Rework XML parser: part 1

This is the first step of the parser refactoring: here we change
	the configuration, next step will change data saving.

bot_sample.xml

@@ -1,11 +1,11 @@
 <nemubotconfig nick="nemubot" realname="nemubot clone" owner="someone">
 
-  <server host="irc.rezosup.org" port="6667" autoconnect="true" caps="znc.in/server-time-iso">
+  <server uri="irc://irc.rezosup.org:6667" autoconnect="true" caps="znc.in/server-time-iso">
     <channel name="#nemutest" />
   </server>
 
   <!--
-  <server host="my_host.local" port="6667" password="secret" autoconnect="true" ip="10.69.42.23" ssl="on">
+  <server host="ircs://my_host.local:6667" password="secret" autoconnect="true">
     <channel name="#nemutest" />
   </server>
   -->

modules/books.py

@@ -15,7 +15,7 @@ from more import Response
 # LOADING #############################################################
 
 def load(context):
-    if not context.config or not context.config.getAttribute("goodreadskey"):
+    if not context.config or "goodreadskey" not in context.config:
         raise ImportError("You need a Goodreads API key in order to use this "
                           "module. Add it to the module configuration file:\n"
                           "<module name=\"books\" goodreadskey=\"XXXXXX\" />\n"

modules/mapquest.py

@@ -16,7 +16,7 @@ from more import Response
 URL_API = "http://open.mapquestapi.com/geocoding/v1/address?key=%s&location=%%s"
 
 def load(context):
-    if not context.config or not context.config.hasAttribute("apikey"):
+    if not context.config or "apikey" not in context.config:
         raise ImportError("You need a MapQuest API key in order to use this "
                           "module. Add it to the module configuration file:\n"
                           "<module name=\"mapquest\" key=\"XXXXXXXXXXXXXXXX\" "

modules/networking/whois.py

@@ -11,7 +11,7 @@ URL_WHOIS = "http://www.whoisxmlapi.com/whoisserver/WhoisService?rid=1&domainNam
 def load(CONF, add_hook):
     global URL_WHOIS
 
-    if not CONF or not CONF.hasNode("whoisxmlapi") or not CONF.getNode("whoisxmlapi").hasAttribute("username") or not CONF.getNode("whoisxmlapi").hasAttribute("password"):
+    if not CONF or not CONF.hasNode("whoisxmlapi") or "username" not in CONF.getNode("whoisxmlapi") or "password" not in CONF.getNode("whoisxmlapi"):
         raise ImportError("You need a WhoisXML API account in order to use "
                           "the !netwhois feature. Add it to the module "
                           "configuration file:\n<whoisxmlapi username=\"XX\" "

modules/syno.py

@@ -21,7 +21,7 @@ def help_full():
 def load(context):
     global lang_binding
 
-    if not context.config or not context.config.hasAttribute("bighugelabskey"):
+    if not context.config or not "bighugelabskey" in context.config:
         logger.error("You need a NigHugeLabs API key in order to have english "
                      "theasorus. Add it to the module configuration file:\n"
                      "<module name=\"syno\" bighugelabskey=\"XXXXXXXXXXXXXXXX\""

modules/tpb.py

@@ -13,7 +13,7 @@ from more import Response
 URL_TPBAPI = None
 
 def load(context):
-    if not context.config or not context.config.hasAttribute("url"):
+    if not context.config or "url" not in context.config:
         raise ImportError("You need a TPB API in order to use the !tpb feature"
                           ". Add it to the module configuration file:\n<module"
                           "name=\"tpb\" url=\"http://tpbapi.org/\" />\nSample "

modules/translate.py

@@ -19,7 +19,7 @@ LANG = ["ar", "zh", "cz", "en", "fr", "gr", "it",
 URL = "http://api.wordreference.com/0.8/%s/json/%%s%%s/%%s"
 
 def load(context):
-    if not context.config or not context.config.hasAttribute("wrapikey"):
+    if not context.config or "wrapikey" not in context.config:
         raise ImportError("You need a WordReference API key in order to use "
                           "this module. Add it to the module configuration "
                           "file:\n<module name=\"translate\" wrapikey=\"XXXXX\""

modules/velib.py

@@ -18,7 +18,7 @@ URL_API = None # http://www.velib.paris.fr/service/stationdetails/paris/%s
 
 def load(context):
     global URL_API
-    if not context.config or not context.config.hasAttribute("url"):
+    if not context.config or "url" not in context.config:
         raise ImportError("Please provide url attribute in the module configuration")
     URL_API = context.config["url"]
     context.data.setIndex("name", "station")

modules/weather.py

@@ -21,7 +21,7 @@ from more import Response
 URL_DSAPI = "https://api.forecast.io/forecast/%s/%%s,%%s"
 
 def load(context):
-    if not context.config or not context.config.hasAttribute("darkskyapikey"):
+    if not context.config or "darkskyapikey" not in context.config:
         raise ImportError("You need a Dark-Sky API key in order to use this "
                           "module. Add it to the module configuration file:\n"
                           "<module name=\"weather\" darkskyapikey=\"XXX\" />\n"

modules/whois.py

@@ -16,7 +16,7 @@ PASSWD_FILE = None
 
 def load(context):
     global PASSWD_FILE
-    if not context.config or not context.config.hasAttribute("passwd"):
+    if not context.config or "passwd" not in context.config:
         print("No passwd file given")
         return None
     PASSWD_FILE = context.config["passwd"]

modules/wolframalpha.py

@@ -19,7 +19,7 @@ URL_API = "http://api.wolframalpha.com/v2/query?input=%%s&appid=%s"
 
 def load(context):
     global URL_API
-    if not context.config or not context.config.hasAttribute("apikey"):
+    if not context.config or "apikey" not in context.config:
         raise ImportError ("You need a Wolfram|Alpha API key in order to use "
                            "this module. Add it to the module configuration: "
                            "\n<module name=\"wolframalpha\" "

nemubot/bot.py

@@ -205,12 +205,62 @@ class Bot(threading.Thread):
                     self.quit()
                 elif action[0] == "loadconf":
                     for path in action[1:]:
-                        from nemubot.tools.config import load_file
+                        self.load_file(path)
-                        load_file(path, self)
                 self.sync_queue.task_done()
 
 
 
+    # Config methods
+
+    def load_file(self, filename):
+        """Load a configuration file
+
+        Arguments:
+        filename -- the path to the file to load
+        """
+
+        import os
+
+        # Unexisting file, assume a name was passed, import the module!
+        if not os.path.isfile(filename):
+            return self.import_module(filename)
+
+        from nemubot.tools.config import config_nodes
+        from nemubot.tools.xmlparser import XMLParser
+
+        try:
+            p = XMLParser(config_nodes)
+            config = p.parse_file(filename)
+        except:
+            logger.exception("Can't load `%s'; this is not a valid nemubot "
+                             "configuration file." % filename)
+            return False
+
+        # Preset each server in this file
+        for server in config.servers:
+            srv = server.server(config)
+            # Add the server in the context
+            if self.add_server(srv, server.autoconnect):
+                logger.info("Server '%s' successfully added." % srv.id)
+            else:
+                logger.error("Can't add server '%s'." % srv.id)
+
+        # Load module and their configuration
+        for mod in config.modules:
+            self.modules_configuration[mod.name] = mod
+            if mod.autoload:
+                try:
+                    __import__(mod.name)
+                except:
+                    logger.exception("Exception occurs when loading module"
+                                     " '%s'", mod.name)
+
+
+        # Load files asked by the configuration file
+        for load in config.includes:
+            self.load_file(load.path)
+
+
     # Events methods
 
     def add_event(self, evt, eid=None, module_src=None):

nemubot/channel.py

@@ -1,5 +1,3 @@
-# coding=utf-8
-
 # Nemubot is a smart and modulable IM bot.
 # Copyright (C) 2012-2015  Mercier Pierre-Olivier
 #
@@ -23,16 +21,18 @@ class Channel:
 
     """A chat room"""
 
-    def __init__(self, name, password=None):
+    def __init__(self, name, password=None, encoding=None):
         """Initialize the channel
 
         Arguments:
         name -- the channel name
         password -- the optional password use to join it
+        encoding -- the optional encoding of the channel
         """
 
         self.name = name
         self.password = password
+        self.encoding = encoding
         self.people = dict()
         self.topic = ""
         self.logger = logging.getLogger("nemubot.channel." + name)

nemubot/prompt/builtins.py

@@ -61,10 +61,8 @@ def liste(toks, context, prompt):
 def load(toks, context, prompt):
     """Load an XML configuration file"""
     if len(toks) > 1:
-        from nemubot.tools.config import load_file
-
         for filename in toks[1:]:
-            load_file(filename, context)
+            context.load_file(filename)
     else:
         print ("Not enough arguments. `load' takes a filename.")
         return 1

nemubot/tools/config.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Nemubot is a smart and modulable IM bot.
 # Copyright (C) 2012-2015  Mercier Pierre-Olivier
 #
@@ -16,123 +14,146 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import logging
+def get_boolean(s):
-
+    if isinstance(s, bool):
-logger = logging.getLogger("nemubot.tools.config")
+        return s
-
-
-def get_boolean(d, k, default=False):
-    return ((k in d and d[k].lower() != "false" and d[k].lower() != "off") or
-            (k not in d and default))
-
-
-def _load_server(config, xmlnode):
-    """Load a server configuration
-
-    Arguments:
-    config -- the global configuration
-    xmlnode -- the current server configuration node
-    """
-
-    opts = {
-        "host": xmlnode["host"],
-        "ssl": xmlnode.hasAttribute("ssl") and xmlnode["ssl"].lower() == "true",
-
-        "nick": xmlnode["nick"] if xmlnode.hasAttribute("nick") else config["nick"],
-        "owner": xmlnode["owner"] if xmlnode.hasAttribute("owner") else config["owner"],
-    }
-
-    # Optional keyword arguments
-    for optional_opt in [ "port", "username", "realname",
-                          "password", "encoding", "caps" ]:
-        if xmlnode.hasAttribute(optional_opt):
-            opts[optional_opt] = xmlnode[optional_opt]
-        elif optional_opt in config:
-            opts[optional_opt] = config[optional_opt]
-
-    # Command to send on connection
-    if "on_connect" in xmlnode:
-        def on_connect():
-            yield xmlnode["on_connect"]
-        opts["on_connect"] = on_connect
-
-    # Channels to autojoin on connection
-    if xmlnode.hasNode("channel"):
-        opts["channels"] = list()
-    for chn in xmlnode.getNodes("channel"):
-        opts["channels"].append((chn["name"], chn["password"])
-                                if chn["password"] is not None
-                                else chn["name"])
-
-    # Server/client capabilities
-    if "caps" in xmlnode or "caps" in config:
-        capsl = (xmlnode["caps"] if xmlnode.hasAttribute("caps")
-                 else config["caps"]).lower()
-        if capsl == "no" or capsl == "off" or capsl == "false":
-            opts["caps"] = None
     else:
-            opts["caps"] = capsl.split(',')
+        return (s and s != "0" and s.lower() != "false" and s.lower() != "off")
+
+
+class GenericNode:
+
+    def __init__(self, tag, **kwargs):
+        self.tag = tag
+        self.attrs = kwargs
+        self.content = ""
+        self.children = []
+        self._cur = None
+        self._deep_cur = 0
+
+
+    def startElement(self, name, attrs):
+        if self._cur is None:
+            self._cur = GenericNode(name, **attrs)
+            self._deep_cur = 0
         else:
-        opts["caps"] = list()
+            self._deep_cur += 1
+            self._cur.startElement(name, attrs)
+        return True
 
-    # Bind the protocol asked to the corresponding implementation
+
-    if "protocol" not in xmlnode or xmlnode["protocol"] == "irc":
+    def characters(self, content):
-        from nemubot.server.IRC import IRC as IRCServer
+        if self._cur is None:
-        srvcls = IRCServer
+            self.content += content
         else:
-        raise Exception("Unhandled protocol '%s'" %
+            self._cur.characters(content)
-                        xmlnode["protocol"])
-
-    # Initialize the server
-    return srvcls(**opts)
 
 
-def load_file(filename, context):
+    def endElement(self, name):
-    """Load the configuration file
+        if name is None:
+            return
 
-    Arguments:
+        if self._deep_cur:
-    filename -- the path to the file to load
+            self._deep_cur -= 1
-    """
+            self._cur.endElement(name)
-
-    import os
-
-    if os.path.isfile(filename):
-        from nemubot.tools.xmlparser import parse_file
-
-        config = parse_file(filename)
-
-        # This is a true nemubot configuration file, load it!
-        if config.getName() == "nemubotconfig":
-            # Preset each server in this file
-            for server in config.getNodes("server"):
-                srv = _load_server(config, server)
-
-                # Add the server in the context
-                if context.add_server(srv, get_boolean(server, "autoconnect")):
-                    logger.info("Server '%s' successfully added." % srv.id)
         else:
-                    logger.error("Can't add server '%s'." % srv.id)
+            self.children.append(self._cur)
-
+            self._cur = None
-            # Load module and their configuration
+        return True
-            for mod in config.getNodes("module"):
-                context.modules_configuration[mod["name"]] = mod
-                if get_boolean(mod, "autoload", default=True):
-                    try:
-                        __import__(mod["name"])
-                    except:
-                        logger.exception("Exception occurs when loading module"
-                                         " '%s'", mod["name"])
 
 
-            # Load files asked by the configuration file
+    def hasNode(self, nodename):
-            for load in config.getNodes("include"):
+        return self.getNode(nodename) is not None
-                load_file(load["path"], context)
 
-        # Other formats
-        else:
-            logger.error("Can't load `%s'; this is not a valid nemubot "
-                         "configuration file." % filename)
 
-    # Unexisting file, assume a name was passed, import the module!
+    def getNode(self, nodename):
-    else:
+        for c in self.children:
-        context.import_module(filename)
+            if c is not None and c.tag == nodename:
+                return c
+        return None
+
+
+    def __getitem__(self, item):
+        return self.attrs[item]
+
+    def __contains__(self, item):
+        return item in self.attrs
+
+
+class NemubotConfig:
+
+    def __init__(self, nick="nemubot", realname="nemubot", owner=None,
+                 ip=None, ssl=False, caps=None, encoding="utf-8"):
+        self.nick = nick
+        self.realname = realname
+        self.owner = owner
+        self.ip = ip
+        self.caps = caps.split(" ") if caps is not None else []
+        self.encoding = encoding
+        self.servers = []
+        self.modules = []
+        self.includes = []
+
+
+    def addChild(self, name, child):
+        if name == "module" and isinstance(child, ModuleConfig):
+            self.modules.append(child)
+            return True
+        elif name == "server" and isinstance(child, ServerConfig):
+            self.servers.append(child)
+            return True
+        elif name == "include" and isinstance(child, IncludeConfig):
+            self.includes.append(child)
+            return True
+
+
+class ServerConfig:
+
+    def __init__(self, uri="irc://nemubot@localhost/", autoconnect=True, caps=None, **kwargs):
+        self.uri = uri
+        self.autoconnect = autoconnect
+        self.caps = caps.split(" ") if caps is not None else []
+        self.args = kwargs
+        self.channels = []
+
+
+    def addChild(self, name, child):
+        if name == "channel" and isinstance(child, Channel):
+            self.channels.append(child)
+            return True
+
+
+    def server(self, parent):
+        from nemubot.server import factory
+
+        for a in ["nick", "owner", "realname", "encoding"]:
+            if a not in self.args:
+                self.args[a] = getattr(parent, a)
+
+        self.caps += parent.caps
+
+        return factory(self.uri, **self.args)
+
+
+class IncludeConfig:
+
+    def __init__(self, path):
+        self.path = path
+
+
+class ModuleConfig(GenericNode):
+
+    def __init__(self, name, autoload=True, **kwargs):
+        super(ModuleConfig, self).__init__(None, **kwargs)
+        self.name = name
+        self.autoload = get_boolean(autoload)
+
+from nemubot.channel import Channel
+
+config_nodes = {
+    "nemubotconfig": NemubotConfig,
+    "server": ServerConfig,
+    "channel": Channel,
+    "module": ModuleConfig,
+    "include": IncludeConfig,
+}

nemubot/tools/test_xmlparser.py

@@ -0,0 +1,82 @@
+import unittest
+
+import xml.parsers.expat
+
+from nemubot.tools.xmlparser import XMLParser
+
+
+class StringNode():
+    def __init__(self):
+        self.string = ""
+
+    def characters(self, content):
+        self.string += content
+
+
+class TestNode():
+    def __init__(self, option=None):
+        self.option = option
+        self.mystr = None
+
+    def addChild(self, name, child):
+        self.mystr = child.string
+
+
+class Test2Node():
+    def __init__(self, option=None):
+        self.option = option
+        self.mystrs = list()
+
+    def startElement(self, name, attrs):
+        if name == "string":
+            self.mystrs.append(attrs["value"])
+            return True
+
+
+class TestXMLParser(unittest.TestCase):
+
+    def test_parser1(self):
+        p = xml.parsers.expat.ParserCreate()
+        mod = XMLParser({"string": StringNode})
+
+        p.StartElementHandler = mod.startElement
+        p.CharacterDataHandler = mod.characters
+        p.EndElementHandler = mod.endElement
+
+        p.Parse("<string>toto</string>", 1)
+
+        self.assertEqual(mod.root.string, "toto")
+
+
+    def test_parser2(self):
+        p = xml.parsers.expat.ParserCreate()
+        mod = XMLParser({"string": StringNode, "test": TestNode})
+
+        p.StartElementHandler = mod.startElement
+        p.CharacterDataHandler = mod.characters
+        p.EndElementHandler = mod.endElement
+
+        p.Parse("<test option='123'><string>toto</string></test>", 1)
+
+        self.assertEqual(mod.root.option, "123")
+        self.assertEqual(mod.root.mystr, "toto")
+
+
+    def test_parser3(self):
+        p = xml.parsers.expat.ParserCreate()
+        mod = XMLParser({"string": StringNode, "test": Test2Node})
+
+        p.StartElementHandler = mod.startElement
+        p.CharacterDataHandler = mod.characters
+        p.EndElementHandler = mod.endElement
+
+        p.Parse("<test><string value='toto' /><string value='toto2' /></test>", 1)
+
+        self.assertEqual(mod.root.option, None)
+        self.assertEqual(len(mod.root.mystrs), 2)
+        self.assertEqual(mod.root.mystrs[0], "toto")
+        self.assertEqual(mod.root.mystrs[1], "toto2")
+
+
+if __name__ == '__main__':
+    unittest.main()

nemubot/tools/xmlparser/__init__.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Nemubot is a smart and modulable IM bot.
 # Copyright (C) 2012-2015  Mercier Pierre-Olivier
 #
@@ -48,9 +46,107 @@ class ModuleStatesFile:
             self.root = child
 
 
+class XMLParser:
+
+    def __init__(self, knodes):
+        self.knodes = knodes
+
+        self.stack = list()
+        self.child = 0
+
+
+    def parse_file(self, path):
+        p = xml.parsers.expat.ParserCreate()
+
+        p.StartElementHandler = self.startElement
+        p.CharacterDataHandler = self.characters
+        p.EndElementHandler = self.endElement
+
+        with open(path, "rb") as f:
+            p.ParseFile(f)
+
+        return self.root
+
+
+    def parse_string(self, s):
+        p = xml.parsers.expat.ParserCreate()
+
+        p.StartElementHandler = self.startElement
+        p.CharacterDataHandler = self.characters
+        p.EndElementHandler = self.endElement
+
+        p.Parse(s, 1)
+
+        return self.root
+
+
+    @property
+    def root(self):
+        if len(self.stack):
+            return self.stack[0]
+        else:
+            return None
+
+
+    @property
+    def current(self):
+        if len(self.stack):
+            return self.stack[-1]
+        else:
+            return None
+
+
+    def display_stack(self):
+        return " in ".join([str(type(s).__name__) for s in reversed(self.stack)])
+
+
+    def startElement(self, name, attrs):
+        if not self.current or not hasattr(self.current, "startElement") or not self.current.startElement(name, attrs):
+            if name not in self.knodes:
+                raise TypeError(name + " is not a known type to decode")
+            else:
+                self.stack.append(self.knodes[name](**attrs))
+        else:
+            self.child += 1
+
+
+    def characters(self, content):
+        if self.current and hasattr(self.current, "characters"):
+            self.current.characters(content)
+
+
+    def endElement(self, name):
+        if self.child:
+            self.child -= 1
+
+            if hasattr(self.current, "endElement"):
+                self.current.endElement(name)
+            return
+
+        if hasattr(self.current, "endElement"):
+            self.current.endElement(None)
+
+        # Don't remove root
+        if len(self.stack) > 1:
+            last = self.stack.pop()
+            if hasattr(self.current, "addChild"):
+                if self.current.addChild(name, last):
+                    return
+            raise TypeError(name + " tag not expected in " + self.display_stack())
+
+
 def parse_file(filename):
-    with open(filename, "r") as f:
+    p = xml.parsers.expat.ParserCreate()
-        return parse_string(f.read())
+    mod = ModuleStatesFile()
+
+    p.StartElementHandler = mod.startElement
+    p.EndElementHandler = mod.endElement
+    p.CharacterDataHandler = mod.characters
+
+    with open(filename, "rb") as f:
+        p.ParseFile(f)
+
+    return mod.root
 
 
 def parse_string(string):

nemubot/tools/xmlparser/node.py

@@ -1,5 +1,3 @@
-# coding=utf-8
-
 # Nemubot is a smart and modulable IM bot.
 # Copyright (C) 2012-2015  Mercier Pierre-Olivier
 #
@@ -37,7 +35,7 @@ class ModuleState:
         """Get the name of the current node"""
         return self.name
 
-    def display(self, level = 0):
+    def display(self, level=0):
         ret = ""
         out = list()
         for k in self.attributes:
@@ -51,6 +49,9 @@ class ModuleState:
     def __str__(self):
         return self.display()
 
+    def __repr__(self):
+        return self.display()
+
     def __getitem__(self, i):
         """Return the attribute asked"""
         return self.getAttribute(i)