New printer and parser for bot data, XML-based

nemubot/config/module.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from nemubot.config import get_boolean
-from nemubot.tools.xmlparser.genericnode import GenericNode
+from nemubot.datastore.nodes.generic import GenericNode
 
 
 class Module(GenericNode):

nemubot/datastore/abstract.py

@@ -23,8 +23,7 @@ class Abstract:
         """Initialize a new empty storage tree
         """
 
-        from nemubot.tools.xmlparser import module_state
+        return None
-        return module_state.ModuleState("nemubotstate")
 
     def open(self):
         return

nemubot/datastore/nodes/__init__.py

@@ -0,0 +1,18 @@
+# Nemubot is a smart and modulable IM bot.
+# Copyright (C) 2012-2016  Mercier Pierre-Olivier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from nemubot.datastore.nodes.generic import ParsingNode
+from nemubot.datastore.nodes.serializable import Serializable

nemubot/datastore/nodes/basic.py

@@ -14,11 +14,16 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-class ListNode:
+from nemubot.datastore.nodes.serializable import Serializable
+
+
+class ListNode(Serializable):
 
     """XML node representing a Python dictionnnary
     """
 
+    serializetag = "list"
+
     def __init__(self, **kwargs):
         self.items = list()
 
@@ -27,6 +32,9 @@ class ListNode:
         self.items.append(child)
         return True
 
+    def parsedForm(self):
+        return self.items
+
 
     def __len__(self):
         return len(self.items)
@@ -44,11 +52,21 @@ class ListNode:
         return self.items.__repr__()
 
 
-class DictNode:
+    def serialize(self):
+        from nemubot.datastore.nodes.generic import ParsingNode
+        node = ParsingNode(tag=self.serializetag)
+        for i in self.items:
+            node.children.append(ParsingNode.serialize_node(i))
+        return node
+
+
+class DictNode(Serializable):
 
     """XML node representing a Python dictionnnary
     """
 
+    serializetag = "dict"
+
     def __init__(self, **kwargs):
         self.items = dict()
         self._cur = None
@@ -56,44 +74,20 @@ class DictNode:
 
     def startElement(self, name, attrs):
         if self._cur is None and "key" in attrs:
-            self._cur = (attrs["key"], "")
+            self._cur = attrs["key"]
-            return True
         return False
 
-
-    def characters(self, content):
-        if self._cur is not None:
-            key, cnt = self._cur
-            if isinstance(cnt, str):
-                cnt += content
-                self._cur = key, cnt
-
-
-    def endElement(self, name):
-        if name is None or self._cur is None:
-            return
-
-        key, cnt = self._cur
-        if isinstance(cnt, list) and len(cnt) == 1:
-            self.items[key] = cnt
-        else:
-            self.items[key] = cnt
-
-        self._cur = None
-        return True
-
-
     def addChild(self, name, child):
         if self._cur is None:
             return False
 
-        key, cnt = self._cur
+        self.items[self._cur] = child
-        if not isinstance(cnt, list):
+        self._cur = None
-            cnt = []
-        cnt.append(child)
-        self._cur = key, cnt
         return True
 
+    def parsedForm(self):
+        return self.items
+
 
     def __getitem__(self, item):
         return self.items[item]
@@ -106,3 +100,13 @@ class DictNode:
 
     def __repr__(self):
         return self.items.__repr__()
+
+
+    def serialize(self):
+        from nemubot.datastore.nodes.generic import ParsingNode
+        node = ParsingNode(tag=self.serializetag)
+        for k in self.items:
+            chld = ParsingNode.serialize_node(self.items[k])
+            chld.attrs["key"] = k
+            node.children.append(chld)
+        return node

nemubot/datastore/nodes/generic.py

@@ -14,6 +14,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from nemubot.datastore.nodes.serializable import Serializable
+
+
 class ParsingNode:
 
     """Allow any kind of subtags, just keep parsed ones
@@ -53,6 +56,47 @@ class ParsingNode:
         return item in self.attrs
 
 
+    def serialize_node(node, **def_kwargs):
+        """Serialize any node or basic data to a ParsingNode instance"""
+
+        if isinstance(node, Serializable):
+            node = node.serialize()
+
+        if isinstance(node, str):
+            from nemubot.datastore.nodes.python import StringNode
+            pn = StringNode(**def_kwargs)
+            pn.value = node
+            return pn
+
+        elif isinstance(node, int):
+            from nemubot.datastore.nodes.python import IntNode
+            pn = IntNode(**def_kwargs)
+            pn.value = node
+            return pn
+
+        elif isinstance(node, float):
+            from nemubot.datastore.nodes.python import FloatNode
+            pn = FloatNode(**def_kwargs)
+            pn.value = node
+            return pn
+
+        elif isinstance(node, list):
+            from nemubot.datastore.nodes.basic import ListNode
+            pn = ListNode(**def_kwargs)
+            pn.items = node
+            return pn.serialize()
+
+        elif isinstance(node, dict):
+            from nemubot.datastore.nodes.basic import DictNode
+            pn = DictNode(**def_kwargs)
+            pn.items = node
+            return pn.serialize()
+
+        else:
+            assert isinstance(node, ParsingNode)
+            return node
+
+
 class GenericNode(ParsingNode):
 
     """Consider all subtags as dictionnary

nemubot/datastore/nodes/python.py

@@ -0,0 +1,91 @@
+# Nemubot is a smart and modulable IM bot.
+# Copyright (C) 2012-2016  Mercier Pierre-Olivier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from nemubot.datastore.nodes.serializable import Serializable
+
+
+class PythonTypeNode(Serializable):
+
+    """XML node representing a Python simple type
+    """
+
+    def __init__(self, **kwargs):
+        self.value = None
+        self._cnt = ""
+
+
+    def characters(self, content):
+        self._cnt += content
+
+
+    def endElement(self, name):
+        raise NotImplemented
+
+
+    def __repr__(self):
+        return self.value.__repr__()
+
+
+    def parsedForm(self):
+        return self.value
+
+    def serialize(self):
+        raise NotImplemented
+
+
+class IntNode(PythonTypeNode):
+
+    serializetag = "int"
+
+    def endElement(self, name):
+        self.value = int(self._cnt)
+        return True
+
+    def serialize(self):
+        from nemubot.datastore.nodes.generic import ParsingNode
+        node = ParsingNode(tag=self.serializetag)
+        node.content = str(self.value)
+        return node
+
+
+class FloatNode(PythonTypeNode):
+
+    serializetag = "float"
+
+    def endElement(self, name):
+        self.value = float(self._cnt)
+        return True
+
+    def serialize(self):
+        from nemubot.datastore.nodes.generic import ParsingNode
+        node = ParsingNode(tag=self.serializetag)
+        node.content = str(self.value)
+        return node
+
+
+class StringNode(PythonTypeNode):
+
+    serializetag = "str"
+
+    def endElement(self, name):
+        self.value = str(self._cnt)
+        return True
+
+    def serialize(self):
+        from nemubot.datastore.nodes.generic import ParsingNode
+        node = ParsingNode(tag=self.serializetag)
+        node.content = str(self.value)
+        return node

nemubot/datastore/nodes/serializable.py

@@ -0,0 +1,22 @@
+# Nemubot is a smart and modulable IM bot.
+# Copyright (C) 2012-2016  Mercier Pierre-Olivier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+class Serializable:
+
+    def serialize(self):
+        # Implementations of this function should return ParsingNode items
+        return NotImplemented

nemubot/datastore/xml.py

@@ -1,5 +1,5 @@
 # Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
+# Copyright (C) 2012-2016  Mercier Pierre-Olivier
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -36,17 +36,24 @@ class XML(Abstract):
         rotate -- auto-backup files?
         """
 
-        self.basedir = basedir
+        self.basedir = os.path.abspath(basedir)
         self.rotate = rotate
         self.nb_save = 0
 
+        logger.info("Initiate XML datastore at %s, rotation %s",
+                    self.basedir,
+                    "enabled" if self.rotate else "disabled")
+
+
     def open(self):
         """Lock the directory"""
 
         if not os.path.isdir(self.basedir):
+            logger.debug("Datastore directory not found, creating: %s", self.basedir)
             os.mkdir(self.basedir)
 
-        lock_path = os.path.join(self.basedir, ".used_by_nemubot")
+        lock_path = self._get_lock_file_path()
+        logger.debug("Locking datastore directory via %s", lock_path)
 
         self.lock_file = open(lock_path, 'a+')
         ok = True
@@ -64,56 +71,91 @@ class XML(Abstract):
         self.lock_file.write(str(os.getpid()))
         self.lock_file.flush()
 
+        logger.info("Datastore successfuly opened at %s", self.basedir)
         return True
 
+
     def close(self):
         """Release a locked path"""
 
         if hasattr(self, "lock_file"):
             self.lock_file.close()
-            lock_path = os.path.join(self.basedir, ".used_by_nemubot")
+            lock_path = self._get_lock_file_path()
             if os.path.isdir(self.basedir) and os.path.exists(lock_path):
                 os.unlink(lock_path)
             del self.lock_file
+            logger.info("Datastore successfully closed at %s", self.basedir)
             return True
+        else:
+            logger.warn("Datastore not open/locked or lock file not found")
         return False
 
+
     def _get_data_file_path(self, module):
         """Get the path to the module data file"""
 
         return os.path.join(self.basedir, module + ".xml")
 
-    def load(self, module):
+
+    def _get_lock_file_path(self):
+        """Get the path to the datastore lock file"""
+
+        return os.path.join(self.basedir, ".used_by_nemubot")
+
+
+    def load(self, module, extendsTags={}):
         """Load data for the given module
 
         Argument:
         module -- the module name of data to load
         """
 
+        logger.debug("Trying to load data for %s%s",
+                     module,
+                     (" with tags: " + ", ".join(extendsTags.keys())) if len(extendsTags) else "")
+
         data_file = self._get_data_file_path(module)
 
+        def parse(path):
+            from nemubot.tools.xmlparser import XMLParser
+            from nemubot.datastore.nodes import basic as basicNodes
+            from nemubot.datastore.nodes import python as pythonNodes
+
+            d = {
+                basicNodes.ListNode.serializetag: basicNodes.ListNode,
+                basicNodes.DictNode.serializetag: basicNodes.DictNode,
+                pythonNodes.IntNode.serializetag: pythonNodes.IntNode,
+                pythonNodes.FloatNode.serializetag: pythonNodes.FloatNode,
+                pythonNodes.StringNode.serializetag: pythonNodes.StringNode,
+            }
+            d.update(extendsTags)
+
+            p = XMLParser(d)
+            return p.parse_file(path)
+
         # Try to load original file
         if os.path.isfile(data_file):
-            from nemubot.tools.xmlparser import parse_file
             try:
-                return parse_file(data_file)
+                return parse(data_file)
             except xml.parsers.expat.ExpatError:
                 # Try to load from backup
                 for i in range(10):
                     path = data_file + "." + str(i)
                     if os.path.isfile(path):
                         try:
-                            cnt = parse_file(path)
+                            cnt = parse(path)
 
-                            logger.warn("Restoring from backup: %s", path)
+                            logger.warn("Restoring data from backup: %s", path)
 
                             return cnt
                         except xml.parsers.expat.ExpatError:
                             continue
 
         # Default case: initialize a new empty datastore
+        logger.warn("No data found in store for %s, creating new set", module)
         return Abstract.load(self, module)
 
+
     def _rotate(self, path):
         """Backup given path
 
@@ -130,6 +172,25 @@ class XML(Abstract):
                 if os.path.isfile(src):
                     os.rename(src, dst)
 
+
+    def _save_node(self, gen, node):
+        from nemubot.datastore.nodes.generic import ParsingNode
+
+        # First, get the serialized form of the node
+        node = ParsingNode.serialize_node(node)
+
+        assert node.tag is not None, "Undefined tag name"
+
+        gen.startElement(node.tag, {k: str(node.attrs[k]) for k in node.attrs})
+
+        gen.characters(node.content)
+
+        for child in node.children:
+            self._save_node(gen, child)
+
+        gen.endElement(node.tag)
+
+
     def save(self, module, data):
         """Load data for the given module
 
@@ -139,8 +200,22 @@ class XML(Abstract):
         """
 
         path = self._get_data_file_path(module)
+        logger.debug("Trying to save data for module %s in %s", module, path)
 
         if self.rotate:
             self._rotate(path)
 
-        return data.save(path)
+        import tempfile
+        _, tmpath = tempfile.mkstemp()
+        with open(tmpath, "w") as f:
+            import xml.sax.saxutils
+            gen = xml.sax.saxutils.XMLGenerator(f, "utf-8")
+            gen.startDocument()
+            self._save_node(gen, data)
+            gen.endDocument()
+
+        # Atomic save
+        import shutil
+        shutil.move(tmpath, path)
+
+        return True

nemubot/modulecontext.py

@@ -39,6 +39,7 @@ class ModuleContext:
 
         self.hooks = list()
         self.events = list()
+        self.extendtags = dict()
         self.debug = context.verbosity > 0 if context is not None else False
 
         from nemubot.hooks import Abstract as AbstractHook
@@ -46,7 +47,7 @@ class ModuleContext:
         # Define some callbacks
         if context is not None:
             def load_data():
-                return context.datastore.load(module_name)
+                return context.datastore.load(module_name, extendsTags=self.extendtags)
 
             def add_hook(hook, *triggers):
                 assert isinstance(hook, AbstractHook), hook
@@ -77,8 +78,7 @@ class ModuleContext:
 
         else:  # Used when using outside of nemubot
             def load_data():
-                from nemubot.tools.xmlparser import module_state
+                return None
-                return module_state.ModuleState("nemubotstate")
 
             def add_hook(hook, *triggers):
                 assert isinstance(hook, AbstractHook), hook
@@ -97,6 +97,8 @@ class ModuleContext:
                 module.logger.info("Send response: %s", res)
 
         def save():
+            # Don't save if no data has been access
+            if hasattr(self, "_data"):
                 context.datastore.save(module_name, self.data)
 
         def subparse(orig, cnt):
@@ -120,6 +122,21 @@ class ModuleContext:
             self._data = self.load_data()
         return self._data
 
+    @data.setter
+    def data(self, value):
+        assert value is not None
+
+        self._data = value
+
+
+    def register_tags(self, **tags):
+        self.extendtags.update(tags)
+
+
+    def unregister_tags(self, *tags):
+        for t in tags:
+            del self.extendtags[t]
+
 
     def unload(self):
         """Perform actions for unloading the module"""

nemubot/tools/xmlparser/__init__.py

@@ -51,11 +51,13 @@ class XMLParser:
     def __init__(self, knodes):
         self.knodes = knodes
 
+    def _reset(self):
         self.stack = list()
         self.child = 0
 
 
     def parse_file(self, path):
+        self._reset()
         p = xml.parsers.expat.ParserCreate()
 
         p.StartElementHandler = self.startElement
@@ -69,6 +71,7 @@ class XMLParser:
 
 
     def parse_string(self, s):
+        self._reset()
         p = xml.parsers.expat.ParserCreate()
 
         p.StartElementHandler = self.startElement
@@ -126,10 +129,13 @@ class XMLParser:
         if hasattr(self.current, "endElement"):
             self.current.endElement(None)
 
+        if hasattr(self.current, "parsedForm") and callable(self.current.parsedForm):
+            self.stack[-1] = self.current.parsedForm()
+
         # Don't remove root
         if len(self.stack) > 1:
             last = self.stack.pop()
-            if hasattr(self.current, "addChild"):
+            if hasattr(self.current, "addChild") and callable(self.current.addChild):
                 if self.current.addChild(name, last):
                     return
             raise TypeError(name + " tag not expected in " + self.display_stack())

setup.py

@@ -63,6 +63,7 @@ setup(
         'nemubot',
         'nemubot.config',
         'nemubot.datastore',
+        'nemubot.datastore.nodes',
         'nemubot.event',
         'nemubot.exception',
         'nemubot.hooks',