diff --git a/nemubot/config/module.py b/nemubot/config/module.py
index ab51971..7586697 100644
--- a/nemubot/config/module.py
+++ b/nemubot/config/module.py
@@ -15,7 +15,7 @@
# along with this program. If not, see .
from nemubot.config import get_boolean
-from nemubot.tools.xmlparser.genericnode import GenericNode
+from nemubot.datastore.nodes.generic import GenericNode
class Module(GenericNode):
diff --git a/nemubot/datastore/abstract.py b/nemubot/datastore/abstract.py
index 96e2c0d..f54bbcd 100644
--- a/nemubot/datastore/abstract.py
+++ b/nemubot/datastore/abstract.py
@@ -23,8 +23,7 @@ class Abstract:
"""Initialize a new empty storage tree
"""
- from nemubot.tools.xmlparser import module_state
- return module_state.ModuleState("nemubotstate")
+ return None
def open(self):
return
diff --git a/nemubot/datastore/nodes/__init__.py b/nemubot/datastore/nodes/__init__.py
new file mode 100644
index 0000000..e4b2788
--- /dev/null
+++ b/nemubot/datastore/nodes/__init__.py
@@ -0,0 +1,18 @@
+# Nemubot is a smart and modulable IM bot.
+# Copyright (C) 2012-2016 Mercier Pierre-Olivier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+from nemubot.datastore.nodes.generic import ParsingNode
+from nemubot.datastore.nodes.serializable import Serializable
diff --git a/nemubot/tools/xmlparser/basic.py b/nemubot/datastore/nodes/basic.py
similarity index 67%
rename from nemubot/tools/xmlparser/basic.py
rename to nemubot/datastore/nodes/basic.py
index 8456629..6fbd136 100644
--- a/nemubot/tools/xmlparser/basic.py
+++ b/nemubot/datastore/nodes/basic.py
@@ -14,11 +14,16 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
-class ListNode:
+from nemubot.datastore.nodes.serializable import Serializable
+
+
+class ListNode(Serializable):
"""XML node representing a Python dictionnnary
"""
+ serializetag = "list"
+
def __init__(self, **kwargs):
self.items = list()
@@ -27,6 +32,9 @@ class ListNode:
self.items.append(child)
return True
+ def parsedForm(self):
+ return self.items
+
def __len__(self):
return len(self.items)
@@ -44,11 +52,21 @@ class ListNode:
return self.items.__repr__()
-class DictNode:
+ def serialize(self):
+ from nemubot.datastore.nodes.generic import ParsingNode
+ node = ParsingNode(tag=self.serializetag)
+ for i in self.items:
+ node.children.append(ParsingNode.serialize_node(i))
+ return node
+
+
+class DictNode(Serializable):
"""XML node representing a Python dictionnnary
"""
+ serializetag = "dict"
+
def __init__(self, **kwargs):
self.items = dict()
self._cur = None
@@ -56,44 +74,20 @@ class DictNode:
def startElement(self, name, attrs):
if self._cur is None and "key" in attrs:
- self._cur = (attrs["key"], "")
- return True
+ self._cur = attrs["key"]
return False
-
- def characters(self, content):
- if self._cur is not None:
- key, cnt = self._cur
- if isinstance(cnt, str):
- cnt += content
- self._cur = key, cnt
-
-
- def endElement(self, name):
- if name is None or self._cur is None:
- return
-
- key, cnt = self._cur
- if isinstance(cnt, list) and len(cnt) == 1:
- self.items[key] = cnt
- else:
- self.items[key] = cnt
-
- self._cur = None
- return True
-
-
def addChild(self, name, child):
if self._cur is None:
return False
- key, cnt = self._cur
- if not isinstance(cnt, list):
- cnt = []
- cnt.append(child)
- self._cur = key, cnt
+ self.items[self._cur] = child
+ self._cur = None
return True
+ def parsedForm(self):
+ return self.items
+
def __getitem__(self, item):
return self.items[item]
@@ -106,3 +100,13 @@ class DictNode:
def __repr__(self):
return self.items.__repr__()
+
+
+ def serialize(self):
+ from nemubot.datastore.nodes.generic import ParsingNode
+ node = ParsingNode(tag=self.serializetag)
+ for k in self.items:
+ chld = ParsingNode.serialize_node(self.items[k])
+ chld.attrs["key"] = k
+ node.children.append(chld)
+ return node
diff --git a/nemubot/tools/xmlparser/genericnode.py b/nemubot/datastore/nodes/generic.py
similarity index 64%
rename from nemubot/tools/xmlparser/genericnode.py
rename to nemubot/datastore/nodes/generic.py
index 9c29a23..c9840bc 100644
--- a/nemubot/tools/xmlparser/genericnode.py
+++ b/nemubot/datastore/nodes/generic.py
@@ -14,6 +14,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
+from nemubot.datastore.nodes.serializable import Serializable
+
+
class ParsingNode:
"""Allow any kind of subtags, just keep parsed ones
@@ -53,6 +56,47 @@ class ParsingNode:
return item in self.attrs
+ def serialize_node(node, **def_kwargs):
+ """Serialize any node or basic data to a ParsingNode instance"""
+
+ if isinstance(node, Serializable):
+ node = node.serialize()
+
+ if isinstance(node, str):
+ from nemubot.datastore.nodes.python import StringNode
+ pn = StringNode(**def_kwargs)
+ pn.value = node
+ return pn
+
+ elif isinstance(node, int):
+ from nemubot.datastore.nodes.python import IntNode
+ pn = IntNode(**def_kwargs)
+ pn.value = node
+ return pn
+
+ elif isinstance(node, float):
+ from nemubot.datastore.nodes.python import FloatNode
+ pn = FloatNode(**def_kwargs)
+ pn.value = node
+ return pn
+
+ elif isinstance(node, list):
+ from nemubot.datastore.nodes.basic import ListNode
+ pn = ListNode(**def_kwargs)
+ pn.items = node
+ return pn.serialize()
+
+ elif isinstance(node, dict):
+ from nemubot.datastore.nodes.basic import DictNode
+ pn = DictNode(**def_kwargs)
+ pn.items = node
+ return pn.serialize()
+
+ else:
+ assert isinstance(node, ParsingNode)
+ return node
+
+
class GenericNode(ParsingNode):
"""Consider all subtags as dictionnary
diff --git a/nemubot/datastore/nodes/python.py b/nemubot/datastore/nodes/python.py
new file mode 100644
index 0000000..6e4278b
--- /dev/null
+++ b/nemubot/datastore/nodes/python.py
@@ -0,0 +1,91 @@
+# Nemubot is a smart and modulable IM bot.
+# Copyright (C) 2012-2016 Mercier Pierre-Olivier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+from nemubot.datastore.nodes.serializable import Serializable
+
+
+class PythonTypeNode(Serializable):
+
+ """XML node representing a Python simple type
+ """
+
+ def __init__(self, **kwargs):
+ self.value = None
+ self._cnt = ""
+
+
+ def characters(self, content):
+ self._cnt += content
+
+
+ def endElement(self, name):
+ raise NotImplemented
+
+
+ def __repr__(self):
+ return self.value.__repr__()
+
+
+ def parsedForm(self):
+ return self.value
+
+ def serialize(self):
+ raise NotImplemented
+
+
+class IntNode(PythonTypeNode):
+
+ serializetag = "int"
+
+ def endElement(self, name):
+ self.value = int(self._cnt)
+ return True
+
+ def serialize(self):
+ from nemubot.datastore.nodes.generic import ParsingNode
+ node = ParsingNode(tag=self.serializetag)
+ node.content = str(self.value)
+ return node
+
+
+class FloatNode(PythonTypeNode):
+
+ serializetag = "float"
+
+ def endElement(self, name):
+ self.value = float(self._cnt)
+ return True
+
+ def serialize(self):
+ from nemubot.datastore.nodes.generic import ParsingNode
+ node = ParsingNode(tag=self.serializetag)
+ node.content = str(self.value)
+ return node
+
+
+class StringNode(PythonTypeNode):
+
+ serializetag = "str"
+
+ def endElement(self, name):
+ self.value = str(self._cnt)
+ return True
+
+ def serialize(self):
+ from nemubot.datastore.nodes.generic import ParsingNode
+ node = ParsingNode(tag=self.serializetag)
+ node.content = str(self.value)
+ return node
diff --git a/nemubot/datastore/nodes/serializable.py b/nemubot/datastore/nodes/serializable.py
new file mode 100644
index 0000000..e543699
--- /dev/null
+++ b/nemubot/datastore/nodes/serializable.py
@@ -0,0 +1,22 @@
+# Nemubot is a smart and modulable IM bot.
+# Copyright (C) 2012-2016 Mercier Pierre-Olivier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+
+
+class Serializable:
+
+ def serialize(self):
+ # Implementations of this function should return ParsingNode items
+ return NotImplemented
diff --git a/nemubot/datastore/xml.py b/nemubot/datastore/xml.py
index 46dca70..266c3ac 100644
--- a/nemubot/datastore/xml.py
+++ b/nemubot/datastore/xml.py
@@ -1,5 +1,5 @@
# Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015 Mercier Pierre-Olivier
+# Copyright (C) 2012-2016 Mercier Pierre-Olivier
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@@ -36,17 +36,24 @@ class XML(Abstract):
rotate -- auto-backup files?
"""
- self.basedir = basedir
+ self.basedir = os.path.abspath(basedir)
self.rotate = rotate
self.nb_save = 0
+ logger.info("Initiate XML datastore at %s, rotation %s",
+ self.basedir,
+ "enabled" if self.rotate else "disabled")
+
+
def open(self):
"""Lock the directory"""
if not os.path.isdir(self.basedir):
+ logger.debug("Datastore directory not found, creating: %s", self.basedir)
os.mkdir(self.basedir)
- lock_path = os.path.join(self.basedir, ".used_by_nemubot")
+ lock_path = self._get_lock_file_path()
+ logger.debug("Locking datastore directory via %s", lock_path)
self.lock_file = open(lock_path, 'a+')
ok = True
@@ -64,56 +71,91 @@ class XML(Abstract):
self.lock_file.write(str(os.getpid()))
self.lock_file.flush()
+ logger.info("Datastore successfuly opened at %s", self.basedir)
return True
+
def close(self):
"""Release a locked path"""
if hasattr(self, "lock_file"):
self.lock_file.close()
- lock_path = os.path.join(self.basedir, ".used_by_nemubot")
+ lock_path = self._get_lock_file_path()
if os.path.isdir(self.basedir) and os.path.exists(lock_path):
os.unlink(lock_path)
del self.lock_file
+ logger.info("Datastore successfully closed at %s", self.basedir)
return True
+ else:
+ logger.warn("Datastore not open/locked or lock file not found")
return False
+
def _get_data_file_path(self, module):
"""Get the path to the module data file"""
return os.path.join(self.basedir, module + ".xml")
- def load(self, module):
+
+ def _get_lock_file_path(self):
+ """Get the path to the datastore lock file"""
+
+ return os.path.join(self.basedir, ".used_by_nemubot")
+
+
+ def load(self, module, extendsTags={}):
"""Load data for the given module
Argument:
module -- the module name of data to load
"""
+ logger.debug("Trying to load data for %s%s",
+ module,
+ (" with tags: " + ", ".join(extendsTags.keys())) if len(extendsTags) else "")
+
data_file = self._get_data_file_path(module)
+ def parse(path):
+ from nemubot.tools.xmlparser import XMLParser
+ from nemubot.datastore.nodes import basic as basicNodes
+ from nemubot.datastore.nodes import python as pythonNodes
+
+ d = {
+ basicNodes.ListNode.serializetag: basicNodes.ListNode,
+ basicNodes.DictNode.serializetag: basicNodes.DictNode,
+ pythonNodes.IntNode.serializetag: pythonNodes.IntNode,
+ pythonNodes.FloatNode.serializetag: pythonNodes.FloatNode,
+ pythonNodes.StringNode.serializetag: pythonNodes.StringNode,
+ }
+ d.update(extendsTags)
+
+ p = XMLParser(d)
+ return p.parse_file(path)
+
# Try to load original file
if os.path.isfile(data_file):
- from nemubot.tools.xmlparser import parse_file
try:
- return parse_file(data_file)
+ return parse(data_file)
except xml.parsers.expat.ExpatError:
# Try to load from backup
for i in range(10):
path = data_file + "." + str(i)
if os.path.isfile(path):
try:
- cnt = parse_file(path)
+ cnt = parse(path)
- logger.warn("Restoring from backup: %s", path)
+ logger.warn("Restoring data from backup: %s", path)
return cnt
except xml.parsers.expat.ExpatError:
continue
# Default case: initialize a new empty datastore
+ logger.warn("No data found in store for %s, creating new set", module)
return Abstract.load(self, module)
+
def _rotate(self, path):
"""Backup given path
@@ -130,6 +172,25 @@ class XML(Abstract):
if os.path.isfile(src):
os.rename(src, dst)
+
+ def _save_node(self, gen, node):
+ from nemubot.datastore.nodes.generic import ParsingNode
+
+ # First, get the serialized form of the node
+ node = ParsingNode.serialize_node(node)
+
+ assert node.tag is not None, "Undefined tag name"
+
+ gen.startElement(node.tag, {k: str(node.attrs[k]) for k in node.attrs})
+
+ gen.characters(node.content)
+
+ for child in node.children:
+ self._save_node(gen, child)
+
+ gen.endElement(node.tag)
+
+
def save(self, module, data):
"""Load data for the given module
@@ -139,8 +200,22 @@ class XML(Abstract):
"""
path = self._get_data_file_path(module)
+ logger.debug("Trying to save data for module %s in %s", module, path)
if self.rotate:
self._rotate(path)
- return data.save(path)
+ import tempfile
+ _, tmpath = tempfile.mkstemp()
+ with open(tmpath, "w") as f:
+ import xml.sax.saxutils
+ gen = xml.sax.saxutils.XMLGenerator(f, "utf-8")
+ gen.startDocument()
+ self._save_node(gen, data)
+ gen.endDocument()
+
+ # Atomic save
+ import shutil
+ shutil.move(tmpath, path)
+
+ return True
diff --git a/nemubot/modulecontext.py b/nemubot/modulecontext.py
index 877b8de..7befe18 100644
--- a/nemubot/modulecontext.py
+++ b/nemubot/modulecontext.py
@@ -26,15 +26,14 @@ class _ModuleContext:
self.hooks = list()
self.events = list()
+ self.extendtags = dict()
self.debug = False
from nemubot.config.module import Module
self.config = Module(self.module_name)
-
def load_data(self):
- from nemubot.tools.xmlparser import module_state
- return module_state.ModuleState("nemubotstate")
+ return None
def add_hook(self, hook, *triggers):
from nemubot.hooks import Abstract as AbstractHook
@@ -64,7 +63,9 @@ class _ModuleContext:
self.module.logger.info("Send response: %s", res)
def save(self):
- self.context.datastore.save(self.module_name, self.data)
+ # Don't save if no data has been access
+ if hasattr(self, "_data"):
+ context.datastore.save(self.module_name, self.data)
def subparse(self, orig, cnt):
if orig.server in self.context.servers:
@@ -76,6 +77,21 @@ class _ModuleContext:
self._data = self.load_data()
return self._data
+ @data.setter
+ def data(self, value):
+ assert value is not None
+
+ self._data = value
+
+
+ def register_tags(self, **tags):
+ self.extendtags.update(tags)
+
+
+ def unregister_tags(self, *tags):
+ for t in tags:
+ del self.extendtags[t]
+
def unload(self):
"""Perform actions for unloading the module"""
@@ -112,7 +128,7 @@ class ModuleContext(_ModuleContext):
def load_data(self):
- return self.context.datastore.load(self.module_name)
+ return self.context.datastore.load(self.module_name, extendsTags=self.extendtags)
def add_hook(self, hook, *triggers):
from nemubot.hooks import Abstract as AbstractHook
diff --git a/nemubot/tools/xmlparser/__init__.py b/nemubot/tools/xmlparser/__init__.py
index abc5bb9..687bf63 100644
--- a/nemubot/tools/xmlparser/__init__.py
+++ b/nemubot/tools/xmlparser/__init__.py
@@ -51,11 +51,13 @@ class XMLParser:
def __init__(self, knodes):
self.knodes = knodes
+ def _reset(self):
self.stack = list()
self.child = 0
def parse_file(self, path):
+ self._reset()
p = xml.parsers.expat.ParserCreate()
p.StartElementHandler = self.startElement
@@ -69,6 +71,7 @@ class XMLParser:
def parse_string(self, s):
+ self._reset()
p = xml.parsers.expat.ParserCreate()
p.StartElementHandler = self.startElement
@@ -126,10 +129,13 @@ class XMLParser:
if hasattr(self.current, "endElement"):
self.current.endElement(None)
+ if hasattr(self.current, "parsedForm") and callable(self.current.parsedForm):
+ self.stack[-1] = self.current.parsedForm()
+
# Don't remove root
if len(self.stack) > 1:
last = self.stack.pop()
- if hasattr(self.current, "addChild"):
+ if hasattr(self.current, "addChild") and callable(self.current.addChild):
if self.current.addChild(name, last):
return
raise TypeError(name + " tag not expected in " + self.display_stack())
diff --git a/setup.py b/setup.py
index 36dddb4..a400c3c 100755
--- a/setup.py
+++ b/setup.py
@@ -63,6 +63,7 @@ setup(
'nemubot',
'nemubot.config',
'nemubot.datastore',
+ 'nemubot.datastore.nodes',
'nemubot.event',
'nemubot.exception',
'nemubot.hooks',