From a8706d62135a39634aeb4ba5206ae54103697c4a Mon Sep 17 00:00:00 2001 From: nemunaire Date: Sat, 2 Apr 2016 15:20:29 +0200 Subject: [PATCH] New printer and parser for bot data, XML-based --- nemubot/config/module.py | 2 +- nemubot/datastore/abstract.py | 3 +- nemubot/datastore/nodes/__init__.py | 18 ++++ .../xmlparser => datastore/nodes}/basic.py | 68 ++++++------- .../nodes/generic.py} | 44 +++++++++ nemubot/datastore/nodes/python.py | 91 ++++++++++++++++++ nemubot/datastore/nodes/serializable.py | 22 +++++ nemubot/datastore/xml.py | 95 +++++++++++++++++-- nemubot/modulecontext.py | 25 ++++- nemubot/tools/xmlparser/__init__.py | 8 +- setup.py | 1 + 11 files changed, 327 insertions(+), 50 deletions(-) create mode 100644 nemubot/datastore/nodes/__init__.py rename nemubot/{tools/xmlparser => datastore/nodes}/basic.py (67%) rename nemubot/{tools/xmlparser/genericnode.py => datastore/nodes/generic.py} (64%) create mode 100644 nemubot/datastore/nodes/python.py create mode 100644 nemubot/datastore/nodes/serializable.py diff --git a/nemubot/config/module.py b/nemubot/config/module.py index ab51971..7586697 100644 --- a/nemubot/config/module.py +++ b/nemubot/config/module.py @@ -15,7 +15,7 @@ # along with this program. If not, see . from nemubot.config import get_boolean -from nemubot.tools.xmlparser.genericnode import GenericNode +from nemubot.datastore.nodes.generic import GenericNode class Module(GenericNode): diff --git a/nemubot/datastore/abstract.py b/nemubot/datastore/abstract.py index 96e2c0d..f54bbcd 100644 --- a/nemubot/datastore/abstract.py +++ b/nemubot/datastore/abstract.py @@ -23,8 +23,7 @@ class Abstract: """Initialize a new empty storage tree """ - from nemubot.tools.xmlparser import module_state - return module_state.ModuleState("nemubotstate") + return None def open(self): return diff --git a/nemubot/datastore/nodes/__init__.py b/nemubot/datastore/nodes/__init__.py new file mode 100644 index 0000000..e4b2788 --- /dev/null +++ b/nemubot/datastore/nodes/__init__.py @@ -0,0 +1,18 @@ +# Nemubot is a smart and modulable IM bot. +# Copyright (C) 2012-2016 Mercier Pierre-Olivier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from nemubot.datastore.nodes.generic import ParsingNode +from nemubot.datastore.nodes.serializable import Serializable diff --git a/nemubot/tools/xmlparser/basic.py b/nemubot/datastore/nodes/basic.py similarity index 67% rename from nemubot/tools/xmlparser/basic.py rename to nemubot/datastore/nodes/basic.py index 8456629..6fbd136 100644 --- a/nemubot/tools/xmlparser/basic.py +++ b/nemubot/datastore/nodes/basic.py @@ -14,11 +14,16 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -class ListNode: +from nemubot.datastore.nodes.serializable import Serializable + + +class ListNode(Serializable): """XML node representing a Python dictionnnary """ + serializetag = "list" + def __init__(self, **kwargs): self.items = list() @@ -27,6 +32,9 @@ class ListNode: self.items.append(child) return True + def parsedForm(self): + return self.items + def __len__(self): return len(self.items) @@ -44,11 +52,21 @@ class ListNode: return self.items.__repr__() -class DictNode: + def serialize(self): + from nemubot.datastore.nodes.generic import ParsingNode + node = ParsingNode(tag=self.serializetag) + for i in self.items: + node.children.append(ParsingNode.serialize_node(i)) + return node + + +class DictNode(Serializable): """XML node representing a Python dictionnnary """ + serializetag = "dict" + def __init__(self, **kwargs): self.items = dict() self._cur = None @@ -56,44 +74,20 @@ class DictNode: def startElement(self, name, attrs): if self._cur is None and "key" in attrs: - self._cur = (attrs["key"], "") - return True + self._cur = attrs["key"] return False - - def characters(self, content): - if self._cur is not None: - key, cnt = self._cur - if isinstance(cnt, str): - cnt += content - self._cur = key, cnt - - - def endElement(self, name): - if name is None or self._cur is None: - return - - key, cnt = self._cur - if isinstance(cnt, list) and len(cnt) == 1: - self.items[key] = cnt - else: - self.items[key] = cnt - - self._cur = None - return True - - def addChild(self, name, child): if self._cur is None: return False - key, cnt = self._cur - if not isinstance(cnt, list): - cnt = [] - cnt.append(child) - self._cur = key, cnt + self.items[self._cur] = child + self._cur = None return True + def parsedForm(self): + return self.items + def __getitem__(self, item): return self.items[item] @@ -106,3 +100,13 @@ class DictNode: def __repr__(self): return self.items.__repr__() + + + def serialize(self): + from nemubot.datastore.nodes.generic import ParsingNode + node = ParsingNode(tag=self.serializetag) + for k in self.items: + chld = ParsingNode.serialize_node(self.items[k]) + chld.attrs["key"] = k + node.children.append(chld) + return node diff --git a/nemubot/tools/xmlparser/genericnode.py b/nemubot/datastore/nodes/generic.py similarity index 64% rename from nemubot/tools/xmlparser/genericnode.py rename to nemubot/datastore/nodes/generic.py index 9c29a23..c9840bc 100644 --- a/nemubot/tools/xmlparser/genericnode.py +++ b/nemubot/datastore/nodes/generic.py @@ -14,6 +14,9 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from nemubot.datastore.nodes.serializable import Serializable + + class ParsingNode: """Allow any kind of subtags, just keep parsed ones @@ -53,6 +56,47 @@ class ParsingNode: return item in self.attrs + def serialize_node(node, **def_kwargs): + """Serialize any node or basic data to a ParsingNode instance""" + + if isinstance(node, Serializable): + node = node.serialize() + + if isinstance(node, str): + from nemubot.datastore.nodes.python import StringNode + pn = StringNode(**def_kwargs) + pn.value = node + return pn + + elif isinstance(node, int): + from nemubot.datastore.nodes.python import IntNode + pn = IntNode(**def_kwargs) + pn.value = node + return pn + + elif isinstance(node, float): + from nemubot.datastore.nodes.python import FloatNode + pn = FloatNode(**def_kwargs) + pn.value = node + return pn + + elif isinstance(node, list): + from nemubot.datastore.nodes.basic import ListNode + pn = ListNode(**def_kwargs) + pn.items = node + return pn.serialize() + + elif isinstance(node, dict): + from nemubot.datastore.nodes.basic import DictNode + pn = DictNode(**def_kwargs) + pn.items = node + return pn.serialize() + + else: + assert isinstance(node, ParsingNode) + return node + + class GenericNode(ParsingNode): """Consider all subtags as dictionnary diff --git a/nemubot/datastore/nodes/python.py b/nemubot/datastore/nodes/python.py new file mode 100644 index 0000000..6e4278b --- /dev/null +++ b/nemubot/datastore/nodes/python.py @@ -0,0 +1,91 @@ +# Nemubot is a smart and modulable IM bot. +# Copyright (C) 2012-2016 Mercier Pierre-Olivier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from nemubot.datastore.nodes.serializable import Serializable + + +class PythonTypeNode(Serializable): + + """XML node representing a Python simple type + """ + + def __init__(self, **kwargs): + self.value = None + self._cnt = "" + + + def characters(self, content): + self._cnt += content + + + def endElement(self, name): + raise NotImplemented + + + def __repr__(self): + return self.value.__repr__() + + + def parsedForm(self): + return self.value + + def serialize(self): + raise NotImplemented + + +class IntNode(PythonTypeNode): + + serializetag = "int" + + def endElement(self, name): + self.value = int(self._cnt) + return True + + def serialize(self): + from nemubot.datastore.nodes.generic import ParsingNode + node = ParsingNode(tag=self.serializetag) + node.content = str(self.value) + return node + + +class FloatNode(PythonTypeNode): + + serializetag = "float" + + def endElement(self, name): + self.value = float(self._cnt) + return True + + def serialize(self): + from nemubot.datastore.nodes.generic import ParsingNode + node = ParsingNode(tag=self.serializetag) + node.content = str(self.value) + return node + + +class StringNode(PythonTypeNode): + + serializetag = "str" + + def endElement(self, name): + self.value = str(self._cnt) + return True + + def serialize(self): + from nemubot.datastore.nodes.generic import ParsingNode + node = ParsingNode(tag=self.serializetag) + node.content = str(self.value) + return node diff --git a/nemubot/datastore/nodes/serializable.py b/nemubot/datastore/nodes/serializable.py new file mode 100644 index 0000000..e543699 --- /dev/null +++ b/nemubot/datastore/nodes/serializable.py @@ -0,0 +1,22 @@ +# Nemubot is a smart and modulable IM bot. +# Copyright (C) 2012-2016 Mercier Pierre-Olivier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +class Serializable: + + def serialize(self): + # Implementations of this function should return ParsingNode items + return NotImplemented diff --git a/nemubot/datastore/xml.py b/nemubot/datastore/xml.py index 46dca70..266c3ac 100644 --- a/nemubot/datastore/xml.py +++ b/nemubot/datastore/xml.py @@ -1,5 +1,5 @@ # Nemubot is a smart and modulable IM bot. -# Copyright (C) 2012-2015 Mercier Pierre-Olivier +# Copyright (C) 2012-2016 Mercier Pierre-Olivier # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -36,17 +36,24 @@ class XML(Abstract): rotate -- auto-backup files? """ - self.basedir = basedir + self.basedir = os.path.abspath(basedir) self.rotate = rotate self.nb_save = 0 + logger.info("Initiate XML datastore at %s, rotation %s", + self.basedir, + "enabled" if self.rotate else "disabled") + + def open(self): """Lock the directory""" if not os.path.isdir(self.basedir): + logger.debug("Datastore directory not found, creating: %s", self.basedir) os.mkdir(self.basedir) - lock_path = os.path.join(self.basedir, ".used_by_nemubot") + lock_path = self._get_lock_file_path() + logger.debug("Locking datastore directory via %s", lock_path) self.lock_file = open(lock_path, 'a+') ok = True @@ -64,56 +71,91 @@ class XML(Abstract): self.lock_file.write(str(os.getpid())) self.lock_file.flush() + logger.info("Datastore successfuly opened at %s", self.basedir) return True + def close(self): """Release a locked path""" if hasattr(self, "lock_file"): self.lock_file.close() - lock_path = os.path.join(self.basedir, ".used_by_nemubot") + lock_path = self._get_lock_file_path() if os.path.isdir(self.basedir) and os.path.exists(lock_path): os.unlink(lock_path) del self.lock_file + logger.info("Datastore successfully closed at %s", self.basedir) return True + else: + logger.warn("Datastore not open/locked or lock file not found") return False + def _get_data_file_path(self, module): """Get the path to the module data file""" return os.path.join(self.basedir, module + ".xml") - def load(self, module): + + def _get_lock_file_path(self): + """Get the path to the datastore lock file""" + + return os.path.join(self.basedir, ".used_by_nemubot") + + + def load(self, module, extendsTags={}): """Load data for the given module Argument: module -- the module name of data to load """ + logger.debug("Trying to load data for %s%s", + module, + (" with tags: " + ", ".join(extendsTags.keys())) if len(extendsTags) else "") + data_file = self._get_data_file_path(module) + def parse(path): + from nemubot.tools.xmlparser import XMLParser + from nemubot.datastore.nodes import basic as basicNodes + from nemubot.datastore.nodes import python as pythonNodes + + d = { + basicNodes.ListNode.serializetag: basicNodes.ListNode, + basicNodes.DictNode.serializetag: basicNodes.DictNode, + pythonNodes.IntNode.serializetag: pythonNodes.IntNode, + pythonNodes.FloatNode.serializetag: pythonNodes.FloatNode, + pythonNodes.StringNode.serializetag: pythonNodes.StringNode, + } + d.update(extendsTags) + + p = XMLParser(d) + return p.parse_file(path) + # Try to load original file if os.path.isfile(data_file): - from nemubot.tools.xmlparser import parse_file try: - return parse_file(data_file) + return parse(data_file) except xml.parsers.expat.ExpatError: # Try to load from backup for i in range(10): path = data_file + "." + str(i) if os.path.isfile(path): try: - cnt = parse_file(path) + cnt = parse(path) - logger.warn("Restoring from backup: %s", path) + logger.warn("Restoring data from backup: %s", path) return cnt except xml.parsers.expat.ExpatError: continue # Default case: initialize a new empty datastore + logger.warn("No data found in store for %s, creating new set", module) return Abstract.load(self, module) + def _rotate(self, path): """Backup given path @@ -130,6 +172,25 @@ class XML(Abstract): if os.path.isfile(src): os.rename(src, dst) + + def _save_node(self, gen, node): + from nemubot.datastore.nodes.generic import ParsingNode + + # First, get the serialized form of the node + node = ParsingNode.serialize_node(node) + + assert node.tag is not None, "Undefined tag name" + + gen.startElement(node.tag, {k: str(node.attrs[k]) for k in node.attrs}) + + gen.characters(node.content) + + for child in node.children: + self._save_node(gen, child) + + gen.endElement(node.tag) + + def save(self, module, data): """Load data for the given module @@ -139,8 +200,22 @@ class XML(Abstract): """ path = self._get_data_file_path(module) + logger.debug("Trying to save data for module %s in %s", module, path) if self.rotate: self._rotate(path) - return data.save(path) + import tempfile + _, tmpath = tempfile.mkstemp() + with open(tmpath, "w") as f: + import xml.sax.saxutils + gen = xml.sax.saxutils.XMLGenerator(f, "utf-8") + gen.startDocument() + self._save_node(gen, data) + gen.endDocument() + + # Atomic save + import shutil + shutil.move(tmpath, path) + + return True diff --git a/nemubot/modulecontext.py b/nemubot/modulecontext.py index 1d1b3d0..9a4b7d4 100644 --- a/nemubot/modulecontext.py +++ b/nemubot/modulecontext.py @@ -39,6 +39,7 @@ class ModuleContext: self.hooks = list() self.events = list() + self.extendtags = dict() self.debug = context.verbosity > 0 if context is not None else False from nemubot.hooks import Abstract as AbstractHook @@ -46,7 +47,7 @@ class ModuleContext: # Define some callbacks if context is not None: def load_data(): - return context.datastore.load(module_name) + return context.datastore.load(module_name, extendsTags=self.extendtags) def add_hook(hook, *triggers): assert isinstance(hook, AbstractHook), hook @@ -77,8 +78,7 @@ class ModuleContext: else: # Used when using outside of nemubot def load_data(): - from nemubot.tools.xmlparser import module_state - return module_state.ModuleState("nemubotstate") + return None def add_hook(hook, *triggers): assert isinstance(hook, AbstractHook), hook @@ -97,7 +97,9 @@ class ModuleContext: module.logger.info("Send response: %s", res) def save(): - context.datastore.save(module_name, self.data) + # Don't save if no data has been access + if hasattr(self, "_data"): + context.datastore.save(module_name, self.data) def subparse(orig, cnt): if orig.server in context.servers: @@ -120,6 +122,21 @@ class ModuleContext: self._data = self.load_data() return self._data + @data.setter + def data(self, value): + assert value is not None + + self._data = value + + + def register_tags(self, **tags): + self.extendtags.update(tags) + + + def unregister_tags(self, *tags): + for t in tags: + del self.extendtags[t] + def unload(self): """Perform actions for unloading the module""" diff --git a/nemubot/tools/xmlparser/__init__.py b/nemubot/tools/xmlparser/__init__.py index abc5bb9..687bf63 100644 --- a/nemubot/tools/xmlparser/__init__.py +++ b/nemubot/tools/xmlparser/__init__.py @@ -51,11 +51,13 @@ class XMLParser: def __init__(self, knodes): self.knodes = knodes + def _reset(self): self.stack = list() self.child = 0 def parse_file(self, path): + self._reset() p = xml.parsers.expat.ParserCreate() p.StartElementHandler = self.startElement @@ -69,6 +71,7 @@ class XMLParser: def parse_string(self, s): + self._reset() p = xml.parsers.expat.ParserCreate() p.StartElementHandler = self.startElement @@ -126,10 +129,13 @@ class XMLParser: if hasattr(self.current, "endElement"): self.current.endElement(None) + if hasattr(self.current, "parsedForm") and callable(self.current.parsedForm): + self.stack[-1] = self.current.parsedForm() + # Don't remove root if len(self.stack) > 1: last = self.stack.pop() - if hasattr(self.current, "addChild"): + if hasattr(self.current, "addChild") and callable(self.current.addChild): if self.current.addChild(name, last): return raise TypeError(name + " tag not expected in " + self.display_stack()) diff --git a/setup.py b/setup.py index 36dddb4..a400c3c 100755 --- a/setup.py +++ b/setup.py @@ -63,6 +63,7 @@ setup( 'nemubot', 'nemubot.config', 'nemubot.datastore', + 'nemubot.datastore.nodes', 'nemubot.event', 'nemubot.exception', 'nemubot.hooks',