New printer and parser for bot data, XML-based
This commit is contained in:
parent
2992c13ca7
commit
e0af09f3c5
|
|
@ -15,7 +15,7 @@
|
|||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from nemubot.config import get_boolean
|
||||
from nemubot.tools.xmlparser.genericnode import GenericNode
|
||||
from nemubot.datastore.nodes.generic import GenericNode
|
||||
|
||||
|
||||
class Module(GenericNode):
|
||||
|
|
|
|||
|
|
@ -23,8 +23,7 @@ class Abstract:
|
|||
"""Initialize a new empty storage tree
|
||||
"""
|
||||
|
||||
from nemubot.tools.xmlparser import module_state
|
||||
return module_state.ModuleState("nemubotstate")
|
||||
return None
|
||||
|
||||
def open(self):
|
||||
return
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
# Nemubot is a smart and modulable IM bot.
|
||||
# Copyright (C) 2012-2016 Mercier Pierre-Olivier
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from nemubot.datastore.nodes.generic import ParsingNode
|
||||
from nemubot.datastore.nodes.serializable import Serializable
|
||||
|
|
@ -14,11 +14,16 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
class ListNode:
|
||||
from nemubot.datastore.nodes.serializable import Serializable
|
||||
|
||||
|
||||
class ListNode(Serializable):
|
||||
|
||||
"""XML node representing a Python dictionnnary
|
||||
"""
|
||||
|
||||
serializetag = "list"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.items = list()
|
||||
|
||||
|
|
@ -27,6 +32,9 @@ class ListNode:
|
|||
self.items.append(child)
|
||||
return True
|
||||
|
||||
def parsedForm(self):
|
||||
return self.items
|
||||
|
||||
|
||||
def __len__(self):
|
||||
return len(self.items)
|
||||
|
|
@ -44,11 +52,21 @@ class ListNode:
|
|||
return self.items.__repr__()
|
||||
|
||||
|
||||
class DictNode:
|
||||
def serialize(self):
|
||||
from nemubot.datastore.nodes.generic import ParsingNode
|
||||
node = ParsingNode(tag=self.serializetag)
|
||||
for i in self.items:
|
||||
node.children.append(ParsingNode.serialize_node(i))
|
||||
return node
|
||||
|
||||
|
||||
class DictNode(Serializable):
|
||||
|
||||
"""XML node representing a Python dictionnnary
|
||||
"""
|
||||
|
||||
serializetag = "dict"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.items = dict()
|
||||
self._cur = None
|
||||
|
|
@ -56,44 +74,20 @@ class DictNode:
|
|||
|
||||
def startElement(self, name, attrs):
|
||||
if self._cur is None and "key" in attrs:
|
||||
self._cur = (attrs["key"], "")
|
||||
return True
|
||||
self._cur = attrs["key"]
|
||||
return False
|
||||
|
||||
|
||||
def characters(self, content):
|
||||
if self._cur is not None:
|
||||
key, cnt = self._cur
|
||||
if isinstance(cnt, str):
|
||||
cnt += content
|
||||
self._cur = key, cnt
|
||||
|
||||
|
||||
def endElement(self, name):
|
||||
if name is None or self._cur is None:
|
||||
return
|
||||
|
||||
key, cnt = self._cur
|
||||
if isinstance(cnt, list) and len(cnt) == 1:
|
||||
self.items[key] = cnt
|
||||
else:
|
||||
self.items[key] = cnt
|
||||
|
||||
self._cur = None
|
||||
return True
|
||||
|
||||
|
||||
def addChild(self, name, child):
|
||||
if self._cur is None:
|
||||
return False
|
||||
|
||||
key, cnt = self._cur
|
||||
if not isinstance(cnt, list):
|
||||
cnt = []
|
||||
cnt.append(child)
|
||||
self._cur = key, cnt
|
||||
self.items[self._cur] = child
|
||||
self._cur = None
|
||||
return True
|
||||
|
||||
def parsedForm(self):
|
||||
return self.items
|
||||
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.items[item]
|
||||
|
|
@ -106,3 +100,13 @@ class DictNode:
|
|||
|
||||
def __repr__(self):
|
||||
return self.items.__repr__()
|
||||
|
||||
|
||||
def serialize(self):
|
||||
from nemubot.datastore.nodes.generic import ParsingNode
|
||||
node = ParsingNode(tag=self.serializetag)
|
||||
for k in self.items:
|
||||
chld = ParsingNode.serialize_node(self.items[k])
|
||||
chld.attrs["key"] = k
|
||||
node.children.append(chld)
|
||||
return node
|
||||
|
|
@ -14,6 +14,9 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from nemubot.datastore.nodes.serializable import Serializable
|
||||
|
||||
|
||||
class ParsingNode:
|
||||
|
||||
"""Allow any kind of subtags, just keep parsed ones
|
||||
|
|
@ -53,6 +56,47 @@ class ParsingNode:
|
|||
return item in self.attrs
|
||||
|
||||
|
||||
def serialize_node(node, **def_kwargs):
|
||||
"""Serialize any node or basic data to a ParsingNode instance"""
|
||||
|
||||
if isinstance(node, Serializable):
|
||||
node = node.serialize()
|
||||
|
||||
if isinstance(node, str):
|
||||
from nemubot.datastore.nodes.python import StringNode
|
||||
pn = StringNode(**def_kwargs)
|
||||
pn.value = node
|
||||
return pn
|
||||
|
||||
elif isinstance(node, int):
|
||||
from nemubot.datastore.nodes.python import IntNode
|
||||
pn = IntNode(**def_kwargs)
|
||||
pn.value = node
|
||||
return pn
|
||||
|
||||
elif isinstance(node, float):
|
||||
from nemubot.datastore.nodes.python import FloatNode
|
||||
pn = FloatNode(**def_kwargs)
|
||||
pn.value = node
|
||||
return pn
|
||||
|
||||
elif isinstance(node, list):
|
||||
from nemubot.datastore.nodes.basic import ListNode
|
||||
pn = ListNode(**def_kwargs)
|
||||
pn.items = node
|
||||
return pn.serialize()
|
||||
|
||||
elif isinstance(node, dict):
|
||||
from nemubot.datastore.nodes.basic import DictNode
|
||||
pn = DictNode(**def_kwargs)
|
||||
pn.items = node
|
||||
return pn.serialize()
|
||||
|
||||
else:
|
||||
assert isinstance(node, ParsingNode)
|
||||
return node
|
||||
|
||||
|
||||
class GenericNode(ParsingNode):
|
||||
|
||||
"""Consider all subtags as dictionnary
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
# Nemubot is a smart and modulable IM bot.
|
||||
# Copyright (C) 2012-2016 Mercier Pierre-Olivier
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from nemubot.datastore.nodes.serializable import Serializable
|
||||
|
||||
|
||||
class PythonTypeNode(Serializable):
|
||||
|
||||
"""XML node representing a Python simple type
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.value = None
|
||||
self._cnt = ""
|
||||
|
||||
|
||||
def characters(self, content):
|
||||
self._cnt += content
|
||||
|
||||
|
||||
def endElement(self, name):
|
||||
raise NotImplemented
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
return self.value.__repr__()
|
||||
|
||||
|
||||
def parsedForm(self):
|
||||
return self.value
|
||||
|
||||
def serialize(self):
|
||||
raise NotImplemented
|
||||
|
||||
|
||||
class IntNode(PythonTypeNode):
|
||||
|
||||
serializetag = "int"
|
||||
|
||||
def endElement(self, name):
|
||||
self.value = int(self._cnt)
|
||||
return True
|
||||
|
||||
def serialize(self):
|
||||
from nemubot.datastore.nodes.generic import ParsingNode
|
||||
node = ParsingNode(tag=self.serializetag)
|
||||
node.content = str(self.value)
|
||||
return node
|
||||
|
||||
|
||||
class FloatNode(PythonTypeNode):
|
||||
|
||||
serializetag = "float"
|
||||
|
||||
def endElement(self, name):
|
||||
self.value = float(self._cnt)
|
||||
return True
|
||||
|
||||
def serialize(self):
|
||||
from nemubot.datastore.nodes.generic import ParsingNode
|
||||
node = ParsingNode(tag=self.serializetag)
|
||||
node.content = str(self.value)
|
||||
return node
|
||||
|
||||
|
||||
class StringNode(PythonTypeNode):
|
||||
|
||||
serializetag = "str"
|
||||
|
||||
def endElement(self, name):
|
||||
self.value = str(self._cnt)
|
||||
return True
|
||||
|
||||
def serialize(self):
|
||||
from nemubot.datastore.nodes.generic import ParsingNode
|
||||
node = ParsingNode(tag=self.serializetag)
|
||||
node.content = str(self.value)
|
||||
return node
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# Nemubot is a smart and modulable IM bot.
|
||||
# Copyright (C) 2012-2016 Mercier Pierre-Olivier
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
class Serializable:
|
||||
|
||||
def serialize(self):
|
||||
# Implementations of this function should return ParsingNode items
|
||||
return NotImplemented
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# Nemubot is a smart and modulable IM bot.
|
||||
# Copyright (C) 2012-2015 Mercier Pierre-Olivier
|
||||
# Copyright (C) 2012-2016 Mercier Pierre-Olivier
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -36,17 +36,24 @@ class XML(Abstract):
|
|||
rotate -- auto-backup files?
|
||||
"""
|
||||
|
||||
self.basedir = basedir
|
||||
self.basedir = os.path.abspath(basedir)
|
||||
self.rotate = rotate
|
||||
self.nb_save = 0
|
||||
|
||||
logger.info("Initiate XML datastore at %s, rotation %s",
|
||||
self.basedir,
|
||||
"enabled" if self.rotate else "disabled")
|
||||
|
||||
|
||||
def open(self):
|
||||
"""Lock the directory"""
|
||||
|
||||
if not os.path.isdir(self.basedir):
|
||||
logger.debug("Datastore directory not found, creating: %s", self.basedir)
|
||||
os.mkdir(self.basedir)
|
||||
|
||||
lock_path = os.path.join(self.basedir, ".used_by_nemubot")
|
||||
lock_path = self._get_lock_file_path()
|
||||
logger.debug("Locking datastore directory via %s", lock_path)
|
||||
|
||||
self.lock_file = open(lock_path, 'a+')
|
||||
ok = True
|
||||
|
|
@ -64,56 +71,91 @@ class XML(Abstract):
|
|||
self.lock_file.write(str(os.getpid()))
|
||||
self.lock_file.flush()
|
||||
|
||||
logger.info("Datastore successfuly opened at %s", self.basedir)
|
||||
return True
|
||||
|
||||
|
||||
def close(self):
|
||||
"""Release a locked path"""
|
||||
|
||||
if hasattr(self, "lock_file"):
|
||||
self.lock_file.close()
|
||||
lock_path = os.path.join(self.basedir, ".used_by_nemubot")
|
||||
lock_path = self._get_lock_file_path()
|
||||
if os.path.isdir(self.basedir) and os.path.exists(lock_path):
|
||||
os.unlink(lock_path)
|
||||
del self.lock_file
|
||||
logger.info("Datastore successfully closed at %s", self.basedir)
|
||||
return True
|
||||
else:
|
||||
logger.warn("Datastore not open/locked or lock file not found")
|
||||
return False
|
||||
|
||||
|
||||
def _get_data_file_path(self, module):
|
||||
"""Get the path to the module data file"""
|
||||
|
||||
return os.path.join(self.basedir, module + ".xml")
|
||||
|
||||
def load(self, module):
|
||||
|
||||
def _get_lock_file_path(self):
|
||||
"""Get the path to the datastore lock file"""
|
||||
|
||||
return os.path.join(self.basedir, ".used_by_nemubot")
|
||||
|
||||
|
||||
def load(self, module, extendsTags={}):
|
||||
"""Load data for the given module
|
||||
|
||||
Argument:
|
||||
module -- the module name of data to load
|
||||
"""
|
||||
|
||||
logger.debug("Trying to load data for %s%s",
|
||||
module,
|
||||
(" with tags: " + ", ".join(extendsTags.keys())) if len(extendsTags) else "")
|
||||
|
||||
data_file = self._get_data_file_path(module)
|
||||
|
||||
def parse(path):
|
||||
from nemubot.tools.xmlparser import XMLParser
|
||||
from nemubot.datastore.nodes import basic as basicNodes
|
||||
from nemubot.datastore.nodes import python as pythonNodes
|
||||
|
||||
d = {
|
||||
basicNodes.ListNode.serializetag: basicNodes.ListNode,
|
||||
basicNodes.DictNode.serializetag: basicNodes.DictNode,
|
||||
pythonNodes.IntNode.serializetag: pythonNodes.IntNode,
|
||||
pythonNodes.FloatNode.serializetag: pythonNodes.FloatNode,
|
||||
pythonNodes.StringNode.serializetag: pythonNodes.StringNode,
|
||||
}
|
||||
d.update(extendsTags)
|
||||
|
||||
p = XMLParser(d)
|
||||
return p.parse_file(path)
|
||||
|
||||
# Try to load original file
|
||||
if os.path.isfile(data_file):
|
||||
from nemubot.tools.xmlparser import parse_file
|
||||
try:
|
||||
return parse_file(data_file)
|
||||
return parse(data_file)
|
||||
except xml.parsers.expat.ExpatError:
|
||||
# Try to load from backup
|
||||
for i in range(10):
|
||||
path = data_file + "." + str(i)
|
||||
if os.path.isfile(path):
|
||||
try:
|
||||
cnt = parse_file(path)
|
||||
cnt = parse(path)
|
||||
|
||||
logger.warn("Restoring from backup: %s", path)
|
||||
logger.warn("Restoring data from backup: %s", path)
|
||||
|
||||
return cnt
|
||||
except xml.parsers.expat.ExpatError:
|
||||
continue
|
||||
|
||||
# Default case: initialize a new empty datastore
|
||||
logger.warn("No data found in store for %s, creating new set", module)
|
||||
return Abstract.load(self, module)
|
||||
|
||||
|
||||
def _rotate(self, path):
|
||||
"""Backup given path
|
||||
|
||||
|
|
@ -130,6 +172,25 @@ class XML(Abstract):
|
|||
if os.path.isfile(src):
|
||||
os.rename(src, dst)
|
||||
|
||||
|
||||
def _save_node(self, gen, node):
|
||||
from nemubot.datastore.nodes.generic import ParsingNode
|
||||
|
||||
# First, get the serialized form of the node
|
||||
node = ParsingNode.serialize_node(node)
|
||||
|
||||
assert node.tag is not None, "Undefined tag name"
|
||||
|
||||
gen.startElement(node.tag, {k: str(node.attrs[k]) for k in node.attrs})
|
||||
|
||||
gen.characters(node.content)
|
||||
|
||||
for child in node.children:
|
||||
self._save_node(gen, child)
|
||||
|
||||
gen.endElement(node.tag)
|
||||
|
||||
|
||||
def save(self, module, data):
|
||||
"""Load data for the given module
|
||||
|
||||
|
|
@ -139,8 +200,22 @@ class XML(Abstract):
|
|||
"""
|
||||
|
||||
path = self._get_data_file_path(module)
|
||||
logger.debug("Trying to save data for module %s in %s", module, path)
|
||||
|
||||
if self.rotate:
|
||||
self._rotate(path)
|
||||
|
||||
return data.save(path)
|
||||
import tempfile
|
||||
_, tmpath = tempfile.mkstemp()
|
||||
with open(tmpath, "w") as f:
|
||||
import xml.sax.saxutils
|
||||
gen = xml.sax.saxutils.XMLGenerator(f, "utf-8")
|
||||
gen.startDocument()
|
||||
self._save_node(gen, data)
|
||||
gen.endDocument()
|
||||
|
||||
# Atomic save
|
||||
import shutil
|
||||
shutil.move(tmpath, path)
|
||||
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -26,15 +26,14 @@ class _ModuleContext:
|
|||
|
||||
self.hooks = list()
|
||||
self.events = list()
|
||||
self.extendtags = dict()
|
||||
self.debug = False
|
||||
|
||||
from nemubot.config.module import Module
|
||||
self.config = Module(self.module_name)
|
||||
|
||||
|
||||
def load_data(self):
|
||||
from nemubot.tools.xmlparser import module_state
|
||||
return module_state.ModuleState("nemubotstate")
|
||||
return None
|
||||
|
||||
def add_hook(self, hook, *triggers):
|
||||
from nemubot.hooks import Abstract as AbstractHook
|
||||
|
|
@ -64,7 +63,9 @@ class _ModuleContext:
|
|||
self.module.logger.info("Send response: %s", res)
|
||||
|
||||
def save(self):
|
||||
self.context.datastore.save(self.module_name, self.data)
|
||||
# Don't save if no data has been access
|
||||
if hasattr(self, "_data"):
|
||||
context.datastore.save(self.module_name, self.data)
|
||||
|
||||
def subparse(self, orig, cnt):
|
||||
if orig.server in self.context.servers:
|
||||
|
|
@ -76,6 +77,21 @@ class _ModuleContext:
|
|||
self._data = self.load_data()
|
||||
return self._data
|
||||
|
||||
@data.setter
|
||||
def data(self, value):
|
||||
assert value is not None
|
||||
|
||||
self._data = value
|
||||
|
||||
|
||||
def register_tags(self, **tags):
|
||||
self.extendtags.update(tags)
|
||||
|
||||
|
||||
def unregister_tags(self, *tags):
|
||||
for t in tags:
|
||||
del self.extendtags[t]
|
||||
|
||||
|
||||
def unload(self):
|
||||
"""Perform actions for unloading the module"""
|
||||
|
|
@ -112,7 +128,7 @@ class ModuleContext(_ModuleContext):
|
|||
|
||||
|
||||
def load_data(self):
|
||||
return self.context.datastore.load(self.module_name)
|
||||
return self.context.datastore.load(self.module_name, extendsTags=self.extendtags)
|
||||
|
||||
def add_hook(self, hook, *triggers):
|
||||
from nemubot.hooks import Abstract as AbstractHook
|
||||
|
|
|
|||
|
|
@ -51,11 +51,13 @@ class XMLParser:
|
|||
def __init__(self, knodes):
|
||||
self.knodes = knodes
|
||||
|
||||
def _reset(self):
|
||||
self.stack = list()
|
||||
self.child = 0
|
||||
|
||||
|
||||
def parse_file(self, path):
|
||||
self._reset()
|
||||
p = xml.parsers.expat.ParserCreate()
|
||||
|
||||
p.StartElementHandler = self.startElement
|
||||
|
|
@ -69,6 +71,7 @@ class XMLParser:
|
|||
|
||||
|
||||
def parse_string(self, s):
|
||||
self._reset()
|
||||
p = xml.parsers.expat.ParserCreate()
|
||||
|
||||
p.StartElementHandler = self.startElement
|
||||
|
|
@ -126,10 +129,13 @@ class XMLParser:
|
|||
if hasattr(self.current, "endElement"):
|
||||
self.current.endElement(None)
|
||||
|
||||
if hasattr(self.current, "parsedForm") and callable(self.current.parsedForm):
|
||||
self.stack[-1] = self.current.parsedForm()
|
||||
|
||||
# Don't remove root
|
||||
if len(self.stack) > 1:
|
||||
last = self.stack.pop()
|
||||
if hasattr(self.current, "addChild"):
|
||||
if hasattr(self.current, "addChild") and callable(self.current.addChild):
|
||||
if self.current.addChild(name, last):
|
||||
return
|
||||
raise TypeError(name + " tag not expected in " + self.display_stack())
|
||||
|
|
|
|||
Loading…
Reference in New Issue