nemunaire/nemubot
1
0
Fork 0
Code Releases Activity

New printer and parser for bot data, XML-based

Browse Source
This commit is contained in:
nemunaire 2016-04-02 15:20:29 +02:00
parent 2992c13ca7
commit e0af09f3c5
11 changed files with 327 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nemubot/config/module.py
View File

@ -15,7 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from nemubot.config import get_boolean
from nemubot.tools.xmlparser.genericnode import GenericNode
from nemubot.datastore.nodes.generic import GenericNode
class Module(GenericNode):

3
nemubot/datastore/abstract.py
View File

@ -23,8 +23,7 @@ class Abstract:
"""Initialize a new empty storage tree
"""
from nemubot.tools.xmlparser import module_state
return module_state.ModuleState("nemubotstate")
return None
def open(self):
return

18
nemubot/datastore/nodes/__init__.py Normal file
View File

@ -0,0 +1,18 @@
# Nemubot is a smart and modulable IM bot.
# Copyright (C) 2012-2016 Mercier Pierre-Olivier
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from nemubot.datastore.nodes.generic import ParsingNode
from nemubot.datastore.nodes.serializable import Serializable

68
nemubot/tools/xmlparser/basic.py → nemubot/datastore/nodes/basic.py
View File

@ -14,11 +14,16 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
class ListNode:
from nemubot.datastore.nodes.serializable import Serializable
class ListNode(Serializable):
"""XML node representing a Python dictionnnary
"""
serializetag = "list"
def __init__(self, **kwargs):
self.items = list()
@ -27,6 +32,9 @@ class ListNode:
self.items.append(child)
return True
def parsedForm(self):
return self.items
def __len__(self):
return len(self.items)
@ -44,11 +52,21 @@ class ListNode:
return self.items.__repr__()
class DictNode:
def serialize(self):
from nemubot.datastore.nodes.generic import ParsingNode
node = ParsingNode(tag=self.serializetag)
for i in self.items:
node.children.append(ParsingNode.serialize_node(i))
return node
class DictNode(Serializable):
"""XML node representing a Python dictionnnary
"""
serializetag = "dict"
def __init__(self, **kwargs):
self.items = dict()
self._cur = None
@ -56,44 +74,20 @@ class DictNode:
def startElement(self, name, attrs):
if self._cur is None and "key" in attrs:
self._cur = (attrs["key"], "")
return True
self._cur = attrs["key"]
return False
def characters(self, content):
if self._cur is not None:
key, cnt = self._cur
if isinstance(cnt, str):
cnt += content
self._cur = key, cnt
def endElement(self, name):
if name is None or self._cur is None:
return
key, cnt = self._cur
if isinstance(cnt, list) and len(cnt) == 1:
self.items[key] = cnt
else:
self.items[key] = cnt
self._cur = None
return True
def addChild(self, name, child):
if self._cur is None:
return False
key, cnt = self._cur
if not isinstance(cnt, list):
cnt = []
cnt.append(child)
self._cur = key, cnt
self.items[self._cur] = child
self._cur = None
return True
def parsedForm(self):
return self.items
def __getitem__(self, item):
return self.items[item]
@ -106,3 +100,13 @@ class DictNode:
def __repr__(self):
return self.items.__repr__()
def serialize(self):
from nemubot.datastore.nodes.generic import ParsingNode
node = ParsingNode(tag=self.serializetag)
for k in self.items:
chld = ParsingNode.serialize_node(self.items[k])
chld.attrs["key"] = k
node.children.append(chld)
return node

44
nemubot/tools/xmlparser/genericnode.py → nemubot/datastore/nodes/generic.py
View File

@ -14,6 +14,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from nemubot.datastore.nodes.serializable import Serializable
class ParsingNode:
"""Allow any kind of subtags, just keep parsed ones
@ -53,6 +56,47 @@ class ParsingNode:
return item in self.attrs
def serialize_node(node, **def_kwargs):
"""Serialize any node or basic data to a ParsingNode instance"""
if isinstance(node, Serializable):
node = node.serialize()
if isinstance(node, str):
from nemubot.datastore.nodes.python import StringNode
pn = StringNode(**def_kwargs)
pn.value = node
return pn
elif isinstance(node, int):
from nemubot.datastore.nodes.python import IntNode
pn = IntNode(**def_kwargs)
pn.value = node
return pn
elif isinstance(node, float):
from nemubot.datastore.nodes.python import FloatNode
pn = FloatNode(**def_kwargs)
pn.value = node
return pn
elif isinstance(node, list):
from nemubot.datastore.nodes.basic import ListNode
pn = ListNode(**def_kwargs)
pn.items = node
return pn.serialize()
elif isinstance(node, dict):
from nemubot.datastore.nodes.basic import DictNode
pn = DictNode(**def_kwargs)
pn.items = node
return pn.serialize()
else:
assert isinstance(node, ParsingNode)
return node
class GenericNode(ParsingNode):
"""Consider all subtags as dictionnary

91
nemubot/datastore/nodes/python.py Normal file
View File

@ -0,0 +1,91 @@
# Nemubot is a smart and modulable IM bot.
# Copyright (C) 2012-2016 Mercier Pierre-Olivier
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from nemubot.datastore.nodes.serializable import Serializable
class PythonTypeNode(Serializable):
"""XML node representing a Python simple type
"""
def __init__(self, **kwargs):
self.value = None
self._cnt = ""
def characters(self, content):
self._cnt += content
def endElement(self, name):
raise NotImplemented
def __repr__(self):
return self.value.__repr__()
def parsedForm(self):
return self.value
def serialize(self):
raise NotImplemented
class IntNode(PythonTypeNode):
serializetag = "int"
def endElement(self, name):
self.value = int(self._cnt)
return True
def serialize(self):
from nemubot.datastore.nodes.generic import ParsingNode
node = ParsingNode(tag=self.serializetag)
node.content = str(self.value)
return node
class FloatNode(PythonTypeNode):
serializetag = "float"
def endElement(self, name):
self.value = float(self._cnt)
return True
def serialize(self):
from nemubot.datastore.nodes.generic import ParsingNode
node = ParsingNode(tag=self.serializetag)
node.content = str(self.value)
return node
class StringNode(PythonTypeNode):
serializetag = "str"
def endElement(self, name):
self.value = str(self._cnt)
return True
def serialize(self):
from nemubot.datastore.nodes.generic import ParsingNode
node = ParsingNode(tag=self.serializetag)
node.content = str(self.value)
return node

22
nemubot/datastore/nodes/serializable.py Normal file
View File

@ -0,0 +1,22 @@
# Nemubot is a smart and modulable IM bot.
# Copyright (C) 2012-2016 Mercier Pierre-Olivier
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
class Serializable:
def serialize(self):
# Implementations of this function should return ParsingNode items
return NotImplemented

95
nemubot/datastore/xml.py
View File

@ -1,5 +1,5 @@
# Nemubot is a smart and modulable IM bot.
# Copyright (C) 2012-2015 Mercier Pierre-Olivier
# Copyright (C) 2012-2016 Mercier Pierre-Olivier
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -36,17 +36,24 @@ class XML(Abstract):
rotate -- auto-backup files?
"""
self.basedir = basedir
self.basedir = os.path.abspath(basedir)
self.rotate = rotate
self.nb_save = 0
logger.info("Initiate XML datastore at %s, rotation %s",
self.basedir,
"enabled" if self.rotate else "disabled")
def open(self):
"""Lock the directory"""
if not os.path.isdir(self.basedir):
logger.debug("Datastore directory not found, creating: %s", self.basedir)
os.mkdir(self.basedir)
lock_path = os.path.join(self.basedir, ".used_by_nemubot")
lock_path = self._get_lock_file_path()
logger.debug("Locking datastore directory via %s", lock_path)
self.lock_file = open(lock_path, 'a+')
ok = True
@ -64,56 +71,91 @@ class XML(Abstract):
self.lock_file.write(str(os.getpid()))
self.lock_file.flush()
logger.info("Datastore successfuly opened at %s", self.basedir)
return True
def close(self):
"""Release a locked path"""
if hasattr(self, "lock_file"):
self.lock_file.close()
lock_path = os.path.join(self.basedir, ".used_by_nemubot")
lock_path = self._get_lock_file_path()
if os.path.isdir(self.basedir) and os.path.exists(lock_path):
os.unlink(lock_path)
del self.lock_file
logger.info("Datastore successfully closed at %s", self.basedir)
return True
else:
logger.warn("Datastore not open/locked or lock file not found")
return False
def _get_data_file_path(self, module):
"""Get the path to the module data file"""
return os.path.join(self.basedir, module + ".xml")
def load(self, module):
def _get_lock_file_path(self):
"""Get the path to the datastore lock file"""
return os.path.join(self.basedir, ".used_by_nemubot")
def load(self, module, extendsTags={}):
"""Load data for the given module
Argument:
module -- the module name of data to load
"""
logger.debug("Trying to load data for %s%s",
module,
(" with tags: " + ", ".join(extendsTags.keys())) if len(extendsTags) else "")
data_file = self._get_data_file_path(module)
def parse(path):
from nemubot.tools.xmlparser import XMLParser
from nemubot.datastore.nodes import basic as basicNodes
from nemubot.datastore.nodes import python as pythonNodes
d = {
basicNodes.ListNode.serializetag: basicNodes.ListNode,
basicNodes.DictNode.serializetag: basicNodes.DictNode,
pythonNodes.IntNode.serializetag: pythonNodes.IntNode,
pythonNodes.FloatNode.serializetag: pythonNodes.FloatNode,
pythonNodes.StringNode.serializetag: pythonNodes.StringNode,
}
d.update(extendsTags)
p = XMLParser(d)
return p.parse_file(path)
# Try to load original file
if os.path.isfile(data_file):
from nemubot.tools.xmlparser import parse_file
try:
return parse_file(data_file)
return parse(data_file)
except xml.parsers.expat.ExpatError:
# Try to load from backup
for i in range(10):
path = data_file + "." + str(i)
if os.path.isfile(path):
try:
cnt = parse_file(path)
cnt = parse(path)
logger.warn("Restoring from backup: %s", path)
logger.warn("Restoring data from backup: %s", path)
return cnt
except xml.parsers.expat.ExpatError:
continue
# Default case: initialize a new empty datastore
logger.warn("No data found in store for %s, creating new set", module)
return Abstract.load(self, module)
def _rotate(self, path):
"""Backup given path
@ -130,6 +172,25 @@ class XML(Abstract):
if os.path.isfile(src):
os.rename(src, dst)
def _save_node(self, gen, node):
from nemubot.datastore.nodes.generic import ParsingNode
# First, get the serialized form of the node
node = ParsingNode.serialize_node(node)
assert node.tag is not None, "Undefined tag name"
gen.startElement(node.tag, {k: str(node.attrs[k]) for k in node.attrs})
gen.characters(node.content)
for child in node.children:
self._save_node(gen, child)
gen.endElement(node.tag)
def save(self, module, data):
"""Load data for the given module
@ -139,8 +200,22 @@ class XML(Abstract):
"""
path = self._get_data_file_path(module)
logger.debug("Trying to save data for module %s in %s", module, path)
if self.rotate:
self._rotate(path)
return data.save(path)
import tempfile
_, tmpath = tempfile.mkstemp()
with open(tmpath, "w") as f:
import xml.sax.saxutils
gen = xml.sax.saxutils.XMLGenerator(f, "utf-8")
gen.startDocument()
self._save_node(gen, data)
gen.endDocument()
# Atomic save
import shutil
shutil.move(tmpath, path)
return True

26
nemubot/modulecontext.py
View File

@ -26,15 +26,14 @@ class _ModuleContext:
self.hooks = list()
self.events = list()
self.extendtags = dict()
self.debug = False
from nemubot.config.module import Module
self.config = Module(self.module_name)
def load_data(self):
from nemubot.tools.xmlparser import module_state
return module_state.ModuleState("nemubotstate")
return None
def add_hook(self, hook, *triggers):
from nemubot.hooks import Abstract as AbstractHook
@ -64,7 +63,9 @@ class _ModuleContext:
self.module.logger.info("Send response: %s", res)
def save(self):
self.context.datastore.save(self.module_name, self.data)
# Don't save if no data has been access
if hasattr(self, "_data"):
context.datastore.save(self.module_name, self.data)
def subparse(self, orig, cnt):
if orig.server in self.context.servers:
@ -76,6 +77,21 @@ class _ModuleContext:
self._data = self.load_data()
return self._data
@data.setter
def data(self, value):
assert value is not None
self._data = value
def register_tags(self, **tags):
self.extendtags.update(tags)
def unregister_tags(self, *tags):
for t in tags:
del self.extendtags[t]
def unload(self):
"""Perform actions for unloading the module"""
@ -112,7 +128,7 @@ class ModuleContext(_ModuleContext):
def load_data(self):
return self.context.datastore.load(self.module_name)
return self.context.datastore.load(self.module_name, extendsTags=self.extendtags)
def add_hook(self, hook, *triggers):
from nemubot.hooks import Abstract as AbstractHook

8
nemubot/tools/xmlparser/__init__.py
View File

@ -51,11 +51,13 @@ class XMLParser:
def __init__(self, knodes):
self.knodes = knodes
def _reset(self):
self.stack = list()
self.child = 0
def parse_file(self, path):
self._reset()
p = xml.parsers.expat.ParserCreate()
p.StartElementHandler = self.startElement
@ -69,6 +71,7 @@ class XMLParser:
def parse_string(self, s):
self._reset()
p = xml.parsers.expat.ParserCreate()
p.StartElementHandler = self.startElement
@ -126,10 +129,13 @@ class XMLParser:
if hasattr(self.current, "endElement"):
self.current.endElement(None)
if hasattr(self.current, "parsedForm") and callable(self.current.parsedForm):
self.stack[-1] = self.current.parsedForm()
# Don't remove root
if len(self.stack) > 1:
last = self.stack.pop()
if hasattr(self.current, "addChild"):
if hasattr(self.current, "addChild") and callable(self.current.addChild):
if self.current.addChild(name, last):
return
raise TypeError(name + " tag not expected in " + self.display_stack())

1
setup.py
View File

@ -63,6 +63,7 @@ setup(
'nemubot',
'nemubot.config',
'nemubot.datastore',
'nemubot.datastore.nodes',
'nemubot.event',
'nemubot.exception',
'nemubot.hooks',