nemunaire/nemubot
1
0
Fork 0
Code Releases Activity

Rework XML parser: part 1

Browse Source 
This is the first step of the parser refactoring: here we change
	the configuration, next step will change data saving.
This commit is contained in:
nemunaire 2015-10-27 18:03:28 +01:00
parent 92530ef1b2
commit c560e13f24
18 changed files with 388 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bot_sample.xml
View File

@ -1,11 +1,11 @@
<nemubotconfig nick="nemubot" realname="nemubot clone" owner="someone">
<server host="irc.rezosup.org" port="6667" autoconnect="true" caps="znc.in/server-time-iso">
<server uri="irc://irc.rezosup.org:6667" autoconnect="true" caps="znc.in/server-time-iso">
<channel name="#nemutest" />
</server>
<!--
<server host="my_host.local" port="6667" password="secret" autoconnect="true" ip="10.69.42.23" ssl="on">
<server host="ircs://my_host.local:6667" password="secret" autoconnect="true">
<channel name="#nemutest" />
</server>
-->

2
modules/books.py
View File

@ -15,7 +15,7 @@ from more import Response
# LOADING #############################################################
def load(context):
if not context.config or not context.config.getAttribute("goodreadskey"):
if not context.config or "goodreadskey" not in context.config:
raise ImportError("You need a Goodreads API key in order to use this "
"module. Add it to the module configuration file:\n"
"<module name=\"books\" goodreadskey=\"XXXXXX\" />\n"

2
modules/mapquest.py
View File

@ -16,7 +16,7 @@ from more import Response
URL_API = "http://open.mapquestapi.com/geocoding/v1/address?key=%s&location=%%s"
def load(context):
if not context.config or not context.config.hasAttribute("apikey"):
if not context.config or "apikey" not in context.config:
raise ImportError("You need a MapQuest API key in order to use this "
"module. Add it to the module configuration file:\n"
"<module name=\"mapquest\" key=\"XXXXXXXXXXXXXXXX\" "

2
modules/networking/whois.py
View File

@ -11,7 +11,7 @@ URL_WHOIS = "http://www.whoisxmlapi.com/whoisserver/WhoisService?rid=1&domainNam
def load(CONF, add_hook):
global URL_WHOIS
if not CONF or not CONF.hasNode("whoisxmlapi") or not CONF.getNode("whoisxmlapi").hasAttribute("username") or not CONF.getNode("whoisxmlapi").hasAttribute("password"):
if not CONF or not CONF.hasNode("whoisxmlapi") or "username" not in CONF.getNode("whoisxmlapi") or "password" not in CONF.getNode("whoisxmlapi"):
raise ImportError("You need a WhoisXML API account in order to use "
"the !netwhois feature. Add it to the module "
"configuration file:\n<whoisxmlapi username=\"XX\" "

2
modules/syno.py
View File

@ -21,7 +21,7 @@ def help_full():
def load(context):
global lang_binding
if not context.config or not context.config.hasAttribute("bighugelabskey"):
if not context.config or not "bighugelabskey" in context.config:
logger.error("You need a NigHugeLabs API key in order to have english "
"theasorus. Add it to the module configuration file:\n"
"<module name=\"syno\" bighugelabskey=\"XXXXXXXXXXXXXXXX\""

2
modules/tpb.py
View File

@ -13,7 +13,7 @@ from more import Response
URL_TPBAPI = None
def load(context):
if not context.config or not context.config.hasAttribute("url"):
if not context.config or "url" not in context.config:
raise ImportError("You need a TPB API in order to use the !tpb feature"
". Add it to the module configuration file:\n<module"
"name=\"tpb\" url=\"http://tpbapi.org/\" />\nSample "

2
modules/translate.py
View File

@ -19,7 +19,7 @@ LANG = ["ar", "zh", "cz", "en", "fr", "gr", "it",
URL = "http://api.wordreference.com/0.8/%s/json/%%s%%s/%%s"
def load(context):
if not context.config or not context.config.hasAttribute("wrapikey"):
if not context.config or "wrapikey" not in context.config:
raise ImportError("You need a WordReference API key in order to use "
"this module. Add it to the module configuration "
"file:\n<module name=\"translate\" wrapikey=\"XXXXX\""

2
modules/velib.py
View File

@ -18,7 +18,7 @@ URL_API = None # http://www.velib.paris.fr/service/stationdetails/paris/%s
def load(context):
global URL_API
if not context.config or not context.config.hasAttribute("url"):
if not context.config or "url" not in context.config:
raise ImportError("Please provide url attribute in the module configuration")
URL_API = context.config["url"]
context.data.setIndex("name", "station")

2
modules/weather.py
View File

@ -21,7 +21,7 @@ from more import Response
URL_DSAPI = "https://api.forecast.io/forecast/%s/%%s,%%s"
def load(context):
if not context.config or not context.config.hasAttribute("darkskyapikey"):
if not context.config or "darkskyapikey" not in context.config:
raise ImportError("You need a Dark-Sky API key in order to use this "
"module. Add it to the module configuration file:\n"
"<module name=\"weather\" darkskyapikey=\"XXX\" />\n"

2
modules/whois.py
View File

@ -16,7 +16,7 @@ PASSWD_FILE = None
def load(context):
global PASSWD_FILE
if not context.config or not context.config.hasAttribute("passwd"):
if not context.config or "passwd" not in context.config:
print("No passwd file given")
return None
PASSWD_FILE = context.config["passwd"]

2
modules/wolframalpha.py
View File

@ -19,7 +19,7 @@ URL_API = "http://api.wolframalpha.com/v2/query?input=%%s&appid=%s"
def load(context):
global URL_API
if not context.config or not context.config.hasAttribute("apikey"):
if not context.config or "apikey" not in context.config:
raise ImportError ("You need a Wolfram|Alpha API key in order to use "
"this module. Add it to the module configuration: "
"\n<module name=\"wolframalpha\" "

54
nemubot/bot.py
View File

@ -205,12 +205,62 @@ class Bot(threading.Thread):
self.quit()
elif action[0] == "loadconf":
for path in action[1:]:
from nemubot.tools.config import load_file
load_file(path, self)
self.load_file(path)
self.sync_queue.task_done()
# Config methods
def load_file(self, filename):
"""Load a configuration file
Arguments:
filename -- the path to the file to load
"""
import os
# Unexisting file, assume a name was passed, import the module!
if not os.path.isfile(filename):
return self.import_module(filename)
from nemubot.tools.config import config_nodes
from nemubot.tools.xmlparser import XMLParser
try:
p = XMLParser(config_nodes)
config = p.parse_file(filename)
except:
logger.exception("Can't load `%s'; this is not a valid nemubot "
"configuration file." % filename)
return False
# Preset each server in this file
for server in config.servers:
srv = server.server(config)
# Add the server in the context
if self.add_server(srv, server.autoconnect):
logger.info("Server '%s' successfully added." % srv.id)
else:
logger.error("Can't add server '%s'." % srv.id)
# Load module and their configuration
for mod in config.modules:
self.modules_configuration[mod.name] = mod
if mod.autoload:
try:
__import__(mod.name)
except:
logger.exception("Exception occurs when loading module"
" '%s'", mod.name)
# Load files asked by the configuration file
for load in config.includes:
self.load_file(load.path)
# Events methods
def add_event(self, evt, eid=None, module_src=None):

6
nemubot/channel.py
View File

@ -1,5 +1,3 @@
# coding=utf-8
# Nemubot is a smart and modulable IM bot.
# Copyright (C) 2012-2015 Mercier Pierre-Olivier
#
@ -23,16 +21,18 @@ class Channel:
"""A chat room"""
def __init__(self, name, password=None):
def __init__(self, name, password=None, encoding=None):
"""Initialize the channel
Arguments:
name -- the channel name
password -- the optional password use to join it
encoding -- the optional encoding of the channel
"""
self.name = name
self.password = password
self.encoding = encoding
self.people = dict()
self.topic = ""
self.logger = logging.getLogger("nemubot.channel." + name)

4
nemubot/prompt/builtins.py
View File

@ -61,10 +61,8 @@ def liste(toks, context, prompt):
def load(toks, context, prompt):
"""Load an XML configuration file"""
if len(toks) > 1:
from nemubot.tools.config import load_file
for filename in toks[1:]:
load_file(filename, context)
context.load_file(filename)
else:
print ("Not enough arguments. `load' takes a filename.")
return 1

241
nemubot/tools/config.py
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
# Nemubot is a smart and modulable IM bot.
# Copyright (C) 2012-2015 Mercier Pierre-Olivier
#
@ -16,123 +14,146 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import logging
logger = logging.getLogger("nemubot.tools.config")
def get_boolean(d, k, default=False):
return ((k in d and d[k].lower() != "false" and d[k].lower() != "off") or
(k not in d and default))
def _load_server(config, xmlnode):
"""Load a server configuration
Arguments:
config -- the global configuration
xmlnode -- the current server configuration node
"""
opts = {
"host": xmlnode["host"],
"ssl": xmlnode.hasAttribute("ssl") and xmlnode["ssl"].lower() == "true",
"nick": xmlnode["nick"] if xmlnode.hasAttribute("nick") else config["nick"],
"owner": xmlnode["owner"] if xmlnode.hasAttribute("owner") else config["owner"],
}
# Optional keyword arguments
for optional_opt in [ "port", "username", "realname",
"password", "encoding", "caps" ]:
if xmlnode.hasAttribute(optional_opt):
opts[optional_opt] = xmlnode[optional_opt]
elif optional_opt in config:
opts[optional_opt] = config[optional_opt]
# Command to send on connection
if "on_connect" in xmlnode:
def on_connect():
yield xmlnode["on_connect"]
opts["on_connect"] = on_connect
# Channels to autojoin on connection
if xmlnode.hasNode("channel"):
opts["channels"] = list()
for chn in xmlnode.getNodes("channel"):
opts["channels"].append((chn["name"], chn["password"])
if chn["password"] is not None
else chn["name"])
# Server/client capabilities
if "caps" in xmlnode or "caps" in config:
capsl = (xmlnode["caps"] if xmlnode.hasAttribute("caps")
else config["caps"]).lower()
if capsl == "no" or capsl == "off" or capsl == "false":
opts["caps"] = None
def get_boolean(s):
if isinstance(s, bool):
return s
else:
opts["caps"] = capsl.split(',')
return (s and s != "0" and s.lower() != "false" and s.lower() != "off")
class GenericNode:
def __init__(self, tag, **kwargs):
self.tag = tag
self.attrs = kwargs
self.content = ""
self.children = []
self._cur = None
self._deep_cur = 0
def startElement(self, name, attrs):
if self._cur is None:
self._cur = GenericNode(name, **attrs)
self._deep_cur = 0
else:
opts["caps"] = list()
self._deep_cur += 1
self._cur.startElement(name, attrs)
return True
# Bind the protocol asked to the corresponding implementation
if "protocol" not in xmlnode or xmlnode["protocol"] == "irc":
from nemubot.server.IRC import IRC as IRCServer
srvcls = IRCServer
def characters(self, content):
if self._cur is None:
self.content += content
else:
raise Exception("Unhandled protocol '%s'" %
xmlnode["protocol"])
# Initialize the server
return srvcls(**opts)
self._cur.characters(content)
def load_file(filename, context):
"""Load the configuration file
def endElement(self, name):
if name is None:
return
Arguments:
filename -- the path to the file to load
"""
import os
if os.path.isfile(filename):
from nemubot.tools.xmlparser import parse_file
config = parse_file(filename)
# This is a true nemubot configuration file, load it!
if config.getName() == "nemubotconfig":
# Preset each server in this file
for server in config.getNodes("server"):
srv = _load_server(config, server)
# Add the server in the context
if context.add_server(srv, get_boolean(server, "autoconnect")):
logger.info("Server '%s' successfully added." % srv.id)
if self._deep_cur:
self._deep_cur -= 1
self._cur.endElement(name)
else:
logger.error("Can't add server '%s'." % srv.id)
# Load module and their configuration
for mod in config.getNodes("module"):
context.modules_configuration[mod["name"]] = mod
if get_boolean(mod, "autoload", default=True):
try:
__import__(mod["name"])
except:
logger.exception("Exception occurs when loading module"
" '%s'", mod["name"])
self.children.append(self._cur)
self._cur = None
return True
# Load files asked by the configuration file
for load in config.getNodes("include"):
load_file(load["path"], context)
def hasNode(self, nodename):
return self.getNode(nodename) is not None
# Other formats
else:
logger.error("Can't load `%s'; this is not a valid nemubot "
"configuration file." % filename)
# Unexisting file, assume a name was passed, import the module!
else:
context.import_module(filename)
def getNode(self, nodename):
for c in self.children:
if c is not None and c.tag == nodename:
return c
return None
def __getitem__(self, item):
return self.attrs[item]
def __contains__(self, item):
return item in self.attrs
class NemubotConfig:
def __init__(self, nick="nemubot", realname="nemubot", owner=None,
ip=None, ssl=False, caps=None, encoding="utf-8"):
self.nick = nick
self.realname = realname
self.owner = owner
self.ip = ip
self.caps = caps.split(" ") if caps is not None else []
self.encoding = encoding
self.servers = []
self.modules = []
self.includes = []
def addChild(self, name, child):
if name == "module" and isinstance(child, ModuleConfig):
self.modules.append(child)
return True
elif name == "server" and isinstance(child, ServerConfig):
self.servers.append(child)
return True
elif name == "include" and isinstance(child, IncludeConfig):
self.includes.append(child)
return True
class ServerConfig:
def __init__(self, uri="irc://nemubot@localhost/", autoconnect=True, caps=None, **kwargs):
self.uri = uri
self.autoconnect = autoconnect
self.caps = caps.split(" ") if caps is not None else []
self.args = kwargs
self.channels = []
def addChild(self, name, child):
if name == "channel" and isinstance(child, Channel):
self.channels.append(child)
return True
def server(self, parent):
from nemubot.server import factory
for a in ["nick", "owner", "realname", "encoding"]:
if a not in self.args:
self.args[a] = getattr(parent, a)
self.caps += parent.caps
return factory(self.uri, **self.args)
class IncludeConfig:
def __init__(self, path):
self.path = path
class ModuleConfig(GenericNode):
def __init__(self, name, autoload=True, **kwargs):
super(ModuleConfig, self).__init__(None, **kwargs)
self.name = name
self.autoload = get_boolean(autoload)
from nemubot.channel import Channel
config_nodes = {
"nemubotconfig": NemubotConfig,
"server": ServerConfig,
"channel": Channel,
"module": ModuleConfig,
"include": IncludeConfig,
}

82
nemubot/tools/test_xmlparser.py Normal file
View File

@ -0,0 +1,82 @@
import unittest
import xml.parsers.expat
from nemubot.tools.xmlparser import XMLParser
class StringNode():
def __init__(self):
self.string = ""
def characters(self, content):
self.string += content
class TestNode():
def __init__(self, option=None):
self.option = option
self.mystr = None
def addChild(self, name, child):
self.mystr = child.string
class Test2Node():
def __init__(self, option=None):
self.option = option
self.mystrs = list()
def startElement(self, name, attrs):
if name == "string":
self.mystrs.append(attrs["value"])
return True
class TestXMLParser(unittest.TestCase):
def test_parser1(self):
p = xml.parsers.expat.ParserCreate()
mod = XMLParser({"string": StringNode})
p.StartElementHandler = mod.startElement
p.CharacterDataHandler = mod.characters
p.EndElementHandler = mod.endElement
p.Parse("<string>toto</string>", 1)
self.assertEqual(mod.root.string, "toto")
def test_parser2(self):
p = xml.parsers.expat.ParserCreate()
mod = XMLParser({"string": StringNode, "test": TestNode})
p.StartElementHandler = mod.startElement
p.CharacterDataHandler = mod.characters
p.EndElementHandler = mod.endElement
p.Parse("<test option='123'><string>toto</string></test>", 1)
self.assertEqual(mod.root.option, "123")
self.assertEqual(mod.root.mystr, "toto")
def test_parser3(self):
p = xml.parsers.expat.ParserCreate()
mod = XMLParser({"string": StringNode, "test": Test2Node})
p.StartElementHandler = mod.startElement
p.CharacterDataHandler = mod.characters
p.EndElementHandler = mod.endElement
p.Parse("<test><string value='toto' /><string value='toto2' /></test>", 1)
self.assertEqual(mod.root.option, None)
self.assertEqual(len(mod.root.mystrs), 2)
self.assertEqual(mod.root.mystrs[0], "toto")
self.assertEqual(mod.root.mystrs[1], "toto2")
if __name__ == '__main__':
unittest.main()

104
nemubot/tools/xmlparser/__init__.py
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
# Nemubot is a smart and modulable IM bot.
# Copyright (C) 2012-2015 Mercier Pierre-Olivier
#
@ -48,9 +46,107 @@ class ModuleStatesFile:
self.root = child
class XMLParser:
def __init__(self, knodes):
self.knodes = knodes
self.stack = list()
self.child = 0
def parse_file(self, path):
p = xml.parsers.expat.ParserCreate()
p.StartElementHandler = self.startElement
p.CharacterDataHandler = self.characters
p.EndElementHandler = self.endElement
with open(path, "rb") as f:
p.ParseFile(f)
return self.root
def parse_string(self, s):
p = xml.parsers.expat.ParserCreate()
p.StartElementHandler = self.startElement
p.CharacterDataHandler = self.characters
p.EndElementHandler = self.endElement
p.Parse(s, 1)
return self.root
@property
def root(self):
if len(self.stack):
return self.stack[0]
else:
return None
@property
def current(self):
if len(self.stack):
return self.stack[-1]
else:
return None
def display_stack(self):
return " in ".join([str(type(s).__name__) for s in reversed(self.stack)])
def startElement(self, name, attrs):
if not self.current or not hasattr(self.current, "startElement") or not self.current.startElement(name, attrs):
if name not in self.knodes:
raise TypeError(name + " is not a known type to decode")
else:
self.stack.append(self.knodes[name](**attrs))
else:
self.child += 1
def characters(self, content):
if self.current and hasattr(self.current, "characters"):
self.current.characters(content)
def endElement(self, name):
if self.child:
self.child -= 1
if hasattr(self.current, "endElement"):
self.current.endElement(name)
return
if hasattr(self.current, "endElement"):
self.current.endElement(None)
# Don't remove root
if len(self.stack) > 1:
last = self.stack.pop()
if hasattr(self.current, "addChild"):
if self.current.addChild(name, last):
return
raise TypeError(name + " tag not expected in " + self.display_stack())
def parse_file(filename):
with open(filename, "r") as f:
return parse_string(f.read())
p = xml.parsers.expat.ParserCreate()
mod = ModuleStatesFile()
p.StartElementHandler = mod.startElement
p.EndElementHandler = mod.endElement
p.CharacterDataHandler = mod.characters
with open(filename, "rb") as f:
p.ParseFile(f)
return mod.root
def parse_string(string):

7
nemubot/tools/xmlparser/node.py
View File

@ -1,5 +1,3 @@
# coding=utf-8
# Nemubot is a smart and modulable IM bot.
# Copyright (C) 2012-2015 Mercier Pierre-Olivier
#
@ -37,7 +35,7 @@ class ModuleState:
"""Get the name of the current node"""
return self.name
def display(self, level = 0):
def display(self, level=0):
ret = ""
out = list()
for k in self.attributes:
@ -51,6 +49,9 @@ class ModuleState:
def __str__(self):
return self.display()
def __repr__(self):
return self.display()
def __getitem__(self, i):
"""Return the attribute asked"""
return self.getAttribute(i)