events: Use the new data parser, knodes based

Closing issue #46

README.md

@@ -7,7 +7,7 @@ An extremely modulable IRC bot, built around XML configuration files!
 Requirements
 ------------
 
-*nemubot* requires at least Python 3.3 to work.
+*nemubot* requires at least Python 3.4 to work, as it uses `asyncio`.
 
 Connecting to SSL server requires [this patch](http://bugs.python.org/issue27629).
 

modules/alias.py

@@ -12,7 +12,7 @@ from nemubot.message import Command
 from nemubot.tools.human import guess
 from nemubot.tools.xmlparser.node import ModuleState
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################
@@ -185,7 +185,7 @@ def cmd_listvars(msg):
 def cmd_set(msg):
     if len(msg.args) < 2:
         raise IMException("!set take two args: the key and the value.")
-    set_variable(msg.args[0], " ".join(msg.args[1:]), msg.nick)
+    set_variable(msg.args[0], " ".join(msg.args[1:]), msg.frm)
     return Response("Variable $%s successfully defined." % msg.args[0],
                     channel=msg.channel)
 
@@ -222,13 +222,13 @@ def cmd_alias(msg):
 
     if alias.cmd in context.data.getNode("aliases").index:
         return Response("%s corresponds to %s" % (alias.cmd, context.data.getNode("aliases").index[alias.cmd]["origin"]),
-                        channel=msg.channel, nick=msg.nick)
+                        channel=msg.channel, nick=msg.frm)
 
     elif len(msg.args) > 1:
         create_alias(alias.cmd,
                      " ".join(msg.args[1:]),
                      channel=msg.channel,
-                     creator=msg.nick)
+                     creator=msg.frm)
         return Response("New alias %s successfully registered." % alias.cmd,
                         channel=msg.channel)
 

modules/birthday.py

@@ -13,7 +13,7 @@ from nemubot.tools.countdown import countdown_format
 from nemubot.tools.date import extractDate
 from nemubot.tools.xmlparser.node import ModuleState
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################
@@ -27,7 +27,7 @@ def load(context):
 def findName(msg):
     if (not len(msg.args) or msg.args[0].lower() == "moi" or
         msg.args[0].lower() == "me"):
-        name = msg.nick.lower()
+        name = msg.frm.lower()
     else:
         name = msg.args[0].lower()
 
@@ -77,7 +77,7 @@ def cmd_anniv(msg):
     else:
         return Response("désolé, je ne connais pas la date d'anniversaire"
                         " de %s. Quand est-il né ?" % name,
-                        msg.channel, msg.nick)
+                        msg.channel, msg.frm)
 
 
 @hook.command("age",
@@ -98,7 +98,7 @@ def cmd_age(msg):
                         msg.channel)
     else:
         return Response("désolé, je ne connais pas l'âge de %s."
-                        " Quand est-il né ?" % name, msg.channel, msg.nick)
+                        " Quand est-il né ?" % name, msg.channel, msg.frm)
     return True
 
 
@@ -106,18 +106,18 @@ def cmd_age(msg):
 
 @hook.ask()
 def parseask(msg):
-    res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)?\s+(birthday|geburtstag|née? |nee? le|born on).*$", msg.text, re.I)
+    res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)?\s+(birthday|geburtstag|née? |nee? le|born on).*$", msg.message, re.I)
     if res is not None:
         try:
-            extDate = extractDate(msg.text)
+            extDate = extractDate(msg.message)
             if extDate is None or extDate.year > datetime.now().year:
                 return Response("la date de naissance ne paraît pas valide...",
                                 msg.channel,
-                                msg.nick)
+                                msg.frm)
             else:
                 nick = res.group(1)
                 if nick == "my" or nick == "I" or nick == "i" or nick == "je" or nick == "mon" or nick == "ma":
-                    nick = msg.nick
+                    nick = msg.frm
                 if nick.lower() in context.data.index:
                     context.data.index[nick.lower()]["born"] = extDate
                 else:
@@ -129,6 +129,6 @@ def parseask(msg):
                 return Response("ok, c'est noté, %s est né le %s"
                                 % (nick, extDate.strftime("%A %d %B %Y à %H:%M")),
                                 msg.channel,
-                                msg.nick)
+                                msg.frm)
         except:
             raise IMException("la date de naissance ne paraît pas valide.")

modules/bonneannee.py

@@ -4,12 +4,10 @@
 
 from datetime import datetime, timezone
 
-from nemubot import context
-from nemubot.event import ModuleEvent
 from nemubot.hooks import hook
 from nemubot.tools.countdown import countdown_format
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # GLOBALS #############################################################
@@ -39,10 +37,8 @@ def load(context):
                 chan = sayon["channel"]
                 context.send_response(srv, Response(txt, chan))
 
-    d = datetime(yrn, 1, 1, 0, 0, 0, 0,
-                 timezone.utc) - datetime.now(timezone.utc)
-    context.add_event(ModuleEvent(interval=0, offset=d.total_seconds(),
-                                  call=bonneannee))
+    context.call_at(datetime(yrn, 1, 1, 0, 0, 0, 0, timezone.utc),
+                    bonneannee)
 
 
 # MODULE INTERFACE ####################################################

modules/books.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################

modules/cat.py

@@ -7,7 +7,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.message import Command, DirectAsk, Text
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MODULE CORE #########################################################

modules/conjugaison.py

@@ -11,7 +11,7 @@ from nemubot.hooks import hook
 from nemubot.tools import web
 from nemubot.tools.web import striphtml
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # GLOBALS #############################################################

modules/ctfs.py

@@ -6,7 +6,7 @@ from bs4 import BeautifulSoup
 
 from nemubot.hooks import hook
 from nemubot.tools.web import getURLContent, striphtml
-from more import Response
+from nemubot.module.more import Response
 
 
 # GLOBALS #############################################################

modules/cve.py

@@ -5,29 +5,68 @@
 from bs4 import BeautifulSoup
 from urllib.parse import quote
 
+from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools.web import getURLContent, striphtml
 
-from more import Response
+from nemubot.module.more import Response
 
-BASEURL_NIST = 'https://web.nvd.nist.gov/view/vuln/detail?vulnId='
+BASEURL_NIST = 'https://nvd.nist.gov/vuln/detail/'
 
 
 # MODULE CORE #########################################################
 
+VULN_DATAS = {
+    "alert-title": "vuln-warning-status-name",
+    "alert-content": "vuln-warning-banner-content",
+
+    "description": "vuln-description",
+    "published": "vuln-published-on",
+    "last_modified": "vuln-last-modified-on",
+    "source": "vuln-source",
+
+    "base_score": "vuln-cvssv3-base-score-link",
+    "severity": "vuln-cvssv3-base-score-severity",
+    "impact_score": "vuln-cvssv3-impact-score",
+    "exploitability_score": "vuln-cvssv3-exploitability-score",
+
+    "av": "vuln-cvssv3-av",
+    "ac": "vuln-cvssv3-ac",
+    "pr": "vuln-cvssv3-pr",
+    "ui": "vuln-cvssv3-ui",
+    "s": "vuln-cvssv3-s",
+    "c": "vuln-cvssv3-c",
+    "i": "vuln-cvssv3-i",
+    "a": "vuln-cvssv3-a",
+}
+
+
 def get_cve(cve_id):
     search_url = BASEURL_NIST + quote(cve_id.upper())
 
     soup = BeautifulSoup(getURLContent(search_url))
-    vuln = soup.body.find(class_="vuln-detail")
-    cvss = vuln.findAll('div')[4]
 
-    return [
-        "Base score: " + cvss.findAll('div')[0].findAll('a')[0].text.strip(),
-        vuln.findAll('p')[0].text, # description
-        striphtml(vuln.findAll('div')[0].text).strip(), # publication date
-        striphtml(vuln.findAll('div')[1].text).strip(), # last revised
-    ]
+    vuln = {}
+
+    for vd in VULN_DATAS:
+        r = soup.body.find(attrs={"data-testid": VULN_DATAS[vd]})
+        if r:
+            vuln[vd] = r.text.strip()
+
+    return vuln
+
+
+def display_metrics(av, ac, pr, ui, s, c, i, a, **kwargs):
+    ret = []
+    if av != "None": ret.append("Attack Vector: \x02%s\x0F" % av)
+    if ac != "None": ret.append("Attack Complexity: \x02%s\x0F" % ac)
+    if pr != "None": ret.append("Privileges Required: \x02%s\x0F" % pr)
+    if ui != "None": ret.append("User Interaction: \x02%s\x0F" % ui)
+    if s != "Unchanged": ret.append("Scope: \x02%s\x0F" % s)
+    if c != "None": ret.append("Confidentiality: \x02%s\x0F" % c)
+    if i != "None": ret.append("Integrity: \x02%s\x0F" % i)
+    if a != "None": ret.append("Availability: \x02%s\x0F" % a)
+    return ', '.join(ret)
 
 
 # MODULE INTERFACE ####################################################
@@ -42,6 +81,20 @@ def get_cve_desc(msg):
         if cve_id[:3].lower() != 'cve':
             cve_id = 'cve-' + cve_id
 
-        res.append_message(get_cve(cve_id))
+        cve = get_cve(cve_id)
+        if not cve:
+            raise IMException("CVE %s doesn't exists." % cve_id)
+
+        if "alert-title" in cve or "alert-content" in cve:
+            alert = "\x02%s:\x0F %s " % (cve["alert-title"] if "alert-title" in cve else "",
+                                         cve["alert-content"] if "alert-content" in cve else "")
+        else:
+            alert = ""
+
+        if "base_score" not in cve and "description" in cve:
+            res.append_message("{alert}From \x02{source}\x0F, last modified on \x02{last_modified}\x0F. {description}".format(alert=alert, **cve), title=cve_id)
+        else:
+            metrics = display_metrics(**cve)
+            res.append_message("{alert}Base score: \x02{base_score} {severity}\x0F (impact: \x02{impact_score}\x0F, exploitability: \x02{exploitability_score}\x0F; {metrics}), from \x02{source}\x0F, last modified on \x02{last_modified}\x0F. {description}".format(alert=alert, metrics=metrics, **cve), title=cve_id)
 
     return res

modules/ddg.py

@@ -8,7 +8,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 # MODULE CORE #########################################################
 

modules/dig.py

@@ -0,0 +1,94 @@
+"""DNS resolver"""
+
+# PYTHON STUFFS #######################################################
+
+import ipaddress
+import socket
+
+import dns.exception
+import dns.name
+import dns.rdataclass
+import dns.rdatatype
+import dns.resolver
+
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+
+from nemubot.module.more import Response
+
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("dig",
+              help="Resolve domain name with a basic syntax similar to dig(1)")
+def dig(msg):
+    lclass = "IN"
+    ltype = "A"
+    ledns = None
+    ltimeout = 6.0
+    ldomain = None
+    lnameservers = []
+    lsearchlist = []
+    loptions = []
+    for a in msg.args:
+        if a in dns.rdatatype._by_text:
+            ltype = a
+        elif a in dns.rdataclass._by_text:
+            lclass = a
+        elif a[0] == "@":
+            try:
+                lnameservers.append(str(ipaddress.ip_address(a[1:])))
+            except ValueError:
+                for r in socket.getaddrinfo(a[1:], 53, proto=socket.IPPROTO_UDP):
+                    lnameservers.append(r[4][0])
+
+        elif a[0:8] == "+domain=":
+            lsearchlist.append(dns.name.from_unicode(a[8:]))
+        elif a[0:6] == "+edns=":
+            ledns = int(a[6:])
+        elif a[0:6] == "+time=":
+            ltimeout = float(a[6:])
+        elif a[0] == "+":
+            loptions.append(a[1:])
+        else:
+            ldomain = a
+
+    if not ldomain:
+        raise IMException("indicate a domain to resolve")
+
+    resolv = dns.resolver.Resolver()
+    if ledns:
+        resolv.edns = ledns
+    resolv.lifetime = ltimeout
+    resolv.timeout = ltimeout
+    resolv.flags = (
+        dns.flags.QR | dns.flags.RA |
+        dns.flags.AA if "aaonly" in loptions or "aaflag" in loptions else 0 |
+        dns.flags.AD if "adflag" in loptions else 0 |
+        dns.flags.CD if "cdflag" in loptions else 0 |
+        dns.flags.RD if "norecurse" not in loptions else 0
+    )
+    if lsearchlist:
+        resolv.search = lsearchlist
+    else:
+        resolv.search = [dns.name.from_text(".")]
+
+    if lnameservers:
+        resolv.nameservers = lnameservers
+
+    try:
+        answers = resolv.query(ldomain, ltype, lclass, tcp="tcp" in loptions)
+    except dns.exception.DNSException as e:
+        raise IMException(str(e))
+
+    res = Response(channel=msg.channel, count=" (%s others entries)")
+    for rdata in answers:
+        res.append_message("%s %s %s %s %s" % (
+            answers.qname.to_text(),
+            answers.ttl if not "nottlid" in loptions else "",
+            dns.rdataclass.to_text(answers.rdclass) if not "nocl" in loptions else "",
+            dns.rdatatype.to_text(answers.rdtype),
+            rdata.to_text())
+        )
+
+    return res

modules/disas.py

@@ -0,0 +1,89 @@
+"""The Ultimate Disassembler Module"""
+
+# PYTHON STUFFS #######################################################
+
+import capstone
+
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+
+from nemubot.module.more import Response
+
+
+# MODULE CORE #########################################################
+
+ARCHITECTURES = {
+    "arm": capstone.CS_ARCH_ARM,
+    "arm64": capstone.CS_ARCH_ARM64,
+    "mips": capstone.CS_ARCH_MIPS,
+    "ppc": capstone.CS_ARCH_PPC,
+    "sparc": capstone.CS_ARCH_SPARC,
+    "sysz": capstone.CS_ARCH_SYSZ,
+    "x86": capstone.CS_ARCH_X86,
+    "xcore": capstone.CS_ARCH_XCORE,
+}
+
+MODES = {
+    "arm": capstone.CS_MODE_ARM,
+    "thumb": capstone.CS_MODE_THUMB,
+    "mips32": capstone.CS_MODE_MIPS32,
+    "mips64": capstone.CS_MODE_MIPS64,
+    "mips32r6": capstone.CS_MODE_MIPS32R6,
+    "16": capstone.CS_MODE_16,
+    "32": capstone.CS_MODE_32,
+    "64": capstone.CS_MODE_64,
+    "le": capstone.CS_MODE_LITTLE_ENDIAN,
+    "be": capstone.CS_MODE_BIG_ENDIAN,
+    "micro": capstone.CS_MODE_MICRO,
+    "mclass": capstone.CS_MODE_MCLASS,
+    "v8": capstone.CS_MODE_V8,
+    "v9": capstone.CS_MODE_V9,
+}
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("disas",
+              help="Display assembly code",
+              help_usage={"CODE": "Display assembly code corresponding to the given CODE"},
+              keywords={
+                  "arch=ARCH": "Specify the architecture of the code to disassemble (default: x86, choose between: %s)" % ', '.join(ARCHITECTURES.keys()),
+                  "modes=MODE[,MODE]": "Specify hardware mode of the code to disassemble (default: 32, between: %s)" % ', '.join(MODES.keys()),
+              })
+def cmd_disas(msg):
+    if not len(msg.args):
+        raise IMException("please give me some code")
+
+    # Determine the architecture
+    if "arch" in msg.kwargs:
+        if msg.kwargs["arch"] not in ARCHITECTURES:
+            raise IMException("unknown architectures '%s'" % msg.kwargs["arch"])
+        architecture = ARCHITECTURES[msg.kwargs["arch"]]
+    else:
+        architecture = capstone.CS_ARCH_X86
+
+    # Determine hardware modes
+    modes = 0
+    if "modes" in msg.kwargs:
+        for mode in msg.kwargs["modes"].split(','):
+            if mode not in MODES:
+                raise IMException("unknown mode '%s'" % mode)
+        modes += MODES[mode]
+    elif architecture == capstone.CS_ARCH_X86 or architecture == capstone.CS_ARCH_PPC:
+        modes = capstone.CS_MODE_32
+    elif architecture == capstone.CS_ARCH_ARM or architecture == capstone.CS_ARCH_ARM64:
+        modes = capstone.CS_MODE_ARM
+    elif architecture == capstone.CS_ARCH_MIPS:
+        modes = capstone.CS_MODE_MIPS32
+
+    # Get the code
+    code = bytearray.fromhex(''.join([a.replace("0x", "") for a in msg.args]))
+
+    # Setup capstone
+    md = capstone.Cs(architecture, modes)
+
+    res = Response(channel=msg.channel, nomore="No more instruction")
+
+    for isn in md.disasm(code, 0x1000):
+        res.append_message("%s %s" %(isn.mnemonic, isn.op_str), title="0x%x" % isn.address)
+
+    return res

modules/events.py

@@ -1,7 +1,9 @@
 """Create countdowns and reminders"""
 
-import re
+import calendar
 from datetime import datetime, timedelta, timezone
+from functools import partial
+import re
 
 from nemubot import context
 from nemubot.exception import IMException
@@ -10,31 +12,84 @@ from nemubot.hooks import hook
 from nemubot.message import Command
 from nemubot.tools.countdown import countdown_format, countdown
 from nemubot.tools.date import extractDate
-from nemubot.tools.xmlparser.node import ModuleState
+from nemubot.tools.xmlparser.basic import DictNode
 
-from more import Response
+from nemubot.module.more import Response
+
+
+class Event:
+
+    def __init__(self, server, channel, creator, start_time, end_time=None):
+        self._server = server
+        self._channel = channel
+        self._creator = creator
+        self._start = datetime.utcfromtimestamp(float(start_time)).replace(tzinfo=timezone.utc) if not isinstance(start_time, datetime) else start_time
+        self._end = datetime.utcfromtimestamp(float(end_time)).replace(tzinfo=timezone.utc) if end_time else None
+        self._evt = None
+
+
+    def __del__(self):
+        if self._evt is not None:
+            context.del_event(self._evt)
+            self._evt = None
+
+
+    def saveElement(self, store, tag="event"):
+        attrs = {
+            "server": str(self._server),
+            "channel": str(self._channel),
+            "creator": str(self._creator),
+            "start_time": str(calendar.timegm(self._start.timetuple())),
+        }
+        if self._end:
+            attrs["end_time"] = str(calendar.timegm(self._end.timetuple()))
+        store.startElement(tag, attrs)
+        store.endElement(tag)
+
+    @property
+    def creator(self):
+        return self._creator
+
+    @property
+    def start(self):
+        return self._start
+
+    @property
+    def end(self):
+        return self._end
+
+    @end.setter
+    def end(self, c):
+        self._end = c
+
+    @end.deleter
+    def end(self):
+        self._end = None
 
 
 def help_full ():
-    return "This module store a lot of events: ny, we, " + (", ".join(context.datas.index.keys())) + "\n!eventslist: gets list of timer\n!start /something/: launch a timer"
+    return "This module store a lot of events: ny, we, " + (", ".join(context.datas.keys()) if hasattr(context, "datas") else "") + "\n!eventslist: gets list of timer\n!start /something/: launch a timer"
 
 
 def load(context):
-    #Define the index
-    context.data.setIndex("name")
+    context.set_knodes({
+        "dict": DictNode,
+        "event": Event,
+    })
 
-    for evt in context.data.index.keys():
-        if context.data.index[evt].hasAttribute("end"):
-            event = ModuleEvent(call=fini, call_data=dict(strend=context.data.index[evt]))
-            event._end = context.data.index[evt].getDate("end")
-            idt = context.add_event(event)
-            if idt is not None:
-                context.data.index[evt]["_id"] = idt
+    if context.data is None:
+        context.data = DictNode()
+
+    # Relaunch all timers
+    for kevt in context.data:
+        if context.data[kevt].end:
+            context.data[kevt]._evt = context.call_at(context.data[kevt].end, partial(fini, kevt, context.data[kevt]))
 
 
-def fini(d, strend):
-    context.send_response(strend["server"], Response("%s arrivé à échéance." % strend["name"], channel=strend["channel"], nick=strend["proprio"]))
-    context.data.delChild(context.data.index[strend["name"]])
+def fini(name, evt):
+    context.send_response(evt._server, Response("%s arrivé à échéance." % name, channel=evt._channel, nick=evt.creator))
+    evt._evt = None
+    del context.data[name]
     context.save()
 
 
@@ -63,18 +118,10 @@ def start_countdown(msg):
     """!start /something/: launch a timer"""
     if len(msg.args) < 1:
         raise IMException("indique le nom d'un événement à chronométrer")
-    if msg.args[0] in context.data.index:
+    if msg.args[0] in context.data:
         raise IMException("%s existe déjà." % msg.args[0])
 
-    strnd = ModuleState("strend")
-    strnd["server"] = msg.server
-    strnd["channel"] = msg.channel
-    strnd["proprio"] = msg.nick
-    strnd["start"] = msg.date
-    strnd["name"] = msg.args[0]
-    context.data.addChild(strnd)
-
-    evt = ModuleEvent(call=fini, call_data=dict(strend=strnd))
+    evt = Event(server=msg.server, channel=msg.channel, creator=msg.frm, start_time=msg.date)
 
     if len(msg.args) > 1:
         result1 = re.findall("([0-9]+)([smhdjwyaSMHDJWYA])?", msg.args[1])
@@ -92,50 +139,48 @@ def start_countdown(msg):
                 if result2 is None or result2.group(4) is None: yea = now.year
                 else: yea = int(result2.group(4))
                 if result2 is not None and result3 is not None:
-                    strnd["end"] = datetime(yea, int(result2.group(3)), int(result2.group(2)), hou, minu, sec, timezone.utc)
+                    evt.end = datetime(yea, int(result2.group(3)), int(result2.group(2)), hou, minu, sec, timezone.utc)
                 elif result2 is not None:
-                    strnd["end"] = datetime(int(result2.group(4)), int(result2.group(3)), int(result2.group(2)), 0, 0, 0, timezone.utc)
+                    evt.end = datetime(int(result2.group(4)), int(result2.group(3)), int(result2.group(2)), 0, 0, 0, timezone.utc)
                 elif result3 is not None:
                   if hou * 3600 + minu * 60 + sec > now.hour * 3600 + now.minute * 60 + now.second:
-                    strnd["end"] = datetime(now.year, now.month, now.day, hou, minu, sec, timezone.utc)
+                    evt.end = datetime(now.year, now.month, now.day, hou, minu, sec, timezone.utc)
                   else:
-                    strnd["end"] = datetime(now.year, now.month, now.day + 1, hou, minu, sec, timezone.utc)
-                evt._end = strnd.getDate("end")
-                strnd["_id"] = context.add_event(evt)
+                    evt.end = datetime(now.year, now.month, now.day + 1, hou, minu, sec, timezone.utc)
             except:
-                context.data.delChild(strnd)
                 raise IMException("Mauvais format de date pour l'événement %s. Il n'a pas été créé." % msg.args[0])
 
         elif result1 is not None and len(result1) > 0:
-            strnd["end"] = msg.date
+            evt.end = msg.date
             for (t, g) in result1:
                 if g is None or g == "" or g == "m" or g == "M":
-                    strnd["end"] += timedelta(minutes=int(t))
+                    evt.end += timedelta(minutes=int(t))
                 elif g == "h" or g == "H":
-                    strnd["end"] += timedelta(hours=int(t))
+                    evt.end += timedelta(hours=int(t))
                 elif g == "d" or g == "D" or g == "j" or g == "J":
-                    strnd["end"] += timedelta(days=int(t))
+                    evt.end += timedelta(days=int(t))
                 elif g == "w" or g == "W":
-                    strnd["end"] += timedelta(days=int(t)*7)
+                    evt.end += timedelta(days=int(t)*7)
                 elif g == "y" or g == "Y" or g == "a" or g == "A":
-                    strnd["end"] += timedelta(days=int(t)*365)
+                    evt.end += timedelta(days=int(t)*365)
                 else:
-                    strnd["end"] += timedelta(seconds=int(t))
-            evt._end = strnd.getDate("end")
-            eid = context.add_event(evt)
-            if eid is not None:
-                strnd["_id"] = eid
+                    evt.end += timedelta(seconds=int(t))
 
+    context.data[msg.args[0]] = evt
     context.save()
-    if "end" in strnd:
+
+    if evt.end is not None:
+        context.add_event(ModuleEvent(partial(fini, msg.args[0], evt),
+                                      offset=evt.end - datetime.now(timezone.utc),
+                                      interval=0))
         return Response("%s commencé le %s et se terminera le %s." %
                         (msg.args[0], msg.date.strftime("%A %d %B %Y à %H:%M:%S"),
-                         strnd.getDate("end").strftime("%A %d %B %Y à %H:%M:%S")),
-                        nick=msg.frm)
+                         evt.end.strftime("%A %d %B %Y à %H:%M:%S")),
+                        channel=msg.channel)
     else:
         return Response("%s commencé le %s"% (msg.args[0],
                             msg.date.strftime("%A %d %B %Y à %H:%M:%S")),
-                        nick=msg.frm)
+                        channel=msg.channel)
 
 
 @hook.command("end")
@@ -144,67 +189,66 @@ def end_countdown(msg):
     if len(msg.args) < 1:
         raise IMException("quel événement terminer ?")
 
-    if msg.args[0] in context.data.index:
-        if context.data.index[msg.args[0]]["proprio"] == msg.nick or (msg.cmd == "forceend" and msg.frm_owner):
-            duration = countdown(msg.date - context.data.index[msg.args[0]].getDate("start"))
-            context.del_event(context.data.index[msg.args[0]]["_id"])
-            context.data.delChild(context.data.index[msg.args[0]])
+    if msg.args[0] in context.data:
+        if context.data[msg.args[0]].creator == msg.frm or (msg.cmd == "forceend" and msg.frm_owner):
+            duration = countdown(msg.date - context.data[msg.args[0]].start)
+            del context.data[msg.args[0]]
             context.save()
             return Response("%s a duré %s." % (msg.args[0], duration),
-                            channel=msg.channel, nick=msg.nick)
+                            channel=msg.channel, nick=msg.frm)
         else:
-            raise IMException("Vous ne pouvez pas terminer le compteur %s, créé par %s." % (msg.args[0], context.data.index[msg.args[0]]["proprio"]))
+            raise IMException("Vous ne pouvez pas terminer le compteur %s, créé par %s." % (msg.args[0], context.data[msg.args[0]].creator))
     else:
-        return Response("%s n'est pas un compteur connu."% (msg.args[0]), channel=msg.channel, nick=msg.nick)
+        return Response("%s n'est pas un compteur connu."% (msg.args[0]), channel=msg.channel, nick=msg.frm)
 
 
 @hook.command("eventslist")
 def liste(msg):
     """!eventslist: gets list of timer"""
     if len(msg.args):
-        res = list()
+        res = Response(channel=msg.channel)
         for user in msg.args:
-            cmptr = [x["name"] for x in context.data.index.values() if x["proprio"] == user]
+            cmptr = [k for k in context.data if context.data[k].creator == user]
             if len(cmptr) > 0:
-                res.append("Compteurs créés par %s : %s" % (user, ", ".join(cmptr)))
+                res.append_message(cmptr, title="Events created by %s" % user)
             else:
-                res.append("%s n'a pas créé de compteur" % user)
-        return Response(" ; ".join(res), channel=msg.channel)
+                res.append_message("%s doesn't have any counting events" % user)
+        return res
     else:
-        return Response("Compteurs connus : %s." % ", ".join(context.data.index.keys()), channel=msg.channel)
+        return Response(list(context.data.keys()), channel=msg.channel, title="Known events")
 
 
-@hook.command(match=lambda msg: isinstance(msg, Command) and msg.cmd in context.data.index)
+@hook.command(match=lambda msg: isinstance(msg, Command) and msg.cmd in context.data)
 def parseanswer(msg):
     res = Response(channel=msg.channel)
 
     # Avoid message starting by ! which can be interpreted as command by other bots
     if msg.cmd[0] == "!":
-        res.nick = msg.nick
+        res.nick = msg.frm
 
-    if context.data.index[msg.cmd].name == "strend":
-        if context.data.index[msg.cmd].hasAttribute("end"):
-            res.append_message("%s commencé il y a %s et se terminera dans %s." % (msg.cmd, countdown(msg.date - context.data.index[msg.cmd].getDate("start")), countdown(context.data.index[msg.cmd].getDate("end") - msg.date)))
+    if msg.cmd in context.data:
+        if context.data[msg.cmd].end:
+            res.append_message("%s commencé il y a %s et se terminera dans %s." % (msg.cmd, countdown(msg.date - context.data[msg.cmd].start), countdown(context.data[msg.cmd].end - msg.date)))
         else:
-            res.append_message("%s commencé il y a %s." % (msg.cmd, countdown(msg.date - context.data.index[msg.cmd].getDate("start"))))
+            res.append_message("%s commencé il y a %s." % (msg.cmd, countdown(msg.date - context.data[msg.cmd].start)))
     else:
-        res.append_message(countdown_format(context.data.index[msg.cmd].getDate("start"), context.data.index[msg.cmd]["msg_before"], context.data.index[msg.cmd]["msg_after"]))
+        res.append_message(countdown_format(context.data[msg.cmd].start, context.data[msg.cmd]["msg_before"], context.data[msg.cmd]["msg_after"]))
     return res
 
 
 RGXP_ask = re.compile(r"^.*((create|new)\s+(a|an|a\s*new|an\s*other)?\s*(events?|commande?)|(nouvel(le)?|ajoute|cr[ée]{1,3})\s+(un)?\s*([eé]v[ée]nements?|commande?)).*$", re.I)
 
-@hook.ask(match=lambda msg: RGXP_ask.match(msg.text))
+@hook.ask(match=lambda msg: RGXP_ask.match(msg.message))
 def parseask(msg):
-    name = re.match("^.*!([^ \"'@!]+).*$", msg.text)
+    name = re.match("^.*!([^ \"'@!]+).*$", msg.message)
     if name is None:
         raise IMException("il faut que tu attribues une commande à l'événement.")
-    if name.group(1) in context.data.index:
+    if name.group(1) in context.data:
         raise IMException("un événement portant ce nom existe déjà.")
 
-    texts = re.match("^[^\"]*(avant|après|apres|before|after)?[^\"]*\"([^\"]+)\"[^\"]*((avant|après|apres|before|after)?.*\"([^\"]+)\".*)?$", msg.text, re.I)
+    texts = re.match("^[^\"]*(avant|après|apres|before|after)?[^\"]*\"([^\"]+)\"[^\"]*((avant|après|apres|before|after)?.*\"([^\"]+)\".*)?$", msg.message, re.I)
     if texts is not None and texts.group(3) is not None:
-        extDate = extractDate(msg.text)
+        extDate = extractDate(msg.message)
         if extDate is None or extDate == "":
             raise IMException("la date de l'événement est invalide !")
 
@@ -223,7 +267,7 @@ def parseask(msg):
         evt = ModuleState("event")
         evt["server"] = msg.server
         evt["channel"] = msg.channel
-        evt["proprio"] = msg.nick
+        evt["proprio"] = msg.frm
         evt["name"] = name.group(1)
         evt["start"] = extDate
         evt["msg_after"] = msg_after
@@ -237,7 +281,7 @@ def parseask(msg):
         evt = ModuleState("event")
         evt["server"] = msg.server
         evt["channel"] = msg.channel
-        evt["proprio"] = msg.nick
+        evt["proprio"] = msg.frm
         evt["name"] = name.group(1)
         evt["msg_before"] = texts.group (2)
         context.data.addChild(evt)

modules/freetarifs.py

@@ -0,0 +1,64 @@
+"""Inform about Free Mobile tarifs"""
+
+# PYTHON STUFFS #######################################################
+
+import urllib.parse
+from bs4 import BeautifulSoup
+
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.tools import web
+
+from nemubot.module.more import Response
+
+
+# MODULE CORE #########################################################
+
+ACT = {
+    "ff_toFixe": "Appel vers les fixes",
+    "ff_toMobile": "Appel vers les mobiles",
+    "ff_smsSendedToCountry": "SMS vers le pays",
+    "ff_mmsSendedToCountry": "MMS vers le pays",
+    "fc_callToFrance": "Appel vers la France",
+    "fc_smsToFrance": "SMS vers la france",
+    "fc_mmsSended": "MMS vers la france",
+    "fc_callToSameCountry": "Réception des appels",
+    "fc_callReceived": "Appel dans le pays",
+    "fc_smsReceived": "SMS (Réception)",
+    "fc_mmsReceived": "MMS (Réception)",
+    "fc_moDataFromCountry": "Data",
+}
+
+def get_land_tarif(country, forfait="pkgFREE"):
+    url = "http://mobile.international.free.fr/?" + urllib.parse.urlencode({'pays': country})
+    page = web.getURLContent(url)
+    soup = BeautifulSoup(page)
+
+    fact = soup.find(class_=forfait)
+
+    if fact is None:
+        raise IMException("Country or forfait not found.")
+
+    res = {}
+    for s in ACT.keys():
+        try:
+            res[s] = fact.find(attrs={"data-bind": "text: " + s}).text + " " + fact.find(attrs={"data-bind": "html: " + s + "Unit"}).text
+        except AttributeError:
+            res[s] = "inclus"
+
+    return res
+
+@hook.command("freetarifs",
+              help="Show Free Mobile tarifs for given contries",
+              help_usage={"COUNTRY": "Show Free Mobile tarifs for given CONTRY"},
+              keywords={
+                  "forfait=FORFAIT": "Related forfait between Free (default) and 2euro"
+              })
+def get_freetarif(msg):
+    res = Response(channel=msg.channel)
+
+    for country in msg.args:
+        t = get_land_tarif(country.lower().capitalize(), "pkg" + (msg.kwargs["forfait"] if "forfait" in msg.kwargs else "FREE").upper())
+        res.append_message(["\x02%s\x0F : %s" % (ACT[k], t[k]) for k in sorted(ACT.keys(), reverse=True)], title=country)
+
+    return res

modules/github.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MODULE CORE #########################################################

modules/grep.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.message import Command, Text
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MODULE CORE #########################################################

modules/imdb.py

@@ -5,63 +5,57 @@
 import re
 import urllib.parse
 
+from bs4 import BeautifulSoup
+
 from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MODULE CORE #########################################################
 
-def get_movie(title=None, year=None, imdbid=None, fullplot=True, tomatoes=False):
+def get_movie_by_id(imdbid):
     """Returns the information about the matching movie"""
 
-    # Built URL
-    url = "http://www.omdbapi.com/?"
-    if title is not None:
-        url += "t=%s&" % urllib.parse.quote(title)
-    if year is not None:
-        url += "y=%s&" % urllib.parse.quote(year)
-    if imdbid is not None:
-        url += "i=%s&" % urllib.parse.quote(imdbid)
-    if fullplot:
-        url += "plot=full&"
-    if tomatoes:
-        url += "tomatoes=true&"
+    url = "http://www.imdb.com/title/" + urllib.parse.quote(imdbid)
+    soup = BeautifulSoup(web.getURLContent(url))
 
-    # Make the request
-    data = web.getJSON(url)
+    return {
+        "imdbID": imdbid,
+        "Title": soup.body.find(attrs={"itemprop": "name"}).next_element.strip(),
+        "Year": soup.body.find(id="titleYear").find("a").text.strip() if soup.body.find(id="titleYear") else ", ".join([y.text.strip() for y in soup.body.find(attrs={"class": "seasons-and-year-nav"}).find_all("div")[3].find_all("a")[:-1]]),
+        "Duration": soup.body.find_all(attrs={"itemprop": "duration"})[-1].text.strip(),
+        "imdbRating": soup.body.find(attrs={"itemprop": "ratingValue"}).text.strip(),
+        "imdbVotes": soup.body.find(attrs={"itemprop": "ratingCount"}).text.strip(),
+        "Plot": re.sub(r"\s+", " ", soup.body.find(id="titleStoryLine").find(attrs={"itemprop": "description"}).text).strip(),
 
-    # Return data
-    if "Error" in data:
-        raise IMException(data["Error"])
-
-    elif "Response" in data and data["Response"] == "True":
-        return data
-
-    else:
-        raise IMException("An error occurs during movie search")
+        "Type": "TV Series" if soup.find(attrs={"class": "np_episode_guide"}) else "Movie",
+        "Country": ", ".join([c.find("a").text.strip() for c in soup.body.find(id="titleDetails").find_all(attrs={"class": "txt-block"}) if c.text.find("Country") != -1]),
+        "Released": soup.body.find(attrs={"itemprop": "datePublished"}).attrs["content"] if "content" in soup.body.find(attrs={"itemprop": "datePublished"}).attrs else "N\A",
+        "Genre": ", ".join([g.text.strip() for g in soup.body.find_all(attrs={"itemprop": "genre"})[:-1]]),
+        "Director": ", ".join([d.find(attrs={"itemprop": "name"}).text.strip() for d in soup.body.find_all(attrs={"itemprop": "director"})]),
+        "Writer": ", ".join([d.find(attrs={"itemprop": "name"}).text.strip() for d in soup.body.find_all(attrs={"itemprop": "creator"})]),
+        "Actors": ", ".join([d.find(attrs={"itemprop": "name"}).text.strip() for d in soup.body.find_all(attrs={"itemprop": "actors"})]),
+    }
 
 
-def find_movies(title):
+def find_movies(title, year=None):
     """Find existing movies matching a approximate title"""
 
+    title = title.lower()
+
     # Built URL
-    url = "http://www.omdbapi.com/?s=%s" % urllib.parse.quote(title)
+    url = "https://v2.sg.media-imdb.com/suggests/%s/%s.json" % (urllib.parse.quote(title[0]), urllib.parse.quote(title.replace(" ", "_")))
 
     # Make the request
-    data = web.getJSON(url)
-
-    # Return data
-    if "Error" in data:
-        raise IMException(data["Error"])
-
-    elif "Search" in data:
-        return data
+    data = web.getJSON(url, remove_callback=True)
 
+    if year is None:
+        return data["d"]
     else:
-        raise IMException("An error occurs during movie search")
+        return [d for d in data["d"] if "y" in d and str(d["y"]) == year]
 
 
 # MODULE INTERFACE ####################################################
@@ -79,23 +73,28 @@ def cmd_imdb(msg):
     title = ' '.join(msg.args)
 
     if re.match("^tt[0-9]{7}$", title) is not None:
-        data = get_movie(imdbid=title)
+        data = get_movie_by_id(imdbid=title)
     else:
         rm = re.match(r"^(.+)\s\(([0-9]{4})\)$", title)
         if rm is not None:
-            data = get_movie(title=rm.group(1), year=rm.group(2))
+            data = find_movies(rm.group(1), year=rm.group(2))
         else:
-            data = get_movie(title=title)
+            data = find_movies(title)
+
+        if not data:
+            raise IMException("Movie/series not found")
+
+        data = get_movie_by_id(data[0]["id"])
 
     res = Response(channel=msg.channel,
                    title="%s (%s)" % (data['Title'], data['Year']),
                    nomore="No more information, more at http://www.imdb.com/title/%s" % data['imdbID'])
 
-    res.append_message("\x02rating\x0F: %s (%s votes); \x02plot\x0F: %s" %
-                       (data['imdbRating'], data['imdbVotes'], data['Plot']))
+    res.append_message("%s \x02genre:\x0F %s; \x02rating\x0F: %s (%s votes); \x02plot\x0F: %s" %
+                       (data['Type'], data['Genre'], data['imdbRating'], data['imdbVotes'], data['Plot']))
 
-    res.append_message("%s \x02from\x0F %s \x02released on\x0F %s; \x02genre:\x0F %s; \x02directed by:\x0F %s; \x02written by:\x0F %s; \x02main actors:\x0F %s"
-                       % (data['Type'], data['Country'], data['Released'], data['Genre'], data['Director'], data['Writer'], data['Actors']))
+    res.append_message("%s \x02from\x0F %s \x02released on\x0F %s; \x02directed by:\x0F %s; \x02written by:\x0F %s; \x02main actors:\x0F %s"
+                       % (data['Type'], data['Country'], data['Released'], data['Director'], data['Writer'], data['Actors']))
     return res
 
 
@@ -111,7 +110,7 @@ def cmd_search(msg):
     data = find_movies(' '.join(msg.args))
 
     movies = list()
-    for m in data['Search']:
-        movies.append("\x02%s\x0F (%s of %s)" % (m['Title'], m['Type'], m['Year']))
+    for m in data:
+        movies.append("\x02%s\x0F%s with %s" % (m['l'], (" (" + str(m['y']) + ")") if "y" in m else "", m['s']))
 
     return Response(movies, title="Titles found", channel=msg.channel)

modules/jsonbot.py

@@ -1,7 +1,7 @@
 from nemubot.hooks import hook
 from nemubot.exception import IMException
 from nemubot.tools import web
-from more import Response
+from nemubot.module.more import Response
 import json
 
 nemubotversion = 3.4

modules/man.py

@@ -8,7 +8,7 @@ import os
 
 from nemubot.hooks import hook
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # GLOBALS #############################################################

modules/mapquest.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 # GLOBALS #############################################################
 
@@ -55,7 +55,7 @@ def cmd_geocode(msg):
     if not len(msg.args):
         raise IMException("indicate a name")
 
-    res = Response(channel=msg.channel, nick=msg.nick,
+    res = Response(channel=msg.channel, nick=msg.frm,
                    nomore="No more geocode", count=" (%s more geocode)")
 
     for loc in geocode(' '.join(msg.args)):

modules/mediawiki.py

@@ -11,7 +11,7 @@ from nemubot.tools import web
 
 nemubotversion = 3.4
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MEDIAWIKI REQUESTS ##################################################

modules/networking/__init__.py

@@ -8,7 +8,7 @@ import re
 from nemubot.exception import IMException
 from nemubot.hooks import hook
 
-from more import Response
+from nemubot.module.more import Response
 
 from . import isup
 from . import page

modules/networking/watchWebsite.py

@@ -12,7 +12,7 @@ from nemubot.tools.xmlparser.node import ModuleState
 
 logger = logging.getLogger("nemubot.module.networking.watchWebsite")
 
-from more import Response
+from nemubot.module.more import Response
 
 from . import page
 

modules/networking/whois.py

@@ -6,7 +6,7 @@ import urllib
 from nemubot.exception import IMException
 from nemubot.tools.web import getJSON
 
-from more import Response
+from nemubot.module.more import Response
 
 URL_AVAIL = "https://www.whoisxmlapi.com/whoisserver/WhoisService?cmd=GET_DN_AVAILABILITY&domainName=%%s&outputFormat=json&username=%s&password=%s"
 URL_WHOIS = "http://www.whoisxmlapi.com/whoisserver/WhoisService?da=2&domainName=%%s&outputFormat=json&userName=%s&password=%s"

modules/news.py

@@ -12,7 +12,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 from nemubot.tools.feed import Feed, AtomEntry
 
 

modules/nextstop.xml

@@ -1,4 +0,0 @@
-<?xml version="1.0" ?>
-<nemubotmodule name="nextstop">
-  <message type="cmd" name="ratp" call="ask_ratp" />
-</nemubotmodule>

modules/openroute.py

@@ -0,0 +1,158 @@
+"""Lost? use our commands to find your way!"""
+
+# PYTHON STUFFS #######################################################
+
+import re
+import urllib.parse
+
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.tools import web
+
+from nemubot.module.more import Response
+
+# GLOBALS #############################################################
+
+URL_DIRECTIONS_API = "https://api.openrouteservice.org/directions?api_key=%s&"
+URL_GEOCODE_API = "https://api.openrouteservice.org/geocoding?api_key=%s&"
+
+waytype = [
+    "unknown",
+    "state road",
+    "road",
+    "street",
+    "path",
+    "track",
+    "cycleway",
+    "footway",
+    "steps",
+    "ferry",
+    "construction",
+]
+
+
+# LOADING #############################################################
+
+def load(context):
+    if not context.config or "apikey" not in context.config:
+        raise ImportError("You need an OpenRouteService API key in order to use this "
+                          "module. Add it to the module configuration file:\n"
+                          "<module name=\"ors\" apikey=\"XXXXXXXXXXXXXXXX\" "
+                          "/>\nRegister at https://developers.openrouteservice.org")
+    global URL_DIRECTIONS_API
+    URL_DIRECTIONS_API = URL_DIRECTIONS_API % context.config["apikey"]
+    global URL_GEOCODE_API
+    URL_GEOCODE_API = URL_GEOCODE_API % context.config["apikey"]
+
+
+# MODULE CORE #########################################################
+
+def approx_distance(lng):
+    if lng > 1111:
+        return "%f km" % (lng / 1000)
+    else:
+        return "%f m" % lng
+
+
+def approx_duration(sec):
+    days = int(sec / 86400)
+    if days > 0:
+        return "%d days %f hours" % (days, (sec % 86400) / 3600)
+    hours = int((sec % 86400) / 3600)
+    if hours > 0:
+        return "%d hours %f minutes" % (hours, (sec % 3600) / 60)
+    minutes = (sec % 3600) / 60
+    if minutes > 0:
+        return "%d minutes" % minutes
+    else:
+        return "%d seconds" % sec
+
+
+def geocode(query, limit=7):
+    obj = web.getJSON(URL_GEOCODE_API + urllib.parse.urlencode({
+        'query': query,
+        'limit': limit,
+    }))
+
+    for f in obj["features"]:
+        yield f["geometry"]["coordinates"], f["properties"]
+
+
+def firstgeocode(query):
+    for g in geocode(query, limit=1):
+        return g
+
+
+def where(loc):
+    return "{name} {city} {state} {county} {country}".format(**loc)
+
+
+def directions(coordinates, **kwargs):
+    kwargs['coordinates'] = '|'.join(coordinates)
+
+    print(URL_DIRECTIONS_API + urllib.parse.urlencode(kwargs))
+    return web.getJSON(URL_DIRECTIONS_API + urllib.parse.urlencode(kwargs), decode_error=True)
+
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("geocode",
+              help="Get GPS coordinates of a place",
+              help_usage={
+                  "PLACE": "Get GPS coordinates of PLACE"
+              })
+def cmd_geocode(msg):
+    res = Response(channel=msg.channel, nick=msg.frm,
+                   nomore="No more geocode", count=" (%s more geocode)")
+
+    for loc in geocode(' '.join(msg.args)):
+        res.append_message("%s is at %s,%s" % (
+            where(loc[1]),
+            loc[0][1], loc[0][0],
+        ))
+
+    return res
+
+
+@hook.command("directions",
+              help="Get routing instructions",
+              help_usage={
+                  "POINT1 POINT2 ...": "Get routing instructions to go from POINT1 to the last POINTX via intermediates POINTX"
+              },
+              keywords={
+                  "profile=PROF": "One of driving-car, driving-hgv, cycling-regular, cycling-road, cycling-safe, cycling-mountain, cycling-tour, cycling-electric, foot-walking, foot-hiking, wheelchair. Default: foot-walking",
+                  "preference=PREF": "One of fastest, shortest, recommended. Default: recommended",
+                  "lang=LANG": "default: en",
+              })
+def cmd_directions(msg):
+    drcts = directions(["{0},{1}".format(*firstgeocode(g)[0]) for g in msg.args],
+                       profile=msg.kwargs["profile"] if "profile" in msg.kwargs else "foot-walking",
+                       preference=msg.kwargs["preference"] if "preference" in msg.kwargs else "recommended",
+                       units="m",
+                       language=msg.kwargs["lang"] if "lang" in msg.kwargs else "en",
+                       geometry=False,
+                       instructions=True,
+                       instruction_format="text")
+    if "error" in drcts and "message" in drcts["error"] and drcts["error"]["message"]:
+        raise IMException(drcts["error"]["message"])
+
+    if "routes" not in drcts or not drcts["routes"]:
+        raise IMException("No route available for this trip")
+
+    myway = drcts["routes"][0]
+    myway["summary"]["strduration"] = approx_duration(myway["summary"]["duration"])
+    myway["summary"]["strdistance"] = approx_distance(myway["summary"]["distance"])
+    res = Response("Trip summary: {strdistance} in approximate {strduration}; elevation +{ascent} m -{descent} m".format(**myway["summary"]), channel=msg.channel, count=" (%d more steps)", nomore="You have arrived!")
+
+    def formatSegments(segments):
+        for segment in segments:
+            for step in segment["steps"]:
+                step["strtype"] = waytype[step["type"]]
+                step["strduration"] = approx_duration(step["duration"])
+                step["strdistance"] = approx_distance(step["distance"])
+                yield "{instruction} for {strdistance} on {strtype} (approximate time: {strduration})".format(**step)
+
+    if "segments" in myway:
+        res.append_message([m for m in formatSegments(myway["segments"])])
+
+    return res

modules/pkgs.py

@@ -0,0 +1,68 @@
+"""Get information about common software"""
+
+# PYTHON STUFFS #######################################################
+
+import portage
+
+from nemubot import context
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+
+from nemubot.module.more import Response
+
+DB = None
+
+# MODULE CORE #########################################################
+
+def get_db():
+    global DB
+    if DB is None:
+        DB = portage.db[portage.root]["porttree"].dbapi
+    return DB
+
+
+def package_info(pkgname):
+    pv = get_db().xmatch("match-all", pkgname)
+    if not pv:
+        raise IMException("No package named '%s' found" % pkgname)
+
+    bv = get_db().xmatch("bestmatch-visible", pkgname)
+    pvsplit = portage.catpkgsplit(bv if bv else pv[-1])
+    info = get_db().aux_get(bv if bv else pv[-1], ["DESCRIPTION", "HOMEPAGE", "LICENSE", "IUSE", "KEYWORDS"])
+
+    return {
+        "pkgname": '/'.join(pvsplit[:2]),
+        "category": pvsplit[0],
+        "shortname": pvsplit[1],
+        "lastvers": '-'.join(pvsplit[2:]) if pvsplit[3] != "r0" else pvsplit[2],
+        "othersvers": ['-'.join(portage.catpkgsplit(p)[2:]) for p in pv if p != bv],
+        "description": info[0],
+        "homepage": info[1],
+        "license": info[2],
+        "uses": info[3],
+        "keywords": info[4],
+    }
+
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("eix",
+      help="Get information about a package",
+      help_usage={
+          "NAME": "Get information about a software NAME"
+      })
+def cmd_eix(msg):
+    if not len(msg.args):
+        raise IMException("please give me a package to search")
+
+    def srch(term):
+        try:
+            yield package_info(term)
+        except portage.exception.AmbiguousPackageName as e:
+            for i in e.args[0]:
+                yield package_info(i)
+
+    res = Response(channel=msg.channel, count=" (%d more packages)", nomore="No more package '%s'" % msg.args[0])
+    for pi in srch(msg.args[0]):
+        res.append_message("\x03\x02{pkgname}:\x03\x02 {description} - {homepage} - {license} - last revisions: \x03\x02{lastvers}\x03\x02{ov}".format(ov=(", " + ', '.join(pi["othersvers"])) if pi["othersvers"] else "", **pi))
+    return res

modules/ratp.py

@@ -4,7 +4,7 @@
 
 from nemubot.exception import IMException
 from nemubot.hooks import hook
-from more import Response
+from nemubot.module.more import Response
 
 from nextstop import ratp
 

modules/reddit.py

@@ -10,7 +10,7 @@ from nemubot.tools import web
 
 nemubotversion = 3.4
 
-from more import Response
+from nemubot.module.more import Response
 
 
 def help_full():
@@ -64,15 +64,22 @@ def cmd_subreddit(msg):
                                         channel=msg.channel))
         else:
             all_res.append(Response("%s is not a valid subreddit" % osub,
-                                    channel=msg.channel, nick=msg.nick))
+                                    channel=msg.channel, nick=msg.frm))
 
     return all_res
 
 
 @hook.message()
 def parselisten(msg):
-    parseresponse(msg)
-    return None
+    global LAST_SUBS
+
+    if hasattr(msg, "message") and msg.message and type(msg.message) == str:
+        urls = re.findall("www.reddit.com(/\w/\w+/?)", msg.message)
+        for url in urls:
+            for recv in msg.to:
+                if recv not in LAST_SUBS:
+                    LAST_SUBS[recv] = list()
+                LAST_SUBS[recv].append(url)
 
 
 @hook.post()

modules/rnd.py

@@ -9,7 +9,7 @@ from nemubot import context
 from nemubot.exception import IMException
 from nemubot.hooks import hook
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MODULE INTERFACE ####################################################
@@ -21,7 +21,7 @@ def cmd_choice(msg):
 
     return Response(random.choice(msg.args),
                     channel=msg.channel,
-                    nick=msg.nick)
+                    nick=msg.frm)
 
 
 @hook.command("choicecmd")

modules/sap.py

@@ -12,7 +12,7 @@ from nemubot.tools import web
 
 nemubotversion = 4.0
 
-from more import Response
+from nemubot.module.more import Response
 
 
 def help_full():

modules/shodan.py

@@ -0,0 +1,104 @@
+"""Search engine for IoT"""
+
+# PYTHON STUFFS #######################################################
+
+from datetime import datetime
+import ipaddress
+import urllib.parse
+
+from nemubot import context
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.tools import web
+
+from nemubot.module.more import Response
+
+
+# GLOBALS #############################################################
+
+BASEURL = "https://api.shodan.io/shodan/"
+
+
+# LOADING #############################################################
+
+def load(context):
+    if not context.config or "apikey" not in context.config:
+        raise ImportError("You need a Shodan API key in order to use this "
+                          "module. Add it to the module configuration file:\n"
+                          "<module name=\"shodan\" apikey=\"XXXXXXXXXXXXXXXX\" "
+                          "/>\nRegister at https://account.shodan.io/register")
+
+
+# MODULE CORE #########################################################
+
+def host_lookup(ip):
+    url = BASEURL + "host/" + urllib.parse.quote(ip) + "?" + urllib.parse.urlencode({'key': context.config["apikey"]})
+    return web.getJSON(url)
+
+
+def search_hosts(query):
+    url = BASEURL + "host/search?" + urllib.parse.urlencode({'query': query, 'key': context.config["apikey"]})
+    return web.getJSON(url, max_size=4194304)
+
+
+def print_ssl(ssl):
+    return (
+        "SSL: " +
+        " ".join([v for v in ssl["versions"] if v[0] != "-"]) +
+        "; cipher used: " + ssl["cipher"]["name"] +
+        ("; certificate: " + ssl["cert"]["sig_alg"] +
+        " issued by: " + ssl["cert"]["issuer"]["CN"] +
+        " expires on: " + str(datetime.strptime(ssl["cert"]["expires"], "%Y%m%d%H%M%SZ")) if "cert" in ssl else "")
+    )
+
+def print_service(svc):
+    ip = ipaddress.ip_address(svc["ip_str"])
+    return ((svc["ip_str"] if ip.version == 4 else "[%s]" % svc["ip_str"]) +
+            ":{port}/{transport} ({module}):" +
+            (" {os}" if svc["os"] else "") +
+            (" {product}" if "product" in svc else "") +
+            (" {version}" if "version" in svc else "") +
+            (" {info}" if "info" in svc else "") +
+            (" Vulns: " + ", ".join(svc["opts"]["vulns"]) if "opts" in svc and "vulns" in svc["opts"] else "") +
+            (" " + print_ssl(svc["ssl"]) if "ssl" in svc else "") +
+            (" \x03\x1D" + svc["data"].replace("\r\n", "\n").split("\n")[0] + "\x03\x1D" if "data" in svc else "") +
+            (" " + svc["title"] if "title" in svc else "")
+           ).format(module=svc["_shodan"]["module"], **svc)
+
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("shodan",
+              help="Use shodan.io to get information on machines connected to Internet",
+              help_usage={
+                  "IP": "retrieve information about the given IP (can be v4 or v6)",
+                  "TERM": "retrieve all hosts matching TERM somewhere in their exposed stuff"
+              })
+def shodan(msg):
+    if not msg.args:
+        raise IMException("indicate an IP or a term to search!")
+
+    terms = " ".join(msg.args)
+
+    try:
+        ip = ipaddress.ip_address(terms)
+    except ValueError:
+        ip = None
+
+    if ip:
+        h = host_lookup(terms)
+        res = Response(channel=msg.channel,
+                       title="%s" % ((h["ip_str"] if ip.version == 4 else "[%s]" % h["ip_str"]) + (" (" + ", ".join(h["hostnames"]) + ")") if h["hostnames"] else ""))
+        res.append_message("{isp} ({asn}) -> {city} ({country_code}), running {os}. Vulns: {vulns_str}. Open ports: {open_ports}. Last update: {last_update}".format(
+            open_ports=", ".join(map(lambda a: str(a), h["ports"])), vulns_str=", ".join(h["vulns"]) if "vulns" in h else None, **h).strip())
+        for d in h["data"]:
+            res.append_message(print_service(d))
+
+    else:
+        q = search_hosts(terms)
+        res = Response(channel=msg.channel,
+                       count=" (%%s/%s results)" % q["total"])
+        for r in q["matches"]:
+            res.append_message(print_service(r))
+
+    return res

modules/sleepytime.py

@@ -10,7 +10,7 @@ from nemubot.hooks import hook
 
 nemubotversion = 3.4
 
-from more import Response
+from nemubot.module.more import Response
 
 
 def help_full():

modules/sms.py

@@ -16,7 +16,7 @@ from nemubot.tools.xmlparser.node import ModuleState
 
 nemubotversion = 3.4
 
-from more import Response
+from nemubot.module.more import Response
 
 def load(context):
     context.data.setIndex("name", "phone")
@@ -73,20 +73,20 @@ def cmd_sms(msg):
             fails.append( "%s: %s" % (u, test) )
 
     if len(fails) > 0:
-        return Response("quelque chose ne s'est pas bien passé durant l'envoi du SMS : " + ", ".join(fails), msg.channel, msg.nick)
+        return Response("quelque chose ne s'est pas bien passé durant l'envoi du SMS : " + ", ".join(fails), msg.channel, msg.frm)
     else:
-        return Response("le SMS a bien été envoyé", msg.channel, msg.nick)
+        return Response("le SMS a bien été envoyé", msg.channel, msg.frm)
 
 apiuser_ask = re.compile(r"(utilisateur|user|numéro|numero|compte|abonne|abone|abonné|account)\s+(est|is)\s+(?P<user>[0-9]{7,})", re.IGNORECASE)
 apikey_ask = re.compile(r"(clef|key|password|mot de passe?)\s+(?:est|is)?\s+(?P<key>[a-zA-Z0-9]{10,})", re.IGNORECASE)
 
 @hook.ask()
 def parseask(msg):
-    if msg.text.find("Free") >= 0 and (
-            msg.text.find("API") >= 0 or msg.text.find("api") >= 0) and (
-                msg.text.find("SMS") >= 0 or msg.text.find("sms") >= 0):
-        resuser = apiuser_ask.search(msg.text)
-        reskey = apikey_ask.search(msg.text)
+    if msg.message.find("Free") >= 0 and (
+            msg.message.find("API") >= 0 or msg.message.find("api") >= 0) and (
+                msg.message.find("SMS") >= 0 or msg.message.find("sms") >= 0):
+        resuser = apiuser_ask.search(msg.message)
+        reskey = apikey_ask.search(msg.message)
         if resuser is not None and reskey is not None:
             apiuser = resuser.group("user")
             apikey = reskey.group("key")
@@ -94,18 +94,18 @@ def parseask(msg):
             test = send_sms("nemubot", apiuser, apikey,
                             "Vous avez enregistré vos codes d'authentification dans nemubot, félicitation !")
             if test is not None:
-                return Response("je n'ai pas pu enregistrer tes identifiants : %s" % test, msg.channel, msg.nick)
+                return Response("je n'ai pas pu enregistrer tes identifiants : %s" % test, msg.channel, msg.frm)
 
-            if msg.nick in context.data.index:
-                context.data.index[msg.nick]["user"] = apiuser
-                context.data.index[msg.nick]["key"] = apikey
+            if msg.frm in context.data.index:
+                context.data.index[msg.frm]["user"] = apiuser
+                context.data.index[msg.frm]["key"] = apikey
             else:
                 ms = ModuleState("phone")
-                ms.setAttribute("name", msg.nick)
+                ms.setAttribute("name", msg.frm)
                 ms.setAttribute("user", apiuser)
                 ms.setAttribute("key", apikey)
                 ms.setAttribute("lastuse", 0)
                 context.data.addChild(ms)
             context.save()
             return Response("ok, c'est noté. Je t'ai envoyé un SMS pour tester ;)",
-                            msg.channel, msg.nick)
+                            msg.channel, msg.frm)

modules/spell/__init__.py

@@ -10,7 +10,7 @@ from nemubot.tools.xmlparser.node import ModuleState
 from .pyaspell import Aspell
 from .pyaspell import AspellError
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################
@@ -64,15 +64,15 @@ def cmd_spell(msg):
             raise IMException("Je n'ai pas le dictionnaire `%s' :(" % lang)
 
         if r == True:
-            add_score(msg.nick, "correct")
+            add_score(msg.frm, "correct")
             res.append_message("l'orthographe de `%s' est correcte" % word)
 
         elif len(r) > 0:
-            add_score(msg.nick, "bad")
+            add_score(msg.frm, "bad")
             res.append_message(r, title="suggestions pour `%s'" % word)
 
         else:
-            add_score(msg.nick, "bad")
+            add_score(msg.frm, "bad")
             res.append_message("aucune suggestion pour `%s'" % word)
 
     return res

modules/suivi.py

@@ -2,23 +2,22 @@
 
 # PYTHON STUFF ############################################
 
-import urllib.request
+import json
 import urllib.parse
 from bs4 import BeautifulSoup
 import re
 
 from nemubot.hooks import hook
 from nemubot.exception import IMException
-from nemubot.tools.web import getURLContent
-from more import Response
+from nemubot.tools.web import getURLContent, getJSON
+from nemubot.module.more import Response
 
 
 # POSTAGE SERVICE PARSERS ############################################
 
 def get_tnt_info(track_id):
     values = []
-    data = getURLContent('www.tnt.fr/public/suivi_colis/recherche/'
-                         'visubontransport.do?bonTransport=%s' % track_id)
+    data = getURLContent('www.tnt.fr/public/suivi_colis/recherche/visubontransport.do?bonTransport=%s' % track_id)
     soup = BeautifulSoup(data)
     status_list = soup.find('div', class_='result__content')
     if not status_list:
@@ -32,8 +31,7 @@ def get_tnt_info(track_id):
 
 
 def get_colissimo_info(colissimo_id):
-    colissimo_data = getURLContent("http://www.colissimo.fr/portail_colissimo/"
-                                   "suivre.do?colispart=%s" % colissimo_id)
+    colissimo_data = getURLContent("http://www.colissimo.fr/portail_colissimo/suivre.do?colispart=%s" % colissimo_id)
     soup = BeautifulSoup(colissimo_data)
 
     dataArray = soup.find(class_='dataArray')
@@ -47,9 +45,8 @@ def get_colissimo_info(colissimo_id):
 
 def get_chronopost_info(track_id):
     data = urllib.parse.urlencode({'listeNumeros': track_id})
-    track_baseurl = "http://www.chronopost.fr/expedier/" \
-        "inputLTNumbersNoJahia.do?lang=fr_FR"
-    track_data = urllib.request.urlopen(track_baseurl, data.encode('utf-8'))
+    track_baseurl = "http://www.chronopost.fr/expedier/inputLTNumbersNoJahia.do?lang=fr_FR"
+    track_data = getURLContent(track_baseurl, data.encode('utf-8'))
     soup = BeautifulSoup(track_data)
 
     infoClass = soup.find(class_='numeroColi2')
@@ -65,9 +62,8 @@ def get_chronopost_info(track_id):
 
 def get_colisprive_info(track_id):
     data = urllib.parse.urlencode({'numColis': track_id})
-    track_baseurl = "https://www.colisprive.com/moncolis/pages/" \
-        "detailColis.aspx"
-    track_data = urllib.request.urlopen(track_baseurl, data.encode('utf-8'))
+    track_baseurl = "https://www.colisprive.com/moncolis/pages/detailColis.aspx"
+    track_data = getURLContent(track_baseurl, data.encode('utf-8'))
     soup = BeautifulSoup(track_data)
 
     dataArray = soup.find(class_='BandeauInfoColis')
@@ -82,8 +78,7 @@ def get_laposte_info(laposte_id):
     data = urllib.parse.urlencode({'id': laposte_id})
     laposte_baseurl = "http://www.part.csuivi.courrier.laposte.fr/suivi/index"
 
-    laposte_data = urllib.request.urlopen(laposte_baseurl,
-                                          data.encode('utf-8'))
+    laposte_data = getURLContent(laposte_baseurl, data.encode('utf-8'))
     soup = BeautifulSoup(laposte_data)
     search_res = soup.find(class_='resultat_rech_simple_table').tbody.tr
     if (soup.find(class_='resultat_rech_simple_table').thead
@@ -112,8 +107,7 @@ def get_postnl_info(postnl_id):
     data = urllib.parse.urlencode({'barcodes': postnl_id})
     postnl_baseurl = "http://www.postnl.post/details/"
 
-    postnl_data = urllib.request.urlopen(postnl_baseurl,
-                                         data.encode('utf-8'))
+    postnl_data = getURLContent(postnl_baseurl, data.encode('utf-8'))
     soup = BeautifulSoup(postnl_data)
     if (soup.find(id='datatables')
             and soup.find(id='datatables').tbody
@@ -132,6 +126,70 @@ def get_postnl_info(postnl_id):
             return (post_status.lower(), post_destination, post_date)
 
 
+def get_usps_info(usps_id):
+    usps_parcelurl = "https://tools.usps.com/go/TrackConfirmAction_input?" + urllib.parse.urlencode({'qtc_tLabels1': usps_id})
+
+    usps_data = getURLContent(usps_parcelurl)
+    soup = BeautifulSoup(usps_data)
+    if (soup.find(class_="tracking_history")
+            and soup.find(class_="tracking_history").find(class_="row_notification")
+            and soup.find(class_="tracking_history").find(class_="row_top").find_all("td")):
+        notification = soup.find(class_="tracking_history").find(class_="row_notification").text.strip()
+        date = re.sub(r"\s+", " ", soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[0].text.strip())
+        status = soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[1].text.strip()
+        last_location = soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[2].text.strip()
+
+        print(notification)
+
+        return (notification, date, status, last_location)
+
+
+def get_fedex_info(fedex_id, lang="en_US"):
+    data = urllib.parse.urlencode({
+        'data': json.dumps({
+            "TrackPackagesRequest": {
+                "appType": "WTRK",
+                "appDeviceType": "DESKTOP",
+                "uniqueKey": "",
+                "processingParameters": {},
+                "trackingInfoList": [
+                    {
+                        "trackNumberInfo": {
+                            "trackingNumber": str(fedex_id),
+                            "trackingQualifier": "",
+                            "trackingCarrier": ""
+                        }
+                    }
+                ]
+            }
+        }),
+        'action': "trackpackages",
+        'locale': lang,
+        'version': 1,
+        'format': "json"
+    })
+    fedex_baseurl = "https://www.fedex.com/trackingCal/track"
+
+    fedex_data = getJSON(fedex_baseurl, data.encode('utf-8'))
+
+    if ("TrackPackagesResponse" in fedex_data and
+        "packageList" in fedex_data["TrackPackagesResponse"] and
+        len(fedex_data["TrackPackagesResponse"]["packageList"]) and
+        not fedex_data["TrackPackagesResponse"]["errorList"][0]["code"] and
+        not fedex_data["TrackPackagesResponse"]["packageList"][0]["errorList"][0]["code"]
+    ):
+        return fedex_data["TrackPackagesResponse"]["packageList"][0]
+
+
+def get_dhl_info(dhl_id, lang="en"):
+    dhl_parcelurl = "http://www.dhl.com/shipmentTracking?" + urllib.parse.urlencode({'AWB': dhl_id})
+
+    dhl_data = getJSON(dhl_parcelurl)
+
+    if "results" in dhl_data and dhl_data["results"]:
+        return dhl_data["results"][0]
+
+
 # TRACKING HANDLERS ###################################################
 
 def handle_tnt(tracknum):
@@ -166,6 +224,13 @@ def handle_postnl(tracknum):
                 ")." % (tracknum, post_status, post_destination, post_date))
 
 
+def handle_usps(tracknum):
+    info = get_usps_info(tracknum)
+    if info:
+        notif, last_date, last_status, last_location = info
+        return ("USPS \x02{tracknum}\x0F is {last_status} in \x02{last_location}\x0F as of {last_date}: {notif}".format(tracknum=tracknum, notif=notif, last_date=last_date, last_status=last_status.lower(), last_location=last_location))
+
+
 def handle_colissimo(tracknum):
     info = get_colissimo_info(tracknum)
     if info:
@@ -189,6 +254,23 @@ def handle_coliprive(tracknum):
         return ("Colis Privé: \x02%s\x0F : \x02%s\x0F." % (tracknum, info))
 
 
+def handle_fedex(tracknum):
+    info = get_fedex_info(tracknum)
+    if info:
+        if info["displayActDeliveryDateTime"] != "":
+            return ("{trackingCarrierDesc}: \x02{statusWithDetails}\x0F: in \x02{statusLocationCity}, {statusLocationCntryCD}\x0F, delivered on: {displayActDeliveryDateTime}.".format(**info))
+        elif info["statusLocationCity"] != "":
+            return ("{trackingCarrierDesc}: \x02{statusWithDetails}\x0F: estimated delivery: {displayEstDeliveryDateTime}.".format(**info))
+        else:
+            return ("{trackingCarrierDesc}: \x02{statusWithDetails}\x0F: in \x02{statusLocationCity}, {statusLocationCntryCD}\x0F, estimated delivery: {displayEstDeliveryDateTime}.".format(**info))
+
+
+def handle_dhl(tracknum):
+    info = get_dhl_info(tracknum)
+    if info:
+        return "DHL {label} {id}: \x02{description}\x0F".format(**info)
+
+
 TRACKING_HANDLERS = {
     'laposte': handle_laposte,
     'postnl': handle_postnl,
@@ -196,6 +278,9 @@ TRACKING_HANDLERS = {
     'chronopost': handle_chronopost,
     'coliprive': handle_coliprive,
     'tnt': handle_tnt,
+    'fedex': handle_fedex,
+    'dhl': handle_dhl,
+    'usps': handle_usps,
 }
 
 

modules/syno.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################

modules/tpb.py

@@ -8,7 +8,7 @@ from nemubot.tools.web import getJSON
 
 nemubotversion = 4.0
 
-from more import Response
+from nemubot.module.more import Response
 
 URL_TPBAPI = None
 

modules/translate.py

@@ -8,7 +8,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # GLOBALS #############################################################

modules/urbandict.py

@@ -8,7 +8,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 # MODULE CORE #########################################################
 

modules/urlreducer.py

@@ -84,8 +84,22 @@ LAST_URLS = dict()
 
 @hook.message()
 def parselisten(msg):
-    parseresponse(msg)
-    return None
+    global LAST_URLS
+    if hasattr(msg, "message") and isinstance(msg.message, str):
+        urls = re.findall("([a-zA-Z0-9+.-]+:(?://)?(?:[^ :/]+:[0-9]+)?[^ :]+)",
+                          msg.message)
+        for url in urls:
+            o = urlparse(web._getNormalizedURL(url), "http")
+
+            # Skip short URLs
+            if (o.netloc == "" or o.netloc in PROVIDERS or
+                    len(o.netloc) + len(o.path) < 17):
+                continue
+
+            for recv in msg.to:
+                if recv not in LAST_URLS:
+                    LAST_URLS[recv] = list()
+                LAST_URLS[recv].append(url)
 
 
 @hook.post()

modules/velib.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################

modules/virtualradar.py

@@ -10,8 +10,8 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
-import mapquest
+from nemubot.module.more import Response
+from nemubot.module import mapquest
 
 # GLOBALS #############################################################
 
@@ -80,7 +80,7 @@ def cmd_flight(msg):
     if not len(msg.args):
         raise IMException("please indicate a flight")
 
-    res = Response(channel=msg.channel, nick=msg.nick,
+    res = Response(channel=msg.channel, nick=msg.frm,
                    nomore="No more flights", count=" (%s more flights)")
 
     for param in msg.args:

modules/weather.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 
-"""The weather module"""
+"""The weather module. Powered by Dark Sky <https://darksky.net/poweredby/>"""
 
 import datetime
 import re
@@ -11,13 +11,13 @@ from nemubot.hooks import hook
 from nemubot.tools import web
 from nemubot.tools.xmlparser.node import ModuleState
 
-import mapquest
+from nemubot.module import mapquest
 
 nemubotversion = 4.0
 
-from more import Response
+from nemubot.module.more import Response
 
-URL_DSAPI = "https://api.forecast.io/forecast/%s/%%s,%%s"
+URL_DSAPI = "https://api.darksky.net/forecast/%s/%%s,%%s?lang=%%s&units=%%s"
 
 def load(context):
     if not context.config or "darkskyapikey" not in context.config:
@@ -30,52 +30,14 @@ def load(context):
     URL_DSAPI = URL_DSAPI % context.config["darkskyapikey"]
 
 
-def help_full ():
-    return "!weather /city/: Display the current weather in /city/."
-
-
-def fahrenheit2celsius(temp):
-    return int((temp - 32) * 50/9)/10
-
-
-def mph2kmph(speed):
-    return int(speed * 160.9344)/100
-
-
-def inh2mmh(size):
-    return int(size * 254)/10
-
-
 def format_wth(wth):
-    return ("%s °C %s; precipitation (%s %% chance) intensity: %s mm/h; relative humidity: %s %%; wind speed: %s km/h %s°; cloud coverage: %s %%; pressure: %s hPa; ozone: %s DU" %
-           (
-               fahrenheit2celsius(wth["temperature"]),
-               wth["summary"],
-               int(wth["precipProbability"] * 100),
-               inh2mmh(wth["precipIntensity"]),
-               int(wth["humidity"] * 100),
-               mph2kmph(wth["windSpeed"]),
-               wth["windBearing"],
-               int(wth["cloudCover"] * 100),
-               int(wth["pressure"]),
-               int(wth["ozone"])
-           ))
+    return ("{temperature} °C {summary}; precipitation ({precipProbability:.0%} chance) intensity: {precipIntensity} mm/h; relative humidity: {humidity:.0%}; wind speed: {windSpeed} km/s {windBearing}°; cloud coverage: {cloudCover:.0%}; pressure: {pressure} hPa; ozone: {ozone} DU"
+            .format(**wth)
+           )
 
 
 def format_forecast_daily(wth):
-    return ("%s; between %s-%s °C; precipitation (%s %% chance) intensity: maximum %s mm/h; relative humidity: %s %%; wind speed: %s km/h %s°; cloud coverage: %s %%; pressure: %s hPa; ozone: %s DU" %
-           (
-               wth["summary"],
-               fahrenheit2celsius(wth["temperatureMin"]), fahrenheit2celsius(wth["temperatureMax"]),
-               int(wth["precipProbability"] * 100),
-               inh2mmh(wth["precipIntensityMax"]),
-               int(wth["humidity"] * 100),
-               mph2kmph(wth["windSpeed"]),
-               wth["windBearing"],
-               int(wth["cloudCover"] * 100),
-               int(wth["pressure"]),
-               int(wth["ozone"])
-           ))
+    return ("{summary}; between {temperatureMin}-{temperatureMax} °C; precipitation ({precipProbability:.0%} chance) intensity: maximum {precipIntensity} mm/h; relative humidity: {humidity:.0%}; wind speed: {windSpeed} km/h {windBearing}°; cloud coverage: {cloudCover:.0%}; pressure: {pressure} hPa; ozone: {ozone} DU".format(**wth))
 
 
 def format_timestamp(timestamp, tzname, tzoffset, format="%c"):
@@ -126,8 +88,8 @@ def treat_coord(msg):
         raise IMException("indique-moi un nom de ville ou des coordonnées.")
 
 
-def get_json_weather(coords):
-    wth = web.getJSON(URL_DSAPI % (float(coords[0]), float(coords[1])))
+def get_json_weather(coords, lang="en", units="auto"):
+    wth = web.getJSON(URL_DSAPI % (float(coords[0]), float(coords[1]), lang, units))
 
     # First read flags
     if wth is None or "darksky-unavailable" in wth["flags"]:
@@ -149,10 +111,16 @@ def cmd_coordinates(msg):
     return Response("Les coordonnées de %s sont %s,%s" % (msg.args[0], coords["lat"], coords["long"]), channel=msg.channel)
 
 
-@hook.command("alert")
+@hook.command("alert",
+              keywords={
+                  "lang=LANG": "change the output language of weather sumarry; default: en",
+                  "units=UNITS": "return weather conditions in the requested units; default: auto",
+              })
 def cmd_alert(msg):
     loc, coords, specific = treat_coord(msg)
-    wth = get_json_weather(coords)
+    wth = get_json_weather(coords,
+                           lang=msg.kwargs["lang"] if "lang" in msg.kwargs else "en",
+                           units=msg.kwargs["units"] if "units" in msg.kwargs else "auto")
 
     res = Response(channel=msg.channel, nomore="No more weather alert", count=" (%d more alerts)")
 
@@ -166,10 +134,20 @@ def cmd_alert(msg):
     return res
 
 
-@hook.command("météo")
+@hook.command("météo",
+              help="Display current weather and previsions",
+              help_usage={
+                  "CITY": "Display the current weather and previsions in CITY",
+              },
+              keywords={
+                  "lang=LANG": "change the output language of weather sumarry; default: en",
+                  "units=UNITS": "return weather conditions in the requested units; default: auto",
+              })
 def cmd_weather(msg):
     loc, coords, specific = treat_coord(msg)
-    wth = get_json_weather(coords)
+    wth = get_json_weather(coords,
+                           lang=msg.kwargs["lang"] if "lang" in msg.kwargs else "en",
+                           units=msg.kwargs["units"] if "units" in msg.kwargs else "auto")
 
     res = Response(channel=msg.channel, nomore="No more weather information")
 
@@ -225,7 +203,7 @@ gps_ask = re.compile(r"^\s*(?P<city>.*\w)\s*(?:(?:se|est)\s+(?:trouve|situ[ée]*
 
 @hook.ask()
 def parseask(msg):
-    res = gps_ask.match(msg.text)
+    res = gps_ask.match(msg.message)
     if res is not None:
         city_name = res.group("city").lower()
         gps_lat = res.group("lat").replace(",", ".")
@@ -242,4 +220,4 @@ def parseask(msg):
             context.data.addChild(ms)
         context.save()
         return Response("ok, j'ai bien noté les coordonnées de %s" % res.group("city"),
-                        msg.channel, msg.nick)
+                        msg.channel, msg.frm)

modules/whois.py

@@ -1,5 +1,6 @@
 # coding=utf-8
 
+import json
 import re
 
 from nemubot import context
@@ -9,17 +10,30 @@ from nemubot.tools.xmlparser.node import ModuleState
 
 nemubotversion = 3.4
 
-from more import Response
-from networking.page import headers
+from nemubot.module.more import Response
+from nemubot.module.networking.page import headers
 
 PASSWD_FILE = None
+# You can get one with: curl -b "sessionid=YOURSESSIONID" 'https://accounts.cri.epita.net/api/users/' > users.json
+APIEXTRACT_FILE = None
 
 def load(context):
     global PASSWD_FILE
     if not context.config or "passwd" not in context.config:
         print("No passwd file given")
+    else:
+        PASSWD_FILE = context.config["passwd"]
+        print("passwd file loaded:", PASSWD_FILE)
+
+    global APIEXTRACT_FILE
+    if not context.config or "apiextract" not in context.config:
+        print("No passwd file given")
+    else:
+        APIEXTRACT_FILE = context.config["apiextract"]
+        print("JSON users file loaded:", APIEXTRACT_FILE)
+
+    if PASSWD_FILE is None and APIEXTRACT_FILE is None:
         return None
-    PASSWD_FILE = context.config["passwd"]
 
     if not context.data.hasNode("aliases"):
         context.data.addChild(ModuleState("aliases"))
@@ -35,16 +49,26 @@ def load(context):
 
 class Login:
 
-    def __init__(self, line):
-        s = line.split(":")
-        self.login = s[0]
-        self.uid = s[2]
-        self.gid = s[3]
-        self.cn = s[4]
-        self.home = s[5]
+    def __init__(self, line=None, login=None, uidNumber=None, cn=None, promo=None, **kwargs):
+        if line is not None:
+            s = line.split(":")
+            self.login = s[0]
+            self.uid = s[2]
+            self.gid = s[3]
+            self.cn = s[4]
+            self.home = s[5]
+        else:
+            self.login = login
+            self.uid = uidNumber
+            self.promo = promo
+            self.cn = cn
+            self.gid = "epita" + promo
 
     def get_promo(self):
-        return self.home.split("/")[2].replace("_", " ")
+        if hasattr(self, "promo"):
+            return self.promo
+        if hasattr(self, "home"):
+            return self.home.split("/")[2].replace("_", " ")
 
     def get_photo(self):
         if self.login in context.data.getNode("pics").index:
@@ -60,17 +84,25 @@ class Login:
         return None
 
 
-def found_login(login, search=False):
+def login_lookup(login, search=False):
     if login in context.data.getNode("aliases").index:
         login = context.data.getNode("aliases").index[login]["to"]
 
+    if APIEXTRACT_FILE:
+        with open(APIEXTRACT_FILE, encoding="utf-8") as f:
+            api = json.load(f)
+            for l in api:
+                if (not search and l["login"] == login) or (search and (("login" in l and l["login"].find(login) != -1) or ("cn" in l and l["cn"].find(login) != -1) or ("uid" in l and str(l["uid"]) == login))):
+                    yield Login(**l)
+
     login_ = login + (":" if not search else "")
     lsize = len(login_)
 
-    with open(PASSWD_FILE, encoding="iso-8859-15") as f:
-        for l in f.readlines():
-            if l[:lsize] == login_:
-                yield Login(l.strip())
+    if PASSWD_FILE:
+        with open(PASSWD_FILE, encoding="iso-8859-15") as f:
+            for l in f.readlines():
+                if l[:lsize] == login_:
+                    yield Login(l.strip())
 
 def cmd_whois(msg):
     if len(msg.args) < 1:
@@ -87,7 +119,7 @@ def cmd_whois(msg):
     res = Response(channel=msg.channel, count=" (%d more logins)", line_treat=format_response)
     for srch in msg.args:
         found = False
-        for l in found_login(srch, "lookup" in msg.kwargs):
+        for l in login_lookup(srch, "lookup" in msg.kwargs):
             found = True
             res.append_message((srch, l))
         if not found:
@@ -98,7 +130,7 @@ def cmd_whois(msg):
 def cmd_nicks(msg):
     if len(msg.args) < 1:
         raise IMException("Provide a login")
-    nick = found_login(msg.args[0])
+    nick = login_lookup(msg.args[0])
     if nick is None:
         nick = msg.args[0]
     else:
@@ -115,12 +147,12 @@ def cmd_nicks(msg):
 
 @hook.ask()
 def parseask(msg):
-    res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)\s+([a-zA-Z0-9_-]{3,8})$", msg.text, re.I)
+    res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)\s+([a-zA-Z0-9_-]{3,8})$", msg.message, re.I)
     if res is not None:
         nick = res.group(1)
         login = res.group(3)
         if nick == "my" or nick == "I" or nick == "i" or nick == "je" or nick == "mon" or nick == "ma":
-            nick = msg.nick
+            nick = msg.frm
         if nick in context.data.getNode("aliases").index:
             context.data.getNode("aliases").index[nick]["to"] = login
         else:
@@ -132,4 +164,4 @@ def parseask(msg):
         return Response("ok, c'est noté, %s est %s"
                         % (nick, login),
                         channel=msg.channel,
-                        nick=msg.nick)
+                        nick=msg.frm)

modules/wolframalpha.py

@@ -10,7 +10,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################

modules/worldcup.py

@@ -15,7 +15,7 @@ from nemubot.tools.xmlparser.node import ModuleState
 
 nemubotversion = 3.4
 
-from more import Response
+from nemubot.module.more import Response
 
 API_URL="http://worldcup.sfg.io/%s"
 
@@ -32,7 +32,7 @@ def start_watch(msg):
     w = ModuleState("watch")
     w["server"] = msg.server
     w["channel"] = msg.channel
-    w["proprio"] = msg.nick
+    w["proprio"] = msg.frm
     w["start"] = datetime.now(timezone.utc)
     context.data.addChild(w)
     context.save()

modules/youtube-title.py

@@ -4,7 +4,7 @@ import re, json, subprocess
 from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools.web import _getNormalizedURL, getURLContent
-from more import Response
+from nemubot.module.more import Response
 
 """Get information of youtube videos"""
 

nemubot/__init__.py

@@ -17,9 +17,9 @@
 __version__ = '4.0.dev3'
 __author__  = 'nemunaire'
 
-from nemubot.modulecontext import ModuleContext
+from nemubot.modulecontext import _ModuleContext
 
-context = ModuleContext(None, None)
+context = _ModuleContext()
 
 
 def requires_version(min=None, max=None):
@@ -53,41 +53,50 @@ def attach(pid, socketfile):
         sys.stderr.write("\n")
         return 1
 
-    from select import select
+    import select
+    mypoll = select.poll()
+
+    mypoll.register(sys.stdin.fileno(), select.POLLIN | select.POLLPRI)
+    mypoll.register(sock.fileno(), select.POLLIN | select.POLLPRI)
     try:
         while True:
-            rl, wl, xl = select([sys.stdin, sock], [], [])
+            for fd, flag in mypoll.poll():
+                if flag & (select.POLLERR | select.POLLHUP | select.POLLNVAL):
+                    sock.close()
+                    print("Connection closed.")
+                    return 1
 
-            if sys.stdin in rl:
-                line = sys.stdin.readline().strip()
-                if line == "exit" or line == "quit":
-                    return 0
-                elif line == "reload":
-                    import os, signal
-                    os.kill(pid, signal.SIGHUP)
-                    print("Reload signal sent. Please wait...")
+                if fd == sys.stdin.fileno():
+                    line = sys.stdin.readline().strip()
+                    if line == "exit" or line == "quit":
+                        return 0
+                    elif line == "reload":
+                        import os, signal
+                        os.kill(pid, signal.SIGHUP)
+                        print("Reload signal sent. Please wait...")
 
-                elif line == "shutdown":
-                    import os, signal
-                    os.kill(pid, signal.SIGTERM)
-                    print("Shutdown signal sent. Please wait...")
+                    elif line == "shutdown":
+                        import os, signal
+                        os.kill(pid, signal.SIGTERM)
+                        print("Shutdown signal sent. Please wait...")
 
-                elif line == "kill":
-                    import os, signal
-                    os.kill(pid, signal.SIGKILL)
-                    print("Signal sent...")
-                    return 0
+                    elif line == "kill":
+                        import os, signal
+                        os.kill(pid, signal.SIGKILL)
+                        print("Signal sent...")
+                        return 0
 
-                elif line == "stack" or line == "stacks":
-                    import os, signal
-                    os.kill(pid, signal.SIGUSR1)
-                    print("Debug signal sent. Consult logs.")
+                    elif line == "stack" or line == "stacks":
+                        import os, signal
+                        os.kill(pid, signal.SIGUSR1)
+                        print("Debug signal sent. Consult logs.")
 
-                else:
-                    sock.send(line.encode() + b'\r\n')
+                    else:
+                        sock.send(line.encode() + b'\r\n')
+
+                if fd == sock.fileno():
+                    sys.stdout.write(sock.recv(2048).decode())
 
-            if sock in rl:
-                sys.stdout.write(sock.recv(2048).decode())
     except KeyboardInterrupt:
         pass
     except:
@@ -97,13 +106,28 @@ def attach(pid, socketfile):
     return 0
 
 
-def daemonize():
+def daemonize(socketfile=None, autoattach=True):
     """Detach the running process to run as a daemon
     """
 
     import os
     import sys
 
+    if socketfile is not None:
+        try:
+            pid = os.fork()
+            if pid > 0:
+                if autoattach:
+                    import time
+                    os.waitpid(pid, 0)
+                    time.sleep(1)
+                    sys.exit(attach(pid, socketfile))
+                else:
+                    sys.exit(0)
+        except OSError as err:
+            sys.stderr.write("Unable to fork: %s\n" % err)
+            sys.exit(1)
+
     try:
         pid = os.fork()
         if pid > 0:

nemubot/__main__.py

@@ -1,5 +1,5 @@
 # Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2016  Mercier Pierre-Olivier
+# Copyright (C) 2012-2017  Mercier Pierre-Olivier
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -37,6 +37,9 @@ def main():
                         default=["./modules/"],
                         help="directory to use as modules store")
 
+    parser.add_argument("-A", "--no-attach", action="store_true",
+                        help="don't attach after fork")
+
     parser.add_argument("-d", "--debug", action="store_true",
                         help="don't deamonize, keep in foreground")
 
@@ -74,28 +77,6 @@ def main():
     args.files = [x for x in map(os.path.abspath, args.files)]
     args.modules_path = [x for x in map(os.path.abspath, args.modules_path)]
 
-    # Check if an instance is already launched
-    if args.pidfile is not None and os.path.isfile(args.pidfile):
-        with open(args.pidfile, "r") as f:
-            pid = int(f.readline())
-        try:
-            os.kill(pid, 0)
-        except OSError:
-            pass
-        else:
-            from nemubot import attach
-            sys.exit(attach(pid, args.socketfile))
-
-    # Daemonize
-    if not args.debug:
-        from nemubot import daemonize
-        daemonize()
-
-    # Store PID to pidfile
-    if args.pidfile is not None:
-        with open(args.pidfile, "w+") as f:
-            f.write(str(os.getpid()))
-
     # Setup logging interface
     import logging
     logger = logging.getLogger("nemubot")
@@ -115,6 +96,18 @@ def main():
     fh.setFormatter(formatter)
     logger.addHandler(fh)
 
+    # Check if an instance is already launched
+    if args.pidfile is not None and os.path.isfile(args.pidfile):
+        with open(args.pidfile, "r") as f:
+            pid = int(f.readline())
+        try:
+            os.kill(pid, 0)
+        except OSError:
+            pass
+        else:
+            from nemubot import attach
+            sys.exit(attach(pid, args.socketfile))
+
     # Add modules dir paths
     modules_paths = list()
     for path in args.modules_path:
@@ -125,10 +118,10 @@ def main():
 
     # Create bot context
     from nemubot import datastore
-    from nemubot.bot import Bot, sync_act
+    from nemubot.bot import Bot
     context = Bot(modules_paths=modules_paths,
                   data_store=datastore.XML(args.data_path),
-                  verbosity=args.verbose)
+                  debug=args.verbose > 0)
 
     if args.no_connect:
         context.noautoconnect = True
@@ -140,35 +133,58 @@ def main():
 
     # Load requested configuration files
     for path in args.files:
-        if os.path.isfile(path):
-            sync_act("loadconf", path)
-        else:
+        if not os.path.isfile(path):
             logger.error("%s is not a readable file", path)
+            continue
+
+        config = load_config(path)
+
+        # Preset each server in this file
+        for server in config.servers:
+            srv = server.server(config)
+            # Add the server in the context
+            if context.add_server(srv):
+                logger.info("Server '%s' successfully added.", srv.name)
+            else:
+                logger.error("Can't add server '%s'.", srv.name)
+
+        # Load module and their configuration
+        for mod in config.modules:
+            context.modules_configuration[mod.name] = mod
+            if mod.autoload:
+                try:
+                    __import__("nemubot.module." + mod.name)
+                except:
+                    logger.exception("Exception occurs when loading module"
+                                     " '%s'", mod.name)
+
+        # Load files asked by the configuration file
+        args.files += config.includes
+
 
     if args.module:
         for module in args.module:
-            __import__(module)
+            __import__("nemubot.module." + module)
+
+    if args.socketfile:
+        from nemubot.server.socket import UnixSocketListener
+        context.add_server(UnixSocketListener(new_server_cb=context.add_server,
+                                              location=args.socketfile,
+                                              name="master_socket"))
+
+    # Daemonize
+    if not args.debug:
+        from nemubot import daemonize
+        daemonize(args.socketfile, not args.no_attach)
 
     # Signals handling
-    def sigtermhandler(signum, frame):
+    def sigtermhandler():
         """On SIGTERM and SIGINT, quit nicely"""
         context.quit()
-    signal.signal(signal.SIGINT, sigtermhandler)
-    signal.signal(signal.SIGTERM, sigtermhandler)
+    context.loop.add_signal_handler(signal.SIGINT, sigtermhandler)
+    context.loop.add_signal_handler(signal.SIGTERM, sigtermhandler)
 
-    def sighuphandler(signum, frame):
-        """On SIGHUP, perform a deep reload"""
-        nonlocal context
-
-        logger.debug("SIGHUP receive, iniate reload procedure...")
-
-        # Reload configuration file
-        for path in args.files:
-            if os.path.isfile(path):
-                sync_act("loadconf", path)
-    signal.signal(signal.SIGHUP, sighuphandler)
-
-    def sigusr1handler(signum, frame):
+    def sigusr1handler():
         """On SIGHUSR1, display stacktraces"""
         import threading, traceback
         for threadId, stack in sys._current_frames().items():
@@ -180,27 +196,50 @@ def main():
             logger.debug("########### Thread %s:\n%s",
                          thName,
                          "".join(traceback.format_stack(stack)))
-    signal.signal(signal.SIGUSR1, sigusr1handler)
+    context.loop.add_signal_handler(signal.SIGUSR1, sigusr1handler)
 
-    if args.socketfile:
-        from nemubot.server.socket import UnixSocketListener
-        context.add_server(UnixSocketListener(new_server_cb=context.add_server,
-                                              location=args.socketfile,
-                                              name="master_socket"))
+    # Store PID to pidfile
+    if args.pidfile is not None:
+        with open(args.pidfile, "w+") as f:
+            f.write(str(os.getpid()))
 
-    # context can change when performing an hotswap, always join the latest context
-    oldcontext = None
-    while oldcontext != context:
-        oldcontext = context
-        context.start()
-        context.join()
+    context.start()
+    context.loop.set_debug(args.verbose > 0)
+    context.loop.run_forever()
+    context.join()
 
     # Wait for consumers
     logger.info("Waiting for other threads shuts down...")
     if args.debug:
-        sigusr1handler(0, None)
+        sigusr1handler()
     sys.exit(0)
 
 
+def load_config(filename):
+    """Load a configuration file
+
+    Arguments:
+    filename -- the path to the file to load
+    """
+
+    from nemubot.channel import Channel
+    from nemubot import config
+    from nemubot.tools.xmlparser import XMLParser
+
+    try:
+        p = XMLParser({
+            "nemubotconfig": config.Nemubot,
+            "server": config.Server,
+            "channel": Channel,
+            "module": config.Module,
+            "include": config.Include,
+        })
+        return p.parse_file(filename)
+    except:
+        logger.exception("Can't load `%s'; this is not a valid nemubot "
+                         "configuration file.", filename)
+    return None
+
+
 if __name__ == "__main__":
     main()

nemubot/bot.py

@@ -14,12 +14,15 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import asyncio
 from datetime import datetime, timezone
 import logging
 from multiprocessing import JoinableQueue
 import threading
+import traceback
 import select
 import sys
+import weakref
 
 from nemubot import __version__
 from nemubot.consumer import Consumer, EventConsumer, MessageConsumer
@@ -39,14 +42,14 @@ class Bot(threading.Thread):
     """Class containing the bot context and ensuring key goals"""
 
     def __init__(self, ip="127.0.0.1", modules_paths=list(),
-                 data_store=datastore.Abstract(), verbosity=0):
+                 data_store=datastore.Abstract(), debug=False, loop=None):
         """Initialize the bot context
 
         Keyword arguments:
         ip -- The external IP of the bot (default: 127.0.0.1)
         modules_paths -- Paths to all directories where looking for modules
         data_store -- An instance of the nemubot datastore for bot's modules
-        verbosity -- verbosity level
+        debug -- enable debug
         """
 
         super().__init__(name="Nemubot main")
@@ -55,9 +58,19 @@ class Bot(threading.Thread):
                     __version__,
                     sys.version_info.major, sys.version_info.minor, sys.version_info.micro)
 
-        self.verbosity = verbosity
+        self.debug = debug
         self.stop = None
 
+        #
+        self.loop = loop if loop is not None else asyncio.get_event_loop()
+
+        # Those events are used to ensure there is always one event in the next 24h, else overflow can occurs on loop timeout
+        def event_sentinel(offset=43210):
+            logger.debug("Defining new event sentinelle in %ss", 43210 + offset)
+            self.loop.call_later(43210 + offset, event_sentinel)
+        event_sentinel(0)
+        event_sentinel(43210)
+
         # External IP for accessing this bot
         import ipaddress
         self.ip = ipaddress.ip_address(ip)
@@ -73,10 +86,6 @@ class Bot(threading.Thread):
         self.modules = dict()
         self.modules_configuration = dict()
 
-        # Events
-        self.events      = list()
-        self.event_timer = None
-
         # Own hooks
         from nemubot.treatment import MessageTreater
         self.treater     = MessageTreater()
@@ -91,23 +100,24 @@ class Bot(threading.Thread):
 
         def in_echo(msg):
             from nemubot.message import Text
-            return Text(msg.nick + ": " + " ".join(msg.args), to=msg.to_response)
+            return Text(msg.frm + ": " + " ".join(msg.args), to=msg.to_response)
         self.treater.hm.add_hook(nemubot.hooks.Command(in_echo, "echo"), "in", "Command")
 
         def _help_msg(msg):
             """Parse and response to help messages"""
-            from more import Response
+            from nemubot.module.more import Response
             res = Response(channel=msg.to_response)
             if len(msg.args) >= 1:
-                if msg.args[0] in self.modules:
-                    if hasattr(self.modules[msg.args[0]], "help_full"):
-                        hlp = self.modules[msg.args[0]].help_full()
+                if "nemubot.module." + msg.args[0] in self.modules and self.modules["nemubot.module." + msg.args[0]]() is not None:
+                    mname = "nemubot.module." + msg.args[0]
+                    if hasattr(self.modules[mname](), "help_full"):
+                        hlp = self.modules[mname]().help_full()
                         if isinstance(hlp, Response):
                             return hlp
                         else:
                             res.append_message(hlp)
                     else:
-                        res.append_message([str(h) for s,h in self.modules[msg.args[0]].__nemubot_context__.hooks], title="Available commands for module " + msg.args[0])
+                        res.append_message([str(h) for s,h in self.modules[mname]().__nemubot_context__.hooks], title="Available commands for module " + msg.args[0])
                 elif msg.args[0][0] == "!":
                     from nemubot.message.command import Command
                     for h in self.treater._in_hooks(Command(msg.args[0][1:])):
@@ -137,7 +147,7 @@ class Bot(threading.Thread):
                 res.append_message(title="Pour plus de détails sur un module, "
                                    "envoyez \"!help nomdumodule\". Voici la liste"
                                    " de tous les modules disponibles localement",
-                                   message=["\x03\x02%s\x03\x02 (%s)" % (im, self.modules[im].__doc__) for im in self.modules if self.modules[im].__doc__])
+                                   message=["\x03\x02%s\x03\x02 (%s)" % (im.replace("nemubot.module.", ""), self.modules[im]().__doc__) for im in self.modules if self.modules[im]() is not None and self.modules[im]().__doc__])
             return res
         self.treater.hm.add_hook(nemubot.hooks.Command(_help_msg, "help"), "in", "Command")
 
@@ -148,7 +158,18 @@ class Bot(threading.Thread):
         self.cnsr_thrd_size = -1
 
 
+    def __del__(self):
+        self.datastore.close()
+
+
     def run(self):
+        global sync_queue
+
+        # Rewrite the sync_queue, as the daemonization process tend to disturb it
+        old_sync_queue, sync_queue = sync_queue, JoinableQueue()
+        while not old_sync_queue.empty():
+            sync_queue.put_nowait(old_sync_queue.get())
+
         self._poll.register(sync_queue._reader, select.POLLIN | select.POLLPRI)
 
         logger.info("Starting main loop")
@@ -188,6 +209,8 @@ class Bot(threading.Thread):
                     args = sync_queue.get()
                     action = args.pop(0)
 
+                    logger.debug("Executing sync_queue action %s%s", action, args)
+
                     if action == "sckt" and len(args) >= 2:
                         try:
                             if args[0] == "write":
@@ -205,11 +228,8 @@ class Bot(threading.Thread):
                     elif action == "exit":
                         self.quit()
 
-                    elif action == "loadconf":
-                        for path in args:
-                            logger.debug("Load configuration from %s", path)
-                            self.load_file(path)
-                            logger.info("Configurations successfully loaded")
+                    elif action == "launch_consumer":
+                        pass  # This is treated after the loop
 
                     sync_queue.task_done()
 
@@ -222,71 +242,33 @@ class Bot(threading.Thread):
                 c = Consumer(self)
                 self.cnsr_thrd.append(c)
                 c.start()
+        sync_queue = None
         logger.info("Ending main loop")
 
 
 
-    # Config methods
-
-    def load_file(self, filename):
-        """Load a configuration file
-
-        Arguments:
-        filename -- the path to the file to load
-        """
-
-        import os
-
-        # Unexisting file, assume a name was passed, import the module!
-        if not os.path.isfile(filename):
-            return self.import_module(filename)
-
-        from nemubot.channel import Channel
-        from nemubot import config
-        from nemubot.tools.xmlparser import XMLParser
-
-        try:
-            p = XMLParser({
-                "nemubotconfig": config.Nemubot,
-                "server": config.Server,
-                "channel": Channel,
-                "module": config.Module,
-                "include": config.Include,
-            })
-            config = p.parse_file(filename)
-        except:
-            logger.exception("Can't load `%s'; this is not a valid nemubot "
-                             "configuration file." % filename)
-            return False
-
-        # Preset each server in this file
-        for server in config.servers:
-            srv = server.server(config)
-            # Add the server in the context
-            if self.add_server(srv, server.autoconnect):
-                logger.info("Server '%s' successfully added." % srv.name)
-            else:
-                logger.error("Can't add server '%s'." % srv.name)
-
-        # Load module and their configuration
-        for mod in config.modules:
-            self.modules_configuration[mod.name] = mod
-            if mod.autoload:
-                try:
-                    __import__(mod.name)
-                except:
-                    logger.exception("Exception occurs when loading module"
-                                     " '%s'", mod.name)
-
-
-        # Load files asked by the configuration file
-        for load in config.includes:
-            self.load_file(load.path)
-
-
     # Events methods
 
-    def add_event(self, evt, eid=None, module_src=None):
+    @asyncio.coroutine
+    def _call_at(self, when, *args, **kwargs):
+        @asyncio.coroutine
+        def _add_event():
+            return self.loop.call_at(when, *args, **kwargs)
+        future = yield from asyncio.run_coroutine_threadsafe(_add_event(), loop=self.loop)
+        logger.debug("New event registered, scheduled in %ss", when - self.loop.time())
+        return future.result()
+
+
+    def call_at(self, when, *args, **kwargs):
+        delay = (when - datetime.now(timezone.utc)).total_seconds()
+        return self._call_at(self.loop.time() + delay, *args, **kwargs)
+
+
+    def call_delay(self, delay, *args, **kwargs):
+        return self._call_at(self.loop.time() + delay, *args, **kwargs)
+
+
+    def add_event(self, evt):
         """Register an event and return its identifiant for futur update
 
         Return:
@@ -295,128 +277,26 @@ class Bot(threading.Thread):
 
         Argument:
         evt -- The event object to add
-
-        Keyword arguments:
-        eid -- The desired event ID (object or string UUID)
-        module_src -- The module to which the event is attached to
         """
 
-        if hasattr(self, "stop") and self.stop:
-            logger.warn("The bot is stopped, can't register new events")
-            return
+        if hasattr(evt, "handle") and evt.handle is not None:
+            raise Exception("Try to launch an already launched event.")
 
-        import uuid
+        def _end_event_timer(event):
+            """Function called at the end of the event timer"""
 
-        # Generate the event id if no given
-        if eid is None:
-            eid = uuid.uuid1()
+            logger.debug("Trigering event")
+            event.handle = None
+            self.cnsr_queue.put_nowait(EventConsumer(event))
+            sync_act("launch_consumer")
 
-        # Fill the id field of the event
-        if type(eid) is uuid.UUID:
-            evt.id = str(eid)
-        else:
-            # Ok, this is quiet useless...
-            try:
-                evt.id = str(uuid.UUID(eid))
-            except ValueError:
-                evt.id = eid
+        evt.start(self.loop)
+        evt.handle = call_at(evt._next, _end_event_timer, evt)
 
-        # TODO: mutex here plz
+        logger.debug("New event registered in %ss", evt._next - self.loop.time())
 
-        # Add the event in its place
-        t = evt.current
-        i = 0 # sentinel
-        for i in range(0, len(self.events)):
-            if self.events[i].current > t:
-                break
-        self.events.insert(i, evt)
+        return evt.handle
 
-        if i == 0:
-            # First event changed, reset timer
-            self._update_event_timer()
-            if len(self.events) <= 0 or self.events[i] != evt:
-                # Our event has been executed and removed from queue
-                return None
-
-        # Register the event in the source module
-        if module_src is not None:
-            module_src.__nemubot_context__.events.append(evt.id)
-        evt.module_src = module_src
-
-        logger.info("New event registered in %d position: %s", i, t)
-        return evt.id
-
-
-    def del_event(self, evt, module_src=None):
-        """Find and remove an event from list
-
-        Return:
-        True if the event has been found and removed, False else
-
-        Argument:
-        evt -- The ModuleEvent object to remove or just the event identifier
-
-        Keyword arguments:
-        module_src -- The module to which the event is attached to (ignored if evt is a ModuleEvent)
-        """
-
-        logger.info("Removing event: %s from %s", evt, module_src)
-
-        from nemubot.event import ModuleEvent
-        if type(evt) is ModuleEvent:
-            id = evt.id
-            module_src = evt.module_src
-        else:
-            id = evt
-
-        if len(self.events) > 0 and id == self.events[0].id:
-            self.events.remove(self.events[0])
-            self._update_event_timer()
-            if module_src is not None:
-                module_src.__nemubot_context__.events.remove(id)
-            return True
-
-        for evt in self.events:
-            if evt.id == id:
-                self.events.remove(evt)
-
-                if module_src is not None:
-                    module_src.__nemubot_context__.events.remove(evt.id)
-                return True
-        return False
-
-
-    def _update_event_timer(self):
-        """(Re)launch the timer to end with the closest event"""
-
-        # Reset the timer if this is the first item
-        if self.event_timer is not None:
-            self.event_timer.cancel()
-
-        if len(self.events):
-            try:
-                remaining = self.events[0].time_left.total_seconds()
-            except:
-                logger.exception("An error occurs during event time calculation:")
-                self.events.pop(0)
-                return self._update_event_timer()
-
-            logger.debug("Update timer: next event in %d seconds", remaining)
-            self.event_timer = threading.Timer(remaining if remaining > 0 else 0, self._end_event_timer)
-            self.event_timer.start()
-
-        else:
-            logger.debug("Update timer: no timer left")
-
-
-    def _end_event_timer(self):
-        """Function called at the end of the event timer"""
-
-        while len(self.events) > 0 and datetime.now(timezone.utc) >= self.events[0].current:
-            evt = self.events.pop(0)
-            self.cnsr_queue.put_nowait(EventConsumer(evt))
-
-        self._update_event_timer()
 
 
     # Consumers methods
@@ -459,6 +339,17 @@ class Bot(threading.Thread):
     def add_module(self, module):
         """Add a module to the context, if already exists, unload the
         old one before"""
+
+        import nemubot.hooks
+
+        self.loop.call_soon_threadsafe(self._add_module,
+                                       module,
+                                       nemubot.hooks.hook.last_registered)
+
+        nemubot.hooks.hook.last_registered = []
+
+
+    def _add_module(self, module, registered_functions):
         module_name = module.__spec__.name if hasattr(module, "__spec__") else module.__name__
 
         if hasattr(self, "stop") and self.stop:
@@ -478,7 +369,7 @@ class Bot(threading.Thread):
         module.print = prnt
 
         # Create module context
-        from nemubot.modulecontext import ModuleContext
+        from nemubot.modulecontext import _ModuleContext, ModuleContext
         module.__nemubot_context__ = ModuleContext(self, module)
 
         if not hasattr(module, "logger"):
@@ -486,14 +377,12 @@ class Bot(threading.Thread):
 
         # Replace imported context by real one
         for attr in module.__dict__:
-            if attr != "__nemubot_context__" and type(module.__dict__[attr]) == ModuleContext:
+            if attr != "__nemubot_context__" and isinstance(module.__dict__[attr], _ModuleContext):
                 module.__dict__[attr] = module.__nemubot_context__
 
         # Register decorated functions
-        import nemubot.hooks
-        for s, h in nemubot.hooks.hook.last_registered:
+        for s, h in registered_functions:
             module.__nemubot_context__.add_hook(h, *s if isinstance(s, list) else s)
-        nemubot.hooks.hook.last_registered = []
 
         # Launch the module
         if hasattr(module, "load"):
@@ -504,18 +393,20 @@ class Bot(threading.Thread):
                 raise
 
         # Save a reference to the module
-        self.modules[module_name] = module
+        self.modules[module_name] = weakref.ref(module)
+        logger.info("Module '%s' successfully loaded.", module_name)
 
 
     def unload_module(self, name):
         """Unload a module"""
-        if name in self.modules:
-            self.modules[name].print("Unloading module %s" % name)
+        if name in self.modules and self.modules[name]() is not None:
+            module = self.modules[name]()
+            module.print("Unloading module %s" % name)
 
             # Call the user defined unload method
-            if hasattr(self.modules[name], "unload"):
-                self.modules[name].unload(self)
-            self.modules[name].__nemubot_context__.unload()
+            if hasattr(module, "unload"):
+                module.unload(self)
+            module.__nemubot_context__.unload()
 
             # Remove from the nemubot dict
             del self.modules[name]
@@ -547,12 +438,8 @@ class Bot(threading.Thread):
     def quit(self):
         """Save and unload modules and disconnect servers"""
 
-        if self.event_timer is not None:
-            logger.info("Stop the event timer...")
-            self.event_timer.cancel()
-
         logger.info("Save and unload all modules...")
-        for mod in self.modules.items():
+        for mod in [m for m in self.modules.keys()]:
             self.unload_module(mod)
 
         logger.info("Close all servers connection...")
@@ -564,11 +451,13 @@ class Bot(threading.Thread):
         for cnsr in k:
             cnsr.stop = True
 
-        self.datastore.close()
+        logger.info("Closing event loop")
+        self.loop.stop()
 
-        self.stop = True
-        sync_act("end")
-        sync_queue.join()
+        if self.stop is False or sync_queue is not None:
+            self.stop = True
+            sync_act("end")
+            sync_queue.join()
 
 
     # Treatment

nemubot/channel.py

@@ -52,11 +52,11 @@ class Channel:
         elif cmd == "MODE":
             self.mode(msg)
         elif cmd == "JOIN":
-            self.join(msg.nick)
+            self.join(msg.frm)
         elif cmd == "NICK":
-            self.nick(msg.nick, msg.text)
+            self.nick(msg.frm, msg.text)
         elif cmd == "PART" or cmd == "QUIT":
-            self.part(msg.nick)
+            self.part(msg.frm)
         elif cmd == "TOPIC":
             self.topic = self.text
 
@@ -120,17 +120,17 @@ class Channel:
             else:
                 self.password = msg.text[1]
         elif msg.text[0] == "+o":
-            self.people[msg.nick] |= 4
+            self.people[msg.frm] |= 4
         elif msg.text[0] == "-o":
-            self.people[msg.nick] &= ~4
+            self.people[msg.frm] &= ~4
         elif msg.text[0] == "+h":
-            self.people[msg.nick] |= 2
+            self.people[msg.frm] |= 2
         elif msg.text[0] == "-h":
-            self.people[msg.nick] &= ~2
+            self.people[msg.frm] &= ~2
         elif msg.text[0] == "+v":
-            self.people[msg.nick] |= 1
+            self.people[msg.frm] |= 1
         elif msg.text[0] == "-v":
-            self.people[msg.nick] &= ~1
+            self.people[msg.frm] &= ~1
 
     def parse332(self, msg):
         """Parse RPL_TOPIC message

nemubot/consumer.py

@@ -88,13 +88,8 @@ class EventConsumer:
             logger.exception("Error during event end")
 
         # Reappend the event in the queue if it has next iteration
-        if self.evt.next is not None:
-            context.add_event(self.evt, eid=self.evt.id)
-
-        # Or remove reference of this event
-        elif (hasattr(self.evt, "module_src") and
-              self.evt.module_src is not None):
-            self.evt.module_src.__nemubot_context__.events.remove(self.evt.id)
+        if self.evt.next():
+            context.add_event(self.evt)
 
 
 

nemubot/datastore/abstract.py

@@ -32,16 +32,20 @@ class Abstract:
     def close(self):
         return
 
-    def load(self, module):
+    def load(self, module, knodes):
         """Load data for the given module
 
         Argument:
         module -- the module name of data to load
+        knodes -- the schema to use to load the datas
 
         Return:
         The loaded data
         """
 
+        if knodes is not None:
+            return None
+
         return self.new()
 
     def save(self, module, data):

nemubot/datastore/xml.py

@@ -83,27 +83,38 @@ class XML(Abstract):
 
         return os.path.join(self.basedir, module + ".xml")
 
-    def load(self, module):
+    def load(self, module, knodes):
         """Load data for the given module
 
         Argument:
         module -- the module name of data to load
+        knodes -- the schema to use to load the datas
         """
 
         data_file = self._get_data_file_path(module)
 
+        if knodes is None:
+            from nemubot.tools.xmlparser import parse_file
+            def _true_load(path):
+                return parse_file(path)
+
+        else:
+            from nemubot.tools.xmlparser import XMLParser
+            p = XMLParser(knodes)
+            def _true_load(path):
+                return p.parse_file(path)
+
         # Try to load original file
         if os.path.isfile(data_file):
-            from nemubot.tools.xmlparser import parse_file
             try:
-                return parse_file(data_file)
+                return _true_load(data_file)
             except xml.parsers.expat.ExpatError:
                 # Try to load from backup
                 for i in range(10):
                     path = data_file + "." + str(i)
                     if os.path.isfile(path):
                         try:
-                            cnt = parse_file(path)
+                            cnt = _true_load(path)
 
                             logger.warn("Restoring from backup: %s", path)
 
@@ -112,7 +123,7 @@ class XML(Abstract):
                             continue
 
         # Default case: initialize a new empty datastore
-        return Abstract.load(self, module)
+        return super().load(module, knodes)
 
     def _rotate(self, path):
         """Backup given path
@@ -143,4 +154,18 @@ class XML(Abstract):
         if self.rotate:
             self._rotate(path)
 
-        return data.save(path)
+        if data is None:
+            return
+
+        import tempfile
+        _, tmpath = tempfile.mkstemp()
+        with open(tmpath, "w") as f:
+            import xml.sax.saxutils
+            gen = xml.sax.saxutils.XMLGenerator(f, "utf-8")
+            gen.startDocument()
+            data.saveElement(gen)
+            gen.endDocument()
+
+        # Atomic save
+        import shutil
+        shutil.move(tmpath, path)

nemubot/event/__init__.py

@@ -21,120 +21,56 @@ class ModuleEvent:
 
     """Representation of a event initiated by a bot module"""
 
-    def __init__(self, call=None, call_data=None, func=None, func_data=None,
-                 cmp=None, cmp_data=None, interval=60, offset=0, times=1):
+    def __init__(self, call=None, cmp=None, interval=60, offset=0, times=1):
 
         """Initialize the event
 
         Keyword arguments:
         call -- Function to call when the event is realized
-        call_data -- Argument(s) (single or dict) to pass as argument
-        func -- Function called to check
-        func_data -- Argument(s) (single or dict) to pass as argument OR if no func, initial data to watch
         cmp -- Boolean function called to check changes
-        cmp_data -- Argument(s) (single or dict) to pass as argument OR if no cmp, data compared to previous
         interval -- Time in seconds between each check (default: 60)
         offset -- Time in seconds added to interval before the first check (default: 0)
         times -- Number of times the event has to be realized before being removed; -1 for no limit (default: 1)
         """
 
-        # What have we to check?
-        self.func = func
-        self.func_data = func_data
-
         # How detect a change?
         self.cmp = cmp
-        self.cmp_data = None
-        if cmp_data is not None:
-            self.cmp_data = cmp_data
-        elif self.func is not None:
-            if self.func_data is None:
-                self.cmp_data = self.func()
-            elif isinstance(self.func_data, dict):
-                self.cmp_data = self.func(**self.func_data)
-            else:
-                self.cmp_data = self.func(self.func_data)
 
         # What should we call when?
         self.call = call
-        if call_data is not None:
-            self.call_data = call_data
-        else:
-            self.call_data = func_data
 
-        # Store times
-        self.offset = timedelta(seconds=offset)  # Time to wait before the first check
+        # Time to wait before the first check
+        if isinstance(offset, timedelta):
+            self.offset = offset
+        else:
+            self.offset = timedelta(seconds=offset)
         self.interval = timedelta(seconds=interval)
-        self._end = None  # Cache
+        self._next = None  # Cache
 
         # How many times do this event?
         self.times = times
 
-    @property
-    def current(self):
-        """Return the date of the near check"""
-        if self.times != 0:
-            if self._end is None:
-                self._end = datetime.now(timezone.utc) + self.offset + self.interval
-            return self._end
-        return None
 
-    @property
+    def start(self, loop):
+        if self._next is None:
+            self._next = loop.time() + self.offset.total_seconds() + self.interval.total_seconds()
+
+
+    def schedule(self, end):
+        self.interval = timedelta(seconds=0)
+        self.offset = end - datetime.now(timezone.utc)
+
+
     def next(self):
-        """Return the date of the next check"""
         if self.times != 0:
-            if self._end is None:
-                return self.current
-            elif self._end < datetime.now(timezone.utc):
-                self._end += self.interval
-            return self._end
-        return None
+            self._next += self.interval.total_seconds()
+            return True
+        return False
 
-    @property
-    def time_left(self):
-        """Return the time left before/after the near check"""
-        if self.current is not None:
-            return self.current - datetime.now(timezone.utc)
-        return timedelta.max
 
     def check(self):
         """Run a check and realized the event if this is time"""
 
-        # Get initial data
-        if self.func is None:
-            d_init = self.func_data
-        elif self.func_data is None:
-            d_init = self.func()
-        elif isinstance(self.func_data, dict):
-            d_init = self.func(**self.func_data)
-        else:
-            d_init = self.func(self.func_data)
-
-        # then compare with current data
-        if self.cmp is None:
-            if self.cmp_data is None:
-                rlz = True
-            else:
-                rlz = (d_init != self.cmp_data)
-        elif self.cmp_data is None:
-            rlz = self.cmp(d_init)
-        elif isinstance(self.cmp_data, dict):
-            rlz = self.cmp(d_init, **self.cmp_data)
-        else:
-            rlz = self.cmp(d_init, self.cmp_data)
-
-        if rlz:
+        if self.cmp():
             self.times -= 1
-
-            # Call attended function
-            if self.call_data is None:
-                if d_init is None:
-                    self.call()
-                else:
-                    self.call(d_init)
-            elif isinstance(self.call_data, dict):
-                self.call(d_init, **self.call_data)
-            elif d_init is None:
-                self.call(self.call_data)
-            else:
-                self.call(d_init, self.call_data)
+            self.call()

nemubot/importer.py

@@ -29,16 +29,16 @@ class ModuleFinder(Finder):
         self.add_module = add_module
 
     def find_module(self, fullname, path=None):
-        # Search only for new nemubot modules (packages init)
-        if path is None:
+        if path is not None and fullname.startswith("nemubot.module."):
+            module_name = fullname.split(".", 2)[2]
             for mpath in self.modules_paths:
-                if os.path.isfile(os.path.join(mpath, fullname + ".py")):
+                if os.path.isfile(os.path.join(mpath, module_name + ".py")):
                     return ModuleLoader(self.add_module, fullname,
-                                        os.path.join(mpath, fullname + ".py"))
-                elif os.path.isfile(os.path.join(os.path.join(mpath, fullname), "__init__.py")):
+                                        os.path.join(mpath, module_name + ".py"))
+                elif os.path.isfile(os.path.join(os.path.join(mpath, module_name), "__init__.py")):
                     return ModuleLoader(self.add_module, fullname,
                                         os.path.join(
-                                            os.path.join(mpath, fullname),
+                                            os.path.join(mpath, module_name),
                                             "__init__.py"))
         return None
 
@@ -53,17 +53,17 @@ class ModuleLoader(SourceFileLoader):
     def _load(self, module, name):
         # Add the module to the global modules list
         self.add_module(module)
-        logger.info("Module '%s' successfully loaded.", name)
+        logger.info("Module '%s' successfully imported from %s.", name.split(".", 2)[2], self.path)
         return module
 
 
     # Python 3.4
     def exec_module(self, module):
-        super(ModuleLoader, self).exec_module(module)
+        super().exec_module(module)
         self._load(module, module.__spec__.name)
 
 
     # Python 3.3
     def load_module(self, fullname):
-        module = super(ModuleLoader, self).load_module(fullname)
+        module = super().load_module(fullname)
         return self._load(module, module.__name__)

nemubot/message/abstract.py

@@ -21,7 +21,7 @@ class Abstract:
 
     """This class represents an abstract message"""
 
-    def __init__(self, server=None, date=None, to=None, to_response=None, frm=None):
+    def __init__(self, server=None, date=None, to=None, to_response=None, frm=None, frm_owner=False):
         """Initialize an abstract message
 
         Arguments:
@@ -40,7 +40,7 @@ class Abstract:
                              else [ to_response ])
         self.frm = frm  # None allowed when it designate this bot
 
-        self.frm_owner = False  # Filled later, in consumer
+        self.frm_owner = frm_owner
 
 
     @property
@@ -59,12 +59,6 @@ class Abstract:
         else:
             return None
 
-    @property
-    def nick(self):
-        # TODO: this is for legacy modules
-        return self.frm
-
-
     def accept(self, visitor):
         visitor.visit(self)
 
@@ -78,7 +72,8 @@ class Abstract:
             "date": self.date,
             "to": self.to,
             "to_response": self._to_response,
-            "frm": self.frm
+            "frm": self.frm,
+            "frm_owner": self.frm_owner,
         }
 
         for w in without:

nemubot/message/command.py

@@ -31,11 +31,6 @@ class Command(Abstract):
     def __str__(self):
         return self.cmd + " @" + ",@".join(self.args)
 
-    @property
-    def cmds(self):
-        # TODO: this is for legacy modules
-        return [self.cmd] + self.args
-
 
 class OwnerCommand(Command):
 

nemubot/message/response.py

@@ -27,8 +27,3 @@ class Response(Abstract):
 
     def __str__(self):
         return self.cmd + " @" + ",@".join(self.args)
-
-    @property
-    def cmds(self):
-        # TODO: this is for legacy modules
-        return [self.cmd] + self.args

nemubot/module/__init__.py

@@ -0,0 +1,7 @@
+#
+# This directory aims to store nemubot core modules.
+#
+# Custom modules should be placed into a separate directory.
+# By default, this is the directory modules in your current directory.
+# Use the --modules-path argument to define a custom directory for your modules.
+#

nemubot/modulecontext.py

@@ -1,5 +1,5 @@
 # Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
+# Copyright (C) 2012-2017  Mercier Pierre-Olivier
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,105 +14,87 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-class ModuleContext:
+import asyncio
 
-    def __init__(self, context, module):
-        """Initialize the module context
 
-        arguments:
-        context -- the bot context
-        module -- the module
-        """
+class _TinyEvent:
+
+    def __init__(self, handle):
+        self.handle = handle
+
+
+class _FakeHandle:
+
+    def __init__(self, true_handle, callback):
+        self.handle = true_handle
+        self.callback = callback
+
+    def cancel(self):
+        self.handle.cancel()
+        if self.callback:
+            return self.callback()
+
+
+class _ModuleContext:
+
+    def __init__(self, module=None, knodes=None):
+        self.module = module
 
         if module is not None:
-            module_name = module.__spec__.name if hasattr(module, "__spec__") else module.__name__
+            self.module_name = (module.__spec__.name if hasattr(module, "__spec__") else module.__name__).replace("nemubot.module.", "")
         else:
-            module_name = ""
+            self.module_name = ""
 
-        # Load module configuration if exists
-        if (context is not None and
-            module_name in context.modules_configuration):
-            self.config = context.modules_configuration[module_name]
-        else:
-            from nemubot.config.module import Module
-            self.config = Module(module_name)
-
-        self.hooks = list()
         self.events = list()
-        self.debug = context.verbosity > 0 if context is not None else False
+        self.hooks = list()
+        self.debug = False
 
+        from nemubot.config.module import Module
+        self.config = Module(self.module_name)
+        self._knodes = knodes
+
+
+    def load_data(self):
+        from nemubot.tools.xmlparser import module_state
+        return module_state.ModuleState("nemubotstate")
+
+
+    def add_hook(self, hook, *triggers):
         from nemubot.hooks import Abstract as AbstractHook
+        assert isinstance(hook, AbstractHook), hook
+        self.hooks.append((triggers, hook))
 
-        # Define some callbacks
-        if context is not None:
-            def load_data():
-                return context.datastore.load(module_name)
+    def del_hook(self, hook, *triggers):
+        from nemubot.hooks import Abstract as AbstractHook
+        assert isinstance(hook, AbstractHook), hook
+        self.hooks.remove((triggers, hook))
 
-            def add_hook(hook, *triggers):
-                assert isinstance(hook, AbstractHook), hook
-                self.hooks.append((triggers, hook))
-                return context.treater.hm.add_hook(hook, *triggers)
 
-            def del_hook(hook, *triggers):
-                assert isinstance(hook, AbstractHook), hook
-                self.hooks.remove((triggers, hook))
-                return context.treater.hm.del_hooks(*triggers, hook=hook)
+    def subtreat(self, msg):
+        return None
 
-            def subtreat(msg):
-                yield from context.treater.treat_msg(msg)
-            def add_event(evt, eid=None):
-                return context.add_event(evt, eid, module_src=module)
-            def del_event(evt):
-                return context.del_event(evt, module_src=module)
 
-            def send_response(server, res):
-                if server in context.servers:
-                    if res.server is not None:
-                        return context.servers[res.server].send_response(res)
-                    else:
-                        return context.servers[server].send_response(res)
-                else:
-                    module.logger.error("Try to send a message to the unknown server: %s", server)
-                    return False
+    def set_knodes(self, knodes):
+        self._knodes = knodes
 
-        else:  # Used when using outside of nemubot
-            def load_data():
-                from nemubot.tools.xmlparser import module_state
-                return module_state.ModuleState("nemubotstate")
 
-            def add_hook(hook, *triggers):
-                assert isinstance(hook, AbstractHook), hook
-                self.hooks.append((triggers, hook))
-            def del_hook(hook, *triggers):
-                assert isinstance(hook, AbstractHook), hook
-                self.hooks.remove((triggers, hook))
-            def subtreat(msg):
-                return None
-            def add_event(evt, eid=None):
-                return context.add_event(evt, eid, module_src=module)
-            def del_event(evt):
-                return context.del_event(evt, module_src=module)
+    def add_event(self, evt):
+        self.events.append(evt)
+        return evt
 
-            def send_response(server, res):
-                module.logger.info("Send response: %s", res)
+    def del_event(self, evt):
+        return self.events.remove(evt)
 
-        def save():
-            context.datastore.save(module_name, self.data)
 
-        def subparse(orig, cnt):
-            if orig.server in context.servers:
-                return context.servers[orig.server].subparse(orig, cnt)
+    def send_response(self, server, res):
+       self.module.logger.info("Send response: %s", res)
 
-        self.load_data = load_data
-        self.add_hook = add_hook
-        self.del_hook = del_hook
-        self.add_event = add_event
-        self.del_event = del_event
-        self.save = save
-        self.send_response = send_response
-        self.subtreat = subtreat
-        self.subparse = subparse
+    def save(self):
+        self.context.datastore.save(self.module_name, self.data)
 
+    def subparse(self, orig, cnt):
+        if orig.server in self.context.servers:
+            return self.context.servers[orig.server].subparse(orig, cnt)
 
     @property
     def data(self):
@@ -120,6 +102,15 @@ class ModuleContext:
             self._data = self.load_data()
         return self._data
 
+    @data.setter
+    def data(self, data):
+        self._data = data
+        return self._data
+
+    @data.deleter
+    def data(self):
+        self._data = None
+
 
     def unload(self):
         """Perform actions for unloading the module"""
@@ -129,7 +120,93 @@ class ModuleContext:
             self.del_hook(h, *s)
 
         # Remove registered events
-        for e in self.events:
-            self.del_event(e)
+        for evt in self.events:
+            self.del_event(evt)
 
         self.save()
+
+
+class ModuleContext(_ModuleContext):
+
+    def __init__(self, context, *args, **kwargs):
+        """Initialize the module context
+
+        arguments:
+        context -- the bot context
+        module -- the module
+        """
+
+        super().__init__(*args, **kwargs)
+
+        # Load module configuration if exists
+        if self.module_name in context.modules_configuration:
+            self.config = context.modules_configuration[self.module_name]
+
+        self.context = context
+        self.debug = context.debug
+
+
+    def load_data(self):
+        return self.context.datastore.load(self.module_name, self._knodes)
+
+    def save(self):
+        self.context.datastore.save(self.module_name, self.data)
+
+
+    def add_hook(self, hook, *triggers):
+        from nemubot.hooks import Abstract as AbstractHook
+        assert isinstance(hook, AbstractHook), hook
+        self.hooks.append((triggers, hook))
+        return self.context.treater.hm.add_hook(hook, *triggers)
+
+    def del_hook(self, hook, *triggers):
+        from nemubot.hooks import Abstract as AbstractHook
+        assert isinstance(hook, AbstractHook), hook
+        self.hooks.remove((triggers, hook))
+        return self.context.treater.hm.del_hooks(*triggers, hook=hook)
+
+
+    def subtreat(self, msg):
+        yield from self.context.treater.treat_msg(msg)
+
+
+    def _add_event(self, evt, call_add, *args, **kwargs):
+        if evt in self.events:
+            return None
+
+        def _cancel_event():
+            self.module.logger.debug("Cancel event")
+            evt.handle = None
+            return super(ModuleContext, self).del_event(evt)
+
+        hd = call_add(*args, **kwargs)
+        evt.handle = _FakeHandle(hd, _cancel_event)
+
+        return super().add_event(evt)
+
+
+    def add_event(self, evt):
+        return self._add_event(evt, self.context.add_event, evt)
+
+    def call_at(self, *args, **kwargs):
+        evt = _TinyEvent(None)
+        return self._add_event(evt, self.context.call_at, *args, **kwargs)
+
+    def call_later(self, *args, **kwargs):
+        evt = _TinyEvent(None)
+        return self._add_event(evt, self.context.call_later, *args, **kwargs)
+
+    def del_event(self, evt):
+        # Call to super().del_event is done in the _FakeHandle.cancel
+        return evt.handle.cancel()
+
+
+    def send_response(self, server, res):
+        if server in self.context.servers:
+            if res.server is not None:
+                return self.context.servers[res.server].send_response(res)
+            else:
+                return self.context.servers[server].send_response(res)
+        else:
+            self.module.logger.error("Try to send a message to the unknown server: %s", server)
+            return False

nemubot/server/IRC.py

@@ -16,6 +16,7 @@
 
 from datetime import datetime
 import re
+import socket
 
 from nemubot.channel import Channel
 from nemubot.message.printer.IRC import IRC as IRCPrinter
@@ -173,10 +174,10 @@ class _IRC:
 
             for chname in msg.params[0].split(b","):
                 if chname in self.channels:
-                    if msg.nick == self.nick:
+                    if msg.frm == self.nick:
                         del self.channels[chname]
-                    elif msg.nick in self.channels[chname].people:
-                        del self.channels[chname].people[msg.nick]
+                    elif msg.frm in self.channels[chname].people:
+                        del self.channels[chname].people[msg.frm]
         self.hookscmd["PART"] = _on_part
         # Respond to 331/RPL_NOTOPIC,332/RPL_TOPIC,TOPIC
         def _on_topic(msg):
@@ -226,7 +227,7 @@ class _IRC:
             else:
                 res = "ERRMSG Unknown or unimplemented CTCP request"
             if res is not None:
-                self.write("NOTICE %s :\x01%s\x01" % (msg.nick, res))
+                self.write("NOTICE %s :\x01%s\x01" % (msg.frm, res))
         self.hookscmd["PRIVMSG"] = _on_ctcp
 
 
@@ -240,7 +241,7 @@ class _IRC:
         if self.capabilities is not None:
             self.write("CAP LS")
         self.write("NICK :" + self.nick)
-        self.write("USER %s %s bla :%s" % (self.username, self.host, self.realname))
+        self.write("USER %s %s bla :%s" % (self.username, socket.getfqdn(), self.realname))
 
 
     def close(self):

nemubot/server/abstract.py

@@ -16,6 +16,7 @@
 
 import logging
 import queue
+import traceback
 
 from nemubot.bot import sync_act
 
@@ -84,7 +85,7 @@ class AbstractServer:
         """
 
         self._sending_queue.put(self.format(message))
-        self.logger.debug("Message '%s' appended to write queue", message)
+        self.logger.debug("Message '%s' appended to write queue coming from %s:%d in %s", message, *traceback.extract_stack(limit=3)[0][:3])
         sync_act("sckt", "write", self.fileno())
 
 

nemubot/server/message/IRC.py

@@ -150,7 +150,8 @@ class IRC(Abstract):
                 "date": self.tags["time"],
                 "to": receivers,
                 "to_response": [r if r != srv.nick else self.nick for r in receivers],
-                "frm": self.nick
+                "frm": self.nick,
+                "frm_owner": self.nick == srv.owner
             }
 
             # If CTCP, remove 0x01

nemubot/server/socket.py

@@ -81,24 +81,17 @@ class _Socket(AbstractServer):
 class _SocketServer(_Socket):
 
     def __init__(self, host, port, bind=None, **kwargs):
-        super().__init__(family=socket.AF_INET, **kwargs)
+        (family, type, proto, canonname, sockaddr) = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)[0]
 
-        assert(host is not None)
-        assert(isinstance(port, int))
+        super().__init__(family=family, type=type, proto=proto, **kwargs)
 
-        self._host = host
-        self._port = port
+        self._sockaddr = sockaddr
         self._bind = bind
 
 
-    @property
-    def host(self):
-        return self._host
-
-
     def connect(self):
-        self.logger.info("Connection to %s:%d", self._host, self._port)
-        super().connect((self._host, self._port))
+        self.logger.info("Connection to %s:%d", *self._sockaddr[:2])
+        super().connect(self._sockaddr)
 
         if self._bind:
             super().bind(self._bind)
@@ -125,9 +118,15 @@ class UnixSocket:
         super().connect(self._socket_path)
 
 
+class SocketClient(_Socket, socket.socket):
+
+    def read(self):
+        return self.recv()
+
+
 class _Listener:
 
-    def __init__(self, new_server_cb, instanciate=_Socket, **kwargs):
+    def __init__(self, new_server_cb, instanciate=SocketClient, **kwargs):
         super().__init__(**kwargs)
 
         self._instanciate = instanciate

nemubot/server/test_IRC.py

@@ -12,7 +12,7 @@ class TestIRCMessage(unittest.TestCase):
 
     def test_parsing(self):
         self.assertEqual(self.msg.prefix, "toto!titi@RZ-3je16g.re")
-        self.assertEqual(self.msg.nick, "toto")
+        self.assertEqual(self.msg.frm, "toto")
         self.assertEqual(self.msg.user, "titi")
         self.assertEqual(self.msg.host, "RZ-3je16g.re")
 

nemubot/tools/test_xmlparser.py

@@ -1,5 +1,6 @@
 import unittest
 
+import io
 import xml.parsers.expat
 
 from nemubot.tools.xmlparser import XMLParser
@@ -12,6 +13,11 @@ class StringNode():
     def characters(self, content):
         self.string += content
 
+    def saveElement(self, store, tag="string"):
+        store.startElement(tag, {})
+        store.characters(self.string)
+        store.endElement(tag)
+
 
 class TestNode():
     def __init__(self, option=None):
@@ -22,6 +28,15 @@ class TestNode():
         self.mystr = child.string
         return True
 
+    def saveElement(self, store, tag="test"):
+        store.startElement(tag, {"option": self.option})
+
+        strNode = StringNode()
+        strNode.string = self.mystr
+        strNode.saveElement(store)
+
+        store.endElement(tag)
+
 
 class Test2Node():
     def __init__(self, option=None):
@@ -33,6 +48,15 @@ class Test2Node():
             self.mystrs.append(attrs["value"])
             return True
 
+    def saveElement(self, store, tag="test"):
+        store.startElement(tag, {"option": self.option} if self.option is not None else {})
+
+        for mystr in self.mystrs:
+            store.startElement("string", {"value": mystr})
+            store.endElement("string")
+
+        store.endElement(tag)
+
 
 class TestXMLParser(unittest.TestCase):
 
@@ -44,9 +68,11 @@ class TestXMLParser(unittest.TestCase):
         p.CharacterDataHandler = mod.characters
         p.EndElementHandler = mod.endElement
 
-        p.Parse("<string>toto</string>", 1)
+        inputstr = "<string>toto</string>"
+        p.Parse(inputstr, 1)
 
         self.assertEqual(mod.root.string, "toto")
+        self.assertEqual(mod.saveDocument(header=False).getvalue(), inputstr)
 
 
     def test_parser2(self):
@@ -57,10 +83,12 @@ class TestXMLParser(unittest.TestCase):
         p.CharacterDataHandler = mod.characters
         p.EndElementHandler = mod.endElement
 
-        p.Parse("<test option='123'><string>toto</string></test>", 1)
+        inputstr = '<test option="123"><string>toto</string></test>'
+        p.Parse(inputstr, 1)
 
         self.assertEqual(mod.root.option, "123")
         self.assertEqual(mod.root.mystr, "toto")
+        self.assertEqual(mod.saveDocument(header=False).getvalue(), inputstr)
 
 
     def test_parser3(self):
@@ -71,12 +99,14 @@ class TestXMLParser(unittest.TestCase):
         p.CharacterDataHandler = mod.characters
         p.EndElementHandler = mod.endElement
 
-        p.Parse("<test><string value='toto' /><string value='toto2' /></test>", 1)
+        inputstr = '<test><string value="toto"/><string value="toto2"/></test>'
+        p.Parse(inputstr, 1)
 
         self.assertEqual(mod.root.option, None)
         self.assertEqual(len(mod.root.mystrs), 2)
         self.assertEqual(mod.root.mystrs[0], "toto")
         self.assertEqual(mod.root.mystrs[1], "toto2")
+        self.assertEqual(mod.saveDocument(header=False, short_empty_elements=True).getvalue(), inputstr)
 
 
 if __name__ == '__main__':

nemubot/tools/web.py

@@ -14,7 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from urllib.parse import urlparse, urlsplit, urlunsplit
+from urllib.parse import urljoin, urlparse, urlsplit, urlunsplit
+import socket
 
 from nemubot.exception import IMException
 
@@ -67,13 +68,16 @@ def getPassword(url):
 
 # Get real pages
 
-def getURLContent(url, body=None, timeout=7, header=None):
+def getURLContent(url, body=None, timeout=7, header=None, decode_error=False,
+                  max_size=524288):
     """Return page content corresponding to URL or None if any error occurs
 
     Arguments:
     url -- the URL to get
     body -- Data to send as POST content
     timeout -- maximum number of seconds to wait before returning an exception
+    decode_error -- raise exception on non-200 pages or ignore it
+    max_size -- maximal size allow for the content
     """
 
     o = urlparse(_getNormalizedURL(url), "http")
@@ -108,6 +112,9 @@ def getURLContent(url, body=None, timeout=7, header=None):
     elif "User-agent" not in header:
         header["User-agent"] = "Nemubot v%s" % __version__
 
+    if body is not None and "Content-Type" not in header:
+        header["Content-Type"] = "application/x-www-form-urlencoded"
+
     import socket
     try:
         if o.query != '':
@@ -120,6 +127,8 @@ def getURLContent(url, body=None, timeout=7, header=None):
                          o.path,
                          body,
                          header)
+    except socket.timeout as e:
+        raise IMException(e)
     except OSError as e:
         raise IMException(e.strerror)
 
@@ -128,7 +137,7 @@ def getURLContent(url, body=None, timeout=7, header=None):
         size = int(res.getheader("Content-Length", 524288))
         cntype = res.getheader("Content-Type")
 
-        if size > 524288 or (cntype is not None and cntype[:4] != "text" and cntype[:4] != "appl"):
+        if size > max_size or (cntype is not None and cntype[:4] != "text" and cntype[:4] != "appl"):
             raise IMException("Content too large to be retrieved")
 
         data = res.read(size)
@@ -156,21 +165,27 @@ def getURLContent(url, body=None, timeout=7, header=None):
     elif ((res.status == http.client.FOUND or
            res.status == http.client.MOVED_PERMANENTLY) and
           res.getheader("Location") != url):
-        return getURLContent(res.getheader("Location"), timeout=timeout)
+        return getURLContent(
+            urljoin(url, res.getheader("Location")),
+            body=body,
+            timeout=timeout,
+            header=header,
+            decode_error=decode_error,
+            max_size=max_size)
+    elif decode_error:
+        return data.decode(charset).strip()
     else:
         raise IMException("A HTTP error occurs: %d - %s" %
                            (res.status, http.client.responses[res.status]))
 
 
-def getXML(url, timeout=7):
+def getXML(*args, **kwargs):
     """Get content page and return XML parsed content
 
-    Arguments:
-    url -- the URL to get
-    timeout -- maximum number of seconds to wait before returning an exception
+    Arguments: same as getURLContent
     """
 
-    cnt = getURLContent(url, timeout=timeout)
+    cnt = getURLContent(*args, **kwargs)
     if cnt is None:
         return None
     else:
@@ -178,19 +193,20 @@ def getXML(url, timeout=7):
         return parseString(cnt)
 
 
-def getJSON(url, timeout=7):
+def getJSON(*args, remove_callback=False, **kwargs):
     """Get content page and return JSON content
 
-    Arguments:
-    url -- the URL to get
-    timeout -- maximum number of seconds to wait before returning an exception
+    Arguments: same as getURLContent
     """
 
-    cnt = getURLContent(url, timeout=timeout)
+    cnt = getURLContent(*args, **kwargs)
     if cnt is None:
         return None
     else:
         import json
+        if remove_callback:
+            import re
+            cnt = re.sub(r"^[^(]+\((.*)\)$", r"\1", cnt)
         return json.loads(cnt)
 
 

nemubot/tools/xmlparser/__init__.py

@@ -134,6 +134,21 @@ class XMLParser:
                     return
             raise TypeError(name + " tag not expected in " + self.display_stack())
 
+    def saveDocument(self, f=None, header=True, short_empty_elements=False):
+        if f is None:
+            import io
+            f = io.StringIO()
+
+        import xml.sax.saxutils
+        gen = xml.sax.saxutils.XMLGenerator(f, "utf-8", short_empty_elements=short_empty_elements)
+        if header:
+            gen.startDocument()
+        self.root.saveElement(gen)
+        if header:
+            gen.endDocument()
+
+        return f
+
 
 def parse_file(filename):
     p = xml.parsers.expat.ParserCreate()

nemubot/tools/xmlparser/basic.py

@@ -44,6 +44,13 @@ class ListNode:
         return self.items.__repr__()
 
 
+    def saveElement(self, store, tag="list"):
+        store.startElement(tag, {})
+        for i in self.items:
+            i.saveElement(store)
+        store.endElement(tag)
+
+
 class DictNode:
 
     """XML node representing a Python dictionnnary
@@ -70,12 +77,12 @@ class DictNode:
 
 
     def endElement(self, name):
-        if name is None or self._cur is None:
+        if name is not None or self._cur is None:
             return
 
         key, cnt = self._cur
         if isinstance(cnt, list) and len(cnt) == 1:
-            self.items[key] = cnt
+            self.items[key] = cnt[0]
         else:
             self.items[key] = cnt
 
@@ -106,3 +113,41 @@ class DictNode:
 
     def __repr__(self):
         return self.items.__repr__()
+
+
+    def saveElement(self, store, tag="dict"):
+        store.startElement(tag, {})
+        for k, v in self.items.items():
+            store.startElement("item", {"key": k})
+            if isinstance(v, str):
+                store.characters(v)
+            else:
+                if hasattr(v, "__iter__"):
+                    for i in v:
+                        i.saveElement(store)
+                else:
+                    v.saveElement(store)
+            store.endElement("item")
+        store.endElement(tag)
+
+
+    def __contain__(self, i):
+        return i in self.items
+
+    def __getitem__(self, i):
+        return self.items[i]
+
+    def __setitem__(self, i, c):
+        self.items[i] = c
+
+    def __delitem__(self, k):
+        del self.items[k]
+
+    def __iter__(self):
+        return self.items.__iter__()
+
+    def keys(self):
+        return self.items.keys()
+
+    def items(self):
+        return self.items.items()

nemubot/tools/xmlparser/genericnode.py

@@ -53,6 +53,14 @@ class ParsingNode:
         return item in self.attrs
 
 
+    def saveElement(self, store, tag=None):
+        store.startElement(tag if tag is not None else self.tag, self.attrs)
+        for child in self.children:
+            child.saveElement(store)
+        store.characters(self.content)
+        store.endElement(tag if tag is not None else self.tag)
+
+
 class GenericNode(ParsingNode):
 
     """Consider all subtags as dictionnary

nemubot/tools/xmlparser/node.py

@@ -196,7 +196,7 @@ class ModuleState:
         if self.index_fieldname is not None:
             self.setIndex(self.index_fieldname, self.index_tagname)
 
-    def save_node(self, gen):
+    def saveElement(self, gen):
         """Serialize this node as a XML node"""
         from datetime import datetime
         attribs = {}
@@ -215,29 +215,9 @@ class ModuleState:
             gen.startElement(self.name, attrs)
 
             for child in self.childs:
-                child.save_node(gen)
+                child.saveElement(gen)
 
             gen.endElement(self.name)
         except:
             logger.exception("Error occured when saving the following "
                              "XML node: %s with %s", self.name, attrs)
-
-    def save(self, filename):
-        """Save the current node as root node in a XML file
-
-        Argument:
-        filename -- location of the file to create/erase
-        """
-
-        import tempfile
-        _, tmpath = tempfile.mkstemp()
-        with open(tmpath, "w") as f:
-            import xml.sax.saxutils
-            gen = xml.sax.saxutils.XMLGenerator(f, "utf-8")
-            gen.startDocument()
-            self.save_node(gen)
-            gen.endDocument()
-
-        # Atomic save
-        import shutil
-        shutil.move(tmpath, filename)

nemubot/treatment.py

@@ -109,6 +109,9 @@ class MessageTreater:
         msg -- message to treat
         """
 
+        if hasattr(msg, "frm_owner"):
+            msg.frm_owner = (not hasattr(msg.server, "owner") or msg.server.owner == msg.frm)
+
         while hook is not None:
             res = hook.run(msg)