bot: Fix sequential message processing with proper consumer pool

Replace the flawed cnsr_thrd_size threshold with cnsr_active, which
tracks the number of consumers currently executing a task. A new
consumer thread is now spawned the moment the queue is non-empty and
all existing consumers are busy, enabling true parallel execution of
slow and fast commands. The pool is capped at os.cpu_count() threads.

- bot.py: replace cnsr_thrd_size with cnsr_active + cnsr_lock + cnsr_max
- consumer.py: increment/decrement cnsr_active around stm.run(), remove
  itself from cnsr_thrd under the lock, mark thread as daemon

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.drone.yml

@@ -0,0 +1,26 @@
+---
+kind: pipeline
+type: docker
+name: default-arm64
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+  - name: build
+    image: python:3.11-alpine
+    commands:
+      - pip install --no-cache-dir -r requirements.txt
+      - pip install .
+
+  - name: docker
+    image: plugins/docker
+    settings:
+      repo: nemunaire/nemubot
+      auto_tag: true
+      auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH}
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "modules/nextstop/external"]
-	path = modules/nextstop/external
-	url = git://github.com/nbr23/NextStop.git

.travis.yml

@@ -1,8 +1,9 @@
 language: python
 python:
-  - 3.3
   - 3.4
   - 3.5
+  - 3.6
+  - 3.7
   - nightly
 install:
   - pip install -r requirements.txt

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.11-alpine
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt /usr/src/app/
+RUN apk add --no-cache bash build-base capstone-dev mandoc-doc man-db w3m youtube-dl aspell aspell-fr py3-matrix-nio && \
+    pip install --no-cache-dir --ignore-installed -r requirements.txt && \
+    pip install bs4 capstone dnspython openai && \
+    apk del build-base capstone-dev && \
+    ln -s /var/lib/nemubot/home /home/nemubot
+
+VOLUME /var/lib/nemubot
+
+COPY . /usr/src/app/
+
+RUN ./setup.py install
+
+WORKDIR /var/lib/nemubot
+USER guest
+ENTRYPOINT [ "python", "-m", "nemubot", "-d", "-P", "", "-M", "/usr/src/app/modules" ]
+CMD [ "-D", "/var/lib/nemubot" ]

README.md

@@ -10,7 +10,7 @@ Requirements
 *nemubot* requires at least Python 3.3 to work.
 
 Some modules (like `cve`, `nextstop` or `laposte`) require the
-[BeautifulSoup module](http://www.crummy.com/software/BeautifulSoup/),
+[BeautifulSoup module](https://www.crummy.com/software/BeautifulSoup/),
 but the core and framework has no dependency.
 
 

modules/alias.py

@@ -12,7 +12,7 @@ from nemubot.message import Command
 from nemubot.tools.human import guess
 from nemubot.tools.xmlparser.node import ModuleState
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################
@@ -76,7 +76,7 @@ def get_variable(name, msg=None):
     elif name in context.data.getNode("variables").index:
         return context.data.getNode("variables").index[name]["value"]
     else:
-        return ""
+        return None
 
 
 def list_variables(user=None):
@@ -108,12 +108,12 @@ def set_variable(name, value, creator):
     context.save()
 
 
-def replace_variables(cnts, msg=None):
+def replace_variables(cnts, msg):
     """Replace variables contained in the content
 
     Arguments:
     cnt -- content where search variables
-    msg -- optional message where pick some variables
+    msg -- Message where pick some variables
     """
 
     unsetCnt = list()
@@ -122,12 +122,12 @@ def replace_variables(cnts, msg=None):
     resultCnt = list()
 
     for cnt in cnts:
-        for res in re.findall("\\$\{(?P<name>[a-zA-Z0-9:]+)\}", cnt):
-            rv = re.match("([0-9]+)(:([0-9]*))?", res)
+        for res, name, default in re.findall("\\$\{(([a-zA-Z0-9:]+)(?:-([^}]+))?)\}", cnt):
+            rv = re.match("([0-9]+)(:([0-9]*))?", name)
             if rv is not None:
                 varI = int(rv.group(1)) - 1
-                if varI > len(msg.args):
-                    cnt = cnt.replace("${%s}" % res, "", 1)
+                if varI >= len(msg.args):
+                    cnt = cnt.replace("${%s}" % res, default, 1)
                 elif rv.group(2) is not None:
                     if rv.group(3) is not None and len(rv.group(3)):
                         varJ = int(rv.group(3)) - 1
@@ -142,9 +142,10 @@ def replace_variables(cnts, msg=None):
                     cnt = cnt.replace("${%s}" % res, msg.args[varI], 1)
                     unsetCnt.append(varI)
             else:
-                cnt = cnt.replace("${%s}" % res, get_variable(res), 1)
+                cnt = cnt.replace("${%s}" % res, get_variable(name) or default, 1)
         resultCnt.append(cnt)
 
+    # Remove used content
     for u in sorted(set(unsetCnt), reverse=True):
         msg.args.pop(u)
 
@@ -184,7 +185,7 @@ def cmd_listvars(msg):
 def cmd_set(msg):
     if len(msg.args) < 2:
         raise IMException("!set take two args: the key and the value.")
-    set_variable(msg.args[0], " ".join(msg.args[1:]), msg.nick)
+    set_variable(msg.args[0], " ".join(msg.args[1:]), msg.frm)
     return Response("Variable $%s successfully defined." % msg.args[0],
                     channel=msg.channel)
 
@@ -221,13 +222,13 @@ def cmd_alias(msg):
 
     if alias.cmd in context.data.getNode("aliases").index:
         return Response("%s corresponds to %s" % (alias.cmd, context.data.getNode("aliases").index[alias.cmd]["origin"]),
-                        channel=msg.channel, nick=msg.nick)
+                        channel=msg.channel, nick=msg.frm)
 
     elif len(msg.args) > 1:
         create_alias(alias.cmd,
                      " ".join(msg.args[1:]),
                      channel=msg.channel,
-                     creator=msg.nick)
+                     creator=msg.frm)
         return Response("New alias %s successfully registered." % alias.cmd,
                         channel=msg.channel)
 
@@ -259,19 +260,18 @@ def cmd_unalias(msg):
 
 @hook.add(["pre","Command"])
 def treat_alias(msg):
-    if msg.cmd in context.data.getNode("aliases").index:
+    if context.data.getNode("aliases") is not None and msg.cmd in context.data.getNode("aliases").index:
         origin = context.data.getNode("aliases").index[msg.cmd]["origin"]
-        rpl_cmd = context.subparse(msg, origin)
-        if isinstance(rpl_cmd, Command):
-            rpl_cmd.args = replace_variables(rpl_cmd.args, msg)
-            rpl_cmd.args += msg.args
-            rpl_cmd.kwargs.update(msg.kwargs)
+        rpl_msg = context.subparse(msg, origin)
+        if isinstance(rpl_msg, Command):
+            rpl_msg.args = replace_variables(rpl_msg.args, msg)
+            rpl_msg.args += msg.args
+            rpl_msg.kwargs.update(msg.kwargs)
         elif len(msg.args) or len(msg.kwargs):
             raise IMException("This kind of alias doesn't take any argument (haven't you forgotten the '!'?).")
 
         # Avoid infinite recursion
-        if msg.cmd != rpl_cmd.cmd:
-            # Also return origin message, if it can be treated as well
-            return [msg, rpl_cmd]
+        if not isinstance(rpl_msg, Command) or msg.cmd != rpl_msg.cmd:
+            return rpl_msg
 
     return msg

modules/birthday.py

@@ -13,7 +13,7 @@ from nemubot.tools.countdown import countdown_format
 from nemubot.tools.date import extractDate
 from nemubot.tools.xmlparser.node import ModuleState
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################
@@ -27,7 +27,7 @@ def load(context):
 def findName(msg):
     if (not len(msg.args) or msg.args[0].lower() == "moi" or
         msg.args[0].lower() == "me"):
-        name = msg.nick.lower()
+        name = msg.frm.lower()
     else:
         name = msg.args[0].lower()
 
@@ -77,7 +77,7 @@ def cmd_anniv(msg):
     else:
         return Response("désolé, je ne connais pas la date d'anniversaire"
                         " de %s. Quand est-il né ?" % name,
-                        msg.channel, msg.nick)
+                        msg.channel, msg.frm)
 
 
 @hook.command("age",
@@ -98,7 +98,7 @@ def cmd_age(msg):
                         msg.channel)
     else:
         return Response("désolé, je ne connais pas l'âge de %s."
-                        " Quand est-il né ?" % name, msg.channel, msg.nick)
+                        " Quand est-il né ?" % name, msg.channel, msg.frm)
     return True
 
 
@@ -106,18 +106,18 @@ def cmd_age(msg):
 
 @hook.ask()
 def parseask(msg):
-    res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)?\s+(birthday|geburtstag|née? |nee? le|born on).*$", msg.text, re.I)
+    res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)?\s+(birthday|geburtstag|née? |nee? le|born on).*$", msg.message, re.I)
     if res is not None:
         try:
-            extDate = extractDate(msg.text)
+            extDate = extractDate(msg.message)
             if extDate is None or extDate.year > datetime.now().year:
                 return Response("la date de naissance ne paraît pas valide...",
                                 msg.channel,
-                                msg.nick)
+                                msg.frm)
             else:
                 nick = res.group(1)
                 if nick == "my" or nick == "I" or nick == "i" or nick == "je" or nick == "mon" or nick == "ma":
-                    nick = msg.nick
+                    nick = msg.frm
                 if nick.lower() in context.data.index:
                     context.data.index[nick.lower()]["born"] = extDate
                 else:
@@ -129,6 +129,6 @@ def parseask(msg):
                 return Response("ok, c'est noté, %s est né le %s"
                                 % (nick, extDate.strftime("%A %d %B %Y à %H:%M")),
                                 msg.channel,
-                                msg.nick)
+                                msg.frm)
         except:
             raise IMException("la date de naissance ne paraît pas valide.")

modules/bonneannee.py

@@ -4,12 +4,11 @@
 
 from datetime import datetime, timezone
 
-from nemubot import context
 from nemubot.event import ModuleEvent
 from nemubot.hooks import hook
 from nemubot.tools.countdown import countdown_format
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # GLOBALS #############################################################

modules/books.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################

modules/cat.py

@@ -7,7 +7,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.message import Command, DirectAsk, Text
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MODULE CORE #########################################################

modules/cmd_server.py

@@ -1,202 +0,0 @@
-# Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import traceback
-import sys
-
-from nemubot.hooks import hook
-
-nemubotversion = 3.4
-NODATA = True
-
-
-def getserver(toks, context, prompt, mandatory=False, **kwargs):
-    """Choose the server in toks or prompt.
-    This function modify the tokens list passed as argument"""
-
-    if len(toks) > 1 and toks[1] in context.servers:
-        return context.servers[toks.pop(1)]
-    elif not mandatory or prompt.selectedServer:
-        return prompt.selectedServer
-    else:
-        from nemubot.prompt.error import PromptError
-        raise PromptError("Please SELECT a server or give its name in argument.")
-
-
-@hook("prompt_cmd", "close")
-def close(toks, context, **kwargs):
-    """Disconnect and forget (remove from the servers list) the server"""
-    srv = getserver(toks, context=context, mandatory=True, **kwargs)
-
-    if srv.close():
-        del context.servers[srv.id]
-        return 0
-    return 1
-
-
-@hook("prompt_cmd", "connect")
-def connect(toks, **kwargs):
-    """Make the connexion to a server"""
-    srv = getserver(toks, mandatory=True, **kwargs)
-
-    return not srv.open()
-
-
-@hook("prompt_cmd", "disconnect")
-def disconnect(toks, **kwargs):
-    """Close the connection to a server"""
-    srv = getserver(toks, mandatory=True, **kwargs)
-
-    return not srv.close()
-
-
-@hook("prompt_cmd", "discover")
-def discover(toks, context, **kwargs):
-    """Discover a new bot on a server"""
-    srv = getserver(toks, context=context, mandatory=True, **kwargs)
-
-    if len(toks) > 1 and "!" in toks[1]:
-        bot = context.add_networkbot(srv, name)
-        return not bot.connect()
-    else:
-        print("  %s is not a valid fullname, for example: "
-              "nemubot!nemubotV3@bot.nemunai.re" % ''.join(toks[1:1]))
-        return 1
-
-
-@hook("prompt_cmd", "join")
-@hook("prompt_cmd", "leave")
-@hook("prompt_cmd", "part")
-def join(toks, **kwargs):
-    """Join or leave a channel"""
-    srv = getserver(toks, mandatory=True, **kwargs)
-
-    if len(toks) <= 2:
-        print("%s: not enough arguments." % toks[0])
-        return 1
-
-    if toks[0] == "join":
-        if len(toks) > 2:
-            srv.write("JOIN %s %s" % (toks[1], toks[2]))
-        else:
-            srv.write("JOIN %s" % toks[1])
-
-    elif toks[0] == "leave" or toks[0] == "part":
-        if len(toks) > 2:
-            srv.write("PART %s :%s" % (toks[1], " ".join(toks[2:])))
-        else:
-            srv.write("PART %s" % toks[1])
-
-    return 0
-
-
-@hook("prompt_cmd", "save")
-def save_mod(toks, context, **kwargs):
-    """Force save module data"""
-    if len(toks) < 2:
-        print("save: not enough arguments.")
-        return 1
-
-    wrn = 0
-    for mod in toks[1:]:
-        if mod in context.modules:
-            context.modules[mod].save()
-            print("save: module `%s´ saved successfully" % mod)
-        else:
-            wrn += 1
-            print("save: no module named `%s´" % mod)
-    return wrn
-
-
-@hook("prompt_cmd", "send")
-def send(toks, **kwargs):
-    """Send a message on a channel"""
-    srv = getserver(toks, mandatory=True, **kwargs)
-
-    # Check the server is connected
-    if not srv.connected:
-        print ("send: server `%s' not connected." % srv.id)
-        return 2
-
-    if len(toks) <= 3:
-        print ("send: not enough arguments.")
-        return 1
-
-    if toks[1] not in srv.channels:
-        print ("send: channel `%s' not authorized in server `%s'."
-               % (toks[1], srv.id))
-        return 3
-
-    from nemubot.message import Text
-    srv.send_response(Text(" ".join(toks[2:]), server=None,
-                           to=[toks[1]]))
-    return 0
-
-
-@hook("prompt_cmd", "zap")
-def zap(toks, **kwargs):
-    """Hard change connexion state"""
-    srv = getserver(toks, mandatory=True, **kwargs)
-
-    srv.connected = not srv.connected
-
-
-@hook("prompt_cmd", "top")
-def top(toks, context, **kwargs):
-    """Display consumers load information"""
-    print("Queue size: %d, %d thread(s) running (counter: %d)" %
-          (context.cnsr_queue.qsize(),
-           len(context.cnsr_thrd),
-           context.cnsr_thrd_size))
-    if len(context.events) > 0:
-        print("Events registered: %d, next in %d seconds" %
-              (len(context.events),
-               context.events[0].time_left.seconds))
-    else:
-        print("No events registered")
-
-    for th in context.cnsr_thrd:
-        if th.is_alive():
-            print(("#" * 15 + " Stack trace for thread %u " + "#" * 15) %
-                  th.ident)
-            traceback.print_stack(sys._current_frames()[th.ident])
-
-
-@hook("prompt_cmd", "netstat")
-def netstat(toks, context, **kwargs):
-    """Display sockets in use and many other things"""
-    if len(context.network) > 0:
-        print("Distant bots connected: %d:" % len(context.network))
-        for name, bot in context.network.items():
-            print("# %s:" % name)
-            print("  * Declared hooks:")
-            lvl = 0
-            for hlvl in bot.hooks:
-                lvl += 1
-                for hook in (hlvl.all_pre + hlvl.all_post + hlvl.cmd_rgxp +
-                             hlvl.cmd_default + hlvl.ask_rgxp +
-                             hlvl.ask_default + hlvl.msg_rgxp +
-                             hlvl.msg_default):
-                    print("  %s- %s" % (' ' * lvl * 2, hook))
-                for kind in ["irc_hook", "cmd_hook", "ask_hook", "msg_hook"]:
-                    print("  %s- <%s> %s" % (' ' * lvl * 2, kind,
-                                             ", ".join(hlvl.__dict__[kind].keys())))
-            print("  * My tag: %d" % bot.my_tag)
-            print("  * Tags in use (%d):" % bot.inc_tag)
-            for tag, (cmd, data) in bot.tags.items():
-                print("    - %11s: %s « %s »" % (tag, cmd, data))
-    else:
-        print("No distant bot connected")

modules/conjugaison.py

@@ -11,7 +11,7 @@ from nemubot.hooks import hook
 from nemubot.tools import web
 from nemubot.tools.web import striphtml
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # GLOBALS #############################################################
@@ -36,7 +36,7 @@ for k, v in s:
 # MODULE CORE #########################################################
 
 def get_conjug(verb, stringTens):
-    url = ("http://leconjugueur.lefigaro.fr/conjugaison/verbe/%s.html" %
+    url = ("https://leconjugueur.lefigaro.fr/conjugaison/verbe/%s.html" %
            quote(verb.encode("ISO-8859-1")))
     page = web.getURLContent(url)
 

modules/ctfs.py

@@ -6,7 +6,7 @@ from bs4 import BeautifulSoup
 
 from nemubot.hooks import hook
 from nemubot.tools.web import getURLContent, striphtml
-from more import Response
+from nemubot.module.more import Response
 
 
 # GLOBALS #############################################################
@@ -25,10 +25,8 @@ def get_info_yt(msg):
 
     for line in soup.body.find_all('tr'):
         n = line.find_all('td')
-        if len(n) == 5:
-            try:
-                res.append_message("\x02%s:\x0F from %s type %s at %s. %s" %
-                                   tuple([striphtml(x.text) for x in n]))
-            except:
-                pass
+        if len(n) == 7:
+            res.append_message("\x02%s:\x0F from %s type %s at %s. Weight: %s. %s%s" %
+                               tuple([striphtml(x.text).strip() for x in n]))
+
     return res

modules/cve.py

@@ -5,29 +5,67 @@
 from bs4 import BeautifulSoup
 from urllib.parse import quote
 
+from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools.web import getURLContent, striphtml
 
-from more import Response
+from nemubot.module.more import Response
 
-BASEURL_NIST = 'https://web.nvd.nist.gov/view/vuln/detail?vulnId='
+BASEURL_NIST = 'https://nvd.nist.gov/vuln/detail/'
 
 
 # MODULE CORE #########################################################
 
+VULN_DATAS = {
+    "alert-title": "vuln-warning-status-name",
+    "alert-content": "vuln-warning-banner-content",
+
+    "description": "vuln-description",
+    "published": "vuln-published-on",
+    "last_modified": "vuln-last-modified-on",
+
+    "base_score": "vuln-cvssv3-base-score-link",
+    "severity": "vuln-cvssv3-base-score-severity",
+    "impact_score": "vuln-cvssv3-impact-score",
+    "exploitability_score": "vuln-cvssv3-exploitability-score",
+
+    "av": "vuln-cvssv3-av",
+    "ac": "vuln-cvssv3-ac",
+    "pr": "vuln-cvssv3-pr",
+    "ui": "vuln-cvssv3-ui",
+    "s": "vuln-cvssv3-s",
+    "c": "vuln-cvssv3-c",
+    "i": "vuln-cvssv3-i",
+    "a": "vuln-cvssv3-a",
+}
+
+
 def get_cve(cve_id):
     search_url = BASEURL_NIST + quote(cve_id.upper())
 
     soup = BeautifulSoup(getURLContent(search_url))
-    vuln = soup.body.find(class_="vuln-detail")
-    cvss = vuln.findAll('div')[4]
 
-    return [
-        "Base score: " + cvss.findAll('div')[0].findAll('a')[0].text.strip(),
-        vuln.findAll('p')[0].text, # description
-        striphtml(vuln.findAll('div')[0].text).strip(), # publication date
-        striphtml(vuln.findAll('div')[1].text).strip(), # last revised
-    ]
+    vuln = {}
+
+    for vd in VULN_DATAS:
+        r = soup.body.find(attrs={"data-testid": VULN_DATAS[vd]})
+        if r:
+            vuln[vd] = r.text.strip()
+
+    return vuln
+
+
+def display_metrics(av, ac, pr, ui, s, c, i, a, **kwargs):
+    ret = []
+    if av != "None": ret.append("Attack Vector: \x02%s\x0F" % av)
+    if ac != "None": ret.append("Attack Complexity: \x02%s\x0F" % ac)
+    if pr != "None": ret.append("Privileges Required: \x02%s\x0F" % pr)
+    if ui != "None": ret.append("User Interaction: \x02%s\x0F" % ui)
+    if s != "Unchanged": ret.append("Scope: \x02%s\x0F" % s)
+    if c != "None": ret.append("Confidentiality: \x02%s\x0F" % c)
+    if i != "None": ret.append("Integrity: \x02%s\x0F" % i)
+    if a != "None": ret.append("Availability: \x02%s\x0F" % a)
+    return ', '.join(ret)
 
 
 # MODULE INTERFACE ####################################################
@@ -42,6 +80,20 @@ def get_cve_desc(msg):
         if cve_id[:3].lower() != 'cve':
             cve_id = 'cve-' + cve_id
 
-        res.append_message(get_cve(cve_id))
+        cve = get_cve(cve_id)
+        if not cve:
+            raise IMException("CVE %s doesn't exists." % cve_id)
+
+        if "alert-title" in cve or "alert-content" in cve:
+            alert = "\x02%s:\x0F %s " % (cve["alert-title"] if "alert-title" in cve else "",
+                                         cve["alert-content"] if "alert-content" in cve else "")
+        else:
+            alert = ""
+
+        if "base_score" not in cve and "description" in cve:
+            res.append_message("{alert}Last modified on \x02{last_modified}\x0F. {description}".format(alert=alert, **cve), title=cve_id)
+        else:
+            metrics = display_metrics(**cve)
+            res.append_message("{alert}Base score: \x02{base_score} {severity}\x0F (impact: \x02{impact_score}\x0F, exploitability: \x02{exploitability_score}\x0F; {metrics}), last modified on \x02{last_modified}\x0F. {description}".format(alert=alert, metrics=metrics, **cve), title=cve_id)
 
     return res

modules/ddg.py

@@ -8,7 +8,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 # MODULE CORE #########################################################
 

modules/dig.py

@@ -0,0 +1,94 @@
+"""DNS resolver"""
+
+# PYTHON STUFFS #######################################################
+
+import ipaddress
+import socket
+
+import dns.exception
+import dns.name
+import dns.rdataclass
+import dns.rdatatype
+import dns.resolver
+
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+
+from nemubot.module.more import Response
+
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("dig",
+              help="Resolve domain name with a basic syntax similar to dig(1)")
+def dig(msg):
+    lclass = "IN"
+    ltype = "A"
+    ledns = None
+    ltimeout = 6.0
+    ldomain = None
+    lnameservers = []
+    lsearchlist = []
+    loptions = []
+    for a in msg.args:
+        if a in dns.rdatatype._by_text:
+            ltype = a
+        elif a in dns.rdataclass._by_text:
+            lclass = a
+        elif a[0] == "@":
+            try:
+                lnameservers.append(str(ipaddress.ip_address(a[1:])))
+            except ValueError:
+                for r in socket.getaddrinfo(a[1:], 53, proto=socket.IPPROTO_UDP):
+                    lnameservers.append(r[4][0])
+
+        elif a[0:8] == "+domain=":
+            lsearchlist.append(dns.name.from_unicode(a[8:]))
+        elif a[0:6] == "+edns=":
+            ledns = int(a[6:])
+        elif a[0:6] == "+time=":
+            ltimeout = float(a[6:])
+        elif a[0] == "+":
+            loptions.append(a[1:])
+        else:
+            ldomain = a
+
+    if not ldomain:
+        raise IMException("indicate a domain to resolve")
+
+    resolv = dns.resolver.Resolver()
+    if ledns:
+        resolv.edns = ledns
+    resolv.lifetime = ltimeout
+    resolv.timeout = ltimeout
+    resolv.flags = (
+        dns.flags.QR | dns.flags.RA |
+        dns.flags.AA if "aaonly" in loptions or "aaflag" in loptions else 0 |
+        dns.flags.AD if "adflag" in loptions else 0 |
+        dns.flags.CD if "cdflag" in loptions else 0 |
+        dns.flags.RD if "norecurse" not in loptions else 0
+    )
+    if lsearchlist:
+        resolv.search = lsearchlist
+    else:
+        resolv.search = [dns.name.from_text(".")]
+
+    if lnameservers:
+        resolv.nameservers = lnameservers
+
+    try:
+        answers = resolv.query(ldomain, ltype, lclass, tcp="tcp" in loptions)
+    except dns.exception.DNSException as e:
+        raise IMException(str(e))
+
+    res = Response(channel=msg.channel, count=" (%s others entries)")
+    for rdata in answers:
+        res.append_message("%s %s %s %s %s" % (
+            answers.qname.to_text(),
+            answers.ttl if not "nottlid" in loptions else "",
+            dns.rdataclass.to_text(answers.rdclass) if not "nocl" in loptions else "",
+            dns.rdatatype.to_text(answers.rdtype),
+            rdata.to_text())
+        )
+
+    return res

modules/disas.py

@@ -0,0 +1,89 @@
+"""The Ultimate Disassembler Module"""
+
+# PYTHON STUFFS #######################################################
+
+import capstone
+
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+
+from nemubot.module.more import Response
+
+
+# MODULE CORE #########################################################
+
+ARCHITECTURES = {
+    "arm": capstone.CS_ARCH_ARM,
+    "arm64": capstone.CS_ARCH_ARM64,
+    "mips": capstone.CS_ARCH_MIPS,
+    "ppc": capstone.CS_ARCH_PPC,
+    "sparc": capstone.CS_ARCH_SPARC,
+    "sysz": capstone.CS_ARCH_SYSZ,
+    "x86": capstone.CS_ARCH_X86,
+    "xcore": capstone.CS_ARCH_XCORE,
+}
+
+MODES = {
+    "arm": capstone.CS_MODE_ARM,
+    "thumb": capstone.CS_MODE_THUMB,
+    "mips32": capstone.CS_MODE_MIPS32,
+    "mips64": capstone.CS_MODE_MIPS64,
+    "mips32r6": capstone.CS_MODE_MIPS32R6,
+    "16": capstone.CS_MODE_16,
+    "32": capstone.CS_MODE_32,
+    "64": capstone.CS_MODE_64,
+    "le": capstone.CS_MODE_LITTLE_ENDIAN,
+    "be": capstone.CS_MODE_BIG_ENDIAN,
+    "micro": capstone.CS_MODE_MICRO,
+    "mclass": capstone.CS_MODE_MCLASS,
+    "v8": capstone.CS_MODE_V8,
+    "v9": capstone.CS_MODE_V9,
+}
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("disas",
+              help="Display assembly code",
+              help_usage={"CODE": "Display assembly code corresponding to the given CODE"},
+              keywords={
+                  "arch=ARCH": "Specify the architecture of the code to disassemble (default: x86, choose between: %s)" % ', '.join(ARCHITECTURES.keys()),
+                  "modes=MODE[,MODE]": "Specify hardware mode of the code to disassemble (default: 32, between: %s)" % ', '.join(MODES.keys()),
+              })
+def cmd_disas(msg):
+    if not len(msg.args):
+        raise IMException("please give me some code")
+
+    # Determine the architecture
+    if "arch" in msg.kwargs:
+        if msg.kwargs["arch"] not in ARCHITECTURES:
+            raise IMException("unknown architectures '%s'" % msg.kwargs["arch"])
+        architecture = ARCHITECTURES[msg.kwargs["arch"]]
+    else:
+        architecture = capstone.CS_ARCH_X86
+
+    # Determine hardware modes
+    modes = 0
+    if "modes" in msg.kwargs:
+        for mode in msg.kwargs["modes"].split(','):
+            if mode not in MODES:
+                raise IMException("unknown mode '%s'" % mode)
+        modes += MODES[mode]
+    elif architecture == capstone.CS_ARCH_X86 or architecture == capstone.CS_ARCH_PPC:
+        modes = capstone.CS_MODE_32
+    elif architecture == capstone.CS_ARCH_ARM or architecture == capstone.CS_ARCH_ARM64:
+        modes = capstone.CS_MODE_ARM
+    elif architecture == capstone.CS_ARCH_MIPS:
+        modes = capstone.CS_MODE_MIPS32
+
+    # Get the code
+    code = bytearray.fromhex(''.join([a.replace("0x", "") for a in msg.args]))
+
+    # Setup capstone
+    md = capstone.Cs(architecture, modes)
+
+    res = Response(channel=msg.channel, nomore="No more instruction")
+
+    for isn in md.disasm(code, 0x1000):
+        res.append_message("%s %s" %(isn.mnemonic, isn.op_str), title="0x%x" % isn.address)
+
+    return res

modules/events.py

@@ -1,7 +1,9 @@
 """Create countdowns and reminders"""
 
-import re
+import calendar
 from datetime import datetime, timedelta, timezone
+from functools import partial
+import re
 
 from nemubot import context
 from nemubot.exception import IMException
@@ -10,31 +12,84 @@ from nemubot.hooks import hook
 from nemubot.message import Command
 from nemubot.tools.countdown import countdown_format, countdown
 from nemubot.tools.date import extractDate
-from nemubot.tools.xmlparser.node import ModuleState
+from nemubot.tools.xmlparser.basic import DictNode
 
-from more import Response
+from nemubot.module.more import Response
+
+
+class Event:
+
+    def __init__(self, server, channel, creator, start_time, end_time=None):
+        self._server = server
+        self._channel = channel
+        self._creator = creator
+        self._start = datetime.utcfromtimestamp(float(start_time)).replace(tzinfo=timezone.utc) if not isinstance(start_time, datetime) else start_time
+        self._end = datetime.utcfromtimestamp(float(end_time)).replace(tzinfo=timezone.utc) if end_time else None
+        self._evt = None
+
+
+    def __del__(self):
+        if self._evt is not None:
+            context.del_event(self._evt)
+            self._evt = None
+
+
+    def saveElement(self, store, tag="event"):
+        attrs = {
+            "server": str(self._server),
+            "channel": str(self._channel),
+            "creator": str(self._creator),
+            "start_time": str(calendar.timegm(self._start.timetuple())),
+        }
+        if self._end:
+            attrs["end_time"] = str(calendar.timegm(self._end.timetuple()))
+        store.startElement(tag, attrs)
+        store.endElement(tag)
+
+    @property
+    def creator(self):
+        return self._creator
+
+    @property
+    def start(self):
+        return self._start
+
+    @property
+    def end(self):
+        return self._end
+
+    @end.setter
+    def end(self, c):
+        self._end = c
+
+    @end.deleter
+    def end(self):
+        self._end = None
 
 
 def help_full ():
-    return "This module store a lot of events: ny, we, " + (", ".join(context.datas.index.keys())) + "\n!eventslist: gets list of timer\n!start /something/: launch a timer"
+    return "This module store a lot of events: ny, we, " + (", ".join(context.datas.keys()) if hasattr(context, "datas") else "") + "\n!eventslist: gets list of timer\n!start /something/: launch a timer"
 
 
 def load(context):
-    #Define the index
-    context.data.setIndex("name")
+    context.set_knodes({
+        "dict": DictNode,
+        "event": Event,
+    })
 
-    for evt in context.data.index.keys():
-        if context.data.index[evt].hasAttribute("end"):
-            event = ModuleEvent(call=fini, call_data=dict(strend=context.data.index[evt]))
-            event._end = context.data.index[evt].getDate("end")
-            idt = context.add_event(event)
-            if idt is not None:
-                context.data.index[evt]["_id"] = idt
+    if context.data is None:
+        context.set_default(DictNode())
+
+    # Relaunch all timers
+    for kevt in context.data:
+        if context.data[kevt].end:
+            context.data[kevt]._evt = context.add_event(ModuleEvent(partial(fini, kevt, context.data[kevt]), offset=context.data[kevt].end - datetime.now(timezone.utc), interval=0))
 
 
-def fini(d, strend):
-    context.send_response(strend["server"], Response("%s arrivé à échéance." % strend["name"], channel=strend["channel"], nick=strend["proprio"]))
-    context.data.delChild(context.data.index[strend["name"]])
+def fini(name, evt):
+    context.send_response(evt._server, Response("%s arrivé à échéance." % name, channel=evt._channel, nick=evt.creator))
+    evt._evt = None
+    del context.data[name]
     context.save()
 
 
@@ -63,18 +118,10 @@ def start_countdown(msg):
     """!start /something/: launch a timer"""
     if len(msg.args) < 1:
         raise IMException("indique le nom d'un événement à chronométrer")
-    if msg.args[0] in context.data.index:
+    if msg.args[0] in context.data:
         raise IMException("%s existe déjà." % msg.args[0])
 
-    strnd = ModuleState("strend")
-    strnd["server"] = msg.server
-    strnd["channel"] = msg.channel
-    strnd["proprio"] = msg.nick
-    strnd["start"] = msg.date
-    strnd["name"] = msg.args[0]
-    context.data.addChild(strnd)
-
-    evt = ModuleEvent(call=fini, call_data=dict(strend=strnd))
+    evt = Event(server=msg.server, channel=msg.channel, creator=msg.frm, start_time=msg.date)
 
     if len(msg.args) > 1:
         result1 = re.findall("([0-9]+)([smhdjwyaSMHDJWYA])?", msg.args[1])
@@ -92,50 +139,51 @@ def start_countdown(msg):
                 if result2 is None or result2.group(4) is None: yea = now.year
                 else: yea = int(result2.group(4))
                 if result2 is not None and result3 is not None:
-                    strnd["end"] = datetime(yea, int(result2.group(3)), int(result2.group(2)), hou, minu, sec, timezone.utc)
+                    evt.end = datetime(yea, int(result2.group(3)), int(result2.group(2)), hou, minu, sec, timezone.utc)
                 elif result2 is not None:
-                    strnd["end"] = datetime(int(result2.group(4)), int(result2.group(3)), int(result2.group(2)), 0, 0, 0, timezone.utc)
+                    evt.end = datetime(int(result2.group(4)), int(result2.group(3)), int(result2.group(2)), 0, 0, 0, timezone.utc)
                 elif result3 is not None:
                   if hou * 3600 + minu * 60 + sec > now.hour * 3600 + now.minute * 60 + now.second:
-                    strnd["end"] = datetime(now.year, now.month, now.day, hou, minu, sec, timezone.utc)
+                    evt.end = datetime(now.year, now.month, now.day, hou, minu, sec, timezone.utc)
                   else:
-                    strnd["end"] = datetime(now.year, now.month, now.day + 1, hou, minu, sec, timezone.utc)
-                evt._end = strnd.getDate("end")
-                strnd["_id"] = context.add_event(evt)
+                    evt.end = datetime(now.year, now.month, now.day + 1, hou, minu, sec, timezone.utc)
             except:
-                context.data.delChild(strnd)
                 raise IMException("Mauvais format de date pour l'événement %s. Il n'a pas été créé." % msg.args[0])
 
         elif result1 is not None and len(result1) > 0:
-            strnd["end"] = msg.date
+            evt.end = msg.date
             for (t, g) in result1:
                 if g is None or g == "" or g == "m" or g == "M":
-                    strnd["end"] += timedelta(minutes=int(t))
+                    evt.end += timedelta(minutes=int(t))
                 elif g == "h" or g == "H":
-                    strnd["end"] += timedelta(hours=int(t))
+                    evt.end += timedelta(hours=int(t))
                 elif g == "d" or g == "D" or g == "j" or g == "J":
-                    strnd["end"] += timedelta(days=int(t))
+                    evt.end += timedelta(days=int(t))
                 elif g == "w" or g == "W":
-                    strnd["end"] += timedelta(days=int(t)*7)
+                    evt.end += timedelta(days=int(t)*7)
                 elif g == "y" or g == "Y" or g == "a" or g == "A":
-                    strnd["end"] += timedelta(days=int(t)*365)
+                    evt.end += timedelta(days=int(t)*365)
                 else:
-                    strnd["end"] += timedelta(seconds=int(t))
-            evt._end = strnd.getDate("end")
-            eid = context.add_event(evt)
-            if eid is not None:
-                strnd["_id"] = eid
+                    evt.end += timedelta(seconds=int(t))
 
+        else:
+            raise IMException("Mauvais format de date pour l'événement %s. Il n'a pas été créé." % msg.args[0])
+
+    context.data[msg.args[0]] = evt
     context.save()
-    if "end" in strnd:
+
+    if evt.end is not None:
+        context.add_event(ModuleEvent(partial(fini, msg.args[0], evt),
+                                      offset=evt.end - datetime.now(timezone.utc),
+                                      interval=0))
         return Response("%s commencé le %s et se terminera le %s." %
                         (msg.args[0], msg.date.strftime("%A %d %B %Y à %H:%M:%S"),
-                         strnd.getDate("end").strftime("%A %d %B %Y à %H:%M:%S")),
-                        nick=msg.frm)
+                         evt.end.strftime("%A %d %B %Y à %H:%M:%S")),
+                        channel=msg.channel)
     else:
         return Response("%s commencé le %s"% (msg.args[0],
                             msg.date.strftime("%A %d %B %Y à %H:%M:%S")),
-                        nick=msg.frm)
+                        channel=msg.channel)
 
 
 @hook.command("end")
@@ -144,67 +192,66 @@ def end_countdown(msg):
     if len(msg.args) < 1:
         raise IMException("quel événement terminer ?")
 
-    if msg.args[0] in context.data.index:
-        if context.data.index[msg.args[0]]["proprio"] == msg.nick or (msg.cmd == "forceend" and msg.frm_owner):
-            duration = countdown(msg.date - context.data.index[msg.args[0]].getDate("start"))
-            context.del_event(context.data.index[msg.args[0]]["_id"])
-            context.data.delChild(context.data.index[msg.args[0]])
+    if msg.args[0] in context.data:
+        if context.data[msg.args[0]].creator == msg.frm or (msg.cmd == "forceend" and msg.frm_owner):
+            duration = countdown(msg.date - context.data[msg.args[0]].start)
+            del context.data[msg.args[0]]
             context.save()
             return Response("%s a duré %s." % (msg.args[0], duration),
-                            channel=msg.channel, nick=msg.nick)
+                            channel=msg.channel, nick=msg.frm)
         else:
-            raise IMException("Vous ne pouvez pas terminer le compteur %s, créé par %s." % (msg.args[0], context.data.index[msg.args[0]]["proprio"]))
+            raise IMException("Vous ne pouvez pas terminer le compteur %s, créé par %s." % (msg.args[0], context.data[msg.args[0]].creator))
     else:
-        return Response("%s n'est pas un compteur connu."% (msg.args[0]), channel=msg.channel, nick=msg.nick)
+        return Response("%s n'est pas un compteur connu."% (msg.args[0]), channel=msg.channel, nick=msg.frm)
 
 
 @hook.command("eventslist")
 def liste(msg):
     """!eventslist: gets list of timer"""
     if len(msg.args):
-        res = list()
+        res = Response(channel=msg.channel)
         for user in msg.args:
-            cmptr = [x["name"] for x in context.data.index.values() if x["proprio"] == user]
+            cmptr = [k for k in context.data if context.data[k].creator == user]
             if len(cmptr) > 0:
-                res.append("Compteurs créés par %s : %s" % (user, ", ".join(cmptr)))
+                res.append_message(cmptr, title="Events created by %s" % user)
             else:
-                res.append("%s n'a pas créé de compteur" % user)
-        return Response(" ; ".join(res), channel=msg.channel)
+                res.append_message("%s doesn't have any counting events" % user)
+        return res
     else:
-        return Response("Compteurs connus : %s." % ", ".join(context.data.index.keys()), channel=msg.channel)
+        return Response(list(context.data.keys()), channel=msg.channel, title="Known events")
 
 
-@hook.command(match=lambda msg: isinstance(msg, Command) and msg.cmd in context.data.index)
+@hook.command(match=lambda msg: isinstance(msg, Command) and msg.cmd in context.data)
 def parseanswer(msg):
     res = Response(channel=msg.channel)
 
     # Avoid message starting by ! which can be interpreted as command by other bots
     if msg.cmd[0] == "!":
-        res.nick = msg.nick
+        res.nick = msg.frm
 
-    if context.data.index[msg.cmd].name == "strend":
-        if context.data.index[msg.cmd].hasAttribute("end"):
-            res.append_message("%s commencé il y a %s et se terminera dans %s." % (msg.cmd, countdown(msg.date - context.data.index[msg.cmd].getDate("start")), countdown(context.data.index[msg.cmd].getDate("end") - msg.date)))
+    if msg.cmd in context.data:
+        if context.data[msg.cmd].end:
+            res.append_message("%s commencé il y a %s et se terminera dans %s." % (msg.cmd, countdown(msg.date - context.data[msg.cmd].start), countdown(context.data[msg.cmd].end - msg.date)))
         else:
-            res.append_message("%s commencé il y a %s." % (msg.cmd, countdown(msg.date - context.data.index[msg.cmd].getDate("start"))))
+            res.append_message("%s commencé il y a %s." % (msg.cmd, countdown(msg.date - context.data[msg.cmd].start)))
     else:
-        res.append_message(countdown_format(context.data.index[msg.cmd].getDate("start"), context.data.index[msg.cmd]["msg_before"], context.data.index[msg.cmd]["msg_after"]))
+        res.append_message(countdown_format(context.data[msg.cmd].start, context.data[msg.cmd]["msg_before"], context.data[msg.cmd]["msg_after"]))
     return res
 
 
 RGXP_ask = re.compile(r"^.*((create|new)\s+(a|an|a\s*new|an\s*other)?\s*(events?|commande?)|(nouvel(le)?|ajoute|cr[ée]{1,3})\s+(un)?\s*([eé]v[ée]nements?|commande?)).*$", re.I)
 
-@hook.ask(match=lambda msg: RGXP_ask.match(msg.text))
+@hook.ask(match=lambda msg: RGXP_ask.match(msg.message))
 def parseask(msg):
-    name = re.match("^.*!([^ \"'@!]+).*$", msg.text)
+    name = re.match("^.*!([^ \"'@!]+).*$", msg.message)
     if name is None:
         raise IMException("il faut que tu attribues une commande à l'événement.")
-    if name.group(1) in context.data.index:
+    if name.group(1) in context.data:
         raise IMException("un événement portant ce nom existe déjà.")
 
-    texts = re.match("^[^\"]*(avant|après|apres|before|after)?[^\"]*\"([^\"]+)\"[^\"]*((avant|après|apres|before|after)?.*\"([^\"]+)\".*)?$", msg.text, re.I)
+    texts = re.match("^[^\"]*(avant|après|apres|before|after)?[^\"]*\"([^\"]+)\"[^\"]*((avant|après|apres|before|after)?.*\"([^\"]+)\".*)?$", msg.message, re.I)
     if texts is not None and texts.group(3) is not None:
-        extDate = extractDate(msg.text)
+        extDate = extractDate(msg.message)
         if extDate is None or extDate == "":
             raise IMException("la date de l'événement est invalide !")
 
@@ -223,7 +270,7 @@ def parseask(msg):
         evt = ModuleState("event")
         evt["server"] = msg.server
         evt["channel"] = msg.channel
-        evt["proprio"] = msg.nick
+        evt["proprio"] = msg.frm
         evt["name"] = name.group(1)
         evt["start"] = extDate
         evt["msg_after"] = msg_after
@@ -237,7 +284,7 @@ def parseask(msg):
         evt = ModuleState("event")
         evt["server"] = msg.server
         evt["channel"] = msg.channel
-        evt["proprio"] = msg.nick
+        evt["proprio"] = msg.frm
         evt["name"] = name.group(1)
         evt["msg_before"] = texts.group (2)
         context.data.addChild(evt)

modules/freetarifs.py

@@ -0,0 +1,64 @@
+"""Inform about Free Mobile tarifs"""
+
+# PYTHON STUFFS #######################################################
+
+import urllib.parse
+from bs4 import BeautifulSoup
+
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.tools import web
+
+from nemubot.module.more import Response
+
+
+# MODULE CORE #########################################################
+
+ACT = {
+    "ff_toFixe": "Appel vers les fixes",
+    "ff_toMobile": "Appel vers les mobiles",
+    "ff_smsSendedToCountry": "SMS vers le pays",
+    "ff_mmsSendedToCountry": "MMS vers le pays",
+    "fc_callToFrance": "Appel vers la France",
+    "fc_smsToFrance": "SMS vers la france",
+    "fc_mmsSended": "MMS vers la france",
+    "fc_callToSameCountry": "Réception des appels",
+    "fc_callReceived": "Appel dans le pays",
+    "fc_smsReceived": "SMS (Réception)",
+    "fc_mmsReceived": "MMS (Réception)",
+    "fc_moDataFromCountry": "Data",
+}
+
+def get_land_tarif(country, forfait="pkgFREE"):
+    url = "http://mobile.international.free.fr/?" + urllib.parse.urlencode({'pays': country})
+    page = web.getURLContent(url)
+    soup = BeautifulSoup(page)
+
+    fact = soup.find(class_=forfait)
+
+    if fact is None:
+        raise IMException("Country or forfait not found.")
+
+    res = {}
+    for s in ACT.keys():
+        try:
+            res[s] = fact.find(attrs={"data-bind": "text: " + s}).text + " " + fact.find(attrs={"data-bind": "html: " + s + "Unit"}).text
+        except AttributeError:
+            res[s] = "inclus"
+
+    return res
+
+@hook.command("freetarifs",
+              help="Show Free Mobile tarifs for given contries",
+              help_usage={"COUNTRY": "Show Free Mobile tarifs for given CONTRY"},
+              keywords={
+                  "forfait=FORFAIT": "Related forfait between Free (default) and 2euro"
+              })
+def get_freetarif(msg):
+    res = Response(channel=msg.channel)
+
+    for country in msg.args:
+        t = get_land_tarif(country.lower().capitalize(), "pkg" + (msg.kwargs["forfait"] if "forfait" in msg.kwargs else "FREE").upper())
+        res.append_message(["\x02%s\x0F : %s" % (ACT[k], t[k]) for k in sorted(ACT.keys(), reverse=True)], title=country)
+
+    return res

modules/github.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MODULE CORE #########################################################

modules/grep.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.message import Command, Text
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MODULE CORE #########################################################
@@ -73,7 +73,7 @@ def cmd_grep(msg):
 
     only = "only" in msg.kwargs
 
-    l = [m for m in grep(msg.args[0] if msg.args[0][0] == "^" else ".*?(" + msg.args[0] + ").*?",
+    l = [m for m in grep(msg.args[0] if len(msg.args[0]) and msg.args[0][0] == "^" else ".*?(" + msg.args[0] + ").*?",
                          " ".join(msg.args[1:]),
                          msg,
                          icase="nocase" in msg.kwargs,

modules/imdb.py

@@ -5,63 +5,56 @@
 import re
 import urllib.parse
 
+from bs4 import BeautifulSoup
+
 from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MODULE CORE #########################################################
 
-def get_movie(title=None, year=None, imdbid=None, fullplot=True, tomatoes=False):
+def get_movie_by_id(imdbid):
     """Returns the information about the matching movie"""
 
-    # Built URL
-    url = "http://www.omdbapi.com/?"
-    if title is not None:
-        url += "t=%s&" % urllib.parse.quote(title)
-    if year is not None:
-        url += "y=%s&" % urllib.parse.quote(year)
-    if imdbid is not None:
-        url += "i=%s&" % urllib.parse.quote(imdbid)
-    if fullplot:
-        url += "plot=full&"
-    if tomatoes:
-        url += "tomatoes=true&"
+    url = "http://www.imdb.com/title/" + urllib.parse.quote(imdbid)
+    soup = BeautifulSoup(web.getURLContent(url))
 
-    # Make the request
-    data = web.getJSON(url)
+    return {
+        "imdbID": imdbid,
+        "Title": soup.body.find('h1').contents[0].strip(),
+        "Year": soup.body.find(id="titleYear").find("a").text.strip() if soup.body.find(id="titleYear") else ", ".join([y.text.strip() for y in soup.body.find(attrs={"class": "seasons-and-year-nav"}).find_all("a")[1:]]),
+        "Duration": soup.body.find(attrs={"class": "title_wrapper"}).find("time").text.strip() if soup.body.find(attrs={"class": "title_wrapper"}).find("time") else None,
+        "imdbRating": soup.body.find(attrs={"class": "ratingValue"}).find("strong").text.strip() if soup.body.find(attrs={"class": "ratingValue"}) else None,
+        "imdbVotes": soup.body.find(attrs={"class": "imdbRating"}).find("a").text.strip() if soup.body.find(attrs={"class": "imdbRating"}) else None,
+        "Plot": re.sub(r"\s+", " ", soup.body.find(attrs={"class": "summary_text"}).text).strip(),
 
-    # Return data
-    if "Error" in data:
-        raise IMException(data["Error"])
-
-    elif "Response" in data and data["Response"] == "True":
-        return data
-
-    else:
-        raise IMException("An error occurs during movie search")
+        "Type": "TV Series" if soup.find(id="title-episode-widget") else "Movie",
+        "Genre": ", ".join([x.text.strip() for x in soup.body.find(id="titleStoryLine").find_all("a") if x.get("href") is not None and x.get("href")[:21] == "/search/title?genres="]),
+        "Country": ", ".join([x.text.strip() for x in soup.body.find(id="titleDetails").find_all("a") if x.get("href") is not None and x.get("href")[:32] == "/search/title?country_of_origin="]),
+        "Credits": " ; ".join([x.find("h4").text.strip() + " " + (", ".join([y.text.strip() for y in x.find_all("a") if y.get("href") is not None and y.get("href")[:6] == "/name/"])) for x in soup.body.find_all(attrs={"class": "credit_summary_item"})]),
+    }
 
 
-def find_movies(title):
+def find_movies(title, year=None):
     """Find existing movies matching a approximate title"""
 
+    title = title.lower()
+
     # Built URL
-    url = "http://www.omdbapi.com/?s=%s" % urllib.parse.quote(title)
+    url = "https://v2.sg.media-imdb.com/suggests/%s/%s.json" % (urllib.parse.quote(title[0]), urllib.parse.quote(title.replace(" ", "_")))
 
     # Make the request
-    data = web.getJSON(url)
-
-    # Return data
-    if "Error" in data:
-        raise IMException(data["Error"])
-
-    elif "Search" in data:
-        return data
+    data = web.getJSON(url, remove_callback=True)
 
+    if "d" not in data:
+        return None
+    elif year is None:
+        return data["d"]
     else:
-        raise IMException("An error occurs during movie search")
+        return [d for d in data["d"] if "y" in d and str(d["y"]) == year]
 
 
 # MODULE INTERFACE ####################################################
@@ -79,23 +72,28 @@ def cmd_imdb(msg):
     title = ' '.join(msg.args)
 
     if re.match("^tt[0-9]{7}$", title) is not None:
-        data = get_movie(imdbid=title)
+        data = get_movie_by_id(imdbid=title)
     else:
         rm = re.match(r"^(.+)\s\(([0-9]{4})\)$", title)
         if rm is not None:
-            data = get_movie(title=rm.group(1), year=rm.group(2))
+            data = find_movies(rm.group(1), year=rm.group(2))
         else:
-            data = get_movie(title=title)
+            data = find_movies(title)
+
+        if not data:
+            raise IMException("Movie/series not found")
+
+        data = get_movie_by_id(data[0]["id"])
 
     res = Response(channel=msg.channel,
                    title="%s (%s)" % (data['Title'], data['Year']),
                    nomore="No more information, more at http://www.imdb.com/title/%s" % data['imdbID'])
 
-    res.append_message("\x02rating\x0F: %s (%s votes); \x02plot\x0F: %s" %
-                       (data['imdbRating'], data['imdbVotes'], data['Plot']))
+    res.append_message("%s \x02genre:\x0F %s; \x02rating\x0F: %s (%s votes); \x02plot\x0F: %s" %
+                       (data['Type'], data['Genre'], data['imdbRating'], data['imdbVotes'], data['Plot']))
+    res.append_message("%s \x02from\x0F %s; %s"
+                       % (data['Type'], data['Country'], data['Credits']))
 
-    res.append_message("%s \x02from\x0F %s \x02released on\x0F %s; \x02genre:\x0F %s; \x02directed by:\x0F %s; \x02written by:\x0F %s; \x02main actors:\x0F %s"
-                       % (data['Type'], data['Country'], data['Released'], data['Genre'], data['Director'], data['Writer'], data['Actors']))
     return res
 
 
@@ -111,7 +109,7 @@ def cmd_search(msg):
     data = find_movies(' '.join(msg.args))
 
     movies = list()
-    for m in data['Search']:
-        movies.append("\x02%s\x0F (%s of %s)" % (m['Title'], m['Type'], m['Year']))
+    for m in data:
+        movies.append("\x02%s\x0F%s with %s" % (m['l'], (" (" + str(m['y']) + ")") if "y" in m else "", m['s']))
 
     return Response(movies, title="Titles found", channel=msg.channel)

modules/jsonbot.py

@@ -1,7 +1,7 @@
 from nemubot.hooks import hook
 from nemubot.exception import IMException
 from nemubot.tools import web
-from more import Response
+from nemubot.module.more import Response
 import json
 
 nemubotversion = 3.4

modules/man.py

@@ -8,7 +8,7 @@ import os
 
 from nemubot.hooks import hook
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # GLOBALS #############################################################

modules/mapquest.py

@@ -9,11 +9,11 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 # GLOBALS #############################################################
 
-URL_API = "http://open.mapquestapi.com/geocoding/v1/address?key=%s&location=%%s"
+URL_API = "https://open.mapquestapi.com/geocoding/v1/address?key=%s&location=%%s"
 
 
 # LOADING #############################################################
@@ -23,7 +23,7 @@ def load(context):
         raise ImportError("You need a MapQuest API key in order to use this "
                           "module. Add it to the module configuration file:\n"
                           "<module name=\"mapquest\" key=\"XXXXXXXXXXXXXXXX\" "
-                          "/>\nRegister at http://developer.mapquest.com/")
+                          "/>\nRegister at https://developer.mapquest.com/")
     global URL_API
     URL_API = URL_API % context.config["apikey"].replace("%", "%%")
 
@@ -55,7 +55,7 @@ def cmd_geocode(msg):
     if not len(msg.args):
         raise IMException("indicate a name")
 
-    res = Response(channel=msg.channel, nick=msg.nick,
+    res = Response(channel=msg.channel, nick=msg.frm,
                    nomore="No more geocode", count=" (%s more geocode)")
 
     for loc in geocode(' '.join(msg.args)):

modules/mediawiki.py

@@ -11,15 +11,15 @@ from nemubot.tools import web
 
 nemubotversion = 3.4
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MEDIAWIKI REQUESTS ##################################################
 
-def get_namespaces(site, ssl=False):
+def get_namespaces(site, ssl=False, path="/w/api.php"):
     # Built URL
-    url = "http%s://%s/w/api.php?format=json&action=query&meta=siteinfo&siprop=namespaces" % (
-        "s" if ssl else "", site)
+    url = "http%s://%s%s?format=json&action=query&meta=siteinfo&siprop=namespaces" % (
+        "s" if ssl else "", site, path)
 
     # Make the request
     data = web.getJSON(url)
@@ -30,10 +30,10 @@ def get_namespaces(site, ssl=False):
     return namespaces
 
 
-def get_raw_page(site, term, ssl=False):
+def get_raw_page(site, term, ssl=False, path="/w/api.php"):
     # Built URL
-    url = "http%s://%s/w/api.php?format=json&redirects&action=query&prop=revisions&rvprop=content&titles=%s" % (
-        "s" if ssl else "", site, urllib.parse.quote(term))
+    url = "http%s://%s%s?format=json&redirects&action=query&prop=revisions&rvprop=content&titles=%s" % (
+        "s" if ssl else "", site, path, urllib.parse.quote(term))
 
     # Make the request
     data = web.getJSON(url)
@@ -45,10 +45,10 @@ def get_raw_page(site, term, ssl=False):
             raise IMException("article not found")
 
 
-def get_unwikitextified(site, wikitext, ssl=False):
+def get_unwikitextified(site, wikitext, ssl=False, path="/w/api.php"):
     # Built URL
-    url = "http%s://%s/w/api.php?format=json&action=expandtemplates&text=%s" % (
-        "s" if ssl else "", site, urllib.parse.quote(wikitext))
+    url = "http%s://%s%s?format=json&action=expandtemplates&text=%s" % (
+        "s" if ssl else "", site, path, urllib.parse.quote(wikitext))
 
     # Make the request
     data = web.getJSON(url)
@@ -58,10 +58,10 @@ def get_unwikitextified(site, wikitext, ssl=False):
 
 ## Search
 
-def opensearch(site, term, ssl=False):
+def opensearch(site, term, ssl=False, path="/w/api.php"):
     # Built URL
-    url = "http%s://%s/w/api.php?format=json&action=opensearch&search=%s" % (
-        "s" if ssl else "", site, urllib.parse.quote(term))
+    url = "http%s://%s%s?format=json&action=opensearch&search=%s" % (
+        "s" if ssl else "", site, path, urllib.parse.quote(term))
 
     # Make the request
     response = web.getJSON(url)
@@ -73,10 +73,10 @@ def opensearch(site, term, ssl=False):
                    response[3][k])
 
 
-def search(site, term, ssl=False):
+def search(site, term, ssl=False, path="/w/api.php"):
     # Built URL
-    url = "http%s://%s/w/api.php?format=json&action=query&list=search&srsearch=%s&srprop=titlesnippet|snippet" % (
-        "s" if ssl else "", site, urllib.parse.quote(term))
+    url = "http%s://%s%s?format=json&action=query&list=search&srsearch=%s&srprop=titlesnippet|snippet" % (
+        "s" if ssl else "", site, path, urllib.parse.quote(term))
 
     # Make the request
     data = web.getJSON(url)
@@ -89,6 +89,11 @@ def search(site, term, ssl=False):
 
 # PARSING FUNCTIONS ###################################################
 
+def get_model(cnt, model="Infobox"):
+    for full in re.findall(r"(\{\{" + model + " .*?(?:\{\{.*?}}.*?)*}})", cnt, flags=re.DOTALL):
+        return full[3 + len(model):-2].replace("\n", " ").strip()
+
+
 def strip_model(cnt):
     # Strip models at begin: mostly useless
     cnt = re.sub(r"^(({{([^{]|\s|({{([^{]|\s|{{.*?}})*?}})*?)*?}}|\[\[([^[]|\s|\[\[.*?\]\])*?\]\])\s*)+", "", cnt, flags=re.DOTALL)
@@ -108,9 +113,9 @@ def strip_model(cnt):
     return cnt
 
 
-def parse_wikitext(site, cnt, namespaces=dict(), ssl=False):
+def parse_wikitext(site, cnt, namespaces=dict(), **kwargs):
     for i, _, _, _ in re.findall(r"({{([^{]|\s|({{(.|\s|{{.*?}})*?}})*?)*?}})", cnt):
-        cnt = cnt.replace(i, get_unwikitextified(site, i, ssl), 1)
+        cnt = cnt.replace(i, get_unwikitextified(site, i, **kwargs), 1)
 
     # Strip [[...]]
     for full, args, lnk in re.findall(r"(\[\[(.*?|)?([^|]*?)\]\])", cnt):
@@ -139,67 +144,101 @@ def irc_format(cnt):
     return cnt.replace("'''", "\x03\x02").replace("''", "\x03\x1f")
 
 
-def get_page(site, term, ssl=False, subpart=None):
-    raw = get_raw_page(site, term, ssl)
+def parse_infobox(cnt):
+    for v in cnt.split("|"):
+        try:
+            yield re.sub(r"^\s*([^=]*[^=\s])\s*=\s*(.+)\s*$", "\x03\x02" + r"\1" + ":\x03\x02 " + r"\2", v).replace("<br />", ", ").replace("<br/>", ", ").strip()
+        except:
+            yield re.sub(r"^\s+(.+)\s+$", "\x03\x02" + r"\1" + "\x03\x02", v).replace("<br />", ", ").replace("<br/>", ", ").strip()
+
+
+def get_page(site, term, subpart=None, **kwargs):
+    raw = get_raw_page(site, term, **kwargs)
 
     if subpart is not None:
         subpart = subpart.replace("_", " ")
         raw = re.sub(r"^.*(?P<title>==+)\s*(" + subpart + r")\s*(?P=title)", r"\1 \2 \1", raw, flags=re.DOTALL)
 
-    return strip_model(raw)
+    return raw
 
 
 # NEMUBOT #############################################################
 
-def mediawiki_response(site, term, to):
-    ns = get_namespaces(site)
+def mediawiki_response(site, term, to, **kwargs):
+    ns = get_namespaces(site, **kwargs)
 
     terms = term.split("#", 1)
 
     try:
         # Print the article if it exists
-        return Response(get_page(site, terms[0], subpart=terms[1] if len(terms) > 1 else None),
-                        line_treat=lambda line: irc_format(parse_wikitext(site, line, ns)),
+        return Response(strip_model(get_page(site, terms[0], subpart=terms[1] if len(terms) > 1 else None, **kwargs)),
+                        line_treat=lambda line: irc_format(parse_wikitext(site, line, ns, **kwargs)),
                         channel=to)
     except:
         pass
 
     # Try looking at opensearch
-    os = [x for x, _, _ in opensearch(site, terms[0])]
+    os = [x for x, _, _ in opensearch(site, terms[0], **kwargs)]
     print(os)
     # Fallback to global search
     if not len(os):
-        os = [x for x, _ in search(site, terms[0]) if x is not None and x != ""]
+        os = [x for x, _ in search(site, terms[0], **kwargs) if x is not None and x != ""]
     return Response(os,
                     channel=to,
                     title="Article not found, would you mean")
 
 
-@hook.command("mediawiki")
+@hook.command("mediawiki",
+              help="Read an article on a MediaWiki",
+              keywords={
+                  "ssl": "query over https instead of http",
+                  "path=PATH": "absolute path to the API",
+              })
 def cmd_mediawiki(msg):
-    """Read an article on a MediaWiki"""
     if len(msg.args) < 2:
         raise IMException("indicate a domain and a term to search")
 
     return mediawiki_response(msg.args[0],
                               " ".join(msg.args[1:]),
-                              msg.to_response)
+                              msg.to_response,
+                              **msg.kwargs)
 
 
-@hook.command("search_mediawiki")
+@hook.command("mediawiki_search",
+              help="Search an article on a MediaWiki",
+              keywords={
+                  "ssl": "query over https instead of http",
+                  "path=PATH": "absolute path to the API",
+              })
 def cmd_srchmediawiki(msg):
-    """Search an article on a MediaWiki"""
     if len(msg.args) < 2:
         raise IMException("indicate a domain and a term to search")
 
     res = Response(channel=msg.to_response, nomore="No more results", count=" (%d more results)")
 
-    for r in search(msg.args[0], " ".join(msg.args[1:])):
+    for r in search(msg.args[0], " ".join(msg.args[1:]), **msg.kwargs):
         res.append_message("%s: %s" % r)
 
     return res
 
 
+@hook.command("mediawiki_infobox",
+              help="Highlight information from an article on a MediaWiki",
+              keywords={
+                  "ssl": "query over https instead of http",
+                  "path=PATH": "absolute path to the API",
+              })
+def cmd_infobox(msg):
+    if len(msg.args) < 2:
+        raise IMException("indicate a domain and a term to search")
+
+    ns = get_namespaces(msg.args[0], **msg.kwargs)
+
+    return Response(", ".join([x for x in parse_infobox(get_model(get_page(msg.args[0], " ".join(msg.args[1:]), **msg.kwargs), "Infobox"))]),
+                    line_treat=lambda line: irc_format(parse_wikitext(msg.args[0], line, ns, **msg.kwargs)),
+                    channel=msg.to_response)
+
+
 @hook.command("wikipedia")
 def cmd_wikipedia(msg):
     if len(msg.args) < 2:

modules/networking/__init__.py

@@ -8,7 +8,7 @@ import re
 from nemubot.exception import IMException
 from nemubot.hooks import hook
 
-from more import Response
+from nemubot.module.more import Response
 
 from . import isup
 from . import page

modules/networking/isup.py

@@ -11,7 +11,7 @@ def isup(url):
 
     o = urllib.parse.urlparse(getNormalizedURL(url), "http")
     if o.netloc != "":
-        isup = getJSON("http://isitup.org/%s.json" % o.netloc)
+        isup = getJSON("https://isitup.org/%s.json" % o.netloc)
         if isup is not None and "status_code" in isup and isup["status_code"] == 1:
             return isup["response_time"]
 

modules/networking/w3c.py

@@ -17,7 +17,7 @@ def validator(url):
         raise IMException("Indicate a valid URL!")
 
     try:
-        req = urllib.request.Request("http://validator.w3.org/check?uri=%s&output=json" % (urllib.parse.quote(o.geturl())), headers={ 'User-Agent' : "Nemubot v%s" % __version__})
+        req = urllib.request.Request("https://validator.w3.org/check?uri=%s&output=json" % (urllib.parse.quote(o.geturl())), headers={ 'User-Agent' : "Nemubot v%s" % __version__})
         raw = urllib.request.urlopen(req, timeout=10)
     except urllib.error.HTTPError as e:
         raise IMException("HTTP error occurs: %s %s" % (e.code, e.reason))

modules/networking/watchWebsite.py

@@ -1,5 +1,6 @@
 """Alert on changes on websites"""
 
+from functools import partial
 import logging
 from random import randint
 import urllib.parse
@@ -12,7 +13,7 @@ from nemubot.tools.xmlparser.node import ModuleState
 
 logger = logging.getLogger("nemubot.module.networking.watchWebsite")
 
-from more import Response
+from nemubot.module.more import Response
 
 from . import page
 
@@ -209,15 +210,14 @@ def start_watching(site, offset=0):
     offset -- offset time to delay the launch of the first check
     """
 
-    o = urlparse(getNormalizedURL(site["url"]), "http")
-    #print_debug("Add %s event for site: %s" % (site["type"], o.netloc))
+    #o = urlparse(getNormalizedURL(site["url"]), "http")
+    #print("Add %s event for site: %s" % (site["type"], o.netloc))
 
     try:
-        evt = ModuleEvent(func=fwatch,
-                          cmp_data=site["lastcontent"],
-                          func_data=site["url"], offset=offset,
-                          interval=site.getInt("time"),
-                          call=alert_change, call_data=site)
+        evt = ModuleEvent(func=partial(fwatch, url=site["url"]),
+                          cmp=site["lastcontent"],
+                          offset=offset, interval=site.getInt("time"),
+                          call=partial(alert_change, site=site))
         site["_evt_id"] = add_event(evt)
     except IMException:
         logger.exception("Unable to watch %s", site["url"])

modules/networking/whois.py

@@ -6,10 +6,10 @@ import urllib
 from nemubot.exception import IMException
 from nemubot.tools.web import getJSON
 
-from more import Response
+from nemubot.module.more import Response
 
 URL_AVAIL = "https://www.whoisxmlapi.com/whoisserver/WhoisService?cmd=GET_DN_AVAILABILITY&domainName=%%s&outputFormat=json&username=%s&password=%s"
-URL_WHOIS = "http://www.whoisxmlapi.com/whoisserver/WhoisService?da=2&domainName=%%s&outputFormat=json&userName=%s&password=%s"
+URL_WHOIS = "https://www.whoisxmlapi.com/whoisserver/WhoisService?da=2&domainName=%%s&outputFormat=json&userName=%s&password=%s"
 
 
 # LOADING #############################################################
@@ -22,7 +22,7 @@ def load(CONF, add_hook):
                           "the !netwhois feature. Add it to the module "
                           "configuration file:\n<whoisxmlapi username=\"XX\" "
                           "password=\"XXX\" />\nRegister at "
-                          "http://www.whoisxmlapi.com/newaccount.php")
+                          "https://www.whoisxmlapi.com/newaccount.php")
 
     URL_AVAIL = URL_AVAIL % (urllib.parse.quote(CONF.getNode("whoisxmlapi")["username"]), urllib.parse.quote(CONF.getNode("whoisxmlapi")["password"]))
     URL_WHOIS = URL_WHOIS % (urllib.parse.quote(CONF.getNode("whoisxmlapi")["username"]), urllib.parse.quote(CONF.getNode("whoisxmlapi")["password"]))

modules/news.py

@@ -12,7 +12,8 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
+from nemubot.module.urlreducer import reduce_inline
 from nemubot.tools.feed import Feed, AtomEntry
 
 
@@ -50,10 +51,11 @@ def cmd_news(msg):
     links = [x for x in find_rss_links(url)]
     if len(links) == 0: links = [ url ]
 
-    res = Response(channel=msg.channel, nomore="No more news from %s" % url)
+    res = Response(channel=msg.channel, nomore="No more news from %s" % url, line_treat=reduce_inline)
     for n in get_last_news(links[0]):
         res.append_message("%s published %s: %s %s" % (("\x02" + web.striphtml(n.title) + "\x0F") if n.title else "An article without title",
                                                        (n.updated.strftime("on %A %d. %B %Y at %H:%M") if n.updated else "someday") if isinstance(n, AtomEntry) else n.pubDate,
                                                        web.striphtml(n.summary) if n.summary else "",
                                                        n.link if n.link else ""))
+
     return res

modules/nextstop.xml

@@ -1,4 +0,0 @@
-<?xml version="1.0" ?>
-<nemubotmodule name="nextstop">
-  <message type="cmd" name="ratp" call="ask_ratp" />
-</nemubotmodule>

modules/nextstop/__init__.py

@@ -1,55 +0,0 @@
-# coding=utf-8
-
-"""Informe les usagers des prochains passages des transports en communs de la RATP"""
-
-from nemubot.exception import IMException
-from nemubot.hooks import hook
-from more import Response
-
-nemubotversion = 3.4
-
-from .external.src import ratp
-
-def help_full ():
-    return "!ratp transport line [station]: Donne des informations sur les prochains passages du transport en commun séléctionné à l'arrêt désiré. Si aucune station n'est précisée, les liste toutes."
-
-
-@hook.command("ratp")
-def ask_ratp(msg):
-    """Hook entry from !ratp"""
-    if len(msg.args) >= 3:
-        transport = msg.args[0]
-        line = msg.args[1]
-        station = msg.args[2]
-        if len(msg.args) == 4:
-            times = ratp.getNextStopsAtStation(transport, line, station, msg.args[3])
-        else:
-            times = ratp.getNextStopsAtStation(transport, line, station)
-
-        if len(times) == 0:
-            raise IMException("la station %s n'existe pas sur le %s ligne %s." % (station, transport, line))
-
-        (time, direction, stationname) = times[0]
-        return Response(message=["\x03\x02%s\x03\x02 direction %s" % (time, direction) for time, direction, stationname in times],
-                        title="Prochains passages du %s ligne %s à l'arrêt %s" % (transport, line, stationname),
-                        channel=msg.channel)
-
-    elif len(msg.args) == 2:
-        stations = ratp.getAllStations(msg.args[0], msg.args[1])
-
-        if len(stations) == 0:
-            raise IMException("aucune station trouvée.")
-        return Response([s for s in stations], title="Stations", channel=msg.channel)
-
-    else:
-        raise IMException("Mauvais usage, merci de spécifier un type de transport et une ligne, ou de consulter l'aide du module.")
-
-@hook.command("ratp_alert")
-def ratp_alert(msg):
-    if len(msg.args) == 2:
-        transport = msg.args[0]
-        cause  = msg.args[1]
-        incidents = ratp.getDisturbance(cause, transport)
-        return Response(incidents, channel=msg.channel, nomore="No more incidents", count=" (%d more incidents)")
-    else:
-        raise IMException("Mauvais usage, merci de spécifier un type de transport et un type d'alerte (alerte, manif, travaux), ou de consulter l'aide du module.")

modules/nextstop/external

@@ -1 +0,0 @@
-Subproject commit 3d5c9b2d52fbd214f5aaad00e5f3952de919b3e5

modules/nntp.py

@@ -0,0 +1,229 @@
+"""The NNTP module"""
+
+# PYTHON STUFFS #######################################################
+
+import email
+import email.policy
+from email.utils import mktime_tz, parseaddr, parsedate_tz
+from functools import partial
+from nntplib import NNTP, decode_header
+import re
+import time
+from datetime import datetime
+from zlib import adler32
+
+from nemubot import context
+from nemubot.event import ModuleEvent
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.tools.xmlparser.node import ModuleState
+
+from nemubot.module.more import Response
+
+
+# LOADING #############################################################
+
+def load(context):
+    for wn in context.data.getNodes("watched_newsgroup"):
+        watch(**wn.attributes)
+
+
+# MODULE CORE #########################################################
+
+def list_groups(group_pattern="*", **server):
+    with NNTP(**server) as srv:
+        response, l = srv.list(group_pattern)
+        for i in l:
+            yield i.group, srv.description(i.group), i.flag
+
+def read_group(group, **server):
+    with NNTP(**server) as srv:
+        response, count, first, last, name = srv.group(group)
+        resp, overviews = srv.over((first, last))
+        for art_num, over in reversed(overviews):
+            yield over
+
+def read_article(msg_id, **server):
+    with NNTP(**server) as srv:
+        response, info = srv.article(msg_id)
+        return email.message_from_bytes(b"\r\n".join(info.lines), policy=email.policy.SMTPUTF8)
+
+
+servers_lastcheck = dict()
+servers_lastseen = dict()
+
+def whatsnew(group="*", **server):
+    fill = dict()
+    if "user" in server: fill["user"] = server["user"]
+    if "password" in server: fill["password"] = server["password"]
+    if "host" in server: fill["host"] = server["host"]
+    if "port" in server: fill["port"] = server["port"]
+
+    idx = _indexServer(**server)
+    if idx in servers_lastcheck and servers_lastcheck[idx] is not None:
+        date_last_check = servers_lastcheck[idx]
+    else:
+        date_last_check = datetime.now()
+
+    if idx not in servers_lastseen:
+        servers_lastseen[idx] = []
+
+    with NNTP(**fill) as srv:
+        response, servers_lastcheck[idx] = srv.date()
+
+        response, groups = srv.newgroups(date_last_check)
+        for g in groups:
+            yield g
+
+        response, articles = srv.newnews(group, date_last_check)
+        for msg_id in articles:
+            if msg_id not in servers_lastseen[idx]:
+                servers_lastseen[idx].append(msg_id)
+                response, info = srv.article(msg_id)
+                yield email.message_from_bytes(b"\r\n".join(info.lines))
+
+        # Clean huge lists
+        if len(servers_lastseen[idx]) > 42:
+            servers_lastseen[idx] = servers_lastseen[idx][23:]
+
+
+def format_article(art, **response_args):
+    art["X-FromName"], art["X-FromEmail"] = parseaddr(art["From"] if "From" in art else "")
+    if art["X-FromName"] == '': art["X-FromName"] = art["X-FromEmail"]
+
+    date = mktime_tz(parsedate_tz(art["Date"]))
+    if date < time.time() - 120:
+        title = "\x0314In \x0F\x03{0:02d}{Newsgroups}\x0F\x0314: on \x0F{Date}\x0314 by \x0F\x03{0:02d}{X-FromName}\x0F \x02{Subject}\x0F"
+    else:
+        title = "\x0314In \x0F\x03{0:02d}{Newsgroups}\x0F\x0314: by \x0F\x03{0:02d}{X-FromName}\x0F \x02{Subject}\x0F"
+
+    return Response(art.get_payload().replace('\n', ' '),
+                    title=title.format(adler32(art["Newsgroups"].encode()) & 0xf, adler32(art["X-FromEmail"].encode()) & 0xf, **{h: decode_header(i) for h,i in art.items()}),
+                    **response_args)
+
+
+watches = dict()
+
+def _indexServer(**kwargs):
+    if "user" not in kwargs: kwargs["user"] = ""
+    if "password" not in kwargs: kwargs["password"] = ""
+    if "host" not in kwargs: kwargs["host"] = ""
+    if "port" not in kwargs: kwargs["port"] = 119
+    return "{user}:{password}@{host}:{port}".format(**kwargs)
+
+def _newevt(**args):
+    context.add_event(ModuleEvent(call=partial(_ticker, **args), interval=42))
+
+def _ticker(to_server, to_channel, group, server):
+    _newevt(to_server=to_server, to_channel=to_channel, group=group, server=server)
+    n = 0
+    for art in whatsnew(group, **server):
+        n += 1
+        if n > 10:
+            continue
+        context.send_response(to_server, format_article(art, channel=to_channel))
+    if n > 10:
+        context.send_response(to_server, Response("... and %s others news" % (n - 10), channel=to_channel))
+
+def watch(to_server, to_channel, group="*", **server):
+    _newevt(to_server=to_server, to_channel=to_channel, group=group, server=server)
+
+
+# MODULE INTERFACE ####################################################
+
+keywords_server = {
+    "host=HOST": "hostname or IP of the NNTP server",
+    "port=PORT": "port of the NNTP server",
+    "user=USERNAME": "username to use to connect to the server",
+    "password=PASSWORD": "password to use to connect to the server",
+}
+
+@hook.command("nntp_groups",
+      help="Show list of existing groups",
+      help_usage={
+          None: "Display all groups",
+          "PATTERN": "Filter on group matching the PATTERN"
+      },
+      keywords=keywords_server)
+def cmd_groups(msg):
+    if "host" not in msg.kwargs:
+        raise IMException("please give a hostname in keywords")
+
+    return Response(["\x02\x03{0:02d}{1}\x0F: {2}".format(adler32(g[0].encode()) & 0xf, *g) for g in list_groups(msg.args[0] if len(msg.args) > 0 else "*", **msg.kwargs)],
+                    channel=msg.channel,
+                    title="Matching groups on %s" % msg.kwargs["host"])
+
+
+@hook.command("nntp_overview",
+              help="Show an overview of articles in given group(s)",
+              help_usage={
+                  "GROUP": "Filter on group matching the PATTERN"
+              },
+              keywords=keywords_server)
+def cmd_overview(msg):
+    if "host" not in msg.kwargs:
+        raise IMException("please give a hostname in keywords")
+
+    if not len(msg.args):
+        raise IMException("which group would you overview?")
+
+    for g in msg.args:
+        arts = []
+        for grp in read_group(g, **msg.kwargs):
+            grp["X-FromName"], grp["X-FromEmail"] = parseaddr(grp["from"] if "from" in grp else "")
+            if grp["X-FromName"] == '': grp["X-FromName"] = grp["X-FromEmail"]
+
+            arts.append("On {date}, from \x03{0:02d}{X-FromName}\x0F \x02{subject}\x0F: \x0314{message-id}\x0F".format(adler32(grp["X-FromEmail"].encode()) & 0xf, **{h: decode_header(i) for h,i in grp.items()}))
+
+        if len(arts):
+            yield Response(arts,
+                           channel=msg.channel,
+                           title="In \x03{0:02d}{1}\x0F".format(adler32(g[0].encode()) & 0xf, g))
+
+
+@hook.command("nntp_read",
+              help="Read an article from a server",
+              help_usage={
+                  "MSG_ID": "Read the given message"
+              },
+              keywords=keywords_server)
+def cmd_read(msg):
+    if "host" not in msg.kwargs:
+        raise IMException("please give a hostname in keywords")
+
+    for msgid in msg.args:
+        if not re.match("<.*>", msgid):
+            msgid = "<" + msgid + ">"
+        art = read_article(msgid, **msg.kwargs)
+        yield format_article(art, channel=msg.channel)
+
+
+@hook.command("nntp_watch",
+              help="Launch an event looking for new groups and articles on a server",
+              help_usage={
+                  None: "Watch all groups",
+                  "PATTERN": "Limit the watch on group matching this PATTERN"
+              },
+              keywords=keywords_server)
+def cmd_watch(msg):
+    if "host" not in msg.kwargs:
+        raise IMException("please give a hostname in keywords")
+
+    if not msg.frm_owner:
+        raise IMException("sorry, this command is currently limited to the owner")
+
+    wnnode = ModuleState("watched_newsgroup")
+    wnnode["id"] = _indexServer(**msg.kwargs)
+    wnnode["to_server"] = msg.server
+    wnnode["to_channel"] = msg.channel
+    wnnode["group"] = msg.args[0] if len(msg.args) > 0 else "*"
+
+    wnnode["user"] = msg.kwargs["user"] if "user" in msg.kwargs else ""
+    wnnode["password"] = msg.kwargs["password"] if "password" in msg.kwargs else ""
+    wnnode["host"] = msg.kwargs["host"] if "host" in msg.kwargs else ""
+    wnnode["port"] = msg.kwargs["port"] if "port" in msg.kwargs else 119
+
+    context.data.addChild(wnnode)
+    watch(**wnnode.attributes)
+
+    return Response("Ok ok, I watch this newsgroup!", channel=msg.channel)

modules/openai.py

@@ -0,0 +1,87 @@
+"""Perform requests to openai"""
+
+# PYTHON STUFFS #######################################################
+
+from openai import OpenAI
+
+from nemubot import context
+from nemubot.hooks import hook
+from nemubot.tools import web
+
+from nemubot.module.more import Response
+
+
+# LOADING #############################################################
+
+CLIENT = None
+MODEL = "gpt-4"
+ENDPOINT = None
+
+def load(context):
+    global CLIENT, ENDPOINT, MODEL
+    if not context.config or ("apikey" not in context.config and "endpoint" not in context.config):
+        raise ImportError ("You need a OpenAI API key in order to use "
+                           "this module. Add it to the module configuration: "
+                           "\n<module name=\"openai\" "
+                           "apikey=\"XXXXXX-XXXXXXXXXX\" endpoint=\"https://...\" model=\"gpt-4\" />")
+    kwargs = {
+        "api_key": context.config["apikey"] or "",
+    }
+
+    if "endpoint" in context.config:
+        ENDPOINT = context.config["endpoint"]
+        kwargs["base_url"] = ENDPOINT
+
+    CLIENT = OpenAI(**kwargs)
+
+    if "model" in context.config:
+        MODEL = context.config["model"]
+
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("list_models",
+      help="list available LLM")
+def cmd_listllm(msg):
+    llms = web.getJSON(ENDPOINT + "/models", timeout=6)
+    return Response(message=[m for m in map(lambda i: i["id"], llms["data"])], title="Here is the available models", channel=msg.channel)
+
+
+@hook.command("set_model",
+      help="Set the model to use when talking to nemubot")
+def cmd_setllm(msg):
+    if len(msg.args) != 1:
+        raise IMException("Indicate 1 model to use")
+
+    wanted_model = msg.args[0]
+
+    llms = web.getJSON(ENDPOINT + "/models", timeout=6)
+    for model in llms["data"]:
+        if wanted_model == model["id"]:
+            break
+    else:
+        raise IMException("Unable to set such model: unknown")
+
+    MODEL = wanted_model
+    return Response("New model in use: " + wanted_model, channel=msg.channel)
+
+
+@hook.ask()
+def parseask(msg):
+    chat_completion = CLIENT.chat.completions.create(
+        messages=[
+            {
+                "role": "system",
+                "content": "You are a kind multilingual assistant. Respond to the user request in 255 characters maximum. Be conscise, go directly to the point. Never add useless terms.",
+            },
+            {
+                "role": "user",
+                "content": msg.message,
+            }
+        ],
+        model=MODEL,
+    )
+
+    return Response(chat_completion.choices[0].message.content,
+                    msg.channel,
+                    msg.frm)

modules/openroute.py

@@ -0,0 +1,158 @@
+"""Lost? use our commands to find your way!"""
+
+# PYTHON STUFFS #######################################################
+
+import re
+import urllib.parse
+
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.tools import web
+
+from nemubot.module.more import Response
+
+# GLOBALS #############################################################
+
+URL_DIRECTIONS_API = "https://api.openrouteservice.org/directions?api_key=%s&"
+URL_GEOCODE_API = "https://api.openrouteservice.org/geocoding?api_key=%s&"
+
+waytype = [
+    "unknown",
+    "state road",
+    "road",
+    "street",
+    "path",
+    "track",
+    "cycleway",
+    "footway",
+    "steps",
+    "ferry",
+    "construction",
+]
+
+
+# LOADING #############################################################
+
+def load(context):
+    if not context.config or "apikey" not in context.config:
+        raise ImportError("You need an OpenRouteService API key in order to use this "
+                          "module. Add it to the module configuration file:\n"
+                          "<module name=\"ors\" apikey=\"XXXXXXXXXXXXXXXX\" "
+                          "/>\nRegister at https://developers.openrouteservice.org")
+    global URL_DIRECTIONS_API
+    URL_DIRECTIONS_API = URL_DIRECTIONS_API % context.config["apikey"]
+    global URL_GEOCODE_API
+    URL_GEOCODE_API = URL_GEOCODE_API % context.config["apikey"]
+
+
+# MODULE CORE #########################################################
+
+def approx_distance(lng):
+    if lng > 1111:
+        return "%f km" % (lng / 1000)
+    else:
+        return "%f m" % lng
+
+
+def approx_duration(sec):
+    days = int(sec / 86400)
+    if days > 0:
+        return "%d days %f hours" % (days, (sec % 86400) / 3600)
+    hours = int((sec % 86400) / 3600)
+    if hours > 0:
+        return "%d hours %f minutes" % (hours, (sec % 3600) / 60)
+    minutes = (sec % 3600) / 60
+    if minutes > 0:
+        return "%d minutes" % minutes
+    else:
+        return "%d seconds" % sec
+
+
+def geocode(query, limit=7):
+    obj = web.getJSON(URL_GEOCODE_API + urllib.parse.urlencode({
+        'query': query,
+        'limit': limit,
+    }))
+
+    for f in obj["features"]:
+        yield f["geometry"]["coordinates"], f["properties"]
+
+
+def firstgeocode(query):
+    for g in geocode(query, limit=1):
+        return g
+
+
+def where(loc):
+    return "{name} {city} {state} {county} {country}".format(**loc)
+
+
+def directions(coordinates, **kwargs):
+    kwargs['coordinates'] = '|'.join(coordinates)
+
+    print(URL_DIRECTIONS_API + urllib.parse.urlencode(kwargs))
+    return web.getJSON(URL_DIRECTIONS_API + urllib.parse.urlencode(kwargs), decode_error=True)
+
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("geocode",
+              help="Get GPS coordinates of a place",
+              help_usage={
+                  "PLACE": "Get GPS coordinates of PLACE"
+              })
+def cmd_geocode(msg):
+    res = Response(channel=msg.channel, nick=msg.frm,
+                   nomore="No more geocode", count=" (%s more geocode)")
+
+    for loc in geocode(' '.join(msg.args)):
+        res.append_message("%s is at %s,%s" % (
+            where(loc[1]),
+            loc[0][1], loc[0][0],
+        ))
+
+    return res
+
+
+@hook.command("directions",
+              help="Get routing instructions",
+              help_usage={
+                  "POINT1 POINT2 ...": "Get routing instructions to go from POINT1 to the last POINTX via intermediates POINTX"
+              },
+              keywords={
+                  "profile=PROF": "One of driving-car, driving-hgv, cycling-regular, cycling-road, cycling-safe, cycling-mountain, cycling-tour, cycling-electric, foot-walking, foot-hiking, wheelchair. Default: foot-walking",
+                  "preference=PREF": "One of fastest, shortest, recommended. Default: recommended",
+                  "lang=LANG": "default: en",
+              })
+def cmd_directions(msg):
+    drcts = directions(["{0},{1}".format(*firstgeocode(g)[0]) for g in msg.args],
+                       profile=msg.kwargs["profile"] if "profile" in msg.kwargs else "foot-walking",
+                       preference=msg.kwargs["preference"] if "preference" in msg.kwargs else "recommended",
+                       units="m",
+                       language=msg.kwargs["lang"] if "lang" in msg.kwargs else "en",
+                       geometry=False,
+                       instructions=True,
+                       instruction_format="text")
+    if "error" in drcts and "message" in drcts["error"] and drcts["error"]["message"]:
+        raise IMException(drcts["error"]["message"])
+
+    if "routes" not in drcts or not drcts["routes"]:
+        raise IMException("No route available for this trip")
+
+    myway = drcts["routes"][0]
+    myway["summary"]["strduration"] = approx_duration(myway["summary"]["duration"])
+    myway["summary"]["strdistance"] = approx_distance(myway["summary"]["distance"])
+    res = Response("Trip summary: {strdistance} in approximate {strduration}; elevation +{ascent} m -{descent} m".format(**myway["summary"]), channel=msg.channel, count=" (%d more steps)", nomore="You have arrived!")
+
+    def formatSegments(segments):
+        for segment in segments:
+            for step in segment["steps"]:
+                step["strtype"] = waytype[step["type"]]
+                step["strduration"] = approx_duration(step["duration"])
+                step["strdistance"] = approx_distance(step["distance"])
+                yield "{instruction} for {strdistance} on {strtype} (approximate time: {strduration})".format(**step)
+
+    if "segments" in myway:
+        res.append_message([m for m in formatSegments(myway["segments"])])
+
+    return res

modules/pkgs.py

@@ -0,0 +1,68 @@
+"""Get information about common software"""
+
+# PYTHON STUFFS #######################################################
+
+import portage
+
+from nemubot import context
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+
+from nemubot.module.more import Response
+
+DB = None
+
+# MODULE CORE #########################################################
+
+def get_db():
+    global DB
+    if DB is None:
+        DB = portage.db[portage.root]["porttree"].dbapi
+    return DB
+
+
+def package_info(pkgname):
+    pv = get_db().xmatch("match-all", pkgname)
+    if not pv:
+        raise IMException("No package named '%s' found" % pkgname)
+
+    bv = get_db().xmatch("bestmatch-visible", pkgname)
+    pvsplit = portage.catpkgsplit(bv if bv else pv[-1])
+    info = get_db().aux_get(bv if bv else pv[-1], ["DESCRIPTION", "HOMEPAGE", "LICENSE", "IUSE", "KEYWORDS"])
+
+    return {
+        "pkgname": '/'.join(pvsplit[:2]),
+        "category": pvsplit[0],
+        "shortname": pvsplit[1],
+        "lastvers": '-'.join(pvsplit[2:]) if pvsplit[3] != "r0" else pvsplit[2],
+        "othersvers": ['-'.join(portage.catpkgsplit(p)[2:]) for p in pv if p != bv],
+        "description": info[0],
+        "homepage": info[1],
+        "license": info[2],
+        "uses": info[3],
+        "keywords": info[4],
+    }
+
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("eix",
+      help="Get information about a package",
+      help_usage={
+          "NAME": "Get information about a software NAME"
+      })
+def cmd_eix(msg):
+    if not len(msg.args):
+        raise IMException("please give me a package to search")
+
+    def srch(term):
+        try:
+            yield package_info(term)
+        except portage.exception.AmbiguousPackageName as e:
+            for i in e.args[0]:
+                yield package_info(i)
+
+    res = Response(channel=msg.channel, count=" (%d more packages)", nomore="No more package '%s'" % msg.args[0])
+    for pi in srch(msg.args[0]):
+        res.append_message("\x03\x02{pkgname}:\x03\x02 {description} - {homepage} - {license} - last revisions: \x03\x02{lastvers}\x03\x02{ov}".format(ov=(", " + ', '.join(pi["othersvers"])) if pi["othersvers"] else "", **pi))
+    return res

modules/ratp.py

@@ -0,0 +1,74 @@
+"""Informe les usagers des prochains passages des transports en communs de la RATP"""
+
+# PYTHON STUFFS #######################################################
+
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.module.more import Response
+
+from nextstop import ratp
+
+@hook.command("ratp",
+              help="Affiche les prochains horaires de passage",
+              help_usage={
+                  "TRANSPORT": "Affiche les lignes du moyen de transport donné",
+                  "TRANSPORT LINE": "Affiche les stations sur la ligne de transport donnée",
+                  "TRANSPORT LINE STATION": "Affiche les prochains horaires de passage à l'arrêt donné",
+                  "TRANSPORT LINE STATION DESTINATION": "Affiche les prochains horaires de passage dans la direction donnée",
+              })
+def ask_ratp(msg):
+    l = len(msg.args)
+
+    transport = msg.args[0] if l > 0 else None
+    line = msg.args[1] if l > 1 else None
+    station = msg.args[2] if l > 2 else None
+    direction = msg.args[3] if l > 3 else None
+
+    if station is not None:
+        times = sorted(ratp.getNextStopsAtStation(transport, line, station, direction), key=lambda i: i[0])
+
+        if len(times) == 0:
+            raise IMException("la station %s n'existe pas sur le %s ligne %s." % (station, transport, line))
+
+        (time, direction, stationname) = times[0]
+        return Response(message=["\x03\x02%s\x03\x02 direction %s" % (time, direction) for time, direction, stationname in times],
+                        title="Prochains passages du %s ligne %s à l'arrêt %s" % (transport, line, stationname),
+                        channel=msg.channel)
+
+    elif line is not None:
+        stations = ratp.getAllStations(transport, line)
+
+        if len(stations) == 0:
+            raise IMException("aucune station trouvée.")
+        return Response(stations, title="Stations", channel=msg.channel)
+
+    elif transport is not None:
+        lines = ratp.getTransportLines(transport)
+        if len(lines) == 0:
+            raise IMException("aucune ligne trouvée.")
+        return Response(lines, title="Lignes", channel=msg.channel)
+
+    else:
+        raise IMException("précise au moins un moyen de transport.")
+
+
+@hook.command("ratp_alert",
+              help="Affiche les perturbations en cours sur le réseau")
+def ratp_alert(msg):
+    if len(msg.args) == 0:
+        raise IMException("précise au moins un moyen de transport.")
+
+    l = len(msg.args)
+    transport = msg.args[0] if l > 0 else None
+    line = msg.args[1] if l > 1 else None
+
+    if line is not None:
+        d = ratp.getDisturbanceFromLine(transport, line)
+        if "date" in d and d["date"] is not None:
+            incidents = "Au {date[date]}, {title}: {message}".format(**d)
+        else:
+            incidents = "{title}: {message}".format(**d)
+    else:
+        incidents = ratp.getDisturbance(None, transport)
+
+    return Response(incidents, channel=msg.channel, nomore="No more incidents", count=" (%d more incidents)")

modules/reddit.py

@@ -10,7 +10,7 @@ from nemubot.tools import web
 
 nemubotversion = 3.4
 
-from more import Response
+from nemubot.module.more import Response
 
 
 def help_full():
@@ -40,7 +40,7 @@ def cmd_subreddit(msg):
             else:
                 where = "r"
 
-            sbr = web.getJSON("http://www.reddit.com/%s/%s/about.json" %
+            sbr = web.getJSON("https://www.reddit.com/%s/%s/about.json" %
                               (where, sub.group(2)))
 
             if sbr is None:
@@ -64,22 +64,29 @@ def cmd_subreddit(msg):
                                         channel=msg.channel))
         else:
             all_res.append(Response("%s is not a valid subreddit" % osub,
-                                    channel=msg.channel, nick=msg.nick))
+                                    channel=msg.channel, nick=msg.frm))
 
     return all_res
 
 
 @hook.message()
 def parselisten(msg):
-    parseresponse(msg)
-    return None
+    global LAST_SUBS
+
+    if hasattr(msg, "message") and msg.message and type(msg.message) == str:
+        urls = re.findall("www.reddit.com(/\w/\w+/?)", msg.message)
+        for url in urls:
+            for recv in msg.to:
+                if recv not in LAST_SUBS:
+                    LAST_SUBS[recv] = list()
+                LAST_SUBS[recv].append(url)
 
 
 @hook.post()
 def parseresponse(msg):
     global LAST_SUBS
 
-    if hasattr(msg, "text") and msg.text:
+    if hasattr(msg, "text") and msg.text and type(msg.text) == str:
         urls = re.findall("www.reddit.com(/\w/\w+/?)", msg.text)
         for url in urls:
             for recv in msg.to:

modules/repology.py

@@ -0,0 +1,94 @@
+# coding=utf-8
+
+"""Repology.org module: the packaging hub"""
+
+import datetime
+import re
+
+from nemubot import context
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.tools import web
+from nemubot.tools.xmlparser.node import ModuleState
+
+nemubotversion = 4.0
+
+from nemubot.module.more import Response
+
+URL_REPOAPI = "https://repology.org/api/v1/project/%s"
+
+def get_json_project(project):
+    prj = web.getJSON(URL_REPOAPI % (project))
+
+    return prj
+
+
+@hook.command("repology",
+              help="Display version information about a package",
+              help_usage={
+                  "PACKAGE_NAME": "Retrieve informations about PACKAGE_NAME",
+              },
+              keywords={
+                  "distro=DISTRO": "filter by disto",
+                  "status=STATUS[,STATUS...]": "filter by status",
+              })
+def cmd_repology(msg):
+    if len(msg.args) == 0:
+        raise IMException("Please provide at least a package name")
+
+    res = Response(channel=msg.channel, nomore="No more information on package")
+
+    for project in msg.args:
+        prj = get_json_project(project)
+        if len(prj) == 0:
+            raise IMException("Unable to find package " + project)
+
+        pkg_versions = {}
+        pkg_maintainers = {}
+        pkg_licenses = {}
+        summary = None
+
+        for repo in prj:
+            # Apply filters
+            if "distro" in msg.kwargs and repo["repo"].find(msg.kwargs["distro"]) < 0:
+                continue
+            if "status" in msg.kwargs and repo["status"] not in msg.kwargs["status"].split(","):
+                continue
+
+            name = repo["visiblename"] if "visiblename" in repo else repo["name"]
+            status = repo["status"] if "status" in repo else "unknown"
+            if name not in pkg_versions:
+                pkg_versions[name] = {}
+            if status not in pkg_versions[name]:
+                pkg_versions[name][status] = []
+            if repo["version"] not in pkg_versions[name][status]:
+                pkg_versions[name][status].append(repo["version"])
+
+            if "maintainers" in repo:
+                if name not in pkg_maintainers:
+                    pkg_maintainers[name] = []
+                for maintainer in repo["maintainers"]:
+                    if maintainer not in pkg_maintainers[name]:
+                        pkg_maintainers[name].append(maintainer)
+
+            if "licenses" in repo:
+                if name not in pkg_licenses:
+                    pkg_licenses[name] = []
+                for lic in repo["licenses"]:
+                    if lic not in pkg_licenses[name]:
+                        pkg_licenses[name].append(lic)
+
+            if "summary" in repo and summary is None:
+                summary = repo["summary"]
+
+        for pkgname in sorted(pkg_versions.keys()):
+            m = "Package " + pkgname + " (" + summary + ")"
+            if pkgname in pkg_licenses:
+                m += " under " + ", ".join(pkg_licenses[pkgname])
+            m += ": " + " - ".join([status + ": " + ", ".join(pkg_versions[pkgname][status]) for status in ["newest", "devel", "unique", "outdated", "legacy", "rolling", "noscheme", "untrusted", "ignored"] if status in pkg_versions[pkgname]])
+            if "distro" in msg.kwargs and pkgname in pkg_maintainers:
+                m += " - Maintained by " + ", ".join(pkg_maintainers[pkgname])
+
+            res.append_message(m)
+
+    return res

modules/rnd.py

@@ -8,9 +8,8 @@ import shlex
 from nemubot import context
 from nemubot.exception import IMException
 from nemubot.hooks import hook
-from nemubot.message import Command
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # MODULE INTERFACE ####################################################
@@ -22,7 +21,7 @@ def cmd_choice(msg):
 
     return Response(random.choice(msg.args),
                     channel=msg.channel,
-                    nick=msg.nick)
+                    nick=msg.frm)
 
 
 @hook.command("choicecmd")
@@ -32,8 +31,24 @@ def cmd_choicecmd(msg):
 
     choice = shlex.split(random.choice(msg.args))
 
-    return [x for x in context.subtreat(Command(choice[0][1:],
-                                                choice[1:],
-                                                to_response=msg.to_response,
-                                                frm=msg.frm,
-                                                server=msg.server))]
+    return [x for x in context.subtreat(context.subparse(msg, choice))]
+
+
+@hook.command("choiceres")
+def cmd_choiceres(msg):
+    if not len(msg.args):
+        raise IMException("indicate some command to pick a message from!")
+
+    rl = [x for x in context.subtreat(context.subparse(msg, " ".join(msg.args)))]
+    if len(rl) <= 0:
+        return rl
+
+    r = random.choice(rl)
+
+    if isinstance(r, Response):
+        for i in range(len(r.messages) - 1, -1, -1):
+            if isinstance(r.messages[i], list):
+                r.messages = [ random.choice(random.choice(r.messages)) ]
+            elif isinstance(r.messages[i], str):
+                r.messages = [ random.choice(r.messages) ]
+    return r

modules/sap.py

@@ -12,7 +12,7 @@ from nemubot.tools import web
 
 nemubotversion = 4.0
 
-from more import Response
+from nemubot.module.more import Response
 
 
 def help_full():
@@ -25,7 +25,7 @@ def cmd_tcode(msg):
         raise IMException("indicate a transaction code or "
                            "a keyword to search!")
 
-    url = ("http://www.tcodesearch.com/tcodes/search?q=%s" %
+    url = ("https://www.tcodesearch.com/tcodes/search?q=%s" %
            urllib.parse.quote(msg.args[0]))
 
     page = web.getURLContent(url)

modules/shodan.py

@@ -0,0 +1,104 @@
+"""Search engine for IoT"""
+
+# PYTHON STUFFS #######################################################
+
+from datetime import datetime
+import ipaddress
+import urllib.parse
+
+from nemubot import context
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.tools import web
+
+from nemubot.module.more import Response
+
+
+# GLOBALS #############################################################
+
+BASEURL = "https://api.shodan.io/shodan/"
+
+
+# LOADING #############################################################
+
+def load(context):
+    if not context.config or "apikey" not in context.config:
+        raise ImportError("You need a Shodan API key in order to use this "
+                          "module. Add it to the module configuration file:\n"
+                          "<module name=\"shodan\" apikey=\"XXXXXXXXXXXXXXXX\" "
+                          "/>\nRegister at https://account.shodan.io/register")
+
+
+# MODULE CORE #########################################################
+
+def host_lookup(ip):
+    url = BASEURL + "host/" + urllib.parse.quote(ip) + "?" + urllib.parse.urlencode({'key': context.config["apikey"]})
+    return web.getJSON(url)
+
+
+def search_hosts(query):
+    url = BASEURL + "host/search?" + urllib.parse.urlencode({'query': query, 'key': context.config["apikey"]})
+    return web.getJSON(url, max_size=4194304)
+
+
+def print_ssl(ssl):
+    return (
+        "SSL: " +
+        " ".join([v for v in ssl["versions"] if v[0] != "-"]) +
+        "; cipher used: " + ssl["cipher"]["name"] +
+        ("; certificate: " + ssl["cert"]["sig_alg"] +
+        " issued by: " + ssl["cert"]["issuer"]["CN"] +
+        " expires on: " + str(datetime.strptime(ssl["cert"]["expires"], "%Y%m%d%H%M%SZ")) if "cert" in ssl else "")
+    )
+
+def print_service(svc):
+    ip = ipaddress.ip_address(svc["ip_str"])
+    return ((svc["ip_str"] if ip.version == 4 else "[%s]" % svc["ip_str"]) +
+            ":{port}/{transport} ({module}):" +
+            (" {os}" if svc["os"] else "") +
+            (" {product}" if "product" in svc else "") +
+            (" {version}" if "version" in svc else "") +
+            (" {info}" if "info" in svc else "") +
+            (" Vulns: " + ", ".join(svc["opts"]["vulns"]) if "opts" in svc and "vulns" in svc["opts"] else "") +
+            (" " + print_ssl(svc["ssl"]) if "ssl" in svc else "") +
+            (" \x03\x1D" + svc["data"].replace("\r\n", "\n").split("\n")[0] + "\x03\x1D" if "data" in svc else "") +
+            (" " + svc["title"] if "title" in svc else "")
+           ).format(module=svc["_shodan"]["module"], **svc)
+
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("shodan",
+              help="Use shodan.io to get information on machines connected to Internet",
+              help_usage={
+                  "IP": "retrieve information about the given IP (can be v4 or v6)",
+                  "TERM": "retrieve all hosts matching TERM somewhere in their exposed stuff"
+              })
+def shodan(msg):
+    if not msg.args:
+        raise IMException("indicate an IP or a term to search!")
+
+    terms = " ".join(msg.args)
+
+    try:
+        ip = ipaddress.ip_address(terms)
+    except ValueError:
+        ip = None
+
+    if ip:
+        h = host_lookup(terms)
+        res = Response(channel=msg.channel,
+                       title="%s" % ((h["ip_str"] if ip.version == 4 else "[%s]" % h["ip_str"]) + (" (" + ", ".join(h["hostnames"]) + ")") if h["hostnames"] else ""))
+        res.append_message("{isp} ({asn}) -> {city} ({country_code}), running {os}. Vulns: {vulns_str}. Open ports: {open_ports}. Last update: {last_update}".format(
+            open_ports=", ".join(map(lambda a: str(a), h["ports"])), vulns_str=", ".join(h["vulns"]) if "vulns" in h else None, **h).strip())
+        for d in h["data"]:
+            res.append_message(print_service(d))
+
+    else:
+        q = search_hosts(terms)
+        res = Response(channel=msg.channel,
+                       count=" (%%s/%s results)" % q["total"])
+        for r in q["matches"]:
+            res.append_message(print_service(r))
+
+    return res

modules/sleepytime.py

@@ -10,7 +10,7 @@ from nemubot.hooks import hook
 
 nemubotversion = 3.4
 
-from more import Response
+from nemubot.module.more import Response
 
 
 def help_full():

modules/smmry.py

@@ -0,0 +1,116 @@
+"""Summarize texts"""
+
+# PYTHON STUFFS #######################################################
+
+from urllib.parse import quote
+
+from nemubot import context
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.tools import web
+
+from nemubot.module.more import Response
+from nemubot.module.urlreducer import LAST_URLS
+
+
+# GLOBALS #############################################################
+
+URL_API = "https://api.smmry.com/?SM_API_KEY=%s"
+
+
+# LOADING #############################################################
+
+def load(context):
+    if not context.config or "apikey" not in context.config:
+        raise ImportError("You need a Smmry API key in order to use this "
+                          "module. Add it to the module configuration file:\n"
+                          "<module name=\"smmry\" apikey=\"XXXXXXXXXXXXXXXX\" "
+                          "/>\nRegister at https://smmry.com/partner")
+    global URL_API
+    URL_API = URL_API % context.config["apikey"]
+
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("smmry",
+              help="Summarize the following words/command return",
+              help_usage={
+                  "WORDS/CMD": ""
+              },
+              keywords={
+                  "keywords?=X": "Returns keywords instead of summary (count optional)",
+                  "length=7": "The number of sentences returned, default 7",
+                  "break": "inserts the string [BREAK] between sentences",
+                  "ignore_length": "returns summary regardless of quality or length",
+                  "quote_avoid": "sentences with quotations will be excluded",
+                  "question_avoid": "sentences with question will be excluded",
+                  "exclamation_avoid": "sentences with exclamation marks will be excluded",
+              })
+def cmd_smmry(msg):
+    if not len(msg.args):
+        global LAST_URLS
+        if msg.channel in LAST_URLS and len(LAST_URLS[msg.channel]) > 0:
+            msg.args.append(LAST_URLS[msg.channel].pop())
+        else:
+            raise IMException("I have no more URL to sum up.")
+
+    URL = URL_API
+    if "length" in msg.kwargs:
+        if int(msg.kwargs["length"]) > 0 :
+            URL += "&SM_LENGTH=" + msg.kwargs["length"]
+        else:
+            msg.kwargs["ignore_length"] = True
+    if "break" in msg.kwargs: URL += "&SM_WITH_BREAK"
+    if "ignore_length" in msg.kwargs: URL += "&SM_IGNORE_LENGTH"
+    if "quote_avoid" in msg.kwargs: URL += "&SM_QUOTE_AVOID"
+    if "question_avoid" in msg.kwargs: URL += "&SM_QUESTION_AVOID"
+    if "exclamation_avoid" in msg.kwargs: URL += "&SM_EXCLAMATION_AVOID"
+    if "keywords" in msg.kwargs and msg.kwargs["keywords"] is not None and int(msg.kwargs["keywords"]) > 0: URL += "&SM_KEYWORD_COUNT=" + msg.kwargs["keywords"]
+
+    res = Response(channel=msg.channel)
+
+    if web.isURL(" ".join(msg.args)):
+        smmry = web.getJSON(URL + "&SM_URL=" + quote(" ".join(msg.args)), timeout=23)
+    else:
+        cnt = ""
+        for r in context.subtreat(context.subparse(msg, " ".join(msg.args))):
+            if isinstance(r, Response):
+                for i in range(len(r.messages) - 1, -1, -1):
+                    if isinstance(r.messages[i], list):
+                        for j in range(len(r.messages[i]) - 1, -1, -1):
+                            cnt += r.messages[i][j] + "\n"
+                    elif isinstance(r.messages[i], str):
+                        cnt += r.messages[i] + "\n"
+                    else:
+                        cnt += str(r.messages) + "\n"
+
+            elif isinstance(r, Text):
+                cnt += r.message + "\n"
+
+            else:
+                cnt += str(r) + "\n"
+
+        smmry = web.getJSON(URL, body="sm_api_input=" + quote(cnt), timeout=23)
+
+    if "sm_api_error" in smmry:
+        if smmry["sm_api_error"] == 0:
+            title = "Internal server problem (not your fault)"
+        elif smmry["sm_api_error"] == 1:
+            title = "Incorrect submission variables"
+        elif smmry["sm_api_error"] == 2:
+            title = "Intentional restriction (low credits?)"
+        elif smmry["sm_api_error"] == 3:
+            title = "Summarization error"
+        else:
+            title = "Unknown error"
+        raise IMException(title + ": " + smmry['sm_api_message'].lower())
+
+    if "keywords" in msg.kwargs:
+        smmry["sm_api_content"] = ", ".join(smmry["sm_api_keyword_array"])
+
+    if "sm_api_title" in smmry and smmry["sm_api_title"] != "":
+        res.append_message(smmry["sm_api_content"], title=smmry["sm_api_title"])
+    else:
+        res.append_message(smmry["sm_api_content"])
+
+    return res

modules/sms.py

@@ -16,7 +16,7 @@ from nemubot.tools.xmlparser.node import ModuleState
 
 nemubotversion = 3.4
 
-from more import Response
+from nemubot.module.more import Response
 
 def load(context):
     context.data.setIndex("name", "phone")
@@ -46,47 +46,89 @@ def send_sms(frm, api_usr, api_key, content):
 
     return None
 
+def check_sms_dests(dests, cur_epoch):
+    """Raise exception if one of the dest is not known or has already receive a SMS recently
+    """
+    for u in dests:
+        if u not in context.data.index:
+            raise IMException("Désolé, je sais pas comment envoyer de SMS à %s." % u)
+        elif cur_epoch - float(context.data.index[u]["lastuse"]) < 42:
+            raise IMException("Un peu de calme, %s a déjà reçu un SMS il n'y a pas si longtemps." % u)
+    return True
+
+
+def send_sms_to_list(msg, frm, dests, content, cur_epoch):
+    fails = list()
+    for u in dests:
+        context.data.index[u]["lastuse"] = cur_epoch
+        test = send_sms(frm, context.data.index[u]["user"], context.data.index[u]["key"], content)
+        if test is not None:
+            fails.append( "%s: %s" % (u, test) )
+
+    if len(fails) > 0:
+        return Response("quelque chose ne s'est pas bien passé durant l'envoi du SMS : " + ", ".join(fails), msg.channel, msg.frm)
+    else:
+        return Response("le SMS a bien été envoyé", msg.channel, msg.frm)
+
 
 @hook.command("sms")
 def cmd_sms(msg):
     if not len(msg.args):
         raise IMException("À qui veux-tu envoyer ce SMS ?")
 
-    # Check dests
-    cur_epoch = time.mktime(time.localtime());
-    for u in msg.args[0].split(","):
-        if u not in context.data.index:
-            raise IMException("Désolé, je sais pas comment envoyer de SMS à %s." % u)
-        elif cur_epoch - float(context.data.index[u]["lastuse"]) < 42:
-            raise IMException("Un peu de calme, %s a déjà reçu un SMS il n'y a pas si longtemps." % u)
+    cur_epoch = time.mktime(time.localtime())
+    dests = msg.args[0].split(",")
+    frm = msg.frm if msg.to_response[0] == msg.frm else msg.frm + "@" + msg.to[0]
+    content = " ".join(msg.args[1:])
 
-    # Go!
-    fails = list()
-    for u in msg.args[0].split(","):
-        context.data.index[u]["lastuse"] = cur_epoch
-        if msg.to_response[0] == msg.frm:
-            frm = msg.frm
-        else:
-            frm = msg.frm + "@" + msg.to[0]
-        test = send_sms(frm, context.data.index[u]["user"], context.data.index[u]["key"], " ".join(msg.args[1:]))
-        if test is not None:
-            fails.append( "%s: %s" % (u, test) )
+    check_sms_dests(dests, cur_epoch)
+    return send_sms_to_list(msg, frm, dests, content, cur_epoch)
+
+
+@hook.command("smscmd")
+def cmd_smscmd(msg):
+    if not len(msg.args):
+        raise IMException("À qui veux-tu envoyer ce SMS ?")
+
+    cur_epoch = time.mktime(time.localtime())
+    dests = msg.args[0].split(",")
+    frm = msg.frm if msg.to_response[0] == msg.frm else msg.frm + "@" + msg.to[0]
+    cmd = " ".join(msg.args[1:])
+
+    content = None
+    for r in context.subtreat(context.subparse(msg, cmd)):
+        if isinstance(r, Response):
+            for m in r.messages:
+                if isinstance(m, list):
+                    for n in m:
+                        content = n
+                        break
+                    if content is not None:
+                        break
+                elif isinstance(m, str):
+                    content = m
+                    break
+
+        elif isinstance(r, Text):
+            content = r.message
+
+    if content is None:
+        raise IMException("Aucun SMS envoyé : le résultat de la commande n'a pas retourné de contenu.")
+
+    check_sms_dests(dests, cur_epoch)
+    return send_sms_to_list(msg, frm, dests, content, cur_epoch)
 
-    if len(fails) > 0:
-        return Response("quelque chose ne s'est pas bien passé durant l'envoi du SMS : " + ", ".join(fails), msg.channel, msg.nick)
-    else:
-        return Response("le SMS a bien été envoyé", msg.channel, msg.nick)
 
 apiuser_ask = re.compile(r"(utilisateur|user|numéro|numero|compte|abonne|abone|abonné|account)\s+(est|is)\s+(?P<user>[0-9]{7,})", re.IGNORECASE)
 apikey_ask = re.compile(r"(clef|key|password|mot de passe?)\s+(?:est|is)?\s+(?P<key>[a-zA-Z0-9]{10,})", re.IGNORECASE)
 
 @hook.ask()
 def parseask(msg):
-    if msg.text.find("Free") >= 0 and (
-            msg.text.find("API") >= 0 or msg.text.find("api") >= 0) and (
-                msg.text.find("SMS") >= 0 or msg.text.find("sms") >= 0):
-        resuser = apiuser_ask.search(msg.text)
-        reskey = apikey_ask.search(msg.text)
+    if msg.message.find("Free") >= 0 and (
+            msg.message.find("API") >= 0 or msg.message.find("api") >= 0) and (
+                msg.message.find("SMS") >= 0 or msg.message.find("sms") >= 0):
+        resuser = apiuser_ask.search(msg.message)
+        reskey = apikey_ask.search(msg.message)
         if resuser is not None and reskey is not None:
             apiuser = resuser.group("user")
             apikey = reskey.group("key")
@@ -94,18 +136,18 @@ def parseask(msg):
             test = send_sms("nemubot", apiuser, apikey,
                             "Vous avez enregistré vos codes d'authentification dans nemubot, félicitation !")
             if test is not None:
-                return Response("je n'ai pas pu enregistrer tes identifiants : %s" % test, msg.channel, msg.nick)
+                return Response("je n'ai pas pu enregistrer tes identifiants : %s" % test, msg.channel, msg.frm)
 
-            if msg.nick in context.data.index:
-                context.data.index[msg.nick]["user"] = apiuser
-                context.data.index[msg.nick]["key"] = apikey
+            if msg.frm in context.data.index:
+                context.data.index[msg.frm]["user"] = apiuser
+                context.data.index[msg.frm]["key"] = apikey
             else:
                 ms = ModuleState("phone")
-                ms.setAttribute("name", msg.nick)
+                ms.setAttribute("name", msg.frm)
                 ms.setAttribute("user", apiuser)
                 ms.setAttribute("key", apikey)
                 ms.setAttribute("lastuse", 0)
                 context.data.addChild(ms)
             context.save()
             return Response("ok, c'est noté. Je t'ai envoyé un SMS pour tester ;)",
-                            msg.channel, msg.nick)
+                            msg.channel, msg.frm)

modules/spell/__init__.py

@@ -10,7 +10,7 @@ from nemubot.tools.xmlparser.node import ModuleState
 from .pyaspell import Aspell
 from .pyaspell import AspellError
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################
@@ -64,15 +64,15 @@ def cmd_spell(msg):
             raise IMException("Je n'ai pas le dictionnaire `%s' :(" % lang)
 
         if r == True:
-            add_score(msg.nick, "correct")
+            add_score(msg.frm, "correct")
             res.append_message("l'orthographe de `%s' est correcte" % word)
 
         elif len(r) > 0:
-            add_score(msg.nick, "bad")
+            add_score(msg.frm, "bad")
             res.append_message(r, title="suggestions pour `%s'" % word)
 
         else:
-            add_score(msg.nick, "bad")
+            add_score(msg.frm, "bad")
             res.append_message("aucune suggestion pour `%s'" % word)
 
     return res

modules/suivi.py

@@ -2,47 +2,52 @@
 
 # PYTHON STUFF ############################################
 
-import urllib.request
+import json
 import urllib.parse
 from bs4 import BeautifulSoup
 import re
 
 from nemubot.hooks import hook
 from nemubot.exception import IMException
-from nemubot.tools.web import getURLContent
-from more import Response
+from nemubot.tools.web import getURLContent, getURLHeaders, getJSON
+from nemubot.module.more import Response
 
 
 # POSTAGE SERVICE PARSERS ############################################
 
 def get_tnt_info(track_id):
-    data = getURLContent('www.tnt.fr/public/suivi_colis/recherche/'
-                         'visubontransport.do?bonTransport=%s' % track_id)
+    values = []
+    data = getURLContent('https://www.tnt.fr/public/suivi_colis/recherche/visubontransport.do?bonTransport=%s' % track_id)
     soup = BeautifulSoup(data)
-    status = soup.find('p', class_='suivi-title-selected')
-    if status:
-        return status.get_text()
+    status_list = soup.find('div', class_='result__content')
+    if not status_list:
+        return None
+    last_status = status_list.find('div', class_='roster')
+    if last_status:
+        for info in last_status.find_all('div', class_='roster__item'):
+            values.append(info.get_text().strip())
+    if len(values) == 3:
+        return (values[0], values[1], values[2])
 
 
 def get_colissimo_info(colissimo_id):
-    colissimo_data = getURLContent("http://www.colissimo.fr/portail_colissimo/"
-                                   "suivre.do?colispart=%s" % colissimo_id)
+    colissimo_data = getURLContent("https://www.laposte.fr/particulier/outils/suivre-vos-envois?code=%s" % colissimo_id)
     soup = BeautifulSoup(colissimo_data)
 
-    dataArray = soup.find(class_='dataArray')
-    if dataArray and dataArray.tbody and dataArray.tbody.tr:
-        date = dataArray.tbody.tr.find(headers="Date").get_text()
-        libelle = re.sub(r'[\n\t\r]', '',
-                         dataArray.tbody.tr.find(headers="Libelle").get_text())
-        site = dataArray.tbody.tr.find(headers="site").get_text().strip()
-        return (date, libelle, site.strip())
+    dataArray = soup.find(class_='results-suivi')
+    if dataArray and dataArray.table and dataArray.table.tbody and dataArray.table.tbody.tr:
+        td = dataArray.table.tbody.tr.find_all('td')
+        if len(td) > 2:
+            date = td[0].get_text()
+            libelle = re.sub(r'[\n\t\r]', '', td[1].get_text())
+            site = td[2].get_text().strip()
+            return (date, libelle, site.strip())
 
 
 def get_chronopost_info(track_id):
     data = urllib.parse.urlencode({'listeNumeros': track_id})
-    track_baseurl = "http://www.chronopost.fr/expedier/" \
-        "inputLTNumbersNoJahia.do?lang=fr_FR"
-    track_data = urllib.request.urlopen(track_baseurl, data.encode('utf-8'))
+    track_baseurl = "https://www.chronopost.fr/expedier/inputLTNumbersNoJahia.do?lang=fr_FR"
+    track_data = getURLContent(track_baseurl, data.encode('utf-8'))
     soup = BeautifulSoup(track_data)
 
     infoClass = soup.find(class_='numeroColi2')
@@ -58,9 +63,8 @@ def get_chronopost_info(track_id):
 
 def get_colisprive_info(track_id):
     data = urllib.parse.urlencode({'numColis': track_id})
-    track_baseurl = "https://www.colisprive.com/moncolis/pages/" \
-        "detailColis.aspx"
-    track_data = urllib.request.urlopen(track_baseurl, data.encode('utf-8'))
+    track_baseurl = "https://www.colisprive.com/moncolis/pages/detailColis.aspx"
+    track_data = getURLContent(track_baseurl, data.encode('utf-8'))
     soup = BeautifulSoup(track_data)
 
     dataArray = soup.find(class_='BandeauInfoColis')
@@ -71,34 +75,117 @@ def get_colisprive_info(track_id):
         return status
 
 
+def get_ups_info(track_id):
+    data = json.dumps({'Locale': "en_US", 'TrackingNumber': [track_id]})
+    track_baseurl = "https://www.ups.com/track/api/Track/GetStatus?loc=en_US"
+    track_data = getJSON(track_baseurl, data.encode('utf-8'), header={"Content-Type": "application/json"})
+    return (track_data["trackDetails"][0]["trackingNumber"],
+            track_data["trackDetails"][0]["packageStatus"],
+            track_data["trackDetails"][0]["shipmentProgressActivities"][0]["date"] + " " + track_data["trackDetails"][0]["shipmentProgressActivities"][0]["time"],
+            track_data["trackDetails"][0]["shipmentProgressActivities"][0]["location"],
+            track_data["trackDetails"][0]["shipmentProgressActivities"][0]["activityScan"])
+
+
 def get_laposte_info(laposte_id):
-    data = urllib.parse.urlencode({'id': laposte_id})
-    laposte_baseurl = "http://www.part.csuivi.courrier.laposte.fr/suivi/index"
+    status, laposte_headers = getURLHeaders("https://www.laposte.fr/outils/suivre-vos-envois?" + urllib.parse.urlencode({'code': laposte_id}))
 
-    laposte_data = urllib.request.urlopen(laposte_baseurl,
-                                          data.encode('utf-8'))
-    soup = BeautifulSoup(laposte_data)
-    search_res = soup.find(class_='resultat_rech_simple_table').tbody.tr
-    if (soup.find(class_='resultat_rech_simple_table').thead
-            and soup.find(class_='resultat_rech_simple_table').thead.tr
-            and len(search_res.find_all('td')) > 3):
-        field = search_res.find('td')
-        poste_id = field.get_text()
+    laposte_cookie = None
+    for k,v in laposte_headers:
+        if k.lower() == "set-cookie" and v.find("access_token") >= 0:
+            laposte_cookie = v.split(";")[0]
 
-        field = field.find_next('td')
-        poste_type = field.get_text()
+    laposte_data = getJSON("https://api.laposte.fr/ssu/v1/suivi-unifie/idship/%s?lang=fr_FR" % urllib.parse.quote(laposte_id), header={"Accept": "application/json", "Cookie": laposte_cookie})
 
-        field = field.find_next('td')
-        poste_date = field.get_text()
+    shipment = laposte_data["shipment"]
+    return (shipment["product"], shipment["idShip"], shipment["event"][0]["label"], shipment["event"][0]["date"])
 
-        field = field.find_next('td')
-        poste_location = field.get_text()
 
-        field = field.find_next('td')
-        poste_status = field.get_text()
+def get_postnl_info(postnl_id):
+    data = urllib.parse.urlencode({'barcodes': postnl_id})
+    postnl_baseurl = "http://www.postnl.post/details/"
 
-        return (poste_type.lower(), poste_id.strip(), poste_status.lower(),
-                poste_location, poste_date)
+    postnl_data = getURLContent(postnl_baseurl, data.encode('utf-8'))
+    soup = BeautifulSoup(postnl_data)
+    if (soup.find(id='datatables')
+            and soup.find(id='datatables').tbody
+            and soup.find(id='datatables').tbody.tr):
+        search_res = soup.find(id='datatables').tbody.tr
+        if len(search_res.find_all('td')) >= 3:
+            field = field.find_next('td')
+            post_date = field.get_text()
+
+            field = field.find_next('td')
+            post_status = field.get_text()
+
+            field = field.find_next('td')
+            post_destination = field.get_text()
+
+            return (post_status.lower(), post_destination, post_date)
+
+
+def get_usps_info(usps_id):
+    usps_parcelurl = "https://tools.usps.com/go/TrackConfirmAction_input?" + urllib.parse.urlencode({'qtc_tLabels1': usps_id})
+
+    usps_data = getURLContent(usps_parcelurl)
+    soup = BeautifulSoup(usps_data)
+    if (soup.find(id="trackingHistory_1")
+            and soup.find(class_="tracking_history").find(class_="row_notification")
+            and soup.find(class_="tracking_history").find(class_="row_top").find_all("td")):
+        notification = soup.find(class_="tracking_history").find(class_="row_notification").text.strip()
+        date = re.sub(r"\s+", " ", soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[0].text.strip())
+        status = soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[1].text.strip()
+        last_location = soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[2].text.strip()
+
+        print(notification)
+
+        return (notification, date, status, last_location)
+
+
+def get_fedex_info(fedex_id, lang="en_US"):
+    data = urllib.parse.urlencode({
+        'data': json.dumps({
+            "TrackPackagesRequest": {
+                "appType": "WTRK",
+                "appDeviceType": "DESKTOP",
+                "uniqueKey": "",
+                "processingParameters": {},
+                "trackingInfoList": [
+                    {
+                        "trackNumberInfo": {
+                            "trackingNumber": str(fedex_id),
+                            "trackingQualifier": "",
+                            "trackingCarrier": ""
+                        }
+                    }
+                ]
+            }
+        }),
+        'action': "trackpackages",
+        'locale': lang,
+        'version': 1,
+        'format': "json"
+    })
+    fedex_baseurl = "https://www.fedex.com/trackingCal/track"
+
+    fedex_data = getJSON(fedex_baseurl, data.encode('utf-8'))
+
+    if ("TrackPackagesResponse" in fedex_data and
+        "packageList" in fedex_data["TrackPackagesResponse"] and
+        len(fedex_data["TrackPackagesResponse"]["packageList"]) and
+        (not fedex_data["TrackPackagesResponse"]["errorList"][0]["code"] or
+        fedex_data["TrackPackagesResponse"]["errorList"][0]["code"] == '0') and
+        not fedex_data["TrackPackagesResponse"]["packageList"][0]["errorList"][0]["code"]
+    ):
+        return fedex_data["TrackPackagesResponse"]["packageList"][0]
+
+
+def get_dhl_info(dhl_id, lang="en"):
+    dhl_parcelurl = "http://www.dhl.com/shipmentTracking?" + urllib.parse.urlencode({'AWB': dhl_id})
+
+    dhl_data = getJSON(dhl_parcelurl)
+
+    if "results" in dhl_data and dhl_data["results"]:
+        return dhl_data["results"][0]
 
 
 # TRACKING HANDLERS ###################################################
@@ -106,18 +193,46 @@ def get_laposte_info(laposte_id):
 def handle_tnt(tracknum):
     info = get_tnt_info(tracknum)
     if info:
+        status, date, place = info
+        placestr = ''
+        if place:
+            placestr = ' à \x02{place}\x0f'
         return ('Le colis \x02{trackid}\x0f a actuellement le status: '
-                '\x02{status}\x0F'.format(trackid=tracknum, status=info))
+                '\x02{status}\x0F mis à jour le \x02{date}\x0f{place}.'
+                .format(trackid=tracknum, status=status,
+                        date=re.sub(r'\s+', ' ', date), place=placestr))
 
 
 def handle_laposte(tracknum):
     info = get_laposte_info(tracknum)
     if info:
-        poste_type, poste_id, poste_status, poste_location, poste_date = info
-        return ("Le courrier de type \x02%s\x0F : \x02%s\x0F est actuellement "
-                "\x02%s\x0F dans la zone \x02%s\x0F (Mis à jour le \x02%s\x0F"
-                ")." % (poste_type, poste_id, poste_status,
-                        poste_location, poste_date))
+        poste_type, poste_id, poste_status, poste_date = info
+        return ("\x02%s\x0F : \x02%s\x0F est actuellement "
+                "\x02%s\x0F (Mis à jour le \x02%s\x0F"
+                ")." % (poste_type, poste_id, poste_status, poste_date))
+
+
+def handle_postnl(tracknum):
+    info = get_postnl_info(tracknum)
+    if info:
+        post_status, post_destination, post_date = info
+        return ("PostNL \x02%s\x0F est actuellement "
+                "\x02%s\x0F vers le pays \x02%s\x0F (Mis à jour le \x02%s\x0F"
+                ")." % (tracknum, post_status, post_destination, post_date))
+
+
+def handle_usps(tracknum):
+    info = get_usps_info(tracknum)
+    if info:
+        notif, last_date, last_status, last_location = info
+        return ("USPS \x02{tracknum}\x0F: {last_status} in \x02{last_location}\x0F as of {last_date}: {notif}".format(tracknum=tracknum, notif=notif, last_date=last_date, last_status=last_status.lower(), last_location=last_location))
+
+
+def handle_ups(tracknum):
+    info = get_ups_info(tracknum)
+    if info:
+        tracknum, status, last_date, last_location, last_status  = info
+        return ("UPS \x02{tracknum}\x0F: {status}: in \x02{last_location}\x0F as of {last_date}: {last_status}".format(tracknum=tracknum, status=status, last_date=last_date, last_status=last_status.lower(), last_location=last_location))
 
 
 def handle_colissimo(tracknum):
@@ -143,12 +258,34 @@ def handle_coliprive(tracknum):
         return ("Colis Privé: \x02%s\x0F : \x02%s\x0F." % (tracknum, info))
 
 
+def handle_fedex(tracknum):
+    info = get_fedex_info(tracknum)
+    if info:
+        if info["displayActDeliveryDateTime"] != "":
+            return ("{trackingCarrierDesc}: \x02{statusWithDetails}\x0F: in \x02{statusLocationCity}, {statusLocationCntryCD}\x0F, delivered on: {displayActDeliveryDateTime}.".format(**info))
+        elif info["statusLocationCity"] != "":
+            return ("{trackingCarrierDesc}: \x02{statusWithDetails}\x0F: estimated delivery: {displayEstDeliveryDateTime}.".format(**info))
+        else:
+            return ("{trackingCarrierDesc}: \x02{statusWithDetails}\x0F: in \x02{statusLocationCity}, {statusLocationCntryCD}\x0F, estimated delivery: {displayEstDeliveryDateTime}.".format(**info))
+
+
+def handle_dhl(tracknum):
+    info = get_dhl_info(tracknum)
+    if info:
+        return "DHL {label} {id}: \x02{description}\x0F".format(**info)
+
+
 TRACKING_HANDLERS = {
     'laposte': handle_laposte,
+    'postnl': handle_postnl,
     'colissimo': handle_colissimo,
     'chronopost': handle_chronopost,
     'coliprive': handle_coliprive,
     'tnt': handle_tnt,
+    'fedex': handle_fedex,
+    'dhl': handle_dhl,
+    'usps': handle_usps,
+    'ups': handle_ups,
 }
 
 

modules/syno.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################
@@ -29,7 +29,7 @@ def load(context):
 # MODULE CORE #########################################################
 
 def get_french_synos(word):
-    url = "http://www.crisco.unicaen.fr/des/synonymes/" + quote(word.encode("ISO-8859-1"))
+    url = "https://crisco.unicaen.fr/des/synonymes/" + quote(word)
     page = web.getURLContent(url)
 
     best = list(); synos = list(); anton = list()
@@ -53,7 +53,7 @@ def get_french_synos(word):
 
 
 def get_english_synos(key, word):
-    cnt = web.getJSON("http://words.bighugelabs.com/api/2/%s/%s/json" %
+    cnt = web.getJSON("https://words.bighugelabs.com/api/2/%s/%s/json" %
                       (quote(key), quote(word.encode("ISO-8859-1"))))
 
     best = list(); synos = list(); anton = list()

modules/tpb.py

@@ -8,7 +8,7 @@ from nemubot.tools.web import getJSON
 
 nemubotversion = 4.0
 
-from more import Response
+from nemubot.module.more import Response
 
 URL_TPBAPI = None
 

modules/translate.py

@@ -8,7 +8,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # GLOBALS #############################################################

modules/urbandict.py

@@ -8,13 +8,13 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 # MODULE CORE #########################################################
 
 def search(terms):
     return web.getJSON(
-        "http://api.urbandictionary.com/v0/define?term=%s"
+        "https://api.urbandictionary.com/v0/define?term=%s"
         % quote(' '.join(terms)))
 
 

modules/urlreducer.py

@@ -21,7 +21,7 @@ def default_reducer(url, data):
 
 
 def ycc_reducer(url, data):
-    return "http://ycc.fr/%s" % default_reducer(url, data)
+    return "https://ycc.fr/%s" % default_reducer(url, data)
 
 def lstu_reducer(url, data):
     json_data = json.loads(web.getURLContent(url, "lsturl=" + quote(data),
@@ -36,8 +36,8 @@ def lstu_reducer(url, data):
 # MODULE VARIABLES ####################################################
 
 PROVIDERS = {
-    "tinyurl": (default_reducer, "http://tinyurl.com/api-create.php?url="),
-    "ycc": (ycc_reducer, "http://ycc.fr/redirection/create/"),
+    "tinyurl": (default_reducer, "https://tinyurl.com/api-create.php?url="),
+    "ycc": (ycc_reducer, "https://ycc.fr/redirection/create/"),
     "framalink": (lstu_reducer, "https://frama.link/a?format=json"),
     "huitre": (lstu_reducer, "https://huit.re/a?format=json"),
     "lstu": (lstu_reducer, "https://lstu.fr/a?format=json"),
@@ -60,12 +60,20 @@ def load(context):
 
 # MODULE CORE #########################################################
 
-def reduce(url, provider=DEFAULT_PROVIDER):
+def reduce_inline(txt, provider=None):
+    for url in re.findall("([a-zA-Z0-9+.-]+:(?://)?(?:[^ :/]+:[0-9]+)?[^ :]+)", txt):
+        txt = txt.replace(url, reduce(url, provider))
+    return txt
+
+
+def reduce(url, provider=None):
     """Ask the url shortner website to reduce given URL
 
     Argument:
     url -- the URL to reduce
     """
+    if provider is None:
+        provider = DEFAULT_PROVIDER
     return PROVIDERS[provider][0](PROVIDERS[provider][1], url)
 
 
@@ -84,15 +92,30 @@ LAST_URLS = dict()
 
 @hook.message()
 def parselisten(msg):
-    parseresponse(msg)
-    return None
+    global LAST_URLS
+    if hasattr(msg, "message") and isinstance(msg.message, str):
+        urls = re.findall("([a-zA-Z0-9+.-]+:(?://)?(?:[^ :/]+:[0-9]+)?[^ :]+)",
+                          msg.message)
+        for url in urls:
+            o = urlparse(web._getNormalizedURL(url), "http")
+
+            # Skip short URLs
+            if (o.netloc == "" or o.netloc in PROVIDERS or
+                    len(o.netloc) + len(o.path) < 17):
+                continue
+
+            for recv in msg.to:
+                if recv not in LAST_URLS:
+                    LAST_URLS[recv] = list()
+                LAST_URLS[recv].append(url)
 
 
 @hook.post()
 def parseresponse(msg):
     global LAST_URLS
     if hasattr(msg, "text") and isinstance(msg.text, str):
-        urls = re.findall("([a-zA-Z0-9+.-]+:(?://)?(?:[^ :/]+:[0-9]+)?[^ :]+)", msg.text)
+        urls = re.findall("([a-zA-Z0-9+.-]+:(?://)?(?:[^ :/]+:[0-9]+)?[^ :]+)",
+                          msg.text)
         for url in urls:
             o = urlparse(web._getNormalizedURL(url), "http")
 
@@ -130,7 +153,7 @@ def cmd_reduceurl(msg):
             raise IMException("I have no more URL to reduce.")
 
     if len(msg.args) > 4:
-        raise IMException("I cannot reduce that maby URLs at once.")
+        raise IMException("I cannot reduce that many URLs at once.")
     else:
         minify += msg.args
 

modules/velib.py

@@ -9,7 +9,7 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################

modules/virtualradar.py

@@ -0,0 +1,100 @@
+"""Retrieve flight information from VirtualRadar APIs"""
+
+# PYTHON STUFFS #######################################################
+
+import re
+from urllib.parse import quote
+import time
+
+from nemubot.exception import IMException
+from nemubot.hooks import hook
+from nemubot.tools import web
+
+from nemubot.module.more import Response
+from nemubot.module import mapquest
+
+# GLOBALS #############################################################
+
+URL_API = "https://public-api.adsbexchange.com/VirtualRadar/AircraftList.json?fcallC=%s"
+
+SPEED_TYPES = {
+        0: 'Ground speed',
+        1: 'Ground speed reversing',
+        2: 'Indicated air speed',
+        3: 'True air speed'}
+
+WTC_CAT = {
+        0: 'None',
+        1: 'Light',
+        2: 'Medium',
+        3: 'Heavy'
+        }
+
+SPECIES = {
+        1: 'Land plane',
+        2: 'Sea plane',
+        3: 'Amphibian',
+        4: 'Helicopter',
+        5: 'Gyrocopter',
+        6: 'Tiltwing',
+        7: 'Ground vehicle',
+        8: 'Tower'}
+
+HANDLER_TABLE = {
+        'From': lambda x: 'From: \x02%s\x0F' % x,
+        'To': lambda x: 'To: \x02%s\x0F' % x,
+        'Op': lambda x: 'Airline: \x02%s\x0F' % x,
+        'Mdl': lambda x: 'Model: \x02%s\x0F' % x,
+        'Call': lambda x: 'Flight: \x02%s\x0F' % x,
+        'PosTime': lambda x: 'Last update: \x02%s\x0F' % (time.ctime(int(x)/1000)),
+        'Alt': lambda x: 'Altitude: \x02%s\x0F ft' % x,
+        'Spd': lambda x: 'Speed: \x02%s\x0F kn' % x,
+        'SpdTyp': lambda x: 'Speed type: \x02%s\x0F' % SPEED_TYPES[x] if x in SPEED_TYPES  else None,
+        'Engines': lambda x: 'Engines: \x02%s\x0F' % x,
+        'Gnd': lambda x: 'On the ground' if x else None,
+        'Mil': lambda x: 'Military aicraft' if x else None,
+        'Species': lambda x: 'Aircraft species: \x02%s\x0F' % SPECIES[x] if x in SPECIES  else None,
+        'WTC': lambda x: 'Turbulence level: \x02%s\x0F' % WTC_CAT[x] if x in WTC_CAT  else None,
+        }
+
+# MODULE CORE #########################################################
+
+def virtual_radar(flight_call):
+    obj = web.getJSON(URL_API % quote(flight_call))
+
+    if "acList" in obj:
+        for flight in obj["acList"]:
+            yield flight
+
+def flight_info(flight):
+    for prop in HANDLER_TABLE:
+        if prop in flight:
+            yield HANDLER_TABLE[prop](flight[prop])
+
+# MODULE INTERFACE ####################################################
+
+@hook.command("flight",
+              help="Get flight information",
+              help_usage={ "FLIGHT": "Get information on FLIGHT" })
+def cmd_flight(msg):
+    if not len(msg.args):
+        raise IMException("please indicate a flight")
+
+    res = Response(channel=msg.channel, nick=msg.frm,
+                   nomore="No more flights", count=" (%s more flights)")
+
+    for param in msg.args:
+        for flight in virtual_radar(param):
+            if 'Lat' in flight and 'Long' in flight:
+                loc = None
+                for location in  mapquest.geocode('{Lat},{Long}'.format(**flight)):
+                    loc = location
+                    break
+                if loc:
+                    res.append_message('\x02{0}\x0F: Position: \x02{1}\x0F, {2}'.format(flight['Call'], \
+                            mapquest.where(loc), \
+                            ', '.join(filter(None, flight_info(flight)))))
+                    continue
+            res.append_message('\x02{0}\x0F: {1}'.format(flight['Call'], \
+                        ', '.join(filter(None, flight_info(flight)))))
+    return res

modules/weather.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 
-"""The weather module"""
+"""The weather module. Powered by Dark Sky <https://darksky.net/poweredby/>"""
 
 import datetime
 import re
@@ -11,71 +11,71 @@ from nemubot.hooks import hook
 from nemubot.tools import web
 from nemubot.tools.xmlparser.node import ModuleState
 
-import mapquest
+from nemubot.module import mapquest
 
 nemubotversion = 4.0
 
-from more import Response
+from nemubot.module.more import Response
 
-URL_DSAPI = "https://api.forecast.io/forecast/%s/%%s,%%s"
+URL_DSAPI = "https://api.darksky.net/forecast/%s/%%s,%%s?lang=%%s&units=%%s"
+
+UNITS = {
+    "ca": {
+        "temperature": "°C",
+        "distance": "km",
+        "precipIntensity": "mm/h",
+        "precip": "cm",
+        "speed": "km/h",
+        "pressure": "hPa",
+    },
+    "uk2": {
+        "temperature": "°C",
+        "distance": "mi",
+        "precipIntensity": "mm/h",
+        "precip": "cm",
+        "speed": "mi/h",
+        "pressure": "hPa",
+    },
+    "us": {
+        "temperature": "°F",
+        "distance": "mi",
+        "precipIntensity": "in/h",
+        "precip": "in",
+        "speed": "mi/h",
+        "pressure": "mbar",
+    },
+    "si": {
+        "temperature": "°C",
+        "distance": "km",
+        "precipIntensity": "mm/h",
+        "precip": "cm",
+        "speed": "m/s",
+        "pressure": "hPa",
+    },
+}
 
 def load(context):
     if not context.config or "darkskyapikey" not in context.config:
         raise ImportError("You need a Dark-Sky API key in order to use this "
                           "module. Add it to the module configuration file:\n"
                           "<module name=\"weather\" darkskyapikey=\"XXX\" />\n"
-                          "Register at http://developer.forecast.io/")
+                          "Register at https://developer.forecast.io/")
     context.data.setIndex("name", "city")
     global URL_DSAPI
     URL_DSAPI = URL_DSAPI % context.config["darkskyapikey"]
 
 
-def help_full ():
-    return "!weather /city/: Display the current weather in /city/."
+def format_wth(wth, flags):
+    units = UNITS[flags["units"] if "units" in flags and flags["units"] in UNITS else "si"]
+    return ("{temperature} {units[temperature]} {summary}; precipitation ({precipProbability:.0%} chance) intensity: {precipIntensity} {units[precipIntensity]}; relative humidity: {humidity:.0%}; wind speed: {windSpeed} {units[speed]} {windBearing}°; cloud coverage: {cloudCover:.0%}; pressure: {pressure} {units[pressure]}; ozone: {ozone} DU"
+            .format(units=units, **wth)
+           )
 
 
-def fahrenheit2celsius(temp):
-    return int((temp - 32) * 50/9)/10
-
-
-def mph2kmph(speed):
-    return int(speed * 160.9344)/100
-
-
-def inh2mmh(size):
-    return int(size * 254)/10
-
-
-def format_wth(wth):
-    return ("%s °C %s; precipitation (%s %% chance) intensity: %s mm/h; relative humidity: %s %%; wind speed: %s km/h %s°; cloud coverage: %s %%; pressure: %s hPa; ozone: %s DU" %
-           (
-               fahrenheit2celsius(wth["temperature"]),
-               wth["summary"],
-               int(wth["precipProbability"] * 100),
-               inh2mmh(wth["precipIntensity"]),
-               int(wth["humidity"] * 100),
-               mph2kmph(wth["windSpeed"]),
-               wth["windBearing"],
-               int(wth["cloudCover"] * 100),
-               int(wth["pressure"]),
-               int(wth["ozone"])
-           ))
-
-
-def format_forecast_daily(wth):
-    return ("%s; between %s-%s °C; precipitation (%s %% chance) intensity: maximum %s mm/h; relative humidity: %s %%; wind speed: %s km/h %s°; cloud coverage: %s %%; pressure: %s hPa; ozone: %s DU" %
-           (
-               wth["summary"],
-               fahrenheit2celsius(wth["temperatureMin"]), fahrenheit2celsius(wth["temperatureMax"]),
-               int(wth["precipProbability"] * 100),
-               inh2mmh(wth["precipIntensityMax"]),
-               int(wth["humidity"] * 100),
-               mph2kmph(wth["windSpeed"]),
-               wth["windBearing"],
-               int(wth["cloudCover"] * 100),
-               int(wth["pressure"]),
-               int(wth["ozone"])
-           ))
+def format_forecast_daily(wth, flags):
+    units = UNITS[flags["units"] if "units" in flags and flags["units"] in UNITS else "si"]
+    print(units)
+    return ("{summary}; between {temperatureMin}-{temperatureMax} {units[temperature]}; precipitation ({precipProbability:.0%} chance) intensity: maximum {precipIntensity} {units[precipIntensity]}; relative humidity: {humidity:.0%}; wind speed: {windSpeed} {units[speed]} {windBearing}°; cloud coverage: {cloudCover:.0%}; pressure: {pressure} {units[pressure]}; ozone: {ozone} DU".format(units=units, **wth))
 
 
 def format_timestamp(timestamp, tzname, tzoffset, format="%c"):
@@ -126,8 +126,8 @@ def treat_coord(msg):
         raise IMException("indique-moi un nom de ville ou des coordonnées.")
 
 
-def get_json_weather(coords):
-    wth = web.getJSON(URL_DSAPI % (float(coords[0]), float(coords[1])))
+def get_json_weather(coords, lang="en", units="ca"):
+    wth = web.getJSON(URL_DSAPI % (float(coords[0]), float(coords[1]), lang, units))
 
     # First read flags
     if wth is None or "darksky-unavailable" in wth["flags"]:
@@ -149,31 +149,53 @@ def cmd_coordinates(msg):
     return Response("Les coordonnées de %s sont %s,%s" % (msg.args[0], coords["lat"], coords["long"]), channel=msg.channel)
 
 
-@hook.command("alert")
+@hook.command("alert",
+              keywords={
+                  "lang=LANG": "change the output language of weather sumarry; default: en",
+                  "units=UNITS": "return weather conditions in the requested units; default: ca",
+              })
 def cmd_alert(msg):
     loc, coords, specific = treat_coord(msg)
-    wth = get_json_weather(coords)
+    wth = get_json_weather(coords,
+                           lang=msg.kwargs["lang"] if "lang" in msg.kwargs else "en",
+                           units=msg.kwargs["units"] if "units" in msg.kwargs else "ca")
 
     res = Response(channel=msg.channel, nomore="No more weather alert", count=" (%d more alerts)")
 
     if "alerts" in wth:
         for alert in wth["alerts"]:
-            res.append_message("\x03\x02%s\x03\x02 (see %s expire on %s): %s" % (alert["title"], alert["uri"], format_timestamp(int(alert["expires"]), wth["timezone"], wth["offset"]), alert["description"].replace("\n", " ")))
+            if "expires" in alert:
+                res.append_message("\x03\x02%s\x03\x02 (see %s expire on %s): %s" % (alert["title"], alert["uri"], format_timestamp(int(alert["expires"]), wth["timezone"], wth["offset"]), alert["description"].replace("\n", " ")))
+            else:
+                res.append_message("\x03\x02%s\x03\x02 (see %s): %s" % (alert["title"], alert["uri"], alert["description"].replace("\n", " ")))
 
     return res
 
 
-@hook.command("météo")
+@hook.command("météo",
+              help="Display current weather and previsions",
+              help_usage={
+                  "CITY": "Display the current weather and previsions in CITY",
+              },
+              keywords={
+                  "lang=LANG": "change the output language of weather sumarry; default: en",
+                  "units=UNITS": "return weather conditions in the requested units; default: ca",
+              })
 def cmd_weather(msg):
     loc, coords, specific = treat_coord(msg)
-    wth = get_json_weather(coords)
+    wth = get_json_weather(coords,
+                           lang=msg.kwargs["lang"] if "lang" in msg.kwargs else "en",
+                           units=msg.kwargs["units"] if "units" in msg.kwargs else "ca")
 
     res = Response(channel=msg.channel, nomore="No more weather information")
 
     if "alerts" in wth:
         alert_msgs = list()
         for alert in wth["alerts"]:
-            alert_msgs.append("\x03\x02%s\x03\x02 expire on %s" % (alert["title"], format_timestamp(int(alert["expires"]), wth["timezone"], wth["offset"])))
+            if "expires" in alert:
+                alert_msgs.append("\x03\x02%s\x03\x02 expire on %s" % (alert["title"], format_timestamp(int(alert["expires"]), wth["timezone"], wth["offset"])))
+            else:
+                alert_msgs.append("\x03\x02%s\x03\x02" % (alert["title"]))
         res.append_message("\x03\x16\x03\x02/!\\\x03\x02 Alert%s:\x03\x16 " % ("s" if len(alert_msgs) > 1 else "") + ", ".join(alert_msgs))
 
     if specific is not None:
@@ -185,17 +207,17 @@ def cmd_weather(msg):
 
         if gr.group(2).lower() == "h" and gr1 < len(wth["hourly"]["data"]):
             hour = wth["hourly"]["data"][gr1]
-            res.append_message("\x03\x02At %sh:\x03\x02 %s" % (format_timestamp(int(hour["time"]), wth["timezone"], wth["offset"], '%H'), format_wth(hour)))
+            res.append_message("\x03\x02At %sh:\x03\x02 %s" % (format_timestamp(int(hour["time"]), wth["timezone"], wth["offset"], '%H'), format_wth(hour, wth["flags"])))
 
         elif gr.group(2).lower() == "d" and gr1 < len(wth["daily"]["data"]):
             day = wth["daily"]["data"][gr1]
-            res.append_message("\x03\x02On %s:\x03\x02 %s" % (format_timestamp(int(day["time"]), wth["timezone"], wth["offset"], '%A'), format_forecast_daily(day)))
+            res.append_message("\x03\x02On %s:\x03\x02 %s" % (format_timestamp(int(day["time"]), wth["timezone"], wth["offset"], '%A'), format_forecast_daily(day, wth["flags"])))
 
         else:
             res.append_message("I don't understand %s or information is not available" % specific)
 
     else:
-        res.append_message("\x03\x02Currently:\x03\x02 " + format_wth(wth["currently"]))
+        res.append_message("\x03\x02Currently:\x03\x02 " + format_wth(wth["currently"], wth["flags"]))
 
         nextres = "\x03\x02Today:\x03\x02 %s " % wth["daily"]["data"][0]["summary"]
         if "minutely" in wth:
@@ -205,11 +227,11 @@ def cmd_weather(msg):
 
         for hour in wth["hourly"]["data"][1:4]:
             res.append_message("\x03\x02At %sh:\x03\x02 %s" % (format_timestamp(int(hour["time"]), wth["timezone"], wth["offset"], '%H'),
-                                                               format_wth(hour)))
+                                                               format_wth(hour, wth["flags"])))
 
         for day in wth["daily"]["data"][1:]:
             res.append_message("\x03\x02On %s:\x03\x02 %s" % (format_timestamp(int(day["time"]), wth["timezone"], wth["offset"], '%A'),
-                                                              format_forecast_daily(day)))
+                                                              format_forecast_daily(day, wth["flags"])))
 
     return res
 
@@ -219,7 +241,7 @@ gps_ask = re.compile(r"^\s*(?P<city>.*\w)\s*(?:(?:se|est)\s+(?:trouve|situ[ée]*
 
 @hook.ask()
 def parseask(msg):
-    res = gps_ask.match(msg.text)
+    res = gps_ask.match(msg.message)
     if res is not None:
         city_name = res.group("city").lower()
         gps_lat = res.group("lat").replace(",", ".")
@@ -236,4 +258,4 @@ def parseask(msg):
             context.data.addChild(ms)
         context.save()
         return Response("ok, j'ai bien noté les coordonnées de %s" % res.group("city"),
-                        msg.channel, msg.nick)
+                        msg.channel, msg.frm)

modules/whois.py

@@ -1,5 +1,6 @@
 # coding=utf-8
 
+import json
 import re
 
 from nemubot import context
@@ -9,47 +10,70 @@ from nemubot.tools.xmlparser.node import ModuleState
 
 nemubotversion = 3.4
 
-from more import Response
-from networking.page import headers
+from nemubot.module.more import Response
+from nemubot.module.networking.page import headers
 
 PASSWD_FILE = None
+# You can get one with: curl -b "sessionid=YOURSESSIONID" 'https://accounts.cri.epita.net/api/users/?limit=10000' > users.json
+APIEXTRACT_FILE = None
 
 def load(context):
     global PASSWD_FILE
     if not context.config or "passwd" not in context.config:
         print("No passwd file given")
+    else:
+        PASSWD_FILE = context.config["passwd"]
+        print("passwd file loaded:", PASSWD_FILE)
+
+    global APIEXTRACT_FILE
+    if not context.config or "apiextract" not in context.config:
+        print("No passwd file given")
+    else:
+        APIEXTRACT_FILE = context.config["apiextract"]
+        print("JSON users file loaded:", APIEXTRACT_FILE)
+
+    if PASSWD_FILE is None and APIEXTRACT_FILE is None:
         return None
-    PASSWD_FILE = context.config["passwd"]
 
     if not context.data.hasNode("aliases"):
         context.data.addChild(ModuleState("aliases"))
     context.data.getNode("aliases").setIndex("from", "alias")
 
-    if not context.data.hasNode("pics"):
-        context.data.addChild(ModuleState("pics"))
-    context.data.getNode("pics").setIndex("login", "pict")
-
     import nemubot.hooks
-    context.add_hook(nemubot.hooks.Command(cmd_whois, "whois"),
+    context.add_hook(nemubot.hooks.Command(cmd_whois, "whois", keywords={"lookup": "Perform a lookup of the begining of the login instead of an exact search."}),
                      "in","Command")
 
 class Login:
 
-    def __init__(self, line):
-        s = line.split(":")
-        self.login = s[0]
-        self.uid = s[2]
-        self.gid = s[3]
-        self.cn = s[4]
-        self.home = s[5]
+    def __init__(self, line=None, login=None, uidNumber=None, firstname=None, lastname=None, promo=None, **kwargs):
+        if line is not None:
+            s = line.split(":")
+            self.login = s[0]
+            self.uid = s[2]
+            self.gid = s[3]
+            self.cn = s[4]
+            self.home = s[5]
+        else:
+            self.login = login
+            self.uid = uidNumber
+            self.promo = promo
+            self.cn = firstname + " " + lastname
+            try:
+                self.gid = "epita" + str(int(promo))
+            except:
+                self.gid = promo
 
     def get_promo(self):
-        return self.home.split("/")[2].replace("_", " ")
+        if hasattr(self, "promo"):
+            return self.promo
+        if hasattr(self, "home"):
+            try:
+                return self.home.split("/")[2].replace("_", " ")
+            except:
+                return self.gid
 
     def get_photo(self):
-        if self.login in context.data.getNode("pics").index:
-            return context.data.getNode("pics").index[self.login]["url"]
-        for url in [ "https://static.acu.epita.fr/photos/%s", "https://static.acu.epita.fr/photos/%s/%%s" % self.gid, "https://intra-bocal.epitech.eu/trombi/%s.jpg", "http://whois.23.tf/p/%s/%%s.jpg" % self.gid ]:
+        for url in [ "https://photos.cri.epita.fr/%s", "https://intra-bocal.epitech.eu/trombi/%s.jpg" ]:
             url = url % self.login
             try:
                 _, status, _, _ = headers(url)
@@ -60,38 +84,53 @@ class Login:
         return None
 
 
-def found_login(login):
+def login_lookup(login, search=False):
     if login in context.data.getNode("aliases").index:
         login = context.data.getNode("aliases").index[login]["to"]
 
-    login_ = login + ":"
+    if APIEXTRACT_FILE:
+        with open(APIEXTRACT_FILE, encoding="utf-8") as f:
+            api = json.load(f)
+            for l in api["results"]:
+                if (not search and l["login"] == login) or (search and (("login" in l and l["login"].find(login) != -1) or ("cn" in l and l["cn"].find(login) != -1) or ("uid" in l and str(l["uid"]) == login))):
+                    yield Login(**l)
+
+    login_ = login + (":" if not search else "")
     lsize = len(login_)
 
-    with open(PASSWD_FILE, encoding="iso-8859-15") as f:
-        for l in f.readlines():
-            if l[:lsize] == login_:
-                return Login(l.strip())
-    return None
+    if PASSWD_FILE:
+        with open(PASSWD_FILE, encoding="iso-8859-15") as f:
+            for l in f.readlines():
+                if l[:lsize] == login_:
+                    yield Login(l.strip())
 
 def cmd_whois(msg):
     if len(msg.args) < 1:
         raise IMException("Provide a name")
 
-    res = Response(channel=msg.channel, count=" (%d more logins)")
-    for srch in msg.args:
-        l = found_login(srch)
-        if l is not None:
+    def format_response(t):
+        srch, l = t
+        if type(l) is Login:
             pic = l.get_photo()
-            res.append_message("%s is %s (%s %s): %s%s" % (srch, l.cn.capitalize(), l.login, l.uid, l.get_promo(), " and looks like %s" % pic if pic is not None else ""))
+            return "%s is %s (%s %s): %s%s" % (srch, l.cn.capitalize(), l.login, l.uid, l.get_promo(), " and looks like %s" % pic if pic is not None else "")
         else:
-            res.append_message("Unknown %s :(" % srch)
+            return l % srch
+
+    res = Response(channel=msg.channel, count=" (%d more logins)", line_treat=format_response)
+    for srch in msg.args:
+        found = False
+        for l in login_lookup(srch, "lookup" in msg.kwargs):
+            found = True
+            res.append_message((srch, l))
+        if not found:
+            res.append_message((srch, "Unknown %s :("))
     return res
 
 @hook.command("nicks")
 def cmd_nicks(msg):
     if len(msg.args) < 1:
         raise IMException("Provide a login")
-    nick = found_login(msg.args[0])
+    nick = login_lookup(msg.args[0])
     if nick is None:
         nick = msg.args[0]
     else:
@@ -108,12 +147,12 @@ def cmd_nicks(msg):
 
 @hook.ask()
 def parseask(msg):
-    res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)\s+([a-zA-Z0-9_-]{3,8})$", msg.text, re.I)
+    res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)\s+([a-zA-Z0-9_-]{3,8})$", msg.message, re.I)
     if res is not None:
         nick = res.group(1)
         login = res.group(3)
         if nick == "my" or nick == "I" or nick == "i" or nick == "je" or nick == "mon" or nick == "ma":
-            nick = msg.nick
+            nick = msg.frm
         if nick in context.data.getNode("aliases").index:
             context.data.getNode("aliases").index[nick]["to"] = login
         else:
@@ -125,4 +164,4 @@ def parseask(msg):
         return Response("ok, c'est noté, %s est %s"
                         % (nick, login),
                         channel=msg.channel,
-                        nick=msg.nick)
+                        nick=msg.frm)

modules/wolframalpha.py

@@ -10,12 +10,12 @@ from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools import web
 
-from more import Response
+from nemubot.module.more import Response
 
 
 # LOADING #############################################################
 
-URL_API = "http://api.wolframalpha.com/v2/query?input=%%s&appid=%s"
+URL_API = "https://api.wolframalpha.com/v2/query?input=%%s&format=plaintext&appid=%s"
 
 def load(context):
     global URL_API
@@ -24,7 +24,7 @@ def load(context):
                            "this module. Add it to the module configuration: "
                            "\n<module name=\"wolframalpha\" "
                            "apikey=\"XXXXXX-XXXXXXXXXX\" />\n"
-                           "Register at http://products.wolframalpha.com/api/")
+                           "Register at https://products.wolframalpha.com/api/")
     URL_API = URL_API % quote(context.config["apikey"]).replace("%", "%%")
 
 

modules/worldcup.py

@@ -1,27 +1,28 @@
 # coding=utf-8
 
-"""The 2014 football worldcup module"""
+"""The 2014,2018 football worldcup module"""
 
 from datetime import datetime, timezone
+from functools import partial
 import json
 import re
 from urllib.parse import quote
 from urllib.request import urlopen
 
 from nemubot import context
+from nemubot.event import ModuleEvent
 from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools.xmlparser.node import ModuleState
 
 nemubotversion = 3.4
 
-from more import Response
+from nemubot.module.more import Response
 
 API_URL="http://worldcup.sfg.io/%s"
 
 def load(context):
-    from nemubot.event import ModuleEvent
-    context.add_event(ModuleEvent(func=lambda url: urlopen(url, timeout=10).read().decode(), func_data=API_URL % "matches/current?by_date=DESC", call=current_match_new_action, interval=30))
+    context.add_event(ModuleEvent(func=partial(lambda url: urlopen(url, timeout=10).read().decode(), API_URL % "matches/current?by_date=DESC"), call=current_match_new_action, interval=30))
 
 
 def help_full ():
@@ -32,7 +33,7 @@ def start_watch(msg):
     w = ModuleState("watch")
     w["server"] = msg.server
     w["channel"] = msg.channel
-    w["proprio"] = msg.nick
+    w["proprio"] = msg.frm
     w["start"] = datetime.now(timezone.utc)
     context.data.addChild(w)
     context.save()
@@ -65,10 +66,10 @@ def cmd_watch(msg):
             context.save()
             raise IMException("This channel will not anymore receives world cup events.")
 
-def current_match_new_action(match_str, osef):
-    context.add_event(ModuleEvent(func=lambda url: urlopen(url).read().decode(), func_data=API_URL % "matches/current?by_date=DESC", call=current_match_new_action, interval=30))
-
-    matches = json.loads(match_str)
+def current_match_new_action(matches):
+    def cmp(om, nm):
+        return len(nm) and (len(om) == 0 or len(nm[0]["home_team_events"]) != len(om[0]["home_team_events"]) or len(nm[0]["away_team_events"]) != len(om[0]["away_team_events"]))
+    context.add_event(ModuleEvent(func=partial(lambda url: json.loads(urlopen(url).read().decode()), API_URL % "matches/current?by_date=DESC"), cmp=partial(cmp, matches), call=current_match_new_action, interval=30))
 
     for match in matches:
         if is_valid(match):
@@ -120,20 +121,19 @@ def detail_event(evt):
     return evt + " par"
 
 def txt_event(e):
-    return "%se minutes : %s %s (%s)" % (e["time"], detail_event(e["type_of_event"]), e["player"], e["team"]["code"])
+    return "%s minute : %s %s (%s)" % (e["time"], detail_event(e["type_of_event"]), e["player"], e["team"]["code"])
 
 def prettify(match):
-    matchdate_local = datetime.strptime(match["datetime"].replace(':', ''), "%Y-%m-%dT%H%M%S.%f%z")
-    matchdate = matchdate_local - (matchdate_local.utcoffset() - datetime.timedelta(hours=2))
+    matchdate = datetime.strptime(match["datetime"].replace(':', ''), "%Y-%m-%dT%H%M%SZ").replace(tzinfo=timezone.utc)
     if match["status"] == "future":
-        return ["Match à venir (%s) le %s : %s vs. %s" % (match["match_number"], matchdate.strftime("%A %d à %H:%M"), match["home_team"]["country"], match["away_team"]["country"])]
+        return ["Match à venir (%s) le %s : %s vs. %s" % (match["fifa_id"], matchdate.strftime("%A %d à %H:%M"), match["home_team"]["country"], match["away_team"]["country"])]
     else:
         msgs = list()
         msg = ""
         if match["status"] == "completed":
-            msg += "Match (%s) du %s terminé : " % (match["match_number"], matchdate.strftime("%A %d à %H:%M"))
+            msg += "Match (%s) du %s terminé : " % (match["fifa_id"], matchdate.strftime("%A %d à %H:%M"))
         else:
-            msg += "Match en cours (%s) depuis %d minutes : " % (match["match_number"], (datetime.now(matchdate.tzinfo) - matchdate_local).seconds / 60)
+            msg += "Match en cours (%s) depuis %d minutes : " % (match["fifa_id"], (datetime.now(tz=timezone.utc) - matchdate).total_seconds() / 60)
 
         msg += "%s %d - %d %s" % (match["home_team"]["country"], match["home_team"]["goals"], match["away_team"]["goals"], match["away_team"]["country"])
 
@@ -163,7 +163,7 @@ def is_valid(match):
 def get_match(url, matchid):
     allm = get_matches(url)
     for m in allm:
-        if int(m["match_number"]) == matchid:
+        if int(m["fifa_id"]) == matchid:
             return [ m ]
 
 def get_matches(url):
@@ -192,7 +192,7 @@ def cmd_worldcup(msg):
         elif len(msg.args[0]) == 3:
             url = "matches/country?fifa_code=%s&by_date=DESC" % msg.args[0]
         elif is_int(msg.args[0]):
-            url = int(msg.arg[0])
+            url = int(msg.args[0])
         else:
             raise IMException("unrecognized request; choose between 'today', 'tomorrow', a FIFA country code or a match identifier")
 

modules/youtube-title.py

@@ -4,7 +4,7 @@ import re, json, subprocess
 from nemubot.exception import IMException
 from nemubot.hooks import hook
 from nemubot.tools.web import _getNormalizedURL, getURLContent
-from more import Response
+from nemubot.module.more import Response
 
 """Get information of youtube videos"""
 
@@ -82,7 +82,7 @@ def parselisten(msg):
 @hook.post()
 def parseresponse(msg):
     global LAST_URLS
-    if hasattr(msg, "text") and msg.text:
+    if hasattr(msg, "text") and msg.text and type(msg.text) == str:
       urls = re.findall("([a-zA-Z0-9+.-]+:(?://)?[^ :]+)", msg.text)
       for url in urls:
         o = urlparse(_getNormalizedURL(url))

nemubot/__init__.py

@@ -17,8 +17,9 @@
 __version__ = '4.0.dev3'
 __author__  = 'nemunaire'
 
-from nemubot.modulecontext import ModuleContext
-context = ModuleContext(None, None)
+from nemubot.modulecontext import _ModuleContext
+
+context = _ModuleContext()
 
 
 def requires_version(min=None, max=None):
@@ -38,62 +39,110 @@ def requires_version(min=None, max=None):
                           "but this is nemubot v%s." % (str(max), __version__))
 
 
-def reload():
-    """Reload code of all Python modules used by nemubot
+def attach(pidfile, socketfile):
+    import socket
+    import sys
+
+    # Read PID from pidfile
+    with open(pidfile, "r") as f:
+        pid = int(f.readline())
+
+    print("nemubot is launched with PID %d. Attaching to Unix socket at: %s" % (pid, socketfile))
+
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    try:
+        sock.connect(socketfile)
+    except socket.error as e:
+        sys.stderr.write(str(e))
+        sys.stderr.write("\n")
+        return 1
+
+    import select
+    mypoll = select.poll()
+
+    mypoll.register(sys.stdin.fileno(), select.POLLIN | select.POLLPRI)
+    mypoll.register(sock.fileno(), select.POLLIN | select.POLLPRI)
+    try:
+        while True:
+            for fd, flag in mypoll.poll():
+                if flag & (select.POLLERR | select.POLLHUP | select.POLLNVAL):
+                    sock.close()
+                    print("Connection closed.")
+                    return 1
+
+                if fd == sys.stdin.fileno():
+                    line = sys.stdin.readline().strip()
+                    if line == "exit" or line == "quit":
+                        return 0
+                    elif line == "reload":
+                        import os, signal
+                        os.kill(pid, signal.SIGHUP)
+                        print("Reload signal sent. Please wait...")
+
+                    elif line == "shutdown":
+                        import os, signal
+                        os.kill(pid, signal.SIGTERM)
+                        print("Shutdown signal sent. Please wait...")
+
+                    elif line == "kill":
+                        import os, signal
+                        os.kill(pid, signal.SIGKILL)
+                        print("Signal sent...")
+                        return 0
+
+                    elif line == "stack" or line == "stacks":
+                        import os, signal
+                        os.kill(pid, signal.SIGUSR1)
+                        print("Debug signal sent. Consult logs.")
+
+                    else:
+                        sock.send(line.encode() + b'\r\n')
+
+                if fd == sock.fileno():
+                    sys.stdout.write(sock.recv(2048).decode())
+
+    except KeyboardInterrupt:
+        pass
+    except:
+        return 1
+    finally:
+        sock.close()
+    return 0
+
+
+def daemonize(socketfile=None):
+    """Detach the running process to run as a daemon
     """
 
-    import imp
+    import os
+    import sys
 
-    import nemubot.channel
-    imp.reload(nemubot.channel)
+    try:
+        pid = os.fork()
+        if pid > 0:
+            sys.exit(0)
+    except OSError as err:
+        sys.stderr.write("Unable to fork: %s\n" % err)
+        sys.exit(1)
 
-    import nemubot.config
-    imp.reload(nemubot.config)
+    os.setsid()
+    os.umask(0)
+    os.chdir('/')
 
-    nemubot.config.reload()
+    try:
+        pid = os.fork()
+        if pid > 0:
+            sys.exit(0)
+    except OSError as err:
+        sys.stderr.write("Unable to fork: %s\n" % err)
+        sys.exit(1)
 
-    import nemubot.consumer
-    imp.reload(nemubot.consumer)
+    sys.stdout.flush()
+    sys.stderr.flush()
+    si = open(os.devnull, 'r')
+    so = open(os.devnull, 'a+')
+    se = open(os.devnull, 'a+')
 
-    import nemubot.datastore
-    imp.reload(nemubot.datastore)
-
-    nemubot.datastore.reload()
-
-    import nemubot.event
-    imp.reload(nemubot.event)
-
-    import nemubot.exception
-    imp.reload(nemubot.exception)
-
-    nemubot.exception.reload()
-
-    import nemubot.hooks
-    imp.reload(nemubot.hooks)
-
-    nemubot.hooks.reload()
-
-    import nemubot.importer
-    imp.reload(nemubot.importer)
-
-    import nemubot.message
-    imp.reload(nemubot.message)
-
-    nemubot.message.reload()
-
-    import nemubot.prompt
-    imp.reload(nemubot.prompt)
-
-    nemubot.prompt.reload()
-
-    import nemubot.server
-    rl, wl, xl = nemubot.server._rlist, nemubot.server._wlist, nemubot.server._xlist
-    imp.reload(nemubot.server)
-    nemubot.server._rlist, nemubot.server._wlist, nemubot.server._xlist = rl, wl, xl
-
-    nemubot.server.reload()
-
-    import nemubot.tools
-    imp.reload(nemubot.tools)
-
-    nemubot.tools.reload()
+    os.dup2(si.fileno(), sys.stdin.fileno())
+    os.dup2(so.fileno(), sys.stdout.fileno())
+    os.dup2(se.fileno(), sys.stderr.fileno())

nemubot/__main__.py

@@ -1,5 +1,5 @@
 # Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
+# Copyright (C) 2012-2017  Mercier Pierre-Olivier
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -16,6 +16,7 @@
 
 def main():
     import os
+    import signal
     import sys
 
     # Parse command line arguments
@@ -36,6 +37,18 @@ def main():
                         default=["./modules/"],
                         help="directory to use as modules store")
 
+    parser.add_argument("-A", "--no-attach", action="store_true",
+                        help="don't attach after fork")
+
+    parser.add_argument("-d", "--debug", action="store_true",
+                        help="don't deamonize, keep in foreground")
+
+    parser.add_argument("-P", "--pidfile", default="./nemubot.pid",
+                        help="Path to the file where store PID")
+
+    parser.add_argument("-S", "--socketfile", default="./nemubot.sock",
+                        help="path where open the socket for internal communication")
+
     parser.add_argument("-l", "--logfile", default="./nemubot.log",
                         help="Path to store logs")
 
@@ -58,11 +71,27 @@ def main():
 
     # Resolve relatives paths
     args.data_path = os.path.abspath(os.path.expanduser(args.data_path))
+    args.pidfile = os.path.abspath(os.path.expanduser(args.pidfile)) if args.pidfile is not None and args.pidfile != "" else None
+    args.socketfile = os.path.abspath(os.path.expanduser(args.socketfile)) if args.socketfile is not None and args.socketfile != "" else None
     args.logfile = os.path.abspath(os.path.expanduser(args.logfile))
-    args.files = [ x for x in map(os.path.abspath, args.files)]
-    args.modules_path = [ x for x in map(os.path.abspath, args.modules_path)]
+    args.files = [x for x in map(os.path.abspath, args.files)]
+    args.modules_path = [x for x in map(os.path.abspath, args.modules_path)]
 
-    # Setup loggin interface
+    # Prepare the attached client, before setting other stuff
+    if not args.debug and not args.no_attach and args.socketfile is not None and args.pidfile is not None:
+        try:
+            pid = os.fork()
+            if pid > 0:
+                import time
+                os.waitpid(pid, 0)
+                time.sleep(1)
+                from nemubot import attach
+                sys.exit(attach(args.pidfile, args.socketfile))
+        except OSError as err:
+            sys.stderr.write("Unable to fork: %s\n" % err)
+            sys.exit(1)
+
+    # Setup logging interface
     import logging
     logger = logging.getLogger("nemubot")
     logger.setLevel(logging.DEBUG)
@@ -70,16 +99,29 @@ def main():
     formatter = logging.Formatter(
         '%(asctime)s %(name)s %(levelname)s %(message)s')
 
-    ch = logging.StreamHandler()
-    ch.setFormatter(formatter)
-    if args.verbose < 2:
-        ch.setLevel(logging.INFO)
-    logger.addHandler(ch)
+    if args.debug:
+        ch = logging.StreamHandler()
+        ch.setFormatter(formatter)
+        if args.verbose < 2:
+            ch.setLevel(logging.INFO)
+        logger.addHandler(ch)
 
     fh = logging.FileHandler(args.logfile)
     fh.setFormatter(formatter)
     logger.addHandler(fh)
 
+    # Check if an instance is already launched
+    if args.pidfile is not None and os.path.isfile(args.pidfile):
+        with open(args.pidfile, "r") as f:
+            pid = int(f.readline())
+        try:
+            os.kill(pid, 0)
+        except OSError:
+            pass
+        else:
+            from nemubot import attach
+            sys.exit(attach(args.pidfile, args.socketfile))
+
     # Add modules dir paths
     modules_paths = list()
     for path in args.modules_path:
@@ -93,61 +135,145 @@ def main():
     from nemubot.bot import Bot
     context = Bot(modules_paths=modules_paths,
                   data_store=datastore.XML(args.data_path),
-                  verbosity=args.verbose)
+                  debug=args.verbose > 0)
 
     if args.no_connect:
         context.noautoconnect = True
 
-    # Load the prompt
-    import nemubot.prompt
-    prmpt = nemubot.prompt.Prompt()
-
     # Register the hook for futur import
     from nemubot.importer import ModuleFinder
-    sys.meta_path.append(ModuleFinder(context.modules_paths, context.add_module))
+    module_finder = ModuleFinder(context.modules_paths, context.add_module)
+    sys.meta_path.append(module_finder)
 
     # Load requested configuration files
     for path in args.files:
-        if os.path.isfile(path):
-            context.sync_queue.put_nowait(["loadconf", path])
-        else:
+        if not os.path.isfile(path):
             logger.error("%s is not a readable file", path)
+            continue
+
+        config = load_config(path)
+
+        # Preset each server in this file
+        for server in config.servers:
+            # Add the server in the context
+            for i in [0,1,2,3]:
+                srv = server.server(config, trynb=i)
+                try:
+                    if context.add_server(srv):
+                        logger.info("Server '%s' successfully added.", srv.name)
+                    else:
+                        logger.error("Can't add server '%s'.", srv.name)
+                except Exception as e:
+                    logger.error("Unable to connect to '%s': %s", srv.name, e)
+                    continue
+                break
+
+        # Load module and their configuration
+        for mod in config.modules:
+            context.modules_configuration[mod.name] = mod
+            if mod.autoload:
+                try:
+                    __import__("nemubot.module." + mod.name)
+                except:
+                    logger.exception("Exception occurs when loading module"
+                                     " '%s'", mod.name)
+
+        # Load files asked by the configuration file
+        args.files += config.includes
+
 
     if args.module:
         for module in args.module:
-            __import__(module)
+            __import__("nemubot.module." + module)
 
-    print ("Nemubot v%s ready, my PID is %i!" % (nemubot.__version__,
-                                                 os.getpid()))
-    while True:
-        from nemubot.prompt.reset import PromptReset
-        try:
-            context.start()
-            if prmpt.run(context):
-                break
-        except PromptReset as e:
-            if e.type == "quit":
-                break
+    if args.socketfile:
+        from nemubot.server.socket import UnixSocketListener
+        context.add_server(UnixSocketListener(new_server_cb=context.add_server,
+                                              location=args.socketfile,
+                                              name="master_socket"))
 
-        try:
-            import imp
-            # Reload all other modules
-            imp.reload(nemubot)
-            imp.reload(nemubot.prompt)
-            nemubot.reload()
-            import nemubot.bot
-            context = nemubot.bot.hotswap(context)
-            prmpt = nemubot.prompt.hotswap(prmpt)
-            print("\033[1;32mContext reloaded\033[0m, now in Nemubot %s" %
-                  nemubot.__version__)
-        except:
-            logger.exception("\033[1;31mUnable to reload the prompt due to "
-                             "errors.\033[0m Fix them before trying to reload "
-                             "the prompt.")
+    # Daemonize
+    if not args.debug:
+        from nemubot import daemonize
+        daemonize(args.socketfile)
 
-    context.quit()
-    print("Waiting for other threads shuts down...")
+    # Signals handling
+    def sigtermhandler(signum, frame):
+        """On SIGTERM and SIGINT, quit nicely"""
+        context.quit()
+    signal.signal(signal.SIGINT, sigtermhandler)
+    signal.signal(signal.SIGTERM, sigtermhandler)
+
+    def sighuphandler(signum, frame):
+        """On SIGHUP, perform a deep reload"""
+        nonlocal context
+
+        logger.debug("SIGHUP receive, iniate reload procedure...")
+
+        # Reload configuration file
+        for path in args.files:
+            if os.path.isfile(path):
+                sync_act("loadconf", path)
+    signal.signal(signal.SIGHUP, sighuphandler)
+
+    def sigusr1handler(signum, frame):
+        """On SIGHUSR1, display stacktraces"""
+        import threading, traceback
+        for threadId, stack in sys._current_frames().items():
+            thName = "#%d" % threadId
+            for th in threading.enumerate():
+                if th.ident == threadId:
+                    thName = th.name
+                    break
+            logger.debug("########### Thread %s:\n%s",
+                         thName,
+                         "".join(traceback.format_stack(stack)))
+    signal.signal(signal.SIGUSR1, sigusr1handler)
+
+    # Store PID to pidfile
+    if args.pidfile is not None:
+        with open(args.pidfile, "w+") as f:
+            f.write(str(os.getpid()))
+
+    # context can change when performing an hotswap, always join the latest context
+    oldcontext = None
+    while oldcontext != context:
+        oldcontext = context
+        context.start()
+        context.join()
+
+    # Wait for consumers
+    logger.info("Waiting for other threads shuts down...")
+    if args.debug:
+        sigusr1handler(0, None)
     sys.exit(0)
 
+
+def load_config(filename):
+    """Load a configuration file
+
+    Arguments:
+    filename -- the path to the file to load
+    """
+
+    from nemubot.channel import Channel
+    from nemubot import config
+    from nemubot.tools.xmlparser import XMLParser
+
+    try:
+        p = XMLParser({
+            "nemubotconfig": config.Nemubot,
+            "server": config.Server,
+            "channel": Channel,
+            "module": config.Module,
+            "include": config.Include,
+        })
+        return p.parse_file(filename)
+    except:
+        logger.exception("Can't load `%s'; this is not a valid nemubot "
+                         "configuration file.", filename)
+    return None
+
+
 if __name__ == "__main__":
     main()

nemubot/bot.py

@@ -16,8 +16,11 @@
 
 from datetime import datetime, timezone
 import logging
+from multiprocessing import JoinableQueue
 import threading
+import select
 import sys
+import weakref
 
 from nemubot import __version__
 from nemubot.consumer import Consumer, EventConsumer, MessageConsumer
@@ -26,29 +29,35 @@ import nemubot.hooks
 
 logger = logging.getLogger("nemubot")
 
+sync_queue = JoinableQueue()
+
+def sync_act(*args):
+    sync_queue.put(list(args))
+
 
 class Bot(threading.Thread):
 
     """Class containing the bot context and ensuring key goals"""
 
     def __init__(self, ip="127.0.0.1", modules_paths=list(),
-                 data_store=datastore.Abstract(), verbosity=0):
+                 data_store=datastore.Abstract(), debug=False):
         """Initialize the bot context
 
         Keyword arguments:
         ip -- The external IP of the bot (default: 127.0.0.1)
-        modules_paths -- Paths to all directories where looking for module
+        modules_paths -- Paths to all directories where looking for modules
         data_store -- An instance of the nemubot datastore for bot's modules
+        debug -- enable debug
         """
 
-        threading.Thread.__init__(self)
+        super().__init__(name="Nemubot main")
 
         logger.info("Initiate nemubot v%s (running on Python %s.%s.%s)",
                     __version__,
                     sys.version_info.major, sys.version_info.minor, sys.version_info.micro)
 
-        self.verbosity = verbosity
-        self.stop = None
+        self.debug = debug
+        self.stop = True
 
         # External IP for accessing this bot
         import ipaddress
@@ -60,6 +69,7 @@ class Bot(threading.Thread):
         self.datastore.open()
 
         # Keep global context: servers and modules
+        self._poll = select.poll()
         self.servers = dict()
         self.modules = dict()
         self.modules_configuration = dict()
@@ -82,23 +92,24 @@ class Bot(threading.Thread):
 
         def in_echo(msg):
             from nemubot.message import Text
-            return Text(msg.nick + ": " + " ".join(msg.args), to=msg.to_response)
+            return Text(msg.frm + ": " + " ".join(msg.args), to=msg.to_response)
         self.treater.hm.add_hook(nemubot.hooks.Command(in_echo, "echo"), "in", "Command")
 
         def _help_msg(msg):
             """Parse and response to help messages"""
-            from more import Response
+            from nemubot.module.more import Response
             res = Response(channel=msg.to_response)
             if len(msg.args) >= 1:
-                if msg.args[0] in self.modules:
-                    if hasattr(self.modules[msg.args[0]], "help_full"):
-                        hlp = self.modules[msg.args[0]].help_full()
+                if "nemubot.module." + msg.args[0] in self.modules and self.modules["nemubot.module." + msg.args[0]]() is not None:
+                    mname = "nemubot.module." + msg.args[0]
+                    if hasattr(self.modules[mname](), "help_full"):
+                        hlp = self.modules[mname]().help_full()
                         if isinstance(hlp, Response):
                             return hlp
                         else:
                             res.append_message(hlp)
                     else:
-                        res.append_message([str(h) for s,h in self.modules[msg.args[0]].__nemubot_context__.hooks], title="Available commands for module " + msg.args[0])
+                        res.append_message([str(h) for s,h in self.modules[mname]().__nemubot_context__.hooks], title="Available commands for module " + msg.args[0])
                 elif msg.args[0][0] == "!":
                     from nemubot.message.command import Command
                     for h in self.treater._in_hooks(Command(msg.args[0][1:])):
@@ -124,152 +135,122 @@ class Bot(threading.Thread):
                                    "Vous pouvez le consulter, le dupliquer, "
                                    "envoyer des rapports de bogues ou bien "
                                    "contribuer au projet sur GitHub : "
-                                   "http://github.com/nemunaire/nemubot/")
+                                   "https://github.com/nemunaire/nemubot/")
                 res.append_message(title="Pour plus de détails sur un module, "
                                    "envoyez \"!help nomdumodule\". Voici la liste"
                                    " de tous les modules disponibles localement",
-                                   message=["\x03\x02%s\x03\x02 (%s)" % (im, self.modules[im].__doc__) for im in self.modules if self.modules[im].__doc__])
+                                   message=["\x03\x02%s\x03\x02 (%s)" % (im, self.modules[im]().__doc__) for im in self.modules if self.modules[im]() is not None and self.modules[im]().__doc__])
             return res
         self.treater.hm.add_hook(nemubot.hooks.Command(_help_msg, "help"), "in", "Command")
 
+        import os
         from queue import Queue
-        # Messages to be treated
-        self.cnsr_queue     = Queue()
-        self.cnsr_thrd      = list()
-        self.cnsr_thrd_size = -1
-        # Synchrone actions to be treated by main thread
-        self.sync_queue     = Queue()
+        # Messages to be treated — shared across all server connections.
+        # cnsr_active tracks consumers currently inside stm.run() (not idle),
+        # which lets us spawn a new thread the moment all existing ones are busy.
+        self.cnsr_queue  = Queue()
+        self.cnsr_thrd   = list()
+        self.cnsr_lock   = threading.Lock()
+        self.cnsr_active = 0                      # consumers currently executing a task
+        self.cnsr_max    = os.cpu_count() or 4    # upper bound on concurrent consumer threads
+
+
+    def __del__(self):
+        self.datastore.close()
 
 
     def run(self):
-        from select import select
-        from nemubot.server import _lock, _rlist, _wlist, _xlist
+        global sync_queue
+
+        # Rewrite the sync_queue, as the daemonization process tend to disturb it
+        old_sync_queue, sync_queue = sync_queue, JoinableQueue()
+        while not old_sync_queue.empty():
+            sync_queue.put_nowait(old_sync_queue.get())
+
+        self._poll.register(sync_queue._reader, select.POLLIN | select.POLLPRI)
+
 
         self.stop = False
+
+        # Relaunch events
+        self._update_event_timer()
+
+        logger.info("Starting main loop")
         while not self.stop:
-            with _lock:
-                try:
-                    rl, wl, xl = select(_rlist, _wlist, _xlist, 0.1)
-                except:
-                    logger.error("Something went wrong in select")
-                    fnd_smth = False
-                    # Looking for invalid server
-                    for r in _rlist:
-                        if not hasattr(r, "fileno") or not isinstance(r.fileno(), int) or r.fileno() < 0:
-                            _rlist.remove(r)
-                            logger.error("Found invalid object in _rlist: " + str(r))
-                            fnd_smth = True
-                    for w in _wlist:
-                        if not hasattr(w, "fileno") or not isinstance(w.fileno(), int) or w.fileno() < 0:
-                            _wlist.remove(w)
-                            logger.error("Found invalid object in _wlist: " + str(w))
-                            fnd_smth = True
-                    for x in _xlist:
-                        if not hasattr(x, "fileno") or not isinstance(x.fileno(), int) or x.fileno() < 0:
-                            _xlist.remove(x)
-                            logger.error("Found invalid object in _xlist: " + str(x))
-                            fnd_smth = True
-                    if not fnd_smth:
-                        logger.exception("Can't continue, sorry")
-                        self.quit()
-                    continue
+            for fd, flag in self._poll.poll():
+                # Handle internal socket passing orders
+                if fd != sync_queue._reader.fileno() and fd in self.servers:
+                    srv = self.servers[fd]
 
-                for x in xl:
-                    try:
-                        x.exception()
-                    except:
-                        logger.exception("Uncatched exception on server exception")
-                for w in wl:
-                    try:
-                        w.write_select()
-                    except:
-                        logger.exception("Uncatched exception on server write")
-                for r in rl:
-                    for i in r.read():
+                    if flag & (select.POLLERR | select.POLLHUP | select.POLLNVAL):
                         try:
-                            self.receive_message(r, i)
+                            srv.exception(flag)
                         except:
-                            logger.exception("Uncatched exception on server read")
+                            logger.exception("Uncatched exception on server exception")
+
+                    if srv.fileno() > 0:
+                        if flag & (select.POLLOUT):
+                            try:
+                                srv.async_write()
+                            except:
+                                logger.exception("Uncatched exception on server write")
+
+                        if flag & (select.POLLIN | select.POLLPRI):
+                            try:
+                                for i in srv.async_read():
+                                    self.receive_message(srv, i)
+                            except:
+                                logger.exception("Uncatched exception on server read")
+
+                    else:
+                        del self.servers[fd]
 
 
-            # Launch new consumer threads if necessary
-            while self.cnsr_queue.qsize() > self.cnsr_thrd_size:
-                # Next launch if two more items in queue
-                self.cnsr_thrd_size += 2
+                # Always check the sync queue
+                while not sync_queue.empty():
+                    args = sync_queue.get()
+                    action = args.pop(0)
 
-                c = Consumer(self)
-                self.cnsr_thrd.append(c)
-                c.start()
+                    logger.debug("Executing sync_queue action %s%s", action, args)
 
-            while self.sync_queue.qsize() > 0:
-                action = self.sync_queue.get_nowait()
-                if action[0] == "exit":
-                    self.quit()
-                elif action[0] == "loadconf":
-                    for path in action[1:]:
-                        logger.debug("Load configuration from %s", path)
-                        self.load_file(path)
-                    logger.info("Configurations successfully loaded")
-                self.sync_queue.task_done()
+                    if action == "sckt" and len(args) >= 2:
+                        try:
+                            if args[0] == "write":
+                                self._poll.modify(int(args[1]), select.POLLOUT | select.POLLIN | select.POLLPRI)
+                            elif args[0] == "unwrite":
+                                self._poll.modify(int(args[1]), select.POLLIN | select.POLLPRI)
+
+                            elif args[0] == "register":
+                                self._poll.register(int(args[1]), select.POLLIN | select.POLLPRI)
+                            elif args[0] == "unregister":
+                                try:
+                                    self._poll.unregister(int(args[1]))
+                                except KeyError:
+                                    pass
+                        except:
+                            logger.exception("Unhandled excpetion during action:")
+
+                    elif action == "exit":
+                        self.quit()
+
+                    elif action == "launch_consumer":
+                        pass  # This is treated after the loop
+
+                    sync_queue.task_done()
 
 
+            # Spawn a new consumer whenever the queue has work and every
+            # existing consumer is already busy executing a task.
+            with self.cnsr_lock:
+                while (not self.cnsr_queue.empty()
+                       and self.cnsr_active >= len(self.cnsr_thrd)
+                       and len(self.cnsr_thrd) < self.cnsr_max):
+                    c = Consumer(self)
+                    self.cnsr_thrd.append(c)
+                    c.start()
+        sync_queue = None
+        logger.info("Ending main loop")
 
-    # Config methods
-
-    def load_file(self, filename):
-        """Load a configuration file
-
-        Arguments:
-        filename -- the path to the file to load
-        """
-
-        import os
-
-        # Unexisting file, assume a name was passed, import the module!
-        if not os.path.isfile(filename):
-            return self.import_module(filename)
-
-        from nemubot.channel import Channel
-        from nemubot import config
-        from nemubot.tools.xmlparser import XMLParser
-
-        try:
-            p = XMLParser({
-                "nemubotconfig": config.Nemubot,
-                "server": config.Server,
-                "channel": Channel,
-                "module": config.Module,
-                "include": config.Include,
-            })
-            config = p.parse_file(filename)
-        except:
-            logger.exception("Can't load `%s'; this is not a valid nemubot "
-                             "configuration file." % filename)
-            return False
-
-        # Preset each server in this file
-        for server in config.servers:
-            srv = server.server(config)
-            # Add the server in the context
-            if self.add_server(srv, server.autoconnect):
-                logger.info("Server '%s' successfully added." % srv.id)
-            else:
-                logger.error("Can't add server '%s'." % srv.id)
-
-        # Load module and their configuration
-        for mod in config.modules:
-            self.modules_configuration[mod.name] = mod
-            if mod.autoload:
-                try:
-                    __import__(mod.name)
-                except:
-                    logger.exception("Exception occurs when loading module"
-                                     " '%s'", mod.name)
-
-
-        # Load files asked by the configuration file
-        for load in config.includes:
-            self.load_file(load.path)
 
 
     # Events methods
@@ -289,10 +270,6 @@ class Bot(threading.Thread):
         module_src -- The module to which the event is attached to
         """
 
-        if hasattr(self, "stop") and self.stop:
-            logger.warn("The bot is stopped, can't register new events")
-            return
-
         import uuid
 
         # Generate the event id if no given
@@ -303,7 +280,7 @@ class Bot(threading.Thread):
         if type(eid) is uuid.UUID:
             evt.id = str(eid)
         else:
-            # Ok, this is quite useless...
+            # Ok, this is quiet useless...
             try:
                 evt.id = str(uuid.UUID(eid))
             except ValueError:
@@ -319,7 +296,7 @@ class Bot(threading.Thread):
                 break
         self.events.insert(i, evt)
 
-        if i == 0:
+        if i == 0 and not self.stop:
             # First event changed, reset timer
             self._update_event_timer()
             if len(self.events) <= 0 or self.events[i] != evt:
@@ -328,10 +305,10 @@ class Bot(threading.Thread):
 
         # Register the event in the source module
         if module_src is not None:
-            module_src.__nemubot_context__.events.append(evt.id)
+            module_src.__nemubot_context__.events.append((evt, evt.id))
         evt.module_src = module_src
 
-        logger.info("New event registered: %s -> %s", evt.id, evt)
+        logger.info("New event registered in %d position: %s", i, t)
         return evt.id
 
 
@@ -358,10 +335,10 @@ class Bot(threading.Thread):
             id = evt
 
         if len(self.events) > 0 and id == self.events[0].id:
+            if module_src is not None:
+                module_src.__nemubot_context__.events.remove((self.events[0], id))
             self.events.remove(self.events[0])
             self._update_event_timer()
-            if module_src is not None:
-                module_src.__nemubot_context__.events.remove(id)
             return True
 
         for evt in self.events:
@@ -369,7 +346,7 @@ class Bot(threading.Thread):
                 self.events.remove(evt)
 
                 if module_src is not None:
-                    module_src.__nemubot_context__.events.remove(evt.id)
+                    module_src.__nemubot_context__.events.remove((evt, evt.id))
                 return True
         return False
 
@@ -382,7 +359,13 @@ class Bot(threading.Thread):
             self.event_timer.cancel()
 
         if len(self.events):
-            remaining = self.events[0].time_left.seconds + self.events[0].time_left.microseconds / 1000000
+            try:
+                remaining = self.events[0].time_left.total_seconds()
+            except:
+                logger.exception("An error occurs during event time calculation:")
+                self.events.pop(0)
+                return self._update_event_timer()
+
             logger.debug("Update timer: next event in %d seconds", remaining)
             self.event_timer = threading.Timer(remaining if remaining > 0 else 0, self._end_event_timer)
             self.event_timer.start()
@@ -397,6 +380,7 @@ class Bot(threading.Thread):
         while len(self.events) > 0 and datetime.now(timezone.utc) >= self.events[0].current:
             evt = self.events.pop(0)
             self.cnsr_queue.put_nowait(EventConsumer(evt))
+            sync_act("launch_consumer")
 
         self._update_event_timer()
 
@@ -411,10 +395,12 @@ class Bot(threading.Thread):
         autoconnect -- connect after add?
         """
 
-        if srv.id not in self.servers:
-            self.servers[srv.id] = srv
+        fileno = srv.fileno()
+        if fileno not in self.servers:
+            self.servers[fileno] = srv
+            self.servers[srv.name] = srv
             if autoconnect and not hasattr(self, "noautoconnect"):
-                srv.open()
+                srv.connect()
             return True
 
         else:
@@ -441,10 +427,6 @@ class Bot(threading.Thread):
         old one before"""
         module_name = module.__spec__.name if hasattr(module, "__spec__") else module.__name__
 
-        if hasattr(self, "stop") and self.stop:
-            logger.warn("The bot is stopped, can't register new modules")
-            return
-
         # Check if the module already exists
         if module_name in self.modules:
             self.unload_module(module_name)
@@ -458,7 +440,7 @@ class Bot(threading.Thread):
         module.print = prnt
 
         # Create module context
-        from nemubot.modulecontext import ModuleContext
+        from nemubot.modulecontext import _ModuleContext, ModuleContext
         module.__nemubot_context__ = ModuleContext(self, module)
 
         if not hasattr(module, "logger"):
@@ -466,7 +448,7 @@ class Bot(threading.Thread):
 
         # Replace imported context by real one
         for attr in module.__dict__:
-            if attr != "__nemubot_context__" and type(module.__dict__[attr]) == ModuleContext:
+            if attr != "__nemubot_context__" and type(module.__dict__[attr]) == _ModuleContext:
                 module.__dict__[attr] = module.__nemubot_context__
 
         # Register decorated functions
@@ -484,18 +466,20 @@ class Bot(threading.Thread):
                 raise
 
         # Save a reference to the module
-        self.modules[module_name] = module
+        self.modules[module_name] = weakref.ref(module)
+        logger.info("Module '%s' successfully loaded.", module_name)
 
 
     def unload_module(self, name):
         """Unload a module"""
-        if name in self.modules:
-            self.modules[name].print("Unloading module %s" % name)
+        if name in self.modules and self.modules[name]() is not None:
+            module = self.modules[name]()
+            module.print("Unloading module %s" % name)
 
             # Call the user defined unload method
-            if hasattr(self.modules[name], "unload"):
-                self.modules[name].unload(self)
-            self.modules[name].__nemubot_context__.unload()
+            if hasattr(module, "unload"):
+                module.unload(self)
+            module.__nemubot_context__.unload()
 
             # Remove from the nemubot dict
             del self.modules[name]
@@ -527,28 +511,28 @@ class Bot(threading.Thread):
     def quit(self):
         """Save and unload modules and disconnect servers"""
 
-        self.datastore.close()
-
         if self.event_timer is not None:
             logger.info("Stop the event timer...")
             self.event_timer.cancel()
 
-        logger.info("Stop consumers")
-        k = self.cnsr_thrd
-        for cnsr in k:
-            cnsr.stop = True
-
         logger.info("Save and unload all modules...")
-        k = list(self.modules.keys())
-        for mod in k:
+        for mod in [m for m in self.modules.keys()]:
             self.unload_module(mod)
 
         logger.info("Close all servers connection...")
-        k = list(self.servers.keys())
-        for srv in k:
-            self.servers[srv].close()
+        for srv in [self.servers[k] for k in self.servers]:
+            srv.close()
 
-        self.stop = True
+        logger.info("Stop consumers")
+        with self.cnsr_lock:
+            k = list(self.cnsr_thrd)
+        for cnsr in k:
+            cnsr.stop = True
+
+        if self.stop is False or sync_queue is not None:
+            self.stop = True
+            sync_act("end")
+            sync_queue.join()
 
 
     # Treatment
@@ -562,20 +546,3 @@ class Bot(threading.Thread):
                     del store[hook.name]
             elif isinstance(store, list):
                 store.remove(hook)
-
-
-def hotswap(bak):
-    bak.stop = True
-    if bak.event_timer is not None:
-        bak.event_timer.cancel()
-    bak.datastore.close()
-
-    new = Bot(str(bak.ip), bak.modules_paths, bak.datastore)
-    new.servers = bak.servers
-    new.modules = bak.modules
-    new.modules_configuration = bak.modules_configuration
-    new.events = bak.events
-    new.hooks = bak.hooks
-
-    new._update_event_timer()
-    return new

nemubot/channel.py

@@ -52,11 +52,11 @@ class Channel:
         elif cmd == "MODE":
             self.mode(msg)
         elif cmd == "JOIN":
-            self.join(msg.nick)
+            self.join(msg.frm)
         elif cmd == "NICK":
-            self.nick(msg.nick, msg.text)
+            self.nick(msg.frm, msg.text)
         elif cmd == "PART" or cmd == "QUIT":
-            self.part(msg.nick)
+            self.part(msg.frm)
         elif cmd == "TOPIC":
             self.topic = self.text
 
@@ -120,17 +120,17 @@ class Channel:
             else:
                 self.password = msg.text[1]
         elif msg.text[0] == "+o":
-            self.people[msg.nick] |= 4
+            self.people[msg.frm] |= 4
         elif msg.text[0] == "-o":
-            self.people[msg.nick] &= ~4
+            self.people[msg.frm] &= ~4
         elif msg.text[0] == "+h":
-            self.people[msg.nick] |= 2
+            self.people[msg.frm] |= 2
         elif msg.text[0] == "-h":
-            self.people[msg.nick] &= ~2
+            self.people[msg.frm] &= ~2
         elif msg.text[0] == "+v":
-            self.people[msg.nick] |= 1
+            self.people[msg.frm] |= 1
         elif msg.text[0] == "-v":
-            self.people[msg.nick] &= ~1
+            self.people[msg.frm] &= ~1
 
     def parse332(self, msg):
         """Parse RPL_TOPIC message

nemubot/config/__init__.py

@@ -24,24 +24,3 @@ from nemubot.config.include import Include
 from nemubot.config.module import Module
 from nemubot.config.nemubot import Nemubot
 from nemubot.config.server import Server
-
-def reload():
-    global Include, Module, Nemubot, Server
-
-    import imp
-
-    import nemubot.config.include
-    imp.reload(nemubot.config.include)
-    Include = nemubot.config.include.Include
-
-    import nemubot.config.module
-    imp.reload(nemubot.config.module)
-    Module = nemubot.config.module.Module
-
-    import nemubot.config.nemubot
-    imp.reload(nemubot.config.nemubot)
-    Nemubot = nemubot.config.nemubot.Nemubot
-
-    import nemubot.config.server
-    imp.reload(nemubot.config.server)
-    Server = nemubot.config.server.Server

nemubot/config/server.py

@@ -33,7 +33,7 @@ class Server:
             return True
 
 
-    def server(self, parent):
+    def server(self, parent, trynb=0):
         from nemubot.server import factory
 
         for a in ["nick", "owner", "realname", "encoding"]:
@@ -42,4 +42,4 @@ class Server:
 
         self.caps += parent.caps
 
-        return factory(self.uri, caps=self.caps, channels=self.channels, **self.args)
+        return factory(self.uri, caps=self.caps, channels=self.channels, trynb=trynb, **self.args)

nemubot/consumer.py

@@ -1,5 +1,5 @@
 # Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
+# Copyright (C) 2012-2016  Mercier Pierre-Olivier
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -38,42 +38,37 @@ class MessageConsumer:
 
         msgs = []
 
-        # Parse the message
+        # Parse message
         try:
             for msg in self.srv.parse(self.orig):
                 msgs.append(msg)
         except:
             logger.exception("Error occurred during the processing of the %s: "
-                             "%s", type(self.msgs[0]).__name__, self.msgs[0])
+                             "%s", type(self.orig).__name__, self.orig)
 
-        if len(msgs) <= 0:
-            return
-
-        # Qualify the message
-        if not hasattr(msg, "server") or msg.server is None:
-            msg.server = self.srv.id
-        if hasattr(msg, "frm_owner"):
-            msg.frm_owner = (not hasattr(self.srv, "owner") or self.srv.owner == msg.frm)
-
-        # Treat the message
+        # Treat message
         for msg in msgs:
             for res in context.treater.treat_msg(msg):
-                # Identify the destination
+                # Identify destination
                 to_server = None
                 if isinstance(res, str):
                     to_server = self.srv
+                elif not hasattr(res, "server"):
+                    logger.error("No server defined for response of type %s: %s", type(res).__name__, res)
+                    continue
                 elif res.server is None:
                     to_server = self.srv
-                    res.server = self.srv.id
-                elif isinstance(res.server, str) and res.server in context.servers:
+                    res.server = self.srv.fileno()
+                elif res.server in context.servers:
                     to_server = context.servers[res.server]
+                else:
+                    to_server = res.server
 
-                if to_server is None:
-                    logger.error("The server defined in this response doesn't "
-                                 "exist: %s", res.server)
+                if to_server is None or not hasattr(to_server, "send_response") or not callable(to_server.send_response):
+                    logger.error("The server defined in this response doesn't exist: %s", res.server)
                     continue
 
-                # Sent the message only if treat_post authorize it
+                # Sent message
                 to_server.send_response(res)
 
 
@@ -99,7 +94,7 @@ class EventConsumer:
         # Or remove reference of this event
         elif (hasattr(self.evt, "module_src") and
               self.evt.module_src is not None):
-            self.evt.module_src.__nemubot_context__.events.remove(self.evt.id)
+            self.evt.module_src.__nemubot_context__.events.remove((self.evt, self.evt.id))
 
 
 
@@ -110,18 +105,25 @@ class Consumer(threading.Thread):
     def __init__(self, context):
         self.context = context
         self.stop = False
-        threading.Thread.__init__(self)
+        super().__init__(name="Nemubot consumer", daemon=True)
 
 
     def run(self):
         try:
             while not self.stop:
-                stm = self.context.cnsr_queue.get(True, 1)
-                stm.run(self.context)
-                self.context.cnsr_queue.task_done()
+                try:
+                    stm = self.context.cnsr_queue.get(True, 1)
+                except queue.Empty:
+                    break
 
-        except queue.Empty:
-            pass
+                with self.context.cnsr_lock:
+                    self.context.cnsr_active += 1
+                try:
+                    stm.run(self.context)
+                finally:
+                    self.context.cnsr_queue.task_done()
+                    with self.context.cnsr_lock:
+                        self.context.cnsr_active -= 1
         finally:
-            self.context.cnsr_thrd_size -= 2
-            self.context.cnsr_thrd.remove(self)
+            with self.context.cnsr_lock:
+                self.context.cnsr_thrd.remove(self)

nemubot/datastore/__init__.py

@@ -16,16 +16,3 @@
 
 from nemubot.datastore.abstract import Abstract
 from nemubot.datastore.xml import XML
-
-
-def reload():
-    global Abstract, XML
-    import imp
-
-    import nemubot.datastore.abstract
-    imp.reload(nemubot.datastore.abstract)
-    Abstract = nemubot.datastore.abstract.Abstract
-
-    import nemubot.datastore.xml
-    imp.reload(nemubot.datastore.xml)
-    XML = nemubot.datastore.xml.XML

nemubot/datastore/abstract.py

@@ -32,16 +32,20 @@ class Abstract:
     def close(self):
         return
 
-    def load(self, module):
+    def load(self, module, knodes):
         """Load data for the given module
 
         Argument:
         module -- the module name of data to load
+        knodes -- the schema to use to load the datas
 
         Return:
         The loaded data
         """
 
+        if knodes is not None:
+            return None
+
         return self.new()
 
     def save(self, module, data):

nemubot/datastore/xml.py

@@ -83,27 +83,38 @@ class XML(Abstract):
 
         return os.path.join(self.basedir, module + ".xml")
 
-    def load(self, module):
+    def load(self, module, knodes):
         """Load data for the given module
 
         Argument:
         module -- the module name of data to load
+        knodes -- the schema to use to load the datas
         """
 
         data_file = self._get_data_file_path(module)
 
+        if knodes is None:
+            from nemubot.tools.xmlparser import parse_file
+            def _true_load(path):
+                return parse_file(path)
+
+        else:
+            from nemubot.tools.xmlparser import XMLParser
+            p = XMLParser(knodes)
+            def _true_load(path):
+                return p.parse_file(path)
+
         # Try to load original file
         if os.path.isfile(data_file):
-            from nemubot.tools.xmlparser import parse_file
             try:
-                return parse_file(data_file)
+                return _true_load(data_file)
             except xml.parsers.expat.ExpatError:
                 # Try to load from backup
                 for i in range(10):
                     path = data_file + "." + str(i)
                     if os.path.isfile(path):
                         try:
-                            cnt = parse_file(path)
+                            cnt = _true_load(path)
 
                             logger.warn("Restoring from backup: %s", path)
 
@@ -112,7 +123,7 @@ class XML(Abstract):
                             continue
 
         # Default case: initialize a new empty datastore
-        return Abstract.load(self, module)
+        return super().load(module, knodes)
 
     def _rotate(self, path):
         """Backup given path
@@ -143,4 +154,18 @@ class XML(Abstract):
         if self.rotate:
             self._rotate(path)
 
-        return data.save(path)
+        if data is None:
+            return
+
+        import tempfile
+        _, tmpath = tempfile.mkstemp()
+        with open(tmpath, "w") as f:
+            import xml.sax.saxutils
+            gen = xml.sax.saxutils.XMLGenerator(f, "utf-8")
+            gen.startDocument()
+            data.saveElement(gen)
+            gen.endDocument()
+
+        # Atomic save
+        import shutil
+        shutil.move(tmpath, path)

nemubot/event/__init__.py

@@ -21,18 +21,14 @@ class ModuleEvent:
 
     """Representation of a event initiated by a bot module"""
 
-    def __init__(self, call=None, call_data=None, func=None, func_data=None,
-                 cmp=None, cmp_data=None, interval=60, offset=0, times=1):
+    def __init__(self, call=None, func=None, cmp=None, interval=60, offset=0, times=1):
 
         """Initialize the event
 
         Keyword arguments:
         call -- Function to call when the event is realized
-        call_data -- Argument(s) (single or dict) to pass as argument
         func -- Function called to check
-        func_data -- Argument(s) (single or dict) to pass as argument OR if no func, initial data to watch
-        cmp -- Boolean function called to check changes
-        cmp_data -- Argument(s) (single or dict) to pass as argument OR if no cmp, data compared to previous
+        cmp -- Boolean function called to check changes or value to compare with
         interval -- Time in seconds between each check (default: 60)
         offset -- Time in seconds added to interval before the first check (default: 0)
         times -- Number of times the event has to be realized before being removed; -1 for no limit (default: 1)
@@ -40,31 +36,22 @@ class ModuleEvent:
 
         # What have we to check?
         self.func = func
-        self.func_data = func_data
 
         # How detect a change?
         self.cmp = cmp
-        self.cmp_data = None
-        if cmp_data is not None:
-            self.cmp_data = cmp_data
-        elif self.func is not None:
-            if self.func_data is None:
-                self.cmp_data = self.func()
-            elif isinstance(self.func_data, dict):
-                self.cmp_data = self.func(**self.func_data)
-            else:
-                self.cmp_data = self.func(self.func_data)
 
         # What should we call when?
         self.call = call
-        if call_data is not None:
-            self.call_data = call_data
-        else:
-            self.call_data = func_data
 
         # Store times
-        self.offset = timedelta(seconds=offset)  # Time to wait before the first check
-        self.interval = timedelta(seconds=interval)
+        if isinstance(offset, timedelta):
+            self.offset = offset  # Time to wait before the first check
+        else:
+            self.offset = timedelta(seconds=offset)  # Time to wait before the first check
+        if isinstance(interval, timedelta):
+            self.interval = interval
+        else:
+            self.interval = timedelta(seconds=interval)
         self._end = None  # Cache
 
         # How many times do this event?
@@ -95,44 +82,23 @@ class ModuleEvent:
         """Return the time left before/after the near check"""
         if self.current is not None:
             return self.current - datetime.now(timezone.utc)
-        return 99999  # TODO: 99999 is not a valid time to return
+        return timedelta.max
 
     def check(self):
         """Run a check and realized the event if this is time"""
 
-        # Get initial data
-        if self.func is None:
-            d_init = self.func_data
-        elif self.func_data is None:
-            d_init = self.func()
-        elif isinstance(self.func_data, dict):
-            d_init = self.func(**self.func_data)
+        # Get new data
+        if self.func is not None:
+            d_new = self.func()
         else:
-            d_init = self.func(self.func_data)
+            d_new = None
 
         # then compare with current data
-        if self.cmp is None:
-            if self.cmp_data is None:
-                rlz = True
-            else:
-                rlz = (d_init != self.cmp_data)
-        elif self.cmp_data is None:
-            rlz = self.cmp(d_init)
-        elif isinstance(self.cmp_data, dict):
-            rlz = self.cmp(d_init, **self.cmp_data)
-        else:
-            rlz = self.cmp(d_init, self.cmp_data)
-
-        if rlz:
+        if self.cmp is None or (callable(self.cmp) and self.cmp(d_new)) or (not callable(self.cmp) and d_new != self.cmp):
             self.times -= 1
 
             # Call attended function
-            if self.call_data is None:
-                if d_init is None:
-                    self.call()
-                else:
-                    self.call(d_init)
-            elif isinstance(self.call_data, dict):
-                self.call(d_init, **self.call_data)
+            if self.func is not None:
+                self.call(d_new)
             else:
-                self.call(d_init, self.call_data)
+                self.call()

nemubot/exception/__init__.py

@@ -32,10 +32,3 @@ class IMException(Exception):
             from nemubot.message import Text
             return Text(*self.args,
                         server=msg.server, to=msg.to_response)
-
-
-def reload():
-    import imp
-
-    import nemubot.exception.Keyword
-    imp.reload(nemubot.exception.printer.IRC)

nemubot/hooks/__init__.py

@@ -49,23 +49,3 @@ class hook:
 
     def pre(*args, store=["pre"], **kwargs):
         return hook._add(store, Abstract, *args, **kwargs)
-
-
-def reload():
-    import imp
-
-    import nemubot.hooks.abstract
-    imp.reload(nemubot.hooks.abstract)
-
-    import nemubot.hooks.command
-    imp.reload(nemubot.hooks.command)
-
-    import nemubot.hooks.message
-    imp.reload(nemubot.hooks.message)
-
-    import nemubot.hooks.keywords
-    imp.reload(nemubot.hooks.keywords)
-    nemubot.hooks.keywords.reload()
-
-    import nemubot.hooks.manager
-    imp.reload(nemubot.hooks.manager)

nemubot/hooks/abstract.py

@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import types
+
 def call_game(call, *args, **kargs):
     """With given args, try to determine the right call to make
 
@@ -119,10 +121,18 @@ class Abstract:
         try:
             if self.check(data1):
                 ret = call_game(self.call, data1, self.data, *args)
+            if isinstance(ret, types.GeneratorType):
+                for r in ret:
+                    yield r
+                ret = None
         except IMException as e:
             ret = e.fill_response(data1)
         finally:
             if self.times == 0:
                 self.call_end(ret)
 
-        return ret
+        if isinstance(ret, list):
+            for r in ret:
+                yield ret
+        elif ret is not None:
+            yield ret

nemubot/hooks/keywords/__init__.py

@@ -26,11 +26,22 @@ class NoKeyword(Abstract):
         return super().check(mkw)
 
 
-def reload():
-    import imp
+class AnyKeyword(Abstract):
 
-    import nemubot.hooks.keywords.abstract
-    imp.reload(nemubot.hooks.keywords.abstract)
+    def __init__(self, h):
+        """Class that accepts any passed keywords
 
-    import nemubot.hooks.keywords.dict
-    imp.reload(nemubot.hooks.keywords.dict)
+        Arguments:
+        h -- Help string
+        """
+
+        super().__init__()
+        self.h = h
+
+
+    def check(self, mkw):
+        return super().check(mkw)
+
+
+    def help(self):
+        return self.h

nemubot/hooks/keywords/dict.py

@@ -43,7 +43,7 @@ class Dict(Abstract):
 
     def check(self, mkw):
         for k in mkw:
-            if (mkw[k] and k not in self.chk_args) or (not mkw[k] and k not in self.chk_noarg):
+            if ((k + "?") not in self.chk_args) and ((mkw[k] and k not in self.chk_args) or (not mkw[k] and k not in self.chk_noarg)):
                 if mkw[k] and k in self.chk_noarg:
                     raise KeywordException("Keyword %s doesn't take value." % k)
                 elif not mkw[k] and k in self.chk_args:

nemubot/importer.py

@@ -29,16 +29,16 @@ class ModuleFinder(Finder):
         self.add_module = add_module
 
     def find_module(self, fullname, path=None):
-        # Search only for new nemubot modules (packages init)
-        if path is None:
+        if path is not None and fullname.startswith("nemubot.module."):
+            module_name = fullname.split(".", 2)[2]
             for mpath in self.modules_paths:
-                if os.path.isfile(os.path.join(mpath, fullname + ".py")):
+                if os.path.isfile(os.path.join(mpath, module_name + ".py")):
                     return ModuleLoader(self.add_module, fullname,
-                                        os.path.join(mpath, fullname + ".py"))
-                elif os.path.isfile(os.path.join(os.path.join(mpath, fullname), "__init__.py")):
+                                        os.path.join(mpath, module_name + ".py"))
+                elif os.path.isfile(os.path.join(os.path.join(mpath, module_name), "__init__.py")):
                     return ModuleLoader(self.add_module, fullname,
                                         os.path.join(
-                                            os.path.join(mpath, fullname),
+                                            os.path.join(mpath, module_name),
                                             "__init__.py"))
         return None
 
@@ -53,17 +53,17 @@ class ModuleLoader(SourceFileLoader):
     def _load(self, module, name):
         # Add the module to the global modules list
         self.add_module(module)
-        logger.info("Module '%s' successfully loaded.", name)
+        logger.info("Module '%s' successfully imported from %s.", name.split(".", 2)[2], self.path)
         return module
 
 
     # Python 3.4
     def exec_module(self, module):
-        super(ModuleLoader, self).exec_module(module)
+        super().exec_module(module)
         self._load(module, module.__spec__.name)
 
 
     # Python 3.3
     def load_module(self, fullname):
-        module = super(ModuleLoader, self).load_module(fullname)
+        module = super().load_module(fullname)
         return self._load(module, module.__name__)

nemubot/message/__init__.py

@@ -19,27 +19,3 @@ from nemubot.message.text import Text
 from nemubot.message.directask import DirectAsk
 from nemubot.message.command import Command
 from nemubot.message.command import OwnerCommand
-
-
-def reload():
-    global Abstract, Text, DirectAsk, Command, OwnerCommand
-    import imp
-
-    import nemubot.message.abstract
-    imp.reload(nemubot.message.abstract)
-    Abstract = nemubot.message.abstract.Abstract
-    imp.reload(nemubot.message.text)
-    Text = nemubot.message.text.Text
-    imp.reload(nemubot.message.directask)
-    DirectAsk = nemubot.message.directask.DirectAsk
-    imp.reload(nemubot.message.command)
-    Command = nemubot.message.command.Command
-    OwnerCommand = nemubot.message.command.OwnerCommand
-
-    import nemubot.message.visitor
-    imp.reload(nemubot.message.visitor)
-
-    import nemubot.message.printer
-    imp.reload(nemubot.message.printer)
-
-    nemubot.message.printer.reload()

nemubot/message/abstract.py

@@ -21,7 +21,7 @@ class Abstract:
 
     """This class represents an abstract message"""
 
-    def __init__(self, server=None, date=None, to=None, to_response=None, frm=None):
+    def __init__(self, server=None, date=None, to=None, to_response=None, frm=None, frm_owner=False):
         """Initialize an abstract message
 
         Arguments:
@@ -40,7 +40,7 @@ class Abstract:
                              else [ to_response ])
         self.frm = frm  # None allowed when it designate this bot
 
-        self.frm_owner = False  # Filled later, in consumer
+        self.frm_owner = frm_owner
 
 
     @property
@@ -59,12 +59,6 @@ class Abstract:
         else:
             return None
 
-    @property
-    def nick(self):
-        # TODO: this is for legacy modules
-        return self.frm
-
-
     def accept(self, visitor):
         visitor.visit(self)
 
@@ -78,7 +72,8 @@ class Abstract:
             "date": self.date,
             "to": self.to,
             "to_response": self._to_response,
-            "frm": self.frm
+            "frm": self.frm,
+            "frm_owner": self.frm_owner,
         }
 
         for w in without:

nemubot/message/command.py

@@ -22,7 +22,7 @@ class Command(Abstract):
     """This class represents a specialized TextMessage"""
 
     def __init__(self, cmd, args=None, kwargs=None, *nargs, **kargs):
-        Abstract.__init__(self, *nargs, **kargs)
+        super().__init__(*nargs, **kargs)
 
         self.cmd = cmd
         self.args = args if args is not None else list()
@@ -31,11 +31,6 @@ class Command(Abstract):
     def __str__(self):
         return self.cmd + " @" + ",@".join(self.args)
 
-    @property
-    def cmds(self):
-        # TODO: this is for legacy modules
-        return [self.cmd] + self.args
-
 
 class OwnerCommand(Command):
 

nemubot/message/directask.py

@@ -28,7 +28,7 @@ class DirectAsk(Text):
         designated -- the user designated by the message
         """
 
-        Text.__init__(self, *args, **kargs)
+        super().__init__(*args, **kargs)
 
         self.designated = designated
 

nemubot/message/printer/IRC.py

@@ -1,25 +0,0 @@
-# Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2016  Mercier Pierre-Olivier
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from nemubot.message import Text
-from nemubot.message.printer.socket import Socket as SocketPrinter
-
-
-class IRC(SocketPrinter):
-
-    def visit_Text(self, msg):
-        self.pp += "PRIVMSG %s :" % ",".join(msg.to)
-        SocketPrinter.visit_Text(self, msg)

nemubot/message/printer/IRCLib.py

@@ -0,0 +1,67 @@
+# Nemubot is a smart and modulable IM bot.
+# Copyright (C) 2012-2026  Mercier Pierre-Olivier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from nemubot.message.visitor import AbstractVisitor
+
+
+class IRCLib(AbstractVisitor):
+
+    """Visitor that sends bot responses via an irc.client.ServerConnection.
+
+    Unlike the socket-based IRC printer (which builds a raw PRIVMSG string),
+    this calls connection.privmsg() directly so the library handles encoding,
+    line-length capping, and any internal locking.
+    """
+
+    def __init__(self, connection):
+        self._conn = connection
+
+    def _send(self, target, text):
+        try:
+            self._conn.privmsg(target, text)
+        except Exception:
+            pass  # drop silently during reconnection
+
+    # Visitor methods
+
+    def visit_Text(self, msg):
+        if isinstance(msg.message, str):
+            for target in msg.to:
+                self._send(target, msg.message)
+        else:
+            msg.message.accept(self)
+
+    def visit_DirectAsk(self, msg):
+        text = msg.message if isinstance(msg.message, str) else str(msg.message)
+        # Mirrors socket.py logic:
+        # rooms that are NOT the designated nick get a "nick: " prefix
+        others = [to for to in msg.to if to != msg.designated]
+        if len(others) == 0 or len(others) != len(msg.to):
+            for target in msg.to:
+                self._send(target, text)
+        if others:
+            for target in others:
+                self._send(target, "%s: %s" % (msg.designated, text))
+
+    def visit_Command(self, msg):
+        parts = ["!" + msg.cmd] + list(msg.args)
+        for target in msg.to:
+            self._send(target, " ".join(parts))
+
+    def visit_OwnerCommand(self, msg):
+        parts = ["`" + msg.cmd] + list(msg.args)
+        for target in msg.to:
+            self._send(target, " ".join(parts))

nemubot/message/printer/Matrix.py

@@ -0,0 +1,69 @@
+# Nemubot is a smart and modulable IM bot.
+# Copyright (C) 2012-2026  Mercier Pierre-Olivier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from nemubot.message.visitor import AbstractVisitor
+
+
+class Matrix(AbstractVisitor):
+
+    """Visitor that sends bot responses as Matrix room messages.
+
+    Instead of accumulating text like the IRC printer does, each visit_*
+    method calls send_func(room_id, text) directly for every destination room.
+    """
+
+    def __init__(self, send_func):
+        """
+        Argument:
+        send_func -- callable(room_id: str, text: str) that sends a plain-text
+                     message to the given Matrix room
+        """
+        self._send = send_func
+
+    def visit_Text(self, msg):
+        if isinstance(msg.message, str):
+            for room in msg.to:
+                self._send(room, msg.message)
+        else:
+            # Nested message object — let it visit itself
+            msg.message.accept(self)
+
+    def visit_DirectAsk(self, msg):
+        text = msg.message if isinstance(msg.message, str) else str(msg.message)
+        # Rooms that are NOT the designated nick → prefix with "nick: "
+        others = [to for to in msg.to if to != msg.designated]
+        if len(others) == 0 or len(others) != len(msg.to):
+            # At least one room IS the designated target → send plain
+            for room in msg.to:
+                self._send(room, text)
+        if len(others):
+            # Other rooms → prefix with nick
+            for room in others:
+                self._send(room, "%s: %s" % (msg.designated, text))
+
+    def visit_Command(self, msg):
+        parts = ["!" + msg.cmd]
+        if msg.args:
+            parts.extend(msg.args)
+        for room in msg.to:
+            self._send(room, " ".join(parts))
+
+    def visit_OwnerCommand(self, msg):
+        parts = ["`" + msg.cmd]
+        if msg.args:
+            parts.extend(msg.args)
+        for room in msg.to:
+            self._send(room, " ".join(parts))

nemubot/message/printer/__init__.py

@@ -13,12 +13,3 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-def reload():
-    import imp
-
-    import nemubot.message.printer.IRC
-    imp.reload(nemubot.message.printer.IRC)
-
-    import nemubot.message.printer.socket
-    imp.reload(nemubot.message.printer.socket)

nemubot/message/printer/socket.py

@@ -35,7 +35,7 @@ class Socket(AbstractVisitor):
         others = [to for to in msg.to if to != msg.designated]
 
         # Avoid nick starting message when discussing on user channel
-        if len(others) != len(msg.to):
+        if len(others) == 0 or len(others) != len(msg.to):
             res = Text(msg.message,
                        server=msg.server, date=msg.date,
                        to=msg.to, frm=msg.frm)

nemubot/message/response.py

@@ -1,5 +1,3 @@
-# coding=utf-8
-
 # Nemubot is a smart and modulable IM bot.
 # Copyright (C) 2012-2015  Mercier Pierre-Olivier
 #
@@ -16,8 +14,16 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-class PromptReset(Exception):
+from nemubot.message.abstract import Abstract
 
-    def __init__(self, type):
-        super(PromptReset, self).__init__("Prompt reset asked")
-        self.type = type
+
+class Response(Abstract):
+
+    def __init__(self, cmd, args=None, *nargs, **kargs):
+        super().__init__(*nargs, **kargs)
+
+        self.cmd = cmd
+        self.args = args if args is not None else list()
+
+    def __str__(self):
+        return self.cmd + " @" + ",@".join(self.args)

nemubot/message/text.py

@@ -28,7 +28,7 @@ class Text(Abstract):
         message -- the parsed message
         """
 
-        Abstract.__init__(self, *args, **kargs)
+        super().__init__(*args, **kargs)
 
         self.message = message
 

nemubot/module/__init__.py

@@ -0,0 +1,7 @@
+#
+# This directory aims to store nemubot core modules.
+#
+# Custom modules should be placed into a separate directory.
+# By default, this is the directory modules in your current directory.
+# Use the --modules-path argument to define a custom directory for your modules.
+#

nemubot/module/more.py

@@ -181,8 +181,16 @@ class Response:
                 return self.nomore
 
         if self.line_treat is not None and self.elt == 0:
-            self.messages[0] = (self.line_treat(self.messages[0])
-                                .replace("\n", " ").strip())
+            try:
+                if isinstance(self.messages[0], list):
+                    for x in self.messages[0]:
+                        print(x, self.line_treat(x))
+                    self.messages[0] = [self.line_treat(x).replace("\n", " ").strip() for x in self.messages[0]]
+                else:
+                    self.messages[0] = (self.line_treat(self.messages[0])
+                                        .replace("\n", " ").strip())
+            except Exception as e:
+                logger.exception(e)
 
         msg = ""
         if self.title is not None:

nemubot/modulecontext.py

@@ -1,5 +1,5 @@
 # Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
+# Copyright (C) 2012-2017  Mercier Pierre-Olivier
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,105 +14,70 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-class ModuleContext:
+class _ModuleContext:
 
-    def __init__(self, context, module):
-        """Initialize the module context
-
-        arguments:
-        context -- the bot context
-        module -- the module
-        """
+    def __init__(self, module=None, knodes=None):
+        self.module = module
 
         if module is not None:
-            module_name = module.__spec__.name if hasattr(module, "__spec__") else module.__name__
+            self.module_name = (module.__spec__.name if hasattr(module, "__spec__") else module.__name__).replace("nemubot.module.", "")
         else:
-            module_name = ""
-
-        # Load module configuration if exists
-        if (context is not None and
-            module_name in context.modules_configuration):
-            self.config = context.modules_configuration[module_name]
-        else:
-            from nemubot.config.module import Module
-            self.config = Module(module_name)
+            self.module_name = ""
 
         self.hooks = list()
         self.events = list()
-        self.debug = context.verbosity > 0 if context is not None else False
+        self.debug = False
 
+        from nemubot.config.module import Module
+        self.config = Module(self.module_name)
+        self._knodes = knodes
+
+
+    def load_data(self):
+        from nemubot.tools.xmlparser import module_state
+        return module_state.ModuleState("nemubotstate")
+
+    def set_knodes(self, knodes):
+        self._knodes = knodes
+
+    def set_default(self, default):
+        # Access to data will trigger the load of data
+        if self.data is None:
+            self._data = default
+
+    def add_hook(self, hook, *triggers):
         from nemubot.hooks import Abstract as AbstractHook
+        assert isinstance(hook, AbstractHook), hook
+        self.hooks.append((triggers, hook))
 
-        # Define some callbacks
-        if context is not None:
-            def load_data():
-                return context.datastore.load(module_name)
+    def del_hook(self, hook, *triggers):
+        from nemubot.hooks import Abstract as AbstractHook
+        assert isinstance(hook, AbstractHook), hook
+        self.hooks.remove((triggers, hook))
 
-            def add_hook(hook, *triggers):
-                assert isinstance(hook, AbstractHook), hook
-                self.hooks.append((triggers, hook))
-                return context.treater.hm.add_hook(hook, *triggers)
+    def subtreat(self, msg):
+        return None
 
-            def del_hook(hook, *triggers):
-                assert isinstance(hook, AbstractHook), hook
-                self.hooks.remove((triggers, hook))
-                return context.treater.hm.del_hooks(*triggers, hook=hook)
+    def add_event(self, evt, eid=None):
+        return self.events.append((evt, eid))
 
-            def subtreat(msg):
-                yield from context.treater.treat_msg(msg)
-            def add_event(evt, eid=None):
-                return context.add_event(evt, eid, module_src=module)
-            def del_event(evt):
-                return context.del_event(evt, module_src=module)
+    def del_event(self, evt):
+        for i in self.events:
+            e, eid = i
+            if e == evt:
+                self.events.remove(i)
+                return True
+        return False
 
-            def send_response(server, res):
-                if server in context.servers:
-                    if res.server is not None:
-                        return context.servers[res.server].send_response(res)
-                    else:
-                        return context.servers[server].send_response(res)
-                else:
-                    module.logger.error("Try to send a message to the unknown server: %s", server)
-                    return False
+    def send_response(self, server, res):
+       self.module.logger.info("Send response: %s", res)
 
-        else:  # Used when using outside of nemubot
-            def load_data():
-                from nemubot.tools.xmlparser import module_state
-                return module_state.ModuleState("nemubotstate")
-
-            def add_hook(hook, *triggers):
-                assert isinstance(hook, AbstractHook), hook
-                self.hooks.append((triggers, hook))
-            def del_hook(hook, *triggers):
-                assert isinstance(hook, AbstractHook), hook
-                self.hooks.remove((triggers, hook))
-            def subtreat(msg):
-                return None
-            def add_event(evt, eid=None):
-                return context.add_event(evt, eid, module_src=module)
-            def del_event(evt):
-                return context.del_event(evt, module_src=module)
-
-            def send_response(server, res):
-                module.logger.info("Send response: %s", res)
-
-        def save():
-            context.datastore.save(module_name, self.data)
-
-        def subparse(orig, cnt):
-            if orig.server in context.servers:
-                return context.servers[orig.server].subparse(orig, cnt)
-
-        self.load_data = load_data
-        self.add_hook = add_hook
-        self.del_hook = del_hook
-        self.add_event = add_event
-        self.del_event = del_event
-        self.save = save
-        self.send_response = send_response
-        self.subtreat = subtreat
-        self.subparse = subparse
+    def save(self):
+        self.context.datastore.save(self.module_name, self.data)
 
+    def subparse(self, orig, cnt):
+        if orig.server in self.context.servers:
+            return self.context.servers[orig.server].subparse(orig, cnt)
 
     @property
     def data(self):
@@ -129,7 +94,62 @@ class ModuleContext:
             self.del_hook(h, *s)
 
         # Remove registered events
-        for e in self.events:
-            self.del_event(e)
+        for evt, eid in self.events:
+            self.del_event(evt)
 
         self.save()
+
+
+class ModuleContext(_ModuleContext):
+
+    def __init__(self, context, *args, **kwargs):
+        """Initialize the module context
+
+        arguments:
+        context -- the bot context
+        module -- the module
+        """
+
+        super().__init__(*args, **kwargs)
+
+        # Load module configuration if exists
+        if self.module_name in context.modules_configuration:
+            self.config = context.modules_configuration[self.module_name]
+
+        self.context = context
+        self.debug = context.debug
+
+
+    def load_data(self):
+        return self.context.datastore.load(self.module_name, self._knodes)
+
+    def add_hook(self, hook, *triggers):
+        from nemubot.hooks import Abstract as AbstractHook
+        assert isinstance(hook, AbstractHook), hook
+        self.hooks.append((triggers, hook))
+        return self.context.treater.hm.add_hook(hook, *triggers)
+
+    def del_hook(self, hook, *triggers):
+        from nemubot.hooks import Abstract as AbstractHook
+        assert isinstance(hook, AbstractHook), hook
+        self.hooks.remove((triggers, hook))
+        return self.context.treater.hm.del_hooks(*triggers, hook=hook)
+
+    def subtreat(self, msg):
+        yield from self.context.treater.treat_msg(msg)
+
+    def add_event(self, evt, eid=None):
+        return self.context.add_event(evt, eid, module_src=self.module)
+
+    def del_event(self, evt):
+        return self.context.del_event(evt, module_src=self.module)
+
+    def send_response(self, server, res):
+        if server in self.context.servers:
+            if res.server is not None:
+                return self.context.servers[res.server].send_response(res)
+            else:
+                return self.context.servers[server].send_response(res)
+        else:
+            self.module.logger.error("Try to send a message to the unknown server: %s", server)
+            return False

nemubot/prompt/__init__.py

@@ -1,142 +0,0 @@
-# Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import shlex
-import sys
-import traceback
-
-from nemubot.prompt import builtins
-
-
-class Prompt:
-
-    def __init__(self):
-        self.selectedServer = None
-        self.lastretcode = 0
-
-        self.HOOKS_CAPS = dict()
-        self.HOOKS_LIST = dict()
-
-    def add_cap_hook(self, name, call, data=None):
-        self.HOOKS_CAPS[name] = lambda t, c: call(t, data=data,
-                                                  context=c, prompt=self)
-
-    def add_list_hook(self, name, call):
-        self.HOOKS_LIST[name] = call
-
-    def lex_cmd(self, line):
-        """Return an array of tokens
-
-        Argument:
-        line -- the line to lex
-        """
-
-        try:
-            cmds = shlex.split(line)
-        except:
-            exc_type, exc_value, _ = sys.exc_info()
-            sys.stderr.write(traceback.format_exception_only(exc_type,
-                                                             exc_value)[0])
-            return
-
-        bgn = 0
-
-        # Separate commands (command separator: ;)
-        for i in range(0, len(cmds)):
-            if cmds[i][-1] == ';':
-                if i != bgn:
-                    yield cmds[bgn:i]
-                bgn = i + 1
-
-        # Return rest of the command (that not end with a ;)
-        if bgn != len(cmds):
-            yield cmds[bgn:]
-
-    def exec_cmd(self, toks, context):
-        """Execute the command
-
-        Arguments:
-        toks -- lexed tokens to executes
-        context -- current bot context
-        """
-
-        if toks[0] in builtins.CAPS:
-            self.lastretcode = builtins.CAPS[toks[0]](toks, context, self)
-        elif toks[0] in self.HOOKS_CAPS:
-            self.lastretcode = self.HOOKS_CAPS[toks[0]](toks, context)
-        else:
-            print("Unknown command: `%s'" % toks[0])
-            self.lastretcode = 127
-
-    def getPS1(self):
-        """Get the PS1 associated to the selected server"""
-        if self.selectedServer is None:
-            return "nemubot"
-        else:
-            return self.selectedServer.id
-
-    def run(self, context):
-        """Launch the prompt
-
-        Argument:
-        context -- current bot context
-        """
-
-        from nemubot.prompt.error import PromptError
-        from nemubot.prompt.reset import PromptReset
-
-        while True:  # Stopped by exception
-            try:
-                line = input("\033[0;33m%s\033[0;%dm§\033[0m " %
-                             (self.getPS1(), 31 if self.lastretcode else 32))
-                cmds = self.lex_cmd(line.strip())
-                for toks in cmds:
-                    try:
-                        self.exec_cmd(toks, context)
-                    except PromptReset:
-                        raise
-                    except PromptError as e:
-                        print(e.message)
-                        self.lastretcode = 128
-                    except:
-                        exc_type, exc_value, exc_traceback = sys.exc_info()
-                        traceback.print_exception(exc_type, exc_value,
-                                                  exc_traceback)
-            except KeyboardInterrupt:
-                print("")
-            except EOFError:
-                print("quit")
-                return True
-
-
-def hotswap(bak):
-    p = Prompt()
-    p.HOOKS_CAPS = bak.HOOKS_CAPS
-    p.HOOKS_LIST = bak.HOOKS_LIST
-    return p
-
-
-def reload():
-    import imp
-
-    import nemubot.prompt.builtins
-    imp.reload(nemubot.prompt.builtins)
-
-    import nemubot.prompt.error
-    imp.reload(nemubot.prompt.error)
-
-    import nemubot.prompt.reset
-    imp.reload(nemubot.prompt.reset)

nemubot/prompt/builtins.py

@@ -1,128 +0,0 @@
-# Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-def end(toks, context, prompt):
-    """Quit the prompt for reload or exit"""
-    from nemubot.prompt.reset import PromptReset
-
-    if toks[0] == "refresh":
-        raise PromptReset("refresh")
-    elif toks[0] == "reset":
-        raise PromptReset("reset")
-    raise PromptReset("quit")
-
-
-def liste(toks, context, prompt):
-    """Show some lists"""
-    if len(toks) > 1:
-        for l in toks[1:]:
-            l = l.lower()
-            if l == "server" or l == "servers":
-                for srv in context.servers.keys():
-                    print ("  - %s (state: %s) ;" % (srv,
-                                                     "connected" if context.servers[srv].connected else "disconnected"))
-                if len(context.servers) == 0:
-                    print ("  > No server loaded")
-
-            elif l == "mod" or l == "mods" or l == "module" or l == "modules":
-                for mod in context.modules.keys():
-                    print ("  - %s ;" % mod)
-                if len(context.modules) == 0:
-                    print ("  > No module loaded")
-
-            elif l in prompt.HOOKS_LIST:
-                f, d = prompt.HOOKS_LIST[l]
-                f(d, context, prompt)
-
-            else:
-                print ("  Unknown list `%s'" % l)
-                return 2
-        return 0
-    else:
-        print ("  Please give a list to show: servers, ...")
-        return 1
-
-
-def load(toks, context, prompt):
-    """Load an XML configuration file"""
-    if len(toks) > 1:
-        for filename in toks[1:]:
-            context.load_file(filename)
-    else:
-        print ("Not enough arguments. `load' takes a filename.")
-        return 1
-
-
-def select(toks, context, prompt):
-    """Select the current server"""
-    if (len(toks) == 2 and toks[1] != "None" and
-        toks[1] != "nemubot" and toks[1] != "none"):
-        if toks[1] in context.servers:
-            prompt.selectedServer = context.servers[toks[1]]
-        else:
-            print ("select: server `%s' not found." % toks[1])
-            return 1
-    else:
-        prompt.selectedServer = None
-
-
-def unload(toks, context, prompt):
-    """Unload a module"""
-    if len(toks) == 2 and toks[1] == "all":
-        for name in context.modules.keys():
-            context.unload_module(name)
-    elif len(toks) > 1:
-        for name in toks[1:]:
-            if context.unload_module(name):
-                print ("  Module `%s' successfully unloaded." % name)
-            else:
-                print ("  No module `%s' loaded, can't unload!" % name)
-                return 2
-    else:
-        print ("Not enough arguments. `unload' takes a module name.")
-        return 1
-
-
-def debug(toks, context, prompt):
-    """Enable/Disable debug mode on a module"""
-    if len(toks) > 1:
-        for name in toks[1:]:
-            if name in context.modules:
-                context.modules[name].DEBUG = not context.modules[name].DEBUG
-                if context.modules[name].DEBUG:
-                    print ("  Module `%s' now in DEBUG mode." % name)
-                else:
-                    print ("  Debug for module module `%s' disabled." % name)
-            else:
-                print ("  No module `%s' loaded, can't debug!" % name)
-                return 2
-    else:
-        print ("Not enough arguments. `debug' takes a module name.")
-        return 1
-
-
-# Register build-ins
-CAPS = {
-    'quit': end,       # Disconnect all server and quit
-    'exit': end,       # Alias for quit
-    'reset': end,      # Reload the prompt
-    'refresh': end,    # Reload the prompt but save modules
-    'load': load,      # Load a servers or module configuration file
-    'unload': unload,  # Unload a module and remove it from the list
-    'select': select,  # Select a server
-    'list': liste,     # Show lists
-    'debug': debug,    # Pass a module in debug mode
-}

nemubot/prompt/error.py

@@ -1,21 +0,0 @@
-# Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-class PromptError(Exception):
-
-    def __init__(self, message):
-        super(PromptError, self).__init__(message)
-        self.message = message

nemubot/server/DCC.py

@@ -1,239 +0,0 @@
-# Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import imp
-import os
-import re
-import socket
-import sys
-import time
-import threading
-import traceback
-
-import nemubot.message as message
-import nemubot.server as server
-
-#Store all used ports
-PORTS = list()
-
-class DCC(server.AbstractServer):
-    def __init__(self, srv, dest, socket=None):
-        server.Server.__init__(self)
-
-        self.error = False # An error has occur, closing the connection?
-        self.messages = list() # Message queued before connexion
-
-        # Informations about the sender
-        self.sender = dest
-        if self.sender is not None:
-            self.nick = (self.sender.split('!'))[0]
-            if self.nick != self.sender:
-                self.realname = (self.sender.split('!'))[1]
-            else:
-                self.realname = self.nick
-
-        # Keep the server
-        self.srv = srv
-        self.treatement = self.treat_msg
-
-        # Found a port for the connection
-        self.port = self.foundPort()
-
-        if self.port is None:
-            self.logger.critical("No more available slot for DCC connection")
-            self.setError("Il n'y a plus de place disponible sur le serveur"
-                          " pour initialiser une session DCC.")
-
-    def foundPort(self):
-        """Found a free port for the connection"""
-        for p in range(65432, 65535):
-            if p not in PORTS:
-                PORTS.append(p)
-                return p
-        return None
-
-    @property
-    def id(self):
-        """Gives the server identifiant"""
-        return self.srv.id + "/" + self.sender
-
-    def setError(self, msg):
-        self.error = True
-        self.srv.send_msg_usr(self.sender, msg)
-
-    def accept_user(self, host, port):
-        """Accept a DCC connection"""
-        self.s = socket.socket()
-        try:
-            self.s.connect((host, port))
-            self.logger.info("Accepted user from %s:%d for %s", host, port, self.sender)
-            self.connected = True
-            self.stop = False
-        except:
-            self.connected = False
-            self.error = True
-            return False
-        self.start()
-        return True
-
-
-    def request_user(self, type="CHAT", filename="CHAT", size=""):
-        """Create a DCC connection"""
-        #Open the port
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        try:
-            s.bind(('', self.port))
-        except:
-            try:
-                self.port = self.foundPort()
-                s.bind(('', self.port))
-            except:
-                self.setError("Une erreur s'est produite durant la tentative"
-                              " d'ouverture d'une session DCC.")
-                return False
-        self.logger.info("Listening on %d for %s", self.port, self.sender)
-
-        #Send CTCP request for DCC
-        self.srv.send_ctcp(self.sender,
-                           "DCC %s %s %d %d %s" % (type, filename, self.srv.ip,
-                                                   self.port, size),
-                           "PRIVMSG")
-
-        s.listen(1)
-        #Waiting for the client
-        (self.s, addr) = s.accept()
-        self.logger.info("Connected by %d", addr)
-        self.connected = True
-        return True
-
-    def send_dcc_raw(self, line):
-        self.s.sendall(line + b'\n')
-
-    def send_dcc(self, msg, to = None):
-        """If we talk to this user, send a message through this connection
-           else, send the message to the server class"""
-        if to is None or to == self.sender or to == self.nick:
-            if self.error:
-                self.srv.send_msg_final(self.nick, msg)
-            elif not self.connected or self.s is None:
-                try:
-                    self.start()
-                except RuntimeError:
-                    pass
-                self.messages.append(msg)
-            else:
-                for line in msg.split("\n"):
-                    self.send_dcc_raw(line.encode())
-        else:
-            self.srv.send_dcc(msg, to)
-
-    def send_file(self, filename):
-        """Send a file over DCC"""
-        if os.path.isfile(filename):
-            self.messages = filename
-            try:
-                self.start()
-            except RuntimeError:
-                pass
-        else:
-            self.logger.error("File not found `%s'", filename)
-
-    def run(self):
-        self.stopping.clear()
-
-        # Send file connection
-        if not isinstance(self.messages, list):
-            self.request_user("SEND",
-                              os.path.basename(self.messages),
-                              os.path.getsize(self.messages))
-            if self.connected:
-                with open(self.messages, 'rb') as f:
-                    d = f.read(268435456) #Packets size: 256Mo
-                    while d:
-                        self.s.sendall(d)
-                        self.s.recv(4) #The client send a confirmation after each packet
-                        d = f.read(268435456) #Packets size: 256Mo
-
-        # Messages connection
-        else:
-            if not self.connected:
-                if not self.request_user():
-                    #TODO: do something here
-                    return False
-
-            #Start by sending all queued messages
-            for mess in self.messages:
-                self.send_dcc(mess)
-
-            time.sleep(1)
-
-            readbuffer = b''
-            self.nicksize = len(self.srv.nick)
-            self.Bnick = self.srv.nick.encode()
-            while not self.stop:
-                raw = self.s.recv(1024) #recieve server messages
-                if not raw:
-                    break
-                readbuffer = readbuffer + raw
-                temp = readbuffer.split(b'\n')
-                readbuffer = temp.pop()
-
-                for line in temp:
-                    self.treatement(line)
-
-        if self.connected:
-            self.s.close()
-            self.connected = False
-
-        #Remove from DCC connections server list
-        if self.realname in self.srv.dcc_clients:
-            del self.srv.dcc_clients[self.realname]
-
-        self.logger.info("Closing connection with %s", self.nick)
-        self.stopping.set()
-        if self.closing_event is not None:
-            self.closing_event()
-        #Rearm Thread
-        threading.Thread.__init__(self)
-
-    def treat_msg(self, line):
-        """Treat a receive message, *can be overwritten*"""
-        if line == b'NEMUBOT###':
-            bot = self.srv.add_networkbot(self.srv, self.sender, self)
-            self.treatement = bot.treat_msg
-            self.send_dcc("NEMUBOT###")
-        elif (line[:self.nicksize] == self.Bnick and
-            line[self.nicksize+1:].strip()[:10] == b'my name is'):
-            name = line[self.nicksize+1:].strip()[11:].decode('utf-8',
-                                                         'replace')
-            if re.match("^[a-zA-Z0-9_-]+$", name):
-                if name not in self.srv.dcc_clients:
-                    del self.srv.dcc_clients[self.sender]
-                    self.nick = name
-                    self.sender = self.nick + "!" + self.realname
-                    self.srv.dcc_clients[self.realname] = self
-                    self.send_dcc("Hi " + self.nick)
-                else:
-                    self.send_dcc("This nickname is already in use"
-                                  ", please choose another one.")
-            else:
-                self.send_dcc("The name you entered contain"
-                              " invalid char.")
-        else:
-            self.srv.treat_msg(
-                (":%s PRIVMSG %s :" % (
-                        self.sender,self.srv.nick)).encode() + line,
-                True)

nemubot/server/IRC.py

@@ -1,275 +0,0 @@
-# Nemubot is a smart and modulable IM bot.
-# Copyright (C) 2012-2015  Mercier Pierre-Olivier
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from datetime import datetime
-import re
-
-from nemubot.channel import Channel
-from nemubot.message.printer.IRC import IRC as IRCPrinter
-from nemubot.server.message.IRC import IRC as IRCMessage
-from nemubot.server.socket import SocketServer
-
-
-class IRC(SocketServer):
-
-    """Concrete implementation of a connexion to an IRC server"""
-
-    def __init__(self, host="localhost", port=6667, ssl=False, owner=None,
-                 nick="nemubot", username=None, password=None,
-                 realname="Nemubot", encoding="utf-8", caps=None,
-                 channels=list(), on_connect=None):
-        """Prepare a connection with an IRC server
-
-        Keyword arguments:
-        host -- host to join
-        port -- port on the host to reach
-        ssl -- is this server using a TLS socket
-        owner -- bot's owner
-        nick -- bot's nick
-        username -- the username as sent to server
-        password -- if a password is required to connect to the server
-        realname -- the bot's realname
-        encoding -- the encoding used on the whole server
-        caps -- client capabilities to register on the server
-        channels -- list of channels to join on connection
-        on_connect -- generator to call when connection is done
-        """
-
-        self.username = username if username is not None else nick
-        self.password = password
-        self.nick     = nick
-        self.owner    = owner
-        self.realname = realname
-
-        self.id       = self.username + "@" + host + ":" + str(port)
-        SocketServer.__init__(self, host=host, port=port, ssl=ssl)
-        self.printer  = IRCPrinter
-
-        self.encoding = encoding
-
-        # Keep a list of joined channels
-        self.channels = dict()
-
-        # Server/client capabilities
-        self.capabilities = caps
-
-        # Register CTCP capabilities
-        self.ctcp_capabilities = dict()
-
-        def _ctcp_clientinfo(msg, cmds):
-            """Response to CLIENTINFO CTCP message"""
-            return " ".join(self.ctcp_capabilities.keys())
-
-        def _ctcp_dcc(msg, cmds):
-            """Response to DCC CTCP message"""
-            try:
-                import ipaddress
-                ip = ipaddress.ip_address(int(cmds[3]))
-                port = int(cmds[4])
-                conn = DCC(srv, msg.sender)
-            except:
-                return "ERRMSG invalid parameters provided as DCC CTCP request"
-
-            self.logger.info("Receive DCC connection request from %s to %s:%d", conn.sender, ip, port)
-
-            if conn.accept_user(ip, port):
-                srv.dcc_clients[conn.sender] = conn
-                conn.send_dcc("Hello %s!" % conn.nick)
-            else:
-                self.logger.error("DCC: unable to connect to %s:%d", ip, port)
-                return "ERRMSG unable to connect to %s:%d" % (ip, port)
-
-        import nemubot
-
-        self.ctcp_capabilities["ACTION"] = lambda msg, cmds: None
-        self.ctcp_capabilities["CLIENTINFO"] = _ctcp_clientinfo
-        #self.ctcp_capabilities["DCC"] = _ctcp_dcc
-        self.ctcp_capabilities["FINGER"] = lambda msg, cmds: "VERSION nemubot v%s" % nemubot.__version__
-        self.ctcp_capabilities["NEMUBOT"] = lambda msg, cmds: "NEMUBOT %s" % nemubot.__version__
-        self.ctcp_capabilities["PING"] = lambda msg, cmds: "PING %s" % " ".join(cmds[1:])
-        self.ctcp_capabilities["SOURCE"] = lambda msg, cmds: "SOURCE https://github.com/nemunaire/nemubot"
-        self.ctcp_capabilities["TIME"] = lambda msg, cmds: "TIME %s" % (datetime.now())
-        self.ctcp_capabilities["USERINFO"] = lambda msg, cmds: "USERINFO %s" % self.realname
-        self.ctcp_capabilities["VERSION"] = lambda msg, cmds: "VERSION nemubot v%s" % nemubot.__version__
-
-        # TODO: Temporary fix, waiting for hook based CTCP management
-        self.ctcp_capabilities["TYPING"] = lambda msg, cmds: None
-
-        self.logger.debug("CTCP capabilities setup: %s", ", ".join(self.ctcp_capabilities))
-
-
-        # Register hooks on some IRC CMD
-        self.hookscmd = dict()
-
-        # Respond to PING
-        def _on_ping(msg):
-            self.write(b"PONG :" + msg.params[0])
-        self.hookscmd["PING"] = _on_ping
-
-        # Respond to 001
-        def _on_connect(msg):
-            # First, send user defined command
-            if on_connect is not None:
-                if callable(on_connect):
-                    toc = on_connect()
-                else:
-                    toc = on_connect
-                if toc is not None:
-                    for oc in toc:
-                        self.write(oc)
-            # Then, JOIN some channels
-            for chn in channels:
-                if chn.password:
-                    self.write("JOIN %s %s" % (chn.name, chn.password))
-                else:
-                    self.write("JOIN %s" % chn.name)
-        self.hookscmd["001"] = _on_connect
-
-        # Respond to ERROR
-        def _on_error(msg):
-            self.close()
-        self.hookscmd["ERROR"] = _on_error
-
-        # Respond to CAP
-        def _on_cap(msg):
-            if len(msg.params) != 3 or msg.params[1] != b"LS":
-                return
-            server_caps = msg.params[2].decode().split(" ")
-            for cap in self.capabilities:
-                if cap not in server_caps:
-                    self.capabilities.remove(cap)
-            if len(self.capabilities) > 0:
-                self.write("CAP REQ :" + " ".join(self.capabilities))
-            self.write("CAP END")
-        self.hookscmd["CAP"] = _on_cap
-
-        # Respond to JOIN
-        def _on_join(msg):
-            if len(msg.params) == 0:
-                return
-
-            for chname in msg.decode(msg.params[0]).split(","):
-                # Register the channel
-                chan = Channel(chname)
-                self.channels[chname] = chan
-        self.hookscmd["JOIN"] = _on_join
-        # Respond to PART
-        def _on_part(msg):
-            if len(msg.params) != 1 and len(msg.params) != 2:
-                return
-
-            for chname in msg.params[0].split(b","):
-                if chname in self.channels:
-                    if msg.nick == self.nick:
-                        del self.channels[chname]
-                    elif msg.nick in self.channels[chname].people:
-                        del self.channels[chname].people[msg.nick]
-        self.hookscmd["PART"] = _on_part
-        # Respond to 331/RPL_NOTOPIC,332/RPL_TOPIC,TOPIC
-        def _on_topic(msg):
-            if len(msg.params) != 1 and len(msg.params) != 2:
-                return
-            if msg.params[0] in self.channels:
-                if len(msg.params) == 1 or len(msg.params[1]) == 0:
-                    self.channels[msg.params[0]].topic = None
-                else:
-                    self.channels[msg.params[0]].topic = msg.decode(msg.params[1])
-        self.hookscmd["331"] = _on_topic
-        self.hookscmd["332"] = _on_topic
-        self.hookscmd["TOPIC"] = _on_topic
-        # Respond to 353/RPL_NAMREPLY
-        def _on_353(msg):
-            if len(msg.params) == 3:
-                msg.params.pop(0)  # 353: like RFC 1459
-            if len(msg.params) != 2:
-                return
-            if msg.params[0] in self.channels:
-                for nk in msg.decode(msg.params[1]).split(" "):
-                    res = re.match("^(?P<level>[^a-zA-Z[\]\\`_^{|}])(?P<nickname>[a-zA-Z[\]\\`_^{|}][a-zA-Z0-9[\]\\`_^{|}-]*)$")
-                    self.channels[msg.params[0]].people[res.group("nickname")] = res.group("level")
-        self.hookscmd["353"] = _on_353
-
-        # Respond to INVITE
-        def _on_invite(msg):
-            if len(msg.params) != 2:
-                return
-            self.write("JOIN " + msg.decode(msg.params[1]))
-        self.hookscmd["INVITE"] = _on_invite
-
-        # Respond to ERR_NICKCOLLISION
-        def _on_nickcollision(msg):
-            self.nick += "_"
-            self.write("NICK " + self.nick)
-        self.hookscmd["433"] = _on_nickcollision
-        self.hookscmd["436"] = _on_nickcollision
-
-        # Handle CTCP requests
-        def _on_ctcp(msg):
-            if len(msg.params) != 2 or not msg.is_ctcp:
-                return
-            cmds = msg.decode(msg.params[1][1:len(msg.params[1])-1]).split(' ')
-            if cmds[0] in self.ctcp_capabilities:
-                res = self.ctcp_capabilities[cmds[0]](msg, cmds)
-            else:
-                res = "ERRMSG Unknown or unimplemented CTCP request"
-            if res is not None:
-                self.write("NOTICE %s :\x01%s\x01" % (msg.nick, res))
-        self.hookscmd["PRIVMSG"] = _on_ctcp
-
-
-    # Open/close
-
-    def _open(self):
-        if SocketServer._open(self):
-            if self.password is not None:
-                self.write("PASS :" + self.password)
-            if self.capabilities is not None:
-                self.write("CAP LS")
-            self.write("NICK :" + self.nick)
-            self.write("USER %s %s bla :%s" % (self.username, self.host, self.realname))
-            return True
-        return False
-
-
-    def _close(self):
-        if self.connected: self.write("QUIT")
-        return SocketServer._close(self)
-
-
-    # Writes: as inherited
-
-    # Read
-
-    def read(self):
-        for line in SocketServer.read(self):
-            # PING should be handled here, so start parsing here :/
-            msg = IRCMessage(line, self.encoding)
-
-            if msg.cmd in self.hookscmd:
-                self.hookscmd[msg.cmd](msg)
-
-            yield msg
-
-
-    def parse(self, msg):
-        mes = msg.to_bot_message(self)
-        if mes is not None:
-            yield mes
-
-
-    def subparse(self, orig, cnt):
-        msg = IRCMessage(("@time=%s :%s!user@host.com PRIVMSG %s :%s" % (orig.date.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), orig.frm, ",".join(orig.to), cnt)).encode(self.encoding), self.encoding)
-        return msg.to_bot_message(self)

nemubot/server/IRCLib.py

@@ -0,0 +1,375 @@
+# Nemubot is a smart and modulable IM bot.
+# Copyright (C) 2012-2026  Mercier Pierre-Olivier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from datetime import datetime
+import shlex
+import threading
+
+import irc.bot
+import irc.client
+import irc.connection
+
+import nemubot.message as message
+from nemubot.server.threaded import ThreadedServer
+
+
+class _IRCBotAdapter(irc.bot.SingleServerIRCBot):
+
+    """Internal adapter that bridges the irc library event model to nemubot.
+
+    Subclasses SingleServerIRCBot to get automatic reconnection, PING/PONG,
+    and nick-collision handling for free.
+    """
+
+    def __init__(self, server_name, push_fn, channels, on_connect_cmds,
+                 nick, server_list, owner=None, realname="Nemubot",
+                 encoding="utf-8", **connect_params):
+        super().__init__(server_list, nick, realname, **connect_params)
+        self._nemubot_name = server_name
+        self._push = push_fn
+        self._channels_to_join = channels
+        self._on_connect_cmds = on_connect_cmds or []
+        self.owner = owner
+        self.encoding = encoding
+        self._stop_event = threading.Event()
+
+
+    # Event loop control
+
+    def start(self):
+        """Run the reactor loop until stop() is called."""
+        self._connect()
+        while not self._stop_event.is_set():
+            self.reactor.process_once(timeout=0.2)
+
+    def stop(self):
+        """Signal the loop to exit and disconnect cleanly."""
+        self._stop_event.set()
+        try:
+            self.connection.disconnect("Goodbye")
+        except Exception:
+            pass
+
+    def on_disconnect(self, connection, event):
+        """Reconnect automatically unless we are shutting down."""
+        if not self._stop_event.is_set():
+            super().on_disconnect(connection, event)
+
+
+    # Connection lifecycle
+
+    def on_welcome(self, connection, event):
+        """001 — run on_connect commands then join channels."""
+        for cmd in self._on_connect_cmds:
+            if callable(cmd):
+                for c in (cmd() or []):
+                    connection.send_raw(c)
+            else:
+                connection.send_raw(cmd)
+
+        for ch in self._channels_to_join:
+            if isinstance(ch, tuple):
+                connection.join(ch[0], ch[1] if len(ch) > 1 else "")
+            elif hasattr(ch, 'name'):
+                connection.join(ch.name, getattr(ch, 'password', "") or "")
+            else:
+                connection.join(str(ch))
+
+    def on_invite(self, connection, event):
+        """Auto-join on INVITE."""
+        if event.arguments:
+            connection.join(event.arguments[0])
+
+
+    # CTCP
+
+    def on_ctcp(self, connection, event):
+        """Handle CTCP requests (irc library >= 19 dispatches all to on_ctcp)."""
+        nick = irc.client.NickMask(event.source).nick
+        ctcp_type = event.arguments[0].upper() if event.arguments else ""
+        ctcp_arg  = event.arguments[1] if len(event.arguments) > 1 else ""
+        self._reply_ctcp(connection, nick, ctcp_type, ctcp_arg)
+
+    # Fallbacks for older irc library versions that dispatch per-type
+    def on_ctcpversion(self, connection, event):
+        import nemubot
+        nick = irc.client.NickMask(event.source).nick
+        connection.ctcp_reply(nick, "VERSION nemubot v%s" % nemubot.__version__)
+
+    def on_ctcpping(self, connection, event):
+        nick = irc.client.NickMask(event.source).nick
+        arg = event.arguments[0] if event.arguments else ""
+        connection.ctcp_reply(nick, "PING %s" % arg)
+
+    def _reply_ctcp(self, connection, nick, ctcp_type, ctcp_arg):
+        import nemubot
+        responses = {
+            "ACTION":     None,  # handled as on_action
+            "CLIENTINFO": "CLIENTINFO ACTION CLIENTINFO FINGER PING SOURCE TIME USERINFO VERSION",
+            "FINGER":     "FINGER nemubot v%s" % nemubot.__version__,
+            "PING":       "PING %s" % ctcp_arg,
+            "SOURCE":     "SOURCE https://github.com/nemunaire/nemubot",
+            "TIME":       "TIME %s" % datetime.now(),
+            "USERINFO":   "USERINFO Nemubot",
+            "VERSION":    "VERSION nemubot v%s" % nemubot.__version__,
+        }
+        if ctcp_type in responses and responses[ctcp_type] is not None:
+            connection.ctcp_reply(nick, responses[ctcp_type])
+
+
+    # Incoming messages
+
+    def _decode(self, text):
+        if isinstance(text, bytes):
+            try:
+                return text.decode("utf-8")
+            except UnicodeDecodeError:
+                return text.decode(self.encoding, "replace")
+        return text
+
+    def _make_message(self, connection, source, target, text):
+        """Convert raw IRC event data into a nemubot bot message."""
+        nick = irc.client.NickMask(source).nick
+        text = self._decode(text)
+        bot_nick = connection.get_nickname()
+        is_channel = irc.client.is_channel(target)
+        to          = [target] if is_channel else [nick]
+        to_response = [target] if is_channel else [nick]
+
+        common = dict(
+            server=self._nemubot_name,
+            to=to,
+            to_response=to_response,
+            frm=nick,
+            frm_owner=(nick == self.owner),
+        )
+
+        # "botname: text" or "botname, text"
+        if (text.startswith(bot_nick + ":") or
+                text.startswith(bot_nick + ",")):
+            inner = text[len(bot_nick) + 1:].strip()
+            return message.DirectAsk(designated=bot_nick, message=inner,
+                                     **common)
+
+        # "!command [args]"
+        if len(text) > 1 and text[0] == '!':
+            inner = text[1:].strip()
+            try:
+                args = shlex.split(inner)
+            except ValueError:
+                args = inner.split()
+            if args:
+                # Extract @key=value named arguments (same logic as IRC.py)
+                kwargs = {}
+                while len(args) > 1:
+                    arg = args[1]
+                    if len(arg) > 2 and arg[0:2] == '\\@':
+                        args[1] = arg[1:]
+                    elif len(arg) > 1 and arg[0] == '@':
+                        arsp = arg[1:].split("=", 1)
+                        kwargs[arsp[0]] = arsp[1] if len(arsp) == 2 else None
+                        args.pop(1)
+                        continue
+                    break
+                return message.Command(cmd=args[0], args=args[1:],
+                                       kwargs=kwargs, **common)
+
+        return message.Text(message=text, **common)
+
+    def on_pubmsg(self, connection, event):
+        msg = self._make_message(
+            connection, event.source, event.target,
+            event.arguments[0] if event.arguments else "",
+        )
+        if msg:
+            self._push(msg)
+
+    def on_privmsg(self, connection, event):
+        nick = irc.client.NickMask(event.source).nick
+        msg = self._make_message(
+            connection, event.source, nick,
+            event.arguments[0] if event.arguments else "",
+        )
+        if msg:
+            self._push(msg)
+
+    def on_action(self, connection, event):
+        """CTCP ACTION (/me) — delivered as a plain Text message."""
+        nick = irc.client.NickMask(event.source).nick
+        text = "/me %s" % (event.arguments[0] if event.arguments else "")
+        is_channel = irc.client.is_channel(event.target)
+        to = [event.target] if is_channel else [nick]
+        self._push(message.Text(
+            message=text,
+            server=self._nemubot_name,
+            to=to, to_response=to,
+            frm=nick, frm_owner=(nick == self.owner),
+        ))
+
+
+class IRCLib(ThreadedServer):
+
+    """IRC server using the irc Python library (jaraco).
+
+    Compared to the hand-rolled IRC.py implementation, this gets:
+      - Automatic exponential-backoff reconnection
+      - PING/PONG handled transparently
+      - Nick-collision suffix logic built-in
+    """
+
+    def __init__(self, host="localhost", port=6667, nick="nemubot",
+                 username=None, password=None, realname="Nemubot",
+                 encoding="utf-8", owner=None, channels=None,
+                 on_connect=None, ssl=False, **kwargs):
+        """Prepare a connection to an IRC server.
+
+        Keyword arguments:
+        host       -- IRC server hostname
+        port       -- IRC server port (default 6667)
+        nick       -- bot's nickname
+        username   -- username for USER command (defaults to nick)
+        password   -- server password (sent as PASS)
+        realname   -- bot's real name
+        encoding   -- fallback encoding for non-UTF-8 servers
+        owner      -- nick of the bot's owner (sets frm_owner on messages)
+        channels   -- list of channel names / (name, key) tuples to join
+        on_connect -- list of raw IRC commands (or a callable returning one)
+                      to send after receiving 001
+        ssl        -- wrap the connection in TLS
+        """
+        name = (username or nick) + "@" + host + ":" + str(port)
+        super().__init__(name=name)
+
+        self._host     = host
+        self._port     = int(port)
+        self._nick     = nick
+        self._username = username or nick
+        self._password = password
+        self._realname = realname
+        self._encoding = encoding
+        self.owner     = owner
+        self._channels = channels or []
+        self._on_connect_cmds = on_connect
+        self._ssl      = ssl
+
+        self._bot    = None
+        self._thread = None
+
+
+    # ThreadedServer hooks
+
+    def _start(self):
+        server_list = [irc.bot.ServerSpec(self._host, self._port,
+                                          self._password)]
+
+        connect_params = {"username": self._username}
+
+        if self._ssl:
+            import ssl as ssl_mod
+            ctx  = ssl_mod.create_default_context()
+            host = self._host  # capture for closure
+            connect_params["connect_factory"] = irc.connection.Factory(
+                wrapper=lambda sock: ctx.wrap_socket(sock,
+                                                     server_hostname=host)
+            )
+
+        self._bot = _IRCBotAdapter(
+            server_name=self.name,
+            push_fn=self._push_message,
+            channels=self._channels,
+            on_connect_cmds=self._on_connect_cmds,
+            nick=self._nick,
+            server_list=server_list,
+            owner=self.owner,
+            realname=self._realname,
+            encoding=self._encoding,
+            **connect_params,
+        )
+        self._thread = threading.Thread(
+            target=self._bot.start,
+            daemon=True,
+            name="nemubot.IRC/" + self.name,
+        )
+        self._thread.start()
+
+    def _stop(self):
+        if self._bot:
+            self._bot.stop()
+        if self._thread:
+            self._thread.join(timeout=5)
+
+
+    # Outgoing messages
+
+    def send_response(self, response):
+        if response is None:
+            return
+        if isinstance(response, list):
+            for r in response:
+                self.send_response(r)
+            return
+        if not self._bot:
+            return
+
+        from nemubot.message.printer.IRCLib import IRCLib as IRCLibPrinter
+        printer = IRCLibPrinter(self._bot.connection)
+        response.accept(printer)
+
+
+    # subparse: re-parse a plain string in the context of an existing message
+    # (used by alias, rnd, grep, cat, smmry, sms modules)
+
+    def subparse(self, orig, cnt):
+        bot_nick = (self._bot.connection.get_nickname()
+                    if self._bot else self._nick)
+        common = dict(
+            server=self.name,
+            to=orig.to,
+            to_response=orig.to_response,
+            frm=orig.frm,
+            frm_owner=orig.frm_owner,
+            date=orig.date,
+        )
+        text = cnt
+
+        if (text.startswith(bot_nick + ":") or
+                text.startswith(bot_nick + ",")):
+            inner = text[len(bot_nick) + 1:].strip()
+            return message.DirectAsk(designated=bot_nick, message=inner,
+                                     **common)
+
+        if len(text) > 1 and text[0] == '!':
+            inner = text[1:].strip()
+            try:
+                args = shlex.split(inner)
+            except ValueError:
+                args = inner.split()
+            if args:
+                kwargs = {}
+                while len(args) > 1:
+                    arg = args[1]
+                    if len(arg) > 2 and arg[0:2] == '\\@':
+                        args[1] = arg[1:]
+                    elif len(arg) > 1 and arg[0] == '@':
+                        arsp = arg[1:].split("=", 1)
+                        kwargs[arsp[0]] = arsp[1] if len(arsp) == 2 else None
+                        args.pop(1)
+                        continue
+                    break
+                return message.Command(cmd=args[0], args=args[1:],
+                                       kwargs=kwargs, **common)
+
+        return message.Text(message=text, **common)

nemubot/server/Matrix.py

@@ -0,0 +1,200 @@
+# Nemubot is a smart and modulable IM bot.
+# Copyright (C) 2012-2026  Mercier Pierre-Olivier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import asyncio
+import shlex
+import threading
+
+import nemubot.message as message
+from nemubot.server.threaded import ThreadedServer
+
+
+class Matrix(ThreadedServer):
+
+    """Matrix server implementation using matrix-nio's AsyncClient.
+
+    Runs an asyncio event loop in a daemon thread.  Incoming room messages are
+    converted to nemubot bot messages and pushed through the pipe; outgoing
+    responses are sent via the async client from the same event loop.
+    """
+
+    def __init__(self, homeserver, user_id, password=None, access_token=None,
+                 owner=None, nick=None, channels=None, **kwargs):
+        """Prepare a connection to a Matrix homeserver.
+
+        Keyword arguments:
+        homeserver   -- base URL of the homeserver, e.g. "https://matrix.org"
+        user_id      -- full MXID (@user:server) or bare localpart
+        password     -- login password (required if no access_token)
+        access_token -- pre-obtained access token (alternative to password)
+        owner        -- MXID of the bot owner (marks frm_owner on messages)
+        nick         -- display name / prefix for DirectAsk detection
+        channels     -- list of room IDs / aliases to join on connect
+        """
+
+        # Ensure fully-qualified MXID
+        if not user_id.startswith("@"):
+            host = homeserver.split("//")[-1].rstrip("/")
+            user_id = "@%s:%s" % (user_id, host)
+
+        super().__init__(name=user_id)
+
+        self.homeserver = homeserver
+        self.user_id = user_id
+        self.password = password
+        self.access_token = access_token
+        self.owner = owner
+        self.nick = nick or user_id
+
+        self._initial_rooms = channels or []
+        self._client = None
+        self._loop = None
+        self._thread = None
+
+
+    # Open/close
+
+    def _start(self):
+        self._thread = threading.Thread(
+            target=self._run_loop,
+            daemon=True,
+            name="nemubot.Matrix/" + self._name,
+        )
+        self._thread.start()
+
+    def _stop(self):
+        if self._client and self._loop and not self._loop.is_closed():
+            try:
+                asyncio.run_coroutine_threadsafe(
+                    self._client.close(), self._loop
+                ).result(timeout=5)
+            except Exception:
+                self._logger.exception("Error while closing Matrix client")
+        if self._thread:
+            self._thread.join(timeout=5)
+
+
+    # Asyncio thread
+
+    def _run_loop(self):
+        self._loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self._loop)
+        try:
+            self._loop.run_until_complete(self._async_main())
+        except Exception:
+            self._logger.exception("Unhandled exception in Matrix event loop")
+        finally:
+            self._loop.close()
+
+    async def _async_main(self):
+        from nio import AsyncClient, LoginError, RoomMessageText
+
+        self._client = AsyncClient(self.homeserver, self.user_id)
+
+        if self.access_token:
+            self._client.access_token = self.access_token
+            self._logger.info("Using provided access token for %s", self.user_id)
+        elif self.password:
+            resp = await self._client.login(self.password)
+            if isinstance(resp, LoginError):
+                self._logger.error("Matrix login failed: %s", resp.message)
+                return
+            self._logger.info("Logged in to Matrix as %s", self.user_id)
+        else:
+            self._logger.error("Need either password or access_token to connect")
+            return
+
+        self._client.add_event_callback(self._on_room_message, RoomMessageText)
+
+        for room in self._initial_rooms:
+            await self._client.join(room)
+            self._logger.info("Joined room %s", room)
+
+        await self._client.sync_forever(timeout=30000, full_state=True)
+
+
+    # Incoming messages
+
+    async def _on_room_message(self, room, event):
+        """Callback invoked by matrix-nio for each m.room.message event."""
+
+        if event.sender == self.user_id:
+            return  # ignore own messages
+
+        text = event.body
+        room_id = room.room_id
+        frm = event.sender
+
+        common_args = {
+            "server": self.name,
+            "to": [room_id],
+            "to_response": [room_id],
+            "frm": frm,
+            "frm_owner": frm == self.owner,
+        }
+
+        if len(text) > 1 and text[0] == '!':
+            text = text[1:].strip()
+            try:
+                args = shlex.split(text)
+            except ValueError:
+                args = text.split(' ')
+            msg = message.Command(cmd=args[0], args=args[1:], **common_args)
+
+        elif (text.lower().startswith(self.nick.lower() + ":")
+              or text.lower().startswith(self.nick.lower() + ",")):
+            text = text[len(self.nick) + 1:].strip()
+            msg = message.DirectAsk(designated=self.nick, message=text,
+                                    **common_args)
+
+        else:
+            msg = message.Text(message=text, **common_args)
+
+        self._push_message(msg)
+
+
+    # Outgoing messages
+
+    def send_response(self, response):
+        if response is None:
+            return
+        if isinstance(response, list):
+            for r in response:
+                self.send_response(r)
+            return
+
+        from nemubot.message.printer.Matrix import Matrix as MatrixPrinter
+        printer = MatrixPrinter(self._send_text)
+        response.accept(printer)
+
+    def _send_text(self, room_id, text):
+        """Thread-safe: schedule a Matrix room_send on the asyncio loop."""
+        if not self._client or not self._loop or self._loop.is_closed():
+            self._logger.warning("Cannot send: Matrix client not ready")
+            return
+        future = asyncio.run_coroutine_threadsafe(
+            self._client.room_send(
+                room_id=room_id,
+                message_type="m.room.message",
+                content={"msgtype": "m.text", "body": text},
+                ignore_unverified_devices=True,
+            ),
+            self._loop,
+        )
+        future.add_done_callback(
+            lambda f: self._logger.warning("Matrix send error: %s", f.exception())
+            if not f.cancelled() and f.exception() else None
+        )