From 293828786980f49f5dfca97283d528487d21e2a0 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 21 Jul 2017 07:26:00 +0200 Subject: [PATCH 001/127] cve: update and clean module, following NIST website changes --- modules/cve.py | 49 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/modules/cve.py b/modules/cve.py index 23a0302..fa09de5 100644 --- a/modules/cve.py +++ b/modules/cve.py @@ -10,7 +10,7 @@ from nemubot.tools.web import getURLContent, striphtml from more import Response -BASEURL_NIST = 'https://web.nvd.nist.gov/view/vuln/detail?vulnId=' +BASEURL_NIST = 'https://nvd.nist.gov/vuln/detail/' # MODULE CORE ######################################################### @@ -19,15 +19,40 @@ def get_cve(cve_id): search_url = BASEURL_NIST + quote(cve_id.upper()) soup = BeautifulSoup(getURLContent(search_url)) - vuln = soup.body.find(class_="vuln-detail") - cvss = vuln.findAll('div')[4] - return [ - "Base score: " + cvss.findAll('div')[0].findAll('a')[0].text.strip(), - vuln.findAll('p')[0].text, # description - striphtml(vuln.findAll('div')[0].text).strip(), # publication date - striphtml(vuln.findAll('div')[1].text).strip(), # last revised - ] + return { + "description": soup.body.find(attrs={"data-testid":"vuln-description"}).text.strip(), + "published": soup.body.find(attrs={"data-testid":"vuln-published-on"}).text.strip(), + "last_modified": soup.body.find(attrs={"data-testid":"vuln-last-modified-on"}).text.strip(), + "source": soup.body.find(attrs={"data-testid":"vuln-source"}).text.strip(), + + "base_score": float(soup.body.find(attrs={"data-testid":"vuln-cvssv3-base-score-link"}).text.strip()), + "severity": soup.body.find(attrs={"data-testid":"vuln-cvssv3-base-score-severity"}).text.strip(), + "impact_score": float(soup.body.find(attrs={"data-testid":"vuln-cvssv3-impact-score"}).text.strip()), + "exploitability_score": float(soup.body.find(attrs={"data-testid":"vuln-cvssv3-exploitability-score"}).text.strip()), + + "av": soup.body.find(attrs={"data-testid":"vuln-cvssv3-av"}).text.strip(), + "ac": soup.body.find(attrs={"data-testid":"vuln-cvssv3-ac"}).text.strip(), + "pr": soup.body.find(attrs={"data-testid":"vuln-cvssv3-pr"}).text.strip(), + "ui": soup.body.find(attrs={"data-testid":"vuln-cvssv3-ui"}).text.strip(), + "s": soup.body.find(attrs={"data-testid":"vuln-cvssv3-s"}).text.strip(), + "c": soup.body.find(attrs={"data-testid":"vuln-cvssv3-c"}).text.strip(), + "i": soup.body.find(attrs={"data-testid":"vuln-cvssv3-i"}).text.strip(), + "a": soup.body.find(attrs={"data-testid":"vuln-cvssv3-a"}).text.strip(), + } + + +def display_metrics(av, ac, pr, ui, s, c, i, a, **kwargs): + ret = [] + if av != "None": ret.append("Attack Vector: \x02%s\x0F" % av) + if ac != "None": ret.append("Attack Complexity: \x02%s\x0F" % ac) + if pr != "None": ret.append("Privileges Required: \x02%s\x0F" % pr) + if ui != "None": ret.append("User Interaction: \x02%s\x0F" % ui) + if s != "Unchanged": ret.append("Scope: \x02%s\x0F" % s) + if c != "None": ret.append("Confidentiality: \x02%s\x0F" % c) + if i != "None": ret.append("Integrity: \x02%s\x0F" % i) + if a != "None": ret.append("Availability: \x02%s\x0F" % a) + return ', '.join(ret) # MODULE INTERFACE #################################################### @@ -42,6 +67,10 @@ def get_cve_desc(msg): if cve_id[:3].lower() != 'cve': cve_id = 'cve-' + cve_id - res.append_message(get_cve(cve_id)) + cve = get_cve(cve_id) + metrics = display_metrics(**cve) + res.append_message("{cveid}: Base score: \x02{base_score} {severity}\x0F (impact: \x02{impact_score}\x0F, exploitability: \x02{exploitability_score}\x0F; {metrics}), from \x02{source}\x0F, last modified on \x02{last_modified}\x0F. {description}".format(cveid=cve_id, metrics=metrics, **cve)) return res + +print(get_cve("CVE-2017-11108")) From 0d52fff64a9739745b8e2344ca8e1bc70b4322ba Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 22 Jul 2017 10:49:38 +0200 Subject: [PATCH 002/127] tools/web: display socket timeout --- nemubot/tools/web.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index 0852664..9ced693 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -15,6 +15,7 @@ # along with this program. If not, see . from urllib.parse import urljoin, urlparse, urlsplit, urlunsplit +import socket from nemubot.exception import IMException @@ -123,6 +124,8 @@ def getURLContent(url, body=None, timeout=7, header=None): o.path, body, header) + except socket.timeout as e: + raise IMException(e) except OSError as e: raise IMException(e.strerror) From bc183bcfa0d10f67345d39f2a158587e389c9342 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 22 Jul 2017 10:53:08 +0200 Subject: [PATCH 003/127] tools/web: new option decode_error to decode non-200 page content (useful on REST API) --- nemubot/tools/web.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index 9ced693..0394aac 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -68,13 +68,14 @@ def getPassword(url): # Get real pages -def getURLContent(url, body=None, timeout=7, header=None): +def getURLContent(url, body=None, timeout=7, header=None, decode_error=False): """Return page content corresponding to URL or None if any error occurs Arguments: url -- the URL to get body -- Data to send as POST content timeout -- maximum number of seconds to wait before returning an exception + decode_error -- raise exception on non-200 pages or ignore it """ o = urlparse(_getNormalizedURL(url), "http") @@ -166,7 +167,10 @@ def getURLContent(url, body=None, timeout=7, header=None): urljoin(url, res.getheader("Location")), body=body, timeout=timeout, - header=header) + header=header, + decode_error=decode_error) + elif decode_error: + return data.decode(charset).strip() else: raise IMException("A HTTP error occurs: %d - %s" % (res.status, http.client.responses[res.status])) From 7eac685a0ab0eb6b7f374a6e849cf02ead2b97b9 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 26 Jul 2017 07:51:35 +0200 Subject: [PATCH 004/127] openroute: new module providing geocode and direction instructions Closing issue #46 --- modules/openroute.py | 158 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 modules/openroute.py diff --git a/modules/openroute.py b/modules/openroute.py new file mode 100644 index 0000000..440b05a --- /dev/null +++ b/modules/openroute.py @@ -0,0 +1,158 @@ +"""Lost? use our commands to find your way!""" + +# PYTHON STUFFS ####################################################### + +import re +import urllib.parse + +from nemubot.exception import IMException +from nemubot.hooks import hook +from nemubot.tools import web + +from more import Response + +# GLOBALS ############################################################# + +URL_DIRECTIONS_API = "https://api.openrouteservice.org/directions?api_key=%s&" +URL_GEOCODE_API = "https://api.openrouteservice.org/geocoding?api_key=%s&" + +waytype = [ + "unknown", + "state road", + "road", + "street", + "path", + "track", + "cycleway", + "footway", + "steps", + "ferry", + "construction", +] + + +# LOADING ############################################################# + +def load(context): + if not context.config or "apikey" not in context.config: + raise ImportError("You need an OpenRouteService API key in order to use this " + "module. Add it to the module configuration file:\n" + "\nRegister at https://developers.openrouteservice.org") + global URL_DIRECTIONS_API + URL_DIRECTIONS_API = URL_DIRECTIONS_API % context.config["apikey"] + global URL_GEOCODE_API + URL_GEOCODE_API = URL_GEOCODE_API % context.config["apikey"] + + +# MODULE CORE ######################################################### + +def approx_distance(lng): + if lng > 1111: + return "%f km" % (lng / 1000) + else: + return "%f m" % lng + + +def approx_duration(sec): + days = int(sec / 86400) + if days > 0: + return "%d days %f hours" % (days, (sec % 86400) / 3600) + hours = int((sec % 86400) / 3600) + if hours > 0: + return "%d hours %f minutes" % (hours, (sec % 3600) / 60) + minutes = (sec % 3600) / 60 + if minutes > 0: + return "%d minutes" % minutes + else: + return "%d seconds" % sec + + +def geocode(query, limit=7): + obj = web.getJSON(URL_GEOCODE_API + urllib.parse.urlencode({ + 'query': query, + 'limit': limit, + })) + + for f in obj["features"]: + yield f["geometry"]["coordinates"], f["properties"] + + +def firstgeocode(query): + for g in geocode(query, limit=1): + return g + + +def where(loc): + return "{name} {city} {state} {county} {country}".format(**loc) + + +def directions(coordinates, **kwargs): + kwargs['coordinates'] = '|'.join(coordinates) + + print(URL_DIRECTIONS_API + urllib.parse.urlencode(kwargs)) + return web.getJSON(URL_DIRECTIONS_API + urllib.parse.urlencode(kwargs), decode_error=True) + + +# MODULE INTERFACE #################################################### + +@hook.command("geocode", + help="Get GPS coordinates of a place", + help_usage={ + "PLACE": "Get GPS coordinates of PLACE" + }) +def cmd_geocode(msg): + res = Response(channel=msg.channel, nick=msg.frm, + nomore="No more geocode", count=" (%s more geocode)") + + for loc in geocode(' '.join(msg.args)): + res.append_message("%s is at %s,%s" % ( + where(loc[1]), + loc[0][1], loc[0][0], + )) + + return res + + +@hook.command("directions", + help="Get routing instructions", + help_usage={ + "POINT1 POINT2 ...": "Get routing instructions to go from POINT1 to the last POINTX via intermediates POINTX" + }, + keywords={ + "profile=PROF": "One of driving-car, driving-hgv, cycling-regular, cycling-road, cycling-safe, cycling-mountain, cycling-tour, cycling-electric, foot-walking, foot-hiking, wheelchair. Default: foot-walking", + "preference=PREF": "One of fastest, shortest, recommended. Default: recommended", + "lang=LANG": "default: en", + }) +def cmd_directions(msg): + drcts = directions(["{0},{1}".format(*firstgeocode(g)[0]) for g in msg.args], + profile=msg.kwargs["profile"] if "profile" in msg.kwargs else "foot-walking", + preference=msg.kwargs["preference"] if "preference" in msg.kwargs else "recommended", + units="m", + language=msg.kwargs["lang"] if "lang" in msg.kwargs else "en", + geometry=False, + instructions=True, + instruction_format="text") + if "error" in drcts and "message" in drcts["error"] and drcts["error"]["message"]: + raise IMException(drcts["error"]["message"]) + + if "routes" not in drcts or not drcts["routes"]: + raise IMException("No route available for this trip") + + myway = drcts["routes"][0] + myway["summary"]["strduration"] = approx_duration(myway["summary"]["duration"]) + myway["summary"]["strdistance"] = approx_distance(myway["summary"]["distance"]) + res = Response("Trip summary: {strdistance} in approximate {strduration}; elevation +{ascent} m -{descent} m".format(**myway["summary"]), channel=msg.channel, count=" (%d more steps)", nomore="You have arrived!") + + def formatSegments(segments): + for segment in segments: + for step in segment["steps"]: + step["strtype"] = waytype[step["type"]] + step["strduration"] = approx_duration(step["duration"]) + step["strdistance"] = approx_distance(step["distance"]) + yield "{instruction} for {strdistance} on {strtype} (approximate time: {strduration})".format(**step) + + if "segments" in myway: + res.append_message([m for m in formatSegments(myway["segments"])]) + + return res From c8afa65dcb3458319a50db08628dec441fe3f482 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 27 Jul 2017 20:44:26 +0200 Subject: [PATCH 005/127] tools/xmlparser: implement writer --- nemubot/tools/test_xmlparser.py | 36 +++++++++++++++++++++++--- nemubot/tools/xmlparser/__init__.py | 15 +++++++++++ nemubot/tools/xmlparser/basic.py | 14 ++++++++++ nemubot/tools/xmlparser/genericnode.py | 8 ++++++ 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/nemubot/tools/test_xmlparser.py b/nemubot/tools/test_xmlparser.py index d7f5a9a..0feda73 100644 --- a/nemubot/tools/test_xmlparser.py +++ b/nemubot/tools/test_xmlparser.py @@ -1,5 +1,6 @@ import unittest +import io import xml.parsers.expat from nemubot.tools.xmlparser import XMLParser @@ -12,6 +13,11 @@ class StringNode(): def characters(self, content): self.string += content + def saveElement(self, store, tag="string"): + store.startElement(tag, {}) + store.characters(self.string) + store.endElement(tag) + class TestNode(): def __init__(self, option=None): @@ -22,6 +28,15 @@ class TestNode(): self.mystr = child.string return True + def saveElement(self, store, tag="test"): + store.startElement(tag, {"option": self.option}) + + strNode = StringNode() + strNode.string = self.mystr + strNode.saveElement(store) + + store.endElement(tag) + class Test2Node(): def __init__(self, option=None): @@ -33,6 +48,15 @@ class Test2Node(): self.mystrs.append(attrs["value"]) return True + def saveElement(self, store, tag="test"): + store.startElement(tag, {"option": self.option} if self.option is not None else {}) + + for mystr in self.mystrs: + store.startElement("string", {"value": mystr}) + store.endElement("string") + + store.endElement(tag) + class TestXMLParser(unittest.TestCase): @@ -44,9 +68,11 @@ class TestXMLParser(unittest.TestCase): p.CharacterDataHandler = mod.characters p.EndElementHandler = mod.endElement - p.Parse("toto", 1) + inputstr = "toto" + p.Parse(inputstr, 1) self.assertEqual(mod.root.string, "toto") + self.assertEqual(mod.saveDocument(header=False).getvalue(), inputstr) def test_parser2(self): @@ -57,10 +83,12 @@ class TestXMLParser(unittest.TestCase): p.CharacterDataHandler = mod.characters p.EndElementHandler = mod.endElement - p.Parse("toto", 1) + inputstr = 'toto' + p.Parse(inputstr, 1) self.assertEqual(mod.root.option, "123") self.assertEqual(mod.root.mystr, "toto") + self.assertEqual(mod.saveDocument(header=False).getvalue(), inputstr) def test_parser3(self): @@ -71,12 +99,14 @@ class TestXMLParser(unittest.TestCase): p.CharacterDataHandler = mod.characters p.EndElementHandler = mod.endElement - p.Parse("", 1) + inputstr = '' + p.Parse(inputstr, 1) self.assertEqual(mod.root.option, None) self.assertEqual(len(mod.root.mystrs), 2) self.assertEqual(mod.root.mystrs[0], "toto") self.assertEqual(mod.root.mystrs[1], "toto2") + self.assertEqual(mod.saveDocument(header=False, short_empty_elements=True).getvalue(), inputstr) if __name__ == '__main__': diff --git a/nemubot/tools/xmlparser/__init__.py b/nemubot/tools/xmlparser/__init__.py index abc5bb9..c8d393a 100644 --- a/nemubot/tools/xmlparser/__init__.py +++ b/nemubot/tools/xmlparser/__init__.py @@ -134,6 +134,21 @@ class XMLParser: return raise TypeError(name + " tag not expected in " + self.display_stack()) + def saveDocument(self, f=None, header=True, short_empty_elements=False): + if f is None: + import io + f = io.StringIO() + + import xml.sax.saxutils + gen = xml.sax.saxutils.XMLGenerator(f, "utf-8", short_empty_elements=short_empty_elements) + if header: + gen.startDocument() + self.root.saveElement(gen) + if header: + gen.endDocument() + + return f + def parse_file(filename): p = xml.parsers.expat.ParserCreate() diff --git a/nemubot/tools/xmlparser/basic.py b/nemubot/tools/xmlparser/basic.py index 8456629..f2d9fd5 100644 --- a/nemubot/tools/xmlparser/basic.py +++ b/nemubot/tools/xmlparser/basic.py @@ -44,6 +44,13 @@ class ListNode: return self.items.__repr__() + def saveElement(self, store, tag="list"): + store.startElement(tag, {}) + for i in self.items: + i.saveElement(store) + store.endElement(tag) + + class DictNode: """XML node representing a Python dictionnnary @@ -106,3 +113,10 @@ class DictNode: def __repr__(self): return self.items.__repr__() + + + def saveElement(self, store, tag="dict"): + store.startElement(tag, {}) + for k, v in self.items.items(): + v.saveElement(store) + store.endElement(tag) diff --git a/nemubot/tools/xmlparser/genericnode.py b/nemubot/tools/xmlparser/genericnode.py index 9c29a23..425934c 100644 --- a/nemubot/tools/xmlparser/genericnode.py +++ b/nemubot/tools/xmlparser/genericnode.py @@ -53,6 +53,14 @@ class ParsingNode: return item in self.attrs + def saveElement(self, store, tag=None): + store.startElement(tag if tag is not None else self.tag, self.attrs) + for child in self.children: + child.saveElement(store) + store.characters(self.content) + store.endElement(tag if tag is not None else self.tag) + + class GenericNode(ParsingNode): """Consider all subtags as dictionnary From 76bea2bc15c6ea2bdad90160c50bb66f8b8b95eb Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 28 Jul 2017 06:55:17 +0200 Subject: [PATCH 006/127] datastore/xml: handle entire file save and be closer with new nemubot XML API --- nemubot/datastore/xml.py | 13 ++++++++++++- nemubot/tools/xmlparser/node.py | 24 ++---------------------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/nemubot/datastore/xml.py b/nemubot/datastore/xml.py index 46dca70..025c0c5 100644 --- a/nemubot/datastore/xml.py +++ b/nemubot/datastore/xml.py @@ -143,4 +143,15 @@ class XML(Abstract): if self.rotate: self._rotate(path) - return data.save(path) + import tempfile + _, tmpath = tempfile.mkstemp() + with open(tmpath, "w") as f: + import xml.sax.saxutils + gen = xml.sax.saxutils.XMLGenerator(f, "utf-8") + gen.startDocument() + data.saveElement(gen) + gen.endDocument() + + # Atomic save + import shutil + shutil.move(tmpath, path) diff --git a/nemubot/tools/xmlparser/node.py b/nemubot/tools/xmlparser/node.py index 965a475..7df255e 100644 --- a/nemubot/tools/xmlparser/node.py +++ b/nemubot/tools/xmlparser/node.py @@ -196,7 +196,7 @@ class ModuleState: if self.index_fieldname is not None: self.setIndex(self.index_fieldname, self.index_tagname) - def save_node(self, gen): + def saveElement(self, gen): """Serialize this node as a XML node""" from datetime import datetime attribs = {} @@ -215,29 +215,9 @@ class ModuleState: gen.startElement(self.name, attrs) for child in self.childs: - child.save_node(gen) + child.saveElement(gen) gen.endElement(self.name) except: logger.exception("Error occured when saving the following " "XML node: %s with %s", self.name, attrs) - - def save(self, filename): - """Save the current node as root node in a XML file - - Argument: - filename -- location of the file to create/erase - """ - - import tempfile - _, tmpath = tempfile.mkstemp() - with open(tmpath, "w") as f: - import xml.sax.saxutils - gen = xml.sax.saxutils.XMLGenerator(f, "utf-8") - gen.startDocument() - self.save_node(gen) - gen.endDocument() - - # Atomic save - import shutil - shutil.move(tmpath, filename) From 9d0ab88c1227256a4462ef0fdaf10cf8f2126d68 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 29 Jul 2017 15:22:57 +0200 Subject: [PATCH 007/127] Store module into weakref --- nemubot/bot.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/nemubot/bot.py b/nemubot/bot.py index b0d3915..febe7d6 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -20,6 +20,7 @@ from multiprocessing import JoinableQueue import threading import select import sys +import weakref from nemubot import __version__ from nemubot.consumer import Consumer, EventConsumer, MessageConsumer @@ -99,15 +100,15 @@ class Bot(threading.Thread): from more import Response res = Response(channel=msg.to_response) if len(msg.args) >= 1: - if msg.args[0] in self.modules: - if hasattr(self.modules[msg.args[0]], "help_full"): - hlp = self.modules[msg.args[0]].help_full() + if msg.args[0] in self.modules and self.modules[msg.args[0]]() is not None: + if hasattr(self.modules[msg.args[0]](), "help_full"): + hlp = self.modules[msg.args[0]]().help_full() if isinstance(hlp, Response): return hlp else: res.append_message(hlp) else: - res.append_message([str(h) for s,h in self.modules[msg.args[0]].__nemubot_context__.hooks], title="Available commands for module " + msg.args[0]) + res.append_message([str(h) for s,h in self.modules[msg.args[0]]().__nemubot_context__.hooks], title="Available commands for module " + msg.args[0]) elif msg.args[0][0] == "!": from nemubot.message.command import Command for h in self.treater._in_hooks(Command(msg.args[0][1:])): @@ -137,7 +138,7 @@ class Bot(threading.Thread): res.append_message(title="Pour plus de détails sur un module, " "envoyez \"!help nomdumodule\". Voici la liste" " de tous les modules disponibles localement", - message=["\x03\x02%s\x03\x02 (%s)" % (im, self.modules[im].__doc__) for im in self.modules if self.modules[im].__doc__]) + message=["\x03\x02%s\x03\x02 (%s)" % (im, self.modules[im]().__doc__) for im in self.modules if self.modules[im]() is not None and self.modules[im]().__doc__]) return res self.treater.hm.add_hook(nemubot.hooks.Command(_help_msg, "help"), "in", "Command") @@ -518,18 +519,20 @@ class Bot(threading.Thread): raise # Save a reference to the module - self.modules[module_name] = module + self.modules[module_name] = weakref.ref(module) + logger.info("Module '%s' successfully loaded.", module_name) def unload_module(self, name): """Unload a module""" - if name in self.modules: - self.modules[name].print("Unloading module %s" % name) + if name in self.modules and self.modules[name]() is not None: + module = self.modules[name]() + module.print("Unloading module %s" % name) # Call the user defined unload method - if hasattr(self.modules[name], "unload"): - self.modules[name].unload(self) - self.modules[name].__nemubot_context__.unload() + if hasattr(module, "unload"): + module.unload(self) + module.__nemubot_context__.unload() # Remove from the nemubot dict del self.modules[name] From ef4a6e9af5341177c84acdc782db1bcc160f4a56 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 29 Jul 2017 15:25:44 +0200 Subject: [PATCH 008/127] Fix module unloading --- nemubot/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemubot/bot.py b/nemubot/bot.py index febe7d6..aa1cb3e 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -569,7 +569,7 @@ class Bot(threading.Thread): self.event_timer.cancel() logger.info("Save and unload all modules...") - for mod in self.modules.items(): + for mod in [m for m in self.modules.keys()]: self.unload_module(mod) logger.info("Close all servers connection...") From 7a527488497f540e793e1d047dd04ab06f749c93 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 8 Aug 2017 23:24:37 +0200 Subject: [PATCH 009/127] pkgs: new module to display quick information about common softwares --- modules/pkgs.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 modules/pkgs.py diff --git a/modules/pkgs.py b/modules/pkgs.py new file mode 100644 index 0000000..5a7b0a9 --- /dev/null +++ b/modules/pkgs.py @@ -0,0 +1,68 @@ +"""Get information about common software""" + +# PYTHON STUFFS ####################################################### + +import portage + +from nemubot import context +from nemubot.exception import IMException +from nemubot.hooks import hook + +from more import Response + +DB = None + +# MODULE CORE ######################################################### + +def get_db(): + global DB + if DB is None: + DB = portage.db[portage.root]["porttree"].dbapi + return DB + + +def package_info(pkgname): + pv = get_db().xmatch("match-all", pkgname) + if not pv: + raise IMException("No package named '%s' found" % pkgname) + + bv = get_db().xmatch("bestmatch-visible", pkgname) + pvsplit = portage.catpkgsplit(bv if bv else pv[-1]) + info = get_db().aux_get(bv if bv else pv[-1], ["DESCRIPTION", "HOMEPAGE", "LICENSE", "IUSE", "KEYWORDS"]) + + return { + "pkgname": '/'.join(pvsplit[:2]), + "category": pvsplit[0], + "shortname": pvsplit[1], + "lastvers": '-'.join(pvsplit[2:]) if pvsplit[3] != "r0" else pvsplit[2], + "othersvers": ['-'.join(portage.catpkgsplit(p)[2:]) for p in pv if p != bv], + "description": info[0], + "homepage": info[1], + "license": info[2], + "uses": info[3], + "keywords": info[4], + } + + +# MODULE INTERFACE #################################################### + +@hook.command("eix", + help="Get information about a package", + help_usage={ + "NAME": "Get information about a software NAME" + }) +def cmd_eix(msg): + if not len(msg.args): + raise IMException("please give me a package to search") + + def srch(term): + try: + yield package_info(term) + except portage.exception.AmbiguousPackageName as e: + for i in e.args[0]: + yield package_info(i) + + res = Response(channel=msg.channel, count=" (%d more packages)", nomore="No more package '%s'" % msg.args[0]) + for pi in srch(msg.args[0]): + res.append_message("\x03\x02{pkgname}:\x03\x02 {description} - {homepage} - {license} - last revisions: \x03\x02{lastvers}\x03\x02{ov}".format(ov=(", " + ', '.join(pi["othersvers"])) if pi["othersvers"] else "", **pi)) + return res From 990599551ca5b9729cb28370a870505fec74870c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 9 Aug 2017 22:53:35 +0200 Subject: [PATCH 010/127] suivi: fix error handling of fedex parcel --- modules/suivi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/suivi.py b/modules/suivi.py index a6f6ab4..24f5bf9 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -156,7 +156,9 @@ def get_fedex_info(fedex_id, lang="en_US"): if ("TrackPackagesResponse" in fedex_data and "packageList" in fedex_data["TrackPackagesResponse"] and - len(fedex_data["TrackPackagesResponse"]["packageList"]) + len(fedex_data["TrackPackagesResponse"]["packageList"]) and + not fedex_data["TrackPackagesResponse"]["errorList"][0]["code"] and + not fedex_data["TrackPackagesResponse"]["packageList"][0]["errorList"][0]["code"] ): return fedex_data["TrackPackagesResponse"]["packageList"][0] From 92702f3995b3e8b0da595ec1be91fca0a5f99fe5 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 10 Aug 2017 00:55:13 +0200 Subject: [PATCH 011/127] suivi: support DHL --- modules/suivi.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/modules/suivi.py b/modules/suivi.py index 24f5bf9..75a065b 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -163,6 +163,15 @@ def get_fedex_info(fedex_id, lang="en_US"): return fedex_data["TrackPackagesResponse"]["packageList"][0] +def get_dhl_info(dhl_id, lang="en"): + dhl_parcelurl = "http://www.dhl.com/shipmentTracking?" + urllib.parse.urlencode({'AWB': dhl_id}) + + dhl_data = getJSON(dhl_parcelurl) + + if "results" in dhl_data and dhl_data["results"]: + return dhl_data["results"][0] + + # TRACKING HANDLERS ################################################### def handle_tnt(tracknum): @@ -231,6 +240,12 @@ def handle_fedex(tracknum): return ("{trackingCarrierDesc}: \x02{statusWithDetails}\x0F: in \x02{statusLocationCity}, {statusLocationCntryCD}\x0F, estimated delivery: {displayEstDeliveryDateTime}.".format(**info)) +def handle_dhl(tracknum): + info = get_dhl_info(tracknum) + if info: + return "DHL {label} {id}: \x02{description}\x0F".format(**info) + + TRACKING_HANDLERS = { 'laposte': handle_laposte, 'postnl': handle_postnl, @@ -239,6 +254,7 @@ TRACKING_HANDLERS = { 'coliprive': handle_coliprive, 'tnt': handle_tnt, 'fedex': handle_fedex, + 'dhl': handle_dhl, } From 55a8f7490018489ada0383e22486084d9e1a91c9 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 10 Aug 2017 06:48:48 +0200 Subject: [PATCH 012/127] suivi: support USPS --- modules/suivi.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/modules/suivi.py b/modules/suivi.py index 75a065b..6ad13e9 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -126,6 +126,24 @@ def get_postnl_info(postnl_id): return (post_status.lower(), post_destination, post_date) +def get_usps_info(usps_id): + usps_parcelurl = "https://tools.usps.com/go/TrackConfirmAction_input?" + urllib.parse.urlencode({'qtc_tLabels1': usps_id}) + + usps_data = getURLContent(usps_parcelurl) + soup = BeautifulSoup(usps_data) + if (soup.find(class_="tracking_history") + and soup.find(class_="tracking_history").find(class_="row_notification") + and soup.find(class_="tracking_history").find(class_="row_top").find_all("td")): + notification = soup.find(class_="tracking_history").find(class_="row_notification").text.strip() + date = re.sub(r"\s+", " ", soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[0].text.strip()) + status = soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[1].text.strip() + last_location = soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[2].text.strip() + + print(notification) + + return (notification, date, status, last_location) + + def get_fedex_info(fedex_id, lang="en_US"): data = urllib.parse.urlencode({ 'data': json.dumps({ @@ -206,6 +224,13 @@ def handle_postnl(tracknum): ")." % (tracknum, post_status, post_destination, post_date)) +def handle_usps(tracknum): + info = get_usps_info(tracknum) + if info: + notif, last_date, last_status, last_location = info + return ("USPS \x02{tracknum}\x0F is {last_status} in \x02{last_location}\x0F as of {last_date}: {notif}".format(tracknum=tracknum, notif=notif, last_date=last_date, last_status=last_status.lower(), last_location=last_location)) + + def handle_colissimo(tracknum): info = get_colissimo_info(tracknum) if info: @@ -255,6 +280,7 @@ TRACKING_HANDLERS = { 'tnt': handle_tnt, 'fedex': handle_fedex, 'dhl': handle_dhl, + 'usps': handle_usps, } From 709128b7aad880e94a6e4654873c791782009807 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 4 Aug 2017 01:22:24 +0200 Subject: [PATCH 013/127] freetarifs: new module --- modules/freetarifs.py | 64 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 modules/freetarifs.py diff --git a/modules/freetarifs.py b/modules/freetarifs.py new file mode 100644 index 0000000..b96a30f --- /dev/null +++ b/modules/freetarifs.py @@ -0,0 +1,64 @@ +"""Inform about Free Mobile tarifs""" + +# PYTHON STUFFS ####################################################### + +import urllib.parse +from bs4 import BeautifulSoup + +from nemubot.exception import IMException +from nemubot.hooks import hook +from nemubot.tools import web + +from more import Response + + +# MODULE CORE ######################################################### + +ACT = { + "ff_toFixe": "Appel vers les fixes", + "ff_toMobile": "Appel vers les mobiles", + "ff_smsSendedToCountry": "SMS vers le pays", + "ff_mmsSendedToCountry": "MMS vers le pays", + "fc_callToFrance": "Appel vers la France", + "fc_smsToFrance": "SMS vers la france", + "fc_mmsSended": "MMS vers la france", + "fc_callToSameCountry": "Réception des appels", + "fc_callReceived": "Appel dans le pays", + "fc_smsReceived": "SMS (Réception)", + "fc_mmsReceived": "MMS (Réception)", + "fc_moDataFromCountry": "Data", +} + +def get_land_tarif(country, forfait="pkgFREE"): + url = "http://mobile.international.free.fr/?" + urllib.parse.urlencode({'pays': country}) + page = web.getURLContent(url) + soup = BeautifulSoup(page) + + fact = soup.find(class_=forfait) + + if fact is None: + raise IMException("Country or forfait not found.") + + res = {} + for s in ACT.keys(): + try: + res[s] = fact.find(attrs={"data-bind": "text: " + s}).text + " " + fact.find(attrs={"data-bind": "html: " + s + "Unit"}).text + except AttributeError: + res[s] = "inclus" + + return res + +@hook.command("freetarifs", + help="Show Free Mobile tarifs for given contries", + help_usage={"COUNTRY": "Show Free Mobile tarifs for given CONTRY"}, + keywords={ + "forfait=FORFAIT": "Related forfait between Free (default) and 2euro" + }) +def get_freetarif(msg): + res = Response(channel=msg.channel) + + for country in msg.args: + t = get_land_tarif(country.lower().capitalize(), "pkg" + (msg.kwargs["forfait"] if "forfait" in msg.kwargs else "FREE").upper()) + res.append_message(["\x02%s\x0F : %s" % (ACT[k], t[k]) for k in sorted(ACT.keys(), reverse=True)], title=country) + + return res From db1e4e92664e50b25f360e9a7e19df94e3cc230a Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 6 Aug 2017 12:27:19 +0200 Subject: [PATCH 014/127] disas: new module, aim to disassemble binary code. Closing #67 --- modules/disas.py | 85 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 modules/disas.py diff --git a/modules/disas.py b/modules/disas.py new file mode 100644 index 0000000..669ccc1 --- /dev/null +++ b/modules/disas.py @@ -0,0 +1,85 @@ +"""The Ultimate Disassembler Module""" + +# PYTHON STUFFS ####################################################### + +import capstone + +from nemubot.exception import IMException +from nemubot.hooks import hook + +from more import Response + + +# MODULE CORE ######################################################### + +ARCHITECTURES = { + "arm": capstone.CS_ARCH_ARM, + "arm64": capstone.CS_ARCH_ARM64, + "mips": capstone.CS_ARCH_MIPS, + "ppc": capstone.CS_ARCH_PPC, + "sparc": capstone.CS_ARCH_SPARC, + "sysz": capstone.CS_ARCH_SYSZ, + "x86": capstone.CS_ARCH_X86, + "xcore": capstone.CS_ARCH_XCORE, +} + +MODES = { + "arm": capstone.CS_MODE_ARM, + "thumb": capstone.CS_MODE_THUMB, + "mips32": capstone.CS_MODE_MIPS32, + "mips64": capstone.CS_MODE_MIPS64, + "mips32r6": capstone.CS_MODE_MIPS32R6, + "16": capstone.CS_MODE_16, + "32": capstone.CS_MODE_32, + "64": capstone.CS_MODE_64, + "le": capstone.CS_MODE_LITTLE_ENDIAN, + "be": capstone.CS_MODE_BIG_ENDIAN, + "micro": capstone.CS_MODE_MICRO, + "mclass": capstone.CS_MODE_MCLASS, + "v8": capstone.CS_MODE_V8, + "v9": capstone.CS_MODE_V9, +} + +# MODULE INTERFACE #################################################### + +@hook.command("disas", + help="Display assembly code", + help_usage={"CODE": "Display assembly code corresponding to the given CODE"}, + keywords={ + "arch=ARCH": "Specify the architecture of the code to disassemble (default: x86, choose between: %s)" % ', '.join(ARCHITECTURES.keys()), + "modes=MODE[,MODE]": "Specify hardware mode of the code to disassemble (default: 32, between: %s)" % ', '.join(MODES.keys()), + }) +def cmd_disas(msg): + if not len(msg.args): + raise IMException("please give me some code") + + # Determine the architecture + if "architecture" in msg.kwargs: + if msg.kwargs["architecture"] not in ARCHITECTURES: + raise IMException("unknown architectures '%s'" % msg.kwargs["architecture"]) + architecture = ARCHITECTURES[msg.kwargs["architecture"]] + else: + architecture = capstone.CS_ARCH_X86 + + # Determine hardware modes + if "modes" in msg.kwargs: + modes = 0 + for mode in msg.kwargs["modes"].split(','): + if mode not in MODES: + raise IMException("unknown mode '%s'" % mode) + modes += MODES[mode] + else: + modes = capstone.CS_MODE_32 + + # Get the code + code = bytearray.fromhex(''.join([a.replace("0x", "") for a in msg.args])) + + # Setup capstone + md = capstone.Cs(architecture, modes) + + res = Response(channel=msg.channel, nomore="No more instruction") + + for isn in md.disasm(code, 0x1000): + res.append_message("%s %s" %(isn.mnemonic, isn.op_str), title="0x%x" % isn.address) + + return res From fecbc54d8cde077660ed95b7c190ac07285d332e Mon Sep 17 00:00:00 2001 From: nemunaire Date: Sun, 16 Jul 2017 21:32:43 +0200 Subject: [PATCH 015/127] WIP --- smsgate.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 smsgate.py diff --git a/smsgate.py b/smsgate.py new file mode 100644 index 0000000..d4b76ce --- /dev/null +++ b/smsgate.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# Nemubot is a smart and modulable IM bot. +# Copyright (C) 2012-2017 Mercier Pierre-Olivier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys + +def fs_event(path): + # open file + + # parse file to extract: + # - correspondant number + # - correspondant name + # - message + + # build a command with given values + +def main(): + import argparse + parser = argparse.ArgumentParser() + + parser.add_argument("-S", "--socketfile", default="./nemubot.sock", + help="path where open the socket for internal communication") + + args = parser.parse_args() + + # Resolve relatives paths + args.socketfile = os.path.abspath(os.path.expanduser(args.socketfile)) + + import socket + import select + mypoll = select.poll() + + mypoll.register(sock.fileno(), select.POLLIN | select.POLLPRI) + + # TODO: inotify on FS + + +if __name__ == "__main__": + main() From aad777058ee8600b12f1e835e4b8db30dc2f7d52 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 21 Jul 2017 07:26:00 +0200 Subject: [PATCH 016/127] cve: update and clean module, following NIST website changes --- modules/cve.py | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/modules/cve.py b/modules/cve.py index 23a0302..c470e29 100644 --- a/modules/cve.py +++ b/modules/cve.py @@ -10,7 +10,7 @@ from nemubot.tools.web import getURLContent, striphtml from more import Response -BASEURL_NIST = 'https://web.nvd.nist.gov/view/vuln/detail?vulnId=' +BASEURL_NIST = 'https://nvd.nist.gov/vuln/detail/' # MODULE CORE ######################################################### @@ -19,15 +19,40 @@ def get_cve(cve_id): search_url = BASEURL_NIST + quote(cve_id.upper()) soup = BeautifulSoup(getURLContent(search_url)) - vuln = soup.body.find(class_="vuln-detail") - cvss = vuln.findAll('div')[4] - return [ - "Base score: " + cvss.findAll('div')[0].findAll('a')[0].text.strip(), - vuln.findAll('p')[0].text, # description - striphtml(vuln.findAll('div')[0].text).strip(), # publication date - striphtml(vuln.findAll('div')[1].text).strip(), # last revised - ] + return { + "description": soup.body.find(attrs={"data-testid":"vuln-description"}).text.strip(), + "published": soup.body.find(attrs={"data-testid":"vuln-published-on"}).text.strip(), + "last_modified": soup.body.find(attrs={"data-testid":"vuln-last-modified-on"}).text.strip(), + "source": soup.body.find(attrs={"data-testid":"vuln-source"}).text.strip(), + + "base_score": float(soup.body.find(attrs={"data-testid":"vuln-cvssv3-base-score-link"}).text.strip()), + "severity": soup.body.find(attrs={"data-testid":"vuln-cvssv3-base-score-severity"}).text.strip(), + "impact_score": float(soup.body.find(attrs={"data-testid":"vuln-cvssv3-impact-score"}).text.strip()), + "exploitability_score": float(soup.body.find(attrs={"data-testid":"vuln-cvssv3-exploitability-score"}).text.strip()), + + "av": soup.body.find(attrs={"data-testid":"vuln-cvssv3-av"}).text.strip(), + "ac": soup.body.find(attrs={"data-testid":"vuln-cvssv3-ac"}).text.strip(), + "pr": soup.body.find(attrs={"data-testid":"vuln-cvssv3-pr"}).text.strip(), + "ui": soup.body.find(attrs={"data-testid":"vuln-cvssv3-ui"}).text.strip(), + "s": soup.body.find(attrs={"data-testid":"vuln-cvssv3-s"}).text.strip(), + "c": soup.body.find(attrs={"data-testid":"vuln-cvssv3-c"}).text.strip(), + "i": soup.body.find(attrs={"data-testid":"vuln-cvssv3-i"}).text.strip(), + "a": soup.body.find(attrs={"data-testid":"vuln-cvssv3-a"}).text.strip(), + } + + +def display_metrics(av, ac, pr, ui, s, c, i, a, **kwargs): + ret = [] + if av != "None": ret.append("Attack Vector: \x02%s\x0F" % av) + if ac != "None": ret.append("Attack Complexity: \x02%s\x0F" % ac) + if pr != "None": ret.append("Privileges Required: \x02%s\x0F" % pr) + if ui != "None": ret.append("User Interaction: \x02%s\x0F" % ui) + if s != "Unchanged": ret.append("Scope: \x02%s\x0F" % s) + if c != "None": ret.append("Confidentiality: \x02%s\x0F" % c) + if i != "None": ret.append("Integrity: \x02%s\x0F" % i) + if a != "None": ret.append("Availability: \x02%s\x0F" % a) + return ', '.join(ret) # MODULE INTERFACE #################################################### @@ -42,6 +67,8 @@ def get_cve_desc(msg): if cve_id[:3].lower() != 'cve': cve_id = 'cve-' + cve_id - res.append_message(get_cve(cve_id)) + cve = get_cve(cve_id) + metrics = display_metrics(**cve) + res.append_message("{cveid}: Base score: \x02{base_score} {severity}\x0F (impact: \x02{impact_score}\x0F, exploitability: \x02{exploitability_score}\x0F; {metrics}), from \x02{source}\x0F, last modified on \x02{last_modified}\x0F. {description}".format(cveid=cve_id, metrics=metrics, **cve)) return res From 3267c3e2e15df50bceb14390bd198cb7fc4f9fcd Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 22 Jul 2017 10:49:38 +0200 Subject: [PATCH 017/127] tools/web: display socket timeout --- nemubot/tools/web.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index 0852664..9ced693 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -15,6 +15,7 @@ # along with this program. If not, see . from urllib.parse import urljoin, urlparse, urlsplit, urlunsplit +import socket from nemubot.exception import IMException @@ -123,6 +124,8 @@ def getURLContent(url, body=None, timeout=7, header=None): o.path, body, header) + except socket.timeout as e: + raise IMException(e) except OSError as e: raise IMException(e.strerror) From 171297b5810a863ed02754850afd2d8a76cd2c04 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 22 Jul 2017 10:53:08 +0200 Subject: [PATCH 018/127] tools/web: new option decode_error to decode non-200 page content (useful on REST API) --- nemubot/tools/web.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index 9ced693..0394aac 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -68,13 +68,14 @@ def getPassword(url): # Get real pages -def getURLContent(url, body=None, timeout=7, header=None): +def getURLContent(url, body=None, timeout=7, header=None, decode_error=False): """Return page content corresponding to URL or None if any error occurs Arguments: url -- the URL to get body -- Data to send as POST content timeout -- maximum number of seconds to wait before returning an exception + decode_error -- raise exception on non-200 pages or ignore it """ o = urlparse(_getNormalizedURL(url), "http") @@ -166,7 +167,10 @@ def getURLContent(url, body=None, timeout=7, header=None): urljoin(url, res.getheader("Location")), body=body, timeout=timeout, - header=header) + header=header, + decode_error=decode_error) + elif decode_error: + return data.decode(charset).strip() else: raise IMException("A HTTP error occurs: %d - %s" % (res.status, http.client.responses[res.status])) From f16dedb320b12634bbadc202a191727d90a5c30c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 26 Jul 2017 07:51:35 +0200 Subject: [PATCH 019/127] openroute: new module providing geocode and direction instructions Closing issue #46 --- modules/openroute.py | 158 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 modules/openroute.py diff --git a/modules/openroute.py b/modules/openroute.py new file mode 100644 index 0000000..440b05a --- /dev/null +++ b/modules/openroute.py @@ -0,0 +1,158 @@ +"""Lost? use our commands to find your way!""" + +# PYTHON STUFFS ####################################################### + +import re +import urllib.parse + +from nemubot.exception import IMException +from nemubot.hooks import hook +from nemubot.tools import web + +from more import Response + +# GLOBALS ############################################################# + +URL_DIRECTIONS_API = "https://api.openrouteservice.org/directions?api_key=%s&" +URL_GEOCODE_API = "https://api.openrouteservice.org/geocoding?api_key=%s&" + +waytype = [ + "unknown", + "state road", + "road", + "street", + "path", + "track", + "cycleway", + "footway", + "steps", + "ferry", + "construction", +] + + +# LOADING ############################################################# + +def load(context): + if not context.config or "apikey" not in context.config: + raise ImportError("You need an OpenRouteService API key in order to use this " + "module. Add it to the module configuration file:\n" + "\nRegister at https://developers.openrouteservice.org") + global URL_DIRECTIONS_API + URL_DIRECTIONS_API = URL_DIRECTIONS_API % context.config["apikey"] + global URL_GEOCODE_API + URL_GEOCODE_API = URL_GEOCODE_API % context.config["apikey"] + + +# MODULE CORE ######################################################### + +def approx_distance(lng): + if lng > 1111: + return "%f km" % (lng / 1000) + else: + return "%f m" % lng + + +def approx_duration(sec): + days = int(sec / 86400) + if days > 0: + return "%d days %f hours" % (days, (sec % 86400) / 3600) + hours = int((sec % 86400) / 3600) + if hours > 0: + return "%d hours %f minutes" % (hours, (sec % 3600) / 60) + minutes = (sec % 3600) / 60 + if minutes > 0: + return "%d minutes" % minutes + else: + return "%d seconds" % sec + + +def geocode(query, limit=7): + obj = web.getJSON(URL_GEOCODE_API + urllib.parse.urlencode({ + 'query': query, + 'limit': limit, + })) + + for f in obj["features"]: + yield f["geometry"]["coordinates"], f["properties"] + + +def firstgeocode(query): + for g in geocode(query, limit=1): + return g + + +def where(loc): + return "{name} {city} {state} {county} {country}".format(**loc) + + +def directions(coordinates, **kwargs): + kwargs['coordinates'] = '|'.join(coordinates) + + print(URL_DIRECTIONS_API + urllib.parse.urlencode(kwargs)) + return web.getJSON(URL_DIRECTIONS_API + urllib.parse.urlencode(kwargs), decode_error=True) + + +# MODULE INTERFACE #################################################### + +@hook.command("geocode", + help="Get GPS coordinates of a place", + help_usage={ + "PLACE": "Get GPS coordinates of PLACE" + }) +def cmd_geocode(msg): + res = Response(channel=msg.channel, nick=msg.frm, + nomore="No more geocode", count=" (%s more geocode)") + + for loc in geocode(' '.join(msg.args)): + res.append_message("%s is at %s,%s" % ( + where(loc[1]), + loc[0][1], loc[0][0], + )) + + return res + + +@hook.command("directions", + help="Get routing instructions", + help_usage={ + "POINT1 POINT2 ...": "Get routing instructions to go from POINT1 to the last POINTX via intermediates POINTX" + }, + keywords={ + "profile=PROF": "One of driving-car, driving-hgv, cycling-regular, cycling-road, cycling-safe, cycling-mountain, cycling-tour, cycling-electric, foot-walking, foot-hiking, wheelchair. Default: foot-walking", + "preference=PREF": "One of fastest, shortest, recommended. Default: recommended", + "lang=LANG": "default: en", + }) +def cmd_directions(msg): + drcts = directions(["{0},{1}".format(*firstgeocode(g)[0]) for g in msg.args], + profile=msg.kwargs["profile"] if "profile" in msg.kwargs else "foot-walking", + preference=msg.kwargs["preference"] if "preference" in msg.kwargs else "recommended", + units="m", + language=msg.kwargs["lang"] if "lang" in msg.kwargs else "en", + geometry=False, + instructions=True, + instruction_format="text") + if "error" in drcts and "message" in drcts["error"] and drcts["error"]["message"]: + raise IMException(drcts["error"]["message"]) + + if "routes" not in drcts or not drcts["routes"]: + raise IMException("No route available for this trip") + + myway = drcts["routes"][0] + myway["summary"]["strduration"] = approx_duration(myway["summary"]["duration"]) + myway["summary"]["strdistance"] = approx_distance(myway["summary"]["distance"]) + res = Response("Trip summary: {strdistance} in approximate {strduration}; elevation +{ascent} m -{descent} m".format(**myway["summary"]), channel=msg.channel, count=" (%d more steps)", nomore="You have arrived!") + + def formatSegments(segments): + for segment in segments: + for step in segment["steps"]: + step["strtype"] = waytype[step["type"]] + step["strduration"] = approx_duration(step["duration"]) + step["strdistance"] = approx_distance(step["distance"]) + yield "{instruction} for {strdistance} on {strtype} (approximate time: {strduration})".format(**step) + + if "segments" in myway: + res.append_message([m for m in formatSegments(myway["segments"])]) + + return res From 39056cf3584cc352a8dcbd472d30bac4e76d673c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 27 Jul 2017 20:44:26 +0200 Subject: [PATCH 020/127] tools/xmlparser: implement writer --- nemubot/tools/test_xmlparser.py | 36 +++++++++++++++++++++++--- nemubot/tools/xmlparser/__init__.py | 15 +++++++++++ nemubot/tools/xmlparser/basic.py | 20 ++++++++++++++ nemubot/tools/xmlparser/genericnode.py | 8 ++++++ 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/nemubot/tools/test_xmlparser.py b/nemubot/tools/test_xmlparser.py index d7f5a9a..0feda73 100644 --- a/nemubot/tools/test_xmlparser.py +++ b/nemubot/tools/test_xmlparser.py @@ -1,5 +1,6 @@ import unittest +import io import xml.parsers.expat from nemubot.tools.xmlparser import XMLParser @@ -12,6 +13,11 @@ class StringNode(): def characters(self, content): self.string += content + def saveElement(self, store, tag="string"): + store.startElement(tag, {}) + store.characters(self.string) + store.endElement(tag) + class TestNode(): def __init__(self, option=None): @@ -22,6 +28,15 @@ class TestNode(): self.mystr = child.string return True + def saveElement(self, store, tag="test"): + store.startElement(tag, {"option": self.option}) + + strNode = StringNode() + strNode.string = self.mystr + strNode.saveElement(store) + + store.endElement(tag) + class Test2Node(): def __init__(self, option=None): @@ -33,6 +48,15 @@ class Test2Node(): self.mystrs.append(attrs["value"]) return True + def saveElement(self, store, tag="test"): + store.startElement(tag, {"option": self.option} if self.option is not None else {}) + + for mystr in self.mystrs: + store.startElement("string", {"value": mystr}) + store.endElement("string") + + store.endElement(tag) + class TestXMLParser(unittest.TestCase): @@ -44,9 +68,11 @@ class TestXMLParser(unittest.TestCase): p.CharacterDataHandler = mod.characters p.EndElementHandler = mod.endElement - p.Parse("toto", 1) + inputstr = "toto" + p.Parse(inputstr, 1) self.assertEqual(mod.root.string, "toto") + self.assertEqual(mod.saveDocument(header=False).getvalue(), inputstr) def test_parser2(self): @@ -57,10 +83,12 @@ class TestXMLParser(unittest.TestCase): p.CharacterDataHandler = mod.characters p.EndElementHandler = mod.endElement - p.Parse("toto", 1) + inputstr = 'toto' + p.Parse(inputstr, 1) self.assertEqual(mod.root.option, "123") self.assertEqual(mod.root.mystr, "toto") + self.assertEqual(mod.saveDocument(header=False).getvalue(), inputstr) def test_parser3(self): @@ -71,12 +99,14 @@ class TestXMLParser(unittest.TestCase): p.CharacterDataHandler = mod.characters p.EndElementHandler = mod.endElement - p.Parse("", 1) + inputstr = '' + p.Parse(inputstr, 1) self.assertEqual(mod.root.option, None) self.assertEqual(len(mod.root.mystrs), 2) self.assertEqual(mod.root.mystrs[0], "toto") self.assertEqual(mod.root.mystrs[1], "toto2") + self.assertEqual(mod.saveDocument(header=False, short_empty_elements=True).getvalue(), inputstr) if __name__ == '__main__': diff --git a/nemubot/tools/xmlparser/__init__.py b/nemubot/tools/xmlparser/__init__.py index abc5bb9..c8d393a 100644 --- a/nemubot/tools/xmlparser/__init__.py +++ b/nemubot/tools/xmlparser/__init__.py @@ -134,6 +134,21 @@ class XMLParser: return raise TypeError(name + " tag not expected in " + self.display_stack()) + def saveDocument(self, f=None, header=True, short_empty_elements=False): + if f is None: + import io + f = io.StringIO() + + import xml.sax.saxutils + gen = xml.sax.saxutils.XMLGenerator(f, "utf-8", short_empty_elements=short_empty_elements) + if header: + gen.startDocument() + self.root.saveElement(gen) + if header: + gen.endDocument() + + return f + def parse_file(filename): p = xml.parsers.expat.ParserCreate() diff --git a/nemubot/tools/xmlparser/basic.py b/nemubot/tools/xmlparser/basic.py index 8456629..86eac3c 100644 --- a/nemubot/tools/xmlparser/basic.py +++ b/nemubot/tools/xmlparser/basic.py @@ -44,6 +44,13 @@ class ListNode: return self.items.__repr__() + def saveElement(self, store, tag="list"): + store.startElement(tag, {}) + for i in self.items: + i.saveElement(store) + store.endElement(tag) + + class DictNode: """XML node representing a Python dictionnnary @@ -106,3 +113,16 @@ class DictNode: def __repr__(self): return self.items.__repr__() + + + def saveElement(self, store, tag="dict"): + store.startElement(tag, {}) + for k, v in self.items.items(): + store.startElement("item", {"key": k}) + if isinstance(v, str): + store.characters(v) + else: + for i in v: + i.saveElement(store) + store.endElement("item") + store.endElement(tag) diff --git a/nemubot/tools/xmlparser/genericnode.py b/nemubot/tools/xmlparser/genericnode.py index 9c29a23..425934c 100644 --- a/nemubot/tools/xmlparser/genericnode.py +++ b/nemubot/tools/xmlparser/genericnode.py @@ -53,6 +53,14 @@ class ParsingNode: return item in self.attrs + def saveElement(self, store, tag=None): + store.startElement(tag if tag is not None else self.tag, self.attrs) + for child in self.children: + child.saveElement(store) + store.characters(self.content) + store.endElement(tag if tag is not None else self.tag) + + class GenericNode(ParsingNode): """Consider all subtags as dictionnary From e3b6c3b85ea552051b0ba839145dcfd67e8bafde Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 30 Jul 2017 23:22:14 +0200 Subject: [PATCH 021/127] Set urlreducer to use https --- modules/urlreducer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/urlreducer.py b/modules/urlreducer.py index bd5dc9a..36fcb3c 100644 --- a/modules/urlreducer.py +++ b/modules/urlreducer.py @@ -21,7 +21,7 @@ def default_reducer(url, data): def ycc_reducer(url, data): - return "http://ycc.fr/%s" % default_reducer(url, data) + return "https://ycc.fr/%s" % default_reducer(url, data) def lstu_reducer(url, data): json_data = json.loads(web.getURLContent(url, "lsturl=" + quote(data), @@ -36,8 +36,8 @@ def lstu_reducer(url, data): # MODULE VARIABLES #################################################### PROVIDERS = { - "tinyurl": (default_reducer, "http://tinyurl.com/api-create.php?url="), - "ycc": (ycc_reducer, "http://ycc.fr/redirection/create/"), + "tinyurl": (default_reducer, "https://tinyurl.com/api-create.php?url="), + "ycc": (ycc_reducer, "https://ycc.fr/redirection/create/"), "framalink": (lstu_reducer, "https://frama.link/a?format=json"), "huitre": (lstu_reducer, "https://huit.re/a?format=json"), "lstu": (lstu_reducer, "https://lstu.fr/a?format=json"), From ce012b70170412c710ae00615e5a7d93d9812268 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 28 Jul 2017 06:55:17 +0200 Subject: [PATCH 022/127] datastore/xml: handle entire file save and be closer with new nemubot XML API --- nemubot/datastore/xml.py | 13 ++++++++++++- nemubot/tools/xmlparser/node.py | 24 ++---------------------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/nemubot/datastore/xml.py b/nemubot/datastore/xml.py index 46dca70..025c0c5 100644 --- a/nemubot/datastore/xml.py +++ b/nemubot/datastore/xml.py @@ -143,4 +143,15 @@ class XML(Abstract): if self.rotate: self._rotate(path) - return data.save(path) + import tempfile + _, tmpath = tempfile.mkstemp() + with open(tmpath, "w") as f: + import xml.sax.saxutils + gen = xml.sax.saxutils.XMLGenerator(f, "utf-8") + gen.startDocument() + data.saveElement(gen) + gen.endDocument() + + # Atomic save + import shutil + shutil.move(tmpath, path) diff --git a/nemubot/tools/xmlparser/node.py b/nemubot/tools/xmlparser/node.py index 965a475..7df255e 100644 --- a/nemubot/tools/xmlparser/node.py +++ b/nemubot/tools/xmlparser/node.py @@ -196,7 +196,7 @@ class ModuleState: if self.index_fieldname is not None: self.setIndex(self.index_fieldname, self.index_tagname) - def save_node(self, gen): + def saveElement(self, gen): """Serialize this node as a XML node""" from datetime import datetime attribs = {} @@ -215,29 +215,9 @@ class ModuleState: gen.startElement(self.name, attrs) for child in self.childs: - child.save_node(gen) + child.saveElement(gen) gen.endElement(self.name) except: logger.exception("Error occured when saving the following " "XML node: %s with %s", self.name, attrs) - - def save(self, filename): - """Save the current node as root node in a XML file - - Argument: - filename -- location of the file to create/erase - """ - - import tempfile - _, tmpath = tempfile.mkstemp() - with open(tmpath, "w") as f: - import xml.sax.saxutils - gen = xml.sax.saxutils.XMLGenerator(f, "utf-8") - gen.startDocument() - self.save_node(gen) - gen.endDocument() - - # Atomic save - import shutil - shutil.move(tmpath, filename) From f81349bbfd48f8fd37b18b80274f8bac59bf9418 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 29 Jul 2017 15:22:57 +0200 Subject: [PATCH 023/127] Store module into weakref --- nemubot/bot.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/nemubot/bot.py b/nemubot/bot.py index b0d3915..febe7d6 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -20,6 +20,7 @@ from multiprocessing import JoinableQueue import threading import select import sys +import weakref from nemubot import __version__ from nemubot.consumer import Consumer, EventConsumer, MessageConsumer @@ -99,15 +100,15 @@ class Bot(threading.Thread): from more import Response res = Response(channel=msg.to_response) if len(msg.args) >= 1: - if msg.args[0] in self.modules: - if hasattr(self.modules[msg.args[0]], "help_full"): - hlp = self.modules[msg.args[0]].help_full() + if msg.args[0] in self.modules and self.modules[msg.args[0]]() is not None: + if hasattr(self.modules[msg.args[0]](), "help_full"): + hlp = self.modules[msg.args[0]]().help_full() if isinstance(hlp, Response): return hlp else: res.append_message(hlp) else: - res.append_message([str(h) for s,h in self.modules[msg.args[0]].__nemubot_context__.hooks], title="Available commands for module " + msg.args[0]) + res.append_message([str(h) for s,h in self.modules[msg.args[0]]().__nemubot_context__.hooks], title="Available commands for module " + msg.args[0]) elif msg.args[0][0] == "!": from nemubot.message.command import Command for h in self.treater._in_hooks(Command(msg.args[0][1:])): @@ -137,7 +138,7 @@ class Bot(threading.Thread): res.append_message(title="Pour plus de détails sur un module, " "envoyez \"!help nomdumodule\". Voici la liste" " de tous les modules disponibles localement", - message=["\x03\x02%s\x03\x02 (%s)" % (im, self.modules[im].__doc__) for im in self.modules if self.modules[im].__doc__]) + message=["\x03\x02%s\x03\x02 (%s)" % (im, self.modules[im]().__doc__) for im in self.modules if self.modules[im]() is not None and self.modules[im]().__doc__]) return res self.treater.hm.add_hook(nemubot.hooks.Command(_help_msg, "help"), "in", "Command") @@ -518,18 +519,20 @@ class Bot(threading.Thread): raise # Save a reference to the module - self.modules[module_name] = module + self.modules[module_name] = weakref.ref(module) + logger.info("Module '%s' successfully loaded.", module_name) def unload_module(self, name): """Unload a module""" - if name in self.modules: - self.modules[name].print("Unloading module %s" % name) + if name in self.modules and self.modules[name]() is not None: + module = self.modules[name]() + module.print("Unloading module %s" % name) # Call the user defined unload method - if hasattr(self.modules[name], "unload"): - self.modules[name].unload(self) - self.modules[name].__nemubot_context__.unload() + if hasattr(module, "unload"): + module.unload(self) + module.__nemubot_context__.unload() # Remove from the nemubot dict del self.modules[name] From b517cac4cfde69e81f8c7228089c56a8c3505348 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 29 Jul 2017 15:25:44 +0200 Subject: [PATCH 024/127] Fix module unloading --- nemubot/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemubot/bot.py b/nemubot/bot.py index febe7d6..aa1cb3e 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -569,7 +569,7 @@ class Bot(threading.Thread): self.event_timer.cancel() logger.info("Save and unload all modules...") - for mod in self.modules.items(): + for mod in [m for m in self.modules.keys()]: self.unload_module(mod) logger.info("Close all servers connection...") From 29817ba1c1fd5f8b5618877e0023a834f145e65b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 8 Aug 2017 23:24:37 +0200 Subject: [PATCH 025/127] pkgs: new module to display quick information about common softwares --- modules/pkgs.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 modules/pkgs.py diff --git a/modules/pkgs.py b/modules/pkgs.py new file mode 100644 index 0000000..5a7b0a9 --- /dev/null +++ b/modules/pkgs.py @@ -0,0 +1,68 @@ +"""Get information about common software""" + +# PYTHON STUFFS ####################################################### + +import portage + +from nemubot import context +from nemubot.exception import IMException +from nemubot.hooks import hook + +from more import Response + +DB = None + +# MODULE CORE ######################################################### + +def get_db(): + global DB + if DB is None: + DB = portage.db[portage.root]["porttree"].dbapi + return DB + + +def package_info(pkgname): + pv = get_db().xmatch("match-all", pkgname) + if not pv: + raise IMException("No package named '%s' found" % pkgname) + + bv = get_db().xmatch("bestmatch-visible", pkgname) + pvsplit = portage.catpkgsplit(bv if bv else pv[-1]) + info = get_db().aux_get(bv if bv else pv[-1], ["DESCRIPTION", "HOMEPAGE", "LICENSE", "IUSE", "KEYWORDS"]) + + return { + "pkgname": '/'.join(pvsplit[:2]), + "category": pvsplit[0], + "shortname": pvsplit[1], + "lastvers": '-'.join(pvsplit[2:]) if pvsplit[3] != "r0" else pvsplit[2], + "othersvers": ['-'.join(portage.catpkgsplit(p)[2:]) for p in pv if p != bv], + "description": info[0], + "homepage": info[1], + "license": info[2], + "uses": info[3], + "keywords": info[4], + } + + +# MODULE INTERFACE #################################################### + +@hook.command("eix", + help="Get information about a package", + help_usage={ + "NAME": "Get information about a software NAME" + }) +def cmd_eix(msg): + if not len(msg.args): + raise IMException("please give me a package to search") + + def srch(term): + try: + yield package_info(term) + except portage.exception.AmbiguousPackageName as e: + for i in e.args[0]: + yield package_info(i) + + res = Response(channel=msg.channel, count=" (%d more packages)", nomore="No more package '%s'" % msg.args[0]) + for pi in srch(msg.args[0]): + res.append_message("\x03\x02{pkgname}:\x03\x02 {description} - {homepage} - {license} - last revisions: \x03\x02{lastvers}\x03\x02{ov}".format(ov=(", " + ', '.join(pi["othersvers"])) if pi["othersvers"] else "", **pi)) + return res From 281d81acc4c1b167dbf89b75408a3a427ea67576 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 9 Aug 2017 22:53:35 +0200 Subject: [PATCH 026/127] suivi: fix error handling of fedex parcel --- modules/suivi.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/suivi.py b/modules/suivi.py index a6f6ab4..24f5bf9 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -156,7 +156,9 @@ def get_fedex_info(fedex_id, lang="en_US"): if ("TrackPackagesResponse" in fedex_data and "packageList" in fedex_data["TrackPackagesResponse"] and - len(fedex_data["TrackPackagesResponse"]["packageList"]) + len(fedex_data["TrackPackagesResponse"]["packageList"]) and + not fedex_data["TrackPackagesResponse"]["errorList"][0]["code"] and + not fedex_data["TrackPackagesResponse"]["packageList"][0]["errorList"][0]["code"] ): return fedex_data["TrackPackagesResponse"]["packageList"][0] From b8f4560780a7570a71d4e17cf0b6f688eb7a2ead Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 10 Aug 2017 00:55:13 +0200 Subject: [PATCH 027/127] suivi: support DHL --- modules/suivi.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/modules/suivi.py b/modules/suivi.py index 24f5bf9..75a065b 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -163,6 +163,15 @@ def get_fedex_info(fedex_id, lang="en_US"): return fedex_data["TrackPackagesResponse"]["packageList"][0] +def get_dhl_info(dhl_id, lang="en"): + dhl_parcelurl = "http://www.dhl.com/shipmentTracking?" + urllib.parse.urlencode({'AWB': dhl_id}) + + dhl_data = getJSON(dhl_parcelurl) + + if "results" in dhl_data and dhl_data["results"]: + return dhl_data["results"][0] + + # TRACKING HANDLERS ################################################### def handle_tnt(tracknum): @@ -231,6 +240,12 @@ def handle_fedex(tracknum): return ("{trackingCarrierDesc}: \x02{statusWithDetails}\x0F: in \x02{statusLocationCity}, {statusLocationCntryCD}\x0F, estimated delivery: {displayEstDeliveryDateTime}.".format(**info)) +def handle_dhl(tracknum): + info = get_dhl_info(tracknum) + if info: + return "DHL {label} {id}: \x02{description}\x0F".format(**info) + + TRACKING_HANDLERS = { 'laposte': handle_laposte, 'postnl': handle_postnl, @@ -239,6 +254,7 @@ TRACKING_HANDLERS = { 'coliprive': handle_coliprive, 'tnt': handle_tnt, 'fedex': handle_fedex, + 'dhl': handle_dhl, } From 09462d0d90b6cd9d2241e6dd18cc77e5424d6de7 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 10 Aug 2017 06:48:48 +0200 Subject: [PATCH 028/127] suivi: support USPS --- modules/suivi.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/modules/suivi.py b/modules/suivi.py index 75a065b..6ad13e9 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -126,6 +126,24 @@ def get_postnl_info(postnl_id): return (post_status.lower(), post_destination, post_date) +def get_usps_info(usps_id): + usps_parcelurl = "https://tools.usps.com/go/TrackConfirmAction_input?" + urllib.parse.urlencode({'qtc_tLabels1': usps_id}) + + usps_data = getURLContent(usps_parcelurl) + soup = BeautifulSoup(usps_data) + if (soup.find(class_="tracking_history") + and soup.find(class_="tracking_history").find(class_="row_notification") + and soup.find(class_="tracking_history").find(class_="row_top").find_all("td")): + notification = soup.find(class_="tracking_history").find(class_="row_notification").text.strip() + date = re.sub(r"\s+", " ", soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[0].text.strip()) + status = soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[1].text.strip() + last_location = soup.find(class_="tracking_history").find(class_="row_top").find_all("td")[2].text.strip() + + print(notification) + + return (notification, date, status, last_location) + + def get_fedex_info(fedex_id, lang="en_US"): data = urllib.parse.urlencode({ 'data': json.dumps({ @@ -206,6 +224,13 @@ def handle_postnl(tracknum): ")." % (tracknum, post_status, post_destination, post_date)) +def handle_usps(tracknum): + info = get_usps_info(tracknum) + if info: + notif, last_date, last_status, last_location = info + return ("USPS \x02{tracknum}\x0F is {last_status} in \x02{last_location}\x0F as of {last_date}: {notif}".format(tracknum=tracknum, notif=notif, last_date=last_date, last_status=last_status.lower(), last_location=last_location)) + + def handle_colissimo(tracknum): info = get_colissimo_info(tracknum) if info: @@ -255,6 +280,7 @@ TRACKING_HANDLERS = { 'tnt': handle_tnt, 'fedex': handle_fedex, 'dhl': handle_dhl, + 'usps': handle_usps, } From 39b7b1ae2fc88db91eb8ddd890082fa77c0fd0fc Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 4 Aug 2017 01:22:24 +0200 Subject: [PATCH 029/127] freetarifs: new module --- modules/freetarifs.py | 64 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 modules/freetarifs.py diff --git a/modules/freetarifs.py b/modules/freetarifs.py new file mode 100644 index 0000000..b96a30f --- /dev/null +++ b/modules/freetarifs.py @@ -0,0 +1,64 @@ +"""Inform about Free Mobile tarifs""" + +# PYTHON STUFFS ####################################################### + +import urllib.parse +from bs4 import BeautifulSoup + +from nemubot.exception import IMException +from nemubot.hooks import hook +from nemubot.tools import web + +from more import Response + + +# MODULE CORE ######################################################### + +ACT = { + "ff_toFixe": "Appel vers les fixes", + "ff_toMobile": "Appel vers les mobiles", + "ff_smsSendedToCountry": "SMS vers le pays", + "ff_mmsSendedToCountry": "MMS vers le pays", + "fc_callToFrance": "Appel vers la France", + "fc_smsToFrance": "SMS vers la france", + "fc_mmsSended": "MMS vers la france", + "fc_callToSameCountry": "Réception des appels", + "fc_callReceived": "Appel dans le pays", + "fc_smsReceived": "SMS (Réception)", + "fc_mmsReceived": "MMS (Réception)", + "fc_moDataFromCountry": "Data", +} + +def get_land_tarif(country, forfait="pkgFREE"): + url = "http://mobile.international.free.fr/?" + urllib.parse.urlencode({'pays': country}) + page = web.getURLContent(url) + soup = BeautifulSoup(page) + + fact = soup.find(class_=forfait) + + if fact is None: + raise IMException("Country or forfait not found.") + + res = {} + for s in ACT.keys(): + try: + res[s] = fact.find(attrs={"data-bind": "text: " + s}).text + " " + fact.find(attrs={"data-bind": "html: " + s + "Unit"}).text + except AttributeError: + res[s] = "inclus" + + return res + +@hook.command("freetarifs", + help="Show Free Mobile tarifs for given contries", + help_usage={"COUNTRY": "Show Free Mobile tarifs for given CONTRY"}, + keywords={ + "forfait=FORFAIT": "Related forfait between Free (default) and 2euro" + }) +def get_freetarif(msg): + res = Response(channel=msg.channel) + + for country in msg.args: + t = get_land_tarif(country.lower().capitalize(), "pkg" + (msg.kwargs["forfait"] if "forfait" in msg.kwargs else "FREE").upper()) + res.append_message(["\x02%s\x0F : %s" % (ACT[k], t[k]) for k in sorted(ACT.keys(), reverse=True)], title=country) + + return res From 128afb5914059c9e88df878576738d768529601b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 6 Aug 2017 12:27:19 +0200 Subject: [PATCH 030/127] disas: new module, aim to disassemble binary code. Closing #67 --- modules/disas.py | 89 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 modules/disas.py diff --git a/modules/disas.py b/modules/disas.py new file mode 100644 index 0000000..7c17907 --- /dev/null +++ b/modules/disas.py @@ -0,0 +1,89 @@ +"""The Ultimate Disassembler Module""" + +# PYTHON STUFFS ####################################################### + +import capstone + +from nemubot.exception import IMException +from nemubot.hooks import hook + +from more import Response + + +# MODULE CORE ######################################################### + +ARCHITECTURES = { + "arm": capstone.CS_ARCH_ARM, + "arm64": capstone.CS_ARCH_ARM64, + "mips": capstone.CS_ARCH_MIPS, + "ppc": capstone.CS_ARCH_PPC, + "sparc": capstone.CS_ARCH_SPARC, + "sysz": capstone.CS_ARCH_SYSZ, + "x86": capstone.CS_ARCH_X86, + "xcore": capstone.CS_ARCH_XCORE, +} + +MODES = { + "arm": capstone.CS_MODE_ARM, + "thumb": capstone.CS_MODE_THUMB, + "mips32": capstone.CS_MODE_MIPS32, + "mips64": capstone.CS_MODE_MIPS64, + "mips32r6": capstone.CS_MODE_MIPS32R6, + "16": capstone.CS_MODE_16, + "32": capstone.CS_MODE_32, + "64": capstone.CS_MODE_64, + "le": capstone.CS_MODE_LITTLE_ENDIAN, + "be": capstone.CS_MODE_BIG_ENDIAN, + "micro": capstone.CS_MODE_MICRO, + "mclass": capstone.CS_MODE_MCLASS, + "v8": capstone.CS_MODE_V8, + "v9": capstone.CS_MODE_V9, +} + +# MODULE INTERFACE #################################################### + +@hook.command("disas", + help="Display assembly code", + help_usage={"CODE": "Display assembly code corresponding to the given CODE"}, + keywords={ + "arch=ARCH": "Specify the architecture of the code to disassemble (default: x86, choose between: %s)" % ', '.join(ARCHITECTURES.keys()), + "modes=MODE[,MODE]": "Specify hardware mode of the code to disassemble (default: 32, between: %s)" % ', '.join(MODES.keys()), + }) +def cmd_disas(msg): + if not len(msg.args): + raise IMException("please give me some code") + + # Determine the architecture + if "arch" in msg.kwargs: + if msg.kwargs["arch"] not in ARCHITECTURES: + raise IMException("unknown architectures '%s'" % msg.kwargs["arch"]) + architecture = ARCHITECTURES[msg.kwargs["arch"]] + else: + architecture = capstone.CS_ARCH_X86 + + # Determine hardware modes + modes = 0 + if "modes" in msg.kwargs: + for mode in msg.kwargs["modes"].split(','): + if mode not in MODES: + raise IMException("unknown mode '%s'" % mode) + modes += MODES[mode] + elif architecture == capstone.CS_ARCH_X86 or architecture == capstone.CS_ARCH_PPC: + modes = capstone.CS_MODE_32 + elif architecture == capstone.CS_ARCH_ARM or architecture == capstone.CS_ARCH_ARM64: + modes = capstone.CS_MODE_ARM + elif architecture == capstone.CS_ARCH_MIPS: + modes = capstone.CS_MODE_MIPS32 + + # Get the code + code = bytearray.fromhex(''.join([a.replace("0x", "") for a in msg.args])) + + # Setup capstone + md = capstone.Cs(architecture, modes) + + res = Response(channel=msg.channel, nomore="No more instruction") + + for isn in md.disasm(code, 0x1000): + res.append_message("%s %s" %(isn.mnemonic, isn.op_str), title="0x%x" % isn.address) + + return res From 0a576410c72951fabc1b0284d98078bbe19acf4f Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 3 Aug 2017 21:28:56 +0200 Subject: [PATCH 031/127] cve: improve read of partial and inexistant CVE --- modules/cve.py | 66 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/modules/cve.py b/modules/cve.py index c470e29..6cdb339 100644 --- a/modules/cve.py +++ b/modules/cve.py @@ -5,6 +5,7 @@ from bs4 import BeautifulSoup from urllib.parse import quote +from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools.web import getURLContent, striphtml @@ -15,31 +16,44 @@ BASEURL_NIST = 'https://nvd.nist.gov/vuln/detail/' # MODULE CORE ######################################################### +VULN_DATAS = { + "alert-title": "vuln-warning-status-name", + "alert-content": "vuln-warning-banner-content", + + "description": "vuln-description", + "published": "vuln-published-on", + "last_modified": "vuln-last-modified-on", + "source": "vuln-source", + + "base_score": "vuln-cvssv3-base-score-link", + "severity": "vuln-cvssv3-base-score-severity", + "impact_score": "vuln-cvssv3-impact-score", + "exploitability_score": "vuln-cvssv3-exploitability-score", + + "av": "vuln-cvssv3-av", + "ac": "vuln-cvssv3-ac", + "pr": "vuln-cvssv3-pr", + "ui": "vuln-cvssv3-ui", + "s": "vuln-cvssv3-s", + "c": "vuln-cvssv3-c", + "i": "vuln-cvssv3-i", + "a": "vuln-cvssv3-a", +} + + def get_cve(cve_id): search_url = BASEURL_NIST + quote(cve_id.upper()) soup = BeautifulSoup(getURLContent(search_url)) - return { - "description": soup.body.find(attrs={"data-testid":"vuln-description"}).text.strip(), - "published": soup.body.find(attrs={"data-testid":"vuln-published-on"}).text.strip(), - "last_modified": soup.body.find(attrs={"data-testid":"vuln-last-modified-on"}).text.strip(), - "source": soup.body.find(attrs={"data-testid":"vuln-source"}).text.strip(), + vuln = {} - "base_score": float(soup.body.find(attrs={"data-testid":"vuln-cvssv3-base-score-link"}).text.strip()), - "severity": soup.body.find(attrs={"data-testid":"vuln-cvssv3-base-score-severity"}).text.strip(), - "impact_score": float(soup.body.find(attrs={"data-testid":"vuln-cvssv3-impact-score"}).text.strip()), - "exploitability_score": float(soup.body.find(attrs={"data-testid":"vuln-cvssv3-exploitability-score"}).text.strip()), + for vd in VULN_DATAS: + r = soup.body.find(attrs={"data-testid": VULN_DATAS[vd]}) + if r: + vuln[vd] = r.text.strip() - "av": soup.body.find(attrs={"data-testid":"vuln-cvssv3-av"}).text.strip(), - "ac": soup.body.find(attrs={"data-testid":"vuln-cvssv3-ac"}).text.strip(), - "pr": soup.body.find(attrs={"data-testid":"vuln-cvssv3-pr"}).text.strip(), - "ui": soup.body.find(attrs={"data-testid":"vuln-cvssv3-ui"}).text.strip(), - "s": soup.body.find(attrs={"data-testid":"vuln-cvssv3-s"}).text.strip(), - "c": soup.body.find(attrs={"data-testid":"vuln-cvssv3-c"}).text.strip(), - "i": soup.body.find(attrs={"data-testid":"vuln-cvssv3-i"}).text.strip(), - "a": soup.body.find(attrs={"data-testid":"vuln-cvssv3-a"}).text.strip(), - } + return vuln def display_metrics(av, ac, pr, ui, s, c, i, a, **kwargs): @@ -68,7 +82,19 @@ def get_cve_desc(msg): cve_id = 'cve-' + cve_id cve = get_cve(cve_id) - metrics = display_metrics(**cve) - res.append_message("{cveid}: Base score: \x02{base_score} {severity}\x0F (impact: \x02{impact_score}\x0F, exploitability: \x02{exploitability_score}\x0F; {metrics}), from \x02{source}\x0F, last modified on \x02{last_modified}\x0F. {description}".format(cveid=cve_id, metrics=metrics, **cve)) + if not cve: + raise IMException("CVE %s doesn't exists." % cve_id) + + if "alert-title" in cve or "alert-content" in cve: + alert = "\x02%s:\x0F %s " % (cve["alert-title"] if "alert-title" in cve else "", + cve["alert-content"] if "alert-content" in cve else "") + else: + alert = "" + + if "base_score" not in cve and "description" in cve: + res.append_message("{alert}From \x02{source}\x0F, last modified on \x02{last_modified}\x0F. {description}".format(alert=alert, **cve), title=cve_id) + else: + metrics = display_metrics(**cve) + res.append_message("{alert}Base score: \x02{base_score} {severity}\x0F (impact: \x02{impact_score}\x0F, exploitability: \x02{exploitability_score}\x0F; {metrics}), from \x02{source}\x0F, last modified on \x02{last_modified}\x0F. {description}".format(alert=alert, metrics=metrics, **cve), title=cve_id) return res From dcb44ca3f24b111de4718db7db5fd652ef9abdc1 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 2 Aug 2017 19:58:49 +0200 Subject: [PATCH 032/127] tools/web: new parameter to choose max content size to retrieve --- nemubot/tools/web.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index 0394aac..164f5da 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -68,7 +68,8 @@ def getPassword(url): # Get real pages -def getURLContent(url, body=None, timeout=7, header=None, decode_error=False): +def getURLContent(url, body=None, timeout=7, header=None, decode_error=False, + max_size=524288): """Return page content corresponding to URL or None if any error occurs Arguments: @@ -76,6 +77,7 @@ def getURLContent(url, body=None, timeout=7, header=None, decode_error=False): body -- Data to send as POST content timeout -- maximum number of seconds to wait before returning an exception decode_error -- raise exception on non-200 pages or ignore it + max_size -- maximal size allow for the content """ o = urlparse(_getNormalizedURL(url), "http") @@ -135,7 +137,7 @@ def getURLContent(url, body=None, timeout=7, header=None, decode_error=False): size = int(res.getheader("Content-Length", 524288)) cntype = res.getheader("Content-Type") - if size > 524288 or (cntype is not None and cntype[:4] != "text" and cntype[:4] != "appl"): + if size > max_size or (cntype is not None and cntype[:4] != "text" and cntype[:4] != "appl"): raise IMException("Content too large to be retrieved") data = res.read(size) @@ -168,7 +170,8 @@ def getURLContent(url, body=None, timeout=7, header=None, decode_error=False): body=body, timeout=timeout, header=header, - decode_error=decode_error) + decode_error=decode_error, + max_size=max_size) elif decode_error: return data.decode(charset).strip() else: From 6dda14218855c7cabd613001ec46ea1f77401c44 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 30 Jul 2017 11:49:21 +0200 Subject: [PATCH 033/127] shodan: introducing new module to search on shodan --- modules/shodan.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 modules/shodan.py diff --git a/modules/shodan.py b/modules/shodan.py new file mode 100644 index 0000000..4b2edae --- /dev/null +++ b/modules/shodan.py @@ -0,0 +1,104 @@ +"""Search engine for IoT""" + +# PYTHON STUFFS ####################################################### + +from datetime import datetime +import ipaddress +import urllib.parse + +from nemubot import context +from nemubot.exception import IMException +from nemubot.hooks import hook +from nemubot.tools import web + +from more import Response + + +# GLOBALS ############################################################# + +BASEURL = "https://api.shodan.io/shodan/" + + +# LOADING ############################################################# + +def load(context): + if not context.config or "apikey" not in context.config: + raise ImportError("You need a Shodan API key in order to use this " + "module. Add it to the module configuration file:\n" + "\nRegister at https://account.shodan.io/register") + + +# MODULE CORE ######################################################### + +def host_lookup(ip): + url = BASEURL + "host/" + urllib.parse.quote(ip) + "?" + urllib.parse.urlencode({'key': context.config["apikey"]}) + return web.getJSON(url) + + +def search_hosts(query): + url = BASEURL + "host/search?" + urllib.parse.urlencode({'query': query, 'key': context.config["apikey"]}) + return web.getJSON(url, max_size=4194304) + + +def print_ssl(ssl): + return ( + "SSL: " + + " ".join([v for v in ssl["versions"] if v[0] != "-"]) + + "; cipher used: " + ssl["cipher"]["name"] + + ("; certificate: " + ssl["cert"]["sig_alg"] + + " issued by: " + ssl["cert"]["issuer"]["CN"] + + " expires on: " + str(datetime.strptime(ssl["cert"]["expires"], "%Y%m%d%H%M%SZ")) if "cert" in ssl else "") + ) + +def print_service(svc): + ip = ipaddress.ip_address(svc["ip_str"]) + return ((svc["ip_str"] if ip.version == 4 else "[%s]" % svc["ip_str"]) + + ":{port}/{transport} ({module}):" + + (" {os}" if svc["os"] else "") + + (" {product}" if "product" in svc else "") + + (" {version}" if "version" in svc else "") + + (" {info}" if "info" in svc else "") + + (" Vulns: " + ", ".join(svc["opts"]["vulns"]) if "opts" in svc and "vulns" in svc["opts"] else "") + + (" " + print_ssl(svc["ssl"]) if "ssl" in svc else "") + + (" \x03\x1D" + svc["data"].replace("\r\n", "\n").split("\n")[0] + "\x03\x1D" if "data" in svc else "") + + (" " + svc["title"] if "title" in svc else "") + ).format(module=svc["_shodan"]["module"], **svc) + + +# MODULE INTERFACE #################################################### + +@hook.command("shodan", + help="Use shodan.io to get information on machines connected to Internet", + help_usage={ + "IP": "retrieve information about the given IP (can be v4 or v6)", + "TERM": "retrieve all hosts matching TERM somewhere in their exposed stuff" + }) +def shodan(msg): + if not msg.args: + raise IMException("indicate an IP or a term to search!") + + terms = " ".join(msg.args) + + try: + ip = ipaddress.ip_address(terms) + except ValueError: + ip = None + + if ip: + h = host_lookup(terms) + res = Response(channel=msg.channel, + title="%s" % ((h["ip_str"] if ip.version == 4 else "[%s]" % h["ip_str"]) + (" (" + ", ".join(h["hostnames"]) + ")") if h["hostnames"] else "")) + res.append_message("{isp} ({asn}) -> {city} ({country_code}), running {os}. Vulns: {vulns_str}. Open ports: {open_ports}. Last update: {last_update}".format( + open_ports=", ".join(map(lambda a: str(a), h["ports"])), vulns_str=", ".join(h["vulns"]) if "vulns" in h else None, **h).strip()) + for d in h["data"]: + res.append_message(print_service(d)) + + else: + q = search_hosts(terms) + res = Response(channel=msg.channel, + count=" (%%s/%s results)" % q["total"]) + for r in q["matches"]: + res.append_message(print_service(r)) + + return res From 3c7ed176c09b8ebb0072cc88a802b8da3856865b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 26 Aug 2017 10:38:52 +0200 Subject: [PATCH 034/127] dig: new module --- modules/dig.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 modules/dig.py diff --git a/modules/dig.py b/modules/dig.py new file mode 100644 index 0000000..3db5581 --- /dev/null +++ b/modules/dig.py @@ -0,0 +1,36 @@ +"""DNS resolver""" + +# PYTHON STUFFS ####################################################### + +import dns.rdtypes.ANY +import dns.rdtypes.IN +import dns.resolver + +from nemubot.exception import IMException +from nemubot.hooks import hook + +from more import Response + + +# MODULE INTERFACE #################################################### + +@hook.command("dig") +def dig(msg): + ltype = "A" + ldomain = None + for a in msg.args: + if a in dns.rdtypes.IN.__all__ or a in dns.rdtypes.ANY.__all__: + ltype = a + else: + ldomain = a + + if not ldomain: + raise IMException("indicate a domain to resolve") + + answers = dns.resolver.query(ldomain, ltype) + + res = Response(channel=msg.channel, title=ldomain, count=" (%s others records)") + for rdata in answers: + res.append_message(type(rdata).__name__ + " " + rdata.to_text()) + + return res From aa81aa4e96041ec45b5fb3bcd59ed110a5ebad1b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 26 Aug 2017 12:14:29 +0200 Subject: [PATCH 035/127] dig: better parse dig syntax @ and some + --- modules/dig.py | 72 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/modules/dig.py b/modules/dig.py index 3db5581..de7b2a3 100644 --- a/modules/dig.py +++ b/modules/dig.py @@ -2,8 +2,13 @@ # PYTHON STUFFS ####################################################### -import dns.rdtypes.ANY -import dns.rdtypes.IN +import ipaddress +import socket + +import dns.exception +import dns.name +import dns.rdataclass +import dns.rdatatype import dns.resolver from nemubot.exception import IMException @@ -14,23 +19,76 @@ from more import Response # MODULE INTERFACE #################################################### -@hook.command("dig") +@hook.command("dig", + help="Resolve domain name with a basic syntax similar to dig(1)") def dig(msg): + lclass = "IN" ltype = "A" + ledns = None + ltimeout = 6.0 ldomain = None + lnameservers = [] + lsearchlist = [] + loptions = [] for a in msg.args: - if a in dns.rdtypes.IN.__all__ or a in dns.rdtypes.ANY.__all__: + if a in dns.rdatatype._by_text: ltype = a + elif a in dns.rdataclass._by_text: + lclass = a + elif a[0] == "@": + try: + lnameservers.append(str(ipaddress.ip_address(a[1:]))) + except ValueError: + for r in socket.getaddrinfo(a[1:], 53, proto=socket.IPPROTO_UDP): + lnameservers.append(r[4][0]) + + elif a[0:8] == "+domain=": + lsearchlist.append(dns.name.from_unicode(a[8:])) + elif a[0:6] == "+edns=": + ledns = int(a[6:]) + elif a[0:6] == "+time=": + ltimeout = float(a[6:]) + elif a[0] == "+": + loptions.append(a[1:]) else: ldomain = a if not ldomain: raise IMException("indicate a domain to resolve") - answers = dns.resolver.query(ldomain, ltype) + resolv = dns.resolver.Resolver() + if ledns: + resolv.edns = ledns + resolv.lifetime = ltimeout + resolv.timeout = ltimeout + resolv.flags = ( + dns.flags.QR | dns.flags.RA | + dns.flags.AA if "aaonly" in loptions or "aaflag" in loptions else 0 | + dns.flags.AD if "adflag" in loptions else 0 | + dns.flags.CD if "cdflag" in loptions else 0 | + dns.flags.RD if "norecurse" not in loptions else 0 + ) + if lsearchlist: + resolv.search = lsearchlist + else: + resolv.search = [dns.name.from_text(".")] - res = Response(channel=msg.channel, title=ldomain, count=" (%s others records)") + if lnameservers: + resolv.nameservers = lnameservers + + try: + answers = resolv.query(ldomain, ltype, lclass, tcp="tcp" in loptions) + except dns.exception.DNSException as e: + raise IMException(str(e)) + + res = Response(channel=msg.channel, count=" (%s others entries)") for rdata in answers: - res.append_message(type(rdata).__name__ + " " + rdata.to_text()) + res.append_message("%s %s %s %s %s" % ( + answers.qname.to_text(), + answers.ttl if not "nottlid" in loptions else "", + dns.rdataclass.to_text(answers.rdclass) if not "nocl" in loptions else "", + dns.rdatatype.to_text(answers.rdtype), + rdata.to_text()) + ) return res From 89772ebce0893eb3b7ec20a1a7a960114a39686f Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 26 Aug 2017 16:56:05 +0200 Subject: [PATCH 036/127] whois: now able to use a CRI API dump --- modules/whois.py | 64 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/modules/whois.py b/modules/whois.py index 52344d1..fb6d250 100644 --- a/modules/whois.py +++ b/modules/whois.py @@ -1,5 +1,6 @@ # coding=utf-8 +import json import re from nemubot import context @@ -13,13 +14,26 @@ from more import Response from networking.page import headers PASSWD_FILE = None +# You can get one with: curl -b "sessionid=YOURSESSIONID" 'https://accounts.cri.epita.net/api/users/' > users.json +APIEXTRACT_FILE = None def load(context): global PASSWD_FILE if not context.config or "passwd" not in context.config: print("No passwd file given") + else: + PASSWD_FILE = context.config["passwd"] + print("passwd file loaded:", PASSWD_FILE) + + global APIEXTRACT_FILE + if not context.config or "apiextract" not in context.config: + print("No passwd file given") + else: + APIEXTRACT_FILE = context.config["apiextract"] + print("JSON users file loaded:", APIEXTRACT_FILE) + + if PASSWD_FILE is None and APIEXTRACT_FILE is None: return None - PASSWD_FILE = context.config["passwd"] if not context.data.hasNode("aliases"): context.data.addChild(ModuleState("aliases")) @@ -35,16 +49,26 @@ def load(context): class Login: - def __init__(self, line): - s = line.split(":") - self.login = s[0] - self.uid = s[2] - self.gid = s[3] - self.cn = s[4] - self.home = s[5] + def __init__(self, line=None, login=None, uidNumber=None, cn=None, promo=None, **kwargs): + if line is not None: + s = line.split(":") + self.login = s[0] + self.uid = s[2] + self.gid = s[3] + self.cn = s[4] + self.home = s[5] + else: + self.login = login + self.uid = uidNumber + self.promo = promo + self.cn = cn + self.gid = "epita" + promo def get_promo(self): - return self.home.split("/")[2].replace("_", " ") + if hasattr(self, "promo"): + return self.promo + if hasattr(self, "home"): + return self.home.split("/")[2].replace("_", " ") def get_photo(self): if self.login in context.data.getNode("pics").index: @@ -60,17 +84,25 @@ class Login: return None -def found_login(login, search=False): +def login_lookup(login, search=False): if login in context.data.getNode("aliases").index: login = context.data.getNode("aliases").index[login]["to"] + if APIEXTRACT_FILE: + with open(APIEXTRACT_FILE, encoding="utf-8") as f: + api = json.load(f) + for l in api: + if (not search and l["login"] == login) or (search and (("login" in l and l["login"].find(login) != -1) or ("cn" in l and l["cn"].find(login) != -1) or ("uid" in l and str(l["uid"]) == login))): + yield Login(**l) + login_ = login + (":" if not search else "") lsize = len(login_) - with open(PASSWD_FILE, encoding="iso-8859-15") as f: - for l in f.readlines(): - if l[:lsize] == login_: - yield Login(l.strip()) + if PASSWD_FILE: + with open(PASSWD_FILE, encoding="iso-8859-15") as f: + for l in f.readlines(): + if l[:lsize] == login_: + yield Login(l.strip()) def cmd_whois(msg): if len(msg.args) < 1: @@ -87,7 +119,7 @@ def cmd_whois(msg): res = Response(channel=msg.channel, count=" (%d more logins)", line_treat=format_response) for srch in msg.args: found = False - for l in found_login(srch, "lookup" in msg.kwargs): + for l in login_lookup(srch, "lookup" in msg.kwargs): found = True res.append_message((srch, l)) if not found: @@ -98,7 +130,7 @@ def cmd_whois(msg): def cmd_nicks(msg): if len(msg.args) < 1: raise IMException("Provide a login") - nick = found_login(msg.args[0]) + nick = login_lookup(msg.args[0]) if nick is None: nick = msg.args[0] else: From 1dae3c713a4095320f637bca89264926571b276c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 25 Aug 2017 23:53:10 +0200 Subject: [PATCH 037/127] tools/web: new option to remove callback from JSON files --- nemubot/tools/web.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index 164f5da..c3ba42a 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -193,7 +193,7 @@ def getXML(*args, **kwargs): return parseString(cnt) -def getJSON(*args, **kwargs): +def getJSON(*args, remove_callback=False, **kwargs): """Get content page and return JSON content Arguments: same as getURLContent @@ -204,6 +204,9 @@ def getJSON(*args, **kwargs): return None else: import json + if remove_callback: + import re + cnt = re.sub(r"^[^(]+\((.*)\)$", r"\1", cnt) return json.loads(cnt) From 694c54a6bc30841bb040b270d8c288a624b3707a Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 26 Aug 2017 00:14:14 +0200 Subject: [PATCH 038/127] imdb: switch to ugly IMDB HTML parsing --- modules/imdb.py | 87 ++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/modules/imdb.py b/modules/imdb.py index 2434a3c..bd1cadf 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -5,6 +5,8 @@ import re import urllib.parse +from bs4 import BeautifulSoup + from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web @@ -14,54 +16,46 @@ from more import Response # MODULE CORE ######################################################### -def get_movie(title=None, year=None, imdbid=None, fullplot=True, tomatoes=False): +def get_movie_by_id(imdbid): """Returns the information about the matching movie""" - # Built URL - url = "http://www.omdbapi.com/?" - if title is not None: - url += "t=%s&" % urllib.parse.quote(title) - if year is not None: - url += "y=%s&" % urllib.parse.quote(year) - if imdbid is not None: - url += "i=%s&" % urllib.parse.quote(imdbid) - if fullplot: - url += "plot=full&" - if tomatoes: - url += "tomatoes=true&" + url = "http://www.imdb.com/title/" + urllib.parse.quote(imdbid) + soup = BeautifulSoup(web.getURLContent(url)) - # Make the request - data = web.getJSON(url) + return { + "imdbID": imdbid, + "Title": soup.body.find(attrs={"itemprop": "name"}).next_element.strip(), + "Year": soup.body.find(id="titleYear").find("a").text.strip() if soup.body.find(id="titleYear") else ", ".join([y.text.strip() for y in soup.body.find(attrs={"class": "seasons-and-year-nav"}).find_all("div")[3].find_all("a")[:-1]]), + "Duration": soup.body.find_all(attrs={"itemprop": "duration"})[-1].text.strip(), + "imdbRating": soup.body.find(attrs={"itemprop": "ratingValue"}).text.strip(), + "imdbVotes": soup.body.find(attrs={"itemprop": "ratingCount"}).text.strip(), + "Plot": re.sub(r"\s+", " ", soup.body.find(id="titleStoryLine").find(attrs={"itemprop": "description"}).text).strip(), - # Return data - if "Error" in data: - raise IMException(data["Error"]) - - elif "Response" in data and data["Response"] == "True": - return data - - else: - raise IMException("An error occurs during movie search") + "Type": "TV Series" if soup.find(attrs={"class": "np_episode_guide"}) else "Movie", + "Country": ", ".join([c.find("a").text.strip() for c in soup.body.find(id="titleDetails").find_all(attrs={"class": "txt-block"}) if c.text.find("Country") != -1]), + "Released": soup.body.find(attrs={"itemprop": "datePublished"}).attrs["content"] if "content" in soup.body.find(attrs={"itemprop": "datePublished"}).attrs else "N\A", + "Genre": ", ".join([g.text.strip() for g in soup.body.find_all(attrs={"itemprop": "genre"})[:-1]]), + "Director": ", ".join([d.find(attrs={"itemprop": "name"}).text.strip() for d in soup.body.find_all(attrs={"itemprop": "director"})]), + "Writer": ", ".join([d.find(attrs={"itemprop": "name"}).text.strip() for d in soup.body.find_all(attrs={"itemprop": "creator"})]), + "Actors": ", ".join([d.find(attrs={"itemprop": "name"}).text.strip() for d in soup.body.find_all(attrs={"itemprop": "actors"})]), + } -def find_movies(title): +def find_movies(title, year=None): """Find existing movies matching a approximate title""" + title = title.lower() + # Built URL - url = "http://www.omdbapi.com/?s=%s" % urllib.parse.quote(title) + url = "https://v2.sg.media-imdb.com/suggests/%s/%s.json" % (urllib.parse.quote(title[0]), urllib.parse.quote(title.replace(" ", "_"))) # Make the request - data = web.getJSON(url) - - # Return data - if "Error" in data: - raise IMException(data["Error"]) - - elif "Search" in data: - return data + data = web.getJSON(url, remove_callback=True) + if year is None: + return data["d"] else: - raise IMException("An error occurs during movie search") + return [d for d in data["d"] if "y" in d and str(d["y"]) == year] # MODULE INTERFACE #################################################### @@ -79,23 +73,28 @@ def cmd_imdb(msg): title = ' '.join(msg.args) if re.match("^tt[0-9]{7}$", title) is not None: - data = get_movie(imdbid=title) + data = get_movie_by_id(imdbid=title) else: rm = re.match(r"^(.+)\s\(([0-9]{4})\)$", title) if rm is not None: - data = get_movie(title=rm.group(1), year=rm.group(2)) + data = find_movies(rm.group(1), year=rm.group(2)) else: - data = get_movie(title=title) + data = find_movies(title) + + if not data: + raise IMException("Movie/series not found") + + data = get_movie_by_id(data[0]["id"]) res = Response(channel=msg.channel, title="%s (%s)" % (data['Title'], data['Year']), nomore="No more information, more at http://www.imdb.com/title/%s" % data['imdbID']) - res.append_message("\x02rating\x0F: %s (%s votes); \x02plot\x0F: %s" % - (data['imdbRating'], data['imdbVotes'], data['Plot'])) + res.append_message("%s \x02genre:\x0F %s; \x02rating\x0F: %s (%s votes); \x02plot\x0F: %s" % + (data['Type'], data['Genre'], data['imdbRating'], data['imdbVotes'], data['Plot'])) - res.append_message("%s \x02from\x0F %s \x02released on\x0F %s; \x02genre:\x0F %s; \x02directed by:\x0F %s; \x02written by:\x0F %s; \x02main actors:\x0F %s" - % (data['Type'], data['Country'], data['Released'], data['Genre'], data['Director'], data['Writer'], data['Actors'])) + res.append_message("%s \x02from\x0F %s \x02released on\x0F %s; \x02directed by:\x0F %s; \x02written by:\x0F %s; \x02main actors:\x0F %s" + % (data['Type'], data['Country'], data['Released'], data['Director'], data['Writer'], data['Actors'])) return res @@ -111,7 +110,7 @@ def cmd_search(msg): data = find_movies(' '.join(msg.args)) movies = list() - for m in data['Search']: - movies.append("\x02%s\x0F (%s of %s)" % (m['Title'], m['Type'], m['Year'])) + for m in data: + movies.append("\x02%s\x0F%s with %s" % (m['l'], (" (" + str(m['y']) + ")") if "y" in m else "", m['s'])) return Response(movies, title="Titles found", channel=msg.channel) From fde459c3fff26bb1f1f982bd5966ec448b13ca29 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 18 Jul 2017 06:32:48 +0200 Subject: [PATCH 039/127] Remove legacy msg.nick --- modules/alias.py | 6 +++--- modules/birthday.py | 12 ++++++------ modules/events.py | 14 +++++++------- modules/mapquest.py | 2 +- modules/reddit.py | 2 +- modules/rnd.py | 2 +- modules/sms.py | 16 ++++++++-------- modules/spell/__init__.py | 6 +++--- modules/virtualradar.py | 2 +- modules/weather.py | 2 +- modules/whois.py | 4 ++-- modules/worldcup.py | 2 +- nemubot/bot.py | 2 +- nemubot/channel.py | 18 +++++++++--------- nemubot/message/abstract.py | 6 ------ nemubot/server/IRC.py | 8 ++++---- 16 files changed, 49 insertions(+), 55 deletions(-) diff --git a/modules/alias.py b/modules/alias.py index 5053783..5aae6bb 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -185,7 +185,7 @@ def cmd_listvars(msg): def cmd_set(msg): if len(msg.args) < 2: raise IMException("!set take two args: the key and the value.") - set_variable(msg.args[0], " ".join(msg.args[1:]), msg.nick) + set_variable(msg.args[0], " ".join(msg.args[1:]), msg.frm) return Response("Variable $%s successfully defined." % msg.args[0], channel=msg.channel) @@ -222,13 +222,13 @@ def cmd_alias(msg): if alias.cmd in context.data.getNode("aliases").index: return Response("%s corresponds to %s" % (alias.cmd, context.data.getNode("aliases").index[alias.cmd]["origin"]), - channel=msg.channel, nick=msg.nick) + channel=msg.channel, nick=msg.frm) elif len(msg.args) > 1: create_alias(alias.cmd, " ".join(msg.args[1:]), channel=msg.channel, - creator=msg.nick) + creator=msg.frm) return Response("New alias %s successfully registered." % alias.cmd, channel=msg.channel) diff --git a/modules/birthday.py b/modules/birthday.py index cb850ac..7a9cdaa 100644 --- a/modules/birthday.py +++ b/modules/birthday.py @@ -27,7 +27,7 @@ def load(context): def findName(msg): if (not len(msg.args) or msg.args[0].lower() == "moi" or msg.args[0].lower() == "me"): - name = msg.nick.lower() + name = msg.frm.lower() else: name = msg.args[0].lower() @@ -77,7 +77,7 @@ def cmd_anniv(msg): else: return Response("désolé, je ne connais pas la date d'anniversaire" " de %s. Quand est-il né ?" % name, - msg.channel, msg.nick) + msg.channel, msg.frm) @hook.command("age", @@ -98,7 +98,7 @@ def cmd_age(msg): msg.channel) else: return Response("désolé, je ne connais pas l'âge de %s." - " Quand est-il né ?" % name, msg.channel, msg.nick) + " Quand est-il né ?" % name, msg.channel, msg.frm) return True @@ -113,11 +113,11 @@ def parseask(msg): if extDate is None or extDate.year > datetime.now().year: return Response("la date de naissance ne paraît pas valide...", msg.channel, - msg.nick) + msg.frm) else: nick = res.group(1) if nick == "my" or nick == "I" or nick == "i" or nick == "je" or nick == "mon" or nick == "ma": - nick = msg.nick + nick = msg.frm if nick.lower() in context.data.index: context.data.index[nick.lower()]["born"] = extDate else: @@ -129,6 +129,6 @@ def parseask(msg): return Response("ok, c'est noté, %s est né le %s" % (nick, extDate.strftime("%A %d %B %Y à %H:%M")), msg.channel, - msg.nick) + msg.frm) except: raise IMException("la date de naissance ne paraît pas valide.") diff --git a/modules/events.py b/modules/events.py index a35c28b..0cc5a44 100644 --- a/modules/events.py +++ b/modules/events.py @@ -69,7 +69,7 @@ def start_countdown(msg): strnd = ModuleState("strend") strnd["server"] = msg.server strnd["channel"] = msg.channel - strnd["proprio"] = msg.nick + strnd["proprio"] = msg.frm strnd["start"] = msg.date strnd["name"] = msg.args[0] context.data.addChild(strnd) @@ -145,17 +145,17 @@ def end_countdown(msg): raise IMException("quel événement terminer ?") if msg.args[0] in context.data.index: - if context.data.index[msg.args[0]]["proprio"] == msg.nick or (msg.cmd == "forceend" and msg.frm_owner): + if context.data.index[msg.args[0]]["proprio"] == msg.frm or (msg.cmd == "forceend" and msg.frm_owner): duration = countdown(msg.date - context.data.index[msg.args[0]].getDate("start")) context.del_event(context.data.index[msg.args[0]]["_id"]) context.data.delChild(context.data.index[msg.args[0]]) context.save() return Response("%s a duré %s." % (msg.args[0], duration), - channel=msg.channel, nick=msg.nick) + channel=msg.channel, nick=msg.frm) else: raise IMException("Vous ne pouvez pas terminer le compteur %s, créé par %s." % (msg.args[0], context.data.index[msg.args[0]]["proprio"])) else: - return Response("%s n'est pas un compteur connu."% (msg.args[0]), channel=msg.channel, nick=msg.nick) + return Response("%s n'est pas un compteur connu."% (msg.args[0]), channel=msg.channel, nick=msg.frm) @hook.command("eventslist") @@ -180,7 +180,7 @@ def parseanswer(msg): # Avoid message starting by ! which can be interpreted as command by other bots if msg.cmd[0] == "!": - res.nick = msg.nick + res.nick = msg.frm if context.data.index[msg.cmd].name == "strend": if context.data.index[msg.cmd].hasAttribute("end"): @@ -223,7 +223,7 @@ def parseask(msg): evt = ModuleState("event") evt["server"] = msg.server evt["channel"] = msg.channel - evt["proprio"] = msg.nick + evt["proprio"] = msg.frm evt["name"] = name.group(1) evt["start"] = extDate evt["msg_after"] = msg_after @@ -237,7 +237,7 @@ def parseask(msg): evt = ModuleState("event") evt["server"] = msg.server evt["channel"] = msg.channel - evt["proprio"] = msg.nick + evt["proprio"] = msg.frm evt["name"] = name.group(1) evt["msg_before"] = texts.group (2) context.data.addChild(evt) diff --git a/modules/mapquest.py b/modules/mapquest.py index 55b87c0..1caa41c 100644 --- a/modules/mapquest.py +++ b/modules/mapquest.py @@ -55,7 +55,7 @@ def cmd_geocode(msg): if not len(msg.args): raise IMException("indicate a name") - res = Response(channel=msg.channel, nick=msg.nick, + res = Response(channel=msg.channel, nick=msg.frm, nomore="No more geocode", count=" (%s more geocode)") for loc in geocode(' '.join(msg.args)): diff --git a/modules/reddit.py b/modules/reddit.py index 7d481b7..31f566c 100644 --- a/modules/reddit.py +++ b/modules/reddit.py @@ -64,7 +64,7 @@ def cmd_subreddit(msg): channel=msg.channel)) else: all_res.append(Response("%s is not a valid subreddit" % osub, - channel=msg.channel, nick=msg.nick)) + channel=msg.channel, nick=msg.frm)) return all_res diff --git a/modules/rnd.py b/modules/rnd.py index 5329b06..6044bd4 100644 --- a/modules/rnd.py +++ b/modules/rnd.py @@ -21,7 +21,7 @@ def cmd_choice(msg): return Response(random.choice(msg.args), channel=msg.channel, - nick=msg.nick) + nick=msg.frm) @hook.command("choicecmd") diff --git a/modules/sms.py b/modules/sms.py index 3a9727f..61e63d6 100644 --- a/modules/sms.py +++ b/modules/sms.py @@ -73,9 +73,9 @@ def cmd_sms(msg): fails.append( "%s: %s" % (u, test) ) if len(fails) > 0: - return Response("quelque chose ne s'est pas bien passé durant l'envoi du SMS : " + ", ".join(fails), msg.channel, msg.nick) + return Response("quelque chose ne s'est pas bien passé durant l'envoi du SMS : " + ", ".join(fails), msg.channel, msg.frm) else: - return Response("le SMS a bien été envoyé", msg.channel, msg.nick) + return Response("le SMS a bien été envoyé", msg.channel, msg.frm) apiuser_ask = re.compile(r"(utilisateur|user|numéro|numero|compte|abonne|abone|abonné|account)\s+(est|is)\s+(?P[0-9]{7,})", re.IGNORECASE) apikey_ask = re.compile(r"(clef|key|password|mot de passe?)\s+(?:est|is)?\s+(?P[a-zA-Z0-9]{10,})", re.IGNORECASE) @@ -94,18 +94,18 @@ def parseask(msg): test = send_sms("nemubot", apiuser, apikey, "Vous avez enregistré vos codes d'authentification dans nemubot, félicitation !") if test is not None: - return Response("je n'ai pas pu enregistrer tes identifiants : %s" % test, msg.channel, msg.nick) + return Response("je n'ai pas pu enregistrer tes identifiants : %s" % test, msg.channel, msg.frm) - if msg.nick in context.data.index: - context.data.index[msg.nick]["user"] = apiuser - context.data.index[msg.nick]["key"] = apikey + if msg.frm in context.data.index: + context.data.index[msg.frm]["user"] = apiuser + context.data.index[msg.frm]["key"] = apikey else: ms = ModuleState("phone") - ms.setAttribute("name", msg.nick) + ms.setAttribute("name", msg.frm) ms.setAttribute("user", apiuser) ms.setAttribute("key", apikey) ms.setAttribute("lastuse", 0) context.data.addChild(ms) context.save() return Response("ok, c'est noté. Je t'ai envoyé un SMS pour tester ;)", - msg.channel, msg.nick) + msg.channel, msg.frm) diff --git a/modules/spell/__init__.py b/modules/spell/__init__.py index a70b016..c15f5fc 100644 --- a/modules/spell/__init__.py +++ b/modules/spell/__init__.py @@ -64,15 +64,15 @@ def cmd_spell(msg): raise IMException("Je n'ai pas le dictionnaire `%s' :(" % lang) if r == True: - add_score(msg.nick, "correct") + add_score(msg.frm, "correct") res.append_message("l'orthographe de `%s' est correcte" % word) elif len(r) > 0: - add_score(msg.nick, "bad") + add_score(msg.frm, "bad") res.append_message(r, title="suggestions pour `%s'" % word) else: - add_score(msg.nick, "bad") + add_score(msg.frm, "bad") res.append_message("aucune suggestion pour `%s'" % word) return res diff --git a/modules/virtualradar.py b/modules/virtualradar.py index ffd5a67..d7448ce 100644 --- a/modules/virtualradar.py +++ b/modules/virtualradar.py @@ -80,7 +80,7 @@ def cmd_flight(msg): if not len(msg.args): raise IMException("please indicate a flight") - res = Response(channel=msg.channel, nick=msg.nick, + res = Response(channel=msg.channel, nick=msg.frm, nomore="No more flights", count=" (%s more flights)") for param in msg.args: diff --git a/modules/weather.py b/modules/weather.py index 8b3540e..8c9ca0e 100644 --- a/modules/weather.py +++ b/modules/weather.py @@ -220,4 +220,4 @@ def parseask(msg): context.data.addChild(ms) context.save() return Response("ok, j'ai bien noté les coordonnées de %s" % res.group("city"), - msg.channel, msg.nick) + msg.channel, msg.frm) diff --git a/modules/whois.py b/modules/whois.py index fb6d250..ae27ccc 100644 --- a/modules/whois.py +++ b/modules/whois.py @@ -152,7 +152,7 @@ def parseask(msg): nick = res.group(1) login = res.group(3) if nick == "my" or nick == "I" or nick == "i" or nick == "je" or nick == "mon" or nick == "ma": - nick = msg.nick + nick = msg.frm if nick in context.data.getNode("aliases").index: context.data.getNode("aliases").index[nick]["to"] = login else: @@ -164,4 +164,4 @@ def parseask(msg): return Response("ok, c'est noté, %s est %s" % (nick, login), channel=msg.channel, - nick=msg.nick) + nick=msg.frm) diff --git a/modules/worldcup.py b/modules/worldcup.py index 7b4f53d..ff3e0c4 100644 --- a/modules/worldcup.py +++ b/modules/worldcup.py @@ -32,7 +32,7 @@ def start_watch(msg): w = ModuleState("watch") w["server"] = msg.server w["channel"] = msg.channel - w["proprio"] = msg.nick + w["proprio"] = msg.frm w["start"] = datetime.now(timezone.utc) context.data.addChild(w) context.save() diff --git a/nemubot/bot.py b/nemubot/bot.py index aa1cb3e..6327afe 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -92,7 +92,7 @@ class Bot(threading.Thread): def in_echo(msg): from nemubot.message import Text - return Text(msg.nick + ": " + " ".join(msg.args), to=msg.to_response) + return Text(msg.frm + ": " + " ".join(msg.args), to=msg.to_response) self.treater.hm.add_hook(nemubot.hooks.Command(in_echo, "echo"), "in", "Command") def _help_msg(msg): diff --git a/nemubot/channel.py b/nemubot/channel.py index a070131..835c22f 100644 --- a/nemubot/channel.py +++ b/nemubot/channel.py @@ -52,11 +52,11 @@ class Channel: elif cmd == "MODE": self.mode(msg) elif cmd == "JOIN": - self.join(msg.nick) + self.join(msg.frm) elif cmd == "NICK": - self.nick(msg.nick, msg.text) + self.nick(msg.frm, msg.text) elif cmd == "PART" or cmd == "QUIT": - self.part(msg.nick) + self.part(msg.frm) elif cmd == "TOPIC": self.topic = self.text @@ -120,17 +120,17 @@ class Channel: else: self.password = msg.text[1] elif msg.text[0] == "+o": - self.people[msg.nick] |= 4 + self.people[msg.frm] |= 4 elif msg.text[0] == "-o": - self.people[msg.nick] &= ~4 + self.people[msg.frm] &= ~4 elif msg.text[0] == "+h": - self.people[msg.nick] |= 2 + self.people[msg.frm] |= 2 elif msg.text[0] == "-h": - self.people[msg.nick] &= ~2 + self.people[msg.frm] &= ~2 elif msg.text[0] == "+v": - self.people[msg.nick] |= 1 + self.people[msg.frm] |= 1 elif msg.text[0] == "-v": - self.people[msg.nick] &= ~1 + self.people[msg.frm] &= ~1 def parse332(self, msg): """Parse RPL_TOPIC message diff --git a/nemubot/message/abstract.py b/nemubot/message/abstract.py index 6ee43d5..3af0511 100644 --- a/nemubot/message/abstract.py +++ b/nemubot/message/abstract.py @@ -59,12 +59,6 @@ class Abstract: else: return None - @property - def nick(self): - # TODO: this is for legacy modules - return self.frm - - def accept(self, visitor): visitor.visit(self) diff --git a/nemubot/server/IRC.py b/nemubot/server/IRC.py index 7469abc..7adc484 100644 --- a/nemubot/server/IRC.py +++ b/nemubot/server/IRC.py @@ -174,10 +174,10 @@ class _IRC: for chname in msg.params[0].split(b","): if chname in self.channels: - if msg.nick == self.nick: + if msg.frm == self.nick: del self.channels[chname] - elif msg.nick in self.channels[chname].people: - del self.channels[chname].people[msg.nick] + elif msg.frm in self.channels[chname].people: + del self.channels[chname].people[msg.frm] self.hookscmd["PART"] = _on_part # Respond to 331/RPL_NOTOPIC,332/RPL_TOPIC,TOPIC def _on_topic(msg): @@ -227,7 +227,7 @@ class _IRC: else: res = "ERRMSG Unknown or unimplemented CTCP request" if res is not None: - self.write("NOTICE %s :\x01%s\x01" % (msg.nick, res)) + self.write("NOTICE %s :\x01%s\x01" % (msg.frm, res)) self.hookscmd["PRIVMSG"] = _on_ctcp From a11ccb2e39b5a1402f82393446aacc0a4c0e51f0 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 18 Jul 2017 06:39:17 +0200 Subject: [PATCH 040/127] Remove legacy msg.cmds --- nemubot/message/command.py | 5 ----- nemubot/message/response.py | 5 ----- 2 files changed, 10 deletions(-) diff --git a/nemubot/message/command.py b/nemubot/message/command.py index 6c208b2..ca87e4c 100644 --- a/nemubot/message/command.py +++ b/nemubot/message/command.py @@ -31,11 +31,6 @@ class Command(Abstract): def __str__(self): return self.cmd + " @" + ",@".join(self.args) - @property - def cmds(self): - # TODO: this is for legacy modules - return [self.cmd] + self.args - class OwnerCommand(Command): diff --git a/nemubot/message/response.py b/nemubot/message/response.py index fba864b..f9353ad 100644 --- a/nemubot/message/response.py +++ b/nemubot/message/response.py @@ -27,8 +27,3 @@ class Response(Abstract): def __str__(self): return self.cmd + " @" + ",@".join(self.args) - - @property - def cmds(self): - # TODO: this is for legacy modules - return [self.cmd] + self.args From e49312e63e3c9b91cf77c2392705a6aabe4316af Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 18 Jul 2017 06:48:15 +0200 Subject: [PATCH 041/127] Remove legacy msg.text --- modules/birthday.py | 4 ++-- modules/events.py | 8 ++++---- modules/reddit.py | 11 +++++++++-- modules/sms.py | 10 +++++----- modules/urlreducer.py | 18 ++++++++++++++++-- modules/weather.py | 2 +- modules/whois.py | 2 +- 7 files changed, 38 insertions(+), 17 deletions(-) diff --git a/modules/birthday.py b/modules/birthday.py index 7a9cdaa..d8093b8 100644 --- a/modules/birthday.py +++ b/modules/birthday.py @@ -106,10 +106,10 @@ def cmd_age(msg): @hook.ask() def parseask(msg): - res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)?\s+(birthday|geburtstag|née? |nee? le|born on).*$", msg.text, re.I) + res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)?\s+(birthday|geburtstag|née? |nee? le|born on).*$", msg.message, re.I) if res is not None: try: - extDate = extractDate(msg.text) + extDate = extractDate(msg.message) if extDate is None or extDate.year > datetime.now().year: return Response("la date de naissance ne paraît pas valide...", msg.channel, diff --git a/modules/events.py b/modules/events.py index 0cc5a44..f6c6621 100644 --- a/modules/events.py +++ b/modules/events.py @@ -194,17 +194,17 @@ def parseanswer(msg): RGXP_ask = re.compile(r"^.*((create|new)\s+(a|an|a\s*new|an\s*other)?\s*(events?|commande?)|(nouvel(le)?|ajoute|cr[ée]{1,3})\s+(un)?\s*([eé]v[ée]nements?|commande?)).*$", re.I) -@hook.ask(match=lambda msg: RGXP_ask.match(msg.text)) +@hook.ask(match=lambda msg: RGXP_ask.match(msg.message)) def parseask(msg): - name = re.match("^.*!([^ \"'@!]+).*$", msg.text) + name = re.match("^.*!([^ \"'@!]+).*$", msg.message) if name is None: raise IMException("il faut que tu attribues une commande à l'événement.") if name.group(1) in context.data.index: raise IMException("un événement portant ce nom existe déjà.") - texts = re.match("^[^\"]*(avant|après|apres|before|after)?[^\"]*\"([^\"]+)\"[^\"]*((avant|après|apres|before|after)?.*\"([^\"]+)\".*)?$", msg.text, re.I) + texts = re.match("^[^\"]*(avant|après|apres|before|after)?[^\"]*\"([^\"]+)\"[^\"]*((avant|après|apres|before|after)?.*\"([^\"]+)\".*)?$", msg.message, re.I) if texts is not None and texts.group(3) is not None: - extDate = extractDate(msg.text) + extDate = extractDate(msg.message) if extDate is None or extDate == "": raise IMException("la date de l'événement est invalide !") diff --git a/modules/reddit.py b/modules/reddit.py index 31f566c..ae28999 100644 --- a/modules/reddit.py +++ b/modules/reddit.py @@ -71,8 +71,15 @@ def cmd_subreddit(msg): @hook.message() def parselisten(msg): - parseresponse(msg) - return None + global LAST_SUBS + + if hasattr(msg, "message") and msg.message and type(msg.message) == str: + urls = re.findall("www.reddit.com(/\w/\w+/?)", msg.message) + for url in urls: + for recv in msg.to: + if recv not in LAST_SUBS: + LAST_SUBS[recv] = list() + LAST_SUBS[recv].append(url) @hook.post() diff --git a/modules/sms.py b/modules/sms.py index 61e63d6..ca7e9f0 100644 --- a/modules/sms.py +++ b/modules/sms.py @@ -82,11 +82,11 @@ apikey_ask = re.compile(r"(clef|key|password|mot de passe?)\s+(?:est|is)?\s+(?P< @hook.ask() def parseask(msg): - if msg.text.find("Free") >= 0 and ( - msg.text.find("API") >= 0 or msg.text.find("api") >= 0) and ( - msg.text.find("SMS") >= 0 or msg.text.find("sms") >= 0): - resuser = apiuser_ask.search(msg.text) - reskey = apikey_ask.search(msg.text) + if msg.message.find("Free") >= 0 and ( + msg.message.find("API") >= 0 or msg.message.find("api") >= 0) and ( + msg.message.find("SMS") >= 0 or msg.message.find("sms") >= 0): + resuser = apiuser_ask.search(msg.message) + reskey = apikey_ask.search(msg.message) if resuser is not None and reskey is not None: apiuser = resuser.group("user") apikey = reskey.group("key") diff --git a/modules/urlreducer.py b/modules/urlreducer.py index 36fcb3c..bd7646b 100644 --- a/modules/urlreducer.py +++ b/modules/urlreducer.py @@ -84,8 +84,22 @@ LAST_URLS = dict() @hook.message() def parselisten(msg): - parseresponse(msg) - return None + global LAST_URLS + if hasattr(msg, "message") and isinstance(msg.message, str): + urls = re.findall("([a-zA-Z0-9+.-]+:(?://)?(?:[^ :/]+:[0-9]+)?[^ :]+)", + msg.message) + for url in urls: + o = urlparse(web._getNormalizedURL(url), "http") + + # Skip short URLs + if (o.netloc == "" or o.netloc in PROVIDERS or + len(o.netloc) + len(o.path) < 17): + continue + + for recv in msg.to: + if recv not in LAST_URLS: + LAST_URLS[recv] = list() + LAST_URLS[recv].append(url) @hook.post() diff --git a/modules/weather.py b/modules/weather.py index 8c9ca0e..3f74b8e 100644 --- a/modules/weather.py +++ b/modules/weather.py @@ -203,7 +203,7 @@ gps_ask = re.compile(r"^\s*(?P.*\w)\s*(?:(?:se|est)\s+(?:trouve|situ[ée]* @hook.ask() def parseask(msg): - res = gps_ask.match(msg.text) + res = gps_ask.match(msg.message) if res is not None: city_name = res.group("city").lower() gps_lat = res.group("lat").replace(",", ".") diff --git a/modules/whois.py b/modules/whois.py index ae27ccc..00eb940 100644 --- a/modules/whois.py +++ b/modules/whois.py @@ -147,7 +147,7 @@ def cmd_nicks(msg): @hook.ask() def parseask(msg): - res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)\s+([a-zA-Z0-9_-]{3,8})$", msg.text, re.I) + res = re.match(r"^(\S+)\s*('s|suis|est|is|was|were)\s+([a-zA-Z0-9_-]{3,8})$", msg.message, re.I) if res is not None: nick = res.group(1) login = res.group(3) From 45fe5b21561e664cac526dd6e8b0f02b8e27545e Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 18 Jul 2017 07:16:54 +0200 Subject: [PATCH 042/127] Refactor configuration loading --- nemubot/__main__.py | 60 ++++++++++++++++++++++++++++--- nemubot/bot.py | 76 ++++------------------------------------ nemubot/modulecontext.py | 2 +- 3 files changed, 63 insertions(+), 75 deletions(-) diff --git a/nemubot/__main__.py b/nemubot/__main__.py index e1576fb..8d51249 100644 --- a/nemubot/__main__.py +++ b/nemubot/__main__.py @@ -118,10 +118,10 @@ def main(): # Create bot context from nemubot import datastore - from nemubot.bot import Bot, sync_act + from nemubot.bot import Bot context = Bot(modules_paths=modules_paths, data_store=datastore.XML(args.data_path), - verbosity=args.verbose) + debug=args.verbose > 0) if args.no_connect: context.noautoconnect = True @@ -133,10 +133,34 @@ def main(): # Load requested configuration files for path in args.files: - if os.path.isfile(path): - sync_act("loadconf", path) - else: + if not os.path.isfile(path): logger.error("%s is not a readable file", path) + continue + + config = load_config(path) + + # Preset each server in this file + for server in config.servers: + srv = server.server(config) + # Add the server in the context + if context.add_server(srv): + logger.info("Server '%s' successfully added.", srv.name) + else: + logger.error("Can't add server '%s'.", srv.name) + + # Load module and their configuration + for mod in config.modules: + context.modules_configuration[mod.name] = mod + if mod.autoload: + try: + __import__(mod.name) + except: + logger.exception("Exception occurs when loading module" + " '%s'", mod.name) + + # Load files asked by the configuration file + args.files += config.includes + if args.module: for module in args.module: @@ -205,5 +229,31 @@ def main(): sys.exit(0) +def load_config(filename): + """Load a configuration file + + Arguments: + filename -- the path to the file to load + """ + + from nemubot.channel import Channel + from nemubot import config + from nemubot.tools.xmlparser import XMLParser + + try: + p = XMLParser({ + "nemubotconfig": config.Nemubot, + "server": config.Server, + "channel": Channel, + "module": config.Module, + "include": config.Include, + }) + return p.parse_file(filename) + except: + logger.exception("Can't load `%s'; this is not a valid nemubot " + "configuration file.", filename) + return None + + if __name__ == "__main__": main() diff --git a/nemubot/bot.py b/nemubot/bot.py index 6327afe..7975958 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -40,14 +40,14 @@ class Bot(threading.Thread): """Class containing the bot context and ensuring key goals""" def __init__(self, ip="127.0.0.1", modules_paths=list(), - data_store=datastore.Abstract(), verbosity=0): + data_store=datastore.Abstract(), debug=False): """Initialize the bot context Keyword arguments: ip -- The external IP of the bot (default: 127.0.0.1) modules_paths -- Paths to all directories where looking for modules data_store -- An instance of the nemubot datastore for bot's modules - verbosity -- verbosity level + debug -- enable debug """ super().__init__(name="Nemubot main") @@ -56,7 +56,7 @@ class Bot(threading.Thread): __version__, sys.version_info.major, sys.version_info.minor, sys.version_info.micro) - self.verbosity = verbosity + self.debug = debug self.stop = None # External IP for accessing this bot @@ -149,6 +149,10 @@ class Bot(threading.Thread): self.cnsr_thrd_size = -1 + def __del__(self): + self.datastore.close() + + def run(self): global sync_queue @@ -218,12 +222,6 @@ class Bot(threading.Thread): elif action == "launch_consumer": pass # This is treated after the loop - elif action == "loadconf": - for path in args: - logger.debug("Load configuration from %s", path) - self.load_file(path) - logger.info("Configurations successfully loaded") - sync_queue.task_done() @@ -240,64 +238,6 @@ class Bot(threading.Thread): - # Config methods - - def load_file(self, filename): - """Load a configuration file - - Arguments: - filename -- the path to the file to load - """ - - import os - - # Unexisting file, assume a name was passed, import the module! - if not os.path.isfile(filename): - return self.import_module(filename) - - from nemubot.channel import Channel - from nemubot import config - from nemubot.tools.xmlparser import XMLParser - - try: - p = XMLParser({ - "nemubotconfig": config.Nemubot, - "server": config.Server, - "channel": Channel, - "module": config.Module, - "include": config.Include, - }) - config = p.parse_file(filename) - except: - logger.exception("Can't load `%s'; this is not a valid nemubot " - "configuration file." % filename) - return False - - # Preset each server in this file - for server in config.servers: - srv = server.server(config) - # Add the server in the context - if self.add_server(srv, server.autoconnect): - logger.info("Server '%s' successfully added." % srv.name) - else: - logger.error("Can't add server '%s'." % srv.name) - - # Load module and their configuration - for mod in config.modules: - self.modules_configuration[mod.name] = mod - if mod.autoload: - try: - __import__(mod.name) - except: - logger.exception("Exception occurs when loading module" - " '%s'", mod.name) - - - # Load files asked by the configuration file - for load in config.includes: - self.load_file(load.path) - - # Events methods def add_event(self, evt, eid=None, module_src=None): @@ -581,8 +521,6 @@ class Bot(threading.Thread): for cnsr in k: cnsr.stop = True - self.datastore.close() - if self.stop is False or sync_queue is not None: self.stop = True sync_act("end") diff --git a/nemubot/modulecontext.py b/nemubot/modulecontext.py index 877b8de..70e4b6f 100644 --- a/nemubot/modulecontext.py +++ b/nemubot/modulecontext.py @@ -108,7 +108,7 @@ class ModuleContext(_ModuleContext): self.config = context.modules_configuration[self.module_name] self.context = context - self.debug = context.verbosity > 0 + self.debug = context.debug def load_data(self): From f60de818f275f71e932d05d8056daf7f0f13cf2b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 27 Aug 2017 18:22:53 +0200 Subject: [PATCH 043/127] Virtualy move all nemubot modules into nemubot.module.* hierarchy, to avoid conflict with system/vendor modules --- modules/alias.py | 2 +- modules/birthday.py | 2 +- modules/bonneannee.py | 3 +-- modules/books.py | 2 +- modules/cat.py | 2 +- modules/conjugaison.py | 2 +- modules/ctfs.py | 2 +- modules/cve.py | 2 +- modules/ddg.py | 2 +- modules/dig.py | 2 +- modules/disas.py | 2 +- modules/events.py | 2 +- modules/freetarifs.py | 2 +- modules/github.py | 2 +- modules/grep.py | 2 +- modules/imdb.py | 2 +- modules/jsonbot.py | 2 +- modules/man.py | 2 +- modules/mapquest.py | 2 +- modules/mediawiki.py | 2 +- modules/networking/__init__.py | 2 +- modules/networking/watchWebsite.py | 2 +- modules/networking/whois.py | 2 +- modules/news.py | 2 +- modules/openroute.py | 2 +- modules/pkgs.py | 2 +- modules/ratp.py | 2 +- modules/reddit.py | 2 +- modules/rnd.py | 2 +- modules/sap.py | 2 +- modules/shodan.py | 2 +- modules/sleepytime.py | 2 +- modules/sms.py | 2 +- modules/spell/__init__.py | 2 +- modules/suivi.py | 2 +- modules/syno.py | 2 +- modules/tpb.py | 2 +- modules/translate.py | 2 +- modules/urbandict.py | 2 +- modules/velib.py | 2 +- modules/virtualradar.py | 4 ++-- modules/weather.py | 4 ++-- modules/whois.py | 4 ++-- modules/wolframalpha.py | 2 +- modules/worldcup.py | 2 +- modules/youtube-title.py | 2 +- nemubot/__main__.py | 4 ++-- nemubot/bot.py | 11 ++++++----- nemubot/importer.py | 18 +++++++++--------- nemubot/module/__init__.py | 7 +++++++ {modules => nemubot/module}/more.py | 0 nemubot/modulecontext.py | 2 +- setup.py | 1 + 53 files changed, 75 insertions(+), 67 deletions(-) create mode 100644 nemubot/module/__init__.py rename {modules => nemubot/module}/more.py (100%) diff --git a/modules/alias.py b/modules/alias.py index 5aae6bb..a246d2c 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -12,7 +12,7 @@ from nemubot.message import Command from nemubot.tools.human import guess from nemubot.tools.xmlparser.node import ModuleState -from more import Response +from nemubot.module.more import Response # LOADING ############################################################# diff --git a/modules/birthday.py b/modules/birthday.py index d8093b8..e1406d4 100644 --- a/modules/birthday.py +++ b/modules/birthday.py @@ -13,7 +13,7 @@ from nemubot.tools.countdown import countdown_format from nemubot.tools.date import extractDate from nemubot.tools.xmlparser.node import ModuleState -from more import Response +from nemubot.module.more import Response # LOADING ############################################################# diff --git a/modules/bonneannee.py b/modules/bonneannee.py index b3b3934..1829bce 100644 --- a/modules/bonneannee.py +++ b/modules/bonneannee.py @@ -4,12 +4,11 @@ from datetime import datetime, timezone -from nemubot import context from nemubot.event import ModuleEvent from nemubot.hooks import hook from nemubot.tools.countdown import countdown_format -from more import Response +from nemubot.module.more import Response # GLOBALS ############################################################# diff --git a/modules/books.py b/modules/books.py index df48056..5ab404b 100644 --- a/modules/books.py +++ b/modules/books.py @@ -9,7 +9,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # LOADING ############################################################# diff --git a/modules/cat.py b/modules/cat.py index 0619cee..5eb3e19 100644 --- a/modules/cat.py +++ b/modules/cat.py @@ -7,7 +7,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.message import Command, DirectAsk, Text -from more import Response +from nemubot.module.more import Response # MODULE CORE ######################################################### diff --git a/modules/conjugaison.py b/modules/conjugaison.py index 25fe242..42d78c6 100644 --- a/modules/conjugaison.py +++ b/modules/conjugaison.py @@ -11,7 +11,7 @@ from nemubot.hooks import hook from nemubot.tools import web from nemubot.tools.web import striphtml -from more import Response +from nemubot.module.more import Response # GLOBALS ############################################################# diff --git a/modules/ctfs.py b/modules/ctfs.py index 1526cbc..169ee46 100644 --- a/modules/ctfs.py +++ b/modules/ctfs.py @@ -6,7 +6,7 @@ from bs4 import BeautifulSoup from nemubot.hooks import hook from nemubot.tools.web import getURLContent, striphtml -from more import Response +from nemubot.module.more import Response # GLOBALS ############################################################# diff --git a/modules/cve.py b/modules/cve.py index 6cdb339..b9cf1c3 100644 --- a/modules/cve.py +++ b/modules/cve.py @@ -9,7 +9,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools.web import getURLContent, striphtml -from more import Response +from nemubot.module.more import Response BASEURL_NIST = 'https://nvd.nist.gov/vuln/detail/' diff --git a/modules/ddg.py b/modules/ddg.py index d94bd61..089409b 100644 --- a/modules/ddg.py +++ b/modules/ddg.py @@ -8,7 +8,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # MODULE CORE ######################################################### diff --git a/modules/dig.py b/modules/dig.py index de7b2a3..bec0a87 100644 --- a/modules/dig.py +++ b/modules/dig.py @@ -14,7 +14,7 @@ import dns.resolver from nemubot.exception import IMException from nemubot.hooks import hook -from more import Response +from nemubot.module.more import Response # MODULE INTERFACE #################################################### diff --git a/modules/disas.py b/modules/disas.py index 7c17907..cb80ef3 100644 --- a/modules/disas.py +++ b/modules/disas.py @@ -7,7 +7,7 @@ import capstone from nemubot.exception import IMException from nemubot.hooks import hook -from more import Response +from nemubot.module.more import Response # MODULE CORE ######################################################### diff --git a/modules/events.py b/modules/events.py index f6c6621..9814aa2 100644 --- a/modules/events.py +++ b/modules/events.py @@ -12,7 +12,7 @@ from nemubot.tools.countdown import countdown_format, countdown from nemubot.tools.date import extractDate from nemubot.tools.xmlparser.node import ModuleState -from more import Response +from nemubot.module.more import Response def help_full (): diff --git a/modules/freetarifs.py b/modules/freetarifs.py index b96a30f..49ad8a6 100644 --- a/modules/freetarifs.py +++ b/modules/freetarifs.py @@ -9,7 +9,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # MODULE CORE ######################################################### diff --git a/modules/github.py b/modules/github.py index ddd0851..5f9a7d9 100644 --- a/modules/github.py +++ b/modules/github.py @@ -9,7 +9,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # MODULE CORE ######################################################### diff --git a/modules/grep.py b/modules/grep.py index 6a26c02..5c25c7d 100644 --- a/modules/grep.py +++ b/modules/grep.py @@ -9,7 +9,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.message import Command, Text -from more import Response +from nemubot.module.more import Response # MODULE CORE ######################################################### diff --git a/modules/imdb.py b/modules/imdb.py index bd1cadf..d5ff158 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -11,7 +11,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # MODULE CORE ######################################################### diff --git a/modules/jsonbot.py b/modules/jsonbot.py index fe25187..3126dc1 100644 --- a/modules/jsonbot.py +++ b/modules/jsonbot.py @@ -1,7 +1,7 @@ from nemubot.hooks import hook from nemubot.exception import IMException from nemubot.tools import web -from more import Response +from nemubot.module.more import Response import json nemubotversion = 3.4 diff --git a/modules/man.py b/modules/man.py index f45e30d..f60e0cf 100644 --- a/modules/man.py +++ b/modules/man.py @@ -8,7 +8,7 @@ import os from nemubot.hooks import hook -from more import Response +from nemubot.module.more import Response # GLOBALS ############################################################# diff --git a/modules/mapquest.py b/modules/mapquest.py index 1caa41c..5662a49 100644 --- a/modules/mapquest.py +++ b/modules/mapquest.py @@ -9,7 +9,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # GLOBALS ############################################################# diff --git a/modules/mediawiki.py b/modules/mediawiki.py index cb3d1da..be608ca 100644 --- a/modules/mediawiki.py +++ b/modules/mediawiki.py @@ -11,7 +11,7 @@ from nemubot.tools import web nemubotversion = 3.4 -from more import Response +from nemubot.module.more import Response # MEDIAWIKI REQUESTS ################################################## diff --git a/modules/networking/__init__.py b/modules/networking/__init__.py index f0df094..3b939ab 100644 --- a/modules/networking/__init__.py +++ b/modules/networking/__init__.py @@ -8,7 +8,7 @@ import re from nemubot.exception import IMException from nemubot.hooks import hook -from more import Response +from nemubot.module.more import Response from . import isup from . import page diff --git a/modules/networking/watchWebsite.py b/modules/networking/watchWebsite.py index 4945981..adedbee 100644 --- a/modules/networking/watchWebsite.py +++ b/modules/networking/watchWebsite.py @@ -12,7 +12,7 @@ from nemubot.tools.xmlparser.node import ModuleState logger = logging.getLogger("nemubot.module.networking.watchWebsite") -from more import Response +from nemubot.module.more import Response from . import page diff --git a/modules/networking/whois.py b/modules/networking/whois.py index d3d30b1..787cd17 100644 --- a/modules/networking/whois.py +++ b/modules/networking/whois.py @@ -6,7 +6,7 @@ import urllib from nemubot.exception import IMException from nemubot.tools.web import getJSON -from more import Response +from nemubot.module.more import Response URL_AVAIL = "https://www.whoisxmlapi.com/whoisserver/WhoisService?cmd=GET_DN_AVAILABILITY&domainName=%%s&outputFormat=json&username=%s&password=%s" URL_WHOIS = "http://www.whoisxmlapi.com/whoisserver/WhoisService?da=2&domainName=%%s&outputFormat=json&userName=%s&password=%s" diff --git a/modules/news.py b/modules/news.py index a8fb8de..40daa92 100644 --- a/modules/news.py +++ b/modules/news.py @@ -12,7 +12,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response from nemubot.tools.feed import Feed, AtomEntry diff --git a/modules/openroute.py b/modules/openroute.py index 440b05a..c280dec 100644 --- a/modules/openroute.py +++ b/modules/openroute.py @@ -9,7 +9,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # GLOBALS ############################################################# diff --git a/modules/pkgs.py b/modules/pkgs.py index 5a7b0a9..386946f 100644 --- a/modules/pkgs.py +++ b/modules/pkgs.py @@ -8,7 +8,7 @@ from nemubot import context from nemubot.exception import IMException from nemubot.hooks import hook -from more import Response +from nemubot.module.more import Response DB = None diff --git a/modules/ratp.py b/modules/ratp.py index 7f4b211..06f5f1d 100644 --- a/modules/ratp.py +++ b/modules/ratp.py @@ -4,7 +4,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook -from more import Response +from nemubot.module.more import Response from nextstop import ratp diff --git a/modules/reddit.py b/modules/reddit.py index ae28999..2de7612 100644 --- a/modules/reddit.py +++ b/modules/reddit.py @@ -10,7 +10,7 @@ from nemubot.tools import web nemubotversion = 3.4 -from more import Response +from nemubot.module.more import Response def help_full(): diff --git a/modules/rnd.py b/modules/rnd.py index 6044bd4..d1c6fe7 100644 --- a/modules/rnd.py +++ b/modules/rnd.py @@ -9,7 +9,7 @@ from nemubot import context from nemubot.exception import IMException from nemubot.hooks import hook -from more import Response +from nemubot.module.more import Response # MODULE INTERFACE #################################################### diff --git a/modules/sap.py b/modules/sap.py index 8691d6a..a6168a2 100644 --- a/modules/sap.py +++ b/modules/sap.py @@ -12,7 +12,7 @@ from nemubot.tools import web nemubotversion = 4.0 -from more import Response +from nemubot.module.more import Response def help_full(): diff --git a/modules/shodan.py b/modules/shodan.py index 4b2edae..9c158c6 100644 --- a/modules/shodan.py +++ b/modules/shodan.py @@ -11,7 +11,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # GLOBALS ############################################################# diff --git a/modules/sleepytime.py b/modules/sleepytime.py index 715b3b9..f7fb626 100644 --- a/modules/sleepytime.py +++ b/modules/sleepytime.py @@ -10,7 +10,7 @@ from nemubot.hooks import hook nemubotversion = 3.4 -from more import Response +from nemubot.module.more import Response def help_full(): diff --git a/modules/sms.py b/modules/sms.py index ca7e9f0..7db172b 100644 --- a/modules/sms.py +++ b/modules/sms.py @@ -16,7 +16,7 @@ from nemubot.tools.xmlparser.node import ModuleState nemubotversion = 3.4 -from more import Response +from nemubot.module.more import Response def load(context): context.data.setIndex("name", "phone") diff --git a/modules/spell/__init__.py b/modules/spell/__init__.py index c15f5fc..da16a80 100644 --- a/modules/spell/__init__.py +++ b/modules/spell/__init__.py @@ -10,7 +10,7 @@ from nemubot.tools.xmlparser.node import ModuleState from .pyaspell import Aspell from .pyaspell import AspellError -from more import Response +from nemubot.module.more import Response # LOADING ############################################################# diff --git a/modules/suivi.py b/modules/suivi.py index 6ad13e9..4bc079e 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -10,7 +10,7 @@ import re from nemubot.hooks import hook from nemubot.exception import IMException from nemubot.tools.web import getURLContent, getJSON -from more import Response +from nemubot.module.more import Response # POSTAGE SERVICE PARSERS ############################################ diff --git a/modules/syno.py b/modules/syno.py index 4bdc990..bda0456 100644 --- a/modules/syno.py +++ b/modules/syno.py @@ -9,7 +9,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # LOADING ############################################################# diff --git a/modules/tpb.py b/modules/tpb.py index ce98b04..a752324 100644 --- a/modules/tpb.py +++ b/modules/tpb.py @@ -8,7 +8,7 @@ from nemubot.tools.web import getJSON nemubotversion = 4.0 -from more import Response +from nemubot.module.more import Response URL_TPBAPI = None diff --git a/modules/translate.py b/modules/translate.py index 9d50966..906ba93 100644 --- a/modules/translate.py +++ b/modules/translate.py @@ -8,7 +8,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # GLOBALS ############################################################# diff --git a/modules/urbandict.py b/modules/urbandict.py index e90c096..a897fad 100644 --- a/modules/urbandict.py +++ b/modules/urbandict.py @@ -8,7 +8,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # MODULE CORE ######################################################### diff --git a/modules/velib.py b/modules/velib.py index 8ef6833..71c472c 100644 --- a/modules/velib.py +++ b/modules/velib.py @@ -9,7 +9,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # LOADING ############################################################# diff --git a/modules/virtualradar.py b/modules/virtualradar.py index d7448ce..9382d3b 100644 --- a/modules/virtualradar.py +++ b/modules/virtualradar.py @@ -10,8 +10,8 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response -import mapquest +from nemubot.module.more import Response +from nemubot.module import mapquest # GLOBALS ############################################################# diff --git a/modules/weather.py b/modules/weather.py index 3f74b8e..bee0d20 100644 --- a/modules/weather.py +++ b/modules/weather.py @@ -11,11 +11,11 @@ from nemubot.hooks import hook from nemubot.tools import web from nemubot.tools.xmlparser.node import ModuleState -import mapquest +from nemubot.module import mapquest nemubotversion = 4.0 -from more import Response +from nemubot.module.more import Response URL_DSAPI = "https://api.darksky.net/forecast/%s/%%s,%%s?lang=%%s&units=%%s" diff --git a/modules/whois.py b/modules/whois.py index 00eb940..d6106dd 100644 --- a/modules/whois.py +++ b/modules/whois.py @@ -10,8 +10,8 @@ from nemubot.tools.xmlparser.node import ModuleState nemubotversion = 3.4 -from more import Response -from networking.page import headers +from nemubot.module.more import Response +from nemubot.module.networking.page import headers PASSWD_FILE = None # You can get one with: curl -b "sessionid=YOURSESSIONID" 'https://accounts.cri.epita.net/api/users/' > users.json diff --git a/modules/wolframalpha.py b/modules/wolframalpha.py index e6bf86c..b7cc7fb 100644 --- a/modules/wolframalpha.py +++ b/modules/wolframalpha.py @@ -10,7 +10,7 @@ from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools import web -from more import Response +from nemubot.module.more import Response # LOADING ############################################################# diff --git a/modules/worldcup.py b/modules/worldcup.py index ff3e0c4..b12ca30 100644 --- a/modules/worldcup.py +++ b/modules/worldcup.py @@ -15,7 +15,7 @@ from nemubot.tools.xmlparser.node import ModuleState nemubotversion = 3.4 -from more import Response +from nemubot.module.more import Response API_URL="http://worldcup.sfg.io/%s" diff --git a/modules/youtube-title.py b/modules/youtube-title.py index fe62cda..41b613a 100644 --- a/modules/youtube-title.py +++ b/modules/youtube-title.py @@ -4,7 +4,7 @@ import re, json, subprocess from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools.web import _getNormalizedURL, getURLContent -from more import Response +from nemubot.module.more import Response """Get information of youtube videos""" diff --git a/nemubot/__main__.py b/nemubot/__main__.py index 8d51249..b79d90e 100644 --- a/nemubot/__main__.py +++ b/nemubot/__main__.py @@ -153,7 +153,7 @@ def main(): context.modules_configuration[mod.name] = mod if mod.autoload: try: - __import__(mod.name) + __import__("nemubot.module." + mod.name) except: logger.exception("Exception occurs when loading module" " '%s'", mod.name) @@ -164,7 +164,7 @@ def main(): if args.module: for module in args.module: - __import__(module) + __import__("nemubot.module." + module) if args.socketfile: from nemubot.server.socket import UnixSocketListener diff --git a/nemubot/bot.py b/nemubot/bot.py index 7975958..d2e042c 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -97,18 +97,19 @@ class Bot(threading.Thread): def _help_msg(msg): """Parse and response to help messages""" - from more import Response + from nemubot.module.more import Response res = Response(channel=msg.to_response) if len(msg.args) >= 1: - if msg.args[0] in self.modules and self.modules[msg.args[0]]() is not None: - if hasattr(self.modules[msg.args[0]](), "help_full"): - hlp = self.modules[msg.args[0]]().help_full() + if "nemubot.module." + msg.args[0] in self.modules and self.modules["nemubot.module." + msg.args[0]]() is not None: + mname = "nemubot.module." + msg.args[0] + if hasattr(self.modules[mname](), "help_full"): + hlp = self.modules[mname]().help_full() if isinstance(hlp, Response): return hlp else: res.append_message(hlp) else: - res.append_message([str(h) for s,h in self.modules[msg.args[0]]().__nemubot_context__.hooks], title="Available commands for module " + msg.args[0]) + res.append_message([str(h) for s,h in self.modules[mname]().__nemubot_context__.hooks], title="Available commands for module " + msg.args[0]) elif msg.args[0][0] == "!": from nemubot.message.command import Command for h in self.treater._in_hooks(Command(msg.args[0][1:])): diff --git a/nemubot/importer.py b/nemubot/importer.py index eaf1535..674ab40 100644 --- a/nemubot/importer.py +++ b/nemubot/importer.py @@ -29,16 +29,16 @@ class ModuleFinder(Finder): self.add_module = add_module def find_module(self, fullname, path=None): - # Search only for new nemubot modules (packages init) - if path is None: + if path is not None and fullname.startswith("nemubot.module."): + module_name = fullname.split(".", 2)[2] for mpath in self.modules_paths: - if os.path.isfile(os.path.join(mpath, fullname + ".py")): + if os.path.isfile(os.path.join(mpath, module_name + ".py")): return ModuleLoader(self.add_module, fullname, - os.path.join(mpath, fullname + ".py")) - elif os.path.isfile(os.path.join(os.path.join(mpath, fullname), "__init__.py")): + os.path.join(mpath, module_name + ".py")) + elif os.path.isfile(os.path.join(os.path.join(mpath, module_name), "__init__.py")): return ModuleLoader(self.add_module, fullname, os.path.join( - os.path.join(mpath, fullname), + os.path.join(mpath, module_name), "__init__.py")) return None @@ -53,17 +53,17 @@ class ModuleLoader(SourceFileLoader): def _load(self, module, name): # Add the module to the global modules list self.add_module(module) - logger.info("Module '%s' successfully loaded.", name) + logger.info("Module '%s' successfully imported from %s.", name.split(".", 2)[2], self.path) return module # Python 3.4 def exec_module(self, module): - super(ModuleLoader, self).exec_module(module) + super().exec_module(module) self._load(module, module.__spec__.name) # Python 3.3 def load_module(self, fullname): - module = super(ModuleLoader, self).load_module(fullname) + module = super().load_module(fullname) return self._load(module, module.__name__) diff --git a/nemubot/module/__init__.py b/nemubot/module/__init__.py new file mode 100644 index 0000000..33f0e41 --- /dev/null +++ b/nemubot/module/__init__.py @@ -0,0 +1,7 @@ +# +# This directory aims to store nemubot core modules. +# +# Custom modules should be placed into a separate directory. +# By default, this is the directory modules in your current directory. +# Use the --modules-path argument to define a custom directory for your modules. +# diff --git a/modules/more.py b/nemubot/module/more.py similarity index 100% rename from modules/more.py rename to nemubot/module/more.py diff --git a/nemubot/modulecontext.py b/nemubot/modulecontext.py index 70e4b6f..d6291c4 100644 --- a/nemubot/modulecontext.py +++ b/nemubot/modulecontext.py @@ -20,7 +20,7 @@ class _ModuleContext: self.module = module if module is not None: - self.module_name = module.__spec__.name if hasattr(module, "__spec__") else module.__name__ + self.module_name = (module.__spec__.name if hasattr(module, "__spec__") else module.__name__).replace("nemubot.module.", "") else: self.module_name = "" diff --git a/setup.py b/setup.py index 36dddb4..94c1274 100755 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ setup( 'nemubot.hooks.keywords', 'nemubot.message', 'nemubot.message.printer', + 'nemubot.module', 'nemubot.server', 'nemubot.server.message', 'nemubot.tools', From 05d20ed6ee70177da1df3286eaa708211c72d10a Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Mon, 4 Sep 2017 23:54:40 +0200 Subject: [PATCH 044/127] weather: handle units --- modules/weather.py | 68 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/modules/weather.py b/modules/weather.py index bee0d20..96fd718 100644 --- a/modules/weather.py +++ b/modules/weather.py @@ -19,6 +19,41 @@ from nemubot.module.more import Response URL_DSAPI = "https://api.darksky.net/forecast/%s/%%s,%%s?lang=%%s&units=%%s" +UNITS = { + "ca": { + "temperature": "°C", + "distance": "km", + "precipIntensity": "mm/h", + "precip": "cm", + "speed": "km/h", + "pressure": "hPa", + }, + "uk2": { + "temperature": "°C", + "distance": "mi", + "precipIntensity": "mm/h", + "precip": "cm", + "speed": "mi/h", + "pressure": "hPa", + }, + "us": { + "temperature": "°F", + "distance": "mi", + "precipIntensity": "in/h", + "precip": "in", + "speed": "mi/h", + "pressure": "mbar", + }, + "si": { + "temperature": "°C", + "distance": "km", + "precipIntensity": "mm/h", + "precip": "cm", + "speed": "m/s", + "pressure": "hPa", + }, +} + def load(context): if not context.config or "darkskyapikey" not in context.config: raise ImportError("You need a Dark-Sky API key in order to use this " @@ -30,14 +65,17 @@ def load(context): URL_DSAPI = URL_DSAPI % context.config["darkskyapikey"] -def format_wth(wth): - return ("{temperature} °C {summary}; precipitation ({precipProbability:.0%} chance) intensity: {precipIntensity} mm/h; relative humidity: {humidity:.0%}; wind speed: {windSpeed} km/s {windBearing}°; cloud coverage: {cloudCover:.0%}; pressure: {pressure} hPa; ozone: {ozone} DU" - .format(**wth) +def format_wth(wth, flags): + units = UNITS[flags["units"] if "units" in flags and flags["units"] in UNITS else "si"] + return ("{temperature} {units[temperature]} {summary}; precipitation ({precipProbability:.0%} chance) intensity: {precipIntensity} {units[precipIntensity]}; relative humidity: {humidity:.0%}; wind speed: {windSpeed} {units[speed]} {windBearing}°; cloud coverage: {cloudCover:.0%}; pressure: {pressure} {units[pressure]}; ozone: {ozone} DU" + .format(units=units, **wth) ) -def format_forecast_daily(wth): - return ("{summary}; between {temperatureMin}-{temperatureMax} °C; precipitation ({precipProbability:.0%} chance) intensity: maximum {precipIntensity} mm/h; relative humidity: {humidity:.0%}; wind speed: {windSpeed} km/h {windBearing}°; cloud coverage: {cloudCover:.0%}; pressure: {pressure} hPa; ozone: {ozone} DU".format(**wth)) +def format_forecast_daily(wth, flags): + units = UNITS[flags["units"] if "units" in flags and flags["units"] in UNITS else "si"] + print(units) + return ("{summary}; between {temperatureMin}-{temperatureMax} {units[temperature]}; precipitation ({precipProbability:.0%} chance) intensity: maximum {precipIntensity} {units[precipIntensity]}; relative humidity: {humidity:.0%}; wind speed: {windSpeed} {units[speed]} {windBearing}°; cloud coverage: {cloudCover:.0%}; pressure: {pressure} {units[pressure]}; ozone: {ozone} DU".format(units=units, **wth)) def format_timestamp(timestamp, tzname, tzoffset, format="%c"): @@ -88,7 +126,7 @@ def treat_coord(msg): raise IMException("indique-moi un nom de ville ou des coordonnées.") -def get_json_weather(coords, lang="en", units="auto"): +def get_json_weather(coords, lang="en", units="ca"): wth = web.getJSON(URL_DSAPI % (float(coords[0]), float(coords[1]), lang, units)) # First read flags @@ -114,13 +152,13 @@ def cmd_coordinates(msg): @hook.command("alert", keywords={ "lang=LANG": "change the output language of weather sumarry; default: en", - "units=UNITS": "return weather conditions in the requested units; default: auto", + "units=UNITS": "return weather conditions in the requested units; default: ca", }) def cmd_alert(msg): loc, coords, specific = treat_coord(msg) wth = get_json_weather(coords, lang=msg.kwargs["lang"] if "lang" in msg.kwargs else "en", - units=msg.kwargs["units"] if "units" in msg.kwargs else "auto") + units=msg.kwargs["units"] if "units" in msg.kwargs else "ca") res = Response(channel=msg.channel, nomore="No more weather alert", count=" (%d more alerts)") @@ -141,13 +179,13 @@ def cmd_alert(msg): }, keywords={ "lang=LANG": "change the output language of weather sumarry; default: en", - "units=UNITS": "return weather conditions in the requested units; default: auto", + "units=UNITS": "return weather conditions in the requested units; default: ca", }) def cmd_weather(msg): loc, coords, specific = treat_coord(msg) wth = get_json_weather(coords, lang=msg.kwargs["lang"] if "lang" in msg.kwargs else "en", - units=msg.kwargs["units"] if "units" in msg.kwargs else "auto") + units=msg.kwargs["units"] if "units" in msg.kwargs else "ca") res = Response(channel=msg.channel, nomore="No more weather information") @@ -169,17 +207,17 @@ def cmd_weather(msg): if gr.group(2).lower() == "h" and gr1 < len(wth["hourly"]["data"]): hour = wth["hourly"]["data"][gr1] - res.append_message("\x03\x02At %sh:\x03\x02 %s" % (format_timestamp(int(hour["time"]), wth["timezone"], wth["offset"], '%H'), format_wth(hour))) + res.append_message("\x03\x02At %sh:\x03\x02 %s" % (format_timestamp(int(hour["time"]), wth["timezone"], wth["offset"], '%H'), format_wth(hour, wth["flags"]))) elif gr.group(2).lower() == "d" and gr1 < len(wth["daily"]["data"]): day = wth["daily"]["data"][gr1] - res.append_message("\x03\x02On %s:\x03\x02 %s" % (format_timestamp(int(day["time"]), wth["timezone"], wth["offset"], '%A'), format_forecast_daily(day))) + res.append_message("\x03\x02On %s:\x03\x02 %s" % (format_timestamp(int(day["time"]), wth["timezone"], wth["offset"], '%A'), format_forecast_daily(day, wth["flags"]))) else: res.append_message("I don't understand %s or information is not available" % specific) else: - res.append_message("\x03\x02Currently:\x03\x02 " + format_wth(wth["currently"])) + res.append_message("\x03\x02Currently:\x03\x02 " + format_wth(wth["currently"], wth["flags"])) nextres = "\x03\x02Today:\x03\x02 %s " % wth["daily"]["data"][0]["summary"] if "minutely" in wth: @@ -189,11 +227,11 @@ def cmd_weather(msg): for hour in wth["hourly"]["data"][1:4]: res.append_message("\x03\x02At %sh:\x03\x02 %s" % (format_timestamp(int(hour["time"]), wth["timezone"], wth["offset"], '%H'), - format_wth(hour))) + format_wth(hour, wth["flags"]))) for day in wth["daily"]["data"][1:]: res.append_message("\x03\x02On %s:\x03\x02 %s" % (format_timestamp(int(day["time"]), wth["timezone"], wth["offset"], '%A'), - format_forecast_daily(day))) + format_forecast_daily(day, wth["flags"]))) return res From 7a4b27510c701be8bc633f075d57e94cca01d8e8 Mon Sep 17 00:00:00 2001 From: nemunaire Date: Wed, 13 Sep 2017 08:03:47 +0200 Subject: [PATCH 045/127] Replace logger by _logger in servers --- nemubot/server/DCC.py | 12 ++++++------ nemubot/server/IRC.py | 6 +++--- nemubot/server/abstract.py | 9 +++++---- nemubot/server/socket.py | 10 +++++----- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/nemubot/server/DCC.py b/nemubot/server/DCC.py index c1a6852..f5d4b8f 100644 --- a/nemubot/server/DCC.py +++ b/nemubot/server/DCC.py @@ -53,7 +53,7 @@ class DCC(server.AbstractServer): self.port = self.foundPort() if self.port is None: - self.logger.critical("No more available slot for DCC connection") + self._logger.critical("No more available slot for DCC connection") self.setError("Il n'y a plus de place disponible sur le serveur" " pour initialiser une session DCC.") @@ -79,7 +79,7 @@ class DCC(server.AbstractServer): self.s = socket.socket() try: self.s.connect((host, port)) - self.logger.info("Accepted user from %s:%d for %s", host, port, self.sender) + self._logger.info("Accepted user from %s:%d for %s", host, port, self.sender) self.connected = True self.stop = False except: @@ -104,7 +104,7 @@ class DCC(server.AbstractServer): self.setError("Une erreur s'est produite durant la tentative" " d'ouverture d'une session DCC.") return False - self.logger.info("Listening on %d for %s", self.port, self.sender) + self._logger.info("Listening on %d for %s", self.port, self.sender) #Send CTCP request for DCC self.srv.send_ctcp(self.sender, @@ -115,7 +115,7 @@ class DCC(server.AbstractServer): s.listen(1) #Waiting for the client (self.s, addr) = s.accept() - self.logger.info("Connected by %d", addr) + self._logger.info("Connected by %d", addr) self.connected = True return True @@ -149,7 +149,7 @@ class DCC(server.AbstractServer): except RuntimeError: pass else: - self.logger.error("File not found `%s'", filename) + self._logger.error("File not found `%s'", filename) def run(self): self.stopping.clear() @@ -202,7 +202,7 @@ class DCC(server.AbstractServer): if self.realname in self.srv.dcc_clients: del self.srv.dcc_clients[self.realname] - self.logger.info("Closing connection with %s", self.nick) + self._logger.info("Closing connection with %s", self.nick) self.stopping.set() if self.closing_event is not None: self.closing_event() diff --git a/nemubot/server/IRC.py b/nemubot/server/IRC.py index 7adc484..6b90bbb 100644 --- a/nemubot/server/IRC.py +++ b/nemubot/server/IRC.py @@ -84,13 +84,13 @@ class _IRC: except: return "ERRMSG invalid parameters provided as DCC CTCP request" - self.logger.info("Receive DCC connection request from %s to %s:%d", conn.sender, ip, port) + self._logger.info("Receive DCC connection request from %s to %s:%d", conn.sender, ip, port) if conn.accept_user(ip, port): srv.dcc_clients[conn.sender] = conn conn.send_dcc("Hello %s!" % conn.nick) else: - self.logger.error("DCC: unable to connect to %s:%d", ip, port) + self._logger.error("DCC: unable to connect to %s:%d", ip, port) return "ERRMSG unable to connect to %s:%d" % (ip, port) import nemubot @@ -109,7 +109,7 @@ class _IRC: # TODO: Temporary fix, waiting for hook based CTCP management self.ctcp_capabilities["TYPING"] = lambda msg, cmds: None - self.logger.debug("CTCP capabilities setup: %s", ", ".join(self.ctcp_capabilities)) + self._logger.debug("CTCP capabilities setup: %s", ", ".join(self.ctcp_capabilities)) # Register hooks on some IRC CMD diff --git a/nemubot/server/abstract.py b/nemubot/server/abstract.py index fd25c2d..814461c 100644 --- a/nemubot/server/abstract.py +++ b/nemubot/server/abstract.py @@ -32,10 +32,11 @@ class AbstractServer: """ self._name = name + self._socket = socket super().__init__(**kwargs) - self.logger = logging.getLogger("nemubot.server." + str(self.name)) + self._logger = logging.getLogger("nemubot.server." + str(self.name)) self._readbuffer = b'' self._sending_queue = queue.Queue() @@ -53,7 +54,7 @@ class AbstractServer: def connect(self, *args, **kwargs): """Register the server in _poll""" - self.logger.info("Opening connection") + self._logger.info("Opening connection") super().connect(*args, **kwargs) @@ -66,7 +67,7 @@ class AbstractServer: def close(self, *args, **kwargs): """Unregister the server from _poll""" - self.logger.info("Closing connection") + self._logger.info("Closing connection") if self.fileno() > 0: sync_act("sckt", "unregister", self.fileno()) @@ -84,7 +85,7 @@ class AbstractServer: """ self._sending_queue.put(self.format(message)) - self.logger.debug("Message '%s' appended to write queue", message) + self._logger.debug("Message '%s' appended to write queue", message) sync_act("sckt", "write", self.fileno()) diff --git a/nemubot/server/socket.py b/nemubot/server/socket.py index 8a0950c..a803bb2 100644 --- a/nemubot/server/socket.py +++ b/nemubot/server/socket.py @@ -90,7 +90,7 @@ class _SocketServer(_Socket): def connect(self): - self.logger.info("Connection to %s:%d", *self._sockaddr[:2]) + self._logger.info("Connection to %s:%d", *self._sockaddr[:2]) super().connect(self._sockaddr) if self._bind: @@ -114,7 +114,7 @@ class UnixSocket: def connect(self): - self.logger.info("Connection to unix://%s", self._socket_path) + self._logger.info("Connection to unix://%s", self._socket_path) super().connect(self._socket_path) @@ -136,7 +136,7 @@ class _Listener: def read(self): conn, addr = self.accept() fileno = conn.fileno() - self.logger.info("Accept new connection from %s (fd=%d)", addr, fileno) + self._logger.info("Accept new connection from %s (fd=%d)", addr, fileno) ss = self._instanciate(name=self.name + "#" + str(fileno), fileno=conn.detach()) ss.connect = ss._on_connect @@ -152,7 +152,7 @@ class UnixSocketListener(_Listener, UnixSocket, _Socket, socket.socket): def connect(self): - self.logger.info("Creating Unix socket at unix://%s", self._socket_path) + self._logger.info("Creating Unix socket at unix://%s", self._socket_path) try: os.remove(self._socket_path) @@ -161,7 +161,7 @@ class UnixSocketListener(_Listener, UnixSocket, _Socket, socket.socket): self.bind(self._socket_path) self.listen(5) - self.logger.info("Socket ready for accepting new connections") + self._logger.info("Socket ready for accepting new connections") self._on_connect() From 12ddf40ef4e05b8b20e4f9826ac127af921734d4 Mon Sep 17 00:00:00 2001 From: nemunaire Date: Sat, 16 Sep 2017 19:32:58 +0200 Subject: [PATCH 046/127] servers: use proxy design pattern instead of inheritance, because Python ssl patch has benn refused --- README.md | 2 -- nemubot/server/IRC.py | 13 +++------ nemubot/server/__init__.py | 28 +++++++++---------- nemubot/server/abstract.py | 32 ++++++++++++---------- nemubot/server/socket.py | 55 ++++++++++++++++---------------------- 5 files changed, 57 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 1d40faf..aa3b141 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,6 @@ Requirements *nemubot* requires at least Python 3.3 to work. -Connecting to SSL server requires [this patch](http://bugs.python.org/issue27629). - Some modules (like `cve`, `nextstop` or `laposte`) require the [BeautifulSoup module](http://www.crummy.com/software/BeautifulSoup/), but the core and framework has no dependency. diff --git a/nemubot/server/IRC.py b/nemubot/server/IRC.py index 6b90bbb..2096a63 100644 --- a/nemubot/server/IRC.py +++ b/nemubot/server/IRC.py @@ -21,10 +21,10 @@ import socket from nemubot.channel import Channel from nemubot.message.printer.IRC import IRC as IRCPrinter from nemubot.server.message.IRC import IRC as IRCMessage -from nemubot.server.socket import SocketServer, SecureSocketServer +from nemubot.server.socket import SocketServer -class _IRC: +class IRC(SocketServer): """Concrete implementation of a connexion to an IRC server""" @@ -245,7 +245,7 @@ class _IRC: def close(self): - if not self._closed: + if not self._fd._closed: self.write("QUIT") return super().close() @@ -274,10 +274,3 @@ class _IRC: def subparse(self, orig, cnt): msg = IRCMessage(("@time=%s :%s!user@host.com PRIVMSG %s :%s" % (orig.date.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), orig.frm, ",".join(orig.to), cnt)).encode(self.encoding), self.encoding) return msg.to_bot_message(self) - - -class IRC(_IRC, SocketServer): - pass - -class IRC_secure(_IRC, SecureSocketServer): - pass diff --git a/nemubot/server/__init__.py b/nemubot/server/__init__.py index a533491..068d152 100644 --- a/nemubot/server/__init__.py +++ b/nemubot/server/__init__.py @@ -32,16 +32,6 @@ def factory(uri, ssl=False, **init_args): if o.username is not None: args["username"] = o.username if o.password is not None: args["password"] = o.password - if ssl: - try: - from ssl import create_default_context - args["_context"] = create_default_context() - except ImportError: - # Python 3.3 compat - from ssl import SSLContext, PROTOCOL_TLSv1 - args["_context"] = SSLContext(PROTOCOL_TLSv1) - args["server_hostname"] = o.hostname - modifiers = o.path.split(",") target = unquote(modifiers.pop(0)[1:]) @@ -68,11 +58,19 @@ def factory(uri, ssl=False, **init_args): if "channels" not in args and "isnick" not in modifiers: args["channels"] = [ target ] + from nemubot.server.IRC import IRC as IRCServer + srv = IRCServer(**args) + if ssl: - from nemubot.server.IRC import IRC_secure as SecureIRCServer - srv = SecureIRCServer(**args) - else: - from nemubot.server.IRC import IRC as IRCServer - srv = IRCServer(**args) + try: + from ssl import create_default_context + context = create_default_context() + except ImportError: + # Python 3.3 compat + from ssl import SSLContext, PROTOCOL_TLSv1 + context = SSLContext(PROTOCOL_TLSv1) + from ssl import wrap_socket + srv._fd = context.wrap_socket(srv._fd, server_hostname=o.hostname) + return srv diff --git a/nemubot/server/abstract.py b/nemubot/server/abstract.py index 814461c..433068d 100644 --- a/nemubot/server/abstract.py +++ b/nemubot/server/abstract.py @@ -24,17 +24,16 @@ class AbstractServer: """An abstract server: handle communication with an IM server""" - def __init__(self, name=None, **kwargs): + def __init__(self, name, fdClass, **kwargs): """Initialize an abstract server Keyword argument: name -- Identifier of the socket, for convinience + fdClass -- Class to instantiate as support file """ self._name = name - self._socket = socket - - super().__init__(**kwargs) + self._fd = fdClass(**kwargs) self._logger = logging.getLogger("nemubot.server." + str(self.name)) self._readbuffer = b'' @@ -46,7 +45,7 @@ class AbstractServer: if self._name is not None: return self._name else: - return self.fileno() + return self._fd.fileno() # Open/close @@ -56,12 +55,12 @@ class AbstractServer: self._logger.info("Opening connection") - super().connect(*args, **kwargs) + self._fd.connect(*args, **kwargs) self._on_connect() def _on_connect(self): - sync_act("sckt", "register", self.fileno()) + sync_act("sckt", "register", self._fd.fileno()) def close(self, *args, **kwargs): @@ -69,10 +68,10 @@ class AbstractServer: self._logger.info("Closing connection") - if self.fileno() > 0: - sync_act("sckt", "unregister", self.fileno()) + if self._fd.fileno() > 0: + sync_act("sckt", "unregister", self._fd.fileno()) - super().close(*args, **kwargs) + self._fd.close(*args, **kwargs) # Writes @@ -86,14 +85,14 @@ class AbstractServer: self._sending_queue.put(self.format(message)) self._logger.debug("Message '%s' appended to write queue", message) - sync_act("sckt", "write", self.fileno()) + sync_act("sckt", "write", self._fd.fileno()) def async_write(self): """Internal function used when the file descriptor is writable""" try: - sync_act("sckt", "unwrite", self.fileno()) + sync_act("sckt", "unwrite", self._fd.fileno()) while not self._sending_queue.empty(): self._write(self._sending_queue.get_nowait()) self._sending_queue.task_done() @@ -131,7 +130,7 @@ class AbstractServer: A list of fully received messages """ - ret, self._readbuffer = self.lex(self._readbuffer + self.read()) + ret, self._readbuffer = self.lex(self._readbuffer + self._fd.read()) for r in ret: yield r @@ -159,4 +158,9 @@ class AbstractServer: def exception(self, flags): """Exception occurs on fd""" - self.close() + self._fd.close() + + # Proxy + + def fileno(self): + return self._fd.fileno() diff --git a/nemubot/server/socket.py b/nemubot/server/socket.py index a803bb2..9cf7c18 100644 --- a/nemubot/server/socket.py +++ b/nemubot/server/socket.py @@ -16,7 +16,6 @@ import os import socket -import ssl import nemubot.message as message from nemubot.message.printer.socket import Socket as SocketPrinter @@ -40,7 +39,7 @@ class _Socket(AbstractServer): # Write def _write(self, cnt): - self.sendall(cnt) + self._fd.sendall(cnt) def format(self, txt): @@ -52,8 +51,8 @@ class _Socket(AbstractServer): # Read - def recv(self, n=1024): - return super().recv(n) + def recv(self, *args, **kwargs): + return self._fd.recv(*args, **kwargs) def parse(self, line): @@ -67,7 +66,7 @@ class _Socket(AbstractServer): args = line.split(' ') if len(args): - yield message.Command(cmd=args[0], args=args[1:], server=self.fileno(), to=["you"], frm="you") + yield message.Command(cmd=args[0], args=args[1:], server=self._fd.fileno(), to=["you"], frm="you") def subparse(self, orig, cnt): @@ -78,50 +77,46 @@ class _Socket(AbstractServer): yield m -class _SocketServer(_Socket): +class SocketServer(_Socket): def __init__(self, host, port, bind=None, **kwargs): - (family, type, proto, canonname, sockaddr) = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)[0] + (family, type, proto, canonname, self._sockaddr) = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)[0] - super().__init__(family=family, type=type, proto=proto, **kwargs) + super().__init__(fdClass=socket.socket, family=family, type=type, proto=proto, **kwargs) - self._sockaddr = sockaddr self._bind = bind def connect(self): - self._logger.info("Connection to %s:%d", *self._sockaddr[:2]) + self._logger.info("Connecting to %s:%d", *self._sockaddr[:2]) super().connect(self._sockaddr) + self._logger.info("Connected to %s:%d", *self._sockaddr[:2]) if self._bind: - super().bind(self._bind) - - -class SocketServer(_SocketServer, socket.socket): - pass - - -class SecureSocketServer(_SocketServer, ssl.SSLSocket): - pass + self._fd.bind(self._bind) class UnixSocket: def __init__(self, location, **kwargs): - super().__init__(family=socket.AF_UNIX, **kwargs) + super().__init__(fdClass=socket.socket, family=socket.AF_UNIX, **kwargs) self._socket_path = location def connect(self): self._logger.info("Connection to unix://%s", self._socket_path) - super().connect(self._socket_path) + self.connect(self._socket_path) -class SocketClient(_Socket, socket.socket): +class SocketClient(_Socket): + + def __init__(self, **kwargs): + super().__init__(fdClass=socket.socket, **kwargs) + def read(self): - return self.recv() + return self._fd.recv() class _Listener: @@ -134,7 +129,7 @@ class _Listener: def read(self): - conn, addr = self.accept() + conn, addr = self._fd.accept() fileno = conn.fileno() self._logger.info("Accept new connection from %s (fd=%d)", addr, fileno) @@ -145,11 +140,7 @@ class _Listener: return b'' -class UnixSocketListener(_Listener, UnixSocket, _Socket, socket.socket): - - def __init__(self, **kwargs): - super().__init__(**kwargs) - +class UnixSocketListener(_Listener, UnixSocket, _Socket): def connect(self): self._logger.info("Creating Unix socket at unix://%s", self._socket_path) @@ -159,8 +150,8 @@ class UnixSocketListener(_Listener, UnixSocket, _Socket, socket.socket): except FileNotFoundError: pass - self.bind(self._socket_path) - self.listen(5) + self._fd.bind(self._socket_path) + self._fd.listen(5) self._logger.info("Socket ready for accepting new connections") self._on_connect() @@ -171,7 +162,7 @@ class UnixSocketListener(_Listener, UnixSocket, _Socket, socket.socket): import socket try: - self.shutdown(socket.SHUT_RDWR) + self._fd.shutdown(socket.SHUT_RDWR) except socket.error: pass From 62cd92e1cbea9b8876634799afc14bb15995703c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 17 Sep 2017 18:09:35 +0200 Subject: [PATCH 047/127] server: Rework factory tests --- nemubot/server/factory_test.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/nemubot/server/factory_test.py b/nemubot/server/factory_test.py index 358591e..efc3130 100644 --- a/nemubot/server/factory_test.py +++ b/nemubot/server/factory_test.py @@ -22,30 +22,32 @@ class TestFactory(unittest.TestCase): def test_IRC1(self): from nemubot.server.IRC import IRC as IRCServer - from nemubot.server.IRC import IRC_secure as IRCSServer + import socket + import ssl # : If omitted, the client must connect to a prespecified default IRC server. server = factory("irc:///") self.assertIsInstance(server, IRCServer) - self.assertEqual(server.host, "localhost") + self.assertIsInstance(server._fd, socket.socket) + self.assertIn(server._sockaddr[0], ["127.0.0.1", "::1"]) - server = factory("ircs:///") - self.assertIsInstance(server, IRCSServer) - self.assertEqual(server.host, "localhost") - - server = factory("irc://host1") + server = factory("irc://2.2.2.2") self.assertIsInstance(server, IRCServer) - self.assertEqual(server.host, "host1") + self.assertEqual(server._sockaddr[0], "2.2.2.2") - server = factory("irc://host2:6667") + server = factory("ircs://1.2.1.2") self.assertIsInstance(server, IRCServer) - self.assertEqual(server.host, "host2") - self.assertEqual(server.port, 6667) + self.assertIsInstance(server._fd, ssl.SSLSocket) - server = factory("ircs://host3:194/") - self.assertIsInstance(server, IRCSServer) - self.assertEqual(server.host, "host3") - self.assertEqual(server.port, 194) + server = factory("irc://1.2.3.4:6667") + self.assertIsInstance(server, IRCServer) + self.assertEqual(server._sockaddr[0], "1.2.3.4") + self.assertEqual(server._sockaddr[1], 6667) + + server = factory("ircs://4.3.2.1:194/") + self.assertIsInstance(server, IRCServer) + self.assertEqual(server._sockaddr[0], "4.3.2.1") + self.assertEqual(server._sockaddr[1], 194) if __name__ == '__main__': From 28d4e507eb9e53654e4516fb298e2110cd9d69e2 Mon Sep 17 00:00:00 2001 From: nemunaire Date: Mon, 25 Sep 2017 23:56:28 +0200 Subject: [PATCH 048/127] servers: call recv late --- nemubot/server/abstract.py | 2 +- nemubot/server/socket.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/nemubot/server/abstract.py b/nemubot/server/abstract.py index 433068d..510a319 100644 --- a/nemubot/server/abstract.py +++ b/nemubot/server/abstract.py @@ -130,7 +130,7 @@ class AbstractServer: A list of fully received messages """ - ret, self._readbuffer = self.lex(self._readbuffer + self._fd.read()) + ret, self._readbuffer = self.lex(self._readbuffer + self.read()) for r in ret: yield r diff --git a/nemubot/server/socket.py b/nemubot/server/socket.py index 9cf7c18..a6be620 100644 --- a/nemubot/server/socket.py +++ b/nemubot/server/socket.py @@ -51,8 +51,8 @@ class _Socket(AbstractServer): # Read - def recv(self, *args, **kwargs): - return self._fd.recv(*args, **kwargs) + def read(self, bufsize=1024, *args, **kwargs): + return self._fd.recv(bufsize, *args, **kwargs) def parse(self, line): @@ -115,10 +115,6 @@ class SocketClient(_Socket): super().__init__(fdClass=socket.socket, **kwargs) - def read(self): - return self._fd.recv() - - class _Listener: def __init__(self, new_server_cb, instanciate=SocketClient, **kwargs): From 30ec9121628950c9b2be4f60ef1e374a27362724 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 26 Sep 2017 19:35:31 +0200 Subject: [PATCH 049/127] daemonize: fork client before loading context --- nemubot/__init__.py | 23 ++++++----------------- nemubot/__main__.py | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/nemubot/__init__.py b/nemubot/__init__.py index 48de6ea..62807c6 100644 --- a/nemubot/__init__.py +++ b/nemubot/__init__.py @@ -39,10 +39,14 @@ def requires_version(min=None, max=None): "but this is nemubot v%s." % (str(max), __version__)) -def attach(pid, socketfile): +def attach(pidfile, socketfile): import socket import sys + # Read PID from pidfile + with open(pidfile, "r") as f: + pid = int(f.readline()) + print("nemubot is launched with PID %d. Attaching to Unix socket at: %s" % (pid, socketfile)) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) @@ -106,28 +110,13 @@ def attach(pid, socketfile): return 0 -def daemonize(socketfile=None, autoattach=True): +def daemonize(socketfile=None): """Detach the running process to run as a daemon """ import os import sys - if socketfile is not None: - try: - pid = os.fork() - if pid > 0: - if autoattach: - import time - os.waitpid(pid, 0) - time.sleep(1) - sys.exit(attach(pid, socketfile)) - else: - sys.exit(0) - except OSError as err: - sys.stderr.write("Unable to fork: %s\n" % err) - sys.exit(1) - try: pid = os.fork() if pid > 0: diff --git a/nemubot/__main__.py b/nemubot/__main__.py index b79d90e..abb290b 100644 --- a/nemubot/__main__.py +++ b/nemubot/__main__.py @@ -77,6 +77,20 @@ def main(): args.files = [x for x in map(os.path.abspath, args.files)] args.modules_path = [x for x in map(os.path.abspath, args.modules_path)] + # Prepare the attached client, before setting other stuff + if not args.debug and not args.no_attach and args.socketfile is not None and args.pidfile is not None: + try: + pid = os.fork() + if pid > 0: + import time + os.waitpid(pid, 0) + time.sleep(1) + from nemubot import attach + sys.exit(attach(args.pidfile, args.socketfile)) + except OSError as err: + sys.stderr.write("Unable to fork: %s\n" % err) + sys.exit(1) + # Setup logging interface import logging logger = logging.getLogger("nemubot") @@ -106,7 +120,7 @@ def main(): pass else: from nemubot import attach - sys.exit(attach(pid, args.socketfile)) + sys.exit(attach(args.pidfile, args.socketfile)) # Add modules dir paths modules_paths = list() @@ -175,7 +189,7 @@ def main(): # Daemonize if not args.debug: from nemubot import daemonize - daemonize(args.socketfile, not args.no_attach) + daemonize(args.socketfile) # Signals handling def sigtermhandler(signum, frame): From 5646850df11be31a9b0e98e2a7bc20348a0e922d Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 26 Sep 2017 23:51:48 +0200 Subject: [PATCH 050/127] Don't launch timer thread before bot launch --- nemubot/bot.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/nemubot/bot.py b/nemubot/bot.py index d2e042c..c9c0e66 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -57,7 +57,7 @@ class Bot(threading.Thread): sys.version_info.major, sys.version_info.minor, sys.version_info.micro) self.debug = debug - self.stop = None + self.stop = True # External IP for accessing this bot import ipaddress @@ -164,8 +164,13 @@ class Bot(threading.Thread): self._poll.register(sync_queue._reader, select.POLLIN | select.POLLPRI) - logger.info("Starting main loop") + self.stop = False + + # Relaunch events + self._update_event_timer() + + logger.info("Starting main loop") while not self.stop: for fd, flag in self._poll.poll(): # Handle internal socket passing orders @@ -256,10 +261,6 @@ class Bot(threading.Thread): module_src -- The module to which the event is attached to """ - if hasattr(self, "stop") and self.stop: - logger.warn("The bot is stopped, can't register new events") - return - import uuid # Generate the event id if no given @@ -286,7 +287,7 @@ class Bot(threading.Thread): break self.events.insert(i, evt) - if i == 0: + if i == 0 and not self.stop: # First event changed, reset timer self._update_event_timer() if len(self.events) <= 0 or self.events[i] != evt: @@ -417,10 +418,6 @@ class Bot(threading.Thread): old one before""" module_name = module.__spec__.name if hasattr(module, "__spec__") else module.__name__ - if hasattr(self, "stop") and self.stop: - logger.warn("The bot is stopped, can't register new modules") - return - # Check if the module already exists if module_name in self.modules: self.unload_module(module_name) From b15d18b3a5c7b0c42e1a8d382c60fdc371a3419a Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 27 Sep 2017 00:13:26 +0200 Subject: [PATCH 051/127] events: fix event removal --- nemubot/bot.py | 8 ++++---- nemubot/consumer.py | 2 +- nemubot/modulecontext.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nemubot/bot.py b/nemubot/bot.py index c9c0e66..28df8ce 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -296,7 +296,7 @@ class Bot(threading.Thread): # Register the event in the source module if module_src is not None: - module_src.__nemubot_context__.events.append(evt.id) + module_src.__nemubot_context__.events.append((evt, evt.id)) evt.module_src = module_src logger.info("New event registered in %d position: %s", i, t) @@ -326,10 +326,10 @@ class Bot(threading.Thread): id = evt if len(self.events) > 0 and id == self.events[0].id: + if module_src is not None: + module_src.__nemubot_context__.events.remove((self.events[0], id)) self.events.remove(self.events[0]) self._update_event_timer() - if module_src is not None: - module_src.__nemubot_context__.events.remove(id) return True for evt in self.events: @@ -337,7 +337,7 @@ class Bot(threading.Thread): self.events.remove(evt) if module_src is not None: - module_src.__nemubot_context__.events.remove(evt.id) + module_src.__nemubot_context__.events.remove((evt, evt.id)) return True return False diff --git a/nemubot/consumer.py b/nemubot/consumer.py index 2765aff..3a58219 100644 --- a/nemubot/consumer.py +++ b/nemubot/consumer.py @@ -94,7 +94,7 @@ class EventConsumer: # Or remove reference of this event elif (hasattr(self.evt, "module_src") and self.evt.module_src is not None): - self.evt.module_src.__nemubot_context__.events.remove(self.evt.id) + self.evt.module_src.__nemubot_context__.events.remove((self.evt, self.evt.id)) diff --git a/nemubot/modulecontext.py b/nemubot/modulecontext.py index d6291c4..f39934c 100644 --- a/nemubot/modulecontext.py +++ b/nemubot/modulecontext.py @@ -85,7 +85,7 @@ class _ModuleContext: self.del_hook(h, *s) # Remove registered events - for evt, eid, module_src in self.events: + for evt, eid in self.events: self.del_event(evt) self.save() From bb0e958118f917907553075426c7d004dac2306c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 3 Oct 2017 07:00:08 +0200 Subject: [PATCH 052/127] grep: allow the pattern to be empty --- modules/grep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/grep.py b/modules/grep.py index 5c25c7d..fde8ecb 100644 --- a/modules/grep.py +++ b/modules/grep.py @@ -73,7 +73,7 @@ def cmd_grep(msg): only = "only" in msg.kwargs - l = [m for m in grep(msg.args[0] if msg.args[0][0] == "^" else ".*?(" + msg.args[0] + ").*?", + l = [m for m in grep(msg.args[0] if len(msg.args[0]) and msg.args[0][0] == "^" else ".*?(" + msg.args[0] + ").*?", " ".join(msg.args[1:]), msg, icase="nocase" in msg.kwargs, From c23dc22ce234fc9c04eac3376fded293e0db97e0 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 26 Sep 2017 02:21:14 +0200 Subject: [PATCH 053/127] [suivi] Fix colissimo tracking --- modules/suivi.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/modules/suivi.py b/modules/suivi.py index 4bc079e..1d04333 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -31,16 +31,17 @@ def get_tnt_info(track_id): def get_colissimo_info(colissimo_id): - colissimo_data = getURLContent("http://www.colissimo.fr/portail_colissimo/suivre.do?colispart=%s" % colissimo_id) + colissimo_data = getURLContent("https://www.laposte.fr/particulier/outils/suivre-vos-envois?code=%s" % colissimo_id) soup = BeautifulSoup(colissimo_data) - dataArray = soup.find(class_='dataArray') - if dataArray and dataArray.tbody and dataArray.tbody.tr: - date = dataArray.tbody.tr.find(headers="Date").get_text() - libelle = re.sub(r'[\n\t\r]', '', - dataArray.tbody.tr.find(headers="Libelle").get_text()) - site = dataArray.tbody.tr.find(headers="site").get_text().strip() - return (date, libelle, site.strip()) + dataArray = soup.find(class_='results-suivi') + if dataArray and dataArray.table and dataArray.table.tbody and dataArray.table.tbody.tr: + td = dataArray.table.tbody.tr.find_all('td') + if len(td) > 2: + date = td[0].get_text() + libelle = re.sub(r'[\n\t\r]', '', td[1].get_text()) + site = td[2].get_text().strip() + return (date, libelle, site.strip()) def get_chronopost_info(track_id): From a7f4ccc959565701f3f0a46aab70980fa69176d0 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 26 Sep 2017 02:25:14 +0200 Subject: [PATCH 054/127] [suivi] Fix Fedex tracking --- modules/suivi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/suivi.py b/modules/suivi.py index 1d04333..100f98b 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -176,7 +176,8 @@ def get_fedex_info(fedex_id, lang="en_US"): if ("TrackPackagesResponse" in fedex_data and "packageList" in fedex_data["TrackPackagesResponse"] and len(fedex_data["TrackPackagesResponse"]["packageList"]) and - not fedex_data["TrackPackagesResponse"]["errorList"][0]["code"] and + (not fedex_data["TrackPackagesResponse"]["errorList"][0]["code"] or + fedex_data["TrackPackagesResponse"]["errorList"][0]["code"] == '0') and not fedex_data["TrackPackagesResponse"]["packageList"][0]["errorList"][0]["code"] ): return fedex_data["TrackPackagesResponse"]["packageList"][0] From ef4c119f1f4aea6ebe3eeb2f7339fa6619c3f7e6 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 26 Sep 2017 02:30:14 +0200 Subject: [PATCH 055/127] [suivi] Add https when supported by service --- modules/suivi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/suivi.py b/modules/suivi.py index 100f98b..27a939a 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -17,7 +17,7 @@ from nemubot.module.more import Response def get_tnt_info(track_id): values = [] - data = getURLContent('www.tnt.fr/public/suivi_colis/recherche/visubontransport.do?bonTransport=%s' % track_id) + data = getURLContent('https://www.tnt.fr/public/suivi_colis/recherche/visubontransport.do?bonTransport=%s' % track_id) soup = BeautifulSoup(data) status_list = soup.find('div', class_='result__content') if not status_list: @@ -46,7 +46,7 @@ def get_colissimo_info(colissimo_id): def get_chronopost_info(track_id): data = urllib.parse.urlencode({'listeNumeros': track_id}) - track_baseurl = "http://www.chronopost.fr/expedier/inputLTNumbersNoJahia.do?lang=fr_FR" + track_baseurl = "https://www.chronopost.fr/expedier/inputLTNumbersNoJahia.do?lang=fr_FR" track_data = getURLContent(track_baseurl, data.encode('utf-8')) soup = BeautifulSoup(track_data) From 99384ad6f7b4de313d130b297f86635080a93ad8 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 28 Sep 2017 02:30:14 +0200 Subject: [PATCH 056/127] [suivi] Fix awkward USPS message --- modules/suivi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/suivi.py b/modules/suivi.py index 27a939a..637f64f 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -230,7 +230,7 @@ def handle_usps(tracknum): info = get_usps_info(tracknum) if info: notif, last_date, last_status, last_location = info - return ("USPS \x02{tracknum}\x0F is {last_status} in \x02{last_location}\x0F as of {last_date}: {notif}".format(tracknum=tracknum, notif=notif, last_date=last_date, last_status=last_status.lower(), last_location=last_location)) + return ("USPS \x02{tracknum}\x0F: {last_status} in \x02{last_location}\x0F as of {last_date}: {notif}".format(tracknum=tracknum, notif=notif, last_date=last_date, last_status=last_status.lower(), last_location=last_location)) def handle_colissimo(tracknum): From e0d7ef13146b78edc4c8a3f7cc2e7d343f8515b5 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 6 Oct 2017 02:54:37 +0200 Subject: [PATCH 057/127] Fix https links when available, everywhere --- README.md | 2 +- modules/conjugaison.py | 2 +- modules/mapquest.py | 4 ++-- modules/networking/isup.py | 2 +- modules/networking/w3c.py | 2 +- modules/networking/whois.py | 4 ++-- modules/reddit.py | 2 +- modules/sap.py | 2 +- modules/syno.py | 2 +- modules/urbandict.py | 2 +- modules/virtualradar.py | 2 +- modules/weather.py | 2 +- modules/wolframalpha.py | 4 ++-- nemubot/bot.py | 2 +- nemubot/server/__init__.py | 4 ++-- 15 files changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index aa3b141..6977c9f 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Requirements *nemubot* requires at least Python 3.3 to work. Some modules (like `cve`, `nextstop` or `laposte`) require the -[BeautifulSoup module](http://www.crummy.com/software/BeautifulSoup/), +[BeautifulSoup module](https://www.crummy.com/software/BeautifulSoup/), but the core and framework has no dependency. diff --git a/modules/conjugaison.py b/modules/conjugaison.py index 42d78c6..c953da3 100644 --- a/modules/conjugaison.py +++ b/modules/conjugaison.py @@ -36,7 +36,7 @@ for k, v in s: # MODULE CORE ######################################################### def get_conjug(verb, stringTens): - url = ("http://leconjugueur.lefigaro.fr/conjugaison/verbe/%s.html" % + url = ("https://leconjugueur.lefigaro.fr/conjugaison/verbe/%s.html" % quote(verb.encode("ISO-8859-1"))) page = web.getURLContent(url) diff --git a/modules/mapquest.py b/modules/mapquest.py index 5662a49..f328e1d 100644 --- a/modules/mapquest.py +++ b/modules/mapquest.py @@ -13,7 +13,7 @@ from nemubot.module.more import Response # GLOBALS ############################################################# -URL_API = "http://open.mapquestapi.com/geocoding/v1/address?key=%s&location=%%s" +URL_API = "https://open.mapquestapi.com/geocoding/v1/address?key=%s&location=%%s" # LOADING ############################################################# @@ -23,7 +23,7 @@ def load(context): raise ImportError("You need a MapQuest API key in order to use this " "module. Add it to the module configuration file:\n" "\nRegister at http://developer.mapquest.com/") + "/>\nRegister at https://developer.mapquest.com/") global URL_API URL_API = URL_API % context.config["apikey"].replace("%", "%%") diff --git a/modules/networking/isup.py b/modules/networking/isup.py index c518900..99e2664 100644 --- a/modules/networking/isup.py +++ b/modules/networking/isup.py @@ -11,7 +11,7 @@ def isup(url): o = urllib.parse.urlparse(getNormalizedURL(url), "http") if o.netloc != "": - isup = getJSON("http://isitup.org/%s.json" % o.netloc) + isup = getJSON("https://isitup.org/%s.json" % o.netloc) if isup is not None and "status_code" in isup and isup["status_code"] == 1: return isup["response_time"] diff --git a/modules/networking/w3c.py b/modules/networking/w3c.py index 83056dd..3c8084f 100644 --- a/modules/networking/w3c.py +++ b/modules/networking/w3c.py @@ -17,7 +17,7 @@ def validator(url): raise IMException("Indicate a valid URL!") try: - req = urllib.request.Request("http://validator.w3.org/check?uri=%s&output=json" % (urllib.parse.quote(o.geturl())), headers={ 'User-Agent' : "Nemubot v%s" % __version__}) + req = urllib.request.Request("https://validator.w3.org/check?uri=%s&output=json" % (urllib.parse.quote(o.geturl())), headers={ 'User-Agent' : "Nemubot v%s" % __version__}) raw = urllib.request.urlopen(req, timeout=10) except urllib.error.HTTPError as e: raise IMException("HTTP error occurs: %s %s" % (e.code, e.reason)) diff --git a/modules/networking/whois.py b/modules/networking/whois.py index 787cd17..999dc01 100644 --- a/modules/networking/whois.py +++ b/modules/networking/whois.py @@ -9,7 +9,7 @@ from nemubot.tools.web import getJSON from nemubot.module.more import Response URL_AVAIL = "https://www.whoisxmlapi.com/whoisserver/WhoisService?cmd=GET_DN_AVAILABILITY&domainName=%%s&outputFormat=json&username=%s&password=%s" -URL_WHOIS = "http://www.whoisxmlapi.com/whoisserver/WhoisService?da=2&domainName=%%s&outputFormat=json&userName=%s&password=%s" +URL_WHOIS = "https://www.whoisxmlapi.com/whoisserver/WhoisService?da=2&domainName=%%s&outputFormat=json&userName=%s&password=%s" # LOADING ############################################################# @@ -22,7 +22,7 @@ def load(CONF, add_hook): "the !netwhois feature. Add it to the module " "configuration file:\n\nRegister at " - "http://www.whoisxmlapi.com/newaccount.php") + "https://www.whoisxmlapi.com/newaccount.php") URL_AVAIL = URL_AVAIL % (urllib.parse.quote(CONF.getNode("whoisxmlapi")["username"]), urllib.parse.quote(CONF.getNode("whoisxmlapi")["password"])) URL_WHOIS = URL_WHOIS % (urllib.parse.quote(CONF.getNode("whoisxmlapi")["username"]), urllib.parse.quote(CONF.getNode("whoisxmlapi")["password"])) diff --git a/modules/reddit.py b/modules/reddit.py index 2de7612..d4def85 100644 --- a/modules/reddit.py +++ b/modules/reddit.py @@ -40,7 +40,7 @@ def cmd_subreddit(msg): else: where = "r" - sbr = web.getJSON("http://www.reddit.com/%s/%s/about.json" % + sbr = web.getJSON("https://www.reddit.com/%s/%s/about.json" % (where, sub.group(2))) if sbr is None: diff --git a/modules/sap.py b/modules/sap.py index a6168a2..0b9017f 100644 --- a/modules/sap.py +++ b/modules/sap.py @@ -25,7 +25,7 @@ def cmd_tcode(msg): raise IMException("indicate a transaction code or " "a keyword to search!") - url = ("http://www.tcodesearch.com/tcodes/search?q=%s" % + url = ("https://www.tcodesearch.com/tcodes/search?q=%s" % urllib.parse.quote(msg.args[0])) page = web.getURLContent(url) diff --git a/modules/syno.py b/modules/syno.py index bda0456..6f6a625 100644 --- a/modules/syno.py +++ b/modules/syno.py @@ -53,7 +53,7 @@ def get_french_synos(word): def get_english_synos(key, word): - cnt = web.getJSON("http://words.bighugelabs.com/api/2/%s/%s/json" % + cnt = web.getJSON("https://words.bighugelabs.com/api/2/%s/%s/json" % (quote(key), quote(word.encode("ISO-8859-1")))) best = list(); synos = list(); anton = list() diff --git a/modules/urbandict.py b/modules/urbandict.py index a897fad..b561e89 100644 --- a/modules/urbandict.py +++ b/modules/urbandict.py @@ -14,7 +14,7 @@ from nemubot.module.more import Response def search(terms): return web.getJSON( - "http://api.urbandictionary.com/v0/define?term=%s" + "https://api.urbandictionary.com/v0/define?term=%s" % quote(' '.join(terms))) diff --git a/modules/virtualradar.py b/modules/virtualradar.py index 9382d3b..2c87e79 100644 --- a/modules/virtualradar.py +++ b/modules/virtualradar.py @@ -15,7 +15,7 @@ from nemubot.module import mapquest # GLOBALS ############################################################# -URL_API = "http://public-api.adsbexchange.com/VirtualRadar/AircraftList.json?fcallC=%s" +URL_API = "https://public-api.adsbexchange.com/VirtualRadar/AircraftList.json?fcallC=%s" SPEED_TYPES = { 0: 'Ground speed', diff --git a/modules/weather.py b/modules/weather.py index 96fd718..9b36470 100644 --- a/modules/weather.py +++ b/modules/weather.py @@ -59,7 +59,7 @@ def load(context): raise ImportError("You need a Dark-Sky API key in order to use this " "module. Add it to the module configuration file:\n" "\n" - "Register at http://developer.forecast.io/") + "Register at https://developer.forecast.io/") context.data.setIndex("name", "city") global URL_DSAPI URL_DSAPI = URL_DSAPI % context.config["darkskyapikey"] diff --git a/modules/wolframalpha.py b/modules/wolframalpha.py index b7cc7fb..fc83815 100644 --- a/modules/wolframalpha.py +++ b/modules/wolframalpha.py @@ -15,7 +15,7 @@ from nemubot.module.more import Response # LOADING ############################################################# -URL_API = "http://api.wolframalpha.com/v2/query?input=%%s&format=plaintext&appid=%s" +URL_API = "https://api.wolframalpha.com/v2/query?input=%%s&format=plaintext&appid=%s" def load(context): global URL_API @@ -24,7 +24,7 @@ def load(context): "this module. Add it to the module configuration: " "\n\n" - "Register at http://products.wolframalpha.com/api/") + "Register at https://products.wolframalpha.com/api/") URL_API = URL_API % quote(context.config["apikey"]).replace("%", "%%") diff --git a/nemubot/bot.py b/nemubot/bot.py index 28df8ce..6aa5ed6 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -135,7 +135,7 @@ class Bot(threading.Thread): "Vous pouvez le consulter, le dupliquer, " "envoyer des rapports de bogues ou bien " "contribuer au projet sur GitHub : " - "http://github.com/nemunaire/nemubot/") + "https://github.com/nemunaire/nemubot/") res.append_message(title="Pour plus de détails sur un module, " "envoyez \"!help nomdumodule\". Voici la liste" " de tous les modules disponibles localement", diff --git a/nemubot/server/__init__.py b/nemubot/server/__init__.py index 068d152..a39c491 100644 --- a/nemubot/server/__init__.py +++ b/nemubot/server/__init__.py @@ -22,8 +22,8 @@ def factory(uri, ssl=False, **init_args): srv = None if o.scheme == "irc" or o.scheme == "ircs": - # http://www.w3.org/Addressing/draft-mirashi-url-irc-01.txt - # http://www-archive.mozilla.org/projects/rt-messaging/chatzilla/irc-urls.html + # https://www.w3.org/Addressing/draft-mirashi-url-irc-01.txt + # https://www-archive.mozilla.org/projects/rt-messaging/chatzilla/irc-urls.html args = init_args if o.scheme == "ircs": ssl = True From 226ee4e34eece827a5fc91389fb144b0b46c63fc Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 11 Oct 2017 08:03:05 +0200 Subject: [PATCH 058/127] ctfs: update module reflecting site changes --- modules/ctfs.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/ctfs.py b/modules/ctfs.py index 169ee46..ac27c4a 100644 --- a/modules/ctfs.py +++ b/modules/ctfs.py @@ -25,10 +25,8 @@ def get_info_yt(msg): for line in soup.body.find_all('tr'): n = line.find_all('td') - if len(n) == 5: - try: - res.append_message("\x02%s:\x0F from %s type %s at %s. %s" % - tuple([striphtml(x.text) for x in n])) - except: - pass + if len(n) == 7: + res.append_message("\x02%s:\x0F from %s type %s at %s. Weight: %s. %s%s" % + tuple([striphtml(x.text).strip() for x in n])) + return res From 5cbad964920ea36ab2442331e5abf7d0fbfbce3c Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 31 Oct 2017 00:13:14 -0500 Subject: [PATCH 059/127] [module] RMS upcoming locations --- modules/rms.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 modules/rms.py diff --git a/modules/rms.py b/modules/rms.py new file mode 100644 index 0000000..e7b89ce --- /dev/null +++ b/modules/rms.py @@ -0,0 +1,35 @@ +"""Finding RMS""" + +# PYTHON STUFFS ####################################################### + +from bs4 import BeautifulSoup + +from nemubot.hooks import hook +from nemubot.tools.web import getURLContent, striphtml +from nemubot.module.more import Response + + +# GLOBALS ############################################################# + +URL = 'https://www.fsf.org/events/rms-speeches.html' + + +# MODULE INTERFACE #################################################### + +@hook.command("rms", + help="Lists upcoming RMS events.") +def cmd_rms(msg): + soup = BeautifulSoup(getURLContent(URL), "lxml") + + res = Response(channel=msg.channel, + nomore="", + count=" (%d more event(s))") + + search_res = soup.find("table", {'class':'listing'}) + for item in search_res.tbody.find_all('tr'): + columns = item.find_all('td') + res.append_message("RMS will be in \x02%s\x0F for \x02%s\x0F on \x02%s\x0F." % ( + columns[1].get_text(), + columns[2].get_text().replace('\n', ''), + columns[0].get_text().replace('\n', ''))) + return res From d528746cb51557ca70c61c5c2d714c9172e39e82 Mon Sep 17 00:00:00 2001 From: nemunaire Date: Sun, 20 Aug 2017 21:17:08 +0200 Subject: [PATCH 060/127] datastore: support custom knodes instead of nemubotstate --- nemubot/datastore/abstract.py | 6 +++++- nemubot/datastore/xml.py | 24 +++++++++++++++++++----- nemubot/modulecontext.py | 8 ++++++-- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/nemubot/datastore/abstract.py b/nemubot/datastore/abstract.py index 96e2c0d..aeaecc6 100644 --- a/nemubot/datastore/abstract.py +++ b/nemubot/datastore/abstract.py @@ -32,16 +32,20 @@ class Abstract: def close(self): return - def load(self, module): + def load(self, module, knodes): """Load data for the given module Argument: module -- the module name of data to load + knodes -- the schema to use to load the datas Return: The loaded data """ + if knodes is not None: + return None + return self.new() def save(self, module, data): diff --git a/nemubot/datastore/xml.py b/nemubot/datastore/xml.py index 025c0c5..aa6cbd0 100644 --- a/nemubot/datastore/xml.py +++ b/nemubot/datastore/xml.py @@ -83,27 +83,38 @@ class XML(Abstract): return os.path.join(self.basedir, module + ".xml") - def load(self, module): + def load(self, module, knodes): """Load data for the given module Argument: module -- the module name of data to load + knodes -- the schema to use to load the datas """ data_file = self._get_data_file_path(module) + if knodes is None: + from nemubot.tools.xmlparser import parse_file + def _true_load(path): + return parse_file(path) + + else: + from nemubot.tools.xmlparser import XMLParser + p = XMLParser(knodes) + def _true_load(path): + return p.parse_file(path) + # Try to load original file if os.path.isfile(data_file): - from nemubot.tools.xmlparser import parse_file try: - return parse_file(data_file) + return _true_load(data_file) except xml.parsers.expat.ExpatError: # Try to load from backup for i in range(10): path = data_file + "." + str(i) if os.path.isfile(path): try: - cnt = parse_file(path) + cnt = _true_load(path) logger.warn("Restoring from backup: %s", path) @@ -112,7 +123,7 @@ class XML(Abstract): continue # Default case: initialize a new empty datastore - return Abstract.load(self, module) + return super().load(module, knodes) def _rotate(self, path): """Backup given path @@ -143,6 +154,9 @@ class XML(Abstract): if self.rotate: self._rotate(path) + if data is None: + return + import tempfile _, tmpath = tempfile.mkstemp() with open(tmpath, "w") as f: diff --git a/nemubot/modulecontext.py b/nemubot/modulecontext.py index f39934c..c7fa3d4 100644 --- a/nemubot/modulecontext.py +++ b/nemubot/modulecontext.py @@ -16,7 +16,7 @@ class _ModuleContext: - def __init__(self, module=None): + def __init__(self, module=None, knodes=None): self.module = module if module is not None: @@ -30,12 +30,16 @@ class _ModuleContext: from nemubot.config.module import Module self.config = Module(self.module_name) + self._knodes = knodes def load_data(self): from nemubot.tools.xmlparser import module_state return module_state.ModuleState("nemubotstate") + def set_knodes(self, knodes): + self._knodes = knodes + def add_hook(self, hook, *triggers): from nemubot.hooks import Abstract as AbstractHook assert isinstance(hook, AbstractHook), hook @@ -112,7 +116,7 @@ class ModuleContext(_ModuleContext): def load_data(self): - return self.context.datastore.load(self.module_name) + return self.context.datastore.load(self.module_name, self._knodes) def add_hook(self, hook, *triggers): from nemubot.hooks import Abstract as AbstractHook From 4cd099e087ac04838de323c6b9d05d513387106f Mon Sep 17 00:00:00 2001 From: nemunaire Date: Fri, 1 Sep 2017 20:45:58 +0200 Subject: [PATCH 061/127] xmlparser: make DictNode more usable --- nemubot/tools/xmlparser/basic.py | 33 ++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/nemubot/tools/xmlparser/basic.py b/nemubot/tools/xmlparser/basic.py index 86eac3c..dadff23 100644 --- a/nemubot/tools/xmlparser/basic.py +++ b/nemubot/tools/xmlparser/basic.py @@ -77,12 +77,12 @@ class DictNode: def endElement(self, name): - if name is None or self._cur is None: + if name is not None or self._cur is None: return key, cnt = self._cur if isinstance(cnt, list) and len(cnt) == 1: - self.items[key] = cnt + self.items[key] = cnt[0] else: self.items[key] = cnt @@ -122,7 +122,32 @@ class DictNode: if isinstance(v, str): store.characters(v) else: - for i in v: - i.saveElement(store) + if hasattr(v, "__iter__"): + for i in v: + i.saveElement(store) + else: + v.saveElement(store) store.endElement("item") store.endElement(tag) + + + def __contain__(self, i): + return i in self.items + + def __getitem__(self, i): + return self.items[i] + + def __setitem__(self, i, c): + self.items[i] = c + + def __delitem__(self, k): + del self.items[k] + + def __iter__(self): + return self.items.__iter__() + + def keys(self): + return self.items.keys() + + def items(self): + return self.items.items() From c3c74847923aae6d470e4989dc880c04c96ad7d6 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 19 Jul 2017 07:51:19 +0200 Subject: [PATCH 062/127] In debug mode, display the last stack element to be able to trace --- nemubot/server/abstract.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemubot/server/abstract.py b/nemubot/server/abstract.py index 510a319..8fbb923 100644 --- a/nemubot/server/abstract.py +++ b/nemubot/server/abstract.py @@ -16,6 +16,7 @@ import logging import queue +import traceback from nemubot.bot import sync_act @@ -84,7 +85,7 @@ class AbstractServer: """ self._sending_queue.put(self.format(message)) - self._logger.debug("Message '%s' appended to write queue", message) + self._logger.debug("Message '%s' appended to write queue coming from %s:%d in %s", message, *traceback.extract_stack(limit=3)[0][:3]) sync_act("sckt", "write", self._fd.fileno()) From f520c67c8927ef4608f399a0cc221d188d90f8af Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 4 Jan 2018 18:12:31 +0100 Subject: [PATCH 063/127] context: new function to define default data, instead of None --- nemubot/modulecontext.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nemubot/modulecontext.py b/nemubot/modulecontext.py index c7fa3d4..4af3731 100644 --- a/nemubot/modulecontext.py +++ b/nemubot/modulecontext.py @@ -40,6 +40,11 @@ class _ModuleContext: def set_knodes(self, knodes): self._knodes = knodes + def set_default(self, default): + # Access to data will trigger the load of data + if self.data is None: + self._data = default + def add_hook(self, hook, *triggers): from nemubot.hooks import Abstract as AbstractHook assert isinstance(hook, AbstractHook), hook From 4275009dead4cac6c2b39219b03bbad79ea434bd Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 4 Jan 2018 18:16:47 +0100 Subject: [PATCH 064/127] events: now support timedelta instead of int/float --- nemubot/event/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nemubot/event/__init__.py b/nemubot/event/__init__.py index 981cf4b..2b3ed6b 100644 --- a/nemubot/event/__init__.py +++ b/nemubot/event/__init__.py @@ -63,8 +63,14 @@ class ModuleEvent: self.call_data = func_data # Store times - self.offset = timedelta(seconds=offset) # Time to wait before the first check - self.interval = timedelta(seconds=interval) + if isinstance(offset, timedelta): + self.offset = offset # Time to wait before the first check + else: + self.offset = timedelta(seconds=offset) # Time to wait before the first check + if isinstance(interval, timedelta): + self.interval = interval + else: + self.interval = timedelta(seconds=interval) self._end = None # Cache # How many times do this event? From 2af56e606a79b82b728b6d07f2a2f1c774ee2114 Mon Sep 17 00:00:00 2001 From: nemunaire Date: Fri, 1 Sep 2017 20:47:38 +0200 Subject: [PATCH 065/127] events: Use the new data parser, knodes based --- modules/events.py | 176 +++++++++++++++++++++++++++++----------------- 1 file changed, 110 insertions(+), 66 deletions(-) diff --git a/modules/events.py b/modules/events.py index 9814aa2..2b6bfb1 100644 --- a/modules/events.py +++ b/modules/events.py @@ -1,7 +1,9 @@ """Create countdowns and reminders""" -import re +import calendar from datetime import datetime, timedelta, timezone +from functools import partial +import re from nemubot import context from nemubot.exception import IMException @@ -10,31 +12,84 @@ from nemubot.hooks import hook from nemubot.message import Command from nemubot.tools.countdown import countdown_format, countdown from nemubot.tools.date import extractDate -from nemubot.tools.xmlparser.node import ModuleState +from nemubot.tools.xmlparser.basic import DictNode from nemubot.module.more import Response +class Event: + + def __init__(self, server, channel, creator, start_time, end_time=None): + self._server = server + self._channel = channel + self._creator = creator + self._start = datetime.utcfromtimestamp(float(start_time)).replace(tzinfo=timezone.utc) if not isinstance(start_time, datetime) else start_time + self._end = datetime.utcfromtimestamp(float(end_time)).replace(tzinfo=timezone.utc) if end_time else None + self._evt = None + + + def __del__(self): + if self._evt is not None: + context.del_event(self._evt) + self._evt = None + + + def saveElement(self, store, tag="event"): + attrs = { + "server": str(self._server), + "channel": str(self._channel), + "creator": str(self._creator), + "start_time": str(calendar.timegm(self._start.timetuple())), + } + if self._end: + attrs["end_time"] = str(calendar.timegm(self._end.timetuple())) + store.startElement(tag, attrs) + store.endElement(tag) + + @property + def creator(self): + return self._creator + + @property + def start(self): + return self._start + + @property + def end(self): + return self._end + + @end.setter + def end(self, c): + self._end = c + + @end.deleter + def end(self): + self._end = None + + def help_full (): - return "This module store a lot of events: ny, we, " + (", ".join(context.datas.index.keys() if hasattr(context, "datas") else [])) + "\n!eventslist: gets list of timer\n!start /something/: launch a timer" + return "This module store a lot of events: ny, we, " + (", ".join(context.datas.keys()) if hasattr(context, "datas") else "") + "\n!eventslist: gets list of timer\n!start /something/: launch a timer" def load(context): - #Define the index - context.data.setIndex("name") + context.set_knodes({ + "dict": DictNode, + "event": Event, + }) - for evt in context.data.index.keys(): - if context.data.index[evt].hasAttribute("end"): - event = ModuleEvent(call=fini, call_data=dict(strend=context.data.index[evt])) - event._end = context.data.index[evt].getDate("end") - idt = context.add_event(event) - if idt is not None: - context.data.index[evt]["_id"] = idt + if context.data is None: + context.set_default(DictNode()) + + # Relaunch all timers + for kevt in context.data: + if context.data[kevt].end: + context.data[kevt]._evt = context.add_event(ModuleEvent(partial(fini, kevt, context.data[kevt]), offset=context.data[kevt].end - datetime.now(timezone.utc), interval=0)) -def fini(d, strend): - context.send_response(strend["server"], Response("%s arrivé à échéance." % strend["name"], channel=strend["channel"], nick=strend["proprio"])) - context.data.delChild(context.data.index[strend["name"]]) +def fini(name, evt): + context.send_response(evt._server, Response("%s arrivé à échéance." % name, channel=evt._channel, nick=evt.creator)) + evt._evt = None + del context.data[name] context.save() @@ -63,18 +118,10 @@ def start_countdown(msg): """!start /something/: launch a timer""" if len(msg.args) < 1: raise IMException("indique le nom d'un événement à chronométrer") - if msg.args[0] in context.data.index: + if msg.args[0] in context.data: raise IMException("%s existe déjà." % msg.args[0]) - strnd = ModuleState("strend") - strnd["server"] = msg.server - strnd["channel"] = msg.channel - strnd["proprio"] = msg.frm - strnd["start"] = msg.date - strnd["name"] = msg.args[0] - context.data.addChild(strnd) - - evt = ModuleEvent(call=fini, call_data=dict(strend=strnd)) + evt = Event(server=msg.server, channel=msg.channel, creator=msg.frm, start_time=msg.date) if len(msg.args) > 1: result1 = re.findall("([0-9]+)([smhdjwyaSMHDJWYA])?", msg.args[1]) @@ -92,50 +139,48 @@ def start_countdown(msg): if result2 is None or result2.group(4) is None: yea = now.year else: yea = int(result2.group(4)) if result2 is not None and result3 is not None: - strnd["end"] = datetime(yea, int(result2.group(3)), int(result2.group(2)), hou, minu, sec, timezone.utc) + evt.end = datetime(yea, int(result2.group(3)), int(result2.group(2)), hou, minu, sec, timezone.utc) elif result2 is not None: - strnd["end"] = datetime(int(result2.group(4)), int(result2.group(3)), int(result2.group(2)), 0, 0, 0, timezone.utc) + evt.end = datetime(int(result2.group(4)), int(result2.group(3)), int(result2.group(2)), 0, 0, 0, timezone.utc) elif result3 is not None: if hou * 3600 + minu * 60 + sec > now.hour * 3600 + now.minute * 60 + now.second: - strnd["end"] = datetime(now.year, now.month, now.day, hou, minu, sec, timezone.utc) + evt.end = datetime(now.year, now.month, now.day, hou, minu, sec, timezone.utc) else: - strnd["end"] = datetime(now.year, now.month, now.day + 1, hou, minu, sec, timezone.utc) - evt._end = strnd.getDate("end") - strnd["_id"] = context.add_event(evt) + evt.end = datetime(now.year, now.month, now.day + 1, hou, minu, sec, timezone.utc) except: - context.data.delChild(strnd) raise IMException("Mauvais format de date pour l'événement %s. Il n'a pas été créé." % msg.args[0]) elif result1 is not None and len(result1) > 0: - strnd["end"] = msg.date + evt.end = msg.date for (t, g) in result1: if g is None or g == "" or g == "m" or g == "M": - strnd["end"] += timedelta(minutes=int(t)) + evt.end += timedelta(minutes=int(t)) elif g == "h" or g == "H": - strnd["end"] += timedelta(hours=int(t)) + evt.end += timedelta(hours=int(t)) elif g == "d" or g == "D" or g == "j" or g == "J": - strnd["end"] += timedelta(days=int(t)) + evt.end += timedelta(days=int(t)) elif g == "w" or g == "W": - strnd["end"] += timedelta(days=int(t)*7) + evt.end += timedelta(days=int(t)*7) elif g == "y" or g == "Y" or g == "a" or g == "A": - strnd["end"] += timedelta(days=int(t)*365) + evt.end += timedelta(days=int(t)*365) else: - strnd["end"] += timedelta(seconds=int(t)) - evt._end = strnd.getDate("end") - eid = context.add_event(evt) - if eid is not None: - strnd["_id"] = eid + evt.end += timedelta(seconds=int(t)) + context.data[msg.args[0]] = evt context.save() - if "end" in strnd: + + if evt.end is not None: + context.add_event(ModuleEvent(partial(fini, msg.args[0], evt), + offset=evt.end - datetime.now(timezone.utc), + interval=0)) return Response("%s commencé le %s et se terminera le %s." % (msg.args[0], msg.date.strftime("%A %d %B %Y à %H:%M:%S"), - strnd.getDate("end").strftime("%A %d %B %Y à %H:%M:%S")), - nick=msg.frm) + evt.end.strftime("%A %d %B %Y à %H:%M:%S")), + channel=msg.channel) else: return Response("%s commencé le %s"% (msg.args[0], msg.date.strftime("%A %d %B %Y à %H:%M:%S")), - nick=msg.frm) + channel=msg.channel) @hook.command("end") @@ -144,16 +189,15 @@ def end_countdown(msg): if len(msg.args) < 1: raise IMException("quel événement terminer ?") - if msg.args[0] in context.data.index: - if context.data.index[msg.args[0]]["proprio"] == msg.frm or (msg.cmd == "forceend" and msg.frm_owner): - duration = countdown(msg.date - context.data.index[msg.args[0]].getDate("start")) - context.del_event(context.data.index[msg.args[0]]["_id"]) - context.data.delChild(context.data.index[msg.args[0]]) + if msg.args[0] in context.data: + if context.data[msg.args[0]].creator == msg.frm or (msg.cmd == "forceend" and msg.frm_owner): + duration = countdown(msg.date - context.data[msg.args[0]].start) + del context.data[msg.args[0]] context.save() return Response("%s a duré %s." % (msg.args[0], duration), channel=msg.channel, nick=msg.frm) else: - raise IMException("Vous ne pouvez pas terminer le compteur %s, créé par %s." % (msg.args[0], context.data.index[msg.args[0]]["proprio"])) + raise IMException("Vous ne pouvez pas terminer le compteur %s, créé par %s." % (msg.args[0], context.data[msg.args[0]].creator)) else: return Response("%s n'est pas un compteur connu."% (msg.args[0]), channel=msg.channel, nick=msg.frm) @@ -162,19 +206,19 @@ def end_countdown(msg): def liste(msg): """!eventslist: gets list of timer""" if len(msg.args): - res = list() + res = Response(channel=msg.channel) for user in msg.args: - cmptr = [x["name"] for x in context.data.index.values() if x["proprio"] == user] + cmptr = [k for k in context.data if context.data[k].creator == user] if len(cmptr) > 0: - res.append("Compteurs créés par %s : %s" % (user, ", ".join(cmptr))) + res.append_message(cmptr, title="Events created by %s" % user) else: - res.append("%s n'a pas créé de compteur" % user) - return Response(" ; ".join(res), channel=msg.channel) + res.append_message("%s doesn't have any counting events" % user) + return res else: - return Response("Compteurs connus : %s." % ", ".join(context.data.index.keys()), channel=msg.channel) + return Response(list(context.data.keys()), channel=msg.channel, title="Known events") -@hook.command(match=lambda msg: isinstance(msg, Command) and msg.cmd in context.data.index) +@hook.command(match=lambda msg: isinstance(msg, Command) and msg.cmd in context.data) def parseanswer(msg): res = Response(channel=msg.channel) @@ -182,13 +226,13 @@ def parseanswer(msg): if msg.cmd[0] == "!": res.nick = msg.frm - if context.data.index[msg.cmd].name == "strend": - if context.data.index[msg.cmd].hasAttribute("end"): - res.append_message("%s commencé il y a %s et se terminera dans %s." % (msg.cmd, countdown(msg.date - context.data.index[msg.cmd].getDate("start")), countdown(context.data.index[msg.cmd].getDate("end") - msg.date))) + if msg.cmd in context.data: + if context.data[msg.cmd].end: + res.append_message("%s commencé il y a %s et se terminera dans %s." % (msg.cmd, countdown(msg.date - context.data[msg.cmd].start), countdown(context.data[msg.cmd].end - msg.date))) else: - res.append_message("%s commencé il y a %s." % (msg.cmd, countdown(msg.date - context.data.index[msg.cmd].getDate("start")))) + res.append_message("%s commencé il y a %s." % (msg.cmd, countdown(msg.date - context.data[msg.cmd].start))) else: - res.append_message(countdown_format(context.data.index[msg.cmd].getDate("start"), context.data.index[msg.cmd]["msg_before"], context.data.index[msg.cmd]["msg_after"])) + res.append_message(countdown_format(context.data[msg.cmd].start, context.data[msg.cmd]["msg_before"], context.data[msg.cmd]["msg_after"])) return res @@ -199,7 +243,7 @@ def parseask(msg): name = re.match("^.*!([^ \"'@!]+).*$", msg.message) if name is None: raise IMException("il faut que tu attribues une commande à l'événement.") - if name.group(1) in context.data.index: + if name.group(1) in context.data: raise IMException("un événement portant ce nom existe déjà.") texts = re.match("^[^\"]*(avant|après|apres|before|after)?[^\"]*\"([^\"]+)\"[^\"]*((avant|après|apres|before|after)?.*\"([^\"]+)\".*)?$", msg.message, re.I) From 342bb9acdc88c779e44adf2e9f4a2b4e675d134c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 10 Feb 2018 09:51:51 +0100 Subject: [PATCH 066/127] Refactor in treatment analysis --- modules/alias.py | 3 +- nemubot/hooks/abstract.py | 12 ++++++- nemubot/treatment.py | 66 +++++++++++++-------------------------- 3 files changed, 34 insertions(+), 47 deletions(-) diff --git a/modules/alias.py b/modules/alias.py index a246d2c..c432a85 100644 --- a/modules/alias.py +++ b/modules/alias.py @@ -272,7 +272,6 @@ def treat_alias(msg): # Avoid infinite recursion if not isinstance(rpl_msg, Command) or msg.cmd != rpl_msg.cmd: - # Also return origin message, if it can be treated as well - return [msg, rpl_msg] + return rpl_msg return msg diff --git a/nemubot/hooks/abstract.py b/nemubot/hooks/abstract.py index eac4b20..ffe79fb 100644 --- a/nemubot/hooks/abstract.py +++ b/nemubot/hooks/abstract.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import types + def call_game(call, *args, **kargs): """With given args, try to determine the right call to make @@ -119,10 +121,18 @@ class Abstract: try: if self.check(data1): ret = call_game(self.call, data1, self.data, *args) + if isinstance(ret, types.GeneratorType): + for r in ret: + yield r + ret = None except IMException as e: ret = e.fill_response(data1) finally: if self.times == 0: self.call_end(ret) - return ret + if isinstance(ret, list): + for r in ret: + yield ret + elif ret is not None: + yield ret diff --git a/nemubot/treatment.py b/nemubot/treatment.py index 4f629e0..ed7cacb 100644 --- a/nemubot/treatment.py +++ b/nemubot/treatment.py @@ -15,7 +15,6 @@ # along with this program. If not, see . import logging -import types logger = logging.getLogger("nemubot.treatment") @@ -79,19 +78,12 @@ class MessageTreater: for h in self.hm.get_hooks("pre", type(msg).__name__): if h.can_read(msg.to, msg.server) and h.match(msg): - res = h.run(msg) + for res in flatify(h.run(msg)): + if res is not None and res != msg: + yield from self._pre_treat(res) - if isinstance(res, list): - for i in range(len(res)): - # Avoid infinite loop - if res[i] != msg: - yield from self._pre_treat(res[i]) - - elif res is not None and res != msg: - yield from self._pre_treat(res) - - elif res is None or res is False: - break + elif res is None or res is False: + break else: yield msg @@ -113,25 +105,10 @@ class MessageTreater: msg.frm_owner = (not hasattr(msg.server, "owner") or msg.server.owner == msg.frm) while hook is not None: - res = hook.run(msg) - - if isinstance(res, list): - for r in res: - yield r - - elif res is not None: - if isinstance(res, types.GeneratorType): - for r in res: - if not hasattr(r, "server") or r.server is None: - r.server = msg.server - - yield r - - else: - if not hasattr(res, "server") or res.server is None: - res.server = msg.server - - yield res + for res in flatify(hook.run(msg)): + if not hasattr(res, "server") or res.server is None: + res.server = msg.server + yield res hook = next(hook_gen, None) @@ -165,19 +142,20 @@ class MessageTreater: for h in self.hm.get_hooks("post", type(msg).__name__): if h.can_write(msg.to, msg.server) and h.match(msg): - res = h.run(msg) + for res in flatify(h.run(msg)): + if res is not None and res != msg: + yield from self._post_treat(res) - if isinstance(res, list): - for i in range(len(res)): - # Avoid infinite loop - if res[i] != msg: - yield from self._post_treat(res[i]) - - elif res is not None and res != msg: - yield from self._post_treat(res) - - elif res is None or res is False: - break + elif res is None or res is False: + break else: yield msg + + +def flatify(g): + if hasattr(g, "__iter__"): + for i in g: + yield from flatify(i) + else: + yield g From 1887e481d2978949a52710ee45926bcdd1bacc8b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 3 Apr 2018 08:02:41 +0200 Subject: [PATCH 067/127] sms: send result of command by SMS --- modules/sms.py | 74 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/modules/sms.py b/modules/sms.py index 7db172b..57ab3ae 100644 --- a/modules/sms.py +++ b/modules/sms.py @@ -46,29 +46,22 @@ def send_sms(frm, api_usr, api_key, content): return None - -@hook.command("sms") -def cmd_sms(msg): - if not len(msg.args): - raise IMException("À qui veux-tu envoyer ce SMS ?") - - # Check dests - cur_epoch = time.mktime(time.localtime()); - for u in msg.args[0].split(","): +def check_sms_dests(dests, cur_epoch): + """Raise exception if one of the dest is not known or has already receive a SMS recently + """ + for u in dests: if u not in context.data.index: raise IMException("Désolé, je sais pas comment envoyer de SMS à %s." % u) elif cur_epoch - float(context.data.index[u]["lastuse"]) < 42: raise IMException("Un peu de calme, %s a déjà reçu un SMS il n'y a pas si longtemps." % u) + return True - # Go! + +def send_sms_to_list(msg, frm, dests, content, cur_epoch): fails = list() - for u in msg.args[0].split(","): + for u in dests: context.data.index[u]["lastuse"] = cur_epoch - if msg.to_response[0] == msg.frm: - frm = msg.frm - else: - frm = msg.frm + "@" + msg.to[0] - test = send_sms(frm, context.data.index[u]["user"], context.data.index[u]["key"], " ".join(msg.args[1:])) + test = send_sms(frm, context.data.index[u]["user"], context.data.index[u]["key"], content) if test is not None: fails.append( "%s: %s" % (u, test) ) @@ -77,6 +70,55 @@ def cmd_sms(msg): else: return Response("le SMS a bien été envoyé", msg.channel, msg.frm) + +@hook.command("sms") +def cmd_sms(msg): + if not len(msg.args): + raise IMException("À qui veux-tu envoyer ce SMS ?") + + cur_epoch = time.mktime(time.localtime()) + dests = msg.args[0].split(",") + frm = msg.frm if msg.to_response[0] == msg.frm else msg.frm + "@" + msg.to[0] + content = " ".join(msg.args[1:]) + + check_sms_dests(dests, cur_epoch) + return send_sms_to_list(msg, frm, dests, content, cur_epoch) + + +@hook.command("smscmd") +def cmd_smscmd(msg): + if not len(msg.args): + raise IMException("À qui veux-tu envoyer ce SMS ?") + + cur_epoch = time.mktime(time.localtime()) + dests = msg.args[0].split(",") + frm = msg.frm if msg.to_response[0] == msg.frm else msg.frm + "@" + msg.to[0] + cmd = " ".join(msg.args[1:]) + + content = None + for r in context.subtreat(context.subparse(msg, cmd)): + if isinstance(r, Response): + for m in r.messages: + if isinstance(m, list): + for n in m: + content = n + break + if content is not None: + break + elif isinstance(m, str): + content = m + break + + elif isinstance(r, Text): + content = r.message + + if content is None: + raise IMException("Aucun SMS envoyé : le résultat de la commande n'a pas retourné de contenu.") + + check_sms_dests(dests, cur_epoch) + return send_sms_to_list(msg, frm, dests, content, cur_epoch) + + apiuser_ask = re.compile(r"(utilisateur|user|numéro|numero|compte|abonne|abone|abonné|account)\s+(est|is)\s+(?P[0-9]{7,})", re.IGNORECASE) apikey_ask = re.compile(r"(clef|key|password|mot de passe?)\s+(?:est|is)?\s+(?P[a-zA-Z0-9]{10,})", re.IGNORECASE) From 8a25ebb45b81f92d9bb1fe9702f5d52318487a56 Mon Sep 17 00:00:00 2001 From: nemunaire Date: Thu, 19 Apr 2018 23:52:35 +0200 Subject: [PATCH 068/127] xmlparser: fix parsing of subchild --- nemubot/tools/xmlparser/__init__.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/nemubot/tools/xmlparser/__init__.py b/nemubot/tools/xmlparser/__init__.py index c8d393a..1bf60a8 100644 --- a/nemubot/tools/xmlparser/__init__.py +++ b/nemubot/tools/xmlparser/__init__.py @@ -83,7 +83,7 @@ class XMLParser: @property def root(self): if len(self.stack): - return self.stack[0] + return self.stack[0][0] else: return None @@ -91,13 +91,13 @@ class XMLParser: @property def current(self): if len(self.stack): - return self.stack[-1] + return self.stack[-1][0] else: return None def display_stack(self): - return " in ".join([str(type(s).__name__) for s in reversed(self.stack)]) + return " in ".join([str(type(s).__name__) for s,c in reversed(self.stack)]) def startElement(self, name, attrs): @@ -105,7 +105,8 @@ class XMLParser: if name not in self.knodes: raise TypeError(name + " is not a known type to decode") else: - self.stack.append(self.knodes[name](**attrs)) + self.stack.append((self.knodes[name](**attrs), self.child)) + self.child = 0 else: self.child += 1 @@ -116,19 +117,15 @@ class XMLParser: def endElement(self, name): - if self.child: - self.child -= 1 - - if hasattr(self.current, "endElement"): - self.current.endElement(name) - return - if hasattr(self.current, "endElement"): self.current.endElement(None) + if self.child: + self.child -= 1 + # Don't remove root - if len(self.stack) > 1: - last = self.stack.pop() + elif len(self.stack) > 1: + last, self.child = self.stack.pop() if hasattr(self.current, "addChild"): if self.current.addChild(name, last): return From 015fb47d90344847b94e3c86d55ac5944909c2d9 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 3 Jun 2018 09:58:38 +0200 Subject: [PATCH 069/127] events: alert on malformed start command --- modules/events.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/events.py b/modules/events.py index 2b6bfb1..acac196 100644 --- a/modules/events.py +++ b/modules/events.py @@ -166,6 +166,9 @@ def start_countdown(msg): else: evt.end += timedelta(seconds=int(t)) + else: + raise IMException("Mauvais format de date pour l'événement %s. Il n'a pas été créé." % msg.args[0]) + context.data[msg.args[0]] = evt context.save() From 125ae6ad0b288e8e2f931794ded32655c40f6fcb Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 2 Jun 2018 14:17:31 +0200 Subject: [PATCH 070/127] feed: fix RSS parsing --- nemubot/tools/feed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nemubot/tools/feed.py b/nemubot/tools/feed.py index 7e63cd2..0404fff 100644 --- a/nemubot/tools/feed.py +++ b/nemubot/tools/feed.py @@ -105,13 +105,13 @@ class Feed: self.updated = None self.entries = list() - if self.feed.tagName == "rss": + if self.feed.tagName == "rdf:RDF": self._parse_rss_feed() elif self.feed.tagName == "feed": self._parse_atom_feed() else: from nemubot.exception import IMException - raise IMException("This is not a valid Atom or RSS feed") + raise IMException("This is not a valid Atom or RSS feed.") def _parse_atom_feed(self): From b8741bb1f71ecf4d91f6a4d0547e292eeb8a4774 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 20 Jun 2018 07:46:46 +0200 Subject: [PATCH 071/127] imdb: fix exception when no movie found --- modules/imdb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/imdb.py b/modules/imdb.py index d5ff158..c5fdf76 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -52,7 +52,9 @@ def find_movies(title, year=None): # Make the request data = web.getJSON(url, remove_callback=True) - if year is None: + if "d" not in data: + return None + elif year is None: return data["d"] else: return [d for d in data["d"] if "y" in d and str(d["y"]) == year] From cd6750154c9270fbf3a9324bbd024f9956107b9b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 15 Jun 2018 00:55:08 +0200 Subject: [PATCH 072/127] worldcup: update module to 2018 worldcup --- modules/worldcup.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/modules/worldcup.py b/modules/worldcup.py index b12ca30..764991d 100644 --- a/modules/worldcup.py +++ b/modules/worldcup.py @@ -1,6 +1,6 @@ # coding=utf-8 -"""The 2014 football worldcup module""" +"""The 2014,2018 football worldcup module""" from datetime import datetime, timezone import json @@ -9,6 +9,7 @@ from urllib.parse import quote from urllib.request import urlopen from nemubot import context +from nemubot.event import ModuleEvent from nemubot.exception import IMException from nemubot.hooks import hook from nemubot.tools.xmlparser.node import ModuleState @@ -20,7 +21,6 @@ from nemubot.module.more import Response API_URL="http://worldcup.sfg.io/%s" def load(context): - from nemubot.event import ModuleEvent context.add_event(ModuleEvent(func=lambda url: urlopen(url, timeout=10).read().decode(), func_data=API_URL % "matches/current?by_date=DESC", call=current_match_new_action, interval=30)) @@ -65,10 +65,10 @@ def cmd_watch(msg): context.save() raise IMException("This channel will not anymore receives world cup events.") -def current_match_new_action(match_str, osef): - context.add_event(ModuleEvent(func=lambda url: urlopen(url).read().decode(), func_data=API_URL % "matches/current?by_date=DESC", call=current_match_new_action, interval=30)) - - matches = json.loads(match_str) +def current_match_new_action(matches, osef): + def cmp(om, nm): + return len(nm) and (len(om) == 0 or len(nm[0]["home_team_events"]) != len(om[0]["home_team_events"]) or len(nm[0]["away_team_events"]) != len(om[0]["away_team_events"])) + context.add_event(ModuleEvent(func=lambda url: json.loads(urlopen(url).read().decode()), func_data=API_URL % "matches/current?by_date=DESC", cmp=cmp, call=current_match_new_action, interval=30)) for match in matches: if is_valid(match): @@ -120,20 +120,19 @@ def detail_event(evt): return evt + " par" def txt_event(e): - return "%se minutes : %s %s (%s)" % (e["time"], detail_event(e["type_of_event"]), e["player"], e["team"]["code"]) + return "%s minute : %s %s (%s)" % (e["time"], detail_event(e["type_of_event"]), e["player"], e["team"]["code"]) def prettify(match): - matchdate_local = datetime.strptime(match["datetime"].replace(':', ''), "%Y-%m-%dT%H%M%S.%f%z") - matchdate = matchdate_local - (matchdate_local.utcoffset() - datetime.timedelta(hours=2)) + matchdate = datetime.strptime(match["datetime"].replace(':', ''), "%Y-%m-%dT%H%M%SZ").replace(tzinfo=timezone.utc) if match["status"] == "future": - return ["Match à venir (%s) le %s : %s vs. %s" % (match["match_number"], matchdate.strftime("%A %d à %H:%M"), match["home_team"]["country"], match["away_team"]["country"])] + return ["Match à venir (%s) le %s : %s vs. %s" % (match["fifa_id"], matchdate.strftime("%A %d à %H:%M"), match["home_team"]["country"], match["away_team"]["country"])] else: msgs = list() msg = "" if match["status"] == "completed": - msg += "Match (%s) du %s terminé : " % (match["match_number"], matchdate.strftime("%A %d à %H:%M")) + msg += "Match (%s) du %s terminé : " % (match["fifa_id"], matchdate.strftime("%A %d à %H:%M")) else: - msg += "Match en cours (%s) depuis %d minutes : " % (match["match_number"], (datetime.now(matchdate.tzinfo) - matchdate_local).total_seconds() / 60) + msg += "Match en cours (%s) depuis %d minutes : " % (match["fifa_id"], (datetime.now(tz=timezone.utc) - matchdate).total_seconds() / 60) msg += "%s %d - %d %s" % (match["home_team"]["country"], match["home_team"]["goals"], match["away_team"]["goals"], match["away_team"]["country"]) @@ -163,7 +162,7 @@ def is_valid(match): def get_match(url, matchid): allm = get_matches(url) for m in allm: - if int(m["match_number"]) == matchid: + if int(m["fifa_id"]) == matchid: return [ m ] def get_matches(url): @@ -192,7 +191,7 @@ def cmd_worldcup(msg): elif len(msg.args[0]) == 3: url = "matches/country?fifa_code=%s&by_date=DESC" % msg.args[0] elif is_int(msg.args[0]): - url = int(msg.arg[0]) + url = int(msg.args[0]) else: raise IMException("unrecognized request; choose between 'today', 'tomorrow', a FIFA country code or a match identifier") From 3b99099b528859db95106eb0309c6e9cd7f09809 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 30 Aug 2018 07:20:00 +0200 Subject: [PATCH 073/127] imdb: fix compatibility with new IMDB version --- modules/imdb.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/modules/imdb.py b/modules/imdb.py index c5fdf76..a938c7b 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -24,20 +24,17 @@ def get_movie_by_id(imdbid): return { "imdbID": imdbid, - "Title": soup.body.find(attrs={"itemprop": "name"}).next_element.strip(), + "Title": soup.body.find('h1').next_element.strip(), "Year": soup.body.find(id="titleYear").find("a").text.strip() if soup.body.find(id="titleYear") else ", ".join([y.text.strip() for y in soup.body.find(attrs={"class": "seasons-and-year-nav"}).find_all("div")[3].find_all("a")[:-1]]), - "Duration": soup.body.find_all(attrs={"itemprop": "duration"})[-1].text.strip(), - "imdbRating": soup.body.find(attrs={"itemprop": "ratingValue"}).text.strip(), - "imdbVotes": soup.body.find(attrs={"itemprop": "ratingCount"}).text.strip(), - "Plot": re.sub(r"\s+", " ", soup.body.find(id="titleStoryLine").find(attrs={"itemprop": "description"}).text).strip(), + "Duration": soup.body.find(attrs={"class": "subtext"}).find("time").text.strip(), + "imdbRating": soup.body.find(attrs={"class": "ratingValue"}).find("strong").text.strip(), + "imdbVotes": soup.body.find(attrs={"class": "imdbRating"}).find("a").text.strip(), + "Plot": re.sub(r"\s+", " ", soup.body.find(attrs={"class": "summary_text"}).text).strip(), - "Type": "TV Series" if soup.find(attrs={"class": "np_episode_guide"}) else "Movie", - "Country": ", ".join([c.find("a").text.strip() for c in soup.body.find(id="titleDetails").find_all(attrs={"class": "txt-block"}) if c.text.find("Country") != -1]), - "Released": soup.body.find(attrs={"itemprop": "datePublished"}).attrs["content"] if "content" in soup.body.find(attrs={"itemprop": "datePublished"}).attrs else "N\A", - "Genre": ", ".join([g.text.strip() for g in soup.body.find_all(attrs={"itemprop": "genre"})[:-1]]), - "Director": ", ".join([d.find(attrs={"itemprop": "name"}).text.strip() for d in soup.body.find_all(attrs={"itemprop": "director"})]), - "Writer": ", ".join([d.find(attrs={"itemprop": "name"}).text.strip() for d in soup.body.find_all(attrs={"itemprop": "creator"})]), - "Actors": ", ".join([d.find(attrs={"itemprop": "name"}).text.strip() for d in soup.body.find_all(attrs={"itemprop": "actors"})]), + "Type": "TV Series" if soup.find(id="title-episode-widget") else "Movie", + "Genre": ", ".join([x.text.strip() for x in soup.body.find(id="titleStoryLine").find_all("a") if x.get("href") is not None and x.get("href")[:7] == "/genre/"]), + "Country": ", ".join([x.text.strip() for x in soup.body.find(id="titleDetails").find_all("a") if x.get("href") is not None and x.get("href")[:32] == "/search/title?country_of_origin="]), + "Credits": " ; ".join([x.find("h4").text.strip() + " " + (", ".join([y.text.strip() for y in x.find_all("a") if y.get("href") is not None and y.get("href")[:6] == "/name/"])) for x in soup.body.find_all(attrs={"class": "credit_summary_item"})]), } @@ -94,9 +91,9 @@ def cmd_imdb(msg): res.append_message("%s \x02genre:\x0F %s; \x02rating\x0F: %s (%s votes); \x02plot\x0F: %s" % (data['Type'], data['Genre'], data['imdbRating'], data['imdbVotes'], data['Plot'])) + res.append_message("%s \x02from\x0F %s; %s" + % (data['Type'], data['Country'], data['Credits'])) - res.append_message("%s \x02from\x0F %s \x02released on\x0F %s; \x02directed by:\x0F %s; \x02written by:\x0F %s; \x02main actors:\x0F %s" - % (data['Type'], data['Country'], data['Released'], data['Director'], data['Writer'], data['Actors'])) return res From 5578e8b86e7bd3f707ee7b2f0bfe81dd14d19be7 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 30 Aug 2018 07:32:15 +0200 Subject: [PATCH 074/127] tools/web: split getURLContent function --- nemubot/tools/web.py | 73 ++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index c3ba42a..9428dd5 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -68,18 +68,7 @@ def getPassword(url): # Get real pages -def getURLContent(url, body=None, timeout=7, header=None, decode_error=False, - max_size=524288): - """Return page content corresponding to URL or None if any error occurs - - Arguments: - url -- the URL to get - body -- Data to send as POST content - timeout -- maximum number of seconds to wait before returning an exception - decode_error -- raise exception on non-200 pages or ignore it - max_size -- maximal size allow for the content - """ - +def _URLConn(cb, url, body=None, timeout=7, header=None): o = urlparse(_getNormalizedURL(url), "http") import http.client @@ -134,6 +123,27 @@ def getURLContent(url, body=None, timeout=7, header=None, decode_error=False, try: res = conn.getresponse() + return cb(res) + except http.client.BadStatusLine: + raise IMException("Invalid HTTP response") + finally: + conn.close() + +def getURLContent(url, body=None, timeout=7, header=None, decode_error=False, + max_size=524288): + """Return page content corresponding to URL or None if any error occurs + + Arguments: + url -- the URL to get + body -- Data to send as POST content + timeout -- maximum number of seconds to wait before returning an exception + decode_error -- raise exception on non-200 pages or ignore it + max_size -- maximal size allow for the content + """ + + import http.client + + def next(res): size = int(res.getheader("Content-Length", 524288)) cntype = res.getheader("Content-Type") @@ -155,28 +165,25 @@ def getURLContent(url, body=None, timeout=7, header=None, decode_error=False, charset = cha[1] else: charset = cha[0] - except http.client.BadStatusLine: - raise IMException("Invalid HTTP response") - finally: - conn.close() - if res.status == http.client.OK or res.status == http.client.SEE_OTHER: - return data.decode(charset).strip() - elif ((res.status == http.client.FOUND or - res.status == http.client.MOVED_PERMANENTLY) and - res.getheader("Location") != url): - return getURLContent( - urljoin(url, res.getheader("Location")), - body=body, - timeout=timeout, - header=header, - decode_error=decode_error, - max_size=max_size) - elif decode_error: - return data.decode(charset).strip() - else: - raise IMException("A HTTP error occurs: %d - %s" % - (res.status, http.client.responses[res.status])) + if res.status == http.client.OK or res.status == http.client.SEE_OTHER: + return data.decode(charset).strip() + elif ((res.status == http.client.FOUND or + res.status == http.client.MOVED_PERMANENTLY) and + res.getheader("Location") != url): + return getURLContent( + urljoin(url, res.getheader("Location")), + body=body, + timeout=timeout, + header=header, + decode_error=decode_error, + max_size=max_size) + elif decode_error: + return data.decode(charset).strip() + else: + raise IMException("A HTTP error occurs: %d - %s" % + (res.status, http.client.responses[res.status])) + return _URLConn(next, url=url, body=body, timeout=timeout) def getXML(*args, **kwargs): From 72bc8d3839a6b51394d23ccc4a385086d679d9ca Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 30 Aug 2018 07:53:38 +0200 Subject: [PATCH 075/127] feed: accept RSS that begins with tag --- nemubot/tools/feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemubot/tools/feed.py b/nemubot/tools/feed.py index 0404fff..2873a65 100644 --- a/nemubot/tools/feed.py +++ b/nemubot/tools/feed.py @@ -105,7 +105,7 @@ class Feed: self.updated = None self.entries = list() - if self.feed.tagName == "rdf:RDF": + if self.feed.tagName == "rdf:RDF" or self.feed.tagName == "rss": self._parse_rss_feed() elif self.feed.tagName == "feed": self._parse_atom_feed() From 4a636b2b119cb52dbe4c8db16412695b199dce00 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 30 Aug 2018 07:59:26 +0200 Subject: [PATCH 076/127] tools/web: follow redirection in URLConn --- nemubot/tools/web.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index 9428dd5..73a3807 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -68,7 +68,7 @@ def getPassword(url): # Get real pages -def _URLConn(cb, url, body=None, timeout=7, header=None): +def _URLConn(cb, url, body=None, timeout=7, header=None, follow_redir=True): o = urlparse(_getNormalizedURL(url), "http") import http.client @@ -123,6 +123,15 @@ def _URLConn(cb, url, body=None, timeout=7, header=None): try: res = conn.getresponse() + if follow_redir and ((res.status == http.client.FOUND or + res.status == http.client.MOVED_PERMANENTLY) and + res.getheader("Location") != url): + return _URLConn(cb, + url=urljoin(url, res.getheader("Location")), + body=body, + timeout=timeout, + header=header, + follow_redir=follow_redir) return cb(res) except http.client.BadStatusLine: raise IMException("Invalid HTTP response") @@ -141,13 +150,11 @@ def getURLContent(url, body=None, timeout=7, header=None, decode_error=False, max_size -- maximal size allow for the content """ - import http.client - - def next(res): + def _nextURLContent(res): size = int(res.getheader("Content-Length", 524288)) cntype = res.getheader("Content-Type") - if size > max_size or (cntype is not None and cntype[:4] != "text" and cntype[:4] != "appl"): + if max_size >= 0 and (size > max_size or (cntype is not None and cntype[:4] != "text" and cntype[:4] != "appl")): raise IMException("Content too large to be retrieved") data = res.read(size) @@ -166,24 +173,17 @@ def getURLContent(url, body=None, timeout=7, header=None, decode_error=False, else: charset = cha[0] + import http.client + if res.status == http.client.OK or res.status == http.client.SEE_OTHER: return data.decode(charset).strip() - elif ((res.status == http.client.FOUND or - res.status == http.client.MOVED_PERMANENTLY) and - res.getheader("Location") != url): - return getURLContent( - urljoin(url, res.getheader("Location")), - body=body, - timeout=timeout, - header=header, - decode_error=decode_error, - max_size=max_size) elif decode_error: return data.decode(charset).strip() else: raise IMException("A HTTP error occurs: %d - %s" % (res.status, http.client.responses[res.status])) - return _URLConn(next, url=url, body=body, timeout=timeout) + + return _URLConn(_nextURLContent, url=url, body=body, timeout=timeout, header=header) def getXML(*args, **kwargs): From 53fe00ed58f3401e7bb1da5964d397e8966288a8 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 30 Aug 2018 07:59:41 +0200 Subject: [PATCH 077/127] tools/web: new function to retrieve only headers --- nemubot/tools/web.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index 73a3807..1a4fbd7 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -138,6 +138,21 @@ def _URLConn(cb, url, body=None, timeout=7, header=None, follow_redir=True): finally: conn.close() + +def getURLHeaders(url, body=None, timeout=7, header=None, follow_redir=True): + """Return page headers corresponding to URL or None if any error occurs + + Arguments: + url -- the URL to get + body -- Data to send as POST content + timeout -- maximum number of seconds to wait before returning an exception + """ + + def next(res): + return res.status, res.getheaders() + return _URLConn(next, url=url, body=body, timeout=timeout, header=header, follow_redir=follow_redir) + + def getURLContent(url, body=None, timeout=7, header=None, decode_error=False, max_size=524288): """Return page content corresponding to URL or None if any error occurs From 31abcc97cfd54c223e1cbe5271d3a8655ab66d1d Mon Sep 17 00:00:00 2001 From: nemunaire Date: Sun, 9 Sep 2018 19:17:56 +0200 Subject: [PATCH 078/127] event: extract dict before call without init data --- nemubot/event/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nemubot/event/__init__.py b/nemubot/event/__init__.py index 2b3ed6b..c471b2e 100644 --- a/nemubot/event/__init__.py +++ b/nemubot/event/__init__.py @@ -138,9 +138,12 @@ class ModuleEvent: self.call() else: self.call(d_init) + elif d_init is None: + if isinstance(self.call_data, dict): + self.call(**self.call_data) + else: + self.call(self.call_data) elif isinstance(self.call_data, dict): self.call(d_init, **self.call_data) - elif d_init is None: - self.call(self.call_data) else: self.call(d_init, self.call_data) From 2fd20d900242325b535c020c6e32f0a8d5af5fa6 Mon Sep 17 00:00:00 2001 From: nemunaire Date: Sun, 9 Sep 2018 19:33:42 +0200 Subject: [PATCH 079/127] nntp: Here it is! --- modules/nntp.py | 209 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 modules/nntp.py diff --git a/modules/nntp.py b/modules/nntp.py new file mode 100644 index 0000000..67757d1 --- /dev/null +++ b/modules/nntp.py @@ -0,0 +1,209 @@ +"""The NNTP module""" + +# PYTHON STUFFS ####################################################### + +import email +from email.utils import mktime_tz, parseaddr, parsedate_tz +from nntplib import NNTP, decode_header +import re +import time +from datetime import datetime +from zlib import adler32 + +from nemubot import context +from nemubot.event import ModuleEvent +from nemubot.exception import IMException +from nemubot.hooks import hook +from nemubot.tools.xmlparser.node import ModuleState + +from nemubot.module.more import Response + + +# LOADING ############################################################# + +def load(context): + for wn in context.data.getNodes("watched_newsgroup"): + watch(**wn.attributes) + + +# MODULE CORE ######################################################### + +def list_groups(group_pattern="*", **server): + with NNTP(**server) as srv: + response, l = srv.list(group_pattern) + for i in l: + yield i.group, srv.description(i.group), i.flag + +def read_group(group, **server): + with NNTP(**server) as srv: + response, count, first, last, name = srv.group(group) + resp, overviews = srv.over((first, last)) + for art_num, over in reversed(overviews): + yield over + +def read_article(msg_id, **server): + with NNTP(**server) as srv: + response, info = srv.article(msg_id) + return email.message_from_bytes(b"\r\n".join(info.lines)) + +def whatsnew(date_last_check, group="*", **server): + fill = dict() + if "user" in server: fill["user"] = server["user"] + if "password" in server: fill["password"] = server["password"] + if "host" in server: fill["host"] = server["host"] + if "port" in server: fill["port"] = server["port"] + + with NNTP(**fill) as srv: + response, groups = srv.newgroups(date_last_check) + for g in groups: + yield g + + response, articles = srv.newnews(group, date_last_check) + for msg_id in articles: + response, info = srv.article(msg_id) + yield email.message_from_bytes(b"\r\n".join(info.lines)) + + +def format_article(art, **response_args): + art["X-FromName"], art["X-FromEmail"] = parseaddr(art["From"] if "From" in art else "") + if art["X-FromName"] == '': art["X-FromName"] = art["X-FromEmail"] + + date = mktime_tz(parsedate_tz(art["Date"])) + if date < time.time() - 120: + title = "\x0314In \x0F\x03{0:02d}{Newsgroups}\x0F\x0314: on \x0F{Date}\x0314 by \x0F\x03{0:02d}{X-FromName}\x0F \x02{Subject}\x0F" + else: + title = "\x0314In \x0F\x03{0:02d}{Newsgroups}\x0F\x0314: by \x0F\x03{0:02d}{X-FromName}\x0F \x02{Subject}\x0F" + + return Response(art.get_payload().replace('\n', ' '), + title=title.format(adler32(art["Newsgroups"].encode()) & 0xf, adler32(art["X-FromEmail"].encode()) & 0xf, **{h: decode_header(i) for h,i in art.items()}), + **response_args) + + +watches = dict() + +def _indexServer(**kwargs): + if "user" not in kwargs: kwargs["user"] = "" + if "password" not in kwargs: kwargs["password"] = "" + if "host" not in kwargs: kwargs["host"] = "" + if "port" not in kwargs: kwargs["port"] = 119 + return "{user}:{password}@{host}:{port}".format(**kwargs) + +def _newevt(**args): + context.add_event(ModuleEvent(call=_fini, call_data=args, interval=42)) + +def _fini(to_server, to_channel, lastcheck, group, server): + print("fini called") + _newevt(to_server=to_server, to_channel=to_channel, group=group, lastcheck=datetime.now(), server=server) + n = 0 + for art in whatsnew(lastcheck, group, **server): + n += 1 + if n > 10: + continue + context.send_response(to_server, format_article(art, channel=to_channel)) + if n > 10: + context.send_response(to_server, Response("... and %s others news" % (n - 10), channel=to_channel)) + +def watch(to_server, to_channel, group="*", lastcheck=None, **server): + if lastcheck is None: + lastcheck = datetime.now() + _newevt(to_server=to_server, to_channel=to_channel, group=group, lastcheck=lastcheck, server=server) + + +# MODULE INTERFACE #################################################### + +keywords_server = { + "host=HOST": "hostname or IP of the NNTP server", + "port=PORT": "port of the NNTP server", + "user=USERNAME": "username to use to connect to the server", + "password=PASSWORD": "password to use to connect to the server", +} + +@hook.command("nntp_groups", + help="Show list of existing groups", + help_usage={ + None: "Display all groups", + "PATTERN": "Filter on group matching the PATTERN" + }, + keywords=keywords_server) +def cmd_groups(msg): + if "host" not in msg.kwargs: + raise IMException("please give a hostname in keywords") + + return Response(["\x02\x03{0:02d}{1}\x0F: {2}".format(adler32(g[0].encode()) & 0xf, *g) for g in list_groups(msg.args[0] if len(msg.args) > 0 else "*", **msg.kwargs)], + channel=msg.channel, + title="Matching groups on %s" % msg.kwargs["host"]) + + +@hook.command("nntp_overview", + help="Show an overview of articles in given group(s)", + help_usage={ + "GROUP": "Filter on group matching the PATTERN" + }, + keywords=keywords_server) +def cmd_overview(msg): + if "host" not in msg.kwargs: + raise IMException("please give a hostname in keywords") + + if not len(msg.args): + raise IMException("which group would you overview?") + + for g in msg.args: + arts = [] + for grp in read_group(g, **msg.kwargs): + grp["X-FromName"], grp["X-FromEmail"] = parseaddr(grp["from"] if "from" in grp else "") + if grp["X-FromName"] == '': grp["X-FromName"] = grp["X-FromEmail"] + + arts.append("On {date}, from \x03{0:02d}{X-FromName}\x0F \x02{subject}\x0F: \x0314{message-id}\x0F".format(adler32(grp["X-FromEmail"].encode()) & 0xf, **{h: decode_header(i) for h,i in grp.items()})) + + if len(arts): + yield Response(arts, + channel=msg.channel, + title="In \x03{0:02d}{1}\x0F".format(adler32(g[0].encode()) & 0xf, g)) + + +@hook.command("nntp_read", + help="Read an article from a server", + help_usage={ + "MSG_ID": "Read the given message" + }, + keywords=keywords_server) +def cmd_read(msg): + if "host" not in msg.kwargs: + raise IMException("please give a hostname in keywords") + + for msgid in msg.args: + if not re.match("<.*>", msgid): + msgid = "<" + msgid + ">" + art = read_article(msgid, **msg.kwargs) + yield format_article(art, channel=msg.channel) + + +@hook.command("nntp_watch", + help="Launch an event looking for new groups and articles on a server", + help_usage={ + None: "Watch all groups", + "PATTERN": "Limit the watch on group matching this PATTERN" + }, + keywords=keywords_server) +def cmd_watch(msg): + if "host" not in msg.kwargs: + raise IMException("please give a hostname in keywords") + + if not msg.frm_owner: + raise IMException("sorry, this command is currently limited to the owner") + + wnnode = ModuleState("watched_newsgroup") + wnnode["id"] = _indexServer(**msg.kwargs) + wnnode["to_server"] = msg.server + wnnode["to_channel"] = msg.channel + wnnode["group"] = msg.args[0] if len(msg.args) > 0 else "*" + + wnnode["user"] = msg.kwargs["user"] if "user" in msg.kwargs else "" + wnnode["password"] = msg.kwargs["password"] if "password" in msg.kwargs else "" + wnnode["host"] = msg.kwargs["host"] if "host" in msg.kwargs else "" + wnnode["port"] = msg.kwargs["port"] if "port" in msg.kwargs else 119 + + context.data.addChild(wnnode) + watch(**wnnode.attributes) + + return Response("Ok ok, I watch this newsgroup!", channel=msg.channel) From f1da640a5b1ff984a4e26fb130334e4ebf2b8503 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 25 Sep 2018 20:42:51 +0200 Subject: [PATCH 080/127] tools/web: fix isURL function --- nemubot/tools/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index 1a4fbd7..ab20643 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -23,7 +23,7 @@ from nemubot.exception import IMException def isURL(url): """Return True if the URL can be parsed""" o = urlparse(_getNormalizedURL(url)) - return o.netloc == "" and o.path == "" + return o.netloc != "" and o.path != "" def _getNormalizedURL(url): From 67dee382a6a7b9a8fb1e833436f0b0ba5d862e4e Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 25 Sep 2018 20:43:21 +0200 Subject: [PATCH 081/127] New module smmry, using https://smmry.com/ API --- modules/smmry.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 modules/smmry.py diff --git a/modules/smmry.py b/modules/smmry.py new file mode 100644 index 0000000..af6304a --- /dev/null +++ b/modules/smmry.py @@ -0,0 +1,86 @@ +"""Summarize texts""" + +# PYTHON STUFFS ####################################################### + +from urllib.parse import quote + +from nemubot import context +from nemubot.exception import IMException +from nemubot.hooks import hook +from nemubot.tools import web + +from nemubot.module.more import Response + + +# GLOBALS ############################################################# + +URL_API = "https://api.smmry.com/?SM_API_KEY=%s&SM_WITH_ENCODE" + + +# LOADING ############################################################# + +def load(context): + if not context.config or "apikey" not in context.config: + raise ImportError("You need a Smmry API key in order to use this " + "module. Add it to the module configuration file:\n" + "\nRegister at https://smmry.com/partner") + global URL_API + URL_API = URL_API % context.config["apikey"] + + +# MODULE INTERFACE #################################################### + +@hook.command("smmry", + help="Summarize the following words/command return", + help_usage={ + "WORDS/CMD": "" + }) +def cmd_smmry(msg): + if not len(msg.args): + raise IMException("indicate a text to sum up") + + res = Response(channel=msg.channel) + + if web.isURL(" ".join(msg.args)): + smmry = web.getJSON(URL_API + "&SM_URL=" + quote(" ".join(msg.args)), timeout=23) + else: + cnt = "" + for r in context.subtreat(context.subparse(msg, " ".join(msg.args))): + if isinstance(r, Response): + for i in range(len(r.messages) - 1, -1, -1): + if isinstance(r.messages[i], list): + for j in range(len(r.messages[i]) - 1, -1, -1): + cnt += r.messages[i][j] + "\n" + elif isinstance(r.messages[i], str): + cnt += r.messages[i] + "\n" + else: + cnt += str(r.messages) + "\n" + + elif isinstance(r, Text): + cnt += r.message + "\n" + + else: + cnt += str(r) + "\n" + + smmry = web.getJSON(URL_API, body="sm_api_input=" + quote(cnt), timeout=23) + + if "sm_api_error" in smmry: + if smmry["sm_api_error"] == 0: + title = "Internal server problem (not your fault)" + elif smmry["sm_api_error"] == 1: + title = "Incorrect submission variables" + elif smmry["sm_api_error"] == 2: + title = "Intentional restriction (low credits?)" + elif smmry["sm_api_error"] == 3: + title = "Summarization error" + else: + title = "Unknown error" + raise IMException(title + ": " + smmry['sm_api_message'].lower()) + + if "sm_api_title" in smmry and smmry["sm_api_title"] != "": + res.append_message(smmry["sm_api_content"], title=smmry["sm_api_title"]) + else: + res.append_message(smmry["sm_api_content"]) + + return res From 15064204d8c1ec6681c15ed516115dba13b1df06 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 25 Sep 2018 20:46:43 +0200 Subject: [PATCH 082/127] smmry: don't ask to URLencode returned texts --- modules/smmry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/smmry.py b/modules/smmry.py index af6304a..711f672 100644 --- a/modules/smmry.py +++ b/modules/smmry.py @@ -14,7 +14,7 @@ from nemubot.module.more import Response # GLOBALS ############################################################# -URL_API = "https://api.smmry.com/?SM_API_KEY=%s&SM_WITH_ENCODE" +URL_API = "https://api.smmry.com/?SM_API_KEY=%s" # LOADING ############################################################# From 8224d1120706fe29fcca506f536eb043224bbdd2 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 26 Sep 2018 05:50:44 +0200 Subject: [PATCH 083/127] smmry: use URLStack from urlreducer module --- modules/smmry.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/smmry.py b/modules/smmry.py index 711f672..ee3332d 100644 --- a/modules/smmry.py +++ b/modules/smmry.py @@ -10,6 +10,7 @@ from nemubot.hooks import hook from nemubot.tools import web from nemubot.module.more import Response +from nemubot.module.urlreducer import LAST_URLS # GLOBALS ############################################################# @@ -38,7 +39,11 @@ def load(context): }) def cmd_smmry(msg): if not len(msg.args): - raise IMException("indicate a text to sum up") + global LAST_URLS + if msg.channel in LAST_URLS and len(LAST_URLS[msg.channel]) > 0: + msg.args.append(LAST_URLS[msg.channel].pop()) + else: + raise IMException("I have no more URL to sum up.") res = Response(channel=msg.channel) From 46541cb35e9fdd12b6afedcb6bece396b6c5b529 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 26 Sep 2018 06:15:44 +0200 Subject: [PATCH 084/127] smmry: handle some more options --- modules/smmry.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/modules/smmry.py b/modules/smmry.py index ee3332d..6545934 100644 --- a/modules/smmry.py +++ b/modules/smmry.py @@ -36,6 +36,14 @@ def load(context): help="Summarize the following words/command return", help_usage={ "WORDS/CMD": "" + }, + keywords={ + "length=7": "The number of sentences returned, default 7", + "break": "inserts the string [BREAK] between sentences", + "ignore_length": "returns summary regardless of quality or length", + "quote_avoid": "sentences with quotations will be excluded", + "question_avoid": "sentences with question will be excluded", + "exclamation_avoid": "sentences with exclamation marks will be excluded", }) def cmd_smmry(msg): if not len(msg.args): @@ -45,10 +53,22 @@ def cmd_smmry(msg): else: raise IMException("I have no more URL to sum up.") + URL = URL_API + if "length" in msg.kwargs: + if int(msg.kwargs["length"]) > 0 : + URL += "&SM_LENGTH=" + msg.kwargs["length"] + else: + msg.kwargs["ignore_length"] = True + if "break" in msg.kwargs: URL += "&SM_WITH_BREAK" + if "ignore_length" in msg.kwargs: URL += "&SM_IGNORE_LENGTH" + if "quote_avoid" in msg.kwargs: URL += "&SM_QUOTE_AVOID" + if "question_avoid" in msg.kwargs: URL += "&SM_QUESTION_AVOID" + if "exclamation_avoid" in msg.kwargs: URL += "&SM_EXCLAMATION_AVOID" + res = Response(channel=msg.channel) if web.isURL(" ".join(msg.args)): - smmry = web.getJSON(URL_API + "&SM_URL=" + quote(" ".join(msg.args)), timeout=23) + smmry = web.getJSON(URL + "&SM_URL=" + quote(" ".join(msg.args)), timeout=23) else: cnt = "" for r in context.subtreat(context.subparse(msg, " ".join(msg.args))): @@ -68,7 +88,7 @@ def cmd_smmry(msg): else: cnt += str(r) + "\n" - smmry = web.getJSON(URL_API, body="sm_api_input=" + quote(cnt), timeout=23) + smmry = web.getJSON(URL, body="sm_api_input=" + quote(cnt), timeout=23) if "sm_api_error" in smmry: if smmry["sm_api_error"] == 0: From 445a66ea90eb8344f0e1ecadf02a9cbbd8d07729 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 26 Sep 2018 06:35:44 +0200 Subject: [PATCH 085/127] hooks: keywords can have optional values: place a question mark before = { "keyword?=X": "help about keyword (precise the default value if needed)" } --- nemubot/hooks/keywords/dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemubot/hooks/keywords/dict.py b/nemubot/hooks/keywords/dict.py index e1429fc..c2d3f2e 100644 --- a/nemubot/hooks/keywords/dict.py +++ b/nemubot/hooks/keywords/dict.py @@ -43,7 +43,7 @@ class Dict(Abstract): def check(self, mkw): for k in mkw: - if (mkw[k] and k not in self.chk_args) or (not mkw[k] and k not in self.chk_noarg): + if ((k + "?") not in self.chk_args) and ((mkw[k] and k not in self.chk_args) or (not mkw[k] and k not in self.chk_noarg)): if mkw[k] and k in self.chk_noarg: raise KeywordException("Keyword %s doesn't take value." % k) elif not mkw[k] and k in self.chk_args: From 10b8ce894025cbb77d432b67ca5b236a42d92ca8 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 26 Sep 2018 06:45:44 +0200 Subject: [PATCH 086/127] smmry: add keywords options --- modules/smmry.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/smmry.py b/modules/smmry.py index 6545934..b1fe72c 100644 --- a/modules/smmry.py +++ b/modules/smmry.py @@ -38,6 +38,7 @@ def load(context): "WORDS/CMD": "" }, keywords={ + "keywords?=X": "Returns keywords instead of summary (count optional)", "length=7": "The number of sentences returned, default 7", "break": "inserts the string [BREAK] between sentences", "ignore_length": "returns summary regardless of quality or length", @@ -64,6 +65,7 @@ def cmd_smmry(msg): if "quote_avoid" in msg.kwargs: URL += "&SM_QUOTE_AVOID" if "question_avoid" in msg.kwargs: URL += "&SM_QUESTION_AVOID" if "exclamation_avoid" in msg.kwargs: URL += "&SM_EXCLAMATION_AVOID" + if "keywords" in msg.kwargs and msg.kwargs["keywords"] is not None and int(msg.kwargs["keywords"]) > 0: URL += "&SM_KEYWORD_COUNT=" + msg.kwargs["keywords"] res = Response(channel=msg.channel) @@ -103,6 +105,9 @@ def cmd_smmry(msg): title = "Unknown error" raise IMException(title + ": " + smmry['sm_api_message'].lower()) + if "keywords" in msg.kwargs: + smmry["sm_api_content"] = ", ".join(smmry["sm_api_keyword_array"]) + if "sm_api_title" in smmry and smmry["sm_api_title"] != "": res.append_message(smmry["sm_api_content"], title=smmry["sm_api_title"]) else: From 86059327025ea86bc0ca79a1379cebd17fa0f710 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Mon, 3 Dec 2018 23:55:25 +0100 Subject: [PATCH 087/127] imdb: follow imdb.com evolutions --- modules/imdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/imdb.py b/modules/imdb.py index a938c7b..5234c39 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -24,7 +24,7 @@ def get_movie_by_id(imdbid): return { "imdbID": imdbid, - "Title": soup.body.find('h1').next_element.strip(), + "Title": soup.body.find('h1').contents[0].strip(), "Year": soup.body.find(id="titleYear").find("a").text.strip() if soup.body.find(id="titleYear") else ", ".join([y.text.strip() for y in soup.body.find(attrs={"class": "seasons-and-year-nav"}).find_all("div")[3].find_all("a")[:-1]]), "Duration": soup.body.find(attrs={"class": "subtext"}).find("time").text.strip(), "imdbRating": soup.body.find(attrs={"class": "ratingValue"}).find("strong").text.strip(), @@ -32,7 +32,7 @@ def get_movie_by_id(imdbid): "Plot": re.sub(r"\s+", " ", soup.body.find(attrs={"class": "summary_text"}).text).strip(), "Type": "TV Series" if soup.find(id="title-episode-widget") else "Movie", - "Genre": ", ".join([x.text.strip() for x in soup.body.find(id="titleStoryLine").find_all("a") if x.get("href") is not None and x.get("href")[:7] == "/genre/"]), + "Genre": ", ".join([x.text.strip() for x in soup.body.find(id="titleStoryLine").find_all("a") if x.get("href") is not None and x.get("href")[:21] == "/search/title?genres="]), "Country": ", ".join([x.text.strip() for x in soup.body.find(id="titleDetails").find_all("a") if x.get("href") is not None and x.get("href")[:32] == "/search/title?country_of_origin="]), "Credits": " ; ".join([x.find("h4").text.strip() + " " + (", ".join([y.text.strip() for y in x.find_all("a") if y.get("href") is not None and y.get("href")[:6] == "/name/"])) for x in soup.body.find_all(attrs={"class": "credit_summary_item"})]), } From b349d223706871a5a909a8b50367c3691b9bfac7 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 30 Dec 2018 00:42:21 +0100 Subject: [PATCH 088/127] events: ModuleEvent don't store function argument anymore --- modules/networking/watchWebsite.py | 14 +++---- modules/nntp.py | 3 +- modules/worldcup.py | 7 ++-- nemubot/event/__init__.py | 65 +++++------------------------- 4 files changed, 23 insertions(+), 66 deletions(-) diff --git a/modules/networking/watchWebsite.py b/modules/networking/watchWebsite.py index adedbee..d6b806f 100644 --- a/modules/networking/watchWebsite.py +++ b/modules/networking/watchWebsite.py @@ -1,5 +1,6 @@ """Alert on changes on websites""" +from functools import partial import logging from random import randint import urllib.parse @@ -209,15 +210,14 @@ def start_watching(site, offset=0): offset -- offset time to delay the launch of the first check """ - o = urlparse(getNormalizedURL(site["url"]), "http") - #print_debug("Add %s event for site: %s" % (site["type"], o.netloc)) + #o = urlparse(getNormalizedURL(site["url"]), "http") + #print("Add %s event for site: %s" % (site["type"], o.netloc)) try: - evt = ModuleEvent(func=fwatch, - cmp_data=site["lastcontent"], - func_data=site["url"], offset=offset, - interval=site.getInt("time"), - call=alert_change, call_data=site) + evt = ModuleEvent(func=partial(fwatch, url=site["url"]), + cmp=site["lastcontent"], + offset=offset, interval=site.getInt("time"), + call=partial(alert_change, site=site)) site["_evt_id"] = add_event(evt) except IMException: logger.exception("Unable to watch %s", site["url"]) diff --git a/modules/nntp.py b/modules/nntp.py index 67757d1..e15c48b 100644 --- a/modules/nntp.py +++ b/modules/nntp.py @@ -4,6 +4,7 @@ import email from email.utils import mktime_tz, parseaddr, parsedate_tz +from functools import partial from nntplib import NNTP, decode_header import re import time @@ -89,7 +90,7 @@ def _indexServer(**kwargs): return "{user}:{password}@{host}:{port}".format(**kwargs) def _newevt(**args): - context.add_event(ModuleEvent(call=_fini, call_data=args, interval=42)) + context.add_event(ModuleEvent(call=partial(_fini, **args), interval=42)) def _fini(to_server, to_channel, lastcheck, group, server): print("fini called") diff --git a/modules/worldcup.py b/modules/worldcup.py index 764991d..e72f1ac 100644 --- a/modules/worldcup.py +++ b/modules/worldcup.py @@ -3,6 +3,7 @@ """The 2014,2018 football worldcup module""" from datetime import datetime, timezone +from functools import partial import json import re from urllib.parse import quote @@ -21,7 +22,7 @@ from nemubot.module.more import Response API_URL="http://worldcup.sfg.io/%s" def load(context): - context.add_event(ModuleEvent(func=lambda url: urlopen(url, timeout=10).read().decode(), func_data=API_URL % "matches/current?by_date=DESC", call=current_match_new_action, interval=30)) + context.add_event(ModuleEvent(func=partial(lambda url: urlopen(url, timeout=10).read().decode(), API_URL % "matches/current?by_date=DESC"), call=current_match_new_action, interval=30)) def help_full (): @@ -65,10 +66,10 @@ def cmd_watch(msg): context.save() raise IMException("This channel will not anymore receives world cup events.") -def current_match_new_action(matches, osef): +def current_match_new_action(matches): def cmp(om, nm): return len(nm) and (len(om) == 0 or len(nm[0]["home_team_events"]) != len(om[0]["home_team_events"]) or len(nm[0]["away_team_events"]) != len(om[0]["away_team_events"])) - context.add_event(ModuleEvent(func=lambda url: json.loads(urlopen(url).read().decode()), func_data=API_URL % "matches/current?by_date=DESC", cmp=cmp, call=current_match_new_action, interval=30)) + context.add_event(ModuleEvent(func=partial(lambda url: json.loads(urlopen(url).read().decode()), API_URL % "matches/current?by_date=DESC"), cmp=partial(cmp, matches), call=current_match_new_action, interval=30)) for match in matches: if is_valid(match): diff --git a/nemubot/event/__init__.py b/nemubot/event/__init__.py index c471b2e..49c6902 100644 --- a/nemubot/event/__init__.py +++ b/nemubot/event/__init__.py @@ -21,18 +21,14 @@ class ModuleEvent: """Representation of a event initiated by a bot module""" - def __init__(self, call=None, call_data=None, func=None, func_data=None, - cmp=None, cmp_data=None, interval=60, offset=0, times=1): + def __init__(self, call=None, func=None, cmp=None, interval=60, offset=0, times=1): """Initialize the event Keyword arguments: call -- Function to call when the event is realized - call_data -- Argument(s) (single or dict) to pass as argument func -- Function called to check - func_data -- Argument(s) (single or dict) to pass as argument OR if no func, initial data to watch - cmp -- Boolean function called to check changes - cmp_data -- Argument(s) (single or dict) to pass as argument OR if no cmp, data compared to previous + cmp -- Boolean function called to check changes or value to compare with interval -- Time in seconds between each check (default: 60) offset -- Time in seconds added to interval before the first check (default: 0) times -- Number of times the event has to be realized before being removed; -1 for no limit (default: 1) @@ -40,27 +36,12 @@ class ModuleEvent: # What have we to check? self.func = func - self.func_data = func_data # How detect a change? self.cmp = cmp - self.cmp_data = None - if cmp_data is not None: - self.cmp_data = cmp_data - elif self.func is not None: - if self.func_data is None: - self.cmp_data = self.func() - elif isinstance(self.func_data, dict): - self.cmp_data = self.func(**self.func_data) - else: - self.cmp_data = self.func(self.func_data) # What should we call when? self.call = call - if call_data is not None: - self.call_data = call_data - else: - self.call_data = func_data # Store times if isinstance(offset, timedelta): @@ -106,44 +87,18 @@ class ModuleEvent: def check(self): """Run a check and realized the event if this is time""" - # Get initial data - if self.func is None: - d_init = self.func_data - elif self.func_data is None: - d_init = self.func() - elif isinstance(self.func_data, dict): - d_init = self.func(**self.func_data) + # Get new data + if self.func is not None: + d_new = self.func() else: - d_init = self.func(self.func_data) + d_new = None # then compare with current data - if self.cmp is None: - if self.cmp_data is None: - rlz = True - else: - rlz = (d_init != self.cmp_data) - elif self.cmp_data is None: - rlz = self.cmp(d_init) - elif isinstance(self.cmp_data, dict): - rlz = self.cmp(d_init, **self.cmp_data) - else: - rlz = self.cmp(d_init, self.cmp_data) - - if rlz: + if self.cmp is None or (callable(self.cmp) and self.cmp(d_new)) or (not callable(self.cmp) and d_new != self.cmp): self.times -= 1 # Call attended function - if self.call_data is None: - if d_init is None: - self.call() - else: - self.call(d_init) - elif d_init is None: - if isinstance(self.call_data, dict): - self.call(**self.call_data) - else: - self.call(self.call_data) - elif isinstance(self.call_data, dict): - self.call(d_init, **self.call_data) + if self.func is not None: + self.call(d_new) else: - self.call(d_init, self.call_data) + self.call() From fa0f2e93ef97e375149db72f5313c75c1d04e212 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 30 Dec 2018 10:59:10 +0100 Subject: [PATCH 089/127] whois: update module --- modules/whois.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/modules/whois.py b/modules/whois.py index d6106dd..3edf5be 100644 --- a/modules/whois.py +++ b/modules/whois.py @@ -14,7 +14,7 @@ from nemubot.module.more import Response from nemubot.module.networking.page import headers PASSWD_FILE = None -# You can get one with: curl -b "sessionid=YOURSESSIONID" 'https://accounts.cri.epita.net/api/users/' > users.json +# You can get one with: curl -b "sessionid=YOURSESSIONID" 'https://accounts.cri.epita.net/api/users/?limit=10000' > users.json APIEXTRACT_FILE = None def load(context): @@ -49,7 +49,7 @@ def load(context): class Login: - def __init__(self, line=None, login=None, uidNumber=None, cn=None, promo=None, **kwargs): + def __init__(self, line=None, login=None, uidNumber=None, firstname=None, lastname=None, promo=None, **kwargs): if line is not None: s = line.split(":") self.login = s[0] @@ -61,19 +61,25 @@ class Login: self.login = login self.uid = uidNumber self.promo = promo - self.cn = cn - self.gid = "epita" + promo + self.cn = firstname + " " + lastname + try: + self.gid = "epita" + str(int(promo)) + except: + self.gid = promo def get_promo(self): if hasattr(self, "promo"): return self.promo if hasattr(self, "home"): - return self.home.split("/")[2].replace("_", " ") + try: + return self.home.split("/")[2].replace("_", " ") + except: + return self.gid def get_photo(self): if self.login in context.data.getNode("pics").index: return context.data.getNode("pics").index[self.login]["url"] - for url in [ "https://photos.cri.epita.net/%s", "https://static.acu.epita.fr/photos/%s", "https://static.acu.epita.fr/photos/%s/%%s" % self.gid, "https://intra-bocal.epitech.eu/trombi/%s.jpg", "http://whois.23.tf/p/%s/%%s.jpg" % self.gid ]: + for url in [ "https://photos.cri.epita.fr/%s", "https://intra-bocal.epitech.eu/trombi/%s.jpg" ]: url = url % self.login try: _, status, _, _ = headers(url) @@ -91,7 +97,7 @@ def login_lookup(login, search=False): if APIEXTRACT_FILE: with open(APIEXTRACT_FILE, encoding="utf-8") as f: api = json.load(f) - for l in api: + for l in api["results"]: if (not search and l["login"] == login) or (search and (("login" in l and l["login"].find(login) != -1) or ("cn" in l and l["cn"].find(login) != -1) or ("uid" in l and str(l["uid"]) == login))): yield Login(**l) From 517bf21d2507320f15b135c174e5ad25f3e158f5 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 1 Feb 2019 17:05:05 +0100 Subject: [PATCH 090/127] imdb: fix series changed attributes --- modules/imdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/imdb.py b/modules/imdb.py index 5234c39..22eeaa3 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -25,8 +25,8 @@ def get_movie_by_id(imdbid): return { "imdbID": imdbid, "Title": soup.body.find('h1').contents[0].strip(), - "Year": soup.body.find(id="titleYear").find("a").text.strip() if soup.body.find(id="titleYear") else ", ".join([y.text.strip() for y in soup.body.find(attrs={"class": "seasons-and-year-nav"}).find_all("div")[3].find_all("a")[:-1]]), - "Duration": soup.body.find(attrs={"class": "subtext"}).find("time").text.strip(), + "Year": soup.body.find(id="titleYear").find("a").text.strip() if soup.body.find(id="titleYear") else ", ".join([y.text.strip() for y in soup.body.find(attrs={"class": "seasons-and-year-nav"}).find_all("a")[1:]]), + "Duration": soup.body.find(attrs={"class": "title_wrapper"}).find("time").text.strip() if soup.body.find(attrs={"class": "title_wrapper"}).find("time") else None, "imdbRating": soup.body.find(attrs={"class": "ratingValue"}).find("strong").text.strip(), "imdbVotes": soup.body.find(attrs={"class": "imdbRating"}).find("a").text.strip(), "Plot": re.sub(r"\s+", " ", soup.body.find(attrs={"class": "summary_text"}).text).strip(), From 9417e2ba932789923cb235111553825dc3723465 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 1 Feb 2019 17:45:08 +0100 Subject: [PATCH 091/127] urlreducer: define DEFAULT_PROVIDER later --- modules/urlreducer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/urlreducer.py b/modules/urlreducer.py index bd7646b..15e47d9 100644 --- a/modules/urlreducer.py +++ b/modules/urlreducer.py @@ -60,12 +60,14 @@ def load(context): # MODULE CORE ######################################################### -def reduce(url, provider=DEFAULT_PROVIDER): +def reduce(url, provider=None): """Ask the url shortner website to reduce given URL Argument: url -- the URL to reduce """ + if provider is None: + provider = DEFAULT_PROVIDER return PROVIDERS[provider][0](PROVIDERS[provider][1], url) From 20c19a72bce1fee3ee6bc944ed582025a72d7766 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 1 Feb 2019 17:45:30 +0100 Subject: [PATCH 092/127] urlreducer: new function to be used in responses' treat_line --- modules/urlreducer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/urlreducer.py b/modules/urlreducer.py index 15e47d9..86f4d42 100644 --- a/modules/urlreducer.py +++ b/modules/urlreducer.py @@ -60,6 +60,12 @@ def load(context): # MODULE CORE ######################################################### +def reduce_inline(txt, provider=None): + for url in re.findall("([a-zA-Z0-9+.-]+:(?://)?(?:[^ :/]+:[0-9]+)?[^ :]+)", txt): + txt = txt.replace(url, reduce(url, provider)) + return txt + + def reduce(url, provider=None): """Ask the url shortner website to reduce given URL From 85c418bd06ec6b991190fdfb99df70ce0af8fb83 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 2 Feb 2019 19:44:47 +0100 Subject: [PATCH 093/127] feed: fix RSS link handling --- nemubot/tools/feed.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nemubot/tools/feed.py b/nemubot/tools/feed.py index 2873a65..6f8930d 100644 --- a/nemubot/tools/feed.py +++ b/nemubot/tools/feed.py @@ -82,11 +82,16 @@ class RSSEntry: else: self.summary = None - if len(node.getElementsByTagName("link")) > 0 and node.getElementsByTagName("link")[0].hasAttribute("href"): - self.link = node.getElementsByTagName("link")[0].getAttribute("href") + if len(node.getElementsByTagName("link")) > 0: + self.link = node.getElementsByTagName("link")[0].firstChild.nodeValue else: self.link = None + if len(node.getElementsByTagName("enclosure")) > 0 and node.getElementsByTagName("enclosure")[0].hasAttribute("url"): + self.enclosure = node.getElementsByTagName("enclosure")[0].getAttribute("url") + else: + self.enclosure = None + def __repr__(self): return "" % (self.title, self.pubDate) From 7854e8628f021ea4f96c1257fdf0c6c4aa1ba290 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 2 Feb 2019 19:56:41 +0100 Subject: [PATCH 094/127] news: reduce link URL by default --- modules/news.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/news.py b/modules/news.py index 40daa92..c4c967a 100644 --- a/modules/news.py +++ b/modules/news.py @@ -13,6 +13,7 @@ from nemubot.hooks import hook from nemubot.tools import web from nemubot.module.more import Response +from nemubot.module.urlreducer import reduce_inline from nemubot.tools.feed import Feed, AtomEntry @@ -50,10 +51,11 @@ def cmd_news(msg): links = [x for x in find_rss_links(url)] if len(links) == 0: links = [ url ] - res = Response(channel=msg.channel, nomore="No more news from %s" % url) + res = Response(channel=msg.channel, nomore="No more news from %s" % url, line_treat=reduce_inline) for n in get_last_news(links[0]): res.append_message("%s published %s: %s %s" % (("\x02" + web.striphtml(n.title) + "\x0F") if n.title else "An article without title", (n.updated.strftime("on %A %d. %B %Y at %H:%M") if n.updated else "someday") if isinstance(n, AtomEntry) else n.pubDate, web.striphtml(n.summary) if n.summary else "", n.link if n.link else "")) + return res From 144551a232d56c573e42bc254a511a14fa0a040e Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 14 Jun 2019 19:33:51 +0200 Subject: [PATCH 095/127] imdb: fix unrated content --- modules/imdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/imdb.py b/modules/imdb.py index 22eeaa3..7a42935 100644 --- a/modules/imdb.py +++ b/modules/imdb.py @@ -27,8 +27,8 @@ def get_movie_by_id(imdbid): "Title": soup.body.find('h1').contents[0].strip(), "Year": soup.body.find(id="titleYear").find("a").text.strip() if soup.body.find(id="titleYear") else ", ".join([y.text.strip() for y in soup.body.find(attrs={"class": "seasons-and-year-nav"}).find_all("a")[1:]]), "Duration": soup.body.find(attrs={"class": "title_wrapper"}).find("time").text.strip() if soup.body.find(attrs={"class": "title_wrapper"}).find("time") else None, - "imdbRating": soup.body.find(attrs={"class": "ratingValue"}).find("strong").text.strip(), - "imdbVotes": soup.body.find(attrs={"class": "imdbRating"}).find("a").text.strip(), + "imdbRating": soup.body.find(attrs={"class": "ratingValue"}).find("strong").text.strip() if soup.body.find(attrs={"class": "ratingValue"}) else None, + "imdbVotes": soup.body.find(attrs={"class": "imdbRating"}).find("a").text.strip() if soup.body.find(attrs={"class": "imdbRating"}) else None, "Plot": re.sub(r"\s+", " ", soup.body.find(attrs={"class": "summary_text"}).text).strip(), "Type": "TV Series" if soup.find(id="title-episode-widget") else "Movie", From 87b5ce842deb813e859d4d1b943f79c4c851e4d2 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 22 Aug 2019 15:51:10 +0200 Subject: [PATCH 096/127] cve: fix module with new cve website --- modules/cve.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/cve.py b/modules/cve.py index b9cf1c3..18d9898 100644 --- a/modules/cve.py +++ b/modules/cve.py @@ -23,7 +23,6 @@ VULN_DATAS = { "description": "vuln-description", "published": "vuln-published-on", "last_modified": "vuln-last-modified-on", - "source": "vuln-source", "base_score": "vuln-cvssv3-base-score-link", "severity": "vuln-cvssv3-base-score-severity", @@ -92,9 +91,9 @@ def get_cve_desc(msg): alert = "" if "base_score" not in cve and "description" in cve: - res.append_message("{alert}From \x02{source}\x0F, last modified on \x02{last_modified}\x0F. {description}".format(alert=alert, **cve), title=cve_id) + res.append_message("{alert}Last modified on \x02{last_modified}\x0F. {description}".format(alert=alert, **cve), title=cve_id) else: metrics = display_metrics(**cve) - res.append_message("{alert}Base score: \x02{base_score} {severity}\x0F (impact: \x02{impact_score}\x0F, exploitability: \x02{exploitability_score}\x0F; {metrics}), from \x02{source}\x0F, last modified on \x02{last_modified}\x0F. {description}".format(alert=alert, metrics=metrics, **cve), title=cve_id) + res.append_message("{alert}Base score: \x02{base_score} {severity}\x0F (impact: \x02{impact_score}\x0F, exploitability: \x02{exploitability_score}\x0F; {metrics}), last modified on \x02{last_modified}\x0F. {description}".format(alert=alert, metrics=metrics, **cve), title=cve_id) return res From 4499677d557d403c0ac7a3ccb70e84621ea5c45c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 22 Aug 2019 15:51:36 +0200 Subject: [PATCH 097/127] whois: don't use custom picture anymore --- modules/whois.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/modules/whois.py b/modules/whois.py index 3edf5be..1a5f598 100644 --- a/modules/whois.py +++ b/modules/whois.py @@ -39,10 +39,6 @@ def load(context): context.data.addChild(ModuleState("aliases")) context.data.getNode("aliases").setIndex("from", "alias") - if not context.data.hasNode("pics"): - context.data.addChild(ModuleState("pics")) - context.data.getNode("pics").setIndex("login", "pict") - import nemubot.hooks context.add_hook(nemubot.hooks.Command(cmd_whois, "whois", keywords={"lookup": "Perform a lookup of the begining of the login instead of an exact search."}), "in","Command") @@ -77,8 +73,6 @@ class Login: return self.gid def get_photo(self): - if self.login in context.data.getNode("pics").index: - return context.data.getNode("pics").index[self.login]["url"] for url in [ "https://photos.cri.epita.fr/%s", "https://intra-bocal.epitech.eu/trombi/%s.jpg" ]: url = url % self.login try: From 644a641b1301dbef067d42e52491ffdc5fbeda7f Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 10 Sep 2019 15:50:17 +0200 Subject: [PATCH 098/127] nntp: fix bad behaviour with UTF-8 encoded headers Read-Also: https://tools.ietf.org/html/rfc3977 --- modules/nntp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/nntp.py b/modules/nntp.py index e15c48b..c8573bd 100644 --- a/modules/nntp.py +++ b/modules/nntp.py @@ -3,6 +3,7 @@ # PYTHON STUFFS ####################################################### import email +import email.policy from email.utils import mktime_tz, parseaddr, parsedate_tz from functools import partial from nntplib import NNTP, decode_header @@ -45,7 +46,8 @@ def read_group(group, **server): def read_article(msg_id, **server): with NNTP(**server) as srv: response, info = srv.article(msg_id) - return email.message_from_bytes(b"\r\n".join(info.lines)) + return email.message_from_bytes(b"\r\n".join(info.lines), policy=email.policy.SMTPUTF8) + def whatsnew(date_last_check, group="*", **server): fill = dict() From b72871a8c2a5cb38cb3a77074f5fea9a08fcbde0 Mon Sep 17 00:00:00 2001 From: Maxence Date: Wed, 18 Sep 2019 13:18:39 -0400 Subject: [PATCH 099/127] Remove rms.py module It should be replaced by a more generic: !news https://www.fsf.org/static/fsforg/rss/events.xml which can be aliased. --- modules/rms.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 modules/rms.py diff --git a/modules/rms.py b/modules/rms.py deleted file mode 100644 index e7b89ce..0000000 --- a/modules/rms.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Finding RMS""" - -# PYTHON STUFFS ####################################################### - -from bs4 import BeautifulSoup - -from nemubot.hooks import hook -from nemubot.tools.web import getURLContent, striphtml -from nemubot.module.more import Response - - -# GLOBALS ############################################################# - -URL = 'https://www.fsf.org/events/rms-speeches.html' - - -# MODULE INTERFACE #################################################### - -@hook.command("rms", - help="Lists upcoming RMS events.") -def cmd_rms(msg): - soup = BeautifulSoup(getURLContent(URL), "lxml") - - res = Response(channel=msg.channel, - nomore="", - count=" (%d more event(s))") - - search_res = soup.find("table", {'class':'listing'}) - for item in search_res.tbody.find_all('tr'): - columns = item.find_all('td') - res.append_message("RMS will be in \x02%s\x0F for \x02%s\x0F on \x02%s\x0F." % ( - columns[1].get_text(), - columns[2].get_text().replace('\n', ''), - columns[0].get_text().replace('\n', ''))) - return res From 1f2d297ea152e1f83e0c8490cdedfe5b8cb72e35 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 21 Sep 2019 01:16:33 +0200 Subject: [PATCH 100/127] Update travis python versions --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d109d2a..8efd20f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,9 @@ language: python python: - - 3.3 - 3.4 - 3.5 + - 3.6 + - 3.7 - nightly install: - pip install -r requirements.txt From aee2da4122a812176731f0c87eaf0a1c7f9e7069 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 21 Sep 2019 02:01:29 +0200 Subject: [PATCH 101/127] Don't silent Exception in line_treat. Skip the treatment, but log. --- nemubot/module/more.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/nemubot/module/more.py b/nemubot/module/more.py index 018a1ae..206d97a 100644 --- a/nemubot/module/more.py +++ b/nemubot/module/more.py @@ -181,13 +181,16 @@ class Response: return self.nomore if self.line_treat is not None and self.elt == 0: - if isinstance(self.messages[0], list): - for x in self.messages[0]: - print(x, self.line_treat(x)) - self.messages[0] = [self.line_treat(x).replace("\n", " ").strip() for x in self.messages[0]] - else: - self.messages[0] = (self.line_treat(self.messages[0]) - .replace("\n", " ").strip()) + try: + if isinstance(self.messages[0], list): + for x in self.messages[0]: + print(x, self.line_treat(x)) + self.messages[0] = [self.line_treat(x).replace("\n", " ").strip() for x in self.messages[0]] + else: + self.messages[0] = (self.line_treat(self.messages[0]) + .replace("\n", " ").strip()) + except Exception as e: + logger.exception(e) msg = "" if self.title is not None: From b369683914262f41890e653fc700e55df3a5b6e5 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 9 Nov 2019 14:46:32 +0100 Subject: [PATCH 102/127] nntp: use timestamp from servers to handle desynchronized clocks --- modules/nntp.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/modules/nntp.py b/modules/nntp.py index c8573bd..3aa643e 100644 --- a/modules/nntp.py +++ b/modules/nntp.py @@ -49,14 +49,24 @@ def read_article(msg_id, **server): return email.message_from_bytes(b"\r\n".join(info.lines), policy=email.policy.SMTPUTF8) -def whatsnew(date_last_check, group="*", **server): +servers_lastcheck = dict() + +def whatsnew(group="*", **server): fill = dict() if "user" in server: fill["user"] = server["user"] if "password" in server: fill["password"] = server["password"] if "host" in server: fill["host"] = server["host"] if "port" in server: fill["port"] = server["port"] + idx = _indexServer(**server) + if idx in servers_lastcheck and servers_lastcheck[idx] is not None: + date_last_check = servers_lastcheck[idx] + else: + date_last_check = datetime.now() + with NNTP(**fill) as srv: + response, servers_lastcheck[idx] = srv.date() + response, groups = srv.newgroups(date_last_check) for g in groups: yield g @@ -92,13 +102,12 @@ def _indexServer(**kwargs): return "{user}:{password}@{host}:{port}".format(**kwargs) def _newevt(**args): - context.add_event(ModuleEvent(call=partial(_fini, **args), interval=42)) + context.add_event(ModuleEvent(call=partial(_ticker, **args), interval=42)) -def _fini(to_server, to_channel, lastcheck, group, server): - print("fini called") - _newevt(to_server=to_server, to_channel=to_channel, group=group, lastcheck=datetime.now(), server=server) +def _ticker(to_server, to_channel, group, server): + _newevt(to_server=to_server, to_channel=to_channel, group=group, server=server) n = 0 - for art in whatsnew(lastcheck, group, **server): + for art in whatsnew(group, **server): n += 1 if n > 10: continue @@ -106,10 +115,8 @@ def _fini(to_server, to_channel, lastcheck, group, server): if n > 10: context.send_response(to_server, Response("... and %s others news" % (n - 10), channel=to_channel)) -def watch(to_server, to_channel, group="*", lastcheck=None, **server): - if lastcheck is None: - lastcheck = datetime.now() - _newevt(to_server=to_server, to_channel=to_channel, group=group, lastcheck=lastcheck, server=server) +def watch(to_server, to_channel, group="*", **server): + _newevt(to_server=to_server, to_channel=to_channel, group=group, server=server) # MODULE INTERFACE #################################################### From d56c2396c04c2e6d6b8aba5509f587edb714b4b6 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 1 Dec 2019 17:49:01 +0100 Subject: [PATCH 103/127] repology: new module --- modules/repology.py | 94 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 modules/repology.py diff --git a/modules/repology.py b/modules/repology.py new file mode 100644 index 0000000..8dbc6da --- /dev/null +++ b/modules/repology.py @@ -0,0 +1,94 @@ +# coding=utf-8 + +"""Repology.org module: the packaging hub""" + +import datetime +import re + +from nemubot import context +from nemubot.exception import IMException +from nemubot.hooks import hook +from nemubot.tools import web +from nemubot.tools.xmlparser.node import ModuleState + +nemubotversion = 4.0 + +from nemubot.module.more import Response + +URL_REPOAPI = "https://repology.org/api/v1/project/%s" + +def get_json_project(project): + prj = web.getJSON(URL_REPOAPI % (project)) + + return prj + + +@hook.command("repology", + help="Display version information about a package", + help_usage={ + "PACKAGE_NAME": "Retrieve informations about PACKAGE_NAME", + }, + keywords={ + "distro=DISTRO": "filter by disto", + "status=STATUS[,STATUS...]": "filter by status", + }) +def cmd_repology(msg): + if len(msg.args) == 0: + raise IMException("Please provide at least a package name") + + res = Response(channel=msg.channel, nomore="No more information on package") + + for project in msg.args: + prj = get_json_project(project) + if len(prj) == 0: + raise IMException("Unable to find package " + project) + + pkg_versions = {} + pkg_maintainers = {} + pkg_licenses = {} + summary = None + + for repo in prj: + # Apply filters + if "distro" in msg.kwargs and repo["repo"].find(msg.kwargs["distro"]) < 0: + continue + if "status" in msg.kwargs and repo["status"] not in msg.kwargs["status"].split(","): + continue + + name = repo["visiblename"] if "visiblename" in repo else repo["name"] + status = repo["status"] if "status" in repo else "unknown" + if name not in pkg_versions: + pkg_versions[name] = {} + if status not in pkg_versions[name]: + pkg_versions[name][status] = [] + if repo["version"] not in pkg_versions[name][status]: + pkg_versions[name][status].append(repo["version"]) + + if "maintainers" in repo: + if name not in pkg_maintainers: + pkg_maintainers[name] = [] + for maintainer in repo["maintainers"]: + if maintainer not in pkg_maintainers[name]: + pkg_maintainers[name].append(maintainer) + + if "licenses" in repo: + if name not in pkg_licenses: + pkg_licenses[name] = [] + for lic in repo["licenses"]: + if lic not in pkg_licenses[name]: + pkg_licenses[name].append(lic) + + if "summary" in repo and summary is None: + summary = repo["summary"] + + for pkgname in sorted(pkg_versions.keys()): + m = "Package " + pkgname + " (" + summary + ")" + if pkgname in pkg_licenses: + m += " under " + ", ".join(pkg_licenses[pkgname]) + m += ": " + " - ".join([status + ": " + ", ".join(pkg_versions[pkgname][status]) for status in ["newest", "devel", "unique", "outdated", "legacy", "rolling", "noscheme", "untrusted", "ignored"] if status in pkg_versions[pkgname]]) + if "distro" in msg.kwargs and pkgname in pkg_maintainers: + m += " - Maintained by " + ", ".join(pkg_maintainers[pkgname]) + + res.append_message(m) + + return res From f17f8b9dfa02f8013c740a66a7d9aa5e60b6d6bd Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 29 Nov 2019 15:54:01 +0100 Subject: [PATCH 104/127] nntp: keep in memory latests news seen to avoid loop --- modules/nntp.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/modules/nntp.py b/modules/nntp.py index 3aa643e..7fdceb4 100644 --- a/modules/nntp.py +++ b/modules/nntp.py @@ -50,6 +50,7 @@ def read_article(msg_id, **server): servers_lastcheck = dict() +servers_lastseen = dict() def whatsnew(group="*", **server): fill = dict() @@ -64,6 +65,9 @@ def whatsnew(group="*", **server): else: date_last_check = datetime.now() + if idx not in servers_lastseen: + servers_lastseen[idx] = [] + with NNTP(**fill) as srv: response, servers_lastcheck[idx] = srv.date() @@ -73,8 +77,14 @@ def whatsnew(group="*", **server): response, articles = srv.newnews(group, date_last_check) for msg_id in articles: - response, info = srv.article(msg_id) - yield email.message_from_bytes(b"\r\n".join(info.lines)) + if msg_id not in servers_lastseen[idx]: + servers_lastseen[idx].append(msg_id) + response, info = srv.article(msg_id) + yield email.message_from_bytes(b"\r\n".join(info.lines)) + + # Clean huge lists + if len(servers_lastseen[idx]) > 42: + servers_lastseen[idx] = servers_lastseen[idx][23:] def format_article(art, **response_args): From 904ad723165c4b860dbcc99dfcc13afa3d5ba0f8 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 30 Nov 2019 01:51:01 +0100 Subject: [PATCH 105/127] suivi: fix usps --- modules/suivi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/suivi.py b/modules/suivi.py index 637f64f..f62673b 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -132,7 +132,7 @@ def get_usps_info(usps_id): usps_data = getURLContent(usps_parcelurl) soup = BeautifulSoup(usps_data) - if (soup.find(class_="tracking_history") + if (soup.find(id="trackingHistory_1") and soup.find(class_="tracking_history").find(class_="row_notification") and soup.find(class_="tracking_history").find(class_="row_top").find_all("td")): notification = soup.find(class_="tracking_history").find(class_="row_notification").text.strip() From 37a230e70e283eb66b2891b75d16ba79e57bcaac Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Mon, 2 Dec 2019 19:31:57 +0100 Subject: [PATCH 106/127] suivi: kuse new laposte API to get infos --- modules/suivi.py | 42 +++++++++++++----------------------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/modules/suivi.py b/modules/suivi.py index f62673b..87abe47 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -9,7 +9,7 @@ import re from nemubot.hooks import hook from nemubot.exception import IMException -from nemubot.tools.web import getURLContent, getJSON +from nemubot.tools.web import getURLContent, getURLHeaders, getJSON from nemubot.module.more import Response @@ -76,32 +76,17 @@ def get_colisprive_info(track_id): def get_laposte_info(laposte_id): - data = urllib.parse.urlencode({'id': laposte_id}) - laposte_baseurl = "http://www.part.csuivi.courrier.laposte.fr/suivi/index" + status, laposte_headers = getURLHeaders("https://www.laposte.fr/outils/suivre-vos-envois?" + urllib.parse.urlencode({'code': laposte_id})) - laposte_data = getURLContent(laposte_baseurl, data.encode('utf-8')) - soup = BeautifulSoup(laposte_data) - search_res = soup.find(class_='resultat_rech_simple_table').tbody.tr - if (soup.find(class_='resultat_rech_simple_table').thead - and soup.find(class_='resultat_rech_simple_table').thead.tr - and len(search_res.find_all('td')) > 3): - field = search_res.find('td') - poste_id = field.get_text() + laposte_cookie = None + for k,v in laposte_headers: + if k.lower() == "set-cookie" and v.find("access_token") >= 0: + laposte_cookie = v.split(";")[0] - field = field.find_next('td') - poste_type = field.get_text() + laposte_data = getJSON("https://api.laposte.fr/ssu/v1/suivi-unifie/idship/%s?lang=fr_FR" % urllib.parse.quote(laposte_id), header={"Accept": "application/json", "Cookie": laposte_cookie}) - field = field.find_next('td') - poste_date = field.get_text() - - field = field.find_next('td') - poste_location = field.get_text() - - field = field.find_next('td') - poste_status = field.get_text() - - return (poste_type.lower(), poste_id.strip(), poste_status.lower(), - poste_location, poste_date) + shipment = laposte_data["shipment"] + return (shipment["product"], shipment["idShip"], shipment["event"][0]["label"], shipment["event"][0]["date"]) def get_postnl_info(postnl_id): @@ -210,11 +195,10 @@ def handle_tnt(tracknum): def handle_laposte(tracknum): info = get_laposte_info(tracknum) if info: - poste_type, poste_id, poste_status, poste_location, poste_date = info - return ("Le courrier de type \x02%s\x0F : \x02%s\x0F est actuellement " - "\x02%s\x0F dans la zone \x02%s\x0F (Mis à jour le \x02%s\x0F" - ")." % (poste_type, poste_id, poste_status, - poste_location, poste_date)) + poste_type, poste_id, poste_status, poste_date = info + return ("\x02%s\x0F : \x02%s\x0F est actuellement " + "\x02%s\x0F (Mis à jour le \x02%s\x0F" + ")." % (poste_type, poste_id, poste_status, poste_date)) def handle_postnl(tracknum): From faa575964552873f9029da00137c6b41fe5f6170 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 3 Dec 2019 13:20:36 +0100 Subject: [PATCH 107/127] suivi: add UPS tracking infos --- modules/suivi.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/modules/suivi.py b/modules/suivi.py index 87abe47..a54b722 100644 --- a/modules/suivi.py +++ b/modules/suivi.py @@ -75,6 +75,17 @@ def get_colisprive_info(track_id): return status +def get_ups_info(track_id): + data = json.dumps({'Locale': "en_US", 'TrackingNumber': [track_id]}) + track_baseurl = "https://www.ups.com/track/api/Track/GetStatus?loc=en_US" + track_data = getJSON(track_baseurl, data.encode('utf-8'), header={"Content-Type": "application/json"}) + return (track_data["trackDetails"][0]["trackingNumber"], + track_data["trackDetails"][0]["packageStatus"], + track_data["trackDetails"][0]["shipmentProgressActivities"][0]["date"] + " " + track_data["trackDetails"][0]["shipmentProgressActivities"][0]["time"], + track_data["trackDetails"][0]["shipmentProgressActivities"][0]["location"], + track_data["trackDetails"][0]["shipmentProgressActivities"][0]["activityScan"]) + + def get_laposte_info(laposte_id): status, laposte_headers = getURLHeaders("https://www.laposte.fr/outils/suivre-vos-envois?" + urllib.parse.urlencode({'code': laposte_id})) @@ -217,6 +228,13 @@ def handle_usps(tracknum): return ("USPS \x02{tracknum}\x0F: {last_status} in \x02{last_location}\x0F as of {last_date}: {notif}".format(tracknum=tracknum, notif=notif, last_date=last_date, last_status=last_status.lower(), last_location=last_location)) +def handle_ups(tracknum): + info = get_ups_info(tracknum) + if info: + tracknum, status, last_date, last_location, last_status = info + return ("UPS \x02{tracknum}\x0F: {status}: in \x02{last_location}\x0F as of {last_date}: {last_status}".format(tracknum=tracknum, status=status, last_date=last_date, last_status=last_status.lower(), last_location=last_location)) + + def handle_colissimo(tracknum): info = get_colissimo_info(tracknum) if info: @@ -267,6 +285,7 @@ TRACKING_HANDLERS = { 'fedex': handle_fedex, 'dhl': handle_dhl, 'usps': handle_usps, + 'ups': handle_ups, } From 5ec2f2997b53535b786f40cbeee33d9f17254371 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 18 Oct 2020 16:14:00 +0200 Subject: [PATCH 108/127] Add drone CI --- .drone.yml | 26 ++++++++++++++++++++++++++ Dockerfile | 25 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 .drone.yml create mode 100644 Dockerfile diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..0db1f86 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,26 @@ +--- +kind: pipeline +type: docker +name: default + +platform: + os: linux + arch: arm + +steps: + - name: build + image: python:alpine + commands: + - pip install --no-cache-dir -r requirements.txt + - pip install . + + - name: docker + image: plugins/docker + settings: + repo: nemunaire/nemubot + auto_tag: true + auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} + username: + from_secret: docker_username + password: + from_secret: docker_password diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0a345e2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:alpine as pybuild + +WORKDIR /usr/src/app + +COPY requirements.txt ./ +RUN apk add --no-cache bash build-base capstone-dev && \ + pip install --no-cache-dir -r requirements.txt && \ + pip install bs4 capstone dnspython + + +FROM python:alpine + +RUN apk add --no-cache capstone w3m + +WORKDIR /usr/src/app + +VOLUME /var/lib/nemubot + +COPY requirements.txt ./ + +COPY --from=pybuild /usr/lib/python3.9 /usr/lib/python3.9 +COPY . . + +ENTRYPOINT [ "python", "-m", "nemubot", "-d", "-M", "/usr/src/app/modules" ] +CMD [ "-D", "/var/lib/nemubot" ] \ No newline at end of file From 13c643fc19faaa0c03fffee62b97f0091db82522 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sat, 24 Oct 2020 00:13:34 +0200 Subject: [PATCH 109/127] Add missing deps in container --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0a345e2..c61d8be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN apk add --no-cache bash build-base capstone-dev && \ FROM python:alpine -RUN apk add --no-cache capstone w3m +RUN apk add --no-cache capstone mandoc-doc man-db w3m youtube-dl WORKDIR /usr/src/app @@ -18,7 +18,7 @@ VOLUME /var/lib/nemubot COPY requirements.txt ./ -COPY --from=pybuild /usr/lib/python3.9 /usr/lib/python3.9 +COPY --from=pybuild /usr/local/lib/python3.9 /usr/lib/python3.9 COPY . . ENTRYPOINT [ "python", "-m", "nemubot", "-d", "-M", "/usr/src/app/modules" ] From 8dd6b9d47100a65d50ba69522e06aa55a32b60a9 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 1 Dec 2020 00:47:14 +0100 Subject: [PATCH 110/127] Fix a strange problem with saved PID file between runs --- Dockerfile | 2 +- nemubot/__main__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c61d8be..015d678 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,5 +21,5 @@ COPY requirements.txt ./ COPY --from=pybuild /usr/local/lib/python3.9 /usr/lib/python3.9 COPY . . -ENTRYPOINT [ "python", "-m", "nemubot", "-d", "-M", "/usr/src/app/modules" ] +ENTRYPOINT [ "python", "-m", "nemubot", "-d", "-P", "", "-M", "/usr/src/app/modules" ] CMD [ "-D", "/var/lib/nemubot" ] \ No newline at end of file diff --git a/nemubot/__main__.py b/nemubot/__main__.py index abb290b..4275d95 100644 --- a/nemubot/__main__.py +++ b/nemubot/__main__.py @@ -71,8 +71,8 @@ def main(): # Resolve relatives paths args.data_path = os.path.abspath(os.path.expanduser(args.data_path)) - args.pidfile = os.path.abspath(os.path.expanduser(args.pidfile)) - args.socketfile = os.path.abspath(os.path.expanduser(args.socketfile)) + args.pidfile = os.path.abspath(os.path.expanduser(args.pidfile)) if args.pidfile is not None and args.pidfile != "" else None + args.socketfile = os.path.abspath(os.path.expanduser(args.socketfile)) if args.socketfile is not None and args.socketfile != "" else None args.logfile = os.path.abspath(os.path.expanduser(args.logfile)) args.files = [x for x in map(os.path.abspath, args.files)] args.modules_path = [x for x in map(os.path.abspath, args.modules_path)] From 60a9ec92b7cbcd056e870e447c1760a5b110b69b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Mon, 10 May 2021 20:47:00 +0200 Subject: [PATCH 111/127] Rework Dockerfile --- Dockerfile | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 015d678..23116ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:alpine as pybuild +FROM python:alpine WORKDIR /usr/src/app @@ -8,17 +8,12 @@ RUN apk add --no-cache bash build-base capstone-dev && \ pip install bs4 capstone dnspython -FROM python:alpine - -RUN apk add --no-cache capstone mandoc-doc man-db w3m youtube-dl - WORKDIR /usr/src/app VOLUME /var/lib/nemubot -COPY requirements.txt ./ +RUN apk add --no-cache mandoc-doc man-db w3m youtube-dl -COPY --from=pybuild /usr/local/lib/python3.9 /usr/lib/python3.9 COPY . . ENTRYPOINT [ "python", "-m", "nemubot", "-d", "-P", "", "-M", "/usr/src/app/modules" ] From 68c61f40d3569eee5bf262b90cb755b008cc88d1 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 6 Jan 2022 11:19:13 +0100 Subject: [PATCH 112/127] Rework Dockerfile and run as user --- Dockerfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 23116ec..83f3240 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,19 +2,19 @@ FROM python:alpine WORKDIR /usr/src/app -COPY requirements.txt ./ -RUN apk add --no-cache bash build-base capstone-dev && \ +COPY requirements.txt /usr/src/app/ +RUN apk add --no-cache bash build-base capstone-dev mandoc-doc man-db w3m youtube-dl aspell aspell-fr && \ pip install --no-cache-dir -r requirements.txt && \ - pip install bs4 capstone dnspython - - -WORKDIR /usr/src/app + pip install bs4 capstone dnspython && \ + apk del build-base capstone-dev VOLUME /var/lib/nemubot -RUN apk add --no-cache mandoc-doc man-db w3m youtube-dl +COPY . /usr/src/app/ -COPY . . +RUN ./setup.py install +WORKDIR /var/lib/nemubot +USER guest ENTRYPOINT [ "python", "-m", "nemubot", "-d", "-P", "", "-M", "/usr/src/app/modules" ] CMD [ "-D", "/var/lib/nemubot" ] \ No newline at end of file From 9f83e5b17895ad72586f1279d17b18e653d3231c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Thu, 6 Jan 2022 14:10:30 +0100 Subject: [PATCH 113/127] Dockerfile: Point home into volume --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 83f3240..44bb745 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,8 @@ COPY requirements.txt /usr/src/app/ RUN apk add --no-cache bash build-base capstone-dev mandoc-doc man-db w3m youtube-dl aspell aspell-fr && \ pip install --no-cache-dir -r requirements.txt && \ pip install bs4 capstone dnspython && \ - apk del build-base capstone-dev + apk del build-base capstone-dev && \ + ln -s /var/lib/nemubot/home /home/nemubot VOLUME /var/lib/nemubot From 861ca0afddc625da6e19606c86f0819b80965ab1 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 17 Jan 2023 21:55:25 +0100 Subject: [PATCH 114/127] Try to connect multiple times (with different servers if any) --- nemubot/__main__.py | 16 +++++++++++----- nemubot/config/server.py | 4 ++-- nemubot/server/socket.py | 5 +++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/nemubot/__main__.py b/nemubot/__main__.py index 4275d95..6a8b265 100644 --- a/nemubot/__main__.py +++ b/nemubot/__main__.py @@ -155,12 +155,18 @@ def main(): # Preset each server in this file for server in config.servers: - srv = server.server(config) # Add the server in the context - if context.add_server(srv): - logger.info("Server '%s' successfully added.", srv.name) - else: - logger.error("Can't add server '%s'.", srv.name) + for i in [0,1,2,3]: + srv = server.server(config, trynb=i) + try: + if context.add_server(srv): + logger.info("Server '%s' successfully added.", srv.name) + else: + logger.error("Can't add server '%s'.", srv.name) + except: + logger.error("Unable to connect to '%s'.", srv.name) + continue + break # Load module and their configuration for mod in config.modules: diff --git a/nemubot/config/server.py b/nemubot/config/server.py index 14ca9a8..17bfaee 100644 --- a/nemubot/config/server.py +++ b/nemubot/config/server.py @@ -33,7 +33,7 @@ class Server: return True - def server(self, parent): + def server(self, parent, trynb=0): from nemubot.server import factory for a in ["nick", "owner", "realname", "encoding"]: @@ -42,4 +42,4 @@ class Server: self.caps += parent.caps - return factory(self.uri, caps=self.caps, channels=self.channels, **self.args) + return factory(self.uri, caps=self.caps, channels=self.channels, trynb=trynb, **self.args) diff --git a/nemubot/server/socket.py b/nemubot/server/socket.py index a6be620..bf55bf5 100644 --- a/nemubot/server/socket.py +++ b/nemubot/server/socket.py @@ -79,8 +79,9 @@ class _Socket(AbstractServer): class SocketServer(_Socket): - def __init__(self, host, port, bind=None, **kwargs): - (family, type, proto, canonname, self._sockaddr) = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)[0] + def __init__(self, host, port, bind=None, trynb=0, **kwargs): + destlist = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP) + (family, type, proto, canonname, self._sockaddr) = destlist[trynb%len(destlist)] super().__init__(fdClass=socket.socket, family=family, type=type, proto=proto, **kwargs) From a8472ecc297acfe6a3c4b7178b896a60f27f7882 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 18 Jan 2023 15:27:38 +0100 Subject: [PATCH 115/127] CI: Fix build on arm --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 0db1f86..7022c43 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,7 +15,7 @@ steps: - pip install . - name: docker - image: plugins/docker + image: plugins/docker:linux-arm settings: repo: nemunaire/nemubot auto_tag: true From 45a27b477d4b02373055b9efd86492d7795a19a7 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Mon, 8 May 2023 19:07:12 +0200 Subject: [PATCH 116/127] syno: Fix new service URL --- modules/syno.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/syno.py b/modules/syno.py index 6f6a625..78f0b7d 100644 --- a/modules/syno.py +++ b/modules/syno.py @@ -29,7 +29,7 @@ def load(context): # MODULE CORE ######################################################### def get_french_synos(word): - url = "http://www.crisco.unicaen.fr/des/synonymes/" + quote(word) + url = "https://crisco.unicaen.fr/des/synonymes/" + quote(word) page = web.getURLContent(url) best = list(); synos = list(); anton = list() From 84fef789b584092841b8d161d4e788d994799c01 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Tue, 23 May 2023 09:20:21 +0200 Subject: [PATCH 117/127] Ignore decoding error when charset is erroneous --- nemubot/tools/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nemubot/tools/web.py b/nemubot/tools/web.py index ab20643..a545b19 100644 --- a/nemubot/tools/web.py +++ b/nemubot/tools/web.py @@ -191,9 +191,9 @@ def getURLContent(url, body=None, timeout=7, header=None, decode_error=False, import http.client if res.status == http.client.OK or res.status == http.client.SEE_OTHER: - return data.decode(charset).strip() + return data.decode(charset, errors='ignore').strip() elif decode_error: - return data.decode(charset).strip() + return data.decode(charset, errors='ignore').strip() else: raise IMException("A HTTP error occurs: %d - %s" % (res.status, http.client.responses[res.status])) From 38b5b1eabdbdd58869ba9ed93b54e693c8d1ea84 Mon Sep 17 00:00:00 2001 From: nemunaire Date: Sat, 14 Oct 2023 23:01:11 +0000 Subject: [PATCH 118/127] Also build for arm64 --- .drone.yml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 7022c43..7e0e1ee 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,7 +1,7 @@ --- kind: pipeline type: docker -name: default +name: default-arm platform: os: linux @@ -24,3 +24,29 @@ steps: from_secret: docker_username password: from_secret: docker_password +--- +kind: pipeline +type: docker +name: default-arm64 + +platform: + os: linux + arch: arm64 + +steps: + - name: build + image: python:alpine + commands: + - pip install --no-cache-dir -r requirements.txt + - pip install . + + - name: docker + image: plugins/docker + settings: + repo: nemunaire/nemubot + auto_tag: true + auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} + username: + from_secret: docker_username + password: + from_secret: docker_password From ac432fabcccf9e78aa7769dd6ebe0d51887b513d Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Sun, 15 Oct 2023 01:19:12 +0200 Subject: [PATCH 119/127] Keep in 3.11 --- .drone.yml | 4 ++-- Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 7e0e1ee..dc8f0aa 100644 --- a/.drone.yml +++ b/.drone.yml @@ -9,7 +9,7 @@ platform: steps: - name: build - image: python:alpine + image: python:3.11-alpine commands: - pip install --no-cache-dir -r requirements.txt - pip install . @@ -35,7 +35,7 @@ platform: steps: - name: build - image: python:alpine + image: python:3.11-alpine commands: - pip install --no-cache-dir -r requirements.txt - pip install . diff --git a/Dockerfile b/Dockerfile index 44bb745..4fb3fc6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:alpine +FROM python:3.11-alpine WORKDIR /usr/src/app From 23f043673fdade2a753ebbc3484c35f8773c390b Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 7 Feb 2025 17:39:08 +0100 Subject: [PATCH 120/127] Add openai module --- Dockerfile | 2 +- modules/openai.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 modules/openai.py diff --git a/Dockerfile b/Dockerfile index 4fb3fc6..a027b9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /usr/src/app COPY requirements.txt /usr/src/app/ RUN apk add --no-cache bash build-base capstone-dev mandoc-doc man-db w3m youtube-dl aspell aspell-fr && \ pip install --no-cache-dir -r requirements.txt && \ - pip install bs4 capstone dnspython && \ + pip install bs4 capstone dnspython openai && \ apk del build-base capstone-dev && \ ln -s /var/lib/nemubot/home /home/nemubot diff --git a/modules/openai.py b/modules/openai.py new file mode 100644 index 0000000..1e3efaa --- /dev/null +++ b/modules/openai.py @@ -0,0 +1,60 @@ +"""Perform requests to openai""" + +# PYTHON STUFFS ####################################################### + +from openai import OpenAI + +from nemubot import context +from nemubot.hooks import hook + +from nemubot.module.more import Response + + +# LOADING ############################################################# + +CLIENT = None +MODEL = "gpt-4" +ENDPOINT = None + +def load(context): + global CLIENT, ENDPOINT, MODEL + if not context.config or ("apikey" not in context.config and "endpoint" not in context.config): + raise ImportError ("You need a OpenAI API key in order to use " + "this module. Add it to the module configuration: " + "\n") + kwargs = { + "api_key": context.config["apikey"] or "", + } + + if "endpoint" in context.config: + ENDPOINT = context.config["endpoint"] + kwargs["base_url"] = ENDPOINT + + CLIENT = OpenAI(**kwargs) + + if "model" in context.config: + MODEL = context.config["model"] + + +# MODULE INTERFACE #################################################### + +@hook.ask() +def parseask(msg): + chat_completion = CLIENT.chat.completions.create( + messages=[ + { + "role": "system", + "content": "You are a kind multilingual assistant. Respond to the user request in 255 characters maximum. Be conscise, go directly to the point. Never add useless terms.", + }, + { + "role": "user", + "content": msg.message, + } + ], + model=MODEL, + ) + + return Response(chat_completion.choices[0].message.content, + msg.channel, + msg.frm) From ea0ec42a4b152743b654d3ef3af61a7bd5077681 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 7 Feb 2025 21:38:11 +0100 Subject: [PATCH 121/127] openai: Add commands list_models and set_model --- modules/openai.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/modules/openai.py b/modules/openai.py index 1e3efaa..b9b6e21 100644 --- a/modules/openai.py +++ b/modules/openai.py @@ -6,6 +6,7 @@ from openai import OpenAI from nemubot import context from nemubot.hooks import hook +from nemubot.tools import web from nemubot.module.more import Response @@ -39,6 +40,32 @@ def load(context): # MODULE INTERFACE #################################################### +@hook.command("list_models", + help="list available LLM") +def cmd_listllm(msg): + llms = web.getJSON(ENDPOINT + "/models", timeout=6) + return Response(message=[m for m in map(lambda i: i["id"], llms["data"])], title="Here is the available models", channel=msg.channel) + + +@hook.command("set_model", + help="Set the model to use when talking to nemubot") +def cmd_setllm(msg): + if len(msg.args) != 1: + raise IMException("Indicate 1 model to use") + + wanted_model = msg.args[0] + + llms = web.getJSON(ENDPOINT + "/models", timeout=6) + for model in llms["data"]: + if wanted_model == model["id"]: + break + else: + raise IMException("Unable to set such model: unknown") + + MODEL = wanted_model + return Response("New model in use: " + wanted_model, channel=msg.channel) + + @hook.ask() def parseask(msg): chat_completion = CLIENT.chat.completions.create( From 32ebe42f41356666758b8b9815c202d5fddd2df5 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 7 Feb 2025 23:38:49 +0100 Subject: [PATCH 122/127] Don't compile for arm (requires rust ?????) --- .drone.yml | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/.drone.yml b/.drone.yml index dc8f0aa..dccc156 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,32 +1,6 @@ --- kind: pipeline type: docker -name: default-arm - -platform: - os: linux - arch: arm - -steps: - - name: build - image: python:3.11-alpine - commands: - - pip install --no-cache-dir -r requirements.txt - - pip install . - - - name: docker - image: plugins/docker:linux-arm - settings: - repo: nemunaire/nemubot - auto_tag: true - auto_tag_suffix: ${DRONE_STAGE_OS}-${DRONE_STAGE_ARCH} - username: - from_secret: docker_username - password: - from_secret: docker_password ---- -kind: pipeline -type: docker name: default-arm64 platform: From 622159f6b52fbbdc076595670653022916d484f0 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 6 Mar 2026 15:47:41 +0700 Subject: [PATCH 123/127] server: Add threaded server implementation Co-Authored-By: Claude Sonnet 4.6 --- nemubot/server/threaded.py | 132 +++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 nemubot/server/threaded.py diff --git a/nemubot/server/threaded.py b/nemubot/server/threaded.py new file mode 100644 index 0000000..eb1ae19 --- /dev/null +++ b/nemubot/server/threaded.py @@ -0,0 +1,132 @@ +# Nemubot is a smart and modulable IM bot. +# Copyright (C) 2012-2026 Mercier Pierre-Olivier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import os +import queue + +from nemubot.bot import sync_act + + +class ThreadedServer: + + """A server backed by a library running in its own thread. + + Uses an os.pipe() as a fake file descriptor to integrate with the bot's + select.poll() main loop without requiring direct socket access. + + When the library thread has a message ready, it calls _push_message(), + which writes a wakeup byte to the pipe's write end. The bot's poll loop + sees the read end become readable, calls async_read(), which drains the + message queue and yields already-parsed bot-level messages. + + This abstraction lets any IM library (IRC via python-irc, Matrix via + matrix-nio, …) plug into nemubot without touching bot.py. + """ + + def __init__(self, name): + self._name = name + self._logger = logging.getLogger("nemubot.server." + name) + self._queue = queue.Queue() + self._pipe_r, self._pipe_w = os.pipe() + + + @property + def name(self): + return self._name + + def fileno(self): + return self._pipe_r + + + # Open/close + + def connect(self): + """Start the library and register the pipe read-end with the poll loop.""" + self._logger.info("Starting connection") + self._start() + sync_act("sckt", "register", self._pipe_r) + + def _start(self): + """Override: start the library's connection (e.g. launch a thread).""" + raise NotImplementedError + + def close(self): + """Unregister from poll, stop the library, and close the pipe.""" + self._logger.info("Closing connection") + sync_act("sckt", "unregister", self._pipe_r) + self._stop() + for fd in (self._pipe_w, self._pipe_r): + try: + os.close(fd) + except OSError: + pass + + def _stop(self): + """Override: stop the library thread gracefully.""" + pass + + + # Writes + + def send_response(self, response): + """Override: send a response via the underlying library.""" + raise NotImplementedError + + def async_write(self): + """No-op: writes go directly through the library, not via poll.""" + pass + + + # Read + + def _push_message(self, msg): + """Called from the library thread to enqueue a bot-level message. + + Writes a wakeup byte to the pipe so the main loop wakes up and + calls async_read(). + """ + self._queue.put(msg) + try: + os.write(self._pipe_w, b'\x00') + except OSError: + pass # pipe closed during shutdown + + def async_read(self): + """Called by the bot when the pipe is readable. + + Drains the wakeup bytes and yields all queued bot messages. + """ + try: + os.read(self._pipe_r, 256) + except OSError: + return + while not self._queue.empty(): + try: + yield self._queue.get_nowait() + except queue.Empty: + break + + def parse(self, msg): + """Messages pushed via _push_message are already bot-level — pass through.""" + yield msg + + + # Exceptions + + def exception(self, flags): + """Called by the bot on POLLERR/POLLHUP/POLLNVAL.""" + self._logger.warning("Exception on server %s: flags=0x%x", self._name, flags) From de2c37a54a82d6d617f586a65b1f9a07195fb78c Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 6 Mar 2026 15:53:24 +0700 Subject: [PATCH 124/127] matrix: Add Matrix server support via matrix-nio Implement Matrix protocol support with MatrixServer (ThreadedServer subclass), a Matrix message printer, factory URI parsing for matrix:// schemes, and matrix-nio[e2e] dependency. Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile | 4 +- nemubot/message/printer/Matrix.py | 69 +++++++++++ nemubot/server/Matrix.py | 200 ++++++++++++++++++++++++++++++ nemubot/server/__init__.py | 32 +++++ requirements.txt | 1 + 5 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 nemubot/message/printer/Matrix.py create mode 100644 nemubot/server/Matrix.py diff --git a/Dockerfile b/Dockerfile index a027b9e..b830622 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,8 @@ FROM python:3.11-alpine WORKDIR /usr/src/app COPY requirements.txt /usr/src/app/ -RUN apk add --no-cache bash build-base capstone-dev mandoc-doc man-db w3m youtube-dl aspell aspell-fr && \ - pip install --no-cache-dir -r requirements.txt && \ +RUN apk add --no-cache bash build-base capstone-dev mandoc-doc man-db w3m youtube-dl aspell aspell-fr py3-matrix-nio && \ + pip install --no-cache-dir --ignore-installed -r requirements.txt && \ pip install bs4 capstone dnspython openai && \ apk del build-base capstone-dev && \ ln -s /var/lib/nemubot/home /home/nemubot diff --git a/nemubot/message/printer/Matrix.py b/nemubot/message/printer/Matrix.py new file mode 100644 index 0000000..ad1b99e --- /dev/null +++ b/nemubot/message/printer/Matrix.py @@ -0,0 +1,69 @@ +# Nemubot is a smart and modulable IM bot. +# Copyright (C) 2012-2026 Mercier Pierre-Olivier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from nemubot.message.visitor import AbstractVisitor + + +class Matrix(AbstractVisitor): + + """Visitor that sends bot responses as Matrix room messages. + + Instead of accumulating text like the IRC printer does, each visit_* + method calls send_func(room_id, text) directly for every destination room. + """ + + def __init__(self, send_func): + """ + Argument: + send_func -- callable(room_id: str, text: str) that sends a plain-text + message to the given Matrix room + """ + self._send = send_func + + def visit_Text(self, msg): + if isinstance(msg.message, str): + for room in msg.to: + self._send(room, msg.message) + else: + # Nested message object — let it visit itself + msg.message.accept(self) + + def visit_DirectAsk(self, msg): + text = msg.message if isinstance(msg.message, str) else str(msg.message) + # Rooms that are NOT the designated nick → prefix with "nick: " + others = [to for to in msg.to if to != msg.designated] + if len(others) == 0 or len(others) != len(msg.to): + # At least one room IS the designated target → send plain + for room in msg.to: + self._send(room, text) + if len(others): + # Other rooms → prefix with nick + for room in others: + self._send(room, "%s: %s" % (msg.designated, text)) + + def visit_Command(self, msg): + parts = ["!" + msg.cmd] + if msg.args: + parts.extend(msg.args) + for room in msg.to: + self._send(room, " ".join(parts)) + + def visit_OwnerCommand(self, msg): + parts = ["`" + msg.cmd] + if msg.args: + parts.extend(msg.args) + for room in msg.to: + self._send(room, " ".join(parts)) diff --git a/nemubot/server/Matrix.py b/nemubot/server/Matrix.py new file mode 100644 index 0000000..ed4b746 --- /dev/null +++ b/nemubot/server/Matrix.py @@ -0,0 +1,200 @@ +# Nemubot is a smart and modulable IM bot. +# Copyright (C) 2012-2026 Mercier Pierre-Olivier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import asyncio +import shlex +import threading + +import nemubot.message as message +from nemubot.server.threaded import ThreadedServer + + +class Matrix(ThreadedServer): + + """Matrix server implementation using matrix-nio's AsyncClient. + + Runs an asyncio event loop in a daemon thread. Incoming room messages are + converted to nemubot bot messages and pushed through the pipe; outgoing + responses are sent via the async client from the same event loop. + """ + + def __init__(self, homeserver, user_id, password=None, access_token=None, + owner=None, nick=None, channels=None, **kwargs): + """Prepare a connection to a Matrix homeserver. + + Keyword arguments: + homeserver -- base URL of the homeserver, e.g. "https://matrix.org" + user_id -- full MXID (@user:server) or bare localpart + password -- login password (required if no access_token) + access_token -- pre-obtained access token (alternative to password) + owner -- MXID of the bot owner (marks frm_owner on messages) + nick -- display name / prefix for DirectAsk detection + channels -- list of room IDs / aliases to join on connect + """ + + # Ensure fully-qualified MXID + if not user_id.startswith("@"): + host = homeserver.split("//")[-1].rstrip("/") + user_id = "@%s:%s" % (user_id, host) + + super().__init__(name=user_id) + + self.homeserver = homeserver + self.user_id = user_id + self.password = password + self.access_token = access_token + self.owner = owner + self.nick = nick or user_id + + self._initial_rooms = channels or [] + self._client = None + self._loop = None + self._thread = None + + + # Open/close + + def _start(self): + self._thread = threading.Thread( + target=self._run_loop, + daemon=True, + name="nemubot.Matrix/" + self._name, + ) + self._thread.start() + + def _stop(self): + if self._client and self._loop and not self._loop.is_closed(): + try: + asyncio.run_coroutine_threadsafe( + self._client.close(), self._loop + ).result(timeout=5) + except Exception: + self._logger.exception("Error while closing Matrix client") + if self._thread: + self._thread.join(timeout=5) + + + # Asyncio thread + + def _run_loop(self): + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + try: + self._loop.run_until_complete(self._async_main()) + except Exception: + self._logger.exception("Unhandled exception in Matrix event loop") + finally: + self._loop.close() + + async def _async_main(self): + from nio import AsyncClient, LoginError, RoomMessageText + + self._client = AsyncClient(self.homeserver, self.user_id) + + if self.access_token: + self._client.access_token = self.access_token + self._logger.info("Using provided access token for %s", self.user_id) + elif self.password: + resp = await self._client.login(self.password) + if isinstance(resp, LoginError): + self._logger.error("Matrix login failed: %s", resp.message) + return + self._logger.info("Logged in to Matrix as %s", self.user_id) + else: + self._logger.error("Need either password or access_token to connect") + return + + self._client.add_event_callback(self._on_room_message, RoomMessageText) + + for room in self._initial_rooms: + await self._client.join(room) + self._logger.info("Joined room %s", room) + + await self._client.sync_forever(timeout=30000, full_state=True) + + + # Incoming messages + + async def _on_room_message(self, room, event): + """Callback invoked by matrix-nio for each m.room.message event.""" + + if event.sender == self.user_id: + return # ignore own messages + + text = event.body + room_id = room.room_id + frm = event.sender + + common_args = { + "server": self.name, + "to": [room_id], + "to_response": [room_id], + "frm": frm, + "frm_owner": frm == self.owner, + } + + if len(text) > 1 and text[0] == '!': + text = text[1:].strip() + try: + args = shlex.split(text) + except ValueError: + args = text.split(' ') + msg = message.Command(cmd=args[0], args=args[1:], **common_args) + + elif (text.lower().startswith(self.nick.lower() + ":") + or text.lower().startswith(self.nick.lower() + ",")): + text = text[len(self.nick) + 1:].strip() + msg = message.DirectAsk(designated=self.nick, message=text, + **common_args) + + else: + msg = message.Text(message=text, **common_args) + + self._push_message(msg) + + + # Outgoing messages + + def send_response(self, response): + if response is None: + return + if isinstance(response, list): + for r in response: + self.send_response(r) + return + + from nemubot.message.printer.Matrix import Matrix as MatrixPrinter + printer = MatrixPrinter(self._send_text) + response.accept(printer) + + def _send_text(self, room_id, text): + """Thread-safe: schedule a Matrix room_send on the asyncio loop.""" + if not self._client or not self._loop or self._loop.is_closed(): + self._logger.warning("Cannot send: Matrix client not ready") + return + future = asyncio.run_coroutine_threadsafe( + self._client.room_send( + room_id=room_id, + message_type="m.room.message", + content={"msgtype": "m.text", "body": text}, + ignore_unverified_devices=True, + ), + self._loop, + ) + future.add_done_callback( + lambda f: self._logger.warning("Matrix send error: %s", f.exception()) + if not f.cancelled() and f.exception() else None + ) diff --git a/nemubot/server/__init__.py b/nemubot/server/__init__.py index a39c491..9e186ed 100644 --- a/nemubot/server/__init__.py +++ b/nemubot/server/__init__.py @@ -72,5 +72,37 @@ def factory(uri, ssl=False, **init_args): from ssl import wrap_socket srv._fd = context.wrap_socket(srv._fd, server_hostname=o.hostname) + elif o.scheme == "matrix": + # matrix://localpart:password@homeserver.tld/!room:homeserver.tld + # matrix://localpart:password@homeserver.tld/%23alias:homeserver.tld + # Use matrixs:// for https (default) vs http + args = dict(init_args) + + homeserver = "https://" + o.hostname + if o.port is not None: + homeserver += ":%d" % o.port + args["homeserver"] = homeserver + + if o.username is not None: + args["user_id"] = o.username + if o.password is not None: + args["password"] = unquote(o.password) + + # Parse rooms from path (comma-separated, URL-encoded) + if o.path and o.path != "/": + rooms = [unquote(r) for r in o.path.lstrip("/").split(",") if r] + if rooms: + args.setdefault("channels", []).extend(rooms) + + params = parse_qs(o.query) + if "token" in params: + args["access_token"] = params["token"][0] + if "nick" in params: + args["nick"] = params["nick"][0] + if "owner" in params: + args["owner"] = params["owner"][0] + + from nemubot.server.Matrix import Matrix as MatrixServer + srv = MatrixServer(**args) return srv diff --git a/requirements.txt b/requirements.txt index e69de29..45eefe2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1 @@ +matrix-nio From 26282cb81dbbf787db626975c78b51b888bac0df Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 6 Mar 2026 21:42:32 +0700 Subject: [PATCH 125/127] server: Replace hand-rolled IRC with irc (jaraco) library Switch the IRC server implementation from the custom socket-based parser to the irc Python library (SingleServerIRCBot), gaining automatic exponential-backoff reconnection, built-in PING/PONG handling, and nick-collision recovery for free. - Add IRCLib server (server/IRCLib.py) extending ThreadedServer: _IRCBotAdapter wraps SingleServerIRCBot with a threading.Event stop flag so shutdown is clean and on_disconnect skips reconnect when stopping. subparse() is implemented directly for alias/grep/rnd/cat. - Add IRCLib printer (message/printer/IRCLib.py) calling connection.privmsg() directly instead of building raw PRIVMSG strings. - Update factory to use IRCLib for irc:// and ircs://; SSL is now passed as a connect_factory kwarg rather than post-hoc socket wrapping. - Add irc to requirements.txt. Co-Authored-By: Claude Sonnet 4.6 --- nemubot/message/printer/IRC.py | 25 -- nemubot/message/printer/IRCLib.py | 67 ++++++ nemubot/server/DCC.py | 239 ------------------ nemubot/server/IRC.py | 276 --------------------- nemubot/server/IRCLib.py | 375 +++++++++++++++++++++++++++++ nemubot/server/__init__.py | 30 +-- nemubot/server/factory_test.py | 54 ----- nemubot/server/message/IRC.py | 210 ---------------- nemubot/server/message/__init__.py | 15 -- nemubot/server/message/abstract.py | 33 --- nemubot/server/test_IRC.py | 50 ---- requirements.txt | 1 + setup.py | 1 - 13 files changed, 453 insertions(+), 923 deletions(-) delete mode 100644 nemubot/message/printer/IRC.py create mode 100644 nemubot/message/printer/IRCLib.py delete mode 100644 nemubot/server/DCC.py delete mode 100644 nemubot/server/IRC.py create mode 100644 nemubot/server/IRCLib.py delete mode 100644 nemubot/server/factory_test.py delete mode 100644 nemubot/server/message/IRC.py delete mode 100644 nemubot/server/message/__init__.py delete mode 100644 nemubot/server/message/abstract.py delete mode 100644 nemubot/server/test_IRC.py diff --git a/nemubot/message/printer/IRC.py b/nemubot/message/printer/IRC.py deleted file mode 100644 index df9cb9f..0000000 --- a/nemubot/message/printer/IRC.py +++ /dev/null @@ -1,25 +0,0 @@ -# Nemubot is a smart and modulable IM bot. -# Copyright (C) 2012-2016 Mercier Pierre-Olivier -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from nemubot.message import Text -from nemubot.message.printer.socket import Socket as SocketPrinter - - -class IRC(SocketPrinter): - - def visit_Text(self, msg): - self.pp += "PRIVMSG %s :" % ",".join(msg.to) - super().visit_Text(msg) diff --git a/nemubot/message/printer/IRCLib.py b/nemubot/message/printer/IRCLib.py new file mode 100644 index 0000000..abd1f2f --- /dev/null +++ b/nemubot/message/printer/IRCLib.py @@ -0,0 +1,67 @@ +# Nemubot is a smart and modulable IM bot. +# Copyright (C) 2012-2026 Mercier Pierre-Olivier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from nemubot.message.visitor import AbstractVisitor + + +class IRCLib(AbstractVisitor): + + """Visitor that sends bot responses via an irc.client.ServerConnection. + + Unlike the socket-based IRC printer (which builds a raw PRIVMSG string), + this calls connection.privmsg() directly so the library handles encoding, + line-length capping, and any internal locking. + """ + + def __init__(self, connection): + self._conn = connection + + def _send(self, target, text): + try: + self._conn.privmsg(target, text) + except Exception: + pass # drop silently during reconnection + + # Visitor methods + + def visit_Text(self, msg): + if isinstance(msg.message, str): + for target in msg.to: + self._send(target, msg.message) + else: + msg.message.accept(self) + + def visit_DirectAsk(self, msg): + text = msg.message if isinstance(msg.message, str) else str(msg.message) + # Mirrors socket.py logic: + # rooms that are NOT the designated nick get a "nick: " prefix + others = [to for to in msg.to if to != msg.designated] + if len(others) == 0 or len(others) != len(msg.to): + for target in msg.to: + self._send(target, text) + if others: + for target in others: + self._send(target, "%s: %s" % (msg.designated, text)) + + def visit_Command(self, msg): + parts = ["!" + msg.cmd] + list(msg.args) + for target in msg.to: + self._send(target, " ".join(parts)) + + def visit_OwnerCommand(self, msg): + parts = ["`" + msg.cmd] + list(msg.args) + for target in msg.to: + self._send(target, " ".join(parts)) diff --git a/nemubot/server/DCC.py b/nemubot/server/DCC.py deleted file mode 100644 index f5d4b8f..0000000 --- a/nemubot/server/DCC.py +++ /dev/null @@ -1,239 +0,0 @@ -# Nemubot is a smart and modulable IM bot. -# Copyright (C) 2012-2015 Mercier Pierre-Olivier -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import imp -import os -import re -import socket -import sys -import time -import threading -import traceback - -import nemubot.message as message -import nemubot.server as server - -#Store all used ports -PORTS = list() - -class DCC(server.AbstractServer): - def __init__(self, srv, dest, socket=None): - super().__init__(name="Nemubot DCC server") - - self.error = False # An error has occur, closing the connection? - self.messages = list() # Message queued before connexion - - # Informations about the sender - self.sender = dest - if self.sender is not None: - self.nick = (self.sender.split('!'))[0] - if self.nick != self.sender: - self.realname = (self.sender.split('!'))[1] - else: - self.realname = self.nick - - # Keep the server - self.srv = srv - self.treatement = self.treat_msg - - # Found a port for the connection - self.port = self.foundPort() - - if self.port is None: - self._logger.critical("No more available slot for DCC connection") - self.setError("Il n'y a plus de place disponible sur le serveur" - " pour initialiser une session DCC.") - - def foundPort(self): - """Found a free port for the connection""" - for p in range(65432, 65535): - if p not in PORTS: - PORTS.append(p) - return p - return None - - @property - def id(self): - """Gives the server identifiant""" - return self.srv.id + "/" + self.sender - - def setError(self, msg): - self.error = True - self.srv.send_msg_usr(self.sender, msg) - - def accept_user(self, host, port): - """Accept a DCC connection""" - self.s = socket.socket() - try: - self.s.connect((host, port)) - self._logger.info("Accepted user from %s:%d for %s", host, port, self.sender) - self.connected = True - self.stop = False - except: - self.connected = False - self.error = True - return False - self.start() - return True - - - def request_user(self, type="CHAT", filename="CHAT", size=""): - """Create a DCC connection""" - #Open the port - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - s.bind(('', self.port)) - except: - try: - self.port = self.foundPort() - s.bind(('', self.port)) - except: - self.setError("Une erreur s'est produite durant la tentative" - " d'ouverture d'une session DCC.") - return False - self._logger.info("Listening on %d for %s", self.port, self.sender) - - #Send CTCP request for DCC - self.srv.send_ctcp(self.sender, - "DCC %s %s %d %d %s" % (type, filename, self.srv.ip, - self.port, size), - "PRIVMSG") - - s.listen(1) - #Waiting for the client - (self.s, addr) = s.accept() - self._logger.info("Connected by %d", addr) - self.connected = True - return True - - def send_dcc_raw(self, line): - self.s.sendall(line + b'\n') - - def send_dcc(self, msg, to = None): - """If we talk to this user, send a message through this connection - else, send the message to the server class""" - if to is None or to == self.sender or to == self.nick: - if self.error: - self.srv.send_msg_final(self.nick, msg) - elif not self.connected or self.s is None: - try: - self.start() - except RuntimeError: - pass - self.messages.append(msg) - else: - for line in msg.split("\n"): - self.send_dcc_raw(line.encode()) - else: - self.srv.send_dcc(msg, to) - - def send_file(self, filename): - """Send a file over DCC""" - if os.path.isfile(filename): - self.messages = filename - try: - self.start() - except RuntimeError: - pass - else: - self._logger.error("File not found `%s'", filename) - - def run(self): - self.stopping.clear() - - # Send file connection - if not isinstance(self.messages, list): - self.request_user("SEND", - os.path.basename(self.messages), - os.path.getsize(self.messages)) - if self.connected: - with open(self.messages, 'rb') as f: - d = f.read(268435456) #Packets size: 256Mo - while d: - self.s.sendall(d) - self.s.recv(4) #The client send a confirmation after each packet - d = f.read(268435456) #Packets size: 256Mo - - # Messages connection - else: - if not self.connected: - if not self.request_user(): - #TODO: do something here - return False - - #Start by sending all queued messages - for mess in self.messages: - self.send_dcc(mess) - - time.sleep(1) - - readbuffer = b'' - self.nicksize = len(self.srv.nick) - self.Bnick = self.srv.nick.encode() - while not self.stop: - raw = self.s.recv(1024) #recieve server messages - if not raw: - break - readbuffer = readbuffer + raw - temp = readbuffer.split(b'\n') - readbuffer = temp.pop() - - for line in temp: - self.treatement(line) - - if self.connected: - self.s.close() - self.connected = False - - #Remove from DCC connections server list - if self.realname in self.srv.dcc_clients: - del self.srv.dcc_clients[self.realname] - - self._logger.info("Closing connection with %s", self.nick) - self.stopping.set() - if self.closing_event is not None: - self.closing_event() - #Rearm Thread - threading.Thread.__init__(self) - - def treat_msg(self, line): - """Treat a receive message, *can be overwritten*""" - if line == b'NEMUBOT###': - bot = self.srv.add_networkbot(self.srv, self.sender, self) - self.treatement = bot.treat_msg - self.send_dcc("NEMUBOT###") - elif (line[:self.nicksize] == self.Bnick and - line[self.nicksize+1:].strip()[:10] == b'my name is'): - name = line[self.nicksize+1:].strip()[11:].decode('utf-8', - 'replace') - if re.match("^[a-zA-Z0-9_-]+$", name): - if name not in self.srv.dcc_clients: - del self.srv.dcc_clients[self.sender] - self.nick = name - self.sender = self.nick + "!" + self.realname - self.srv.dcc_clients[self.realname] = self - self.send_dcc("Hi " + self.nick) - else: - self.send_dcc("This nickname is already in use" - ", please choose another one.") - else: - self.send_dcc("The name you entered contain" - " invalid char.") - else: - self.srv.treat_msg( - (":%s PRIVMSG %s :" % ( - self.sender,self.srv.nick)).encode() + line, - True) diff --git a/nemubot/server/IRC.py b/nemubot/server/IRC.py deleted file mode 100644 index 2096a63..0000000 --- a/nemubot/server/IRC.py +++ /dev/null @@ -1,276 +0,0 @@ -# Nemubot is a smart and modulable IM bot. -# Copyright (C) 2012-2015 Mercier Pierre-Olivier -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from datetime import datetime -import re -import socket - -from nemubot.channel import Channel -from nemubot.message.printer.IRC import IRC as IRCPrinter -from nemubot.server.message.IRC import IRC as IRCMessage -from nemubot.server.socket import SocketServer - - -class IRC(SocketServer): - - """Concrete implementation of a connexion to an IRC server""" - - def __init__(self, host="localhost", port=6667, owner=None, - nick="nemubot", username=None, password=None, - realname="Nemubot", encoding="utf-8", caps=None, - channels=list(), on_connect=None, **kwargs): - """Prepare a connection with an IRC server - - Keyword arguments: - host -- host to join - port -- port on the host to reach - ssl -- is this server using a TLS socket - owner -- bot's owner - nick -- bot's nick - username -- the username as sent to server - password -- if a password is required to connect to the server - realname -- the bot's realname - encoding -- the encoding used on the whole server - caps -- client capabilities to register on the server - channels -- list of channels to join on connection - on_connect -- generator to call when connection is done - """ - - self.username = username if username is not None else nick - self.password = password - self.nick = nick - self.owner = owner - self.realname = realname - - super().__init__(name=self.username + "@" + host + ":" + str(port), - host=host, port=port, **kwargs) - self.printer = IRCPrinter - - self.encoding = encoding - - # Keep a list of joined channels - self.channels = dict() - - # Server/client capabilities - self.capabilities = caps - - # Register CTCP capabilities - self.ctcp_capabilities = dict() - - def _ctcp_clientinfo(msg, cmds): - """Response to CLIENTINFO CTCP message""" - return " ".join(self.ctcp_capabilities.keys()) - - def _ctcp_dcc(msg, cmds): - """Response to DCC CTCP message""" - try: - import ipaddress - ip = ipaddress.ip_address(int(cmds[3])) - port = int(cmds[4]) - conn = DCC(srv, msg.sender) - except: - return "ERRMSG invalid parameters provided as DCC CTCP request" - - self._logger.info("Receive DCC connection request from %s to %s:%d", conn.sender, ip, port) - - if conn.accept_user(ip, port): - srv.dcc_clients[conn.sender] = conn - conn.send_dcc("Hello %s!" % conn.nick) - else: - self._logger.error("DCC: unable to connect to %s:%d", ip, port) - return "ERRMSG unable to connect to %s:%d" % (ip, port) - - import nemubot - - self.ctcp_capabilities["ACTION"] = lambda msg, cmds: None - self.ctcp_capabilities["CLIENTINFO"] = _ctcp_clientinfo - #self.ctcp_capabilities["DCC"] = _ctcp_dcc - self.ctcp_capabilities["FINGER"] = lambda msg, cmds: "VERSION nemubot v%s" % nemubot.__version__ - self.ctcp_capabilities["NEMUBOT"] = lambda msg, cmds: "NEMUBOT %s" % nemubot.__version__ - self.ctcp_capabilities["PING"] = lambda msg, cmds: "PING %s" % " ".join(cmds[1:]) - self.ctcp_capabilities["SOURCE"] = lambda msg, cmds: "SOURCE https://github.com/nemunaire/nemubot" - self.ctcp_capabilities["TIME"] = lambda msg, cmds: "TIME %s" % (datetime.now()) - self.ctcp_capabilities["USERINFO"] = lambda msg, cmds: "USERINFO %s" % self.realname - self.ctcp_capabilities["VERSION"] = lambda msg, cmds: "VERSION nemubot v%s" % nemubot.__version__ - - # TODO: Temporary fix, waiting for hook based CTCP management - self.ctcp_capabilities["TYPING"] = lambda msg, cmds: None - - self._logger.debug("CTCP capabilities setup: %s", ", ".join(self.ctcp_capabilities)) - - - # Register hooks on some IRC CMD - self.hookscmd = dict() - - # Respond to PING - def _on_ping(msg): - self.write(b"PONG :" + msg.params[0]) - self.hookscmd["PING"] = _on_ping - - # Respond to 001 - def _on_connect(msg): - # First, send user defined command - if on_connect is not None: - if callable(on_connect): - toc = on_connect() - else: - toc = on_connect - if toc is not None: - for oc in toc: - self.write(oc) - # Then, JOIN some channels - for chn in channels: - if chn.password: - self.write("JOIN %s %s" % (chn.name, chn.password)) - else: - self.write("JOIN %s" % chn.name) - self.hookscmd["001"] = _on_connect - - # Respond to ERROR - def _on_error(msg): - self.close() - self.hookscmd["ERROR"] = _on_error - - # Respond to CAP - def _on_cap(msg): - if len(msg.params) != 3 or msg.params[1] != b"LS": - return - server_caps = msg.params[2].decode().split(" ") - for cap in self.capabilities: - if cap not in server_caps: - self.capabilities.remove(cap) - if len(self.capabilities) > 0: - self.write("CAP REQ :" + " ".join(self.capabilities)) - self.write("CAP END") - self.hookscmd["CAP"] = _on_cap - - # Respond to JOIN - def _on_join(msg): - if len(msg.params) == 0: - return - - for chname in msg.decode(msg.params[0]).split(","): - # Register the channel - chan = Channel(chname) - self.channels[chname] = chan - self.hookscmd["JOIN"] = _on_join - # Respond to PART - def _on_part(msg): - if len(msg.params) != 1 and len(msg.params) != 2: - return - - for chname in msg.params[0].split(b","): - if chname in self.channels: - if msg.frm == self.nick: - del self.channels[chname] - elif msg.frm in self.channels[chname].people: - del self.channels[chname].people[msg.frm] - self.hookscmd["PART"] = _on_part - # Respond to 331/RPL_NOTOPIC,332/RPL_TOPIC,TOPIC - def _on_topic(msg): - if len(msg.params) != 1 and len(msg.params) != 2: - return - if msg.params[0] in self.channels: - if len(msg.params) == 1 or len(msg.params[1]) == 0: - self.channels[msg.params[0]].topic = None - else: - self.channels[msg.params[0]].topic = msg.decode(msg.params[1]) - self.hookscmd["331"] = _on_topic - self.hookscmd["332"] = _on_topic - self.hookscmd["TOPIC"] = _on_topic - # Respond to 353/RPL_NAMREPLY - def _on_353(msg): - if len(msg.params) == 3: - msg.params.pop(0) # 353: like RFC 1459 - if len(msg.params) != 2: - return - if msg.params[0] in self.channels: - for nk in msg.decode(msg.params[1]).split(" "): - res = re.match("^(?P[^a-zA-Z[\]\\`_^{|}])(?P[a-zA-Z[\]\\`_^{|}][a-zA-Z0-9[\]\\`_^{|}-]*)$") - self.channels[msg.params[0]].people[res.group("nickname")] = res.group("level") - self.hookscmd["353"] = _on_353 - - # Respond to INVITE - def _on_invite(msg): - if len(msg.params) != 2: - return - self.write("JOIN " + msg.decode(msg.params[1])) - self.hookscmd["INVITE"] = _on_invite - - # Respond to ERR_NICKCOLLISION - def _on_nickcollision(msg): - self.nick += "_" - self.write("NICK " + self.nick) - self.hookscmd["433"] = _on_nickcollision - self.hookscmd["436"] = _on_nickcollision - - # Handle CTCP requests - def _on_ctcp(msg): - if len(msg.params) != 2 or not msg.is_ctcp: - return - cmds = msg.decode(msg.params[1][1:len(msg.params[1])-1]).split(' ') - if cmds[0] in self.ctcp_capabilities: - res = self.ctcp_capabilities[cmds[0]](msg, cmds) - else: - res = "ERRMSG Unknown or unimplemented CTCP request" - if res is not None: - self.write("NOTICE %s :\x01%s\x01" % (msg.frm, res)) - self.hookscmd["PRIVMSG"] = _on_ctcp - - - # Open/close - - def connect(self): - super().connect() - - if self.password is not None: - self.write("PASS :" + self.password) - if self.capabilities is not None: - self.write("CAP LS") - self.write("NICK :" + self.nick) - self.write("USER %s %s bla :%s" % (self.username, socket.getfqdn(), self.realname)) - - - def close(self): - if not self._fd._closed: - self.write("QUIT") - return super().close() - - - # Writes: as inherited - - # Read - - def async_read(self): - for line in super().async_read(): - # PING should be handled here, so start parsing here :/ - msg = IRCMessage(line, self.encoding) - - if msg.cmd in self.hookscmd: - self.hookscmd[msg.cmd](msg) - - yield msg - - - def parse(self, msg): - mes = msg.to_bot_message(self) - if mes is not None: - yield mes - - - def subparse(self, orig, cnt): - msg = IRCMessage(("@time=%s :%s!user@host.com PRIVMSG %s :%s" % (orig.date.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), orig.frm, ",".join(orig.to), cnt)).encode(self.encoding), self.encoding) - return msg.to_bot_message(self) diff --git a/nemubot/server/IRCLib.py b/nemubot/server/IRCLib.py new file mode 100644 index 0000000..cdd13cf --- /dev/null +++ b/nemubot/server/IRCLib.py @@ -0,0 +1,375 @@ +# Nemubot is a smart and modulable IM bot. +# Copyright (C) 2012-2026 Mercier Pierre-Olivier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from datetime import datetime +import shlex +import threading + +import irc.bot +import irc.client +import irc.connection + +import nemubot.message as message +from nemubot.server.threaded import ThreadedServer + + +class _IRCBotAdapter(irc.bot.SingleServerIRCBot): + + """Internal adapter that bridges the irc library event model to nemubot. + + Subclasses SingleServerIRCBot to get automatic reconnection, PING/PONG, + and nick-collision handling for free. + """ + + def __init__(self, server_name, push_fn, channels, on_connect_cmds, + nick, server_list, owner=None, realname="Nemubot", + encoding="utf-8", **connect_params): + super().__init__(server_list, nick, realname, **connect_params) + self._nemubot_name = server_name + self._push = push_fn + self._channels_to_join = channels + self._on_connect_cmds = on_connect_cmds or [] + self.owner = owner + self.encoding = encoding + self._stop_event = threading.Event() + + + # Event loop control + + def start(self): + """Run the reactor loop until stop() is called.""" + self._connect() + while not self._stop_event.is_set(): + self.reactor.process_once(timeout=0.2) + + def stop(self): + """Signal the loop to exit and disconnect cleanly.""" + self._stop_event.set() + try: + self.connection.disconnect("Goodbye") + except Exception: + pass + + def on_disconnect(self, connection, event): + """Reconnect automatically unless we are shutting down.""" + if not self._stop_event.is_set(): + super().on_disconnect(connection, event) + + + # Connection lifecycle + + def on_welcome(self, connection, event): + """001 — run on_connect commands then join channels.""" + for cmd in self._on_connect_cmds: + if callable(cmd): + for c in (cmd() or []): + connection.send_raw(c) + else: + connection.send_raw(cmd) + + for ch in self._channels_to_join: + if isinstance(ch, tuple): + connection.join(ch[0], ch[1] if len(ch) > 1 else "") + elif hasattr(ch, 'name'): + connection.join(ch.name, getattr(ch, 'password', "") or "") + else: + connection.join(str(ch)) + + def on_invite(self, connection, event): + """Auto-join on INVITE.""" + if event.arguments: + connection.join(event.arguments[0]) + + + # CTCP + + def on_ctcp(self, connection, event): + """Handle CTCP requests (irc library >= 19 dispatches all to on_ctcp).""" + nick = irc.client.NickMask(event.source).nick + ctcp_type = event.arguments[0].upper() if event.arguments else "" + ctcp_arg = event.arguments[1] if len(event.arguments) > 1 else "" + self._reply_ctcp(connection, nick, ctcp_type, ctcp_arg) + + # Fallbacks for older irc library versions that dispatch per-type + def on_ctcpversion(self, connection, event): + import nemubot + nick = irc.client.NickMask(event.source).nick + connection.ctcp_reply(nick, "VERSION nemubot v%s" % nemubot.__version__) + + def on_ctcpping(self, connection, event): + nick = irc.client.NickMask(event.source).nick + arg = event.arguments[0] if event.arguments else "" + connection.ctcp_reply(nick, "PING %s" % arg) + + def _reply_ctcp(self, connection, nick, ctcp_type, ctcp_arg): + import nemubot + responses = { + "ACTION": None, # handled as on_action + "CLIENTINFO": "CLIENTINFO ACTION CLIENTINFO FINGER PING SOURCE TIME USERINFO VERSION", + "FINGER": "FINGER nemubot v%s" % nemubot.__version__, + "PING": "PING %s" % ctcp_arg, + "SOURCE": "SOURCE https://github.com/nemunaire/nemubot", + "TIME": "TIME %s" % datetime.now(), + "USERINFO": "USERINFO Nemubot", + "VERSION": "VERSION nemubot v%s" % nemubot.__version__, + } + if ctcp_type in responses and responses[ctcp_type] is not None: + connection.ctcp_reply(nick, responses[ctcp_type]) + + + # Incoming messages + + def _decode(self, text): + if isinstance(text, bytes): + try: + return text.decode("utf-8") + except UnicodeDecodeError: + return text.decode(self.encoding, "replace") + return text + + def _make_message(self, connection, source, target, text): + """Convert raw IRC event data into a nemubot bot message.""" + nick = irc.client.NickMask(source).nick + text = self._decode(text) + bot_nick = connection.get_nickname() + is_channel = irc.client.is_channel(target) + to = [target] if is_channel else [nick] + to_response = [target] if is_channel else [nick] + + common = dict( + server=self._nemubot_name, + to=to, + to_response=to_response, + frm=nick, + frm_owner=(nick == self.owner), + ) + + # "botname: text" or "botname, text" + if (text.startswith(bot_nick + ":") or + text.startswith(bot_nick + ",")): + inner = text[len(bot_nick) + 1:].strip() + return message.DirectAsk(designated=bot_nick, message=inner, + **common) + + # "!command [args]" + if len(text) > 1 and text[0] == '!': + inner = text[1:].strip() + try: + args = shlex.split(inner) + except ValueError: + args = inner.split() + if args: + # Extract @key=value named arguments (same logic as IRC.py) + kwargs = {} + while len(args) > 1: + arg = args[1] + if len(arg) > 2 and arg[0:2] == '\\@': + args[1] = arg[1:] + elif len(arg) > 1 and arg[0] == '@': + arsp = arg[1:].split("=", 1) + kwargs[arsp[0]] = arsp[1] if len(arsp) == 2 else None + args.pop(1) + continue + break + return message.Command(cmd=args[0], args=args[1:], + kwargs=kwargs, **common) + + return message.Text(message=text, **common) + + def on_pubmsg(self, connection, event): + msg = self._make_message( + connection, event.source, event.target, + event.arguments[0] if event.arguments else "", + ) + if msg: + self._push(msg) + + def on_privmsg(self, connection, event): + nick = irc.client.NickMask(event.source).nick + msg = self._make_message( + connection, event.source, nick, + event.arguments[0] if event.arguments else "", + ) + if msg: + self._push(msg) + + def on_action(self, connection, event): + """CTCP ACTION (/me) — delivered as a plain Text message.""" + nick = irc.client.NickMask(event.source).nick + text = "/me %s" % (event.arguments[0] if event.arguments else "") + is_channel = irc.client.is_channel(event.target) + to = [event.target] if is_channel else [nick] + self._push(message.Text( + message=text, + server=self._nemubot_name, + to=to, to_response=to, + frm=nick, frm_owner=(nick == self.owner), + )) + + +class IRCLib(ThreadedServer): + + """IRC server using the irc Python library (jaraco). + + Compared to the hand-rolled IRC.py implementation, this gets: + - Automatic exponential-backoff reconnection + - PING/PONG handled transparently + - Nick-collision suffix logic built-in + """ + + def __init__(self, host="localhost", port=6667, nick="nemubot", + username=None, password=None, realname="Nemubot", + encoding="utf-8", owner=None, channels=None, + on_connect=None, ssl=False, **kwargs): + """Prepare a connection to an IRC server. + + Keyword arguments: + host -- IRC server hostname + port -- IRC server port (default 6667) + nick -- bot's nickname + username -- username for USER command (defaults to nick) + password -- server password (sent as PASS) + realname -- bot's real name + encoding -- fallback encoding for non-UTF-8 servers + owner -- nick of the bot's owner (sets frm_owner on messages) + channels -- list of channel names / (name, key) tuples to join + on_connect -- list of raw IRC commands (or a callable returning one) + to send after receiving 001 + ssl -- wrap the connection in TLS + """ + name = (username or nick) + "@" + host + ":" + str(port) + super().__init__(name=name) + + self._host = host + self._port = int(port) + self._nick = nick + self._username = username or nick + self._password = password + self._realname = realname + self._encoding = encoding + self.owner = owner + self._channels = channels or [] + self._on_connect_cmds = on_connect + self._ssl = ssl + + self._bot = None + self._thread = None + + + # ThreadedServer hooks + + def _start(self): + server_list = [irc.bot.ServerSpec(self._host, self._port, + self._password)] + + connect_params = {"username": self._username} + + if self._ssl: + import ssl as ssl_mod + ctx = ssl_mod.create_default_context() + host = self._host # capture for closure + connect_params["connect_factory"] = irc.connection.Factory( + wrapper=lambda sock: ctx.wrap_socket(sock, + server_hostname=host) + ) + + self._bot = _IRCBotAdapter( + server_name=self.name, + push_fn=self._push_message, + channels=self._channels, + on_connect_cmds=self._on_connect_cmds, + nick=self._nick, + server_list=server_list, + owner=self.owner, + realname=self._realname, + encoding=self._encoding, + **connect_params, + ) + self._thread = threading.Thread( + target=self._bot.start, + daemon=True, + name="nemubot.IRC/" + self.name, + ) + self._thread.start() + + def _stop(self): + if self._bot: + self._bot.stop() + if self._thread: + self._thread.join(timeout=5) + + + # Outgoing messages + + def send_response(self, response): + if response is None: + return + if isinstance(response, list): + for r in response: + self.send_response(r) + return + if not self._bot: + return + + from nemubot.message.printer.IRCLib import IRCLib as IRCLibPrinter + printer = IRCLibPrinter(self._bot.connection) + response.accept(printer) + + + # subparse: re-parse a plain string in the context of an existing message + # (used by alias, rnd, grep, cat, smmry, sms modules) + + def subparse(self, orig, cnt): + bot_nick = (self._bot.connection.get_nickname() + if self._bot else self._nick) + common = dict( + server=self.name, + to=orig.to, + to_response=orig.to_response, + frm=orig.frm, + frm_owner=orig.frm_owner, + date=orig.date, + ) + text = cnt + + if (text.startswith(bot_nick + ":") or + text.startswith(bot_nick + ",")): + inner = text[len(bot_nick) + 1:].strip() + return message.DirectAsk(designated=bot_nick, message=inner, + **common) + + if len(text) > 1 and text[0] == '!': + inner = text[1:].strip() + try: + args = shlex.split(inner) + except ValueError: + args = inner.split() + if args: + kwargs = {} + while len(args) > 1: + arg = args[1] + if len(arg) > 2 and arg[0:2] == '\\@': + args[1] = arg[1:] + elif len(arg) > 1 and arg[0] == '@': + arsp = arg[1:].split("=", 1) + kwargs[arsp[0]] = arsp[1] if len(arsp) == 2 else None + args.pop(1) + continue + break + return message.Command(cmd=args[0], args=args[1:], + kwargs=kwargs, **common) + + return message.Text(message=text, **common) diff --git a/nemubot/server/__init__.py b/nemubot/server/__init__.py index 9e186ed..db9ad87 100644 --- a/nemubot/server/__init__.py +++ b/nemubot/server/__init__.py @@ -24,13 +24,13 @@ def factory(uri, ssl=False, **init_args): if o.scheme == "irc" or o.scheme == "ircs": # https://www.w3.org/Addressing/draft-mirashi-url-irc-01.txt # https://www-archive.mozilla.org/projects/rt-messaging/chatzilla/irc-urls.html - args = init_args + args = dict(init_args) if o.scheme == "ircs": ssl = True if o.hostname is not None: args["host"] = o.hostname if o.port is not None: args["port"] = o.port if o.username is not None: args["username"] = o.username - if o.password is not None: args["password"] = o.password + if o.password is not None: args["password"] = unquote(o.password) modifiers = o.path.split(",") target = unquote(modifiers.pop(0)[1:]) @@ -41,37 +41,27 @@ def factory(uri, ssl=False, **init_args): if "msg" in params: if "on_connect" not in args: args["on_connect"] = [] - args["on_connect"].append("PRIVMSG %s :%s" % (target, params["msg"])) + args["on_connect"].append("PRIVMSG %s :%s" % (target, params["msg"][0])) if "key" in params: if "channels" not in args: args["channels"] = [] - args["channels"].append((target, params["key"])) + args["channels"].append((target, params["key"][0])) if "pass" in params: - args["password"] = params["pass"] + args["password"] = params["pass"][0] if "charset" in params: - args["encoding"] = params["charset"] + args["encoding"] = params["charset"][0] - # if "channels" not in args and "isnick" not in modifiers: - args["channels"] = [ target ] + args["channels"] = [target] - from nemubot.server.IRC import IRC as IRCServer + args["ssl"] = ssl + + from nemubot.server.IRCLib import IRCLib as IRCServer srv = IRCServer(**args) - if ssl: - try: - from ssl import create_default_context - context = create_default_context() - except ImportError: - # Python 3.3 compat - from ssl import SSLContext, PROTOCOL_TLSv1 - context = SSLContext(PROTOCOL_TLSv1) - from ssl import wrap_socket - srv._fd = context.wrap_socket(srv._fd, server_hostname=o.hostname) - elif o.scheme == "matrix": # matrix://localpart:password@homeserver.tld/!room:homeserver.tld # matrix://localpart:password@homeserver.tld/%23alias:homeserver.tld diff --git a/nemubot/server/factory_test.py b/nemubot/server/factory_test.py deleted file mode 100644 index efc3130..0000000 --- a/nemubot/server/factory_test.py +++ /dev/null @@ -1,54 +0,0 @@ -# Nemubot is a smart and modulable IM bot. -# Copyright (C) 2012-2015 Mercier Pierre-Olivier -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import unittest - -from nemubot.server import factory - -class TestFactory(unittest.TestCase): - - def test_IRC1(self): - from nemubot.server.IRC import IRC as IRCServer - import socket - import ssl - - # : If omitted, the client must connect to a prespecified default IRC server. - server = factory("irc:///") - self.assertIsInstance(server, IRCServer) - self.assertIsInstance(server._fd, socket.socket) - self.assertIn(server._sockaddr[0], ["127.0.0.1", "::1"]) - - server = factory("irc://2.2.2.2") - self.assertIsInstance(server, IRCServer) - self.assertEqual(server._sockaddr[0], "2.2.2.2") - - server = factory("ircs://1.2.1.2") - self.assertIsInstance(server, IRCServer) - self.assertIsInstance(server._fd, ssl.SSLSocket) - - server = factory("irc://1.2.3.4:6667") - self.assertIsInstance(server, IRCServer) - self.assertEqual(server._sockaddr[0], "1.2.3.4") - self.assertEqual(server._sockaddr[1], 6667) - - server = factory("ircs://4.3.2.1:194/") - self.assertIsInstance(server, IRCServer) - self.assertEqual(server._sockaddr[0], "4.3.2.1") - self.assertEqual(server._sockaddr[1], 194) - - -if __name__ == '__main__': - unittest.main() diff --git a/nemubot/server/message/IRC.py b/nemubot/server/message/IRC.py deleted file mode 100644 index 5ccd735..0000000 --- a/nemubot/server/message/IRC.py +++ /dev/null @@ -1,210 +0,0 @@ -# Nemubot is a smart and modulable IM bot. -# Copyright (C) 2012-2016 Mercier Pierre-Olivier -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from datetime import datetime, timezone -import re -import shlex - -import nemubot.message as message -from nemubot.server.message.abstract import Abstract - -mgx = re.compile(b'''^(?:@(?P[^ ]+)\ )? - (?::(?P - (?P[^!@ ]+) - (?: !(?P[^@ ]+))? - (?:@(?P[^ ]*))? - )\ )? - (?P(?:[a-zA-Z]+|[0-9]{3})) - (?P(?:\ [^:][^ ]*)*)(?:\ :(?P.*))? - $''', re.X) - -class IRC(Abstract): - - """Class responsible for parsing IRC messages""" - - def __init__(self, raw, encoding="utf-8"): - self.encoding = encoding - self.tags = { 'time': datetime.now(timezone.utc) } - self.params = list() - - p = mgx.match(raw.rstrip()) - - if p is None: - raise Exception("Not a valid IRC message: %s" % raw) - - # Parse tags if exists: @aaa=bbb;ccc;example.com/ddd=eee - if p.group("tags"): - for tgs in self.decode(p.group("tags")).split(';'): - tag = tgs.split('=') - if len(tag) > 1: - self.add_tag(tag[0], tag[1]) - else: - self.add_tag(tag[0]) - - # Parse prefix if exists: :nick!user@host.com - self.prefix = self.decode(p.group("prefix")) - self.nick = self.decode(p.group("nick")) - self.user = self.decode(p.group("user")) - self.host = self.decode(p.group("host")) - - # Parse command - self.cmd = self.decode(p.group("command")) - - # Parse params - if p.group("params") is not None and p.group("params") != b'': - for param in p.group("params").strip().split(b' '): - self.params.append(param) - - if p.group("trailing") is not None: - self.params.append(p.group("trailing")) - - - def add_tag(self, key, value=None): - """Add an IRCv3.2 Message Tags - - Arguments: - key -- tag identifier (unique for the message) - value -- optional value for the tag - """ - - # Treat special tags - if key == "time" and value is not None: - import calendar, time - value = datetime.fromtimestamp(calendar.timegm(time.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ")), timezone.utc) - - # Store tag - self.tags[key] = value - - - @property - def is_ctcp(self): - """Analyze a message, to determine if this is a CTCP one""" - return self.cmd == "PRIVMSG" and len(self.params) == 2 and len(self.params[1]) > 1 and (self.params[1][0] == 0x01 or self.params[1][1] == 0x01) - - - def decode(self, s): - """Decode the content string usign a specific encoding - - Argument: - s -- string to decode - """ - - if isinstance(s, bytes): - try: - s = s.decode() - except UnicodeDecodeError: - s = s.decode(self.encoding, 'replace') - return s - - - - def to_server_string(self, client=True): - """Pretty print the message to close to original input string - - Keyword argument: - client -- export as a client-side string if true - """ - - res = ";".join(["@%s=%s" % (k, v if not isinstance(v, datetime) else v.strftime("%Y-%m-%dT%H:%M:%S.%fZ")) for k, v in self.tags.items()]) - - if not client: - res += " :%s!%s@%s" % (self.nick, self.user, self.host) - - res += " " + self.cmd - - if len(self.params) > 0: - - if len(self.params) > 1: - res += " " + self.decode(b" ".join(self.params[:-1])) - res += " :" + self.decode(self.params[-1]) - - return res - - - def to_bot_message(self, srv): - """Convert to one of concrete implementation of AbstractMessage - - Argument: - srv -- the server from the message was received - """ - - if self.cmd == "PRIVMSG" or self.cmd == "NOTICE": - - receivers = self.decode(self.params[0]).split(',') - - common_args = { - "server": srv.name, - "date": self.tags["time"], - "to": receivers, - "to_response": [r if r != srv.nick else self.nick for r in receivers], - "frm": self.nick, - "frm_owner": self.nick == srv.owner - } - - # If CTCP, remove 0x01 - if self.is_ctcp: - text = self.decode(self.params[1][1:len(self.params[1])-1]) - else: - text = self.decode(self.params[1]) - - if text.find(srv.nick) == 0 and len(text) > len(srv.nick) + 2 and text[len(srv.nick)] == ":": - designated = srv.nick - text = text[len(srv.nick) + 1:].strip() - else: - designated = None - - # Is this a command? - if len(text) > 1 and text[0] == '!': - text = text[1:].strip() - - # Split content by words - try: - args = shlex.split(text) - except ValueError: - args = text.split(' ') - - # Extract explicit named arguments: @key=value or just @key, only at begening - kwargs = {} - while len(args) > 1: - arg = args[1] - if len(arg) > 2: - if arg[0:2] == '\\@': - args[1] = arg[1:] - elif arg[0] == '@': - arsp = arg[1:].split("=", 1) - if len(arsp) == 2: - kwargs[arsp[0]] = arsp[1] - else: - kwargs[arg[1:]] = None - args.pop(1) - continue - # Futher argument are considered as normal argument (this helps for subcommand treatment) - break - - return message.Command(cmd=args[0], - args=args[1:], - kwargs=kwargs, - **common_args) - - # Is this an ask for this bot? - elif designated is not None: - return message.DirectAsk(designated=designated, message=text, **common_args) - - # Normal message - else: - return message.Text(message=text, **common_args) - - return None diff --git a/nemubot/server/message/__init__.py b/nemubot/server/message/__init__.py deleted file mode 100644 index 57f3468..0000000 --- a/nemubot/server/message/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Nemubot is a smart and modulable IM bot. -# Copyright (C) 2012-2015 Mercier Pierre-Olivier -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . diff --git a/nemubot/server/message/abstract.py b/nemubot/server/message/abstract.py deleted file mode 100644 index 624e453..0000000 --- a/nemubot/server/message/abstract.py +++ /dev/null @@ -1,33 +0,0 @@ -# Nemubot is a smart and modulable IM bot. -# Copyright (C) 2012-2016 Mercier Pierre-Olivier -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -class Abstract: - - def to_bot_message(self, srv): - """Convert to one of concrete implementation of AbstractMessage - - Argument: - srv -- the server from the message was received - """ - - raise NotImplemented - - - def to_server_string(self, **kwargs): - """Pretty print the message to close to original input string - """ - - raise NotImplemented diff --git a/nemubot/server/test_IRC.py b/nemubot/server/test_IRC.py deleted file mode 100644 index 552a1d3..0000000 --- a/nemubot/server/test_IRC.py +++ /dev/null @@ -1,50 +0,0 @@ -import unittest - -import nemubot.server.IRC as IRC - - -class TestIRCMessage(unittest.TestCase): - - - def setUp(self): - self.msg = IRC.IRCMessage(b":toto!titi@RZ-3je16g.re PRIVMSG #the-channel :Can you parse this message?") - - - def test_parsing(self): - self.assertEqual(self.msg.prefix, "toto!titi@RZ-3je16g.re") - self.assertEqual(self.msg.nick, "toto") - self.assertEqual(self.msg.user, "titi") - self.assertEqual(self.msg.host, "RZ-3je16g.re") - - self.assertEqual(len(self.msg.params), 2) - - self.assertEqual(self.msg.params[0], b"#the-channel") - self.assertEqual(self.msg.params[1], b"Can you parse this message?") - - - def test_prettyprint(self): - bst1 = self.msg.to_server_string(False) - msg2 = IRC.IRCMessage(bst1.encode()) - - bst2 = msg2.to_server_string(False) - msg3 = IRC.IRCMessage(bst2.encode()) - - bst3 = msg3.to_server_string(False) - - self.assertEqual(bst2, bst3) - - - def test_tags(self): - self.assertEqual(len(self.msg.tags), 1) - self.assertIn("time", self.msg.tags) - - self.msg.add_tag("time") - self.assertEqual(len(self.msg.tags), 1) - - self.msg.add_tag("toto") - self.assertEqual(len(self.msg.tags), 2) - self.assertIn("toto", self.msg.tags) - - -if __name__ == '__main__': - unittest.main() diff --git a/requirements.txt b/requirements.txt index 45eefe2..e037895 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ +irc matrix-nio diff --git a/setup.py b/setup.py index 94c1274..7b5bdcd 100755 --- a/setup.py +++ b/setup.py @@ -71,7 +71,6 @@ setup( 'nemubot.message.printer', 'nemubot.module', 'nemubot.server', - 'nemubot.server.message', 'nemubot.tools', 'nemubot.tools.xmlparser', ], From 310f9330914e5a1b90122e24ae07509c51c80432 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 6 Mar 2026 22:14:20 +0700 Subject: [PATCH 126/127] bot: fix duplicate unregister KeyError and improve connection error logging Silently ignore KeyError when unregistering an already-removed FD from the poll loop (servers can queue multiple close events). Also include the exception message when a server connection fails at startup. Co-Authored-By: Claude Sonnet 4.6 --- nemubot/__main__.py | 4 ++-- nemubot/bot.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/nemubot/__main__.py b/nemubot/__main__.py index 6a8b265..7070639 100644 --- a/nemubot/__main__.py +++ b/nemubot/__main__.py @@ -163,8 +163,8 @@ def main(): logger.info("Server '%s' successfully added.", srv.name) else: logger.error("Can't add server '%s'.", srv.name) - except: - logger.error("Unable to connect to '%s'.", srv.name) + except Exception as e: + logger.error("Unable to connect to '%s': %s", srv.name, e) continue break diff --git a/nemubot/bot.py b/nemubot/bot.py index 6aa5ed6..21f7178 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -218,7 +218,10 @@ class Bot(threading.Thread): elif args[0] == "register": self._poll.register(int(args[1]), select.POLLIN | select.POLLPRI) elif args[0] == "unregister": - self._poll.unregister(int(args[1])) + try: + self._poll.unregister(int(args[1])) + except KeyError: + pass except: logger.exception("Unhandled excpetion during action:") From 9d7c278d1aa1eece9468933e0d353e1473a68349 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Fri, 6 Mar 2026 23:08:27 +0700 Subject: [PATCH 127/127] bot: Fix sequential message processing with proper consumer pool Replace the flawed cnsr_thrd_size threshold with cnsr_active, which tracks the number of consumers currently executing a task. A new consumer thread is now spawned the moment the queue is non-empty and all existing consumers are busy, enabling true parallel execution of slow and fast commands. The pool is capped at os.cpu_count() threads. - bot.py: replace cnsr_thrd_size with cnsr_active + cnsr_lock + cnsr_max - consumer.py: increment/decrement cnsr_active around stm.run(), remove itself from cnsr_thrd under the lock, mark thread as daemon Co-Authored-By: Claude Sonnet 4.6 --- nemubot/bot.py | 33 ++++++++++++++++++++------------- nemubot/consumer.py | 23 +++++++++++++++-------- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/nemubot/bot.py b/nemubot/bot.py index 21f7178..2b6e15c 100644 --- a/nemubot/bot.py +++ b/nemubot/bot.py @@ -143,11 +143,16 @@ class Bot(threading.Thread): return res self.treater.hm.add_hook(nemubot.hooks.Command(_help_msg, "help"), "in", "Command") + import os from queue import Queue - # Messages to be treated - self.cnsr_queue = Queue() - self.cnsr_thrd = list() - self.cnsr_thrd_size = -1 + # Messages to be treated — shared across all server connections. + # cnsr_active tracks consumers currently inside stm.run() (not idle), + # which lets us spawn a new thread the moment all existing ones are busy. + self.cnsr_queue = Queue() + self.cnsr_thrd = list() + self.cnsr_lock = threading.Lock() + self.cnsr_active = 0 # consumers currently executing a task + self.cnsr_max = os.cpu_count() or 4 # upper bound on concurrent consumer threads def __del__(self): @@ -234,14 +239,15 @@ class Bot(threading.Thread): sync_queue.task_done() - # Launch new consumer threads if necessary - while self.cnsr_queue.qsize() > self.cnsr_thrd_size: - # Next launch if two more items in queue - self.cnsr_thrd_size += 2 - - c = Consumer(self) - self.cnsr_thrd.append(c) - c.start() + # Spawn a new consumer whenever the queue has work and every + # existing consumer is already busy executing a task. + with self.cnsr_lock: + while (not self.cnsr_queue.empty() + and self.cnsr_active >= len(self.cnsr_thrd) + and len(self.cnsr_thrd) < self.cnsr_max): + c = Consumer(self) + self.cnsr_thrd.append(c) + c.start() sync_queue = None logger.info("Ending main loop") @@ -518,7 +524,8 @@ class Bot(threading.Thread): srv.close() logger.info("Stop consumers") - k = self.cnsr_thrd + with self.cnsr_lock: + k = list(self.cnsr_thrd) for cnsr in k: cnsr.stop = True diff --git a/nemubot/consumer.py b/nemubot/consumer.py index 3a58219..a9a4146 100644 --- a/nemubot/consumer.py +++ b/nemubot/consumer.py @@ -105,18 +105,25 @@ class Consumer(threading.Thread): def __init__(self, context): self.context = context self.stop = False - super().__init__(name="Nemubot consumer") + super().__init__(name="Nemubot consumer", daemon=True) def run(self): try: while not self.stop: - stm = self.context.cnsr_queue.get(True, 1) - stm.run(self.context) - self.context.cnsr_queue.task_done() + try: + stm = self.context.cnsr_queue.get(True, 1) + except queue.Empty: + break - except queue.Empty: - pass + with self.context.cnsr_lock: + self.context.cnsr_active += 1 + try: + stm.run(self.context) + finally: + self.context.cnsr_queue.task_done() + with self.context.cnsr_lock: + self.context.cnsr_active -= 1 finally: - self.context.cnsr_thrd_size -= 2 - self.context.cnsr_thrd.remove(self) + with self.context.cnsr_lock: + self.context.cnsr_thrd.remove(self)