Use urllib instead of own tools for modules ycc and networking

modules/ddg/__init__.py

@@ -134,5 +134,5 @@ def wiki(msg):
         return res
     else:
         return Response(msg.sender,
-                        "No information about " + msg.cmds[1],
+                        "No information about " + " ".join(msg.cmds[1:]),
                         msg.channel)

modules/networking.py

@@ -1,6 +1,9 @@
 # coding=utf-8
 
-from tools import web
+import http.client
+import json
+from urllib.parse import urlparse
+from urllib.request import urlopen
 
 nemubotversion = 3.3
 
@@ -21,11 +24,8 @@ def cmd_traceurl(msg):
     if 1 < len(msg.cmds) < 6:
         res = list()
         for url in msg.cmds[1:]:
-            if web.isURL(url):
+            trace = traceURL(url)
-                trace = web.traceURL(url)
+            res.append(Response(msg.sender, trace, channel=msg.channel, title="TraceURL"))
-                res.append(Response(msg.sender, trace, channel=msg.channel, title="TraceURL"))
-            else:
-                res.append(Response(msg.sender, "%s n'est pas une URL valide" % url, channel=msg.channel))
         return res
     else:
         return Response(msg.sender, "Indiquer une URL a tracer !", channel=msg.channel)
@@ -34,9 +34,12 @@ def cmd_isup(msg):
     if 1 < len(msg.cmds) < 6:
         res = list()
         for url in msg.cmds[1:]:
-            host = web.getHost(url)
+            o = urlparse(url, "http")
-            if host is not None:
+            if o.netloc == "":
-                isup = web.getJSON("http://isitup.org/" + host + ".json")
+                o = urlparse("http://" + url)
+            if o.netloc != "":
+                raw = urlopen("http://isitup.org/" + o.netloc + ".json", timeout=10)
+                isup = json.loads(raw.read().decode())
                 if "status_code" in isup and isup["status_code"] == 1:
                     res.append(Response(msg.sender, "%s est accessible (temps de reponse : %ss)" % (isup["domain"], isup["response_time"]), channel=msg.channel))
                 else:
@@ -46,3 +49,46 @@ def cmd_isup(msg):
         return res
     else:
         return Response(msg.sender, "Indiquer une URL a verifier !", channel=msg.channel)
+
+
+def traceURL(url, timeout=5, stack=None):
+    """Follow redirections and return the redirections stack"""
+    if stack is None:
+        stack = list()
+    stack.append(url)
+
+    o = urlparse(url, "http")
+    if o.netloc == "":
+        return stack
+    if o.scheme == "http":
+        conn = http.client.HTTPConnection(o.netloc, port=o.port, timeout=timeout)
+    else:
+        conn = http.client.HTTPSConnection(o.netloc, port=o.port, timeout=timeout)
+    try:
+        conn.request("HEAD", o.path, None, {"User-agent": "Nemubot v3"})
+    except socket.timeout:
+        stack.append("Timeout")
+        return stack
+    except socket.gaierror:
+        print ("<tools.web> Unable to receive page %s from %s on %d."
+               % (o.path, o.netloc, o.port))
+        return None
+
+    try:
+        res = conn.getresponse()
+    except http.client.BadStatusLine:
+        return None
+    finally:
+        conn.close()
+
+    if res.status == http.client.OK:
+        return stack
+    elif res.status == http.client.FOUND or res.status == http.client.MOVED_PERMANENTLY or res.status == http.client.SEE_OTHER:
+        url = res.getheader("Location")
+        if url in stack:
+            stack.append(url)
+            return stack
+        else:
+            return traceURL(url, timeout, stack)
+    else:
+        return None

modules/ycc.py

@@ -1,8 +1,8 @@
 # coding=utf-8
 
 import re
-
+from urllib.parse import urlparse
-from tools import web
+from urllib.request import urlopen
 
 nemubotversion = 3.3
 
@@ -39,11 +39,14 @@ def cmd_ycc(msg):
     if len(msg.cmds) < 6:
         res = list()
         for url in msg.cmds[1:]:
-            srv = web.getHost(url)
+            o = urlparse(url, "http")
-            if srv is not None:
+            if o.scheme != "":
-                res.append(gen_response(
+                raw = urlopen("http://ycc.fr/redirection/create/" + url,
-                    web.getURLContent("http://ycc.fr/redirection/create/"
+                              timeout=10)
-                                      + url).decode(), msg, srv))
+                if o.netloc == "":
+                    res.append(gen_response(raw.read().decode(), msg, o.scheme))
+                else:
+                    res.append(gen_response(raw.read().decode(), msg, o.netloc))
             else:
                 res.append(gen_response(False, msg, url))
         return res
@@ -56,13 +59,13 @@ def parselisten(msg):
     res = re.match(".*([a-zA-Z0-9+.-]+):(//)?([^ ]*).*", msg.content)
     if res is not None:
         url = res.group(1)
-        srv = web.getHost(url)
+        o = urlparse(url)
-        if srv is not None:
+        if o.scheme != "":
-            if srv == "ycc.fr":
+            if o.netloc == "ycc.fr":
                 return False
             if msg.channel not in LAST_URLS:
                 LAST_URLS[msg.channel] = list()
-            LAST_URLS[msg.channel].append(url)
+            LAST_URLS[msg.channel].append(o.geturl())
             return True
     return False