2012-06-29 17:20:29 +00:00
|
|
|
# coding=utf-8
|
|
|
|
|
2014-08-27 23:39:31 +00:00
|
|
|
"""Alert on changes on websites"""
|
|
|
|
|
2012-09-01 14:16:31 +00:00
|
|
|
from datetime import datetime
|
|
|
|
from datetime import timedelta
|
|
|
|
import http.client
|
|
|
|
import hashlib
|
2014-09-19 17:28:48 +00:00
|
|
|
from random import randint
|
2012-09-01 14:16:31 +00:00
|
|
|
import re
|
|
|
|
import socket
|
|
|
|
import sys
|
2012-11-07 17:55:17 +00:00
|
|
|
import urllib.parse
|
|
|
|
from urllib.parse import urlparse
|
2012-06-29 17:20:29 +00:00
|
|
|
|
2014-08-12 18:10:19 +00:00
|
|
|
from hooks import hook
|
|
|
|
|
2014-09-26 16:00:22 +00:00
|
|
|
nemubotversion = 3.4
|
|
|
|
|
2014-09-19 22:15:06 +00:00
|
|
|
from networking import w3m
|
2012-09-01 14:16:31 +00:00
|
|
|
from .atom import Atom
|
2014-09-26 16:00:22 +00:00
|
|
|
from more import Response
|
2012-06-29 17:20:29 +00:00
|
|
|
|
|
|
|
def help_full ():
|
|
|
|
return "This module is autonomous you can't interract with it."
|
|
|
|
|
2012-09-01 14:16:31 +00:00
|
|
|
def load(context):
|
|
|
|
"""Register watched website"""
|
2012-11-07 17:55:17 +00:00
|
|
|
DATAS.setIndex("url", "watch")
|
2012-09-01 14:16:31 +00:00
|
|
|
for site in DATAS.getNodes("watch"):
|
2012-12-21 10:31:03 +00:00
|
|
|
if site.hasNode("alert"):
|
2014-09-19 17:28:48 +00:00
|
|
|
start_watching(site, randint(-30, 30))
|
2012-12-21 10:31:03 +00:00
|
|
|
else:
|
|
|
|
print("No alert defined for this site: " + site["url"])
|
|
|
|
#DATAS.delChild(site)
|
2012-09-01 14:16:31 +00:00
|
|
|
|
2014-09-19 17:28:48 +00:00
|
|
|
def start_watching(site, offset=0):
|
2012-11-07 17:55:17 +00:00
|
|
|
o = urlparse(site["url"], "http")
|
|
|
|
print_debug("Add event for site: %s" % o.netloc)
|
2014-09-19 22:15:06 +00:00
|
|
|
evt = ModuleEvent(func=lambda url: w3m(url), cmp_data=site["lastcontent"],
|
2014-09-19 17:28:48 +00:00
|
|
|
func_data=site["url"], offset=offset,
|
2014-09-23 20:27:53 +00:00
|
|
|
interval=site.getInt("time"),
|
2012-09-01 14:16:31 +00:00
|
|
|
call=alert_change, call_data=site)
|
2012-11-07 17:55:17 +00:00
|
|
|
site["_evt_id"] = add_event(evt)
|
2012-09-01 14:16:31 +00:00
|
|
|
|
|
|
|
|
2014-08-12 18:10:19 +00:00
|
|
|
@hook("cmd_hook", "unwatch")
|
2012-09-01 14:16:31 +00:00
|
|
|
def del_site(msg):
|
2012-11-04 14:32:39 +00:00
|
|
|
if len(msg.cmds) <= 1:
|
2014-07-25 16:02:30 +00:00
|
|
|
raise IRCException("quel site dois-je arrêter de surveiller ?")
|
2012-09-01 14:16:31 +00:00
|
|
|
|
2012-11-07 17:55:17 +00:00
|
|
|
url = msg.cmds[1]
|
|
|
|
|
|
|
|
o = urlparse(url, "http")
|
|
|
|
if o.scheme != "" and url in DATAS.index:
|
|
|
|
site = DATAS.index[url]
|
|
|
|
for a in site.getNodes("alert"):
|
|
|
|
if a["channel"] == msg.channel:
|
2014-08-06 14:20:51 +00:00
|
|
|
if not (msg.sender == a["sender"] or msg.is_owner):
|
2014-07-25 16:02:30 +00:00
|
|
|
raise IRCException("vous ne pouvez pas supprimer cette URL.")
|
|
|
|
site.delChild(a)
|
|
|
|
if not site.hasNode("alert"):
|
|
|
|
del_event(site["_evt_id"])
|
|
|
|
DATAS.delChild(site)
|
|
|
|
save()
|
2014-09-18 05:57:06 +00:00
|
|
|
return Response("je ne surveille désormais plus cette URL.",
|
2014-07-25 16:02:30 +00:00
|
|
|
channel=msg.channel, nick=msg.nick)
|
|
|
|
raise IRCException("je ne surveillais pas cette URL !")
|
2012-09-01 14:16:31 +00:00
|
|
|
|
2014-08-12 18:10:19 +00:00
|
|
|
|
|
|
|
@hook("cmd_hook", "watch", data="diff")
|
|
|
|
@hook("cmd_hook", "updown", data="updown")
|
2012-12-07 20:18:21 +00:00
|
|
|
def add_site(msg, diffType="diff"):
|
2012-12-08 08:21:27 +00:00
|
|
|
print (diffType)
|
2012-11-04 14:32:39 +00:00
|
|
|
if len(msg.cmds) <= 1:
|
2014-07-25 16:02:30 +00:00
|
|
|
raise IRCException("quel site dois-je surveiller ?")
|
2012-09-01 14:16:31 +00:00
|
|
|
|
2012-11-07 17:55:17 +00:00
|
|
|
url = msg.cmds[1]
|
|
|
|
|
|
|
|
o = urlparse(url, "http")
|
2014-07-25 16:02:30 +00:00
|
|
|
if o.netloc == "":
|
|
|
|
raise IRCException("je ne peux pas surveiller cette URL")
|
|
|
|
|
|
|
|
alert = ModuleState("alert")
|
|
|
|
alert["sender"] = msg.sender
|
|
|
|
alert["server"] = msg.server
|
|
|
|
alert["channel"] = msg.channel
|
2014-09-10 10:19:25 +00:00
|
|
|
alert["message"] = "{url} a changé !"
|
2014-07-25 16:02:30 +00:00
|
|
|
|
|
|
|
if url not in DATAS.index:
|
|
|
|
watch = ModuleState("watch")
|
|
|
|
watch["type"] = diffType
|
|
|
|
watch["url"] = url
|
|
|
|
watch["time"] = 123
|
|
|
|
DATAS.addChild(watch)
|
|
|
|
watch.addChild(alert)
|
|
|
|
start_watching(watch)
|
2012-11-07 17:55:17 +00:00
|
|
|
else:
|
2014-07-25 16:02:30 +00:00
|
|
|
DATAS.index[url].addChild(alert)
|
2012-09-01 14:16:31 +00:00
|
|
|
|
|
|
|
save()
|
2014-09-18 05:57:06 +00:00
|
|
|
return Response(channel=msg.channel, nick=msg.nick,
|
2012-09-01 14:16:31 +00:00
|
|
|
message="ce site est maintenant sous ma surveillance.")
|
|
|
|
|
2014-09-10 10:19:25 +00:00
|
|
|
def format_response(site, link='%s', title='%s', categ='%s', content='%s'):
|
2012-11-07 17:55:17 +00:00
|
|
|
for a in site.getNodes("alert"):
|
2014-09-18 05:57:06 +00:00
|
|
|
send_response(a["server"], Response(a["message"].format(url=site["url"], link=link, title=title, categ=categ, content=content),
|
2012-11-07 17:55:17 +00:00
|
|
|
channel=a["channel"], server=a["server"]))
|
|
|
|
|
2012-09-01 14:16:31 +00:00
|
|
|
def alert_change(content, site):
|
|
|
|
"""Alert when a change is detected"""
|
2012-12-07 20:18:21 +00:00
|
|
|
if site["type"] == "updown":
|
|
|
|
if site["lastcontent"] is None:
|
|
|
|
site["lastcontent"] = content is not None
|
|
|
|
|
2012-12-08 08:21:27 +00:00
|
|
|
if (content is not None) != site.getBool("lastcontent"):
|
2012-12-07 20:18:21 +00:00
|
|
|
format_response(site, link=site["url"])
|
|
|
|
site["lastcontent"] = content is not None
|
2012-12-08 08:21:27 +00:00
|
|
|
start_watching(site)
|
2012-12-07 20:18:21 +00:00
|
|
|
return
|
|
|
|
|
2012-09-01 14:16:31 +00:00
|
|
|
if content is None:
|
2012-10-15 00:49:48 +00:00
|
|
|
start_watching(site)
|
2012-09-01 14:16:31 +00:00
|
|
|
return
|
|
|
|
|
|
|
|
if site["type"] == "atom":
|
|
|
|
if site["_lastpage"] is None:
|
2013-01-25 16:47:11 +00:00
|
|
|
if site["lastcontent"] is None or site["lastcontent"] == "":
|
|
|
|
site["lastcontent"] = content
|
|
|
|
site["_lastpage"] = Atom(site["lastcontent"])
|
2012-11-01 08:27:19 +00:00
|
|
|
try:
|
|
|
|
page = Atom(content)
|
|
|
|
except:
|
|
|
|
print ("An error occurs during Atom parsing. Restart event...")
|
|
|
|
start_watching(site)
|
|
|
|
return
|
2012-09-01 14:16:31 +00:00
|
|
|
diff = site["_lastpage"].diff(page)
|
|
|
|
if len(diff) > 0:
|
|
|
|
site["_lastpage"] = page
|
|
|
|
diff.reverse()
|
|
|
|
for d in diff:
|
2012-09-07 12:59:22 +00:00
|
|
|
site.setIndex("term", "category")
|
|
|
|
categories = site.index
|
2012-09-01 14:16:31 +00:00
|
|
|
|
2012-11-07 17:55:17 +00:00
|
|
|
if len(categories) > 0:
|
2012-09-01 14:16:31 +00:00
|
|
|
if d.category is None or d.category not in categories:
|
2012-12-01 01:20:09 +00:00
|
|
|
format_response(site, link=d.link, categ=categories[""]["part"], title=d.title)
|
2012-09-01 14:16:31 +00:00
|
|
|
else:
|
2012-12-01 01:20:09 +00:00
|
|
|
format_response(site, link=d.link, categ=categories[d.category]["part"], title=d.title)
|
2012-09-01 14:16:31 +00:00
|
|
|
else:
|
2012-12-01 01:20:09 +00:00
|
|
|
format_response(site, link=d.link, title=urllib.parse.unquote(d.title))
|
2012-09-01 14:16:31 +00:00
|
|
|
else:
|
2012-10-15 00:49:48 +00:00
|
|
|
start_watching(site)
|
2012-09-01 14:16:31 +00:00
|
|
|
return #Stop here, no changes, so don't save
|
2012-06-29 17:20:29 +00:00
|
|
|
|
2012-09-01 14:16:31 +00:00
|
|
|
else: # Just looking for any changes
|
2014-09-10 10:19:25 +00:00
|
|
|
format_response(site, link=site["url"], content=content)
|
2012-09-01 14:16:31 +00:00
|
|
|
site["lastcontent"] = content
|
2012-10-15 00:49:48 +00:00
|
|
|
start_watching(site)
|
2012-09-01 14:16:31 +00:00
|
|
|
save()
|