2014-09-02 19:19:08 +00:00
|
|
|
# coding=utf-8
|
|
|
|
|
|
|
|
"""Use MediaWiki API to get pages"""
|
|
|
|
|
|
|
|
import json
|
|
|
|
import re
|
|
|
|
import urllib.parse
|
|
|
|
import urllib.request
|
|
|
|
|
|
|
|
from hooks import hook
|
2014-09-30 22:16:19 +00:00
|
|
|
from tools import web
|
2014-09-02 19:19:08 +00:00
|
|
|
|
|
|
|
nemubotversion = 3.4
|
|
|
|
|
2014-09-26 16:00:22 +00:00
|
|
|
from more import Response
|
|
|
|
|
2014-11-13 01:51:49 +00:00
|
|
|
|
2014-10-22 05:38:53 +00:00
|
|
|
def get_namespaces(site, ssl=False):
|
|
|
|
# Built URL
|
|
|
|
url = "http%s://%s/w/api.php?format=json&action=query&meta=siteinfo&siprop=namespaces" % (
|
|
|
|
"s" if ssl else "", site)
|
|
|
|
print_debug(url)
|
|
|
|
|
|
|
|
# Make the request
|
|
|
|
raw = urllib.request.urlopen(url)
|
|
|
|
data = json.loads(raw.read().decode())
|
|
|
|
|
|
|
|
namespaces = dict()
|
|
|
|
for ns in data["query"]["namespaces"]:
|
|
|
|
namespaces[data["query"]["namespaces"][ns]["*"]] = data["query"]["namespaces"][ns]
|
|
|
|
return namespaces
|
|
|
|
|
2014-11-13 01:51:49 +00:00
|
|
|
|
2014-09-02 19:19:08 +00:00
|
|
|
def get_raw_page(site, term, ssl=False):
|
2014-09-30 22:16:19 +00:00
|
|
|
# Built URL
|
2014-09-02 19:19:08 +00:00
|
|
|
url = "http%s://%s/w/api.php?format=json&redirects&action=query&prop=revisions&rvprop=content&titles=%s" % (
|
|
|
|
"s" if ssl else "", site, urllib.parse.quote(term))
|
|
|
|
print_debug(url)
|
|
|
|
|
|
|
|
# Make the request
|
|
|
|
raw = urllib.request.urlopen(url)
|
|
|
|
data = json.loads(raw.read().decode())
|
|
|
|
|
|
|
|
for k in data["query"]["pages"]:
|
2014-09-19 05:59:11 +00:00
|
|
|
try:
|
|
|
|
return data["query"]["pages"][k]["revisions"][0]["*"]
|
|
|
|
except:
|
|
|
|
raise IRCException("article not found")
|
2014-09-02 19:19:08 +00:00
|
|
|
|
2014-11-13 01:51:49 +00:00
|
|
|
|
2014-09-02 19:19:08 +00:00
|
|
|
def get_unwikitextified(site, wikitext, ssl=False):
|
2014-09-30 22:16:19 +00:00
|
|
|
# Built URL
|
2014-09-02 19:19:08 +00:00
|
|
|
url = "http%s://%s/w/api.php?format=json&action=expandtemplates&text=%s" % (
|
|
|
|
"s" if ssl else "", site, urllib.parse.quote(wikitext))
|
|
|
|
print_debug(url)
|
|
|
|
|
|
|
|
# Make the request
|
|
|
|
raw = urllib.request.urlopen(url)
|
|
|
|
data = json.loads(raw.read().decode())
|
|
|
|
|
|
|
|
return data["expandtemplates"]["*"]
|
|
|
|
|
|
|
|
|
|
|
|
def strip_model(cnt):
|
|
|
|
# Strip models at begin and end: mostly useless
|
|
|
|
cnt = re.sub(r"^(({{([^{]|\s|({{(.|\s|{{.*?}})*?}})*?)*?}}|\[\[(.|\s|\[\[.*?\]\])*?\]\])\s*)+", "", cnt)
|
|
|
|
|
|
|
|
# Strip HTML comments
|
|
|
|
cnt = re.sub(r"<!--.*?-->", "", cnt)
|
|
|
|
|
|
|
|
# Strip ref
|
|
|
|
cnt = re.sub(r"<ref.*?/ref>", "", cnt)
|
|
|
|
return cnt
|
|
|
|
|
2014-11-13 01:51:49 +00:00
|
|
|
|
2014-10-22 05:38:53 +00:00
|
|
|
def parse_wikitext(site, cnt, namespaces=dict(), ssl=False):
|
2014-11-13 01:51:49 +00:00
|
|
|
for i, _, _, _ in re.findall(r"({{([^{]|\s|({{(.|\s|{{.*?}})*?}})*?)*?}})", cnt):
|
2014-09-02 19:19:08 +00:00
|
|
|
cnt = cnt.replace(i, get_unwikitextified(site, i, ssl), 1)
|
|
|
|
|
|
|
|
# Strip [[...]]
|
2014-11-13 01:51:49 +00:00
|
|
|
for full, args, lnk in re.findall(r"(\[\[(.*?|)?([^|]*?)\]\])", cnt):
|
2014-10-22 05:38:53 +00:00
|
|
|
ns = lnk.find(":")
|
|
|
|
if lnk == "":
|
|
|
|
cnt = cnt.replace(full, args[:-1], 1)
|
|
|
|
elif ns > 0:
|
|
|
|
namespace = lnk[:ns]
|
|
|
|
if namespace in namespaces and namespaces[namespace]["canonical"] == "Category":
|
|
|
|
cnt = cnt.replace(full, "", 1)
|
|
|
|
continue
|
|
|
|
cnt = cnt.replace(full, lnk, 1)
|
|
|
|
else:
|
|
|
|
cnt = cnt.replace(full, lnk, 1)
|
2014-09-02 19:19:08 +00:00
|
|
|
|
|
|
|
# Strip HTML tags
|
2014-09-30 22:16:19 +00:00
|
|
|
cnt = web.striphtml(cnt)
|
2014-09-02 19:19:08 +00:00
|
|
|
|
|
|
|
return cnt
|
|
|
|
|
2014-11-13 01:51:49 +00:00
|
|
|
|
2014-09-02 19:19:08 +00:00
|
|
|
def irc_format(cnt):
|
2014-09-18 06:22:59 +00:00
|
|
|
cnt, _ = re.subn(r"(?P<title>==+)\s*(.*?)\s*(?P=title)\n*", "\x03\x16" + r"\2" + " :\x03\x16 ", cnt)
|
2014-09-02 19:19:08 +00:00
|
|
|
return cnt.replace("'''", "\x03\x02").replace("''", "\x03\x1f")
|
|
|
|
|
2014-11-13 01:51:49 +00:00
|
|
|
|
2014-09-02 19:19:08 +00:00
|
|
|
def get_page(site, term, ssl=False):
|
|
|
|
return strip_model(get_raw_page(site, term, ssl))
|
|
|
|
|
|
|
|
|
2014-09-30 22:16:19 +00:00
|
|
|
def opensearch(site, term, ssl=False):
|
|
|
|
# Built URL
|
|
|
|
url = "http%s://%s/w/api.php?format=xml&action=opensearch&search=%s" % (
|
|
|
|
"s" if ssl else "", site, urllib.parse.quote(term))
|
|
|
|
print_debug(url)
|
|
|
|
|
|
|
|
# Make the request
|
|
|
|
response = web.getXML(url)
|
|
|
|
|
|
|
|
if response is not None and response.hasNode("Section"):
|
|
|
|
for itm in response.getNode("Section").getNodes("Item"):
|
|
|
|
yield (itm.getNode("Text").getContent(),
|
|
|
|
itm.getNode("Description").getContent(),
|
|
|
|
itm.getNode("Url").getContent())
|
|
|
|
|
|
|
|
|
|
|
|
def search(site, term, ssl=False):
|
|
|
|
# Built URL
|
|
|
|
url = "http%s://%s/w/api.php?format=json&action=query&list=search&srsearch=%s&srprop=titlesnippet|snippet" % (
|
|
|
|
"s" if ssl else "", site, urllib.parse.quote(term))
|
|
|
|
print_debug(url)
|
|
|
|
|
|
|
|
# Make the request
|
|
|
|
raw = urllib.request.urlopen(url)
|
|
|
|
data = json.loads(raw.read().decode())
|
|
|
|
|
|
|
|
if data is not None and "query" in data and "search" in data["query"]:
|
|
|
|
for itm in data["query"]["search"]:
|
|
|
|
yield (web.striphtml(itm["titlesnippet"].replace("<span class='searchmatch'>", "\x03\x02").replace("</span>", "\x03\x02")),
|
|
|
|
web.striphtml(itm["snippet"].replace("<span class='searchmatch'>", "\x03\x02").replace("</span>", "\x03\x02")))
|
|
|
|
|
|
|
|
|
2014-10-05 16:19:20 +00:00
|
|
|
@hook("cmd_hook", "mediawiki")
|
2014-09-02 19:19:08 +00:00
|
|
|
def cmd_mediawiki(msg):
|
|
|
|
"""Read an article on a MediaWiki"""
|
|
|
|
if len(msg.cmds) < 3:
|
|
|
|
raise IRCException("indicate a domain and a term to search")
|
|
|
|
|
2014-10-22 05:38:53 +00:00
|
|
|
site = msg.cmds[1]
|
|
|
|
|
|
|
|
ns = get_namespaces(site)
|
|
|
|
|
|
|
|
return Response(get_page(site, " ".join(msg.cmds[2:])),
|
|
|
|
line_treat=lambda line: irc_format(parse_wikitext(msg.cmds[1], line, ns)),
|
2014-09-02 19:19:08 +00:00
|
|
|
channel=msg.receivers)
|
|
|
|
|
|
|
|
|
2014-10-05 16:19:20 +00:00
|
|
|
@hook("cmd_hook", "search_mediawiki")
|
2014-09-30 22:16:19 +00:00
|
|
|
def cmd_srchmediawiki(msg):
|
|
|
|
"""Search an article on a MediaWiki"""
|
|
|
|
if len(msg.cmds) < 3:
|
|
|
|
raise IRCException("indicate a domain and a term to search")
|
|
|
|
|
|
|
|
res = Response(channel=msg.receivers, nomore="No more results", count=" (%d more results)")
|
|
|
|
|
|
|
|
for r in search(msg.cmds[1], " ".join(msg.cmds[2:])):
|
|
|
|
res.append_message("%s: %s" % r)
|
|
|
|
|
|
|
|
return res
|
|
|
|
|
|
|
|
|
2014-10-05 16:19:20 +00:00
|
|
|
@hook("cmd_hook", "wikipedia")
|
2014-09-02 19:19:08 +00:00
|
|
|
def cmd_wikipedia(msg):
|
|
|
|
if len(msg.cmds) < 3:
|
|
|
|
raise IRCException("indicate a lang and a term to search")
|
|
|
|
|
2014-09-18 06:22:59 +00:00
|
|
|
site = msg.cmds[1] + ".wikipedia.org"
|
|
|
|
|
2014-10-22 05:38:53 +00:00
|
|
|
ns = get_namespaces(site)
|
|
|
|
|
2014-09-18 06:22:59 +00:00
|
|
|
return Response(get_page(site, " ".join(msg.cmds[2:])),
|
2014-10-22 05:38:53 +00:00
|
|
|
line_treat=lambda line: irc_format(parse_wikitext(site, line, ns)),
|
2014-09-18 05:57:06 +00:00
|
|
|
channel=msg.receivers)
|