From daabc5f39a1ed741ca88941d39856d6211f729ac Mon Sep 17 00:00:00 2001 From: sfan5 Date: Sun, 20 Jul 2014 16:13:59 +0200 Subject: [PATCH] =?UTF-8?q?=E2=86=AA=20Python=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + COMMANDS.md | 7 +- admin.py | 2 +- calc.py | 29 +++++--- devwiki.py | 190 +++++++++++--------------------------------------- forumutils.py | 4 +- log.py | 28 ++++---- ping.py | 4 +- reload.py | 4 +- rssnotify.py | 8 +-- rutils.py | 50 +------------ search.py | 60 +++++----------- seen.py | 12 ++-- server.py | 2 +- serverup.py | 2 +- shorten.py | 38 ---------- shortutils.py | 9 +-- startup.py | 18 ++--- tell.py | 3 +- title.py | 112 ++++++++--------------------- twitter.py | 19 +++-- wiki.py | 190 +++++++++++--------------------------------------- 22 files changed, 206 insertions(+), 586 deletions(-) mode change 100644 => 100755 log.py delete mode 100755 shorten.py diff --git a/.gitignore b/.gitignore index e7afa1b..2844c3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.py[cod] *~ +__pycache__ diff --git a/COMMANDS.md b/COMMANDS.md index f029c87..d84cb3c 100644 --- a/COMMANDS.md +++ b/COMMANDS.md @@ -45,9 +45,7 @@ Required arguments are enclosed in { and }, optional arguments are enclosed in [ !b32d {string} Base32-decode a string Anyone !b16e {string} Base16-encode a string Anyone !b16d {string} Base16-decode a string Anyone - !crc32 {string} Hash a string using crc32 Anyone - !hash {hashtype} {string} Hash a string using various hash functions Anyone - !re {regex}`{string} check if regex matches, if it does print groups Anyone + !hash {hashtype} {string} Hash a string, supports various hash functions Anyone !rand [min] {max} Says a random number between(incl.) min and max Anyone search.py !g {string} Output first Google result for string Anyone @@ -74,9 +72,6 @@ Required arguments are enclosed in { and }, optional arguments are enclosed in [ serverup.py !up {IP/hostname} [port] Check if server at IP/hostname is up Anyone Supports multiple Ports e.g. 123-456,999 - shorten.py - !sh {service} {url} Shortens URL Anyone - Currently supports: is.gd, v.gd title.py !title [link] Query Page Title Anyone twitter.py diff --git a/admin.py b/admin.py index 58d4ad8..68a29a2 100755 --- a/admin.py +++ b/admin.py @@ -80,4 +80,4 @@ py.commands = ['py'] py.priority = 'high' if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/calc.py b/calc.py index bfcdbc1..9921100 100755 --- a/calc.py +++ b/calc.py @@ -14,13 +14,24 @@ class SomeObject(object): pass env = { - "bin": bin, "abs": abs, "oct": oct, "int": int, "sum": sum, - "tuple": tuple, "divmod": divmod, "hash": hash, "hex": hex, - "len": len, "list": list, "long": long, "max": max, - "range": range, "round": round, "min": min, "map": map, - "zip": zip, "xrange": xrange, "unicode": unicode, - "unichr": unichr, "type": type, "slice": slice, "ord": ord, - "chr": chr, "str": str, "float": float + "abs": abs, "all": all, "any": any, + "ascii": ascii, "bin": bin, "bool": bool, + "bytearray": bytearray, "bytes": bytes, + "callable": callable, "chr": chr, + "complex": complex, "dict": dict, + "divmod": divmod, + "enumerate": enumerate, "filter": filter, + "float": format, "hex": hex, "int": int, + "iter": iter, "len": len, "list": list, + "map": map, "max": max, "min": min, + "next": next, "object": object, + "oct": oct, "ord": ord, "pow": pow, + "range": range, "repr": repr, + "reversed": reversed, "round": round, + "set": set, "slice": slice, + "sorted": sorted, "str": str, + "sum": sum, "tuple": tuple, + "type": type, "zip": zip, } libs = [ @@ -37,7 +48,7 @@ for lib in libs: def c(phenny, input): if not input.group(2): return phenny.reply("Nothing to calculate.") - q = input.group(2).encode('ascii', 'ignore') + q = input.group(2) if '__' in q: return phenny.reply("Sorry, but no double underscores.") log.log("event", "%s calculated '%s'" % (log.fmt_user(input), q), phenny) @@ -67,4 +78,4 @@ c.commands = ['c'] c.example = '.c 5 + 3' if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/devwiki.py b/devwiki.py index 419f2ca..10f7400 100755 --- a/devwiki.py +++ b/devwiki.py @@ -1,164 +1,56 @@ #!/usr/bin/env python """ -devwiki.py - Phenny Wiki Module -Copyright 2008-9, Sean B. Palmer, inamidst.com -Modified by Sfan5 2013 -Licensed under the Eiffel Forum License 2. - -http://inamidst.com/phenny/ +wiki.py - Phenny Wiki Module +Copyright 2014, sfan5 """ -import re, urllib, gzip, StringIO +import re import web +import urllib.parse -devwikiuri = 'http://dev.minetest.net/%s' +wikiuri_g = 'http://dev.minetest.net/index.php?title=%s&printable=yes' +wikiuri_r = 'http://dev.minetest.net/index.php?title=%s' -r_tr = re.compile(r'(?ims)]*>.*?') -r_paragraph = re.compile(r'(?ims)]*>.*?

|]*>.*?') -r_tag = re.compile(r'<(?!!)[^>]+>') -r_whitespace = re.compile(r'[\t\r\n ]+') -r_redirect = re.compile( - r'(?ims)class=.redirectText.>\s*]+class=.mw-content-ltr.>') +r_paragraph = re.compile(r'(?ims)

(.+?)

') +r_sentenceend = re.compile(r'\.[^\.]') +transforms = [ + re.compile(r'(?i)]+>(.+?)'), + re.compile(r'(?i)(.+?)'), + re.compile(r'(?i)(.+?)'), +] -abbrs = ['etc', 'ca', 'cf', 'Co', 'Ltd', 'Inc', 'Mt', 'Mr', 'Mrs', - 'Dr', 'Ms', 'Rev', 'Fr', 'St', 'Sgt', 'pron', 'approx', 'lit', - 'syn', 'transl', 'sess', 'fl', 'Op', 'Dec', 'Brig', 'Gen'] \ - + list('ABCDEFGHIJKLMNOPQRSTUVWXYZ') \ - + list('abcdefghijklmnopqrstuvwxyz') -t_sentence = r'^.{5,}?(?') - s = s.replace('<', '<') - s = s.replace('&', '&') - s = s.replace(' ', ' ') - return s + log.log("event", "%s queried Developer Wiki for '%s'" % (log.fmt_user(input), term), phenny) + term = web.urlencode(term) -def text(html): - html = r_tag.sub('', html) - html = r_whitespace.sub(' ', html) - return unescape(html).strip() + data, scode = web.get(wikiuri_g % term) + if scode == 404: + return phenny.say("No such page.") + data = str(data, "utf-8") -def devwikipedia(term, language='en', last=False): - global devwikiuri - if not '%' in term: - if isinstance(term, unicode): - t = term.encode('utf-8') - else: t = term - q = urllib.quote(t) - u = devwikiuri % (q) - bytes = web.get(u) - else: bytes = web.get(devwikiuri % (term)) + m = re.search(r_content, data) + if not m: + return phenny.say("Sorry, did not find anything.") + data = data[m.span()[1]:] - if bytes.startswith('\x1f\x8b\x08\x00\x00\x00\x00\x00'): - f = StringIO.StringIO(bytes) - f.seek(0) - gzip_file = gzip.GzipFile(fileobj=f) - bytes = gzip_file.read() - gzip_file.close() - f.close() + m = re.search(r_paragraph, data) + if not m: + return phenny.say("Sorry, did not find anything.") + data = m.group(1) + for transform in transforms: + data = re.sub(transform, '\g<1>', data) + m = re.search(r_sentenceend, data) + if m: + data = data[:m.span()[1]-1] + phenny.say('"%s" - %s ' % (web.decode(data), wikiuri_r % term)) - bytes = r_tr.sub('', bytes) - - if not last: - r = r_redirect.search(bytes[:4096]) - if r: - term = urllib.unquote(r.group(1)) - return devwikipedia(term, language=language, last=True) - - paragraphs = r_paragraph.findall(bytes) - - if not paragraphs: - if not last: - term = search(term) - return devwikipedia(term, language=language, last=True) - return None - - # Pre-process - paragraphs = [para for para in paragraphs - if (para and 'technical limitations' not in para - and 'window.showTocToggle' not in para - and 'Deletion_policy' not in para - and 'Template:AfD_footer' not in para - and not (para.startswith('

') and - para.endswith('

')) - and not 'disambiguation)"' in para) - and not '(images and media)' in para - and not 'This article contains a' in para - and not 'id="coordinates"' in para - and not 'class="thumb' in para] - # and not 'style="display:none"' in para] - - for i, para in enumerate(paragraphs): - para = para.replace('', '|') - para = para.replace('', '|') - paragraphs[i] = text(para).strip() - - # Post-process - paragraphs = [para for para in paragraphs if - (para and not (para.endswith(':') and len(para) < 150))] - - para = text(paragraphs[0]) - m = r_sentence.match(para) - - if not m: - if not last: - term = search(term) - return devwikipedia(term, language=language, last=True) - return None - sentence = m.group(0) - - maxlength = 275 - if len(sentence) > maxlength: - sentence = sentence[:maxlength] - words = sentence[:-5].split(' ') - words.pop() - sentence = ' '.join(words) + ' [...]' - - if (('using the Article Wizard if you wish' in sentence) - or ('or add a request for it' in sentence) - or ('in existing articles' in sentence)): - if not last: - term = search(term) - return devwikipedia(term, language=language, last=True) - return None - - sentence = '"' + sentence.replace('"', "'") + '"' - sentence = sentence.decode('utf-8').encode('utf-8') - devwikiuri = devwikiuri.decode('utf-8').encode('utf-8') - term = term.decode('utf-8').encode('utf-8') - return sentence + ' - ' + (devwikiuri % (term)) - -def devwik(phenny, input): - origterm = input.groups()[1] - if not origterm: - return phenny.say('Perhaps you meant "!devwik Zen"?') - origterm = origterm.encode('utf-8') - log.log("event", "%s queried Devwiki for '%s'" % (log.fmt_user(input), origterm), phenny) - - term = urllib.unquote(origterm) - language = 'en' - if term.startswith(':') and (' ' in term): - a, b = term.split(' ', 1) - a = a.lstrip(':') - if a.isalpha(): - language, term = a, b - term = term.replace(' ', '_') - - try: result = devwikipedia(term, language) - except IOError: - args = (language, devwikiuri % (term)) - error = "Can't connect to dev.minetest.net (%s)" % args - return phenny.say(error) - - if result is not None: - phenny.say(result) - else: phenny.say('Can\'t find anything in Dev Wiki for "%s".' % origterm) - -devwik.commands = ['dev', 'devwik', 'devwiki'] -devwik.priority = 'high' +wiki.commands = ['devwik', 'devwiki'] +wiki.priority = 'high' if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/forumutils.py b/forumutils.py index d9392c5..cda849f 100755 --- a/forumutils.py +++ b/forumutils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ forumutils.py - Phenny Minetest Forum Module -Copyright 2012, Sfan5 +Copyright 2012, sfan5 """ import web @@ -137,4 +137,4 @@ search_forumuser_limit.commands = ['searchforumuserlimit', 'sfulimit'] search_forumuser_limit.priority = 'low' if __name__ == '__main__': - print __doc__ + print(__doc__) diff --git a/log.py b/log.py old mode 100644 new mode 100755 index 6072071..41d0037 --- a/log.py +++ b/log.py @@ -7,36 +7,36 @@ Copyright 2014, sfan5 import time loglevels = { - 'TEXT': 'TXT', # TEXT: purely informal message that appears in masses e.g. a command being executed - 'EVENT': 'EVT', # EVENT: like TEXT events but more likely someone wants to see those - 'ACTION': 'ACT', # ACTION: something the bot decided on doing automatically, requires attention - 'WARNING': 'WRN', # WARNING: a warning + 'TEXT': 'TXT', # TEXT: purely informal message that appears in masses e.g. a command being executed + 'EVENT': 'EVT', # EVENT: like TEXT events but more likely someone wants to see those + 'ACTION': 'ACT', # ACTION: something the bot decided on doing automatically, requires attention + 'WARNING': 'WRN', # WARNING: a warning } actionchannel = "##minetestbot" actionhighlight = "sfan5" def log(level, text, phenny): - level = level.upper() - f = open("bot.log", "ab") - f.write(time.strftime("%F %H:%M:%S %z") + "\t" + loglevels[level] + "\t" + text.encode('utf-8') + "\n") - f.close() - if level == 'ACTION': - phenny.write(['PRIVMSG', actionchannel], actionhighlight + ": " + text) - print(level + " " + text) + level = level.upper() + f = open("bot.log", "a") + f.write(time.strftime("%F %H:%M:%S %z") + "\t" + loglevels[level] + "\t" + text + "\n") + f.close() + if level == 'ACTION': + phenny.write(['PRIVMSG', actionchannel], actionhighlight + ": " + text) + print(level + " " + text) def fmt_user(input): - return "%s(%s)" % (input.nick, input.hostmask) + return "%s(%s)" % (input.nick, input.hostmask) class SomeObject(object): - pass + pass log_api = SomeObject() log_api.log = log log_api.fmt_user = fmt_user _export = { - 'log': log_api, + 'log': log_api, } def log_text(phenny, input, func): diff --git a/ping.py b/ping.py index 5b679ce..e674974 100755 --- a/ping.py +++ b/ping.py @@ -20,8 +20,8 @@ interjection.priority = 'high' def l3(phenny, input): phenny.say('<3 ' + input.nick) -l3.rule = r'<3 $nickname' +l3.rule = r'<3 $nickname\s*' l3.priority = 'low' if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/reload.py b/reload.py index a52f4ee..c28a456 100755 --- a/reload.py +++ b/reload.py @@ -24,7 +24,7 @@ def f_reload(phenny, input): phenny.setup() return phenny.reply('done') - if not sys.modules.has_key(name): + if not name in sys.modules: return phenny.reply('%s: no such module!' % name) # Thanks to moot for prodding me on this @@ -53,4 +53,4 @@ f_reload.priority = 'low' f_reload.thread = False if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/rssnotify.py b/rssnotify.py index 0bd3f36..43dd971 100755 --- a/rssnotify.py +++ b/rssnotify.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ rssnotify.py - Phenny RssNotify Module -Copyright 2013, Sfan5 +Copyright 2013, sfan5 """ import time, urllib, re import feedparser # sudo pip install feedparser @@ -51,7 +51,7 @@ def rsscheck(phenny, input): ('https://github.com/Uberi/MineTest-WorldEdit/commits/master.atom', allchans), ('https://github.com/Jeija/minetest-mod-mesecons/commits/master.atom', allchans), ] - for v in xrange(0, len(feeds)): + for v in range(0, len(feeds)): url = feeds[v][0] feednum = v options = { @@ -111,7 +111,7 @@ def rsscheck(phenny, input): phenny.write(['PRIVMSG', ch],"GIT: %s commited to %s: %s %s %s %s" % (commiter, reponame, feed_entry.title, commit_hash, commit_time, commit_link)) if len(feed.entries) > 0: m = -1 - for i in xrange(0, len(feed.entries)): + for i in range(0, len(feed.entries)): if to_unix_time(feed.entries[i].updated) > m: m = to_unix_time(feed.entries[i].updated) rssnotify["last_updated_feeds"][feednum] = m @@ -128,4 +128,4 @@ rsscheck.event = '*' rsscheck.nohook = True if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/rutils.py b/rutils.py index 11a78a7..3b63fa8 100755 --- a/rutils.py +++ b/rutils.py @@ -12,7 +12,7 @@ def rev(phenny, input): """reverse string""" if not input.group(2): return phenny.reply("Nothing to reverse.") - q = input.group(2).encode('utf-8') + q = input.group(2) s = "" for i in range(1,len(q)): s += q[-i] @@ -27,7 +27,7 @@ def make_thing(cmds, func): if not input.group(2): return q = input.group(2).encode('utf-8') try: - phenny.say(rs(func(q))) + phenny.say(rs(func(q).decode('utf-8'))) except BaseException as e: phenny.reply("Failed to handle data") m.commands = cmds @@ -41,17 +41,6 @@ b32d = make_thing(['b32d','base32decode'], base64.b32decode) b16e = make_thing(['b16e','base16encode'], base64.b16encode) b16d = make_thing(['b16d','base16decode'], base64.b16decode) -def crc32(phenny, input): - """crc32 hash""" - if not input.group(2): - return phenny.reply("Nothing to hash.") - q = input.group(2).encode('utf-8') - h = binascii.crc32(q) - return phenny.say(rs(str(h) + "(" + hex(h) + ")")) - -crc32.commands = ['crc32'] -crc32.priority = 'low' - def hash_(phenny, input): if not input.group(2): return phenny.reply("Usage: hash | Get available hash funcs with ?") @@ -74,39 +63,6 @@ def hash_(phenny, input): hash_.commands = ['hash'] hash_.priority = 'low' -def regex(phenny, input): - """regex""" - if not input.group(2): - return phenny.reply("Give me a regex and a message seperated by `") - q = input.group(2).encode('utf-8') - rgx = q[:q.find("`")] - txt = q[q.find("`")+1:] - if rgx == "" or txt == "": - return phenny.reply("Give me a regex and a message seperated by `") - try: - r = re.compile(rgx) - except BaseException as e: - return phenny.reply("Failed to compile regex: " + e.message) - q = multiprocessing.Queue() - def compute(q, re, rgx, txt): - q.put(list(e.groups()[0] for e in list(re.finditer(rgx, txt)))) - t = multiprocessing.Process(target=compute, args=(q,re,rgx,txt)) - t.start() - t.join(3.0) - if t.is_alive(): - t.terminate() - phenny.reply("Regex took to long to compute") - return - m = q.get() - if m == []: - return phenny.say("false") - else: - return phenny.say("true groups=[" + ', '.join((repr(e) for e in m)) + "]") - -regex.commands = ['re','regex'] -regex.priority = 'low' -regex.thread = True - def rand(phenny, input): """Returns a random number""" if not input.group(2): @@ -138,4 +94,4 @@ rand.commands = ['rand', 'random'] rand.priority = 'low' if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/search.py b/search.py index 81a5f70..d67dcd0 100755 --- a/search.py +++ b/search.py @@ -8,59 +8,30 @@ Licensed under the Eiffel Forum License 2. http://inamidst.com/phenny/ """ -import web, re - -class Grab(web.urllib.URLopener): - def __init__(self, *args): - self.version = 'Mozilla/5.0 (MinetestBot)' - web.urllib.URLopener.__init__(self, *args) - self.addheader('Referer', 'http://minetest.net') - def http_error_default(self, url, fp, errcode, errmsg, headers): - return web.urllib.addinfourl(fp, [headers, errcode], "http:" + url) +import web +import re def google_ajax(query): """Search using AjaxSearch, and return its JSON.""" - if isinstance(query, unicode): - query = query.encode('utf-8') uri = 'http://ajax.googleapis.com/ajax/services/search/web' - args = '?v=1.0&safe=off&q=' + web.urllib.quote(query) - handler = web.urllib._urlopener - web.urllib._urlopener = Grab() - bytes = web.get(uri + args) - web.urllib._urlopener = handler - return web.json(bytes) + args = '?v=1.0&safe=off&q=' + web.urlencode(query) + data, sc = web.get(uri + args) + data = str(data, 'utf-8') + return web.json(data) def google_search(query): results = google_ajax(query) try: return results['responseData']['results'][0]['unescapedUrl'] except IndexError: return None except TypeError: - print results return False -def google_count(query): - results = google_ajax(query) - if not results.has_key('responseData'): return '0' - if not results['responseData'].has_key('cursor'): return '0' - if not results['responseData']['cursor'].has_key('estimatedResultCount'): - return '0' - return results['responseData']['cursor']['estimatedResultCount'] - -def formatnumber(n): - """Format a number with beautiful commas.""" - parts = list(str(n)) - for i in range((len(parts) - 3), 0, -3): - parts.insert(i, ',') - return ''.join(parts) - - def g(phenny, input): """Queries Google for the specified input.""" query = input.group(2) if not query: return phenny.reply('.g what?') - query = query.encode('utf-8') - log.log("%s searched Google for '%s'" % (log.fmt_user(input), query)) + log.log("event", "%s searched Google for '%s'" % (log.fmt_user(input), query), phenny) uri = google_search(query) if uri: phenny.reply(uri) @@ -72,10 +43,10 @@ g.priority = 'high' g.example = '.g minetest' def gc(phenny, input): - if not input.group(2): + query = input.group(2) + if not query: return phenny.reply("No query term.") - query = input.group(2).encode('utf-8') - log.log("%s searched Google for '%s'" % (log.fmt_user(input), query)) + log.log("event", "%s searched Google for '%s'" % (log.fmt_user(input), query), phenny) result = new_gc(query) if result: phenny.say(query + ": " + result) @@ -83,12 +54,13 @@ def gc(phenny, input): def new_gc(query): uri = 'https://www.google.com/search?hl=en&q=' - uri = uri + web.urllib.quote(query).replace('+', '%2B') + uri = uri + web.urlencode(query).replace('+', '%2B') if '"' in query: uri += '&tbs=li:1' - bytes = web.get(uri) - if "did not match any documents" in bytes: + data, sc = web.get(uri) + data = str(data, 'utf-8') + if "did not match any documents" in data: return "0" - for result in re.compile(r'(?ims)([0-9,]+) results?').findall(bytes): + for result in re.compile(r'(?ims)([0-9,]+) results?').findall(data): return result return None @@ -97,4 +69,4 @@ gc.priority = 'high' gc.example = '.gc minetest' if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/seen.py b/seen.py index e2c4d79..eae355f 100755 --- a/seen.py +++ b/seen.py @@ -9,13 +9,12 @@ http://inamidst.com/phenny/ """ import time -from tools import deprecated -from thread import start_new_thread, allocate_lock +from threading import Thread, Lock import sqlite3 updates = list() -update_l = allocate_lock() -dblock = allocate_lock() +update_l = Lock() +dblock = Lock() def opendb(): db = sqlite3.connect("seen.sqlite") @@ -130,7 +129,8 @@ c.execute('''CREATE TABLE IF NOT EXISTS seen (nick text, channel text, time int) c.close() db.close() -start_new_thread(updatethread, ()) +t = Thread(target=updatethread, name="seen.py database thread") +t.start() if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/server.py b/server.py index 988832b..8e6a0ee 100755 --- a/server.py +++ b/server.py @@ -148,4 +148,4 @@ def server(phenny, input): server.commands = ['sv', 'server'] if __name__ == '__main__': - print __doc__ + print(__doc__) diff --git a/serverup.py b/serverup.py index a77d50f..7a3cca3 100755 --- a/serverup.py +++ b/serverup.py @@ -80,4 +80,4 @@ serverup.commands = ['up'] serverup.thread = True if __name__ == '__main__': - print __doc__ + print(__doc__) diff --git a/shorten.py b/shorten.py deleted file mode 100755 index d824d95..0000000 --- a/shorten.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -""" -shorten.py - URL Shorten Module -Copyright 2013, sfan5 -""" -import urllib - -def shorten(phenny, input): - arg = input.group(2) - if not arg: - arg = "" # Function continues and prints Help Message - arg = arg.split(' ') - if len(arg) < 2: - phenny.reply("Give me an url shorten service and an address") - return phenny.reply("Supported Services: is.gd, v.gd") - else: - if arg[0].lower() == "is.gd": - p = urllib.urlencode({'format' :"simple", 'url': arg[1]}) - try: - u = urllib.urlopen("http://is.gd/create.php?%s" % p) - return phenny.reply(u.read()) - except: - return phenny.reply("Problems accessing is.gd, please try a different Service") - if arg[0].lower() == "v.gd": - p = urllib.urlencode({'format' :"simple", 'url': arg[1]}) - try: - u = urllib.urlopen("http://v.gd/create.php?%s" % p) - return phenny.reply(u.read()) - except: - return phenny.reply("Problems accessing v.gd, please try a different Service") - return phenny.reply("Unknown Service") - - -shorten.commands = ['shorten','sh'] -shorten.thread = True - -if __name__ == '__main__': - print __doc__ diff --git a/shortutils.py b/shortutils.py index d2e67cd..393b2ce 100755 --- a/shortutils.py +++ b/shortutils.py @@ -10,8 +10,7 @@ http://inamidst.com/phenny/ """ import random -import urllib2 -import json +import web def make_cmd(cmd, txt): def m(phenny, input): @@ -41,10 +40,8 @@ next.commands = ['next'] def doge(phenny, input): """much wow, very function, such programming""" if random.randint(0, 1) == 0: - f = urllib2.urlopen('http://pubapi.cryptsy.com/api.php?method=singlemarketdata&marketid=132') - data = f.read() - f.close() - data = json.loads(data) + data = web.get('http://pubapi.cryptsy.com/api.php?method=singlemarketdata&marketid=132') + data = web.json(data) phenny.say("DOGE is at " + data['return']['markets']['DOGE']['lasttradeprice'] + " BTC") else: links = [ diff --git a/startup.py b/startup.py index 81b3ecf..e40202f 100755 --- a/startup.py +++ b/startup.py @@ -9,7 +9,7 @@ http://inamidst.com/phenny/ import threading, time -def setup(phenny): +def setup(phenny): print("Setting up phenny") # by clsn phenny.data = {} @@ -20,9 +20,9 @@ def setup(phenny): except: pass def close(): - print "Nobody PONGed our PING, restarting" + print("Nobody PONGed our PING, restarting") phenny.handle_close() - + def pingloop(): timer = threading.Timer(refresh_delay, close, ()) phenny.data['startup.setup.timer'] = timer @@ -43,27 +43,27 @@ def setup(phenny): pong.rule = r'.*' phenny.variables['pong'] = pong -def startup(phenny, input): +def startup(phenny, input): import time # Start the ping loop. Has to be done after USER on e.g. quakenet if phenny.data.get('startup.setup.pingloop'): phenny.data['startup.setup.pingloop']() - if hasattr(phenny.config, 'serverpass'): + if hasattr(phenny.config, 'serverpass'): phenny.write(('PASS', phenny.config.serverpass)) - if hasattr(phenny.config, 'password'): + if hasattr(phenny.config, 'password'): phenny.msg('NickServ', 'IDENTIFY %s' % phenny.config.password) time.sleep(5) # Cf. http://swhack.com/logs/2005-12-05#T19-32-36 - for channel in phenny.channels: + for channel in phenny.channels: phenny.write(('JOIN', channel)) time.sleep(0.5) startup.rule = r'(.*)' startup.event = '251' startup.priority = 'low' -if __name__ == '__main__': - print __doc__.strip() +if __name__ == '__main__': + print(__doc__.strip()) diff --git a/tell.py b/tell.py index 13150b6..05e96f1 100755 --- a/tell.py +++ b/tell.py @@ -5,7 +5,6 @@ Copyright 2013, sfan5 """ import random -from thread import start_new_thread, allocate_lock import sqlite3 import time import hashlib @@ -118,4 +117,4 @@ db.commit() db.close() if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/title.py b/title.py index 37e0ebe..7c1eb4e 100755 --- a/title.py +++ b/title.py @@ -1,100 +1,46 @@ #!/usr/bin/env python """ title.py - Phenny URL Title Module -Copyright 2008, Sean B. Palmer, inamidst.com -Modified by sfan5, 2013 -Licensed under the Eiffel Forum License 2. - -http://inamidst.com/phenny/ +Copyright 2014, sfan5 """ -import re, urllib2, urlparse +import re +import web -r_title = re.compile(r'(?ims)]*>(.*?)') - -def f_title(phenny, input): - uri = input.group(2) - uri = (uri or '').encode('utf-8') - - if not uri and hasattr(phenny.bot, 'last_seen_uri'): - uri = phenny.bot.last_seen_uri - if not uri: - return phenny.reply('I need a URI to give the title of...') - - if not ':' in uri: - uri = 'http://' + uri +r_title = re.compile(r'(?ims)<\s*title[^>]*>(.*?)<\s*/\s*title\s*>') +def title(phenny, input): + uri = input.group(2).strip() + if uri: + pass + elif hasattr(phenny.bot, 'last_seen_uri'): + uri = phenny.bot.last_seen_uri + else: + return phenny.reply("Give me an URI..") + data, sc = web.get(uri, 4096) + if sc != 200: + return phenny.say("HTTP error %d" % sc) try: - redirects = 0 - while True: - headers = { - 'Accept': 'text/html', - 'User-Agent': 'Mozilla/5.0 (MinetestBot)' - } - req = urllib2.Request(uri, headers=headers) - u = urllib2.urlopen(req) - info = u.info() - u.close() + data = str(data, 'utf-8') + except UnicodeDecodeError: + return phenny.say("Doesn't seem to be HTML..") + m = re.search(r_title, data) + if not m: + return phenny.say("No title found.") + title = m.group(1).strip() + if len(title) > 75: + title = title[:75] + "[...]" + phenny.reply(title) - if not isinstance(info, list): - status = '200' - else: - status = str(info[1]) - info = info[0] - if status.startswith('3'): - uri = urlparse.urljoin(uri, info['Location']) - else: break - - redirects += 1 - if redirects >= 20: - return phenny.reply("Too many redirects") - - try: mtype = info['content-type'] - except: - return phenny.reply("Couldn't get the Content-Type, sorry") - if not (('/html' in mtype) or ('/xhtml' in mtype)): - return phenny.reply("Document isn't HTML") - - u = urllib2.urlopen(req) - bytes = u.read(262144) - u.close() - - except IOError: - return phenny.reply("Can't connect to %s" % uri) - - m = r_title.search(bytes) - if m: - title = m.group(1) - title = title.strip() - title = title.replace('\n', ' ') - title = title.replace('\r', ' ') - while ' ' in title: - title = title.replace(' ', ' ') - if len(title) > 100: - title = title[:100] + '[...]' - - if title: - try: title.decode('utf-8') - except: - try: title = title.decode('iso-8859-1').encode('utf-8') - except: title = title.decode('cp1252').encode('utf-8') - else: pass - else: title = '[The title is empty.]' - - title = title.replace('\n', '') - title = title.replace('\r', '') - return phenny.reply(title) - else: return phenny.reply('No title found') - -f_title.commands = ['title'] +title.commands = ['title'] def noteuri(phenny, input): - uri = input.group(1).encode('utf-8') + uri = input.group(1) phenny.bot.last_seen_uri = uri -noteuri.rule = r'.*(https?://[^<> "\x01]+).*' +noteuri.rule = r'(https?://[^<> "\x01]+)' noteuri.priority = 'low' noteuri.nohook = True if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip()) diff --git a/twitter.py b/twitter.py index fbe407b..05a32a0 100755 --- a/twitter.py +++ b/twitter.py @@ -21,7 +21,7 @@ r_whiteline = re.compile(r'(?ims)[ \t]+[\r\n]+') r_breaks = re.compile(r'(?ims)[\r\n]+') def entity(*args, **kargs): - return web.entity(*args, **kargs).encode('utf-8') + return web.entity(*args, **kargs) def decode(html): return web.r_entity.sub(entity, html) @@ -35,7 +35,8 @@ def expand(tweet): return r_anchor.sub(replacement, tweet) def read_tweet(url): - bytes = web.get(url) + bytes, sc = web.get(url) + bytes = str(bytes, 'utf-8') shim = '
' if shim in bytes: bytes = bytes.split(shim, 1).pop() @@ -58,11 +59,11 @@ def user_tweet(username): def id_tweet(tid): link = 'https://twitter.com/twitter/status/' + tid - data = web.head(link) - message, status = tuple(data) + headers, status = web.head(link) if status == 301: - url = message.get("Location") - if not url: return "Sorry, couldn't get a tweet from %s" % link + if not "Location" in headers: + return "Sorry, couldn't get a tweet from %s" % link + url = headers["Location"] username = url.split('/')[3] tweet = read_tweet(url) return format(tweet, username) @@ -74,9 +75,7 @@ def twitter(phenny, input): return phenny.reply("Give me a link, a @username, or a tweet id") arg = arg.strip() - if isinstance(arg, unicode): - arg = arg.encode('utf-8') - log.log("%s queried Twitter for '%s'" % (log.fmt_user(input), arg)) + log.log("event", "%s queried Twitter for '%s'" % (log.fmt_user(input), arg), phenny) if arg.isdigit(): phenny.say(id_tweet(arg)) elif r_username.match(arg): @@ -91,4 +90,4 @@ twitter.commands = ['tw', 'twitter'] twitter.thread = True if __name__ == '__main__': - print __doc__ + print(__doc__) diff --git a/wiki.py b/wiki.py index b0d9ced..3a135ac 100755 --- a/wiki.py +++ b/wiki.py @@ -1,166 +1,56 @@ #!/usr/bin/env python """ wiki.py - Phenny Wiki Module -Copyright 2008-9, Sean B. Palmer, inamidst.com -Modified by sfan5 2013 -Licensed under the Eiffel Forum License 2. - -http://inamidst.com/phenny/ +Copyright 2014, sfan5 """ -import re, urllib, gzip, StringIO +import re import web +import urllib.parse -wikiuri = 'http://wiki.minetest.net/%s' +wikiuri_g = 'http://wiki.minetest.net/%s?printable=yes' +wikiuri_r = 'http://wiki.minetest.net/%s' -r_tr = re.compile(r'(?ims)]*>.*?') -r_paragraph = re.compile(r'(?ims)]*>.*?

|]*>.*?') -r_tag = re.compile(r'<(?!!)[^>]+>') -r_whitespace = re.compile(r'[\t\r\n ]+') -r_redirect = re.compile( - r'(?ims)class=.redirectText.>\s*]+class=.mw-content-ltr.>') +r_paragraph = re.compile(r'(?ims)

(.+?)

') +r_sentenceend = re.compile(r'\.[^\.]') +transforms = [ + re.compile(r'(?i)]+>(.+?)'), + re.compile(r'(?i)(.+?)'), + re.compile(r'(?i)(.+?)'), +] -abbrs = ['etc', 'ca', 'cf', 'Co', 'Ltd', 'Inc', 'Mt', 'Mr', 'Mrs', - 'Dr', 'Ms', 'Rev', 'Fr', 'St', 'Sgt', 'pron', 'approx', 'lit', - 'syn', 'transl', 'sess', 'fl', 'Op', 'Dec', 'Brig', 'Gen'] \ - + list('ABCDEFGHIJKLMNOPQRSTUVWXYZ') \ - + list('abcdefghijklmnopqrstuvwxyz') -t_sentence = r'^.{5,}?(?') - s = s.replace('<', '<') - s = s.replace('&', '&') - s = s.replace(' ', ' ') - return s + log.log("event", "%s queried Wiki for '%s'" % (log.fmt_user(input), term), phenny) + term = web.urlencode(term) -def text(html): - html = r_tag.sub('', html) - html = r_whitespace.sub(' ', html) - return unescape(html).strip() + data, scode = web.get(wikiuri_g % term) + if scode == 404: + return phenny.say("No such page.") + data = str(data, "utf-8") -def wikipedia(term, language='en', last=False): - global wikiuri - if not '%' in term: - if isinstance(term, unicode): - t = term.encode('utf-8') - else: t = term - q = urllib.quote(t) - u = wikiuri % (q) - bytes = web.get(u) - else: bytes = web.get(wikiuri % (term)) + m = re.search(r_content, data) + if not m: + return phenny.say("Sorry, did not find anything.") + data = data[m.span()[1]:] - if bytes.startswith('\x1f\x8b\x08\x00\x00\x00\x00\x00'): - f = StringIO.StringIO(bytes) - f.seek(0) - gzip_file = gzip.GzipFile(fileobj=f) - bytes = gzip_file.read() - gzip_file.close() - f.close() + m = re.search(r_paragraph, data) + if not m: + return phenny.say("Sorry, did not find anything.") + data = m.group(1) + for transform in transforms: + data = re.sub(transform, '\g<1>', data) + m = re.search(r_sentenceend, data) + if m: + data = data[:m.span()[1]-1] + phenny.say('"%s" - %s ' % (web.decode(data), wikiuri_r % term)) - bytes = r_tr.sub('', bytes) - - if not last: - r = r_redirect.search(bytes[:4096]) - if r: - term = urllib.unquote(r.group(1)) - return wikipedia(term, language=language, last=True) - - paragraphs = r_paragraph.findall(bytes) - - if not paragraphs: - if not last: - term = search(term) - return wikipedia(term, language=language, last=True) - return None - - # Pre-process - paragraphs = [para for para in paragraphs - if (para and 'technical limitations' not in para - and 'window.showTocToggle' not in para - and 'Deletion_policy' not in para - and 'Template:AfD_footer' not in para - and not (para.startswith('

') and - para.endswith('

')) - and not 'disambiguation)"' in para) - and not '(images and media)' in para - and not 'This article contains a' in para - and not 'id="coordinates"' in para - and not 'class="thumb' in para] - # and not 'style="display:none"' in para] - - for i, para in enumerate(paragraphs): - para = para.replace('', '|') - para = para.replace('', '|') - paragraphs[i] = text(para).strip() - - # Post-process - paragraphs = [para for para in paragraphs if - (para and not (para.endswith(':') and len(para) < 150))] - - para = text(paragraphs[0]) - m = r_sentence.match(para) - - if not m: - if not last: - term = search(term) - return wikipedia(term, language=language, last=True) - return None - sentence = m.group(0) - - maxlength = 275 - if len(sentence) > maxlength: - sentence = sentence[:maxlength] - words = sentence[:-5].split(' ') - words.pop() - sentence = ' '.join(words) + ' [...]' - - if (('using the Article Wizard if you wish' in sentence) - or ('or add a request for it' in sentence) - or ('in existing articles' in sentence)): - if not last: - term = search(term) - return wikipedia(term, language=language, last=True) - return None - - sentence = '"' + sentence.replace('"', "'") + '"' - sentence = sentence.decode('utf-8').encode('utf-8') - wikiuri = wikiuri.decode('utf-8').encode('utf-8') - term = term.decode('utf-8').encode('utf-8') - return sentence + ' - ' + (wikiuri % (term)) - -def wik(phenny, input): - origterm = input.groups()[1] - if not origterm: - return phenny.say('Perhaps you meant "!wik Zen"?') - origterm = origterm.encode('utf-8') - - log.log("event", "%s queried Wiki for '%s'" % (log.fmt_user(input), origterm), phenny) - - term = urllib.unquote(origterm) - language = 'en' - if term.startswith(':') and (' ' in term): - a, b = term.split(' ', 1) - a = a.lstrip(':') - if a.isalpha(): - language, term = a, b - term = term[0].upper() + term[1:] - term = term.replace(' ', '_') - - try: result = wikipedia(term, language) - except IOError: - args = (language, wikiuri % (term)) - error = "Can't connect to wiki.minetest.com (%s)" % args - return phenny.say(error) - - if result is not None: - phenny.say(result) - else: phenny.say('Can\'t find anything in Wiki for "%s".' % origterm) - -wik.commands = ['wik', 'wiki'] -wik.priority = 'high' +wiki.commands = ['wik', 'wiki'] +wiki.priority = 'high' if __name__ == '__main__': - print __doc__.strip() + print(__doc__.strip())