Make rssnotify module suck significantly less
sfan5
parent
12d6c11e6b
commit
dfe8fdc6c2
226
rssnotify.py
226
rssnotify.py
|
|
@ -1,15 +1,16 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
"""
|
"""
|
||||||
rssnotify.py - Phenny RssNotify Module
|
rssnotify.py - Phenny RSSNotify Module
|
||||||
Copyright 2013, sfan5
|
Copyright 2016, sfan5
|
||||||
Licensed under GNU General Public License v2.0
|
Licensed under GNU General Public License v2.0
|
||||||
"""
|
"""
|
||||||
import time
|
import time
|
||||||
import re
|
import re
|
||||||
import web
|
import web
|
||||||
import os
|
import os
|
||||||
|
import threading
|
||||||
|
|
||||||
import feedparser # sudo pip install feedparser
|
import feedparser # sudo pip install feedparser
|
||||||
rssnotify = {}
|
|
||||||
|
|
||||||
def to_unix_time(tstr):
|
def to_unix_time(tstr):
|
||||||
if tstr.endswith("Z"):
|
if tstr.endswith("Z"):
|
||||||
|
|
@ -31,125 +32,120 @@ def to_unix_time(tstr):
|
||||||
ts += int(g[3]) * 60
|
ts += int(g[3]) * 60
|
||||||
return ts
|
return ts
|
||||||
|
|
||||||
def excepta(arr, exclude):
|
class RssNotify():
|
||||||
o = []
|
def __init__(self, config):
|
||||||
for el in arr:
|
self.config = config
|
||||||
if not el in exclude:
|
self.last_updated = {}
|
||||||
o.append(el)
|
self.last_check = 0
|
||||||
return o
|
self.firstrun = True
|
||||||
|
for i in range(len(self.config["feeds"])):
|
||||||
|
self.last_updated[i] = 0
|
||||||
|
def needs_check(self):
|
||||||
|
return time.time() > self.last_check + self.config["check_interval"]
|
||||||
|
def check(self, phenny):
|
||||||
|
start = self.last_check = time.time()
|
||||||
|
print("[RssNotify]: Checking RSS feeds...")
|
||||||
|
for fid, feedspec in enumerate(self.config["feeds"]):
|
||||||
|
feed = feedparser.parse(feedspec[0], agent="Mozilla/5.0 (compatible; MinetestBot)")
|
||||||
|
updated = 0
|
||||||
|
for entry in feed.entries:
|
||||||
|
if self.last_updated[fid] >= to_unix_time(entry.updated):
|
||||||
|
continue
|
||||||
|
if self.firstrun:
|
||||||
|
continue
|
||||||
|
message = self._format_msg(entry)
|
||||||
|
self._announce(phenny, message, feedspec[1])
|
||||||
|
if self.config["logfile"] is not None:
|
||||||
|
with open(self.config["logfile"], "a") as f:
|
||||||
|
message = self._format_msg(entry, log_format=True)
|
||||||
|
f.write(message)
|
||||||
|
f.write("\n")
|
||||||
|
updated += 1
|
||||||
|
self.last_updated[fid] = max((to_unix_time(e.updated) for e in feed.entries), default=0)
|
||||||
|
if updated > 0:
|
||||||
|
print("[RssNotify]: Found %d update(s) for '%s'" % (updated, feedspec[0]))
|
||||||
|
if self.firstrun:
|
||||||
|
self.firstrun = False
|
||||||
|
print("[RssNotify]: Checked %d RSS feeds in %0.3f seconds" % (len(self.config["feeds"]), time.time()-start))
|
||||||
|
def _shorten(self, link):
|
||||||
|
# We can utilitze git.io to shorten *.github.com links
|
||||||
|
l, code = web.post("https://git.io/create", {'url': link})
|
||||||
|
if code != 200:
|
||||||
|
return None
|
||||||
|
l = str(l, 'utf-8')
|
||||||
|
if ' ' in l: # spaces means there was an error :(
|
||||||
|
return None
|
||||||
|
return "https://git.io/" + l
|
||||||
|
def _format_msg(self, feed_entry, log_format=False):
|
||||||
|
if log_format:
|
||||||
|
f_cshort = "[color=#cc0000]%s[/color]"
|
||||||
|
f_clong = "[color=#cc0000]%s[/color] ([color=#cc0000]%s[/color])"
|
||||||
|
f_all = "[color=#3465a4][git][/color] %s -> [color=#73d216]%s[/color]: [b]%s[/b] [color=#a04265]%s[/color] %s ([color=#888a85]%s[/color])"
|
||||||
|
else:
|
||||||
|
f_cshort = "\x0304%s\x0f"
|
||||||
|
f_clong = "\x0304%s\x0f (\x0304%s\x0f)"
|
||||||
|
f_all = "\x0302[git]\x0f %s -> \x0303%s\x0f: \x02%s\x0f \x0313%s\x0f %s (\x0315%s\x0f)"
|
||||||
|
committer_realname = feed_entry.authors[0].name
|
||||||
|
if committer_realname == "":
|
||||||
|
try:
|
||||||
|
committer_realname = feed_entry.authors[0].email
|
||||||
|
except AttributeError:
|
||||||
|
committer_realname = ""
|
||||||
|
try:
|
||||||
|
committer = feed_entry.authors[0].href.replace('https://github.com/',"")
|
||||||
|
except AttributeError:
|
||||||
|
committer = committer_realname # This will only use the realname if the nickname couldn't be obtained
|
||||||
|
m = re.search(r'/([a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+)/commit/([a-f0-9]{7})', feed_entry.links[0].href)
|
||||||
|
repo_name = m.group(1) if m else "?"
|
||||||
|
commit_hash = m.group(2) if m else "???????"
|
||||||
|
commit_time = feed_entry.updated
|
||||||
|
commit_text = feed_entry.title
|
||||||
|
if self.config["show_link"]:
|
||||||
|
commit_link = feed_entry.link
|
||||||
|
if self.config["shorten_link"]:
|
||||||
|
commit_link = self._shorten(commit_link) or commit_link
|
||||||
|
else:
|
||||||
|
commit_link = ""
|
||||||
|
if committer_realname == "" or committer_realname.lower() == committer.lower():
|
||||||
|
committer_final = f_cshort % committer
|
||||||
|
else:
|
||||||
|
committer_final = f_clong % (committer, committer_realname)
|
||||||
|
return f_all % (committer_final, repo_name, commit_text, commit_hash, commit_link, commit_time)
|
||||||
|
def _announce(self, phenny, message, chans):
|
||||||
|
if chans == "*":
|
||||||
|
chans = phenny.bot.channels
|
||||||
|
assert(type(chans) == list)
|
||||||
|
for ch in chans:
|
||||||
|
phenny.write(['PRIVMSG', ch], message)
|
||||||
|
|
||||||
rssnotify["last_updated_feeds"] = {}
|
rssn = RssNotify({
|
||||||
rssnotify["logfilepath"] = os.getcwd() + "/rssnotify.log"
|
"check_interval": 120,
|
||||||
rssnotify["dont_print_first_message"] = True # prevents spam when restarting the bot/reloading the module
|
"show_link": True,
|
||||||
rssnotify["update_cooldown"] = 60 # in seconds
|
"shorten_link": True,
|
||||||
rssnotify["show_commit_link"] = True
|
"logfile": os.getcwd() + "/rssnotify.log",
|
||||||
rssnotify["use_git.io"] = True
|
"feeds": [
|
||||||
rssnotify["last_update"] = time.time() - rssnotify["update_cooldown"]
|
('https://github.com/minetest/minetest/commits/master.atom', "*"),
|
||||||
|
('https://github.com/minetest/minetest_game/commits/master.atom', "*"),
|
||||||
def rsscheck(phenny, input):
|
('https://github.com/minetest/minetestmapper/commits/master.atom', "*"),
|
||||||
t = time.time()
|
('https://github.com/minetest/master-server/commits/master.atom', "*"),
|
||||||
if rssnotify["last_update"] > t-rssnotify["update_cooldown"]:
|
('https://github.com/Uberi/Minetest-WorldEdit/commits/master.atom', "*"),
|
||||||
return
|
('https://github.com/Jeija/minetest-mod-mesecons/commits/master.atom', "*"),
|
||||||
rssnotify["last_update"] = t
|
|
||||||
print("[RssNotify]: Checking RSS Feeds...")
|
|
||||||
start = time.time()
|
|
||||||
allchans = excepta(phenny.bot.channels, ['##minebest'])
|
|
||||||
feeds = [
|
|
||||||
('https://github.com/minetest/minetest/commits/master.atom', allchans),
|
|
||||||
('https://github.com/minetest/minetest_game/commits/master.atom', allchans),
|
|
||||||
('https://github.com/minetest/minetestmapper/commits/master.atom', allchans),
|
|
||||||
('https://github.com/minetest/master-server/commits/master.atom', allchans),
|
|
||||||
('https://github.com/Uberi/Minetest-WorldEdit/commits/master.atom', allchans),
|
|
||||||
('https://github.com/Jeija/minetest-mod-mesecons/commits/master.atom', allchans),
|
|
||||||
('https://github.com/sfan5/phenny/commits/master.atom', ['##minetestbot']),
|
('https://github.com/sfan5/phenny/commits/master.atom', ['##minetestbot']),
|
||||||
('https://github.com/sfan5/minetestbot-modules/commits/master.atom', ['##minetestbot']),
|
('https://github.com/sfan5/minetestbot-modules/commits/master.atom', ['##minetestbot']),
|
||||||
('https://github.com/BlockMen/minetest_next/commits/master.atom', allchans),
|
('https://github.com/BlockMen/minetest_next/commits/master.atom', "*"),
|
||||||
]
|
],
|
||||||
for v in range(0, len(feeds)):
|
})
|
||||||
url = feeds[v][0]
|
|
||||||
feednum = v
|
|
||||||
options = {
|
|
||||||
'agent': 'Mozilla/5.0 (MinetestBot)',
|
|
||||||
'referrer': 'http://minetest.net'
|
|
||||||
}
|
|
||||||
feed = feedparser.parse(url, **options)
|
|
||||||
updcnt = 0
|
|
||||||
for feed_entry in feed.entries:
|
|
||||||
if not feednum in rssnotify["last_updated_feeds"].keys():
|
|
||||||
rssnotify["last_updated_feeds"][feednum] = -1
|
|
||||||
if rssnotify["last_updated_feeds"][feednum] < to_unix_time(feed_entry.updated):
|
|
||||||
if rssnotify["dont_print_first_message"]:
|
|
||||||
continue
|
|
||||||
commiter_realname = feed_entry.authors[0].name
|
|
||||||
if commiter_realname == "":
|
|
||||||
try:
|
|
||||||
commiter_realname = feed_entry.authors[0].email
|
|
||||||
except AttributeError:
|
|
||||||
commiter_realname = "Unknown"
|
|
||||||
try:
|
|
||||||
commiter = feed_entry.authors[0].href.replace('https://github.com/',"")
|
|
||||||
except AttributeError:
|
|
||||||
commiter = commiter_realname # This will only use the realname if the nickname couldn't be obtained
|
|
||||||
reponame = url.replace("https://github.com/","").replace("/commits/master.atom","")
|
|
||||||
commit_hash = re.search(r"/([a-f0-9]{7})[a-f0-9]*$", feed_entry.links[0].href)
|
|
||||||
if commit_hash:
|
|
||||||
commit_hash = commit_hash.group(1)
|
|
||||||
else:
|
|
||||||
commit_hash = "?" * 7
|
|
||||||
commit_time = feed_entry.updated
|
|
||||||
updcnt += 1
|
|
||||||
if rssnotify["show_commit_link"]:
|
|
||||||
if rssnotify["use_git.io"]:
|
|
||||||
# Side note: git.io only works with *.github.com links
|
|
||||||
l, code = web.post("https://git.io/create", {'url': feed_entry.link})
|
|
||||||
if code == 200:
|
|
||||||
l = str(l, 'utf-8')
|
|
||||||
if not ' ' in l: # If there are spaces it's an error
|
|
||||||
commit_link = "https://git.io/" + l
|
|
||||||
else:
|
|
||||||
commit_link = feed_entry.link
|
|
||||||
else:
|
|
||||||
commit_link = feed_entry.link
|
|
||||||
else:
|
|
||||||
commit_link = feed_entry.link
|
|
||||||
else:
|
|
||||||
commit_link = ""
|
|
||||||
|
|
||||||
chans = []
|
def rsscheck(phenny, input):
|
||||||
if feeds[v][1] == '*':
|
if not rssn.needs_check():
|
||||||
chans = phenny.bot.channels
|
return
|
||||||
elif type(feeds[v][1]) == type([]):
|
t = threading.Thread(target=rssn.check, args=(phenny, ))
|
||||||
chans = feeds[v][1]
|
t.start()
|
||||||
else:
|
|
||||||
print("[RssNotify]: Something went wrong!")
|
|
||||||
if rssnotify["logfilepath"] != "":
|
|
||||||
lf = open(rssnotify["logfilepath"], "a")
|
|
||||||
if commiter.lower() != commiter_realname.lower():
|
|
||||||
lf.write("[color=#3465a4][git][/color] [color=#cc0000]%s[/color] ([color=#cc0000]%s[/color]) -> [color=#73d216]%s[/color]: [b]%s[/b] [color=#a04265]%s[/color] %s ([color=#888a85]%s[/color])\n" % (commiter, commiter_realname, reponame, feed_entry.title, commit_hash, commit_link, commit_time))
|
|
||||||
else:
|
|
||||||
lf.write("[color=#3465a4][git][/color] [color=#cc0000]%s[/color] -> [color=#73d216]%s[/color]: [b]%s[/b] [color=#a04265]%s[/color] %s ([color=#888a85]%s[/color])\n" % (commiter, reponame, feed_entry.title, commit_hash, commit_link, commit_time))
|
|
||||||
lf.close()
|
|
||||||
for ch in chans:
|
|
||||||
if commiter.lower() != commiter_realname.lower():
|
|
||||||
phenny.write(['PRIVMSG', ch], "\x0302[git]\x0f \x0304%s\x0f (\x0304%s\x0f) -> \x0303%s\x0f: \x02%s\x0f \x0313%s\x0f %s (\x0315%s\x0f)" % (commiter, commiter_realname, reponame, feed_entry.title, commit_hash, commit_link, commit_time))
|
|
||||||
else:
|
|
||||||
phenny.write(['PRIVMSG', ch], "\x0302[git]\x0f \x0304%s\x0f -> \x0303%s\x0f: \x02%s\x0f \x0313%s\x0f %s (\x0315%s\x0f)" % (commiter, reponame, feed_entry.title, commit_hash, commit_link, commit_time))
|
|
||||||
if len(feed.entries) > 0:
|
|
||||||
m = -1
|
|
||||||
for i in range(0, len(feed.entries)):
|
|
||||||
if to_unix_time(feed.entries[i].updated) > m:
|
|
||||||
m = to_unix_time(feed.entries[i].updated)
|
|
||||||
rssnotify["last_updated_feeds"][feednum] = m
|
|
||||||
if updcnt > 0:
|
|
||||||
print("[RssNotify]: Found %i RSS Update(s) for URL '%s'" % (updcnt, url))
|
|
||||||
end = time.time()
|
|
||||||
if rssnotify["dont_print_first_message"]:
|
|
||||||
rssnotify["dont_print_first_message"] = False
|
|
||||||
print("[RssNotify]: Checked " + str(len(feeds)) + " RSS Feeds in %0.3f seconds" % (end-start))
|
|
||||||
|
|
||||||
rsscheck.priority = 'medium'
|
rsscheck.priority = 'low'
|
||||||
rsscheck.rule = r'.*'
|
rsscheck.rule = r'.*'
|
||||||
rsscheck.event = '*'
|
rsscheck.event = '*'
|
||||||
|
rsscheck.thread = False
|
||||||
rsscheck.nohook = True
|
rsscheck.nohook = True
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue