catdog2/urlbot-native
1
0
Fork 0
Code Issues Pull Requests Releases Activity

all death to the tab character

Browse Source
This commit is contained in:
Thorsten
2015-11-30 19:17:40 +01:00
parent d94d62335f
commit a7b53d855a
7 changed files with 1328 additions and 1329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

266
common.py
View File

@@ -26,195 +26,195 @@ USER_AGENT = '''Mozilla/5.0 (X11; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/
basedir = '.' basedir = '.'
if 2 == len(sys.argv): if 2 == len(sys.argv):
basedir = sys.argv[1] basedir = sys.argv[1]
def conf_save(obj): def conf_save(obj):
with open(conf('persistent_storage'), 'wb') as fd: with open(conf('persistent_storage'), 'wb') as fd:
return pickle.dump(obj, fd) return pickle.dump(obj, fd)
def conf_load(): def conf_load():
path = conf('persistent_storage') path = conf('persistent_storage')
if os.path.isfile(path): if os.path.isfile(path):
with open(path, 'rb') as fd: with open(path, 'rb') as fd:
fd.seek(0) fd.seek(0)
return pickle.load(fd) return pickle.load(fd)
else: else:
return {} return {}
def conf_set(key, value): def conf_set(key, value):
blob = conf_load() blob = conf_load()
blob[key] = value blob[key] = value
conf_save(blob) conf_save(blob)
def conf_get(key, default=None): def conf_get(key, default=None):
blob = conf_load() blob = conf_load()
return blob.get(key, default) return blob.get(key, default)
Bucket = namedtuple("BucketConfig", ["history", "period", "max_hist_len"]) Bucket = namedtuple("BucketConfig", ["history", "period", "max_hist_len"])
buckets = { buckets = {
# everything else # everything else
RATE_GLOBAL: Bucket(history=[], period=60, max_hist_len=10), RATE_GLOBAL: Bucket(history=[], period=60, max_hist_len=10),
# bot writes with no visible stimuli # bot writes with no visible stimuli
RATE_NO_SILENCE: Bucket(history=[], period=10, max_hist_len=5), RATE_NO_SILENCE: Bucket(history=[], period=10, max_hist_len=5),
# interactive stuff like ping # interactive stuff like ping
RATE_INTERACTIVE: Bucket(history=[], period=30, max_hist_len=5), RATE_INTERACTIVE: Bucket(history=[], period=30, max_hist_len=5),
# chitty-chat, master volume control # chitty-chat, master volume control
RATE_CHAT: Bucket(history=[], period=10, max_hist_len=5), RATE_CHAT: Bucket(history=[], period=10, max_hist_len=5),
# reacting on URLs # reacting on URLs
RATE_URL: Bucket(history=[], period=10, max_hist_len=5), RATE_URL: Bucket(history=[], period=10, max_hist_len=5),
# triggering events # triggering events
RATE_EVENT: Bucket(history=[], period=60, max_hist_len=10), RATE_EVENT: Bucket(history=[], period=60, max_hist_len=10),
# bot blames people, produces cake and entertains # bot blames people, produces cake and entertains
RATE_FUN: Bucket(history=[], period=180, max_hist_len=5), RATE_FUN: Bucket(history=[], period=180, max_hist_len=5),
} }
rate_limit_classes = buckets.keys() rate_limit_classes = buckets.keys()
def rate_limit(rate_class=RATE_GLOBAL): def rate_limit(rate_class=RATE_GLOBAL):
""" """
Remember N timestamps, Remember N timestamps,
if N[0] newer than now()-T then do not output, do not append. if N[0] newer than now()-T then do not output, do not append.
else pop(0); append() else pop(0); append()
:param rate_class: the type of message to verify :param rate_class: the type of message to verify
:return: False if blocked, True if allowed :return: False if blocked, True if allowed
""" """
if rate_class not in rate_limit_classes: if rate_class not in rate_limit_classes:
return all(rate_limit(c) for c in rate_limit_classes if c & rate_class) return all(rate_limit(c) for c in rate_limit_classes if c & rate_class)
now = time.time() now = time.time()
bucket = buckets[rate_class] bucket = buckets[rate_class]
logging.getLogger(__name__).debug("[ratelimit][bucket=%x][time=%s]%s" % (rate_class, now, bucket.history)) logging.getLogger(__name__).debug("[ratelimit][bucket=%x][time=%s]%s" % (rate_class, now, bucket.history))
if len(bucket.history) >= bucket.max_hist_len and bucket.history[0] > (now - bucket.period): if len(bucket.history) >= bucket.max_hist_len and bucket.history[0] > (now - bucket.period):
# print("blocked") # print("blocked")
return False return False
else: else:
if bucket.history and len(bucket.history) > bucket.max_hist_len: if bucket.history and len(bucket.history) > bucket.max_hist_len:
bucket.history.pop(0) bucket.history.pop(0)
bucket.history.append(now) bucket.history.append(now)
return True return True
def rate_limited(max_per_second): def rate_limited(max_per_second):
""" """
very simple flow control context manager very simple flow control context manager
:param max_per_second: how many events per second may be executed - more are delayed :param max_per_second: how many events per second may be executed - more are delayed
:return: :return:
""" """
min_interval = 1.0 / float(max_per_second) min_interval = 1.0 / float(max_per_second)
def decorate(func): def decorate(func):
lasttimecalled = [0.0] lasttimecalled = [0.0]
def ratelimitedfunction(*args, **kargs): def ratelimitedfunction(*args, **kargs):
elapsed = time.clock() - lasttimecalled[0] elapsed = time.clock() - lasttimecalled[0]
lefttowait = min_interval - elapsed lefttowait = min_interval - elapsed
if lefttowait > 0: if lefttowait > 0:
time.sleep(lefttowait) time.sleep(lefttowait)
ret = func(*args, **kargs) ret = func(*args, **kargs)
lasttimecalled[0] = time.clock() lasttimecalled[0] = time.clock()
return ret return ret
return ratelimitedfunction return ratelimitedfunction
return decorate return decorate
def get_version_git(): def get_version_git():
import subprocess import subprocess
cmd = ['git', 'log', '--oneline', '--abbrev-commit'] cmd = ['git', 'log', '--oneline', '--abbrev-commit']
try: try:
p = subprocess.Popen(cmd, bufsize=1, stdout=subprocess.PIPE) p = subprocess.Popen(cmd, bufsize=1, stdout=subprocess.PIPE)
first_line = p.stdout.readline() first_line = p.stdout.readline()
line_count = len(p.stdout.readlines()) + 1 line_count = len(p.stdout.readlines()) + 1
if 0 == p.wait(): if 0 == p.wait():
# skip this 1st, 2nd, 3rd stuff and use always [0-9]th # skip this 1st, 2nd, 3rd stuff and use always [0-9]th
return "version (Git, %dth rev) '%s'" % ( return "version (Git, %dth rev) '%s'" % (
line_count, str(first_line.strip(), encoding='utf8') line_count, str(first_line.strip(), encoding='utf8')
) )
else: else:
return "(unknown version)" return "(unknown version)"
except: except:
return "cannot determine version" return "cannot determine version"
VERSION = get_version_git() VERSION = get_version_git()
def fetch_page(url): def fetch_page(url):
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
log.info('fetching page ' + url) log.info('fetching page ' + url)
try: try:
request = urllib.request.Request(url) request = urllib.request.Request(url)
request.add_header('User-Agent', USER_AGENT) request.add_header('User-Agent', USER_AGENT)
response = urllib.request.urlopen(request) response = urllib.request.urlopen(request)
html_text = response.read(BUFSIZ) # ignore more than BUFSIZ html_text = response.read(BUFSIZ) # ignore more than BUFSIZ
response.close() response.close()
return 0, html_text, response.headers return 0, html_text, response.headers
except Exception as e: except Exception as e:
log.warn('failed: %s' % e) log.warn('failed: %s' % e)
return 1, str(e), 'dummy' return 1, str(e), 'dummy'
def extract_title(url): def extract_title(url):
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
global parser global parser
if 'repo/urlbot-native.git' in url: if 'repo/urlbot-native.git' in url:
log.info('repo URL found: ' + url) log.info('repo URL found: ' + url)
return 3, 'wee, that looks like my home repo!' return 3, 'wee, that looks like my home repo!'
log.info('extracting title from ' + url) log.info('extracting title from ' + url)
(code, html_text, headers) = fetch_page(url) (code, html_text, headers) = fetch_page(url)
if 1 == code: if 1 == code:
return 3, 'failed: %s for %s' % (html_text, url) return 3, 'failed: %s for %s' % (html_text, url)
if not html_text: if not html_text:
return -1, 'error' return -1, 'error'
charset = '' charset = ''
if 'content-type' in headers: if 'content-type' in headers:
log.debug('content-type: ' + headers['content-type']) log.debug('content-type: ' + headers['content-type'])
if 'text/' != headers['content-type'][:len('text/')]: if 'text/' != headers['content-type'][:len('text/')]:
return 1, headers['content-type'] return 1, headers['content-type']
charset = re.sub( charset = re.sub(
r'.*charset=(?P<charset>\S+).*', r'.*charset=(?P<charset>\S+).*',
r'\g<charset>', headers['content-type'], re.IGNORECASE r'\g<charset>', headers['content-type'], re.IGNORECASE
) )
if '' != charset: if '' != charset:
try: try:
html_text = html_text.decode(charset) html_text = html_text.decode(charset)
except LookupError: except LookupError:
log.warn("invalid charset in '%s': '%s'" % (headers['content-type'], charset)) log.warn("invalid charset in '%s': '%s'" % (headers['content-type'], charset))
if str != type(html_text): if str != type(html_text):
html_text = str(html_text) html_text = str(html_text)
result = re.match(r'.*?<title.*?>(.*?)</title>.*?', html_text, re.S | re.M | re.IGNORECASE) result = re.match(r'.*?<title.*?>(.*?)</title>.*?', html_text, re.S | re.M | re.IGNORECASE)
if result: if result:
match = result.groups()[0] match = result.groups()[0]
parser = html.parser.HTMLParser() parser = html.parser.HTMLParser()
try: try:
expanded_html = parser.unescape(match) expanded_html = parser.unescape(match)
except UnicodeDecodeError as e: # idk why this can happen, but it does except UnicodeDecodeError as e: # idk why this can happen, but it does
log.warn('parser.unescape() expoded here: ' + str(e)) log.warn('parser.unescape() expoded here: ' + str(e))
expanded_html = match expanded_html = match
return 0, expanded_html return 0, expanded_html
else: else:
return 2, 'no title' return 2, 'no title'

170
idlebot.py
View File

@@ -6,18 +6,18 @@ import sys
from common import VERSION, EVENTLOOP_DELAY, conf_load from common import VERSION, EVENTLOOP_DELAY, conf_load
try: try:
from local_config import conf, set_conf from local_config import conf, set_conf
except ImportError: except ImportError:
sys.stderr.write(''' sys.stderr.write('''
%s: E: local_config.py isn't tracked because of included secrets and %s: E: local_config.py isn't tracked because of included secrets and
%s site specific configurations. Rename local_config.py.skel and %s site specific configurations. Rename local_config.py.skel and
%s adjust to you needs. %s adjust to you needs.
'''[1:] % ( '''[1:] % (
sys.argv[0], sys.argv[0],
' ' * len(sys.argv[0]), ' ' * len(sys.argv[0]),
' ' * len(sys.argv[0]) ' ' * len(sys.argv[0])
)) ))
sys.exit(1) sys.exit(1)
from sleekxmpp import ClientXMPP from sleekxmpp import ClientXMPP
@@ -25,98 +25,98 @@ got_hangup = False
class IdleBot(ClientXMPP): class IdleBot(ClientXMPP):
def __init__(self, jid, password, rooms, nick): def __init__(self, jid, password, rooms, nick):
ClientXMPP.__init__(self, jid, password) ClientXMPP.__init__(self, jid, password)
self.rooms = rooms self.rooms = rooms
self.nick = nick self.nick = nick
self.add_event_handler('session_start', self.session_start) self.add_event_handler('session_start', self.session_start)
self.add_event_handler('groupchat_message', self.muc_message) self.add_event_handler('groupchat_message', self.muc_message)
self.priority = 0 self.priority = 0
self.status = None self.status = None
self.show = None self.show = None
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
def session_start(self, _): def session_start(self, _):
self.get_roster() self.get_roster()
self.send_presence(ppriority=self.priority, pstatus=self.status, pshow=self.show) self.send_presence(ppriority=self.priority, pstatus=self.status, pshow=self.show)
for room in self.rooms: for room in self.rooms:
self.logger.info('%s: joining' % room) self.logger.info('%s: joining' % room)
ret = self.plugin['xep_0045'].joinMUC( ret = self.plugin['xep_0045'].joinMUC(
room, room,
self.nick, self.nick,
wait=True wait=True
) )
self.logger.info('%s: joined with code %s' % (room, ret)) self.logger.info('%s: joined with code %s' % (room, ret))
def muc_message(self, msg_obj): def muc_message(self, msg_obj):
""" """
Handle muc messages, return if irrelevant content or die by hangup. Handle muc messages, return if irrelevant content or die by hangup.
:param msg_obj: :param msg_obj:
:return: :return:
""" """
# don't talk to yourself # don't talk to yourself
if msg_obj['mucnick'] == self.nick or 'groupchat' != msg_obj['type']: if msg_obj['mucnick'] == self.nick or 'groupchat' != msg_obj['type']:
return False return False
elif msg_obj['body'].startswith(conf('bot_user')) and 'hangup' in msg_obj['body']: elif msg_obj['body'].startswith(conf('bot_user')) and 'hangup' in msg_obj['body']:
self.logger.warn("got 'hangup' from '%s': '%s'" % ( self.logger.warn("got 'hangup' from '%s': '%s'" % (
msg_obj['mucnick'], msg_obj['body'] msg_obj['mucnick'], msg_obj['body']
)) ))
global got_hangup global got_hangup
got_hangup = True got_hangup = True
return False return False
elif msg_obj['mucnick'] in conf_load().get("other_bots", ()): elif msg_obj['mucnick'] in conf_load().get("other_bots", ()):
# not talking to the other bot. # not talking to the other bot.
return False return False
else: else:
return True return True
def start(botclass, active=False): def start(botclass, active=False):
logging.basicConfig( logging.basicConfig(
level=conf('loglevel', logging.INFO), level=conf('loglevel', logging.INFO),
format=sys.argv[0] + ' %(asctime)s %(levelname).1s %(funcName)-15s %(message)s' format=sys.argv[0] + ' %(asctime)s %(levelname).1s %(funcName)-15s %(message)s'
) )
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logger.info(VERSION) logger.info(VERSION)
jid = conf('jid') jid = conf('jid')
if '/' not in jid: if '/' not in jid:
jid = '%s/%s' % (jid, botclass.__name__) jid = '%s/%s' % (jid, botclass.__name__)
bot = botclass( bot = botclass(
jid=jid, jid=jid,
password=conf('password'), password=conf('password'),
rooms=conf('rooms'), rooms=conf('rooms'),
nick=conf('bot_user') nick=conf('bot_user')
) )
import plugins import plugins
if active: if active:
plugins.register_all() plugins.register_all()
if plugins.plugin_enabled_get(plugins.command_dsa_watcher): if plugins.plugin_enabled_get(plugins.command_dsa_watcher):
# first result is lost. # first result is lost.
plugins.command_dsa_watcher(['dsa-watcher', 'crawl']) plugins.command_dsa_watcher(['dsa-watcher', 'crawl'])
bot.connect() bot.connect()
bot.register_plugin('xep_0045') bot.register_plugin('xep_0045')
bot.process() bot.process()
global got_hangup global got_hangup
while 1: while 1:
try: try:
# print("hangup: %s" % got_hangup) # print("hangup: %s" % got_hangup)
if got_hangup or not plugins.event_trigger(): if got_hangup or not plugins.event_trigger():
bot.disconnect() bot.disconnect()
sys.exit(1) sys.exit(1)
time.sleep(EVENTLOOP_DELAY) time.sleep(EVENTLOOP_DELAY)
except KeyboardInterrupt: except KeyboardInterrupt:
print('') print('')
exit(130) exit(130)
if '__main__' == __name__: if '__main__' == __name__:
start(IdleBot) start(IdleBot)

82
local_config.py.skel
View File

@@ -3,63 +3,63 @@
import time import time
if '__main__' == __name__: if '__main__' == __name__:
print('''this is a config file, which is not meant to be executed''') print('''this is a config file, which is not meant to be executed''')
exit(-1) exit(-1)
config = { config = {
'jid': 'FIXME', 'jid': 'FIXME',
'password': 'FIXME', 'password': 'FIXME',
'rooms': ['FIXME'], 'rooms': ['FIXME'],
'src-url': 'http://aero2k.de/t/repos/urlbot-native.git', 'src-url': 'http://aero2k.de/t/repos/urlbot-native.git',
'bot_user': 'native-urlbot', 'bot_user': 'native-urlbot',
'bot_owner': 'FIXME', 'bot_owner': 'FIXME',
'hist_max_count': 5, 'hist_max_count': 5,
'hist_max_time': 10 * 60, 'hist_max_time': 10 * 60,
'uptime': -time.time(), 'uptime': -time.time(),
'request_counter': 0, 'request_counter': 0,
'persistent_storage': 'urlbot.persistent', 'persistent_storage': 'urlbot.persistent',
'persistent_locked': False, 'persistent_locked': False,
'url_blacklist': [ 'url_blacklist': [
r'^.*heise\.de/.*-[0-9]+\.html$', r'^.*heise\.de/.*-[0-9]+\.html$',
r'^.*wikipedia\.org/wiki/.*$', r'^.*wikipedia\.org/wiki/.*$',
r'^.*blog\.fefe\.de/\?ts=[0-9a-f]+$', r'^.*blog\.fefe\.de/\?ts=[0-9a-f]+$',
r'^.*ibash\.de/zitat.*$', r'^.*ibash\.de/zitat.*$',
r'^.*golem\.de/news/.*$' r'^.*golem\.de/news/.*$'
r'^.*paste\.debian\.net/((hidden|plainh?)/)?[0-9a-f]+/?$', r'^.*paste\.debian\.net/((hidden|plainh?)/)?[0-9a-f]+/?$',
r'^.*example\.(org|net|com).*$', r'^.*example\.(org|net|com).*$',
r'^.*sprunge\.us/.*$', r'^.*sprunge\.us/.*$',
r'^.*ftp\...\.debian\.org.*$' r'^.*ftp\...\.debian\.org.*$'
], ],
# the "dice" feature will use more efficient random data (0) for given users # the "dice" feature will use more efficient random data (0) for given users
'enhanced-random-user': ('FIXME', 'FIXME'), 'enhanced-random-user': ('FIXME', 'FIXME'),
# the "moin" feature will be "disabled" for given users # the "moin" feature will be "disabled" for given users
'moin-modified-user': (), 'moin-modified-user': (),
'moin-disabled-user': (), 'moin-disabled-user': (),
'tea_steep_time': (3*60 + 40), 'tea_steep_time': (3 * 60 + 40),
'image_preview': True, 'image_preview': True,
'dsa_watcher_interval': 15 * 60 'dsa_watcher_interval': 15 * 60
} }
def conf(val): def conf(val):
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
if val in list(config.keys()): if val in list(config.keys()):
return config[val] return config[val]
logger.warn('conf(): unknown key ' + str(val)) logger.warn('conf(): unknown key ' + str(val))
return None return None
def set_conf(key, val): def set_conf(key, val):
config[key] = val config[key] = val
return None return None

1602
plugins.py
View File

File diff suppressed because it is too large Load Diff

44
string_constants.py
View File

@@ -470,31 +470,31 @@ DNS server drank too much and had a hiccup
'''.split('\n')[1:-1] '''.split('\n')[1:-1]
moin_strings_hi = [ moin_strings_hi = [
'Hi', 'Hi',
'Guten Morgen', 'Morgen', 'Guten Morgen', 'Morgen',
'Moin', 'Moin',
'Tag', 'Tach', 'Tag', 'Tach',
'NAbend', 'Abend', 'NAbend', 'Abend',
'Hallo', 'Hello' 'Hallo', 'Hello'
] ]
moin_strings_bye = [ moin_strings_bye = [
'Nacht', 'gN8', 'N8', 'Nacht', 'gN8', 'N8',
'bye', 'bye',
] ]
cakes = [ cakes = [
"No cake for you!", "No cake for you!",
("The Enrichment Center is required to remind you " ("The Enrichment Center is required to remind you "
"that you will be baked, and then there will be cake."), "that you will be baked, and then there will be cake."),
"The cake is a lie!", "The cake is a lie!",
("This is your fault. I'm going to kill you. " ("This is your fault. I'm going to kill you. "
"And all the cake is gone. You don't even care, do you?"), "And all the cake is gone. You don't even care, do you?"),
"Quit now and cake will be served immediately.", "Quit now and cake will be served immediately.",
("Enrichment Center regulations require both hands to be " ("Enrichment Center regulations require both hands to be "
"empty before any cake..."), "empty before any cake..."),
("Uh oh. Somebody cut the cake. I told them to wait for " ("Uh oh. Somebody cut the cake. I told them to wait for "
"you, but they did it anyway. There is still some left, " "you, but they did it anyway. There is still some left, "
"though, if you hurry back."), "though, if you hurry back."),
"I'm going to kill you, and all the cake is gone.", "I'm going to kill you, and all the cake is gone.",
"Who's gonna make the cake when I'm gone? You?" "Who's gonna make the cake when I'm gone? You?"
] ]

70
test_urlbot.py
View File

@@ -3,19 +3,18 @@ To be executed with nose
""" """
import unittest import unittest
import time import time
from common import buckets, rate_limit, RATE_GLOBAL from common import buckets, rate_limit, RATE_GLOBAL
class TestEventlooper(unittest.TestCase): class TestEventlooper(unittest.TestCase):
def test_broken_url(self): def test_broken_url(self):
""" """
Test that broken socket calls are not breaking Test that broken socket calls are not breaking
""" """
from common import fetch_page from common import fetch_page
broken_url = 'http://foo' broken_url = 'http://foo'
result = fetch_page(url=broken_url) result = fetch_page(url=broken_url)
self.assertEqual(result, (None, None)) self.assertEqual(result, (None, None))
from collections import namedtuple from collections import namedtuple
@@ -24,34 +23,33 @@ Bucket = namedtuple("BucketConfig", ["history", "period", "max_hist_len"])
class TestRateLimiting(unittest.TestCase): class TestRateLimiting(unittest.TestCase):
def setUp(self):
# just for assertions
self.called = {
RATE_GLOBAL: [],
}
def setUp(self): def say(self, msg, rate_class=RATE_GLOBAL):
# just for assertions if rate_limit(rate_class):
self.called = { self.called[rate_class].append(msg)
RATE_GLOBAL: [], # print(msg)
} time.sleep(0.1)
def say(self, msg, rate_class=RATE_GLOBAL): def test_simple_burst(self):
if rate_limit(rate_class): messages = ["x_%d" % i for i in range(1, 9)]
self.called[rate_class].append(msg) for m in messages:
# print(msg) self.say(msg=m)
time.sleep(0.1) self.assertEqual(messages[:buckets[RATE_GLOBAL].max_hist_len], self.called[RATE_GLOBAL])
def test_simple_burst(self): def test_msg_two_bursts(self):
messages = ["x_%d" % i for i in range(1, 9)] # custom bucket, just for testing
for m in messages: buckets[0x42] = Bucket(history=[], period=1, max_hist_len=5)
self.say(msg=m) self.called[0x42] = []
self.assertEqual(messages[:buckets[RATE_GLOBAL].max_hist_len], self.called[RATE_GLOBAL])
def test_msg_two_bursts(self): bucket = buckets[0x42]
# custom bucket, just for testing messages = ["x_%d" % i for i in range(0, 15)]
buckets[0x42] = Bucket(history=[], period=1, max_hist_len=5) for i, m in enumerate(messages):
self.called[0x42] = [] if i % bucket.max_hist_len == 0:
time.sleep(bucket.period)
bucket = buckets[0x42] self.say(msg=m, rate_class=0x42)
messages = ["x_%d" % i for i in range(0, 15)] self.assertEqual(messages, self.called[0x42])
for i, m in enumerate(messages):
if i % bucket.max_hist_len == 0:
time.sleep(bucket.period)
self.say(msg=m, rate_class=0x42)
self.assertEqual(messages, self.called[0x42])

423
urlbot.py
View File

@@ -6,276 +6,277 @@ import re
import sys import sys
from common import ( from common import (
conf_load, conf_save, conf_load, conf_save,
extract_title, extract_title,
rate_limit_classes, rate_limit_classes,
RATE_GLOBAL, RATE_GLOBAL,
RATE_CHAT, RATE_CHAT,
RATE_NO_SILENCE, RATE_NO_SILENCE,
RATE_EVENT, RATE_EVENT,
# rate_limited, # rate_limited,
rate_limit, rate_limit,
RATE_URL, conf_set) RATE_URL, conf_set)
from idlebot import IdleBot, start from idlebot import IdleBot, start
from plugins import ( from plugins import (
plugins as plugin_storage, plugins as plugin_storage,
ptypes_COMMAND, ptypes_COMMAND,
plugin_enabled_get, plugin_enabled_get,
ptypes_PARSE, ptypes_PARSE,
register_event, register_event,
else_command else_command
) )
try: try:
from local_config import conf, set_conf from local_config import conf, set_conf
except ImportError: except ImportError:
sys.stderr.write(''' sys.stderr.write('''
%s: E: local_config.py isn't tracked because of included secrets and %s: E: local_config.py isn't tracked because of included secrets and
%s site specific configurations. Rename local_config.py.skel and %s site specific configurations. Rename local_config.py.skel and
%s adjust to your needs. %s adjust to your needs.
'''[1:] % ( '''[1:] % (
sys.argv[0], sys.argv[0],
' ' * len(sys.argv[0]), ' ' * len(sys.argv[0]),
' ' * len(sys.argv[0]) ' ' * len(sys.argv[0])
)) ))
sys.exit(1) sys.exit(1)
class UrlBot(IdleBot): class UrlBot(IdleBot):
def __init__(self, jid, password, rooms, nick): def __init__(self, jid, password, rooms, nick):
super(UrlBot, self).__init__(jid, password, rooms, nick) super(UrlBot, self).__init__(jid, password, rooms, nick)
self.hist_ts = {p: [] for p in rate_limit_classes} self.hist_ts = {p: [] for p in rate_limit_classes}
self.hist_flag = {p: True for p in rate_limit_classes} self.hist_flag = {p: True for p in rate_limit_classes}
self.add_event_handler('message', self.message) self.add_event_handler('message', self.message)
self.priority = 100 self.priority = 100
for r in self.rooms: for r in self.rooms:
self.add_event_handler('muc::%s::got_online' % r, self.muc_online) self.add_event_handler('muc::%s::got_online' % r, self.muc_online)
def muc_message(self, msg_obj): def muc_message(self, msg_obj):
return super(UrlBot, self).muc_message(msg_obj) and self.handle_msg(msg_obj) return super(UrlBot, self).muc_message(msg_obj) and self.handle_msg(msg_obj)
def message(self, msg_obj): def message(self, msg_obj):
if 'groupchat' == msg_obj['type']: if 'groupchat' == msg_obj['type']:
return return
else: else:
self.logger.info("Got the following PM: %s" % str(msg_obj)) self.logger.info("Got the following PM: %s" % str(msg_obj))
def muc_online(self, msg_obj): def muc_online(self, msg_obj):
""" """
Hook for muc event "user joins" Hook for muc event "user joins"
""" """
# don't react to yourself # don't react to yourself
if msg_obj['muc']['nick'] == self.nick: if msg_obj['muc']['nick'] == self.nick:
return return
# TODO: move this to a undirected plugin, maybe new plugin type # TODO: move this to a undirected plugin, maybe new plugin type
arg_user = msg_obj['muc']['nick'] arg_user = msg_obj['muc']['nick']
arg_user_key = arg_user.lower() arg_user_key = arg_user.lower()
blob_userrecords = conf_load().get('user_records', {}) blob_userrecords = conf_load().get('user_records', {})
if arg_user_key in blob_userrecords: if arg_user_key in blob_userrecords:
records = blob_userrecords[arg_user_key] records = blob_userrecords[arg_user_key]
if not records: if not records:
return return
self.send_message( self.send_message(
mto=msg_obj['from'].bare, mto=msg_obj['from'].bare,
mbody='%s, there %s %d message%s for you:\n%s' % ( mbody='%s, there %s %d message%s for you:\n%s' % (
arg_user, arg_user,
'is' if 1 == len(records) else 'are', 'is' if 1 == len(records) else 'are',
len(records), len(records),
'' if 1 == len(records) else 's', '' if 1 == len(records) else 's',
'\n'.join(records) '\n'.join(records)
), ),
mtype='groupchat' mtype='groupchat'
) )
self.logger.info('sent %d offline records to room %s' % ( self.logger.info('sent %d offline records to room %s' % (
len(records), msg_obj['from'].bare len(records), msg_obj['from'].bare
)) ))
if conf('persistent_locked'): if conf('persistent_locked'):
self.logger.warn("couldn't get exclusive lock") self.logger.warn("couldn't get exclusive lock")
return False return False
set_conf('persistent_locked', True) set_conf('persistent_locked', True)
blob = conf_load() blob = conf_load()
if 'user_records' not in blob: if 'user_records' not in blob:
blob['user_records'] = {} blob['user_records'] = {}
if arg_user_key in blob['user_records']: if arg_user_key in blob['user_records']:
blob['user_records'].pop(arg_user_key) blob['user_records'].pop(arg_user_key)
conf_save(blob) conf_save(blob)
set_conf('persistent_locked', False) set_conf('persistent_locked', False)
# @rate_limited(10) # @rate_limited(10)
def send_reply(self, message, msg_obj=None): def send_reply(self, message, msg_obj=None):
""" """
Send a reply to a message Send a reply to a message
""" """
if self.show: if self.show:
self.logger.warn("I'm muted! (status: %s)" % self.show) self.logger.warn("I'm muted! (status: %s)" % self.show)
return return
set_conf('request_counter', conf('request_counter') + 1) set_conf('request_counter', conf('request_counter') + 1)
if str is not type(message): if str is not type(message):
message = '\n'.join(message) message = '\n'.join(message)
# check other bots, add nospoiler with urls # check other bots, add nospoiler with urls
def _prevent_panic(message, room): def _prevent_panic(message, room):
if 'http' in message: if 'http' in message:
other_bots = conf_load().get("other_bots", ()) other_bots = conf_load().get("other_bots", ())
users = self.plugin['xep_0045'].getRoster(room) users = self.plugin['xep_0045'].getRoster(room)
if set(users).intersection(set(other_bots)): if set(users).intersection(set(other_bots)):
message = '(nospoiler) %s' % message message = '(nospoiler) %s' % message
return message return message
if conf('debug_mode', False): if conf('debug_mode', False):
print(message) print(message)
else: else:
if msg_obj: if msg_obj:
message = _prevent_panic(message, msg_obj['from'].bare) message = _prevent_panic(message, msg_obj['from'].bare)
self.send_message( self.send_message(
mto=msg_obj['from'].bare, mto=msg_obj['from'].bare,
mbody=message, mbody=message,
mtype='groupchat' mtype='groupchat'
) )
else: # unset msg_obj == broadcast else: # unset msg_obj == broadcast
for room in self.rooms: for room in self.rooms:
message = _prevent_panic(message, room) message = _prevent_panic(message, room)
self.send_message( self.send_message(
mto=room, mto=room,
mbody=message, mbody=message,
mtype='groupchat' mtype='groupchat'
) )
def handle_msg(self, msg_obj): def handle_msg(self, msg_obj):
""" """
called for incoming messages called for incoming messages
:param msg_obj: :param msg_obj:
:returns nothing :returns nothing
""" """
content = msg_obj['body'] content = msg_obj['body']
if 'has set the subject to:' in content: if 'has set the subject to:' in content:
return return
if sys.argv[0] in content: if sys.argv[0] in content:
self.logger.info('silenced, this is my own log') self.logger.info('silenced, this is my own log')
return return
if 'nospoiler' in content: if 'nospoiler' in content:
self.logger.info('no spoiler for: ' + content) self.logger.info('no spoiler for: ' + content)
return return
self.data_parse_commands(msg_obj) self.data_parse_commands(msg_obj)
self.data_parse_other(msg_obj) self.data_parse_other(msg_obj)
def data_parse_commands(self, msg_obj): def data_parse_commands(self, msg_obj):
""" """
react to a message with the bots nick react to a message with the bots nick
:param msg_obj: dictionary with incoming message parameters :param msg_obj: dictionary with incoming message parameters
:returns: nothing :returns: nothing
""" """
global got_hangup global got_hangup
data = msg_obj['body'] data = msg_obj['body']
words = data.split() words = data.split()
if 2 > len(words): # need at least two words if 2 > len(words): # need at least two words
return None return None
# don't reply if beginning of the text matches bot_user # don't reply if beginning of the text matches bot_user
if not data.startswith(conf('bot_user')): if not data.startswith(conf('bot_user')):
return None return None
if 'hangup' in data: if 'hangup' in data:
self.logger.warn('received hangup: ' + data) self.logger.warn('received hangup: ' + data)
got_hangup = True got_hangup = True
sys.exit(1) sys.exit(1)
reply_user = msg_obj['mucnick'] reply_user = msg_obj['mucnick']
# TODO: check how several commands/plugins in a single message behave (also with rate limiting) # TODO: check how several commands/plugins in a single message behave (also with rate limiting)
reacted = False reacted = False
for plugin in plugin_storage[ptypes_COMMAND]: for plugin in plugin_storage[ptypes_COMMAND]:
if not plugin_enabled_get(plugin): if not plugin_enabled_get(plugin):
continue continue
ret = plugin( ret = plugin(
data=data, data=data,
cmd_list=[pl.plugin_name for pl in plugin_storage[ptypes_COMMAND]], cmd_list=[pl.plugin_name for pl in plugin_storage[ptypes_COMMAND]],
parser_list=[pl.plugin_name for pl in plugin_storage[ptypes_PARSE]], parser_list=[pl.plugin_name for pl in plugin_storage[ptypes_PARSE]],
reply_user=reply_user, reply_user=reply_user,
msg_obj=msg_obj, msg_obj=msg_obj,
argv=words[1:] argv=words[1:]
) )
if ret: if ret:
self._run_action(ret, plugin, msg_obj) self._run_action(ret, plugin, msg_obj)
reacted = True reacted = True
if not reacted and rate_limit(RATE_GLOBAL): if not reacted and rate_limit(RATE_GLOBAL):
ret = else_command({'reply_user': reply_user}) ret = else_command({'reply_user': reply_user})
if ret: if ret:
if 'msg' in ret: if 'msg' in ret:
self.send_reply(ret['msg'], msg_obj) self.send_reply(ret['msg'], msg_obj)
def data_parse_other(self, msg_obj): def data_parse_other(self, msg_obj):
""" """
react to any message react to any message
:param msg_obj: incoming message parameters :param msg_obj: incoming message parameters
:return: :return:
""" """
data = msg_obj['body'] data = msg_obj['body']
reply_user = msg_obj['mucnick'] reply_user = msg_obj['mucnick']
for plugin in plugin_storage[ptypes_PARSE]: for plugin in plugin_storage[ptypes_PARSE]:
if not plugin_enabled_get(plugin): if not plugin_enabled_get(plugin):
continue continue
ret = plugin(reply_user=reply_user, data=data) ret = plugin(reply_user=reply_user, data=data)
if ret: if ret:
self._run_action(ret, plugin, msg_obj) self._run_action(ret, plugin, msg_obj)
def _run_action(self, action, plugin, msg_obj): def _run_action(self, action, plugin, msg_obj):
""" """
Execute the plugin's execution plan Execute the plugin's execution plan
:param action: dict with event and/or msg :param action: dict with event and/or msg
:param plugin: plugin obj :param plugin: plugin obj
:param msg_obj: xmpp message obj :param msg_obj: xmpp message obj
""" """
if 'event' in action: if 'event' in action:
event = action["event"] event = action["event"]
if 'msg' in event: if 'msg' in event:
register_event(event["time"], self.send_reply, [event['msg']]) register_event(event["time"], self.send_reply, [event['msg']])
elif 'command' in event: elif 'command' in event:
command = event["command"] command = event["command"]
if rate_limit(RATE_EVENT): if rate_limit(RATE_EVENT):
register_event(event["time"], command[0], command[1]) register_event(event["time"], command[0], command[1])
if 'msg' in action and rate_limit(RATE_CHAT | plugin.ratelimit_class): if 'msg' in action and rate_limit(RATE_CHAT | plugin.ratelimit_class):
self.send_reply(action['msg'], msg_obj) self.send_reply(action['msg'], msg_obj)
if 'presence' in action: if 'presence' in action:
presence = action['presence'] presence = action['presence']
conf_set('presence', presence) conf_set('presence', presence)
self.status = presence.get('msg') self.status = presence.get('msg')
self.show = presence.get('status') self.show = presence.get('status')
self.send_presence(pstatus=self.status, pshow=self.show)
# self.reconnect(wait=True)
self.send_presence(pstatus=self.status, pshow=self.show)
# self.reconnect(wait=True)
if '__main__' == __name__: if '__main__' == __name__:
start(UrlBot, True) start(UrlBot, True)