tox.ini: change dep to dnspython3

see also https://github.com/mozilla/http-observatory/pull/86/files

LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2017 Thorsten Sperber
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

common.py

@@ -39,7 +39,7 @@ VERSION = get_version_git()
 def fetch_page(url):
     log = logging.getLogger(__name__)
     log.info('fetching page ' + url)
-    response = requests.get(url, headers={'User-Agent': USER_AGENT}, stream=True)
+    response = requests.get(url, headers={'User-Agent': USER_AGENT}, stream=True, timeout=15)
     content = response.raw.read(BUFSIZ, decode_content=True)
     return content.decode(response.encoding or 'utf-8'), response.headers
 

deploy/hosts

@@ -1,2 +1,2 @@
 [bots]
-aero2k.de ansible_host=2a01:4f8:d16:130c::2
+aero2k.de ansible_host=2a01:4f8:d16:130c::2 ansible_become_method=su

events.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import logging
 import time
 import sched
 import threading
@@ -8,7 +9,7 @@ EVENTLOOP_DELAY = 0.100  # seconds
 event_list = sched.scheduler(time.time, time.sleep)
 
 
-def register_active_event(t, callback, args, action_runner, plugin, msg_obj):
+def register_active_event(t, callback, args, action_runner, plugin, msg_obj, mutex=None):
     """
     Execute a callback at a given time and react on the output
 
@@ -24,10 +25,14 @@ def register_active_event(t, callback, args, action_runner, plugin, msg_obj):
         action = callback(*func_args)
         if action:
             action_runner(action=action, plugin=plugin, msg_obj=msg_obj)
-    event_list.enterabs(t, 0, func, args)
+    register_event(t, func, args, mutex=mutex)
 
 
-def register_event(t, callback, args):
+def register_event(t, callback, args, **kwargs):
+    for pending_event in event_list.queue:
+        if kwargs.get('mutex') and pending_event.kwargs.get('mutex', None) == kwargs.get('mutex'):
+            logging.debug("Dropped event: %s", kwargs.get('mutex'))
+            return
     event_list.enterabs(t, 0, callback, args)
 
 

idlebot.py

@@ -1,25 +1,31 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 import logging
-import time
 import sys
+import time
+import _ssl
+
+from sleekxmpp import ClientXMPP
+
 import config
 import events
 from common import VERSION
 
-from sleekxmpp import ClientXMPP
-
 
 class IdleBot(ClientXMPP):
     def __init__(self, jid, password, rooms, nick):
         ClientXMPP.__init__(self, jid, password)
 
+        self.ssl_version = _ssl.PROTOCOL_TLSv1_2
+
         self.rooms = rooms
         self.nick = nick
 
         self.add_event_handler('session_start', self.session_start)
         self.add_event_handler('groupchat_message', self.muc_message)
         self.add_event_handler('disconnected', self.disconnected)
+        self.add_event_handler('presence_error', self.disconnected)
+        self.add_event_handler('session_end', self.disconnected)
         self.priority = 0
         self.status = None
         self.show = None
@@ -29,7 +35,8 @@ class IdleBot(ClientXMPP):
             self.add_event_handler('muc::%s::got_offline' % room, self.muc_offline)
 
     def disconnected(self, _):
-        exit(0)
+        self.logger.warn("Disconnected! dbg: {}".format(str(_)))
+        self.disconnect(wait=True)
 
     def session_start(self, _):
         self.get_roster()
@@ -81,8 +88,7 @@ class IdleBot(ClientXMPP):
         """
         disconnect and exit
         """
-        self.disconnect()
+        self.disconnect(wait=True)
-        sys.exit(1)
 
 
 def start(botclass, active=False):
@@ -105,12 +111,14 @@ def start(botclass, active=False):
 
     bot.connect()
     bot.register_plugin('xep_0045')
+    bot.register_plugin('xep_0199', {'keepalive': True})
+    bot.register_plugin('xep_0308')
     bot.process()
 
     config.runtimeconf_set('start_time', -time.time())
 
     if active:
-        import plugins
+        pass
 
     events.event_loop.start()
 

plugins/cake.py

@@ -6,28 +6,48 @@ from plugin_system import pluginfunction, ptypes
 from rate_limit import RATE_FUN, RATE_GLOBAL
 
 
+def give_item(user, item_name, search_word=None):
+    if not search_word:
+        search_word = item_name
+    return {'msg': '{} for {}: {}'.format(item_name, user, giphy(search_word, 'dc6zaTOxFJmzC'))}
+
+
+def cake_excuse(user):
+    return {
+        'msg': '{}: {}'.format(user, random.choice(cakes))
+    }
+
+
 @pluginfunction('cake', 'displays a cake ASCII art', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
 def command_cake(argv, **args):
     if {'please', 'bitte'}.intersection(set(argv)):
-        return {
+        return give_item(args['reply_user'], 'cake')
-            'msg': 'cake for {}: {}'.format(args['reply_user'], giphy('cake', 'dc6zaTOxFJmzC'))
+    else:
-        }
+        return cake_excuse(args['reply_user'])
-
-    return {
-        'msg': args['reply_user'] + ': %s' % random.choice(cakes)
-    }
 
 
 @pluginfunction('keks', 'keks!', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
 def command_cookie(argv, **args):
     if {'please', 'bitte'}.intersection(set(argv)):
-        return {
+        return give_item(args['reply_user'], 'keks', 'cookie')
-            'msg': 'keks für {}: {}'.format(args['reply_user'], giphy('cookie', 'dc6zaTOxFJmzC'))
+    else:
-        }
+        return cake_excuse(args['reply_user'])
 
-    return {
+
-        'msg': args['reply_user'] + ': %s' % random.choice(cakes)
+@pluginfunction('schnitzel', 'schnitzel!', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
-    }
+def command_schnitzel(argv, **args):
+    if {'please', 'bitte'}.intersection(set(argv)):
+        return give_item(args['reply_user'], 'schnitzel')
+    else:
+        return cake_excuse(args['reply_user'])
+
+
+@pluginfunction('kaffee', 'kaffee!', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
+def command_coffee(argv, **args):
+    if {'please', 'bitte'}.intersection(set(argv)):
+        return give_item(args['reply_user'], 'kaffee', 'coffee')
+    else:
+        return cake_excuse(args['reply_user'])
 
 
 cakes = [
@@ -46,4 +66,3 @@ cakes = [
     "I'm going to kill you, and all the cake is gone.",
     "Who's gonna make the cake when I'm gone? You?"
 ]
-

plugins/commands.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import logging
 
+import re
 import events
 import json
 import random
@@ -13,6 +14,7 @@ from lxml import etree
 
 import config
 from common import VERSION
+from plugins.searx import searx
 from rate_limit import RATE_FUN, RATE_GLOBAL, RATE_INTERACTIVE, RATE_NO_SILENCE, RATE_NO_LIMIT
 from plugin_system import pluginfunction, ptypes, plugin_storage, plugin_enabled_get, plugin_enabled_set
 
@@ -279,15 +281,158 @@ def command_dice(argv, **args):
     }
 
 
+@pluginfunction('xchoose', 'chooses randomly between nested option groups', ptypes.COMMAND, ratelimit_class=RATE_INTERACTIVE)
+def command_xchoose(argv, **args):
+
+    class ChooseTree():
+        def __init__(self, item=None):
+            self.item = item
+            self.tree = None
+            self.closed = False
+
+            # opening our root node
+            if self.item is None:
+                self.open()
+
+        def open(self):
+            if self.tree is None:
+                self.tree = []
+            elif self.closed:
+                raise Exception("cannot re-open group for item '%s'" % (self.item))
+
+        def close(self):
+            if self.tree is None:
+                raise Exception("close on unopened bracket")
+            elif len(self.tree) == 0:
+                raise Exception("item '%s' has a group without sub options" % (self.item))
+            else:
+                self.closed = True
+
+        def last(self):
+            return self.tree[-1]
+
+        def choose(self):
+            if self.item:
+                yield self.item
+            
+            if self.tree:
+                sel = random.choice(self.tree)
+                for sub in sel.choose():
+                    yield sub
+
+        def add(self, item):
+            self.tree.append( ChooseTree(item) )
+        
+    # because of error handling we're nesting this function here
+    def xchoose(line):
+        item = ''
+        quote = None
+        choose_tree = ChooseTree()
+        choose_stack = [ choose_tree ]
+        bracket_stack = []
+
+        for pos, c in enumerate(line, 1):
+            try:
+                if quote:
+                    if c == quote:
+                        quote = None
+                    else:
+                        item += c
+
+                elif c == ' ':
+                    if item:
+                        choose_stack[-1].add(item)
+                        item = ''
+
+                elif c in ('(', '[', '{', '<'):
+                    if item:
+                        choose_stack[-1].add(item)
+                        item = ''
+
+                    try:
+                        last = choose_stack[-1].last()
+                        last.open()
+                        choose_stack.append(last)
+                        bracket_stack.append(c)
+                    except IndexError:
+                        raise Exception("cannot open group without preceding option")
+
+                elif c in (')', ']', '}', '>'):
+                    if not bracket_stack:
+                        raise Exception("missing leading bracket for '%s'" % (c))
+
+                    opening_bracket = bracket_stack.pop(-1)
+                    wanted_closing_bracket = { '(':')', '[':']', '{':'}', '<':'>' }[opening_bracket]
+                    if c != wanted_closing_bracket:
+                        raise Exception("bracket mismatch, wanted bracket '%s' but got '%s'" % (
+                            wanted_closing_bracket, c))
+
+                    if item:
+                        choose_stack[-1].add(item)
+                        item = ''
+
+                    choose_stack[-1].close()
+                    choose_stack.pop(-1)
+
+                elif c in ('"', "'"):
+                    quote = c
+
+                else:
+                    item += c
+
+            except Exception as e:
+                raise Exception("%s (at pos %d)" % (e, pos))
+
+        if bracket_stack:
+            raise Exception("missing closing bracket for '%s'" % (bracket_stack[-1]))
+
+        if quote:
+            raise Exception("missing closing quote (%s)" % (quote))
+
+        if item:
+            choose_stack[-1].add(item)
+
+        return ' '.join(choose_tree.choose())
+
+
+    # start of command_xchoose
+    line = re.sub('.*xchoose *', '', args['data'])
+    if not line:
+        return {
+            'msg': '%s: %s' % (args['reply_user'], 'missing options')
+        }
+    try:
+        return {
+            'msg': '%s: %s' % (args['reply_user'], xchoose(line))
+        }
+    except Exception as e:
+        return {
+            'msg': '%s: %s' % (args['reply_user'], str(e))
+        }
+
+
 @pluginfunction('choose', 'chooses randomly between arguments', ptypes.COMMAND, ratelimit_class=RATE_INTERACTIVE)
 def command_choose(argv, **args):
     alternatives = argv
-    binary = ['Yes', 'No', 'Maybe']
+    binary = (
+        (('Yes.', 'Yeah!', 'Ok!', 'Aye!', 'Great!'), 4),
+        (('No.', 'Naah..', 'Meh.', 'Nay.', 'You stupid?'), 4),
+        (('Maybe.', 'Dunno.', 'I don\'t care.'), 2)
+    )
+
+    def weighted_choice(choices):
+        total = sum(w for c, w in choices)
+        r = random.uniform(0, total)
+        upto = 0
+        for c, w in choices:
+            if upto + w >= r:
+                return c
+            upto += w
 
     # single or no choice
     if len(alternatives) < 2:
         return {
-            'msg': '{}: {}.'.format(args['reply_user'], random.choice(binary))
+            'msg': '{}: {}'.format(args['reply_user'], random.choice(weighted_choice(binary)))
         }
     elif 'choose' not in alternatives:
         choice = random.choice(alternatives)
@@ -302,14 +447,14 @@ def command_choose(argv, **args):
         for item in options:
             if item == 'choose':
                 if len(current_choices) < 2:
-                    responses.append(random.choice(binary))
+                    responses.append(random.choice(weighted_choice(binary)))
                 else:
                     responses.append(random.choice(current_choices))
                 current_choices = []
             else:
                 current_choices.append(item)
         if len(current_choices) < 2:
-            responses.append(random.choice(binary))
+            responses.append(random.choice(weighted_choice(binary)))
         else:
             responses.append(random.choice(current_choices))
         return responses
@@ -350,7 +495,8 @@ def command_teatimer(argv, **args):
         ),
         'event': {
             'time': ready,
-            'msg': (args['reply_user'] + ': Your tea is ready!')
+            'msg': (args['reply_user'] + ': Your tea is ready!'),
+            'mutex': 'teatimer_{}'.format(args['reply_user'])
         }
     }
 
@@ -616,9 +762,11 @@ def command_dsa_watcher(argv=None, **_):
     msg = 'next crawl set to %s' % time.strftime('%Y-%m-%d %H:%M', time.localtime(crawl_at))
     out.append(msg)
     return {
+        # 'msg': out,
         'event': {
             'time': crawl_at,
-            'command': (command_dsa_watcher, ([],))
+            'command': (command_dsa_watcher, ([],)),
+            'mutex': 'dsa'
         }
     }
 
@@ -734,37 +882,8 @@ def reload_runtimeconfig(argv, **args):
         return {'msg': 'done'}
 
 
-@pluginfunction('snitch', "tell on a spammy user", ptypes.COMMAND)
+@pluginfunction('ducksearch', 'search the web (using duckduckgo)', ptypes.COMMAND)
-def ignore_user(argv, **args):
+def search_the_duck(argv, **args):
-    if not argv:
-        return {'msg': 'syntax: "{}: snitch username"'.format(config.conf_get("bot_nickname"))}
-
-    then = time.time() + 15 * 60
-    spammer = argv[0]
-
-    if spammer == config.conf_get("bot_owner"):
-        return {
-            'msg': 'My owner does not spam, he is just very informative.'
-        }
-
-    if spammer not in config.runtime_config_store['spammers']:
-        config.runtime_config_store['spammers'].append(spammer)
-
-    def unblock_user(user):
-        if user not in config.runtime_config_store['spammers']:
-            config.runtime_config_store['spammers'].append(user)
-
-    return {
-        'msg': 'user reported and ignored till {}'.format(time.strftime('%H:%M', time.localtime(then))),
-        'event': {
-            'time': then,
-            'command': (unblock_user, ([spammer],))
-        }
-    }
-
-
-@pluginfunction('search', 'search the web (using duckduckgo)', ptypes.COMMAND)
-def search_the_web(argv, **args):
     url = 'http://api.duckduckgo.com/'
     params = dict(
         q=' '.join(argv),
@@ -795,6 +914,24 @@ def search_the_web(argv, **args):
         return {'msg': 'Sorry, no results.'}
 
 
+@pluginfunction('search', 'search the web (using searx)', ptypes.COMMAND)
+def search_the_web(argv, **args):
+    result = searx(' '.join(argv))
+    if not result:
+        return {'msg': 'Sorry, no results.'}
+    else:
+        abstract, url = result
+
+    if len(abstract) > 150:
+        suffix = '…'
+    else:
+        suffix = ''
+    return {
+        'msg': '{}{} ({})'.format(abstract[:150], suffix, url)
+    }
+    pass
+
+
 @pluginfunction('raise', 'only for debugging', ptypes.COMMAND)
 def raise_an_error(argv, **args):
     if args['reply_user'] == config.conf_get("bot_owner"):

plugins/morse.py

@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+import logging
+import re
+
+from plugin_system import pluginfunction, ptypes
+from rate_limit import RATE_FUN, RATE_GLOBAL
+
+log = logging.getLogger(__name__)
+
+# copy from https://de.wikipedia.org/wiki/Morsezeichen
+raw_wiki_copy = """
+A· −
+B− · · ·
+C− · − ·
+D− · ·
+E·
+F· · − ·
+G− − ·
+H· · · ·
+I· ·
+J· − − −
+K− · −
+L· − · ·
+M− −
+N− ·
+O− − −
+P· − − ·
+Q− − · −
+R· − ·
+S· · ·
+T−
+U· · −
+V· · · −
+W· − −
+X− · · −
+Y− · − −
+Z− − · ·
+1· − − − −
+2· · − − −
+3· · · − −
+4· · · · −
+5· · · · ·
+6− · · · ·
+7− − · · ·
+8− − − · ·
+9− − − − ·
+0− − − − −
+"""
+
+
+# machen dictionary aus wikipaste
+def wiki_paste_to_morse_dict(wikicopy):
+    wikicopy = wikicopy.replace(' ', '')
+    morse_dict = {l[0]: l[1:] for l in wikicopy.splitlines() if l}
+    return morse_dict
+
+
+ascii_morse = wiki_paste_to_morse_dict(raw_wiki_copy)
+morse_ascii = {v: k for k, v in ascii_morse.items()}
+
+
+# return a dictionary of possible morse-chars as key
+# and their count as value
+def possible_morse_chars(string):
+    """
+    returns dit,dah or None
+    """
+    stats = {}
+
+    for c in re.sub("[\w\d ]", '', string):
+        try:
+            stats[c] += 1
+        except KeyError:
+            stats[c] = 1
+
+    return stats
+
+
+# return morse-encoded string
+def morse_encode(string, dot='·', dash='−', sep=' ', ignore_unknown=False):
+    morse_codes = []
+
+    for char in string.upper():
+        try:
+            morse_codes.append(ascii_morse[char].replace('·', dot).replace('−', dash))
+        except KeyError:
+            if not ignore_unknown:
+                morse_codes.append(char)
+
+    return sep.join(morse_codes)
+
+
+# return morse-decoded string with number of errors as tuple
+# -> (decoded string, num errors)
+def morse_decode(string, dot=None, dash=None):
+    """
+    decode a "morse string" to ascii text
+    uses \s{2,} as word separator
+    """
+    # dot and dash given, just decode
+    if dot and dash:
+        errors = 0
+
+        words = []
+        # drawback: does not allow single characters.
+        for match in re.finditer('([{dit}{dah}]+((?:\\s)[{dit}{dah}]+)+|\w+)'.format(dit=dot, dah=dash), string):
+            word = match.group()
+            log.debug("morse word: ", word)
+            if any([dot in word, dash in word]):
+                w = []
+                for morse_character in word.split():
+                    try:
+                        character = morse_ascii[morse_character.replace(dot, '·').replace(dash, '−')]
+                        print("Converted \t{} \tto {}".format(morse_character, character))
+                    except KeyError:
+                        character = morse_character
+                        errors += 1
+                    w.append(character)
+                words.append(''.join(w))
+            # words.append(''.join([morse_ascii[x.replace(dot, '·').replace(dash, '−')] for x in word.split()]))
+            else:
+                words.append(word)
+        return ' '.join(words), errors
+
+    # dot/dash given, search for dash/dot
+    else:
+        if not dash:
+            dash_stats = {x: string.count(x) for x in '-−_'}
+            dash = max(dash_stats, key=dash_stats.get)
+        if not dot:
+            dot_stats = {x: string.count(x) for x in '.·*'}
+            dot = max(dot_stats, key=dot_stats.get)
+
+        return morse_decode(string, dot=dot, dash=dash)
+
+
+@pluginfunction('morse-encode', 'encode string to morse', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
+def command_morse_encode(argv, **args):
+    if not argv:
+        return {
+            'msg': args['reply_user'] + "usage: morse-encode <string>"
+        }
+
+    if len(argv) == 1 and argv[0] == 'that':
+        message_stack = args['stack']
+        if not message_stack[-1]:
+            return
+        message = message_stack[-1]['body']
+    else:
+        message = ' '.join(argv)
+
+    return {
+        'msg': args['reply_user'] + ': %s' % morse_encode(message)
+    }
+
+
+@pluginfunction('morse-decode', 'decode morse encoded string', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
+def command_morse_decode(argv, **args):
+    if not argv:
+        return {
+            'msg': args['reply_user'] + "usage: morse-decode <string>"
+        }
+
+    if len(argv) == 1 and argv[0] == 'that':
+        message_stack = args['stack']
+        if not message_stack[-1]:
+            return
+        message = message_stack[-1]['body']
+    else:
+        message = ' '.join(argv)
+
+    decoded, errors = morse_decode(message, dot='·', dash='-')
+
+    return {
+        'msg': args['reply_user'] + ': %s (%d errors)' % (decoded, errors)
+    }

plugins/parsers.py

@@ -39,6 +39,14 @@ def parse_mental_ill(**args):
         }
 
 
+@pluginfunction('woof', '*puts sunglasses on*', ptypes.PARSE, ratelimit_class=RATE_NO_SILENCE | RATE_GLOBAL)
+def command_woof(**args):
+    if 'who let the bots out' in args['data']:
+        return {
+            'msg': 'beeep! beep! beep! beep! beep!'
+        }
+
+
 @pluginfunction('debbug', 'parse Debian bug numbers', ptypes.PARSE, ratelimit_class=RATE_NO_SILENCE | RATE_GLOBAL)
 def parse_debbug(**args):
     bugs = re.findall(r'#(\d{4,})', args['data'])
@@ -50,14 +58,11 @@ def parse_debbug(**args):
         log.info('detected Debian bug #%s' % b)
 
         url = 'https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s' % b
-        status, title = extract_title(url)
 
-        if 0 == status:
+        title = extract_title(url)
+
+        if title:
             out.append('Debian Bug: %s: %s' % (title, url))
-        elif 3 == status:
-            out.append('error for #%s: %s' % (b, title))
-        else:
-            log.info('unknown status %d' % status)
 
     return {
         'msg': out
@@ -122,34 +127,6 @@ def parse_slash_me(**args):
         }
 
 
-@pluginfunction("recognize_bots", "got ya", ptypes.PARSE, enabled=False)
-def recognize_bots(**args):
-    # disabled until channel separation
-    return
-    unique_standard_phrases = (
-        'independent bot and have nothing to do with other artificial intelligence systems',
-        'new Debian Security Announce',
-        'I\'m a bot (highlight me',
-    )
-
-    def _add_to_list(username, message):
-        if username not in config.runtime_config_store['other_bots']:
-            config.runtime_config_store['other_bots'].append(username)
-            config.runtimeconf_persist()
-            log.info("Adding {} to the list of bots (now {})".format(username, config.runtime_config_store['other_bots']))
-            return {
-                'event': {
-                    'time': time.time() + 3,
-                    'msg': message
-                }
-            }
-
-    if any([phrase in args['data'] for phrase in unique_standard_phrases]):
-        return _add_to_list(args['reply_user'], 'Making notes...')
-    elif 'I\'ll be back' in args['data']:
-        return _add_to_list(args['reply_user'], 'Hey there, buddy!')
-
-
 @pluginfunction('resolve-url-title', 'extract titles from urls', ptypes.PARSE, ratelimit_class=RATE_URL)
 def resolve_url_title(**args):
     user = args['reply_user']
@@ -165,7 +142,7 @@ def resolve_url_title(**args):
     url_blacklist = config.runtime_config_store['url_blacklist'].values()
 
     out = []
-    for url in result:
+    for url in result[:10]:
         if any([re.match(b, url) for b in url_blacklist]):
             log.info('url blacklist match for ' + url)
             break

plugins/searx.py

@@ -0,0 +1,99 @@
+import logging
+import time
+from functools import wraps
+import json
+import requests
+from lxml import etree, html
+from requests import HTTPError
+
+search_list = []
+
+if not hasattr(json, 'JSONDecodeError'):
+    json.JSONDecodeError = ValueError
+
+class RateLimitingError(HTTPError):
+    pass
+
+
+def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
+    """Retry calling the decorated function using an exponential backoff.
+
+    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+
+    :param ExceptionToCheck: the exception to check. may be a tuple of
+        exceptions to check
+    :type ExceptionToCheck: Exception or tuple
+    :param tries: number of times to try (not retry) before giving up
+    :type tries: int
+    :param delay: initial delay between retries in seconds
+    :type delay: int
+    :param backoff: backoff multiplier e.g. value of 2 will double the delay
+        each retry
+    :type backoff: int
+    :param logger: logger to use. If None, print
+    :type logger: logging.Logger instance
+    """
+
+    def deco_retry(f):
+
+        @wraps(f)
+        def f_retry(*args, **kwargs):
+            mtries, mdelay = tries, delay
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except ExceptionToCheck as e:
+                    msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
+                    if logger:
+                        logger.warning(msg)
+                    else:
+                        print(msg)
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+
+        return f_retry  # true decorator
+
+    return deco_retry
+
+
+def fetch_all_searx_engines():
+    # error handling is for pussies
+    tree = etree.XML(
+        requests.get("http://stats.searx.oe5tpo.com").content,
+        parser=html.HTMLParser()
+    )
+    searxes = [str(x) for x in tree.xpath('//span[text()[contains(.,"200 - OK")]]/../..//a/text()')]
+
+    return searxes
+
+
+@retry(ExceptionToCheck=(RateLimitingError, json.JSONDecodeError))
+def searx(text):
+    global search_list
+    if not search_list:
+        search_list = fetch_all_searx_engines()
+    logger = logging.getLogger(__name__)
+
+    url = search_list[0]
+    logger.info('Currently feeding from {} (of {} in stock)'.format(url, len(search_list)))
+    response = requests.get(url, params={
+        'q': text,
+        'format': 'json',
+        'lang': 'de'
+    })
+    if response.status_code == 429:
+        search_list.pop(0)
+        raise RateLimitingError(response=response, request=response.request)
+    try:
+        response = response.json()
+    except json.JSONDecodeError:
+        # "maintenance" they say...
+        search_list.pop(0)
+        raise
+
+    if not response['results']:
+        return
+    return [(r.get('content', ''), r['url']) for r in response['results']][0]

tox.ini

@@ -0,0 +1,32 @@
+# defaults to tests
+# run with tox -e urlbot,  tox -e idlebot, etc
+[tox]
+envlist = test
+# we have no setup.py
+skipsdist = true
+
+[testenv]
+
+envdir = {toxinidir}/.env
+
+deps=nose
+     fasteners
+     sleekxmpp
+     configobj
+     requests
+     lxml
+     dnspython3
+     pyasn1
+     pyasn1-modules
+
+commands=
+  test: nosetests []  # substitute with tox' positional arguments
+  idlebot: python idlebot.py []
+  urlbot: python urlbot.py []
+  sh: sh []
+  bash: bash []
+  zsh: zsh []
+
+whitelist_externals = zsh 
+                      bash 
+		      sh

urlbot.py

@@ -12,7 +12,9 @@ from collections import deque
 from lxml import etree
 
 import requests
+from sleekxmpp.plugins import PluginNotFound
 
+import plugins  # force initialization
 from plugin_system import plugin_storage, ptypes, plugin_enabled_get
 from rate_limit import rate_limit_classes, RATE_GLOBAL, RATE_CHAT, RATE_EVENT, rate_limit
 
@@ -44,6 +46,9 @@ class UrlBot(IdleBot):
         for room in self.rooms:
             self.add_event_handler('muc::%s::got_online' % room, self.muc_online)
 
+        dsa_plugin = list(filter(lambda x: x.plugin_name == 'dsa-watcher', plugin_storage[ptypes.COMMAND]))[0]
+        self._run_action(dsa_plugin(), dsa_plugin, None)
+
     def muc_message(self, msg_obj):
         """
         Handle muc messages, return if irrelevant content or die by hangup.
@@ -83,7 +88,7 @@ class UrlBot(IdleBot):
         request_counter = int(config.runtimeconf_get('request_counter'))
         config.runtimeconf_set('request_counter', request_counter + 1)
 
-        if str is not type(message):
+        if not isinstance(message, str):
             message = '\n'.join(message)
 
         def cached(function, ttl=60):
@@ -109,7 +114,10 @@ class UrlBot(IdleBot):
             other_bots = config.runtimeconf_get("other_bots")
             if not other_bots:
                 return False
+            try:
                 users = self.plugin['xep_0045'].getRoster(room)
+            except PluginNotFound:
+                users = []
             return set(users).intersection(set(other_bots))
 
         def _prevent_panic(message, room):
@@ -292,7 +300,7 @@ class UrlBot(IdleBot):
             if not plugin_enabled_get(plugin):
                 continue
 
-            ret = plugin(reply_user=reply_user, data=data)
+            ret = plugin(reply_user=reply_user, data=data, sender=msg_obj['from'])
 
             if ret:
                 self._run_action(ret, plugin, msg_obj)
@@ -349,7 +357,8 @@ class UrlBot(IdleBot):
                         args=command[1],
                         action_runner=self._run_action,
                         plugin=plugin,
-                        msg_obj=msg_obj
+                        msg_obj=msg_obj,
+                        mutex=event.get('mutex')
                     )
 
         if 'msg' in action and rate_limit(RATE_CHAT | plugin.ratelimit_class):