tox.ini: change dep to dnspython3

see also https://github.com/mozilla/http-observatory/pull/86/files

LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2017 Thorsten Sperber
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

common.py

@@ -39,7 +39,7 @@ VERSION = get_version_git()
 def fetch_page(url):
     log = logging.getLogger(__name__)
     log.info('fetching page ' + url)
-    response = requests.get(url, headers={'User-Agent': USER_AGENT}, stream=True)
+    response = requests.get(url, headers={'User-Agent': USER_AGENT}, stream=True, timeout=15)
     content = response.raw.read(BUFSIZ, decode_content=True)
     return content.decode(response.encoding or 'utf-8'), response.headers
 

deploy/deploy.yml

@@ -20,6 +20,7 @@
   vars:
   - botrepo: http://aero2k.de/t/repos/urlbot-native.git
   - pypi_mirror: http://pypi.fcio.net/simple/
+  - systemd: true
   tasks:
   - include_vars: credentials.yml
     tags: [render_config]
@@ -27,6 +28,7 @@
     shell: virtualenv -p python3 --system-site-packages ~/botenv creates=~/botenv/bin/activate
   - name: virtualenv for supervisord
     shell: virtualenv -p python2 ~/svenv creates=~/svenv/bin/activate
+    when: not systemd
   - name: clone repository
     git: repo="{{botrepo}}" dest=~/urlbot force=yes update=yes
     register: source_code
@@ -34,6 +36,7 @@
     pip: requirements="~/urlbot/requirements.txt" virtualenv=~/botenv extra_args="-i {{pypi_mirror}}"
   - name: install supervisor
     pip: name=supervisor virtualenv=~/svenv extra_args="-i {{pypi_mirror}}"
+    when: not systemd
 
   - name: set configuration
     lineinfile: dest=~/urlbot/local_config.ini create=yes line="{{item.key}} = {{item.value}}" regexp="^{{item.key}}.="
@@ -60,29 +63,94 @@
   - name: create supervisor config
     copy: src=supervisord.conf dest=~/supervisord.conf
     register: supervisord
+    when: not systemd
+
+  - name: create directory for systemd unit file
+    shell: mkdir -p ~/.config/systemd/user/ creates=~/.config/systemd/user/
+    when: systemd
+
+  - name: create unitfile
+    copy: src=urlbug@.service dest=~/.config/systemd/user/urlbug@.service
+    when: systemd
+    register: unitfile
+
+  # crapshit does not work
+  - name: reload unitfiles
+    become: true
+    shell: systemctl daemon-reload
+    when: unitfile.changed
+    ignore_errors: true
+
+  - name: enable services
+    shell: "systemctl --user enable urlbug@{{item}}.service"
+    with_items:
+      - idlebot
+      - urlbot
+    when: systemd
 
   - name: verify supervisor running
     shell: nc -z 127.0.0.1 9004; echo $? executable=/bin/bash
     register: supervisor_running
     changed_when: false
+    when: not systemd
 
   - name: start supervisord
     shell: source ~/svenv/bin/activate && supervisord executable=/bin/bash
     register: start_supervisor
-    when: supervisord.changed or supervisor_running.stdout == "1"
+    when:
+      - not systemd
+      - supervisord.changed or supervisor_running.stdout == "1"
     #changed_when: "'already listening' not in start_supervisor.stdout"
 
   - name: activate supervisord changes
-    when: supervisord.changed
     shell: source ~/svenv/bin/activate && supervisorctl reload executable=/bin/bash
+    when:
+      - not systemd
+      - supervisord.changed
 
   - name: idlebot started
     supervisorctl: name=idlebot state=restarted supervisorctl_path=~/svenv/bin/supervisorctl
-    when: (source_code.changed or urlbot_config.changed) and not supervisord.changed
+    when:
+      - not systemd
+      - (source_code.changed or urlbot_config.changed) and not supervisord.changed
 
-  - pause: seconds=30
+  # following tasks are workaround for missing ansible systemd-user support
-    when: (source_code.changed or urlbot_config.changed) and not supervisord.changed
+  - name: get systemd unit status
+    shell: systemctl --user status urlbug.slice
+    register: systemd_unit_status
+
+  - debug: var=systemd_unit_status
+  - debug: msg="{{'{{item}}.service' not in systemd_unit_status.stdout}}"
+    with_items:
+      - idlebot
+      - urlbot
+
+  - name: bots started
+    shell: "systemctl --user start urlbug@{{item}}.service && sleep 20"
+    with_items:
+      - idlebot
+      - urlbot
+    when: systemd and '{{item}}.service' not in systemd_unit_status.stdout
+    register: started_bots
+
+  - debug: var=started_bots
+
+  - name: bots restarted
+    shell: "systemctl --user restart urlbug@{{item}}.service && sleep 10"
+    with_items:
+      - idlebot
+      - urlbot
+    when:
+      - systemd
+      - source_code.changed or urlbot_config.changed
+
+  - pause: seconds=20
+    when:
+      - not systemd
+      - (source_code.changed or urlbot_config.changed) and not supervisord.changed
 
   - name: urlbot started
     supervisorctl: name=bot state=restarted supervisorctl_path=~/svenv/bin/supervisorctl
-    when: (source_code.changed or urlbot_config.changed) and not supervisord.changed
+    when:
+      - not systemd
+      - (source_code.changed or urlbot_config.changed) and not supervisord.changed

deploy/hosts

@@ -1,2 +1,2 @@
 [bots]
-aero2k.de ansible_host=2a01:4f8:d16:130c::2
+aero2k.de ansible_host=2a01:4f8:d16:130c::2 ansible_become_method=su

deploy/requirements-deploy.yml

@@ -0,0 +1,2 @@
+ansible
+markupsafe

deploy/urlbug@.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=jabber bot entertaining and supporting activity on jabber MUCs
+
+[Service]
+ExecStart=/home/jabberbot/botenv/bin/python3 /home/jabberbot/urlbot/%i.py
+WorkingDirectory=/home/jabberbot/urlbot/
+StandardOutput=journal+console
+StandardError=journal+console
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

events.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import logging
 import time
 import sched
 import threading
@@ -8,7 +9,7 @@ EVENTLOOP_DELAY = 0.100  # seconds
 event_list = sched.scheduler(time.time, time.sleep)
 
 
-def register_active_event(t, callback, args, action_runner, plugin, msg_obj):
+def register_active_event(t, callback, args, action_runner, plugin, msg_obj, mutex=None):
     """
     Execute a callback at a given time and react on the output
 
@@ -24,10 +25,14 @@ def register_active_event(t, callback, args, action_runner, plugin, msg_obj):
         action = callback(*func_args)
         if action:
             action_runner(action=action, plugin=plugin, msg_obj=msg_obj)
-    event_list.enterabs(t, 0, func, args)
+    register_event(t, func, args, mutex=mutex)
 
 
-def register_event(t, callback, args):
+def register_event(t, callback, args, **kwargs):
+    for pending_event in event_list.queue:
+        if kwargs.get('mutex') and pending_event.kwargs.get('mutex', None) == kwargs.get('mutex'):
+            logging.debug("Dropped event: %s", kwargs.get('mutex'))
+            return
     event_list.enterabs(t, 0, callback, args)
 
 

idlebot.py

@@ -1,25 +1,31 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 import logging
-import time
 import sys
+import time
+import _ssl
+
+from sleekxmpp import ClientXMPP
+
 import config
 import events
 from common import VERSION
 
-from sleekxmpp import ClientXMPP
-
 
 class IdleBot(ClientXMPP):
     def __init__(self, jid, password, rooms, nick):
         ClientXMPP.__init__(self, jid, password)
 
+        self.ssl_version = _ssl.PROTOCOL_TLSv1_2
+
         self.rooms = rooms
         self.nick = nick
 
         self.add_event_handler('session_start', self.session_start)
         self.add_event_handler('groupchat_message', self.muc_message)
         self.add_event_handler('disconnected', self.disconnected)
+        self.add_event_handler('presence_error', self.disconnected)
+        self.add_event_handler('session_end', self.disconnected)
         self.priority = 0
         self.status = None
         self.show = None
@@ -29,7 +35,8 @@ class IdleBot(ClientXMPP):
             self.add_event_handler('muc::%s::got_offline' % room, self.muc_offline)
 
     def disconnected(self, _):
-        exit(0)
+        self.logger.warn("Disconnected! dbg: {}".format(str(_)))
+        self.disconnect(wait=True)
 
     def session_start(self, _):
         self.get_roster()
@@ -81,8 +88,7 @@ class IdleBot(ClientXMPP):
         """
         disconnect and exit
         """
-        self.disconnect()
+        self.disconnect(wait=True)
-        sys.exit(1)
 
 
 def start(botclass, active=False):
@@ -105,12 +111,14 @@ def start(botclass, active=False):
 
     bot.connect()
     bot.register_plugin('xep_0045')
+    bot.register_plugin('xep_0199', {'keepalive': True})
+    bot.register_plugin('xep_0308')
     bot.process()
 
     config.runtimeconf_set('start_time', -time.time())
 
     if active:
-        import plugins
+        pass
 
     events.event_loop.start()
 

plugins/cake.py

@@ -6,28 +6,48 @@ from plugin_system import pluginfunction, ptypes
 from rate_limit import RATE_FUN, RATE_GLOBAL
 
 
+def give_item(user, item_name, search_word=None):
+    if not search_word:
+        search_word = item_name
+    return {'msg': '{} for {}: {}'.format(item_name, user, giphy(search_word, 'dc6zaTOxFJmzC'))}
+
+
+def cake_excuse(user):
+    return {
+        'msg': '{}: {}'.format(user, random.choice(cakes))
+    }
+
+
 @pluginfunction('cake', 'displays a cake ASCII art', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
 def command_cake(argv, **args):
     if {'please', 'bitte'}.intersection(set(argv)):
-        return {
+        return give_item(args['reply_user'], 'cake')
-            'msg': 'cake for {}: {}'.format(args['reply_user'], giphy('cake', 'dc6zaTOxFJmzC'))
+    else:
-        }
+        return cake_excuse(args['reply_user'])
-
-    return {
-        'msg': args['reply_user'] + ': %s' % random.choice(cakes)
-    }
 
 
 @pluginfunction('keks', 'keks!', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
 def command_cookie(argv, **args):
     if {'please', 'bitte'}.intersection(set(argv)):
-        return {
+        return give_item(args['reply_user'], 'keks', 'cookie')
-            'msg': 'keks für {}: {}'.format(args['reply_user'], giphy('cookie', 'dc6zaTOxFJmzC'))
+    else:
-        }
+        return cake_excuse(args['reply_user'])
 
-    return {
+
-        'msg': args['reply_user'] + ': %s' % random.choice(cakes)
+@pluginfunction('schnitzel', 'schnitzel!', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
-    }
+def command_schnitzel(argv, **args):
+    if {'please', 'bitte'}.intersection(set(argv)):
+        return give_item(args['reply_user'], 'schnitzel')
+    else:
+        return cake_excuse(args['reply_user'])
+
+
+@pluginfunction('kaffee', 'kaffee!', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
+def command_coffee(argv, **args):
+    if {'please', 'bitte'}.intersection(set(argv)):
+        return give_item(args['reply_user'], 'kaffee', 'coffee')
+    else:
+        return cake_excuse(args['reply_user'])
 
 
 cakes = [
@@ -46,4 +66,3 @@ cakes = [
     "I'm going to kill you, and all the cake is gone.",
     "Who's gonna make the cake when I'm gone? You?"
 ]
-

plugins/commands.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import logging
 
+import re
 import events
 import json
 import random
@@ -13,8 +14,10 @@ from lxml import etree
 
 import config
 from common import VERSION
+from plugins.searx import searx
 from rate_limit import RATE_FUN, RATE_GLOBAL, RATE_INTERACTIVE, RATE_NO_SILENCE, RATE_NO_LIMIT
 from plugin_system import pluginfunction, ptypes, plugin_storage, plugin_enabled_get, plugin_enabled_set
+
 log = logging.getLogger(__name__)
 
 
@@ -92,7 +95,6 @@ def command_plugin_activation(argv, **args):
 
 @pluginfunction('list', 'list plugin and parser status', ptypes.COMMAND)
 def command_list(argv, **args):
-
     log.info('list plugin called')
 
     if 'enabled' in argv and 'disabled' in argv:
@@ -279,18 +281,187 @@ def command_dice(argv, **args):
     }
 
 
+@pluginfunction('xchoose', 'chooses randomly between nested option groups', ptypes.COMMAND, ratelimit_class=RATE_INTERACTIVE)
+def command_xchoose(argv, **args):
+
+    class ChooseTree():
+        def __init__(self, item=None):
+            self.item = item
+            self.tree = None
+            self.closed = False
+
+            # opening our root node
+            if self.item is None:
+                self.open()
+
+        def open(self):
+            if self.tree is None:
+                self.tree = []
+            elif self.closed:
+                raise Exception("cannot re-open group for item '%s'" % (self.item))
+
+        def close(self):
+            if self.tree is None:
+                raise Exception("close on unopened bracket")
+            elif len(self.tree) == 0:
+                raise Exception("item '%s' has a group without sub options" % (self.item))
+            else:
+                self.closed = True
+
+        def last(self):
+            return self.tree[-1]
+
+        def choose(self):
+            if self.item:
+                yield self.item
+            
+            if self.tree:
+                sel = random.choice(self.tree)
+                for sub in sel.choose():
+                    yield sub
+
+        def add(self, item):
+            self.tree.append( ChooseTree(item) )
+        
+    # because of error handling we're nesting this function here
+    def xchoose(line):
+        item = ''
+        quote = None
+        choose_tree = ChooseTree()
+        choose_stack = [ choose_tree ]
+        bracket_stack = []
+
+        for pos, c in enumerate(line, 1):
+            try:
+                if quote:
+                    if c == quote:
+                        quote = None
+                    else:
+                        item += c
+
+                elif c == ' ':
+                    if item:
+                        choose_stack[-1].add(item)
+                        item = ''
+
+                elif c in ('(', '[', '{', '<'):
+                    if item:
+                        choose_stack[-1].add(item)
+                        item = ''
+
+                    try:
+                        last = choose_stack[-1].last()
+                        last.open()
+                        choose_stack.append(last)
+                        bracket_stack.append(c)
+                    except IndexError:
+                        raise Exception("cannot open group without preceding option")
+
+                elif c in (')', ']', '}', '>'):
+                    if not bracket_stack:
+                        raise Exception("missing leading bracket for '%s'" % (c))
+
+                    opening_bracket = bracket_stack.pop(-1)
+                    wanted_closing_bracket = { '(':')', '[':']', '{':'}', '<':'>' }[opening_bracket]
+                    if c != wanted_closing_bracket:
+                        raise Exception("bracket mismatch, wanted bracket '%s' but got '%s'" % (
+                            wanted_closing_bracket, c))
+
+                    if item:
+                        choose_stack[-1].add(item)
+                        item = ''
+
+                    choose_stack[-1].close()
+                    choose_stack.pop(-1)
+
+                elif c in ('"', "'"):
+                    quote = c
+
+                else:
+                    item += c
+
+            except Exception as e:
+                raise Exception("%s (at pos %d)" % (e, pos))
+
+        if bracket_stack:
+            raise Exception("missing closing bracket for '%s'" % (bracket_stack[-1]))
+
+        if quote:
+            raise Exception("missing closing quote (%s)" % (quote))
+
+        if item:
+            choose_stack[-1].add(item)
+
+        return ' '.join(choose_tree.choose())
+
+
+    # start of command_xchoose
+    line = re.sub('.*xchoose *', '', args['data'])
+    if not line:
+        return {
+            'msg': '%s: %s' % (args['reply_user'], 'missing options')
+        }
+    try:
+        return {
+            'msg': '%s: %s' % (args['reply_user'], xchoose(line))
+        }
+    except Exception as e:
+        return {
+            'msg': '%s: %s' % (args['reply_user'], str(e))
+        }
+
+
 @pluginfunction('choose', 'chooses randomly between arguments', ptypes.COMMAND, ratelimit_class=RATE_INTERACTIVE)
 def command_choose(argv, **args):
     alternatives = argv
+    binary = (
+        (('Yes.', 'Yeah!', 'Ok!', 'Aye!', 'Great!'), 4),
+        (('No.', 'Naah..', 'Meh.', 'Nay.', 'You stupid?'), 4),
+        (('Maybe.', 'Dunno.', 'I don\'t care.'), 2)
+    )
+
+    def weighted_choice(choices):
+        total = sum(w for c, w in choices)
+        r = random.uniform(0, total)
+        upto = 0
+        for c, w in choices:
+            if upto + w >= r:
+                return c
+            upto += w
+
+    # single or no choice
     if len(alternatives) < 2:
         return {
-            'msg': '{}: {}.'.format(args['reply_user'], random.choice(['Yes', 'No']))
+            'msg': '{}: {}'.format(args['reply_user'], random.choice(weighted_choice(binary)))
+        }
+    elif 'choose' not in alternatives:
+        choice = random.choice(alternatives)
+        return {
+            'msg': '%s: I prefer %s!' % (args['reply_user'], choice)
         }
 
-    choice = random.choice(alternatives)
+    def choose_between(options):
-    log.info('sent random choice')
+        responses = []
+        current_choices = []
+
+        for item in options:
+            if item == 'choose':
+                if len(current_choices) < 2:
+                    responses.append(random.choice(weighted_choice(binary)))
+                else:
+                    responses.append(random.choice(current_choices))
+                current_choices = []
+            else:
+                current_choices.append(item)
+        if len(current_choices) < 2:
+            responses.append(random.choice(weighted_choice(binary)))
+        else:
+            responses.append(random.choice(current_choices))
+        return responses
+
+    log.info('sent multiple random choices')
     return {
-        'msg': '%s: I prefer %s!' % (args['reply_user'], choice)
+        'msg': '%s: My choices are: %s!' % (args['reply_user'], ', '.join(choose_between(alternatives)))
     }
 
 
@@ -324,7 +495,8 @@ def command_teatimer(argv, **args):
         ),
         'event': {
             'time': ready,
-            'msg': (args['reply_user'] + ': Your tea is ready!')
+            'msg': (args['reply_user'] + ': Your tea is ready!'),
+            'mutex': 'teatimer_{}'.format(args['reply_user'])
         }
     }
 
@@ -564,8 +736,8 @@ def command_dsa_watcher(argv=None, **_):
         for dsa_about in reversed(dsa_about_list):
             dsa_id = get_id_from_about_string(dsa_about)
             title = xmldoc.xpath(
-                    '//purl:item[@rdf:about="{}"]/purl:title/text()'.format(dsa_about),
+                '//purl:item[@rdf:about="{}"]/purl:title/text()'.format(dsa_about),
-                    namespaces=nsmap
+                namespaces=nsmap
             )[0]
             if after and dsa_id <= after:
                 continue
@@ -590,9 +762,11 @@ def command_dsa_watcher(argv=None, **_):
     msg = 'next crawl set to %s' % time.strftime('%Y-%m-%d %H:%M', time.localtime(crawl_at))
     out.append(msg)
     return {
+        # 'msg': out,
         'event': {
             'time': crawl_at,
-            'command': (command_dsa_watcher, ([],))
+            'command': (command_dsa_watcher, ([],)),
+            'mutex': 'dsa'
         }
     }
 
@@ -621,8 +795,9 @@ def remove_from_botlist(argv, **args):
         return False
 
 
-@pluginfunction("add-to-botlist", "add a user to the botlist", ptypes.COMMAND)
+@pluginfunction("add-to-botlist", "add a user to the botlist", ptypes.COMMAND, enabled=False)
 def add_to_botlist(argv, **args):
+    return {'msg': 'feature disabled until channel separation'}
     if not argv:
         return {'msg': "wrong number of arguments!"}
     suspect = argv[0]
@@ -707,37 +882,8 @@ def reload_runtimeconfig(argv, **args):
         return {'msg': 'done'}
 
 
-@pluginfunction('snitch', "tell on a spammy user", ptypes.COMMAND)
+@pluginfunction('ducksearch', 'search the web (using duckduckgo)', ptypes.COMMAND)
-def ignore_user(argv, **args):
+def search_the_duck(argv, **args):
-    if not argv:
-        return {'msg': 'syntax: "{}: snitch username"'.format(config.conf_get("bot_nickname"))}
-
-    then = time.time() + 15 * 60
-    spammer = argv[0]
-
-    if spammer == config.conf_get("bot_owner"):
-        return {
-            'msg': 'My owner does not spam, he is just very informative.'
-        }
-
-    if spammer not in config.runtime_config_store['spammers']:
-        config.runtime_config_store['spammers'].append(spammer)
-
-    def unblock_user(user):
-        if user not in config.runtime_config_store['spammers']:
-            config.runtime_config_store['spammers'].append(user)
-
-    return {
-        'msg': 'user reported and ignored till {}'.format(time.strftime('%H:%M', time.localtime(then))),
-        'event': {
-            'time': then,
-            'command': (unblock_user, ([spammer],))
-        }
-    }
-
-
-@pluginfunction('search', 'search the web (using duckduckgo)', ptypes.COMMAND)
-def search_the_web(argv, **args):
     url = 'http://api.duckduckgo.com/'
     params = dict(
         q=' '.join(argv),
@@ -768,6 +914,24 @@ def search_the_web(argv, **args):
         return {'msg': 'Sorry, no results.'}
 
 
+@pluginfunction('search', 'search the web (using searx)', ptypes.COMMAND)
+def search_the_web(argv, **args):
+    result = searx(' '.join(argv))
+    if not result:
+        return {'msg': 'Sorry, no results.'}
+    else:
+        abstract, url = result
+
+    if len(abstract) > 150:
+        suffix = '…'
+    else:
+        suffix = ''
+    return {
+        'msg': '{}{} ({})'.format(abstract[:150], suffix, url)
+    }
+    pass
+
+
 @pluginfunction('raise', 'only for debugging', ptypes.COMMAND)
 def raise_an_error(argv, **args):
     if args['reply_user'] == config.conf_get("bot_owner"):
@@ -776,9 +940,10 @@ def raise_an_error(argv, **args):
 
 @pluginfunction('repeat', 'repeat the last message', ptypes.COMMAND)
 def repeat_message(argv, **args):
-    return {
+    if args['stack']:
-        'msg': args['stack'][-1]['body']
+        return {
-    }
+            'msg': args['stack'][-1]['body']
+        }
 
 
 @pluginfunction('isdown', 'check if a website is reachable', ptypes.COMMAND)

plugins/comment_joins.py

@@ -17,7 +17,7 @@ comment_joins_strings = [
 def comment_joins(**args):
     # max elapsed time between the latest and the N latest join
     timespan = 120
-    max_joins = config.runtime_config_store
+    max_joins = 6
 
     current_timestamp = int(time.time())
 

plugins/morse.py

@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+import logging
+import re
+
+from plugin_system import pluginfunction, ptypes
+from rate_limit import RATE_FUN, RATE_GLOBAL
+
+log = logging.getLogger(__name__)
+
+# copy from https://de.wikipedia.org/wiki/Morsezeichen
+raw_wiki_copy = """
+A· −
+B− · · ·
+C− · − ·
+D− · ·
+E·
+F· · − ·
+G− − ·
+H· · · ·
+I· ·
+J· − − −
+K− · −
+L· − · ·
+M− −
+N− ·
+O− − −
+P· − − ·
+Q− − · −
+R· − ·
+S· · ·
+T−
+U· · −
+V· · · −
+W· − −
+X− · · −
+Y− · − −
+Z− − · ·
+1· − − − −
+2· · − − −
+3· · · − −
+4· · · · −
+5· · · · ·
+6− · · · ·
+7− − · · ·
+8− − − · ·
+9− − − − ·
+0− − − − −
+"""
+
+
+# machen dictionary aus wikipaste
+def wiki_paste_to_morse_dict(wikicopy):
+    wikicopy = wikicopy.replace(' ', '')
+    morse_dict = {l[0]: l[1:] for l in wikicopy.splitlines() if l}
+    return morse_dict
+
+
+ascii_morse = wiki_paste_to_morse_dict(raw_wiki_copy)
+morse_ascii = {v: k for k, v in ascii_morse.items()}
+
+
+# return a dictionary of possible morse-chars as key
+# and their count as value
+def possible_morse_chars(string):
+    """
+    returns dit,dah or None
+    """
+    stats = {}
+
+    for c in re.sub("[\w\d ]", '', string):
+        try:
+            stats[c] += 1
+        except KeyError:
+            stats[c] = 1
+
+    return stats
+
+
+# return morse-encoded string
+def morse_encode(string, dot='·', dash='−', sep=' ', ignore_unknown=False):
+    morse_codes = []
+
+    for char in string.upper():
+        try:
+            morse_codes.append(ascii_morse[char].replace('·', dot).replace('−', dash))
+        except KeyError:
+            if not ignore_unknown:
+                morse_codes.append(char)
+
+    return sep.join(morse_codes)
+
+
+# return morse-decoded string with number of errors as tuple
+# -> (decoded string, num errors)
+def morse_decode(string, dot=None, dash=None):
+    """
+    decode a "morse string" to ascii text
+    uses \s{2,} as word separator
+    """
+    # dot and dash given, just decode
+    if dot and dash:
+        errors = 0
+
+        words = []
+        # drawback: does not allow single characters.
+        for match in re.finditer('([{dit}{dah}]+((?:\\s)[{dit}{dah}]+)+|\w+)'.format(dit=dot, dah=dash), string):
+            word = match.group()
+            log.debug("morse word: ", word)
+            if any([dot in word, dash in word]):
+                w = []
+                for morse_character in word.split():
+                    try:
+                        character = morse_ascii[morse_character.replace(dot, '·').replace(dash, '−')]
+                        print("Converted \t{} \tto {}".format(morse_character, character))
+                    except KeyError:
+                        character = morse_character
+                        errors += 1
+                    w.append(character)
+                words.append(''.join(w))
+            # words.append(''.join([morse_ascii[x.replace(dot, '·').replace(dash, '−')] for x in word.split()]))
+            else:
+                words.append(word)
+        return ' '.join(words), errors
+
+    # dot/dash given, search for dash/dot
+    else:
+        if not dash:
+            dash_stats = {x: string.count(x) for x in '-−_'}
+            dash = max(dash_stats, key=dash_stats.get)
+        if not dot:
+            dot_stats = {x: string.count(x) for x in '.·*'}
+            dot = max(dot_stats, key=dot_stats.get)
+
+        return morse_decode(string, dot=dot, dash=dash)
+
+
+@pluginfunction('morse-encode', 'encode string to morse', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
+def command_morse_encode(argv, **args):
+    if not argv:
+        return {
+            'msg': args['reply_user'] + "usage: morse-encode <string>"
+        }
+
+    if len(argv) == 1 and argv[0] == 'that':
+        message_stack = args['stack']
+        if not message_stack[-1]:
+            return
+        message = message_stack[-1]['body']
+    else:
+        message = ' '.join(argv)
+
+    return {
+        'msg': args['reply_user'] + ': %s' % morse_encode(message)
+    }
+
+
+@pluginfunction('morse-decode', 'decode morse encoded string', ptypes.COMMAND, ratelimit_class=RATE_FUN | RATE_GLOBAL)
+def command_morse_decode(argv, **args):
+    if not argv:
+        return {
+            'msg': args['reply_user'] + "usage: morse-decode <string>"
+        }
+
+    if len(argv) == 1 and argv[0] == 'that':
+        message_stack = args['stack']
+        if not message_stack[-1]:
+            return
+        message = message_stack[-1]['body']
+    else:
+        message = ' '.join(argv)
+
+    decoded, errors = morse_decode(message, dot='·', dash='-')
+
+    return {
+        'msg': args['reply_user'] + ': %s (%d errors)' % (decoded, errors)
+    }

plugins/parsers.py

@@ -39,6 +39,14 @@ def parse_mental_ill(**args):
         }
 
 
+@pluginfunction('woof', '*puts sunglasses on*', ptypes.PARSE, ratelimit_class=RATE_NO_SILENCE | RATE_GLOBAL)
+def command_woof(**args):
+    if 'who let the bots out' in args['data']:
+        return {
+            'msg': 'beeep! beep! beep! beep! beep!'
+        }
+
+
 @pluginfunction('debbug', 'parse Debian bug numbers', ptypes.PARSE, ratelimit_class=RATE_NO_SILENCE | RATE_GLOBAL)
 def parse_debbug(**args):
     bugs = re.findall(r'#(\d{4,})', args['data'])
@@ -50,14 +58,11 @@ def parse_debbug(**args):
         log.info('detected Debian bug #%s' % b)
 
         url = 'https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=%s' % b
-        status, title = extract_title(url)
 
-        if 0 == status:
+        title = extract_title(url)
+
+        if title:
             out.append('Debian Bug: %s: %s' % (title, url))
-        elif 3 == status:
-            out.append('error for #%s: %s' % (b, title))
-        else:
-            log.info('unknown status %d' % status)
 
     return {
         'msg': out
@@ -122,32 +127,6 @@ def parse_slash_me(**args):
         }
 
 
-@pluginfunction("recognize_bots", "got ya", ptypes.PARSE)
-def recognize_bots(**args):
-    unique_standard_phrases = (
-        'independent bot and have nothing to do with other artificial intelligence systems',
-        'new Debian Security Announce',
-        'I\'m a bot (highlight me',
-    )
-
-    def _add_to_list(username, message):
-        if username not in config.runtime_config_store['other_bots']:
-            config.runtime_config_store['other_bots'].append(username)
-            config.runtimeconf_persist()
-            log.info("Adding {} to the list of bots (now {})".format(username, config.runtime_config_store['other_bots']))
-            return {
-                'event': {
-                    'time': time.time() + 3,
-                    'msg': message
-                }
-            }
-
-    if any([phrase in args['data'] for phrase in unique_standard_phrases]):
-        return _add_to_list(args['reply_user'], 'Making notes...')
-    elif 'I\'ll be back' in args['data']:
-        return _add_to_list(args['reply_user'], 'Hey there, buddy!')
-
-
 @pluginfunction('resolve-url-title', 'extract titles from urls', ptypes.PARSE, ratelimit_class=RATE_URL)
 def resolve_url_title(**args):
     user = args['reply_user']
@@ -163,7 +142,7 @@ def resolve_url_title(**args):
     url_blacklist = config.runtime_config_store['url_blacklist'].values()
 
     out = []
-    for url in result:
+    for url in result[:10]:
         if any([re.match(b, url) for b in url_blacklist]):
             log.info('url blacklist match for ' + url)
             break
@@ -184,3 +163,10 @@ def resolve_url_title(**args):
         'msg': out
     }
 
+
+@pluginfunction('doctor', 'parse doctor', ptypes.PARSE, ratelimit_class=RATE_FUN | RATE_GLOBAL)
+def parse_doctor(**args):
+    if 'doctor' in args['data'].lower() or 'doktor' in args['data'].lower():
+        return {
+            'msg': 'ELIMINIEREN! ELIMINIEREN!'
+        }

plugins/searx.py

@@ -0,0 +1,99 @@
+import logging
+import time
+from functools import wraps
+import json
+import requests
+from lxml import etree, html
+from requests import HTTPError
+
+search_list = []
+
+if not hasattr(json, 'JSONDecodeError'):
+    json.JSONDecodeError = ValueError
+
+class RateLimitingError(HTTPError):
+    pass
+
+
+def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
+    """Retry calling the decorated function using an exponential backoff.
+
+    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+
+    :param ExceptionToCheck: the exception to check. may be a tuple of
+        exceptions to check
+    :type ExceptionToCheck: Exception or tuple
+    :param tries: number of times to try (not retry) before giving up
+    :type tries: int
+    :param delay: initial delay between retries in seconds
+    :type delay: int
+    :param backoff: backoff multiplier e.g. value of 2 will double the delay
+        each retry
+    :type backoff: int
+    :param logger: logger to use. If None, print
+    :type logger: logging.Logger instance
+    """
+
+    def deco_retry(f):
+
+        @wraps(f)
+        def f_retry(*args, **kwargs):
+            mtries, mdelay = tries, delay
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except ExceptionToCheck as e:
+                    msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
+                    if logger:
+                        logger.warning(msg)
+                    else:
+                        print(msg)
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+
+        return f_retry  # true decorator
+
+    return deco_retry
+
+
+def fetch_all_searx_engines():
+    # error handling is for pussies
+    tree = etree.XML(
+        requests.get("http://stats.searx.oe5tpo.com").content,
+        parser=html.HTMLParser()
+    )
+    searxes = [str(x) for x in tree.xpath('//span[text()[contains(.,"200 - OK")]]/../..//a/text()')]
+
+    return searxes
+
+
+@retry(ExceptionToCheck=(RateLimitingError, json.JSONDecodeError))
+def searx(text):
+    global search_list
+    if not search_list:
+        search_list = fetch_all_searx_engines()
+    logger = logging.getLogger(__name__)
+
+    url = search_list[0]
+    logger.info('Currently feeding from {} (of {} in stock)'.format(url, len(search_list)))
+    response = requests.get(url, params={
+        'q': text,
+        'format': 'json',
+        'lang': 'de'
+    })
+    if response.status_code == 429:
+        search_list.pop(0)
+        raise RateLimitingError(response=response, request=response.request)
+    try:
+        response = response.json()
+    except json.JSONDecodeError:
+        # "maintenance" they say...
+        search_list.pop(0)
+        raise
+
+    if not response['results']:
+        return
+    return [(r.get('content', ''), r['url']) for r in response['results']][0]

plugins/translate.py

@@ -22,6 +22,8 @@ def translate(argv, **args):
         if not api_key:
             return
         message_stack = args['stack']
+        if not message_stack[-1]:
+            return
         last_message = message_stack[-1]['body']
         data = {
             'q': last_message,

tox.ini

@@ -0,0 +1,32 @@
+# defaults to tests
+# run with tox -e urlbot,  tox -e idlebot, etc
+[tox]
+envlist = test
+# we have no setup.py
+skipsdist = true
+
+[testenv]
+
+envdir = {toxinidir}/.env
+
+deps=nose
+     fasteners
+     sleekxmpp
+     configobj
+     requests
+     lxml
+     dnspython3
+     pyasn1
+     pyasn1-modules
+
+commands=
+  test: nosetests []  # substitute with tox' positional arguments
+  idlebot: python idlebot.py []
+  urlbot: python urlbot.py []
+  sh: sh []
+  bash: bash []
+  zsh: zsh []
+
+whitelist_externals = zsh 
+                      bash 
+		      sh

urlbot.py

@@ -7,10 +7,14 @@ import re
 import shlex
 import sys
 import time
+from collections import deque
+
 from lxml import etree
 
 import requests
+from sleekxmpp.plugins import PluginNotFound
 
+import plugins  # force initialization
 from plugin_system import plugin_storage, ptypes, plugin_enabled_get
 from rate_limit import rate_limit_classes, RATE_GLOBAL, RATE_CHAT, RATE_EVENT, rate_limit
 
@@ -34,7 +38,7 @@ class UrlBot(IdleBot):
 
         self.hist_ts = {p: [] for p in rate_limit_classes}
         self.hist_flag = {p: True for p in rate_limit_classes}
-        self.message_stack = []
+        self.message_stack = {str(room): deque(maxlen=5) for room in self.rooms}
 
         self.add_event_handler('message', self.message)
         self.priority = 100
@@ -42,6 +46,9 @@ class UrlBot(IdleBot):
         for room in self.rooms:
             self.add_event_handler('muc::%s::got_online' % room, self.muc_online)
 
+        dsa_plugin = list(filter(lambda x: x.plugin_name == 'dsa-watcher', plugin_storage[ptypes.COMMAND]))[0]
+        self._run_action(dsa_plugin(), dsa_plugin, None)
+
     def muc_message(self, msg_obj):
         """
         Handle muc messages, return if irrelevant content or die by hangup.
@@ -81,7 +88,7 @@ class UrlBot(IdleBot):
         request_counter = int(config.runtimeconf_get('request_counter'))
         config.runtimeconf_set('request_counter', request_counter + 1)
 
-        if str is not type(message):
+        if not isinstance(message, str):
             message = '\n'.join(message)
 
         def cached(function, ttl=60):
@@ -107,7 +114,10 @@ class UrlBot(IdleBot):
             other_bots = config.runtimeconf_get("other_bots")
             if not other_bots:
                 return False
-            users = self.plugin['xep_0045'].getRoster(room)
+            try:
+                users = self.plugin['xep_0045'].getRoster(room)
+            except PluginNotFound:
+                users = []
             return set(users).intersection(set(other_bots))
 
         def _prevent_panic(message, room):
@@ -196,9 +206,8 @@ class UrlBot(IdleBot):
         except Exception as e:
             self.logger.exception(e)
         finally:
-            if len(self.message_stack) > 4:
+            if msg_obj['from'].bare in self.rooms:
-                self.message_stack.pop(0)
+                self.message_stack[msg_obj['from'].bare].append(msg_obj)
-            self.message_stack.append(msg_obj)
 
     def handle_muc_online(self, msg_obj):
         """
@@ -268,7 +277,7 @@ class UrlBot(IdleBot):
                 reply_user=reply_user,
                 msg_obj=msg_obj,
                 argv=words[2:] if len(words) > 1 else [],
-                stack=self.message_stack
+                stack=self.message_stack.get(msg_obj['from'].bare, [])
             )
 
             if ret:
@@ -291,7 +300,7 @@ class UrlBot(IdleBot):
             if not plugin_enabled_get(plugin):
                 continue
 
-            ret = plugin(reply_user=reply_user, data=data)
+            ret = plugin(reply_user=reply_user, data=data, sender=msg_obj['from'])
 
             if ret:
                 self._run_action(ret, plugin, msg_obj)
@@ -348,7 +357,8 @@ class UrlBot(IdleBot):
                         args=command[1],
                         action_runner=self._run_action,
                         plugin=plugin,
-                        msg_obj=msg_obj
+                        msg_obj=msg_obj,
+                        mutex=event.get('mutex')
                     )
 
         if 'msg' in action and rate_limit(RATE_CHAT | plugin.ratelimit_class):