implement new version of the dsa watcher

2017-09-06 15:25:38 +02:00 · 2016-01-04 14:27:02 +01:00
parent d1bf544ff9
commit e1f451c094
5 changed files with 101 additions and 97 deletions
--- a/local_config.ini.spec
+++ b/local_config.ini.spec
@@ -24,7 +24,6 @@ moin-disabled-user = string_list(default=list())
 tea_steep_time = integer(default=220)

 image_preview = boolean(default=true)
-dsa_watcher_interval = integer(default=900)
 loglevel = option('ERROR', WARN', 'INFO', 'DEBUG', default='INFO')

 debug_mode = boolean(default=false)
--- a/persistent_config.ini.spec
+++ b/persistent_config.ini.spec
@@ -6,7 +6,9 @@ start_time = integer(default=0)
 [plugins]
  [[info]]
    enabled = boolean(default=true)
-  last_dsa = integer(default=0)  # TODO broken
+  [[dsa-watcher]]
+    last_dsa = integer(default=0)
+    interval = integer(default=900)

 [user_pref]

--- a/plugins/init.py
+++ b/plugins/init.py
@@ -26,7 +26,6 @@ def plugin_enabled_set(plugin, enabled):
        log.warn("couldn't get exclusive lock")

    config.conf_set('persistent_locked', True)
-    # blob = conf_load()

    if plugin.plugin_name not in config.runtime_config_store['plugins']:
        config.runtime_config_store['plugins'][plugin.plugin_name] = {}
@@ -36,6 +35,25 @@ def plugin_enabled_set(plugin, enabled):
    config.conf_set('persistent_locked', False)


+def register_active_event(t, callback, args, action_runner, plugin, msg_obj):
+    """
+    Execute a callback at a given time and react on the output
+
+    :param t: when to execute the job
+    :param callback: the function to execute
+    :param args: parameters for said function
+    :param action_runner: bots action dict parser
+    :param plugin: pass-through object for action parser
+    :param msg_obj: pass-through object for action parser
+    :return:
+    """
+    def func(func_args):
+        action = callback(*func_args)
+        if action:
+            action_runner(action=action, plugin=plugin, msg_obj=msg_obj)
+    joblist.append((t, func, args))
+
+
 def register_event(t, callback, args):
    joblist.append((t, callback, args))

--- a/plugins/commands.py
+++ b/plugins/commands.py
@@ -6,13 +6,14 @@ import traceback
 import unicodedata

 import requests
+from lxml import etree

 import config
 from common import (
    VERSION, RATE_FUN, RATE_GLOBAL, RATE_INTERACTIVE, RATE_NO_LIMIT,
    giphy, pluginfunction,
-    ptypes_COMMAND
-)
+    ptypes_COMMAND,
+    RATE_NO_SILENCE)
 from string_constants import cakes, excuses, moin_strings_hi, moin_strings_bye

 log = logging.getLogger(__name__)
@@ -28,7 +29,6 @@ def command_version(argv, **args):

@pluginfunction('uptime', 'prints uptime', ptypes_COMMAND)
 def command_uptime(argv, **args):
-
    u = int(config.runtimeconf_get('start_time') + time.time())
    plural_uptime = 's'
    plural_request = 's'
@@ -47,7 +47,6 @@ def command_uptime(argv, **args):

@pluginfunction('info', 'prints info message', ptypes_COMMAND)
 def command_info(argv, **args):
-
    log.info('sent long info')
    return {
        'msg': args['reply_user'] + (
@@ -61,7 +60,6 @@ def command_info(argv, **args):

@pluginfunction('ping', 'sends pong', ptypes_COMMAND, ratelimit_class=RATE_INTERACTIVE)
 def command_ping(argv, **args):
-
    rnd = random.randint(0, 3)  # 1:4
    if 0 == rnd:
        msg = args['reply_user'] + ''': peng (You're dead now.)'''
@@ -338,7 +336,6 @@ def usersetting_get(argv, args):

@pluginfunction('set', 'modify a user setting', ptypes_COMMAND, ratelimit_class=RATE_NO_LIMIT)
 def command_usersetting(argv, **args):
-
    settings = ['spoiler']
    arg_user = args['reply_user']
    arg_key = argv[0] if len(argv) > 0 else None
@@ -529,90 +526,67 @@ def command_show_recordlist(argv, **args):
            )
    }

-# TODO: disabled until rewrite
-# @pluginfunction('dsa-watcher', 'automatically crawls for newly published Debian Security Announces', ptypes_COMMAND,
-#                 ratelimit_class=RATE_NO_SILENCE)
-# def command_dsa_watcher(argv, **_):
-#     """
-#     TODO: rewrite so that a last_dsa_date is used instead, then all DSAs since then printed and the date set to now()
-#     """
-#
-#     if 2 != len(argv):
-#         msg = 'wrong number of arguments'
-#         log.warn(msg)
-#         return {'msg': msg}
-#
-#     if 'crawl' == argv[1]:
-#         out = []
-#         # TODO: this is broken... the default should neither be part of the code,
-#         # but rather be determined at runtime (like "latest" or similar)
-#         dsa = config.runtime_config_store.deepget('plugins.last_dsa', 1000)
-#
-#         url = 'https://security-tracker.debian.org/tracker/DSA-%d-1' % dsa
-#
-#         try:
-#             request = urllib.request.Request(url)
-#             request.add_header('User-Agent', USER_AGENT)
-#             response = urllib.request.urlopen(request)
-#             html_text = response.read(BUFSIZ)  # ignore more than BUFSIZ
-#         except Exception as e:
-#             err = e
-#             if '404' not in str(err):
-#                 msg = 'error for %s: %s' % (url, err)
-#                 log.warn(msg)
-#                 out.append(msg)
-#         else:
-#             if str != type(html_text):
-#                 html_text = str(html_text)
-#
-#             result = re.match(r'.*?Description</b></td><td>(.*?)</td>.*?', html_text, re.S | re.M | re.IGNORECASE)
-#
-#             package = 'error extracting package name'
-#             if result:
-#                 package = result.groups()[0]
-#
-#             if config.get('persistent_locked'):
-#                 msg = "couldn't get exclusive lock"
-#                 log.warn(msg)
-#                 out.append(msg)
-#             else:
-#                 config.set('persistent_locked', True)
-#                 blob = conf_load()
-#
-#                 if 'plugin_conf' not in blob:
-#                     blob['plugin_conf'] = {}
-#
-#                 if 'last_dsa' not in blob['plugin_conf']:
-#                     blob['plugin_conf']['last_dsa'] = 3308  # FIXME: fixed value
-#
-#                 blob['plugin_conf']['last_dsa'] += 1
-#
-#                 runtimeconf_save(blob)
-#                 config.set('persistent_locked', False)
-#
-#             msg = (
-#                 'new Debian Security Announce found (%s): %s' % (str(package).replace(' - security update', ''), url))
-#             out.append(msg)
-#
-#             log.info('no dsa for %d, trying again...' % dsa)
-#         # that's good, no error, just 404 -> DSA not released yet
-#
-#         crawl_at = time.time() + config.get('dsa_watcher_interval')
-#         # register_event(crawl_at, command_dsa_watcher, (['dsa-watcher', 'crawl'],))
-#
-#         msg = 'next crawl set to %s' % time.strftime('%Y-%m-%d %H:%M', time.localtime(crawl_at))
-#         out.append(msg)
-#         return {
-#             'msg': out,
-#             'event': {
-#                 'time': crawl_at,
-#                 'command': (command_dsa_watcher, (['dsa-watcher', 'crawl'],))
-#             }
-#         }
-#     else:
-#         msg = 'wrong argument'
-#         log.warn(msg)
-#         return {'msg': msg}
+
+@pluginfunction(
+    'dsa-watcher',
+    'automatically crawls for newly published Debian Security Announces', ptypes_COMMAND,
+    ratelimit_class=RATE_NO_SILENCE, enabled=True)
+def command_dsa_watcher(argv=None, **_):
+    """
+    TODO: rewrite so that a last_dsa_date is used instead,
+    then all DSAs since then printed and the date set to now()
+    :param argv:
+    :param _:
+    """
+    log.debug("Called command_dsa_watcher")
+
+    def get_id_from_about_string(about):
+        return int(about.split('/')[-1].split('-')[1])
+
+    def get_dsa_list(after):
+        """
+        Get a list of dsa items in form of id and package, retrieved from the RSS feed
+        :param after: optional integer to filter on (only DSA's after that will be returned)
+        :returns list of id, package (with DSA prefix)
+        """
+        nsmap = {
+            "purl": "http://purl.org/rss/1.0/",
+            "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+        }
+        dsa_response = requests.get("https://www.debian.org/security/dsa-long")
+        xmldoc = etree.fromstring(dsa_response.content)
+        dsa_about_list = xmldoc.xpath('//purl:item/@rdf:about', namespaces=nsmap)
+        for dsa_about in reversed(dsa_about_list):
+            dsa_id = get_id_from_about_string(dsa_about)
+            if after and dsa_id <= after:
+                continue
+            else:
+                yield dsa_id, str(dsa_about).replace(' - security update', '')
+
+    out = []
+    last_dsa = config.runtimeconf_deepget('plugins.dsa-watcher.last_dsa')
+    log.debug('Searching for DSA after ID {}'.format(last_dsa))
+    for dsa, package in get_dsa_list(after=last_dsa):
+        url = 'https://security-tracker.debian.org/tracker/DSA-%d-1' % dsa
+
+        msg = 'new Debian Security Announce found ({}): {}'.format(package, url)
+        out.append(msg)
+
+        last_dsa = dsa
+
+    config.runtime_config_store['plugins']['dsa-watcher']['last_dsa'] = last_dsa
+    config.runtimeconf_persist()
+    crawl_at = time.time() + config.runtimeconf_deepget('plugins.dsa-watcher.interval')
+
+    msg = 'next crawl set to %s' % time.strftime('%Y-%m-%d %H:%M', time.localtime(crawl_at))
+    out.append(msg)
+    return {
+        'msg': out,
+        'event': {
+            'time': crawl_at,
+            'command': (command_dsa_watcher, ([],))
+        }
+    }


@pluginfunction("provoke-bots", "search for other bots", ptypes_COMMAND)
@@ -667,7 +641,8 @@ def set_status(argv, **args):
        return {
            'presence': {
                'status': 'xa',
-                'msg': 'I\'m muted now. You can unmute me with "%s: set_status unmute"' % config.conf_get("bot_nickname")
+                'msg': 'I\'m muted now. You can unmute me with "%s: set_status unmute"' % config.conf_get(
+                    "bot_nickname")
            }
        }
    elif command == 'unmute' and args['reply_user'] == config.conf_get('bot_owner'):
@@ -720,7 +695,7 @@ def ignore_user(argv, **args):
    if not argv:
        return {'msg': 'syntax: "{}: snitch username"'.format(config.conf_get("bot_nickname"))}

-    then = time.time() + 15*60
+    then = time.time() + 15 * 60
    spammer = argv[0]

    if spammer == config.conf_get("bot_owner"):
--- a/urlbot.py
+++ b/urlbot.py
@@ -26,7 +26,8 @@ from plugins import (
    plugin_enabled_get,
    ptypes_PARSE,
    register_event,
-    else_command
+    register_active_event,
+    else_command,
 )
 import config

@@ -355,7 +356,16 @@ class UrlBot(IdleBot):
            elif 'command' in event:
                command = event["command"]
                if rate_limit(RATE_EVENT):
-                    register_event(event["time"], command[0], command[1])
+                    register_event(t=event["time"], callback=command[0], args=command[1])
+                    # kind of ugly..
+                    register_active_event(
+                        t=event['time'],
+                        callback=command[0],
+                        args=command[1],
+                        action_runner=self._run_action,
+                        plugin=plugin,
+                        msg_obj=msg_obj
+                    )

        if 'msg' in action and rate_limit(RATE_CHAT | plugin.ratelimit_class):
            self.send_reply(action['msg'], msg_obj)