implement new version of the dsa watcher

local_config.ini.spec

@@ -24,7 +24,6 @@ moin-disabled-user = string_list(default=list())
 tea_steep_time = integer(default=220)
 
 image_preview = boolean(default=true)
-dsa_watcher_interval = integer(default=900)
 loglevel = option('ERROR', WARN', 'INFO', 'DEBUG', default='INFO')
 
 debug_mode = boolean(default=false)

persistent_config.ini.spec

@@ -6,7 +6,9 @@ start_time = integer(default=0)
 [plugins]
   [[info]]
     enabled = boolean(default=true)
-  last_dsa = integer(default=0)  # TODO broken
+  [[dsa-watcher]]
+    last_dsa = integer(default=0)
+    interval = integer(default=900)
 
 [user_pref]
 

plugins/__init__.py

@@ -26,7 +26,6 @@ def plugin_enabled_set(plugin, enabled):
         log.warn("couldn't get exclusive lock")
 
     config.conf_set('persistent_locked', True)
-    # blob = conf_load()
 
     if plugin.plugin_name not in config.runtime_config_store['plugins']:
         config.runtime_config_store['plugins'][plugin.plugin_name] = {}
@@ -36,6 +35,25 @@ def plugin_enabled_set(plugin, enabled):
     config.conf_set('persistent_locked', False)
 
 
+def register_active_event(t, callback, args, action_runner, plugin, msg_obj):
+    """
+    Execute a callback at a given time and react on the output
+
+    :param t: when to execute the job
+    :param callback: the function to execute
+    :param args: parameters for said function
+    :param action_runner: bots action dict parser
+    :param plugin: pass-through object for action parser
+    :param msg_obj: pass-through object for action parser
+    :return:
+    """
+    def func(func_args):
+        action = callback(*func_args)
+        if action:
+            action_runner(action=action, plugin=plugin, msg_obj=msg_obj)
+    joblist.append((t, func, args))
+
+
 def register_event(t, callback, args):
     joblist.append((t, callback, args))
 

plugins/commands.py

@@ -6,13 +6,14 @@ import traceback
 import unicodedata
 
 import requests
+from lxml import etree
 
 import config
 from common import (
     VERSION, RATE_FUN, RATE_GLOBAL, RATE_INTERACTIVE, RATE_NO_LIMIT,
     giphy, pluginfunction,
-    ptypes_COMMAND
+    ptypes_COMMAND,
-)
+    RATE_NO_SILENCE)
 from string_constants import cakes, excuses, moin_strings_hi, moin_strings_bye
 
 log = logging.getLogger(__name__)
@@ -28,7 +29,6 @@ def command_version(argv, **args):
 
 @pluginfunction('uptime', 'prints uptime', ptypes_COMMAND)
 def command_uptime(argv, **args):
-
     u = int(config.runtimeconf_get('start_time') + time.time())
     plural_uptime = 's'
     plural_request = 's'
@@ -47,7 +47,6 @@ def command_uptime(argv, **args):
 
 @pluginfunction('info', 'prints info message', ptypes_COMMAND)
 def command_info(argv, **args):
-
     log.info('sent long info')
     return {
         'msg': args['reply_user'] + (
@@ -61,7 +60,6 @@ def command_info(argv, **args):
 
 @pluginfunction('ping', 'sends pong', ptypes_COMMAND, ratelimit_class=RATE_INTERACTIVE)
 def command_ping(argv, **args):
-
     rnd = random.randint(0, 3)  # 1:4
     if 0 == rnd:
         msg = args['reply_user'] + ''': peng (You're dead now.)'''
@@ -338,7 +336,6 @@ def usersetting_get(argv, args):
 
 @pluginfunction('set', 'modify a user setting', ptypes_COMMAND, ratelimit_class=RATE_NO_LIMIT)
 def command_usersetting(argv, **args):
-
     settings = ['spoiler']
     arg_user = args['reply_user']
     arg_key = argv[0] if len(argv) > 0 else None
@@ -529,90 +526,67 @@ def command_show_recordlist(argv, **args):
             )
     }
 
-# TODO: disabled until rewrite
+
-# @pluginfunction('dsa-watcher', 'automatically crawls for newly published Debian Security Announces', ptypes_COMMAND,
+@pluginfunction(
-#                 ratelimit_class=RATE_NO_SILENCE)
+    'dsa-watcher',
-# def command_dsa_watcher(argv, **_):
+    'automatically crawls for newly published Debian Security Announces', ptypes_COMMAND,
-#     """
+    ratelimit_class=RATE_NO_SILENCE, enabled=True)
-#     TODO: rewrite so that a last_dsa_date is used instead, then all DSAs since then printed and the date set to now()
+def command_dsa_watcher(argv=None, **_):
-#     """
+    """
-#
+    TODO: rewrite so that a last_dsa_date is used instead,
-#     if 2 != len(argv):
+    then all DSAs since then printed and the date set to now()
-#         msg = 'wrong number of arguments'
+    :param argv:
-#         log.warn(msg)
+    :param _:
-#         return {'msg': msg}
+    """
-#
+    log.debug("Called command_dsa_watcher")
-#     if 'crawl' == argv[1]:
+
-#         out = []
+    def get_id_from_about_string(about):
-#         # TODO: this is broken... the default should neither be part of the code,
+        return int(about.split('/')[-1].split('-')[1])
-#         # but rather be determined at runtime (like "latest" or similar)
+
-#         dsa = config.runtime_config_store.deepget('plugins.last_dsa', 1000)
+    def get_dsa_list(after):
-#
+        """
-#         url = 'https://security-tracker.debian.org/tracker/DSA-%d-1' % dsa
+        Get a list of dsa items in form of id and package, retrieved from the RSS feed
-#
+        :param after: optional integer to filter on (only DSA's after that will be returned)
-#         try:
+        :returns list of id, package (with DSA prefix)
-#             request = urllib.request.Request(url)
+        """
-#             request.add_header('User-Agent', USER_AGENT)
+        nsmap = {
-#             response = urllib.request.urlopen(request)
+            "purl": "http://purl.org/rss/1.0/",
-#             html_text = response.read(BUFSIZ)  # ignore more than BUFSIZ
+            "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
-#         except Exception as e:
+        }
-#             err = e
+        dsa_response = requests.get("https://www.debian.org/security/dsa-long")
-#             if '404' not in str(err):
+        xmldoc = etree.fromstring(dsa_response.content)
-#                 msg = 'error for %s: %s' % (url, err)
+        dsa_about_list = xmldoc.xpath('//purl:item/@rdf:about', namespaces=nsmap)
-#                 log.warn(msg)
+        for dsa_about in reversed(dsa_about_list):
-#                 out.append(msg)
+            dsa_id = get_id_from_about_string(dsa_about)
-#         else:
+            if after and dsa_id <= after:
-#             if str != type(html_text):
+                continue
-#                 html_text = str(html_text)
+            else:
-#
+                yield dsa_id, str(dsa_about).replace(' - security update', '')
-#             result = re.match(r'.*?Description</b></td><td>(.*?)</td>.*?', html_text, re.S | re.M | re.IGNORECASE)
+
-#
+    out = []
-#             package = 'error extracting package name'
+    last_dsa = config.runtimeconf_deepget('plugins.dsa-watcher.last_dsa')
-#             if result:
+    log.debug('Searching for DSA after ID {}'.format(last_dsa))
-#                 package = result.groups()[0]
+    for dsa, package in get_dsa_list(after=last_dsa):
-#
+        url = 'https://security-tracker.debian.org/tracker/DSA-%d-1' % dsa
-#             if config.get('persistent_locked'):
+
-#                 msg = "couldn't get exclusive lock"
+        msg = 'new Debian Security Announce found ({}): {}'.format(package, url)
-#                 log.warn(msg)
+        out.append(msg)
-#                 out.append(msg)
+
-#             else:
+        last_dsa = dsa
-#                 config.set('persistent_locked', True)
+
-#                 blob = conf_load()
+    config.runtime_config_store['plugins']['dsa-watcher']['last_dsa'] = last_dsa
-#
+    config.runtimeconf_persist()
-#                 if 'plugin_conf' not in blob:
+    crawl_at = time.time() + config.runtimeconf_deepget('plugins.dsa-watcher.interval')
-#                     blob['plugin_conf'] = {}
+
-#
+    msg = 'next crawl set to %s' % time.strftime('%Y-%m-%d %H:%M', time.localtime(crawl_at))
-#                 if 'last_dsa' not in blob['plugin_conf']:
+    out.append(msg)
-#                     blob['plugin_conf']['last_dsa'] = 3308  # FIXME: fixed value
+    return {
-#
+        'msg': out,
-#                 blob['plugin_conf']['last_dsa'] += 1
+        'event': {
-#
+            'time': crawl_at,
-#                 runtimeconf_save(blob)
+            'command': (command_dsa_watcher, ([],))
-#                 config.set('persistent_locked', False)
+        }
-#
+    }
-#             msg = (
-#                 'new Debian Security Announce found (%s): %s' % (str(package).replace(' - security update', ''), url))
-#             out.append(msg)
-#
-#             log.info('no dsa for %d, trying again...' % dsa)
-#         # that's good, no error, just 404 -> DSA not released yet
-#
-#         crawl_at = time.time() + config.get('dsa_watcher_interval')
-#         # register_event(crawl_at, command_dsa_watcher, (['dsa-watcher', 'crawl'],))
-#
-#         msg = 'next crawl set to %s' % time.strftime('%Y-%m-%d %H:%M', time.localtime(crawl_at))
-#         out.append(msg)
-#         return {
-#             'msg': out,
-#             'event': {
-#                 'time': crawl_at,
-#                 'command': (command_dsa_watcher, (['dsa-watcher', 'crawl'],))
-#             }
-#         }
-#     else:
-#         msg = 'wrong argument'
-#         log.warn(msg)
-#         return {'msg': msg}
 
 
 @pluginfunction("provoke-bots", "search for other bots", ptypes_COMMAND)
@@ -667,7 +641,8 @@ def set_status(argv, **args):
         return {
             'presence': {
                 'status': 'xa',
-                'msg': 'I\'m muted now. You can unmute me with "%s: set_status unmute"' % config.conf_get("bot_nickname")
+                'msg': 'I\'m muted now. You can unmute me with "%s: set_status unmute"' % config.conf_get(
+                    "bot_nickname")
             }
         }
     elif command == 'unmute' and args['reply_user'] == config.conf_get('bot_owner'):

urlbot.py

@@ -26,7 +26,8 @@ from plugins import (
     plugin_enabled_get,
     ptypes_PARSE,
     register_event,
-    else_command
+    register_active_event,
+    else_command,
 )
 import config
 
@@ -355,7 +356,16 @@ class UrlBot(IdleBot):
             elif 'command' in event:
                 command = event["command"]
                 if rate_limit(RATE_EVENT):
-                    register_event(event["time"], command[0], command[1])
+                    register_event(t=event["time"], callback=command[0], args=command[1])
+                    # kind of ugly..
+                    register_active_event(
+                        t=event['time'],
+                        callback=command[0],
+                        args=command[1],
+                        action_runner=self._run_action,
+                        plugin=plugin,
+                        msg_obj=msg_obj
+                    )
 
         if 'msg' in action and rate_limit(RATE_CHAT | plugin.ratelimit_class):
             self.send_reply(action['msg'], msg_obj)