catdog2/urlbot-native
1
0
Fork 0
Code Issues Pull Requests Releases Activity

implement new version of the dsa watcher

Browse Source
This commit is contained in:
Thorsten
2016-01-04 14:27:02 +01:00
parent d1bf544ff9
commit e1f451c094
5 changed files with 101 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
local_config.ini.spec
View File

@@ -24,7 +24,6 @@ moin-disabled-user = string_list(default=list())
tea_steep_time = integer(default=220)
image_preview = boolean(default=true)
dsa_watcher_interval = integer(default=900)
loglevel = option('ERROR', WARN', 'INFO', 'DEBUG', default='INFO')
debug_mode = boolean(default=false)

4
persistent_config.ini.spec
View File

@@ -6,7 +6,9 @@ start_time = integer(default=0)
[plugins]
[[info]]
enabled = boolean(default=true)
last_dsa = integer(default=0) # TODO broken
[[dsa-watcher]]
last_dsa = integer(default=0)
interval = integer(default=900)
[user_pref]

20
plugins/__init__.py
View File

@@ -26,7 +26,6 @@ def plugin_enabled_set(plugin, enabled):
log.warn("couldn't get exclusive lock")
config.conf_set('persistent_locked', True)
# blob = conf_load()
if plugin.plugin_name not in config.runtime_config_store['plugins']:
config.runtime_config_store['plugins'][plugin.plugin_name] = {}
@@ -36,6 +35,25 @@ def plugin_enabled_set(plugin, enabled):
config.conf_set('persistent_locked', False)
def register_active_event(t, callback, args, action_runner, plugin, msg_obj):
"""
Execute a callback at a given time and react on the output
:param t: when to execute the job
:param callback: the function to execute
:param args: parameters for said function
:param action_runner: bots action dict parser
:param plugin: pass-through object for action parser
:param msg_obj: pass-through object for action parser
:return:
"""
def func(func_args):
action = callback(*func_args)
if action:
action_runner(action=action, plugin=plugin, msg_obj=msg_obj)
joblist.append((t, func, args))
def register_event(t, callback, args):
joblist.append((t, callback, args))

157
plugins/commands.py
View File

@@ -6,13 +6,14 @@ import traceback
import unicodedata
import requests
from lxml import etree
import config
from common import (
VERSION, RATE_FUN, RATE_GLOBAL, RATE_INTERACTIVE, RATE_NO_LIMIT,
giphy, pluginfunction,
ptypes_COMMAND
)
ptypes_COMMAND,
RATE_NO_SILENCE)
from string_constants import cakes, excuses, moin_strings_hi, moin_strings_bye
log = logging.getLogger(__name__)
@@ -28,7 +29,6 @@ def command_version(argv, **args):
@pluginfunction('uptime', 'prints uptime', ptypes_COMMAND)
def command_uptime(argv, **args):
u = int(config.runtimeconf_get('start_time') + time.time())
plural_uptime = 's'
plural_request = 's'
@@ -47,7 +47,6 @@ def command_uptime(argv, **args):
@pluginfunction('info', 'prints info message', ptypes_COMMAND)
def command_info(argv, **args):
log.info('sent long info')
return {
'msg': args['reply_user'] + (
@@ -61,7 +60,6 @@ def command_info(argv, **args):
@pluginfunction('ping', 'sends pong', ptypes_COMMAND, ratelimit_class=RATE_INTERACTIVE)
def command_ping(argv, **args):
rnd = random.randint(0, 3) # 1:4
if 0 == rnd:
msg = args['reply_user'] + ''': peng (You're dead now.)'''
@@ -338,7 +336,6 @@ def usersetting_get(argv, args):
@pluginfunction('set', 'modify a user setting', ptypes_COMMAND, ratelimit_class=RATE_NO_LIMIT)
def command_usersetting(argv, **args):
settings = ['spoiler']
arg_user = args['reply_user']
arg_key = argv[0] if len(argv) > 0 else None
@@ -529,90 +526,67 @@ def command_show_recordlist(argv, **args):
)
}
# TODO: disabled until rewrite
# @pluginfunction('dsa-watcher', 'automatically crawls for newly published Debian Security Announces', ptypes_COMMAND,
# ratelimit_class=RATE_NO_SILENCE)
# def command_dsa_watcher(argv, **_):
# """
# TODO: rewrite so that a last_dsa_date is used instead, then all DSAs since then printed and the date set to now()
# """
#
# if 2 != len(argv):
# msg = 'wrong number of arguments'
# log.warn(msg)
# return {'msg': msg}
#
# if 'crawl' == argv[1]:
# out = []
# # TODO: this is broken... the default should neither be part of the code,
# # but rather be determined at runtime (like "latest" or similar)
# dsa = config.runtime_config_store.deepget('plugins.last_dsa', 1000)
#
# url = 'https://security-tracker.debian.org/tracker/DSA-%d-1' % dsa
#
# try:
# request = urllib.request.Request(url)
# request.add_header('User-Agent', USER_AGENT)
# response = urllib.request.urlopen(request)
# html_text = response.read(BUFSIZ) # ignore more than BUFSIZ
# except Exception as e:
# err = e
# if '404' not in str(err):
# msg = 'error for %s: %s' % (url, err)
# log.warn(msg)
# out.append(msg)
# else:
# if str != type(html_text):
# html_text = str(html_text)
#
# result = re.match(r'.*?Description</b></td><td>(.*?)</td>.*?', html_text, re.S | re.M | re.IGNORECASE)
#
# package = 'error extracting package name'
# if result:
# package = result.groups()[0]
#
# if config.get('persistent_locked'):
# msg = "couldn't get exclusive lock"
# log.warn(msg)
# out.append(msg)
# else:
# config.set('persistent_locked', True)
# blob = conf_load()
#
# if 'plugin_conf' not in blob:
# blob['plugin_conf'] = {}
#
# if 'last_dsa' not in blob['plugin_conf']:
# blob['plugin_conf']['last_dsa'] = 3308 # FIXME: fixed value
#
# blob['plugin_conf']['last_dsa'] += 1
#
# runtimeconf_save(blob)
# config.set('persistent_locked', False)
#
# msg = (
# 'new Debian Security Announce found (%s): %s' % (str(package).replace(' - security update', ''), url))
# out.append(msg)
#
# log.info('no dsa for %d, trying again...' % dsa)
# # that's good, no error, just 404 -> DSA not released yet
#
# crawl_at = time.time() + config.get('dsa_watcher_interval')
# # register_event(crawl_at, command_dsa_watcher, (['dsa-watcher', 'crawl'],))
#
# msg = 'next crawl set to %s' % time.strftime('%Y-%m-%d %H:%M', time.localtime(crawl_at))
# out.append(msg)
# return {
# 'msg': out,
# 'event': {
# 'time': crawl_at,
# 'command': (command_dsa_watcher, (['dsa-watcher', 'crawl'],))
# }
# }
# else:
# msg = 'wrong argument'
# log.warn(msg)
# return {'msg': msg}
@pluginfunction(
'dsa-watcher',
'automatically crawls for newly published Debian Security Announces', ptypes_COMMAND,
ratelimit_class=RATE_NO_SILENCE, enabled=True)
def command_dsa_watcher(argv=None, **_):
"""
TODO: rewrite so that a last_dsa_date is used instead,
then all DSAs since then printed and the date set to now()
:param argv:
:param _:
"""
log.debug("Called command_dsa_watcher")
def get_id_from_about_string(about):
return int(about.split('/')[-1].split('-')[1])
def get_dsa_list(after):
"""
Get a list of dsa items in form of id and package, retrieved from the RSS feed
:param after: optional integer to filter on (only DSA's after that will be returned)
:returns list of id, package (with DSA prefix)
"""
nsmap = {
"purl": "http://purl.org/rss/1.0/",
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
}
dsa_response = requests.get("https://www.debian.org/security/dsa-long")
xmldoc = etree.fromstring(dsa_response.content)
dsa_about_list = xmldoc.xpath('//purl:item/@rdf:about', namespaces=nsmap)
for dsa_about in reversed(dsa_about_list):
dsa_id = get_id_from_about_string(dsa_about)
if after and dsa_id <= after:
continue
else:
yield dsa_id, str(dsa_about).replace(' - security update', '')
out = []
last_dsa = config.runtimeconf_deepget('plugins.dsa-watcher.last_dsa')
log.debug('Searching for DSA after ID {}'.format(last_dsa))
for dsa, package in get_dsa_list(after=last_dsa):
url = 'https://security-tracker.debian.org/tracker/DSA-%d-1' % dsa
msg = 'new Debian Security Announce found ({}): {}'.format(package, url)
out.append(msg)
last_dsa = dsa
config.runtime_config_store['plugins']['dsa-watcher']['last_dsa'] = last_dsa
config.runtimeconf_persist()
crawl_at = time.time() + config.runtimeconf_deepget('plugins.dsa-watcher.interval')
msg = 'next crawl set to %s' % time.strftime('%Y-%m-%d %H:%M', time.localtime(crawl_at))
out.append(msg)
return {
'msg': out,
'event': {
'time': crawl_at,
'command': (command_dsa_watcher, ([],))
}
}
@pluginfunction("provoke-bots", "search for other bots", ptypes_COMMAND)
@@ -667,7 +641,8 @@ def set_status(argv, **args):
return {
'presence': {
'status': 'xa',
'msg': 'I\'m muted now. You can unmute me with "%s: set_status unmute"' % config.conf_get("bot_nickname")
'msg': 'I\'m muted now. You can unmute me with "%s: set_status unmute"' % config.conf_get(
"bot_nickname")
}
}
elif command == 'unmute' and args['reply_user'] == config.conf_get('bot_owner'):

14
urlbot.py
View File

@@ -26,7 +26,8 @@ from plugins import (
plugin_enabled_get,
ptypes_PARSE,
register_event,
else_command
register_active_event,
else_command,
)
import config
@@ -355,7 +356,16 @@ class UrlBot(IdleBot):
elif 'command' in event:
command = event["command"]
if rate_limit(RATE_EVENT):
register_event(event["time"], command[0], command[1])
register_event(t=event["time"], callback=command[0], args=command[1])
# kind of ugly..
register_active_event(
t=event['time'],
callback=command[0],
args=command[1],
action_runner=self._run_action,
plugin=plugin,
msg_obj=msg_obj
)
if 'msg' in action and rate_limit(RATE_CHAT | plugin.ratelimit_class):
self.send_reply(action['msg'], msg_obj)