diff --git a/.gitignore b/.gitignore index ef5090e..e3212ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .*swp *.pyc local_config.ini +persistent_config.ini # legacy local_config.py diff --git a/common.py b/common.py index d815d2d..0bc02df 100644 --- a/common.py +++ b/common.py @@ -2,14 +2,10 @@ """ Common functions for urlbot """ import html.parser import logging -import os -import pickle import re -import sys import time import urllib.request from collections import namedtuple -import config RATE_NO_LIMIT = 0x00 RATE_GLOBAL = 0x01 @@ -25,32 +21,6 @@ EVENTLOOP_DELAY = 0.100 # seconds USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:31.0) ' \ 'Gecko/20100101 Firefox/31.0 Iceweasel/31.0' - -def conf_save(obj): - with open(config.get('persistent_storage'), 'wb') as config_file: - return pickle.dump(obj, config_file) - - -def conf_load(): - path = config.get('persistent_storage') - if os.path.isfile(path): - with open(path, 'rb') as fd: - fd.seek(0) - return pickle.load(fd) - else: - return {} - - -def conf_set(key, value): - blob = conf_load() - blob[key] = value - conf_save(blob) - - -def conf_get(key, default=None): - blob = conf_load() - return blob.get(key, default) - Bucket = namedtuple("BucketConfig", ["history", "period", "max_hist_len"]) buckets = { diff --git a/config.py b/config.py index 81b282a..a88a310 100644 --- a/config.py +++ b/config.py @@ -18,9 +18,11 @@ from validate import Validator __initialized = False __config_store = ConfigObj('local_config.ini', configspec='local_config.ini.spec') +runtime_config_store = ConfigObj('persistent_config.ini', configspec='persistent_config.ini.spec') validator = Validator() result = __config_store.validate(validator) +runtime_config_store.validate(validator) if not result: print('Config file validation failed!') @@ -30,7 +32,7 @@ else: __config_store.write() -def get(key): +def conf_get(key): if not __initialized: raise RuntimeError("not __initialized") try: @@ -42,7 +44,32 @@ def get(key): raise -def set(key, val): +def conf_set(key, val): __config_store[key] = val __config_store.write() return None + + +def runtimeconf_set(key, value): + runtime_config_store[key] = value + runtime_config_store.write() + + +def runtimeconf_get(key, default=None): + if key is None: + return runtime_config_store + else: + return runtime_config_store.get(key, default=default) + + +def runtimeconf_deepget(key, default=None): + if '.' not in key: + return runtimeconf_get(key, default) + else: + path = key.split('.') + value = runtimeconf_get(path.pop(0)) + for p in path: + value = value.get(p) + if value is None: + return None + return value diff --git a/idlebot.py b/idlebot.py index 8e79800..bafbf2a 100755 --- a/idlebot.py +++ b/idlebot.py @@ -3,14 +3,11 @@ import logging import time import sys -from common import VERSION, EVENTLOOP_DELAY, conf_load - +from common import VERSION, EVENTLOOP_DELAY import config from sleekxmpp import ClientXMPP -# got_hangup = False - class IdleBot(ClientXMPP): def __init__(self, jid, password, rooms, nick): @@ -49,13 +46,13 @@ class IdleBot(ClientXMPP): # don't talk to yourself if msg_obj['mucnick'] == self.nick or 'groupchat' != msg_obj['type']: return False - elif msg_obj['body'].startswith(config.get('bot_nickname')) and 'hangup' in msg_obj['body']: + elif msg_obj['body'].startswith(config.conf_get('bot_nickname')) and 'hangup' in msg_obj['body']: self.logger.warn("got 'hangup' from '%s': '%s'" % ( msg_obj['mucnick'], msg_obj['body'] )) self.hangup() return False - elif msg_obj['mucnick'] in conf_load().get("other_bots", ()): + elif msg_obj['mucnick'] in config.runtime_config_store["other_bots"]: # not talking to the other bot. return False else: @@ -71,28 +68,28 @@ class IdleBot(ClientXMPP): def start(botclass, active=False): logging.basicConfig( - level=config.get('loglevel'), + level=config.conf_get('loglevel'), format=sys.argv[0] + ' %(asctime)s %(levelname).1s %(funcName)-15s %(message)s' ) logger = logging.getLogger(__name__) logger.info(VERSION) - jid = config.get('jid') + jid = config.conf_get('jid') if '/' not in jid: jid = '%s/%s' % (jid, botclass.__name__) bot = botclass( jid=jid, - password=config.get('password'), - rooms=config.get('rooms'), - nick=config.get('bot_nickname') + password=config.conf_get('password'), + rooms=config.conf_get('rooms'), + nick=config.conf_get('bot_nickname') ) import plugins if active: plugins.register_all() - if plugins.plugin_enabled_get(plugins.command_dsa_watcher): + # if plugins.plugin_enabled_get(plugins.command_dsa_watcher): # first result is lost. - plugins.command_dsa_watcher(['dsa-watcher', 'crawl']) + # plugins.command_dsa_watcher(['dsa-watcher', 'crawl']) bot.connect() bot.register_plugin('xep_0045') diff --git a/local_config.ini.spec b/local_config.ini.spec index dc8d6cf..de182b0 100644 --- a/local_config.ini.spec +++ b/local_config.ini.spec @@ -29,7 +29,6 @@ tea_steep_time = integer(default=220) image_preview = boolean(default=true) dsa_watcher_interval = integer(default=900) -last_dsa = integer # TODO broken loglevel = option('ERROR', WARN', 'INFO', 'DEBUG', default='INFO') debug_mode = boolean(default=false) diff --git a/persistent_config.ini.spec b/persistent_config.ini.spec new file mode 100644 index 0000000..6171efe --- /dev/null +++ b/persistent_config.ini.spec @@ -0,0 +1,11 @@ +# [main] +other_bots = string_list(default=list()) + +[plugins] + [[info]] + enabled = boolean(default=true) + last_dsa = integer(default=0) # TODO broken + +[user_pref] + +[user_records] diff --git a/plugins.py b/plugins.py index 97e128c..0560ba9 100644 --- a/plugins.py +++ b/plugins.py @@ -10,8 +10,9 @@ import unicodedata import urllib.parse import urllib.request -from common import conf_load, conf_save, RATE_GLOBAL, RATE_NO_SILENCE, VERSION, RATE_INTERACTIVE, BUFSIZ, \ - USER_AGENT, extract_title, RATE_FUN, RATE_NO_LIMIT, conf_get, RATE_URL +from common import RATE_GLOBAL, RATE_NO_SILENCE, VERSION, RATE_INTERACTIVE, BUFSIZ, \ + USER_AGENT, extract_title, RATE_FUN, RATE_NO_LIMIT, RATE_URL +from config import runtimeconf_get import config from string_constants import excuses, moin_strings_hi, moin_strings_bye, cakes @@ -27,35 +28,26 @@ log = logging.getLogger(__name__) def plugin_enabled_get(urlbot_plugin): - blob = conf_load() - - if 'plugin_conf' in blob: - if urlbot_plugin.plugin_name in blob['plugin_conf']: - return blob['plugin_conf'][urlbot_plugin.plugin_name].get('enabled', urlbot_plugin.is_enabled) - - return urlbot_plugin.is_enabled + is_enabled = config.runtimeconf_deepget('plugins.{}.enabled'.format(urlbot_plugin.plugin_name), None) + if is_enabled is None: + return urlbot_plugin.is_enabled + else: + return is_enabled def plugin_enabled_set(plugin, enabled): - if config.get('persistent_locked'): + if config.conf_get('persistent_locked'): log.warn("couldn't get exclusive lock") - return False - config.set('persistent_locked', True) - blob = conf_load() + config.conf_set('persistent_locked', True) + # blob = conf_load() - if 'plugin_conf' not in blob: - blob['plugin_conf'] = {} + if plugin.plugin_name not in config.runtime_config_store['plugins']: + config.runtime_config_store['plugins'][plugin.plugin_name] = {} - if plugin.plugin_name not in blob['plugin_conf']: - blob['plugin_conf'][plugin.plugin_name] = {} - - blob['plugin_conf'][plugin.plugin_name]['enabled'] = enabled - - conf_save(blob) - config.set('persistent_locked', False) - - return True + config.runtime_config_store['plugins'][plugin.plugin_name]['enabled'] = enabled + config.runtime_config_store.write() + config.conf_set('persistent_locked', False) def pluginfunction(name, desc, plugin_type, ratelimit_class=RATE_GLOBAL, enabled=True): @@ -105,7 +97,7 @@ def parse_mental_ill(**args): log.info('sent mental illness reply') return { 'msg': ( - '''Multiple exclamation/question marks are a sure sign of mental disease, with %s as a living example.''' % + 'Multiple exclamation/question marks are a sure sign of mental disease, with %s as a living example.' % args['reply_user'] ) } @@ -180,11 +172,11 @@ def parse_moin(**args): for w in words: if d.lower() == w.lower(): - if args['reply_user'] in config.get('moin-disabled-user'): + if args['reply_user'] in config.conf_get('moin-disabled-user'): log.info('moin blacklist match') return - if args['reply_user'] in config.get('moin-modified-user'): + if args['reply_user'] in config.conf_get('moin-modified-user'): log.info('being "quiet" for %s' % w) return { 'msg': '/me %s' % random.choice([ @@ -215,7 +207,7 @@ def parse_latex(**args): @pluginfunction('me-action', 'reacts to /me.*%{bot_nickname}', ptypes_PARSE, ratelimit_class=RATE_FUN | RATE_GLOBAL) def parse_slash_me(**args): - if args['data'].lower().startswith('/me') and (config.get('bot_nickname') in args['data'].lower()): + if args['data'].lower().startswith('/me') and (config.conf_get('bot_nickname') in args['data'].lower()): log.info('sent /me reply') me_replys = [ @@ -324,7 +316,7 @@ def command_source(argv, **_): log.info('sent source URL') return { - 'msg': 'My source code can be found at %s' % config.get('src-url') + 'msg': 'My source code can be found at %s' % config.conf_get('src-url') } @@ -353,7 +345,7 @@ def command_dice(argv, **args): ) for i in range(count): - if args['reply_user'] in config.get('enhanced-random-user'): + if args['reply_user'] in config.conf_get('enhanced-random-user'): rnd = 0 # this might confuse users. good. log.info('sent random (enhanced)') else: @@ -394,19 +386,19 @@ def command_uptime(argv, **args): if 'uptime' != argv[0]: return - u = int(config.get('uptime') + time.time()) + u = int(config.conf_get('uptime') + time.time()) plural_uptime = 's' plural_request = 's' if 1 == u: plural_uptime = '' - if 1 == config.get('request_counter'): + if 1 == config.conf_get('request_counter'): plural_request = '' log.info('sent statistics') return { 'msg': args['reply_user'] + (''': happily serving for %d second%s, %d request%s so far.''' % ( - u, plural_uptime, int(config.get('request_counter')), plural_request)) + u, plural_uptime, int(config.conf_get('request_counter')), plural_request)) } @@ -443,16 +435,16 @@ def command_info(argv, **args): questions, please talk to my master %s. I'm rate limited. To make me exit immediately, highlight me with 'hangup' in the message (emergency only, please). For other commands, highlight me with 'help'.''' % ( - config.get('bot_owner'))) + config.conf_get('bot_owner'))) } -@pluginfunction('teatimer', 'sets a tea timer to $1 or currently %d seconds' % config.get('tea_steep_time'), ptypes_COMMAND) +@pluginfunction('teatimer', 'sets a tea timer to $1 or currently %d seconds' % config.conf_get('tea_steep_time'), ptypes_COMMAND) def command_teatimer(argv, **args): if 'teatimer' != argv[0]: return - steep = config.get('tea_steep_time') + steep = config.conf_get('tea_steep_time') if len(argv) > 1: try: @@ -544,18 +536,16 @@ def command_show_blacklist(argv, **args): '' if not argv1 else ' (limited to %s)' % argv1 ) ] + [ - b for b in config.get('url_blacklist') if not argv1 or argv1 in b + b for b in config.conf_get('url_blacklist') if not argv1 or argv1 in b ] } def usersetting_get(argv, args): - blob = conf_load() - arg_user = args['reply_user'] arg_key = argv[1] - if arg_user not in blob['user_pref']: + if arg_user not in config.runtime_config_store['user_pref']: return { 'msg': args['reply_user'] + ': user key not found' } @@ -563,7 +553,7 @@ def usersetting_get(argv, args): return { 'msg': args['reply_user'] + ': %s == %s' % ( arg_key, - 'on' if blob['user_pref'][arg_user][arg_key] else 'off' + 'on' if config.runtime_config_store['user_pref'][arg_user][arg_key] else 'off' ) } @@ -592,24 +582,20 @@ def command_usersetting(argv, **args): # display current value return usersetting_get(argv, args) - if config.get('persistent_locked'): + if config.conf_get('persistent_locked'): return { 'msg': args['reply_user'] + ''': couldn't get exclusive lock''' } - config.set('persistent_locked', True) - blob = conf_load() + config.conf_set('persistent_locked', True) - if 'user_pref' not in blob: - blob['user_pref'] = {} + if arg_user not in config.runtime_config_store['user_pref']: + config.runtime_config_store['user_pref'][arg_user] = {} - if arg_user not in blob['user_pref']: - blob['user_pref'][arg_user] = {} + config.runtime_config_store['user_pref'][arg_user][arg_key] = 'on' == arg_val - blob['user_pref'][arg_user][arg_key] = 'on' == arg_val - - conf_save(blob) - config.set('persistent_locked', False) + config.runtime_config_store.write() + config.conf_set('persistent_locked', False) # display value written to db return usersetting_get(argv, args) @@ -824,24 +810,20 @@ def command_record(argv, **args): message = '%s (%s): ' % (args['reply_user'], time.strftime('%F.%T')) message += ' '.join(argv[2:]) - if config.get('persistent_locked'): + if config.conf_get('persistent_locked'): return { 'msg': "%s: couldn't get exclusive lock" % args['reply_user'] } - config.set('persistent_locked', True) - blob = conf_load() + config.conf_set('persistent_locked', True) - if 'user_records' not in blob: - blob['user_records'] = {} + if target_user not in config.runtime_config_store['user_records']: + config.runtime_config_store['user_records'][target_user] = [] - if target_user not in blob['user_records']: - blob['user_records'][target_user] = [] + config.runtime_config_store['user_records'][target_user].append(message) - blob['user_records'][target_user].append(message) - - conf_save(blob) - config.set('persistent_locked', False) + config.runtime_config_store.write() + config.conf_set('persistent_locked', False) return { 'msg': '%s: message saved for %s' % (args['reply_user'], target_user) @@ -862,97 +844,102 @@ def command_show_recordlist(argv, **args): '%s: offline records%s: %s' % ( args['reply_user'], '' if not argv1 else ' (limited to %s)' % argv1, - ', '.join([ - '%s (%d)' % (key, len(val)) for key, val in conf_load().get('user_records').items() - if not argv1 or argv1.lower() in key.lower() - ]) + ', '.join( + [ + '%s (%d)' % (key, len(val)) for key, val in config.runtime_config_store['user_records'].items() + if not argv1 or argv1.lower() in key.lower() + ] + ) ) } -@pluginfunction('dsa-watcher', 'automatically crawls for newly published Debian Security Announces', ptypes_COMMAND, - ratelimit_class=RATE_NO_SILENCE) -def command_dsa_watcher(argv, **_): - """ - TODO: rewrite so that a last_dsa_date is used instead, then all DSAs since then printed and the date set to now() - """ - if 'dsa-watcher' != argv[0]: - return - - if 2 != len(argv): - msg = 'wrong number of arguments' - log.warn(msg) - return {'msg': msg} - - if 'crawl' == argv[1]: - out = [] - dsa = conf_load().get('plugin_conf', {}).get('last_dsa', 1000) - - url = 'https://security-tracker.debian.org/tracker/DSA-%d-1' % dsa - - try: - request = urllib.request.Request(url) - request.add_header('User-Agent', USER_AGENT) - response = urllib.request.urlopen(request) - html_text = response.read(BUFSIZ) # ignore more than BUFSIZ - except Exception as e: - err = e - if '404' not in str(err): - msg = 'error for %s: %s' % (url, err) - log.warn(msg) - out.append(msg) - else: - if str != type(html_text): - html_text = str(html_text) - - result = re.match(r'.*?Description