From 15c78581f60d8f1fc0ae8f831ac52817ce2e2b8d Mon Sep 17 00:00:00 2001 From: Thorsten S Date: Sun, 20 Dec 2015 23:35:34 +0100 Subject: [PATCH] deployment, wohoo. also, cleanup of old stuff. --- .gitignore | 2 + deploy/README | 23 ++++++++++ deploy/deploy.sh | 4 ++ deploy/deploy.yml | 86 +++++++++++++++++++++++++++++++++++++ deploy/hosts | 2 + deploy/supervisord.conf | 46 ++++++++++++++++++++ related/bot.py | 80 ---------------------------------- related/levenshtein_test.py | 38 ---------------- related/strsim.py | 69 ----------------------------- 9 files changed, 163 insertions(+), 187 deletions(-) create mode 100644 deploy/README create mode 100644 deploy/deploy.sh create mode 100644 deploy/deploy.yml create mode 100644 deploy/hosts create mode 100644 deploy/supervisord.conf delete mode 100755 related/bot.py delete mode 100755 related/levenshtein_test.py delete mode 100755 related/strsim.py diff --git a/.gitignore b/.gitignore index e3212ca..9e984d1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ persistent_config.ini # legacy local_config.py urlbot.persistent + +deploy/credentials.yml diff --git a/deploy/README b/deploy/README new file mode 100644 index 0000000..bb3ad50 --- /dev/null +++ b/deploy/README @@ -0,0 +1,23 @@ +To use the playbook, create a yaml file credentials.yml +with the following content (you can use vault to encrypt): + +jid: yourjabber@id.to.use +password: yourpasswordforthisjabber +rooms: + - debianforum.de@chat.debianforum.de # your channel + - spielwiese@chat.debianforum.de # bot playground +bot_nickname: T800 # your bots nickname +bot_owner: MASTER # your nickname (for info and admin stuff) + + +Further, you need a hosts-file with the following content: + +[bots] +yourserverip +# or alternatively, +derpy_name ansible_host=yourserverip + + +There is deploy.sh which I created so I have a single file to +deploy my stuff - it uses a virtualenv (py2!) with ansible and + some vault file declaration. diff --git a/deploy/deploy.sh b/deploy/deploy.sh new file mode 100644 index 0000000..3c94c96 --- /dev/null +++ b/deploy/deploy.sh @@ -0,0 +1,4 @@ +#!/bin/bash +source ~/urlbot-native/venv/bin/activate + +ansible-playbook -i hosts deploy.yml --vault-password-file ~/.vaultpass -D diff --git a/deploy/deploy.yml b/deploy/deploy.yml new file mode 100644 index 0000000..fd6ae68 --- /dev/null +++ b/deploy/deploy.yml @@ -0,0 +1,86 @@ +- hosts: bots + remote_user: root + tasks: + - name: create user for bot + user: name=jabberbot comment="Account for the urlbot" shell=/bin/bash + - name: local user can log in with ssh key + authorized_key: user=jabberbot key="{{ lookup('file', '~/.ssh/id_rsa.pub') }}" + + - name: git, python3, virtualenv have to be installed + apt: name="{{item}}" state=installed + with_items: + - git + - python3 + - virtualenv + - python3-virtualenv + +- hosts: bots + remote_user: jabberbot + vars: + - botrepo: http://aero2k.de/t/repos/urlbot-native.git + - pypi_mirror: http://pypi.fcio.net/simple/ + tasks: + - include_vars: credentials.yml + tags: [render_config] + - name: virtualenv for the bot + shell: virtualenv -p python3 ~/botenv creates=~/botenv/bin/activate + - name: virtualenv for supervisord + shell: virtualenv -p python2 ~/svenv creates=~/svenv/bin/activate + - name: clone repository + git: repo="{{botrepo}}" dest=~/urlbot force=yes update=yes + register: source_code + - name: install bot dependencies into virtualenv + pip: name="{{item}}" virtualenv=~/botenv extra_args="-i {{pypi_mirror}}" + with_items: + - sleekxmpp + - configobj + - name: install supervisor + pip: name=supervisor virtualenv=~/svenv extra_args="-i {{pypi_mirror}}" + + - name: set configuration + lineinfile: dest=~/urlbot/local_config.ini create=yes line="{{item.key}} = {{item.value}}" regexp="^{{item.key}}.=" + with_items: + - key: "jid" + value: "{{jid}}" + - key: "password" + value: "{{password}}" + - key: "rooms" + value: "{{rooms | join(', ')}}" + - key: "src-url" + value: "{{botrepo}}" + - key: "bot_nickname" + value: "{{bot_nickname}}" + - key: "bot_owner" + value: "{{bot_owner}}" + tags: [render_config] + register: urlbot_config + + - name: create supervisor config + copy: src=supervisord.conf dest=~/supervisord.conf + register: supervisord + + - name: verify supervisor running + shell: nc -z 127.0.0.1 9004; echo $? executable=/bin/bash + register: supervisor_running + changed_when: false + + - name: start supervisord + shell: source ~/svenv/bin/activate && supervisord executable=/bin/bash + register: start_supervisor + when: supervisord.changed or supervisor_running.stdout == "1" + #changed_when: "'already listening' not in start_supervisor.stdout" + + - name: activate supervisord changes + when: supervisord.changed + shell: source ~/svenv/bin/activate && supervisorctl reload executable=/bin/bash + + - name: idlebot started + supervisorctl: name=idlebot state=restarted supervisorctl_path=~/svenv/bin/supervisorctl + when: (source_code.changed or urlbot_config.changed) and not supervisord.changed + + - pause: seconds=30 + when: (source_code.changed or urlbot_config.changed) and not supervisord.changed + + - name: urlbot started + supervisorctl: name=bot state=restarted supervisorctl_path=~/svenv/bin/supervisorctl + when: (source_code.changed or urlbot_config.changed) and not supervisord.changed diff --git a/deploy/hosts b/deploy/hosts new file mode 100644 index 0000000..d7902ca --- /dev/null +++ b/deploy/hosts @@ -0,0 +1,2 @@ +[bots] +aero2k.de ansible_host=2a01:4f8:d16:130c::2 diff --git a/deploy/supervisord.conf b/deploy/supervisord.conf new file mode 100644 index 0000000..716ca0a --- /dev/null +++ b/deploy/supervisord.conf @@ -0,0 +1,46 @@ +; Sample supervisor config file. +; +; For more information on the config file, please see: +; http://supervisord.org/configuration.html +; +; Notes: +; - Shell expansion ("~" or "$HOME") is not supported. Environment +; variables can be expanded using this syntax: "%(ENV_HOME)s". +; - Comments must have a leading space: "a=b ;comment" not "a=b;comment". + +[unix_http_server] +file=/home/jabberbot/supervisor.sock ; (the path to the socket file) + +[inet_http_server] +port=127.0.0.1:9004 + +[supervisord] +logfile=/home/jabberbot/supervisord.log ; (main log file;default $CWD/supervisord.log) +logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) +logfile_backups=10 ; (num of main logfile rotation backups;default 10) +loglevel=info ; (log level;default info; others: debug,warn,trace) +pidfile=/home/jabberbot/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +nodaemon=false ; (start in foreground if true;default false) +minfds=1024 ; (min. avail startup file descriptors;default 1024) +minprocs=200 ; (min. avail process descriptors;default 200) + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///home/jabberbot/supervisor.sock ; use a unix:// URL for a unix socket + +;[include] +;files = relative/directory/*.ini + +[program:bot] +command=/home/jabberbot/botenv/bin/python3 /home/jabberbot/urlbot/urlbot.py +directory=/home/jabberbot/urlbot/ +stderr_logfile=/home/jabberbot/urlbot.err +stdout_logfile=/home/jabberbot/urlbot.log + +[program:idlebot] +directory=/home/jabberbot/urlbot/ +command=/home/jabberbot/botenv/bin/python3 /home/jabberbot/urlbot/idlebot.py +stderr_logfile=/home/jabberbot/idlebot.err +stdout_logfile=/home/jabberbot/idlebot.log diff --git a/related/bot.py b/related/bot.py deleted file mode 100755 index 1827151..0000000 --- a/related/bot.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/python3 - -import logging - -from sleekxmpp import ClientXMPP - -try: - from local_config import conf -except ImportError: - import sys - sys.stderr.write(''' -%s: E: local_config.py isn't tracked because of included secrets and -%s site specific configurations. Rename local_config.py.skel and -%s adjust to you needs. -'''[1:] % ( - sys.argv[0], - ' ' * len(sys.argv[0]), - ' ' * len(sys.argv[0]) - ) - ) - - sys.exit(10) - -import time -t = -time.time() - - -class Bot(ClientXMPP): - def __init__(self, jid, password, room, nick): - ClientXMPP.__init__(self, jid, password) - - self.room = room - self.nick = nick - - self.add_event_handler('session_start', self.session_start) - self.add_event_handler('groupchat_message', self.muc_message) - - def session_start(self, event): - self.get_roster() - self.send_presence() - - self.plugin['xep_0045'].joinMUC( - self.room, - self.nick, - wait=True - ) - - def muc_message(self, msg): - print(msg['mucnick']) - print(msg['body']) - print((msg['from'], msg['from'].bare)) - - print(conf('room') == msg['from'].bare) - - # don't talk to yourself - if msg['mucnick'] == self.nick: - return - - self.send_message( - mto=msg['from'].bare, - mbody='got[%s]' % msg['body'], - mtype='groupchat' - ) - -if '__main__' == __name__: - logging.basicConfig( - level=logging.DEBUG, - format='%(levelname)-8s %(message)s' - ) - - xmpp = Bot( - jid=conf('jid'), - password=conf('password'), - room=conf('room'), - nick=conf('bot_user') - ) - - xmpp.connect() - xmpp.register_plugin('xep_0045') - xmpp.process(threaded=False) diff --git a/related/levenshtein_test.py b/related/levenshtein_test.py deleted file mode 100755 index a29abc7..0000000 --- a/related/levenshtein_test.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/python3 - -from common import levenshtein - -(a, b) = ('foo barbaz', 'foobar baz') -(a, b) = ('sitting', 'kitten') -(a, b) = ('Monte Kali (Heringen)', 'http://de.wikipedia.org/wiki/Monte_Kali_%28Heringen%29') - -(matrix, ret) = levenshtein(a, b, return_table=True) - -sep = ' '*0 -out = '' -for B in b: - out += sep + '%2s' % B -print(sep + ' '*4 + out) - -for i in range(len(matrix)): - if 0 == i: - out = ' ' - else: - out = '%2s' % a[i-1] - - for j in range(len(matrix[i])): - if 0 == i or 0 == j: - col = '30;42' - elif i == j: - col = '41' - else: - col = 0 - - if 0 != col: - out += sep + '\x1b[%sm%2d\x1b[m' %(col, matrix[i][j]) - else: - out += sep + '%2d' % matrix[i][j] - - print(out) - -print(ret) diff --git a/related/strsim.py b/related/strsim.py deleted file mode 100755 index f03573b..0000000 --- a/related/strsim.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -import re - -def str_sim(a, b, do_print=False): - a = a.lower() - b = b.lower() - - a_parts = re.split('[\W_]+', a) - b_parts = re.split('[\W_]+', b) - - # this is a "simple" way to declare out[a][b] - out = list(map(list, [[0]*len(b_parts)]*len(a_parts))) - - for i in range(0, len(a_parts)-1): - for j in range(0, len(b_parts)-1): - if a_parts[i] == b_parts[j]: - out[i][j] += 1 - - if do_print: - i = 0 - for j in range(0, len(b_parts)): - print(' |'*i + ' '*2 + '.- ' + b_parts[j]) - i += 1 - print(' |'*i) - - for i in range(0, len(a_parts)): - print(' ' + str(out[i]) + ' ' + a_parts[i]) - - return out - -def sum_array(array): - _sum = 0 - for a in array: - if list == type(a) or tuple == type(a) or hash == type(a): - _sum += sum_array(a) - elif int == type(a) or float == type(a): - _sum += a - return _sum - -def wrapper_print(a, b, comment=''): - ret = str_sim(a, b, do_print=True) - - if '' != comment: - comment = ' ^ ' + comment - - print('[%2dx%2d::%2d]%s' %(len(ret), len(ret[0]), sum_array(ret), comment)) - -if '__main__' == __name__: - pairs = ( - ( - 'http://de.wikipedia.org/wiki/Monte_Kali_%28Heringen%29', - 'Monte Kali (Heringen)' - ), - ( - 'http://www.spiegel.de/politik/ausland/buddhisten-treffen-in-colombo-blitzender-moench-a-994447.html', - 'Buddhisten-Treffen in Colombo: Blitzender Mönch - SPIEGEL ONLINE' - ) - ) - - wrapper_print('foo bar baz', 'foo bar boom') - - for (url, title) in pairs: - wrapper_print(title, url, comment='raw') - url_no_proto = re.sub(r'https?://[^/]*/', '', url) - wrapper_print(title, url_no_proto, comment='no proto/domain') - url_no_proto_no_digits = re.sub(r'[0-9]*', '', url_no_proto) - wrapper_print(title, url_no_proto_no_digits, comment='no proto/domain/[0-9]')