deployment, wohoo. also, cleanup of old stuff.

.gitignore

@@ -6,3 +6,5 @@ persistent_config.ini
 # legacy
 local_config.py
 urlbot.persistent
+
+deploy/credentials.yml

deploy/README

@@ -0,0 +1,23 @@
+To use the playbook, create a yaml file credentials.yml
+with the following content (you can use vault to encrypt):
+
+jid: yourjabber@id.to.use
+password: yourpasswordforthisjabber
+rooms:
+   - debianforum.de@chat.debianforum.de # your channel
+   - spielwiese@chat.debianforum.de # bot playground
+bot_nickname: T800 # your bots nickname
+bot_owner: MASTER # your nickname (for info and admin stuff)
+
+
+Further, you need a hosts-file with the following content:
+
+[bots]
+yourserverip
+# or alternatively,
+derpy_name ansible_host=yourserverip
+
+
+There is deploy.sh which I created so I have a single file to
+deploy my stuff - it uses a virtualenv (py2!) with ansible and
+ some vault file declaration.

deploy/deploy.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+source ~/urlbot-native/venv/bin/activate
+
+ansible-playbook -i hosts deploy.yml --vault-password-file ~/.vaultpass -D

deploy/deploy.yml

@@ -0,0 +1,86 @@
+- hosts: bots
+  remote_user: root
+  tasks:
+  - name: create user for bot
+    user: name=jabberbot comment="Account for the urlbot" shell=/bin/bash
+  - name: local user can log in with ssh key
+    authorized_key: user=jabberbot key="{{ lookup('file', '~/.ssh/id_rsa.pub') }}"
+
+  - name: git, python3, virtualenv have to be installed
+    apt: name="{{item}}" state=installed
+    with_items:
+    - git
+    - python3
+    - virtualenv
+    - python3-virtualenv
+
+- hosts: bots
+  remote_user: jabberbot
+  vars:
+  - botrepo: http://aero2k.de/t/repos/urlbot-native.git
+  - pypi_mirror: http://pypi.fcio.net/simple/
+  tasks:
+  - include_vars: credentials.yml
+    tags: [render_config]
+  - name: virtualenv for the bot
+    shell: virtualenv -p python3 ~/botenv creates=~/botenv/bin/activate
+  - name: virtualenv for supervisord
+    shell: virtualenv -p python2 ~/svenv creates=~/svenv/bin/activate
+  - name: clone repository
+    git: repo="{{botrepo}}" dest=~/urlbot force=yes update=yes
+    register: source_code
+  - name: install bot dependencies into virtualenv
+    pip: name="{{item}}" virtualenv=~/botenv extra_args="-i {{pypi_mirror}}"
+    with_items:
+    - sleekxmpp
+    - configobj
+  - name: install supervisor
+    pip: name=supervisor virtualenv=~/svenv extra_args="-i {{pypi_mirror}}"
+
+  - name: set configuration
+    lineinfile: dest=~/urlbot/local_config.ini create=yes line="{{item.key}} = {{item.value}}" regexp="^{{item.key}}.="
+    with_items:
+      - key: "jid"
+        value: "{{jid}}"
+      - key: "password"
+        value: "{{password}}"
+      - key: "rooms"
+        value: "{{rooms | join(', ')}}"
+      - key: "src-url"
+        value: "{{botrepo}}"
+      - key: "bot_nickname"
+        value: "{{bot_nickname}}"
+      - key: "bot_owner"
+        value: "{{bot_owner}}"
+    tags: [render_config]
+    register: urlbot_config
+
+  - name: create supervisor config
+    copy: src=supervisord.conf dest=~/supervisord.conf
+    register: supervisord
+
+  - name: verify supervisor running
+    shell: nc -z 127.0.0.1 9004; echo $? executable=/bin/bash
+    register: supervisor_running
+    changed_when: false
+
+  - name: start supervisord
+    shell: source ~/svenv/bin/activate && supervisord executable=/bin/bash
+    register: start_supervisor
+    when: supervisord.changed or supervisor_running.stdout == "1"
+    #changed_when: "'already listening' not in start_supervisor.stdout"
+
+  - name: activate supervisord changes
+    when: supervisord.changed
+    shell: source ~/svenv/bin/activate && supervisorctl reload executable=/bin/bash
+
+  - name: idlebot started
+    supervisorctl: name=idlebot state=restarted supervisorctl_path=~/svenv/bin/supervisorctl
+    when: (source_code.changed or urlbot_config.changed) and not supervisord.changed
+
+  - pause: seconds=30
+    when: (source_code.changed or urlbot_config.changed) and not supervisord.changed
+
+  - name: urlbot started
+    supervisorctl: name=bot state=restarted supervisorctl_path=~/svenv/bin/supervisorctl
+    when: (source_code.changed or urlbot_config.changed) and not supervisord.changed

deploy/hosts

@@ -0,0 +1,2 @@
+[bots]
+aero2k.de ansible_host=2a01:4f8:d16:130c::2

deploy/supervisord.conf

@@ -0,0 +1,46 @@
+; Sample supervisor config file.
+;
+; For more information on the config file, please see:
+; http://supervisord.org/configuration.html
+;
+; Notes:
+;  - Shell expansion ("~" or "$HOME") is not supported.  Environment
+;    variables can be expanded using this syntax: "%(ENV_HOME)s".
+;  - Comments must have a leading space: "a=b ;comment" not "a=b;comment".
+
+[unix_http_server]
+file=/home/jabberbot/supervisor.sock   ; (the path to the socket file)
+
+[inet_http_server]
+port=127.0.0.1:9004
+
+[supervisord]
+logfile=/home/jabberbot/supervisord.log ; (main log file;default $CWD/supervisord.log)
+logfile_maxbytes=50MB        ; (max main logfile bytes b4 rotation;default 50MB)
+logfile_backups=10           ; (num of main logfile rotation backups;default 10)
+loglevel=info                ; (log level;default info; others: debug,warn,trace)
+pidfile=/home/jabberbot/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+nodaemon=false               ; (start in foreground if true;default false)
+minfds=1024                  ; (min. avail startup file descriptors;default 1024)
+minprocs=200                 ; (min. avail process descriptors;default 200)
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///home/jabberbot/supervisor.sock ; use a unix:// URL  for a unix socket
+
+;[include]
+;files = relative/directory/*.ini
+
+[program:bot]
+command=/home/jabberbot/botenv/bin/python3 /home/jabberbot/urlbot/urlbot.py
+directory=/home/jabberbot/urlbot/
+stderr_logfile=/home/jabberbot/urlbot.err
+stdout_logfile=/home/jabberbot/urlbot.log
+
+[program:idlebot]
+directory=/home/jabberbot/urlbot/
+command=/home/jabberbot/botenv/bin/python3 /home/jabberbot/urlbot/idlebot.py
+stderr_logfile=/home/jabberbot/idlebot.err
+stdout_logfile=/home/jabberbot/idlebot.log

related/bot.py

@@ -1,80 +0,0 @@
-#!/usr/bin/python3
-
-import logging
-
-from sleekxmpp import ClientXMPP
-
-try:
-	from local_config import conf
-except ImportError:
-	import sys
-	sys.stderr.write('''
-%s: E: local_config.py isn't tracked because of included secrets and
-%s     site specific configurations. Rename local_config.py.skel and
-%s     adjust to you needs.
-'''[1:] % (
-		sys.argv[0],
-		' ' * len(sys.argv[0]),
-		' ' * len(sys.argv[0])
-	)
-	)
-
-	sys.exit(10)
-
-import time
-t = -time.time()
-
-
-class Bot(ClientXMPP):
-	def __init__(self, jid, password, room, nick):
-		ClientXMPP.__init__(self, jid, password)
-
-		self.room = room
-		self.nick = nick
-
-		self.add_event_handler('session_start', self.session_start)
-		self.add_event_handler('groupchat_message', self.muc_message)
-
-	def session_start(self, event):
-		self.get_roster()
-		self.send_presence()
-
-		self.plugin['xep_0045'].joinMUC(
-			self.room,
-			self.nick,
-			wait=True
-		)
-
-	def muc_message(self, msg):
-		print(msg['mucnick'])
-		print(msg['body'])
-		print((msg['from'], msg['from'].bare))
-
-		print(conf('room') == msg['from'].bare)
-
-		# don't talk to yourself
-		if msg['mucnick'] == self.nick:
-			return
-
-		self.send_message(
-			mto=msg['from'].bare,
-			mbody='got[%s]' % msg['body'],
-			mtype='groupchat'
-		)
-
-if '__main__' == __name__:
-	logging.basicConfig(
-		level=logging.DEBUG,
-		format='%(levelname)-8s %(message)s'
-	)
-
-	xmpp = Bot(
-		jid=conf('jid'),
-		password=conf('password'),
-		room=conf('room'),
-		nick=conf('bot_user')
-	)
-
-	xmpp.connect()
-	xmpp.register_plugin('xep_0045')
-	xmpp.process(threaded=False)

related/levenshtein_test.py

@@ -1,38 +0,0 @@
-#!/usr/bin/python3
-
-from common import levenshtein
-
-(a, b) = ('foo barbaz', 'foobar baz')
-(a, b) = ('sitting', 'kitten')
-(a, b) = ('Monte Kali (Heringen)', 'http://de.wikipedia.org/wiki/Monte_Kali_%28Heringen%29')
-
-(matrix, ret) = levenshtein(a, b, return_table=True)
-
-sep = ' '*0
-out = ''
-for B in b:
-	out += sep + '%2s' % B
-print(sep + ' '*4 + out)
-
-for i in range(len(matrix)):
-	if 0 == i:
-		out = '  '
-	else:
-		out = '%2s' % a[i-1]
-
-	for j in range(len(matrix[i])):
-		if 0 == i or 0 == j:
-			col = '30;42'
-		elif i == j:
-			col = '41'
-		else:
-			col = 0
-
-		if 0 != col:
-			out += sep + '\x1b[%sm%2d\x1b[m' %(col, matrix[i][j])
-		else:
-			out += sep + '%2d' % matrix[i][j]
-
-	print(out)
-
-print(ret)

related/strsim.py

@@ -1,69 +0,0 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
-
-import re
-
-def str_sim(a, b, do_print=False):
-	a = a.lower()
-	b = b.lower()
-
-	a_parts = re.split('[\W_]+', a)
-	b_parts = re.split('[\W_]+', b)
-
-	# this is a "simple" way to declare out[a][b]
-	out = list(map(list, [[0]*len(b_parts)]*len(a_parts)))
-
-	for i in range(0, len(a_parts)-1):
-		for j in range(0, len(b_parts)-1):
-			if a_parts[i] == b_parts[j]:
-				out[i][j] += 1
-
-	if do_print:
-		i = 0
-		for j in range(0, len(b_parts)):
-			print('  |'*i + ' '*2 + '.- ' + b_parts[j])
-			i += 1
-		print('  |'*i)
-
-		for i in range(0, len(a_parts)):
-			print(' ' + str(out[i]) + ' ' + a_parts[i])
-
-	return out
-
-def sum_array(array):
-	_sum = 0
-	for a in array:
-		if list == type(a) or tuple == type(a) or hash == type(a):
-			_sum += sum_array(a)
-		elif int == type(a) or float == type(a):
-			_sum += a
-	return _sum
-
-def wrapper_print(a, b, comment=''):
-	ret = str_sim(a, b, do_print=True)
-
-	if '' != comment:
-		comment = ' ^ ' + comment
-
-	print('[%2dx%2d::%2d]%s' %(len(ret), len(ret[0]), sum_array(ret), comment))
-
-if '__main__' == __name__:
-	pairs = (
-		(
-			'http://de.wikipedia.org/wiki/Monte_Kali_%28Heringen%29',
-			'Monte Kali (Heringen)'
-		),
-		(
-			'http://www.spiegel.de/politik/ausland/buddhisten-treffen-in-colombo-blitzender-moench-a-994447.html',
-			'Buddhisten-Treffen in Colombo: Blitzender Mönch - SPIEGEL ONLINE'
-		)
-	)
-
-	wrapper_print('foo bar baz', 'foo bar boom')
-
-	for (url, title) in pairs:
-		wrapper_print(title, url, comment='raw')
-		url_no_proto = re.sub(r'https?://[^/]*/', '', url)
-		wrapper_print(title, url_no_proto, comment='no proto/domain')
-		url_no_proto_no_digits = re.sub(r'[0-9]*', '', url_no_proto)
-		wrapper_print(title, url_no_proto_no_digits, comment='no proto/domain/[0-9]')