catdog2/urlbot-native
1
0
Fork 0
Code Issues Pull Requests Releases Activity

levenshtein: strip domain; write to persistent struct

Browse Source
This commit is contained in:
urlbot
2014-09-30 17:39:52 +02:00
parent 5a7601b108
commit 126b238aeb
1 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
urlbot.py
View File

@@ -133,9 +133,16 @@ def extract_url(data):
(status, title) = extract_title(r)
if 0 == status:
message = 'lev=%d/%d:%d Title: %s: %s' %(
levenshtein(r, title.strip()), len(title.strip()), len(r), title.strip(), r
)
title = title.strip()
lev_url = re.sub(r'https?://[^/]*/', '', r)
lev_res = levenshtein(lev_url, title)
obj = conf_load()
obj['lev'].append((lev_res, title, lev_url))
conf_save(obj)
lev_str = 'lev=%d/%d:%d ' %(lev_res, len(title), len(lev_url))
message = lev_str + 'Title: %s: %s' %(title, r)
elif 1 == status:
logger('info', 'no message sent for non-text %s (%s)' %(r, title))
continue