added some more error handling because error handling is cool

This commit is contained in:
chat
2014-07-21 02:58:29 +02:00
parent b8cf1a0d58
commit 8aa8485621

View File

@@ -29,14 +29,19 @@ def logger(severity, message):
def fetch_page(url):
logger('info', 'fetching page ' + url)
try:
response = urllib.urlopen(url)
html = response.read(BUFSIZ)
response.close()
return html
except IOError as e:
logger('warn', 'failed: ' + e.errno)
def extract_title(url):
logger('info', 'extracting title from ' + url)
html = fetch_page(url)
if html:
result = re.match(r'.*?<title.*?>(.*?)</title>.*?', html, re.S|re.M)
if result:
return result.groups()[0]
@@ -55,7 +60,11 @@ def extract_url(data):
for r in result:
title = extract_title(r)
if title:
message = 'Title: %s: %s' % (title, e(r))
else:
message = 'some error occured when fetching %s' % e(r)
logger('info', 'printing ' + message)
if debug_enabled():