catdog2/urlbot-native
1
0
Fork 0
Code Issues Pull Requests Releases Activity

add send error replys from urllib

Browse Source
This commit is contained in:
urlbot
2014-10-01 10:21:27 +02:00
parent 351da575db
commit 6ce9e177de
1 changed files with 9 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
urlbot.py
View File

@@ -22,11 +22,12 @@ def fetch_page(url):
response = urllib.request.urlopen(url) response = urllib.request.urlopen(url)
html_text = response.read(BUFSIZ) # ignore more than BUFSIZ html_text = response.read(BUFSIZ) # ignore more than BUFSIZ
response.close() response.close()
return (html_text, response.headers) return (0, html_text, response.headers)
except IOError as e: except IOError as e:
logger('warn', 'failed: ' + str(e.errno)) logger('warn', 'failed: ' + str(e))
return (1, str(e), 'dummy')
return (None, None) return (-1, None, None)
def extract_title(url): def extract_title(url):
global parser global parser
@@ -37,7 +38,11 @@ def extract_title(url):
logger('info', 'extracting title from ' + url) logger('info', 'extracting title from ' + url)
(html_text, headers) = fetch_page(url) (code, html_text, headers) = fetch_page(url)
if 1 == code:
return (3, 'failed: %s for %s' %(html_text, url))
if html_text: if html_text:
charset = '' charset = ''
if 'content-type' in headers: if 'content-type' in headers: