catdog2/urlbot-native-trex
1
0
Fork 0
mirror of http://aero2k.de/t/repos/urlbot-native.git synced 2017-09-06 15:25:38 +02:00
Code Issues Releases Activity

Python3 post-migration: fix User-Agent

Browse Source
This commit is contained in:
urlbot
2014-10-05 13:58:44 +02:00
parent 6d11929e54
commit 09f96101f0
1 changed files with 3 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
urlbot.py
View File

@@ -12,14 +12,12 @@ hist_flag = True
parser = None
class urllib_user_agent_wrapper(urllib.request.FancyURLopener):
version = '''Mozilla/5.0 (X11; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0 Iceweasel/31.0'''
def fetch_page(url):
logger('info', 'fetching page ' + url)
try:
urllib.request._urlopener = urllib_user_agent_wrapper()
response = urllib.request.urlopen(url)
request = urllib.request.Request(url)
request.add_header('User-Agent', '''Mozilla/5.0 (X11; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0 Iceweasel/31.0''')
response = urllib.request.urlopen(request)
html_text = response.read(BUFSIZ) # ignore more than BUFSIZ
response.close()
return (0, html_text, response.headers)