catdog2/urlbot-native-trex
1
0
Fork 0
mirror of http://aero2k.de/t/repos/urlbot-native.git synced 2017-09-06 15:25:38 +02:00
Code Issues Releases Activity

schei? encoding - doing workarounds

Browse Source
This commit is contained in:
Thorsten
2016-01-04 22:42:18 +01:00
parent 98d5c82d82
commit 3480491013
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common.py
View File

@@ -127,7 +127,7 @@ def fetch_page(url):
log.info('fetching page ' + url)
response = requests.get(url, headers={'User-Agent': USER_AGENT}, stream=True)
content = response.raw.read(BUFSIZ, decode_content=True)
return content.decode(response.encoding), response.headers
return content.decode(response.encoding or 'utf-8'), response.headers
def extract_title(url):