catdog2/urlbot-native
1
0
Fork 0
Code Issues Pull Requests Releases Activity

schei? encoding - doing workarounds

Browse Source
This commit is contained in:
Thorsten
2016-01-04 22:42:18 +01:00
parent 98d5c82d82
commit 3480491013
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common.py
View File

@@ -127,7 +127,7 @@ def fetch_page(url):
log.info('fetching page ' + url) log.info('fetching page ' + url)
response = requests.get(url, headers={'User-Agent': USER_AGENT}, stream=True) response = requests.get(url, headers={'User-Agent': USER_AGENT}, stream=True)
content = response.raw.read(BUFSIZ, decode_content=True) content = response.raw.read(BUFSIZ, decode_content=True)
return content.decode(response.encoding), response.headers return content.decode(response.encoding or 'utf-8'), response.headers
def extract_title(url): def extract_title(url):