mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-14 07:04:27 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: Improve headers handling

Browse source
This commit is contained in:
Cristian 2020-09-24 08:37:27 -05:00 • committed by Cristian Vargas
parent a40af98ced
commit 62ed11a5ca
6 changed files with 60 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
archivebox/util.py
View file

@ -186,13 +186,17 @@ def get_headers(url: str, timeout: int=None) -> str:
headers={'User-Agent': WGET_USER_AGENT},
verify=CHECK_SSL_VALIDITY,
timeout=timeout,
allow_redirects=True
)
if response.status_code >= 400:
raise RequestException
except RequestException:
response = requests.get(
url,
headers={'User-Agent': WGET_USER_AGENT},
verify=CHECK_SSL_VALIDITY,
timeout=timeout,
stream=True
)
return pyjson.dumps(dict(response.headers), indent=4)