mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-20 18:05:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

accept gzipped responses when using curl

Browse source
This commit is contained in:
Nick Sweeting 2020-07-28 05:55:54 -04:00
parent af9084ee95
commit 273059f054
3 changed files with 6 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

1
archivebox/extractors/archive_org.py
View file

@ -48,6 +48,7 @@ def save_archive_dot_org(link: Link, out_dir: Optional[str]=None, timeout: int=T
'--silent', '--silent',
'--location', '--location',
'--head', '--head',
'--compressed',
'--max-time', str(timeout), '--max-time', str(timeout),
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []), *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']), *([] if CHECK_SSL_VALIDITY else ['--insecure']),

1
archivebox/extractors/favicon.py
View file

@ -37,6 +37,7 @@ def save_favicon(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT)
'--silent', '--silent',
'--max-time', str(timeout), '--max-time', str(timeout),
'--location', '--location',
'--compressed',
'--output', str(output), '--output', str(output),
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []), *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']), *([] if CHECK_SSL_VALIDITY else ['--insecure']),

7
archivebox/extractors/title.py
View file

@ -43,18 +43,19 @@ def should_save_title(link: Link, out_dir: Optional[str]=None) -> bool:
def save_title(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult: def save_title(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
"""try to guess the page's title from its content""" """try to guess the page's title from its content"""
setup_django(out_dir=out_dir)
from core.models import Snapshot
output: ArchiveOutput = None output: ArchiveOutput = None
cmd = [ cmd = [
CURL_BINARY, CURL_BINARY,
'--silent', '--silent',
'--max-time', str(timeout), '--max-time', str(timeout),
'--location', '--location',
'--compressed',
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []), *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']), *([] if CHECK_SSL_VALIDITY else ['--insecure']),
link.url, link.url,
'|',
'grep',
'<title',
] ]
status = 'succeeded' status = 'succeeded'
timer = TimedProgress(timeout, prefix=' ') timer = TimedProgress(timeout, prefix=' ')