mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-14 07:04:27 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: Use CURL_ARGS on header extractor

Browse source
This commit is contained in:
Cristian 2020-10-15 08:47:32 -05:00 • committed by Cristian Vargas
parent 972d57bd08
commit 2e1cdca789
1 changed files with 2 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

6
archivebox/extractors/headers.py
View file

@ -13,6 +13,7 @@ from ..util import (
from ..config import ( from ..config import (
TIMEOUT, TIMEOUT,
CURL_BINARY, CURL_BINARY,
CURL_ARGS,
CURL_USER_AGENT, CURL_USER_AGENT,
CURL_VERSION, CURL_VERSION,
CHECK_SSL_VALIDITY, CHECK_SSL_VALIDITY,
@ -41,11 +42,8 @@ def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT)
cmd = [ cmd = [
CURL_BINARY, CURL_BINARY,
'--silent', *CURL_ARGS,
'--max-time', str(timeout), '--max-time', str(timeout),
'--location',
'--compressed',
'--head',
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []), *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']), *([] if CHECK_SSL_VALIDITY else ['--insecure']),
link.url, link.url,