mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 06:34:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: Use CURL_ARGS in favicon extractor

Browse source
This commit is contained in:
Cristian 2020-10-15 08:49:54 -05:00 • committed by Cristian Vargas
parent 2e1cdca789
commit 563d0f94ec
4 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

1
archivebox/config/__init__.py
View file

@ -133,7 +133,6 @@ CONFIG_DEFAULTS: Dict[str, ConfigDefaultDict] = {
]}, ]},
'CURL_ARGS': {'type': list, 'default': ['--silent', 'CURL_ARGS': {'type': list, 'default': ['--silent',
'--location', '--location',
'--head',
'--compressed' '--compressed'
]} ]}
}, },

1
archivebox/extractors/archive_org.py
View file

@ -47,6 +47,7 @@ def save_archive_dot_org(link: Link, out_dir: Optional[Path]=None, timeout: int=
cmd = [ cmd = [
CURL_BINARY, CURL_BINARY,
*CURL_ARGS, *CURL_ARGS,
'--head',
'--max-time', str(timeout), '--max-time', str(timeout),
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []), *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']), *([] if CHECK_SSL_VALIDITY else ['--insecure']),

5
archivebox/extractors/favicon.py
View file

@ -11,6 +11,7 @@ from ..config import (
TIMEOUT, TIMEOUT,
SAVE_FAVICON, SAVE_FAVICON,
CURL_BINARY, CURL_BINARY,
CURL_ARGS,
CURL_VERSION, CURL_VERSION,
CHECK_SSL_VALIDITY, CHECK_SSL_VALIDITY,
CURL_USER_AGENT, CURL_USER_AGENT,
@ -34,10 +35,8 @@ def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
output: ArchiveOutput = 'favicon.ico' output: ArchiveOutput = 'favicon.ico'
cmd = [ cmd = [
CURL_BINARY, CURL_BINARY,
'--silent', *CURL_ARGS,
'--max-time', str(timeout), '--max-time', str(timeout),
'--location',
'--compressed',
'--output', str(output), '--output', str(output),
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []), *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']), *([] if CHECK_SSL_VALIDITY else ['--insecure']),

1
archivebox/extractors/headers.py
View file

@ -43,6 +43,7 @@ def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT)
cmd = [ cmd = [
CURL_BINARY, CURL_BINARY,
*CURL_ARGS, *CURL_ARGS,
'--head',
'--max-time', str(timeout), '--max-time', str(timeout),
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []), *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']), *([] if CHECK_SSL_VALIDITY else ['--insecure']),