mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 06:34:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: Use CURL_ARGS on title extractor

Browse source
This commit is contained in:
Cristian 2020-10-15 08:58:22 -05:00 • committed by Cristian Vargas
parent 563d0f94ec
commit c12fe0e3d7
4 changed files with 8 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

3
archivebox/config/__init__.py
View file

@ -134,7 +134,8 @@ CONFIG_DEFAULTS: Dict[str, ConfigDefaultDict] = {
'CURL_ARGS': {'type': list, 'default': ['--silent', 'CURL_ARGS': {'type': list, 'default': ['--silent',
'--location', '--location',
'--compressed' '--compressed'
]} ]},
'GIT_ARGS': {'type': list, 'default': ['--recursive']},
}, },
'DEPENDENCY_CONFIG': { 'DEPENDENCY_CONFIG': {

2
archivebox/config/stubs.py
View file

@ -96,6 +96,8 @@ class ConfigDict(BaseConfig, total=False):
YOUTUBEDL_ARGS: Optional[str] YOUTUBEDL_ARGS: Optional[str]
WGET_ARGS: Optional[str] WGET_ARGS: Optional[str]
CURL_ARGS: Optional[str]
GIT_ARGS: Optional[str]
ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue] ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue]

3
archivebox/extractors/git.py
View file

@ -18,6 +18,7 @@ from ..config import (
TIMEOUT, TIMEOUT,
SAVE_GIT, SAVE_GIT,
GIT_BINARY, GIT_BINARY,
GIT_ARGS,
GIT_VERSION, GIT_VERSION,
GIT_DOMAINS, GIT_DOMAINS,
CHECK_SSL_VALIDITY CHECK_SSL_VALIDITY
@ -56,7 +57,7 @@ def save_git(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
cmd = [ cmd = [
GIT_BINARY, GIT_BINARY,
'clone', 'clone',
'--recursive', *GIT_ARGS,
*([] if CHECK_SSL_VALIDITY else ['-c', 'http.sslVerify=false']), *([] if CHECK_SSL_VALIDITY else ['-c', 'http.sslVerify=false']),
without_query(without_fragment(link.url)), without_query(without_fragment(link.url)),
] ]

5
archivebox/extractors/title.py
View file

@ -16,6 +16,7 @@ from ..config import (
CHECK_SSL_VALIDITY, CHECK_SSL_VALIDITY,
SAVE_TITLE, SAVE_TITLE,
CURL_BINARY, CURL_BINARY,
CURL_ARGS,
CURL_VERSION, CURL_VERSION,
CURL_USER_AGENT, CURL_USER_AGENT,
setup_django, setup_django,
@ -51,10 +52,8 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -
output: ArchiveOutput = None output: ArchiveOutput = None
cmd = [ cmd = [
CURL_BINARY, CURL_BINARY,
'--silent', *CURL_ARGS,
'--max-time', str(timeout), '--max-time', str(timeout),
'--location',
'--compressed',
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []), *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']), *([] if CHECK_SSL_VALIDITY else ['--insecure']),
link.url, link.url,