mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-12 22:25:44 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Flip dedupe precedence order

Browse source
This commit is contained in:
Ben Muthalaly 2024-03-01 14:50:32 -06:00
parent 4d9c5a7b4b
commit d74ddd42ae
8 changed files with 33 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

6
archivebox/extractors/wget.py
View file

@ -57,8 +57,10 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
# WGET CLI Docs: https://www.gnu.org/software/wget/manual/wget.html
output: ArchiveOutput = None
# earlier options take precedence
# later options take precedence
options = [
*WGET_ARGS,
*WGET_EXTRA_ARGS,
'--timeout={}'.format(timeout),
*(['--restrict-file-names={}'.format(RESTRICT_FILE_NAMES)] if RESTRICT_FILE_NAMES else []),
*(['--warc-file={}'.format(str(warc_path))] if SAVE_WARC else []),
@ -69,8 +71,6 @@ def save_wget(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
*([] if SAVE_WARC else ['--timestamping']),
*([] if CHECK_SSL_VALIDITY else ['--no-check-certificate', '--no-hsts']),
# '--server-response', # print headers for better error parsing
*WGET_EXTRA_ARGS,
*WGET_ARGS,
]
cmd = [
WGET_BINARY,