mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-14 23:24:30 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

15042020

Browse source
This commit is contained in:
misha 2020-04-15 11:54:53 +03:00
parent bb580533f7
commit 1aa2a5b069
3 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
archivebox/archive_methods.py
View file

@ -30,6 +30,7 @@ from config import (
OUTPUT_DIR,
GIT_DOMAINS,
GIT_SHA,
RESTRICT_FILE_NAMES,
CURL_USER_AGENT,
WGET_USER_AGENT,
CHECK_SSL_VALIDITY,
@ -227,7 +228,7 @@ def fetch_wget(link_dir, link, timeout=TIMEOUT):
'--span-hosts',
'--no-parent',
'-e', 'robots=off',
'--restrict-file-names=nocontrol',
*(('--restrict-file-names={}'.format(RESTRICT_FILE_NAMES),) if RESTRICT_FILE_NAMES else ()),
'--timeout={}'.format(timeout),
*(('--compression=auto',) if WGET_AUTO_COMPRESSION else ()),
*(() if FETCH_WARC else ('--timestamping',)),