mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-14 23:24:30 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

05042020

Browse source
This commit is contained in:
misha 2020-04-05 18:22:59 +03:00
parent 83197ef88e
commit 18f0f66f1e
3 changed files with 8 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
archivebox/archive_methods.py
View file

@ -30,6 +30,7 @@ from config import (
OUTPUT_DIR,
GIT_DOMAINS,
GIT_SHA,
CURL_USER_AGENT,
WGET_USER_AGENT,
CHECK_SSL_VALIDITY,
COOKIES_FILE,
@ -226,7 +227,7 @@ def fetch_wget(link_dir, link, timeout=TIMEOUT):
'--span-hosts',
'--no-parent',
'-e', 'robots=off',
'--restrict-file-names=windows',
'--restrict-file-names=nocontrol',
'--timeout={}'.format(timeout),
*(('--compression=auto',) if WGET_AUTO_COMPRESSION else ()),
*(() if FETCH_WARC else ('--timestamping',)),
@ -561,7 +562,7 @@ def archive_dot_org(link_dir, link, timeout=TIMEOUT):
CURL_BINARY,
'--location',
'--head',
'--user-agent', 'ArchiveBox/{} (+https://github.com/pirate/ArchiveBox/)'.format(GIT_SHA), # be nice to the Archive.org people and show them where all this ArchiveBox traffic is coming from
*(('--user-agent', '{}'.format(CURL_USER_AGENT),) if CURL_USER_AGENT else ()), # be nice to the Archive.org people and show them where all this ArchiveBox traffic is coming from
'--max-time', str(timeout),
*(() if CHECK_SSL_VALIDITY else ('--insecure',)),
submit_url,