mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-15 07:34:27 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into dev

Browse source
This commit is contained in:
Nick Sweeting 2019-03-27 10:36:29 -04:00
commit 7ae680a461
2 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
archivebox/archive_methods.py
View file

@ -41,6 +41,7 @@ from config import (
GIT_VERSION, GIT_VERSION,
YOUTUBEDL_VERSION, YOUTUBEDL_VERSION,
ONLY_NEW, ONLY_NEW,
WGET_AUTO_COMPRESSION,
) )
from util import ( from util import (
enforce_types, enforce_types,
@ -251,7 +252,6 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult
'--backup-converted', '--backup-converted',
'--span-hosts', '--span-hosts',
'--no-parent', '--no-parent',
'--compression=auto',
'-e', 'robots=off', '-e', 'robots=off',
'--restrict-file-names=unix', '--restrict-file-names=unix',
'--timeout={}'.format(timeout), '--timeout={}'.format(timeout),
@ -260,6 +260,7 @@ def fetch_wget(link_dir: str, link: Link, timeout: int=TIMEOUT) -> ArchiveResult
*(('--page-requisites',) if FETCH_WGET_REQUISITES else ()), *(('--page-requisites',) if FETCH_WGET_REQUISITES else ()),
*(('--user-agent={}'.format(WGET_USER_AGENT),) if WGET_USER_AGENT else ()), *(('--user-agent={}'.format(WGET_USER_AGENT),) if WGET_USER_AGENT else ()),
*(('--load-cookies', COOKIES_FILE) if COOKIES_FILE else ()), *(('--load-cookies', COOKIES_FILE) if COOKIES_FILE else ()),
*(('--compression=auto',) if WGET_AUTO_COMPRESSION else ()),
*((() if CHECK_SSL_VALIDITY else ('--no-check-certificate', '--no-hsts'))), *((() if CHECK_SSL_VALIDITY else ('--no-check-certificate', '--no-hsts'))),
link.url, link.url,
] ]

2
archivebox/config.py
View file

@ -213,8 +213,10 @@ try:
else: else:
FETCH_WGET = FETCH_WARC = False FETCH_WGET = FETCH_WARC = False
WGET_VERSION = None WGET_VERSION = None
WGET_AUTO_COMPRESSION = False
if USE_WGET: if USE_WGET:
WGET_VERSION = check_version(WGET_BINARY) WGET_VERSION = check_version(WGET_BINARY)
WGET_AUTO_COMPRESSION = not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode
WGET_USER_AGENT = WGET_USER_AGENT.format( WGET_USER_AGENT = WGET_USER_AGENT.format(
GIT_SHA=GIT_SHA[:9], GIT_SHA=GIT_SHA[:9],