mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-14 07:04:27 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

ignore robots.txt when using wget

Browse source
This commit is contained in:
Nick Sweeting 2019-02-06 22:06:28 -08:00
parent 56d382235f
commit e6d5cd4432
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
archivebox/archive_methods.py
View file

@ -217,6 +217,7 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, warc=FETCH_WARC
'--backup-converted', '--backup-converted',
'--span-hosts', '--span-hosts',
'--no-parent', '--no-parent',
'-e', 'robots=off',
'--restrict-file-names=unix', '--restrict-file-names=unix',
'--timeout={}'.format(timeout), '--timeout={}'.format(timeout),
*(() if warc else ('--timestamping',)), *(() if warc else ('--timestamping',)),