mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-16 08:04:26 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

make URL_BLACKLIST empty by default

Browse source
This commit is contained in:
Nick Sweeting 2019-03-30 14:56:19 -04:00 • committed by GitHub
parent 8502fa5cc3
commit 066b36b6a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

12
archivebox/config.py
View file

@ -47,7 +47,7 @@ WGET_BINARY = os.getenv('WGET_BINARY', 'wget')
YOUTUBEDL_BINARY = os.getenv('YOUTUBEDL_BINARY', 'youtube-dl') YOUTUBEDL_BINARY = os.getenv('YOUTUBEDL_BINARY', 'youtube-dl')
CHROME_BINARY = os.getenv('CHROME_BINARY', None) CHROME_BINARY = os.getenv('CHROME_BINARY', None)
URL_BLACKLIST = os.getenv('URL_BLACKLIST', '.*youtube.com.*,.*facebook.com/.*,.*.exe') URL_BLACKLIST = os.getenv('URL_BLACKLIST', None)
try: try:
OUTPUT_DIR = os.path.abspath(os.getenv('OUTPUT_DIR')) OUTPUT_DIR = os.path.abspath(os.getenv('OUTPUT_DIR'))
@ -76,6 +76,8 @@ USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM
USE_WGET = FETCH_WGET or FETCH_WGET_REQUISITES or FETCH_WARC USE_WGET = FETCH_WGET or FETCH_WGET_REQUISITES or FETCH_WARC
WGET_AUTO_COMPRESSION = USE_WGET and WGET_BINARY and (not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode) WGET_AUTO_COMPRESSION = USE_WGET and WGET_BINARY and (not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode)
URL_BLACKLIST = URL_BLACKLIST and re.compile(URL_BLACKLIST, re.IGNORECASE)
########################### Environment & Dependencies ######################### ########################### Environment & Dependencies #########################
try: try:
@ -268,11 +270,3 @@ except KeyboardInterrupt:
except: except:
print('[X] There was an error during the startup procedure, your archive data is unaffected.') print('[X] There was an error during the startup procedure, your archive data is unaffected.')
raise raise
URL_BLACKLIST = re.compile(
r'(.*\.youtube\.com)|'
r'(.*\.facebook\.com)|'
r'(.*\.amazon\.com)|'
r'(.*\.reddit\.com)',
re.IGNORECASE,
)