mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-25 04:04:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #181 from n0ncetonic/feature/chrome_user-agent

Browse source 
Feature/chrome user agent
This commit is contained in:
Nick Sweeting 2019-03-19 11:16:18 -04:00 • committed by GitHub
commit 7acc12ded0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 6 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
archivebox/archive_methods.py
View file

@ -30,6 +30,7 @@ from config import (
SUBMIT_ARCHIVE_DOT_ORG, SUBMIT_ARCHIVE_DOT_ORG,
COOKIES_FILE, COOKIES_FILE,
WGET_USER_AGENT, WGET_USER_AGENT,
CHROME_USER_AGENT,
CHROME_USER_DATA_DIR, CHROME_USER_DATA_DIR,
CHROME_HEADLESS, CHROME_HEADLESS,
CHROME_SANDBOX, CHROME_SANDBOX,
@ -266,6 +267,7 @@ def fetch_pdf(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DI
'--hide-scrollbars', '--hide-scrollbars',
'--timeout={}'.format((timeout) * 1000), '--timeout={}'.format((timeout) * 1000),
*(() if CHECK_SSL_VALIDITY else ('--disable-web-security', '--ignore-certificate-errors')), *(() if CHECK_SSL_VALIDITY else ('--disable-web-security', '--ignore-certificate-errors')),
*(('--user-agent={}'.format(CHROME_USER_AGENT),) if CHROME_USER_AGENT else ()),
link['url'] link['url']
] ]
end = progress(timeout, prefix=' ') end = progress(timeout, prefix=' ')
@ -304,6 +306,7 @@ def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_
'--hide-scrollbars', '--hide-scrollbars',
'--timeout={}'.format((timeout) * 1000), '--timeout={}'.format((timeout) * 1000),
*(() if CHECK_SSL_VALIDITY else ('--disable-web-security', '--ignore-certificate-errors')), *(() if CHECK_SSL_VALIDITY else ('--disable-web-security', '--ignore-certificate-errors')),
*(('--user-agent={}'.format(CHROME_USER_AGENT),) if CHROME_USER_AGENT else ()),
# '--full-page', # TODO: make this actually work using ./bin/screenshot fullPage: true # '--full-page', # TODO: make this actually work using ./bin/screenshot fullPage: true
link['url'], link['url'],
] ]
@ -342,6 +345,7 @@ def fetch_dom(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DI
*chrome_headless(user_data_dir=user_data_dir), *chrome_headless(user_data_dir=user_data_dir),
'--dump-dom', '--dump-dom',
'--timeout={}'.format((timeout) * 1000), '--timeout={}'.format((timeout) * 1000),
*(('--user-agent={}'.format(CHROME_USER_AGENT),) if CHROME_USER_AGENT else ()),
link['url'] link['url']
] ]
end = progress(timeout, prefix=' ') end = progress(timeout, prefix=' ')

3
etc/ArchiveBox.conf.default
View file

@ -40,7 +40,8 @@
#CHECK_SSL_VALIDITY=True #CHECK_SSL_VALIDITY=True
#FETCH_WGET_REQUISITES=True #FETCH_WGET_REQUISITES=True
#RESOLUTION="1440,900" #RESOLUTION="1440,900"
#WGET_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36" #WGET_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36"
#CHROME_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36"
#GIT_DOMAINS="github.com,bitbucket.org,gitlab.com" #GIT_DOMAINS="github.com,bitbucket.org,gitlab.com"
#COOKIES_FILE="path/to/cookies.txt" #COOKIES_FILE="path/to/cookies.txt"
#CHROME_USER_DATA_DIR="~/.config/google-chrome/Default" #CHROME_USER_DATA_DIR="~/.config/google-chrome/Default"