From 7fd7dced9a7b2fe1412432f4731465286407f6d9 Mon Sep 17 00:00:00 2001 From: Angel Rey Date: Fri, 11 Sep 2020 14:28:04 -0500 Subject: [PATCH] Added curl params --- archivebox/extractors/headers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/archivebox/extractors/headers.py b/archivebox/extractors/headers.py index 9987d4ed..90a9af0d 100644 --- a/archivebox/extractors/headers.py +++ b/archivebox/extractors/headers.py @@ -46,10 +46,11 @@ def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) cmd = [ CURL_BINARY, - '-s', - '-I', - '-X', - '-D', + '--silent', + '--max-time', str(timeout), + '--location', + '--compressed', + '--head', *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []), *([] if CHECK_SSL_VALIDITY else ['--insecure']), link.url,