From 2e1cdca7890b83b8d1cdc4ef9f505857d104af3f Mon Sep 17 00:00:00 2001
From: Cristian <cristian@swapps.com>
Date: Thu, 15 Oct 2020 08:47:32 -0500
Subject: [PATCH] feat: Use CURL_ARGS on header extractor
---
archivebox/extractors/headers.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/archivebox/extractors/headers.py b/archivebox/extractors/headers.py
index 533e6bb3..0cc366e3 100644
--- a/archivebox/extractors/headers.py
+++ b/archivebox/extractors/headers.py
@@ -13,6 +13,7 @@ from ..util import (
from ..config import (
TIMEOUT,
CURL_BINARY,
+ CURL_ARGS,
CURL_USER_AGENT,
CURL_VERSION,
CHECK_SSL_VALIDITY,
@@ -41,11 +42,8 @@ def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT)
cmd = [
CURL_BINARY,
- '--silent',
+ *CURL_ARGS,
'--max-time', str(timeout),
- '--location',
- '--compressed',
- '--head',
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']),
link.url,