Merge branch 'master' into django

archivebox/extractors/favicon.py

@@ -13,6 +13,7 @@ from ..config import (
     CURL_BINARY,
     CURL_VERSION,
     CHECK_SSL_VALIDITY,
+    CURL_USER_AGENT,
 )
 from ..cli.logging import TimedProgress
 
@@ -37,14 +38,16 @@ def save_favicon(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT)
         '--max-time', str(timeout),
         '--location',
         '--output', str(output),
+        *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else [],
         *([] if CHECK_SSL_VALIDITY else ['--insecure']),
         'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)),
     ]
-    status = 'succeeded'
+    status = 'pending'
     timer = TimedProgress(timeout, prefix='      ')
     try:
         run(cmd, stdout=PIPE, stderr=PIPE, cwd=out_dir, timeout=timeout)
         chmod_file(output, cwd=out_dir)
+        status = 'succeeded'
     except Exception as err:
         status = 'failed'
         output = err