mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 06:34:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix archive_org header rename

Browse source
This commit is contained in:
Nick Sweeting 2020-07-22 01:46:38 -04:00
parent 25e0cba0cc
commit 0965031d8f
2 changed files with 7 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

2
archivebox/extractors/archive_org.py
View file

@ -106,7 +106,7 @@ def parse_archive_dot_org_response(response: bytes) -> Tuple[List[str], List[str
headers[name.lower().strip()].append(val.strip()) headers[name.lower().strip()].append(val.strip())
# Get successful archive url in "content-location" header or any errors # Get successful archive url in "content-location" header or any errors
content_location = headers['content-location'] content_location = headers.get('content-location', headers['location'])
errors = headers['x-archive-wayback-runtime-error'] errors = headers['x-archive-wayback-runtime-error']
return content_location, errors return content_location, errors

9
archivebox/extractors/wget.py
View file

@ -97,21 +97,20 @@ def save_wget(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) ->
if 'Downloaded:' in output_tail[-1] if 'Downloaded:' in output_tail[-1]
else 0 else 0
) )
# Check for common failure cases
if result.returncode > 0 and files_downloaded < 1:
hints = ( hints = (
'Got wget response code: {}.'.format(result.returncode), 'Got wget response code: {}.'.format(result.returncode),
*output_tail, *output_tail,
) )
# Check for common failure cases
if (result.returncode > 0 and files_downloaded < 1) or output is None:
if b'403: Forbidden' in result.stderr: if b'403: Forbidden' in result.stderr:
raise ArchiveError('403 Forbidden (try changing WGET_USER_AGENT)', hints) raise ArchiveError('403 Forbidden (try changing WGET_USER_AGENT)', hints)
if b'404: Not Found' in result.stderr: if b'404: Not Found' in result.stderr:
raise ArchiveError('404 Not Found', hints) raise ArchiveError('404 Not Found', hints)
if b'ERROR 500: Internal Server Error' in result.stderr: if b'ERROR 500: Internal Server Error' in result.stderr:
raise ArchiveError('500 Internal Server Error', hints) raise ArchiveError('500 Internal Server Error', hints)
raise ArchiveError('Got an error from the server', hints) raise ArchiveError('Wget failed or got an error from the server', hints)
chmod_file(output, cwd=out_dir) chmod_file(output, cwd=out_dir)
except Exception as err: except Exception as err:
status = 'failed' status = 'failed'