mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-14 07:04:27 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

properly handle querystrings for wget .html appended links

Browse source
This commit is contained in:
Nick Sweeting 2017-07-05 17:15:56 -05:00
parent 6bb91fbb45
commit 2265f2aaf0
1 changed files with 8 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

10
parse.py
View file

@ -175,16 +175,22 @@ def html_appended_url(link):
See docs on wget --adjust-extension.""" See docs on wget --adjust-extension."""
split_url = link['url'].split('#', 1) split_url = link['url'].split('#', 1)
query = ('%3F' + link['url'].split('?', 1)[-1]) if '?' in link['url'] else ''
if re.search(".+\\.[Hh][Tt][Mm][Ll]?$", split_url[0], re.I | re.M): if re.search(".+\\.[Hh][Tt][Mm][Ll]?$", split_url[0], re.I | re.M):
# already ends in .html # already ends in .html
return link['base_url'] return link['base_url']
else: else:
# .html needs to be appended # .html needs to be appended
without_scheme = split_url[0].split('://', 1)[-1] without_scheme = split_url[0].split('://', 1)[-1].split('?', 1)[0]
if without_scheme.endswith('/'): if without_scheme.endswith('/'):
if query:
return '#'.join([without_scheme + 'index.html' + query + '.html', *split_url[1:]])
return '#'.join([without_scheme + 'index.html', *split_url[1:]]) return '#'.join([without_scheme + 'index.html', *split_url[1:]])
return '#'.join([without_scheme + '.html', *split_url[1:]]) else:
if query:
return '#'.join([without_scheme + 'index.html' + query + '.html', *split_url[1:]])
return '#'.join([without_scheme + '.html', *split_url[1:]])
def derived_link_info(link): def derived_link_info(link):