mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-16 16:14:28 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix broken function name

Browse source
This commit is contained in:
Nick Sweeting 2019-03-30 15:00:21 -04:00 • committed by GitHub
parent 066b36b6a9
commit 529a0f8bb2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

17
archivebox/links.py
View file

@ -37,7 +37,6 @@ def validate_links(links):
links = archivable_links(links) # remove chrome://, about:, mailto: etc. links = archivable_links(links) # remove chrome://, about:, mailto: etc.
links = uniquefied_links(links) # merge/dedupe duplicate timestamps & urls links = uniquefied_links(links) # merge/dedupe duplicate timestamps & urls
links = sorted_links(links) # deterministically sort the links based on timstamp, url links = sorted_links(links) # deterministically sort the links based on timstamp, url
links = list(exclude_links(links)) # exclude URLs that match the blacklisted url pattern regex
if not links: if not links:
print('[X] No links found :(') print('[X] No links found :(')
@ -52,11 +51,11 @@ def validate_links(links):
def archivable_links(links): def archivable_links(links):
"""remove chrome://, about:// or other schemed links that cant be archived""" """remove chrome://, about:// or other schemed links that cant be archived"""
return ( for link in links:
link scheme_is_valid = scheme(url) in ('http', 'https', 'ftp)
for link in links not_blacklisted = (not URL_BLACKLIST.match(link['url'])) if URL_BLACKLIST else True
if any(link['url'].lower().startswith(s) for s in ('http://', 'https://', 'ftp://')) if scheme_is_valid and not_blacklisted:
) yield link
def uniquefied_links(sorted_links): def uniquefied_links(sorted_links):
@ -119,9 +118,5 @@ def lowest_uniq_timestamp(used_timestamps, timestamp):
new_timestamp = '{}.{}'.format(timestamp, nonce) new_timestamp = '{}.{}'.format(timestamp, nonce)
return new_timestamp return new_timestamp
def exclude_blacklisted(links):
"""exclude URLs that match the blacklisted url pattern regex"""
return (link for link in links if not URL_BLACKLIST.match(link['url']))