mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-15 07:34:27 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make O(n^2) loop to an O(n) problem.

Browse source
This commit is contained in:
Aaron Fischer 2018-10-21 22:36:32 +02:00
parent b1b6be4f13
commit ebc327bb89
1 changed files with 4 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

15
archiver/links.py
View file

@ -74,21 +74,14 @@ def validate_links(links):
return list(links) return list(links)
def new_links(imported_links, existing_links): def new_links(all_links, existing_links):
""" """
Return all links which are in the imported_links but not in the existing_links. Return all links which are in the all_links but not in the existing_links.
This is used to determine which links are new and not indexed jet. Set the This is used to determine which links are new and not indexed jet. Set the
ONLY_NEW environment variable to activate this filter mechanism. ONLY_NEW environment variable to activate this filter mechanism.
""" """
new_links = [] existing_urls = list(map(lambda l: l['url'], existing_links))
for i_link in imported_links: return list(filter(lambda l: l['url'] not in existing_urls, all_links))
found_link_in_existing_links = False
for e_link in existing_links:
if i_link['url'] == e_link['url']:
found_link_in_existing_links = True
if not found_link_in_existing_links:
new_links.append(i_link)
return new_links
def archivable_links(links): def archivable_links(links):
"""remove chrome://, about:// or other schemed links that cant be archived""" """remove chrome://, about:// or other schemed links that cant be archived"""