mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-31 23:08:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: Refactor add method to use querysets

Browse source
This commit is contained in:
Cristian 2020-08-21 09:57:29 -05:00 • committed by Cristian Vargas
parent 6a2e6aad2f
commit be520d137a
4 changed files with 71 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

31
archivebox/extractors/__init__.py
View file

@ -9,6 +9,7 @@ from ..index.schema import Link
from ..index import (
load_link_details,
write_link_details,
write_main_index,
)
from ..util import enforce_types
from ..logging_util import (
@ -128,24 +129,34 @@ def archive_link(link: Link, overwrite: bool=False, methods: Optional[Iterable[s
return link
@enforce_types
def archive_links(links: List[Link], overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[str]=None) -> List[Link]:
if not links:
def archive_links(all_links: any, overwrite: bool=False, methods: Optional[Iterable[str]]=None, out_dir: Optional[str]=None) -> List[Link]:
if type(all_links) is list:
num_links: int = len(all_links)
get_link = lambda x: x
get_iter = lambda x: x
else:
num_links: int = all_links.count()
get_link = lambda x: x.as_link()
get_iter = lambda x: x.iterator()
if num_links == 0:
return []
log_archiving_started(len(links))
log_archiving_started(num_links)
idx: int = 0
link: Link = links[0]
try:
for idx, link in enumerate(links):
archive_link(link, overwrite=overwrite, methods=methods, out_dir=link.link_dir)
for link in get_iter(all_links):
idx += 1
to_archive = get_link(link)
archive_link(to_archive, overwrite=overwrite, methods=methods, out_dir=link.link_dir)
except KeyboardInterrupt:
log_archiving_paused(len(links), idx, link.timestamp)
log_archiving_paused(num_links, idx, link.timestamp)
raise SystemExit(0)
except BaseException:
print()
raise
log_archiving_finished(len(links))
return links
log_archiving_finished(num_links)
return all_links