mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-12 22:25:44 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

load EXTRACTORS dynamically using importlib.import_module

Browse source
This commit is contained in:
Nick Sweeting 2024-05-11 22:28:59 -07:00
parent c7f55fc3ba
commit 457c42bf84
No known key found for this signature in database
18 changed files with 198 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

6
archivebox/index/html.py
View file

@ -121,7 +121,7 @@ def snapshot_icons(snapshot) -> str:
cache_key = f'{snapshot.id}-{(snapshot.updated or snapshot.added).timestamp()}-snapshot-icons'
def calc_snapshot_icons():
from core.models import EXTRACTORS
from core.models import EXTRACTOR_CHOICES
# start = datetime.now(timezone.utc)
archive_results = snapshot.archiveresult_set.filter(status="succeeded", output__isnull=False)
@ -147,12 +147,12 @@ def snapshot_icons(snapshot) -> str:
# Missing specific entry for WARC
extractor_outputs = defaultdict(lambda: None)
for extractor, _ in EXTRACTORS:
for extractor, _ in EXTRACTOR_CHOICES:
for result in archive_results:
if result.extractor == extractor and result:
extractor_outputs[extractor] = result
for extractor, _ in EXTRACTORS:
for extractor, _ in EXTRACTOR_CHOICES:
if extractor not in exclude:
existing = extractor_outputs[extractor] and extractor_outputs[extractor].status == 'succeeded' and extractor_outputs[extractor].output
# Check filesystsem to see if anything is actually present (too slow, needs optimization/caching)