mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-16 16:14:28 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Exception handling for indexing and searching

Browse source
This commit is contained in:
JDC 2020-11-21 09:37:13 -05:00 • committed by Nick Sweeting
parent 0ed53cc117
commit 4eeedae815
2 changed files with 38 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

9
archivebox/index/__init__.py
View file

@ -386,7 +386,7 @@ def search_filter(snapshots: QuerySet, filter_patterns: List[str], filter_type:
if not search_backend_enabled(): if not search_backend_enabled():
stderr() stderr()
stderr( stderr(
'[X] The search backend is not enabled', '[X] The search backend is not enabled, set config.USE_SEARCHING_BACKEND = True',
color='red', color='red',
) )
raise SystemExit(2) raise SystemExit(2)
@ -395,12 +395,7 @@ def search_filter(snapshots: QuerySet, filter_patterns: List[str], filter_type:
for pattern in filter_patterns: for pattern in filter_patterns:
try: try:
qsearch |= query_search_index(pattern) qsearch |= query_search_index(pattern)
except Exception as err: except:
stderr()
stderr(
f'[X] The search backend threw an exception={err}:',
color='red',
)
raise SystemExit(2) raise SystemExit(2)
return snapshots & qsearch return snapshots & qsearch

50
archivebox/search/__init__.py
View file

@ -6,7 +6,7 @@ from django.db.models import QuerySet
from archivebox.index.schema import Link from archivebox.index.schema import Link
from archivebox.util import enforce_types from archivebox.util import enforce_types
from archivebox.config import setup_django, OUTPUT_DIR, USE_INDEXING_BACKEND, USE_SEARCHING_BACKEND, SEARCH_BACKEND_ENGINE from archivebox.config import setup_django,stderr, OUTPUT_DIR, USE_INDEXING_BACKEND, USE_SEARCHING_BACKEND, SEARCH_BACKEND_ENGINE
def indexing_enabled(): def indexing_enabled():
return USE_INDEXING_BACKEND return USE_INDEXING_BACKEND
@ -37,21 +37,37 @@ def write_search_index(link: Link, texts: Union[List[str], None]=None, out_dir:
snap = Snapshot.objects.filter(url=link.url).first() snap = Snapshot.objects.filter(url=link.url).first()
backend = import_backend() backend = import_backend()
if snap: if snap:
backend.index(snapshot_id=str(snap.id), texts=texts) try:
backend.index(snapshot_id=str(snap.id), texts=texts)
except Exception as err:
stderr()
stderr(
f'[X] The search backend threw an exception={err}:',
color='red',
)
@enforce_types @enforce_types
def query_search_index(query: str, out_dir: Path=OUTPUT_DIR) -> QuerySet: def query_search_index(query: str, out_dir: Path=OUTPUT_DIR) -> QuerySet:
if search_backend_enabled(): setup_django(out_dir, check_db=True)
setup_django(out_dir, check_db=True) from core.models import Snapshot
from core.models import Snapshot
if search_backend_enabled():
backend = import_backend() backend = import_backend()
snapshot_ids = backend.search(query) try:
# TODO preserve ordering from backend snapshot_ids = backend.search(query)
qsearch = Snapshot.objects.filter(pk__in=snapshot_ids) except Exception as err:
return qsearch stderr()
else: stderr(
return Snapshot.objects.none() f'[X] The search backend threw an exception={err}:',
color='red',
)
raise
else:
# TODO preserve ordering from backend
qsearch = Snapshot.objects.filter(pk__in=snapshot_ids)
return qsearch
return Snapshot.objects.none()
@enforce_types @enforce_types
def flush_search_index(snapshots: QuerySet): def flush_search_index(snapshots: QuerySet):
@ -59,5 +75,11 @@ def flush_search_index(snapshots: QuerySet):
return return
backend = import_backend() backend = import_backend()
snapshot_ids=(str(pk) for pk in snapshots.values_list('pk',flat=True)) snapshot_ids=(str(pk) for pk in snapshots.values_list('pk',flat=True))
try:
backend.flush(snapshot_ids) backend.flush(snapshot_ids)
except Exception as err:
stderr()
stderr(
f'[X] The search backend threw an exception={err}:',
color='red',
)