mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-16 08:04:26 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Exception handling for indexing and searching

Browse source
This commit is contained in:
JDC 2020-11-21 09:37:13 -05:00 • committed by Nick Sweeting
parent 0ed53cc117
commit 4eeedae815
2 changed files with 38 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

9
archivebox/index/__init__.py
View file

@ -386,7 +386,7 @@ def search_filter(snapshots: QuerySet, filter_patterns: List[str], filter_type:
if not search_backend_enabled():
stderr()
stderr(
'[X] The search backend is not enabled',
'[X] The search backend is not enabled, set config.USE_SEARCHING_BACKEND = True',
color='red',
)
raise SystemExit(2)
@ -395,12 +395,7 @@ def search_filter(snapshots: QuerySet, filter_patterns: List[str], filter_type:
for pattern in filter_patterns:
try:
qsearch |= query_search_index(pattern)
except Exception as err:
stderr()
stderr(
f'[X] The search backend threw an exception={err}:',
color='red',
)
except:
raise SystemExit(2)
return snapshots & qsearch

50
archivebox/search/__init__.py
View file

@ -6,7 +6,7 @@ from django.db.models import QuerySet
from archivebox.index.schema import Link
from archivebox.util import enforce_types
from archivebox.config import setup_django, OUTPUT_DIR, USE_INDEXING_BACKEND, USE_SEARCHING_BACKEND, SEARCH_BACKEND_ENGINE
from archivebox.config import setup_django,stderr, OUTPUT_DIR, USE_INDEXING_BACKEND, USE_SEARCHING_BACKEND, SEARCH_BACKEND_ENGINE
def indexing_enabled():
return USE_INDEXING_BACKEND
@ -37,21 +37,37 @@ def write_search_index(link: Link, texts: Union[List[str], None]=None, out_dir:
snap = Snapshot.objects.filter(url=link.url).first()
backend = import_backend()
if snap:
backend.index(snapshot_id=str(snap.id), texts=texts)
try:
backend.index(snapshot_id=str(snap.id), texts=texts)
except Exception as err:
stderr()
stderr(
f'[X] The search backend threw an exception={err}:',
color='red',
)
@enforce_types
def query_search_index(query: str, out_dir: Path=OUTPUT_DIR) -> QuerySet:
if search_backend_enabled():
setup_django(out_dir, check_db=True)
from core.models import Snapshot
def query_search_index(query: str, out_dir: Path=OUTPUT_DIR) -> QuerySet:
setup_django(out_dir, check_db=True)
from core.models import Snapshot
if search_backend_enabled():
backend = import_backend()
snapshot_ids = backend.search(query)
# TODO preserve ordering from backend
qsearch = Snapshot.objects.filter(pk__in=snapshot_ids)
return qsearch
else:
return Snapshot.objects.none()
try:
snapshot_ids = backend.search(query)
except Exception as err:
stderr()
stderr(
f'[X] The search backend threw an exception={err}:',
color='red',
)
raise
else:
# TODO preserve ordering from backend
qsearch = Snapshot.objects.filter(pk__in=snapshot_ids)
return qsearch
return Snapshot.objects.none()
@enforce_types
def flush_search_index(snapshots: QuerySet):
@ -59,5 +75,11 @@ def flush_search_index(snapshots: QuerySet):
return
backend = import_backend()
snapshot_ids=(str(pk) for pk in snapshots.values_list('pk',flat=True))
backend.flush(snapshot_ids)
try:
backend.flush(snapshot_ids)
except Exception as err:
stderr()
stderr(
f'[X] The search backend threw an exception={err}:',
color='red',
)