mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-14 23:24:30 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add config for search backend

Browse source
This commit is contained in:
JDC 2020-11-19 08:06:13 -05:00 • committed by Nick Sweeting
parent 5f6673c72c
commit c2c01af3ad
3 changed files with 23 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

12
archivebox/config.py
View file

@ -139,6 +139,18 @@ CONFIG_DEFAULTS: Dict[str, ConfigDefaultDict] = {
'GIT_ARGS': {'type': list, 'default': ['--recursive']}, 'GIT_ARGS': {'type': list, 'default': ['--recursive']},
}, },
'SEARCH_BACKEND_CONFIG' : {
'USE_INDEXING_BACKEND': {'type': bool, 'default': True},
'USE_SEARCHING_BACKEND': {'type': bool, 'default': True},
'SEARCH_BACKEND_ENGINE': {'type': str, 'default': 'sonic'},
'SEARCH_BACKEND_HOST_NAME': {'type': str, 'default': 'localhost'},
'SEARCH_BACKEND_PORT': {'type': int, 'default': 1491},
'SEARCH_BACKEND_PASSWORD': {'type': str, 'default': 'SecretPassword'},
# SONIC
'SONIC_BUCKET': {'type': str, 'default': 'archivebox'},
'SONIC_COLLECTION': {'type': str, 'default': 'snapshots'},
},
'DEPENDENCY_CONFIG': { 'DEPENDENCY_CONFIG': {
'USE_CURL': {'type': bool, 'default': True}, 'USE_CURL': {'type': bool, 'default': True},
'USE_WGET': {'type': bool, 'default': True}, 'USE_WGET': {'type': bool, 'default': True},

11
archivebox/search/__init__.py
View file

@ -5,19 +5,16 @@ from importlib import import_module
from archivebox.index.schema import Link from archivebox.index.schema import Link
from archivebox.util import enforce_types from archivebox.util import enforce_types
from archivebox.config import setup_django, OUTPUT_DIR from archivebox.config import setup_django, OUTPUT_DIR, USE_INDEXING_BACKEND, USE_SEARCHING_BACKEND, SEARCH_BACKEND_ENGINE
def indexing_enabled(): def indexing_enabled():
return True return USE_INDEXING_BACKEND
# return FULLTEXT_INDEXING_ENABLED
def search_backend_enabled(): def search_backend_enabled():
return True return USE_SEARCHING_BACKEND
# return FULLTEXT_SEARCH_ENABLED
def get_backend(): def get_backend():
return 'search.backends.sonic' return f'search.backends.{SEARCH_BACKEND_ENGINE}'
def import_backend(): def import_backend():
backend_string = get_backend() backend_string = get_backend()

11
archivebox/search/backends/sonic.py
View file

@ -3,17 +3,18 @@ from typing import List
from sonic import IngestClient, SearchClient from sonic import IngestClient, SearchClient
from archivebox.util import enforce_types from archivebox.util import enforce_types
from archivebox.config import SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD, SONIC_BUCKET, SONIC_COLLECTION
@enforce_types @enforce_types
def index(snapshot_id: str, texts: List[str]): def index(snapshot_id: str, texts: List[str]):
# TODO add variables to localhost, port, password, bucket, collection with IngestClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as ingestcl:
with IngestClient("localhost", 1491, "SecretPassword") as ingestcl:
for text in texts: for text in texts:
ingestcl.push("archivebox", "snapshots", snapshot_id, str(text)) ingestcl.push(SONIC_BUCKET, SONIC_COLLECTION, snapshot_id, str(text))
@enforce_types @enforce_types
def search(text: str) -> List: def search(text: str) -> List:
with SearchClient("localhost", 1491, "SecretPassword") as querycl: with SearchClient(SEARCH_BACKEND_HOST_NAME, SEARCH_BACKEND_PORT, SEARCH_BACKEND_PASSWORD) as querycl:
snap_ids = querycl.query("archivebox", "snapshots", text) snap_ids = querycl.query(SONIC_BUCKET, SONIC_COLLECTION, text)
return snap_ids return snap_ids