mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-23 03:06:55 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

more settings loading tweaks and improvements

Browse source
This commit is contained in:
Nick Sweeting 2024-09-24 15:13:54 -07:00
parent fbfd16e195
commit 97695bda5e
No known key found for this signature in database
10 changed files with 350 additions and 260 deletions
Download patch file Download diff file Expand all files Collapse all files

29
archivebox/plugins_search/sonic/apps.py
View file

@ -1,5 +1,6 @@
__package__ = 'archivebox.plugins_search.sonic'
import os
import sys
from typing import List, Dict, ClassVar, Generator, cast
@ -38,16 +39,24 @@ class SonicConfig(BaseConfigSet):
SONIC_PASSWORD: str = Field(default='SecretPassword', alias='SEARCH_BACKEND_PASSWORD')
SONIC_COLLECTION: str = Field(default='archivebox')
SONIC_BUCKET: str = Field(default='archivebox')
SONIC_MAX_CHUNK_LENGTH: int = Field(default=2000)
SONIC_MAX_TEXT_LENGTH: int = Field(default=100000000)
SONIC_MAX_RETRIES: int = Field(default=5)
@model_validator(mode='after')
def validate_sonic_port(self):
if SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE == 'sonic':
if SONIC_LIB is None:
sys.stderr.write('[!] Sonic search backend is enabled but not installed. Install Sonic to use the Sonic search backend.\n')
if SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE == 'sonic' and SONIC_LIB is None:
sys.stderr.write('[X] Error: Sonic search backend is enabled but sonic-client lib is not installed. You may need to run: pip install archivebox[sonic]\n')
# dont hard exit here. in case the user is just running "archivebox version" or "archivebox help", we still want those to work despite broken ldap
# sys.exit(1)
SEARCH_BACKEND_CONFIG.update_in_place(SEARCH_BACKEND_ENGINE='ripgrep')
return self
SONIC_CONFIG = SonicConfig()
class SonicBinary(BaseBinary):
name: BinName = SONIC_CONFIG.SONIC_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [brew, env] # TODO: add cargo
@ -57,6 +66,7 @@ class SonicBinary(BaseBinary):
# cargo.name: {'packages': lambda: ['sonic-server']}, # TODO: add cargo
}
# TODO: add version checking over protocol? for when sonic backend is on remote server and binary is not installed locally
# def on_get_version(self):
# with sonic.IngestClient(SONIC_CONFIG.SONIC_HOST, str(SONIC_CONFIG.SONIC_PORT), SONIC_CONFIG.SONIC_PASSWORD) as ingestcl:
# return SemVer.parse(str(ingestcl.protocol))
@ -64,11 +74,6 @@ class SonicBinary(BaseBinary):
SONIC_BINARY = SonicBinary()
MAX_SONIC_TEXT_TOTAL_LENGTH = 100000000 # dont index more than 100 million characters per text
MAX_SONIC_TEXT_CHUNK_LENGTH = 2000 # dont index more than 2000 characters per chunk
MAX_SONIC_ERRORS_BEFORE_ABORT = 5
class SonicSearchBackend(BaseSearchBackend):
name: str = 'sonic'
@ -80,11 +85,11 @@ class SonicSearchBackend(BaseSearchBackend):
with sonic.IngestClient(SONIC_CONFIG.SONIC_HOST, str(SONIC_CONFIG.SONIC_PORT), SONIC_CONFIG.SONIC_PASSWORD) as ingestcl:
for text in texts:
chunks = (
text[i:i+MAX_SONIC_TEXT_CHUNK_LENGTH]
text[i:i+SONIC_CONFIG.SONIC_MAX_CHUNK_LENGTH]
for i in range(
0,
min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH),
MAX_SONIC_TEXT_CHUNK_LENGTH,
min(len(text), SONIC_CONFIG.SONIC_MAX_TEXT_LENGTH),
SONIC_CONFIG.SONIC_MAX_CHUNK_LENGTH,
)
)
try:
@ -93,7 +98,7 @@ class SonicSearchBackend(BaseSearchBackend):
except Exception as err:
print(f'[!] Sonic search backend threw an error while indexing: {err.__class__.__name__} {err}')
error_count += 1
if error_count > MAX_SONIC_ERRORS_BEFORE_ABORT:
if error_count > SONIC_CONFIG.SONIC_MAX_RETRIES:
raise
@staticmethod