mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-05-21 18:35:23 -04:00
more settings loading tweaks and improvements
This commit is contained in:
Nick Sweeting
parent
fbfd16e195
commit
97695bda5e
10 changed files with 350 additions and 260 deletions
|
|
@ -27,9 +27,22 @@ class RipgrepConfig(BaseConfigSet):
|
|||
section: ClassVar[ConfigSectionName] = 'DEPENDENCY_CONFIG'
|
||||
|
||||
RIPGREP_BINARY: str = Field(default='rg')
|
||||
|
||||
RIPGREP_IGNORE_EXTENSIONS: str = Field(default='css,js,orig,svg')
|
||||
RIPGREP_ARGS_DEFAULT: List[str] = Field(default=lambda c: [
|
||||
# https://github.com/BurntSushi/ripgrep/blob/master/GUIDE.md
|
||||
f'--type-add=ignore:*.{{{c.RIPGREP_IGNORE_EXTENSIONS}}}',
|
||||
'--type-not=ignore',
|
||||
'--ignore-case',
|
||||
'--files-with-matches',
|
||||
'--regexp',
|
||||
])
|
||||
RIPGREP_SEARCH_DIR: str = Field(default=lambda: str(settings.ARCHIVE_DIR))
|
||||
|
||||
RIPGREP_CONFIG = RipgrepConfig()
|
||||
|
||||
|
||||
|
||||
class RipgrepBinary(BaseBinary):
|
||||
name: BinName = RIPGREP_CONFIG.RIPGREP_BINARY
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
|
||||
|
|
@ -41,17 +54,8 @@ class RipgrepBinary(BaseBinary):
|
|||
|
||||
RIPGREP_BINARY = RipgrepBinary()
|
||||
|
||||
|
||||
RG_IGNORE_EXTENSIONS = ('css','js','orig','svg')
|
||||
|
||||
RG_ADD_TYPE = '--type-add'
|
||||
RG_IGNORE_ARGUMENTS = f"ignore:*.{{{','.join(RG_IGNORE_EXTENSIONS)}}}"
|
||||
RG_DEFAULT_ARGUMENTS = "-ilTignore" # Case insensitive(i), matching files results(l)
|
||||
RG_REGEX_ARGUMENT = '-e'
|
||||
|
||||
TIMESTAMP_REGEX = r'\/([\d]+\.[\d]+)\/'
|
||||
ts_regex = re.compile(TIMESTAMP_REGEX)
|
||||
|
||||
# regex to match archive/<ts>/... snapshot dir names
|
||||
TIMESTAMP_REGEX = re.compile(r'\/([\d]+\.[\d]+)\/')
|
||||
|
||||
class RipgrepSearchBackend(BaseSearchBackend):
|
||||
name: str = 'ripgrep'
|
||||
|
|
@ -67,30 +71,29 @@ class RipgrepSearchBackend(BaseSearchBackend):
|
|||
|
||||
@staticmethod
|
||||
def search(text: str) -> List[str]:
|
||||
rg_bin = RIPGREP_BINARY.load()
|
||||
if not rg_bin.version:
|
||||
from core.models import Snapshot
|
||||
|
||||
ripgrep_binary = RIPGREP_BINARY.load()
|
||||
if not ripgrep_binary.version:
|
||||
raise Exception("ripgrep binary not found, install ripgrep to use this search backend")
|
||||
|
||||
rg_cmd = [
|
||||
rg_bin.abspath,
|
||||
RG_ADD_TYPE,
|
||||
RG_IGNORE_ARGUMENTS,
|
||||
RG_DEFAULT_ARGUMENTS,
|
||||
RG_REGEX_ARGUMENT,
|
||||
text,
|
||||
str(settings.ARCHIVE_DIR)
|
||||
cmd = [
|
||||
ripgrep_binary.abspath,
|
||||
*RIPGREP_CONFIG.RIPGREP_ARGS_DEFAULT,
|
||||
text,
|
||||
RIPGREP_CONFIG.RIPGREP_SEARCH_DIR,
|
||||
]
|
||||
rg = run(rg_cmd, timeout=SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_TIMEOUT, capture_output=True, text=True)
|
||||
proc = run(cmd, timeout=SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_TIMEOUT, capture_output=True, text=True)
|
||||
timestamps = set()
|
||||
for path in rg.stdout.splitlines():
|
||||
ts = ts_regex.findall(path)
|
||||
for path in proc.stdout.splitlines():
|
||||
ts = TIMESTAMP_REGEX.findall(path)
|
||||
if ts:
|
||||
timestamps.add(ts[0])
|
||||
|
||||
snap_ids = [str(id) for id in Snapshot.objects.filter(timestamp__in=timestamps).values_list('pk', flat=True)]
|
||||
|
||||
return snap_ids
|
||||
|
||||
|
||||
RIPGREP_SEARCH_BACKEND = RipgrepSearchBackend()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
__package__ = 'archivebox.plugins_search.sonic'
|
||||
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Dict, ClassVar, Generator, cast
|
||||
|
||||
|
|
@ -38,16 +39,24 @@ class SonicConfig(BaseConfigSet):
|
|||
SONIC_PASSWORD: str = Field(default='SecretPassword', alias='SEARCH_BACKEND_PASSWORD')
|
||||
SONIC_COLLECTION: str = Field(default='archivebox')
|
||||
SONIC_BUCKET: str = Field(default='archivebox')
|
||||
|
||||
SONIC_MAX_CHUNK_LENGTH: int = Field(default=2000)
|
||||
SONIC_MAX_TEXT_LENGTH: int = Field(default=100000000)
|
||||
SONIC_MAX_RETRIES: int = Field(default=5)
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate_sonic_port(self):
|
||||
if SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE == 'sonic':
|
||||
if SONIC_LIB is None:
|
||||
sys.stderr.write('[!] Sonic search backend is enabled but not installed. Install Sonic to use the Sonic search backend.\n')
|
||||
if SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE == 'sonic' and SONIC_LIB is None:
|
||||
sys.stderr.write('[X] Error: Sonic search backend is enabled but sonic-client lib is not installed. You may need to run: pip install archivebox[sonic]\n')
|
||||
# dont hard exit here. in case the user is just running "archivebox version" or "archivebox help", we still want those to work despite broken ldap
|
||||
# sys.exit(1)
|
||||
SEARCH_BACKEND_CONFIG.update_in_place(SEARCH_BACKEND_ENGINE='ripgrep')
|
||||
return self
|
||||
|
||||
SONIC_CONFIG = SonicConfig()
|
||||
|
||||
|
||||
|
||||
class SonicBinary(BaseBinary):
|
||||
name: BinName = SONIC_CONFIG.SONIC_BINARY
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [brew, env] # TODO: add cargo
|
||||
|
|
@ -57,6 +66,7 @@ class SonicBinary(BaseBinary):
|
|||
# cargo.name: {'packages': lambda: ['sonic-server']}, # TODO: add cargo
|
||||
}
|
||||
|
||||
# TODO: add version checking over protocol? for when sonic backend is on remote server and binary is not installed locally
|
||||
# def on_get_version(self):
|
||||
# with sonic.IngestClient(SONIC_CONFIG.SONIC_HOST, str(SONIC_CONFIG.SONIC_PORT), SONIC_CONFIG.SONIC_PASSWORD) as ingestcl:
|
||||
# return SemVer.parse(str(ingestcl.protocol))
|
||||
|
|
@ -64,11 +74,6 @@ class SonicBinary(BaseBinary):
|
|||
SONIC_BINARY = SonicBinary()
|
||||
|
||||
|
||||
MAX_SONIC_TEXT_TOTAL_LENGTH = 100000000 # dont index more than 100 million characters per text
|
||||
MAX_SONIC_TEXT_CHUNK_LENGTH = 2000 # dont index more than 2000 characters per chunk
|
||||
MAX_SONIC_ERRORS_BEFORE_ABORT = 5
|
||||
|
||||
|
||||
|
||||
class SonicSearchBackend(BaseSearchBackend):
|
||||
name: str = 'sonic'
|
||||
|
|
@ -80,11 +85,11 @@ class SonicSearchBackend(BaseSearchBackend):
|
|||
with sonic.IngestClient(SONIC_CONFIG.SONIC_HOST, str(SONIC_CONFIG.SONIC_PORT), SONIC_CONFIG.SONIC_PASSWORD) as ingestcl:
|
||||
for text in texts:
|
||||
chunks = (
|
||||
text[i:i+MAX_SONIC_TEXT_CHUNK_LENGTH]
|
||||
text[i:i+SONIC_CONFIG.SONIC_MAX_CHUNK_LENGTH]
|
||||
for i in range(
|
||||
0,
|
||||
min(len(text), MAX_SONIC_TEXT_TOTAL_LENGTH),
|
||||
MAX_SONIC_TEXT_CHUNK_LENGTH,
|
||||
min(len(text), SONIC_CONFIG.SONIC_MAX_TEXT_LENGTH),
|
||||
SONIC_CONFIG.SONIC_MAX_CHUNK_LENGTH,
|
||||
)
|
||||
)
|
||||
try:
|
||||
|
|
@ -93,7 +98,7 @@ class SonicSearchBackend(BaseSearchBackend):
|
|||
except Exception as err:
|
||||
print(f'[!] Sonic search backend threw an error while indexing: {err.__class__.__name__} {err}')
|
||||
error_count += 1
|
||||
if error_count > MAX_SONIC_ERRORS_BEFORE_ABORT:
|
||||
if error_count > SONIC_CONFIG.SONIC_MAX_RETRIES:
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
__package__ = 'archivebox.plugins_search.sqlite'
|
||||
|
||||
import sys
|
||||
import sqlite3
|
||||
import codecs
|
||||
from typing import List, ClassVar, Generator, Callable
|
||||
from typing import List, ClassVar, Iterable, Callable
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import connection as database
|
||||
|
|
@ -17,7 +18,7 @@ from plugantic.base_hook import BaseHook
|
|||
from plugantic.base_searchbackend import BaseSearchBackend
|
||||
|
||||
# Depends on Other Plugins:
|
||||
# from plugins_sys.config.apps import SEARCH_BACKEND_CONFIG
|
||||
from plugins_sys.config.apps import SEARCH_BACKEND_CONFIG
|
||||
|
||||
|
||||
|
||||
|
|
@ -26,19 +27,21 @@ from plugantic.base_searchbackend import BaseSearchBackend
|
|||
class SqliteftsConfig(BaseConfigSet):
|
||||
section: ClassVar[ConfigSectionName] = 'DEPENDENCY_CONFIG'
|
||||
|
||||
SQLITEFTS_SEPARATE_DATABASE: bool = Field(default=True, alias='FTS_SEPARATE_DATABASE')
|
||||
SQLITEFTS_TOKENIZERS: str = Field(default='porter unicode61 remove_diacritics 2', alias='FTS_TOKENIZERS')
|
||||
SQLITEFTS_MAX_LENGTH: int = Field(default=int(1e9), alias='FTS_SQLITE_MAX_LENGTH')
|
||||
SQLITEFTS_SEPARATE_DATABASE: bool = Field(default=True, alias='FTS_SEPARATE_DATABASE')
|
||||
SQLITEFTS_TOKENIZERS: str = Field(default='porter unicode61 remove_diacritics 2', alias='FTS_TOKENIZERS')
|
||||
SQLITEFTS_MAX_LENGTH: int = Field(default=int(1e9), alias='FTS_SQLITE_MAX_LENGTH')
|
||||
|
||||
SQLITEFTS_DB: str = Field(default='search.sqlite3')
|
||||
SQLITEFTS_TABLE: str = Field(default='snapshot_fts')
|
||||
SQLITEFTS_ID_TABLE: str = Field(default='snapshot_id_fts')
|
||||
SQLITEFTS_COLUMN: str = Field(default='texts')
|
||||
# Not really meant to be user-modified, just here as constants
|
||||
SQLITEFTS_DB: str = Field(default='search.sqlite3')
|
||||
SQLITEFTS_TABLE: str = Field(default='snapshot_fts')
|
||||
SQLITEFTS_ID_TABLE: str = Field(default='snapshot_id_fts')
|
||||
SQLITEFTS_COLUMN: str = Field(default='texts')
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate_fts_separate_database(self):
|
||||
if self.SQLITEFTS_SEPARATE_DATABASE:
|
||||
assert self.SQLITEFTS_DB, "SQLITEFTS_DB must be set if SQLITEFTS_SEPARATE_DATABASE is True"
|
||||
if SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE == 'sqlite' and self.SQLITEFTS_SEPARATE_DATABASE and not self.SQLITEFTS_DB:
|
||||
sys.stderr.write('[X] Error: SQLITEFTS_DB must be set if SQLITEFTS_SEPARATE_DATABASE is True\n')
|
||||
SEARCH_BACKEND_CONFIG.update_in_place(SEARCH_BACKEND_ENGINE='ripgrep')
|
||||
return self
|
||||
|
||||
@property
|
||||
|
|
@ -84,8 +87,7 @@ def _escape_sqlite3(value: str, *, quote: str, errors='strict') -> str:
|
|||
|
||||
nul_index = encodable.find("\x00")
|
||||
if nul_index >= 0:
|
||||
error = UnicodeEncodeError("NUL-terminated utf-8", encodable,
|
||||
nul_index, nul_index + 1, "NUL not allowed")
|
||||
error = UnicodeEncodeError("NUL-terminated utf-8", encodable, nul_index, nul_index + 1, "NUL not allowed")
|
||||
error_handler = codecs.lookup_error(errors)
|
||||
replacement, _ = error_handler(error)
|
||||
assert isinstance(replacement, str), "handling a UnicodeEncodeError should return a str replacement"
|
||||
|
|
@ -224,7 +226,7 @@ class SqliteftsSearchBackend(BaseSearchBackend):
|
|||
return snap_ids
|
||||
|
||||
@staticmethod
|
||||
def flush(snapshot_ids: Generator[str, None, None]):
|
||||
def flush(snapshot_ids: Iterable[str]):
|
||||
snapshot_ids = list(snapshot_ids) # type: ignore[assignment]
|
||||
|
||||
id_table = _escape_sqlite3_identifier(SQLITEFTS_CONFIG.SQLITEFTS_ID_TABLE)
|
||||
|
|
@ -243,7 +245,7 @@ SQLITEFTS_SEARCH_BACKEND = SqliteftsSearchBackend()
|
|||
|
||||
class SqliteftsSearchPlugin(BasePlugin):
|
||||
app_label: str ='sqlitefts'
|
||||
verbose_name: str = 'Sqlitefts'
|
||||
verbose_name: str = 'SQLite FTS5 Search'
|
||||
|
||||
hooks: List[InstanceOf[BaseHook]] = [
|
||||
SQLITEFTS_CONFIG,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue