move abx plugins inside vendor dir

archivebox/vendor/abx-plugin-sonic-search/abx_plugin_sonic_search/__init__.py

@@ -0,0 +1,37 @@
+__package__ = 'abx_plugin_sonic_search'
+__label__ = 'Sonic Search'
+__homepage__ = 'https://github.com/valeriansaliou/sonic'
+
+import abx
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import SONIC_CONFIG
+    
+    return {
+        'SONIC_CONFIG': SONIC_CONFIG
+    }
+
+
+@abx.hookimpl
+def get_BINARIES():
+    from .binaries import SONIC_BINARY
+    
+    return {
+        'sonic': SONIC_BINARY
+    }
+
+
+@abx.hookimpl
+def get_SEARCHBACKENDS():
+    from .searchbackend import SONIC_SEARCH_BACKEND
+    
+    return {
+        'sonic': SONIC_SEARCH_BACKEND,
+    }
+
+@abx.hookimpl
+def ready():
+    from .config import SONIC_CONFIG
+    SONIC_CONFIG.validate()

archivebox/vendor/abx-plugin-sonic-search/abx_plugin_sonic_search/binaries.py

@@ -0,0 +1,27 @@
+__package__ = 'abx_plugin_sonic_search'
+
+from typing import List
+
+from pydantic import InstanceOf
+from pydantic_pkgr import BinProvider, BinaryOverrides, BinName, Binary
+
+from abx_plugin_default_binproviders import brew, env
+
+from .config import SONIC_CONFIG
+
+
+class SonicBinary(Binary):
+    name: BinName = SONIC_CONFIG.SONIC_BINARY
+    binproviders_supported: List[InstanceOf[BinProvider]] = [brew, env]   # TODO: add cargo
+
+    overrides: BinaryOverrides = {
+        brew.name: {'packages': ['sonic']},
+        # cargo.name: {'packages': ['sonic-server']},                     # TODO: add cargo
+    }
+    
+    # TODO: add version checking over protocol? for when sonic backend is on remote server and binary is not installed locally
+    # def on_get_version(self):
+    #     with sonic.IngestClient(SONIC_CONFIG.SONIC_HOST, str(SONIC_CONFIG.SONIC_PORT), SONIC_CONFIG.SONIC_PASSWORD) as ingestcl:
+    #         return SemVer.parse(str(ingestcl.protocol))
+
+SONIC_BINARY = SonicBinary()

archivebox/vendor/abx-plugin-sonic-search/abx_plugin_sonic_search/config.py

@@ -0,0 +1,41 @@
+__package__ = 'abx_plugin_sonic_search'
+
+import sys
+
+from pydantic import Field
+
+from abx_spec_config.base_configset import BaseConfigSet
+
+from archivebox.config.common import SEARCH_BACKEND_CONFIG
+
+
+SONIC_LIB = None
+try:
+    import sonic
+    SONIC_LIB = sonic
+except ImportError:
+    SONIC_LIB = None
+
+###################### Config ##########################
+
+
+class SonicConfig(BaseConfigSet):
+    SONIC_BINARY: str       = Field(default='sonic')
+    
+    SONIC_HOST: str         = Field(default='localhost', alias='SEARCH_BACKEND_HOST_NAME')
+    SONIC_PORT: int         = Field(default=1491, alias='SEARCH_BACKEND_PORT')
+    SONIC_PASSWORD: str     = Field(default='SecretPassword', alias='SEARCH_BACKEND_PASSWORD')
+    SONIC_COLLECTION: str   = Field(default='archivebox')
+    SONIC_BUCKET: str       = Field(default='archivebox')
+    
+    SONIC_MAX_CHUNK_LENGTH: int     = Field(default=2000)
+    SONIC_MAX_TEXT_LENGTH: int      = Field(default=100000000)
+    SONIC_MAX_RETRIES: int          = Field(default=5)
+    
+    def validate(self):
+        if SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE == 'sonic' and SONIC_LIB is None:
+            sys.stderr.write('[X] Error: Sonic search backend is enabled but sonic-client lib is not installed. You may need to run: pip install archivebox[sonic]\n')
+            # dont hard exit here. in case the user is just running "archivebox version" or "archivebox help", we still want those to work despite broken ldap
+            SEARCH_BACKEND_CONFIG.update_in_place(SEARCH_BACKEND_ENGINE='ripgrep')
+
+SONIC_CONFIG = SonicConfig()

archivebox/vendor/abx-plugin-sonic-search/abx_plugin_sonic_search/searchbackend.py

@@ -0,0 +1,51 @@
+__package__ = 'plugins_search.sonic'
+
+from typing import List, Generator, cast
+
+from abx_spec_searchbackend import BaseSearchBackend
+
+
+from .config import SONIC_CONFIG, SONIC_LIB
+
+
+class SonicSearchBackend(BaseSearchBackend):
+    name: str = 'sonic'
+    docs_url: str = 'https://github.com/valeriansaliou/sonic'
+    
+    @staticmethod
+    def index(snapshot_id: str, texts: List[str]):
+        error_count = 0
+        with SONIC_LIB.IngestClient(SONIC_CONFIG.SONIC_HOST, str(SONIC_CONFIG.SONIC_PORT), SONIC_CONFIG.SONIC_PASSWORD) as ingestcl:
+            for text in texts:
+                chunks = (
+                    text[i:i+SONIC_CONFIG.SONIC_MAX_CHUNK_LENGTH]
+                    for i in range(
+                        0,
+                        min(len(text), SONIC_CONFIG.SONIC_MAX_TEXT_LENGTH),
+                        SONIC_CONFIG.SONIC_MAX_CHUNK_LENGTH,
+                    )
+                )
+                try:
+                    for chunk in chunks:
+                        ingestcl.push(SONIC_CONFIG.SONIC_COLLECTION, SONIC_CONFIG.SONIC_BUCKET, snapshot_id, str(chunk))
+                except Exception as err:
+                    print(f'[!] Sonic search backend threw an error while indexing: {err.__class__.__name__} {err}')
+                    error_count += 1
+                    if error_count > SONIC_CONFIG.SONIC_MAX_RETRIES:
+                        raise
+
+    @staticmethod
+    def flush(snapshot_ids: Generator[str, None, None]):
+        with SONIC_LIB.IngestClient(SONIC_CONFIG.SONIC_HOST, str(SONIC_CONFIG.SONIC_PORT), SONIC_CONFIG.SONIC_PASSWORD) as ingestcl:
+            for id in snapshot_ids:
+                ingestcl.flush_object(SONIC_CONFIG.SONIC_COLLECTION, SONIC_CONFIG.SONIC_BUCKET, str(id))
+    
+
+    @staticmethod
+    def search(text: str) -> List[str]:
+        with SONIC_LIB.SearchClient(SONIC_CONFIG.SONIC_HOST, SONIC_CONFIG.SONIC_PORT, SONIC_CONFIG.SONIC_PASSWORD) as querycl:
+            snap_ids = cast(List[str], querycl.query(SONIC_CONFIG.SONIC_COLLECTION, SONIC_CONFIG.SONIC_BUCKET, text))
+        return [str(id) for id in snap_ids]
+    
+    
+SONIC_SEARCH_BACKEND = SonicSearchBackend()

archivebox/vendor/abx-plugin-sonic-search/pyproject.toml

@@ -0,0 +1,20 @@
+[project]
+name = "abx-plugin-sonic-search"
+version = "2024.10.28"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "abx>=0.1.0",
+    "abx-spec-config>=0.1.0",
+    "abx-spec-pydantic-pkgr>=0.1.0",
+    "abx-spec-searchbackend>=0.1.0",
+    "pydantic-pkgr>=0.5.4",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project.entry-points.abx]
+abx_plugin_sonic_search = "abx_plugin_sonic_search"