fully migrate all search backends to new plugin system

archivebox/plugins_search/ripgrep/apps.py

@@ -1,6 +1,8 @@
 __package__ = 'archivebox.plugins_search.ripgrep'
 
-from typing import List, Dict, ClassVar
+import re
+from subprocess import run
+from typing import List, Dict, ClassVar, Iterable
 # from typing_extensions import Self
 
 from django.conf import settings
@@ -14,10 +16,10 @@ from plugantic.base_plugin import BasePlugin
 from plugantic.base_configset import BaseConfigSet, ConfigSectionName
 from plugantic.base_binary import BaseBinary, env, apt, brew
 from plugantic.base_hook import BaseHook
-# from plugantic.base_search import BaseSearchBackend
+from plugantic.base_searchbackend import BaseSearchBackend
 
 # Depends on Other Plugins:
-# from plugins_sys.config.apps import SEARCH_BACKEND_CONFIG
+from plugins_sys.config.apps import SEARCH_BACKEND_CONFIG
 
 ###################### Config ##########################
 
@@ -39,11 +41,59 @@ class RipgrepBinary(BaseBinary):
 
 RIPGREP_BINARY = RipgrepBinary()
 
-# TODO:
-# class RipgrepSearchBackend(BaseSearchBackend):
-#     name: str = 'ripgrep'
 
-# RIPGREP_SEARCH_BACKEND = RipgrepSearchBackend()
+RG_IGNORE_EXTENSIONS = ('css','js','orig','svg')
+
+RG_ADD_TYPE = '--type-add'
+RG_IGNORE_ARGUMENTS = f"ignore:*.{{{','.join(RG_IGNORE_EXTENSIONS)}}}"
+RG_DEFAULT_ARGUMENTS = "-ilTignore" # Case insensitive(i), matching files results(l)
+RG_REGEX_ARGUMENT = '-e'
+
+TIMESTAMP_REGEX = r'\/([\d]+\.[\d]+)\/'
+ts_regex =  re.compile(TIMESTAMP_REGEX)
+
+
+class RipgrepSearchBackend(BaseSearchBackend):
+    name: str = 'ripgrep'
+    docs_url: str = 'https://github.com/BurntSushi/ripgrep'
+    
+    @staticmethod
+    def index(snapshot_id: str, texts: List[str]):
+        return
+
+    @staticmethod
+    def flush(snapshot_ids: Iterable[str]):
+        return
+
+    @staticmethod
+    def search(text: str) -> List[str]:
+        rg_bin = RIPGREP_BINARY.load()
+        if not rg_bin.version:
+            raise Exception("ripgrep binary not found, install ripgrep to use this search backend")
+    
+        rg_cmd = [
+            rg_bin.abspath, 
+            RG_ADD_TYPE, 
+            RG_IGNORE_ARGUMENTS, 
+            RG_DEFAULT_ARGUMENTS, 
+            RG_REGEX_ARGUMENT, 
+            text, 
+            str(settings.ARCHIVE_DIR)
+        ]
+        rg = run(rg_cmd, timeout=SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_TIMEOUT, capture_output=True, text=True)
+        timestamps = set()
+        for path in rg.stdout.splitlines():
+            ts = ts_regex.findall(path)
+            if ts:
+                timestamps.add(ts[0])
+        
+        snap_ids = [str(id) for id in Snapshot.objects.filter(timestamp__in=timestamps).values_list('pk', flat=True)]
+    
+        return snap_ids
+    
+RIPGREP_SEARCH_BACKEND = RipgrepSearchBackend()
+
+
 
 
 class RipgrepSearchPlugin(BasePlugin):
@@ -53,6 +103,7 @@ class RipgrepSearchPlugin(BasePlugin):
     hooks: List[InstanceOf[BaseHook]] = [
         RIPGREP_CONFIG,
         RIPGREP_BINARY,
+        RIPGREP_SEARCH_BACKEND,
     ]
 
 

archivebox/plugins_search/ripgrep/ripgrep.py

@@ -1,45 +0,0 @@
-import re
-from subprocess import run, PIPE
-from typing import List, Generator
-
-from archivebox.config import ARCHIVE_DIR, RIPGREP_VERSION, SEARCH_BACKEND_TIMEOUT
-from archivebox.util import enforce_types
-
-RG_IGNORE_EXTENSIONS = ('css','js','orig','svg')
-
-RG_ADD_TYPE = '--type-add'
-RG_IGNORE_ARGUMENTS = f"ignore:*.{{{','.join(RG_IGNORE_EXTENSIONS)}}}"
-RG_DEFAULT_ARGUMENTS = "-ilTignore" # Case insensitive(i), matching files results(l)
-RG_REGEX_ARGUMENT = '-e'
-
-TIMESTAMP_REGEX = r'\/([\d]+\.[\d]+)\/'
-
-ts_regex =  re.compile(TIMESTAMP_REGEX)
-
-@enforce_types
-def index(snapshot_id: str, texts: List[str]):
-    return
-
-@enforce_types
-def flush(snapshot_ids: Generator[str, None, None]):
-    return
-
-@enforce_types
-def search(text: str) -> List[str]:
-    if not RIPGREP_VERSION:
-        raise Exception("ripgrep binary not found, install ripgrep to use this search backend")
-
-    from core.models import Snapshot
-
-    rg_cmd = ['rg', RG_ADD_TYPE, RG_IGNORE_ARGUMENTS, RG_DEFAULT_ARGUMENTS, RG_REGEX_ARGUMENT, text, str(ARCHIVE_DIR)]
-    rg = run(rg_cmd, stdout=PIPE, stderr=PIPE, timeout=SEARCH_BACKEND_TIMEOUT)
-    file_paths = [p.decode() for p in rg.stdout.splitlines()]
-    timestamps = set()
-    for path in file_paths:
-        ts = ts_regex.findall(path)
-        if ts:
-            timestamps.add(ts[0])
-    
-    snap_ids = [str(id) for id in Snapshot.objects.filter(timestamp__in=timestamps).values_list('pk', flat=True)]
-
-    return snap_ids