fix npm and pip binprovider setup and paths search

archivebox/builtin_plugins/npm/apps.py

@@ -1,16 +1,19 @@
 __package__ = 'archivebox.builtin_plugins.npm'
 
+from pathlib import Path
 from typing import List, Optional
-from pydantic import InstanceOf, Field
 
 from django.conf import settings
+from pydantic import InstanceOf, Field
+
+from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr, BinProviderName
 
-from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr
 from plugantic.base_plugin import BasePlugin
 from plugantic.base_configset import BaseConfigSet, ConfigSectionName
 from plugantic.base_binary import BaseBinary, BaseBinProvider, env, apt, brew
 from plugantic.base_hook import BaseHook
 
+
 from ...config import CONFIG
 
 ###################### Config ##########################
@@ -31,11 +34,22 @@ DEFAULT_GLOBAL_CONFIG = {
 NPM_CONFIG = NpmDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
 
 
-class CustomNpmProvider(NpmProvider, BaseBinProvider):
+class SystemNpmProvider(NpmProvider, BaseBinProvider):
+    name: BinProviderName = "npm"
     PATH: PATHStr = str(CONFIG.NODE_BIN_PATH)
+    
+    npm_prefix: Optional[Path] = None
 
-NPM_BINPROVIDER = CustomNpmProvider(PATH=str(CONFIG.NODE_BIN_PATH))
+class LibNpmProvider(NpmProvider, BaseBinProvider):
-npm = NPM_BINPROVIDER
+    name: BinProviderName = "lib_npm"
+    PATH: PATHStr = str(CONFIG.NODE_BIN_PATH)
+    
+    npm_prefix: Optional[Path] = settings.CONFIG.LIB_DIR / 'npm'
+
+
+SYS_NPM_BINPROVIDER = SystemNpmProvider()
+LIB_NPM_BINPROVIDER = LibNpmProvider()
+npm = LIB_NPM_BINPROVIDER
 
 class NpmBinary(BaseBinary):
     name: BinName = 'npm'
@@ -59,7 +73,8 @@ class NpmPlugin(BasePlugin):
     
     hooks: List[InstanceOf[BaseHook]] = [
         NPM_CONFIG,
-        NPM_BINPROVIDER,
+        SYS_NPM_BINPROVIDER,
+        LIB_NPM_BINPROVIDER,
         NODE_BINARY,
         NPM_BINARY,
     ]

archivebox/builtin_plugins/pip/apps.py

@@ -7,11 +7,11 @@ from pydantic import InstanceOf, Field
 
 import django
 
-from django.db.backends.sqlite3.base import Database as sqlite3     # type: ignore[import-type]
+from django.db.backends.sqlite3.base import Database as django_sqlite3     # type: ignore[import-type]
 from django.core.checks import Error, Tags
 from django.conf import settings
 
-from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer
+from pydantic_pkgr import BinProvider, PipProvider, BinName, BinProviderName, ProviderLookupDict, SemVer
 from plugantic.base_plugin import BasePlugin
 from plugantic.base_configset import BaseConfigSet, ConfigSectionName
 from plugantic.base_check import BaseCheck
@@ -36,37 +36,41 @@ DEFAULT_GLOBAL_CONFIG = {
 }
 PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
 
-class CustomPipProvider(PipProvider, BaseBinProvider):
+class SystemPipBinProvider(PipProvider, BaseBinProvider):
-    name: str = 'pip'
+    name: BinProviderName = "pip"
-    INSTALLER_BIN: str = 'pip'
+    INSTALLER_BIN: BinName = "pip"
-    PATH: PATHStr = str(Path(sys.executable).parent)
+    
+    pip_venv: Optional[Path] = None        # global pip scope
+    
+
+class SystemPipxBinProvider(PipProvider, BaseBinProvider):
+    name: BinProviderName = "pipx"
+    INSTALLER_BIN: BinName = "pipx"
 
 
-PIP_BINPROVIDER = CustomPipProvider(PATH=str(Path(sys.executable).parent))
+class LibPipBinProvider(PipProvider, BaseBinProvider):
-pip = PIP_BINPROVIDER
+    name: BinProviderName = "lib_pip"
-
+    INSTALLER_BIN: BinName = "pip"
-class PipBinary(BaseBinary):
+    
-    name: BinName = 'pip'
+    pip_venv: Optional[Path] = settings.CONFIG.OUTPUT_DIR / 'lib' / 'pip' / 'venv'
-    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
-
-PIP_BINARY = PipBinary()
-
 
+SYS_PIP_BINPROVIDER = SystemPipBinProvider()
+SYS_PIPX_BINPROVIDER = SystemPipxBinProvider()
+LIB_PIP_BINPROVIDER = LibPipBinProvider()
+pip = LIB_PIP_BINPROVIDER
 
 
 
 class PythonBinary(BaseBinary):
     name: BinName = 'python'
 
-    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
+    binproviders_supported: List[InstanceOf[BinProvider]] = [SYS_PIP_BINPROVIDER, apt, brew, env]
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
-        'apt': {
+        SYS_PIP_BINPROVIDER.name: {
-            'packages': \
+            'abspath': lambda:
-                lambda: 'python3 python3-minimal python3-pip python3-setuptools python3-virtualenv',
+                sys.executable,
-            'abspath': \
+            'version': lambda: 
-                lambda: sys.executable,
+                '{}.{}.{}'.format(*sys.version_info[:3]),
-            'version': \
-                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
         },
     }
 
@@ -74,13 +78,13 @@ PYTHON_BINARY = PythonBinary()
 
 class SqliteBinary(BaseBinary):
     name: BinName = 'sqlite'
-    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[SYS_PIP_BINPROVIDER])
     provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'pip': {
+        SYS_PIP_BINPROVIDER.name: {
-            'abspath': \
+            'abspath': lambda:
-                lambda: Path(inspect.getfile(sqlite3)),
+                Path(inspect.getfile(django_sqlite3)),
-            'version': \
+            'version': lambda:
-                lambda: SemVer(sqlite3.version),
+                SemVer(django_sqlite3.version),
         },
     }
 
@@ -90,18 +94,25 @@ SQLITE_BINARY = SqliteBinary()
 class DjangoBinary(BaseBinary):
     name: BinName = 'django'
 
-    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[SYS_PIP_BINPROVIDER])
     provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'pip': {
+        SYS_PIP_BINPROVIDER.name: {
-            'abspath': \
+            'abspath': lambda:
-                lambda: inspect.getfile(django),
+                inspect.getfile(django),
-            'version': \
+            'version': lambda:
-                lambda: django.VERSION[:3],
+                django.VERSION[:3],
         },
     }
 
 DJANGO_BINARY = DjangoBinary()
 
+class PipBinary(BaseBinary):
+    name: BinName = "pip"
+    binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
+
+
+PIP_BINARY = PipBinary()
+
 
 class CheckUserIsNotRoot(BaseCheck):
     label: str = 'CheckUserIsNotRoot'
@@ -120,9 +131,30 @@ class CheckUserIsNotRoot(BaseCheck):
             )
         logger.debug('[√] UID is not root')
         return errors
+    
+class CheckPipEnvironment(BaseCheck):
+    label: str = "CheckPipEnvironment"
+    tag: str = Tags.database
+
+    @staticmethod
+    def check(settings, logger) -> List[Warning]:
+        errors = []
+       
+        LIB_PIP_BINPROVIDER.setup()
+        if not LIB_PIP_BINPROVIDER.INSTALLER_BIN_ABSPATH:
+            errors.append(
+                Error(
+                    "Failed to setup data/lib/pip virtualenv for runtime dependencies!",
+                    id="pip.P001",
+                    hint="Make sure the data dir is writable and make sure python3-pip and python3-venv are installed & available on the host.",
+                )
+            )
+        logger.debug("[√] CheckPipEnvironment: data/lib/pip virtualenv is setup properly")
+        return errors
 
 
 USER_IS_NOT_ROOT_CHECK = CheckUserIsNotRoot()
+PIP_ENVIRONMENT_CHECK = CheckPipEnvironment()
 
 
 class PipPlugin(BasePlugin):
@@ -131,12 +163,15 @@ class PipPlugin(BasePlugin):
 
     hooks: List[InstanceOf[BaseHook]] = [
         PIP_CONFIG,
-        PIP_BINPROVIDER,
+        SYS_PIP_BINPROVIDER,
+        SYS_PIPX_BINPROVIDER,
+        LIB_PIP_BINPROVIDER,
         PIP_BINARY,
         PYTHON_BINARY,
         SQLITE_BINARY,
         DJANGO_BINARY,
         USER_IS_NOT_ROOT_CHECK,
+        PIP_ENVIRONMENT_CHECK,
     ]
 
 PLUGIN = PipPlugin()

archivebox/plugantic/base_binary.py

@@ -1,8 +1,9 @@
 __package__ = "archivebox.plugantic"
 
 from typing import Dict, List
+from typing_extensions import Self
 
-from pydantic import Field, InstanceOf
+from pydantic import Field, InstanceOf, validate_call
 from pydantic_pkgr import (
     Binary,
     BinProvider,
@@ -13,6 +14,7 @@ from pydantic_pkgr import (
     EnvProvider,
 )
 
+from django.conf import settings
 
 from .base_hook import BaseHook, HookType
 from ..config_stubs import AttrDict
@@ -40,7 +42,7 @@ class BaseBinProvider(BaseHook, BinProvider):
         settings.BINPROVIDERS[self.id] = self
 
         super().register(settings, parent_plugin=parent_plugin)
-
+        
 
 
 class BaseBinary(BaseHook, Binary):
@@ -57,6 +59,34 @@ class BaseBinary(BaseHook, Binary):
 
         super().register(settings, parent_plugin=parent_plugin)
 
+    @staticmethod
+    def symlink_to_lib(binary, bin_dir=settings.CONFIG.BIN_DIR) -> None:
+        if not (binary.abspath and binary.abspath.exists()):
+            return
+        
+        bin_dir.mkdir(parents=True, exist_ok=True)
+        
+        symlink = bin_dir / binary.name
+        symlink.unlink(missing_ok=True)
+        symlink.symlink_to(binary.abspath)
+
+    @validate_call
+    def load(self, **kwargs) -> Self:
+        binary = super().load(**kwargs)
+        self.symlink_to_lib(binary=binary, bin_dir=settings.CONFIG.BIN_DIR)
+        return binary
+    
+    @validate_call
+    def install(self, **kwargs) -> Self:
+        binary = super().install(**kwargs)
+        self.symlink_to_lib(binary=binary, bin_dir=settings.CONFIG.BIN_DIR)
+        return binary
+    
+    @validate_call
+    def load_or_install(self, **kwargs) -> Self:
+        binary = super().load_or_install(**kwargs)
+        self.symlink_to_lib(binary=binary, bin_dir=settings.CONFIG.BIN_DIR)
+        return binary
 
 apt = AptProvider()
 brew = BrewProvider()