wip

packages/abx-plugin-pip-binprovider/abx_plugin_pip_binprovider/.plugin_order

@@ -0,0 +1 @@
+0

packages/abx-plugin-pip-binprovider/abx_plugin_pip_binprovider/__init__.py

@@ -0,0 +1,37 @@
+__package__ = 'abx_plugin_pip_binprovider'
+__id__ = 'pip'
+__label__ = 'PIP'
+
+import abx
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import PIP_CONFIG
+    
+    return {
+        __id__: PIP_CONFIG
+    }
+
+@abx.hookimpl(tryfirst=True)
+def get_BINARIES():
+    from .binaries import ARCHIVEBOX_BINARY, PYTHON_BINARY, DJANGO_BINARY, SQLITE_BINARY, PIP_BINARY, PIPX_BINARY
+    
+    return {
+        'archivebox': ARCHIVEBOX_BINARY,
+        'python': PYTHON_BINARY,
+        'django': DJANGO_BINARY,
+        'sqlite': SQLITE_BINARY,
+        'pip': PIP_BINARY,
+        'pipx': PIPX_BINARY,
+    }
+
+@abx.hookimpl
+def get_BINPROVIDERS():
+    from .binproviders import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER
+    
+    return {
+        'sys_pip': SYS_PIP_BINPROVIDER,
+        'venv_pip': VENV_PIP_BINPROVIDER,
+        'lib_pip': LIB_PIP_BINPROVIDER,
+    }

packages/abx-plugin-pip-binprovider/abx_plugin_pip_binprovider/binaries.py

@@ -0,0 +1,162 @@
+__package__ = 'abx_plugin_pip_binprovider'
+
+import sys
+from pathlib import Path
+from typing import List
+from pydantic import InstanceOf, Field, model_validator
+
+
+import django
+import django.db.backends.sqlite3.base
+from django.db.backends.sqlite3.base import Database as django_sqlite3     # type: ignore[import-type]
+from pydantic_pkgr import BinProvider, Binary, BinName, BinaryOverrides, SemVer
+
+
+from .binproviders import LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, env, apt, brew
+
+###################### Config ##########################
+
+def get_archivebox_version():
+    try:
+        from archivebox import VERSION
+        return VERSION
+    except Exception:
+        return None
+
+
+class ArchiveboxBinary(Binary):
+    name: BinName = 'archivebox'
+
+    binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
+    overrides: BinaryOverrides = {
+        VENV_PIP_BINPROVIDER.name:  {'packages': [], 'version': get_archivebox_version},
+        SYS_PIP_BINPROVIDER.name:   {'packages': [], 'version': get_archivebox_version},
+        apt.name:                   {'packages': [], 'version': get_archivebox_version},
+        brew.name:                  {'packages': [], 'version': get_archivebox_version},
+    }
+    
+    # @validate_call
+    def install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+    
+    # @validate_call
+    def load_or_install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+
+ARCHIVEBOX_BINARY = ArchiveboxBinary()
+
+
+class PythonBinary(Binary):
+    name: BinName = 'python'
+
+    binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
+    overrides: BinaryOverrides = {
+        SYS_PIP_BINPROVIDER.name: {
+            'abspath': sys.executable,
+            'version': '{}.{}.{}'.format(*sys.version_info[:3]),
+        },
+    }
+    
+    # @validate_call
+    def install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+    
+    # @validate_call
+    def load_or_install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+
+PYTHON_BINARY = PythonBinary()
+
+
+LOADED_SQLITE_PATH = Path(django.db.backends.sqlite3.base.__file__)
+LOADED_SQLITE_VERSION = SemVer(django_sqlite3.version)
+LOADED_SQLITE_FROM_VENV = str(LOADED_SQLITE_PATH.absolute().resolve()).startswith(str(VENV_PIP_BINPROVIDER.pip_venv.absolute().resolve()))
+
+class SqliteBinary(Binary):
+    name: BinName = 'sqlite'
+    binproviders_supported: List[InstanceOf[BinProvider]] = Field(default=[VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER])
+    overrides: BinaryOverrides = {
+        VENV_PIP_BINPROVIDER.name: {
+            "abspath": LOADED_SQLITE_PATH if LOADED_SQLITE_FROM_VENV else None,
+            "version": LOADED_SQLITE_VERSION if LOADED_SQLITE_FROM_VENV else None,
+        },
+        SYS_PIP_BINPROVIDER.name: {
+            "abspath": LOADED_SQLITE_PATH if not LOADED_SQLITE_FROM_VENV else None,
+            "version": LOADED_SQLITE_VERSION if not LOADED_SQLITE_FROM_VENV else None,
+        },
+    }
+    
+    @model_validator(mode='after')
+    def validate_json_extension_is_available(self):
+        # Check to make sure JSON extension is available in our Sqlite3 instance
+        try:
+            cursor = django_sqlite3.connect(':memory:').cursor()
+            cursor.execute('SELECT JSON(\'{"a": "b"}\')')
+        except django_sqlite3.OperationalError as exc:
+            print(f'[red][X] Your SQLite3 version is missing the required JSON1 extension: {exc}[/red]')
+            print(
+                '[violet]Hint:[/violet] Upgrade your Python version or install the extension manually:\n' +
+                '      https://code.djangoproject.com/wiki/JSON1Extension\n'
+            )
+        return self
+    
+    # @validate_call
+    def install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+    
+    # @validate_call
+    def load_or_install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+
+SQLITE_BINARY = SqliteBinary()
+
+
+LOADED_DJANGO_PATH = Path(django.__file__)
+LOADED_DJANGO_VERSION = SemVer(django.VERSION[:3])
+LOADED_DJANGO_FROM_VENV = str(LOADED_DJANGO_PATH.absolute().resolve()).startswith(str(VENV_PIP_BINPROVIDER.pip_venv and VENV_PIP_BINPROVIDER.pip_venv.absolute().resolve()))
+
+class DjangoBinary(Binary):
+    name: BinName = 'django'
+
+    binproviders_supported: List[InstanceOf[BinProvider]] = Field(default=[VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER])
+    overrides: BinaryOverrides = {
+        VENV_PIP_BINPROVIDER.name: {
+            "abspath": LOADED_DJANGO_PATH if LOADED_DJANGO_FROM_VENV else None,
+            "version": LOADED_DJANGO_VERSION if LOADED_DJANGO_FROM_VENV else None,
+        },
+        SYS_PIP_BINPROVIDER.name: {
+            "abspath": LOADED_DJANGO_PATH if not LOADED_DJANGO_FROM_VENV else None,
+            "version": LOADED_DJANGO_VERSION if not LOADED_DJANGO_FROM_VENV else None,
+        },
+    }
+    
+    # @validate_call
+    def install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+    
+    # @validate_call
+    def load_or_install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+
+DJANGO_BINARY = DjangoBinary()
+
+class PipBinary(Binary):
+    name: BinName = "pip"
+    binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
+
+    # @validate_call
+    def install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+    
+    # @validate_call
+    def load_or_install(self, **kwargs):
+        return self.load()                  # obviously it's already installed if we are running this ;)
+
+PIP_BINARY = PipBinary()
+
+
+class PipxBinary(Binary):
+    name: BinName = "pipx"
+    binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
+
+PIPX_BINARY = PipxBinary()

packages/abx-plugin-pip-binprovider/abx_plugin_pip_binprovider/binproviders.py

@@ -0,0 +1,91 @@
+import os
+import sys
+import site
+from pathlib import Path
+from typing import Optional
+
+from benedict import benedict
+
+from pydantic_pkgr import PipProvider, BinName, BinProviderName
+
+import abx
+
+from abx_plugin_default_binproviders import get_BINPROVIDERS
+
+DEFAULT_BINPROVIDERS = benedict(get_BINPROVIDERS())
+env = DEFAULT_BINPROVIDERS.env
+apt = DEFAULT_BINPROVIDERS.apt
+brew = DEFAULT_BINPROVIDERS.brew
+
+
+###################### Config ##########################
+
+class SystemPipBinProvider(PipProvider):
+    name: BinProviderName = "sys_pip"
+    INSTALLER_BIN: BinName = "pip"
+    
+    pip_venv: Optional[Path] = None        # global pip scope
+    
+    def on_install(self, bin_name: str, **kwargs):
+        # never modify system pip packages
+        return 'refusing to install packages globally with system pip, use a venv instead'
+
+class SystemPipxBinProvider(PipProvider):
+    name: BinProviderName = "pipx"
+    INSTALLER_BIN: BinName = "pipx"
+    
+    pip_venv: Optional[Path] = None        # global pipx scope
+
+
+IS_INSIDE_VENV = sys.prefix != sys.base_prefix
+
+class VenvPipBinProvider(PipProvider):
+    name: BinProviderName = "venv_pip"
+    INSTALLER_BIN: BinName = "pip"
+
+    pip_venv: Optional[Path] = Path(sys.prefix if IS_INSIDE_VENV else os.environ.get("VIRTUAL_ENV", '/tmp/NotInsideAVenv/lib'))
+    
+    def setup(self):
+        """never attempt to create a venv here, this is just used to detect if we are inside an existing one"""
+        return None
+    
+
+class LibPipBinProvider(PipProvider):
+    name: BinProviderName = "lib_pip"
+    INSTALLER_BIN: BinName = "pip"
+    
+    pip_venv: Optional[Path] = Path('/usr/local/share/abx/pip/venv')
+    
+    def setup(self) -> None:
+        # update venv path to match most up-to-date LIB_DIR based on runtime config
+        LIB_DIR = abx.pm.hook.get_FLAT_CONFIG().LIB_DIR
+        self.pip_venv = LIB_DIR / 'pip' / 'venv'
+        super().setup()
+
+SYS_PIP_BINPROVIDER = SystemPipBinProvider()
+PIPX_PIP_BINPROVIDER = SystemPipxBinProvider()
+VENV_PIP_BINPROVIDER = VenvPipBinProvider()
+LIB_PIP_BINPROVIDER = LibPipBinProvider()
+pip = LIB_PIP_BINPROVIDER
+
+# ensure python libraries are importable from these locations (if archivebox wasnt executed from one of these then they wont already be in sys.path)
+assert VENV_PIP_BINPROVIDER.pip_venv is not None
+assert LIB_PIP_BINPROVIDER.pip_venv is not None
+
+major, minor, patch = sys.version_info[:3]
+site_packages_dir = f'lib/python{major}.{minor}/site-packages'
+
+LIB_SITE_PACKAGES = (LIB_PIP_BINPROVIDER.pip_venv / site_packages_dir,)
+VENV_SITE_PACKAGES = (VENV_PIP_BINPROVIDER.pip_venv / site_packages_dir,)
+USER_SITE_PACKAGES = site.getusersitepackages()
+SYS_SITE_PACKAGES = site.getsitepackages()
+
+ALL_SITE_PACKAGES = (
+    *LIB_SITE_PACKAGES,
+    *VENV_SITE_PACKAGES,
+    *USER_SITE_PACKAGES,
+    *SYS_SITE_PACKAGES,
+)
+for site_packages_dir in ALL_SITE_PACKAGES:
+    if site_packages_dir not in sys.path:
+        sys.path.append(str(site_packages_dir))

packages/abx-plugin-pip-binprovider/abx_plugin_pip_binprovider/config.py

@@ -0,0 +1,16 @@
+__package__ = 'pip'
+
+from typing import List, Optional
+from pydantic import Field
+
+from abx.archivebox.base_configset import BaseConfigSet
+
+
+class PipDependencyConfigs(BaseConfigSet):
+    USE_PIP: bool = True
+    PIP_BINARY: str = Field(default='pip')
+    PIP_ARGS: Optional[List[str]] = Field(default=None)
+    PIP_EXTRA_ARGS: List[str] = []
+    PIP_DEFAULT_ARGS: List[str] = []
+    
+PIP_CONFIG = PipDependencyConfigs()