From dd6d7e49759947169c21dbe49bcc13f950934572 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sat, 21 Sep 2024 01:52:36 -0700 Subject: [PATCH] fix npm and pip binprovider setup and paths search --- archivebox/builtin_plugins/npm/apps.py | 27 +++++-- archivebox/builtin_plugins/pip/apps.py | 107 ++++++++++++++++--------- archivebox/plugantic/base_binary.py | 34 +++++++- 3 files changed, 124 insertions(+), 44 deletions(-) diff --git a/archivebox/builtin_plugins/npm/apps.py b/archivebox/builtin_plugins/npm/apps.py index cd3f5826..c6c43660 100644 --- a/archivebox/builtin_plugins/npm/apps.py +++ b/archivebox/builtin_plugins/npm/apps.py @@ -1,16 +1,19 @@ __package__ = 'archivebox.builtin_plugins.npm' +from pathlib import Path from typing import List, Optional -from pydantic import InstanceOf, Field from django.conf import settings +from pydantic import InstanceOf, Field + +from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr, BinProviderName -from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr from plugantic.base_plugin import BasePlugin from plugantic.base_configset import BaseConfigSet, ConfigSectionName from plugantic.base_binary import BaseBinary, BaseBinProvider, env, apt, brew from plugantic.base_hook import BaseHook + from ...config import CONFIG ###################### Config ########################## @@ -31,11 +34,22 @@ DEFAULT_GLOBAL_CONFIG = { NPM_CONFIG = NpmDependencyConfigs(**DEFAULT_GLOBAL_CONFIG) -class CustomNpmProvider(NpmProvider, BaseBinProvider): +class SystemNpmProvider(NpmProvider, BaseBinProvider): + name: BinProviderName = "npm" PATH: PATHStr = str(CONFIG.NODE_BIN_PATH) + + npm_prefix: Optional[Path] = None -NPM_BINPROVIDER = CustomNpmProvider(PATH=str(CONFIG.NODE_BIN_PATH)) -npm = NPM_BINPROVIDER +class LibNpmProvider(NpmProvider, BaseBinProvider): + name: BinProviderName = "lib_npm" + PATH: PATHStr = str(CONFIG.NODE_BIN_PATH) + + npm_prefix: Optional[Path] = settings.CONFIG.LIB_DIR / 'npm' + + +SYS_NPM_BINPROVIDER = SystemNpmProvider() +LIB_NPM_BINPROVIDER = LibNpmProvider() +npm = LIB_NPM_BINPROVIDER class NpmBinary(BaseBinary): name: BinName = 'npm' @@ -59,7 +73,8 @@ class NpmPlugin(BasePlugin): hooks: List[InstanceOf[BaseHook]] = [ NPM_CONFIG, - NPM_BINPROVIDER, + SYS_NPM_BINPROVIDER, + LIB_NPM_BINPROVIDER, NODE_BINARY, NPM_BINARY, ] diff --git a/archivebox/builtin_plugins/pip/apps.py b/archivebox/builtin_plugins/pip/apps.py index 965f370e..a0b661c7 100644 --- a/archivebox/builtin_plugins/pip/apps.py +++ b/archivebox/builtin_plugins/pip/apps.py @@ -7,11 +7,11 @@ from pydantic import InstanceOf, Field import django -from django.db.backends.sqlite3.base import Database as sqlite3 # type: ignore[import-type] +from django.db.backends.sqlite3.base import Database as django_sqlite3 # type: ignore[import-type] from django.core.checks import Error, Tags from django.conf import settings -from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer +from pydantic_pkgr import BinProvider, PipProvider, BinName, BinProviderName, ProviderLookupDict, SemVer from plugantic.base_plugin import BasePlugin from plugantic.base_configset import BaseConfigSet, ConfigSectionName from plugantic.base_check import BaseCheck @@ -36,37 +36,41 @@ DEFAULT_GLOBAL_CONFIG = { } PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG) -class CustomPipProvider(PipProvider, BaseBinProvider): - name: str = 'pip' - INSTALLER_BIN: str = 'pip' - PATH: PATHStr = str(Path(sys.executable).parent) +class SystemPipBinProvider(PipProvider, BaseBinProvider): + name: BinProviderName = "pip" + INSTALLER_BIN: BinName = "pip" + + pip_venv: Optional[Path] = None # global pip scope + + +class SystemPipxBinProvider(PipProvider, BaseBinProvider): + name: BinProviderName = "pipx" + INSTALLER_BIN: BinName = "pipx" -PIP_BINPROVIDER = CustomPipProvider(PATH=str(Path(sys.executable).parent)) -pip = PIP_BINPROVIDER - -class PipBinary(BaseBinary): - name: BinName = 'pip' - binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env] - -PIP_BINARY = PipBinary() - +class LibPipBinProvider(PipProvider, BaseBinProvider): + name: BinProviderName = "lib_pip" + INSTALLER_BIN: BinName = "pip" + + pip_venv: Optional[Path] = settings.CONFIG.OUTPUT_DIR / 'lib' / 'pip' / 'venv' +SYS_PIP_BINPROVIDER = SystemPipBinProvider() +SYS_PIPX_BINPROVIDER = SystemPipxBinProvider() +LIB_PIP_BINPROVIDER = LibPipBinProvider() +pip = LIB_PIP_BINPROVIDER class PythonBinary(BaseBinary): name: BinName = 'python' - binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env] + binproviders_supported: List[InstanceOf[BinProvider]] = [SYS_PIP_BINPROVIDER, apt, brew, env] provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { - 'apt': { - 'packages': \ - lambda: 'python3 python3-minimal python3-pip python3-setuptools python3-virtualenv', - 'abspath': \ - lambda: sys.executable, - 'version': \ - lambda: '{}.{}.{}'.format(*sys.version_info[:3]), + SYS_PIP_BINPROVIDER.name: { + 'abspath': lambda: + sys.executable, + 'version': lambda: + '{}.{}.{}'.format(*sys.version_info[:3]), }, } @@ -74,13 +78,13 @@ PYTHON_BINARY = PythonBinary() class SqliteBinary(BaseBinary): name: BinName = 'sqlite' - binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip]) + binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[SYS_PIP_BINPROVIDER]) provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { - 'pip': { - 'abspath': \ - lambda: Path(inspect.getfile(sqlite3)), - 'version': \ - lambda: SemVer(sqlite3.version), + SYS_PIP_BINPROVIDER.name: { + 'abspath': lambda: + Path(inspect.getfile(django_sqlite3)), + 'version': lambda: + SemVer(django_sqlite3.version), }, } @@ -90,18 +94,25 @@ SQLITE_BINARY = SqliteBinary() class DjangoBinary(BaseBinary): name: BinName = 'django' - binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip]) + binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[SYS_PIP_BINPROVIDER]) provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { - 'pip': { - 'abspath': \ - lambda: inspect.getfile(django), - 'version': \ - lambda: django.VERSION[:3], + SYS_PIP_BINPROVIDER.name: { + 'abspath': lambda: + inspect.getfile(django), + 'version': lambda: + django.VERSION[:3], }, } DJANGO_BINARY = DjangoBinary() +class PipBinary(BaseBinary): + name: BinName = "pip" + binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env] + + +PIP_BINARY = PipBinary() + class CheckUserIsNotRoot(BaseCheck): label: str = 'CheckUserIsNotRoot' @@ -120,9 +131,30 @@ class CheckUserIsNotRoot(BaseCheck): ) logger.debug('[√] UID is not root') return errors + +class CheckPipEnvironment(BaseCheck): + label: str = "CheckPipEnvironment" + tag: str = Tags.database + + @staticmethod + def check(settings, logger) -> List[Warning]: + errors = [] + + LIB_PIP_BINPROVIDER.setup() + if not LIB_PIP_BINPROVIDER.INSTALLER_BIN_ABSPATH: + errors.append( + Error( + "Failed to setup data/lib/pip virtualenv for runtime dependencies!", + id="pip.P001", + hint="Make sure the data dir is writable and make sure python3-pip and python3-venv are installed & available on the host.", + ) + ) + logger.debug("[√] CheckPipEnvironment: data/lib/pip virtualenv is setup properly") + return errors USER_IS_NOT_ROOT_CHECK = CheckUserIsNotRoot() +PIP_ENVIRONMENT_CHECK = CheckPipEnvironment() class PipPlugin(BasePlugin): @@ -131,12 +163,15 @@ class PipPlugin(BasePlugin): hooks: List[InstanceOf[BaseHook]] = [ PIP_CONFIG, - PIP_BINPROVIDER, + SYS_PIP_BINPROVIDER, + SYS_PIPX_BINPROVIDER, + LIB_PIP_BINPROVIDER, PIP_BINARY, PYTHON_BINARY, SQLITE_BINARY, DJANGO_BINARY, USER_IS_NOT_ROOT_CHECK, + PIP_ENVIRONMENT_CHECK, ] PLUGIN = PipPlugin() diff --git a/archivebox/plugantic/base_binary.py b/archivebox/plugantic/base_binary.py index 9adeb350..810f56b9 100644 --- a/archivebox/plugantic/base_binary.py +++ b/archivebox/plugantic/base_binary.py @@ -1,8 +1,9 @@ __package__ = "archivebox.plugantic" from typing import Dict, List +from typing_extensions import Self -from pydantic import Field, InstanceOf +from pydantic import Field, InstanceOf, validate_call from pydantic_pkgr import ( Binary, BinProvider, @@ -13,6 +14,7 @@ from pydantic_pkgr import ( EnvProvider, ) +from django.conf import settings from .base_hook import BaseHook, HookType from ..config_stubs import AttrDict @@ -40,7 +42,7 @@ class BaseBinProvider(BaseHook, BinProvider): settings.BINPROVIDERS[self.id] = self super().register(settings, parent_plugin=parent_plugin) - + class BaseBinary(BaseHook, Binary): @@ -57,6 +59,34 @@ class BaseBinary(BaseHook, Binary): super().register(settings, parent_plugin=parent_plugin) + @staticmethod + def symlink_to_lib(binary, bin_dir=settings.CONFIG.BIN_DIR) -> None: + if not (binary.abspath and binary.abspath.exists()): + return + + bin_dir.mkdir(parents=True, exist_ok=True) + + symlink = bin_dir / binary.name + symlink.unlink(missing_ok=True) + symlink.symlink_to(binary.abspath) + + @validate_call + def load(self, **kwargs) -> Self: + binary = super().load(**kwargs) + self.symlink_to_lib(binary=binary, bin_dir=settings.CONFIG.BIN_DIR) + return binary + + @validate_call + def install(self, **kwargs) -> Self: + binary = super().install(**kwargs) + self.symlink_to_lib(binary=binary, bin_dir=settings.CONFIG.BIN_DIR) + return binary + + @validate_call + def load_or_install(self, **kwargs) -> Self: + binary = super().load_or_install(**kwargs) + self.symlink_to_lib(binary=binary, bin_dir=settings.CONFIG.BIN_DIR) + return binary apt = AptProvider() brew = BrewProvider()