diff --git a/archivebox/abx/archivebox/base_binary.py b/archivebox/abx/archivebox/base_binary.py index 7f352fd8..2533025b 100644 --- a/archivebox/abx/archivebox/base_binary.py +++ b/archivebox/abx/archivebox/base_binary.py @@ -95,10 +95,15 @@ class BaseBinary(BaseHook, Binary): return binary @validate_call - def load_or_install(self, **kwargs) -> Self: - binary = super().load_or_install(**kwargs) - self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR) - return binary + def load_or_install(self, fresh=False, **kwargs) -> Self: + try: + binary = self.load(fresh=fresh) + if binary and binary.version: + self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR) + return binary + except Exception: + pass + return self.install(**kwargs) @property def admin_url(self) -> str: diff --git a/archivebox/main.py b/archivebox/main.py index 548799bb..8a8fc59a 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -961,7 +961,7 @@ def install(out_dir: Path=DATA_DIR) -> None: providers = ' [grey53]or[/grey53] '.join(provider.name for provider in binary.binproviders_supported) print(f'[+] Locating / Installing [yellow]{binary.name}[/yellow] using [red]{providers}[/red]...') try: - print(binary.load_or_install().model_dump(exclude={'binproviders_supported', 'loaded_binprovider', 'provider_overrides', 'loaded_abspaths', 'bin_dir', 'loaded_respath', 'hook_type'})) + print(binary.load_or_install(fresh=True).model_dump(exclude={'provider_overrides', 'bin_dir', 'hook_type'})) except Exception as e: print(f'[X] Failed to install {binary.name}: {e}') diff --git a/archivebox/plugins_extractor/readability/apps.py b/archivebox/plugins_extractor/readability/apps.py index 3e27587a..c7a84009 100644 --- a/archivebox/plugins_extractor/readability/apps.py +++ b/archivebox/plugins_extractor/readability/apps.py @@ -1,7 +1,7 @@ __package__ = 'archivebox.plugins_extractor.readability' from pathlib import Path -from typing import List, Dict, Optional, ClassVar +from typing import List, Dict, Optional # from typing_extensions import Self # Depends on other PyPI/vendor packages: @@ -45,17 +45,17 @@ class ReadabilityBinary(BaseBinary): } @validate_call - def install(self, binprovider_name: Optional[BinProviderName]=None) -> ShallowBinary: + def install(self, binprovider_name: Optional[BinProviderName]=None, **kwargs) -> ShallowBinary: # force install to only use lib/npm provider, we never want to modify global NPM packages - return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name) + return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs) @validate_call - def load_or_install(self, binprovider_name: Optional[BinProviderName] = None) -> ShallowBinary: - # force install to only use lib/npm provider, we never want to modify global NPM packages + def load_or_install(self, binprovider_name: Optional[BinProviderName] = None, fresh=False, **kwargs) -> ShallowBinary: try: - return self.load() + return self.load(fresh=fresh) except Exception: - return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name) + # force install to only use lib/npm provider, we never want to modify global NPM packages + return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs) diff --git a/archivebox/plugins_extractor/singlefile/apps.py b/archivebox/plugins_extractor/singlefile/apps.py index 66ae69cc..e3535ded 100644 --- a/archivebox/plugins_extractor/singlefile/apps.py +++ b/archivebox/plugins_extractor/singlefile/apps.py @@ -66,18 +66,16 @@ class SinglefileBinary(BaseBinary): }, } - @validate_call - def install(self, binprovider_name: Optional[BinProviderName]=None) -> ShallowBinary: + def install(self, binprovider_name: Optional[BinProviderName]=None, **kwargs) -> ShallowBinary: # force install to only use lib/npm provider, we never want to modify global NPM packages - return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name) + return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs) - @validate_call - def load_or_install(self, binprovider_name: Optional[BinProviderName] = None) -> ShallowBinary: - # force install to only use lib/npm provider, we never want to modify global NPM packages + def load_or_install(self, binprovider_name: Optional[BinProviderName]=None, fresh=False, **kwargs) -> ShallowBinary: try: - return self.load() + return self.load(fresh=fresh) except Exception: - return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name) + # force install to only use lib/npm provider, we never want to modify global NPM packages + return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs) diff --git a/archivebox/plugins_pkg/pip/apps.py b/archivebox/plugins_pkg/pip/apps.py index 5ab28251..31c05786 100644 --- a/archivebox/plugins_pkg/pip/apps.py +++ b/archivebox/plugins_pkg/pip/apps.py @@ -5,17 +5,16 @@ import sys import inspect from pathlib import Path from typing import List, Dict, Optional -from pydantic import InstanceOf, Field, model_validator +from pydantic import InstanceOf, Field, model_validator, validate_call import django from django.db.backends.sqlite3.base import Database as django_sqlite3 # type: ignore[import-type] from django.core.checks import Error, Tags -from pydantic_pkgr import BinProvider, PipProvider, BinName, BinProviderName, ProviderLookupDict, SemVer +from pydantic_pkgr import BinProvider, PipProvider, BinName, BinProviderName, ProviderLookupDict, SemVer, bin_abspath from archivebox.config import CONSTANTS, VERSION -import abx from abx.archivebox.base_plugin import BasePlugin from abx.archivebox.base_configset import BaseConfigSet from abx.archivebox.base_check import BaseCheck @@ -35,12 +34,9 @@ class PipDependencyConfigs(BaseConfigSet): PIP_EXTRA_ARGS: List[str] = [] PIP_DEFAULT_ARGS: List[str] = [] +PIP_CONFIG = PipDependencyConfigs() -DEFAULT_GLOBAL_CONFIG = { -} -PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG) - class SystemPipBinProvider(PipProvider, BaseBinProvider): name: BinProviderName = "sys_pip" INSTALLER_BIN: BinName = "pip" @@ -83,11 +79,19 @@ class ArchiveboxBinary(BaseBinary): binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env] provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { - VENV_PIP_BINPROVIDER.name: {'packages': lambda: [], 'version': lambda: VERSION}, - SYS_PIP_BINPROVIDER.name: {'packages': lambda: [], 'version': lambda: VERSION}, - apt.name: {'packages': lambda: [], 'version': lambda: VERSION}, - brew.name: {'packages': lambda: [], 'version': lambda: VERSION}, + VENV_PIP_BINPROVIDER.name: {'packages': lambda: [], 'version': lambda: VERSION, 'abspath': lambda: bin_abspath('archivebox')}, + SYS_PIP_BINPROVIDER.name: {'packages': lambda: [], 'version': lambda: VERSION, 'abspath': lambda: bin_abspath('archivebox')}, + apt.name: {'packages': lambda: [], 'version': lambda: VERSION, 'abspath': lambda: bin_abspath('archivebox')}, + brew.name: {'packages': lambda: [], 'version': lambda: VERSION, 'abspath': lambda: bin_abspath('archivebox')}, } + + @validate_call + def install(self, **kwargs): + return self.load() # obviously it's already installed if we are running this ;) + + @validate_call + def load_or_install(self, **kwargs): + return self.load() # obviously it's already installed if we are running this ;) ARCHIVEBOX_BINARY = ArchiveboxBinary() @@ -98,12 +102,18 @@ class PythonBinary(BaseBinary): binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env] provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { SYS_PIP_BINPROVIDER.name: { - 'abspath': lambda: - sys.executable, - 'version': lambda: - '{}.{}.{}'.format(*sys.version_info[:3]), + 'abspath': lambda: sys.executable, + 'version': lambda: '{}.{}.{}'.format(*sys.version_info[:3]), }, } + + @validate_call + def install(self, **kwargs): + return self.load() # obviously it's already installed if we are running this ;) + + @validate_call + def load_or_install(self, **kwargs): + return self.load() # obviously it's already installed if we are running this ;) PYTHON_BINARY = PythonBinary() @@ -134,6 +144,14 @@ class SqliteBinary(BaseBinary): 'https://code.djangoproject.com/wiki/JSON1Extension' ]) return self + + @validate_call + def install(self, **kwargs): + return self.load() # obviously it's already installed if we are running this ;) + + @validate_call + def load_or_install(self, **kwargs): + return self.load() # obviously it's already installed if we are running this ;) SQLITE_BINARY = SqliteBinary() @@ -152,6 +170,14 @@ class DjangoBinary(BaseBinary): "version": lambda: django.VERSION[:3], }, } + + @validate_call + def install(self, **kwargs): + return self.load() # obviously it's already installed if we are running this ;) + + @validate_call + def load_or_install(self, **kwargs): + return self.load() # obviously it's already installed if we are running this ;) DJANGO_BINARY = DjangoBinary() @@ -159,6 +185,13 @@ class PipBinary(BaseBinary): name: BinName = "pip" binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env] + @validate_call + def install(self, **kwargs): + return self.load() # obviously it's already installed if we are running this ;) + + @validate_call + def load_or_install(self, **kwargs): + return self.load() # obviously it's already installed if we are running this ;) PIP_BINARY = PipBinary() @@ -242,9 +275,3 @@ class PipPlugin(BasePlugin): PLUGIN = PipPlugin() # PLUGIN.register(settings) DJANGO_APP = PLUGIN.AppConfig - - -@abx.hookimpl -def register_django_checks(settings): - USER_IS_NOT_ROOT_CHECK.register_with_django_check_system(settings) - PIP_ENVIRONMENT_CHECK.register_with_django_check_system(settings) diff --git a/archivebox/vendor/pydantic-pkgr b/archivebox/vendor/pydantic-pkgr index 5bb42056..7647d375 160000 --- a/archivebox/vendor/pydantic-pkgr +++ b/archivebox/vendor/pydantic-pkgr @@ -1 +1 @@ -Subproject commit 5bb42056bda9269e600885d83369b89f8dd916a5 +Subproject commit 7647d3757ef0655bd8ba106c4c5b27ae16909e03