diff --git a/archivebox/builtin_plugins/npm/apps.py b/archivebox/builtin_plugins/npm/apps.py index dbdf15fb..5b137f9a 100644 --- a/archivebox/builtin_plugins/npm/apps.py +++ b/archivebox/builtin_plugins/npm/apps.py @@ -40,11 +40,18 @@ npm = NpmProvider(PATH=str(CONFIG.NODE_BIN_PATH)) class NpmBinary(BaseBinary): name: BinName = 'npm' - binproviders_supported: List[InstanceOf[BinProvider]] = [env, apt, brew] + binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env] NPM_BINARY = NpmBinary() +class NodeBinary(BaseBinary): + name: BinName = 'node' + binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env] + + +NODE_BINARY = NodeBinary() + class NpmPlugin(BasePlugin): @@ -54,7 +61,7 @@ class NpmPlugin(BasePlugin): configs: List[InstanceOf[BaseConfigSet]] = [NPM_CONFIG] binproviders: List[InstanceOf[BaseBinProvider]] = [npm] - binaries: List[InstanceOf[BaseBinary]] = [NPM_BINARY] + binaries: List[InstanceOf[BaseBinary]] = [NODE_BINARY, NPM_BINARY] PLUGIN = NpmPlugin() diff --git a/archivebox/builtin_plugins/pip/apps.py b/archivebox/builtin_plugins/pip/apps.py index 101cab52..f6a77408 100644 --- a/archivebox/builtin_plugins/pip/apps.py +++ b/archivebox/builtin_plugins/pip/apps.py @@ -1,13 +1,19 @@ import sys +import inspect from pathlib import Path from typing import List, Dict, Optional from pydantic import InstanceOf, Field +import django from django.apps import AppConfig -from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr +from django.db.backends.sqlite3.base import Database as sqlite3 +from django.core.checks import Error, Tags, register + +from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider from plugantic.base_configset import ConfigSectionName +from plugantic.base_check import BaseCheck from pkg.settings import env, apt, brew @@ -37,13 +43,75 @@ pip = PipProvider(PATH=str(Path(sys.executable).parent)) class PipBinary(BaseBinary): name: BinName = 'pip' - binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew] + binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env] PIP_BINARY = PipBinary() +class PythonBinary(BaseBinary): + name: BinName = 'python' + + binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env] + provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { + 'apt': { + 'subdeps': \ + lambda: 'python3 python3-minimal python3-pip python3-virtualenv', + 'abspath': \ + lambda: sys.executable, + 'version': \ + lambda: '{}.{}.{}'.format(*sys.version_info[:3]), + }, + } + +class SqliteBinary(BaseBinary): + name: BinName = 'sqlite' + binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip]) + provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { + 'pip': { + 'abspath': \ + lambda: Path(inspect.getfile(sqlite3)), + 'version': \ + lambda: SemVer(sqlite3.version), + }, + } + + +class DjangoBinary(BaseBinary): + name: BinName = 'django' + + binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip]) + provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { + 'pip': { + 'abspath': \ + lambda: inspect.getfile(django), + 'version': \ + lambda: django.VERSION[:3], + }, + } + + + + +class CheckUserIsNotRoot(BaseCheck): + label: str = 'CheckUserIsNotRoot' + tag = Tags.database + + @staticmethod + def check(settings, logger) -> List[Warning]: + errors = [] + if getattr(settings, "USER", None) == 'root' or getattr(settings, "PUID", None) == 0: + errors.append( + Error( + "Cannot run as root!", + id="core.S001", + hint=f'Run ArchiveBox as a non-root user with a UID greater than 500. (currently running as UID {os.getuid()}).', + ) + ) + logger.debug('[√] UID is not root') + return errors + @@ -54,7 +122,8 @@ class PipPlugin(BasePlugin): configs: List[InstanceOf[BaseConfigSet]] = [PIP_CONFIG] binproviders: List[InstanceOf[BaseBinProvider]] = [pip] - binaries: List[InstanceOf[BaseBinary]] = [PIP_BINARY] + binaries: List[InstanceOf[BaseBinary]] = [PIP_BINARY, PythonBinary(), SqliteBinary(), DjangoBinary()] + checks: List[InstanceOf[BaseCheck]] = [CheckUserIsNotRoot()] PLUGIN = PipPlugin() diff --git a/archivebox/builtin_plugins/singlefile/apps.py b/archivebox/builtin_plugins/singlefile/apps.py index 8c60419c..3b1924f3 100644 --- a/archivebox/builtin_plugins/singlefile/apps.py +++ b/archivebox/builtin_plugins/singlefile/apps.py @@ -71,7 +71,7 @@ def get_singlefile_abspath() -> Optional[Path]: class SinglefileBinary(BaseBinary): name: BinName = 'single-file' - binproviders_supported: List[InstanceOf[BinProvider]] = [env, npm] + binproviders_supported: List[InstanceOf[BinProvider]] = [npm, env] provider_overrides: Dict[BinProviderName, ProviderLookupDict] ={ # 'env': { diff --git a/archivebox/builtin_plugins/systempython/__init__.py b/archivebox/builtin_plugins/systempython/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/archivebox/builtin_plugins/systempython/admin.py b/archivebox/builtin_plugins/systempython/admin.py deleted file mode 100644 index 8c38f3f3..00000000 --- a/archivebox/builtin_plugins/systempython/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/archivebox/builtin_plugins/systempython/apps.py b/archivebox/builtin_plugins/systempython/apps.py deleted file mode 100644 index 24939e82..00000000 --- a/archivebox/builtin_plugins/systempython/apps.py +++ /dev/null @@ -1,116 +0,0 @@ -__package__ = 'archivebox.builtin_plugins.systempython' - -import os -import sys -import inspect -from typing import List, Dict, Any, Callable, ClassVar -from pathlib import Path - -import django -from django.apps import AppConfig -from django.core.checks import Tags, Warning, register -from django.utils.functional import classproperty -from django.db.backends.sqlite3.base import Database as sqlite3 -from django.core.checks import Tags, Error, register - -from pydantic import InstanceOf, Field - -from pydantic_pkgr import SemVer, BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider - -from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider, BaseExtractor, BaseReplayer -from plugantic.base_check import BaseCheck - -from pkg.settings import env, apt, brew - -from builtin_plugins.pip.apps import pip - -class PythonBinary(BaseBinary): - name: BinName = 'python' - - binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env] - provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { - 'apt': { - 'subdeps': \ - lambda: 'python3 python3-minimal python3-pip python3-virtualenv', - 'abspath': \ - lambda: sys.executable, - 'version': \ - lambda: '{}.{}.{}'.format(*sys.version_info[:3]), - }, - } - -class SqliteBinary(BaseBinary): - name: BinName = 'sqlite' - binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip]) - provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { - 'pip': { - 'abspath': \ - lambda: Path(inspect.getfile(sqlite3)), - 'version': \ - lambda: SemVer(sqlite3.version), - }, - } - - -class DjangoBinary(BaseBinary): - name: BinName = 'django' - - binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip]) - provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { - 'pip': { - 'abspath': \ - lambda: inspect.getfile(django), - 'version': \ - lambda: django.VERSION[:3], - }, - } - - -class BasicReplayer(BaseReplayer): - name: str = 'basic' - - - - -class CheckUserIsNotRoot(BaseCheck): - label: str = 'CheckUserIsNotRoot' - tag = Tags.database - - @staticmethod - def check(settings, logger) -> List[Warning]: - errors = [] - if getattr(settings, "USER", None) == 'root' or getattr(settings, "PUID", None) == 0: - errors.append( - Error( - "Cannot run as root!", - id="core.S001", - hint=f'Run ArchiveBox as a non-root user with a UID greater than 500. (currently running as UID {os.getuid()}).', - ) - ) - logger.debug('[√] UID is not root') - return errors - - - -class SystemPythonPlugin(BasePlugin): - name: str = 'builtin_plugins.systempython' - app_label: str = 'systempython' - verbose_name: str = 'System Python' - - configs: List[InstanceOf[BaseConfigSet]] = [] - binaries: List[InstanceOf[BaseBinary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()] - extractors: List[InstanceOf[BaseExtractor]] = [] - replayers: List[InstanceOf[BaseReplayer]] = [BasicReplayer()] - checks: List[InstanceOf[BaseCheck]] = [CheckUserIsNotRoot()] - - -PLUGIN = SystemPythonPlugin() -DJANGO_APP = PLUGIN.AppConfig -# CONFIGS = PLUGIN.configs -# BINARIES = PLUGIN.binaries -# EXTRACTORS = PLUGIN.extractors -# REPLAYERS = PLUGIN.replayers -# PARSERS = PLUGIN.parsers -# DAEMONS = PLUGIN.daemons -# MODELS = PLUGIN.models -# CHECKS = PLUGIN.checks diff --git a/archivebox/builtin_plugins/systempython/migrations/__init__.py b/archivebox/builtin_plugins/systempython/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/archivebox/builtin_plugins/systempython/models.py b/archivebox/builtin_plugins/systempython/models.py deleted file mode 100644 index 71a83623..00000000 --- a/archivebox/builtin_plugins/systempython/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/archivebox/builtin_plugins/systempython/tests.py b/archivebox/builtin_plugins/systempython/tests.py deleted file mode 100644 index 7ce503c2..00000000 --- a/archivebox/builtin_plugins/systempython/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/archivebox/builtin_plugins/systempython/views.py b/archivebox/builtin_plugins/systempython/views.py deleted file mode 100644 index 91ea44a2..00000000 --- a/archivebox/builtin_plugins/systempython/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/archivebox/builtin_plugins/ytdlp/apps.py b/archivebox/builtin_plugins/ytdlp/apps.py index cb7758f4..a635b17f 100644 --- a/archivebox/builtin_plugins/ytdlp/apps.py +++ b/archivebox/builtin_plugins/ytdlp/apps.py @@ -1,4 +1,5 @@ import sys +import shutil from pathlib import Path from typing import List, Dict, Optional from subprocess import run, PIPE, CompletedProcess @@ -31,21 +32,24 @@ YTDLP_CONFIG = YtdlpDependencyConfigs(**DEFAULT_GLOBAL_CONFIG) class YtdlpBinary(BaseBinary): name: BinName = YTDLP_CONFIG.YTDLP_BINARY - binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew] + binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env] class FfmpegBinary(BaseBinary): name: BinName = 'ffmpeg' - binproviders_supported: List[InstanceOf[BinProvider]] = [env, apt, brew] + binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env] provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { 'env': { - 'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout, + # 'abspath': lambda: shutil.which('ffmpeg', PATH=env.PATH), + # 'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout, }, 'apt': { - 'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout, + # 'abspath': lambda: shutil.which('ffmpeg', PATH=apt.PATH), + 'version': lambda: run(['apt', 'show', 'ffmpeg'], stdout=PIPE, stderr=PIPE, text=True).stdout, }, 'brew': { - 'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout, + # 'abspath': lambda: shutil.which('ffmpeg', PATH=brew.PATH), + 'version': lambda: run(['brew', 'info', 'ffmpeg', '--quiet'], stdout=PIPE, stderr=PIPE, text=True).stdout, }, } diff --git a/archivebox/vendor/pydantic-pkgr b/archivebox/vendor/pydantic-pkgr index 36aaa4f9..ce9c3319 160000 --- a/archivebox/vendor/pydantic-pkgr +++ b/archivebox/vendor/pydantic-pkgr @@ -1 +1 @@ -Subproject commit 36aaa4f9098e5987e23394398aa56154582bd2d2 +Subproject commit ce9c33192df319f843655c80018c4126b5d3fad1