From 9af260df162adad4f45c434e768be0fb9bffbb75 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 3 Sep 2024 00:58:50 -0700 Subject: [PATCH] BasePlugin system expanded and registration system improved --- archivebox/builtin_plugins/base/apps.py | 83 ------- .../builtin_plugins/{base => npm}/__init__.py | 0 archivebox/builtin_plugins/npm/apps.py | 66 ++++++ .../{base/migrations => pip}/__init__.py | 0 archivebox/builtin_plugins/pip/apps.py | 66 ++++++ archivebox/builtin_plugins/singlefile/apps.py | 123 ++++++----- .../builtin_plugins/singlefile/config.yaml | 66 ------ .../builtin_plugins/singlefile/tests.py | 3 - .../builtin_plugins/systempython/__init__.py | 0 .../{base => systempython}/admin.py | 0 .../builtin_plugins/systempython/apps.py | 116 ++++++++++ .../systempython/migrations/__init__.py | 0 .../{base => systempython}/models.py | 0 .../{base => systempython}/tests.py | 0 .../{base => systempython}/views.py | 0 archivebox/builtin_plugins/ytdlp/__init__.py | 0 archivebox/builtin_plugins/ytdlp/apps.py | 48 +++++ archivebox/core/settings.py | 149 +++++++++---- archivebox/main.py | 2 +- archivebox/pkg/settings.py | 71 +----- archivebox/plugantic/__init__.py | 19 +- archivebox/plugantic/apps.py | 14 +- archivebox/plugantic/base_admindataview.py | 34 +++ archivebox/plugantic/base_binary.py | 99 +++++++++ archivebox/plugantic/base_check.py | 55 +++++ archivebox/plugantic/base_configset.py | 81 +++++++ .../{extractors.py => base_extractor.py} | 67 +++--- archivebox/plugantic/base_plugin.py | 202 ++++++++++++++++++ .../{replayers.py => base_replayer.py} | 17 +- archivebox/plugantic/binaries.py | 65 ------ archivebox/plugantic/configs.py | 53 ----- .../plugantic/migrations/0001_initial.py | 38 ---- .../migrations/0002_alter_plugin_schema.py | 21 -- .../migrations/0003_alter_plugin_schema.py | 21 -- ...lugin_schema_plugin_configs_plugin_name.py | 32 --- .../0005_customplugin_delete_plugin.py | 39 ---- .../0006_alter_customplugin_path.py | 19 -- .../0007_alter_customplugin_path.py | 19 -- .../0008_alter_customplugin_path.py | 19 -- .../0009_alter_customplugin_path.py | 18 -- .../0010_alter_customplugin_path.py | 18 -- .../0011_alter_customplugin_path.py | 18 -- .../0012_alter_customplugin_path.py | 18 -- .../0013_alter_customplugin_path.py | 18 -- .../0014_alter_customplugin_path.py | 18 -- .../0015_alter_customplugin_path.py | 18 -- .../migrations/0016_delete_customplugin.py | 16 -- archivebox/plugantic/plugins.py | 122 ----------- archivebox/plugantic/views.py | 62 ++++-- archivebox/vendor/pydantic-pkgr | 2 +- 50 files changed, 1062 insertions(+), 973 deletions(-) delete mode 100644 archivebox/builtin_plugins/base/apps.py rename archivebox/builtin_plugins/{base => npm}/__init__.py (100%) create mode 100644 archivebox/builtin_plugins/npm/apps.py rename archivebox/builtin_plugins/{base/migrations => pip}/__init__.py (100%) create mode 100644 archivebox/builtin_plugins/pip/apps.py delete mode 100644 archivebox/builtin_plugins/singlefile/config.yaml delete mode 100644 archivebox/builtin_plugins/singlefile/tests.py create mode 100644 archivebox/builtin_plugins/systempython/__init__.py rename archivebox/builtin_plugins/{base => systempython}/admin.py (100%) create mode 100644 archivebox/builtin_plugins/systempython/apps.py create mode 100644 archivebox/builtin_plugins/systempython/migrations/__init__.py rename archivebox/builtin_plugins/{base => systempython}/models.py (100%) rename archivebox/builtin_plugins/{base => systempython}/tests.py (100%) rename archivebox/builtin_plugins/{base => systempython}/views.py (100%) create mode 100644 archivebox/builtin_plugins/ytdlp/__init__.py create mode 100644 archivebox/builtin_plugins/ytdlp/apps.py create mode 100644 archivebox/plugantic/base_admindataview.py create mode 100644 archivebox/plugantic/base_binary.py create mode 100644 archivebox/plugantic/base_check.py create mode 100644 archivebox/plugantic/base_configset.py rename archivebox/plugantic/{extractors.py => base_extractor.py} (60%) create mode 100644 archivebox/plugantic/base_plugin.py rename archivebox/plugantic/{replayers.py => base_replayer.py} (62%) delete mode 100644 archivebox/plugantic/binaries.py delete mode 100644 archivebox/plugantic/configs.py delete mode 100644 archivebox/plugantic/migrations/0001_initial.py delete mode 100644 archivebox/plugantic/migrations/0002_alter_plugin_schema.py delete mode 100644 archivebox/plugantic/migrations/0003_alter_plugin_schema.py delete mode 100644 archivebox/plugantic/migrations/0004_remove_plugin_schema_plugin_configs_plugin_name.py delete mode 100644 archivebox/plugantic/migrations/0005_customplugin_delete_plugin.py delete mode 100644 archivebox/plugantic/migrations/0006_alter_customplugin_path.py delete mode 100644 archivebox/plugantic/migrations/0007_alter_customplugin_path.py delete mode 100644 archivebox/plugantic/migrations/0008_alter_customplugin_path.py delete mode 100644 archivebox/plugantic/migrations/0009_alter_customplugin_path.py delete mode 100644 archivebox/plugantic/migrations/0010_alter_customplugin_path.py delete mode 100644 archivebox/plugantic/migrations/0011_alter_customplugin_path.py delete mode 100644 archivebox/plugantic/migrations/0012_alter_customplugin_path.py delete mode 100644 archivebox/plugantic/migrations/0013_alter_customplugin_path.py delete mode 100644 archivebox/plugantic/migrations/0014_alter_customplugin_path.py delete mode 100644 archivebox/plugantic/migrations/0015_alter_customplugin_path.py delete mode 100644 archivebox/plugantic/migrations/0016_delete_customplugin.py delete mode 100644 archivebox/plugantic/plugins.py diff --git a/archivebox/builtin_plugins/base/apps.py b/archivebox/builtin_plugins/base/apps.py deleted file mode 100644 index 291bbe50..00000000 --- a/archivebox/builtin_plugins/base/apps.py +++ /dev/null @@ -1,83 +0,0 @@ -import sys -import inspect -from typing import List, Dict, Any, Optional -from pathlib import Path - -import django -from django.apps import AppConfig -from django.core.checks import Tags, Warning, register -from django.db.backends.sqlite3.base import Database as sqlite3 - -from pydantic import ( - Field, - SerializeAsAny, -) - -from pydantic_pkgr import SemVer, BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider - -from plugantic.extractors import Extractor, ExtractorName -from plugantic.plugins import Plugin -from plugantic.configs import ConfigSet, ConfigSectionName -from plugantic.replayers import Replayer - - -class PythonBinary(Binary): - name: BinName = 'python' - - providers_supported: List[BinProvider] = [EnvProvider()] - provider_overrides: Dict[str, Any] = { - 'env': { - 'subdeps': \ - lambda: 'python3 python3-minimal python3-pip python3-virtualenv', - 'abspath': \ - lambda: sys.executable, - 'version': \ - lambda: '{}.{}.{}'.format(*sys.version_info[:3]), - }, - } - -class SqliteBinary(Binary): - name: BinName = 'sqlite' - providers_supported: List[BinProvider] = [EnvProvider()] - provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { - 'env': { - 'abspath': \ - lambda: Path(inspect.getfile(sqlite3)), - 'version': \ - lambda: SemVer(sqlite3.version), - }, - } - - -class DjangoBinary(Binary): - name: BinName = 'django' - - providers_supported: List[BinProvider] = [EnvProvider()] - provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { - 'env': { - 'abspath': \ - lambda: inspect.getfile(django), - 'version': \ - lambda: django.VERSION[:3], - }, - } - - -class BasicReplayer(Replayer): - name: str = 'basic' - - -class BasePlugin(Plugin): - name: str = 'base' - configs: List[SerializeAsAny[ConfigSet]] = [] - binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()] - extractors: List[SerializeAsAny[Extractor]] = [] - replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()] - - -PLUGINS = [BasePlugin()] - - -class BaseConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'builtin_plugins.base' diff --git a/archivebox/builtin_plugins/base/__init__.py b/archivebox/builtin_plugins/npm/__init__.py similarity index 100% rename from archivebox/builtin_plugins/base/__init__.py rename to archivebox/builtin_plugins/npm/__init__.py diff --git a/archivebox/builtin_plugins/npm/apps.py b/archivebox/builtin_plugins/npm/apps.py new file mode 100644 index 00000000..dbdf15fb --- /dev/null +++ b/archivebox/builtin_plugins/npm/apps.py @@ -0,0 +1,66 @@ +__package__ = 'archivebox.builtin_plugins.npm' + +from pathlib import Path +from typing import List, Dict, Optional +from pydantic import InstanceOf, Field + +from django.apps import AppConfig +from django.conf import settings + +from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr +from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider +from plugantic.base_configset import ConfigSectionName + +from pkg.settings import env, apt, brew + +from ...config import CONFIG + +###################### Config ########################## + + +class NpmDependencyConfigs(BaseConfigSet): + section: ConfigSectionName = 'DEPENDENCY_CONFIG' + + USE_NPM: bool = True + NPM_BINARY: str = Field(default='npm') + NPM_ARGS: Optional[List[str]] = Field(default=None) + NPM_EXTRA_ARGS: List[str] = [] + NPM_DEFAULT_ARGS: List[str] = [] + + +DEFAULT_GLOBAL_CONFIG = { +} +NPM_CONFIG = NpmDependencyConfigs(**DEFAULT_GLOBAL_CONFIG) + + +class NpmProvider(NpmProvider, BaseBinProvider): + PATH: PATHStr = str(CONFIG.NODE_BIN_PATH) + +npm = NpmProvider(PATH=str(CONFIG.NODE_BIN_PATH)) + +class NpmBinary(BaseBinary): + name: BinName = 'npm' + binproviders_supported: List[InstanceOf[BinProvider]] = [env, apt, brew] + + +NPM_BINARY = NpmBinary() + + + +class NpmPlugin(BasePlugin): + name: str = 'builtin_plugins.npm' + app_label: str = 'npm' + verbose_name: str = 'NPM' + + configs: List[InstanceOf[BaseConfigSet]] = [NPM_CONFIG] + binproviders: List[InstanceOf[BaseBinProvider]] = [npm] + binaries: List[InstanceOf[BaseBinary]] = [NPM_BINARY] + + +PLUGIN = NpmPlugin() +DJANGO_APP = PLUGIN.AppConfig +# CONFIGS = PLUGIN.configs +# BINARIES = PLUGIN.binaries +# EXTRACTORS = PLUGIN.extractors +# REPLAYERS = PLUGIN.replayers +# CHECKS = PLUGIN.checks diff --git a/archivebox/builtin_plugins/base/migrations/__init__.py b/archivebox/builtin_plugins/pip/__init__.py similarity index 100% rename from archivebox/builtin_plugins/base/migrations/__init__.py rename to archivebox/builtin_plugins/pip/__init__.py diff --git a/archivebox/builtin_plugins/pip/apps.py b/archivebox/builtin_plugins/pip/apps.py new file mode 100644 index 00000000..101cab52 --- /dev/null +++ b/archivebox/builtin_plugins/pip/apps.py @@ -0,0 +1,66 @@ +import sys +from pathlib import Path +from typing import List, Dict, Optional +from pydantic import InstanceOf, Field + +from django.apps import AppConfig + +from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr +from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider +from plugantic.base_configset import ConfigSectionName + +from pkg.settings import env, apt, brew + + +###################### Config ########################## + + +class PipDependencyConfigs(BaseConfigSet): + section: ConfigSectionName = 'DEPENDENCY_CONFIG' + + USE_PIP: bool = True + PIP_BINARY: str = Field(default='pip') + PIP_ARGS: Optional[List[str]] = Field(default=None) + PIP_EXTRA_ARGS: List[str] = [] + PIP_DEFAULT_ARGS: List[str] = [] + + +DEFAULT_GLOBAL_CONFIG = { +} +PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG) + +class PipProvider(PipProvider, BaseBinProvider): + PATH: PATHStr = str(Path(sys.executable).parent) + +pip = PipProvider(PATH=str(Path(sys.executable).parent)) + + +class PipBinary(BaseBinary): + name: BinName = 'pip' + binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew] +PIP_BINARY = PipBinary() + + + + + + + + +class PipPlugin(BasePlugin): + name: str = 'builtin_plugins.pip' + app_label: str = 'pip' + verbose_name: str = 'PIP' + + configs: List[InstanceOf[BaseConfigSet]] = [PIP_CONFIG] + binproviders: List[InstanceOf[BaseBinProvider]] = [pip] + binaries: List[InstanceOf[BaseBinary]] = [PIP_BINARY] + + +PLUGIN = PipPlugin() +DJANGO_APP = PLUGIN.AppConfig +# CONFIGS = PLUGIN.configs +# BINARIES = PLUGIN.binaries +# EXTRACTORS = PLUGIN.extractors +# REPLAYERS = PLUGIN.replayers +# CHECKS = PLUGIN.checks diff --git a/archivebox/builtin_plugins/singlefile/apps.py b/archivebox/builtin_plugins/singlefile/apps.py index 1d40e8a7..8c60419c 100644 --- a/archivebox/builtin_plugins/singlefile/apps.py +++ b/archivebox/builtin_plugins/singlefile/apps.py @@ -1,42 +1,31 @@ -from typing import List, Optional, Dict from pathlib import Path +from typing import List, Dict, Optional from django.apps import AppConfig -from django.core.checks import Tags, Warning, register -from pydantic import ( - Field, - SerializeAsAny, -) - -from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider +# Depends on other PyPI/vendor packages: +from pydantic import InstanceOf, Field +from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName from pydantic_pkgr.binprovider import bin_abspath -from pydantic_pkgr.binary import BinProviderName, ProviderLookupDict -from plugantic.extractors import Extractor, ExtractorName -from plugantic.plugins import Plugin -from plugantic.configs import ConfigSet, ConfigSectionName +# Depends on other Django apps: +from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseExtractor, BaseReplayer +from plugantic.base_configset import ConfigSectionName +# Depends on Other Plugins: from pkg.settings import env +from builtin_plugins.npm.apps import npm ###################### Config ########################## -class SinglefileToggleConfig(ConfigSet): +class SinglefileToggleConfigs(BaseConfigSet): section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES' SAVE_SINGLEFILE: bool = True -class SinglefileDependencyConfig(ConfigSet): - section: ConfigSectionName = 'DEPENDENCY_CONFIG' - - SINGLEFILE_BINARY: str = Field(default='wget') - SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None) - SINGLEFILE_EXTRA_ARGS: List[str] = [] - SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}'] - -class SinglefileOptionsConfig(ConfigSet): +class SinglefileOptionsConfigs(BaseConfigSet): section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS' # loaded from shared config @@ -47,67 +36,83 @@ class SinglefileOptionsConfig(ConfigSet): SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE') +class SinglefileDependencyConfigs(BaseConfigSet): + section: ConfigSectionName = 'DEPENDENCY_CONFIG' -DEFAULT_CONFIG = { + SINGLEFILE_BINARY: str = Field(default='wget') + SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None) + SINGLEFILE_EXTRA_ARGS: List[str] = [] + SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}'] + +class SinglefileConfigs(SinglefileToggleConfigs, SinglefileOptionsConfigs, SinglefileDependencyConfigs): + # section: ConfigSectionName = 'ALL_CONFIGS' + pass + +DEFAULT_GLOBAL_CONFIG = { 'CHECK_SSL_VALIDITY': False, 'SAVE_SINGLEFILE': True, 'TIMEOUT': 120, } -PLUGIN_CONFIG = [ - SinglefileToggleConfig(**DEFAULT_CONFIG), - SinglefileDependencyConfig(**DEFAULT_CONFIG), - SinglefileOptionsConfig(**DEFAULT_CONFIG), +SINGLEFILE_CONFIGS = [ + SinglefileToggleConfigs(**DEFAULT_GLOBAL_CONFIG), + SinglefileDependencyConfigs(**DEFAULT_GLOBAL_CONFIG), + SinglefileOptionsConfigs(**DEFAULT_GLOBAL_CONFIG), ] -###################### Binaries ############################ + min_version: str = "1.1.54" max_version: str = "2.0.0" -class SinglefileBinary(Binary): - name: BinName = 'single-file' - providers_supported: List[BinProvider] = [NpmProvider()] +def get_singlefile_abspath() -> Optional[Path]: + return +class SinglefileBinary(BaseBinary): + name: BinName = 'single-file' + binproviders_supported: List[InstanceOf[BinProvider]] = [env, npm] + provider_overrides: Dict[BinProviderName, ProviderLookupDict] ={ - 'env': { - 'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH), - }, - 'npm': { - # 'abspath': lambda: bin_abspath('single-file', PATH=NpmProvider().PATH) or bin_abspath('single-file', PATH=env.PATH), - 'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}', - }, + # 'env': { + # 'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH), + # }, + # 'npm': { + # 'abspath': lambda: bin_abspath('single-file', PATH=npm.PATH) or bin_abspath('single-file-node.js', PATH=npm.PATH), + # 'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}', + # }, } +SINGLEFILE_BINARY = SinglefileBinary() -###################### Extractors ########################## +PLUGIN_BINARIES = [SINGLEFILE_BINARY] -class SinglefileExtractor(Extractor): - name: ExtractorName = 'singlefile' - binary: Binary = SinglefileBinary() +class SinglefileExtractor(BaseExtractor): + name: str = 'singlefile' + binary: BinName = SINGLEFILE_BINARY.name def get_output_path(self, snapshot) -> Path: return Path(snapshot.link_dir) / 'singlefile.html' -###################### Plugins ############################# +SINGLEFILE_BINARY = SinglefileBinary() +SINGLEFILE_EXTRACTOR = SinglefileExtractor() + +class SinglefilePlugin(BasePlugin): + name: str = 'builtin_plugins.singlefile' + app_label: str ='singlefile' + verbose_name: str = 'SingleFile' + + configs: List[InstanceOf[BaseConfigSet]] = SINGLEFILE_CONFIGS + binaries: List[InstanceOf[BaseBinary]] = [SINGLEFILE_BINARY] + extractors: List[InstanceOf[BaseExtractor]] = [SINGLEFILE_EXTRACTOR] -class SinglefilePlugin(Plugin): - name: str = 'singlefile' - configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG] - binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()] - extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()] -PLUGINS = [SinglefilePlugin()] - -###################### Django Apps ######################### - -class SinglefileConfig(AppConfig): - name = 'builtin_plugins.singlefile' - verbose_name = 'SingleFile' - - def ready(self): - pass - # print('Loaded singlefile plugin') +PLUGIN = SinglefilePlugin() +DJANGO_APP = PLUGIN.AppConfig +# CONFIGS = PLUGIN.configs +# BINARIES = PLUGIN.binaries +# EXTRACTORS = PLUGIN.extractors +# REPLAYERS = PLUGIN.replayers +# CHECKS = PLUGIN.checks diff --git a/archivebox/builtin_plugins/singlefile/config.yaml b/archivebox/builtin_plugins/singlefile/config.yaml deleted file mode 100644 index b4d80f06..00000000 --- a/archivebox/builtin_plugins/singlefile/config.yaml +++ /dev/null @@ -1,66 +0,0 @@ -name: singlefile -plugin_version: '0.0.1' -plugin_spec: '0.0.1' - -binaries: - singlefile: - providers: - - env - - npm - -commands: - - singlefile.exec - - singlefile.extract - - singlefile.should_extract - - singlefile.get_output_path - -extractors: - singlefile: - binary: singlefile - test: singlefile.should_extract - extract: singlefile.extract - output_files: - - singlefile.html - -configs: - ARCHIVE_METHOD_TOGGLES: - SAVE_SINGLEFILE: - type: bool - default: true - - DEPENDENCY_CONFIG: - SINGLEFILE_BINARY: - type: str - default: wget - SINGLEFILE_ARGS: - type: Optional[List[str]] - default: null - SINGLEFILE_EXTRA_ARGS: - type: List[str] - default: [] - SINGLEFILE_DEFAULT_ARGS: - type: List[str] - default: - - "--timeout={TIMEOUT-10}" - - ARCHIVE_METHOD_OPTIONS: - SINGLEFILE_USER_AGENT: - type: str - default: "" - alias: USER_AGENT - SINGLEFILE_TIMEOUT: - type: int - default: 60 - alias: TIMEOUT - SINGLEFILE_CHECK_SSL_VALIDITY: - type: bool - default: true - alias: CHECK_SSL_VALIDITY - SINGLEFILE_RESTRICT_FILE_NAMES: - type: str - default: windows - alias: RESTRICT_FILE_NAMES - SINGLEFILE_COOKIES_FILE: - type: Optional[Path] - default: null - alias: COOKIES_FILE diff --git a/archivebox/builtin_plugins/singlefile/tests.py b/archivebox/builtin_plugins/singlefile/tests.py deleted file mode 100644 index 7ce503c2..00000000 --- a/archivebox/builtin_plugins/singlefile/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/archivebox/builtin_plugins/systempython/__init__.py b/archivebox/builtin_plugins/systempython/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/archivebox/builtin_plugins/base/admin.py b/archivebox/builtin_plugins/systempython/admin.py similarity index 100% rename from archivebox/builtin_plugins/base/admin.py rename to archivebox/builtin_plugins/systempython/admin.py diff --git a/archivebox/builtin_plugins/systempython/apps.py b/archivebox/builtin_plugins/systempython/apps.py new file mode 100644 index 00000000..24939e82 --- /dev/null +++ b/archivebox/builtin_plugins/systempython/apps.py @@ -0,0 +1,116 @@ +__package__ = 'archivebox.builtin_plugins.systempython' + +import os +import sys +import inspect +from typing import List, Dict, Any, Callable, ClassVar +from pathlib import Path + +import django +from django.apps import AppConfig +from django.core.checks import Tags, Warning, register +from django.utils.functional import classproperty +from django.db.backends.sqlite3.base import Database as sqlite3 +from django.core.checks import Tags, Error, register + +from pydantic import InstanceOf, Field + +from pydantic_pkgr import SemVer, BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider + +from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider, BaseExtractor, BaseReplayer +from plugantic.base_check import BaseCheck + +from pkg.settings import env, apt, brew + +from builtin_plugins.pip.apps import pip + +class PythonBinary(BaseBinary): + name: BinName = 'python' + + binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env] + provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { + 'apt': { + 'subdeps': \ + lambda: 'python3 python3-minimal python3-pip python3-virtualenv', + 'abspath': \ + lambda: sys.executable, + 'version': \ + lambda: '{}.{}.{}'.format(*sys.version_info[:3]), + }, + } + +class SqliteBinary(BaseBinary): + name: BinName = 'sqlite' + binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip]) + provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { + 'pip': { + 'abspath': \ + lambda: Path(inspect.getfile(sqlite3)), + 'version': \ + lambda: SemVer(sqlite3.version), + }, + } + + +class DjangoBinary(BaseBinary): + name: BinName = 'django' + + binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip]) + provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { + 'pip': { + 'abspath': \ + lambda: inspect.getfile(django), + 'version': \ + lambda: django.VERSION[:3], + }, + } + + +class BasicReplayer(BaseReplayer): + name: str = 'basic' + + + + +class CheckUserIsNotRoot(BaseCheck): + label: str = 'CheckUserIsNotRoot' + tag = Tags.database + + @staticmethod + def check(settings, logger) -> List[Warning]: + errors = [] + if getattr(settings, "USER", None) == 'root' or getattr(settings, "PUID", None) == 0: + errors.append( + Error( + "Cannot run as root!", + id="core.S001", + hint=f'Run ArchiveBox as a non-root user with a UID greater than 500. (currently running as UID {os.getuid()}).', + ) + ) + logger.debug('[√] UID is not root') + return errors + + + +class SystemPythonPlugin(BasePlugin): + name: str = 'builtin_plugins.systempython' + app_label: str = 'systempython' + verbose_name: str = 'System Python' + + configs: List[InstanceOf[BaseConfigSet]] = [] + binaries: List[InstanceOf[BaseBinary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()] + extractors: List[InstanceOf[BaseExtractor]] = [] + replayers: List[InstanceOf[BaseReplayer]] = [BasicReplayer()] + checks: List[InstanceOf[BaseCheck]] = [CheckUserIsNotRoot()] + + +PLUGIN = SystemPythonPlugin() +DJANGO_APP = PLUGIN.AppConfig +# CONFIGS = PLUGIN.configs +# BINARIES = PLUGIN.binaries +# EXTRACTORS = PLUGIN.extractors +# REPLAYERS = PLUGIN.replayers +# PARSERS = PLUGIN.parsers +# DAEMONS = PLUGIN.daemons +# MODELS = PLUGIN.models +# CHECKS = PLUGIN.checks diff --git a/archivebox/builtin_plugins/systempython/migrations/__init__.py b/archivebox/builtin_plugins/systempython/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/archivebox/builtin_plugins/base/models.py b/archivebox/builtin_plugins/systempython/models.py similarity index 100% rename from archivebox/builtin_plugins/base/models.py rename to archivebox/builtin_plugins/systempython/models.py diff --git a/archivebox/builtin_plugins/base/tests.py b/archivebox/builtin_plugins/systempython/tests.py similarity index 100% rename from archivebox/builtin_plugins/base/tests.py rename to archivebox/builtin_plugins/systempython/tests.py diff --git a/archivebox/builtin_plugins/base/views.py b/archivebox/builtin_plugins/systempython/views.py similarity index 100% rename from archivebox/builtin_plugins/base/views.py rename to archivebox/builtin_plugins/systempython/views.py diff --git a/archivebox/builtin_plugins/ytdlp/__init__.py b/archivebox/builtin_plugins/ytdlp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/archivebox/builtin_plugins/ytdlp/apps.py b/archivebox/builtin_plugins/ytdlp/apps.py new file mode 100644 index 00000000..5fb7d3a8 --- /dev/null +++ b/archivebox/builtin_plugins/ytdlp/apps.py @@ -0,0 +1,48 @@ +import sys +from pathlib import Path +from typing import List, Dict, Optional +from pydantic import InstanceOf, Field + +from django.apps import AppConfig + +from pydantic_pkgr import BinProvider, BinName, PATHStr +from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider +from plugantic.base_configset import ConfigSectionName + +from pkg.settings import env, apt, brew + +from builtin_plugins.pip.apps import pip + +###################### Config ########################## + + +class YtdlpDependencyConfigs(BaseConfigSet): + section: ConfigSectionName = 'DEPENDENCY_CONFIG' + + USE_YTDLP: bool = True + + YTDLP_BINARY: str = Field(default='yt-dlp') + +DEFAULT_GLOBAL_CONFIG = {} +YTDLP_CONFIG = YtdlpDependencyConfigs(**DEFAULT_GLOBAL_CONFIG) + + + +class YtdlpBinary(BaseBinary): + name: BinName = YTDLP_CONFIG.YTDLP_BINARY + binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew] + +YTDLP_BINARY = YtdlpBinary() + + +class YtdlpPlugin(BasePlugin): + name: str = 'builtin_plugins.ytdlp' + app_label: str = 'ytdlp' + verbose_name: str = 'YTDLP' + + configs: List[InstanceOf[BaseConfigSet]] = [YTDLP_CONFIG] + binaries: List[InstanceOf[BaseBinary]] = [YTDLP_BINARY] + + +PLUGIN = YtdlpPlugin() +DJANGO_APP = PLUGIN.AppConfig diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 707e17a1..1cf6e3bc 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -19,6 +19,46 @@ IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3] IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3] + +################################################################################ +### ArchiveBox Plugin Settings +################################################################################ + +BUILTIN_PLUGINS_DIR = CONFIG.PACKAGE_DIR / 'builtin_plugins' # /app/archivebox/builtin_plugins +USERDATA_PLUGINS_DIR = CONFIG.OUTPUT_DIR / 'user_plugins' # /data/user_plugins + +def find_plugins_in_dir(plugins_dir, prefix: str) -> Dict[str, Path]: + return { + f'{prefix}.{plugin_entrypoint.parent.name}': plugin_entrypoint.parent + for plugin_entrypoint in sorted(plugins_dir.glob('*/apps.py')) + } + +INSTALLED_PLUGINS = { + **find_plugins_in_dir(BUILTIN_PLUGINS_DIR, prefix='builtin_plugins'), + **find_plugins_in_dir(USERDATA_PLUGINS_DIR, prefix='user_plugins'), +} + +### Plugins Globals (filled by plugantic.apps.load_plugins() after Django startup) +PLUGINS = AttrDict({}) + +CONFIGS = AttrDict({}) +BINPROVIDERS = AttrDict({}) +BINARIES = AttrDict({}) +EXTRACTORS = AttrDict({}) +REPLAYERS = AttrDict({}) +CHECKS = AttrDict({}) +ADMINDATAVIEWS = AttrDict({}) + +PLUGIN_KEYS = AttrDict({ + 'CONFIGS': CONFIGS, + 'BINPROVIDERS': BINPROVIDERS, + 'BINARIES': BINARIES, + 'EXTRACTORS': EXTRACTORS, + 'REPLAYERS': REPLAYERS, + 'CHECKS': CHECKS, + 'ADMINDATAVIEWS': ADMINDATAVIEWS, +}) + ################################################################################ ### Django Core Settings ################################################################################ @@ -35,52 +75,35 @@ APPEND_SLASH = True DEBUG = CONFIG.DEBUG or ('--debug' in sys.argv) -BUILTIN_PLUGINS_DIR = CONFIG.PACKAGE_DIR / 'builtin_plugins' -USER_PLUGINS_DIR = CONFIG.OUTPUT_DIR / 'user_plugins' - -def find_plugins(plugins_dir, prefix: str) -> Dict[str, Any]: - plugins = { - f'{prefix}.{plugin_entrypoint.parent.name}': plugin_entrypoint.parent - for plugin_entrypoint in plugins_dir.glob('*/apps.py') - } - # print(f'Found {prefix} plugins:\n', '\n '.join(plugins.keys())) - return plugins - -INSTALLED_PLUGINS = { - **find_plugins(BUILTIN_PLUGINS_DIR, prefix='builtin_plugins'), - **find_plugins(USER_PLUGINS_DIR, prefix='user_plugins'), -} - - INSTALLED_APPS = [ + # Django default apps 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.admin', - 'django_jsonform', + + # 3rd-party apps from PyPI + 'django_jsonform', # handles rendering Pydantic models to Django HTML widgets/forms + 'signal_webhooks', # handles REST API outbound webhooks - 'signal_webhooks', - 'abid_utils', - 'plugantic', - 'core', - 'api', - 'pkg', + # our own apps + 'abid_utils', # handles ABID ID creation, handling, and models + 'plugantic', # ArchiveBox plugin API definition + finding/registering/calling interface + 'core', # core django model with Snapshot, ArchiveResult, etc. + 'api', # Django-Ninja-based Rest API interfaces, config, APIToken model, etc. + 'pkg', # ArchiveBox runtime package management interface for subdependencies - *INSTALLED_PLUGINS.keys(), + # ArchiveBox plugins + *INSTALLED_PLUGINS.keys(), # all plugin django-apps found in archivebox/builtin_plugins and data/user_plugins - 'admin_data_views', - 'django_extensions', + # 3rd-party apps from PyPI that need to be loaded last + 'admin_data_views', # handles rendering some convenient automatic read-only views of data in Django admin + 'django_extensions', # provides Django Debug Toolbar (and other non-debug helpers) ] -# For usage with https://www.jetadmin.io/integrations/django -# INSTALLED_APPS += ['jet_django'] -# JET_PROJECT = 'archivebox' -# JET_TOKEN = 'some-api-token-here' - - MIDDLEWARE = [ 'core.middleware.TimezoneMiddleware', 'django.middleware.security.SecurityMiddleware', @@ -371,8 +394,11 @@ LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', + "console": { + "level": "DEBUG", + "filters": [], + 'formatter': 'simple', + "class": "logging.StreamHandler", }, 'logfile': { 'level': 'ERROR', @@ -380,14 +406,57 @@ LOGGING = { 'filename': ERROR_LOG, 'maxBytes': 1024 * 1024 * 25, # 25 MB 'backupCount': 10, + 'formatter': 'verbose', }, + # "mail_admins": { + # "level": "ERROR", + # "filters": ["require_debug_false"], + # "class": "django.utils.log.AdminEmailHandler", + # }, }, 'filters': { 'noisyrequestsfilter': { '()': NoisyRequestsFilter, - } + }, + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", + }, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", + }, + }, + 'formatters': { + 'verbose': { + 'format': '{name} {levelname} {asctime} {module} {process:d} {thread:d} {message}', + 'style': '{', + }, + 'simple': { + 'format': '{name} {message}', + 'style': '{', + }, + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[{server_time}] {message}", + "style": "{", + }, }, 'loggers': { + 'api': { + 'handlers': ['console', 'logfile'], + 'level': 'DEBUG', + }, + 'checks': { + 'handlers': ['console', 'logfile'], + 'level': 'DEBUG', + }, + 'core': { + 'handlers': ['console', 'logfile'], + 'level': 'DEBUG', + }, + 'builtin_plugins': { + 'handlers': ['console', 'logfile'], + 'level': 'DEBUG', + }, 'django': { 'handlers': ['console', 'logfile'], 'level': 'INFO', @@ -397,6 +466,8 @@ LOGGING = { 'handlers': ['console', 'logfile'], 'level': 'INFO', 'filters': ['noisyrequestsfilter'], + 'propagate': False, + "formatter": "django.server", } }, } @@ -541,3 +612,9 @@ if DEBUG_REQUESTS_TRACKER: # https://docs.pydantic.dev/logfire/integrations/django/ (similar to DataDog / NewRelic / etc.) DEBUG_LOGFIRE = False DEBUG_LOGFIRE = DEBUG_LOGFIRE and (Path(CONFIG.OUTPUT_DIR) / '.logfire').is_dir() + + +# For usage with https://www.jetadmin.io/integrations/django +# INSTALLED_APPS += ['jet_django'] +# JET_PROJECT = 'archivebox' +# JET_TOKEN = 'some-api-token-here' diff --git a/archivebox/main.py b/archivebox/main.py index 5ab175bb..02d377b1 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -318,7 +318,7 @@ def init(force: bool=False, quick: bool=False, setup: bool=False, out_dir: Path= print('{green}----------------------------------------------------------------------{reset}'.format(**ANSI)) elif existing_index: # TODO: properly detect and print the existing version in current index as well - print('{green}[^] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **ANSI)) + print('{green}[*] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **ANSI)) print('{green}----------------------------------------------------------------------{reset}'.format(**ANSI)) else: if force: diff --git a/archivebox/pkg/settings.py b/archivebox/pkg/settings.py index 7f13d125..972fd91a 100644 --- a/archivebox/pkg/settings.py +++ b/archivebox/pkg/settings.py @@ -10,77 +10,24 @@ import django from django.conf import settings from django.db.backends.sqlite3.base import Database as sqlite3 -from pydantic_pkgr import Binary, BinProvider, BrewProvider, EnvProvider, SemVer +from pydantic_pkgr import Binary, BinProvider, BrewProvider, PipProvider, NpmProvider, AptProvider, EnvProvider, SemVer from pydantic_pkgr.binprovider import bin_abspath from ..config import NODE_BIN_PATH, bin_path -env = EnvProvider(PATH=NODE_BIN_PATH + ':' + os.environ.get('PATH', '/bin')) +apt = AptProvider() +brew = BrewProvider() +env = EnvProvider(PATH=os.environ.get('PATH', '/bin')) +# Defined in their own plugins: +#pip = PipProvider(PATH=str(Path(sys.executable).parent)) +#npm = NpmProvider(PATH=NODE_BIN_PATH) LOADED_DEPENDENCIES = {} -for bin_key, dependency in settings.CONFIG.DEPENDENCIES.items(): - # 'PYTHON_BINARY': { - # 'path': bin_path(config['PYTHON_BINARY']), - # 'version': config['PYTHON_VERSION'], - # 'hash': bin_hash(config['PYTHON_BINARY']), - # 'enabled': True, - # 'is_valid': bool(config['PYTHON_VERSION']), - # }, - - - bin_name = settings.CONFIG[bin_key] - - if bin_name.endswith('django/__init__.py'): - binary_spec = Binary(name='django', providers=[env], provider_overrides={ - 'env': { - 'abspath': lambda: Path(inspect.getfile(django)), - 'version': lambda: SemVer('{}.{}.{} {} ({})'.format(*django.VERSION)), - } - }) - elif bin_name.endswith('sqlite3/dbapi2.py'): - binary_spec = Binary(name='sqlite3', providers=[env], provider_overrides={ - 'env': { - 'abspath': lambda: Path(inspect.getfile(sqlite3)), - 'version': lambda: SemVer(sqlite3.version), - } - }) - elif bin_name.endswith('archivebox'): - binary_spec = Binary(name='archivebox', providers=[env], provider_overrides={ - 'env': { - 'abspath': lambda: shutil.which(str(Path('archivebox').expanduser())), - 'version': lambda: settings.CONFIG.VERSION, - } - }) - elif bin_name.endswith('postlight/parser/cli.js'): - binary_spec = Binary(name='postlight-parser', providers=[env], provider_overrides={ - 'env': { - 'abspath': lambda: bin_path('postlight-parser'), - 'version': lambda: SemVer('1.0.0'), - } - }) - else: - binary_spec = Binary(name=bin_name, providers=[env]) - +for bin_name, binary_spec in settings.BINARIES.items(): try: - binary = binary_spec.load() + settings.BINARIES[bin_name] = binary_spec.load() except Exception as e: # print(f"- ❌ Binary {bin_name} failed to load with error: {e}") continue - - assert isinstance(binary.loaded_version, SemVer) - - try: - assert str(binary.loaded_version) == dependency['version'], f"Expected {bin_name} version {dependency['version']}, got {binary.loaded_version}" - assert str(binary.loaded_respath) == str(bin_abspath(dependency['path']).resolve()), f"Expected {bin_name} abspath {bin_abspath(dependency['path']).resolve()}, got {binary.loaded_respath}" - assert binary.is_valid == dependency['is_valid'], f"Expected {bin_name} is_valid={dependency['is_valid']}, got {binary.is_valid}" - except Exception as e: - pass - # print(f"WARNING: Error loading {bin_name}: {e}") - # import ipdb; ipdb.set_trace() - - # print(f"- ✅ Binary {bin_name} loaded successfully") - LOADED_DEPENDENCIES[bin_key] = binary - - diff --git a/archivebox/plugantic/__init__.py b/archivebox/plugantic/__init__.py index c8f37e05..950a947c 100644 --- a/archivebox/plugantic/__init__.py +++ b/archivebox/plugantic/__init__.py @@ -1,16 +1,9 @@ __package__ = 'archivebox.plugantic' -from .binaries import Binary -from .extractors import Extractor -from .replayers import Replayer -from .configs import ConfigSet -from .plugins import Plugin +from .base_plugin import BasePlugin +from .base_configset import BaseConfigSet +from .base_binary import BaseBinary +from .base_extractor import BaseExtractor +from .base_replayer import BaseReplayer +from .base_check import BaseCheck -# __all__ = [ -# 'BinProvider', -# 'Binary', -# 'Extractor', -# 'Replayer', -# 'ConfigSet', -# 'Plugin', -# ] diff --git a/archivebox/plugantic/apps.py b/archivebox/plugantic/apps.py index 57d57cd8..1212b0a3 100644 --- a/archivebox/plugantic/apps.py +++ b/archivebox/plugantic/apps.py @@ -1,6 +1,9 @@ -import importlib -from django.apps import AppConfig +__package__ = 'archivebox.plugantic' +import json +import importlib + +from django.apps import AppConfig class PluganticConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' @@ -8,10 +11,5 @@ class PluganticConfig(AppConfig): def ready(self) -> None: from django.conf import settings - from .plugins import PLUGINS - for plugin_name in settings.INSTALLED_PLUGINS.keys(): - lib = importlib.import_module(f'{plugin_name}.apps') - if hasattr(lib, 'PLUGINS'): - for plugin_instance in lib.PLUGINS: - PLUGINS.append(plugin_instance) + print(f'[🧩] Detected {len(settings.INSTALLED_PLUGINS)} settings.INSTALLED_PLUGINS to load...') diff --git a/archivebox/plugantic/base_admindataview.py b/archivebox/plugantic/base_admindataview.py new file mode 100644 index 00000000..d3b117e8 --- /dev/null +++ b/archivebox/plugantic/base_admindataview.py @@ -0,0 +1,34 @@ +from typing import List, Type, Any, Dict + +from pydantic_core import core_schema +from pydantic import GetCoreSchemaHandler, BaseModel + +from django.utils.functional import classproperty +from django.core.checks import Warning, Tags, register + +class BaseAdminDataView(BaseModel): + name: str = 'NPM Installed Packages' + route: str = '/npm/installed/' + view: str = 'builtin_plugins.npm.admin.installed_list_view' + items: Dict[str, str] = { + "name": "installed_npm_pkg", + 'route': '/', + 'view': 'builtin_plugins.npm.admin.installed_detail_view', + } + + def as_route(self) -> Dict[str, str | Dict[str, str]]: + return { + 'route': self.route, + 'view': self.view, + 'name': self.name, + 'items': self.items, + } + + def register(self, settings, parent_plugin=None): + """Regsiter AdminDataViews.as_route() in settings.ADMIN_DATA_VIEWS.URLS at runtime""" + self._plugin = parent_plugin # circular ref to parent only here for easier debugging! never depend on circular backref to parent in real code! + + route = self.as_route() + if route not in settings.ADMIN_DATA_VIEWS.URLS: + settings.ADMIN_DATA_VIEWS.URLS += [route] # append our route (update in place) + diff --git a/archivebox/plugantic/base_binary.py b/archivebox/plugantic/base_binary.py new file mode 100644 index 00000000..0f2d47d0 --- /dev/null +++ b/archivebox/plugantic/base_binary.py @@ -0,0 +1,99 @@ +__package__ = 'archivebox.plugantic' + +import sys +import inspect +import importlib +from pathlib import Path + + +from typing import Any, Optional, Dict, List +from typing_extensions import Self +from subprocess import run, PIPE + +from pydantic import Field, InstanceOf +from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict +from pydantic_pkgr.binprovider import HostBinPath + +import django +from django.core.cache import cache +from django.db.backends.sqlite3.base import Database as sqlite3 + + +class BaseBinProvider(BinProvider): + # def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]: + # Class = super() + # get_abspath_func = lambda: Class.on_get_abspath(bin_name, **context) + # # return cache.get_or_set(f'bin:abspath:{bin_name}', get_abspath_func) + # return get_abspath_func() + + # def on_get_version(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> SemVer | None: + # Class = super() + # get_version_func = lambda: Class.on_get_version(bin_name, abspath, **context) + # # return cache.get_or_set(f'bin:version:{bin_name}:{abspath}', get_version_func) + # return get_version_func() + + def register(self, settings, parent_plugin=None): + if settings is None: + from django.conf import settings as django_settings + settings = django_settings + + self._plugin = parent_plugin # for debugging only, never rely on this! + settings.BINPROVIDERS[self.name] = self + + +class BaseBinary(Binary): + binproviders_supported: List[InstanceOf[BinProvider]] = Field(default_factory=list, alias='binproviders') + + def register(self, settings, parent_plugin=None): + if settings is None: + from django.conf import settings as django_settings + settings = django_settings + + self._plugin = parent_plugin # for debugging only, never rely on this! + settings.BINARIES[self.name] = self + +# def get_ytdlp_version() -> str: +# import yt_dlp +# return yt_dlp.version.__version__ + + + + +# class YtdlpBinary(Binary): +# name: BinName = 'yt-dlp' +# providers_supported: List[BinProvider] = [ +# EnvProvider(), +# PipProvider(), +# BrewProvider(), +# AptProvider(), +# ] +# provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { +# 'pip': { +# 'version': get_ytdlp_version, +# }, +# 'brew': { +# 'subdeps': lambda: 'yt-dlp ffmpeg', +# }, +# 'apt': { +# 'subdeps': lambda: 'yt-dlp ffmpeg', +# } +# } + +# class WgetBinary(Binary): +# name: BinName = 'wget' +# providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()] + + +# if __name__ == '__main__': +# PYTHON_BINARY = PythonBinary() +# SQLITE_BINARY = SqliteBinary() +# DJANGO_BINARY = DjangoBinary() +# WGET_BINARY = WgetBinary() +# YTDLP_BINARY = YtdlpPBinary() + +# print('-------------------------------------DEFINING BINARIES---------------------------------') +# print(PYTHON_BINARY) +# print(SQLITE_BINARY) +# print(DJANGO_BINARY) +# print(WGET_BINARY) +# print(YTDLP_BINARY) diff --git a/archivebox/plugantic/base_check.py b/archivebox/plugantic/base_check.py new file mode 100644 index 00000000..542b1957 --- /dev/null +++ b/archivebox/plugantic/base_check.py @@ -0,0 +1,55 @@ +from typing import List, Type, Any + +from pydantic_core import core_schema +from pydantic import GetCoreSchemaHandler + +from django.utils.functional import classproperty +from django.core.checks import Warning, Tags, register + +class BaseCheck: + label: str = '' + tag = Tags.database + + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: + return core_schema.typed_dict_schema( + { + 'name': core_schema.typed_dict_field(core_schema.str_schema()), + 'tag': core_schema.typed_dict_field(core_schema.str_schema()), + }, + ) + + + @classproperty + def name(cls) -> str: + return cls.label or cls.__name__ + + @staticmethod + def check(settings, logger) -> List[Warning]: + """Override this method to implement your custom runtime check.""" + errors = [] + # if not hasattr(settings, 'SOME_KEY'): + # errors.extend(Error( + # 'Missing settings.SOME_KEY after django_setup(), did SOME_KEY get loaded?', + # id='core.C001', + # hint='Make sure to run django_setup() is able to load settings.SOME_KEY.', + # )) + # logger.debug('[√] Loaded settings.PLUGINS succesfully.') + return errors + + def register(self, settings, parent_plugin=None): + # Regsiter in ArchiveBox plugins runtime settings + self._plugin = parent_plugin + settings.CHECKS[self.name] = self + + # Register using Django check framework + def run_check(app_configs, **kwargs) -> List[Warning]: + from django.conf import settings + import logging + settings = settings + logger = logging.getLogger('checks') + return self.check(settings, logger) + + run_check.__name__ = self.label or self.__class__.__name__ + run_check.tags = [self.tag] + register(self.tag)(run_check) diff --git a/archivebox/plugantic/base_configset.py b/archivebox/plugantic/base_configset.py new file mode 100644 index 00000000..5edd1407 --- /dev/null +++ b/archivebox/plugantic/base_configset.py @@ -0,0 +1,81 @@ +__package__ = 'archivebox.plugantic' + + +from typing import Optional, List, Literal +from pathlib import Path +from pydantic import BaseModel, Field, ConfigDict, computed_field + + +ConfigSectionName = Literal[ + 'GENERAL_CONFIG', + 'ARCHIVE_METHOD_TOGGLES', + 'ARCHIVE_METHOD_OPTIONS', + 'DEPENDENCY_CONFIG', +] +ConfigSectionNames: List[ConfigSectionName] = [ + 'GENERAL_CONFIG', + 'ARCHIVE_METHOD_TOGGLES', + 'ARCHIVE_METHOD_OPTIONS', + 'DEPENDENCY_CONFIG', +] + + +class BaseConfigSet(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow', populate_by_name=True) + + section: ConfigSectionName = 'GENERAL_CONFIG' + + @computed_field + @property + def name(self) -> str: + return self.__class__.__name__ + + def register(self, settings, parent_plugin=None): + if settings is None: + from django.conf import settings as django_settings + settings = django_settings + + self._plugin = parent_plugin # for debugging only, never rely on this! + settings.CONFIGS[self.name] = self + + + +# class WgetToggleConfig(ConfigSet): +# section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES' + +# SAVE_WGET: bool = True +# SAVE_WARC: bool = True + +# class WgetDependencyConfig(ConfigSet): +# section: ConfigSectionName = 'DEPENDENCY_CONFIG' + +# WGET_BINARY: str = Field(default='wget') +# WGET_ARGS: Optional[List[str]] = Field(default=None) +# WGET_EXTRA_ARGS: List[str] = [] +# WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}'] + +# class WgetOptionsConfig(ConfigSet): +# section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS' + +# # loaded from shared config +# WGET_AUTO_COMPRESSION: bool = Field(default=True) +# SAVE_WGET_REQUISITES: bool = Field(default=True) +# WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT') +# WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT') +# WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY') +# WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES') +# WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE') + + +# CONFIG = { +# 'CHECK_SSL_VALIDITY': False, +# 'SAVE_WARC': False, +# 'TIMEOUT': 999, +# } + + +# WGET_CONFIG = [ +# WgetToggleConfig(**CONFIG), +# WgetDependencyConfig(**CONFIG), +# WgetOptionsConfig(**CONFIG), +# ] diff --git a/archivebox/plugantic/extractors.py b/archivebox/plugantic/base_extractor.py similarity index 60% rename from archivebox/plugantic/extractors.py rename to archivebox/plugantic/base_extractor.py index 56d594f3..d091ca6a 100644 --- a/archivebox/plugantic/extractors.py +++ b/archivebox/plugantic/base_extractor.py @@ -6,13 +6,14 @@ from typing_extensions import Self from abc import ABC from pathlib import Path -from pydantic import BaseModel, model_validator, field_serializer, AfterValidator +from pydantic import BaseModel, model_validator, field_serializer, AfterValidator, Field +from pydantic_pkgr import BinName -from .binaries import ( - Binary, - YtdlpBinary, - WgetBinary, -) +# from .binaries import ( +# Binary, +# YtdlpBinary, +# WgetBinary, +# ) # stubs @@ -37,9 +38,9 @@ HandlerFuncStr = Annotated[str, AfterValidator(lambda s: s.startswith('self.'))] CmdArgsList = Annotated[List[str], AfterValidator(no_empty_args)] -class Extractor(ABC, BaseModel): +class BaseExtractor(ABC, BaseModel): name: ExtractorName - binary: Binary + binary: BinName output_path_func: HandlerFuncStr = 'self.get_output_path' should_extract_func: HandlerFuncStr = 'self.should_extract' @@ -55,10 +56,14 @@ class Extractor(ABC, BaseModel): if self.args is None: self.args = [*self.default_args, *self.extra_args] return self + + def register(self, settings, parent_plugin=None): + if settings is None: + from django.conf import settings as django_settings + settings = django_settings - @field_serializer('binary', when_used='json') - def dump_binary(binary) -> str: - return binary.name + self._plugin = parent_plugin # for debugging only, never rely on this! + settings.EXTRACTORS[self.name] = self def get_output_path(self, snapshot) -> Path: return Path(self.name) @@ -86,33 +91,37 @@ class Extractor(ABC, BaseModel): 'returncode': proc.returncode, } - def exec(self, args: CmdArgsList, pwd: Optional[Path]=None): + def exec(self, args: CmdArgsList, pwd: Optional[Path]=None, settings=None): pwd = pwd or Path('.') - assert self.binary.loaded_provider - return self.binary.exec(args, pwd=pwd) + if settings is None: + from django.conf import settings as django_settings + settings = django_settings + + binary = settings.BINARIES[self.binary] + return binary.exec(args, pwd=pwd) -class YtdlpExtractor(Extractor): - name: ExtractorName = 'media' - binary: Binary = YtdlpBinary() +# class YtdlpExtractor(Extractor): +# name: ExtractorName = 'media' +# binary: Binary = YtdlpBinary() - def get_output_path(self, snapshot) -> Path: - return Path(self.name) +# def get_output_path(self, snapshot) -> Path: +# return Path(self.name) -class WgetExtractor(Extractor): - name: ExtractorName = 'wget' - binary: Binary = WgetBinary() +# class WgetExtractor(Extractor): +# name: ExtractorName = 'wget' +# binary: Binary = WgetBinary() - def get_output_path(self, snapshot) -> Path: - return get_wget_output_path(snapshot) +# def get_output_path(self, snapshot) -> Path: +# return get_wget_output_path(snapshot) -class WarcExtractor(Extractor): - name: ExtractorName = 'warc' - binary: Binary = WgetBinary() +# class WarcExtractor(Extractor): +# name: ExtractorName = 'warc' +# binary: Binary = WgetBinary() - def get_output_path(self, snapshot) -> Path: - return get_wget_output_path(snapshot) +# def get_output_path(self, snapshot) -> Path: +# return get_wget_output_path(snapshot) diff --git a/archivebox/plugantic/base_plugin.py b/archivebox/plugantic/base_plugin.py new file mode 100644 index 00000000..cdad499c --- /dev/null +++ b/archivebox/plugantic/base_plugin.py @@ -0,0 +1,202 @@ +__package__ = 'archivebox.plugantic' + +import json + +from django.apps import AppConfig +from django.core.checks import register + +from typing import List, ClassVar, Type, Dict +from typing_extensions import Self + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + model_validator, + InstanceOf, + computed_field, + validate_call, +) + +from .base_configset import BaseConfigSet +from .base_binary import BaseBinProvider, BaseBinary +from .base_extractor import BaseExtractor +from .base_replayer import BaseReplayer +from .base_check import BaseCheck +from .base_admindataview import BaseAdminDataView + +from ..config import ANSI, AttrDict + + +class BasePlugin(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True) + + # Required by AppConfig: + name: str = Field() # e.g. 'builtin_plugins.singlefile' + app_label: str = Field() # e.g. 'singlefile' + verbose_name: str = Field() # e.g. 'SingleFile' + default_auto_field: ClassVar[str] = 'django.db.models.AutoField' + + # Required by Plugantic: + configs: List[InstanceOf[BaseConfigSet]] = Field(default=[]) + binproviders: List[InstanceOf[BaseBinProvider]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')] + binaries: List[InstanceOf[BaseBinary]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')] + extractors: List[InstanceOf[BaseExtractor]] = Field(default=[]) + replayers: List[InstanceOf[BaseReplayer]] = Field(default=[]) + checks: List[InstanceOf[BaseCheck]] = Field(default=[]) + admindataviews: List[InstanceOf[BaseAdminDataView]] = Field(default=[]) + + @model_validator(mode='after') + def validate(self) -> Self: + """Validate the plugin's build-time configuration here before it's registered in Django at runtime.""" + + assert self.name and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name' + + assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.' + + @property + def AppConfig(plugin_self) -> Type[AppConfig]: + """Generate a Django AppConfig class for this plugin.""" + + class PluginAppConfig(AppConfig): + name = plugin_self.name + app_label = plugin_self.app_label + verbose_name = plugin_self.verbose_name + + def ready(self): + from django.conf import settings + + plugin_self.validate() + plugin_self.register(settings) + + return PluginAppConfig + + @computed_field + @property + def BINPROVIDERS(self) -> Dict[str, BaseBinProvider]: + return AttrDict({binprovider.name: binprovider for binprovider in self.binproviders}) + + @computed_field + @property + def BINARIES(self) -> Dict[str, BaseBinary]: + return AttrDict({binary.python_name: binary for binary in self.binaries}) + + @computed_field + @property + def CONFIGS(self) -> Dict[str, BaseConfigSet]: + return AttrDict({config.name: config for config in self.configs}) + + @computed_field + @property + def EXTRACTORS(self) -> Dict[str, BaseExtractor]: + return AttrDict({extractor.name: extractor for extractor in self.extractors}) + + @computed_field + @property + def REPLAYERS(self) -> Dict[str, BaseReplayer]: + return AttrDict({replayer.name: replayer for replayer in self.replayers}) + + @computed_field + @property + def CHECKS(self) -> Dict[str, BaseCheck]: + return AttrDict({check.name: check for check in self.checks}) + + @computed_field + @property + def ADMINDATAVIEWS(self) -> Dict[str, BaseCheck]: + return AttrDict({admindataview.name: admindataview for admindataview in self.admindataviews}) + + @computed_field + @property + def PLUGIN_KEYS(self) -> List[str]: + return + + def register(self, settings=None): + """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime.""" + + if settings is None: + from django.conf import settings as django_settings + settings = django_settings + + assert all(hasattr(settings, key) for key in ['PLUGINS', 'CONFIGS', 'BINARIES', 'EXTRACTORS', 'REPLAYERS', 'ADMINDATAVIEWS']), 'Tried to register plugin in settings but couldnt find required global dicts in settings.' + + assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.' + + assert self.app_label not in settings.PLUGINS, f'Tried to register plugin {self.name} but it conflicts with existing plugin of the same name ({self.app_label}).' + + ### Mutate django.conf.settings... values in-place to include plugin-provided overrides + settings.PLUGINS[self.app_label] = self + + for config in self.CONFIGS.values(): + config.register(settings, parent_plugin=self) + + for binprovider in self.BINPROVIDERS.values(): + binprovider.register(settings, parent_plugin=self) + + for binary in self.BINARIES.values(): + binary.register(settings, parent_plugin=self) + + for extractor in self.EXTRACTORS.values(): + extractor.register(settings, parent_plugin=self) + + for replayer in self.REPLAYERS.values(): + replayer.register(settings, parent_plugin=self) + + for check in self.CHECKS.values(): + check.register(settings, parent_plugin=self) + + for admindataview in self.ADMINDATAVIEWS.values(): + admindataview.register(settings, parent_plugin=self) + + # TODO: add parsers? custom templates? persona fixtures? + + plugin_prefix, plugin_shortname = self.name.split('.', 1) + + print( + f' > {ANSI.black}{plugin_prefix.upper().replace("_PLUGINS", "").ljust(15)} ' + + f'{ANSI.lightyellow}{plugin_shortname.ljust(12)} ' + + f'{ANSI.black}CONFIGSx{len(self.configs)} BINARIESx{len(self.binaries)} EXTRACTORSx{len(self.extractors)} REPLAYERSx{len(self.replayers)} CHECKSx{len(self.CHECKS)} ADMINDATAVIEWSx{len(self.ADMINDATAVIEWS)}{ANSI.reset}' + ) + + # @validate_call + # def install_binaries(self) -> Self: + # new_binaries = [] + # for idx, binary in enumerate(self.binaries): + # new_binaries.append(binary.install() or binary) + # return self.model_copy(update={ + # 'binaries': new_binaries, + # }) + + @validate_call + def load_binaries(self, cache=True) -> Self: + new_binaries = [] + for idx, binary in enumerate(self.binaries): + new_binaries.append(binary.load(cache=cache) or binary) + return self.model_copy(update={ + 'binaries': new_binaries, + }) + + # @validate_call + # def load_or_install_binaries(self, cache=True) -> Self: + # new_binaries = [] + # for idx, binary in enumerate(self.binaries): + # new_binaries.append(binary.load_or_install(cache=cache) or binary) + # return self.model_copy(update={ + # 'binaries': new_binaries, + # }) + + + + +# class YtdlpPlugin(BasePlugin): +# name: str = 'ytdlp' +# configs: List[SerializeAsAny[BaseConfigSet]] = [] +# binaries: List[SerializeAsAny[BaseBinary]] = [YtdlpBinary()] +# extractors: List[SerializeAsAny[BaseExtractor]] = [YtdlpExtractor()] +# replayers: List[SerializeAsAny[BaseReplayer]] = [MEDIA_REPLAYER] + +# class WgetPlugin(BasePlugin): +# name: str = 'wget' +# configs: List[SerializeAsAny[BaseConfigSet]] = [*WGET_CONFIG] +# binaries: List[SerializeAsAny[BaseBinary]] = [WgetBinary()] +# extractors: List[SerializeAsAny[BaseExtractor]] = [WgetExtractor(), WarcExtractor()] diff --git a/archivebox/plugantic/replayers.py b/archivebox/plugantic/base_replayer.py similarity index 62% rename from archivebox/plugantic/replayers.py rename to archivebox/plugantic/base_replayer.py index 08f1cd88..4f18415f 100644 --- a/archivebox/plugantic/replayers.py +++ b/archivebox/plugantic/base_replayer.py @@ -3,10 +3,9 @@ __package__ = 'archivebox.plugantic' from pydantic import BaseModel -# from .binproviders import LazyImportStr -class Replayer(BaseModel): +class BaseReplayer(BaseModel): """Describes how to render an ArchiveResult in several contexts""" name: str = 'GenericReplayer' url_pattern: str = '*' @@ -21,5 +20,17 @@ class Replayer(BaseModel): # icon_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon' # thumbnail_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon' + def register(self, settings, parent_plugin=None): + if settings is None: + from django.conf import settings as django_settings + settings = django_settings -MEDIA_REPLAYER = Replayer(name='media') + self._plugin = parent_plugin # for debugging only, never rely on this! + settings.REPLAYERS[self.name] = self + + +# class MediaReplayer(BaseReplayer): +# name: str = 'MediaReplayer' + + +# MEDIA_REPLAYER = MediaReplayer() diff --git a/archivebox/plugantic/binaries.py b/archivebox/plugantic/binaries.py deleted file mode 100644 index 76bd63ac..00000000 --- a/archivebox/plugantic/binaries.py +++ /dev/null @@ -1,65 +0,0 @@ -__package__ = 'archivebox.plugantic' - -import sys -import inspect -import importlib -from pathlib import Path - - -from typing import Any, Optional, Dict, List -from typing_extensions import Self -from subprocess import run, PIPE - -from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict - -import django -from django.db.backends.sqlite3.base import Database as sqlite3 - - - - -def get_ytdlp_version() -> str: - import yt_dlp - return yt_dlp.version.__version__ - - - - -class YtdlpBinary(Binary): - name: BinName = 'yt-dlp' - providers_supported: List[BinProvider] = [ - EnvProvider(), - PipProvider(), - BrewProvider(), - AptProvider(), - ] - provider_overrides: Dict[BinProviderName, ProviderLookupDict] = { - 'pip': { - 'version': get_ytdlp_version, - }, - 'brew': { - 'subdeps': lambda: 'yt-dlp ffmpeg', - }, - 'apt': { - 'subdeps': lambda: 'yt-dlp ffmpeg', - } - } - -class WgetBinary(Binary): - name: BinName = 'wget' - providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()] - - -# if __name__ == '__main__': -# PYTHON_BINARY = PythonBinary() -# SQLITE_BINARY = SqliteBinary() -# DJANGO_BINARY = DjangoBinary() -# WGET_BINARY = WgetBinary() -# YTDLP_BINARY = YtdlpPBinary() - -# print('-------------------------------------DEFINING BINARIES---------------------------------') -# print(PYTHON_BINARY) -# print(SQLITE_BINARY) -# print(DJANGO_BINARY) -# print(WGET_BINARY) -# print(YTDLP_BINARY) diff --git a/archivebox/plugantic/configs.py b/archivebox/plugantic/configs.py deleted file mode 100644 index 671f105c..00000000 --- a/archivebox/plugantic/configs.py +++ /dev/null @@ -1,53 +0,0 @@ -__package__ = 'archivebox.plugantic' - - -from typing import Optional, List, Literal -from pathlib import Path -from pydantic import BaseModel, Field - - -ConfigSectionName = Literal['GENERAL_CONFIG', 'ARCHIVE_METHOD_TOGGLES', 'ARCHIVE_METHOD_OPTIONS', 'DEPENDENCY_CONFIG'] - - -class ConfigSet(BaseModel): - section: ConfigSectionName = 'GENERAL_CONFIG' - -class WgetToggleConfig(ConfigSet): - section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES' - - SAVE_WGET: bool = True - SAVE_WARC: bool = True - -class WgetDependencyConfig(ConfigSet): - section: ConfigSectionName = 'DEPENDENCY_CONFIG' - - WGET_BINARY: str = Field(default='wget') - WGET_ARGS: Optional[List[str]] = Field(default=None) - WGET_EXTRA_ARGS: List[str] = [] - WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}'] - -class WgetOptionsConfig(ConfigSet): - section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS' - - # loaded from shared config - WGET_AUTO_COMPRESSION: bool = Field(default=True) - SAVE_WGET_REQUISITES: bool = Field(default=True) - WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT') - WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT') - WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY') - WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES') - WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE') - - -CONFIG = { - 'CHECK_SSL_VALIDITY': False, - 'SAVE_WARC': False, - 'TIMEOUT': 999, -} - - -WGET_CONFIG = [ - WgetToggleConfig(**CONFIG), - WgetDependencyConfig(**CONFIG), - WgetOptionsConfig(**CONFIG), -] diff --git a/archivebox/plugantic/migrations/0001_initial.py b/archivebox/plugantic/migrations/0001_initial.py deleted file mode 100644 index 7e209f59..00000000 --- a/archivebox/plugantic/migrations/0001_initial.py +++ /dev/null @@ -1,38 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 00:16 - -import abid_utils.models -import archivebox.plugantic.plugins -import charidfield.fields -import django.core.serializers.json -import django.db.models.deletion -import django_pydantic_field.fields -import uuid -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name='Plugin', - fields=[ - ('created', models.DateTimeField(auto_now_add=True)), - ('modified', models.DateTimeField(auto_now=True)), - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('uuid', models.UUIDField(blank=True, null=True, unique=True)), - ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='plg_', unique=True)), - ('schema', django_pydantic_field.fields.PydanticSchemaField(config=None, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.plugins.Plugin)), - ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'abstract': False, - }, - ), - ] diff --git a/archivebox/plugantic/migrations/0002_alter_plugin_schema.py b/archivebox/plugantic/migrations/0002_alter_plugin_schema.py deleted file mode 100644 index 152e2eb3..00000000 --- a/archivebox/plugantic/migrations/0002_alter_plugin_schema.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:16 - -import archivebox.plugantic.plugins -import django.core.serializers.json -import django_pydantic_field.fields -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='plugin', - name='schema', - field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.plugins.Plugin), - ), - ] diff --git a/archivebox/plugantic/migrations/0003_alter_plugin_schema.py b/archivebox/plugantic/migrations/0003_alter_plugin_schema.py deleted file mode 100644 index 754ec3b0..00000000 --- a/archivebox/plugantic/migrations/0003_alter_plugin_schema.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:25 - -import archivebox.plugantic.replayers -import django.core.serializers.json -import django_pydantic_field.fields -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0002_alter_plugin_schema'), - ] - - operations = [ - migrations.AlterField( - model_name='plugin', - name='schema', - field=django_pydantic_field.fields.PydanticSchemaField(config=None, default={'embed_template': 'plugins/generic_replayer/templates/embed.html', 'fullpage_template': 'plugins/generic_replayer/templates/fullpage.html', 'name': 'GenericReplayer', 'row_template': 'plugins/generic_replayer/templates/row.html', 'url_pattern': '*'}, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.replayers.Replayer), - ), - ] diff --git a/archivebox/plugantic/migrations/0004_remove_plugin_schema_plugin_configs_plugin_name.py b/archivebox/plugantic/migrations/0004_remove_plugin_schema_plugin_configs_plugin_name.py deleted file mode 100644 index fce99723..00000000 --- a/archivebox/plugantic/migrations/0004_remove_plugin_schema_plugin_configs_plugin_name.py +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:28 - -import archivebox.plugantic.configs -import django.core.serializers.json -import django_pydantic_field.compat.django -import django_pydantic_field.fields -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0003_alter_plugin_schema'), - ] - - operations = [ - migrations.RemoveField( - model_name='plugin', - name='schema', - ), - migrations.AddField( - model_name='plugin', - name='configs', - field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=[], encoder=django.core.serializers.json.DjangoJSONEncoder, schema=django_pydantic_field.compat.django.GenericContainer(list, (archivebox.plugantic.configs.ConfigSet,))), - ), - migrations.AddField( - model_name='plugin', - name='name', - field=models.CharField(default='name', max_length=64, unique=True), - preserve_default=False, - ), - ] diff --git a/archivebox/plugantic/migrations/0005_customplugin_delete_plugin.py b/archivebox/plugantic/migrations/0005_customplugin_delete_plugin.py deleted file mode 100644 index 31ac4a94..00000000 --- a/archivebox/plugantic/migrations/0005_customplugin_delete_plugin.py +++ /dev/null @@ -1,39 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:42 - -import abid_utils.models -import charidfield.fields -import django.db.models.deletion -import pathlib -import uuid -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0004_remove_plugin_schema_plugin_configs_plugin_name'), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name='CustomPlugin', - fields=[ - ('created', models.DateTimeField(auto_now_add=True)), - ('modified', models.DateTimeField(auto_now=True)), - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('uuid', models.UUIDField(blank=True, null=True, unique=True)), - ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='plg_', unique=True)), - ('name', models.CharField(max_length=64, unique=True)), - ('path', models.FilePathField(path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/archivebox/plugins'))), - ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'abstract': False, - }, - ), - migrations.DeleteModel( - name='Plugin', - ), - ] diff --git a/archivebox/plugantic/migrations/0006_alter_customplugin_path.py b/archivebox/plugantic/migrations/0006_alter_customplugin_path.py deleted file mode 100644 index facf6604..00000000 --- a/archivebox/plugantic/migrations/0006_alter_customplugin_path.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:45 - -import pathlib -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0005_customplugin_delete_plugin'), - ] - - operations = [ - migrations.AlterField( - model_name='customplugin', - name='path', - field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/archivebox/plugins'), recursive=True), - ), - ] diff --git a/archivebox/plugantic/migrations/0007_alter_customplugin_path.py b/archivebox/plugantic/migrations/0007_alter_customplugin_path.py deleted file mode 100644 index 0c78fad8..00000000 --- a/archivebox/plugantic/migrations/0007_alter_customplugin_path.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:46 - -import pathlib -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0006_alter_customplugin_path'), - ] - - operations = [ - migrations.AlterField( - model_name='customplugin', - name='path', - field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins'), recursive=True), - ), - ] diff --git a/archivebox/plugantic/migrations/0008_alter_customplugin_path.py b/archivebox/plugantic/migrations/0008_alter_customplugin_path.py deleted file mode 100644 index 087fe0fc..00000000 --- a/archivebox/plugantic/migrations/0008_alter_customplugin_path.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:47 - -import pathlib -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0007_alter_customplugin_path'), - ] - - operations = [ - migrations.AlterField( - model_name='customplugin', - name='path', - field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data'), recursive=True), - ), - ] diff --git a/archivebox/plugantic/migrations/0009_alter_customplugin_path.py b/archivebox/plugantic/migrations/0009_alter_customplugin_path.py deleted file mode 100644 index 57ab3e79..00000000 --- a/archivebox/plugantic/migrations/0009_alter_customplugin_path.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:48 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0008_alter_customplugin_path'), - ] - - operations = [ - migrations.AlterField( - model_name='customplugin', - name='path', - field=models.FilePathField(allow_files=False, allow_folders=True, path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True), - ), - ] diff --git a/archivebox/plugantic/migrations/0010_alter_customplugin_path.py b/archivebox/plugantic/migrations/0010_alter_customplugin_path.py deleted file mode 100644 index 4a8fbd88..00000000 --- a/archivebox/plugantic/migrations/0010_alter_customplugin_path.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:48 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0009_alter_customplugin_path'), - ] - - operations = [ - migrations.AlterField( - model_name='customplugin', - name='path', - field=models.FilePathField(allow_files=False, allow_folders=True, match='/plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True), - ), - ] diff --git a/archivebox/plugantic/migrations/0011_alter_customplugin_path.py b/archivebox/plugantic/migrations/0011_alter_customplugin_path.py deleted file mode 100644 index e89b7137..00000000 --- a/archivebox/plugantic/migrations/0011_alter_customplugin_path.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:48 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0010_alter_customplugin_path'), - ] - - operations = [ - migrations.AlterField( - model_name='customplugin', - name='path', - field=models.FilePathField(allow_files=False, allow_folders=True, match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True), - ), - ] diff --git a/archivebox/plugantic/migrations/0012_alter_customplugin_path.py b/archivebox/plugantic/migrations/0012_alter_customplugin_path.py deleted file mode 100644 index 0e3fe5a5..00000000 --- a/archivebox/plugantic/migrations/0012_alter_customplugin_path.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:49 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0011_alter_customplugin_path'), - ] - - operations = [ - migrations.AlterField( - model_name='customplugin', - name='path', - field=models.FilePathField(allow_files=False, allow_folders=True, default='example_plugin', match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True), - ), - ] diff --git a/archivebox/plugantic/migrations/0013_alter_customplugin_path.py b/archivebox/plugantic/migrations/0013_alter_customplugin_path.py deleted file mode 100644 index 4c4069ed..00000000 --- a/archivebox/plugantic/migrations/0013_alter_customplugin_path.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:49 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0012_alter_customplugin_path'), - ] - - operations = [ - migrations.AlterField( - model_name='customplugin', - name='path', - field=models.FilePathField(allow_files=False, allow_folders=True, default='/plugins/example_plugin', match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True), - ), - ] diff --git a/archivebox/plugantic/migrations/0014_alter_customplugin_path.py b/archivebox/plugantic/migrations/0014_alter_customplugin_path.py deleted file mode 100644 index f3424dc6..00000000 --- a/archivebox/plugantic/migrations/0014_alter_customplugin_path.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:50 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0013_alter_customplugin_path'), - ] - - operations = [ - migrations.AlterField( - model_name='customplugin', - name='path', - field=models.FilePathField(allow_files=False, allow_folders=True, default='/plugins/example_plugin', match='*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins', recursive=True), - ), - ] diff --git a/archivebox/plugantic/migrations/0015_alter_customplugin_path.py b/archivebox/plugantic/migrations/0015_alter_customplugin_path.py deleted file mode 100644 index a6c9a270..00000000 --- a/archivebox/plugantic/migrations/0015_alter_customplugin_path.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:51 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0014_alter_customplugin_path'), - ] - - operations = [ - migrations.AlterField( - model_name='customplugin', - name='path', - field=models.FilePathField(allow_files=False, allow_folders=True, match='*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins', recursive=True), - ), - ] diff --git a/archivebox/plugantic/migrations/0016_delete_customplugin.py b/archivebox/plugantic/migrations/0016_delete_customplugin.py deleted file mode 100644 index 2d06d6c5..00000000 --- a/archivebox/plugantic/migrations/0016_delete_customplugin.py +++ /dev/null @@ -1,16 +0,0 @@ -# Generated by Django 5.0.6 on 2024-05-18 01:57 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('plugantic', '0015_alter_customplugin_path'), - ] - - operations = [ - migrations.DeleteModel( - name='CustomPlugin', - ), - ] diff --git a/archivebox/plugantic/plugins.py b/archivebox/plugantic/plugins.py deleted file mode 100644 index d213fced..00000000 --- a/archivebox/plugantic/plugins.py +++ /dev/null @@ -1,122 +0,0 @@ -__package__ = 'archivebox.plugantic' - -from typing import List -from typing_extensions import Self - -from pydantic import ( - BaseModel, - ConfigDict, - Field, - model_validator, - validate_call, - SerializeAsAny, -) - -from .binaries import ( - Binary, - WgetBinary, - YtdlpBinary, -) -from .extractors import ( - Extractor, - YtdlpExtractor, - WgetExtractor, - WarcExtractor, -) -from .replayers import ( - Replayer, - MEDIA_REPLAYER, -) -from .configs import ( - ConfigSet, - WGET_CONFIG, -) - - -class Plugin(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True) - - name: str = Field(default='baseplugin') # e.g. media - description: str = Field(default='') # e.g. get media using yt-dlp - - configs: List[SerializeAsAny[ConfigSet]] = Field(default=[]) - binaries: List[SerializeAsAny[Binary]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')] - extractors: List[SerializeAsAny[Extractor]] = Field(default=[]) - replayers: List[SerializeAsAny[Replayer]] = Field(default=[]) - - @model_validator(mode='after') - def validate(self): - self.description = self.description or self.name - - @validate_call - def install(self) -> Self: - new_binaries = [] - for idx, binary in enumerate(self.binaries): - new_binaries.append(binary.install() or binary) - return self.model_copy(update={ - 'binaries': new_binaries, - }) - - @validate_call - def load(self, cache=True) -> Self: - new_binaries = [] - for idx, binary in enumerate(self.binaries): - new_binaries.append(binary.load(cache=cache) or binary) - return self.model_copy(update={ - 'binaries': new_binaries, - }) - - @validate_call - def load_or_install(self, cache=True) -> Self: - new_binaries = [] - for idx, binary in enumerate(self.binaries): - new_binaries.append(binary.load_or_install(cache=cache) or binary) - return self.model_copy(update={ - 'binaries': new_binaries, - }) - - - -class YtdlpPlugin(Plugin): - name: str = 'ytdlp' - configs: List[SerializeAsAny[ConfigSet]] = [] - binaries: List[SerializeAsAny[Binary]] = [YtdlpBinary()] - extractors: List[SerializeAsAny[Extractor]] = [YtdlpExtractor()] - replayers: List[SerializeAsAny[Replayer]] = [MEDIA_REPLAYER] - -class WgetPlugin(Plugin): - name: str = 'wget' - configs: List[SerializeAsAny[ConfigSet]] = [*WGET_CONFIG] - binaries: List[SerializeAsAny[Binary]] = [WgetBinary()] - extractors: List[SerializeAsAny[Extractor]] = [WgetExtractor(), WarcExtractor()] - - -YTDLP_PLUGIN = YtdlpPlugin() -WGET_PLUGIN = WgetPlugin() -PLUGINS = [ - YTDLP_PLUGIN, - WGET_PLUGIN, -] -LOADED_PLUGINS = PLUGINS - - -import json - -for plugin in PLUGINS: - try: - json.dumps(plugin.model_json_schema(), indent=4) - # print(json.dumps(plugin.model_json_schema(), indent=4)) - except Exception as err: - print(f'Failed to generate JSON schema for {plugin.name}') - raise - -# print('-------------------------------------BEFORE INSTALL---------------------------------') -# for plugin in PLUGINS: -# print(plugin.model_dump_json(indent=4)) -# print('-------------------------------------DURING LOAD/INSTALL---------------------------------') -# for plugin in PLUGINS: - # LOADED_PLUGINS.append(plugin.install()) -# print('-------------------------------------AFTER INSTALL---------------------------------') -# for plugin in LOADED_PLUGINS: - # print(plugin.model_dump_json(indent=4)) - diff --git a/archivebox/plugantic/views.py b/archivebox/plugantic/views.py index 24f256de..168c8564 100644 --- a/archivebox/plugantic/views.py +++ b/archivebox/plugantic/views.py @@ -4,17 +4,19 @@ import inspect from typing import Any from django.http import HttpRequest +from django.conf import settings from django.utils.html import format_html, mark_safe from admin_data_views.typing import TableContext, ItemContext from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink -from plugantic.plugins import LOADED_PLUGINS from django.conf import settings def obj_to_yaml(obj: Any, indent: int=0) -> str: indent_str = " " * indent + if indent == 0: + indent_str = '\n' # put extra newline between top-level entries if isinstance(obj, dict): if not obj: @@ -74,22 +76,34 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext: if '_BINARY' in key or '_VERSION' in key } - for plugin in LOADED_PLUGINS: + for plugin in settings.PLUGINS.values(): for binary in plugin.binaries: - binary = binary.load_or_install() + try: + binary = binary.load() + except Exception as e: + print(e) rows['Binary'].append(ItemLink(binary.name, key=binary.name)) - rows['Found Version'].append(binary.loaded_version) + rows['Found Version'].append(f'✅ {binary.loaded_version}' if binary.loaded_version else '❌ missing') rows['From Plugin'].append(plugin.name) - rows['Provided By'].append(binary.loaded_provider) - rows['Found Abspath'].append(binary.loaded_abspath) + rows['Provided By'].append( + ', '.join( + f'[{binprovider.name}]' if binprovider.name == getattr(binary.loaded_binprovider, 'name', None) else binprovider.name + for binprovider in binary.binproviders_supported + if binprovider + ) + # binary.loaded_binprovider.name + # if binary.loaded_binprovider else + # ', '.join(getattr(provider, 'name', str(provider)) for provider in binary.binproviders_supported) + ) + rows['Found Abspath'].append(binary.loaded_abspath or '❌ missing') rows['Related Configuration'].append(mark_safe(', '.join( f'{config_key}' for config_key, config_value in relevant_configs.items() if binary.name.lower().replace('-', '').replace('_', '').replace('ytdlp', 'youtubedl') in config_key.lower() # or binary.name.lower().replace('-', '').replace('_', '') in str(config_value).lower() ))) - rows['Overrides'].append(obj_to_yaml(binary.provider_overrides)) + rows['Overrides'].append(str(obj_to_yaml(binary.provider_overrides))[:200]) # rows['Description'].append(binary.description) return TableContext( @@ -104,7 +118,7 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: binary = None plugin = None - for loaded_plugin in LOADED_PLUGINS: + for loaded_plugin in settings.PLUGINS.values(): for loaded_binary in loaded_plugin.binaries: if loaded_binary.name == key: binary = loaded_binary @@ -112,7 +126,10 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: assert plugin and binary, f'Could not find a binary matching the specified name: {key}' - binary = binary.load_or_install() + try: + binary = binary.load() + except Exception as e: + print(e) return ItemContext( slug=key, @@ -120,14 +137,14 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: data=[ { "name": binary.name, - "description": binary.description, + "description": binary.abspath, "fields": { 'plugin': plugin.name, - 'binprovider': binary.loaded_provider, + 'binprovider': binary.loaded_binprovider, 'abspath': binary.loaded_abspath, 'version': binary.loaded_version, 'overrides': obj_to_yaml(binary.provider_overrides), - 'providers': obj_to_yaml(binary.providers_supported), + 'providers': obj_to_yaml(binary.binproviders_supported), }, "help_texts": { # TODO @@ -148,12 +165,15 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext: "extractors": [], "replayers": [], "configs": [], - "description": [], + "verbose_name": [], } - for plugin in LOADED_PLUGINS: - plugin = plugin.load_or_install() + for plugin in settings.PLUGINS.values(): + try: + plugin = plugin.load_binaries() + except Exception as e: + print(e) rows['Name'].append(ItemLink(plugin.name, key=plugin.name)) rows['binaries'].append(mark_safe(', '.join( @@ -168,7 +188,7 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext: for config_key in configset.__fields__.keys() if config_key != 'section' and config_key in settings.CONFIG ))) - rows['description'].append(str(plugin.description)) + rows['verbose_name'].append(str(plugin.verbose_name)) return TableContext( title="Installed plugins", @@ -181,13 +201,16 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: assert request.user.is_superuser, 'Must be a superuser to view configuration settings.' plugin = None - for loaded_plugin in LOADED_PLUGINS: + for loaded_plugin in settings.PLUGINS.values(): if loaded_plugin.name == key: plugin = loaded_plugin assert plugin, f'Could not find a plugin matching the specified name: {key}' - plugin = plugin.load_or_install() + try: + plugin = plugin.load_binaries() + except Exception as e: + print(e) return ItemContext( slug=key, @@ -195,12 +218,13 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext: data=[ { "name": plugin.name, - "description": plugin.description, + "description": plugin.verbose_name, "fields": { 'configs': plugin.configs, 'binaries': plugin.binaries, 'extractors': plugin.extractors, 'replayers': plugin.replayers, + 'schema': obj_to_yaml(plugin.model_dump(include=('name', 'verbose_name', 'app_label', settings.PLUGIN_KEYS.keys()))), }, "help_texts": { # TODO diff --git a/archivebox/vendor/pydantic-pkgr b/archivebox/vendor/pydantic-pkgr index c97de57f..36aaa4f9 160000 --- a/archivebox/vendor/pydantic-pkgr +++ b/archivebox/vendor/pydantic-pkgr @@ -1 +1 @@ -Subproject commit c97de57f8df5f36a0f8cd1e51645f114e74bffb0 +Subproject commit 36aaa4f9098e5987e23394398aa56154582bd2d2