mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-05-27 13:14:24 -04:00
BasePlugin system expanded and registration system improved
This commit is contained in:
Nick Sweeting
parent
f1579bfdcd
commit
9af260df16
50 changed files with 1062 additions and 973 deletions
|
|
@ -1,16 +1,9 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
from .binaries import Binary
|
||||
from .extractors import Extractor
|
||||
from .replayers import Replayer
|
||||
from .configs import ConfigSet
|
||||
from .plugins import Plugin
|
||||
from .base_plugin import BasePlugin
|
||||
from .base_configset import BaseConfigSet
|
||||
from .base_binary import BaseBinary
|
||||
from .base_extractor import BaseExtractor
|
||||
from .base_replayer import BaseReplayer
|
||||
from .base_check import BaseCheck
|
||||
|
||||
# __all__ = [
|
||||
# 'BinProvider',
|
||||
# 'Binary',
|
||||
# 'Extractor',
|
||||
# 'Replayer',
|
||||
# 'ConfigSet',
|
||||
# 'Plugin',
|
||||
# ]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import importlib
|
||||
from django.apps import AppConfig
|
||||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
import json
|
||||
import importlib
|
||||
|
||||
from django.apps import AppConfig
|
||||
|
||||
class PluganticConfig(AppConfig):
|
||||
default_auto_field = 'django.db.models.BigAutoField'
|
||||
|
|
@ -8,10 +11,5 @@ class PluganticConfig(AppConfig):
|
|||
|
||||
def ready(self) -> None:
|
||||
from django.conf import settings
|
||||
from .plugins import PLUGINS
|
||||
|
||||
for plugin_name in settings.INSTALLED_PLUGINS.keys():
|
||||
lib = importlib.import_module(f'{plugin_name}.apps')
|
||||
if hasattr(lib, 'PLUGINS'):
|
||||
for plugin_instance in lib.PLUGINS:
|
||||
PLUGINS.append(plugin_instance)
|
||||
print(f'[🧩] Detected {len(settings.INSTALLED_PLUGINS)} settings.INSTALLED_PLUGINS to load...')
|
||||
|
|
|
|||
34
archivebox/plugantic/base_admindataview.py
Normal file
34
archivebox/plugantic/base_admindataview.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from typing import List, Type, Any, Dict
|
||||
|
||||
from pydantic_core import core_schema
|
||||
from pydantic import GetCoreSchemaHandler, BaseModel
|
||||
|
||||
from django.utils.functional import classproperty
|
||||
from django.core.checks import Warning, Tags, register
|
||||
|
||||
class BaseAdminDataView(BaseModel):
|
||||
name: str = 'NPM Installed Packages'
|
||||
route: str = '/npm/installed/'
|
||||
view: str = 'builtin_plugins.npm.admin.installed_list_view'
|
||||
items: Dict[str, str] = {
|
||||
"name": "installed_npm_pkg",
|
||||
'route': '<str:key>/',
|
||||
'view': 'builtin_plugins.npm.admin.installed_detail_view',
|
||||
}
|
||||
|
||||
def as_route(self) -> Dict[str, str | Dict[str, str]]:
|
||||
return {
|
||||
'route': self.route,
|
||||
'view': self.view,
|
||||
'name': self.name,
|
||||
'items': self.items,
|
||||
}
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
"""Regsiter AdminDataViews.as_route() in settings.ADMIN_DATA_VIEWS.URLS at runtime"""
|
||||
self._plugin = parent_plugin # circular ref to parent only here for easier debugging! never depend on circular backref to parent in real code!
|
||||
|
||||
route = self.as_route()
|
||||
if route not in settings.ADMIN_DATA_VIEWS.URLS:
|
||||
settings.ADMIN_DATA_VIEWS.URLS += [route] # append our route (update in place)
|
||||
|
||||
99
archivebox/plugantic/base_binary.py
Normal file
99
archivebox/plugantic/base_binary.py
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
import sys
|
||||
import inspect
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
from typing import Any, Optional, Dict, List
|
||||
from typing_extensions import Self
|
||||
from subprocess import run, PIPE
|
||||
|
||||
from pydantic import Field, InstanceOf
|
||||
from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict
|
||||
from pydantic_pkgr.binprovider import HostBinPath
|
||||
|
||||
import django
|
||||
from django.core.cache import cache
|
||||
from django.db.backends.sqlite3.base import Database as sqlite3
|
||||
|
||||
|
||||
class BaseBinProvider(BinProvider):
|
||||
# def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
|
||||
# Class = super()
|
||||
# get_abspath_func = lambda: Class.on_get_abspath(bin_name, **context)
|
||||
# # return cache.get_or_set(f'bin:abspath:{bin_name}', get_abspath_func)
|
||||
# return get_abspath_func()
|
||||
|
||||
# def on_get_version(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> SemVer | None:
|
||||
# Class = super()
|
||||
# get_version_func = lambda: Class.on_get_version(bin_name, abspath, **context)
|
||||
# # return cache.get_or_set(f'bin:version:{bin_name}:{abspath}', get_version_func)
|
||||
# return get_version_func()
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
settings.BINPROVIDERS[self.name] = self
|
||||
|
||||
|
||||
class BaseBinary(Binary):
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = Field(default_factory=list, alias='binproviders')
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
settings.BINARIES[self.name] = self
|
||||
|
||||
# def get_ytdlp_version() -> str:
|
||||
# import yt_dlp
|
||||
# return yt_dlp.version.__version__
|
||||
|
||||
|
||||
|
||||
|
||||
# class YtdlpBinary(Binary):
|
||||
# name: BinName = 'yt-dlp'
|
||||
# providers_supported: List[BinProvider] = [
|
||||
# EnvProvider(),
|
||||
# PipProvider(),
|
||||
# BrewProvider(),
|
||||
# AptProvider(),
|
||||
# ]
|
||||
# provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
# 'pip': {
|
||||
# 'version': get_ytdlp_version,
|
||||
# },
|
||||
# 'brew': {
|
||||
# 'subdeps': lambda: 'yt-dlp ffmpeg',
|
||||
# },
|
||||
# 'apt': {
|
||||
# 'subdeps': lambda: 'yt-dlp ffmpeg',
|
||||
# }
|
||||
# }
|
||||
|
||||
# class WgetBinary(Binary):
|
||||
# name: BinName = 'wget'
|
||||
# providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()]
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# PYTHON_BINARY = PythonBinary()
|
||||
# SQLITE_BINARY = SqliteBinary()
|
||||
# DJANGO_BINARY = DjangoBinary()
|
||||
# WGET_BINARY = WgetBinary()
|
||||
# YTDLP_BINARY = YtdlpPBinary()
|
||||
|
||||
# print('-------------------------------------DEFINING BINARIES---------------------------------')
|
||||
# print(PYTHON_BINARY)
|
||||
# print(SQLITE_BINARY)
|
||||
# print(DJANGO_BINARY)
|
||||
# print(WGET_BINARY)
|
||||
# print(YTDLP_BINARY)
|
||||
55
archivebox/plugantic/base_check.py
Normal file
55
archivebox/plugantic/base_check.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
from typing import List, Type, Any
|
||||
|
||||
from pydantic_core import core_schema
|
||||
from pydantic import GetCoreSchemaHandler
|
||||
|
||||
from django.utils.functional import classproperty
|
||||
from django.core.checks import Warning, Tags, register
|
||||
|
||||
class BaseCheck:
|
||||
label: str = ''
|
||||
tag = Tags.database
|
||||
|
||||
@classmethod
|
||||
def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
|
||||
return core_schema.typed_dict_schema(
|
||||
{
|
||||
'name': core_schema.typed_dict_field(core_schema.str_schema()),
|
||||
'tag': core_schema.typed_dict_field(core_schema.str_schema()),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@classproperty
|
||||
def name(cls) -> str:
|
||||
return cls.label or cls.__name__
|
||||
|
||||
@staticmethod
|
||||
def check(settings, logger) -> List[Warning]:
|
||||
"""Override this method to implement your custom runtime check."""
|
||||
errors = []
|
||||
# if not hasattr(settings, 'SOME_KEY'):
|
||||
# errors.extend(Error(
|
||||
# 'Missing settings.SOME_KEY after django_setup(), did SOME_KEY get loaded?',
|
||||
# id='core.C001',
|
||||
# hint='Make sure to run django_setup() is able to load settings.SOME_KEY.',
|
||||
# ))
|
||||
# logger.debug('[√] Loaded settings.PLUGINS succesfully.')
|
||||
return errors
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
# Regsiter in ArchiveBox plugins runtime settings
|
||||
self._plugin = parent_plugin
|
||||
settings.CHECKS[self.name] = self
|
||||
|
||||
# Register using Django check framework
|
||||
def run_check(app_configs, **kwargs) -> List[Warning]:
|
||||
from django.conf import settings
|
||||
import logging
|
||||
settings = settings
|
||||
logger = logging.getLogger('checks')
|
||||
return self.check(settings, logger)
|
||||
|
||||
run_check.__name__ = self.label or self.__class__.__name__
|
||||
run_check.tags = [self.tag]
|
||||
register(self.tag)(run_check)
|
||||
81
archivebox/plugantic/base_configset.py
Normal file
81
archivebox/plugantic/base_configset.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
|
||||
from typing import Optional, List, Literal
|
||||
from pathlib import Path
|
||||
from pydantic import BaseModel, Field, ConfigDict, computed_field
|
||||
|
||||
|
||||
ConfigSectionName = Literal[
|
||||
'GENERAL_CONFIG',
|
||||
'ARCHIVE_METHOD_TOGGLES',
|
||||
'ARCHIVE_METHOD_OPTIONS',
|
||||
'DEPENDENCY_CONFIG',
|
||||
]
|
||||
ConfigSectionNames: List[ConfigSectionName] = [
|
||||
'GENERAL_CONFIG',
|
||||
'ARCHIVE_METHOD_TOGGLES',
|
||||
'ARCHIVE_METHOD_OPTIONS',
|
||||
'DEPENDENCY_CONFIG',
|
||||
]
|
||||
|
||||
|
||||
class BaseConfigSet(BaseModel):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow', populate_by_name=True)
|
||||
|
||||
section: ConfigSectionName = 'GENERAL_CONFIG'
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
settings.CONFIGS[self.name] = self
|
||||
|
||||
|
||||
|
||||
# class WgetToggleConfig(ConfigSet):
|
||||
# section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
|
||||
|
||||
# SAVE_WGET: bool = True
|
||||
# SAVE_WARC: bool = True
|
||||
|
||||
# class WgetDependencyConfig(ConfigSet):
|
||||
# section: ConfigSectionName = 'DEPENDENCY_CONFIG'
|
||||
|
||||
# WGET_BINARY: str = Field(default='wget')
|
||||
# WGET_ARGS: Optional[List[str]] = Field(default=None)
|
||||
# WGET_EXTRA_ARGS: List[str] = []
|
||||
# WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
|
||||
|
||||
# class WgetOptionsConfig(ConfigSet):
|
||||
# section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
|
||||
|
||||
# # loaded from shared config
|
||||
# WGET_AUTO_COMPRESSION: bool = Field(default=True)
|
||||
# SAVE_WGET_REQUISITES: bool = Field(default=True)
|
||||
# WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT')
|
||||
# WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
|
||||
# WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
|
||||
# WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
|
||||
# WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
|
||||
|
||||
|
||||
# CONFIG = {
|
||||
# 'CHECK_SSL_VALIDITY': False,
|
||||
# 'SAVE_WARC': False,
|
||||
# 'TIMEOUT': 999,
|
||||
# }
|
||||
|
||||
|
||||
# WGET_CONFIG = [
|
||||
# WgetToggleConfig(**CONFIG),
|
||||
# WgetDependencyConfig(**CONFIG),
|
||||
# WgetOptionsConfig(**CONFIG),
|
||||
# ]
|
||||
|
|
@ -6,13 +6,14 @@ from typing_extensions import Self
|
|||
from abc import ABC
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, model_validator, field_serializer, AfterValidator
|
||||
from pydantic import BaseModel, model_validator, field_serializer, AfterValidator, Field
|
||||
from pydantic_pkgr import BinName
|
||||
|
||||
from .binaries import (
|
||||
Binary,
|
||||
YtdlpBinary,
|
||||
WgetBinary,
|
||||
)
|
||||
# from .binaries import (
|
||||
# Binary,
|
||||
# YtdlpBinary,
|
||||
# WgetBinary,
|
||||
# )
|
||||
|
||||
|
||||
# stubs
|
||||
|
|
@ -37,9 +38,9 @@ HandlerFuncStr = Annotated[str, AfterValidator(lambda s: s.startswith('self.'))]
|
|||
CmdArgsList = Annotated[List[str], AfterValidator(no_empty_args)]
|
||||
|
||||
|
||||
class Extractor(ABC, BaseModel):
|
||||
class BaseExtractor(ABC, BaseModel):
|
||||
name: ExtractorName
|
||||
binary: Binary
|
||||
binary: BinName
|
||||
|
||||
output_path_func: HandlerFuncStr = 'self.get_output_path'
|
||||
should_extract_func: HandlerFuncStr = 'self.should_extract'
|
||||
|
|
@ -55,10 +56,14 @@ class Extractor(ABC, BaseModel):
|
|||
if self.args is None:
|
||||
self.args = [*self.default_args, *self.extra_args]
|
||||
return self
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
|
||||
@field_serializer('binary', when_used='json')
|
||||
def dump_binary(binary) -> str:
|
||||
return binary.name
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
settings.EXTRACTORS[self.name] = self
|
||||
|
||||
def get_output_path(self, snapshot) -> Path:
|
||||
return Path(self.name)
|
||||
|
|
@ -86,33 +91,37 @@ class Extractor(ABC, BaseModel):
|
|||
'returncode': proc.returncode,
|
||||
}
|
||||
|
||||
def exec(self, args: CmdArgsList, pwd: Optional[Path]=None):
|
||||
def exec(self, args: CmdArgsList, pwd: Optional[Path]=None, settings=None):
|
||||
pwd = pwd or Path('.')
|
||||
assert self.binary.loaded_provider
|
||||
return self.binary.exec(args, pwd=pwd)
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
|
||||
binary = settings.BINARIES[self.binary]
|
||||
return binary.exec(args, pwd=pwd)
|
||||
|
||||
|
||||
class YtdlpExtractor(Extractor):
|
||||
name: ExtractorName = 'media'
|
||||
binary: Binary = YtdlpBinary()
|
||||
# class YtdlpExtractor(Extractor):
|
||||
# name: ExtractorName = 'media'
|
||||
# binary: Binary = YtdlpBinary()
|
||||
|
||||
def get_output_path(self, snapshot) -> Path:
|
||||
return Path(self.name)
|
||||
# def get_output_path(self, snapshot) -> Path:
|
||||
# return Path(self.name)
|
||||
|
||||
|
||||
class WgetExtractor(Extractor):
|
||||
name: ExtractorName = 'wget'
|
||||
binary: Binary = WgetBinary()
|
||||
# class WgetExtractor(Extractor):
|
||||
# name: ExtractorName = 'wget'
|
||||
# binary: Binary = WgetBinary()
|
||||
|
||||
def get_output_path(self, snapshot) -> Path:
|
||||
return get_wget_output_path(snapshot)
|
||||
# def get_output_path(self, snapshot) -> Path:
|
||||
# return get_wget_output_path(snapshot)
|
||||
|
||||
|
||||
class WarcExtractor(Extractor):
|
||||
name: ExtractorName = 'warc'
|
||||
binary: Binary = WgetBinary()
|
||||
# class WarcExtractor(Extractor):
|
||||
# name: ExtractorName = 'warc'
|
||||
# binary: Binary = WgetBinary()
|
||||
|
||||
def get_output_path(self, snapshot) -> Path:
|
||||
return get_wget_output_path(snapshot)
|
||||
# def get_output_path(self, snapshot) -> Path:
|
||||
# return get_wget_output_path(snapshot)
|
||||
|
||||
|
||||
202
archivebox/plugantic/base_plugin.py
Normal file
202
archivebox/plugantic/base_plugin.py
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
import json
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.core.checks import register
|
||||
|
||||
from typing import List, ClassVar, Type, Dict
|
||||
from typing_extensions import Self
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Field,
|
||||
model_validator,
|
||||
InstanceOf,
|
||||
computed_field,
|
||||
validate_call,
|
||||
)
|
||||
|
||||
from .base_configset import BaseConfigSet
|
||||
from .base_binary import BaseBinProvider, BaseBinary
|
||||
from .base_extractor import BaseExtractor
|
||||
from .base_replayer import BaseReplayer
|
||||
from .base_check import BaseCheck
|
||||
from .base_admindataview import BaseAdminDataView
|
||||
|
||||
from ..config import ANSI, AttrDict
|
||||
|
||||
|
||||
class BasePlugin(BaseModel):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
|
||||
|
||||
# Required by AppConfig:
|
||||
name: str = Field() # e.g. 'builtin_plugins.singlefile'
|
||||
app_label: str = Field() # e.g. 'singlefile'
|
||||
verbose_name: str = Field() # e.g. 'SingleFile'
|
||||
default_auto_field: ClassVar[str] = 'django.db.models.AutoField'
|
||||
|
||||
# Required by Plugantic:
|
||||
configs: List[InstanceOf[BaseConfigSet]] = Field(default=[])
|
||||
binproviders: List[InstanceOf[BaseBinProvider]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
|
||||
binaries: List[InstanceOf[BaseBinary]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
|
||||
extractors: List[InstanceOf[BaseExtractor]] = Field(default=[])
|
||||
replayers: List[InstanceOf[BaseReplayer]] = Field(default=[])
|
||||
checks: List[InstanceOf[BaseCheck]] = Field(default=[])
|
||||
admindataviews: List[InstanceOf[BaseAdminDataView]] = Field(default=[])
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate(self) -> Self:
|
||||
"""Validate the plugin's build-time configuration here before it's registered in Django at runtime."""
|
||||
|
||||
assert self.name and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
|
||||
|
||||
assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
|
||||
|
||||
@property
|
||||
def AppConfig(plugin_self) -> Type[AppConfig]:
|
||||
"""Generate a Django AppConfig class for this plugin."""
|
||||
|
||||
class PluginAppConfig(AppConfig):
|
||||
name = plugin_self.name
|
||||
app_label = plugin_self.app_label
|
||||
verbose_name = plugin_self.verbose_name
|
||||
|
||||
def ready(self):
|
||||
from django.conf import settings
|
||||
|
||||
plugin_self.validate()
|
||||
plugin_self.register(settings)
|
||||
|
||||
return PluginAppConfig
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def BINPROVIDERS(self) -> Dict[str, BaseBinProvider]:
|
||||
return AttrDict({binprovider.name: binprovider for binprovider in self.binproviders})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def BINARIES(self) -> Dict[str, BaseBinary]:
|
||||
return AttrDict({binary.python_name: binary for binary in self.binaries})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def CONFIGS(self) -> Dict[str, BaseConfigSet]:
|
||||
return AttrDict({config.name: config for config in self.configs})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def EXTRACTORS(self) -> Dict[str, BaseExtractor]:
|
||||
return AttrDict({extractor.name: extractor for extractor in self.extractors})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def REPLAYERS(self) -> Dict[str, BaseReplayer]:
|
||||
return AttrDict({replayer.name: replayer for replayer in self.replayers})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def CHECKS(self) -> Dict[str, BaseCheck]:
|
||||
return AttrDict({check.name: check for check in self.checks})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def ADMINDATAVIEWS(self) -> Dict[str, BaseCheck]:
|
||||
return AttrDict({admindataview.name: admindataview for admindataview in self.admindataviews})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def PLUGIN_KEYS(self) -> List[str]:
|
||||
return
|
||||
|
||||
def register(self, settings=None):
|
||||
"""Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime."""
|
||||
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
|
||||
assert all(hasattr(settings, key) for key in ['PLUGINS', 'CONFIGS', 'BINARIES', 'EXTRACTORS', 'REPLAYERS', 'ADMINDATAVIEWS']), 'Tried to register plugin in settings but couldnt find required global dicts in settings.'
|
||||
|
||||
assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
|
||||
|
||||
assert self.app_label not in settings.PLUGINS, f'Tried to register plugin {self.name} but it conflicts with existing plugin of the same name ({self.app_label}).'
|
||||
|
||||
### Mutate django.conf.settings... values in-place to include plugin-provided overrides
|
||||
settings.PLUGINS[self.app_label] = self
|
||||
|
||||
for config in self.CONFIGS.values():
|
||||
config.register(settings, parent_plugin=self)
|
||||
|
||||
for binprovider in self.BINPROVIDERS.values():
|
||||
binprovider.register(settings, parent_plugin=self)
|
||||
|
||||
for binary in self.BINARIES.values():
|
||||
binary.register(settings, parent_plugin=self)
|
||||
|
||||
for extractor in self.EXTRACTORS.values():
|
||||
extractor.register(settings, parent_plugin=self)
|
||||
|
||||
for replayer in self.REPLAYERS.values():
|
||||
replayer.register(settings, parent_plugin=self)
|
||||
|
||||
for check in self.CHECKS.values():
|
||||
check.register(settings, parent_plugin=self)
|
||||
|
||||
for admindataview in self.ADMINDATAVIEWS.values():
|
||||
admindataview.register(settings, parent_plugin=self)
|
||||
|
||||
# TODO: add parsers? custom templates? persona fixtures?
|
||||
|
||||
plugin_prefix, plugin_shortname = self.name.split('.', 1)
|
||||
|
||||
print(
|
||||
f' > {ANSI.black}{plugin_prefix.upper().replace("_PLUGINS", "").ljust(15)} ' +
|
||||
f'{ANSI.lightyellow}{plugin_shortname.ljust(12)} ' +
|
||||
f'{ANSI.black}CONFIGSx{len(self.configs)} BINARIESx{len(self.binaries)} EXTRACTORSx{len(self.extractors)} REPLAYERSx{len(self.replayers)} CHECKSx{len(self.CHECKS)} ADMINDATAVIEWSx{len(self.ADMINDATAVIEWS)}{ANSI.reset}'
|
||||
)
|
||||
|
||||
# @validate_call
|
||||
# def install_binaries(self) -> Self:
|
||||
# new_binaries = []
|
||||
# for idx, binary in enumerate(self.binaries):
|
||||
# new_binaries.append(binary.install() or binary)
|
||||
# return self.model_copy(update={
|
||||
# 'binaries': new_binaries,
|
||||
# })
|
||||
|
||||
@validate_call
|
||||
def load_binaries(self, cache=True) -> Self:
|
||||
new_binaries = []
|
||||
for idx, binary in enumerate(self.binaries):
|
||||
new_binaries.append(binary.load(cache=cache) or binary)
|
||||
return self.model_copy(update={
|
||||
'binaries': new_binaries,
|
||||
})
|
||||
|
||||
# @validate_call
|
||||
# def load_or_install_binaries(self, cache=True) -> Self:
|
||||
# new_binaries = []
|
||||
# for idx, binary in enumerate(self.binaries):
|
||||
# new_binaries.append(binary.load_or_install(cache=cache) or binary)
|
||||
# return self.model_copy(update={
|
||||
# 'binaries': new_binaries,
|
||||
# })
|
||||
|
||||
|
||||
|
||||
|
||||
# class YtdlpPlugin(BasePlugin):
|
||||
# name: str = 'ytdlp'
|
||||
# configs: List[SerializeAsAny[BaseConfigSet]] = []
|
||||
# binaries: List[SerializeAsAny[BaseBinary]] = [YtdlpBinary()]
|
||||
# extractors: List[SerializeAsAny[BaseExtractor]] = [YtdlpExtractor()]
|
||||
# replayers: List[SerializeAsAny[BaseReplayer]] = [MEDIA_REPLAYER]
|
||||
|
||||
# class WgetPlugin(BasePlugin):
|
||||
# name: str = 'wget'
|
||||
# configs: List[SerializeAsAny[BaseConfigSet]] = [*WGET_CONFIG]
|
||||
# binaries: List[SerializeAsAny[BaseBinary]] = [WgetBinary()]
|
||||
# extractors: List[SerializeAsAny[BaseExtractor]] = [WgetExtractor(), WarcExtractor()]
|
||||
|
|
@ -3,10 +3,9 @@ __package__ = 'archivebox.plugantic'
|
|||
|
||||
from pydantic import BaseModel
|
||||
|
||||
# from .binproviders import LazyImportStr
|
||||
|
||||
|
||||
class Replayer(BaseModel):
|
||||
class BaseReplayer(BaseModel):
|
||||
"""Describes how to render an ArchiveResult in several contexts"""
|
||||
name: str = 'GenericReplayer'
|
||||
url_pattern: str = '*'
|
||||
|
|
@ -21,5 +20,17 @@ class Replayer(BaseModel):
|
|||
# icon_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
|
||||
# thumbnail_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
|
||||
MEDIA_REPLAYER = Replayer(name='media')
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
settings.REPLAYERS[self.name] = self
|
||||
|
||||
|
||||
# class MediaReplayer(BaseReplayer):
|
||||
# name: str = 'MediaReplayer'
|
||||
|
||||
|
||||
# MEDIA_REPLAYER = MediaReplayer()
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
import sys
|
||||
import inspect
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
from typing import Any, Optional, Dict, List
|
||||
from typing_extensions import Self
|
||||
from subprocess import run, PIPE
|
||||
|
||||
from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict
|
||||
|
||||
import django
|
||||
from django.db.backends.sqlite3.base import Database as sqlite3
|
||||
|
||||
|
||||
|
||||
|
||||
def get_ytdlp_version() -> str:
|
||||
import yt_dlp
|
||||
return yt_dlp.version.__version__
|
||||
|
||||
|
||||
|
||||
|
||||
class YtdlpBinary(Binary):
|
||||
name: BinName = 'yt-dlp'
|
||||
providers_supported: List[BinProvider] = [
|
||||
EnvProvider(),
|
||||
PipProvider(),
|
||||
BrewProvider(),
|
||||
AptProvider(),
|
||||
]
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
'pip': {
|
||||
'version': get_ytdlp_version,
|
||||
},
|
||||
'brew': {
|
||||
'subdeps': lambda: 'yt-dlp ffmpeg',
|
||||
},
|
||||
'apt': {
|
||||
'subdeps': lambda: 'yt-dlp ffmpeg',
|
||||
}
|
||||
}
|
||||
|
||||
class WgetBinary(Binary):
|
||||
name: BinName = 'wget'
|
||||
providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()]
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# PYTHON_BINARY = PythonBinary()
|
||||
# SQLITE_BINARY = SqliteBinary()
|
||||
# DJANGO_BINARY = DjangoBinary()
|
||||
# WGET_BINARY = WgetBinary()
|
||||
# YTDLP_BINARY = YtdlpPBinary()
|
||||
|
||||
# print('-------------------------------------DEFINING BINARIES---------------------------------')
|
||||
# print(PYTHON_BINARY)
|
||||
# print(SQLITE_BINARY)
|
||||
# print(DJANGO_BINARY)
|
||||
# print(WGET_BINARY)
|
||||
# print(YTDLP_BINARY)
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
|
||||
from typing import Optional, List, Literal
|
||||
from pathlib import Path
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
ConfigSectionName = Literal['GENERAL_CONFIG', 'ARCHIVE_METHOD_TOGGLES', 'ARCHIVE_METHOD_OPTIONS', 'DEPENDENCY_CONFIG']
|
||||
|
||||
|
||||
class ConfigSet(BaseModel):
|
||||
section: ConfigSectionName = 'GENERAL_CONFIG'
|
||||
|
||||
class WgetToggleConfig(ConfigSet):
|
||||
section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
|
||||
|
||||
SAVE_WGET: bool = True
|
||||
SAVE_WARC: bool = True
|
||||
|
||||
class WgetDependencyConfig(ConfigSet):
|
||||
section: ConfigSectionName = 'DEPENDENCY_CONFIG'
|
||||
|
||||
WGET_BINARY: str = Field(default='wget')
|
||||
WGET_ARGS: Optional[List[str]] = Field(default=None)
|
||||
WGET_EXTRA_ARGS: List[str] = []
|
||||
WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
|
||||
|
||||
class WgetOptionsConfig(ConfigSet):
|
||||
section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
|
||||
|
||||
# loaded from shared config
|
||||
WGET_AUTO_COMPRESSION: bool = Field(default=True)
|
||||
SAVE_WGET_REQUISITES: bool = Field(default=True)
|
||||
WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT')
|
||||
WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
|
||||
WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
|
||||
WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
|
||||
WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
|
||||
|
||||
|
||||
CONFIG = {
|
||||
'CHECK_SSL_VALIDITY': False,
|
||||
'SAVE_WARC': False,
|
||||
'TIMEOUT': 999,
|
||||
}
|
||||
|
||||
|
||||
WGET_CONFIG = [
|
||||
WgetToggleConfig(**CONFIG),
|
||||
WgetDependencyConfig(**CONFIG),
|
||||
WgetOptionsConfig(**CONFIG),
|
||||
]
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 00:16
|
||||
|
||||
import abid_utils.models
|
||||
import archivebox.plugantic.plugins
|
||||
import charidfield.fields
|
||||
import django.core.serializers.json
|
||||
import django.db.models.deletion
|
||||
import django_pydantic_field.fields
|
||||
import uuid
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='Plugin',
|
||||
fields=[
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('modified', models.DateTimeField(auto_now=True)),
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('uuid', models.UUIDField(blank=True, null=True, unique=True)),
|
||||
('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='plg_', unique=True)),
|
||||
('schema', django_pydantic_field.fields.PydanticSchemaField(config=None, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.plugins.Plugin)),
|
||||
('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'abstract': False,
|
||||
},
|
||||
),
|
||||
]
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:16
|
||||
|
||||
import archivebox.plugantic.plugins
|
||||
import django.core.serializers.json
|
||||
import django_pydantic_field.fields
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='plugin',
|
||||
name='schema',
|
||||
field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.plugins.Plugin),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:25
|
||||
|
||||
import archivebox.plugantic.replayers
|
||||
import django.core.serializers.json
|
||||
import django_pydantic_field.fields
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0002_alter_plugin_schema'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='plugin',
|
||||
name='schema',
|
||||
field=django_pydantic_field.fields.PydanticSchemaField(config=None, default={'embed_template': 'plugins/generic_replayer/templates/embed.html', 'fullpage_template': 'plugins/generic_replayer/templates/fullpage.html', 'name': 'GenericReplayer', 'row_template': 'plugins/generic_replayer/templates/row.html', 'url_pattern': '*'}, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.replayers.Replayer),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:28
|
||||
|
||||
import archivebox.plugantic.configs
|
||||
import django.core.serializers.json
|
||||
import django_pydantic_field.compat.django
|
||||
import django_pydantic_field.fields
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0003_alter_plugin_schema'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='plugin',
|
||||
name='schema',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='plugin',
|
||||
name='configs',
|
||||
field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=[], encoder=django.core.serializers.json.DjangoJSONEncoder, schema=django_pydantic_field.compat.django.GenericContainer(list, (archivebox.plugantic.configs.ConfigSet,))),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='plugin',
|
||||
name='name',
|
||||
field=models.CharField(default='name', max_length=64, unique=True),
|
||||
preserve_default=False,
|
||||
),
|
||||
]
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:42
|
||||
|
||||
import abid_utils.models
|
||||
import charidfield.fields
|
||||
import django.db.models.deletion
|
||||
import pathlib
|
||||
import uuid
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0004_remove_plugin_schema_plugin_configs_plugin_name'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='CustomPlugin',
|
||||
fields=[
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('modified', models.DateTimeField(auto_now=True)),
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('uuid', models.UUIDField(blank=True, null=True, unique=True)),
|
||||
('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='plg_', unique=True)),
|
||||
('name', models.CharField(max_length=64, unique=True)),
|
||||
('path', models.FilePathField(path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/archivebox/plugins'))),
|
||||
('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'abstract': False,
|
||||
},
|
||||
),
|
||||
migrations.DeleteModel(
|
||||
name='Plugin',
|
||||
),
|
||||
]
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:45
|
||||
|
||||
import pathlib
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0005_customplugin_delete_plugin'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='customplugin',
|
||||
name='path',
|
||||
field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/archivebox/plugins'), recursive=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:46
|
||||
|
||||
import pathlib
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0006_alter_customplugin_path'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='customplugin',
|
||||
name='path',
|
||||
field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins'), recursive=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:47
|
||||
|
||||
import pathlib
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0007_alter_customplugin_path'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='customplugin',
|
||||
name='path',
|
||||
field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data'), recursive=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:48
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0008_alter_customplugin_path'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='customplugin',
|
||||
name='path',
|
||||
field=models.FilePathField(allow_files=False, allow_folders=True, path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:48
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0009_alter_customplugin_path'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='customplugin',
|
||||
name='path',
|
||||
field=models.FilePathField(allow_files=False, allow_folders=True, match='/plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:48
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0010_alter_customplugin_path'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='customplugin',
|
||||
name='path',
|
||||
field=models.FilePathField(allow_files=False, allow_folders=True, match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:49
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0011_alter_customplugin_path'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='customplugin',
|
||||
name='path',
|
||||
field=models.FilePathField(allow_files=False, allow_folders=True, default='example_plugin', match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:49
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0012_alter_customplugin_path'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='customplugin',
|
||||
name='path',
|
||||
field=models.FilePathField(allow_files=False, allow_folders=True, default='/plugins/example_plugin', match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:50
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0013_alter_customplugin_path'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='customplugin',
|
||||
name='path',
|
||||
field=models.FilePathField(allow_files=False, allow_folders=True, default='/plugins/example_plugin', match='*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins', recursive=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:51
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0014_alter_customplugin_path'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='customplugin',
|
||||
name='path',
|
||||
field=models.FilePathField(allow_files=False, allow_folders=True, match='*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins', recursive=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
# Generated by Django 5.0.6 on 2024-05-18 01:57
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('plugantic', '0015_alter_customplugin_path'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.DeleteModel(
|
||||
name='CustomPlugin',
|
||||
),
|
||||
]
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
from typing import List
|
||||
from typing_extensions import Self
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Field,
|
||||
model_validator,
|
||||
validate_call,
|
||||
SerializeAsAny,
|
||||
)
|
||||
|
||||
from .binaries import (
|
||||
Binary,
|
||||
WgetBinary,
|
||||
YtdlpBinary,
|
||||
)
|
||||
from .extractors import (
|
||||
Extractor,
|
||||
YtdlpExtractor,
|
||||
WgetExtractor,
|
||||
WarcExtractor,
|
||||
)
|
||||
from .replayers import (
|
||||
Replayer,
|
||||
MEDIA_REPLAYER,
|
||||
)
|
||||
from .configs import (
|
||||
ConfigSet,
|
||||
WGET_CONFIG,
|
||||
)
|
||||
|
||||
|
||||
class Plugin(BaseModel):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
|
||||
|
||||
name: str = Field(default='baseplugin') # e.g. media
|
||||
description: str = Field(default='') # e.g. get media using yt-dlp
|
||||
|
||||
configs: List[SerializeAsAny[ConfigSet]] = Field(default=[])
|
||||
binaries: List[SerializeAsAny[Binary]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
|
||||
extractors: List[SerializeAsAny[Extractor]] = Field(default=[])
|
||||
replayers: List[SerializeAsAny[Replayer]] = Field(default=[])
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate(self):
|
||||
self.description = self.description or self.name
|
||||
|
||||
@validate_call
|
||||
def install(self) -> Self:
|
||||
new_binaries = []
|
||||
for idx, binary in enumerate(self.binaries):
|
||||
new_binaries.append(binary.install() or binary)
|
||||
return self.model_copy(update={
|
||||
'binaries': new_binaries,
|
||||
})
|
||||
|
||||
@validate_call
|
||||
def load(self, cache=True) -> Self:
|
||||
new_binaries = []
|
||||
for idx, binary in enumerate(self.binaries):
|
||||
new_binaries.append(binary.load(cache=cache) or binary)
|
||||
return self.model_copy(update={
|
||||
'binaries': new_binaries,
|
||||
})
|
||||
|
||||
@validate_call
|
||||
def load_or_install(self, cache=True) -> Self:
|
||||
new_binaries = []
|
||||
for idx, binary in enumerate(self.binaries):
|
||||
new_binaries.append(binary.load_or_install(cache=cache) or binary)
|
||||
return self.model_copy(update={
|
||||
'binaries': new_binaries,
|
||||
})
|
||||
|
||||
|
||||
|
||||
class YtdlpPlugin(Plugin):
|
||||
name: str = 'ytdlp'
|
||||
configs: List[SerializeAsAny[ConfigSet]] = []
|
||||
binaries: List[SerializeAsAny[Binary]] = [YtdlpBinary()]
|
||||
extractors: List[SerializeAsAny[Extractor]] = [YtdlpExtractor()]
|
||||
replayers: List[SerializeAsAny[Replayer]] = [MEDIA_REPLAYER]
|
||||
|
||||
class WgetPlugin(Plugin):
|
||||
name: str = 'wget'
|
||||
configs: List[SerializeAsAny[ConfigSet]] = [*WGET_CONFIG]
|
||||
binaries: List[SerializeAsAny[Binary]] = [WgetBinary()]
|
||||
extractors: List[SerializeAsAny[Extractor]] = [WgetExtractor(), WarcExtractor()]
|
||||
|
||||
|
||||
YTDLP_PLUGIN = YtdlpPlugin()
|
||||
WGET_PLUGIN = WgetPlugin()
|
||||
PLUGINS = [
|
||||
YTDLP_PLUGIN,
|
||||
WGET_PLUGIN,
|
||||
]
|
||||
LOADED_PLUGINS = PLUGINS
|
||||
|
||||
|
||||
import json
|
||||
|
||||
for plugin in PLUGINS:
|
||||
try:
|
||||
json.dumps(plugin.model_json_schema(), indent=4)
|
||||
# print(json.dumps(plugin.model_json_schema(), indent=4))
|
||||
except Exception as err:
|
||||
print(f'Failed to generate JSON schema for {plugin.name}')
|
||||
raise
|
||||
|
||||
# print('-------------------------------------BEFORE INSTALL---------------------------------')
|
||||
# for plugin in PLUGINS:
|
||||
# print(plugin.model_dump_json(indent=4))
|
||||
# print('-------------------------------------DURING LOAD/INSTALL---------------------------------')
|
||||
# for plugin in PLUGINS:
|
||||
# LOADED_PLUGINS.append(plugin.install())
|
||||
# print('-------------------------------------AFTER INSTALL---------------------------------')
|
||||
# for plugin in LOADED_PLUGINS:
|
||||
# print(plugin.model_dump_json(indent=4))
|
||||
|
||||
|
|
@ -4,17 +4,19 @@ import inspect
|
|||
from typing import Any
|
||||
|
||||
from django.http import HttpRequest
|
||||
from django.conf import settings
|
||||
from django.utils.html import format_html, mark_safe
|
||||
|
||||
from admin_data_views.typing import TableContext, ItemContext
|
||||
from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
|
||||
|
||||
|
||||
from plugantic.plugins import LOADED_PLUGINS
|
||||
from django.conf import settings
|
||||
|
||||
def obj_to_yaml(obj: Any, indent: int=0) -> str:
|
||||
indent_str = " " * indent
|
||||
if indent == 0:
|
||||
indent_str = '\n' # put extra newline between top-level entries
|
||||
|
||||
if isinstance(obj, dict):
|
||||
if not obj:
|
||||
|
|
@ -74,22 +76,34 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
|||
if '_BINARY' in key or '_VERSION' in key
|
||||
}
|
||||
|
||||
for plugin in LOADED_PLUGINS:
|
||||
for plugin in settings.PLUGINS.values():
|
||||
for binary in plugin.binaries:
|
||||
binary = binary.load_or_install()
|
||||
try:
|
||||
binary = binary.load()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
rows['Binary'].append(ItemLink(binary.name, key=binary.name))
|
||||
rows['Found Version'].append(binary.loaded_version)
|
||||
rows['Found Version'].append(f'✅ {binary.loaded_version}' if binary.loaded_version else '❌ missing')
|
||||
rows['From Plugin'].append(plugin.name)
|
||||
rows['Provided By'].append(binary.loaded_provider)
|
||||
rows['Found Abspath'].append(binary.loaded_abspath)
|
||||
rows['Provided By'].append(
|
||||
', '.join(
|
||||
f'[{binprovider.name}]' if binprovider.name == getattr(binary.loaded_binprovider, 'name', None) else binprovider.name
|
||||
for binprovider in binary.binproviders_supported
|
||||
if binprovider
|
||||
)
|
||||
# binary.loaded_binprovider.name
|
||||
# if binary.loaded_binprovider else
|
||||
# ', '.join(getattr(provider, 'name', str(provider)) for provider in binary.binproviders_supported)
|
||||
)
|
||||
rows['Found Abspath'].append(binary.loaded_abspath or '❌ missing')
|
||||
rows['Related Configuration'].append(mark_safe(', '.join(
|
||||
f'<a href="/admin/environment/config/{config_key}/">{config_key}</a>'
|
||||
for config_key, config_value in relevant_configs.items()
|
||||
if binary.name.lower().replace('-', '').replace('_', '').replace('ytdlp', 'youtubedl') in config_key.lower()
|
||||
# or binary.name.lower().replace('-', '').replace('_', '') in str(config_value).lower()
|
||||
)))
|
||||
rows['Overrides'].append(obj_to_yaml(binary.provider_overrides))
|
||||
rows['Overrides'].append(str(obj_to_yaml(binary.provider_overrides))[:200])
|
||||
# rows['Description'].append(binary.description)
|
||||
|
||||
return TableContext(
|
||||
|
|
@ -104,7 +118,7 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
|
|||
|
||||
binary = None
|
||||
plugin = None
|
||||
for loaded_plugin in LOADED_PLUGINS:
|
||||
for loaded_plugin in settings.PLUGINS.values():
|
||||
for loaded_binary in loaded_plugin.binaries:
|
||||
if loaded_binary.name == key:
|
||||
binary = loaded_binary
|
||||
|
|
@ -112,7 +126,10 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
|
|||
|
||||
assert plugin and binary, f'Could not find a binary matching the specified name: {key}'
|
||||
|
||||
binary = binary.load_or_install()
|
||||
try:
|
||||
binary = binary.load()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
return ItemContext(
|
||||
slug=key,
|
||||
|
|
@ -120,14 +137,14 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
|
|||
data=[
|
||||
{
|
||||
"name": binary.name,
|
||||
"description": binary.description,
|
||||
"description": binary.abspath,
|
||||
"fields": {
|
||||
'plugin': plugin.name,
|
||||
'binprovider': binary.loaded_provider,
|
||||
'binprovider': binary.loaded_binprovider,
|
||||
'abspath': binary.loaded_abspath,
|
||||
'version': binary.loaded_version,
|
||||
'overrides': obj_to_yaml(binary.provider_overrides),
|
||||
'providers': obj_to_yaml(binary.providers_supported),
|
||||
'providers': obj_to_yaml(binary.binproviders_supported),
|
||||
},
|
||||
"help_texts": {
|
||||
# TODO
|
||||
|
|
@ -148,12 +165,15 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
|||
"extractors": [],
|
||||
"replayers": [],
|
||||
"configs": [],
|
||||
"description": [],
|
||||
"verbose_name": [],
|
||||
}
|
||||
|
||||
|
||||
for plugin in LOADED_PLUGINS:
|
||||
plugin = plugin.load_or_install()
|
||||
for plugin in settings.PLUGINS.values():
|
||||
try:
|
||||
plugin = plugin.load_binaries()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
rows['Name'].append(ItemLink(plugin.name, key=plugin.name))
|
||||
rows['binaries'].append(mark_safe(', '.join(
|
||||
|
|
@ -168,7 +188,7 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
|||
for config_key in configset.__fields__.keys()
|
||||
if config_key != 'section' and config_key in settings.CONFIG
|
||||
)))
|
||||
rows['description'].append(str(plugin.description))
|
||||
rows['verbose_name'].append(str(plugin.verbose_name))
|
||||
|
||||
return TableContext(
|
||||
title="Installed plugins",
|
||||
|
|
@ -181,13 +201,16 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
|
|||
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
|
||||
|
||||
plugin = None
|
||||
for loaded_plugin in LOADED_PLUGINS:
|
||||
for loaded_plugin in settings.PLUGINS.values():
|
||||
if loaded_plugin.name == key:
|
||||
plugin = loaded_plugin
|
||||
|
||||
assert plugin, f'Could not find a plugin matching the specified name: {key}'
|
||||
|
||||
plugin = plugin.load_or_install()
|
||||
try:
|
||||
plugin = plugin.load_binaries()
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
return ItemContext(
|
||||
slug=key,
|
||||
|
|
@ -195,12 +218,13 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
|
|||
data=[
|
||||
{
|
||||
"name": plugin.name,
|
||||
"description": plugin.description,
|
||||
"description": plugin.verbose_name,
|
||||
"fields": {
|
||||
'configs': plugin.configs,
|
||||
'binaries': plugin.binaries,
|
||||
'extractors': plugin.extractors,
|
||||
'replayers': plugin.replayers,
|
||||
'schema': obj_to_yaml(plugin.model_dump(include=('name', 'verbose_name', 'app_label', settings.PLUGIN_KEYS.keys()))),
|
||||
},
|
||||
"help_texts": {
|
||||
# TODO
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue