cleanup plugantic and pkg apps, make BaseHook actually create its own settings

archivebox/abid_utils/models.py

@@ -89,6 +89,8 @@ class ABIDModel(models.Model):
     # created_at = AutoDateTimeField(default=None, null=False, db_index=True)
     # modified_at = models.DateTimeField(auto_now=True)
 
+    _prefetched_objects_cache: Dict[str, Any]
+
     class Meta(TypedModelMeta):
         abstract = True
 

archivebox/builtin_plugins/npm/apps.py

@@ -1,17 +1,14 @@
 __package__ = 'archivebox.builtin_plugins.npm'
 
-from pathlib import Path
+from typing import List, Optional
-from typing import List, Dict, Optional
 from pydantic import InstanceOf, Field
 
-from django.apps import AppConfig
-from django.conf import settings
 
 from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr
-from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
+from plugantic.base_plugin import BasePlugin
-from plugantic.base_configset import ConfigSectionName
+from plugantic.base_configset import BaseConfigSet, ConfigSectionName
-
+from plugantic.base_binary import BaseBinary, BaseBinProvider, env, apt, brew
-from pkg.settings import env, apt, brew
+from plugantic.base_hook import BaseHook
 
 from ...config import CONFIG
 
@@ -33,10 +30,11 @@ DEFAULT_GLOBAL_CONFIG = {
 NPM_CONFIG = NpmDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
 
 
-class NpmProvider(NpmProvider, BaseBinProvider):
+class CustomNpmProvider(NpmProvider, BaseBinProvider):
     PATH: PATHStr = str(CONFIG.NODE_BIN_PATH)
 
-npm = NpmProvider(PATH=str(CONFIG.NODE_BIN_PATH))
+NPM_BINPROVIDER = CustomNpmProvider(PATH=str(CONFIG.NODE_BIN_PATH))
+npm = NPM_BINPROVIDER
 
 class NpmBinary(BaseBinary):
     name: BinName = 'npm'
@@ -55,19 +53,16 @@ NODE_BINARY = NodeBinary()
 
 
 class NpmPlugin(BasePlugin):
-    name: str = 'builtin_plugins.npm'
     app_label: str = 'npm'
     verbose_name: str = 'NPM'
     
-    configs: List[InstanceOf[BaseConfigSet]] = [NPM_CONFIG]
+    hooks: List[InstanceOf[BaseHook]] = [
-    binproviders: List[InstanceOf[BaseBinProvider]] = [npm]
+        NPM_CONFIG,
-    binaries: List[InstanceOf[BaseBinary]] = [NODE_BINARY, NPM_BINARY]
+        NPM_BINPROVIDER,
+        NODE_BINARY,
+        NPM_BINARY,
+    ]
 
 
 PLUGIN = NpmPlugin()
 DJANGO_APP = PLUGIN.AppConfig
-# CONFIGS = PLUGIN.configs
-# BINARIES = PLUGIN.binaries
-# EXTRACTORS = PLUGIN.extractors
-# REPLAYERS = PLUGIN.replayers
-# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/pip/apps.py

@@ -6,17 +6,16 @@ from typing import List, Dict, Optional
 from pydantic import InstanceOf, Field
 
 import django
-from django.apps import AppConfig
 
-from django.db.backends.sqlite3.base import Database as sqlite3
+from django.db.backends.sqlite3.base import Database as sqlite3     # type: ignore[import-type]
-from django.core.checks import Error, Tags, register
+from django.core.checks import Error, Tags
 
 from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer
-from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
+from plugantic.base_plugin import BasePlugin
-from plugantic.base_configset import ConfigSectionName
+from plugantic.base_configset import BaseConfigSet, ConfigSectionName
 from plugantic.base_check import BaseCheck
-
+from plugantic.base_binary import BaseBinary, BaseBinProvider, env, apt, brew
-from pkg.settings import env, apt, brew
+from plugantic.base_hook import BaseHook
 
 
 ###################### Config ##########################
@@ -36,15 +35,17 @@ DEFAULT_GLOBAL_CONFIG = {
 }
 PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
 
-class PipProvider(PipProvider, BaseBinProvider):
+class CustomPipProvider(PipProvider, BaseBinProvider):
     PATH: PATHStr = str(Path(sys.executable).parent)
 
-pip = PipProvider(PATH=str(Path(sys.executable).parent))
 
+PIP_BINPROVIDER = CustomPipProvider(PATH=str(Path(sys.executable).parent))
+pip = PIP_BINPROVIDER
 
 class PipBinary(BaseBinary):
     name: BinName = 'pip'
     binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
+
 PIP_BINARY = PipBinary()
 
 
@@ -57,8 +58,8 @@ class PythonBinary(BaseBinary):
     binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
         'apt': {
-            'subdeps': \
+            'packages': \
-                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
+                lambda: 'python3 python3-minimal python3-pip python3-setuptools python3-virtualenv',
             'abspath': \
                 lambda: sys.executable,
             'version': \
@@ -66,6 +67,8 @@ class PythonBinary(BaseBinary):
         },
     }
 
+PYTHON_BINARY = PythonBinary()
+
 class SqliteBinary(BaseBinary):
     name: BinName = 'sqlite'
     binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
@@ -78,6 +81,8 @@ class SqliteBinary(BaseBinary):
         },
     }
 
+SQLITE_BINARY = SqliteBinary()
+
 
 class DjangoBinary(BaseBinary):
     name: BinName = 'django'
@@ -92,12 +97,12 @@ class DjangoBinary(BaseBinary):
         },
     }
 
-
+DJANGO_BINARY = DjangoBinary()
 
 
 class CheckUserIsNotRoot(BaseCheck):
     label: str = 'CheckUserIsNotRoot'
-    tag = Tags.database
+    tag: str = Tags.database
 
     @staticmethod
     def check(settings, logger) -> List[Warning]:
@@ -114,23 +119,22 @@ class CheckUserIsNotRoot(BaseCheck):
         return errors
 
 
+USER_IS_NOT_ROOT_CHECK = CheckUserIsNotRoot()
 
 
 class PipPlugin(BasePlugin):
-    name: str = 'builtin_plugins.pip'
     app_label: str = 'pip'
     verbose_name: str = 'PIP'
 
-    configs: List[InstanceOf[BaseConfigSet]] = [PIP_CONFIG]
+    hooks: List[InstanceOf[BaseHook]] = [
-    binproviders: List[InstanceOf[BaseBinProvider]] = [pip]
+        PIP_CONFIG,
-    binaries: List[InstanceOf[BaseBinary]] = [PIP_BINARY, PythonBinary(), SqliteBinary(), DjangoBinary()]
+        PIP_BINPROVIDER,
-    checks: List[InstanceOf[BaseCheck]] = [CheckUserIsNotRoot()]
+        PIP_BINARY,
-
+        PYTHON_BINARY,
+        SQLITE_BINARY,
+        DJANGO_BINARY,
+        USER_IS_NOT_ROOT_CHECK,
+    ]
 
 PLUGIN = PipPlugin()
 DJANGO_APP = PLUGIN.AppConfig
-# CONFIGS = PLUGIN.configs
-# BINARIES = PLUGIN.binaries
-# EXTRACTORS = PLUGIN.extractors
-# REPLAYERS = PLUGIN.replayers
-# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/singlefile/apps.py

@@ -1,19 +1,18 @@
 from pathlib import Path
 from typing import List, Dict, Optional
 
-from django.apps import AppConfig
-
 # Depends on other PyPI/vendor packages:
 from pydantic import InstanceOf, Field
 from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName
-from pydantic_pkgr.binprovider import bin_abspath
 
 # Depends on other Django apps:
-from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseExtractor, BaseReplayer
+from plugantic.base_plugin import BasePlugin
-from plugantic.base_configset import ConfigSectionName
+from plugantic.base_configset import BaseConfigSet, ConfigSectionName
+from plugantic.base_binary import BaseBinary, env
+from plugantic.base_extractor import BaseExtractor
+from plugantic.base_hook import BaseHook
 
 # Depends on Other Plugins:
-from pkg.settings import env
 from builtin_plugins.npm.apps import npm
 
 
@@ -54,11 +53,7 @@ DEFAULT_GLOBAL_CONFIG = {
     'TIMEOUT': 120,
 }
 
-SINGLEFILE_CONFIGS = [
+SINGLEFILE_CONFIG = SinglefileConfigs(**DEFAULT_GLOBAL_CONFIG)
-    SinglefileToggleConfigs(**DEFAULT_GLOBAL_CONFIG),
-    SinglefileDependencyConfigs(**DEFAULT_GLOBAL_CONFIG),
-    SinglefileOptionsConfigs(**DEFAULT_GLOBAL_CONFIG),
-]
 
 
 
@@ -79,7 +74,7 @@ class SinglefileBinary(BaseBinary):
         # },
         # 'npm': {
         #     'abspath': lambda: bin_abspath('single-file', PATH=npm.PATH) or bin_abspath('single-file-node.js', PATH=npm.PATH),
-        #     'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}',
+        #     'packages': lambda: f'single-file-cli@>={min_version} <{max_version}',
         # },
     }
 
@@ -99,20 +94,16 @@ SINGLEFILE_BINARY = SinglefileBinary()
 SINGLEFILE_EXTRACTOR = SinglefileExtractor()
 
 class SinglefilePlugin(BasePlugin):
-    name: str = 'builtin_plugins.singlefile'
     app_label: str ='singlefile'
     verbose_name: str = 'SingleFile'
 
-    configs: List[InstanceOf[BaseConfigSet]] = SINGLEFILE_CONFIGS
+    hooks: List[InstanceOf[BaseHook]] = [
-    binaries: List[InstanceOf[BaseBinary]] = [SINGLEFILE_BINARY]
+        SINGLEFILE_CONFIG,
-    extractors: List[InstanceOf[BaseExtractor]] = [SINGLEFILE_EXTRACTOR]
+        SINGLEFILE_BINARY,
+        SINGLEFILE_EXTRACTOR,
+    ]
 
 
 
 PLUGIN = SinglefilePlugin()
 DJANGO_APP = PLUGIN.AppConfig
-# CONFIGS = PLUGIN.configs
-# BINARIES = PLUGIN.binaries
-# EXTRACTORS = PLUGIN.extractors
-# REPLAYERS = PLUGIN.replayers
-# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/ytdlp/apps.py

@@ -1,17 +1,13 @@
-import sys
+from typing import List, Dict
-import shutil
+from subprocess import run, PIPE
-from pathlib import Path
-from typing import List, Dict, Optional
-from subprocess import run, PIPE, CompletedProcess
 from pydantic import InstanceOf, Field
 
-from django.apps import AppConfig
 
-from pydantic_pkgr import BinProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict
+from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict
-from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
+from plugantic.base_plugin import BasePlugin
-from plugantic.base_configset import ConfigSectionName
+from plugantic.base_configset import BaseConfigSet, ConfigSectionName
-
+from plugantic.base_binary import BaseBinary, env, apt, brew
-from pkg.settings import env, apt, brew
+from plugantic.base_hook import BaseHook
 
 from builtin_plugins.pip.apps import pip
 
@@ -67,12 +63,14 @@ FFMPEG_BINARY = FfmpegBinary()
 
 
 class YtdlpPlugin(BasePlugin):
-    name: str = 'builtin_plugins.ytdlp'
     app_label: str = 'ytdlp'
     verbose_name: str = 'YTDLP'
 
-    configs: List[InstanceOf[BaseConfigSet]] = [YTDLP_CONFIG]
+    hooks: List[InstanceOf[BaseHook]] = [
-    binaries: List[InstanceOf[BaseBinary]] = [YTDLP_BINARY, FFMPEG_BINARY]
+        YTDLP_CONFIG,
+        YTDLP_BINARY,
+        FFMPEG_BINARY,
+    ]
 
 
 PLUGIN = YtdlpPlugin()

archivebox/core/models.py

@@ -1,7 +1,7 @@
 __package__ = 'archivebox.core'
 
 
-from typing import Optional, List, Dict, Iterable
+from typing import Optional, Dict, Iterable
 from django_stubs_ext.db.models import TypedModelMeta
 
 import json
@@ -9,7 +9,6 @@ import json
 from pathlib import Path
 
 from django.db import models
-from django.utils import timezone
 from django.utils.functional import cached_property
 from django.utils.text import slugify
 from django.core.cache import cache
@@ -107,7 +106,7 @@ class Tag(ABIDModel):
 
     @property
     def api_docs_url(self) -> str:
-        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_tag'
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_tag'
 
 class SnapshotTag(models.Model):
     id = models.AutoField(primary_key=True)
@@ -215,7 +214,7 @@ class Snapshot(ABIDModel):
     
     @property
     def api_docs_url(self) -> str:
-        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
     
     def get_absolute_url(self):
         return f'/{self.archive_path}'
@@ -483,7 +482,7 @@ class ArchiveResult(ABIDModel):
 
     @property
     def api_docs_url(self) -> str:
-        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
 
     def get_absolute_url(self):
         return f'/{self.snapshot.archive_path}/{self.output_path()}'

archivebox/core/settings.py

@@ -40,27 +40,18 @@ INSTALLED_PLUGINS = {
     **find_plugins_in_dir(USERDATA_PLUGINS_DIR, prefix='user_plugins'),
 }
 
-### Plugins Globals (filled by plugantic.apps.load_plugins() after Django startup)
+### Plugins Globals (filled by builtin_plugins.npm.apps.NpmPlugin.register() after Django startup)
 PLUGINS = AttrDict({})
 HOOKS = AttrDict({})
 
-CONFIGS = AttrDict({})
+# CONFIGS = AttrDict({})
-BINPROVIDERS = AttrDict({})
+# BINPROVIDERS = AttrDict({})
-BINARIES = AttrDict({})
+# BINARIES = AttrDict({})
-EXTRACTORS = AttrDict({})
+# EXTRACTORS = AttrDict({})
-REPLAYERS = AttrDict({})
+# REPLAYERS = AttrDict({})
-CHECKS = AttrDict({})
+# CHECKS = AttrDict({})
-ADMINDATAVIEWS = AttrDict({})
+# ADMINDATAVIEWS = AttrDict({})
 
-PLUGIN_KEYS = AttrDict({
-    'CONFIGS': CONFIGS,
-    'BINPROVIDERS': BINPROVIDERS,
-    'BINARIES': BINARIES,
-    'EXTRACTORS': EXTRACTORS,
-    'REPLAYERS': REPLAYERS,
-    'CHECKS': CHECKS,
-    'ADMINDATAVIEWS': ADMINDATAVIEWS,
-})
 
 ################################################################################
 ### Django Core Settings
@@ -95,12 +86,11 @@ INSTALLED_APPS = [
     'signal_webhooks',           # handles REST API outbound webhooks                              https://github.com/MrThearMan/django-signal-webhooks
     'django_object_actions',     # provides easy Django Admin action buttons on change views       https://github.com/crccheck/django-object-actions
     
-    # our own apps
+    # Our ArchiveBox-provided apps
     'abid_utils',                # handles ABID ID creation, handling, and models
     'plugantic',                 # ArchiveBox plugin API definition + finding/registering/calling interface
     'core',                      # core django model with Snapshot, ArchiveResult, etc.
     'api',                       # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
-    'pkg',                       # ArchiveBox runtime package management interface for subdependencies
 
     # ArchiveBox plugins
     *INSTALLED_PLUGINS.keys(),   # all plugin django-apps found in archivebox/builtin_plugins and data/user_plugins

archivebox/pkg/admin.py

@@ -1,3 +0,0 @@
-from django.contrib import admin
-
-# Register your models here.

archivebox/pkg/apps.py

@@ -1,16 +0,0 @@
-__package__ = 'archivebox.pkg'
-
-from django.apps import AppConfig
-
-
-class PkgsConfig(AppConfig):
-    name = 'pkg'
-    verbose_name = 'Package Management'
-    
-    default_auto_field = 'django.db.models.BigAutoField'
-
-    def ready(self):
-        from .settings import LOADED_DEPENDENCIES
-
-        # print(LOADED_DEPENDENCIES)
-        

archivebox/pkg/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

archivebox/pkg/settings.py

@@ -1,33 +0,0 @@
-__package__ = 'archivebox.pkg'
-
-import os
-import sys
-import shutil
-import inspect
-from pathlib import Path
-
-import django
-from django.conf import settings
-from django.db.backends.sqlite3.base import Database as sqlite3
-
-from pydantic_pkgr import Binary, BinProvider, BrewProvider, PipProvider, NpmProvider, AptProvider, EnvProvider, SemVer
-from pydantic_pkgr.binprovider import bin_abspath
-
-from ..config import NODE_BIN_PATH, bin_path
-
-apt = AptProvider()
-brew = BrewProvider()
-env = EnvProvider(PATH=os.environ.get('PATH', '/bin'))
-
-# Defined in their own plugins:
-#pip = PipProvider(PATH=str(Path(sys.executable).parent))
-#npm = NpmProvider(PATH=NODE_BIN_PATH)
-
-LOADED_DEPENDENCIES = {}
-
-for bin_name, binary_spec in settings.BINARIES.items():
-    try:
-        settings.BINARIES[bin_name] = binary_spec.load()
-    except Exception as e:
-        # print(f"- ❌ Binary {bin_name} failed to load with error: {e}")
-        continue

archivebox/pkg/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/pkg/views.py

@@ -1,3 +0,0 @@
-from django.shortcuts import render
-
-# Create your views here.

archivebox/plugantic/apps.py

@@ -1,8 +1,5 @@
 __package__ = 'archivebox.plugantic'
 
-import json
-import importlib
-
 from django.apps import AppConfig
 
 class PluganticConfig(AppConfig):
@@ -10,6 +7,6 @@ class PluganticConfig(AppConfig):
     name = 'plugantic'
 
     def ready(self) -> None:
-        from django.conf import settings
+        pass
-
+        # from django.conf import settings
-        print(f'[🧩] Detected {len(settings.INSTALLED_PLUGINS)} settings.INSTALLED_PLUGINS to load...')
+        # print(f'[🧩] Detected {len(settings.INSTALLED_PLUGINS)} settings.INSTALLED_PLUGINS to load...')

archivebox/plugantic/base_admindataview.py

@@ -1,13 +1,14 @@
-from typing import List, Type, Any, Dict
+__package__ = 'archivebox.plugantic'
 
-from pydantic_core import core_schema
+from typing import Dict
-from pydantic import GetCoreSchemaHandler, BaseModel
 
-from django.utils.functional import classproperty
+from .base_hook import BaseHook, HookType
-from django.core.checks import Warning, Tags, register
+from ..config_stubs import AttrDict
 
-class BaseAdminDataView(BaseModel):
+class BaseAdminDataView(BaseHook):
-    name: str = 'NPM Installed Packages'
+    hook_type: HookType = "ADMINDATAVIEW"
+    
+    verbose_name: str = 'NPM Installed Packages'
     route: str = '/npm/installed/'
     view: str = 'builtin_plugins.npm.admin.installed_list_view'
     items: Dict[str, str] = {
@@ -16,19 +17,22 @@ class BaseAdminDataView(BaseModel):
         'view': 'builtin_plugins.npm.admin.installed_detail_view',
     }
 
-    def as_route(self) -> Dict[str, str | Dict[str, str]]:
-        return {
-            'route': self.route,
-            'view': self.view,
-            'name': self.name,
-            'items': self.items,
-        }
-
     def register(self, settings, parent_plugin=None):
-        """Regsiter AdminDataViews.as_route() in settings.ADMIN_DATA_VIEWS.URLS at runtime"""
+        # self._plugin = parent_plugin                          # circular ref to parent only here for easier debugging! never depend on circular backref to parent in real code!
-        self._plugin = parent_plugin                          # circular ref to parent only here for easier debugging! never depend on circular backref to parent in real code!
 
-        route = self.as_route()
+        self.register_route_in_admin_data_view_urls(settings)
+
+        settings.ADMINDATAVIEWS = getattr(settings, "ADMINDATAVIEWS", None) or AttrDict({})
+        settings.ADMINDATAVIEWS[self.id] = self
+
+        super().register(settings, parent_plugin)
+
+    def register_route_in_admin_data_view_urls(self, settings):
+        route = {
+            "route": self.route,
+            "view": self.view,
+            "name": self.verbose_name,
+            "items": self.items,
+        }
         if route not in settings.ADMIN_DATA_VIEWS.URLS:
             settings.ADMIN_DATA_VIEWS.URLS += [route]  # append our route (update in place)
-

archivebox/plugantic/base_binary.py

@@ -1,25 +1,18 @@
 __package__ = 'archivebox.plugantic'
 
-import sys
+import os
-import inspect
+from typing import Dict, List
-import importlib
-from pathlib import Path
-
-
-from typing import Any, Optional, Dict, List
-from typing_extensions import Self
-from subprocess import run, PIPE
 
 from pydantic import Field, InstanceOf
-from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict
+from pydantic_pkgr import Binary, BinProvider, BinProviderName, ProviderLookupDict, AptProvider, BrewProvider, EnvProvider
-from pydantic_pkgr.binprovider import HostBinPath
 
-import django
+from .base_hook import BaseHook, HookType
-from django.core.cache import cache
+from ..config_stubs import AttrDict
-from django.db.backends.sqlite3.base import Database as sqlite3
 
 
-class BaseBinProvider(BinProvider):
+class BaseBinProvider(BaseHook, BinProvider):
+    hook_type: HookType = 'BINPROVIDER'
+    
     # def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
     #     Class = super()
     #     get_abspath_func = lambda: Class.on_get_abspath(bin_name, **context)
@@ -33,68 +26,30 @@ class BaseBinProvider(BinProvider):
     #     return get_version_func()
 
     def register(self, settings, parent_plugin=None):
-        if settings is None:
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
-            from django.conf import settings as django_settings
-            settings = django_settings
 
-        self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+        settings.BINPROVIDERS = getattr(settings, "BINPROVIDERS", None) or AttrDict({})
-        settings.BINPROVIDERS[self.name] = self
+        settings.BINPROVIDERS[self.id] = self
+
+        super().register(settings, parent_plugin=parent_plugin)
 
 
-class BaseBinary(Binary):
+class BaseBinary(BaseHook, Binary):
+    hook_type: HookType = "BINARY"
+    
     binproviders_supported: List[InstanceOf[BinProvider]] = Field(default_factory=list, alias='binproviders')
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] = Field(default_factory=dict, alias='overrides')
 
     def register(self, settings, parent_plugin=None):
-        if settings is None:
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
-            from django.conf import settings as django_settings
-            settings = django_settings
 
-        self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+        settings.BINARIES = getattr(settings, "BINARIES", None) or AttrDict({})
-        settings.BINARIES[self.name] = self
+        settings.BINARIES[self.id] = self
 
-# def get_ytdlp_version() -> str:
+        super().register(settings, parent_plugin=parent_plugin)
-#     import yt_dlp
-#     return yt_dlp.version.__version__
 
 
 
-
+apt = AptProvider()
-# class YtdlpBinary(Binary):
+brew = BrewProvider()
-#     name: BinName = 'yt-dlp'
+env = EnvProvider(PATH=os.environ.get("PATH", "/bin"))
-#     providers_supported: List[BinProvider] = [
-#         EnvProvider(),
-#         PipProvider(),
-#         BrewProvider(),
-#         AptProvider(),
-#     ]
-#     provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-#         'pip': {
-#             'version': get_ytdlp_version,
-#         },
-#         'brew': {
-#             'subdeps': lambda: 'yt-dlp ffmpeg',
-#         },
-#         'apt': {
-#             'subdeps': lambda: 'yt-dlp ffmpeg',
-#         }
-#     }
-
-# class WgetBinary(Binary):
-#     name: BinName = 'wget'
-#     providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()]
-
-
-# if __name__ == '__main__':
-#     PYTHON_BINARY = PythonBinary()
-#     SQLITE_BINARY = SqliteBinary()
-#     DJANGO_BINARY = DjangoBinary()
-#     WGET_BINARY = WgetBinary()
-#     YTDLP_BINARY = YtdlpPBinary()
-
-#     print('-------------------------------------DEFINING BINARIES---------------------------------')
-#     print(PYTHON_BINARY)
-#     print(SQLITE_BINARY)
-#     print(DJANGO_BINARY)
-#     print(WGET_BINARY)
-#     print(YTDLP_BINARY)

archivebox/plugantic/base_check.py

@@ -1,29 +1,17 @@
-from typing import List, Type, Any
+__package__ = "archivebox.plugantic"
 
-from pydantic_core import core_schema
+from typing import List
-from pydantic import GetCoreSchemaHandler, BaseModel
 
-from django.utils.functional import classproperty
 from django.core.checks import Warning, Tags, register
 
-class BaseCheck:
+from .base_hook import BaseHook, HookType
-    label: str = ''
+from ..config_stubs import AttrDict
+
+class BaseCheck(BaseHook):
+    hook_type: HookType = "CHECK"
+    
     tag: str = Tags.database
     
-    @classmethod
-    def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
-        return core_schema.typed_dict_schema(
-            {
-                'name': core_schema.typed_dict_field(core_schema.str_schema()),
-                'tag': core_schema.typed_dict_field(core_schema.str_schema()),
-            },
-        )
-
-
-    @classproperty
-    def name(cls) -> str:
-        return cls.label or cls.__name__
-    
     @staticmethod
     def check(settings, logger) -> List[Warning]:
         """Override this method to implement your custom runtime check."""
@@ -38,18 +26,26 @@ class BaseCheck:
         return errors
 
     def register(self, settings, parent_plugin=None):
-        # Regsiter in ArchiveBox plugins runtime settings
+        # self._plugin = parent_plugin  # backref to parent is for debugging only, never rely on this!
-        self._plugin = parent_plugin
+
-        settings.CHECKS[self.name] = self
+        self.register_with_django_check_system()  # (SIDE EFFECT)
+
+        # install hook into settings.CHECKS
+        settings.CHECKS = getattr(settings, "CHECKS", None) or AttrDict({})
+        settings.CHECKS[self.id] = self
+
+        # record installed hook in settings.HOOKS
+        super().register(settings, parent_plugin=parent_plugin)
+
+    def register_with_django_check_system(self):
 
-        # Register using Django check framework
         def run_check(app_configs, **kwargs) -> List[Warning]:
             from django.conf import settings
             import logging
-            settings = settings
-            logger = logging.getLogger('checks')
-            return self.check(settings, logger)
 
-        run_check.__name__ = self.label or self.__class__.__name__
+            return self.check(settings, logging.getLogger("checks"))
+
+        run_check.__name__ = self.id
         run_check.tags = [self.tag]
         register(self.tag)(run_check)
+

archivebox/plugantic/base_configset.py

@@ -2,9 +2,10 @@ __package__ = 'archivebox.plugantic'
 
 
 from typing import List, Literal
-from pydantic import ConfigDict
 
 from .base_hook import BaseHook, HookType
+from ..config_stubs import AttrDict
+
 
 ConfigSectionName = Literal[
     'GENERAL_CONFIG',
@@ -21,23 +22,16 @@ ConfigSectionNames: List[ConfigSectionName] = [
 
 
 class BaseConfigSet(BaseHook):
-    model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow', populate_by_name=True)
     hook_type: HookType = 'CONFIG'
 
     section: ConfigSectionName = 'GENERAL_CONFIG'
 
     def register(self, settings, parent_plugin=None):
-        """Installs the ConfigSet into Django settings.CONFIGS (and settings.HOOKS)."""
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
-        if settings is None:
-            from django.conf import settings as django_settings
-            settings = django_settings
 
-        self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+        settings.CONFIGS = getattr(settings, "CONFIGS", None) or AttrDict({})
+        settings.CONFIGS[self.id] = self
 
-        # install hook into settings.CONFIGS
-        settings.CONFIGS[self.name] = self
-
-        # record installed hook in settings.HOOKS
         super().register(settings, parent_plugin=parent_plugin)
 
 

archivebox/plugantic/base_extractor.py

@@ -3,28 +3,13 @@ __package__ = 'archivebox.plugantic'
 from typing import Optional, List, Literal, Annotated, Dict, Any
 from typing_extensions import Self
 
-from abc import ABC
 from pathlib import Path
 
-from pydantic import BaseModel, model_validator, field_serializer, AfterValidator, Field
+from pydantic import model_validator, AfterValidator
 from pydantic_pkgr import BinName
 
-# from .binaries import (
+from .base_hook import BaseHook, HookType
-#     Binary,
+from ..config_stubs import AttrDict
-#     YtdlpBinary,
-#     WgetBinary,
-# )
-
-
-# stubs
-class Snapshot:
-    pass
-
-class ArchiveResult:
-    pass
-
-def get_wget_output_path(*args, **kwargs) -> Path:
-    return Path('.').resolve()
 
 
 
@@ -38,7 +23,9 @@ HandlerFuncStr = Annotated[str, AfterValidator(lambda s: s.startswith('self.'))]
 CmdArgsList = Annotated[List[str], AfterValidator(no_empty_args)]
 
 
-class BaseExtractor(ABC, BaseModel):
+class BaseExtractor(BaseHook):
+    hook_type: HookType = 'EXTRACTOR'
+    
     name: ExtractorName
     binary: BinName
 
@@ -57,16 +44,19 @@ class BaseExtractor(ABC, BaseModel):
             self.args = [*self.default_args, *self.extra_args]
         return self
 
-    def register(self, settings, parent_plugin=None):
-        if settings is None:
-            from django.conf import settings as django_settings
-            settings = django_settings
 
-        self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+    def register(self, settings, parent_plugin=None):
-        settings.EXTRACTORS[self.name] = self
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+
+        settings.EXTRACTORS = getattr(settings, "EXTRACTORS", None) or AttrDict({})
+        settings.EXTRACTORS[self.id] = self
+
+        super().register(settings, parent_plugin=parent_plugin)
+
+
 
     def get_output_path(self, snapshot) -> Path:
-        return Path(self.name)
+        return Path(self.id.lower())
 
     def should_extract(self, snapshot) -> bool:
         output_dir = self.get_output_path(snapshot)
@@ -106,7 +96,7 @@ class BaseExtractor(ABC, BaseModel):
 #     binary: Binary = YtdlpBinary()
 
 #     def get_output_path(self, snapshot) -> Path:
-#         return Path(self.name)
+#         return 'media/'
 
 
 # class WgetExtractor(Extractor):

archivebox/plugantic/base_hook.py

@@ -1,9 +1,8 @@
 __package__ = 'archivebox.plugantic'
 
 import json
-from typing import Optional, List, Literal, ClassVar
+from typing import List, Literal
-from pathlib import Path
+from pydantic import BaseModel, ConfigDict, Field, computed_field
-from pydantic import BaseModel, Field, ConfigDict, computed_field
 
 
 HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW']
@@ -50,31 +49,39 @@ class BaseHook(BaseModel):
 
     """
     model_config = ConfigDict(
-        extra='allow',
+        extra="allow",
         arbitrary_types_allowed=True,
         from_attributes=True,
         populate_by_name=True,
         validate_defaults=True,
         validate_assignment=True,
+        revalidate_instances="always",
     )
     
-    hook_type: HookType = 'CONFIG'
+    # verbose_name: str = Field()
 
+
+    @computed_field
     @property
-    def name(self) -> str:
+    def id(self) -> str:
-        return f'{self.__module__}.{__class__.__name__}'
+        return self.__class__.__name__
+    
+    @computed_field
+    @property
+    def hook_module(self) -> str:
+        return f'{self.__module__}.{self.__class__.__name__}'
+    
+    hook_type: HookType = Field()
+    
+    
 
     def register(self, settings, parent_plugin=None):
         """Load a record of an installed hook into global Django settings.HOOKS at runtime."""
+        self._plugin = parent_plugin         # for debugging only, never rely on this!
 
-        assert json.dumps(self.model_json_schema(), indent=4), f'Hook {self.name} has invalid JSON schema.'
+        # assert json.dumps(self.model_json_schema(), indent=4), f"Hook {self.hook_module} has invalid JSON schema."
-
-        if settings is None:
-            from django.conf import settings as django_settings
-            settings = django_settings
 
         # record installed hook in settings.HOOKS
-        self._plugin = parent_plugin         # for debugging only, never rely on this!
+        settings.HOOKS[self.id] = self
-        settings.HOOKS[self.name] = self
 
-        print('REGISTERED HOOK:', self.name)
+        # print("REGISTERED HOOK:", self.hook_module)

archivebox/plugantic/base_plugin.py

@@ -5,9 +5,8 @@ import inspect
 from pathlib import Path
 
 from django.apps import AppConfig
-from django.core.checks import register
 
-from typing import List, ClassVar, Type, Dict
+from typing import List, Type, Dict
 from typing_extensions import Self
 
 from pydantic import (
@@ -20,95 +19,80 @@ from pydantic import (
     validate_call,
 )
 
-from .base_configset import BaseConfigSet
+from .base_hook import BaseHook, HookType
-from .base_binary import BaseBinProvider, BaseBinary
-from .base_extractor import BaseExtractor
-from .base_replayer import BaseReplayer
-from .base_check import BaseCheck
-from .base_admindataview import BaseAdminDataView
 
-from ..config import ANSI, AttrDict
+from ..config import AttrDict
 
 
 class BasePlugin(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
 
     # Required by AppConfig:
-    name: str = Field()                           # e.g. 'builtin_plugins.singlefile'  (DottedImportPath)
     app_label: str = Field()                      # e.g. 'singlefile'                  (one-word machine-readable representation, to use as url-safe id/db-table prefix_/attr name)
     verbose_name: str = Field()                   # e.g. 'SingleFile'                  (human-readable *short* label, for use in column names, form labels, etc.)
     
     # All the hooks the plugin will install:
-    configs: List[InstanceOf[BaseConfigSet]] = Field(default=[])
+    hooks: List[InstanceOf[BaseHook]] = Field(default=[])
-    binproviders: List[InstanceOf[BaseBinProvider]] = Field(default=[])                # e.g. [Binary(name='yt-dlp')]
+    
-    binaries: List[InstanceOf[BaseBinary]] = Field(default=[])                # e.g. [Binary(name='yt-dlp')]
+    @computed_field
-    extractors: List[InstanceOf[BaseExtractor]] = Field(default=[])
+    @property
-    replayers: List[InstanceOf[BaseReplayer]] = Field(default=[])
+    def id(self) -> str:
-    checks: List[InstanceOf[BaseCheck]] = Field(default=[])
+        return self.__class__.__name__
-    admindataviews: List[InstanceOf[BaseAdminDataView]] = Field(default=[])
+    
+    @computed_field
+    @property
+    def plugin_module(self) -> str:  # DottedImportPath
+        """ "
+        Dotted import path of the plugin's module (after its loaded via settings.INSTALLED_APPS).
+        e.g. 'archivebox.builtin_plugins.npm.apps.NpmPlugin' -> 'builtin_plugins.npm'
+        """
+        return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit('.apps.', 1)[0]
+
+    @computed_field
+    @property
+    def plugin_dir(self) -> Path:
+        return Path(inspect.getfile(self.__class__)).parent.resolve()
     
     @model_validator(mode='after')
     def validate(self) -> Self:
         """Validate the plugin's build-time configuration here before it's registered in Django at runtime."""
         
-        assert self.name and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
+        assert self.app_label and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
         
-        assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
+        assert json.dumps(self.model_json_schema(), indent=4), f"Plugin {self.plugin_module} has invalid JSON schema."
         return self
     
     @property
     def AppConfig(plugin_self) -> Type[AppConfig]:
         """Generate a Django AppConfig class for this plugin."""
 
+
         class PluginAppConfig(AppConfig):
             """Django AppConfig for plugin, allows it to be loaded as a Django app listed in settings.INSTALLED_APPS."""
-            name = plugin_self.name
+            name = plugin_self.plugin_module
             app_label = plugin_self.app_label
             verbose_name = plugin_self.verbose_name
+
             default_auto_field = 'django.db.models.AutoField'
 
             def ready(self):
                 from django.conf import settings
-                
-                # plugin_self.validate()
                 plugin_self.register(settings)
 
         return PluginAppConfig
 
-    @computed_field
     @property
-    def BINPROVIDERS(self) -> Dict[str, BaseBinProvider]:
+    def HOOKS_BY_ID(self) -> Dict[str, InstanceOf[BaseHook]]:
-        return AttrDict({binprovider.name: binprovider for binprovider in self.binproviders})
+        return AttrDict({hook.id: hook for hook in self.hooks})
 
-    @computed_field
     @property
-    def BINARIES(self) -> Dict[str, BaseBinary]:
+    def HOOKS_BY_TYPE(self) -> Dict[HookType, Dict[str, InstanceOf[BaseHook]]]:
-        return AttrDict({binary.python_name: binary for binary in self.binaries})
+        hooks = AttrDict({})
+        for hook in self.hooks:
+            hooks[hook.hook_type] = hooks.get(hook.hook_type) or AttrDict({})
+            hooks[hook.hook_type][hook.id] = hook
+        return hooks
 
-    @computed_field
-    @property
-    def CONFIGS(self) -> Dict[str, BaseConfigSet]:
-        return AttrDict({config.name: config for config in self.configs})
-    
-    @computed_field
-    @property
-    def EXTRACTORS(self) -> Dict[str, BaseExtractor]:
-        return AttrDict({extractor.name: extractor for extractor in self.extractors})
-    
-    @computed_field
-    @property
-    def REPLAYERS(self) -> Dict[str, BaseReplayer]:
-        return AttrDict({replayer.name: replayer for replayer in self.replayers})
-    
-    @computed_field
-    @property
-    def CHECKS(self) -> Dict[str, BaseCheck]:
-        return AttrDict({check.name: check for check in self.checks})
-    
-    @computed_field
-    @property
-    def ADMINDATAVIEWS(self) -> Dict[str, BaseCheck]:
-        return AttrDict({admindataview.name: admindataview for admindataview in self.admindataviews})
 
     def register(self, settings=None):
         """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime."""
@@ -117,45 +101,17 @@ class BasePlugin(BaseModel):
             from django.conf import settings as django_settings
             settings = django_settings
 
-        assert all(hasattr(settings, key) for key in ['PLUGINS', 'CONFIGS', 'BINARIES', 'EXTRACTORS', 'REPLAYERS', 'ADMINDATAVIEWS']), 'Tried to register plugin in settings but couldnt find required global dicts in settings.'
+        assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.plugin_module} has invalid JSON schema.'
 
-        assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
+        assert self.id not in settings.PLUGINS, f'Tried to register plugin {self.plugin_module} but it conflicts with existing plugin of the same name ({self.app_label}).'
-
-        assert self.app_label not in settings.PLUGINS, f'Tried to register plugin {self.name} but it conflicts with existing plugin of the same name ({self.app_label}).'
 
         ### Mutate django.conf.settings... values in-place to include plugin-provided overrides
-        settings.PLUGINS[self.app_label] = self
+        settings.PLUGINS[self.id] = self
 
-        for config in self.CONFIGS.values():
+        for hook in self.hooks:
-            config.register(settings, parent_plugin=self)
+            hook.register(settings, parent_plugin=self)
 
-        for binprovider in self.BINPROVIDERS.values():
+        print('√ REGISTERED PLUGIN:', self.plugin_module)
-            binprovider.register(settings, parent_plugin=self)
-        
-        for binary in self.BINARIES.values():
-            binary.register(settings, parent_plugin=self)
-        
-        for extractor in self.EXTRACTORS.values():
-            extractor.register(settings, parent_plugin=self)
-
-        for replayer in self.REPLAYERS.values():
-            replayer.register(settings, parent_plugin=self)
-
-        for check in self.CHECKS.values():
-            check.register(settings, parent_plugin=self)
-
-        for admindataview in self.ADMINDATAVIEWS.values():
-            admindataview.register(settings, parent_plugin=self)
-
-        # TODO: add parsers? custom templates? persona fixtures?
-
-        plugin_prefix, plugin_shortname = self.name.split('.', 1)
-
-        print(
-            f'    > {ANSI.black}{plugin_prefix.upper().replace("_PLUGINS", "").ljust(15)} ' +
-            f'{ANSI.lightyellow}{plugin_shortname.ljust(12)} ' + 
-            f'{ANSI.black}CONFIGSx{len(self.configs)}  BINARIESx{len(self.binaries)}  EXTRACTORSx{len(self.extractors)}  REPLAYERSx{len(self.replayers)}  CHECKSx{len(self.CHECKS)}  ADMINDATAVIEWSx{len(self.ADMINDATAVIEWS)}{ANSI.reset}'
-        )
 
     # @validate_call
     # def install_binaries(self) -> Self:
@@ -169,7 +125,7 @@ class BasePlugin(BaseModel):
     @validate_call
     def load_binaries(self, cache=True) -> Self:
         new_binaries = []
-        for idx, binary in enumerate(self.binaries):
+        for idx, binary in enumerate(self.HOOKS_BY_TYPE['BINARY'].values()):
             new_binaries.append(binary.load(cache=cache) or binary)
         return self.model_copy(update={
             'binaries': new_binaries,
@@ -184,20 +140,6 @@ class BasePlugin(BaseModel):
     #         'binaries': new_binaries,
     #     })
 
-    @computed_field
-    @property
-    def module_dir(self) -> Path:
-        return Path(inspect.getfile(self.__class__)).parent.resolve()
-    
-    @computed_field
-    @property
-    def module_path(self) -> str:  # DottedImportPath
-        """"
-        Dotted import path of the plugin's module (after its loaded via settings.INSTALLED_APPS).
-        e.g. 'archivebox.builtin_plugins.npm'
-        """
-        return self.name.strip('archivebox.')
-
 
 
 

archivebox/plugantic/base_replayer.py

@@ -1,13 +1,15 @@
 __package__ = 'archivebox.plugantic'
 
 
-from pydantic import BaseModel
+from .base_hook import BaseHook, HookType
+from ..config_stubs import AttrDict
 
 
-
+class BaseReplayer(BaseHook):
-class BaseReplayer(BaseModel):
     """Describes how to render an ArchiveResult in several contexts"""
-    name: str = 'GenericReplayer'
+    
+    hook_type: HookType = 'REPLAYER'
+    
     url_pattern: str = '*'
 
     row_template: str = 'plugins/generic_replayer/templates/row.html'
@@ -21,13 +23,12 @@ class BaseReplayer(BaseModel):
     # thumbnail_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
 
     def register(self, settings, parent_plugin=None):
-        if settings is None:
+        # self._plugin = parent_plugin                                      # for debugging only, never rely on this!
-            from django.conf import settings as django_settings
-            settings = django_settings
 
-        self._plugin = parent_plugin                                      # for debugging only, never rely on this!
+        settings.REPLAYERS = getattr(settings, 'REPLAYERS', None) or AttrDict({})
-        settings.REPLAYERS[self.name] = self
+        settings.REPLAYERS[self.id] = self
 
+        super().register(settings, parent_plugin=parent_plugin)
 
 # class MediaReplayer(BaseReplayer):
 #     name: str = 'MediaReplayer'

archivebox/plugantic/management/commands/pkg.py

@@ -1,4 +1,4 @@
-__package__ = 'archivebox.pkg.management.commands'
+__package__ = 'archivebox.plugantic.management.commands'
 
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -7,8 +7,7 @@ from pydantic_pkgr import Binary, BinProvider, BrewProvider, EnvProvider, SemVer
 from pydantic_pkgr.binprovider import bin_abspath
 
 from ....config import NODE_BIN_PATH, bin_path
-
+from ...base_binary import env
-from pkg.settings import env
 
 
 class Command(BaseCommand):

archivebox/plugantic/views.py

@@ -235,7 +235,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
                     'binaries': plugin.binaries,
                     'extractors': plugin.extractors,
                     'replayers': plugin.replayers,
-                    'schema': obj_to_yaml(plugin.model_dump(include=('name', 'verbose_name', 'app_label', *settings.PLUGIN_KEYS.keys()))),
+                    'schema': obj_to_yaml(plugin.model_dump(include=('name', 'verbose_name', 'app_label', 'hooks'))),
                 },
                 "help_texts": {
                     # TODO

pyproject.toml

@@ -120,6 +120,9 @@ target-version = "py310"
 src = ["archivebox"]
 exclude = ["*.pyi", "typings/", "migrations/", "vendor/"]
 
+[tool.ruff.lint]
+ignore = ["E731"]
+
 [tool.pytest.ini_options]
 testpaths = [ "tests" ]