BasePlugin system expanded and registration system improved

archivebox/builtin_plugins/base/apps.py

@@ -1,83 +0,0 @@
-import sys
-import inspect
-from typing import List, Dict, Any, Optional
-from pathlib import Path
-
-import django
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
-from django.db.backends.sqlite3.base import Database as sqlite3
-
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import SemVer, BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
-
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
-from plugantic.replayers import Replayer
-
-
-class PythonBinary(Binary):
-    name: BinName = 'python'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides: Dict[str, Any] = {
-        'env': {
-            'subdeps': \
-                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
-            'abspath': \
-                lambda: sys.executable,
-            'version': \
-                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
-        },
-    }
-
-class SqliteBinary(Binary):
-    name: BinName = 'sqlite'
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: Path(inspect.getfile(sqlite3)),
-            'version': \
-                lambda: SemVer(sqlite3.version),
-        },
-    }
-
-
-class DjangoBinary(Binary):
-    name: BinName = 'django'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: inspect.getfile(django),
-            'version': \
-                lambda: django.VERSION[:3],
-        },
-    }
-
-
-class BasicReplayer(Replayer):
-    name: str = 'basic'
-
-
-class BasePlugin(Plugin):
-    name: str = 'base'
-    configs: List[SerializeAsAny[ConfigSet]] = []
-    binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = []
-    replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
-
-
-PLUGINS = [BasePlugin()]
-
-
-class BaseConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'builtin_plugins.base'

archivebox/builtin_plugins/npm/apps.py

@@ -0,0 +1,66 @@
+__package__ = 'archivebox.builtin_plugins.npm'
+
+from pathlib import Path
+from typing import List, Dict, Optional
+from pydantic import InstanceOf, Field
+
+from django.apps import AppConfig
+from django.conf import settings
+
+from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr
+from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
+from plugantic.base_configset import ConfigSectionName
+
+from pkg.settings import env, apt, brew
+
+from ...config import CONFIG
+
+###################### Config ##########################
+
+
+class NpmDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    USE_NPM: bool = True
+    NPM_BINARY: str = Field(default='npm')
+    NPM_ARGS: Optional[List[str]] = Field(default=None)
+    NPM_EXTRA_ARGS: List[str] = []
+    NPM_DEFAULT_ARGS: List[str] = []
+
+
+DEFAULT_GLOBAL_CONFIG = {
+}
+NPM_CONFIG = NpmDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+
+class NpmProvider(NpmProvider, BaseBinProvider):
+    PATH: PATHStr = str(CONFIG.NODE_BIN_PATH)
+
+npm = NpmProvider(PATH=str(CONFIG.NODE_BIN_PATH))
+
+class NpmBinary(BaseBinary):
+    name: BinName = 'npm'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [env, apt, brew]
+
+
+NPM_BINARY = NpmBinary()
+
+
+
+class NpmPlugin(BasePlugin):
+    name: str = 'builtin_plugins.npm'
+    app_label: str = 'npm'
+    verbose_name: str = 'NPM'
+
+    configs: List[InstanceOf[BaseConfigSet]] = [NPM_CONFIG]
+    binproviders: List[InstanceOf[BaseBinProvider]] = [npm]
+    binaries: List[InstanceOf[BaseBinary]] = [NPM_BINARY]
+
+
+PLUGIN = NpmPlugin()
+DJANGO_APP = PLUGIN.AppConfig
+# CONFIGS = PLUGIN.configs
+# BINARIES = PLUGIN.binaries
+# EXTRACTORS = PLUGIN.extractors
+# REPLAYERS = PLUGIN.replayers
+# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/pip/apps.py

@@ -0,0 +1,66 @@
+import sys
+from pathlib import Path
+from typing import List, Dict, Optional
+from pydantic import InstanceOf, Field
+
+from django.apps import AppConfig
+
+from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr
+from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
+from plugantic.base_configset import ConfigSectionName
+
+from pkg.settings import env, apt, brew
+
+
+###################### Config ##########################
+
+
+class PipDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    USE_PIP: bool = True
+    PIP_BINARY: str = Field(default='pip')
+    PIP_ARGS: Optional[List[str]] = Field(default=None)
+    PIP_EXTRA_ARGS: List[str] = []
+    PIP_DEFAULT_ARGS: List[str] = []
+
+
+DEFAULT_GLOBAL_CONFIG = {
+}
+PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+class PipProvider(PipProvider, BaseBinProvider):
+    PATH: PATHStr = str(Path(sys.executable).parent)
+
+pip = PipProvider(PATH=str(Path(sys.executable).parent))
+
+
+class PipBinary(BaseBinary):
+    name: BinName = 'pip'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew]
+PIP_BINARY = PipBinary()
+
+
+
+
+
+
+
+
+class PipPlugin(BasePlugin):
+    name: str = 'builtin_plugins.pip'
+    app_label: str = 'pip'
+    verbose_name: str = 'PIP'
+
+    configs: List[InstanceOf[BaseConfigSet]] = [PIP_CONFIG]
+    binproviders: List[InstanceOf[BaseBinProvider]] = [pip]
+    binaries: List[InstanceOf[BaseBinary]] = [PIP_BINARY]
+
+
+PLUGIN = PipPlugin()
+DJANGO_APP = PLUGIN.AppConfig
+# CONFIGS = PLUGIN.configs
+# BINARIES = PLUGIN.binaries
+# EXTRACTORS = PLUGIN.extractors
+# REPLAYERS = PLUGIN.replayers
+# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/singlefile/apps.py

@@ -1,42 +1,31 @@
-from typing import List, Optional, Dict
 from pathlib import Path
+from typing import List, Dict, Optional
 
 from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
 
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
+# Depends on other PyPI/vendor packages:
+from pydantic import InstanceOf, Field
+from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName
 from pydantic_pkgr.binprovider import bin_abspath
-from pydantic_pkgr.binary import BinProviderName, ProviderLookupDict
 
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
+# Depends on other Django apps:
+from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseExtractor, BaseReplayer
+from plugantic.base_configset import ConfigSectionName
 
+# Depends on Other Plugins:
 from pkg.settings import env
+from builtin_plugins.npm.apps import npm
 
 
 ###################### Config ##########################
 
-class SinglefileToggleConfig(ConfigSet):
+class SinglefileToggleConfigs(BaseConfigSet):
     section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
 
     SAVE_SINGLEFILE: bool = True
 
 
-class SinglefileDependencyConfig(ConfigSet):
-    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
-
-    SINGLEFILE_BINARY: str = Field(default='wget')
-    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
-    SINGLEFILE_EXTRA_ARGS: List[str] = []
-    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
-
-class SinglefileOptionsConfig(ConfigSet):
+class SinglefileOptionsConfigs(BaseConfigSet):
     section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
 
     # loaded from shared config
@@ -47,67 +36,83 @@ class SinglefileOptionsConfig(ConfigSet):
     SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
 
 
+class SinglefileDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
 
-DEFAULT_CONFIG = {
+    SINGLEFILE_BINARY: str = Field(default='wget')
+    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
+    SINGLEFILE_EXTRA_ARGS: List[str] = []
+    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+class SinglefileConfigs(SinglefileToggleConfigs, SinglefileOptionsConfigs, SinglefileDependencyConfigs):
+    # section: ConfigSectionName = 'ALL_CONFIGS'
+    pass
+
+DEFAULT_GLOBAL_CONFIG = {
     'CHECK_SSL_VALIDITY': False,
     'SAVE_SINGLEFILE': True,
     'TIMEOUT': 120,
 }
 
-PLUGIN_CONFIG = [
-    SinglefileToggleConfig(**DEFAULT_CONFIG),
-    SinglefileDependencyConfig(**DEFAULT_CONFIG),
-    SinglefileOptionsConfig(**DEFAULT_CONFIG),
+SINGLEFILE_CONFIGS = [
+    SinglefileToggleConfigs(**DEFAULT_GLOBAL_CONFIG),
+    SinglefileDependencyConfigs(**DEFAULT_GLOBAL_CONFIG),
+    SinglefileOptionsConfigs(**DEFAULT_GLOBAL_CONFIG),
 ]
 
-###################### Binaries ############################
+
 
 min_version: str = "1.1.54"
 max_version: str = "2.0.0"
 
-class SinglefileBinary(Binary):
-    name: BinName = 'single-file'
-    providers_supported: List[BinProvider] = [NpmProvider()]
+def get_singlefile_abspath() -> Optional[Path]:
+    return 
 
 
+class SinglefileBinary(BaseBinary):
+    name: BinName = 'single-file'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [env, npm]
+
     provider_overrides: Dict[BinProviderName, ProviderLookupDict] ={
-        'env': {
-            'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH),
-        },
-        'npm': {
-            # 'abspath': lambda: bin_abspath('single-file', PATH=NpmProvider().PATH) or bin_abspath('single-file', PATH=env.PATH),
-            'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}',
-        },
+        # 'env': {
+        #     'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH),
+        # },
+        # 'npm': {
+        #     'abspath': lambda: bin_abspath('single-file', PATH=npm.PATH) or bin_abspath('single-file-node.js', PATH=npm.PATH),
+        #     'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}',
+        # },
     }
 
+SINGLEFILE_BINARY = SinglefileBinary()
 
-###################### Extractors ##########################
+PLUGIN_BINARIES = [SINGLEFILE_BINARY]
 
-class SinglefileExtractor(Extractor):
-    name: ExtractorName = 'singlefile'
-    binary: Binary = SinglefileBinary()
+class SinglefileExtractor(BaseExtractor):
+    name: str = 'singlefile'
+    binary: BinName = SINGLEFILE_BINARY.name
 
     def get_output_path(self, snapshot) -> Path:
         return Path(snapshot.link_dir) / 'singlefile.html'
 
 
-###################### Plugins #############################
+SINGLEFILE_BINARY = SinglefileBinary()
+SINGLEFILE_EXTRACTOR = SinglefileExtractor()
+
+class SinglefilePlugin(BasePlugin):
+    name: str = 'builtin_plugins.singlefile'
+    app_label: str ='singlefile'
+    verbose_name: str = 'SingleFile'
+
+    configs: List[InstanceOf[BaseConfigSet]] = SINGLEFILE_CONFIGS
+    binaries: List[InstanceOf[BaseBinary]] = [SINGLEFILE_BINARY]
+    extractors: List[InstanceOf[BaseExtractor]] = [SINGLEFILE_EXTRACTOR]
 
 
-class SinglefilePlugin(Plugin):
-    name: str = 'singlefile'
-    configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
-    binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
 
-PLUGINS = [SinglefilePlugin()]
-
-###################### Django Apps #########################
-
-class SinglefileConfig(AppConfig):
-    name = 'builtin_plugins.singlefile'
-    verbose_name = 'SingleFile'
-
-    def ready(self):
-        pass
-        # print('Loaded singlefile plugin')
+PLUGIN = SinglefilePlugin()
+DJANGO_APP = PLUGIN.AppConfig
+# CONFIGS = PLUGIN.configs
+# BINARIES = PLUGIN.binaries
+# EXTRACTORS = PLUGIN.extractors
+# REPLAYERS = PLUGIN.replayers
+# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/singlefile/config.yaml

@@ -1,66 +0,0 @@
-name: singlefile
-plugin_version: '0.0.1'
-plugin_spec: '0.0.1'
-
-binaries:
-    singlefile:
-        providers:
-            - env
-            - npm
-
-commands:
-    - singlefile.exec
-    - singlefile.extract
-    - singlefile.should_extract
-    - singlefile.get_output_path
-
-extractors:
-    singlefile:
-        binary: singlefile
-        test: singlefile.should_extract
-        extract: singlefile.extract
-        output_files:
-            - singlefile.html
-
-configs:
-    ARCHIVE_METHOD_TOGGLES:
-        SAVE_SINGLEFILE:
-            type: bool
-            default: true
-
-    DEPENDENCY_CONFIG:
-        SINGLEFILE_BINARY:
-            type: str
-            default: wget
-        SINGLEFILE_ARGS:
-            type: Optional[List[str]]
-            default: null
-        SINGLEFILE_EXTRA_ARGS:
-            type: List[str]
-            default: []
-        SINGLEFILE_DEFAULT_ARGS:
-            type: List[str]
-            default: 
-            - "--timeout={TIMEOUT-10}"
-
-    ARCHIVE_METHOD_OPTIONS:
-        SINGLEFILE_USER_AGENT:
-            type: str
-            default: ""
-            alias: USER_AGENT
-        SINGLEFILE_TIMEOUT:
-            type: int
-            default: 60
-            alias: TIMEOUT
-        SINGLEFILE_CHECK_SSL_VALIDITY:
-            type: bool
-            default: true
-            alias: CHECK_SSL_VALIDITY
-        SINGLEFILE_RESTRICT_FILE_NAMES:
-            type: str
-            default: windows
-            alias: RESTRICT_FILE_NAMES
-        SINGLEFILE_COOKIES_FILE:
-            type: Optional[Path]
-            default: null
-            alias: COOKIES_FILE

archivebox/builtin_plugins/singlefile/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/builtin_plugins/systempython/apps.py

@@ -0,0 +1,116 @@
+__package__ = 'archivebox.builtin_plugins.systempython'
+
+import os
+import sys
+import inspect
+from typing import List, Dict, Any, Callable, ClassVar
+from pathlib import Path
+
+import django
+from django.apps import AppConfig
+from django.core.checks import Tags, Warning, register
+from django.utils.functional import classproperty
+from django.db.backends.sqlite3.base import Database as sqlite3
+from django.core.checks import Tags, Error, register
+
+from pydantic import InstanceOf, Field
+
+from pydantic_pkgr import SemVer, BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
+
+from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider, BaseExtractor, BaseReplayer
+from plugantic.base_check import BaseCheck
+
+from pkg.settings import env, apt, brew
+
+from builtin_plugins.pip.apps import pip
+
+class PythonBinary(BaseBinary):
+    name: BinName = 'python'
+
+    binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
+    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
+        'apt': {
+            'subdeps': \
+                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
+            'abspath': \
+                lambda: sys.executable,
+            'version': \
+                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
+        },
+    }
+
+class SqliteBinary(BaseBinary):
+    name: BinName = 'sqlite'
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'pip': {
+            'abspath': \
+                lambda: Path(inspect.getfile(sqlite3)),
+            'version': \
+                lambda: SemVer(sqlite3.version),
+        },
+    }
+
+
+class DjangoBinary(BaseBinary):
+    name: BinName = 'django'
+
+    binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'pip': {
+            'abspath': \
+                lambda: inspect.getfile(django),
+            'version': \
+                lambda: django.VERSION[:3],
+        },
+    }
+
+
+class BasicReplayer(BaseReplayer):
+    name: str = 'basic'
+
+
+
+
+class CheckUserIsNotRoot(BaseCheck):
+    label: str = 'CheckUserIsNotRoot'
+    tag = Tags.database
+
+    @staticmethod
+    def check(settings, logger) -> List[Warning]:
+        errors = []
+        if getattr(settings, "USER", None) == 'root' or getattr(settings, "PUID", None) == 0:
+            errors.append(
+                Error(
+                    "Cannot run as root!",
+                    id="core.S001",
+                    hint=f'Run ArchiveBox as a non-root user with a UID greater than 500. (currently running as UID {os.getuid()}).',
+                )
+            )
+        logger.debug('[√] UID is not root')
+        return errors
+
+
+
+class SystemPythonPlugin(BasePlugin):
+    name: str = 'builtin_plugins.systempython'
+    app_label: str = 'systempython'
+    verbose_name: str = 'System Python'
+
+    configs: List[InstanceOf[BaseConfigSet]] = []
+    binaries: List[InstanceOf[BaseBinary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
+    extractors: List[InstanceOf[BaseExtractor]] = []
+    replayers: List[InstanceOf[BaseReplayer]] = [BasicReplayer()]
+    checks: List[InstanceOf[BaseCheck]] = [CheckUserIsNotRoot()]
+
+
+PLUGIN = SystemPythonPlugin()
+DJANGO_APP = PLUGIN.AppConfig
+# CONFIGS = PLUGIN.configs
+# BINARIES = PLUGIN.binaries
+# EXTRACTORS = PLUGIN.extractors
+# REPLAYERS = PLUGIN.replayers
+# PARSERS = PLUGIN.parsers
+# DAEMONS = PLUGIN.daemons
+# MODELS = PLUGIN.models
+# CHECKS = PLUGIN.checks

archivebox/builtin_plugins/ytdlp/apps.py

@@ -0,0 +1,48 @@
+import sys
+from pathlib import Path
+from typing import List, Dict, Optional
+from pydantic import InstanceOf, Field
+
+from django.apps import AppConfig
+
+from pydantic_pkgr import BinProvider, BinName, PATHStr
+from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
+from plugantic.base_configset import ConfigSectionName
+
+from pkg.settings import env, apt, brew
+
+from builtin_plugins.pip.apps import pip
+
+###################### Config ##########################
+
+
+class YtdlpDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    USE_YTDLP: bool = True
+
+    YTDLP_BINARY: str = Field(default='yt-dlp')
+
+DEFAULT_GLOBAL_CONFIG = {}
+YTDLP_CONFIG = YtdlpDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+
+
+class YtdlpBinary(BaseBinary):
+    name: BinName = YTDLP_CONFIG.YTDLP_BINARY
+    binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew]
+
+YTDLP_BINARY = YtdlpBinary()
+
+
+class YtdlpPlugin(BasePlugin):
+    name: str = 'builtin_plugins.ytdlp'
+    app_label: str = 'ytdlp'
+    verbose_name: str = 'YTDLP'
+
+    configs: List[InstanceOf[BaseConfigSet]] = [YTDLP_CONFIG]
+    binaries: List[InstanceOf[BaseBinary]] = [YTDLP_BINARY]
+
+
+PLUGIN = YtdlpPlugin()
+DJANGO_APP = PLUGIN.AppConfig