move abx plugins inside vendor dir

archivebox/__init__.py

@@ -15,7 +15,7 @@ import os
 import sys
 
 from pathlib import Path
-
+from typing import cast
 ASCII_LOGO = """
  █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
 ██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
@@ -52,6 +52,50 @@ load_vendored_libs()
 # print('DONE LOADING VENDORED LIBRARIES')
 
 
+import abx                                       # noqa
+import abx_spec_archivebox                       # noqa
+import abx_spec_config                            # noqa
+import abx_spec_pydantic_pkgr                    # noqa
+import abx_spec_django                            # noqa
+import abx_spec_searchbackend                     # noqa
+
+
+abx.pm.add_hookspecs(abx_spec_config.PLUGIN_SPEC)
+abx.pm.register(abx_spec_config.PLUGIN_SPEC())
+
+abx.pm.add_hookspecs(abx_spec_pydantic_pkgr.PLUGIN_SPEC)
+abx.pm.register(abx_spec_pydantic_pkgr.PLUGIN_SPEC())
+
+abx.pm.add_hookspecs(abx_spec_django.PLUGIN_SPEC)
+abx.pm.register(abx_spec_django.PLUGIN_SPEC())
+
+abx.pm.add_hookspecs(abx_spec_searchbackend.PLUGIN_SPEC)
+abx.pm.register(abx_spec_searchbackend.PLUGIN_SPEC())
+
+
+abx.pm = cast(abx.ABXPluginManager[abx_spec_archivebox.ArchiveBoxPluginSpec], abx.pm)
+pm = abx.pm
+
+
+# Load all installed ABX-compatible plugins
+ABX_ECOSYSTEM_PLUGINS = abx.get_pip_installed_plugins(group='abx')
+# Load all ArchiveBox-specific plugins
+ARCHIVEBOX_BUILTIN_PLUGINS = {
+    'config': PACKAGE_DIR / 'config',
+    'core': PACKAGE_DIR / 'core',
+    # 'search': PACKAGE_DIR / 'search',
+    # 'core': PACKAGE_DIR / 'core',
+}
+# Load all user-defined ArchiveBox plugins
+USER_PLUGINS = abx.find_plugins_in_dir(Path(os.getcwd()) / 'user_plugins')
+# Merge all plugins together
+ALL_PLUGINS = {**ABX_ECOSYSTEM_PLUGINS, **ARCHIVEBOX_BUILTIN_PLUGINS, **USER_PLUGINS}
+
+
+# Load ArchiveBox plugins
+LOADED_PLUGINS = abx.load_plugins(ALL_PLUGINS)
+
+
 from .config.constants import CONSTANTS                         # noqa
 from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR    # noqa
 from .config.version import VERSION                             # noqa

archivebox/config/__init__.py

@@ -1,4 +1,5 @@
-__package__ = 'archivebox.config'
+__package__ = 'config'
+__order__ = 200
 
 from .paths import (
     PACKAGE_DIR,                                    # noqa
@@ -9,30 +10,3 @@ from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHI
 from .version import VERSION                        # noqa
 
 
-import abx
-
-
-# @abx.hookimpl
-# def get_INSTALLED_APPS():
-#     return ['config']
-
-
-@abx.hookimpl
-def get_CONFIG():
-    from .common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-    return {
-        'SHELL_CONFIG': SHELL_CONFIG,
-        'STORAGE_CONFIG': STORAGE_CONFIG,
-        'GENERAL_CONFIG': GENERAL_CONFIG,
-        'SERVER_CONFIG': SERVER_CONFIG,
-        'ARCHIVING_CONFIG': ARCHIVING_CONFIG,
-        'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
-    }
-

archivebox/config/collection.py

@@ -9,6 +9,8 @@ from configparser import ConfigParser
 
 from benedict import benedict
 
+import archivebox
+
 from archivebox.config.constants import CONSTANTS
 
 from archivebox.misc.logging import stderr
@@ -16,9 +18,9 @@ from archivebox.misc.logging import stderr
 
 def get_real_name(key: str) -> str:
     """get the up-to-date canonical name for a given old alias or current key"""
-    from django.conf import settings
+    CONFIGS = archivebox.pm.hook.get_CONFIGS()
     
-    for section in settings.CONFIGS.values():
+    for section in CONFIGS.values():
         try:
             return section.aliases[key]
         except KeyError:
@@ -115,17 +117,15 @@ def load_config_file() -> Optional[benedict]:
 
 
 def section_for_key(key: str) -> Any:
-    from django.conf import settings
+    for config_section in archivebox.pm.hook.get_CONFIGS().values():
-    for config_section in settings.CONFIGS.values():
         if hasattr(config_section, key):
             return config_section
-    return None
+    raise ValueError(f'No config section found for key: {key}')
 
 
 def write_config_file(config: Dict[str, str]) -> benedict:
     """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
 
-    import abx.archivebox.reads
     from archivebox.misc.system import atomic_write
 
     CONFIG_HEADER = (
@@ -175,7 +175,7 @@ def write_config_file(config: Dict[str, str]) -> benedict:
     updated_config = {}
     try:
         # validate the updated_config by attempting to re-parse it
-        updated_config = {**load_all_config(), **abx.archivebox.reads.get_FLAT_CONFIG()}
+        updated_config = {**load_all_config(), **archivebox.pm.hook.get_FLAT_CONFIG()}
     except BaseException:                                                       # lgtm [py/catch-base-exception]
         # something went horribly wrong, revert to the previous version
         with open(f'{config_path}.bak', 'r', encoding='utf-8') as old:
@@ -233,11 +233,11 @@ def load_config(defaults: Dict[str, Any],
     return benedict(extended_config)
 
 def load_all_config():
-    import abx.archivebox.reads
+    import abx
     
     flat_config = benedict()
     
-    for config_section in abx.archivebox.reads.get_CONFIGS().values():
+    for config_section in abx.pm.hook.get_CONFIGS().values():
         config_section.__init__()
         flat_config.update(config_section.model_dump())
         

archivebox/config/common.py

@@ -7,10 +7,10 @@ from typing import Dict, Optional, List
 from pathlib import Path
 
 from rich import print
-from pydantic import Field, field_validator, computed_field
+from pydantic import Field, field_validator
 from django.utils.crypto import get_random_string
 
-from abx.archivebox.base_configset import BaseConfigSet
+from abx_spec_config.base_configset import BaseConfigSet
 
 from .constants import CONSTANTS
 from .version import get_COMMIT_HASH, get_BUILD_TIME
@@ -31,22 +31,19 @@ class ShellConfig(BaseConfigSet):
 
     ANSI: Dict[str, str]                = Field(default=lambda c: CONSTANTS.DEFAULT_CLI_COLORS if c.USE_COLOR else CONSTANTS.DISABLED_CLI_COLORS)
 
-    VERSIONS_AVAILABLE: bool = False             # .check_for_update.get_versions_available_on_github(c)},
+    # VERSIONS_AVAILABLE: bool = False             # .check_for_update.get_versions_available_on_github(c)},
-    CAN_UPGRADE: bool = False                    # .check_for_update.can_upgrade(c)},
+    # CAN_UPGRADE: bool = False                    # .check_for_update.can_upgrade(c)},
 
-    @computed_field
     @property
     def TERM_WIDTH(self) -> int:
         if not self.IS_TTY:
             return 200
         return shutil.get_terminal_size((140, 10)).columns
     
-    @computed_field
     @property
     def COMMIT_HASH(self) -> Optional[str]:
         return get_COMMIT_HASH()
     
-    @computed_field
     @property
     def BUILD_TIME(self) -> str:
         return get_BUILD_TIME()

archivebox/config/django.py

@@ -97,7 +97,7 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
                 except Exception as e:
                     bump_startup_progress_bar(advance=1000)
                     
-                    is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ('help', 'version', '--help', '--version', 'init'))
+                    is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ('help', 'version', '--help', '--version'))
                     if not is_using_meta_cmd:
                         # show error message to user only if they're not running a meta command / just trying to get help
                         STDERR.print()

archivebox/config/views.py

@@ -14,8 +14,8 @@ from django.utils.html import format_html, mark_safe
 from admin_data_views.typing import TableContext, ItemContext
 from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
 
-import abx.archivebox.reads
+import abx
-
+import archivebox
 from archivebox.config import CONSTANTS
 from archivebox.misc.util import parse_date
 
@@ -65,7 +65,7 @@ def obj_to_yaml(obj: Any, indent: int=0) -> str:
 
 @render_with_table_view
 def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
-
+    FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
     assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
 
     rows = {
@@ -81,12 +81,11 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     relevant_configs = {
         key: val
-        for key, val in settings.FLAT_CONFIG.items()
+        for key, val in FLAT_CONFIG.items()
         if '_BINARY' in key or '_VERSION' in key
     }
 
-    for plugin_id, plugin in abx.archivebox.reads.get_PLUGINS().items():
+    for plugin_id, plugin in abx.get_all_plugins().items():
-        plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
         if not plugin.hooks.get('get_BINARIES'):
             continue
         
@@ -131,17 +130,16 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 @render_with_item_view
 def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
 
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
+    assert request.user and request.user.is_superuser, 'Must be a superuser to view configuration settings.'
 
     binary = None
     plugin = None
-    for plugin_id in abx.archivebox.reads.get_PLUGINS().keys():
+    for plugin_id, plugin in abx.get_all_plugins().items():
-        loaded_plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
         try:
-            for loaded_binary in loaded_plugin.hooks.get_BINARIES().values():
+            for loaded_binary in plugin['hooks'].get_BINARIES().values():
                 if loaded_binary.name == key:
                     binary = loaded_binary
-                    plugin = loaded_plugin
+                    plugin = plugin
                     # break  # last write wins
         except Exception as e:
             print(e)
@@ -161,7 +159,7 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
                 "name": binary.name,
                 "description": binary.abspath,
                 "fields": {
-                    'plugin': plugin.package,
+                    'plugin': plugin['package'],
                     'binprovider': binary.loaded_binprovider,
                     'abspath': binary.loaded_abspath,
                     'version': binary.loaded_version,
@@ -215,9 +213,7 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 return color
         return 'black'
 
-    for plugin_id in settings.PLUGINS.keys():
+    for plugin_id, plugin in abx.get_all_plugins().items():
-        
-        plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
         plugin.hooks.get_BINPROVIDERS = plugin.hooks.get('get_BINPROVIDERS', lambda: {})
         plugin.hooks.get_BINARIES = plugin.hooks.get('get_BINARIES', lambda: {})
         plugin.hooks.get_CONFIG = plugin.hooks.get('get_CONFIG', lambda: {})
@@ -263,7 +259,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
 
     assert plugin_id, f'Could not find a plugin matching the specified name: {key}'
 
-    plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
+    plugin = abx.get_plugin(plugin_id)
 
     return ItemContext(
         slug=key,

archivebox/core/__init__.py

@@ -1,2 +1,31 @@
 __package__ = 'archivebox.core'
 
+import abx
+
+@abx.hookimpl
+def register_admin(admin_site):
+    """Register the core.models views (Snapshot, ArchiveResult, Tag, etc.) with the admin site"""
+    from core.admin import register_admin
+    register_admin(admin_site)
+
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from archivebox.config.common import (
+        SHELL_CONFIG,
+        STORAGE_CONFIG,
+        GENERAL_CONFIG,
+        SERVER_CONFIG,
+        ARCHIVING_CONFIG,
+        SEARCH_BACKEND_CONFIG,
+    )
+    return {
+        'SHELL_CONFIG': SHELL_CONFIG,
+        'STORAGE_CONFIG': STORAGE_CONFIG,
+        'GENERAL_CONFIG': GENERAL_CONFIG,
+        'SERVER_CONFIG': SERVER_CONFIG,
+        'ARCHIVING_CONFIG': ARCHIVING_CONFIG,
+        'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
+    }
+

archivebox/core/admin_site.py

@@ -2,7 +2,7 @@ __package__ = 'archivebox.core'
 
 from django.contrib import admin
 
-import abx.django.use
+import archivebox
 
 class ArchiveBoxAdmin(admin.AdminSite):
     site_header = 'ArchiveBox'
@@ -37,6 +37,6 @@ def register_admin_site():
     sites.site = archivebox_admin
     
     # register all plugins admin classes
-    abx.django.use.register_admin(archivebox_admin)
+    archivebox.pm.hook.register_admin(admin_site=archivebox_admin)
     
     return archivebox_admin

archivebox/core/apps.py

@@ -2,7 +2,7 @@ __package__ = 'archivebox.core'
 
 from django.apps import AppConfig
 
-import abx
+import archivebox
 
 
 class CoreConfig(AppConfig):
@@ -10,16 +10,11 @@ class CoreConfig(AppConfig):
 
     def ready(self):
         """Register the archivebox.core.admin_site as the main django admin site"""
+        from django.conf import settings
+        archivebox.pm.hook.ready(settings=settings)
+        
         from core.admin_site import register_admin_site
         register_admin_site()
         
-        abx.pm.hook.ready()
 
 
-
-
-@abx.hookimpl
-def register_admin(admin_site):
-    """Register the core.models views (Snapshot, ArchiveResult, Tag, etc.) with the admin site"""
-    from core.admin import register_admin
-    register_admin(admin_site)

archivebox/core/settings.py

@@ -9,10 +9,12 @@ from pathlib import Path
 from django.utils.crypto import get_random_string
 
 import abx
+import archivebox
 
-from archivebox.config import DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS
+from archivebox.config import DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS  # noqa
 from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG      # noqa
 
+
 IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
 IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
 IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
@@ -22,24 +24,8 @@ IS_GETTING_VERSION_OR_HELP = 'version' in sys.argv or 'help' in sys.argv or '--v
 ### ArchiveBox Plugin Settings
 ################################################################################
 
-PLUGIN_HOOKSPECS = [
+ALL_PLUGINS = archivebox.ALL_PLUGINS
-    'abx_spec_django',
+LOADED_PLUGINS = archivebox.LOADED_PLUGINS
-    'abx_spec_pydantic_pkgr',
-    'abx_spec_config',
-    'abx_spec_archivebox',
-]
-abx.register_hookspecs(PLUGIN_HOOKSPECS)
-
-SYSTEM_PLUGINS = abx.get_pip_installed_plugins(group='abx')
-USER_PLUGINS = abx.find_plugins_in_dir(DATA_DIR / 'user_plugins')
-
-ALL_PLUGINS = {**SYSTEM_PLUGINS, **USER_PLUGINS}
-
-# Load ArchiveBox plugins
-abx.load_plugins(ALL_PLUGINS)
-
-# # Load ArchiveBox config from plugins
-
 
 ################################################################################
 ### Django Core Settings
@@ -101,6 +87,7 @@ INSTALLED_APPS = [
 
 
 
+
 MIDDLEWARE = [
     'core.middleware.TimezoneMiddleware',
     'django.middleware.security.SecurityMiddleware',

archivebox/core/settings_logging.py

@@ -163,11 +163,6 @@ SETTINGS_LOGGING = {
             "level": "DEBUG",
             "propagate": False,
         },
-        "plugins_extractor": {
-            "handlers": ["default", "logfile"],
-            "level": "DEBUG",
-            "propagate": False,
-        },
         "httpx": {
             "handlers": ["outbound_webhooks"],
             "level": "INFO",

archivebox/core/views.py

@@ -21,6 +21,7 @@ from django.utils.decorators import method_decorator
 from admin_data_views.typing import TableContext, ItemContext
 from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
 
+import archivebox
 
 from core.models import Snapshot
 from core.forms import AddLinkForm
@@ -32,9 +33,8 @@ from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG
 from archivebox.misc.util import base_url, htmlencode, ts_to_date_str
 from archivebox.misc.serve_static import serve_static_with_byterange_support
 
-from ..plugins_extractor.archivedotorg.config import ARCHIVEDOTORG_CONFIG
+from archivebox.logging_util import printable_filesize
-from ..logging_util import printable_filesize
+from archivebox.search import query_search_index
-from ..search import query_search_index
 
 
 class HomepageView(View):
@@ -154,7 +154,7 @@ class SnapshotView(View):
             'status_color': 'success' if link.is_archived else 'danger',
             'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
             'warc_path': warc_path,
-            'SAVE_ARCHIVE_DOT_ORG': ARCHIVEDOTORG_CONFIG.SAVE_ARCHIVE_DOT_ORG,
+            'SAVE_ARCHIVE_DOT_ORG': archivebox.pm.hook.get_FLAT_CONFIG().SAVE_ARCHIVE_DOT_ORG,
             'PREVIEW_ORIGINALS': SERVER_CONFIG.PREVIEW_ORIGINALS,
             'archiveresults': sorted(archiveresults.values(), key=lambda r: all_types.index(r['name']) if r['name'] in all_types else -r['size']),
             'best_result': best_result,
@@ -500,21 +500,25 @@ class HealthCheckView(View):
 
 
 def find_config_section(key: str) -> str:
+    CONFIGS = archivebox.pm.hook.get_CONFIGS()
+    
     if key in CONSTANTS_CONFIG:
         return 'CONSTANT'
     matching_sections = [
-        section_id for section_id, section in settings.CONFIGS.items() if key in section.model_fields
+        section_id for section_id, section in CONFIGS.items() if key in section.model_fields
     ]
     section = matching_sections[0] if matching_sections else 'DYNAMIC'
     return section
 
 def find_config_default(key: str) -> str:
+    CONFIGS = archivebox.pm.hook.get_CONFIGS()
+    
     if key in CONSTANTS_CONFIG:
         return str(CONSTANTS_CONFIG[key])
     
     default_val = None
 
-    for config in settings.CONFIGS.values():
+    for config in CONFIGS.values():
         if key in config.model_fields:
             default_val = config.model_fields[key].default
             break
@@ -530,7 +534,9 @@ def find_config_default(key: str) -> str:
     return default_val
 
 def find_config_type(key: str) -> str:
-    for config in settings.CONFIGS.values():
+    CONFIGS = archivebox.pm.hook.get_CONFIGS()
+    
+    for config in CONFIGS.values():
         if hasattr(config, key):
             type_hints = get_type_hints(config)
             try:
@@ -547,6 +553,7 @@ def key_is_safe(key: str) -> bool:
 
 @render_with_table_view
 def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
+    CONFIGS = archivebox.pm.hook.get_CONFIGS()
     
     assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
 
@@ -560,7 +567,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
         # "Aliases": [],
     }
 
-    for section_id, section in reversed(list(settings.CONFIGS.items())):
+    for section_id, section in reversed(list(CONFIGS.items())):
         for key, field in section.model_fields.items():
             rows['Section'].append(section_id)   # section.replace('_', ' ').title().replace(' Config', '')
             rows['Key'].append(ItemLink(key, key=key))
@@ -570,7 +577,6 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
             # rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
             # rows['Aliases'].append(', '.join(find_config_aliases(key)))
 
-   
     section = 'CONSTANT'
     for key in CONSTANTS_CONFIG.keys():
         rows['Section'].append(section)   # section.replace('_', ' ').title().replace(' Config', '')
@@ -589,6 +595,8 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
 @render_with_item_view
 def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
+    CONFIGS = archivebox.pm.hook.get_CONFIGS()
+    FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
     
     assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
 
@@ -597,7 +605,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
 
     if key in CONSTANTS_CONFIG:
         section_header = mark_safe(f'[CONSTANTS]   &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>')
-    elif key in settings.FLAT_CONFIG:
+    elif key in FLAT_CONFIG:
         section_header = mark_safe(f'data / ArchiveBox.conf &nbsp; [{find_config_section(key)}]  &nbsp; <b><code style="color: lightgray">{key}</code></b>')
     else:
         section_header = mark_safe(f'[DYNAMIC CONFIG]   &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, calculated at runtime)</small>')
@@ -613,7 +621,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
                 "fields": {
                     'Key': key,
                     'Type': find_config_type(key),
-                    'Value': settings.FLAT_CONFIG.get(key, settings.CONFIGS.get(key, None)) if key_is_safe(key) else '********',
+                    'Value': FLAT_CONFIG.get(key, CONFIGS.get(key, None)) if key_is_safe(key) else '********',
                 },
                 "help_texts": {
                     'Key': mark_safe(f'''
@@ -635,13 +643,13 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
                             <code>{find_config_default(key) or '↗️ See in ArchiveBox source code...'}</code>
                         </a>
                         <br/><br/>
-                        <p style="display: {"block" if key in settings.FLAT_CONFIG else "none"}">
+                        <p style="display: {"block" if key in FLAT_CONFIG else "none"}">
                             <i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
                             <br/><br/>
                             <code>archivebox config --set {key}="{
                                 val.strip("'")
                                 if (val := find_config_default(key)) else
-                                (repr(settings.FLAT_CONFIG[key] if key_is_safe(key) else '********')).strip("'")
+                                (repr(FLAT_CONFIG[key] if key_is_safe(key) else '********')).strip("'")
                             }"</code>
                         </p>
                     '''),

archivebox/extractors/__init__.py

@@ -27,43 +27,29 @@ from ..logging_util import (
     log_archive_method_finished,
 )
 
-from .title import should_save_title, save_title
-from .favicon import should_save_favicon, save_favicon
-from .wget import should_save_wget, save_wget
-from .singlefile import should_save_singlefile, save_singlefile
-from .readability import should_save_readability, save_readability
-from .mercury import should_save_mercury, save_mercury
-from .htmltotext import should_save_htmltotext, save_htmltotext
-from .pdf import should_save_pdf, save_pdf
-from .screenshot import should_save_screenshot, save_screenshot
-from .dom import should_save_dom, save_dom
-from .git import should_save_git, save_git
-from .media import should_save_media, save_media
-from .archive_org import should_save_archive_dot_org, save_archive_dot_org
-from .headers import should_save_headers, save_headers
-
 
 ShouldSaveFunction = Callable[[Link, Optional[Path], Optional[bool]], bool]
 SaveFunction = Callable[[Link, Optional[Path], int], ArchiveResult]
 ArchiveMethodEntry = tuple[str, ShouldSaveFunction, SaveFunction]
 
 def get_default_archive_methods() -> List[ArchiveMethodEntry]:
+    # TODO: move to abx.pm.hook.get_EXTRACTORS()
     return [
-        ('favicon', should_save_favicon, save_favicon),
+        # ('favicon', should_save_favicon, save_favicon),
-        ('headers', should_save_headers, save_headers),
+        # ('headers', should_save_headers, save_headers),
-        ('singlefile', should_save_singlefile, save_singlefile),
+        # ('singlefile', should_save_singlefile, save_singlefile),
-        ('pdf', should_save_pdf, save_pdf),
+        # ('pdf', should_save_pdf, save_pdf),
-        ('screenshot', should_save_screenshot, save_screenshot),
+        # ('screenshot', should_save_screenshot, save_screenshot),
-        ('dom', should_save_dom, save_dom),
+        # ('dom', should_save_dom, save_dom),
-        ('wget', should_save_wget, save_wget),
+        # ('wget', should_save_wget, save_wget),
-        # keep title, readability, and htmltotext below wget and singlefile, as they depend on them
+        # # keep title, readability, and htmltotext below wget and singlefile, as they depend on them
-        ('title', should_save_title, save_title),
+        # ('title', should_save_title, save_title),
-        ('readability', should_save_readability, save_readability),
+        # ('readability', should_save_readability, save_readability),
-        ('mercury', should_save_mercury, save_mercury),
+        # ('mercury', should_save_mercury, save_mercury),
-        ('htmltotext', should_save_htmltotext, save_htmltotext),
+        # ('htmltotext', should_save_htmltotext, save_htmltotext),
-        ('git', should_save_git, save_git),
+        # ('git', should_save_git, save_git),
-        ('media', should_save_media, save_media),
+        # ('media', should_save_media, save_media),
-        ('archive_org', should_save_archive_dot_org, save_archive_dot_org),
+        # ('archive_org', should_save_archive_dot_org, save_archive_dot_org),
     ]
 
 ARCHIVE_METHODS_INDEXING_PRECEDENCE = [

archivebox/index/html.py

@@ -8,6 +8,8 @@ from typing import List, Optional, Iterator, Mapping
 from django.utils.html import format_html, mark_safe   # type: ignore
 from django.core.cache import cache
 
+import abx
+
 from archivebox.misc.system import atomic_write
 from archivebox.misc.util import (
     enforce_types,
@@ -19,7 +21,6 @@ from archivebox.misc.util import (
 from archivebox.config import CONSTANTS, DATA_DIR, VERSION
 from archivebox.config.common import SERVER_CONFIG
 from archivebox.config.version import get_COMMIT_HASH
-from archivebox.plugins_extractor.archivedotorg.config import ARCHIVEDOTORG_CONFIG
 
 from .schema import Link
 from ..logging_util import printable_filesize
@@ -80,7 +81,9 @@ def write_html_link_details(link: Link, out_dir: Optional[str]=None) -> None:
 @enforce_types
 def link_details_template(link: Link) -> str:
     
-    from ..extractors.wget import wget_output_path
+    from abx_plugin_wget_extractor.wget import wget_output_path
+    
+    SAVE_ARCHIVE_DOT_ORG = abx.pm.hook.get_FLAT_CONFIG().SAVE_ARCHIVE_DOT_ORG
 
     link_info = link._asdict(extended=True)
 
@@ -102,7 +105,7 @@ def link_details_template(link: Link) -> str:
         'status': 'archived' if link.is_archived else 'not yet archived',
         'status_color': 'success' if link.is_archived else 'danger',
         'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
-        'SAVE_ARCHIVE_DOT_ORG': ARCHIVEDOTORG_CONFIG.SAVE_ARCHIVE_DOT_ORG,
+        'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
         'PREVIEW_ORIGINALS': SERVER_CONFIG.PREVIEW_ORIGINALS,
     })
 

archivebox/index/json.py

@@ -8,7 +8,7 @@ from pathlib import Path
 from datetime import datetime, timezone
 from typing import List, Optional, Iterator, Any, Union
 
-import abx.archivebox.reads
+import abx
 
 from archivebox.config import VERSION, DATA_DIR, CONSTANTS
 from archivebox.config.common import SERVER_CONFIG, SHELL_CONFIG
@@ -33,7 +33,7 @@ def generate_json_index_from_links(links: List[Link], with_headers: bool):
             'docs': 'https://github.com/ArchiveBox/ArchiveBox/wiki',
             'source': 'https://github.com/ArchiveBox/ArchiveBox',
             'issues': 'https://github.com/ArchiveBox/ArchiveBox/issues',
-            'dependencies': dict(abx.archivebox.reads.get_BINARIES()),
+            'dependencies': dict(abx.pm.hook.get_BINARIES()),
         },
     }
     

archivebox/index/schema.py

@@ -17,9 +17,9 @@ from dataclasses import dataclass, asdict, field, fields
 
 from django.utils.functional import cached_property
 
-from archivebox.config import ARCHIVE_DIR, CONSTANTS
+import abx
 
-from plugins_extractor.favicon.config import FAVICON_CONFIG
+from archivebox.config import ARCHIVE_DIR, CONSTANTS
 
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.util import ts_to_date_str, parse_date
@@ -426,7 +426,10 @@ class Link:
     def canonical_outputs(self) -> Dict[str, Optional[str]]:
         """predict the expected output paths that should be present after archiving"""
 
-        from ..extractors.wget import wget_output_path
+        from abx_plugin_wget.wget import wget_output_path
+        
+        FAVICON_CONFIG = abx.pm.hook.get_CONFIGS().favicon
+        
         # TODO: banish this awful duplication from the codebase and import these
         # from their respective extractor files
         canonical = {

archivebox/machine/models.py

@@ -8,9 +8,10 @@ from django.db import models
 from django.utils import timezone
 from django.utils.functional import cached_property
 
-import abx.archivebox.reads
+import abx
+import archivebox
 
-from abx.archivebox.base_binary import BaseBinary, BaseBinProvider
+from pydantic_pkgr import Binary, BinProvider
 from archivebox.abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithHealthStats
 
 from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
@@ -180,7 +181,7 @@ class NetworkInterface(ABIDModel, ModelWithHealthStats):
 
 
 class InstalledBinaryManager(models.Manager):
-    def get_from_db_or_cache(self, binary: BaseBinary) -> 'InstalledBinary':
+    def get_from_db_or_cache(self, binary: Binary) -> 'InstalledBinary':
         """Get or create an InstalledBinary record for a Binary on the local machine"""
         
         global _CURRENT_BINARIES
@@ -216,7 +217,7 @@ class InstalledBinaryManager(models.Manager):
             # if binary was not yet loaded from filesystem, do it now
             # this is expensive, we have to find it's abspath, version, and sha256, but it's necessary
             # to make sure we have a good, up-to-date record of it in the DB & in-memroy cache
-            binary = binary.load(fresh=True)
+            binary = archivebox.pm.hook.binary_load(binary=binary, fresh=True)
 
         assert binary.loaded_binprovider and binary.loaded_abspath and binary.loaded_version and binary.loaded_sha256, f'Failed to load binary {binary.name} abspath, version, and sha256'
         
@@ -291,8 +292,8 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
         if not hasattr(self, 'machine'):
             self.machine = Machine.objects.current()
         if not self.binprovider:
-            all_known_binproviders = list(abx.archivebox.reads.get_BINPROVIDERS().values())
+            all_known_binproviders = list(abx.as_dict(archivebox.pm.hook.get_BINPROVIDERS()).values())
-            binary = BaseBinary(name=self.name, binproviders=all_known_binproviders).load(fresh=True)
+            binary = archivebox.pm.hook.binary_load(binary=Binary(name=self.name, binproviders=all_known_binproviders), fresh=True)
             self.binprovider = binary.loaded_binprovider.name if binary.loaded_binprovider else None
         if not self.abspath:
             self.abspath = self.BINPROVIDER.get_abspath(self.name)
@@ -304,16 +305,16 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
         super().clean(*args, **kwargs)
 
     @cached_property
-    def BINARY(self) -> BaseBinary:
+    def BINARY(self) -> Binary:
-        for binary in abx.archivebox.reads.get_BINARIES().values():
+        for binary in abx.as_dict(archivebox.pm.hook.get_BINARIES()).values():
             if binary.name == self.name:
                 return binary
         raise Exception(f'Orphaned InstalledBinary {self.name} {self.binprovider} was found in DB, could not find any plugin that defines it')
         # TODO: we could technically reconstruct it from scratch, but why would we ever want to do that?
 
     @cached_property
-    def BINPROVIDER(self) -> BaseBinProvider:
+    def BINPROVIDER(self) -> BinProvider:
-        for binprovider in abx.archivebox.reads.get_BINPROVIDERS().values():
+        for binprovider in abx.as_dict(archivebox.pm.hook.get_BINPROVIDERS()).values():
             if binprovider.name == self.binprovider:
                 return binprovider
         raise Exception(f'Orphaned InstalledBinary(name={self.name}) was found in DB, could not find any plugin that defines BinProvider(name={self.binprovider})')
@@ -321,7 +322,7 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
     # maybe not a good idea to provide this? Binary in DB is a record of the binary's config
     # whereas a loaded binary is a not-yet saved instance that may not have the same config
     # why would we want to load a binary record from the db when it could be freshly loaded?
-    def load_from_db(self) -> BaseBinary:
+    def load_from_db(self) -> Binary:
         # TODO: implement defaults arg in pydantic_pkgr
         # return self.BINARY.load(defaults={
         #     'binprovider': self.BINPROVIDER,
@@ -330,7 +331,7 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
         #     'sha256': self.sha256,
         # })
         
-        return BaseBinary.model_validate({
+        return Binary.model_validate({
             **self.BINARY.model_dump(),
             'abspath': self.abspath and Path(self.abspath),
             'version': self.version,
@@ -340,5 +341,5 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
             'overrides': self.BINARY.overrides,
         })
 
-    def load_fresh(self) -> BaseBinary:
+    def load_fresh(self) -> Binary:
-        return self.BINARY.load(fresh=True)
+        return archivebox.pm.hook.binary_load(binary=self.BINARY, fresh=True)

archivebox/main.py

@@ -14,6 +14,10 @@ from crontab import CronTab, CronSlices
 from django.db.models import QuerySet
 from django.utils import timezone
 
+from pydantic_pkgr import Binary
+
+import abx
+import archivebox
 from archivebox.misc.checks import check_data_folder
 from archivebox.misc.util import enforce_types                         # type: ignore
 from archivebox.misc.system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT
@@ -197,13 +201,13 @@ def version(quiet: bool=False,
     
     from django.conf import settings
     
-    from abx.archivebox.base_binary import BaseBinary, apt, brew, env
+    from abx_plugin_default_binproviders import apt, brew, env
     
     from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
     from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID
     from archivebox.config.paths import get_data_locations, get_code_locations
     
-    from plugins_auth.ldap.config import LDAP_CONFIG
+    LDAP_ENABLED = archivebox.pm.hook.get_FLAT_CONFIG().LDAP_ENABLED
 
 
     # 0.7.1
@@ -242,7 +246,7 @@ def version(quiet: bool=False,
         f'SUDO={CONSTANTS.IS_ROOT}',
         f'ID={CONSTANTS.MACHINE_ID}:{CONSTANTS.COLLECTION_ID}',
         f'SEARCH_BACKEND={SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}',
-        f'LDAP={LDAP_CONFIG.LDAP_ENABLED}',
+        f'LDAP={LDAP_ENABLED}',
         #f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})',  # add this if we have more useful info to show eventually
     )
     prnt()
@@ -264,7 +268,8 @@ def version(quiet: bool=False,
 
     prnt('[pale_green1][i] Binary Dependencies:[/pale_green1]')
     failures = []
-    for name, binary in list(settings.BINARIES.items()):
+    BINARIES = abx.as_dict(archivebox.pm.hook.get_BINARIES())
+    for name, binary in list(BINARIES.items()):
         if binary.name == 'archivebox':
             continue
         
@@ -295,14 +300,15 @@ def version(quiet: bool=False,
             
     prnt()
     prnt('[gold3][i] Package Managers:[/gold3]')
-    for name, binprovider in list(settings.BINPROVIDERS.items()):
+    BINPROVIDERS = abx.as_dict(archivebox.pm.hook.get_BINPROVIDERS())
+    for name, binprovider in list(BINPROVIDERS.items()):
         err = None
         
         if binproviders and binprovider.name not in binproviders:
             continue
         
         # TODO: implement a BinProvider.BINARY() method that gets the loaded binary for a binprovider's INSTALLER_BIN
-        loaded_bin = binprovider.INSTALLER_BINARY or BaseBinary(name=binprovider.INSTALLER_BIN, binproviders=[env, apt, brew])
+        loaded_bin = binprovider.INSTALLER_BINARY or Binary(name=binprovider.INSTALLER_BIN, binproviders=[env, apt, brew])
         
         abspath = None
         if loaded_bin.abspath:
@@ -1050,10 +1056,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
     #    - recommend user re-run with sudo if any deps need to be installed as root
 
     from rich import print
-    from django.conf import settings
     
-    
-    import abx.archivebox.reads
     from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
     from archivebox.config.paths import get_or_create_working_lib_dir
 
@@ -1076,11 +1079,11 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
     
     package_manager_names = ', '.join(
         f'[yellow]{binprovider.name}[/yellow]'
-        for binprovider in reversed(list(abx.archivebox.reads.get_BINPROVIDERS().values()))
+        for binprovider in reversed(list(abx.as_dict(abx.pm.hook.get_BINPROVIDERS()).values()))
         if not binproviders or (binproviders and binprovider.name in binproviders)
     )
     print(f'[+] Setting up package managers {package_manager_names}...')
-    for binprovider in reversed(list(abx.archivebox.reads.get_BINPROVIDERS().values())):
+    for binprovider in reversed(list(abx.as_dict(abx.pm.hook.get_BINPROVIDERS()).values())):
         if binproviders and binprovider.name not in binproviders:
             continue
         try:
@@ -1093,7 +1096,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
     
     print()
     
-    for binary in reversed(list(abx.archivebox.reads.get_BINARIES().values())):
+    for binary in reversed(list(abx.as_dict(abx.pm.hook.get_BINARIES()).values())):
         if binary.name in ('archivebox', 'django', 'sqlite', 'python'):
             # obviously must already be installed if we are running
             continue
@@ -1123,7 +1126,8 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
                                 result = binary.install(binproviders=[binprovider_name], dry_run=dry_run).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
                                 sys.stderr.write("\033[00m\n")     # reset
                             else:
-                                result = binary.load_or_install(binproviders=[binprovider_name], fresh=True, dry_run=dry_run, quiet=False).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
+                                loaded_binary = archivebox.pm.hook.binary_load_or_install(binary=binary, binproviders=[binprovider_name], fresh=True, dry_run=dry_run, quiet=False)
+                                result = loaded_binary.model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
                             if result and result['loaded_version']:
                                 break
                         except Exception as e:
@@ -1134,7 +1138,8 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
                         binary.install(dry_run=dry_run).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
                         sys.stderr.write("\033[00m\n")  # reset
                     else:
-                        binary.load_or_install(fresh=True, dry_run=dry_run).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
+                        loaded_binary = archivebox.pm.hook.binary_load_or_install(binary=binary, fresh=True, dry_run=dry_run)
+                        result = loaded_binary.model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
             if IS_ROOT and LIB_DIR:
                 with SudoPermission(uid=0):
                     if ARCHIVEBOX_USER == 0:
@@ -1158,7 +1163,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
     
     print('\n[green][√] Set up ArchiveBox and its dependencies successfully.[/green]\n', file=sys.stderr)
     
-    from plugins_pkg.pip.binaries import ARCHIVEBOX_BINARY
+    from abx_plugin_pip.binaries import ARCHIVEBOX_BINARY
     
     extra_args = []
     if binproviders:
@@ -1184,8 +1189,6 @@ def config(config_options_str: Optional[str]=None,
            out_dir: Path=DATA_DIR) -> None:
     """Get and set your ArchiveBox project configuration values"""
 
-    import abx.archivebox.reads
-
     from rich import print
 
     check_data_folder()
@@ -1199,7 +1202,8 @@ def config(config_options_str: Optional[str]=None,
     elif config_options_str:
         config_options = config_options_str.split('\n')
 
-    from django.conf import settings
+    FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
+    CONFIGS = archivebox.pm.hook.get_CONFIGS()
     
     config_options = config_options or []
 
@@ -1209,8 +1213,8 @@ def config(config_options_str: Optional[str]=None,
     if search:
         if config_options:
             config_options = [get_real_name(key) for key in config_options]
-            matching_config = {key: settings.FLAT_CONFIG[key] for key in config_options if key in settings.FLAT_CONFIG}
+            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            for config_section in settings.CONFIGS.values():
+            for config_section in CONFIGS.values():
                 aliases = config_section.aliases
                 
                 for search_key in config_options:
@@ -1229,15 +1233,15 @@ def config(config_options_str: Optional[str]=None,
     elif get or no_args:
         if config_options:
             config_options = [get_real_name(key) for key in config_options]
-            matching_config = {key: settings.FLAT_CONFIG[key] for key in config_options if key in settings.FLAT_CONFIG}
+            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in settings.FLAT_CONFIG]
+            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
             if failed_config:
                 stderr()
                 stderr('[X] These options failed to get', color='red')
                 stderr('    {}'.format('\n    '.join(config_options)))
                 raise SystemExit(1)
         else:
-            matching_config = settings.FLAT_CONFIG
+            matching_config = FLAT_CONFIG
         
         print(printable_config(matching_config))
         raise SystemExit(not matching_config)
@@ -1258,20 +1262,20 @@ def config(config_options_str: Optional[str]=None,
             if key != raw_key:
                 stderr(f'[i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.', color='lightyellow')
 
-            if key in settings.FLAT_CONFIG:
+            if key in FLAT_CONFIG:
                 new_config[key] = val.strip()
             else:
                 failed_options.append(line)
 
         if new_config:
-            before = settings.FLAT_CONFIG
+            before = FLAT_CONFIG
             matching_config = write_config_file(new_config)
-            after = {**load_all_config(), **abx.archivebox.reads.get_FLAT_CONFIG()}
+            after = {**load_all_config(), **archivebox.pm.hook.get_FLAT_CONFIG()}
             print(printable_config(matching_config))
 
             side_effect_changes = {}
             for key, val in after.items():
-                if key in settings.FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
+                if key in FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
                     side_effect_changes[key] = after[key]
                     # import ipdb; ipdb.set_trace()
 
@@ -1313,7 +1317,7 @@ def schedule(add: bool=False,
     """Set ArchiveBox to regularly import URLs at specific times using cron"""
     
     check_data_folder()
-    from archivebox.plugins_pkg.pip.binaries import ARCHIVEBOX_BINARY
+    from abx_plugin_pip.binaries import ARCHIVEBOX_BINARY
     from archivebox.config.permissions import USER
 
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)

archivebox/misc/checks.py

@@ -201,6 +201,7 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True):
 
 
 def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_exist=True):
+    import archivebox
     from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
     from archivebox.misc.logging import STDERR
     from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir
@@ -209,6 +210,8 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
     
     lib_dir = lib_dir or STORAGE_CONFIG.LIB_DIR
     
+    assert lib_dir == archivebox.pm.hook.get_LIB_DIR(), "lib_dir is not the same as the one in the flat config"
+    
     if not must_exist and not os.path.isdir(lib_dir):
         return True
     

archivebox/misc/shell_welcome_message.py

@@ -23,7 +23,7 @@ from archivebox import CONSTANTS           # noqa
 from ..main import *                       # noqa
 from ..cli import CLI_SUBCOMMANDS
 
-CONFIG = settings.FLAT_CONFIG
+CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
 CLI_COMMAND_NAMES = ", ".join(CLI_SUBCOMMANDS.keys())
 
 if __name__ == '__main__':
@@ -55,6 +55,5 @@ if __name__ == '__main__':
     prnt('    add[blink][deep_sky_blue4]?[/deep_sky_blue4][/blink]                                                                        [grey53]# add ? after anything to get help[/]')
     prnt('    add("https://example.com/some/new/url")                                     [grey53]# call CLI methods from the shell[/]')
     prnt('    snap = Snapshot.objects.filter(url__contains="https://example.com").last()  [grey53]# query for individual snapshots[/]')
-    prnt('    archivebox.plugins_extractor.wget.apps.WGET_EXTRACTOR.extract(snap.id)      [grey53]# call an extractor directly[/]')
     prnt('    snap.archiveresult_set.all()                                                [grey53]# see extractor results[/]')
     prnt('    bool(re.compile(CONFIG.URL_DENYLIST).search("https://example.com/abc.exe")) [grey53]# test out a config change[/]')

archivebox/parsers/pocket_api.py

@@ -6,8 +6,7 @@ import re
 from typing import IO, Iterable, Optional
 from configparser import ConfigParser
 
-from pocket import Pocket
+import archivebox
-
 from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types
 from archivebox.misc.system import atomic_write
@@ -22,7 +21,7 @@ API_DB_PATH = CONSTANTS.SOURCES_DIR / 'pocket_api.db'
 _BROKEN_PROTOCOL_RE = re.compile('^(http[s]?)(:/(?!/))')
 
 
-def get_pocket_articles(api: Pocket, since=None, page=0):
+def get_pocket_articles(api, since=None, page=0):
     body, headers = api.get(
         state='archive',
         sort='oldest',
@@ -94,7 +93,9 @@ def should_parse_as_pocket_api(text: str) -> bool:
 def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse bookmarks from the Pocket API"""
 
-    from archivebox.plugins_extractor.pocket.config import POCKET_CONFIG
+    from pocket import Pocket
+
+    FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
 
     input_buffer.seek(0)
     pattern = re.compile(r"^pocket:\/\/(\w+)")
@@ -102,7 +103,7 @@ def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
         if should_parse_as_pocket_api(line):
             
             username = pattern.search(line).group(1)
-            api = Pocket(POCKET_CONFIG.POCKET_CONSUMER_KEY, POCKET_CONFIG.POCKET_ACCESS_TOKENS[username])
+            api = Pocket(FLAT_CONFIG.POCKET_CONSUMER_KEY, FLAT_CONFIG.POCKET_ACCESS_TOKENS[username])
             api.last_since = None
     
             for article in get_pocket_articles(api, since=read_since(username)):

archivebox/parsers/readwise_reader_api.py

@@ -8,9 +8,10 @@ from datetime import datetime
 from typing import IO, Iterable, Optional
 from configparser import ConfigParser
 
+import abx
+
 from archivebox.misc.util import enforce_types
 from archivebox.misc.system import atomic_write
-from archivebox.plugins_extractor.readwise.config import READWISE_CONFIG
 
 from ..index.schema import Link
 
@@ -62,26 +63,30 @@ def link_from_article(article: dict, sources: list):
 
 
 def write_cursor(username: str, since: str):
-    if not READWISE_CONFIG.READWISE_DB_PATH.exists():
+    READWISE_DB_PATH = abx.pm.hook.get_CONFIG().READWISE_DB_PATH
-        atomic_write(READWISE_CONFIG.READWISE_DB_PATH, "")
+    
+    if not READWISE_DB_PATH.exists():
+        atomic_write(READWISE_DB_PATH, "")
 
     since_file = ConfigParser()
     since_file.optionxform = str
-    since_file.read(READWISE_CONFIG.READWISE_DB_PATH)
+    since_file.read(READWISE_DB_PATH)
 
     since_file[username] = {"since": since}
 
-    with open(READWISE_CONFIG.READWISE_DB_PATH, "w+") as new:
+    with open(READWISE_DB_PATH, "w+") as new:
         since_file.write(new)
 
 
 def read_cursor(username: str) -> Optional[str]:
-    if not READWISE_CONFIG.READWISE_DB_PATH.exists():
+    READWISE_DB_PATH = abx.pm.hook.get_CONFIG().READWISE_DB_PATH
-        atomic_write(READWISE_CONFIG.READWISE_DB_PATH, "")
+    
+    if not READWISE_DB_PATH.exists():
+        atomic_write(READWISE_DB_PATH, "")
 
     config_file = ConfigParser()
     config_file.optionxform = str
-    config_file.read(READWISE_CONFIG.READWISE_DB_PATH)
+    config_file.read(READWISE_DB_PATH)
 
     return config_file.get(username, "since", fallback=None)
 
@@ -97,12 +102,14 @@ def should_parse_as_readwise_reader_api(text: str) -> bool:
 def parse_readwise_reader_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
     """Parse bookmarks from the Readwise Reader API"""
 
+    READWISE_READER_TOKENS = abx.pm.hook.get_CONFIG().READWISE_READER_TOKENS
+
     input_buffer.seek(0)
     pattern = re.compile(r"^readwise-reader:\/\/(\w+)")
     for line in input_buffer:
         if should_parse_as_readwise_reader_api(line):
             username = pattern.search(line).group(1)
-            api = ReadwiseReaderAPI(READWISE_CONFIG.READWISE_READER_TOKENS[username], cursor=read_cursor(username))
+            api = ReadwiseReaderAPI(READWISE_READER_TOKENS[username], cursor=read_cursor(username))
 
             for article in get_readwise_reader_articles(api):
                 yield link_from_article(article, sources=[line])

archivebox/search/__init__.py

@@ -6,8 +6,8 @@ from typing import List, Union
 from django.db.models import QuerySet
 from django.conf import settings
 
-import abx.archivebox.reads
+import abx
-
+import archivebox
 from archivebox.index.schema import Link
 from archivebox.misc.util import enforce_types
 from archivebox.misc.logging import stderr
@@ -57,7 +57,7 @@ def get_indexable_content(results: QuerySet):
 
 
 def import_backend():
-    for backend in abx.archivebox.reads.get_SEARCHBACKENDS().values():
+    for backend in abx.as_dict(archivebox.pm.hook.get_SEARCHBACKENDS()).values():
         if backend.name == SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE:
             return backend
     raise Exception(f'Could not load {SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE} as search backend')

archivebox/vendor/__init__.py

@@ -4,23 +4,27 @@ from pathlib import Path
 
 VENDOR_DIR = Path(__file__).parent
 
-VENDORED_LIBS = {
+VENDORED_LIBS = [
-    # sys.path dir:         library name
+    'abx',
-    #'python-atomicwrites':  'atomicwrites',
+    'pydantic-pkgr',
-    #'django-taggit':        'taggit',
+    'pocket',
-    # 'pydantic-pkgr':        'pydantic_pkgr',
+]
-    # 'pocket':               'pocket',
+
-    #'base32-crockford':     'base32_crockford',
+for subdir in reversed(sorted(VENDOR_DIR.iterdir())):
-}
+    if subdir.is_dir() and subdir.name not in VENDORED_LIBS and not subdir.name.startswith('_'):
+        VENDORED_LIBS.append(subdir.name)
 
 def load_vendored_libs():
-    for lib_subdir, lib_name in VENDORED_LIBS.items():
+    if str(VENDOR_DIR) not in sys.path:
-        lib_dir = VENDOR_DIR / lib_subdir
+        sys.path.append(str(VENDOR_DIR))
-        assert lib_dir.is_dir(), 'Expected vendor libary {lib_name} could not be found in {lib_dir}'
+    
+    for lib_name in VENDORED_LIBS:
+        lib_dir = VENDOR_DIR / lib_name
+        assert lib_dir.is_dir(), f'Expected vendor libary {lib_name} could not be found in {lib_dir}'
 
         try:
             lib = importlib.import_module(lib_name)
-            # print(f"Successfully imported lib from environment {lib_name}: {inspect.getfile(lib)}")
+            # print(f"Successfully imported lib from environment {lib_name}")
         except ImportError:
             sys.path.append(str(lib_dir))
             try:

archivebox/vendor/abx-plugin-archivedotorg/abx_plugin_archivedotorg/__init__.py

@@ -0,0 +1,21 @@
+__label__ = 'Archive.org'
+__homepage__ = 'https://archive.org'
+
+import abx
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import ARCHIVEDOTORG_CONFIG
+    
+    return {
+        'ARCHIVEDOTORG_CONFIG': ARCHIVEDOTORG_CONFIG
+    }
+
+
+# @abx.hookimpl
+# def get_EXTRACTORS():
+#     from .extractors import ARCHIVEDOTORG_EXTRACTOR
+#
+#     return {
+#         'archivedotorg': ARCHIVEDOTORG_EXTRACTOR,
+#     }

archivebox/vendor/abx-plugin-archivedotorg/abx_plugin_archivedotorg/config.py

@@ -1,7 +1,4 @@
-__package__ = 'plugins_extractor.archivedotorg'
+from abx_spec_config.base_configset import BaseConfigSet
-
-
-from abx.archivebox.base_configset import BaseConfigSet
 
 
 class ArchivedotorgConfig(BaseConfigSet):

archivebox/vendor/abx-plugin-archivedotorg/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "abx-plugin-archivedotorg"
+version = "2024.10.28"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "abx>=0.1.0",
+    "abx-spec-config>=0.1.0",
+    "abx-plugin-curl>=2024.10.24",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project.entry-points.abx]
+abx_plugin_archivedotorg = "abx_plugin_archivedotorg"

archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/__init__.py

@@ -0,0 +1,34 @@
+__label__ = 'Chrome'
+__author__ = 'ArchiveBox'
+
+import abx
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import CHROME_CONFIG
+    
+    return {
+        'CHROME_CONFIG': CHROME_CONFIG
+    }
+
+@abx.hookimpl
+def get_BINARIES():
+    from .binaries import CHROME_BINARY
+    
+    return {
+        'chrome': CHROME_BINARY,
+    }
+
+@abx.hookimpl
+def ready():
+    from .config import CHROME_CONFIG
+    CHROME_CONFIG.validate()
+
+
+# @abx.hookimpl
+# def get_EXTRACTORS():
+#     return {
+#         'pdf': PDF_EXTRACTOR,
+#         'screenshot': SCREENSHOT_EXTRACTOR,
+#         'dom': DOM_EXTRACTOR,
+#     }

archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/binaries.py

@@ -1,5 +1,3 @@
-__package__ = 'plugins_extractor.chrome'
-
 import os
 import platform
 from pathlib import Path
@@ -7,17 +5,18 @@ from typing import List, Optional
 
 from pydantic import InstanceOf
 from pydantic_pkgr import (
+    Binary,
     BinProvider,
     BinName,
     BinaryOverrides,
     bin_abspath,
 )
 
-import abx.archivebox.reads
+import abx
-from abx.archivebox.base_binary import BaseBinary, env, apt, brew
 
-from abx_puppeteer_binprovider.binproviders import PUPPETEER_BINPROVIDER
+from abx_plugin_default_binproviders import apt, brew, env
-from abx_playwright_binprovider.binproviders import PLAYWRIGHT_BINPROVIDER
+from abx_plugin_puppeteer.binproviders import PUPPETEER_BINPROVIDER
+from abx_plugin_playwright.binproviders import PLAYWRIGHT_BINPROVIDER
 
 
 from .config import CHROME_CONFIG
@@ -81,7 +80,7 @@ def create_macos_app_symlink(target: Path, shortcut: Path):
 ###################### Config ##########################
 
 
-class ChromeBinary(BaseBinary):
+class ChromeBinary(Binary):
     name: BinName = CHROME_CONFIG.CHROME_BINARY
     binproviders_supported: List[InstanceOf[BinProvider]] = [PUPPETEER_BINPROVIDER, env, PLAYWRIGHT_BINPROVIDER, apt, brew]
     
@@ -105,7 +104,7 @@ class ChromeBinary(BaseBinary):
 
     @staticmethod
     def symlink_to_lib(binary, bin_dir=None) -> None:
-        bin_dir = bin_dir or abx.archivebox.reads.get_CONFIGS().STORAGE_CONFIG.LIB_DIR / 'bin'
+        bin_dir = bin_dir or abx.pm.hook.get_BIN_DIR()
         
         if not (binary.abspath and os.path.isfile(binary.abspath)):
             return

archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/config.py

@@ -1,5 +1,3 @@
-__package__ = 'plugins_extractor.chrome'
-
 import os
 from pathlib import Path
 from typing import List, Optional
@@ -7,8 +5,8 @@ from typing import List, Optional
 from pydantic import Field
 from pydantic_pkgr import bin_abspath
 
-from abx.archivebox.base_configset import BaseConfigSet
+from abx_spec_config.base_configset import BaseConfigSet
-from abx.archivebox.base_binary import env
+from abx_plugin_default_binproviders import env
 
 from archivebox.config import CONSTANTS
 from archivebox.config.common import ARCHIVING_CONFIG, SHELL_CONFIG

archivebox/vendor/abx-plugin-chrome/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "abx-plugin-chrome"
+version = "2024.10.28"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "abx>=0.1.0",
+    "abx-spec-config>=0.1.0",
+    "abx-spec-pydantic-pkgr>=0.1.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project.entry-points.abx]
+abx_plugin_chrome = "abx_plugin_chrome"

archivebox/vendor/abx-plugin-curl/abx_plugin_curl/__init__.py

@@ -0,0 +1,18 @@
+import abx
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import CURL_CONFIG
+    
+    return {
+        'curl': CURL_CONFIG
+    }
+
+@abx.hookimpl
+def get_BINARIES():
+    from .binaries import CURL_BINARY
+    
+    return {
+        'curl': CURL_BINARY,
+    }

archivebox/vendor/abx-plugin-curl/abx_plugin_curl/binaries.py

@@ -1,17 +1,17 @@
-__package__ = 'plugins_extractor.curl'
+__package__ = 'abx_plugin_curl'
 
 from typing import List
 
 from pydantic import InstanceOf
-from pydantic_pkgr import BinProvider, BinName
+from pydantic_pkgr import BinProvider, BinName, Binary
 
-from abx.archivebox.base_binary import BaseBinary, env, apt, brew
+from abx_plugin_default_binproviders import apt, brew, env
 
 
 from .config import CURL_CONFIG
 
 
-class CurlBinary(BaseBinary):
+class CurlBinary(Binary):
     name: BinName = CURL_CONFIG.CURL_BINARY
     binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
 

archivebox/vendor/abx-plugin-curl/abx_plugin_curl/config.py

@@ -1,11 +1,11 @@
-__package__ = 'plugins_extractor.curl'
+__package__ = 'abx_plugin_curl'
 
 from typing import List, Optional
 from pathlib import Path
 
 from pydantic import Field
 
-from abx.archivebox.base_configset import BaseConfigSet
+from abx_spec_config.base_configset import BaseConfigSet
 
 from archivebox.config.common import ARCHIVING_CONFIG
 

archivebox/vendor/abx-plugin-curl/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "abx-plugin-curl"
+version = "2024.10.24"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "abx>=0.1.0",
+    "abx-spec-config>=0.1.0",
+    "abx-spec-pydantic-pkgr>=0.1.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project.entry-points.abx]
+abx_plugin_curl = "abx_plugin_curl"

archivebox/vendor/abx-plugin-default-binproviders/abx_plugin_default_binproviders.py

@@ -16,7 +16,6 @@ env = ENV_BINPROVIDER = EnvProvider()
 
 @abx.hookimpl(tryfirst=True)
 def get_BINPROVIDERS() -> Dict[str, BinProvider]:
-
     return {
         'apt': APT_BINPROVIDER,
         'brew': BREW_BINPROVIDER,

archivebox/vendor/abx-plugin-favicon/abx_plugin_favicon/__init__.py

@@ -0,0 +1,29 @@
+__label__ = 'Favicon'
+__version__ = '2024.10.24'
+__author__ = 'ArchiveBox'
+__homepage__ = 'https://github.com/ArchiveBox/archivebox'
+__dependencies__ = [
+    'abx>=0.1.0',
+    'abx-spec-config>=0.1.0',
+    'abx-plugin-curl-extractor>=2024.10.24',
+]
+
+import abx
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import FAVICON_CONFIG
+    
+    return {
+        'FAVICON_CONFIG': FAVICON_CONFIG
+    }
+
+
+# @abx.hookimpl
+# def get_EXTRACTORS():
+#     from .extractors import FAVICON_EXTRACTOR
+    
+#     return {
+#         'favicon': FAVICON_EXTRACTOR,
+#     }

archivebox/vendor/abx-plugin-favicon/abx_plugin_favicon/config.py

@@ -1,7 +1,4 @@
-__package__ = 'plugins_extractor.favicon'
+from abx_spec_config.base_configset import BaseConfigSet
-
-
-from abx.archivebox.base_configset import BaseConfigSet
 
 
 class FaviconConfig(BaseConfigSet):

archivebox/vendor/abx-plugin-favicon/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "abx-plugin-favicon"
+version = "2024.10.28"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "abx>=0.1.0",
+    "abx-spec-config>=0.1.0",
+    "abx-plugin-curl>=2024.10.28",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project.entry-points.abx]
+abx_plugin_favicon = "abx_plugin_favicon"

archivebox/vendor/abx-plugin-git/abx_plugin_git/__init__.py

@@ -0,0 +1,29 @@
+__package__ = 'abx_plugin_git'
+__label__ = 'Git'
+
+import abx
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import GIT_CONFIG
+    
+    return {
+        'GIT_CONFIG': GIT_CONFIG
+    }
+
+@abx.hookimpl
+def get_BINARIES():
+    from .binaries import GIT_BINARY
+    
+    return {
+        'git': GIT_BINARY,
+    }
+
+@abx.hookimpl
+def get_EXTRACTORS():
+    from .extractors import GIT_EXTRACTOR
+    
+    return {
+        'git': GIT_EXTRACTOR,
+    }

archivebox/vendor/abx-plugin-git/abx_plugin_git/binaries.py

@@ -1,17 +1,17 @@
-__package__ = 'plugins_extractor.git'
+__package__ = 'abx_plugin_git'
 
 from typing import List
 
 from pydantic import InstanceOf
-from pydantic_pkgr import BinProvider, BinName
+from pydantic_pkgr import BinProvider, BinName, Binary
 
-from abx.archivebox.base_binary import BaseBinary, env, apt, brew
+from abx_plugin_default_binproviders import apt, brew, env
 
 from .config import GIT_CONFIG
 
 
 
-class GitBinary(BaseBinary):
+class GitBinary(Binary):
     name: BinName = GIT_CONFIG.GIT_BINARY
     binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
 

archivebox/vendor/abx-plugin-git/abx_plugin_git/config.py

@@ -1,10 +1,10 @@
-__package__ = 'plugins_extractor.git'
+__package__ = 'abx_plugin_git'
 
 from typing import List
 
 from pydantic import Field
 
-from abx.archivebox.base_configset import BaseConfigSet
+from abx_spec_config.base_configset import BaseConfigSet
 
 from archivebox.config.common import ARCHIVING_CONFIG
 

archivebox/vendor/abx-plugin-git/abx_plugin_git/extractors.py

@@ -0,0 +1,15 @@
+__package__ = 'abx_plugin_git'
+
+# from pathlib import Path
+
+# from .binaries import GIT_BINARY
+
+
+# class GitExtractor(BaseExtractor):
+#     name: ExtractorName = 'git'
+#     binary: str = GIT_BINARY.name
+
+#     def get_output_path(self, snapshot) -> Path | None:
+#         return snapshot.as_link() / 'git'
+
+# GIT_EXTRACTOR = GitExtractor()

archivebox/vendor/abx-plugin-git/abx_plugin_git/git.py

@@ -16,8 +16,8 @@ from archivebox.misc.util import (
 from ..logging_util import TimedProgress
 from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
 
-from archivebox.plugins_extractor.git.config import GIT_CONFIG
+from abx_plugin_git.config import GIT_CONFIG
-from archivebox.plugins_extractor.git.binaries import GIT_BINARY
+from abx_plugin_git.binaries import GIT_BINARY
 
 
 def get_output_path():

archivebox/vendor/abx-plugin-git/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "abx-plugin-git"
+version = "2024.10.28"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "abx>=0.1.0",
+    "abx-spec-config>=0.1.0",
+    "abx-spec-pydantic-pkgr>=0.1.0",
+    "abx-plugin-default-binproviders>=2024.10.24",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project.entry-points.abx]
+abx_plugin_git = "abx_plugin_git"

archivebox/vendor/abx-plugin-htmltotext/abx_plugin_htmltotext/__init__.py

@@ -0,0 +1,22 @@
+__package__ = 'abx_plugin_htmltotext'
+__label__ = 'HTML-to-Text'
+
+import abx
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import HTMLTOTEXT_CONFIG
+    
+    return {
+        'HTMLTOTEXT_CONFIG': HTMLTOTEXT_CONFIG
+    }
+
+
+# @abx.hookimpl
+# def get_EXTRACTORS():
+#     from .extractors import FAVICON_EXTRACTOR
+    
+#     return {
+#         'htmltotext': FAVICON_EXTRACTOR,
+#     }

archivebox/vendor/abx-plugin-htmltotext/abx_plugin_htmltotext/config.py

@@ -1,7 +1,4 @@
-__package__ = 'plugins_extractor.htmltotext'
+from abx_spec_config.base_configset import BaseConfigSet
-
-
-from abx.archivebox.base_configset import BaseConfigSet
 
 
 class HtmltotextConfig(BaseConfigSet):

archivebox/vendor/abx-plugin-htmltotext/pyproject.toml

@@ -1,22 +1,17 @@
 [project]
-name = "abx-ldap-auth"
+name = "abx-plugin-htmltotext"
-version = "0.1.0"
+version = "2024.10.28"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.10"
-dependencies = []
+dependencies = [
-
+    "abx>=0.1.0",
-
+    "abx-spec-config>=0.1.0",
-[project.entry-points.abx]
+]
-ldap = "abx_ldap_auth"
-
 
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
-[tool.hatch.build.targets.sdist]
+[project.entry-points.abx]
-packages = ["."]
+abx_plugin_htmltotext = "abx_plugin_htmltotext"
-
-[tool.hatch.build.targets.wheel]
-packages = ["."]

archivebox/vendor/abx-plugin-ldap-auth/abx_plugin_ldap_auth/__init__.py

@@ -1,36 +1,15 @@
-__package__ = 'plugins_auth.ldap'
+__package__ = 'abx_plugin_ldap_auth'
-__id__ = 'ldap'
 __label__ = 'LDAP'
-__version__ = '2024.10.14'
-__author__ = 'ArchiveBox'
 __homepage__ = 'https://github.com/django-auth-ldap/django-auth-ldap'
-__dependencies__ = ['pip']
 
 import abx
 
-
-@abx.hookimpl
-def get_PLUGIN():
-    return {
-        __id__: {
-            'id': __id__,
-            'package': __package__,
-            'label': __label__,
-            'version': __version__,
-            'author': __author__,
-            'homepage': __homepage__,
-            'dependencies': __dependencies__,
-        }
-    }
-
-
-
 @abx.hookimpl
 def get_CONFIG():
     from .config import LDAP_CONFIG
     
     return {
-        __id__: LDAP_CONFIG
+        'LDAP_CONFIG': LDAP_CONFIG
     }
 
 @abx.hookimpl
@@ -48,12 +27,12 @@ def create_superuser_from_ldap_user(sender, user=None, ldap_user=None, **kwargs)
     ArchiveBox requires staff/superuser status to view the admin at all, so we must create a user
     + set staff and superuser when LDAP authenticates a new person.
     """
-    from django.conf import settings
+    from .config import LDAP_CONFIG
     
     if user is None:
         return                        # not authenticated at all
     
-    if not user.id and settings.CONFIGS.ldap.LDAP_CREATE_SUPERUSER:
+    if not user.id and LDAP_CONFIG.LDAP_CREATE_SUPERUSER:
         user.is_superuser = True      # authenticated via LDAP, but user is not set up in DB yet
 
     user.is_staff = True
@@ -69,9 +48,7 @@ def ready():
     
     LDAP_CONFIG.validate()
     
-    from django.conf import settings
+    if LDAP_CONFIG.LDAP_ENABLED:
-    
-    if settings.CONFIGS.ldap.LDAP_ENABLED:
         # tell django-auth-ldap to call our function when a user is authenticated via LDAP
         import django_auth_ldap.backend
         django_auth_ldap.backend.populate_user.connect(create_superuser_from_ldap_user)

archivebox/vendor/abx-plugin-ldap-auth/abx_plugin_ldap_auth/binaries.py

@@ -1,5 +1,4 @@
-__package__ = 'plugins_auth.ldap'
+__package__ = 'abx_plugin_ldap_auth'
-
 
 import inspect
 
@@ -7,12 +6,10 @@ from typing import List
 from pathlib import Path
 from pydantic import InstanceOf
 
-from pydantic_pkgr import BinaryOverrides, SemVer
+from pydantic_pkgr import BinaryOverrides, SemVer, Binary, BinProvider
 
-
+from abx_plugin_default_binproviders import apt
-from abx.archivebox.base_binary import BaseBinary, BaseBinProvider, apt
+from abx_plugin_pip.binproviders import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, VENV_SITE_PACKAGES, LIB_SITE_PACKAGES, USER_SITE_PACKAGES, SYS_SITE_PACKAGES
-
-from plugins_pkg.pip.binproviders import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, VENV_SITE_PACKAGES, LIB_SITE_PACKAGES, USER_SITE_PACKAGES, SYS_SITE_PACKAGES
 
 from .config import get_ldap_lib
 
@@ -39,10 +36,10 @@ def get_LDAP_LIB_version():
     return LDAP_LIB and SemVer(LDAP_LIB.__version__)
 
 
-class LdapBinary(BaseBinary):
+class LdapBinary(Binary):
     name: str = 'ldap'
     description: str = 'LDAP Authentication'
-    binproviders_supported: List[InstanceOf[BaseBinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, apt]
+    binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, apt]
 
     overrides: BinaryOverrides = {
         LIB_PIP_BINPROVIDER.name: {

archivebox/vendor/abx-plugin-ldap-auth/abx_plugin_ldap_auth/config.py

@@ -1,11 +1,11 @@
-__package__ = 'plugins_auth.ldap'
+__package__ = 'abx_plugin_ldap_auth'
 
 import sys
 
 from typing import Dict, List, Optional
-from pydantic import Field, model_validator, computed_field
+from pydantic import Field, computed_field
 
-from abx.archivebox.base_configset import BaseConfigSet
+from abx_spec_config.base_configset import BaseConfigSet
 
 LDAP_LIB = None
 LDAP_SEARCH = None

archivebox/vendor/abx-plugin-ldap-auth/pyproject.toml

@@ -0,0 +1,20 @@
+[project]
+name = "abx-plugin-ldap-auth"
+version = "2024.10.28"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "abx>=0.1.0",
+    "abx-spec-config>=0.1.0",
+    "abx-spec-django>=0.1.0",
+]
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+
+[project.entry-points.abx]
+abx_plugin_ldap_auth = "abx_plugin_ldap_auth"

archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/__init__.py

@@ -0,0 +1,29 @@
+__package__ = 'abx_plugin_mercury'
+__label__ = 'Postlight Parser'
+__homepage__ = 'https://github.com/postlight/mercury-parser'
+
+import abx
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import MERCURY_CONFIG
+    
+    return {
+        'MERCURY_CONFIG': MERCURY_CONFIG
+    }
+
+@abx.hookimpl
+def get_BINARIES():
+    from .binaries import MERCURY_BINARY
+    
+    return {
+        'mercury': MERCURY_BINARY,
+    }
+
+@abx.hookimpl
+def get_EXTRACTORS():
+    from .extractors import MERCURY_EXTRACTOR
+    
+    return {
+        'mercury': MERCURY_EXTRACTOR,
+    }

archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/binaries.py

@@ -1,18 +1,18 @@
-__package__ = 'plugins_extractor.mercury'
+__package__ = 'abx_plugin_mercury'
 
 from typing import List
 
 from pydantic import InstanceOf
-from pydantic_pkgr import BinProvider, BinName, BinaryOverrides, bin_abspath
+from pydantic_pkgr import BinProvider, BinName, BinaryOverrides, bin_abspath, Binary
 
-from abx.archivebox.base_binary import BaseBinary, env
+from abx_plugin_default_binproviders import env
 
-from archivebox.plugins_pkg.npm.binproviders import SYS_NPM_BINPROVIDER, LIB_NPM_BINPROVIDER
+from abx_plugin_npm.binproviders import SYS_NPM_BINPROVIDER, LIB_NPM_BINPROVIDER
 
 from .config import MERCURY_CONFIG
 
 
-class MercuryBinary(BaseBinary):
+class MercuryBinary(Binary):
     name: BinName = MERCURY_CONFIG.MERCURY_BINARY
     binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_NPM_BINPROVIDER, SYS_NPM_BINPROVIDER, env]
 

archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/config.py

@@ -1,11 +1,11 @@
-__package__ = 'plugins_extractor.mercury'
+__package__ = 'abx_plugin_mercury'
 
 from typing import List, Optional
 from pathlib import Path
 
 from pydantic import Field
 
-from abx.archivebox.base_configset import BaseConfigSet
+from abx_spec_config.base_configset import BaseConfigSet
 
 from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG
 

archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/extractors.py

@@ -0,0 +1,17 @@
+__package__ = 'abx_plugin_mercury'
+
+# from pathlib import Path
+
+# from .binaries import MERCURY_BINARY
+
+
+
+# class MercuryExtractor(BaseExtractor):
+#     name: ExtractorName = 'mercury'
+#     binary: str = MERCURY_BINARY.name
+
+#     def get_output_path(self, snapshot) -> Path | None:
+#         return snapshot.link_dir / 'mercury' / 'content.html'
+
+
+# MERCURY_EXTRACTOR = MercuryExtractor()

archivebox/vendor/abx-plugin-mercury/pyproject.toml

@@ -0,0 +1,17 @@
+[project]
+name = "abx-plugin-mercury"
+version = "2024.10.28"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "abx>=0.1.0",
+    "abx-spec-config>=0.1.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project.entry-points.abx]
+abx_plugin_mercury = "abx_plugin_mercury"

archivebox/vendor/abx-plugin-npm/abx_plugin_npm/__init__.py

@@ -1,5 +1,3 @@
-__package__ = 'abx_plugin_npm_binprovider'
-__id__ = 'npm'
 __label__ = 'NPM'
 __author__ = 'ArchiveBox'
 __homepage__ = 'https://www.npmjs.com/'
@@ -10,9 +8,8 @@ import abx
 @abx.hookimpl
 def get_CONFIG():
     from .config import NPM_CONFIG
-    
     return {
-        __id__: NPM_CONFIG,
+        'NPM_CONFIG': NPM_CONFIG,
     }
 
 @abx.hookimpl

archivebox/vendor/abx-plugin-npm/abx_plugin_npm/binproviders.py

@@ -26,8 +26,7 @@ class LibNpmBinProvider(NpmProvider):
     
     def setup(self) -> None:
         # update paths from config at runtime
-        LIB_DIR = abx.pm.hook.get_CONFIG().LIB_DIR
+        LIB_DIR = abx.pm.hook.get_LIB_DIR()
-    
         self.npm_prefix = LIB_DIR / 'npm'
         self.PATH = f'{LIB_DIR / "npm" / "node_modules" / ".bin"}:{NEW_NODE_BIN_PATH}:{OLD_NODE_BIN_PATH}'
 

archivebox/vendor/abx-plugin-npm/pyproject.toml

@@ -1,5 +1,5 @@
 [project]
-name = "abx-plugin-npm-binprovider"
+name = "abx-plugin-npm"
 version = "2024.10.24"
 description = "NPM binary provider plugin for ABX"
 readme = "README.md"
@@ -17,4 +17,4 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [project.entry-points.abx]
-abx_plugin_npm_binprovider = "abx_plugin_npm_binprovider"
+abx_plugin_npm = "abx_plugin_npm"

archivebox/vendor/abx-plugin-pip/abx_plugin_pip/__init__.py

@@ -1,5 +1,4 @@
-__package__ = 'abx_plugin_pip_binprovider'
+__package__ = 'abx_plugin_pip'
-__id__ = 'pip'
 __label__ = 'PIP'
 
 import abx
@@ -10,7 +9,7 @@ def get_CONFIG():
     from .config import PIP_CONFIG
     
     return {
-        __id__: PIP_CONFIG
+        'PIP_CONFIG': PIP_CONFIG
     }
 
 @abx.hookimpl(tryfirst=True)

archivebox/vendor/abx-plugin-pip/abx_plugin_pip/binaries.py

@@ -1,4 +1,4 @@
-__package__ = 'abx_plugin_pip_binprovider'
+__package__ = 'abx_plugin_pip'
 
 import sys
 from pathlib import Path

archivebox/vendor/abx-plugin-pip/abx_plugin_pip/binproviders.py

@@ -58,7 +58,7 @@ class LibPipBinProvider(PipProvider):
     
     def setup(self) -> None:
         # update venv path to match most up-to-date LIB_DIR based on runtime config
-        LIB_DIR = abx.pm.hook.get_FLAT_CONFIG().LIB_DIR
+        LIB_DIR = abx.pm.hook.get_LIB_DIR()
         self.pip_venv = LIB_DIR / 'pip' / 'venv'
         super().setup()
 

archivebox/vendor/abx-plugin-pip/abx_plugin_pip/config.py

@@ -3,7 +3,7 @@ __package__ = 'pip'
 from typing import List, Optional
 from pydantic import Field
 
-from abx.archivebox.base_configset import BaseConfigSet
+from abx_spec_config.base_configset import BaseConfigSet
 
 
 class PipDependencyConfigs(BaseConfigSet):

archivebox/vendor/abx-plugin-pip/pyproject.toml

@@ -1,5 +1,5 @@
 [project]
-name = "abx-plugin-pip-binprovider"
+name = "abx-plugin-pip"
 version = "2024.10.24"
 description = "Add your description here"
 readme = "README.md"
@@ -19,4 +19,4 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [project.entry-points.abx]
-abx_plugin_pip_binprovider = "abx_plugin_pip_binprovider"
+abx_plugin_pip = "abx_plugin_pip"

archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/__init__.py

@@ -1,7 +1,4 @@
-__package__ = 'abx_plugin_playwright_binprovider'
-__id__ = 'playwright'
 __label__ = 'Playwright'
-__author__ = 'ArchiveBox'
 __homepage__ = 'https://github.com/microsoft/playwright-python'
 
 import abx
@@ -10,9 +7,8 @@ import abx
 @abx.hookimpl
 def get_CONFIG():
     from .config import PLAYWRIGHT_CONFIG
-    
     return {
-        __id__: PLAYWRIGHT_CONFIG
+        'PLAYWRIGHT_CONFIG': PLAYWRIGHT_CONFIG
     }
 
 @abx.hookimpl

archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/binaries.py

@@ -1,4 +1,4 @@
-__package__ = 'abx_plugin_playwright_binprovider'
+__package__ = 'abx_plugin_playwright'
 
 from typing import List
 
@@ -6,7 +6,7 @@ from pydantic import InstanceOf
 from pydantic_pkgr import BinName, BinProvider, Binary
 
 
-from abx_plugin_pip_binprovider.binproviders import LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER
+from abx_plugin_pip.binproviders import LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER
 from abx_plugin_default_binproviders import env
 
 from .config import PLAYWRIGHT_CONFIG

archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/binproviders.py

@@ -1,4 +1,4 @@
-__package__ = 'abx_plugin_playwright_binprovider'
+__package__ = 'abx_plugin_playwright'
 
 import os
 import shutil
@@ -59,7 +59,7 @@ class PlaywrightBinProvider(BinProvider):
 
     def setup(self) -> None:
         # update paths from config at runtime
-        LIB_DIR = abx.pm.hook.get_FLAT_CONFIG().LIB_DIR
+        LIB_DIR = abx.pm.hook.get_LIB_DIR()
         
         self.PATH = f"{LIB_DIR / 'bin'}:{DEFAULT_ENV_PATH}"
 

archivebox/vendor/abx-plugin-playwright/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
-name = "abx-plugin-playwright-binprovider"
+name = "abx-plugin-playwright"
-version = "2024.10.24"
+version = "2024.10.28"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -17,4 +17,4 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [project.entry-points.abx]
-abx_plugin_playwright_binprovider = "abx_plugin_playwright_binprovider"
+abx_plugin_playwright = "abx_plugin_playwright"

archivebox/vendor/abx-plugin-pocket/abx_plugin_pocket/__init__.py

@@ -0,0 +1,18 @@
+__package__ = 'abx_plugin_pocket'
+__label__ = 'Pocket'
+
+import abx
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import POCKET_CONFIG
+    
+    return {
+        'POCKET_CONFIG': POCKET_CONFIG
+    }
+
+@abx.hookimpl
+def ready():
+    from .config import POCKET_CONFIG
+    POCKET_CONFIG.validate()

archivebox/vendor/abx-plugin-pocket/abx_plugin_pocket/config.py

@@ -1,10 +1,10 @@
-__package__ = 'plugins_extractor.pocket'
+__package__ = 'abx_plugin_pocket'
 
 from typing import Dict
 
 from pydantic import Field
 
-from abx.archivebox.base_configset import BaseConfigSet
+from abx_spec_config.base_configset import BaseConfigSet
 
 
 class PocketConfig(BaseConfigSet):

archivebox/vendor/abx-plugin-pocket/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "abx-plugin-pocket"
+version = "2024.10.28"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "abx>=0.1.0",
+    "abx-spec-config>=0.1.0",
+    "pocket>=0.3.6",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project.entry-points.abx]
+abx_plugin_pocket = "abx_plugin_pocket"

archivebox/vendor/abx-plugin-puppeteer/abx_plugin_puppeteer/__init__.py

@@ -0,0 +1,30 @@
+__package__ = 'abx_plugin_puppeteer'
+__label__ = 'Puppeteer'
+__homepage__ = 'https://github.com/puppeteer/puppeteer'
+
+import abx
+
+
+@abx.hookimpl
+def get_CONFIG():
+    from .config import PUPPETEER_CONFIG
+    
+    return {
+        'PUPPETEER_CONFIG': PUPPETEER_CONFIG
+    }
+
+@abx.hookimpl
+def get_BINARIES():
+    from .binaries import PUPPETEER_BINARY
+    
+    return {
+        'puppeteer': PUPPETEER_BINARY,
+    }
+
+@abx.hookimpl
+def get_BINPROVIDERS():
+    from .binproviders import PUPPETEER_BINPROVIDER
+    
+    return {
+        'puppeteer': PUPPETEER_BINPROVIDER,
+    }