mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-09 12:21:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix imports and deps

Browse source
This commit is contained in:
Nick Sweeting 2024-11-18 18:07:34 -08:00
parent 6b83b4c995
commit 0acd388c02
No known key found for this signature in database
7 changed files with 614 additions and 695 deletions
Download patch file Download diff file Expand all files Collapse all files

3
archivebox/cli/archivebox_add.py
View file

@ -21,7 +21,6 @@ from archivebox.misc.checks import check_data_folder
from archivebox.parsers import PARSERS
from archivebox.logging_util import SmartFormatter, accept_stdin, stderr
from abid_utils.models import get_or_create_system_user_pk
if TYPE_CHECKING:
from core.models import Snapshot
@ -55,6 +54,8 @@ def add(urls: str | list[str],
from seeds.models import Seed
from crawls.models import Crawl
from actors.orchestrator import Orchestrator
from abid_utils.models import get_or_create_system_user_pk
created_by_id = created_by_id or get_or_create_system_user_pk()

2
archivebox/core/models.py
View file

@ -449,6 +449,8 @@ class Snapshot(ModelWithOutputDir, ModelWithStateMachine, ABIDModel):
for extractor in EXTRACTORS:
if not extractor:
continue
if ArchiveResult.objects.filter(snapshot=self, extractor=extractor).exists():
continue
archiveresult, created = ArchiveResult.objects.get_or_create(
snapshot=self,
extractor=extractor,

8
archivebox/pkgs/abx-spec-config/abx_spec_config/__init__.py
View file

@ -43,7 +43,7 @@ class ConfigPluginSpec:
@staticmethod
@abx.hookspec(firstresult=True)
@abx.hookimpl
def get_CONFIGS() -> dict[abx.PluginId, BaseConfigSet]:
def get_CONFIGS() -> dict[abx.PluginId, 'BaseConfigSet | ConstantsDict']:
"""Get the config for all plugins by plugin_id -> {plugin_abc: PluginABCConfigSet(), plugin_xyz: PluginXYZConfigSet(), ...}"""
return abx.as_dict(pm.hook.get_CONFIG())
@ -117,7 +117,7 @@ class ConfigPluginSpec:
return benedict({
key: value
for configset in pm.hook.get_CONFIGS().values()
for key, value in configset.from_collection().items()
for key, value in (configset.from_collection().items() if isinstance(configset, BaseConfigSet) else {})
}) if collection == ... else collection
@staticmethod
@ -129,7 +129,7 @@ class ConfigPluginSpec:
return benedict({
key: value
for configset in pm.hook.get_CONFIGS().values()
for key, value in configset.from_environment().items()
for key, value in (configset.from_environment().items() if isinstance(configset, BaseConfigSet) else ())
}) if environment == ... else environment
@staticmethod
@ -151,7 +151,7 @@ class ConfigPluginSpec:
return benedict({
key: value
for configset in pm.hook.get_CONFIGS().values()
for key, value in configset.from_defaults().items()
for key, value in (configset.from_defaults().items() if isinstance(configset, BaseConfigSet) else configset.items())
}) if default == ... else default

6
archivebox/pkgs/abx/abx.py
View file

@ -5,7 +5,7 @@ __author__ = 'Nick Sweeting'
__homepage__ = 'https://github.com/ArchiveBox/ArchiveBox'
__order__ = 0
import sys
import inspect
import importlib
import itertools
@ -446,9 +446,11 @@ def load_plugins(plugins: Iterable[PluginId | ModuleType | Type] | Dict[PluginId
PLUGINS_TO_LOAD = sorted(PLUGINS_TO_LOAD, key=lambda x: x['order'])
for plugin_info in PLUGINS_TO_LOAD:
if '--version' not in sys.argv and '--help' not in sys.argv:
print(f'🧩 Loading plugin: {plugin_info["id"]}...', end='\r', flush=True, file=sys.stderr)
pm.register(plugin_info['module'])
LOADED_PLUGINS[plugin_info['id']] = plugin_info
print(f' √ Loaded plugin: {plugin_info["id"]}')
# print('\x1b[2K', end='\r', flush=True, file=sys.stderr)
return benedict(LOADED_PLUGINS)
@cache