mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-09 12:21:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix imports and deps

Browse source
This commit is contained in:
Nick Sweeting 2024-11-18 18:07:34 -08:00
parent 6b83b4c995
commit 0acd388c02
No known key found for this signature in database
7 changed files with 614 additions and 695 deletions
Download patch file Download diff file Expand all files Collapse all files

3
archivebox/cli/archivebox_add.py
View file

@ -21,7 +21,6 @@ from archivebox.misc.checks import check_data_folder
from archivebox.parsers import PARSERS from archivebox.parsers import PARSERS
from archivebox.logging_util import SmartFormatter, accept_stdin, stderr from archivebox.logging_util import SmartFormatter, accept_stdin, stderr
from abid_utils.models import get_or_create_system_user_pk
if TYPE_CHECKING: if TYPE_CHECKING:
from core.models import Snapshot from core.models import Snapshot
@ -55,6 +54,8 @@ def add(urls: str | list[str],
from seeds.models import Seed from seeds.models import Seed
from crawls.models import Crawl from crawls.models import Crawl
from actors.orchestrator import Orchestrator from actors.orchestrator import Orchestrator
from abid_utils.models import get_or_create_system_user_pk
created_by_id = created_by_id or get_or_create_system_user_pk() created_by_id = created_by_id or get_or_create_system_user_pk()

2
archivebox/core/models.py
View file

@ -449,6 +449,8 @@ class Snapshot(ModelWithOutputDir, ModelWithStateMachine, ABIDModel):
for extractor in EXTRACTORS: for extractor in EXTRACTORS:
if not extractor: if not extractor:
continue continue
if ArchiveResult.objects.filter(snapshot=self, extractor=extractor).exists():
continue
archiveresult, created = ArchiveResult.objects.get_or_create( archiveresult, created = ArchiveResult.objects.get_or_create(
snapshot=self, snapshot=self,
extractor=extractor, extractor=extractor,

8
archivebox/pkgs/abx-spec-config/abx_spec_config/__init__.py
View file

@ -43,7 +43,7 @@ class ConfigPluginSpec:
@staticmethod @staticmethod
@abx.hookspec(firstresult=True) @abx.hookspec(firstresult=True)
@abx.hookimpl @abx.hookimpl
def get_CONFIGS() -> dict[abx.PluginId, BaseConfigSet]: def get_CONFIGS() -> dict[abx.PluginId, 'BaseConfigSet | ConstantsDict']:
"""Get the config for all plugins by plugin_id -> {plugin_abc: PluginABCConfigSet(), plugin_xyz: PluginXYZConfigSet(), ...}""" """Get the config for all plugins by plugin_id -> {plugin_abc: PluginABCConfigSet(), plugin_xyz: PluginXYZConfigSet(), ...}"""
return abx.as_dict(pm.hook.get_CONFIG()) return abx.as_dict(pm.hook.get_CONFIG())
@ -117,7 +117,7 @@ class ConfigPluginSpec:
return benedict({ return benedict({
key: value key: value
for configset in pm.hook.get_CONFIGS().values() for configset in pm.hook.get_CONFIGS().values()
for key, value in configset.from_collection().items() for key, value in (configset.from_collection().items() if isinstance(configset, BaseConfigSet) else {})
}) if collection == ... else collection }) if collection == ... else collection
@staticmethod @staticmethod
@ -129,7 +129,7 @@ class ConfigPluginSpec:
return benedict({ return benedict({
key: value key: value
for configset in pm.hook.get_CONFIGS().values() for configset in pm.hook.get_CONFIGS().values()
for key, value in configset.from_environment().items() for key, value in (configset.from_environment().items() if isinstance(configset, BaseConfigSet) else ())
}) if environment == ... else environment }) if environment == ... else environment
@staticmethod @staticmethod
@ -151,7 +151,7 @@ class ConfigPluginSpec:
return benedict({ return benedict({
key: value key: value
for configset in pm.hook.get_CONFIGS().values() for configset in pm.hook.get_CONFIGS().values()
for key, value in configset.from_defaults().items() for key, value in (configset.from_defaults().items() if isinstance(configset, BaseConfigSet) else configset.items())
}) if default == ... else default }) if default == ... else default

6
archivebox/pkgs/abx/abx.py
View file

@ -5,7 +5,7 @@ __author__ = 'Nick Sweeting'
__homepage__ = 'https://github.com/ArchiveBox/ArchiveBox' __homepage__ = 'https://github.com/ArchiveBox/ArchiveBox'
__order__ = 0 __order__ = 0
import sys
import inspect import inspect
import importlib import importlib
import itertools import itertools
@ -446,9 +446,11 @@ def load_plugins(plugins: Iterable[PluginId | ModuleType | Type] | Dict[PluginId
PLUGINS_TO_LOAD = sorted(PLUGINS_TO_LOAD, key=lambda x: x['order']) PLUGINS_TO_LOAD = sorted(PLUGINS_TO_LOAD, key=lambda x: x['order'])
for plugin_info in PLUGINS_TO_LOAD: for plugin_info in PLUGINS_TO_LOAD:
if '--version' not in sys.argv and '--help' not in sys.argv:
print(f'🧩 Loading plugin: {plugin_info["id"]}...', end='\r', flush=True, file=sys.stderr)
pm.register(plugin_info['module']) pm.register(plugin_info['module'])
LOADED_PLUGINS[plugin_info['id']] = plugin_info LOADED_PLUGINS[plugin_info['id']] = plugin_info
print(f' √ Loaded plugin: {plugin_info["id"]}') # print('\x1b[2K', end='\r', flush=True, file=sys.stderr)
return benedict(LOADED_PLUGINS) return benedict(LOADED_PLUGINS)
@cache @cache

8
pyproject.toml
View file

@ -143,6 +143,9 @@ all = [
] ]
[tool.uv] [tool.uv]
environments = ["sys_platform == 'darwin'", "sys_platform == 'linux'"]
package = true
# compile-bytecode = true
dev-dependencies = [ dev-dependencies = [
### BUILD ### BUILD
"uv>=0.4.26", "uv>=0.4.26",
@ -219,6 +222,11 @@ abx-plugin-htmltotext = { workspace = true }
members = ["archivebox/pkgs/*"] members = ["archivebox/pkgs/*"]
exclude = ["archivebox/pkgs/__pycache__"] exclude = ["archivebox/pkgs/__pycache__"]
[tool.uv.pip]
all-extras = true
python-version = "3.10"
# compile-bytecode = true
[build-system] [build-system]
requires = ["pdm-backend"] requires = ["pdm-backend"]
build-backend = "pdm.backend" build-backend = "pdm.backend"

88
requirements.txt
View file

@ -1,6 +1,6 @@
# This file was autogenerated by uv via the following command: # This file was autogenerated by uv via the following command:
# uv pip compile pyproject.toml --all-extras -o requirements.txt # uv pip compile pyproject.toml --all-extras -o requirements.txt
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx
# via # via
# archivebox (pyproject.toml) # archivebox (pyproject.toml)
# abx-plugin-archivedotorg # abx-plugin-archivedotorg
@ -30,59 +30,74 @@
# abx-spec-django # abx-spec-django
# abx-spec-extractor # abx-spec-extractor
# abx-spec-searchbackend # abx-spec-searchbackend
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-archivedotorg -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-archivedotorg
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-chrome -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-chrome
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-curl -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-curl
# via # via
# archivebox (pyproject.toml) # archivebox (pyproject.toml)
# abx-plugin-archivedotorg # abx-plugin-archivedotorg
# abx-plugin-favicon # abx-plugin-favicon
# abx-plugin-title # abx-plugin-title
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-default-binproviders -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-default-binproviders
# via # via
# archivebox (pyproject.toml) # archivebox (pyproject.toml)
# abx-plugin-git # abx-plugin-git
# abx-plugin-npm # abx-plugin-npm
# abx-plugin-pip # abx-plugin-pip
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-favicon -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-favicon
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-git -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-git
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-htmltotext -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-htmltotext
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-ldap-auth -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-ldap-auth
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-mercury -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-mercury
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-npm -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-npm
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-pip -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-pip
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-playwright -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-playwright
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-puppeteer -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-puppeteer
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-readability -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-readability
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-ripgrep-search -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-ripgrep-search
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-singlefile -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-singlefile
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-sonic-search -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-sonic-search
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-sqlitefts-search -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-sqlitefts-search
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-title -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-title
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-wget -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-wget
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-plugin-ytdlp -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-plugin-ytdlp
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-archivebox -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-abx-pkg
# via
# archivebox (pyproject.toml)
# abx-plugin-chrome
# abx-plugin-curl
# abx-plugin-default-binproviders
# abx-plugin-git
# abx-plugin-npm
# abx-plugin-pip
# abx-plugin-playwright
# abx-plugin-puppeteer
# abx-plugin-singlefile
# abx-plugin-sonic-search
# abx-plugin-wget
# abx-plugin-ytdlp
-e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-archivebox
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-config -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-config
# via # via
# archivebox (pyproject.toml) # archivebox (pyproject.toml)
# abx-plugin-archivedotorg # abx-plugin-archivedotorg
@ -105,28 +120,13 @@
# abx-plugin-title # abx-plugin-title
# abx-plugin-wget # abx-plugin-wget
# abx-plugin-ytdlp # abx-plugin-ytdlp
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-django -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-django
# via # via
# archivebox (pyproject.toml) # archivebox (pyproject.toml)
# abx-plugin-ldap-auth # abx-plugin-ldap-auth
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-extractor -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-extractor
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-abx-pkg -e file:///Users/squash/Local/Code/archiveboxes/archivebox7/archivebox/pkgs/abx-spec-searchbackend
# via
# archivebox (pyproject.toml)
# abx-plugin-chrome
# abx-plugin-curl
# abx-plugin-default-binproviders
# abx-plugin-git
# abx-plugin-npm
# abx-plugin-pip
# abx-plugin-playwright
# abx-plugin-puppeteer
# abx-plugin-singlefile
# abx-plugin-sonic-search
# abx-plugin-wget
# abx-plugin-ytdlp
-e file:///Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox7/archivebox/pkgs/abx-spec-searchbackend
# via # via
# archivebox (pyproject.toml) # archivebox (pyproject.toml)
# abx-plugin-ripgrep-search # abx-plugin-ripgrep-search
@ -308,7 +308,7 @@ ipython==8.29.0
# ipdb # ipdb
jedi==0.19.2 jedi==0.19.2
# via ipython # via ipython
libcst==1.5.0 libcst==1.5.1
# via django-autotyping # via django-autotyping
mailchecker==6.0.11 mailchecker==6.0.11
# via python-benedict # via python-benedict
@ -515,7 +515,7 @@ xlrd==2.0.1
# via python-benedict # via python-benedict
xmltodict==0.14.2 xmltodict==0.14.2
# via python-benedict # via python-benedict
yt-dlp==2024.11.4 yt-dlp==2024.11.18
# via archivebox (pyproject.toml) # via archivebox (pyproject.toml)
zope-interface==7.1.1 zope-interface==7.1.1
# via twisted # via twisted

1194
uv.lock generated
View file

File diff suppressed because it is too large Load diff