mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-27 21:24:16 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix plugin loading and admin config display

Browse source
This commit is contained in:
Nick Sweeting 2024-01-24 02:15:02 -08:00
parent d0e3c9502e
commit 8e41aec099
19 changed files with 316 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

1
archivebox/plugins/defaults/__init__.py Normal file
View file

@ -0,0 +1 @@
default_app_config = 'plugins.defaults.apps.DefaultsPluginAppConfig'

5
archivebox/plugins/defaults/admin.py
View file

@ -8,11 +8,10 @@ from .models import (
class DependencyAdmin(SingletonModelAdmin):
readonly_fields = ('REQUIRED', 'ENABLED', 'BINARY', 'ARGS', 'bin_path', 'bin_version', 'is_valid', 'is_enabled')
readonly_fields = ('id', 'NAME', 'LABEL', 'REQUIRED', 'bin_path', 'bin_version', 'is_valid', 'is_enabled')
class ExtractorAdmin(SingletonModelAdmin):
# readonly_fields = ('REQUIRED', 'ENABLED', 'BINARY', 'ARGS', 'bin_path', 'bin_version', 'is_valid', 'is_enabled')
pass
readonly_fields = ('id', 'NAME', 'LABEL', 'DEFAULT_ENABLED', 'DEFAULT_CMD', 'CMD', 'ARGS', 'TIMEOUT', 'dependency', 'is_valid', 'is_enabled')
print('DefaultsPluginConfig.admin')

10
archivebox/plugins/defaults/apps.py
View file

@ -1,13 +1,15 @@
__package__ = 'archivebox.plugins.defaults'
# __package__ = 'archivebox.plugins.defaults'
from django.apps import AppConfig
class DefaultsPluginConfig(AppConfig):
label = "ArchiveBox Defaults"
name = "defaults"
class DefaultsPluginAppConfig(AppConfig):
name = "plugins.defaults"
# label = "ArchiveBox Defaults"
verbose_name = "Plugin Configuration Defaults"
default_auto_field = "django.db.models.AutoField"

4
archivebox/plugins/defaults/migrations/0001_initial.py
View file

@ -1,4 +1,4 @@
# Generated by Django 3.1.14 on 2024-01-24 08:06
# Generated by Django 3.1.14 on 2024-01-24 08:56
from django.db import migrations, models
@ -20,6 +20,7 @@ class Migration(migrations.Migration):
('id', models.AutoField(default=1, primary_key=True, serialize=False)),
],
options={
'verbose_name': 'Dependency Configuration Defaults',
'abstract': False,
},
),
@ -33,6 +34,7 @@ class Migration(migrations.Migration):
('id', models.AutoField(default=1, primary_key=True, serialize=False)),
],
options={
'verbose_name': 'Extractor Configuration Defaults',
'abstract': False,
},
),

31
archivebox/plugins/defaults/migrations/0002_auto_20240124_0943.py Normal file
View file

@ -0,0 +1,31 @@
# Generated by Django 3.1.14 on 2024-01-24 09:43
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('defaults', '0001_initial'),
]
operations = [
migrations.AlterModelOptions(
name='archiveboxdefaultdependency',
options={'verbose_name': 'Default Configuration: Dependencies'},
),
migrations.AlterModelOptions(
name='archiveboxdefaultextractor',
options={'verbose_name': 'Default Configuration: Extractors'},
),
migrations.AlterField(
model_name='archiveboxdefaultdependency',
name='BINARY',
field=models.CharField(default='/bin/bash', max_length=255),
),
migrations.AlterField(
model_name='archiveboxdefaultdependency',
name='ENABLED',
field=models.BooleanField(default=True),
),
]

37
archivebox/plugins/defaults/models.py
View file

@ -10,17 +10,20 @@ from django.utils.functional import cached_property
from solo.models import SingletonModel
from config import bin_path, bin_version
ConfigDict = Dict[str, Any]
def bin_path(binary: str) -> str | None:
return shutil.which(str(Path(binary).expanduser())) or shutil.which(str(binary)) or binary
# def bin_path(binary: str) -> str | None:
# return shutil.which(str(Path(binary).expanduser())) or shutil.which(str(binary)) or binary
def bin_version(bin_path: str, cmd: str | None=None) -> str | None:
return '0.0.0'
# def bin_version(bin_path: str, cmd: str | None=None) -> str | None:
# return '0.0.0'
class ArchiveBoxBaseDependency(SingletonModel):
class ArchiveBoxBaseDependency(models.Model):
singleton_instance_id = 1
id = models.AutoField(default=singleton_instance_id, primary_key=True)
@ -37,11 +40,11 @@ class ArchiveBoxBaseDependency(SingletonModel):
PIP_DEPENDENCIES = []
NPM_DEPENDENCIES = []
DEFAULT_BINARY = '/bin/false'
DEFAULT_START_CMD = '/bin/false'
DEFAULT_BINARY = '/bin/bash'
DEFAULT_START_CMD = '/bin/bash -c "while true; do sleep 1; done"'
DEFAULT_PID_FILE = 'logs/{NAME}_WORKER.pid'
DEFAULT_STOP_CMD = 'kill "$(<{PID_FILE})"'
DEFAULT_VERSION_COMMAND = '{CMD} --version'
DEFAULT_VERSION_COMMAND = '{BINARY} --version'
DEFAULT_ARGS = ''
VERSION_CMD = '{BINARY} --version'
@ -58,7 +61,7 @@ class ArchiveBoxBaseDependency(SingletonModel):
app_label = 'defaults'
def __str__(self):
return "{self.LABEL} Dependency Configuration"
return f"{self.LABEL} Dependency Configuration"
def __json__(self):
return {
@ -79,7 +82,9 @@ class ArchiveBoxBaseDependency(SingletonModel):
@cached_property
def bin_version(self):
return bin_version(self.bin_path, cmd=self.VERSION_CMD)
print(f'ArchiveBoxBaseDependency.bin_version({self.bin_path}, cmd={self.VERSION_CMD.format(BINARY=self.BINARY)})')
return bin_version(self.bin_path, cmd=self.VERSION_CMD.format(BINARY=self.BINARY))
# return bin_version(self.bin_path, cmd=self.VERSION_CMD)
@cached_property
def is_valid(self):
@ -157,12 +162,15 @@ class ArchiveBoxDefaultDependency(ArchiveBoxBaseDependency, SingletonModel):
id = models.AutoField(default=singleton_instance_id, primary_key=True)
ENABLED = models.BooleanField(default=True, editable=True)
class Meta:
abstract = False
app_label = 'defaults'
verbose_name = 'Default Configuration: Dependencies'
class ArchiveBoxBaseExtractor(SingletonModel):
class ArchiveBoxBaseExtractor(models.Model):
singleton_instance_id = 1
id = models.AutoField(default=singleton_instance_id, primary_key=True)
@ -258,7 +266,7 @@ class ArchiveBoxBaseExtractor(SingletonModel):
def save(self, *args, **kwargs):
assert self.is_valid
# assert self.is_valid
with transaction.atomic():
result = super().save(*args, **kwargs)
@ -356,6 +364,11 @@ class ArchiveBoxDefaultExtractor(ArchiveBoxBaseExtractor, SingletonModel):
id = models.AutoField(default=singleton_instance_id, primary_key=True)
DEPENDENCY = ArchiveBoxDefaultDependency
ENABLED = models.BooleanField(default=True, editable=True)
class Meta:
abstract = False
app_label = 'defaults'
verbose_name = 'Default Configuration: Extractors'

7
archivebox/plugins/defaults/settings.py
View file

@ -1,12 +1,15 @@
from pathlib import Path
from django.conf import settings
def register_plugin_settings(settings=settings, name='defaults'):
settings.STATICFILES_DIRS += [
str(Path(PACKAGE_DIR) / f'plugins/{name}/static'),
str(Path(settings.PACKAGE_DIR) / f'plugins/{name}/static'),
]
settings.TEMPLATE_DIRS += [
str(Path(PACKAGE_DIR) / f'plugins/{name}/templates'),
str(Path(settings.PACKAGE_DIR) / f'plugins/{name}/templates'),
]
print('REGISTERED PLUGIN SETTINGS', name)

1
archivebox/plugins/system/__init__.py Normal file
View file

@ -0,0 +1 @@
default_app_config = 'plugins.system.apps.SystemPluginAppConfig'

17
archivebox/plugins/system/admin.py
View file

@ -5,6 +5,9 @@ from plugins.defaults.admin import DependencyAdmin, ExtractorAdmin
from .models import (
BashEnvironmentDependency,
PythonEnvironmentDependency,
NodeJSEnvironmentDependency,
AptEnvironmentDependency,
BrewEnvironmentDependency,
PipEnvironmentDependency,
@ -20,8 +23,20 @@ from .models import (
print('DefaultsPluginConfig.admin')
class MultiDependencyAdmin(admin.ModelAdmin):
readonly_fields = DependencyAdmin.readonly_fields
list_display = ('id', 'NAME', 'ENABLED', 'BINARY', 'ARGS', 'bin_path', 'bin_version', 'is_valid', 'is_enabled')
class MultiExtractorAdmin(admin.ModelAdmin):
readonly_fields = DependencyAdmin.readonly_fields
list_display = ('id', 'NAME', 'CMD', 'ARGS', 'is_valid', 'is_enabled')
# admin.site.register(BashEnvironmentDependency, DependencyAdmin)
admin.site.register(BashEnvironmentDependency, MultiDependencyAdmin)
admin.site.register(PythonEnvironmentDependency, DependencyAdmin)
admin.site.register(NodeJSEnvironmentDependency, DependencyAdmin)
admin.site.register(BashEnvironmentDependency, DependencyAdmin)
admin.site.register(AptEnvironmentDependency, DependencyAdmin)
admin.site.register(BrewEnvironmentDependency, DependencyAdmin)
admin.site.register(PipEnvironmentDependency, DependencyAdmin)

9
archivebox/plugins/system/apps.py
View file

@ -4,9 +4,10 @@
from django.apps import AppConfig
class SystemPluginConfig(AppConfig):
label = "ArchiveBox System"
name = "system"
class SystemPluginAppConfig(AppConfig):
name = "plugins.system"
# label = "ArchiveBox System"
verbose_name = "Host System Configuration"
default_auto_field = "django.db.models.AutoField"
@ -15,7 +16,7 @@ class SystemPluginConfig(AppConfig):
from django.conf import settings
from .settings import register_plugin_settings
from plugins.defaults.settings import register_plugin_settings
register_plugin_settings(settings, name=self.name)

36
archivebox/plugins/system/migrations/0001_initial.py
View file

@ -1,4 +1,4 @@
# Generated by Django 3.1.14 on 2024-01-24 08:06
# Generated by Django 3.1.14 on 2024-01-24 08:56
from django.db import migrations, models
@ -20,6 +20,7 @@ class Migration(migrations.Migration):
('ARGS', models.CharField(default='-qq', max_length=255)),
],
options={
'verbose_name': 'Package Manager: apt',
'abstract': False,
},
),
@ -32,6 +33,7 @@ class Migration(migrations.Migration):
('ARGS', models.CharField(default=[], editable=False, max_length=255)),
],
options={
'verbose_name': 'Internal Dependency: ArchiveBox Package',
'abstract': False,
},
),
@ -44,6 +46,7 @@ class Migration(migrations.Migration):
('ARGS', models.CharField(default='-c', max_length=255)),
],
options={
'verbose_name': 'Shell Environment: bash',
'abstract': False,
},
),
@ -56,6 +59,7 @@ class Migration(migrations.Migration):
('ARGS', models.CharField(default='', max_length=255)),
],
options={
'verbose_name': 'Package Manager: brew',
'abstract': False,
},
),
@ -68,6 +72,20 @@ class Migration(migrations.Migration):
('ARGS', models.CharField(default=[], editable=False, max_length=255)),
],
options={
'verbose_name': 'Internal Dependency: Django Package',
'abstract': False,
},
),
migrations.CreateModel(
name='NodeJSEnvironmentDependency',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('ENABLED', models.BooleanField(default=True)),
('BINARY', models.CharField(default='node', max_length=255)),
('ARGS', models.CharField(default='-c', max_length=255)),
],
options={
'verbose_name': 'Shell Environment: NodeJS',
'abstract': False,
},
),
@ -80,6 +98,7 @@ class Migration(migrations.Migration):
('ARGS', models.CharField(default='', max_length=255)),
],
options={
'verbose_name': 'Package Manager: npm',
'abstract': False,
},
),
@ -92,6 +111,20 @@ class Migration(migrations.Migration):
('ARGS', models.CharField(default='', max_length=255)),
],
options={
'verbose_name': 'Package Manager: pip',
'abstract': False,
},
),
migrations.CreateModel(
name='PythonEnvironmentDependency',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('ENABLED', models.BooleanField(default=True, editable=False)),
('BINARY', models.CharField(default='python3', max_length=255)),
('ARGS', models.CharField(default='-c', max_length=255)),
],
options={
'verbose_name': 'Shell Environment: Python3',
'abstract': False,
},
),
@ -104,6 +137,7 @@ class Migration(migrations.Migration):
('ARGS', models.CharField(default=[], editable=False, max_length=255)),
],
options={
'verbose_name': 'Internal Dependency: SQLite3 Package',
'abstract': False,
},
),

33
archivebox/plugins/system/migrations/0002_auto_20240124_0943.py Normal file
View file

@ -0,0 +1,33 @@
# Generated by Django 3.1.14 on 2024-01-24 09:43
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('system', '0001_initial'),
]
operations = [
migrations.AlterModelOptions(
name='archiveboxdependency',
options={'verbose_name': 'Internal Dependency: archivebox'},
),
migrations.AlterModelOptions(
name='djangodependency',
options={'verbose_name': 'Internal Dependency: django'},
),
migrations.AlterModelOptions(
name='nodejsenvironmentdependency',
options={'verbose_name': 'Shell Environment: node'},
),
migrations.AlterModelOptions(
name='pythonenvironmentdependency',
options={'verbose_name': 'Shell Environment: python3'},
),
migrations.AlterModelOptions(
name='sqlitedependency',
options={'verbose_name': 'Internal Dependency: sqlite3'},
),
]

22
archivebox/plugins/system/migrations/0003_auto_20240124_0956.py Normal file
View file

@ -0,0 +1,22 @@
# Generated by Django 3.1.14 on 2024-01-24 09:56
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('system', '0002_auto_20240124_0943'),
]
operations = [
migrations.AlterModelOptions(
name='bashenvironmentdependency',
options={'verbose_name': 'Shell Environment: bash', 'verbose_name_plural': 'Shell Environments: bash'},
),
migrations.AddField(
model_name='bashenvironmentdependency',
name='VERSION_CMD',
field=models.CharField(default='{BINARY} --version', max_length=255),
),
]

109
archivebox/plugins/system/models.py
View file

@ -6,7 +6,7 @@ import shutil
import sys
import inspect
import django
import sqlite3
from sqlite3 import dbapi2 as sqlite3
from pathlib import Path
from typing import List, Dict, Any
@ -16,12 +16,14 @@ from django.utils.functional import cached_property
from solo.models import SingletonModel
from plugins.defaults.models import ArchiveBoxBaseDependency, bin_path, bin_version
from config import bin_path, bin_version, VERSION
from plugins.defaults.models import ArchiveBoxBaseDependency
ConfigDict = Dict[str, Any]
class BashEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
class BashEnvironmentDependency(ArchiveBoxBaseDependency):
singleton_instance_id = 1
id = models.AutoField(primary_key=True)
@ -43,6 +45,53 @@ class BashEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
DEFAULT_STOP_CMD = None
DEFAULT_PID_FILE = None
DEFAULT_ARGS = '-c'
ENABLED = models.BooleanField(default=True, editable=not REQUIRED)
BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
VERSION_CMD = models.CharField(max_length=255, default='{BINARY} --version')
# START_CMD = models.CharField(max_length=255, default=DEFAULT_START_CMD)
# WORKERS = models.IntegerField(default=1)
class Meta:
abstract = False
app_label = 'system'
verbose_name = "Shell Environment: bash"
verbose_name_plural = "Shell Environments: bash"
# @task
def install_pkgs(self, os_pkgs=()):
assert self.is_valid, 'Bash environment is not available on this host'
for os_dependency in os_pkgs:
assert bin_path(os_dependency)
return True
class PythonEnvironmentDependency(ArchiveBoxBaseDependency):
singleton_instance_id = 1
id = models.AutoField(primary_key=True)
NAME = 'PYTHON'
LABEL = "Python"
REQUIRED = True
PARENT_DEPENDENCIES = []
BIN_DEPENDENCIES = ['python3']
APT_DEPENDENCIES = []
BREW_DEPENDENCIES = []
PIP_DEPENDENCIES = []
NPM_DEPENDENCIES = []
DEFAULT_BINARY = 'python3'
DEFAULT_START_CMD = None
DEFAULT_STOP_CMD = None
DEFAULT_PID_FILE = None
DEFAULT_ARGS = '-c'
VERSION_CMD = '{BINARY} --version'
ENABLED = models.BooleanField(default=True, editable=not REQUIRED)
@ -55,15 +104,44 @@ class BashEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
class Meta:
abstract = False
app_label = 'system'
verbose_name = "Shell Environment: python3"
# @task
def install_pkgs(self, os_pkgs=()):
assert self.is_valid, 'Bash environment is not available on this host'
class NodeJSEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
singleton_instance_id = 1
for os_dependency in os_pkgs:
assert bin_path(os_dependency)
id = models.AutoField(primary_key=True)
NAME = 'NODEJS'
LABEL = "NodeJS"
REQUIRED = True
PARENT_DEPENDENCIES = []
BIN_DEPENDENCIES = ['node']
APT_DEPENDENCIES = []
BREW_DEPENDENCIES = []
PIP_DEPENDENCIES = []
NPM_DEPENDENCIES = []
DEFAULT_BINARY = 'node'
DEFAULT_START_CMD = None
DEFAULT_STOP_CMD = None
DEFAULT_PID_FILE = None
DEFAULT_ARGS = '-c'
VERSION_CMD = '{BINARY} --version'
ENABLED = models.BooleanField(default=True, editable=True)
BINARY = models.CharField(max_length=255, default=DEFAULT_BINARY)
ARGS = models.CharField(max_length=255, default=DEFAULT_ARGS)
# START_CMD = models.CharField(max_length=255, default=DEFAULT_START_CMD)
# WORKERS = models.IntegerField(default=1)
class Meta:
abstract = False
app_label = 'system'
verbose_name = "Shell Environment: node"
return True
class AptEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
singleton_instance_id = 1
@ -95,6 +173,7 @@ class AptEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
class Meta:
abstract = False
app_label = 'system'
verbose_name = "Package Manager: apt"
# @task
def install_pkgs(self, apt_pkgs=()):
@ -136,6 +215,7 @@ class BrewEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
class Meta:
abstract = False
app_label = 'system'
verbose_name = "Package Manager: brew"
# @task
def install_pkgs(self, brew_pkgs=()):
@ -182,6 +262,7 @@ class PipEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
class Meta:
abstract = False
app_label = 'system'
verbose_name = "Package Manager: pip"
# @task
def install_pkgs(self, pip_pkgs=()):
@ -224,6 +305,7 @@ class NPMEnvironmentDependency(ArchiveBoxBaseDependency, SingletonModel):
class Meta:
abstract = False
app_label = 'system'
verbose_name = "Package Manager: npm"
# @task
def install_pkgs(self, npm_pkgs=()):
@ -266,6 +348,7 @@ class DjangoDependency(ArchiveBoxBaseDependency, SingletonModel):
class Meta:
abstract = False
app_label = 'system'
verbose_name = "Internal Dependency: django"
@cached_property
def bin_path(self):
@ -273,7 +356,7 @@ class DjangoDependency(ArchiveBoxBaseDependency, SingletonModel):
@cached_property
def bin_version(self):
return django.VERSION
return '.'.join(str(v) for v in django.VERSION[:3])
class SQLiteDependency(ArchiveBoxBaseDependency, SingletonModel):
@ -307,6 +390,7 @@ class SQLiteDependency(ArchiveBoxBaseDependency, SingletonModel):
class Meta:
abstract = False
app_label = 'system'
verbose_name = "Internal Dependency: sqlite3"
@cached_property
def bin_path(self):
@ -316,7 +400,7 @@ class SQLiteDependency(ArchiveBoxBaseDependency, SingletonModel):
def bin_version(self):
return sqlite3.version
class ArchiveBoxDependency(ArchiveBoxBaseDependency, SingletonModel):
class ArchiveBoxDependency(ArchiveBoxBaseDependency):
singleton_instance_id = 1
id = models.AutoField(primary_key=True)
@ -349,6 +433,7 @@ class ArchiveBoxDependency(ArchiveBoxBaseDependency, SingletonModel):
class Meta:
abstract = False
app_label = 'system'
verbose_name = "Internal Dependency: archivebox"
@cached_property
def bin_path(self):
@ -357,5 +442,5 @@ class ArchiveBoxDependency(ArchiveBoxBaseDependency, SingletonModel):
@cached_property
def bin_version(self):
# return config['VERSION']
return '0.7.3+editable'
return VERSION