mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-05-23 03:06:55 -04:00
BasePlugin system expanded and registration system improved
This commit is contained in:
Nick Sweeting
parent
f1579bfdcd
commit
9af260df16
50 changed files with 1062 additions and 973 deletions
|
|
@ -1,42 +1,31 @@
|
|||
from typing import List, Optional, Dict
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.core.checks import Tags, Warning, register
|
||||
|
||||
from pydantic import (
|
||||
Field,
|
||||
SerializeAsAny,
|
||||
)
|
||||
|
||||
from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
|
||||
# Depends on other PyPI/vendor packages:
|
||||
from pydantic import InstanceOf, Field
|
||||
from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName
|
||||
from pydantic_pkgr.binprovider import bin_abspath
|
||||
from pydantic_pkgr.binary import BinProviderName, ProviderLookupDict
|
||||
|
||||
from plugantic.extractors import Extractor, ExtractorName
|
||||
from plugantic.plugins import Plugin
|
||||
from plugantic.configs import ConfigSet, ConfigSectionName
|
||||
# Depends on other Django apps:
|
||||
from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseExtractor, BaseReplayer
|
||||
from plugantic.base_configset import ConfigSectionName
|
||||
|
||||
# Depends on Other Plugins:
|
||||
from pkg.settings import env
|
||||
from builtin_plugins.npm.apps import npm
|
||||
|
||||
|
||||
###################### Config ##########################
|
||||
|
||||
class SinglefileToggleConfig(ConfigSet):
|
||||
class SinglefileToggleConfigs(BaseConfigSet):
|
||||
section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
|
||||
|
||||
SAVE_SINGLEFILE: bool = True
|
||||
|
||||
|
||||
class SinglefileDependencyConfig(ConfigSet):
|
||||
section: ConfigSectionName = 'DEPENDENCY_CONFIG'
|
||||
|
||||
SINGLEFILE_BINARY: str = Field(default='wget')
|
||||
SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
|
||||
SINGLEFILE_EXTRA_ARGS: List[str] = []
|
||||
SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
|
||||
|
||||
class SinglefileOptionsConfig(ConfigSet):
|
||||
class SinglefileOptionsConfigs(BaseConfigSet):
|
||||
section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
|
||||
|
||||
# loaded from shared config
|
||||
|
|
@ -47,67 +36,83 @@ class SinglefileOptionsConfig(ConfigSet):
|
|||
SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
|
||||
|
||||
|
||||
class SinglefileDependencyConfigs(BaseConfigSet):
|
||||
section: ConfigSectionName = 'DEPENDENCY_CONFIG'
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
SINGLEFILE_BINARY: str = Field(default='wget')
|
||||
SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
|
||||
SINGLEFILE_EXTRA_ARGS: List[str] = []
|
||||
SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
|
||||
|
||||
class SinglefileConfigs(SinglefileToggleConfigs, SinglefileOptionsConfigs, SinglefileDependencyConfigs):
|
||||
# section: ConfigSectionName = 'ALL_CONFIGS'
|
||||
pass
|
||||
|
||||
DEFAULT_GLOBAL_CONFIG = {
|
||||
'CHECK_SSL_VALIDITY': False,
|
||||
'SAVE_SINGLEFILE': True,
|
||||
'TIMEOUT': 120,
|
||||
}
|
||||
|
||||
PLUGIN_CONFIG = [
|
||||
SinglefileToggleConfig(**DEFAULT_CONFIG),
|
||||
SinglefileDependencyConfig(**DEFAULT_CONFIG),
|
||||
SinglefileOptionsConfig(**DEFAULT_CONFIG),
|
||||
SINGLEFILE_CONFIGS = [
|
||||
SinglefileToggleConfigs(**DEFAULT_GLOBAL_CONFIG),
|
||||
SinglefileDependencyConfigs(**DEFAULT_GLOBAL_CONFIG),
|
||||
SinglefileOptionsConfigs(**DEFAULT_GLOBAL_CONFIG),
|
||||
]
|
||||
|
||||
###################### Binaries ############################
|
||||
|
||||
|
||||
min_version: str = "1.1.54"
|
||||
max_version: str = "2.0.0"
|
||||
|
||||
class SinglefileBinary(Binary):
|
||||
name: BinName = 'single-file'
|
||||
providers_supported: List[BinProvider] = [NpmProvider()]
|
||||
def get_singlefile_abspath() -> Optional[Path]:
|
||||
return
|
||||
|
||||
|
||||
class SinglefileBinary(BaseBinary):
|
||||
name: BinName = 'single-file'
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [env, npm]
|
||||
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] ={
|
||||
'env': {
|
||||
'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH),
|
||||
},
|
||||
'npm': {
|
||||
# 'abspath': lambda: bin_abspath('single-file', PATH=NpmProvider().PATH) or bin_abspath('single-file', PATH=env.PATH),
|
||||
'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}',
|
||||
},
|
||||
# 'env': {
|
||||
# 'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH),
|
||||
# },
|
||||
# 'npm': {
|
||||
# 'abspath': lambda: bin_abspath('single-file', PATH=npm.PATH) or bin_abspath('single-file-node.js', PATH=npm.PATH),
|
||||
# 'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}',
|
||||
# },
|
||||
}
|
||||
|
||||
SINGLEFILE_BINARY = SinglefileBinary()
|
||||
|
||||
###################### Extractors ##########################
|
||||
PLUGIN_BINARIES = [SINGLEFILE_BINARY]
|
||||
|
||||
class SinglefileExtractor(Extractor):
|
||||
name: ExtractorName = 'singlefile'
|
||||
binary: Binary = SinglefileBinary()
|
||||
class SinglefileExtractor(BaseExtractor):
|
||||
name: str = 'singlefile'
|
||||
binary: BinName = SINGLEFILE_BINARY.name
|
||||
|
||||
def get_output_path(self, snapshot) -> Path:
|
||||
return Path(snapshot.link_dir) / 'singlefile.html'
|
||||
|
||||
|
||||
###################### Plugins #############################
|
||||
SINGLEFILE_BINARY = SinglefileBinary()
|
||||
SINGLEFILE_EXTRACTOR = SinglefileExtractor()
|
||||
|
||||
class SinglefilePlugin(BasePlugin):
|
||||
name: str = 'builtin_plugins.singlefile'
|
||||
app_label: str ='singlefile'
|
||||
verbose_name: str = 'SingleFile'
|
||||
|
||||
configs: List[InstanceOf[BaseConfigSet]] = SINGLEFILE_CONFIGS
|
||||
binaries: List[InstanceOf[BaseBinary]] = [SINGLEFILE_BINARY]
|
||||
extractors: List[InstanceOf[BaseExtractor]] = [SINGLEFILE_EXTRACTOR]
|
||||
|
||||
|
||||
class SinglefilePlugin(Plugin):
|
||||
name: str = 'singlefile'
|
||||
configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
|
||||
binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
|
||||
extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
|
||||
|
||||
PLUGINS = [SinglefilePlugin()]
|
||||
|
||||
###################### Django Apps #########################
|
||||
|
||||
class SinglefileConfig(AppConfig):
|
||||
name = 'builtin_plugins.singlefile'
|
||||
verbose_name = 'SingleFile'
|
||||
|
||||
def ready(self):
|
||||
pass
|
||||
# print('Loaded singlefile plugin')
|
||||
PLUGIN = SinglefilePlugin()
|
||||
DJANGO_APP = PLUGIN.AppConfig
|
||||
# CONFIGS = PLUGIN.configs
|
||||
# BINARIES = PLUGIN.binaries
|
||||
# EXTRACTORS = PLUGIN.extractors
|
||||
# REPLAYERS = PLUGIN.replayers
|
||||
# CHECKS = PLUGIN.checks
|
||||
|
|
|
|||
|
|
@ -1,66 +0,0 @@
|
|||
name: singlefile
|
||||
plugin_version: '0.0.1'
|
||||
plugin_spec: '0.0.1'
|
||||
|
||||
binaries:
|
||||
singlefile:
|
||||
providers:
|
||||
- env
|
||||
- npm
|
||||
|
||||
commands:
|
||||
- singlefile.exec
|
||||
- singlefile.extract
|
||||
- singlefile.should_extract
|
||||
- singlefile.get_output_path
|
||||
|
||||
extractors:
|
||||
singlefile:
|
||||
binary: singlefile
|
||||
test: singlefile.should_extract
|
||||
extract: singlefile.extract
|
||||
output_files:
|
||||
- singlefile.html
|
||||
|
||||
configs:
|
||||
ARCHIVE_METHOD_TOGGLES:
|
||||
SAVE_SINGLEFILE:
|
||||
type: bool
|
||||
default: true
|
||||
|
||||
DEPENDENCY_CONFIG:
|
||||
SINGLEFILE_BINARY:
|
||||
type: str
|
||||
default: wget
|
||||
SINGLEFILE_ARGS:
|
||||
type: Optional[List[str]]
|
||||
default: null
|
||||
SINGLEFILE_EXTRA_ARGS:
|
||||
type: List[str]
|
||||
default: []
|
||||
SINGLEFILE_DEFAULT_ARGS:
|
||||
type: List[str]
|
||||
default:
|
||||
- "--timeout={TIMEOUT-10}"
|
||||
|
||||
ARCHIVE_METHOD_OPTIONS:
|
||||
SINGLEFILE_USER_AGENT:
|
||||
type: str
|
||||
default: ""
|
||||
alias: USER_AGENT
|
||||
SINGLEFILE_TIMEOUT:
|
||||
type: int
|
||||
default: 60
|
||||
alias: TIMEOUT
|
||||
SINGLEFILE_CHECK_SSL_VALIDITY:
|
||||
type: bool
|
||||
default: true
|
||||
alias: CHECK_SSL_VALIDITY
|
||||
SINGLEFILE_RESTRICT_FILE_NAMES:
|
||||
type: str
|
||||
default: windows
|
||||
alias: RESTRICT_FILE_NAMES
|
||||
SINGLEFILE_COOKIES_FILE:
|
||||
type: Optional[Path]
|
||||
default: null
|
||||
alias: COOKIES_FILE
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
from django.test import TestCase
|
||||
|
||||
# Create your tests here.
|
||||
Loading…
Add table
Add a link
Reference in a new issue