mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-05-19 09:25:11 -04:00
new vastly simplified plugin spec without pydantic
Some checks are pending
Build Debian package / build (push) Waiting to run
Build Docker image / buildx (push) Waiting to run
Build Homebrew package / build (push) Waiting to run
Run linters / lint (push) Waiting to run
Build Pip package / build (push) Waiting to run
Run tests / python_tests (ubuntu-22.04, 3.11) (push) Waiting to run
Run tests / docker_tests (push) Waiting to run
Some checks are pending
Build Debian package / build (push) Waiting to run
Build Docker image / buildx (push) Waiting to run
Build Homebrew package / build (push) Waiting to run
Run linters / lint (push) Waiting to run
Build Pip package / build (push) Waiting to run
Run tests / python_tests (ubuntu-22.04, 3.11) (push) Waiting to run
Run tests / docker_tests (push) Waiting to run
This commit is contained in:
Nick Sweeting
parent
abf75f49f4
commit
01ba6d49d3
115 changed files with 2466 additions and 2301 deletions
|
|
@ -0,0 +1,37 @@
|
|||
__package__ = 'plugins_extractor.ytdlp'
|
||||
__label__ = 'YT-DLP'
|
||||
__version__ = '2024.10.14'
|
||||
__author__ = 'Nick Sweeting'
|
||||
__homepage__ = 'https://github.com/yt-dlp/yt-dlp'
|
||||
|
||||
import abx
|
||||
|
||||
|
||||
@abx.hookimpl
|
||||
def get_PLUGIN():
|
||||
return {
|
||||
'ytdlp': {
|
||||
'PACKAGE': __package__,
|
||||
'LABEL': __label__,
|
||||
'VERSION': __version__,
|
||||
'AUTHOR': __author__,
|
||||
'HOMEPAGE': __homepage__,
|
||||
}
|
||||
}
|
||||
|
||||
@abx.hookimpl
|
||||
def get_CONFIG():
|
||||
from .config import YTDLP_CONFIG
|
||||
|
||||
return {
|
||||
'ytdlp': YTDLP_CONFIG
|
||||
}
|
||||
|
||||
@abx.hookimpl
|
||||
def get_BINARIES():
|
||||
from .binaries import YTDLP_BINARY, FFMPEG_BINARY
|
||||
|
||||
return {
|
||||
'ytdlp': YTDLP_BINARY,
|
||||
'ffmpeg': FFMPEG_BINARY,
|
||||
}
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
import sys
|
||||
from typing import List
|
||||
from subprocess import run, PIPE
|
||||
|
||||
from rich import print
|
||||
from pydantic import InstanceOf, Field, model_validator, AliasChoices
|
||||
from pydantic_pkgr import BinProvider, BinName, BinaryOverrides
|
||||
|
||||
from abx.archivebox.base_plugin import BasePlugin
|
||||
from abx.archivebox.base_configset import BaseConfigSet
|
||||
from abx.archivebox.base_binary import BaseBinary, env, apt, brew
|
||||
from abx.archivebox.base_hook import BaseHook
|
||||
|
||||
from archivebox.config.common import ARCHIVING_CONFIG
|
||||
from plugins_pkg.pip.apps import pip
|
||||
|
||||
###################### Config ##########################
|
||||
|
||||
|
||||
class YtdlpConfig(BaseConfigSet):
|
||||
USE_YTDLP: bool = Field(default=True, validation_alias=AliasChoices('USE_YOUTUBEDL', 'SAVE_MEDIA'))
|
||||
|
||||
YTDLP_BINARY: str = Field(default='yt-dlp', alias='YOUTUBEDL_BINARY')
|
||||
YTDLP_EXTRA_ARGS: List[str] = Field(default=[], alias='YOUTUBEDL_EXTRA_ARGS')
|
||||
|
||||
YTDLP_CHECK_SSL_VALIDITY: bool = Field(default=lambda: ARCHIVING_CONFIG.CHECK_SSL_VALIDITY)
|
||||
YTDLP_TIMEOUT: int = Field(default=lambda: ARCHIVING_CONFIG.MEDIA_TIMEOUT)
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate_use_ytdlp(self):
|
||||
if self.USE_YTDLP and self.YTDLP_TIMEOUT < 20:
|
||||
print(f'[red][!] Warning: MEDIA_TIMEOUT is set too low! (currently set to MEDIA_TIMEOUT={self.YTDLP_TIMEOUT} seconds)[/red]', file=sys.stderr)
|
||||
print(' youtube-dl/yt-dlp will fail to archive any media if set to less than ~20 seconds.', file=sys.stderr)
|
||||
print(' (Setting it somewhere over 60 seconds is recommended)', file=sys.stderr)
|
||||
print(file=sys.stderr)
|
||||
print(' If you want to disable media archiving entirely, set SAVE_MEDIA=False instead:', file=sys.stderr)
|
||||
print(' https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_media', file=sys.stderr)
|
||||
print(file=sys.stderr)
|
||||
return self
|
||||
|
||||
|
||||
YTDLP_CONFIG = YtdlpConfig()
|
||||
|
||||
|
||||
|
||||
class YtdlpBinary(BaseBinary):
|
||||
name: BinName = YTDLP_CONFIG.YTDLP_BINARY
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
|
||||
|
||||
YTDLP_BINARY = YtdlpBinary()
|
||||
|
||||
|
||||
class FfmpegBinary(BaseBinary):
|
||||
name: BinName = 'ffmpeg'
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
|
||||
|
||||
overrides: BinaryOverrides = {
|
||||
'env': {
|
||||
# 'abspath': lambda: shutil.which('ffmpeg', PATH=env.PATH),
|
||||
'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout,
|
||||
},
|
||||
'apt': {
|
||||
# 'abspath': lambda: shutil.which('ffmpeg', PATH=apt.PATH),
|
||||
'version': lambda: run(['apt', 'show', 'ffmpeg'], stdout=PIPE, stderr=PIPE, text=True).stdout,
|
||||
},
|
||||
'brew': {
|
||||
# 'abspath': lambda: shutil.which('ffmpeg', PATH=brew.PATH),
|
||||
'version': lambda: run(['brew', 'info', 'ffmpeg', '--quiet'], stdout=PIPE, stderr=PIPE, text=True).stdout,
|
||||
},
|
||||
}
|
||||
|
||||
# def get_ffmpeg_version(self) -> Optional[str]:
|
||||
# return self.exec(cmd=['-version']).stdout
|
||||
|
||||
FFMPEG_BINARY = FfmpegBinary()
|
||||
|
||||
|
||||
# class YtdlpExtractor(BaseExtractor):
|
||||
# name: str = 'ytdlp'
|
||||
# binary: str = 'ytdlp'
|
||||
|
||||
|
||||
|
||||
class YtdlpPlugin(BasePlugin):
|
||||
app_label: str = 'ytdlp'
|
||||
verbose_name: str = 'YT-DLP'
|
||||
docs_url: str = 'https://github.com/yt-dlp/yt-dlp'
|
||||
|
||||
hooks: List[InstanceOf[BaseHook]] = [
|
||||
YTDLP_CONFIG,
|
||||
YTDLP_BINARY,
|
||||
FFMPEG_BINARY,
|
||||
]
|
||||
|
||||
|
||||
PLUGIN = YtdlpPlugin()
|
||||
# PLUGIN.register(settings)
|
||||
DJANGO_APP = PLUGIN.AppConfig
|
||||
42
archivebox/plugins_extractor/ytdlp/binaries.py
Normal file
42
archivebox/plugins_extractor/ytdlp/binaries.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
__package__ = 'plugins_extractor.ytdlp'
|
||||
|
||||
import subprocess
|
||||
from typing import List
|
||||
|
||||
from pydantic import InstanceOf
|
||||
from pydantic_pkgr import BinProvider, BinName, BinaryOverrides
|
||||
|
||||
from abx.archivebox.base_binary import BaseBinary, env, apt, brew
|
||||
|
||||
from plugins_pkg.pip.binproviders import LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER
|
||||
|
||||
from .config import YTDLP_CONFIG
|
||||
|
||||
|
||||
class YtdlpBinary(BaseBinary):
|
||||
name: BinName = YTDLP_CONFIG.YTDLP_BINARY
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
|
||||
|
||||
YTDLP_BINARY = YtdlpBinary()
|
||||
|
||||
|
||||
class FfmpegBinary(BaseBinary):
|
||||
name: BinName = 'ffmpeg'
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
|
||||
|
||||
overrides: BinaryOverrides = {
|
||||
'env': {
|
||||
# 'abspath': lambda: shutil.which('ffmpeg', PATH=env.PATH),
|
||||
'version': lambda: subprocess.run(['ffmpeg', '-version'], capture_output=True, text=True).stdout,
|
||||
},
|
||||
'apt': {
|
||||
# 'abspath': lambda: shutil.which('ffmpeg', PATH=apt.PATH),
|
||||
'version': lambda: subprocess.run(['apt', 'show', 'ffmpeg'], capture_output=True, text=True).stdout,
|
||||
},
|
||||
'brew': {
|
||||
# 'abspath': lambda: shutil.which('ffmpeg', PATH=brew.PATH),
|
||||
'version': lambda: subprocess.run(['brew', 'info', 'ffmpeg', '--quiet'], capture_output=True, text=True).stdout,
|
||||
},
|
||||
}
|
||||
|
||||
FFMPEG_BINARY = FfmpegBinary()
|
||||
35
archivebox/plugins_extractor/ytdlp/config.py
Normal file
35
archivebox/plugins_extractor/ytdlp/config.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
__package__ = 'plugins_extractor.ytdlp'
|
||||
|
||||
from typing import List
|
||||
|
||||
from pydantic import Field, model_validator, AliasChoices
|
||||
|
||||
from abx.archivebox.base_configset import BaseConfigSet
|
||||
|
||||
from archivebox.config.common import ARCHIVING_CONFIG
|
||||
from archivebox.misc.logging import STDERR
|
||||
|
||||
|
||||
class YtdlpConfig(BaseConfigSet):
|
||||
USE_YTDLP: bool = Field(default=True, validation_alias=AliasChoices('USE_YOUTUBEDL', 'SAVE_MEDIA'))
|
||||
|
||||
YTDLP_BINARY: str = Field(default='yt-dlp', alias='YOUTUBEDL_BINARY')
|
||||
YTDLP_EXTRA_ARGS: List[str] = Field(default=[], alias='YOUTUBEDL_EXTRA_ARGS')
|
||||
|
||||
YTDLP_CHECK_SSL_VALIDITY: bool = Field(default=lambda: ARCHIVING_CONFIG.CHECK_SSL_VALIDITY)
|
||||
YTDLP_TIMEOUT: int = Field(default=lambda: ARCHIVING_CONFIG.MEDIA_TIMEOUT)
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate_use_ytdlp(self):
|
||||
if self.USE_YTDLP and self.YTDLP_TIMEOUT < 20:
|
||||
STDERR.print(f'[red][!] Warning: MEDIA_TIMEOUT is set too low! (currently set to MEDIA_TIMEOUT={self.YTDLP_TIMEOUT} seconds)[/red]')
|
||||
STDERR.print(' youtube-dl/yt-dlp will fail to archive any media if set to less than ~20 seconds.')
|
||||
STDERR.print(' (Setting it somewhere over 60 seconds is recommended)')
|
||||
STDERR.print()
|
||||
STDERR.print(' If you want to disable media archiving entirely, set SAVE_MEDIA=False instead:')
|
||||
STDERR.print(' https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_media')
|
||||
STDERR.print()
|
||||
return self
|
||||
|
||||
|
||||
YTDLP_CONFIG = YtdlpConfig()
|
||||
Loading…
Add table
Add a link
Reference in a new issue