mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-05-23 19:27:00 -04:00
add new binproviders and binaries args to install and version, bump pydantic-pkgr version
This commit is contained in:
Nick Sweeting
parent
fbd2c458c3
commit
6e7071bd19
24 changed files with 318 additions and 235 deletions
|
|
@ -1,11 +1,11 @@
|
|||
__package__ = 'archivebox.plugins_pkg.npm'
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Dict
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import InstanceOf, model_validator
|
||||
|
||||
from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict
|
||||
from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr, BinProviderName, BinaryOverrides
|
||||
|
||||
from archivebox.config import DATA_DIR, CONSTANTS
|
||||
|
||||
|
|
@ -60,8 +60,8 @@ class NodeBinary(BaseBinary):
|
|||
name: BinName = 'node'
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
|
||||
|
||||
overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
apt.name: {'packages': lambda c: ['nodejs']},
|
||||
overrides: BinaryOverrides = {
|
||||
apt.name: {'packages': ['nodejs']},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -72,7 +72,7 @@ class NpmBinary(BaseBinary):
|
|||
name: BinName = 'npm'
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
|
||||
|
||||
overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
overrides: BinaryOverrides = {
|
||||
apt.name: {'install': lambda: None}, # already installed when nodejs is installed
|
||||
brew.name: {'install': lambda: None}, # already installed when nodejs is installed
|
||||
}
|
||||
|
|
@ -84,7 +84,7 @@ class NpxBinary(BaseBinary):
|
|||
name: BinName = 'npx'
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
|
||||
|
||||
overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
overrides: BinaryOverrides = {
|
||||
apt.name: {'install': lambda: None}, # already installed when nodejs is installed
|
||||
brew.name: {'install': lambda: None}, # already installed when nodejs is installed
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,14 +4,14 @@ import os
|
|||
import sys
|
||||
import site
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
from typing import List, Optional
|
||||
from pydantic import InstanceOf, Field, model_validator, validate_call
|
||||
|
||||
|
||||
import django
|
||||
import django.db.backends.sqlite3.base
|
||||
from django.db.backends.sqlite3.base import Database as django_sqlite3 # type: ignore[import-type]
|
||||
from pydantic_pkgr import BinProvider, PipProvider, BinName, BinProviderName, ProviderLookupDict, SemVer
|
||||
from pydantic_pkgr import BinProvider, PipProvider, BinName, BinProviderName, BinaryOverrides, SemVer
|
||||
|
||||
from archivebox.config import CONSTANTS, VERSION
|
||||
|
||||
|
|
@ -105,18 +105,18 @@ class ArchiveboxBinary(BaseBinary):
|
|||
name: BinName = 'archivebox'
|
||||
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
VENV_PIP_BINPROVIDER.name: {'packages': lambda: [], 'version': lambda: VERSION},
|
||||
SYS_PIP_BINPROVIDER.name: {'packages': lambda: [], 'version': lambda: VERSION},
|
||||
apt.name: {'packages': lambda: [], 'version': lambda: VERSION},
|
||||
brew.name: {'packages': lambda: [], 'version': lambda: VERSION},
|
||||
overrides: BinaryOverrides = {
|
||||
VENV_PIP_BINPROVIDER.name: {'packages': [], 'version': VERSION},
|
||||
SYS_PIP_BINPROVIDER.name: {'packages': [], 'version': VERSION},
|
||||
apt.name: {'packages': [], 'version': VERSION},
|
||||
brew.name: {'packages': [], 'version': VERSION},
|
||||
}
|
||||
|
||||
@validate_call
|
||||
# @validate_call
|
||||
def install(self, **kwargs):
|
||||
return self.load() # obviously it's already installed if we are running this ;)
|
||||
|
||||
@validate_call
|
||||
# @validate_call
|
||||
def load_or_install(self, **kwargs):
|
||||
return self.load() # obviously it's already installed if we are running this ;)
|
||||
|
||||
|
|
@ -127,18 +127,18 @@ class PythonBinary(BaseBinary):
|
|||
name: BinName = 'python'
|
||||
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
overrides: BinaryOverrides = {
|
||||
SYS_PIP_BINPROVIDER.name: {
|
||||
'abspath': lambda: sys.executable,
|
||||
'version': lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
|
||||
'abspath': sys.executable,
|
||||
'version': '{}.{}.{}'.format(*sys.version_info[:3]),
|
||||
},
|
||||
}
|
||||
|
||||
@validate_call
|
||||
# @validate_call
|
||||
def install(self, **kwargs):
|
||||
return self.load() # obviously it's already installed if we are running this ;)
|
||||
|
||||
@validate_call
|
||||
# @validate_call
|
||||
def load_or_install(self, **kwargs):
|
||||
return self.load() # obviously it's already installed if we are running this ;)
|
||||
|
||||
|
|
@ -152,14 +152,14 @@ LOADED_SQLITE_FROM_VENV = str(LOADED_SQLITE_PATH.absolute().resolve()).startswit
|
|||
class SqliteBinary(BaseBinary):
|
||||
name: BinName = 'sqlite'
|
||||
binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER])
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
overrides: BinaryOverrides = {
|
||||
VENV_PIP_BINPROVIDER.name: {
|
||||
"abspath": lambda: LOADED_SQLITE_PATH if LOADED_SQLITE_FROM_VENV else None,
|
||||
"version": lambda: LOADED_SQLITE_VERSION if LOADED_SQLITE_FROM_VENV else None,
|
||||
"abspath": LOADED_SQLITE_PATH if LOADED_SQLITE_FROM_VENV else None,
|
||||
"version": LOADED_SQLITE_VERSION if LOADED_SQLITE_FROM_VENV else None,
|
||||
},
|
||||
SYS_PIP_BINPROVIDER.name: {
|
||||
"abspath": lambda: LOADED_SQLITE_PATH if not LOADED_SQLITE_FROM_VENV else None,
|
||||
"version": lambda: LOADED_SQLITE_VERSION if not LOADED_SQLITE_FROM_VENV else None,
|
||||
"abspath": LOADED_SQLITE_PATH if not LOADED_SQLITE_FROM_VENV else None,
|
||||
"version": LOADED_SQLITE_VERSION if not LOADED_SQLITE_FROM_VENV else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -177,11 +177,11 @@ class SqliteBinary(BaseBinary):
|
|||
])
|
||||
return self
|
||||
|
||||
@validate_call
|
||||
# @validate_call
|
||||
def install(self, **kwargs):
|
||||
return self.load() # obviously it's already installed if we are running this ;)
|
||||
|
||||
@validate_call
|
||||
# @validate_call
|
||||
def load_or_install(self, **kwargs):
|
||||
return self.load() # obviously it's already installed if we are running this ;)
|
||||
|
||||
|
|
@ -196,22 +196,22 @@ class DjangoBinary(BaseBinary):
|
|||
name: BinName = 'django'
|
||||
|
||||
binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER])
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
overrides: BinaryOverrides = {
|
||||
VENV_PIP_BINPROVIDER.name: {
|
||||
"abspath": lambda: LOADED_DJANGO_PATH if LOADED_DJANGO_FROM_VENV else None,
|
||||
"version": lambda: LOADED_DJANGO_VERSION if LOADED_DJANGO_FROM_VENV else None,
|
||||
"abspath": LOADED_DJANGO_PATH if LOADED_DJANGO_FROM_VENV else None,
|
||||
"version": LOADED_DJANGO_VERSION if LOADED_DJANGO_FROM_VENV else None,
|
||||
},
|
||||
SYS_PIP_BINPROVIDER.name: {
|
||||
"abspath": lambda: LOADED_DJANGO_PATH if not LOADED_DJANGO_FROM_VENV else None,
|
||||
"version": lambda: LOADED_DJANGO_VERSION if not LOADED_DJANGO_FROM_VENV else None,
|
||||
"abspath": LOADED_DJANGO_PATH if not LOADED_DJANGO_FROM_VENV else None,
|
||||
"version": LOADED_DJANGO_VERSION if not LOADED_DJANGO_FROM_VENV else None,
|
||||
},
|
||||
}
|
||||
|
||||
@validate_call
|
||||
# @validate_call
|
||||
def install(self, **kwargs):
|
||||
return self.load() # obviously it's already installed if we are running this ;)
|
||||
|
||||
@validate_call
|
||||
# @validate_call
|
||||
def load_or_install(self, **kwargs):
|
||||
return self.load() # obviously it's already installed if we are running this ;)
|
||||
|
||||
|
|
@ -221,11 +221,11 @@ class PipBinary(BaseBinary):
|
|||
name: BinName = "pip"
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
|
||||
|
||||
@validate_call
|
||||
# @validate_call
|
||||
def install(self, **kwargs):
|
||||
return self.load() # obviously it's already installed if we are running this ;)
|
||||
|
||||
@validate_call
|
||||
# @validate_call
|
||||
def load_or_install(self, **kwargs):
|
||||
return self.load() # obviously it's already installed if we are running this ;)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from pydantic_pkgr import (
|
|||
BinName,
|
||||
BinProvider,
|
||||
BinProviderName,
|
||||
ProviderLookupDict,
|
||||
BinProviderOverrides,
|
||||
InstallArgs,
|
||||
PATHStr,
|
||||
HostBinPath,
|
||||
|
|
@ -66,15 +66,15 @@ class PlaywrightBinProvider(BaseBinProvider):
|
|||
|
||||
PATH: PATHStr = f"{CONSTANTS.LIB_BIN_DIR}:{DEFAULT_ENV_PATH}"
|
||||
|
||||
playwright_browsers_dir: Optional[Path] = (
|
||||
playwright_browsers_dir: Path = (
|
||||
Path("~/Library/Caches/ms-playwright").expanduser() # macos playwright cache dir
|
||||
if OPERATING_SYSTEM == "darwin" else
|
||||
Path("~/.cache/ms-playwright").expanduser() # linux playwright cache dir
|
||||
)
|
||||
playwright_install_args: List[str] = ["install"] # --with-deps
|
||||
|
||||
packages_handler: ProviderLookupDict = Field(default={
|
||||
"chrome": lambda: ["chromium"],
|
||||
packages_handler: BinProviderOverrides = Field(default={
|
||||
"chrome": ["chromium"],
|
||||
}, exclude=True)
|
||||
|
||||
_browser_abspaths: ClassVar[Dict[str, HostBinPath]] = {}
|
||||
|
|
@ -104,9 +104,17 @@ class PlaywrightBinProvider(BaseBinProvider):
|
|||
)
|
||||
|
||||
# ~/Library/caches/ms-playwright/chromium-1097/chrome-linux/chromium
|
||||
return sorted(self.playwright_browsers_dir.glob(f"{browser_name}-*/*-linux/*"))
|
||||
paths = []
|
||||
for path in sorted(self.playwright_browsers_dir.glob(f"{browser_name}-*/*-linux/*")):
|
||||
if 'xdg-settings' in str(path):
|
||||
continue
|
||||
if 'ffmpeg' in str(path):
|
||||
continue
|
||||
if '/chrom' in str(path) and 'chrom' in path.name.lower():
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
|
||||
def default_abspath_handler(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
|
||||
assert bin_name == "chrome", "Only chrome is supported using the @puppeteer/browsers install method currently."
|
||||
|
||||
# already loaded, return abspath from cache
|
||||
|
|
@ -128,7 +136,7 @@ class PlaywrightBinProvider(BaseBinProvider):
|
|||
|
||||
return None
|
||||
|
||||
def on_install(self, bin_name: str, packages: Optional[InstallArgs] = None, **context) -> str:
|
||||
def default_install_handler(self, bin_name: str, packages: Optional[InstallArgs] = None, **context) -> str:
|
||||
"""playwright install chrome"""
|
||||
self.setup()
|
||||
assert bin_name == "chrome", "Only chrome is supported using the playwright install method currently."
|
||||
|
|
@ -137,7 +145,7 @@ class PlaywrightBinProvider(BaseBinProvider):
|
|||
raise Exception(
|
||||
f"{self.__class__.__name__} install method is not available on this host ({self.INSTALLER_BIN} not found in $PATH)"
|
||||
)
|
||||
packages = packages or self.on_get_packages(bin_name)
|
||||
packages = packages or self.get_packages(bin_name)
|
||||
|
||||
# print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}')
|
||||
|
||||
|
|
@ -155,7 +163,7 @@ class PlaywrightBinProvider(BaseBinProvider):
|
|||
output_lines = [
|
||||
line for line in proc.stdout.strip().split('\n')
|
||||
if '/chrom' in line
|
||||
and 'chrom' in line.rsplit('/', 1)[-1].lower() # make final path segment (filename) contains chrome or chromium
|
||||
and 'chrom' in line.rsplit('/', 1)[-1].lower() # if final path segment (filename) contains chrome or chromium
|
||||
and 'xdg-settings' not in line
|
||||
and 'ffmpeg' not in line
|
||||
]
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from pydantic_pkgr import (
|
|||
BinProvider,
|
||||
BinName,
|
||||
BinProviderName,
|
||||
ProviderLookupDict,
|
||||
BinProviderOverrides,
|
||||
InstallArgs,
|
||||
PATHStr,
|
||||
HostBinPath,
|
||||
|
|
@ -65,10 +65,10 @@ class PuppeteerBinProvider(BaseBinProvider):
|
|||
|
||||
euid: Optional[int] = ARCHIVEBOX_USER
|
||||
|
||||
puppeteer_browsers_dir: Optional[Path] = LIB_DIR_BROWSERS
|
||||
puppeteer_browsers_dir: Path = LIB_DIR_BROWSERS
|
||||
puppeteer_install_args: List[str] = ["@puppeteer/browsers", "install", "--path", str(LIB_DIR_BROWSERS)]
|
||||
|
||||
packages_handler: ProviderLookupDict = Field(default={
|
||||
packages_handler: BinProviderOverrides = Field(default={
|
||||
"chrome": lambda:
|
||||
['chrome@stable'],
|
||||
}, exclude=True)
|
||||
|
|
@ -90,7 +90,7 @@ class PuppeteerBinProvider(BaseBinProvider):
|
|||
# /data/lib/browsers/chrome/linux-131.0.6730.0/chrome-linux64/chrome
|
||||
return sorted(self.puppeteer_browsers_dir.glob(f"{browser_name}/linux*/chrome*/chrome"))
|
||||
|
||||
def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
|
||||
def default_abspath_handler(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
|
||||
assert bin_name == 'chrome', 'Only chrome is supported using the @puppeteer/browsers install method currently.'
|
||||
|
||||
# already loaded, return abspath from cache
|
||||
|
|
@ -106,7 +106,7 @@ class PuppeteerBinProvider(BaseBinProvider):
|
|||
|
||||
return None
|
||||
|
||||
def on_install(self, bin_name: str, packages: Optional[InstallArgs] = None, **context) -> str:
|
||||
def default_install_handler(self, bin_name: str, packages: Optional[InstallArgs] = None, **context) -> str:
|
||||
"""npx @puppeteer/browsers install chrome@stable"""
|
||||
self.setup()
|
||||
assert bin_name == 'chrome', 'Only chrome is supported using the @puppeteer/browsers install method currently.'
|
||||
|
|
@ -115,7 +115,7 @@ class PuppeteerBinProvider(BaseBinProvider):
|
|||
raise Exception(
|
||||
f"{self.__class__.__name__} install method is not available on this host ({self.INSTALLER_BIN} not found in $PATH)"
|
||||
)
|
||||
packages = packages or self.on_get_packages(bin_name)
|
||||
packages = packages or self.get_packages(bin_name)
|
||||
assert packages, f"No packages specified for installation of {bin_name}"
|
||||
|
||||
# print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue