mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-05-21 02:15:10 -04:00
add new binproviders and binaries args to install and version, bump pydantic-pkgr version
This commit is contained in:
Nick Sweeting
parent
fbd2c458c3
commit
6e7071bd19
24 changed files with 318 additions and 235 deletions
|
|
@ -13,7 +13,7 @@ from pydantic_pkgr import (
|
|||
BinProvider,
|
||||
BinName,
|
||||
BinProviderName,
|
||||
ProviderLookupDict,
|
||||
BinaryOverrides,
|
||||
bin_abspath,
|
||||
)
|
||||
|
||||
|
|
@ -204,15 +204,15 @@ class ChromeBinary(BaseBinary):
|
|||
name: BinName = CHROME_CONFIG.CHROME_BINARY
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [PUPPETEER_BINPROVIDER, env, PLAYWRIGHT_BINPROVIDER]
|
||||
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
overrides: BinaryOverrides = {
|
||||
env.name: {
|
||||
'abspath': lambda: autodetect_system_chrome_install(PATH=env.PATH), # /usr/bin/google-chrome-stable
|
||||
},
|
||||
PUPPETEER_BINPROVIDER.name: {
|
||||
'packages': lambda: ['chrome@stable'], # npx @puppeteer/browsers install chrome@stable
|
||||
'packages': ['chrome@stable'], # npx @puppeteer/browsers install chrome@stable
|
||||
},
|
||||
PLAYWRIGHT_BINPROVIDER.name: {
|
||||
'packages': lambda: ['chromium'], # playwright install chromium
|
||||
'packages': ['chromium'], # playwright install chromium
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
__package__ = 'plugins_extractor.mercury'
|
||||
|
||||
from typing import List, Optional, Dict
|
||||
from typing import List, Optional
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import InstanceOf, Field
|
||||
from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict, bin_abspath
|
||||
from pydantic_pkgr import BinProvider, BinName, BinaryOverrides, bin_abspath
|
||||
|
||||
from abx.archivebox.base_plugin import BasePlugin, BaseHook
|
||||
from abx.archivebox.base_configset import BaseConfigSet
|
||||
|
|
@ -38,13 +38,13 @@ class MercuryBinary(BaseBinary):
|
|||
name: BinName = MERCURY_CONFIG.MERCURY_BINARY
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_NPM_BINPROVIDER, SYS_NPM_BINPROVIDER, env]
|
||||
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
overrides: BinaryOverrides = {
|
||||
LIB_NPM_BINPROVIDER.name: {
|
||||
'packages': lambda: ['@postlight/parser@^2.2.3'],
|
||||
'packages': ['@postlight/parser@^2.2.3'],
|
||||
},
|
||||
SYS_NPM_BINPROVIDER.name: {
|
||||
'packages': lambda: ['@postlight/parser@^2.2.3'],
|
||||
'install': lambda: False, # never try to install things into global prefix
|
||||
'packages': ['@postlight/parser@^2.2.3'],
|
||||
'install': lambda: None, # never try to install things into global prefix
|
||||
},
|
||||
env.name: {
|
||||
'version': lambda: '999.999.999' if bin_abspath('postlight-parser', PATH=env.PATH) else None,
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
__package__ = 'archivebox.plugins_extractor.readability'
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
from typing import List
|
||||
# from typing_extensions import Self
|
||||
|
||||
# Depends on other PyPI/vendor packages:
|
||||
from pydantic import InstanceOf, Field, validate_call
|
||||
from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, ShallowBinary
|
||||
from pydantic import InstanceOf, Field
|
||||
from pydantic_pkgr import BinProvider, BinaryOverrides, BinName
|
||||
|
||||
# Depends on other Django apps:
|
||||
from abx.archivebox.base_plugin import BasePlugin
|
||||
|
|
@ -39,23 +39,10 @@ class ReadabilityBinary(BaseBinary):
|
|||
name: BinName = READABILITY_CONFIG.READABILITY_BINARY
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_NPM_BINPROVIDER, SYS_NPM_BINPROVIDER, env]
|
||||
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
LIB_NPM_BINPROVIDER.name: {"packages": lambda: [READABILITY_PACKAGE_NAME]},
|
||||
SYS_NPM_BINPROVIDER.name: {"packages": lambda: []}, # prevent modifying system global npm packages
|
||||
overrides: BinaryOverrides = {
|
||||
LIB_NPM_BINPROVIDER.name: {"packages": [READABILITY_PACKAGE_NAME]},
|
||||
SYS_NPM_BINPROVIDER.name: {"packages": [READABILITY_PACKAGE_NAME], "install": lambda: None}, # prevent modifying system global npm packages
|
||||
}
|
||||
|
||||
@validate_call
|
||||
def install(self, binprovider_name: Optional[BinProviderName]=None, **kwargs) -> ShallowBinary:
|
||||
# force install to only use lib/npm provider, we never want to modify global NPM packages
|
||||
return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs)
|
||||
|
||||
@validate_call
|
||||
def load_or_install(self, binprovider_name: Optional[BinProviderName] = None, fresh=False, **kwargs) -> ShallowBinary:
|
||||
try:
|
||||
return self.load(fresh=fresh)
|
||||
except Exception:
|
||||
# force install to only use lib/npm provider, we never want to modify global NPM packages
|
||||
return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs)
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
__package__ = 'archivebox.plugins_extractor.singlefile'
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
from typing import List, Optional
|
||||
# from typing_extensions import Self
|
||||
|
||||
# Depends on other PyPI/vendor packages:
|
||||
from pydantic import InstanceOf, Field
|
||||
from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, bin_abspath, ShallowBinary
|
||||
from pydantic_pkgr import BinProvider, BinaryOverrides, BinName, bin_abspath
|
||||
|
||||
# Depends on other Django apps:
|
||||
from abx.archivebox.base_plugin import BasePlugin
|
||||
|
|
@ -45,22 +45,21 @@ class SinglefileBinary(BaseBinary):
|
|||
name: BinName = SINGLEFILE_CONFIG.SINGLEFILE_BINARY
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_NPM_BINPROVIDER, SYS_NPM_BINPROVIDER, env]
|
||||
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
overrides: BinaryOverrides = {
|
||||
LIB_NPM_BINPROVIDER.name: {
|
||||
"abspath": lambda:
|
||||
bin_abspath(SINGLEFILE_CONFIG.SINGLEFILE_BINARY, PATH=LIB_NPM_BINPROVIDER.PATH)
|
||||
or bin_abspath("single-file", PATH=LIB_NPM_BINPROVIDER.PATH)
|
||||
or bin_abspath("single-file-node.js", PATH=LIB_NPM_BINPROVIDER.PATH),
|
||||
"packages": lambda:
|
||||
[f"single-file-cli@>={SINGLEFILE_MIN_VERSION} <{SINGLEFILE_MAX_VERSION}"],
|
||||
"packages": [f"single-file-cli@>={SINGLEFILE_MIN_VERSION} <{SINGLEFILE_MAX_VERSION}"],
|
||||
},
|
||||
SYS_NPM_BINPROVIDER.name: {
|
||||
"abspath": lambda:
|
||||
bin_abspath(SINGLEFILE_CONFIG.SINGLEFILE_BINARY, PATH=SYS_NPM_BINPROVIDER.PATH)
|
||||
or bin_abspath("single-file", PATH=SYS_NPM_BINPROVIDER.PATH)
|
||||
or bin_abspath("single-file-node.js", PATH=SYS_NPM_BINPROVIDER.PATH),
|
||||
"packages": lambda:
|
||||
[], # prevent modifying system global npm packages
|
||||
"packages": [f"single-file-cli@>={SINGLEFILE_MIN_VERSION} <{SINGLEFILE_MAX_VERSION}"],
|
||||
"install": lambda: None,
|
||||
},
|
||||
env.name: {
|
||||
'abspath': lambda:
|
||||
|
|
@ -69,18 +68,6 @@ class SinglefileBinary(BaseBinary):
|
|||
or bin_abspath('single-file-node.js', PATH=env.PATH),
|
||||
},
|
||||
}
|
||||
|
||||
def install(self, binprovider_name: Optional[BinProviderName]=None, **kwargs) -> ShallowBinary:
|
||||
# force install to only use lib/npm provider, we never want to modify global NPM packages
|
||||
return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs)
|
||||
|
||||
def load_or_install(self, binprovider_name: Optional[BinProviderName]=None, fresh=False, **kwargs) -> ShallowBinary:
|
||||
try:
|
||||
return self.load(fresh=fresh)
|
||||
except Exception:
|
||||
# force install to only use lib/npm provider, we never want to modify global NPM packages
|
||||
return BaseBinary.install(self, binprovider_name=binprovider_name or LIB_NPM_BINPROVIDER.name, **kwargs)
|
||||
|
||||
|
||||
|
||||
SINGLEFILE_BINARY = SinglefileBinary()
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import sys
|
||||
from typing import List, Dict
|
||||
from typing import List
|
||||
from subprocess import run, PIPE
|
||||
|
||||
from rich import print
|
||||
from pydantic import InstanceOf, Field, model_validator, AliasChoices
|
||||
from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict
|
||||
from pydantic_pkgr import BinProvider, BinName, BinaryOverrides
|
||||
|
||||
from abx.archivebox.base_plugin import BasePlugin
|
||||
from abx.archivebox.base_configset import BaseConfigSet
|
||||
|
|
@ -54,10 +54,10 @@ class FfmpegBinary(BaseBinary):
|
|||
name: BinName = 'ffmpeg'
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
|
||||
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
overrides: BinaryOverrides = {
|
||||
'env': {
|
||||
# 'abspath': lambda: shutil.which('ffmpeg', PATH=env.PATH),
|
||||
# 'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout,
|
||||
'version': lambda: run(['ffmpeg', '-version'], stdout=PIPE, stderr=PIPE, text=True).stdout,
|
||||
},
|
||||
'apt': {
|
||||
# 'abspath': lambda: shutil.which('ffmpeg', PATH=apt.PATH),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue