mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 22:54:27 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix playwright and puppeteer browser detection after install

Browse source
This commit is contained in:
Nick Sweeting 2024-10-09 03:46:28 -07:00
parent 6100685cbb
commit 31c66a1068
No known key found for this signature in database
6 changed files with 35 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

4
archivebox/main.py
View file

@ -1006,7 +1006,7 @@ def install(out_dir: Path=DATA_DIR) -> None:
from django.conf import settings
from archivebox import CONSTANTS
from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, USER
from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()):
run_subcommand('init', stdin=None, pwd=out_dir) # must init full index because we need a db to store InstalledBinary entries in
@ -1019,7 +1019,7 @@ def install(out_dir: Path=DATA_DIR) -> None:
# if we have sudo/root permissions, take advantage of them just while installing dependencies
print()
print(f'[yellow]:warning: Running as [blue]{USER}[/blue] ({EUID}) with [red]sudo[/red] only for dependencies that need it.[/yellow]')
print(f'[yellow]:warning: Running as UID=[blue]{EUID}[/blue] with [red]sudo[/red] only for dependencies that need it.[/yellow]')
print(f' DATA_DIR, LIB_DIR, and TMP_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].')
print()

30
archivebox/plugins_pkg/playwright/apps.py
View file

@ -1,5 +1,6 @@
__package__ = 'archivebox.plugins_pkg.playwright'
import os
import platform
from pathlib import Path
from typing import List, Optional, Dict, ClassVar
@ -65,12 +66,12 @@ class PlaywrightBinProvider(BaseBinProvider):
PATH: PATHStr = f"{CONSTANTS.LIB_BIN_DIR}:{DEFAULT_ENV_PATH}"
puppeteer_browsers_dir: Optional[Path] = (
playwright_browsers_dir: Optional[Path] = (
Path("~/Library/Caches/ms-playwright").expanduser() # macos playwright cache dir
if OPERATING_SYSTEM == "darwin" else
Path("~/.cache/ms-playwright").expanduser() # linux playwright cache dir
)
puppeteer_install_args: List[str] = ["install"] # --with-deps
playwright_install_args: List[str] = ["install"] # --with-deps
packages_handler: ProviderLookupDict = Field(default={
"chrome": lambda: ["chromium"],
@ -86,8 +87,8 @@ class PlaywrightBinProvider(BaseBinProvider):
def setup(self) -> None:
assert SYS_PIP_BINPROVIDER.INSTALLER_BIN_ABSPATH, "Pip bin provider not initialized"
if self.puppeteer_browsers_dir:
self.puppeteer_browsers_dir.mkdir(parents=True, exist_ok=True)
if self.playwright_browsers_dir:
self.playwright_browsers_dir.mkdir(parents=True, exist_ok=True)
def installed_browser_bins(self, browser_name: str = "*") -> List[Path]:
if browser_name == 'chrome':
@ -97,13 +98,13 @@ class PlaywrightBinProvider(BaseBinProvider):
if platform.system().lower() == "darwin":
# ~/Library/caches/ms-playwright/chromium-1097/chrome-mac/Chromium.app/Contents/MacOS/Chromium
return sorted(
self.puppeteer_browsers_dir.glob(
self.playwright_browsers_dir.glob(
f"{browser_name}-*/*-mac*/*.app/Contents/MacOS/*"
)
)
# ~/Library/caches/ms-playwright/chromium-1097/chrome-linux/chromium
return sorted(self.puppeteer_browsers_dir.glob(f"{browser_name}-*/*-linux/*"))
return sorted(self.playwright_browsers_dir.glob(f"{browser_name}-*/*-linux/*"))
def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
assert bin_name == "chrome", "Only chrome is supported using the @puppeteer/browsers install method currently."
@ -112,7 +113,7 @@ class PlaywrightBinProvider(BaseBinProvider):
if bin_name in self._browser_abspaths:
return self._browser_abspaths[bin_name]
# first time loading, find browser in self.puppeteer_browsers_dir by searching filesystem for installed binaries
# first time loading, find browser in self.playwright_browsers_dir by searching filesystem for installed binaries
matching_bins = [abspath for abspath in self.installed_browser_bins() if bin_name in str(abspath)]
if matching_bins:
newest_bin = matching_bins[-1] # already sorted alphabetically, last should theoretically be highest version number
@ -140,7 +141,7 @@ class PlaywrightBinProvider(BaseBinProvider):
# print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}')
install_args = [*self.puppeteer_install_args]
install_args = [*self.playwright_install_args]
proc = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[*install_args, *packages])
@ -150,11 +151,14 @@ class PlaywrightBinProvider(BaseBinProvider):
raise Exception(f"{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages}")
# chrome@129.0.6668.58 /data/lib/browsers/chrome/mac_arm-129.0.6668.58/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing
output_info = proc.stdout.strip().split("\n")[-1]
browser_abspath = output_info.split(" ", 1)[-1]
# browser_version = output_info.split('@', 1)[-1].split(' ', 1)[0]
self._browser_abspaths[bin_name] = Path(browser_abspath)
# playwright build v1010 downloaded to /home/squash/.cache/ms-playwright/ffmpeg-1010
output_lines = [line for line in proc.stdout.strip().split('\n') if '/chrome-' in line]
if output_lines:
relpath = output_lines[0].split(self.playwright_browsers_dir)[-1]
abspath = self.playwright_browsers_dir / relpath
if os.path.isfile(abspath) and os.access(abspath, os.X_OK):
self._browser_abspaths[bin_name] = abspath
return abspath
return proc.stderr.strip() + "\n" + proc.stdout.strip()

11
archivebox/plugins_pkg/puppeteer/apps.py
View file

@ -1,5 +1,6 @@
__package__ = 'archivebox.plugins_pkg.puppeteer'
import os
import platform
from pathlib import Path
from typing import List, Optional, Dict, ClassVar
@ -128,12 +129,14 @@ class PuppeteerBinProvider(BaseBinProvider):
print(proc.stderr.strip())
raise Exception(f"{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages}")
# to proceed? (y) chrome@129.0.6668.91 /tmp/test3/lib/x86_64-linux/browsers/chrome/linux-129.0.6668.91/chrome-linux64/chrome
# chrome@129.0.6668.58 /data/lib/browsers/chrome/mac_arm-129.0.6668.58/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing
output_info = proc.stdout.strip().split('\n')[-1]
browser_abspath = output_info.split(' ', 1)[-1]
# browser_version = output_info.split('@', 1)[-1].split(' ', 1)[0]
relpath = proc.stdout.strip().split(str(self.puppeteer_browsers_dir))[-1]
abspath = self.puppeteer_browsers_dir / relpath
self._browser_abspaths[bin_name] = Path(browser_abspath)
if os.path.isfile(abspath) and os.access(abspath, os.X_OK):
self._browser_abspaths[bin_name] = abspath
return abspath
return proc.stderr.strip() + "\n" + proc.stdout.strip()

4
pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.8.5rc27"
version = "0.8.5rc28"
requires-python = ">=3.10"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@ -78,7 +78,7 @@ dependencies = [
"django-taggit==1.3.0",
"base32-crockford==0.3.0",
# "pocket@git+https://github.com/tapanpandita/pocket.git@v0.3.7",
"pydantic-pkgr>=0.4.20",
"pydantic-pkgr>=0.4.21",
############# Plugin Dependencies ################
"sonic-client>=1.0.0",
"yt-dlp>=2024.8.6", # for: media"

2
requirements.txt
View file

@ -203,7 +203,7 @@ pydantic-core==2.23.4
# via
# pydantic
# pydantic-pkgr
pydantic-pkgr==0.4.20
pydantic-pkgr==0.4.21
# via archivebox (pyproject.toml)
pydantic-settings==2.5.2
# via archivebox (pyproject.toml)

10
uv.lock generated
View file

@ -41,7 +41,7 @@ wheels = [
[[package]]
name = "archivebox"
version = "0.8.5rc27"
version = "0.8.5rc28"
source = { editable = "." }
dependencies = [
{ name = "atomicwrites" },
@ -148,7 +148,7 @@ requires-dist = [
{ name = "pluggy", specifier = ">=1.5.0" },
{ name = "psutil", specifier = ">=6.0.0" },
{ name = "py-machineid", specifier = ">=0.6.0" },
{ name = "pydantic-pkgr", specifier = ">=0.4.20" },
{ name = "pydantic-pkgr", specifier = ">=0.4.21" },
{ name = "pydantic-settings", specifier = ">=2.5.2" },
{ name = "python-benedict", extras = ["io", "parse"], specifier = ">=0.33.2" },
{ name = "python-crontab", specifier = ">=3.2.0" },
@ -1834,16 +1834,16 @@ wheels = [
[[package]]
name = "pydantic-pkgr"
version = "0.4.20"
version = "0.4.21"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pydantic" },
{ name = "pydantic-core" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/0c/3b/d2efaa2c4f39d1bf16a7ab514e0d2ce3abb4c1b3f8b78114588b6d8a9511/pydantic_pkgr-0.4.20.tar.gz", hash = "sha256:75ddd2ef77457e078dcf61dbadc63857500b258b4aa9d776318c018abfac8775", size = 38287 }
sdist = { url = "https://files.pythonhosted.org/packages/a4/5d/25288d8eb7386e68ac3e19ed2874219dffc7225b5db61e7749668d4cb2fd/pydantic_pkgr-0.4.21.tar.gz", hash = "sha256:ace74f3d4ab1d69d16f52d40ee8897b47deb89f4c82e659c29fd242407adb64c", size = 38567 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/70/d2/9078976dba39688c6a4a16ad26b36c1e25f9e0d0d22be1ab7a5334cf02be/pydantic_pkgr-0.4.20-py3-none-any.whl", hash = "sha256:723c0afd7e2a64f4de1f8bf5709c8c7dd6f695476bccda972d730c38df8a27a7", size = 41365 },
{ url = "https://files.pythonhosted.org/packages/8b/b5/1f36fc3fbd667ff6d1da581689fe9de2ef700a645d4f10f3f5c1afc7a4a6/pydantic_pkgr-0.4.21-py3-none-any.whl", hash = "sha256:6138ad0f979a6ea233ba3abc4010a0a974931796e94256f3c09e7fde5b020488", size = 41633 },
]
[[package]]