diff --git a/archivebox/main.py b/archivebox/main.py index af4521c9..9a9e3ed5 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -1006,7 +1006,7 @@ def install(out_dir: Path=DATA_DIR) -> None: from django.conf import settings from archivebox import CONSTANTS - from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, USER + from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()): run_subcommand('init', stdin=None, pwd=out_dir) # must init full index because we need a db to store InstalledBinary entries in @@ -1019,7 +1019,7 @@ def install(out_dir: Path=DATA_DIR) -> None: # if we have sudo/root permissions, take advantage of them just while installing dependencies print() - print(f'[yellow]:warning: Running as [blue]{USER}[/blue] ({EUID}) with [red]sudo[/red] only for dependencies that need it.[/yellow]') + print(f'[yellow]:warning: Running as UID=[blue]{EUID}[/blue] with [red]sudo[/red] only for dependencies that need it.[/yellow]') print(f' DATA_DIR, LIB_DIR, and TMP_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].') print() diff --git a/archivebox/plugins_pkg/playwright/apps.py b/archivebox/plugins_pkg/playwright/apps.py index a065be43..96fac341 100644 --- a/archivebox/plugins_pkg/playwright/apps.py +++ b/archivebox/plugins_pkg/playwright/apps.py @@ -1,5 +1,6 @@ __package__ = 'archivebox.plugins_pkg.playwright' +import os import platform from pathlib import Path from typing import List, Optional, Dict, ClassVar @@ -65,12 +66,12 @@ class PlaywrightBinProvider(BaseBinProvider): PATH: PATHStr = f"{CONSTANTS.LIB_BIN_DIR}:{DEFAULT_ENV_PATH}" - puppeteer_browsers_dir: Optional[Path] = ( + playwright_browsers_dir: Optional[Path] = ( Path("~/Library/Caches/ms-playwright").expanduser() # macos playwright cache dir if OPERATING_SYSTEM == "darwin" else Path("~/.cache/ms-playwright").expanduser() # linux playwright cache dir ) - puppeteer_install_args: List[str] = ["install"] # --with-deps + playwright_install_args: List[str] = ["install"] # --with-deps packages_handler: ProviderLookupDict = Field(default={ "chrome": lambda: ["chromium"], @@ -86,8 +87,8 @@ class PlaywrightBinProvider(BaseBinProvider): def setup(self) -> None: assert SYS_PIP_BINPROVIDER.INSTALLER_BIN_ABSPATH, "Pip bin provider not initialized" - if self.puppeteer_browsers_dir: - self.puppeteer_browsers_dir.mkdir(parents=True, exist_ok=True) + if self.playwright_browsers_dir: + self.playwright_browsers_dir.mkdir(parents=True, exist_ok=True) def installed_browser_bins(self, browser_name: str = "*") -> List[Path]: if browser_name == 'chrome': @@ -97,13 +98,13 @@ class PlaywrightBinProvider(BaseBinProvider): if platform.system().lower() == "darwin": # ~/Library/caches/ms-playwright/chromium-1097/chrome-mac/Chromium.app/Contents/MacOS/Chromium return sorted( - self.puppeteer_browsers_dir.glob( + self.playwright_browsers_dir.glob( f"{browser_name}-*/*-mac*/*.app/Contents/MacOS/*" ) ) # ~/Library/caches/ms-playwright/chromium-1097/chrome-linux/chromium - return sorted(self.puppeteer_browsers_dir.glob(f"{browser_name}-*/*-linux/*")) + return sorted(self.playwright_browsers_dir.glob(f"{browser_name}-*/*-linux/*")) def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]: assert bin_name == "chrome", "Only chrome is supported using the @puppeteer/browsers install method currently." @@ -112,7 +113,7 @@ class PlaywrightBinProvider(BaseBinProvider): if bin_name in self._browser_abspaths: return self._browser_abspaths[bin_name] - # first time loading, find browser in self.puppeteer_browsers_dir by searching filesystem for installed binaries + # first time loading, find browser in self.playwright_browsers_dir by searching filesystem for installed binaries matching_bins = [abspath for abspath in self.installed_browser_bins() if bin_name in str(abspath)] if matching_bins: newest_bin = matching_bins[-1] # already sorted alphabetically, last should theoretically be highest version number @@ -140,7 +141,7 @@ class PlaywrightBinProvider(BaseBinProvider): # print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}') - install_args = [*self.puppeteer_install_args] + install_args = [*self.playwright_install_args] proc = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[*install_args, *packages]) @@ -150,12 +151,15 @@ class PlaywrightBinProvider(BaseBinProvider): raise Exception(f"{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages}") # chrome@129.0.6668.58 /data/lib/browsers/chrome/mac_arm-129.0.6668.58/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing - output_info = proc.stdout.strip().split("\n")[-1] - browser_abspath = output_info.split(" ", 1)[-1] - # browser_version = output_info.split('@', 1)[-1].split(' ', 1)[0] - - self._browser_abspaths[bin_name] = Path(browser_abspath) - + # playwright build v1010 downloaded to /home/squash/.cache/ms-playwright/ffmpeg-1010 + output_lines = [line for line in proc.stdout.strip().split('\n') if '/chrome-' in line] + if output_lines: + relpath = output_lines[0].split(self.playwright_browsers_dir)[-1] + abspath = self.playwright_browsers_dir / relpath + if os.path.isfile(abspath) and os.access(abspath, os.X_OK): + self._browser_abspaths[bin_name] = abspath + return abspath + return proc.stderr.strip() + "\n" + proc.stdout.strip() PLAYWRIGHT_BINPROVIDER = PlaywrightBinProvider() diff --git a/archivebox/plugins_pkg/puppeteer/apps.py b/archivebox/plugins_pkg/puppeteer/apps.py index 15946e87..f44927c1 100644 --- a/archivebox/plugins_pkg/puppeteer/apps.py +++ b/archivebox/plugins_pkg/puppeteer/apps.py @@ -1,5 +1,6 @@ __package__ = 'archivebox.plugins_pkg.puppeteer' +import os import platform from pathlib import Path from typing import List, Optional, Dict, ClassVar @@ -128,12 +129,14 @@ class PuppeteerBinProvider(BaseBinProvider): print(proc.stderr.strip()) raise Exception(f"{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages}") + # to proceed? (y) chrome@129.0.6668.91 /tmp/test3/lib/x86_64-linux/browsers/chrome/linux-129.0.6668.91/chrome-linux64/chrome # chrome@129.0.6668.58 /data/lib/browsers/chrome/mac_arm-129.0.6668.58/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing - output_info = proc.stdout.strip().split('\n')[-1] - browser_abspath = output_info.split(' ', 1)[-1] - # browser_version = output_info.split('@', 1)[-1].split(' ', 1)[0] + relpath = proc.stdout.strip().split(str(self.puppeteer_browsers_dir))[-1] + abspath = self.puppeteer_browsers_dir / relpath - self._browser_abspaths[bin_name] = Path(browser_abspath) + if os.path.isfile(abspath) and os.access(abspath, os.X_OK): + self._browser_abspaths[bin_name] = abspath + return abspath return proc.stderr.strip() + "\n" + proc.stdout.strip() diff --git a/pyproject.toml b/pyproject.toml index 0722d533..f8353b23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "archivebox" -version = "0.8.5rc27" +version = "0.8.5rc28" requires-python = ">=3.10" description = "Self-hosted internet archiving solution." authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}] @@ -78,7 +78,7 @@ dependencies = [ "django-taggit==1.3.0", "base32-crockford==0.3.0", # "pocket@git+https://github.com/tapanpandita/pocket.git@v0.3.7", - "pydantic-pkgr>=0.4.20", + "pydantic-pkgr>=0.4.21", ############# Plugin Dependencies ################ "sonic-client>=1.0.0", "yt-dlp>=2024.8.6", # for: media" diff --git a/requirements.txt b/requirements.txt index e455571b..22e1e8b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -203,7 +203,7 @@ pydantic-core==2.23.4 # via # pydantic # pydantic-pkgr -pydantic-pkgr==0.4.20 +pydantic-pkgr==0.4.21 # via archivebox (pyproject.toml) pydantic-settings==2.5.2 # via archivebox (pyproject.toml) diff --git a/uv.lock b/uv.lock index ec925c17..e3b66ecb 100644 --- a/uv.lock +++ b/uv.lock @@ -41,7 +41,7 @@ wheels = [ [[package]] name = "archivebox" -version = "0.8.5rc27" +version = "0.8.5rc28" source = { editable = "." } dependencies = [ { name = "atomicwrites" }, @@ -148,7 +148,7 @@ requires-dist = [ { name = "pluggy", specifier = ">=1.5.0" }, { name = "psutil", specifier = ">=6.0.0" }, { name = "py-machineid", specifier = ">=0.6.0" }, - { name = "pydantic-pkgr", specifier = ">=0.4.20" }, + { name = "pydantic-pkgr", specifier = ">=0.4.21" }, { name = "pydantic-settings", specifier = ">=2.5.2" }, { name = "python-benedict", extras = ["io", "parse"], specifier = ">=0.33.2" }, { name = "python-crontab", specifier = ">=3.2.0" }, @@ -1834,16 +1834,16 @@ wheels = [ [[package]] name = "pydantic-pkgr" -version = "0.4.20" +version = "0.4.21" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "pydantic-core" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0c/3b/d2efaa2c4f39d1bf16a7ab514e0d2ce3abb4c1b3f8b78114588b6d8a9511/pydantic_pkgr-0.4.20.tar.gz", hash = "sha256:75ddd2ef77457e078dcf61dbadc63857500b258b4aa9d776318c018abfac8775", size = 38287 } +sdist = { url = "https://files.pythonhosted.org/packages/a4/5d/25288d8eb7386e68ac3e19ed2874219dffc7225b5db61e7749668d4cb2fd/pydantic_pkgr-0.4.21.tar.gz", hash = "sha256:ace74f3d4ab1d69d16f52d40ee8897b47deb89f4c82e659c29fd242407adb64c", size = 38567 } wheels = [ - { url = "https://files.pythonhosted.org/packages/70/d2/9078976dba39688c6a4a16ad26b36c1e25f9e0d0d22be1ab7a5334cf02be/pydantic_pkgr-0.4.20-py3-none-any.whl", hash = "sha256:723c0afd7e2a64f4de1f8bf5709c8c7dd6f695476bccda972d730c38df8a27a7", size = 41365 }, + { url = "https://files.pythonhosted.org/packages/8b/b5/1f36fc3fbd667ff6d1da581689fe9de2ef700a645d4f10f3f5c1afc7a4a6/pydantic_pkgr-0.4.21-py3-none-any.whl", hash = "sha256:6138ad0f979a6ea233ba3abc4010a0a974931796e94256f3c09e7fde5b020488", size = 41633 }, ] [[package]]