fix playwright and puppeteer browser detection after install

archivebox/main.py

@@ -1006,7 +1006,7 @@ def install(out_dir: Path=DATA_DIR) -> None:
     from django.conf import settings
     
     from archivebox import CONSTANTS
-    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, USER
+    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
 
     if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()):
         run_subcommand('init', stdin=None, pwd=out_dir)  # must init full index because we need a db to store InstalledBinary entries in
@@ -1019,7 +1019,7 @@ def install(out_dir: Path=DATA_DIR) -> None:
         
         # if we have sudo/root permissions, take advantage of them just while installing dependencies
         print()
-        print(f'[yellow]:warning:  Running as [blue]{USER}[/blue] ({EUID}) with [red]sudo[/red] only for dependencies that need it.[/yellow]')
+        print(f'[yellow]:warning:  Running as UID=[blue]{EUID}[/blue] with [red]sudo[/red] only for dependencies that need it.[/yellow]')
         print(f'    DATA_DIR, LIB_DIR, and TMP_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].')
         print()
     

archivebox/plugins_pkg/playwright/apps.py

@@ -1,5 +1,6 @@
 __package__ = 'archivebox.plugins_pkg.playwright'
 
+import os
 import platform
 from pathlib import Path
 from typing import List, Optional, Dict, ClassVar
@@ -65,12 +66,12 @@ class PlaywrightBinProvider(BaseBinProvider):
 
     PATH: PATHStr = f"{CONSTANTS.LIB_BIN_DIR}:{DEFAULT_ENV_PATH}"
 
-    puppeteer_browsers_dir: Optional[Path] = (
+    playwright_browsers_dir: Optional[Path] = (
         Path("~/Library/Caches/ms-playwright").expanduser()      # macos playwright cache dir
         if OPERATING_SYSTEM == "darwin" else
         Path("~/.cache/ms-playwright").expanduser()              # linux playwright cache dir
     )
-    puppeteer_install_args: List[str] = ["install"]              # --with-deps
+    playwright_install_args: List[str] = ["install"]              # --with-deps
 
     packages_handler: ProviderLookupDict = Field(default={
         "chrome": lambda: ["chromium"],
@@ -86,8 +87,8 @@ class PlaywrightBinProvider(BaseBinProvider):
     def setup(self) -> None:
         assert SYS_PIP_BINPROVIDER.INSTALLER_BIN_ABSPATH, "Pip bin provider not initialized"
 
-        if self.puppeteer_browsers_dir:
+        if self.playwright_browsers_dir:
-            self.puppeteer_browsers_dir.mkdir(parents=True, exist_ok=True)
+            self.playwright_browsers_dir.mkdir(parents=True, exist_ok=True)
 
     def installed_browser_bins(self, browser_name: str = "*") -> List[Path]:
         if browser_name == 'chrome':
@@ -97,13 +98,13 @@ class PlaywrightBinProvider(BaseBinProvider):
         if platform.system().lower() == "darwin":
             # ~/Library/caches/ms-playwright/chromium-1097/chrome-mac/Chromium.app/Contents/MacOS/Chromium
             return sorted(
-                self.puppeteer_browsers_dir.glob(
+                self.playwright_browsers_dir.glob(
                     f"{browser_name}-*/*-mac*/*.app/Contents/MacOS/*"
                 )
             )
 
         # ~/Library/caches/ms-playwright/chromium-1097/chrome-linux/chromium
-        return sorted(self.puppeteer_browsers_dir.glob(f"{browser_name}-*/*-linux/*"))
+        return sorted(self.playwright_browsers_dir.glob(f"{browser_name}-*/*-linux/*"))
 
     def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
         assert bin_name == "chrome", "Only chrome is supported using the @puppeteer/browsers install method currently."
@@ -112,7 +113,7 @@ class PlaywrightBinProvider(BaseBinProvider):
         if bin_name in self._browser_abspaths:
             return self._browser_abspaths[bin_name]
 
-        # first time loading, find browser in self.puppeteer_browsers_dir by searching filesystem for installed binaries
+        # first time loading, find browser in self.playwright_browsers_dir by searching filesystem for installed binaries
         matching_bins = [abspath for abspath in self.installed_browser_bins() if bin_name in str(abspath)]
         if matching_bins:
             newest_bin = matching_bins[-1]  # already sorted alphabetically, last should theoretically be highest version number
@@ -140,7 +141,7 @@ class PlaywrightBinProvider(BaseBinProvider):
 
         # print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}')
 
-        install_args = [*self.puppeteer_install_args]
+        install_args = [*self.playwright_install_args]
 
         proc = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=[*install_args, *packages])
 
@@ -150,11 +151,14 @@ class PlaywrightBinProvider(BaseBinProvider):
             raise Exception(f"{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages}")
 
         # chrome@129.0.6668.58 /data/lib/browsers/chrome/mac_arm-129.0.6668.58/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing
-        output_info = proc.stdout.strip().split("\n")[-1]
+        # playwright build v1010 downloaded to /home/squash/.cache/ms-playwright/ffmpeg-1010
-        browser_abspath = output_info.split(" ", 1)[-1]
+        output_lines = [line for line in proc.stdout.strip().split('\n') if '/chrome-' in line]
-        # browser_version = output_info.split('@', 1)[-1].split(' ', 1)[0]
+        if output_lines:
-
+            relpath = output_lines[0].split(self.playwright_browsers_dir)[-1]
-        self._browser_abspaths[bin_name] = Path(browser_abspath)
+            abspath = self.playwright_browsers_dir / relpath
+            if os.path.isfile(abspath) and os.access(abspath, os.X_OK):
+                self._browser_abspaths[bin_name] = abspath
+                return abspath
         
         return proc.stderr.strip() + "\n" + proc.stdout.strip()
 

archivebox/plugins_pkg/puppeteer/apps.py

@@ -1,5 +1,6 @@
 __package__ = 'archivebox.plugins_pkg.puppeteer'
 
+import os
 import platform
 from pathlib import Path
 from typing import List, Optional, Dict, ClassVar
@@ -128,12 +129,14 @@ class PuppeteerBinProvider(BaseBinProvider):
             print(proc.stderr.strip())
             raise Exception(f"{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages}")
 
+        # to proceed? (y) chrome@129.0.6668.91 /tmp/test3/lib/x86_64-linux/browsers/chrome/linux-129.0.6668.91/chrome-linux64/chrome
         # chrome@129.0.6668.58 /data/lib/browsers/chrome/mac_arm-129.0.6668.58/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing
-        output_info = proc.stdout.strip().split('\n')[-1]
+        relpath = proc.stdout.strip().split(str(self.puppeteer_browsers_dir))[-1]
-        browser_abspath = output_info.split(' ', 1)[-1]
+        abspath = self.puppeteer_browsers_dir / relpath
-        # browser_version = output_info.split('@', 1)[-1].split(' ', 1)[0]
         
-        self._browser_abspaths[bin_name] = Path(browser_abspath)
+        if os.path.isfile(abspath) and os.access(abspath, os.X_OK):
+            self._browser_abspaths[bin_name] = abspath
+            return abspath
 
         return proc.stderr.strip() + "\n" + proc.stdout.strip()
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.8.5rc27"
+version = "0.8.5rc28"
 requires-python = ">=3.10"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -78,7 +78,7 @@ dependencies = [
     "django-taggit==1.3.0",
     "base32-crockford==0.3.0",
     # "pocket@git+https://github.com/tapanpandita/pocket.git@v0.3.7",
-    "pydantic-pkgr>=0.4.20",
+    "pydantic-pkgr>=0.4.21",
     ############# Plugin Dependencies ################
     "sonic-client>=1.0.0",
     "yt-dlp>=2024.8.6",               # for: media"

requirements.txt

@@ -203,7 +203,7 @@ pydantic-core==2.23.4
     # via
     #   pydantic
     #   pydantic-pkgr
-pydantic-pkgr==0.4.20
+pydantic-pkgr==0.4.21
     # via archivebox (pyproject.toml)
 pydantic-settings==2.5.2
     # via archivebox (pyproject.toml)

uv.lock

@@ -41,7 +41,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.8.5rc27"
+version = "0.8.5rc28"
 source = { editable = "." }
 dependencies = [
     { name = "atomicwrites" },
@@ -148,7 +148,7 @@ requires-dist = [
     { name = "pluggy", specifier = ">=1.5.0" },
     { name = "psutil", specifier = ">=6.0.0" },
     { name = "py-machineid", specifier = ">=0.6.0" },
-    { name = "pydantic-pkgr", specifier = ">=0.4.20" },
+    { name = "pydantic-pkgr", specifier = ">=0.4.21" },
     { name = "pydantic-settings", specifier = ">=2.5.2" },
     { name = "python-benedict", extras = ["io", "parse"], specifier = ">=0.33.2" },
     { name = "python-crontab", specifier = ">=3.2.0" },
@@ -1834,16 +1834,16 @@ wheels = [
 
 [[package]]
 name = "pydantic-pkgr"
-version = "0.4.20"
+version = "0.4.21"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "pydantic" },
     { name = "pydantic-core" },
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/0c/3b/d2efaa2c4f39d1bf16a7ab514e0d2ce3abb4c1b3f8b78114588b6d8a9511/pydantic_pkgr-0.4.20.tar.gz", hash = "sha256:75ddd2ef77457e078dcf61dbadc63857500b258b4aa9d776318c018abfac8775", size = 38287 }
+sdist = { url = "https://files.pythonhosted.org/packages/a4/5d/25288d8eb7386e68ac3e19ed2874219dffc7225b5db61e7749668d4cb2fd/pydantic_pkgr-0.4.21.tar.gz", hash = "sha256:ace74f3d4ab1d69d16f52d40ee8897b47deb89f4c82e659c29fd242407adb64c", size = 38567 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/70/d2/9078976dba39688c6a4a16ad26b36c1e25f9e0d0d22be1ab7a5334cf02be/pydantic_pkgr-0.4.20-py3-none-any.whl", hash = "sha256:723c0afd7e2a64f4de1f8bf5709c8c7dd6f695476bccda972d730c38df8a27a7", size = 41365 },
+    { url = "https://files.pythonhosted.org/packages/8b/b5/1f36fc3fbd667ff6d1da581689fe9de2ef700a645d4f10f3f5c1afc7a4a6/pydantic_pkgr-0.4.21-py3-none-any.whl", hash = "sha256:6138ad0f979a6ea233ba3abc4010a0a974931796e94256f3c09e7fde5b020488", size = 41633 },
 ]
 
 [[package]]