consolidate ansible setup into roles dir

archivebox/builtin_plugins/puppeteer/apps.py

@@ -0,0 +1,85 @@
+from typing import List, Optional
+
+from django.conf import settings
+
+# Depends on other PyPI/vendor packages:
+from pydantic import InstanceOf, Field
+from pydantic_pkgr import BinProvider, BinName
+
+# Depends on other Django apps:
+from plugantic.base_plugin import BasePlugin
+from plugantic.base_configset import BaseConfigSet, ConfigSectionName
+from plugantic.base_binary import BaseBinary, env
+# from plugantic.base_extractor import BaseExtractor
+# from plugantic.base_queue import BaseQueue
+from plugantic.base_hook import BaseHook
+from plugantic.ansible_utils import run_playbook
+
+# Depends on Other Plugins:
+from builtin_plugins.npm.apps import npm
+
+
+###################### Config ##########################
+
+
+class PuppeteerDependencyConfigs(BaseConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    PUPPETEER_BINARY: str = Field(default='wget')
+    PUPPETEER_ARGS: Optional[List[str]] = Field(default=None)
+    PUPPETEER_EXTRA_ARGS: List[str] = []
+    PUPPETEER_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+class PuppeteerConfigs(PuppeteerDependencyConfigs):
+    # section: ConfigSectionName = 'ALL_CONFIGS'
+    pass
+
+DEFAULT_GLOBAL_CONFIG = {
+}
+
+PUPPETEER_CONFIG = PuppeteerConfigs(**DEFAULT_GLOBAL_CONFIG)
+
+
+INSTALL_BIN = './install_puppeteer.yml'
+
+
+class ChromeBinary(BaseBinary):
+    name: BinName = 'chrome'
+    binproviders_supported: List[InstanceOf[BinProvider]] = [npm, env]
+
+    
+    def install(self, *args, quiet=False) -> "ChromeBinary":
+        
+        install_playbook = self.plugin_dir / 'install_puppeteer.yml'
+        
+        chrome_bin = run_playbook(install_playbook, data_dir=settings.CONFIG.OUTPUT_DIR, quiet=quiet).BINARIES.chrome
+
+        return self.__class__.model_validate(
+            {
+                **self.model_dump(),
+                "loaded_abspath": chrome_bin.symlink,
+                "loaded_version": chrome_bin.version,
+                "loaded_binprovider": env,
+                "binproviders_supported": self.binproviders_supported,
+            }
+        )
+        
+
+CHROME_BINARY = ChromeBinary()
+
+PLUGIN_BINARIES = [CHROME_BINARY]
+
+class PuppeteerPlugin(BasePlugin):
+    app_label: str ='puppeteer'
+    verbose_name: str = 'SingleFile'
+
+    hooks: List[InstanceOf[BaseHook]] = [
+        PUPPETEER_CONFIG,
+        CHROME_BINARY,
+    ]
+
+
+
+PLUGIN = PuppeteerPlugin()
+PLUGIN.register(settings)
+DJANGO_APP = PLUGIN.AppConfig

archivebox/builtin_plugins/puppeteer/install_puppeteer.yml

@@ -0,0 +1,127 @@
+#!/usr/bin/env ansible-playbook
+---
+- name: "Install puppeteer, puppeteer/browsers, and chrome"
+  hosts: localhost
+  gather_facts: true
+  vars:
+    LIB_DIR: '{{DATA_DIR}}/lib'
+    LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+    LIB_DIR_BROWSERS: '{{LIB_DIR}}/browsers'
+    CHROME_RELEASE_CHANNEL: 'chrome@stable'
+    CHROME_VERSION_MIN: '128.0.6613.137'
+  tasks:
+    - include_role:
+        name: setup_lib_npm
+      vars:
+        MIN_NODE_VERSION: '20.0.0'
+        MIN_NPM_VERSION: '10.0.0'
+
+    - name: "Install npm packages: [puppeteer, @puppeteer/browsers]"
+      community.general.npm:
+        name: '{{item}}'
+        state: "present"
+        path: '{{BINPROVIDERS.npm.lib_dir_npm}}'
+      loop:
+        - 'puppeteer'
+        - '@puppeteer/browsers'
+      
+    - name: Make sure prerequisite folders exist
+      file:
+        path: '{{LIB_DIR_BROWSERS}}'
+        state: directory
+        recurse: true
+
+    - name: Load puppeteer binary from installed NPM package
+      include_role:
+        name: load_binary
+      vars:
+        name: puppeteer
+        PATH: '{{BINPROVIDERS.npm.PATH}}'
+
+    - name: Load chrome binaries from environment PATH
+      include_tasks: roles/load_binary/tasks/main.yml
+      vars:
+        name: 'chrome'
+        bin_name: '{{chrome_executable}}'
+        PATH: '{{ansible_env.PATH}}'
+      with_items:
+        - chrome
+        # - chrome-browser
+        # - chromium
+        # - chromium-browser
+        # - google-chrome
+        # - google-chrome-browser
+        # - google-chrome-stable
+        # - google-chrome-beta
+        # - google-chrome-canary
+        # - google-chrome-unstable
+        # - google-chrome-dev
+      loop_control:
+        loop_var: chrome_executable
+        # break_when: 
+        #   - BINARIES.chrome.version|default('')
+
+    # - debug:
+    #     msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"
+
+
+    ###################################################################################
+    - name: 'Install Chrome browser: npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}}'
+      command: 'npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}} --path {{LIB_DIR_BROWSERS}}'
+      register: CHROME_VERSION_FULL
+      environment:
+        PATH: "{{BINPROVIDERS.npm.PATH}}:{{ ansible_env.PATH }}"
+      changed_when: CHROME_VERSION_MIN not in CHROME_VERSION_FULL.stdout
+      when: not BINARIES.chrome.version|default('')
+      #         -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...'
+
+    ###################################################################################
+    - name: Parse Chrome version and abspath from npx @puppeteer/browsers install output
+      set_fact:
+        CHROME_ABSPATH: "{{ CHROME_VERSION_FULL.stdout_lines|last|split(' ', 1)|last }}"
+        CHROME_VERSION: "{{ CHROME_VERSION_FULL.stdout_lines|last|split('@', 1)|last|split(' ', 1)|first }}"
+      when: not BINARIES.chrome.version|default('')
+
+    - name: Create ./bin/chrome symlink to ./browsers/chrome/... binary
+      copy:
+        content: |
+                 #!/bin/bash
+                 exec '{{CHROME_ABSPATH|default(BINARIES.chrome.abspath)}}' "$@"
+        dest: "{{LIB_DIR_BIN}}/chrome"
+      changed_when: False
+    
+    - name: Ensure ./bin/chrome symlink is executable
+      file:
+        path: "{{LIB_DIR_BIN}}/chrome"
+        mode: u+rx,g-rx,o-rwx
+        state: 'file'
+      changed_when: False
+
+    ###################################################################################
+    - set_fact:
+        PUPPETEER_BINARIES:
+          chrome:
+            bin_name: 'chrome'
+            abspath: "{{CHROME_ABSPATH|default(BINARIES.chrome.abspath) or None}}"
+            version: "{{CHROME_VERSION|default(BINARIES.chrome.version) or None}}"
+            symlink: "{{LIB_DIR_BIN}}/chrome"
+            version_cmd: "chrome --version"
+            binprovider: 'puppeteer'
+        PUPPETEER_BINPROVIDERS:
+          puppeteer:
+            installer_abspath: "{{BINARIES.puppeteer.abspath}}"
+            installer_version: "{{BINARIES.puppeteer.version}}"
+            PATH: "{{LIB_DIR_BIN}}"
+            lib_dir_browsers: "{{LIB_DIR_BROWSERS}}"
+
+    - name: Check that installed Chrome matches expected version
+      assert:
+        that: PUPPETEER_BINARIES.chrome.version is version(CHROME_VERSION_MIN, '>=')
+        quiet: true
+    
+    - set_fact:
+        BINARIES: "{{ BINARIES | combine(PUPPETEER_BINARIES) }}"
+        cacheable: true
+
+    - debug:
+        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/builtin_plugins/puppeteer/roles

@@ -0,0 +1 @@
+../ansible/roles