move unused ansible folder

archivebox/_ansible_unused/ansible_utils.py

@@ -0,0 +1,59 @@
+# import os
+
+# from pathlib import Path
+
+# from benedict import benedict
+# from rich.pretty import pprint
+
+# from ansible_runner import Runner, RunnerConfig
+
+# GLOBAL_CACHE = {}
+
+
+# def run_playbook(playbook_path, data_dir, quiet=False, **kwargs):
+#     ANSIBLE_TMP_DIR = str(Path(data_dir) / "tmp" / "ansible")
+#     os.environ['ANSIBLE_INVENTORY_UNPARSED_WARNING'] = 'False'
+#     os.environ['ANSIBLE_LOCALHOST_WARNING'] = 'False'
+#     os.environ["ANSIBLE_HOME"] = ANSIBLE_TMP_DIR
+#     # os.environ["ANSIBLE_COLLECTIONS_PATH"] = str(Path(data_dir).parent / 'archivebox')
+#     os.environ["ANSIBLE_ROLES_PATH"] = (
+#         './roles'
+#     )
+    
+#     rc = RunnerConfig(
+#         private_data_dir=ANSIBLE_TMP_DIR,
+#         playbook=str(playbook_path),
+#         rotate_artifacts=50000,
+#         host_pattern="localhost",
+#         extravars={
+#             "DATA_DIR": str(data_dir),
+#             **kwargs,
+#         },
+#         quiet=quiet,
+#     )
+#     rc.prepare()
+#     r = Runner(config=rc)
+#     r.set_fact_cache('localhost', GLOBAL_CACHE)
+#     r.run()
+#     last_run_facts = r.get_fact_cache('localhost')
+#     GLOBAL_CACHE.update(filtered_facts(last_run_facts))
+#     return benedict({
+#         key: val
+#         for key, val in last_run_facts.items()
+#         if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup'))
+#     })
+
+# def filtered_facts(facts):
+#     return benedict({
+#         key: val
+#         for key, val in facts.items()
+#         if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup'))
+#     })
+
+# def print_globals():
+#     pprint(filtered_facts(GLOBAL_CACHE), expand_all=True)
+
+
+
+# # YTDLP_OUTPUT = run_playbook('extract.yml', {'url': 'https://www.youtube.com/watch?v=cK4REjqGc9w&t=27s'})
+# # pprint(YTDLP_OUTPUT)

archivebox/_ansible_unused/install_puppeteer.yml

@@ -0,0 +1,125 @@
+#!/usr/bin/env ansible-playbook
+---
+- name: "Install puppeteer, puppeteer/browsers, and chrome"
+  hosts: localhost
+  gather_facts: true
+  vars:
+    DATA_DIR: '{{playbook_dir}}'
+    LIB_DIR: '{{DATA_DIR}}/lib'
+    LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+    LIB_DIR_BROWSERS: '{{LIB_DIR}}/browsers'
+    CHROME_RELEASE_CHANNEL: 'chrome@stable'
+    CHROME_VERSION_MIN: '128.0.6613.137'
+  tasks:
+    - include_role:
+        name: setup_lib_npm
+      vars:
+        TARGET_NODE_VERSION: '21'
+        MIN_NODE_VERSION: '20.0.0'
+        MIN_NPM_VERSION: '10.0.0'
+
+    - name: "Install npm packages: [puppeteer, @puppeteer/browsers]"
+      community.general.npm:
+        name: '{{item}}'
+        state: "present"
+        path: '{{BINPROVIDERS.npm.lib_dir_npm}}'
+      loop:
+        - 'puppeteer'
+        - '@puppeteer/browsers'
+      
+    - name: Make sure prerequisite folders exist
+      file:
+        path: '{{LIB_DIR_BROWSERS}}'
+        state: directory
+        recurse: true
+
+    - name: Load puppeteer binary from installed NPM package
+      include_role:
+        name: load_binary
+      vars:
+        name: puppeteer
+        PATH: '{{BINPROVIDERS.npm.PATH}}'
+
+    # - name: Find existing chrome binaries in environment PATH
+    #   include_role:
+    #     name: load_binary
+    #   vars:
+    #     name: 'chrome'
+    #     bin_name: '{{chrome_executable}}'
+    #     PATH: '{{ansible_env.PATH}}'
+    #   loop:
+    #     - chrome
+    #     - chrome-browser
+    #     - chromium
+    #     - chromium-browser
+    #     - google-chrome
+    #     - google-chrome-browser
+    #     - google-chrome-stable
+    #     - google-chrome-beta
+    #     - google-chrome-canary
+    #     - google-chrome-unstable
+    #     - google-chrome-dev
+    #   loop_control:
+    #     loop_var: chrome_executable
+    #     break_when: 
+    #       - BINARIES.chrome.version|default('')
+
+
+    ###################################################################################
+    - name: 'Install Chrome browser: npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}}'
+      command: 'npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}} --path {{LIB_DIR_BROWSERS}}'
+      register: CHROME_VERSION_FULL
+      environment:
+        PATH: "{{BINPROVIDERS.npm.PATH}}:{{ ansible_env.PATH }}"
+      changed_when: CHROME_VERSION_MIN not in CHROME_VERSION_FULL.stdout
+      when: not BINARIES.chrome.version|default('')
+      #         -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...'
+
+    ###################################################################################
+    - name: Parse Chrome version and abspath from npx @puppeteer/browsers install output
+      set_fact:
+        CHROME_ABSPATH: "{{ CHROME_VERSION_FULL.stdout_lines|last|split(' ', 1)|last }}"
+        CHROME_VERSION: "{{ CHROME_VERSION_FULL.stdout_lines|last|split('@', 1)|last|split(' ', 1)|first }}"
+      when: not BINARIES.chrome.version|default('')
+
+    - name: Create ./bin/chrome symlink to ./browsers/chrome/... binary
+      # normal symlink doesn't work for .app on macOS because it fails to load ../Framworks/..., so we create a tiny bash script to open it in its correct cwd instead
+      copy:
+        content: |
+                 #!/bin/bash
+                 exec '{{CHROME_ABSPATH|default(BINARIES.chrome.abspath)}}' "$@"
+        dest: "{{LIB_DIR_BIN}}/chrome"
+      changed_when: False
+    
+    - name: Ensure ./bin/chrome symlink is executable
+      file:
+        path: "{{LIB_DIR_BIN}}/chrome"
+        mode: u+rx,g-rx,o-rwx
+        state: 'file'
+      changed_when: False
+
+    ###################################################################################
+    - set_fact:
+        PUPPETEER_BINARIES:
+          chrome:
+            name: 'chrome'
+            bin_name: 'chrome'
+            abspath: "{{CHROME_ABSPATH|default(BINARIES.chrome.abspath) or None}}"
+            version: "{{CHROME_VERSION|default(BINARIES.chrome.version) or None}}"
+            symlink: "{{LIB_DIR_BIN}}/chrome"
+            version_cmd: "chrome --version"
+            version_stdout: "{{CHROME_VERSION_FULL.stdout}}"
+            binprovider: 'puppeteer'
+            PATH: "{{LIB_DIR_BIN}}"
+
+    - name: Check that installed Chrome matches expected version
+      assert:
+        that: PUPPETEER_BINARIES.chrome.version is version(CHROME_VERSION_MIN, '>=')
+        quiet: true
+    
+    - set_fact:
+        BINARIES: "{{ BINARIES | combine(PUPPETEER_BINARIES) }}"
+        cacheable: true
+
+    - debug:
+        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/_ansible_unused/install_singlefile.yml

@@ -0,0 +1,40 @@
+#!/usr/bin/env ansible-playbook
+---
+
+- import_playbook: ../puppeteer/install_puppeteer.yml
+  when: not BINARIES.chrome.version|default('')
+
+- name: "Install Singlefile"
+  hosts: localhost
+  gather_facts: no
+  vars:
+    SINGLEFILE_VERSION_EXACT: '1.1.54'
+  tasks:
+    - include_role:
+        name: setup_lib_npm
+      vars:
+        MIN_NODE_VERSION: '20.0.0'
+        MIN_NPM_VERSION: '10.0.0'
+
+    - name: "Install npm packages: [single-file-cli]"
+      community.general.npm:
+        name: 'single-file-cli@1.1.54'
+        state: "present"
+        path: '{{BINPROVIDERS.npm.lib_dir_npm}}'
+
+    - name: Load single-file binary from installed NPM package
+      include_role:
+        name: load_binary
+      vars:
+        name: singlefile
+        bin_name: 'single-file'
+        version_cmd: 'single-file --version'
+        PATH: '{{BINPROVIDERS.npm.PATH}}'
+
+    - name: Check that installed Singlefile version matches expected version
+      assert:
+        that: SINGLEFILE_VERSION_EXACT is version(BINARIES.singlefile.version, '==')
+        quiet: true
+
+    - debug:
+        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/_ansible_unused/install_ytdlp.yml

@@ -0,0 +1,45 @@
+#!/usr/bin/env ansible-playbook
+---
+- name: "Install YT-DLP"
+  hosts: localhost
+  gather_facts: True
+  vars:
+    YTDLP_VERSION_MIN: '2024.8.6'
+  tasks:
+    - include_role:
+        name: setup_lib_pip
+      vars:
+        MIN_PYTHON_VERSION: '3.10.0'
+        MIN_PIP_VERSION: '22.0'
+    
+    - name: "Install pip packages: yt-dlp"
+      ansible.builtin.pip:
+        name: 'yt-dlp'
+        state: 'latest'
+        virtualenv: '{{BINPROVIDERS.pip.virtualenv}}'
+        virtualenv_python: "{{BINPROVIDERS.pip.virtualenv_python}}"
+        virtualenv_site_packages: no
+
+    - name: Load YTDLP binary
+      include_role:
+        name: load_binary
+      vars:
+        name: ytdlp
+        bin_name: yt-dlp
+        PATH: '{{BINPROVIDERS.pip.PATH}}'
+
+    - name: Load ffmpeg binary
+      include_role:
+        name: load_binary
+      vars:
+        name: ffmpeg
+        version_cmd: 'ffmpeg -version'
+        PATH: '{{BINPROVIDERS.pip.PATH}}:{{ansible_env.PATH}}'
+
+    - name: Check that installed YT-DLP matches expected version
+      assert:
+        that: BINARIES.ytdlp.version is version(YTDLP_VERSION_MIN, '>=')
+        quiet: true
+
+    - debug:
+        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/_ansible_unused/roles/load_binary/meta/argument_specs.yml

@@ -0,0 +1,31 @@
+---
+argument_specs:
+  main:
+    short_description: Load a specified binary from the environment/PATH into BINARIES fact.
+
+    options:
+
+      name:
+        type: "str"
+        required: true
+        description: "A string key for the binary"
+
+      bin_name:
+        type: "str"
+        required: false
+        description: "The basename of the binary file (optional, defaults to name)"
+
+      abspath:
+        type: "str"
+        required: false
+        description: "An absolute path to the binary (overrides any auto-detected one)"
+
+      version_cmd:
+        type: "str"
+        required: false
+        description: "The command to run to get the binary's version (optional, defaults to $ <bin_name> --version)"
+
+      PATH:
+        type: "str"
+        required: false
+        description: "The PATH to search for the binary (optional, defaults to environment $PATH)"

archivebox/_ansible_unused/roles/load_binary/tasks/main.yml

@@ -0,0 +1,75 @@
+
+---
+- name: Make sure ./data/lib/bin folder exists
+  file:
+    path: '{{LIB_DIR_BIN}}'
+    state: directory
+    recurse: true
+
+###################################################################################
+
+- set_fact:
+    PATH: "{{PATH or DEFAULT_PATH}}"
+    bin_name: "{{bin_name or name}}"
+    BINARY_ABSPATH: null
+    BINARY_VERSION: null
+
+- name: 'Get installed binary abspath: {{name}}'
+  command: 'env PATH="{{PATH}}:$PATH" which {{bin_name}}'
+  register: BINARY_ABSPATH
+  changed_when: False
+
+- set_fact:
+    bin_name: "{{BINARY_ABSPATH.stdout|basename or bin_name}}"
+- set_fact:
+    version_cmd: "{{version_cmd or (bin_name + ' --version')}}"
+
+- name: 'Get installed binary version: {{name}}'
+  command: 'env PATH="{{PATH}}:$PATH" {{version_cmd}}'
+  register: BINARY_VERSION
+  changed_when: False
+
+
+- name: 'Updating BINARIES with loaded abspaths & versions: {{name}}'
+  set_fact:
+    BINARIES: "{{
+      BINARIES
+      | default({})
+      | combine({
+          name: {
+            'name': name,
+            'bin_name': bin_name,
+            'version_cmd': version_cmd,
+            'symlink': LIB_DIR_BIN + '/' + name,
+            'abspath': BINARY_ABSPATH.stdout or abspath or None,
+            'version': BINARY_VERSION.stdout_lines|first|regex_replace('^.*?([\\d+\\.]+).*$', '\\1') or version or None,
+            'version_stdout': BINARY_VERSION.stdout or BINARY_VERSION.stderr,
+            'PATH': BINARY_ABSPATH.stdout|dirname or PATH,
+          },
+      })
+    }}"
+    cacheable: true
+  changed_when: True
+
+- name: 'Symlink installed binary into lib bin folder: {{name}}'
+  file:
+    src: "{{ BINARY_ABSPATH.stdout }}"
+    dest: "{{ LIB_DIR_BIN }}/{{ name }}"
+    state: link
+    force: true
+  when: BINARY_VERSION.stdout_lines|first|regex_replace('^.*?([\\d+\\.]+).*$', '\\1')|length
+
+- debug:
+    msg:
+      - '{{BINARIES}}'
+
+- name: Unset variables
+  set_fact:
+    name:
+    bin_name:
+    version_cmd:
+    abspath:
+    version:
+    PATH:
+    BINARY_ABSPATH:
+    BINARY_VERSION:

archivebox/_ansible_unused/roles/load_binary/vars/main.yml

@@ -0,0 +1,12 @@
+DATA_DIR: '{{playbook_dir}}'
+
+LIB_DIR: '{{DATA_DIR}}/lib'
+LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+DEFAULT_PATH: /bin
+
+name:
+bin_name:
+version_cmd:
+PATH:
+abspath:
+version:

archivebox/_ansible_unused/roles/setup_lib_npm/meta/argument_specs.yml

@@ -0,0 +1,31 @@
+---
+argument_specs:
+  main:
+    short_description: Main entry point for the npm role
+
+    options:
+    
+      state:
+        type: "str"
+        required: false
+        default: 'present'
+        description:
+          - "The desired state: present | latest"
+
+      npm_packages:
+        type: "list"
+        elements: "dict"
+        required: false
+        default: []
+        description: "A list of dicts with a defined structure and with default a value."
+        options:
+          key:
+            type: "str"
+            required: true
+            description: "A string name for the dependency"
+
+          packages:
+            type: "list"
+            elements: "str"
+            required: true
+            description: "What npm packages to install for the given dependency."

archivebox/_ansible_unused/roles/setup_lib_npm/tasks/main.yml

@@ -0,0 +1,99 @@
+---
+- name: Make sure lib folders exist
+  file:
+    path: '{{item}}'
+    state: directory
+    recurse: true
+  loop:
+    - '{{LIB_DIR_NPM_BIN}}'
+    - '{{LIB_DIR_BIN}}'
+
+###################################################################################
+
+- name: Ensure dependencies are present.
+  when: ansible_facts['os_family']|lower == 'debian'
+  ansible.builtin.apt:
+    name:
+      - apt-transport-https
+      - python3-debian
+      - gnupg2
+    state: present
+
+- name: Download NodeSource's signing key.
+  # NodeSource's web server discriminates the User-Agent used by the deb822_repository module.
+  # https://github.com/nodesource/distributions/issues/1723
+  when: ansible_facts['os_family']|lower == 'debian'
+  ansible.builtin.get_url:
+    url: https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key
+    dest: /etc/apt/signing-key-nodesource-repo.asc
+    owner: root
+    group: root
+    mode: '0444'
+  register: node_signing_key
+
+- name: Add NodeSource repositories for Node.js.
+  when: ansible_facts['os_family']|lower == 'debian'
+  ansible.builtin.deb822_repository:
+    name: nodesource_{{ TARGET_NODE_VERSION }}
+    uris: "https://deb.nodesource.com/node_{{ TARGET_NODE_VERSION }}.x"
+    types: deb
+    suites: nodistro
+    components: main
+    signed_by: "{{ node_signing_key.dest }}"
+    state: present
+  register: node_repo
+
+- name: Update apt cache if repo was added.
+  ansible.builtin.apt: update_cache=yes
+  when: ansible_facts['os_family']|lower == 'debian' and node_repo is changed
+
+- name: Ensure Node.js and npm are installed.
+  when: ansible_facts['os_family']|lower == 'debian'
+  ansible.builtin.apt:
+    name: "nodejs={{ TARGET_NODE_VERSION | regex_replace('x', '') }}*"
+    state: present
+
+- name: Load NPM and Node binaries
+  include_role:
+    name: load_binary
+  vars:
+    name: '{{item}}'
+  loop:
+    - node
+    - npm
+
+- name: Check that installed Node version matches expected version
+  assert:
+    that:
+      - BINARIES.node.version is version(MIN_NODE_VERSION, '>=')
+      - BINARIES.npm.version is version(MIN_NPM_VERSION, '>=')
+    quiet: true
+
+###################################################################################
+
+
+# - name: "Install npm packages: {{install_npm}}"
+#   community.general.npm:
+#     name: '{{item}}'
+#     state: "{{state}}"
+#     path: '{{LIB_DIR_NPM}}'
+#   loop: "{{install_npm|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}"
+
+###################################################################################
+
+
+###################################################################################
+- set_fact:
+    NODE_BINPROVIDERS:
+      npm:
+        installer_abspath: "{{BINARIES.npm.abspath}}"
+        installer_version: "{{BINARIES.npm.version}}"
+        PATH: "{{LIB_DIR_NPM_BIN}}"
+        lib_dir_npm: "{{LIB_DIR_NPM}}"
+
+- set_fact:
+    BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(NODE_BINPROVIDERS) }}"
+    cacheable: true
+
+- debug:
+    msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/_ansible_unused/roles/setup_lib_npm/vars/main.yml

@@ -0,0 +1,10 @@
+DATA_DIR: '{{playbook_dir}}'
+
+LIB_DIR: '{{DATA_DIR}}/lib'
+LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+LIB_DIR_NPM: '{{LIB_DIR}}/npm'
+LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin'
+
+TARGET_NODE_VERSION: '21'
+MIN_NODE_VERSION: '20.0.0'
+MIN_NPM_VERSION: '10.0.0'

archivebox/_ansible_unused/roles/setup_lib_pip/meta/argument_specs.yml

@@ -0,0 +1,31 @@
+---
+argument_specs:
+  main:
+    short_description: Main entry point for the npm role
+
+    options:
+    
+      state:
+        type: "str"
+        required: false
+        default: 'present'
+        description:
+          - "The desired state: present | latest"
+
+      npm_packages:
+        type: "list"
+        elements: "dict"
+        required: false
+        default: []
+        description: "A list of dicts with a defined structure and with default a value."
+        options:
+          key:
+            type: "str"
+            required: true
+            description: "A string name for the dependency"
+
+          packages:
+            type: "list"
+            elements: "str"
+            required: true
+            description: "What npm packages to install for the given dependency."

archivebox/_ansible_unused/roles/setup_lib_pip/tasks/main.yml

@@ -0,0 +1,69 @@
+---
+- name: Make sure lib folders exist
+  file:
+    path: '{{item}}'
+    state: directory
+    recurse: true
+  loop:
+    - '{{LIB_DIR_PIP}}'
+    - '{{LIB_DIR_BIN}}'
+  when: BINPROVIDERS.pip is not defined
+
+- name: Load Python and Pip binaries
+  include_role:
+    name: load_binary
+  vars:
+    name: '{{item}}'
+  loop:
+    - python
+    - pip
+  when: BINARIES.python is not defined or BINARIES.pip is not defined
+
+- assert:
+    that:
+      - BINARIES.python.version is version(MIN_PYTHON_VERSION, '>=')
+      - BINARIES.pip.version is version(MIN_PIP_VERSION, '>=')
+    quiet: true
+  when: BINPROVIDERS.pip is not defined
+
+###################################################################################
+
+
+# - name: "Install pip packages: {{install_pip}}"
+#   ansible.builtin.pip:
+#     name: '{{item}}'
+#     state: "{{state}}"
+#     virtualenv: '{{LIB_DIR_PIP}}/venv'
+#     virtualenv_python: "{{BINARIES.python.abspath}}"
+#     virtualenv_site_packages: yes
+#   loop: "{{install_pip|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}"
+
+
+###################################################################################
+- set_fact:
+    PIP_BINPROVIDERS:
+      pip:
+        installer_abspath: "{{BINARIES.pip.abspath}}"
+        installer_version: "{{BINARIES.pip.version}}"
+        PATH: "{{LIB_DIR_PIP_BIN}}"
+        virtualenv: "{{LIB_DIR_PIP}}/venv"
+        virtualenv_python: "{{BINARIES.python.abspath}}"
+  when: BINPROVIDERS.pip is not defined
+
+- set_fact:
+    BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PIP_BINPROVIDERS) }}"
+    cacheable: true
+  changed_when: False
+
+- name: Load Python and Pip binaries from venv
+  include_role:
+    name: load_binary
+  vars:
+    name: '{{item}}'
+    PATH: '{{BINPROVIDERS.pip.PATH}}'
+  loop:
+    - python
+    - pip
+
+- debug:
+    msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/_ansible_unused/roles/setup_lib_pip/vars/main.yml

@@ -0,0 +1,9 @@
+DATA_DIR: '{{playbook_dir}}'
+
+LIB_DIR: '{{DATA_DIR}}/lib'
+LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+LIB_DIR_PIP: '{{LIB_DIR}}/pip'
+LIB_DIR_PIP_BIN: '{{LIB_DIR_PIP}}/venv/bin'
+
+MIN_PYTHON_VERSION: '3.10.0'
+MIN_PIP_VERSION: '22.0'