From da274fd8e87944a66c38d38e7a9abc50e3da59c4 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Fri, 4 Oct 2024 14:48:20 -0700 Subject: [PATCH] remove dead code --- archivebox/_ansible_unused/ansible_utils.py | 59 -- .../_ansible_unused/install_puppeteer.yml | 125 ---- .../_ansible_unused/install_singlefile.yml | 40 -- archivebox/_ansible_unused/install_ytdlp.yml | 45 -- .../roles/load_binary/meta/argument_specs.yml | 31 - .../roles/load_binary/tasks/main.yml | 75 -- .../roles/load_binary/vars/main.yml | 12 - .../setup_lib_npm/meta/argument_specs.yml | 31 - .../roles/setup_lib_npm/tasks/main.yml | 99 --- .../roles/setup_lib_npm/vars/main.yml | 10 - .../setup_lib_pip/meta/argument_specs.yml | 31 - .../roles/setup_lib_pip/tasks/main.yml | 69 -- .../roles/setup_lib_pip/vars/main.yml | 9 - archivebox/misc/tests.py | 642 +++++++++--------- 14 files changed, 321 insertions(+), 957 deletions(-) delete mode 100644 archivebox/_ansible_unused/ansible_utils.py delete mode 100755 archivebox/_ansible_unused/install_puppeteer.yml delete mode 100755 archivebox/_ansible_unused/install_singlefile.yml delete mode 100755 archivebox/_ansible_unused/install_ytdlp.yml delete mode 100644 archivebox/_ansible_unused/roles/load_binary/meta/argument_specs.yml delete mode 100755 archivebox/_ansible_unused/roles/load_binary/tasks/main.yml delete mode 100644 archivebox/_ansible_unused/roles/load_binary/vars/main.yml delete mode 100644 archivebox/_ansible_unused/roles/setup_lib_npm/meta/argument_specs.yml delete mode 100755 archivebox/_ansible_unused/roles/setup_lib_npm/tasks/main.yml delete mode 100644 archivebox/_ansible_unused/roles/setup_lib_npm/vars/main.yml delete mode 100644 archivebox/_ansible_unused/roles/setup_lib_pip/meta/argument_specs.yml delete mode 100755 archivebox/_ansible_unused/roles/setup_lib_pip/tasks/main.yml delete mode 100644 archivebox/_ansible_unused/roles/setup_lib_pip/vars/main.yml diff --git a/archivebox/_ansible_unused/ansible_utils.py b/archivebox/_ansible_unused/ansible_utils.py deleted file mode 100644 index 8957fe5d..00000000 --- a/archivebox/_ansible_unused/ansible_utils.py +++ /dev/null @@ -1,59 +0,0 @@ -# import os - -# from pathlib import Path - -# from benedict import benedict -# from rich.pretty import pprint - -# from ansible_runner import Runner, RunnerConfig - -# GLOBAL_CACHE = {} - - -# def run_playbook(playbook_path, data_dir, quiet=False, **kwargs): -# ANSIBLE_TMP_DIR = str(Path(data_dir) / "tmp" / "ansible") -# os.environ['ANSIBLE_INVENTORY_UNPARSED_WARNING'] = 'False' -# os.environ['ANSIBLE_LOCALHOST_WARNING'] = 'False' -# os.environ["ANSIBLE_HOME"] = ANSIBLE_TMP_DIR -# # os.environ["ANSIBLE_COLLECTIONS_PATH"] = str(Path(data_dir).parent / 'archivebox') -# os.environ["ANSIBLE_ROLES_PATH"] = ( -# './roles' -# ) - -# rc = RunnerConfig( -# private_data_dir=ANSIBLE_TMP_DIR, -# playbook=str(playbook_path), -# rotate_artifacts=50000, -# host_pattern="localhost", -# extravars={ -# "DATA_DIR": str(data_dir), -# **kwargs, -# }, -# quiet=quiet, -# ) -# rc.prepare() -# r = Runner(config=rc) -# r.set_fact_cache('localhost', GLOBAL_CACHE) -# r.run() -# last_run_facts = r.get_fact_cache('localhost') -# GLOBAL_CACHE.update(filtered_facts(last_run_facts)) -# return benedict({ -# key: val -# for key, val in last_run_facts.items() -# if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup')) -# }) - -# def filtered_facts(facts): -# return benedict({ -# key: val -# for key, val in facts.items() -# if not (key.startswith('ansible_') or key in ('gather_subset', 'module_setup')) -# }) - -# def print_globals(): -# pprint(filtered_facts(GLOBAL_CACHE), expand_all=True) - - - -# # YTDLP_OUTPUT = run_playbook('extract.yml', {'url': 'https://www.youtube.com/watch?v=cK4REjqGc9w&t=27s'}) -# # pprint(YTDLP_OUTPUT) diff --git a/archivebox/_ansible_unused/install_puppeteer.yml b/archivebox/_ansible_unused/install_puppeteer.yml deleted file mode 100755 index 29cdaf3f..00000000 --- a/archivebox/_ansible_unused/install_puppeteer.yml +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env ansible-playbook ---- -- name: "Install puppeteer, puppeteer/browsers, and chrome" - hosts: localhost - gather_facts: true - vars: - DATA_DIR: '{{playbook_dir}}' - LIB_DIR: '{{DATA_DIR}}/lib' - LIB_DIR_BIN: '{{LIB_DIR}}/bin' - LIB_DIR_BROWSERS: '{{LIB_DIR}}/browsers' - CHROME_RELEASE_CHANNEL: 'chrome@stable' - CHROME_VERSION_MIN: '128.0.6613.137' - tasks: - - include_role: - name: setup_lib_npm - vars: - TARGET_NODE_VERSION: '21' - MIN_NODE_VERSION: '20.0.0' - MIN_NPM_VERSION: '10.0.0' - - - name: "Install npm packages: [puppeteer, @puppeteer/browsers]" - community.general.npm: - name: '{{item}}' - state: "present" - path: '{{BINPROVIDERS.npm.lib_dir_npm}}' - loop: - - 'puppeteer' - - '@puppeteer/browsers' - - - name: Make sure prerequisite folders exist - file: - path: '{{LIB_DIR_BROWSERS}}' - state: directory - recurse: true - - - name: Load puppeteer binary from installed NPM package - include_role: - name: load_binary - vars: - name: puppeteer - PATH: '{{BINPROVIDERS.npm.PATH}}' - - # - name: Find existing chrome binaries in environment PATH - # include_role: - # name: load_binary - # vars: - # name: 'chrome' - # bin_name: '{{chrome_executable}}' - # PATH: '{{ansible_env.PATH}}' - # loop: - # - chrome - # - chrome-browser - # - chromium - # - chromium-browser - # - google-chrome - # - google-chrome-browser - # - google-chrome-stable - # - google-chrome-beta - # - google-chrome-canary - # - google-chrome-unstable - # - google-chrome-dev - # loop_control: - # loop_var: chrome_executable - # break_when: - # - BINARIES.chrome.version|default('') - - - ################################################################################### - - name: 'Install Chrome browser: npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}}' - command: 'npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}} --path {{LIB_DIR_BROWSERS}}' - register: CHROME_VERSION_FULL - environment: - PATH: "{{BINPROVIDERS.npm.PATH}}:{{ ansible_env.PATH }}" - changed_when: CHROME_VERSION_MIN not in CHROME_VERSION_FULL.stdout - when: not BINARIES.chrome.version|default('') - # -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...' - - ################################################################################### - - name: Parse Chrome version and abspath from npx @puppeteer/browsers install output - set_fact: - CHROME_ABSPATH: "{{ CHROME_VERSION_FULL.stdout_lines|last|split(' ', 1)|last }}" - CHROME_VERSION: "{{ CHROME_VERSION_FULL.stdout_lines|last|split('@', 1)|last|split(' ', 1)|first }}" - when: not BINARIES.chrome.version|default('') - - - name: Create ./bin/chrome symlink to ./browsers/chrome/... binary - # normal symlink doesn't work for .app on macOS because it fails to load ../Framworks/..., so we create a tiny bash script to open it in its correct cwd instead - copy: - content: | - #!/bin/bash - exec '{{CHROME_ABSPATH|default(BINARIES.chrome.abspath)}}' "$@" - dest: "{{LIB_DIR_BIN}}/chrome" - changed_when: False - - - name: Ensure ./bin/chrome symlink is executable - file: - path: "{{LIB_DIR_BIN}}/chrome" - mode: u+rx,g-rx,o-rwx - state: 'file' - changed_when: False - - ################################################################################### - - set_fact: - PUPPETEER_BINARIES: - chrome: - name: 'chrome' - bin_name: 'chrome' - abspath: "{{CHROME_ABSPATH|default(BINARIES.chrome.abspath) or None}}" - version: "{{CHROME_VERSION|default(BINARIES.chrome.version) or None}}" - symlink: "{{LIB_DIR_BIN}}/chrome" - version_cmd: "chrome --version" - version_stdout: "{{CHROME_VERSION_FULL.stdout}}" - binprovider: 'puppeteer' - PATH: "{{LIB_DIR_BIN}}" - - - name: Check that installed Chrome matches expected version - assert: - that: PUPPETEER_BINARIES.chrome.version is version(CHROME_VERSION_MIN, '>=') - quiet: true - - - set_fact: - BINARIES: "{{ BINARIES | combine(PUPPETEER_BINARIES) }}" - cacheable: true - - - debug: - msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/_ansible_unused/install_singlefile.yml b/archivebox/_ansible_unused/install_singlefile.yml deleted file mode 100755 index b155d434..00000000 --- a/archivebox/_ansible_unused/install_singlefile.yml +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env ansible-playbook ---- - -- import_playbook: ../puppeteer/install_puppeteer.yml - when: not BINARIES.chrome.version|default('') - -- name: "Install Singlefile" - hosts: localhost - gather_facts: no - vars: - SINGLEFILE_VERSION_EXACT: '1.1.54' - tasks: - - include_role: - name: setup_lib_npm - vars: - MIN_NODE_VERSION: '20.0.0' - MIN_NPM_VERSION: '10.0.0' - - - name: "Install npm packages: [single-file-cli]" - community.general.npm: - name: 'single-file-cli@1.1.54' - state: "present" - path: '{{BINPROVIDERS.npm.lib_dir_npm}}' - - - name: Load single-file binary from installed NPM package - include_role: - name: load_binary - vars: - name: singlefile - bin_name: 'single-file' - version_cmd: 'single-file --version' - PATH: '{{BINPROVIDERS.npm.PATH}}' - - - name: Check that installed Singlefile version matches expected version - assert: - that: SINGLEFILE_VERSION_EXACT is version(BINARIES.singlefile.version, '==') - quiet: true - - - debug: - msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/_ansible_unused/install_ytdlp.yml b/archivebox/_ansible_unused/install_ytdlp.yml deleted file mode 100755 index b2aa9cc9..00000000 --- a/archivebox/_ansible_unused/install_ytdlp.yml +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env ansible-playbook ---- -- name: "Install YT-DLP" - hosts: localhost - gather_facts: True - vars: - YTDLP_VERSION_MIN: '2024.8.6' - tasks: - - include_role: - name: setup_lib_pip - vars: - MIN_PYTHON_VERSION: '3.10.0' - MIN_PIP_VERSION: '22.0' - - - name: "Install pip packages: yt-dlp" - ansible.builtin.pip: - name: 'yt-dlp' - state: 'latest' - virtualenv: '{{BINPROVIDERS.pip.virtualenv}}' - virtualenv_python: "{{BINPROVIDERS.pip.virtualenv_python}}" - virtualenv_site_packages: no - - - name: Load YTDLP binary - include_role: - name: load_binary - vars: - name: ytdlp - bin_name: yt-dlp - PATH: '{{BINPROVIDERS.pip.PATH}}' - - - name: Load ffmpeg binary - include_role: - name: load_binary - vars: - name: ffmpeg - version_cmd: 'ffmpeg -version' - PATH: '{{BINPROVIDERS.pip.PATH}}:{{ansible_env.PATH}}' - - - name: Check that installed YT-DLP matches expected version - assert: - that: BINARIES.ytdlp.version is version(YTDLP_VERSION_MIN, '>=') - quiet: true - - - debug: - msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/_ansible_unused/roles/load_binary/meta/argument_specs.yml b/archivebox/_ansible_unused/roles/load_binary/meta/argument_specs.yml deleted file mode 100644 index 7f95fa87..00000000 --- a/archivebox/_ansible_unused/roles/load_binary/meta/argument_specs.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -argument_specs: - main: - short_description: Load a specified binary from the environment/PATH into BINARIES fact. - - options: - - name: - type: "str" - required: true - description: "A string key for the binary" - - bin_name: - type: "str" - required: false - description: "The basename of the binary file (optional, defaults to name)" - - abspath: - type: "str" - required: false - description: "An absolute path to the binary (overrides any auto-detected one)" - - version_cmd: - type: "str" - required: false - description: "The command to run to get the binary's version (optional, defaults to $ --version)" - - PATH: - type: "str" - required: false - description: "The PATH to search for the binary (optional, defaults to environment $PATH)" diff --git a/archivebox/_ansible_unused/roles/load_binary/tasks/main.yml b/archivebox/_ansible_unused/roles/load_binary/tasks/main.yml deleted file mode 100755 index 9139bb8d..00000000 --- a/archivebox/_ansible_unused/roles/load_binary/tasks/main.yml +++ /dev/null @@ -1,75 +0,0 @@ - ---- -- name: Make sure ./data/lib/bin folder exists - file: - path: '{{LIB_DIR_BIN}}' - state: directory - recurse: true - -################################################################################### - -- set_fact: - PATH: "{{PATH or DEFAULT_PATH}}" - bin_name: "{{bin_name or name}}" - BINARY_ABSPATH: null - BINARY_VERSION: null - -- name: 'Get installed binary abspath: {{name}}' - command: 'env PATH="{{PATH}}:$PATH" which {{bin_name}}' - register: BINARY_ABSPATH - changed_when: False - -- set_fact: - bin_name: "{{BINARY_ABSPATH.stdout|basename or bin_name}}" -- set_fact: - version_cmd: "{{version_cmd or (bin_name + ' --version')}}" - -- name: 'Get installed binary version: {{name}}' - command: 'env PATH="{{PATH}}:$PATH" {{version_cmd}}' - register: BINARY_VERSION - changed_when: False - - -- name: 'Updating BINARIES with loaded abspaths & versions: {{name}}' - set_fact: - BINARIES: "{{ - BINARIES - | default({}) - | combine({ - name: { - 'name': name, - 'bin_name': bin_name, - 'version_cmd': version_cmd, - 'symlink': LIB_DIR_BIN + '/' + name, - 'abspath': BINARY_ABSPATH.stdout or abspath or None, - 'version': BINARY_VERSION.stdout_lines|first|regex_replace('^.*?([\\d+\\.]+).*$', '\\1') or version or None, - 'version_stdout': BINARY_VERSION.stdout or BINARY_VERSION.stderr, - 'PATH': BINARY_ABSPATH.stdout|dirname or PATH, - }, - }) - }}" - cacheable: true - changed_when: True - -- name: 'Symlink installed binary into lib bin folder: {{name}}' - file: - src: "{{ BINARY_ABSPATH.stdout }}" - dest: "{{ LIB_DIR_BIN }}/{{ name }}" - state: link - force: true - when: BINARY_VERSION.stdout_lines|first|regex_replace('^.*?([\\d+\\.]+).*$', '\\1')|length - -- debug: - msg: - - '{{BINARIES}}' - -- name: Unset variables - set_fact: - name: - bin_name: - version_cmd: - abspath: - version: - PATH: - BINARY_ABSPATH: - BINARY_VERSION: diff --git a/archivebox/_ansible_unused/roles/load_binary/vars/main.yml b/archivebox/_ansible_unused/roles/load_binary/vars/main.yml deleted file mode 100644 index 99c921f0..00000000 --- a/archivebox/_ansible_unused/roles/load_binary/vars/main.yml +++ /dev/null @@ -1,12 +0,0 @@ -DATA_DIR: '{{playbook_dir}}' - -LIB_DIR: '{{DATA_DIR}}/lib' -LIB_DIR_BIN: '{{LIB_DIR}}/bin' -DEFAULT_PATH: /bin - -name: -bin_name: -version_cmd: -PATH: -abspath: -version: diff --git a/archivebox/_ansible_unused/roles/setup_lib_npm/meta/argument_specs.yml b/archivebox/_ansible_unused/roles/setup_lib_npm/meta/argument_specs.yml deleted file mode 100644 index df0647d0..00000000 --- a/archivebox/_ansible_unused/roles/setup_lib_npm/meta/argument_specs.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -argument_specs: - main: - short_description: Main entry point for the npm role - - options: - - state: - type: "str" - required: false - default: 'present' - description: - - "The desired state: present | latest" - - npm_packages: - type: "list" - elements: "dict" - required: false - default: [] - description: "A list of dicts with a defined structure and with default a value." - options: - key: - type: "str" - required: true - description: "A string name for the dependency" - - packages: - type: "list" - elements: "str" - required: true - description: "What npm packages to install for the given dependency." diff --git a/archivebox/_ansible_unused/roles/setup_lib_npm/tasks/main.yml b/archivebox/_ansible_unused/roles/setup_lib_npm/tasks/main.yml deleted file mode 100755 index bf5b627d..00000000 --- a/archivebox/_ansible_unused/roles/setup_lib_npm/tasks/main.yml +++ /dev/null @@ -1,99 +0,0 @@ ---- -- name: Make sure lib folders exist - file: - path: '{{item}}' - state: directory - recurse: true - loop: - - '{{LIB_DIR_NPM_BIN}}' - - '{{LIB_DIR_BIN}}' - -################################################################################### - -- name: Ensure dependencies are present. - when: ansible_facts['os_family']|lower == 'debian' - ansible.builtin.apt: - name: - - apt-transport-https - - python3-debian - - gnupg2 - state: present - -- name: Download NodeSource's signing key. - # NodeSource's web server discriminates the User-Agent used by the deb822_repository module. - # https://github.com/nodesource/distributions/issues/1723 - when: ansible_facts['os_family']|lower == 'debian' - ansible.builtin.get_url: - url: https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key - dest: /etc/apt/signing-key-nodesource-repo.asc - owner: root - group: root - mode: '0444' - register: node_signing_key - -- name: Add NodeSource repositories for Node.js. - when: ansible_facts['os_family']|lower == 'debian' - ansible.builtin.deb822_repository: - name: nodesource_{{ TARGET_NODE_VERSION }} - uris: "https://deb.nodesource.com/node_{{ TARGET_NODE_VERSION }}.x" - types: deb - suites: nodistro - components: main - signed_by: "{{ node_signing_key.dest }}" - state: present - register: node_repo - -- name: Update apt cache if repo was added. - ansible.builtin.apt: update_cache=yes - when: ansible_facts['os_family']|lower == 'debian' and node_repo is changed - -- name: Ensure Node.js and npm are installed. - when: ansible_facts['os_family']|lower == 'debian' - ansible.builtin.apt: - name: "nodejs={{ TARGET_NODE_VERSION | regex_replace('x', '') }}*" - state: present - -- name: Load NPM and Node binaries - include_role: - name: load_binary - vars: - name: '{{item}}' - loop: - - node - - npm - -- name: Check that installed Node version matches expected version - assert: - that: - - BINARIES.node.version is version(MIN_NODE_VERSION, '>=') - - BINARIES.npm.version is version(MIN_NPM_VERSION, '>=') - quiet: true - -################################################################################### - - -# - name: "Install npm packages: {{install_npm}}" -# community.general.npm: -# name: '{{item}}' -# state: "{{state}}" -# path: '{{LIB_DIR_NPM}}' -# loop: "{{install_npm|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}" - -################################################################################### - - -################################################################################### -- set_fact: - NODE_BINPROVIDERS: - npm: - installer_abspath: "{{BINARIES.npm.abspath}}" - installer_version: "{{BINARIES.npm.version}}" - PATH: "{{LIB_DIR_NPM_BIN}}" - lib_dir_npm: "{{LIB_DIR_NPM}}" - -- set_fact: - BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(NODE_BINPROVIDERS) }}" - cacheable: true - -- debug: - msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/_ansible_unused/roles/setup_lib_npm/vars/main.yml b/archivebox/_ansible_unused/roles/setup_lib_npm/vars/main.yml deleted file mode 100644 index 9ad00c1e..00000000 --- a/archivebox/_ansible_unused/roles/setup_lib_npm/vars/main.yml +++ /dev/null @@ -1,10 +0,0 @@ -DATA_DIR: '{{playbook_dir}}' - -LIB_DIR: '{{DATA_DIR}}/lib' -LIB_DIR_BIN: '{{LIB_DIR}}/bin' -LIB_DIR_NPM: '{{LIB_DIR}}/npm' -LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin' - -TARGET_NODE_VERSION: '21' -MIN_NODE_VERSION: '20.0.0' -MIN_NPM_VERSION: '10.0.0' diff --git a/archivebox/_ansible_unused/roles/setup_lib_pip/meta/argument_specs.yml b/archivebox/_ansible_unused/roles/setup_lib_pip/meta/argument_specs.yml deleted file mode 100644 index df0647d0..00000000 --- a/archivebox/_ansible_unused/roles/setup_lib_pip/meta/argument_specs.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -argument_specs: - main: - short_description: Main entry point for the npm role - - options: - - state: - type: "str" - required: false - default: 'present' - description: - - "The desired state: present | latest" - - npm_packages: - type: "list" - elements: "dict" - required: false - default: [] - description: "A list of dicts with a defined structure and with default a value." - options: - key: - type: "str" - required: true - description: "A string name for the dependency" - - packages: - type: "list" - elements: "str" - required: true - description: "What npm packages to install for the given dependency." diff --git a/archivebox/_ansible_unused/roles/setup_lib_pip/tasks/main.yml b/archivebox/_ansible_unused/roles/setup_lib_pip/tasks/main.yml deleted file mode 100755 index 052da4b7..00000000 --- a/archivebox/_ansible_unused/roles/setup_lib_pip/tasks/main.yml +++ /dev/null @@ -1,69 +0,0 @@ ---- -- name: Make sure lib folders exist - file: - path: '{{item}}' - state: directory - recurse: true - loop: - - '{{LIB_DIR_PIP}}' - - '{{LIB_DIR_BIN}}' - when: BINPROVIDERS.pip is not defined - -- name: Load Python and Pip binaries - include_role: - name: load_binary - vars: - name: '{{item}}' - loop: - - python - - pip - when: BINARIES.python is not defined or BINARIES.pip is not defined - -- assert: - that: - - BINARIES.python.version is version(MIN_PYTHON_VERSION, '>=') - - BINARIES.pip.version is version(MIN_PIP_VERSION, '>=') - quiet: true - when: BINPROVIDERS.pip is not defined - -################################################################################### - - -# - name: "Install pip packages: {{install_pip}}" -# ansible.builtin.pip: -# name: '{{item}}' -# state: "{{state}}" -# virtualenv: '{{LIB_DIR_PIP}}/venv' -# virtualenv_python: "{{BINARIES.python.abspath}}" -# virtualenv_site_packages: yes -# loop: "{{install_pip|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}" - - -################################################################################### -- set_fact: - PIP_BINPROVIDERS: - pip: - installer_abspath: "{{BINARIES.pip.abspath}}" - installer_version: "{{BINARIES.pip.version}}" - PATH: "{{LIB_DIR_PIP_BIN}}" - virtualenv: "{{LIB_DIR_PIP}}/venv" - virtualenv_python: "{{BINARIES.python.abspath}}" - when: BINPROVIDERS.pip is not defined - -- set_fact: - BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PIP_BINPROVIDERS) }}" - cacheable: true - changed_when: False - -- name: Load Python and Pip binaries from venv - include_role: - name: load_binary - vars: - name: '{{item}}' - PATH: '{{BINPROVIDERS.pip.PATH}}' - loop: - - python - - pip - -- debug: - msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/_ansible_unused/roles/setup_lib_pip/vars/main.yml b/archivebox/_ansible_unused/roles/setup_lib_pip/vars/main.yml deleted file mode 100644 index 317cf8b4..00000000 --- a/archivebox/_ansible_unused/roles/setup_lib_pip/vars/main.yml +++ /dev/null @@ -1,9 +0,0 @@ -DATA_DIR: '{{playbook_dir}}' - -LIB_DIR: '{{DATA_DIR}}/lib' -LIB_DIR_BIN: '{{LIB_DIR}}/bin' -LIB_DIR_PIP: '{{LIB_DIR}}/pip' -LIB_DIR_PIP_BIN: '{{LIB_DIR_PIP}}/venv/bin' - -MIN_PYTHON_VERSION: '3.10.0' -MIN_PIP_VERSION: '22.0' diff --git a/archivebox/misc/tests.py b/archivebox/misc/tests.py index 4cb34b2e..74bbbb94 100644 --- a/archivebox/misc/tests.py +++ b/archivebox/misc/tests.py @@ -1,332 +1,332 @@ __package__ = 'abx.archivebox' -from django.test import TestCase +# from django.test import TestCase -from .toml_util import convert, TOML_HEADER +# from .toml_util import convert, TOML_HEADER -TEST_INPUT = """ -[SERVER_CONFIG] -IS_TTY=False -USE_COLOR=False -SHOW_PROGRESS=False -IN_DOCKER=False -IN_QEMU=False -PUID=501 -PGID=20 -CONFIG_FILE=/opt/archivebox/data/ArchiveBox.conf -ONLY_NEW=True -TIMEOUT=60 -MEDIA_TIMEOUT=3600 -OUTPUT_PERMISSIONS=644 -RESTRICT_FILE_NAMES=windows -URL_DENYLIST=\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$ -URL_ALLOWLIST=None -ADMIN_USERNAME=None -ADMIN_PASSWORD=None -ENFORCE_ATOMIC_WRITES=True -TAG_SEPARATOR_PATTERN=[,] -SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -BIND_ADDR=127.0.0.1:8000 -ALLOWED_HOSTS=* -DEBUG=False -PUBLIC_INDEX=True -PUBLIC_SNAPSHOTS=True -PUBLIC_ADD_VIEW=False -FOOTER_INFO=Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests. -SNAPSHOTS_PER_PAGE=40 -CUSTOM_TEMPLATES_DIR=None -TIME_ZONE=UTC -TIMEZONE=UTC -REVERSE_PROXY_USER_HEADER=Remote-User -REVERSE_PROXY_WHITELIST= -LOGOUT_REDIRECT_URL=/ -PREVIEW_ORIGINALS=True -LDAP=False -LDAP_SERVER_URI=None -LDAP_BIND_DN=None -LDAP_BIND_PASSWORD=None -LDAP_USER_BASE=None -LDAP_USER_FILTER=None -LDAP_USERNAME_ATTR=None -LDAP_FIRSTNAME_ATTR=None -LDAP_LASTNAME_ATTR=None -LDAP_EMAIL_ATTR=None -LDAP_CREATE_SUPERUSER=False -SAVE_TITLE=True -SAVE_FAVICON=True -SAVE_WGET=True -SAVE_WGET_REQUISITES=True -SAVE_SINGLEFILE=True -SAVE_READABILITY=True -SAVE_MERCURY=True -SAVE_HTMLTOTEXT=True -SAVE_PDF=True -SAVE_SCREENSHOT=True -SAVE_DOM=True -SAVE_HEADERS=True -SAVE_WARC=True -SAVE_GIT=True -SAVE_MEDIA=True -SAVE_ARCHIVE_DOT_ORG=True -RESOLUTION=1440,2000 -GIT_DOMAINS=github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht -CHECK_SSL_VALIDITY=True -MEDIA_MAX_SIZE=750m -USER_AGENT=None -CURL_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.4.0 (x86_64-apple-darwin23.0) -WGET_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.24.5 -CHROME_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) -COOKIES_FILE=None -CHROME_USER_DATA_DIR=None -CHROME_TIMEOUT=0 -CHROME_HEADLESS=True -CHROME_SANDBOX=True -CHROME_EXTRA_ARGS=[] -YOUTUBEDL_ARGS=['--restrict-filenames', '--trim-filenames', '128', '--write-description', '--write-info-json', '--write-annotations', '--write-thumbnail', '--no-call-home', '--write-sub', '--write-auto-subs', '--convert-subs=srt', '--yes-playlist', '--continue', '--no-abort-on-error', '--ignore-errors', '--geo-bypass', '--add-metadata', '--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)'] -YOUTUBEDL_EXTRA_ARGS=[] -WGET_ARGS=['--no-verbose', '--adjust-extension', '--convert-links', '--force-directories', '--backup-converted', '--span-hosts', '--no-parent', '-e', 'robots=off'] -WGET_EXTRA_ARGS=[] -CURL_ARGS=['--silent', '--location', '--compressed'] -CURL_EXTRA_ARGS=[] -GIT_ARGS=['--recursive'] -SINGLEFILE_ARGS=[] -SINGLEFILE_EXTRA_ARGS=[] -MERCURY_ARGS=['--format=text'] -MERCURY_EXTRA_ARGS=[] -FAVICON_PROVIDER=https://www.google.com/s2/favicons?domain={} -USE_INDEXING_BACKEND=True -USE_SEARCHING_BACKEND=True -SEARCH_BACKEND_ENGINE=ripgrep -SEARCH_BACKEND_HOST_NAME=localhost -SEARCH_BACKEND_PORT=1491 -SEARCH_BACKEND_PASSWORD=SecretPassword -SEARCH_PROCESS_HTML=True -SONIC_COLLECTION=archivebox -SONIC_BUCKET=snapshots -SEARCH_BACKEND_TIMEOUT=90 -FTS_SEPARATE_DATABASE=True -FTS_TOKENIZERS=porter unicode61 remove_diacritics 2 -FTS_SQLITE_MAX_LENGTH=1000000000 -USE_CURL=True -USE_WGET=True -USE_SINGLEFILE=True -USE_READABILITY=True -USE_MERCURY=True -USE_GIT=True -USE_CHROME=True -USE_NODE=True -USE_YOUTUBEDL=True -USE_RIPGREP=True -CURL_BINARY=curl -GIT_BINARY=git -WGET_BINARY=wget -SINGLEFILE_BINARY=single-file -READABILITY_BINARY=readability-extractor -MERCURY_BINARY=postlight-parser -YOUTUBEDL_BINARY=yt-dlp -NODE_BINARY=node -RIPGREP_BINARY=rg -CHROME_BINARY=chrome -POCKET_CONSUMER_KEY=None -USER=squash -PACKAGE_DIR=/opt/archivebox/archivebox -TEMPLATES_DIR=/opt/archivebox/archivebox/templates -ARCHIVE_DIR=/opt/archivebox/data/archive -SOURCES_DIR=/opt/archivebox/data/sources -LOGS_DIR=/opt/archivebox/data/logs -PERSONAS_DIR=/opt/archivebox/data/personas -URL_DENYLIST_PTN=re.compile('\\.(css|js|otf|ttf|woff|woff2|gstatic\\.com|googleapis\\.com/css)(\\?.*)?$', re.IGNORECASE|re.MULTILINE) -URL_ALLOWLIST_PTN=None -DIR_OUTPUT_PERMISSIONS=755 -ARCHIVEBOX_BINARY=/opt/archivebox/.venv/bin/archivebox -VERSION=0.8.0 -COMMIT_HASH=102e87578c6036bb0132dd1ebd17f8f05ffc880f -BUILD_TIME=2024-05-15 03:28:05 1715768885 -VERSIONS_AVAILABLE=None -CAN_UPGRADE=False -PYTHON_BINARY=/opt/archivebox/.venv/bin/python3.10 -PYTHON_VERSION=3.10.14 -DJANGO_BINARY=/opt/archivebox/.venv/lib/python3.10/site-packages/django/__init__.py -DJANGO_VERSION=5.0.6 final (0) -SQLITE_BINARY=/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/sqlite3/dbapi2.py -SQLITE_VERSION=2.6.0 -CURL_VERSION=curl 8.4.0 (x86_64-apple-darwin23.0) -WGET_VERSION=GNU Wget 1.24.5 -WGET_AUTO_COMPRESSION=True -RIPGREP_VERSION=ripgrep 14.1.0 -SINGLEFILE_VERSION=None -READABILITY_VERSION=None -MERCURY_VERSION=None -GIT_VERSION=git version 2.44.0 -YOUTUBEDL_VERSION=2024.04.09 -CHROME_VERSION=Google Chrome 124.0.6367.207 -NODE_VERSION=v21.7.3 -""" +# TEST_INPUT = """ +# [SERVER_CONFIG] +# IS_TTY=False +# USE_COLOR=False +# SHOW_PROGRESS=False +# IN_DOCKER=False +# IN_QEMU=False +# PUID=501 +# PGID=20 +# CONFIG_FILE=/opt/archivebox/data/ArchiveBox.conf +# ONLY_NEW=True +# TIMEOUT=60 +# MEDIA_TIMEOUT=3600 +# OUTPUT_PERMISSIONS=644 +# RESTRICT_FILE_NAMES=windows +# URL_DENYLIST=\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$ +# URL_ALLOWLIST=None +# ADMIN_USERNAME=None +# ADMIN_PASSWORD=None +# ENFORCE_ATOMIC_WRITES=True +# TAG_SEPARATOR_PATTERN=[,] +# SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +# BIND_ADDR=127.0.0.1:8000 +# ALLOWED_HOSTS=* +# DEBUG=False +# PUBLIC_INDEX=True +# PUBLIC_SNAPSHOTS=True +# PUBLIC_ADD_VIEW=False +# FOOTER_INFO=Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests. +# SNAPSHOTS_PER_PAGE=40 +# CUSTOM_TEMPLATES_DIR=None +# TIME_ZONE=UTC +# TIMEZONE=UTC +# REVERSE_PROXY_USER_HEADER=Remote-User +# REVERSE_PROXY_WHITELIST= +# LOGOUT_REDIRECT_URL=/ +# PREVIEW_ORIGINALS=True +# LDAP=False +# LDAP_SERVER_URI=None +# LDAP_BIND_DN=None +# LDAP_BIND_PASSWORD=None +# LDAP_USER_BASE=None +# LDAP_USER_FILTER=None +# LDAP_USERNAME_ATTR=None +# LDAP_FIRSTNAME_ATTR=None +# LDAP_LASTNAME_ATTR=None +# LDAP_EMAIL_ATTR=None +# LDAP_CREATE_SUPERUSER=False +# SAVE_TITLE=True +# SAVE_FAVICON=True +# SAVE_WGET=True +# SAVE_WGET_REQUISITES=True +# SAVE_SINGLEFILE=True +# SAVE_READABILITY=True +# SAVE_MERCURY=True +# SAVE_HTMLTOTEXT=True +# SAVE_PDF=True +# SAVE_SCREENSHOT=True +# SAVE_DOM=True +# SAVE_HEADERS=True +# SAVE_WARC=True +# SAVE_GIT=True +# SAVE_MEDIA=True +# SAVE_ARCHIVE_DOT_ORG=True +# RESOLUTION=1440,2000 +# GIT_DOMAINS=github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht +# CHECK_SSL_VALIDITY=True +# MEDIA_MAX_SIZE=750m +# USER_AGENT=None +# CURL_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.4.0 (x86_64-apple-darwin23.0) +# WGET_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.24.5 +# CHROME_USER_AGENT=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) +# COOKIES_FILE=None +# CHROME_USER_DATA_DIR=None +# CHROME_TIMEOUT=0 +# CHROME_HEADLESS=True +# CHROME_SANDBOX=True +# CHROME_EXTRA_ARGS=[] +# YOUTUBEDL_ARGS=['--restrict-filenames', '--trim-filenames', '128', '--write-description', '--write-info-json', '--write-annotations', '--write-thumbnail', '--no-call-home', '--write-sub', '--write-auto-subs', '--convert-subs=srt', '--yes-playlist', '--continue', '--no-abort-on-error', '--ignore-errors', '--geo-bypass', '--add-metadata', '--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)'] +# YOUTUBEDL_EXTRA_ARGS=[] +# WGET_ARGS=['--no-verbose', '--adjust-extension', '--convert-links', '--force-directories', '--backup-converted', '--span-hosts', '--no-parent', '-e', 'robots=off'] +# WGET_EXTRA_ARGS=[] +# CURL_ARGS=['--silent', '--location', '--compressed'] +# CURL_EXTRA_ARGS=[] +# GIT_ARGS=['--recursive'] +# SINGLEFILE_ARGS=[] +# SINGLEFILE_EXTRA_ARGS=[] +# MERCURY_ARGS=['--format=text'] +# MERCURY_EXTRA_ARGS=[] +# FAVICON_PROVIDER=https://www.google.com/s2/favicons?domain={} +# USE_INDEXING_BACKEND=True +# USE_SEARCHING_BACKEND=True +# SEARCH_BACKEND_ENGINE=ripgrep +# SEARCH_BACKEND_HOST_NAME=localhost +# SEARCH_BACKEND_PORT=1491 +# SEARCH_BACKEND_PASSWORD=SecretPassword +# SEARCH_PROCESS_HTML=True +# SONIC_COLLECTION=archivebox +# SONIC_BUCKET=snapshots +# SEARCH_BACKEND_TIMEOUT=90 +# FTS_SEPARATE_DATABASE=True +# FTS_TOKENIZERS=porter unicode61 remove_diacritics 2 +# FTS_SQLITE_MAX_LENGTH=1000000000 +# USE_CURL=True +# USE_WGET=True +# USE_SINGLEFILE=True +# USE_READABILITY=True +# USE_MERCURY=True +# USE_GIT=True +# USE_CHROME=True +# USE_NODE=True +# USE_YOUTUBEDL=True +# USE_RIPGREP=True +# CURL_BINARY=curl +# GIT_BINARY=git +# WGET_BINARY=wget +# SINGLEFILE_BINARY=single-file +# READABILITY_BINARY=readability-extractor +# MERCURY_BINARY=postlight-parser +# YOUTUBEDL_BINARY=yt-dlp +# NODE_BINARY=node +# RIPGREP_BINARY=rg +# CHROME_BINARY=chrome +# POCKET_CONSUMER_KEY=None +# USER=squash +# PACKAGE_DIR=/opt/archivebox/archivebox +# TEMPLATES_DIR=/opt/archivebox/archivebox/templates +# ARCHIVE_DIR=/opt/archivebox/data/archive +# SOURCES_DIR=/opt/archivebox/data/sources +# LOGS_DIR=/opt/archivebox/data/logs +# PERSONAS_DIR=/opt/archivebox/data/personas +# URL_DENYLIST_PTN=re.compile('\\.(css|js|otf|ttf|woff|woff2|gstatic\\.com|googleapis\\.com/css)(\\?.*)?$', re.IGNORECASE|re.MULTILINE) +# URL_ALLOWLIST_PTN=None +# DIR_OUTPUT_PERMISSIONS=755 +# ARCHIVEBOX_BINARY=/opt/archivebox/.venv/bin/archivebox +# VERSION=0.8.0 +# COMMIT_HASH=102e87578c6036bb0132dd1ebd17f8f05ffc880f +# BUILD_TIME=2024-05-15 03:28:05 1715768885 +# VERSIONS_AVAILABLE=None +# CAN_UPGRADE=False +# PYTHON_BINARY=/opt/archivebox/.venv/bin/python3.10 +# PYTHON_VERSION=3.10.14 +# DJANGO_BINARY=/opt/archivebox/.venv/lib/python3.10/site-packages/django/__init__.py +# DJANGO_VERSION=5.0.6 final (0) +# SQLITE_BINARY=/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/sqlite3/dbapi2.py +# SQLITE_VERSION=2.6.0 +# CURL_VERSION=curl 8.4.0 (x86_64-apple-darwin23.0) +# WGET_VERSION=GNU Wget 1.24.5 +# WGET_AUTO_COMPRESSION=True +# RIPGREP_VERSION=ripgrep 14.1.0 +# SINGLEFILE_VERSION=None +# READABILITY_VERSION=None +# MERCURY_VERSION=None +# GIT_VERSION=git version 2.44.0 +# YOUTUBEDL_VERSION=2024.04.09 +# CHROME_VERSION=Google Chrome 124.0.6367.207 +# NODE_VERSION=v21.7.3 +# """ -EXPECTED_OUTPUT = TOML_HEADER + '''[SERVER_CONFIG] -IS_TTY = false -USE_COLOR = false -SHOW_PROGRESS = false -IN_DOCKER = false -IN_QEMU = false -PUID = 501 -PGID = 20 -CONFIG_FILE = "/opt/archivebox/data/ArchiveBox.conf" -ONLY_NEW = true -TIMEOUT = 60 -MEDIA_TIMEOUT = 3600 -OUTPUT_PERMISSIONS = 644 -RESTRICT_FILE_NAMES = "windows" -URL_DENYLIST = "\\\\.(css|js|otf|ttf|woff|woff2|gstatic\\\\.com|googleapis\\\\.com/css)(\\\\?.*)?$" -URL_ALLOWLIST = null -ADMIN_USERNAME = null -ADMIN_PASSWORD = null -ENFORCE_ATOMIC_WRITES = true -TAG_SEPARATOR_PATTERN = "[,]" -SECRET_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" -BIND_ADDR = "127.0.0.1:8000" -ALLOWED_HOSTS = "*" -DEBUG = false -PUBLIC_INDEX = true -PUBLIC_SNAPSHOTS = true -PUBLIC_ADD_VIEW = false -FOOTER_INFO = "Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests." -SNAPSHOTS_PER_PAGE = 40 -CUSTOM_TEMPLATES_DIR = null -TIME_ZONE = "UTC" -TIMEZONE = "UTC" -REVERSE_PROXY_USER_HEADER = "Remote-User" -REVERSE_PROXY_WHITELIST = "" -LOGOUT_REDIRECT_URL = "/" -PREVIEW_ORIGINALS = true -LDAP = false -LDAP_SERVER_URI = null -LDAP_BIND_DN = null -LDAP_BIND_PASSWORD = null -LDAP_USER_BASE = null -LDAP_USER_FILTER = null -LDAP_USERNAME_ATTR = null -LDAP_FIRSTNAME_ATTR = null -LDAP_LASTNAME_ATTR = null -LDAP_EMAIL_ATTR = null -LDAP_CREATE_SUPERUSER = false -SAVE_TITLE = true -SAVE_FAVICON = true -SAVE_WGET = true -SAVE_WGET_REQUISITES = true -SAVE_SINGLEFILE = true -SAVE_READABILITY = true -SAVE_MERCURY = true -SAVE_HTMLTOTEXT = true -SAVE_PDF = true -SAVE_SCREENSHOT = true -SAVE_DOM = true -SAVE_HEADERS = true -SAVE_WARC = true -SAVE_GIT = true -SAVE_MEDIA = true -SAVE_ARCHIVE_DOT_ORG = true -RESOLUTION = [1440, 2000] -GIT_DOMAINS = "github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht" -CHECK_SSL_VALIDITY = true -MEDIA_MAX_SIZE = "750m" -USER_AGENT = null -CURL_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.4.0 (x86_64-apple-darwin23.0)" -WGET_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.24.5" -CHROME_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/)" -COOKIES_FILE = null -CHROME_USER_DATA_DIR = null -CHROME_TIMEOUT = false -CHROME_HEADLESS = true -CHROME_SANDBOX = true -CHROME_EXTRA_ARGS = [] -YOUTUBEDL_ARGS = ["--restrict-filenames", "--trim-filenames", "128", "--write-description", "--write-info-json", "--write-annotations", "--write-thumbnail", "--no-call-home", "--write-sub", "--write-auto-subs", "--convert-subs=srt", "--yes-playlist", "--continue", "--no-abort-on-error", "--ignore-errors", "--geo-bypass", "--add-metadata", "--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)"] -YOUTUBEDL_EXTRA_ARGS = [] -WGET_ARGS = ["--no-verbose", "--adjust-extension", "--convert-links", "--force-directories", "--backup-converted", "--span-hosts", "--no-parent", "-e", "robots=off"] -WGET_EXTRA_ARGS = [] -CURL_ARGS = ["--silent", "--location", "--compressed"] -CURL_EXTRA_ARGS = [] -GIT_ARGS = ["--recursive"] -SINGLEFILE_ARGS = [] -SINGLEFILE_EXTRA_ARGS = [] -MERCURY_ARGS = ["--format=text"] -MERCURY_EXTRA_ARGS = [] -FAVICON_PROVIDER = "https://www.google.com/s2/favicons?domain={}" -USE_INDEXING_BACKEND = true -USE_SEARCHING_BACKEND = true -SEARCH_BACKEND_ENGINE = "ripgrep" -SEARCH_BACKEND_HOST_NAME = "localhost" -SEARCH_BACKEND_PORT = 1491 -SEARCH_BACKEND_PASSWORD = "SecretPassword" -SEARCH_PROCESS_HTML = true -SONIC_COLLECTION = "archivebox" -SONIC_BUCKET = "snapshots" -SEARCH_BACKEND_TIMEOUT = 90 -FTS_SEPARATE_DATABASE = true -FTS_TOKENIZERS = "porter unicode61 remove_diacritics 2" -FTS_SQLITE_MAX_LENGTH = 1000000000 -USE_CURL = true -USE_WGET = true -USE_SINGLEFILE = true -USE_READABILITY = true -USE_MERCURY = true -USE_GIT = true -USE_CHROME = true -USE_NODE = true -USE_YOUTUBEDL = true -USE_RIPGREP = true -CURL_BINARY = "curl" -GIT_BINARY = "git" -WGET_BINARY = "wget" -SINGLEFILE_BINARY = "single-file" -READABILITY_BINARY = "readability-extractor" -MERCURY_BINARY = "postlight-parser" -YOUTUBEDL_BINARY = "yt-dlp" -NODE_BINARY = "node" -RIPGREP_BINARY = "rg" -CHROME_BINARY = "chrome" -POCKET_CONSUMER_KEY = null -USER = "squash" -PACKAGE_DIR = "/opt/archivebox/archivebox" -TEMPLATES_DIR = "/opt/archivebox/archivebox/templates" -ARCHIVE_DIR = "/opt/archivebox/data/archive" -SOURCES_DIR = "/opt/archivebox/data/sources" -LOGS_DIR = "/opt/archivebox/data/logs" -PERSONAS_DIR = "/opt/archivebox/data/personas" -URL_DENYLIST_PTN = "re.compile(\'\\\\.(css|js|otf|ttf|woff|woff2|gstatic\\\\.com|googleapis\\\\.com/css)(\\\\?.*)?$\', re.IGNORECASE|re.MULTILINE)" -URL_ALLOWLIST_PTN = null -DIR_OUTPUT_PERMISSIONS = 755 -ARCHIVEBOX_BINARY = "/opt/archivebox/.venv/bin/archivebox" -VERSION = "0.8.0" -COMMIT_HASH = "102e87578c6036bb0132dd1ebd17f8f05ffc880f" -BUILD_TIME = "2024-05-15 03:28:05 1715768885" -VERSIONS_AVAILABLE = null -CAN_UPGRADE = false -PYTHON_BINARY = "/opt/archivebox/.venv/bin/python3.10" -PYTHON_VERSION = "3.10.14" -DJANGO_BINARY = "/opt/archivebox/.venv/lib/python3.10/site-packages/django/__init__.py" -DJANGO_VERSION = "5.0.6 final (0)" -SQLITE_BINARY = "/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/sqlite3/dbapi2.py" -SQLITE_VERSION = "2.6.0" -CURL_VERSION = "curl 8.4.0 (x86_64-apple-darwin23.0)" -WGET_VERSION = "GNU Wget 1.24.5" -WGET_AUTO_COMPRESSION = true -RIPGREP_VERSION = "ripgrep 14.1.0" -SINGLEFILE_VERSION = null -READABILITY_VERSION = null -MERCURY_VERSION = null -GIT_VERSION = "git version 2.44.0" -YOUTUBEDL_VERSION = "2024.04.09" -CHROME_VERSION = "Google Chrome 124.0.6367.207" -NODE_VERSION = "v21.7.3"''' +# EXPECTED_OUTPUT = TOML_HEADER + '''[SERVER_CONFIG] +# IS_TTY = false +# USE_COLOR = false +# SHOW_PROGRESS = false +# IN_DOCKER = false +# IN_QEMU = false +# PUID = 501 +# PGID = 20 +# CONFIG_FILE = "/opt/archivebox/data/ArchiveBox.conf" +# ONLY_NEW = true +# TIMEOUT = 60 +# MEDIA_TIMEOUT = 3600 +# OUTPUT_PERMISSIONS = 644 +# RESTRICT_FILE_NAMES = "windows" +# URL_DENYLIST = "\\\\.(css|js|otf|ttf|woff|woff2|gstatic\\\\.com|googleapis\\\\.com/css)(\\\\?.*)?$" +# URL_ALLOWLIST = null +# ADMIN_USERNAME = null +# ADMIN_PASSWORD = null +# ENFORCE_ATOMIC_WRITES = true +# TAG_SEPARATOR_PATTERN = "[,]" +# SECRET_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +# BIND_ADDR = "127.0.0.1:8000" +# ALLOWED_HOSTS = "*" +# DEBUG = false +# PUBLIC_INDEX = true +# PUBLIC_SNAPSHOTS = true +# PUBLIC_ADD_VIEW = false +# FOOTER_INFO = "Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests." +# SNAPSHOTS_PER_PAGE = 40 +# CUSTOM_TEMPLATES_DIR = null +# TIME_ZONE = "UTC" +# TIMEZONE = "UTC" +# REVERSE_PROXY_USER_HEADER = "Remote-User" +# REVERSE_PROXY_WHITELIST = "" +# LOGOUT_REDIRECT_URL = "/" +# PREVIEW_ORIGINALS = true +# LDAP = false +# LDAP_SERVER_URI = null +# LDAP_BIND_DN = null +# LDAP_BIND_PASSWORD = null +# LDAP_USER_BASE = null +# LDAP_USER_FILTER = null +# LDAP_USERNAME_ATTR = null +# LDAP_FIRSTNAME_ATTR = null +# LDAP_LASTNAME_ATTR = null +# LDAP_EMAIL_ATTR = null +# LDAP_CREATE_SUPERUSER = false +# SAVE_TITLE = true +# SAVE_FAVICON = true +# SAVE_WGET = true +# SAVE_WGET_REQUISITES = true +# SAVE_SINGLEFILE = true +# SAVE_READABILITY = true +# SAVE_MERCURY = true +# SAVE_HTMLTOTEXT = true +# SAVE_PDF = true +# SAVE_SCREENSHOT = true +# SAVE_DOM = true +# SAVE_HEADERS = true +# SAVE_WARC = true +# SAVE_GIT = true +# SAVE_MEDIA = true +# SAVE_ARCHIVE_DOT_ORG = true +# RESOLUTION = [1440, 2000] +# GIT_DOMAINS = "github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht" +# CHECK_SSL_VALIDITY = true +# MEDIA_MAX_SIZE = "750m" +# USER_AGENT = null +# CURL_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) curl/curl 8.4.0 (x86_64-apple-darwin23.0)" +# WGET_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/) wget/GNU Wget 1.24.5" +# CHROME_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 ArchiveBox/0.8.0 (+https://github.com/ArchiveBox/ArchiveBox/)" +# COOKIES_FILE = null +# CHROME_USER_DATA_DIR = null +# CHROME_TIMEOUT = false +# CHROME_HEADLESS = true +# CHROME_SANDBOX = true +# CHROME_EXTRA_ARGS = [] +# YOUTUBEDL_ARGS = ["--restrict-filenames", "--trim-filenames", "128", "--write-description", "--write-info-json", "--write-annotations", "--write-thumbnail", "--no-call-home", "--write-sub", "--write-auto-subs", "--convert-subs=srt", "--yes-playlist", "--continue", "--no-abort-on-error", "--ignore-errors", "--geo-bypass", "--add-metadata", "--format=(bv*+ba/b)[filesize<=750m][filesize_approx<=?750m]/(bv*+ba/b)"] +# YOUTUBEDL_EXTRA_ARGS = [] +# WGET_ARGS = ["--no-verbose", "--adjust-extension", "--convert-links", "--force-directories", "--backup-converted", "--span-hosts", "--no-parent", "-e", "robots=off"] +# WGET_EXTRA_ARGS = [] +# CURL_ARGS = ["--silent", "--location", "--compressed"] +# CURL_EXTRA_ARGS = [] +# GIT_ARGS = ["--recursive"] +# SINGLEFILE_ARGS = [] +# SINGLEFILE_EXTRA_ARGS = [] +# MERCURY_ARGS = ["--format=text"] +# MERCURY_EXTRA_ARGS = [] +# FAVICON_PROVIDER = "https://www.google.com/s2/favicons?domain={}" +# USE_INDEXING_BACKEND = true +# USE_SEARCHING_BACKEND = true +# SEARCH_BACKEND_ENGINE = "ripgrep" +# SEARCH_BACKEND_HOST_NAME = "localhost" +# SEARCH_BACKEND_PORT = 1491 +# SEARCH_BACKEND_PASSWORD = "SecretPassword" +# SEARCH_PROCESS_HTML = true +# SONIC_COLLECTION = "archivebox" +# SONIC_BUCKET = "snapshots" +# SEARCH_BACKEND_TIMEOUT = 90 +# FTS_SEPARATE_DATABASE = true +# FTS_TOKENIZERS = "porter unicode61 remove_diacritics 2" +# FTS_SQLITE_MAX_LENGTH = 1000000000 +# USE_CURL = true +# USE_WGET = true +# USE_SINGLEFILE = true +# USE_READABILITY = true +# USE_MERCURY = true +# USE_GIT = true +# USE_CHROME = true +# USE_NODE = true +# USE_YOUTUBEDL = true +# USE_RIPGREP = true +# CURL_BINARY = "curl" +# GIT_BINARY = "git" +# WGET_BINARY = "wget" +# SINGLEFILE_BINARY = "single-file" +# READABILITY_BINARY = "readability-extractor" +# MERCURY_BINARY = "postlight-parser" +# YOUTUBEDL_BINARY = "yt-dlp" +# NODE_BINARY = "node" +# RIPGREP_BINARY = "rg" +# CHROME_BINARY = "chrome" +# POCKET_CONSUMER_KEY = null +# USER = "squash" +# PACKAGE_DIR = "/opt/archivebox/archivebox" +# TEMPLATES_DIR = "/opt/archivebox/archivebox/templates" +# ARCHIVE_DIR = "/opt/archivebox/data/archive" +# SOURCES_DIR = "/opt/archivebox/data/sources" +# LOGS_DIR = "/opt/archivebox/data/logs" +# PERSONAS_DIR = "/opt/archivebox/data/personas" +# URL_DENYLIST_PTN = "re.compile(\'\\\\.(css|js|otf|ttf|woff|woff2|gstatic\\\\.com|googleapis\\\\.com/css)(\\\\?.*)?$\', re.IGNORECASE|re.MULTILINE)" +# URL_ALLOWLIST_PTN = null +# DIR_OUTPUT_PERMISSIONS = 755 +# ARCHIVEBOX_BINARY = "/opt/archivebox/.venv/bin/archivebox" +# VERSION = "0.8.0" +# COMMIT_HASH = "102e87578c6036bb0132dd1ebd17f8f05ffc880f" +# BUILD_TIME = "2024-05-15 03:28:05 1715768885" +# VERSIONS_AVAILABLE = null +# CAN_UPGRADE = false +# PYTHON_BINARY = "/opt/archivebox/.venv/bin/python3.10" +# PYTHON_VERSION = "3.10.14" +# DJANGO_BINARY = "/opt/archivebox/.venv/lib/python3.10/site-packages/django/__init__.py" +# DJANGO_VERSION = "5.0.6 final (0)" +# SQLITE_BINARY = "/opt/homebrew/Cellar/python@3.10/3.10.14/Frameworks/Python.framework/Versions/3.10/lib/python3.10/sqlite3/dbapi2.py" +# SQLITE_VERSION = "2.6.0" +# CURL_VERSION = "curl 8.4.0 (x86_64-apple-darwin23.0)" +# WGET_VERSION = "GNU Wget 1.24.5" +# WGET_AUTO_COMPRESSION = true +# RIPGREP_VERSION = "ripgrep 14.1.0" +# SINGLEFILE_VERSION = null +# READABILITY_VERSION = null +# MERCURY_VERSION = null +# GIT_VERSION = "git version 2.44.0" +# YOUTUBEDL_VERSION = "2024.04.09" +# CHROME_VERSION = "Google Chrome 124.0.6367.207" +# NODE_VERSION = "v21.7.3"''' -class IniToTomlTests(TestCase): - def test_convert(self): - first_output = convert(TEST_INPUT) # make sure ini -> toml parses correctly - second_output = convert(first_output) # make sure toml -> toml parses/dumps consistently - assert first_output == second_output == EXPECTED_OUTPUT # make sure parsing is indempotent +# class IniToTomlTests(TestCase): +# def test_convert(self): +# first_output = convert(TEST_INPUT) # make sure ini -> toml parses correctly +# second_output = convert(first_output) # make sure toml -> toml parses/dumps consistently +# assert first_output == second_output == EXPECTED_OUTPUT # make sure parsing is indempotent # # DEBUGGING # import sys