mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-09 12:21:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

make chrome binary and configs directly runnable and make extractor use external bin

Browse source
This commit is contained in:
Nick Sweeting 2024-12-06 02:04:05 -08:00
parent a572db307b
commit ac53fdf677
No known key found for this signature in database
7 changed files with 316 additions and 83 deletions
Download patch file Download diff file Expand all files Collapse all files

22
archivebox/base_models/models.py
View file

@ -3,11 +3,12 @@ This file provides the Django ABIDField and ABIDModel base model to inherit from
"""
from typing import Any, Dict, Union, List, Set, cast
import json
from uuid import uuid4
from functools import partial
from pathlib import Path
from typing import Any, Dict, Union, List, Set, cast
from charidfield import CharIDField # type: ignore[import-untyped]
from django.contrib import admin
@ -27,6 +28,7 @@ from django_stubs_ext.db.models import TypedModelMeta
from archivebox.index.json import to_json
from archivebox.misc.hashing import get_dir_info
from .abid import (
ABID,
@ -590,18 +592,20 @@ class ModelWithOutputDir(ABIDModel):
"""Write the ./.index.merkle file to the output dir"""
# write self.generate_merkle_tree() to self.output_dir / '.index.merkle'
print(f'{type(self).__name__}[{self.ABID}].save_merkle_index()')
dir_info = get_dir_info(self.OUTPUT_DIR, max_depth=6)
with open(self.OUTPUT_DIR / '.hashes.json', 'w') as f:
json.dump(dir_info, f)
pass
def save_html_index(self, **kwargs) -> None:
# write self.as_html() to self.output_dir / 'index.html'
print(f'{type(self).__name__}[{self.ABID}].save_html_index()')
pass
(self.OUTPUT_DIR / 'index.html').write_text(self.as_html())
def save_json_index(self, **kwargs) -> None:
print(f'{type(self).__name__}[{self.ABID}].save_json_index()')
# write self.as_json() to self.output_dir / 'index.json'
(self.OUTPUT_DIR / 'index.json').write_text(to_json(self.as_json()))
pass
def save_symlinks_index(self) -> None:
print(f'{type(self).__name__}[{self.ABID}].save_symlinks_index()')
@ -610,26 +614,26 @@ class ModelWithOutputDir(ABIDModel):
# ln -s self.output_dir data/archive/1453452234234.21445
pass
def as_json(self) -> dict:
def as_json(self, *keys) -> dict:
"""Get the object's properties as a dict"""
# dump the object's properties to a json-ready dict
return {
'TYPE': self.TYPE,
'id': self.id,
'id': str(self.id),
'abid': str(self.ABID),
'str': str(self),
'modified_at': self.modified_at,
'created_at': self.created_at,
'created_by_id': self.created_by_id,
'created_at': self.created_at,
'modified_at': self.modified_at,
'status': getattr(self, 'status', None),
'retry_at': getattr(self, 'retry_at', None),
'notes': getattr(self, 'notes', None),
**{key: getattr(self, key) for key in keys},
}
def as_html(self) -> str:
"""Get the object's properties as a html string"""
# render snapshot_detail.html template with self as context and return html string
return ''
return str(self)
####################################################

1
archivebox/config/common.py
View file

@ -104,6 +104,7 @@ SERVER_CONFIG = ServerConfig()
class ArchivingConfig(BaseConfigSet):
ONLY_NEW: bool = Field(default=True)
OVERWRITE: bool = Field(default=False)
TIMEOUT: int = Field(default=60)
MEDIA_TIMEOUT: int = Field(default=3600)

119
archivebox/core/models.py
View file

@ -16,7 +16,7 @@ from django.utils.text import slugify
from django.utils import timezone
from django.core.cache import cache
from django.urls import reverse, reverse_lazy
from django.db.models import Case, When, Value, IntegerField
from django.db.models import Case, When, IntegerField
from django.contrib import admin
from django.conf import settings
@ -25,7 +25,8 @@ import abx
from archivebox.config import CONSTANTS
from archivebox.misc.system import get_dir_size
from archivebox.misc.util import parse_date, base_url
from archivebox.misc.util import parse_date, base_url, domain as url_domain
from archivebox.misc.hashing import get_dir_info
from archivebox.index.schema import Link
from archivebox.index.html import snapshot_icons
from archivebox.extractors import ARCHIVE_METHODS_INDEXING_PRECEDENCE
@ -142,8 +143,20 @@ def validate_timestamp(value):
assert value.replace('.', '').isdigit(), f'timestamp must be a float str, got: "{value}"'
class SnapshotManager(models.Manager):
def filter(self, *args, **kwargs):
"""add support for .filter(domain='example.com') to Snapshot queryset"""
domain = kwargs.pop('domain', None)
qs = super().filter(*args, **kwargs)
if domain:
qs = qs.filter(url__icontains=f'://{domain}')
return qs
def get_queryset(self):
return super().get_queryset().prefetch_related('tags', 'archiveresult_set') # .annotate(archiveresult_count=models.Count('archiveresult')).distinct()
return (
super().get_queryset()
.prefetch_related('tags', 'archiveresult_set')
# .annotate(archiveresult_count=models.Count('archiveresult')).distinct()
)
class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithStateMachine, ABIDModel):
abid_prefix = 'snp_'
@ -256,6 +269,13 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithStateMachine, ABIDM
if self.crawl and self.url not in self.crawl.urls:
self.crawl.urls += f'\n{self.url}'
self.crawl.save()
def output_dir_parent(self) -> str:
return 'archive'
def output_dir_name(self) -> str:
return str(self.timestamp)
def archive(self, overwrite=False, methods=None):
result = bg_archive_snapshot(self, overwrite=overwrite, methods=methods)
@ -338,6 +358,10 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithStateMachine, ABIDM
def bookmarked_date(self):
# TODO: remove this
return self.bookmarked
@cached_property
def domain(self) -> str:
return url_domain(self.url)
@cached_property
def is_archived(self):
@ -659,7 +683,8 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
notes = models.TextField(blank=True, null=False, default='', help_text='Any extra notes this ArchiveResult should have')
# the network interface that was used to download this result
# uplink = models.ForeignKey(NetworkInterface, on_delete=models.SET_NULL, null=True, blank=True, verbose_name='Network Interface Used')
# machine = models.ForeignKey(Machine, on_delete=models.SET_NULL, null=True, blank=True, verbose_name='Machine Used')
# network = models.ForeignKey(NetworkInterface, on_delete=models.SET_NULL, null=True, blank=True, verbose_name='Network Interface Used')
objects = ArchiveResultManager()
@ -742,8 +767,7 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
return None
def legacy_output_path(self):
link = self.snapshot.as_link()
return link.canonical_outputs().get(f'{self.extractor}_path')
return self.canonical_outputs().get(f'{self.extractor}_path')
def output_exists(self) -> bool:
output_path = Path(self.snapshot_dir) / self.extractor
@ -761,6 +785,89 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
for key in args
}
def canonical_outputs(self) -> Dict[str, Optional[str]]:
"""Predict the expected output paths that should be present after archiving"""
# You'll need to implement the actual logic based on your requirements
# TODO: banish this awful duplication from the codebase and import these
# from their respective extractor files
from abx_plugin_favicon.config import FAVICON_CONFIG
canonical = {
'index_path': 'index.html',
'favicon_path': 'favicon.ico',
'google_favicon_path': FAVICON_CONFIG.FAVICON_PROVIDER.format(self.domain),
'wget_path': f'warc/{self.timestamp}',
'warc_path': 'warc/',
'singlefile_path': 'singlefile.html',
'readability_path': 'readability/content.html',
'mercury_path': 'mercury/content.html',
'htmltotext_path': 'htmltotext.txt',
'pdf_path': 'output.pdf',
'screenshot_path': 'screenshot.png',
'dom_path': 'output.html',
'archive_org_path': f'https://web.archive.org/web/{self.base_url}',
'git_path': 'git/',
'media_path': 'media/',
'headers_path': 'headers.json',
}
if self.is_static:
static_path = f'warc/{self.timestamp}'
canonical.update({
'title': self.basename,
'wget_path': static_path,
'pdf_path': static_path,
'screenshot_path': static_path,
'dom_path': static_path,
'singlefile_path': static_path,
'readability_path': static_path,
'mercury_path': static_path,
'htmltotext_path': static_path,
})
return canonical
@property
def output_dir_name(self) -> str:
return self.extractor
@property
def output_dir_parent(self) -> str:
return str(self.snapshot.OUTPUT_DIR.relative_to(CONSTANTS.DATA_DIR))
@cached_property
def output_files(self) -> dict[str, dict]:
dir_info = get_dir_info(self.OUTPUT_DIR, max_depth=6)
with open(self.OUTPUT_DIR / '.hashes.json', 'w') as f:
json.dump(dir_info, f)
return dir_info
def announce_event(self, output_type: str, event: dict):
event = {
**event,
'type': output_type,
}
# if event references a file, make sure it exists on disk
if 'path' in event:
file_path = Path(self.OUTPUT_DIR) / event['path']
assert file_path.exists(), f'ArchiveResult[{self.ABID}].announce_event(): File does not exist: {file_path} ({event})'
with open(self.OUTPUT_DIR / '.events.jsonl', 'a') as f:
f.write(json.dumps(event, sort_keys=True, default=str) + '\n')
def events(self, filter_type: str | None=None) -> list[dict]:
events = []
try:
with open(self.OUTPUT_DIR / '.events.jsonl', 'r') as f:
for line in f:
event = json.loads(line)
if filter_type is None or event['type'] == filter_type:
events.append(event)
except FileNotFoundError:
pass
return events
def write_indexes(self):
"""Write the ArchiveResult json, html, and merkle indexes to output dir, and pass searchable text to the search backend"""
super().write_indexes()

9
archivebox/extractors/extractor.py
View file

@ -2,12 +2,15 @@ import hashlib
import mimetypes
import os
import subprocess
from typing import ClassVar
from datetime import timedelta
from zipfile import Path
from django.utils import timezone
from archivebox.misc.hashing import get_dir_info
from core.models import ArchiveResult
import abx
@ -205,9 +208,7 @@ class Extractor:
def after_extract(self, error: Exception | None=None):
status, retry_at = self.determine_status()
self.archiveresult.outputs = []
self.archiveresult.error = f'{type(error).__name__}: {error}' if error else None
self.archiveresult.status = self.archiveresult.StatusChoices.FAILED if error else self.archiveresult.StatusChoices.SUCCEEDED
@ -216,4 +217,4 @@ class Extractor:
self.archiveresult.output = self.archiveresult.outputs[0].path
self.archiveresult.save()
self.archiveresult.write_indexes()

149
archivebox/filestore/models.py
View file

@ -1,67 +1,110 @@
# import mimetypes
# import uuid
import mimetypes
import uuid
from datetime import timedelta
from pathlib import Path
from django.db import models
from django.conf import settings
from django.utils import timezone
# from django.db import models
# from django.conf import settings
# from django.utils import timezone
# from archivebox import DATA_DIR
# from archivebox.misc.hashing import get_dir_info, hash_file
# from base_models.abid import DEFAULT_ABID_URI_SALT
# from base_models.models import ABIDModel, ABIDField, get_or_create_system_user_pk
from archivebox import DATA_DIR
from archivebox.misc.hashing import get_dir_info, hash_file
from base_models.abid import DEFAULT_ABID_URI_SALT
from base_models.models import ABIDModel, ABIDField, get_or_create_system_user_pk
# class File(ABIDModel):
# abid_prefix = 'fil_'
# abid_ts_src = 'self.created_at'
# abid_uri_src = 'self.path'
# abid_subtype_src = 'self.mime_type'
# abid_rand_src = 'self.id'
# abid_salt: str = DEFAULT_ABID_URI_SALT # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
# abid_drift_allowed: bool = False
class File(ABIDModel):
abid_prefix = 'fil_'
abid_ts_src = 'self.created_at'
abid_uri_src = 'self.path'
abid_subtype_src = 'self.mime_type'
abid_rand_src = 'self.id'
abid_salt: str = DEFAULT_ABID_URI_SALT # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
abid_drift_allowed: bool = False
# id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, null=False)
# abid = ABIDField(prefix=abid_prefix)
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, null=False)
abid = ABIDField(prefix=abid_prefix)
# created_at = models.DateTimeField(default=timezone.now, null=False)
# modified_at = models.DateTimeField(default=timezone.now, null=False)
# created_by = models.ForeignKey(settings.USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
created_at = models.DateTimeField(default=timezone.now, null=False)
modified_at = models.DateTimeField(default=timezone.now, null=False)
created_by = models.ForeignKey(settings.USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
# path = models.FilePathField(path=str(DATA_DIR), recursive=True, allow_files=True, allow_folders=True, db_index=True, unique=True)
class StatusChoices(models.TextChoices):
UNLOCKED = 'unlocked'
LOCKED = 'locked'
# basename = models.CharField(max_length=255, default=None, null=False) # e.g. 'index'
# extension = models.CharField(max_length=63, default='', null=False) # e.g. 'html'
# mime_type = models.CharField(max_length=63, default=None, null=False, db_index=True) # e.g. 'inode/directory' or 'text/html'
# num_subpaths = models.IntegerField(default=None, null=False) # e.g. 3
# num_bytes = models.IntegerField(default=None, null=False) # e.g. 123456
status = models.CharField(max_length=16, choices=StatusChoices.choices, default=StatusChoices.UNLOCKED, null=False)
retry_at = models.DateTimeField(default=None, null=True)
version = models.CharField(max_length=16, default='unknown', null=False)
# hash_sha256 = models.CharField(max_length=64, default=None, null=False, db_index=True) # e.g. '5994471abb01112afcc1815994471abb01112afcc1815994471abb01112afcc181'
# # hash_blake3 = models.CharField(max_length=64, default=None, null=False, db_index=True) # e.g. '5994471abb01112afcc1815994471abb01112afcc1815994471abb01112afcc181'
file = models.FileField(null=False)
# DIR = 'inode/directory'
basename = models.CharField(max_length=255, default=None, null=False) # e.g. 'index'
extension = models.CharField(max_length=63, default='', null=False) # e.g. 'html'
mime_type = models.CharField(max_length=63, default=None, null=False, db_index=True) # e.g. 'inode/directory' or 'text/html'
num_subpaths = models.IntegerField(default=None, null=False) # e.g. 3
num_bytes = models.IntegerField(default=None, null=False) # e.g. 123456
sha256 = models.CharField(max_length=64, default=None, null=False, db_index=True) # e.g. '5994471abb01112afcc1815994471abb01112afcc1815994471abb01112afcc181'
# blake3 = models.CharField(max_length=64, default=None, null=False, db_index=True) # e.g. '5994471abb01112afcc1815994471abb01112afcc1815994471abb01112afcc181'
DIR = 'inode/directory'
@classmethod
def release_expired_locks(cls):
cls.objects.filter(status='locked', retry_at__lt=timezone.now()).update(status='unlocked', retry_at=None)
# @property
# def parent(self) -> 'File':
# return File.objects.get(path=self.path.parent) or File(path=self.path.parent)
@property
def parent(self) -> 'File':
return File.objects.get(path=str(self.PATH.parent)) or File(path=str(self.PATH.parent))
@property
def relpath(self) -> Path:
return Path(self.file.name)
@property
def abspath(self) -> Path:
return DATA_DIR / self.file.name
# def save(self, *args, **kwargs):
# assert self.path.exists()
def save(self, *args, **kwargs):
assert self.abspath.exists()
# if self.path.is_dir():
# self.basename = self.path.name
# self.extension = ''
# self.mime_type = self.DIR
# dir_info = get_dir_info(self.path)
# self.num_subpaths = dir_info['.']['num_subpaths']
# self.num_bytes = dir_info['.']['num_bytes']
# self.hash_sha256 = dir_info['.']['hash_sha256']
# # TODO: hash_blake3 = dir_info['.']['hash_blake3']
# else:
# self.basename = self.path.name
# self.extension = self.path.suffix
# self.mime_type = mimetypes.guess_type(self.path)[0]
# self.num_bytes = self.path.stat().st_size
# self.hash_sha256, self.hash_blake3 = hash_file(self.path)
# super().save(*args, **kwargs)
if self.abspath.is_dir():
self.basename = self.relpath.name
self.extension = ''
self.mime_type = self.DIR
dir_info = get_dir_info(self.abspath)
self.num_subpaths = dir_info['.']['num_subpaths']
self.num_bytes = dir_info['.']['num_bytes']
self.hash_sha256 = dir_info['.']['hash_sha256']
# TODO: hash_blake3 = dir_info['.']['hash_blake3']
else:
self.basename = self.relpath.name
self.extension = self.relpath.suffix
self.mime_type = mimetypes.guess_type(self.abspath)[0]
self.num_bytes = self.abspath.stat().st_size
self.hash_sha256, self.hash_blake3 = hash_file(self.abspath)
super().save(*args, **kwargs)
def acquire_lock(self, timeout_seconds: int = 60):
self.status = 'locked'
self.retry_at = timezone.now() + timedelta(seconds=timeout_seconds)
self.save()
def release_lock(self):
self.status = 'unlocked'
self.retry_at = None
self.save()
def move_to(self, new_path: Path):
if str(new_path).startswith(str(DATA_DIR)):
new_relpath = new_path.relative_to(DATA_DIR)
new_abspath = new_path
else:
new_relpath = new_path
new_abspath = DATA_DIR / new_path
new_abspath.parent.mkdir(parents=True, exist_ok=True)
self.abspath.rename(new_abspath)
self.file.name = new_relpath
self.save()

7
archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/binaries.py Normal file → Executable file
View file

@ -1,3 +1,6 @@
#!/usr/bin/env python3
__package__ = 'abx_plugin_chrome'
import os
import platform
from pathlib import Path
@ -147,3 +150,7 @@ class ChromeBinary(Binary):
CHROME_BINARY = ChromeBinary()
if __name__ == '__main__':
binary = CHROME_BINARY.load()
print(binary.version, ' ', binary.abspath)

92
archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/config.py Normal file → Executable file
View file

@ -1,5 +1,8 @@
#!/usr/bin/env python3
import os
from pathlib import Path
import sys
from typing import List, Optional
from pydantic import Field
@ -79,16 +82,67 @@ class ChromeConfig(BaseConfigSet):
# Chrome Binary
CHROME_BINARY: str = Field(default='chrome')
CHROME_DEFAULT_ARGS: List[str] = Field(default=[
"--disable-sync",
"--no-pings",
"--no-first-run", # dont show any first run ui / setup prompts
'--virtual-time-budget=15000', # accellerate any animations on the page by 15s into the future
'--disable-features=DarkMode', # disable dark mode for archiving
"--run-all-compositor-stages-before-draw", # dont draw partially rendered content, wait until everything is ready
"--hide-scrollbars", # hide scrollbars to prevent layout shift / scrollbar visible in screenshots
"--autoplay-policy=no-user-gesture-required", # allow media autoplay without user gesture (e.g. on mobile)
"--use-fake-ui-for-media-stream", # provide fake camera if site tries to request camera access
"--no-default-browser-check",
"--disable-default-apps",
"--ash-no-nudges",
"--disable-infobars",
"--disable-blink-features=AutomationControlled",
"--js-flags=--random-seed=1157259159",
"--deterministic-mode",
"--deterministic-fetch",
"--start-maximized",
"--test-type=gpu",
"--disable-search-engine-choice-screen",
"--disable-session-crashed-bubble",
"--hide-crash-restore-bubble",
"--suppress-message-center-popups",
"--disable-client-side-phishing-detection",
"--disable-domain-reliability",
"--disable-component-update",
"--disable-datasaver-prompt",
"--disable-hang-monitor",
"--disable-session-crashed-bubble",
"--disable-speech-synthesis-api",
"--disable-speech-api",
"--disable-print-preview",
"--safebrowsing-disable-auto-update",
"--deny-permission-prompts",
"--disable-external-intent-requests",
"--disable-notifications",
"--disable-desktop-notifications",
"--noerrdialogs",
"--disable-popup-blocking",
"--disable-prompt-on-repost",
"--silent-debugger-extension-api",
"--block-new-web-contents",
"--metrics-recording-only",
"--disable-breakpad",
"--run-all-compositor-stages-before-draw",
"--use-fake-device-for-media-stream", # provide fake camera if site tries to request camera access
"--simulate-outdated-no-au='Tue, 31 Dec 2099 23:59:59 GMT'", # ignore chrome updates
"--force-gpu-mem-available-mb=4096", # allows for longer full page screenshots https://github.com/puppeteer/puppeteer/issues/5530
"--simulate-outdated-no-au=Tue, 31 Dec 2099 23:59:59 GMT", # ignore chrome updates
"--force-gpu-mem-available-mb=4096", # allows for longer full page screenshots https://github.com/puppeteer/puppeteer/issues/5530
"--password-store=basic",
"--use-mock-keychain",
"--disable-cookie-encryption",
"--allow-legacy-extension-manifests",
"--disable-gesture-requirement-for-media-playback",
"--font-render-hinting=none",
"--force-color-profile=srgb",
"--disable-partial-raster",
"--disable-skia-runtime-opts",
"--disable-2d-canvas-clip-aa",
"--disable-lazy-loading",
"--disable-renderer-backgrounding",
"--disable-background-networking",
"--disable-background-timer-throttling",
"--disable-backgrounding-occluded-windows",
"--disable-ipc-flooding-protection",
"--disable-extensions-http-throttling",
"--disable-field-trial-config",
"--disable-back-forward-cache",
])
CHROME_EXTRA_ARGS: List[str] = Field(default=[])
@ -99,6 +153,7 @@ class ChromeConfig(BaseConfigSet):
CHROME_RESOLUTION: str = Field(default=lambda: ARCHIVING_CONFIG.RESOLUTION)
CHROME_CHECK_SSL_VALIDITY: bool = Field(default=lambda: ARCHIVING_CONFIG.CHECK_SSL_VALIDITY)
# Cookies & Auth
CHROME_USER_AGENT: str = Field(default=lambda: ARCHIVING_CONFIG.USER_AGENT)
CHROME_USER_DATA_DIR: Path | None = Field(default=CONSTANTS.PERSONAS_DIR / 'Default' / 'chrome_profile')
@ -108,6 +163,8 @@ class ChromeConfig(BaseConfigSet):
SAVE_SCREENSHOT: bool = Field(default=True, alias='FETCH_SCREENSHOT')
SAVE_DOM: bool = Field(default=True, alias='FETCH_DOM')
SAVE_PDF: bool = Field(default=True, alias='FETCH_PDF')
OVERWRITE: bool = Field(default=lambda: ARCHIVING_CONFIG.OVERWRITE)
def validate(self):
from archivebox.config.paths import create_and_chown_dir
@ -147,7 +204,11 @@ class ChromeConfig(BaseConfigSet):
self.update_in_place(CHROME_USER_DATA_DIR=None)
@property
def CHROME_ARGS(self) -> str:
# import shlex
# return '\n'.join(shlex.quote(arg) for arg in self.chrome_args())
return '\n'.join(self.chrome_args())
def chrome_args(self, **options) -> List[str]:
"""helper to build up a chrome shell command with arguments"""
@ -157,8 +218,8 @@ class ChromeConfig(BaseConfigSet):
cmd_args = [*options.CHROME_DEFAULT_ARGS, *options.CHROME_EXTRA_ARGS]
if options.CHROME_HEADLESS:
cmd_args += ["--headless=new"] # expects chrome version >= 111
# if options.CHROME_HEADLESS:
# cmd_args += ["--headless"] # expects chrome version >= 111
if not options.CHROME_SANDBOX:
# assume this means we are running inside a docker container
@ -205,3 +266,12 @@ class ChromeConfig(BaseConfigSet):
CHROME_CONFIG = ChromeConfig()
if __name__ == '__main__':
if len(sys.argv) > 1:
result = getattr(CHROME_CONFIG, sys.argv[1], '')
if callable(result):
result = result()
print(result)
else:
print(CHROME_CONFIG.model_dump_json(indent=4))