diff --git a/archivebox/base_models/models.py b/archivebox/base_models/models.py
index 7ffc4eea..058eeaae 100644
--- a/archivebox/base_models/models.py
+++ b/archivebox/base_models/models.py
@@ -3,11 +3,12 @@ This file provides the Django ABIDField and ABIDModel base model to inherit from
"""
-from typing import Any, Dict, Union, List, Set, cast
+import json
from uuid import uuid4
from functools import partial
from pathlib import Path
+from typing import Any, Dict, Union, List, Set, cast
from charidfield import CharIDField # type: ignore[import-untyped]
from django.contrib import admin
@@ -27,6 +28,7 @@ from django_stubs_ext.db.models import TypedModelMeta
from archivebox.index.json import to_json
+from archivebox.misc.hashing import get_dir_info
from .abid import (
ABID,
@@ -590,18 +592,20 @@ class ModelWithOutputDir(ABIDModel):
"""Write the ./.index.merkle file to the output dir"""
# write self.generate_merkle_tree() to self.output_dir / '.index.merkle'
print(f'{type(self).__name__}[{self.ABID}].save_merkle_index()')
+ dir_info = get_dir_info(self.OUTPUT_DIR, max_depth=6)
+ with open(self.OUTPUT_DIR / '.hashes.json', 'w') as f:
+ json.dump(dir_info, f)
pass
def save_html_index(self, **kwargs) -> None:
# write self.as_html() to self.output_dir / 'index.html'
print(f'{type(self).__name__}[{self.ABID}].save_html_index()')
- pass
+ (self.OUTPUT_DIR / 'index.html').write_text(self.as_html())
def save_json_index(self, **kwargs) -> None:
print(f'{type(self).__name__}[{self.ABID}].save_json_index()')
# write self.as_json() to self.output_dir / 'index.json'
(self.OUTPUT_DIR / 'index.json').write_text(to_json(self.as_json()))
- pass
def save_symlinks_index(self) -> None:
print(f'{type(self).__name__}[{self.ABID}].save_symlinks_index()')
@@ -610,26 +614,26 @@ class ModelWithOutputDir(ABIDModel):
# ln -s self.output_dir data/archive/1453452234234.21445
pass
- def as_json(self) -> dict:
+ def as_json(self, *keys) -> dict:
"""Get the object's properties as a dict"""
- # dump the object's properties to a json-ready dict
return {
'TYPE': self.TYPE,
- 'id': self.id,
+ 'id': str(self.id),
'abid': str(self.ABID),
'str': str(self),
- 'modified_at': self.modified_at,
- 'created_at': self.created_at,
'created_by_id': self.created_by_id,
+ 'created_at': self.created_at,
+ 'modified_at': self.modified_at,
'status': getattr(self, 'status', None),
'retry_at': getattr(self, 'retry_at', None),
'notes': getattr(self, 'notes', None),
+ **{key: getattr(self, key) for key in keys},
}
def as_html(self) -> str:
"""Get the object's properties as a html string"""
# render snapshot_detail.html template with self as context and return html string
- return ''
+ return str(self)
####################################################
diff --git a/archivebox/config/common.py b/archivebox/config/common.py
index 36d32705..89edba33 100644
--- a/archivebox/config/common.py
+++ b/archivebox/config/common.py
@@ -104,6 +104,7 @@ SERVER_CONFIG = ServerConfig()
class ArchivingConfig(BaseConfigSet):
ONLY_NEW: bool = Field(default=True)
+ OVERWRITE: bool = Field(default=False)
TIMEOUT: int = Field(default=60)
MEDIA_TIMEOUT: int = Field(default=3600)
diff --git a/archivebox/core/models.py b/archivebox/core/models.py
index bedb32d6..fe984bb2 100644
--- a/archivebox/core/models.py
+++ b/archivebox/core/models.py
@@ -16,7 +16,7 @@ from django.utils.text import slugify
from django.utils import timezone
from django.core.cache import cache
from django.urls import reverse, reverse_lazy
-from django.db.models import Case, When, Value, IntegerField
+from django.db.models import Case, When, IntegerField
from django.contrib import admin
from django.conf import settings
@@ -25,7 +25,8 @@ import abx
from archivebox.config import CONSTANTS
from archivebox.misc.system import get_dir_size
-from archivebox.misc.util import parse_date, base_url
+from archivebox.misc.util import parse_date, base_url, domain as url_domain
+from archivebox.misc.hashing import get_dir_info
from archivebox.index.schema import Link
from archivebox.index.html import snapshot_icons
from archivebox.extractors import ARCHIVE_METHODS_INDEXING_PRECEDENCE
@@ -142,8 +143,20 @@ def validate_timestamp(value):
assert value.replace('.', '').isdigit(), f'timestamp must be a float str, got: "{value}"'
class SnapshotManager(models.Manager):
+ def filter(self, *args, **kwargs):
+ """add support for .filter(domain='example.com') to Snapshot queryset"""
+ domain = kwargs.pop('domain', None)
+ qs = super().filter(*args, **kwargs)
+ if domain:
+ qs = qs.filter(url__icontains=f'://{domain}')
+ return qs
+
def get_queryset(self):
- return super().get_queryset().prefetch_related('tags', 'archiveresult_set') # .annotate(archiveresult_count=models.Count('archiveresult')).distinct()
+ return (
+ super().get_queryset()
+ .prefetch_related('tags', 'archiveresult_set')
+ # .annotate(archiveresult_count=models.Count('archiveresult')).distinct()
+ )
class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithStateMachine, ABIDModel):
abid_prefix = 'snp_'
@@ -256,6 +269,13 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithStateMachine, ABIDM
if self.crawl and self.url not in self.crawl.urls:
self.crawl.urls += f'\n{self.url}'
self.crawl.save()
+
+
+ def output_dir_parent(self) -> str:
+ return 'archive'
+
+ def output_dir_name(self) -> str:
+ return str(self.timestamp)
def archive(self, overwrite=False, methods=None):
result = bg_archive_snapshot(self, overwrite=overwrite, methods=methods)
@@ -338,6 +358,10 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithStateMachine, ABIDM
def bookmarked_date(self):
# TODO: remove this
return self.bookmarked
+
+ @cached_property
+ def domain(self) -> str:
+ return url_domain(self.url)
@cached_property
def is_archived(self):
@@ -659,7 +683,8 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
notes = models.TextField(blank=True, null=False, default='', help_text='Any extra notes this ArchiveResult should have')
# the network interface that was used to download this result
- # uplink = models.ForeignKey(NetworkInterface, on_delete=models.SET_NULL, null=True, blank=True, verbose_name='Network Interface Used')
+ # machine = models.ForeignKey(Machine, on_delete=models.SET_NULL, null=True, blank=True, verbose_name='Machine Used')
+ # network = models.ForeignKey(NetworkInterface, on_delete=models.SET_NULL, null=True, blank=True, verbose_name='Network Interface Used')
objects = ArchiveResultManager()
@@ -742,8 +767,7 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
return None
def legacy_output_path(self):
- link = self.snapshot.as_link()
- return link.canonical_outputs().get(f'{self.extractor}_path')
+ return self.canonical_outputs().get(f'{self.extractor}_path')
def output_exists(self) -> bool:
output_path = Path(self.snapshot_dir) / self.extractor
@@ -761,6 +785,89 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
for key in args
}
+ def canonical_outputs(self) -> Dict[str, Optional[str]]:
+ """Predict the expected output paths that should be present after archiving"""
+ # You'll need to implement the actual logic based on your requirements
+ # TODO: banish this awful duplication from the codebase and import these
+ # from their respective extractor files
+
+
+ from abx_plugin_favicon.config import FAVICON_CONFIG
+ canonical = {
+ 'index_path': 'index.html',
+ 'favicon_path': 'favicon.ico',
+ 'google_favicon_path': FAVICON_CONFIG.FAVICON_PROVIDER.format(self.domain),
+ 'wget_path': f'warc/{self.timestamp}',
+ 'warc_path': 'warc/',
+ 'singlefile_path': 'singlefile.html',
+ 'readability_path': 'readability/content.html',
+ 'mercury_path': 'mercury/content.html',
+ 'htmltotext_path': 'htmltotext.txt',
+ 'pdf_path': 'output.pdf',
+ 'screenshot_path': 'screenshot.png',
+ 'dom_path': 'output.html',
+ 'archive_org_path': f'https://web.archive.org/web/{self.base_url}',
+ 'git_path': 'git/',
+ 'media_path': 'media/',
+ 'headers_path': 'headers.json',
+ }
+
+ if self.is_static:
+ static_path = f'warc/{self.timestamp}'
+ canonical.update({
+ 'title': self.basename,
+ 'wget_path': static_path,
+ 'pdf_path': static_path,
+ 'screenshot_path': static_path,
+ 'dom_path': static_path,
+ 'singlefile_path': static_path,
+ 'readability_path': static_path,
+ 'mercury_path': static_path,
+ 'htmltotext_path': static_path,
+ })
+ return canonical
+
+ @property
+ def output_dir_name(self) -> str:
+ return self.extractor
+
+ @property
+ def output_dir_parent(self) -> str:
+ return str(self.snapshot.OUTPUT_DIR.relative_to(CONSTANTS.DATA_DIR))
+
+ @cached_property
+ def output_files(self) -> dict[str, dict]:
+ dir_info = get_dir_info(self.OUTPUT_DIR, max_depth=6)
+ with open(self.OUTPUT_DIR / '.hashes.json', 'w') as f:
+ json.dump(dir_info, f)
+ return dir_info
+
+ def announce_event(self, output_type: str, event: dict):
+ event = {
+ **event,
+ 'type': output_type,
+ }
+
+ # if event references a file, make sure it exists on disk
+ if 'path' in event:
+ file_path = Path(self.OUTPUT_DIR) / event['path']
+ assert file_path.exists(), f'ArchiveResult[{self.ABID}].announce_event(): File does not exist: {file_path} ({event})'
+
+ with open(self.OUTPUT_DIR / '.events.jsonl', 'a') as f:
+ f.write(json.dumps(event, sort_keys=True, default=str) + '\n')
+
+ def events(self, filter_type: str | None=None) -> list[dict]:
+ events = []
+ try:
+ with open(self.OUTPUT_DIR / '.events.jsonl', 'r') as f:
+ for line in f:
+ event = json.loads(line)
+ if filter_type is None or event['type'] == filter_type:
+ events.append(event)
+ except FileNotFoundError:
+ pass
+ return events
+
def write_indexes(self):
"""Write the ArchiveResult json, html, and merkle indexes to output dir, and pass searchable text to the search backend"""
super().write_indexes()
diff --git a/archivebox/extractors/extractor.py b/archivebox/extractors/extractor.py
index ea4e0b89..d43325e4 100644
--- a/archivebox/extractors/extractor.py
+++ b/archivebox/extractors/extractor.py
@@ -2,12 +2,15 @@ import hashlib
import mimetypes
import os
+import subprocess
from typing import ClassVar
from datetime import timedelta
from zipfile import Path
from django.utils import timezone
+from archivebox.misc.hashing import get_dir_info
+
from core.models import ArchiveResult
import abx
@@ -205,9 +208,7 @@ class Extractor:
def after_extract(self, error: Exception | None=None):
status, retry_at = self.determine_status()
-
- self.archiveresult.outputs = []
-
+
self.archiveresult.error = f'{type(error).__name__}: {error}' if error else None
self.archiveresult.status = self.archiveresult.StatusChoices.FAILED if error else self.archiveresult.StatusChoices.SUCCEEDED
@@ -216,4 +217,4 @@ class Extractor:
self.archiveresult.output = self.archiveresult.outputs[0].path
self.archiveresult.save()
self.archiveresult.write_indexes()
-
+
\ No newline at end of file
diff --git a/archivebox/filestore/models.py b/archivebox/filestore/models.py
index 2293c1d0..063c4421 100644
--- a/archivebox/filestore/models.py
+++ b/archivebox/filestore/models.py
@@ -1,67 +1,110 @@
-# import mimetypes
-# import uuid
+import mimetypes
+import uuid
+from datetime import timedelta
+from pathlib import Path
+from django.db import models
+from django.conf import settings
+from django.utils import timezone
-# from django.db import models
-# from django.conf import settings
-# from django.utils import timezone
-
-# from archivebox import DATA_DIR
-# from archivebox.misc.hashing import get_dir_info, hash_file
-# from base_models.abid import DEFAULT_ABID_URI_SALT
-# from base_models.models import ABIDModel, ABIDField, get_or_create_system_user_pk
+from archivebox import DATA_DIR
+from archivebox.misc.hashing import get_dir_info, hash_file
+from base_models.abid import DEFAULT_ABID_URI_SALT
+from base_models.models import ABIDModel, ABIDField, get_or_create_system_user_pk
-# class File(ABIDModel):
-# abid_prefix = 'fil_'
-# abid_ts_src = 'self.created_at'
-# abid_uri_src = 'self.path'
-# abid_subtype_src = 'self.mime_type'
-# abid_rand_src = 'self.id'
-# abid_salt: str = DEFAULT_ABID_URI_SALT # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
-# abid_drift_allowed: bool = False
+class File(ABIDModel):
+ abid_prefix = 'fil_'
+ abid_ts_src = 'self.created_at'
+ abid_uri_src = 'self.path'
+ abid_subtype_src = 'self.mime_type'
+ abid_rand_src = 'self.id'
+ abid_salt: str = DEFAULT_ABID_URI_SALT # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
+ abid_drift_allowed: bool = False
-# id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, null=False)
-# abid = ABIDField(prefix=abid_prefix)
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, null=False)
+ abid = ABIDField(prefix=abid_prefix)
-# created_at = models.DateTimeField(default=timezone.now, null=False)
-# modified_at = models.DateTimeField(default=timezone.now, null=False)
-# created_by = models.ForeignKey(settings.USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+ created_at = models.DateTimeField(default=timezone.now, null=False)
+ modified_at = models.DateTimeField(default=timezone.now, null=False)
+ created_by = models.ForeignKey(settings.USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-# path = models.FilePathField(path=str(DATA_DIR), recursive=True, allow_files=True, allow_folders=True, db_index=True, unique=True)
+ class StatusChoices(models.TextChoices):
+ UNLOCKED = 'unlocked'
+ LOCKED = 'locked'
-# basename = models.CharField(max_length=255, default=None, null=False) # e.g. 'index'
-# extension = models.CharField(max_length=63, default='', null=False) # e.g. 'html'
-# mime_type = models.CharField(max_length=63, default=None, null=False, db_index=True) # e.g. 'inode/directory' or 'text/html'
-# num_subpaths = models.IntegerField(default=None, null=False) # e.g. 3
-# num_bytes = models.IntegerField(default=None, null=False) # e.g. 123456
+ status = models.CharField(max_length=16, choices=StatusChoices.choices, default=StatusChoices.UNLOCKED, null=False)
+ retry_at = models.DateTimeField(default=None, null=True)
+ version = models.CharField(max_length=16, default='unknown', null=False)
-# hash_sha256 = models.CharField(max_length=64, default=None, null=False, db_index=True) # e.g. '5994471abb01112afcc1815994471abb01112afcc1815994471abb01112afcc181'
-# # hash_blake3 = models.CharField(max_length=64, default=None, null=False, db_index=True) # e.g. '5994471abb01112afcc1815994471abb01112afcc1815994471abb01112afcc181'
+ file = models.FileField(null=False)
-# DIR = 'inode/directory'
+ basename = models.CharField(max_length=255, default=None, null=False) # e.g. 'index'
+ extension = models.CharField(max_length=63, default='', null=False) # e.g. 'html'
+ mime_type = models.CharField(max_length=63, default=None, null=False, db_index=True) # e.g. 'inode/directory' or 'text/html'
+ num_subpaths = models.IntegerField(default=None, null=False) # e.g. 3
+ num_bytes = models.IntegerField(default=None, null=False) # e.g. 123456
+
+ sha256 = models.CharField(max_length=64, default=None, null=False, db_index=True) # e.g. '5994471abb01112afcc1815994471abb01112afcc1815994471abb01112afcc181'
+ # blake3 = models.CharField(max_length=64, default=None, null=False, db_index=True) # e.g. '5994471abb01112afcc1815994471abb01112afcc1815994471abb01112afcc181'
+
+ DIR = 'inode/directory'
+ @classmethod
+ def release_expired_locks(cls):
+ cls.objects.filter(status='locked', retry_at__lt=timezone.now()).update(status='unlocked', retry_at=None)
-# @property
-# def parent(self) -> 'File':
-# return File.objects.get(path=self.path.parent) or File(path=self.path.parent)
+ @property
+ def parent(self) -> 'File':
+ return File.objects.get(path=str(self.PATH.parent)) or File(path=str(self.PATH.parent))
+
+ @property
+ def relpath(self) -> Path:
+ return Path(self.file.name)
+
+ @property
+ def abspath(self) -> Path:
+ return DATA_DIR / self.file.name
-# def save(self, *args, **kwargs):
-# assert self.path.exists()
+ def save(self, *args, **kwargs):
+ assert self.abspath.exists()
-# if self.path.is_dir():
-# self.basename = self.path.name
-# self.extension = ''
-# self.mime_type = self.DIR
-# dir_info = get_dir_info(self.path)
-# self.num_subpaths = dir_info['.']['num_subpaths']
-# self.num_bytes = dir_info['.']['num_bytes']
-# self.hash_sha256 = dir_info['.']['hash_sha256']
-# # TODO: hash_blake3 = dir_info['.']['hash_blake3']
-# else:
-# self.basename = self.path.name
-# self.extension = self.path.suffix
-# self.mime_type = mimetypes.guess_type(self.path)[0]
-# self.num_bytes = self.path.stat().st_size
-# self.hash_sha256, self.hash_blake3 = hash_file(self.path)
-# super().save(*args, **kwargs)
+ if self.abspath.is_dir():
+ self.basename = self.relpath.name
+ self.extension = ''
+ self.mime_type = self.DIR
+ dir_info = get_dir_info(self.abspath)
+ self.num_subpaths = dir_info['.']['num_subpaths']
+ self.num_bytes = dir_info['.']['num_bytes']
+ self.hash_sha256 = dir_info['.']['hash_sha256']
+ # TODO: hash_blake3 = dir_info['.']['hash_blake3']
+ else:
+ self.basename = self.relpath.name
+ self.extension = self.relpath.suffix
+ self.mime_type = mimetypes.guess_type(self.abspath)[0]
+ self.num_bytes = self.abspath.stat().st_size
+ self.hash_sha256, self.hash_blake3 = hash_file(self.abspath)
+ super().save(*args, **kwargs)
+
+ def acquire_lock(self, timeout_seconds: int = 60):
+ self.status = 'locked'
+ self.retry_at = timezone.now() + timedelta(seconds=timeout_seconds)
+ self.save()
+
+ def release_lock(self):
+ self.status = 'unlocked'
+ self.retry_at = None
+ self.save()
+
+ def move_to(self, new_path: Path):
+ if str(new_path).startswith(str(DATA_DIR)):
+ new_relpath = new_path.relative_to(DATA_DIR)
+ new_abspath = new_path
+ else:
+ new_relpath = new_path
+ new_abspath = DATA_DIR / new_path
+
+ new_abspath.parent.mkdir(parents=True, exist_ok=True)
+ self.abspath.rename(new_abspath)
+ self.file.name = new_relpath
+ self.save()
diff --git a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/binaries.py b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/binaries.py
old mode 100644
new mode 100755
index 29cfc13a..93c8a3d7
--- a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/binaries.py
+++ b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/binaries.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python3
+
+__package__ = 'abx_plugin_chrome'
import os
import platform
from pathlib import Path
@@ -147,3 +150,7 @@ class ChromeBinary(Binary):
CHROME_BINARY = ChromeBinary()
+
+if __name__ == '__main__':
+ binary = CHROME_BINARY.load()
+ print(binary.version, ' ', binary.abspath)
diff --git a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/config.py b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/config.py
old mode 100644
new mode 100755
index 3efea3b3..89e05e5e
--- a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/config.py
+++ b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/config.py
@@ -1,5 +1,8 @@
+#!/usr/bin/env python3
+
import os
from pathlib import Path
+import sys
from typing import List, Optional
from pydantic import Field
@@ -79,16 +82,67 @@ class ChromeConfig(BaseConfigSet):
# Chrome Binary
CHROME_BINARY: str = Field(default='chrome')
CHROME_DEFAULT_ARGS: List[str] = Field(default=[
+ "--disable-sync",
+ "--no-pings",
"--no-first-run", # dont show any first run ui / setup prompts
- '--virtual-time-budget=15000', # accellerate any animations on the page by 15s into the future
- '--disable-features=DarkMode', # disable dark mode for archiving
- "--run-all-compositor-stages-before-draw", # dont draw partially rendered content, wait until everything is ready
- "--hide-scrollbars", # hide scrollbars to prevent layout shift / scrollbar visible in screenshots
- "--autoplay-policy=no-user-gesture-required", # allow media autoplay without user gesture (e.g. on mobile)
- "--use-fake-ui-for-media-stream", # provide fake camera if site tries to request camera access
+ "--no-default-browser-check",
+ "--disable-default-apps",
+ "--ash-no-nudges",
+ "--disable-infobars",
+ "--disable-blink-features=AutomationControlled",
+ "--js-flags=--random-seed=1157259159",
+ "--deterministic-mode",
+ "--deterministic-fetch",
+ "--start-maximized",
+ "--test-type=gpu",
+ "--disable-search-engine-choice-screen",
+ "--disable-session-crashed-bubble",
+ "--hide-crash-restore-bubble",
+ "--suppress-message-center-popups",
+ "--disable-client-side-phishing-detection",
+ "--disable-domain-reliability",
+ "--disable-component-update",
+ "--disable-datasaver-prompt",
+ "--disable-hang-monitor",
+ "--disable-session-crashed-bubble",
+ "--disable-speech-synthesis-api",
+ "--disable-speech-api",
+ "--disable-print-preview",
+ "--safebrowsing-disable-auto-update",
+ "--deny-permission-prompts",
+ "--disable-external-intent-requests",
+ "--disable-notifications",
+ "--disable-desktop-notifications",
+ "--noerrdialogs",
+ "--disable-popup-blocking",
+ "--disable-prompt-on-repost",
+ "--silent-debugger-extension-api",
+ "--block-new-web-contents",
+ "--metrics-recording-only",
+ "--disable-breakpad",
+ "--run-all-compositor-stages-before-draw",
"--use-fake-device-for-media-stream", # provide fake camera if site tries to request camera access
- "--simulate-outdated-no-au='Tue, 31 Dec 2099 23:59:59 GMT'", # ignore chrome updates
- "--force-gpu-mem-available-mb=4096", # allows for longer full page screenshots https://github.com/puppeteer/puppeteer/issues/5530
+ "--simulate-outdated-no-au=Tue, 31 Dec 2099 23:59:59 GMT", # ignore chrome updates
+ "--force-gpu-mem-available-mb=4096", # allows for longer full page screenshots https://github.com/puppeteer/puppeteer/issues/5530
+ "--password-store=basic",
+ "--use-mock-keychain",
+ "--disable-cookie-encryption",
+ "--allow-legacy-extension-manifests",
+ "--disable-gesture-requirement-for-media-playback",
+ "--font-render-hinting=none",
+ "--force-color-profile=srgb",
+ "--disable-partial-raster",
+ "--disable-skia-runtime-opts",
+ "--disable-2d-canvas-clip-aa",
+ "--disable-lazy-loading",
+ "--disable-renderer-backgrounding",
+ "--disable-background-networking",
+ "--disable-background-timer-throttling",
+ "--disable-backgrounding-occluded-windows",
+ "--disable-ipc-flooding-protection",
+ "--disable-extensions-http-throttling",
+ "--disable-field-trial-config",
+ "--disable-back-forward-cache",
])
CHROME_EXTRA_ARGS: List[str] = Field(default=[])
@@ -99,6 +153,7 @@ class ChromeConfig(BaseConfigSet):
CHROME_RESOLUTION: str = Field(default=lambda: ARCHIVING_CONFIG.RESOLUTION)
CHROME_CHECK_SSL_VALIDITY: bool = Field(default=lambda: ARCHIVING_CONFIG.CHECK_SSL_VALIDITY)
+
# Cookies & Auth
CHROME_USER_AGENT: str = Field(default=lambda: ARCHIVING_CONFIG.USER_AGENT)
CHROME_USER_DATA_DIR: Path | None = Field(default=CONSTANTS.PERSONAS_DIR / 'Default' / 'chrome_profile')
@@ -108,6 +163,8 @@ class ChromeConfig(BaseConfigSet):
SAVE_SCREENSHOT: bool = Field(default=True, alias='FETCH_SCREENSHOT')
SAVE_DOM: bool = Field(default=True, alias='FETCH_DOM')
SAVE_PDF: bool = Field(default=True, alias='FETCH_PDF')
+
+ OVERWRITE: bool = Field(default=lambda: ARCHIVING_CONFIG.OVERWRITE)
def validate(self):
from archivebox.config.paths import create_and_chown_dir
@@ -147,7 +204,11 @@ class ChromeConfig(BaseConfigSet):
self.update_in_place(CHROME_USER_DATA_DIR=None)
-
+ @property
+ def CHROME_ARGS(self) -> str:
+ # import shlex
+ # return '\n'.join(shlex.quote(arg) for arg in self.chrome_args())
+ return '\n'.join(self.chrome_args())
def chrome_args(self, **options) -> List[str]:
"""helper to build up a chrome shell command with arguments"""
@@ -157,8 +218,8 @@ class ChromeConfig(BaseConfigSet):
cmd_args = [*options.CHROME_DEFAULT_ARGS, *options.CHROME_EXTRA_ARGS]
- if options.CHROME_HEADLESS:
- cmd_args += ["--headless=new"] # expects chrome version >= 111
+ # if options.CHROME_HEADLESS:
+ # cmd_args += ["--headless"] # expects chrome version >= 111
if not options.CHROME_SANDBOX:
# assume this means we are running inside a docker container
@@ -205,3 +266,12 @@ class ChromeConfig(BaseConfigSet):
CHROME_CONFIG = ChromeConfig()
+
+if __name__ == '__main__':
+ if len(sys.argv) > 1:
+ result = getattr(CHROME_CONFIG, sys.argv[1], '')
+ if callable(result):
+ result = result()
+ print(result)
+ else:
+ print(CHROME_CONFIG.model_dump_json(indent=4))