diff --git a/archivebox/abx/archivebox/base_binary.py b/archivebox/abx/archivebox/base_binary.py
index afa4f192..7890c05b 100644
--- a/archivebox/abx/archivebox/base_binary.py
+++ b/archivebox/abx/archivebox/base_binary.py
@@ -14,11 +14,8 @@ from pydantic_pkgr import (
EnvProvider,
)
-from archivebox.config import CONSTANTS
from archivebox.config.permissions import ARCHIVEBOX_USER
-import abx
-
class BaseBinProvider(BinProvider):
@@ -29,10 +26,6 @@ class BaseBinProvider(BinProvider):
# e.g. /admin/environment/binproviders/NpmBinProvider/ TODO
return "/admin/environment/binaries/"
- @abx.hookimpl
- def get_BINPROVIDERS(self):
- return [self]
-
class BaseBinary(Binary):
@staticmethod
@@ -96,10 +89,6 @@ class BaseBinary(Binary):
# e.g. /admin/environment/config/LdapConfig/
return f"/admin/environment/binaries/{self.name}/"
- @abx.hookimpl
- def get_BINARIES(self):
- return [self]
-
class AptBinProvider(AptProvider, BaseBinProvider):
name: BinProviderName = "apt"
diff --git a/archivebox/config/__init__.py b/archivebox/config/__init__.py
index a3e96681..1a3f8a7b 100644
--- a/archivebox/config/__init__.py
+++ b/archivebox/config/__init__.py
@@ -36,3 +36,7 @@ def get_CONFIG():
'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
}
+@abx.hookimpl
+def ready():
+ for config in get_CONFIG().values():
+ config.validate()
diff --git a/archivebox/config/common.py b/archivebox/config/common.py
index 2deccb0d..15f575f4 100644
--- a/archivebox/config/common.py
+++ b/archivebox/config/common.py
@@ -7,13 +7,13 @@ from typing import Dict, Optional, List
from pathlib import Path
from rich import print
-from pydantic import Field, field_validator, computed_field
+from pydantic import Field, field_validator
from django.utils.crypto import get_random_string
from abx.archivebox.base_configset import BaseConfigSet
from .constants import CONSTANTS
-from .version import get_COMMIT_HASH, get_BUILD_TIME
+from .version import get_COMMIT_HASH, get_BUILD_TIME, VERSION
from .permissions import IN_DOCKER
###################### Config ##########################
@@ -31,25 +31,22 @@ class ShellConfig(BaseConfigSet):
ANSI: Dict[str, str] = Field(default=lambda c: CONSTANTS.DEFAULT_CLI_COLORS if c.USE_COLOR else CONSTANTS.DISABLED_CLI_COLORS)
- VERSIONS_AVAILABLE: bool = False # .check_for_update.get_versions_available_on_github(c)},
- CAN_UPGRADE: bool = False # .check_for_update.can_upgrade(c)},
-
- @computed_field
@property
def TERM_WIDTH(self) -> int:
if not self.IS_TTY:
return 200
return shutil.get_terminal_size((140, 10)).columns
- @computed_field
@property
def COMMIT_HASH(self) -> Optional[str]:
return get_COMMIT_HASH()
- @computed_field
@property
def BUILD_TIME(self) -> str:
return get_BUILD_TIME()
+
+ # def VERSIONS_AVAILABLE() -> bool # .check_for_update.get_versions_available_on_github(c)},
+ # def CAN_UPGRADE() -> bool # .check_for_update.can_upgrade(c)},
SHELL_CONFIG = ShellConfig()
@@ -79,7 +76,6 @@ STORAGE_CONFIG = StorageConfig()
class GeneralConfig(BaseConfigSet):
TAG_SEPARATOR_PATTERN: str = Field(default=r'[,]')
-
GENERAL_CONFIG = GeneralConfig()
@@ -90,6 +86,7 @@ class ServerConfig(BaseConfigSet):
CSRF_TRUSTED_ORIGINS: str = Field(default=lambda c: 'http://localhost:8000,http://127.0.0.1:8000,http://0.0.0.0:8000,http://{}'.format(c.BIND_ADDR))
SNAPSHOTS_PER_PAGE: int = Field(default=40)
+ PREVIEW_ORIGINALS: bool = Field(default=True)
FOOTER_INFO: str = Field(default='Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.')
# CUSTOM_TEMPLATES_DIR: Path = Field(default=None) # this is now a constant
@@ -99,10 +96,10 @@ class ServerConfig(BaseConfigSet):
ADMIN_USERNAME: str = Field(default=None)
ADMIN_PASSWORD: str = Field(default=None)
+
REVERSE_PROXY_USER_HEADER: str = Field(default='Remote-User')
REVERSE_PROXY_WHITELIST: str = Field(default='')
LOGOUT_REDIRECT_URL: str = Field(default='/')
- PREVIEW_ORIGINALS: bool = Field(default=True)
SERVER_CONFIG = ServerConfig()
@@ -116,7 +113,7 @@ class ArchivingConfig(BaseConfigSet):
MEDIA_MAX_SIZE: str = Field(default='750m')
RESOLUTION: str = Field(default='1440,2000')
CHECK_SSL_VALIDITY: bool = Field(default=True)
- USER_AGENT: str = Field(default='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)')
+ USER_AGENT: str = Field(default=f'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)')
COOKIES_FILE: Path | None = Field(default=None)
URL_DENYLIST: str = Field(default=r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$', alias='URL_BLACKLIST')
@@ -134,17 +131,15 @@ class ArchivingConfig(BaseConfigSet):
# CHROME_HEADLESS: bool = Field(default=True)
# CHROME_SANDBOX: bool = Field(default=lambda: not SHELL_CONFIG.IN_DOCKER)
- @field_validator('TIMEOUT', mode='after')
- def validate_timeout(cls, v):
- if int(v) < 5:
- print(f'[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={v} seconds)[/red]', file=sys.stderr)
+ def validate(self):
+ if int(self.TIMEOUT) < 5:
+ print(f'[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={self.TIMEOUT} seconds)[/red]', file=sys.stderr)
print(' You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.', file=sys.stderr)
print(' (Setting it to somewhere between 30 and 3000 seconds is recommended)', file=sys.stderr)
print(file=sys.stderr)
print(' If you want to make ArchiveBox run faster, disable specific archive methods instead:', file=sys.stderr)
print(' https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles', file=sys.stderr)
print(file=sys.stderr)
- return v
@field_validator('CHECK_SSL_VALIDITY', mode='after')
def validate_check_ssl_validity(cls, v):
@@ -167,15 +162,17 @@ class ArchivingConfig(BaseConfigSet):
@property
def SAVE_ALLOWLIST_PTNS(self) -> Dict[re.Pattern, List[str]]:
return {
- re.compile(k, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): v
- for k, v in self.SAVE_ALLOWLIST.items()
+ # regexp: methods list
+ re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
+ for key, val in self.SAVE_ALLOWLIST.items()
} if self.SAVE_ALLOWLIST else {}
@property
def SAVE_DENYLIST_PTNS(self) -> Dict[re.Pattern, List[str]]:
return {
- re.compile(k, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): v
- for k, v in self.SAVE_DENYLIST.items()
+ # regexp: methods list
+ re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
+ for key, val in self.SAVE_DENYLIST.items()
} if self.SAVE_DENYLIST else {}
ARCHIVING_CONFIG = ArchivingConfig()
diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py
index 2bd08421..84558632 100644
--- a/archivebox/core/admin_snapshots.py
+++ b/archivebox/core/admin_snapshots.py
@@ -13,7 +13,7 @@ from django.template import Template, RequestContext
from django.contrib.admin.helpers import ActionForm
from django.contrib.admin.widgets import FilteredSelectMultiple
-from archivebox.config import DATA_DIR, VERSION
+from archivebox.config import DATA_DIR
from archivebox.config.common import SERVER_CONFIG
from archivebox.misc.util import htmldecode, urldecode
from archivebox.misc.paginators import AccelleratedPaginator
@@ -32,8 +32,8 @@ from core.admin_tags import TagInline
from core.admin_archiveresults import ArchiveResultInline, result_url
-GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
-
+# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
+GLOBAL_CONTEXT = {}
class SnapshotActionForm(ActionForm):
diff --git a/archivebox/index/json.py b/archivebox/index/json.py
index 017dbc94..8671369a 100644
--- a/archivebox/index/json.py
+++ b/archivebox/index/json.py
@@ -33,7 +33,7 @@ def generate_json_index_from_links(links: List[Link], with_headers: bool):
'docs': 'https://github.com/ArchiveBox/ArchiveBox/wiki',
'source': 'https://github.com/ArchiveBox/ArchiveBox',
'issues': 'https://github.com/ArchiveBox/ArchiveBox/issues',
- 'dependencies': settings.BINARIES.to_dict(),
+ 'dependencies': settings.BINARIES,
},
}
diff --git a/archivebox/main.py b/archivebox/main.py
index 7f196a3c..a3db809f 100755
--- a/archivebox/main.py
+++ b/archivebox/main.py
@@ -1075,11 +1075,11 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
package_manager_names = ', '.join(
f'[yellow]{binprovider.name}[/yellow]'
- for binprovider in reversed(list(settings.BINPROVIDERS.values()))
+ for binprovider in list(settings.BINPROVIDERS.values())
if not binproviders or (binproviders and binprovider.name in binproviders)
)
print(f'[+] Setting up package managers {package_manager_names}...')
- for binprovider in reversed(list(settings.BINPROVIDERS.values())):
+ for binprovider in list(settings.BINPROVIDERS.values()):
if binproviders and binprovider.name not in binproviders:
continue
try:
@@ -1092,7 +1092,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
print()
- for binary in reversed(list(settings.BINARIES.values())):
+ for binary in list(settings.BINARIES.values()):
if binary.name in ('archivebox', 'django', 'sqlite', 'python'):
# obviously must already be installed if we are running
continue
diff --git a/archivebox/plugins_extractor/chrome/config.py b/archivebox/plugins_extractor/chrome/config.py
index be62f360..a28c530f 100644
--- a/archivebox/plugins_extractor/chrome/config.py
+++ b/archivebox/plugins_extractor/chrome/config.py
@@ -189,6 +189,10 @@ class ChromeConfig(BaseConfigSet):
# cmd_args += ('--timeout={}'.format(options.CHROME_TIMEOUT * 1000),)
if options.CHROME_USER_DATA_DIR:
+ # remove SingletonLock file
+ lockfile = options.CHROME_USER_DATA_DIR / options.CHROME_PROFILE_NAME / 'SingletonLock'
+ lockfile.unlink(missing_ok=True)
+
cmd_args.append('--user-data-dir={}'.format(options.CHROME_USER_DATA_DIR))
cmd_args.append('--profile-directory={}'.format(options.CHROME_PROFILE_NAME or 'Default'))
diff --git a/archivebox/templates/admin/base.html b/archivebox/templates/admin/base.html
index 8c97e10f..192cc323 100644
--- a/archivebox/templates/admin/base.html
+++ b/archivebox/templates/admin/base.html
@@ -148,6 +148,7 @@
{% block footer %}<div id="footer"></div>{% endblock %}
</div>
+ {% comment %}
{% if user.is_authenticated and user.is_superuser and CAN_UPGRADE %}
<script>
if (!localStorage.getItem("bannerDismissed")) {
@@ -184,6 +185,7 @@
}
</script>
{% endif %}
+ {% endcomment %}
<script>
$ = django.jQuery;