mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-05-31 23:08:25 -04:00
finish migrating almost all config to new system
This commit is contained in:
Nick Sweeting
parent
4b6a2a3e50
commit
d21bc86075
25 changed files with 246 additions and 349 deletions
|
|
@ -36,7 +36,7 @@ from main import remove
|
|||
from extractors import archive_links
|
||||
|
||||
|
||||
CONFIG = settings.CONFIG
|
||||
CONFIG = settings.FLAT_CONFIG
|
||||
|
||||
GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
__package__ = 'archivebox.core'
|
||||
|
||||
|
||||
from ..config.legacy import (
|
||||
LDAP
|
||||
)
|
||||
from archivebox.plugins_auth.ldap.apps import LDAP_CONFIG
|
||||
|
||||
def register_signals():
|
||||
|
||||
if LDAP:
|
||||
if LDAP_CONFIG.LDAP_ENABLED:
|
||||
import django_auth_ldap.backend
|
||||
from .auth_ldap import create_user
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
from ..config.legacy import (
|
||||
LDAP_CREATE_SUPERUSER
|
||||
)
|
||||
from archivebox.plugins_auth.ldap.apps import LDAP_CONFIG
|
||||
|
||||
def create_user(sender, user=None, ldap_user=None, **kwargs):
|
||||
if not user.id and LDAP_CREATE_SUPERUSER:
|
||||
if not user.id and LDAP_CONFIG.LDAP_CREATE_SUPERUSER:
|
||||
user.is_superuser = True
|
||||
|
||||
user.is_staff = True
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from django.utils import timezone
|
|||
from django.contrib.auth.middleware import RemoteUserMiddleware
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
|
||||
from ..config.legacy import PUBLIC_SNAPSHOTS, REVERSE_PROXY_USER_HEADER, REVERSE_PROXY_WHITELIST
|
||||
from archivebox.config import SERVER_CONFIG
|
||||
|
||||
|
||||
def detect_timezone(request, activate: bool=True):
|
||||
|
|
@ -32,7 +32,7 @@ def CacheControlMiddleware(get_response):
|
|||
response = get_response(request)
|
||||
|
||||
if '/archive/' in request.path or '/static/' in request.path:
|
||||
policy = 'public' if PUBLIC_SNAPSHOTS else 'private'
|
||||
policy = 'public' if SERVER_CONFIG.PUBLIC_SNAPSHOTS else 'private'
|
||||
response['Cache-Control'] = f'{policy}, max-age=60, stale-while-revalidate=300'
|
||||
# print('Set Cache-Control header to', response['Cache-Control'])
|
||||
return response
|
||||
|
|
@ -40,15 +40,15 @@ def CacheControlMiddleware(get_response):
|
|||
return middleware
|
||||
|
||||
class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
|
||||
header = 'HTTP_{normalized}'.format(normalized=REVERSE_PROXY_USER_HEADER.replace('-', '_').upper())
|
||||
header = 'HTTP_{normalized}'.format(normalized=SERVER_CONFIG.REVERSE_PROXY_USER_HEADER.replace('-', '_').upper())
|
||||
|
||||
def process_request(self, request):
|
||||
if REVERSE_PROXY_WHITELIST == '':
|
||||
if SERVER_CONFIG.REVERSE_PROXY_WHITELIST == '':
|
||||
return
|
||||
|
||||
ip = request.META.get('REMOTE_ADDR')
|
||||
|
||||
for cidr in REVERSE_PROXY_WHITELIST.split(','):
|
||||
for cidr in SERVER_CONFIG.REVERSE_PROXY_WHITELIST.split(','):
|
||||
try:
|
||||
network = ipaddress.ip_network(cidr)
|
||||
except ValueError:
|
||||
|
|
|
|||
|
|
@ -13,9 +13,7 @@ import abx.archivebox
|
|||
import abx.archivebox.use
|
||||
import abx.django.use
|
||||
|
||||
from archivebox.config import VERSION, DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS # noqa
|
||||
|
||||
from ..config.legacy import CONFIG
|
||||
from archivebox.config import VERSION, DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS, SHELL_CONFIG, SERVER_CONFIG # noqa
|
||||
|
||||
IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
|
||||
IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
|
||||
|
|
@ -80,7 +78,7 @@ LOGOUT_REDIRECT_URL = os.environ.get('LOGOUT_REDIRECT_URL', '/')
|
|||
PASSWORD_RESET_URL = '/accounts/password_reset/'
|
||||
APPEND_SLASH = True
|
||||
|
||||
DEBUG = CONFIG.DEBUG or ('--debug' in sys.argv)
|
||||
DEBUG = SHELL_CONFIG.DEBUG or ('--debug' in sys.argv)
|
||||
|
||||
|
||||
INSTALLED_APPS = [
|
||||
|
|
@ -364,10 +362,10 @@ STORAGES = {
|
|||
### Security Settings
|
||||
################################################################################
|
||||
|
||||
SECRET_KEY = CONFIG.SECRET_KEY or get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789_')
|
||||
SECRET_KEY = SERVER_CONFIG.SECRET_KEY or get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789_')
|
||||
|
||||
ALLOWED_HOSTS = CONFIG.ALLOWED_HOSTS.split(',')
|
||||
CSRF_TRUSTED_ORIGINS = list(set(CONFIG.CSRF_TRUSTED_ORIGINS.split(',')))
|
||||
ALLOWED_HOSTS = SERVER_CONFIG.ALLOWED_HOSTS.split(',')
|
||||
CSRF_TRUSTED_ORIGINS = list(set(SERVER_CONFIG.CSRF_TRUSTED_ORIGINS.split(',')))
|
||||
|
||||
# automatically fix case when user sets ALLOWED_HOSTS (e.g. to archivebox.example.com)
|
||||
# but forgets to add https://archivebox.example.com to CSRF_TRUSTED_ORIGINS
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from .views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthC
|
|||
from .serve_static import serve_static
|
||||
|
||||
# GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
|
||||
# from .config.legacy import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
|
||||
# from archivebox.config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
|
||||
# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
__package__ = 'archivebox.core'
|
||||
|
||||
from typing import Callable
|
||||
from benedict import benedict
|
||||
import inspect
|
||||
from typing import Callable, get_type_hints
|
||||
from pathlib import Path
|
||||
|
||||
from django.shortcuts import render, redirect
|
||||
|
|
@ -27,21 +27,13 @@ from core.admin import result_url
|
|||
|
||||
from queues.tasks import bg_add
|
||||
|
||||
from archivebox.config import CONSTANTS, DATA_DIR, VERSION, SHELL_CONFIG, SERVER_CONFIG
|
||||
from ..plugins_extractor.archivedotorg.apps import ARCHIVEDOTORG_CONFIG
|
||||
|
||||
from ..config.legacy import (
|
||||
CONFIG_SCHEMA,
|
||||
DYNAMIC_CONFIG_SCHEMA,
|
||||
USER_CONFIG,
|
||||
CONFIG,
|
||||
)
|
||||
from ..logging_util import printable_filesize
|
||||
from archivebox.config import CONSTANTS_CONFIG, DATA_DIR, VERSION, SHELL_CONFIG, SERVER_CONFIG
|
||||
from archivebox.misc.util import base_url, htmlencode, ts_to_date_str
|
||||
from ..search import query_search_index
|
||||
from .serve_static import serve_static_with_byterange_support
|
||||
|
||||
CONFIG = benedict({**CONSTANTS, **CONFIG, **settings.FLAT_CONFIG})
|
||||
from .serve_static import serve_static_with_byterange_support
|
||||
from ..plugins_extractor.archivedotorg.apps import ARCHIVEDOTORG_CONFIG
|
||||
from ..logging_util import printable_filesize
|
||||
from ..search import query_search_index
|
||||
|
||||
|
||||
class HomepageView(View):
|
||||
|
|
@ -502,27 +494,43 @@ class HealthCheckView(View):
|
|||
|
||||
|
||||
def find_config_section(key: str) -> str:
|
||||
if key in CONSTANTS:
|
||||
if key in CONSTANTS_CONFIG:
|
||||
return 'CONSTANT'
|
||||
matching_sections = [
|
||||
name for name, opts in CONFIG_SCHEMA.items() if key in opts
|
||||
section.id for section in settings.CONFIGS.values() if key in section.model_fields
|
||||
]
|
||||
section = matching_sections[0] if matching_sections else 'DYNAMIC'
|
||||
return section
|
||||
|
||||
def find_config_default(key: str) -> str:
|
||||
default_val = USER_CONFIG.get(key, {}).get('default', lambda: None)
|
||||
if key in CONSTANTS_CONFIG:
|
||||
return str(CONSTANTS_CONFIG[key])
|
||||
|
||||
default_val = None
|
||||
|
||||
for config in settings.CONFIGS.values():
|
||||
if key in config.model_fields:
|
||||
default_val = config.model_fields[key].default
|
||||
break
|
||||
|
||||
if isinstance(default_val, Callable):
|
||||
return None
|
||||
default_val = inspect.getsource(default_val).split('lambda', 1)[-1].split(':', 1)[-1].replace('\n', ' ').strip()
|
||||
if default_val.count(')') > default_val.count('('):
|
||||
default_val = default_val[:-1]
|
||||
else:
|
||||
default_val = repr(default_val)
|
||||
default_val = str(default_val)
|
||||
|
||||
|
||||
return default_val
|
||||
|
||||
def find_config_type(key: str) -> str:
|
||||
if key in USER_CONFIG:
|
||||
return str(USER_CONFIG[key]['type'])
|
||||
elif key in DYNAMIC_CONFIG_SCHEMA:
|
||||
return str(type(CONFIG[key]))
|
||||
for config in settings.CONFIGS.values():
|
||||
if hasattr(config, key):
|
||||
type_hints = get_type_hints(config)
|
||||
try:
|
||||
return str(type_hints[key].__name__)
|
||||
except AttributeError:
|
||||
return str(type_hints[key])
|
||||
return 'str'
|
||||
|
||||
def key_is_safe(key: str) -> bool:
|
||||
|
|
@ -543,40 +551,29 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
|
|||
"Value": [],
|
||||
"Default": [],
|
||||
# "Documentation": [],
|
||||
"Aliases": [],
|
||||
# "Aliases": [],
|
||||
}
|
||||
|
||||
for section in CONFIG_SCHEMA.keys():
|
||||
for key in CONFIG_SCHEMA[section].keys():
|
||||
rows['Section'].append(section) # section.replace('_', ' ').title().replace(' Config', '')
|
||||
for section in reversed(list(settings.CONFIGS.values())):
|
||||
for key, field in section.model_fields.items():
|
||||
rows['Section'].append(section.id) # section.replace('_', ' ').title().replace(' Config', '')
|
||||
rows['Key'].append(ItemLink(key, key=key))
|
||||
rows['Type'].append(mark_safe(f'<code>{find_config_type(key)}</code>'))
|
||||
rows['Value'].append(mark_safe(f'<code>{CONFIG[key]}</code>') if key_is_safe(key) else '******** (redacted)')
|
||||
rows['Default'].append(mark_safe(f'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig.py+{key}&type=code"><code style="text-decoration: underline">{find_config_default(key) or "See here..."}</code></a>'))
|
||||
rows['Type'].append(format_html('<code>{}</code>', find_config_type(key)))
|
||||
rows['Value'].append(mark_safe(f'<code>{getattr(section, key)}</code>') if key_is_safe(key) else '******** (redacted)')
|
||||
rows['Default'].append(mark_safe(f'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{key}&type=code"><code style="text-decoration: underline">{find_config_default(key) or "See here..."}</code></a>'))
|
||||
# rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
|
||||
rows['Aliases'].append(', '.join(CONFIG_SCHEMA[section][key].get('aliases', [])))
|
||||
|
||||
section = 'DYNAMIC'
|
||||
for key in DYNAMIC_CONFIG_SCHEMA.keys():
|
||||
if key in CONSTANTS:
|
||||
continue
|
||||
rows['Section'].append(section) # section.replace('_', ' ').title().replace(' Config', '')
|
||||
rows['Key'].append(ItemLink(key, key=key))
|
||||
rows['Type'].append(mark_safe(f'<code>{find_config_type(key)}</code>'))
|
||||
rows['Value'].append(mark_safe(f'<code>{CONFIG[key]}</code>') if key_is_safe(key) else '******** (redacted)')
|
||||
rows['Default'].append(mark_safe(f'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig.py+{key}&type=code"><code style="text-decoration: underline">{find_config_default(key) or "See here..."}</code></a>'))
|
||||
# rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
|
||||
rows['Aliases'].append(ItemLink(key, key=key) if key in USER_CONFIG else '')
|
||||
# rows['Aliases'].append(', '.join(find_config_aliases(key)))
|
||||
|
||||
|
||||
section = 'CONSTANT'
|
||||
for key in CONSTANTS.keys():
|
||||
for key in CONSTANTS_CONFIG.keys():
|
||||
rows['Section'].append(section) # section.replace('_', ' ').title().replace(' Config', '')
|
||||
rows['Key'].append(ItemLink(key, key=key))
|
||||
rows['Type'].append(mark_safe(f'<code>{find_config_type(key)}</code>'))
|
||||
rows['Value'].append(mark_safe(f'<code>{CONFIG[key]}</code>') if key_is_safe(key) else '******** (redacted)')
|
||||
rows['Default'].append(mark_safe(f'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig.py+{key}&type=code"><code style="text-decoration: underline">{find_config_default(key) or "See here..."}</code></a>'))
|
||||
rows['Type'].append(format_html('<code>{}</code>', getattr(type(CONSTANTS_CONFIG[key]), '__name__', repr(CONSTANTS_CONFIG[key]))))
|
||||
rows['Value'].append(format_html('<code>{}</code>', CONSTANTS_CONFIG[key]) if key_is_safe(key) else '******** (redacted)')
|
||||
rows['Default'].append(mark_safe(f'<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{key}&type=code"><code style="text-decoration: underline">{find_config_default(key) or "See here..."}</code></a>'))
|
||||
# rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
|
||||
rows['Aliases'].append(ItemLink(key, key=key) if key in USER_CONFIG else '')
|
||||
# rows['Aliases'].append('')
|
||||
|
||||
|
||||
return TableContext(
|
||||
|
|
@ -589,11 +586,12 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
|
|||
|
||||
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
|
||||
|
||||
aliases = USER_CONFIG.get(key, {}).get("aliases", [])
|
||||
# aliases = USER_CONFIG.get(key, {}).get("aliases", [])
|
||||
aliases = []
|
||||
|
||||
if key in CONSTANTS:
|
||||
if key in CONSTANTS_CONFIG:
|
||||
section_header = mark_safe(f'[CONSTANTS] <b><code style="color: lightgray">{key}</code></b> <small>(read-only, hardcoded by ArchiveBox)</small>')
|
||||
elif key in USER_CONFIG:
|
||||
elif key in settings.FLAT_CONFIG:
|
||||
section_header = mark_safe(f'data / ArchiveBox.conf [{find_config_section(key)}] <b><code style="color: lightgray">{key}</code></b>')
|
||||
else:
|
||||
section_header = mark_safe(f'[DYNAMIC CONFIG] <b><code style="color: lightgray">{key}</code></b> <small>(read-only, calculated at runtime)</small>')
|
||||
|
|
@ -609,7 +607,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
|
|||
"fields": {
|
||||
'Key': key,
|
||||
'Type': find_config_type(key),
|
||||
'Value': CONFIG[key] if key_is_safe(key) else '********',
|
||||
'Value': settings.FLAT_CONFIG[key] if key_is_safe(key) else '********',
|
||||
},
|
||||
"help_texts": {
|
||||
'Key': mark_safe(f'''
|
||||
|
|
@ -619,25 +617,25 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
|
|||
</span>
|
||||
'''),
|
||||
'Type': mark_safe(f'''
|
||||
<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig.py+{key}&type=code">
|
||||
See full definition in <code>archivebox/config.py</code>...
|
||||
<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{key}&type=code">
|
||||
See full definition in <code>archivebox/config</code>...
|
||||
</a>
|
||||
'''),
|
||||
'Value': mark_safe(f'''
|
||||
{'<b style="color: red">Value is redacted for your security. (Passwords, secrets, API tokens, etc. cannot be viewed in the Web UI)</b><br/><br/>' if not key_is_safe(key) else ''}
|
||||
<br/><hr/><br/>
|
||||
Default:
|
||||
<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig.py+{key}&type=code">
|
||||
<a href="https://github.com/search?q=repo%3AArchiveBox%2FArchiveBox+path%3Aconfig+{key}&type=code">
|
||||
<code>{find_config_default(key) or '↗️ See in ArchiveBox source code...'}</code>
|
||||
</a>
|
||||
<br/><br/>
|
||||
<p style="display: {"block" if key in USER_CONFIG else "none"}">
|
||||
<p style="display: {"block" if key in settings.FLAT_CONFIG else "none"}">
|
||||
<i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
|
||||
<br/><br/>
|
||||
<code>archivebox config --set {key}="{
|
||||
val.strip("'")
|
||||
if (val := find_config_default(key)) else
|
||||
(repr(CONFIG[key] if key_is_safe(key) else '********')).strip("'")
|
||||
(repr(settings.FLAT_CONFIG[key] if key_is_safe(key) else '********')).strip("'")
|
||||
}"</code>
|
||||
</p>
|
||||
'''),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue