From 917af4b1d7106d494c7d529b9d87b827a9c704b9 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 26 Aug 2024 20:14:47 -0700 Subject: [PATCH] add pydantic logfire and improve settings for debugging --- archivebox/config.py | 12 +++ archivebox/core/settings.py | 172 ++++++++++++++++++++++-------------- 2 files changed, 120 insertions(+), 64 deletions(-) diff --git a/archivebox/config.py b/archivebox/config.py index eb6611fa..8fcc5352 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -1525,5 +1525,17 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG, assert sql_index_path.exists(), ( f'No database file {SQL_INDEX_FILENAME} found in: {config["OUTPUT_DIR"]} (Are you in an ArchiveBox collection directory?)') + + # https://docs.pydantic.dev/logfire/integrations/django/ Logfire Debugging + if settings.DEBUG_LOGFIRE: + from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor + SQLite3Instrumentor().instrument() + + import logfire + + logfire.configure() + logfire.instrument_django(is_sql_commentor_enabled=True) + logfire.info(f'Started ArchiveBox v{CONFIG.VERSION}', argv=sys.argv) + except KeyboardInterrupt: raise SystemExit(2) diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 983b135f..9131f367 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -4,6 +4,7 @@ import os import sys import re import logging +import inspect import tempfile from typing import Any, Dict @@ -143,64 +144,6 @@ if CONFIG.LDAP: # sys.exit(1) -################################################################################ -### Debug Settings -################################################################################ - -# only enable debug toolbar when in DEBUG mode with --nothreading (it doesnt work in multithreaded mode) -DEBUG_TOOLBAR = DEBUG and ('--nothreading' in sys.argv) and ('--reload' not in sys.argv) -if DEBUG_TOOLBAR: - try: - import debug_toolbar # noqa - DEBUG_TOOLBAR = True - except ImportError: - DEBUG_TOOLBAR = False - -if DEBUG_TOOLBAR: - INSTALLED_APPS = [*INSTALLED_APPS, 'debug_toolbar'] - INTERNAL_IPS = ['0.0.0.0', '127.0.0.1', '*'] - DEBUG_TOOLBAR_CONFIG = { - "SHOW_TOOLBAR_CALLBACK": lambda request: True, - "RENDER_PANELS": True, - } - DEBUG_TOOLBAR_PANELS = [ - 'debug_toolbar.panels.history.HistoryPanel', - 'debug_toolbar.panels.versions.VersionsPanel', - 'debug_toolbar.panels.timer.TimerPanel', - 'debug_toolbar.panels.settings.SettingsPanel', - 'debug_toolbar.panels.headers.HeadersPanel', - 'debug_toolbar.panels.request.RequestPanel', - 'debug_toolbar.panels.sql.SQLPanel', - 'debug_toolbar.panels.staticfiles.StaticFilesPanel', - # 'debug_toolbar.panels.templates.TemplatesPanel', - 'debug_toolbar.panels.cache.CachePanel', - 'debug_toolbar.panels.signals.SignalsPanel', - 'debug_toolbar.panels.logging.LoggingPanel', - 'debug_toolbar.panels.redirects.RedirectsPanel', - 'debug_toolbar.panels.profiling.ProfilingPanel', - 'djdt_flamegraph.FlamegraphPanel', - ] - MIDDLEWARE = [*MIDDLEWARE, 'debug_toolbar.middleware.DebugToolbarMiddleware'] - -if DEBUG: - from django_autotyping.typing import AutotypingSettingsDict - - INSTALLED_APPS += ['django_autotyping'] - AUTOTYPING: AutotypingSettingsDict = { - "STUBS_GENERATION": { - "LOCAL_STUBS_DIR": Path(CONFIG.PACKAGE_DIR) / "typings", - } - } - -# https://github.com/bensi94/Django-Requests-Tracker (improved version of django-debug-toolbar) -# Must delete archivebox/templates/admin to use because it relies on some things we override -# visit /__requests_tracker__/ to access -DEBUG_REQUESTS_TRACKER = False -if DEBUG_REQUESTS_TRACKER: - INSTALLED_APPS += ["requests_tracker"] - MIDDLEWARE += ["requests_tracker.middleware.requests_tracker_middleware"] - INTERNAL_IPS = ["127.0.0.1", "10.0.2.2", "0.0.0.0", "*"] - ################################################################################ ### Staticfile and Template Settings @@ -346,6 +289,8 @@ AUTH_PASSWORD_VALIDATORS = [ {'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'}, ] +DATA_UPLOAD_MAX_NUMBER_FIELDS = None + ################################################################################ ### Shell Settings ################################################################################ @@ -386,6 +331,10 @@ IGNORABLE_404_URLS = [ re.compile(r'robots\.txt$'), re.compile(r'.*\.(css|js)\.map$'), ] +IGNORABLE_200_URLS = [ + re.compile(r'^"GET /static/.* HTTP/.*" (200|30.) .+', re.I | re.M), + re.compile(r'^"GET /admin/jsi18n/ HTTP/.*" (200|30.) .+', re.I | re.M), +] class NoisyRequestsFilter(logging.Filter): def filter(self, record) -> bool: @@ -397,19 +346,26 @@ class NoisyRequestsFilter(logging.Filter): if ignorable_log_pattern.match(logline): return False - # ignore staticfile requests that 200 or 30* - ignoreable_200_log_pattern = re.compile(r'"GET /static/.* HTTP/.*" (200|30.) .+', re.I | re.M) - if ignoreable_200_log_pattern.match(logline): - return False + ignorable_log_pattern = re.compile(f'^Not Found: /.*/?{ignorable_url_pattern.pattern}', re.I | re.M) + if ignorable_log_pattern.match(logline): + return False + # ignore staticfile requests that 200 or 30* + for ignorable_url_pattern in IGNORABLE_200_URLS: + if ignorable_log_pattern.match(logline): + return False + return True + +ERROR_LOG = tempfile.NamedTemporaryFile().name + if CONFIG.LOGS_DIR.exists(): ERROR_LOG = (CONFIG.LOGS_DIR / 'errors.log') else: # historically too many edge cases here around creating log dir w/ correct permissions early on # if there's an issue on startup, we trash the log and let user figure it out via stdout/stderr - ERROR_LOG = tempfile.NamedTemporaryFile().name + print(f'[!] WARNING: data/logs dir does not exist. Logging to temp file: {ERROR_LOG}') LOGGING = { 'version': 1, @@ -446,6 +402,10 @@ LOGGING = { } +################################################################################ +### REST API Outbound Webhooks settings +################################################################################ + # Add default webhook configuration to the User model SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook' SIGNAL_WEBHOOKS = { @@ -459,7 +419,9 @@ SIGNAL_WEBHOOKS = { }, } -DATA_UPLOAD_MAX_NUMBER_FIELDS = None +################################################################################ +### Admin Data View Settings +################################################################################ ADMIN_DATA_VIEWS = { "NAME": "Environment", @@ -496,3 +458,85 @@ ADMIN_DATA_VIEWS = { }, ], } + + +################################################################################ +### Debug Settings +################################################################################ + +# only enable debug toolbar when in DEBUG mode with --nothreading (it doesnt work in multithreaded mode) +DEBUG_TOOLBAR = True +DEBUG_TOOLBAR = DEBUG_TOOLBAR and DEBUG and ('--nothreading' in sys.argv) and ('--reload' not in sys.argv) +if DEBUG_TOOLBAR: + try: + import debug_toolbar # noqa + DEBUG_TOOLBAR = True + except ImportError: + DEBUG_TOOLBAR = False + +if DEBUG_TOOLBAR: + INSTALLED_APPS = [*INSTALLED_APPS, 'debug_toolbar'] + INTERNAL_IPS = ['0.0.0.0', '127.0.0.1', '*'] + DEBUG_TOOLBAR_CONFIG = { + "SHOW_TOOLBAR_CALLBACK": lambda request: True, + "RENDER_PANELS": True, + } + DEBUG_TOOLBAR_PANELS = [ + 'debug_toolbar.panels.history.HistoryPanel', + 'debug_toolbar.panels.versions.VersionsPanel', + 'debug_toolbar.panels.timer.TimerPanel', + 'debug_toolbar.panels.settings.SettingsPanel', + 'debug_toolbar.panels.headers.HeadersPanel', + 'debug_toolbar.panels.request.RequestPanel', + 'debug_toolbar.panels.sql.SQLPanel', + 'debug_toolbar.panels.staticfiles.StaticFilesPanel', + # 'debug_toolbar.panels.templates.TemplatesPanel', + 'debug_toolbar.panels.cache.CachePanel', + 'debug_toolbar.panels.signals.SignalsPanel', + 'debug_toolbar.panels.logging.LoggingPanel', + 'debug_toolbar.panels.redirects.RedirectsPanel', + 'debug_toolbar.panels.profiling.ProfilingPanel', + 'djdt_flamegraph.FlamegraphPanel', + ] + MIDDLEWARE = [*MIDDLEWARE, 'debug_toolbar.middleware.DebugToolbarMiddleware'] + +if DEBUG: + from django_autotyping.typing import AutotypingSettingsDict + + INSTALLED_APPS += ['django_autotyping'] + AUTOTYPING: AutotypingSettingsDict = { + "STUBS_GENERATION": { + "LOCAL_STUBS_DIR": Path(CONFIG.PACKAGE_DIR) / "typings", + } + } + +# https://github.com/bensi94/Django-Requests-Tracker (improved version of django-debug-toolbar) +# Must delete archivebox/templates/admin to use because it relies on some things we override +# visit /__requests_tracker__/ to access +DEBUG_REQUESTS_TRACKER = True +if DEBUG_REQUESTS_TRACKER: + import requests_tracker + + INSTALLED_APPS += ["requests_tracker"] + MIDDLEWARE += ["requests_tracker.middleware.requests_tracker_middleware"] + INTERNAL_IPS = ["127.0.0.1", "10.0.2.2", "0.0.0.0", "*"] + + TEMPLATE_DIRS.insert(0, str(Path(inspect.getfile(requests_tracker)).parent / "templates")) + + REQUESTS_TRACKER_CONFIG = { + "TRACK_SQL": True, + "ENABLE_STACKTRACES": False, + "IGNORE_PATHS_PATTERNS": ( + r".*/favicon\.ico", + r".*\.png", + r"/admin/jsi18n/", + ), + "IGNORE_SQL_PATTERNS": ( + r"^SELECT .* FROM django_migrations WHERE app = 'requests_tracker'", + r"^SELECT .* FROM django_migrations WHERE app = 'auth'", + ), + } + +# https://docs.pydantic.dev/logfire/integrations/django/ (similar to DataDog / NewRelic / etc.) +DEBUG_LOGFIRE = False +DEBUG_LOGFIRE = DEBUG_LOGFIRE and (Path(CONFIG.OUTPUT_DIR) / '.logfire').is_dir()