mirrors/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2025-05-13 06:34:25 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

change supervisord to always start non-daemonized by default
Some checks are pending
CodeQL / Analyze (python) (push) Waiting to run
Details
Build Debian package / build (push) Waiting to run
Details
Build Docker image / buildx (push) Waiting to run
Details
Build Homebrew package / build (push) Waiting to run
Details
Build GitHub Pages website / build (push) Waiting to run
Details
Build GitHub Pages website / deploy (push) Blocked by required conditions
Details
Run linters / lint (push) Waiting to run
Details
Build Pip package / build (push) Waiting to run
Details
Run tests / python_tests (ubuntu-22.04, 3.11) (push) Waiting to run
Details
Run tests / docker_tests (push) Waiting to run
Details

Browse source
This commit is contained in:
Nick Sweeting 2024-09-24 22:22:03 -07:00
parent de2ba890ea
commit 5e4b78d9e0
No known key found for this signature in database
4 changed files with 55 additions and 88 deletions
Download patch file Download diff file Expand all files Collapse all files

56
archivebox/config.py
View file

@ -1098,34 +1098,34 @@ def get_data_locations(config: ConfigDict) -> ConfigValue:
def get_dependency_info(config: ConfigDict) -> ConfigValue: def get_dependency_info(config: ConfigDict) -> ConfigValue:
return { return {
'PYTHON_BINARY': { # 'PYTHON_BINARY': {
'path': bin_path(config['PYTHON_BINARY']), # 'path': bin_path(config['PYTHON_BINARY']),
'version': config['PYTHON_VERSION'], # 'version': config['PYTHON_VERSION'],
'hash': bin_hash(config['PYTHON_BINARY']), # 'hash': bin_hash(config['PYTHON_BINARY']),
'enabled': True, # 'enabled': True,
'is_valid': bool(config['PYTHON_VERSION']), # 'is_valid': bool(config['PYTHON_VERSION']),
}, # },
'SQLITE_BINARY': { # 'SQLITE_BINARY': {
'path': bin_path(config['SQLITE_BINARY']), # 'path': bin_path(config['SQLITE_BINARY']),
'version': config['SQLITE_VERSION'], # 'version': config['SQLITE_VERSION'],
'hash': bin_hash(config['SQLITE_BINARY']), # 'hash': bin_hash(config['SQLITE_BINARY']),
'enabled': True, # 'enabled': True,
'is_valid': bool(config['SQLITE_VERSION']), # 'is_valid': bool(config['SQLITE_VERSION']),
}, # },
'DJANGO_BINARY': { # 'DJANGO_BINARY': {
'path': bin_path(config['DJANGO_BINARY']), # 'path': bin_path(config['DJANGO_BINARY']),
'version': config['DJANGO_VERSION'], # 'version': config['DJANGO_VERSION'],
'hash': bin_hash(config['DJANGO_BINARY']), # 'hash': bin_hash(config['DJANGO_BINARY']),
'enabled': True, # 'enabled': True,
'is_valid': bool(config['DJANGO_VERSION']), # 'is_valid': bool(config['DJANGO_VERSION']),
}, # },
'ARCHIVEBOX_BINARY': { # 'ARCHIVEBOX_BINARY': {
'path': bin_path(config['ARCHIVEBOX_BINARY']), # 'path': bin_path(config['ARCHIVEBOX_BINARY']),
'version': config['VERSION'], # 'version': config['VERSION'],
'hash': bin_hash(config['ARCHIVEBOX_BINARY']), # 'hash': bin_hash(config['ARCHIVEBOX_BINARY']),
'enabled': True, # 'enabled': True,
'is_valid': True, # 'is_valid': True,
}, # },
'CURL_BINARY': { 'CURL_BINARY': {
'path': bin_path(config['CURL_BINARY']), 'path': bin_path(config['CURL_BINARY']),

44
archivebox/main.py
View file

@ -1372,51 +1372,11 @@ def server(runserver_args: Optional[List[str]]=None,
print(f' > Starting ArchiveBox webserver on http://{host}:{port}/') print(f' > Starting ArchiveBox webserver on http://{host}:{port}/')
from queues.supervisor_util import get_or_create_supervisord_process, start_worker, stop_worker, watch_worker from queues.supervisor_util import start_server_workers
print() print()
supervisor = get_or_create_supervisord_process(daemonize=False) start_server_workers(host=host, port=port)
bg_workers = [
{
"name": "worker_system_tasks",
"command": "archivebox manage djangohuey --queue system_tasks",
"autostart": "true",
"autorestart": "true",
"stdout_logfile": "logs/worker_system_tasks.log",
"redirect_stderr": "true",
},
]
fg_worker = {
"name": "worker_daphne",
"command": f"daphne --bind={host} --port={port} --application-close-timeout=600 archivebox.core.asgi:application",
"autostart": "false",
"autorestart": "true",
"stdout_logfile": "logs/worker_daphne.log",
"redirect_stderr": "true",
}
print()
for worker in bg_workers:
start_worker(supervisor, worker)
print()
start_worker(supervisor, fg_worker)
print()
try:
watch_worker(supervisor, "worker_daphne")
except KeyboardInterrupt:
print("\n[🛑] Got Ctrl+C, stopping gracefully...")
except SystemExit:
pass
except BaseException as e:
print(f"\n[🛑] Got {e.__class__.__name__} exception, stopping web server gracefully...")
raise
finally:
stop_worker(supervisor, "worker_daphne")
time.sleep(0.5)
print("\n[🟩] ArchiveBox server shut down gracefully.") print("\n[🟩] ArchiveBox server shut down gracefully.")

35
archivebox/queues/supervisor_util.py
View file

@ -57,8 +57,7 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
files = %(here)s/{WORKER_DIR.name}/*.conf files = %(here)s/{WORKER_DIR.name}/*.conf
""" """
with open(CONFIG_FILE, "w") as f: CONFIG_FILE.write_text(config_content)
f.write(config_content)
def create_worker_config(daemon): def create_worker_config(daemon):
Path.mkdir(WORKER_DIR, exist_ok=True) Path.mkdir(WORKER_DIR, exist_ok=True)
@ -111,7 +110,7 @@ def stop_existing_supervisord_process():
except FileNotFoundError: except FileNotFoundError:
pass pass
def start_new_supervisord_process(daemonize=True): def start_new_supervisord_process(daemonize=False):
print(f"[🦸‍♂️] Supervisord starting{' in background' if daemonize else ''}...") print(f"[🦸‍♂️] Supervisord starting{' in background' if daemonize else ''}...")
# Create a config file in the current working directory # Create a config file in the current working directory
create_supervisord_config() create_supervisord_config()
@ -142,7 +141,7 @@ def start_new_supervisord_process(daemonize=True):
return get_existing_supervisord_process() return get_existing_supervisord_process()
def get_or_create_supervisord_process(daemonize=True): def get_or_create_supervisord_process(daemonize=False):
supervisor = get_existing_supervisord_process() supervisor = get_existing_supervisord_process()
if supervisor is None: if supervisor is None:
stop_existing_supervisord_process() stop_existing_supervisord_process()
@ -213,7 +212,7 @@ def watch_worker(supervisor, daemon_name, interval=5):
continue continue
def tail_worker_logs(log_path: str): def tail_worker_logs(log_path: str):
get_or_create_supervisord_process(daemonize=True) get_or_create_supervisord_process(daemonize=False)
from rich.live import Live from rich.live import Live
from rich.table import Table from rich.table import Table
@ -271,9 +270,17 @@ def start_server_workers(host='0.0.0.0', port='8000'):
supervisor = get_or_create_supervisord_process(daemonize=False) supervisor = get_or_create_supervisord_process(daemonize=False)
bg_workers = [ bg_workers = [
{
"name": "worker_scheduler",
"command": "archivebox manage djangohuey --queue system_tasks -w 4 -k thread --disable-health-check --flush-locks",
"autostart": "true",
"autorestart": "true",
"stdout_logfile": "logs/worker_scheduler.log",
"redirect_stderr": "true",
},
{ {
"name": "worker_system_tasks", "name": "worker_system_tasks",
"command": "archivebox manage djangohuey --queue system_tasks", "command": "archivebox manage djangohuey --queue system_tasks -w 4 -k thread --no-periodic --disable-health-check",
"autostart": "true", "autostart": "true",
"autorestart": "true", "autorestart": "true",
"stdout_logfile": "logs/worker_system_tasks.log", "stdout_logfile": "logs/worker_system_tasks.log",
@ -289,12 +296,11 @@ def start_server_workers(host='0.0.0.0', port='8000'):
"redirect_stderr": "true", "redirect_stderr": "true",
} }
print()
start_worker(supervisor, fg_worker)
print() print()
for worker in bg_workers: for worker in bg_workers:
start_worker(supervisor, worker) start_worker(supervisor, worker)
print()
start_worker(supervisor, fg_worker)
print() print()
try: try:
@ -337,17 +343,18 @@ def start_cli_workers(watch=False):
raise raise
finally: finally:
stop_worker(supervisor, "worker_system_tasks") stop_worker(supervisor, "worker_system_tasks")
stop_worker(supervisor, "worker_scheduler")
time.sleep(0.5) time.sleep(0.5)
return fg_worker return fg_worker
def main(daemons): # def main(daemons):
supervisor = get_or_create_supervisord_process(daemonize=True) # supervisor = get_or_create_supervisord_process(daemonize=False)
worker = start_worker(supervisor, daemons["webworker"]) # worker = start_worker(supervisor, daemons["webworker"])
pprint(worker) # pprint(worker)
print("All processes started in background.") # print("All processes started in background.")
# Optionally you can block the main thread until an exit signal is received: # Optionally you can block the main thread until an exit signal is received:
# try: # try:

8
archivebox/queues/tasks.py
View file

@ -10,7 +10,7 @@ from .supervisor_util import get_or_create_supervisord_process
@db_task(queue="system_tasks", context=True) @db_task(queue="system_tasks", context=True)
def bg_add(add_kwargs, task=None, parent_task_id=None): def bg_add(add_kwargs, task=None, parent_task_id=None):
get_or_create_supervisord_process(daemonize=True) get_or_create_supervisord_process(daemonize=False)
from ..main import add from ..main import add
@ -29,7 +29,7 @@ def bg_add(add_kwargs, task=None, parent_task_id=None):
@task(queue="system_tasks", context=True) @task(queue="system_tasks", context=True)
def bg_archive_links(args, kwargs=None, task=None, parent_task_id=None): def bg_archive_links(args, kwargs=None, task=None, parent_task_id=None):
get_or_create_supervisord_process(daemonize=True) get_or_create_supervisord_process(daemonize=False)
from ..extractors import archive_links from ..extractors import archive_links
@ -50,7 +50,7 @@ def bg_archive_links(args, kwargs=None, task=None, parent_task_id=None):
@task(queue="system_tasks", context=True) @task(queue="system_tasks", context=True)
def bg_archive_link(args, kwargs=None,task=None, parent_task_id=None): def bg_archive_link(args, kwargs=None,task=None, parent_task_id=None):
get_or_create_supervisord_process(daemonize=True) get_or_create_supervisord_process(daemonize=False)
from ..extractors import archive_link from ..extractors import archive_link
@ -71,7 +71,7 @@ def bg_archive_link(args, kwargs=None,task=None, parent_task_id=None):
@task(queue="system_tasks", context=True) @task(queue="system_tasks", context=True)
def bg_archive_snapshot(snapshot, overwrite=False, methods=None, task=None, parent_task_id=None): def bg_archive_snapshot(snapshot, overwrite=False, methods=None, task=None, parent_task_id=None):
# get_or_create_supervisord_process(daemonize=True) # get_or_create_supervisord_process(daemonize=False)
from ..extractors import archive_link from ..extractors import archive_link