diff --git a/archivebox/config/constants.py b/archivebox/config/constants.py index f324259d..e56c8456 100644 --- a/archivebox/config/constants.py +++ b/archivebox/config/constants.py @@ -18,6 +18,7 @@ PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent # archivebox sourc DATA_DIR: Path = Path(os.curdir).resolve() # archivebox user data dir ARCHIVE_DIR: Path = DATA_DIR / 'archive' # archivebox snapshot data dir +IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'yes') def _detect_installed_version(PACKAGE_DIR: Path): """Autodetect the installed archivebox version by using pip package metadata, pyproject.toml file, or package.json file""" @@ -79,8 +80,8 @@ class ConstantsDict(Mapping): PERSONAS_DIR: Path = DATA_DIR / PERSONAS_DIR_NAME CACHE_DIR: Path = DATA_DIR / CACHE_DIR_NAME LOGS_DIR: Path = DATA_DIR / LOGS_DIR_NAME - LIB_DIR: Path = DATA_DIR / LIB_DIR_NAME - TMP_DIR: Path = DATA_DIR / TMP_DIR_NAME + LIB_DIR: Path = (Path('/tmp') if IN_DOCKER else DATA_DIR) / LIB_DIR_NAME + TMP_DIR: Path = (Path('/tmp') if IN_DOCKER else DATA_DIR) / TMP_DIR_NAME CUSTOM_TEMPLATES_DIR: Path = DATA_DIR / CUSTOM_TEMPLATES_DIR_NAME USER_PLUGINS_DIR: Path = DATA_DIR / USER_PLUGINS_DIR_NAME diff --git a/archivebox/config/legacy.py b/archivebox/config/legacy.py index 08c9d798..af9a9c8b 100644 --- a/archivebox/config/legacy.py +++ b/archivebox/config/legacy.py @@ -756,7 +756,9 @@ def setup_django(out_dir: Path | None=None, check_db=False, config: benedict=CON global DJANGO_SET_UP if DJANGO_SET_UP: - raise Exception('django is already set up!') + # raise Exception('django is already set up!') + # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes + return with Progress(transient=True, expand=True, console=CONSOLE) as INITIAL_STARTUP_PROGRESS: INITIAL_STARTUP_PROGRESS_TASK = INITIAL_STARTUP_PROGRESS.add_task("[green]Loading modules...", total=25) diff --git a/archivebox/queues/settings.py b/archivebox/queues/settings.py index 8634cf93..d9566d27 100644 --- a/archivebox/queues/settings.py +++ b/archivebox/queues/settings.py @@ -1,13 +1,14 @@ from pathlib import Path -from archivebox.config import DATA_DIR, CONSTANTS +from archivebox.config import CONSTANTS +DATA_DIR = CONSTANTS.DATA_DIR LOGS_DIR = CONSTANTS.LOGS_DIR TMP_DIR = CONSTANTS.TMP_DIR Path.mkdir(TMP_DIR, exist_ok=True) -CONFIG_FILE = TMP_DIR / "supervisord.conf" +SUPERVISORD_CONFIG_FILE = TMP_DIR / "supervisord.conf" PID_FILE = TMP_DIR / "supervisord.pid" SOCK_FILE = TMP_DIR / "supervisord.sock" LOG_FILE = TMP_DIR / "supervisord.log" -WORKER_DIR = TMP_DIR / "workers" +WORKERS_DIR = TMP_DIR / "workers" diff --git a/archivebox/queues/supervisor_util.py b/archivebox/queues/supervisor_util.py index a109828e..4e3d749b 100644 --- a/archivebox/queues/supervisor_util.py +++ b/archivebox/queues/supervisor_util.py @@ -12,7 +12,7 @@ from typing import Dict, cast from supervisor.xmlrpc import SupervisorTransport from xmlrpc.client import ServerProxy -from .settings import CONFIG_FILE, PID_FILE, SOCK_FILE, LOG_FILE, WORKER_DIR, TMP_DIR, LOGS_DIR +from .settings import SUPERVISORD_CONFIG_FILE, DATA_DIR, PID_FILE, SOCK_FILE, LOG_FILE, WORKERS_DIR, TMP_DIR, LOGS_DIR from typing import Iterator @@ -36,38 +36,39 @@ def create_supervisord_config(): [supervisord] nodaemon = true environment = IS_SUPERVISORD_PARENT="true" -pidfile = %(here)s/{PID_FILE.name} -logfile = %(here)s/../{LOGS_DIR.name}/{LOG_FILE.name} -childlogdir = %(here)s/../{LOGS_DIR.name} -directory = %(here)s/.. +pidfile = {TMP_DIR}/{PID_FILE.name} +logfile = {LOGS_DIR}/{LOG_FILE.name} +childlogdir = {LOGS_DIR} +directory = {DATA_DIR} strip_ansi = true nocleanup = true [unix_http_server] -file = %(here)s/{SOCK_FILE.name} +file = {TMP_DIR}/{SOCK_FILE.name} chmod = 0700 [supervisorctl] -serverurl = unix://%(here)s/{SOCK_FILE.name} +serverurl = unix://{TMP_DIR}/{SOCK_FILE.name} [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface [include] -files = %(here)s/{WORKER_DIR.name}/*.conf +files = {WORKERS_DIR}/*.conf """ - CONFIG_FILE.write_text(config_content) + SUPERVISORD_CONFIG_FILE.write_text(config_content) def create_worker_config(daemon): - Path.mkdir(WORKER_DIR, exist_ok=True) + Path.mkdir(WORKERS_DIR, exist_ok=True) name = daemon['name'] - configfile = WORKER_DIR / f"{name}.conf" + configfile = WORKERS_DIR / f"{name}.conf" config_content = f"[program:{name}]\n" for key, value in daemon.items(): - if key == 'name': continue + if key == 'name': + continue config_content += f"{key}={value}\n" config_content += "\n" @@ -117,7 +118,7 @@ def start_new_supervisord_process(daemonize=False): # Start supervisord subprocess.Popen( - f"supervisord --configuration={CONFIG_FILE}", + f"supervisord --configuration={SUPERVISORD_CONFIG_FILE}", stdin=None, shell=True, start_new_session=daemonize, @@ -146,8 +147,11 @@ def get_or_create_supervisord_process(daemonize=False): if supervisor is None: stop_existing_supervisord_process() supervisor = start_new_supervisord_process(daemonize=daemonize) + time.sleep(0.5) - assert supervisor and supervisor.getPID(), "Failed to start supervisord or connect to it!" + assert supervisor, "Failed to start supervisord or connect to it!" + supervisor.getPID() # make sure it doesn't throw an exception + return supervisor def start_worker(supervisor, daemon, lazy=False):