From d680c48942317fdb901b88c25d4f8a44b72a25f8 Mon Sep 17 00:00:00 2001
From: Nick Sweeting <github@sweeting.me>
Date: Tue, 10 Sep 2024 00:19:32 -0700
Subject: [PATCH] avoid auto-starting all supervisord workers on startup
---
archivebox/plugantic/base_hook.py | 10 +++++++---
archivebox/plugantic/base_queue.py | 13 +++++++++----
2 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/archivebox/plugantic/base_hook.py b/archivebox/plugantic/base_hook.py
index 3495ff25..3622c3bb 100644
--- a/archivebox/plugantic/base_hook.py
+++ b/archivebox/plugantic/base_hook.py
@@ -77,12 +77,16 @@ class BaseHook(BaseModel):
"""e.g. builtin_plugins.singlefile.apps.SinglefileConfigSet"""
return f'{self.__module__}.{self.__class__.__name__}'
+ @property
+ def hook_file(self) -> Path:
+ """e.g. builtin_plugins.singlefile.apps.SinglefileConfigSet"""
+ return Path(inspect.getfile(self.__class__))
+
@property
def plugin_module(self) -> str:
"""e.g. builtin_plugins.singlefile"""
return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit(".apps.", 1)[0]
-
- @computed_field
+
@property
def plugin_dir(self) -> Path:
return Path(inspect.getfile(self.__class__)).parent.resolve()
@@ -97,7 +101,7 @@ class BaseHook(BaseModel):
# record installed hook in settings.HOOKS
settings.HOOKS[self.id] = self
-
+
if settings.HOOKS[self.id].is_registered:
raise Exception(f"Tried to run {self.hook_module}.register() but its already been called!")
diff --git a/archivebox/plugantic/base_queue.py b/archivebox/plugantic/base_queue.py
index 39593297..a3409b6e 100644
--- a/archivebox/plugantic/base_queue.py
+++ b/archivebox/plugantic/base_queue.py
@@ -77,6 +77,11 @@ class BaseQueue(BaseHook):
return None
print()
worker = start_worker(supervisor, self.get_supervisor_config(settings), lazy=lazy)
+
+ # Update settings.WORKERS to include this worker
+ settings.WORKERS = getattr(settings, "WORKERS", None) or AttrDict({})
+ settings.WORKERS[self.id] = self.start_supervisord_worker(settings, lazy=True)
+
return worker
def register(self, settings, parent_plugin=None):
@@ -90,10 +95,6 @@ class BaseQueue(BaseHook):
# on_startup(queue=self.name)(self.on_startup_task)
# db_periodic_task(crontab(minute='*/5'))(self.on_periodic_task)
- # Side effect: start consumer worker process under supervisord
- settings.WORKERS = getattr(settings, "WORKERS", None) or AttrDict({})
- settings.WORKERS[self.id] = self.start_supervisord_worker(settings, lazy=True)
-
# Install queue into settings.QUEUES
settings.QUEUES = getattr(settings, "QUEUES", None) or AttrDict({})
settings.QUEUES[self.id] = self
@@ -101,6 +102,10 @@ class BaseQueue(BaseHook):
# Record installed hook into settings.HOOKS
super().register(settings, parent_plugin=parent_plugin)
+ # def ready(self, settings):
+ # self.start_supervisord_worker(settings, lazy=True)
+ # super().ready(settings)
+
# class WgetToggleConfig(ConfigSet):
# section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'