mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2025-05-12 22:25:44 -04:00
c8e186f21b
Some checks failed
Run linters / lint (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Build Debian package / build (push) Has been cancelled
Build Docker image / buildx (push) Has been cancelled
Deploy static content to Pages / deploy (push) Has been cancelled
Build Homebrew package / build (push) Has been cancelled
Build GitHub Pages website / build (push) Has been cancelled
Build Pip package / build (push) Has been cancelled
Run tests / python_tests (ubuntu-22.04, 3.11) (push) Has been cancelled
Run tests / docker_tests (push) Has been cancelled
Build GitHub Pages website / deploy (push) Has been cancelled
65 lines
2.1 KiB
Python
65 lines
2.1 KiB
Python
__package__ = 'archivebox.crawls'
|
|
|
|
from django.utils import timezone
|
|
|
|
from statemachine import State, StateMachine
|
|
|
|
from crawls.models import Crawl
|
|
|
|
# State Machine Definitions
|
|
#################################################
|
|
|
|
|
|
class CrawlMachine(StateMachine, strict_states=True):
|
|
"""State machine for managing Crawl lifecycle."""
|
|
|
|
model: Crawl
|
|
|
|
# States
|
|
queued = State(value=Crawl.StatusChoices.QUEUED, initial=True)
|
|
started = State(value=Crawl.StatusChoices.STARTED)
|
|
sealed = State(value=Crawl.StatusChoices.SEALED, final=True)
|
|
|
|
# Tick Event
|
|
tick = (
|
|
queued.to.itself(unless='can_start') |
|
|
queued.to(started, cond='can_start') |
|
|
started.to.itself(unless='is_finished') |
|
|
started.to(sealed, cond='is_finished')
|
|
)
|
|
|
|
def __init__(self, crawl, *args, **kwargs):
|
|
self.crawl = crawl
|
|
super().__init__(crawl, *args, **kwargs)
|
|
|
|
def can_start(self) -> bool:
|
|
return bool(self.crawl.seed and self.crawl.seed.uri and (self.retry_at < timezone.now()))
|
|
|
|
def is_finished(self) -> bool:
|
|
if not self.crawl.snapshot_set.exists():
|
|
return False
|
|
if self.crawl.pending_snapshots().exists():
|
|
return False
|
|
if self.crawl.pending_archiveresults().exists():
|
|
return False
|
|
return True
|
|
|
|
# def before_transition(self, event, state):
|
|
# print(f"Before '{event}', on the '{state.id}' state.")
|
|
# return "before_transition_return"
|
|
|
|
@started.enter
|
|
def enter_started(self):
|
|
print(f'CrawlMachine[{self.crawl.ABID}].on_started(): crawl.create_root_snapshot() + crawl.bump_retry_at(+10s)')
|
|
self.crawl.status = Crawl.StatusChoices.STARTED
|
|
self.crawl.bump_retry_at(seconds=10)
|
|
self.crawl.save()
|
|
self.crawl.create_root_snapshot()
|
|
|
|
@sealed.enter
|
|
def enter_sealed(self):
|
|
print(f'CrawlMachine[{self.crawl.ABID}].on_sealed(): crawl.retry_at=None')
|
|
self.crawl.status = Crawl.StatusChoices.SEALED
|
|
self.crawl.retry_at = None
|
|
self.crawl.save()
|
|
|