From bd5dd2f9490df8a9c92c025830697b70eeb4826b Mon Sep 17 00:00:00 2001
From: Nick Sweeting <git@sweeting.me>
Date: Thu, 12 Dec 2024 21:34:48 -0800
Subject: [PATCH] clearer core models separation of concerns using new
basemodels
---
archivebox/core/models.py | 691 +++++++++++++++++++++++++++++++-------
1 file changed, 575 insertions(+), 116 deletions(-)
diff --git a/archivebox/core/models.py b/archivebox/core/models.py
index fe984bb2..d3a3d63a 100644
--- a/archivebox/core/models.py
+++ b/archivebox/core/models.py
@@ -11,6 +11,7 @@ from pathlib import Path
from django.db import models
from django.db.models import QuerySet
+from django.core.validators import MinValueValidator, MaxValueValidator
from django.utils.functional import cached_property
from django.utils.text import slugify
from django.utils import timezone
@@ -30,18 +31,23 @@ from archivebox.misc.hashing import get_dir_info
from archivebox.index.schema import Link
from archivebox.index.html import snapshot_icons
from archivebox.extractors import ARCHIVE_METHODS_INDEXING_PRECEDENCE
-from archivebox.base_models.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithOutputDir, ModelWithConfig
-
+from archivebox.base_models.models import (
+ ABIDModel, ABIDField, AutoDateTimeField, get_or_create_system_user_pk,
+ ModelWithReadOnlyFields, ModelWithSerializers, ModelWithUUID, ModelWithKVTags # ModelWithStateMachine
+ ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats
+)
from workers.models import ModelWithStateMachine
from workers.tasks import bg_archive_snapshot
-from crawls.models import Crawl
+from tags.models import KVTag
# from machine.models import Machine, NetworkInterface
-class Tag(ABIDModel):
+class Tag(ModelWithReadOnlyFields, ModelWithSerializers, ModelWithUUID, ABIDModel):
"""
- Loosely based on django-taggit model + ABID base.
+ Old tag model, loosely based on django-taggit model + ABID base.
+
+ Being phazed out in favor of archivebox.tags.models.ATag
"""
abid_prefix = 'tag_'
abid_ts_src = 'self.created_at'
@@ -49,11 +55,13 @@ class Tag(ABIDModel):
abid_subtype_src = '"03"'
abid_rand_src = 'self.id'
abid_drift_allowed = True
+
+ read_only_fields = ('id', 'abid', 'created_at', 'created_by', 'slug')
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
abid = ABIDField(prefix=abid_prefix)
- created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='tag_set')
+ created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False, related_name='tag_set')
created_at = AutoDateTimeField(default=None, null=False, db_index=True)
modified_at = models.DateTimeField(auto_now=True)
@@ -125,15 +133,400 @@ class SnapshotTag(models.Model):
unique_together = [('snapshot', 'tag')]
-# class CrawlTag(models.Model):
-# id = models.AutoField(primary_key=True)
+class Seed(ModelWithReadOnlyFields, ModelWithSerializers, ModelWithUUID, ModelWithKVTags, ABIDModel, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats):
+ """
+ A fountain that produces URLs (+metadata) each time it's queried e.g.
+ - file:///data/sources/2024-01-02_11-57-51__cli_add.txt
+ - file:///data/sources/2024-01-02_11-57-51__web_ui_add.txt
+ - file:///Users/squash/Library/Application Support/Google/Chrome/Default/Bookmarks
+ - https://getpocket.com/user/nikisweeting/feed
+ - https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml
+ - ...
+ Each query of a Seed can produce the same list of URLs, or a different list each time.
+ The list of URLs it returns is used to create a new Crawl and seed it with new pending Snapshots.
+
+ When a crawl is created, a root_snapshot is initially created with a URI set to the Seed URI.
+ The seed's preferred extractor is executed on that URI, which produces an ArchiveResult containing outlinks.
+ The outlinks then get turned into new pending Snapshots under the same crawl,
+ and the cycle repeats until Crawl.max_depth.
-# crawl = models.ForeignKey('Crawl', db_column='crawl_id', on_delete=models.CASCADE, to_field='id')
-# tag = models.ForeignKey(Tag, db_column='tag_id', on_delete=models.CASCADE, to_field='id')
+ Each consumption of a Seed by an Extractor can produce new urls, as Seeds can point to
+ stateful remote services, files with contents that change, directories that have new files within, etc.
+ """
+
+ ### ModelWithReadOnlyFields:
+ read_only_fields = ('id', 'abid', 'created_at', 'created_by', 'uri')
+
+ ### Immutable fields
+ id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+ abid = ABIDField(prefix=abid_prefix)
+ created_at = AutoDateTimeField(default=None, null=False, db_index=True) # unique source location where URLs will be loaded from
+ created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
+
+ ### Mutable fields:
+ extractor = models.CharField(default='auto', max_length=32, help_text='The parser / extractor to use to load URLs from this source (default: auto)')
+ tags_str = models.CharField(max_length=255, null=False, blank=True, default='', help_text='An optional comma-separated list of tags to attach to any URLs that come from this source')
+ label = models.CharField(max_length=255, null=False, blank=True, default='', help_text='A human-readable label for this seed')
+ modified_at = models.DateTimeField(auto_now=True)
-# class Meta:
-# db_table = 'core_crawl_tags'
-# unique_together = [('crawl', 'tag')]
+ ### ModelWithConfig:
+ config = models.JSONField(default=dict, help_text='An optional JSON object containing extra config to put in scope when loading URLs from this source')
+
+ ### ModelWithOutputDir:
+ output_dir = models.CharField(max_length=255, null=False, blank=True, default='', help_text='The directory to store the output of this seed')
+
+ ### ModelWithNotes:
+ notes = models.TextField(blank=True, null=False, default='', help_text='Any extra notes this seed should have')
+
+ ### ModelWithKVTags:
+ tag_set = GenericRelation(
+ KVTag,
+ related_query_name="seed",
+ content_type_field="obj_type",
+ object_id_field="obj_id",
+ order_by=('name',),
+ )
+
+ ### ABIDModel:
+ abid_prefix = 'src_'
+ abid_ts_src = 'self.created_at'
+ abid_uri_src = 'self.uri'
+ abid_subtype_src = 'self.extractor'
+ abid_rand_src = 'self.id'
+ abid_drift_allowed = True
+
+ ### Managers:
+ crawl_set: models.Manager['Crawl']
+
+ class Meta:
+ verbose_name = 'Seed'
+ verbose_name_plural = 'Seeds'
+
+ unique_together = (('created_by', 'uri', 'extractor'),('created_by', 'label'))
+
+
+ @classmethod
+ def from_file(cls, source_file: Path, label: str='', parser: str='auto', tag: str='', created_by: int|None=None, config: dict|None=None):
+ source_path = str(source_file.resolve()).replace(str(CONSTANTS.DATA_DIR), '/data')
+
+ seed, _ = cls.objects.get_or_create(
+ label=label or source_file.name,
+ uri=f'file://{source_path}',
+ created_by_id=getattr(created_by, 'pk', created_by) or get_or_create_system_user_pk(),
+ extractor=parser,
+ tags_str=tag,
+ config=config or {},
+ )
+ seed.save()
+ return seed
+
+ @property
+ def source_type(self):
+ # e.g. http/https://
+ # file://
+ # pocketapi://
+ # s3://
+ # etc..
+ return self.uri.split('://', 1)[0].lower()
+
+ @property
+ def api_url(self) -> str:
+ # /api/v1/core/seed/{uulid}
+ return reverse_lazy('api-1:get_seed', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}'
+
+ @property
+ def api_docs_url(self) -> str:
+ return '/api/v1/docs#/Core%20Models/api_v1_core_get_seed'
+
+ @property
+ def scheduled_crawl_set(self) -> QuerySet['CrawlSchedule']:
+ from crawls.models import CrawlSchedule
+ return CrawlSchedule.objects.filter(template__seed_id=self.pk)
+
+ @property
+ def snapshot_set(self) -> QuerySet['Snapshot']:
+ from core.models import Snapshot
+
+ crawl_ids = self.crawl_set.values_list('pk', flat=True)
+ return Snapshot.objects.filter(crawl_id__in=crawl_ids)
+
+
+
+
+class CrawlSchedule(ModelWithReadOnlyFields, ModelWithSerializers, ModelWithUUID, ModelWithKVTags, ABIDModel, ModelWithNotes, ModelWithHealthStats):
+ """
+ A record for a job that should run repeatedly on a given schedule.
+
+ It pulls from a given Seed and creates a new Crawl for each scheduled run.
+ The new Crawl will inherit all the properties of the crawl_template Crawl.
+ """
+ ### ModelWithReadOnlyFields:
+ read_only_fields = ('id', 'abid', 'created_at', 'created_by', 'template_id')
+
+ ### Immutable fields:
+ id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+ abid = ABIDField(prefix=abid_prefix)
+ created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+ created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
+ template: 'Crawl' = models.ForeignKey('Crawl', on_delete=models.CASCADE, null=False, blank=False, help_text='The base crawl that each new scheduled job should copy as a template') # type: ignore
+
+ ### Mutable fields
+ schedule = models.CharField(max_length=64, blank=False, null=False, help_text='The schedule to run this crawl on in CRON syntax e.g. 0 0 * * * (see https://crontab.guru/)')
+ is_enabled = models.BooleanField(default=True)
+ label = models.CharField(max_length=64, blank=True, null=False, default='', help_text='A human-readable label for this scheduled crawl')
+ notes = models.TextField(blank=True, null=False, default='', help_text='Any extra notes this crawl should have')
+ modified_at = models.DateTimeField(auto_now=True)
+
+ ### ModelWithKVTags:
+ tag_set = GenericRelation(
+ KVTag,
+ related_query_name="crawlschedule",
+ content_type_field="obj_type",
+ object_id_field="obj_id",
+ order_by=('name',),
+ )
+
+ ### ABIDModel:
+ abid_prefix = 'cws_'
+ abid_ts_src = 'self.created_at'
+ abid_uri_src = 'self.template.seed.uri'
+ abid_subtype_src = 'self.template.persona'
+ abid_rand_src = 'self.id'
+ abid_drift_allowed = True
+
+ ### Managers:
+ crawl_set: models.Manager['Crawl']
+
+ class Meta(TypedModelMeta):
+ verbose_name = 'Scheduled Crawl'
+ verbose_name_plural = 'Scheduled Crawls'
+
+ def __str__(self) -> str:
+ uri = (self.template and self.template.seed and self.template.seed.uri) or '<no url set>'
+ crawl_label = self.label or (self.template and self.template.seed and self.template.seed.label) or 'Untitled Crawl'
+ if self.id and self.template:
+ return f'[{self.ABID}] {uri[:64]} @ {self.schedule} (Scheduled {crawl_label})'
+ return f'[{self.abid_prefix}****not*saved*yet****] {uri[:64]} @ {self.schedule} (Scheduled {crawl_label})'
+
+ @property
+ def api_url(self) -> str:
+ # /api/v1/core/crawlschedule/{uulid}
+ return reverse_lazy('api-1:get_any', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}'
+
+ @property
+ def api_docs_url(self) -> str:
+ return '/api/v1/docs#/Core%20Models/api_v1_core_get_any'
+
+ def save(self, *args, **kwargs):
+ self.label = self.label or self.template.seed.label or self.template.seed.uri
+ super().save(*args, **kwargs)
+
+ # make sure the template crawl points to this schedule as its schedule
+ self.template.schedule = self
+ self.template.save()
+
+ @property
+ def snapshot_set(self) -> QuerySet['Snapshot']:
+ from core.models import Snapshot
+
+ crawl_ids = self.crawl_set.values_list('pk', flat=True)
+ return Snapshot.objects.filter(crawl_id__in=crawl_ids)
+
+
+class CrawlManager(models.Manager):
+ pass
+
+class CrawlQuerySet(models.QuerySet):
+ """
+ Enhanced QuerySet for Crawl that adds some useful methods.
+
+ To get all the snapshots for a given set of Crawls:
+ Crawl.objects.filter(seed__uri='https://example.com/some/rss.xml').snapshots() -> QuerySet[Snapshot]
+
+ To get all the archiveresults for a given set of Crawls:
+ Crawl.objects.filter(seed__uri='https://example.com/some/rss.xml').archiveresults() -> QuerySet[ArchiveResult]
+
+ To export the list of Crawls as a CSV or JSON:
+ Crawl.objects.filter(seed__uri='https://example.com/some/rss.xml').export_as_csv() -> str
+ Crawl.objects.filter(seed__uri='https://example.com/some/rss.xml').export_as_json() -> str
+ """
+ def snapshots(self, **filter_kwargs) -> QuerySet['Snapshot']:
+ return Snapshot.objects.filter(crawl_id__in=self.values_list('pk', flat=True), **filter_kwargs)
+
+ def archiveresults(self) -> QuerySet['ArchiveResult']:
+ return ArchiveResult.objects.filter(snapshot__crawl_id__in=self.values_list('pk', flat=True))
+
+ def as_csv_str(self, keys: Iterable[str]=()) -> str:
+ return '\n'.join(
+ row.as_csv(keys=keys)
+ for row in self.all()
+ )
+
+ def as_jsonl_str(self, keys: Iterable[str]=()) -> str:
+ return '\n'.join([
+ row.as_jsonl_row(keys=keys)
+ for row in self.all()
+ ])
+
+
+
+class Crawl(ModelWithReadOnlyFields, ModelWithSerializers, ModelWithUUID, ModelWithKVTags, ABIDModel, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithStateMachine):
+ """
+ A single session of URLs to archive starting from a given Seed and expanding outwards. An "archiving session" so to speak.
+
+ A new Crawl should be created for each loading from a Seed (because it can produce a different set of URLs every time its loaded).
+ E.g. every scheduled import from an RSS feed should create a new Crawl, and more loadings from the same seed each create a new Crawl
+
+ Every "Add" task triggered from the Web UI, CLI, or Scheduled Crawl should create a new Crawl with the seed set to a
+ file URI e.g. file:///sources/<date>_{ui,cli}_add.txt containing the user's input.
+ """
+
+ ### ModelWithReadOnlyFields:
+ read_only_fields = ('id', 'abid', 'created_at', 'created_by', 'seed')
+
+ ### Immutable fields:
+ id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+ abid = ABIDField(prefix=abid_prefix)
+ created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+ created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
+ seed = models.ForeignKey(Seed, on_delete=models.PROTECT, related_name='crawl_set', null=False, blank=False)
+
+ ### Mutable fields:
+ urls = models.TextField(blank=True, null=False, default='', help_text='The log of URLs discovered in this crawl, one per line, should be 1:1 with snapshot_set')
+ config = models.JSONField(default=dict)
+ max_depth = models.PositiveSmallIntegerField(default=0, validators=[MinValueValidator(0), MaxValueValidator(4)])
+ tags_str = models.CharField(max_length=1024, blank=True, null=False, default='')
+ persona_id = models.UUIDField(null=True, blank=True) # TODO: replace with self.persona = models.ForeignKey(Persona, on_delete=models.SET_NULL, null=True, blank=True, editable=True)
+ label = models.CharField(max_length=64, blank=True, null=False, default='', help_text='A human-readable label for this crawl')
+ notes = models.TextField(blank=True, null=False, default='', help_text='Any extra notes this crawl should have')
+ schedule = models.ForeignKey(CrawlSchedule, on_delete=models.SET_NULL, null=True, blank=True, editable=True)
+ modified_at = models.DateTimeField(auto_now=True)
+
+ ### ModelWithKVTags:
+ tag_set = GenericRelation(
+ KVTag,
+ related_query_name="crawl",
+ content_type_field="obj_type",
+ object_id_field="obj_id",
+ order_by=('name',),
+ )
+
+ ### ModelWithStateMachine:
+ state_machine_name = 'crawls.statemachines.CrawlMachine'
+ retry_at_field_name = 'retry_at'
+ state_field_name = 'status'
+ StatusChoices = ModelWithStateMachine.StatusChoices
+ active_state = StatusChoices.STARTED
+
+ status = ModelWithStateMachine.StatusField(choices=StatusChoices, default=StatusChoices.QUEUED)
+ retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
+
+ ### ABIDModel:
+ abid_prefix = 'cwl_'
+ abid_ts_src = 'self.created_at'
+ abid_uri_src = 'self.seed.uri'
+ abid_subtype_src = 'self.persona'
+ abid_rand_src = 'self.id'
+ abid_drift_allowed = True
+
+ ### Managers:
+ snapshot_set: models.Manager['Snapshot']
+
+ # @property
+ # def persona(self) -> Persona:
+ # # TODO: replace with self.persona = models.ForeignKey(Persona, on_delete=models.SET_NULL, null=True, blank=True, editable=True)
+ # return self.persona_id
+
+
+ class Meta(TypedModelMeta):
+ verbose_name = 'Crawl'
+ verbose_name_plural = 'Crawls'
+
+ def __str__(self):
+ url = (self.seed and self.seed.uri) or '<no url set>'
+ parser = (self.seed and self.seed.extractor) or 'auto'
+ created_at = self.created_at.strftime("%Y-%m-%d %H:%M") if self.created_at else '<no timestamp set>'
+ if self.id and self.seed:
+ return f'[{self.ABID}] {url[:64]} ({parser}) @ {created_at} ({self.label or "Untitled Crawl"})'
+ return f'[{self.abid_prefix}****not*saved*yet****] {url[:64]} ({parser}) @ {created_at} ({self.label or "Untitled Crawl"})'
+
+ @classmethod
+ def from_seed(cls, seed: Seed, max_depth: int=0, persona: str='Default', tags_str: str='', config: dict|None=None, created_by: int|None=None):
+ crawl, _ = cls.objects.get_or_create(
+ seed=seed,
+ max_depth=max_depth,
+ tags_str=tags_str or seed.tags_str,
+ persona=persona or seed.config.get('DEFAULT_PERSONA') or 'Default',
+ config=seed.config or config or {},
+ created_by_id=getattr(created_by, 'pk', created_by) or seed.created_by_id,
+ )
+ crawl.save()
+ return crawl
+
+ @property
+ def template(self):
+ """If this crawl was created under a ScheduledCrawl, returns the original template Crawl it was based off"""
+ if not self.schedule:
+ return None
+ return self.schedule.template
+
+ @property
+ def api_url(self) -> str:
+ # /api/v1/core/crawl/{uulid}
+ # TODO: implement get_crawl
+ return reverse_lazy('api-1:get_crawl', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}'
+
+ @property
+ def api_docs_url(self) -> str:
+ return '/api/v1/docs#/Core%20Models/api_v1_core_get_crawl'
+
+ def pending_snapshots(self) -> QuerySet['Snapshot']:
+ return self.snapshot_set.filter(retry_at__isnull=False)
+
+ def pending_archiveresults(self) -> QuerySet['ArchiveResult']:
+ from core.models import ArchiveResult
+
+ snapshot_ids = self.snapshot_set.values_list('id', flat=True)
+ pending_archiveresults = ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, retry_at__isnull=False)
+ return pending_archiveresults
+
+ def create_root_snapshot(self) -> 'Snapshot':
+ print(f'Crawl[{self.ABID}].create_root_snapshot()')
+ from core.models import Snapshot
+
+ try:
+ return Snapshot.objects.get(crawl=self, url=self.seed.uri)
+ except Snapshot.DoesNotExist:
+ pass
+
+ root_snapshot, _ = Snapshot.objects.update_or_create(
+ crawl=self,
+ url=self.seed.uri,
+ defaults={
+ 'status': Snapshot.INITIAL_STATE,
+ 'retry_at': timezone.now(),
+ 'timestamp': str(timezone.now().timestamp()),
+ # 'config': self.seed.config,
+ },
+ )
+ root_snapshot.save()
+ return root_snapshot
+
+
+class Outlink(ModelWithReadOnlyFields, ModelWithSerializers, ModelWithUUID, ModelWithKVTags):
+ """A record of a link found on a page, pointing to another page."""
+ read_only_fields = ('id', 'src', 'dst', 'crawl', 'via')
+
+ id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+
+ src = models.URLField() # parent page where the outlink/href was found e.g. https://example.com/downloads
+ dst = models.URLField() # remote location the child outlink/href points to e.g. https://example.com/downloads/some_file.pdf
+
+ crawl = models.ForeignKey(Crawl, on_delete=models.CASCADE, null=False, blank=False, related_name='outlink_set')
+ via = models.ForeignKey('core.ArchiveResult', on_delete=models.SET_NULL, null=True, blank=True, related_name='outlink_set')
+
+ class Meta:
+ unique_together = (('src', 'dst', 'via'),)
@@ -158,86 +551,83 @@ class SnapshotManager(models.Manager):
# .annotate(archiveresult_count=models.Count('archiveresult')).distinct()
)
-class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithStateMachine, ABIDModel):
+
+class Snapshot(
+ ModelWithReadOnlyFields,
+ ModelWithSerializers,
+ ModelWithUUID,
+ ModelWithKVTags,
+ ABIDModel,
+ ModelWithOutputDir,
+ ModelWithConfig,
+ ModelWithNotes,
+ ModelWithHealthStats,
+ ModelWithStateMachine,
+):
+
+ ### ModelWithSerializers
+ # cls.from_dict() -> Self
+ # self.as_json() -> dict[str, Any]
+ # self.as_jsonl_row() -> str
+ # self.as_csv_row() -> str
+ # self.as_html_icon(), .as_html_embed(), .as_html_row(), ...
+
+ ### ModelWithReadOnlyFields
+ read_only_fields = ('id', 'abid', 'created_at', 'created_by_id', 'url', 'timestamp', 'bookmarked_at', 'crawl_id')
+
+ ### Immutable fields:
+ id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+ abid = ABIDField(prefix=abid_prefix)
+ created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='snapshot_set', db_index=True)
+ created_at = AutoDateTimeField(default=None, null=False, db_index=True) # loaded from self._init_timestamp
+
+ url = models.URLField(unique=True, db_index=True)
+ timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False, validators=[validate_timestamp])
+ bookmarked_at = AutoDateTimeField(default=None, null=False, editable=True, db_index=True)
+ crawl: Crawl = models.ForeignKey(Crawl, on_delete=models.CASCADE, default=None, null=True, blank=True, related_name='snapshot_set', db_index=True) # type: ignore
+
+ ### Mutable fields:
+ title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
+ downloaded_at = models.DateTimeField(default=None, null=True, editable=False, db_index=True, blank=True)
+ modified_at = models.DateTimeField(auto_now=True)
+
+ ### ModelWithStateMachine
+ retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
+ status = ModelWithStateMachine.StatusField(choices=StatusChoices, default=StatusChoices.QUEUED)
+
+ ### ModelWithConfig
+ config = models.JSONField(default=dict, null=False, blank=False, editable=True)
+
+ ### ModelWithNotes
+ notes = models.TextField(blank=True, null=False, default='', help_text='Any extra notes this snapshot should have')
+
+ ### ModelWithOutputDir
+ output_dir = models.FilePathField(path=CONSTANTS.ARCHIVE_DIR, recursive=True, match='.*', default=None, null=True, blank=True, editable=True)
+ # self.output_dir_parent -> str 'archive/snapshots/<YYYY-MM-DD>/<example.com>'
+ # self.output_dir_name -> '<abid>'
+ # self.output_dir_str -> 'archive/snapshots/<YYYY-MM-DD>/<example.com>/<abid>'
+ # self.OUTPUT_DIR -> Path('/data/archive/snapshots/<YYYY-MM-DD>/<example.com>/<abid>')
+ # self.save(): creates OUTPUT_DIR, writes index.json, writes indexes
+
+ # old-style tags (dedicated ManyToMany Tag model above):
+ tags = models.ManyToManyField(Tag, blank=True, through=SnapshotTag, related_name='snapshot_set', through_fields=('snapshot', 'tag'))
+
+ # new-style tags (new key-value tags defined by tags.models.KVTag & ModelWithKVTags):
+ kvtag_set = tag_set = GenericRelation(
+ KVTag,
+ related_query_name="snapshot",
+ content_type_field="obj_type",
+ object_id_field="obj_id",
+ order_by=('created_at',),
+ )
+
+ ### ABIDModel
abid_prefix = 'snp_'
abid_ts_src = 'self.created_at'
abid_uri_src = 'self.url'
abid_subtype_src = '"01"'
abid_rand_src = 'self.id'
abid_drift_allowed = True
-
- state_machine_name = 'core.statemachines.SnapshotMachine'
- state_field_name = 'status'
- retry_at_field_name = 'retry_at'
- StatusChoices = ModelWithStateMachine.StatusChoices
- active_state = StatusChoices.STARTED
-
- id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
- abid = ABIDField(prefix=abid_prefix)
-
- created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='snapshot_set', db_index=True)
- created_at = AutoDateTimeField(default=None, null=False, db_index=True) # loaded from self._init_timestamp
- modified_at = models.DateTimeField(auto_now=True)
-
- retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
- status = ModelWithStateMachine.StatusField(choices=StatusChoices, default=StatusChoices.QUEUED)
- notes = models.TextField(blank=True, null=False, default='', help_text='Any extra notes this snapshot should have')
-
- bookmarked_at = AutoDateTimeField(default=None, null=False, editable=True, db_index=True)
- downloaded_at = models.DateTimeField(default=None, null=True, editable=False, db_index=True, blank=True)
-
- crawl: Crawl = models.ForeignKey(Crawl, on_delete=models.CASCADE, default=None, null=True, blank=True, related_name='snapshot_set', db_index=True) # type: ignore
-
- url = models.URLField(unique=True, db_index=True)
- timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False, validators=[validate_timestamp])
- tags = models.ManyToManyField(Tag, blank=True, through=SnapshotTag, related_name='snapshot_set', through_fields=('snapshot', 'tag'))
- title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
-
-
- keys = ('url', 'timestamp', 'title', 'tags', 'downloaded_at', 'created_at', 'status', 'retry_at', 'abid', 'id')
-
- archiveresult_set: models.Manager['ArchiveResult']
-
- objects = SnapshotManager()
-
- ### Inherited from ModelWithStateMachine #################################
- # StatusChoices = ModelWithStateMachine.StatusChoices
- #
- # status = ModelWithStateMachine.StatusField(choices=StatusChoices, default=StatusChoices.QUEUED)
- # retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
- #
- # state_machine_name = 'core.statemachines.SnapshotMachine'
- # state_field_name = 'status'
- # retry_at_field_name = 'retry_at'
- # active_state = StatusChoices.STARTED
- ########################################################################
-
- ### Inherited from ModelWithConfig #######################################
- # config = models.JSONField(default=dict, null=False, blank=False, editable=True)
- ########################################################################
-
- ### Inherited from ModelWithOutputDir:
- # output_dir = models.FilePathField(path=CONSTANTS.ARCHIVE_DIR, recursive=True, match='.*', default=None, null=True, blank=True, editable=True)
-
- # self.save(): creates OUTPUT_DIR, writes index.json, writes indexes
- # self.output_dir_parent -> str 'archive/snapshots/<YYYY-MM-DD>/<example.com>'
- # self.output_dir_name -> '<abid>'
- # self.output_dir_str -> 'archive/snapshots/<YYYY-MM-DD>/<example.com>/<abid>'
- # self.OUTPUT_DIR -> Path('/data/archive/snapshots/<YYYY-MM-DD>/<example.com>/<abid>')
-
- ### Inherited from ABIDModel:
- # id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
- # abid = ABIDField(prefix=abid_prefix)
- # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='snapshot_set', db_index=True)
- # created_at = AutoDateTimeField(default=None, null=False, db_index=True) # loaded from self._init_timestamp
- # modified_at = models.DateTimeField(auto_now=True)
-
- # abid_prefix = 'snp_'
- # abid_ts_src = 'self.created_at'
- # abid_uri_src = 'self.url'
- # abid_subtype_src = '"01"'
- # abid_rand_src = 'self.id'
- # abid_drift_allowed = True
# self.clean() -> sets self._timestamp
# self.save() -> issues new ABID if creating new, otherwise uses existing ABID
# self.ABID -> ABID
@@ -246,9 +636,18 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithStateMachine, ABIDM
# self.admin_change_url -> '/admin/core/snapshot/{pk}/change/'
# self.get_absolute_url() -> '/{self.archive_path}'
# self.update_for_workers() -> bool
- # self.as_json() -> dict[str, Any]
-
+ ### ModelWithStateMachine
+ state_machine_name = 'core.statemachines.SnapshotMachine'
+ state_field_name = 'status'
+ retry_at_field_name = 'retry_at'
+ StatusChoices = ModelWithStateMachine.StatusChoices
+ active_state = StatusChoices.STARTED
+
+ ### Relations & Managers
+ objects = SnapshotManager()
+ archiveresult_set: models.Manager['ArchiveResult']
+
def save(self, *args, **kwargs):
print(f'Snapshot[{self.ABID}].save()')
if self.pk:
@@ -292,16 +691,15 @@ class Snapshot(ModelWithOutputDir, ModelWithConfig, ModelWithStateMachine, ABIDM
return repr(self)
@classmethod
- def from_json(cls, info: dict):
- info = {k: v for k, v in info.items() if k in cls.keys}
- return cls(**info)
+ def from_json(cls, fields: dict[str, Any]) -> Self:
+ # print('LEGACY from_json()')
+ return cls.from_dict(fields)
- def as_json(self, *args) -> dict:
- args = args or self.keys
- return {
- key: getattr(self, key) if key != 'tags' else self.tags_str(nocache=False)
- for key in args
- }
+ def as_json(self, *args, **kwargs) -> dict:
+ json_dict = super().as_json(*args, **kwargs)
+ if 'tags' in json_dict:
+ json_dict['tags'] = self.tags_str(nocache=False)
+ return json_dict
def as_link(self) -> Link:
return Link.from_json(self.as_json())
@@ -620,7 +1018,12 @@ class ArchiveResultManager(models.Manager):
).order_by('indexing_precedence')
return qs
-class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine, ABIDModel):
+
+class ArchiveResult(
+ ModelWithReadOnlyFields, ModelWithSerializers, ModelWithUUID, ModelWithKVTags, ABIDModel,
+ ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithStateMachine
+):
+ ### ABIDModel
abid_prefix = 'res_'
abid_ts_src = 'self.snapshot.created_at'
abid_uri_src = 'self.snapshot.url'
@@ -628,6 +1031,7 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
abid_rand_src = 'self.id'
abid_drift_allowed = True
+ ### ModelWithStateMachine
class StatusChoices(models.TextChoices):
QUEUED = 'queued', 'Queued' # pending, initial
STARTED = 'started', 'Started' # active
@@ -658,33 +1062,48 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
('title', 'title'),
('wget', 'wget'),
)
+
+ ### ModelWithReadOnlyFields
+ read_only_fields = ('id', 'abid', 'created_at', 'created_by', 'snapshot', 'extractor', 'pwd')
-
+ ### Immutable fields:
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
abid = ABIDField(prefix=abid_prefix)
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='archiveresult_set', db_index=True)
created_at = AutoDateTimeField(default=None, null=False, db_index=True)
- modified_at = models.DateTimeField(auto_now=True)
- status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED)
- retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
-
snapshot: Snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE) # type: ignore
-
extractor = models.CharField(choices=EXTRACTOR_CHOICES, max_length=32, blank=False, null=False, db_index=True)
- cmd = models.JSONField(default=None, null=True, blank=True)
pwd = models.CharField(max_length=256, default=None, null=True, blank=True)
+
+
+ ### Mutable fields:
+ cmd = models.JSONField(default=None, null=True, blank=True)
+ modified_at = models.DateTimeField(auto_now=True)
cmd_version = models.CharField(max_length=128, default=None, null=True, blank=True)
output = models.CharField(max_length=1024, default=None, null=True, blank=True)
start_ts = models.DateTimeField(default=None, null=True, blank=True)
end_ts = models.DateTimeField(default=None, null=True, blank=True)
+
+ ### ModelWithStateMachine
+ status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED)
+ retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
+ ### ModelWithNotes
notes = models.TextField(blank=True, null=False, default='', help_text='Any extra notes this ArchiveResult should have')
- # the network interface that was used to download this result
+ ### ModelWithHealthStats
+ # ...
+
+ ### ModelWithKVTags
+ # tag_set = GenericRelation(KVTag, related_query_name='archiveresult')
+
+ ### ModelWithOutputDir
+ output_dir = models.CharField(max_length=256, default=None, null=True, blank=True)
+
# machine = models.ForeignKey(Machine, on_delete=models.SET_NULL, null=True, blank=True, verbose_name='Machine Used')
- # network = models.ForeignKey(NetworkInterface, on_delete=models.SET_NULL, null=True, blank=True, verbose_name='Network Interface Used')
+ iface = models.ForeignKey(NetworkInterface, on_delete=models.SET_NULL, null=True, blank=True, verbose_name='Network Interface Used')
objects = ArchiveResultManager()
@@ -718,7 +1137,7 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
super().save(*args, **kwargs)
# DONT DO THIS:
# self.snapshot.update_for_workers() # this should be done manually wherever its needed, not in here as a side-effect on save()
-
+
# TODO: finish connecting machine.models
# @cached_property
@@ -777,13 +1196,6 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
output_dir = Path(self.snapshot_dir) / self.extractor
output_dir.mkdir(parents=True, exist_ok=True)
return output_dir
-
- def as_json(self, *args) -> dict:
- args = args or self.keys
- return {
- key: getattr(self, key)
- for key in args
- }
def canonical_outputs(self) -> Dict[str, Optional[str]]:
"""Predict the expected output paths that should be present after archiving"""
@@ -958,3 +1370,50 @@ class ArchiveResult(ModelWithConfig, ModelWithOutputDir, ModelWithStateMachine,
# def symlink_index(self, create=True):
# abs_result_dir = self.get_storage_dir(create=create)
+
+
+
+
+
+
+# @abx.hookimpl.on_archiveresult_created
+# def exec_archiveresult_extractor_effects(archiveresult):
+# config = get_scope_config(...)
+
+# # abx.archivebox.writes.update_archiveresult_started(archiveresult, start_ts=timezone.now())
+# # abx.archivebox.events.on_archiveresult_updated(archiveresult)
+
+# # check if it should be skipped
+# if not abx.archivebox.reads.get_archiveresult_should_run(archiveresult, config):
+# abx.archivebox.writes.update_archiveresult_skipped(archiveresult, status='skipped')
+# abx.archivebox.events.on_archiveresult_skipped(archiveresult, config)
+# return
+
+# # run the extractor method and save the output back to the archiveresult
+# try:
+# output = abx.archivebox.effects.exec_archiveresult_extractor(archiveresult, config)
+# abx.archivebox.writes.update_archiveresult_succeeded(archiveresult, output=output, error=None, end_ts=timezone.now())
+# except Exception as e:
+# abx.archivebox.writes.update_archiveresult_failed(archiveresult, error=e, end_ts=timezone.now())
+
+# # bump the modified time on the archiveresult and Snapshot
+# abx.archivebox.events.on_archiveresult_updated(archiveresult)
+# abx.archivebox.events.on_snapshot_updated(archiveresult.snapshot)
+
+
+# @abx.hookimpl.reads.get_outlink_parents
+# def get_outlink_parents(url, crawl_pk=None, config=None):
+# scope = Q(dst=url)
+# if crawl_pk:
+# scope = scope | Q(via__snapshot__crawl_id=crawl_pk)
+
+# parent = list(Outlink.objects.filter(scope))
+# if not parent:
+# # base case: we reached the top of the chain, no more parents left
+# return []
+
+# # recursive case: there is another parent above us, get its parents
+# yield parent[0]
+# yield from get_outlink_parents(parent[0].src, crawl_pk=crawl_pk, config=config)
+
+