add notes and label fields, fix model getters

archivebox/crawls/models.py

@@ -3,9 +3,8 @@ __package__ = 'archivebox.crawls'
 from typing import TYPE_CHECKING
 from django_stubs_ext.db.models import TypedModelMeta
 
-from datetime import timedelta
-
 from django.db import models
+from django.db.models import QuerySet
 from django.core.validators import MaxValueValidator, MinValueValidator 
 from django.conf import settings
 from django.urls import reverse_lazy
@@ -14,7 +13,7 @@ from django.utils import timezone
 from actors.models import ModelWithStateMachine
 
 if TYPE_CHECKING:
-    from core.models import Snapshot
+    from core.models import Snapshot, ArchiveResult
 
 from seeds.models import Seed
 
@@ -28,25 +27,64 @@ class CrawlSchedule(ABIDModel, ModelWithHealthStats):
     It pulls from a given Seed and creates a new Crawl for each scheduled run.
     The new Crawl will inherit all the properties of the crawl_template Crawl.
     """
-    abid_prefix = 'sch_'
+    abid_prefix = 'cws_'
     abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.created_by_id'
     abid_subtype_src = 'self.schedule'
     abid_rand_src = 'self.id'
     
-    schedule = models.CharField(max_length=64, blank=False, null=False)
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    abid = ABIDField(prefix=abid_prefix)
     
-    is_enabled = models.BooleanField(default=True)
+    schedule = models.CharField(max_length=64, blank=False, null=False, help_text='The schedule to run this crawl on in CRON syntax e.g. 0 0 * * * (see https://crontab.guru/)')
+    label = models.CharField(max_length=64, blank=True, null=False, default='', help_text='A human-readable label for this scheduled crawl')
+    notes = models.TextField(blank=True, null=False, default='', help_text='Any extra notes this crawl should have')
+    
+    template: 'Crawl' = models.ForeignKey('Crawl', on_delete=models.CASCADE, null=False, blank=False, help_text='The base crawl that each new scheduled job should copy as a template')  # type: ignore
+    
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
     created_at = AutoDateTimeField(default=None, null=False, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
+    
+    is_enabled = models.BooleanField(default=True)
     
     crawl_set: models.Manager['Crawl']
     
+    class Meta(TypedModelMeta):
+        verbose_name = 'Scheduled Crawl'
+        verbose_name_plural = 'Scheduled Crawls'
+        
+    def __str__(self) -> str:
+        uri = (self.template and self.template.seed and self.template.seed.uri) or '<no url set>'
+        crawl_label = self.label or (self.template and self.template.seed and self.template.seed.label) or 'Untitled Crawl'
+        if self.id and self.template:
+            return f'[{self.ABID}] {uri[:64]} @ {self.schedule} (Scheduled {crawl_label})'
+        return f'[{self.abid_prefix}****not*saved*yet****] {uri[:64]} @ {self.schedule} (Scheduled {crawl_label})'
+    
     @property
-    def template(self):
-        """The base crawl that each new scheduled job should copy as a template"""
-        return self.crawl_set.first()
+    def api_url(self) -> str:
+        # /api/v1/core/crawlschedule/{uulid}
+        return reverse_lazy('api-1:get_any', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
+
+    @property
+    def api_docs_url(self) -> str:
+        return '/api/v1/docs#/Core%20Models/api_v1_core_get_any'
+    
+    def save(self, *args, **kwargs):
+        self.label = self.label or self.template.seed.label or self.template.seed.uri
+        super().save(*args, **kwargs)
+        
+        # make sure the template crawl points to this schedule as its schedule
+        self.template.schedule = self
+        self.template.save()
+        
+    @property
+    def snapshot_set(self) -> QuerySet['Snapshot']:
+        from core.models import Snapshot
+        
+        crawl_ids = self.crawl_set.values_list('pk', flat=True)
+        return Snapshot.objects.filter(crawl_id__in=crawl_ids)
+    
 
     
 
@@ -60,7 +98,7 @@ class Crawl(ABIDModel, ModelWithHealthStats, ModelWithStateMachine):
     Every "Add" task triggered from the Web UI, CLI, or Scheduled Crawl should create a new Crawl with the seed set to a 
     file URI e.g. file:///sources/<date>_{ui,cli}_add.txt containing the user's input.
     """
-    abid_prefix = 'crl_'
+    abid_prefix = 'cwl_'
     abid_ts_src = 'self.created_at'
     abid_uri_src = 'self.seed.uri'
     abid_subtype_src = 'self.persona'
@@ -84,6 +122,10 @@ class Crawl(ABIDModel, ModelWithHealthStats, ModelWithStateMachine):
     retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
 
     seed = models.ForeignKey(Seed, on_delete=models.PROTECT, related_name='crawl_set', null=False, blank=False)
+    
+    label = models.CharField(max_length=64, blank=True, null=False, default='', help_text='A human-readable label for this crawl')
+    notes = models.TextField(blank=True, null=False, default='', help_text='Any extra notes this crawl should have')
+    
     max_depth = models.PositiveSmallIntegerField(default=0, validators=[MinValueValidator(0), MaxValueValidator(4)])
     tags_str = models.CharField(max_length=1024, blank=True, null=False, default='')
     persona = models.CharField(max_length=32, blank=True, null=False, default='auto')
@@ -103,6 +145,27 @@ class Crawl(ABIDModel, ModelWithHealthStats, ModelWithStateMachine):
         verbose_name = 'Crawl'
         verbose_name_plural = 'Crawls'
         
+    def __str__(self):
+        url = (self.seed and self.seed.uri) or '<no url set>'
+        parser = (self.seed and self.seed.extractor) or 'auto'
+        created_at = self.created_at.strftime("%Y-%m-%d %H:%M") if self.created_at else '<no timestamp set>'
+        if self.id and self.seed:
+            return f'[{self.ABID}] {url[:64]} ({parser}) @ {created_at} ({self.label or "Untitled Crawl"})'
+        return f'[{self.abid_prefix}****not*saved*yet****] {url[:64]} ({parser}) @ {created_at} ({self.label or "Untitled Crawl"})'
+        
+    @classmethod
+    def from_seed(cls, seed: Seed, max_depth: int=0, persona: str='Default', tags_str: str='', config: dict|None=None, created_by: int|None=None):
+        crawl, _ = cls.objects.get_or_create(
+            seed=seed,
+            max_depth=max_depth,
+            tags_str=tags_str or seed.tags_str,
+            persona=persona or seed.config.get('DEFAULT_PERSONA') or 'Default',
+            config=seed.config or config or {},
+            created_by_id=getattr(created_by, 'pk', created_by) or seed.created_by_id,
+        )
+        crawl.save()
+        return crawl
+        
     @property
     def template(self):
         """If this crawl was created under a ScheduledCrawl, returns the original template Crawl it was based off"""
@@ -120,12 +183,16 @@ class Crawl(ABIDModel, ModelWithHealthStats, ModelWithStateMachine):
     def api_docs_url(self) -> str:
         return '/api/v1/docs#/Core%20Models/api_v1_core_get_crawl'
     
-    def has_pending_archiveresults(self) -> bool:
+    def pending_snapshots(self) -> QuerySet['Snapshot']:
+        from core.models import Snapshot
+        return self.snapshot_set.exclude(status__in=Snapshot.FINAL_OR_ACTIVE_STATES)
+    
+    def pending_archiveresults(self) -> QuerySet['ArchiveResult']:
         from core.models import ArchiveResult
         
         snapshot_ids = self.snapshot_set.values_list('id', flat=True)
         pending_archiveresults = ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids).exclude(status__in=ArchiveResult.FINAL_OR_ACTIVE_STATES)
-        return pending_archiveresults.exists()
+        return pending_archiveresults
     
     def create_root_snapshot(self) -> 'Snapshot':
         from core.models import Snapshot
@@ -134,6 +201,9 @@ class Crawl(ABIDModel, ModelWithHealthStats, ModelWithStateMachine):
             crawl=self,
             url=self.seed.uri,
             status=Snapshot.INITIAL_STATE,
+            retry_at=timezone.now(),
+            timestamp=str(timezone.now().timestamp()),
+            # config=self.seed.config,
         )
         return root_snapshot