add created, modified, updated, created_by and update django admin

archivebox/abid_utils/models.py

@@ -1,14 +1,16 @@
-from typing import Any, Dict, Union, List, Set, cast
+from typing import Any, Dict, Union, List, Set, NamedTuple, cast
 
-import ulid
+from ulid import ULID
-from uuid import UUID
+from uuid import uuid4, UUID
 from typeid import TypeID            # type: ignore[import-untyped]
 from datetime import datetime
 from functools import partial
 from charidfield import CharIDField  # type: ignore[import-untyped]
 
+from django.conf import settings
 from django.db import models
 from django.db.utils import OperationalError
+from django.contrib.auth import get_user_model
 
 from django_stubs_ext.db.models import TypedModelMeta
 
@@ -37,6 +39,19 @@ ABIDField = partial(
     unique=True,
 )
 
+def get_or_create_system_user_pk(username='system'):
+    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
+
+    User = get_user_model()
+
+    # if only one user exists total, return that user
+    if User.objects.filter(is_superuser=True).count() == 1:
+        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
+
+    # otherwise, create a dedicated "system" user
+    user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
+    return user.pk
+
 
 class ABIDModel(models.Model):
     abid_prefix: str = DEFAULT_ABID_PREFIX  # e.g. 'tag_'
@@ -45,11 +60,13 @@ class ABIDModel(models.Model):
     abid_subtype_src = 'None'               # e.g. 'self.extractor'
     abid_rand_src = 'None'                  # e.g. 'self.uuid' or 'self.id'
 
-    # abid = ABIDField(prefix=abid_prefix, db_index=True, unique=True, null=True, blank=True, editable=True)
+    id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
+    uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
+    abid = ABIDField(prefix=abid_prefix)
 
-    # created = models.DateTimeField(auto_now_add=True, blank=True, null=True, db_index=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    # modified = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
+    created = models.DateTimeField(auto_now_add=True)
-    # created_by = models.ForeignKeyField(get_user_model(), blank=True, null=True, db_index=True)
+    modified = models.DateTimeField(auto_now=True)
 
     class Meta(TypedModelMeta):
         abstract = True
@@ -64,15 +81,21 @@ class ABIDModel(models.Model):
         
         super().save(*args, **kwargs)
 
-    def calculate_abid(self) -> ABID:
+    @property
+    def abid_values(self) -> Dict[str, Any]:
+        return {
+            'prefix': self.abid_prefix,
+            'ts': eval(self.abid_ts_src),
+            'uri': eval(self.abid_uri_src),
+            'subtype': eval(self.abid_subtype_src),
+            'rand': eval(self.abid_rand_src),
+        }
+
+    def get_abid(self) -> ABID:
         """
         Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
         """
-        prefix = self.abid_prefix
+        prefix, ts, uri, subtype, rand = self.abid_values.values()
-        ts = eval(self.abid_ts_src)
-        uri = eval(self.abid_uri_src)
-        subtype = eval(self.abid_subtype_src)
-        rand = eval(self.abid_rand_src)
 
         if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
             suggested_abid = self.__class__.__name__[:3].lower()
@@ -112,7 +135,7 @@ class ABIDModel(models.Model):
         return ABID.parse(self.abid) if getattr(self, 'abid', None) else self.calculate_abid()
 
     @property
-    def ULID(self) -> ulid.ULID:
+    def ULID(self) -> ULID:
         """
         Get a ulid.ULID representation of the object's ABID.
         """

archivebox/api/models.py

@@ -21,7 +21,11 @@ def generate_secret_token() -> str:
 
 
 class APIToken(ABIDModel):
-    abid_prefix = 'apt'
+    """
+    A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
+    """
+    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
+    abid_prefix = 'apt_'
     abid_ts_src = 'self.created'
     abid_uri_src = 'self.token'
     abid_subtype_src = 'self.user_id'
@@ -31,12 +35,13 @@ class APIToken(ABIDModel):
     uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
     abid = ABIDField(prefix=abid_prefix)
 
-    user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
     token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
     
     created = models.DateTimeField(auto_now_add=True)
     expires = models.DateTimeField(null=True, blank=True)
     
+
     class Meta(TypedModelMeta):
         verbose_name = "API Key"
         verbose_name_plural = "API Keys"
@@ -86,12 +91,13 @@ class OutboundWebhook(ABIDModel, WebhookBase):
     Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
         settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
     """
-    abid_prefix = 'whk'
+    abid_prefix = 'whk_'
     abid_ts_src = 'self.created'
     abid_uri_src = 'self.endpoint'
     abid_subtype_src = 'self.ref'
     abid_rand_src = 'self.id'
 
+    id = models.UUIDField(blank=True, null=True, unique=True, editable=True)
     uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True)
     abid = ABIDField(prefix=abid_prefix)
 

archivebox/core/admin.py

@@ -160,14 +160,41 @@ class SnapshotActionForm(ActionForm):
     # )
 
 
+def get_abid_info(self, obj):
+    return format_html(
+        # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+        '''
+        &nbsp; &nbsp; ABID:&nbsp; <code style="font-size: 16px; user-select: all"><b>{}</b></code><br/>
+        &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
+        &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
+        &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
+        &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/><br/>
+        &nbsp; &nbsp; ABID AS UUID:&nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;<br/><br/>
+
+        &nbsp; &nbsp; .uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;<br/>
+        &nbsp; &nbsp; .id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;<br/>
+        &nbsp; &nbsp; .pk: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;<br/><br/>
+        ''',
+        obj.abid,
+        obj.ABID.ts, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'],
+        obj.ABID.uri, str(obj.abid_values['uri']),
+        obj.ABID.subtype, str(obj.abid_values['subtype']),
+        obj.ABID.rand, str(obj.abid_values['rand'])[-7:],
+        obj.ABID.uuid,
+        obj.uuid,
+        obj.id,
+        obj.pk,
+    )
+
+
 @admin.register(Snapshot, site=archivebox_admin)
 class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
     list_display = ('added', 'title_str', 'files', 'size', 'url_str')
     sort_fields = ('title_str', 'url_str', 'added', 'files')
-    readonly_fields = ('info', 'pk', 'uuid', 'abid', 'calculate_abid', 'bookmarked', 'added', 'updated')
+    readonly_fields = ('admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'identifiers')
     search_fields = ('id', 'url', 'timestamp', 'title', 'tags__name')
-    fields = ('timestamp', 'url', 'title', 'tags', *readonly_fields)
+    fields = ('url', 'timestamp', 'created_by', 'tags', 'title', *readonly_fields)
-    list_filter = ('added', 'updated', 'tags', 'archiveresult__status')
+    list_filter = ('added', 'updated', 'tags', 'archiveresult__status', 'created_by')
     ordering = ['-added']
     actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
     autocomplete_fields = ['tags']
@@ -216,29 +243,30 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
     #         obj.pk,
     #     )
 
-    def info(self, obj):
+    def admin_actions(self, obj):
         return format_html(
+            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+            '''
+            <a class="btn" style="font-size: 15px; display: inline-block; border-radius: 10px; border: 2px solid #eee; padding: 4px 8px" href="/archive/{}">Summary page ➡️</a> &nbsp; &nbsp;
+            <a class="btn" style="font-size: 15px; display: inline-block; border-radius: 10px; border: 2px solid #eee; padding: 4px 8px" href="/archive/{}/index.html#all">Result files 📑</a> &nbsp; &nbsp;
+            <a class="btn" style="font-size: 15px; display: inline-block; border-radius: 10px; border: 2px solid #eee; padding: 4px 8px" href="/admin/core/snapshot/?id__exact={}">Admin actions ⚙️</a>
+            ''',
+            obj.timestamp,
+            obj.timestamp,
+            obj.pk,
+        )
+
+    def status_info(self, obj):
+        return format_html(
+            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
             '''
-            PK: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
-            ABID: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
-            UUID: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
-            Timestamp: <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;
-            URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
             Archived: {} ({} files {}) &nbsp; &nbsp;
             Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
-            Status code: {} &nbsp; &nbsp;
+            Status code: {} &nbsp; &nbsp;<br/>
             Server: {} &nbsp; &nbsp;
             Content type: {} &nbsp; &nbsp;
             Extension: {} &nbsp; &nbsp;
-            <br/><br/>
-            <a href="/archive/{}">View Snapshot index ➡️</a> &nbsp; &nbsp;
-            <a href="/admin/core/snapshot/?uuid__exact={}">View actions ⚙️</a>
             ''',
-            obj.pk,
-            obj.ABID,
-            obj.uuid,
-            obj.timestamp,
-            obj.url_hash,
             '✅' if obj.is_archived else '❌',
             obj.num_outputs,
             self.size(obj),
@@ -247,10 +275,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
             obj.headers and obj.headers.get('Server') or '?',
             obj.headers and obj.headers.get('Content-Type') or '?',
             obj.extension or '?',
-            obj.timestamp,
-            obj.uuid,
         )
 
+    def identifiers(self, obj):
+        return get_abid_info(self, obj)
+
     @admin.display(
         description='Title',
         ordering='title',
@@ -310,7 +339,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
         return format_html(
             '<a href="{}"><code style="user-select: all;">{}</code></a>',
             obj.url,
-            obj.url,
+            obj.url[:128],
         )
 
     def grid_view(self, request, extra_context=None):
@@ -413,14 +442,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
 @admin.register(Tag, site=archivebox_admin)
 class TagAdmin(admin.ModelAdmin):
-    list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'id')
+    list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'abid')
-    sort_fields = ('id', 'name', 'slug')
+    sort_fields = ('id', 'name', 'slug', 'abid')
-    readonly_fields = ('id', 'pk', 'abid', 'calculate_abid', 'num_snapshots', 'snapshots')
+    readonly_fields = ('created', 'modified', 'identifiers', 'num_snapshots', 'snapshots')
-    search_fields = ('id', 'name', 'slug')
+    search_fields = ('id', 'abid', 'uuid', 'name', 'slug')
-    fields = (*readonly_fields, 'name', 'slug')
+    fields = ('name', 'slug', 'created_by', *readonly_fields, )
     actions = ['delete_selected']
     ordering = ['-id']
 
+    def identifiers(self, obj):
+        return get_abid_info(self, obj)
+
     def num_snapshots(self, tag):
         return format_html(
             '<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
@@ -444,11 +476,11 @@ class TagAdmin(admin.ModelAdmin):
 
 @admin.register(ArchiveResult, site=archivebox_admin)
 class ArchiveResultAdmin(admin.ModelAdmin):
-    list_display = ('id', 'start_ts', 'extractor', 'snapshot_str', 'tags_str', 'cmd_str', 'status', 'output_str')
+    list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
     sort_fields = ('start_ts', 'extractor', 'status')
-    readonly_fields = ('id', 'ABID', 'snapshot_str', 'tags_str')
+    readonly_fields = ('snapshot_info', 'tags_str', 'created_by', 'created', 'modified', 'identifiers')
     search_fields = ('id', 'uuid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
-    fields = (*readonly_fields, 'snapshot', 'extractor', 'status', 'start_ts', 'end_ts', 'output', 'pwd', 'cmd', 'cmd_version')
+    fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'cmd',  'start_ts', 'end_ts', 'cmd_version', *readonly_fields)
     autocomplete_fields = ['snapshot']
 
     list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
@@ -456,19 +488,22 @@ class ArchiveResultAdmin(admin.ModelAdmin):
     list_per_page = SNAPSHOTS_PER_PAGE
 
     @admin.display(
-        description='snapshot'
+        description='Snapshot Info'
     )
-    def snapshot_str(self, result):
+    def snapshot_info(self, result):
         return format_html(
-            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b></a><br/>'
+            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            '<small>{}</small>',
-            result.snapshot.timestamp,
             result.snapshot.timestamp,
+            result.snapshot.abid,
+            result.snapshot.added.strftime('%Y-%m-%d %H:%M'),
             result.snapshot.url[:128],
         )
 
+    def identifiers(self, obj):
+        return get_abid_info(self, obj)
+
     @admin.display(
-        description='tags'
+        description='Snapshot Tags'
     )
     def tags_str(self, result):
         return result.snapshot.tags_str()

archivebox/core/models.py

@@ -53,19 +53,20 @@ class Tag(ABIDModel):
     Based on django-taggit model
     """
     abid_prefix = 'tag_'
-    abid_ts_src = 'None'          # TODO: add created/modified time
+    abid_ts_src = 'self.created'          # TODO: add created/modified time
     abid_uri_src = 'self.name'
     abid_subtype_src = '"03"'
     abid_rand_src = 'self.id'
 
+    # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True)
     id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
+    uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
     abid = ABIDField(prefix=abid_prefix)
-    # no uuid on Tags
+
 
     name = models.CharField(unique=True, blank=False, max_length=100)
-
-    # slug is autoset on save from name, never set it manually
     slug = models.SlugField(unique=True, blank=True, max_length=100)
+    # slug is autoset on save from name, never set it manually
 
 
     class Meta(TypedModelMeta):
@@ -325,8 +326,9 @@ class ArchiveResult(ABIDModel):
     abid_rand_src = 'self.uuid'
     EXTRACTOR_CHOICES = EXTRACTOR_CHOICES
 
+    # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
     id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')   # legacy pk
-    uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)      # legacy uuid
+    uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
     abid = ABIDField(prefix=abid_prefix)
 
     snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE)

archivebox/index/sql.py

@@ -143,7 +143,7 @@ def list_migrations(out_dir: Path=OUTPUT_DIR) -> List[Tuple[bool, str]]:
 def apply_migrations(out_dir: Path=OUTPUT_DIR) -> List[str]:
     from django.core.management import call_command
     null, out = StringIO(), StringIO()
-    call_command("makemigrations", interactive=False, stdout=null)
+    # call_command("makemigrations", interactive=False, stdout=null)
     call_command("migrate", interactive=False, stdout=out)
     out.seek(0)