From 241a7c6ab2980a37185c0cc1618779df1a0d2ee2 Mon Sep 17 00:00:00 2001
From: Nick Sweeting <github@sweeting.me>
Date: Mon, 13 May 2024 07:50:07 -0700
Subject: [PATCH] add created, modified, updated, created_by and update django
admin
---
archivebox/abid_utils/models.py | 51 +++++++++++-----
archivebox/api/models.py | 12 +++-
archivebox/core/admin.py | 105 +++++++++++++++++++++-----------
archivebox/core/models.py | 12 ++--
archivebox/index/sql.py | 2 +-
5 files changed, 124 insertions(+), 58 deletions(-)
diff --git a/archivebox/abid_utils/models.py b/archivebox/abid_utils/models.py
index 917b5283..0645a32f 100644
--- a/archivebox/abid_utils/models.py
+++ b/archivebox/abid_utils/models.py
@@ -1,14 +1,16 @@
-from typing import Any, Dict, Union, List, Set, cast
+from typing import Any, Dict, Union, List, Set, NamedTuple, cast
-import ulid
-from uuid import UUID
+from ulid import ULID
+from uuid import uuid4, UUID
from typeid import TypeID # type: ignore[import-untyped]
from datetime import datetime
from functools import partial
from charidfield import CharIDField # type: ignore[import-untyped]
+from django.conf import settings
from django.db import models
from django.db.utils import OperationalError
+from django.contrib.auth import get_user_model
from django_stubs_ext.db.models import TypedModelMeta
@@ -37,6 +39,19 @@ ABIDField = partial(
unique=True,
)
+def get_or_create_system_user_pk(username='system'):
+ """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
+
+ User = get_user_model()
+
+ # if only one user exists total, return that user
+ if User.objects.filter(is_superuser=True).count() == 1:
+ return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
+
+ # otherwise, create a dedicated "system" user
+ user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
+ return user.pk
+
class ABIDModel(models.Model):
abid_prefix: str = DEFAULT_ABID_PREFIX # e.g. 'tag_'
@@ -45,11 +60,13 @@ class ABIDModel(models.Model):
abid_subtype_src = 'None' # e.g. 'self.extractor'
abid_rand_src = 'None' # e.g. 'self.uuid' or 'self.id'
- # abid = ABIDField(prefix=abid_prefix, db_index=True, unique=True, null=True, blank=True, editable=True)
+ id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
+ uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
+ abid = ABIDField(prefix=abid_prefix)
- # created = models.DateTimeField(auto_now_add=True, blank=True, null=True, db_index=True)
- # modified = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
- # created_by = models.ForeignKeyField(get_user_model(), blank=True, null=True, db_index=True)
+ created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+ created = models.DateTimeField(auto_now_add=True)
+ modified = models.DateTimeField(auto_now=True)
class Meta(TypedModelMeta):
abstract = True
@@ -64,15 +81,21 @@ class ABIDModel(models.Model):
super().save(*args, **kwargs)
- def calculate_abid(self) -> ABID:
+ @property
+ def abid_values(self) -> Dict[str, Any]:
+ return {
+ 'prefix': self.abid_prefix,
+ 'ts': eval(self.abid_ts_src),
+ 'uri': eval(self.abid_uri_src),
+ 'subtype': eval(self.abid_subtype_src),
+ 'rand': eval(self.abid_rand_src),
+ }
+
+ def get_abid(self) -> ABID:
"""
Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
"""
- prefix = self.abid_prefix
- ts = eval(self.abid_ts_src)
- uri = eval(self.abid_uri_src)
- subtype = eval(self.abid_subtype_src)
- rand = eval(self.abid_rand_src)
+ prefix, ts, uri, subtype, rand = self.abid_values.values()
if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
suggested_abid = self.__class__.__name__[:3].lower()
@@ -112,7 +135,7 @@ class ABIDModel(models.Model):
return ABID.parse(self.abid) if getattr(self, 'abid', None) else self.calculate_abid()
@property
- def ULID(self) -> ulid.ULID:
+ def ULID(self) -> ULID:
"""
Get a ulid.ULID representation of the object's ABID.
"""
diff --git a/archivebox/api/models.py b/archivebox/api/models.py
index 87593bea..8d286a8b 100644
--- a/archivebox/api/models.py
+++ b/archivebox/api/models.py
@@ -21,7 +21,11 @@ def generate_secret_token() -> str:
class APIToken(ABIDModel):
- abid_prefix = 'apt'
+ """
+ A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
+ """
+ # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
+ abid_prefix = 'apt_'
abid_ts_src = 'self.created'
abid_uri_src = 'self.token'
abid_subtype_src = 'self.user_id'
@@ -31,11 +35,12 @@ class APIToken(ABIDModel):
uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
abid = ABIDField(prefix=abid_prefix)
- user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
+ created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
created = models.DateTimeField(auto_now_add=True)
expires = models.DateTimeField(null=True, blank=True)
+
class Meta(TypedModelMeta):
verbose_name = "API Key"
@@ -86,12 +91,13 @@ class OutboundWebhook(ABIDModel, WebhookBase):
Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
"""
- abid_prefix = 'whk'
+ abid_prefix = 'whk_'
abid_ts_src = 'self.created'
abid_uri_src = 'self.endpoint'
abid_subtype_src = 'self.ref'
abid_rand_src = 'self.id'
+ id = models.UUIDField(blank=True, null=True, unique=True, editable=True)
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True)
abid = ABIDField(prefix=abid_prefix)
diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py
index 4f84ebcf..15822478 100644
--- a/archivebox/core/admin.py
+++ b/archivebox/core/admin.py
@@ -160,14 +160,41 @@ class SnapshotActionForm(ActionForm):
# )
+def get_abid_info(self, obj):
+ return format_html(
+ # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+ '''
+ ABID: <code style="font-size: 16px; user-select: all"><b>{}</b></code><br/>
+ TS: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
+ URI: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
+ SUBTYPE: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
+ RAND: <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/><br/>
+ ABID AS UUID: <code style="font-size: 10px; user-select: all">{}</code> <br/><br/>
+
+ .uuid: <code style="font-size: 10px; user-select: all">{}</code> <br/>
+ .id: <code style="font-size: 10px; user-select: all">{}</code> <br/>
+ .pk: <code style="font-size: 10px; user-select: all">{}</code> <br/><br/>
+ ''',
+ obj.abid,
+ obj.ABID.ts, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'],
+ obj.ABID.uri, str(obj.abid_values['uri']),
+ obj.ABID.subtype, str(obj.abid_values['subtype']),
+ obj.ABID.rand, str(obj.abid_values['rand'])[-7:],
+ obj.ABID.uuid,
+ obj.uuid,
+ obj.id,
+ obj.pk,
+ )
+
+
@admin.register(Snapshot, site=archivebox_admin)
class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
list_display = ('added', 'title_str', 'files', 'size', 'url_str')
sort_fields = ('title_str', 'url_str', 'added', 'files')
- readonly_fields = ('info', 'pk', 'uuid', 'abid', 'calculate_abid', 'bookmarked', 'added', 'updated')
+ readonly_fields = ('admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'identifiers')
search_fields = ('id', 'url', 'timestamp', 'title', 'tags__name')
- fields = ('timestamp', 'url', 'title', 'tags', *readonly_fields)
- list_filter = ('added', 'updated', 'tags', 'archiveresult__status')
+ fields = ('url', 'timestamp', 'created_by', 'tags', 'title', *readonly_fields)
+ list_filter = ('added', 'updated', 'tags', 'archiveresult__status', 'created_by')
ordering = ['-added']
actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
autocomplete_fields = ['tags']
@@ -216,29 +243,30 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
# obj.pk,
# )
- def info(self, obj):
+ def admin_actions(self, obj):
return format_html(
+ # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+ '''
+ <a class="btn" style="font-size: 15px; display: inline-block; border-radius: 10px; border: 2px solid #eee; padding: 4px 8px" href="/archive/{}">Summary page ➡️</a>
+ <a class="btn" style="font-size: 15px; display: inline-block; border-radius: 10px; border: 2px solid #eee; padding: 4px 8px" href="/archive/{}/index.html#all">Result files 📑</a>
+ <a class="btn" style="font-size: 15px; display: inline-block; border-radius: 10px; border: 2px solid #eee; padding: 4px 8px" href="/admin/core/snapshot/?id__exact={}">Admin actions ⚙️</a>
+ ''',
+ obj.timestamp,
+ obj.timestamp,
+ obj.pk,
+ )
+
+ def status_info(self, obj):
+ return format_html(
+ # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
'''
- PK: <code style="font-size: 10px; user-select: all">{}</code>
- ABID: <code style="font-size: 10px; user-select: all">{}</code>
- UUID: <code style="font-size: 10px; user-select: all">{}</code>
- Timestamp: <code style="font-size: 10px; user-select: all">{}</code>
- URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
Archived: {} ({} files {})
Favicon: <img src="{}" style="height: 20px"/>
- Status code: {}
+ Status code: {} <br/>
Server: {}
Content type: {}
Extension: {}
- <br/><br/>
- <a href="/archive/{}">View Snapshot index ➡️</a>
- <a href="/admin/core/snapshot/?uuid__exact={}">View actions ⚙️</a>
''',
- obj.pk,
- obj.ABID,
- obj.uuid,
- obj.timestamp,
- obj.url_hash,
'✅' if obj.is_archived else '❌',
obj.num_outputs,
self.size(obj),
@@ -247,10 +275,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
obj.headers and obj.headers.get('Server') or '?',
obj.headers and obj.headers.get('Content-Type') or '?',
obj.extension or '?',
- obj.timestamp,
- obj.uuid,
)
+ def identifiers(self, obj):
+ return get_abid_info(self, obj)
+
@admin.display(
description='Title',
ordering='title',
@@ -310,7 +339,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
return format_html(
'<a href="{}"><code style="user-select: all;">{}</code></a>',
obj.url,
- obj.url,
+ obj.url[:128],
)
def grid_view(self, request, extra_context=None):
@@ -413,14 +442,17 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
@admin.register(Tag, site=archivebox_admin)
class TagAdmin(admin.ModelAdmin):
- list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'id')
- sort_fields = ('id', 'name', 'slug')
- readonly_fields = ('id', 'pk', 'abid', 'calculate_abid', 'num_snapshots', 'snapshots')
- search_fields = ('id', 'name', 'slug')
- fields = (*readonly_fields, 'name', 'slug')
+ list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'abid')
+ sort_fields = ('id', 'name', 'slug', 'abid')
+ readonly_fields = ('created', 'modified', 'identifiers', 'num_snapshots', 'snapshots')
+ search_fields = ('id', 'abid', 'uuid', 'name', 'slug')
+ fields = ('name', 'slug', 'created_by', *readonly_fields, )
actions = ['delete_selected']
ordering = ['-id']
+ def identifiers(self, obj):
+ return get_abid_info(self, obj)
+
def num_snapshots(self, tag):
return format_html(
'<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
@@ -444,11 +476,11 @@ class TagAdmin(admin.ModelAdmin):
@admin.register(ArchiveResult, site=archivebox_admin)
class ArchiveResultAdmin(admin.ModelAdmin):
- list_display = ('id', 'start_ts', 'extractor', 'snapshot_str', 'tags_str', 'cmd_str', 'status', 'output_str')
+ list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
sort_fields = ('start_ts', 'extractor', 'status')
- readonly_fields = ('id', 'ABID', 'snapshot_str', 'tags_str')
+ readonly_fields = ('snapshot_info', 'tags_str', 'created_by', 'created', 'modified', 'identifiers')
search_fields = ('id', 'uuid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
- fields = (*readonly_fields, 'snapshot', 'extractor', 'status', 'start_ts', 'end_ts', 'output', 'pwd', 'cmd', 'cmd_version')
+ fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'cmd', 'start_ts', 'end_ts', 'cmd_version', *readonly_fields)
autocomplete_fields = ['snapshot']
list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
@@ -456,19 +488,22 @@ class ArchiveResultAdmin(admin.ModelAdmin):
list_per_page = SNAPSHOTS_PER_PAGE
@admin.display(
- description='snapshot'
+ description='Snapshot Info'
)
- def snapshot_str(self, result):
+ def snapshot_info(self, result):
return format_html(
- '<a href="/archive/{}/index.html"><b><code>[{}]</code></b></a><br/>'
- '<small>{}</small>',
- result.snapshot.timestamp,
+ '<a href="/archive/{}/index.html"><b><code>[{}]</code></b> {} {}</a><br/>',
result.snapshot.timestamp,
+ result.snapshot.abid,
+ result.snapshot.added.strftime('%Y-%m-%d %H:%M'),
result.snapshot.url[:128],
)
+ def identifiers(self, obj):
+ return get_abid_info(self, obj)
+
@admin.display(
- description='tags'
+ description='Snapshot Tags'
)
def tags_str(self, result):
return result.snapshot.tags_str()
diff --git a/archivebox/core/models.py b/archivebox/core/models.py
index 8fced67d..0761985f 100644
--- a/archivebox/core/models.py
+++ b/archivebox/core/models.py
@@ -53,19 +53,20 @@ class Tag(ABIDModel):
Based on django-taggit model
"""
abid_prefix = 'tag_'
- abid_ts_src = 'None' # TODO: add created/modified time
+ abid_ts_src = 'self.created' # TODO: add created/modified time
abid_uri_src = 'self.name'
abid_subtype_src = '"03"'
abid_rand_src = 'self.id'
+ # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True)
id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
+ uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
abid = ABIDField(prefix=abid_prefix)
- # no uuid on Tags
+
name = models.CharField(unique=True, blank=False, max_length=100)
-
- # slug is autoset on save from name, never set it manually
slug = models.SlugField(unique=True, blank=True, max_length=100)
+ # slug is autoset on save from name, never set it manually
class Meta(TypedModelMeta):
@@ -325,8 +326,9 @@ class ArchiveResult(ABIDModel):
abid_rand_src = 'self.uuid'
EXTRACTOR_CHOICES = EXTRACTOR_CHOICES
+ # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID') # legacy pk
- uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) # legacy uuid
+ uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
abid = ABIDField(prefix=abid_prefix)
snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE)
diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py
index 3c4c2a96..8a67f109 100644
--- a/archivebox/index/sql.py
+++ b/archivebox/index/sql.py
@@ -143,7 +143,7 @@ def list_migrations(out_dir: Path=OUTPUT_DIR) -> List[Tuple[bool, str]]:
def apply_migrations(out_dir: Path=OUTPUT_DIR) -> List[str]:
from django.core.management import call_command
null, out = StringIO(), StringIO()
- call_command("makemigrations", interactive=False, stdout=null)
+ # call_command("makemigrations", interactive=False, stdout=null)
call_command("migrate", interactive=False, stdout=out)
out.seek(0)