From 849b4963a1e308aa085885ea210ef430ec8a8d29 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Tue, 20 Aug 2024 01:58:44 -0700 Subject: [PATCH] add migrations --- .../core/migrations/0055_alter_tag_slug.py | 18 +++++ .../core/migrations/0056_remove_tag_uuid.py | 17 ++++ .../migrations/0057_rename_id_tag_old_id.py | 18 +++++ .../core/migrations/0058_alter_tag_old_id.py | 19 +++++ archivebox/core/migrations/0059_tag_id.py | 81 +++++++++++++++++++ .../core/migrations/0060_alter_tag_id.py | 19 +++++ ...rename_tag_snapshottag_old_tag_and_more.py | 22 +++++ .../0062_alter_snapshottag_old_tag.py | 19 +++++ ...apshottag_tag_alter_snapshottag_old_tag.py | 40 +++++++++ ...er_snapshottag_unique_together_and_more.py | 27 +++++++ .../0065_remove_snapshottag_old_tag.py | 17 ++++ ...ottag_tag_alter_tag_id_alter_tag_old_id.py | 31 +++++++ .../migrations/0067_alter_snapshottag_tag.py | 19 +++++ .../0068_alter_archiveresult_options.py | 17 ++++ 14 files changed, 364 insertions(+) create mode 100644 archivebox/core/migrations/0055_alter_tag_slug.py create mode 100644 archivebox/core/migrations/0056_remove_tag_uuid.py create mode 100644 archivebox/core/migrations/0057_rename_id_tag_old_id.py create mode 100644 archivebox/core/migrations/0058_alter_tag_old_id.py create mode 100644 archivebox/core/migrations/0059_tag_id.py create mode 100644 archivebox/core/migrations/0060_alter_tag_id.py create mode 100644 archivebox/core/migrations/0061_rename_tag_snapshottag_old_tag_and_more.py create mode 100644 archivebox/core/migrations/0062_alter_snapshottag_old_tag.py create mode 100644 archivebox/core/migrations/0063_snapshottag_tag_alter_snapshottag_old_tag.py create mode 100644 archivebox/core/migrations/0064_alter_snapshottag_unique_together_and_more.py create mode 100644 archivebox/core/migrations/0065_remove_snapshottag_old_tag.py create mode 100644 archivebox/core/migrations/0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id.py create mode 100644 archivebox/core/migrations/0067_alter_snapshottag_tag.py create mode 100644 archivebox/core/migrations/0068_alter_archiveresult_options.py diff --git a/archivebox/core/migrations/0055_alter_tag_slug.py b/archivebox/core/migrations/0055_alter_tag_slug.py new file mode 100644 index 00000000..741b1365 --- /dev/null +++ b/archivebox/core/migrations/0055_alter_tag_slug.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0054_alter_snapshot_timestamp'), + ] + + operations = [ + migrations.AlterField( + model_name='tag', + name='slug', + field=models.SlugField(editable=False, max_length=100, unique=True), + ), + ] diff --git a/archivebox/core/migrations/0056_remove_tag_uuid.py b/archivebox/core/migrations/0056_remove_tag_uuid.py new file mode 100644 index 00000000..9c01507e --- /dev/null +++ b/archivebox/core/migrations/0056_remove_tag_uuid.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:25 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0055_alter_tag_slug'), + ] + + operations = [ + migrations.RemoveField( + model_name='tag', + name='uuid', + ), + ] diff --git a/archivebox/core/migrations/0057_rename_id_tag_old_id.py b/archivebox/core/migrations/0057_rename_id_tag_old_id.py new file mode 100644 index 00000000..ebe20b01 --- /dev/null +++ b/archivebox/core/migrations/0057_rename_id_tag_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0056_remove_tag_uuid'), + ] + + operations = [ + migrations.RenameField( + model_name='tag', + old_name='id', + new_name='old_id', + ), + ] diff --git a/archivebox/core/migrations/0058_alter_tag_old_id.py b/archivebox/core/migrations/0058_alter_tag_old_id.py new file mode 100644 index 00000000..4cc291c0 --- /dev/null +++ b/archivebox/core/migrations/0058_alter_tag_old_id.py @@ -0,0 +1,19 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:30 + +import core.models +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0057_rename_id_tag_old_id'), + ] + + operations = [ + migrations.AlterField( + model_name='tag', + name='old_id', + field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='Old ID'), + ), + ] diff --git a/archivebox/core/migrations/0059_tag_id.py b/archivebox/core/migrations/0059_tag_id.py new file mode 100644 index 00000000..004ac541 --- /dev/null +++ b/archivebox/core/migrations/0059_tag_id.py @@ -0,0 +1,81 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:33 + +from django.db import migrations, models +from abid_utils.models import ABID, abid_from_values + + +def calculate_abid(self): + """ + Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src). + """ + prefix = self.abid_prefix + ts = eval(self.abid_ts_src) + uri = eval(self.abid_uri_src) + subtype = eval(self.abid_subtype_src) + rand = eval(self.abid_rand_src) + + if (not prefix) or prefix == 'obj_': + suggested_abid = self.__class__.__name__[:3].lower() + raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})') + + if not ts: + ts = datetime.utcfromtimestamp(0) + print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat()) + + if not uri: + uri = str(self) + print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri) + + if not subtype: + subtype = self.__class__.__name__ + print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype) + + if not rand: + rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk') + print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand) + + abid = abid_from_values( + prefix=prefix, + ts=ts, + uri=uri, + subtype=subtype, + rand=rand, + ) + assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}' + return abid + + +def update_archiveresult_ids(apps, schema_editor): + Tag = apps.get_model("core", "Tag") + num_total = Tag.objects.all().count() + print(f' Updating {num_total} Tag.id, ArchiveResult.uuid values in place...') + for idx, tag in enumerate(Tag.objects.all().iterator()): + assert tag.name + tag.abid_prefix = 'tag_' + tag.abid_ts_src = 'self.created' + tag.abid_uri_src = 'self.slug' + tag.abid_subtype_src = '"03"' + tag.abid_rand_src = 'self.old_id' + tag.abid = calculate_abid(tag) + tag.id = tag.abid.uuid + tag.save(update_fields=["abid", "id"]) + assert str(ABID.parse(tag.abid).uuid) == str(tag.id) + if idx % 10 == 0: + print(f'Migrated {idx}/{num_total} Tag objects...') + + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0058_alter_tag_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='tag', + name='id', + field=models.UUIDField(blank=True, null=True), + ), + migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop), + ] diff --git a/archivebox/core/migrations/0060_alter_tag_id.py b/archivebox/core/migrations/0060_alter_tag_id.py new file mode 100644 index 00000000..aeabefdc --- /dev/null +++ b/archivebox/core/migrations/0060_alter_tag_id.py @@ -0,0 +1,19 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:42 + +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0059_tag_id'), + ] + + operations = [ + migrations.AlterField( + model_name='tag', + name='id', + field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True), + ), + ] diff --git a/archivebox/core/migrations/0061_rename_tag_snapshottag_old_tag_and_more.py b/archivebox/core/migrations/0061_rename_tag_snapshottag_old_tag_and_more.py new file mode 100644 index 00000000..e29c8081 --- /dev/null +++ b/archivebox/core/migrations/0061_rename_tag_snapshottag_old_tag_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:43 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0060_alter_tag_id'), + ] + + operations = [ + migrations.RenameField( + model_name='snapshottag', + old_name='tag', + new_name='old_tag', + ), + migrations.AlterUniqueTogether( + name='snapshottag', + unique_together={('snapshot', 'old_tag')}, + ), + ] diff --git a/archivebox/core/migrations/0062_alter_snapshottag_old_tag.py b/archivebox/core/migrations/0062_alter_snapshottag_old_tag.py new file mode 100644 index 00000000..561d739c --- /dev/null +++ b/archivebox/core/migrations/0062_alter_snapshottag_old_tag.py @@ -0,0 +1,19 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:44 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0061_rename_tag_snapshottag_old_tag_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='snapshottag', + name='old_tag', + field=models.ForeignKey(db_column='old_tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag'), + ), + ] diff --git a/archivebox/core/migrations/0063_snapshottag_tag_alter_snapshottag_old_tag.py b/archivebox/core/migrations/0063_snapshottag_tag_alter_snapshottag_old_tag.py new file mode 100644 index 00000000..6c574669 --- /dev/null +++ b/archivebox/core/migrations/0063_snapshottag_tag_alter_snapshottag_old_tag.py @@ -0,0 +1,40 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:45 + +import django.db.models.deletion +from django.db import migrations, models + + +def update_snapshottag_ids(apps, schema_editor): + Tag = apps.get_model("core", "Tag") + SnapshotTag = apps.get_model("core", "SnapshotTag") + num_total = SnapshotTag.objects.all().count() + print(f' Updating {num_total} SnapshotTag.tag_id values in place... (may take an hour or longer for large collections...)') + for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('old_tag_id').iterator()): + assert snapshottag.old_tag_id + tag = Tag.objects.get(old_id=snapshottag.old_tag_id) + snapshottag.tag_id = tag.id + snapshottag.save(update_fields=["tag_id"]) + assert str(snapshottag.tag_id) == str(tag.id) + if idx % 100 == 0: + print(f'Migrated {idx}/{num_total} SnapshotTag objects...') + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0062_alter_snapshottag_old_tag'), + ] + + operations = [ + migrations.AddField( + model_name='snapshottag', + name='tag', + field=models.ForeignKey(blank=True, db_column='tag_id', null=True, on_delete=django.db.models.deletion.CASCADE, to='core.tag', to_field='id'), + ), + migrations.AlterField( + model_name='snapshottag', + name='old_tag', + field=models.ForeignKey(db_column='old_tag_id', on_delete=django.db.models.deletion.CASCADE, related_name='snapshottags_old', to='core.tag'), + ), + migrations.RunPython(update_snapshottag_ids, reverse_code=migrations.RunPython.noop), + ] diff --git a/archivebox/core/migrations/0064_alter_snapshottag_unique_together_and_more.py b/archivebox/core/migrations/0064_alter_snapshottag_unique_together_and_more.py new file mode 100644 index 00000000..911bf68b --- /dev/null +++ b/archivebox/core/migrations/0064_alter_snapshottag_unique_together_and_more.py @@ -0,0 +1,27 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:50 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0063_snapshottag_tag_alter_snapshottag_old_tag'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='snapshottag', + unique_together=set(), + ), + migrations.AlterField( + model_name='snapshottag', + name='tag', + field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag', to_field='id'), + ), + migrations.AlterUniqueTogether( + name='snapshottag', + unique_together={('snapshot', 'tag')}, + ), + ] diff --git a/archivebox/core/migrations/0065_remove_snapshottag_old_tag.py b/archivebox/core/migrations/0065_remove_snapshottag_old_tag.py new file mode 100644 index 00000000..16b2eea0 --- /dev/null +++ b/archivebox/core/migrations/0065_remove_snapshottag_old_tag.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:51 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0064_alter_snapshottag_unique_together_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='snapshottag', + name='old_tag', + ), + ] diff --git a/archivebox/core/migrations/0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id.py b/archivebox/core/migrations/0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id.py new file mode 100644 index 00000000..e6022eab --- /dev/null +++ b/archivebox/core/migrations/0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id.py @@ -0,0 +1,31 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:52 + +import core.models +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0065_remove_snapshottag_old_tag'), + ] + + operations = [ + migrations.AlterField( + model_name='snapshottag', + name='tag', + field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag', to_field='id'), + ), + migrations.AlterField( + model_name='tag', + name='id', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True), + ), + migrations.AlterField( + model_name='tag', + name='old_id', + field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, unique=True, verbose_name='Old ID'), + ), + ] diff --git a/archivebox/core/migrations/0067_alter_snapshottag_tag.py b/archivebox/core/migrations/0067_alter_snapshottag_tag.py new file mode 100644 index 00000000..b1c9f6a5 --- /dev/null +++ b/archivebox/core/migrations/0067_alter_snapshottag_tag.py @@ -0,0 +1,19 @@ +# Generated by Django 5.0.6 on 2024-08-20 03:53 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id'), + ] + + operations = [ + migrations.AlterField( + model_name='snapshottag', + name='tag', + field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag'), + ), + ] diff --git a/archivebox/core/migrations/0068_alter_archiveresult_options.py b/archivebox/core/migrations/0068_alter_archiveresult_options.py new file mode 100644 index 00000000..d5606592 --- /dev/null +++ b/archivebox/core/migrations/0068_alter_archiveresult_options.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.6 on 2024-08-20 07:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0067_alter_snapshottag_tag'), + ] + + operations = [ + migrations.AlterModelOptions( + name='archiveresult', + options={'verbose_name': 'Archive Result', 'verbose_name_plural': 'Archive Results Log'}, + ), + ]