Stream block uploads and compress cached mouth crops

clips/management/commands/compress_crop_blocks.py

@@ -0,0 +1,57 @@
+"""Compress existing raw mouth crop blocks without changing their public bytes."""
+
+import hashlib
+import zlib
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+from django.db.models.deletion import ProtectedError
+
+from clips import blobs
+from clips.models import Blob, Block
+
+
+class Command(BaseCommand):
+    help = "Compress existing source/crops blobs and remove unreferenced raw copies"
+
+    def handle(self, *args, **options):
+        converted = 0
+        before = after = 0
+        for block in Block.objects.filter(role="source/crops").select_related("data"):
+            old = block.data
+            if old.media_type == blobs.CROP_MEDIA_TYPE:
+                continue
+            old_digest = old.digest
+            with open(blobs.path_for(old_digest), "rb") as source:
+                digest, size = blobs.write_compressed_stream(
+                    iter(lambda: source.read(blobs.CHUNK), b"")
+                )
+            check = hashlib.sha256()
+            decompressor = zlib.decompressobj()
+            with open(blobs.path_for(digest), "rb") as compressed:
+                while chunk := compressed.read(blobs.CHUNK):
+                    check.update(decompressor.decompress(chunk))
+                check.update(decompressor.flush())
+            if not decompressor.eof or check.hexdigest() != old_digest:
+                raise CommandError(f"crop compression failed verification: {block.key}")
+            with transaction.atomic():
+                new, _ = Blob.objects.get_or_create(
+                    digest=digest,
+                    defaults={"size": size, "media_type": blobs.CROP_MEDIA_TYPE},
+                )
+                changed = Block.objects.filter(key=block.key, data=old).update(data=new)
+            if not changed:
+                continue
+            converted += 1
+            before += old.size
+            after += size
+            if old_digest != new.digest:
+                try:
+                    old.delete()
+                except ProtectedError:
+                    pass
+                else:
+                    blobs.path_for(old_digest).unlink(missing_ok=True)
+        self.stdout.write(
+            f"Compressed {converted} crop blocks: {before:,} -> {after:,} bytes"
+        )