"""Compress existing raw mouth crop blocks without changing their public bytes.""" import hashlib import zlib from django.core.management.base import BaseCommand, CommandError from django.db import transaction from django.db.models.deletion import ProtectedError from clips import blobs from clips.models import Blob, Block class Command(BaseCommand): help = "Compress existing source/crops blobs and remove unreferenced raw copies" def handle(self, *args, **options): converted = 0 before = after = 0 for block in Block.objects.filter(role="source/crops").select_related("data"): old = block.data if old.media_type == blobs.CROP_MEDIA_TYPE: continue old_digest = old.digest with open(blobs.path_for(old_digest), "rb") as source: digest, size = blobs.write_compressed_stream( iter(lambda: source.read(blobs.CHUNK), b"") ) check = hashlib.sha256() decompressor = zlib.decompressobj() with open(blobs.path_for(digest), "rb") as compressed: while chunk := compressed.read(blobs.CHUNK): check.update(decompressor.decompress(chunk)) check.update(decompressor.flush()) if not decompressor.eof or check.hexdigest() != old_digest: raise CommandError(f"crop compression failed verification: {block.key}") with transaction.atomic(): new, _ = Blob.objects.get_or_create( digest=digest, defaults={"size": size, "media_type": blobs.CROP_MEDIA_TYPE}, ) changed = Block.objects.filter(key=block.key, data=old).update(data=new) if not changed: continue converted += 1 before += old.size after += size if old_digest != new.digest: try: old.delete() except ProtectedError: pass else: blobs.path_for(old_digest).unlink(missing_ok=True) self.stdout.write( f"Compressed {converted} crop blocks: {before:,} -> {after:,} bytes" )