58 lines
2.2 KiB
Python
58 lines
2.2 KiB
Python
|
"""Compress existing raw mouth crop blocks without changing their public bytes."""
|
||
|
|
|
||
|
|
import hashlib
|
||
|
|
import zlib
|
||
|
|
|
||
|
|
from django.core.management.base import BaseCommand, CommandError
|
||
|
|
from django.db import transaction
|
||
|
|
from django.db.models.deletion import ProtectedError
|
||
|
|
|
||
|
|
from clips import blobs
|
||
|
|
from clips.models import Blob, Block
|
||
|
|
|
||
|
|
|
||
|
|
class Command(BaseCommand):
|
||
|
|
help = "Compress existing source/crops blobs and remove unreferenced raw copies"
|
||
|
|
|
||
|
|
def handle(self, *args, **options):
|
||
|
|
converted = 0
|
||
|
|
before = after = 0
|
||
|
|
for block in Block.objects.filter(role="source/crops").select_related("data"):
|
||
|
|
old = block.data
|
||
|
|
if old.media_type == blobs.CROP_MEDIA_TYPE:
|
||
|
|
continue
|
||
|
|
old_digest = old.digest
|
||
|
|
with open(blobs.path_for(old_digest), "rb") as source:
|
||
|
|
digest, size = blobs.write_compressed_stream(
|
||
|
|
iter(lambda: source.read(blobs.CHUNK), b"")
|
||
|
|
)
|
||
|
|
check = hashlib.sha256()
|
||
|
|
decompressor = zlib.decompressobj()
|
||
|
|
with open(blobs.path_for(digest), "rb") as compressed:
|
||
|
|
while chunk := compressed.read(blobs.CHUNK):
|
||
|
|
check.update(decompressor.decompress(chunk))
|
||
|
|
check.update(decompressor.flush())
|
||
|
|
if not decompressor.eof or check.hexdigest() != old_digest:
|
||
|
|
raise CommandError(f"crop compression failed verification: {block.key}")
|
||
|
|
with transaction.atomic():
|
||
|
|
new, _ = Blob.objects.get_or_create(
|
||
|
|
digest=digest,
|
||
|
|
defaults={"size": size, "media_type": blobs.CROP_MEDIA_TYPE},
|
||
|
|
)
|
||
|
|
changed = Block.objects.filter(key=block.key, data=old).update(data=new)
|
||
|
|
if not changed:
|
||
|
|
continue
|
||
|
|
converted += 1
|
||
|
|
before += old.size
|
||
|
|
after += size
|
||
|
|
if old_digest != new.digest:
|
||
|
|
try:
|
||
|
|
old.delete()
|
||
|
|
except ProtectedError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
blobs.path_for(old_digest).unlink(missing_ok=True)
|
||
|
|
self.stdout.write(
|
||
|
|
f"Compressed {converted} crop blocks: {before:,} -> {after:,} bytes"
|
||
|
|
)
|