Stream block uploads and compress cached mouth crops

clips/views.py

@@ -26,6 +26,7 @@ tool got worse", with no event to attach it to.
 import hashlib
 import json
 import re
+import zlib
 from functools import lru_cache
 from pathlib import Path
 from uuid import UUID
@@ -98,6 +99,22 @@ def _blob(b64, media_type="application/octet-stream"):
     return blob
 
 
+def _uploaded_blob(upload, media_type="application/octet-stream"):
+    digest, size = blobs.write_stream(upload.chunks())
+    blob, _ = Blob.objects.get_or_create(
+        digest=digest, defaults={"size": size, "media_type": media_type}
+    )
+    return blob
+
+
+def _crop_blob(chunks):
+    digest, size = blobs.write_compressed_stream(chunks)
+    blob, _ = Blob.objects.get_or_create(
+        digest=digest, defaults={"size": size, "media_type": blobs.CROP_MEDIA_TYPE}
+    )
+    return blob
+
+
 # ---------------------------------------------------------------------------
 # the page
 
@@ -456,7 +473,10 @@ def blocks(request):
     """Store one dense block: its bytes, its optional absence mask, and the
     descriptor its key is the hash of."""
     try:
-        data = _body(request)
+        multipart = request.content_type == "multipart/form-data"
+        data = request.POST if multipart else _body(request)
+        upload = request.FILES.get("data") if multipart else None
+        state_upload = request.FILES.get("state") if multipart else None
         key = data.get("key")
         descriptor = data.get("descriptor")
         parsed = _check_key(key, descriptor)
@@ -478,17 +498,27 @@ def blocks(request):
                 "version that produced it",
                 analysis=analysis_key,
             )
-        if not data.get("data"):
+        if not (upload and upload.size) and not data.get("data"):
             raise Bad("a block with no bytes")
         with transaction.atomic():
+            if role == "source/crops":
+                if upload:
+                    data_blob = _crop_blob(upload.chunks())
+                else:
+                    import base64
+
+                    data_blob = _crop_blob([base64.b64decode(data["data"])])
+            else:
+                data_blob = _uploaded_blob(upload) if upload else _blob(data["data"])
             row, created = Block.objects.get_or_create(
                 key=key,
                 defaults={
                     "descriptor": descriptor,
                     "role": role,
                     "analysis": analysis,
-                    "data": _blob(data["data"]),
-                    "state": _blob(data["state"]) if data.get("state") else None,
+                    "data": data_blob,
+                    "state": (_uploaded_blob(state_upload) if state_upload else
+                              _blob(data["state"]) if data.get("state") else None),
                 },
             )
         return JsonResponse({"key": row.key, "created": created}, status=201 if created else 200)
@@ -504,10 +534,13 @@ def block_detail(request, key):
         row = Block.objects.select_related("data", "state").get(key=key)
     except Block.DoesNotExist:
         return JsonResponse({"error": "no such block"}, status=404)
+    data = blobs.read(row.data.digest)
+    if row.data.media_type == blobs.CROP_MEDIA_TYPE:
+        data = zlib.decompress(data)
     out = {
         "key": row.key,
         "descriptor": row.descriptor,
-        "data": base64.b64encode(blobs.read(row.data.digest)).decode("ascii"),
+        "data": base64.b64encode(data).decode("ascii"),
     }
     if row.state_id:
         out["state"] = base64.b64encode(blobs.read(row.state.digest)).decode("ascii")