Stream block uploads and compress cached mouth crops

clips/tests/test_api.py

@@ -16,6 +16,7 @@ of the system rather than a convention in ClojureScript:
 The rest is the load/save round trip, the conditional write, and the footage
 manifest that makes the frames the backend's to serve.
 """
+import base64
 import hashlib
 import json
 import shutil
@@ -23,11 +24,13 @@ import struct
 import subprocess
 import tempfile
 import zlib
+from io import StringIO
 from pathlib import Path
 from unittest import skipUnless
 from unittest.mock import Mock, patch
 
 from django.core.files.uploadedfile import SimpleUploadedFile
+from django.core.management import call_command
 from django.test import TestCase, override_settings
 
 from clips import blobs, extraction
@@ -215,6 +218,47 @@ class Tier2Tests(TestCase):
         self.assertEqual("AAE=", fetched["state"])
         self.assertEqual(descriptor, fetched["descriptor"])
 
+    @override_settings(DATA_UPLOAD_MAX_MEMORY_SIZE=1024, FILE_UPLOAD_MAX_MEMORY_SIZE=1024)
+    def test_large_block_upload_streams_past_json_body_limit(self):
+        analysis = self.register_analysis()
+        descriptor = block_descriptor(analysis, role="source/crops")
+        key = key_for(descriptor)
+        payload = bytes(range(256)) * 16
+        response = self.client.post("/api/blocks", {
+            "key": key,
+            "descriptor": descriptor,
+            "data": SimpleUploadedFile("block.bin", payload),
+            "state": SimpleUploadedFile("state.bin", b"\x00\x01"),
+        })
+        self.assertEqual(201, response.status_code, response.content)
+        row = Block.objects.get(key=key)
+        self.assertEqual(blobs.CROP_MEDIA_TYPE, row.data.media_type)
+        self.assertLess(row.data.size, len(payload))
+        self.assertEqual(payload, zlib.decompress(blobs.read(row.data_id)))
+        self.assertEqual(base64.b64encode(payload).decode(),
+                         self.client.get(f"/api/blocks/{key}").json()["data"])
+        self.assertEqual(b"\x00\x01", blobs.read(row.state_id))
+
+    def test_existing_raw_crop_block_is_compressed_without_changing_its_key_or_read(self):
+        analysis = self.register_analysis()
+        descriptor = block_descriptor(analysis, role="source/crops")
+        key = key_for(descriptor)
+        payload = b"raw crop pixels" * 100
+        digest, size = blobs.write(payload)
+        old = Blob.objects.create(digest=digest, size=size)
+        Block.objects.create(key=key, descriptor=descriptor, role="source/crops",
+                             analysis_id=analysis, data=old)
+
+        call_command("compress_crop_blocks", stdout=StringIO())
+        row = Block.objects.select_related("data").get(key=key)
+        self.assertEqual(blobs.CROP_MEDIA_TYPE, row.data.media_type)
+        self.assertEqual(base64.b64encode(payload).decode(),
+                         self.client.get(f"/api/blocks/{key}").json()["data"])
+        self.assertFalse(blobs.path_for(digest).exists())
+        compressed_digest = row.data_id
+        call_command("compress_crop_blocks", stdout=StringIO())
+        self.assertEqual(compressed_digest, Block.objects.get(key=key).data_id)
+
     def test_a_block_whose_analysis_is_unknown_is_refused(self):
         descriptor = block_descriptor("sha256:" + "f" * 64)
         response = self.post("/api/blocks", {