Measure the video, not a PNG per frame

Detection now walks a browser-seekable H.264 proxy in MediaPipe's VIDEO
running mode. The PNG sequence it replaces was 112MB for 7.6 seconds at
1440x1920 and 1.1GB at the 900-frame limit; the proxy is 6MB, and landmarks
detected off decoded H.264 rather than off the PNGs moved at most 0.0033 of
frame width.

Three things had to be true for video mode to work, and each was measured
against the same footage decoded to PNGs:

/blob/<digest> answers byte ranges. Django's FileResponse does no Range
handling, and a media element handed 200 with no Accept-Ranges reports an
empty `seekable`, no-ops every currentTime write, and detects frame one
ninety times without raising.

A seek aims at the MIDDLE of its frame. Aiming at i/fps sits on a frame
boundary and landed one frame early 31 times in 91; (i + 0.5)/fps was exact
on all 91.

Timestamps are strictly increasing footage milliseconds. Video mode is a
tracker: a repeat leaves the graph in an error state every later call
re-throws, so the landmarker is discarded on failure, and passing the frame
index instead of i*1000/fps moved landmarks six times further from the
per-frame answer.

Frames are verified rather than trusted. requestVideoFrameCallback states
which frame it handed over, the walker discards any other and fails loudly
if the one it asked for never arrives — a stale presentation from the tail
of a previous seek is what produced "asked for frame 1 and it presented
frame 2" on a video whose seeks were in fact exact.

The proxy is re-encoded even when the upload is already H.264: HEVC is not
decodable everywhere, and footage identity is the proxy's digest. The JPEG
stills beside it are tracing references, outside the footage digest because
re-rendering them at another size is not different footage.

Verified end to end in a real browser against real footage: 228/228 frames
detected, a drawn roto face, 37 backend and 234 frontend tests green.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

clips/tests/test_api.py

@@ -105,6 +105,46 @@ class BlobStoreTests(TestCase):
         self.assertEqual(404, self.client.get("/blob/" + "0" * 64).status_code)
         self.assertEqual(404, self.client.get("/blob/nonsense").status_code)
 
+    def test_a_blob_serves_byte_ranges(self):
+        # NOT AN OPTIMISATION. A <video> that is handed 200 with no Accept-Ranges
+        # reports an empty `seekable`, every currentTime write is a no-op, and the
+        # detector then measures frame one over and over without anything raising.
+        # Django's FileResponse does no Range handling, so this is the whole of
+        # what makes the analysis source seekable.
+        digest, _ = blobs.write(b"0123456789")
+        Blob.objects.create(digest=digest, size=10, media_type="video/mp4")
+
+        whole = self.client.get(f"/blob/{digest}")
+        self.assertEqual(200, whole.status_code)
+        self.assertEqual("bytes", whole["Accept-Ranges"])
+
+        part = self.client.get(f"/blob/{digest}", headers={"range": "bytes=2-5"})
+        self.assertEqual(206, part.status_code)
+        self.assertEqual("bytes 2-5/10", part["Content-Range"])
+        self.assertEqual("4", part["Content-Length"])
+        self.assertEqual(b"2345", b"".join(part.streaming_content))
+
+        # An open end, which is what a media element actually sends first.
+        tail = self.client.get(f"/blob/{digest}", headers={"range": "bytes=7-"})
+        self.assertEqual(206, tail.status_code)
+        self.assertEqual("bytes 7-9/10", tail["Content-Range"])
+        self.assertEqual(b"789", b"".join(tail.streaming_content))
+
+        # A suffix range asks a different question: the LAST n bytes.
+        suffix = self.client.get(f"/blob/{digest}", headers={"range": "bytes=-3"})
+        self.assertEqual(206, suffix.status_code)
+        self.assertEqual("bytes 7-9/10", suffix["Content-Range"])
+
+        # Past the end is a 416 with the real length, so the client can recover.
+        over = self.client.get(f"/blob/{digest}", headers={"range": "bytes=50-60"})
+        self.assertEqual(416, over.status_code)
+        self.assertEqual("bytes */10", over["Content-Range"])
+
+        # Unparsable is not an error: RFC 9110 says ignore it and send it all.
+        junk = self.client.get(f"/blob/{digest}", headers={"range": "furlongs=1-2"})
+        self.assertEqual(200, junk.status_code)
+        self.assertEqual(b"0123456789", b"".join(junk.streaming_content))
+
 
 @override_settings(BLOB_ROOT=BLOB_DIR)
 class Tier2Tests(TestCase):
@@ -509,11 +549,12 @@ class PageTests(TestCase):
 @skipUnless(shutil.which("ffmpeg") and shutil.which("ffprobe"), "ffmpeg is required")
 @override_settings(BLOB_ROOT=BLOB_DIR)
 class UploadTests(TestCase):
-    def test_frame_decode_reports_live_progress(self):
+    def test_an_ffmpeg_stage_reports_live_progress_within_its_own_span(self):
+        # The job's percentage is shared between the encode and the stills, so a
+        # stage reports its own fraction of its own span rather than of the job.
+        # Half of the frames through a stage that owns 0-55 is 27.
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)
-            frames = root / "frames"
-            frames.mkdir()
             job = Mock(progress=0)
 
             class FakeProcess:
@@ -523,7 +564,7 @@ class UploadTests(TestCase):
                 def poll(self):
                     self.calls += 1
                     if self.calls == 1:
-                        (root / "frames.progress").write_text("frame=2\nprogress=continue\n")
+                        (root / "proxy.progress").write_text("frame=2\nprogress=continue\n")
                         return None
                     return 0
 
@@ -532,10 +573,9 @@ class UploadTests(TestCase):
 
             with patch("clips.extraction.subprocess.Popen", return_value=FakeProcess()), \
                  patch("clips.extraction.time.sleep"):
-                extraction._decode_frames(job, root / "source.mp4", frames,
-                                          {"reported_frames": 4, "duration": 1, "fps": 4},
-                                          root)
-            self.assertEqual(30, job.progress)
+                extraction._run_with_progress(job, ["-i", "in.mp4", "out.mp4"],
+                                              root, "proxy", 4, (0, 55))
+            self.assertEqual(27, job.progress)
             job.save.assert_called_once_with(update_fields=["progress", "updated"])
 
     def test_uploaded_video_extracts_to_reopenable_footage(self):
@@ -568,6 +608,61 @@ class UploadTests(TestCase):
         self.assertEqual("done", job["state"], job)
         footage = self.client.get(f"/api/footage/{job['footage']}").json()
         self.assertEqual((4, 64, 48), (footage["frames"], footage["width"], footage["height"]))
+
+        # THE PROXY IS THE ANALYSIS SOURCE. The page seeks this URL frame by
+        # frame, so it has to exist, be a video, and answer a Range request —
+        # without the last of those a media element cannot seek it at all.
+        self.assertTrue(footage["video"].startswith("/blob/"), footage)
+        proxy = self.client.get(footage["video"])
+        self.assertEqual(200, proxy.status_code)
+        self.assertEqual("video/mp4", proxy["Content-Type"])
+        self.assertEqual("bytes", proxy["Accept-Ranges"])
+        self.assertEqual(206, self.client.get(footage["video"],
+                                              headers={"range": "bytes=0-31"}).status_code)
+
+        # And the stills beside it are JPEGs for tracing, one per frame.
         self.assertEqual(4, len(footage["urls"]))
-        self.assertEqual(200, self.client.get(footage["urls"][0]).status_code)
+        still = self.client.get(footage["urls"][0])
+        self.assertEqual(200, still.status_code)
+        self.assertEqual("image/jpeg", still["Content-Type"])
         self.assertEqual(200, self.client.get(footage["audio"]).status_code)
+
+    def test_the_proxy_is_re_encoded_rather_than_the_upload_re_served(self):
+        # The footage's identity is the proxy's digest, and the proxy is produced
+        # by one ffmpeg invocation whatever the upload was. If the upload were
+        # passed through when it happened to be playable, identity would depend on
+        # which branch ran — and an HEVC upload would reach a browser that cannot
+        # decode it.
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "already-h264.mp4"
+            subprocess.run([
+                "ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
+                "-f", "lavfi", "-i", "testsrc=s=64x48:r=4:d=1",
+                "-c:v", "libx264", "-pix_fmt", "yuv420p", str(path),
+            ], check=True, capture_output=True)
+            payload = path.read_bytes()
+
+        uploaded = self.client.post("/api/sources", {
+            "file": SimpleUploadedFile("already-h264.mp4", payload, content_type="video/mp4")})
+        with patch("clips.extraction.enqueue", side_effect=extraction.run):
+            queued = self.client.post("/api/extractions", json.dumps({
+                "source": uploaded.json()["id"], "settings": {},
+            }), content_type="application/json")
+        job = self.client.get(f"/api/extractions/{queued.json()['key']}").json()
+        self.assertEqual("done", job["state"], job)
+
+        footage = Footage.objects.get(id=job["footage"])
+        self.assertIsNotNone(footage.video)
+        self.assertNotEqual(Source.objects.get(id=uploaded.json()["id"]).blob_id,
+                            footage.video_id)
+
+    def test_footage_without_a_proxy_says_so_rather_than_serving_nothing(self):
+        # Footage ingested before the proxy existed. The manifest reports a null
+        # video so the loader can name the fix; it does not omit the field and let
+        # the client discover it somewhere inside MediaPipe.
+        audio, size = blobs.write(b"RIFF....WAVEfmt ")
+        blob = Blob.objects.create(digest=audio, size=size, media_type="audio/wav")
+        footage = Footage.objects.create(
+            digest="e" * 64, fps=12, frames=3, width=8, height=6, audio=blob)
+        manifest = self.client.get(f"/api/footage/{footage.id}").json()
+        self.assertIsNone(manifest["video"])