noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Decode uploaded footage in order with WebCodecs

Browse source
This commit is contained in:
Olive Vaughn 2026-09-28 14:18:09 -04:00
parent 131b39bff0
commit 65ad67c129
12 changed files with 454 additions and 364 deletions
Download patch file Download diff file Expand all files Collapse all files

49
clips/extraction.py
View file

@ -106,7 +106,10 @@ def _encode_proxy(job, source_path, proxy_path, facts, root):
# step that makes the thing the page measures not be.
"-fps_mode", "cfr", "-r", facts.get("rate") or str(facts["fps"]),
"-c:v", "libx264", "-preset", "veryfast", "-crf", PROXY_CRF,
# NO B-FRAMES, AND THIS IS THE LOAD-BEARING FLAG. With them x264 has a
# NO B-FRAMES, AND THIS IS THE LOAD-BEARING FLAG. It is what makes
# decode order presentation order, so the page can treat access unit k
# of the elementary stream as frame k without demuxing a container or
# consulting a timestamp. With them x264 has a
# two-frame reordering delay, ffmpeg compensates by writing an edit list
# (`elst` media_time 1024 at timebase 1/15360 — exactly two frames), and
# the browser then lives on two timelines at once: `currentTime` obeys the
@ -125,6 +128,25 @@ def _encode_proxy(job, source_path, proxy_path, facts, root):
root, "proxy", total, (0, 55))
def _elementary_stream(proxy_path, out_path):
"""The proxy's video, unwrapped into a raw Annex-B H.264 stream.
A STREAM COPY, not a second encode: the same coded frames as the MP4, with
the container's length-prefixed NAL units rewritten as start-code-delimited
ones. It costs a file read and nothing else.
This exists because the page decodes with WebCodecs, and `VideoDecoder` takes
demuxed chunks rather than a container. Handing it Annex-B means the client
needs no demuxer: NAL start codes are findable in a loop, and because the
proxy is encoded with no B-frames, decode order is presentation order — so
access unit k IS frame k, with no container timing to consult and no clock to
reconcile. That is the whole reason this file is worth the bytes it costs.
"""
_command(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
"-i", str(proxy_path), "-an", "-c:v", "copy",
"-bsf:v", "h264_mp4toannexb", "-f", "h264", str(out_path)])
def _extract_stills(job, proxy_path, frames_dir, frames, root):
"""The proxy -> one tracing JPEG per frame, long edge capped."""
_run_with_progress(
@ -234,13 +256,16 @@ def count_frames(path):
def extraction_key(source, settings):
text = json.dumps({"scheme": 2, "source": source.blob_id, "settings": settings},
# Scheme 3: the extraction now also produces the elementary stream the page
# decodes, so a job run under scheme 2 did not make everything this one does.
text = json.dumps({"scheme": 3, "source": source.blob_id, "settings": settings},
sort_keys=True, separators=(",", ":"))
return "sha256:" + hashlib.sha256(text.encode()).hexdigest()
def _register(job, proxy_path, stills, audio_path, facts):
def _register(job, proxy_path, stream_path, stills, audio_path, facts):
proxy_digest, proxy_size = blobs.adopt(proxy_path)
stream_digest, stream_size = blobs.adopt(stream_path)
audio_digest, audio_size = blobs.adopt(audio_path)
still_blobs = [(index, *blobs.adopt(path)) for index, path in enumerate(stills)]
width, height, fps, frames = facts["width"], facts["height"], facts["fps"], facts["frames"]
@ -256,13 +281,24 @@ def _register(job, proxy_path, stills, audio_path, facts):
with transaction.atomic():
proxy_blob, _ = Blob.objects.get_or_create(
digest=proxy_digest, defaults={"size": proxy_size, "media_type": "video/mp4"})
stream_blob, _ = Blob.objects.get_or_create(
digest=stream_digest, defaults={"size": stream_size, "media_type": "video/h264"})
audio_blob, _ = Blob.objects.get_or_create(
digest=audio_digest, defaults={"size": audio_size, "media_type": "audio/wav"})
footage, created = Footage.objects.get_or_create(
digest=h.hexdigest(),
defaults={"label": job.source.filename[:200], "source": job.source.filename[:200],
"fps": fps, "frames": frames, "width": width, "height": height,
"audio": audio_blob, "video": proxy_blob})
"audio": audio_blob, "video": proxy_blob, "stream": stream_blob})
if not created and not footage.stream_id:
# The same footage by identity, extracted before the elementary
# stream existed. Its digest is over the proxy and the audio, which
# have not changed — so this is the same footage gaining a file it
# was always entitled to, not a different one.
footage.stream = stream_blob
if not footage.video_id:
footage.video = proxy_blob
footage.save(update_fields=["stream", "video"])
if created:
rows = []
for index, digest, size in still_blobs:
@ -306,6 +342,9 @@ def run(key):
"audio would drift")
proxy_facts["frames"] = frames
stream_path = root / "proxy.h264"
_elementary_stream(proxy_path, stream_path)
frames_dir = root / "stills"
frames_dir.mkdir()
_extract_stills(job, proxy_path, frames_dir, frames, root)
@ -325,7 +364,7 @@ def run(key):
"-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono",
"-t", str(frames / proxy_facts["fps"]), "-c:a", "pcm_s16le",
str(audio_path)])
footage = _register(job, proxy_path, stills, audio_path, proxy_facts)
footage = _register(job, proxy_path, stream_path, stills, audio_path, proxy_facts)
job.footage, job.state, job.progress = footage, "done", 100
job.save(update_fields=["footage", "state", "progress", "updated"])
except Exception as exc:

24
clips/migrations/0005_footage_stream_alter_footage_video.py Normal file
View file

@ -0,0 +1,24 @@
# Generated by Django 5.2.17 on 2026-09-28 17:11
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('clips', '0004_footage_video_alter_footageframe_index'),
]
operations = [
migrations.AddField(
model_name='footage',
name='stream',
field=models.ForeignKey(blank=True, help_text="the proxy's video as raw Annex-B H.264: what the page DECODES, one access unit per frame; null on footage extracted before it", null=True, on_delete=django.db.models.deletion.PROTECT, related_name='stream_for', to='clips.blob'),
),
migrations.AlterField(
model_name='footage',
name='video',
field=models.ForeignKey(blank=True, help_text='the browser-safe proxy, playable and seekable', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='video_for', to='clips.blob'),
),
]

7
clips/models.py
View file

@ -102,7 +102,12 @@ class Footage(models.Model):
audio = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="audio_for")
video = models.ForeignKey(
Blob, null=True, blank=True, on_delete=models.PROTECT, related_name="video_for",
help_text="the browser-safe proxy the page detects from; null on pre-proxy footage",
help_text="the browser-safe proxy, playable and seekable",
)
stream = models.ForeignKey(
Blob, null=True, blank=True, on_delete=models.PROTECT, related_name="stream_for",
help_text="the proxy's video as raw Annex-B H.264: what the page DECODES, "
"one access unit per frame; null on footage extracted before it",
)
feature_absence = models.JSONField(default=dict, blank=True)
created = models.DateTimeField(auto_now_add=True)

4
clips/views.py
View file

@ -240,6 +240,8 @@ def _footage_json(footage: Footage, urls=True):
# existed, which the loader reports as "re-extract this" rather than
# failing somewhere inside MediaPipe.
"video": f"/blob/{footage.video.digest}" if footage.video_id else None,
# What the page actually decodes: one access unit per frame, no container.
"stream": f"/blob/{footage.stream.digest}" if footage.stream_id else None,
"feature-absence": footage.feature_absence or {},
}
if urls:
@ -257,7 +259,7 @@ def footage_list(request):
@require_http_methods(["GET"])
def footage_detail(request, footage_id):
try:
footage = Footage.objects.select_related("audio", "video").get(id=footage_id)
footage = Footage.objects.select_related("audio", "video", "stream").get(id=footage_id)
except Footage.DoesNotExist:
return JsonResponse({"error": "no such footage"}, status=404)
return JsonResponse(_footage_json(footage))