noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Decode uploaded footage in order with WebCodecs

Browse source
This commit is contained in:
Olive Vaughn 2026-09-28 14:18:09 -04:00
parent 131b39bff0
commit 65ad67c129
12 changed files with 454 additions and 364 deletions
Download patch file Download diff file Expand all files Collapse all files

49
clips/extraction.py
View file

@ -106,7 +106,10 @@ def _encode_proxy(job, source_path, proxy_path, facts, root):
# step that makes the thing the page measures not be.
"-fps_mode", "cfr", "-r", facts.get("rate") or str(facts["fps"]),
"-c:v", "libx264", "-preset", "veryfast", "-crf", PROXY_CRF,
# NO B-FRAMES, AND THIS IS THE LOAD-BEARING FLAG. With them x264 has a
# NO B-FRAMES, AND THIS IS THE LOAD-BEARING FLAG. It is what makes
# decode order presentation order, so the page can treat access unit k
# of the elementary stream as frame k without demuxing a container or
# consulting a timestamp. With them x264 has a
# two-frame reordering delay, ffmpeg compensates by writing an edit list
# (`elst` media_time 1024 at timebase 1/15360 — exactly two frames), and
# the browser then lives on two timelines at once: `currentTime` obeys the
@ -125,6 +128,25 @@ def _encode_proxy(job, source_path, proxy_path, facts, root):
root, "proxy", total, (0, 55))
def _elementary_stream(proxy_path, out_path):
"""The proxy's video, unwrapped into a raw Annex-B H.264 stream.
A STREAM COPY, not a second encode: the same coded frames as the MP4, with
the container's length-prefixed NAL units rewritten as start-code-delimited
ones. It costs a file read and nothing else.
This exists because the page decodes with WebCodecs, and `VideoDecoder` takes
demuxed chunks rather than a container. Handing it Annex-B means the client
needs no demuxer: NAL start codes are findable in a loop, and because the
proxy is encoded with no B-frames, decode order is presentation order — so
access unit k IS frame k, with no container timing to consult and no clock to
reconcile. That is the whole reason this file is worth the bytes it costs.
"""
_command(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
"-i", str(proxy_path), "-an", "-c:v", "copy",
"-bsf:v", "h264_mp4toannexb", "-f", "h264", str(out_path)])
def _extract_stills(job, proxy_path, frames_dir, frames, root):
"""The proxy -> one tracing JPEG per frame, long edge capped."""
_run_with_progress(
@ -234,13 +256,16 @@ def count_frames(path):
def extraction_key(source, settings):
text = json.dumps({"scheme": 2, "source": source.blob_id, "settings": settings},
# Scheme 3: the extraction now also produces the elementary stream the page
# decodes, so a job run under scheme 2 did not make everything this one does.
text = json.dumps({"scheme": 3, "source": source.blob_id, "settings": settings},
sort_keys=True, separators=(",", ":"))
return "sha256:" + hashlib.sha256(text.encode()).hexdigest()
def _register(job, proxy_path, stills, audio_path, facts):
def _register(job, proxy_path, stream_path, stills, audio_path, facts):
proxy_digest, proxy_size = blobs.adopt(proxy_path)
stream_digest, stream_size = blobs.adopt(stream_path)
audio_digest, audio_size = blobs.adopt(audio_path)
still_blobs = [(index, *blobs.adopt(path)) for index, path in enumerate(stills)]
width, height, fps, frames = facts["width"], facts["height"], facts["fps"], facts["frames"]
@ -256,13 +281,24 @@ def _register(job, proxy_path, stills, audio_path, facts):
with transaction.atomic():
proxy_blob, _ = Blob.objects.get_or_create(
digest=proxy_digest, defaults={"size": proxy_size, "media_type": "video/mp4"})
stream_blob, _ = Blob.objects.get_or_create(
digest=stream_digest, defaults={"size": stream_size, "media_type": "video/h264"})
audio_blob, _ = Blob.objects.get_or_create(
digest=audio_digest, defaults={"size": audio_size, "media_type": "audio/wav"})
footage, created = Footage.objects.get_or_create(
digest=h.hexdigest(),
defaults={"label": job.source.filename[:200], "source": job.source.filename[:200],
"fps": fps, "frames": frames, "width": width, "height": height,
"audio": audio_blob, "video": proxy_blob})
"audio": audio_blob, "video": proxy_blob, "stream": stream_blob})
if not created and not footage.stream_id:
# The same footage by identity, extracted before the elementary
# stream existed. Its digest is over the proxy and the audio, which
# have not changed — so this is the same footage gaining a file it
# was always entitled to, not a different one.
footage.stream = stream_blob
if not footage.video_id:
footage.video = proxy_blob
footage.save(update_fields=["stream", "video"])
if created:
rows = []
for index, digest, size in still_blobs:
@ -306,6 +342,9 @@ def run(key):
"audio would drift")
proxy_facts["frames"] = frames
stream_path = root / "proxy.h264"
_elementary_stream(proxy_path, stream_path)
frames_dir = root / "stills"
frames_dir.mkdir()
_extract_stills(job, proxy_path, frames_dir, frames, root)
@ -325,7 +364,7 @@ def run(key):
"-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono",
"-t", str(frames / proxy_facts["fps"]), "-c:a", "pcm_s16le",
str(audio_path)])
footage = _register(job, proxy_path, stills, audio_path, proxy_facts)
footage = _register(job, proxy_path, stream_path, stills, audio_path, proxy_facts)
job.footage, job.state, job.progress = footage, "done", 100
job.save(update_fields=["footage", "state", "progress", "updated"])
except Exception as exc: