noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Decode uploaded footage in order with WebCodecs

Browse source
This commit is contained in:
Olive Vaughn 2026-09-28 14:18:09 -04:00
parent 131b39bff0
commit 65ad67c129
12 changed files with 454 additions and 364 deletions
Download patch file Download diff file Expand all files Collapse all files

80
frontend/test/arthur/flow/ingest_test.cljs
View file

@ -15,50 +15,52 @@
(is (thrown? ExceptionInfo (ingest/feature-presence 8 bad)))))
;; ---------------------------------------------------------------------------
;; walking the proxy
;; decoding the proxy
;;
;; These three are arithmetic, and all three were wrong in a way that no error
;; reported. A seek that lands one frame early, a timestamp read back with the
;; wrong inverse, or a frame time given as an index all produce a take that runs
;; to completion and is quietly off — which is why the numbers are asserted here
;; rather than left inline where they read as obvious.
;; What used to be here was arithmetic mapping a browser clock to a frame number,
;; and it is gone with the seeking that needed it. `access-units` is the only pure
;; thing left in the decode path, and it is worth testing precisely because it is
;; the step that makes "chunk k is frame k" true.
(deftest a-seek-aims-at-the-middle-of-its-frame
;; THE BUG THIS ENCODES: seeking to `i/fps` sits exactly on the boundary between
;; two frames, and over 91 frames of real footage it produced the PREVIOUS frame
;; 31 times. Every seek must land strictly inside its own frame's interval, with
;; half a frame of slack on each side.
(doseq [fps [12 23.976 24 25 29.97 30 60]]
(testing (str fps "fps")
(doseq [i [0 1 2 41 90 899]]
(let [t (ingest/seek-time fps i)]
(is (< (/ i fps) t (/ (inc i) fps))
(str "frame " i " at " fps "fps seeks to " t
", outside [" (/ i fps) ", " (/ (inc i) fps) ")"))
;; Within a rounding error of dead centre. Not `=`: `(/ i fps)` and
;; `(/ (+ i 0.5) fps)` are two different divisions, so their difference
;; carries the last bit of both and 0.4999999999999982 is a pass.
(is (< (abs (- 0.5 (* fps (- t (/ i fps))))) 1e-9)
"the aim point drifted off the middle of the frame"))))))
(defn- annexb
"A tiny Annex-B stream: `[[nal-type ...] ...]`, one vector per access unit."
[units]
(js/Uint8Array.
(into-array (mapcat (fn [types]
(mapcat (fn [t] [0 0 0 1 t 0x42]) types))
units))))
(deftest a-presented-media-time-reads-back-as-its-own-frame
;; The inverse of `i/fps` and NOT of `seek-time`: requestVideoFrameCallback
;; reports the frame's START. Rounding the wrong one is a silent off-by-one
;; between the landmarks and the audio.
(doseq [fps [12 24 29.97 30]]
(doseq [i [0 1 2 41 90]]
(is (= i (ingest/presented-frame fps (/ i fps)))
(str "frame " i " at " fps "fps did not read back as itself")))))
(deftest access-units-cut-the-stream-at-coded-frames
;; Types 1 and 5 are slices; 7, 8 and 6 are SPS, PPS and SEI, which belong to
;; the frame that FOLLOWS them. Getting that boundary wrong would shift every
;; parameter set onto the previous frame and make the first chunk undecodable.
(let [bytes (annexb [[7 8 5] [1] [1] [7 8 5] [1]])
units (ingest/access-units bytes)]
(is (= 5 (count units)) "one access unit per coded slice")
(is (= [true false false true false] (mapv :key? units))
"an IDR slice makes its access unit a keyframe")
(is (apply < (mapv :from units)) "units are in stream order")
(is (= (mapv :to units)
(mapv :from (concat (rest units) [{:from (.-length bytes)}])))
"units tile the stream with no bytes dropped between them")))
(deftest a-frame-time-is-footage-milliseconds-and-not-a-frame-number
(deftest a-stream-of-one-frame-is-one-unit
(is (= 1 (count (ingest/access-units (annexb [[7 8 5]]))))))
(deftest three-byte-and-four-byte-start-codes-are-both-found
;; Both are legal and ffmpeg emits both: four bytes before parameter sets and
;; three between slices. Missing the short form merges two frames into one.
(let [bytes (js/Uint8Array. (into-array [0 0 0 1 5 0x42 0 0 1 1 0x42 0 0 1 1 0x42]))]
(is (= 3 (count (ingest/access-units bytes))))))
(deftest a-frame-time-is-footage-milliseconds-and-strictly-increasing
;; MediaPipe's video mode is a tracker that reads the gap between timestamps as
;; motion. Handing it `i` still runs, and measurably moved landmarks six times
;; further from the per-frame answer. It must be real elapsed time.
;; motion, and its input stream refuses one that does not advance — an error the
;; landmarker never recovers from.
(is (= 0.0 (ingest/frame-ms 12 0)))
(is (= 1000.0 (ingest/frame-ms 12 12)))
(is (= (/ 1000 30) (ingest/frame-ms 30 1)))
(testing "strictly increasing, which the input stream requires"
(doseq [fps [12 23.976 30 60 240]]
(let [times (mapv #(ingest/frame-ms fps %) (range 200))]
(is (every? pos? (map - (rest times) times))
(str "two frames at " fps "fps shared a timestamp"))))))
(doseq [fps [12 23.976 30 60 240]]
(let [times (mapv #(ingest/frame-ms fps %) (range 200))]
(is (every? pos? (map - (rest times) times))
(str "two frames at " fps "fps shared a timestamp")))))