Decode uploaded footage in order with WebCodecs
This commit is contained in:
Olive Vaughn
parent
131b39bff0
commit
65ad67c129
12 changed files with 454 additions and 364 deletions
|
|
@ -115,18 +115,18 @@ and real footage use `src/arthur/flow/take.cljs` for the measurement order and
|
|||
### Real footage
|
||||
|
||||
Choose a video in the **footage** file input. The server probes it, re-encodes it
|
||||
to a browser-seekable H.264 proxy, pulls WAV audio and one tracing JPEG per frame,
|
||||
to an H.264 proxy and a raw stream of the same coded frames, pulls WAV audio and one tracing JPEG per frame,
|
||||
then makes the resulting footage selectable. Click **load frames** to detect and
|
||||
freeze it. Extraction progress is currently read from `/api/extractions/<key>`; a
|
||||
future WebSocket can push the same job state. The uploaded bytes, extraction job,
|
||||
and decoded footage have separate records, so the same uploaded video can be
|
||||
reopened without decoding it again.
|
||||
|
||||
**The proxy is what gets measured, and the stills are not.** `flow/ingest` steps
|
||||
the proxy one frame at a time — seek to `(i + 0.5) / fps`, wait for
|
||||
`requestVideoFrameCallback`, check the `mediaTime` it reports is the frame that
|
||||
was asked for — and `flow/detect` hands each frame to MediaPipe in **VIDEO**
|
||||
running mode at `i * 1000 / fps` milliseconds. That timestamp has to increase
|
||||
**The proxy is what gets measured, and the stills are not.** `flow/ingest` cuts
|
||||
its raw H.264 stream into coded frames and decodes them in order with WebCodecs.
|
||||
The proxy has no B-frames, so decode order matches frame order. `flow/detect`
|
||||
hands each decoded frame to MediaPipe in **VIDEO** running mode at
|
||||
`i * 1000 / fps` milliseconds. That timestamp has to increase
|
||||
strictly and has to be real footage time: video mode is a tracker, it reads the
|
||||
gap between timestamps as motion, and a repeat leaves the graph in an error state
|
||||
that every later call re-throws. The JPEGs beside the proxy are reference images
|
||||
|
|
@ -136,7 +136,8 @@ digest.
|
|||
It is re-encoded even when the upload is already H.264, for two reasons: an
|
||||
iPhone's HEVC is not decodable in every browser, and the footage's identity is the
|
||||
proxy's digest — one produced by one ffmpeg invocation, not one that depends on
|
||||
which branch the source happened to take.
|
||||
which branch the source happened to take. Its raw stream is copied from that
|
||||
proxy without another encode.
|
||||
|
||||
The command-line route is also available for an existing extracted bundle:
|
||||
|
||||
|
|
|
|||
|
|
@ -16,15 +16,35 @@
|
|||
[arthur.fx.http :as http]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
(defonce ^:private clock (atom 0))
|
||||
|
||||
(defn- mark!
|
||||
"Where the time goes, on the console, one line per stage.
|
||||
|
||||
Kept rather than removed after it earned its keep. Progress only paints every
|
||||
fourth frame, so a stall near the end looks identical whether the decoder has
|
||||
stopped delivering or the work after it is holding the main thread — and those
|
||||
two were confused for each other three times before this printed the answer:
|
||||
decoding was finished, and `measure-crops` was ten seconds of synchronous
|
||||
arithmetic with nothing able to repaint."
|
||||
[label]
|
||||
(let [now (js/Date.now)
|
||||
since (- now @clock)]
|
||||
(reset! clock now)
|
||||
(js/console.log (str "arthur ⏱ " label " +" since "ms"))))
|
||||
|
||||
(defn- detect-frames!
|
||||
"Walk the proxy once, forward, and measure every frame as it goes.
|
||||
"Decode the proxy once, in order, and measure every frame as it goes.
|
||||
|
||||
ONCE AND FORWARD IS A REQUIREMENT, NOT A STYLE. MediaPipe's video mode is a
|
||||
tracker whose input stream refuses a timestamp that does not advance, and the
|
||||
error it raises is terminal for the landmarker — so there is no re-reading a
|
||||
frame, no retry of frame 40, and no second pass. Each frame is decoded, handed
|
||||
to the detector at its own time in the take, and its mouth crop read off the
|
||||
same canvas before the loop moves on."
|
||||
frame and no second pass. That used to be a constraint the frame walk had to be
|
||||
careful about; with a decoder it is simply what decoding is.
|
||||
|
||||
The work happens inside `decode!`'s callback, and the promise it returns is the
|
||||
backpressure: the decoder does not run ahead of the detector, so a 900-frame
|
||||
take does not hold 900 decoded frames at 1440x1920 in memory."
|
||||
[manifest model]
|
||||
(let [[w h] [(:width manifest) (:height manifest)]
|
||||
canvas (.createElement js/document "canvas")
|
||||
|
|
@ -32,51 +52,58 @@
|
|||
fps (:fps manifest)
|
||||
raw (atom [])
|
||||
crops (atom [])
|
||||
inner (atom [])
|
||||
total (:frames manifest)]
|
||||
(set! (.-width canvas) w)
|
||||
(set! (.-height canvas) h)
|
||||
(-> (ingest/video! (ingest/video-url manifest) fps w h)
|
||||
(rf/dispatch [::progress "loading the video…"])
|
||||
(-> (ingest/stream! (ingest/stream-url manifest) total)
|
||||
(.then
|
||||
(fn [video]
|
||||
(js/Promise.
|
||||
(fn [resolve reject]
|
||||
(letfn [(next-frame [i]
|
||||
(if (= i total)
|
||||
(try
|
||||
(resolve (assoc (detect/fill-gaps @raw)
|
||||
:dimensions [w h] :crops @crops))
|
||||
(catch :default error (reject error)))
|
||||
(-> (ingest/frame! video fps i)
|
||||
(.then
|
||||
(fn [_]
|
||||
(.drawImage ctx (:el video) 0 0)
|
||||
(let [face (detect/detect! model canvas
|
||||
(ingest/frame-ms fps i))
|
||||
ring (when face (mapv #(nth face %) lm/LIPS-INNER))
|
||||
box (when ring (interior/crop ring [w h]))
|
||||
pixels (when box
|
||||
(.-data (.getImageData ctx (:x box) (:y box)
|
||||
(:w box) (:h box))))]
|
||||
(swap! raw conj face)
|
||||
(swap! crops conj (when box {:box box :data pixels})))
|
||||
(when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total))
|
||||
(rf/dispatch [::progress (str "detecting " (inc i) "/" total)]))
|
||||
;; Let the status paint between synchronous
|
||||
;; MediaPipe calls.
|
||||
(js/setTimeout #(next-frame (inc i)) 0)))
|
||||
(.catch reject))))]
|
||||
(next-frame 0)))))))))
|
||||
(fn [stream]
|
||||
(ingest/decode!
|
||||
stream fps w h
|
||||
(fn [i frame]
|
||||
(.drawImage ctx frame 0 0)
|
||||
(let [face (detect/detect! model canvas (ingest/frame-ms fps i))
|
||||
ring (when face (mapv #(nth face %) lm/LIPS-INNER))
|
||||
box (when ring (interior/crop ring [w h]))
|
||||
pixels (when box
|
||||
(.-data (.getImageData ctx (:x box) (:y box)
|
||||
(:w box) (:h box))))]
|
||||
(swap! raw conj face)
|
||||
(let [crop (when box {:box box :data pixels})]
|
||||
(swap! crops conj crop)
|
||||
;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work
|
||||
;; either way, but done after the fact it is ten seconds of
|
||||
;; synchronous arithmetic with the main thread held and the
|
||||
;; frame counter frozen on its last value — which reads as the
|
||||
;; decoder hanging, and was diagnosed as that twice.
|
||||
(swap! inner conj (source/measure-crop take/knobs crop))))
|
||||
(when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total))
|
||||
(rf/dispatch [::progress (str "detecting " (inc i) "/" total)]))
|
||||
;; Yield, so the status and the transport paint between synchronous
|
||||
;; MediaPipe calls. `decode!` waits on this before feeding more.
|
||||
(js/Promise. (fn [done] (js/setTimeout done 0)))))))
|
||||
(.then (fn [_]
|
||||
(mark! (str "DECODE FINISHED — " (count @raw) " frames"))
|
||||
(let [gaps (detect/fill-gaps @raw)]
|
||||
(mark! "fill-gaps")
|
||||
(assoc gaps :dimensions [w h] :crops @crops
|
||||
:interior @inner)))))))
|
||||
|
||||
(defn- build-clip [manifest detector
|
||||
{:keys [dense detected dimensions crops missing first-real] :as source-inputs}]
|
||||
{:keys [dense detected dimensions interior missing first-real]
|
||||
:as source-inputs}]
|
||||
(let [[w h] dimensions
|
||||
interior (source/measure-crops take/knobs crops)
|
||||
_ (mark! "build-clip: start")
|
||||
frozen (take/footage manifest {:dense dense :detected detected
|
||||
:dimensions dimensions :interior interior
|
||||
:presence (:presence manifest)
|
||||
:detector detector})
|
||||
_ (mark! "build-clip: freeze")
|
||||
built (:clip frozen)
|
||||
source-blocks (source/pack (:id (:analysis built)) source-inputs)]
|
||||
source-blocks (source/pack (:id (:analysis built)) source-inputs)
|
||||
_ (mark! "build-clip: pack source blocks")]
|
||||
(assoc (select-keys built [:fps :width :height])
|
||||
:frames (clip/frames built)
|
||||
:display-fps (:fps built)
|
||||
|
|
@ -111,25 +138,50 @@
|
|||
(.catch (fn [error]
|
||||
(if (= 404 (:status (ex-data error))) nil (throw error)))))))
|
||||
|
||||
(defn- measure-cached! [track]
|
||||
;; Cached source blocks hold crop pixels, but no interior measurements. Spread
|
||||
;; their measurement over event-loop turns just like the fresh decode path.
|
||||
(let [crops (:crops track)
|
||||
total (count crops)
|
||||
interior (atom [])]
|
||||
(js/Promise.
|
||||
(fn [resolve reject]
|
||||
(letfn [(step [i]
|
||||
(if (= i total)
|
||||
(resolve (assoc track :interior @interior))
|
||||
(try
|
||||
(swap! interior conj (source/measure-crop take/knobs (nth crops i)))
|
||||
(when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total))
|
||||
(rf/dispatch [::progress
|
||||
(str "measuring " (inc i) "/" total)]))
|
||||
(js/setTimeout #(step (inc i)) 0)
|
||||
(catch :default error (reject error)))))]
|
||||
(step 0))))))
|
||||
|
||||
(rf/reg-fx
|
||||
::begin!
|
||||
(fn [footage-id]
|
||||
(-> (js/Promise.all #js [(ingest/manifest! footage-id) (ingest/detector!)])
|
||||
(.then (fn [[manifest detector]]
|
||||
(reset! clock (js/Date.now))
|
||||
(rf/dispatch [::progress "looking for saved analysis…"])
|
||||
(-> (cached-source! manifest detector)
|
||||
(.then (fn [track]
|
||||
(if track
|
||||
(do (rf/dispatch [::progress "reusing saved analysis…"])
|
||||
(build-clip manifest detector track))
|
||||
(-> (measure-cached! track)
|
||||
(.then (fn [measured]
|
||||
(build-clip manifest detector measured)))))
|
||||
(do (rf/dispatch [::progress "loading MediaPipe…"])
|
||||
(-> (detect/landmarker!)
|
||||
(.then (fn [model]
|
||||
(mark! "MediaPipe ready")
|
||||
(rf/dispatch [::progress "opening the video…"])
|
||||
(-> (detect-frames! manifest model)
|
||||
(.then (fn [fresh]
|
||||
(build-clip manifest detector fresh))))))))))))))
|
||||
(.then (fn [entry]
|
||||
(mark! "build-clip: done")
|
||||
(let [id (store/install! entry)]
|
||||
(rf/dispatch [::loaded id (:summary entry)]))))
|
||||
(.catch (fn [error]
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@
|
|||
[arthur.footage.store :as store]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.source :as source]
|
||||
[arthur.flow.ingest :as ingest]
|
||||
[arthur.fx.http :as http]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
|
|
@ -68,42 +67,28 @@
|
|||
(.then (fn [^js created] (.-id created))))))
|
||||
|
||||
(defn- opened-entry! [^js clip-json]
|
||||
(let [footage-id (.-footage clip-json)
|
||||
analysis-key (.-analysis clip-json)]
|
||||
(let [footage-id (.-footage clip-json)]
|
||||
(-> (js/Promise.all
|
||||
#js [(js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %))
|
||||
(array-seq (.-blocks clip-json)))))
|
||||
(if footage-id (ingest/manifest! footage-id) (js/Promise.resolve nil))
|
||||
(if analysis-key (http/GET (str "/api/analyses/" analysis-key))
|
||||
(js/Promise.resolve nil))])
|
||||
(.then (fn [[blocks manifest analysis]]
|
||||
(let [source-keys (when analysis (array-seq (.-source_blocks ^js analysis)))]
|
||||
(-> (js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %)) source-keys)))
|
||||
(.then (fn [source-responses]
|
||||
(let [cid (.-cid clip-json)
|
||||
loaded (project/load
|
||||
cid #js {:leaves (.-leaves clip-json)
|
||||
:blocks blocks})
|
||||
built (:clip loaded)
|
||||
inputs (when (seq source-keys)
|
||||
(when-not manifest
|
||||
(throw (ex-info "saved analysis has no footage"
|
||||
{:analysis analysis-key})))
|
||||
(source/unpack source-responses
|
||||
[(:width manifest) (:height manifest)]))]
|
||||
(merge (select-keys built [:fps :width :height])
|
||||
{:label (str (or (.-name clip-json) cid) " (saved)")
|
||||
:cid cid :frames (clip/frames built)
|
||||
:display-fps (:fps built)
|
||||
:clip built :store (:store loaded)
|
||||
:footage-id footage-id
|
||||
:source-inputs inputs
|
||||
:source-blocks (when inputs
|
||||
(source/pack analysis-key inputs))
|
||||
:audio (if manifest (:audio manifest)
|
||||
"/static/arthur/audio.wav")})))))))))))
|
||||
(if footage-id
|
||||
(http/GET (str "/api/footage/" footage-id))
|
||||
(js/Promise.resolve nil))])
|
||||
(.then (fn [[blocks ^js footage]]
|
||||
(let [cid (.-cid clip-json)
|
||||
loaded (project/load
|
||||
cid #js {:leaves (.-leaves clip-json)
|
||||
:blocks blocks})
|
||||
built (:clip loaded)]
|
||||
(merge (select-keys built [:fps :width :height])
|
||||
{:label (str (or (.-name clip-json) cid) " (saved)")
|
||||
:cid cid :frames (clip/frames built)
|
||||
:display-fps (:fps built)
|
||||
:clip built :store (:store loaded)
|
||||
:footage-id footage-id
|
||||
:audio (if footage (.-audio footage)
|
||||
"/static/arthur/audio.wav")})))))))
|
||||
|
||||
(rf/reg-fx
|
||||
::save!
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@
|
|||
|
||||
THE MEASURED PIXELS COME OUT OF A VIDEO NOW, not out of a PNG per frame. The old
|
||||
arrangement stored 112MB for a 7.6-second take and 1.1GB at the 900-frame limit;
|
||||
the same footage is a 6MB H.264 proxy the page steps through. What that costs is
|
||||
the property a PNG sequence gave for free — that asking for frame 12 gets frame
|
||||
12 — so `frame!` below buys it back explicitly, and refuses to guess."
|
||||
the same footage is a 6MB H.264 proxy. What that cost was the property a PNG
|
||||
sequence gave for free — that asking for frame 12 gets frame 12 — and `decode!`
|
||||
below is how it is bought back."
|
||||
(:require [arthur.fx.http :as http]
|
||||
[clojure.string :as str]))
|
||||
|
||||
|
|
@ -50,8 +50,8 @@
|
|||
;; it has a specific cause and a specific fix: this footage was extracted
|
||||
;; before the proxy existed, and its frames were stored as PNGs that the
|
||||
;; measurement path no longer reads.
|
||||
(when-not (and (string? (:video m)) (seq (:video m)))
|
||||
(throw (ex-info "this footage has no video to measure — re-extract it from its source"
|
||||
(when-not (and (string? (:stream m)) (seq (:stream m)))
|
||||
(throw (ex-info "this footage has no decodable stream — re-extract it from its source"
|
||||
{:footage (:id m)})))
|
||||
(when-not (= frames (count urls))
|
||||
;; The count is the manifest's and the URLs are the manifest's, so a
|
||||
|
|
@ -92,6 +92,9 @@
|
|||
(defn video-url [manifest]
|
||||
(:video manifest))
|
||||
|
||||
(defn stream-url [manifest]
|
||||
(:stream manifest))
|
||||
|
||||
(defn frame-url
|
||||
"The tracing still for one frame. A reference image for drawing over — the
|
||||
landmarks and the mouth crops come from the video, not from these."
|
||||
|
|
@ -108,219 +111,194 @@
|
|||
(set! (.-src image) src)))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; walking the proxy, one frame at a time
|
||||
|
||||
(defn seek-time
|
||||
"When to ask the video for source frame `i`: the MIDDLE of the frame, not its
|
||||
start.
|
||||
|
||||
A frame occupies the half-open interval `[i/fps, (i+1)/fps)`, so `i/fps` sits
|
||||
exactly on a boundary — and a boundary is where a seek lands on whichever side
|
||||
the container's timebase rounds to. Measured over 91 frames: seeking to `i/fps`
|
||||
produced the previous frame 31 times, and seeking to the middle was exact on all
|
||||
91. Half a frame of slack in both directions is the entire fix."
|
||||
[fps i]
|
||||
(/ (+ i 0.5) fps))
|
||||
|
||||
(defn presented-frame
|
||||
"Which source frame a `mediaTime` from requestVideoFrameCallback refers to.
|
||||
|
||||
`mediaTime` is the presented frame's own START — measured on 91 frames at 12fps
|
||||
it came back as an exact multiple of the frame duration — so this is the inverse
|
||||
of `i/fps` and NOT of `seek-time`. The two are asymmetric on purpose: we aim
|
||||
half a frame late because a seek target is fuzzy, and read back exactly because
|
||||
a presentation timestamp is not."
|
||||
[fps media-time]
|
||||
(js/Math.round (* media-time fps)))
|
||||
;; decoding the proxy
|
||||
;;
|
||||
;; WEBCODECS, NOT SEEKING, and the seeking is worth a paragraph because three
|
||||
;; separate failures came out of it. A `<video>` cannot be asked for frame 12: it
|
||||
;; can be asked for a TIME, and which frame that lands on is up to the engine.
|
||||
;; Measured on a video whose every frame carries its own index in its pixels,
|
||||
;; Firefox 156 returned the wrong frame for 4 of 40 seeks aimed at the middle of
|
||||
;; each frame, and 15 of 40 aimed at the start — and `requestVideoFrameCallback`,
|
||||
;; the thing that is supposed to say which frame arrived, reported a different
|
||||
;; frame from the one actually on screen 27 times out of 40. There is no way to
|
||||
;; verify a seek when the verification is the part that is wrong.
|
||||
;;
|
||||
;; So the page decodes instead. `VideoDecoder` takes coded chunks and returns
|
||||
;; exactly one frame per chunk, in order — measured 120 of 120 in order in both
|
||||
;; Firefox and Chrome. The server hands us the proxy's video as a raw Annex-B
|
||||
;; stream, which needs no demuxer: NAL start codes are findable in a loop. And
|
||||
;; because the proxy is encoded with no B-frames, decode order is presentation
|
||||
;; order, so ACCESS UNIT k IS FRAME k. No timestamps are interpreted, no clock is
|
||||
;; reconciled, and nothing here can be off by one.
|
||||
|
||||
(defn frame-ms
|
||||
"When source frame `i` happens, in milliseconds of footage.
|
||||
|
||||
What `flow/detect` hands MediaPipe as the frame's timestamp. Real elapsed time
|
||||
rather than the frame number, because the tracker reads the gap between
|
||||
timestamps as motion — see `detect/detect!`."
|
||||
timestamps as motion — see `detect/detect!` — and strictly increasing, which its
|
||||
input stream requires."
|
||||
[fps i]
|
||||
(/ (* i 1000) fps))
|
||||
|
||||
(defn access-units
|
||||
"An Annex-B H.264 stream -> one `{:from :to :key?}` per coded frame.
|
||||
|
||||
(def ^:private seek-timeout-ms
|
||||
"How long one frame may take to arrive before the run gives up.
|
||||
A NAL unit starts at a three- or four-byte start code, and an access unit is the
|
||||
parameter sets and SEI leading up to and including one coded slice. So: a new
|
||||
unit begins at the first non-slice NAL after a slice. Types 1 and 5 are the
|
||||
slice types — 5 is an IDR, which is what makes a chunk a keyframe.
|
||||
|
||||
Long, because the first seek of a take also opens the file and fills a buffer,
|
||||
and short enough that a video the browser cannot decode fails with a sentence
|
||||
instead of hanging with a spinner."
|
||||
10000)
|
||||
Twenty-five lines instead of a demuxer, and the reason it is only twenty-five is
|
||||
that the stream was produced for this: constant rate, no B-frames, one slice per
|
||||
frame."
|
||||
[^js bytes]
|
||||
(let [n (.-length bytes)
|
||||
starts (loop [i 0 out (transient [])]
|
||||
(if (>= i (- n 3))
|
||||
(persistent! out)
|
||||
(let [a (aget bytes i) b (aget bytes (inc i)) c (aget bytes (+ i 2))]
|
||||
(cond
|
||||
(and (zero? a) (zero? b) (= 1 c))
|
||||
(recur (inc i) (conj! out i))
|
||||
|
||||
(defn- await-frame!
|
||||
"One presentation, with its raw index. `nil` if none arrives in time."
|
||||
[^js video fps at]
|
||||
(and (zero? a) (zero? b) (zero? c) (= 1 (aget bytes (+ i 3))))
|
||||
(recur (+ i 2) (conj! out i))
|
||||
|
||||
:else (recur (inc i) out)))))]
|
||||
(loop [ks 0 current nil saw-slice? false out []]
|
||||
(if (= ks (count starts))
|
||||
(if current (conj out current) out)
|
||||
(let [start (nth starts ks)
|
||||
end (if (< (inc ks) (count starts)) (nth starts (inc ks)) n)
|
||||
header (aget bytes (+ start (if (= 1 (aget bytes (+ start 2))) 3 4)))
|
||||
kind (bit-and header 0x1f)
|
||||
slice? (or (= 1 kind) (= 5 kind))
|
||||
[out current saw-slice?] (if (and slice? saw-slice?)
|
||||
[(conj out current) nil false]
|
||||
[out current saw-slice?])
|
||||
current (or current {:from start :to end :key? false})]
|
||||
(recur (inc ks)
|
||||
(assoc current :to end :key? (or (:key? current) (= 5 kind)))
|
||||
(or saw-slice? slice?)
|
||||
out))))))
|
||||
|
||||
(defn stream!
|
||||
"Fetch the elementary stream and cut it into one chunk per frame."
|
||||
[src frames]
|
||||
(-> (js/fetch src)
|
||||
(.then (fn [^js response]
|
||||
(when-not (.-ok response)
|
||||
(throw (ex-info (str "the footage's video stream did not load: "
|
||||
(.-status response))
|
||||
{:src src})))
|
||||
(.arrayBuffer response)))
|
||||
(.then (fn [buffer]
|
||||
(let [units (access-units (js/Uint8Array. buffer))]
|
||||
(when-not (= (count units) frames)
|
||||
(throw (ex-info (str "the video stream holds " (count units)
|
||||
" coded frames and the manifest says " frames)
|
||||
{:units (count units) :frames frames})))
|
||||
{:bytes (js/Uint8Array. buffer) :units (vec units)})))))
|
||||
|
||||
(def ^:private decode-lookahead
|
||||
"How many chunks may be in the decoder at once.
|
||||
|
||||
Backpressure, and not a tuning knob to leave at infinity: a 900-frame take at
|
||||
1440x1920 is gigabytes of decoded frames, so feeding the whole stream in and
|
||||
letting the output callback keep up is how the tab dies. Small enough to bound
|
||||
that, and more than one so the decoder is never idle waiting on us."
|
||||
4)
|
||||
|
||||
(defn decode!
|
||||
"Decode every frame in order, calling `(on-frame i frame)` for each.
|
||||
|
||||
`on-frame` runs while the frame is alive and must not retain it — this closes it
|
||||
as soon as the call returns, because a `VideoFrame` holds decoder memory and the
|
||||
decoder stalls when it runs out. It may return a promise, which the walk waits
|
||||
for before feeding more; that is what lets a synchronous MediaPipe call and a
|
||||
repaint happen between frames without the decoder running ahead.
|
||||
|
||||
Resolves when the last frame has been handed over."
|
||||
[{:keys [bytes units]} fps width height on-frame]
|
||||
(js/Promise.
|
||||
(fn [resolve _reject]
|
||||
(let [settled (volatile! false)
|
||||
give (fn [v] (when-not @settled (vreset! settled true) (resolve v)))]
|
||||
(.requestVideoFrameCallback
|
||||
video (fn [_now metadata]
|
||||
(give (presented-frame fps (.-mediaTime metadata)))))
|
||||
(js/setTimeout #(give nil) seek-timeout-ms)
|
||||
(set! (.-currentTime video) at)))))
|
||||
|
||||
(defn calibrate!
|
||||
"What the browser CALLS the first frame it will show us.
|
||||
|
||||
Not always zero, and that is not the browser being wrong. A container can carry
|
||||
an edit list — ffmpeg writes one to absorb an encoder's reordering delay — and
|
||||
then `currentTime` counts from the start of the edited presentation while the
|
||||
`mediaTime` on a frame counts from the start of the media. The two differ by a
|
||||
constant, and the frame at `currentTime` 0 can honestly report a `mediaTime` of
|
||||
two frames in.
|
||||
|
||||
SO THE CONSTANT IS MEASURED ONCE AND SUBTRACTED, rather than corrected for by
|
||||
seeking. Seeking cannot fix it: when the offset is positive, source frame 0
|
||||
would have to be found BEFORE the start of the video, every attempt clamps at
|
||||
zero, and the walk reports `never presented frame 1; it offered 2` forever. The
|
||||
first frame the element presents IS frame 0 — it is what a viewer sees at time
|
||||
zero, and the audio clock this take plays against starts in the same place — so
|
||||
its own label is the origin everything else is counted from."
|
||||
[^js video fps]
|
||||
(-> (await-frame! video fps (seek-time fps 0))
|
||||
(.then (fn [base]
|
||||
(when (nil? base)
|
||||
(throw (ex-info "the video presented no frame at all; it cannot be walked"
|
||||
{})))
|
||||
base))))
|
||||
|
||||
(defn video!
|
||||
"Load the proxy as a decodable, seekable element, and find its origin.
|
||||
|
||||
`preload=auto` and nothing else: the element is never added to the document and
|
||||
never played. It is a decoder with a seek function, and the only reason it is a
|
||||
DOM element rather than a `VideoDecoder` is that a `VideoDecoder` needs the
|
||||
container demuxed before it can be handed a single frame, and this does not."
|
||||
[src fps width height]
|
||||
(-> (js/Promise.
|
||||
(fn [resolve reject]
|
||||
(let [video (.createElement js/document "video")]
|
||||
(set! (.-muted video) true)
|
||||
(set! (.-playsInline video) true)
|
||||
(set! (.-preload video) "auto")
|
||||
(set! (.-crossOrigin video) "anonymous")
|
||||
(set! (.-onerror video)
|
||||
(fn [_]
|
||||
(reject (ex-info (str "the browser could not decode this footage's video"
|
||||
(when-let [e (.-error video)]
|
||||
(str " (" (.-message e) ")")))
|
||||
{:src src}))))
|
||||
(set! (.-onloadeddata video)
|
||||
(fn [_]
|
||||
(cond
|
||||
(not (fn? (.-requestVideoFrameCallback video)))
|
||||
(reject (ex-info (str "this browser has no requestVideoFrameCallback, so "
|
||||
"which frame is on screen cannot be established")
|
||||
{}))
|
||||
|
||||
(not= [(.-videoWidth video) (.-videoHeight video)] [width height])
|
||||
(reject (ex-info "the video's size disagrees with the footage manifest"
|
||||
{:manifest [width height]
|
||||
:video [(.-videoWidth video) (.-videoHeight video)]}))
|
||||
|
||||
:else (resolve video))))
|
||||
(set! (.-src video) src))))
|
||||
(.then (fn [video]
|
||||
(-> (calibrate! video fps)
|
||||
;; Calibration left the element ON frame 0, so the walk starts
|
||||
;; already holding it. `current` is what is on screen now.
|
||||
(.then (fn [base] {:el video :base base :current (atom 0)})))))))
|
||||
|
||||
(def ^:private seek-attempts
|
||||
"How many times one frame may be asked for before the run gives up.
|
||||
|
||||
More than one because the browser is allowed to disagree with us about where a
|
||||
frame starts, and few because each attempt corrects by the exact size of the
|
||||
disagreement — so a constant offset is gone on the second try and anything still
|
||||
wrong on the sixth is not an offset."
|
||||
6)
|
||||
|
||||
(defn frame!
|
||||
"Seek to source frame `i` and resolve once the browser has PRESENTED it.
|
||||
|
||||
COUNTED FROM THE CALIBRATED ORIGIN. `base` is what the browser called the first
|
||||
frame it showed (see `calibrate!`), so the frame we want is the one whose raw
|
||||
index is `base + i`. Subtracting a measured constant is what makes a container
|
||||
with an edit list walk the same as one without, and it is the half that seeking
|
||||
cannot do: when the offset is positive, frame 0 lies before the start of the
|
||||
video and no amount of re-seeking will reach it.
|
||||
|
||||
IT STILL CORRECTS TOWARDS THE FRAME IT WANTS, for whatever the constant does not
|
||||
explain. A wrong frame is not merely an error to report, it is a MEASUREMENT of
|
||||
how far off the aim was, and the next attempt shifts by exactly that. Re-seeking
|
||||
rather than only re-listening is load-bearing: nothing further is ever presented
|
||||
to a paused video that has not been asked to move, so an earlier version that
|
||||
re-armed the callback without seeking again starved until its timeout.
|
||||
|
||||
It fails loudly rather than accepting a near miss. A one-frame slip between the
|
||||
landmarks and the audio is not something anyone finds by looking at the result,
|
||||
so exhausting the attempts ends the run and reports every frame that was offered
|
||||
and where it was asked from."
|
||||
[{:keys [^js el base current]} fps i]
|
||||
(if (= @current i)
|
||||
;; Already on screen. Seeking to where we already are presents NOTHING — a
|
||||
;; paused element with an unchanged frame fires no callback — so asking again
|
||||
;; would wait out the timeout. This is frame 0 straight after `calibrate!`,
|
||||
;; and it is the difference between a walk that starts and one that hangs.
|
||||
(js/Promise.resolve el)
|
||||
(js/Promise.
|
||||
(fn [resolve reject]
|
||||
(let [settled (volatile! false)
|
||||
offered (volatile! [])
|
||||
finish (fn [f] (when-not @settled (vreset! settled true) (f)))
|
||||
duration (or (.-duration el) 0)
|
||||
describe (fn []
|
||||
(if (seq @offered)
|
||||
(str/join ", "
|
||||
(map (fn [[idx at]]
|
||||
(str (inc idx) " (asked at " (.toFixed at 4) "s)"))
|
||||
@offered))
|
||||
"nothing at all"))
|
||||
timer (js/setTimeout
|
||||
#(finish
|
||||
(fn []
|
||||
(reject (ex-info (str "the video never presented frame " (inc i)
|
||||
"; it offered " (describe))
|
||||
{:frame i :base base :offered @offered}))))
|
||||
seek-timeout-ms)
|
||||
seek! (fn [at]
|
||||
;; A repeat of the current position is not a seek and presents
|
||||
;; nothing, so nudge within the frame rather than stall.
|
||||
(let [at (min (max at 0) (max 0 (- duration 1e-3)))
|
||||
at (if (= at (.-currentTime el)) (+ at (/ 0.25 fps)) at)]
|
||||
(set! (.-currentTime el) at)))]
|
||||
(letfn [(attempt [n at]
|
||||
(.requestVideoFrameCallback
|
||||
el
|
||||
(fn [_now metadata]
|
||||
(when-not @settled
|
||||
(let [presented (- (presented-frame fps (.-mediaTime metadata)) base)]
|
||||
(cond
|
||||
(= presented i)
|
||||
(do (js/clearTimeout timer)
|
||||
(reset! current i)
|
||||
(finish #(resolve el)))
|
||||
|
||||
(< n seek-attempts)
|
||||
(let [next-at (- at (/ (- presented i) fps))]
|
||||
(vswap! offered conj [presented at])
|
||||
(attempt (inc n) next-at)
|
||||
(seek! next-at))
|
||||
|
||||
:else
|
||||
(do (vswap! offered conj [presented at])
|
||||
(js/clearTimeout timer)
|
||||
(finish
|
||||
(fn []
|
||||
(reject (ex-info
|
||||
(str "the video kept presenting the wrong frame for "
|
||||
(inc i) "; it offered " (describe))
|
||||
{:frame i :base base :offered @offered})))))))))))]
|
||||
(let [at (seek-time fps i)]
|
||||
(attempt 1 at)
|
||||
(seek! at))))))))
|
||||
(if-not (exists? js/VideoDecoder)
|
||||
(reject (ex-info (str "this browser has no WebCodecs VideoDecoder, which is "
|
||||
"what reads footage a frame at a time")
|
||||
{}))
|
||||
(let [total (count units)
|
||||
next-in (volatile! 0)
|
||||
done-out (volatile! 0)
|
||||
failed (volatile! false)
|
||||
;; `taken` is claimed as a frame ARRIVES. `done-out` counts frames
|
||||
;; FINISHED, and is what backpressure and completion read. They are
|
||||
;; two different numbers whenever `on-frame` yields, which it does —
|
||||
;; reading `done-out` as the index let frames 0 and 1 both claim 0,
|
||||
;; call the detector twice at timestamp 0, and get the run killed by
|
||||
;; `Packet timestamp mismatch ... expected 1 but received 0`.
|
||||
taken (volatile! 0)
|
||||
;; And the work is CHAINED rather than started, because the detector
|
||||
;; is a tracker fed one frame at a time in order. Two overlapping
|
||||
;; `detect!` calls are not slow, they are wrong.
|
||||
chain (volatile! (js/Promise.resolve))
|
||||
decoder (volatile! nil)
|
||||
fail! (fn [error]
|
||||
(when-not @failed
|
||||
(vreset! failed true)
|
||||
(when-let [^js d @decoder]
|
||||
(try (when-not (= "closed" (.-state d)) (.close d))
|
||||
(catch :default _ nil)))
|
||||
(reject error)))]
|
||||
(letfn [(feed! []
|
||||
(while (and (not @failed)
|
||||
(< @next-in total)
|
||||
(< (- @next-in @done-out) decode-lookahead))
|
||||
(let [i @next-in
|
||||
{:keys [from to key?]} (nth units i)]
|
||||
(vreset! next-in (inc i))
|
||||
(.decode ^js @decoder
|
||||
(js/EncodedVideoChunk.
|
||||
#js {:type (if key? "key" "delta")
|
||||
:timestamp (js/Math.round (/ (* i 1e6) fps))
|
||||
:duration (js/Math.round (/ 1e6 fps))
|
||||
:data (.subarray bytes from to)})))))]
|
||||
(vreset!
|
||||
decoder
|
||||
(js/VideoDecoder.
|
||||
#js {:output
|
||||
(fn [^js frame]
|
||||
(if @failed
|
||||
(.close frame)
|
||||
(let [i @taken]
|
||||
(vreset! taken (inc i))
|
||||
(vreset!
|
||||
chain
|
||||
(.then
|
||||
@chain
|
||||
(fn []
|
||||
(if @failed
|
||||
(do (try (.close frame) (catch :default _ nil)) nil)
|
||||
(-> (js/Promise.resolve
|
||||
(try (on-frame i frame)
|
||||
(catch :default error (js/Promise.reject error))))
|
||||
(.then (fn [_]
|
||||
(try (.close frame) (catch :default _ nil))
|
||||
(vreset! done-out (inc i))
|
||||
(if (= (inc i) total)
|
||||
(do (.close ^js @decoder)
|
||||
(resolve total))
|
||||
(feed!))))
|
||||
(.catch (fn [error]
|
||||
(try (.close frame) (catch :default _ nil))
|
||||
(fail! error)
|
||||
nil))))))))))
|
||||
:error (fn [^js e]
|
||||
(fail! (ex-info (str "the browser could not decode this "
|
||||
"footage: " (.-message e))
|
||||
{})))}))
|
||||
(.configure ^js @decoder
|
||||
#js {:codec "avc1.640028"
|
||||
:codedWidth width :codedHeight height
|
||||
:optimizeForLatency true})
|
||||
(feed!)))))))
|
||||
|
|
|
|||
|
|
@ -9,12 +9,20 @@
|
|||
|
||||
(def roles ["source/dense" "source/detected" "source/crops"])
|
||||
|
||||
(defn measure-crop
|
||||
"One mouth crop's interior, or the empty measurement for a frame with no face.
|
||||
|
||||
Singular because the caller that matters measures a crop the moment it has one,
|
||||
while the rest of the frame's pixels are still on the canvas — see
|
||||
`events/footage`. Measuring all of them afterwards is the same arithmetic and
|
||||
ten seconds of a frozen page."
|
||||
[params crop]
|
||||
(if crop
|
||||
(interior/measure params (:box crop) #js {:data (:data crop)})
|
||||
{:contour nil :contrast 0 :area 0 :debug nil}))
|
||||
|
||||
(defn measure-crops [params crops]
|
||||
(mapv (fn [crop]
|
||||
(if crop
|
||||
(interior/measure params (:box crop) #js {:data (:data crop)})
|
||||
{:contour nil :contrast 0 :area 0 :debug nil}))
|
||||
crops))
|
||||
(mapv #(measure-crop params %) crops))
|
||||
|
||||
(defn- named [role analysis tracks layout data]
|
||||
(merge (address/block {:role role :analysis analysis :params {}
|
||||
|
|
|
|||
|
|
@ -15,50 +15,52 @@
|
|||
(is (thrown? ExceptionInfo (ingest/feature-presence 8 bad)))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; walking the proxy
|
||||
;; decoding the proxy
|
||||
;;
|
||||
;; These three are arithmetic, and all three were wrong in a way that no error
|
||||
;; reported. A seek that lands one frame early, a timestamp read back with the
|
||||
;; wrong inverse, or a frame time given as an index all produce a take that runs
|
||||
;; to completion and is quietly off — which is why the numbers are asserted here
|
||||
;; rather than left inline where they read as obvious.
|
||||
;; What used to be here was arithmetic mapping a browser clock to a frame number,
|
||||
;; and it is gone with the seeking that needed it. `access-units` is the only pure
|
||||
;; thing left in the decode path, and it is worth testing precisely because it is
|
||||
;; the step that makes "chunk k is frame k" true.
|
||||
|
||||
(deftest a-seek-aims-at-the-middle-of-its-frame
|
||||
;; THE BUG THIS ENCODES: seeking to `i/fps` sits exactly on the boundary between
|
||||
;; two frames, and over 91 frames of real footage it produced the PREVIOUS frame
|
||||
;; 31 times. Every seek must land strictly inside its own frame's interval, with
|
||||
;; half a frame of slack on each side.
|
||||
(doseq [fps [12 23.976 24 25 29.97 30 60]]
|
||||
(testing (str fps "fps")
|
||||
(doseq [i [0 1 2 41 90 899]]
|
||||
(let [t (ingest/seek-time fps i)]
|
||||
(is (< (/ i fps) t (/ (inc i) fps))
|
||||
(str "frame " i " at " fps "fps seeks to " t
|
||||
", outside [" (/ i fps) ", " (/ (inc i) fps) ")"))
|
||||
;; Within a rounding error of dead centre. Not `=`: `(/ i fps)` and
|
||||
;; `(/ (+ i 0.5) fps)` are two different divisions, so their difference
|
||||
;; carries the last bit of both and 0.4999999999999982 is a pass.
|
||||
(is (< (abs (- 0.5 (* fps (- t (/ i fps))))) 1e-9)
|
||||
"the aim point drifted off the middle of the frame"))))))
|
||||
(defn- annexb
|
||||
"A tiny Annex-B stream: `[[nal-type ...] ...]`, one vector per access unit."
|
||||
[units]
|
||||
(js/Uint8Array.
|
||||
(into-array (mapcat (fn [types]
|
||||
(mapcat (fn [t] [0 0 0 1 t 0x42]) types))
|
||||
units))))
|
||||
|
||||
(deftest a-presented-media-time-reads-back-as-its-own-frame
|
||||
;; The inverse of `i/fps` and NOT of `seek-time`: requestVideoFrameCallback
|
||||
;; reports the frame's START. Rounding the wrong one is a silent off-by-one
|
||||
;; between the landmarks and the audio.
|
||||
(doseq [fps [12 24 29.97 30]]
|
||||
(doseq [i [0 1 2 41 90]]
|
||||
(is (= i (ingest/presented-frame fps (/ i fps)))
|
||||
(str "frame " i " at " fps "fps did not read back as itself")))))
|
||||
(deftest access-units-cut-the-stream-at-coded-frames
|
||||
;; Types 1 and 5 are slices; 7, 8 and 6 are SPS, PPS and SEI, which belong to
|
||||
;; the frame that FOLLOWS them. Getting that boundary wrong would shift every
|
||||
;; parameter set onto the previous frame and make the first chunk undecodable.
|
||||
(let [bytes (annexb [[7 8 5] [1] [1] [7 8 5] [1]])
|
||||
units (ingest/access-units bytes)]
|
||||
(is (= 5 (count units)) "one access unit per coded slice")
|
||||
(is (= [true false false true false] (mapv :key? units))
|
||||
"an IDR slice makes its access unit a keyframe")
|
||||
(is (apply < (mapv :from units)) "units are in stream order")
|
||||
(is (= (mapv :to units)
|
||||
(mapv :from (concat (rest units) [{:from (.-length bytes)}])))
|
||||
"units tile the stream with no bytes dropped between them")))
|
||||
|
||||
(deftest a-frame-time-is-footage-milliseconds-and-not-a-frame-number
|
||||
(deftest a-stream-of-one-frame-is-one-unit
|
||||
(is (= 1 (count (ingest/access-units (annexb [[7 8 5]]))))))
|
||||
|
||||
(deftest three-byte-and-four-byte-start-codes-are-both-found
|
||||
;; Both are legal and ffmpeg emits both: four bytes before parameter sets and
|
||||
;; three between slices. Missing the short form merges two frames into one.
|
||||
(let [bytes (js/Uint8Array. (into-array [0 0 0 1 5 0x42 0 0 1 1 0x42 0 0 1 1 0x42]))]
|
||||
(is (= 3 (count (ingest/access-units bytes))))))
|
||||
|
||||
(deftest a-frame-time-is-footage-milliseconds-and-strictly-increasing
|
||||
;; MediaPipe's video mode is a tracker that reads the gap between timestamps as
|
||||
;; motion. Handing it `i` still runs, and measurably moved landmarks six times
|
||||
;; further from the per-frame answer. It must be real elapsed time.
|
||||
;; motion, and its input stream refuses one that does not advance — an error the
|
||||
;; landmarker never recovers from.
|
||||
(is (= 0.0 (ingest/frame-ms 12 0)))
|
||||
(is (= 1000.0 (ingest/frame-ms 12 12)))
|
||||
(is (= (/ 1000 30) (ingest/frame-ms 30 1)))
|
||||
(testing "strictly increasing, which the input stream requires"
|
||||
(doseq [fps [12 23.976 30 60 240]]
|
||||
(let [times (mapv #(ingest/frame-ms fps %) (range 200))]
|
||||
(is (every? pos? (map - (rest times) times))
|
||||
(str "two frames at " fps "fps shared a timestamp"))))))
|
||||
(doseq [fps [12 23.976 30 60 240]]
|
||||
(let [times (mapv #(ingest/frame-ms fps %) (range 200))]
|
||||
(is (every? pos? (map - (rest times) times))
|
||||
(str "two frames at " fps "fps shared a timestamp")))))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue