diff --git a/clips/extraction.py b/clips/extraction.py index ef012ae..2863aae 100644 --- a/clips/extraction.py +++ b/clips/extraction.py @@ -106,6 +106,18 @@ def _encode_proxy(job, source_path, proxy_path, facts, root): # step that makes the thing the page measures not be. "-fps_mode", "cfr", "-r", facts.get("rate") or str(facts["fps"]), "-c:v", "libx264", "-preset", "veryfast", "-crf", PROXY_CRF, + # NO B-FRAMES, AND THIS IS THE LOAD-BEARING FLAG. With them x264 has a + # two-frame reordering delay, ffmpeg compensates by writing an edit list + # (`elst` media_time 1024 at timebase 1/15360 — exactly two frames), and + # the browser then lives on two timelines at once: `currentTime` obeys the + # edit list and the `mediaTime` reported by requestVideoFrameCallback does + # not. Seek to frame 0 and the browser correctly hands back a frame whose + # mediaTime says 2. Software decoding hides it; hardware decoding does + # not, which is the worst possible way for it to be wrong. Without + # B-frames DTS equals PTS, no edit list is written, and the two timelines + # are the same one. It also makes decode order presentation order, should + # this ever be fed to a WebCodecs VideoDecoder. + "-bf", "0", # yuv420p and an even frame size are what makes this playable everywhere # rather than only in the browser that happened to be tested. "-pix_fmt", "yuv420p", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", @@ -183,6 +195,27 @@ def probe(path): "vfr": nominal != average} +def _refuse_a_shifted_timeline(path): + """The proxy must put frame `i` at `i / fps` on BOTH of the browser's clocks. + + Asserted rather than assumed, because the failure is silent and the symptom is + unrecognisable. An encoder delay makes ffmpeg write an edit list, `currentTime` + then obeys it while `requestVideoFrameCallback`'s `mediaTime` does not, and the + page's frame walk is uniformly off by the delay — on hardware decoding only. It + cost two wrong diagnoses to find, so it does not get to come back silently if + somebody changes an encoder flag. + """ + data = json.loads(_command(["ffprobe", "-v", "error", "-select_streams", "v:0", + "-show_streams", "-of", "json", str(path)])) + stream = data["streams"][0] + if int(stream.get("has_b_frames") or 0): + raise ValueError( + "the proxy was encoded with B-frames, whose reordering delay makes the " + "browser's seek clock and its frame-timestamp clock disagree") + if float(stream.get("start_time") or 0) != 0: + raise ValueError(f"the proxy starts at {stream['start_time']}s rather than 0") + + def count_frames(path): """How many frames a file really holds, counted rather than reported. @@ -255,6 +288,7 @@ def run(key): # Everything downstream describes the PROXY, not the upload. proxy_facts = probe(proxy_path) + _refuse_a_shifted_timeline(proxy_path) frames = count_frames(proxy_path) if not 1 <= frames <= 900: raise ValueError(f"the proxy holds {frames} frames; the limit is 1–900") diff --git a/frontend/src/arthur/events/footage.cljs b/frontend/src/arthur/events/footage.cljs index 3629a51..d8efcf8 100644 --- a/frontend/src/arthur/events/footage.cljs +++ b/frontend/src/arthur/events/footage.cljs @@ -35,7 +35,7 @@ total (:frames manifest)] (set! (.-width canvas) w) (set! (.-height canvas) h) - (-> (ingest/video! (ingest/video-url manifest) w h) + (-> (ingest/video! (ingest/video-url manifest) fps w h) (.then (fn [video] (js/Promise. @@ -49,7 +49,7 @@ (-> (ingest/frame! video fps i) (.then (fn [_] - (.drawImage ctx video 0 0) + (.drawImage ctx (:el video) 0 0) (let [face (detect/detect! model canvas (ingest/frame-ms fps i)) ring (when face (mapv #(nth face %) lm/LIPS-INNER)) diff --git a/frontend/src/arthur/flow/ingest.cljs b/frontend/src/arthur/flow/ingest.cljs index 8464215..bb607e9 100644 --- a/frontend/src/arthur/flow/ingest.cljs +++ b/frontend/src/arthur/flow/ingest.cljs @@ -110,51 +110,6 @@ ;; --------------------------------------------------------------------------- ;; walking the proxy, one frame at a time -(def ^:private seek-timeout-ms - "How long one frame may take to arrive before the run gives up. - - Long, because the first seek of a take also opens the file and fills a buffer, - and short enough that a video the browser cannot decode fails with a sentence - instead of hanging with a spinner." - 10000) - -(defn video! - "Load the proxy as a decodable, seekable element. - - `preload=auto` and nothing else: the element is never added to the document and - never played. It is a decoder with a seek function, and the only reason it is a - DOM element rather than a `VideoDecoder` is that a `VideoDecoder` needs the - container demuxed before it can be handed a single frame, and this does not." - [src width height] - (js/Promise. - (fn [resolve reject] - (let [video (.createElement js/document "video")] - (set! (.-muted video) true) - (set! (.-playsInline video) true) - (set! (.-preload video) "auto") - (set! (.-crossOrigin video) "anonymous") - (set! (.-onerror video) - (fn [_] - (reject (ex-info (str "the browser could not decode this footage's video" - (when-let [e (.-error video)] - (str " (" (.-message e) ")"))) - {:src src})))) - (set! (.-onloadeddata video) - (fn [_] - (cond - (not (fn? (.-requestVideoFrameCallback video))) - (reject (ex-info (str "this browser has no requestVideoFrameCallback, so " - "which frame is on screen cannot be established") - {})) - - (not= [(.-videoWidth video) (.-videoHeight video)] [width height]) - (reject (ex-info "the video's size disagrees with the footage manifest" - {:manifest [width height] - :video [(.-videoWidth video) (.-videoHeight video)]})) - - :else (resolve video)))) - (set! (.-src video) src))))) - (defn seek-time "When to ask the video for source frame `i`: the MIDDLE of the frame, not its start. @@ -187,45 +142,185 @@ [fps i] (/ (* i 1000) fps)) + +(def ^:private seek-timeout-ms + "How long one frame may take to arrive before the run gives up. + + Long, because the first seek of a take also opens the file and fills a buffer, + and short enough that a video the browser cannot decode fails with a sentence + instead of hanging with a spinner." + 10000) + +(defn- await-frame! + "One presentation, with its raw index. `nil` if none arrives in time." + [^js video fps at] + (js/Promise. + (fn [resolve _reject] + (let [settled (volatile! false) + give (fn [v] (when-not @settled (vreset! settled true) (resolve v)))] + (.requestVideoFrameCallback + video (fn [_now metadata] + (give (presented-frame fps (.-mediaTime metadata))))) + (js/setTimeout #(give nil) seek-timeout-ms) + (set! (.-currentTime video) at))))) + +(defn calibrate! + "What the browser CALLS the first frame it will show us. + + Not always zero, and that is not the browser being wrong. A container can carry + an edit list — ffmpeg writes one to absorb an encoder's reordering delay — and + then `currentTime` counts from the start of the edited presentation while the + `mediaTime` on a frame counts from the start of the media. The two differ by a + constant, and the frame at `currentTime` 0 can honestly report a `mediaTime` of + two frames in. + + SO THE CONSTANT IS MEASURED ONCE AND SUBTRACTED, rather than corrected for by + seeking. Seeking cannot fix it: when the offset is positive, source frame 0 + would have to be found BEFORE the start of the video, every attempt clamps at + zero, and the walk reports `never presented frame 1; it offered 2` forever. The + first frame the element presents IS frame 0 — it is what a viewer sees at time + zero, and the audio clock this take plays against starts in the same place — so + its own label is the origin everything else is counted from." + [^js video fps] + (-> (await-frame! video fps (seek-time fps 0)) + (.then (fn [base] + (when (nil? base) + (throw (ex-info "the video presented no frame at all; it cannot be walked" + {}))) + base)))) + +(defn video! + "Load the proxy as a decodable, seekable element, and find its origin. + + `preload=auto` and nothing else: the element is never added to the document and + never played. It is a decoder with a seek function, and the only reason it is a + DOM element rather than a `VideoDecoder` is that a `VideoDecoder` needs the + container demuxed before it can be handed a single frame, and this does not." + [src fps width height] + (-> (js/Promise. + (fn [resolve reject] + (let [video (.createElement js/document "video")] + (set! (.-muted video) true) + (set! (.-playsInline video) true) + (set! (.-preload video) "auto") + (set! (.-crossOrigin video) "anonymous") + (set! (.-onerror video) + (fn [_] + (reject (ex-info (str "the browser could not decode this footage's video" + (when-let [e (.-error video)] + (str " (" (.-message e) ")"))) + {:src src})))) + (set! (.-onloadeddata video) + (fn [_] + (cond + (not (fn? (.-requestVideoFrameCallback video))) + (reject (ex-info (str "this browser has no requestVideoFrameCallback, so " + "which frame is on screen cannot be established") + {})) + + (not= [(.-videoWidth video) (.-videoHeight video)] [width height]) + (reject (ex-info "the video's size disagrees with the footage manifest" + {:manifest [width height] + :video [(.-videoWidth video) (.-videoHeight video)]})) + + :else (resolve video)))) + (set! (.-src video) src)))) + (.then (fn [video] + (-> (calibrate! video fps) + ;; Calibration left the element ON frame 0, so the walk starts + ;; already holding it. `current` is what is on screen now. + (.then (fn [base] {:el video :base base :current (atom 0)}))))))) + +(def ^:private seek-attempts + "How many times one frame may be asked for before the run gives up. + + More than one because the browser is allowed to disagree with us about where a + frame starts, and few because each attempt corrects by the exact size of the + disagreement — so a constant offset is gone on the second try and anything still + wrong on the sixth is not an offset." + 6) + (defn frame! "Seek to source frame `i` and resolve once the browser has PRESENTED it. - IT WAITS FOR THE FRAME IT ASKED FOR, rather than trusting the first callback. - `requestVideoFrameCallback` is a queue of presentations, not an answer to our - seek: a frame presented while the file was still opening, or the tail of the - previous seek, arrives on the next callback we happen to have registered. Taking - it at face value is what produced `asked the video for frame 1 and it presented - frame 2` on a video whose seeks were in fact exact. So a callback whose - `mediaTime` is not this frame's is DISCARDED and the wait re-armed — the - browser states which frame it handed over, and that is the only frame we let - through. + COUNTED FROM THE CALIBRATED ORIGIN. `base` is what the browser called the first + frame it showed (see `calibrate!`), so the frame we want is the one whose raw + index is `base + i`. Subtracting a measured constant is what makes a container + with an edit list walk the same as one without, and it is the half that seeking + cannot do: when the offset is positive, frame 0 lies before the start of the + video and no amount of re-seeking will reach it. - It still fails loudly. A silent one-frame slip between the landmarks and the - audio is not something anyone finds by looking at the result, so a frame that - never arrives inside `seek-timeout-ms` ends the run and says which one." - [^js video fps i] - (js/Promise. + IT STILL CORRECTS TOWARDS THE FRAME IT WANTS, for whatever the constant does not + explain. A wrong frame is not merely an error to report, it is a MEASUREMENT of + how far off the aim was, and the next attempt shifts by exactly that. Re-seeking + rather than only re-listening is load-bearing: nothing further is ever presented + to a paused video that has not been asked to move, so an earlier version that + re-armed the callback without seeking again starved until its timeout. + + It fails loudly rather than accepting a near miss. A one-frame slip between the + landmarks and the audio is not something anyone finds by looking at the result, + so exhausting the attempts ends the run and reports every frame that was offered + and where it was asked from." + [{:keys [^js el base current]} fps i] + (if (= @current i) + ;; Already on screen. Seeking to where we already are presents NOTHING — a + ;; paused element with an unchanged frame fires no callback — so asking again + ;; would wait out the timeout. This is frame 0 straight after `calibrate!`, + ;; and it is the difference between a walk that starts and one that hangs. + (js/Promise.resolve el) + (js/Promise. (fn [resolve reject] (let [settled (volatile! false) - seen (volatile! []) + offered (volatile! []) finish (fn [f] (when-not @settled (vreset! settled true) (f))) + duration (or (.-duration el) 0) + describe (fn [] + (if (seq @offered) + (str/join ", " + (map (fn [[idx at]] + (str (inc idx) " (asked at " (.toFixed at 4) "s)")) + @offered)) + "nothing at all")) timer (js/setTimeout #(finish (fn [] (reject (ex-info (str "the video never presented frame " (inc i) - (when (seq @seen) - (str "; it offered " - (str/join ", " (map inc @seen))))) - {:frame i :offered @seen})))) - seek-timeout-ms)] - (letfn [(listen [] + "; it offered " (describe)) + {:frame i :base base :offered @offered})))) + seek-timeout-ms) + seek! (fn [at] + ;; A repeat of the current position is not a seek and presents + ;; nothing, so nudge within the frame rather than stall. + (let [at (min (max at 0) (max 0 (- duration 1e-3))) + at (if (= at (.-currentTime el)) (+ at (/ 0.25 fps)) at)] + (set! (.-currentTime el) at)))] + (letfn [(attempt [n at] (.requestVideoFrameCallback - video + el (fn [_now metadata] (when-not @settled - (let [presented (presented-frame fps (.-mediaTime metadata))] - (if (= presented i) - (do (js/clearTimeout timer) (finish #(resolve video))) - (do (vswap! seen conj presented) (listen))))))))] - (listen)) - (set! (.-currentTime video) (seek-time fps i)))))) + (let [presented (- (presented-frame fps (.-mediaTime metadata)) base)] + (cond + (= presented i) + (do (js/clearTimeout timer) + (reset! current i) + (finish #(resolve el))) + + (< n seek-attempts) + (let [next-at (- at (/ (- presented i) fps))] + (vswap! offered conj [presented at]) + (attempt (inc n) next-at) + (seek! next-at)) + + :else + (do (vswap! offered conj [presented at]) + (js/clearTimeout timer) + (finish + (fn [] + (reject (ex-info + (str "the video kept presenting the wrong frame for " + (inc i) "; it offered " (describe)) + {:frame i :base base :offered @offered})))))))))))] + (let [at (seek-time fps i)] + (attempt 1 at) + (seek! at))))))))