Play the audio mixdown instead of encoding a WAV of it
Opening the 8625 study froze the main thread for 4.7 seconds and settled at 615MB of heap. A profile put three quarters of a project open inside mix/wav-bytes, which playback had no business calling at all. Two separate causes. The peak scan built a lazy sequence of one boxed double per SAMPLE -- ten million of them for a seven-minute mix -- to compute a single maximum over data already sitting in Float32Arrays; the hand-written loop is 73x faster and agrees to the bit. The rest was structural: the WAV existed only because an <audio> element can hold a URL and nothing else, and the element existed only to be the clock. So a mixdown that was already rendered got encoded to 73MB of 16-bit PCM, on the main thread, on open, on every tab switch and on every edit to a track -- and a symbol with no sound got silence synthesized and encoded full length so the element had a duration to report. arthur.clock keeps its interface and all of its arithmetic; the position now comes from a backend behind a protocol. clock.graph plays the AudioBuffer through an AudioBufferSourceNode and derives the frame from the context's own clock, which is the audio device's position in double precision rather than whatever the media pipeline last published. clock.element is the old path, kept switchable while the new one earns trust -- BACKEND, or use-backend! -- which is also why every one of the original clock tests passes unchanged: the derivation they assert is shared, and the backends can only disagree about the position under it. 6.5s to 1.8s, 4.7s of blocking to 370ms, 615MB to 68MB. THE POSITION IS COMPENSATED FOR OUTPUT LATENCY, and piecewise because of it. currentTime is the quantum being rendered, which the speaker is tens of milliseconds behind; report the renderer and the picture leads the sound, which in a lip-sync tool is the only artefact that matters. Audio already rendered cannot be re-rated, though, so reading it back at a new rate jumped the playhead backwards by three latencies on every press of the rate button. Each play, pause, seek and rate change now records a segment and a position is read against whichever segment was in force when that audio was rendered. One duplicate fell out of this. Opening a project asked for its clock twice -- once from ::opened and once from a ::refresh-clock the shell raised because it compared symbol ids, and two different documents both open on :main. The sounds subscription carries the clip id now, so "an edit under the same symbol" means what it says. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
90b1fbe2f8
commit
5b5b9ae4c3
10 changed files with 846 additions and 98 deletions
254
frontend/test/arthur/clock/graph_test.cljs
Normal file
254
frontend/test/arthur/clock/graph_test.cljs
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
(ns arthur.clock.graph-test
|
||||
"The graph clock, asserted against a hand-moved context clock.
|
||||
|
||||
The context's `currentTime` is set by these tests rather than advanced by a
|
||||
device, which is exactly the coupling worth asserting: if any of this counted
|
||||
time instead of deriving it, moving the clock by hand would not move the
|
||||
position. A drift bug is invisible for the first second and unmistakable by
|
||||
the tenth, which is the worst possible shape for a bug to have."
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[arthur.clock.element :as element]
|
||||
[arthur.clock.graph :as graph]
|
||||
[arthur.clock.transport :as t]))
|
||||
|
||||
(defn- fake-node [log]
|
||||
(let [n #js {:loop false :playbackRate #js {:value 1.0}}]
|
||||
(set! (.-connect n) (fn [_] nil))
|
||||
(set! (.-disconnect n) (fn [] nil))
|
||||
(set! (.-start n) (fn [_when offset] (swap! log conj [:start offset]) nil))
|
||||
(set! (.-stop n) (fn [] (swap! log conj [:stop]) nil))
|
||||
n))
|
||||
|
||||
(defn- fake-ctx
|
||||
"Stands in for an `AudioContext`. `currentTime` is a plain writable property
|
||||
so a test can move it; the rest is the three methods the backend calls."
|
||||
[latency log]
|
||||
(let [ctx #js {:currentTime 0.0 :outputLatency latency :baseLatency 0}]
|
||||
(set! (.-resume ctx) (fn [] nil))
|
||||
(set! (.-createBufferSource ctx) (fn [] (fake-node log)))
|
||||
ctx))
|
||||
|
||||
(defn- rig
|
||||
"A backend, its context, and the log of what it did to its source nodes."
|
||||
[& {:keys [seconds latency buffer?] :or {seconds 10.0 latency 0 buffer? true}}]
|
||||
(let [log (atom [])
|
||||
ctx (fake-ctx latency log)
|
||||
out #js {:gain #js {:value 1}}]
|
||||
{:ctx ctx :log log :out out
|
||||
:g (graph/backend ctx out (when buffer? #js {}) seconds)}))
|
||||
|
||||
(defn- at! [{:keys [^js ctx]} t] (set! (.-currentTime ctx) t))
|
||||
|
||||
(defn- close?
|
||||
"Within a frame's worth of a microsecond. Positions compound a multiply and a
|
||||
subtract, so the last bit of a double is not something to assert on."
|
||||
[a b]
|
||||
(< (js/Math.abs (- a b)) 1e-9))
|
||||
|
||||
(deftest the-position-is-derived-from-the-context-clock-not-counted
|
||||
;; Nothing here is ticked. The context moves and the position follows, which
|
||||
;; is the whole property: a stalled paint loop cannot lose the clock, because
|
||||
;; the clock was never in the paint loop.
|
||||
(let [{:keys [g] :as r} (rig)]
|
||||
(t/-play! g)
|
||||
(doseq [[now want] [[0.0 0.0] [0.5 0.5] [2.0 2.0] [9.25 9.25]]]
|
||||
(at! r now)
|
||||
(is (= want (t/-position g)) (str "context at " now "s")))))
|
||||
|
||||
(deftest a-stall-lands-where-the-audio-already-is
|
||||
;; THE property the derivation buys, same as the element's. A loop that
|
||||
;; stalled for a third of a second resumes at the position the audio reached,
|
||||
;; not a third of a second behind it.
|
||||
(let [{:keys [g] :as r} (rig)]
|
||||
(t/-play! g)
|
||||
(at! r 1.0)
|
||||
(is (= 1.0 (t/-position g)))
|
||||
(at! r 1.5) ; fifteen frames' worth of stall
|
||||
(is (= 1.5 (t/-position g)) "and nothing was counted in between")))
|
||||
|
||||
(deftest the-position-is-the-speaker-not-the-renderer
|
||||
;; `currentTime` is the quantum being RENDERED; the speaker is one output
|
||||
;; latency behind it. Report the renderer and the picture leads the sound by
|
||||
;; that much, which in a lip-sync tool is the only artefact that matters.
|
||||
(let [{:keys [g] :as r} (rig :latency 0.05)]
|
||||
(t/-play! g)
|
||||
(at! r 2.0)
|
||||
(is (= 1.95 (t/-position g)) "50ms of output latency, subtracted")
|
||||
(testing "and scaled by the rate, because a latency is wall time"
|
||||
(t/-set-rate! g 4.0)
|
||||
(at! r 3.0)
|
||||
;; Banked at 1.95 + 0.05 = 2.0, then four seconds of buffer per second of
|
||||
;; wall clock, less the 4 x 50ms still in the output buffer.
|
||||
(is (close? 5.8 (t/-position g))))))
|
||||
|
||||
(deftest a-rate-change-does-not-lurch-the-playhead
|
||||
;; THE bug the segments exist for. Audio already rendered cannot be re-rated:
|
||||
;; read the samples still travelling to the speaker at the NEW rate and the
|
||||
;; position jumps backwards by three output latencies the instant 1x becomes
|
||||
;; 4x — fourteen frames at 30fps with a laptop's 150ms, which is a visible
|
||||
;; lurch on every press of the rate button.
|
||||
(let [{:keys [g] :as r} (rig :seconds 100.0 :latency 0.15)]
|
||||
(t/-play! g)
|
||||
(at! r 2.0)
|
||||
(let [before (t/-position g)]
|
||||
(t/-set-rate! g 4.0)
|
||||
(is (close? before (t/-position g))
|
||||
"the same instant, read either side of the change")
|
||||
(testing "and it goes FORWARD from there, at the new rate"
|
||||
(at! r 2.5)
|
||||
(is (> (t/-position g) before))
|
||||
(at! r 3.0)
|
||||
;; At 3.0s of context the speaker is hearing what was rendered at
|
||||
;; 2.85s. The rate changed when the renderer was at buffer position
|
||||
;; 2.0, which is 0.85s of wall clock earlier — so 4 x 0.85 of buffer
|
||||
;; since, and nothing before that boundary re-rated.
|
||||
(is (close? (+ 2.0 (* 4 0.85)) (t/-position g)))))
|
||||
(testing "and back to 1x is continuous too"
|
||||
(let [before (t/-position g)]
|
||||
(t/-set-rate! g 1.0)
|
||||
(is (close? before (t/-position g)))))))
|
||||
|
||||
(deftest segments-do-not-pile-up
|
||||
;; One per rate change, pruned to the in-flight window. Unbounded history
|
||||
;; would grow for the life of the document.
|
||||
(let [{:keys [g] :as r} (rig :seconds 1000.0 :latency 0.05)]
|
||||
(t/-play! g)
|
||||
(dotimes [i 200]
|
||||
(at! r (+ 1.0 i))
|
||||
(t/-set-rate! g (if (even? i) 4.0 1.0)))
|
||||
(at! r 500.0)
|
||||
(is (number? (t/-position g)))
|
||||
(is (<= (count (:segs @(.-state g))) 3)
|
||||
"the one in force, plus whatever is still in flight")))
|
||||
|
||||
(deftest the-first-samples-are-not-the-last-ones
|
||||
;; Right after a looped play the audible position is a few milliseconds
|
||||
;; NEGATIVE — the first samples are still in the output buffer. Wrapping
|
||||
;; before clamping would read that as the very end of the take and jump the
|
||||
;; playhead there on every play.
|
||||
(let [{:keys [g] :as r} (rig :seconds 2.0 :latency 0.05)]
|
||||
(t/-set-loop! g true)
|
||||
(t/-play! g)
|
||||
(at! r 0.0)
|
||||
(is (= 0 (t/-position g)))))
|
||||
|
||||
(deftest a-rate-change-is-banked-so-nothing-accumulates
|
||||
;; 1x -> 4x -> 1x across a take. The anchor moves to where the graph had got
|
||||
;; to and the new rate applies from there, so switching rates mid-take cannot
|
||||
;; leave an offset behind — which is what a multiplier on a counted frame
|
||||
;; would do.
|
||||
(let [{:keys [g] :as r} (rig :seconds 100.0)]
|
||||
(t/-play! g)
|
||||
(at! r 2.0)
|
||||
(is (= 2.0 (t/-position g)))
|
||||
(t/-set-rate! g 4.0)
|
||||
(at! r 3.0)
|
||||
(is (= 6.0 (t/-position g)) "one second of wall clock, four of buffer")
|
||||
(t/-set-rate! g 1.0)
|
||||
(at! r 4.0)
|
||||
(is (= 7.0 (t/-position g)) "and back to one for one, from six")
|
||||
(is (= 1.0 (t/-rate g)))))
|
||||
|
||||
(deftest a-seek-round-trips
|
||||
;; Reading back exactly what was seeked to is what makes a scrub idempotent.
|
||||
(let [{:keys [g]} (rig)]
|
||||
(doseq [t [0.0 0.5 3.25 9.0]]
|
||||
(t/-seek! g t)
|
||||
(is (= t (t/-position g)) (str "seek to " t "s")))))
|
||||
|
||||
(deftest a-seek-while-playing-restarts-the-source-at-the-new-offset
|
||||
;; Source nodes are single-use, so a seek under a playing take has to stop the
|
||||
;; old one and start a new one at the offset. Checked through the log because
|
||||
;; an offset that silently stayed at zero would still read back correctly from
|
||||
;; the arithmetic while playing the wrong audio.
|
||||
(let [{:keys [g log] :as r} (rig)]
|
||||
(t/-play! g)
|
||||
(at! r 1.0)
|
||||
(reset! log [])
|
||||
(t/-seek! g 4.0)
|
||||
(is (= [[:stop] [:start 4.0]] @log))
|
||||
(is (= 4.0 (t/-position g)))))
|
||||
|
||||
(deftest looping-wraps-rather-than-stopping
|
||||
(let [{:keys [g] :as r} (rig :seconds 2.0)]
|
||||
(t/-set-loop! g true)
|
||||
(t/-play! g)
|
||||
(at! r 5.0)
|
||||
(is (= 1.0 (t/-position g)) "five seconds into a two-second loop")
|
||||
(is (t/-playing? g) "and a loop never ends")))
|
||||
|
||||
(deftest running-off-the-end-stops-being-playing
|
||||
;; The end of the sound is the authority on playback having stopped — nothing
|
||||
;; counts frames to notice it. `ui/player`'s loop reads this to correct the
|
||||
;; document.
|
||||
(let [{:keys [g] :as r} (rig :seconds 2.0)]
|
||||
(t/-play! g)
|
||||
(at! r 1.9)
|
||||
(is (t/-playing? g))
|
||||
(at! r 2.5)
|
||||
(is (not (t/-playing? g)))
|
||||
(is (= 2.0 (t/-position g)) "and the position stops at the end, not past it")
|
||||
(testing "and playing again starts over, as the element does"
|
||||
(t/-play! g)
|
||||
(is (= 0.0 (t/-position g))))))
|
||||
|
||||
(deftest a-symbol-with-no-sound-still-keeps-time
|
||||
;; NO BUFFER AT ALL. The element backend had to synthesize silence and encode
|
||||
;; it to a WAV, full length, so that it had a duration to report; here the
|
||||
;; length is simply told to the clock and the context does the rest.
|
||||
(let [{:keys [g log] :as r} (rig :buffer? false :seconds 3.0)]
|
||||
(t/-play! g)
|
||||
(is (= [] @log) "and no source node was made for silence")
|
||||
(at! r 1.0)
|
||||
(is (= 1.0 (t/-position g)))
|
||||
(is (t/-playing? g))
|
||||
(is (= 3.0 (t/-duration g)))
|
||||
(at! r 3.5)
|
||||
(is (not (t/-playing? g)) "and silence ends when the symbol does")))
|
||||
|
||||
(deftest pause-anchors-at-what-was-heard
|
||||
;; Resuming replays the last few milliseconds rather than skipping them, which
|
||||
;; is the kinder of the two roundings.
|
||||
(let [{:keys [g] :as r} (rig :latency 0.05)]
|
||||
(t/-play! g)
|
||||
(at! r 2.0)
|
||||
(t/-pause! g)
|
||||
(is (= 1.95 (t/-position g)))
|
||||
(is (not (t/-playing? g)))
|
||||
(testing "and the position holds while paused, however the context moves"
|
||||
(at! r 8.0)
|
||||
(is (= 1.95 (t/-position g))))))
|
||||
|
||||
(deftest releasing-stops-the-source
|
||||
;; A backend dropped while playing would go on being heard under the one that
|
||||
;; replaced it: the source is wired to the output the whole app shares.
|
||||
(let [{:keys [g log] :as r} (rig)]
|
||||
(t/-play! g)
|
||||
(at! r 1.0)
|
||||
(reset! log [])
|
||||
(t/-release! g)
|
||||
(is (= [[:stop]] @log))
|
||||
(is (not (t/-playing? g)))))
|
||||
|
||||
(deftest mute-is-the-shared-output-and-not-the-buffer
|
||||
(let [{:keys [g ^js out]} (rig)]
|
||||
(t/-set-muted! g true)
|
||||
(is (= 0 (.-value (.-gain out))))
|
||||
(t/-set-muted! g false)
|
||||
(is (= 1 (.-value (.-gain out))))))
|
||||
|
||||
(deftest the-two-backends-report-the-same-position
|
||||
;; The point of keeping both switchable. Everything BELOW the position — the
|
||||
;; frame derivation, the clamp, the exposure grid — is in `arthur.clock` and
|
||||
;; shared, so the two backends can only disagree about the position itself. At
|
||||
;; zero output latency they must not; `clock-test` asserts what is built on
|
||||
;; top of it, once, for both.
|
||||
(let [{:keys [g] :as r} (rig :seconds 7.633)]
|
||||
(t/-play! g)
|
||||
(doseq [now [0.0 0.034 1.0 1.999 2.0 7.0]]
|
||||
(at! r now)
|
||||
(let [e (element/backend #js {:currentTime now :paused false :ended false
|
||||
:playbackRate 1.0 :duration 7.633})]
|
||||
(is (= (t/-position e) (t/-position g)) (str "at " now "s"))
|
||||
(is (= (t/-duration e) (t/-duration g)))
|
||||
(is (= (t/-playing? e) (t/-playing? g)))))))
|
||||
Loading…
Add table
Add a link
Reference in a new issue