From 8a068358950a3af1e917e4c06888182a8a88cd88 Mon Sep 17 00:00:00 2001 From: Olive Vaughn Date: Sun, 27 Sep 2026 19:01:39 -0400 Subject: [PATCH] Port step 5: freeze measured mouth into playable channels --- .gitignore | 2 + frontend/README.md | 108 ++-- frontend/package.json | 3 +- frontend/src/arthur/db.cljs | 48 +- frontend/src/arthur/demo.cljs | 3 - frontend/src/arthur/demo/scene.edn | 6 + frontend/src/arthur/demo/swarm.cljs | 2 + frontend/src/arthur/demo/take.cljs | 112 +++++ frontend/src/arthur/domain/channel.cljs | 94 +++- frontend/src/arthur/domain/scene.cljs | 26 +- frontend/src/arthur/events/playback.cljs | 7 +- frontend/src/arthur/flow/freeze.cljs | 440 +++++++++++++++++ frontend/src/arthur/flow/measure/anchor.cljs | 5 +- frontend/src/arthur/subs/playback.cljs | 5 + frontend/{test => src}/arthur/synth.cljs | 16 +- frontend/src/arthur/ui/player.cljs | 11 +- frontend/src/arthur/ui/shell.cljs | 19 +- frontend/test/arthur/domain/channel_test.cljs | 58 +++ frontend/test/arthur/domain/scene_test.cljs | 8 +- frontend/test/arthur/flow/freeze_test.cljs | 463 ++++++++++++++++++ frontend/test/arthur/parity_test.cljs | 245 --------- frontend/test/browser/take.mjs | 300 ++++++++++++ frontend/test/parity/.gitignore | 4 - frontend/test/parity/oracle.mjs | 146 ------ 24 files changed, 1625 insertions(+), 506 deletions(-) create mode 100644 frontend/src/arthur/demo/take.cljs create mode 100644 frontend/src/arthur/flow/freeze.cljs rename frontend/{test => src}/arthur/synth.cljs (92%) create mode 100644 frontend/test/arthur/flow/freeze_test.cljs delete mode 100644 frontend/test/arthur/parity_test.cljs create mode 100644 frontend/test/browser/take.mjs delete mode 100644 frontend/test/parity/.gitignore delete mode 100644 frontend/test/parity/oracle.mjs diff --git a/.gitignore b/.gitignore index f2a6ea3..f5fd260 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ frames/ # CLJS build frontend/node_modules/ +# screenshots from the browser suite; regenerated by `npm run browser` +frontend/test/browser/out/ frontend/.shadow-cljs/ frontend/out/ frontend/.cpcache/ diff --git a/frontend/README.md b/frontend/README.md index fc9677d..11652cb 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -17,85 +17,107 @@ shadow-cljs bug and is not one. `mise install` is what prevents it. ## The tests ```sh -cd frontend && npm test +cd frontend && mise exec -- npm test ``` -That is three things in order: regenerate the JS oracle's answers, compile the -`:test` build, run it under node. +Two things: compile the `:test` build, run it under node. ``` -node test/parity/oracle.mjs # drives js/ and writes test/parity/oracle.json shadow-cljs compile test node out/node-tests.js ``` -Run them separately if a compile error is in the way. The oracle JSON is -generated, not committed. +Run them separately if a compile error is in the way. -**Run them through `mise`**, or make sure `mise`'s node is first on PATH. The -oracle imports `js/*.js` directly, and those are ES modules in a directory with -no `package.json`, so node needs the module detection that became default in -20.19. On an older node 20 — an nvm install shadowing the pinned one is the easy -way to get there — every import fails with "Named export not found ... is a -CommonJS module", which reads like the oracle being broken and is not. +**Run them through `mise`**, or make sure `mise`'s node is first on PATH. `java` +must be 21+ and node 20.19+. On an nvm node 20.11 shadowing the pinned one, +things fail in ways that read like the code being broken and are not. + +### And the browser one + +Step 5's done-criterion is a PICTURE, and no assertion in `cljs.test` can check +one: a take that resolves to the right numbers and draws nothing would pass every +test in `arthur.flow.freeze-test`. A blank canvas under a perfectly correct +transport is the bug class unit tests miss, and it has happened here once. + +So there is a second suite that drives a real Chrome over CDP. It needs the dev +server up: + +```sh +cd frontend && mise exec -- npx shadow-cljs watch app # in one shell +cd frontend && mise exec -- npm run browser # in another +``` + +No dependencies. Playwright is not installed and CDP needs none — +`node --experimental-websocket` has a global `WebSocket` and +`--headless=new --remote-debugging-port=N` is the whole of the other side. It +reads the canvas's own pixels rather than a screenshot, because the CSS scales +the stage up by 2 and a screenshot is four pixels per raster pixel; it writes +PNGs into `test/browser/out/` anyway, so "it drew something" can be checked by +eye as well as by count. ## The app ```sh -cd frontend && npx shadow-cljs watch app +cd frontend && mise exec -- npx shadow-cljs watch app ``` Then open **** — with the `/index.html`, not bare `/`. This shadow-cljs does no directory-index resolution, so `/` is a 404 whatever the roots are. -The page is the hand-written scene from step 2, scrubbed by hand. There is -deliberately no clock: the audio clock, the rAF loop, the `::resolver` -subscription and the transport are step 3, and the question this step answers is -whether the data model evaluates correctly, not whether it evaluates at 30fps. -A scrubber answers the first and nothing else, which is what makes a failure -here unambiguous. +Four clips, on buttons in the transport: -The scene itself is `src/arthur/demo/scene.edn`, and it is not a face. It is the -smallest scene that exercises every mechanism the model claims to have — -inherited exposure, a sparse held `[:xform :pos]`, composition through a group, -rotation about an anchor, a stencil chain, a keyed `[:vis]`, a `:span`, and -fractional z among siblings — chosen so that each one is visible when it breaks. +| | | +| --- | --- | +| `take` | the synthetic take, head **as filmed**. Step 5's deliverable: a moving mouth, frozen into dense channels, with no video file anywhere. | +| `locked` | the same freeze, head **locked**. The same blocks — `:head`'s channels are written as framed identity instead of as a dense track, and nothing in tier 2 differs. | +| `demo` | the hand-written scene from step 2. Not a face: the smallest scene that exercises every mechanism the model claims to have, so that each one is visible when it breaks. | +| `swarm` | a hundred and twenty dense nodes. Not useful; it is the load test. | + +`take` and `locked` are the pair worth looking at together, because switching +between them is the whole of what "stabilisation is a channel, not a mode" means. + +The demo scene itself is `src/arthur/demo/scene.edn`; the take is built in +`src/arthur/demo/take.cljs`, which is also the only place the seven stages are +composed in order. Port 8778 is deliberately not 8777. `python3 serve.py` from the repo root still -runs the old JS tool on 8777, and the two are meant to run side by side — that is -the whole reason `js/` is still in the tree. +runs the old JS tool on 8777, and the two are meant to run side by side. From step 9 Django serves the page and `:dev-http` goes away. -## The oracle +## The oracle, which is finished -`js/` is the numeric oracle, not dead weight. `test/parity/` runs both -implementations on the same synthetic track and diffs them: `fit-similarity` and -`procrustes-mean` agree to 1e-9, the raster pixel-for-pixel, and `stabilize`'s -whole output agrees across the three namespaces it was split into. +`js/` was the numeric oracle through step 4: `test/parity/` ran both +implementations on the same synthetic track and diffed `fit-similarity`, +`procrustes-mean`, the raster and `stabilize` to 1e-9. -The oracle drives `stabilize` at aspect 0.5625 as well as at 1. Aspect 1 makes the -anisotropy correction the identity, so a port that dropped it entirely would pass -— which is the one thing a parity suite on normalised landmarks can be blind to. +**It was deleted at step 5, on purpose.** Parity proves the port is FAITHFUL, not +that the answer is RIGHT. The JS is a prototype and several of its conclusions +contradict each other; a parity test pins behaviour while code moves, and keeping +it afterwards would bake the prototype's mistakes into the rewrite and make them +permanent. `docs/port-plan.md` says to delete it in one commit once the CLJS +player renders the synthetic take, and that is what happened. -Both sides get the identical track because `js/synth.js` reads `Math.random` at -call time, so `oracle.mjs` stubs it to a constant and the CLJS side passes -`:rand-fn (constantly 0.5)`. `js/` itself is never modified. - -**`test/parity/` and `arthur.parity-test` get deleted in one commit at step 5.** -A parity test pins behaviour while code moves; keeping it afterwards would bake -the prototype's mistakes into the rewrite and make them permanent. +`js/` itself stays. It is not an oracle any more, it is the SOURCE for steps 6 +and 7 — the MediaPipe setup, the eye and brow signals, the interior extraction — +and its comments encode bugs that actually happened. ## Layout ``` src/arthur/domain/ pure. No re-frame, no DOM, no flow/. src/arthur/flow/ the stages. `(f params inputs) -> output`, no state. +src/arthur/synth.cljs the synthetic track. In src/ because the take PLAYS it — + it stands in for flow/detect, and a tool that needs a + video file before it shows you anything is one you + cannot debug. src/arthur/demo.cljs the hand-written scene, read from demo/scene.edn +src/arthur/demo/take.cljs the seven stages composed in order, and the only place + they are src/arthur/ui/canvas.cljs the one imperative sink — the only DOM canvas call -test/arthur/synth.cljs the synthetic track — test infrastructure, not src -test/parity/ the JS oracle harness. Deletable at step 5. +test/browser/ drives a real Chrome over CDP. Not run by `npm test`. public/index.html dev host page. Django replaces it at step 9. ``` diff --git a/frontend/package.json b/frontend/package.json index 72b27f2..2bdaea0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,7 +5,8 @@ "scripts": { "watch": "shadow-cljs watch app", "release": "shadow-cljs release app", - "test": "node test/parity/oracle.mjs && shadow-cljs compile test && node out/node-tests.js" + "test": "shadow-cljs compile test && node out/node-tests.js", + "browser": "node --experimental-websocket test/browser/take.mjs" }, "dependencies": { "react": "^18.3.1", diff --git a/frontend/src/arthur/db.cljs b/frontend/src/arthur/db.cljs index bb92312..2f07566 100644 --- a/frontend/src/arthur/db.cljs +++ b/frontend/src/arthur/db.cljs @@ -7,30 +7,50 @@ into this map and every mounted layer-2 sub compares the result. So the scene is here (it is a document — a human placed every node) and dense - channel blocks are not; they live behind a handle in `store`. Today the demo - scene has no dense blocks and the store is empty, which is why it is a map and - not yet a namespace." + channel blocks are not; they live behind a handle in `store`. The hand-written + demo scene has no dense blocks and its store is empty; the swarm and the take + are entirely dense." (:require [arthur.demo :as demo] - [arthur.demo.swarm :as swarm])) + [arthur.demo.swarm :as swarm] + [arthur.demo.take :as take])) + +(defn- clip + "A scene plus the clip-level facts the transport and the stage need. + + Read OFF the scene rather than written again beside it. `:fps`, `:frames` and the + stage dimensions belong to the CLIP and not to the timeline — a timeline has a + frame space, not a rate and not a size — and they sit on the scene map only + because there is one clip per scene today. Copying them by hand into this table + is how one of them comes to disagree with the scene it describes." + [label scene store] + (merge {:label label :scene scene :store store} + (select-keys scene [:fps :frames :width :height]))) (def scenes - "Two hand-made clips, selectable from the transport. + "The hand-made clips, selectable from the transport. - `:swarm` holds its geometry in DENSE blocks behind store handles, which is the - shape freeze produces at step 5 — so the fun one is also the load test." - {:demo {:label "demo" :scene demo/scene :store nil - :fps demo/fps :frames demo/frames} - :swarm {:label "swarm" :scene @swarm/scene :store @swarm/store - :fps swarm/fps :frames swarm/frames}}) + `:swarm` is the load test: a hundred and twenty nodes, entirely dense. The two + takes are step 5's deliverable and they are ONE freeze — the same blocks, with + `:head` written as a dense track in one and as framed identity in the other, so + the button that switches between them switches a document field and nothing + else." + {:demo (clip "demo" demo/scene nil) + :swarm (clip "swarm" @swarm/scene @swarm/store) + :take (clip "take" @take/scene @take/store) + :take-locked (clip "locked" @take/locked @take/store)}) (def default {;; --- the document --- - :scene/current :demo + :scene/current :take :palette :arthur/default ; a NAME; the ramp itself is project data ;; --- the clip --- - :clip {:fps demo/fps - :frames demo/frames} + ;; + ;; Including the STAGE DIMENSIONS, which are the project's and not the + ;; footage's. That is what deleting `makeXform` buys — the framing became a + ;; transform on a node, so nothing downstream of the freeze knows the frame + ;; size — and it is why ui/player no longer hardcodes 320x200. + :clip (select-keys (:take scenes) [:fps :frames :width :height]) ;; --- transport --- ;; diff --git a/frontend/src/arthur/demo.cljs b/frontend/src/arthur/demo.cljs index 8181d62..cb60c9c 100644 --- a/frontend/src/arthur/demo.cljs +++ b/frontend/src/arthur/demo.cljs @@ -13,9 +13,6 @@ (def scene (reader/read-string source)) -(def width 320) -(def height 200) - (def fps (:fps scene)) (def frames (:frames scene)) diff --git a/frontend/src/arthur/demo/scene.edn b/frontend/src/arthur/demo/scene.edn index f358f13..840b23a 100644 --- a/frontend/src/arthur/demo/scene.edn +++ b/frontend/src/arthur/demo/scene.edn @@ -26,6 +26,12 @@ ;; frame space, not a rate — and it is here only because there is one clip. :frames 229 :fps 30 + ;; The STAGE, in pixels. The project's dimensions, not the footage's — which is + ;; what makes `makeXform` deletable: placement is a transform on a node and the + ;; stage clips whatever hangs off. Here everything is authored in stage pixels + ;; already, because a hand-written scene is a painted one. + :width 320 + :height 200 :nodes {;; The clip root. EXPOSURE LIVES HERE and is inherited, because diff --git a/frontend/src/arthur/demo/swarm.cljs b/frontend/src/arthur/demo/swarm.cljs index 23db5b5..bf53b0f 100644 --- a/frontend/src/arthur/demo/swarm.cljs +++ b/frontend/src/arthur/demo/swarm.cljs @@ -152,6 +152,8 @@ {:name "swarm" :frames frames :fps fps + :width 320 + :height 200 :nodes (into {:root {:id :root :kind :group :parent nil :z "a1" ;; On 2s, like everything else. A hundred and twenty shapes diff --git a/frontend/src/arthur/demo/take.cljs b/frontend/src/arthur/demo/take.cljs new file mode 100644 index 0000000..4db3548 --- /dev/null +++ b/frontend/src/arthur/demo/take.cljs @@ -0,0 +1,112 @@ +(ns arthur.demo.take + "The synthetic take: the whole vertical slice, with no video file in it. + + This is port-plan step 5's deliverable and it is the first thing in the tree + that runs every stage in order — + + synth ──▶ measure/anchor ──▶ condition/anchor + │ │ + └──▶ measure/mouth ◀─┘ + │ + condition/contours + │ + FREEZE ──▶ channels on nodes + │ + scene/resolver ──▶ raster + + — and the order of that diagram is the whole argument for the stage split. The + anchor fit is knob-free. Conditioning smooths its four parameters. The rings are + then measured THROUGH the conditioned transform, so `anchor avg` does re-run the + ring mapping — a few hundred frames of twenty points, free — and does not re-run + anything that reads a source pixel, because that part takes the landmarks and + the frames and never the transform. + + TWO SCENES, ONE STORE. `:take` carries the head as filmed and `:take-locked` + carries it locked, and they are the same dense blocks with one node's channels + written two ways. That is the claim \"stabilisation is a channel, not a mode\" + made checkable by eye: switching between them is a document edit, tier 1, and + not one byte of tier 2 differs." + (:require [arthur.flow.condition :as condition] + [arthur.flow.freeze :as freeze] + [arthur.flow.measure.anchor :as anchor] + [arthur.flow.measure.mouth :as mouth] + [arthur.synth :as synth])) + +(def frames 229) +(def fps 30) + +(def ^:private stage + ;; The project's dimensions, and NOT the footage's. This is what deleting + ;; `makeXform` buys: the head is placed and scaled on the stage by a transform + ;; on a node, so a 1440x1920 portrait clip and a 320x200 stage are not a + ;; conflict to resolve. Whatever hangs off the edge is clipped. + [320 200]) + +(def ^:private aspect + ;; The synth writes x and y in the SAME unit, so its normalised space is already + ;; isotropic and the anisotropy correction is the identity here. Real footage + ;; passes W/H from the manifest at step 6. Worth knowing while reading anything + ;; here: aspect 1 is the one setting at which a port that dropped the + ;; anisotropy correction entirely would still look right, which is why + ;; anchor-test exercises 0.5625 and this does not. + 1) + +(def ^:private knobs + "The prototype's own defaults, from index.html, so the first thing anyone sees + is the thing it was tuned to look like." + {:anchor-avg 2 ; smoothWin + :contour-avg 1 ; contourSmooth + :verts 8 ; vertices + :aperture-cut 0.12}) ; apertureThresh 120/1000 + +(def analysis + "Stage 2's output, synthesised. A seeded generator, so a wrong pose is + reproducible rather than something that happened once." + (delay (synth/synth-dense frames {:seed 1}))) + +(def measured + "Stages 3 and 4, in the order the stage split requires." + (delay + (let [dense @analysis + fitted (anchor/fit {:aspect aspect} {:dense dense}) + anchored (condition/anchor knobs fitted) + rings (mouth/measure {:aspect aspect} + {:dense dense :transforms (:transforms anchored)})] + (assoc anchored + :outer (condition/contours knobs (:outer rings)) + :inner (condition/contours knobs (:inner rings)) + ;; NOT smoothed. The aperture is the inner ring's own height read off + ;; as a scalar, and `contour avg` is a spatial-correspondence + ;; smoother over a ring; running it over a scalar would be a second, + ;; undocumented low-pass on a signal that then decides visibility. + :aperture (:aperture rings))))) + +(def params + "What the freeze was handed. Public because it is the honest way to re-freeze at + other settings — a test that built its own copy would be asserting about a clip + nobody looks at." + (merge knobs + {:name "take" + :fps fps + :stage stage + ;; On 2s. docs/design.md is emphatic that everything rides ONE grid: a + ;; head cutting on odd frames against a mouth cutting on even ones reads + ;; as two performances, so exposure lives on the clip root and inherits. + :expose 2 + ;; The head as filmed. `:take-locked` is the same freeze with this one + ;; field changed, which is the point. + :head :as-filmed + ;; Provenance. A content hash once the analysis is an artifact the + ;; backend stores; until then, honest about what it actually is. + :analysis "synth:mulberry32/seed-1"})) + +(def frozen + (delay (freeze/clip params @measured))) + +(def store (delay (:store @frozen))) + +(def scene (delay (:scene @frozen))) + +(def locked + "The same blocks, with `:head` written as framed identity instead." + (delay (freeze/head-mode {:mode :locked} @frozen))) diff --git a/frontend/src/arthur/domain/channel.cljs b/frontend/src/arthur/domain/channel.cljs index 9a5a5ff..a280449 100644 --- a/frontend/src/arthur/domain/channel.cljs +++ b/frontend/src/arthur/domain/channel.cljs @@ -109,11 +109,18 @@ ;; --------------------------------------------------------------------------- ;; dense blocks -(defn- dense-state - [state f] - (if (nil? state) - present - (aget state f))) +(defn- absent-at? + "Is the subject absent on frame f of this block's slice? + + INDEXED THE WAY THE DATA IS. A block is node-major — offset(node i) = + i·frames·stride — so a block holding several nodes holds several mask regions, + and the one belonging to this channel starts at offset/stride. Indexing the + mask by f alone reads the FIRST node's absence for every node in the block, + which is not a subtly wrong pose: it is every part in the block vanishing on + the frames where one of them was occluded." + [state offset stride f] + (and (some? state) + (pos? (bit-and (aget state (+ (quot offset stride) f)) absent-bit)))) (defn dense-at "Read frame f out of a dense block. @@ -130,21 +137,37 @@ stride 1 yields a number; anything wider yields a SUBARRAY VIEW over the block, not a copy. Fixed topology is what makes that possible — the frame's - data is a rectangular slice at a known offset with no per-frame header." - [{:keys [store offset stride] nf :frames} f st] - (let [{:keys [data state]} (get st store)] - (when (nil? data) - (throw (ex-info "dense channel's store key is not in the store" - {:store store :have (vec (sort (map str (keys st))))}))) - (let [f (-> f (max 0) (min (dec nf))) - sm (dense-state state f)] - (cond - (pos? (bit-and sm absent-bit)) absent - :else - (let [o (+ offset (* f stride))] - (if (= 1 stride) - (aget data o) - (.subarray data o (+ o stride)))))))) + data is a rectangular slice at a known offset with no per-frame header. + + FIXED POINT. `:scale` in the block header means the stored integers are the + value times that scale, so a block of geometry in image-height units fills an + Int16 usefully and a block of stage pixels — which wants a different scale + entirely — fills one too. It is in the header rather than agreed by convention + for exactly that reason, and it is why the block in memory is byte for byte the + block on the wire: a handle that names a sha256 has to name the bytes you + actually hold. + + Decoding costs the view. `out` is a stride-sized destination the caller owns — + `cursor` allocates one per channel — because a copy per node per frame is the + allocation this whole model is arranged to avoid; passing nil allocates, which + is what `value-at`, the specification, does." + ([blk f st] (dense-at blk f st nil)) + ([{:keys [store offset stride scale] nf :frames} f st out] + (let [{:keys [data state]} (get st store)] + (when (nil? data) + (throw (ex-info "dense channel's store key is not in the store" + {:store store :have (vec (sort (map str (keys st))))}))) + (let [f (-> f (max 0) (min (dec nf)))] + (if (absent-at? state offset stride f) + absent + (let [o (+ offset (* f stride))] + (cond + (= 1 stride) (let [v (aget data o)] (if scale (/ v scale) v)) + (nil? scale) (.subarray data o (+ o stride)) + :else (let [dst (or out (js/Float64Array. stride))] + (dotimes [k stride] + (aset dst k (/ (aget data (+ o k)) scale))) + dst)))))))) ;; --------------------------------------------------------------------------- ;; the specification @@ -197,20 +220,28 @@ (recur (inc mid) hi mid) (recur lo (dec mid) best)))))) -(deftype Cursor [ch ks store ^:mutable i] +(deftype Cursor [ch ks store buf ^:mutable i] Object (toString [_] (str "#Cursor{" (pr-str (if ks :keyed (if (:dense ch) :dense :framed))) " i=" i "}"))) (defn cursor "A reading head on one channel. Build once per channel per resolver, then `sample!` it per frame. Holds the sorted key index, which is why the index is - built here and not in the document." + built here and not in the document — and the decode buffer a fixed-point block + needs, for the same reason the resolver owns one point buffer per node. + + Only a wide fixed-point block gets a buffer: a stride-1 block decodes to a + number and a block with no `:scale` is handed back as a view." ([ch] (cursor ch nil)) ([ch store] (check-unimplemented! ch) - (->Cursor ch (when (and (:animated? ch) (not (:dense ch)) (seq (:keys ch))) - (frames ch)) - store 0))) + (let [d (:dense ch)] + (->Cursor ch + (when (and (:animated? ch) (not d) (seq (:keys ch))) (frames ch)) + store + (when (and d (:scale d) (> (:stride d) 1)) + (js/Float64Array. (:stride d))) + 0)))) (defn sample! "Value of the cursor's channel at f. O(1) when f is at or one key past where @@ -222,7 +253,7 @@ ks (.-ks cur)] (cond (not (:animated? ch)) (:value ch) - (:dense ch) (dense-at (:dense ch) f (.-store cur)) + (:dense ch) (dense-at (:dense ch) f (.-store cur) (.-buf cur)) (nil? ks) absent ; animated with an empty key map :else (let [n (count ks) @@ -281,5 +312,14 @@ " requires hold of every cut part and tweening reads as puppet software")) (and (map? ch) (seq (:over ch))) - (conj ":over layers are not implemented (port-plan step 2 scope)"))) + (conj ":over layers are not implemented (port-plan step 2 scope)") + + ;; A scale of zero divides every value in the block by zero, and a negative + ;; one mirrors the geometry. Both are authored-data bugs that present as a + ;; part drawn nowhere or inside out, not as an error. + (and (map? ch) (:dense ch) (contains? (:dense ch) :scale) + (not (and (number? (:scale (:dense ch))) (pos? (:scale (:dense ch)))))) + (conj (str ":dense :scale is " (pr-str (:scale (:dense ch))) + " — a fixed-point scale is a positive number the stored integers" + " were multiplied by")))) diff --git a/frontend/src/arthur/domain/scene.cljs b/frontend/src/arthur/domain/scene.cljs index c9a5bca..5df011a 100644 --- a/frontend/src/arthur/domain/scene.cljs +++ b/frontend/src/arthur/domain/scene.cljs @@ -190,6 +190,30 @@ (ch/nothing? skw) (ch/nothing? anc)) [pos rot scl skw anc]))) +(defn- visible? + "Is the node switched on this frame? + + `[:vis]` IS A BOOLEAN, and this insists on it rather than testing truthiness, + because the two obvious implementations are both wrong about a DENSE `[:vis]`. + A dense block yields 0 or 1, and 0 is TRUTHY in CLJS — so `(if v …)` shows a + hidden frame, and `(true? v)` hides every frame. Neither reads as an error. + + docs/animation-model.md's parts table says `:mouth-in` carries `[:vis]` dense; + flow/freeze writes it KEYED, because a threshold crossing is a handful of + transitions and hold is the default, and because a human has to be able to fix + one frame of it. When something does want a dense one it will land here loudly + instead of blanking the scene. + + Absence is not a boolean and is not an error: a subject that is not on the + frame has nothing to show." + [id v] + (cond + (true? v) true + (false? v) false + (ch/nothing? v) false + :else (throw (ex-info "[:vis] must sample to a boolean" + {:node id :value v})))) + (defn- place "Where a node sits this frame, as {:m world :f local-frame :rd reader}, or nil when it is not on the frame at all. @@ -210,7 +234,7 @@ chs (node/channels n) lf (node/local-frame n pf) rd (fn [path] (read id path (get chs path) lf))] - (when (true? (rd [:vis])) + (when (visible? id (rd [:vis])) (when-let [[pos rot scl skw anc] (xform-at rd)] ;; dest aliases `local` here, which mul! allows: it reads both ;; operands fully before writing either. diff --git a/frontend/src/arthur/events/playback.cljs b/frontend/src/arthur/events/playback.cljs index 6e5e921..f0d7129 100644 --- a/frontend/src/arthur/events/playback.cljs +++ b/frontend/src/arthur/events/playback.cljs @@ -92,9 +92,12 @@ ;; Changing the clip changes the resolver, the frame count and the rate all ;; at once, so the playhead goes home rather than being left pointing at a ;; frame the new clip may not have. - (let [{:keys [fps frames]} (get db/scenes id)] + (let [{:keys [fps frames] :as clip} (get db/scenes id)] {:db (-> db (assoc :scene/current id) - (assoc :clip {:fps fps :frames frames}) + ;; The stage travels with the clip: two clips may be different + ;; sizes, and the raster the loop paints into is the clip's, not + ;; the app's. + (assoc :clip (select-keys clip [:fps :frames :width :height])) (assoc-in [:playback :frame] 0)) ::seek! [fps frames 0]}))) diff --git a/frontend/src/arthur/flow/freeze.cljs b/frontend/src/arthur/flow/freeze.cljs new file mode 100644 index 0000000..4681479 --- /dev/null +++ b/frontend/src/arthur/flow/freeze.cljs @@ -0,0 +1,440 @@ +(ns arthur.flow.freeze + "Stage 5, the freeze: measurements become CHANNELS. + + This is the hinge the whole model turns on. Freezing is NOT a conversion into a + second format — there is one format, and freezing fills it in. That is what + makes \"the only difference between rotoscoped and hand-authored is a flag\" + literally true: what comes out of here is the same `:channels` map a hand fills + in sparsely, and the flag is `:generated`, which nothing in the renderer reads. + + Three conversions happen here and nowhere else. + + MAPS BECOME FLAT. `flow/measure/*` speaks {:x :y}, because it is the numeric + oracle and a faithful port was worth more there than a fast one. A channel + value is FLAT — [x0 y0 x1 y1 …] — in authored vectors and dense blocks alike. + `rings->flat` is the only place that crossing is made. + + FLOATS BECOME FIXED POINT. Geometry goes into an Int16 block with the scale in + its header; see `geom-scale`. + + A SIMILARITY BECOMES THREE CHANNELS. `{s θ tx ty}` IS `[:xform :scale]`, + `[:xform :rot]` and `[:xform :pos]`, so the anchor drops onto `:head` with no + adapter — which is the sign the decomposition is the right one. + + `makeXform` IS NOT HERE AND IS NOT COMING. The prototype centres on the face + oval's bbox and zooms until the face is 80% of the raster height, so every + stored vertex carries a cropping decision made once, at analysis time, from one + frame's landmarks. Here the geometry stays in the node's own local space and the + framing is `[:xform :*]` on an authored `:face` node, which the stage clips. + Project dimensions are therefore independent of the footage — see + `face-placement`, and \"What space geometry is in\" in docs/animation-model.md. + + Not `flow/key`. A traced mouth costs nothing, so it gets a key on EVERY frame; + only a plate, which a human draws, is worth decimating. The one stage-5 policy + the mouth does want is the aperture threshold, and that is `visibility` below." + (:require [arthur.domain.channel :as ch] + [arthur.domain.geom :as geom] + [arthur.domain.ring :as ring])) + +;; --------------------------------------------------------------------------- +;; fixed point + +(def ^:const geom-scale + "Q14: the stored integer is the value times 16384. + + Geometry is head-local and its unit is ONE IMAGE HEIGHT, so 16384 covers ±2 + image heights in an Int16 and quantises to 1/16384 = 6.1e-5 of an image height. + Against a face placed so that its 0.29-image-height oval fills most of a 200px + stage — around 850 stage pixels per image height — that is 0.05px, two orders + below anything the rasteriser can express. + + A power of two, so the decode is a floating-point exact division and freezing + the same numbers twice cannot drift. + + It is a CONSTANT here and a FIELD in the block header, and the difference + matters: a painted cel's geometry is in stage pixels, where ±2 would be absurd + and 1/16384 of a pixel is waste. Each block says what its own space needs." + 16384) + +(def ^:private ^:const int16-max 32767) + +;; --------------------------------------------------------------------------- +;; blocks + +(defn- pack + "Tracks -> one dense block, NODE-MAJOR and FRAME-MINOR: + + offset(track i) = i · frames · stride + value(i, f) = data[offset(i) + f · stride] + + No per-frame header and no indirection, which FIXED TOPOLOGY is what buys: every + frame of a part carries the same component count with the same meanings, so a + frame is a rectangular slice at an arithmetic offset. A variable vertex count + would force an offset table and a scan per frame, so the aesthetic constraint is + a performance asset rather than a cost. `demo/swarm` holds the same layout and + is the load test for it. + + `:ctor` makes the array — a function and not a type, because `new` is not + something a value can carry — `:scale` is the fixed-point scale or nil, and + `:absent` an optional per-frame predicate. Absence is the SUBJECT's, not the + part's, so it is written into every track of the block: the mouth outline and + the mouth interior are one face, and a frame that face was not on has no pose + for either. + + Written with `dotimes` and `aset` rather than as a fold, and that is the + exception rather than the rule in this codebase: the destination is a typed + array, so there is nothing to accumulate into and a collection idiom here would + allocate a seq per frame to throw away." + [{:keys [ctor scale absent]} tracks] + (let [n (count tracks) + nf (count (first tracks)) + stride (count (first (first tracks))) + data (ctor (* n nf stride)) + state (when absent (js/Uint8Array. (* n nf)))] + (dotimes [i n] + (let [track (vec (nth tracks i)) + base (* i nf stride)] + (dotimes [f nf] + (let [vs (nth track f) + o (+ base (* f stride))] + (dotimes [k stride] + (let [v (nth vs k)] + (aset data (+ o k) + (if scale + (let [q (js/Math.round (* v scale))] + ;; Saturating would read as articulation flattening off + ;; at the extremes — a bad detection, not a bad scale. + (when (> (abs q) int16-max) + (throw (ex-info "value does not fit the block's fixed point" + {:value v :scale scale :quantised q + :track i :frame f :component k}))) + q) + v)))) + (when (and state (absent f)) + (aset state (+ (* i nf) f) ch/absent-bit)))))) + {:data data :state state :stride stride :frames nf :scale scale + :offsets (mapv #(* % nf stride) (range n))})) + +(defn- dense + "Track i of a packed block, as a DENSE channel definition." + [store-key blk i generated] + {:animated? true :interp :hold + :dense (cond-> {:store store-key + :offset (nth (:offsets blk) i) + :stride (:stride blk) + :frames (:frames blk)} + (:scale blk) (assoc :scale (:scale blk))) + :generated generated + :over []}) + +;; --------------------------------------------------------------------------- +;; geometry + +(defn rings->flat + "Ring track -> one flat [x0 y0 x1 y1 …] per frame, at the vertex budget. + + The vertex budget is a fixed-index SUBSAMPLE, never adaptive decimation: slot k + means the same anatomy on every frame of the shot, and that is what makes + temporal correspondence possible at all. It is applied here, after stage 4's + contour average, and the two commute because both are per-slot — which is the + whole reason the vertex knob can sit downstream of the smoothing knob instead + of alongside it. `mouth-test` asserts that rather than leaving it to look + obvious. + + An ODD budget is refused. It lands off the cardinal slots — the corners and the + lip centres — and a wrongly-ordered ring self-intersects INVISIBLY at odd vertex + counts and obviously at even ones, so an odd budget is the one setting at which + the simplicity assertion stops protecting anything." + [rings verts] + (let [len (count (first rings))] + (when-not (and (integer? verts) (even? verts) (>= verts 4) (<= verts len)) + (throw (ex-info "vertex budget must be even and between 4 and the ring's slot count" + {:verts verts :slots len}))) + (let [slots (ring/subsample-slots len verts)] + (mapv (fn [r] + (into [] (mapcat (fn [s] (let [p (nth r s)] [(:x p) (:y p)]))) slots)) + rings)))) + +;; --------------------------------------------------------------------------- +;; the anchor, onto :head + +(defn invert + "The inverse of a similarity, STILL FACTORED. + + The anchor fit maps each frame onto the shot's mean pose, so it is what takes + the head's motion OUT; geometry is stored in the space it produces. Putting the + head's motion back — the \"as filmed\" mode — is therefore the fit's inverse, and + it has to stay a {s θ t} rather than becoming a matrix, because the three + components land on three independently keyframable channels and interpolating + matrix entries is meaningless. + + p ↦ s·R(θ)·p + t inverts to q ↦ (1/s)·R(-θ)·(q - t)" + [{:keys [s theta tx ty]}] + (let [s' (/ 1.0 s) + c (js/Math.cos theta) + sn (js/Math.sin theta)] + {:s s' + :theta (- theta) + :tx (* (- s') (+ (* c tx) (* sn ty))) + :ty (* (- s') (+ (* (- sn) tx) (* c ty)))})) + +(def ^:private head-modes #{:locked :as-filmed :per-plate}) + +(defn head-mode + "Rewrite `:head`'s transform channels into one of the three shapes. + + STABILISATION IS A CHANNEL, NOT A MODE. `{s θ tx ty}` per frame IS + `[:xform :scale]`, `[:xform :rot]` and `[:xform :pos]`, so removing the head's + motion is not a pipeline setting — it is a question of which shape one node's + channels carry: + + :locked framed identity — the head sits still, for tracing and for + judging articulation + :as-filmed dense — the head moves around the stage + :per-plate keyed at the kept frames — the head pose is stable for exactly as + long as a drawing is on screen, which is what a plate strip wants + + ALWAYS MEASURE, ALWAYS STORE FACTORED, whatever the toggle says. The blocks are + written once and every mode reads them or ignores them; making this an + analysis-time switch would lose two things downstream. \"Smooth the transform, + never the contour\" only means anything while the two are separate. And a + velocity minimum is \"articulation paused\" in head-local space and \"the head + happened to be still\" in image space, so key selection needs the split to exist + in storage. + + So this is a DOCUMENT EDIT: tier 1, undoable, syncable, instant, and not a + reason to re-analyse. It rewrites `:head` and touches nothing else — in + particular not `:face`, which a hand placed, and not one byte of any block. + + `:per-plate` samples the dense track at the kept frames and stores plain + vectors. It may not store what `value-at` handed it: a wide dense read is a + VIEW into the block, and a view into tier 2 sitting in the document is a value + that changes when a re-freeze rewrites the array under it." + [{:keys [mode kept]} {:keys [scene store]}] + (when-not (contains? head-modes mode) + (throw (ex-info "head mode is not one of the three channel shapes" + {:mode mode :modes head-modes}))) + (let [base (get-in scene [:nodes :head :measured]) + at (fn [path f] + (let [v (ch/value-at (get base path) f store) + n (:stride (:dense (get base path)))] + (if (= 1 n) v (mapv #(ch/component v %) (range n))))) + keys-of (fn [path] + (-> (get base path) + (dissoc :dense) + (assoc :keys (into {} (map (juxt identity #(at path %))) (sort kept)))))] + (when (and (= mode :per-plate) (empty? kept)) + (throw (ex-info "the per-plate head mode needs a kept-frame set; it is the plate strip's, not measurement's" + {:mode mode}))) + (assoc-in scene [:nodes :head :channels] + (case mode + ;; No `:generated` on the locked shape, and that is not an + ;; oversight: nothing generated this identity. It is a decision, + ;; and provenance that claimed otherwise would offer a re-freeze + ;; button that could only undo it. + :locked {[:xform :pos] (ch/framed [0.0 0.0]) + [:xform :rot] (ch/framed 0.0) + [:xform :scale] (ch/framed [1.0 1.0])} + :as-filmed base + :per-plate (into {} (map (juxt key #(keys-of (key %)))) base))))) + +;; --------------------------------------------------------------------------- +;; the aperture, onto [:vis] of :mouth-in + +(defn visibility + "The aperture track -> `[:vis]` keys on the mouth interior. + + `flow/measure/mouth` reports the aperture and deliberately does not threshold + it: the measurement is the inner ring's own height and the threshold is a + policy, which is stage 5. Relative to the take's PEAK aperture, not absolute, so + one number works across faces and framings — the prototype's + `ap.map(v => v / apMax < apertureThresh)`, which lives in its `app.js` and not + in its `pipeline.js`, and is easy to miss when porting from the latter. + + KEYED, not dense, although docs/animation-model.md's parts table says dense. A + threshold crossing is a handful of transitions over a take, hold is the default, + and keys are what a human can correct — \"this frame's mouth should be shut\" is + the single most likely hand edit on a lip-sync take, and a dense block in tier 2 + is the one shape that cannot receive it. `:generated` still rides along, which is + the point of provenance being on the channel rather than implied by its shape. + + A key on frame 0 always, because the first key is the pose the part starts in." + [{:keys [aperture-cut]} {:keys [aperture]} generated] + (let [peak (reduce max aperture) + ;; A take with no mouth at all has no peak to be a fraction of. Present + ;; rather than absent: an all-zero aperture is a shut mouth, and the + ;; interior of a shut mouth is simply not drawn. + shown (mapv (fn [v] (and (pos? peak) (>= (/ v peak) aperture-cut))) aperture)] + (assoc (ch/keyed (into {} (keep (fn [f] + (when (or (zero? f) + (not= (nth shown f) (nth shown (dec f)))) + [f (nth shown f)]))) + (range (count shown)))) + :generated generated))) + +;; --------------------------------------------------------------------------- +;; the face, onto the stage + +(defn face-placement + "The face's transform on the stage, as FRAMED channels. + + AUTHORED, and that is the whole difference from `makeXform`. What comes back is + a DEFAULT — the placement a human would otherwise have to make from scratch on + first open — and from then on it is an ordinary hand-placed transform on an + ordinary node. `makeXform` made the same decision and then baked it into every + vertex, where nothing could ever revise it. + + Derived from the reference rigid configuration, which is what the freeze has: + the face oval is not measured, because its only consumers in the prototype were + that transform and the placeholder plate outline. Two numbers: + + SCALE is stage pixels per image height, set so the reference's eye-corner span + is 40% of the stage width. Landmark-free — it is the rigid configuration's own + bounding box — and it is a fraction of the STAGE, so a 1440x1920 portrait clip + composited onto a 320x200 stage is not a problem to solve. + + ANCHOR is the reference centroid, and this is where `:anchor` earns its place. + MediaPipe's normalised space has its origin at the image's TOP-LEFT CORNER, so + head-local geometry is not centred on anything; the registration point of the + face is the head's own centre, and rotation and scale have to happen about that + rather than about a corner of the footage. Getting that wrong is why hand-placed + parts swing rather than turn. + + POSITION puts the anchor a QUARTER of the way down the stage, because the rigid + landmarks are eyes and nose — the upper middle of a face — so a quarter down + leaves the jaw and the mouth on the stage. Whatever hangs off is clipped, which + is not a feature to add: every fill in `domain/raster` clamps already." + [{:keys [stage]} {:keys [ref]}] + (let [[w h] stage + c (geom/centroid ref) + span (- (reduce max (map :x ref)) (reduce min (map :x ref))) + k (/ (* 0.4 w) span)] + {[:xform :anchor] (ch/framed [(:x c) (:y c)]) + [:xform :scale] (ch/framed [k k]) + [:xform :pos] (ch/framed [(- (/ w 2) (:x c)) + (- (* 0.25 h) (:y c))])})) + +;; --------------------------------------------------------------------------- +;; the clip + +(defn clip + "Conditioned measurements -> a clip: nodes with frozen channels, plus the dense + blocks they read. `(f params inputs)`, no state. + + params + :name names the clip and its store keys + :fps the clip's rate. FRAMES is not a parameter — it is + `(count outer)`, because a freeze that could disagree with its + own input about the length of the take would. + :stage [w h] project dimensions, INDEPENDENT of the footage + :expose the clip root's exposure grid, inherited by everything + :verts the lip rings' vertex budget + :aperture-cut fraction of the take's peak aperture below which the mouth + interior is not present + :head :locked | :as-filmed | :per-plate + :kept frames, for :per-plate only + :analysis which analysis artifact these measurements came from + :anchor-avg + :contour-avg the stage-4 knobs. Freeze does not use them; it RECORDS them, + because `:generated` is what lets the UI offer a re-freeze at + different parameters instead of raw keys. + + inputs + :ref the Procrustes reference configuration + :transforms the CONDITIONED anchor transforms + :outer :inner the CONDITIONED head-local lip rings + :aperture head-local aperture per frame + :detected optional per-frame booleans; absent frames get the state mask + + The node tree is the one docs/animation-model.md specifies, and the two groups + are two different things wanting the same transform: + + :root the clip. EXPOSURE LIVES HERE and is inherited strictly. + :face AUTHORED. where the face sits on the stage, and how big. + :head MEASURED. the head's motion, or identity. + :mouth + :mouth-in + + `:mouth-in`'s parent is `:mouth` and that transform is identity today, so + composing it is composing identity. It is documented intent, and it is the one + place in the tree where the parent pointer is not yet doing work — the moment a + painted cel rides a moving plate it will be. + + `:head` keeps its measured channels under `:measured` as well as in + `:channels`, so `head-mode` can switch shapes without the blocks or the + provenance having to be rebuilt." + [{:keys [name fps stage expose verts aperture-cut head kept + analysis anchor-avg contour-avg] :as params} + {:keys [transforms outer inner detected] :as inputs}] + (let [nf (count outer) + absent (when detected #(not (nth detected % true))) + geom-k (str name "/geom") + head-k #(str name "/head-" %) + prov (fn [by extra] + {:by by :analysis analysis + :params (merge {:anchor-avg anchor-avg} extra)}) + rings (pack {:ctor #(js/Int16Array. %) :scale geom-scale :absent absent} + [(rings->flat outer verts) (rings->flat inner verts)]) + ;; The anchor, inverted and split into its three components. Three blocks + ;; and not one: they are three channels, they have three strides, and a + ;; single block would need a per-component offset table to say so. + inv (mapv invert transforms) + xf (fn [f] (pack {:ctor #(js/Float32Array. %) :absent absent} [(mapv f inv)])) + pos (xf (fn [t] [(:tx t) (:ty t)])) + rot (xf (fn [t] [(:theta t)])) + scale (xf (fn [t] [(:s t) (:s t)])) + ;; Which knobs each channel records is not decoration, it is the + ;; invalidation table written down where a re-freeze can read it. The + ;; anchor depends on `anchor avg` alone. The rings depend on it and on + ;; `contour avg` and on the vertex budget. The aperture depends on + ;; `anchor avg` and on its own cut and NOT on `contour avg`, because + ;; measure reports the inner ring's own height and nothing smooths it. + anchor-prov (prov :anchor/similarity nil) + roto (fn [by] (prov by {:verts verts :contour-avg contour-avg})) + scene {:name name + :frames nf + :fps fps + ;; Stage dimensions, on the clip and not on the footage. See + ;; face-placement: nothing below here knows the frame size. + :width (first stage) + :height (second stage) + :nodes + {:root + {:id :root :name "clip" :kind :group :parent nil :z "a1" + :time {:mode :map :expose expose}} + + :face + {:id :face :name "face" :kind :group :parent :root :z "a1" + :channels (face-placement params inputs)} + + :head + {:id :head :name "head" :kind :group :parent :face :z "a1" + :measured {[:xform :pos] (dense (head-k "pos") pos 0 anchor-prov) + [:xform :rot] (dense (head-k "rot") rot 0 anchor-prov) + [:xform :scale] (dense (head-k "scale") scale 0 anchor-prov)}} + + ;; The outer lip ring is the dark band OUTSIDE the interior, and + ;; that three-layer structure — dark ring, pale interior, teeth + ;; — is what makes a flat shape read as an opening rather than + ;; as a blob. So it keeps every frame and is never hidden. + :mouth + {:id :mouth :name "mouth" :kind :poly :parent :head :z "a1" + :channels {[:geom :pts] (dense geom-k rings 0 (roto :roto/lips-outer)) + [:style :color] (ch/framed :skin-dark)}} + + :mouth-in + {:id :mouth-in :name "mouth interior" :kind :poly + :parent :mouth :z "a2" + :channels {[:geom :pts] (dense geom-k rings 1 (roto :roto/lips-inner)) + [:style :color] (ch/framed :mouth-dark) + [:vis] (visibility params inputs + (prov :roto/mouth-aperture + {:aperture-cut aperture-cut}))}}}} + ;; Tier 2, behind a handle. The keys are descriptive because there is no + ;; hashing yet; they become the blocks' sha256 when the backend arrives + ;; and nothing above here changes, which is the point of a handle. + store (into {} (map (fn [[k blk]] [k (select-keys blk [:data :state])])) + {geom-k rings + (head-k "pos") pos, (head-k "rot") rot, (head-k "scale") scale})] + {:store store + :scene (head-mode {:mode head :kept kept} {:scene scene :store store})})) diff --git a/frontend/src/arthur/flow/measure/anchor.cljs b/frontend/src/arthur/flow/measure/anchor.cljs index e3a311b..bc5a1cc 100644 --- a/frontend/src/arthur/flow/measure/anchor.cljs +++ b/frontend/src/arthur/flow/measure/anchor.cljs @@ -37,8 +37,9 @@ "RMS misfit per frame, in the isotropic space's units — one image height. Taken against the transforms it is HANDED rather than against a fit of its own, - so the same function serves the stage-3 reading and the parity diff. The - prototype took it against the SMOOTHED transforms, which folds the smoothing + which is what let the parity diff assert on exactly what the prototype handed + it while that diff existed. The prototype took the residual against the + SMOOTHED transforms, which folds the smoothing error into a number whose whole job is to say whether the footage is stabilisable at all; `fit` takes it against the raw fit instead." [ref rigid tfs] diff --git a/frontend/src/arthur/subs/playback.cljs b/frontend/src/arthur/subs/playback.cljs index 2f0c344..9bdd5ff 100644 --- a/frontend/src/arthur/subs/playback.cljs +++ b/frontend/src/arthur/subs/playback.cljs @@ -11,3 +11,8 @@ (rf/reg-sub ::muted? (fn [db _] (get-in db [:playback :muted?]))) (rf/reg-sub ::fps (fn [db _] (get-in db [:clip :fps]))) (rf/reg-sub ::frames (fn [db _] (get-in db [:clip :frames]))) +;; The stage, in pixels. On the clip because project dimensions are independent +;; of the footage — see flow/freeze/face-placement — so the canvas and the raster +;; take their size from the document rather than from a constant. +(rf/reg-sub ::width (fn [db _] (get-in db [:clip :width]))) +(rf/reg-sub ::height (fn [db _] (get-in db [:clip :height]))) diff --git a/frontend/test/arthur/synth.cljs b/frontend/src/arthur/synth.cljs similarity index 92% rename from frontend/test/arthur/synth.cljs rename to frontend/src/arthur/synth.cljs index fa2eb52..f7361a9 100644 --- a/frontend/test/arthur/synth.cljs +++ b/frontend/src/arthur/synth.cljs @@ -7,12 +7,18 @@ stabilisation against a KNOWN head motion, since real footage gives no ground truth to compare against. + In `src/` and not in `test/`, and that moved at step 5: the synthetic take is + what the tool plays before any footage has been detected, so the generator is + app code that stands in for `flow/detect` rather than a test fixture. Step 6 + puts MediaPipe beside it, not in place of it — a tool that needs a video file + before it will show you anything is a tool you cannot debug. + Differs from js/synth.js in exactly one way, deliberately: the jitter comes - from a SEEDED generator rather than Math.random. Two reasons. A failing - assertion has to be reproducible to be worth anything, and the JS is the - numeric oracle - parity is only checkable if both sides can be handed the same - track. `:rand-fn` takes the generator over, so stubbing js/Math.random in a - node harness makes the two implementations agree exactly." + from a SEEDED generator rather than Math.random. A failing assertion has to be + reproducible to be worth anything, and a wrong pose that happened once is not + a bug report. `:rand-fn` takes the generator over entirely, which is what let + the JS-parity harness hand both implementations the identical track while it + existed." (:require [arthur.domain.landmarks :as lm])) ;; mulberry32. Chosen for being four lines of int32 arithmetic that port diff --git a/frontend/src/arthur/ui/player.cljs b/frontend/src/arthur/ui/player.cljs index 309d299..308395e 100644 --- a/frontend/src/arthur/ui/player.cljs +++ b/frontend/src/arthur/ui/player.cljs @@ -107,6 +107,8 @@ :ramp @(rf/subscribe [::render/ramp]) :fps @(rf/subscribe [::sub/fps]) :frames @(rf/subscribe [::sub/frames]) + :width @(rf/subscribe [::sub/width]) + :height @(rf/subscribe [::sub/height]) :frame @(rf/subscribe [::sub/frame]) :playing? @(rf/subscribe [::sub/playing?])}) ;; A new resolver means a new scene or a new palette, and neither @@ -133,13 +135,16 @@ rasterised before the next frame is asked for." [f] (let [{:keys [canvas]} @state - {:keys [resolver palette ramp]} @snapshot] - (when (and canvas resolver) + {:keys [resolver palette ramp width height]} @snapshot] + (when (and canvas resolver width height) ;; User Timing, so a profile in the DevTools performance panel has named ;; spans in the Timings track instead of a wall of anonymous frames. Three ;; lines, and the difference between reading a profile and guessing at one. (js/performance.mark "arthur/paint:start") - (let [ras (raster-for 320 200)] + ;; THE STAGE IS THE CLIP'S, not a constant. Project dimensions are + ;; independent of the footage, so the size the frame is rasterised at comes + ;; out of the document like everything else. + (let [ras (raster-for width height)] (-> ras (raster/clear! (get palette :bg 0)) (raster/draw-ops! (resolver f))) diff --git a/frontend/src/arthur/ui/shell.cljs b/frontend/src/arthur/ui/shell.cljs index b64bb0c..cc63a9f 100644 --- a/frontend/src/arthur/ui/shell.cljs +++ b/frontend/src/arthur/ui/shell.cljs @@ -7,7 +7,6 @@ why scrubbing at speed does not re-render the page." (:require [arthur.clock :as clock] [arthur.db :as db] - [arthur.demo :as demo] [arthur.events.playback :as pb] [arthur.subs.playback :as sub] [arthur.subs.render :as render] @@ -80,14 +79,22 @@ (when (and drop (pos? drop)) (str " · " (.toFixed drop 2) " frames/paint")))])]])) +(defn- stage [] + ;; The canvas is the STAGE's size, and the stage is the clip's — not a constant + ;; and not the footage's. Reactive, so selecting a clip of another size resizes + ;; it; ui/canvas guards the width assignment, which reallocates the backing + ;; store, so this being a re-render costs nothing per frame. + (let [w @(rf/subscribe [::sub/width]) + h @(rf/subscribe [::sub/height])] + [:canvas.stage + {:ref #(player/set-canvas! %) + :width w :height h + :style {:width (str (* zoom w) "px") :height (str (* zoom h) "px")}}])) + (defn view [] [:main [:h1 "arthur"] - [:canvas.stage - {:ref #(player/set-canvas! %) - :width demo/width :height demo/height - :style {:width (str (* zoom demo/width) "px") - :height (str (* zoom demo/height) "px")}}] + [stage] [audio] [transport] [:p.note diff --git a/frontend/test/arthur/domain/channel_test.cljs b/frontend/test/arthur/domain/channel_test.cljs index b423b52..83d4877 100644 --- a/frontend/test/arthur/domain/channel_test.cljs +++ b/frontend/test/arthur/domain/channel_test.cljs @@ -95,6 +95,27 @@ (is (not (ch/nothing? 0)) "zero is a value, not an absence") (is (not (ch/nothing? false)) "and so is false"))) +(deftest the-mask-belongs-to-the-block-s-slice-and-not-to-frame-zero + ;; A block is NODE-MAJOR, so one block holds several nodes' tracks and therefore + ;; several mask regions. Indexing the mask by the frame alone reads the FIRST + ;; track's absence for every track in the block — which is not a subtly wrong + ;; pose, it is every part in the block vanishing on the frames where one of them + ;; was occluded, and `demo/swarm` drew nothing at all for its first thirty-four + ;; frames on account of it. + (let [nf 3 + ;; Track 0 absent on frame 1, track 1 absent on frame 2. + state (js/Uint8Array. #js [ch/present ch/absent-bit ch/present + ch/present ch/present ch/absent-bit]) + store {"blk" {:data (js/Int16Array. #js [10 11, 20 21, 30 31, + 40 41, 50 51, 60 61]) + :state state}} + track (fn [i] {:animated? true + :dense {:store "blk" :offset (* i nf 2) :stride 2 :frames nf}}) + read (fn [i f] (let [v (ch/value-at (track i) f store)] + (if (ch/nothing? v) :absent [(ch/component v 0) (ch/component v 1)])))] + (is (= [[10 11] :absent [30 31]] (mapv #(read 0 %) (range nf)))) + (is (= [[40 41] [50 51] :absent] (mapv #(read 1 %) (range nf)))))) + (deftest a-block-with-no-mask-is-present-throughout ;; The mask is optional: a generator that cannot fail to detect has nothing to ;; say, and allocating a zeroed byte per frame to say it would be noise. @@ -129,6 +150,43 @@ (is (= (mapv spec fs) (via-cursor c fs)) (str label " / " order-name))))))) +(deftest a-fixed-point-block-decodes-through-its-own-scale + ;; `:scale` is in the block HEADER and not agreed by convention, because a block + ;; in image-height units and a block in stage pixels need different ones to fill + ;; an Int16 usefully. It is what lets the block in memory be byte for byte the + ;; block on the wire, which a handle naming a sha256 requires. + (let [store {"blk" {:data (js/Int16Array. #js [16384 -8192, 4096 32767]) :state nil}} + wide {:animated? true :dense {:store "blk" :offset 0 :stride 2 :frames 2 + :scale 16384}} + thin {:animated? true :dense {:store "blk" :offset 0 :stride 1 :frames 4 + :scale 16384}}] + (is (= [[1.0 -0.5] [0.25 (/ 32767 16384)]] + (mapv (fn [f] (let [v (ch/value-at wide f store)] + [(ch/component v 0) (ch/component v 1)])) + (range 2)))) + (is (= [1.0 -0.5 0.25] (mapv #(ch/value-at thin % store) (range 3))) + "a stride-1 block decodes to a number and needs no buffer") + (is (seq (ch/problems (assoc-in wide [:dense :scale] 0))) + "a scale of zero divides every value in the block by zero") + (is (seq (ch/problems (assoc-in wide [:dense :scale] -16384)))))) + +(deftest the-cursor-decodes-into-a-buffer-it-owns-and-agrees-with-the-spec + ;; Decoding costs the subarray view, so the reader owns one destination per + ;; channel — the same bargain the resolver makes with its point buffers, and it + ;; carries the same contract: a value has to be CONSUMED before the next frame + ;; is asked for, because the next read overwrites it. + (let [store {"blk" {:data (js/Int16Array. #js [16384 0, 0 16384, -16384 0]) :state nil}} + c {:animated? true :dense {:store "blk" :offset 0 :stride 2 :frames 3 + :scale 16384}} + cur (ch/cursor c store) + pair (fn [v] [(ch/component v 0) (ch/component v 1)])] + (is (= (mapv #(pair (ch/value-at c % store)) (range 3)) + (mapv #(pair (ch/sample! cur %)) (range 3)))) + (let [held (ch/sample! cur 0)] + (ch/sample! cur 2) + (is (= [-1.0 0.0] (pair held)) + "the buffer is reused, which is the contract and not a bug")))) + (deftest the-cursor-reads-a-dense-block-too (let [store {"blk" {:data (js/Float32Array. #js [1 2 3 4 5]) :state nil}} c {:animated? true :dense {:store "blk" :offset 0 :stride 1 :frames 5}} diff --git a/frontend/test/arthur/domain/scene_test.cljs b/frontend/test/arthur/domain/scene_test.cljs index f09744b..b783a42 100644 --- a/frontend/test/arthur/domain/scene_test.cljs +++ b/frontend/test/arthur/domain/scene_test.cljs @@ -301,7 +301,7 @@ ;; same on every frame. (let [res (scene/resolver demo/scene) render (fn [f] - (let [r (raster/make demo/width demo/height)] + (let [r (raster/make (:width demo/scene) (:height demo/scene))] (raster/clear! r (:bg pal/index-of)) (raster/draw-ops! r (res f)) r)) @@ -319,7 +319,7 @@ ;; other than the frame the channels are sampled at. (let [res (scene/resolver demo/scene) render (fn [f] - (let [r (raster/make demo/width demo/height)] + (let [r (raster/make (:width demo/scene) (:height demo/scene))] (raster/clear! r (:bg pal/index-of)) (raster/draw-ops! r (res f)) (vec (array-seq (:buf r)))))] @@ -336,8 +336,8 @@ ;; pupil by the iris, and neither is expressed anywhere as a chain. (let [res (scene/resolver demo/scene)] (doseq [f (range 0 (:frames demo/scene) 4)] - (let [before (raster/make demo/width demo/height) - after (raster/make demo/width demo/height) + (let [before (raster/make (:width demo/scene) (:height demo/scene)) + after (raster/make (:width demo/scene) (:height demo/scene)) ops (res f) card? (fn [op] (= :card (:node op)))] (raster/clear! before (:bg pal/index-of)) diff --git a/frontend/test/arthur/flow/freeze_test.cljs b/frontend/test/arthur/flow/freeze_test.cljs new file mode 100644 index 0000000..e9a5e62 --- /dev/null +++ b/frontend/test/arthur/flow/freeze_test.cljs @@ -0,0 +1,463 @@ +(ns arthur.flow.freeze-test + "The freeze is where the model's central claim is either true or false: + measurement and a hand produce THE SAME DATA, and the only difference is a flag + nothing in the renderer reads. Most of what is asserted here is that claim, + taken apart into the pieces that could quietly stop holding. + + It is also the first step whose done-criterion is a PICTURE, so nothing in here + is the proof that step 5 is done — a take that resolves to the right numbers and + draws nothing would pass every assertion below. See test/browser/take.mjs, which + drives a real Chrome." + (:require [cljs.test :refer [deftest is testing]] + [arthur.demo.take :as take] + [arthur.domain.channel :as ch] + [arthur.domain.geom :as geom] + [arthur.domain.node :as node] + [arthur.domain.palette :as pal] + [arthur.domain.raster :as raster] + [arthur.domain.ring :as ring] + [arthur.domain.scene :as scene] + [arthur.flow.freeze :as freeze])) + +(def ^:private W 320) +(def ^:private H 200) + +;; The whole vertical slice, exactly as the page builds it. Asserting against the +;; page's own clip rather than against a fixture built here is deliberate: a +;; fixture is a second scene nobody looks at, and the one that renders is the one +;; that has to be right. +(def clip (delay @take/frozen)) +(def scene* (delay (:scene @clip))) +(def store (delay (:store @clip))) + +(defn- node [id] (get-in @scene* [:nodes id])) +(defn- chan [id path] (get-in (node id) [:channels path])) + +(defn- pts-at + "The mouth's [:geom :pts] at frame f, as a flat CLJS vector." + [id f] + (let [v (ch/value-at (chan id [:geom :pts]) f @store)] + (mapv #(ch/component v %) (range (.-length v))))) + +(defn- ops-at [sc f] + ((scene/resolver sc @store pal/index-of) f)) + +(defn- render + "One frame of a scene into a byte buffer. The stage's size comes off the clip, + because project dimensions are the project's and not the footage's." + [sc f] + (let [r (raster/make (:width sc) (:height sc))] + (raster/clear! r (get pal/index-of :bg)) + (raster/draw-ops! r (ops-at sc f)) + (vec (array-seq (:buf r))))) + +(defn- drawn + "How many pixels are not background." + [buf] + (count (remove #(= % (get pal/index-of :bg)) buf))) + +;; --------------------------------------------------------------------------- +;; the shape of what came out + +(deftest the-frozen-take-is-a-valid-scene-in-every-head-mode + ;; `scene/problems` is total by construction, so this is safe to run over data + ;; before the data is trusted — which is what it is for. + (doseq [mode [:as-filmed :locked]] + (let [sc (freeze/head-mode {:mode mode} @clip)] + (is (empty? (scene/problems sc)) (str mode ": " (pr-str (scene/problems sc)))))) + (let [sc (freeze/head-mode {:mode :per-plate :kept #{0 12 40 88 150}} @clip)] + (is (empty? (scene/problems sc)) (pr-str (scene/problems sc))))) + +(deftest the-tree-is-the-one-the-model-specifies + ;; :face is AUTHORED and :head is MEASURED, and they are two nodes because two + ;; different things want that transform. A group node is free; keeping the + ;; hand-placed and the measured transform apart is the whole reason the + ;; transform is decomposed in the first place. + (is (= [:face :root] (scene/lineage (:nodes @scene*) :face))) + (is (= [:head :face :root] (scene/lineage (:nodes @scene*) :head))) + (is (= [:mouth :head :face :root] (scene/lineage (:nodes @scene*) :mouth))) + (is (= [:mouth-in :mouth :head :face :root] + (scene/lineage (:nodes @scene*) :mouth-in))) + ;; Exposure lives on the clip root and inherits strictly. + (is (= {:mode :map :expose 2} (:time (node :root)))) + (is (every? #(nil? (:time (node %))) [:face :head :mouth :mouth-in]))) + +(deftest geometry-is-flat-and-dense-and-the-interior-shares-the-outline-s-block + (doseq [id [:mouth :mouth-in]] + (is (= :dense (ch/describe (chan id [:geom :pts]))))) + ;; ONE block, two tracks, node-major. offset(track 1) = frames · stride, which + ;; is the layout demo/swarm holds and the reason a frame is a rectangular slice + ;; at an arithmetic offset rather than a lookup into a table. + (let [a (:dense (chan :mouth [:geom :pts])) + b (:dense (chan :mouth-in [:geom :pts]))] + (is (= (:store a) (:store b))) + (is (zero? (:offset a))) + (is (= (* (:frames a) (:stride a)) (:offset b)))) + ;; Flat: [x0 y0 x1 y1 …], so stride is twice the vertex budget and a value reads + ;; the same way an authored vector does. + (is (= (* 2 8) (:stride (:dense (chan :mouth [:geom :pts]))))) + (is (= (* 2 8) (count (pts-at :mouth 0))))) + +(deftest the-fixed-point-block-round-trips-to-within-its-own-quantum + ;; The one thing fixed point can cost is precision, so it is measured rather + ;; than assumed. `rings->flat` is re-run here against the same measured rings, + ;; which is what the block was written from. + (let [want (freeze/rings->flat (:outer @take/measured) 8) + q (/ 1.0 freeze/geom-scale) + gap (reduce max (for [f (range 0 take/frames 7) + [a b] (map vector (pts-at :mouth f) (nth want f))] + (abs (- a b))))] + (is (<= gap (/ q 2)) + (str "worst quantisation error " gap " against a quantum of " q)) + ;; And in stage pixels, which is the unit anyone can judge. `:face`'s scale is + ;; stage px per image height, so the whole quantum is q·k — a twentieth of a + ;; pixel at this placement, two orders below anything the rasteriser can + ;; express, which is the argument for Int16 geometry stated as a measurement. + (let [k (first (:value (chan :face [:xform :scale])))] + (is (< (* q k) 0.1) + (str "the quantum is " (* q k) " stage pixels at " k " px per image height")) + (is (< (* gap k) (* q k)))))) + +(deftest the-scale-is-in-the-header-and-not-agreed-by-convention + ;; A block in image-height units and a block in stage pixels want different + ;; scales, which is why it is a field. Transform blocks carry none at all: they + ;; are Float32, because an angle and a scale factor have no natural fixed point + ;; and there are four numbers a frame of them rather than forty. + (is (= freeze/geom-scale (:scale (:dense (chan :mouth [:geom :pts]))))) + (doseq [path [[:xform :pos] [:xform :rot] [:xform :scale]]] + (is (nil? (:scale (:dense (get-in (node :head) [:measured path])))) + (str path " should be plain Float32"))) + (is (instance? js/Int16Array (:data (get @store "take/geom")))) + (is (instance? js/Float32Array (:data (get @store "take/head-pos"))))) + +(deftest a-value-past-the-block-s-range-is-refused-rather-than-saturated + ;; Saturating reads as articulation flattening off at the extremes — a bad + ;; detection, not a bad scale — so it has to be loud. A ring three image heights + ;; wide cannot be real, and that is the point: if it happens, the block's space + ;; is wrong and there is nothing to be gained by drawing something. + (is (thrown-with-msg? + ExceptionInfo #"does not fit the block's fixed point" + (freeze/clip (assoc take/params :name "huge") + (update @take/measured :outer + (fn [rings] (mapv (fn [r] (mapv #(update % :x + 3) r)) rings))))))) + +;; --------------------------------------------------------------------------- +;; the anchor: three channels, and the inverse + +(deftest the-similarity-inverse-undoes-the-fit-exactly + ;; The fit takes the head's motion OUT and geometry is stored in the space it + ;; produces, so putting the motion back — "as filmed" — is the fit's inverse. + ;; Still factored, because the three components land on three independently + ;; keyframable channels. + (let [gap (reduce max + (for [tf (:transforms @take/measured) + p [{:x 0.5 :y 0.6} {:x 0.0 :y 0.0} {:x -0.3 :y 1.2}]] + (let [q (geom/apply-sim (freeze/invert tf) (geom/apply-sim tf p))] + (js/Math.hypot (- (:x q) (:x p)) (- (:y q) (:y p))))))] + (is (< gap 1e-12) (str "invert ∘ fit is off by " gap)))) + +(deftest the-head-carries-the-inverse-fit-split-into-its-three-components + ;; Float32 storage, so this is a tolerance and not equality — four bytes a + ;; number is the spec's choice for transform blocks and it costs about seven + ;; decimal digits, which at 850 stage pixels per image height is far below a + ;; pixel. + (let [measured (get-in (node :head) [:measured]) + at (fn [path f] (ch/value-at (get measured path) f @store))] + (doseq [f (range 0 take/frames 11)] + (let [want (freeze/invert (nth (:transforms @take/measured) f)) + pos (at [:xform :pos] f)] + (is (< (abs (- (ch/component pos 0) (:tx want))) 1e-5)) + (is (< (abs (- (ch/component pos 1) (:ty want))) 1e-5)) + (is (< (abs (- (at [:xform :rot] f) (:theta want))) 1e-6)) + (is (< (abs (- (ch/component (at [:xform :scale] f) 0) (:s want))) 1e-6)))))) + +(deftest head-local-geometry-composed-through-head-and-face-lands-on-the-stage + ;; The end-to-end claim of the split, asserted against the OPS the resolver + ;; actually emits rather than against an intermediate: stored head-local, put + ;; back through `:head`, placed by `:face`, the mouth is where the composition + ;; of the two says it is. A test that recomputed the chain would only be + ;; checking arithmetic against itself; this checks `node/local!`, `node/world!` + ;; and `emit` as well. + (let [sc (freeze/head-mode {:mode :as-filmed} @clip) + res (scene/resolver sc @store pal/index-of) + k (first (:value (chan :face [:xform :scale]))) + anc (:value (chan :face [:xform :anchor])) + pos (:value (chan :face [:xform :pos])) + tfs (:transforms @take/measured)] + (doseq [f (range 0 take/frames 13)] + (let [ops (res f) + op (first (filter #(= :mouth (:node %)) ops)) + ;; EXPOSURE FIRST. The clip root is on 2s and exposure inherits + ;; strictly, so frame 13 shows frame 12's pose — which is also the + ;; cheapest place to assert that the grid is actually being applied, + ;; since reading the unexposed frame here misses by half a pixel and + ;; looks like a rounding problem. + ef (node/expose f 2) + ;; The frozen, quantised vertex — so the fixed point is not part of + ;; what is being asserted here; it has its own test. + flat (pts-at :mouth ef) + g {:x (nth flat 0) :y (nth flat 1)} + ;; Where the anchor fit says that head-local point was in the image: + ;; the fit removed the head's motion, so putting it back is the fit's + ;; inverse. Node :head carries exactly this. + im (geom/apply-sim (freeze/invert (nth tfs ef)) g) + ;; And where :face puts it: scaled about the anchor, then translated, + ;; which is p ↦ k(p - anchor) + anchor + pos. + wx (+ (* k (- (:x im) (nth anc 0))) (nth anc 0) (nth pos 0)) + wy (+ (* k (- (:y im) (nth anc 1))) (nth anc 1) (nth pos 1))] + (is (some? op) (str "frame " f " emitted no mouth op")) + ;; Tolerance is the Float32 transform block's, scaled to stage pixels, and + ;; it is three orders below a pixel. + (is (< (js/Math.hypot (- (aget (:pts op) 0) wx) + (- (aget (:pts op) 1) wy)) + 0.01) + (str "frame " f ": op has [" + (aget (:pts op) 0) " " (aget (:pts op) 1) + "], the composition says [" wx " " wy "]")))))) + +;; --------------------------------------------------------------------------- +;; the three modes are the three channel shapes + +(deftest the-three-head-modes-are-the-three-channel-shapes + (let [kept #{0 12 40 88 150} + of (fn [sc path] (get-in sc [:nodes :head :channels path]))] + (testing "locked is framed identity" + (let [sc (freeze/head-mode {:mode :locked} @clip)] + (is (= [:framed :framed :framed] + (mapv #(ch/describe (of sc %)) + [[:xform :pos] [:xform :rot] [:xform :scale]]))) + (is (= [0.0 0.0] (:value (of sc [:xform :pos])))) + (is (= 0.0 (:value (of sc [:xform :rot])))) + (is (= [1.0 1.0] (:value (of sc [:xform :scale])))))) + (testing "as filmed is dense" + (let [sc (freeze/head-mode {:mode :as-filmed} @clip)] + (is (= [:dense :dense :dense] + (mapv #(ch/describe (of sc %)) + [[:xform :pos] [:xform :rot] [:xform :scale]]))))) + (testing "per plate is keyed at exactly the kept frames" + (let [sc (freeze/head-mode {:mode :per-plate :kept kept} @clip)] + (doseq [path [[:xform :pos] [:xform :rot] [:xform :scale]]] + (is (= :keyed (ch/describe (of sc path)))) + (is (= (sort kept) (sort (keys (:keys (of sc path)))))) + ;; A dense read is a VIEW into tier 2. Storing one in the document would + ;; be storing a value that changes when a re-freeze rewrites the array + ;; under it, so the keys hold plain data. + (doseq [[_ v] (:keys (of sc path))] + (is (or (number? v) (vector? v)) (str path " key is " (pr-str v))))) + ;; And the keys are the dense track sampled at those frames, which is the + ;; whole of what "per plate" means. + (is (= (mapv #(ch/value-at (get-in (node :head) [:measured [:xform :rot]]) % @store) + (sort kept)) + (mapv (:keys (of sc [:xform :rot])) (sort kept)))))))) + +(deftest switching-modes-rewrites-the-head-and-nothing-else + ;; It has to be impossible for the toggle to move something a hand placed, and + ;; it has to be a DOCUMENT edit: tier 1, undoable, syncable, instant, and not a + ;; reason to re-analyse. + (let [a (freeze/head-mode {:mode :as-filmed} @clip) + b (freeze/head-mode {:mode :locked} @clip) + c (freeze/head-mode {:mode :per-plate :kept #{0 40}} @clip)] + (doseq [sc [b c]] + (is (= (get-in a [:nodes :face]) (get-in sc [:nodes :face])) + ":face moved") + (is (= (dissoc (:nodes a) :head) (dissoc (:nodes sc) :head)) + "a node other than :head changed") + ;; The measurement does not go away when the head is locked: always measure, + ;; always store factored, toggle the parent. + (is (= (get-in a [:nodes :head :measured]) (get-in sc [:nodes :head :measured])))))) + +(deftest a-head-mode-that-is-not-one-of-the-three-is-refused + (is (thrown-with-msg? ExceptionInfo #"not one of the three channel shapes" + (freeze/head-mode {:mode :stabilised} @clip))) + ;; The kept-frame set belongs to the plate strip, not to measurement, so freeze + ;; cannot invent one. + (is (thrown-with-msg? ExceptionInfo #"kept-frame set" + (freeze/head-mode {:mode :per-plate} @clip)))) + +;; --------------------------------------------------------------------------- +;; the face: authored, and what makes makeXform deletable + +(deftest the-face-is-authored-and-carries-no-provenance + ;; The difference from `makeXform` in one assertion: the placement is a FRAMED + ;; transform on a node, which a hand can revise, and it claims no generator that + ;; would offer to overwrite it. + (doseq [path [[:xform :pos] [:xform :rot] [:xform :scale] [:xform :anchor]]] + (let [c (get (node/channels (get-in @scene* [:nodes :face])) path)] + (is (= :framed (ch/describe c)) (str path " is not framed")) + (is (nil? (:generated c)) (str path " claims provenance"))))) + +(deftest the-face-puts-the-head-s-centre-where-it-says-it-does + ;; anchor + pos is where the anchor lands in the parent, which is what makes + ;; `:anchor` the registration point: scale and rotation happen about the head's + ;; centre rather than about the corner of the footage, where MediaPipe's + ;; normalised space has its origin. + (let [anc (:value (chan :face [:xform :anchor])) + pos (:value (chan :face [:xform :pos])) + c (geom/centroid (:ref @take/measured))] + (is (< (abs (- (nth anc 0) (:x c))) 1e-12)) + (is (< (abs (- (nth anc 1) (:y c))) 1e-12)) + (is (< (abs (- (+ (nth anc 0) (nth pos 0)) (/ W 2))) 1e-9)) + (is (< (abs (- (+ (nth anc 1) (nth pos 1)) (* 0.25 H))) 1e-9)))) + +(deftest the-stage-is-the-clip-s-and-not-the-footage-s + ;; Project dimensions are independent of the footage, which is precisely what + ;; dropping makeXform buys. Nothing below the freeze knows the frame size, so + ;; asking for a different stage moves and rescales the same geometry rather than + ;; re-measuring anything. + (let [big (freeze/clip (assoc take/params :stage [640 480] :name "big") + @take/measured)] + (is (= [640 480] [(:width (:scene big)) (:height (:scene big))])) + (is (= (vec (array-seq (:data (get @store "take/geom")))) + (vec (array-seq (:data (get (:store big) "big/geom"))))) + "the geometry is the same numbers at either stage size") + (is (not= (:value (get-in (:scene big) [:nodes :face :channels [:xform :scale]])) + (:value (chan :face [:xform :scale])))))) + +;; --------------------------------------------------------------------------- +;; the aperture, as [:vis] + +(deftest the-mouth-interior-is-hidden-below-the-aperture-cut + (let [c (chan :mouth-in [:vis]) + ap (:aperture @take/measured) + peak (reduce max ap) + want (mapv #(>= (/ % peak) 0.12) ap)] + (is (= :keyed (ch/describe c))) + (is (= want (mapv #(ch/value-at c %) (range take/frames))) + "the held keys do not reproduce the threshold") + ;; The reason it is keyed: a threshold crossing is a handful of transitions, + ;; hold is the default, and keys are the shape a human can correct. A dense + ;; block would be 229 bytes in tier 2 to say the same thing, un-editable. + (is (< (count (:keys c)) 40) + (str (count (:keys c)) " keys for " take/frames " frames")) + (is (contains? (:keys c) 0) "the first key is the pose the part starts in") + ;; It really does both, or the assertion above is vacuous. + (is (some true? want)) + (is (some false? want)))) + +(deftest the-mouth-outline-is-never-hidden + ;; The dark band OUTSIDE the interior is what makes a flat shape read as an + ;; opening rather than a blob, so the outline keeps every frame; only the + ;; interior comes and goes. + (is (nil? (chan :mouth [:vis]))) + (doseq [f (range 0 take/frames 9)] + (is (some #(= :mouth (:node %)) (ops-at @scene* f)) + (str "frame " f " drew no mouth outline")))) + +;; --------------------------------------------------------------------------- +;; provenance + +(deftest every-generated-channel-says-who-generated-it-and-under-which-knobs + ;; `:generated` is what lets the UI offer a parameter panel and a re-freeze + ;; instead of raw keys, and which knobs it names is the invalidation table + ;; written where a re-freeze can read it. + (is (= :roto/lips-outer (:by (:generated (chan :mouth [:geom :pts]))))) + (is (= :roto/lips-inner (:by (:generated (chan :mouth-in [:geom :pts]))))) + (is (= :roto/mouth-aperture (:by (:generated (chan :mouth-in [:vis]))))) + (is (= :anchor/similarity + (:by (:generated (get-in (node :head) [:measured [:xform :pos]]))))) + (is (= {:anchor-avg 2 :contour-avg 1 :verts 8} + (:params (:generated (chan :mouth [:geom :pts]))))) + ;; The aperture does NOT depend on `contour avg`: measure reports the inner + ;; ring's own height and nothing smooths it. + (is (= {:anchor-avg 2 :aperture-cut 0.12} + (:params (:generated (chan :mouth-in [:vis]))))) + (is (every? #(some? (:analysis (:generated %))) + [(chan :mouth [:geom :pts]) (chan :mouth-in [:vis])]))) + +(deftest the-renderer-never-reads-provenance + ;; The load-bearing claim, asserted rather than trusted: strip every + ;; `:generated` out of the document and the frame is the same bytes. If this + ;; ever fails, a rotoscoped part and a hand-drawn one have stopped being the + ;; same data. + (let [stripped (update @scene* :nodes + (fn [ns] (into {} (map (fn [[id n]] + [id (update n :channels + #(into {} (map (fn [[p c]] [p (dissoc c :generated)])) %))])) + ns)))] + (doseq [f (range 0 take/frames 17)] + (is (= (render @scene* f) (render stripped f)) + (str "frame " f " differs with provenance removed"))))) + +;; --------------------------------------------------------------------------- +;; presence is not visibility + +(deftest an-undetected-frame-has-no-pose-at-all + ;; A subject that was not on the frame has NO VALUE, which is different from a + ;; part being switched off. The mask lands on every block of the freeze, so an + ;; absent frame takes the head's transform with it — and a node with no + ;; transform gives its children nowhere to be, so the whole face goes. + (let [gap (set (range 40 60)) + det (mapv #(not (contains? gap %)) (range take/frames)) + c (freeze/clip (assoc take/params :name "gappy") + (assoc @take/measured :detected det)) + sc (:scene c) + res (scene/resolver sc (:store c) pal/index-of)] + (doseq [f [39 40 50 59 60]] + (let [ops (res f)] + (if (contains? gap f) + (is (empty? ops) (str "frame " f " is absent and drew " (count ops) " ops")) + (is (seq ops) (str "frame " f " is present and drew nothing"))))) + ;; And it is the MASK doing it, not a hidden flag: `[:vis]` on :mouth-in is + ;; unchanged across the gap, because hiding and absence are different + ;; questions with different answers. + (is (= (mapv #(ch/value-at (get-in sc [:nodes :mouth-in :channels [:vis]]) %) + (range take/frames)) + (mapv #(ch/value-at (chan :mouth-in [:vis]) %) (range take/frames)))))) + +;; --------------------------------------------------------------------------- +;; the rings are still rings + +(deftest a-frozen-ring-is-simple-at-every-vertex-budget + ;; Because hold parts CUT between poses rather than interpolating, a ring whose + ;; vertex order is wrong renders as blocks meeting at corners rather than as an + ;; error. It is invisible at odd vertex counts and obvious at even ones, so it + ;; needs an assertion rather than an eyeball — and the subsample is the one + ;; operation in the freeze that could reorder a traversal. + (doseq [verts [4 6 8 10 16 20] + which [:outer :inner]] + (let [flat (freeze/rings->flat (get @take/measured which) verts) + bad (first (for [f (range 0 take/frames 3) + :let [r (mapv (fn [k] {:x (nth (nth flat f) (* 2 k)) + :y (nth (nth flat f) (inc (* 2 k)))}) + (range verts)) + hits (ring/self-intersections r)] + :when (seq hits)] + {:verts verts :ring which :frame f :edges (first hits)}))] + (is (nil? bad) (str "self-intersection: " (pr-str bad)))))) + +(deftest an-odd-vertex-budget-is-refused + ;; It lands off the cardinal slots, and it is the one setting at which the + ;; simplicity assertion above stops protecting anything. + (doseq [bad [3 5 7 2 22 8.5]] + (is (thrown-with-msg? ExceptionInfo #"vertex budget" + (freeze/rings->flat (:outer @take/measured) bad)) + (str bad " was accepted")))) + +;; --------------------------------------------------------------------------- +;; it draws, and it moves + +(deftest the-take-draws-something-on-every-frame + (doseq [f (range 0 take/frames 5)] + (let [n (drawn (render @scene* f))] + (is (> n 200) (str "frame " f " drew only " n " pixels"))))) + +(deftest the-mouth-moves + ;; The synth holds each pose for nine frames in a four-beat cycle, so frames + ;; from different beats are genuinely different mouths and frames inside one + ;; beat are not. This is the numeric half of step 5's done-criterion; the other + ;; half is a picture and lives in test/browser/take.mjs. + (let [locked (freeze/head-mode {:mode :locked} @clip) + shot (fn [f] (render locked f)) + differ (fn [a b] (count (remove true? (map = a b))))] + ;; Beat 1 is wide open and beat 3 is shut. Rendered with the head LOCKED, so + ;; what differs is articulation and not the head wandering across the stage. + (is (> (differ (shot 10) (shot 28)) 300) + "the open and the shut mouth rasterise the same") + ;; And within a beat, on the exposure grid, it holds. + (is (= (shot 10) (shot 10))) + (is (< (differ (shot 10) (shot 12)) 200) + "a held pose is moving more than the detector noise it should have lost")) + ;; As filmed, the head carries it around the stage as well. + (let [filmed (freeze/head-mode {:mode :as-filmed} @clip)] + (is (> (count (remove true? (map = (render filmed 10) (render filmed 120)))) 300) + "the head does not move across the take"))) diff --git a/frontend/test/arthur/parity_test.cljs b/frontend/test/arthur/parity_test.cljs deleted file mode 100644 index 8122a43..0000000 --- a/frontend/test/arthur/parity_test.cljs +++ /dev/null @@ -1,245 +0,0 @@ -(ns arthur.parity-test - "Diffs the CLJS port against js/ — the numeric oracle — on the identical - synthetic track. - - DELETABLE, and deliberately so. This namespace and test/parity/oracle.mjs go - together, in one commit, once the CLJS player renders the synthetic take - correctly (port-plan step 5). - - Parity proves the port is FAITHFUL, not that the answer is RIGHT. The JS is a - prototype and several of its conclusions contradict each other; a parity test - pins behaviour while the code moves, and a correctness test asserts something - that has been decided and stays. Conflating the two bakes the prototype's - mistakes into the rewrite and makes them permanent — so nothing in here is - allowed to outlive the move, and nothing in here is evidence that a number is - the number we want. - - Run test/parity/oracle.mjs first; `npm test` does." - (:require [cljs.test :refer [deftest is testing]] - [arthur.domain.geom :as geom] - [arthur.domain.landmarks :as lm] - [arthur.domain.palette :as pal] - [arthur.domain.raster :as raster] - [arthur.domain.ring :as ring] - [arthur.flow.condition :as condition] - [arthur.flow.measure.anchor :as anchor] - [arthur.flow.measure.mouth :as mouth] - [arthur.synth :as synth])) - -;; The port plan's number. A larger gap than this is a port bug, not float noise. -(def TOL 1e-9) - -(def oracle - (delay - (let [fs (js/require "fs") - path (str js/__dirname "/../test/parity/oracle.json")] - (when-not (.existsSync fs path) - (throw (ex-info (str "no oracle at " path - " — run `node test/parity/oracle.mjs` first") - {:path path}))) - (js->clj (js/JSON.parse (.readFileSync fs path "utf8")) :keywordize-keys true)))) - -;; Both sides get jitter of exactly zero, which is what makes the tracks -;; comparable at all: the JS uses Math.random and the CLJS a seeded generator. -(def track (delay (synth/synth-dense (:frames @oracle) {:rand-fn (constantly 0.5)}))) - -(defn- worst - "The largest absolute difference between two equally-shaped nested numeric - structures, and where it was, so a failure names the case." - [a b] - (let [seen (atom {:d -1 :at nil})] - (letfn [(walk [x y path] - (cond - (number? x) - (let [d (abs (- x y))] - (when (> d (:d @seen)) (reset! seen {:d d :at path :got x :want y}))) - (map? x) - (doseq [k (keys x)] (walk (get x k) (get y k) (conj path k))) - (sequential? x) - (do (when (not= (count x) (count y)) - (throw (ex-info "shape mismatch" {:at path :got (count x) :want (count y)}))) - (dotimes [i (count x)] (walk (nth x i) (nth y i) (conj path i)))) - :else nil))] - (walk a b [])) - @seen)) - -(defn- agrees? - "Assert two structures agree to TOL, reporting the worst offender." - [label a b] - (let [{:keys [d at got want]} (worst a b)] - (is (< d TOL) - (str label ": worst gap " d " at " (pr-str at) " (" got " vs " want ")")))) - -;; ---- the track itself ---- -;; -;; Everything below is meaningless if the two synths disagree, so this is -;; asserted first and separately: a track mismatch would otherwise surface as a -;; dozen numeric failures pointing nowhere near the cause. - -(deftest the-two-synths-produce-the-same-track - (let [js-track (:track @oracle)] - (is (= (count js-track) (count @track))) - (is (= (count (first js-track)) (count (first @track)))) - (agrees? "synthetic track" @track js-track))) - -;; ---- the tables ---- - -(deftest tables-were-transcribed-without-a-typo - (let [t (:tables @oracle)] - (is (= lm/RIGID (:RIGID t))) - (is (= lm/LIPS-OUTER (:LIPS_OUTER t))) - (is (= lm/EYE-R-RING (:EYE_R_RING t))) - (is (= lm/BROW-A-RING (:BROW_A_RING t))) - (is (= lm/FACE-OVAL (:FACE_OVAL t))))) - -;; ---- domain/ring ---- - -(deftest subsample-slots-agrees - ;; The oracle keys are "len/n", which js->clj reads as a NAMESPACED keyword — - ;; so the length is the namespace, not the first half of the name. - (doseq [[k want] (:subsampleSlots @oracle)] - (let [len (js/parseInt (namespace k)) - n (js/parseInt (name k))] - (is (= want (ring/subsample-slots len n)) - (str "subsample-slots(" len "," n ")"))))) - -(deftest offset-ring-agrees - (doseq [{:keys [d ring shutLid collapsed]} (:offsetRing @oracle)] - (let [src (mapv #(nth (first @track) %) lm/LIPS-OUTER)] - (agrees? (str "offset-ring(lips, " d ")") - (mapv #(select-keys % [:x :y]) (ring/offset-ring src d)) - (mapv #(select-keys % [:x :y]) ring))) - (agrees? (str "offset-ring(shut lid, " d ")") - (mapv #(select-keys % [:x :y]) - (ring/offset-ring [{:x -10 :y 0} {:x 0 :y -0.02} - {:x 10 :y 0} {:x 0 :y 0.02}] d)) - (mapv #(select-keys % [:x :y]) shutLid)) - ;; A vertex on the centroid has no outward direction. Both sides must leave - ;; it alone rather than emit NaN, and NaN != NaN would slip past `worst`. - (let [got (ring/offset-ring [{:x 0 :y 0} {:x 0 :y 0} {:x 0 :y 0}] d)] - (is (every? #(and (not (js/isNaN (:x %))) (not (js/isNaN (:y %)))) got) - (str "offset-ring(collapsed, " d ") is NaN-free")) - (is (every? #(and (not (js/isNaN (:x %))) (not (js/isNaN (:y %)))) collapsed) - "...and the oracle's is too, so this is parity and not a shared bug")))) - -;; ---- domain/geom: the two the port plan names ---- - -(deftest fit-similarity-agrees-on-a-known-transform - (let [{:keys [src dst fit]} (:known @oracle)] - (agrees? "fit-similarity (known transform)" - (geom/fit-similarity src dst) - fit))) - -(deftest fit-similarity-agrees-over-the-whole-shot - (let [rigid (mapv (fn [fr] (mapv #(nth fr %) lm/RIGID)) @track) - ref (:rigidRef @oracle)] - ;; Fitted against the ORACLE's reference, so this isolates fit-similarity - ;; from procrustes-mean instead of compounding the two. - (agrees? "fit-similarity over 72 frames" - (mapv #(geom/fit-similarity % ref) rigid) - (:transforms @oracle)))) - -(deftest procrustes-mean-agrees - (let [rigid (mapv (fn [fr] (mapv #(nth fr %) lm/RIGID)) @track)] - (agrees? "procrustes-mean" - (mapv #(select-keys % [:x :y]) (geom/procrustes-mean rigid)) - (mapv #(select-keys % [:x :y]) (:rigidRef @oracle))))) - -(deftest fit-residual-agrees - (let [rigid (mapv (fn [fr] (mapv #(nth fr %) lm/RIGID)) @track) - ref (:rigidRef @oracle)] - (agrees? "fit-residual" - (mapv (fn [r tf] (geom/fit-residual tf r ref)) - rigid (:transforms @oracle)) - (:residuals @oracle)))) - -;; ---- domain/geom: the smoothing knobs ---- - -(deftest moving-average-agrees-at-every-radius - (let [tx (mapv :tx (:transforms @oracle))] - (doseq [{:keys [radius vals]} (:movingAverage @oracle)] - (agrees? (str "moving-average radius " radius) - (geom/moving-average tx radius) - vals)))) - -(deftest smooth-transforms-agrees-at-every-radius - (doseq [{:keys [radius tfs]} (:smoothed @oracle)] - (agrees? (str "smooth-transforms radius " radius) - (geom/smooth-transforms (:transforms @oracle) radius) - tfs))) - -;; ---- domain/raster ---- - -(deftest raster-agrees-pixel-for-pixel - ;; Integer output, so this is EXACT equality and not TOL. The whole buffer is - ;; diffed rather than a pixel count: one scanline a pixel wide of the JS would - ;; read as a seam between two parts, not as an error, and a count would miss it. - (let [{:keys [w h buf]} (:raster @oracle) - fr (first @track) - ras (-> (raster/make w h) (raster/clear! 0))] - (raster/fill-poly! ras (mapv (fn [i] {:x (- (* (:x (nth fr i)) 320) 100) - :y (- (* (:y (nth fr i)) 200) 40)}) - lm/LIPS-OUTER) 2) - (raster/fill-poly! ras [{:x 8.5 :y 8.5} {:x 56.25 :y 8.5} - {:x 56.25 :y 40.75} {:x 8.5 :y 40.75}] 1) - (raster/fill-disc! ras 30.4 24.6 9.2 3 1) - (raster/fill-disc! ras 5.5 44.5 4 4) - (raster/fill-rect! ras 30.49 24.51 3 5 3) - (raster/fill-rect! ras 1 1 0 6) - (let [got (vec (array-seq (:buf ras))) - diff (keep-indexed (fn [i v] (when (not= v (nth buf i)) - {:at [(mod i w) (quot i w)] - :got v :want (nth buf i)})) - got)] - (is (empty? diff) - (str (count diff) " of " (* w h) " pixels differ, first few: " - (pr-str (vec (take 5 diff))))) - ;; A buffer that agreed because both sides drew nothing would pass the - ;; above, so check the drawing actually happened. - (is (> (count (distinct got)) 3) - (str "only " (pr-str (distinct got)) " indices present"))))) - -(deftest hex-to-rgb-agrees - (agrees? "palette rgb" pal/rgb (:paletteRgb @oracle))) - -;; ---- flow: stage 3 measure + stage 4 condition ---- -;; -;; The prototype's `stabilize` is three things: the anchor fit, the smoothing of -;; its parameters, and the mouth measured through the result. Here they are three -;; functions in two stages, so what is diffed is the COMPOSITION — a split that -;; agreed on every piece and not on the whole would be a split and not a port. - -(deftest stabilize-agrees-across-the-split-stages - (doseq [{:keys [aspect radius ref rigid transforms residual outer inner aperture]} - (:stabilize @oracle)] - (let [label (str "stabilize(aspect " aspect ", radius " radius ")") - fitted (anchor/fit {:aspect aspect} {:dense @track}) - anchd (condition/anchor {:anchor-avg radius} fitted) - got (mouth/measure {:aspect aspect} - {:dense @track :transforms (:transforms anchd)})] - (agrees? (str label " ref") (:ref fitted) ref) - (agrees? (str label " rigid") (:rigid fitted) rigid) - (agrees? (str label " transforms") (:transforms anchd) transforms) - (agrees? (str label " outer") (:outer got) outer) - (agrees? (str label " inner") (:inner got) inner) - (agrees? (str label " aperture") (:aperture got) aperture) - ;; The prototype takes the residual against the SMOOTHED transforms, because - ;; those were the ones in scope. `anchor/fit` takes it against the raw fit, - ;; deliberately: the number's job is to say whether the footage is - ;; stabilisable, and folding the smoothing error into it makes a setting look - ;; like a property of the shot. Parity is on the function, handed what the - ;; prototype handed it — so the divergence is a decision and not a drift. - (agrees? (str label " residual") - (anchor/residuals (:ref fitted) (:rigid fitted) (:transforms anchd)) - residual) - (when (zero? radius) - (agrees? (str label " residual, as fit reports it") (:residual fitted) residual))))) - -(deftest smooth-contours-agrees-at-every-radius - (let [fitted (anchor/fit {:aspect 0.5625} {:dense @track}) - rings (:outer (mouth/measure {:aspect 0.5625} - {:dense @track :transforms (:transforms fitted)}))] - (doseq [{:keys [radius outer]} (:smoothContours @oracle)] - (agrees? (str "condition/contours radius " radius) - (condition/contours {:contour-avg radius} rings) - outer)))) diff --git a/frontend/test/browser/take.mjs b/frontend/test/browser/take.mjs new file mode 100644 index 0000000..5f39d7d --- /dev/null +++ b/frontend/test/browser/take.mjs @@ -0,0 +1,300 @@ +// Drives a real Chrome at the running dev server and checks that the frozen +// take is a MOVING MOUTH on a canvas. +// +// This exists because port-plan step 5 is the first step whose done-criterion is +// a picture, and a picture cannot be asserted from cljs.test. A take that +// resolves to the right numbers and draws nothing would pass every assertion in +// arthur.flow.freeze-test: a blank canvas under a perfectly correct transport is +// the bug class unit tests miss, and it has already happened once here. +// +// No dependencies, deliberately. Playwright is not installed and CDP needs +// nothing: `node --experimental-websocket` has a global WebSocket and +// `--headless=new --remote-debugging-port=N` is the whole of the other side. +// +// cd frontend +// mise exec -- npx shadow-cljs compile app +// mise exec -- npx shadow-cljs watch app # or `server`, for :dev-http +// mise exec -- node --experimental-websocket test/browser/take.mjs +// +// Writes a PNG per sampled frame into test/browser/out/ so that "it drew +// something" can be checked by eye as well as by pixel count. + +import { spawn } from 'node:child_process'; +import { mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const OUT = join(HERE, 'out'); +const URL_ = process.env.ARTHUR_URL ?? 'http://localhost:8778/index.html'; +const PORT = 9333; +const CHROME = process.env.CHROME ?? + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'; + +// The palette's background, from domain/palette. A pixel of this colour is +// nothing drawn, and every check below is a count of pixels that are not it. +const BG = [0x12, 0x14, 0x1c]; + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +let failures = 0; +function check(ok, label, detail = '') { + console.log(`${ok ? ' ok ' : ' FAIL'} ${label}${detail ? ` — ${detail}` : ''}`); + if (!ok) failures++; +} + +// --------------------------------------------------------------------------- +// the CDP connection + +async function connect() { + const profile = mkdtempSync(join(tmpdir(), 'arthur-chrome-')); + const chrome = spawn(CHROME, [ + '--headless=new', + `--remote-debugging-port=${PORT}`, + `--user-data-dir=${profile}`, + // The take plays against audio.wav, and the clock IS the audio element, so + // without these the frame never advances and "plays back" cannot be checked + // at all — the failure would look like a broken rAF loop. + '--autoplay-policy=no-user-gesture-required', + '--mute-audio', + '--no-first-run', + '--no-default-browser-check', + '--disable-gpu', + '--window-size=1200,900', + URL_, + ], { stdio: ['ignore', 'ignore', 'pipe'] }); + chrome.stderr.on('data', () => {}); + + let wsUrl = null; + for (let i = 0; i < 100 && !wsUrl; i++) { + await sleep(100); + try { + const targets = await fetch(`http://127.0.0.1:${PORT}/json/list`).then((r) => r.json()); + wsUrl = targets.find((t) => t.type === 'page' && t.url.includes('index.html')) + ?.webSocketDebuggerUrl; + } catch { /* not up yet */ } + } + if (!wsUrl) throw new Error(`Chrome never offered a page target on ${PORT}`); + + const ws = new WebSocket(wsUrl); + await new Promise((res, rej) => { ws.onopen = res; ws.onerror = rej; }); + + let id = 0; + const pending = new Map(); + const logs = []; + ws.onmessage = (ev) => { + const msg = JSON.parse(ev.data); + if (msg.id && pending.has(msg.id)) { + const { res, rej } = pending.get(msg.id); + pending.delete(msg.id); + msg.error ? rej(new Error(JSON.stringify(msg.error))) : res(msg.result); + } else if (msg.method === 'Runtime.consoleAPICalled' && msg.params.type === 'error') { + logs.push(msg.params.args.map((a) => a.value ?? a.description).join(' ')); + } else if (msg.method === 'Runtime.exceptionThrown') { + logs.push(msg.params.exceptionDetails.text + ' ' + + (msg.params.exceptionDetails.exception?.description ?? '')); + } + }; + const send = (method, params = {}) => + new Promise((res, rej) => { + const n = ++id; + pending.set(n, { res, rej }); + ws.send(JSON.stringify({ id: n, method, params })); + }); + + await send('Runtime.enable'); + await send('Page.enable'); + return { + send, logs, + async eval(expr) { + const r = await this.send('Runtime.evaluate', { + expression: expr, awaitPromise: true, returnByValue: true, + }); + if (r.exceptionDetails) { + throw new Error(r.exceptionDetails.exception?.description ?? + r.exceptionDetails.text); + } + return r.result.value; + }, + async shot(name) { + const r = await this.send('Page.captureScreenshot', { format: 'png' }); + writeFileSync(join(OUT, `${name}.png`), Buffer.from(r.data, 'base64')); + }, + close() { ws.close(); chrome.kill(); rmSync(profile, { recursive: true, force: true }); }, + }; +} + +// --------------------------------------------------------------------------- +// what we ask the page +// +// Every probe reads the canvas's own pixels rather than a screenshot: the CSS +// scales the stage up by 2 with image-rendering:pixelated, so a screenshot is +// four pixels per raster pixel and is for looking at, not for counting. + +const PROBE = `(() => { + const c = document.querySelector('canvas.stage'); + if (!c) return null; + const d = c.getContext('2d').getImageData(0, 0, c.width, c.height).data; + const bg = [${BG.join(',')}]; + let drawn = 0; + const tones = new Set(); + let cx = 0, cy = 0; + for (let i = 0; i < d.length; i += 4) { + if (d[i] === bg[0] && d[i+1] === bg[1] && d[i+2] === bg[2]) continue; + drawn++; + tones.add((d[i] << 16) | (d[i+1] << 8) | d[i+2]); + const p = i / 4; + cx += p % c.width; cy += Math.floor(p / c.width); + } + // A cheap 32-bit hash of the whole buffer: two frames with the same drawn + // count can still be different pictures, and "the mouth moved" is a question + // about the picture. + let h = 2166136261; + for (let i = 0; i < d.length; i += 4) { h ^= d[i] + d[i+1] * 31 + d[i+2] * 131; h = Math.imul(h, 16777619); } + return { + w: c.width, h: c.height, drawn, tones: [...tones].length, + cx: drawn ? cx / drawn : null, cy: drawn ? cy / drawn : null, + hash: h >>> 0, + frame: document.querySelector('.readout span')?.textContent ?? '', + scene: [...document.querySelectorAll('.transport .row button')] + .filter((b) => b.classList.contains('on')).map((b) => b.textContent), + }; +})()`; + +const SEEK = (f) => `(() => { + const el = document.querySelector('input.scrub'); + // React installs its own value setter on the element, so assigning .value + // directly updates the DOM and not React's idea of it, and onChange never + // fires. The prototype-level setter plus a bubbling 'input' event is what + // React's synthetic onChange actually listens for. + const set = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set; + set.call(el, '${f}'); + el.dispatchEvent(new Event('input', { bubbles: true })); + return el.value; +})()`; + +const CLICK = (label) => `(() => { + const b = [...document.querySelectorAll('.transport button')] + .find((b) => b.textContent.trim() === ${JSON.stringify(label)}); + if (!b) return false; + b.click(); + return true; +})()`; + +// --------------------------------------------------------------------------- + +async function main() { + rmSync(OUT, { recursive: true, force: true }); + mkdirSync(OUT, { recursive: true }); + + const page = await connect(); + try { + // Mounted, and painting. Polled rather than waited on a fixed delay: the + // canvas :ref fires after the loop starts, so there genuinely is a window in + // which the page is up and the canvas is blank. + let probe = null; + for (let i = 0; i < 100; i++) { + probe = await page.eval(PROBE); + if (probe && probe.drawn > 0) break; + await sleep(100); + } + if (!probe) throw new Error('no canvas.stage on the page — is `shadow-cljs watch app` running?'); + + console.log(`\ncanvas ${probe.w}x${probe.h}, scene ${JSON.stringify(probe.scene)}`); + + check(probe.scene.includes('take'), 'the take is the scene that opens'); + check(probe.w === 320 && probe.h === 200, 'the canvas is the stage size', + `${probe.w}x${probe.h}`); + check(probe.drawn > 200, 'the first frame is not blank', `${probe.drawn} px drawn`); + + // --- it is a mouth: two tones, one inside the other --- + // + // The three-layer structure is what makes a flat shape read as an opening + // rather than a blob, so the interior being a SECOND tone is the check that + // this is a mouth and not one polygon. + await page.eval(SEEK(10)); // beat 1: wide open + await sleep(120); + const open = await page.eval(PROBE); + await page.shot('take-open'); + check(open.tones >= 2, 'an open mouth draws an outline and an interior', + `${open.tones} tones`); + + await page.eval(SEEK(28)); // beat 3: shut + await sleep(120); + const shut = await page.eval(PROBE); + await page.shot('take-shut'); + check(shut.tones === 1, 'a shut mouth draws the outline alone', + `${shut.tones} tones`); + check(open.hash !== shut.hash, 'the open and the shut mouth are different pictures'); + check(open.drawn > shut.drawn, 'the open mouth covers more of the stage', + `${open.drawn} vs ${shut.drawn} px`); + + // --- it moves under the head, and the head is a channel --- + const filmed = []; + for (const f of [0, 40, 80, 120, 160, 200]) { + await page.eval(SEEK(f)); + await sleep(120); + filmed.push(await page.eval(PROBE)); + } + check(new Set(filmed.map((p) => p.hash)).size === filmed.length, + 'every sampled frame is a different picture'); + const xs = filmed.map((p) => p.cx); + check(Math.max(...xs) - Math.min(...xs) > 8, + 'as filmed, the head carries the mouth across the stage', + `centroid x spans ${(Math.max(...xs) - Math.min(...xs)).toFixed(1)} px`); + + check(await page.eval(CLICK('locked')), 'the locked take is selectable'); + await sleep(200); + const locked = []; + for (const f of [0, 40, 80, 120, 160, 200]) { + await page.eval(SEEK(f)); + await sleep(120); + locked.push(await page.eval(PROBE)); + } + await page.shot('take-locked'); + const lxs = locked.map((p) => p.cx); + check(locked.every((p) => p.drawn > 200), 'the locked take draws too'); + // The same blocks with one node's channels written differently: the mouth + // still articulates, and the head no longer wanders. That is the claim + // "stabilisation is a channel, not a mode", by eye. + check(Math.max(...lxs) - Math.min(...lxs) < (Math.max(...xs) - Math.min(...xs)) / 2, + 'locked, the head holds still while the mouth still articulates', + `centroid x spans ${(Math.max(...lxs) - Math.min(...lxs)).toFixed(1)} px`); + check(new Set(locked.map((p) => p.hash)).size > 3, + 'and it is still a performance, not a still frame'); + + // --- it PLAYS, against the audio clock --- + check(await page.eval(CLICK('take')), 'back to the take'); + await sleep(150); + await page.eval(SEEK(0)); + await sleep(150); + check(await page.eval(CLICK('play')), 'play is clickable'); + const during = []; + for (let i = 0; i < 8; i++) { await sleep(180); during.push(await page.eval(PROBE)); } + await page.eval(CLICK('pause')); + // The readout is "frame 12 / 229", so the first run of digits is the + // playhead. Parsed rather than reached for in app-db on purpose: what the + // page SHOWS is what a person would check, and the readout agreeing with the + // picture is half of what the transport is for. + const nums = during.map((p) => parseInt((p.frame.match(/\d+/) ?? [NaN])[0], 10)); + check(nums[nums.length - 1] > nums[0] + 5, 'the playhead advances under the clock', + `frame ${nums[0]} -> ${nums[nums.length - 1]}`); + check(new Set(during.map((p) => p.hash)).size > 4, + 'and the picture changes while it runs', + `${new Set(during.map((p) => p.hash)).size} distinct of ${during.length}`); + await page.shot('take-playing'); + + check(page.logs.length === 0, 'no errors on the console', + page.logs.slice(0, 3).join(' | ')); + } finally { + page.close(); + } + + console.log(`\n${failures ? `${failures} FAILED` : 'all checks passed'}` + + ` — screenshots in test/browser/out/\n`); + process.exit(failures ? 1 : 0); +} + +main().catch((e) => { console.error(e); process.exit(2); }); diff --git a/frontend/test/parity/.gitignore b/frontend/test/parity/.gitignore deleted file mode 100644 index 8e8c2b9..0000000 --- a/frontend/test/parity/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Generated by oracle.mjs on every `npm test`. Not committed: it is 1.2MB of -# derived numbers, and a stale copy would make the parity suite pass against -# yesterday's oracle. -oracle.json diff --git a/frontend/test/parity/oracle.mjs b/frontend/test/parity/oracle.mjs deleted file mode 100644 index a8ed287..0000000 --- a/frontend/test/parity/oracle.mjs +++ /dev/null @@ -1,146 +0,0 @@ -// Runs the JS prototype — the numeric oracle — and writes its answers to JSON -// for arthur.parity-test to diff against the CLJS port. -// -// DELETABLE. This file and arthur.parity-test go together, in one commit, once -// the CLJS player renders the synthetic take correctly (port-plan step 5). A -// parity test pins behaviour while the code moves; keeping it afterwards would -// bake the prototype's mistakes into the rewrite and make them permanent. -// -// Math.random is stubbed to a constant so both sides get the IDENTICAL track: -// js/synth.js reads Math.random at call time, not at import time, so assigning -// it here — before synthDense is called below — is enough, and js/ stays -// untouched. 0.5 makes every (Math.random() - 0.5) jitter term exactly zero, -// which is also what `:rand-fn (constantly 0.5)` does on the CLJS side. -Math.random = () => 0.5; - -import { writeFileSync } from 'node:fs'; -import { fileURLToPath } from 'node:url'; -import { dirname, join } from 'node:path'; - -import { RIGID, LIPS_OUTER, EYE_R_RING, BROW_A_RING, FACE_OVAL, - subsampleSlots } from '../../../js/landmarks.js'; -import { fitSimilarity, applySim, fitResidual, procrustesMean, - movingAverage, smoothTransforms, offsetRing } from '../../../js/mathutil.js'; -import { stabilize, smoothContours } from '../../../js/pipeline.js'; -import { synthDense } from '../../../js/synth.js'; -import { IndexedRaster, hexToRgb } from '../../../js/raster.js'; - -const FRAMES = 72; -const track = synthDense(FRAMES); -const rigid = track.map((f) => RIGID.map((i) => f[i])); - -// The two the port plan names explicitly, plus everything else in mathutil.js: -// a function nobody diffed is a function nobody ported. -const ref = procrustesMean(rigid); -const tfs = rigid.map((r) => fitSimilarity(r, ref)); - -const strip = (p) => ({ x: p.x, y: p.y, z: p.z ?? 0 }); -const xy = (p) => ({ x: p.x, y: p.y }); -const ring = (r) => r.map(xy); -const stripTf = (t) => ({ s: t.s, theta: t.theta, tx: t.tx, ty: t.ty }); - -// A known transform recovered exactly, which is the same case the CLJS unit test -// asserts — here so a disagreement can be localised to the fit rather than to -// the track. -const knownSrc = [{ x: 0, y: 0 }, { x: 1, y: 0 }, { x: 0, y: 1 }, { x: 2, y: 3 }]; -const knownTruth = { s: 1.7, theta: 0.6, tx: 4, ty: -2 }; -const knownDst = knownSrc.map((p) => applySim(knownTruth, p)); - -const out = { - frames: FRAMES, - track: track.map((f) => f.map(strip)), - rigidRef: ref.map(strip), - transforms: tfs.map(stripTf), - residuals: rigid.map((r, i) => fitResidual(tfs[i], r, ref)), - smoothed: [0, 1, 2, 5].map((radius) => ({ - radius, tfs: smoothTransforms(tfs, radius).map(stripTf), - })), - known: { src: knownSrc, truth: knownTruth, dst: knownDst, - fit: stripTf(fitSimilarity(knownSrc, knownDst)) }, - // tx over the shot is the sway; it is the one-dimensional series the smoothing - // knob actually acts on, so it is what movingAverage gets diffed on. - movingAverage: [0, 1, 2, 3, 7].map((radius) => ({ - radius, vals: movingAverage(tfs.map((t) => t.tx), radius), - })), - // stabilize(), which the CLJS side reaches as three stages: the anchor fit, - // the conditioning of its parameters, and the mouth measured through the - // result. Diffing the composition is the point — a split that agreed on each - // piece and not on the whole would be a split, not a port. - // - // aspect 1 is in here to isolate the rest, and 0.5625 (a 1080x1920 phone clip) - // because it is the only value that exercises the anisotropy correction at all: - // at aspect 1 `pick` is the identity and a port that dropped it entirely would - // pass. radius 0 and 2 because the split moved the smoothing OUT of the middle - // of this function, so agreeing only at radius 0 would prove nothing about it. - stabilize: [{ aspect: 1, radius: 0 }, - { aspect: 0.5625, radius: 0 }, - { aspect: 0.5625, radius: 2 }].map(({ aspect, radius }) => { - const st = stabilize(track, radius, aspect); - return { - aspect, radius, - ref: st.ref.map(xy), - rigid: st.rigid.map(ring), - transforms: st.transforms.map(stripTf), - residual: st.residual, - outer: st.outer.map(ring), - inner: st.inner.map(ring), - aperture: st.aperture, - }; - }), - // The contour knob, on the ring it is actually dragged for. The rings are the - // full 20 slots and not a subsample, which is where the CLJS side differs in - // arrangement and must not differ in numbers: the prototype subsamples before - // smoothing, the port smooths before subsampling, and the two commute because - // both operations are per-slot. - smoothContours: (() => { - const st = stabilize(track, 0, 0.5625); - return [0, 1, 3].map((radius) => ({ - radius, outer: smoothContours(st.outer, radius).map(ring), - })); - })(), - offsetRing: [0, 0.5, 2, -1].map((d) => ({ - d, - ring: offsetRing(LIPS_OUTER.map((i) => track[0][i]), d).map(strip), - // The degenerate case: a shut lid is a flat sliver and must still open into - // a band, and a ring collapsed onto its own centroid must not emit NaN. - shutLid: offsetRing([{ x: -10, y: 0 }, { x: 0, y: -0.02 }, - { x: 10, y: 0 }, { x: 0, y: 0.02 }], d).map(strip), - collapsed: offsetRing([{ x: 0, y: 0 }, { x: 0, y: 0 }, { x: 0, y: 0 }], d).map(strip), - })), - subsampleSlots: Object.fromEntries( - [[20, 4], [20, 6], [20, 8], [20, 10], [20, 16], [16, 4], [16, 6], [16, 12], - [10, 4], [10, 6], [10, 10], [36, 8]] - .map(([len, n]) => [`${len}/${n}`, subsampleSlots(len, n)])), - tables: { RIGID, LIPS_OUTER, EYE_R_RING, BROW_A_RING, FACE_OVAL }, - // The raster is integer output, so parity here is EXACT equality, not 1e-9. - // One scanline drawn one pixel wide of the JS would read as a seam between two - // parts rather than as an error, which is why the whole buffer is diffed and - // not a pixel count. - // - // toImageData is not exercised: it needs an ImageData, the CLJS side returns - // plain bytes on purpose so domain/ stays DOM-free, and the palette expansion - // is asserted directly in arthur.domain.raster-test instead. - raster: (() => { - const r = new IndexedRaster(64, 48); - r.clear(0); - // A real mouth ring at raster scale, so the scanline fill is diffed on a - // shape with fractional coordinates and non-convex spans rather than on an - // axis-aligned box that would agree even if the rounding were wrong. - r.fillPoly(LIPS_OUTER.map((i) => ({ x: track[0][i].x * 320 - 100, - y: track[0][i].y * 200 - 40 })), 2); - r.fillPoly([{ x: 8.5, y: 8.5 }, { x: 56.25, y: 8.5 }, - { x: 56.25, y: 40.75 }, { x: 8.5, y: 40.75 }], 1); - r.fillDisc(30.4, 24.6, 9.2, 3, 1); // stencilled by the box - r.fillDisc(5.5, 44.5, 4, 4); // unstencilled, clipped by the edge - r.fillRect(30.49, 24.51, 3, 5, 3); // stencilled by the disc - r.fillRect(1, 1, 0, 6); // size 0 draws nothing - return { w: r.w, h: r.h, buf: Array.from(r.buf) }; - })(), - paletteRgb: ['#12141c', '#b07a5a', '#7a4f3a', '#24161a', '#d9cfc2', - '#c9c3b4', '#4a5468', '#171a22', '#3a2a22'].map(hexToRgb), -}; - -const here = dirname(fileURLToPath(import.meta.url)); -const path = join(here, 'oracle.json'); -writeFileSync(path, JSON.stringify(out)); -console.log(`oracle: ${FRAMES} frames -> ${path}`);