noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Split clips from timelines

Browse source
This commit is contained in:
Olive Vaughn 2026-09-28 02:33:26 -04:00
parent 27bfe18bee
commit 9778b9023b
31 changed files with 794 additions and 444 deletions
Download patch file Download diff file Expand all files Collapse all files

170
frontend/test/arthur/flow/freeze_test.cljs
View file

@ -11,12 +11,13 @@
(:require [cljs.test :refer [deftest is testing]]
[arthur.demo.take :as take]
[arthur.domain.channel :as ch]
[arthur.domain.clip :as clip]
[arthur.domain.geom :as geom]
[arthur.domain.node :as node]
[arthur.domain.palette :as pal]
[arthur.domain.raster :as raster]
[arthur.domain.ring :as ring]
[arthur.domain.scene :as scene]
[arthur.domain.timeline :as timeline]
[arthur.flow.freeze :as freeze]))
(def ^:private W 320)
@ -24,13 +25,17 @@
;; The whole vertical slice, exactly as the page builds it. Asserting against the
;; page's own clip rather than against a fixture built here is deliberate: a
;; fixture is a second scene nobody looks at, and the one that renders is the one
;; fixture is a second document nobody looks at, and the one that renders is the one
;; that has to be right.
(def clip (delay @take/frozen))
(def scene* (delay (:scene @clip)))
(def store (delay (:store @clip)))
(def frozen (delay @take/frozen))
(def clip* (delay (:clip @frozen)))
;; The ROOT TIMELINE, which is what an evaluator takes. `clip*` is the document —
;; `:fps`, the stage, the analysis, the tracking identities — and `tl*` is the bag
;; of nodes in its frame space. `timeline/resolver` refuses the wrong one loudly.
(def tl* (delay (clip/root @clip*)))
(def store (delay (:store @frozen)))
(defn- node [id] (get-in @scene* [:nodes id]))
(defn- node [id] (get-in @tl* [:nodes id]))
(defn- chan [id path] (get-in (node id) [:channels path]))
(defn- block-of
@ -44,16 +49,19 @@
(let [v (ch/value-at (chan id [:geom :pts]) f @store)]
(mapv #(ch/component v %) (range (.-length v)))))
(defn- ops-at [sc f]
((scene/resolver sc @store pal/index-of) f))
(defn- ops-at
"Ops for one frame of a TIMELINE."
[tl f]
((timeline/resolver tl @store pal/index-of) f))
(defn- render
"One frame of a scene into a byte buffer. The stage's size comes off the clip,
because project dimensions are the project's and not the footage's."
[sc f]
(let [r (raster/make (:width sc) (:height sc))]
"One frame of a CLIP into a byte buffer. The stage's size comes off the clip and
the ops off its root timeline, because project dimensions are the project's and
not the footage's — and not a timeline's either."
[c f]
(let [r (raster/make (:width c) (:height c))]
(raster/clear! r (get pal/index-of :bg))
(raster/draw-ops! r (ops-at sc f))
(raster/draw-ops! r (ops-at (clip/root c) f))
(vec (array-seq (:buf r)))))
(defn- drawn
@ -64,25 +72,27 @@
;; ---------------------------------------------------------------------------
;; the shape of what came out
(deftest the-frozen-take-is-a-valid-scene-in-every-head-mode
;; `scene/problems` is total by construction, so this is safe to run over data
;; before the data is trusted — which is what it is for.
(deftest the-frozen-take-is-a-valid-clip-in-every-head-mode
;; `clip/problems` is total by construction, so this is safe to run over data
;; before the data is trusted — which is what it is for. It checks the clip's
;; fields, every timeline in it and the tracking identities, so it is the whole
;; of what a save would refuse.
(doseq [mode [:as-filmed :locked]]
(let [sc (freeze/head-mode {:mode mode} @clip)]
(is (empty? (scene/problems sc)) (str mode ": " (pr-str (scene/problems sc))))))
(let [sc (freeze/head-mode {:mode :per-plate :kept #{0 12 40 88 150}} @clip)]
(is (empty? (scene/problems sc)) (pr-str (scene/problems sc)))))
(let [c (freeze/head-mode {:mode mode} @frozen)]
(is (empty? (clip/problems c)) (str mode ": " (pr-str (clip/problems c))))))
(let [c (freeze/head-mode {:mode :per-plate :kept #{0 12 40 88 150}} @frozen)]
(is (empty? (clip/problems c)) (pr-str (clip/problems c)))))
(deftest the-tree-is-the-one-the-model-specifies
;; :face is AUTHORED and :head is MEASURED, and they are two nodes because two
;; different things want that transform. A group node is free; keeping the
;; hand-placed and the measured transform apart is the whole reason the
;; transform is decomposed in the first place.
(is (= [:face :root] (scene/lineage (:nodes @scene*) :face)))
(is (= [:head :face :root] (scene/lineage (:nodes @scene*) :head)))
(is (= [:mouth :head :face :root] (scene/lineage (:nodes @scene*) :mouth)))
(is (= [:face :root] (timeline/lineage (:nodes @tl*) :face)))
(is (= [:head :face :root] (timeline/lineage (:nodes @tl*) :head)))
(is (= [:mouth :head :face :root] (timeline/lineage (:nodes @tl*) :mouth)))
(is (= [:mouth-in :mouth :head :face :root]
(scene/lineage (:nodes @scene*) :mouth-in)))
(timeline/lineage (:nodes @tl*) :mouth-in)))
;; Exposure lives on the clip root and inherits strictly.
(is (= {:mode :map :expose 2} (:time (node :root))))
(is (every? #(nil? (:time (node %))) [:face :head :mouth :mouth-in])))
@ -186,8 +196,8 @@
;; of the two says it is. A test that recomputed the chain would only be
;; checking arithmetic against itself; this checks `node/local!`, `node/world!`
;; and `emit` as well.
(let [sc (freeze/head-mode {:mode :as-filmed} @clip)
res (scene/resolver sc @store pal/index-of)
(let [c (freeze/head-mode {:mode :as-filmed} @frozen)
res (timeline/resolver (clip/root c) @store pal/index-of)
k (first (:value (chan :face [:xform :scale])))
anc (:value (chan :face [:xform :anchor]))
pos (:value (chan :face [:xform :pos]))
@ -228,9 +238,9 @@
(deftest the-three-head-modes-are-the-three-channel-shapes
(let [kept #{0 12 40 88 150}
of (fn [sc path] (get-in sc [:nodes :head :channels path]))]
of (fn [c path] (get-in (clip/nodes c) [:head :channels path]))]
(testing "locked is framed identity"
(let [sc (freeze/head-mode {:mode :locked} @clip)]
(let [sc (freeze/head-mode {:mode :locked} @frozen)]
(is (= [:framed :framed :framed]
(mapv #(ch/describe (of sc %))
[[:xform :pos] [:xform :rot] [:xform :scale]])))
@ -238,12 +248,12 @@
(is (= 0.0 (:value (of sc [:xform :rot]))))
(is (= [1.0 1.0] (:value (of sc [:xform :scale]))))))
(testing "as filmed is dense"
(let [sc (freeze/head-mode {:mode :as-filmed} @clip)]
(let [sc (freeze/head-mode {:mode :as-filmed} @frozen)]
(is (= [:dense :dense :dense]
(mapv #(ch/describe (of sc %))
[[:xform :pos] [:xform :rot] [:xform :scale]])))))
(testing "per plate is keyed at exactly the kept frames"
(let [sc (freeze/head-mode {:mode :per-plate :kept kept} @clip)]
(let [sc (freeze/head-mode {:mode :per-plate :kept kept} @frozen)]
(doseq [path [[:xform :pos] [:xform :rot] [:xform :scale]]]
(is (= :keyed (ch/describe (of sc path))))
(is (= (sort kept) (sort (keys (:keys (of sc path))))))
@ -262,25 +272,29 @@
;; It has to be impossible for the toggle to move something a hand placed, and
;; it has to be a DOCUMENT edit: tier 1, undoable, syncable, instant, and not a
;; reason to re-analyse.
(let [a (freeze/head-mode {:mode :as-filmed} @clip)
b (freeze/head-mode {:mode :locked} @clip)
c (freeze/head-mode {:mode :per-plate :kept #{0 40}} @clip)]
(doseq [sc [b c]]
(is (= (get-in a [:nodes :face]) (get-in sc [:nodes :face]))
(let [a (freeze/head-mode {:mode :as-filmed} @frozen)
b (freeze/head-mode {:mode :locked} @frozen)
c (freeze/head-mode {:mode :per-plate :kept #{0 40}} @frozen)]
(doseq [x [b c]]
(is (= (get (clip/nodes a) :face) (get (clip/nodes x) :face))
":face moved")
(is (= (dissoc (:nodes a) :head) (dissoc (:nodes sc) :head))
(is (= (dissoc (clip/nodes a) :head) (dissoc (clip/nodes x) :head))
"a node other than :head changed")
;; The measurement does not go away when the head is locked: always measure,
;; always store factored, toggle the parent.
(is (= (get-in a [:nodes :head :measured]) (get-in sc [:nodes :head :measured]))))))
(is (= (get-in (clip/nodes a) [:head :measured])
(get-in (clip/nodes x) [:head :measured])))
;; And nothing above the timeline moved either: the toggle is one node's
;; channels, so the clip's own fields and its other timelines are untouched.
(is (= (dissoc a :timelines) (dissoc x :timelines))))))
(deftest a-head-mode-that-is-not-one-of-the-three-is-refused
(is (thrown-with-msg? ExceptionInfo #"not one of the three channel shapes"
(freeze/head-mode {:mode :stabilised} @clip)))
(freeze/head-mode {:mode :stabilised} @frozen)))
;; The kept-frame set belongs to the plate strip, not to measurement, so freeze
;; cannot invent one.
(is (thrown-with-msg? ExceptionInfo #"kept-frame set"
(freeze/head-mode {:mode :per-plate} @clip))))
(freeze/head-mode {:mode :per-plate} @frozen))))
;; ---------------------------------------------------------------------------
;; the face: authored, and what makes makeXform deletable
@ -290,7 +304,7 @@
;; transform on a node, which a hand can revise, and it claims no generator that
;; would offer to overwrite it.
(doseq [path [[:xform :pos] [:xform :rot] [:xform :scale] [:xform :anchor]]]
(let [c (get (node/channels (get-in @scene* [:nodes :face])) path)]
(let [c (get (node/channels (node :face)) path)]
(is (= :framed (ch/describe c)) (str path " is not framed"))
(is (nil? (:generated c)) (str path " claims provenance")))))
@ -314,18 +328,20 @@
;; re-measuring anything.
(let [big (freeze/clip (assoc take/params :stage [640 480] :name "big")
@take/measured)
key-of (fn [sc] (:store (:dense (get-in sc [:nodes :mouth :channels [:geom :pts]]))))]
(is (= [640 480] [(:width (:scene big)) (:height (:scene big))]))
key-of (fn [c] (:store (:dense (get-in (clip/nodes c)
[:mouth :channels [:geom :pts]]))))]
(is (= [640 480] [(:width (:clip big)) (:height (:clip big))]))
;; Stronger than it was, and for free: the stage is not an input to tier 2, so
;; the two clips do not merely hold equal bytes — they name the SAME BLOCK, and
;; a stage change cannot invalidate a bake. The clip's name is not an input
;; either, which is why "big" and "take" still agree.
(is (= (key-of @scene*) (key-of (:scene big)))
(is (= (key-of @clip*) (key-of (:clip big)))
"a different stage is a different document over the same tier 2")
(is (= (vec (array-seq (:data (get @store (key-of @scene*)))))
(vec (array-seq (:data (get (:store big) (key-of (:scene big)))))))
(is (= (vec (array-seq (:data (get @store (key-of @clip*)))))
(vec (array-seq (:data (get (:store big) (key-of (:clip big)))))))
"the geometry is the same numbers at either stage size")
(is (not= (:value (get-in (:scene big) [:nodes :face :channels [:xform :scale]]))
(is (not= (:value (get-in (clip/nodes (:clip big))
[:face :channels [:xform :scale]]))
(:value (chan :face [:xform :scale]))))))
;; ---------------------------------------------------------------------------
@ -355,7 +371,7 @@
;; interior comes and goes.
(is (nil? (chan :mouth [:vis])))
(doseq [f (range 0 take/frames 9)]
(is (some #(= :mouth (:node %)) (ops-at @scene* f))
(is (some #(= :mouth (:node %)) (ops-at @tl* f))
(str "frame " f " drew no mouth outline"))))
;; ---------------------------------------------------------------------------
@ -384,13 +400,15 @@
;; `:generated` out of the document and the frame is the same bytes. If this
;; ever fails, a rotoscoped part and a hand-drawn one have stopped being the
;; same data.
(let [stripped (update @scene* :nodes
(fn [ns] (into {} (map (fn [[id n]]
[id (update n :channels
#(into {} (map (fn [[p c]] [p (dissoc c :generated)])) %))]))
ns)))]
(let [strip-node (fn [n]
(update n :channels
#(into {} (map (fn [[p c]] [p (dissoc c :generated)])) %)))
stripped (clip/update-root
@clip*
update :nodes
#(into {} (map (fn [[id n]] [id (strip-node n)])) %))]
(doseq [f (range 0 take/frames 17)]
(is (= (render @scene* f) (render stripped f))
(is (= (render @clip* f) (render stripped f))
(str "frame " f " differs with provenance removed")))))
;; ---------------------------------------------------------------------------
@ -401,21 +419,23 @@
presence {:eye-r (mapv #(not (contains? gap %)) (range take/frames))}
c (freeze/clip (assoc take/params :name "one-eye-gappy")
(assoc @take/measured :presence presence))
sc (:scene c)
tl (clip/root (:clip c))
at (fn [id f]
(ch/value-at (get-in sc [:nodes id :channels [:geom :pts]]) f (:store c)))]
(is (= [:eye-r :eye-l] (get-in sc [:groups :eyes-1 :members])))
(is (= :eye-r (get-in sc [:features :eye-r :id])))
(ch/value-at (get-in (:nodes tl) [id :channels [:geom :pts]]) f (:store c)))]
;; The identities are the CLIP's, and a gap does not touch them: a feature keeps
;; its id and its pair membership across the frames it was not observed on.
(is (= [:eye-r :eye-l] (get-in (:clip c) [:groups :eyes-1 :members])))
(is (= :eye-r (get-in (:clip c) [:features :eye-r :id])))
(is (not (ch/nothing? (at :eye-r 39))))
(is (ch/nothing? (at :eye-r 50)))
(is (not (ch/nothing? (at :eye-r 60))))
(is (not (ch/nothing? (at :eye-l 50))))
(is (not (ch/nothing? (at :mouth 50))))
(let [drawn-nodes (into #{} (map :node) ((scene/resolver sc (:store c) pal/index-of) 50))]
(let [drawn-nodes (into #{} (map :node) ((timeline/resolver tl (:store c) pal/index-of) 50))]
(is (not (contains? drawn-nodes :eye-r)))
(is (contains? drawn-nodes :eye-l))
(is (contains? drawn-nodes :mouth)))
(is (empty? (scene/problems sc)))))
(is (empty? (clip/problems (:clip c))))))
(deftest each-dense-track-follows-its-own-features-presence
;; Every feature is given a DIFFERENT gap, so a track wired to the wrong one
@ -437,9 +457,9 @@
windows)
c (freeze/clip (assoc take/params :name "per-track-gaps")
(assoc @take/measured :presence presence))
sc (:scene c)
tl (clip/root (:clip c))
at (fn [id path f]
(ch/value-at (get-in sc [:nodes id :channels path]) f (:store c)))
(ch/value-at (get-in (:nodes tl) [id :channels path]) f (:store c)))
;; Node, channel, and the feature whose gap it must follow. Every dense
;; track of the eye, iris, brow and brow-position blocks appears once.
tracks [[:eye-r [:geom :pts] :eye-r]
@ -452,7 +472,7 @@
[:brow-l [:geom :pts] :brow-l]
[:brow-r [:xform :pos] :brow-r]
[:brow-l [:xform :pos] :brow-l]]]
(is (empty? (scene/problems sc)))
(is (empty? (clip/problems (:clip c))))
(doseq [[id path owner] tracks
[feature gap] windows
f gap]
@ -465,10 +485,10 @@
(deftest teeth-follow-the-mouth-through-their-stencil-and-not-through-a-mask
;; Teeth are their own feature, so that they can carry their own :area :teeth
;; parameters — which means an occluded MOUTH sets no absence bit on them. They
;; do not need one: they are stencilled by :mouth-in, and scene/finish drops a
;; do not need one: they are stencilled by :mouth-in, and timeline/finish drops a
;; node whose stencil drew nothing. So the coupling is real, and it is the
;; stencil rule rather than the mask that enforces it. The rule itself is
;; asserted in scene-test; what is pinned here is the wiring that relies on it.
;; asserted in timeline-test; what is pinned here is the wiring that relies on it.
(let [gap (set (range 40 60))
teeth-in {;; The mouth's own inner ring standing in for a pixel-derived
;; contour: real geometry and no nils, so every frame has a
@ -485,7 +505,7 @@
;; Hoisted: the resolver caches its order and reuses its buffers, so the
;; node ids come out before the next frame is asked for.
nodes-at (fn [c]
(let [r (scene/resolver (:scene c) (:store c) pal/index-of)]
(let [r (timeline/resolver (clip/root (:clip c)) (:store c) pal/index-of)]
(fn [f] (into #{} (map :node) (r f)))))
ref-at (nodes-at ref)
occ-at (nodes-at occ)
@ -495,7 +515,7 @@
open (filter #(contains? (ref-at %) :teeth) (range take/frames))
inside (first (filter gap open))
outside (first (remove gap open))]
(is (= :mouth-in (get-in (:scene occ) [:nodes :teeth :stencil]))
(is (= :mouth-in (get-in (clip/nodes (:clip occ)) [:teeth :stencil]))
"teeth stop inheriting the mouth's absence if this stops being their stencil")
(is (some? inside) "no open-mouth frame inside the gap to test with")
(is (some? outside) "no open-mouth frame outside the gap to test with")
@ -503,14 +523,14 @@
;; Annotating the MOUTH sets no bit on the teeth: they are a feature of
;; their own and nothing named them.
(is (not (ch/nothing?
(ch/value-at (get-in (:scene occ) [:nodes :teeth :channels [:geom :pts]])
(ch/value-at (get-in (clip/nodes (:clip occ)) [:teeth :channels [:geom :pts]])
inside (:store occ)))))
;; They are dropped anyway — :mouth-in drew nothing to clip them against.
(is (not (contains? (occ-at inside) :mouth-in)))
(is (not (contains? (occ-at inside) :teeth)))
;; And the same articulation outside the gap still draws them.
(is (contains? (occ-at outside) :teeth)))
(is (empty? (scene/problems (:scene occ))))))
(is (empty? (clip/problems (:clip occ))))))
(deftest an-undetected-frame-has-no-pose-at-all
;; A subject that was not on the frame has NO VALUE, which is different from a
@ -521,8 +541,8 @@
det (mapv #(not (contains? gap %)) (range take/frames))
c (freeze/clip (assoc take/params :name "gappy")
(assoc @take/measured :detected det))
sc (:scene c)
res (scene/resolver sc (:store c) pal/index-of)]
tl (clip/root (:clip c))
res (timeline/resolver tl (:store c) pal/index-of)]
(doseq [f [39 40 50 59 60]]
(let [ops (res f)]
(if (contains? gap f)
@ -531,7 +551,7 @@
;; And it is the MASK doing it, not a hidden flag: `[:vis]` on :mouth-in is
;; unchanged across the gap, because hiding and absence are different
;; questions with different answers.
(is (= (mapv #(ch/value-at (get-in sc [:nodes :mouth-in :channels [:vis]]) %)
(is (= (mapv #(ch/value-at (get-in (:nodes tl) [:mouth-in :channels [:vis]]) %)
(range take/frames))
(mapv #(ch/value-at (chan :mouth-in [:vis]) %) (range take/frames))))))
@ -569,7 +589,7 @@
(deftest the-take-draws-something-on-every-frame
(doseq [f (range 0 take/frames 5)]
(let [n (drawn (render @scene* f))]
(let [n (drawn (render @clip* f))]
(is (> n 200) (str "frame " f " drew only " n " pixels")))))
(deftest the-mouth-moves
@ -577,10 +597,10 @@
;; from different beats are genuinely different mouths and frames inside one
;; beat are not. This is the numeric half of step 5's done-criterion; the other
;; half is a picture and lives in test/browser/take.mjs.
(let [locked (freeze/head-mode {:mode :locked} @clip)
(let [locked (freeze/head-mode {:mode :locked} @frozen)
shot (fn [f]
(let [r (raster/make W H)
mouth (filter #(= :mouth (:node %)) (ops-at locked f))]
mouth (filter #(= :mouth (:node %)) (ops-at (clip/root locked) f))]
(raster/clear! r (get pal/index-of :bg))
(raster/draw-ops! r mouth)
(vec (array-seq (:buf r)))))
@ -594,6 +614,6 @@
(is (< (differ (shot 10) (shot 12)) 200)
"a held pose is moving more than the detector noise it should have lost"))
;; As filmed, the head carries it around the stage as well.
(let [filmed (freeze/head-mode {:mode :as-filmed} @clip)]
(let [filmed (freeze/head-mode {:mode :as-filmed} @frozen)]
(is (> (count (remove true? (map = (render filmed 10) (render filmed 120)))) 300)
"the head does not move across the take")))