noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Model head anchors and independent pose timing

Browse source
This commit is contained in:
Olive Vaughn 2026-09-29 00:46:08 -04:00
parent a45e89f4e4
commit 49ece8dee6
17 changed files with 521 additions and 215 deletions
Download patch file Download diff file Expand all files Collapse all files

14
frontend/src/arthur/demo/take.cljs
View file

@ -21,11 +21,9 @@
anything that reads a source pixel, because that part takes the landmarks and
the frames and never the transform.
TWO CLIPS, ONE STORE. `:take` carries the head as filmed and `:take-locked`
carries it locked, and they are the same dense blocks with one node's channels
written two ways. That is the claim \"stabilisation is a channel, not a mode\"
made checkable by eye: switching between them is a document edit, tier 1, and
not one byte of tier 2 differs."
TWO CLIPS, ONE STORE. `:take` reads each measured head transform in time;
`:take-locked` holds the measured transform from frame zero. They share the
same dense blocks; only the head node's anchor map differs."
(:require [arthur.flow.address :as address]
[arthur.flow.freeze :as freeze]
[arthur.flow.take :as take]
@ -74,7 +72,7 @@
:expose 2
;; The head as filmed. `:take-locked` is the same freeze with this one
;; field changed, which is the point.
:head :as-filmed
:head :free
;; Provenance, and now a content address. There is no detector here, so
;; the generator IS the detector and its seed is the source: two synth
;; takes at different seeds are different analyses, which is the same
@ -94,5 +92,5 @@
(def clip (delay (:clip @frozen)))
(def locked
"The same blocks, with `:head` written as framed identity instead."
(delay (freeze/head-mode {:mode :locked} @frozen)))
"The same blocks, with `:head` held at measured frame zero."
(delay (freeze/head-mode {:mode :anchored :anchors {0 0}} @frozen)))

27
frontend/src/arthur/domain/clip.cljs
View file

@ -36,6 +36,7 @@
(:require [arthur.domain.feature :as feature]
[arthur.domain.node :as node]
[arthur.domain.palette :as pal]
[arthur.domain.pose :as pose]
[arthur.domain.timeline :as timeline]))
(def ^:const root-id
@ -118,8 +119,9 @@
inside it still resolve, because this is the function that knows how to do that."
([clip store] (resolver clip store pal/index-of root-id))
([clip store palette] (resolver clip store palette root-id))
([clip store palette root]
(letfn [(build [tid chain]
([clip store palette root] (resolver clip store palette root nil))
([clip store palette root {:keys [picture-fps] :as opts}]
(letfn [(build [tid chain pose-tracks]
(when (some #{tid} chain)
(throw (ex-info "symbol timeline cycle" {:chain (conj chain tid)})))
(let [tl (or (timeline clip tid)
@ -127,10 +129,12 @@
nodes (:nodes tl)
rank (timeline/draw-rank nodes (timeline/order nodes))
ids (sort-by rank (keys nodes))
own (timeline/resolver tl store palette)
own (timeline/resolver tl store palette pose-tracks
(assoc opts :source-fps (:fps clip)))
children (into {}
(for [[id n] nodes :when (= :symbol (:kind n))]
[id (build (:of n) (conj chain tid))]))]
[id (build (:of n) (conj chain tid)
(get-in n [:playback :tracks]))]))]
(fn [f]
(let [by-id (into {} (map (juxt :node identity)) (own f))]
(into []
@ -151,7 +155,7 @@
[]))
(when-let [op (get by-id id)] [op]))))
ids))))))]
(build root []))))
(build root [] nil))))
(defn problems
"Human-readable reasons this clip will not evaluate or will not save. Empty
@ -184,6 +188,19 @@
(not (contains? (:timelines clip) (:of n))))]
(str "timeline " (pr-str tid) " symbol " (pr-str id)
" names missing timeline " (pr-str (:of n))))
(for [[tid tl] (:timelines clip)
[id n] (:nodes tl)
:when (= :symbol (:kind n))
:let [target (get-in clip [:timelines (:of n)])
active (filter (fn [node]
(some :pose-sampled? (vals (:channels node))))
(vals (:nodes target)))
groups (set (concat
(map #(or (:pose-group %) (:id %)) active)
(map #(vector :node (:id %)) active)))]
p (pose/problems (get-in n [:playback :tracks])
(:frames target) groups)]
(str "timeline " (pr-str tid) " symbol " (pr-str id) ": " p))
(for [[tid tl] (:timelines clip)
[id n] (:nodes tl)
:when (and (= :audio (:kind n)) (:linked-to n)

4
frontend/src/arthur/domain/leaf.cljs
View file

@ -44,7 +44,9 @@
WHY `measured` IS ONE LEAF AND CHANNELS ARE NOT. `:head`'s measured channels are
not authored: they are written together by a freeze and replaced together by a
re-freeze, and `head-mode` reads them to write `:channels`. A leaf per measured
re-freeze, and `head-mode` exposes them through `:channels`. The optional
`:anchors` map on the head node chooses which measured frame those channels
read. A leaf per measured
channel would offer a write nobody can make. The authored channels beside them
are one leaf each, because a hand writes one at a time.

94
frontend/src/arthur/domain/pose.cljs Normal file
View file

@ -0,0 +1,94 @@
(ns arthur.domain.pose
"An instance's explicit, held choices of source pose for each shape group.
A track is {local-frame -> source-frame}. The key is when the cut happens;
the value is the frozen pose to read. Skipped source frames remain available.")
(defn prepare
"Sort exposure tracks once when building a resolver."
[tracks]
(into {}
(map (fn [[group entries]]
[group (vec (sort-by first entries))]))
tracks))
(defn held-frame
"Last value keyed at or before f, or default before the first key."
[entries f default-frame]
(loop [lo 0 hi (dec (count entries)) hit nil]
(if (> lo hi)
(if (some? hit) (second (nth entries hit)) default-frame)
(let [mid (bit-shift-right (+ lo hi) 1)]
(if (<= (first (nth entries mid)) f)
(recur (inc mid) hi mid)
(recur lo (dec mid) hit))))))
(defn source-frame
"Read an explicit cut if one has happened; otherwise read the default pose.
This keeps the existing motion before the first edited cut."
[prepared group f default-frame]
(if-let [entries (get prepared group)]
(held-frame entries f default-frame)
default-frame))
(defn put-cut
"Set one held pose on a symbol instance. Earlier motion stays untouched."
[clip instance group at source]
(let [node (get-in clip [:timelines :main :nodes instance])
symbol (get-in clip [:timelines (:of node)])
length (:frames symbol)
active (filter (fn [n] (some :pose-sampled? (vals (:channels n))))
(vals (:nodes symbol)))
groups (set (map #(or (:pose-group %) (:id %)) active))
ids (set (map :id active))]
(when-not (and (= :symbol (:kind node))
(or (contains? groups group)
(and (vector? group) (= 2 (count group))
(= :node (first group))
(contains? ids (second group))))
(integer? at) (<= 0 at) (< at length)
(integer? source) (<= 0 source) (< source length))
(throw (ex-info "invalid stage pose cut"
{:instance instance :group group :at at :source source})))
(update-in clip [:timelines :main :nodes instance :playback :tracks group]
#(assoc (or % {}) at source))))
(defn remove-cut
"Remove a cut; an empty track again follows the normal generated motion."
[clip instance group at]
(let [path [:timelines :main :nodes instance :playback :tracks group]]
(if-let [entries (get-in clip path)]
(if-let [remaining (not-empty (dissoc entries at))]
(assoc-in clip path remaining)
(update-in clip [:timelines :main :nodes instance :playback :tracks]
dissoc group))
clip)))
(defn problems
"Errors in one symbol instance's exposure tracks."
[tracks source-frames groups]
(cond
(nil? tracks) []
(not (map? tracks)) [":playback :tracks must be a map"]
:else
(vec
(mapcat
(fn [[group entries]]
(cond
(not (contains? groups group))
[(str "pose track " (pr-str group) " names no generated shape")]
(not (map? entries))
[(str "pose track " (pr-str group) " must map local frames to source frames")]
:else
(concat
(when-not (every? #(and (integer? %) (<= 0 %)
(or (nil? source-frames) (< % source-frames)))
(keys entries))
[(str "pose track " (pr-str group) " has an invalid change frame")])
(when-not (every? #(and (integer? %) (<= 0 %)
(or (nil? source-frames) (< % source-frames)))
(vals entries))
[(str "pose track " (pr-str group) " names a pose outside the source")]))))
tracks))))

62
frontend/src/arthur/domain/timeline.cljs
View file

@ -55,6 +55,7 @@
fill in the same channel rather than convert into a second format."
(:require [arthur.domain.channel :as ch]
[arthur.domain.node :as node]
[arthur.domain.pose :as pose]
[arthur.domain.palette :as pal]))
;; ---------------------------------------------------------------------------
@ -335,6 +336,29 @@
{:keys (vec (sort-by str (keys tl)))})))
nodes))
(defn- channel-frame
"Only marked generated channels read picture cadence and instance pose cuts."
[choices head-anchors nodes source-fps picture-fps id path c lf]
(cond
(and (= id :head) (contains? #{[:xform :pos] [:xform :rot] [:xform :scale]} path)
(contains? head-anchors id))
(pose/held-frame (get head-anchors id) lf lf)
(:pose-sampled? c)
(pose/source-frame choices
(if (contains? choices [:node id])
[:node id]
(or (:pose-group (get nodes id)) id))
lf
(node/sample-frame lf source-fps picture-fps))
:else lf))
(defn- prepared-head-anchors [nodes]
(into {}
(for [[id n] nodes :when (seq (:anchors n))]
[id (vec (sort-by first (:anchors n)))])))
(defn- eval-into
"One frame, as a fold over the nodes in topological order.
@ -381,10 +405,17 @@
This is the definition of what a frame means. `resolver` is what plays it."
([tl f] (eval-frame tl f nil pal/index-of))
([tl f store] (eval-frame tl f store pal/index-of))
([tl f store palette]
([tl f store palette] (eval-frame tl f store palette nil nil))
([tl f store palette pose-tracks opts]
(let [nodes (nodes-of tl)
choices (pose/prepare pose-tracks)
head-anchors (prepared-head-anchors nodes)
{:keys [source-fps picture-fps]} opts
ord (order nodes)]
(eval-into {:read (fn [_id _path c lf] (ch/value-at c lf store))
(eval-into {:read (fn [id path c lf]
(ch/value-at c (channel-frame choices head-anchors nodes
source-fps picture-fps id path c lf)
store))
:palette palette
:mat-for (fn [_id] (node/mat))
:pinv-for (fn [id] (node/pinv (get nodes id)))
@ -435,10 +466,14 @@
The op maps themselves are allocated fresh, and deliberately: there are a dozen
of them per frame against hundreds of points, so pooling them would buy
nothing and cost the ability to hand an op list around as plain data."
([tl] (resolver tl nil pal/index-of))
([tl store] (resolver tl store pal/index-of))
([tl store palette]
([tl] (resolver tl nil pal/index-of nil nil))
([tl store] (resolver tl store pal/index-of nil nil))
([tl store palette] (resolver tl store palette nil nil))
([tl store palette pose-tracks] (resolver tl store palette pose-tracks nil))
([tl store palette pose-tracks {:keys [source-fps picture-fps]}]
(let [nodes (nodes-of tl)
choices (pose/prepare pose-tracks)
head-anchors (prepared-head-anchors nodes)
ord (order nodes)
rank (draw-rank nodes ord)
cursors (into {}
@ -460,7 +495,10 @@
;; eval-into having to report it — and it covers groups, which are
;; placed but emit no op, and which are exactly what an underlay rides.
placed (volatile! {})
ctx {:read (fn [id path _c lf] (ch/sample! (get-in cursors [id path]) lf))
ctx {:read (fn [id path c lf]
(ch/sample! (get-in cursors [id path])
(channel-frame choices head-anchors nodes
source-fps picture-fps id path c lf)))
:palette palette
:mat-for (fn [id] (get mats id))
:on-place (fn [id p] (vswap! placed assoc id p))
@ -521,6 +559,18 @@
(into (for [[id n] nodes
p (node/problems n)]
(str "node " (pr-str id) ": " p)))
(into (for [[id n] nodes
:let [anchors (:anchors n)]
:when (some? anchors)
:when (not (and (= id :head)
(map? anchors) (contains? anchors 0)
(integer? (:frames tl))
(every? #(and (integer? %) (<= 0 %)
(< % (:frames tl)))
(concat (keys anchors) (vals anchors)))
(= (:channels n) (:measured n))))]
(str "node " (pr-str id)
": :anchors must start at frame 0, name valid measured frames, and read the measured head channels")))
(into (for [k (remove timeline-keys (keys tl))]
(str "timeline has a field with no leaf to save it in: " (pr-str k))))
(into (when-not (or (nil? (:frames tl)) (and (integer? (:frames tl)) (pos? (:frames tl))))

29
frontend/src/arthur/export.cljs
View file

@ -12,7 +12,8 @@
reachable and neither should be the other's special case.
What is genuinely shared is everything above the sink, and it is most of the
work: rooting the resolver at the chosen timeline, the picture-rate time map,
work: rooting the resolver at the chosen timeline, generated-channel picture
sampling,
the raster, the frame loop, the audio mix, the progress reporting and the
yielding that lets the page paint. So `run!` owns all of that and calls three
methods.
@ -126,22 +127,6 @@
(update tl :nodes select-keys (kin (:nodes tl) id))
tl))
(defn- sampled
"The timeline with picture-rate sampling written onto its root node's time map.
The same transform `subs/render/::timeline` applies for playback, and
deliberately the same one: an export that sampled poses differently from the
preview would make the preview a lie about the deliverable. Only the root node's
time map changes — the dense source track stays at its native rate, and the
frame COUNT is untouched, which is what keeps the duration and therefore the
audio right."
[tl source-fps picture-fps]
(if (and picture-fps source-fps (< picture-fps source-fps))
(-> tl
(assoc-in [:nodes :root :time :source-fps] source-fps)
(assoc-in [:nodes :root :time :sample-fps] picture-fps))
tl))
(defn- yield!
"Hand the event loop a turn between frames.
@ -183,8 +168,8 @@
:width (* (:width clip) zoom)
:height (* (:height clip) zoom)
:seconds (/ (:frames tl) (:fps clip))
;; The poses the picture actually holds, which is what a lower picture rate
;; changes. The frame COUNT does not move — see the namespace docstring.
;; The unedited picture-grid count. A per-instance pose track can add or
;; remove changes, so this is only the grid's nominal count.
:poses (if (and picture-fps (< picture-fps (:fps clip)))
(js/Math.ceil (* (/ (:frames tl) (:fps clip)) picture-fps))
(:frames tl))})))
@ -207,9 +192,9 @@
;; Rooted at the chosen timeline, so exporting a symbol is exporting a
;; clip whose root that symbol is. Nested symbols inside it still
;; resolve — clip/resolver is the function that knows how.
doc (assoc-in clip [:timelines timeline]
(sampled tl (:fps clip) picture-fps))
resolve-frame (clip/resolver doc store palette timeline)
doc (assoc-in clip [:timelines timeline] tl)
resolve-frame (clip/resolver doc store palette timeline
{:picture-fps picture-fps})
ras (raster/make (:width clip) (:height clip))
bg (get palette :bg 0)]
(-> (audio! doc timeline store audio-url)

130
frontend/src/arthur/flow/freeze.cljs
View file

@ -245,65 +245,34 @@
:tx (* (- s') (+ (* c tx) (* sn ty)))
:ty (* (- s') (+ (* (- sn) tx) (* c ty)))}))
(def ^:private head-modes #{:locked :as-filmed :per-plate})
(def ^:private head-modes #{:free :anchored})
(defn head-mode
"Rewrite `:head`'s transform channels into one of the three shapes.
"Keep the measured transform dense; optionally hold chosen source frames.
STABILISATION IS A CHANNEL, NOT A MODE. `{s θ tx ty}` per frame IS
`[:xform :scale]`, `[:xform :rot]` and `[:xform :pos]`, so removing the head's
motion is not a pipeline setting — it is a question of which shape one node's
channels carry:
:locked framed identity — the head sits still, for tracing and for
judging articulation
:as-filmed dense — the head moves around the stage
:per-plate keyed at the kept frames — the head pose is stable for exactly as
long as a drawing is on screen, which is what a plate strip wants
ALWAYS MEASURE, ALWAYS STORE FACTORED, whatever the toggle says. The blocks are
written once and every mode reads them or ignores them; making this an
analysis-time switch would lose two things downstream. \"Smooth the transform,
never the contour\" only means anything while the two are separate. And a
velocity minimum is \"articulation paused\" in head-local space and \"the head
happened to be still\" in image space, so key selection needs the split to exist
in storage.
So this is a DOCUMENT EDIT: tier 1, undoable, syncable, instant, and not a
reason to re-analyse. It rewrites `:head` and touches nothing else — in
particular not `:face`, which a hand placed, and not one byte of any block.
`:per-plate` samples the dense track at the kept frames and stores plain
vectors. It may not store what `value-at` handed it: a wide dense read is a
VIEW into the block, and a view into tier 2 sitting in the document is a value
that changes when a re-freeze rewrites the array under it."
[{:keys [mode kept]} {:keys [clip store]}]
A nil anchor map reads measured frame f at frame f (free movement).
`{0 12}` locks to the measured transform of source frame 12. `{0 12, 40 42}`
cuts to source frame 42 at local frame 40. The same map selects position,
rotation and scale, so the head and registered photo cannot drift apart.
No analysis block or authored face placement changes."
[{:keys [mode anchors]} {:keys [clip]}]
(when-not (contains? head-modes mode)
(throw (ex-info "head mode is not one of the three channel shapes"
(throw (ex-info "head mode must be free or anchored"
{:mode mode :modes head-modes})))
(let [base (get-in (clip/root clip) [:nodes :head :measured])
at (fn [path f]
(let [v (ch/value-at (get base path) f store)
n (:stride (:dense (get base path)))]
(if (= 1 n) v (mapv #(ch/component v %) (range n)))))
keys-of (fn [path]
(-> (get base path)
(dissoc :dense)
(assoc :keys (into {} (map (juxt identity #(at path %))) (sort kept)))))]
(when (and (= mode :per-plate) (empty? kept))
(throw (ex-info "the per-plate head mode needs a kept-frame set; it is the plate strip's, not measurement's"
{:mode mode})))
(assoc-in clip [:timelines clip/root-id :nodes :head :channels]
(case mode
;; No `:generated` on the locked shape, and that is not an
;; oversight: nothing generated this identity. It is a decision,
;; and provenance that claimed otherwise would offer a re-freeze
;; button that could only undo it.
:locked {[:xform :pos] (ch/framed [0.0 0.0])
[:xform :rot] (ch/framed 0.0)
[:xform :scale] (ch/framed [1.0 1.0])}
:as-filmed base
:per-plate (into {} (map (juxt key #(keys-of (key %)))) base)))))
(when (and (= mode :free) (some? anchors))
(throw (ex-info "free head motion has no anchors" {:anchors anchors})))
(let [frames (clip/frames clip)]
(when (and (= mode :anchored)
(not (and (map? anchors) (contains? anchors 0)
(every? #(and (integer? %) (<= 0 %) (< % frames))
(concat (keys anchors) (vals anchors))))))
(throw (ex-info "anchored head needs a frame-zero key and valid source frames"
{:anchors anchors :frames frames})))
(update-in clip [:timelines clip/root-id :nodes :head]
(fn [n]
(cond-> (assoc n :channels (:measured n))
(= mode :anchored) (assoc :anchors anchors)
(= mode :free) (dissoc :anchors))))))
;; ---------------------------------------------------------------------------
;; the aperture, onto [:vis] of :mouth-in
@ -347,6 +316,32 @@
(range (count values))))
:generated generated))
(def ^:private pose-groups
{:mouth :mouth :mouth-in :mouth :teeth :mouth
:eye-r :eye-r :eye-r-in :eye-r :iris-r :eye-r :pupil-r :eye-r
:eye-l :eye-l :eye-l-in :eye-l :iris-l :eye-l :pupil-l :eye-l
:brow-r :brow-r :brow-l :brow-l})
(defn- performance-nodes
"Mark the generated moving channels that may read an instance's pose choice.
Provenance helps identify them here; the renderer reads only :pose-sampled?."
[nodes]
(into {}
(map (fn [[id n]]
[id (cond-> n
(contains? pose-groups id)
(assoc :pose-group (get pose-groups id))
(contains? pose-groups id)
(update :channels
(fn [channels]
(into {}
(map (fn [[path ch]]
[path (cond-> ch
(and (:generated ch) (:animated? ch))
(assoc :pose-sampled? true))]))
channels))))]))
nodes))
;; ---------------------------------------------------------------------------
;; the face, onto the stage
@ -586,12 +581,13 @@
(and (contains? presence id)
(not (nth (get presence id) f))))))
obs {:detected detected :presence presence}]
(case area
:mouth (mouth-part absent? obs params measured)
:eye (feature-parts absent? obs params (select-keys measured [:eyes]))
:brow (feature-parts absent? obs params (select-keys measured [:brows]))
:teeth (interior-part params absent? obs (:teeth measured))
(throw (ex-info "unknown frozen feature type" {:area area})))))
(update (case area
:mouth (mouth-part absent? obs params measured)
:eye (feature-parts absent? obs params (select-keys measured [:eyes]))
:brow (feature-parts absent? obs params (select-keys measured [:brows]))
:teeth (interior-part params absent? obs (:teeth measured))
(throw (ex-info "unknown frozen feature type" {:area area})))
:nodes performance-nodes)))
(defn head-part
"Freeze only the measured head transform from an already conditioned anchor."
@ -638,8 +634,8 @@
:brow-verts the brow rings' vertex budget
:aperture-cut fraction of the take's peak aperture below which the mouth
interior is not present
:head :locked | :as-filmed | :per-plate
:kept frames, for :per-plate only
:head :free | :anchored
:anchors optional map of local change frame -> measured source frame
:analysis the analysis record these measurements came from — detector,
VERSION, source, source cadence and pixel aspect. Its `:id` is
a content address over all of that, every block's key is a hash
@ -682,9 +678,8 @@
painted cel rides a moving plate it will be.
`:head` keeps its measured channels under `:measured` as well as in
`:channels`, so `head-mode` can switch shapes without the blocks or the
provenance having to be rebuilt."
[{:keys [name fps stage expose verts aperture-cut head kept
`:channels`; its optional `:anchors` map changes only their frame address."
[{:keys [name fps stage expose verts aperture-cut head anchors
analysis anchor-avg contour-avg] :as params}
{:keys [transforms outer inner detected presence eyes brows teeth] :as inputs}]
(let [nf (count outer)
@ -796,4 +791,7 @@
(throw (ex-info "presence track names no feature in this clip"
{:feature id :features (keys (:features built))}))))
{:store store
:clip (head-mode {:mode head :kept kept} {:clip built :store store})}))
:clip (head-mode {:mode head :anchors anchors}
{:clip (update-in built [:timelines clip/root-id :nodes]
performance-nodes)
:store store})}))

2
frontend/src/arthur/flow/take.cljs
View file

@ -112,7 +112,7 @@
{:name (or (:source manifest) "footage")
:fps (:fps manifest) :aspect (/ w h)
:stage [320 200] :fit-motion? true
:expose 1 :head :as-filmed
:expose 1 :head :free
;; The detector's identity comes from the server, which
;; hashes the model asset it serves rather than trusting a
;; version string somebody has to remember to bump. See

22
frontend/src/arthur/subs/render.cljs
View file

@ -26,21 +26,7 @@
(rf/reg-sub
::timeline
:<- [::clip]
:<- [::playback/display-fps]
(fn [[clip picture-fps] _]
;; The ROOT timeline, with picture sampling written onto its root node's time
;; map. Only that node's time map changes: the dense source track stays at its
;; native rate and the audio clock still advances through source time.
;;
;; `:fps` is read off the CLIP and `:frames` off the timeline, which is the
;; whole reason the two came apart — the source cadence is a fact about how fast
;; the clip plays against its audio, and a nested timeline will not have one.
(when-let [tl (clip/root clip)]
(if (and picture-fps (< picture-fps (:fps clip)))
(-> tl
(assoc-in [:nodes :root :time :source-fps] (:fps clip))
(assoc-in [:nodes :root :time :sample-fps] picture-fps))
tl))))
(fn [clip _] (some-> clip clip/root)))
(rf/reg-sub
::exposure
@ -85,6 +71,8 @@
:<- [::timeline]
:<- [::store]
:<- [::palette]
(fn [[document tl store palette] _]
:<- [::playback/display-fps]
(fn [[document tl store palette picture-fps] _]
(when tl (clip/resolver (assoc-in document [:timelines :main] tl)
store palette))))
store palette clip/root-id
{:picture-fps picture-fps}))))