noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

multi fce stuff

Browse source
This commit is contained in:
Olive Vaughn 2026-09-29 02:34:53 -04:00
parent 49ece8dee6
commit ddabfbeaa8
32 changed files with 1309 additions and 700 deletions
Download patch file Download diff file Expand all files Collapse all files

13
frontend/src/arthur/demo/stage.cljs
View file

@ -67,12 +67,7 @@
:channels (cond-> {[:audio :gain] gain}
pan (assoc [:audio :pan] pan))}])
audio))]
(cond-> (assoc source :name name :width width :height height
:timelines {:main {:id :main :frames frames :nodes nodes}
symbol (assoc original :id symbol)})
;; These are the same tracked features in the shared symbol. Every
;; placement reads them; tuning one changes all instances.
(seq (:features source))
(assoc :features (into {} (map (fn [[id f]]
[id (assoc f :timeline symbol)]))
(:features source))))))
(assoc source :name name :width width :height height
:timelines (assoc (:timelines source)
:main {:id :main :frames frames :nodes nodes}
symbol (assoc original :id symbol)))))

4
frontend/src/arthur/demo/take.cljs
View file

@ -84,8 +84,10 @@
:fps fps
:aspect aspect})}))
(def subject :face-1)
(def frozen
(delay (freeze/clip params @measured)))
(delay (freeze/clip params {subject @measured})))
(def store (delay (:store @frozen)))

13
frontend/src/arthur/domain/clip.cljs
View file

@ -101,7 +101,7 @@
:disc (let [[x y] (at (:cx op) (:cy op))]
(assoc op :cx x :cy y :r (* scale (:r op))))
:rect (let [[x y] (at (:cx op) (:cy op))]
(assoc op :cx x :cy y :size (js/Math.round (* scale (:size op)))))
(assoc op :cx x :cy y :size (* scale (:size op))))
op)))
(defn resolver
@ -158,13 +158,7 @@
(build root [] nil))))
(defn problems
"Human-readable reasons this clip will not evaluate or will not save. Empty
means it will.
The tracking identities are checked HERE and not in `domain/timeline`, because a
feature names nodes and only the root timeline's nodes were tracked into: a
library symbol is drawn, not detected. So `feature/problems` is asked about the
root, once, rather than about every timeline."
"Human-readable reasons this clip will not evaluate or save."
[clip]
(vec
(concat
@ -207,5 +201,4 @@
(not (contains? (:nodes tl) (:linked-to n))))]
(str "timeline " (pr-str tid) " audio " (pr-str id)
" links to missing node " (pr-str (:linked-to n))))
(when (map? (root clip))
(feature/problems clip (nodes clip))))))
(feature/problems clip))))

36
frontend/src/arthur/domain/feature.cljs
View file

@ -1,14 +1,13 @@
(ns arthur.domain.feature
"Stable tracked identities and explicit eye-pair settings associations.
These maps are the CLIP's — `:subjects`, `:features`, `:groups` — and not a
timeline's, because they describe what a camera saw and a library symbol is
drawn rather than detected. Rendering never reads them; it reads nodes and
channels. `problems` therefore takes the clip AND the node map to check
references against, rather than reaching for `(:nodes clip)`: a clip holds
several timelines and only the root one was tracked into."
"Tracked subjects, feature ownership, and eye-pair settings.
Features name their timeline explicitly; node ids are local to that timeline."
(:require [arthur.domain.params :as params]))
(defn owned
"A deterministic clip-level feature or group id. Nodes keep local names."
[subject role]
(keyword (subs (str subject) 1) (name role)))
(defn group-for [clip feature-id]
(first (filter (fn [[_ group]] (some #{feature-id} (:members group)))
(:groups clip))))
@ -45,17 +44,14 @@
clip))
(defn problems
"Check identity references and pair membership before storing a clip.
`nodes` is the node map a feature's `:nodes` are resolved against — the root
timeline's, passed in rather than looked up, so this namespace does not have to
know which timeline a clip plays."
[clip nodes]
"Check tracked identities and timeline-local node ownership."
[clip]
(let [subjects (:subjects clip)
features (:features clip)
groups (:groups clip)
memberships (mapcat (comp :members val) groups)
node-owners (mapcat (comp :nodes val) features)]
node-owners (for [[_ f] features n (:nodes f)]
[(:timeline f) n])]
(vec
(concat
(for [[id s] subjects :when (not= id (:id s))]
@ -63,6 +59,9 @@
(for [[id s] subjects
:when (not (params/valid-settings? :subject (or (:params s) {})))]
(str "subject " (pr-str id) " has invalid settings"))
(for [[id _] subjects
:when (not (seq (get-in clip [:timelines id :nodes :head :measured])))]
(str "subject " (pr-str id) " has no measured head in its timeline"))
(for [[id f] features :when (not= id (:id f))]
(str "feature " (pr-str id) " has a different :id"))
(for [[id f] features :when (not (contains? subjects (:subject f)))]
@ -72,10 +71,11 @@
(for [[id f] features
:when (not (params/valid-settings? (:area f) (or (:params f) {})))]
(str "feature " (pr-str id) " has invalid settings for " (pr-str (:area f))))
(for [[id f] features
:when (not (contains? (:timelines clip) (:timeline f)))]
(str "feature " (pr-str id) " names a missing timeline"))
(for [[id f] features node-id (:nodes f)
:let [owned-nodes (if-let [timeline (:timeline f)]
(get-in clip [:timelines timeline :nodes])
nodes)]
:let [owned-nodes (get-in clip [:timelines (:timeline f) :nodes])]
:when (not (contains? owned-nodes node-id))]
(str "feature " (pr-str id) " refers to missing node " (pr-str node-id)))
(for [[id n] (frequencies node-owners) :when (> n 1)]

9
frontend/src/arthur/domain/raster.cljs
View file

@ -134,10 +134,11 @@
breathing."
([r cx cy size index] (fill-rect! r cx cy size index nil))
([{:keys [w h buf] :as r} cx cy size index over]
(when (>= size 1)
(let [x0 (js/Math.round (- cx (/ size 2)))
y0 (js/Math.round (- cy (/ size 2)))]
(let [ya (max 0 y0) yb (min h (+ y0 size))
(let [size (js/Math.round size)]
(when (>= size 1)
(let [x0 (js/Math.round (- cx (/ size 2)))
y0 (js/Math.round (- cy (/ size 2)))
ya (max 0 y0) yb (min h (+ y0 size))
xa (max 0 x0) xb (min w (+ x0 size))]
(dotimes [iy (- yb ya)]
(dotimes [ix (- xb xa)]

45
frontend/src/arthur/domain/timeline.cljs
View file

@ -271,14 +271,8 @@
:rd rd})))))))
(defn- emit
"The draw op for a placed node, or nil when it has nothing to draw. A group
never draws; it exists to carry a transform.
Three branches that rhyme, deliberately left as three: a poly writes vertices
into a buffer it was handed, a disc carries a scaled radius, a rect a rounded
pixel count. They are three different marks, and the shared skeleton is two
cheap lines each — folding them into one shape driven by a table would buy
those lines back by coupling this to whatever the table was for."
"Emit geometry in the timeline's space. Rect sizes stay fractional until
rasterization, so enclosing symbol transforms can still scale them."
[{:keys [palette buf-for]} n {:keys [m rd]} base]
(let [colour #(colour-index palette (rd [:style :color]))]
(case (:kind n)
@ -310,10 +304,7 @@
(when-not (ch/nothing? size)
(assoc base :kind :rect
:cx (aget m 4) :cy (aget m 5)
;; ROUNDED, because :size is a pixel count: a scaled square
;; 3.4px wide would be 3px on one frame and 4 on the next,
;; which reads as the pupil breathing. See raster/fill-rect!.
:size (js/Math.round (* size (node/mean-scale m)))
:size (* size (node/mean-scale m))
:color (colour))))
(throw (ex-info "node kind is not implemented"
@ -337,12 +328,11 @@
nodes))
(defn- channel-frame
"Only marked generated channels read picture cadence and instance pose cuts."
[choices head-anchors nodes source-fps picture-fps id path c lf]
"Anchors select measured frames; marked channels read instance pose choices."
[choices anchors nodes source-fps picture-fps id c lf]
(cond
(and (= id :head) (contains? #{[:xform :pos] [:xform :rot] [:xform :scale]} path)
(contains? head-anchors id))
(pose/held-frame (get head-anchors id) lf lf)
(contains? anchors id)
(pose/held-frame (get anchors id) lf lf)
(:pose-sampled? c)
(pose/source-frame choices
@ -354,7 +344,7 @@
:else lf))
(defn- prepared-head-anchors [nodes]
(defn- prepared-anchors [nodes]
(into {}
(for [[id n] nodes :when (seq (:anchors n))]
[id (vec (sort-by first (:anchors n)))])))
@ -409,12 +399,12 @@
([tl f store palette pose-tracks opts]
(let [nodes (nodes-of tl)
choices (pose/prepare pose-tracks)
head-anchors (prepared-head-anchors nodes)
anchors (prepared-anchors nodes)
{:keys [source-fps picture-fps]} opts
ord (order nodes)]
(eval-into {:read (fn [id path c lf]
(ch/value-at c (channel-frame choices head-anchors nodes
source-fps picture-fps id path c lf)
(ch/value-at c (channel-frame choices anchors nodes
source-fps picture-fps id c lf)
store))
:palette palette
:mat-for (fn [_id] (node/mat))
@ -473,7 +463,7 @@
([tl store palette pose-tracks {:keys [source-fps picture-fps]}]
(let [nodes (nodes-of tl)
choices (pose/prepare pose-tracks)
head-anchors (prepared-head-anchors nodes)
anchors (prepared-anchors nodes)
ord (order nodes)
rank (draw-rank nodes ord)
cursors (into {}
@ -497,8 +487,8 @@
placed (volatile! {})
ctx {:read (fn [id path c lf]
(ch/sample! (get-in cursors [id path])
(channel-frame choices head-anchors nodes
source-fps picture-fps id path c lf)))
(channel-frame choices anchors nodes
source-fps picture-fps id c lf)))
:palette palette
:mat-for (fn [id] (get mats id))
:on-place (fn [id p] (vswap! placed assoc id p))
@ -559,18 +549,19 @@
(into (for [[id n] nodes
p (node/problems n)]
(str "node " (pr-str id) ": " p)))
;; Anchors re-address the node's measurement, regardless of its name.
(into (for [[id n] nodes
:let [anchors (:anchors n)]
:when (some? anchors)
:when (not (and (= id :head)
(map? anchors) (contains? anchors 0)
:when (not (and (map? anchors) (contains? anchors 0)
(integer? (:frames tl))
(every? #(and (integer? %) (<= 0 %)
(< % (:frames tl)))
(concat (keys anchors) (vals anchors)))
(seq (:measured n))
(= (:channels n) (:measured n))))]
(str "node " (pr-str id)
": :anchors must start at frame 0, name valid measured frames, and read the measured head channels")))
": :anchors must start at frame 0, name valid measured frames, and read that node's own measured channels")))
(into (for [k (remove timeline-keys (keys tl))]
(str "timeline has a field with no leaf to save it in: " (pr-str k))))
(into (when-not (or (nil? (:frames tl)) (and (integer? (:frames tl)) (pos? (:frames tl))))

154
frontend/src/arthur/events/footage.cljs
View file

@ -64,21 +64,32 @@
stream fps w h
(fn [i frame]
(.drawImage ctx frame 0 0)
(let [face (detect/detect! model canvas (ingest/frame-ms fps i))
ring (when face (mapv #(nth face %) lm/LIPS-INNER))
box (when ring (interior/crop ring [w h]))
pixels (when box
(.-data (.getImageData ctx (:x box) (:y box)
(:w box) (:h box))))]
(swap! raw conj face)
(let [crop (when box {:box box :data pixels})]
(swap! crops conj crop)
;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work
;; either way, but done after the fact it is ten seconds of
;; synchronous arithmetic with the main thread held and the
;; frame counter frozen on its last value — which reads as the
;; decoder hanging, and was diagnosed as that twice.
(swap! inner conj (source/measure-crop take/knobs crop))))
;; EVERY FACE ON THIS FRAME, each with its own mouth crop taken
;; while the frame's pixels are still on the canvas. Which of these
;; detections belongs to which subject is not decided here — the
;; answer needs the whole take — so all three vectors stay in
;; DETECTION ORDER and `detect/tracks` re-keys them afterwards.
(let [faces (detect/detect! model canvas (ingest/frame-ms fps i))
boxes (mapv (fn [face]
(interior/crop (mapv #(nth face %) lm/LIPS-INNER)
[w h]))
faces)
frame-crops (mapv (fn [box]
(when box
{:box box
:data (.-data (.getImageData
ctx (:x box) (:y box)
(:w box) (:h box)))}))
boxes)]
(swap! raw conj faces)
(swap! crops conj frame-crops)
;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work
;; either way, but done after the fact it is ten seconds of
;; synchronous arithmetic with the main thread held and the
;; frame counter frozen on its last value — which reads as the
;; decoder hanging, and was diagnosed as that twice.
(swap! inner conj (mapv #(source/measure-crop take/knobs %)
frame-crops)))
(when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total))
(rf/dispatch [::progress (str "detecting " (inc i) "/" total)]))
;; Yield, so the status and the transport paint between synchronous
@ -86,30 +97,56 @@
(js/Promise. (fn [done] (js/setTimeout done 0)))))))
(.then (fn [_]
(mark! (str "DECODE FINISHED — " (count @raw) " frames"))
(let [gaps (detect/fill-gaps @raw)]
(let [slots (detect/tracks @raw)
subjects (into {}
(map (fn [[id track]]
[id (assoc (detect/fill-gaps
(detect/pick @raw track))
:crops (detect/pick @crops track)
:interior (detect/pick @inner track)
:interior-settings take/knobs)]))
slots)]
(mark! "fill-gaps")
(assoc gaps :dimensions [w h] :crops @crops
:interior @inner :interior-settings take/knobs)))))))
{:subjects subjects
:dimensions [w h]
;; Frames where NOBODY was found, which is a fact about the
;; footage. A frame one of two faces is missing from is a gap
;; in that subject's own detection mask and is reported there.
:missing (count (remove seq @raw))
:first-real (first (keep-indexed (fn [i faces] (when (seq faces) i))
@raw))}))))))
(defn presence-for
"Select a subject's masks and express them in its local feature names.
Unqualified manifest ids refer to the first face, as in single-face footage."
[manifest subject]
(not-empty
(into {} (keep (fn [[id mask]]
(when (= (or (namespace id) "face-1") (subs (str subject) 1))
[(keyword (name id)) mask])))
(:presence manifest))))
(defn- build-clip [manifest detector
{:keys [dense detected dimensions interior missing first-real]
:as source-inputs}]
{:keys [subjects dimensions missing first-real] :as source-inputs}]
(let [[w h] dimensions
_ (mark! "build-clip: start")
frozen (take/footage manifest {:dense dense :detected detected
:dimensions dimensions :interior interior
:presence (:presence manifest)
:detector detector})
with-presence (into {}
(map (fn [[id inputs]]
[id (assoc inputs :presence (presence-for manifest id))]))
subjects)
frozen (take/footage manifest {:dimensions dimensions
:subjects with-presence
:detector detector})
_ (mark! "build-clip: freeze")
built (:clip frozen)
source-blocks (source/pack (:id (:analysis built)) source-inputs)
source-blocks (source/pack-subjects (:id (:analysis built)) subjects)
_ (mark! "build-clip: pack source blocks")]
(assoc (select-keys built [:fps :width :height])
:frames (clip/frames built)
:display-fps (:fps built)
:clip built :store (:store frozen)
:source-blocks source-blocks
:source-inputs (assoc source-inputs :presence (:presence manifest))
:source-inputs (assoc source-inputs :subjects with-presence)
;; No cache-buster. The audio is a blob named by the hash of its own
;; bytes, so re-extracting gives it a different URL rather than
;; overwriting this one — which is what the `?v=` here used to work
@ -120,14 +157,18 @@
:cid (or (:id manifest) "footage")
:summary (str (:frames manifest) " frames · " w "×" h " · "
(:fps manifest) " fps"
(when (> (count subjects) 1)
(str " · " (count subjects) " faces"))
(when (pos? missing)
(str " · " missing " without a face"
(when (pos? first-real)
(str " (first found on " (inc first-real) ")"))))))))
(defn- with-default-interior! [analysis track]
(defn- with-default-interior!
"Backfill one subject's retained interior measurements at the default knobs."
[analysis subject track]
(let [frames (count (:crops track))
key (source/interior-key analysis take/knobs frames)]
key (source/interior-key analysis subject take/knobs frames)]
(-> (http/GET (str "/api/blocks/" key))
(.then (fn [block]
(assoc track :interior (source/unpack-interior block take/knobs frames)
@ -137,16 +178,29 @@
(if (= 404 (:status (ex-data error))) track (throw error)))))))
(defn saved-source!
"Restore retained source tracks by analysis id, for regeneration after open."
"Restore retained source tracks by analysis id, for regeneration after open.
An analysis holds one set of blocks per tracked subject, so the count is a
multiple of `source/roles` rather than equal to it; `source/unpack` reads each
block's own descriptor to find out whose it is."
[key dimensions]
(-> (http/GET (str "/api/analyses/" key))
(.then (fn [^js analysis]
(let [keys (array-seq (.-source_blocks analysis))]
(when (= (count keys) (count source/roles))
(when (and (seq keys) (zero? (mod (count keys) (count source/roles))))
(-> (js/Promise.all
(into-array (map #(http/GET (str "/api/blocks/" %)) keys)))
(.then (fn [blocks] (source/unpack blocks dimensions)))
(.then (fn [track] (with-default-interior! key track))))))))
(.then (fn [{:keys [subjects]}]
(-> (js/Promise.all
(into-array
(map (fn [[id track]]
(.then (with-default-interior! key id track)
(fn [filled] [id filled])))
subjects)))
(.then (fn [pairs]
{:subjects (into {} (array-seq pairs))
:dimensions dimensions}))))))))))
(.catch (fn [error]
(if (= 404 (:status (ex-data error))) nil (throw error))))))
@ -154,19 +208,8 @@
(saved-source! (:id (take/analysis-for manifest detector))
[(:width manifest) (:height manifest)]))
(defn measure-crops!
"Measure every retained crop's interior, ONE PER EVENT-LOOP TURN.
Done in a tight loop instead, it is ten seconds of synchronous arithmetic with
the main thread held and the frame counter frozen on its last value — which
reads as the decoder hanging, and was diagnosed as that twice.
Two callers, and the only difference between them is who is waiting: opening an
older analysis that predates the interior block backfills it behind a progress
line, and a knob drag that needs the pixels measured at new settings does the
same work with nobody watching, so it passes no `on-step`. The settings ride
back on the track, because a measurement and the knobs it was taken at are one
fact — `source/pack` will not address an interior block without them."
(defn measure-one!
"One subject's crops, ONE PER EVENT-LOOP TURN."
[settings track on-step]
(if (or (:interior track) (not (:crops track)))
(js/Promise.resolve track)
@ -186,6 +229,29 @@
(catch :default error (reject error)))))]
(step 0)))))))
(defn measure-crops!
"Measure every retained crop's interior, one subject after another and one
crop per event-loop turn.
Done in a tight loop instead, it is ten seconds of synchronous arithmetic with
the main thread held and the frame counter frozen on its last value — which
reads as the decoder hanging, and was diagnosed as that twice.
Two callers, and the only difference between them is who is waiting: opening an
older analysis that predates the interior block backfills it behind a progress
line, and a knob drag that needs the pixels measured at new settings does the
same work with nobody watching, so it passes no `on-step`. The settings ride
back on each track, because a measurement and the knobs it was taken at are one
fact — `source/pack` will not address an interior block without them."
[settings track on-step]
(reduce (fn [chain [id one]]
(.then chain
(fn [acc]
(.then (measure-one! settings one on-step)
(fn [measured] (assoc-in acc [:subjects id] measured))))))
(js/Promise.resolve track)
(:subjects track)))
(rf/reg-fx
::begin!
(fn [footage-id]

68
frontend/src/arthur/events/project.cljs
View file

@ -135,9 +135,12 @@
(.then (fn [_]
(http/PUT
(str "/api/analyses/" (:id analysis))
;; One set per tracked subject, in
;; the order `source/unpack` does
;; not depend on.
#js {:source_blocks
(into-array (map #(get-in source-blocks [% :key])
source/roles))})))))))
(into-array
(source/block-keys source-blocks))})))))))
(.then (fn [_] (upload-missing! doc)))
(.then (fn [uploaded]
(-> (http/PUT (str "/api/projects/" pid)
@ -210,9 +213,9 @@
(defonce ^:private retained-source (atom nil))
(defonce ^:private retained-interior (atom nil))
(defn- retained-interior! [analysis settings inputs]
(defn- retained-interior! [analysis subject settings inputs]
(let [frames (count (:crops inputs))
block-key (source/interior-key analysis settings frames)]
block-key (source/interior-key analysis subject settings frames)]
(-> (http/GET (str "/api/blocks/" block-key))
(.then (fn [block]
(assoc inputs
@ -220,10 +223,10 @@
:interior-key block-key)))
(.catch (fn [error]
(if (= 404 (:status (ex-data error)))
(-> (footage/measure-crops! settings (dissoc inputs :interior) nil)
(-> (footage/measure-one! settings (dissoc inputs :interior) nil)
(.then (fn [measured]
(let [{:keys [key descriptor data]}
(source/interior-block analysis settings
(source/interior-block analysis subject settings
(:interior measured))]
(-> (upload-missing!
#js {:blocks #js [#js {:key key
@ -241,8 +244,13 @@
(:promise @retained-source)
(let [promise
(if (= "synth" (:detector analysis))
;; The synthetic take tracks one face and regenerating it reads
;; that face's landmarks, so it arrives in the same shape real
;; footage does rather than in a flat one only this branch uses.
(js/Promise.resolve
{:dense (synth/synth-dense (:frames analysis) {:seed (:seed analysis)})})
{:subjects
{:face-1 {:dense (synth/synth-dense (:frames analysis)
{:seed (:seed analysis)})}}})
(-> (ingest/manifest! (:footage-id entry))
(.then (fn [manifest]
(-> (footage/saved-source! (:id analysis)
@ -251,29 +259,43 @@
(.then (fn [inputs]
(when-not inputs
(throw (ex-info "saved analysis has no source blocks" {})))
(assoc inputs :presence (:presence manifest)))))))))]
(update inputs :subjects
(fn [subjects]
(into {}
(map (fn [[id one]]
[id (assoc one :presence
(footage/presence-for
manifest id))]))
subjects))))))))))]
(do
(reset! retained-source {:id (:id analysis) :promise promise})
promise))))))
(defn- inputs-for-edit! [entry edit inputs]
(let [{clip :changed fids :features} (regenerate/plan (:clip entry) edit)
teeth (first (filter #(= :teeth (get-in clip [:features % :area])) fids))]
(if (and teeth (:crops inputs))
(defn- inputs-for-edit!
"Bring the EDITED SUBJECT's retained pixel measurements up to the settings this
edit needs. Only that subject's: a knob dragged on the second face does not
re-measure the first face's mouth."
[entry edit inputs]
(let [{clip :changed fids :features subject :subject} (regenerate/plan (:clip entry) edit)
teeth (first (filter #(= :teeth (get-in clip [:features % :area])) fids))
one (get-in inputs [:subjects subject])]
(if (and teeth (:crops one))
(let [settings (merge take/knobs (feature/effective-params clip teeth))
analysis (get-in clip [:analysis :id])
frames (count (:crops inputs))
key (source/interior-key analysis settings frames)]
(if (and (:interior inputs)
(or (= (:interior-key inputs) key)
(and (nil? (:interior-key inputs))
(= key (source/interior-key analysis take/knobs frames)))))
frames (count (:crops one))
key (source/interior-key analysis subject settings frames)
done (fn [measured] (assoc-in inputs [:subjects subject] measured))]
(if (and (:interior one)
(or (= (:interior-key one) key)
(and (nil? (:interior-key one))
(= key (source/interior-key analysis subject take/knobs frames)))))
(js/Promise.resolve inputs)
(if (= key (:key @retained-interior))
(:promise @retained-interior)
(let [promise (retained-interior! analysis settings inputs)]
(reset! retained-interior {:key key :promise promise})
promise))))
(.then (if (= key (:key @retained-interior))
(:promise @retained-interior)
(let [promise (retained-interior! analysis subject settings one)]
(reset! retained-interior {:key key :promise promise})
promise))
done)))
(js/Promise.resolve inputs))))
(rf/reg-fx

17
frontend/src/arthur/flow/address.cljs
View file

@ -68,7 +68,7 @@
width, which is a visible difference on a mouth. Optional, because the synthetic
take has no running mode to declare and an absent field is how the other
optional inputs already say \"not applicable\"."
[{:keys [detector version source footage frames fps aspect seed mode]}]
[{:keys [detector version source footage frames fps aspect seed mode tracking]}]
(when-not (and (string? detector) (seq detector) (string? version) (seq version))
(throw (ex-info "an analysis names its detector and the detector's VERSION: an upgrade that silently reuses old landmarks is the failure content addressing exists to prevent"
{:detector detector :version version})))
@ -81,7 +81,8 @@
source (assoc :source source)
footage (assoc :footage footage)
seed (assoc :seed seed)
mode (assoc :mode mode))))
mode (assoc :mode mode)
tracking (assoc :tracking tracking))))
(defn analysis
"An analysis record with its `:id` filled in. The record is tier 1 — it says
@ -94,12 +95,9 @@
;; the observation masks
(defn- feature-name
"A feature id as a descriptor string. `canon` refuses a keyword value on
purpose — so that \"mouth\" and :mouth cannot address the same block — and this
is the naming it insists happens at the call site. `nil` is the whole face,
which is what a track with no feature follows."
"An explicit subject or feature id as a descriptor string."
[id]
(if id (subs (str id) 1) "face"))
(subs (str id) 1))
(defn observation
"A digest of exactly the absence data one block reads.
@ -111,10 +109,9 @@
block reads `:eye-r` and `:eye-l`'s presence and no other feature's, so a gap in
one brow does not rewrite the mouth's address for nothing.
`nil` features mean the track follows the whole face's detection rather than a
feature's presence, which is what the head's three blocks do."
Head tracks name their subject and follow its detection mask."
[features {:keys [detected presence]}]
(let [wanted (sort-by str (distinct (keep identity features)))
(let [wanted (sort-by str (distinct features))
masks (into {} (map (fn [id] [id (mapv boolean (get presence id))]))
(filter #(contains? presence %) wanted))]
(when (or detected (seq masks))

122
frontend/src/arthur/flow/detect.cljs
View file

@ -4,6 +4,10 @@
(defonce ^:private instance (atom nil))
(defonce ^:private pending (atom nil))
(def settings
"Detection and assignment inputs, also included in the analysis address."
{:max-faces 4 :assignment "nearest-centroid-v1" :gate 0.2})
(defn discard!
"Throw away the cached landmarker so the next run builds a fresh one.
@ -43,7 +47,7 @@
#js {:modelAssetPath "/static/mediapipe/face_landmarker.task"
:delegate "CPU"}
:runningMode "VIDEO"
:numFaces 1})))
:numFaces (:max-faces settings)})))
(.then (fn [model]
(reset! instance model)
model))
@ -55,8 +59,8 @@
(js/Promise.reject (js/Error. "local MediaPipe script did not load"))))))
(defn detect!
"Detect one already-drawn canvas frame at its own time in the take. nil means no
face was detected.
"Detect one already-drawn canvas frame at its own time in the take. An empty
vector means no face was detected.
`at-ms` IS THE FRAME'S REAL PRESENTATION TIME, and all three words are
load-bearing. In VIDEO mode the graph is a tracker: it runs face DETECTION only
@ -75,14 +79,95 @@
against 0.013), because a tracker told that every frame is 1ms apart expects a
face that has barely moved."
[model canvas at-ms]
(when-let [face (aget (aget (.call (aget model "detectForVideo") model canvas at-ms)
"faceLandmarks") 0)]
(mapv (fn [p] {:x (.-x p) :y (.-y p) :z (.-z p)}) face)))
(let [faces (aget (.call (aget model "detectForVideo") model canvas at-ms)
"faceLandmarks")]
(mapv (fn [face] (mapv (fn [p] {:x (.-x p) :y (.-y p) :z (.-z p)}) face))
(array-seq faces))))
;; ---------------------------------------------------------------------------
;; which face is which
;;
;; MediaPipe hands back a LIST, and a list has an order rather than an identity.
;; Nothing in the API promises that slot 0 is the same person on frame 41 as on
;; frame 40 — and when two faces cross, or one is briefly lost and re-detected,
;; it is not. Left unassigned, the two faces' geometry would swap mid-shot inside
;; one dense block, which reads as both heads snapping and is invisible in any
;; per-frame assertion.
;;
;; So identity is assigned HERE, once, by nearest centroid to each track's last
;; known position. Greedy and cheap: a handful of faces, one pass, and the
;; ordering it produces is the `:subjects` map every later stage keys on.
(defn- centroid [face]
(let [n (count face)]
{:x (/ (reduce + (map :x face)) n)
:y (/ (reduce + (map :y face)) n)}))
(defn- distance [a b]
(let [dx (- (:x a) (:x b)) dy (- (:y a) (:y b))]
(js/Math.sqrt (+ (* dx dx) (* dy dy)))))
(defn assign
"Per-frame lists of faces -> one vector per TRACK of `detection index or nil`.
INDICES AND NOT FACES, because a detection is more than its landmarks: the
mouth crop and the pixel measurement taken beside it on the same frame have to
follow the same face, and handing back the index is what lets one assignment
re-key all three. Whoever holds the per-frame lists does the lookup.
A track is claimed by the unclaimed detection nearest its last known centroid,
nearest pair first, so a frame where MediaPipe swaps its slot order does not
swap the tracks. A detection that matches no existing track within `gate`
starts a new one — which is how a second person walking into the shot on frame
200 gets their own subject instead of stealing the first one's.
`gate` is in normalised image units: two centroids closer than this on
consecutive frames are one face moving, and further apart are not."
([frames] (assign frames (:gate settings)))
([frames gate]
(let [step (fn [[rows last] faces]
(let [cs (mapv centroid faces)
pairs (sort-by :d
(for [[t at] (map-indexed vector last)
:when at
[d c] (map-indexed vector cs)
:let [gap (distance at c)]
:when (< gap gate)]
{:d gap :track t :face d}))
claim (reduce (fn [{:keys [by-track taken] :as acc}
{:keys [track face]}]
(if (or (contains? by-track track)
(contains? taken face))
acc
{:by-track (assoc by-track track face)
:taken (conj taken face)}))
{:by-track {} :taken #{}}
pairs)
opened (map-indexed (fn [i d] [(+ (count last) i) d])
(remove (:taken claim) (range (count faces))))
by-track (into (:by-track claim) opened)
width (+ (count last) (count opened))]
[(conj rows (mapv by-track (range width)))
(mapv (fn [t] (if-let [d (get by-track t)]
(nth cs d)
(nth last t nil)))
(range width))]))
[rows _] (reduce step [[] []] frames)
width (reduce max 0 (map count rows))]
;; Transposed to column-major: one vector per subject, padded to the final
;; width so a track that opened late still spans the whole take.
(mapv (fn [t] (mapv (fn [row] (nth row t nil)) rows))
(range width)))))
(defn fill-gaps
"Keep the detection mask while supplying real poses for measurement. A leading
gap uses the first observed face; later gaps hold the previous observed pose.
Freeze uses the mask to mark those frames absent in the channel blocks."
Freeze uses the mask to mark those frames absent in the channel blocks.
ONE TRACK, which is one subject: the mask says when THIS face was on screen,
and two faces in a shot have two of them. A frame where the second person has
not walked in yet is a frame their track is absent on, which is the same fact
as a frame nobody was found on and needs no second mechanism."
[raw]
(let [first-real (first (keep-indexed (fn [i frame] (when frame i)) raw))]
(when-not first-real
@ -95,3 +180,26 @@
:detected (mapv some? raw)
:missing (count (remove some? raw))
:first-real first-real}))
(defn subject-id
"Track index -> the subject that owns it. `:face-1` is the first face found,
which is also the id a single-face take has always used."
[i]
(keyword (str "face-" (inc i))))
(defn tracks
"Per-frame detection lists -> `{:face-1 <slots>, …}`, one entry per tracked
face, each a per-frame detection index or nil."
[frames]
(let [assigned (assign frames)]
(when (empty? assigned)
(throw (ex-info "no face found in any frame — check framing and light" {})))
(into {} (map-indexed (fn [i slots] [(subject-id i) slots])) assigned)))
(defn pick
"One track's slots applied to anything measured PER DETECTION — the landmarks,
the mouth crop, the pixel measurement taken beside it. All three were recorded
in detection order on a frame where nobody yet knew whose face was whose, and
this is the single lookup that turns any of them into one subject's track."
[per-frame slots]
(mapv (fn [row slot] (when slot (nth row slot))) per-frame slots))

492
frontend/src/arthur/flow/freeze.cljs
View file

@ -34,6 +34,7 @@
decisions about the mouth cavity, blink and teeth without thinning geometry."
(:require [arthur.domain.channel :as ch]
[arthur.domain.clip :as clip]
[arthur.domain.feature :as feature]
[arthur.domain.geom :as geom]
[arthur.domain.ring :as ring]
[arthur.flow.address :as address]))
@ -82,7 +83,7 @@
ABSENCE IS PER TRACK, and `:features` is what says whose. Each track names the
feature it follows, so one occluded eye can be absent while its partner still
has a value; `nil` means the track follows the whole face's detection and no
has a value; a subject id means the track follows whole-face detection and no
feature, which is what the head's blocks do. `absent?` is then asked
`(absent? feature f)` and never about a track index.
@ -248,31 +249,44 @@
(def ^:private head-modes #{:free :anchored})
(defn head-mode
"Keep the measured transform dense; optionally hold chosen source frames.
"Keep a subject's measured transform dense; optionally hold chosen source
frames.
A nil anchor map reads measured frame f at frame f (free movement).
`{0 12}` locks to the measured transform of source frame 12. `{0 12, 40 42}`
cuts to source frame 42 at local frame 40. The same map selects position,
rotation and scale, so the head and registered photo cannot drift apart.
No analysis block or authored face placement changes."
[{:keys [mode anchors]} {:keys [clip]}]
No analysis block or authored face placement changes.
ONE SUBJECT AT A TIME when `:subject` is given, and EVERY subject when it is
not. Two faces in one shot were filmed together and are posed apart: choosing
frame 12 for the second face must leave the first one running, and it does,
because an anchor map lives on that subject's own head node and
`domain/timeline` reads anchors off whatever node carries them."
[{:keys [subject mode anchors]} {:keys [clip]}]
(when-not (contains? head-modes mode)
(throw (ex-info "head mode must be free or anchored"
{:mode mode :modes head-modes})))
(when (and (= mode :free) (some? anchors))
(throw (ex-info "free head motion has no anchors" {:anchors anchors})))
(let [frames (clip/frames clip)]
(when (and (= mode :anchored)
(not (and (map? anchors) (contains? anchors 0)
(every? #(and (integer? %) (<= 0 %) (< % frames))
(concat (keys anchors) (vals anchors))))))
(throw (ex-info "anchored head needs a frame-zero key and valid source frames"
{:anchors anchors :frames frames})))
(update-in clip [:timelines clip/root-id :nodes :head]
(fn [n]
(cond-> (assoc n :channels (:measured n))
(= mode :anchored) (assoc :anchors anchors)
(= mode :free) (dissoc :anchors))))))
(when (and subject (not (contains? (:subjects clip) subject)))
(throw (ex-info "head mode names a subject this clip did not track"
{:subject subject :subjects (vec (sort-by str (keys (:subjects clip))))})))
(reduce
(fn [c sid]
(let [frames (get-in c [:timelines sid :frames])]
(when (and (= mode :anchored)
(not (and (map? anchors) (contains? anchors 0)
(every? #(and (integer? %) (<= 0 %) (< % frames))
(concat (keys anchors) (vals anchors))))))
(throw (ex-info "anchored head needs a frame-zero key and valid source frames"
{:subject sid :anchors anchors :frames frames})))
(update-in c [:timelines sid :nodes :head]
(fn [n]
(cond-> (assoc n :channels (:measured n))
(= mode :anchored) (assoc :anchors anchors)
(= mode :free) (dissoc :anchors))))))
clip (if subject [subject] (sort-by str (keys (:subjects clip))))))
;; ---------------------------------------------------------------------------
;; the aperture, onto [:vis] of :mouth-in
@ -323,53 +337,56 @@
:brow-r :brow-r :brow-l :brow-l})
(defn- performance-nodes
"Mark the generated moving channels that may read an instance's pose choice.
Provenance helps identify them here; the renderer reads only :pose-sampled?."
"Mark channels that read the containing instance's pose choices."
[nodes]
(into {}
(map (fn [[id n]]
[id (cond-> n
(contains? pose-groups id)
(assoc :pose-group (get pose-groups id))
(contains? pose-groups id)
(update :channels
(fn [channels]
(into {}
(map (fn [[path ch]]
[path (cond-> ch
(and (:generated ch) (:animated? ch))
(assoc :pose-sampled? true))]))
channels))))]))
[id (if-let [group (get pose-groups id)]
(-> n
(assoc :pose-group group)
(update :channels
(fn [channels]
(into {}
(map (fn [[path ch]]
[path (cond-> ch
(and (:generated ch) (:animated? ch))
(assoc :pose-sampled? true))]))
channels))))
n)]))
nodes))
;; ---------------------------------------------------------------------------
;; the face, onto the stage
(defn- motion-placement
"An editable default framing for a real take. Fit every drawn face feature's
observed motion inside the stage, including raised brows and a moving mouth."
[[w h] {:keys [rigid transforms outer eyes brows detected]}]
(let [points (mapcat (fn [i]
(when (or (nil? detected) (nth detected i))
(let [local (concat (nth outer i)
(when eyes (concat (nth (:lash-r eyes) i)
(nth (:lash-l eyes) i)))
(when brows (concat (nth (:ring-r brows) i)
(nth (:ring-l brows) i))))]
(concat (nth rigid i)
(geom/apply-sim-all (invert (nth transforms i))
local)))))
(range (count outer)))
xs (map :x points)
ys (map :y points)
x0 (reduce min xs)
x1 (reduce max xs)
y0 (reduce min ys)
y1 (reduce max ys)
cx (/ (+ x0 x1) 2)
cy (/ (+ y0 y1) 2)
k (min (/ (* 0.8 w) (max 1e-9 (- x1 x0)))
(/ (* 0.8 h) (max 1e-9 (- y1 y0))))]
(defn- motion-points
"Every source-space point one subject's drawn features visit over the shot,
including raised brows and a moving mouth."
[{:keys [rigid transforms outer eyes brows detected]}]
(mapcat (fn [i]
(when (or (nil? detected) (nth detected i))
(let [local (concat (nth outer i)
(when eyes (concat (nth (:lash-r eyes) i)
(nth (:lash-l eyes) i)))
(when brows (concat (nth (:ring-r brows) i)
(nth (:ring-l brows) i))))]
(concat (nth rigid i)
(geom/apply-sim-all (invert (nth transforms i))
local)))))
(range (count outer))))
(defn- fit-points
"Source-space points -> the placement that puts their bounding box on stage."
[[w h] points]
(let [xs (map :x points)
ys (map :y points)
x0 (reduce min xs)
x1 (reduce max xs)
y0 (reduce min ys)
y1 (reduce max ys)
cx (/ (+ x0 x1) 2)
cy (/ (+ y0 y1) 2)
k (min (/ (* 0.8 w) (max 1e-9 (- x1 x0)))
(/ (* 0.8 h) (max 1e-9 (- y1 y0))))]
{[:xform :anchor] (ch/framed [cx cy])
[:xform :scale] (ch/framed [k k])
[:xform :pos] (ch/framed [(- (/ w 2) cx) (- (/ h 2) cy)])}))
@ -407,26 +424,33 @@
POSITION puts the anchor a QUARTER of the way down the stage, because the rigid
landmarks are eyes and nose — the upper middle of a face — so a quarter down
leaves the jaw and the mouth on the stage. Whatever hangs off is clipped, which
is not a feature to add: every fill in `domain/raster` clamps already."
[{:keys [stage fit-motion?]} {:keys [ref] :as inputs}]
(if fit-motion?
(motion-placement stage inputs)
(let [[w h] stage
c (geom/centroid ref)
span (- (reduce max (map :x ref)) (reduce min (map :x ref)))
k (/ (* 0.4 w) span)]
{[:xform :anchor] (ch/framed [(:x c) (:y c)])
[:xform :scale] (ch/framed [k k])
[:xform :pos] (ch/framed [(- (/ w 2) (:x c))
(- (* 0.25 h) (:y c))])})))
is not a feature to add: every fill in `domain/raster` clamps already.
(defn- mouth-part [absent? obs
All subjects share one source-to-stage mapping on the :face group. Each
instance can then be placed independently with ordinary transform channels."
[{:keys [stage fit-motion?]} subjects]
(let [[w h] stage
inputs (vals subjects)]
(if fit-motion?
(fit-points stage (mapcat motion-points inputs))
(let [ref (mapcat :ref inputs)
c (geom/centroid ref)
span (- (reduce max (map :x ref)) (reduce min (map :x ref)))
k (/ (* 0.4 w) span)]
{[:xform :anchor] (ch/framed [(:x c) (:y c)])
[:xform :scale] (ch/framed [k k])
[:xform :pos] (ch/framed [(- (/ w 2) (:x c))
(- (* 0.25 h) (:y c))])}))))
(defn- mouth-part [subject absent? obs
{:keys [analysis verts anchor-avg contour-avg aperture-cut] :as params}
{:keys [outer inner] :as inputs}]
(let [rings (block {:role "geom" :analysis (:id analysis) :params params
(let [own (partial feature/owned subject)
mouth (own :mouth)
rings (block {:role "geom" :analysis (:id analysis) :params params
:tracks ["outer" "inner"]}
{:type "int16" :scale geom-scale
:features [:mouth :mouth] :absent? absent?}
:features [mouth mouth] :absent? absent?}
obs [(rings->flat outer verts) (rings->flat inner verts)])
prov (fn [by]
{:by by :analysis (:id analysis)
@ -452,9 +476,11 @@
"Freeze eyes and brows into their own dense blocks and scene nodes. This owns
only representation: the landmark correspondence, blink and pose choices have
already been settled by measure and condition."
[absent? obs {:keys [eye-verts brow-verts analysis contour-avg anchor-avg] :as params}
[subject absent? obs
{:keys [eye-verts brow-verts analysis contour-avg anchor-avg] :as params}
{:keys [eyes brows]}]
(let [provenance (fn [by extra]
(let [own (partial feature/owned subject)
provenance (fn [by extra]
{:by by :analysis (:id analysis)
:params (merge {:anchor-avg anchor-avg :contour-avg contour-avg}
extra)})
@ -470,20 +496,23 @@
;; and `each-dense-track-follows-its-own-features-presence` is what pins it.
eye-block (when eyes (named "eyes"
["lash-r" "lid-r" "lash-l" "lid-l"]
[:eye-r :eye-r :eye-l :eye-l]
[(own :eye-r) (own :eye-r) (own :eye-l) (own :eye-l)]
"int16"
(mapv #(rings->flat % eye-verts)
[(:lash-r eyes) (:lid-r eyes)
(:lash-l eyes) (:lid-l eyes)])))
iris-block (when eyes
(named "iris-pos" ["iris-r" "iris-l"] [:eye-r :eye-l] "float32"
(named "iris-pos" ["iris-r" "iris-l"]
[(own :eye-r) (own :eye-l)] "float32"
[(:iris-r eyes) (:iris-l eyes)]))
brow-block (when brows
(named "brows" ["ring-r" "ring-l"] [:brow-r :brow-l] "int16"
(named "brows" ["ring-r" "ring-l"]
[(own :brow-r) (own :brow-l)] "int16"
(mapv #(rings->flat % brow-verts)
[(:ring-r brows) (:ring-l brows)])))
brow-pos-block (when brows
(named "brow-pos" ["pos-r" "pos-l"] [:brow-r :brow-l] "float32"
(named "brow-pos" ["pos-r" "pos-l"]
[(own :brow-r) (own :brow-l)] "float32"
[(:pos-r brows) (:pos-l brows)]))
eye-node (fn [id z track]
{:id id :name (clojure.core/name id) :kind :poly :parent :head :z z
@ -525,14 +554,14 @@
[:style :color] (ch/framed :brow)}})]
{:nodes (merge
(when eyes
{:eye-r (eye-node :eye-r "a2" 0)
:eye-r-in (inner-node :eye-r-in :eye-r "a1" 1 (:shut-r eyes))
:iris-r (iris-node :iris-r :eye-r-in 0 (:radius-r eyes))
:pupil-r (pupil-node :pupil-r :iris-r)
:eye-l (eye-node :eye-l "a3" 2)
:eye-l-in (inner-node :eye-l-in :eye-l "a1" 3 (:shut-l eyes))
:iris-l (iris-node :iris-l :eye-l-in 1 (:radius-l eyes))
:pupil-l (pupil-node :pupil-l :iris-l)})
{:eye-r (eye-node :eye-r "a2" 0)
:eye-r-in (inner-node :eye-r-in :eye-r "a1" 1 (:shut-r eyes))
:iris-r (iris-node :iris-r :eye-r-in 0 (:radius-r eyes))
:pupil-r (pupil-node :pupil-r :iris-r)
:eye-l (eye-node :eye-l "a3" 2)
:eye-l-in (inner-node :eye-l-in :eye-l "a1" 3 (:shut-l eyes))
:iris-l (iris-node :iris-l :eye-l-in 1 (:radius-l eyes))
:pupil-l (pupil-node :pupil-l :iris-l)})
(when brows
{:brow-r (brow-node :brow-r "a4" 0)
:brow-l (brow-node :brow-l "a5" 1)}))
@ -542,10 +571,11 @@
(defn- interior-part
"Freeze the pixel-derived radial contour under the mouth cavity. Missing
contours use the dense block's absence bit; contrast decides editable :vis."
[{:keys [analysis teeth-verts cavity-erode tongue-reject blob-grow
top-bias teeth-on teeth-smooth] :as params} absent? obs
[subject {:keys [analysis teeth-verts cavity-erode tongue-reject blob-grow
top-bias teeth-on teeth-smooth] :as params} absent? obs
{:keys [contours shown]}]
(let [empty-points (vec (repeat (* 2 teeth-verts) 0))
(let [own (partial feature/owned subject)
empty-points (vec (repeat (* 2 teeth-verts) 0))
values (mapv (fn [ring]
(if ring
(into [] (mapcat (juxt :x :y)) ring)
@ -553,7 +583,7 @@
blk (block {:role "teeth" :analysis (:id analysis) :params params
:tracks ["contour"]}
{:type "int16" :scale geom-scale
:features [:teeth] :absent? absent?
:features [(own :teeth)] :absent? absent?
;; Not the feature's absence: a frame no contour could be
;; extracted from has no teeth to draw whether or not the
;; teeth were occluded, and the two reasons are different facts.
@ -565,7 +595,8 @@
:top-bias top-bias :teeth-verts teeth-verts
:teeth-on teeth-on :teeth-smooth teeth-smooth}}]
{:nodes {:teeth
{:id :teeth :name "teeth" :kind :poly :parent :mouth-in :z "a1"
{:id :teeth :name "teeth" :kind :poly
:parent :mouth-in :z "a1"
:stencil :mouth-in
:channels {[:geom :pts] (dense blk 0 generated)
[:style :color] (ch/framed :teeth)
@ -573,32 +604,40 @@
:store (stored blk)}))
(defn part
"A feature type's nodes and addressed blocks, with no clip scaffold."
[area params {:keys [detected presence] :as measured}]
(let [absent? (when (or detected presence)
"One feature type: local nodes and blocks addressed by subject and feature."
[subject area params {:keys [detected presence] :as measured}]
(let [presence (into {} (map (fn [[role mask]] [(feature/owned subject role) mask])) presence)
absent? (when (or detected presence)
(fn [id f]
(or (and detected (not (nth detected f true)))
(and (contains? presence id)
(not (nth (get presence id) f))))))
obs {:detected detected :presence presence}]
(update (case area
:mouth (mouth-part absent? obs params measured)
:eye (feature-parts absent? obs params (select-keys measured [:eyes]))
:brow (feature-parts absent? obs params (select-keys measured [:brows]))
:teeth (interior-part params absent? obs (:teeth measured))
:mouth (mouth-part subject absent? obs params measured)
:eye (feature-parts subject absent? obs params (select-keys measured [:eyes]))
:brow (feature-parts subject absent? obs params (select-keys measured [:brows]))
:teeth (interior-part subject params absent? obs (:teeth measured))
(throw (ex-info "unknown frozen feature type" {:area area})))
:nodes performance-nodes)))
(defn head-part
"Freeze only the measured head transform from an already conditioned anchor."
[{:keys [analysis anchor-avg] :as params} {:keys [transforms detected presence]}]
"Freeze one subject's measured head transform from its conditioned anchor.
THE THREE BLOCKS NAME THE SUBJECT, and that is not decoration. A block's key is
a hash over its descriptor, and the head follows DETECTION rather than any
feature's presence — so with `nil` in the feature slot, two faces tracked in one
analysis, both detected on every frame, produced byte-for-byte different
transforms under one identical key, and the second freeze's block silently
replaced the first's. The subject is the feature the head follows."
[subject {:keys [analysis anchor-avg] :as params} {:keys [transforms detected presence]}]
(let [absent? (when (or detected presence)
(fn [_ f] (and detected (not (nth detected f true)))))
inv (mapv invert transforms)
xf (fn [role f]
(block {:role role :analysis (:id analysis) :params params
:tracks [role]}
{:type "float32" :features [nil] :absent? absent?}
{:type "float32" :features [subject] :absent? absent?}
{:detected detected}
[(mapv f inv)]))
pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)]))
@ -614,184 +653,83 @@
;; ---------------------------------------------------------------------------
;; the clip
(defn- subject-part
"A subject's drawing, metadata and blocks. Node names are timeline-local."
[params subject {:keys [outer eyes brows teeth] :as inputs}]
(let [own (partial feature/owned subject)
areas (cond-> [:mouth] (and eyes brows) (into [:eye :brow]) teeth (conj :teeth))
parts (mapv #(part subject % params inputs) areas)
head (head-part subject params inputs)
features (cond-> {:mouth [:mouth [:mouth :mouth-in]]}
(and eyes brows)
(merge {:eye-r [:eye [:eye-r :eye-r-in :iris-r :pupil-r]]
:eye-l [:eye [:eye-l :eye-l-in :iris-l :pupil-l]]
:brow-r [:brow [:brow-r]] :brow-l [:brow [:brow-l]]})
teeth (assoc :teeth [:teeth [:teeth]]))]
{:timeline {:id subject :frames (count outer)
:nodes (into {:head {:id :head :name "head" :kind :group :z "a1"
:measured (:measured head)}}
(mapcat :nodes) parts)}
:features (into {} (map (fn [[role [area nodes]]]
[(own role) {:id (own role) :subject subject
:timeline subject :area area
:nodes nodes :params {}}])) features)
:groups (if (and eyes brows)
{(own :eyes) {:id (own :eyes) :kind :eye-pair :subject subject
:members [(own :eye-r) (own :eye-l)] :params {}}}
{})
:store (into (:store head) (mapcat :store) parts)}))
(defn clip
"Conditioned measurements -> a clip: nodes with frozen channels, plus the dense
blocks they read. `(f params inputs)`, no state.
"Subject-id -> conditioned measurements becomes a library of face timelines.
params
:name labels the clip. It is NOT in any key: two clips of the same
footage under different names are the same analysis and the
same blocks, and sharing them is the return on addressing.
:fps the clip's rate. FRAMES is not a parameter — it is
`(count outer)`, because a freeze that could disagree with its
own input about the length of the take would.
:stage [w h] project dimensions, INDEPENDENT of the footage
:fit-motion? choose an editable real-footage default that keeps the
observed feature motion on stage
:expose the clip root's exposure grid, inherited by everything
:verts the lip rings' vertex budget
:eye-verts the eyelid rings' vertex budget
:brow-verts the brow rings' vertex budget
:aperture-cut fraction of the take's peak aperture below which the mouth
interior is not present
:head :free | :anchored
:anchors optional map of local change frame -> measured source frame
:analysis the analysis record these measurements came from — detector,
VERSION, source, source cadence and pixel aspect. Its `:id` is
a content address over all of that, every block's key is a hash
over that id, and `flow/address` says why the version being in
there is the one field that must not be forgotten.
:anchor-avg
:contour-avg the stage-4 knobs. Freeze does not use them; it RECORDS them,
because `:generated` is what lets the UI offer a re-freeze at
different parameters instead of raw keys.
:main holds exposure and a shared source-to-stage placement. Each subject is
placed by an ordinary symbol instance, so pose choices and transforms have
their existing instance scope. Features name local nodes in that subject's
timeline; block descriptors still name globally distinct features.
inputs
:ref the Procrustes reference configuration
:transforms the CONDITIONED anchor transforms
:outer :inner the CONDITIONED head-local lip rings
:eyes :brows the CONDITIONED head-local feature measurements
:teeth optional pixel-derived, conditioned radial contour
:aperture head-local aperture per frame
:detected optional per-frame face booleans
:presence optional feature-id -> per-frame booleans; false means an
unobserved feature, even if the rest of the face was found
The node tree is the one docs/animation-model.md specifies, and the two groups
are two different things wanting the same transform:
:root the clip. EXPOSURE LIVES HERE and is inherited strictly.
:face AUTHORED. where the face sits on the stage, and how big.
:head MEASURED. the head's motion, or identity.
:mouth
:mouth-in
:teeth
:eye-r / :eye-l
:eye-*-in
:iris-*
:pupil-*
:brow-r / :brow-l
`:mouth-in`'s parent is `:mouth` and that transform is identity today, so
composing it is composing identity. It is documented intent, and it is the one
place in the tree where the parent pointer is not yet doing work — the moment a
painted cel rides a moving plate it will be.
`:head` keeps its measured channels under `:measured` as well as in
`:channels`; its optional `:anchors` map changes only their frame address."
[{:keys [name fps stage expose verts aperture-cut head anchors
analysis anchor-avg contour-avg] :as params}
{:keys [transforms outer inner detected presence eyes brows teeth] :as inputs}]
(let [nf (count outer)
_ (doseq [[id track] presence]
(when (not= nf (count track))
(throw (ex-info "feature presence track must match the clip"
{:feature id :frames nf :actual (count track)}))))
;; Asked `(absent? feature f)`, where a nil feature is the whole face. A
;; feature with no presence track is present whenever a face was found.
absent? (when (or detected presence)
(fn [id f]
(or (and detected (not (nth detected f true)))
(and (contains? presence id)
(not (nth (get presence id) f))))))
obs {:detected detected :presence presence}
prov (fn [by extra]
{:by by :analysis (:id analysis)
:params (merge {:anchor-avg anchor-avg} extra)})
mouth (mouth-part absent? obs params inputs)
;; The anchor, inverted and split into its three components. Three blocks
;; and not one: they are three channels, they have three strides, and a
;; single block would need a per-component offset table to say so.
inv (mapv invert transforms)
xf (fn [role f]
;; The head follows DETECTION and no feature's presence: an
;; occluded eye does not mean the head was not there. That is
;; what the nil feature says.
(block {:role role :analysis (:id analysis) :params params
:tracks [role]}
{:type "float32" :features [nil] :absent? absent?}
{:detected detected}
[(mapv f inv)]))
pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)]))
rot (xf "head-rot" (fn [t] [(:theta t)]))
scale (xf "head-scale" (fn [t] [(:s t) (:s t)]))
;; Which knobs each channel records is not decoration, it is the
;; invalidation table written down where a re-freeze can read it. The
;; anchor depends on `anchor avg` alone. The rings depend on it and on
;; `contour avg` and on the vertex budget. The aperture depends on
;; `anchor avg` and on its own cut and NOT on `contour avg`, because
;; measure reports the inner ring's own height and nothing smooths it.
anchor-prov (prov :anchor/similarity nil)
features (when (and eyes brows) (feature-parts absent? obs params inputs))
interior (when teeth (interior-part params absent? obs teeth))
built {:name name
:fps fps
;; Tier 1 says which analysis its channels came out of, in full.
;; The id alone would make the document unreadable the first time
;; a detector upgrade orphaned a block: "sha256:7f2…" is not an
;; answer to "which model produced this take".
:analysis analysis
;; A gap changes channel state, never these IDs or pair links.
:subjects {:face-1 {:id :face-1 :params {}}}
:features (merge
{:mouth {:id :mouth :subject :face-1 :area :mouth
:nodes [:mouth :mouth-in] :params {}}}
(when features
{:eye-r {:id :eye-r :subject :face-1 :area :eye
:nodes [:eye-r :eye-r-in :iris-r :pupil-r] :params {}}
:eye-l {:id :eye-l :subject :face-1 :area :eye
:nodes [:eye-l :eye-l-in :iris-l :pupil-l] :params {}}
:brow-r {:id :brow-r :subject :face-1 :area :brow
:nodes [:brow-r] :params {}}
:brow-l {:id :brow-l :subject :face-1 :area :brow
:nodes [:brow-l] :params {}}})
(when teeth
{:teeth {:id :teeth :subject :face-1 :area :teeth
:nodes [:teeth] :params {}}}))
:groups (if features
{:eyes-1 {:id :eyes-1 :kind :eye-pair :subject :face-1
:members [:eye-r :eye-l] :params {}}}
{})
;; Stage dimensions, on the clip and not on the footage. See
;; face-placement: nothing below here knows the frame size.
:width (first stage)
:height (second stage)
;; ONE TIMELINE, and the frame count is ITS. A clip is a rate and
;; a timeline is a frame space — see arthur.domain.clip — so `nf`
;; lands here and `:fps` above, and the library a symbol will live
;; in is this same map with a second entry.
:timelines
{clip/root-id
{:id clip/root-id
:frames nf
:nodes
(merge
{:root
{:id :root :name "clip" :kind :group :parent nil :z "a1"
:time {:mode :map :expose expose}}
:face
{:id :face :name "face" :kind :group :parent :root :z "a1"
:channels (face-placement params inputs)}
:head
{:id :head :name "head" :kind :group :parent :face :z "a1"
:measured {[:xform :pos] (dense pos 0 anchor-prov)
[:xform :rot] (dense rot 0 anchor-prov)
[:xform :scale] (dense scale 0 anchor-prov)}}}
(:nodes mouth) (:nodes features) (:nodes interior))}}}
;; Tier 2, behind a handle, and now behind a content address: every key is
;; a sha256 over the analysis, the settings and the absence data that
;; produced the bytes under it. Nothing above this line changed when they
;; stopped being "take/geom", which is the point of a handle.
store (merge (stored pos rot scale) (:store mouth)
(:store features) (:store interior))]
(doseq [id (keys presence)]
(when-not (contains? (:features built) id)
(throw (ex-info "presence track names no feature in this clip"
{:feature id :features (keys (:features built))}))))
{:store store
:clip (head-mode {:mode head :anchors anchors}
{:clip (update-in built [:timelines clip/root-id :nodes]
performance-nodes)
:store store})}))
Subjects share a source frame space. :head and :anchors may be overridden
per subject; all other freeze settings come from params."
[{:keys [name fps stage expose head anchors] :as params} subjects]
(when-not (and (map? subjects) (seq subjects)
(every? keyword? (keys subjects))
(not-any? #{:main :root :face} (keys subjects)))
(throw (ex-info "a freeze needs subjects with ids distinct from :main, :root and :face" {})))
(let [ordered (sort-by (comp str key) subjects)
parts (mapv (fn [[id inputs]] [id (subject-part params id inputs)]) ordered)
lengths (distinct (map #(get-in % [1 :timeline :frames]) parts))
_ (when-not (and (= 1 (count lengths)) (pos? (first lengths)))
(throw (ex-info "subjects need the same positive frame count"
{:frames (vec lengths)})))
nf (first lengths)
merged (fn [k] (into {} (mapcat (comp k second)) parts))
built {:name name :fps fps :analysis (:analysis params)
:width (first stage) :height (second stage)
:subjects (into {} (map (fn [[id _]] [id {:id id :params {}}])) ordered)
:features (merged :features) :groups (merged :groups)
:timelines
(into {clip/root-id
{:id clip/root-id :frames nf
:nodes (into {:root {:id :root :name "clip" :kind :group :z "a1"
:time {:mode :map :expose expose}}
:face {:id :face :name "source placement" :kind :group
:parent :root :z "a1"
:channels (face-placement params subjects)}}
(map-indexed
(fn [i [id _]]
[id {:id id :kind :symbol :of id :parent :face
:z (str "a" i)}]))
ordered)}}
(map (fn [[id part]] [id (:timeline part)])) parts)}]
(doseq [[subject inputs] ordered
[id track] (:presence inputs)]
(when-not (and (= nf (count track))
(= subject (get-in built [:features (feature/owned subject id) :subject])))
(throw (ex-info "presence must name this subject's feature and span the take"
{:subject subject :feature id :frames nf :actual (count track)}))))
{:store (merged :store)
:clip (reduce (fn [c [subject inputs]]
(head-mode {:subject subject :mode (or (:head inputs) head)
:anchors (get inputs :anchors anchors)}
{:clip c}))
built ordered)}))

56
frontend/src/arthur/flow/regenerate.cljs
View file

@ -1,8 +1,7 @@
(ns arthur.flow.regenerate
"Recompute a changed feature from retained source tracks, then replace only
channels owned by that feature. Upload remains project/save's ordinary job."
(:require [arthur.domain.clip :as clip]
[arthur.domain.feature :as feature]
(:require [arthur.domain.feature :as feature]
[arthur.domain.params :as params]
[arthur.flow.address :as address]
[arthur.flow.freeze :as freeze]
@ -40,7 +39,8 @@
:when (:generated channel)]
[id prop channel])]
(-> (reduce (fn [entry [id prop channel]]
(let [at [:clip :timelines clip/root-id :nodes id :channels prop]
(let [at [:clip :timelines (get-in entry [:clip :features fid :timeline])
:nodes id :channels prop]
old (get-in entry at)]
(assoc-in entry at
(cond-> channel
@ -85,24 +85,25 @@
owns those input edges, and neither one is a request to freeze the other
feature."
[base-params base source-inputs entry fid]
(let [area (get-in entry [:clip :features fid :area])
(let [{:keys [area subject]} (get-in entry [:clip :features fid])
params (merge base-params (settings (:clip entry) fid))]
(when (and (= :teeth area) (nil? (:interior source-inputs)))
(throw (ex-info "teeth regeneration needs retained pixel measurements"
{:feature fid})))
(replace-feature entry
(freeze/part area params
(freeze/part subject area params
(take/measure-part area params source-inputs @base))
fid)))
(defn- regenerate-head
"Re-freeze the head transform, which is a different job from a feature's: its
only input is the conditioned anchor, and it owns no channels to replace. The
authored `:channels` follow the measurement while they still ARE the
measurement, and are left alone once somebody has placed the head by hand."
[entry params base]
(let [baked (freeze/head-part params @base)
at [:clip :timelines clip/root-id :nodes :head]
"Re-freeze ONE SUBJECT's head transform, which is a different job from a
feature's: its only input is that subject's conditioned anchor, and it owns no
channels to replace. The authored `:channels` follow the measurement while they
still ARE the measurement, and are left alone once somebody has placed the head
by hand."
[entry params base subject]
(let [baked (freeze/head-part subject params @base)
at [:clip :timelines subject :nodes :head]
old (get-in entry at)
measured (:measured baked)]
(cond-> (-> entry
@ -111,13 +112,19 @@
(= (:channels old) (:measured old))
(assoc-in (conj at :channels) measured))))
(defn- change-take
(defn change
"One scoped static edit. `source-inputs` holds dense landmarks and, when the
teeth are dirty, retained pixel measurements. No IO or app-db here."
[{:keys [clip source-inputs] :as entry} edit]
(when-not (:dense source-inputs)
(throw (ex-info "regeneration needs retained source landmarks" {})))
(let [{:keys [changed subject features]} (plan clip edit)
;; THE EDITED SUBJECT'S OWN LANDMARKS. Retained source is per subject —
;; one video, one dense track per tracked face — so re-measuring the
;; second face through the first one's anchor is the mistake this lookup
;; exists to prevent.
source-inputs (get-in source-inputs [:subjects subject])
_ (when-not (:dense source-inputs)
(throw (ex-info "regeneration needs retained source landmarks"
{:subject subject})))
base-params (merge take/knobs
{:fps (:fps changed)
:aspect (get-in changed [:analysis :aspect])
@ -131,21 +138,4 @@
base (delay (take/anchor-base anchor-params source-inputs))]
(cond-> (reduce (partial regenerate-feature base-params base source-inputs)
(assoc entry :clip changed) features)
(= :anchor-avg (:knob edit)) (regenerate-head anchor-params base))))
(defn change
"A take edits its root timeline. A stage edits the shared symbol timeline;
every instance then reads the same new channels on its next paint."
[{:keys [clip] :as entry} edit]
(if-let [symbol (some :timeline (vals (:features clip)))]
(let [original (:timelines clip)
take (assoc clip :timelines
{clip/root-id (assoc (get original symbol) :id clip/root-id)})
changed (change-take (assoc entry :clip take) edit)]
(assoc changed :clip
(-> (:clip changed)
(assoc :timelines
(assoc original symbol
(assoc (get-in changed [:clip :timelines clip/root-id])
:id symbol))))))
(change-take entry edit)))
(= :anchor-avg (:knob edit)) (regenerate-head anchor-params base subject))))

131
frontend/src/arthur/flow/source.cljs
View file

@ -9,25 +9,26 @@
(def roles ["source/dense" "source/detected" "source/crops"])
(defn- interior-address [analysis settings frames]
(defn- interior-address [analysis subject settings frames]
(let [verts (:teeth-verts settings)
stride (+ 2 (* 2 verts))]
(address/block {:role "source/interior" :analysis analysis
:params settings
:tracks ["contrast" "area" "contour"]
:features [] :observation nil
:features [subject] :observation nil
:layout {:type "float64" :frames frames :verts verts
:stride stride}})))
(defn interior-key [analysis settings frames]
(:key (interior-address analysis settings frames)))
(defn interior-key [analysis subject settings frames]
(:key (interior-address analysis subject settings frames)))
(defn interior-block
"Retain pixel measurements before any head-local transform or smoothing."
[analysis settings measures]
"Retain one subject's pixel measurements before any head-local transform or
smoothing."
[analysis subject settings measures]
(let [frames (count measures)
{:keys [key descriptor]}
(interior-address analysis settings frames)
(interior-address analysis subject settings frames)
verts (:teeth-verts settings)
stride (+ 2 (* 2 verts))
data (js/Float64Array. (* frames stride))]
@ -82,20 +83,28 @@
(interior/measure params (:box crop) #js {:data (:data crop)})
{:contour nil :contrast 0 :area 0 :debug nil}))
(defn- named [role analysis tracks layout data]
(defn- named [role analysis subject tracks layout data]
(merge (address/block {:role role :analysis analysis :params {}
:tracks tracks :features [] :observation nil
:tracks tracks :features [subject] :observation nil
:layout layout})
{:role role :data data}))
(defn pack
"Dense landmarks, mask, crops and optional pixel measurements -> source blocks.
"One SUBJECT's dense landmarks, mask, crops and optional pixel measurements ->
source blocks.
`:interior-settings` is not optional when `:interior` is present: the block is
addressed BY those knobs, and defaulting them here would name a block after
settings its bytes did not come from — a content address that lies, which is the
one failure the key exists to prevent."
[analysis {:keys [dense detected crops interior interior-settings]}]
one failure the key exists to prevent.
`subject` is in every block's descriptor, and it has to be. These four blocks
carry no feature — they are the raw analysis, upstream of any part — so two
faces tracked in one video would otherwise produce different bytes under one
identical key, and the second face's landmarks would replace the first's in the
store. That is the same failure `freeze/head-part` guards against one stage
later, for the same reason."
[analysis subject {:keys [dense detected crops interior interior-settings]}]
(let [frames (count dense)
points (count (first dense))]
(when-not (and (pos? frames) (pos? points)
@ -124,52 +133,66 @@
(when-let [crop (nth crops f)]
(.set pixels (:data crop) (nth offsets f))))
(cond-> {"source/dense"
(named "source/dense" analysis ["landmarks"]
(named "source/dense" analysis subject ["landmarks"]
{:type "float64" :frames frames :points points
:stride (* 3 points)}
landmarks)
"source/detected"
(named "source/detected" analysis ["detected"]
(named "source/detected" analysis subject ["detected"]
{:type "uint8" :frames frames :stride 1} mask)
"source/crops"
(named "source/crops" analysis ["rgba"]
(named "source/crops" analysis subject ["rgba"]
{:type "uint8" :frames frames :boxes boxes :offsets offsets}
pixels)}
interior (assoc "source/interior"
(interior-block
analysis
analysis subject
(or interior-settings
(throw (ex-info "an interior block needs the settings its measurements were taken at"
{:frames (count interior)})))
interior))))))
(defn wire-blocks [blocks]
(into-array
(map (fn [role]
(let [{:keys [key descriptor data]} (get blocks role)]
#js {:key key :descriptor descriptor :data (wire/base64 data)}))
roles)))
(defn pack-subjects
"Every tracked subject's source blocks: `{subject {role block}}`.
(defn upload-blocks [blocks]
(into-array
(map (fn [role]
(let [{:keys [key descriptor data]} (get blocks role)]
#js {:key key :descriptor descriptor :data data}))
(filter #(contains? blocks %) (conj roles "source/interior")))))
One video, one analysis, one set of blocks PER FACE. They are separate blocks
and not one block with a subject axis, because a face can be re-measured on its
own — a knob dragged on the second person re-reads the second person's
landmarks — and because that is what makes each one's content address name
exactly the bytes it covers."
[analysis by-subject]
(into {} (map (fn [[subject inputs] ] [subject (pack analysis subject inputs)]))
by-subject))
(defn unpack
"The three block-detail responses -> inputs for measurement and freeze."
[^js responses [width height]]
(let [by-role (into {}
(map (fn [^js response]
(let [descriptor (js->clj (js/JSON.parse (.-descriptor response))
:keywordize-keys true)]
[(:role descriptor)
{:layout (:layout descriptor)
:data (wire/typed (get-in descriptor [:layout :type])
(.-data response))}]))
(array-seq responses)))
_ (when-not (= (set roles) (set (keys by-role)))
(defn- flatten-blocks
"`{subject {role block}}` -> one flat seq, in a stable order."
[by-subject wanted]
(for [[_ blocks] (sort-by (comp str key) by-subject)
role wanted
:when (contains? blocks role)]
(get blocks role)))
(defn block-keys
"The keys of every subject's source blocks, in the order they upload."
[by-subject]
(mapv :key (flatten-blocks by-subject roles)))
(defn wire-blocks [by-subject]
(into-array
(map (fn [{:keys [key descriptor data]}]
#js {:key key :descriptor descriptor :data (wire/base64 data)})
(flatten-blocks by-subject roles))))
(defn upload-blocks [by-subject]
(into-array
(map (fn [{:keys [key descriptor data]}]
#js {:key key :descriptor descriptor :data data})
(flatten-blocks by-subject (conj roles "source/interior")))))
(defn- unpack-one
"One subject's three blocks -> its inputs for measurement and freeze."
[by-role [width height]]
(let [_ (when-not (= (set roles) (set (keys by-role)))
(throw (ex-info "analysis is missing source blocks" {:roles (keys by-role)})))
{:keys [layout data]} (get by-role "source/dense")
frames (:frames layout)
@ -206,3 +229,29 @@
{:dense dense :detected detected :crops crops :dimensions [width height]
:missing (count (remove true? detected))
:first-real (first (keep-indexed (fn [i present?] (when present? i)) detected))})))
(defn unpack
"Every subject's block-detail responses -> `{:subjects {id inputs}}`.
WHICH SUBJECT A BLOCK BELONGS TO IS READ OFF ITS DESCRIPTOR, not off the order
the server returned it in. The descriptor is the thing the key is a hash of, so
it is the only statement about a block that cannot have drifted from its bytes."
[^js responses dimensions]
(let [parsed (map (fn [^js response]
(let [descriptor (js->clj (js/JSON.parse (.-descriptor response))
:keywordize-keys true)]
{:subject (keyword (first (:features descriptor)))
:role (:role descriptor)
:block {:layout (:layout descriptor)
:data (wire/typed (get-in descriptor [:layout :type])
(.-data response))}}))
(array-seq responses))]
(when (some (comp nil? :subject) parsed)
(throw (ex-info "a source block does not say which subject it came from"
{:roles (mapv :role parsed)})))
{:subjects
(into {}
(map (fn [[subject entries]]
[subject (unpack-one (into {} (map (juxt :role :block)) entries)
dimensions)]))
(group-by :subject parsed))}))

25
frontend/src/arthur/flow/take.cljs
View file

@ -1,6 +1,7 @@
(ns arthur.flow.take
"The shared landmark-to-channel path for synthetic and detected takes."
(:require [arthur.flow.address :as address]
[arthur.flow.detect :as detect]
[arthur.flow.condition :as condition]
[arthur.flow.condition.brows :as condition-brows]
[arthur.flow.condition.eyes :as condition-eyes]
@ -27,7 +28,7 @@
:aspect aspect
;; How the detector was run, not just which one it was. See
;; `flow/address/analysis-descriptor`.
:mode "video"}))))
:mode "video" :tracking detect/settings}))))
(defn measure
"Condition the anchor before measuring rings through it."
@ -98,15 +99,28 @@
aperture)})
(throw (ex-info "unknown measured feature type" {:area area}))))))
(defn subjects
"Per-subject source landmarks -> per-subject conditioned measurements.
Every tracked face in one shot goes through the same measurement path and is
conditioned on its OWN anchor fit, because the fit maps a face onto its own
mean pose: sharing one would register the second face against the first one's
head. What they share is the video, and that is already expressed by them
sharing the analysis record."
[params by-subject]
(into {} (map (fn [[id inputs]] [id (measure params inputs)])) by-subject))
(defn build
[params inputs]
(freeze/clip params (measure params inputs)))
"`by-subject` is subject id -> that face's source tracks, one entry per
tracked face."
[params by-subject]
(freeze/clip params (subjects params by-subject)))
(defn footage
"A real manifest and its detected landmarks through the same measurement and
freeze path as the synthetic take. The source cadence stays in :fps; picture
sampling is a root time map applied only after this artifact exists."
[manifest {:keys [dense detected dimensions interior presence detector]}]
[manifest {:keys [dimensions detector subjects]}]
(let [[w h] dimensions
params (merge knobs
{:name (or (:source manifest) "footage")
@ -120,5 +134,4 @@
;; these landmarks is the failure this prevents.
:analysis (analysis-for (assoc manifest :width w :height h)
detector)})]
(build params {:dense dense :detected detected :interior interior
:presence presence})))
(build params subjects)))

35
frontend/src/arthur/ui/shell.cljs
View file

@ -24,29 +24,12 @@
(defonce ^:private selected-owner (r/atom nil))
(defonce ^:private drafts (r/atom {}))
(defn- instance-label
"What to call a symbol instance in the UI.
Its `:name`, because a placement's id is a UUID and a uuid is not something to
show anyone or to sort by. The id is the fallback only so that a node with no
name still has a row rather than a blank one."
[clip id]
(or (get-in clip [:timelines :main :nodes id :name]) (str id)))
(defn- setting-owners [clip]
(let [instances (->> (get-in clip [:timelines :main :nodes])
(filter (comp #{:symbol} :kind val))
;; By LABEL, not by id: ordering by a uuid is ordering at
;; random, and this list is read top to bottom by a person.
(sort-by (fn [[id _]] [(instance-label clip id) (str id)]))
(mapv key))
instances (if (seq instances) instances [nil])]
(vec (for [instance instances
[scope ids] [[:subject (:subjects clip)]
[:feature (:features clip)]
[:group (:groups clip)]]
id (sort-by str (keys ids))]
[scope id instance]))))
(vec (for [[scope objects] [[:subject (:subjects clip)]
[:feature (:features clip)]
[:group (:groups clip)]]
id (sort-by str (keys objects))]
[scope id])))
(defn- slider-bounds [knob {:keys [type default min even?] upper :max}]
{:min (or min (if (= knob :blob-grow) -3 0))
@ -61,7 +44,7 @@
busy? (:busy? @(rf/subscribe [::sub/project]))
report @(rf/subscribe [::project/regeneration])
owners (setting-owners clip)
[scope id instance :as owner] (if (some #{(deref selected-owner)} owners)
[scope id :as owner] (if (some #{(deref selected-owner)} owners)
@selected-owner (first owners))
area (case scope
:subject :subject
@ -84,12 +67,8 @@
(if (seq owners)
(doall (for [[i [kind object-id]] (map-indexed vector owners)]
^{:key i} [:option {:value i}
(str (when-let [instance (nth (nth owners i) 2)]
(str (instance-label clip instance) " / "))
(name kind) " · " (name object-id))]))
(str (name kind) " · " (subs (str object-id) 1))]))
[:option {:value 0} "no tracked objects"])] ]
(when instance
[:div.shared-note "shared symbol · changes all seven stage instances"])
(when area
[:div.control-list
(doall