Add scoped feature regeneration and retained pixel measurements
This commit is contained in:
Olive Vaughn
parent
39ee37db02
commit
3058b9a5f2
15 changed files with 889 additions and 81 deletions
|
|
@ -45,8 +45,12 @@
|
|||
:channels (cond-> {[:audio :gain] gain}
|
||||
pan (assoc [:audio :pan] pan))}])
|
||||
audio))]
|
||||
(-> source
|
||||
(assoc :name name :width width :height height
|
||||
:timelines {:main {:id :main :frames frames :nodes nodes}
|
||||
symbol (assoc original :id symbol)})
|
||||
(dissoc :subjects :features :groups))))
|
||||
(cond-> (assoc source :name name :width width :height height
|
||||
:timelines {:main {:id :main :frames frames :nodes nodes}
|
||||
symbol (assoc original :id symbol)})
|
||||
;; These are the same tracked features in the shared symbol. Every
|
||||
;; placement reads them; tuning one changes all instances.
|
||||
(seq (:features source))
|
||||
(assoc :features (into {} (map (fn [[id f]]
|
||||
[id (assoc f :timeline symbol)]))
|
||||
(:features source))))))
|
||||
|
|
|
|||
|
|
@ -73,7 +73,10 @@
|
|||
:when (not (params/valid-settings? (:area f) (or (:params f) {})))]
|
||||
(str "feature " (pr-str id) " has invalid settings for " (pr-str (:area f))))
|
||||
(for [[id f] features node-id (:nodes f)
|
||||
:when (not (contains? nodes node-id))]
|
||||
:let [owned-nodes (if-let [timeline (:timeline f)]
|
||||
(get-in clip [:timelines timeline :nodes])
|
||||
nodes)]
|
||||
:when (not (contains? owned-nodes node-id))]
|
||||
(str "feature " (pr-str id) " refers to missing node " (pr-str node-id)))
|
||||
(for [[id n] (frequencies node-owners) :when (> n 1)]
|
||||
(str "node " (pr-str id) " belongs to more than one feature"))
|
||||
|
|
|
|||
|
|
@ -108,7 +108,8 @@
|
|||
:frames (clip/frames built)
|
||||
:display-fps (:fps built)
|
||||
:clip built :store (:store frozen)
|
||||
:source-blocks source-blocks :source-inputs source-inputs
|
||||
:source-blocks source-blocks
|
||||
:source-inputs (assoc source-inputs :presence (:presence manifest))
|
||||
;; No cache-buster. The audio is a blob named by the hash of its own
|
||||
;; bytes, so re-extracting gives it a different URL rather than
|
||||
;; overwriting this one — which is what the `?v=` here used to work
|
||||
|
|
@ -124,24 +125,40 @@
|
|||
(when (pos? first-real)
|
||||
(str " (first found on " (inc first-real) ")"))))))))
|
||||
|
||||
(defn- cached-source! [manifest detector]
|
||||
(let [key (:id (take/analysis-for manifest detector))]
|
||||
(-> (http/GET (str "/api/analyses/" key))
|
||||
(.then (fn [^js analysis]
|
||||
(let [keys (array-seq (.-source_blocks analysis))]
|
||||
(when (= (count keys) (count source/roles))
|
||||
(-> (js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %)) keys)))
|
||||
(.then (fn [blocks]
|
||||
(source/unpack blocks
|
||||
[(:width manifest) (:height manifest)]))))))))
|
||||
(defn- with-default-interior! [analysis track]
|
||||
(let [frames (count (:crops track))
|
||||
key (source/interior-key analysis take/knobs frames)]
|
||||
(-> (http/GET (str "/api/blocks/" key))
|
||||
(.then (fn [block]
|
||||
(assoc track :interior (source/unpack-interior block take/knobs frames)
|
||||
:interior-key key)))
|
||||
(.catch (fn [error]
|
||||
(if (= 404 (:status (ex-data error))) nil (throw error)))))))
|
||||
(if (= 404 (:status (ex-data error))) track (throw error)))))))
|
||||
|
||||
(defn saved-source!
|
||||
"Restore retained source tracks by analysis id, for regeneration after open."
|
||||
[key dimensions]
|
||||
(-> (http/GET (str "/api/analyses/" key))
|
||||
(.then (fn [^js analysis]
|
||||
(let [keys (array-seq (.-source_blocks analysis))]
|
||||
(when (= (count keys) (count source/roles))
|
||||
(-> (js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %)) keys)))
|
||||
(.then (fn [blocks] (source/unpack blocks dimensions)))
|
||||
(.then (fn [track] (with-default-interior! key track))))))))
|
||||
(.catch (fn [error]
|
||||
(if (= 404 (:status (ex-data error))) nil (throw error))))))
|
||||
|
||||
(defn- cached-source! [manifest detector]
|
||||
(saved-source! (:id (take/analysis-for manifest detector))
|
||||
[(:width manifest) (:height manifest)]))
|
||||
|
||||
(defn- measure-cached! [track]
|
||||
;; Cached source blocks hold crop pixels, but no interior measurements. Spread
|
||||
;; their measurement over event-loop turns just like the fresh decode path.
|
||||
(let [crops (:crops track)
|
||||
;; Older analyses may lack the optional interior block. Spread their one-time
|
||||
;; measurement over event-loop turns just like the fresh decode path.
|
||||
(if (:interior track)
|
||||
(js/Promise.resolve track)
|
||||
(let [crops (:crops track)
|
||||
total (count crops)
|
||||
interior (atom [])]
|
||||
(js/Promise.
|
||||
|
|
@ -156,7 +173,7 @@
|
|||
(str "measuring " (inc i) "/" total)]))
|
||||
(js/setTimeout #(step (inc i)) 0)
|
||||
(catch :default error (reject error)))))]
|
||||
(step 0))))))
|
||||
(step 0)))))))
|
||||
|
||||
(rf/reg-fx
|
||||
::begin!
|
||||
|
|
|
|||
|
|
@ -24,13 +24,19 @@
|
|||
(:require [arthur.domain.clip :as clip]
|
||||
[arthur.audio.mix :as mix]
|
||||
[arthur.demo.stage :as stage]
|
||||
[arthur.domain.feature :as feature]
|
||||
[arthur.domain.project :as project]
|
||||
[arthur.domain.wire :as wire]
|
||||
[arthur.events.footage :as footage]
|
||||
[arthur.events.playback :as pb]
|
||||
[arthur.footage.store :as store]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.ingest :as ingest]
|
||||
[arthur.flow.regenerate :as regenerate]
|
||||
[arthur.flow.source :as source]
|
||||
[arthur.flow.take :as take]
|
||||
[arthur.fx.http :as http]
|
||||
[arthur.synth :as synth]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
(defn- analysis-payload [analysis]
|
||||
|
|
@ -130,7 +136,8 @@
|
|||
(http/PUT
|
||||
(str "/api/analyses/" (:id analysis))
|
||||
#js {:source_blocks
|
||||
(into-array (map :key (vals source-blocks)))})))))))
|
||||
(into-array (map #(get-in source-blocks [% :key])
|
||||
source/roles))})))))))
|
||||
(.then (fn [_] (upload-missing! doc)))
|
||||
(.then (fn [uploaded]
|
||||
(-> (http/PUT (str "/api/projects/" pid)
|
||||
|
|
@ -200,6 +207,137 @@
|
|||
(js/console.error error)
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))]))))))
|
||||
|
||||
(defonce ^:private retained-source (atom nil))
|
||||
(defonce ^:private retained-interior (atom nil))
|
||||
|
||||
(defn- measure-crops! [inputs settings]
|
||||
;; Existing analyses without an interior block pay this once. One crop per
|
||||
;; event-loop turn keeps playback responsive during that backfill.
|
||||
(if (or (:interior inputs) (not (:crops inputs)))
|
||||
(js/Promise.resolve inputs)
|
||||
(js/Promise.
|
||||
(fn [resolve reject]
|
||||
(let [crops (:crops inputs)
|
||||
measured (atom [])]
|
||||
(letfn [(step [i]
|
||||
(if (= i (count crops))
|
||||
(resolve (assoc inputs :interior @measured))
|
||||
(try
|
||||
(swap! measured conj (source/measure-crop settings (nth crops i)))
|
||||
(js/setTimeout #(step (inc i)) 0)
|
||||
(catch :default error (reject error)))))]
|
||||
(step 0)))))))
|
||||
|
||||
(defn- retained-interior! [analysis settings inputs]
|
||||
(let [frames (count (:crops inputs))
|
||||
block-key (source/interior-key analysis settings frames)]
|
||||
(-> (http/GET (str "/api/blocks/" block-key))
|
||||
(.then (fn [block]
|
||||
(assoc inputs
|
||||
:interior (source/unpack-interior block settings frames)
|
||||
:interior-key block-key)))
|
||||
(.catch (fn [error]
|
||||
(if (= 404 (:status (ex-data error)))
|
||||
(-> (measure-crops! (dissoc inputs :interior) settings)
|
||||
(.then (fn [measured]
|
||||
(let [{:keys [key descriptor data]}
|
||||
(source/interior-block analysis settings
|
||||
(:interior measured))]
|
||||
(-> (upload-missing!
|
||||
#js {:blocks #js [#js {:key key
|
||||
:descriptor descriptor
|
||||
:data data}]})
|
||||
(.then (fn [_]
|
||||
(assoc measured :interior-key block-key))))))))
|
||||
(throw error)))))))
|
||||
|
||||
(defn- source-for! [entry]
|
||||
(if-let [inputs (:source-inputs entry)]
|
||||
(js/Promise.resolve inputs)
|
||||
(let [analysis (get-in entry [:clip :analysis])]
|
||||
(if (= (:id analysis) (:id @retained-source))
|
||||
(:promise @retained-source)
|
||||
(let [promise
|
||||
(if (= "synth" (:detector analysis))
|
||||
(js/Promise.resolve
|
||||
{:dense (synth/synth-dense (:frames analysis) {:seed (:seed analysis)})})
|
||||
(-> (ingest/manifest! (:footage-id entry))
|
||||
(.then (fn [manifest]
|
||||
(-> (footage/saved-source! (:id analysis)
|
||||
[(:width manifest)
|
||||
(:height manifest)])
|
||||
(.then (fn [inputs]
|
||||
(when-not inputs
|
||||
(throw (ex-info "saved analysis has no source blocks" {})))
|
||||
(assoc inputs :presence (:presence manifest)))))))))]
|
||||
(do
|
||||
(reset! retained-source {:id (:id analysis) :promise promise})
|
||||
promise))))))
|
||||
|
||||
(defn- inputs-for-edit! [entry edit inputs]
|
||||
(let [{clip :changed fids :features} (regenerate/plan (:clip entry) edit)
|
||||
teeth (first (filter #(= :teeth (get-in clip [:features % :area])) fids))]
|
||||
(if (and teeth (:crops inputs))
|
||||
(let [settings (merge take/knobs (feature/effective-params clip teeth))
|
||||
analysis (get-in clip [:analysis :id])
|
||||
frames (count (:crops inputs))
|
||||
key (source/interior-key analysis settings frames)]
|
||||
(if (and (:interior inputs)
|
||||
(or (= (:interior-key inputs) key)
|
||||
(and (nil? (:interior-key inputs))
|
||||
(= key (source/interior-key analysis take/knobs frames)))))
|
||||
(js/Promise.resolve inputs)
|
||||
(if (= key (:key @retained-interior))
|
||||
(:promise @retained-interior)
|
||||
(let [promise (retained-interior! analysis settings inputs)]
|
||||
(reset! retained-interior {:key key :promise promise})
|
||||
promise))))
|
||||
(js/Promise.resolve inputs))))
|
||||
|
||||
(rf/reg-fx
|
||||
::preview-settings!
|
||||
(fn [{:keys [id entry edit request]}]
|
||||
(-> (source-for! entry)
|
||||
(.then (fn [inputs] (inputs-for-edit! entry edit inputs)))
|
||||
(.then (fn [inputs]
|
||||
(regenerate/change (assoc entry :source-inputs inputs) edit)))
|
||||
(.then (fn [changed]
|
||||
(rf/dispatch [::settings-previewed id request changed])))
|
||||
(.catch (fn [error]
|
||||
(js/console.error error)
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))]))))))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::preview-settings
|
||||
(fn [{:keys [db]} [_ edit]]
|
||||
(let [id (:clip/current db)
|
||||
entry (store/entry id)]
|
||||
(if (or (get-in db [:project :busy?]) (nil? (:analysis (:clip entry))))
|
||||
{}
|
||||
(let [plan (regenerate/plan (:clip entry) edit)
|
||||
report (select-keys plan [:features :roles])
|
||||
request (inc (or (:preview-request db) 0))]
|
||||
(js/console.info "arthur regeneration" (clj->js (assoc report :edit edit)))
|
||||
{:db (-> db
|
||||
(assoc :preview-request request)
|
||||
(update :project merge {:status "previewing…"})
|
||||
(assoc :regeneration (assoc report :edit edit)))
|
||||
::preview-settings! {:id id :entry entry :edit edit
|
||||
:request request}})))))
|
||||
|
||||
(rf/reg-sub ::regeneration (fn [db _] (:regeneration db)))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::settings-previewed
|
||||
(fn [{:keys [db]} [_ previous request entry]]
|
||||
(if (and (= previous (:clip/current db))
|
||||
(= request (:preview-request db)))
|
||||
(let [id (store/install! entry "edited")]
|
||||
{:db (-> db
|
||||
(assoc :clip/current id)
|
||||
(update :project merge {:status "preview · unsaved"}))})
|
||||
{})))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; events
|
||||
|
||||
|
|
|
|||
|
|
@ -148,6 +148,8 @@
|
|||
"source/dense" []
|
||||
"source/detected" []
|
||||
"source/crops" []
|
||||
"source/interior" [:blob-grow :cavity-erode :min-area :teeth-verts
|
||||
:tongue-reject :top-bias]
|
||||
"head-pos" [:anchor-avg]
|
||||
"head-rot" [:anchor-avg]
|
||||
"head-scale" [:anchor-avg]
|
||||
|
|
|
|||
|
|
@ -425,6 +425,34 @@
|
|||
[:xform :pos] (ch/framed [(- (/ w 2) (:x c))
|
||||
(- (* 0.25 h) (:y c))])})))
|
||||
|
||||
(defn- mouth-part [absent? obs
|
||||
{:keys [analysis verts anchor-avg contour-avg aperture-cut] :as params}
|
||||
{:keys [outer inner] :as inputs}]
|
||||
(let [rings (block {:role "geom" :analysis (:id analysis) :params params
|
||||
:tracks ["outer" "inner"]}
|
||||
{:type "int16" :scale geom-scale
|
||||
:features [:mouth :mouth] :absent? absent?}
|
||||
obs [(rings->flat outer verts) (rings->flat inner verts)])
|
||||
prov (fn [by]
|
||||
{:by by :analysis (:id analysis)
|
||||
:params {:anchor-avg anchor-avg :contour-avg contour-avg
|
||||
:verts verts}})]
|
||||
{:nodes {:mouth
|
||||
{:id :mouth :name "mouth" :kind :poly :parent :head :z "a1"
|
||||
:channels {[:geom :pts] (dense rings 0 (prov :roto/lips-outer))
|
||||
[:style :color] (ch/framed :skin-dark)}}
|
||||
:mouth-in
|
||||
{:id :mouth-in :name "mouth interior" :kind :poly
|
||||
:parent :mouth :z "a2"
|
||||
:channels {[:geom :pts] (dense rings 1 (prov :roto/lips-inner))
|
||||
[:style :color] (ch/framed :mouth-dark)
|
||||
[:vis] (visibility params inputs
|
||||
{:by :roto/mouth-aperture
|
||||
:analysis (:id analysis)
|
||||
:params {:anchor-avg anchor-avg
|
||||
:aperture-cut aperture-cut}})}}}
|
||||
:store (stored rings)}))
|
||||
|
||||
(defn- feature-parts
|
||||
"Freeze eyes and brows into their own dense blocks and scene nodes. This owns
|
||||
only representation: the landmark correspondence, blink and pose choices have
|
||||
|
|
@ -445,20 +473,23 @@
|
|||
;; feature each one follows, in the same order. The two vectors are read
|
||||
;; together on purpose: this is the mapping `pack` cannot check for itself,
|
||||
;; and `each-dense-track-follows-its-own-features-presence` is what pins it.
|
||||
eye-block (named "eyes"
|
||||
eye-block (when eyes (named "eyes"
|
||||
["lash-r" "lid-r" "lash-l" "lid-l"]
|
||||
[:eye-r :eye-r :eye-l :eye-l]
|
||||
"int16"
|
||||
(mapv #(rings->flat % eye-verts)
|
||||
[(:lash-r eyes) (:lid-r eyes)
|
||||
(:lash-l eyes) (:lid-l eyes)]))
|
||||
iris-block (named "iris-pos" ["iris-r" "iris-l"] [:eye-r :eye-l] "float32"
|
||||
[(:iris-r eyes) (:iris-l eyes)])
|
||||
brow-block (named "brows" ["ring-r" "ring-l"] [:brow-r :brow-l] "int16"
|
||||
(:lash-l eyes) (:lid-l eyes)])))
|
||||
iris-block (when eyes
|
||||
(named "iris-pos" ["iris-r" "iris-l"] [:eye-r :eye-l] "float32"
|
||||
[(:iris-r eyes) (:iris-l eyes)]))
|
||||
brow-block (when brows
|
||||
(named "brows" ["ring-r" "ring-l"] [:brow-r :brow-l] "int16"
|
||||
(mapv #(rings->flat % brow-verts)
|
||||
[(:ring-r brows) (:ring-l brows)]))
|
||||
brow-pos-block (named "brow-pos" ["pos-r" "pos-l"] [:brow-r :brow-l] "float32"
|
||||
[(:pos-r brows) (:pos-l brows)])
|
||||
[(:ring-r brows) (:ring-l brows)])))
|
||||
brow-pos-block (when brows
|
||||
(named "brow-pos" ["pos-r" "pos-l"] [:brow-r :brow-l] "float32"
|
||||
[(:pos-r brows) (:pos-l brows)]))
|
||||
eye-node (fn [id z track]
|
||||
{:id id :name (clojure.core/name id) :kind :poly :parent :head :z z
|
||||
:channels {[:geom :pts] (dense eye-block track
|
||||
|
|
@ -477,12 +508,18 @@
|
|||
:stencil parent
|
||||
:channels {[:xform :pos] (dense iris-block track
|
||||
(provenance :roto/gaze nil))
|
||||
[:geom :radius] (ch/framed radius)
|
||||
[:geom :radius] (assoc (ch/framed radius)
|
||||
:generated
|
||||
(provenance :roto/iris-size
|
||||
{:iris-size (:iris-size params)}))
|
||||
[:style :color] (ch/framed :iris)}})
|
||||
pupil-node (fn [id parent]
|
||||
{:id id :name (clojure.core/name id) :kind :rect :parent parent :z "a1"
|
||||
:stencil parent
|
||||
:channels {[:geom :size] (ch/framed (:pupil-size eyes))
|
||||
:channels {[:geom :size] (assoc (ch/framed (:pupil-size eyes))
|
||||
:generated
|
||||
(provenance :roto/pupil-size
|
||||
{:pupil-size (:pupil-size params)}))
|
||||
[:style :color] (ch/framed :pupil)}})
|
||||
brow-node (fn [id z track]
|
||||
{:id id :name (clojure.core/name id) :kind :poly :parent :head :z z
|
||||
|
|
@ -491,17 +528,21 @@
|
|||
[:xform :pos] (dense brow-pos-block track
|
||||
(provenance :roto/brow-raise nil))
|
||||
[:style :color] (ch/framed :brow)}})]
|
||||
{:nodes {:eye-r (eye-node :eye-r "a2" 0)
|
||||
{:nodes (merge
|
||||
(when eyes
|
||||
{:eye-r (eye-node :eye-r "a2" 0)
|
||||
:eye-r-in (inner-node :eye-r-in :eye-r "a1" 1 (:shut-r eyes))
|
||||
:iris-r (iris-node :iris-r :eye-r-in 0 (:radius-r eyes))
|
||||
:pupil-r (pupil-node :pupil-r :iris-r)
|
||||
:eye-l (eye-node :eye-l "a3" 2)
|
||||
:eye-l-in (inner-node :eye-l-in :eye-l "a1" 3 (:shut-l eyes))
|
||||
:iris-l (iris-node :iris-l :eye-l-in 1 (:radius-l eyes))
|
||||
:pupil-l (pupil-node :pupil-l :iris-l)
|
||||
:brow-r (brow-node :brow-r "a4" 0)
|
||||
:brow-l (brow-node :brow-l "a5" 1)}
|
||||
:store (stored eye-block iris-block brow-block brow-pos-block)}))
|
||||
:pupil-l (pupil-node :pupil-l :iris-l)})
|
||||
(when brows
|
||||
{:brow-r (brow-node :brow-r "a4" 0)
|
||||
:brow-l (brow-node :brow-l "a5" 1)}))
|
||||
:store (apply stored (remove nil?
|
||||
[eye-block iris-block brow-block brow-pos-block]))}))
|
||||
|
||||
(defn- interior-part
|
||||
"Freeze the pixel-derived radial contour under the mouth cavity. Missing
|
||||
|
|
@ -536,6 +577,44 @@
|
|||
[:vis] (keyed-visibility shown generated)}}}
|
||||
:store (stored blk)}))
|
||||
|
||||
(defn part
|
||||
"A feature type's nodes and addressed blocks, with no clip scaffold."
|
||||
[area params {:keys [detected presence] :as measured}]
|
||||
(let [absent? (when (or detected presence)
|
||||
(fn [id f]
|
||||
(or (and detected (not (nth detected f true)))
|
||||
(and (contains? presence id)
|
||||
(not (nth (get presence id) f))))))
|
||||
obs {:detected detected :presence presence}]
|
||||
(case area
|
||||
:mouth (mouth-part absent? obs params measured)
|
||||
:eye (feature-parts absent? obs params (select-keys measured [:eyes]))
|
||||
:brow (feature-parts absent? obs params (select-keys measured [:brows]))
|
||||
:teeth (interior-part params absent? obs (:teeth measured))
|
||||
(throw (ex-info "unknown frozen feature type" {:area area})))))
|
||||
|
||||
(defn head-part
|
||||
"Freeze only the measured head transform from an already conditioned anchor."
|
||||
[{:keys [analysis anchor-avg] :as params} {:keys [transforms detected presence]}]
|
||||
(let [absent? (when (or detected presence)
|
||||
(fn [_ f] (and detected (not (nth detected f true)))))
|
||||
inv (mapv invert transforms)
|
||||
xf (fn [role f]
|
||||
(block {:role role :analysis (:id analysis) :params params
|
||||
:tracks [role]}
|
||||
{:type "float32" :features [nil] :absent? absent?}
|
||||
{:detected detected}
|
||||
[(mapv f inv)]))
|
||||
pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)]))
|
||||
rot (xf "head-rot" (fn [t] [(:theta t)]))
|
||||
scale (xf "head-scale" (fn [t] [(:s t) (:s t)]))
|
||||
prov {:by :anchor/similarity :analysis (:id analysis)
|
||||
:params {:anchor-avg anchor-avg}}]
|
||||
{:measured {[:xform :pos] (dense pos 0 prov)
|
||||
[:xform :rot] (dense rot 0 prov)
|
||||
[:xform :scale] (dense scale 0 prov)}
|
||||
:store (stored pos rot scale)}))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the clip
|
||||
|
||||
|
|
@ -624,12 +703,7 @@
|
|||
prov (fn [by extra]
|
||||
{:by by :analysis (:id analysis)
|
||||
:params (merge {:anchor-avg anchor-avg} extra)})
|
||||
rings (block {:role "geom" :analysis (:id analysis) :params params
|
||||
:tracks ["outer" "inner"]}
|
||||
{:type "int16" :scale geom-scale
|
||||
:features [:mouth :mouth] :absent? absent?}
|
||||
obs
|
||||
[(rings->flat outer verts) (rings->flat inner verts)])
|
||||
mouth (mouth-part absent? obs params inputs)
|
||||
;; The anchor, inverted and split into its three components. Three blocks
|
||||
;; and not one: they are three channels, they have three strides, and a
|
||||
;; single block would need a per-component offset table to say so.
|
||||
|
|
@ -653,7 +727,6 @@
|
|||
;; `anchor avg` and on its own cut and NOT on `contour avg`, because
|
||||
;; measure reports the inner ring's own height and nothing smooths it.
|
||||
anchor-prov (prov :anchor/similarity nil)
|
||||
roto (fn [by] (prov by {:verts verts :contour-avg contour-avg}))
|
||||
features (when (and eyes brows) (feature-parts absent? obs params inputs))
|
||||
interior (when teeth (interior-part params absent? obs teeth))
|
||||
built {:name name
|
||||
|
|
@ -710,31 +783,13 @@
|
|||
{:id :head :name "head" :kind :group :parent :face :z "a1"
|
||||
:measured {[:xform :pos] (dense pos 0 anchor-prov)
|
||||
[:xform :rot] (dense rot 0 anchor-prov)
|
||||
[:xform :scale] (dense scale 0 anchor-prov)}}
|
||||
|
||||
;; The outer lip ring is the dark band OUTSIDE the interior, and
|
||||
;; that three-layer structure — dark ring, pale interior, teeth
|
||||
;; — is what makes a flat shape read as an opening rather than
|
||||
;; as a blob. So it keeps every frame and is never hidden.
|
||||
:mouth
|
||||
{:id :mouth :name "mouth" :kind :poly :parent :head :z "a1"
|
||||
:channels {[:geom :pts] (dense rings 0 (roto :roto/lips-outer))
|
||||
[:style :color] (ch/framed :skin-dark)}}
|
||||
|
||||
:mouth-in
|
||||
{:id :mouth-in :name "mouth interior" :kind :poly
|
||||
:parent :mouth :z "a2"
|
||||
:channels {[:geom :pts] (dense rings 1 (roto :roto/lips-inner))
|
||||
[:style :color] (ch/framed :mouth-dark)
|
||||
[:vis] (visibility params inputs
|
||||
(prov :roto/mouth-aperture
|
||||
{:aperture-cut aperture-cut}))}}}
|
||||
(:nodes features) (:nodes interior))}}}
|
||||
[:xform :scale] (dense scale 0 anchor-prov)}}}
|
||||
(:nodes mouth) (:nodes features) (:nodes interior))}}}
|
||||
;; Tier 2, behind a handle, and now behind a content address: every key is
|
||||
;; a sha256 over the analysis, the settings and the absence data that
|
||||
;; produced the bytes under it. Nothing above this line changed when they
|
||||
;; stopped being "take/geom", which is the point of a handle.
|
||||
store (merge (stored rings pos rot scale)
|
||||
store (merge (stored pos rot scale) (:store mouth)
|
||||
(:store features) (:store interior))]
|
||||
(doseq [id (keys presence)]
|
||||
(when-not (contains? (:features built) id)
|
||||
|
|
|
|||
125
frontend/src/arthur/flow/regenerate.cljs
Normal file
125
frontend/src/arthur/flow/regenerate.cljs
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
(ns arthur.flow.regenerate
|
||||
"Recompute a changed feature from retained source tracks, then replace only
|
||||
channels owned by that feature. Upload remains project/save's ordinary job."
|
||||
(:require [arthur.domain.clip :as clip]
|
||||
[arthur.domain.feature :as feature]
|
||||
[arthur.domain.params :as params]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.freeze :as freeze]
|
||||
[arthur.flow.take :as take]))
|
||||
|
||||
(defn- settings [clip fid]
|
||||
(let [f (get-in clip [:features fid])
|
||||
mouth (first (for [[id peer] (:features clip)
|
||||
:when (and (= :mouth (:area peer))
|
||||
(= (:subject f) (:subject peer)))] id))]
|
||||
(cond-> (feature/effective-params clip fid)
|
||||
(and (= :teeth (:area f)) mouth)
|
||||
(assoc :aperture-cut (:aperture-cut (feature/effective-params clip mouth))))))
|
||||
|
||||
(defn- replace-feature [entry fragment fid]
|
||||
(let [paths (for [id (get-in entry [:clip :features fid :nodes])
|
||||
[prop channel] (get-in fragment [:nodes id :channels])
|
||||
:when (:generated channel)]
|
||||
[id prop channel])]
|
||||
(-> (reduce (fn [entry [id prop channel]]
|
||||
(let [at [:clip :timelines clip/root-id :nodes id :channels prop]
|
||||
old (get-in entry at)]
|
||||
(assoc-in entry at
|
||||
(cond-> channel
|
||||
(contains? old :over) (assoc :over (:over old))))))
|
||||
entry paths)
|
||||
(update :store merge (:store fragment)))))
|
||||
|
||||
(defn plan
|
||||
"The changed document and feature IDs an edit dirties. Also reports tier-2
|
||||
block roles from the existing address table for the debug UI."
|
||||
[clip {:keys [scope id knob value]}]
|
||||
(let [area (get-in params/definitions [knob :area])
|
||||
collection (case scope
|
||||
:subject :subjects :feature :features :group :groups
|
||||
(throw (ex-info "unknown setting scope" {:scope scope})))
|
||||
owner (get-in clip [collection id])]
|
||||
(when-not (and owner (params/valid-value? knob value)
|
||||
(case scope
|
||||
:subject (= area :subject)
|
||||
:feature (= area (:area owner))
|
||||
:group (and (= area :eye) (= :eye-pair (:kind owner)))))
|
||||
(throw (ex-info "invalid scoped setting" {:scope scope :id id
|
||||
:knob knob :value value})))
|
||||
(let [changed (assoc-in clip [collection id :params knob] value)
|
||||
subject (if (= scope :subject) id (:subject owner))
|
||||
members (case scope
|
||||
:subject (for [[fid f] (:features clip)
|
||||
:when (= subject (:subject f))] fid)
|
||||
:feature [id]
|
||||
:group (:members owner))
|
||||
members (if (= knob :contour-avg)
|
||||
(remove #(= :teeth (get-in clip [:features % :area])) members)
|
||||
members)
|
||||
members (if (= knob :aperture-cut)
|
||||
(concat members (for [[fid f] (:features clip)
|
||||
:when (and (= subject (:subject f))
|
||||
(= :teeth (:area f)))] fid))
|
||||
members)]
|
||||
{:changed changed
|
||||
:features (vec (filter #(not= (settings clip %) (settings changed %))
|
||||
(distinct members)))
|
||||
:roles (address/invalidates knob)})))
|
||||
|
||||
(defn- change-take
|
||||
"One scoped static edit. `source-inputs` contains dense landmarks and, when
|
||||
teeth are dirty, retained pixel measurements. No IO or app-db here."
|
||||
[{:keys [clip source-inputs] :as entry} edit]
|
||||
(let [{changed :changed fids :features} (plan clip edit)]
|
||||
(when-not (:dense source-inputs)
|
||||
(throw (ex-info "regeneration needs retained source landmarks" {})))
|
||||
(if (empty? fids)
|
||||
(assoc entry :clip changed)
|
||||
(let [base-params (merge take/knobs
|
||||
{:fps (:fps clip)
|
||||
:aspect (get-in clip [:analysis :aspect])
|
||||
:analysis (:analysis clip)})
|
||||
base (delay (take/anchor-base
|
||||
(merge base-params (settings changed (first fids)))
|
||||
source-inputs))
|
||||
entry (reduce
|
||||
(fn [entry fid]
|
||||
(let [part-area (get-in changed [:features fid :area])
|
||||
p (merge base-params (settings changed fid))
|
||||
_ (when (and (= part-area :teeth)
|
||||
(nil? (:interior source-inputs)))
|
||||
(throw (ex-info "teeth regeneration needs retained pixel measurements"
|
||||
{:feature fid})))
|
||||
measured (take/measure-part part-area p source-inputs @base)]
|
||||
(replace-feature entry (freeze/part part-area p measured) fid)))
|
||||
(assoc entry :clip changed) fids)]
|
||||
(if (= :anchor-avg (:knob edit))
|
||||
(let [p (merge base-params
|
||||
(settings changed (first fids)))
|
||||
baked (freeze/head-part p @base)
|
||||
old-head (get-in clip [:timelines :main :nodes :head])
|
||||
measured (:measured baked)]
|
||||
(cond-> (-> entry
|
||||
(assoc-in [:clip :timelines :main :nodes :head :measured] measured)
|
||||
(update :store merge (:store baked)))
|
||||
(= (:channels old-head) (:measured old-head))
|
||||
(assoc-in [:clip :timelines :main :nodes :head :channels] measured)))
|
||||
entry)))))
|
||||
|
||||
(defn change
|
||||
"A take edits its root timeline. A stage edits the shared symbol timeline;
|
||||
every instance then reads the same new channels on its next paint."
|
||||
[{:keys [clip] :as entry} edit]
|
||||
(if-let [symbol (some :timeline (vals (:features clip)))]
|
||||
(let [original (:timelines clip)
|
||||
take (assoc clip :timelines
|
||||
{:main (assoc (get original symbol) :id :main)})
|
||||
changed (change-take (assoc entry :clip take) edit)]
|
||||
(assoc changed :clip
|
||||
(-> (:clip changed)
|
||||
(assoc :timelines
|
||||
(assoc original symbol
|
||||
(assoc (get-in changed [:clip :timelines :main])
|
||||
:id symbol))))))
|
||||
(change-take entry edit)))
|
||||
|
|
@ -1,14 +1,76 @@
|
|||
(ns arthur.flow.source
|
||||
"The pixel-dependent result of analyzing footage, stored as three ordinary
|
||||
content-addressed blocks. Everything after this boundary can run without PNGs
|
||||
or MediaPipe. The mouth crops hold raw pixels and an un-eroded lip ring."
|
||||
"Retained analysis: three raw source blocks and an optional addressed interior
|
||||
measurement block. Everything after this boundary can run without PNGs or
|
||||
MediaPipe. Mouth crops retain pixels for settings that change the measurement."
|
||||
(:require [arthur.domain.landmarks :as lm]
|
||||
[arthur.domain.params :as params]
|
||||
[arthur.domain.wire :as wire]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.measure.interior :as interior]))
|
||||
|
||||
(def roles ["source/dense" "source/detected" "source/crops"])
|
||||
|
||||
(defn- interior-address [analysis settings frames]
|
||||
(let [verts (:teeth-verts settings)
|
||||
stride (+ 2 (* 2 verts))]
|
||||
(address/block {:role "source/interior" :analysis analysis
|
||||
:params settings
|
||||
:tracks ["contrast" "area" "contour"]
|
||||
:features [] :observation nil
|
||||
:layout {:type "float64" :frames frames :verts verts
|
||||
:stride stride}})))
|
||||
|
||||
(defn interior-key [analysis settings frames]
|
||||
(:key (interior-address analysis settings frames)))
|
||||
|
||||
(defn interior-block
|
||||
"Retain pixel measurements before any head-local transform or smoothing."
|
||||
[analysis settings measures]
|
||||
(let [frames (count measures)
|
||||
{:keys [key descriptor]}
|
||||
(interior-address analysis settings frames)
|
||||
verts (:teeth-verts settings)
|
||||
stride (+ 2 (* 2 verts))
|
||||
data (js/Float64Array. (* frames stride))]
|
||||
(doseq [f (range frames)]
|
||||
(let [{:keys [contrast area contour]} (nth measures f)
|
||||
at (* f stride)]
|
||||
(aset data at (or contrast 0))
|
||||
(aset data (inc at) (or area 0))
|
||||
(if contour
|
||||
(do
|
||||
(when-not (= verts (count contour))
|
||||
(throw (ex-info "interior contour vertex count disagrees with settings"
|
||||
{:frame f :expected verts :actual (count contour)})))
|
||||
(doseq [i (range verts)]
|
||||
(aset data (+ at 2 (* 2 i)) (:x (nth contour i)))
|
||||
(aset data (+ at 3 (* 2 i)) (:y (nth contour i)))))
|
||||
(aset data (+ at 2) js/NaN))))
|
||||
{:role "source/interior" :key key :descriptor descriptor :data data}))
|
||||
|
||||
(defn unpack-interior [^js response settings frames]
|
||||
(let [descriptor (js->clj (js/JSON.parse (.-descriptor response))
|
||||
:keywordize-keys true)
|
||||
layout (:layout descriptor)
|
||||
data (wire/typed (:type layout) (.-data response))
|
||||
verts (:teeth-verts settings)
|
||||
stride (+ 2 (* 2 verts))]
|
||||
(when-not (and (= "source/interior" (:role descriptor))
|
||||
(= frames (:frames layout)) (= verts (:verts layout))
|
||||
(= stride (:stride layout))
|
||||
(= (* frames stride) (.-length data)))
|
||||
(throw (ex-info "saved interior block has the wrong layout"
|
||||
{:layout layout :frames frames :verts verts})))
|
||||
(mapv (fn [f]
|
||||
(let [at (* f stride)]
|
||||
{:contrast (aget data at) :area (aget data (inc at))
|
||||
:contour (when-not (js/Number.isNaN (aget data (+ at 2)))
|
||||
(mapv (fn [i]
|
||||
{:x (aget data (+ at 2 (* 2 i)))
|
||||
:y (aget data (+ at 3 (* 2 i)))})
|
||||
(range verts)))}))
|
||||
(range frames))))
|
||||
|
||||
(defn measure-crop
|
||||
"One mouth crop's interior, or the empty measurement for a frame with no face.
|
||||
|
||||
|
|
@ -21,9 +83,6 @@
|
|||
(interior/measure params (:box crop) #js {:data (:data crop)})
|
||||
{:contour nil :contrast 0 :area 0 :debug nil}))
|
||||
|
||||
(defn measure-crops [params crops]
|
||||
(mapv #(measure-crop params %) crops))
|
||||
|
||||
(defn- named [role analysis tracks layout data]
|
||||
(merge (address/block {:role role :analysis analysis :params {}
|
||||
:tracks tracks :features [] :observation nil
|
||||
|
|
@ -31,12 +90,13 @@
|
|||
{:role role :data data}))
|
||||
|
||||
(defn pack
|
||||
"Dense landmarks, detection mask and raw RGBA mouth crops -> source blocks."
|
||||
[analysis {:keys [dense detected crops]}]
|
||||
"Dense landmarks, mask, crops and optional pixel measurements -> source blocks."
|
||||
[analysis {:keys [dense detected crops interior]}]
|
||||
(let [frames (count dense)
|
||||
points (count (first dense))]
|
||||
(when-not (and (pos? frames) (pos? points)
|
||||
(= frames (count detected) (count crops))
|
||||
(or (nil? interior) (= frames (count interior)))
|
||||
(every? #(= points (count %)) dense))
|
||||
(throw (ex-info "source tracks must have the same frame and landmark counts"
|
||||
{:frames frames :points points})))
|
||||
|
|
@ -59,14 +119,16 @@
|
|||
(aset landmarks (+ base 2) z)))
|
||||
(when-let [crop (nth crops f)]
|
||||
(.set pixels (:data crop) (nth offsets f))))
|
||||
{"source/dense" (named "source/dense" analysis ["landmarks"]
|
||||
(cond-> {"source/dense" (named "source/dense" analysis ["landmarks"]
|
||||
{:type "float64" :frames frames :points points :stride (* 3 points)}
|
||||
landmarks)
|
||||
"source/detected" (named "source/detected" analysis ["detected"]
|
||||
{:type "uint8" :frames frames :stride 1} mask)
|
||||
"source/crops" (named "source/crops" analysis ["rgba"]
|
||||
{:type "uint8" :frames frames :boxes boxes :offsets offsets}
|
||||
pixels)})))
|
||||
pixels)}
|
||||
interior (assoc "source/interior"
|
||||
(interior-block analysis params/defaults interior))))))
|
||||
|
||||
(defn wire-blocks [blocks]
|
||||
(into-array
|
||||
|
|
@ -80,7 +142,7 @@
|
|||
(map (fn [role]
|
||||
(let [{:keys [key descriptor data]} (get blocks role)]
|
||||
#js {:key key :descriptor descriptor :data data}))
|
||||
roles)))
|
||||
(filter #(contains? blocks %) (conj roles "source/interior")))))
|
||||
|
||||
(defn unpack
|
||||
"The three block-detail responses -> inputs for measurement and freeze."
|
||||
|
|
|
|||
|
|
@ -60,6 +60,44 @@
|
|||
:detected detected
|
||||
:presence presence)))
|
||||
|
||||
(defn anchor-base
|
||||
"Shared upstream measurement for a feature regeneration."
|
||||
[{:keys [aspect] :as params} {:keys [dense detected presence]}]
|
||||
(let [fitted (anchor/fit {:aspect aspect} {:dense dense})]
|
||||
(assoc (condition/anchor params fitted)
|
||||
:detected detected :presence presence)))
|
||||
|
||||
(defn measure-part
|
||||
"Recompute one feature type through the shared conditioned anchor. A brow
|
||||
reads the eye corners, and teeth read mouth aperture; these are input edges,
|
||||
not requests to freeze those other features."
|
||||
[area {:keys [aspect] :as params}
|
||||
{:keys [dense detected interior presence]} {:keys [transforms] :as base}]
|
||||
(let [mouth (fn []
|
||||
(mouth/measure {:aspect aspect}
|
||||
{:dense dense :transforms transforms}))
|
||||
eyes (fn []
|
||||
(eyes/measure params {:dense dense :transforms transforms
|
||||
:detected detected :presence presence}))]
|
||||
(merge (select-keys base [:detected :presence])
|
||||
(case area
|
||||
:mouth (let [m (mouth)]
|
||||
{:outer (condition/contours params (:outer m))
|
||||
:inner (condition/contours params (:inner m))
|
||||
:aperture (:aperture m)})
|
||||
:eye {:eyes (condition-eyes/apply-defaults params (eyes))}
|
||||
:brow {:brows (condition-brows/apply-defaults
|
||||
params
|
||||
(brows/measure params {:dense dense :transforms transforms})
|
||||
(eyes))}
|
||||
:teeth (let [aperture (:aperture (mouth))]
|
||||
{:teeth (condition-interior/apply-defaults
|
||||
params
|
||||
(mapv interior/head-local interior transforms
|
||||
(repeat aspect))
|
||||
aperture)})
|
||||
(throw (ex-info "unknown measured feature type" {:area area}))))))
|
||||
|
||||
(defn build
|
||||
[params inputs]
|
||||
(freeze/clip params (measure params inputs)))
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@
|
|||
([entry kind]
|
||||
(let [id (keyword kind (str (swap! serial inc)))]
|
||||
(when-let [old (:audio @loaded)]
|
||||
(when (.startsWith old "blob:") (js/URL.revokeObjectURL old)))
|
||||
(when (and (not= old (:audio entry)) (.startsWith old "blob:"))
|
||||
(js/URL.revokeObjectURL old)))
|
||||
(reset! loaded (assoc entry :id id))
|
||||
id)))
|
||||
|
||||
|
|
|
|||
|
|
@ -7,15 +7,103 @@
|
|||
why scrubbing at speed does not re-render the page."
|
||||
(:require [arthur.clock :as clock]
|
||||
[arthur.db :as db]
|
||||
[arthur.domain.feature :as feature]
|
||||
[arthur.domain.params :as params]
|
||||
[arthur.events.footage :as footage]
|
||||
[arthur.events.playback :as pb]
|
||||
[arthur.events.project :as project]
|
||||
[arthur.subs.playback :as sub]
|
||||
[arthur.subs.render :as render]
|
||||
[arthur.ui.player :as player]
|
||||
[re-frame.core :as rf]))
|
||||
[re-frame.core :as rf]
|
||||
[reagent.core :as r]))
|
||||
|
||||
(def ^:private zoom 2)
|
||||
(defonce ^:private selected-owner (r/atom nil))
|
||||
(defonce ^:private drafts (r/atom {}))
|
||||
|
||||
(defn- setting-owners [clip]
|
||||
(let [instances (for [[id node] (get-in clip [:timelines :main :nodes])
|
||||
:when (= :symbol (:kind node))] id)
|
||||
instances (if (seq instances) (sort-by str instances) [nil])]
|
||||
(vec (for [instance instances
|
||||
[scope ids] [[:subject (:subjects clip)]
|
||||
[:feature (:features clip)]
|
||||
[:group (:groups clip)]]
|
||||
id (sort-by str (keys ids))]
|
||||
[scope id instance]))))
|
||||
|
||||
(defn- slider-bounds [knob {:keys [type default min even?] upper :max}]
|
||||
{:min (or min (if (= knob :blob-grow) -3 0))
|
||||
:max (or upper (get {:verts 20 :eye-verts 16 :brow-verts 10
|
||||
:teeth-verts 20 :blob-grow 3 :top-bias 2
|
||||
:gaze-gain 4} knob)
|
||||
(max 1 (* 2 default)))
|
||||
:step (cond even? 2 (= type :integer) 1 :else 0.01)})
|
||||
|
||||
(defn- controls []
|
||||
(let [clip @(rf/subscribe [::render/clip])
|
||||
busy? (:busy? @(rf/subscribe [::sub/project]))
|
||||
report @(rf/subscribe [::project/regeneration])
|
||||
owners (setting-owners clip)
|
||||
[scope id instance :as owner] (if (some #{(deref selected-owner)} owners)
|
||||
@selected-owner (first owners))
|
||||
area (case scope
|
||||
:subject :subject
|
||||
:feature (get-in clip [:features id :area])
|
||||
:group :eye
|
||||
nil)
|
||||
values (case scope
|
||||
:subject (merge (params/for-area :subject)
|
||||
(get-in clip [:subjects id :params]))
|
||||
:feature (when id (feature/effective-params clip id))
|
||||
:group (merge (params/for-area :eye)
|
||||
(get-in clip [:groups id :params]))
|
||||
nil)]
|
||||
[:section.controls
|
||||
[:label "active object "
|
||||
[:select {:value (or (first (keep-indexed
|
||||
(fn [i option] (when (= option owner) i)) owners)) 0)
|
||||
:on-change #(reset! selected-owner
|
||||
(nth owners (js/parseInt (.. % -target -value) 10)))}
|
||||
(if (seq owners)
|
||||
(doall (for [[i [kind object-id]] (map-indexed vector owners)]
|
||||
^{:key i} [:option {:value i}
|
||||
(str (when-let [instance (nth (nth owners i) 2)]
|
||||
(str (name instance) " / "))
|
||||
(name kind) " · " (name object-id))]))
|
||||
[:option {:value 0} "no tracked objects"])] ]
|
||||
(when instance
|
||||
[:div.shared-note "shared symbol · changes all seven stage instances"])
|
||||
(when area
|
||||
[:div.control-list
|
||||
(doall
|
||||
(for [[knob spec] (sort-by (comp str key) params/definitions)
|
||||
:when (= area (:area spec))]
|
||||
(let [draft-key [scope id knob]
|
||||
value (get @drafts draft-key (get values knob))
|
||||
{:keys [min max step]} (slider-bounds knob spec)]
|
||||
^{:key (str draft-key)}
|
||||
[:label.control-row
|
||||
[:span (name knob)]
|
||||
[:input {:type "range" :min min :max max :step step :value value
|
||||
:disabled busy?
|
||||
:on-change (fn [event]
|
||||
(let [s (.. event -target -value)
|
||||
v (if (= :integer (:type spec))
|
||||
(js/parseInt s 10)
|
||||
(js/parseFloat s))]
|
||||
(swap! drafts assoc draft-key v)
|
||||
(rf/dispatch [::project/preview-settings
|
||||
{:scope scope :id id :knob knob
|
||||
:value v}]))) }]
|
||||
[:output (str value)]])))])
|
||||
(when report
|
||||
[:pre.regeneration-debug
|
||||
(str "dirty features: " (pr-str (:features report)) "\n"
|
||||
"new block roles: " (if (seq (:roles report))
|
||||
(pr-str (sort (:roles report))) "none (tier 1 only)")
|
||||
"\n" (get-in @(rf/subscribe [::sub/project]) [:status]))])]))
|
||||
|
||||
(defn- audio []
|
||||
(let [src @(rf/subscribe [::sub/audio])]
|
||||
|
|
@ -153,6 +241,7 @@
|
|||
[stage]
|
||||
[audio]
|
||||
[transport]
|
||||
[controls]
|
||||
[:p.note
|
||||
"Upload a video, choose its footage, then load frames. Save the project to "
|
||||
"share the analyzed take without detecting frames again."]])
|
||||
|
|
|
|||
101
frontend/test/arthur/flow/regenerate_test.cljs
Normal file
101
frontend/test/arthur/flow/regenerate_test.cljs
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
(ns arthur.flow.regenerate-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[arthur.domain.project :as project]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.regenerate :as regenerate]
|
||||
[arthur.flow.take :as take]
|
||||
[arthur.synth :as synth]))
|
||||
|
||||
(def ^:private frames 40)
|
||||
(def ^:private inputs (delay {:dense (synth/synth-dense frames {:seed 9})}))
|
||||
(def ^:private initial
|
||||
(delay
|
||||
(let [p (merge take/knobs
|
||||
{:name "regen" :fps 30 :aspect 1 :stage [320 200]
|
||||
:expose 1 :head :as-filmed
|
||||
:analysis (address/analysis
|
||||
{:detector "synth" :version "mulberry32"
|
||||
:seed 9 :frames frames :fps 30 :aspect 1})})]
|
||||
(assoc (take/build p @inputs) :source-inputs @inputs))))
|
||||
|
||||
(defn- full-at [change]
|
||||
(take/build
|
||||
(merge take/knobs
|
||||
{:name "regen" :fps 30 :aspect 1 :stage [320 200]
|
||||
:expose 1 :head :as-filmed
|
||||
:analysis (get-in @initial [:clip :analysis])}
|
||||
change)
|
||||
@inputs))
|
||||
|
||||
(defn- channel [entry node path]
|
||||
(get-in entry [:clip :timelines :main :nodes node :channels path]))
|
||||
|
||||
(deftest eye-rebuild-is-confined-to-the-edited-feature
|
||||
(let [before @initial
|
||||
after (regenerate/change before
|
||||
{:scope :feature :id :eye-r :knob :gaze-gain :value 2})]
|
||||
(is (not= (channel before :iris-r [:xform :pos])
|
||||
(channel after :iris-r [:xform :pos])))
|
||||
(is (= (channel before :iris-l [:xform :pos])
|
||||
(channel after :iris-l [:xform :pos])))
|
||||
(is (= (channel before :mouth [:geom :pts])
|
||||
(channel after :mouth [:geom :pts])))
|
||||
(is (= (get-in before [:clip :timelines :main :nodes :face])
|
||||
(get-in after [:clip :timelines :main :nodes :face])))
|
||||
(is (= (channel (full-at {:gaze-gain 2}) :iris-r [:xform :pos])
|
||||
(channel after :iris-r [:xform :pos])))))
|
||||
|
||||
(deftest framed-eye-size-needs-no-new-dense-block
|
||||
(let [before @initial
|
||||
after (regenerate/change before
|
||||
{:scope :feature :id :eye-r :knob :iris-size :value 0.6})]
|
||||
(is (not= (channel before :iris-r [:geom :radius])
|
||||
(channel after :iris-r [:geom :radius])))
|
||||
(is (= (channel before :iris-l [:geom :radius])
|
||||
(channel after :iris-l [:geom :radius])))
|
||||
(is (= (channel (full-at {:iris-size 0.6}) :iris-r [:geom :radius])
|
||||
(channel after :iris-r [:geom :radius])))
|
||||
(is (= (set (keys (:store before))) (set (keys (:store after)))))))
|
||||
|
||||
(deftest mouth-edit-leaves-the-head-and-eye-alone
|
||||
(let [before @initial
|
||||
after (regenerate/change before
|
||||
{:scope :feature :id :mouth :knob :verts :value 10})]
|
||||
(is (not= (channel before :mouth [:geom :pts])
|
||||
(channel after :mouth [:geom :pts])))
|
||||
(is (= (get-in before [:clip :timelines :main :nodes :head])
|
||||
(get-in after [:clip :timelines :main :nodes :head])))
|
||||
(is (= (channel before :eye-r [:geom :pts])
|
||||
(channel after :eye-r [:geom :pts])))
|
||||
(is (= (channel (full-at {:verts 10}) :mouth [:geom :pts])
|
||||
(channel after :mouth [:geom :pts])))))
|
||||
|
||||
(deftest subject-edit-recomputes-head-and-keeps-authored-placement
|
||||
(let [before @initial
|
||||
after (regenerate/change before
|
||||
{:scope :subject :id :face-1 :knob :anchor-avg :value 4})
|
||||
full (full-at {:anchor-avg 4})]
|
||||
(is (not= (get-in before [:clip :timelines :main :nodes :head :measured])
|
||||
(get-in after [:clip :timelines :main :nodes :head :measured])))
|
||||
(is (= (get-in full [:clip :timelines :main :nodes :head :measured])
|
||||
(get-in after [:clip :timelines :main :nodes :head :measured])))
|
||||
(is (= (get-in before [:clip :timelines :main :nodes :face])
|
||||
(get-in after [:clip :timelines :main :nodes :face])))))
|
||||
|
||||
(deftest contour-edit-does-not-rebuild-head-or-teeth
|
||||
(let [before @initial
|
||||
after (regenerate/change before
|
||||
{:scope :subject :id :face-1 :knob :contour-avg :value 3})]
|
||||
(is (= (get-in before [:clip :timelines :main :nodes :head])
|
||||
(get-in after [:clip :timelines :main :nodes :head])))
|
||||
(is (= (channel before :teeth [:geom :pts])
|
||||
(channel after :teeth [:geom :pts])))))
|
||||
|
||||
(deftest edited-take-still-round-trips-through-normal-save
|
||||
(let [after (regenerate/change @initial
|
||||
{:scope :group :id :eyes-1 :knob :iris-size :value 0.6})
|
||||
wire (project/save "regen" after)
|
||||
loaded (project/load "regen" (js/JSON.parse (js/JSON.stringify wire)))]
|
||||
(is (= (:clip after) (:clip loaded)))
|
||||
(is (= (channel after :iris-r [:geom :radius])
|
||||
(channel loaded :iris-r [:geom :radius])))))
|
||||
|
|
@ -1,7 +1,25 @@
|
|||
(ns arthur.flow.source-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[arthur.domain.params :as params]
|
||||
[arthur.domain.wire :as wire]
|
||||
[arthur.flow.source :as source]))
|
||||
|
||||
(deftest pixel-measurements-have-their-own-addressed-block
|
||||
(let [settings params/defaults
|
||||
ring (mapv (fn [i] {:x (/ i 10) :y (/ (+ i 1) 20)})
|
||||
(range (:teeth-verts settings)))
|
||||
measures [{:contour ring :contrast 0.7 :area 42}
|
||||
{:contour nil :contrast 0 :area 0}]
|
||||
block (source/interior-block "sha256:analysis" settings measures)
|
||||
response #js {:descriptor (:descriptor block)
|
||||
:data (wire/base64 (:data block))}]
|
||||
(is (= (:key block)
|
||||
(source/interior-key "sha256:analysis" settings 2)))
|
||||
(is (= measures (source/unpack-interior response settings 2)))
|
||||
(is (not= (:key block)
|
||||
(source/interior-key "sha256:analysis"
|
||||
(assoc settings :cavity-erode 0.2) 2)))))
|
||||
|
||||
(deftest source-blocks-round-trip-without-source-images
|
||||
(let [face (vec (repeat 478 {:x 0.25 :y 0.5 :z -0.125}))
|
||||
pixels (js/Uint8ClampedArray. #js [12 24 36 255])
|
||||
|
|
@ -19,3 +37,12 @@
|
|||
(is (nil? (second (:crops out))))
|
||||
(is (= (mapv :key (vals blocks))
|
||||
(mapv :key (vals (source/pack "sha256:analysis" out)))))))
|
||||
|
||||
(deftest fresh-source-includes-its-pixel-measurement-block
|
||||
(let [face (vec (repeat 478 {:x 0.25 :y 0.5 :z -0.125}))
|
||||
input {:dense [face] :detected [false] :crops [nil]
|
||||
:interior [{:contour nil :contrast 0 :area 0}]}
|
||||
blocks (source/pack "sha256:analysis" input)]
|
||||
(is (contains? blocks "source/interior"))
|
||||
(is (= 4 (.-length (source/upload-blocks blocks))))
|
||||
(is (= 3 (count source/roles)))))
|
||||
|
|
|
|||
|
|
@ -233,7 +233,8 @@ async function main() {
|
|||
if (probe && probe.drawn > 0) break;
|
||||
await sleep(100);
|
||||
}
|
||||
if (!probe) throw new Error('no canvas.stage on the page — is `shadow-cljs watch app` running?');
|
||||
if (!probe) throw new Error('no canvas.stage on the page — ' +
|
||||
(page.logs.slice(0, 3).join(' | ') || 'is `shadow-cljs watch app` running?'));
|
||||
|
||||
console.log(`\ncanvas ${probe.w}x${probe.h}, clip ${JSON.stringify(probe.selectedClip)}`);
|
||||
|
||||
|
|
@ -393,6 +394,138 @@ async function main() {
|
|||
check(!back[0].selectedClip.includes('take'), 'the picture is the reopened document',
|
||||
JSON.stringify(back[0].selectedClip));
|
||||
|
||||
// The local 8625 study is optional in a fresh test database. When present,
|
||||
// tune its shared symbol while its stage is playing.
|
||||
const stageSource = await page.eval(`fetch('/api/projects/4379f900-bdd2-409b-acf6-32081f8ce01f')
|
||||
.then(r => r.ok)`);
|
||||
if (stageSource) {
|
||||
check(await page.eval(CLICK('stage 8625')), 'the 8625 stage loads');
|
||||
const stageLoaded = await statusMatching(/loaded 8625 stage study/, 160);
|
||||
check(stageLoaded !== null, 'the 8625 stage is ready', stageLoaded ?? (await page.eval(STATUS)));
|
||||
const eyeSelected = await page.eval(`(() => {
|
||||
const select = document.querySelector('.controls select');
|
||||
const option = [...select.options]
|
||||
.find(o => o.textContent.trim() === 'left / feature · eye-r');
|
||||
if (!option) return false;
|
||||
select.value = option.value;
|
||||
select.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
})()`);
|
||||
check(eyeSelected, 'a stage instance exposes its tracked right eye');
|
||||
await sleep(100);
|
||||
const irisSlider = await page.eval(`(() => {
|
||||
const row = [...document.querySelectorAll('.control-row')]
|
||||
.find(row => row.querySelector('span')?.textContent === 'iris-size');
|
||||
if (!row) return null;
|
||||
const r = row.querySelector('input').getBoundingClientRect();
|
||||
return { x: r.x + r.width * 0.9, y: r.y + r.height / 2 };
|
||||
})()`);
|
||||
check(irisSlider !== null, 'the stage eye has an iris-size slider');
|
||||
if (irisSlider) {
|
||||
check(await page.eval(CLICK('play')), 'the stage starts playing');
|
||||
const startFrame = await page.eval(`Number(document.querySelector('.readout span').textContent.match(/\\d+/)[0])`);
|
||||
await page.send('Input.dispatchMouseEvent', {
|
||||
type: 'mousePressed', x: irisSlider.x, y: irisSlider.y, button: 'left', clickCount: 1,
|
||||
});
|
||||
await page.send('Input.dispatchMouseEvent', {
|
||||
type: 'mouseReleased', x: irisSlider.x, y: irisSlider.y, button: 'left', clickCount: 1,
|
||||
});
|
||||
const preview = await statusMatching(/preview · unsaved/, 160);
|
||||
const debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`);
|
||||
check(preview !== null, 'the slider updates the stage preview', preview ?? debug);
|
||||
check(debug.includes(':eye-r') && debug.includes('tier 1 only'),
|
||||
'the panel reports the affected feature and tier', debug);
|
||||
const endFrame = await page.eval(`Number(document.querySelector('.readout span').textContent.match(/\\d+/)[0])`);
|
||||
check(endFrame > startFrame, 'playback continues during tuning', `${startFrame} -> ${endFrame}`);
|
||||
await page.eval(CLICK('pause'));
|
||||
}
|
||||
const subjectSelected = await page.eval(`(() => {
|
||||
const select = document.querySelector('.controls select');
|
||||
const option = [...select.options]
|
||||
.find(o => o.textContent.includes('subject · face-1'));
|
||||
if (!option) return false;
|
||||
select.value = option.value;
|
||||
select.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
})()`);
|
||||
check(subjectSelected, 'the stage exposes its tracked subject');
|
||||
if (subjectSelected) {
|
||||
await sleep(100);
|
||||
const anchorSlider = await page.eval(`(() => {
|
||||
const row = [...document.querySelectorAll('.control-row')]
|
||||
.find(row => row.querySelector('span')?.textContent === 'anchor-avg');
|
||||
if (!row) return null;
|
||||
const r = row.querySelector('input').getBoundingClientRect();
|
||||
return { x: r.x + r.width * 0.9, y: r.y + r.height / 2 };
|
||||
})()`);
|
||||
check(anchorSlider !== null, 'the subject has an anchor slider');
|
||||
if (anchorSlider) {
|
||||
const anchorStart = Date.now();
|
||||
await page.send('Input.dispatchMouseEvent', {
|
||||
type: 'mousePressed', x: anchorSlider.x, y: anchorSlider.y,
|
||||
button: 'left', clickCount: 1,
|
||||
});
|
||||
await page.send('Input.dispatchMouseEvent', {
|
||||
type: 'mouseReleased', x: anchorSlider.x, y: anchorSlider.y,
|
||||
button: 'left', clickCount: 1,
|
||||
});
|
||||
let debug = '';
|
||||
for (let i = 0; i < 160; i++) {
|
||||
debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`);
|
||||
if (debug.includes('head-pos')) break;
|
||||
await sleep(250);
|
||||
}
|
||||
check(debug.includes('head-pos') && debug.includes(':teeth'),
|
||||
'anchor invalidates the head and teeth', debug);
|
||||
const preview = await statusMatching(/preview · unsaved/, 160);
|
||||
check(preview !== null, 'the anchor edit finishes previewing',
|
||||
preview ? `${Date.now() - anchorStart}ms` : (await page.eval(STATUS)));
|
||||
}
|
||||
}
|
||||
const teethSelected = await page.eval(`(() => {
|
||||
const select = document.querySelector('.controls select');
|
||||
const option = [...select.options]
|
||||
.find(o => o.textContent.trim() === 'left / feature · teeth');
|
||||
if (!option) return false;
|
||||
select.value = option.value;
|
||||
select.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
})()`);
|
||||
check(teethSelected, 'the stage exposes its tracked teeth');
|
||||
if (teethSelected) {
|
||||
await sleep(100);
|
||||
const slider = await page.eval(`(() => {
|
||||
const row = [...document.querySelectorAll('.control-row')]
|
||||
.find(row => row.querySelector('span')?.textContent === 'cavity-erode');
|
||||
if (!row) return null;
|
||||
const r = row.querySelector('input').getBoundingClientRect();
|
||||
return { x: r.x + r.width * 0.9, y: r.y + r.height / 2 };
|
||||
})()`);
|
||||
check(slider !== null, 'the teeth have a pixel-setting slider');
|
||||
if (slider) {
|
||||
await page.send('Input.dispatchMouseEvent', {
|
||||
type: 'mousePressed', x: slider.x, y: slider.y,
|
||||
button: 'left', clickCount: 1,
|
||||
});
|
||||
await page.send('Input.dispatchMouseEvent', {
|
||||
type: 'mouseReleased', x: slider.x, y: slider.y,
|
||||
button: 'left', clickCount: 1,
|
||||
});
|
||||
let debug = '';
|
||||
for (let i = 0; i < 160; i++) {
|
||||
debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`);
|
||||
if (debug.includes('dirty features: [:teeth]')) break;
|
||||
await sleep(250);
|
||||
}
|
||||
check(debug.includes('dirty features: [:teeth]'),
|
||||
'the pixel setting invalidates only teeth', debug);
|
||||
const preview = await statusMatching(/preview · unsaved/, 160);
|
||||
check(preview !== null, 'the teeth edit finishes previewing',
|
||||
preview ?? (await page.eval(STATUS)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise the actual file input and FormData path, then wait for the
|
||||
// extraction job's footage to appear in the server list.
|
||||
const uploadDir = mkdtempSync(join(tmpdir(), 'arthur-upload-'));
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue