From 3058b9a5f292e34cee300a16079336a19168abce Mon Sep 17 00:00:00 2001 From: Olive Vaughn Date: Mon, 28 Sep 2026 16:31:27 -0400 Subject: [PATCH] Add scoped feature regeneration and retained pixel measurements --- clips/templates/clips/index.html | 13 ++ frontend/src/arthur/demo/stage.cljs | 14 +- frontend/src/arthur/domain/feature.cljs | 5 +- frontend/src/arthur/events/footage.cljs | 51 ++++--- frontend/src/arthur/events/project.cljs | 140 ++++++++++++++++- frontend/src/arthur/flow/address.cljs | 2 + frontend/src/arthur/flow/freeze.cljs | 141 ++++++++++++------ frontend/src/arthur/flow/regenerate.cljs | 125 ++++++++++++++++ frontend/src/arthur/flow/source.cljs | 84 +++++++++-- frontend/src/arthur/flow/take.cljs | 38 +++++ frontend/src/arthur/footage/store.cljs | 3 +- frontend/src/arthur/ui/shell.cljs | 91 ++++++++++- .../test/arthur/flow/regenerate_test.cljs | 101 +++++++++++++ frontend/test/arthur/flow/source_test.cljs | 27 ++++ frontend/test/browser/take.mjs | 135 ++++++++++++++++- 15 files changed, 889 insertions(+), 81 deletions(-) create mode 100644 frontend/src/arthur/flow/regenerate.cljs create mode 100644 frontend/test/arthur/flow/regenerate_test.cljs diff --git a/clips/templates/clips/index.html b/clips/templates/clips/index.html index 2a75220..6de64b3 100644 --- a/clips/templates/clips/index.html +++ b/clips/templates/clips/index.html @@ -48,6 +48,19 @@ and PUTs with ordinary CSRF protection — no endpoint in this app is exempt. color: var(--fg); background: #1c1f2b; border: 1px solid #2b3040; font: inherit; max-width: 360px; } .load-status { margin-top: 6px; font-size: 12px; opacity: .75; } + .controls { width: 640px; margin-top: 18px; padding-top: 12px; + border-top: 1px solid #2b3040; font-size: 12px; } + .controls select { margin-left: 8px; padding: 3px 5px; color: var(--fg); + background: #1c1f2b; border: 1px solid #2b3040; font: inherit; } + .shared-note { margin-top: 6px; color: #d0ba86; } + .control-list { display: grid; grid-template-columns: 1fr 1fr; gap: 6px 16px; + margin-top: 10px; } + .control-row { display: grid; grid-template-columns: 115px 1fr 42px; + align-items: center; gap: 6px; } + .control-row input { width: 100%; } + .control-row output { text-align: right; } + .regeneration-debug { padding: 8px; margin-top: 10px; background: #1c1f2b; + white-space: pre-wrap; color: #d0ba86; } .note { opacity: .35; font-size: 12px; max-width: 640px; } diff --git a/frontend/src/arthur/demo/stage.cljs b/frontend/src/arthur/demo/stage.cljs index 16066ad..58bd0a1 100644 --- a/frontend/src/arthur/demo/stage.cljs +++ b/frontend/src/arthur/demo/stage.cljs @@ -45,8 +45,12 @@ :channels (cond-> {[:audio :gain] gain} pan (assoc [:audio :pan] pan))}]) audio))] - (-> source - (assoc :name name :width width :height height - :timelines {:main {:id :main :frames frames :nodes nodes} - symbol (assoc original :id symbol)}) - (dissoc :subjects :features :groups)))) + (cond-> (assoc source :name name :width width :height height + :timelines {:main {:id :main :frames frames :nodes nodes} + symbol (assoc original :id symbol)}) + ;; These are the same tracked features in the shared symbol. Every + ;; placement reads them; tuning one changes all instances. + (seq (:features source)) + (assoc :features (into {} (map (fn [[id f]] + [id (assoc f :timeline symbol)])) + (:features source)))))) diff --git a/frontend/src/arthur/domain/feature.cljs b/frontend/src/arthur/domain/feature.cljs index b56e84f..0a4c97b 100644 --- a/frontend/src/arthur/domain/feature.cljs +++ b/frontend/src/arthur/domain/feature.cljs @@ -73,7 +73,10 @@ :when (not (params/valid-settings? (:area f) (or (:params f) {})))] (str "feature " (pr-str id) " has invalid settings for " (pr-str (:area f)))) (for [[id f] features node-id (:nodes f) - :when (not (contains? nodes node-id))] + :let [owned-nodes (if-let [timeline (:timeline f)] + (get-in clip [:timelines timeline :nodes]) + nodes)] + :when (not (contains? owned-nodes node-id))] (str "feature " (pr-str id) " refers to missing node " (pr-str node-id))) (for [[id n] (frequencies node-owners) :when (> n 1)] (str "node " (pr-str id) " belongs to more than one feature")) diff --git a/frontend/src/arthur/events/footage.cljs b/frontend/src/arthur/events/footage.cljs index 1e5d8bd..c0f95ad 100644 --- a/frontend/src/arthur/events/footage.cljs +++ b/frontend/src/arthur/events/footage.cljs @@ -108,7 +108,8 @@ :frames (clip/frames built) :display-fps (:fps built) :clip built :store (:store frozen) - :source-blocks source-blocks :source-inputs source-inputs + :source-blocks source-blocks + :source-inputs (assoc source-inputs :presence (:presence manifest)) ;; No cache-buster. The audio is a blob named by the hash of its own ;; bytes, so re-extracting gives it a different URL rather than ;; overwriting this one — which is what the `?v=` here used to work @@ -124,24 +125,40 @@ (when (pos? first-real) (str " (first found on " (inc first-real) ")")))))))) -(defn- cached-source! [manifest detector] - (let [key (:id (take/analysis-for manifest detector))] - (-> (http/GET (str "/api/analyses/" key)) - (.then (fn [^js analysis] - (let [keys (array-seq (.-source_blocks analysis))] - (when (= (count keys) (count source/roles)) - (-> (js/Promise.all - (into-array (map #(http/GET (str "/api/blocks/" %)) keys))) - (.then (fn [blocks] - (source/unpack blocks - [(:width manifest) (:height manifest)])))))))) +(defn- with-default-interior! [analysis track] + (let [frames (count (:crops track)) + key (source/interior-key analysis take/knobs frames)] + (-> (http/GET (str "/api/blocks/" key)) + (.then (fn [block] + (assoc track :interior (source/unpack-interior block take/knobs frames) + :interior-key key))) (.catch (fn [error] - (if (= 404 (:status (ex-data error))) nil (throw error))))))) + (if (= 404 (:status (ex-data error))) track (throw error))))))) + +(defn saved-source! + "Restore retained source tracks by analysis id, for regeneration after open." + [key dimensions] + (-> (http/GET (str "/api/analyses/" key)) + (.then (fn [^js analysis] + (let [keys (array-seq (.-source_blocks analysis))] + (when (= (count keys) (count source/roles)) + (-> (js/Promise.all + (into-array (map #(http/GET (str "/api/blocks/" %)) keys))) + (.then (fn [blocks] (source/unpack blocks dimensions))) + (.then (fn [track] (with-default-interior! key track)))))))) + (.catch (fn [error] + (if (= 404 (:status (ex-data error))) nil (throw error)))))) + +(defn- cached-source! [manifest detector] + (saved-source! (:id (take/analysis-for manifest detector)) + [(:width manifest) (:height manifest)])) (defn- measure-cached! [track] - ;; Cached source blocks hold crop pixels, but no interior measurements. Spread - ;; their measurement over event-loop turns just like the fresh decode path. - (let [crops (:crops track) + ;; Older analyses may lack the optional interior block. Spread their one-time + ;; measurement over event-loop turns just like the fresh decode path. + (if (:interior track) + (js/Promise.resolve track) + (let [crops (:crops track) total (count crops) interior (atom [])] (js/Promise. @@ -156,7 +173,7 @@ (str "measuring " (inc i) "/" total)])) (js/setTimeout #(step (inc i)) 0) (catch :default error (reject error)))))] - (step 0)))))) + (step 0))))))) (rf/reg-fx ::begin! diff --git a/frontend/src/arthur/events/project.cljs b/frontend/src/arthur/events/project.cljs index 351eadf..8905c99 100644 --- a/frontend/src/arthur/events/project.cljs +++ b/frontend/src/arthur/events/project.cljs @@ -24,13 +24,19 @@ (:require [arthur.domain.clip :as clip] [arthur.audio.mix :as mix] [arthur.demo.stage :as stage] + [arthur.domain.feature :as feature] [arthur.domain.project :as project] [arthur.domain.wire :as wire] + [arthur.events.footage :as footage] [arthur.events.playback :as pb] [arthur.footage.store :as store] [arthur.flow.address :as address] + [arthur.flow.ingest :as ingest] + [arthur.flow.regenerate :as regenerate] [arthur.flow.source :as source] + [arthur.flow.take :as take] [arthur.fx.http :as http] + [arthur.synth :as synth] [re-frame.core :as rf])) (defn- analysis-payload [analysis] @@ -130,7 +136,8 @@ (http/PUT (str "/api/analyses/" (:id analysis)) #js {:source_blocks - (into-array (map :key (vals source-blocks)))}))))))) + (into-array (map #(get-in source-blocks [% :key]) + source/roles))}))))))) (.then (fn [_] (upload-missing! doc))) (.then (fn [uploaded] (-> (http/PUT (str "/api/projects/" pid) @@ -200,6 +207,137 @@ (js/console.error error) (rf/dispatch [::failed (or (ex-message error) (str error))])))))) +(defonce ^:private retained-source (atom nil)) +(defonce ^:private retained-interior (atom nil)) + +(defn- measure-crops! [inputs settings] + ;; Existing analyses without an interior block pay this once. One crop per + ;; event-loop turn keeps playback responsive during that backfill. + (if (or (:interior inputs) (not (:crops inputs))) + (js/Promise.resolve inputs) + (js/Promise. + (fn [resolve reject] + (let [crops (:crops inputs) + measured (atom [])] + (letfn [(step [i] + (if (= i (count crops)) + (resolve (assoc inputs :interior @measured)) + (try + (swap! measured conj (source/measure-crop settings (nth crops i))) + (js/setTimeout #(step (inc i)) 0) + (catch :default error (reject error)))))] + (step 0))))))) + +(defn- retained-interior! [analysis settings inputs] + (let [frames (count (:crops inputs)) + block-key (source/interior-key analysis settings frames)] + (-> (http/GET (str "/api/blocks/" block-key)) + (.then (fn [block] + (assoc inputs + :interior (source/unpack-interior block settings frames) + :interior-key block-key))) + (.catch (fn [error] + (if (= 404 (:status (ex-data error))) + (-> (measure-crops! (dissoc inputs :interior) settings) + (.then (fn [measured] + (let [{:keys [key descriptor data]} + (source/interior-block analysis settings + (:interior measured))] + (-> (upload-missing! + #js {:blocks #js [#js {:key key + :descriptor descriptor + :data data}]}) + (.then (fn [_] + (assoc measured :interior-key block-key)))))))) + (throw error))))))) + +(defn- source-for! [entry] + (if-let [inputs (:source-inputs entry)] + (js/Promise.resolve inputs) + (let [analysis (get-in entry [:clip :analysis])] + (if (= (:id analysis) (:id @retained-source)) + (:promise @retained-source) + (let [promise + (if (= "synth" (:detector analysis)) + (js/Promise.resolve + {:dense (synth/synth-dense (:frames analysis) {:seed (:seed analysis)})}) + (-> (ingest/manifest! (:footage-id entry)) + (.then (fn [manifest] + (-> (footage/saved-source! (:id analysis) + [(:width manifest) + (:height manifest)]) + (.then (fn [inputs] + (when-not inputs + (throw (ex-info "saved analysis has no source blocks" {}))) + (assoc inputs :presence (:presence manifest)))))))))] + (do + (reset! retained-source {:id (:id analysis) :promise promise}) + promise)))))) + +(defn- inputs-for-edit! [entry edit inputs] + (let [{clip :changed fids :features} (regenerate/plan (:clip entry) edit) + teeth (first (filter #(= :teeth (get-in clip [:features % :area])) fids))] + (if (and teeth (:crops inputs)) + (let [settings (merge take/knobs (feature/effective-params clip teeth)) + analysis (get-in clip [:analysis :id]) + frames (count (:crops inputs)) + key (source/interior-key analysis settings frames)] + (if (and (:interior inputs) + (or (= (:interior-key inputs) key) + (and (nil? (:interior-key inputs)) + (= key (source/interior-key analysis take/knobs frames))))) + (js/Promise.resolve inputs) + (if (= key (:key @retained-interior)) + (:promise @retained-interior) + (let [promise (retained-interior! analysis settings inputs)] + (reset! retained-interior {:key key :promise promise}) + promise)))) + (js/Promise.resolve inputs)))) + +(rf/reg-fx + ::preview-settings! + (fn [{:keys [id entry edit request]}] + (-> (source-for! entry) + (.then (fn [inputs] (inputs-for-edit! entry edit inputs))) + (.then (fn [inputs] + (regenerate/change (assoc entry :source-inputs inputs) edit))) + (.then (fn [changed] + (rf/dispatch [::settings-previewed id request changed]))) + (.catch (fn [error] + (js/console.error error) + (rf/dispatch [::failed (or (ex-message error) (str error))])))))) + +(rf/reg-event-fx + ::preview-settings + (fn [{:keys [db]} [_ edit]] + (let [id (:clip/current db) + entry (store/entry id)] + (if (or (get-in db [:project :busy?]) (nil? (:analysis (:clip entry)))) + {} + (let [plan (regenerate/plan (:clip entry) edit) + report (select-keys plan [:features :roles]) + request (inc (or (:preview-request db) 0))] + (js/console.info "arthur regeneration" (clj->js (assoc report :edit edit))) + {:db (-> db + (assoc :preview-request request) + (update :project merge {:status "previewing…"}) + (assoc :regeneration (assoc report :edit edit))) + ::preview-settings! {:id id :entry entry :edit edit + :request request}}))))) + +(rf/reg-sub ::regeneration (fn [db _] (:regeneration db))) + +(rf/reg-event-fx + ::settings-previewed + (fn [{:keys [db]} [_ previous request entry]] + (if (and (= previous (:clip/current db)) + (= request (:preview-request db))) + (let [id (store/install! entry "edited")] + {:db (-> db + (assoc :clip/current id) + (update :project merge {:status "preview · unsaved"}))}) + {}))) + ;; --------------------------------------------------------------------------- ;; events diff --git a/frontend/src/arthur/flow/address.cljs b/frontend/src/arthur/flow/address.cljs index 29fe8b6..f5171d1 100644 --- a/frontend/src/arthur/flow/address.cljs +++ b/frontend/src/arthur/flow/address.cljs @@ -148,6 +148,8 @@ "source/dense" [] "source/detected" [] "source/crops" [] + "source/interior" [:blob-grow :cavity-erode :min-area :teeth-verts + :tongue-reject :top-bias] "head-pos" [:anchor-avg] "head-rot" [:anchor-avg] "head-scale" [:anchor-avg] diff --git a/frontend/src/arthur/flow/freeze.cljs b/frontend/src/arthur/flow/freeze.cljs index b35cf8d..0080f4e 100644 --- a/frontend/src/arthur/flow/freeze.cljs +++ b/frontend/src/arthur/flow/freeze.cljs @@ -425,6 +425,34 @@ [:xform :pos] (ch/framed [(- (/ w 2) (:x c)) (- (* 0.25 h) (:y c))])}))) +(defn- mouth-part [absent? obs + {:keys [analysis verts anchor-avg contour-avg aperture-cut] :as params} + {:keys [outer inner] :as inputs}] + (let [rings (block {:role "geom" :analysis (:id analysis) :params params + :tracks ["outer" "inner"]} + {:type "int16" :scale geom-scale + :features [:mouth :mouth] :absent? absent?} + obs [(rings->flat outer verts) (rings->flat inner verts)]) + prov (fn [by] + {:by by :analysis (:id analysis) + :params {:anchor-avg anchor-avg :contour-avg contour-avg + :verts verts}})] + {:nodes {:mouth + {:id :mouth :name "mouth" :kind :poly :parent :head :z "a1" + :channels {[:geom :pts] (dense rings 0 (prov :roto/lips-outer)) + [:style :color] (ch/framed :skin-dark)}} + :mouth-in + {:id :mouth-in :name "mouth interior" :kind :poly + :parent :mouth :z "a2" + :channels {[:geom :pts] (dense rings 1 (prov :roto/lips-inner)) + [:style :color] (ch/framed :mouth-dark) + [:vis] (visibility params inputs + {:by :roto/mouth-aperture + :analysis (:id analysis) + :params {:anchor-avg anchor-avg + :aperture-cut aperture-cut}})}}} + :store (stored rings)})) + (defn- feature-parts "Freeze eyes and brows into their own dense blocks and scene nodes. This owns only representation: the landmark correspondence, blink and pose choices have @@ -445,20 +473,23 @@ ;; feature each one follows, in the same order. The two vectors are read ;; together on purpose: this is the mapping `pack` cannot check for itself, ;; and `each-dense-track-follows-its-own-features-presence` is what pins it. - eye-block (named "eyes" + eye-block (when eyes (named "eyes" ["lash-r" "lid-r" "lash-l" "lid-l"] [:eye-r :eye-r :eye-l :eye-l] "int16" (mapv #(rings->flat % eye-verts) [(:lash-r eyes) (:lid-r eyes) - (:lash-l eyes) (:lid-l eyes)])) - iris-block (named "iris-pos" ["iris-r" "iris-l"] [:eye-r :eye-l] "float32" - [(:iris-r eyes) (:iris-l eyes)]) - brow-block (named "brows" ["ring-r" "ring-l"] [:brow-r :brow-l] "int16" + (:lash-l eyes) (:lid-l eyes)]))) + iris-block (when eyes + (named "iris-pos" ["iris-r" "iris-l"] [:eye-r :eye-l] "float32" + [(:iris-r eyes) (:iris-l eyes)])) + brow-block (when brows + (named "brows" ["ring-r" "ring-l"] [:brow-r :brow-l] "int16" (mapv #(rings->flat % brow-verts) - [(:ring-r brows) (:ring-l brows)])) - brow-pos-block (named "brow-pos" ["pos-r" "pos-l"] [:brow-r :brow-l] "float32" - [(:pos-r brows) (:pos-l brows)]) + [(:ring-r brows) (:ring-l brows)]))) + brow-pos-block (when brows + (named "brow-pos" ["pos-r" "pos-l"] [:brow-r :brow-l] "float32" + [(:pos-r brows) (:pos-l brows)])) eye-node (fn [id z track] {:id id :name (clojure.core/name id) :kind :poly :parent :head :z z :channels {[:geom :pts] (dense eye-block track @@ -477,12 +508,18 @@ :stencil parent :channels {[:xform :pos] (dense iris-block track (provenance :roto/gaze nil)) - [:geom :radius] (ch/framed radius) + [:geom :radius] (assoc (ch/framed radius) + :generated + (provenance :roto/iris-size + {:iris-size (:iris-size params)})) [:style :color] (ch/framed :iris)}}) pupil-node (fn [id parent] {:id id :name (clojure.core/name id) :kind :rect :parent parent :z "a1" :stencil parent - :channels {[:geom :size] (ch/framed (:pupil-size eyes)) + :channels {[:geom :size] (assoc (ch/framed (:pupil-size eyes)) + :generated + (provenance :roto/pupil-size + {:pupil-size (:pupil-size params)})) [:style :color] (ch/framed :pupil)}}) brow-node (fn [id z track] {:id id :name (clojure.core/name id) :kind :poly :parent :head :z z @@ -491,17 +528,21 @@ [:xform :pos] (dense brow-pos-block track (provenance :roto/brow-raise nil)) [:style :color] (ch/framed :brow)}})] - {:nodes {:eye-r (eye-node :eye-r "a2" 0) + {:nodes (merge + (when eyes + {:eye-r (eye-node :eye-r "a2" 0) :eye-r-in (inner-node :eye-r-in :eye-r "a1" 1 (:shut-r eyes)) :iris-r (iris-node :iris-r :eye-r-in 0 (:radius-r eyes)) :pupil-r (pupil-node :pupil-r :iris-r) :eye-l (eye-node :eye-l "a3" 2) :eye-l-in (inner-node :eye-l-in :eye-l "a1" 3 (:shut-l eyes)) :iris-l (iris-node :iris-l :eye-l-in 1 (:radius-l eyes)) - :pupil-l (pupil-node :pupil-l :iris-l) - :brow-r (brow-node :brow-r "a4" 0) - :brow-l (brow-node :brow-l "a5" 1)} - :store (stored eye-block iris-block brow-block brow-pos-block)})) + :pupil-l (pupil-node :pupil-l :iris-l)}) + (when brows + {:brow-r (brow-node :brow-r "a4" 0) + :brow-l (brow-node :brow-l "a5" 1)})) + :store (apply stored (remove nil? + [eye-block iris-block brow-block brow-pos-block]))})) (defn- interior-part "Freeze the pixel-derived radial contour under the mouth cavity. Missing @@ -536,6 +577,44 @@ [:vis] (keyed-visibility shown generated)}}} :store (stored blk)})) +(defn part + "A feature type's nodes and addressed blocks, with no clip scaffold." + [area params {:keys [detected presence] :as measured}] + (let [absent? (when (or detected presence) + (fn [id f] + (or (and detected (not (nth detected f true))) + (and (contains? presence id) + (not (nth (get presence id) f)))))) + obs {:detected detected :presence presence}] + (case area + :mouth (mouth-part absent? obs params measured) + :eye (feature-parts absent? obs params (select-keys measured [:eyes])) + :brow (feature-parts absent? obs params (select-keys measured [:brows])) + :teeth (interior-part params absent? obs (:teeth measured)) + (throw (ex-info "unknown frozen feature type" {:area area}))))) + +(defn head-part + "Freeze only the measured head transform from an already conditioned anchor." + [{:keys [analysis anchor-avg] :as params} {:keys [transforms detected presence]}] + (let [absent? (when (or detected presence) + (fn [_ f] (and detected (not (nth detected f true))))) + inv (mapv invert transforms) + xf (fn [role f] + (block {:role role :analysis (:id analysis) :params params + :tracks [role]} + {:type "float32" :features [nil] :absent? absent?} + {:detected detected} + [(mapv f inv)])) + pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)])) + rot (xf "head-rot" (fn [t] [(:theta t)])) + scale (xf "head-scale" (fn [t] [(:s t) (:s t)])) + prov {:by :anchor/similarity :analysis (:id analysis) + :params {:anchor-avg anchor-avg}}] + {:measured {[:xform :pos] (dense pos 0 prov) + [:xform :rot] (dense rot 0 prov) + [:xform :scale] (dense scale 0 prov)} + :store (stored pos rot scale)})) + ;; --------------------------------------------------------------------------- ;; the clip @@ -624,12 +703,7 @@ prov (fn [by extra] {:by by :analysis (:id analysis) :params (merge {:anchor-avg anchor-avg} extra)}) - rings (block {:role "geom" :analysis (:id analysis) :params params - :tracks ["outer" "inner"]} - {:type "int16" :scale geom-scale - :features [:mouth :mouth] :absent? absent?} - obs - [(rings->flat outer verts) (rings->flat inner verts)]) + mouth (mouth-part absent? obs params inputs) ;; The anchor, inverted and split into its three components. Three blocks ;; and not one: they are three channels, they have three strides, and a ;; single block would need a per-component offset table to say so. @@ -653,7 +727,6 @@ ;; `anchor avg` and on its own cut and NOT on `contour avg`, because ;; measure reports the inner ring's own height and nothing smooths it. anchor-prov (prov :anchor/similarity nil) - roto (fn [by] (prov by {:verts verts :contour-avg contour-avg})) features (when (and eyes brows) (feature-parts absent? obs params inputs)) interior (when teeth (interior-part params absent? obs teeth)) built {:name name @@ -710,31 +783,13 @@ {:id :head :name "head" :kind :group :parent :face :z "a1" :measured {[:xform :pos] (dense pos 0 anchor-prov) [:xform :rot] (dense rot 0 anchor-prov) - [:xform :scale] (dense scale 0 anchor-prov)}} - - ;; The outer lip ring is the dark band OUTSIDE the interior, and - ;; that three-layer structure — dark ring, pale interior, teeth - ;; — is what makes a flat shape read as an opening rather than - ;; as a blob. So it keeps every frame and is never hidden. - :mouth - {:id :mouth :name "mouth" :kind :poly :parent :head :z "a1" - :channels {[:geom :pts] (dense rings 0 (roto :roto/lips-outer)) - [:style :color] (ch/framed :skin-dark)}} - - :mouth-in - {:id :mouth-in :name "mouth interior" :kind :poly - :parent :mouth :z "a2" - :channels {[:geom :pts] (dense rings 1 (roto :roto/lips-inner)) - [:style :color] (ch/framed :mouth-dark) - [:vis] (visibility params inputs - (prov :roto/mouth-aperture - {:aperture-cut aperture-cut}))}}} - (:nodes features) (:nodes interior))}}} + [:xform :scale] (dense scale 0 anchor-prov)}}} + (:nodes mouth) (:nodes features) (:nodes interior))}}} ;; Tier 2, behind a handle, and now behind a content address: every key is ;; a sha256 over the analysis, the settings and the absence data that ;; produced the bytes under it. Nothing above this line changed when they ;; stopped being "take/geom", which is the point of a handle. - store (merge (stored rings pos rot scale) + store (merge (stored pos rot scale) (:store mouth) (:store features) (:store interior))] (doseq [id (keys presence)] (when-not (contains? (:features built) id) diff --git a/frontend/src/arthur/flow/regenerate.cljs b/frontend/src/arthur/flow/regenerate.cljs new file mode 100644 index 0000000..7a01b64 --- /dev/null +++ b/frontend/src/arthur/flow/regenerate.cljs @@ -0,0 +1,125 @@ +(ns arthur.flow.regenerate + "Recompute a changed feature from retained source tracks, then replace only + channels owned by that feature. Upload remains project/save's ordinary job." + (:require [arthur.domain.clip :as clip] + [arthur.domain.feature :as feature] + [arthur.domain.params :as params] + [arthur.flow.address :as address] + [arthur.flow.freeze :as freeze] + [arthur.flow.take :as take])) + +(defn- settings [clip fid] + (let [f (get-in clip [:features fid]) + mouth (first (for [[id peer] (:features clip) + :when (and (= :mouth (:area peer)) + (= (:subject f) (:subject peer)))] id))] + (cond-> (feature/effective-params clip fid) + (and (= :teeth (:area f)) mouth) + (assoc :aperture-cut (:aperture-cut (feature/effective-params clip mouth)))))) + +(defn- replace-feature [entry fragment fid] + (let [paths (for [id (get-in entry [:clip :features fid :nodes]) + [prop channel] (get-in fragment [:nodes id :channels]) + :when (:generated channel)] + [id prop channel])] + (-> (reduce (fn [entry [id prop channel]] + (let [at [:clip :timelines clip/root-id :nodes id :channels prop] + old (get-in entry at)] + (assoc-in entry at + (cond-> channel + (contains? old :over) (assoc :over (:over old)))))) + entry paths) + (update :store merge (:store fragment))))) + +(defn plan + "The changed document and feature IDs an edit dirties. Also reports tier-2 + block roles from the existing address table for the debug UI." + [clip {:keys [scope id knob value]}] + (let [area (get-in params/definitions [knob :area]) + collection (case scope + :subject :subjects :feature :features :group :groups + (throw (ex-info "unknown setting scope" {:scope scope}))) + owner (get-in clip [collection id])] + (when-not (and owner (params/valid-value? knob value) + (case scope + :subject (= area :subject) + :feature (= area (:area owner)) + :group (and (= area :eye) (= :eye-pair (:kind owner))))) + (throw (ex-info "invalid scoped setting" {:scope scope :id id + :knob knob :value value}))) + (let [changed (assoc-in clip [collection id :params knob] value) + subject (if (= scope :subject) id (:subject owner)) + members (case scope + :subject (for [[fid f] (:features clip) + :when (= subject (:subject f))] fid) + :feature [id] + :group (:members owner)) + members (if (= knob :contour-avg) + (remove #(= :teeth (get-in clip [:features % :area])) members) + members) + members (if (= knob :aperture-cut) + (concat members (for [[fid f] (:features clip) + :when (and (= subject (:subject f)) + (= :teeth (:area f)))] fid)) + members)] + {:changed changed + :features (vec (filter #(not= (settings clip %) (settings changed %)) + (distinct members))) + :roles (address/invalidates knob)}))) + +(defn- change-take + "One scoped static edit. `source-inputs` contains dense landmarks and, when + teeth are dirty, retained pixel measurements. No IO or app-db here." + [{:keys [clip source-inputs] :as entry} edit] + (let [{changed :changed fids :features} (plan clip edit)] + (when-not (:dense source-inputs) + (throw (ex-info "regeneration needs retained source landmarks" {}))) + (if (empty? fids) + (assoc entry :clip changed) + (let [base-params (merge take/knobs + {:fps (:fps clip) + :aspect (get-in clip [:analysis :aspect]) + :analysis (:analysis clip)}) + base (delay (take/anchor-base + (merge base-params (settings changed (first fids))) + source-inputs)) + entry (reduce + (fn [entry fid] + (let [part-area (get-in changed [:features fid :area]) + p (merge base-params (settings changed fid)) + _ (when (and (= part-area :teeth) + (nil? (:interior source-inputs))) + (throw (ex-info "teeth regeneration needs retained pixel measurements" + {:feature fid}))) + measured (take/measure-part part-area p source-inputs @base)] + (replace-feature entry (freeze/part part-area p measured) fid))) + (assoc entry :clip changed) fids)] + (if (= :anchor-avg (:knob edit)) + (let [p (merge base-params + (settings changed (first fids))) + baked (freeze/head-part p @base) + old-head (get-in clip [:timelines :main :nodes :head]) + measured (:measured baked)] + (cond-> (-> entry + (assoc-in [:clip :timelines :main :nodes :head :measured] measured) + (update :store merge (:store baked))) + (= (:channels old-head) (:measured old-head)) + (assoc-in [:clip :timelines :main :nodes :head :channels] measured))) + entry))))) + +(defn change + "A take edits its root timeline. A stage edits the shared symbol timeline; + every instance then reads the same new channels on its next paint." + [{:keys [clip] :as entry} edit] + (if-let [symbol (some :timeline (vals (:features clip)))] + (let [original (:timelines clip) + take (assoc clip :timelines + {:main (assoc (get original symbol) :id :main)}) + changed (change-take (assoc entry :clip take) edit)] + (assoc changed :clip + (-> (:clip changed) + (assoc :timelines + (assoc original symbol + (assoc (get-in changed [:clip :timelines :main]) + :id symbol)))))) + (change-take entry edit))) diff --git a/frontend/src/arthur/flow/source.cljs b/frontend/src/arthur/flow/source.cljs index c196577..47c1aaa 100644 --- a/frontend/src/arthur/flow/source.cljs +++ b/frontend/src/arthur/flow/source.cljs @@ -1,14 +1,76 @@ (ns arthur.flow.source - "The pixel-dependent result of analyzing footage, stored as three ordinary - content-addressed blocks. Everything after this boundary can run without PNGs - or MediaPipe. The mouth crops hold raw pixels and an un-eroded lip ring." + "Retained analysis: three raw source blocks and an optional addressed interior + measurement block. Everything after this boundary can run without PNGs or + MediaPipe. Mouth crops retain pixels for settings that change the measurement." (:require [arthur.domain.landmarks :as lm] + [arthur.domain.params :as params] [arthur.domain.wire :as wire] [arthur.flow.address :as address] [arthur.flow.measure.interior :as interior])) (def roles ["source/dense" "source/detected" "source/crops"]) +(defn- interior-address [analysis settings frames] + (let [verts (:teeth-verts settings) + stride (+ 2 (* 2 verts))] + (address/block {:role "source/interior" :analysis analysis + :params settings + :tracks ["contrast" "area" "contour"] + :features [] :observation nil + :layout {:type "float64" :frames frames :verts verts + :stride stride}}))) + +(defn interior-key [analysis settings frames] + (:key (interior-address analysis settings frames))) + +(defn interior-block + "Retain pixel measurements before any head-local transform or smoothing." + [analysis settings measures] + (let [frames (count measures) + {:keys [key descriptor]} + (interior-address analysis settings frames) + verts (:teeth-verts settings) + stride (+ 2 (* 2 verts)) + data (js/Float64Array. (* frames stride))] + (doseq [f (range frames)] + (let [{:keys [contrast area contour]} (nth measures f) + at (* f stride)] + (aset data at (or contrast 0)) + (aset data (inc at) (or area 0)) + (if contour + (do + (when-not (= verts (count contour)) + (throw (ex-info "interior contour vertex count disagrees with settings" + {:frame f :expected verts :actual (count contour)}))) + (doseq [i (range verts)] + (aset data (+ at 2 (* 2 i)) (:x (nth contour i))) + (aset data (+ at 3 (* 2 i)) (:y (nth contour i))))) + (aset data (+ at 2) js/NaN)))) + {:role "source/interior" :key key :descriptor descriptor :data data})) + +(defn unpack-interior [^js response settings frames] + (let [descriptor (js->clj (js/JSON.parse (.-descriptor response)) + :keywordize-keys true) + layout (:layout descriptor) + data (wire/typed (:type layout) (.-data response)) + verts (:teeth-verts settings) + stride (+ 2 (* 2 verts))] + (when-not (and (= "source/interior" (:role descriptor)) + (= frames (:frames layout)) (= verts (:verts layout)) + (= stride (:stride layout)) + (= (* frames stride) (.-length data))) + (throw (ex-info "saved interior block has the wrong layout" + {:layout layout :frames frames :verts verts}))) + (mapv (fn [f] + (let [at (* f stride)] + {:contrast (aget data at) :area (aget data (inc at)) + :contour (when-not (js/Number.isNaN (aget data (+ at 2))) + (mapv (fn [i] + {:x (aget data (+ at 2 (* 2 i))) + :y (aget data (+ at 3 (* 2 i)))}) + (range verts)))})) + (range frames)))) + (defn measure-crop "One mouth crop's interior, or the empty measurement for a frame with no face. @@ -21,9 +83,6 @@ (interior/measure params (:box crop) #js {:data (:data crop)}) {:contour nil :contrast 0 :area 0 :debug nil})) -(defn measure-crops [params crops] - (mapv #(measure-crop params %) crops)) - (defn- named [role analysis tracks layout data] (merge (address/block {:role role :analysis analysis :params {} :tracks tracks :features [] :observation nil @@ -31,12 +90,13 @@ {:role role :data data})) (defn pack - "Dense landmarks, detection mask and raw RGBA mouth crops -> source blocks." - [analysis {:keys [dense detected crops]}] + "Dense landmarks, mask, crops and optional pixel measurements -> source blocks." + [analysis {:keys [dense detected crops interior]}] (let [frames (count dense) points (count (first dense))] (when-not (and (pos? frames) (pos? points) (= frames (count detected) (count crops)) + (or (nil? interior) (= frames (count interior))) (every? #(= points (count %)) dense)) (throw (ex-info "source tracks must have the same frame and landmark counts" {:frames frames :points points}))) @@ -59,14 +119,16 @@ (aset landmarks (+ base 2) z))) (when-let [crop (nth crops f)] (.set pixels (:data crop) (nth offsets f)))) - {"source/dense" (named "source/dense" analysis ["landmarks"] + (cond-> {"source/dense" (named "source/dense" analysis ["landmarks"] {:type "float64" :frames frames :points points :stride (* 3 points)} landmarks) "source/detected" (named "source/detected" analysis ["detected"] {:type "uint8" :frames frames :stride 1} mask) "source/crops" (named "source/crops" analysis ["rgba"] {:type "uint8" :frames frames :boxes boxes :offsets offsets} - pixels)}))) + pixels)} + interior (assoc "source/interior" + (interior-block analysis params/defaults interior)))))) (defn wire-blocks [blocks] (into-array @@ -80,7 +142,7 @@ (map (fn [role] (let [{:keys [key descriptor data]} (get blocks role)] #js {:key key :descriptor descriptor :data data})) - roles))) + (filter #(contains? blocks %) (conj roles "source/interior"))))) (defn unpack "The three block-detail responses -> inputs for measurement and freeze." diff --git a/frontend/src/arthur/flow/take.cljs b/frontend/src/arthur/flow/take.cljs index 37f50b2..2312820 100644 --- a/frontend/src/arthur/flow/take.cljs +++ b/frontend/src/arthur/flow/take.cljs @@ -60,6 +60,44 @@ :detected detected :presence presence))) +(defn anchor-base + "Shared upstream measurement for a feature regeneration." + [{:keys [aspect] :as params} {:keys [dense detected presence]}] + (let [fitted (anchor/fit {:aspect aspect} {:dense dense})] + (assoc (condition/anchor params fitted) + :detected detected :presence presence))) + +(defn measure-part + "Recompute one feature type through the shared conditioned anchor. A brow + reads the eye corners, and teeth read mouth aperture; these are input edges, + not requests to freeze those other features." + [area {:keys [aspect] :as params} + {:keys [dense detected interior presence]} {:keys [transforms] :as base}] + (let [mouth (fn [] + (mouth/measure {:aspect aspect} + {:dense dense :transforms transforms})) + eyes (fn [] + (eyes/measure params {:dense dense :transforms transforms + :detected detected :presence presence}))] + (merge (select-keys base [:detected :presence]) + (case area + :mouth (let [m (mouth)] + {:outer (condition/contours params (:outer m)) + :inner (condition/contours params (:inner m)) + :aperture (:aperture m)}) + :eye {:eyes (condition-eyes/apply-defaults params (eyes))} + :brow {:brows (condition-brows/apply-defaults + params + (brows/measure params {:dense dense :transforms transforms}) + (eyes))} + :teeth (let [aperture (:aperture (mouth))] + {:teeth (condition-interior/apply-defaults + params + (mapv interior/head-local interior transforms + (repeat aspect)) + aperture)}) + (throw (ex-info "unknown measured feature type" {:area area})))))) + (defn build [params inputs] (freeze/clip params (measure params inputs))) diff --git a/frontend/src/arthur/footage/store.cljs b/frontend/src/arthur/footage/store.cljs index 75f3a38..8e60f26 100644 --- a/frontend/src/arthur/footage/store.cljs +++ b/frontend/src/arthur/footage/store.cljs @@ -22,7 +22,8 @@ ([entry kind] (let [id (keyword kind (str (swap! serial inc)))] (when-let [old (:audio @loaded)] - (when (.startsWith old "blob:") (js/URL.revokeObjectURL old))) + (when (and (not= old (:audio entry)) (.startsWith old "blob:")) + (js/URL.revokeObjectURL old))) (reset! loaded (assoc entry :id id)) id))) diff --git a/frontend/src/arthur/ui/shell.cljs b/frontend/src/arthur/ui/shell.cljs index 0a810eb..504d230 100644 --- a/frontend/src/arthur/ui/shell.cljs +++ b/frontend/src/arthur/ui/shell.cljs @@ -7,15 +7,103 @@ why scrubbing at speed does not re-render the page." (:require [arthur.clock :as clock] [arthur.db :as db] + [arthur.domain.feature :as feature] + [arthur.domain.params :as params] [arthur.events.footage :as footage] [arthur.events.playback :as pb] [arthur.events.project :as project] [arthur.subs.playback :as sub] [arthur.subs.render :as render] [arthur.ui.player :as player] - [re-frame.core :as rf])) + [re-frame.core :as rf] + [reagent.core :as r])) (def ^:private zoom 2) +(defonce ^:private selected-owner (r/atom nil)) +(defonce ^:private drafts (r/atom {})) + +(defn- setting-owners [clip] + (let [instances (for [[id node] (get-in clip [:timelines :main :nodes]) + :when (= :symbol (:kind node))] id) + instances (if (seq instances) (sort-by str instances) [nil])] + (vec (for [instance instances + [scope ids] [[:subject (:subjects clip)] + [:feature (:features clip)] + [:group (:groups clip)]] + id (sort-by str (keys ids))] + [scope id instance])))) + +(defn- slider-bounds [knob {:keys [type default min even?] upper :max}] + {:min (or min (if (= knob :blob-grow) -3 0)) + :max (or upper (get {:verts 20 :eye-verts 16 :brow-verts 10 + :teeth-verts 20 :blob-grow 3 :top-bias 2 + :gaze-gain 4} knob) + (max 1 (* 2 default))) + :step (cond even? 2 (= type :integer) 1 :else 0.01)}) + +(defn- controls [] + (let [clip @(rf/subscribe [::render/clip]) + busy? (:busy? @(rf/subscribe [::sub/project])) + report @(rf/subscribe [::project/regeneration]) + owners (setting-owners clip) + [scope id instance :as owner] (if (some #{(deref selected-owner)} owners) + @selected-owner (first owners)) + area (case scope + :subject :subject + :feature (get-in clip [:features id :area]) + :group :eye + nil) + values (case scope + :subject (merge (params/for-area :subject) + (get-in clip [:subjects id :params])) + :feature (when id (feature/effective-params clip id)) + :group (merge (params/for-area :eye) + (get-in clip [:groups id :params])) + nil)] + [:section.controls + [:label "active object " + [:select {:value (or (first (keep-indexed + (fn [i option] (when (= option owner) i)) owners)) 0) + :on-change #(reset! selected-owner + (nth owners (js/parseInt (.. % -target -value) 10)))} + (if (seq owners) + (doall (for [[i [kind object-id]] (map-indexed vector owners)] + ^{:key i} [:option {:value i} + (str (when-let [instance (nth (nth owners i) 2)] + (str (name instance) " / ")) + (name kind) " · " (name object-id))])) + [:option {:value 0} "no tracked objects"])] ] + (when instance + [:div.shared-note "shared symbol · changes all seven stage instances"]) + (when area + [:div.control-list + (doall + (for [[knob spec] (sort-by (comp str key) params/definitions) + :when (= area (:area spec))] + (let [draft-key [scope id knob] + value (get @drafts draft-key (get values knob)) + {:keys [min max step]} (slider-bounds knob spec)] + ^{:key (str draft-key)} + [:label.control-row + [:span (name knob)] + [:input {:type "range" :min min :max max :step step :value value + :disabled busy? + :on-change (fn [event] + (let [s (.. event -target -value) + v (if (= :integer (:type spec)) + (js/parseInt s 10) + (js/parseFloat s))] + (swap! drafts assoc draft-key v) + (rf/dispatch [::project/preview-settings + {:scope scope :id id :knob knob + :value v}]))) }] + [:output (str value)]])))]) + (when report + [:pre.regeneration-debug + (str "dirty features: " (pr-str (:features report)) "\n" + "new block roles: " (if (seq (:roles report)) + (pr-str (sort (:roles report))) "none (tier 1 only)") + "\n" (get-in @(rf/subscribe [::sub/project]) [:status]))])])) (defn- audio [] (let [src @(rf/subscribe [::sub/audio])] @@ -153,6 +241,7 @@ [stage] [audio] [transport] + [controls] [:p.note "Upload a video, choose its footage, then load frames. Save the project to " "share the analyzed take without detecting frames again."]]) diff --git a/frontend/test/arthur/flow/regenerate_test.cljs b/frontend/test/arthur/flow/regenerate_test.cljs new file mode 100644 index 0000000..0bfffbc --- /dev/null +++ b/frontend/test/arthur/flow/regenerate_test.cljs @@ -0,0 +1,101 @@ +(ns arthur.flow.regenerate-test + (:require [cljs.test :refer [deftest is]] + [arthur.domain.project :as project] + [arthur.flow.address :as address] + [arthur.flow.regenerate :as regenerate] + [arthur.flow.take :as take] + [arthur.synth :as synth])) + +(def ^:private frames 40) +(def ^:private inputs (delay {:dense (synth/synth-dense frames {:seed 9})})) +(def ^:private initial + (delay + (let [p (merge take/knobs + {:name "regen" :fps 30 :aspect 1 :stage [320 200] + :expose 1 :head :as-filmed + :analysis (address/analysis + {:detector "synth" :version "mulberry32" + :seed 9 :frames frames :fps 30 :aspect 1})})] + (assoc (take/build p @inputs) :source-inputs @inputs)))) + +(defn- full-at [change] + (take/build + (merge take/knobs + {:name "regen" :fps 30 :aspect 1 :stage [320 200] + :expose 1 :head :as-filmed + :analysis (get-in @initial [:clip :analysis])} + change) + @inputs)) + +(defn- channel [entry node path] + (get-in entry [:clip :timelines :main :nodes node :channels path])) + +(deftest eye-rebuild-is-confined-to-the-edited-feature + (let [before @initial + after (regenerate/change before + {:scope :feature :id :eye-r :knob :gaze-gain :value 2})] + (is (not= (channel before :iris-r [:xform :pos]) + (channel after :iris-r [:xform :pos]))) + (is (= (channel before :iris-l [:xform :pos]) + (channel after :iris-l [:xform :pos]))) + (is (= (channel before :mouth [:geom :pts]) + (channel after :mouth [:geom :pts]))) + (is (= (get-in before [:clip :timelines :main :nodes :face]) + (get-in after [:clip :timelines :main :nodes :face]))) + (is (= (channel (full-at {:gaze-gain 2}) :iris-r [:xform :pos]) + (channel after :iris-r [:xform :pos]))))) + +(deftest framed-eye-size-needs-no-new-dense-block + (let [before @initial + after (regenerate/change before + {:scope :feature :id :eye-r :knob :iris-size :value 0.6})] + (is (not= (channel before :iris-r [:geom :radius]) + (channel after :iris-r [:geom :radius]))) + (is (= (channel before :iris-l [:geom :radius]) + (channel after :iris-l [:geom :radius]))) + (is (= (channel (full-at {:iris-size 0.6}) :iris-r [:geom :radius]) + (channel after :iris-r [:geom :radius]))) + (is (= (set (keys (:store before))) (set (keys (:store after))))))) + +(deftest mouth-edit-leaves-the-head-and-eye-alone + (let [before @initial + after (regenerate/change before + {:scope :feature :id :mouth :knob :verts :value 10})] + (is (not= (channel before :mouth [:geom :pts]) + (channel after :mouth [:geom :pts]))) + (is (= (get-in before [:clip :timelines :main :nodes :head]) + (get-in after [:clip :timelines :main :nodes :head]))) + (is (= (channel before :eye-r [:geom :pts]) + (channel after :eye-r [:geom :pts]))) + (is (= (channel (full-at {:verts 10}) :mouth [:geom :pts]) + (channel after :mouth [:geom :pts]))))) + +(deftest subject-edit-recomputes-head-and-keeps-authored-placement + (let [before @initial + after (regenerate/change before + {:scope :subject :id :face-1 :knob :anchor-avg :value 4}) + full (full-at {:anchor-avg 4})] + (is (not= (get-in before [:clip :timelines :main :nodes :head :measured]) + (get-in after [:clip :timelines :main :nodes :head :measured]))) + (is (= (get-in full [:clip :timelines :main :nodes :head :measured]) + (get-in after [:clip :timelines :main :nodes :head :measured]))) + (is (= (get-in before [:clip :timelines :main :nodes :face]) + (get-in after [:clip :timelines :main :nodes :face]))))) + +(deftest contour-edit-does-not-rebuild-head-or-teeth + (let [before @initial + after (regenerate/change before + {:scope :subject :id :face-1 :knob :contour-avg :value 3})] + (is (= (get-in before [:clip :timelines :main :nodes :head]) + (get-in after [:clip :timelines :main :nodes :head]))) + (is (= (channel before :teeth [:geom :pts]) + (channel after :teeth [:geom :pts]))))) + +(deftest edited-take-still-round-trips-through-normal-save + (let [after (regenerate/change @initial + {:scope :group :id :eyes-1 :knob :iris-size :value 0.6}) + wire (project/save "regen" after) + loaded (project/load "regen" (js/JSON.parse (js/JSON.stringify wire)))] + (is (= (:clip after) (:clip loaded))) + (is (= (channel after :iris-r [:geom :radius]) + (channel loaded :iris-r [:geom :radius]))))) diff --git a/frontend/test/arthur/flow/source_test.cljs b/frontend/test/arthur/flow/source_test.cljs index d5f3626..0f4312b 100644 --- a/frontend/test/arthur/flow/source_test.cljs +++ b/frontend/test/arthur/flow/source_test.cljs @@ -1,7 +1,25 @@ (ns arthur.flow.source-test (:require [cljs.test :refer [deftest is]] + [arthur.domain.params :as params] + [arthur.domain.wire :as wire] [arthur.flow.source :as source])) +(deftest pixel-measurements-have-their-own-addressed-block + (let [settings params/defaults + ring (mapv (fn [i] {:x (/ i 10) :y (/ (+ i 1) 20)}) + (range (:teeth-verts settings))) + measures [{:contour ring :contrast 0.7 :area 42} + {:contour nil :contrast 0 :area 0}] + block (source/interior-block "sha256:analysis" settings measures) + response #js {:descriptor (:descriptor block) + :data (wire/base64 (:data block))}] + (is (= (:key block) + (source/interior-key "sha256:analysis" settings 2))) + (is (= measures (source/unpack-interior response settings 2))) + (is (not= (:key block) + (source/interior-key "sha256:analysis" + (assoc settings :cavity-erode 0.2) 2))))) + (deftest source-blocks-round-trip-without-source-images (let [face (vec (repeat 478 {:x 0.25 :y 0.5 :z -0.125})) pixels (js/Uint8ClampedArray. #js [12 24 36 255]) @@ -19,3 +37,12 @@ (is (nil? (second (:crops out)))) (is (= (mapv :key (vals blocks)) (mapv :key (vals (source/pack "sha256:analysis" out))))))) + +(deftest fresh-source-includes-its-pixel-measurement-block + (let [face (vec (repeat 478 {:x 0.25 :y 0.5 :z -0.125})) + input {:dense [face] :detected [false] :crops [nil] + :interior [{:contour nil :contrast 0 :area 0}]} + blocks (source/pack "sha256:analysis" input)] + (is (contains? blocks "source/interior")) + (is (= 4 (.-length (source/upload-blocks blocks)))) + (is (= 3 (count source/roles))))) diff --git a/frontend/test/browser/take.mjs b/frontend/test/browser/take.mjs index 1adf6ea..f1209d4 100644 --- a/frontend/test/browser/take.mjs +++ b/frontend/test/browser/take.mjs @@ -233,7 +233,8 @@ async function main() { if (probe && probe.drawn > 0) break; await sleep(100); } - if (!probe) throw new Error('no canvas.stage on the page — is `shadow-cljs watch app` running?'); + if (!probe) throw new Error('no canvas.stage on the page — ' + + (page.logs.slice(0, 3).join(' | ') || 'is `shadow-cljs watch app` running?')); console.log(`\ncanvas ${probe.w}x${probe.h}, clip ${JSON.stringify(probe.selectedClip)}`); @@ -393,6 +394,138 @@ async function main() { check(!back[0].selectedClip.includes('take'), 'the picture is the reopened document', JSON.stringify(back[0].selectedClip)); + // The local 8625 study is optional in a fresh test database. When present, + // tune its shared symbol while its stage is playing. + const stageSource = await page.eval(`fetch('/api/projects/4379f900-bdd2-409b-acf6-32081f8ce01f') + .then(r => r.ok)`); + if (stageSource) { + check(await page.eval(CLICK('stage 8625')), 'the 8625 stage loads'); + const stageLoaded = await statusMatching(/loaded 8625 stage study/, 160); + check(stageLoaded !== null, 'the 8625 stage is ready', stageLoaded ?? (await page.eval(STATUS))); + const eyeSelected = await page.eval(`(() => { + const select = document.querySelector('.controls select'); + const option = [...select.options] + .find(o => o.textContent.trim() === 'left / feature · eye-r'); + if (!option) return false; + select.value = option.value; + select.dispatchEvent(new Event('change', { bubbles: true })); + return true; + })()`); + check(eyeSelected, 'a stage instance exposes its tracked right eye'); + await sleep(100); + const irisSlider = await page.eval(`(() => { + const row = [...document.querySelectorAll('.control-row')] + .find(row => row.querySelector('span')?.textContent === 'iris-size'); + if (!row) return null; + const r = row.querySelector('input').getBoundingClientRect(); + return { x: r.x + r.width * 0.9, y: r.y + r.height / 2 }; + })()`); + check(irisSlider !== null, 'the stage eye has an iris-size slider'); + if (irisSlider) { + check(await page.eval(CLICK('play')), 'the stage starts playing'); + const startFrame = await page.eval(`Number(document.querySelector('.readout span').textContent.match(/\\d+/)[0])`); + await page.send('Input.dispatchMouseEvent', { + type: 'mousePressed', x: irisSlider.x, y: irisSlider.y, button: 'left', clickCount: 1, + }); + await page.send('Input.dispatchMouseEvent', { + type: 'mouseReleased', x: irisSlider.x, y: irisSlider.y, button: 'left', clickCount: 1, + }); + const preview = await statusMatching(/preview · unsaved/, 160); + const debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`); + check(preview !== null, 'the slider updates the stage preview', preview ?? debug); + check(debug.includes(':eye-r') && debug.includes('tier 1 only'), + 'the panel reports the affected feature and tier', debug); + const endFrame = await page.eval(`Number(document.querySelector('.readout span').textContent.match(/\\d+/)[0])`); + check(endFrame > startFrame, 'playback continues during tuning', `${startFrame} -> ${endFrame}`); + await page.eval(CLICK('pause')); + } + const subjectSelected = await page.eval(`(() => { + const select = document.querySelector('.controls select'); + const option = [...select.options] + .find(o => o.textContent.includes('subject · face-1')); + if (!option) return false; + select.value = option.value; + select.dispatchEvent(new Event('change', { bubbles: true })); + return true; + })()`); + check(subjectSelected, 'the stage exposes its tracked subject'); + if (subjectSelected) { + await sleep(100); + const anchorSlider = await page.eval(`(() => { + const row = [...document.querySelectorAll('.control-row')] + .find(row => row.querySelector('span')?.textContent === 'anchor-avg'); + if (!row) return null; + const r = row.querySelector('input').getBoundingClientRect(); + return { x: r.x + r.width * 0.9, y: r.y + r.height / 2 }; + })()`); + check(anchorSlider !== null, 'the subject has an anchor slider'); + if (anchorSlider) { + const anchorStart = Date.now(); + await page.send('Input.dispatchMouseEvent', { + type: 'mousePressed', x: anchorSlider.x, y: anchorSlider.y, + button: 'left', clickCount: 1, + }); + await page.send('Input.dispatchMouseEvent', { + type: 'mouseReleased', x: anchorSlider.x, y: anchorSlider.y, + button: 'left', clickCount: 1, + }); + let debug = ''; + for (let i = 0; i < 160; i++) { + debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`); + if (debug.includes('head-pos')) break; + await sleep(250); + } + check(debug.includes('head-pos') && debug.includes(':teeth'), + 'anchor invalidates the head and teeth', debug); + const preview = await statusMatching(/preview · unsaved/, 160); + check(preview !== null, 'the anchor edit finishes previewing', + preview ? `${Date.now() - anchorStart}ms` : (await page.eval(STATUS))); + } + } + const teethSelected = await page.eval(`(() => { + const select = document.querySelector('.controls select'); + const option = [...select.options] + .find(o => o.textContent.trim() === 'left / feature · teeth'); + if (!option) return false; + select.value = option.value; + select.dispatchEvent(new Event('change', { bubbles: true })); + return true; + })()`); + check(teethSelected, 'the stage exposes its tracked teeth'); + if (teethSelected) { + await sleep(100); + const slider = await page.eval(`(() => { + const row = [...document.querySelectorAll('.control-row')] + .find(row => row.querySelector('span')?.textContent === 'cavity-erode'); + if (!row) return null; + const r = row.querySelector('input').getBoundingClientRect(); + return { x: r.x + r.width * 0.9, y: r.y + r.height / 2 }; + })()`); + check(slider !== null, 'the teeth have a pixel-setting slider'); + if (slider) { + await page.send('Input.dispatchMouseEvent', { + type: 'mousePressed', x: slider.x, y: slider.y, + button: 'left', clickCount: 1, + }); + await page.send('Input.dispatchMouseEvent', { + type: 'mouseReleased', x: slider.x, y: slider.y, + button: 'left', clickCount: 1, + }); + let debug = ''; + for (let i = 0; i < 160; i++) { + debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`); + if (debug.includes('dirty features: [:teeth]')) break; + await sleep(250); + } + check(debug.includes('dirty features: [:teeth]'), + 'the pixel setting invalidates only teeth', debug); + const preview = await statusMatching(/preview · unsaved/, 160); + check(preview !== null, 'the teeth edit finishes previewing', + preview ?? (await page.eval(STATUS))); + } + } + } + // Exercise the actual file input and FormData path, then wait for the // extraction job's footage to appear in the server list. const uploadDir = mkdtempSync(join(tmpdir(), 'arthur-upload-'));