From 41b4bdf1106222c302400a55e09d3005f343a292 Mon Sep 17 00:00:00 2001 From: Olive Vaughn Date: Tue, 29 Sep 2026 13:21:08 -0400 Subject: [PATCH] Make a symbol from a dropped video Dropping footage on the stage or the timeline asks which frames and what name: a dialog plays the video with start and end handles that seek it. Detection runs on that range only (decoding stops at the end, frames before the start are skipped) and the range is part of the analysis address, so a partial analysis is never served as a whole one. The frozen take comes in as one named symbol, carrying its sound as an audio node, placed where it was dropped; tracking and tuning come with it when the document has no analysis of its own. clip/adopt copies symbols between documents, renaming ids that collide, and clip/audio-tracks carries sounds out of nested instances so a placed symbol is heard where it is placed. Fixes the stage going blank after a conversion: ::store recomputed only when the clip id changed, so blocks merged into the loaded entry were invisible to the resolver. And the paint loop now schedules its next frame before painting and reports a frame it cannot draw instead of stopping. Co-Authored-By: Claude Opus 5.5 --- frontend/src/arthur/audio/mix.cljs | 6 +- frontend/src/arthur/domain/clip.cljs | 124 +++++++++- frontend/src/arthur/events/edit.cljs | 13 +- frontend/src/arthur/events/footage.cljs | 225 +++++++++++------- frontend/src/arthur/events/playback.cljs | 11 + frontend/src/arthur/flow/address.cljs | 14 +- frontend/src/arthur/flow/ingest.cljs | 14 ++ frontend/src/arthur/flow/take.cljs | 21 +- frontend/src/arthur/footage/store.cljs | 19 +- frontend/src/arthur/subs/render.cljs | 11 +- frontend/src/arthur/subs/ui.cljs | 1 + frontend/src/arthur/ui/convert.cljs | 90 +++++++ frontend/src/arthur/ui/drag.cljs | 8 +- frontend/src/arthur/ui/player.cljs | 17 +- frontend/src/arthur/ui/pool.cljs | 4 +- frontend/src/arthur/ui/shell.cljs | 2 + .../test/arthur/domain/instance_test.cljs | 31 +++ static/arthur/app.css | 71 ++++++ 18 files changed, 562 insertions(+), 120 deletions(-) create mode 100644 frontend/src/arthur/ui/convert.cljs diff --git a/frontend/src/arthur/audio/mix.cljs b/frontend/src/arthur/audio/mix.cljs index 29d4809..9706eda 100644 --- a/frontend/src/arthur/audio/mix.cljs +++ b/frontend/src/arthur/audio/mix.cljs @@ -91,10 +91,10 @@ (.setValueAtTime param (* factor v) (/ f fps))))))) (defn tracks-of - "The audio nodes of one of the clip's symbols. Any symbol may carry its own - sound, and playback mixes the open one's." + "The sounds symbol `sid` plays, including those inside what it places — see + `clip/audio-tracks`. Playback mixes the open symbol's." [document sid] - (filter #(= :audio (:kind %)) (vals (:nodes (clip/symbol document sid))))) + (clip/audio-tracks document sid)) (defn- render! [document sid sources store] (let [fps (:fps document) diff --git a/frontend/src/arthur/domain/clip.cljs b/frontend/src/arthur/domain/clip.cljs index f5d6a98..84c2d0e 100644 --- a/frontend/src/arthur/domain/clip.cljs +++ b/frontend/src/arthur/domain/clip.cljs @@ -37,7 +37,8 @@ [arthur.domain.node :as node] [arthur.domain.palette :as pal] [arthur.domain.pose :as pose] - [arthur.domain.symbol :as symbol])) + [arthur.domain.symbol :as symbol] + [clojure.string :as string])) (def clip-keys "Every top-level field of a clip, and the reason `arthur.domain.leaf` refuses @@ -174,6 +175,127 @@ (assoc-in [:symbols sid] {:id sid :name (name sid) :frames (- end frame) :nodes {}}) (place-symbol host sid frame uuid [0 0]))))) +(defn- free-id + "`wanted`, or the first `wanted-2`, `wanted-3`… `taken?` does not claim. + Keeps the namespace, so `:sym/face` becomes `:sym/face-2`." + [taken? wanted] + (first (remove taken? + (cons wanted + (map #(keyword (namespace wanted) (str (name wanted) "-" %)) + (iterate inc 2)))))) + +(defn adopt + "Copy symbols `roots` of clip `other`, and every symbol they place, into + `clip`. Returns `{:clip :ids}`, where `:ids` maps each copied symbol's id in + `other` to its id here. + + AN ID THAT IS TAKEN IS RENAMED, never merged: two symbols that happen to share + an id are two drawings, and an instance's `:of` inside the copy is rewritten to + follow. `wanted` maps a root's id in `other` to the id it should preferably get, + which is how a symbol made from footage is called what the person typed rather + than `:main`. + + Only symbols travel. What else `other` holds — tracking identities, an analysis + — is the caller's decision, because whether it can come too depends on what + `clip` already has." + [clip other roots wanted] + (let [;; A tree walk is safe because placing cannot make a cycle. + reach (into #{} (mapcat #(tree-seq any? (partial places other) %)) roots) + ids (reduce (fn [ids sid] + (let [taken? #(or (contains? (:symbols clip) %) + (some #{%} (vals ids)))] + (assoc ids sid (free-id taken? (get wanted sid sid))))) + {} (sort-by str reach)) + copy (fn [sid] + (-> (symbol other sid) + (assoc :id (ids sid)) + (update :nodes #(into {} (map (fn [[id n]] + [id (cond-> n (:of n) (update :of ids))])) + %))))] + {:clip (reduce (fn [c sid] (assoc-in c [:symbols (ids sid)] (copy sid))) clip reach) + :ids ids})) + +(defn symbol-id + "An id for a symbol a person has named: the name, lower-cased and hyphenated, + or `:symbol` when nothing of it survives." + [label] + (let [slug (-> (str label) string/lower-case + (string/replace #"[^a-z0-9]+" "-") + (string/replace #"^-+|-+$" ""))] + (keyword (if (seq slug) slug "symbol")))) + +(defn add-take + "Put a frozen take into `clip` as ONE symbol called `label`. Returns + `{:clip :sid :tracked?}`. + + `take` is what `flow/freeze/clip` makes: a `:main` that places one symbol per + tracked face. `:main` becomes the named symbol — it is what holds the faces in + stage pixels, so it is the thing worth placing — and it gets the take's SOUND + as an audio node of its own, source frames `range` of footage `footage-id`, so + wherever the symbol is placed it is heard. + + The tracking identities, and the analysis they were measured by, come along + only when `clip` has no analysis of its own and no face had to be renamed. A + document holds one analysis, and regeneration finds a face's symbol by its + subject id, so either condition failing means the take comes in as drawings + that play but cannot be re-tuned — `:tracked? false` says so." + [clip take label footage-id range] + (let [{c :clip ids :ids} (adopt clip take [:main] {:main (symbol-id label)}) + sid (ids :main) + tracked? (and (nil? (:analysis clip)) + (every? #(= % (ids %)) (keys (:subjects take))))] + {:sid sid + :tracked? tracked? + :clip (cond-> (-> c + (assoc-in [:symbols sid :name] (str label)) + (assoc-in [:symbols sid :nodes :sound] + {:id :sound :name "sound" :kind :audio :parent nil + :z "z-sound" :source {:footage footage-id} + :span range :time {:mode :map :at 0 :rate 1}})) + tracked? (-> (assoc :analysis (:analysis take)) + (update :subjects merge (:subjects take)) + (update :features merge (:features take)) + (update :groups merge (:groups take))))})) + +(defn audio-tracks + "Every sound symbol `sid` plays, as audio nodes in `sid`'s own frames: its own + and, recursively, those inside the instances it places. + + A sound inside a placed symbol is heard where the instance puts it, so each one + is carried OUT through the instance's time map — `:at` and `:rate`, and clipped + to the instance's own span — until it is in the frames of the symbol being + played. Keyed automation moves with it. What comes back is what a mixer that + only knows flat tracks can play as it is." + [clip sid] + (let [sym (symbol clip sid)] + (into (vec (filter #(= :audio (:kind %)) (vals (:nodes sym)))) + (mapcat + (fn [inst] + (let [{:keys [mode at rate] :or {at 0 rate 1}} (:time inst) + inner (symbol clip (:of inst)) + [in out] (or (:span inst) [0 (:frames inner)]) + in (if (= mode :map) in 0) + ->outer (fn [x] (+ at (/ (- x in) rate)))] + (keep (fn [a] + (let [{a-at :at a-rate :rate :or {a-at 0 a-rate 1}} (:time a) + [s0 s1] (or (:span a) [0 (:frames inner)]) + x0 (max a-at in) + x1 (min (+ a-at (/ (- s1 s0) a-rate)) out)] + (when (< x0 x1) + (-> a + (assoc :span [(+ s0 (* a-rate (- x0 a-at))) + (+ s0 (* a-rate (- x1 a-at)))] + :time {:mode :map :at (->outer x0) + :rate (* a-rate rate)}) + (update :channels + (fn [chs] + (into {} (map (fn [[p ch]] + [p (cond-> ch (:keys ch) + (update :keys #(into {} (map (fn [[f v]] [(->outer f) v])) %)))])) + chs))))))) + (audio-tracks clip (:of inst))))) + (filter #(= :instance (:kind %)) (vals (:nodes sym))))))) + (defn frame-inside "Carry frame `f` of symbol `sid` down through the nodes named by `path`, one per level, the way a timeline row's path names them. Returns `[symbol frame]`: diff --git a/frontend/src/arthur/events/edit.cljs b/frontend/src/arthur/events/edit.cljs index 42ce07d..f1c065a 100644 --- a/frontend/src/arthur/events/edit.cljs +++ b/frontend/src/arthur/events/edit.cljs @@ -14,13 +14,20 @@ were the only thing anyone could edit. They are not." (:require [arthur.footage.store :as store])) -(defn edit - "Apply `f` to the loaded clip and return the new db." +(defn edit-entry + "Apply `f` to the loaded ENTRY — the document and the blocks, footage and + source tracks beside it — and return the new db. For an edit that brings tier-2 + data in with it, which a document edit alone cannot." [db f] - (let [id (store/edit-clip! (:clip/current db) f)] + (let [id (store/edit-entry! (:clip/current db) f)] (if id (-> db (assoc :clip/current id) (update :paint/revision (fnil inc 0)) (update :project merge {:status "edited · unsaved"})) db))) + +(defn edit + "Apply `f` to the loaded clip and return the new db." + [db f] + (edit-entry db #(update % :clip f))) diff --git a/frontend/src/arthur/events/footage.cljs b/frontend/src/arthur/events/footage.cljs index ca54b58..22f05cf 100644 --- a/frontend/src/arthur/events/footage.cljs +++ b/frontend/src/arthur/events/footage.cljs @@ -5,6 +5,7 @@ detector's identity comes from the server too, because it goes into the content address of every block this produces." (:require [arthur.domain.clip :as clip] + [arthur.events.edit :as edit] [arthur.events.playback :as pb] [arthur.flow.detect :as detect] [arthur.flow.ingest :as ingest] @@ -14,6 +15,7 @@ [arthur.footage.store :as store] [arthur.domain.landmarks :as lm] [arthur.fx.http :as http] + [clojure.string :as string] [re-frame.core :as rf])) (defonce ^:private clock (atom 0)) @@ -44,8 +46,12 @@ The work happens inside `decode!`'s callback, and the promise it returns is the backpressure: the decoder does not run ahead of the detector, so a 900-frame - take does not hold 900 decoded frames at 1440x1920 in memory." - [manifest model] + take does not hold 900 decoded frames at 1440x1920 in memory. + + Only source frames `[start end)` are measured. Decoding still begins at frame + 0, because every frame after the first is coded against the ones before it, + and it stops at `end`; frames before `start` are decoded and dropped unread." + [manifest model [start end]] (let [[w h] [(:width manifest) (:height manifest)] canvas (.createElement js/document "canvas") ctx (.getContext canvas "2d" #js {:willReadFrequently true}) @@ -53,45 +59,48 @@ raw (atom []) crops (atom []) inner (atom []) - total (:frames manifest)] + total end] (set! (.-width canvas) w) (set! (.-height canvas) h) (rf/dispatch [::progress "loading the video…"]) - (-> (ingest/stream! (ingest/stream-url manifest) total) + (-> (ingest/stream! (ingest/stream-url manifest) (:frames manifest)) (.then (fn [stream] (ingest/decode! - stream fps w h + (update stream :units subvec 0 end) fps w h (fn [i frame] - (.drawImage ctx frame 0 0) - ;; EVERY FACE ON THIS FRAME, each with its own mouth crop taken - ;; while the frame's pixels are still on the canvas. Which of these - ;; detections belongs to which subject is not decided here — the - ;; answer needs the whole take — so all three vectors stay in - ;; DETECTION ORDER and `detect/tracks` re-keys them afterwards. - (let [faces (detect/detect! model canvas (ingest/frame-ms fps i)) - boxes (mapv (fn [face] - (interior/crop (mapv #(nth face %) lm/LIPS-INNER) - [w h])) - faces) - frame-crops (mapv (fn [box] - (when box - {:box box - :data (.-data (.getImageData - ctx (:x box) (:y box) - (:w box) (:h box)))})) - boxes)] - (swap! raw conj faces) - (swap! crops conj frame-crops) - ;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work - ;; either way, but done after the fact it is ten seconds of - ;; synchronous arithmetic with the main thread held and the - ;; frame counter frozen on its last value — which reads as the - ;; decoder hanging, and was diagnosed as that twice. - (swap! inner conj (mapv #(source/measure-crop take/knobs %) - frame-crops))) + (when (>= i start) + (.drawImage ctx frame 0 0) + ;; EVERY FACE ON THIS FRAME, each with its own mouth crop taken + ;; while the frame's pixels are still on the canvas. Which of + ;; these detections belongs to which subject is not decided here + ;; — the answer needs the whole take — so all three vectors stay + ;; in DETECTION ORDER and `detect/tracks` re-keys them afterwards. + (let [faces (detect/detect! model canvas (ingest/frame-ms fps i)) + boxes (mapv (fn [face] + (interior/crop (mapv #(nth face %) lm/LIPS-INNER) + [w h])) + faces) + frame-crops (mapv (fn [box] + (when box + {:box box + :data (.-data (.getImageData + ctx (:x box) (:y box) + (:w box) (:h box)))})) + boxes)] + (swap! raw conj faces) + (swap! crops conj frame-crops) + ;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work + ;; either way, but done after the fact it is ten seconds of + ;; synchronous arithmetic with the main thread held and the + ;; frame counter frozen on its last value — which reads as the + ;; decoder hanging, and was diagnosed as that twice. + (swap! inner conj (mapv #(source/measure-crop take/knobs %) + frame-crops)))) (when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total)) - (rf/dispatch [::progress (str "detecting " (inc i) "/" total)])) + (rf/dispatch [::progress (if (< i start) + (str "seeking " (inc i) "/" start) + (str "detecting " (- (inc i) start) "/" (- end start)))])) ;; Yield, so the status and the transport paint between synchronous ;; MediaPipe calls. `decode!` waits on this before feeding more. (js/Promise. (fn [done] (js/setTimeout done 0))))))) @@ -251,38 +260,44 @@ (js/Promise.resolve track) (:subjects track))) +(defn- analyse! + "Promise of source frames `[start end)` of footage `footage-id`, detected, + measured and frozen — `{:clip :store ...}` as `build-clip` makes it, with the + footage reading as though it were only those frames. A saved analysis of + exactly that range is reused instead of detecting again." + [footage-id [start end]] + (-> (js/Promise.all #js [(ingest/manifest! footage-id) (ingest/detector!)]) + (.then (fn [[full detector]] + (let [manifest (ingest/slice full start end)] + (reset! clock (js/Date.now)) + (rf/dispatch [::progress "looking for saved analysis…"]) + (-> (cached-source! manifest detector) + (.then (fn [track] + (if track + (do (rf/dispatch [::progress "reusing saved analysis…"]) + (-> (measure-crops! + take/knobs track + (fn [done total] + (when (or (= 1 done) (zero? (mod done 4)) + (= done total)) + (rf/dispatch + [::progress (str "measuring " done "/" total)])))) + (.then #(build-clip manifest detector %)))) + (do (rf/dispatch [::progress "loading MediaPipe…"]) + (-> (detect/landmarker!) + (.then (fn [model] + (mark! "MediaPipe ready") + (rf/dispatch [::progress "opening the video…"]) + (detect-frames! full model [start end]))) + (.then #(build-clip manifest detector %))))))))))))) + (rf/reg-fx - ::begin! - (fn [footage-id] - (-> (js/Promise.all #js [(ingest/manifest! footage-id) (ingest/detector!)]) - (.then (fn [[manifest detector]] - (reset! clock (js/Date.now)) - (rf/dispatch [::progress "looking for saved analysis…"]) - (-> (cached-source! manifest detector) - (.then (fn [track] - (if track - (do (rf/dispatch [::progress "reusing saved analysis…"]) - (-> (measure-crops! - take/knobs track - (fn [done total] - (when (or (= 1 done) (zero? (mod done 4)) - (= done total)) - (rf/dispatch - [::progress (str "measuring " done "/" total)])))) - (.then (fn [measured] - (build-clip manifest detector measured))))) - (do (rf/dispatch [::progress "loading MediaPipe…"]) - (-> (detect/landmarker!) - (.then (fn [model] - (mark! "MediaPipe ready") - (rf/dispatch [::progress "opening the video…"]) - (-> (detect-frames! manifest model) - (.then (fn [fresh] - (build-clip manifest detector fresh)))))))))))))) - (.then (fn [entry] + ::convert! + (fn [{:keys [footage-id range] :as request}] + (-> (analyse! footage-id range) + (.then (fn [built] (mark! "build-clip: done") - (let [id (store/install! entry)] - (rf/dispatch [::loaded id (:summary entry)])))) + (rf/dispatch [::converted request built]))) (.catch (fn [error] (js/console.error error) ;; A run that ended badly may have ended on a MediaPipe graph @@ -361,19 +376,6 @@ ::choose (fn [db [_ id]] (assoc-in db [:footage :chosen] id))) -(rf/reg-event-fx - ::load - (fn [{:keys [db]} _] - (let [chosen (get-in db [:footage :chosen])] - (cond - (get-in db [:footage :loading?]) {} - (nil? chosen) - {:db (assoc-in db [:footage :status] "upload a video to begin")} - :else - {:db (update db :footage merge {:loading? true :status "reading the manifest…"}) - ::pb/pause! nil - ::begin! chosen})))) - (rf/reg-event-db ::progress (fn [db [_ message]] (assoc-in db [:footage :status] message))) @@ -384,11 +386,68 @@ (assoc db :footage (assoc (:footage db) :loading? false :status (str "footage failed: " message))))) + +;; --------------------------------------------------------------------------- +;; footage -> a symbol +;; +;; Dropping a video asks first. `[:ui :convert]` is the question — which footage, +;; which of its frames, what to call the result — and where the answer will be +;; placed: `:host`, `:frame` and `:pos` are the drop's, captured when it happened +;; so that switching tabs while detection runs does not move where it lands. + +(rf/reg-event-db + ::ask-convert + (fn [db [_ {:keys [frames label] :as footage} frame pos]] + (assoc-in db [:ui :convert] + (merge (select-keys footage [:id :label :frames :fps :video]) + {:range [0 frames] + :name (string/replace (str label) #"\.[^.]*$" "") + :host (get-in db [:ui :open]) :frame frame :pos pos})))) + +(rf/reg-event-db + ::convert-set + (fn [db [_ k v]] (assoc-in db [:ui :convert k] v))) + +(rf/reg-event-db + ::convert-cancel + (fn [db _] + (if (get-in db [:footage :loading?]) db (update db :ui dissoc :convert)))) + (rf/reg-event-fx - ::loaded - (fn [{:keys [db]} [_ id summary]] - (let [clip (store/entry id)] - {:db (-> (pb/show db id) - (assoc :footage (assoc (:footage db) :id id :label (:label clip) - :loading? false :status summary))) - ::pb/pause! nil}))) + ::convert + (fn [{:keys [db]} _] + (let [{:keys [id range] :as request} (get-in db [:ui :convert])] + (if (or (nil? request) (get-in db [:footage :loading?])) + {} + {:db (update db :footage merge {:loading? true :status "starting…"}) + ::pb/pause! nil + ::convert! {:footage-id id :range range :request request}})))) + +(rf/reg-event-fx + ::converted + (fn [{:keys [db]} [_ {{:keys [name host frame pos range]} :request footage-id :footage-id} + built]] + (let [uuid (random-uuid) + fps (get-in db [:clip :fps]) + {:keys [clip sid tracked?]} + (clip/add-take (:clip (store/entry (:clip/current db))) (:clip built) + name footage-id range) + db (edit/edit-entry + db + #(cond-> (-> % + (assoc :clip (clip/place-symbol clip host sid frame uuid pos)) + (update :store merge (:store built))) + tracked? (merge (select-keys built [:footage-id :source-blocks + :source-inputs]))))] + {:db (-> db + (update :ui dissoc :convert) + (assoc-in [:ui :selection] [:node host uuid [uuid]]) + (update :footage merge + {:loading? false + :status (str "made " name " · " (- (second range) (first range)) " frames" + (when-not tracked? + " · as drawings: this project already tracks other footage") + (when (not= fps (get-in built [:clip :fps])) + (str " · footage is " (get-in built [:clip :fps]) + " fps, this project " fps)))})) + :dispatch [::pb/refresh-clock]}))) diff --git a/frontend/src/arthur/events/playback.cljs b/frontend/src/arthur/events/playback.cljs index 9cc14b1..d0cbdce 100644 --- a/frontend/src/arthur/events/playback.cljs +++ b/frontend/src/arthur/events/playback.cljs @@ -168,6 +168,17 @@ {:db (assoc-in db [:clip :audio] url)}) {}))) +(rf/reg-event-db + ::paint-failed + (fn [db [_ message]] + (update db :project merge {:status (str "cannot draw: " message)}))) + +(rf/reg-event-fx + ::refresh-clock + ;; After an edit that changes what the open symbol sounds like. + (fn [{:keys [db]} _] + {::clock! {:id (:clip/current db) :sid (get-in db [:ui :open])}})) + (rf/reg-event-fx ::open-symbol (fn [{:keys [db]} [_ sid]] diff --git a/frontend/src/arthur/flow/address.cljs b/frontend/src/arthur/flow/address.cljs index 6cd5b53..1e6146e 100644 --- a/frontend/src/arthur/flow/address.cljs +++ b/frontend/src/arthur/flow/address.cljs @@ -67,8 +67,15 @@ over the same frames and the same model they disagree by up to 0.013 of frame width, which is a visible difference on a mouth. Optional, because the synthetic take has no running mode to declare and an absent field is how the other - optional inputs already say \"not applicable\"." - [{:keys [detector version source footage frames fps aspect seed mode tracking]}] + optional inputs already say \"not applicable\". + + `:range` is `[first end)` source frames when only part of the footage was + analysed, and absent for all of it — so every analysis made before ranges + existed keeps its address. It is in here because a partial analysis is a + different artifact: without it, detecting frames 40–90 would be saved under the + same key as the whole take, and the next conversion of the whole take would be + handed fifty frames." + [{:keys [detector version source footage frames fps aspect seed mode tracking range]}] (when-not (and (string? detector) (seq detector) (string? version) (seq version)) (throw (ex-info "an analysis names its detector and the detector's VERSION: an upgrade that silently reuses old landmarks is the failure content addressing exists to prevent" {:detector detector :version version}))) @@ -82,7 +89,8 @@ footage (assoc :footage footage) seed (assoc :seed seed) mode (assoc :mode mode) - tracking (assoc :tracking tracking)))) + tracking (assoc :tracking tracking) + range (assoc :range range)))) (defn analysis "An analysis record with its `:id` filled in. The record is tier 1 — it says diff --git a/frontend/src/arthur/flow/ingest.cljs b/frontend/src/arthur/flow/ingest.cljs index 07a7170..bbc32c9 100644 --- a/frontend/src/arthur/flow/ingest.cljs +++ b/frontend/src/arthur/flow/ingest.cljs @@ -86,6 +86,20 @@ (-> (http/GET "/api/detector") (.then (fn [json] (js->clj json :keywordize-keys true))))) +(defn slice + "The manifest of source frames `[start end)` of this footage, as though that + were all of it: its frame count, its tracing stills and its absence tracks cut + to the range, and `:range` saying where it came from — which goes into the + analysis address, see `flow/address`. The whole footage is returned unchanged, + so a full-length conversion reuses analyses made before ranges existed." + [m start end] + (if (= [start end] [0 (:frames m)]) + m + (-> m + (assoc :frames (- end start) :range [start end]) + (update :urls #(some-> % vec (subvec start end))) + (update :presence #(into {} (map (fn [[id track]] [id (subvec (vec track) start end)])) %))))) + (defn audio-url [manifest] (:audio manifest)) diff --git a/frontend/src/arthur/flow/take.cljs b/frontend/src/arthur/flow/take.cljs index a8530e8..d04dd1f 100644 --- a/frontend/src/arthur/flow/take.cljs +++ b/frontend/src/arthur/flow/take.cljs @@ -19,16 +19,17 @@ (defn analysis-for [manifest detector] (let [aspect (/ (:width manifest) (:height manifest))] (address/analysis - (merge {:detector "mediapipe" :version "unknown"} - detector - {:source (:source manifest) - :footage (:footage manifest) - :frames (:frames manifest) - :fps (:fps manifest) - :aspect aspect - ;; How the detector was run, not just which one it was. See - ;; `flow/address/analysis-descriptor`. - :mode "video" :tracking detect/settings})))) + (cond-> (merge {:detector "mediapipe" :version "unknown"} + detector + {:source (:source manifest) + :footage (:footage manifest) + :frames (:frames manifest) + :fps (:fps manifest) + :aspect aspect + ;; How the detector was run, not just which one it was. See + ;; `flow/address/analysis-descriptor`. + :mode "video" :tracking detect/settings}) + (:range manifest) (assoc :range (:range manifest)))))) (defn measure "Condition the anchor before measuring rings through it." diff --git a/frontend/src/arthur/footage/store.cljs b/frontend/src/arthur/footage/store.cljs index aafc1a2..3e620a4 100644 --- a/frontend/src/arthur/footage/store.cljs +++ b/frontend/src/arthur/footage/store.cljs @@ -32,12 +32,17 @@ @loaded (db/clip-entry id))) -(defn edit-clip! - "Edit the loaded document. Built-in clips are copied into the runtime store - on first edit, so their delayed source values remain reusable." +(defn edit-entry! + "Edit the loaded entry — the document and what travels with it: its blocks, + its footage, its retained source tracks. Built-in clips are copied into the + runtime store on first edit, so their delayed source values remain reusable." [id f] (if (= id (:id @loaded)) - (do (swap! loaded update :clip f) id) - (let [entry (entry id)] - (when entry - (install! (update entry :clip f) "paint"))))) + (do (swap! loaded f) id) + (when-let [entry (entry id)] + (install! (f entry) "paint")))) + +(defn edit-clip! + "Edit the loaded document and nothing beside it." + [id f] + (edit-entry! id #(update % :clip f))) diff --git a/frontend/src/arthur/subs/render.cljs b/frontend/src/arthur/subs/render.cljs index c0b1e1b..49b7032 100644 --- a/frontend/src/arthur/subs/render.cljs +++ b/frontend/src/arthur/subs/render.cljs @@ -69,10 +69,15 @@ (rf/reg-sub ::store :<- [::clip-id] - (fn [id _] + :<- [::paint-revision] + (fn [[id _] _] ;; Tier 2, behind a handle, and never in app-db itself — what is in the db is - ;; the id of the clip whose blocks these are. The hand-written demo has none; - ;; the swarm is entirely dense. + ;; the id of the clip whose blocks these are. + ;; + ;; ON THE REVISION AS WELL AS THE ID, like `::clip`. An edit can bring blocks + ;; in with it — a video made into a symbol does — without the id changing, and + ;; a store read only when the id changed is the old one: the document names the + ;; new blocks, the resolver cannot find them, and the frame throws. (:store (footage/entry id)))) (rf/reg-sub diff --git a/frontend/src/arthur/subs/ui.cljs b/frontend/src/arthur/subs/ui.cljs index c9c3a08..ab9258c 100644 --- a/frontend/src/arthur/subs/ui.cljs +++ b/frontend/src/arthur/subs/ui.cljs @@ -13,6 +13,7 @@ (rf/reg-sub ::tone (fn [db _] (get-in db [:ui :tone]))) (rf/reg-sub ::tool (fn [db _] (get-in db [:ui :tool]))) (rf/reg-sub ::draft (fn [db _] (get-in db [:ui :draft]))) +(rf/reg-sub ::convert (fn [db _] (get-in db [:ui :convert]))) (rf/reg-sub ::drop (fn [db _] (get-in db [:ui :drop]))) (rf/reg-sub ::tabs (fn [db _] (get-in db [:ui :tabs]))) (rf/reg-sub ::expanded (fn [db _] (get-in db [:ui :expanded]))) diff --git a/frontend/src/arthur/ui/convert.cljs b/frontend/src/arthur/ui/convert.cljs new file mode 100644 index 0000000..1a12ee5 --- /dev/null +++ b/frontend/src/arthur/ui/convert.cljs @@ -0,0 +1,90 @@ +(ns arthur.ui.convert + "The question a dropped video asks: which of its frames become a symbol, and + what is that symbol called. + + The video plays in the dialog so the frames can be chosen by looking at them. + The two handles under it are the range, `[start end)` in source frames; moving + either seeks the video to the frame it is on, so the edge being chosen is the + picture being shown. Detection then runs on those frames only — see + `events/footage/analyse!` — and the result lands where the video was dropped." + (:require [arthur.events.footage :as footage] + [arthur.subs.playback :as playback] + [arthur.subs.ui :as sub] + [re-frame.core :as rf] + [reagent.core :as r])) + +(defn- frame-at [^js track x frames] + (let [box (.getBoundingClientRect track)] + (-> (/ (* (- x (.-left box)) frames) (.-width box)) + js/Math.round (max 0) (min frames)))) + +(defn- range-bar + "Two handles over the footage's length. `player` holds the element to seek." + [{:keys [frames fps range]} player] + (r/with-let [held (atom nil) + track (atom nil)] + (let [[start end] range + pct #(str (* 100 (/ % (max 1 frames))) "%") + move! (fn [^js e] + (when-let [which @held] + (let [f (frame-at @track (.-clientX e) frames) + [s' e'] (if (= :start which) + [(min f (dec end)) end] + [start (max f (inc start))])] + (rf/dispatch [::footage/convert-set :range [s' e']]) + (when-let [^js v @player] + (set! (.-currentTime v) + (/ (if (= :start which) s' (dec e')) fps)))))) + grab (fn [which] + (fn [^js e] + (.preventDefault e) + (reset! held which) + (try (.setPointerCapture (.-currentTarget e) (.-pointerId e)) + (catch :default _ nil))))] + [:div.convert-range {:ref #(reset! track %)} + [:div.convert-kept {:style {:left (pct start) :width (pct (- end start))}}] + (doall + (for [[which f] [[:start start] [:end end]]] + ^{:key which} + [:div {:class (str "convert-handle " (name which)) + :style {:left (pct f)} + :title (str (name which) " · frame " f) + :on-pointer-down (grab which) + :on-pointer-move move! + :on-pointer-up #(reset! held nil) + :on-pointer-cancel #(reset! held nil)}]))]))) + +(defn view [] + (r/with-let [player (atom nil)] + (when-let [{:keys [label frames fps video range name] :as request} + @(rf/subscribe [::sub/convert])] + (let [{:keys [loading? status]} @(rf/subscribe [::playback/footage]) + [start end] range + n (- end start)] + [:div.convert-scrim + [:div.convert + [:div.pane-head (str "make a symbol from " label)] + [:div.convert-body + [:video.convert-video + {:ref #(reset! player %) + :src video :controls true :muted false :preload "auto" + :plays-inline true}] + [range-bar request player] + [:div.row.dim + (str "frames " start " … " end " · " n " frames · " + (.toFixed (/ n fps) 1) "s of " frames)] + [:label.convert-name + [:span.dim "name"] + [:input {:type "text" :value name :disabled loading? + :auto-focus true + :on-change #(rf/dispatch [::footage/convert-set :name + (.. % -target -value)]) + :on-key-down #(when (= "Enter" (.-key %)) + (rf/dispatch [::footage/convert]))}]] + (when loading? [:div.dim status])] + [:div.convert-actions + [:button {:disabled loading? + :on-click #(rf/dispatch [::footage/convert-cancel])} "cancel"] + [:button.on {:disabled (or loading? (< n 1) (empty? name)) + :on-click #(rf/dispatch [::footage/convert])} + (if loading? "working…" "make symbol")]]]])))) diff --git a/frontend/src/arthur/ui/drag.cljs b/frontend/src/arthur/ui/drag.cljs index b1d2660..8079fc1 100644 --- a/frontend/src/arthur/ui/drag.cljs +++ b/frontend/src/arthur/ui/drag.cljs @@ -11,6 +11,7 @@ pointer — is `[:ui :drop]`, set by `::ui/drop-hover`." (:require [arthur.domain.clip :as clip] [arthur.domain.palette :as pal] + [arthur.events.footage :as footage] [arthur.events.ui :as ui] [arthur.footage.store :as store] [re-frame.core :as rf])) @@ -91,8 +92,11 @@ "Drop what is being carried at `frame` of the open symbol, with its instance at `pos`." [frame pos] - (when-let [{:keys [kind sid]} (when (accepts?) @carrying)] + (when-let [{:keys [kind sid] :as c} (when (accepts?) @carrying)] (case kind - :symbol (rf/dispatch [::ui/drop-symbol sid frame pos]) + :symbol (rf/dispatch [::ui/drop-symbol sid frame pos]) + ;; Video is asked about before anything happens: which frames, and what + ;; the symbol they become is called. + :footage (rf/dispatch [::footage/ask-convert c frame pos]) nil)) (done!)) diff --git a/frontend/src/arthur/ui/player.cljs b/frontend/src/arthur/ui/player.cljs index dfadf37..63956c3 100644 --- a/frontend/src/arthur/ui/player.cljs +++ b/frontend/src/arthur/ui/player.cljs @@ -184,7 +184,17 @@ ;; painted when it was not is how the canvas stays empty forever. (when (and ready? f (not= f (:last @state))) (swap! state assoc :last f) - (paint! f) + ;; A frame that cannot be drawn is REPORTED and the clock carries on. Let + ;; it throw and the transport freezes with it — the playhead stops, the + ;; stage stays on whatever it last showed, and the one line saying why is + ;; buried under the next sixty identical ones. Reported once per message. + (try + (paint! f) + (catch :default error + (when (not= (ex-message error) (:failed @state)) + (swap! state assoc :failed (ex-message error)) + (js/console.error "arthur: frame" f "could not be drawn:" error) + (rf/dispatch [::pb/paint-failed (ex-message error)])))) (when live? (meter! f)) (when live? (rf/dispatch [::pb/tick f]))) ;; The audio ending is the authority on playback having stopped; nothing @@ -193,8 +203,9 @@ (rf/dispatch [::pb/pause])))) (defn- frame-loop [] - (tick!) - (swap! state assoc :raf (js/requestAnimationFrame frame-loop))) + ;; Scheduled BEFORE the tick, so nothing the tick does can end the loop. + (swap! state assoc :raf (js/requestAnimationFrame frame-loop)) + (tick!)) (defn start! [] (refresh-subs!) diff --git a/frontend/src/arthur/ui/pool.cljs b/frontend/src/arthur/ui/pool.cljs index eb03cdc..d792856 100644 --- a/frontend/src/arthur/ui/pool.cljs +++ b/frontend/src/arthur/ui/pool.cljs @@ -68,7 +68,7 @@ :style {:width 40 :height 30 :max-width 40 :max-height 30}}] [:span.thumb])) -(defn- footage-row [{:keys [id label frames fps] :as f} chosen] +(defn- footage-row [{:keys [id label frames fps video] :as f} chosen] [item (merge {:label label :sub (str frames "f @ " fps) :thumb [thumbnail f] @@ -76,7 +76,7 @@ :on-click #(rf/dispatch [::footage/choose id])} (carrying (str "footage:" id) #(drag/other! {:kind :footage :id id :label label - :frames frames :fps fps})))]) + :frames frames :fps fps :video video})))]) (defn- folder [title & children] (into [:details.pool-folder {:open true} [:summary title]] children)) diff --git a/frontend/src/arthur/ui/shell.cljs b/frontend/src/arthur/ui/shell.cljs index fe176d0..49b69a9 100644 --- a/frontend/src/arthur/ui/shell.cljs +++ b/frontend/src/arthur/ui/shell.cljs @@ -6,6 +6,7 @@ the canvas by `ui/player`'s loop rather than by anything here re-rendering, which is why scrubbing at speed does not touch React at all." (:require [arthur.clock :as clock] + [arthur.ui.convert :as convert] [arthur.events.playback :as pb] [arthur.subs.playback :as playback] [arthur.ui.palette :as palette] @@ -39,4 +40,5 @@ [stage/view]] [params/view] [timeline/view] + [convert/view] [audio]]) diff --git a/frontend/test/arthur/domain/instance_test.cljs b/frontend/test/arthur/domain/instance_test.cljs index 791c07c..7c3914c 100644 --- a/frontend/test/arthur/domain/instance_test.cljs +++ b/frontend/test/arthur/domain/instance_test.cljs @@ -270,3 +270,34 @@ (is (= [:outer 12] (clip/frame-inside c :outer [] 12))) (is (= [:inner 7] (clip/frame-inside c :outer [id] 12)) "the instance starts at 5, so frame 12 outside is frame 7 inside"))) + +(deftest adopting-symbols-renames-what-collides + (let [here (nested) + there (-> (clip/blank) + (assoc-in [:symbols :inner] {:id :inner :frames 4 :nodes {}}) + (clip/place-symbol :main :inner 0 #uuid "00000000-0000-4000-8000-0000000000aa" [0 0])) + {:keys [clip ids]} (clip/adopt here there [:main] {:main :take})] + (is (= {:main :take :inner :inner-2} ids) + "the root gets the name asked for; a taken id gets the next free one") + (is (= 10 (clip/frames clip :inner)) "what was already here is untouched") + (is (= :inner-2 (:of (first (vals (get-in clip [:symbols :take :nodes]))))) + "and the copy's instance follows its renamed symbol") + (is (empty? (clip/problems clip))))) + +(deftest a-placed-symbols-sound-is-heard-where-it-is-placed + (let [voice {:id :v :kind :audio :source {:footage "f"} :z "a1" + :span [10 40] :time {:mode :map :at 0 :rate 1} + :channels {[:audio :gain] (ch/keyed {0 0.0 5 1.0})}} + c (-> (clip/blank) + (assoc-in [:symbols :talk] {:id :talk :frames 30 :nodes {:v voice}}) + (clip/place-symbol :main :talk 50 #uuid "00000000-0000-4000-8000-0000000000bb" [0 0])) + [t] (clip/audio-tracks c :main)] + (is (= [10 40] (:span t)) "the same frames of the source") + (is (= [50 80] (node/placed-span t)) "starting where the instance starts") + (is (= #{50 55} (set (keys (get-in t [:channels [:audio :gain] :keys])))) + "with its automation moved along") + (testing "and cut off where the instance's own span ends" + (let [c (assoc-in c [:symbols :main :nodes #uuid "00000000-0000-4000-8000-0000000000bb" :span] [0 12]) + [t] (clip/audio-tracks c :main)] + (is (= [10 22] (:span t))) + (is (= [50 62] (node/placed-span t))))))) diff --git a/static/arthur/app.css b/static/arthur/app.css index d996758..83a6def 100644 --- a/static/arthur/app.css +++ b/static/arthur/app.css @@ -654,3 +654,74 @@ input[type="range"] { width: 100%; accent-color: var(--sel); } } .tl-empty { padding: 9px; color: var(--dim); } + +/* -------------------------------------------------------------------------- + the video -> symbol dialog */ + +.convert-scrim { + position: fixed; + inset: 0; + z-index: 20; + display: grid; + place-items: center; + background: rgba(0, 0, 0, .35); +} + +.convert { + width: min(560px, calc(100vw - 32px)); + background: var(--pane); + border: 1px solid var(--line); + border-radius: 3px; + box-shadow: 0 6px 24px rgba(0, 0, 0, .3); +} + +.convert-body { display: grid; gap: 8px; padding: 10px; } + +.convert-video { + width: 100%; + max-height: 46vh; + background: var(--stage); + border-radius: 2px; +} + +/* The range: the kept frames lit, a handle at each end. */ +.convert-range { + position: relative; + height: 18px; + margin: 0 7px; + background: var(--sunk); + border: 1px solid var(--hair); + border-radius: 2px; + touch-action: none; +} + +.convert-kept { + position: absolute; + top: 0; + bottom: 0; + background: var(--sel-bg); + border-left: 1px solid var(--sel); + border-right: 1px solid var(--sel); +} + +.convert-handle { + position: absolute; + top: -3px; + bottom: -3px; + width: 10px; + margin-left: -5px; + background: var(--sel); + border-radius: 2px; + cursor: ew-resize; +} + +.convert-name { display: flex; align-items: center; gap: 8px; } +.convert-name input { flex: 1; } + +.convert-actions { + display: flex; + justify-content: flex-end; + gap: 6px; + padding: 8px 10px; + border-top: 1px solid var(--hair); +}