diff --git a/frontend/src/arthur/audio/mix.cljs b/frontend/src/arthur/audio/mix.cljs index 29d4809..9706eda 100644 --- a/frontend/src/arthur/audio/mix.cljs +++ b/frontend/src/arthur/audio/mix.cljs @@ -91,10 +91,10 @@ (.setValueAtTime param (* factor v) (/ f fps))))))) (defn tracks-of - "The audio nodes of one of the clip's symbols. Any symbol may carry its own - sound, and playback mixes the open one's." + "The sounds symbol `sid` plays, including those inside what it places — see + `clip/audio-tracks`. Playback mixes the open symbol's." [document sid] - (filter #(= :audio (:kind %)) (vals (:nodes (clip/symbol document sid))))) + (clip/audio-tracks document sid)) (defn- render! [document sid sources store] (let [fps (:fps document) diff --git a/frontend/src/arthur/domain/clip.cljs b/frontend/src/arthur/domain/clip.cljs index f5d6a98..84c2d0e 100644 --- a/frontend/src/arthur/domain/clip.cljs +++ b/frontend/src/arthur/domain/clip.cljs @@ -37,7 +37,8 @@ [arthur.domain.node :as node] [arthur.domain.palette :as pal] [arthur.domain.pose :as pose] - [arthur.domain.symbol :as symbol])) + [arthur.domain.symbol :as symbol] + [clojure.string :as string])) (def clip-keys "Every top-level field of a clip, and the reason `arthur.domain.leaf` refuses @@ -174,6 +175,127 @@ (assoc-in [:symbols sid] {:id sid :name (name sid) :frames (- end frame) :nodes {}}) (place-symbol host sid frame uuid [0 0]))))) +(defn- free-id + "`wanted`, or the first `wanted-2`, `wanted-3`… `taken?` does not claim. + Keeps the namespace, so `:sym/face` becomes `:sym/face-2`." + [taken? wanted] + (first (remove taken? + (cons wanted + (map #(keyword (namespace wanted) (str (name wanted) "-" %)) + (iterate inc 2)))))) + +(defn adopt + "Copy symbols `roots` of clip `other`, and every symbol they place, into + `clip`. Returns `{:clip :ids}`, where `:ids` maps each copied symbol's id in + `other` to its id here. + + AN ID THAT IS TAKEN IS RENAMED, never merged: two symbols that happen to share + an id are two drawings, and an instance's `:of` inside the copy is rewritten to + follow. `wanted` maps a root's id in `other` to the id it should preferably get, + which is how a symbol made from footage is called what the person typed rather + than `:main`. + + Only symbols travel. What else `other` holds — tracking identities, an analysis + — is the caller's decision, because whether it can come too depends on what + `clip` already has." + [clip other roots wanted] + (let [;; A tree walk is safe because placing cannot make a cycle. + reach (into #{} (mapcat #(tree-seq any? (partial places other) %)) roots) + ids (reduce (fn [ids sid] + (let [taken? #(or (contains? (:symbols clip) %) + (some #{%} (vals ids)))] + (assoc ids sid (free-id taken? (get wanted sid sid))))) + {} (sort-by str reach)) + copy (fn [sid] + (-> (symbol other sid) + (assoc :id (ids sid)) + (update :nodes #(into {} (map (fn [[id n]] + [id (cond-> n (:of n) (update :of ids))])) + %))))] + {:clip (reduce (fn [c sid] (assoc-in c [:symbols (ids sid)] (copy sid))) clip reach) + :ids ids})) + +(defn symbol-id + "An id for a symbol a person has named: the name, lower-cased and hyphenated, + or `:symbol` when nothing of it survives." + [label] + (let [slug (-> (str label) string/lower-case + (string/replace #"[^a-z0-9]+" "-") + (string/replace #"^-+|-+$" ""))] + (keyword (if (seq slug) slug "symbol")))) + +(defn add-take + "Put a frozen take into `clip` as ONE symbol called `label`. Returns + `{:clip :sid :tracked?}`. + + `take` is what `flow/freeze/clip` makes: a `:main` that places one symbol per + tracked face. `:main` becomes the named symbol — it is what holds the faces in + stage pixels, so it is the thing worth placing — and it gets the take's SOUND + as an audio node of its own, source frames `range` of footage `footage-id`, so + wherever the symbol is placed it is heard. + + The tracking identities, and the analysis they were measured by, come along + only when `clip` has no analysis of its own and no face had to be renamed. A + document holds one analysis, and regeneration finds a face's symbol by its + subject id, so either condition failing means the take comes in as drawings + that play but cannot be re-tuned — `:tracked? false` says so." + [clip take label footage-id range] + (let [{c :clip ids :ids} (adopt clip take [:main] {:main (symbol-id label)}) + sid (ids :main) + tracked? (and (nil? (:analysis clip)) + (every? #(= % (ids %)) (keys (:subjects take))))] + {:sid sid + :tracked? tracked? + :clip (cond-> (-> c + (assoc-in [:symbols sid :name] (str label)) + (assoc-in [:symbols sid :nodes :sound] + {:id :sound :name "sound" :kind :audio :parent nil + :z "z-sound" :source {:footage footage-id} + :span range :time {:mode :map :at 0 :rate 1}})) + tracked? (-> (assoc :analysis (:analysis take)) + (update :subjects merge (:subjects take)) + (update :features merge (:features take)) + (update :groups merge (:groups take))))})) + +(defn audio-tracks + "Every sound symbol `sid` plays, as audio nodes in `sid`'s own frames: its own + and, recursively, those inside the instances it places. + + A sound inside a placed symbol is heard where the instance puts it, so each one + is carried OUT through the instance's time map — `:at` and `:rate`, and clipped + to the instance's own span — until it is in the frames of the symbol being + played. Keyed automation moves with it. What comes back is what a mixer that + only knows flat tracks can play as it is." + [clip sid] + (let [sym (symbol clip sid)] + (into (vec (filter #(= :audio (:kind %)) (vals (:nodes sym)))) + (mapcat + (fn [inst] + (let [{:keys [mode at rate] :or {at 0 rate 1}} (:time inst) + inner (symbol clip (:of inst)) + [in out] (or (:span inst) [0 (:frames inner)]) + in (if (= mode :map) in 0) + ->outer (fn [x] (+ at (/ (- x in) rate)))] + (keep (fn [a] + (let [{a-at :at a-rate :rate :or {a-at 0 a-rate 1}} (:time a) + [s0 s1] (or (:span a) [0 (:frames inner)]) + x0 (max a-at in) + x1 (min (+ a-at (/ (- s1 s0) a-rate)) out)] + (when (< x0 x1) + (-> a + (assoc :span [(+ s0 (* a-rate (- x0 a-at))) + (+ s0 (* a-rate (- x1 a-at)))] + :time {:mode :map :at (->outer x0) + :rate (* a-rate rate)}) + (update :channels + (fn [chs] + (into {} (map (fn [[p ch]] + [p (cond-> ch (:keys ch) + (update :keys #(into {} (map (fn [[f v]] [(->outer f) v])) %)))])) + chs))))))) + (audio-tracks clip (:of inst))))) + (filter #(= :instance (:kind %)) (vals (:nodes sym))))))) + (defn frame-inside "Carry frame `f` of symbol `sid` down through the nodes named by `path`, one per level, the way a timeline row's path names them. Returns `[symbol frame]`: diff --git a/frontend/src/arthur/events/edit.cljs b/frontend/src/arthur/events/edit.cljs index 42ce07d..f1c065a 100644 --- a/frontend/src/arthur/events/edit.cljs +++ b/frontend/src/arthur/events/edit.cljs @@ -14,13 +14,20 @@ were the only thing anyone could edit. They are not." (:require [arthur.footage.store :as store])) -(defn edit - "Apply `f` to the loaded clip and return the new db." +(defn edit-entry + "Apply `f` to the loaded ENTRY — the document and the blocks, footage and + source tracks beside it — and return the new db. For an edit that brings tier-2 + data in with it, which a document edit alone cannot." [db f] - (let [id (store/edit-clip! (:clip/current db) f)] + (let [id (store/edit-entry! (:clip/current db) f)] (if id (-> db (assoc :clip/current id) (update :paint/revision (fnil inc 0)) (update :project merge {:status "edited · unsaved"})) db))) + +(defn edit + "Apply `f` to the loaded clip and return the new db." + [db f] + (edit-entry db #(update % :clip f))) diff --git a/frontend/src/arthur/events/footage.cljs b/frontend/src/arthur/events/footage.cljs index ca54b58..22f05cf 100644 --- a/frontend/src/arthur/events/footage.cljs +++ b/frontend/src/arthur/events/footage.cljs @@ -5,6 +5,7 @@ detector's identity comes from the server too, because it goes into the content address of every block this produces." (:require [arthur.domain.clip :as clip] + [arthur.events.edit :as edit] [arthur.events.playback :as pb] [arthur.flow.detect :as detect] [arthur.flow.ingest :as ingest] @@ -14,6 +15,7 @@ [arthur.footage.store :as store] [arthur.domain.landmarks :as lm] [arthur.fx.http :as http] + [clojure.string :as string] [re-frame.core :as rf])) (defonce ^:private clock (atom 0)) @@ -44,8 +46,12 @@ The work happens inside `decode!`'s callback, and the promise it returns is the backpressure: the decoder does not run ahead of the detector, so a 900-frame - take does not hold 900 decoded frames at 1440x1920 in memory." - [manifest model] + take does not hold 900 decoded frames at 1440x1920 in memory. + + Only source frames `[start end)` are measured. Decoding still begins at frame + 0, because every frame after the first is coded against the ones before it, + and it stops at `end`; frames before `start` are decoded and dropped unread." + [manifest model [start end]] (let [[w h] [(:width manifest) (:height manifest)] canvas (.createElement js/document "canvas") ctx (.getContext canvas "2d" #js {:willReadFrequently true}) @@ -53,45 +59,48 @@ raw (atom []) crops (atom []) inner (atom []) - total (:frames manifest)] + total end] (set! (.-width canvas) w) (set! (.-height canvas) h) (rf/dispatch [::progress "loading the video…"]) - (-> (ingest/stream! (ingest/stream-url manifest) total) + (-> (ingest/stream! (ingest/stream-url manifest) (:frames manifest)) (.then (fn [stream] (ingest/decode! - stream fps w h + (update stream :units subvec 0 end) fps w h (fn [i frame] - (.drawImage ctx frame 0 0) - ;; EVERY FACE ON THIS FRAME, each with its own mouth crop taken - ;; while the frame's pixels are still on the canvas. Which of these - ;; detections belongs to which subject is not decided here — the - ;; answer needs the whole take — so all three vectors stay in - ;; DETECTION ORDER and `detect/tracks` re-keys them afterwards. - (let [faces (detect/detect! model canvas (ingest/frame-ms fps i)) - boxes (mapv (fn [face] - (interior/crop (mapv #(nth face %) lm/LIPS-INNER) - [w h])) - faces) - frame-crops (mapv (fn [box] - (when box - {:box box - :data (.-data (.getImageData - ctx (:x box) (:y box) - (:w box) (:h box)))})) - boxes)] - (swap! raw conj faces) - (swap! crops conj frame-crops) - ;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work - ;; either way, but done after the fact it is ten seconds of - ;; synchronous arithmetic with the main thread held and the - ;; frame counter frozen on its last value — which reads as the - ;; decoder hanging, and was diagnosed as that twice. - (swap! inner conj (mapv #(source/measure-crop take/knobs %) - frame-crops))) + (when (>= i start) + (.drawImage ctx frame 0 0) + ;; EVERY FACE ON THIS FRAME, each with its own mouth crop taken + ;; while the frame's pixels are still on the canvas. Which of + ;; these detections belongs to which subject is not decided here + ;; — the answer needs the whole take — so all three vectors stay + ;; in DETECTION ORDER and `detect/tracks` re-keys them afterwards. + (let [faces (detect/detect! model canvas (ingest/frame-ms fps i)) + boxes (mapv (fn [face] + (interior/crop (mapv #(nth face %) lm/LIPS-INNER) + [w h])) + faces) + frame-crops (mapv (fn [box] + (when box + {:box box + :data (.-data (.getImageData + ctx (:x box) (:y box) + (:w box) (:h box)))})) + boxes)] + (swap! raw conj faces) + (swap! crops conj frame-crops) + ;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work + ;; either way, but done after the fact it is ten seconds of + ;; synchronous arithmetic with the main thread held and the + ;; frame counter frozen on its last value — which reads as the + ;; decoder hanging, and was diagnosed as that twice. + (swap! inner conj (mapv #(source/measure-crop take/knobs %) + frame-crops)))) (when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total)) - (rf/dispatch [::progress (str "detecting " (inc i) "/" total)])) + (rf/dispatch [::progress (if (< i start) + (str "seeking " (inc i) "/" start) + (str "detecting " (- (inc i) start) "/" (- end start)))])) ;; Yield, so the status and the transport paint between synchronous ;; MediaPipe calls. `decode!` waits on this before feeding more. (js/Promise. (fn [done] (js/setTimeout done 0))))))) @@ -251,38 +260,44 @@ (js/Promise.resolve track) (:subjects track))) +(defn- analyse! + "Promise of source frames `[start end)` of footage `footage-id`, detected, + measured and frozen — `{:clip :store ...}` as `build-clip` makes it, with the + footage reading as though it were only those frames. A saved analysis of + exactly that range is reused instead of detecting again." + [footage-id [start end]] + (-> (js/Promise.all #js [(ingest/manifest! footage-id) (ingest/detector!)]) + (.then (fn [[full detector]] + (let [manifest (ingest/slice full start end)] + (reset! clock (js/Date.now)) + (rf/dispatch [::progress "looking for saved analysis…"]) + (-> (cached-source! manifest detector) + (.then (fn [track] + (if track + (do (rf/dispatch [::progress "reusing saved analysis…"]) + (-> (measure-crops! + take/knobs track + (fn [done total] + (when (or (= 1 done) (zero? (mod done 4)) + (= done total)) + (rf/dispatch + [::progress (str "measuring " done "/" total)])))) + (.then #(build-clip manifest detector %)))) + (do (rf/dispatch [::progress "loading MediaPipe…"]) + (-> (detect/landmarker!) + (.then (fn [model] + (mark! "MediaPipe ready") + (rf/dispatch [::progress "opening the video…"]) + (detect-frames! full model [start end]))) + (.then #(build-clip manifest detector %))))))))))))) + (rf/reg-fx - ::begin! - (fn [footage-id] - (-> (js/Promise.all #js [(ingest/manifest! footage-id) (ingest/detector!)]) - (.then (fn [[manifest detector]] - (reset! clock (js/Date.now)) - (rf/dispatch [::progress "looking for saved analysis…"]) - (-> (cached-source! manifest detector) - (.then (fn [track] - (if track - (do (rf/dispatch [::progress "reusing saved analysis…"]) - (-> (measure-crops! - take/knobs track - (fn [done total] - (when (or (= 1 done) (zero? (mod done 4)) - (= done total)) - (rf/dispatch - [::progress (str "measuring " done "/" total)])))) - (.then (fn [measured] - (build-clip manifest detector measured))))) - (do (rf/dispatch [::progress "loading MediaPipe…"]) - (-> (detect/landmarker!) - (.then (fn [model] - (mark! "MediaPipe ready") - (rf/dispatch [::progress "opening the video…"]) - (-> (detect-frames! manifest model) - (.then (fn [fresh] - (build-clip manifest detector fresh)))))))))))))) - (.then (fn [entry] + ::convert! + (fn [{:keys [footage-id range] :as request}] + (-> (analyse! footage-id range) + (.then (fn [built] (mark! "build-clip: done") - (let [id (store/install! entry)] - (rf/dispatch [::loaded id (:summary entry)])))) + (rf/dispatch [::converted request built]))) (.catch (fn [error] (js/console.error error) ;; A run that ended badly may have ended on a MediaPipe graph @@ -361,19 +376,6 @@ ::choose (fn [db [_ id]] (assoc-in db [:footage :chosen] id))) -(rf/reg-event-fx - ::load - (fn [{:keys [db]} _] - (let [chosen (get-in db [:footage :chosen])] - (cond - (get-in db [:footage :loading?]) {} - (nil? chosen) - {:db (assoc-in db [:footage :status] "upload a video to begin")} - :else - {:db (update db :footage merge {:loading? true :status "reading the manifest…"}) - ::pb/pause! nil - ::begin! chosen})))) - (rf/reg-event-db ::progress (fn [db [_ message]] (assoc-in db [:footage :status] message))) @@ -384,11 +386,68 @@ (assoc db :footage (assoc (:footage db) :loading? false :status (str "footage failed: " message))))) + +;; --------------------------------------------------------------------------- +;; footage -> a symbol +;; +;; Dropping a video asks first. `[:ui :convert]` is the question — which footage, +;; which of its frames, what to call the result — and where the answer will be +;; placed: `:host`, `:frame` and `:pos` are the drop's, captured when it happened +;; so that switching tabs while detection runs does not move where it lands. + +(rf/reg-event-db + ::ask-convert + (fn [db [_ {:keys [frames label] :as footage} frame pos]] + (assoc-in db [:ui :convert] + (merge (select-keys footage [:id :label :frames :fps :video]) + {:range [0 frames] + :name (string/replace (str label) #"\.[^.]*$" "") + :host (get-in db [:ui :open]) :frame frame :pos pos})))) + +(rf/reg-event-db + ::convert-set + (fn [db [_ k v]] (assoc-in db [:ui :convert k] v))) + +(rf/reg-event-db + ::convert-cancel + (fn [db _] + (if (get-in db [:footage :loading?]) db (update db :ui dissoc :convert)))) + (rf/reg-event-fx - ::loaded - (fn [{:keys [db]} [_ id summary]] - (let [clip (store/entry id)] - {:db (-> (pb/show db id) - (assoc :footage (assoc (:footage db) :id id :label (:label clip) - :loading? false :status summary))) - ::pb/pause! nil}))) + ::convert + (fn [{:keys [db]} _] + (let [{:keys [id range] :as request} (get-in db [:ui :convert])] + (if (or (nil? request) (get-in db [:footage :loading?])) + {} + {:db (update db :footage merge {:loading? true :status "starting…"}) + ::pb/pause! nil + ::convert! {:footage-id id :range range :request request}})))) + +(rf/reg-event-fx + ::converted + (fn [{:keys [db]} [_ {{:keys [name host frame pos range]} :request footage-id :footage-id} + built]] + (let [uuid (random-uuid) + fps (get-in db [:clip :fps]) + {:keys [clip sid tracked?]} + (clip/add-take (:clip (store/entry (:clip/current db))) (:clip built) + name footage-id range) + db (edit/edit-entry + db + #(cond-> (-> % + (assoc :clip (clip/place-symbol clip host sid frame uuid pos)) + (update :store merge (:store built))) + tracked? (merge (select-keys built [:footage-id :source-blocks + :source-inputs]))))] + {:db (-> db + (update :ui dissoc :convert) + (assoc-in [:ui :selection] [:node host uuid [uuid]]) + (update :footage merge + {:loading? false + :status (str "made " name " · " (- (second range) (first range)) " frames" + (when-not tracked? + " · as drawings: this project already tracks other footage") + (when (not= fps (get-in built [:clip :fps])) + (str " · footage is " (get-in built [:clip :fps]) + " fps, this project " fps)))})) + :dispatch [::pb/refresh-clock]}))) diff --git a/frontend/src/arthur/events/playback.cljs b/frontend/src/arthur/events/playback.cljs index 9cc14b1..d0cbdce 100644 --- a/frontend/src/arthur/events/playback.cljs +++ b/frontend/src/arthur/events/playback.cljs @@ -168,6 +168,17 @@ {:db (assoc-in db [:clip :audio] url)}) {}))) +(rf/reg-event-db + ::paint-failed + (fn [db [_ message]] + (update db :project merge {:status (str "cannot draw: " message)}))) + +(rf/reg-event-fx + ::refresh-clock + ;; After an edit that changes what the open symbol sounds like. + (fn [{:keys [db]} _] + {::clock! {:id (:clip/current db) :sid (get-in db [:ui :open])}})) + (rf/reg-event-fx ::open-symbol (fn [{:keys [db]} [_ sid]] diff --git a/frontend/src/arthur/flow/address.cljs b/frontend/src/arthur/flow/address.cljs index 6cd5b53..1e6146e 100644 --- a/frontend/src/arthur/flow/address.cljs +++ b/frontend/src/arthur/flow/address.cljs @@ -67,8 +67,15 @@ over the same frames and the same model they disagree by up to 0.013 of frame width, which is a visible difference on a mouth. Optional, because the synthetic take has no running mode to declare and an absent field is how the other - optional inputs already say \"not applicable\"." - [{:keys [detector version source footage frames fps aspect seed mode tracking]}] + optional inputs already say \"not applicable\". + + `:range` is `[first end)` source frames when only part of the footage was + analysed, and absent for all of it — so every analysis made before ranges + existed keeps its address. It is in here because a partial analysis is a + different artifact: without it, detecting frames 40–90 would be saved under the + same key as the whole take, and the next conversion of the whole take would be + handed fifty frames." + [{:keys [detector version source footage frames fps aspect seed mode tracking range]}] (when-not (and (string? detector) (seq detector) (string? version) (seq version)) (throw (ex-info "an analysis names its detector and the detector's VERSION: an upgrade that silently reuses old landmarks is the failure content addressing exists to prevent" {:detector detector :version version}))) @@ -82,7 +89,8 @@ footage (assoc :footage footage) seed (assoc :seed seed) mode (assoc :mode mode) - tracking (assoc :tracking tracking)))) + tracking (assoc :tracking tracking) + range (assoc :range range)))) (defn analysis "An analysis record with its `:id` filled in. The record is tier 1 — it says diff --git a/frontend/src/arthur/flow/ingest.cljs b/frontend/src/arthur/flow/ingest.cljs index 07a7170..bbc32c9 100644 --- a/frontend/src/arthur/flow/ingest.cljs +++ b/frontend/src/arthur/flow/ingest.cljs @@ -86,6 +86,20 @@ (-> (http/GET "/api/detector") (.then (fn [json] (js->clj json :keywordize-keys true))))) +(defn slice + "The manifest of source frames `[start end)` of this footage, as though that + were all of it: its frame count, its tracing stills and its absence tracks cut + to the range, and `:range` saying where it came from — which goes into the + analysis address, see `flow/address`. The whole footage is returned unchanged, + so a full-length conversion reuses analyses made before ranges existed." + [m start end] + (if (= [start end] [0 (:frames m)]) + m + (-> m + (assoc :frames (- end start) :range [start end]) + (update :urls #(some-> % vec (subvec start end))) + (update :presence #(into {} (map (fn [[id track]] [id (subvec (vec track) start end)])) %))))) + (defn audio-url [manifest] (:audio manifest)) diff --git a/frontend/src/arthur/flow/take.cljs b/frontend/src/arthur/flow/take.cljs index a8530e8..d04dd1f 100644 --- a/frontend/src/arthur/flow/take.cljs +++ b/frontend/src/arthur/flow/take.cljs @@ -19,16 +19,17 @@ (defn analysis-for [manifest detector] (let [aspect (/ (:width manifest) (:height manifest))] (address/analysis - (merge {:detector "mediapipe" :version "unknown"} - detector - {:source (:source manifest) - :footage (:footage manifest) - :frames (:frames manifest) - :fps (:fps manifest) - :aspect aspect - ;; How the detector was run, not just which one it was. See - ;; `flow/address/analysis-descriptor`. - :mode "video" :tracking detect/settings})))) + (cond-> (merge {:detector "mediapipe" :version "unknown"} + detector + {:source (:source manifest) + :footage (:footage manifest) + :frames (:frames manifest) + :fps (:fps manifest) + :aspect aspect + ;; How the detector was run, not just which one it was. See + ;; `flow/address/analysis-descriptor`. + :mode "video" :tracking detect/settings}) + (:range manifest) (assoc :range (:range manifest)))))) (defn measure "Condition the anchor before measuring rings through it." diff --git a/frontend/src/arthur/footage/store.cljs b/frontend/src/arthur/footage/store.cljs index aafc1a2..3e620a4 100644 --- a/frontend/src/arthur/footage/store.cljs +++ b/frontend/src/arthur/footage/store.cljs @@ -32,12 +32,17 @@ @loaded (db/clip-entry id))) -(defn edit-clip! - "Edit the loaded document. Built-in clips are copied into the runtime store - on first edit, so their delayed source values remain reusable." +(defn edit-entry! + "Edit the loaded entry — the document and what travels with it: its blocks, + its footage, its retained source tracks. Built-in clips are copied into the + runtime store on first edit, so their delayed source values remain reusable." [id f] (if (= id (:id @loaded)) - (do (swap! loaded update :clip f) id) - (let [entry (entry id)] - (when entry - (install! (update entry :clip f) "paint"))))) + (do (swap! loaded f) id) + (when-let [entry (entry id)] + (install! (f entry) "paint")))) + +(defn edit-clip! + "Edit the loaded document and nothing beside it." + [id f] + (edit-entry! id #(update % :clip f))) diff --git a/frontend/src/arthur/subs/render.cljs b/frontend/src/arthur/subs/render.cljs index c0b1e1b..49b7032 100644 --- a/frontend/src/arthur/subs/render.cljs +++ b/frontend/src/arthur/subs/render.cljs @@ -69,10 +69,15 @@ (rf/reg-sub ::store :<- [::clip-id] - (fn [id _] + :<- [::paint-revision] + (fn [[id _] _] ;; Tier 2, behind a handle, and never in app-db itself — what is in the db is - ;; the id of the clip whose blocks these are. The hand-written demo has none; - ;; the swarm is entirely dense. + ;; the id of the clip whose blocks these are. + ;; + ;; ON THE REVISION AS WELL AS THE ID, like `::clip`. An edit can bring blocks + ;; in with it — a video made into a symbol does — without the id changing, and + ;; a store read only when the id changed is the old one: the document names the + ;; new blocks, the resolver cannot find them, and the frame throws. (:store (footage/entry id)))) (rf/reg-sub diff --git a/frontend/src/arthur/subs/ui.cljs b/frontend/src/arthur/subs/ui.cljs index c9c3a08..ab9258c 100644 --- a/frontend/src/arthur/subs/ui.cljs +++ b/frontend/src/arthur/subs/ui.cljs @@ -13,6 +13,7 @@ (rf/reg-sub ::tone (fn [db _] (get-in db [:ui :tone]))) (rf/reg-sub ::tool (fn [db _] (get-in db [:ui :tool]))) (rf/reg-sub ::draft (fn [db _] (get-in db [:ui :draft]))) +(rf/reg-sub ::convert (fn [db _] (get-in db [:ui :convert]))) (rf/reg-sub ::drop (fn [db _] (get-in db [:ui :drop]))) (rf/reg-sub ::tabs (fn [db _] (get-in db [:ui :tabs]))) (rf/reg-sub ::expanded (fn [db _] (get-in db [:ui :expanded]))) diff --git a/frontend/src/arthur/ui/convert.cljs b/frontend/src/arthur/ui/convert.cljs new file mode 100644 index 0000000..1a12ee5 --- /dev/null +++ b/frontend/src/arthur/ui/convert.cljs @@ -0,0 +1,90 @@ +(ns arthur.ui.convert + "The question a dropped video asks: which of its frames become a symbol, and + what is that symbol called. + + The video plays in the dialog so the frames can be chosen by looking at them. + The two handles under it are the range, `[start end)` in source frames; moving + either seeks the video to the frame it is on, so the edge being chosen is the + picture being shown. Detection then runs on those frames only — see + `events/footage/analyse!` — and the result lands where the video was dropped." + (:require [arthur.events.footage :as footage] + [arthur.subs.playback :as playback] + [arthur.subs.ui :as sub] + [re-frame.core :as rf] + [reagent.core :as r])) + +(defn- frame-at [^js track x frames] + (let [box (.getBoundingClientRect track)] + (-> (/ (* (- x (.-left box)) frames) (.-width box)) + js/Math.round (max 0) (min frames)))) + +(defn- range-bar + "Two handles over the footage's length. `player` holds the element to seek." + [{:keys [frames fps range]} player] + (r/with-let [held (atom nil) + track (atom nil)] + (let [[start end] range + pct #(str (* 100 (/ % (max 1 frames))) "%") + move! (fn [^js e] + (when-let [which @held] + (let [f (frame-at @track (.-clientX e) frames) + [s' e'] (if (= :start which) + [(min f (dec end)) end] + [start (max f (inc start))])] + (rf/dispatch [::footage/convert-set :range [s' e']]) + (when-let [^js v @player] + (set! (.-currentTime v) + (/ (if (= :start which) s' (dec e')) fps)))))) + grab (fn [which] + (fn [^js e] + (.preventDefault e) + (reset! held which) + (try (.setPointerCapture (.-currentTarget e) (.-pointerId e)) + (catch :default _ nil))))] + [:div.convert-range {:ref #(reset! track %)} + [:div.convert-kept {:style {:left (pct start) :width (pct (- end start))}}] + (doall + (for [[which f] [[:start start] [:end end]]] + ^{:key which} + [:div {:class (str "convert-handle " (name which)) + :style {:left (pct f)} + :title (str (name which) " · frame " f) + :on-pointer-down (grab which) + :on-pointer-move move! + :on-pointer-up #(reset! held nil) + :on-pointer-cancel #(reset! held nil)}]))]))) + +(defn view [] + (r/with-let [player (atom nil)] + (when-let [{:keys [label frames fps video range name] :as request} + @(rf/subscribe [::sub/convert])] + (let [{:keys [loading? status]} @(rf/subscribe [::playback/footage]) + [start end] range + n (- end start)] + [:div.convert-scrim + [:div.convert + [:div.pane-head (str "make a symbol from " label)] + [:div.convert-body + [:video.convert-video + {:ref #(reset! player %) + :src video :controls true :muted false :preload "auto" + :plays-inline true}] + [range-bar request player] + [:div.row.dim + (str "frames " start " … " end " · " n " frames · " + (.toFixed (/ n fps) 1) "s of " frames)] + [:label.convert-name + [:span.dim "name"] + [:input {:type "text" :value name :disabled loading? + :auto-focus true + :on-change #(rf/dispatch [::footage/convert-set :name + (.. % -target -value)]) + :on-key-down #(when (= "Enter" (.-key %)) + (rf/dispatch [::footage/convert]))}]] + (when loading? [:div.dim status])] + [:div.convert-actions + [:button {:disabled loading? + :on-click #(rf/dispatch [::footage/convert-cancel])} "cancel"] + [:button.on {:disabled (or loading? (< n 1) (empty? name)) + :on-click #(rf/dispatch [::footage/convert])} + (if loading? "working…" "make symbol")]]]])))) diff --git a/frontend/src/arthur/ui/drag.cljs b/frontend/src/arthur/ui/drag.cljs index b1d2660..8079fc1 100644 --- a/frontend/src/arthur/ui/drag.cljs +++ b/frontend/src/arthur/ui/drag.cljs @@ -11,6 +11,7 @@ pointer — is `[:ui :drop]`, set by `::ui/drop-hover`." (:require [arthur.domain.clip :as clip] [arthur.domain.palette :as pal] + [arthur.events.footage :as footage] [arthur.events.ui :as ui] [arthur.footage.store :as store] [re-frame.core :as rf])) @@ -91,8 +92,11 @@ "Drop what is being carried at `frame` of the open symbol, with its instance at `pos`." [frame pos] - (when-let [{:keys [kind sid]} (when (accepts?) @carrying)] + (when-let [{:keys [kind sid] :as c} (when (accepts?) @carrying)] (case kind - :symbol (rf/dispatch [::ui/drop-symbol sid frame pos]) + :symbol (rf/dispatch [::ui/drop-symbol sid frame pos]) + ;; Video is asked about before anything happens: which frames, and what + ;; the symbol they become is called. + :footage (rf/dispatch [::footage/ask-convert c frame pos]) nil)) (done!)) diff --git a/frontend/src/arthur/ui/player.cljs b/frontend/src/arthur/ui/player.cljs index dfadf37..63956c3 100644 --- a/frontend/src/arthur/ui/player.cljs +++ b/frontend/src/arthur/ui/player.cljs @@ -184,7 +184,17 @@ ;; painted when it was not is how the canvas stays empty forever. (when (and ready? f (not= f (:last @state))) (swap! state assoc :last f) - (paint! f) + ;; A frame that cannot be drawn is REPORTED and the clock carries on. Let + ;; it throw and the transport freezes with it — the playhead stops, the + ;; stage stays on whatever it last showed, and the one line saying why is + ;; buried under the next sixty identical ones. Reported once per message. + (try + (paint! f) + (catch :default error + (when (not= (ex-message error) (:failed @state)) + (swap! state assoc :failed (ex-message error)) + (js/console.error "arthur: frame" f "could not be drawn:" error) + (rf/dispatch [::pb/paint-failed (ex-message error)])))) (when live? (meter! f)) (when live? (rf/dispatch [::pb/tick f]))) ;; The audio ending is the authority on playback having stopped; nothing @@ -193,8 +203,9 @@ (rf/dispatch [::pb/pause])))) (defn- frame-loop [] - (tick!) - (swap! state assoc :raf (js/requestAnimationFrame frame-loop))) + ;; Scheduled BEFORE the tick, so nothing the tick does can end the loop. + (swap! state assoc :raf (js/requestAnimationFrame frame-loop)) + (tick!)) (defn start! [] (refresh-subs!) diff --git a/frontend/src/arthur/ui/pool.cljs b/frontend/src/arthur/ui/pool.cljs index eb03cdc..d792856 100644 --- a/frontend/src/arthur/ui/pool.cljs +++ b/frontend/src/arthur/ui/pool.cljs @@ -68,7 +68,7 @@ :style {:width 40 :height 30 :max-width 40 :max-height 30}}] [:span.thumb])) -(defn- footage-row [{:keys [id label frames fps] :as f} chosen] +(defn- footage-row [{:keys [id label frames fps video] :as f} chosen] [item (merge {:label label :sub (str frames "f @ " fps) :thumb [thumbnail f] @@ -76,7 +76,7 @@ :on-click #(rf/dispatch [::footage/choose id])} (carrying (str "footage:" id) #(drag/other! {:kind :footage :id id :label label - :frames frames :fps fps})))]) + :frames frames :fps fps :video video})))]) (defn- folder [title & children] (into [:details.pool-folder {:open true} [:summary title]] children)) diff --git a/frontend/src/arthur/ui/shell.cljs b/frontend/src/arthur/ui/shell.cljs index fe176d0..49b69a9 100644 --- a/frontend/src/arthur/ui/shell.cljs +++ b/frontend/src/arthur/ui/shell.cljs @@ -6,6 +6,7 @@ the canvas by `ui/player`'s loop rather than by anything here re-rendering, which is why scrubbing at speed does not touch React at all." (:require [arthur.clock :as clock] + [arthur.ui.convert :as convert] [arthur.events.playback :as pb] [arthur.subs.playback :as playback] [arthur.ui.palette :as palette] @@ -39,4 +40,5 @@ [stage/view]] [params/view] [timeline/view] + [convert/view] [audio]]) diff --git a/frontend/test/arthur/domain/instance_test.cljs b/frontend/test/arthur/domain/instance_test.cljs index 791c07c..7c3914c 100644 --- a/frontend/test/arthur/domain/instance_test.cljs +++ b/frontend/test/arthur/domain/instance_test.cljs @@ -270,3 +270,34 @@ (is (= [:outer 12] (clip/frame-inside c :outer [] 12))) (is (= [:inner 7] (clip/frame-inside c :outer [id] 12)) "the instance starts at 5, so frame 12 outside is frame 7 inside"))) + +(deftest adopting-symbols-renames-what-collides + (let [here (nested) + there (-> (clip/blank) + (assoc-in [:symbols :inner] {:id :inner :frames 4 :nodes {}}) + (clip/place-symbol :main :inner 0 #uuid "00000000-0000-4000-8000-0000000000aa" [0 0])) + {:keys [clip ids]} (clip/adopt here there [:main] {:main :take})] + (is (= {:main :take :inner :inner-2} ids) + "the root gets the name asked for; a taken id gets the next free one") + (is (= 10 (clip/frames clip :inner)) "what was already here is untouched") + (is (= :inner-2 (:of (first (vals (get-in clip [:symbols :take :nodes]))))) + "and the copy's instance follows its renamed symbol") + (is (empty? (clip/problems clip))))) + +(deftest a-placed-symbols-sound-is-heard-where-it-is-placed + (let [voice {:id :v :kind :audio :source {:footage "f"} :z "a1" + :span [10 40] :time {:mode :map :at 0 :rate 1} + :channels {[:audio :gain] (ch/keyed {0 0.0 5 1.0})}} + c (-> (clip/blank) + (assoc-in [:symbols :talk] {:id :talk :frames 30 :nodes {:v voice}}) + (clip/place-symbol :main :talk 50 #uuid "00000000-0000-4000-8000-0000000000bb" [0 0])) + [t] (clip/audio-tracks c :main)] + (is (= [10 40] (:span t)) "the same frames of the source") + (is (= [50 80] (node/placed-span t)) "starting where the instance starts") + (is (= #{50 55} (set (keys (get-in t [:channels [:audio :gain] :keys])))) + "with its automation moved along") + (testing "and cut off where the instance's own span ends" + (let [c (assoc-in c [:symbols :main :nodes #uuid "00000000-0000-4000-8000-0000000000bb" :span] [0 12]) + [t] (clip/audio-tracks c :main)] + (is (= [10 22] (:span t))) + (is (= [50 62] (node/placed-span t))))))) diff --git a/static/arthur/app.css b/static/arthur/app.css index d996758..83a6def 100644 --- a/static/arthur/app.css +++ b/static/arthur/app.css @@ -654,3 +654,74 @@ input[type="range"] { width: 100%; accent-color: var(--sel); } } .tl-empty { padding: 9px; color: var(--dim); } + +/* -------------------------------------------------------------------------- + the video -> symbol dialog */ + +.convert-scrim { + position: fixed; + inset: 0; + z-index: 20; + display: grid; + place-items: center; + background: rgba(0, 0, 0, .35); +} + +.convert { + width: min(560px, calc(100vw - 32px)); + background: var(--pane); + border: 1px solid var(--line); + border-radius: 3px; + box-shadow: 0 6px 24px rgba(0, 0, 0, .3); +} + +.convert-body { display: grid; gap: 8px; padding: 10px; } + +.convert-video { + width: 100%; + max-height: 46vh; + background: var(--stage); + border-radius: 2px; +} + +/* The range: the kept frames lit, a handle at each end. */ +.convert-range { + position: relative; + height: 18px; + margin: 0 7px; + background: var(--sunk); + border: 1px solid var(--hair); + border-radius: 2px; + touch-action: none; +} + +.convert-kept { + position: absolute; + top: 0; + bottom: 0; + background: var(--sel-bg); + border-left: 1px solid var(--sel); + border-right: 1px solid var(--sel); +} + +.convert-handle { + position: absolute; + top: -3px; + bottom: -3px; + width: 10px; + margin-left: -5px; + background: var(--sel); + border-radius: 2px; + cursor: ew-resize; +} + +.convert-name { display: flex; align-items: center; gap: 8px; } +.convert-name input { flex: 1; } + +.convert-actions { + display: flex; + justify-content: flex-end; + gap: 6px; + padding: 8px 10px; + border-top: 1px solid var(--hair); +}