Preserve source cadence and sample picture fps after analysis
This commit is contained in:
Olive Vaughn
parent
32683efccf
commit
06cf02db83
20 changed files with 303 additions and 87 deletions
|
|
@ -87,16 +87,32 @@ and real footage use `src/arthur/flow/take.cljs` for the measurement order and
|
|||
From the repo root, extract a clip, then click **load frames** in the CLJS app:
|
||||
|
||||
```sh
|
||||
./extract.sh /path/to/clip.mov 12
|
||||
./extract.sh /path/to/clip.mov
|
||||
```
|
||||
|
||||
This writes `frames/0001.png` onward, `audio.wav`, and `manifest.json` at the
|
||||
repo root. The manifest supplies the exact frame count, fps and audio path.
|
||||
This keeps every source frame and writes `frames/0001.png` onward, `audio.wav`,
|
||||
and `manifest.json` at the repo root. The manifest supplies the exact frame
|
||||
count, source fps and audio path. Variable frame rate sources are rejected until
|
||||
the manifest and clock carry per-frame timestamps.
|
||||
|
||||
To keep multiple takes or compare with a previous extraction, pass a bundle
|
||||
directory and enter its manifest path in the app:
|
||||
|
||||
```sh
|
||||
./extract.sh /path/to/clip.mov scratch/my-take
|
||||
# source manifest field: /scratch/my-take/manifest.json
|
||||
```
|
||||
|
||||
`scratch/` is ignored by Git. The directory contains its own frames, audio and
|
||||
manifest, so extracting it does not replace another take's files.
|
||||
Loading detects one face per frame, freezes the measured mouth into channels,
|
||||
and adds a button for the footage clip. Detection happens once when you load;
|
||||
playback only resolves channels and paints. Frames without a detection remain
|
||||
marked absent even though their neighbouring poses are used to condition the
|
||||
track. The stage stays 320×200 regardless of the footage dimensions.
|
||||
track. The stage stays 320×200 regardless of the footage dimensions. Real
|
||||
footage starts at the source picture rate. The **picture fps** buttons sample the
|
||||
frozen roto at lower rates while the source track, duration and audio clock stay
|
||||
unchanged. Picking frames to trace into cels is a separate future editing step.
|
||||
|
||||
MediaPipe's JS, wasm and model are under `public/mediapipe/` and served locally.
|
||||
No CDN is used by this app. See that directory's README for provenance.
|
||||
|
|
|
|||
|
|
@ -31,6 +31,12 @@
|
|||
.scrub { width: 100%; margin: 10px 0 6px; }
|
||||
.readout { display: flex; gap: 18px; opacity: .55; font-size: 12px; }
|
||||
.readout .warn { color: #d98f5a; opacity: 1; }
|
||||
.picture-rate { display: flex; align-items: center; gap: 6px; margin-top: 7px;
|
||||
font-size: 12px; }
|
||||
.source-path { display: block; margin-top: 8px; font-size: 12px; opacity: .7; }
|
||||
.source-path input { width: 300px; margin-left: 8px; padding: 3px 5px;
|
||||
color: var(--fg); background: #1c1f2b; border: 1px solid #2b3040;
|
||||
font: inherit; }
|
||||
.load-status { margin-top: 6px; font-size: 12px; opacity: .75; }
|
||||
.note { opacity: .35; font-size: 12px; max-width: 640px; }
|
||||
</style>
|
||||
|
|
|
|||
|
|
@ -97,3 +97,8 @@
|
|||
rather than only inside the scene."
|
||||
[f expose]
|
||||
(node/expose f expose))
|
||||
|
||||
(defn picture-frame
|
||||
"The source pose displayed at f after picture-rate sampling and exposure."
|
||||
[f source-fps picture-fps expose]
|
||||
(node/expose (node/sample-frame f source-fps picture-fps) expose))
|
||||
|
|
|
|||
|
|
@ -23,7 +23,8 @@
|
|||
because there is one clip per scene today. Copying them by hand into this table
|
||||
is how one of them comes to disagree with the scene it describes."
|
||||
[label scene store]
|
||||
(merge {:label label :scene scene :store store :audio "/audio.wav"}
|
||||
(merge {:label label :scene scene :store store :audio "/audio.wav"
|
||||
:display-fps (:fps scene)}
|
||||
(select-keys scene [:fps :frames :width :height])))
|
||||
|
||||
(def scenes
|
||||
|
|
@ -50,9 +51,10 @@
|
|||
;; footage's. That is what deleting `makeXform` buys — the framing became a
|
||||
;; transform on a node, so nothing downstream of the freeze knows the frame
|
||||
;; size — and it is why ui/player no longer hardcodes 320x200.
|
||||
:clip (select-keys (:take scenes) [:fps :frames :width :height :audio])
|
||||
:clip (select-keys (:take scenes) [:fps :frames :width :height :audio :display-fps])
|
||||
|
||||
:footage {:id nil :label nil :loading? false :status nil}
|
||||
:footage {:id nil :label nil :loading? false :status nil
|
||||
:manifest-path "/manifest.json"}
|
||||
|
||||
;; --- transport ---
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -85,6 +85,22 @@
|
|||
[f n]
|
||||
(if (and n (> n 1)) (* (js/Math.floor (/ f n)) n) f))
|
||||
|
||||
(defn sample-frame
|
||||
"Pick a source frame for a lower picture rate without changing clip time.
|
||||
|
||||
The input is already an integer source frame from the audio clock. Its time is
|
||||
f/source-fps. Quantise that time to the picture grid, then read the latest
|
||||
source frame at or before it. The result is always an integer and never from
|
||||
the future, including when the rates do not divide (30 source → 24 picture)."
|
||||
[f source-fps picture-fps]
|
||||
(if (and source-fps picture-fps
|
||||
(pos? source-fps) (pos? picture-fps)
|
||||
(< picture-fps source-fps))
|
||||
(min f (js/Math.floor
|
||||
(* (js/Math.floor (/ (* f picture-fps) source-fps))
|
||||
(/ source-fps picture-fps))))
|
||||
f))
|
||||
|
||||
(defn local-frame
|
||||
"Apply a node's time map to the frame it was handed by its parent.
|
||||
|
||||
|
|
@ -99,7 +115,8 @@
|
|||
as two performances; offset is PER-NODE by design, because mouth lead applies
|
||||
to performance nodes and not to the plate, which is the entire point of it."
|
||||
[n f]
|
||||
(let [{:keys [mode offset rate] ex :expose :or {mode :inherit}} (:time n)]
|
||||
(let [{:keys [mode offset rate source-fps sample-fps]
|
||||
ex :expose :or {mode :inherit}} (:time n)]
|
||||
(if (= mode :inherit)
|
||||
f
|
||||
(do
|
||||
|
|
@ -110,7 +127,11 @@
|
|||
(when (and rate (not= rate 1.0) (not= rate 1))
|
||||
(throw (ex-info "time map :rate is symbol timing and symbols are not built (port-plan step 2 scope)"
|
||||
{:node (:id n) :time (:time n)})))
|
||||
(when (and sample-fps (not (and source-fps (pos? source-fps))))
|
||||
(throw (ex-info "picture sampling needs a positive source fps"
|
||||
{:node (:id n) :time (:time n)})))
|
||||
(cond-> f
|
||||
sample-fps (sample-frame source-fps sample-fps)
|
||||
ex (expose ex)
|
||||
offset (+ offset))))))
|
||||
|
||||
|
|
@ -256,4 +277,3 @@
|
|||
(into (for [[path c] (:channels n)
|
||||
p (ch/problems c)]
|
||||
(str "channel " (pr-str path) ": " p))))))
|
||||
|
||||
|
|
|
|||
|
|
@ -43,13 +43,11 @@
|
|||
|
||||
(defn- build-clip [manifest {:keys [dense detected dimensions missing first-real]}]
|
||||
(let [[w h] dimensions
|
||||
params (merge take/knobs
|
||||
{:name "footage" :fps (:fps manifest) :aspect (/ w h)
|
||||
:stage [320 200] :expose 2 :head :as-filmed
|
||||
:analysis (str "mediapipe:1.0.1/" (:source manifest))})
|
||||
frozen (take/build params {:dense dense :detected detected})
|
||||
frozen (take/footage manifest {:dense dense :detected detected
|
||||
:dimensions dimensions})
|
||||
scene (:scene frozen)]
|
||||
(assoc (select-keys scene [:fps :frames :width :height])
|
||||
:display-fps (:fps scene)
|
||||
:scene scene :store (:store frozen)
|
||||
;; Re-extraction often overwrites audio.wav under the same name. A new
|
||||
;; URL makes the element fetch the new sound when this clip is loaded.
|
||||
|
|
@ -64,8 +62,8 @@
|
|||
|
||||
(rf/reg-fx
|
||||
::begin!
|
||||
(fn [_]
|
||||
(-> (ingest/manifest!)
|
||||
(fn [path]
|
||||
(-> (ingest/manifest! path)
|
||||
(.then (fn [manifest]
|
||||
(rf/dispatch [::progress "loading MediaPipe…"])
|
||||
(-> (detect/landmarker!)
|
||||
|
|
@ -88,7 +86,11 @@
|
|||
{:db (assoc db :footage (assoc (:footage db) :loading? true
|
||||
:status "reading manifest.json…"))
|
||||
::pb/pause! nil
|
||||
::begin! nil})))
|
||||
::begin! (get-in db [:footage :manifest-path])})))
|
||||
|
||||
(rf/reg-event-db
|
||||
::set-manifest-path
|
||||
(fn [db [_ path]] (assoc-in db [:footage :manifest-path] path)))
|
||||
|
||||
(rf/reg-event-db
|
||||
::progress
|
||||
|
|
@ -106,9 +108,9 @@
|
|||
(let [clip (store/entry id)]
|
||||
{:db (-> db
|
||||
(assoc :scene/current id
|
||||
:clip (select-keys clip [:fps :frames :width :height :audio])
|
||||
:footage {:id id :label (:label clip)
|
||||
:loading? false :status summary})
|
||||
:clip (select-keys clip [:fps :frames :width :height :audio :display-fps])
|
||||
:footage (assoc (:footage db) :id id :label (:label clip)
|
||||
:loading? false :status summary))
|
||||
(assoc-in [:playback :frame] 0)
|
||||
(assoc-in [:playback :playing?] false))
|
||||
::pb/pause! nil})))
|
||||
|
|
|
|||
|
|
@ -65,6 +65,13 @@
|
|||
{:db (assoc-in db [:playback :rate] r)
|
||||
::rate! r}))
|
||||
|
||||
(rf/reg-event-db
|
||||
::set-picture-fps
|
||||
(fn [db [_ target]]
|
||||
(if (and (number? target) (pos? target) (<= target (fps db)))
|
||||
(assoc-in db [:clip :display-fps] target)
|
||||
db)))
|
||||
|
||||
;; --- effects: every DOM touch on the audio element is one of these ---
|
||||
|
||||
(rf/reg-fx ::play! (fn [_] (clock/play!)))
|
||||
|
|
@ -98,7 +105,7 @@
|
|||
;; The stage travels with the clip: two clips may be different
|
||||
;; sizes, and the raster the loop paints into is the clip's, not
|
||||
;; the app's.
|
||||
(assoc :clip (select-keys clip [:fps :frames :width :height :audio]))
|
||||
(assoc :clip (select-keys clip [:fps :frames :width :height :audio :display-fps]))
|
||||
(assoc-in [:playback :frame] 0)
|
||||
(assoc-in [:playback :playing?] false))
|
||||
::pause! nil
|
||||
|
|
|
|||
|
|
@ -275,6 +275,31 @@
|
|||
;; ---------------------------------------------------------------------------
|
||||
;; the face, onto the stage
|
||||
|
||||
(defn- motion-placement
|
||||
"An editable default framing for a real take. Fit the observed eye/nose and
|
||||
mouth motion within the stage; otherwise a downward head move can push the
|
||||
mouth below a fixed 320×200 canvas even while it stays in the source image."
|
||||
[[w h] {:keys [rigid transforms outer detected]}]
|
||||
(let [points (mapcat (fn [i]
|
||||
(when (or (nil? detected) (nth detected i))
|
||||
(concat (nth rigid i)
|
||||
(geom/apply-sim-all (invert (nth transforms i))
|
||||
(nth outer i)))))
|
||||
(range (count outer)))
|
||||
xs (map :x points)
|
||||
ys (map :y points)
|
||||
x0 (reduce min xs)
|
||||
x1 (reduce max xs)
|
||||
y0 (reduce min ys)
|
||||
y1 (reduce max ys)
|
||||
cx (/ (+ x0 x1) 2)
|
||||
cy (/ (+ y0 y1) 2)
|
||||
k (min (/ (* 0.8 w) (max 1e-9 (- x1 x0)))
|
||||
(/ (* 0.8 h) (max 1e-9 (- y1 y0))))]
|
||||
{[:xform :anchor] (ch/framed [cx cy])
|
||||
[:xform :scale] (ch/framed [k k])
|
||||
[:xform :pos] (ch/framed [(- (/ w 2) cx) (- (/ h 2) cy)])}))
|
||||
|
||||
(defn face-placement
|
||||
"The face's transform on the stage, as FRAMED channels.
|
||||
|
||||
|
|
@ -284,9 +309,14 @@
|
|||
ordinary node. `makeXform` made the same decision and then baked it into every
|
||||
vertex, where nothing could ever revise it.
|
||||
|
||||
Derived from the reference rigid configuration, which is what the freeze has:
|
||||
the face oval is not measured, because its only consumers in the prototype were
|
||||
that transform and the placeholder plate outline. Two numbers:
|
||||
The synthetic take uses the reference rigid configuration for its default.
|
||||
Real footage can request `:fit-motion?`: its default fits the observed rigid
|
||||
and lip motion in the stage so a head move does not send the mouth off canvas.
|
||||
Both are ordinary editable transforms on :face, never baked into the geometry.
|
||||
The face oval is not measured, because its only consumers in the prototype were
|
||||
the old baked framing transform and the placeholder plate outline.
|
||||
|
||||
For the synthetic default, two numbers:
|
||||
|
||||
SCALE is stage pixels per image height, set so the reference's eye-corner span
|
||||
is 40% of the stage width. Landmark-free — it is the rigid configuration's own
|
||||
|
|
@ -304,15 +334,17 @@
|
|||
landmarks are eyes and nose — the upper middle of a face — so a quarter down
|
||||
leaves the jaw and the mouth on the stage. Whatever hangs off is clipped, which
|
||||
is not a feature to add: every fill in `domain/raster` clamps already."
|
||||
[{:keys [stage]} {:keys [ref]}]
|
||||
(let [[w h] stage
|
||||
c (geom/centroid ref)
|
||||
span (- (reduce max (map :x ref)) (reduce min (map :x ref)))
|
||||
k (/ (* 0.4 w) span)]
|
||||
{[:xform :anchor] (ch/framed [(:x c) (:y c)])
|
||||
[:xform :scale] (ch/framed [k k])
|
||||
[:xform :pos] (ch/framed [(- (/ w 2) (:x c))
|
||||
(- (* 0.25 h) (:y c))])}))
|
||||
[{:keys [stage fit-motion?]} {:keys [ref] :as inputs}]
|
||||
(if fit-motion?
|
||||
(motion-placement stage inputs)
|
||||
(let [[w h] stage
|
||||
c (geom/centroid ref)
|
||||
span (- (reduce max (map :x ref)) (reduce min (map :x ref)))
|
||||
k (/ (* 0.4 w) span)]
|
||||
{[:xform :anchor] (ch/framed [(:x c) (:y c)])
|
||||
[:xform :scale] (ch/framed [k k])
|
||||
[:xform :pos] (ch/framed [(- (/ w 2) (:x c))
|
||||
(- (* 0.25 h) (:y c))])})))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the clip
|
||||
|
|
@ -327,6 +359,8 @@
|
|||
`(count outer)`, because a freeze that could disagree with its
|
||||
own input about the length of the take would.
|
||||
:stage [w h] project dimensions, INDEPENDENT of the footage
|
||||
:fit-motion? choose an editable real-footage default that keeps the
|
||||
observed feature motion on stage
|
||||
:expose the clip root's exposure grid, inherited by everything
|
||||
:verts the lip rings' vertex budget
|
||||
:aperture-cut fraction of the take's peak aperture below which the mouth
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@
|
|||
(assoc m :fps fps :frames frames)))
|
||||
|
||||
(defn manifest!
|
||||
[]
|
||||
(-> (js/fetch "/manifest.json" #js {:cache "no-store"})
|
||||
[path]
|
||||
(-> (js/fetch (str "/" (str/replace path #"^/+" "")) #js {:cache "no-store"})
|
||||
(.then (fn [response]
|
||||
(when-not (.-ok response)
|
||||
(throw (ex-info "manifest.json was not found; run extract.sh first"
|
||||
|
|
|
|||
|
|
@ -24,3 +24,16 @@
|
|||
(defn build
|
||||
[params inputs]
|
||||
(freeze/clip params (measure params inputs)))
|
||||
|
||||
(defn footage
|
||||
"A real manifest and its detected landmarks through the same measurement and
|
||||
freeze path as the synthetic take. The source cadence stays in :fps; picture
|
||||
sampling is a root time map applied only after this artifact exists."
|
||||
[manifest {:keys [dense detected dimensions]}]
|
||||
(let [[w h] dimensions
|
||||
params (merge knobs
|
||||
{:name "footage" :fps (:fps manifest) :aspect (/ w h)
|
||||
:stage [320 200] :fit-motion? true
|
||||
:expose 1 :head :as-filmed
|
||||
:analysis (str "mediapipe:1.0.1/" (:source manifest))})]
|
||||
(build params {:dense dense :detected detected})))
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
(rf/reg-sub ::loop? (fn [db _] (get-in db [:playback :loop?])))
|
||||
(rf/reg-sub ::muted? (fn [db _] (get-in db [:playback :muted?])))
|
||||
(rf/reg-sub ::fps (fn [db _] (get-in db [:clip :fps])))
|
||||
(rf/reg-sub ::display-fps (fn [db _] (get-in db [:clip :display-fps])))
|
||||
(rf/reg-sub ::frames (fn [db _] (get-in db [:clip :frames])))
|
||||
;; The stage, in pixels. On the clip because project dimensions are independent
|
||||
;; of the footage — see flow/freeze/face-placement — so the canvas and the raster
|
||||
|
|
|
|||
|
|
@ -10,11 +10,28 @@
|
|||
(:require [arthur.domain.palette :as pal]
|
||||
[arthur.domain.scene :as scene]
|
||||
[arthur.footage.store :as footage]
|
||||
[arthur.subs.playback :as playback]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
(rf/reg-sub ::scene-id (fn [db _] (:scene/current db)))
|
||||
|
||||
(rf/reg-sub ::scene (fn [db _] (:scene (footage/entry (:scene/current db)))))
|
||||
(rf/reg-sub
|
||||
::base-scene
|
||||
:<- [::scene-id]
|
||||
(fn [id _] (:scene (footage/entry id))))
|
||||
|
||||
(rf/reg-sub
|
||||
::scene
|
||||
:<- [::base-scene]
|
||||
:<- [::playback/display-fps]
|
||||
(fn [[scene picture-fps] _]
|
||||
;; This changes only the scene's root time map. The dense source track stays
|
||||
;; at its native rate, and the audio clock still advances through source time.
|
||||
(if (and scene picture-fps (< picture-fps (:fps scene)))
|
||||
(-> scene
|
||||
(assoc-in [:nodes :root :time :source-fps] (:fps scene))
|
||||
(assoc-in [:nodes :root :time :sample-fps] picture-fps))
|
||||
scene)))
|
||||
|
||||
(rf/reg-sub
|
||||
::exposure
|
||||
|
|
@ -45,11 +62,12 @@
|
|||
|
||||
(rf/reg-sub
|
||||
::store
|
||||
(fn [db _]
|
||||
:<- [::scene-id]
|
||||
(fn [id _]
|
||||
;; Tier 2, behind a handle, and never in app-db itself — what is in the db is
|
||||
;; the id of the clip whose blocks these are. The hand-written demo has none;
|
||||
;; the swarm is entirely dense.
|
||||
(:store (footage/entry (:scene/current db)))))
|
||||
(:store (footage/entry id))))
|
||||
|
||||
(rf/reg-sub
|
||||
::resolver
|
||||
|
|
|
|||
|
|
@ -167,12 +167,18 @@
|
|||
;; the next one lands where the audio already is, so the failure mode is
|
||||
;; a visible stutter rather than an invisible slide out of sync.
|
||||
f (if live? (clock/frame fps frames) frame)]
|
||||
;; A paused scrub is an explicit seek, not a playback sample. Clear the
|
||||
;; meter before the next play so its first frame cannot count the seek as
|
||||
;; dropped frames or leave a stale low rate in the transport.
|
||||
(when (and (not live?) (:t0 @meter-state))
|
||||
(reset! meter-state {:t0 nil :paints 0 :samples 0 :advanced 0 :prev nil})
|
||||
(reset! meter {:fps 0 :drop 0}))
|
||||
;; `ready?` gates the bookkeeping as well as the draw: recording a frame as
|
||||
;; painted when it was not is how the canvas stays empty forever.
|
||||
(when (and ready? f (not= f (:last @state)))
|
||||
(swap! state assoc :last f)
|
||||
(paint! f)
|
||||
(meter! f)
|
||||
(when live? (meter! f))
|
||||
(when live? (rf/dispatch [::pb/tick f])))
|
||||
;; The audio ending is the authority on playback having stopped; nothing
|
||||
;; counts frames to notice it.
|
||||
|
|
|
|||
|
|
@ -35,8 +35,10 @@
|
|||
frame @(rf/subscribe [::sub/frame])
|
||||
frames @(rf/subscribe [::sub/frames])
|
||||
fps @(rf/subscribe [::sub/fps])
|
||||
picture-fps @(rf/subscribe [::sub/display-fps])
|
||||
current @(rf/subscribe [::render/scene-id])
|
||||
expose @(rf/subscribe [::render/exposure])
|
||||
{:keys [id label loading? status]} @(rf/subscribe [::sub/footage])]
|
||||
{:keys [id label loading? status manifest-path]} @(rf/subscribe [::sub/footage])]
|
||||
[:div.transport
|
||||
[:div.row
|
||||
[:button {:on-click #(rf/dispatch [::pb/toggle])}
|
||||
|
|
@ -77,8 +79,9 @@
|
|||
:on-change #(rf/dispatch [::pb/seek (js/parseInt (.. % -target -value) 10)])}]
|
||||
[:div.readout
|
||||
[:span (str "frame " frame " / " frames)]
|
||||
[:span (str fps " fps")]
|
||||
[:span (str "exposure " expose " → holds " (clock/exposed-frame frame expose))]
|
||||
[:span (str "source " fps " fps")]
|
||||
[:span (str "picture " picture-fps " fps")]
|
||||
[:span (str "pose " (clock/picture-frame frame fps picture-fps expose))]
|
||||
[:span (str (js/Math.round (* 100 rate)) "%")]
|
||||
;; Measured in the loop, not derived from the clock: the whole question
|
||||
;; while profiling is whether the painting keeps up with the clock, so a
|
||||
|
|
@ -88,6 +91,19 @@
|
|||
(str (.toFixed (or fps 0) 1) " paint/s"
|
||||
(when (and drop (pos? drop))
|
||||
(str " · " (.toFixed drop 2) " frames/paint")))])]
|
||||
(when (= id current)
|
||||
[:div.picture-rate
|
||||
[:span "picture fps "]
|
||||
(doall
|
||||
(for [r (distinct (filter #(<= % fps) [8 12 15 24 fps]))]
|
||||
^{:key r}
|
||||
[:button {:class (when (= r picture-fps) "on")
|
||||
:on-click #(rf/dispatch [::pb/set-picture-fps r])}
|
||||
(if (= r fps) "source" (str r))]))])
|
||||
[:label.source-path "source manifest "
|
||||
[:input {:type "text" :value manifest-path :disabled loading?
|
||||
:on-change #(rf/dispatch [::footage/set-manifest-path
|
||||
(.. % -target -value)])}]]
|
||||
(when status [:div.load-status status])]))
|
||||
|
||||
(defn- stage []
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue