Add video upload, extraction progress, and reusable analysis sources
This commit is contained in:
Olive Vaughn
parent
690de21fa4
commit
686f897401
24 changed files with 927 additions and 137 deletions
|
|
@ -41,8 +41,9 @@ things fail in ways that read like the code being broken and are not.
|
|||
mise exec -- python manage.py test clips # from the REPO ROOT
|
||||
```
|
||||
|
||||
Thirty-one tests over the API: the blob store, key verification, the load/save
|
||||
round trip, the conditional write, and the footage manifest. The two groups worth
|
||||
Tests cover the API: the blob store, key verification, the load/save round trip,
|
||||
the conditional write, source analysis blocks, and video upload and extraction.
|
||||
The two groups worth
|
||||
reading are the ones that make the tier split a property of the system rather than
|
||||
a convention in ClojureScript — the server recomputes every tier-2 key it is
|
||||
handed, and refuses a block whose analysis does not declare a detector version.
|
||||
|
|
@ -111,10 +112,16 @@ The demo scene itself is `src/arthur/demo/scene.edn`. Both the synthetic take
|
|||
and real footage use `src/arthur/flow/take.cljs` for the measurement order and
|
||||
`src/arthur/flow/freeze.cljs` for the landmark-to-channel conversion.
|
||||
|
||||
### Real footage (port steps 6–7, served by the backend since step 9)
|
||||
### Real footage
|
||||
|
||||
Two commands from the repo root, then pick the take in the app and click
|
||||
**load frames**:
|
||||
Choose a video in the **footage** file input. The server probes it, extracts one
|
||||
PNG per source frame and WAV audio, then makes the resulting footage selectable.
|
||||
Click **load frames** to detect and freeze it. Extraction progress is currently
|
||||
read from `/api/extractions/<key>`; a future WebSocket can push the same job state.
|
||||
The uploaded bytes, extraction job, and decoded footage have separate records, so
|
||||
the same uploaded video can be reopened without decoding it again.
|
||||
|
||||
The command-line route is also available for an existing extracted bundle:
|
||||
|
||||
```sh
|
||||
./extract.sh /path/to/clip.mov # decode to frames + audio + manifest
|
||||
|
|
@ -161,6 +168,9 @@ stays 320×200 regardless of the footage dimensions. Real
|
|||
footage starts at the source picture rate. The **picture fps** buttons sample the
|
||||
frozen roto at lower rates while the source track, duration and audio clock stay
|
||||
unchanged. Picking frames to trace into cels is a separate future editing step.
|
||||
**save** also stores the detection mask, dense landmarks and raw RGBA mouth crops
|
||||
as three analysis blocks. **open** restores these without running MediaPipe or
|
||||
loading source PNGs. The frozen shapes remain separate channel blocks.
|
||||
|
||||
For known occlusion intervals, an extracted manifest may add
|
||||
`"feature-absence": {"eye-r": [[10, 14]]}`. Frame numbers are one-based and
|
||||
|
|
@ -184,13 +194,14 @@ runs the old JS tool on 8777, and the two are meant to run side by side.
|
|||
|
||||
## Saving
|
||||
|
||||
**save** and **open** in the transport. A save is three requests, in an order that
|
||||
**save** and **open** in the transport. A save has three ordered stages:
|
||||
is the tier split:
|
||||
|
||||
1. the **analysis** record, so every block stored afterwards can name the detector
|
||||
version that produced it. The server refuses a block whose analysis it does not
|
||||
know.
|
||||
2. ask which **blocks** are missing, and upload only those.
|
||||
2. ask which **blocks** are missing, upload the source analysis blocks and frozen
|
||||
channel blocks, then link the source blocks to the analysis.
|
||||
3. the **document** — tier 1, as leaves. The server refuses a clip that names
|
||||
blocks it does not hold, so a saved document cannot load into a blank stage
|
||||
somewhere else.
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@
|
|||
(case type
|
||||
"int16" (js/Int16Array. (.-buffer u8))
|
||||
"float32" (js/Float32Array. (.-buffer u8))
|
||||
"float64" (js/Float64Array. (.-buffer u8))
|
||||
"uint8" u8
|
||||
(throw (ex-info "a block's element type is \"int16\", \"float32\" or \"uint8\""
|
||||
(throw (ex-info "a block's element type is \"int16\", \"float32\", \"float64\" or \"uint8\""
|
||||
{:type type})))))
|
||||
|
|
|
|||
|
|
@ -9,16 +9,18 @@
|
|||
[arthur.flow.detect :as detect]
|
||||
[arthur.flow.ingest :as ingest]
|
||||
[arthur.flow.measure.interior :as interior]
|
||||
[arthur.flow.source :as source]
|
||||
[arthur.flow.take :as take]
|
||||
[arthur.footage.store :as store]
|
||||
[arthur.domain.landmarks :as lm]
|
||||
[arthur.fx.http :as http]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
(defn- detect-frames! [manifest model]
|
||||
(let [canvas (.createElement js/document "canvas")
|
||||
ctx (.getContext canvas "2d")
|
||||
raw (atom [])
|
||||
interiors (atom [])
|
||||
crops (atom [])
|
||||
dims (atom nil)
|
||||
total (:frames manifest)]
|
||||
(js/Promise.
|
||||
|
|
@ -27,7 +29,7 @@
|
|||
(if (= i total)
|
||||
(try
|
||||
(resolve (assoc (detect/fill-gaps @raw)
|
||||
:dimensions @dims :interior @interiors))
|
||||
:dimensions @dims :crops @crops))
|
||||
(catch :default error (reject error)))
|
||||
(-> (ingest/image! (ingest/frame-url manifest i))
|
||||
(.then
|
||||
|
|
@ -43,13 +45,11 @@
|
|||
(let [face (detect/detect! model canvas)
|
||||
ring (when face (mapv #(nth face %) lm/LIPS-INNER))
|
||||
box (when ring (interior/crop ring wh))
|
||||
measured (if box
|
||||
(interior/measure take/knobs box
|
||||
(.getImageData ctx (:x box) (:y box)
|
||||
(:w box) (:h box)))
|
||||
{:contour nil :contrast 0 :area 0 :debug nil})]
|
||||
pixels (when box
|
||||
(.-data (.getImageData ctx (:x box) (:y box)
|
||||
(:w box) (:h box))))]
|
||||
(swap! raw conj face)
|
||||
(swap! interiors conj measured))
|
||||
(swap! crops conj (when box {:box box :data pixels})))
|
||||
(when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total))
|
||||
(rf/dispatch [::progress (str "detecting " (inc i) "/" total)]))
|
||||
;; Let the status and the transport paint between sync
|
||||
|
|
@ -59,43 +59,67 @@
|
|||
(next-frame 0))))))
|
||||
|
||||
(defn- build-clip [manifest detector
|
||||
{:keys [dense detected dimensions interior missing first-real]}]
|
||||
{:keys [dense detected dimensions crops missing first-real] :as source-inputs}]
|
||||
(let [[w h] dimensions
|
||||
interior (source/measure-crops take/knobs crops)
|
||||
frozen (take/footage manifest {:dense dense :detected detected
|
||||
:dimensions dimensions :interior interior
|
||||
:presence (:presence manifest)
|
||||
:detector detector})
|
||||
built (:clip frozen)]
|
||||
built (:clip frozen)
|
||||
source-blocks (source/pack (:id (:analysis built)) source-inputs)]
|
||||
(assoc (select-keys built [:fps :width :height])
|
||||
:frames (clip/frames built)
|
||||
:display-fps (:fps built)
|
||||
:clip built :store (:store frozen)
|
||||
:source-blocks source-blocks :source-inputs source-inputs
|
||||
;; No cache-buster. The audio is a blob named by the hash of its own
|
||||
;; bytes, so re-extracting gives it a different URL rather than
|
||||
;; overwriting this one — which is what the `?v=` here used to work
|
||||
;; around.
|
||||
:audio (ingest/audio-url manifest)
|
||||
:label (or (:label manifest) (:source manifest) "footage")
|
||||
:footage-id (:id manifest)
|
||||
:cid (or (:id manifest) "footage")
|
||||
:summary (str (:frames manifest) " frames · " w "×" h " · "
|
||||
(:fps manifest) " fps"
|
||||
(when (pos? missing)
|
||||
(str " · " missing " without a face"
|
||||
(when (pos? first-real)
|
||||
(str " (first found on " (inc first-real) ")"))))))))
|
||||
(str " (first found on " (inc first-real) ")"))))))))
|
||||
|
||||
(defn- cached-source! [manifest detector]
|
||||
(let [key (:id (take/analysis-for manifest detector))]
|
||||
(-> (http/GET (str "/api/analyses/" key))
|
||||
(.then (fn [^js analysis]
|
||||
(let [keys (array-seq (.-source_blocks analysis))]
|
||||
(when (= (count keys) (count source/roles))
|
||||
(-> (js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %)) keys)))
|
||||
(.then (fn [blocks]
|
||||
(source/unpack blocks
|
||||
[(:width manifest) (:height manifest)]))))))))
|
||||
(.catch (fn [error]
|
||||
(if (= 404 (:status (ex-data error))) nil (throw error)))))))
|
||||
|
||||
(rf/reg-fx
|
||||
::begin!
|
||||
(fn [footage-id]
|
||||
(-> (js/Promise.all #js [(ingest/manifest! footage-id) (ingest/detector!)])
|
||||
(.then (fn [[manifest detector]]
|
||||
(rf/dispatch [::progress "loading MediaPipe…"])
|
||||
(-> (detect/landmarker!)
|
||||
(.then (fn [model]
|
||||
(rf/dispatch [::progress "loading frames…"])
|
||||
(-> (detect-frames! manifest model)
|
||||
(.then (fn [track]
|
||||
(build-clip manifest detector track)))))))))
|
||||
(rf/dispatch [::progress "looking for saved analysis…"])
|
||||
(-> (cached-source! manifest detector)
|
||||
(.then (fn [track]
|
||||
(if track
|
||||
(do (rf/dispatch [::progress "reusing saved analysis…"])
|
||||
(build-clip manifest detector track))
|
||||
(do (rf/dispatch [::progress "loading MediaPipe…"])
|
||||
(-> (detect/landmarker!)
|
||||
(.then (fn [model]
|
||||
(rf/dispatch [::progress "loading frames…"])
|
||||
(-> (detect-frames! manifest model)
|
||||
(.then (fn [fresh]
|
||||
(build-clip manifest detector fresh))))))))))))))
|
||||
(.then (fn [entry]
|
||||
(let [id (store/install! entry)]
|
||||
(rf/dispatch [::loaded id (:summary entry)]))))
|
||||
|
|
@ -111,6 +135,47 @@
|
|||
(.catch (fn [error]
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))]))))))
|
||||
|
||||
(defn- poll-extraction! [key]
|
||||
(-> (http/GET (str "/api/extractions/" key))
|
||||
(.then (fn [^js job]
|
||||
(case (.-state job)
|
||||
"done" (rf/dispatch [::uploaded (.-footage job)])
|
||||
"failed" (rf/dispatch [::failed (.-error job)])
|
||||
(do (rf/dispatch [::progress
|
||||
(str "extracting " (.-progress job) "%")])
|
||||
(js/setTimeout #(poll-extraction! key) 800)))))
|
||||
(.catch (fn [error]
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))])))))
|
||||
|
||||
(rf/reg-fx
|
||||
::upload!
|
||||
(fn [file]
|
||||
(let [form (js/FormData.)]
|
||||
(.append form "file" file)
|
||||
(-> (http/POST-form "/api/sources" form)
|
||||
(.then (fn [^js source]
|
||||
(rf/dispatch [::progress "queued for extraction…"])
|
||||
(http/POST "/api/extractions" #js {:source (.-id source)
|
||||
:settings #js {}})))
|
||||
(.then (fn [^js job] (poll-extraction! (.-key job))))
|
||||
(.catch (fn [error]
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))])))))))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::upload
|
||||
(fn [{:keys [db]} [_ file]]
|
||||
(if (or (nil? file) (get-in db [:footage :loading?]))
|
||||
{}
|
||||
{:db (update db :footage merge {:loading? true :status "uploading video…"})
|
||||
::upload! file})))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::uploaded
|
||||
(fn [{:keys [db]} [_ footage-id]]
|
||||
{:db (update db :footage merge {:loading? false :chosen footage-id
|
||||
:status "video extracted — load frames to analyze"})
|
||||
:dispatch [::refresh]}))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::refresh
|
||||
(fn [_ _] {::list! nil}))
|
||||
|
|
@ -119,10 +184,9 @@
|
|||
::listed
|
||||
(fn [db [_ footage]]
|
||||
(update db :footage merge
|
||||
{:available (vec footage)
|
||||
:chosen (or (:chosen (:footage db)) (:id (first footage)))
|
||||
:status (when (empty? footage)
|
||||
"no footage ingested — ./extract.sh, then manage.py ingest_bundle")})))
|
||||
(cond-> {:available (vec footage)
|
||||
:chosen (or (:chosen (:footage db)) (:id (first footage)))}
|
||||
(empty? footage) (assoc :status "upload a video to begin")))))
|
||||
|
||||
(rf/reg-event-db
|
||||
::choose
|
||||
|
|
@ -135,8 +199,7 @@
|
|||
(cond
|
||||
(get-in db [:footage :loading?]) {}
|
||||
(nil? chosen)
|
||||
{:db (assoc-in db [:footage :status]
|
||||
"no footage ingested — ./extract.sh, then manage.py ingest_bundle")}
|
||||
{:db (assoc-in db [:footage :status] "upload a video to begin")}
|
||||
:else
|
||||
{:db (update db :footage merge {:loading? true :status "reading the manifest…"})
|
||||
::pb/pause! nil
|
||||
|
|
@ -150,7 +213,7 @@
|
|||
::failed
|
||||
(fn [db [_ message]]
|
||||
(assoc db :footage (assoc (:footage db)
|
||||
:loading? false :status (str "load failed: " message)))))
|
||||
:loading? false :status (str "footage failed: " message)))))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::loaded
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@
|
|||
[arthur.events.playback :as pb]
|
||||
[arthur.footage.store :as store]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.source :as source]
|
||||
[arthur.flow.ingest :as ingest]
|
||||
[arthur.fx.http :as http]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
|
|
@ -65,16 +67,64 @@
|
|||
(-> (http/POST "/api/projects" #js {:name name})
|
||||
(.then (fn [^js created] (.-id created))))))
|
||||
|
||||
(defn- opened-entry! [^js clip-json]
|
||||
(let [footage-id (.-footage clip-json)
|
||||
analysis-key (.-analysis clip-json)]
|
||||
(-> (js/Promise.all
|
||||
#js [(js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %))
|
||||
(array-seq (.-blocks clip-json)))))
|
||||
(if footage-id (ingest/manifest! footage-id) (js/Promise.resolve nil))
|
||||
(if analysis-key (http/GET (str "/api/analyses/" analysis-key))
|
||||
(js/Promise.resolve nil))])
|
||||
(.then (fn [[blocks manifest analysis]]
|
||||
(let [source-keys (when analysis (array-seq (.-source_blocks ^js analysis)))]
|
||||
(-> (js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %)) source-keys)))
|
||||
(.then (fn [source-responses]
|
||||
(let [cid (.-cid clip-json)
|
||||
loaded (project/load
|
||||
cid #js {:leaves (.-leaves clip-json)
|
||||
:blocks blocks})
|
||||
built (:clip loaded)
|
||||
inputs (when (seq source-keys)
|
||||
(when-not manifest
|
||||
(throw (ex-info "saved analysis has no footage"
|
||||
{:analysis analysis-key})))
|
||||
(source/unpack source-responses
|
||||
[(:width manifest) (:height manifest)]))]
|
||||
(merge (select-keys built [:fps :width :height])
|
||||
{:label (str (or (.-name clip-json) cid) " (saved)")
|
||||
:cid cid :frames (clip/frames built)
|
||||
:display-fps (:fps built)
|
||||
:clip built :store (:store loaded)
|
||||
:footage-id footage-id
|
||||
:source-inputs inputs
|
||||
:source-blocks (when inputs
|
||||
(source/pack analysis-key inputs))
|
||||
:audio (if manifest (:audio manifest)
|
||||
"/static/arthur/audio.wav")})))))))))))
|
||||
|
||||
(rf/reg-fx
|
||||
::save!
|
||||
(fn [{:keys [id cid label clip]}]
|
||||
(let [analysis (:analysis (:clip clip))
|
||||
doc (project/save cid clip)]
|
||||
doc (project/save cid clip)
|
||||
source-blocks (:source-blocks clip)]
|
||||
(-> (ensure-project! id label)
|
||||
(.then (fn [pid]
|
||||
(-> (if analysis
|
||||
(http/POST "/api/analyses" (analysis-payload analysis))
|
||||
(js/Promise.resolve nil))
|
||||
(.then (fn [_]
|
||||
(when (seq source-blocks)
|
||||
(-> (upload-missing!
|
||||
#js {:blocks (source/wire-blocks source-blocks)})
|
||||
(.then (fn [_]
|
||||
(http/PUT
|
||||
(str "/api/analyses/" (:id analysis))
|
||||
#js {:source_blocks
|
||||
(into-array (map :key (vals source-blocks)))})))))))
|
||||
(.then (fn [_] (upload-missing! doc)))
|
||||
(.then (fn [uploaded]
|
||||
(-> (http/PUT (str "/api/projects/" pid)
|
||||
|
|
@ -82,6 +132,7 @@
|
|||
:clips #js [#js {:cid cid
|
||||
:name label
|
||||
:analysis (:id analysis)
|
||||
:footage (:footage-id clip)
|
||||
:leaves (.-leaves doc)
|
||||
:blocks (block-keys doc)}]})
|
||||
(.then (fn [^js saved]
|
||||
|
|
@ -109,35 +160,13 @@
|
|||
(let [^js clip-json (first (array-seq (.-clips loaded)))]
|
||||
(when-not clip-json
|
||||
(throw (ex-info "that project has no clips" {})))
|
||||
(-> (js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %))
|
||||
(array-seq (.-blocks clip-json)))))
|
||||
(.then (fn [blocks]
|
||||
(let [doc #js {:leaves (.-leaves clip-json) :blocks blocks}
|
||||
cid (.-cid clip-json)
|
||||
loaded' (project/load cid doc)
|
||||
built (:clip loaded')
|
||||
entry (merge
|
||||
(select-keys built [:fps :width :height])
|
||||
{:label (str (or (.-name clip-json) cid) " (saved)")
|
||||
:cid cid
|
||||
:frames (clip/frames built)
|
||||
:display-fps (:fps built)
|
||||
:clip built
|
||||
:store (:store loaded')
|
||||
;; The audio is the clip's, and a
|
||||
;; document does not carry it: tier 3
|
||||
;; is by hash and the clip names the
|
||||
;; analysis, not the sound. Until the
|
||||
;; footage id is in the document, the
|
||||
;; synthetic take's is the one that
|
||||
;; keeps the clock running.
|
||||
:audio "/static/arthur/audio.wav"})]
|
||||
(rf/dispatch [::opened
|
||||
(store/install! entry "project")
|
||||
(.-id loaded)
|
||||
(.-name loaded)
|
||||
(.-seq loaded)]))))))))
|
||||
(-> (opened-entry! clip-json)
|
||||
(.then (fn [entry]
|
||||
(rf/dispatch [::opened
|
||||
(store/install! entry "project")
|
||||
(.-id loaded)
|
||||
(.-name loaded)
|
||||
(.-seq loaded)])))))))
|
||||
(.catch (fn [error]
|
||||
(js/console.error error)
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))]))))))
|
||||
|
|
|
|||
|
|
@ -137,6 +137,9 @@
|
|||
budget does not reach the teeth at all (the crop is taken from raw landmarks).
|
||||
It reads like a mistake in both directions and is neither."
|
||||
{"geom" [:anchor-avg :contour-avg :verts]
|
||||
"source/dense" []
|
||||
"source/detected" []
|
||||
"source/crops" []
|
||||
"head-pos" [:anchor-avg]
|
||||
"head-rot" [:anchor-avg]
|
||||
"head-scale" [:anchor-avg]
|
||||
|
|
|
|||
|
|
@ -1,24 +1,7 @@
|
|||
(ns arthur.flow.ingest
|
||||
"Read an ingested take. The manifest owns timing, the exact frame count, and —
|
||||
since step 9 — the URL of every frame.
|
||||
|
||||
WHAT CHANGED, AND WHY IT IS NOT A DETAIL. This used to fetch `/manifest.json` off
|
||||
the filesystem and then build `frames/0001.png` itself, with shadow-cljs serving
|
||||
the repo root. So the frame layout was a shared secret between a shell script and
|
||||
this namespace, and \"where are the frames\" was answered by a directory listing
|
||||
that nothing could version.
|
||||
|
||||
Now the server names every frame and this asks it. The manifest carries a URL per
|
||||
frame, so the frames can live in a content-addressed blob store — or, when the
|
||||
in-browser wasm-ffmpeg extraction docs/architecture.md describes arrives, be
|
||||
uploaded into the same store by the app itself — and nothing in here learns
|
||||
anything new. That is the whole point of tier 3 being addressed rather than
|
||||
located.
|
||||
|
||||
The cache-busting `?v=` that used to hang off every frame URL went with it. It
|
||||
was there because re-extracting overwrote `frames/0001.png` under the same name;
|
||||
a blob's name IS the hash of its bytes, so a stale copy is not a thing that can
|
||||
happen."
|
||||
"Read footage from the server. Its response gives source timing and one
|
||||
content-addressed URL per frame. The upload path derives this response from
|
||||
stored frame records; no `manifest.json` file is part of that path."
|
||||
(:require [arthur.fx.http :as http]))
|
||||
|
||||
(defn feature-presence
|
||||
|
|
@ -67,8 +50,7 @@
|
|||
:presence (feature-presence frames (:feature-absence m)))))
|
||||
|
||||
(defn available!
|
||||
"Every ingested take the server holds. `manage.py ingest_bundle` is what puts one
|
||||
there."
|
||||
"Every extracted take the server holds."
|
||||
[]
|
||||
(-> (http/GET "/api/footage")
|
||||
(.then (fn [json] (:footage (js->clj json :keywordize-keys true))))))
|
||||
|
|
|
|||
118
frontend/src/arthur/flow/source.cljs
Normal file
118
frontend/src/arthur/flow/source.cljs
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
(ns arthur.flow.source
|
||||
"The pixel-dependent result of analyzing footage, stored as three ordinary
|
||||
content-addressed blocks. Everything after this boundary can run without PNGs
|
||||
or MediaPipe. The mouth crops hold raw pixels and an un-eroded lip ring."
|
||||
(:require [arthur.domain.landmarks :as lm]
|
||||
[arthur.domain.wire :as wire]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.measure.interior :as interior]))
|
||||
|
||||
(def roles ["source/dense" "source/detected" "source/crops"])
|
||||
|
||||
(defn measure-crops [params crops]
|
||||
(mapv (fn [crop]
|
||||
(if crop
|
||||
(interior/measure params (:box crop) #js {:data (:data crop)})
|
||||
{:contour nil :contrast 0 :area 0 :debug nil}))
|
||||
crops))
|
||||
|
||||
(defn- named [role analysis tracks layout data]
|
||||
(merge (address/block {:role role :analysis analysis :params {}
|
||||
:tracks tracks :features [] :observation nil
|
||||
:layout layout})
|
||||
{:role role :data data}))
|
||||
|
||||
(defn pack
|
||||
"Dense landmarks, detection mask and raw RGBA mouth crops -> source blocks."
|
||||
[analysis {:keys [dense detected crops]}]
|
||||
(let [frames (count dense)
|
||||
points (count (first dense))]
|
||||
(when-not (and (pos? frames) (pos? points)
|
||||
(= frames (count detected) (count crops))
|
||||
(every? #(= points (count %)) dense))
|
||||
(throw (ex-info "source tracks must have the same frame and landmark counts"
|
||||
{:frames frames :points points})))
|
||||
(let [landmarks (js/Float64Array. (* frames points 3))
|
||||
mask (js/Uint8Array. frames)
|
||||
lengths (mapv #(if % (.-length ^js (:data %)) 0) crops)
|
||||
offsets (vec (reductions + 0 lengths))
|
||||
pixels (js/Uint8Array. (max 1 (peek offsets)))
|
||||
boxes (mapv (fn [crop]
|
||||
(when-let [box (:box crop)]
|
||||
(mapv box [:x :y :w :h])))
|
||||
crops)]
|
||||
(doseq [f (range frames)]
|
||||
(aset mask f (if (nth detected f) 1 0))
|
||||
(doseq [i (range points)]
|
||||
(let [{:keys [x y z]} (nth (nth dense f) i)
|
||||
base (* 3 (+ (* f points) i))]
|
||||
(aset landmarks base x)
|
||||
(aset landmarks (inc base) y)
|
||||
(aset landmarks (+ base 2) z)))
|
||||
(when-let [crop (nth crops f)]
|
||||
(.set pixels (:data crop) (nth offsets f))))
|
||||
{"source/dense" (named "source/dense" analysis ["landmarks"]
|
||||
{:type "float64" :frames frames :points points :stride (* 3 points)}
|
||||
landmarks)
|
||||
"source/detected" (named "source/detected" analysis ["detected"]
|
||||
{:type "uint8" :frames frames :stride 1} mask)
|
||||
"source/crops" (named "source/crops" analysis ["rgba"]
|
||||
{:type "uint8" :frames frames :boxes boxes :offsets offsets}
|
||||
pixels)})))
|
||||
|
||||
(defn wire-blocks [blocks]
|
||||
(into-array
|
||||
(map (fn [role]
|
||||
(let [{:keys [key descriptor data]} (get blocks role)]
|
||||
#js {:key key :descriptor descriptor :data (wire/base64 data)}))
|
||||
roles)))
|
||||
|
||||
(defn unpack
|
||||
"The three block-detail responses -> inputs for measurement and freeze."
|
||||
[^js responses [width height]]
|
||||
(let [by-role (into {}
|
||||
(map (fn [^js response]
|
||||
(let [descriptor (js->clj (js/JSON.parse (.-descriptor response))
|
||||
:keywordize-keys true)]
|
||||
[(:role descriptor)
|
||||
{:layout (:layout descriptor)
|
||||
:data (wire/typed (get-in descriptor [:layout :type])
|
||||
(.-data response))}]))
|
||||
(array-seq responses)))
|
||||
_ (when-not (= (set roles) (set (keys by-role)))
|
||||
(throw (ex-info "analysis is missing source blocks" {:roles (keys by-role)})))
|
||||
{:keys [layout data]} (get by-role "source/dense")
|
||||
frames (:frames layout)
|
||||
points (:points layout)
|
||||
mask (:data (get by-role "source/detected"))
|
||||
crop-data (:data (get by-role "source/crops"))
|
||||
crop-layout (:layout (get by-role "source/crops"))]
|
||||
(when-not (and (= (* frames points 3) (.-length data))
|
||||
(= frames (.-length mask) (:frames crop-layout))
|
||||
(= frames (count (:boxes crop-layout)))
|
||||
(= (inc frames) (count (:offsets crop-layout))))
|
||||
(throw (ex-info "source block layouts disagree" {:frames frames :points points})))
|
||||
(let [dense (mapv (fn [f]
|
||||
(mapv (fn [i]
|
||||
(let [base (* 3 (+ (* f points) i))]
|
||||
{:x (aget data base) :y (aget data (inc base))
|
||||
:z (aget data (+ base 2))}))
|
||||
(range points)))
|
||||
(range frames))
|
||||
detected (mapv #(pos? (aget mask %)) (range frames))
|
||||
crops (mapv
|
||||
(fn [f]
|
||||
(when-let [[x y w h] (nth (:boxes crop-layout) f)]
|
||||
(let [start (nth (:offsets crop-layout) f)
|
||||
end (nth (:offsets crop-layout) (inc f))
|
||||
shape (mapv #(nth (nth dense f) %) lm/LIPS-INNER)]
|
||||
(when-not (= (* 4 w h) (- end start))
|
||||
(throw (ex-info "mouth crop byte count disagrees with its box"
|
||||
{:frame f :expected (* 4 w h) :actual (- end start)})))
|
||||
{:box {:x x :y y :w w :h h :shape shape
|
||||
:source-width width :source-height height}
|
||||
:data (js/Uint8ClampedArray. (.-buffer crop-data) start (- end start))})))
|
||||
(range frames))]
|
||||
{:dense dense :detected detected :crops crops :dimensions [width height]
|
||||
:missing (count (remove true? detected))
|
||||
:first-real (first (keep-indexed (fn [i present?] (when present? i)) detected))})))
|
||||
|
|
@ -15,6 +15,17 @@
|
|||
|
||||
(def knobs params/defaults)
|
||||
|
||||
(defn analysis-for [manifest detector]
|
||||
(let [aspect (/ (:width manifest) (:height manifest))]
|
||||
(address/analysis
|
||||
(merge {:detector "mediapipe" :version "unknown"}
|
||||
detector
|
||||
{:source (:source manifest)
|
||||
:footage (:footage manifest)
|
||||
:frames (:frames manifest)
|
||||
:fps (:fps manifest)
|
||||
:aspect aspect}))))
|
||||
|
||||
(defn measure
|
||||
"Condition the anchor before measuring rings through it."
|
||||
[{:keys [aspect] :as params} {:keys [dense detected interior presence]}]
|
||||
|
|
@ -66,13 +77,7 @@
|
|||
;; version string somebody has to remember to bump. See
|
||||
;; `flow/address`: a model upgrade that silently reused
|
||||
;; these landmarks is the failure this prevents.
|
||||
:analysis (address/analysis
|
||||
(merge {:detector "mediapipe" :version "unknown"}
|
||||
detector
|
||||
{:source (:source manifest)
|
||||
:footage (:footage manifest)
|
||||
:frames (:frames manifest)
|
||||
:fps (:fps manifest)
|
||||
:aspect (/ w h)}))})]
|
||||
:analysis (analysis-for (assoc manifest :width w :height h)
|
||||
detector)})]
|
||||
(build params {:dense dense :detected detected :interior interior
|
||||
:presence presence})))
|
||||
|
|
|
|||
|
|
@ -34,14 +34,16 @@
|
|||
(defn request!
|
||||
([method url] (request! method url nil))
|
||||
([method url body]
|
||||
(-> (js/fetch url
|
||||
(clj->js (cond-> {:method method
|
||||
:cache "no-store"
|
||||
:headers (cond-> {"Accept" "application/json"}
|
||||
body (assoc "Content-Type" "application/json")
|
||||
(not= "GET" method)
|
||||
(assoc "X-CSRFToken" (or (csrf-token) "")))}
|
||||
body (assoc :body (js/JSON.stringify body)))))
|
||||
(let [form? (instance? js/FormData body)
|
||||
headers #js {"Accept" "application/json"}
|
||||
options #js {:method method :cache "no-store" :headers headers}]
|
||||
(when (and body (not form?))
|
||||
(aset headers "Content-Type" "application/json"))
|
||||
(when (not= "GET" method)
|
||||
(aset headers "X-CSRFToken" (or (csrf-token) "")))
|
||||
(when body
|
||||
(aset options "body" (if form? body (js/JSON.stringify body))))
|
||||
(-> (js/fetch url options)
|
||||
(.then (fn [response]
|
||||
(-> (.text response)
|
||||
(.then (fn [text]
|
||||
|
|
@ -49,8 +51,9 @@
|
|||
(try (js/JSON.parse text) (catch :default _ nil)))]
|
||||
(if (.-ok response)
|
||||
parsed
|
||||
(fail response parsed)))))))))))
|
||||
(fail response parsed))))))))))))
|
||||
|
||||
(defn GET [url] (request! "GET" url))
|
||||
(defn POST [url body] (request! "POST" url body))
|
||||
(defn POST-form [url body] (request! "POST" url body))
|
||||
(defn PUT [url body] (request! "PUT" url body))
|
||||
|
|
|
|||
|
|
@ -108,10 +108,14 @@
|
|||
[:button {:class (when (= r picture-fps) "on")
|
||||
:on-click #(rf/dispatch [::pb/set-picture-fps r])}
|
||||
(if (= r fps) "source" (str r))]))])
|
||||
;; The takes the SERVER holds, since step 9. There is no path to type any
|
||||
;; more: `./extract.sh` decodes a clip and `manage.py ingest_bundle` registers
|
||||
;; it, and from then on the frames are addressed rather than located.
|
||||
;; Upload a video or choose existing server footage. Both routes produce the
|
||||
;; same immutable frame and audio manifest.
|
||||
[:label.source-path "footage "
|
||||
[:input {:type "file" :accept "video/*" :disabled loading?
|
||||
:on-change (fn [event]
|
||||
(when-let [file (aget (.. event -target -files) 0)]
|
||||
(rf/dispatch [::footage/upload file])
|
||||
(set! (.. event -target -value) "")))}]
|
||||
[:select {:value (or chosen "") :disabled loading?
|
||||
:on-change #(rf/dispatch [::footage/choose (.. % -target -value)])}
|
||||
(if (seq available)
|
||||
|
|
@ -119,7 +123,7 @@
|
|||
^{:key id}
|
||||
[:option {:value id}
|
||||
(str label " · " frames "f @" fps)]))
|
||||
[:option {:value ""} "nothing ingested"])]
|
||||
[:option {:value ""} "no footage yet"])]
|
||||
[:button {:disabled loading?
|
||||
:on-click #(rf/dispatch [::footage/refresh])} "refresh"]]
|
||||
(when status [:div.load-status status])
|
||||
|
|
@ -148,5 +152,5 @@
|
|||
[audio]
|
||||
[transport]
|
||||
[:p.note
|
||||
"Extract a clip with extract.sh, then load frames. The manifest supplies "
|
||||
"the frame count, audio and fps."]])
|
||||
"Upload a video, choose its footage, then load frames. Save the project to "
|
||||
"share the analyzed take without detecting frames again."]])
|
||||
|
|
|
|||
|
|
@ -88,4 +88,4 @@
|
|||
(is (= [3 4] (vec (array-seq (wire/typed "int16" (wire/base64 view))))))))
|
||||
|
||||
(deftest an-unknown-element-type-is-refused
|
||||
(is (thrown-with-msg? ExceptionInfo #"int16" (wire/typed "float64" "AA=="))))
|
||||
(is (thrown-with-msg? ExceptionInfo #"int16" (wire/typed "float16" "AA=="))))
|
||||
|
|
|
|||
|
|
@ -119,9 +119,13 @@
|
|||
;; the thing it is derived FROM is the thing the biconditional above asserts.
|
||||
;; These two assertions are what keep the derivation honest in the only two ways
|
||||
;; it could rot.
|
||||
(testing "every role in the table is reachable from some knob"
|
||||
(is (= (set (keys address/block-knobs))
|
||||
(testing "every knob-dependent role is reachable from some knob"
|
||||
(is (= (into #{} (keep (fn [[role knobs]] (when (seq knobs) role)))
|
||||
address/block-knobs)
|
||||
(into #{} (mapcat val) address/knob-roles))))
|
||||
(testing "source roles have no knob dependencies"
|
||||
(doseq [role ["source/dense" "source/detected" "source/crops"]]
|
||||
(is (= [] (get address/block-knobs role)))))
|
||||
(testing "every knob a block declares is a knob the registry defines"
|
||||
;; The other direction of the same edge: a typo in `block-knobs` would name a
|
||||
;; setting no slider can move, and `block-descriptor` only checks that the
|
||||
|
|
|
|||
21
frontend/test/arthur/flow/source_test.cljs
Normal file
21
frontend/test/arthur/flow/source_test.cljs
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
(ns arthur.flow.source-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[arthur.flow.source :as source]))
|
||||
|
||||
(deftest source-blocks-round-trip-without-source-images
|
||||
(let [face (vec (repeat 478 {:x 0.25 :y 0.5 :z -0.125}))
|
||||
pixels (js/Uint8ClampedArray. #js [12 24 36 255])
|
||||
input {:dense [face face] :detected [true false]
|
||||
:crops [{:box {:x 3 :y 4 :w 1 :h 1} :data pixels} nil]}
|
||||
blocks (source/pack "sha256:analysis" input)
|
||||
out (source/unpack (source/wire-blocks blocks) [100 80])]
|
||||
(is (= (set source/roles) (set (keys blocks))))
|
||||
(is (= (:dense input) (:dense out)))
|
||||
(is (= (:detected input) (:detected out)))
|
||||
(is (= [3 4 1 1]
|
||||
((juxt :x :y :w :h) (:box (first (:crops out))))))
|
||||
(is (= [12 24 36 255]
|
||||
(vec (array-seq (:data (first (:crops out)))))))
|
||||
(is (nil? (second (:crops out))))
|
||||
(is (= (mapv :key (vals blocks))
|
||||
(mapv :key (vals (source/pack "sha256:analysis" out)))))))
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
// Writes a PNG per sampled frame into test/browser/out/ so that "it drew
|
||||
// something" can be checked by eye as well as by pixel count.
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { mkdtempSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
|
@ -356,9 +356,15 @@ async function main() {
|
|||
check(await page.eval(CLICK('save')), 'save is clickable');
|
||||
const saved = await statusMatching(/saved r\d+/);
|
||||
check(saved !== null, 'the document saves', saved ?? (await page.eval(STATUS)));
|
||||
// Eleven blocks the first time. The COUNT is not asserted — that is a fact
|
||||
// about the freeze, not about saving — but that some went up is.
|
||||
check(/· [1-9]\d* blocks?/.test(saved ?? ''), 'and its tier 2 went with it', saved ?? '');
|
||||
// Upload count may be zero when the content-addressed blocks already exist
|
||||
// on this server. The saved clip must still reference them.
|
||||
const savedBlocks = await page.eval(`(async () => {
|
||||
const listed = await fetch('/api/projects').then(r => r.json());
|
||||
const project = await fetch('/api/projects/' + listed.projects[0].id).then(r => r.json());
|
||||
return project.clips[0].blocks.length;
|
||||
})()`);
|
||||
check(savedBlocks > 0, 'the saved clip references its tier 2 blocks',
|
||||
`${savedBlocks} blocks`);
|
||||
|
||||
// Again, unchanged. Content addressing means the second save uploads nothing
|
||||
// and rewrites nothing: this is the assertion that the keys are stable across
|
||||
|
|
@ -387,6 +393,33 @@ async function main() {
|
|||
check(!back[0].selectedClip.includes('take'), 'the picture is the reopened document',
|
||||
JSON.stringify(back[0].selectedClip));
|
||||
|
||||
// Exercise the actual file input and FormData path, then wait for the
|
||||
// extraction job's footage to appear in the server list.
|
||||
const uploadDir = mkdtempSync(join(tmpdir(), 'arthur-upload-'));
|
||||
try {
|
||||
const video = join(uploadDir, 'browser-upload.mp4');
|
||||
const encoded = spawnSync('ffmpeg', [
|
||||
'-hide_banner', '-loglevel', 'error', '-y', '-f', 'lavfi',
|
||||
'-i', 'color=c=blue:s=64x48:r=4:d=1', '-c:v', 'mpeg4', video,
|
||||
]);
|
||||
if (encoded.status !== 0) throw new Error(`ffmpeg failed: ${encoded.stderr}`);
|
||||
const { root } = await page.send('DOM.getDocument');
|
||||
const { nodeId } = await page.send('DOM.querySelector', {
|
||||
nodeId: root.nodeId, selector: 'input[type=file]',
|
||||
});
|
||||
await page.send('DOM.setFileInputFiles', { files: [video], nodeId });
|
||||
const extracted = await statusMatching(/video extracted/, 160);
|
||||
check(extracted !== null, 'the page uploads and extracts a video',
|
||||
extracted ?? (await page.eval(STATUS)));
|
||||
const uploadedFootage = await page.eval(`(async () => {
|
||||
const listed = await fetch('/api/footage').then(r => r.json());
|
||||
return listed.footage.some(f => f.label === 'browser-upload.mp4' && f.frames === 4);
|
||||
})()`);
|
||||
check(uploadedFootage, 'the uploaded footage is selectable');
|
||||
} finally {
|
||||
rmSync(uploadDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
check(page.logs.length === 0, 'no errors on the console',
|
||||
page.logs.slice(0, 3).join(' | '));
|
||||
} finally {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue