Port step 6: detect real footage with local MediaPipe

README.md

@@ -14,6 +14,14 @@ Animator Pro, where this started, but they are why the output looks right —
 modern conveniences belong in the workflow, not the output. See
 [docs/design.md](docs/design.md).
 
+## ClojureScript port
+
+The active port has reached [step 6 of the port plan](docs/port-plan.md): it plays
+the synthetic take and can load extracted real footage as a rotoscoped mouth.
+See [frontend/README.md](frontend/README.md) for setup and the **load frames**
+workflow. The rest of this README describes the older JS prototype, which still
+runs separately on port 8777.
+
 ## Run
 
 ```sh

docs/port-plan.md

@@ -2,6 +2,11 @@
 
 Self-contained. You should not need any prior conversation to execute this.
 
+**Implementation status (2026-09-27):** steps 0–6 are in the CLJS frontend.
+Step 6 reads extracted footage from the manifest, detects landmarks with local
+MediaPipe assets, and runs the same freeze path as the synthetic take. Step 7 is
+next: eyes, brows and the pixel-derived mouth interior.
+
 ## What arthur is
 
 A tool that turns live-action video into 2D animation that reads as

frontend/README.md

@@ -66,7 +66,7 @@ Then open **<http://localhost:8778/index.html>** — with the `/index.html`, not
 bare `/`. This shadow-cljs does no directory-index resolution, so `/` is a 404
 whatever the roots are.
 
-Four clips, on buttons in the transport:
+Four built-in clips, on buttons in the transport:
 
 | | |
 | --- | --- |
@@ -78,9 +78,28 @@ Four clips, on buttons in the transport:
 `take` and `locked` are the pair worth looking at together, because switching
 between them is the whole of what "stabilisation is a channel, not a mode" means.
 
-The demo scene itself is `src/arthur/demo/scene.edn`; the take is built in
+The demo scene itself is `src/arthur/demo/scene.edn`. Both the synthetic take
-`src/arthur/demo/take.cljs`, which is also the only place the seven stages are
+and real footage use `src/arthur/flow/take.cljs` for the measurement order and
-composed in order.
+`src/arthur/flow/freeze.cljs` for the landmark-to-channel conversion.
+
+### Real footage (port step 6)
+
+From the repo root, extract a clip, then click **load frames** in the CLJS app:
+
+```sh
+./extract.sh /path/to/clip.mov 12
+```
+
+This writes `frames/0001.png` onward, `audio.wav`, and `manifest.json` at the
+repo root. The manifest supplies the exact frame count, fps and audio path.
+Loading detects one face per frame, freezes the measured mouth into channels,
+and adds a button for the footage clip. Detection happens once when you load;
+playback only resolves channels and paints. Frames without a detection remain
+marked absent even though their neighbouring poses are used to condition the
+track. The stage stays 320×200 regardless of the footage dimensions.
+
+MediaPipe's JS, wasm and model are under `public/mediapipe/` and served locally.
+No CDN is used by this app. See that directory's README for provenance.
 
 Port 8778 is deliberately not 8777. `python3 serve.py` from the repo root still
 runs the old JS tool on 8777, and the two are meant to run side by side.

frontend/package-lock.json

@@ -8,6 +8,7 @@
       "name": "arthur-frontend",
       "version": "0.0.1",
       "dependencies": {
+        "@mediapipe/tasks-vision": "1.0.1",
         "react": "^18.3.1",
         "react-dom": "^18.3.1"
       },
@@ -15,6 +16,12 @@
         "shadow-cljs": "^2.28.21"
       }
     },
+    "node_modules/@mediapipe/tasks-vision": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-1.0.1.tgz",
+      "integrity": "sha512-rvRE2FmAZ6ZxKSw7wq+e+jQDpN3t1B/tD2mJz9SmAzb1msoDkd4dMoE4wAh8Z30Um0PQwLiHr9QtomhmXk3aUQ==",
+      "license": "Apache-2.0"
+    },
     "node_modules/asn1.js": {
       "version": "4.10.1",
       "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz",

frontend/package.json

@@ -9,6 +9,7 @@
     "browser": "node --experimental-websocket test/browser/take.mjs"
   },
   "dependencies": {
+    "@mediapipe/tasks-vision": "1.0.1",
     "react": "^18.3.1",
     "react-dom": "^18.3.1"
   },

frontend/public/index.html

@@ -19,22 +19,25 @@
       .stage { display: block; background: #12141c; }
       audio { display: none; }
       .transport { margin-top: 12px; width: 640px; }
-      .transport .row { display: flex; gap: 6px; align-items: center; }
+      .transport .row { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; }
       .transport .gap { flex: 1; }
       button {
         font: inherit; color: var(--fg); background: #1c1f2b;
         border: 1px solid #2b3040; padding: 3px 10px; cursor: pointer;
       }
       button:hover { background: #242836; }
+      button:disabled { opacity: .45; cursor: wait; }
       button.on { background: #3a4258; border-color: #556080; }
       .scrub { width: 100%; margin: 10px 0 6px; }
       .readout { display: flex; gap: 18px; opacity: .55; font-size: 12px; }
       .readout .warn { color: #d98f5a; opacity: 1; }
+      .load-status { margin-top: 6px; font-size: 12px; opacity: .75; }
       .note { opacity: .35; font-size: 12px; max-width: 640px; }
     </style>
   </head>
   <body>
     <div id="app"></div>
+    <script src="/mediapipe/vision_bundle.js"></script>
     <script src="/static/arthur/js/main.js"></script>
   </body>
 </html>

frontend/public/mediapipe/LICENSE

@@ -0,0 +1,218 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+===========================================================================
+For files under tasks/cc/text/language_detector/custom_ops/utils/utf/
+===========================================================================
+/*
+ * The authors of this software are Rob Pike and Ken Thompson.
+ *              Copyright (c) 2002 by Lucent Technologies.
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose without fee is hereby granted, provided that this entire notice
+ * is included in all copies of any software which is or includes a copy
+ * or modification of this software and in all copies of the supporting
+ * documentation for such software.
+ * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED
+ * WARRANTY.  IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY
+ * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY
+ * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE.
+ */

frontend/public/mediapipe/README.md

@@ -0,0 +1,17 @@
+# Local MediaPipe assets
+
+`vision_bundle.js` and the four wasm loader/binary files under `wasm/` come
+from `@mediapipe/tasks-vision` **1.0.1**, pinned in `frontend/package.json`.
+`FilesetResolver.forVisionTasks` uses the SIMD pair where supported and the
+no-SIMD pair elsewhere. The package and these files are Apache-2.0; see
+[LICENSE](LICENSE).
+
+`face_landmarker.task` is the official [Face Landmarker model](https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task).
+Its SHA-256 is
+`64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff`.
+
+These are served from `/mediapipe/` so detection needs no CDN at runtime. The
+browser bundle is loaded as a script before the CLJS app because Shadow CLJS
+cannot parse the package's CommonJS bundle (its dynamic `import()` is unsupported
+by the current compiler). `flow/detect.cljs` is the sole call site for its
+`Vision` global.

frontend/shadow-cljs.edn

@@ -13,12 +13,12 @@
                 [re-frame "1.4.3"]]
 
  ;; Dev server for the CLJS half, on a different port from serve.py (8777) so the
- ;; old tool and the port can run side by side — which is the whole point of
+ ;; old tool and the port can run side by side. js/ remains the source for the
- ;; keeping js/ around as the numeric oracle.
+ ;; measurement stages not yet ported.
  ;;
  ;; Two roots: `public` holds the host page, `..` is the repo root so the bundle
  ;; at /static/arthur/js/ resolves, and so manifest.json, audio.wav and frames/
- ;; are reachable when step 6 needs real footage.
+ ;; are reachable for real footage.
  ;;
  ;; THE ORDER IS LOAD-BEARING. The repo root has an index.html too — the old
  ;; tool's — so with `..` first, /index.html would quietly serve the prototype

frontend/src/arthur/core.cljs

@@ -4,6 +4,7 @@
   port-plan step 3: the hand-written scene plays at 30fps against audio, scrubs,
   and runs at ½× and ¼×."
   (:require [arthur.db :as db]
+            [arthur.events.footage]
             [arthur.events.playback]
             [arthur.subs.playback]
             [arthur.subs.render]

frontend/src/arthur/db.cljs

@@ -23,7 +23,7 @@
   because there is one clip per scene today. Copying them by hand into this table
   is how one of them comes to disagree with the scene it describes."
   [label scene store]
-  (merge {:label label :scene scene :store store}
+  (merge {:label label :scene scene :store store :audio "/audio.wav"}
          (select-keys scene [:fps :frames :width :height])))
 
 (def scenes
@@ -50,7 +50,9 @@
    ;; footage's. That is what deleting `makeXform` buys — the framing became a
    ;; transform on a node, so nothing downstream of the freeze knows the frame
    ;; size — and it is why ui/player no longer hardcodes 320x200.
-   :clip    (select-keys (:take scenes) [:fps :frames :width :height])
+   :clip    (select-keys (:take scenes) [:fps :frames :width :height :audio])
+
+   :footage {:id nil :label nil :loading? false :status nil}
 
    ;; --- transport ---
    ;;

frontend/src/arthur/demo/take.cljs

@@ -1,8 +1,8 @@
 (ns arthur.demo.take
   "The synthetic take: the whole vertical slice, with no video file in it.
 
-  This is port-plan step 5's deliverable and it is the first thing in the tree
+  This is port-plan step 5's deliverable. `flow/take` now composes the shared
-  that runs every stage in order —
+  measurement path for both this generator and real footage —
 
       synth  ──▶  measure/anchor  ──▶  condition/anchor
                         │                     │
@@ -26,10 +26,8 @@
   written two ways. That is the claim \"stabilisation is a channel, not a mode\"
   made checkable by eye: switching between them is a document edit, tier 1, and
   not one byte of tier 2 differs."
-  (:require [arthur.flow.condition :as condition]
+  (:require [arthur.flow.freeze :as freeze]
-            [arthur.flow.freeze :as freeze]
+            [arthur.flow.take :as take]
-            [arthur.flow.measure.anchor :as anchor]
-            [arthur.flow.measure.mouth :as mouth]
             [arthur.synth :as synth]))
 
 (def frames 229)
@@ -51,14 +49,6 @@
   ;; anchor-test exercises 0.5625 and this does not.
   1)
 
-(def ^:private knobs
-  "The prototype's own defaults, from index.html, so the first thing anyone sees
-  is the thing it was tuned to look like."
-  {:anchor-avg   2       ; smoothWin
-   :contour-avg  1       ; contourSmooth
-   :verts        8       ; vertices
-   :aperture-cut 0.12})  ; apertureThresh 120/1000
-
 (def analysis
   "Stage 2's output, synthesised. A seeded generator, so a wrong pose is
   reproducible rather than something that happened once."
@@ -66,26 +56,13 @@
 
 (def measured
   "Stages 3 and 4, in the order the stage split requires."
-  (delay
+  (delay (take/measure (assoc take/knobs :aspect aspect) {:dense @analysis})))
-    (let [dense    @analysis
-          fitted   (anchor/fit {:aspect aspect} {:dense dense})
-          anchored (condition/anchor knobs fitted)
-          rings    (mouth/measure {:aspect aspect}
-                                  {:dense dense :transforms (:transforms anchored)})]
-      (assoc anchored
-             :outer (condition/contours knobs (:outer rings))
-             :inner (condition/contours knobs (:inner rings))
-             ;; NOT smoothed. The aperture is the inner ring's own height read off
-             ;; as a scalar, and `contour avg` is a spatial-correspondence
-             ;; smoother over a ring; running it over a scalar would be a second,
-             ;; undocumented low-pass on a signal that then decides visibility.
-             :aperture (:aperture rings)))))
 
 (def params
   "What the freeze was handed. Public because it is the honest way to re-freeze at
   other settings — a test that built its own copy would be asserting about a clip
   nobody looks at."
-  (merge knobs
+  (merge take/knobs
          {:name   "take"
           :fps    fps
           :stage  stage

frontend/src/arthur/events/footage.cljs

@@ -0,0 +1,114 @@
+(ns arthur.events.footage
+  "Load and freeze extracted footage once, outside the playback loop."
+  (:require [arthur.events.playback :as pb]
+            [arthur.flow.detect :as detect]
+            [arthur.flow.ingest :as ingest]
+            [arthur.flow.take :as take]
+            [arthur.footage.store :as store]
+            [re-frame.core :as rf]))
+
+(defn- detect-frames! [manifest model]
+  (let [canvas  (.createElement js/document "canvas")
+        ctx     (.getContext canvas "2d")
+        raw     (atom [])
+        dims    (atom nil)
+        total   (:frames manifest)
+        load-id (.now js/Date)]
+    (js/Promise.
+     (fn [resolve reject]
+       (letfn [(next-frame [i]
+                 (if (= i total)
+                   (try
+                     (resolve (assoc (detect/fill-gaps @raw) :dimensions @dims))
+                     (catch :default error (reject error)))
+                   (-> (ingest/image! (ingest/frame-url manifest i load-id))
+                       (.then
+                        (fn [image]
+                          (let [wh [(.-naturalWidth image) (.-naturalHeight image)]]
+                            (when (and @dims (not= @dims wh))
+                              (throw (ex-info (str "frame " (inc i) " has different dimensions")
+                                              {:expected @dims :actual wh})))
+                            (reset! dims wh)
+                            (set! (.-width canvas) (first wh))
+                            (set! (.-height canvas) (second wh))
+                            (.drawImage ctx image 0 0)
+                            (swap! raw conj (detect/detect! model canvas))
+                            (when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total))
+                              (rf/dispatch [::progress (str "detecting " (inc i) "/" total)]))
+                            ;; Let the status and the transport paint between sync
+                            ;; MediaPipe calls, and release each decoded PNG.
+                            (js/setTimeout #(next-frame (inc i)) 0))))
+                       (.catch reject))))]
+         (next-frame 0))))))
+
+(defn- build-clip [manifest {:keys [dense detected dimensions missing first-real]}]
+  (let [[w h] dimensions
+        params (merge take/knobs
+                      {:name "footage" :fps (:fps manifest) :aspect (/ w h)
+                       :stage [320 200] :expose 2 :head :as-filmed
+                       :analysis (str "mediapipe:1.0.1/" (:source manifest))})
+        frozen (take/build params {:dense dense :detected detected})
+        scene (:scene frozen)]
+    (assoc (select-keys scene [:fps :frames :width :height])
+           :scene scene :store (:store frozen)
+           ;; Re-extraction often overwrites audio.wav under the same name. A new
+           ;; URL makes the element fetch the new sound when this clip is loaded.
+           :audio (str (ingest/audio-url manifest) "?v=" (.now js/Date))
+           :label (or (:source manifest) "footage")
+           :summary (str (:frames manifest) " frames · " w "×" h " · "
+                         (:fps manifest) " fps"
+                         (when (pos? missing)
+                           (str " · " missing " without a face"
+                                (when (pos? first-real)
+                                  (str " (first found on " (inc first-real) ")"))))))))
+
+(rf/reg-fx
+ ::begin!
+ (fn [_]
+   (-> (ingest/manifest!)
+       (.then (fn [manifest]
+                (rf/dispatch [::progress "loading MediaPipe…"])
+                (-> (detect/landmarker!)
+                    (.then (fn [model]
+                             (rf/dispatch [::progress "loading frames…"])
+                             (-> (detect-frames! manifest model)
+                                 (.then (fn [track] (build-clip manifest track)))))))))
+       (.then (fn [entry]
+                (let [id (store/install! entry)]
+                  (rf/dispatch [::loaded id (:summary entry)]))))
+       (.catch (fn [error]
+                 (js/console.error error)
+                 (rf/dispatch [::failed (or (.-message error) (str error))]))))))
+
+(rf/reg-event-fx
+ ::load
+ (fn [{:keys [db]} _]
+   (if (get-in db [:footage :loading?])
+     {}
+     {:db (assoc db :footage (assoc (:footage db) :loading? true
+                                   :status "reading manifest.json…"))
+      ::pb/pause! nil
+      ::begin! nil})))
+
+(rf/reg-event-db
+ ::progress
+ (fn [db [_ message]] (assoc-in db [:footage :status] message)))
+
+(rf/reg-event-db
+ ::failed
+ (fn [db [_ message]]
+   (assoc db :footage (assoc (:footage db)
+                            :loading? false :status (str "load failed: " message)))))
+
+(rf/reg-event-fx
+ ::loaded
+ (fn [{:keys [db]} [_ id summary]]
+   (let [clip (store/entry id)]
+     {:db (-> db
+              (assoc :scene/current id
+                     :clip (select-keys clip [:fps :frames :width :height :audio])
+                     :footage {:id id :label (:label clip)
+                               :loading? false :status summary})
+              (assoc-in [:playback :frame] 0)
+              (assoc-in [:playback :playing?] false))
+      ::pb/pause! nil})))

frontend/src/arthur/events/playback.cljs

@@ -11,7 +11,7 @@
   a small app-db. If global interceptors are added later they are added to a
   chain these events are excluded from, not to `reg-global-interceptor`."
   (:require [arthur.clock :as clock]
-            [arthur.db :as db]
+            [arthur.footage.store :as footage]
             [re-frame.core :as rf]))
 
 (defn- fps [db] (get-in db [:clip :fps]))
@@ -92,12 +92,14 @@
    ;; Changing the clip changes the resolver, the frame count and the rate all
    ;; at once, so the playhead goes home rather than being left pointing at a
    ;; frame the new clip may not have.
-   (let [{:keys [fps frames] :as clip} (get db/scenes id)]
+   (let [{:keys [fps frames] :as clip} (footage/entry id)]
      {:db (-> db
               (assoc :scene/current id)
               ;; The stage travels with the clip: two clips may be different
               ;; sizes, and the raster the loop paints into is the clip's, not
               ;; the app's.
-              (assoc :clip (select-keys clip [:fps :frames :width :height]))
+              (assoc :clip (select-keys clip [:fps :frames :width :height :audio]))
-              (assoc-in [:playback :frame] 0))
+              (assoc-in [:playback :frame] 0)
+              (assoc-in [:playback :playing?] false))
+      ::pause! nil
       ::seek! [fps frames 0]})))

frontend/src/arthur/flow/detect.cljs

@@ -0,0 +1,58 @@
+(ns arthur.flow.detect
+  "The only MediaPipe boundary. Landmarks leave here as ordinary CLJS values.")
+
+(defonce ^:private instance (atom nil))
+(defonce ^:private pending (atom nil))
+
+(defn landmarker!
+  "Initialize once, using the vendored wasm and the local model. CPU also works
+  in browsers where a GPU delegate initializes but fails on its first frame."
+  []
+  (if-let [model @instance]
+    (js/Promise.resolve model)
+    (or @pending
+        (if-let [vision (aget js/window "Vision")]
+          (let [resolver (aget vision "FilesetResolver")
+                landmarker (aget vision "FaceLandmarker")
+                ready (-> (.call (aget resolver "forVisionTasks") resolver "/mediapipe/wasm")
+                        (.then (fn [fileset]
+                                 (.call (aget landmarker "createFromOptions")
+                                        landmarker fileset
+                                        #js {:baseOptions
+                                             #js {:modelAssetPath "/mediapipe/face_landmarker.task"
+                                                  :delegate "CPU"}
+                                             :runningMode "IMAGE"
+                                             :numFaces 1})))
+                        (.then (fn [model]
+                                 (reset! instance model)
+                                 model))
+                        (.catch (fn [e]
+                                  (reset! pending nil)
+                                  (throw e))))]
+            (reset! pending ready)
+            ready)
+          (js/Promise.reject (js/Error. "local MediaPipe script did not load"))))))
+
+(defn detect!
+  "Detect one already-drawn canvas frame. nil means no face was detected."
+  [model canvas]
+  (when-let [face (aget (aget (.call (aget model "detect") model canvas)
+                              "faceLandmarks") 0)]
+    (mapv (fn [p] {:x (.-x p) :y (.-y p) :z (.-z p)}) face)))
+
+(defn fill-gaps
+  "Keep the detection mask while supplying real poses for measurement. A leading
+  gap uses the first observed face; later gaps hold the previous observed pose.
+  Freeze uses the mask to mark those frames absent in the channel blocks."
+  [raw]
+  (let [first-real (first (keep-indexed (fn [i frame] (when frame i)) raw))]
+    (when-not first-real
+      (throw (ex-info "no face found in any frame — check framing and light" {})))
+    {:dense (loop [i 0 last-face (nth raw first-real) out []]
+              (if (= i (count raw))
+                out
+                (let [face (or (nth raw i) last-face)]
+                  (recur (inc i) face (conj out face)))))
+     :detected (mapv some? raw)
+     :missing (count (remove some? raw))
+     :first-real first-real}))

frontend/src/arthur/flow/ingest.cljs

@@ -0,0 +1,44 @@
+(ns arthur.flow.ingest
+  "Read a pre-extracted take. The manifest owns timing and the exact frame count."
+  (:require [clojure.string :as str]))
+
+(defn- valid-manifest [m]
+  (let [fps (js/Number (:fps m))
+        frames (js/Number (:frames m))]
+    (when-not (and (js/Number.isFinite fps) (pos? fps)
+                   (js/Number.isInteger frames) (<= 1 frames 900)
+                   (string? (:dir m)) (seq (:dir m))
+                   (string? (:audio m)) (seq (:audio m)))
+      (throw (ex-info "manifest.json needs fps, frames (1–900), dir and audio" {:manifest m})))
+    (assoc m :fps fps :frames frames)))
+
+(defn manifest!
+  []
+  (-> (js/fetch "/manifest.json" #js {:cache "no-store"})
+      (.then (fn [response]
+               (when-not (.-ok response)
+                 (throw (ex-info "manifest.json was not found; run extract.sh first"
+                                 {:status (.-status response)})))
+               (.json response)))
+      (.then (fn [json] (valid-manifest (js->clj json :keywordize-keys true))))))
+
+(defn- asset-url [path]
+  ;; Manifest paths are relative to the extraction root, served at / in dev.
+  (str "/" (str/replace path #"^/+" "")))
+
+(defn audio-url [manifest]
+  (asset-url (:audio manifest)))
+
+(defn frame-url [manifest i load-id]
+  (str (asset-url (str (str/replace (:dir manifest) #"/+$" "")
+                       "/" (.padStart (str (inc i)) 4 "0") ".png"))
+       "?v=" load-id))
+
+(defn image! [src]
+  (js/Promise.
+   (fn [resolve reject]
+     (let [image (js/Image.)]
+       (set! (.-onload image) #(resolve image))
+       (set! (.-onerror image) #(reject (ex-info (str "frame did not load: " src)
+                                                {:src src})))
+       (set! (.-src image) src)))))

frontend/src/arthur/flow/take.cljs

@@ -0,0 +1,26 @@
+(ns arthur.flow.take
+  "The shared landmark-to-channel path for synthetic and detected takes."
+  (:require [arthur.flow.condition :as condition]
+            [arthur.flow.freeze :as freeze]
+            [arthur.flow.measure.anchor :as anchor]
+            [arthur.flow.measure.mouth :as mouth]))
+
+(def knobs
+  {:anchor-avg 2 :contour-avg 1 :verts 8 :aperture-cut 0.12})
+
+(defn measure
+  "Condition the anchor before measuring rings through it."
+  [{:keys [aspect] :as params} {:keys [dense detected]}]
+  (let [fitted   (anchor/fit {:aspect aspect} {:dense dense})
+        anchored (condition/anchor params fitted)
+        rings    (mouth/measure {:aspect aspect}
+                                {:dense dense :transforms (:transforms anchored)})]
+    (assoc anchored
+           :outer (condition/contours params (:outer rings))
+           :inner (condition/contours params (:inner rings))
+           :aperture (:aperture rings)
+           :detected detected)))
+
+(defn build
+  [params inputs]
+  (freeze/clip params (measure params inputs)))

frontend/src/arthur/footage/store.cljs

@@ -0,0 +1,16 @@
+(ns arthur.footage.store
+  "Loaded clip artifacts live outside app-db. The db keeps only their id."
+  (:require [arthur.db :as db]))
+
+(defonce ^:private loaded (atom nil))
+(defonce ^:private serial (atom 0))
+
+(defn install! [entry]
+  (let [id (keyword "footage" (str (swap! serial inc)))]
+    (reset! loaded (assoc entry :id id))
+    id))
+
+(defn entry [id]
+  (if (= id (:id @loaded))
+    @loaded
+    (get db/scenes id)))

frontend/src/arthur/subs/playback.cljs

@@ -16,3 +16,5 @@
 ;; take their size from the document rather than from a constant.
 (rf/reg-sub ::width    (fn [db _] (get-in db [:clip :width])))
 (rf/reg-sub ::height   (fn [db _] (get-in db [:clip :height])))
+(rf/reg-sub ::audio    (fn [db _] (get-in db [:clip :audio])))
+(rf/reg-sub ::footage  (fn [db _] (:footage db)))

frontend/src/arthur/subs/render.cljs

@@ -7,14 +7,14 @@
   CLOSURE that produces geometry at a frame. So a scene edit costs one
   recomputation here and a frame costs a lookup and a blit — and, crucially, the
   playhead is not an input, so moving it cannot invalidate this."
-  (:require [arthur.db :as db]
+  (:require [arthur.domain.palette :as pal]
-            [arthur.domain.palette :as pal]
             [arthur.domain.scene :as scene]
+            [arthur.footage.store :as footage]
             [re-frame.core :as rf]))
 
 (rf/reg-sub ::scene-id (fn [db _] (:scene/current db)))
 
-(rf/reg-sub ::scene (fn [db _] (get-in db/scenes [(:scene/current db) :scene])))
+(rf/reg-sub ::scene (fn [db _] (:scene (footage/entry (:scene/current db)))))
 
 (rf/reg-sub
  ::exposure
@@ -49,7 +49,7 @@
    ;; Tier 2, behind a handle, and never in app-db itself — what is in the db is
    ;; the id of the clip whose blocks these are. The hand-written demo has none;
    ;; the swarm is entirely dense.
-   (get-in db/scenes [(:scene/current db) :store])))
+   (:store (footage/entry (:scene/current db)))))
 
 (rf/reg-sub
  ::resolver

frontend/src/arthur/ui/shell.cljs

@@ -7,6 +7,7 @@
   why scrubbing at speed does not re-render the page."
   (:require [arthur.clock :as clock]
             [arthur.db :as db]
+            [arthur.events.footage :as footage]
             [arthur.events.playback :as pb]
             [arthur.subs.playback :as sub]
             [arthur.subs.render :as render]
@@ -16,16 +17,17 @@
 (def ^:private zoom 2)
 
 (defn- audio []
+  (let [src @(rf/subscribe [::sub/audio])]
     [:audio
    {:ref      #(when % (clock/attach! %))
-    :src      "/audio.wav"
+    :src      src
     :preload  "auto"
     ;; Transport state follows the ELEMENT, not the other way round: the audio
     ;; is the clock, so anything that can change its state — the end of the
     ;; file, the OS media keys, a browser autoplay block — has to be able to
     ;; correct the document rather than be contradicted by it.
     :on-play  #(rf/dispatch [::pb/play])
-    :on-pause #(rf/dispatch [::pb/pause])}])
+    :on-pause #(rf/dispatch [::pb/pause])}]))
 
 (defn- transport []
   (let [playing? @(rf/subscribe [::sub/playing?])
@@ -33,7 +35,8 @@
         frame    @(rf/subscribe [::sub/frame])
         frames   @(rf/subscribe [::sub/frames])
         fps      @(rf/subscribe [::sub/fps])
-        expose   @(rf/subscribe [::render/exposure])]
+        expose   @(rf/subscribe [::render/exposure])
+        {:keys [id label loading? status]} @(rf/subscribe [::sub/footage])]
     [:div.transport
      [:div.row
       [:button {:on-click #(rf/dispatch [::pb/toggle])}
@@ -52,6 +55,13 @@
          [:button {:class    (when (= id @(rf/subscribe [::render/scene-id])) "on")
                    :on-click #(rf/dispatch [::pb/select-scene id])}
           label]))
+      (when id
+        [:button {:class (when (= id @(rf/subscribe [::render/scene-id])) "on")
+                  :on-click #(rf/dispatch [::pb/select-scene id])}
+         (or label "footage")])
+      [:button {:disabled loading?
+                :on-click #(rf/dispatch [::footage/load])}
+       (if loading? "loading…" "load frames")]
       [:span.gap]
       (doall
        (for [r db/rates]
@@ -77,7 +87,8 @@
         [:span {:class (when (and drop (> drop 1.35)) "warn")}
          (str (.toFixed (or fps 0) 1) " paint/s"
               (when (and drop (pos? drop))
-                (str " · " (.toFixed drop 2) " frames/paint")))])]]))
+                (str " · " (.toFixed drop 2) " frames/paint")))])]
+     (when status [:div.load-status status])]))
 
 (defn- stage []
   ;; The canvas is the STAGE's size, and the stage is the clip's — not a constant
@@ -98,5 +109,5 @@
    [audio]
    [transport]
    [:p.note
-    "Audio-clocked: the frame is ⌊currentTime · fps⌋, so a slow loop drops "
+    "Extract a clip with extract.sh, then load frames. The manifest supplies "
-    "frames instead of drifting. ½× and ¼× are playbackRate."]])
+    "the frame count, audio and fps."]])