noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

multi fce stuff

Browse source
This commit is contained in:
Olive Vaughn 2026-09-29 02:34:53 -04:00
parent 49ece8dee6
commit ddabfbeaa8
32 changed files with 1309 additions and 700 deletions
Download patch file Download diff file Expand all files Collapse all files

28
clips/tests/test_api.py
View file

@ -327,6 +327,34 @@ class Tier2Tests(TestCase):
f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys[:2]}), f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys[:2]}),
content_type="application/json").status_code) content_type="application/json").status_code)
def test_source_roles_are_complete_and_unique_per_subject(self):
analysis = self.register_analysis()
keys = []
for subject in ("face-1", "face-2"):
for role in ("source/dense", "source/detected", "source/crops"):
desc = json.loads(block_descriptor(analysis, role=role))
desc["features"] = [subject]
descriptor = json.dumps(desc, sort_keys=True, separators=(",", ":"))
key = key_for(descriptor)
self.assertEqual(201, self.post("/api/blocks", {
"key": key, "descriptor": descriptor, "data": "AA==",
}).status_code)
keys.append(key)
def put(keys):
return self.client.put(f"/api/analyses/{analysis}",
json.dumps({"source_blocks": keys}),
content_type="application/json")
self.assertEqual(400, put(keys[:-1]).status_code)
self.assertEqual(400, put(keys + keys[:1]).status_code)
self.assertEqual(200, put(keys).status_code)
self.assertEqual(200, put(list(reversed(keys))).status_code)
self.assertEqual(409, put(keys[:3]).status_code)
self.assertEqual(set(keys), set(self.client.get(
f"/api/analyses/{analysis}").json()["source_blocks"]))
@override_settings(BLOB_ROOT=BLOB_DIR) @override_settings(BLOB_ROOT=BLOB_DIR)
class DocumentTests(TestCase): class DocumentTests(TestCase):

21
clips/views.py
View file

@ -433,12 +433,23 @@ def analysis_detail(request, key):
try: try:
keys = _body(request).get("source_blocks") keys = _body(request).get("source_blocks")
roles = {"source/dense", "source/detected", "source/crops"} roles = {"source/dense", "source/detected", "source/crops"}
if not isinstance(keys, list) or len(keys) != len(roles) or len(set(keys)) != len(roles): if (not isinstance(keys, list) or not keys
raise Bad("an analysis needs one block for each source role") or not all(isinstance(k, str) for k in keys) or len(set(keys)) != len(keys)):
raise Bad("an analysis needs distinct source block keys")
blocks = list(Block.objects.filter(key__in=keys)) blocks = list(Block.objects.filter(key__in=keys))
if (len(blocks) != len(roles) or {b.role for b in blocks} != roles if len(blocks) != len(keys) or any(b.analysis_id != key for b in blocks):
or any(b.analysis_id != key for b in blocks)): raise Bad("source blocks must exist and name this analysis")
raise Bad("source blocks must have distinct source roles and name this analysis") by_subject = {}
for block in blocks:
subjects = json.loads(block.descriptor).get("features", [])
if (not isinstance(subjects, list) or len(subjects) > 1
or any(not isinstance(s, str) or not s for s in subjects)):
raise Bad("a source block must name one subject")
# Older single-face analyses used an empty feature list.
by_subject.setdefault(tuple(subjects), []).append(block.role)
if any(len(found) != len(roles) or set(found) != roles
for found in by_subject.values()):
raise Bad("each subject needs one block for each source role")
with transaction.atomic(): with transaction.atomic():
row = Analysis.objects.select_for_update().get(key=key) row = Analysis.objects.select_for_update().get(key=key)
current = set(row.source_blocks.values_list("key", flat=True)) current = set(row.source_blocks.values_list("key", flat=True))

101
docs/multi-face-representation.md Normal file
View file

@ -0,0 +1,101 @@
# Multi-face representation
Status (2026-09-29): representation work complete. Reopen it for a concrete
requirement, rather than another round of abstract alternatives.
Implemented: each tracked face has a drawing timeline, placed by an ordinary
symbol instance. Timelines already provide local node names, independent playback,
and persistence. No new kind of scene container is needed.
```clojure
:timelines
{:main {:nodes {:root {:time {:mode :map :expose 2}}
:face {:parent :root :channels <source-to-stage placement>}
:face-1 {:kind :symbol :of :face-1 :parent :face :z "a0"}
:face-2 {:kind :symbol :of :face-2 :parent :face :z "a1"}}}
:face-1 {:nodes {:head {...} :mouth {:parent :head ...} ...}}
:face-2 {:nodes {:head {...} :mouth {:parent :head ...} ...}}}
:features
{:face-1/mouth {:subject :face-1 :timeline :face-1 :area :mouth
:nodes [:mouth :mouth-in]}
:face-2/mouth {:subject :face-2 :timeline :face-2 :area :mouth
:nodes [:mouth :mouth-in]}}
```
The example omits ordinary ids, frame counts and channel details.
## What belongs where
- A **subject** identifies a source track and supplies shared measurement settings.
Its timeline has the same id and contains its measured `:head`.
- A **feature** owns nodes in an explicitly named timeline. Its clip-level id is
qualified when generated; ownership is read from fields, never parsed from ids.
- A **node** has a local name. Parents, stencils and pose groups use local names too.
- An **instance** places and retimes a drawing. Its pose tracks can hold one face's
mouth while the other face continues moving.
Subject metadata and drawing timelines remain separate facts. Hand-drawn timelines
need no subject. Features retain explicit timeline references, so their locations
are not inferred from their labels.
Both filmed faces share one source-to-stage transform. Fitting them independently
would stack them at the center. Additional placement uses each instance's ordinary
channels. Cross-face draw order is the instances' `:z` order.
## Consequences
Regeneration updates the addressed timeline directly. There is no temporary swap
into `:main`, no special stage regeneration path, and no renaming of parents or
stencils. Composing a stage moves the take's root into the library and preserves
its child timelines. Settings appear once per tracked object, since all placements
read that same drawing.
Presence masks inside a subject use local feature names, matching measurement.
Freeze qualifies them when building block descriptors. Head and retained-source
blocks explicitly name their subject; otherwise two faces with identical detection
masks could produce different bytes under the same key. Analysis addresses also
include detection capacity and assignment settings, so old single-face detection
results cannot satisfy a new multi-face request.
Validation counts node ownership by `[timeline node]`. The server requires one
complete set of retained source roles **per subject**, rather than exactly three
blocks for the entire analysis.
Nested rectangles retain fractional sizes until rasterization. Rounding inside a
face timeline discarded small head-local pupils before the source-to-stage scale
was applied. This was a real rendering error missed by the earlier proposal's
coordinate-only benchmark; the regression now compares all mark extents as well.
## Verification and limits
`frontend/test/arthur/flow/multi_face_test.cljs` exercises two distinct subjects,
source block separation, detection and feature gaps, independent pose cuts and
regeneration, nested stage save/load, and equivalence to a flat single-face scene.
Existing geometry, raster, source, and regeneration tests cover the same paths.
`clips/tests/test_api.py` checks complete source roles per subject and immutability.
The browser suite checks rendering, pupils, playback, save/open, drawing and upload.
Assignment remains a nearest-centroid heuristic, with a version and distance gate
recorded in the analysis. Reordered detections, late arrivals and gaps are tested;
identity through crossings or long disappearances is not guaranteed. Assignment
happens before measurement, so correcting it requires measuring again.
This changes the freeze and retained-source contracts. It does not migrate older
flat captures; reanalyze their footage to use the new regeneration path. Existing
rendering and leaf codecs still understand their node/channel representation.
## Next steps
1. Commit the verified checkpoint: 319 frontend tests, 44 API tests, browser
checks, and app/test builds passed. Builds reported no warnings.
2. Exercise real two-person footage, especially crossings, late arrivals and
disappearances. Check assignment before treating the resulting geometry as
evidence about the representation.
3. Build performance-pose selection and instance-scoped picture rates, reusing
the existing held-frame lookup and pose groups.
4. Then build plate-drawing selection and independent tracing references.
The [timing handoff](timing-handoff.md) owns the detailed next implementation
sequence. Older flat captures need reanalysis unless a migration is separately
undertaken to preserve their authored edits.

37
docs/port-plan.md
View file

@ -2,7 +2,7 @@
Self-contained. You should not need any prior conversation to execute this. Self-contained. You should not need any prior conversation to execute this.
**Implementation status (2026-09-28):** steps 0–9 are in. Step 6 reads extracted **Implementation status (2026-09-29):** steps 0–9 are in. Step 6 reads extracted
footage, detects landmarks with local MediaPipe assets at full source cadence, and footage, detects landmarks with local MediaPipe assets at full source cadence, and
runs the same freeze path as the synthetic take. The scene time map can sample the runs the same freeze path as the synthetic take. The scene time map can sample the
frozen roto at a lower picture fps without changing source analysis, duration or frozen roto at a lower picture fps without changing source analysis, duration or
@ -13,9 +13,24 @@ absence intervals through measurement and freeze. Step 9 adds the Django backend
the three-tier split, content-addressed tier 2 with the detector version inside the three-tier split, content-addressed tier 2 with the detector version inside
every key, leaf addressing for tier 1, and project load/save that round-trips. every key, leaf addressing for tier 1, and project load/save that round-trips.
**Still open.** Step 8's parameter UI and scoped regeneration, and automatic Step 8 now has parameter controls and scoped regeneration from retained source.
per-feature detection. Everything under "Out, and do not build it" below, which Multi-face representation is complete: each tracked subject has a drawing
step 9 did not touch. timeline, placed by an ordinary symbol instance. See
[multi-face representation](multi-face-representation.md) for the implemented
model, verification and compatibility limits.
**Next, in order:** commit the verified checkpoint; exercise real two-person
footage, including crossings and disappearances; build performance-pose
Suggest/Keep/Drop and instance-scoped picture rates; then add plate-drawing
selection and independent tracing references. The
[timing handoff](timing-handoff.md) records current code and implementation order.
Reopen the representation only for a concrete requirement it cannot express.
**Still open:** real-footage identity validation, automatic per-feature detection,
the timing and tracing work above, and time-varying parameter settings. Older flat
captures need reanalysis for the new regeneration path; no migration is included.
The step descriptions below retain the original port scope; this status and the
linked handoffs describe subsequent work.
## What arthur is ## What arthur is
@ -355,7 +370,7 @@ stencilled by the sclera.
minus paint. The fixed pixel thresholds remain provisional; step 8 exposes their minus paint. The fixed pixel thresholds remain provisional; step 8 exposes their
parameters for tuning without changing the source track or picture timing. parameters for tuning without changing the source track or picture timing.
### 8 — knobs ### 8 — knobs — DONE for static settings and scoped regeneration
Build the parameter model before its UI. Define each parameter once with its Build the parameter model before its UI. Define each parameter once with its
default, validation, applicable area and regeneration dependencies. Store values default, validation, applicable area and regeneration dependencies. Store values
by stable subject and feature ID. Represent an eye pair as one group with one or by stable subject and feature ID. Represent an eye pair as one group with one or
@ -376,8 +391,9 @@ visible eye. This is an input format, not a control UI or an automatic detector.
Use leaf-addressable settings under the clip, subject, feature and optional Use leaf-addressable settings under the clip, subject, feature and optional
group. Retain source measurements so a setting change can regenerate affected group. Retain source measurements so a setting change can regenerate affected
channels without re-detecting footage. Time-varying parameter values and all channels without re-detecting footage. Static parameter controls and scoped
parameter controls are deferred to the UI pass. regeneration are implemented for takes and composed stages. Time-varying
parameter values remain deferred.
### 9 — backend — DONE ### 9 — backend — DONE
Django project, the `clips` app, models for Django project, the `clips` app, models for
@ -431,7 +447,7 @@ nobody should pre-empt by porting the old one.
## Two things to not foreclose ## Two things to not foreclose
Feature controls will later handle more than one face and editing presence. Feature controls now handle more than one face; editing presence remains future work.
The underlying identity, occlusion and group association model begins in step 8: The underlying identity, occlusion and group association model begins in step 8:
- **Presence is not visibility.** An occluded feature has *no value* on a frame, - **Presence is not visibility.** An occluded feature has *no value* on a frame,
@ -440,5 +456,6 @@ The underlying identity, occlusion and group association model begins in step 8:
- **Params carry stable identity.** A subject and its features keep their IDs - **Params carry stable identity.** A subject and its features keep their IDs
across observation gaps. A run of visible frames is not a new identity. across observation gaps. A run of visible frames is not a new identity.
The identity tracker, when it comes, should use the same pattern the iris and brow The current identity tracker uses nearest-centroid assignment. Validate it on
correspondences already use: vote across every frame rather than trusting one. real crossings and disappearances before choosing a more elaborate policy; the
iris and brow correspondence code offers whole-take voting as one option.

128
docs/timing-handoff.md Normal file
View file

@ -0,0 +1,128 @@
# Timing and frame-selection handoff
Status (2026-09-29): the multi-face representation is complete. Each face has a
local drawing timeline and an ordinary symbol instance. Keep that model; the next
feature is performance-pose selection, followed by plate drawings and tracing.
See [multi-face representation](multi-face-representation.md) for verification
and compatibility limits.
## Next steps, in order
1. **Commit the verified checkpoint.** Representation, scoped regeneration,
nested stage composition and source persistence are implemented and tested.
2. **Exercise real two-person footage.** Include crossings, late arrivals and
disappearances. Assignment is still a nearest-centroid heuristic; inspect
whether identities, landmarks and mouth crops stay together. Correcting an
assignment requires measuring again. Do not redesign the representation to
compensate for an assignment failure.
3. **Build performance-pose selection.** Propose frames from a target picture
rate, allow explicit Keep/Drop edits, and apply requests per instance. Reuse
the existing held-frame lookup and generated pose groups. Keep authored keys
and audio timing intact.
4. **Then build plate drawings and tracing.** Suggest drawing frames from head
displacement, allow manual choices, and give each cel an independently
selectable tracing reference.
Older flat captures need reanalysis for the new regeneration path. Migrating their
existing authored edits is separate work; it is not implemented by this checkpoint.
## Timing decisions
Keep the dense analyzed frames. Generated motion holds the most recent selected
source pose; removing a selected pose never deletes source data or shortens the
clip. Store edits in the animation's local frame space, so moving an instance
does not move its edits. Authored keys follow intentional instance retiming but
must not be quantized by a picture-rate request. Clip FPS and audio duration stay
fixed.
There are two selections with different owners, sharing held-frame lookup:
- **Performance poses:** propose a kept-frame list from the target picture rate,
then apply explicit keep/drop edits. A parent instance may request a lower
rate. Mouth outline, interior, teeth and visibility read the same selected
source frame; likewise each eye's coupled parts. Use group overrides when
needed, rather than a setting on every channel. Head motion currently has its
own anchor selection; do not silently put it under mouth timing.
- **Plate drawings:** start with frame 0, walk measured rigid head poses, and
suggest a frame when maximum landmark displacement from the last kept pose
exceeds tolerance. Let the artist add/remove frames. A removed drawing stays
stored so it can reappear if restored. This selection does not thin the mouth.
A target rate is approximate. Pin a useful closed-mouth pose at its actual frame,
even if that produces more changes than the target. Do not show a future pose
early to fit a grid. Manual drop wins over an automatic suggestion; make removal
of the only closed pose in a beat visible in the UI. Skip missing detections when
suggesting a replacement. Keep a frame-zero selection and hold the last selection
through the end. A skipped pose (hold), `[:vis] false` (hidden), and an absent
measurement remain different facts.
Store manual edits separately from generated proposals so changing the rate or
tolerance retains hand decisions. Selection edits change the document, not dense
blocks or analysis addresses. Verify save/open for every new field; extend leaf
handling and the relevant key whitelist if its storage location requires it.
## Current code: reuse these mechanisms
- `domain/pose.cljs` already has `prepare`, `held-frame` and `source-frame`.
Instance `:playback :tracks` map local change frames to held source frames,
keyed by pose group. Reuse this lookup; frame suggestion and Keep/Drop policy
are the missing layer. An explicit cut is not itself a complete selection UI.
- `freeze/performance-nodes` marks generated animated channels with
`:pose-sampled?` and local `:pose-group` names. This includes keyed visibility
as well as dense geometry. `:generated` remains provenance for regeneration.
- `timeline/channel-frame` already applies explicit pose choices and default
picture sampling to marked channels. Playback and export both use
`clip/resolver` with `:picture-fps`; there is no need for a second sampling
implementation. Export's pose count is still a rate-based estimate.
- The picture-rate option is currently passed through the resolver tree
unchanged. Instance-specific parent requests are still to be implemented.
Instance offset/rate must apply before selecting the local source pose.
- `pose/put-cut` and `remove-cut` currently address instances in `:main`.
A take's face instances are there, but a composed stage nests them inside a
shared source timeline. Make the editing scope explicit when adding nested
controls. A request on one outer placement must not rewrite the shared
drawing's playback settings for every placement.
- Generic root `:time :expose` still retimes descendants, and frozen takes still
store it. Paint nodes are rootless to escape it. When the selection path
replaces take picture cadence, remove that redundant quantization from the
take default; preserve intentional generic time maps. Moving exposure to
`:head` would still retime authored children.
- `freeze/head-mode` supports `:free` and `:anchored`. It keeps measured channels
dense and writes optional per-subject `:anchors` maps; it does **not** implement
`:per-plate` mode or materialize transform keys from `:kept`. Plate selection
should reuse held measured-frame addresses where appropriate, without
rerunning analysis or copying the measurements.
- `:over` hand corrections are currently refused by the channel reader. Their
future application belongs after generated pose selection.
## Performance-pose implementation sequence
1. Add pure proposal and Keep/Drop policy around the existing held-frame lookup.
Cover frame zero, nondivisible rates, manual precedence, missing poses and a
protected mouth closure. Preserve all source frames.
2. Feed instance requests and group selections into the existing channel read
path. Cover two faces, two differently timed placements of one source, nested
instances, coupled visibility/geometry, and authored keys at their normal time.
Use this same path for preview and export; keep audio duration unchanged.
3. Wire the performance strip's Suggest/Keep/Drop controls and persistence.
Replace the export pose estimate with the actual selection count. Retire the
take's redundant root exposure only when this path replaces its behavior.
## Plate drawings and tracing, afterward
The old suggestion algorithm is `js/pipeline.js:suggestPlateFrames`; the strip,
worksheet and tracing photo are in `js/app.js`. Port the useful policy over the
measured head poses and reuse held-frame lookup for the resulting drawing set.
Give a cel an editor-only source-frame reference, defaulting to its plate frame
but independently changeable. It may point to a frame omitted from either rendered
selection. Register the photo using that source frame's measured transform. The
old prototype coupled photo and cel addresses; independent tracing is new work.
The old iris socket lock, gaze origin, CLJS head anchors and registration pivot
are separate settings. Clarify what an "origin-lock" request means before adding
that control.
Keep the UI to two scopes: **performance poses** and **plate drawings**, each with
Suggest/Keep/Drop. Tracing reference and lock controls live with the cel or feature
they affect. No general keyframe framework is needed for this work.

13
frontend/src/arthur/demo/stage.cljs
View file

@ -67,12 +67,7 @@
:channels (cond-> {[:audio :gain] gain} :channels (cond-> {[:audio :gain] gain}
pan (assoc [:audio :pan] pan))}]) pan (assoc [:audio :pan] pan))}])
audio))] audio))]
(cond-> (assoc source :name name :width width :height height (assoc source :name name :width width :height height
:timelines {:main {:id :main :frames frames :nodes nodes} :timelines (assoc (:timelines source)
symbol (assoc original :id symbol)}) :main {:id :main :frames frames :nodes nodes}
;; These are the same tracked features in the shared symbol. Every symbol (assoc original :id symbol)))))
;; placement reads them; tuning one changes all instances.
(seq (:features source))
(assoc :features (into {} (map (fn [[id f]]
[id (assoc f :timeline symbol)]))
(:features source))))))

4
frontend/src/arthur/demo/take.cljs
View file

@ -84,8 +84,10 @@
:fps fps :fps fps
:aspect aspect})})) :aspect aspect})}))
(def subject :face-1)
(def frozen (def frozen
(delay (freeze/clip params @measured))) (delay (freeze/clip params {subject @measured})))
(def store (delay (:store @frozen))) (def store (delay (:store @frozen)))

13
frontend/src/arthur/domain/clip.cljs
View file

@ -101,7 +101,7 @@
:disc (let [[x y] (at (:cx op) (:cy op))] :disc (let [[x y] (at (:cx op) (:cy op))]
(assoc op :cx x :cy y :r (* scale (:r op)))) (assoc op :cx x :cy y :r (* scale (:r op))))
:rect (let [[x y] (at (:cx op) (:cy op))] :rect (let [[x y] (at (:cx op) (:cy op))]
(assoc op :cx x :cy y :size (js/Math.round (* scale (:size op))))) (assoc op :cx x :cy y :size (* scale (:size op))))
op))) op)))
(defn resolver (defn resolver
@ -158,13 +158,7 @@
(build root [] nil)))) (build root [] nil))))
(defn problems (defn problems
"Human-readable reasons this clip will not evaluate or will not save. Empty "Human-readable reasons this clip will not evaluate or save."
means it will.
The tracking identities are checked HERE and not in `domain/timeline`, because a
feature names nodes and only the root timeline's nodes were tracked into: a
library symbol is drawn, not detected. So `feature/problems` is asked about the
root, once, rather than about every timeline."
[clip] [clip]
(vec (vec
(concat (concat
@ -207,5 +201,4 @@
(not (contains? (:nodes tl) (:linked-to n))))] (not (contains? (:nodes tl) (:linked-to n))))]
(str "timeline " (pr-str tid) " audio " (pr-str id) (str "timeline " (pr-str tid) " audio " (pr-str id)
" links to missing node " (pr-str (:linked-to n)))) " links to missing node " (pr-str (:linked-to n))))
(when (map? (root clip)) (feature/problems clip))))
(feature/problems clip (nodes clip))))))

36
frontend/src/arthur/domain/feature.cljs
View file

@ -1,14 +1,13 @@
(ns arthur.domain.feature (ns arthur.domain.feature
"Stable tracked identities and explicit eye-pair settings associations. "Tracked subjects, feature ownership, and eye-pair settings.
Features name their timeline explicitly; node ids are local to that timeline."
These maps are the CLIP's — `:subjects`, `:features`, `:groups` — and not a
timeline's, because they describe what a camera saw and a library symbol is
drawn rather than detected. Rendering never reads them; it reads nodes and
channels. `problems` therefore takes the clip AND the node map to check
references against, rather than reaching for `(:nodes clip)`: a clip holds
several timelines and only the root one was tracked into."
(:require [arthur.domain.params :as params])) (:require [arthur.domain.params :as params]))
(defn owned
"A deterministic clip-level feature or group id. Nodes keep local names."
[subject role]
(keyword (subs (str subject) 1) (name role)))
(defn group-for [clip feature-id] (defn group-for [clip feature-id]
(first (filter (fn [[_ group]] (some #{feature-id} (:members group))) (first (filter (fn [[_ group]] (some #{feature-id} (:members group)))
(:groups clip)))) (:groups clip))))
@ -45,17 +44,14 @@
clip)) clip))
(defn problems (defn problems
"Check identity references and pair membership before storing a clip. "Check tracked identities and timeline-local node ownership."
[clip]
`nodes` is the node map a feature's `:nodes` are resolved against — the root
timeline's, passed in rather than looked up, so this namespace does not have to
know which timeline a clip plays."
[clip nodes]
(let [subjects (:subjects clip) (let [subjects (:subjects clip)
features (:features clip) features (:features clip)
groups (:groups clip) groups (:groups clip)
memberships (mapcat (comp :members val) groups) memberships (mapcat (comp :members val) groups)
node-owners (mapcat (comp :nodes val) features)] node-owners (for [[_ f] features n (:nodes f)]
[(:timeline f) n])]
(vec (vec
(concat (concat
(for [[id s] subjects :when (not= id (:id s))] (for [[id s] subjects :when (not= id (:id s))]
@ -63,6 +59,9 @@
(for [[id s] subjects (for [[id s] subjects
:when (not (params/valid-settings? :subject (or (:params s) {})))] :when (not (params/valid-settings? :subject (or (:params s) {})))]
(str "subject " (pr-str id) " has invalid settings")) (str "subject " (pr-str id) " has invalid settings"))
(for [[id _] subjects
:when (not (seq (get-in clip [:timelines id :nodes :head :measured])))]
(str "subject " (pr-str id) " has no measured head in its timeline"))
(for [[id f] features :when (not= id (:id f))] (for [[id f] features :when (not= id (:id f))]
(str "feature " (pr-str id) " has a different :id")) (str "feature " (pr-str id) " has a different :id"))
(for [[id f] features :when (not (contains? subjects (:subject f)))] (for [[id f] features :when (not (contains? subjects (:subject f)))]
@ -72,10 +71,11 @@
(for [[id f] features (for [[id f] features
:when (not (params/valid-settings? (:area f) (or (:params f) {})))] :when (not (params/valid-settings? (:area f) (or (:params f) {})))]
(str "feature " (pr-str id) " has invalid settings for " (pr-str (:area f)))) (str "feature " (pr-str id) " has invalid settings for " (pr-str (:area f))))
(for [[id f] features
:when (not (contains? (:timelines clip) (:timeline f)))]
(str "feature " (pr-str id) " names a missing timeline"))
(for [[id f] features node-id (:nodes f) (for [[id f] features node-id (:nodes f)
:let [owned-nodes (if-let [timeline (:timeline f)] :let [owned-nodes (get-in clip [:timelines (:timeline f) :nodes])]
(get-in clip [:timelines timeline :nodes])
nodes)]
:when (not (contains? owned-nodes node-id))] :when (not (contains? owned-nodes node-id))]
(str "feature " (pr-str id) " refers to missing node " (pr-str node-id))) (str "feature " (pr-str id) " refers to missing node " (pr-str node-id)))
(for [[id n] (frequencies node-owners) :when (> n 1)] (for [[id n] (frequencies node-owners) :when (> n 1)]

9
frontend/src/arthur/domain/raster.cljs
View file

@ -134,10 +134,11 @@
breathing." breathing."
([r cx cy size index] (fill-rect! r cx cy size index nil)) ([r cx cy size index] (fill-rect! r cx cy size index nil))
([{:keys [w h buf] :as r} cx cy size index over] ([{:keys [w h buf] :as r} cx cy size index over]
(when (>= size 1) (let [size (js/Math.round size)]
(let [x0 (js/Math.round (- cx (/ size 2))) (when (>= size 1)
y0 (js/Math.round (- cy (/ size 2)))] (let [x0 (js/Math.round (- cx (/ size 2)))
(let [ya (max 0 y0) yb (min h (+ y0 size)) y0 (js/Math.round (- cy (/ size 2)))
ya (max 0 y0) yb (min h (+ y0 size))
xa (max 0 x0) xb (min w (+ x0 size))] xa (max 0 x0) xb (min w (+ x0 size))]
(dotimes [iy (- yb ya)] (dotimes [iy (- yb ya)]
(dotimes [ix (- xb xa)] (dotimes [ix (- xb xa)]

45
frontend/src/arthur/domain/timeline.cljs
View file

@ -271,14 +271,8 @@
:rd rd}))))))) :rd rd})))))))
(defn- emit (defn- emit
"The draw op for a placed node, or nil when it has nothing to draw. A group "Emit geometry in the timeline's space. Rect sizes stay fractional until
never draws; it exists to carry a transform. rasterization, so enclosing symbol transforms can still scale them."
Three branches that rhyme, deliberately left as three: a poly writes vertices
into a buffer it was handed, a disc carries a scaled radius, a rect a rounded
pixel count. They are three different marks, and the shared skeleton is two
cheap lines each — folding them into one shape driven by a table would buy
those lines back by coupling this to whatever the table was for."
[{:keys [palette buf-for]} n {:keys [m rd]} base] [{:keys [palette buf-for]} n {:keys [m rd]} base]
(let [colour #(colour-index palette (rd [:style :color]))] (let [colour #(colour-index palette (rd [:style :color]))]
(case (:kind n) (case (:kind n)
@ -310,10 +304,7 @@
(when-not (ch/nothing? size) (when-not (ch/nothing? size)
(assoc base :kind :rect (assoc base :kind :rect
:cx (aget m 4) :cy (aget m 5) :cx (aget m 4) :cy (aget m 5)
;; ROUNDED, because :size is a pixel count: a scaled square :size (* size (node/mean-scale m))
;; 3.4px wide would be 3px on one frame and 4 on the next,
;; which reads as the pupil breathing. See raster/fill-rect!.
:size (js/Math.round (* size (node/mean-scale m)))
:color (colour)))) :color (colour))))
(throw (ex-info "node kind is not implemented" (throw (ex-info "node kind is not implemented"
@ -337,12 +328,11 @@
nodes)) nodes))
(defn- channel-frame (defn- channel-frame
"Only marked generated channels read picture cadence and instance pose cuts." "Anchors select measured frames; marked channels read instance pose choices."
[choices head-anchors nodes source-fps picture-fps id path c lf] [choices anchors nodes source-fps picture-fps id c lf]
(cond (cond
(and (= id :head) (contains? #{[:xform :pos] [:xform :rot] [:xform :scale]} path) (contains? anchors id)
(contains? head-anchors id)) (pose/held-frame (get anchors id) lf lf)
(pose/held-frame (get head-anchors id) lf lf)
(:pose-sampled? c) (:pose-sampled? c)
(pose/source-frame choices (pose/source-frame choices
@ -354,7 +344,7 @@
:else lf)) :else lf))
(defn- prepared-head-anchors [nodes] (defn- prepared-anchors [nodes]
(into {} (into {}
(for [[id n] nodes :when (seq (:anchors n))] (for [[id n] nodes :when (seq (:anchors n))]
[id (vec (sort-by first (:anchors n)))]))) [id (vec (sort-by first (:anchors n)))])))
@ -409,12 +399,12 @@
([tl f store palette pose-tracks opts] ([tl f store palette pose-tracks opts]
(let [nodes (nodes-of tl) (let [nodes (nodes-of tl)
choices (pose/prepare pose-tracks) choices (pose/prepare pose-tracks)
head-anchors (prepared-head-anchors nodes) anchors (prepared-anchors nodes)
{:keys [source-fps picture-fps]} opts {:keys [source-fps picture-fps]} opts
ord (order nodes)] ord (order nodes)]
(eval-into {:read (fn [id path c lf] (eval-into {:read (fn [id path c lf]
(ch/value-at c (channel-frame choices head-anchors nodes (ch/value-at c (channel-frame choices anchors nodes
source-fps picture-fps id path c lf) source-fps picture-fps id c lf)
store)) store))
:palette palette :palette palette
:mat-for (fn [_id] (node/mat)) :mat-for (fn [_id] (node/mat))
@ -473,7 +463,7 @@
([tl store palette pose-tracks {:keys [source-fps picture-fps]}] ([tl store palette pose-tracks {:keys [source-fps picture-fps]}]
(let [nodes (nodes-of tl) (let [nodes (nodes-of tl)
choices (pose/prepare pose-tracks) choices (pose/prepare pose-tracks)
head-anchors (prepared-head-anchors nodes) anchors (prepared-anchors nodes)
ord (order nodes) ord (order nodes)
rank (draw-rank nodes ord) rank (draw-rank nodes ord)
cursors (into {} cursors (into {}
@ -497,8 +487,8 @@
placed (volatile! {}) placed (volatile! {})
ctx {:read (fn [id path c lf] ctx {:read (fn [id path c lf]
(ch/sample! (get-in cursors [id path]) (ch/sample! (get-in cursors [id path])
(channel-frame choices head-anchors nodes (channel-frame choices anchors nodes
source-fps picture-fps id path c lf))) source-fps picture-fps id c lf)))
:palette palette :palette palette
:mat-for (fn [id] (get mats id)) :mat-for (fn [id] (get mats id))
:on-place (fn [id p] (vswap! placed assoc id p)) :on-place (fn [id p] (vswap! placed assoc id p))
@ -559,18 +549,19 @@
(into (for [[id n] nodes (into (for [[id n] nodes
p (node/problems n)] p (node/problems n)]
(str "node " (pr-str id) ": " p))) (str "node " (pr-str id) ": " p)))
;; Anchors re-address the node's measurement, regardless of its name.
(into (for [[id n] nodes (into (for [[id n] nodes
:let [anchors (:anchors n)] :let [anchors (:anchors n)]
:when (some? anchors) :when (some? anchors)
:when (not (and (= id :head) :when (not (and (map? anchors) (contains? anchors 0)
(map? anchors) (contains? anchors 0)
(integer? (:frames tl)) (integer? (:frames tl))
(every? #(and (integer? %) (<= 0 %) (every? #(and (integer? %) (<= 0 %)
(< % (:frames tl))) (< % (:frames tl)))
(concat (keys anchors) (vals anchors))) (concat (keys anchors) (vals anchors)))
(seq (:measured n))
(= (:channels n) (:measured n))))] (= (:channels n) (:measured n))))]
(str "node " (pr-str id) (str "node " (pr-str id)
": :anchors must start at frame 0, name valid measured frames, and read the measured head channels"))) ": :anchors must start at frame 0, name valid measured frames, and read that node's own measured channels")))
(into (for [k (remove timeline-keys (keys tl))] (into (for [k (remove timeline-keys (keys tl))]
(str "timeline has a field with no leaf to save it in: " (pr-str k)))) (str "timeline has a field with no leaf to save it in: " (pr-str k))))
(into (when-not (or (nil? (:frames tl)) (and (integer? (:frames tl)) (pos? (:frames tl)))) (into (when-not (or (nil? (:frames tl)) (and (integer? (:frames tl)) (pos? (:frames tl))))

154
frontend/src/arthur/events/footage.cljs
View file

@ -64,21 +64,32 @@
stream fps w h stream fps w h
(fn [i frame] (fn [i frame]
(.drawImage ctx frame 0 0) (.drawImage ctx frame 0 0)
(let [face (detect/detect! model canvas (ingest/frame-ms fps i)) ;; EVERY FACE ON THIS FRAME, each with its own mouth crop taken
ring (when face (mapv #(nth face %) lm/LIPS-INNER)) ;; while the frame's pixels are still on the canvas. Which of these
box (when ring (interior/crop ring [w h])) ;; detections belongs to which subject is not decided here — the
pixels (when box ;; answer needs the whole take — so all three vectors stay in
(.-data (.getImageData ctx (:x box) (:y box) ;; DETECTION ORDER and `detect/tracks` re-keys them afterwards.
(:w box) (:h box))))] (let [faces (detect/detect! model canvas (ingest/frame-ms fps i))
(swap! raw conj face) boxes (mapv (fn [face]
(let [crop (when box {:box box :data pixels})] (interior/crop (mapv #(nth face %) lm/LIPS-INNER)
(swap! crops conj crop) [w h]))
;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work faces)
;; either way, but done after the fact it is ten seconds of frame-crops (mapv (fn [box]
;; synchronous arithmetic with the main thread held and the (when box
;; frame counter frozen on its last value — which reads as the {:box box
;; decoder hanging, and was diagnosed as that twice. :data (.-data (.getImageData
(swap! inner conj (source/measure-crop take/knobs crop)))) ctx (:x box) (:y box)
(:w box) (:h box)))}))
boxes)]
(swap! raw conj faces)
(swap! crops conj frame-crops)
;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work
;; either way, but done after the fact it is ten seconds of
;; synchronous arithmetic with the main thread held and the
;; frame counter frozen on its last value — which reads as the
;; decoder hanging, and was diagnosed as that twice.
(swap! inner conj (mapv #(source/measure-crop take/knobs %)
frame-crops)))
(when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total)) (when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total))
(rf/dispatch [::progress (str "detecting " (inc i) "/" total)])) (rf/dispatch [::progress (str "detecting " (inc i) "/" total)]))
;; Yield, so the status and the transport paint between synchronous ;; Yield, so the status and the transport paint between synchronous
@ -86,30 +97,56 @@
(js/Promise. (fn [done] (js/setTimeout done 0))))))) (js/Promise. (fn [done] (js/setTimeout done 0)))))))
(.then (fn [_] (.then (fn [_]
(mark! (str "DECODE FINISHED — " (count @raw) " frames")) (mark! (str "DECODE FINISHED — " (count @raw) " frames"))
(let [gaps (detect/fill-gaps @raw)] (let [slots (detect/tracks @raw)
subjects (into {}
(map (fn [[id track]]
[id (assoc (detect/fill-gaps
(detect/pick @raw track))
:crops (detect/pick @crops track)
:interior (detect/pick @inner track)
:interior-settings take/knobs)]))
slots)]
(mark! "fill-gaps") (mark! "fill-gaps")
(assoc gaps :dimensions [w h] :crops @crops {:subjects subjects
:interior @inner :interior-settings take/knobs))))))) :dimensions [w h]
;; Frames where NOBODY was found, which is a fact about the
;; footage. A frame one of two faces is missing from is a gap
;; in that subject's own detection mask and is reported there.
:missing (count (remove seq @raw))
:first-real (first (keep-indexed (fn [i faces] (when (seq faces) i))
@raw))}))))))
(defn presence-for
"Select a subject's masks and express them in its local feature names.
Unqualified manifest ids refer to the first face, as in single-face footage."
[manifest subject]
(not-empty
(into {} (keep (fn [[id mask]]
(when (= (or (namespace id) "face-1") (subs (str subject) 1))
[(keyword (name id)) mask])))
(:presence manifest))))
(defn- build-clip [manifest detector (defn- build-clip [manifest detector
{:keys [dense detected dimensions interior missing first-real] {:keys [subjects dimensions missing first-real] :as source-inputs}]
:as source-inputs}]
(let [[w h] dimensions (let [[w h] dimensions
_ (mark! "build-clip: start") _ (mark! "build-clip: start")
frozen (take/footage manifest {:dense dense :detected detected with-presence (into {}
:dimensions dimensions :interior interior (map (fn [[id inputs]]
:presence (:presence manifest) [id (assoc inputs :presence (presence-for manifest id))]))
:detector detector}) subjects)
frozen (take/footage manifest {:dimensions dimensions
:subjects with-presence
:detector detector})
_ (mark! "build-clip: freeze") _ (mark! "build-clip: freeze")
built (:clip frozen) built (:clip frozen)
source-blocks (source/pack (:id (:analysis built)) source-inputs) source-blocks (source/pack-subjects (:id (:analysis built)) subjects)
_ (mark! "build-clip: pack source blocks")] _ (mark! "build-clip: pack source blocks")]
(assoc (select-keys built [:fps :width :height]) (assoc (select-keys built [:fps :width :height])
:frames (clip/frames built) :frames (clip/frames built)
:display-fps (:fps built) :display-fps (:fps built)
:clip built :store (:store frozen) :clip built :store (:store frozen)
:source-blocks source-blocks :source-blocks source-blocks
:source-inputs (assoc source-inputs :presence (:presence manifest)) :source-inputs (assoc source-inputs :subjects with-presence)
;; No cache-buster. The audio is a blob named by the hash of its own ;; No cache-buster. The audio is a blob named by the hash of its own
;; bytes, so re-extracting gives it a different URL rather than ;; bytes, so re-extracting gives it a different URL rather than
;; overwriting this one — which is what the `?v=` here used to work ;; overwriting this one — which is what the `?v=` here used to work
@ -120,14 +157,18 @@
:cid (or (:id manifest) "footage") :cid (or (:id manifest) "footage")
:summary (str (:frames manifest) " frames · " w "×" h " · " :summary (str (:frames manifest) " frames · " w "×" h " · "
(:fps manifest) " fps" (:fps manifest) " fps"
(when (> (count subjects) 1)
(str " · " (count subjects) " faces"))
(when (pos? missing) (when (pos? missing)
(str " · " missing " without a face" (str " · " missing " without a face"
(when (pos? first-real) (when (pos? first-real)
(str " (first found on " (inc first-real) ")")))))))) (str " (first found on " (inc first-real) ")"))))))))
(defn- with-default-interior! [analysis track] (defn- with-default-interior!
"Backfill one subject's retained interior measurements at the default knobs."
[analysis subject track]
(let [frames (count (:crops track)) (let [frames (count (:crops track))
key (source/interior-key analysis take/knobs frames)] key (source/interior-key analysis subject take/knobs frames)]
(-> (http/GET (str "/api/blocks/" key)) (-> (http/GET (str "/api/blocks/" key))
(.then (fn [block] (.then (fn [block]
(assoc track :interior (source/unpack-interior block take/knobs frames) (assoc track :interior (source/unpack-interior block take/knobs frames)
@ -137,16 +178,29 @@
(if (= 404 (:status (ex-data error))) track (throw error))))))) (if (= 404 (:status (ex-data error))) track (throw error)))))))
(defn saved-source! (defn saved-source!
"Restore retained source tracks by analysis id, for regeneration after open." "Restore retained source tracks by analysis id, for regeneration after open.
An analysis holds one set of blocks per tracked subject, so the count is a
multiple of `source/roles` rather than equal to it; `source/unpack` reads each
block's own descriptor to find out whose it is."
[key dimensions] [key dimensions]
(-> (http/GET (str "/api/analyses/" key)) (-> (http/GET (str "/api/analyses/" key))
(.then (fn [^js analysis] (.then (fn [^js analysis]
(let [keys (array-seq (.-source_blocks analysis))] (let [keys (array-seq (.-source_blocks analysis))]
(when (= (count keys) (count source/roles)) (when (and (seq keys) (zero? (mod (count keys) (count source/roles))))
(-> (js/Promise.all (-> (js/Promise.all
(into-array (map #(http/GET (str "/api/blocks/" %)) keys))) (into-array (map #(http/GET (str "/api/blocks/" %)) keys)))
(.then (fn [blocks] (source/unpack blocks dimensions))) (.then (fn [blocks] (source/unpack blocks dimensions)))
(.then (fn [track] (with-default-interior! key track)))))))) (.then (fn [{:keys [subjects]}]
(-> (js/Promise.all
(into-array
(map (fn [[id track]]
(.then (with-default-interior! key id track)
(fn [filled] [id filled])))
subjects)))
(.then (fn [pairs]
{:subjects (into {} (array-seq pairs))
:dimensions dimensions}))))))))))
(.catch (fn [error] (.catch (fn [error]
(if (= 404 (:status (ex-data error))) nil (throw error)))))) (if (= 404 (:status (ex-data error))) nil (throw error))))))
@ -154,19 +208,8 @@
(saved-source! (:id (take/analysis-for manifest detector)) (saved-source! (:id (take/analysis-for manifest detector))
[(:width manifest) (:height manifest)])) [(:width manifest) (:height manifest)]))
(defn measure-crops! (defn measure-one!
"Measure every retained crop's interior, ONE PER EVENT-LOOP TURN. "One subject's crops, ONE PER EVENT-LOOP TURN."
Done in a tight loop instead, it is ten seconds of synchronous arithmetic with
the main thread held and the frame counter frozen on its last value — which
reads as the decoder hanging, and was diagnosed as that twice.
Two callers, and the only difference between them is who is waiting: opening an
older analysis that predates the interior block backfills it behind a progress
line, and a knob drag that needs the pixels measured at new settings does the
same work with nobody watching, so it passes no `on-step`. The settings ride
back on the track, because a measurement and the knobs it was taken at are one
fact — `source/pack` will not address an interior block without them."
[settings track on-step] [settings track on-step]
(if (or (:interior track) (not (:crops track))) (if (or (:interior track) (not (:crops track)))
(js/Promise.resolve track) (js/Promise.resolve track)
@ -186,6 +229,29 @@
(catch :default error (reject error)))))] (catch :default error (reject error)))))]
(step 0))))))) (step 0)))))))
(defn measure-crops!
"Measure every retained crop's interior, one subject after another and one
crop per event-loop turn.
Done in a tight loop instead, it is ten seconds of synchronous arithmetic with
the main thread held and the frame counter frozen on its last value — which
reads as the decoder hanging, and was diagnosed as that twice.
Two callers, and the only difference between them is who is waiting: opening an
older analysis that predates the interior block backfills it behind a progress
line, and a knob drag that needs the pixels measured at new settings does the
same work with nobody watching, so it passes no `on-step`. The settings ride
back on each track, because a measurement and the knobs it was taken at are one
fact — `source/pack` will not address an interior block without them."
[settings track on-step]
(reduce (fn [chain [id one]]
(.then chain
(fn [acc]
(.then (measure-one! settings one on-step)
(fn [measured] (assoc-in acc [:subjects id] measured))))))
(js/Promise.resolve track)
(:subjects track)))
(rf/reg-fx (rf/reg-fx
::begin! ::begin!
(fn [footage-id] (fn [footage-id]

68
frontend/src/arthur/events/project.cljs
View file

@ -135,9 +135,12 @@
(.then (fn [_] (.then (fn [_]
(http/PUT (http/PUT
(str "/api/analyses/" (:id analysis)) (str "/api/analyses/" (:id analysis))
;; One set per tracked subject, in
;; the order `source/unpack` does
;; not depend on.
#js {:source_blocks #js {:source_blocks
(into-array (map #(get-in source-blocks [% :key]) (into-array
source/roles))}))))))) (source/block-keys source-blocks))})))))))
(.then (fn [_] (upload-missing! doc))) (.then (fn [_] (upload-missing! doc)))
(.then (fn [uploaded] (.then (fn [uploaded]
(-> (http/PUT (str "/api/projects/" pid) (-> (http/PUT (str "/api/projects/" pid)
@ -210,9 +213,9 @@
(defonce ^:private retained-source (atom nil)) (defonce ^:private retained-source (atom nil))
(defonce ^:private retained-interior (atom nil)) (defonce ^:private retained-interior (atom nil))
(defn- retained-interior! [analysis settings inputs] (defn- retained-interior! [analysis subject settings inputs]
(let [frames (count (:crops inputs)) (let [frames (count (:crops inputs))
block-key (source/interior-key analysis settings frames)] block-key (source/interior-key analysis subject settings frames)]
(-> (http/GET (str "/api/blocks/" block-key)) (-> (http/GET (str "/api/blocks/" block-key))
(.then (fn [block] (.then (fn [block]
(assoc inputs (assoc inputs
@ -220,10 +223,10 @@
:interior-key block-key))) :interior-key block-key)))
(.catch (fn [error] (.catch (fn [error]
(if (= 404 (:status (ex-data error))) (if (= 404 (:status (ex-data error)))
(-> (footage/measure-crops! settings (dissoc inputs :interior) nil) (-> (footage/measure-one! settings (dissoc inputs :interior) nil)
(.then (fn [measured] (.then (fn [measured]
(let [{:keys [key descriptor data]} (let [{:keys [key descriptor data]}
(source/interior-block analysis settings (source/interior-block analysis subject settings
(:interior measured))] (:interior measured))]
(-> (upload-missing! (-> (upload-missing!
#js {:blocks #js [#js {:key key #js {:blocks #js [#js {:key key
@ -241,8 +244,13 @@
(:promise @retained-source) (:promise @retained-source)
(let [promise (let [promise
(if (= "synth" (:detector analysis)) (if (= "synth" (:detector analysis))
;; The synthetic take tracks one face and regenerating it reads
;; that face's landmarks, so it arrives in the same shape real
;; footage does rather than in a flat one only this branch uses.
(js/Promise.resolve (js/Promise.resolve
{:dense (synth/synth-dense (:frames analysis) {:seed (:seed analysis)})}) {:subjects
{:face-1 {:dense (synth/synth-dense (:frames analysis)
{:seed (:seed analysis)})}}})
(-> (ingest/manifest! (:footage-id entry)) (-> (ingest/manifest! (:footage-id entry))
(.then (fn [manifest] (.then (fn [manifest]
(-> (footage/saved-source! (:id analysis) (-> (footage/saved-source! (:id analysis)
@ -251,29 +259,43 @@
(.then (fn [inputs] (.then (fn [inputs]
(when-not inputs (when-not inputs
(throw (ex-info "saved analysis has no source blocks" {}))) (throw (ex-info "saved analysis has no source blocks" {})))
(assoc inputs :presence (:presence manifest)))))))))] (update inputs :subjects
(fn [subjects]
(into {}
(map (fn [[id one]]
[id (assoc one :presence
(footage/presence-for
manifest id))]))
subjects))))))))))]
(do (do
(reset! retained-source {:id (:id analysis) :promise promise}) (reset! retained-source {:id (:id analysis) :promise promise})
promise)))))) promise))))))
(defn- inputs-for-edit! [entry edit inputs] (defn- inputs-for-edit!
(let [{clip :changed fids :features} (regenerate/plan (:clip entry) edit) "Bring the EDITED SUBJECT's retained pixel measurements up to the settings this
teeth (first (filter #(= :teeth (get-in clip [:features % :area])) fids))] edit needs. Only that subject's: a knob dragged on the second face does not
(if (and teeth (:crops inputs)) re-measure the first face's mouth."
[entry edit inputs]
(let [{clip :changed fids :features subject :subject} (regenerate/plan (:clip entry) edit)
teeth (first (filter #(= :teeth (get-in clip [:features % :area])) fids))
one (get-in inputs [:subjects subject])]
(if (and teeth (:crops one))
(let [settings (merge take/knobs (feature/effective-params clip teeth)) (let [settings (merge take/knobs (feature/effective-params clip teeth))
analysis (get-in clip [:analysis :id]) analysis (get-in clip [:analysis :id])
frames (count (:crops inputs)) frames (count (:crops one))
key (source/interior-key analysis settings frames)] key (source/interior-key analysis subject settings frames)
(if (and (:interior inputs) done (fn [measured] (assoc-in inputs [:subjects subject] measured))]
(or (= (:interior-key inputs) key) (if (and (:interior one)
(and (nil? (:interior-key inputs)) (or (= (:interior-key one) key)
(= key (source/interior-key analysis take/knobs frames))))) (and (nil? (:interior-key one))
(= key (source/interior-key analysis subject take/knobs frames)))))
(js/Promise.resolve inputs) (js/Promise.resolve inputs)
(if (= key (:key @retained-interior)) (.then (if (= key (:key @retained-interior))
(:promise @retained-interior) (:promise @retained-interior)
(let [promise (retained-interior! analysis settings inputs)] (let [promise (retained-interior! analysis subject settings one)]
(reset! retained-interior {:key key :promise promise}) (reset! retained-interior {:key key :promise promise})
promise)))) promise))
done)))
(js/Promise.resolve inputs)))) (js/Promise.resolve inputs))))
(rf/reg-fx (rf/reg-fx

17
frontend/src/arthur/flow/address.cljs
View file

@ -68,7 +68,7 @@
width, which is a visible difference on a mouth. Optional, because the synthetic width, which is a visible difference on a mouth. Optional, because the synthetic
take has no running mode to declare and an absent field is how the other take has no running mode to declare and an absent field is how the other
optional inputs already say \"not applicable\"." optional inputs already say \"not applicable\"."
[{:keys [detector version source footage frames fps aspect seed mode]}] [{:keys [detector version source footage frames fps aspect seed mode tracking]}]
(when-not (and (string? detector) (seq detector) (string? version) (seq version)) (when-not (and (string? detector) (seq detector) (string? version) (seq version))
(throw (ex-info "an analysis names its detector and the detector's VERSION: an upgrade that silently reuses old landmarks is the failure content addressing exists to prevent" (throw (ex-info "an analysis names its detector and the detector's VERSION: an upgrade that silently reuses old landmarks is the failure content addressing exists to prevent"
{:detector detector :version version}))) {:detector detector :version version})))
@ -81,7 +81,8 @@
source (assoc :source source) source (assoc :source source)
footage (assoc :footage footage) footage (assoc :footage footage)
seed (assoc :seed seed) seed (assoc :seed seed)
mode (assoc :mode mode)))) mode (assoc :mode mode)
tracking (assoc :tracking tracking))))
(defn analysis (defn analysis
"An analysis record with its `:id` filled in. The record is tier 1 — it says "An analysis record with its `:id` filled in. The record is tier 1 — it says
@ -94,12 +95,9 @@
;; the observation masks ;; the observation masks
(defn- feature-name (defn- feature-name
"A feature id as a descriptor string. `canon` refuses a keyword value on "An explicit subject or feature id as a descriptor string."
purpose — so that \"mouth\" and :mouth cannot address the same block — and this
is the naming it insists happens at the call site. `nil` is the whole face,
which is what a track with no feature follows."
[id] [id]
(if id (subs (str id) 1) "face")) (subs (str id) 1))
(defn observation (defn observation
"A digest of exactly the absence data one block reads. "A digest of exactly the absence data one block reads.
@ -111,10 +109,9 @@
block reads `:eye-r` and `:eye-l`'s presence and no other feature's, so a gap in block reads `:eye-r` and `:eye-l`'s presence and no other feature's, so a gap in
one brow does not rewrite the mouth's address for nothing. one brow does not rewrite the mouth's address for nothing.
`nil` features mean the track follows the whole face's detection rather than a Head tracks name their subject and follow its detection mask."
feature's presence, which is what the head's three blocks do."
[features {:keys [detected presence]}] [features {:keys [detected presence]}]
(let [wanted (sort-by str (distinct (keep identity features))) (let [wanted (sort-by str (distinct features))
masks (into {} (map (fn [id] [id (mapv boolean (get presence id))])) masks (into {} (map (fn [id] [id (mapv boolean (get presence id))]))
(filter #(contains? presence %) wanted))] (filter #(contains? presence %) wanted))]
(when (or detected (seq masks)) (when (or detected (seq masks))

122
frontend/src/arthur/flow/detect.cljs
View file

@ -4,6 +4,10 @@
(defonce ^:private instance (atom nil)) (defonce ^:private instance (atom nil))
(defonce ^:private pending (atom nil)) (defonce ^:private pending (atom nil))
(def settings
"Detection and assignment inputs, also included in the analysis address."
{:max-faces 4 :assignment "nearest-centroid-v1" :gate 0.2})
(defn discard! (defn discard!
"Throw away the cached landmarker so the next run builds a fresh one. "Throw away the cached landmarker so the next run builds a fresh one.
@ -43,7 +47,7 @@
#js {:modelAssetPath "/static/mediapipe/face_landmarker.task" #js {:modelAssetPath "/static/mediapipe/face_landmarker.task"
:delegate "CPU"} :delegate "CPU"}
:runningMode "VIDEO" :runningMode "VIDEO"
:numFaces 1}))) :numFaces (:max-faces settings)})))
(.then (fn [model] (.then (fn [model]
(reset! instance model) (reset! instance model)
model)) model))
@ -55,8 +59,8 @@
(js/Promise.reject (js/Error. "local MediaPipe script did not load")))))) (js/Promise.reject (js/Error. "local MediaPipe script did not load"))))))
(defn detect! (defn detect!
"Detect one already-drawn canvas frame at its own time in the take. nil means no "Detect one already-drawn canvas frame at its own time in the take. An empty
face was detected. vector means no face was detected.
`at-ms` IS THE FRAME'S REAL PRESENTATION TIME, and all three words are `at-ms` IS THE FRAME'S REAL PRESENTATION TIME, and all three words are
load-bearing. In VIDEO mode the graph is a tracker: it runs face DETECTION only load-bearing. In VIDEO mode the graph is a tracker: it runs face DETECTION only
@ -75,14 +79,95 @@
against 0.013), because a tracker told that every frame is 1ms apart expects a against 0.013), because a tracker told that every frame is 1ms apart expects a
face that has barely moved." face that has barely moved."
[model canvas at-ms] [model canvas at-ms]
(when-let [face (aget (aget (.call (aget model "detectForVideo") model canvas at-ms) (let [faces (aget (.call (aget model "detectForVideo") model canvas at-ms)
"faceLandmarks") 0)] "faceLandmarks")]
(mapv (fn [p] {:x (.-x p) :y (.-y p) :z (.-z p)}) face))) (mapv (fn [face] (mapv (fn [p] {:x (.-x p) :y (.-y p) :z (.-z p)}) face))
(array-seq faces))))
;; ---------------------------------------------------------------------------
;; which face is which
;;
;; MediaPipe hands back a LIST, and a list has an order rather than an identity.
;; Nothing in the API promises that slot 0 is the same person on frame 41 as on
;; frame 40 — and when two faces cross, or one is briefly lost and re-detected,
;; it is not. Left unassigned, the two faces' geometry would swap mid-shot inside
;; one dense block, which reads as both heads snapping and is invisible in any
;; per-frame assertion.
;;
;; So identity is assigned HERE, once, by nearest centroid to each track's last
;; known position. Greedy and cheap: a handful of faces, one pass, and the
;; ordering it produces is the `:subjects` map every later stage keys on.
(defn- centroid [face]
(let [n (count face)]
{:x (/ (reduce + (map :x face)) n)
:y (/ (reduce + (map :y face)) n)}))
(defn- distance [a b]
(let [dx (- (:x a) (:x b)) dy (- (:y a) (:y b))]
(js/Math.sqrt (+ (* dx dx) (* dy dy)))))
(defn assign
"Per-frame lists of faces -> one vector per TRACK of `detection index or nil`.
INDICES AND NOT FACES, because a detection is more than its landmarks: the
mouth crop and the pixel measurement taken beside it on the same frame have to
follow the same face, and handing back the index is what lets one assignment
re-key all three. Whoever holds the per-frame lists does the lookup.
A track is claimed by the unclaimed detection nearest its last known centroid,
nearest pair first, so a frame where MediaPipe swaps its slot order does not
swap the tracks. A detection that matches no existing track within `gate`
starts a new one — which is how a second person walking into the shot on frame
200 gets their own subject instead of stealing the first one's.
`gate` is in normalised image units: two centroids closer than this on
consecutive frames are one face moving, and further apart are not."
([frames] (assign frames (:gate settings)))
([frames gate]
(let [step (fn [[rows last] faces]
(let [cs (mapv centroid faces)
pairs (sort-by :d
(for [[t at] (map-indexed vector last)
:when at
[d c] (map-indexed vector cs)
:let [gap (distance at c)]
:when (< gap gate)]
{:d gap :track t :face d}))
claim (reduce (fn [{:keys [by-track taken] :as acc}
{:keys [track face]}]
(if (or (contains? by-track track)
(contains? taken face))
acc
{:by-track (assoc by-track track face)
:taken (conj taken face)}))
{:by-track {} :taken #{}}
pairs)
opened (map-indexed (fn [i d] [(+ (count last) i) d])
(remove (:taken claim) (range (count faces))))
by-track (into (:by-track claim) opened)
width (+ (count last) (count opened))]
[(conj rows (mapv by-track (range width)))
(mapv (fn [t] (if-let [d (get by-track t)]
(nth cs d)
(nth last t nil)))
(range width))]))
[rows _] (reduce step [[] []] frames)
width (reduce max 0 (map count rows))]
;; Transposed to column-major: one vector per subject, padded to the final
;; width so a track that opened late still spans the whole take.
(mapv (fn [t] (mapv (fn [row] (nth row t nil)) rows))
(range width)))))
(defn fill-gaps (defn fill-gaps
"Keep the detection mask while supplying real poses for measurement. A leading "Keep the detection mask while supplying real poses for measurement. A leading
gap uses the first observed face; later gaps hold the previous observed pose. gap uses the first observed face; later gaps hold the previous observed pose.
Freeze uses the mask to mark those frames absent in the channel blocks." Freeze uses the mask to mark those frames absent in the channel blocks.
ONE TRACK, which is one subject: the mask says when THIS face was on screen,
and two faces in a shot have two of them. A frame where the second person has
not walked in yet is a frame their track is absent on, which is the same fact
as a frame nobody was found on and needs no second mechanism."
[raw] [raw]
(let [first-real (first (keep-indexed (fn [i frame] (when frame i)) raw))] (let [first-real (first (keep-indexed (fn [i frame] (when frame i)) raw))]
(when-not first-real (when-not first-real
@ -95,3 +180,26 @@
:detected (mapv some? raw) :detected (mapv some? raw)
:missing (count (remove some? raw)) :missing (count (remove some? raw))
:first-real first-real})) :first-real first-real}))
(defn subject-id
"Track index -> the subject that owns it. `:face-1` is the first face found,
which is also the id a single-face take has always used."
[i]
(keyword (str "face-" (inc i))))
(defn tracks
"Per-frame detection lists -> `{:face-1 <slots>, …}`, one entry per tracked
face, each a per-frame detection index or nil."
[frames]
(let [assigned (assign frames)]
(when (empty? assigned)
(throw (ex-info "no face found in any frame — check framing and light" {})))
(into {} (map-indexed (fn [i slots] [(subject-id i) slots])) assigned)))
(defn pick
"One track's slots applied to anything measured PER DETECTION — the landmarks,
the mouth crop, the pixel measurement taken beside it. All three were recorded
in detection order on a frame where nobody yet knew whose face was whose, and
this is the single lookup that turns any of them into one subject's track."
[per-frame slots]
(mapv (fn [row slot] (when slot (nth row slot))) per-frame slots))

492
frontend/src/arthur/flow/freeze.cljs
View file

@ -34,6 +34,7 @@
decisions about the mouth cavity, blink and teeth without thinning geometry." decisions about the mouth cavity, blink and teeth without thinning geometry."
(:require [arthur.domain.channel :as ch] (:require [arthur.domain.channel :as ch]
[arthur.domain.clip :as clip] [arthur.domain.clip :as clip]
[arthur.domain.feature :as feature]
[arthur.domain.geom :as geom] [arthur.domain.geom :as geom]
[arthur.domain.ring :as ring] [arthur.domain.ring :as ring]
[arthur.flow.address :as address])) [arthur.flow.address :as address]))
@ -82,7 +83,7 @@
ABSENCE IS PER TRACK, and `:features` is what says whose. Each track names the ABSENCE IS PER TRACK, and `:features` is what says whose. Each track names the
feature it follows, so one occluded eye can be absent while its partner still feature it follows, so one occluded eye can be absent while its partner still
has a value; `nil` means the track follows the whole face's detection and no has a value; a subject id means the track follows whole-face detection and no
feature, which is what the head's blocks do. `absent?` is then asked feature, which is what the head's blocks do. `absent?` is then asked
`(absent? feature f)` and never about a track index. `(absent? feature f)` and never about a track index.
@ -248,31 +249,44 @@
(def ^:private head-modes #{:free :anchored}) (def ^:private head-modes #{:free :anchored})
(defn head-mode (defn head-mode
"Keep the measured transform dense; optionally hold chosen source frames. "Keep a subject's measured transform dense; optionally hold chosen source
frames.
A nil anchor map reads measured frame f at frame f (free movement). A nil anchor map reads measured frame f at frame f (free movement).
`{0 12}` locks to the measured transform of source frame 12. `{0 12, 40 42}` `{0 12}` locks to the measured transform of source frame 12. `{0 12, 40 42}`
cuts to source frame 42 at local frame 40. The same map selects position, cuts to source frame 42 at local frame 40. The same map selects position,
rotation and scale, so the head and registered photo cannot drift apart. rotation and scale, so the head and registered photo cannot drift apart.
No analysis block or authored face placement changes." No analysis block or authored face placement changes.
[{:keys [mode anchors]} {:keys [clip]}]
ONE SUBJECT AT A TIME when `:subject` is given, and EVERY subject when it is
not. Two faces in one shot were filmed together and are posed apart: choosing
frame 12 for the second face must leave the first one running, and it does,
because an anchor map lives on that subject's own head node and
`domain/timeline` reads anchors off whatever node carries them."
[{:keys [subject mode anchors]} {:keys [clip]}]
(when-not (contains? head-modes mode) (when-not (contains? head-modes mode)
(throw (ex-info "head mode must be free or anchored" (throw (ex-info "head mode must be free or anchored"
{:mode mode :modes head-modes}))) {:mode mode :modes head-modes})))
(when (and (= mode :free) (some? anchors)) (when (and (= mode :free) (some? anchors))
(throw (ex-info "free head motion has no anchors" {:anchors anchors}))) (throw (ex-info "free head motion has no anchors" {:anchors anchors})))
(let [frames (clip/frames clip)] (when (and subject (not (contains? (:subjects clip) subject)))
(when (and (= mode :anchored) (throw (ex-info "head mode names a subject this clip did not track"
(not (and (map? anchors) (contains? anchors 0) {:subject subject :subjects (vec (sort-by str (keys (:subjects clip))))})))
(every? #(and (integer? %) (<= 0 %) (< % frames)) (reduce
(concat (keys anchors) (vals anchors)))))) (fn [c sid]
(throw (ex-info "anchored head needs a frame-zero key and valid source frames" (let [frames (get-in c [:timelines sid :frames])]
{:anchors anchors :frames frames}))) (when (and (= mode :anchored)
(update-in clip [:timelines clip/root-id :nodes :head] (not (and (map? anchors) (contains? anchors 0)
(fn [n] (every? #(and (integer? %) (<= 0 %) (< % frames))
(cond-> (assoc n :channels (:measured n)) (concat (keys anchors) (vals anchors))))))
(= mode :anchored) (assoc :anchors anchors) (throw (ex-info "anchored head needs a frame-zero key and valid source frames"
(= mode :free) (dissoc :anchors)))))) {:subject sid :anchors anchors :frames frames})))
(update-in c [:timelines sid :nodes :head]
(fn [n]
(cond-> (assoc n :channels (:measured n))
(= mode :anchored) (assoc :anchors anchors)
(= mode :free) (dissoc :anchors))))))
clip (if subject [subject] (sort-by str (keys (:subjects clip))))))
;; --------------------------------------------------------------------------- ;; ---------------------------------------------------------------------------
;; the aperture, onto [:vis] of :mouth-in ;; the aperture, onto [:vis] of :mouth-in
@ -323,53 +337,56 @@
:brow-r :brow-r :brow-l :brow-l}) :brow-r :brow-r :brow-l :brow-l})
(defn- performance-nodes (defn- performance-nodes
"Mark the generated moving channels that may read an instance's pose choice. "Mark channels that read the containing instance's pose choices."
Provenance helps identify them here; the renderer reads only :pose-sampled?."
[nodes] [nodes]
(into {} (into {}
(map (fn [[id n]] (map (fn [[id n]]
[id (cond-> n [id (if-let [group (get pose-groups id)]
(contains? pose-groups id) (-> n
(assoc :pose-group (get pose-groups id)) (assoc :pose-group group)
(contains? pose-groups id) (update :channels
(update :channels (fn [channels]
(fn [channels] (into {}
(into {} (map (fn [[path ch]]
(map (fn [[path ch]] [path (cond-> ch
[path (cond-> ch (and (:generated ch) (:animated? ch))
(and (:generated ch) (:animated? ch)) (assoc :pose-sampled? true))]))
(assoc :pose-sampled? true))])) channels))))
channels))))])) n)]))
nodes)) nodes))
;; --------------------------------------------------------------------------- ;; ---------------------------------------------------------------------------
;; the face, onto the stage ;; the face, onto the stage
(defn- motion-placement (defn- motion-points
"An editable default framing for a real take. Fit every drawn face feature's "Every source-space point one subject's drawn features visit over the shot,
observed motion inside the stage, including raised brows and a moving mouth." including raised brows and a moving mouth."
[[w h] {:keys [rigid transforms outer eyes brows detected]}] [{:keys [rigid transforms outer eyes brows detected]}]
(let [points (mapcat (fn [i] (mapcat (fn [i]
(when (or (nil? detected) (nth detected i)) (when (or (nil? detected) (nth detected i))
(let [local (concat (nth outer i) (let [local (concat (nth outer i)
(when eyes (concat (nth (:lash-r eyes) i) (when eyes (concat (nth (:lash-r eyes) i)
(nth (:lash-l eyes) i))) (nth (:lash-l eyes) i)))
(when brows (concat (nth (:ring-r brows) i) (when brows (concat (nth (:ring-r brows) i)
(nth (:ring-l brows) i))))] (nth (:ring-l brows) i))))]
(concat (nth rigid i) (concat (nth rigid i)
(geom/apply-sim-all (invert (nth transforms i)) (geom/apply-sim-all (invert (nth transforms i))
local))))) local)))))
(range (count outer))) (range (count outer))))
xs (map :x points)
ys (map :y points) (defn- fit-points
x0 (reduce min xs) "Source-space points -> the placement that puts their bounding box on stage."
x1 (reduce max xs) [[w h] points]
y0 (reduce min ys) (let [xs (map :x points)
y1 (reduce max ys) ys (map :y points)
cx (/ (+ x0 x1) 2) x0 (reduce min xs)
cy (/ (+ y0 y1) 2) x1 (reduce max xs)
k (min (/ (* 0.8 w) (max 1e-9 (- x1 x0))) y0 (reduce min ys)
(/ (* 0.8 h) (max 1e-9 (- y1 y0))))] y1 (reduce max ys)
cx (/ (+ x0 x1) 2)
cy (/ (+ y0 y1) 2)
k (min (/ (* 0.8 w) (max 1e-9 (- x1 x0)))
(/ (* 0.8 h) (max 1e-9 (- y1 y0))))]
{[:xform :anchor] (ch/framed [cx cy]) {[:xform :anchor] (ch/framed [cx cy])
[:xform :scale] (ch/framed [k k]) [:xform :scale] (ch/framed [k k])
[:xform :pos] (ch/framed [(- (/ w 2) cx) (- (/ h 2) cy)])})) [:xform :pos] (ch/framed [(- (/ w 2) cx) (- (/ h 2) cy)])}))
@ -407,26 +424,33 @@
POSITION puts the anchor a QUARTER of the way down the stage, because the rigid POSITION puts the anchor a QUARTER of the way down the stage, because the rigid
landmarks are eyes and nose — the upper middle of a face — so a quarter down landmarks are eyes and nose — the upper middle of a face — so a quarter down
leaves the jaw and the mouth on the stage. Whatever hangs off is clipped, which leaves the jaw and the mouth on the stage. Whatever hangs off is clipped, which
is not a feature to add: every fill in `domain/raster` clamps already." is not a feature to add: every fill in `domain/raster` clamps already.
[{:keys [stage fit-motion?]} {:keys [ref] :as inputs}]
(if fit-motion?
(motion-placement stage inputs)
(let [[w h] stage
c (geom/centroid ref)
span (- (reduce max (map :x ref)) (reduce min (map :x ref)))
k (/ (* 0.4 w) span)]
{[:xform :anchor] (ch/framed [(:x c) (:y c)])
[:xform :scale] (ch/framed [k k])
[:xform :pos] (ch/framed [(- (/ w 2) (:x c))
(- (* 0.25 h) (:y c))])})))
(defn- mouth-part [absent? obs All subjects share one source-to-stage mapping on the :face group. Each
instance can then be placed independently with ordinary transform channels."
[{:keys [stage fit-motion?]} subjects]
(let [[w h] stage
inputs (vals subjects)]
(if fit-motion?
(fit-points stage (mapcat motion-points inputs))
(let [ref (mapcat :ref inputs)
c (geom/centroid ref)
span (- (reduce max (map :x ref)) (reduce min (map :x ref)))
k (/ (* 0.4 w) span)]
{[:xform :anchor] (ch/framed [(:x c) (:y c)])
[:xform :scale] (ch/framed [k k])
[:xform :pos] (ch/framed [(- (/ w 2) (:x c))
(- (* 0.25 h) (:y c))])}))))
(defn- mouth-part [subject absent? obs
{:keys [analysis verts anchor-avg contour-avg aperture-cut] :as params} {:keys [analysis verts anchor-avg contour-avg aperture-cut] :as params}
{:keys [outer inner] :as inputs}] {:keys [outer inner] :as inputs}]
(let [rings (block {:role "geom" :analysis (:id analysis) :params params (let [own (partial feature/owned subject)
mouth (own :mouth)
rings (block {:role "geom" :analysis (:id analysis) :params params
:tracks ["outer" "inner"]} :tracks ["outer" "inner"]}
{:type "int16" :scale geom-scale {:type "int16" :scale geom-scale
:features [:mouth :mouth] :absent? absent?} :features [mouth mouth] :absent? absent?}
obs [(rings->flat outer verts) (rings->flat inner verts)]) obs [(rings->flat outer verts) (rings->flat inner verts)])
prov (fn [by] prov (fn [by]
{:by by :analysis (:id analysis) {:by by :analysis (:id analysis)
@ -452,9 +476,11 @@
"Freeze eyes and brows into their own dense blocks and scene nodes. This owns "Freeze eyes and brows into their own dense blocks and scene nodes. This owns
only representation: the landmark correspondence, blink and pose choices have only representation: the landmark correspondence, blink and pose choices have
already been settled by measure and condition." already been settled by measure and condition."
[absent? obs {:keys [eye-verts brow-verts analysis contour-avg anchor-avg] :as params} [subject absent? obs
{:keys [eye-verts brow-verts analysis contour-avg anchor-avg] :as params}
{:keys [eyes brows]}] {:keys [eyes brows]}]
(let [provenance (fn [by extra] (let [own (partial feature/owned subject)
provenance (fn [by extra]
{:by by :analysis (:id analysis) {:by by :analysis (:id analysis)
:params (merge {:anchor-avg anchor-avg :contour-avg contour-avg} :params (merge {:anchor-avg anchor-avg :contour-avg contour-avg}
extra)}) extra)})
@ -470,20 +496,23 @@
;; and `each-dense-track-follows-its-own-features-presence` is what pins it. ;; and `each-dense-track-follows-its-own-features-presence` is what pins it.
eye-block (when eyes (named "eyes" eye-block (when eyes (named "eyes"
["lash-r" "lid-r" "lash-l" "lid-l"] ["lash-r" "lid-r" "lash-l" "lid-l"]
[:eye-r :eye-r :eye-l :eye-l] [(own :eye-r) (own :eye-r) (own :eye-l) (own :eye-l)]
"int16" "int16"
(mapv #(rings->flat % eye-verts) (mapv #(rings->flat % eye-verts)
[(:lash-r eyes) (:lid-r eyes) [(:lash-r eyes) (:lid-r eyes)
(:lash-l eyes) (:lid-l eyes)]))) (:lash-l eyes) (:lid-l eyes)])))
iris-block (when eyes iris-block (when eyes
(named "iris-pos" ["iris-r" "iris-l"] [:eye-r :eye-l] "float32" (named "iris-pos" ["iris-r" "iris-l"]
[(own :eye-r) (own :eye-l)] "float32"
[(:iris-r eyes) (:iris-l eyes)])) [(:iris-r eyes) (:iris-l eyes)]))
brow-block (when brows brow-block (when brows
(named "brows" ["ring-r" "ring-l"] [:brow-r :brow-l] "int16" (named "brows" ["ring-r" "ring-l"]
[(own :brow-r) (own :brow-l)] "int16"
(mapv #(rings->flat % brow-verts) (mapv #(rings->flat % brow-verts)
[(:ring-r brows) (:ring-l brows)]))) [(:ring-r brows) (:ring-l brows)])))
brow-pos-block (when brows brow-pos-block (when brows
(named "brow-pos" ["pos-r" "pos-l"] [:brow-r :brow-l] "float32" (named "brow-pos" ["pos-r" "pos-l"]
[(own :brow-r) (own :brow-l)] "float32"
[(:pos-r brows) (:pos-l brows)])) [(:pos-r brows) (:pos-l brows)]))
eye-node (fn [id z track] eye-node (fn [id z track]
{:id id :name (clojure.core/name id) :kind :poly :parent :head :z z {:id id :name (clojure.core/name id) :kind :poly :parent :head :z z
@ -525,14 +554,14 @@
[:style :color] (ch/framed :brow)}})] [:style :color] (ch/framed :brow)}})]
{:nodes (merge {:nodes (merge
(when eyes (when eyes
{:eye-r (eye-node :eye-r "a2" 0) {:eye-r (eye-node :eye-r "a2" 0)
:eye-r-in (inner-node :eye-r-in :eye-r "a1" 1 (:shut-r eyes)) :eye-r-in (inner-node :eye-r-in :eye-r "a1" 1 (:shut-r eyes))
:iris-r (iris-node :iris-r :eye-r-in 0 (:radius-r eyes)) :iris-r (iris-node :iris-r :eye-r-in 0 (:radius-r eyes))
:pupil-r (pupil-node :pupil-r :iris-r) :pupil-r (pupil-node :pupil-r :iris-r)
:eye-l (eye-node :eye-l "a3" 2) :eye-l (eye-node :eye-l "a3" 2)
:eye-l-in (inner-node :eye-l-in :eye-l "a1" 3 (:shut-l eyes)) :eye-l-in (inner-node :eye-l-in :eye-l "a1" 3 (:shut-l eyes))
:iris-l (iris-node :iris-l :eye-l-in 1 (:radius-l eyes)) :iris-l (iris-node :iris-l :eye-l-in 1 (:radius-l eyes))
:pupil-l (pupil-node :pupil-l :iris-l)}) :pupil-l (pupil-node :pupil-l :iris-l)})
(when brows (when brows
{:brow-r (brow-node :brow-r "a4" 0) {:brow-r (brow-node :brow-r "a4" 0)
:brow-l (brow-node :brow-l "a5" 1)})) :brow-l (brow-node :brow-l "a5" 1)}))
@ -542,10 +571,11 @@
(defn- interior-part (defn- interior-part
"Freeze the pixel-derived radial contour under the mouth cavity. Missing "Freeze the pixel-derived radial contour under the mouth cavity. Missing
contours use the dense block's absence bit; contrast decides editable :vis." contours use the dense block's absence bit; contrast decides editable :vis."
[{:keys [analysis teeth-verts cavity-erode tongue-reject blob-grow [subject {:keys [analysis teeth-verts cavity-erode tongue-reject blob-grow
top-bias teeth-on teeth-smooth] :as params} absent? obs top-bias teeth-on teeth-smooth] :as params} absent? obs
{:keys [contours shown]}] {:keys [contours shown]}]
(let [empty-points (vec (repeat (* 2 teeth-verts) 0)) (let [own (partial feature/owned subject)
empty-points (vec (repeat (* 2 teeth-verts) 0))
values (mapv (fn [ring] values (mapv (fn [ring]
(if ring (if ring
(into [] (mapcat (juxt :x :y)) ring) (into [] (mapcat (juxt :x :y)) ring)
@ -553,7 +583,7 @@
blk (block {:role "teeth" :analysis (:id analysis) :params params blk (block {:role "teeth" :analysis (:id analysis) :params params
:tracks ["contour"]} :tracks ["contour"]}
{:type "int16" :scale geom-scale {:type "int16" :scale geom-scale
:features [:teeth] :absent? absent? :features [(own :teeth)] :absent? absent?
;; Not the feature's absence: a frame no contour could be ;; Not the feature's absence: a frame no contour could be
;; extracted from has no teeth to draw whether or not the ;; extracted from has no teeth to draw whether or not the
;; teeth were occluded, and the two reasons are different facts. ;; teeth were occluded, and the two reasons are different facts.
@ -565,7 +595,8 @@
:top-bias top-bias :teeth-verts teeth-verts :top-bias top-bias :teeth-verts teeth-verts
:teeth-on teeth-on :teeth-smooth teeth-smooth}}] :teeth-on teeth-on :teeth-smooth teeth-smooth}}]
{:nodes {:teeth {:nodes {:teeth
{:id :teeth :name "teeth" :kind :poly :parent :mouth-in :z "a1" {:id :teeth :name "teeth" :kind :poly
:parent :mouth-in :z "a1"
:stencil :mouth-in :stencil :mouth-in
:channels {[:geom :pts] (dense blk 0 generated) :channels {[:geom :pts] (dense blk 0 generated)
[:style :color] (ch/framed :teeth) [:style :color] (ch/framed :teeth)
@ -573,32 +604,40 @@
:store (stored blk)})) :store (stored blk)}))
(defn part (defn part
"A feature type's nodes and addressed blocks, with no clip scaffold." "One feature type: local nodes and blocks addressed by subject and feature."
[area params {:keys [detected presence] :as measured}] [subject area params {:keys [detected presence] :as measured}]
(let [absent? (when (or detected presence) (let [presence (into {} (map (fn [[role mask]] [(feature/owned subject role) mask])) presence)
absent? (when (or detected presence)
(fn [id f] (fn [id f]
(or (and detected (not (nth detected f true))) (or (and detected (not (nth detected f true)))
(and (contains? presence id) (and (contains? presence id)
(not (nth (get presence id) f)))))) (not (nth (get presence id) f))))))
obs {:detected detected :presence presence}] obs {:detected detected :presence presence}]
(update (case area (update (case area
:mouth (mouth-part absent? obs params measured) :mouth (mouth-part subject absent? obs params measured)
:eye (feature-parts absent? obs params (select-keys measured [:eyes])) :eye (feature-parts subject absent? obs params (select-keys measured [:eyes]))
:brow (feature-parts absent? obs params (select-keys measured [:brows])) :brow (feature-parts subject absent? obs params (select-keys measured [:brows]))
:teeth (interior-part params absent? obs (:teeth measured)) :teeth (interior-part subject params absent? obs (:teeth measured))
(throw (ex-info "unknown frozen feature type" {:area area}))) (throw (ex-info "unknown frozen feature type" {:area area})))
:nodes performance-nodes))) :nodes performance-nodes)))
(defn head-part (defn head-part
"Freeze only the measured head transform from an already conditioned anchor." "Freeze one subject's measured head transform from its conditioned anchor.
[{:keys [analysis anchor-avg] :as params} {:keys [transforms detected presence]}]
THE THREE BLOCKS NAME THE SUBJECT, and that is not decoration. A block's key is
a hash over its descriptor, and the head follows DETECTION rather than any
feature's presence — so with `nil` in the feature slot, two faces tracked in one
analysis, both detected on every frame, produced byte-for-byte different
transforms under one identical key, and the second freeze's block silently
replaced the first's. The subject is the feature the head follows."
[subject {:keys [analysis anchor-avg] :as params} {:keys [transforms detected presence]}]
(let [absent? (when (or detected presence) (let [absent? (when (or detected presence)
(fn [_ f] (and detected (not (nth detected f true))))) (fn [_ f] (and detected (not (nth detected f true)))))
inv (mapv invert transforms) inv (mapv invert transforms)
xf (fn [role f] xf (fn [role f]
(block {:role role :analysis (:id analysis) :params params (block {:role role :analysis (:id analysis) :params params
:tracks [role]} :tracks [role]}
{:type "float32" :features [nil] :absent? absent?} {:type "float32" :features [subject] :absent? absent?}
{:detected detected} {:detected detected}
[(mapv f inv)])) [(mapv f inv)]))
pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)])) pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)]))
@ -614,184 +653,83 @@
;; --------------------------------------------------------------------------- ;; ---------------------------------------------------------------------------
;; the clip ;; the clip
(defn- subject-part
"A subject's drawing, metadata and blocks. Node names are timeline-local."
[params subject {:keys [outer eyes brows teeth] :as inputs}]
(let [own (partial feature/owned subject)
areas (cond-> [:mouth] (and eyes brows) (into [:eye :brow]) teeth (conj :teeth))
parts (mapv #(part subject % params inputs) areas)
head (head-part subject params inputs)
features (cond-> {:mouth [:mouth [:mouth :mouth-in]]}
(and eyes brows)
(merge {:eye-r [:eye [:eye-r :eye-r-in :iris-r :pupil-r]]
:eye-l [:eye [:eye-l :eye-l-in :iris-l :pupil-l]]
:brow-r [:brow [:brow-r]] :brow-l [:brow [:brow-l]]})
teeth (assoc :teeth [:teeth [:teeth]]))]
{:timeline {:id subject :frames (count outer)
:nodes (into {:head {:id :head :name "head" :kind :group :z "a1"
:measured (:measured head)}}
(mapcat :nodes) parts)}
:features (into {} (map (fn [[role [area nodes]]]
[(own role) {:id (own role) :subject subject
:timeline subject :area area
:nodes nodes :params {}}])) features)
:groups (if (and eyes brows)
{(own :eyes) {:id (own :eyes) :kind :eye-pair :subject subject
:members [(own :eye-r) (own :eye-l)] :params {}}}
{})
:store (into (:store head) (mapcat :store) parts)}))
(defn clip (defn clip
"Conditioned measurements -> a clip: nodes with frozen channels, plus the dense "Subject-id -> conditioned measurements becomes a library of face timelines.
blocks they read. `(f params inputs)`, no state.
params :main holds exposure and a shared source-to-stage placement. Each subject is
:name labels the clip. It is NOT in any key: two clips of the same placed by an ordinary symbol instance, so pose choices and transforms have
footage under different names are the same analysis and the their existing instance scope. Features name local nodes in that subject's
same blocks, and sharing them is the return on addressing. timeline; block descriptors still name globally distinct features.
:fps the clip's rate. FRAMES is not a parameter — it is
`(count outer)`, because a freeze that could disagree with its
own input about the length of the take would.
:stage [w h] project dimensions, INDEPENDENT of the footage
:fit-motion? choose an editable real-footage default that keeps the
observed feature motion on stage
:expose the clip root's exposure grid, inherited by everything
:verts the lip rings' vertex budget
:eye-verts the eyelid rings' vertex budget
:brow-verts the brow rings' vertex budget
:aperture-cut fraction of the take's peak aperture below which the mouth
interior is not present
:head :free | :anchored
:anchors optional map of local change frame -> measured source frame
:analysis the analysis record these measurements came from — detector,
VERSION, source, source cadence and pixel aspect. Its `:id` is
a content address over all of that, every block's key is a hash
over that id, and `flow/address` says why the version being in
there is the one field that must not be forgotten.
:anchor-avg
:contour-avg the stage-4 knobs. Freeze does not use them; it RECORDS them,
because `:generated` is what lets the UI offer a re-freeze at
different parameters instead of raw keys.
inputs Subjects share a source frame space. :head and :anchors may be overridden
:ref the Procrustes reference configuration per subject; all other freeze settings come from params."
:transforms the CONDITIONED anchor transforms [{:keys [name fps stage expose head anchors] :as params} subjects]
:outer :inner the CONDITIONED head-local lip rings (when-not (and (map? subjects) (seq subjects)
:eyes :brows the CONDITIONED head-local feature measurements (every? keyword? (keys subjects))
:teeth optional pixel-derived, conditioned radial contour (not-any? #{:main :root :face} (keys subjects)))
:aperture head-local aperture per frame (throw (ex-info "a freeze needs subjects with ids distinct from :main, :root and :face" {})))
:detected optional per-frame face booleans (let [ordered (sort-by (comp str key) subjects)
:presence optional feature-id -> per-frame booleans; false means an parts (mapv (fn [[id inputs]] [id (subject-part params id inputs)]) ordered)
unobserved feature, even if the rest of the face was found lengths (distinct (map #(get-in % [1 :timeline :frames]) parts))
_ (when-not (and (= 1 (count lengths)) (pos? (first lengths)))
The node tree is the one docs/animation-model.md specifies, and the two groups (throw (ex-info "subjects need the same positive frame count"
are two different things wanting the same transform: {:frames (vec lengths)})))
nf (first lengths)
:root the clip. EXPOSURE LIVES HERE and is inherited strictly. merged (fn [k] (into {} (mapcat (comp k second)) parts))
:face AUTHORED. where the face sits on the stage, and how big. built {:name name :fps fps :analysis (:analysis params)
:head MEASURED. the head's motion, or identity. :width (first stage) :height (second stage)
:mouth :subjects (into {} (map (fn [[id _]] [id {:id id :params {}}])) ordered)
:mouth-in :features (merged :features) :groups (merged :groups)
:teeth :timelines
:eye-r / :eye-l (into {clip/root-id
:eye-*-in {:id clip/root-id :frames nf
:iris-* :nodes (into {:root {:id :root :name "clip" :kind :group :z "a1"
:pupil-* :time {:mode :map :expose expose}}
:brow-r / :brow-l :face {:id :face :name "source placement" :kind :group
:parent :root :z "a1"
`:mouth-in`'s parent is `:mouth` and that transform is identity today, so :channels (face-placement params subjects)}}
composing it is composing identity. It is documented intent, and it is the one (map-indexed
place in the tree where the parent pointer is not yet doing work — the moment a (fn [i [id _]]
painted cel rides a moving plate it will be. [id {:id id :kind :symbol :of id :parent :face
:z (str "a" i)}]))
`:head` keeps its measured channels under `:measured` as well as in ordered)}}
`:channels`; its optional `:anchors` map changes only their frame address." (map (fn [[id part]] [id (:timeline part)])) parts)}]
[{:keys [name fps stage expose verts aperture-cut head anchors (doseq [[subject inputs] ordered
analysis anchor-avg contour-avg] :as params} [id track] (:presence inputs)]
{:keys [transforms outer inner detected presence eyes brows teeth] :as inputs}] (when-not (and (= nf (count track))
(let [nf (count outer) (= subject (get-in built [:features (feature/owned subject id) :subject])))
_ (doseq [[id track] presence] (throw (ex-info "presence must name this subject's feature and span the take"
(when (not= nf (count track)) {:subject subject :feature id :frames nf :actual (count track)}))))
(throw (ex-info "feature presence track must match the clip" {:store (merged :store)
{:feature id :frames nf :actual (count track)})))) :clip (reduce (fn [c [subject inputs]]
;; Asked `(absent? feature f)`, where a nil feature is the whole face. A (head-mode {:subject subject :mode (or (:head inputs) head)
;; feature with no presence track is present whenever a face was found. :anchors (get inputs :anchors anchors)}
absent? (when (or detected presence) {:clip c}))
(fn [id f] built ordered)}))
(or (and detected (not (nth detected f true)))
(and (contains? presence id)
(not (nth (get presence id) f))))))
obs {:detected detected :presence presence}
prov (fn [by extra]
{:by by :analysis (:id analysis)
:params (merge {:anchor-avg anchor-avg} extra)})
mouth (mouth-part absent? obs params inputs)
;; The anchor, inverted and split into its three components. Three blocks
;; and not one: they are three channels, they have three strides, and a
;; single block would need a per-component offset table to say so.
inv (mapv invert transforms)
xf (fn [role f]
;; The head follows DETECTION and no feature's presence: an
;; occluded eye does not mean the head was not there. That is
;; what the nil feature says.
(block {:role role :analysis (:id analysis) :params params
:tracks [role]}
{:type "float32" :features [nil] :absent? absent?}
{:detected detected}
[(mapv f inv)]))
pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)]))
rot (xf "head-rot" (fn [t] [(:theta t)]))
scale (xf "head-scale" (fn [t] [(:s t) (:s t)]))
;; Which knobs each channel records is not decoration, it is the
;; invalidation table written down where a re-freeze can read it. The
;; anchor depends on `anchor avg` alone. The rings depend on it and on
;; `contour avg` and on the vertex budget. The aperture depends on
;; `anchor avg` and on its own cut and NOT on `contour avg`, because
;; measure reports the inner ring's own height and nothing smooths it.
anchor-prov (prov :anchor/similarity nil)
features (when (and eyes brows) (feature-parts absent? obs params inputs))
interior (when teeth (interior-part params absent? obs teeth))
built {:name name
:fps fps
;; Tier 1 says which analysis its channels came out of, in full.
;; The id alone would make the document unreadable the first time
;; a detector upgrade orphaned a block: "sha256:7f2…" is not an
;; answer to "which model produced this take".
:analysis analysis
;; A gap changes channel state, never these IDs or pair links.
:subjects {:face-1 {:id :face-1 :params {}}}
:features (merge
{:mouth {:id :mouth :subject :face-1 :area :mouth
:nodes [:mouth :mouth-in] :params {}}}
(when features
{:eye-r {:id :eye-r :subject :face-1 :area :eye
:nodes [:eye-r :eye-r-in :iris-r :pupil-r] :params {}}
:eye-l {:id :eye-l :subject :face-1 :area :eye
:nodes [:eye-l :eye-l-in :iris-l :pupil-l] :params {}}
:brow-r {:id :brow-r :subject :face-1 :area :brow
:nodes [:brow-r] :params {}}
:brow-l {:id :brow-l :subject :face-1 :area :brow
:nodes [:brow-l] :params {}}})
(when teeth
{:teeth {:id :teeth :subject :face-1 :area :teeth
:nodes [:teeth] :params {}}}))
:groups (if features
{:eyes-1 {:id :eyes-1 :kind :eye-pair :subject :face-1
:members [:eye-r :eye-l] :params {}}}
{})
;; Stage dimensions, on the clip and not on the footage. See
;; face-placement: nothing below here knows the frame size.
:width (first stage)
:height (second stage)
;; ONE TIMELINE, and the frame count is ITS. A clip is a rate and
;; a timeline is a frame space — see arthur.domain.clip — so `nf`
;; lands here and `:fps` above, and the library a symbol will live
;; in is this same map with a second entry.
:timelines
{clip/root-id
{:id clip/root-id
:frames nf
:nodes
(merge
{:root
{:id :root :name "clip" :kind :group :parent nil :z "a1"
:time {:mode :map :expose expose}}
:face
{:id :face :name "face" :kind :group :parent :root :z "a1"
:channels (face-placement params inputs)}
:head
{:id :head :name "head" :kind :group :parent :face :z "a1"
:measured {[:xform :pos] (dense pos 0 anchor-prov)
[:xform :rot] (dense rot 0 anchor-prov)
[:xform :scale] (dense scale 0 anchor-prov)}}}
(:nodes mouth) (:nodes features) (:nodes interior))}}}
;; Tier 2, behind a handle, and now behind a content address: every key is
;; a sha256 over the analysis, the settings and the absence data that
;; produced the bytes under it. Nothing above this line changed when they
;; stopped being "take/geom", which is the point of a handle.
store (merge (stored pos rot scale) (:store mouth)
(:store features) (:store interior))]
(doseq [id (keys presence)]
(when-not (contains? (:features built) id)
(throw (ex-info "presence track names no feature in this clip"
{:feature id :features (keys (:features built))}))))
{:store store
:clip (head-mode {:mode head :anchors anchors}
{:clip (update-in built [:timelines clip/root-id :nodes]
performance-nodes)
:store store})}))

56
frontend/src/arthur/flow/regenerate.cljs
View file

@ -1,8 +1,7 @@
(ns arthur.flow.regenerate (ns arthur.flow.regenerate
"Recompute a changed feature from retained source tracks, then replace only "Recompute a changed feature from retained source tracks, then replace only
channels owned by that feature. Upload remains project/save's ordinary job." channels owned by that feature. Upload remains project/save's ordinary job."
(:require [arthur.domain.clip :as clip] (:require [arthur.domain.feature :as feature]
[arthur.domain.feature :as feature]
[arthur.domain.params :as params] [arthur.domain.params :as params]
[arthur.flow.address :as address] [arthur.flow.address :as address]
[arthur.flow.freeze :as freeze] [arthur.flow.freeze :as freeze]
@ -40,7 +39,8 @@
:when (:generated channel)] :when (:generated channel)]
[id prop channel])] [id prop channel])]
(-> (reduce (fn [entry [id prop channel]] (-> (reduce (fn [entry [id prop channel]]
(let [at [:clip :timelines clip/root-id :nodes id :channels prop] (let [at [:clip :timelines (get-in entry [:clip :features fid :timeline])
:nodes id :channels prop]
old (get-in entry at)] old (get-in entry at)]
(assoc-in entry at (assoc-in entry at
(cond-> channel (cond-> channel
@ -85,24 +85,25 @@
owns those input edges, and neither one is a request to freeze the other owns those input edges, and neither one is a request to freeze the other
feature." feature."
[base-params base source-inputs entry fid] [base-params base source-inputs entry fid]
(let [area (get-in entry [:clip :features fid :area]) (let [{:keys [area subject]} (get-in entry [:clip :features fid])
params (merge base-params (settings (:clip entry) fid))] params (merge base-params (settings (:clip entry) fid))]
(when (and (= :teeth area) (nil? (:interior source-inputs))) (when (and (= :teeth area) (nil? (:interior source-inputs)))
(throw (ex-info "teeth regeneration needs retained pixel measurements" (throw (ex-info "teeth regeneration needs retained pixel measurements"
{:feature fid}))) {:feature fid})))
(replace-feature entry (replace-feature entry
(freeze/part area params (freeze/part subject area params
(take/measure-part area params source-inputs @base)) (take/measure-part area params source-inputs @base))
fid))) fid)))
(defn- regenerate-head (defn- regenerate-head
"Re-freeze the head transform, which is a different job from a feature's: its "Re-freeze ONE SUBJECT's head transform, which is a different job from a
only input is the conditioned anchor, and it owns no channels to replace. The feature's: its only input is that subject's conditioned anchor, and it owns no
authored `:channels` follow the measurement while they still ARE the channels to replace. The authored `:channels` follow the measurement while they
measurement, and are left alone once somebody has placed the head by hand." still ARE the measurement, and are left alone once somebody has placed the head
[entry params base] by hand."
(let [baked (freeze/head-part params @base) [entry params base subject]
at [:clip :timelines clip/root-id :nodes :head] (let [baked (freeze/head-part subject params @base)
at [:clip :timelines subject :nodes :head]
old (get-in entry at) old (get-in entry at)
measured (:measured baked)] measured (:measured baked)]
(cond-> (-> entry (cond-> (-> entry
@ -111,13 +112,19 @@
(= (:channels old) (:measured old)) (= (:channels old) (:measured old))
(assoc-in (conj at :channels) measured)))) (assoc-in (conj at :channels) measured))))
(defn- change-take (defn change
"One scoped static edit. `source-inputs` holds dense landmarks and, when the "One scoped static edit. `source-inputs` holds dense landmarks and, when the
teeth are dirty, retained pixel measurements. No IO or app-db here." teeth are dirty, retained pixel measurements. No IO or app-db here."
[{:keys [clip source-inputs] :as entry} edit] [{:keys [clip source-inputs] :as entry} edit]
(when-not (:dense source-inputs)
(throw (ex-info "regeneration needs retained source landmarks" {})))
(let [{:keys [changed subject features]} (plan clip edit) (let [{:keys [changed subject features]} (plan clip edit)
;; THE EDITED SUBJECT'S OWN LANDMARKS. Retained source is per subject —
;; one video, one dense track per tracked face — so re-measuring the
;; second face through the first one's anchor is the mistake this lookup
;; exists to prevent.
source-inputs (get-in source-inputs [:subjects subject])
_ (when-not (:dense source-inputs)
(throw (ex-info "regeneration needs retained source landmarks"
{:subject subject})))
base-params (merge take/knobs base-params (merge take/knobs
{:fps (:fps changed) {:fps (:fps changed)
:aspect (get-in changed [:analysis :aspect]) :aspect (get-in changed [:analysis :aspect])
@ -131,21 +138,4 @@
base (delay (take/anchor-base anchor-params source-inputs))] base (delay (take/anchor-base anchor-params source-inputs))]
(cond-> (reduce (partial regenerate-feature base-params base source-inputs) (cond-> (reduce (partial regenerate-feature base-params base source-inputs)
(assoc entry :clip changed) features) (assoc entry :clip changed) features)
(= :anchor-avg (:knob edit)) (regenerate-head anchor-params base)))) (= :anchor-avg (:knob edit)) (regenerate-head anchor-params base subject))))
(defn change
"A take edits its root timeline. A stage edits the shared symbol timeline;
every instance then reads the same new channels on its next paint."
[{:keys [clip] :as entry} edit]
(if-let [symbol (some :timeline (vals (:features clip)))]
(let [original (:timelines clip)
take (assoc clip :timelines
{clip/root-id (assoc (get original symbol) :id clip/root-id)})
changed (change-take (assoc entry :clip take) edit)]
(assoc changed :clip
(-> (:clip changed)
(assoc :timelines
(assoc original symbol
(assoc (get-in changed [:clip :timelines clip/root-id])
:id symbol))))))
(change-take entry edit)))

131
frontend/src/arthur/flow/source.cljs
View file

@ -9,25 +9,26 @@
(def roles ["source/dense" "source/detected" "source/crops"]) (def roles ["source/dense" "source/detected" "source/crops"])
(defn- interior-address [analysis settings frames] (defn- interior-address [analysis subject settings frames]
(let [verts (:teeth-verts settings) (let [verts (:teeth-verts settings)
stride (+ 2 (* 2 verts))] stride (+ 2 (* 2 verts))]
(address/block {:role "source/interior" :analysis analysis (address/block {:role "source/interior" :analysis analysis
:params settings :params settings
:tracks ["contrast" "area" "contour"] :tracks ["contrast" "area" "contour"]
:features [] :observation nil :features [subject] :observation nil
:layout {:type "float64" :frames frames :verts verts :layout {:type "float64" :frames frames :verts verts
:stride stride}}))) :stride stride}})))
(defn interior-key [analysis settings frames] (defn interior-key [analysis subject settings frames]
(:key (interior-address analysis settings frames))) (:key (interior-address analysis subject settings frames)))
(defn interior-block (defn interior-block
"Retain pixel measurements before any head-local transform or smoothing." "Retain one subject's pixel measurements before any head-local transform or
[analysis settings measures] smoothing."
[analysis subject settings measures]
(let [frames (count measures) (let [frames (count measures)
{:keys [key descriptor]} {:keys [key descriptor]}
(interior-address analysis settings frames) (interior-address analysis subject settings frames)
verts (:teeth-verts settings) verts (:teeth-verts settings)
stride (+ 2 (* 2 verts)) stride (+ 2 (* 2 verts))
data (js/Float64Array. (* frames stride))] data (js/Float64Array. (* frames stride))]
@ -82,20 +83,28 @@
(interior/measure params (:box crop) #js {:data (:data crop)}) (interior/measure params (:box crop) #js {:data (:data crop)})
{:contour nil :contrast 0 :area 0 :debug nil})) {:contour nil :contrast 0 :area 0 :debug nil}))
(defn- named [role analysis tracks layout data] (defn- named [role analysis subject tracks layout data]
(merge (address/block {:role role :analysis analysis :params {} (merge (address/block {:role role :analysis analysis :params {}
:tracks tracks :features [] :observation nil :tracks tracks :features [subject] :observation nil
:layout layout}) :layout layout})
{:role role :data data})) {:role role :data data}))
(defn pack (defn pack
"Dense landmarks, mask, crops and optional pixel measurements -> source blocks. "One SUBJECT's dense landmarks, mask, crops and optional pixel measurements ->
source blocks.
`:interior-settings` is not optional when `:interior` is present: the block is `:interior-settings` is not optional when `:interior` is present: the block is
addressed BY those knobs, and defaulting them here would name a block after addressed BY those knobs, and defaulting them here would name a block after
settings its bytes did not come from — a content address that lies, which is the settings its bytes did not come from — a content address that lies, which is the
one failure the key exists to prevent." one failure the key exists to prevent.
[analysis {:keys [dense detected crops interior interior-settings]}]
`subject` is in every block's descriptor, and it has to be. These four blocks
carry no feature — they are the raw analysis, upstream of any part — so two
faces tracked in one video would otherwise produce different bytes under one
identical key, and the second face's landmarks would replace the first's in the
store. That is the same failure `freeze/head-part` guards against one stage
later, for the same reason."
[analysis subject {:keys [dense detected crops interior interior-settings]}]
(let [frames (count dense) (let [frames (count dense)
points (count (first dense))] points (count (first dense))]
(when-not (and (pos? frames) (pos? points) (when-not (and (pos? frames) (pos? points)
@ -124,52 +133,66 @@
(when-let [crop (nth crops f)] (when-let [crop (nth crops f)]
(.set pixels (:data crop) (nth offsets f)))) (.set pixels (:data crop) (nth offsets f))))
(cond-> {"source/dense" (cond-> {"source/dense"
(named "source/dense" analysis ["landmarks"] (named "source/dense" analysis subject ["landmarks"]
{:type "float64" :frames frames :points points {:type "float64" :frames frames :points points
:stride (* 3 points)} :stride (* 3 points)}
landmarks) landmarks)
"source/detected" "source/detected"
(named "source/detected" analysis ["detected"] (named "source/detected" analysis subject ["detected"]
{:type "uint8" :frames frames :stride 1} mask) {:type "uint8" :frames frames :stride 1} mask)
"source/crops" "source/crops"
(named "source/crops" analysis ["rgba"] (named "source/crops" analysis subject ["rgba"]
{:type "uint8" :frames frames :boxes boxes :offsets offsets} {:type "uint8" :frames frames :boxes boxes :offsets offsets}
pixels)} pixels)}
interior (assoc "source/interior" interior (assoc "source/interior"
(interior-block (interior-block
analysis analysis subject
(or interior-settings (or interior-settings
(throw (ex-info "an interior block needs the settings its measurements were taken at" (throw (ex-info "an interior block needs the settings its measurements were taken at"
{:frames (count interior)}))) {:frames (count interior)})))
interior)))))) interior))))))
(defn wire-blocks [blocks] (defn pack-subjects
(into-array "Every tracked subject's source blocks: `{subject {role block}}`.
(map (fn [role]
(let [{:keys [key descriptor data]} (get blocks role)]
#js {:key key :descriptor descriptor :data (wire/base64 data)}))
roles)))
(defn upload-blocks [blocks] One video, one analysis, one set of blocks PER FACE. They are separate blocks
(into-array and not one block with a subject axis, because a face can be re-measured on its
(map (fn [role] own — a knob dragged on the second person re-reads the second person's
(let [{:keys [key descriptor data]} (get blocks role)] landmarks — and because that is what makes each one's content address name
#js {:key key :descriptor descriptor :data data})) exactly the bytes it covers."
(filter #(contains? blocks %) (conj roles "source/interior"))))) [analysis by-subject]
(into {} (map (fn [[subject inputs] ] [subject (pack analysis subject inputs)]))
by-subject))
(defn unpack (defn- flatten-blocks
"The three block-detail responses -> inputs for measurement and freeze." "`{subject {role block}}` -> one flat seq, in a stable order."
[^js responses [width height]] [by-subject wanted]
(let [by-role (into {} (for [[_ blocks] (sort-by (comp str key) by-subject)
(map (fn [^js response] role wanted
(let [descriptor (js->clj (js/JSON.parse (.-descriptor response)) :when (contains? blocks role)]
:keywordize-keys true)] (get blocks role)))
[(:role descriptor)
{:layout (:layout descriptor) (defn block-keys
:data (wire/typed (get-in descriptor [:layout :type]) "The keys of every subject's source blocks, in the order they upload."
(.-data response))}])) [by-subject]
(array-seq responses))) (mapv :key (flatten-blocks by-subject roles)))
_ (when-not (= (set roles) (set (keys by-role)))
(defn wire-blocks [by-subject]
(into-array
(map (fn [{:keys [key descriptor data]}]
#js {:key key :descriptor descriptor :data (wire/base64 data)})
(flatten-blocks by-subject roles))))
(defn upload-blocks [by-subject]
(into-array
(map (fn [{:keys [key descriptor data]}]
#js {:key key :descriptor descriptor :data data})
(flatten-blocks by-subject (conj roles "source/interior")))))
(defn- unpack-one
"One subject's three blocks -> its inputs for measurement and freeze."
[by-role [width height]]
(let [_ (when-not (= (set roles) (set (keys by-role)))
(throw (ex-info "analysis is missing source blocks" {:roles (keys by-role)}))) (throw (ex-info "analysis is missing source blocks" {:roles (keys by-role)})))
{:keys [layout data]} (get by-role "source/dense") {:keys [layout data]} (get by-role "source/dense")
frames (:frames layout) frames (:frames layout)
@ -206,3 +229,29 @@
{:dense dense :detected detected :crops crops :dimensions [width height] {:dense dense :detected detected :crops crops :dimensions [width height]
:missing (count (remove true? detected)) :missing (count (remove true? detected))
:first-real (first (keep-indexed (fn [i present?] (when present? i)) detected))}))) :first-real (first (keep-indexed (fn [i present?] (when present? i)) detected))})))
(defn unpack
"Every subject's block-detail responses -> `{:subjects {id inputs}}`.
WHICH SUBJECT A BLOCK BELONGS TO IS READ OFF ITS DESCRIPTOR, not off the order
the server returned it in. The descriptor is the thing the key is a hash of, so
it is the only statement about a block that cannot have drifted from its bytes."
[^js responses dimensions]
(let [parsed (map (fn [^js response]
(let [descriptor (js->clj (js/JSON.parse (.-descriptor response))
:keywordize-keys true)]
{:subject (keyword (first (:features descriptor)))
:role (:role descriptor)
:block {:layout (:layout descriptor)
:data (wire/typed (get-in descriptor [:layout :type])
(.-data response))}}))
(array-seq responses))]
(when (some (comp nil? :subject) parsed)
(throw (ex-info "a source block does not say which subject it came from"
{:roles (mapv :role parsed)})))
{:subjects
(into {}
(map (fn [[subject entries]]
[subject (unpack-one (into {} (map (juxt :role :block)) entries)
dimensions)]))
(group-by :subject parsed))}))

25
frontend/src/arthur/flow/take.cljs
View file

@ -1,6 +1,7 @@
(ns arthur.flow.take (ns arthur.flow.take
"The shared landmark-to-channel path for synthetic and detected takes." "The shared landmark-to-channel path for synthetic and detected takes."
(:require [arthur.flow.address :as address] (:require [arthur.flow.address :as address]
[arthur.flow.detect :as detect]
[arthur.flow.condition :as condition] [arthur.flow.condition :as condition]
[arthur.flow.condition.brows :as condition-brows] [arthur.flow.condition.brows :as condition-brows]
[arthur.flow.condition.eyes :as condition-eyes] [arthur.flow.condition.eyes :as condition-eyes]
@ -27,7 +28,7 @@
:aspect aspect :aspect aspect
;; How the detector was run, not just which one it was. See ;; How the detector was run, not just which one it was. See
;; `flow/address/analysis-descriptor`. ;; `flow/address/analysis-descriptor`.
:mode "video"})))) :mode "video" :tracking detect/settings}))))
(defn measure (defn measure
"Condition the anchor before measuring rings through it." "Condition the anchor before measuring rings through it."
@ -98,15 +99,28 @@
aperture)}) aperture)})
(throw (ex-info "unknown measured feature type" {:area area})))))) (throw (ex-info "unknown measured feature type" {:area area}))))))
(defn subjects
"Per-subject source landmarks -> per-subject conditioned measurements.
Every tracked face in one shot goes through the same measurement path and is
conditioned on its OWN anchor fit, because the fit maps a face onto its own
mean pose: sharing one would register the second face against the first one's
head. What they share is the video, and that is already expressed by them
sharing the analysis record."
[params by-subject]
(into {} (map (fn [[id inputs]] [id (measure params inputs)])) by-subject))
(defn build (defn build
[params inputs] "`by-subject` is subject id -> that face's source tracks, one entry per
(freeze/clip params (measure params inputs))) tracked face."
[params by-subject]
(freeze/clip params (subjects params by-subject)))
(defn footage (defn footage
"A real manifest and its detected landmarks through the same measurement and "A real manifest and its detected landmarks through the same measurement and
freeze path as the synthetic take. The source cadence stays in :fps; picture freeze path as the synthetic take. The source cadence stays in :fps; picture
sampling is a root time map applied only after this artifact exists." sampling is a root time map applied only after this artifact exists."
[manifest {:keys [dense detected dimensions interior presence detector]}] [manifest {:keys [dimensions detector subjects]}]
(let [[w h] dimensions (let [[w h] dimensions
params (merge knobs params (merge knobs
{:name (or (:source manifest) "footage") {:name (or (:source manifest) "footage")
@ -120,5 +134,4 @@
;; these landmarks is the failure this prevents. ;; these landmarks is the failure this prevents.
:analysis (analysis-for (assoc manifest :width w :height h) :analysis (analysis-for (assoc manifest :width w :height h)
detector)})] detector)})]
(build params {:dense dense :detected detected :interior interior (build params subjects)))
:presence presence})))

35
frontend/src/arthur/ui/shell.cljs
View file

@ -24,29 +24,12 @@
(defonce ^:private selected-owner (r/atom nil)) (defonce ^:private selected-owner (r/atom nil))
(defonce ^:private drafts (r/atom {})) (defonce ^:private drafts (r/atom {}))
(defn- instance-label
"What to call a symbol instance in the UI.
Its `:name`, because a placement's id is a UUID and a uuid is not something to
show anyone or to sort by. The id is the fallback only so that a node with no
name still has a row rather than a blank one."
[clip id]
(or (get-in clip [:timelines :main :nodes id :name]) (str id)))
(defn- setting-owners [clip] (defn- setting-owners [clip]
(let [instances (->> (get-in clip [:timelines :main :nodes]) (vec (for [[scope objects] [[:subject (:subjects clip)]
(filter (comp #{:symbol} :kind val)) [:feature (:features clip)]
;; By LABEL, not by id: ordering by a uuid is ordering at [:group (:groups clip)]]
;; random, and this list is read top to bottom by a person. id (sort-by str (keys objects))]
(sort-by (fn [[id _]] [(instance-label clip id) (str id)])) [scope id])))
(mapv key))
instances (if (seq instances) instances [nil])]
(vec (for [instance instances
[scope ids] [[:subject (:subjects clip)]
[:feature (:features clip)]
[:group (:groups clip)]]
id (sort-by str (keys ids))]
[scope id instance]))))
(defn- slider-bounds [knob {:keys [type default min even?] upper :max}] (defn- slider-bounds [knob {:keys [type default min even?] upper :max}]
{:min (or min (if (= knob :blob-grow) -3 0)) {:min (or min (if (= knob :blob-grow) -3 0))
@ -61,7 +44,7 @@
busy? (:busy? @(rf/subscribe [::sub/project])) busy? (:busy? @(rf/subscribe [::sub/project]))
report @(rf/subscribe [::project/regeneration]) report @(rf/subscribe [::project/regeneration])
owners (setting-owners clip) owners (setting-owners clip)
[scope id instance :as owner] (if (some #{(deref selected-owner)} owners) [scope id :as owner] (if (some #{(deref selected-owner)} owners)
@selected-owner (first owners)) @selected-owner (first owners))
area (case scope area (case scope
:subject :subject :subject :subject
@ -84,12 +67,8 @@
(if (seq owners) (if (seq owners)
(doall (for [[i [kind object-id]] (map-indexed vector owners)] (doall (for [[i [kind object-id]] (map-indexed vector owners)]
^{:key i} [:option {:value i} ^{:key i} [:option {:value i}
(str (when-let [instance (nth (nth owners i) 2)] (str (name kind) " · " (subs (str object-id) 1))]))
(str (instance-label clip instance) " / "))
(name kind) " · " (name object-id))]))
[:option {:value 0} "no tracked objects"])] ] [:option {:value 0} "no tracked objects"])] ]
(when instance
[:div.shared-note "shared symbol · changes all seven stage instances"])
(when area (when area
[:div.control-list [:div.control-list
(doall (doall

11
frontend/test/arthur/domain/feature_test.cljs
View file

@ -1,6 +1,5 @@
(ns arthur.domain.feature-test (ns arthur.domain.feature-test
(:require [cljs.test :refer [deftest is]] (:require [cljs.test :refer [deftest is]]
[arthur.domain.clip :as clip]
[arthur.domain.feature :as feature] [arthur.domain.feature :as feature]
[arthur.domain.params :as params])) [arthur.domain.params :as params]))
@ -12,16 +11,22 @@
{:subjects (into {} (map (fn [id] [id {:id id :params {}}]) people)) {:subjects (into {} (map (fn [id] [id {:id id :params {}}]) people))
:features (into {} (map-indexed (fn [i id] :features (into {} (map-indexed (fn [i id]
[id {:id id :subject (nth people (quot i 2)) [id {:id id :subject (nth people (quot i 2))
:timeline (nth people (quot i 2))
:area :eye :nodes [] :params {}}]) eyes)) :area :eye :nodes [] :params {}}]) eyes))
:groups (into {} (map-indexed (fn [i ids] :groups (into {} (map-indexed (fn [i ids]
(let [id (keyword (str "pair-" i))] (let [id (keyword (str "pair-" i))]
[id {:id id :kind :eye-pair [id {:id id :kind :eye-pair
:subject (nth people i) :subject (nth people i)
:members ids :params {}}])) members)) :members ids :params {}}])) members))
:timelines {:main {:id :main :frames 1 :nodes {}}}})) :timelines (into {} (map (fn [id]
[id {:id id :frames 1
:nodes {:head {:id :head :kind :group :z "a1"
:measured {[:xform :rot]
{:animated? false :value 0.0}}}}}]))
people)}))
(defn- problems [c] (defn- problems [c]
(feature/problems c (clip/nodes c))) (feature/problems c))
(deftest five-people-can-have-nine-identified-eyes (deftest five-people-can-have-nine-identified-eyes
(let [scene (nine-eyes)] (let [scene (nine-eyes)]

29
frontend/test/arthur/domain/leaf_test.cljs
View file

@ -40,16 +40,17 @@
(is (contains? ls "clip/c7/timeline/main")) (is (contains? ls "clip/c7/timeline/main"))
(is (= {:frames 229} (get ls "clip/c7/timeline/main")) (is (= {:frames 229} (get ls "clip/c7/timeline/main"))
"a timeline's leaf is its frame space; :fps is the clip's") "a timeline's leaf is its frame space; :fps is the clip's")
(is (contains? ls "clip/c7/timeline/main/node/mouth")) ;; Nodes are local to the face timeline; feature and group ids are clip-wide.
(is (contains? ls "clip/c7/timeline/main/channel/mouth/geom.pts")) (is (contains? ls "clip/c7/timeline/face-1/node/mouth"))
(is (contains? ls "clip/c7/timeline/main/channel/mouth-in/vis")) (is (contains? ls "clip/c7/timeline/face-1/channel/mouth/geom.pts"))
(is (contains? ls "clip/c7/feature/eye-r")) (is (contains? ls "clip/c7/timeline/face-1/channel/mouth-in/vis"))
(is (contains? ls "clip/c7/group/eyes-1")) (is (contains? ls "clip/c7/feature/face-1~eye-r"))
(is (contains? ls "clip/c7/group/face-1~eyes"))
(is (contains? ls "clip/c7/subject/face-1")) (is (contains? ls "clip/c7/subject/face-1"))
;; `:head`'s measured channels are written together by a freeze and replaced ;; A head's measured channels are written together by a freeze and replaced
;; together by a re-freeze, so they are one leaf and not three. ;; together by a re-freeze, so they are one leaf and not three.
(is (contains? ls "clip/c7/timeline/main/measured/head")) (is (contains? ls "clip/c7/timeline/face-1/measured/head"))
(is (= 3 (count (get ls "clip/c7/timeline/main/measured/head")))) (is (= 3 (count (get ls "clip/c7/timeline/face-1/measured/head"))))
;; :frames is NOT in `timing` any more. A timeline is a frame space and a clip ;; :frames is NOT in `timing` any more. A timeline is a frame space and a clip
;; is a rate, so the one leaf that held both was the persistence half of the ;; is a rate, so the one leaf that held both was the persistence half of the
;; conflation `domain/clip` exists to undo. ;; conflation `domain/clip` exists to undo.
@ -59,11 +60,11 @@
;; The boundary that lets two people key different parts without meeting. A node ;; The boundary that lets two people key different parts without meeting. A node
;; leaf carries structure and no geometry. ;; leaf carries structure and no geometry.
(let [ls (leaf/leaves :c1 @take/clip) (let [ls (leaf/leaves :c1 @take/clip)
n (get ls "clip/c1/timeline/main/node/mouth")] n (get ls "clip/c1/timeline/face-1/node/mouth")]
(is (= {:id :mouth :name "mouth" :kind :poly :parent :head :z "a1" (is (= {:id :mouth :name "mouth" :kind :poly :parent :head
:pose-group :mouth} n)) :z "a1" :pose-group :mouth} n))
(is (nil? (:channels n))) (is (nil? (:channels n)))
(is (:animated? (get ls "clip/c1/timeline/main/channel/mouth/geom.pts"))))) (is (:animated? (get ls "clip/c1/timeline/face-1/channel/mouth/geom.pts")))))
(deftest a-field-with-no-leaf-is-refused-rather-than-dropped (deftest a-field-with-no-leaf-is-refused-rather-than-dropped
;; The invariant that keeps the round trip exact as the model grows: a field ;; The invariant that keeps the round trip exact as the model grows: a field
@ -154,7 +155,7 @@
(is (empty? (leaf/problems (leaf/leaves :c1 @take/locked))))) (is (empty? (leaf/problems (leaf/leaves :c1 @take/locked)))))
(deftest a-channel-leaf-for-a-node-that-is-not-there-is-named (deftest a-channel-leaf-for-a-node-that-is-not-there-is-named
(let [ls (dissoc (leaf/leaves :c1 @take/clip) "clip/c1/timeline/main/node/mouth")] (let [ls (dissoc (leaf/leaves :c1 @take/clip) "clip/c1/timeline/face-1/node/mouth")]
(is (some #(re-find #"node with no node leaf" %) (leaf/problems ls))))) (is (some #(re-find #"node with no node leaf" %) (leaf/problems ls)))))
(deftest a-node-leaf-is-scoped-to-its-own-timeline (deftest a-node-leaf-is-scoped-to-its-own-timeline
@ -166,7 +167,7 @@
(assoc "clip/c1/timeline/sym~blink" {:frames 3} (assoc "clip/c1/timeline/sym~blink" {:frames 3}
"clip/c1/timeline/sym~blink/node/mouth" "clip/c1/timeline/sym~blink/node/mouth"
{:id :mouth :kind :poly :parent nil :z "a1"}) {:id :mouth :kind :poly :parent nil :z "a1"})
(dissoc "clip/c1/timeline/main/node/mouth"))] (dissoc "clip/c1/timeline/face-1/node/mouth"))]
(is (some #(re-find #"node with no node leaf" %) (leaf/problems ls))))) (is (some #(re-find #"node with no node leaf" %) (leaf/problems ls)))))
(deftest a-property-with-path-punctuation-in-it-is-refused (deftest a-property-with-path-punctuation-in-it-is-refused

26
frontend/test/arthur/domain/project_test.cljs
View file

@ -26,6 +26,8 @@
[arthur.flow.freeze :as freeze] [arthur.flow.freeze :as freeze]
[arthur.support.ops :as ops])) [arthur.support.ops :as ops]))
(defn- face-timeline [c] (clip/timeline c :face-1))
(defn- wired (defn- wired
"A clip out and back, over a wire that is really only JSON." "A clip out and back, over a wire that is really only JSON."
[cid clip] [cid clip]
@ -57,8 +59,8 @@
"spec vs playback, after" [ops/specified ops/resolved]}] "spec vs playback, after" [ops/specified ops/resolved]}]
(doseq [[label [f g]] paths (doseq [[label [f g]] paths
[order fs] (ops/orders n)] [order fs] (ops/orders n)]
(let [a (f (clip/root (:clip @before)) (:store @before)) (let [a (f (face-timeline (:clip @before)) (:store @before))
b (g (clip/root (:clip @after)) (:store @after))] b (g (face-timeline (:clip @after)) (:store @after))]
(testing (str label ", " order) (testing (str label ", " order)
(doseq [frame fs] (doseq [frame fs]
(is (= (a frame) (b frame)) (is (= (a frame) (b frame))
@ -82,8 +84,8 @@
;; everything above and lose the locked take's identity transform. ;; everything above and lose the locked take's identity transform.
(let [locked {:clip @take/locked :store @take/store} (let [locked {:clip @take/locked :store @take/store}
back (wired :c1 locked) back (wired :c1 locked)
a (ops/resolved (clip/root (:clip locked)) (:store locked)) a (ops/resolved (face-timeline (:clip locked)) (:store locked))
b (ops/resolved (clip/root (:clip back)) (:store back))] b (ops/resolved (face-timeline (:clip back)) (:store back))]
(is (= (:clip locked) (:clip back))) (is (= (:clip locked) (:clip back)))
(doseq [frame (range 0 take/frames 7)] (doseq [frame (range 0 take/frames 7)]
(is (= (a frame) (b frame)) (str "frame " frame))))) (is (= (a frame) (b frame)) (str "frame " frame)))))
@ -104,17 +106,17 @@
(def ^:private gappy (def ^:private gappy
(delay (freeze/clip (assoc take/params :name "gappy") (delay (freeze/clip (assoc take/params :name "gappy")
(assoc @take/measured {:face-1 (assoc @take/measured
:presence :presence
(into {} (map (fn [[id gap]] (into {} (map (fn [[id gap]]
[id (mapv #(not (contains? gap %)) [id (mapv #(not (contains? gap %))
(range take/frames))])) (range take/frames))]))
windows))))) windows))})))
(deftest an-absence-mask-survives-the-wire (deftest an-absence-mask-survives-the-wire
(let [back (wired :c1 @gappy) (let [back (wired :c1 @gappy)
at (fn [entry id path f] at (fn [entry id path f]
(ch/value-at (get-in (clip/nodes (:clip entry)) [id :channels path]) (ch/value-at (get-in (:nodes (face-timeline (:clip entry))) [id :channels path])
f (:store entry))) f (:store entry)))
;; Every dense track of the eye, iris, brow and brow-position blocks, and ;; Every dense track of the eye, iris, brow and brow-position blocks, and
;; the feature whose gap it must follow — the same table ;; the feature whose gap it must follow — the same table
@ -145,7 +147,7 @@
;; frame rather than hidden, and its partner is not. ;; frame rather than hidden, and its partner is not.
(let [back (wired :c1 @gappy) (let [back (wired :c1 @gappy)
drawn (into #{} (map :node) drawn (into #{} (map :node)
((timeline/resolver (clip/root (:clip back)) (:store back)) 12))] ((timeline/resolver (face-timeline (:clip back)) (:store back)) 12))]
(is (not (contains? drawn :eye-r))) (is (not (contains? drawn :eye-r)))
(is (contains? drawn :eye-l)) (is (contains? drawn :eye-l))
(is (contains? drawn :mouth)))) (is (contains? drawn :mouth))))

5
frontend/test/arthur/domain/raster_test.cljs
View file

@ -109,9 +109,10 @@
(deftest a-three-pixel-pupil-is-three-by-three-at-every-centre (deftest a-three-pixel-pupil-is-three-by-three-at-every-centre
;; A square pupil is only worth having if it is the SAME square every frame: ;; A square pupil is only worth having if it is the SAME square every frame:
;; exactly its nominal size at any centre, or it breathes as the gaze moves. ;; exactly its nominal size at any centre, or it breathes as the gaze moves.
(doseq [[cx cy] [[20 20] [20.5 20.5] [20.49 19.51] [21 20] [20.9 20.1]]] (doseq [[cx cy] [[20 20] [20.5 20.5] [20.49 19.51] [21 20] [20.9 20.1]]
size [2.6 3 3.4]]
(let [ras (-> (r/make 40 40) (r/clear! 0))] (let [ras (-> (r/make 40 40) (r/clear! 0))]
(r/fill-rect! ras cx cy 3 1) (r/fill-rect! ras cx cy size 1)
(let [[x0 x1 y0 y1 n] (bbox ras 1)] (let [[x0 x1 y0 y1 n] (bbox ras 1)]
(is (= [3 3 9] [(inc (- x1 x0)) (inc (- y1 y0)) n]) (is (= [3 3 9] [(inc (- x1 x0)) (inc (- y1 y0)) n])
(str "centre " cx "," cy " gave " (inc (- x1 x0)) "x" (inc (- y1 y0)) ":" n)))))) (str "centre " cx "," cy " gave " (inc (- x1 x0)) "x" (inc (- y1 y0)) ":" n))))))

6
frontend/test/arthur/domain/timeline_test.cljs
View file

@ -244,7 +244,7 @@
;; ---- discs and rects ---- ;; ---- discs and rects ----
(deftest a-discs-radius-takes-the-mean-scale-and-a-rects-size-is-rounded (deftest disc-and-rect-extents-retain-precision-for-enclosing-instances
(let [s (sc {:id :g :kind :group :z "a1" (let [s (sc {:id :g :kind :group :z "a1"
:channels {[:xform :pos] (ch/framed [50 60]) [:xform :scale] (ch/framed [2 2])}} :channels {[:xform :pos] (ch/framed [50 60]) [:xform :scale] (ch/framed [2 2])}}
{:id :d :kind :disc :parent :g :z "a1" {:id :d :kind :disc :parent :g :z "a1"
@ -253,9 +253,7 @@
:channels {[:geom :size] (ch/framed 1.7) [:style :color] (ch/framed :pupil)}}) :channels {[:geom :size] (ch/framed 1.7) [:style :color] (ch/framed :pupil)}})
[d r] (timeline/eval-frame s 0)] [d r] (timeline/eval-frame s 0)]
(is (= [50 60 6] [(:cx d) (:cy d) (:r d)])) (is (= [50 60 6] [(:cx d) (:cy d) (:r d)]))
;; 1.7 x 2 is 3.4, and a block 3.4px wide would be 3px on one frame and 4 on (is (= 3.4 (:size r)))))
;; the next, which reads as the pupil breathing.
(is (= 3 (:size r)))))
;; ---- the fast path and the specification agree ---- ;; ---- the fast path and the specification agree ----

9
frontend/test/arthur/flow/address_test.cljs
View file

@ -50,7 +50,7 @@
:seed 3 :frames frames :fps (:fps p) :seed 3 :frames frames :fps (:fps p)
:aspect (:aspect p)} :aspect (:aspect p)}
(:detector overrides))))] (:detector overrides))))]
(freeze/clip p (take/measure p {:dense @dense-track})))) (freeze/clip p {:face-1 (take/measure p {:dense @dense-track})})))
(defn- bytes-of [{:keys [data state]}] (defn- bytes-of [{:keys [data state]}]
(str (sha/of-bytes (js/Uint8Array. (.-buffer data))) (str (sha/of-bytes (js/Uint8Array. (.-buffer data)))
@ -228,9 +228,10 @@
(assoc (params-at {}) :analysis (assoc (params-at {}) :analysis
(address/analysis {:detector "synth" :version "mulberry32" (address/analysis {:detector "synth" :version "mulberry32"
:seed 3 :frames frames :fps fps :aspect 1})) :seed 3 :frames frames :fps fps :aspect 1}))
(assoc (take/measure (params-at {}) {:dense @dense-track}) {:face-1 (assoc (take/measure (params-at {}) {:dense @dense-track})
:presence {:brow-r (mapv #(not (contains? gap %)) :presence
(range frames))})))] {:brow-r (mapv #(not (contains? gap %))
(range frames))})}))]
(is (not= (:key (get @base "brows")) (:key (get with "brows")))) (is (not= (:key (get @base "brows")) (:key (get with "brows"))))
(is (not= (:key (get @base "brow-pos")) (:key (get with "brow-pos")))) (is (not= (:key (get @base "brow-pos")) (:key (get with "brow-pos"))))
(doseq [role ["geom" "eyes" "iris-pos" "head-pos" "head-rot" "head-scale"]] (doseq [role ["geom" "eyes" "iris-pos" "head-pos" "head-rot" "head-scale"]]

8
frontend/test/arthur/flow/eye_occlusion_test.cljs
View file

@ -48,18 +48,18 @@
(let [presence (ingest/feature-presence take/frames {:eye-r [[10 14]]}) (let [presence (ingest/feature-presence take/frames {:eye-r [[10 14]]})
{:keys [clip store]} (flow-take/build {:keys [clip store]} (flow-take/build
(assoc take/params :aspect 1 :name "observed-gap") (assoc take/params :aspect 1 :name "observed-gap")
{:dense @take/analysis :presence presence}) {:face-1 {:dense @take/analysis :presence presence}})
sample (fn [id frame] sample (fn [id frame]
(ch/value-at (get-in (clip/nodes clip) [id :channels [:geom :pts]]) (ch/value-at (get-in (:nodes (clip/timeline clip :face-1)) [id :channels [:geom :pts]])
frame store))] frame store))]
(is (empty? (clip/problems clip))) (is (empty? (clip/problems clip)))
(is (= [:eye-r :eye-l] (get-in clip [:groups :eyes-1 :members]))) (is (= [:face-1/eye-r :face-1/eye-l] (get-in clip [:groups :face-1/eyes :members])))
(doseq [f (range 9 14)] (doseq [f (range 9 14)]
(is (ch/nothing? (sample :eye-r f))) (is (ch/nothing? (sample :eye-r f)))
(is (not (ch/nothing? (sample :eye-l f)))) (is (not (ch/nothing? (sample :eye-l f))))
(is (not (ch/nothing? (sample :mouth f))))) (is (not (ch/nothing? (sample :mouth f)))))
(let [drawn (into #{} (map :node) (let [drawn (into #{} (map :node)
((timeline/resolver (clip/root clip) store pal/index-of) 11))] ((timeline/resolver (clip/timeline clip :face-1) store pal/index-of) 11))]
(is (not (contains? drawn :eye-r))) (is (not (contains? drawn :eye-r)))
(is (not (contains? drawn :iris-r))) (is (not (contains? drawn :iris-r)))
(is (contains? drawn :eye-l)) (is (contains? drawn :eye-l))

94
frontend/test/arthur/flow/freeze_test.cljs
View file

@ -30,13 +30,14 @@
;; that has to be right. ;; that has to be right.
(def frozen (delay @take/frozen)) (def frozen (delay @take/frozen))
(def clip* (delay (:clip @frozen))) (def clip* (delay (:clip @frozen)))
;; The ROOT TIMELINE, which is what an evaluator takes. `clip*` is the document — ;; Geometry assertions read the face timeline. Rendering assertions resolve
;; `:fps`, the stage, the analysis, the tracking identities — and `tl*` is the bag ;; the whole clip, including placement and inherited exposure.
;; of nodes in its frame space. `timeline/resolver` refuses the wrong one loudly. (defn- face-timeline [c] (clip/timeline c :face-1))
(def tl* (delay (clip/root @clip*))) (defn- nodes [c] (merge (clip/nodes c) (:nodes (face-timeline c))))
(def tl* (delay (face-timeline @clip*)))
(def store (delay (:store @frozen))) (def store (delay (:store @frozen)))
(defn- node [id] (get-in @tl* [:nodes id])) (defn- node [id] (get (nodes @clip*) id))
(defn- chan [id path] (get-in (node id) [:channels path])) (defn- chan [id path] (get-in (node id) [:channels path]))
(defn- block-of (defn- block-of
@ -57,12 +58,11 @@
(defn- render (defn- render
"One frame of a CLIP into a byte buffer. The stage's size comes off the clip and "One frame of a CLIP into a byte buffer. The stage's size comes off the clip and
the ops off its root timeline, because project dimensions are the project's and the ops from the complete clip, including nested face timelines."
not the footage's — and not a timeline's either."
[c f] [c f]
(let [r (raster/make (:width c) (:height c))] (let [r (raster/make (:width c) (:height c))]
(raster/clear! r (get pal/index-of :bg)) (raster/clear! r (get pal/index-of :bg))
(raster/draw-ops! r (ops-at (clip/root c) f)) (raster/draw-ops! r ((clip/resolver c @store pal/index-of) f))
(vec (array-seq (:buf r))))) (vec (array-seq (:buf r)))))
(defn- drawn (defn- drawn
@ -86,16 +86,11 @@
(is (empty? (clip/problems c)) (pr-str (clip/problems c))))) (is (empty? (clip/problems c)) (pr-str (clip/problems c)))))
(deftest the-tree-is-the-one-the-model-specifies (deftest the-tree-is-the-one-the-model-specifies
;; :face is AUTHORED and :head is MEASURED, and they are two nodes because two (is (= [:face :root] (timeline/lineage (clip/nodes @clip*) :face)))
;; different things want that transform. A group node is free; keeping the (is (= :face-1 (get-in @clip* [:timelines :main :nodes :face-1 :of])))
;; hand-placed and the measured transform apart is the whole reason the (is (= [:head] (timeline/lineage (:nodes @tl*) :head)))
;; transform is decomposed in the first place. (is (= [:mouth :head] (timeline/lineage (:nodes @tl*) :mouth)))
(is (= [:face :root] (timeline/lineage (:nodes @tl*) :face))) (is (= [:mouth-in :mouth :head] (timeline/lineage (:nodes @tl*) :mouth-in)))
(is (= [:head :face :root] (timeline/lineage (:nodes @tl*) :head)))
(is (= [:mouth :head :face :root] (timeline/lineage (:nodes @tl*) :mouth)))
(is (= [:mouth-in :mouth :head :face :root]
(timeline/lineage (:nodes @tl*) :mouth-in)))
;; Exposure lives on the clip root and inherits strictly.
(is (= {:mode :map :expose 2} (:time (node :root)))) (is (= {:mode :map :expose 2} (:time (node :root))))
(is (every? #(nil? (:time (node %))) [:face :head :mouth :mouth-in]))) (is (every? #(nil? (:time (node %))) [:face :head :mouth :mouth-in])))
@ -158,8 +153,9 @@
(is (thrown-with-msg? (is (thrown-with-msg?
ExceptionInfo #"does not fit the block's fixed point" ExceptionInfo #"does not fit the block's fixed point"
(freeze/clip (assoc take/params :name "huge") (freeze/clip (assoc take/params :name "huge")
(update @take/measured :outer {:face-1 (update @take/measured :outer
(fn [rings] (mapv (fn [r] (mapv #(update % :x + 3) r)) rings))))))) (fn [rings]
(mapv (fn [r] (mapv #(update % :x + 3) r)) rings)))}))))
;; --------------------------------------------------------------------------- ;; ---------------------------------------------------------------------------
;; the anchor: three channels, and the inverse ;; the anchor: three channels, and the inverse
@ -199,14 +195,14 @@
;; checking arithmetic against itself; this checks `node/local!`, `node/world!` ;; checking arithmetic against itself; this checks `node/local!`, `node/world!`
;; and `emit` as well. ;; and `emit` as well.
(let [c (freeze/head-mode {:mode :free} @frozen) (let [c (freeze/head-mode {:mode :free} @frozen)
res (timeline/resolver (clip/root c) @store pal/index-of) res (clip/resolver c @store pal/index-of)
k (first (:value (chan :face [:xform :scale]))) k (first (:value (chan :face [:xform :scale])))
anc (:value (chan :face [:xform :anchor])) anc (:value (chan :face [:xform :anchor]))
pos (:value (chan :face [:xform :pos])) pos (:value (chan :face [:xform :pos]))
tfs (:transforms @take/measured)] tfs (:transforms @take/measured)]
(doseq [f (range 0 take/frames 13)] (doseq [f (range 0 take/frames 13)]
(let [ops (res f) (let [ops (res f)
op (first (filter #(= :mouth (:node %)) ops)) op (first (filter #(= [:face-1 :mouth] (:node %)) ops))
;; EXPOSURE FIRST. The clip root is on 2s and exposure inherits ;; EXPOSURE FIRST. The clip root is on 2s and exposure inherits
;; strictly, so frame 13 shows frame 12's pose — which is also the ;; strictly, so frame 13 shows frame 12's pose — which is also the
;; cheapest place to assert that the grid is actually being applied, ;; cheapest place to assert that the grid is actually being applied,
@ -243,23 +239,23 @@
one (freeze/head-mode {:mode :anchored :anchors {0 12}} @frozen) one (freeze/head-mode {:mode :anchored :anchors {0 12}} @frozen)
keyed (freeze/head-mode {:mode :anchored keyed (freeze/head-mode {:mode :anchored
:anchors {0 12, 40 88, 150 150}} @frozen) :anchors {0 12, 40 88, 150 150}} @frozen)
of (fn [c path] (get-in (clip/nodes c) [:head :channels path]))] of (fn [c path] (get-in (nodes c) [:head :channels path]))]
(doseq [c [free one keyed] (doseq [c [free one keyed]
path [[:xform :pos] [:xform :rot] [:xform :scale]]] path [[:xform :pos] [:xform :rot] [:xform :scale]]]
(is (= :dense (ch/describe (of c path)))) (is (= :dense (ch/describe (of c path))))
(is (= (of free path) (of c path)) "anchor edits do not copy measurements")) (is (= (of free path) (of c path)) "anchor edits do not copy measurements"))
(is (nil? (get-in (clip/nodes free) [:head :anchors]))) (is (nil? (get-in (nodes free) [:head :anchors])))
(is (= {0 12} (get-in (clip/nodes one) [:head :anchors]))) (is (= {0 12} (get-in (nodes one) [:head :anchors])))
(is (= {0 12, 40 88, 150 150} (is (= {0 12, 40 88, 150 150}
(get-in (clip/nodes keyed) [:head :anchors]))) (get-in (nodes keyed) [:head :anchors])))
(is (= keyed (leaf/clip "head" (leaf/leaves "head" keyed))) (is (= keyed (leaf/clip "head" (leaf/leaves "head" keyed)))
"anchor source addresses survive the document round trip"))) "anchor source addresses survive the document round trip")))
(deftest head-anchor-keys-hold-the-whole-measured-transform (deftest head-anchor-keys-hold-the-whole-measured-transform
(let [free (timeline/resolver (clip/root (let [free (timeline/resolver (face-timeline
(freeze/head-mode {:mode :free} @frozen)) (freeze/head-mode {:mode :free} @frozen))
@store pal/index-of) @store pal/index-of)
held (timeline/resolver (clip/root held (timeline/resolver (face-timeline
(freeze/head-mode {:mode :anchored (freeze/head-mode {:mode :anchored
:anchors {0 12, 40 88}} @frozen)) :anchors {0 12, 40 88}} @frozen))
@store pal/index-of) @store pal/index-of)
@ -279,14 +275,14 @@
b (freeze/head-mode {:mode :anchored :anchors {0 0}} @frozen) b (freeze/head-mode {:mode :anchored :anchors {0 0}} @frozen)
c (freeze/head-mode {:mode :anchored :anchors {0 0, 40 40}} @frozen)] c (freeze/head-mode {:mode :anchored :anchors {0 0, 40 40}} @frozen)]
(doseq [x [b c]] (doseq [x [b c]]
(is (= (get (clip/nodes a) :face) (get (clip/nodes x) :face)) (is (= (get (nodes a) :face) (get (nodes x) :face))
":face moved") ":face moved")
(is (= (dissoc (clip/nodes a) :head) (dissoc (clip/nodes x) :head)) (is (= (dissoc (nodes a) :head) (dissoc (nodes x) :head))
"a node other than :head changed") "a node other than :head changed")
;; The measurement does not go away when the head is locked: always measure, ;; The measurement does not go away when the head is locked: always measure,
;; always store factored, toggle the parent. ;; always store factored, toggle the parent.
(is (= (get-in (clip/nodes a) [:head :measured]) (is (= (get-in (nodes a) [:head :measured])
(get-in (clip/nodes x) [:head :measured]))) (get-in (nodes x) [:head :measured])))
;; And nothing above the timeline moved either: the toggle is one node's ;; And nothing above the timeline moved either: the toggle is one node's
;; channels, so the clip's own fields and its other timelines are untouched. ;; channels, so the clip's own fields and its other timelines are untouched.
(is (= (dissoc a :timelines) (dissoc x :timelines)))))) (is (= (dissoc a :timelines) (dissoc x :timelines))))))
@ -331,8 +327,8 @@
;; asking for a different stage moves and rescales the same geometry rather than ;; asking for a different stage moves and rescales the same geometry rather than
;; re-measuring anything. ;; re-measuring anything.
(let [big (freeze/clip (assoc take/params :stage [640 480] :name "big") (let [big (freeze/clip (assoc take/params :stage [640 480] :name "big")
@take/measured) {:face-1 @take/measured})
key-of (fn [c] (:store (:dense (get-in (clip/nodes c) key-of (fn [c] (:store (:dense (get-in (nodes c)
[:mouth :channels [:geom :pts]]))))] [:mouth :channels [:geom :pts]]))))]
(is (= [640 480] [(:width (:clip big)) (:height (:clip big))])) (is (= [640 480] [(:width (:clip big)) (:height (:clip big))]))
;; Stronger than it was, and for free: the stage is not an input to tier 2, so ;; Stronger than it was, and for free: the stage is not an input to tier 2, so
@ -344,7 +340,7 @@
(is (= (vec (array-seq (:data (get @store (key-of @clip*))))) (is (= (vec (array-seq (:data (get @store (key-of @clip*)))))
(vec (array-seq (:data (get (:store big) (key-of (:clip big))))))) (vec (array-seq (:data (get (:store big) (key-of (:clip big)))))))
"the geometry is the same numbers at either stage size") "the geometry is the same numbers at either stage size")
(is (not= (:value (get-in (clip/nodes (:clip big)) (is (not= (:value (get-in (nodes (:clip big))
[:face :channels [:xform :scale]])) [:face :channels [:xform :scale]]))
(:value (chan :face [:xform :scale])))))) (:value (chan :face [:xform :scale]))))))
@ -422,14 +418,14 @@
(let [gap (set (range 40 60)) (let [gap (set (range 40 60))
presence {:eye-r (mapv #(not (contains? gap %)) (range take/frames))} presence {:eye-r (mapv #(not (contains? gap %)) (range take/frames))}
c (freeze/clip (assoc take/params :name "one-eye-gappy") c (freeze/clip (assoc take/params :name "one-eye-gappy")
(assoc @take/measured :presence presence)) {:face-1 (assoc @take/measured :presence presence)})
tl (clip/root (:clip c)) tl (face-timeline (:clip c))
at (fn [id f] at (fn [id f]
(ch/value-at (get-in (:nodes tl) [id :channels [:geom :pts]]) f (:store c)))] (ch/value-at (get-in (:nodes tl) [id :channels [:geom :pts]]) f (:store c)))]
;; The identities are the CLIP's, and a gap does not touch them: a feature keeps ;; The identities are the CLIP's, and a gap does not touch them: a feature keeps
;; its id and its pair membership across the frames it was not observed on. ;; its id and its pair membership across the frames it was not observed on.
(is (= [:eye-r :eye-l] (get-in (:clip c) [:groups :eyes-1 :members]))) (is (= [:face-1/eye-r :face-1/eye-l] (get-in (:clip c) [:groups :face-1/eyes :members])))
(is (= :eye-r (get-in (:clip c) [:features :eye-r :id]))) (is (= :face-1/eye-r (get-in (:clip c) [:features :face-1/eye-r :id])))
(is (not (ch/nothing? (at :eye-r 39)))) (is (not (ch/nothing? (at :eye-r 39))))
(is (ch/nothing? (at :eye-r 50))) (is (ch/nothing? (at :eye-r 50)))
(is (not (ch/nothing? (at :eye-r 60)))) (is (not (ch/nothing? (at :eye-r 60))))
@ -460,8 +456,8 @@
(range take/frames))])) (range take/frames))]))
windows) windows)
c (freeze/clip (assoc take/params :name "per-track-gaps") c (freeze/clip (assoc take/params :name "per-track-gaps")
(assoc @take/measured :presence presence)) {:face-1 (assoc @take/measured :presence presence)})
tl (clip/root (:clip c)) tl (face-timeline (:clip c))
at (fn [id path f] at (fn [id path f]
(ch/value-at (get-in (:nodes tl) [id :channels path]) f (:store c))) (ch/value-at (get-in (:nodes tl) [id :channels path]) f (:store c)))
;; Node, channel, and the feature whose gap it must follow. Every dense ;; Node, channel, and the feature whose gap it must follow. Every dense
@ -501,7 +497,8 @@
:shown (vec (repeat take/frames true))} :shown (vec (repeat take/frames true))}
clip (fn [nm extra] clip (fn [nm extra]
(freeze/clip (assoc take/params :name nm) (freeze/clip (assoc take/params :name nm)
(merge (assoc @take/measured :teeth teeth-in) extra))) {:face-1 (merge (assoc @take/measured :teeth teeth-in)
extra)}))
ref (clip "teeth-reference" nil) ref (clip "teeth-reference" nil)
occ (clip "teeth-mouth-gap" occ (clip "teeth-mouth-gap"
{:presence {:mouth (mapv #(not (contains? gap %)) {:presence {:mouth (mapv #(not (contains? gap %))
@ -509,7 +506,7 @@
;; Hoisted: the resolver caches its order and reuses its buffers, so the ;; Hoisted: the resolver caches its order and reuses its buffers, so the
;; node ids come out before the next frame is asked for. ;; node ids come out before the next frame is asked for.
nodes-at (fn [c] nodes-at (fn [c]
(let [r (timeline/resolver (clip/root (:clip c)) (:store c) pal/index-of)] (let [r (timeline/resolver (face-timeline (:clip c)) (:store c) pal/index-of)]
(fn [f] (into #{} (map :node) (r f))))) (fn [f] (into #{} (map :node) (r f)))))
ref-at (nodes-at ref) ref-at (nodes-at ref)
occ-at (nodes-at occ) occ-at (nodes-at occ)
@ -519,7 +516,7 @@
open (filter #(contains? (ref-at %) :teeth) (range take/frames)) open (filter #(contains? (ref-at %) :teeth) (range take/frames))
inside (first (filter gap open)) inside (first (filter gap open))
outside (first (remove gap open))] outside (first (remove gap open))]
(is (= :mouth-in (get-in (clip/nodes (:clip occ)) [:teeth :stencil])) (is (= :mouth-in (get-in (nodes (:clip occ)) [:teeth :stencil]))
"teeth stop inheriting the mouth's absence if this stops being their stencil") "teeth stop inheriting the mouth's absence if this stops being their stencil")
(is (some? inside) "no open-mouth frame inside the gap to test with") (is (some? inside) "no open-mouth frame inside the gap to test with")
(is (some? outside) "no open-mouth frame outside the gap to test with") (is (some? outside) "no open-mouth frame outside the gap to test with")
@ -527,7 +524,7 @@
;; Annotating the MOUTH sets no bit on the teeth: they are a feature of ;; Annotating the MOUTH sets no bit on the teeth: they are a feature of
;; their own and nothing named them. ;; their own and nothing named them.
(is (not (ch/nothing? (is (not (ch/nothing?
(ch/value-at (get-in (clip/nodes (:clip occ)) [:teeth :channels [:geom :pts]]) (ch/value-at (get-in (nodes (:clip occ)) [:teeth :channels [:geom :pts]])
inside (:store occ))))) inside (:store occ)))))
;; They are dropped anyway — :mouth-in drew nothing to clip them against. ;; They are dropped anyway — :mouth-in drew nothing to clip them against.
(is (not (contains? (occ-at inside) :mouth-in))) (is (not (contains? (occ-at inside) :mouth-in)))
@ -544,8 +541,8 @@
(let [gap (set (range 40 60)) (let [gap (set (range 40 60))
det (mapv #(not (contains? gap %)) (range take/frames)) det (mapv #(not (contains? gap %)) (range take/frames))
c (freeze/clip (assoc take/params :name "gappy") c (freeze/clip (assoc take/params :name "gappy")
(assoc @take/measured :detected det)) {:face-1 (assoc @take/measured :detected det)})
tl (clip/root (:clip c)) tl (face-timeline (:clip c))
res (timeline/resolver tl (:store c) pal/index-of)] res (timeline/resolver tl (:store c) pal/index-of)]
(doseq [f [39 40 50 59 60]] (doseq [f [39 40 50 59 60]]
(let [ops (res f)] (let [ops (res f)]
@ -604,7 +601,8 @@
(let [locked (freeze/head-mode {:mode :anchored :anchors {0 0}} @frozen) (let [locked (freeze/head-mode {:mode :anchored :anchors {0 0}} @frozen)
shot (fn [f] shot (fn [f]
(let [r (raster/make W H) (let [r (raster/make W H)
mouth (filter #(= :mouth (:node %)) (ops-at (clip/root locked) f))] mouth (filter #(= [:face-1 :mouth] (:node %))
((clip/resolver locked @store pal/index-of) f))]
(raster/clear! r (get pal/index-of :bg)) (raster/clear! r (get pal/index-of :bg))
(raster/draw-ops! r mouth) (raster/draw-ops! r mouth)
(vec (array-seq (:buf r))))) (vec (array-seq (:buf r)))))

190
frontend/test/arthur/flow/multi_face_test.cljs Normal file
View file

@ -0,0 +1,190 @@
(ns arthur.flow.multi-face-test
(:require [cljs.test :refer [deftest is testing]]
[arthur.demo.stage :as stage]
[arthur.domain.channel :as ch]
[arthur.domain.clip :as clip]
[arthur.domain.feature :as feature]
[arthur.domain.pose :as pose]
[arthur.domain.project :as project]
[arthur.events.footage :as footage]
[arthur.flow.address :as address]
[arthur.flow.detect :as detect]
[arthur.flow.freeze :as freeze]
[arthur.flow.regenerate :as regenerate]
[arthur.flow.source :as source]
[arthur.flow.take :as take]
[arthur.support.ops :as ops]
[arthur.synth :as synth]))
(def frames 40)
(def settings
(merge take/knobs {:name "two faces" :fps 30 :aspect 1 :stage [320 200]
:fit-motion? true :expose 1 :head :anchored :anchors {0 0}
:analysis (address/analysis {:detector "synth" :version "two-faces-v1"
:seed 9 :frames frames :fps 30 :aspect 1})}))
(def inputs
(delay
(into {} (for [[id seed dx] [[:face-1 9 -0.3] [:face-2 12 0.3]]]
[id {:dense (mapv (fn [face] (mapv #(update % :x + dx) face))
(synth/synth-dense frames {:seed seed}))
:detected (vec (repeat frames true))
:crops (vec (repeat frames nil))}]))))
(def initial
(delay (assoc (take/build settings @inputs) :source-inputs {:subjects @inputs})))
(defn channel [entry subject node path]
(get-in entry [:clip :timelines subject :nodes node :channels path]))
(defn snapshot [c store f] (ops/snapshot ((clip/resolver c store) f)))
(defn by-node [c store f] (into {} (map (juxt :node identity)) (snapshot c store f)))
(deftest subjects-share-local-names-without-sharing-blocks
(let [{:keys [clip store]} @initial
a (get-in clip [:timelines :face-1 :nodes])
b (get-in clip [:timelines :face-2 :nodes])]
(is (empty? (clip/problems clip)))
(is (= (set (keys a)) (set (keys b))))
(is (= :head (get-in b [:mouth :parent])))
(is (= :eye-r-in (get-in b [:iris-r :stencil])))
(is (= :mouth (get-in b [:mouth :pose-group])))
(is (= :face-2 (get-in clip [:features :face-2/mouth :timeline])))
(doseq [path [[:xform :pos] [:xform :rot] [:xform :scale]]]
(is (not= (:store (:dense (get-in a [:head :measured path])))
(:store (:dense (get-in b [:head :measured path]))))))
(let [drawn (by-node clip store 12)
left (get drawn [:face-1 :mouth])
right (get drawn [:face-2 :mouth])]
(is (seq (:points left)))
(is (< (ffirst (:points left)) (ffirst (:points right)))
"one shared placement preserves the filmed separation"))
(is (some #(re-find #"more than one feature" %)
(feature/problems
(assoc-in clip [:features :duplicate]
(assoc (get-in clip [:features :face-2/mouth]) :id :duplicate)))))))
(deftest one-subjects-edit-and-anchor-do-not-change-its-neighbor
(let [before @initial
at [:clip :timelines :face-2 :nodes :iris-r :channels [:style :color]]
before (assoc-in before at (ch/framed :brow))
after (regenerate/change before {:scope :feature :id :face-2/eye-r
:knob :gaze-gain :value 2})
anchored (freeze/head-mode {:subject :face-2 :mode :anchored :anchors {0 12}} after)]
(is (= (get-in before at) (get-in after at)) "authored channels survive regeneration")
(is (= (get-in before [:clip :timelines :face-1])
(get-in after [:clip :timelines :face-1])
(get-in anchored [:timelines :face-1])))
(is (not= (channel before :face-2 :iris-r [:xform :pos])
(channel after :face-2 :iris-r [:xform :pos])))
(is (= (channel before :face-2 :iris-l [:xform :pos])
(channel after :face-2 :iris-l [:xform :pos])))
(is (= {0 12} (get-in anchored [:timelines :face-2 :nodes :head :anchors])))
(is (empty? (clip/problems anchored)))))
(deftest the-second-subject-regenerates-inside-a-composed-stage
(let [before (update @initial :clip stage/compose)
after (regenerate/change before {:scope :subject :id :face-2
:knob :anchor-avg :value 4})
full (take/build (assoc settings :anchor-avg 4) @inputs)]
(is (= (get-in full [:clip :timelines :face-2 :nodes :head :measured])
(get-in after [:clip :timelines :face-2 :nodes :head :measured])))
(doseq [tid [:main :sym/face-8625 :face-1]]
(is (= (get-in before [:clip :timelines tid])
(get-in after [:clip :timelines tid]))))
(is (empty? (clip/problems (:clip after))))
(is (thrown? ExceptionInfo
(freeze/head-mode {:subject :face-2 :mode :anchored :anchors {0 frames}}
after))
"anchors are bounded by the source timeline, even on a longer stage")))
(deftest an-instance-pose-cut-only-holds-that-faces-mouth
(let [{:keys [clip store]} @initial
cut (pose/put-cut clip :face-2 :mouth 0 20)
original (by-node clip store 4)
held (by-node cut store 4)
source (by-node clip store 20)]
(is (empty? (clip/problems cut)))
(is (= (get held [:face-2 :mouth]) (get source [:face-2 :mouth])))
(is (not= (get held [:face-2 :mouth]) (get original [:face-2 :mouth])))
(doseq [id [[:face-1 :mouth] [:face-1 :eye-r] [:face-2 :eye-r]]]
(is (= (get held id) (get original id))))))
(deftest detection-and-feature-gaps-are-per-subject
(let [inputs (-> @inputs
(assoc-in [:face-2 :detected 8] false)
(assoc-in [:face-2 :presence :eye-r]
(assoc (vec (repeat frames true)) 12 false)))
{:keys [clip store]} (take/build settings inputs)
clip (freeze/head-mode {:mode :free} {:clip clip})
at8 (by-node clip store 8)
at12 (by-node clip store 12)]
(is (some? (get at8 [:face-1 :mouth])))
(is (nil? (get at8 [:face-2 :mouth])))
(is (nil? (get at12 [:face-2 :eye-r])))
(is (nil? (get at12 [:face-2 :iris-r])))
(is (some? (get at12 [:face-1 :eye-r])))
(is (some? (get at12 [:face-2 :eye-l])))
(is (some? (get (by-node clip store 13) [:face-2 :eye-r])))))
(deftest nested-faces-survive-save-and-resolve-through-the-stage
(let [entry (update @initial :clip stage/compose)
back (project/load "two" (js/JSON.parse (js/JSON.stringify (project/save "two" entry))))]
(is (= (:clip entry) (:clip back)))
(is (empty? (clip/problems (:clip back))))
(doseq [f [0 12 30 10]]
(let [before (snapshot (:clip entry) (:store entry) f)
after (snapshot (:clip back) (:store back) f)]
(is (seq before))
(is (some #(= [:face-2 :mouth] (second (:node %))) before)))
(is (= (snapshot (:clip entry) (:store entry) f)
(snapshot (:clip back) (:store back) f))))))
(deftest source-round-trip-keeps-two-faces-with-identical-detection-masks
(let [blocks (source/pack-subjects (get-in settings [:analysis :id]) @inputs)
wire (source/wire-blocks blocks)
back (:subjects (source/unpack (.reverse wire) [320 200]))]
(is (= 6 (count (set (source/block-keys blocks)))) )
(doseq [subject [:face-1 :face-2]]
(is (= (select-keys (get @inputs subject) [:dense :detected])
(select-keys (get back subject) [:dense :detected]))))))
(deftest reordered-detections-and-gaps-retain-their-track
(let [a [{:x 0.1 :y 0.5}] b [{:x 0.8 :y 0.5}]
frames [[a] [b a] [] [a b]]
tracks (detect/tracks frames)]
(is (= {:face-1 [0 1 nil 0] :face-2 [nil 0 nil 1]} tracks))
(is (= [false true false true]
(:detected (detect/fill-gaps (detect/pick frames (:face-2 tracks))))))
(is (thrown? ExceptionInfo (detect/tracks [[] []])))))
(deftest manifest-masks-enter-measurement-with-local-feature-names
(let [manifest {:presence {:eye-l [false true]
:face-1/eye-r [true false]
:face-2/eye-r [false true]}}]
(is (= {:eye-l [false true] :eye-r [true false]}
(footage/presence-for manifest :face-1)))
(is (= {:eye-r [false true]} (footage/presence-for manifest :face-2)))))
(deftest tracking-settings-participate-in-the-analysis-address
(let [manifest {:width 320 :height 200 :frames 40 :fps 30 :source "source"}
analysis (take/analysis-for manifest {:version "test"})]
(is (= detect/settings (:tracking analysis)))
(is (not= (:id analysis) (:id (address/analysis (dissoc analysis :tracking)))))
(is (not= (:id analysis)
(:id (address/analysis (assoc-in analysis [:tracking :gate] 0.3)))))))
(deftest the-instance-boundary-preserves-the-single-face-geometry
(let [{:keys [clip store]} (take/build settings (select-keys @inputs [:face-1]))
flat (assoc-in clip [:timelines :main :nodes]
(merge (dissoc (clip/nodes clip) :face-1)
(assoc-in (get-in clip [:timelines :face-1 :nodes])
[:head :parent] :face)))
nested (clip/resolver clip store)
reference (clip/resolver flat store)]
(doseq [f [0 1 7 20 39]]
(let [a (ops/snapshot (nested f)) b (ops/snapshot (reference f))]
(is (= (mapv (comp second :node) a) (mapv :node b)))
(doseq [[x y] (map vector a b)]
(is (= (:kind x) (:kind y)))
(doseq [[u v] (map vector
(concat (mapcat identity (:points x)) (keep x [:cx :cy :r :size]))
(concat (mapcat identity (:points y)) (keep y [:cx :cy :r :size])))]
(is (< (abs (- u v)) 1e-8))))))))

57
frontend/test/arthur/flow/regenerate_test.cljs
View file

@ -21,7 +21,7 @@
:analysis (address/analysis :analysis (address/analysis
{:detector "synth" :version "mulberry32" {:detector "synth" :version "mulberry32"
:seed 9 :frames frames :fps 30 :aspect 1})})] :seed 9 :frames frames :fps 30 :aspect 1})})]
(assoc (take/build p @inputs) :source-inputs @inputs)))) (assoc (take/build p {:face-1 @inputs}) :source-inputs {:subjects {:face-1 @inputs}}))))
(defn- full-at [change] (defn- full-at [change]
(take/build (take/build
@ -30,15 +30,15 @@
:expose 1 :head :free :expose 1 :head :free
:analysis (get-in @initial [:clip :analysis])} :analysis (get-in @initial [:clip :analysis])}
change) change)
@inputs)) {:face-1 @inputs}))
(defn- channel [entry node path] (defn- channel [entry node path]
(get-in entry [:clip :timelines :main :nodes node :channels path])) (get-in entry [:clip :timelines :face-1 :nodes node :channels path]))
(deftest eye-rebuild-is-confined-to-the-edited-feature (deftest eye-rebuild-is-confined-to-the-edited-feature
(let [before @initial (let [before @initial
after (regenerate/change before after (regenerate/change before
{:scope :feature :id :eye-r :knob :gaze-gain :value 2})] {:scope :feature :id :face-1/eye-r :knob :gaze-gain :value 2})]
(is (not= (channel before :iris-r [:xform :pos]) (is (not= (channel before :iris-r [:xform :pos])
(channel after :iris-r [:xform :pos]))) (channel after :iris-r [:xform :pos])))
(is (= (channel before :iris-l [:xform :pos]) (is (= (channel before :iris-l [:xform :pos])
@ -53,7 +53,7 @@
(deftest framed-eye-size-needs-no-new-dense-block (deftest framed-eye-size-needs-no-new-dense-block
(let [before @initial (let [before @initial
after (regenerate/change before after (regenerate/change before
{:scope :feature :id :eye-r :knob :iris-size :value 0.6})] {:scope :feature :id :face-1/eye-r :knob :iris-size :value 0.6})]
(is (not= (channel before :iris-r [:geom :radius]) (is (not= (channel before :iris-r [:geom :radius])
(channel after :iris-r [:geom :radius]))) (channel after :iris-r [:geom :radius])))
(is (= (channel before :iris-l [:geom :radius]) (is (= (channel before :iris-l [:geom :radius])
@ -65,11 +65,11 @@
(deftest mouth-edit-leaves-the-head-and-eye-alone (deftest mouth-edit-leaves-the-head-and-eye-alone
(let [before @initial (let [before @initial
after (regenerate/change before after (regenerate/change before
{:scope :feature :id :mouth :knob :verts :value 10})] {:scope :feature :id :face-1/mouth :knob :verts :value 10})]
(is (not= (channel before :mouth [:geom :pts]) (is (not= (channel before :mouth [:geom :pts])
(channel after :mouth [:geom :pts]))) (channel after :mouth [:geom :pts])))
(is (= (get-in before [:clip :timelines :main :nodes :head]) (is (= (get-in before [:clip :timelines :face-1 :nodes :head])
(get-in after [:clip :timelines :main :nodes :head]))) (get-in after [:clip :timelines :face-1 :nodes :head])))
(is (= (channel before :eye-r [:geom :pts]) (is (= (channel before :eye-r [:geom :pts])
(channel after :eye-r [:geom :pts]))) (channel after :eye-r [:geom :pts])))
(is (= (channel (full-at {:verts 10}) :mouth [:geom :pts]) (is (= (channel (full-at {:verts 10}) :mouth [:geom :pts])
@ -80,10 +80,10 @@
after (regenerate/change before after (regenerate/change before
{:scope :subject :id :face-1 :knob :anchor-avg :value 4}) {:scope :subject :id :face-1 :knob :anchor-avg :value 4})
full (full-at {:anchor-avg 4})] full (full-at {:anchor-avg 4})]
(is (not= (get-in before [:clip :timelines :main :nodes :head :measured]) (is (not= (get-in before [:clip :timelines :face-1 :nodes :head :measured])
(get-in after [:clip :timelines :main :nodes :head :measured]))) (get-in after [:clip :timelines :face-1 :nodes :head :measured])))
(is (= (get-in full [:clip :timelines :main :nodes :head :measured]) (is (= (get-in full [:clip :timelines :face-1 :nodes :head :measured])
(get-in after [:clip :timelines :main :nodes :head :measured]))) (get-in after [:clip :timelines :face-1 :nodes :head :measured])))
(is (= (get-in before [:clip :timelines :main :nodes :face]) (is (= (get-in before [:clip :timelines :main :nodes :face])
(get-in after [:clip :timelines :main :nodes :face]))))) (get-in after [:clip :timelines :main :nodes :face])))))
@ -91,14 +91,14 @@
(let [before @initial (let [before @initial
after (regenerate/change before after (regenerate/change before
{:scope :subject :id :face-1 :knob :contour-avg :value 3})] {:scope :subject :id :face-1 :knob :contour-avg :value 3})]
(is (= (get-in before [:clip :timelines :main :nodes :head]) (is (= (get-in before [:clip :timelines :face-1 :nodes :head])
(get-in after [:clip :timelines :main :nodes :head]))) (get-in after [:clip :timelines :face-1 :nodes :head])))
(is (= (channel before :teeth [:geom :pts]) (is (= (channel before :teeth [:geom :pts])
(channel after :teeth [:geom :pts]))))) (channel after :teeth [:geom :pts])))))
(deftest edited-take-still-round-trips-through-normal-save (deftest edited-take-still-round-trips-through-normal-save
(let [after (regenerate/change @initial (let [after (regenerate/change @initial
{:scope :group :id :eyes-1 :knob :iris-size :value 0.6}) {:scope :group :id :face-1/eyes :knob :iris-size :value 0.6})
wire (project/save "regen" after) wire (project/save "regen" after)
loaded (project/load "regen" (js/JSON.parse (js/JSON.stringify wire)))] loaded (project/load "regen" (js/JSON.parse (js/JSON.stringify wire)))]
(is (= (:clip after) (:clip loaded))) (is (= (:clip after) (:clip loaded)))
@ -172,7 +172,7 @@
{:fps 30 :aspect 1 :analysis (get-in @initial [:clip :analysis])} {:fps 30 :aspect 1 :analysis (get-in @initial [:clip :analysis])}
overrides) overrides)
inputs (assoc @inputs :interior @interior-track)] inputs (assoc @inputs :interior @interior-track)]
(freeze/part area p (take/measure-part area p inputs (take/anchor-base p inputs))))) (freeze/part :face-1 area p (take/measure-part area p inputs (take/anchor-base p inputs)))))
(def ^:private swept (def ^:private swept
"Per area, the knobs whose bytes or channels that area's freeze could plausibly "Per area, the knobs whose bytes or channels that area's freeze could plausibly
@ -202,22 +202,9 @@
;; ---- the stage: a shared symbol behind many placements ---- ;; ---- the stage: a shared symbol behind many placements ----
(defn- staged (defn- staged [] (update @initial :clip stage/compose))
"The take composed onto the 8625 stage: its timeline becomes the shared symbol,
and seven uuid-keyed placements sit on `:main`.
This path had no coverage, and it is the one that differs structurally: `change` (defn- sym-channel [entry node path] (channel entry node path))
re-roots the SYMBOL as the take, edits that, and puts it back, so every
assumption `change-take` makes about `:main` holding the tracked nodes is only
true of the re-rooted document and not of the stage itself."
[]
;; The ENTRY with its clip composed, not the composed clip: `change` takes an
;; entry (`:clip`, `:store`, `:source-inputs`) and the store is what the
;; regenerated blocks merge into.
(update @initial :clip stage/compose))
(defn- sym-channel [entry node path]
(get-in entry [:clip :timelines :sym/face-8625 :nodes node :channels path]))
(deftest a-stage-edit-is-previewable-at-all (deftest a-stage-edit-is-previewable-at-all
;; The guard `events/project/::preview-settings` bails on, stated here so a ;; The guard `events/project/::preview-settings` bails on, stated here so a
@ -228,11 +215,11 @@
"the composed stage keeps the analysis the edit needs") "the composed stage keeps the analysis the edit needs")
(is (some? (:source-inputs entry))) (is (some? (:source-inputs entry)))
(testing "and the features still say which timeline they live in" (testing "and the features still say which timeline they live in"
(is (every? #(= :sym/face-8625 (:timeline %)) (is (every? #(= :face-1 (:timeline %))
(vals (:features (:clip entry)))))))) (vals (:features (:clip entry))))))))
(deftest a-stage-edit-plans-the-same-features-as-a-take-edit (deftest a-stage-edit-plans-the-same-features-as-a-take-edit
(let [edit {:scope :feature :id :eye-r :knob :iris-size :value 0.6}] (let [edit {:scope :feature :id :face-1/eye-r :knob :iris-size :value 0.6}]
(is (= (:features (regenerate/plan (:clip @initial) edit)) (is (= (:features (regenerate/plan (:clip @initial) edit))
(:features (regenerate/plan (:clip (staged)) edit))) (:features (regenerate/plan (:clip (staged)) edit)))
"the same knob dirties the same features on a stage as on a take"))) "the same knob dirties the same features on a stage as on a take")))
@ -243,7 +230,7 @@
;; and `:main` — which holds only placements — must come back untouched. ;; and `:main` — which holds only placements — must come back untouched.
(let [before (staged) (let [before (staged)
after (regenerate/change before after (regenerate/change before
{:scope :feature :id :eye-r :knob :iris-size :value 0.6})] {:scope :feature :id :face-1/eye-r :knob :iris-size :value 0.6})]
(is (not= (sym-channel before :iris-r [:geom :radius]) (is (not= (sym-channel before :iris-r [:geom :radius])
(sym-channel after :iris-r [:geom :radius])) (sym-channel after :iris-r [:geom :radius]))
"the shared drawing is what changed") "the shared drawing is what changed")
@ -261,7 +248,7 @@
;; faces off the stage, or dangle the voice links, while looking like a ;; faces off the stage, or dangle the voice links, while looking like a
;; successful edit of the drawing. ;; successful edit of the drawing.
(let [after (regenerate/change (staged) (let [after (regenerate/change (staged)
{:scope :feature :id :eye-r :knob :iris-size :value 0.6}) {:scope :feature :id :face-1/eye-r :knob :iris-size :value 0.6})
nodes (get-in after [:clip :timelines :main :nodes]) nodes (get-in after [:clip :timelines :main :nodes])
syms (filter (comp #{:symbol} :kind val) nodes)] syms (filter (comp #{:symbol} :kind val) nodes)]
(is (= 7 (count syms))) (is (= 7 (count syms)))

19
frontend/test/arthur/flow/source_test.cljs
View file

@ -10,14 +10,14 @@
(range (:teeth-verts settings))) (range (:teeth-verts settings)))
measures [{:contour ring :contrast 0.7 :area 42} measures [{:contour ring :contrast 0.7 :area 42}
{:contour nil :contrast 0 :area 0}] {:contour nil :contrast 0 :area 0}]
block (source/interior-block "sha256:analysis" settings measures) block (source/interior-block "sha256:analysis" :face-1 settings measures)
response #js {:descriptor (:descriptor block) response #js {:descriptor (:descriptor block)
:data (wire/base64 (:data block))}] :data (wire/base64 (:data block))}]
(is (= (:key block) (is (= (:key block)
(source/interior-key "sha256:analysis" settings 2))) (source/interior-key "sha256:analysis" :face-1 settings 2)))
(is (= measures (source/unpack-interior response settings 2))) (is (= measures (source/unpack-interior response settings 2)))
(is (not= (:key block) (is (not= (:key block)
(source/interior-key "sha256:analysis" (source/interior-key "sha256:analysis" :face-1
(assoc settings :cavity-erode 0.2) 2))))) (assoc settings :cavity-erode 0.2) 2)))))
(deftest source-blocks-round-trip-without-source-images (deftest source-blocks-round-trip-without-source-images
@ -25,8 +25,9 @@
pixels (js/Uint8ClampedArray. #js [12 24 36 255]) pixels (js/Uint8ClampedArray. #js [12 24 36 255])
input {:dense [face face] :detected [true false] input {:dense [face face] :detected [true false]
:crops [{:box {:x 3 :y 4 :w 1 :h 1} :data pixels} nil]} :crops [{:box {:x 3 :y 4 :w 1 :h 1} :data pixels} nil]}
blocks (source/pack "sha256:analysis" input) blocks (source/pack "sha256:analysis" :face-1 input)
out (source/unpack (source/wire-blocks blocks) [100 80])] out (get-in (source/unpack (source/wire-blocks {:face-1 blocks}) [100 80])
[:subjects :face-1])]
(is (= (set source/roles) (set (keys blocks)))) (is (= (set source/roles) (set (keys blocks))))
(is (= (:dense input) (:dense out))) (is (= (:dense input) (:dense out)))
(is (= (:detected input) (:detected out))) (is (= (:detected input) (:detected out)))
@ -36,20 +37,20 @@
(vec (array-seq (:data (first (:crops out))))))) (vec (array-seq (:data (first (:crops out)))))))
(is (nil? (second (:crops out)))) (is (nil? (second (:crops out))))
(is (= (mapv :key (vals blocks)) (is (= (mapv :key (vals blocks))
(mapv :key (vals (source/pack "sha256:analysis" out))))))) (mapv :key (vals (source/pack "sha256:analysis" :face-1 out)))))))
(deftest fresh-source-includes-its-pixel-measurement-block (deftest fresh-source-includes-its-pixel-measurement-block
(let [face (vec (repeat 478 {:x 0.25 :y 0.5 :z -0.125})) (let [face (vec (repeat 478 {:x 0.25 :y 0.5 :z -0.125}))
input {:dense [face] :detected [false] :crops [nil] input {:dense [face] :detected [false] :crops [nil]
:interior [{:contour nil :contrast 0 :area 0}] :interior [{:contour nil :contrast 0 :area 0}]
:interior-settings params/defaults} :interior-settings params/defaults}
blocks (source/pack "sha256:analysis" input)] blocks (source/pack "sha256:analysis" :face-1 input)]
(is (contains? blocks "source/interior")) (is (contains? blocks "source/interior"))
(is (= 4 (.-length (source/upload-blocks blocks)))) (is (= 4 (.-length (source/upload-blocks {:face-1 blocks}))))
(is (= 3 (count source/roles))) (is (= 3 (count source/roles)))
(testing "and cannot be packed without the settings it was measured at" (testing "and cannot be packed without the settings it was measured at"
;; The block is addressed BY those knobs. Defaulting them would name it ;; The block is addressed BY those knobs. Defaulting them would name it
;; after settings its bytes did not come from, which is a key that lies. ;; after settings its bytes did not come from, which is a key that lies.
(is (thrown-with-msg? (is (thrown-with-msg?
ExceptionInfo #"needs the settings" ExceptionInfo #"needs the settings"
(source/pack "sha256:analysis" (dissoc input :interior-settings))))))) (source/pack "sha256:analysis" :face-1 (dissoc input :interior-settings)))))))

20
frontend/test/browser/take.mjs
View file

@ -254,6 +254,7 @@ async function main() {
await sleep(120); await sleep(120);
const open = await page.eval(PROBE); const open = await page.eval(PROBE);
await page.shot('take-open'); await page.shot('take-open');
check(open.toneSet.includes(0x171a22), 'nested face pupils reach the rasterizer');
check(open.toneSet.includes(MOUTH_DARK), check(open.toneSet.includes(MOUTH_DARK),
'an open mouth draws an outline and an interior', 'an open mouth draws an outline and an interior',
`${open.tones} tones, interior ${open.toneSet.includes(MOUTH_DARK) ? 'present' : 'MISSING'}`); `${open.tones} tones, interior ${open.toneSet.includes(MOUTH_DARK) ? 'present' : 'MISSING'}`);
@ -493,13 +494,9 @@ async function main() {
check(stageLoaded !== null, 'the 8625 stage is ready', stageLoaded ?? (await page.eval(STATUS))); check(stageLoaded !== null, 'the 8625 stage is ready', stageLoaded ?? (await page.eval(STATUS)));
const eyeSelected = await page.eval(`(() => { const eyeSelected = await page.eval(`(() => {
const select = document.querySelector('.controls select'); const select = document.querySelector('.controls select');
// The instance prefix is the placement's NAME ("8625 left"), not its id: // Settings belong to tracked features shared by the stage placements.
// a placement is keyed by a uuid now, and a uuid is not something to show
// anyone. So this matches the feature and requires SOME instance prefix,
// rather than pinning the label a rename is free to change.
const option = [...select.options] const option = [...select.options]
.find(o => o.textContent.trim().endsWith('/ feature · eye-r') .find(o => o.textContent.trim().endsWith('feature · face-1/eye-r'));
&& o.textContent.includes(' / '));
if (!option) return false; if (!option) return false;
select.value = option.value; select.value = option.value;
select.dispatchEvent(new Event('change', { bubbles: true })); select.dispatchEvent(new Event('change', { bubbles: true }));
@ -533,7 +530,7 @@ async function main() {
const preview = await statusMatching(/preview · unsaved/, 160); const preview = await statusMatching(/preview · unsaved/, 160);
const debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`); const debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`);
check(preview !== null, 'the slider updates the stage preview', preview ?? debug); check(preview !== null, 'the slider updates the stage preview', preview ?? debug);
check(debug.includes(':eye-r') && debug.includes('tier 1 only'), check(debug.includes(':face-1/eye-r') && debug.includes('tier 1 only'),
'the panel reports the affected feature and tier', debug); 'the panel reports the affected feature and tier', debug);
const endFrame = await page.eval(`Number(document.querySelector('.readout span').textContent.match(/\\d+/)[0])`); const endFrame = await page.eval(`Number(document.querySelector('.readout span').textContent.match(/\\d+/)[0])`);
check(endFrame > startFrame, 'playback continues during tuning', `${startFrame} -> ${endFrame}`); check(endFrame > startFrame, 'playback continues during tuning', `${startFrame} -> ${endFrame}`);
@ -581,7 +578,7 @@ async function main() {
if (debug.includes('head-pos')) break; if (debug.includes('head-pos')) break;
await sleep(250); await sleep(250);
} }
check(debug.includes('head-pos') && debug.includes(':teeth'), check(debug.includes('head-pos') && debug.includes(':face-1/teeth'),
'anchor invalidates the head and teeth', debug); 'anchor invalidates the head and teeth', debug);
const preview = await statusMatching(/preview · unsaved/, 160); const preview = await statusMatching(/preview · unsaved/, 160);
check(preview !== null, 'the anchor edit finishes previewing', check(preview !== null, 'the anchor edit finishes previewing',
@ -591,8 +588,7 @@ async function main() {
const teethSelected = await page.eval(`(() => { const teethSelected = await page.eval(`(() => {
const select = document.querySelector('.controls select'); const select = document.querySelector('.controls select');
const option = [...select.options] const option = [...select.options]
.find(o => o.textContent.trim().endsWith('/ feature · teeth') .find(o => o.textContent.trim().endsWith('feature · face-1/teeth'));
&& o.textContent.includes(' / '));
if (!option) return false; if (!option) return false;
select.value = option.value; select.value = option.value;
select.dispatchEvent(new Event('change', { bubbles: true })); select.dispatchEvent(new Event('change', { bubbles: true }));
@ -627,10 +623,10 @@ async function main() {
let debug = ''; let debug = '';
for (let i = 0; i < 160; i++) { for (let i = 0; i < 160; i++) {
debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`); debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`);
if (debug.includes('dirty features: [:teeth]')) break; if (debug.includes('dirty features: [:face-1/teeth]')) break;
await sleep(250); await sleep(250);
} }
check(debug.includes('dirty features: [:teeth]'), check(debug.includes('dirty features: [:face-1/teeth]'),
'the pixel setting invalidates only teeth', debug); 'the pixel setting invalidates only teeth', debug);
const preview = await statusMatching(/preview · unsaved/, 160); const preview = await statusMatching(/preview · unsaved/, 160);
check(preview !== null, 'the teeth edit finishes previewing', check(preview !== null, 'the teeth edit finishes previewing',