multi fce stuff
This commit is contained in:
Olive Vaughn
parent
49ece8dee6
commit
ddabfbeaa8
32 changed files with 1309 additions and 700 deletions
|
|
@ -327,6 +327,34 @@ class Tier2Tests(TestCase):
|
|||
f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys[:2]}),
|
||||
content_type="application/json").status_code)
|
||||
|
||||
def test_source_roles_are_complete_and_unique_per_subject(self):
|
||||
analysis = self.register_analysis()
|
||||
keys = []
|
||||
for subject in ("face-1", "face-2"):
|
||||
for role in ("source/dense", "source/detected", "source/crops"):
|
||||
desc = json.loads(block_descriptor(analysis, role=role))
|
||||
desc["features"] = [subject]
|
||||
descriptor = json.dumps(desc, sort_keys=True, separators=(",", ":"))
|
||||
key = key_for(descriptor)
|
||||
self.assertEqual(201, self.post("/api/blocks", {
|
||||
"key": key, "descriptor": descriptor, "data": "AA==",
|
||||
}).status_code)
|
||||
keys.append(key)
|
||||
|
||||
def put(keys):
|
||||
return self.client.put(f"/api/analyses/{analysis}",
|
||||
json.dumps({"source_blocks": keys}),
|
||||
content_type="application/json")
|
||||
|
||||
self.assertEqual(400, put(keys[:-1]).status_code)
|
||||
self.assertEqual(400, put(keys + keys[:1]).status_code)
|
||||
self.assertEqual(200, put(keys).status_code)
|
||||
self.assertEqual(200, put(list(reversed(keys))).status_code)
|
||||
self.assertEqual(409, put(keys[:3]).status_code)
|
||||
self.assertEqual(set(keys), set(self.client.get(
|
||||
f"/api/analyses/{analysis}").json()["source_blocks"]))
|
||||
|
||||
|
||||
|
||||
@override_settings(BLOB_ROOT=BLOB_DIR)
|
||||
class DocumentTests(TestCase):
|
||||
|
|
|
|||
|
|
@ -433,12 +433,23 @@ def analysis_detail(request, key):
|
|||
try:
|
||||
keys = _body(request).get("source_blocks")
|
||||
roles = {"source/dense", "source/detected", "source/crops"}
|
||||
if not isinstance(keys, list) or len(keys) != len(roles) or len(set(keys)) != len(roles):
|
||||
raise Bad("an analysis needs one block for each source role")
|
||||
if (not isinstance(keys, list) or not keys
|
||||
or not all(isinstance(k, str) for k in keys) or len(set(keys)) != len(keys)):
|
||||
raise Bad("an analysis needs distinct source block keys")
|
||||
blocks = list(Block.objects.filter(key__in=keys))
|
||||
if (len(blocks) != len(roles) or {b.role for b in blocks} != roles
|
||||
or any(b.analysis_id != key for b in blocks)):
|
||||
raise Bad("source blocks must have distinct source roles and name this analysis")
|
||||
if len(blocks) != len(keys) or any(b.analysis_id != key for b in blocks):
|
||||
raise Bad("source blocks must exist and name this analysis")
|
||||
by_subject = {}
|
||||
for block in blocks:
|
||||
subjects = json.loads(block.descriptor).get("features", [])
|
||||
if (not isinstance(subjects, list) or len(subjects) > 1
|
||||
or any(not isinstance(s, str) or not s for s in subjects)):
|
||||
raise Bad("a source block must name one subject")
|
||||
# Older single-face analyses used an empty feature list.
|
||||
by_subject.setdefault(tuple(subjects), []).append(block.role)
|
||||
if any(len(found) != len(roles) or set(found) != roles
|
||||
for found in by_subject.values()):
|
||||
raise Bad("each subject needs one block for each source role")
|
||||
with transaction.atomic():
|
||||
row = Analysis.objects.select_for_update().get(key=key)
|
||||
current = set(row.source_blocks.values_list("key", flat=True))
|
||||
|
|
|
|||
101
docs/multi-face-representation.md
Normal file
101
docs/multi-face-representation.md
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
# Multi-face representation
|
||||
|
||||
Status (2026-09-29): representation work complete. Reopen it for a concrete
|
||||
requirement, rather than another round of abstract alternatives.
|
||||
|
||||
Implemented: each tracked face has a drawing timeline, placed by an ordinary
|
||||
symbol instance. Timelines already provide local node names, independent playback,
|
||||
and persistence. No new kind of scene container is needed.
|
||||
|
||||
```clojure
|
||||
:timelines
|
||||
{:main {:nodes {:root {:time {:mode :map :expose 2}}
|
||||
:face {:parent :root :channels <source-to-stage placement>}
|
||||
:face-1 {:kind :symbol :of :face-1 :parent :face :z "a0"}
|
||||
:face-2 {:kind :symbol :of :face-2 :parent :face :z "a1"}}}
|
||||
:face-1 {:nodes {:head {...} :mouth {:parent :head ...} ...}}
|
||||
:face-2 {:nodes {:head {...} :mouth {:parent :head ...} ...}}}
|
||||
|
||||
:features
|
||||
{:face-1/mouth {:subject :face-1 :timeline :face-1 :area :mouth
|
||||
:nodes [:mouth :mouth-in]}
|
||||
:face-2/mouth {:subject :face-2 :timeline :face-2 :area :mouth
|
||||
:nodes [:mouth :mouth-in]}}
|
||||
```
|
||||
|
||||
The example omits ordinary ids, frame counts and channel details.
|
||||
|
||||
## What belongs where
|
||||
|
||||
- A **subject** identifies a source track and supplies shared measurement settings.
|
||||
Its timeline has the same id and contains its measured `:head`.
|
||||
- A **feature** owns nodes in an explicitly named timeline. Its clip-level id is
|
||||
qualified when generated; ownership is read from fields, never parsed from ids.
|
||||
- A **node** has a local name. Parents, stencils and pose groups use local names too.
|
||||
- An **instance** places and retimes a drawing. Its pose tracks can hold one face's
|
||||
mouth while the other face continues moving.
|
||||
|
||||
Subject metadata and drawing timelines remain separate facts. Hand-drawn timelines
|
||||
need no subject. Features retain explicit timeline references, so their locations
|
||||
are not inferred from their labels.
|
||||
|
||||
Both filmed faces share one source-to-stage transform. Fitting them independently
|
||||
would stack them at the center. Additional placement uses each instance's ordinary
|
||||
channels. Cross-face draw order is the instances' `:z` order.
|
||||
|
||||
## Consequences
|
||||
|
||||
Regeneration updates the addressed timeline directly. There is no temporary swap
|
||||
into `:main`, no special stage regeneration path, and no renaming of parents or
|
||||
stencils. Composing a stage moves the take's root into the library and preserves
|
||||
its child timelines. Settings appear once per tracked object, since all placements
|
||||
read that same drawing.
|
||||
|
||||
Presence masks inside a subject use local feature names, matching measurement.
|
||||
Freeze qualifies them when building block descriptors. Head and retained-source
|
||||
blocks explicitly name their subject; otherwise two faces with identical detection
|
||||
masks could produce different bytes under the same key. Analysis addresses also
|
||||
include detection capacity and assignment settings, so old single-face detection
|
||||
results cannot satisfy a new multi-face request.
|
||||
|
||||
Validation counts node ownership by `[timeline node]`. The server requires one
|
||||
complete set of retained source roles **per subject**, rather than exactly three
|
||||
blocks for the entire analysis.
|
||||
|
||||
Nested rectangles retain fractional sizes until rasterization. Rounding inside a
|
||||
face timeline discarded small head-local pupils before the source-to-stage scale
|
||||
was applied. This was a real rendering error missed by the earlier proposal's
|
||||
coordinate-only benchmark; the regression now compares all mark extents as well.
|
||||
|
||||
## Verification and limits
|
||||
|
||||
`frontend/test/arthur/flow/multi_face_test.cljs` exercises two distinct subjects,
|
||||
source block separation, detection and feature gaps, independent pose cuts and
|
||||
regeneration, nested stage save/load, and equivalence to a flat single-face scene.
|
||||
Existing geometry, raster, source, and regeneration tests cover the same paths.
|
||||
`clips/tests/test_api.py` checks complete source roles per subject and immutability.
|
||||
The browser suite checks rendering, pupils, playback, save/open, drawing and upload.
|
||||
|
||||
Assignment remains a nearest-centroid heuristic, with a version and distance gate
|
||||
recorded in the analysis. Reordered detections, late arrivals and gaps are tested;
|
||||
identity through crossings or long disappearances is not guaranteed. Assignment
|
||||
happens before measurement, so correcting it requires measuring again.
|
||||
|
||||
This changes the freeze and retained-source contracts. It does not migrate older
|
||||
flat captures; reanalyze their footage to use the new regeneration path. Existing
|
||||
rendering and leaf codecs still understand their node/channel representation.
|
||||
|
||||
## Next steps
|
||||
|
||||
1. Commit the verified checkpoint: 319 frontend tests, 44 API tests, browser
|
||||
checks, and app/test builds passed. Builds reported no warnings.
|
||||
2. Exercise real two-person footage, especially crossings, late arrivals and
|
||||
disappearances. Check assignment before treating the resulting geometry as
|
||||
evidence about the representation.
|
||||
3. Build performance-pose selection and instance-scoped picture rates, reusing
|
||||
the existing held-frame lookup and pose groups.
|
||||
4. Then build plate-drawing selection and independent tracing references.
|
||||
|
||||
The [timing handoff](timing-handoff.md) owns the detailed next implementation
|
||||
sequence. Older flat captures need reanalysis unless a migration is separately
|
||||
undertaken to preserve their authored edits.
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Self-contained. You should not need any prior conversation to execute this.
|
||||
|
||||
**Implementation status (2026-09-28):** steps 0–9 are in. Step 6 reads extracted
|
||||
**Implementation status (2026-09-29):** steps 0–9 are in. Step 6 reads extracted
|
||||
footage, detects landmarks with local MediaPipe assets at full source cadence, and
|
||||
runs the same freeze path as the synthetic take. The scene time map can sample the
|
||||
frozen roto at a lower picture fps without changing source analysis, duration or
|
||||
|
|
@ -13,9 +13,24 @@ absence intervals through measurement and freeze. Step 9 adds the Django backend
|
|||
the three-tier split, content-addressed tier 2 with the detector version inside
|
||||
every key, leaf addressing for tier 1, and project load/save that round-trips.
|
||||
|
||||
**Still open.** Step 8's parameter UI and scoped regeneration, and automatic
|
||||
per-feature detection. Everything under "Out, and do not build it" below, which
|
||||
step 9 did not touch.
|
||||
Step 8 now has parameter controls and scoped regeneration from retained source.
|
||||
Multi-face representation is complete: each tracked subject has a drawing
|
||||
timeline, placed by an ordinary symbol instance. See
|
||||
[multi-face representation](multi-face-representation.md) for the implemented
|
||||
model, verification and compatibility limits.
|
||||
|
||||
**Next, in order:** commit the verified checkpoint; exercise real two-person
|
||||
footage, including crossings and disappearances; build performance-pose
|
||||
Suggest/Keep/Drop and instance-scoped picture rates; then add plate-drawing
|
||||
selection and independent tracing references. The
|
||||
[timing handoff](timing-handoff.md) records current code and implementation order.
|
||||
Reopen the representation only for a concrete requirement it cannot express.
|
||||
|
||||
**Still open:** real-footage identity validation, automatic per-feature detection,
|
||||
the timing and tracing work above, and time-varying parameter settings. Older flat
|
||||
captures need reanalysis for the new regeneration path; no migration is included.
|
||||
The step descriptions below retain the original port scope; this status and the
|
||||
linked handoffs describe subsequent work.
|
||||
|
||||
## What arthur is
|
||||
|
||||
|
|
@ -355,7 +370,7 @@ stencilled by the sclera.
|
|||
minus paint. The fixed pixel thresholds remain provisional; step 8 exposes their
|
||||
parameters for tuning without changing the source track or picture timing.
|
||||
|
||||
### 8 — knobs
|
||||
### 8 — knobs — DONE for static settings and scoped regeneration
|
||||
Build the parameter model before its UI. Define each parameter once with its
|
||||
default, validation, applicable area and regeneration dependencies. Store values
|
||||
by stable subject and feature ID. Represent an eye pair as one group with one or
|
||||
|
|
@ -376,8 +391,9 @@ visible eye. This is an input format, not a control UI or an automatic detector.
|
|||
|
||||
Use leaf-addressable settings under the clip, subject, feature and optional
|
||||
group. Retain source measurements so a setting change can regenerate affected
|
||||
channels without re-detecting footage. Time-varying parameter values and all
|
||||
parameter controls are deferred to the UI pass.
|
||||
channels without re-detecting footage. Static parameter controls and scoped
|
||||
regeneration are implemented for takes and composed stages. Time-varying
|
||||
parameter values remain deferred.
|
||||
|
||||
### 9 — backend — DONE
|
||||
Django project, the `clips` app, models for
|
||||
|
|
@ -431,7 +447,7 @@ nobody should pre-empt by porting the old one.
|
|||
|
||||
## Two things to not foreclose
|
||||
|
||||
Feature controls will later handle more than one face and editing presence.
|
||||
Feature controls now handle more than one face; editing presence remains future work.
|
||||
The underlying identity, occlusion and group association model begins in step 8:
|
||||
|
||||
- **Presence is not visibility.** An occluded feature has *no value* on a frame,
|
||||
|
|
@ -440,5 +456,6 @@ The underlying identity, occlusion and group association model begins in step 8:
|
|||
- **Params carry stable identity.** A subject and its features keep their IDs
|
||||
across observation gaps. A run of visible frames is not a new identity.
|
||||
|
||||
The identity tracker, when it comes, should use the same pattern the iris and brow
|
||||
correspondences already use: vote across every frame rather than trusting one.
|
||||
The current identity tracker uses nearest-centroid assignment. Validate it on
|
||||
real crossings and disappearances before choosing a more elaborate policy; the
|
||||
iris and brow correspondence code offers whole-take voting as one option.
|
||||
|
|
|
|||
128
docs/timing-handoff.md
Normal file
128
docs/timing-handoff.md
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
# Timing and frame-selection handoff
|
||||
|
||||
Status (2026-09-29): the multi-face representation is complete. Each face has a
|
||||
local drawing timeline and an ordinary symbol instance. Keep that model; the next
|
||||
feature is performance-pose selection, followed by plate drawings and tracing.
|
||||
See [multi-face representation](multi-face-representation.md) for verification
|
||||
and compatibility limits.
|
||||
|
||||
## Next steps, in order
|
||||
|
||||
1. **Commit the verified checkpoint.** Representation, scoped regeneration,
|
||||
nested stage composition and source persistence are implemented and tested.
|
||||
2. **Exercise real two-person footage.** Include crossings, late arrivals and
|
||||
disappearances. Assignment is still a nearest-centroid heuristic; inspect
|
||||
whether identities, landmarks and mouth crops stay together. Correcting an
|
||||
assignment requires measuring again. Do not redesign the representation to
|
||||
compensate for an assignment failure.
|
||||
3. **Build performance-pose selection.** Propose frames from a target picture
|
||||
rate, allow explicit Keep/Drop edits, and apply requests per instance. Reuse
|
||||
the existing held-frame lookup and generated pose groups. Keep authored keys
|
||||
and audio timing intact.
|
||||
4. **Then build plate drawings and tracing.** Suggest drawing frames from head
|
||||
displacement, allow manual choices, and give each cel an independently
|
||||
selectable tracing reference.
|
||||
|
||||
Older flat captures need reanalysis for the new regeneration path. Migrating their
|
||||
existing authored edits is separate work; it is not implemented by this checkpoint.
|
||||
|
||||
## Timing decisions
|
||||
|
||||
Keep the dense analyzed frames. Generated motion holds the most recent selected
|
||||
source pose; removing a selected pose never deletes source data or shortens the
|
||||
clip. Store edits in the animation's local frame space, so moving an instance
|
||||
does not move its edits. Authored keys follow intentional instance retiming but
|
||||
must not be quantized by a picture-rate request. Clip FPS and audio duration stay
|
||||
fixed.
|
||||
|
||||
There are two selections with different owners, sharing held-frame lookup:
|
||||
|
||||
- **Performance poses:** propose a kept-frame list from the target picture rate,
|
||||
then apply explicit keep/drop edits. A parent instance may request a lower
|
||||
rate. Mouth outline, interior, teeth and visibility read the same selected
|
||||
source frame; likewise each eye's coupled parts. Use group overrides when
|
||||
needed, rather than a setting on every channel. Head motion currently has its
|
||||
own anchor selection; do not silently put it under mouth timing.
|
||||
- **Plate drawings:** start with frame 0, walk measured rigid head poses, and
|
||||
suggest a frame when maximum landmark displacement from the last kept pose
|
||||
exceeds tolerance. Let the artist add/remove frames. A removed drawing stays
|
||||
stored so it can reappear if restored. This selection does not thin the mouth.
|
||||
|
||||
A target rate is approximate. Pin a useful closed-mouth pose at its actual frame,
|
||||
even if that produces more changes than the target. Do not show a future pose
|
||||
early to fit a grid. Manual drop wins over an automatic suggestion; make removal
|
||||
of the only closed pose in a beat visible in the UI. Skip missing detections when
|
||||
suggesting a replacement. Keep a frame-zero selection and hold the last selection
|
||||
through the end. A skipped pose (hold), `[:vis] false` (hidden), and an absent
|
||||
measurement remain different facts.
|
||||
|
||||
Store manual edits separately from generated proposals so changing the rate or
|
||||
tolerance retains hand decisions. Selection edits change the document, not dense
|
||||
blocks or analysis addresses. Verify save/open for every new field; extend leaf
|
||||
handling and the relevant key whitelist if its storage location requires it.
|
||||
|
||||
## Current code: reuse these mechanisms
|
||||
|
||||
- `domain/pose.cljs` already has `prepare`, `held-frame` and `source-frame`.
|
||||
Instance `:playback :tracks` map local change frames to held source frames,
|
||||
keyed by pose group. Reuse this lookup; frame suggestion and Keep/Drop policy
|
||||
are the missing layer. An explicit cut is not itself a complete selection UI.
|
||||
- `freeze/performance-nodes` marks generated animated channels with
|
||||
`:pose-sampled?` and local `:pose-group` names. This includes keyed visibility
|
||||
as well as dense geometry. `:generated` remains provenance for regeneration.
|
||||
- `timeline/channel-frame` already applies explicit pose choices and default
|
||||
picture sampling to marked channels. Playback and export both use
|
||||
`clip/resolver` with `:picture-fps`; there is no need for a second sampling
|
||||
implementation. Export's pose count is still a rate-based estimate.
|
||||
- The picture-rate option is currently passed through the resolver tree
|
||||
unchanged. Instance-specific parent requests are still to be implemented.
|
||||
Instance offset/rate must apply before selecting the local source pose.
|
||||
- `pose/put-cut` and `remove-cut` currently address instances in `:main`.
|
||||
A take's face instances are there, but a composed stage nests them inside a
|
||||
shared source timeline. Make the editing scope explicit when adding nested
|
||||
controls. A request on one outer placement must not rewrite the shared
|
||||
drawing's playback settings for every placement.
|
||||
- Generic root `:time :expose` still retimes descendants, and frozen takes still
|
||||
store it. Paint nodes are rootless to escape it. When the selection path
|
||||
replaces take picture cadence, remove that redundant quantization from the
|
||||
take default; preserve intentional generic time maps. Moving exposure to
|
||||
`:head` would still retime authored children.
|
||||
- `freeze/head-mode` supports `:free` and `:anchored`. It keeps measured channels
|
||||
dense and writes optional per-subject `:anchors` maps; it does **not** implement
|
||||
`:per-plate` mode or materialize transform keys from `:kept`. Plate selection
|
||||
should reuse held measured-frame addresses where appropriate, without
|
||||
rerunning analysis or copying the measurements.
|
||||
- `:over` hand corrections are currently refused by the channel reader. Their
|
||||
future application belongs after generated pose selection.
|
||||
|
||||
## Performance-pose implementation sequence
|
||||
|
||||
1. Add pure proposal and Keep/Drop policy around the existing held-frame lookup.
|
||||
Cover frame zero, nondivisible rates, manual precedence, missing poses and a
|
||||
protected mouth closure. Preserve all source frames.
|
||||
2. Feed instance requests and group selections into the existing channel read
|
||||
path. Cover two faces, two differently timed placements of one source, nested
|
||||
instances, coupled visibility/geometry, and authored keys at their normal time.
|
||||
Use this same path for preview and export; keep audio duration unchanged.
|
||||
3. Wire the performance strip's Suggest/Keep/Drop controls and persistence.
|
||||
Replace the export pose estimate with the actual selection count. Retire the
|
||||
take's redundant root exposure only when this path replaces its behavior.
|
||||
|
||||
## Plate drawings and tracing, afterward
|
||||
|
||||
The old suggestion algorithm is `js/pipeline.js:suggestPlateFrames`; the strip,
|
||||
worksheet and tracing photo are in `js/app.js`. Port the useful policy over the
|
||||
measured head poses and reuse held-frame lookup for the resulting drawing set.
|
||||
|
||||
Give a cel an editor-only source-frame reference, defaulting to its plate frame
|
||||
but independently changeable. It may point to a frame omitted from either rendered
|
||||
selection. Register the photo using that source frame's measured transform. The
|
||||
old prototype coupled photo and cel addresses; independent tracing is new work.
|
||||
|
||||
The old iris socket lock, gaze origin, CLJS head anchors and registration pivot
|
||||
are separate settings. Clarify what an "origin-lock" request means before adding
|
||||
that control.
|
||||
|
||||
Keep the UI to two scopes: **performance poses** and **plate drawings**, each with
|
||||
Suggest/Keep/Drop. Tracing reference and lock controls live with the cel or feature
|
||||
they affect. No general keyframe framework is needed for this work.
|
||||
|
|
@ -67,12 +67,7 @@
|
|||
:channels (cond-> {[:audio :gain] gain}
|
||||
pan (assoc [:audio :pan] pan))}])
|
||||
audio))]
|
||||
(cond-> (assoc source :name name :width width :height height
|
||||
:timelines {:main {:id :main :frames frames :nodes nodes}
|
||||
symbol (assoc original :id symbol)})
|
||||
;; These are the same tracked features in the shared symbol. Every
|
||||
;; placement reads them; tuning one changes all instances.
|
||||
(seq (:features source))
|
||||
(assoc :features (into {} (map (fn [[id f]]
|
||||
[id (assoc f :timeline symbol)]))
|
||||
(:features source))))))
|
||||
(assoc source :name name :width width :height height
|
||||
:timelines (assoc (:timelines source)
|
||||
:main {:id :main :frames frames :nodes nodes}
|
||||
symbol (assoc original :id symbol)))))
|
||||
|
|
|
|||
|
|
@ -84,8 +84,10 @@
|
|||
:fps fps
|
||||
:aspect aspect})}))
|
||||
|
||||
(def subject :face-1)
|
||||
|
||||
(def frozen
|
||||
(delay (freeze/clip params @measured)))
|
||||
(delay (freeze/clip params {subject @measured})))
|
||||
|
||||
(def store (delay (:store @frozen)))
|
||||
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@
|
|||
:disc (let [[x y] (at (:cx op) (:cy op))]
|
||||
(assoc op :cx x :cy y :r (* scale (:r op))))
|
||||
:rect (let [[x y] (at (:cx op) (:cy op))]
|
||||
(assoc op :cx x :cy y :size (js/Math.round (* scale (:size op)))))
|
||||
(assoc op :cx x :cy y :size (* scale (:size op))))
|
||||
op)))
|
||||
|
||||
(defn resolver
|
||||
|
|
@ -158,13 +158,7 @@
|
|||
(build root [] nil))))
|
||||
|
||||
(defn problems
|
||||
"Human-readable reasons this clip will not evaluate or will not save. Empty
|
||||
means it will.
|
||||
|
||||
The tracking identities are checked HERE and not in `domain/timeline`, because a
|
||||
feature names nodes and only the root timeline's nodes were tracked into: a
|
||||
library symbol is drawn, not detected. So `feature/problems` is asked about the
|
||||
root, once, rather than about every timeline."
|
||||
"Human-readable reasons this clip will not evaluate or save."
|
||||
[clip]
|
||||
(vec
|
||||
(concat
|
||||
|
|
@ -207,5 +201,4 @@
|
|||
(not (contains? (:nodes tl) (:linked-to n))))]
|
||||
(str "timeline " (pr-str tid) " audio " (pr-str id)
|
||||
" links to missing node " (pr-str (:linked-to n))))
|
||||
(when (map? (root clip))
|
||||
(feature/problems clip (nodes clip))))))
|
||||
(feature/problems clip))))
|
||||
|
|
|
|||
|
|
@ -1,14 +1,13 @@
|
|||
(ns arthur.domain.feature
|
||||
"Stable tracked identities and explicit eye-pair settings associations.
|
||||
|
||||
These maps are the CLIP's — `:subjects`, `:features`, `:groups` — and not a
|
||||
timeline's, because they describe what a camera saw and a library symbol is
|
||||
drawn rather than detected. Rendering never reads them; it reads nodes and
|
||||
channels. `problems` therefore takes the clip AND the node map to check
|
||||
references against, rather than reaching for `(:nodes clip)`: a clip holds
|
||||
several timelines and only the root one was tracked into."
|
||||
"Tracked subjects, feature ownership, and eye-pair settings.
|
||||
Features name their timeline explicitly; node ids are local to that timeline."
|
||||
(:require [arthur.domain.params :as params]))
|
||||
|
||||
(defn owned
|
||||
"A deterministic clip-level feature or group id. Nodes keep local names."
|
||||
[subject role]
|
||||
(keyword (subs (str subject) 1) (name role)))
|
||||
|
||||
(defn group-for [clip feature-id]
|
||||
(first (filter (fn [[_ group]] (some #{feature-id} (:members group)))
|
||||
(:groups clip))))
|
||||
|
|
@ -45,17 +44,14 @@
|
|||
clip))
|
||||
|
||||
(defn problems
|
||||
"Check identity references and pair membership before storing a clip.
|
||||
|
||||
`nodes` is the node map a feature's `:nodes` are resolved against — the root
|
||||
timeline's, passed in rather than looked up, so this namespace does not have to
|
||||
know which timeline a clip plays."
|
||||
[clip nodes]
|
||||
"Check tracked identities and timeline-local node ownership."
|
||||
[clip]
|
||||
(let [subjects (:subjects clip)
|
||||
features (:features clip)
|
||||
groups (:groups clip)
|
||||
memberships (mapcat (comp :members val) groups)
|
||||
node-owners (mapcat (comp :nodes val) features)]
|
||||
node-owners (for [[_ f] features n (:nodes f)]
|
||||
[(:timeline f) n])]
|
||||
(vec
|
||||
(concat
|
||||
(for [[id s] subjects :when (not= id (:id s))]
|
||||
|
|
@ -63,6 +59,9 @@
|
|||
(for [[id s] subjects
|
||||
:when (not (params/valid-settings? :subject (or (:params s) {})))]
|
||||
(str "subject " (pr-str id) " has invalid settings"))
|
||||
(for [[id _] subjects
|
||||
:when (not (seq (get-in clip [:timelines id :nodes :head :measured])))]
|
||||
(str "subject " (pr-str id) " has no measured head in its timeline"))
|
||||
(for [[id f] features :when (not= id (:id f))]
|
||||
(str "feature " (pr-str id) " has a different :id"))
|
||||
(for [[id f] features :when (not (contains? subjects (:subject f)))]
|
||||
|
|
@ -72,10 +71,11 @@
|
|||
(for [[id f] features
|
||||
:when (not (params/valid-settings? (:area f) (or (:params f) {})))]
|
||||
(str "feature " (pr-str id) " has invalid settings for " (pr-str (:area f))))
|
||||
(for [[id f] features
|
||||
:when (not (contains? (:timelines clip) (:timeline f)))]
|
||||
(str "feature " (pr-str id) " names a missing timeline"))
|
||||
(for [[id f] features node-id (:nodes f)
|
||||
:let [owned-nodes (if-let [timeline (:timeline f)]
|
||||
(get-in clip [:timelines timeline :nodes])
|
||||
nodes)]
|
||||
:let [owned-nodes (get-in clip [:timelines (:timeline f) :nodes])]
|
||||
:when (not (contains? owned-nodes node-id))]
|
||||
(str "feature " (pr-str id) " refers to missing node " (pr-str node-id)))
|
||||
(for [[id n] (frequencies node-owners) :when (> n 1)]
|
||||
|
|
|
|||
|
|
@ -134,10 +134,11 @@
|
|||
breathing."
|
||||
([r cx cy size index] (fill-rect! r cx cy size index nil))
|
||||
([{:keys [w h buf] :as r} cx cy size index over]
|
||||
(when (>= size 1)
|
||||
(let [x0 (js/Math.round (- cx (/ size 2)))
|
||||
y0 (js/Math.round (- cy (/ size 2)))]
|
||||
(let [ya (max 0 y0) yb (min h (+ y0 size))
|
||||
(let [size (js/Math.round size)]
|
||||
(when (>= size 1)
|
||||
(let [x0 (js/Math.round (- cx (/ size 2)))
|
||||
y0 (js/Math.round (- cy (/ size 2)))
|
||||
ya (max 0 y0) yb (min h (+ y0 size))
|
||||
xa (max 0 x0) xb (min w (+ x0 size))]
|
||||
(dotimes [iy (- yb ya)]
|
||||
(dotimes [ix (- xb xa)]
|
||||
|
|
|
|||
|
|
@ -271,14 +271,8 @@
|
|||
:rd rd})))))))
|
||||
|
||||
(defn- emit
|
||||
"The draw op for a placed node, or nil when it has nothing to draw. A group
|
||||
never draws; it exists to carry a transform.
|
||||
|
||||
Three branches that rhyme, deliberately left as three: a poly writes vertices
|
||||
into a buffer it was handed, a disc carries a scaled radius, a rect a rounded
|
||||
pixel count. They are three different marks, and the shared skeleton is two
|
||||
cheap lines each — folding them into one shape driven by a table would buy
|
||||
those lines back by coupling this to whatever the table was for."
|
||||
"Emit geometry in the timeline's space. Rect sizes stay fractional until
|
||||
rasterization, so enclosing symbol transforms can still scale them."
|
||||
[{:keys [palette buf-for]} n {:keys [m rd]} base]
|
||||
(let [colour #(colour-index palette (rd [:style :color]))]
|
||||
(case (:kind n)
|
||||
|
|
@ -310,10 +304,7 @@
|
|||
(when-not (ch/nothing? size)
|
||||
(assoc base :kind :rect
|
||||
:cx (aget m 4) :cy (aget m 5)
|
||||
;; ROUNDED, because :size is a pixel count: a scaled square
|
||||
;; 3.4px wide would be 3px on one frame and 4 on the next,
|
||||
;; which reads as the pupil breathing. See raster/fill-rect!.
|
||||
:size (js/Math.round (* size (node/mean-scale m)))
|
||||
:size (* size (node/mean-scale m))
|
||||
:color (colour))))
|
||||
|
||||
(throw (ex-info "node kind is not implemented"
|
||||
|
|
@ -337,12 +328,11 @@
|
|||
nodes))
|
||||
|
||||
(defn- channel-frame
|
||||
"Only marked generated channels read picture cadence and instance pose cuts."
|
||||
[choices head-anchors nodes source-fps picture-fps id path c lf]
|
||||
"Anchors select measured frames; marked channels read instance pose choices."
|
||||
[choices anchors nodes source-fps picture-fps id c lf]
|
||||
(cond
|
||||
(and (= id :head) (contains? #{[:xform :pos] [:xform :rot] [:xform :scale]} path)
|
||||
(contains? head-anchors id))
|
||||
(pose/held-frame (get head-anchors id) lf lf)
|
||||
(contains? anchors id)
|
||||
(pose/held-frame (get anchors id) lf lf)
|
||||
|
||||
(:pose-sampled? c)
|
||||
(pose/source-frame choices
|
||||
|
|
@ -354,7 +344,7 @@
|
|||
|
||||
:else lf))
|
||||
|
||||
(defn- prepared-head-anchors [nodes]
|
||||
(defn- prepared-anchors [nodes]
|
||||
(into {}
|
||||
(for [[id n] nodes :when (seq (:anchors n))]
|
||||
[id (vec (sort-by first (:anchors n)))])))
|
||||
|
|
@ -409,12 +399,12 @@
|
|||
([tl f store palette pose-tracks opts]
|
||||
(let [nodes (nodes-of tl)
|
||||
choices (pose/prepare pose-tracks)
|
||||
head-anchors (prepared-head-anchors nodes)
|
||||
anchors (prepared-anchors nodes)
|
||||
{:keys [source-fps picture-fps]} opts
|
||||
ord (order nodes)]
|
||||
(eval-into {:read (fn [id path c lf]
|
||||
(ch/value-at c (channel-frame choices head-anchors nodes
|
||||
source-fps picture-fps id path c lf)
|
||||
(ch/value-at c (channel-frame choices anchors nodes
|
||||
source-fps picture-fps id c lf)
|
||||
store))
|
||||
:palette palette
|
||||
:mat-for (fn [_id] (node/mat))
|
||||
|
|
@ -473,7 +463,7 @@
|
|||
([tl store palette pose-tracks {:keys [source-fps picture-fps]}]
|
||||
(let [nodes (nodes-of tl)
|
||||
choices (pose/prepare pose-tracks)
|
||||
head-anchors (prepared-head-anchors nodes)
|
||||
anchors (prepared-anchors nodes)
|
||||
ord (order nodes)
|
||||
rank (draw-rank nodes ord)
|
||||
cursors (into {}
|
||||
|
|
@ -497,8 +487,8 @@
|
|||
placed (volatile! {})
|
||||
ctx {:read (fn [id path c lf]
|
||||
(ch/sample! (get-in cursors [id path])
|
||||
(channel-frame choices head-anchors nodes
|
||||
source-fps picture-fps id path c lf)))
|
||||
(channel-frame choices anchors nodes
|
||||
source-fps picture-fps id c lf)))
|
||||
:palette palette
|
||||
:mat-for (fn [id] (get mats id))
|
||||
:on-place (fn [id p] (vswap! placed assoc id p))
|
||||
|
|
@ -559,18 +549,19 @@
|
|||
(into (for [[id n] nodes
|
||||
p (node/problems n)]
|
||||
(str "node " (pr-str id) ": " p)))
|
||||
;; Anchors re-address the node's measurement, regardless of its name.
|
||||
(into (for [[id n] nodes
|
||||
:let [anchors (:anchors n)]
|
||||
:when (some? anchors)
|
||||
:when (not (and (= id :head)
|
||||
(map? anchors) (contains? anchors 0)
|
||||
:when (not (and (map? anchors) (contains? anchors 0)
|
||||
(integer? (:frames tl))
|
||||
(every? #(and (integer? %) (<= 0 %)
|
||||
(< % (:frames tl)))
|
||||
(concat (keys anchors) (vals anchors)))
|
||||
(seq (:measured n))
|
||||
(= (:channels n) (:measured n))))]
|
||||
(str "node " (pr-str id)
|
||||
": :anchors must start at frame 0, name valid measured frames, and read the measured head channels")))
|
||||
": :anchors must start at frame 0, name valid measured frames, and read that node's own measured channels")))
|
||||
(into (for [k (remove timeline-keys (keys tl))]
|
||||
(str "timeline has a field with no leaf to save it in: " (pr-str k))))
|
||||
(into (when-not (or (nil? (:frames tl)) (and (integer? (:frames tl)) (pos? (:frames tl))))
|
||||
|
|
|
|||
|
|
@ -64,21 +64,32 @@
|
|||
stream fps w h
|
||||
(fn [i frame]
|
||||
(.drawImage ctx frame 0 0)
|
||||
(let [face (detect/detect! model canvas (ingest/frame-ms fps i))
|
||||
ring (when face (mapv #(nth face %) lm/LIPS-INNER))
|
||||
box (when ring (interior/crop ring [w h]))
|
||||
pixels (when box
|
||||
(.-data (.getImageData ctx (:x box) (:y box)
|
||||
(:w box) (:h box))))]
|
||||
(swap! raw conj face)
|
||||
(let [crop (when box {:box box :data pixels})]
|
||||
(swap! crops conj crop)
|
||||
;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work
|
||||
;; either way, but done after the fact it is ten seconds of
|
||||
;; synchronous arithmetic with the main thread held and the
|
||||
;; frame counter frozen on its last value — which reads as the
|
||||
;; decoder hanging, and was diagnosed as that twice.
|
||||
(swap! inner conj (source/measure-crop take/knobs crop))))
|
||||
;; EVERY FACE ON THIS FRAME, each with its own mouth crop taken
|
||||
;; while the frame's pixels are still on the canvas. Which of these
|
||||
;; detections belongs to which subject is not decided here — the
|
||||
;; answer needs the whole take — so all three vectors stay in
|
||||
;; DETECTION ORDER and `detect/tracks` re-keys them afterwards.
|
||||
(let [faces (detect/detect! model canvas (ingest/frame-ms fps i))
|
||||
boxes (mapv (fn [face]
|
||||
(interior/crop (mapv #(nth face %) lm/LIPS-INNER)
|
||||
[w h]))
|
||||
faces)
|
||||
frame-crops (mapv (fn [box]
|
||||
(when box
|
||||
{:box box
|
||||
:data (.-data (.getImageData
|
||||
ctx (:x box) (:y box)
|
||||
(:w box) (:h box)))}))
|
||||
boxes)]
|
||||
(swap! raw conj faces)
|
||||
(swap! crops conj frame-crops)
|
||||
;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work
|
||||
;; either way, but done after the fact it is ten seconds of
|
||||
;; synchronous arithmetic with the main thread held and the
|
||||
;; frame counter frozen on its last value — which reads as the
|
||||
;; decoder hanging, and was diagnosed as that twice.
|
||||
(swap! inner conj (mapv #(source/measure-crop take/knobs %)
|
||||
frame-crops)))
|
||||
(when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total))
|
||||
(rf/dispatch [::progress (str "detecting " (inc i) "/" total)]))
|
||||
;; Yield, so the status and the transport paint between synchronous
|
||||
|
|
@ -86,30 +97,56 @@
|
|||
(js/Promise. (fn [done] (js/setTimeout done 0)))))))
|
||||
(.then (fn [_]
|
||||
(mark! (str "DECODE FINISHED — " (count @raw) " frames"))
|
||||
(let [gaps (detect/fill-gaps @raw)]
|
||||
(let [slots (detect/tracks @raw)
|
||||
subjects (into {}
|
||||
(map (fn [[id track]]
|
||||
[id (assoc (detect/fill-gaps
|
||||
(detect/pick @raw track))
|
||||
:crops (detect/pick @crops track)
|
||||
:interior (detect/pick @inner track)
|
||||
:interior-settings take/knobs)]))
|
||||
slots)]
|
||||
(mark! "fill-gaps")
|
||||
(assoc gaps :dimensions [w h] :crops @crops
|
||||
:interior @inner :interior-settings take/knobs)))))))
|
||||
{:subjects subjects
|
||||
:dimensions [w h]
|
||||
;; Frames where NOBODY was found, which is a fact about the
|
||||
;; footage. A frame one of two faces is missing from is a gap
|
||||
;; in that subject's own detection mask and is reported there.
|
||||
:missing (count (remove seq @raw))
|
||||
:first-real (first (keep-indexed (fn [i faces] (when (seq faces) i))
|
||||
@raw))}))))))
|
||||
|
||||
(defn presence-for
|
||||
"Select a subject's masks and express them in its local feature names.
|
||||
Unqualified manifest ids refer to the first face, as in single-face footage."
|
||||
[manifest subject]
|
||||
(not-empty
|
||||
(into {} (keep (fn [[id mask]]
|
||||
(when (= (or (namespace id) "face-1") (subs (str subject) 1))
|
||||
[(keyword (name id)) mask])))
|
||||
(:presence manifest))))
|
||||
|
||||
(defn- build-clip [manifest detector
|
||||
{:keys [dense detected dimensions interior missing first-real]
|
||||
:as source-inputs}]
|
||||
{:keys [subjects dimensions missing first-real] :as source-inputs}]
|
||||
(let [[w h] dimensions
|
||||
_ (mark! "build-clip: start")
|
||||
frozen (take/footage manifest {:dense dense :detected detected
|
||||
:dimensions dimensions :interior interior
|
||||
:presence (:presence manifest)
|
||||
:detector detector})
|
||||
with-presence (into {}
|
||||
(map (fn [[id inputs]]
|
||||
[id (assoc inputs :presence (presence-for manifest id))]))
|
||||
subjects)
|
||||
frozen (take/footage manifest {:dimensions dimensions
|
||||
:subjects with-presence
|
||||
:detector detector})
|
||||
_ (mark! "build-clip: freeze")
|
||||
built (:clip frozen)
|
||||
source-blocks (source/pack (:id (:analysis built)) source-inputs)
|
||||
source-blocks (source/pack-subjects (:id (:analysis built)) subjects)
|
||||
_ (mark! "build-clip: pack source blocks")]
|
||||
(assoc (select-keys built [:fps :width :height])
|
||||
:frames (clip/frames built)
|
||||
:display-fps (:fps built)
|
||||
:clip built :store (:store frozen)
|
||||
:source-blocks source-blocks
|
||||
:source-inputs (assoc source-inputs :presence (:presence manifest))
|
||||
:source-inputs (assoc source-inputs :subjects with-presence)
|
||||
;; No cache-buster. The audio is a blob named by the hash of its own
|
||||
;; bytes, so re-extracting gives it a different URL rather than
|
||||
;; overwriting this one — which is what the `?v=` here used to work
|
||||
|
|
@ -120,14 +157,18 @@
|
|||
:cid (or (:id manifest) "footage")
|
||||
:summary (str (:frames manifest) " frames · " w "×" h " · "
|
||||
(:fps manifest) " fps"
|
||||
(when (> (count subjects) 1)
|
||||
(str " · " (count subjects) " faces"))
|
||||
(when (pos? missing)
|
||||
(str " · " missing " without a face"
|
||||
(when (pos? first-real)
|
||||
(str " (first found on " (inc first-real) ")"))))))))
|
||||
|
||||
(defn- with-default-interior! [analysis track]
|
||||
(defn- with-default-interior!
|
||||
"Backfill one subject's retained interior measurements at the default knobs."
|
||||
[analysis subject track]
|
||||
(let [frames (count (:crops track))
|
||||
key (source/interior-key analysis take/knobs frames)]
|
||||
key (source/interior-key analysis subject take/knobs frames)]
|
||||
(-> (http/GET (str "/api/blocks/" key))
|
||||
(.then (fn [block]
|
||||
(assoc track :interior (source/unpack-interior block take/knobs frames)
|
||||
|
|
@ -137,16 +178,29 @@
|
|||
(if (= 404 (:status (ex-data error))) track (throw error)))))))
|
||||
|
||||
(defn saved-source!
|
||||
"Restore retained source tracks by analysis id, for regeneration after open."
|
||||
"Restore retained source tracks by analysis id, for regeneration after open.
|
||||
|
||||
An analysis holds one set of blocks per tracked subject, so the count is a
|
||||
multiple of `source/roles` rather than equal to it; `source/unpack` reads each
|
||||
block's own descriptor to find out whose it is."
|
||||
[key dimensions]
|
||||
(-> (http/GET (str "/api/analyses/" key))
|
||||
(.then (fn [^js analysis]
|
||||
(let [keys (array-seq (.-source_blocks analysis))]
|
||||
(when (= (count keys) (count source/roles))
|
||||
(when (and (seq keys) (zero? (mod (count keys) (count source/roles))))
|
||||
(-> (js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %)) keys)))
|
||||
(.then (fn [blocks] (source/unpack blocks dimensions)))
|
||||
(.then (fn [track] (with-default-interior! key track))))))))
|
||||
(.then (fn [{:keys [subjects]}]
|
||||
(-> (js/Promise.all
|
||||
(into-array
|
||||
(map (fn [[id track]]
|
||||
(.then (with-default-interior! key id track)
|
||||
(fn [filled] [id filled])))
|
||||
subjects)))
|
||||
(.then (fn [pairs]
|
||||
{:subjects (into {} (array-seq pairs))
|
||||
:dimensions dimensions}))))))))))
|
||||
(.catch (fn [error]
|
||||
(if (= 404 (:status (ex-data error))) nil (throw error))))))
|
||||
|
||||
|
|
@ -154,19 +208,8 @@
|
|||
(saved-source! (:id (take/analysis-for manifest detector))
|
||||
[(:width manifest) (:height manifest)]))
|
||||
|
||||
(defn measure-crops!
|
||||
"Measure every retained crop's interior, ONE PER EVENT-LOOP TURN.
|
||||
|
||||
Done in a tight loop instead, it is ten seconds of synchronous arithmetic with
|
||||
the main thread held and the frame counter frozen on its last value — which
|
||||
reads as the decoder hanging, and was diagnosed as that twice.
|
||||
|
||||
Two callers, and the only difference between them is who is waiting: opening an
|
||||
older analysis that predates the interior block backfills it behind a progress
|
||||
line, and a knob drag that needs the pixels measured at new settings does the
|
||||
same work with nobody watching, so it passes no `on-step`. The settings ride
|
||||
back on the track, because a measurement and the knobs it was taken at are one
|
||||
fact — `source/pack` will not address an interior block without them."
|
||||
(defn measure-one!
|
||||
"One subject's crops, ONE PER EVENT-LOOP TURN."
|
||||
[settings track on-step]
|
||||
(if (or (:interior track) (not (:crops track)))
|
||||
(js/Promise.resolve track)
|
||||
|
|
@ -186,6 +229,29 @@
|
|||
(catch :default error (reject error)))))]
|
||||
(step 0)))))))
|
||||
|
||||
(defn measure-crops!
|
||||
"Measure every retained crop's interior, one subject after another and one
|
||||
crop per event-loop turn.
|
||||
|
||||
Done in a tight loop instead, it is ten seconds of synchronous arithmetic with
|
||||
the main thread held and the frame counter frozen on its last value — which
|
||||
reads as the decoder hanging, and was diagnosed as that twice.
|
||||
|
||||
Two callers, and the only difference between them is who is waiting: opening an
|
||||
older analysis that predates the interior block backfills it behind a progress
|
||||
line, and a knob drag that needs the pixels measured at new settings does the
|
||||
same work with nobody watching, so it passes no `on-step`. The settings ride
|
||||
back on each track, because a measurement and the knobs it was taken at are one
|
||||
fact — `source/pack` will not address an interior block without them."
|
||||
[settings track on-step]
|
||||
(reduce (fn [chain [id one]]
|
||||
(.then chain
|
||||
(fn [acc]
|
||||
(.then (measure-one! settings one on-step)
|
||||
(fn [measured] (assoc-in acc [:subjects id] measured))))))
|
||||
(js/Promise.resolve track)
|
||||
(:subjects track)))
|
||||
|
||||
(rf/reg-fx
|
||||
::begin!
|
||||
(fn [footage-id]
|
||||
|
|
|
|||
|
|
@ -135,9 +135,12 @@
|
|||
(.then (fn [_]
|
||||
(http/PUT
|
||||
(str "/api/analyses/" (:id analysis))
|
||||
;; One set per tracked subject, in
|
||||
;; the order `source/unpack` does
|
||||
;; not depend on.
|
||||
#js {:source_blocks
|
||||
(into-array (map #(get-in source-blocks [% :key])
|
||||
source/roles))})))))))
|
||||
(into-array
|
||||
(source/block-keys source-blocks))})))))))
|
||||
(.then (fn [_] (upload-missing! doc)))
|
||||
(.then (fn [uploaded]
|
||||
(-> (http/PUT (str "/api/projects/" pid)
|
||||
|
|
@ -210,9 +213,9 @@
|
|||
(defonce ^:private retained-source (atom nil))
|
||||
(defonce ^:private retained-interior (atom nil))
|
||||
|
||||
(defn- retained-interior! [analysis settings inputs]
|
||||
(defn- retained-interior! [analysis subject settings inputs]
|
||||
(let [frames (count (:crops inputs))
|
||||
block-key (source/interior-key analysis settings frames)]
|
||||
block-key (source/interior-key analysis subject settings frames)]
|
||||
(-> (http/GET (str "/api/blocks/" block-key))
|
||||
(.then (fn [block]
|
||||
(assoc inputs
|
||||
|
|
@ -220,10 +223,10 @@
|
|||
:interior-key block-key)))
|
||||
(.catch (fn [error]
|
||||
(if (= 404 (:status (ex-data error)))
|
||||
(-> (footage/measure-crops! settings (dissoc inputs :interior) nil)
|
||||
(-> (footage/measure-one! settings (dissoc inputs :interior) nil)
|
||||
(.then (fn [measured]
|
||||
(let [{:keys [key descriptor data]}
|
||||
(source/interior-block analysis settings
|
||||
(source/interior-block analysis subject settings
|
||||
(:interior measured))]
|
||||
(-> (upload-missing!
|
||||
#js {:blocks #js [#js {:key key
|
||||
|
|
@ -241,8 +244,13 @@
|
|||
(:promise @retained-source)
|
||||
(let [promise
|
||||
(if (= "synth" (:detector analysis))
|
||||
;; The synthetic take tracks one face and regenerating it reads
|
||||
;; that face's landmarks, so it arrives in the same shape real
|
||||
;; footage does rather than in a flat one only this branch uses.
|
||||
(js/Promise.resolve
|
||||
{:dense (synth/synth-dense (:frames analysis) {:seed (:seed analysis)})})
|
||||
{:subjects
|
||||
{:face-1 {:dense (synth/synth-dense (:frames analysis)
|
||||
{:seed (:seed analysis)})}}})
|
||||
(-> (ingest/manifest! (:footage-id entry))
|
||||
(.then (fn [manifest]
|
||||
(-> (footage/saved-source! (:id analysis)
|
||||
|
|
@ -251,29 +259,43 @@
|
|||
(.then (fn [inputs]
|
||||
(when-not inputs
|
||||
(throw (ex-info "saved analysis has no source blocks" {})))
|
||||
(assoc inputs :presence (:presence manifest)))))))))]
|
||||
(update inputs :subjects
|
||||
(fn [subjects]
|
||||
(into {}
|
||||
(map (fn [[id one]]
|
||||
[id (assoc one :presence
|
||||
(footage/presence-for
|
||||
manifest id))]))
|
||||
subjects))))))))))]
|
||||
(do
|
||||
(reset! retained-source {:id (:id analysis) :promise promise})
|
||||
promise))))))
|
||||
|
||||
(defn- inputs-for-edit! [entry edit inputs]
|
||||
(let [{clip :changed fids :features} (regenerate/plan (:clip entry) edit)
|
||||
teeth (first (filter #(= :teeth (get-in clip [:features % :area])) fids))]
|
||||
(if (and teeth (:crops inputs))
|
||||
(defn- inputs-for-edit!
|
||||
"Bring the EDITED SUBJECT's retained pixel measurements up to the settings this
|
||||
edit needs. Only that subject's: a knob dragged on the second face does not
|
||||
re-measure the first face's mouth."
|
||||
[entry edit inputs]
|
||||
(let [{clip :changed fids :features subject :subject} (regenerate/plan (:clip entry) edit)
|
||||
teeth (first (filter #(= :teeth (get-in clip [:features % :area])) fids))
|
||||
one (get-in inputs [:subjects subject])]
|
||||
(if (and teeth (:crops one))
|
||||
(let [settings (merge take/knobs (feature/effective-params clip teeth))
|
||||
analysis (get-in clip [:analysis :id])
|
||||
frames (count (:crops inputs))
|
||||
key (source/interior-key analysis settings frames)]
|
||||
(if (and (:interior inputs)
|
||||
(or (= (:interior-key inputs) key)
|
||||
(and (nil? (:interior-key inputs))
|
||||
(= key (source/interior-key analysis take/knobs frames)))))
|
||||
frames (count (:crops one))
|
||||
key (source/interior-key analysis subject settings frames)
|
||||
done (fn [measured] (assoc-in inputs [:subjects subject] measured))]
|
||||
(if (and (:interior one)
|
||||
(or (= (:interior-key one) key)
|
||||
(and (nil? (:interior-key one))
|
||||
(= key (source/interior-key analysis subject take/knobs frames)))))
|
||||
(js/Promise.resolve inputs)
|
||||
(if (= key (:key @retained-interior))
|
||||
(:promise @retained-interior)
|
||||
(let [promise (retained-interior! analysis settings inputs)]
|
||||
(reset! retained-interior {:key key :promise promise})
|
||||
promise))))
|
||||
(.then (if (= key (:key @retained-interior))
|
||||
(:promise @retained-interior)
|
||||
(let [promise (retained-interior! analysis subject settings one)]
|
||||
(reset! retained-interior {:key key :promise promise})
|
||||
promise))
|
||||
done)))
|
||||
(js/Promise.resolve inputs))))
|
||||
|
||||
(rf/reg-fx
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@
|
|||
width, which is a visible difference on a mouth. Optional, because the synthetic
|
||||
take has no running mode to declare and an absent field is how the other
|
||||
optional inputs already say \"not applicable\"."
|
||||
[{:keys [detector version source footage frames fps aspect seed mode]}]
|
||||
[{:keys [detector version source footage frames fps aspect seed mode tracking]}]
|
||||
(when-not (and (string? detector) (seq detector) (string? version) (seq version))
|
||||
(throw (ex-info "an analysis names its detector and the detector's VERSION: an upgrade that silently reuses old landmarks is the failure content addressing exists to prevent"
|
||||
{:detector detector :version version})))
|
||||
|
|
@ -81,7 +81,8 @@
|
|||
source (assoc :source source)
|
||||
footage (assoc :footage footage)
|
||||
seed (assoc :seed seed)
|
||||
mode (assoc :mode mode))))
|
||||
mode (assoc :mode mode)
|
||||
tracking (assoc :tracking tracking))))
|
||||
|
||||
(defn analysis
|
||||
"An analysis record with its `:id` filled in. The record is tier 1 — it says
|
||||
|
|
@ -94,12 +95,9 @@
|
|||
;; the observation masks
|
||||
|
||||
(defn- feature-name
|
||||
"A feature id as a descriptor string. `canon` refuses a keyword value on
|
||||
purpose — so that \"mouth\" and :mouth cannot address the same block — and this
|
||||
is the naming it insists happens at the call site. `nil` is the whole face,
|
||||
which is what a track with no feature follows."
|
||||
"An explicit subject or feature id as a descriptor string."
|
||||
[id]
|
||||
(if id (subs (str id) 1) "face"))
|
||||
(subs (str id) 1))
|
||||
|
||||
(defn observation
|
||||
"A digest of exactly the absence data one block reads.
|
||||
|
|
@ -111,10 +109,9 @@
|
|||
block reads `:eye-r` and `:eye-l`'s presence and no other feature's, so a gap in
|
||||
one brow does not rewrite the mouth's address for nothing.
|
||||
|
||||
`nil` features mean the track follows the whole face's detection rather than a
|
||||
feature's presence, which is what the head's three blocks do."
|
||||
Head tracks name their subject and follow its detection mask."
|
||||
[features {:keys [detected presence]}]
|
||||
(let [wanted (sort-by str (distinct (keep identity features)))
|
||||
(let [wanted (sort-by str (distinct features))
|
||||
masks (into {} (map (fn [id] [id (mapv boolean (get presence id))]))
|
||||
(filter #(contains? presence %) wanted))]
|
||||
(when (or detected (seq masks))
|
||||
|
|
|
|||
|
|
@ -4,6 +4,10 @@
|
|||
(defonce ^:private instance (atom nil))
|
||||
(defonce ^:private pending (atom nil))
|
||||
|
||||
(def settings
|
||||
"Detection and assignment inputs, also included in the analysis address."
|
||||
{:max-faces 4 :assignment "nearest-centroid-v1" :gate 0.2})
|
||||
|
||||
(defn discard!
|
||||
"Throw away the cached landmarker so the next run builds a fresh one.
|
||||
|
||||
|
|
@ -43,7 +47,7 @@
|
|||
#js {:modelAssetPath "/static/mediapipe/face_landmarker.task"
|
||||
:delegate "CPU"}
|
||||
:runningMode "VIDEO"
|
||||
:numFaces 1})))
|
||||
:numFaces (:max-faces settings)})))
|
||||
(.then (fn [model]
|
||||
(reset! instance model)
|
||||
model))
|
||||
|
|
@ -55,8 +59,8 @@
|
|||
(js/Promise.reject (js/Error. "local MediaPipe script did not load"))))))
|
||||
|
||||
(defn detect!
|
||||
"Detect one already-drawn canvas frame at its own time in the take. nil means no
|
||||
face was detected.
|
||||
"Detect one already-drawn canvas frame at its own time in the take. An empty
|
||||
vector means no face was detected.
|
||||
|
||||
`at-ms` IS THE FRAME'S REAL PRESENTATION TIME, and all three words are
|
||||
load-bearing. In VIDEO mode the graph is a tracker: it runs face DETECTION only
|
||||
|
|
@ -75,14 +79,95 @@
|
|||
against 0.013), because a tracker told that every frame is 1ms apart expects a
|
||||
face that has barely moved."
|
||||
[model canvas at-ms]
|
||||
(when-let [face (aget (aget (.call (aget model "detectForVideo") model canvas at-ms)
|
||||
"faceLandmarks") 0)]
|
||||
(mapv (fn [p] {:x (.-x p) :y (.-y p) :z (.-z p)}) face)))
|
||||
(let [faces (aget (.call (aget model "detectForVideo") model canvas at-ms)
|
||||
"faceLandmarks")]
|
||||
(mapv (fn [face] (mapv (fn [p] {:x (.-x p) :y (.-y p) :z (.-z p)}) face))
|
||||
(array-seq faces))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; which face is which
|
||||
;;
|
||||
;; MediaPipe hands back a LIST, and a list has an order rather than an identity.
|
||||
;; Nothing in the API promises that slot 0 is the same person on frame 41 as on
|
||||
;; frame 40 — and when two faces cross, or one is briefly lost and re-detected,
|
||||
;; it is not. Left unassigned, the two faces' geometry would swap mid-shot inside
|
||||
;; one dense block, which reads as both heads snapping and is invisible in any
|
||||
;; per-frame assertion.
|
||||
;;
|
||||
;; So identity is assigned HERE, once, by nearest centroid to each track's last
|
||||
;; known position. Greedy and cheap: a handful of faces, one pass, and the
|
||||
;; ordering it produces is the `:subjects` map every later stage keys on.
|
||||
|
||||
(defn- centroid [face]
|
||||
(let [n (count face)]
|
||||
{:x (/ (reduce + (map :x face)) n)
|
||||
:y (/ (reduce + (map :y face)) n)}))
|
||||
|
||||
(defn- distance [a b]
|
||||
(let [dx (- (:x a) (:x b)) dy (- (:y a) (:y b))]
|
||||
(js/Math.sqrt (+ (* dx dx) (* dy dy)))))
|
||||
|
||||
(defn assign
|
||||
"Per-frame lists of faces -> one vector per TRACK of `detection index or nil`.
|
||||
|
||||
INDICES AND NOT FACES, because a detection is more than its landmarks: the
|
||||
mouth crop and the pixel measurement taken beside it on the same frame have to
|
||||
follow the same face, and handing back the index is what lets one assignment
|
||||
re-key all three. Whoever holds the per-frame lists does the lookup.
|
||||
|
||||
A track is claimed by the unclaimed detection nearest its last known centroid,
|
||||
nearest pair first, so a frame where MediaPipe swaps its slot order does not
|
||||
swap the tracks. A detection that matches no existing track within `gate`
|
||||
starts a new one — which is how a second person walking into the shot on frame
|
||||
200 gets their own subject instead of stealing the first one's.
|
||||
|
||||
`gate` is in normalised image units: two centroids closer than this on
|
||||
consecutive frames are one face moving, and further apart are not."
|
||||
([frames] (assign frames (:gate settings)))
|
||||
([frames gate]
|
||||
(let [step (fn [[rows last] faces]
|
||||
(let [cs (mapv centroid faces)
|
||||
pairs (sort-by :d
|
||||
(for [[t at] (map-indexed vector last)
|
||||
:when at
|
||||
[d c] (map-indexed vector cs)
|
||||
:let [gap (distance at c)]
|
||||
:when (< gap gate)]
|
||||
{:d gap :track t :face d}))
|
||||
claim (reduce (fn [{:keys [by-track taken] :as acc}
|
||||
{:keys [track face]}]
|
||||
(if (or (contains? by-track track)
|
||||
(contains? taken face))
|
||||
acc
|
||||
{:by-track (assoc by-track track face)
|
||||
:taken (conj taken face)}))
|
||||
{:by-track {} :taken #{}}
|
||||
pairs)
|
||||
opened (map-indexed (fn [i d] [(+ (count last) i) d])
|
||||
(remove (:taken claim) (range (count faces))))
|
||||
by-track (into (:by-track claim) opened)
|
||||
width (+ (count last) (count opened))]
|
||||
[(conj rows (mapv by-track (range width)))
|
||||
(mapv (fn [t] (if-let [d (get by-track t)]
|
||||
(nth cs d)
|
||||
(nth last t nil)))
|
||||
(range width))]))
|
||||
[rows _] (reduce step [[] []] frames)
|
||||
width (reduce max 0 (map count rows))]
|
||||
;; Transposed to column-major: one vector per subject, padded to the final
|
||||
;; width so a track that opened late still spans the whole take.
|
||||
(mapv (fn [t] (mapv (fn [row] (nth row t nil)) rows))
|
||||
(range width)))))
|
||||
|
||||
(defn fill-gaps
|
||||
"Keep the detection mask while supplying real poses for measurement. A leading
|
||||
gap uses the first observed face; later gaps hold the previous observed pose.
|
||||
Freeze uses the mask to mark those frames absent in the channel blocks."
|
||||
Freeze uses the mask to mark those frames absent in the channel blocks.
|
||||
|
||||
ONE TRACK, which is one subject: the mask says when THIS face was on screen,
|
||||
and two faces in a shot have two of them. A frame where the second person has
|
||||
not walked in yet is a frame their track is absent on, which is the same fact
|
||||
as a frame nobody was found on and needs no second mechanism."
|
||||
[raw]
|
||||
(let [first-real (first (keep-indexed (fn [i frame] (when frame i)) raw))]
|
||||
(when-not first-real
|
||||
|
|
@ -95,3 +180,26 @@
|
|||
:detected (mapv some? raw)
|
||||
:missing (count (remove some? raw))
|
||||
:first-real first-real}))
|
||||
|
||||
(defn subject-id
|
||||
"Track index -> the subject that owns it. `:face-1` is the first face found,
|
||||
which is also the id a single-face take has always used."
|
||||
[i]
|
||||
(keyword (str "face-" (inc i))))
|
||||
|
||||
(defn tracks
|
||||
"Per-frame detection lists -> `{:face-1 <slots>, …}`, one entry per tracked
|
||||
face, each a per-frame detection index or nil."
|
||||
[frames]
|
||||
(let [assigned (assign frames)]
|
||||
(when (empty? assigned)
|
||||
(throw (ex-info "no face found in any frame — check framing and light" {})))
|
||||
(into {} (map-indexed (fn [i slots] [(subject-id i) slots])) assigned)))
|
||||
|
||||
(defn pick
|
||||
"One track's slots applied to anything measured PER DETECTION — the landmarks,
|
||||
the mouth crop, the pixel measurement taken beside it. All three were recorded
|
||||
in detection order on a frame where nobody yet knew whose face was whose, and
|
||||
this is the single lookup that turns any of them into one subject's track."
|
||||
[per-frame slots]
|
||||
(mapv (fn [row slot] (when slot (nth row slot))) per-frame slots))
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@
|
|||
decisions about the mouth cavity, blink and teeth without thinning geometry."
|
||||
(:require [arthur.domain.channel :as ch]
|
||||
[arthur.domain.clip :as clip]
|
||||
[arthur.domain.feature :as feature]
|
||||
[arthur.domain.geom :as geom]
|
||||
[arthur.domain.ring :as ring]
|
||||
[arthur.flow.address :as address]))
|
||||
|
|
@ -82,7 +83,7 @@
|
|||
|
||||
ABSENCE IS PER TRACK, and `:features` is what says whose. Each track names the
|
||||
feature it follows, so one occluded eye can be absent while its partner still
|
||||
has a value; `nil` means the track follows the whole face's detection and no
|
||||
has a value; a subject id means the track follows whole-face detection and no
|
||||
feature, which is what the head's blocks do. `absent?` is then asked
|
||||
`(absent? feature f)` and never about a track index.
|
||||
|
||||
|
|
@ -248,31 +249,44 @@
|
|||
(def ^:private head-modes #{:free :anchored})
|
||||
|
||||
(defn head-mode
|
||||
"Keep the measured transform dense; optionally hold chosen source frames.
|
||||
"Keep a subject's measured transform dense; optionally hold chosen source
|
||||
frames.
|
||||
|
||||
A nil anchor map reads measured frame f at frame f (free movement).
|
||||
`{0 12}` locks to the measured transform of source frame 12. `{0 12, 40 42}`
|
||||
cuts to source frame 42 at local frame 40. The same map selects position,
|
||||
rotation and scale, so the head and registered photo cannot drift apart.
|
||||
No analysis block or authored face placement changes."
|
||||
[{:keys [mode anchors]} {:keys [clip]}]
|
||||
No analysis block or authored face placement changes.
|
||||
|
||||
ONE SUBJECT AT A TIME when `:subject` is given, and EVERY subject when it is
|
||||
not. Two faces in one shot were filmed together and are posed apart: choosing
|
||||
frame 12 for the second face must leave the first one running, and it does,
|
||||
because an anchor map lives on that subject's own head node and
|
||||
`domain/timeline` reads anchors off whatever node carries them."
|
||||
[{:keys [subject mode anchors]} {:keys [clip]}]
|
||||
(when-not (contains? head-modes mode)
|
||||
(throw (ex-info "head mode must be free or anchored"
|
||||
{:mode mode :modes head-modes})))
|
||||
(when (and (= mode :free) (some? anchors))
|
||||
(throw (ex-info "free head motion has no anchors" {:anchors anchors})))
|
||||
(let [frames (clip/frames clip)]
|
||||
(when (and (= mode :anchored)
|
||||
(not (and (map? anchors) (contains? anchors 0)
|
||||
(every? #(and (integer? %) (<= 0 %) (< % frames))
|
||||
(concat (keys anchors) (vals anchors))))))
|
||||
(throw (ex-info "anchored head needs a frame-zero key and valid source frames"
|
||||
{:anchors anchors :frames frames})))
|
||||
(update-in clip [:timelines clip/root-id :nodes :head]
|
||||
(fn [n]
|
||||
(cond-> (assoc n :channels (:measured n))
|
||||
(= mode :anchored) (assoc :anchors anchors)
|
||||
(= mode :free) (dissoc :anchors))))))
|
||||
(when (and subject (not (contains? (:subjects clip) subject)))
|
||||
(throw (ex-info "head mode names a subject this clip did not track"
|
||||
{:subject subject :subjects (vec (sort-by str (keys (:subjects clip))))})))
|
||||
(reduce
|
||||
(fn [c sid]
|
||||
(let [frames (get-in c [:timelines sid :frames])]
|
||||
(when (and (= mode :anchored)
|
||||
(not (and (map? anchors) (contains? anchors 0)
|
||||
(every? #(and (integer? %) (<= 0 %) (< % frames))
|
||||
(concat (keys anchors) (vals anchors))))))
|
||||
(throw (ex-info "anchored head needs a frame-zero key and valid source frames"
|
||||
{:subject sid :anchors anchors :frames frames})))
|
||||
(update-in c [:timelines sid :nodes :head]
|
||||
(fn [n]
|
||||
(cond-> (assoc n :channels (:measured n))
|
||||
(= mode :anchored) (assoc :anchors anchors)
|
||||
(= mode :free) (dissoc :anchors))))))
|
||||
clip (if subject [subject] (sort-by str (keys (:subjects clip))))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the aperture, onto [:vis] of :mouth-in
|
||||
|
|
@ -323,53 +337,56 @@
|
|||
:brow-r :brow-r :brow-l :brow-l})
|
||||
|
||||
(defn- performance-nodes
|
||||
"Mark the generated moving channels that may read an instance's pose choice.
|
||||
Provenance helps identify them here; the renderer reads only :pose-sampled?."
|
||||
"Mark channels that read the containing instance's pose choices."
|
||||
[nodes]
|
||||
(into {}
|
||||
(map (fn [[id n]]
|
||||
[id (cond-> n
|
||||
(contains? pose-groups id)
|
||||
(assoc :pose-group (get pose-groups id))
|
||||
(contains? pose-groups id)
|
||||
(update :channels
|
||||
(fn [channels]
|
||||
(into {}
|
||||
(map (fn [[path ch]]
|
||||
[path (cond-> ch
|
||||
(and (:generated ch) (:animated? ch))
|
||||
(assoc :pose-sampled? true))]))
|
||||
channels))))]))
|
||||
[id (if-let [group (get pose-groups id)]
|
||||
(-> n
|
||||
(assoc :pose-group group)
|
||||
(update :channels
|
||||
(fn [channels]
|
||||
(into {}
|
||||
(map (fn [[path ch]]
|
||||
[path (cond-> ch
|
||||
(and (:generated ch) (:animated? ch))
|
||||
(assoc :pose-sampled? true))]))
|
||||
channels))))
|
||||
n)]))
|
||||
nodes))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the face, onto the stage
|
||||
|
||||
(defn- motion-placement
|
||||
"An editable default framing for a real take. Fit every drawn face feature's
|
||||
observed motion inside the stage, including raised brows and a moving mouth."
|
||||
[[w h] {:keys [rigid transforms outer eyes brows detected]}]
|
||||
(let [points (mapcat (fn [i]
|
||||
(when (or (nil? detected) (nth detected i))
|
||||
(let [local (concat (nth outer i)
|
||||
(when eyes (concat (nth (:lash-r eyes) i)
|
||||
(nth (:lash-l eyes) i)))
|
||||
(when brows (concat (nth (:ring-r brows) i)
|
||||
(nth (:ring-l brows) i))))]
|
||||
(concat (nth rigid i)
|
||||
(geom/apply-sim-all (invert (nth transforms i))
|
||||
local)))))
|
||||
(range (count outer)))
|
||||
xs (map :x points)
|
||||
ys (map :y points)
|
||||
x0 (reduce min xs)
|
||||
x1 (reduce max xs)
|
||||
y0 (reduce min ys)
|
||||
y1 (reduce max ys)
|
||||
cx (/ (+ x0 x1) 2)
|
||||
cy (/ (+ y0 y1) 2)
|
||||
k (min (/ (* 0.8 w) (max 1e-9 (- x1 x0)))
|
||||
(/ (* 0.8 h) (max 1e-9 (- y1 y0))))]
|
||||
(defn- motion-points
|
||||
"Every source-space point one subject's drawn features visit over the shot,
|
||||
including raised brows and a moving mouth."
|
||||
[{:keys [rigid transforms outer eyes brows detected]}]
|
||||
(mapcat (fn [i]
|
||||
(when (or (nil? detected) (nth detected i))
|
||||
(let [local (concat (nth outer i)
|
||||
(when eyes (concat (nth (:lash-r eyes) i)
|
||||
(nth (:lash-l eyes) i)))
|
||||
(when brows (concat (nth (:ring-r brows) i)
|
||||
(nth (:ring-l brows) i))))]
|
||||
(concat (nth rigid i)
|
||||
(geom/apply-sim-all (invert (nth transforms i))
|
||||
local)))))
|
||||
(range (count outer))))
|
||||
|
||||
(defn- fit-points
|
||||
"Source-space points -> the placement that puts their bounding box on stage."
|
||||
[[w h] points]
|
||||
(let [xs (map :x points)
|
||||
ys (map :y points)
|
||||
x0 (reduce min xs)
|
||||
x1 (reduce max xs)
|
||||
y0 (reduce min ys)
|
||||
y1 (reduce max ys)
|
||||
cx (/ (+ x0 x1) 2)
|
||||
cy (/ (+ y0 y1) 2)
|
||||
k (min (/ (* 0.8 w) (max 1e-9 (- x1 x0)))
|
||||
(/ (* 0.8 h) (max 1e-9 (- y1 y0))))]
|
||||
{[:xform :anchor] (ch/framed [cx cy])
|
||||
[:xform :scale] (ch/framed [k k])
|
||||
[:xform :pos] (ch/framed [(- (/ w 2) cx) (- (/ h 2) cy)])}))
|
||||
|
|
@ -407,26 +424,33 @@
|
|||
POSITION puts the anchor a QUARTER of the way down the stage, because the rigid
|
||||
landmarks are eyes and nose — the upper middle of a face — so a quarter down
|
||||
leaves the jaw and the mouth on the stage. Whatever hangs off is clipped, which
|
||||
is not a feature to add: every fill in `domain/raster` clamps already."
|
||||
[{:keys [stage fit-motion?]} {:keys [ref] :as inputs}]
|
||||
(if fit-motion?
|
||||
(motion-placement stage inputs)
|
||||
(let [[w h] stage
|
||||
c (geom/centroid ref)
|
||||
span (- (reduce max (map :x ref)) (reduce min (map :x ref)))
|
||||
k (/ (* 0.4 w) span)]
|
||||
{[:xform :anchor] (ch/framed [(:x c) (:y c)])
|
||||
[:xform :scale] (ch/framed [k k])
|
||||
[:xform :pos] (ch/framed [(- (/ w 2) (:x c))
|
||||
(- (* 0.25 h) (:y c))])})))
|
||||
is not a feature to add: every fill in `domain/raster` clamps already.
|
||||
|
||||
(defn- mouth-part [absent? obs
|
||||
All subjects share one source-to-stage mapping on the :face group. Each
|
||||
instance can then be placed independently with ordinary transform channels."
|
||||
[{:keys [stage fit-motion?]} subjects]
|
||||
(let [[w h] stage
|
||||
inputs (vals subjects)]
|
||||
(if fit-motion?
|
||||
(fit-points stage (mapcat motion-points inputs))
|
||||
(let [ref (mapcat :ref inputs)
|
||||
c (geom/centroid ref)
|
||||
span (- (reduce max (map :x ref)) (reduce min (map :x ref)))
|
||||
k (/ (* 0.4 w) span)]
|
||||
{[:xform :anchor] (ch/framed [(:x c) (:y c)])
|
||||
[:xform :scale] (ch/framed [k k])
|
||||
[:xform :pos] (ch/framed [(- (/ w 2) (:x c))
|
||||
(- (* 0.25 h) (:y c))])}))))
|
||||
|
||||
(defn- mouth-part [subject absent? obs
|
||||
{:keys [analysis verts anchor-avg contour-avg aperture-cut] :as params}
|
||||
{:keys [outer inner] :as inputs}]
|
||||
(let [rings (block {:role "geom" :analysis (:id analysis) :params params
|
||||
(let [own (partial feature/owned subject)
|
||||
mouth (own :mouth)
|
||||
rings (block {:role "geom" :analysis (:id analysis) :params params
|
||||
:tracks ["outer" "inner"]}
|
||||
{:type "int16" :scale geom-scale
|
||||
:features [:mouth :mouth] :absent? absent?}
|
||||
:features [mouth mouth] :absent? absent?}
|
||||
obs [(rings->flat outer verts) (rings->flat inner verts)])
|
||||
prov (fn [by]
|
||||
{:by by :analysis (:id analysis)
|
||||
|
|
@ -452,9 +476,11 @@
|
|||
"Freeze eyes and brows into their own dense blocks and scene nodes. This owns
|
||||
only representation: the landmark correspondence, blink and pose choices have
|
||||
already been settled by measure and condition."
|
||||
[absent? obs {:keys [eye-verts brow-verts analysis contour-avg anchor-avg] :as params}
|
||||
[subject absent? obs
|
||||
{:keys [eye-verts brow-verts analysis contour-avg anchor-avg] :as params}
|
||||
{:keys [eyes brows]}]
|
||||
(let [provenance (fn [by extra]
|
||||
(let [own (partial feature/owned subject)
|
||||
provenance (fn [by extra]
|
||||
{:by by :analysis (:id analysis)
|
||||
:params (merge {:anchor-avg anchor-avg :contour-avg contour-avg}
|
||||
extra)})
|
||||
|
|
@ -470,20 +496,23 @@
|
|||
;; and `each-dense-track-follows-its-own-features-presence` is what pins it.
|
||||
eye-block (when eyes (named "eyes"
|
||||
["lash-r" "lid-r" "lash-l" "lid-l"]
|
||||
[:eye-r :eye-r :eye-l :eye-l]
|
||||
[(own :eye-r) (own :eye-r) (own :eye-l) (own :eye-l)]
|
||||
"int16"
|
||||
(mapv #(rings->flat % eye-verts)
|
||||
[(:lash-r eyes) (:lid-r eyes)
|
||||
(:lash-l eyes) (:lid-l eyes)])))
|
||||
iris-block (when eyes
|
||||
(named "iris-pos" ["iris-r" "iris-l"] [:eye-r :eye-l] "float32"
|
||||
(named "iris-pos" ["iris-r" "iris-l"]
|
||||
[(own :eye-r) (own :eye-l)] "float32"
|
||||
[(:iris-r eyes) (:iris-l eyes)]))
|
||||
brow-block (when brows
|
||||
(named "brows" ["ring-r" "ring-l"] [:brow-r :brow-l] "int16"
|
||||
(named "brows" ["ring-r" "ring-l"]
|
||||
[(own :brow-r) (own :brow-l)] "int16"
|
||||
(mapv #(rings->flat % brow-verts)
|
||||
[(:ring-r brows) (:ring-l brows)])))
|
||||
brow-pos-block (when brows
|
||||
(named "brow-pos" ["pos-r" "pos-l"] [:brow-r :brow-l] "float32"
|
||||
(named "brow-pos" ["pos-r" "pos-l"]
|
||||
[(own :brow-r) (own :brow-l)] "float32"
|
||||
[(:pos-r brows) (:pos-l brows)]))
|
||||
eye-node (fn [id z track]
|
||||
{:id id :name (clojure.core/name id) :kind :poly :parent :head :z z
|
||||
|
|
@ -525,14 +554,14 @@
|
|||
[:style :color] (ch/framed :brow)}})]
|
||||
{:nodes (merge
|
||||
(when eyes
|
||||
{:eye-r (eye-node :eye-r "a2" 0)
|
||||
:eye-r-in (inner-node :eye-r-in :eye-r "a1" 1 (:shut-r eyes))
|
||||
:iris-r (iris-node :iris-r :eye-r-in 0 (:radius-r eyes))
|
||||
:pupil-r (pupil-node :pupil-r :iris-r)
|
||||
:eye-l (eye-node :eye-l "a3" 2)
|
||||
:eye-l-in (inner-node :eye-l-in :eye-l "a1" 3 (:shut-l eyes))
|
||||
:iris-l (iris-node :iris-l :eye-l-in 1 (:radius-l eyes))
|
||||
:pupil-l (pupil-node :pupil-l :iris-l)})
|
||||
{:eye-r (eye-node :eye-r "a2" 0)
|
||||
:eye-r-in (inner-node :eye-r-in :eye-r "a1" 1 (:shut-r eyes))
|
||||
:iris-r (iris-node :iris-r :eye-r-in 0 (:radius-r eyes))
|
||||
:pupil-r (pupil-node :pupil-r :iris-r)
|
||||
:eye-l (eye-node :eye-l "a3" 2)
|
||||
:eye-l-in (inner-node :eye-l-in :eye-l "a1" 3 (:shut-l eyes))
|
||||
:iris-l (iris-node :iris-l :eye-l-in 1 (:radius-l eyes))
|
||||
:pupil-l (pupil-node :pupil-l :iris-l)})
|
||||
(when brows
|
||||
{:brow-r (brow-node :brow-r "a4" 0)
|
||||
:brow-l (brow-node :brow-l "a5" 1)}))
|
||||
|
|
@ -542,10 +571,11 @@
|
|||
(defn- interior-part
|
||||
"Freeze the pixel-derived radial contour under the mouth cavity. Missing
|
||||
contours use the dense block's absence bit; contrast decides editable :vis."
|
||||
[{:keys [analysis teeth-verts cavity-erode tongue-reject blob-grow
|
||||
top-bias teeth-on teeth-smooth] :as params} absent? obs
|
||||
[subject {:keys [analysis teeth-verts cavity-erode tongue-reject blob-grow
|
||||
top-bias teeth-on teeth-smooth] :as params} absent? obs
|
||||
{:keys [contours shown]}]
|
||||
(let [empty-points (vec (repeat (* 2 teeth-verts) 0))
|
||||
(let [own (partial feature/owned subject)
|
||||
empty-points (vec (repeat (* 2 teeth-verts) 0))
|
||||
values (mapv (fn [ring]
|
||||
(if ring
|
||||
(into [] (mapcat (juxt :x :y)) ring)
|
||||
|
|
@ -553,7 +583,7 @@
|
|||
blk (block {:role "teeth" :analysis (:id analysis) :params params
|
||||
:tracks ["contour"]}
|
||||
{:type "int16" :scale geom-scale
|
||||
:features [:teeth] :absent? absent?
|
||||
:features [(own :teeth)] :absent? absent?
|
||||
;; Not the feature's absence: a frame no contour could be
|
||||
;; extracted from has no teeth to draw whether or not the
|
||||
;; teeth were occluded, and the two reasons are different facts.
|
||||
|
|
@ -565,7 +595,8 @@
|
|||
:top-bias top-bias :teeth-verts teeth-verts
|
||||
:teeth-on teeth-on :teeth-smooth teeth-smooth}}]
|
||||
{:nodes {:teeth
|
||||
{:id :teeth :name "teeth" :kind :poly :parent :mouth-in :z "a1"
|
||||
{:id :teeth :name "teeth" :kind :poly
|
||||
:parent :mouth-in :z "a1"
|
||||
:stencil :mouth-in
|
||||
:channels {[:geom :pts] (dense blk 0 generated)
|
||||
[:style :color] (ch/framed :teeth)
|
||||
|
|
@ -573,32 +604,40 @@
|
|||
:store (stored blk)}))
|
||||
|
||||
(defn part
|
||||
"A feature type's nodes and addressed blocks, with no clip scaffold."
|
||||
[area params {:keys [detected presence] :as measured}]
|
||||
(let [absent? (when (or detected presence)
|
||||
"One feature type: local nodes and blocks addressed by subject and feature."
|
||||
[subject area params {:keys [detected presence] :as measured}]
|
||||
(let [presence (into {} (map (fn [[role mask]] [(feature/owned subject role) mask])) presence)
|
||||
absent? (when (or detected presence)
|
||||
(fn [id f]
|
||||
(or (and detected (not (nth detected f true)))
|
||||
(and (contains? presence id)
|
||||
(not (nth (get presence id) f))))))
|
||||
obs {:detected detected :presence presence}]
|
||||
(update (case area
|
||||
:mouth (mouth-part absent? obs params measured)
|
||||
:eye (feature-parts absent? obs params (select-keys measured [:eyes]))
|
||||
:brow (feature-parts absent? obs params (select-keys measured [:brows]))
|
||||
:teeth (interior-part params absent? obs (:teeth measured))
|
||||
:mouth (mouth-part subject absent? obs params measured)
|
||||
:eye (feature-parts subject absent? obs params (select-keys measured [:eyes]))
|
||||
:brow (feature-parts subject absent? obs params (select-keys measured [:brows]))
|
||||
:teeth (interior-part subject params absent? obs (:teeth measured))
|
||||
(throw (ex-info "unknown frozen feature type" {:area area})))
|
||||
:nodes performance-nodes)))
|
||||
|
||||
(defn head-part
|
||||
"Freeze only the measured head transform from an already conditioned anchor."
|
||||
[{:keys [analysis anchor-avg] :as params} {:keys [transforms detected presence]}]
|
||||
"Freeze one subject's measured head transform from its conditioned anchor.
|
||||
|
||||
THE THREE BLOCKS NAME THE SUBJECT, and that is not decoration. A block's key is
|
||||
a hash over its descriptor, and the head follows DETECTION rather than any
|
||||
feature's presence — so with `nil` in the feature slot, two faces tracked in one
|
||||
analysis, both detected on every frame, produced byte-for-byte different
|
||||
transforms under one identical key, and the second freeze's block silently
|
||||
replaced the first's. The subject is the feature the head follows."
|
||||
[subject {:keys [analysis anchor-avg] :as params} {:keys [transforms detected presence]}]
|
||||
(let [absent? (when (or detected presence)
|
||||
(fn [_ f] (and detected (not (nth detected f true)))))
|
||||
inv (mapv invert transforms)
|
||||
xf (fn [role f]
|
||||
(block {:role role :analysis (:id analysis) :params params
|
||||
:tracks [role]}
|
||||
{:type "float32" :features [nil] :absent? absent?}
|
||||
{:type "float32" :features [subject] :absent? absent?}
|
||||
{:detected detected}
|
||||
[(mapv f inv)]))
|
||||
pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)]))
|
||||
|
|
@ -614,184 +653,83 @@
|
|||
;; ---------------------------------------------------------------------------
|
||||
;; the clip
|
||||
|
||||
(defn- subject-part
|
||||
"A subject's drawing, metadata and blocks. Node names are timeline-local."
|
||||
[params subject {:keys [outer eyes brows teeth] :as inputs}]
|
||||
(let [own (partial feature/owned subject)
|
||||
areas (cond-> [:mouth] (and eyes brows) (into [:eye :brow]) teeth (conj :teeth))
|
||||
parts (mapv #(part subject % params inputs) areas)
|
||||
head (head-part subject params inputs)
|
||||
features (cond-> {:mouth [:mouth [:mouth :mouth-in]]}
|
||||
(and eyes brows)
|
||||
(merge {:eye-r [:eye [:eye-r :eye-r-in :iris-r :pupil-r]]
|
||||
:eye-l [:eye [:eye-l :eye-l-in :iris-l :pupil-l]]
|
||||
:brow-r [:brow [:brow-r]] :brow-l [:brow [:brow-l]]})
|
||||
teeth (assoc :teeth [:teeth [:teeth]]))]
|
||||
{:timeline {:id subject :frames (count outer)
|
||||
:nodes (into {:head {:id :head :name "head" :kind :group :z "a1"
|
||||
:measured (:measured head)}}
|
||||
(mapcat :nodes) parts)}
|
||||
:features (into {} (map (fn [[role [area nodes]]]
|
||||
[(own role) {:id (own role) :subject subject
|
||||
:timeline subject :area area
|
||||
:nodes nodes :params {}}])) features)
|
||||
:groups (if (and eyes brows)
|
||||
{(own :eyes) {:id (own :eyes) :kind :eye-pair :subject subject
|
||||
:members [(own :eye-r) (own :eye-l)] :params {}}}
|
||||
{})
|
||||
:store (into (:store head) (mapcat :store) parts)}))
|
||||
|
||||
(defn clip
|
||||
"Conditioned measurements -> a clip: nodes with frozen channels, plus the dense
|
||||
blocks they read. `(f params inputs)`, no state.
|
||||
"Subject-id -> conditioned measurements becomes a library of face timelines.
|
||||
|
||||
params
|
||||
:name labels the clip. It is NOT in any key: two clips of the same
|
||||
footage under different names are the same analysis and the
|
||||
same blocks, and sharing them is the return on addressing.
|
||||
:fps the clip's rate. FRAMES is not a parameter — it is
|
||||
`(count outer)`, because a freeze that could disagree with its
|
||||
own input about the length of the take would.
|
||||
:stage [w h] project dimensions, INDEPENDENT of the footage
|
||||
:fit-motion? choose an editable real-footage default that keeps the
|
||||
observed feature motion on stage
|
||||
:expose the clip root's exposure grid, inherited by everything
|
||||
:verts the lip rings' vertex budget
|
||||
:eye-verts the eyelid rings' vertex budget
|
||||
:brow-verts the brow rings' vertex budget
|
||||
:aperture-cut fraction of the take's peak aperture below which the mouth
|
||||
interior is not present
|
||||
:head :free | :anchored
|
||||
:anchors optional map of local change frame -> measured source frame
|
||||
:analysis the analysis record these measurements came from — detector,
|
||||
VERSION, source, source cadence and pixel aspect. Its `:id` is
|
||||
a content address over all of that, every block's key is a hash
|
||||
over that id, and `flow/address` says why the version being in
|
||||
there is the one field that must not be forgotten.
|
||||
:anchor-avg
|
||||
:contour-avg the stage-4 knobs. Freeze does not use them; it RECORDS them,
|
||||
because `:generated` is what lets the UI offer a re-freeze at
|
||||
different parameters instead of raw keys.
|
||||
:main holds exposure and a shared source-to-stage placement. Each subject is
|
||||
placed by an ordinary symbol instance, so pose choices and transforms have
|
||||
their existing instance scope. Features name local nodes in that subject's
|
||||
timeline; block descriptors still name globally distinct features.
|
||||
|
||||
inputs
|
||||
:ref the Procrustes reference configuration
|
||||
:transforms the CONDITIONED anchor transforms
|
||||
:outer :inner the CONDITIONED head-local lip rings
|
||||
:eyes :brows the CONDITIONED head-local feature measurements
|
||||
:teeth optional pixel-derived, conditioned radial contour
|
||||
:aperture head-local aperture per frame
|
||||
:detected optional per-frame face booleans
|
||||
:presence optional feature-id -> per-frame booleans; false means an
|
||||
unobserved feature, even if the rest of the face was found
|
||||
|
||||
The node tree is the one docs/animation-model.md specifies, and the two groups
|
||||
are two different things wanting the same transform:
|
||||
|
||||
:root the clip. EXPOSURE LIVES HERE and is inherited strictly.
|
||||
:face AUTHORED. where the face sits on the stage, and how big.
|
||||
:head MEASURED. the head's motion, or identity.
|
||||
:mouth
|
||||
:mouth-in
|
||||
:teeth
|
||||
:eye-r / :eye-l
|
||||
:eye-*-in
|
||||
:iris-*
|
||||
:pupil-*
|
||||
:brow-r / :brow-l
|
||||
|
||||
`:mouth-in`'s parent is `:mouth` and that transform is identity today, so
|
||||
composing it is composing identity. It is documented intent, and it is the one
|
||||
place in the tree where the parent pointer is not yet doing work — the moment a
|
||||
painted cel rides a moving plate it will be.
|
||||
|
||||
`:head` keeps its measured channels under `:measured` as well as in
|
||||
`:channels`; its optional `:anchors` map changes only their frame address."
|
||||
[{:keys [name fps stage expose verts aperture-cut head anchors
|
||||
analysis anchor-avg contour-avg] :as params}
|
||||
{:keys [transforms outer inner detected presence eyes brows teeth] :as inputs}]
|
||||
(let [nf (count outer)
|
||||
_ (doseq [[id track] presence]
|
||||
(when (not= nf (count track))
|
||||
(throw (ex-info "feature presence track must match the clip"
|
||||
{:feature id :frames nf :actual (count track)}))))
|
||||
;; Asked `(absent? feature f)`, where a nil feature is the whole face. A
|
||||
;; feature with no presence track is present whenever a face was found.
|
||||
absent? (when (or detected presence)
|
||||
(fn [id f]
|
||||
(or (and detected (not (nth detected f true)))
|
||||
(and (contains? presence id)
|
||||
(not (nth (get presence id) f))))))
|
||||
obs {:detected detected :presence presence}
|
||||
prov (fn [by extra]
|
||||
{:by by :analysis (:id analysis)
|
||||
:params (merge {:anchor-avg anchor-avg} extra)})
|
||||
mouth (mouth-part absent? obs params inputs)
|
||||
;; The anchor, inverted and split into its three components. Three blocks
|
||||
;; and not one: they are three channels, they have three strides, and a
|
||||
;; single block would need a per-component offset table to say so.
|
||||
inv (mapv invert transforms)
|
||||
xf (fn [role f]
|
||||
;; The head follows DETECTION and no feature's presence: an
|
||||
;; occluded eye does not mean the head was not there. That is
|
||||
;; what the nil feature says.
|
||||
(block {:role role :analysis (:id analysis) :params params
|
||||
:tracks [role]}
|
||||
{:type "float32" :features [nil] :absent? absent?}
|
||||
{:detected detected}
|
||||
[(mapv f inv)]))
|
||||
pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)]))
|
||||
rot (xf "head-rot" (fn [t] [(:theta t)]))
|
||||
scale (xf "head-scale" (fn [t] [(:s t) (:s t)]))
|
||||
;; Which knobs each channel records is not decoration, it is the
|
||||
;; invalidation table written down where a re-freeze can read it. The
|
||||
;; anchor depends on `anchor avg` alone. The rings depend on it and on
|
||||
;; `contour avg` and on the vertex budget. The aperture depends on
|
||||
;; `anchor avg` and on its own cut and NOT on `contour avg`, because
|
||||
;; measure reports the inner ring's own height and nothing smooths it.
|
||||
anchor-prov (prov :anchor/similarity nil)
|
||||
features (when (and eyes brows) (feature-parts absent? obs params inputs))
|
||||
interior (when teeth (interior-part params absent? obs teeth))
|
||||
built {:name name
|
||||
:fps fps
|
||||
;; Tier 1 says which analysis its channels came out of, in full.
|
||||
;; The id alone would make the document unreadable the first time
|
||||
;; a detector upgrade orphaned a block: "sha256:7f2…" is not an
|
||||
;; answer to "which model produced this take".
|
||||
:analysis analysis
|
||||
;; A gap changes channel state, never these IDs or pair links.
|
||||
:subjects {:face-1 {:id :face-1 :params {}}}
|
||||
:features (merge
|
||||
{:mouth {:id :mouth :subject :face-1 :area :mouth
|
||||
:nodes [:mouth :mouth-in] :params {}}}
|
||||
(when features
|
||||
{:eye-r {:id :eye-r :subject :face-1 :area :eye
|
||||
:nodes [:eye-r :eye-r-in :iris-r :pupil-r] :params {}}
|
||||
:eye-l {:id :eye-l :subject :face-1 :area :eye
|
||||
:nodes [:eye-l :eye-l-in :iris-l :pupil-l] :params {}}
|
||||
:brow-r {:id :brow-r :subject :face-1 :area :brow
|
||||
:nodes [:brow-r] :params {}}
|
||||
:brow-l {:id :brow-l :subject :face-1 :area :brow
|
||||
:nodes [:brow-l] :params {}}})
|
||||
(when teeth
|
||||
{:teeth {:id :teeth :subject :face-1 :area :teeth
|
||||
:nodes [:teeth] :params {}}}))
|
||||
:groups (if features
|
||||
{:eyes-1 {:id :eyes-1 :kind :eye-pair :subject :face-1
|
||||
:members [:eye-r :eye-l] :params {}}}
|
||||
{})
|
||||
;; Stage dimensions, on the clip and not on the footage. See
|
||||
;; face-placement: nothing below here knows the frame size.
|
||||
:width (first stage)
|
||||
:height (second stage)
|
||||
;; ONE TIMELINE, and the frame count is ITS. A clip is a rate and
|
||||
;; a timeline is a frame space — see arthur.domain.clip — so `nf`
|
||||
;; lands here and `:fps` above, and the library a symbol will live
|
||||
;; in is this same map with a second entry.
|
||||
:timelines
|
||||
{clip/root-id
|
||||
{:id clip/root-id
|
||||
:frames nf
|
||||
:nodes
|
||||
(merge
|
||||
{:root
|
||||
{:id :root :name "clip" :kind :group :parent nil :z "a1"
|
||||
:time {:mode :map :expose expose}}
|
||||
|
||||
:face
|
||||
{:id :face :name "face" :kind :group :parent :root :z "a1"
|
||||
:channels (face-placement params inputs)}
|
||||
|
||||
:head
|
||||
{:id :head :name "head" :kind :group :parent :face :z "a1"
|
||||
:measured {[:xform :pos] (dense pos 0 anchor-prov)
|
||||
[:xform :rot] (dense rot 0 anchor-prov)
|
||||
[:xform :scale] (dense scale 0 anchor-prov)}}}
|
||||
(:nodes mouth) (:nodes features) (:nodes interior))}}}
|
||||
;; Tier 2, behind a handle, and now behind a content address: every key is
|
||||
;; a sha256 over the analysis, the settings and the absence data that
|
||||
;; produced the bytes under it. Nothing above this line changed when they
|
||||
;; stopped being "take/geom", which is the point of a handle.
|
||||
store (merge (stored pos rot scale) (:store mouth)
|
||||
(:store features) (:store interior))]
|
||||
(doseq [id (keys presence)]
|
||||
(when-not (contains? (:features built) id)
|
||||
(throw (ex-info "presence track names no feature in this clip"
|
||||
{:feature id :features (keys (:features built))}))))
|
||||
{:store store
|
||||
:clip (head-mode {:mode head :anchors anchors}
|
||||
{:clip (update-in built [:timelines clip/root-id :nodes]
|
||||
performance-nodes)
|
||||
:store store})}))
|
||||
Subjects share a source frame space. :head and :anchors may be overridden
|
||||
per subject; all other freeze settings come from params."
|
||||
[{:keys [name fps stage expose head anchors] :as params} subjects]
|
||||
(when-not (and (map? subjects) (seq subjects)
|
||||
(every? keyword? (keys subjects))
|
||||
(not-any? #{:main :root :face} (keys subjects)))
|
||||
(throw (ex-info "a freeze needs subjects with ids distinct from :main, :root and :face" {})))
|
||||
(let [ordered (sort-by (comp str key) subjects)
|
||||
parts (mapv (fn [[id inputs]] [id (subject-part params id inputs)]) ordered)
|
||||
lengths (distinct (map #(get-in % [1 :timeline :frames]) parts))
|
||||
_ (when-not (and (= 1 (count lengths)) (pos? (first lengths)))
|
||||
(throw (ex-info "subjects need the same positive frame count"
|
||||
{:frames (vec lengths)})))
|
||||
nf (first lengths)
|
||||
merged (fn [k] (into {} (mapcat (comp k second)) parts))
|
||||
built {:name name :fps fps :analysis (:analysis params)
|
||||
:width (first stage) :height (second stage)
|
||||
:subjects (into {} (map (fn [[id _]] [id {:id id :params {}}])) ordered)
|
||||
:features (merged :features) :groups (merged :groups)
|
||||
:timelines
|
||||
(into {clip/root-id
|
||||
{:id clip/root-id :frames nf
|
||||
:nodes (into {:root {:id :root :name "clip" :kind :group :z "a1"
|
||||
:time {:mode :map :expose expose}}
|
||||
:face {:id :face :name "source placement" :kind :group
|
||||
:parent :root :z "a1"
|
||||
:channels (face-placement params subjects)}}
|
||||
(map-indexed
|
||||
(fn [i [id _]]
|
||||
[id {:id id :kind :symbol :of id :parent :face
|
||||
:z (str "a" i)}]))
|
||||
ordered)}}
|
||||
(map (fn [[id part]] [id (:timeline part)])) parts)}]
|
||||
(doseq [[subject inputs] ordered
|
||||
[id track] (:presence inputs)]
|
||||
(when-not (and (= nf (count track))
|
||||
(= subject (get-in built [:features (feature/owned subject id) :subject])))
|
||||
(throw (ex-info "presence must name this subject's feature and span the take"
|
||||
{:subject subject :feature id :frames nf :actual (count track)}))))
|
||||
{:store (merged :store)
|
||||
:clip (reduce (fn [c [subject inputs]]
|
||||
(head-mode {:subject subject :mode (or (:head inputs) head)
|
||||
:anchors (get inputs :anchors anchors)}
|
||||
{:clip c}))
|
||||
built ordered)}))
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
(ns arthur.flow.regenerate
|
||||
"Recompute a changed feature from retained source tracks, then replace only
|
||||
channels owned by that feature. Upload remains project/save's ordinary job."
|
||||
(:require [arthur.domain.clip :as clip]
|
||||
[arthur.domain.feature :as feature]
|
||||
(:require [arthur.domain.feature :as feature]
|
||||
[arthur.domain.params :as params]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.freeze :as freeze]
|
||||
|
|
@ -40,7 +39,8 @@
|
|||
:when (:generated channel)]
|
||||
[id prop channel])]
|
||||
(-> (reduce (fn [entry [id prop channel]]
|
||||
(let [at [:clip :timelines clip/root-id :nodes id :channels prop]
|
||||
(let [at [:clip :timelines (get-in entry [:clip :features fid :timeline])
|
||||
:nodes id :channels prop]
|
||||
old (get-in entry at)]
|
||||
(assoc-in entry at
|
||||
(cond-> channel
|
||||
|
|
@ -85,24 +85,25 @@
|
|||
owns those input edges, and neither one is a request to freeze the other
|
||||
feature."
|
||||
[base-params base source-inputs entry fid]
|
||||
(let [area (get-in entry [:clip :features fid :area])
|
||||
(let [{:keys [area subject]} (get-in entry [:clip :features fid])
|
||||
params (merge base-params (settings (:clip entry) fid))]
|
||||
(when (and (= :teeth area) (nil? (:interior source-inputs)))
|
||||
(throw (ex-info "teeth regeneration needs retained pixel measurements"
|
||||
{:feature fid})))
|
||||
(replace-feature entry
|
||||
(freeze/part area params
|
||||
(freeze/part subject area params
|
||||
(take/measure-part area params source-inputs @base))
|
||||
fid)))
|
||||
|
||||
(defn- regenerate-head
|
||||
"Re-freeze the head transform, which is a different job from a feature's: its
|
||||
only input is the conditioned anchor, and it owns no channels to replace. The
|
||||
authored `:channels` follow the measurement while they still ARE the
|
||||
measurement, and are left alone once somebody has placed the head by hand."
|
||||
[entry params base]
|
||||
(let [baked (freeze/head-part params @base)
|
||||
at [:clip :timelines clip/root-id :nodes :head]
|
||||
"Re-freeze ONE SUBJECT's head transform, which is a different job from a
|
||||
feature's: its only input is that subject's conditioned anchor, and it owns no
|
||||
channels to replace. The authored `:channels` follow the measurement while they
|
||||
still ARE the measurement, and are left alone once somebody has placed the head
|
||||
by hand."
|
||||
[entry params base subject]
|
||||
(let [baked (freeze/head-part subject params @base)
|
||||
at [:clip :timelines subject :nodes :head]
|
||||
old (get-in entry at)
|
||||
measured (:measured baked)]
|
||||
(cond-> (-> entry
|
||||
|
|
@ -111,13 +112,19 @@
|
|||
(= (:channels old) (:measured old))
|
||||
(assoc-in (conj at :channels) measured))))
|
||||
|
||||
(defn- change-take
|
||||
(defn change
|
||||
"One scoped static edit. `source-inputs` holds dense landmarks and, when the
|
||||
teeth are dirty, retained pixel measurements. No IO or app-db here."
|
||||
[{:keys [clip source-inputs] :as entry} edit]
|
||||
(when-not (:dense source-inputs)
|
||||
(throw (ex-info "regeneration needs retained source landmarks" {})))
|
||||
(let [{:keys [changed subject features]} (plan clip edit)
|
||||
;; THE EDITED SUBJECT'S OWN LANDMARKS. Retained source is per subject —
|
||||
;; one video, one dense track per tracked face — so re-measuring the
|
||||
;; second face through the first one's anchor is the mistake this lookup
|
||||
;; exists to prevent.
|
||||
source-inputs (get-in source-inputs [:subjects subject])
|
||||
_ (when-not (:dense source-inputs)
|
||||
(throw (ex-info "regeneration needs retained source landmarks"
|
||||
{:subject subject})))
|
||||
base-params (merge take/knobs
|
||||
{:fps (:fps changed)
|
||||
:aspect (get-in changed [:analysis :aspect])
|
||||
|
|
@ -131,21 +138,4 @@
|
|||
base (delay (take/anchor-base anchor-params source-inputs))]
|
||||
(cond-> (reduce (partial regenerate-feature base-params base source-inputs)
|
||||
(assoc entry :clip changed) features)
|
||||
(= :anchor-avg (:knob edit)) (regenerate-head anchor-params base))))
|
||||
|
||||
(defn change
|
||||
"A take edits its root timeline. A stage edits the shared symbol timeline;
|
||||
every instance then reads the same new channels on its next paint."
|
||||
[{:keys [clip] :as entry} edit]
|
||||
(if-let [symbol (some :timeline (vals (:features clip)))]
|
||||
(let [original (:timelines clip)
|
||||
take (assoc clip :timelines
|
||||
{clip/root-id (assoc (get original symbol) :id clip/root-id)})
|
||||
changed (change-take (assoc entry :clip take) edit)]
|
||||
(assoc changed :clip
|
||||
(-> (:clip changed)
|
||||
(assoc :timelines
|
||||
(assoc original symbol
|
||||
(assoc (get-in changed [:clip :timelines clip/root-id])
|
||||
:id symbol))))))
|
||||
(change-take entry edit)))
|
||||
(= :anchor-avg (:knob edit)) (regenerate-head anchor-params base subject))))
|
||||
|
|
|
|||
|
|
@ -9,25 +9,26 @@
|
|||
|
||||
(def roles ["source/dense" "source/detected" "source/crops"])
|
||||
|
||||
(defn- interior-address [analysis settings frames]
|
||||
(defn- interior-address [analysis subject settings frames]
|
||||
(let [verts (:teeth-verts settings)
|
||||
stride (+ 2 (* 2 verts))]
|
||||
(address/block {:role "source/interior" :analysis analysis
|
||||
:params settings
|
||||
:tracks ["contrast" "area" "contour"]
|
||||
:features [] :observation nil
|
||||
:features [subject] :observation nil
|
||||
:layout {:type "float64" :frames frames :verts verts
|
||||
:stride stride}})))
|
||||
|
||||
(defn interior-key [analysis settings frames]
|
||||
(:key (interior-address analysis settings frames)))
|
||||
(defn interior-key [analysis subject settings frames]
|
||||
(:key (interior-address analysis subject settings frames)))
|
||||
|
||||
(defn interior-block
|
||||
"Retain pixel measurements before any head-local transform or smoothing."
|
||||
[analysis settings measures]
|
||||
"Retain one subject's pixel measurements before any head-local transform or
|
||||
smoothing."
|
||||
[analysis subject settings measures]
|
||||
(let [frames (count measures)
|
||||
{:keys [key descriptor]}
|
||||
(interior-address analysis settings frames)
|
||||
(interior-address analysis subject settings frames)
|
||||
verts (:teeth-verts settings)
|
||||
stride (+ 2 (* 2 verts))
|
||||
data (js/Float64Array. (* frames stride))]
|
||||
|
|
@ -82,20 +83,28 @@
|
|||
(interior/measure params (:box crop) #js {:data (:data crop)})
|
||||
{:contour nil :contrast 0 :area 0 :debug nil}))
|
||||
|
||||
(defn- named [role analysis tracks layout data]
|
||||
(defn- named [role analysis subject tracks layout data]
|
||||
(merge (address/block {:role role :analysis analysis :params {}
|
||||
:tracks tracks :features [] :observation nil
|
||||
:tracks tracks :features [subject] :observation nil
|
||||
:layout layout})
|
||||
{:role role :data data}))
|
||||
|
||||
(defn pack
|
||||
"Dense landmarks, mask, crops and optional pixel measurements -> source blocks.
|
||||
"One SUBJECT's dense landmarks, mask, crops and optional pixel measurements ->
|
||||
source blocks.
|
||||
|
||||
`:interior-settings` is not optional when `:interior` is present: the block is
|
||||
addressed BY those knobs, and defaulting them here would name a block after
|
||||
settings its bytes did not come from — a content address that lies, which is the
|
||||
one failure the key exists to prevent."
|
||||
[analysis {:keys [dense detected crops interior interior-settings]}]
|
||||
one failure the key exists to prevent.
|
||||
|
||||
`subject` is in every block's descriptor, and it has to be. These four blocks
|
||||
carry no feature — they are the raw analysis, upstream of any part — so two
|
||||
faces tracked in one video would otherwise produce different bytes under one
|
||||
identical key, and the second face's landmarks would replace the first's in the
|
||||
store. That is the same failure `freeze/head-part` guards against one stage
|
||||
later, for the same reason."
|
||||
[analysis subject {:keys [dense detected crops interior interior-settings]}]
|
||||
(let [frames (count dense)
|
||||
points (count (first dense))]
|
||||
(when-not (and (pos? frames) (pos? points)
|
||||
|
|
@ -124,52 +133,66 @@
|
|||
(when-let [crop (nth crops f)]
|
||||
(.set pixels (:data crop) (nth offsets f))))
|
||||
(cond-> {"source/dense"
|
||||
(named "source/dense" analysis ["landmarks"]
|
||||
(named "source/dense" analysis subject ["landmarks"]
|
||||
{:type "float64" :frames frames :points points
|
||||
:stride (* 3 points)}
|
||||
landmarks)
|
||||
"source/detected"
|
||||
(named "source/detected" analysis ["detected"]
|
||||
(named "source/detected" analysis subject ["detected"]
|
||||
{:type "uint8" :frames frames :stride 1} mask)
|
||||
"source/crops"
|
||||
(named "source/crops" analysis ["rgba"]
|
||||
(named "source/crops" analysis subject ["rgba"]
|
||||
{:type "uint8" :frames frames :boxes boxes :offsets offsets}
|
||||
pixels)}
|
||||
interior (assoc "source/interior"
|
||||
(interior-block
|
||||
analysis
|
||||
analysis subject
|
||||
(or interior-settings
|
||||
(throw (ex-info "an interior block needs the settings its measurements were taken at"
|
||||
{:frames (count interior)})))
|
||||
interior))))))
|
||||
|
||||
(defn wire-blocks [blocks]
|
||||
(into-array
|
||||
(map (fn [role]
|
||||
(let [{:keys [key descriptor data]} (get blocks role)]
|
||||
#js {:key key :descriptor descriptor :data (wire/base64 data)}))
|
||||
roles)))
|
||||
(defn pack-subjects
|
||||
"Every tracked subject's source blocks: `{subject {role block}}`.
|
||||
|
||||
(defn upload-blocks [blocks]
|
||||
(into-array
|
||||
(map (fn [role]
|
||||
(let [{:keys [key descriptor data]} (get blocks role)]
|
||||
#js {:key key :descriptor descriptor :data data}))
|
||||
(filter #(contains? blocks %) (conj roles "source/interior")))))
|
||||
One video, one analysis, one set of blocks PER FACE. They are separate blocks
|
||||
and not one block with a subject axis, because a face can be re-measured on its
|
||||
own — a knob dragged on the second person re-reads the second person's
|
||||
landmarks — and because that is what makes each one's content address name
|
||||
exactly the bytes it covers."
|
||||
[analysis by-subject]
|
||||
(into {} (map (fn [[subject inputs] ] [subject (pack analysis subject inputs)]))
|
||||
by-subject))
|
||||
|
||||
(defn unpack
|
||||
"The three block-detail responses -> inputs for measurement and freeze."
|
||||
[^js responses [width height]]
|
||||
(let [by-role (into {}
|
||||
(map (fn [^js response]
|
||||
(let [descriptor (js->clj (js/JSON.parse (.-descriptor response))
|
||||
:keywordize-keys true)]
|
||||
[(:role descriptor)
|
||||
{:layout (:layout descriptor)
|
||||
:data (wire/typed (get-in descriptor [:layout :type])
|
||||
(.-data response))}]))
|
||||
(array-seq responses)))
|
||||
_ (when-not (= (set roles) (set (keys by-role)))
|
||||
(defn- flatten-blocks
|
||||
"`{subject {role block}}` -> one flat seq, in a stable order."
|
||||
[by-subject wanted]
|
||||
(for [[_ blocks] (sort-by (comp str key) by-subject)
|
||||
role wanted
|
||||
:when (contains? blocks role)]
|
||||
(get blocks role)))
|
||||
|
||||
(defn block-keys
|
||||
"The keys of every subject's source blocks, in the order they upload."
|
||||
[by-subject]
|
||||
(mapv :key (flatten-blocks by-subject roles)))
|
||||
|
||||
(defn wire-blocks [by-subject]
|
||||
(into-array
|
||||
(map (fn [{:keys [key descriptor data]}]
|
||||
#js {:key key :descriptor descriptor :data (wire/base64 data)})
|
||||
(flatten-blocks by-subject roles))))
|
||||
|
||||
(defn upload-blocks [by-subject]
|
||||
(into-array
|
||||
(map (fn [{:keys [key descriptor data]}]
|
||||
#js {:key key :descriptor descriptor :data data})
|
||||
(flatten-blocks by-subject (conj roles "source/interior")))))
|
||||
|
||||
(defn- unpack-one
|
||||
"One subject's three blocks -> its inputs for measurement and freeze."
|
||||
[by-role [width height]]
|
||||
(let [_ (when-not (= (set roles) (set (keys by-role)))
|
||||
(throw (ex-info "analysis is missing source blocks" {:roles (keys by-role)})))
|
||||
{:keys [layout data]} (get by-role "source/dense")
|
||||
frames (:frames layout)
|
||||
|
|
@ -206,3 +229,29 @@
|
|||
{:dense dense :detected detected :crops crops :dimensions [width height]
|
||||
:missing (count (remove true? detected))
|
||||
:first-real (first (keep-indexed (fn [i present?] (when present? i)) detected))})))
|
||||
|
||||
(defn unpack
|
||||
"Every subject's block-detail responses -> `{:subjects {id inputs}}`.
|
||||
|
||||
WHICH SUBJECT A BLOCK BELONGS TO IS READ OFF ITS DESCRIPTOR, not off the order
|
||||
the server returned it in. The descriptor is the thing the key is a hash of, so
|
||||
it is the only statement about a block that cannot have drifted from its bytes."
|
||||
[^js responses dimensions]
|
||||
(let [parsed (map (fn [^js response]
|
||||
(let [descriptor (js->clj (js/JSON.parse (.-descriptor response))
|
||||
:keywordize-keys true)]
|
||||
{:subject (keyword (first (:features descriptor)))
|
||||
:role (:role descriptor)
|
||||
:block {:layout (:layout descriptor)
|
||||
:data (wire/typed (get-in descriptor [:layout :type])
|
||||
(.-data response))}}))
|
||||
(array-seq responses))]
|
||||
(when (some (comp nil? :subject) parsed)
|
||||
(throw (ex-info "a source block does not say which subject it came from"
|
||||
{:roles (mapv :role parsed)})))
|
||||
{:subjects
|
||||
(into {}
|
||||
(map (fn [[subject entries]]
|
||||
[subject (unpack-one (into {} (map (juxt :role :block)) entries)
|
||||
dimensions)]))
|
||||
(group-by :subject parsed))}))
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
(ns arthur.flow.take
|
||||
"The shared landmark-to-channel path for synthetic and detected takes."
|
||||
(:require [arthur.flow.address :as address]
|
||||
[arthur.flow.detect :as detect]
|
||||
[arthur.flow.condition :as condition]
|
||||
[arthur.flow.condition.brows :as condition-brows]
|
||||
[arthur.flow.condition.eyes :as condition-eyes]
|
||||
|
|
@ -27,7 +28,7 @@
|
|||
:aspect aspect
|
||||
;; How the detector was run, not just which one it was. See
|
||||
;; `flow/address/analysis-descriptor`.
|
||||
:mode "video"}))))
|
||||
:mode "video" :tracking detect/settings}))))
|
||||
|
||||
(defn measure
|
||||
"Condition the anchor before measuring rings through it."
|
||||
|
|
@ -98,15 +99,28 @@
|
|||
aperture)})
|
||||
(throw (ex-info "unknown measured feature type" {:area area}))))))
|
||||
|
||||
(defn subjects
|
||||
"Per-subject source landmarks -> per-subject conditioned measurements.
|
||||
|
||||
Every tracked face in one shot goes through the same measurement path and is
|
||||
conditioned on its OWN anchor fit, because the fit maps a face onto its own
|
||||
mean pose: sharing one would register the second face against the first one's
|
||||
head. What they share is the video, and that is already expressed by them
|
||||
sharing the analysis record."
|
||||
[params by-subject]
|
||||
(into {} (map (fn [[id inputs]] [id (measure params inputs)])) by-subject))
|
||||
|
||||
(defn build
|
||||
[params inputs]
|
||||
(freeze/clip params (measure params inputs)))
|
||||
"`by-subject` is subject id -> that face's source tracks, one entry per
|
||||
tracked face."
|
||||
[params by-subject]
|
||||
(freeze/clip params (subjects params by-subject)))
|
||||
|
||||
(defn footage
|
||||
"A real manifest and its detected landmarks through the same measurement and
|
||||
freeze path as the synthetic take. The source cadence stays in :fps; picture
|
||||
sampling is a root time map applied only after this artifact exists."
|
||||
[manifest {:keys [dense detected dimensions interior presence detector]}]
|
||||
[manifest {:keys [dimensions detector subjects]}]
|
||||
(let [[w h] dimensions
|
||||
params (merge knobs
|
||||
{:name (or (:source manifest) "footage")
|
||||
|
|
@ -120,5 +134,4 @@
|
|||
;; these landmarks is the failure this prevents.
|
||||
:analysis (analysis-for (assoc manifest :width w :height h)
|
||||
detector)})]
|
||||
(build params {:dense dense :detected detected :interior interior
|
||||
:presence presence})))
|
||||
(build params subjects)))
|
||||
|
|
|
|||
|
|
@ -24,29 +24,12 @@
|
|||
(defonce ^:private selected-owner (r/atom nil))
|
||||
(defonce ^:private drafts (r/atom {}))
|
||||
|
||||
(defn- instance-label
|
||||
"What to call a symbol instance in the UI.
|
||||
|
||||
Its `:name`, because a placement's id is a UUID and a uuid is not something to
|
||||
show anyone or to sort by. The id is the fallback only so that a node with no
|
||||
name still has a row rather than a blank one."
|
||||
[clip id]
|
||||
(or (get-in clip [:timelines :main :nodes id :name]) (str id)))
|
||||
|
||||
(defn- setting-owners [clip]
|
||||
(let [instances (->> (get-in clip [:timelines :main :nodes])
|
||||
(filter (comp #{:symbol} :kind val))
|
||||
;; By LABEL, not by id: ordering by a uuid is ordering at
|
||||
;; random, and this list is read top to bottom by a person.
|
||||
(sort-by (fn [[id _]] [(instance-label clip id) (str id)]))
|
||||
(mapv key))
|
||||
instances (if (seq instances) instances [nil])]
|
||||
(vec (for [instance instances
|
||||
[scope ids] [[:subject (:subjects clip)]
|
||||
[:feature (:features clip)]
|
||||
[:group (:groups clip)]]
|
||||
id (sort-by str (keys ids))]
|
||||
[scope id instance]))))
|
||||
(vec (for [[scope objects] [[:subject (:subjects clip)]
|
||||
[:feature (:features clip)]
|
||||
[:group (:groups clip)]]
|
||||
id (sort-by str (keys objects))]
|
||||
[scope id])))
|
||||
|
||||
(defn- slider-bounds [knob {:keys [type default min even?] upper :max}]
|
||||
{:min (or min (if (= knob :blob-grow) -3 0))
|
||||
|
|
@ -61,7 +44,7 @@
|
|||
busy? (:busy? @(rf/subscribe [::sub/project]))
|
||||
report @(rf/subscribe [::project/regeneration])
|
||||
owners (setting-owners clip)
|
||||
[scope id instance :as owner] (if (some #{(deref selected-owner)} owners)
|
||||
[scope id :as owner] (if (some #{(deref selected-owner)} owners)
|
||||
@selected-owner (first owners))
|
||||
area (case scope
|
||||
:subject :subject
|
||||
|
|
@ -84,12 +67,8 @@
|
|||
(if (seq owners)
|
||||
(doall (for [[i [kind object-id]] (map-indexed vector owners)]
|
||||
^{:key i} [:option {:value i}
|
||||
(str (when-let [instance (nth (nth owners i) 2)]
|
||||
(str (instance-label clip instance) " / "))
|
||||
(name kind) " · " (name object-id))]))
|
||||
(str (name kind) " · " (subs (str object-id) 1))]))
|
||||
[:option {:value 0} "no tracked objects"])] ]
|
||||
(when instance
|
||||
[:div.shared-note "shared symbol · changes all seven stage instances"])
|
||||
(when area
|
||||
[:div.control-list
|
||||
(doall
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
(ns arthur.domain.feature-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[arthur.domain.clip :as clip]
|
||||
[arthur.domain.feature :as feature]
|
||||
[arthur.domain.params :as params]))
|
||||
|
||||
|
|
@ -12,16 +11,22 @@
|
|||
{:subjects (into {} (map (fn [id] [id {:id id :params {}}]) people))
|
||||
:features (into {} (map-indexed (fn [i id]
|
||||
[id {:id id :subject (nth people (quot i 2))
|
||||
:timeline (nth people (quot i 2))
|
||||
:area :eye :nodes [] :params {}}]) eyes))
|
||||
:groups (into {} (map-indexed (fn [i ids]
|
||||
(let [id (keyword (str "pair-" i))]
|
||||
[id {:id id :kind :eye-pair
|
||||
:subject (nth people i)
|
||||
:members ids :params {}}])) members))
|
||||
:timelines {:main {:id :main :frames 1 :nodes {}}}}))
|
||||
:timelines (into {} (map (fn [id]
|
||||
[id {:id id :frames 1
|
||||
:nodes {:head {:id :head :kind :group :z "a1"
|
||||
:measured {[:xform :rot]
|
||||
{:animated? false :value 0.0}}}}}]))
|
||||
people)}))
|
||||
|
||||
(defn- problems [c]
|
||||
(feature/problems c (clip/nodes c)))
|
||||
(feature/problems c))
|
||||
|
||||
(deftest five-people-can-have-nine-identified-eyes
|
||||
(let [scene (nine-eyes)]
|
||||
|
|
|
|||
|
|
@ -40,16 +40,17 @@
|
|||
(is (contains? ls "clip/c7/timeline/main"))
|
||||
(is (= {:frames 229} (get ls "clip/c7/timeline/main"))
|
||||
"a timeline's leaf is its frame space; :fps is the clip's")
|
||||
(is (contains? ls "clip/c7/timeline/main/node/mouth"))
|
||||
(is (contains? ls "clip/c7/timeline/main/channel/mouth/geom.pts"))
|
||||
(is (contains? ls "clip/c7/timeline/main/channel/mouth-in/vis"))
|
||||
(is (contains? ls "clip/c7/feature/eye-r"))
|
||||
(is (contains? ls "clip/c7/group/eyes-1"))
|
||||
;; Nodes are local to the face timeline; feature and group ids are clip-wide.
|
||||
(is (contains? ls "clip/c7/timeline/face-1/node/mouth"))
|
||||
(is (contains? ls "clip/c7/timeline/face-1/channel/mouth/geom.pts"))
|
||||
(is (contains? ls "clip/c7/timeline/face-1/channel/mouth-in/vis"))
|
||||
(is (contains? ls "clip/c7/feature/face-1~eye-r"))
|
||||
(is (contains? ls "clip/c7/group/face-1~eyes"))
|
||||
(is (contains? ls "clip/c7/subject/face-1"))
|
||||
;; `:head`'s measured channels are written together by a freeze and replaced
|
||||
;; A head's measured channels are written together by a freeze and replaced
|
||||
;; together by a re-freeze, so they are one leaf and not three.
|
||||
(is (contains? ls "clip/c7/timeline/main/measured/head"))
|
||||
(is (= 3 (count (get ls "clip/c7/timeline/main/measured/head"))))
|
||||
(is (contains? ls "clip/c7/timeline/face-1/measured/head"))
|
||||
(is (= 3 (count (get ls "clip/c7/timeline/face-1/measured/head"))))
|
||||
;; :frames is NOT in `timing` any more. A timeline is a frame space and a clip
|
||||
;; is a rate, so the one leaf that held both was the persistence half of the
|
||||
;; conflation `domain/clip` exists to undo.
|
||||
|
|
@ -59,11 +60,11 @@
|
|||
;; The boundary that lets two people key different parts without meeting. A node
|
||||
;; leaf carries structure and no geometry.
|
||||
(let [ls (leaf/leaves :c1 @take/clip)
|
||||
n (get ls "clip/c1/timeline/main/node/mouth")]
|
||||
(is (= {:id :mouth :name "mouth" :kind :poly :parent :head :z "a1"
|
||||
:pose-group :mouth} n))
|
||||
n (get ls "clip/c1/timeline/face-1/node/mouth")]
|
||||
(is (= {:id :mouth :name "mouth" :kind :poly :parent :head
|
||||
:z "a1" :pose-group :mouth} n))
|
||||
(is (nil? (:channels n)))
|
||||
(is (:animated? (get ls "clip/c1/timeline/main/channel/mouth/geom.pts")))))
|
||||
(is (:animated? (get ls "clip/c1/timeline/face-1/channel/mouth/geom.pts")))))
|
||||
|
||||
(deftest a-field-with-no-leaf-is-refused-rather-than-dropped
|
||||
;; The invariant that keeps the round trip exact as the model grows: a field
|
||||
|
|
@ -154,7 +155,7 @@
|
|||
(is (empty? (leaf/problems (leaf/leaves :c1 @take/locked)))))
|
||||
|
||||
(deftest a-channel-leaf-for-a-node-that-is-not-there-is-named
|
||||
(let [ls (dissoc (leaf/leaves :c1 @take/clip) "clip/c1/timeline/main/node/mouth")]
|
||||
(let [ls (dissoc (leaf/leaves :c1 @take/clip) "clip/c1/timeline/face-1/node/mouth")]
|
||||
(is (some #(re-find #"node with no node leaf" %) (leaf/problems ls)))))
|
||||
|
||||
(deftest a-node-leaf-is-scoped-to-its-own-timeline
|
||||
|
|
@ -166,7 +167,7 @@
|
|||
(assoc "clip/c1/timeline/sym~blink" {:frames 3}
|
||||
"clip/c1/timeline/sym~blink/node/mouth"
|
||||
{:id :mouth :kind :poly :parent nil :z "a1"})
|
||||
(dissoc "clip/c1/timeline/main/node/mouth"))]
|
||||
(dissoc "clip/c1/timeline/face-1/node/mouth"))]
|
||||
(is (some #(re-find #"node with no node leaf" %) (leaf/problems ls)))))
|
||||
|
||||
(deftest a-property-with-path-punctuation-in-it-is-refused
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@
|
|||
[arthur.flow.freeze :as freeze]
|
||||
[arthur.support.ops :as ops]))
|
||||
|
||||
(defn- face-timeline [c] (clip/timeline c :face-1))
|
||||
|
||||
(defn- wired
|
||||
"A clip out and back, over a wire that is really only JSON."
|
||||
[cid clip]
|
||||
|
|
@ -57,8 +59,8 @@
|
|||
"spec vs playback, after" [ops/specified ops/resolved]}]
|
||||
(doseq [[label [f g]] paths
|
||||
[order fs] (ops/orders n)]
|
||||
(let [a (f (clip/root (:clip @before)) (:store @before))
|
||||
b (g (clip/root (:clip @after)) (:store @after))]
|
||||
(let [a (f (face-timeline (:clip @before)) (:store @before))
|
||||
b (g (face-timeline (:clip @after)) (:store @after))]
|
||||
(testing (str label ", " order)
|
||||
(doseq [frame fs]
|
||||
(is (= (a frame) (b frame))
|
||||
|
|
@ -82,8 +84,8 @@
|
|||
;; everything above and lose the locked take's identity transform.
|
||||
(let [locked {:clip @take/locked :store @take/store}
|
||||
back (wired :c1 locked)
|
||||
a (ops/resolved (clip/root (:clip locked)) (:store locked))
|
||||
b (ops/resolved (clip/root (:clip back)) (:store back))]
|
||||
a (ops/resolved (face-timeline (:clip locked)) (:store locked))
|
||||
b (ops/resolved (face-timeline (:clip back)) (:store back))]
|
||||
(is (= (:clip locked) (:clip back)))
|
||||
(doseq [frame (range 0 take/frames 7)]
|
||||
(is (= (a frame) (b frame)) (str "frame " frame)))))
|
||||
|
|
@ -104,17 +106,17 @@
|
|||
|
||||
(def ^:private gappy
|
||||
(delay (freeze/clip (assoc take/params :name "gappy")
|
||||
(assoc @take/measured
|
||||
:presence
|
||||
(into {} (map (fn [[id gap]]
|
||||
[id (mapv #(not (contains? gap %))
|
||||
(range take/frames))]))
|
||||
windows)))))
|
||||
{:face-1 (assoc @take/measured
|
||||
:presence
|
||||
(into {} (map (fn [[id gap]]
|
||||
[id (mapv #(not (contains? gap %))
|
||||
(range take/frames))]))
|
||||
windows))})))
|
||||
|
||||
(deftest an-absence-mask-survives-the-wire
|
||||
(let [back (wired :c1 @gappy)
|
||||
at (fn [entry id path f]
|
||||
(ch/value-at (get-in (clip/nodes (:clip entry)) [id :channels path])
|
||||
(ch/value-at (get-in (:nodes (face-timeline (:clip entry))) [id :channels path])
|
||||
f (:store entry)))
|
||||
;; Every dense track of the eye, iris, brow and brow-position blocks, and
|
||||
;; the feature whose gap it must follow — the same table
|
||||
|
|
@ -145,7 +147,7 @@
|
|||
;; frame rather than hidden, and its partner is not.
|
||||
(let [back (wired :c1 @gappy)
|
||||
drawn (into #{} (map :node)
|
||||
((timeline/resolver (clip/root (:clip back)) (:store back)) 12))]
|
||||
((timeline/resolver (face-timeline (:clip back)) (:store back)) 12))]
|
||||
(is (not (contains? drawn :eye-r)))
|
||||
(is (contains? drawn :eye-l))
|
||||
(is (contains? drawn :mouth))))
|
||||
|
|
|
|||
|
|
@ -109,9 +109,10 @@
|
|||
(deftest a-three-pixel-pupil-is-three-by-three-at-every-centre
|
||||
;; A square pupil is only worth having if it is the SAME square every frame:
|
||||
;; exactly its nominal size at any centre, or it breathes as the gaze moves.
|
||||
(doseq [[cx cy] [[20 20] [20.5 20.5] [20.49 19.51] [21 20] [20.9 20.1]]]
|
||||
(doseq [[cx cy] [[20 20] [20.5 20.5] [20.49 19.51] [21 20] [20.9 20.1]]
|
||||
size [2.6 3 3.4]]
|
||||
(let [ras (-> (r/make 40 40) (r/clear! 0))]
|
||||
(r/fill-rect! ras cx cy 3 1)
|
||||
(r/fill-rect! ras cx cy size 1)
|
||||
(let [[x0 x1 y0 y1 n] (bbox ras 1)]
|
||||
(is (= [3 3 9] [(inc (- x1 x0)) (inc (- y1 y0)) n])
|
||||
(str "centre " cx "," cy " gave " (inc (- x1 x0)) "x" (inc (- y1 y0)) ":" n))))))
|
||||
|
|
|
|||
|
|
@ -244,7 +244,7 @@
|
|||
|
||||
;; ---- discs and rects ----
|
||||
|
||||
(deftest a-discs-radius-takes-the-mean-scale-and-a-rects-size-is-rounded
|
||||
(deftest disc-and-rect-extents-retain-precision-for-enclosing-instances
|
||||
(let [s (sc {:id :g :kind :group :z "a1"
|
||||
:channels {[:xform :pos] (ch/framed [50 60]) [:xform :scale] (ch/framed [2 2])}}
|
||||
{:id :d :kind :disc :parent :g :z "a1"
|
||||
|
|
@ -253,9 +253,7 @@
|
|||
:channels {[:geom :size] (ch/framed 1.7) [:style :color] (ch/framed :pupil)}})
|
||||
[d r] (timeline/eval-frame s 0)]
|
||||
(is (= [50 60 6] [(:cx d) (:cy d) (:r d)]))
|
||||
;; 1.7 x 2 is 3.4, and a block 3.4px wide would be 3px on one frame and 4 on
|
||||
;; the next, which reads as the pupil breathing.
|
||||
(is (= 3 (:size r)))))
|
||||
(is (= 3.4 (:size r)))))
|
||||
|
||||
;; ---- the fast path and the specification agree ----
|
||||
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@
|
|||
:seed 3 :frames frames :fps (:fps p)
|
||||
:aspect (:aspect p)}
|
||||
(:detector overrides))))]
|
||||
(freeze/clip p (take/measure p {:dense @dense-track}))))
|
||||
(freeze/clip p {:face-1 (take/measure p {:dense @dense-track})})))
|
||||
|
||||
(defn- bytes-of [{:keys [data state]}]
|
||||
(str (sha/of-bytes (js/Uint8Array. (.-buffer data)))
|
||||
|
|
@ -228,9 +228,10 @@
|
|||
(assoc (params-at {}) :analysis
|
||||
(address/analysis {:detector "synth" :version "mulberry32"
|
||||
:seed 3 :frames frames :fps fps :aspect 1}))
|
||||
(assoc (take/measure (params-at {}) {:dense @dense-track})
|
||||
:presence {:brow-r (mapv #(not (contains? gap %))
|
||||
(range frames))})))]
|
||||
{:face-1 (assoc (take/measure (params-at {}) {:dense @dense-track})
|
||||
:presence
|
||||
{:brow-r (mapv #(not (contains? gap %))
|
||||
(range frames))})}))]
|
||||
(is (not= (:key (get @base "brows")) (:key (get with "brows"))))
|
||||
(is (not= (:key (get @base "brow-pos")) (:key (get with "brow-pos"))))
|
||||
(doseq [role ["geom" "eyes" "iris-pos" "head-pos" "head-rot" "head-scale"]]
|
||||
|
|
|
|||
|
|
@ -48,18 +48,18 @@
|
|||
(let [presence (ingest/feature-presence take/frames {:eye-r [[10 14]]})
|
||||
{:keys [clip store]} (flow-take/build
|
||||
(assoc take/params :aspect 1 :name "observed-gap")
|
||||
{:dense @take/analysis :presence presence})
|
||||
{:face-1 {:dense @take/analysis :presence presence}})
|
||||
sample (fn [id frame]
|
||||
(ch/value-at (get-in (clip/nodes clip) [id :channels [:geom :pts]])
|
||||
(ch/value-at (get-in (:nodes (clip/timeline clip :face-1)) [id :channels [:geom :pts]])
|
||||
frame store))]
|
||||
(is (empty? (clip/problems clip)))
|
||||
(is (= [:eye-r :eye-l] (get-in clip [:groups :eyes-1 :members])))
|
||||
(is (= [:face-1/eye-r :face-1/eye-l] (get-in clip [:groups :face-1/eyes :members])))
|
||||
(doseq [f (range 9 14)]
|
||||
(is (ch/nothing? (sample :eye-r f)))
|
||||
(is (not (ch/nothing? (sample :eye-l f))))
|
||||
(is (not (ch/nothing? (sample :mouth f)))))
|
||||
(let [drawn (into #{} (map :node)
|
||||
((timeline/resolver (clip/root clip) store pal/index-of) 11))]
|
||||
((timeline/resolver (clip/timeline clip :face-1) store pal/index-of) 11))]
|
||||
(is (not (contains? drawn :eye-r)))
|
||||
(is (not (contains? drawn :iris-r)))
|
||||
(is (contains? drawn :eye-l))
|
||||
|
|
|
|||
|
|
@ -30,13 +30,14 @@
|
|||
;; that has to be right.
|
||||
(def frozen (delay @take/frozen))
|
||||
(def clip* (delay (:clip @frozen)))
|
||||
;; The ROOT TIMELINE, which is what an evaluator takes. `clip*` is the document —
|
||||
;; `:fps`, the stage, the analysis, the tracking identities — and `tl*` is the bag
|
||||
;; of nodes in its frame space. `timeline/resolver` refuses the wrong one loudly.
|
||||
(def tl* (delay (clip/root @clip*)))
|
||||
;; Geometry assertions read the face timeline. Rendering assertions resolve
|
||||
;; the whole clip, including placement and inherited exposure.
|
||||
(defn- face-timeline [c] (clip/timeline c :face-1))
|
||||
(defn- nodes [c] (merge (clip/nodes c) (:nodes (face-timeline c))))
|
||||
(def tl* (delay (face-timeline @clip*)))
|
||||
(def store (delay (:store @frozen)))
|
||||
|
||||
(defn- node [id] (get-in @tl* [:nodes id]))
|
||||
(defn- node [id] (get (nodes @clip*) id))
|
||||
(defn- chan [id path] (get-in (node id) [:channels path]))
|
||||
|
||||
(defn- block-of
|
||||
|
|
@ -57,12 +58,11 @@
|
|||
|
||||
(defn- render
|
||||
"One frame of a CLIP into a byte buffer. The stage's size comes off the clip and
|
||||
the ops off its root timeline, because project dimensions are the project's and
|
||||
not the footage's — and not a timeline's either."
|
||||
the ops from the complete clip, including nested face timelines."
|
||||
[c f]
|
||||
(let [r (raster/make (:width c) (:height c))]
|
||||
(raster/clear! r (get pal/index-of :bg))
|
||||
(raster/draw-ops! r (ops-at (clip/root c) f))
|
||||
(raster/draw-ops! r ((clip/resolver c @store pal/index-of) f))
|
||||
(vec (array-seq (:buf r)))))
|
||||
|
||||
(defn- drawn
|
||||
|
|
@ -86,16 +86,11 @@
|
|||
(is (empty? (clip/problems c)) (pr-str (clip/problems c)))))
|
||||
|
||||
(deftest the-tree-is-the-one-the-model-specifies
|
||||
;; :face is AUTHORED and :head is MEASURED, and they are two nodes because two
|
||||
;; different things want that transform. A group node is free; keeping the
|
||||
;; hand-placed and the measured transform apart is the whole reason the
|
||||
;; transform is decomposed in the first place.
|
||||
(is (= [:face :root] (timeline/lineage (:nodes @tl*) :face)))
|
||||
(is (= [:head :face :root] (timeline/lineage (:nodes @tl*) :head)))
|
||||
(is (= [:mouth :head :face :root] (timeline/lineage (:nodes @tl*) :mouth)))
|
||||
(is (= [:mouth-in :mouth :head :face :root]
|
||||
(timeline/lineage (:nodes @tl*) :mouth-in)))
|
||||
;; Exposure lives on the clip root and inherits strictly.
|
||||
(is (= [:face :root] (timeline/lineage (clip/nodes @clip*) :face)))
|
||||
(is (= :face-1 (get-in @clip* [:timelines :main :nodes :face-1 :of])))
|
||||
(is (= [:head] (timeline/lineage (:nodes @tl*) :head)))
|
||||
(is (= [:mouth :head] (timeline/lineage (:nodes @tl*) :mouth)))
|
||||
(is (= [:mouth-in :mouth :head] (timeline/lineage (:nodes @tl*) :mouth-in)))
|
||||
(is (= {:mode :map :expose 2} (:time (node :root))))
|
||||
(is (every? #(nil? (:time (node %))) [:face :head :mouth :mouth-in])))
|
||||
|
||||
|
|
@ -158,8 +153,9 @@
|
|||
(is (thrown-with-msg?
|
||||
ExceptionInfo #"does not fit the block's fixed point"
|
||||
(freeze/clip (assoc take/params :name "huge")
|
||||
(update @take/measured :outer
|
||||
(fn [rings] (mapv (fn [r] (mapv #(update % :x + 3) r)) rings)))))))
|
||||
{:face-1 (update @take/measured :outer
|
||||
(fn [rings]
|
||||
(mapv (fn [r] (mapv #(update % :x + 3) r)) rings)))}))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the anchor: three channels, and the inverse
|
||||
|
|
@ -199,14 +195,14 @@
|
|||
;; checking arithmetic against itself; this checks `node/local!`, `node/world!`
|
||||
;; and `emit` as well.
|
||||
(let [c (freeze/head-mode {:mode :free} @frozen)
|
||||
res (timeline/resolver (clip/root c) @store pal/index-of)
|
||||
res (clip/resolver c @store pal/index-of)
|
||||
k (first (:value (chan :face [:xform :scale])))
|
||||
anc (:value (chan :face [:xform :anchor]))
|
||||
pos (:value (chan :face [:xform :pos]))
|
||||
tfs (:transforms @take/measured)]
|
||||
(doseq [f (range 0 take/frames 13)]
|
||||
(let [ops (res f)
|
||||
op (first (filter #(= :mouth (:node %)) ops))
|
||||
op (first (filter #(= [:face-1 :mouth] (:node %)) ops))
|
||||
;; EXPOSURE FIRST. The clip root is on 2s and exposure inherits
|
||||
;; strictly, so frame 13 shows frame 12's pose — which is also the
|
||||
;; cheapest place to assert that the grid is actually being applied,
|
||||
|
|
@ -243,23 +239,23 @@
|
|||
one (freeze/head-mode {:mode :anchored :anchors {0 12}} @frozen)
|
||||
keyed (freeze/head-mode {:mode :anchored
|
||||
:anchors {0 12, 40 88, 150 150}} @frozen)
|
||||
of (fn [c path] (get-in (clip/nodes c) [:head :channels path]))]
|
||||
of (fn [c path] (get-in (nodes c) [:head :channels path]))]
|
||||
(doseq [c [free one keyed]
|
||||
path [[:xform :pos] [:xform :rot] [:xform :scale]]]
|
||||
(is (= :dense (ch/describe (of c path))))
|
||||
(is (= (of free path) (of c path)) "anchor edits do not copy measurements"))
|
||||
(is (nil? (get-in (clip/nodes free) [:head :anchors])))
|
||||
(is (= {0 12} (get-in (clip/nodes one) [:head :anchors])))
|
||||
(is (nil? (get-in (nodes free) [:head :anchors])))
|
||||
(is (= {0 12} (get-in (nodes one) [:head :anchors])))
|
||||
(is (= {0 12, 40 88, 150 150}
|
||||
(get-in (clip/nodes keyed) [:head :anchors])))
|
||||
(get-in (nodes keyed) [:head :anchors])))
|
||||
(is (= keyed (leaf/clip "head" (leaf/leaves "head" keyed)))
|
||||
"anchor source addresses survive the document round trip")))
|
||||
|
||||
(deftest head-anchor-keys-hold-the-whole-measured-transform
|
||||
(let [free (timeline/resolver (clip/root
|
||||
(let [free (timeline/resolver (face-timeline
|
||||
(freeze/head-mode {:mode :free} @frozen))
|
||||
@store pal/index-of)
|
||||
held (timeline/resolver (clip/root
|
||||
held (timeline/resolver (face-timeline
|
||||
(freeze/head-mode {:mode :anchored
|
||||
:anchors {0 12, 40 88}} @frozen))
|
||||
@store pal/index-of)
|
||||
|
|
@ -279,14 +275,14 @@
|
|||
b (freeze/head-mode {:mode :anchored :anchors {0 0}} @frozen)
|
||||
c (freeze/head-mode {:mode :anchored :anchors {0 0, 40 40}} @frozen)]
|
||||
(doseq [x [b c]]
|
||||
(is (= (get (clip/nodes a) :face) (get (clip/nodes x) :face))
|
||||
(is (= (get (nodes a) :face) (get (nodes x) :face))
|
||||
":face moved")
|
||||
(is (= (dissoc (clip/nodes a) :head) (dissoc (clip/nodes x) :head))
|
||||
(is (= (dissoc (nodes a) :head) (dissoc (nodes x) :head))
|
||||
"a node other than :head changed")
|
||||
;; The measurement does not go away when the head is locked: always measure,
|
||||
;; always store factored, toggle the parent.
|
||||
(is (= (get-in (clip/nodes a) [:head :measured])
|
||||
(get-in (clip/nodes x) [:head :measured])))
|
||||
(is (= (get-in (nodes a) [:head :measured])
|
||||
(get-in (nodes x) [:head :measured])))
|
||||
;; And nothing above the timeline moved either: the toggle is one node's
|
||||
;; channels, so the clip's own fields and its other timelines are untouched.
|
||||
(is (= (dissoc a :timelines) (dissoc x :timelines))))))
|
||||
|
|
@ -331,8 +327,8 @@
|
|||
;; asking for a different stage moves and rescales the same geometry rather than
|
||||
;; re-measuring anything.
|
||||
(let [big (freeze/clip (assoc take/params :stage [640 480] :name "big")
|
||||
@take/measured)
|
||||
key-of (fn [c] (:store (:dense (get-in (clip/nodes c)
|
||||
{:face-1 @take/measured})
|
||||
key-of (fn [c] (:store (:dense (get-in (nodes c)
|
||||
[:mouth :channels [:geom :pts]]))))]
|
||||
(is (= [640 480] [(:width (:clip big)) (:height (:clip big))]))
|
||||
;; Stronger than it was, and for free: the stage is not an input to tier 2, so
|
||||
|
|
@ -344,7 +340,7 @@
|
|||
(is (= (vec (array-seq (:data (get @store (key-of @clip*)))))
|
||||
(vec (array-seq (:data (get (:store big) (key-of (:clip big)))))))
|
||||
"the geometry is the same numbers at either stage size")
|
||||
(is (not= (:value (get-in (clip/nodes (:clip big))
|
||||
(is (not= (:value (get-in (nodes (:clip big))
|
||||
[:face :channels [:xform :scale]]))
|
||||
(:value (chan :face [:xform :scale]))))))
|
||||
|
||||
|
|
@ -422,14 +418,14 @@
|
|||
(let [gap (set (range 40 60))
|
||||
presence {:eye-r (mapv #(not (contains? gap %)) (range take/frames))}
|
||||
c (freeze/clip (assoc take/params :name "one-eye-gappy")
|
||||
(assoc @take/measured :presence presence))
|
||||
tl (clip/root (:clip c))
|
||||
{:face-1 (assoc @take/measured :presence presence)})
|
||||
tl (face-timeline (:clip c))
|
||||
at (fn [id f]
|
||||
(ch/value-at (get-in (:nodes tl) [id :channels [:geom :pts]]) f (:store c)))]
|
||||
;; The identities are the CLIP's, and a gap does not touch them: a feature keeps
|
||||
;; its id and its pair membership across the frames it was not observed on.
|
||||
(is (= [:eye-r :eye-l] (get-in (:clip c) [:groups :eyes-1 :members])))
|
||||
(is (= :eye-r (get-in (:clip c) [:features :eye-r :id])))
|
||||
(is (= [:face-1/eye-r :face-1/eye-l] (get-in (:clip c) [:groups :face-1/eyes :members])))
|
||||
(is (= :face-1/eye-r (get-in (:clip c) [:features :face-1/eye-r :id])))
|
||||
(is (not (ch/nothing? (at :eye-r 39))))
|
||||
(is (ch/nothing? (at :eye-r 50)))
|
||||
(is (not (ch/nothing? (at :eye-r 60))))
|
||||
|
|
@ -460,8 +456,8 @@
|
|||
(range take/frames))]))
|
||||
windows)
|
||||
c (freeze/clip (assoc take/params :name "per-track-gaps")
|
||||
(assoc @take/measured :presence presence))
|
||||
tl (clip/root (:clip c))
|
||||
{:face-1 (assoc @take/measured :presence presence)})
|
||||
tl (face-timeline (:clip c))
|
||||
at (fn [id path f]
|
||||
(ch/value-at (get-in (:nodes tl) [id :channels path]) f (:store c)))
|
||||
;; Node, channel, and the feature whose gap it must follow. Every dense
|
||||
|
|
@ -501,7 +497,8 @@
|
|||
:shown (vec (repeat take/frames true))}
|
||||
clip (fn [nm extra]
|
||||
(freeze/clip (assoc take/params :name nm)
|
||||
(merge (assoc @take/measured :teeth teeth-in) extra)))
|
||||
{:face-1 (merge (assoc @take/measured :teeth teeth-in)
|
||||
extra)}))
|
||||
ref (clip "teeth-reference" nil)
|
||||
occ (clip "teeth-mouth-gap"
|
||||
{:presence {:mouth (mapv #(not (contains? gap %))
|
||||
|
|
@ -509,7 +506,7 @@
|
|||
;; Hoisted: the resolver caches its order and reuses its buffers, so the
|
||||
;; node ids come out before the next frame is asked for.
|
||||
nodes-at (fn [c]
|
||||
(let [r (timeline/resolver (clip/root (:clip c)) (:store c) pal/index-of)]
|
||||
(let [r (timeline/resolver (face-timeline (:clip c)) (:store c) pal/index-of)]
|
||||
(fn [f] (into #{} (map :node) (r f)))))
|
||||
ref-at (nodes-at ref)
|
||||
occ-at (nodes-at occ)
|
||||
|
|
@ -519,7 +516,7 @@
|
|||
open (filter #(contains? (ref-at %) :teeth) (range take/frames))
|
||||
inside (first (filter gap open))
|
||||
outside (first (remove gap open))]
|
||||
(is (= :mouth-in (get-in (clip/nodes (:clip occ)) [:teeth :stencil]))
|
||||
(is (= :mouth-in (get-in (nodes (:clip occ)) [:teeth :stencil]))
|
||||
"teeth stop inheriting the mouth's absence if this stops being their stencil")
|
||||
(is (some? inside) "no open-mouth frame inside the gap to test with")
|
||||
(is (some? outside) "no open-mouth frame outside the gap to test with")
|
||||
|
|
@ -527,7 +524,7 @@
|
|||
;; Annotating the MOUTH sets no bit on the teeth: they are a feature of
|
||||
;; their own and nothing named them.
|
||||
(is (not (ch/nothing?
|
||||
(ch/value-at (get-in (clip/nodes (:clip occ)) [:teeth :channels [:geom :pts]])
|
||||
(ch/value-at (get-in (nodes (:clip occ)) [:teeth :channels [:geom :pts]])
|
||||
inside (:store occ)))))
|
||||
;; They are dropped anyway — :mouth-in drew nothing to clip them against.
|
||||
(is (not (contains? (occ-at inside) :mouth-in)))
|
||||
|
|
@ -544,8 +541,8 @@
|
|||
(let [gap (set (range 40 60))
|
||||
det (mapv #(not (contains? gap %)) (range take/frames))
|
||||
c (freeze/clip (assoc take/params :name "gappy")
|
||||
(assoc @take/measured :detected det))
|
||||
tl (clip/root (:clip c))
|
||||
{:face-1 (assoc @take/measured :detected det)})
|
||||
tl (face-timeline (:clip c))
|
||||
res (timeline/resolver tl (:store c) pal/index-of)]
|
||||
(doseq [f [39 40 50 59 60]]
|
||||
(let [ops (res f)]
|
||||
|
|
@ -604,7 +601,8 @@
|
|||
(let [locked (freeze/head-mode {:mode :anchored :anchors {0 0}} @frozen)
|
||||
shot (fn [f]
|
||||
(let [r (raster/make W H)
|
||||
mouth (filter #(= :mouth (:node %)) (ops-at (clip/root locked) f))]
|
||||
mouth (filter #(= [:face-1 :mouth] (:node %))
|
||||
((clip/resolver locked @store pal/index-of) f))]
|
||||
(raster/clear! r (get pal/index-of :bg))
|
||||
(raster/draw-ops! r mouth)
|
||||
(vec (array-seq (:buf r)))))
|
||||
|
|
|
|||
190
frontend/test/arthur/flow/multi_face_test.cljs
Normal file
190
frontend/test/arthur/flow/multi_face_test.cljs
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
(ns arthur.flow.multi-face-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[arthur.demo.stage :as stage]
|
||||
[arthur.domain.channel :as ch]
|
||||
[arthur.domain.clip :as clip]
|
||||
[arthur.domain.feature :as feature]
|
||||
[arthur.domain.pose :as pose]
|
||||
[arthur.domain.project :as project]
|
||||
[arthur.events.footage :as footage]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.detect :as detect]
|
||||
[arthur.flow.freeze :as freeze]
|
||||
[arthur.flow.regenerate :as regenerate]
|
||||
[arthur.flow.source :as source]
|
||||
[arthur.flow.take :as take]
|
||||
[arthur.support.ops :as ops]
|
||||
[arthur.synth :as synth]))
|
||||
|
||||
(def frames 40)
|
||||
(def settings
|
||||
(merge take/knobs {:name "two faces" :fps 30 :aspect 1 :stage [320 200]
|
||||
:fit-motion? true :expose 1 :head :anchored :anchors {0 0}
|
||||
:analysis (address/analysis {:detector "synth" :version "two-faces-v1"
|
||||
:seed 9 :frames frames :fps 30 :aspect 1})}))
|
||||
(def inputs
|
||||
(delay
|
||||
(into {} (for [[id seed dx] [[:face-1 9 -0.3] [:face-2 12 0.3]]]
|
||||
[id {:dense (mapv (fn [face] (mapv #(update % :x + dx) face))
|
||||
(synth/synth-dense frames {:seed seed}))
|
||||
:detected (vec (repeat frames true))
|
||||
:crops (vec (repeat frames nil))}]))))
|
||||
(def initial
|
||||
(delay (assoc (take/build settings @inputs) :source-inputs {:subjects @inputs})))
|
||||
|
||||
(defn channel [entry subject node path]
|
||||
(get-in entry [:clip :timelines subject :nodes node :channels path]))
|
||||
(defn snapshot [c store f] (ops/snapshot ((clip/resolver c store) f)))
|
||||
(defn by-node [c store f] (into {} (map (juxt :node identity)) (snapshot c store f)))
|
||||
|
||||
(deftest subjects-share-local-names-without-sharing-blocks
|
||||
(let [{:keys [clip store]} @initial
|
||||
a (get-in clip [:timelines :face-1 :nodes])
|
||||
b (get-in clip [:timelines :face-2 :nodes])]
|
||||
(is (empty? (clip/problems clip)))
|
||||
(is (= (set (keys a)) (set (keys b))))
|
||||
(is (= :head (get-in b [:mouth :parent])))
|
||||
(is (= :eye-r-in (get-in b [:iris-r :stencil])))
|
||||
(is (= :mouth (get-in b [:mouth :pose-group])))
|
||||
(is (= :face-2 (get-in clip [:features :face-2/mouth :timeline])))
|
||||
(doseq [path [[:xform :pos] [:xform :rot] [:xform :scale]]]
|
||||
(is (not= (:store (:dense (get-in a [:head :measured path])))
|
||||
(:store (:dense (get-in b [:head :measured path]))))))
|
||||
(let [drawn (by-node clip store 12)
|
||||
left (get drawn [:face-1 :mouth])
|
||||
right (get drawn [:face-2 :mouth])]
|
||||
(is (seq (:points left)))
|
||||
(is (< (ffirst (:points left)) (ffirst (:points right)))
|
||||
"one shared placement preserves the filmed separation"))
|
||||
(is (some #(re-find #"more than one feature" %)
|
||||
(feature/problems
|
||||
(assoc-in clip [:features :duplicate]
|
||||
(assoc (get-in clip [:features :face-2/mouth]) :id :duplicate)))))))
|
||||
|
||||
(deftest one-subjects-edit-and-anchor-do-not-change-its-neighbor
|
||||
(let [before @initial
|
||||
at [:clip :timelines :face-2 :nodes :iris-r :channels [:style :color]]
|
||||
before (assoc-in before at (ch/framed :brow))
|
||||
after (regenerate/change before {:scope :feature :id :face-2/eye-r
|
||||
:knob :gaze-gain :value 2})
|
||||
anchored (freeze/head-mode {:subject :face-2 :mode :anchored :anchors {0 12}} after)]
|
||||
(is (= (get-in before at) (get-in after at)) "authored channels survive regeneration")
|
||||
(is (= (get-in before [:clip :timelines :face-1])
|
||||
(get-in after [:clip :timelines :face-1])
|
||||
(get-in anchored [:timelines :face-1])))
|
||||
(is (not= (channel before :face-2 :iris-r [:xform :pos])
|
||||
(channel after :face-2 :iris-r [:xform :pos])))
|
||||
(is (= (channel before :face-2 :iris-l [:xform :pos])
|
||||
(channel after :face-2 :iris-l [:xform :pos])))
|
||||
(is (= {0 12} (get-in anchored [:timelines :face-2 :nodes :head :anchors])))
|
||||
(is (empty? (clip/problems anchored)))))
|
||||
|
||||
(deftest the-second-subject-regenerates-inside-a-composed-stage
|
||||
(let [before (update @initial :clip stage/compose)
|
||||
after (regenerate/change before {:scope :subject :id :face-2
|
||||
:knob :anchor-avg :value 4})
|
||||
full (take/build (assoc settings :anchor-avg 4) @inputs)]
|
||||
(is (= (get-in full [:clip :timelines :face-2 :nodes :head :measured])
|
||||
(get-in after [:clip :timelines :face-2 :nodes :head :measured])))
|
||||
(doseq [tid [:main :sym/face-8625 :face-1]]
|
||||
(is (= (get-in before [:clip :timelines tid])
|
||||
(get-in after [:clip :timelines tid]))))
|
||||
(is (empty? (clip/problems (:clip after))))
|
||||
(is (thrown? ExceptionInfo
|
||||
(freeze/head-mode {:subject :face-2 :mode :anchored :anchors {0 frames}}
|
||||
after))
|
||||
"anchors are bounded by the source timeline, even on a longer stage")))
|
||||
|
||||
(deftest an-instance-pose-cut-only-holds-that-faces-mouth
|
||||
(let [{:keys [clip store]} @initial
|
||||
cut (pose/put-cut clip :face-2 :mouth 0 20)
|
||||
original (by-node clip store 4)
|
||||
held (by-node cut store 4)
|
||||
source (by-node clip store 20)]
|
||||
(is (empty? (clip/problems cut)))
|
||||
(is (= (get held [:face-2 :mouth]) (get source [:face-2 :mouth])))
|
||||
(is (not= (get held [:face-2 :mouth]) (get original [:face-2 :mouth])))
|
||||
(doseq [id [[:face-1 :mouth] [:face-1 :eye-r] [:face-2 :eye-r]]]
|
||||
(is (= (get held id) (get original id))))))
|
||||
|
||||
(deftest detection-and-feature-gaps-are-per-subject
|
||||
(let [inputs (-> @inputs
|
||||
(assoc-in [:face-2 :detected 8] false)
|
||||
(assoc-in [:face-2 :presence :eye-r]
|
||||
(assoc (vec (repeat frames true)) 12 false)))
|
||||
{:keys [clip store]} (take/build settings inputs)
|
||||
clip (freeze/head-mode {:mode :free} {:clip clip})
|
||||
at8 (by-node clip store 8)
|
||||
at12 (by-node clip store 12)]
|
||||
(is (some? (get at8 [:face-1 :mouth])))
|
||||
(is (nil? (get at8 [:face-2 :mouth])))
|
||||
(is (nil? (get at12 [:face-2 :eye-r])))
|
||||
(is (nil? (get at12 [:face-2 :iris-r])))
|
||||
(is (some? (get at12 [:face-1 :eye-r])))
|
||||
(is (some? (get at12 [:face-2 :eye-l])))
|
||||
(is (some? (get (by-node clip store 13) [:face-2 :eye-r])))))
|
||||
|
||||
(deftest nested-faces-survive-save-and-resolve-through-the-stage
|
||||
(let [entry (update @initial :clip stage/compose)
|
||||
back (project/load "two" (js/JSON.parse (js/JSON.stringify (project/save "two" entry))))]
|
||||
(is (= (:clip entry) (:clip back)))
|
||||
(is (empty? (clip/problems (:clip back))))
|
||||
(doseq [f [0 12 30 10]]
|
||||
(let [before (snapshot (:clip entry) (:store entry) f)
|
||||
after (snapshot (:clip back) (:store back) f)]
|
||||
(is (seq before))
|
||||
(is (some #(= [:face-2 :mouth] (second (:node %))) before)))
|
||||
(is (= (snapshot (:clip entry) (:store entry) f)
|
||||
(snapshot (:clip back) (:store back) f))))))
|
||||
|
||||
(deftest source-round-trip-keeps-two-faces-with-identical-detection-masks
|
||||
(let [blocks (source/pack-subjects (get-in settings [:analysis :id]) @inputs)
|
||||
wire (source/wire-blocks blocks)
|
||||
back (:subjects (source/unpack (.reverse wire) [320 200]))]
|
||||
(is (= 6 (count (set (source/block-keys blocks)))) )
|
||||
(doseq [subject [:face-1 :face-2]]
|
||||
(is (= (select-keys (get @inputs subject) [:dense :detected])
|
||||
(select-keys (get back subject) [:dense :detected]))))))
|
||||
|
||||
(deftest reordered-detections-and-gaps-retain-their-track
|
||||
(let [a [{:x 0.1 :y 0.5}] b [{:x 0.8 :y 0.5}]
|
||||
frames [[a] [b a] [] [a b]]
|
||||
tracks (detect/tracks frames)]
|
||||
(is (= {:face-1 [0 1 nil 0] :face-2 [nil 0 nil 1]} tracks))
|
||||
(is (= [false true false true]
|
||||
(:detected (detect/fill-gaps (detect/pick frames (:face-2 tracks))))))
|
||||
(is (thrown? ExceptionInfo (detect/tracks [[] []])))))
|
||||
|
||||
(deftest manifest-masks-enter-measurement-with-local-feature-names
|
||||
(let [manifest {:presence {:eye-l [false true]
|
||||
:face-1/eye-r [true false]
|
||||
:face-2/eye-r [false true]}}]
|
||||
(is (= {:eye-l [false true] :eye-r [true false]}
|
||||
(footage/presence-for manifest :face-1)))
|
||||
(is (= {:eye-r [false true]} (footage/presence-for manifest :face-2)))))
|
||||
|
||||
(deftest tracking-settings-participate-in-the-analysis-address
|
||||
(let [manifest {:width 320 :height 200 :frames 40 :fps 30 :source "source"}
|
||||
analysis (take/analysis-for manifest {:version "test"})]
|
||||
(is (= detect/settings (:tracking analysis)))
|
||||
(is (not= (:id analysis) (:id (address/analysis (dissoc analysis :tracking)))))
|
||||
(is (not= (:id analysis)
|
||||
(:id (address/analysis (assoc-in analysis [:tracking :gate] 0.3)))))))
|
||||
|
||||
(deftest the-instance-boundary-preserves-the-single-face-geometry
|
||||
(let [{:keys [clip store]} (take/build settings (select-keys @inputs [:face-1]))
|
||||
flat (assoc-in clip [:timelines :main :nodes]
|
||||
(merge (dissoc (clip/nodes clip) :face-1)
|
||||
(assoc-in (get-in clip [:timelines :face-1 :nodes])
|
||||
[:head :parent] :face)))
|
||||
nested (clip/resolver clip store)
|
||||
reference (clip/resolver flat store)]
|
||||
(doseq [f [0 1 7 20 39]]
|
||||
(let [a (ops/snapshot (nested f)) b (ops/snapshot (reference f))]
|
||||
(is (= (mapv (comp second :node) a) (mapv :node b)))
|
||||
(doseq [[x y] (map vector a b)]
|
||||
(is (= (:kind x) (:kind y)))
|
||||
(doseq [[u v] (map vector
|
||||
(concat (mapcat identity (:points x)) (keep x [:cx :cy :r :size]))
|
||||
(concat (mapcat identity (:points y)) (keep y [:cx :cy :r :size])))]
|
||||
(is (< (abs (- u v)) 1e-8))))))))
|
||||
|
|
@ -21,7 +21,7 @@
|
|||
:analysis (address/analysis
|
||||
{:detector "synth" :version "mulberry32"
|
||||
:seed 9 :frames frames :fps 30 :aspect 1})})]
|
||||
(assoc (take/build p @inputs) :source-inputs @inputs))))
|
||||
(assoc (take/build p {:face-1 @inputs}) :source-inputs {:subjects {:face-1 @inputs}}))))
|
||||
|
||||
(defn- full-at [change]
|
||||
(take/build
|
||||
|
|
@ -30,15 +30,15 @@
|
|||
:expose 1 :head :free
|
||||
:analysis (get-in @initial [:clip :analysis])}
|
||||
change)
|
||||
@inputs))
|
||||
{:face-1 @inputs}))
|
||||
|
||||
(defn- channel [entry node path]
|
||||
(get-in entry [:clip :timelines :main :nodes node :channels path]))
|
||||
(get-in entry [:clip :timelines :face-1 :nodes node :channels path]))
|
||||
|
||||
(deftest eye-rebuild-is-confined-to-the-edited-feature
|
||||
(let [before @initial
|
||||
after (regenerate/change before
|
||||
{:scope :feature :id :eye-r :knob :gaze-gain :value 2})]
|
||||
{:scope :feature :id :face-1/eye-r :knob :gaze-gain :value 2})]
|
||||
(is (not= (channel before :iris-r [:xform :pos])
|
||||
(channel after :iris-r [:xform :pos])))
|
||||
(is (= (channel before :iris-l [:xform :pos])
|
||||
|
|
@ -53,7 +53,7 @@
|
|||
(deftest framed-eye-size-needs-no-new-dense-block
|
||||
(let [before @initial
|
||||
after (regenerate/change before
|
||||
{:scope :feature :id :eye-r :knob :iris-size :value 0.6})]
|
||||
{:scope :feature :id :face-1/eye-r :knob :iris-size :value 0.6})]
|
||||
(is (not= (channel before :iris-r [:geom :radius])
|
||||
(channel after :iris-r [:geom :radius])))
|
||||
(is (= (channel before :iris-l [:geom :radius])
|
||||
|
|
@ -65,11 +65,11 @@
|
|||
(deftest mouth-edit-leaves-the-head-and-eye-alone
|
||||
(let [before @initial
|
||||
after (regenerate/change before
|
||||
{:scope :feature :id :mouth :knob :verts :value 10})]
|
||||
{:scope :feature :id :face-1/mouth :knob :verts :value 10})]
|
||||
(is (not= (channel before :mouth [:geom :pts])
|
||||
(channel after :mouth [:geom :pts])))
|
||||
(is (= (get-in before [:clip :timelines :main :nodes :head])
|
||||
(get-in after [:clip :timelines :main :nodes :head])))
|
||||
(is (= (get-in before [:clip :timelines :face-1 :nodes :head])
|
||||
(get-in after [:clip :timelines :face-1 :nodes :head])))
|
||||
(is (= (channel before :eye-r [:geom :pts])
|
||||
(channel after :eye-r [:geom :pts])))
|
||||
(is (= (channel (full-at {:verts 10}) :mouth [:geom :pts])
|
||||
|
|
@ -80,10 +80,10 @@
|
|||
after (regenerate/change before
|
||||
{:scope :subject :id :face-1 :knob :anchor-avg :value 4})
|
||||
full (full-at {:anchor-avg 4})]
|
||||
(is (not= (get-in before [:clip :timelines :main :nodes :head :measured])
|
||||
(get-in after [:clip :timelines :main :nodes :head :measured])))
|
||||
(is (= (get-in full [:clip :timelines :main :nodes :head :measured])
|
||||
(get-in after [:clip :timelines :main :nodes :head :measured])))
|
||||
(is (not= (get-in before [:clip :timelines :face-1 :nodes :head :measured])
|
||||
(get-in after [:clip :timelines :face-1 :nodes :head :measured])))
|
||||
(is (= (get-in full [:clip :timelines :face-1 :nodes :head :measured])
|
||||
(get-in after [:clip :timelines :face-1 :nodes :head :measured])))
|
||||
(is (= (get-in before [:clip :timelines :main :nodes :face])
|
||||
(get-in after [:clip :timelines :main :nodes :face])))))
|
||||
|
||||
|
|
@ -91,14 +91,14 @@
|
|||
(let [before @initial
|
||||
after (regenerate/change before
|
||||
{:scope :subject :id :face-1 :knob :contour-avg :value 3})]
|
||||
(is (= (get-in before [:clip :timelines :main :nodes :head])
|
||||
(get-in after [:clip :timelines :main :nodes :head])))
|
||||
(is (= (get-in before [:clip :timelines :face-1 :nodes :head])
|
||||
(get-in after [:clip :timelines :face-1 :nodes :head])))
|
||||
(is (= (channel before :teeth [:geom :pts])
|
||||
(channel after :teeth [:geom :pts])))))
|
||||
|
||||
(deftest edited-take-still-round-trips-through-normal-save
|
||||
(let [after (regenerate/change @initial
|
||||
{:scope :group :id :eyes-1 :knob :iris-size :value 0.6})
|
||||
{:scope :group :id :face-1/eyes :knob :iris-size :value 0.6})
|
||||
wire (project/save "regen" after)
|
||||
loaded (project/load "regen" (js/JSON.parse (js/JSON.stringify wire)))]
|
||||
(is (= (:clip after) (:clip loaded)))
|
||||
|
|
@ -172,7 +172,7 @@
|
|||
{:fps 30 :aspect 1 :analysis (get-in @initial [:clip :analysis])}
|
||||
overrides)
|
||||
inputs (assoc @inputs :interior @interior-track)]
|
||||
(freeze/part area p (take/measure-part area p inputs (take/anchor-base p inputs)))))
|
||||
(freeze/part :face-1 area p (take/measure-part area p inputs (take/anchor-base p inputs)))))
|
||||
|
||||
(def ^:private swept
|
||||
"Per area, the knobs whose bytes or channels that area's freeze could plausibly
|
||||
|
|
@ -202,22 +202,9 @@
|
|||
|
||||
;; ---- the stage: a shared symbol behind many placements ----
|
||||
|
||||
(defn- staged
|
||||
"The take composed onto the 8625 stage: its timeline becomes the shared symbol,
|
||||
and seven uuid-keyed placements sit on `:main`.
|
||||
(defn- staged [] (update @initial :clip stage/compose))
|
||||
|
||||
This path had no coverage, and it is the one that differs structurally: `change`
|
||||
re-roots the SYMBOL as the take, edits that, and puts it back, so every
|
||||
assumption `change-take` makes about `:main` holding the tracked nodes is only
|
||||
true of the re-rooted document and not of the stage itself."
|
||||
[]
|
||||
;; The ENTRY with its clip composed, not the composed clip: `change` takes an
|
||||
;; entry (`:clip`, `:store`, `:source-inputs`) and the store is what the
|
||||
;; regenerated blocks merge into.
|
||||
(update @initial :clip stage/compose))
|
||||
|
||||
(defn- sym-channel [entry node path]
|
||||
(get-in entry [:clip :timelines :sym/face-8625 :nodes node :channels path]))
|
||||
(defn- sym-channel [entry node path] (channel entry node path))
|
||||
|
||||
(deftest a-stage-edit-is-previewable-at-all
|
||||
;; The guard `events/project/::preview-settings` bails on, stated here so a
|
||||
|
|
@ -228,11 +215,11 @@
|
|||
"the composed stage keeps the analysis the edit needs")
|
||||
(is (some? (:source-inputs entry)))
|
||||
(testing "and the features still say which timeline they live in"
|
||||
(is (every? #(= :sym/face-8625 (:timeline %))
|
||||
(is (every? #(= :face-1 (:timeline %))
|
||||
(vals (:features (:clip entry))))))))
|
||||
|
||||
(deftest a-stage-edit-plans-the-same-features-as-a-take-edit
|
||||
(let [edit {:scope :feature :id :eye-r :knob :iris-size :value 0.6}]
|
||||
(let [edit {:scope :feature :id :face-1/eye-r :knob :iris-size :value 0.6}]
|
||||
(is (= (:features (regenerate/plan (:clip @initial) edit))
|
||||
(:features (regenerate/plan (:clip (staged)) edit)))
|
||||
"the same knob dirties the same features on a stage as on a take")))
|
||||
|
|
@ -243,7 +230,7 @@
|
|||
;; and `:main` — which holds only placements — must come back untouched.
|
||||
(let [before (staged)
|
||||
after (regenerate/change before
|
||||
{:scope :feature :id :eye-r :knob :iris-size :value 0.6})]
|
||||
{:scope :feature :id :face-1/eye-r :knob :iris-size :value 0.6})]
|
||||
(is (not= (sym-channel before :iris-r [:geom :radius])
|
||||
(sym-channel after :iris-r [:geom :radius]))
|
||||
"the shared drawing is what changed")
|
||||
|
|
@ -261,7 +248,7 @@
|
|||
;; faces off the stage, or dangle the voice links, while looking like a
|
||||
;; successful edit of the drawing.
|
||||
(let [after (regenerate/change (staged)
|
||||
{:scope :feature :id :eye-r :knob :iris-size :value 0.6})
|
||||
{:scope :feature :id :face-1/eye-r :knob :iris-size :value 0.6})
|
||||
nodes (get-in after [:clip :timelines :main :nodes])
|
||||
syms (filter (comp #{:symbol} :kind val) nodes)]
|
||||
(is (= 7 (count syms)))
|
||||
|
|
|
|||
|
|
@ -10,14 +10,14 @@
|
|||
(range (:teeth-verts settings)))
|
||||
measures [{:contour ring :contrast 0.7 :area 42}
|
||||
{:contour nil :contrast 0 :area 0}]
|
||||
block (source/interior-block "sha256:analysis" settings measures)
|
||||
block (source/interior-block "sha256:analysis" :face-1 settings measures)
|
||||
response #js {:descriptor (:descriptor block)
|
||||
:data (wire/base64 (:data block))}]
|
||||
(is (= (:key block)
|
||||
(source/interior-key "sha256:analysis" settings 2)))
|
||||
(source/interior-key "sha256:analysis" :face-1 settings 2)))
|
||||
(is (= measures (source/unpack-interior response settings 2)))
|
||||
(is (not= (:key block)
|
||||
(source/interior-key "sha256:analysis"
|
||||
(source/interior-key "sha256:analysis" :face-1
|
||||
(assoc settings :cavity-erode 0.2) 2)))))
|
||||
|
||||
(deftest source-blocks-round-trip-without-source-images
|
||||
|
|
@ -25,8 +25,9 @@
|
|||
pixels (js/Uint8ClampedArray. #js [12 24 36 255])
|
||||
input {:dense [face face] :detected [true false]
|
||||
:crops [{:box {:x 3 :y 4 :w 1 :h 1} :data pixels} nil]}
|
||||
blocks (source/pack "sha256:analysis" input)
|
||||
out (source/unpack (source/wire-blocks blocks) [100 80])]
|
||||
blocks (source/pack "sha256:analysis" :face-1 input)
|
||||
out (get-in (source/unpack (source/wire-blocks {:face-1 blocks}) [100 80])
|
||||
[:subjects :face-1])]
|
||||
(is (= (set source/roles) (set (keys blocks))))
|
||||
(is (= (:dense input) (:dense out)))
|
||||
(is (= (:detected input) (:detected out)))
|
||||
|
|
@ -36,20 +37,20 @@
|
|||
(vec (array-seq (:data (first (:crops out)))))))
|
||||
(is (nil? (second (:crops out))))
|
||||
(is (= (mapv :key (vals blocks))
|
||||
(mapv :key (vals (source/pack "sha256:analysis" out)))))))
|
||||
(mapv :key (vals (source/pack "sha256:analysis" :face-1 out)))))))
|
||||
|
||||
(deftest fresh-source-includes-its-pixel-measurement-block
|
||||
(let [face (vec (repeat 478 {:x 0.25 :y 0.5 :z -0.125}))
|
||||
input {:dense [face] :detected [false] :crops [nil]
|
||||
:interior [{:contour nil :contrast 0 :area 0}]
|
||||
:interior-settings params/defaults}
|
||||
blocks (source/pack "sha256:analysis" input)]
|
||||
blocks (source/pack "sha256:analysis" :face-1 input)]
|
||||
(is (contains? blocks "source/interior"))
|
||||
(is (= 4 (.-length (source/upload-blocks blocks))))
|
||||
(is (= 4 (.-length (source/upload-blocks {:face-1 blocks}))))
|
||||
(is (= 3 (count source/roles)))
|
||||
(testing "and cannot be packed without the settings it was measured at"
|
||||
;; The block is addressed BY those knobs. Defaulting them would name it
|
||||
;; after settings its bytes did not come from, which is a key that lies.
|
||||
(is (thrown-with-msg?
|
||||
ExceptionInfo #"needs the settings"
|
||||
(source/pack "sha256:analysis" (dissoc input :interior-settings)))))))
|
||||
(source/pack "sha256:analysis" :face-1 (dissoc input :interior-settings)))))))
|
||||
|
|
|
|||
|
|
@ -254,6 +254,7 @@ async function main() {
|
|||
await sleep(120);
|
||||
const open = await page.eval(PROBE);
|
||||
await page.shot('take-open');
|
||||
check(open.toneSet.includes(0x171a22), 'nested face pupils reach the rasterizer');
|
||||
check(open.toneSet.includes(MOUTH_DARK),
|
||||
'an open mouth draws an outline and an interior',
|
||||
`${open.tones} tones, interior ${open.toneSet.includes(MOUTH_DARK) ? 'present' : 'MISSING'}`);
|
||||
|
|
@ -493,13 +494,9 @@ async function main() {
|
|||
check(stageLoaded !== null, 'the 8625 stage is ready', stageLoaded ?? (await page.eval(STATUS)));
|
||||
const eyeSelected = await page.eval(`(() => {
|
||||
const select = document.querySelector('.controls select');
|
||||
// The instance prefix is the placement's NAME ("8625 left"), not its id:
|
||||
// a placement is keyed by a uuid now, and a uuid is not something to show
|
||||
// anyone. So this matches the feature and requires SOME instance prefix,
|
||||
// rather than pinning the label a rename is free to change.
|
||||
// Settings belong to tracked features shared by the stage placements.
|
||||
const option = [...select.options]
|
||||
.find(o => o.textContent.trim().endsWith('/ feature · eye-r')
|
||||
&& o.textContent.includes(' / '));
|
||||
.find(o => o.textContent.trim().endsWith('feature · face-1/eye-r'));
|
||||
if (!option) return false;
|
||||
select.value = option.value;
|
||||
select.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
|
|
@ -533,7 +530,7 @@ async function main() {
|
|||
const preview = await statusMatching(/preview · unsaved/, 160);
|
||||
const debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`);
|
||||
check(preview !== null, 'the slider updates the stage preview', preview ?? debug);
|
||||
check(debug.includes(':eye-r') && debug.includes('tier 1 only'),
|
||||
check(debug.includes(':face-1/eye-r') && debug.includes('tier 1 only'),
|
||||
'the panel reports the affected feature and tier', debug);
|
||||
const endFrame = await page.eval(`Number(document.querySelector('.readout span').textContent.match(/\\d+/)[0])`);
|
||||
check(endFrame > startFrame, 'playback continues during tuning', `${startFrame} -> ${endFrame}`);
|
||||
|
|
@ -581,7 +578,7 @@ async function main() {
|
|||
if (debug.includes('head-pos')) break;
|
||||
await sleep(250);
|
||||
}
|
||||
check(debug.includes('head-pos') && debug.includes(':teeth'),
|
||||
check(debug.includes('head-pos') && debug.includes(':face-1/teeth'),
|
||||
'anchor invalidates the head and teeth', debug);
|
||||
const preview = await statusMatching(/preview · unsaved/, 160);
|
||||
check(preview !== null, 'the anchor edit finishes previewing',
|
||||
|
|
@ -591,8 +588,7 @@ async function main() {
|
|||
const teethSelected = await page.eval(`(() => {
|
||||
const select = document.querySelector('.controls select');
|
||||
const option = [...select.options]
|
||||
.find(o => o.textContent.trim().endsWith('/ feature · teeth')
|
||||
&& o.textContent.includes(' / '));
|
||||
.find(o => o.textContent.trim().endsWith('feature · face-1/teeth'));
|
||||
if (!option) return false;
|
||||
select.value = option.value;
|
||||
select.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
|
|
@ -627,10 +623,10 @@ async function main() {
|
|||
let debug = '';
|
||||
for (let i = 0; i < 160; i++) {
|
||||
debug = await page.eval(`document.querySelector('.regeneration-debug')?.textContent ?? ''`);
|
||||
if (debug.includes('dirty features: [:teeth]')) break;
|
||||
if (debug.includes('dirty features: [:face-1/teeth]')) break;
|
||||
await sleep(250);
|
||||
}
|
||||
check(debug.includes('dirty features: [:teeth]'),
|
||||
check(debug.includes('dirty features: [:face-1/teeth]'),
|
||||
'the pixel setting invalidates only teeth', debug);
|
||||
const preview = await statusMatching(/preview · unsaved/, 160);
|
||||
check(preview !== null, 'the teeth edit finishes previewing',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue