From 03df81fa0ba569c92912b3f6be13d8e6caa30134 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 24 Sep 2026 18:11:25 -0400 Subject: [PATCH] Brows: traced ring, quantised raise A brow at 320x200 is fourteen pixels wide and three tall. Its shape carries almost nothing at that size; its height above the eye carries the expression, and a brow raise is the most legible beat on a face. So the ring is traced and the height is quantised - the split the eyes already got, where the lid is a traced feature and the iris a quantised primitive. The decomposition is the point. The traced ring already contains the real height, so adding a quantised raise on top would move the brow twice. The height is measured OUT of the ring, quantised, and put back, so the shape that renders is his at a height that snaps between a few levels and holds. Measured at both ends rather than as one number, because raise and tilt are different expressions out of one mechanism: both ends up is surprise, inner up alone is worry, inner down is anger. They share a dwell - the gaze quantiser, renamed quantizeSnap now that it has two callers - so the brow hits its pose in one frame instead of crawling into it with one end arriving before the other. Measured against the eye's corner midpoint, never its lid. Same trap the gaze origin has and worth avoiding twice: brows and lids move together constantly, so a brow that jumped on every blink would read as a tic. Rest pose from the take median rather than the neutral frame, for the reason gaze learned the hard way - that frame is picked by minimum mouth aperture and says nothing about the brows. Two correspondences resolved from geometry, not declared: which ring is which brow, and which end is the outer one. The second matters more - backwards, the tilt mirrors and worry renders as its own opposite, which reads as a directed performance choice rather than a bug and would never be questioned. Which EDGE is upper is deliberately left unresolved: it traverses the same ring the other way, an even-odd fill has no winding, and both ends still land on fixed slots. Also fixes a bug from the exposure work: the live render applied exposure to the plate and the mouth but not to the eyes, so on 2s the preview and the export disagreed. A preview that disagrees with the export is the one bug this tool cannot afford. perfIndex now exists as a named thing so the two paths cannot drift apart again. 91 -> 105 assertions. Ground truth on all four synthetic brow poses, tilt separating worry from anger by sign, a blink not faking a raise, and a shared dwell never emitting a half-raised brow. Co-Authored-By: Claude Opus 5 --- README.md | 45 +++++++++++++++++- docs/design.md | 12 ++++- index.html | 14 +++++- js/app.js | 123 +++++++++++++++++++++++++++++++++++++++++++++--- js/landmarks.js | 31 ++++++++++++ js/pipeline.js | 86 +++++++++++++++++++++++++++++++-- js/selftest.js | 110 +++++++++++++++++++++++++++++++++++++++---- js/synth.js | 24 +++++++++- 8 files changed, 419 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 4cb8dda..009e817 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,47 @@ other way round to prove it actually looks. | iris size | Diameter as a percentage of eye width. | | pupil | Square pupil in whole pixels. 0 = off. | +## Brows + +A brow at 320×200 is about fourteen pixels wide and three tall. Its **shape** +carries almost nothing at that size; its **height above the eye** carries the +expression, and a brow raise is the most legible beat on a face. So the ring is +traced and the height is quantised — the same split the eyes got, where the lid +is a traced feature and the iris a quantised primitive. + +The decomposition matters. The traced ring already contains the real height, so +adding a quantised raise on top would move the brow twice. Instead the height is +measured *out* of the ring, quantised, and put back: the shape that renders is +his, at a height that snaps between a few levels and holds. + +Height is measured at **both ends**, not as one number, because raise and tilt +are different expressions out of one mechanism — both ends up is surprise, inner +up alone is worry, inner down is anger. They share a dwell, so the brow hits its +pose in one frame instead of crawling into it with one end arriving first. + +It is measured against the eye's **corner midpoint**, never its lid — the same +trap the gaze origin has, and worth avoiding twice: brows and lids move together +constantly, so a brow that jumped on every blink would read as a tic. The rest +pose comes from the take **median**, not the neutral frame, for the same reason +gaze does: that frame is chosen by minimum mouth aperture and says nothing +whatever about the brows. + +Two correspondences are resolved from geometry rather than declared: which ring +is which brow, and which end of a ring is the outer one. The second matters more +— get it backwards and the tilt mirrors, so worry renders as its own opposite, +which reads as a directed performance choice and would never be questioned. +Which *edge* of the brow is the upper one is deliberately left unresolved: it +traverses the same ring the other way round, an even-odd fill has no winding, +and the two ends still land on fixed slots either way. + +| Knob | What it does | +| --- | --- | +| brow vertices | Ring vertex budget, off a 10-slot ring. | +| brow weight | Thickens the ring outward. It needs it at three pixels tall. | +| brow raise gain | Exaggerates or damps the raise. | +| brow step | The pixel grid the height snaps to. 0 = off. | +| brow dwell | How long a new height must hold. Shared across both ends. | + ## The plate is reference, not art The plate layer has several representations because its job changes. Cycle with @@ -268,7 +309,7 @@ chromium --headless --virtual-time-budget=8000 --dump-dom \ http://127.0.0.1:8777/selftest.html | grep -oE '(PASS|FAIL) [0-9/]+' ``` -Or open `selftest.html`. 88 assertions over the stages below detection, plus a +Or open `selftest.html`. 105 assertions over the stages below detection, plus a wiring cross-check: every `el('id')` in `app.js` must exist in `index.html`. A knob wired in one but not the other throws during wiring, which aborts the rest of the module and leaves a blank page — a symptom that points nowhere near its @@ -282,7 +323,7 @@ eyeball. ## Not done yet -Brows; hand-drawn head plates and per-plate mouth slots (the strip +Hand-drawn head plates and per-plate mouth slots (the strip decides *which frames need one*, but you cannot yet supply the drawing); real performer→character calibration (currently identity, fitting the face oval to the canvas); the override layer; anything on the Animator Pro side. The plate is a diff --git a/docs/design.md b/docs/design.md index bc7cdbe..9fd57d9 100644 --- a/docs/design.md +++ b/docs/design.md @@ -52,9 +52,17 @@ artist or a fixed authored table. | Kind | Source | Vocabulary | Interp | | --- | --- | --- | --- | | **Plate** — head, hair, body | Hand-drawn | Closed: a few drawings per character | hold | -| **Feature** — mouth, lids | Rotoscoped from landmarks | Open: derived from this take | hold | +| **Feature** — mouth, lids, brows | Rotoscoped from landmarks | Open: derived from this take | hold | | **Interior** — mouth interior, teeth | Image content within a feature | Open | hold | | **Primitive** — iris | Landmark centroid as a disc | Quantised | hold | +| **Scalar** — brow raise, gaze | One number out of a feature | Quantised | hold | + +The last row took the longest to see. A brow is a feature *and* a scalar: the +ring is traced because the shape should be his, but at three pixels tall the +shape carries almost nothing while the height above the eye carries the +expression. So the height is measured out of the traced ring, quantised, and put +back. Extracting the scalar without removing it first would move the part twice, +because the traced ring already contains the height. The asymmetry is deliberate, and it is the opposite choice in each case. @@ -290,7 +298,7 @@ Current modules: - **A paint surface.** The plates have nowhere to be drawn. This is the largest gap between "tool" and "suite": a pixel paint canvas with onion skin, palette constraint, and the registered underlay behind it. -- Brows as parts, and a tongue. +- A tongue. - Plate libraries with per-plate mouth slots. - Real performer→character calibration (currently identity). - The override layer. diff --git a/index.html b/index.html index 5902f86..9df6db8 100644 --- a/index.html +++ b/index.html @@ -102,7 +102,8 @@

source + landmarks

outer lip — · inner lip — · - lids — · iris —
+ lids — · iris — · + brows —

stabilised (head-local)

@@ -161,6 +162,11 @@ + + + + +
mouth lead shifts the performance tracks earlier (positive) against @@ -183,6 +189,12 @@ value carries across takes. blink hold is the minimum length of a blink: a real blink is one frame at 12fps and a single frame of closed eye reads as a dropout, so it is extended to a beat. + brow step and brow dwell quantise the brow's HEIGHT above + the eye, not its shape — the traced ring is his, the height snaps between + a few levels and holds. Both ends move independently, so raise and tilt + come out of one control: both up is surprise, inner up is worry, inner + down is anger. brow weight thickens the ring, which it needs at + three pixels tall.
pupil is a square, in whole pixels, 0 to turn it off: at this size a circle of radius 1.5 is a plus sign with the corners gnawed off and it changes shape as it moves, where a square stays the mark you drew.
diff --git a/js/app.js b/js/app.js index c52fac1..266ca86 100644 --- a/js/app.js +++ b/js/app.js @@ -1,8 +1,10 @@ import { FaceLandmarker, FilesetResolver } from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/vision_bundle.mjs'; import { LIPS_OUTER, LIPS_INNER, FACE_OVAL, - EYE_R_RING, EYE_L_RING, IRIS_A, IRIS_B } from './landmarks.js'; + EYE_R_RING, EYE_L_RING, IRIS_A, IRIS_B, + BROW_A_RING, BROW_B_RING } from './landmarks.js'; import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame, shiftIndex, - exposeIndex, eyeSignals, gazeOrigin, quantizeGaze, resolveBlink } from './pipeline.js'; + exposeIndex, eyeSignals, gazeOrigin, quantizeSnap, resolveBlink, + browSignals } from './pipeline.js'; import { IndexedRaster } from './raster.js'; import { drawRegistered, posterizeInto } from './underlay.js'; import { extractTeeth } from './interior.js'; @@ -31,8 +33,12 @@ const PALETTE = [ // step inside it reads as a hole. { name: 'iris', hex: '#4a5468' }, { name: 'pupil', hex: '#171a22' }, + // Brows get their own entry rather than sharing skin_dark with the lash line. + // They are hair, not shadow: when hair plates exist they want to match those, + // and tying them to the lash means you cannot change one without the other. + { name: 'brow', hex: '#3a2a22' }, ]; -const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, teeth: 4, white: 5, iris: 6, pupil: 7 }; +const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, teeth: 4, white: 5, iris: 6, pupil: 7, brow: 8 }; const state = { dense: null, images: [], stab: null, xform: null, @@ -46,6 +52,7 @@ const state = { interior: null, // per-frame teeth measurement from image content teeth: null, // resolved per-frame {show, t} after knobs eyes: null, // resolved per-frame lid rings, shut flags, iris discs + brows: null, // resolved per-frame brow rings after quantised raise eyeSig: null, // raw eye measurement, kept for the gaze readout }; @@ -72,6 +79,11 @@ const opts = () => ({ apertureThresh: +el('apertureThresh').value / 1000, tol: +el('tol').value / 1000, exposure: +el('exposure').value, + browVerts: +el('browVerts').value, + browWeight: +el('browWeight').value, + browGain: +el('browGain').value / 100, + browStep: +el('browStep').value, + browDwell: +el('browDwell').value, irisAnchor: el('irisAnchor').value, gazeOrigin: el('gazeOrigin').value, eyeVerts: +el('eyeVerts').value, @@ -226,6 +238,7 @@ function rebuild(resetKeep) { } state.teeth = resolveTeeth(o); state.eyes = buildEyes(o); + state.brows = buildBrows(o); // Plate outline per frame, so a kept frame shows its own head shape. state.plates = state.stab.oval.map((r) => r.map(state.xform)); @@ -341,7 +354,7 @@ function buildEyes(o) { x: (g.x - origin.x) * o.gazeGain * w, y: (g.y - origin.y) * o.gazeGain * w, })); - const gaze = quantizeGaze(px, o.gazeStep, o.gazeDwell); + const gaze = quantizeSnap(px, o.gazeStep, o.gazeDwell); const eye = (sk, lids, shut, rad, f) => { const e = sk(f); @@ -371,6 +384,70 @@ function buildEyes(o) { }; } +// Brows: ring traced every frame, HEIGHT quantised. +// +// The decomposition is the point. The traced ring already contains the brow's +// real height, so adding a quantised raise on top would move it twice. Instead +// the height is measured out of the ring, quantised, and put back - the shape +// that renders is his, at a height that snaps between a few authored levels and +// holds. That is the same split the eyes got: lid traced as a feature, iris +// position quantised as a primitive. +// +// Two ends, not one height, warped linearly between them. Raise and tilt are +// different expressions out of one mechanism: both ends up is surprise, inner +// up alone is worry, inner down is anger. +function buildBrows(o) { + const st = state.stab, N = state.dense.length; + const sig = browSignals(st); + state.browSig = sig; + + const ringOf = (side) => (side === 'R' ? sig.pairing.right : sig.pairing.left); + const table = (side) => (ringOf(side) === 'browA' ? BROW_A_RING : BROW_B_RING); + + const build = (side, corners) => { + const rings = smoothContours( + st[ringOf(side)].map((r) => toRasterRing(r, table(side), o.browVerts, state.xform)), + o.contourSmooth); + + // Eye width in raster pixels, so the raise converts from eye widths into the + // units the grid is expressed in and the knob means the same on any framing. + const wpx = (f) => { + const a = state.xform(corners[f][0]), b = state.xform(corners[f][1]); + return Math.hypot(a.x - b.x, a.y - b.y); + }; + const meanW = st.cornersR.reduce((a, _, f) => a + wpx(f), 0) / N; + + // Rest pose from the take MEDIAN, never from the neutral frame. That frame + // is chosen by minimum mouth aperture and says nothing about the brows, and + // the same mistake on the gaze origin re-pointed an entire performance. + const rest = gazeOrigin(sig[side], 'median'); + const px = sig[side].map((g) => ({ + x: (g.x - rest.x) * o.browGain * meanW, + y: (g.y - rest.y) * o.browGain * meanW, + })); + const q = quantizeSnap(px, o.browStep, o.browDwell); + + const frames = rings.map((ring, f) => { + // Raise is measured upward but y grows downward, so a positive raise is a + // negative y offset. + const dOuter = -(q[f].x - px[f].x), dInner = -(q[f].y - px[f].y); + const a = state.xform(corners[f][0]), b = state.xform(corners[f][1]); + const span = b.x - a.x; + const warped = ring.map((p) => { + // Position along the brow's own axis, outer end to inner end. Taken from + // x against the eye corners rather than from ring slots, because + // subsampling does not keep the end slots at any given budget. + const t = span === 0 ? 0 : Math.min(1, Math.max(0, (p.x - a.x) / span)); + return { x: p.x, y: p.y + dOuter + (dInner - dOuter) * t }; + }); + return offsetRing(warped, o.browWeight); + }); + return { frames, px, q }; + }; + + return { R: build('R', st.cornersR), L: build('L', st.cornersL), pairing: sig.pairing }; +} + // Presence gets hysteresis and a minimum dwell, the same treatment plate // selection gets: a teeth block that blinks on and off for single frames is // worse than one that is simply absent. Appearing needs a clear signal, staying @@ -440,7 +517,14 @@ function renderFrame(f, mode = plateMode()) { // device: it exists because a mouth shape anticipates the sound it makes. // Nothing about a blink or a glance is tied to the audio, so shifting the // eyes would only slide them off the head that carries them. - if (state.eyes) drawEyes(r, state.eyes.frames[f]); + // Eyes and brows ride the exposure grid but NOT the mouth lead: the lead is a + // lip-sync device and nothing about a blink or a brow is tied to the audio. + const ef = perfIndex(f); + if (state.eyes) drawEyes(r, state.eyes.frames[ef]); + if (state.brows) { + r.fillPoly(state.brows.R.frames[ef], IDX.brow); + r.fillPoly(state.brows.L.frames[ef], IDX.brow); + } const mf = leadIndex(f); // performance frame, possibly ahead r.fillPoly(state.outer[mf], IDX.dark); // mouth keeps every frame @@ -541,6 +625,12 @@ function leadIndex(f) { // even ones would read as two performances laid over each other. const plateIndex = (f) => heldFrame(keptSorted(), exposeIndex(f, state.exposure)); +// Performance tracks that do not take the mouth lead still ride the grid. This +// existing as a named thing is what stopped the eyes holding on 1s in the +// preview while the export held them on 2s - a preview that disagrees with the +// export is the one bug this tool cannot afford. +const perfIndex = (f) => exposeIndex(f, state.exposure); + function blit(canvas, raster, zoom) { canvas.width = RW * zoom; canvas.height = RH * zoom; canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0); @@ -563,6 +653,7 @@ function drawReadout() { `teeth on ${teethFrames}f · ` + `${blinkRuns(state.eyes.shutR).length}/${blinkRuns(state.eyes.shutL).length} blinks R/L · ` + `${gazeCells(state.eyes.gaze)} gaze cells · ` + + `${gazeCells(state.brows.R.q)} brow poses · ` + (state.exposure > 1 ? `on ${state.exposure}s = ${(state.fps / state.exposure).toFixed(4).replace(/\.?0+$/, '')}fps · ` : '') + @@ -688,6 +779,9 @@ function drawEyeOverlay(g, map, f) { for (const ring of [EYE_R_RING, EYE_L_RING]) { strokePts(g, ring.map((i) => map(lm[i])), '#60a5fa'); } + for (const ring of [BROW_A_RING, BROW_B_RING]) { + strokePts(g, ring.map((i) => map(lm[i])), '#c084fc'); + } if (!state.eyes.hasIris) return; for (const iris of [IRIS_A, IRIS_B]) { strokePts(g, iris.slice(1).map((i) => map(lm[i])), '#fbbf24'); @@ -839,7 +933,7 @@ function eyeParts(grid) { const out = []; // Eyes ride the exposure grid but NOT the mouth lead: the lead is a lip-sync // device and nothing about a blink is tied to the audio. - const src = (f) => exposeIndex(f, state.exposure); + const src = perfIndex; [['r', 20], ['l', 23]].forEach(([side, z]) => { const at = (f) => state.eyes.frames[src(f)][side]; out.push( @@ -871,6 +965,16 @@ function eyeParts(grid) { return out; } +// Brows are a traced ring like the lids, so they keep every frame on the grid. +// The quantised raise is already baked into the points - the renderer is handed +// a polygon, not a shape plus an offset it would have to recombine. +function browParts(grid) { + return [['r', 26, 'R'], ['l', 27, 'L']].map(([name, z, side]) => ({ + name: `brow_${name}`, kind: 'poly', z, color: 'brow', interp: 'hold', + keys: grid.map((f) => ({ f, src: perfIndex(f), pts: state.brows[side].frames[perfIndex(f)] })), + })); +} + function exportTake() { const kept = keptSorted(); const N = state.dense.length; @@ -903,6 +1007,7 @@ function exportTake() { // key stream is dense but the VALUES change only on saccades, so a // hold-interpolating renderer cuts between fixations by itself. ...eyeParts(grid), + ...browParts(grid), { name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in', keys: grid.map((f) => { const m = leadIndex(f), te = state.teeth[m]; @@ -991,6 +1096,9 @@ const FMT = { irisSize: (v) => `${v}%`, gazeGain: (v) => (v / 100).toFixed(2), gazeStep: (v) => (v ? `${v}px` : 'off'), + browStep: (v) => (v ? `${v}px` : 'off'), + browWeight: (v) => `${v}px`, + browGain: (v) => (v / 100).toFixed(2), pupilPx: (v) => (v ? `${v}px` : 'off'), lashPx: (v) => `${v}px`, lead: (v) => (v > 0 ? `+${v}` : String(v)), @@ -1000,7 +1108,8 @@ for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol' 'teethOn', 'teethDwell', 'teethErode', 'tongueReject', 'blobGrow', 'topBias', 'teethVerts', 'teethSmooth', 'lead', 'eyeVerts', 'lashPx', 'irisSize', 'pupilPx', 'gazeGain', - 'gazeStep', 'gazeDwell', 'blinkCut', 'blinkHold', 'blinkDwell']) { + 'gazeStep', 'gazeDwell', 'blinkCut', 'blinkHold', 'blinkDwell', + 'browVerts', 'browWeight', 'browGain', 'browStep', 'browDwell']) { const show = () => { el(id + 'v').textContent = FMT[id] ? FMT[id](+el(id).value) : el(id).value; }; diff --git a/js/landmarks.js b/js/landmarks.js index ee0094b..e8e7b4a 100644 --- a/js/landmarks.js +++ b/js/landmarks.js @@ -95,3 +95,34 @@ export const EYE_L_LIDS = [386, 374]; // the geometry instead. export const IRIS_A = [468, 469, 470, 471, 472]; export const IRIS_B = [473, 474, 475, 476, 477]; + +// ---- brows ---- +// +// Each brow is two five-point chains, an upper edge and a lower edge, which +// close into a ten-point ring: out along one edge from the outer end to the +// inner, back along the other. +// +// WHICH EDGE IS UPPER IS DELIBERATELY NOT DECLARED, and unlike the iris it does +// not need to be. Swapping them traverses the same ring the other way round, +// and an even-odd fill has no winding, so the shape is identical either way. +// What the ring guarantees instead is that the two ENDS land on fixed slots: +// 0 and 9 are one end, 4 and 5 the other. Averaging a pair therefore gives the +// brow's height at that end whichever edge is on top, which is all the raise +// and tilt measurement needs. +// +// Which end is the OUTER one is resolved from geometry in pipeline.js, because +// getting it backwards mirrors the tilt - inner-up "worried" would render as +// outer-up - and that is a expression error, not a glitch, so it would read as +// a directed performance choice rather than as a bug. +export const BROW_A_RING = [ + 70, 63, 105, 66, 107, + 55, 65, 52, 53, 46, +]; +export const BROW_B_RING = [ + 300, 293, 334, 296, 336, + 285, 295, 282, 283, 276, +]; + +// The slots at each end of a brow ring, as pairs to average. +export const BROW_END_0 = [0, 9]; +export const BROW_END_1 = [4, 5]; diff --git a/js/pipeline.js b/js/pipeline.js index e004e72..85bfd30 100644 --- a/js/pipeline.js +++ b/js/pipeline.js @@ -4,7 +4,8 @@ import { RIGID, LIPS_OUTER, LIPS_INNER, APERTURE, FACE_OVAL, EYE_INNER, EYE_R_RING, EYE_L_RING, EYE_R_CORNERS, EYE_L_CORNERS, - EYE_R_LIDS, EYE_L_LIDS, IRIS_A, IRIS_B, subsampleSlots } from './landmarks.js'; + EYE_R_LIDS, EYE_L_LIDS, IRIS_A, IRIS_B, + BROW_A_RING, BROW_B_RING, BROW_END_0, BROW_END_1, subsampleSlots } from './landmarks.js'; import { fitSimilarity, applySimAll, applySim, fitResidual, procrustesMean, smoothTransforms, movingAverage } from './mathutil.js'; // MediaPipe normalises x by image WIDTH and y by image HEIGHT, so its normalised @@ -60,9 +61,81 @@ export function stabilize(dense, smoothRadius, aspect = 1) { lidsR: map(EYE_R_LIDS), lidsL: map(EYE_L_LIDS), irisA: hasIris ? map(IRIS_A) : null, irisB: hasIris ? map(IRIS_B) : null, + browA: map(BROW_A_RING), browB: map(BROW_B_RING), }; } +/* ---------- brows ---------- */ + +// Two correspondences resolved from geometry, for the same reason the iris +// pairing is: a wrong guess here is survivable enough to escape notice. +// +// Which ring is which brow follows MediaPipe's left/right naming, which is the +// naming that would have put the irises on the wrong eyes. Which END of a ring +// is the OUTER one matters more: get it backwards and the tilt mirrors, so +// inner-up "worried" renders as outer-up, which is a different expression +// rather than a broken one. It would read as a directed performance choice and +// never be questioned. +// +// Both are decided by voting across every frame against landmarks already known +// to be rigid, so one bad detection cannot swing them. +export function pairBrows(stab) { + const N = stab.browA.length; + const cen = (ring) => { + let x = 0; + for (const p of ring) x += p.x; + return x / ring.length; + }; + let side = 0, ends = 0; + for (let f = 0; f < N; f++) { + const cR = mid(stab.cornersR[f][0], stab.cornersR[f][1]).x; + const cL = mid(stab.cornersL[f][0], stab.cornersL[f][1]).x; + side += Math.abs(cen(stab.browA[f]) - cR) < Math.abs(cen(stab.browA[f]) - cL) ? 1 : -1; + + // EYE_R_CORNERS is [outer, inner], so this asks whether slot 0 of the ring + // sits nearer the eye's outer corner than its inner one. + const ring = side > 0 ? stab.browA[f] : stab.browB[f]; + const co = side > 0 ? stab.cornersR[f] : stab.cornersL[f]; + const s0 = ring[BROW_END_0[0]]; + ends += Math.abs(s0.x - co[0].x) < Math.abs(s0.x - co[1].x) ? 1 : -1; + } + return { + right: side > 0 ? 'browA' : 'browB', + left: side > 0 ? 'browB' : 'browA', + outerAtSlot0: ends > 0, + }; +} + +// Brow height above its own eye, at each end, in eye widths. +// +// Measured against the eye's CORNER MIDPOINT, not the lid: the corners are +// rigid, so a blink cannot read as a brow raise. That is the same trap the gaze +// origin has and it is worth avoiding twice - brows and lids move together +// constantly, and a brow that jumped on every blink would look like a tic. +// +// Two ends rather than one height, because raise and tilt are different +// expressions built from the same measurement: both ends up is surprise, inner +// up alone is worry, inner down is anger. One number could not tell them apart. +export function browSignals(stab) { + const N = stab.browA.length; + const pairing = pairBrows(stab); + const endOuter = pairing.outerAtSlot0 ? BROW_END_0 : BROW_END_1; + const endInner = pairing.outerAtSlot0 ? BROW_END_1 : BROW_END_0; + const out = { R: [], L: [], pairing }; + + for (let f = 0; f < N; f++) { + for (const [side, corners] of [['R', stab.cornersR], ['L', stab.cornersL]]) { + const ring = stab[pairing[side === 'R' ? 'right' : 'left']][f]; + const c = mid(corners[f][0], corners[f][1]); + const w = dist(corners[f][0], corners[f][1]); + const at = (pair) => (ring[pair[0]].y + ring[pair[1]].y) / 2; + // y grows downward, so a brow ABOVE the eye gives a positive raise. + out[side].push({ x: (c.y - at(endOuter)) / w, y: (c.y - at(endInner)) / w }); + } + } + return out; +} + /* ---------- eyes ---------- */ const mid = (a, b) => ({ x: (a.x + b.x) / 2, y: (a.y + b.y) / 2 }); @@ -200,7 +273,10 @@ export function gazeOrigin(gazeRaw, mode = 'median', neutral = 0, radius = 2) { return { x: mid1(gazeRaw.map((g) => g.x)), y: mid1(gazeRaw.map((g) => g.y)) }; } -// Snap gaze onto a grid, then require a new cell to hold before it takes. +// Snap a two-channel track onto a grid, then require a new cell to hold before +// it takes. Gaze uses it for (x, y); brows use it for (outer raise, inner raise), +// where sharing the dwell is the point - a brow whose inner end arrived a frame +// before its outer end would crawl instead of snapping. // // This is the "Primitive - quantised" row of the part table in docs/design.md, // and it is not a stylisation imposed on the truth: real eyes move in saccades, @@ -212,9 +288,9 @@ export function gazeOrigin(gazeRaw, mode = 'median', neutral = 0, radius = 2) { // The dwell is what stops a gaze parked on a cell boundary from chattering // between two cells forever. It is meaningless without a grid, because // continuous values never repeat, so step 0 short-circuits both. -export function quantizeGaze(gaze, step, dwell) { - if (!(step > 0)) return gaze.map((g) => ({ x: g.x, y: g.y })); - const q = gaze.map((g) => ({ +export function quantizeSnap(track, step, dwell) { + if (!(step > 0)) return track.map((g) => ({ x: g.x, y: g.y })); + const q = track.map((g) => ({ x: Math.round(g.x / step) * step, y: Math.round(g.y / step) * step, })); diff --git a/js/selftest.js b/js/selftest.js index ed58aa4..3ddf456 100644 --- a/js/selftest.js +++ b/js/selftest.js @@ -9,10 +9,12 @@ import { LIPS_OUTER, LIPS_INNER, FACE_OVAL, RIGID, subsampleSlots, subsampleRing, EYE_R_RING, EYE_L_RING, EYE_R_CORNERS, EYE_L_CORNERS, - EYE_R_LIDS, EYE_L_LIDS } from './landmarks.js'; + EYE_R_LIDS, EYE_L_LIDS, BROW_A_RING, BROW_B_RING, + BROW_END_0, BROW_END_1 } from './landmarks.js'; import { fitSimilarity, applySim, procrustesMean, smoothTransforms, offsetRing } from './mathutil.js'; import { stabilize, toRasterRing, smoothContours, selectKeys, activeKey, shiftIndex, - exposeIndex, eyeSignals, pairIrises, gazeOrigin, quantizeGaze, resolveBlink } from './pipeline.js'; + exposeIndex, eyeSignals, pairIrises, gazeOrigin, quantizeSnap, resolveBlink, + browSignals, pairBrows } from './pipeline.js'; import { IndexedRaster, hexToRgb } from './raster.js'; import { writeTake } from './take.js'; import { otsuForTest, scaleRing } from './interior.js'; @@ -479,19 +481,111 @@ export function run() { const px = sig.gazeRaw.map((g) => ({ x: (g.x - org.x) * 30, y: (g.y - org.y) * 30 })); const cells = (a) => new Set(a.map((g) => `${g.x},${g.y}`)).size; ok('quantisation collapses drift into a few fixations', - cells(quantizeGaze(px, 2, 2)) <= 6 && cells(px) > 40, - `${cells(px)} raw -> ${cells(quantizeGaze(px, 2, 2))} cells`); + cells(quantizeSnap(px, 2, 2)) <= 6 && cells(px) > 40, + `${cells(px)} raw -> ${cells(quantizeSnap(px, 2, 2))} cells`); ok('gaze step 0 leaves the track untouched', - quantizeGaze(px, 0, 2).every((g, i) => g.x === px[i].x && g.y === px[i].y)); + quantizeSnap(px, 0, 2).every((g, i) => g.x === px[i].x && g.y === px[i].y)); ok('quantised values land on the grid', - quantizeGaze(px, 2, 0).every((g) => Math.abs(g.x % 2) < 1e-9 && Math.abs(g.y % 2) < 1e-9)); + quantizeSnap(px, 2, 0).every((g) => Math.abs(g.x % 2) < 1e-9 && Math.abs(g.y % 2) < 1e-9)); // A one-frame excursion is noise; the dwell must swallow it. { const spike = [{ x: 0, y: 0 }, { x: 0, y: 0 }, { x: 4, y: 0 }, { x: 0, y: 0 }, { x: 0, y: 0 }]; ok('the dwell suppresses a one-frame gaze spike', - quantizeGaze(spike, 2, 1).every((g) => g.x === 0)); + quantizeSnap(spike, 2, 1).every((g) => g.x === 0)); ok('a sustained move still gets through', - quantizeGaze([...spike, { x: 4, y: 0 }, { x: 4, y: 0 }, { x: 4, y: 0 }], 2, 1).pop().x === 4); + quantizeSnap([...spike, { x: 4, y: 0 }, { x: 4, y: 0 }, { x: 4, y: 0 }], 2, 1).pop().x === 4); + } + } + + /* ---- brows ---- */ + + ok('brow rings have 10 distinct ids each', + new Set(BROW_A_RING).size === 10 && new Set(BROW_B_RING).size === 10); + ok('the brow rings share no landmark with each other, RIGID, or the lids', + !BROW_A_RING.some((i) => BROW_B_RING.includes(i)) && + ![...BROW_A_RING, ...BROW_B_RING].some((i) => + RIGID.includes(i) || EYE_R_RING.includes(i) || EYE_L_RING.includes(i)), + 'a brow in RIGID would bleed expression into the stabilisation'); + + { + let bad = null; + for (const [label, table] of [['A', BROW_A_RING], ['B', BROW_B_RING]]) { + for (let n = 4; n <= 10 && !bad; n += 2) { + const slots = subsampleSlots(table.length, n); + for (let f = 0; f < dense.length; f++) { + if (ringSelfIntersections(slots.map((sl) => dense[f][table[sl]])).length) { + bad = `${label} verts=${n} frame=${f}`; break; + } + } + } + } + ok('brow rings are simple at every vertex budget', !bad, bad || ''); + } + + { + const stB = stabilize(dense, 2); + const pr = pairBrows(stB); + ok('brow-to-eye pairing is resolved from geometry', + pr.right === 'browA' && pr.left === 'browB', JSON.stringify(pr)); + // Getting this backwards mirrors the tilt, so inner-up "worried" renders as + // outer-up. That is a different expression, not a broken one, which is + // exactly why it needs an assertion rather than an eyeball. + ok('the outer end of the brow ring is resolved from geometry', pr.outerAtSlot0 === true); + + // Both ends land on fixed slots whichever edge of the brow is on top, which + // is what lets the upper/lower ambiguity go unresolved without consequence. + ok('brow end slots are disjoint and cover both ends', + !BROW_END_0.some((i) => BROW_END_1.includes(i)) && + BROW_END_0.length === 2 && BROW_END_1.length === 2); + + // Ground truth: synth commands rest, surprise, worry and anger as heights + // above the eye centre in eye widths, holding each for thirteen frames. + const b = browSignals(stB); + const at = (f) => [b.R[f].x, b.R[f].y]; + const near = (v, want) => Math.abs(v - want) < 0.02; + ok('brow raise recovers the commanded rest pose', near(at(0)[0], 0.30) && near(at(0)[1], 0.30), + at(0).map((v) => v.toFixed(3)).join(', ')); + ok('brow raise recovers surprise - both ends up', + near(at(14)[0], 0.46) && near(at(14)[1], 0.46), at(14).map((v) => v.toFixed(3)).join(', ')); + ok('brow raise recovers worry - inner end only', + near(at(27)[0], 0.30) && near(at(27)[1], 0.44), at(27).map((v) => v.toFixed(3)).join(', ')); + ok('brow raise recovers anger - inner end down', + near(at(40)[0], 0.30) && near(at(40)[1], 0.18), at(40).map((v) => v.toFixed(3)).join(', ')); + + // Tilt must be a signed quantity that separates worry from anger. If the + // outer/inner resolution were mirrored these two would swap. + ok('tilt separates worry from anger by sign', + (at(27)[1] - at(27)[0]) > 0.08 && (at(40)[1] - at(40)[0]) < -0.08, + `worry ${(at(27)[1] - at(27)[0]).toFixed(3)}, anger ${(at(40)[1] - at(40)[0]).toFixed(3)}`); + + // A blink must not read as a brow raise: the raise is measured against the + // eye's rigid corners, not its lid, which is the same trap the gaze origin + // has and worth avoiding twice. + const dR = Math.abs(b.R[19].x - b.R[18].x); + ok('a blink does not fake a brow raise', dR < 0.01, `f18 -> f19 delta ${dR.toFixed(4)}`); + + // Quantisation: four sustained poses must come back as a handful of levels. + const rest = gazeOrigin(b.R, 'median'); + const px = b.R.map((g) => ({ x: (g.x - rest.x) * 60, y: (g.y - rest.y) * 60 })); + const cells = (a) => new Set(a.map((g) => `${g.x},${g.y}`)).size; + ok('brow quantisation collapses drift into a few poses', + cells(quantizeSnap(px, 2, 2)) <= 6 && cells(px) > 20, + `${cells(px)} raw -> ${cells(quantizeSnap(px, 2, 2))} poses`); + // The dwell is SHARED across both channels, and that is the whole reason + // brows reuse the gaze quantiser rather than running two independent ones. + // Here the outer end moves one frame before the inner: with a shared dwell + // the half-raised pose (2,0) is transient and never commits, so the brow + // snaps once. Two independent dwells would emit it and the brow would crawl + // into position over two frames instead of hitting it. + { + const staggered = [ + { x: 0, y: 0 }, { x: 0, y: 0 }, { x: 2, y: 0 }, + { x: 2, y: 2 }, { x: 2, y: 2 }, { x: 2, y: 2 }, + ]; + const out = quantizeSnap(staggered, 2, 1); + ok('a shared dwell never emits a half-raised brow', + !out.some((g) => g.x === 2 && g.y === 0), + out.map((g) => `${g.x},${g.y}`).join(' ')); } } diff --git a/js/synth.js b/js/synth.js index 6de5606..84252ba 100644 --- a/js/synth.js +++ b/js/synth.js @@ -6,7 +6,8 @@ // stabilisation against a KNOWN head motion, since real footage gives no ground // truth to compare against. import { LIPS_OUTER, LIPS_INNER, FACE_OVAL, RIGID, - EYE_R_RING, EYE_L_RING, IRIS_A, IRIS_B } from './landmarks.js'; + EYE_R_RING, EYE_L_RING, IRIS_A, IRIS_B, + BROW_A_RING, BROW_B_RING } from './landmarks.js'; const NUM = 478; @@ -84,6 +85,27 @@ export function synthDense(nFrames = 72, { swapIris = false } = {}) { eye(EYE_R_RING, -0.0515, 1, swapIris ? IRIS_B : IRIS_A); eye(EYE_L_RING, 0.0515, -1, swapIris ? IRIS_A : IRIS_B); + // Brows, held in four sustained poses so raise quantisation has genuine + // plateaux to find: rest, surprise (both ends up), worry (inner up only), + // anger (inner down). Commanded in eye widths above the eye centre so the + // measurement can be checked against a number rather than an eyeball. + const BROW = [[0.30, 0.30], [0.46, 0.46], [0.30, 0.44], [0.30, 0.18]]; + const [bOut, bIn] = BROW[Math.floor(t / 13) % BROW.length]; + const EYE_W = EYE_RX * 2, HALF = 0.006; // ring half-thickness + const brow = (ring, cx, outerSign) => { + // Slots 0-4 are one edge outer->inner, 5-9 the other inner->outer, so the + // ends land on {0,9} and {4,5} exactly as the table promises. + const n = ring.length, half = n / 2; + for (let k = 0; k < n; k++) { + const along = k < half ? k / (half - 1) : (n - 1 - k) / (half - 1); + const rise = bOut + (bIn - bOut) * along; + place(ring[k], cx + outerSign * (EYE_RX - along * EYE_W) * 1.05, + EYE_Y - rise * EYE_W + (k < half ? -HALF : HALF)); + } + }; + brow(BROW_A_RING, -0.0515, -1); + brow(BROW_B_RING, 0.0515, 1); + // Lip rings as ellipse arcs, traversed so ring ORDER matches the tables: // slot 0 = right corner, 5 = top centre, 10 = left corner, 15 = bottom // centre, with y growing downward. Getting this convention wrong swaps two