noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Brows: traced ring, quantised raise

Browse source 
A brow at 320x200 is fourteen pixels wide and three tall. Its shape carries
almost nothing at that size; its height above the eye carries the expression,
and a brow raise is the most legible beat on a face. So the ring is traced and
the height is quantised - the split the eyes already got, where the lid is a
traced feature and the iris a quantised primitive.

The decomposition is the point. The traced ring already contains the real
height, so adding a quantised raise on top would move the brow twice. The
height is measured OUT of the ring, quantised, and put back, so the shape that
renders is his at a height that snaps between a few levels and holds.

Measured at both ends rather than as one number, because raise and tilt are
different expressions out of one mechanism: both ends up is surprise, inner up
alone is worry, inner down is anger. They share a dwell - the gaze quantiser,
renamed quantizeSnap now that it has two callers - so the brow hits its pose in
one frame instead of crawling into it with one end arriving before the other.

Measured against the eye's corner midpoint, never its lid. Same trap the gaze
origin has and worth avoiding twice: brows and lids move together constantly,
so a brow that jumped on every blink would read as a tic. Rest pose from the
take median rather than the neutral frame, for the reason gaze learned the hard
way - that frame is picked by minimum mouth aperture and says nothing about the
brows.

Two correspondences resolved from geometry, not declared: which ring is which
brow, and which end is the outer one. The second matters more - backwards, the
tilt mirrors and worry renders as its own opposite, which reads as a directed
performance choice rather than a bug and would never be questioned. Which EDGE
is upper is deliberately left unresolved: it traverses the same ring the other
way, an even-odd fill has no winding, and both ends still land on fixed slots.

Also fixes a bug from the exposure work: the live render applied exposure to
the plate and the mouth but not to the eyes, so on 2s the preview and the
export disagreed. A preview that disagrees with the export is the one bug this
tool cannot afford. perfIndex now exists as a named thing so the two paths
cannot drift apart again.

91 -> 105 assertions. Ground truth on all four synthetic brow poses, tilt
separating worry from anger by sign, a blink not faking a raise, and a shared
dwell never emitting a half-raised brow.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Your Name 2026-09-24 18:11:25 -04:00
parent 2e4406d579
commit 03df81fa0b
8 changed files with 419 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

123
js/app.js
View file

@ -1,8 +1,10 @@
import { FaceLandmarker, FilesetResolver } from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/vision_bundle.mjs';
import { LIPS_OUTER, LIPS_INNER, FACE_OVAL,
EYE_R_RING, EYE_L_RING, IRIS_A, IRIS_B } from './landmarks.js';
EYE_R_RING, EYE_L_RING, IRIS_A, IRIS_B,
BROW_A_RING, BROW_B_RING } from './landmarks.js';
import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame, shiftIndex,
exposeIndex, eyeSignals, gazeOrigin, quantizeGaze, resolveBlink } from './pipeline.js';
exposeIndex, eyeSignals, gazeOrigin, quantizeSnap, resolveBlink,
browSignals } from './pipeline.js';
import { IndexedRaster } from './raster.js';
import { drawRegistered, posterizeInto } from './underlay.js';
import { extractTeeth } from './interior.js';
@ -31,8 +33,12 @@ const PALETTE = [
// step inside it reads as a hole.
{ name: 'iris', hex: '#4a5468' },
{ name: 'pupil', hex: '#171a22' },
// Brows get their own entry rather than sharing skin_dark with the lash line.
// They are hair, not shadow: when hair plates exist they want to match those,
// and tying them to the lash means you cannot change one without the other.
{ name: 'brow', hex: '#3a2a22' },
];
const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, teeth: 4, white: 5, iris: 6, pupil: 7 };
const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, teeth: 4, white: 5, iris: 6, pupil: 7, brow: 8 };
const state = {
dense: null, images: [], stab: null, xform: null,
@ -46,6 +52,7 @@ const state = {
interior: null, // per-frame teeth measurement from image content
teeth: null, // resolved per-frame {show, t} after knobs
eyes: null, // resolved per-frame lid rings, shut flags, iris discs
brows: null, // resolved per-frame brow rings after quantised raise
eyeSig: null, // raw eye measurement, kept for the gaze readout
};
@ -72,6 +79,11 @@ const opts = () => ({
apertureThresh: +el('apertureThresh').value / 1000,
tol: +el('tol').value / 1000,
exposure: +el('exposure').value,
browVerts: +el('browVerts').value,
browWeight: +el('browWeight').value,
browGain: +el('browGain').value / 100,
browStep: +el('browStep').value,
browDwell: +el('browDwell').value,
irisAnchor: el('irisAnchor').value,
gazeOrigin: el('gazeOrigin').value,
eyeVerts: +el('eyeVerts').value,
@ -226,6 +238,7 @@ function rebuild(resetKeep) {
}
state.teeth = resolveTeeth(o);
state.eyes = buildEyes(o);
state.brows = buildBrows(o);
// Plate outline per frame, so a kept frame shows its own head shape.
state.plates = state.stab.oval.map((r) => r.map(state.xform));
@ -341,7 +354,7 @@ function buildEyes(o) {
x: (g.x - origin.x) * o.gazeGain * w,
y: (g.y - origin.y) * o.gazeGain * w,
}));
const gaze = quantizeGaze(px, o.gazeStep, o.gazeDwell);
const gaze = quantizeSnap(px, o.gazeStep, o.gazeDwell);
const eye = (sk, lids, shut, rad, f) => {
const e = sk(f);
@ -371,6 +384,70 @@ function buildEyes(o) {
};
}
// Brows: ring traced every frame, HEIGHT quantised.
//
// The decomposition is the point. The traced ring already contains the brow's
// real height, so adding a quantised raise on top would move it twice. Instead
// the height is measured out of the ring, quantised, and put back - the shape
// that renders is his, at a height that snaps between a few authored levels and
// holds. That is the same split the eyes got: lid traced as a feature, iris
// position quantised as a primitive.
//
// Two ends, not one height, warped linearly between them. Raise and tilt are
// different expressions out of one mechanism: both ends up is surprise, inner
// up alone is worry, inner down is anger.
function buildBrows(o) {
const st = state.stab, N = state.dense.length;
const sig = browSignals(st);
state.browSig = sig;
const ringOf = (side) => (side === 'R' ? sig.pairing.right : sig.pairing.left);
const table = (side) => (ringOf(side) === 'browA' ? BROW_A_RING : BROW_B_RING);
const build = (side, corners) => {
const rings = smoothContours(
st[ringOf(side)].map((r) => toRasterRing(r, table(side), o.browVerts, state.xform)),
o.contourSmooth);
// Eye width in raster pixels, so the raise converts from eye widths into the
// units the grid is expressed in and the knob means the same on any framing.
const wpx = (f) => {
const a = state.xform(corners[f][0]), b = state.xform(corners[f][1]);
return Math.hypot(a.x - b.x, a.y - b.y);
};
const meanW = st.cornersR.reduce((a, _, f) => a + wpx(f), 0) / N;
// Rest pose from the take MEDIAN, never from the neutral frame. That frame
// is chosen by minimum mouth aperture and says nothing about the brows, and
// the same mistake on the gaze origin re-pointed an entire performance.
const rest = gazeOrigin(sig[side], 'median');
const px = sig[side].map((g) => ({
x: (g.x - rest.x) * o.browGain * meanW,
y: (g.y - rest.y) * o.browGain * meanW,
}));
const q = quantizeSnap(px, o.browStep, o.browDwell);
const frames = rings.map((ring, f) => {
// Raise is measured upward but y grows downward, so a positive raise is a
// negative y offset.
const dOuter = -(q[f].x - px[f].x), dInner = -(q[f].y - px[f].y);
const a = state.xform(corners[f][0]), b = state.xform(corners[f][1]);
const span = b.x - a.x;
const warped = ring.map((p) => {
// Position along the brow's own axis, outer end to inner end. Taken from
// x against the eye corners rather than from ring slots, because
// subsampling does not keep the end slots at any given budget.
const t = span === 0 ? 0 : Math.min(1, Math.max(0, (p.x - a.x) / span));
return { x: p.x, y: p.y + dOuter + (dInner - dOuter) * t };
});
return offsetRing(warped, o.browWeight);
});
return { frames, px, q };
};
return { R: build('R', st.cornersR), L: build('L', st.cornersL), pairing: sig.pairing };
}
// Presence gets hysteresis and a minimum dwell, the same treatment plate
// selection gets: a teeth block that blinks on and off for single frames is
// worse than one that is simply absent. Appearing needs a clear signal, staying
@ -440,7 +517,14 @@ function renderFrame(f, mode = plateMode()) {
// device: it exists because a mouth shape anticipates the sound it makes.
// Nothing about a blink or a glance is tied to the audio, so shifting the
// eyes would only slide them off the head that carries them.
if (state.eyes) drawEyes(r, state.eyes.frames[f]);
// Eyes and brows ride the exposure grid but NOT the mouth lead: the lead is a
// lip-sync device and nothing about a blink or a brow is tied to the audio.
const ef = perfIndex(f);
if (state.eyes) drawEyes(r, state.eyes.frames[ef]);
if (state.brows) {
r.fillPoly(state.brows.R.frames[ef], IDX.brow);
r.fillPoly(state.brows.L.frames[ef], IDX.brow);
}
const mf = leadIndex(f); // performance frame, possibly ahead
r.fillPoly(state.outer[mf], IDX.dark); // mouth keeps every frame
@ -541,6 +625,12 @@ function leadIndex(f) {
// even ones would read as two performances laid over each other.
const plateIndex = (f) => heldFrame(keptSorted(), exposeIndex(f, state.exposure));
// Performance tracks that do not take the mouth lead still ride the grid. This
// existing as a named thing is what stopped the eyes holding on 1s in the
// preview while the export held them on 2s - a preview that disagrees with the
// export is the one bug this tool cannot afford.
const perfIndex = (f) => exposeIndex(f, state.exposure);
function blit(canvas, raster, zoom) {
canvas.width = RW * zoom; canvas.height = RH * zoom;
canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0);
@ -563,6 +653,7 @@ function drawReadout() {
`teeth on ${teethFrames}f · ` +
`${blinkRuns(state.eyes.shutR).length}/${blinkRuns(state.eyes.shutL).length} blinks R/L · ` +
`${gazeCells(state.eyes.gaze)} gaze cells · ` +
`${gazeCells(state.brows.R.q)} brow poses · ` +
(state.exposure > 1
? `on ${state.exposure}s = ${(state.fps / state.exposure).toFixed(4).replace(/\.?0+$/, '')}fps · `
: '') +
@ -688,6 +779,9 @@ function drawEyeOverlay(g, map, f) {
for (const ring of [EYE_R_RING, EYE_L_RING]) {
strokePts(g, ring.map((i) => map(lm[i])), '#60a5fa');
}
for (const ring of [BROW_A_RING, BROW_B_RING]) {
strokePts(g, ring.map((i) => map(lm[i])), '#c084fc');
}
if (!state.eyes.hasIris) return;
for (const iris of [IRIS_A, IRIS_B]) {
strokePts(g, iris.slice(1).map((i) => map(lm[i])), '#fbbf24');
@ -839,7 +933,7 @@ function eyeParts(grid) {
const out = [];
// Eyes ride the exposure grid but NOT the mouth lead: the lead is a lip-sync
// device and nothing about a blink is tied to the audio.
const src = (f) => exposeIndex(f, state.exposure);
const src = perfIndex;
[['r', 20], ['l', 23]].forEach(([side, z]) => {
const at = (f) => state.eyes.frames[src(f)][side];
out.push(
@ -871,6 +965,16 @@ function eyeParts(grid) {
return out;
}
// Brows are a traced ring like the lids, so they keep every frame on the grid.
// The quantised raise is already baked into the points - the renderer is handed
// a polygon, not a shape plus an offset it would have to recombine.
function browParts(grid) {
return [['r', 26, 'R'], ['l', 27, 'L']].map(([name, z, side]) => ({
name: `brow_${name}`, kind: 'poly', z, color: 'brow', interp: 'hold',
keys: grid.map((f) => ({ f, src: perfIndex(f), pts: state.brows[side].frames[perfIndex(f)] })),
}));
}
function exportTake() {
const kept = keptSorted();
const N = state.dense.length;
@ -903,6 +1007,7 @@ function exportTake() {
// key stream is dense but the VALUES change only on saccades, so a
// hold-interpolating renderer cuts between fixations by itself.
...eyeParts(grid),
...browParts(grid),
{ name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in',
keys: grid.map((f) => {
const m = leadIndex(f), te = state.teeth[m];
@ -991,6 +1096,9 @@ const FMT = {
irisSize: (v) => `${v}%`,
gazeGain: (v) => (v / 100).toFixed(2),
gazeStep: (v) => (v ? `${v}px` : 'off'),
browStep: (v) => (v ? `${v}px` : 'off'),
browWeight: (v) => `${v}px`,
browGain: (v) => (v / 100).toFixed(2),
pupilPx: (v) => (v ? `${v}px` : 'off'),
lashPx: (v) => `${v}px`,
lead: (v) => (v > 0 ? `+${v}` : String(v)),
@ -1000,7 +1108,8 @@ for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol'
'teethOn', 'teethDwell', 'teethErode', 'tongueReject', 'blobGrow',
'topBias', 'teethVerts', 'teethSmooth', 'lead',
'eyeVerts', 'lashPx', 'irisSize', 'pupilPx', 'gazeGain',
'gazeStep', 'gazeDwell', 'blinkCut', 'blinkHold', 'blinkDwell']) {
'gazeStep', 'gazeDwell', 'blinkCut', 'blinkHold', 'blinkDwell',
'browVerts', 'browWeight', 'browGain', 'browStep', 'browDwell']) {
const show = () => {
el(id + 'v').textContent = FMT[id] ? FMT[id](+el(id).value) : el(id).value;
};