noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make the mouth lead legible and assertable

Browse source 
"It feels like it does nothing" is indistinguishable from "it does nothing",
and at 24fps a lead of 1 is 42ms - small enough to reasonably doubt. So the
shift is now provable and visible rather than taken on trust:

- shiftIndex is a pure exported function with assertions covering identity,
  both directions, and clamping at each end
- the frame label always shows the mouth frame, not only when shifted, so the
  number can be watched diverging from f; non-zero leads also report in ms
- the slider readout carries an explicit + sign
- the stabilised pane draws the unshifted contour as a dark-green ghost when a
  lead is set, so the offset is something you can see

Also: leadIndex called opts() on every invocation - ~14 DOM reads, once per
strip thumbnail, so ~1000 per redraw at 74 frames. It reads a cached scalar now.

The "different pose" assertion checks the whole track rather than one pair:
synthetic poses hold for nine-frame beats, so a single pair can legitimately be
identical while the shift works correctly.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Your Name 2026-09-24 15:38:02 -04:00
parent f9b8ec8617
commit dc0a3f34e6
4 changed files with 56 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

28
js/app.js
View file

@ -1,6 +1,6 @@
import { FaceLandmarker, FilesetResolver } from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/vision_bundle.mjs';
import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js';
import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js';
import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame, shiftIndex } from './pipeline.js';
import { IndexedRaster } from './raster.js';
import { drawRegistered, posterizeInto } from './underlay.js';
import { extractTeeth } from './interior.js';
@ -26,6 +26,7 @@ const state = {
frame: 0, playing: false, faceBox: null,
fps: 12, audio: null, // fps comes from manifest.json, never guessed
aspect: 1, // imgW/imgH; converts MediaPipe's anisotropic space
lead: 0, // performance-track offset in frames
interior: null, // per-frame teeth measurement from image content
teeth: null, // resolved per-frame {show, t} after knobs
};
@ -165,6 +166,7 @@ function makeXform(stab, neutral) {
function rebuild(resetKeep) {
if (!state.dense) return;
const o = opts();
state.lead = o.lead;
const N = state.dense.length;
state.stab = stabilize(state.dense, o.smoothWin, state.aspect);
@ -345,8 +347,9 @@ const keptSorted = () => [...state.keep].sort((a, b) => a - b);
// Positive lead = the mouth arrives earlier. Only performance parts shift; the
// head stays with the audio, because it is the mouth that should anticipate.
function leadIndex(f) {
const N = state.dense.length;
return Math.min(N - 1, Math.max(0, f + opts().lead));
// Reads a cached scalar, not opts(): this runs once per strip thumbnail, and
// calling opts() here meant ~14 DOM reads x 74 frames on every redraw.
return shiftIndex(f, state.lead, state.dense.length);
}
function blit(canvas, raster, zoom) {
@ -364,7 +367,7 @@ function drawAll() {
function drawReadout() {
const kept = keptSorted();
const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k);
const lead = opts().lead;
const lead = state.lead;
const teethFrames = state.teeth ? state.teeth.filter((t) => t.show).length : 0;
el('readout').textContent =
`${state.dense.length} frames → ${kept.length} drawings · ` +
@ -378,10 +381,13 @@ function drawReadout() {
function drawPanes() {
const f = state.frame, kept = keptSorted();
const pf = heldFrame(kept, f);
const lead = opts().lead;
// The mouth frame is always shown, not only when shifted, so the number can be
// watched diverging from f rather than taken on trust.
const lead = state.lead;
el('framelabel').textContent =
`f ${f} / ${state.dense.length - 1} · ${(f / state.fps).toFixed(2)}s · plate f${pf}` +
(lead ? ` · mouth f${leadIndex(f)} (${lead > 0 ? '+' : ''}${lead})` : '') +
`f ${f} / ${state.dense.length - 1} · ${(f / state.fps).toFixed(2)}s · ` +
`plate f${pf} · mouth f${leadIndex(f)}` +
(lead ? ` (${lead > 0 ? '+' : ''}${lead} = ${(lead / state.fps * 1000).toFixed(0)}ms)` : '') +
(state.keep.has(f) ? ' · KEPT' : ' · held');
const c1 = el('cv-source'), g1 = c1.getContext('2d');
@ -415,6 +421,9 @@ function drawPanes() {
const z = (pts) => pts.map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM }));
const mf = leadIndex(f);
strokePts(g2, z(state.plates[pf]), '#3b4a63');
// With a lead set, the unshifted contour is drawn as a ghost so the offset is
// something you can see rather than something you have to trust.
if (mf !== f) strokePts(g2, z(state.outer[f]), '#2f6b46');
strokePts(g2, z(state.outer[mf]), '#4ade80');
if (!state.hidden[mf]) strokePts(g2, z(state.inner[mf]), '#f87171');
@ -611,6 +620,7 @@ function runSynthetic() {
attachAudio(null);
state.fps = 12;
state.aspect = 1; // synthetic landmarks are generated square
state.lead = 0;
state.interior = null; // no pixels, so no teeth
state.dense = synthDense(72);
@ -628,6 +638,7 @@ for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol'
? (+el(id).value / 1000).toFixed(3)
: ['teethOn', 'teethErode', 'tongueReject', 'topBias'].includes(id)
? (+el(id).value / 100).toFixed(2)
: id === 'lead' && +el(id).value > 0 ? `+${el(id).value}`
: el(id).value;
if (id === 'tol') return; // tol only matters when you ask for a suggestion
rebuild(false);
@ -699,7 +710,8 @@ window.addEventListener('keydown', (e) => {
} else if (e.key === '[' || e.key === ']') {
const n = el('lead');
n.value = Math.max(+n.min, Math.min(+n.max, +n.value + (e.key === ']' ? 1 : -1)));
el('leadv').textContent = n.value;
state.lead = +n.value;
el('leadv').textContent = state.lead > 0 ? `+${state.lead}` : String(state.lead);
drawAll();
} else if (e.key === 'b') {
const sel = el('plateMode');

9
js/pipeline.js
View file

@ -177,3 +177,12 @@ export function heldFrame(kept, f) {
for (const k of kept) { if (k <= f) hit = k; else break; }
return hit;
}
// Shift a performance track against the clock, clamped at the ends.
//
// Pure and exported so the shift can actually be asserted: "the slider feels
// like it does nothing" is otherwise indistinguishable from "the slider does
// nothing", and at 24fps a lead of 1 is 42ms, which is small enough to doubt.
export function shiftIndex(f, lead, n) {
return Math.min(n - 1, Math.max(0, f + lead));
}

27
js/selftest.js
View file

@ -9,7 +9,7 @@
import { LIPS_OUTER, LIPS_INNER, FACE_OVAL, RIGID, subsampleSlots, subsampleRing } from './landmarks.js';
import { fitSimilarity, applySim, procrustesMean, smoothTransforms } from './mathutil.js';
import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js';
import { stabilize, toRasterRing, selectKeys, activeKey, shiftIndex } from './pipeline.js';
import { IndexedRaster, hexToRgb } from './raster.js';
import { writeTake } from './take.js';
import { otsuForTest, scaleRing } from './interior.js';
@ -206,6 +206,31 @@ export function run() {
ok('activeKey holds between keys',
activeKey(sel.keys, sel.keys[1].f - 1).f === sel.keys[0].f);
// mouth lead: a shift that "feels like it does nothing" is indistinguishable
// from one that does nothing, so assert the arithmetic directly.
ok('lead 0 is identity', [0, 5, 71].every((f) => shiftIndex(f, 0, 72) === f));
ok('positive lead moves the source frame forward', shiftIndex(10, 2, 72) === 12);
ok('negative lead moves it back', shiftIndex(10, -3, 72) === 7);
ok('lead clamps at the start', shiftIndex(1, -6, 72) === 0);
ok('lead clamps at the end', shiftIndex(70, 6, 72) === 71);
{
// ...and that it selects different POSES, not merely different indices.
// Checked across the whole track rather than at one pair: synthetic poses
// hold for nine-frame beats, so any single pair can legitimately be
// identical while the shift works perfectly.
const N = shapes.length;
let moved = 0, total = 0;
for (let f = 0; f < N; f++) {
const a = shapes[shiftIndex(f, 0, N)], b = shapes[shiftIndex(f, 3, N)];
let d = 0;
for (let i = 0; i < a.length; i++) d += Math.hypot(a[i].x - b[i].x, a[i].y - b[i].y);
total++;
if (d / a.length > 0.5) moved++;
}
ok('a lead of 3 changes the pose on a good share of frames', moved / total > 0.2,
`${moved}/${total} frames differ`);
}
// rasteriser: indexed, hard-edged, no blending
const r = new IndexedRaster(64, 48);
r.clear(0);