noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Mouth lead: nudge performance tracks ahead of the clock

Browse source 
Smoothing does not literally delay anything - a centred moving average has zero
phase lag - but it blurs onsets, so the visually salient moment of a mouth
opening moves later even though the mean does not. Animators also draw mouth
shapes a frame or two ahead of the sound as standard practice, so this is the
normal control rather than a workaround.

Only the performance tracks shift; the head and audio stay put, since it is the
mouth that should anticipate. In photo-underlay modes the vector mouth will
therefore no longer match the frame behind it, which is expected.

The lead is baked into the exported take - key f carries the pose from source
frame f+lead - so the Animator Pro renderer never needs to know about it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Your Name 2026-09-24 15:34:08 -04:00
parent a146d271b6
commit d18791e6ad
4 changed files with 64 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

61
js/app.js
View file

@ -39,6 +39,7 @@ const el = (id) => {
};
const opts = () => ({
verts: +el('verts').value,
lead: +el('lead').value,
teethOn: +el('teethOn').value / 100, // minimum Otsu class separation
teethDwell: +el('teethDwell').value,
teethSmooth: +el('teethSmooth').value,
@ -282,10 +283,11 @@ function renderFrame(f, mode = plateMode()) {
if (mode === 'oval' || mode === 'oval+photo') r.fillPoly(state.plates[pf], IDX.base);
}
r.fillPoly(state.outer[f], IDX.dark); // mouth keeps every frame
if (!state.hidden[f]) {
r.fillPoly(state.inner[f], IDX.mouth);
const te = state.teeth[f];
const mf = leadIndex(f); // performance frame, possibly ahead
r.fillPoly(state.outer[mf], IDX.dark); // mouth keeps every frame
if (!state.hidden[mf]) {
r.fillPoly(state.inner[mf], IDX.mouth);
const te = state.teeth[mf];
if (te.show && te.pts && te.pts.length >= 3) r.fillPoly(te.pts, IDX.teeth);
}
return r;
@ -332,6 +334,21 @@ function compositeRender(canvas, f, zoom) {
const keptSorted = () => [...state.keep].sort((a, b) => a - b);
// Performance tracks can lead the clock.
//
// A centred moving average has no phase lag, so smoothing does not literally
// delay anything - but it blurs onsets, and the visually salient moment of a
// mouth opening moves later even though the mean does not. Animators also draw
// mouth shapes one or two frames ahead of the sound as a matter of course, so
// this is the normal control rather than a correction.
//
// Positive lead = the mouth arrives earlier. Only performance parts shift; the
// head stays with the audio, because it is the mouth that should anticipate.
function leadIndex(f) {
const N = state.dense.length;
return Math.min(N - 1, Math.max(0, f + opts().lead));
}
function blit(canvas, raster, zoom) {
canvas.width = RW * zoom; canvas.height = RH * zoom;
canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0);
@ -347,10 +364,12 @@ function drawAll() {
function drawReadout() {
const kept = keptSorted();
const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k);
const lead = opts().lead;
const teethFrames = state.teeth ? state.teeth.filter((t) => t.show).length : 0;
el('readout').textContent =
`${state.dense.length} frames → ${kept.length} drawings · ` +
`teeth on ${teethFrames}f · ` +
(lead ? `mouth leads ${lead}f (${(lead / state.fps * 1000).toFixed(0)}ms) · ` : '') +
`holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` +
`neutral f${state.neutral} · residual ` +
`${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`;
@ -359,8 +378,10 @@ function drawReadout() {
function drawPanes() {
const f = state.frame, kept = keptSorted();
const pf = heldFrame(kept, f);
const lead = opts().lead;
el('framelabel').textContent =
`f ${f} / ${state.dense.length - 1} · ${(f / state.fps).toFixed(2)}s · plate f${pf}` +
(lead ? ` · mouth f${leadIndex(f)} (${lead > 0 ? '+' : ''}${lead})` : '') +
(state.keep.has(f) ? ' · KEPT' : ' · held');
const c1 = el('cv-source'), g1 = c1.getContext('2d');
@ -392,9 +413,10 @@ function drawPanes() {
g2.moveTo(c2.width / 2, 0); g2.lineTo(c2.width / 2, c2.height);
g2.moveTo(0, c2.height / 2); g2.lineTo(c2.width, c2.height / 2); g2.stroke();
const z = (pts) => pts.map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM }));
const mf = leadIndex(f);
strokePts(g2, z(state.plates[pf]), '#3b4a63');
strokePts(g2, z(state.outer[f]), '#4ade80');
if (!state.hidden[f]) strokePts(g2, z(state.inner[f]), '#f87171');
strokePts(g2, z(state.outer[mf]), '#4ade80');
if (!state.hidden[mf]) strokePts(g2, z(state.inner[mf]), '#f87171');
compositeRender(el('cv-render'), f, ZOOM);
drawInteriorDebug(f);
@ -403,7 +425,8 @@ function drawPanes() {
// What the teeth measurement actually saw: sampled region, pixels above
// threshold in green, the resolved line in amber. Recomputed for the current
// frame only, so it costs nothing to keep on screen.
function drawInteriorDebug(f) {
function drawInteriorDebug(fRaw) {
const f = leadIndex(fRaw);
const host = el('cv-teeth');
const img = state.images[f];
if (!img || state.hidden[f]) {
@ -518,14 +541,21 @@ function exportTake() {
// Sparse: one plate key per frame a human draws.
{ name: 'head', kind: 'plate', z: 0, interp: 'hold',
keys: kept.map((f, i) => ({ f: i, plate: i, src: f })) },
// Dense: the traced mouth keeps every frame.
// Dense: the traced mouth keeps every frame. The lead is baked in here -
// key f carries the pose from source frame f+lead - so the renderer never
// needs to know about it.
{ name: 'mouth', kind: 'poly', z: 30, color: 'skin_dark', interp: 'hold',
keys: state.outer.map((pts, f) => ({ f, src: f, pts })) },
keys: state.outer.map((_, f) => ({ f, src: leadIndex(f), pts: state.outer[leadIndex(f)] })) },
{ name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) },
keys: state.inner.map((_, f) => {
const m = leadIndex(f);
return state.hidden[m] ? { f, hidden: true } : { f, src: m, pts: state.inner[m] };
}) },
{ name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in',
keys: state.teeth.map((te, f) =>
(te.show && te.pts ? { f, src: f, pts: te.pts } : { f, hidden: true })) },
keys: state.teeth.map((_, f) => {
const m = leadIndex(f), te = state.teeth[m];
return te.show && te.pts ? { f, src: m, pts: te.pts } : { f, hidden: true };
}) },
],
};
const text = writeTake(take)
@ -592,7 +622,7 @@ function runSynthetic() {
for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol',
'teethOn', 'teethDwell', 'teethErode', 'tongueReject', 'blobGrow',
'topBias', 'teethVerts', 'teethSmooth']) {
'topBias', 'teethVerts', 'teethSmooth', 'lead']) {
el(id).addEventListener('input', () => {
el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol'
? (+el(id).value / 1000).toFixed(3)
@ -666,6 +696,11 @@ window.addEventListener('keydown', (e) => {
else if (e.key === 'ArrowLeft') { seekTo(Math.max(0, state.frame - 1)); drawAll(); }
else if (e.key === 'Backspace' || e.key === 'Delete' || e.key === 'x') {
state.keep.delete(state.frame === 0 ? -1 : state.frame); drawAll();
} else if (e.key === '[' || e.key === ']') {
const n = el('lead');
n.value = Math.max(+n.min, Math.min(+n.max, +n.value + (e.key === ']' ? 1 : -1)));
el('leadv').textContent = n.value;
drawAll();
} else if (e.key === 'b') {
const sel = el('plateMode');
sel.selectedIndex = (sel.selectedIndex + 1) % sel.options.length;