Mouth lead: nudge performance tracks ahead of the clock

Smoothing does not literally delay anything - a centred moving average has zero
phase lag - but it blurs onsets, so the visually salient moment of a mouth
opening moves later even though the mean does not. Animators also draw mouth
shapes a frame or two ahead of the sound as standard practice, so this is the
normal control rather than a workaround.

Only the performance tracks shift; the head and audio stay put, since it is the
mouth that should anticipate. In photo-underlay modes the vector mouth will
therefore no longer match the frame behind it, which is expected.

The lead is baked into the exported take - key f carries the pose from source
frame f+lead - so the Animator Pro renderer never needs to know about it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

README.md

@@ -57,6 +57,7 @@ is hand-drawn head plates, which this tool does not yet do.
 | Knob | What it does |
 | --- | --- |
 | vertices | Lip vertex budget. The reduction past what the footage supports *is* the style. |
+| mouth lead ±f | Shifts the performance tracks earlier against the audio and the head. `[` `]`. |
 | contour avg ±f | Radius in frames. 0 off, 1 = ±1. Removes per-frame landmark jitter. |
 | anchor avg ±f | Radius on the four similarity parameters. Smooths the *transform*. |
 | closed-mouth cut | Aperture below which the mouth interior is emitted as `hidden`. |
@@ -134,6 +135,14 @@ their own plate drawing. Everything starts kept; delete what you don't want.
 **Suggest** runs error-tolerance decimation over head pose as a starting point,
 then you hand-correct.
 
+**Mouth lead** is not a correction for a bug. A centred moving average has no
+phase lag, so smoothing does not delay anything — but it blurs onsets, and the
+visually salient moment of a mouth opening moves later even though the mean does
+not. Animators also draw mouth shapes one or two frames ahead of the sound as
+standard practice. Only the performance tracks shift; the head stays with the
+audio, since it is the mouth that should anticipate. The lead is baked into the
+exported take, so the renderer never needs to know about it.
+
 `contour avg` is a deliberate, bounded exception to "never smooth the contour" in
 `../docs/roto-puppet.md`. That rule held while keys were sparse, because sampling
 at velocity minima rejected detector noise for free. With a key on every frame it

index.html

@@ -109,6 +109,7 @@
     <input type="range" id="scrub" min="0" max="0" value="0" style="width:100%;margin-top:10px">
     <div class="legend">
       <kbd>←</kbd> <kbd>→</kbd> step · <kbd>X</kbd> delete · <kbd>K</kbd> keep · <kbd>B</kbd> background ·
+      <kbd>[</kbd> <kbd>]</kbd> lead ·
       click to select · double-click or shift-click to toggle ·
       the mouth keeps <b>every</b> frame regardless
     </div>
@@ -119,6 +120,7 @@
     <div class="panel" style="flex:1 1 400px">
       <h2>knobs</h2>
       <label class="ctl"><span>vertices</span><input type="range" id="verts" min="4" max="16" step="2" value="8"><output id="vertsv"></output></label>
+      <label class="ctl"><span>mouth lead ±f</span><input type="range" id="lead" min="-6" max="6" value="0"><output id="leadv"></output></label>
       <label class="ctl"><span>contour avg ±f</span><input type="range" id="contourSmooth" min="0" max="4" value="1"><output id="contourSmoothv"></output></label>
       <label class="ctl"><span>anchor avg ±f</span><input type="range" id="smoothWin" min="0" max="8" value="2"><output id="smoothWinv"></output></label>
       <label class="ctl"><span>closed-mouth cut</span><input type="range" id="apertureThresh" min="0" max="400" value="120"><output id="apertureThreshv"></output></label>
@@ -132,6 +134,10 @@
       <label class="ctl"><span>teeth dwell</span><input type="range" id="teethDwell" min="0" max="6" value="1"><output id="teethDwellv"></output></label>
       <label class="ctl"><span>suggest tolerance</span><input type="range" id="tol" min="2" max="60" value="14"><output id="tolv"></output></label>
       <div class="legend">
+        <b>mouth lead</b> shifts the performance tracks earlier (positive) against
+        the audio and the head. Averaging has no phase lag but it blurs onsets, so
+        an opening reads later than it is; animators also draw mouths a frame or
+        two ahead of the sound as standard practice. <kbd>[</kbd> <kbd>]</kbd>.<br>
         <b>contour avg</b> 0 = off, 1 = ±1 frame. Removes per-frame landmark
         jitter. Push past 2 and it starts eating articulation.<br>
         <b>anchor avg</b> smooths the head transform only — never the contour.<br>

js/app.js

@@ -39,6 +39,7 @@ const el = (id) => {
 };
 const opts = () => ({
   verts: +el('verts').value,
+  lead: +el('lead').value,
   teethOn: +el('teethOn').value / 100,       // minimum Otsu class separation
   teethDwell: +el('teethDwell').value,
   teethSmooth: +el('teethSmooth').value,
@@ -282,10 +283,11 @@ function renderFrame(f, mode = plateMode()) {
     if (mode === 'oval' || mode === 'oval+photo') r.fillPoly(state.plates[pf], IDX.base);
   }
 
-  r.fillPoly(state.outer[f], IDX.dark);    // mouth keeps every frame
+  const mf = leadIndex(f);                 // performance frame, possibly ahead
-  if (!state.hidden[f]) {
+  r.fillPoly(state.outer[mf], IDX.dark);   // mouth keeps every frame
-    r.fillPoly(state.inner[f], IDX.mouth);
+  if (!state.hidden[mf]) {
-    const te = state.teeth[f];
+    r.fillPoly(state.inner[mf], IDX.mouth);
+    const te = state.teeth[mf];
     if (te.show && te.pts && te.pts.length >= 3) r.fillPoly(te.pts, IDX.teeth);
   }
   return r;
@@ -332,6 +334,21 @@ function compositeRender(canvas, f, zoom) {
 
 const keptSorted = () => [...state.keep].sort((a, b) => a - b);
 
+// Performance tracks can lead the clock.
+//
+// A centred moving average has no phase lag, so smoothing does not literally
+// delay anything - but it blurs onsets, and the visually salient moment of a
+// mouth opening moves later even though the mean does not. Animators also draw
+// mouth shapes one or two frames ahead of the sound as a matter of course, so
+// this is the normal control rather than a correction.
+//
+// Positive lead = the mouth arrives earlier. Only performance parts shift; the
+// head stays with the audio, because it is the mouth that should anticipate.
+function leadIndex(f) {
+  const N = state.dense.length;
+  return Math.min(N - 1, Math.max(0, f + opts().lead));
+}
+
 function blit(canvas, raster, zoom) {
   canvas.width = RW * zoom; canvas.height = RH * zoom;
   canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0);
@@ -347,10 +364,12 @@ function drawAll() {
 function drawReadout() {
   const kept = keptSorted();
   const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k);
+  const lead = opts().lead;
   const teethFrames = state.teeth ? state.teeth.filter((t) => t.show).length : 0;
   el('readout').textContent =
     `${state.dense.length} frames → ${kept.length} drawings · ` +
     `teeth on ${teethFrames}f · ` +
+    (lead ? `mouth leads ${lead}f (${(lead / state.fps * 1000).toFixed(0)}ms) · ` : '') +
     `holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` +
     `neutral f${state.neutral} · residual ` +
     `${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`;
@@ -359,8 +378,10 @@ function drawReadout() {
 function drawPanes() {
   const f = state.frame, kept = keptSorted();
   const pf = heldFrame(kept, f);
+  const lead = opts().lead;
   el('framelabel').textContent =
     `f ${f} / ${state.dense.length - 1} · ${(f / state.fps).toFixed(2)}s · plate f${pf}` +
+    (lead ? ` · mouth f${leadIndex(f)} (${lead > 0 ? '+' : ''}${lead})` : '') +
     (state.keep.has(f) ? ' · KEPT' : ' · held');
 
   const c1 = el('cv-source'), g1 = c1.getContext('2d');
@@ -392,9 +413,10 @@ function drawPanes() {
   g2.moveTo(c2.width / 2, 0); g2.lineTo(c2.width / 2, c2.height);
   g2.moveTo(0, c2.height / 2); g2.lineTo(c2.width, c2.height / 2); g2.stroke();
   const z = (pts) => pts.map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM }));
+  const mf = leadIndex(f);
   strokePts(g2, z(state.plates[pf]), '#3b4a63');
-  strokePts(g2, z(state.outer[f]), '#4ade80');
+  strokePts(g2, z(state.outer[mf]), '#4ade80');
-  if (!state.hidden[f]) strokePts(g2, z(state.inner[f]), '#f87171');
+  if (!state.hidden[mf]) strokePts(g2, z(state.inner[mf]), '#f87171');
 
   compositeRender(el('cv-render'), f, ZOOM);
   drawInteriorDebug(f);
@@ -403,7 +425,8 @@ function drawPanes() {
 // What the teeth measurement actually saw: sampled region, pixels above
 // threshold in green, the resolved line in amber. Recomputed for the current
 // frame only, so it costs nothing to keep on screen.
-function drawInteriorDebug(f) {
+function drawInteriorDebug(fRaw) {
+  const f = leadIndex(fRaw);
   const host = el('cv-teeth');
   const img = state.images[f];
   if (!img || state.hidden[f]) {
@@ -518,14 +541,21 @@ function exportTake() {
       // Sparse: one plate key per frame a human draws.
       { name: 'head', kind: 'plate', z: 0, interp: 'hold',
         keys: kept.map((f, i) => ({ f: i, plate: i, src: f })) },
-      // Dense: the traced mouth keeps every frame.
+      // Dense: the traced mouth keeps every frame. The lead is baked in here -
+      // key f carries the pose from source frame f+lead - so the renderer never
+      // needs to know about it.
       { name: 'mouth', kind: 'poly', z: 30, color: 'skin_dark', interp: 'hold',
-        keys: state.outer.map((pts, f) => ({ f, src: f, pts })) },
+        keys: state.outer.map((_, f) => ({ f, src: leadIndex(f), pts: state.outer[leadIndex(f)] })) },
       { name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
-        keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) },
+        keys: state.inner.map((_, f) => {
+          const m = leadIndex(f);
+          return state.hidden[m] ? { f, hidden: true } : { f, src: m, pts: state.inner[m] };
+        }) },
       { name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in',
-        keys: state.teeth.map((te, f) =>
+        keys: state.teeth.map((_, f) => {
-          (te.show && te.pts ? { f, src: f, pts: te.pts } : { f, hidden: true })) },
+          const m = leadIndex(f), te = state.teeth[m];
+          return te.show && te.pts ? { f, src: m, pts: te.pts } : { f, hidden: true };
+        }) },
     ],
   };
   const text = writeTake(take)
@@ -592,7 +622,7 @@ function runSynthetic() {
 
 for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol',
                   'teethOn', 'teethDwell', 'teethErode', 'tongueReject', 'blobGrow',
-                  'topBias', 'teethVerts', 'teethSmooth']) {
+                  'topBias', 'teethVerts', 'teethSmooth', 'lead']) {
   el(id).addEventListener('input', () => {
     el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol'
       ? (+el(id).value / 1000).toFixed(3)
@@ -666,6 +696,11 @@ window.addEventListener('keydown', (e) => {
   else if (e.key === 'ArrowLeft') { seekTo(Math.max(0, state.frame - 1)); drawAll(); }
   else if (e.key === 'Backspace' || e.key === 'Delete' || e.key === 'x') {
     state.keep.delete(state.frame === 0 ? -1 : state.frame); drawAll();
+  } else if (e.key === '[' || e.key === ']') {
+    const n = el('lead');
+    n.value = Math.max(+n.min, Math.min(+n.max, +n.value + (e.key === ']' ? 1 : -1)));
+    el('leadv').textContent = n.value;
+    drawAll();
   } else if (e.key === 'b') {
     const sel = el('plateMode');
     sel.selectedIndex = (sel.selectedIndex + 1) % sel.options.length;

manifest.json

@@ -1 +1 @@
-{"fps":12,"frames":37,"dir":"frames","audio":"audio.wav","source":"IMG_8486.MOV"}
+{"fps":24,"frames":74,"dir":"frames","audio":"audio.wav","source":"IMG_8486.MOV"}