From d18791e6ad4e864933898c47e0fc7aa46d4b6b9b Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 24 Sep 2026 15:34:08 -0400 Subject: [PATCH] Mouth lead: nudge performance tracks ahead of the clock Smoothing does not literally delay anything - a centred moving average has zero phase lag - but it blurs onsets, so the visually salient moment of a mouth opening moves later even though the mean does not. Animators also draw mouth shapes a frame or two ahead of the sound as standard practice, so this is the normal control rather than a workaround. Only the performance tracks shift; the head and audio stay put, since it is the mouth that should anticipate. In photo-underlay modes the vector mouth will therefore no longer match the frame behind it, which is expected. The lead is baked into the exported take - key f carries the pose from source frame f+lead - so the Animator Pro renderer never needs to know about it. Co-Authored-By: Claude Opus 5 --- README.md | 9 ++++++++ index.html | 6 +++++ js/app.js | 61 ++++++++++++++++++++++++++++++++++++++++----------- manifest.json | 2 +- 4 files changed, 64 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 6ced56b..e2a9174 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ is hand-drawn head plates, which this tool does not yet do. | Knob | What it does | | --- | --- | | vertices | Lip vertex budget. The reduction past what the footage supports *is* the style. | +| mouth lead ±f | Shifts the performance tracks earlier against the audio and the head. `[` `]`. | | contour avg ±f | Radius in frames. 0 off, 1 = ±1. Removes per-frame landmark jitter. | | anchor avg ±f | Radius on the four similarity parameters. Smooths the *transform*. | | closed-mouth cut | Aperture below which the mouth interior is emitted as `hidden`. | @@ -134,6 +135,14 @@ their own plate drawing. Everything starts kept; delete what you don't want. **Suggest** runs error-tolerance decimation over head pose as a starting point, then you hand-correct. +**Mouth lead** is not a correction for a bug. A centred moving average has no +phase lag, so smoothing does not delay anything — but it blurs onsets, and the +visually salient moment of a mouth opening moves later even though the mean does +not. Animators also draw mouth shapes one or two frames ahead of the sound as +standard practice. Only the performance tracks shift; the head stays with the +audio, since it is the mouth that should anticipate. The lead is baked into the +exported take, so the renderer never needs to know about it. + `contour avg` is a deliberate, bounded exception to "never smooth the contour" in `../docs/roto-puppet.md`. That rule held while keys were sparse, because sampling at velocity minima rejected detector noise for free. With a key on every frame it diff --git a/index.html b/index.html index b6c44e3..4479db2 100644 --- a/index.html +++ b/index.html @@ -109,6 +109,7 @@
← → step · X delete · K keep · B background · + [ ] lead · click to select · double-click or shift-click to toggle · the mouth keeps every frame regardless
@@ -119,6 +120,7 @@

knobs

+ @@ -132,6 +134,10 @@
+ mouth lead shifts the performance tracks earlier (positive) against + the audio and the head. Averaging has no phase lag but it blurs onsets, so + an opening reads later than it is; animators also draw mouths a frame or + two ahead of the sound as standard practice. [ ].
contour avg 0 = off, 1 = ±1 frame. Removes per-frame landmark jitter. Push past 2 and it starts eating articulation.
anchor avg smooths the head transform only — never the contour.
diff --git a/js/app.js b/js/app.js index e15e444..a213979 100644 --- a/js/app.js +++ b/js/app.js @@ -39,6 +39,7 @@ const el = (id) => { }; const opts = () => ({ verts: +el('verts').value, + lead: +el('lead').value, teethOn: +el('teethOn').value / 100, // minimum Otsu class separation teethDwell: +el('teethDwell').value, teethSmooth: +el('teethSmooth').value, @@ -282,10 +283,11 @@ function renderFrame(f, mode = plateMode()) { if (mode === 'oval' || mode === 'oval+photo') r.fillPoly(state.plates[pf], IDX.base); } - r.fillPoly(state.outer[f], IDX.dark); // mouth keeps every frame - if (!state.hidden[f]) { - r.fillPoly(state.inner[f], IDX.mouth); - const te = state.teeth[f]; + const mf = leadIndex(f); // performance frame, possibly ahead + r.fillPoly(state.outer[mf], IDX.dark); // mouth keeps every frame + if (!state.hidden[mf]) { + r.fillPoly(state.inner[mf], IDX.mouth); + const te = state.teeth[mf]; if (te.show && te.pts && te.pts.length >= 3) r.fillPoly(te.pts, IDX.teeth); } return r; @@ -332,6 +334,21 @@ function compositeRender(canvas, f, zoom) { const keptSorted = () => [...state.keep].sort((a, b) => a - b); +// Performance tracks can lead the clock. +// +// A centred moving average has no phase lag, so smoothing does not literally +// delay anything - but it blurs onsets, and the visually salient moment of a +// mouth opening moves later even though the mean does not. Animators also draw +// mouth shapes one or two frames ahead of the sound as a matter of course, so +// this is the normal control rather than a correction. +// +// Positive lead = the mouth arrives earlier. Only performance parts shift; the +// head stays with the audio, because it is the mouth that should anticipate. +function leadIndex(f) { + const N = state.dense.length; + return Math.min(N - 1, Math.max(0, f + opts().lead)); +} + function blit(canvas, raster, zoom) { canvas.width = RW * zoom; canvas.height = RH * zoom; canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0); @@ -347,10 +364,12 @@ function drawAll() { function drawReadout() { const kept = keptSorted(); const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k); + const lead = opts().lead; const teethFrames = state.teeth ? state.teeth.filter((t) => t.show).length : 0; el('readout').textContent = `${state.dense.length} frames → ${kept.length} drawings · ` + `teeth on ${teethFrames}f · ` + + (lead ? `mouth leads ${lead}f (${(lead / state.fps * 1000).toFixed(0)}ms) · ` : '') + `holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` + `neutral f${state.neutral} · residual ` + `${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`; @@ -359,8 +378,10 @@ function drawReadout() { function drawPanes() { const f = state.frame, kept = keptSorted(); const pf = heldFrame(kept, f); + const lead = opts().lead; el('framelabel').textContent = `f ${f} / ${state.dense.length - 1} · ${(f / state.fps).toFixed(2)}s · plate f${pf}` + + (lead ? ` · mouth f${leadIndex(f)} (${lead > 0 ? '+' : ''}${lead})` : '') + (state.keep.has(f) ? ' · KEPT' : ' · held'); const c1 = el('cv-source'), g1 = c1.getContext('2d'); @@ -392,9 +413,10 @@ function drawPanes() { g2.moveTo(c2.width / 2, 0); g2.lineTo(c2.width / 2, c2.height); g2.moveTo(0, c2.height / 2); g2.lineTo(c2.width, c2.height / 2); g2.stroke(); const z = (pts) => pts.map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM })); + const mf = leadIndex(f); strokePts(g2, z(state.plates[pf]), '#3b4a63'); - strokePts(g2, z(state.outer[f]), '#4ade80'); - if (!state.hidden[f]) strokePts(g2, z(state.inner[f]), '#f87171'); + strokePts(g2, z(state.outer[mf]), '#4ade80'); + if (!state.hidden[mf]) strokePts(g2, z(state.inner[mf]), '#f87171'); compositeRender(el('cv-render'), f, ZOOM); drawInteriorDebug(f); @@ -403,7 +425,8 @@ function drawPanes() { // What the teeth measurement actually saw: sampled region, pixels above // threshold in green, the resolved line in amber. Recomputed for the current // frame only, so it costs nothing to keep on screen. -function drawInteriorDebug(f) { +function drawInteriorDebug(fRaw) { + const f = leadIndex(fRaw); const host = el('cv-teeth'); const img = state.images[f]; if (!img || state.hidden[f]) { @@ -518,14 +541,21 @@ function exportTake() { // Sparse: one plate key per frame a human draws. { name: 'head', kind: 'plate', z: 0, interp: 'hold', keys: kept.map((f, i) => ({ f: i, plate: i, src: f })) }, - // Dense: the traced mouth keeps every frame. + // Dense: the traced mouth keeps every frame. The lead is baked in here - + // key f carries the pose from source frame f+lead - so the renderer never + // needs to know about it. { name: 'mouth', kind: 'poly', z: 30, color: 'skin_dark', interp: 'hold', - keys: state.outer.map((pts, f) => ({ f, src: f, pts })) }, + keys: state.outer.map((_, f) => ({ f, src: leadIndex(f), pts: state.outer[leadIndex(f)] })) }, { name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth', - keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) }, + keys: state.inner.map((_, f) => { + const m = leadIndex(f); + return state.hidden[m] ? { f, hidden: true } : { f, src: m, pts: state.inner[m] }; + }) }, { name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in', - keys: state.teeth.map((te, f) => - (te.show && te.pts ? { f, src: f, pts: te.pts } : { f, hidden: true })) }, + keys: state.teeth.map((_, f) => { + const m = leadIndex(f), te = state.teeth[m]; + return te.show && te.pts ? { f, src: m, pts: te.pts } : { f, hidden: true }; + }) }, ], }; const text = writeTake(take) @@ -592,7 +622,7 @@ function runSynthetic() { for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell', 'teethErode', 'tongueReject', 'blobGrow', - 'topBias', 'teethVerts', 'teethSmooth']) { + 'topBias', 'teethVerts', 'teethSmooth', 'lead']) { el(id).addEventListener('input', () => { el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol' ? (+el(id).value / 1000).toFixed(3) @@ -666,6 +696,11 @@ window.addEventListener('keydown', (e) => { else if (e.key === 'ArrowLeft') { seekTo(Math.max(0, state.frame - 1)); drawAll(); } else if (e.key === 'Backspace' || e.key === 'Delete' || e.key === 'x') { state.keep.delete(state.frame === 0 ? -1 : state.frame); drawAll(); + } else if (e.key === '[' || e.key === ']') { + const n = el('lead'); + n.value = Math.max(+n.min, Math.min(+n.max, +n.value + (e.key === ']' ? 1 : -1))); + el('leadv').textContent = n.value; + drawAll(); } else if (e.key === 'b') { const sel = el('plateMode'); sel.selectedIndex = (sel.selectedIndex + 1) % sel.options.length; diff --git a/manifest.json b/manifest.json index 3efb3c1..9bf7b85 100644 --- a/manifest.json +++ b/manifest.json @@ -1 +1 @@ -{"fps":12,"frames":37,"dir":"frames","audio":"audio.wav","source":"IMG_8486.MOV"} +{"fps":24,"frames":74,"dir":"frames","audio":"audio.wav","source":"IMG_8486.MOV"}