From a082208ad584f6054724e14e70392abf11ef8555 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 24 Sep 2026 14:51:15 -0400 Subject: [PATCH] Frame removal for plates; mouth keeps every frame Sparseness was being applied for two different reasons at once. Aesthetic sparseness is set by the extraction rate; labour sparseness only binds on the plate, because a human draws each one. The mouth is traced and therefore free, and in limited animation lip sync is routinely the densest element - on 1s while heads hold on 2s and 3s. So: the mouth gets a key on every frame, and the frame strip is now the editing surface for deciding which frames need their own plate drawing. All frames start kept; delete the ones you don't want. - strip of face-cropped thumbnails, keep/drop per frame, keyboard driven - worksheet panel lists the drawings needed and the range each one holds - Suggest runs error-tolerance decimation on head pose as a starting point - export writes sparse plate keys + dense mouth keys, with a hold manifest - smoothContours: bounded exception to "never smooth the contour", which held only while keys were sparse enough to reject detector noise by sampling - averages are now a RADIUS in frames: 0 is off, 1 is +-1 - GPU delegate falls back to CPU instead of failing - #synth / #frames autorun for headless smoke tests Co-Authored-By: Claude Opus 5 --- gl.html | 6 + index.html | 146 +++++++++-------- js/app.js | 434 ++++++++++++++++++++++++++++--------------------- js/mathutil.js | 21 ++- js/pipeline.js | 66 +++++++- js/selftest.js | 2 +- probe.html | 79 +++++++++ 7 files changed, 489 insertions(+), 265 deletions(-) create mode 100644 gl.html create mode 100644 probe.html diff --git a/gl.html b/gl.html new file mode 100644 index 0000000..b2b042c --- /dev/null +++ b/gl.html @@ -0,0 +1,6 @@ +
?
diff --git a/index.html b/index.html index 9ff3d62..dfad8d9 100644 --- a/index.html +++ b/index.html @@ -5,57 +5,68 @@ roto — take builder
-

roto — video → take builder

- - +

roto — frame removal + take builder

+ + - + + + +
@@ -70,7 +81,7 @@

stabilised (head-local)

-
live contour · — active key pose · should sit still except the mouth
+
should sit still except the mouth · grey = held plate outline

flat render — 320×200 indexed

@@ -79,19 +90,35 @@
+
+

frames — green kept (gets its own drawing) · dim held from the last kept frame

+
+ +
+ ← → step · X delete · K keep · + click to select · double-click or shift-click to toggle · + the mouth keeps every frame regardless +
+
+
+
-
+

knobs

- - - - - + + -
+ +
+ contour avg 0 = off, 1 = ±1 frame. Removes per-frame landmark + jitter. Push past 2 and it starts eating articulation.
+ anchor avg smooths the head transform only — never the contour.
+ suggest tolerance only affects the Suggest button: max head movement + allowed before a new drawing is required. +
-
+

palette

@@ -102,18 +129,7 @@
-

timeline

- - -
- velocity curve · | candidate minima · - | key as rendered (f) · - | true extreme (src) — a large gap means min-hold and exposure are fighting -
-
- -
-

selected poses — the vocabulary the take actually contains

+

drawings needed — one per kept frame, with the range it holds

diff --git a/js/app.js b/js/app.js index b7b1e23..10c6b03 100644 --- a/js/app.js +++ b/js/app.js @@ -1,11 +1,11 @@ import { FaceLandmarker, FilesetResolver } from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/vision_bundle.mjs'; import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js'; -import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js'; +import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js'; import { IndexedRaster } from './raster.js'; import { writeTake } from './take.js'; import { synthDense } from './synth.js'; -const RW = 320, RH = 200, ZOOM = 2; +const RW = 320, RH = 200, ZOOM = 2, THUMB = 92; const PALETTE = [ { name: 'bg', hex: '#12141c' }, @@ -18,44 +18,37 @@ const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, lite: 4 }; const state = { dense: null, images: [], stab: null, xform: null, - plate: null, neutral: 0, keysOuter: null, keysInner: null, - frame: 0, playing: false, source: 'none', + outer: null, inner: null, plates: null, hidden: null, + keep: new Set(), // frames that get their own plate drawing + frame: 0, playing: false, faceBox: null, }; const el = (id) => document.getElementById(id); const opts = () => ({ verts: +el('verts').value, - minHold: +el('minHold').value, - distThresh: +el('distThresh').value / 10, - velSmooth: +el('velSmooth').value, smoothWin: +el('smoothWin').value, - exposure: +el('exposure').value, + contourSmooth: +el('contourSmooth').value, apertureThresh: +el('apertureThresh').value / 1000, + tol: +el('tol').value / 1000, }); function status(msg, kind = '') { - const s = el('status'); - s.textContent = msg; - s.className = kind; + el('status').textContent = msg; + el('status').className = kind; } -/* ---------- frame loading ---------- */ +/* ---------- loading ---------- */ -function loadImage(src) { - return new Promise((res) => { - const im = new Image(); - im.onload = () => res(im); - im.onerror = () => res(null); - im.src = src; - }); -} +const loadImage = (src) => new Promise((r) => { + const im = new Image(); + im.onload = () => r(im); im.onerror = () => r(null); im.src = src; +}); async function loadFrameSequence() { const dir = el('framedir').value.replace(/\/$/, ''); const imgs = []; for (let i = 1; i <= 900; i++) { - const name = `${dir}/${String(i).padStart(4, '0')}.png`; - const im = await loadImage(name); + const im = await loadImage(`${dir}/${String(i).padStart(4, '0')}.png`); if (!im) break; imgs.push(im); if (i % 10 === 0) status(`loading frames… ${i}`); @@ -63,184 +56,188 @@ async function loadFrameSequence() { return imgs; } -/* ---------- detection ---------- */ - let landmarker = null; - async function initLandmarker() { if (landmarker) return landmarker; status('loading MediaPipe wasm…'); const fileset = await FilesetResolver.forVisionTasks( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/wasm'); - landmarker = await FaceLandmarker.createFromOptions(fileset, { - baseOptions: { modelAssetPath: './face_landmarker.task', delegate: 'GPU' }, - runningMode: 'IMAGE', - numFaces: 1, - }); - return landmarker; + // GPU is faster but unavailable in some contexts; fall back rather than fail. + for (const delegate of ['GPU', 'CPU']) { + try { + landmarker = await FaceLandmarker.createFromOptions(fileset, { + baseOptions: { modelAssetPath: './face_landmarker.task', delegate }, + runningMode: 'IMAGE', numFaces: 1, + }); + status(`landmarker ready (${delegate})`); + return landmarker; + } catch (e) { + if (delegate === 'CPU') throw e; + console.warn('GPU delegate failed, falling back to CPU:', e.message); + } + } } async function detectAll(images) { const lm = await initLandmarker(); const cv = document.createElement('canvas'); - const dense = []; - const missing = []; + const dense = [], missing = []; for (let i = 0; i < images.length; i++) { const im = images[i]; cv.width = im.naturalWidth; cv.height = im.naturalHeight; cv.getContext('2d').drawImage(im, 0, 0); const out = lm.detect(cv); - if (out.faceLandmarks && out.faceLandmarks.length) { - dense.push(out.faceLandmarks[0]); - } else { - // Hold the previous frame rather than dropping it, so frame indices stay - // aligned with the source sequence. A gap is reported, not hidden. - missing.push(i); - dense.push(dense.length ? dense[dense.length - 1] : null); - } - if (i % 5 === 0) status(`detecting… ${i + 1}/${images.length}`); + if (out.faceLandmarks && out.faceLandmarks.length) dense.push(out.faceLandmarks[0]); + else { missing.push(i); dense.push(dense.length ? dense[dense.length - 1] : null); } + if (i % 4 === 0) status(`detecting… ${i + 1}/${images.length}`); } if (dense[0] === null) throw new Error('no face found in the first frame'); return { dense, missing }; } -/* ---------- calibration ---------- */ +/* ---------- build ---------- */ -// v1 calibration: fit the reference face oval's bounding box to a target box on -// the character raster. Identity retargeting - it makes any clip frame sensibly, -// but a real project replaces this with a measured performer->character map. -function makeXform(stab) { - const oval = stab.oval[state.neutral]; +function makeXform(stab, neutral) { + const oval = stab.oval[neutral]; let x0 = Infinity, y0 = Infinity, x1 = -Infinity, y1 = -Infinity; for (const p of oval) { x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y); x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y); } - const targetH = RH * 0.80; - const s = targetH / (y1 - y0); + const s = (RH * 0.80) / (y1 - y0); const cx = (x0 + x1) / 2, cy = (y0 + y1) / 2; return (p) => ({ x: (p.x - cx) * s + RW / 2, y: (p.y - cy) * s + RH / 2 }); } -/* ---------- build ---------- */ - -function rebuild() { +function rebuild(resetKeep) { if (!state.dense) return; const o = opts(); + const N = state.dense.length; state.stab = stabilize(state.dense, o.smoothWin); - // Neutral frame = most closed mouth in the first quarter of the shot, which is - // where the performer is asked to hold a neutral closed mouth. const ap = state.stab.aperture; - const head = Math.max(1, Math.floor(ap.length / 4)); - let best = 0; - for (let i = 0; i < head; i++) if (ap[i] < ap[best]) best = i; - state.neutral = best; + const head = Math.max(1, Math.floor(N / 4)); + let neutral = 0; + for (let i = 0; i < head; i++) if (ap[i] < ap[neutral]) neutral = i; + state.neutral = neutral; + state.xform = makeXform(state.stab, neutral); - state.xform = makeXform(state.stab); - state.plate = state.stab.oval[state.neutral].map(state.xform); - - const outer = state.stab.outer.map((r) => toRasterRing(r, LIPS_OUTER, o.verts, state.xform)); - const inner = state.stab.inner.map((r) => toRasterRing(r, LIPS_INNER, o.verts, state.xform)); - - state.shapesOuter = outer; - state.shapesInner = inner; - state.keysOuter = selectKeys(outer, o); - // The interior is a child of the lip silhouette: it keys on exactly its - // parent's frames, never independently, or it swims. - state.keysInner = { - ...state.keysOuter, - keys: state.keysOuter.keys.map((k) => ({ ...k })), - }; + // Mouth is traced, so it costs nothing: a key on EVERY frame. Only the plate, + // which a human draws, gets decimated. + state.outer = smoothContours( + state.stab.outer.map((r) => toRasterRing(r, LIPS_OUTER, o.verts, state.xform)), o.contourSmooth); + state.inner = smoothContours( + state.stab.inner.map((r) => toRasterRing(r, LIPS_INNER, o.verts, state.xform)), o.contourSmooth); const apMax = Math.max(...ap); - state.hidden = state.keysOuter.keys.map((k) => ap[k.src] / apMax < o.apertureThresh); + state.hidden = ap.map((v) => v / apMax < o.apertureThresh); - el('readout').textContent = - `${state.dense.length} frames · ${state.keysOuter.candidates.length} candidates → ` + - `${state.keysOuter.keys.length} keys · ${o.verts} verts · neutral f${state.neutral} · ` + - `residual ${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`; + // Plate outline per frame, so a kept frame shows its own head shape. + state.plates = state.stab.oval.map((r) => r.map(state.xform)); - drawTimeline(); - drawContactSheet(); - draw(); + if (resetKeep || !state.keep.size) { + state.keep = new Set(Array.from({ length: N }, (_, i) => i)); + } + // Frame 0 must always be kept: something has to be on screen at the start. + state.keep.add(0); + + state.faceBox = faceBoxes(); + drawAll(); } -/* ---------- drawing ---------- */ +// Face bounding box per frame in image space, for legible strip thumbnails. +function faceBoxes() { + return state.dense.map((lm) => { + let x0 = 1, y0 = 1, x1 = 0, y1 = 0; + for (const i of FACE_OVAL) { + x0 = Math.min(x0, lm[i].x); y0 = Math.min(y0, lm[i].y); + x1 = Math.max(x1, lm[i].x); y1 = Math.max(y1, lm[i].y); + } + const mx = (x1 - x0) * 0.18, my = (y1 - y0) * 0.14; + return { x0: x0 - mx, y0: y0 - my, x1: x1 + mx, y1: y1 + my }; + }); +} + +/* ---------- render ---------- */ function renderFrame(f) { const r = new IndexedRaster(RW, RH); r.clear(IDX.bg); - if (state.plate) r.fillPoly(state.plate, IDX.base); - - const ko = activeKey(state.keysOuter.keys, f); - const ki = activeKey(state.keysInner.keys, f); - if (ko) r.fillPoly(state.shapesOuter[ko.src], IDX.dark); - if (ki) { - const slot = state.keysOuter.keys.indexOf(ki); - if (!state.hidden[slot]) r.fillPoly(state.shapesInner[ki.src], IDX.mouth); - } + const pf = heldFrame(keptSorted(), f); // the plate that is on screen + r.fillPoly(state.plates[pf], IDX.base); + r.fillPoly(state.outer[f], IDX.dark); // mouth at full rate + if (!state.hidden[f]) r.fillPoly(state.inner[f], IDX.mouth); return r; } +const keptSorted = () => [...state.keep].sort((a, b) => a - b); + function blit(canvas, raster, zoom) { canvas.width = RW * zoom; canvas.height = RH * zoom; canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0); } -function draw() { - const f = state.frame; - el('framelabel').textContent = - `f ${String(f).padStart(3)} / ${state.dense.length - 1} ` + - `key ${activeKey(state.keysOuter.keys, f).f}`; +function drawAll() { + drawPanes(); + drawStrip(); + drawWorksheet(); + drawReadout(); +} + +function drawReadout() { + const kept = keptSorted(); + const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k); + el('readout').textContent = + `${state.dense.length} frames → ${kept.length} drawings · ` + + `holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` + + `neutral f${state.neutral} · residual ` + + `${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`; +} + +function drawPanes() { + const f = state.frame, kept = keptSorted(); + const pf = heldFrame(kept, f); + el('framelabel').textContent = + `f ${f} / ${state.dense.length - 1} · plate from f${pf}` + + (state.keep.has(f) ? ' · KEPT' : ' · held'); - // pane 1: source with the raw contour overlaid const c1 = el('cv-source'), g1 = c1.getContext('2d'); c1.width = RW * ZOOM; c1.height = RH * ZOOM; g1.fillStyle = '#000'; g1.fillRect(0, 0, c1.width, c1.height); const im = state.images[f]; if (im) { - const s = Math.min(c1.width / im.naturalWidth, c1.height / im.naturalHeight); - const w = im.naturalWidth * s, h = im.naturalHeight * s; - g1.drawImage(im, (c1.width - w) / 2, (c1.height - h) / 2, w, h); - g1.save(); - g1.translate((c1.width - w) / 2, (c1.height - h) / 2); - strokeRing(g1, LIPS_OUTER.map((i) => state.dense[f][i]), w, h, '#4ade80'); - strokeRing(g1, LIPS_INNER.map((i) => state.dense[f][i]), w, h, '#f87171'); - g1.restore(); + const b = state.faceBox[f]; + const sx = b.x0 * im.naturalWidth, sy = b.y0 * im.naturalHeight; + const sw = (b.x1 - b.x0) * im.naturalWidth, sh = (b.y1 - b.y0) * im.naturalHeight; + const s = Math.min(c1.width / sw, c1.height / sh); + const dw = sw * s, dh = sh * s, dx = (c1.width - dw) / 2, dy = (c1.height - dh) / 2; + g1.drawImage(im, sx, sy, sw, sh, dx, dy, dw, dh); + const map = (p) => ({ x: dx + (p.x * im.naturalWidth - sx) * s, y: dy + (p.y * im.naturalHeight - sy) * s }); + strokePts(g1, LIPS_OUTER.map((i) => map(state.dense[f][i])), '#4ade80'); + strokePts(g1, LIPS_INNER.map((i) => map(state.dense[f][i])), '#f87171'); } else { g1.fillStyle = '#555'; g1.font = '13px system-ui'; g1.fillText('synthetic — no source frames', 14, 24); - strokeRing(g1, LIPS_OUTER.map((i) => state.dense[f][i]), c1.width, c1.height, '#4ade80'); - strokeRing(g1, LIPS_INNER.map((i) => state.dense[f][i]), c1.width, c1.height, '#f87171'); + const sc = (p) => ({ x: p.x * c1.width, y: p.y * c1.height }); + strokePts(g1, LIPS_OUTER.map((i) => sc(state.dense[f][i])), '#4ade80'); + strokePts(g1, LIPS_INNER.map((i) => sc(state.dense[f][i])), '#f87171'); } - // pane 2: stabilised contour against a fixed reference cross. - // If stabilisation works, this contour stays put except for mouth motion. const c2 = el('cv-stab'), g2 = c2.getContext('2d'); c2.width = RW * ZOOM; c2.height = RH * ZOOM; g2.fillStyle = '#0d0f16'; g2.fillRect(0, 0, c2.width, c2.height); - g2.strokeStyle = '#2a2f3e'; g2.lineWidth = 1; - g2.beginPath(); + g2.strokeStyle = '#2a2f3e'; g2.lineWidth = 1; g2.beginPath(); g2.moveTo(c2.width / 2, 0); g2.lineTo(c2.width / 2, c2.height); - g2.moveTo(0, c2.height / 2); g2.lineTo(c2.width, c2.height / 2); - g2.stroke(); - const pl = (pts) => pts.map((p) => { const q = state.xform(p); return { x: q.x * ZOOM, y: q.y * ZOOM }; }); - strokePts(g2, pl(state.stab.oval[f]), '#3b4a63'); - strokePts(g2, pl(state.stab.outer[f]), '#4ade80'); - strokePts(g2, pl(state.stab.inner[f]), '#f87171'); - const ko = activeKey(state.keysOuter.keys, f); - strokePts(g2, state.shapesOuter[ko.src].map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM })), '#fbbf24', 2); + g2.moveTo(0, c2.height / 2); g2.lineTo(c2.width, c2.height / 2); g2.stroke(); + const z = (pts) => pts.map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM })); + strokePts(g2, z(state.plates[pf]), '#3b4a63'); + strokePts(g2, z(state.outer[f]), '#4ade80'); + if (!state.hidden[f]) strokePts(g2, z(state.inner[f]), '#f87171'); - // pane 3: the flat indexed render blit(el('cv-render'), renderFrame(f), ZOOM); } -function strokeRing(g, pts, w, h, color) { - strokePts(g, pts.map((p) => ({ x: p.x * w, y: p.y * h })), color); -} - function strokePts(g, pts, color, lw = 1) { g.strokeStyle = color; g.lineWidth = lw; g.beginPath(); @@ -248,51 +245,80 @@ function strokePts(g, pts, color, lw = 1) { g.closePath(); g.stroke(); } -function drawTimeline() { - const c = el('cv-timeline'), g = c.getContext('2d'); +/* ---------- the frame strip: this is the editing surface ---------- */ + +function drawStrip() { + const host = el('strip'); + host.innerHTML = ''; const N = state.dense.length; - const W = Math.max(640, N * 8), H = 90; - c.width = W; c.height = H; - const px = W / N; - g.fillStyle = '#0d0f16'; g.fillRect(0, 0, W, H); + for (let f = 0; f < N; f++) { + const cell = document.createElement('div'); + cell.className = 'fr' + (state.keep.has(f) ? ' keep' : ' drop') + (f === state.frame ? ' cur' : ''); + cell.dataset.f = f; - const vel = state.keysOuter.velocity; - const vmax = Math.max(...vel) || 1; - g.strokeStyle = '#3b4a63'; g.lineWidth = 1; g.beginPath(); - vel.forEach((v, i) => { - const x = i * px + px / 2, y = H - 18 - (v / vmax) * (H - 34); - i ? g.lineTo(x, y) : g.moveTo(x, y); - }); - g.stroke(); - - g.fillStyle = '#5b6478'; - for (const t of state.keysOuter.candidates) g.fillRect(t * px + px / 2 - 0.5, H - 18, 1, 6); - - for (const k of state.keysOuter.keys) { - g.fillStyle = '#fbbf24'; - g.fillRect(k.f * px + px / 2 - 1, 6, 2, H - 24); // f: what renders - if (k.src !== k.f) { - g.fillStyle = '#f87171'; - g.fillRect(k.src * px + px / 2 - 0.5, 6, 1, 10); // src: the true extreme + const cv = document.createElement('canvas'); + const im = state.images[f]; + cv.width = THUMB; cv.height = THUMB; + const g = cv.getContext('2d'); + g.fillStyle = '#0d0f16'; g.fillRect(0, 0, THUMB, THUMB); + if (im) { + const b = state.faceBox[f]; + const sx = b.x0 * im.naturalWidth, sy = b.y0 * im.naturalHeight; + const sw = (b.x1 - b.x0) * im.naturalWidth, sh = (b.y1 - b.y0) * im.naturalHeight; + const s = Math.min(THUMB / sw, THUMB / sh); + g.drawImage(im, sx, sy, sw, sh, (THUMB - sw * s) / 2, (THUMB - sh * s) / 2, sw * s, sh * s); + } else { + const r = new IndexedRaster(RW, RH); + r.clear(IDX.bg); r.fillPoly(state.plates[f], IDX.base); + const tmp = document.createElement('canvas'); + blit(tmp, r, 1); + g.drawImage(tmp, 0, 0, RW, RH, 0, 0, THUMB, THUMB * (RH / RW)); } + + const tag = document.createElement('span'); + tag.textContent = f; + cell.append(cv, tag); + cell.onclick = (ev) => { + state.frame = f; + if (ev.shiftKey) toggle(f); + drawAll(); + }; + cell.ondblclick = () => { toggle(f); drawAll(); }; + host.append(cell); } - g.fillStyle = '#e8eaf0'; - g.fillRect(state.frame * px + px / 2 - 0.5, 0, 1, H); } -function drawContactSheet() { +function toggle(f) { + if (f === 0) return; // frame 0 always has a drawing + if (state.keep.has(f)) state.keep.delete(f); else state.keep.add(f); +} + +/* ---------- worksheet: the frames a human must draw ---------- */ + +function drawWorksheet() { const host = el('sheet'); host.innerHTML = ''; - state.keysOuter.keys.forEach((k, i) => { + const kept = keptSorted(); + kept.forEach((f, i) => { + const until = (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - 1; const cell = document.createElement('div'); cell.className = 'cell'; const cv = document.createElement('canvas'); - blit(cv, renderFrame(k.f), 1); - cv.style.width = '104px'; + const im = state.images[f]; + if (im) { + const b = state.faceBox[f]; + const sx = b.x0 * im.naturalWidth, sy = b.y0 * im.naturalHeight; + const sw = (b.x1 - b.x0) * im.naturalWidth, sh = (b.y1 - b.y0) * im.naturalHeight; + cv.width = 150; cv.height = Math.round(150 * (sh / sw)); + cv.getContext('2d').drawImage(im, sx, sy, sw, sh, 0, 0, cv.width, cv.height); + } else { + blit(cv, renderFrame(f), 1); + cv.style.width = '150px'; + } const cap = document.createElement('span'); - cap.textContent = `f${k.f}${k.src !== k.f ? ` ←${k.src}` : ''}${state.hidden[i] ? ' ·closed' : ''}`; + cap.textContent = `f${f}` + (until > f ? ` → ${until}` : '') + ` (${until - f + 1}f)`; cell.append(cv, cap); - cell.onclick = () => { state.frame = k.f; draw(); drawTimeline(); }; + cell.onclick = () => { state.frame = f; drawAll(); }; host.append(cell); }); } @@ -300,92 +326,118 @@ function drawContactSheet() { /* ---------- export ---------- */ function exportTake() { - const keys = state.keysOuter.keys; + const kept = keptSorted(); + const N = state.dense.length; const take = { name: el('takename').value || 'line_01', - frames: state.dense.length, width: RW, height: RH, - exposure: opts().exposure, + frames: N, width: RW, height: RH, exposure: 1, palette: PALETTE, slot: { x: RW / 2, y: RH / 2 }, parts: [ - { name: 'head', kind: 'plate', z: 0, interp: 'hold', keys: [{ f: 0, plate: 0 }] }, + // Sparse: one plate key per frame a human draws. + { name: 'head', kind: 'plate', z: 0, interp: 'hold', + keys: kept.map((f, i) => ({ f: i, plate: i, src: f })) }, + // Dense: the traced mouth keeps every frame. { name: 'mouth', kind: 'poly', z: 30, color: 'skin_dark', interp: 'hold', - keys: keys.map((k) => ({ f: k.f, src: k.src, pts: state.shapesOuter[k.src] })) }, + keys: state.outer.map((pts, f) => ({ f, src: f, pts })) }, { name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth', - keys: keys.map((k, i) => state.hidden[i] - ? { f: k.f, hidden: true } - : { f: k.f, src: k.src, pts: state.shapesInner[k.src] }) }, + keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) }, ], }; - const text = writeTake(take); + const text = writeTake(take) + + `\n# plate drawings needed: ${kept.length} of ${N} frames\n` + + kept.map((f, i) => { + const until = (i + 1 < kept.length ? kept[i + 1] : N) - 1; + return `# plate ${i} = source frame ${f}, holds f${f}..${until}`; + }).join('\n') + '\n'; const a = document.createElement('a'); a.href = URL.createObjectURL(new Blob([text], { type: 'text/plain' })); a.download = `${take.name}.take`; a.click(); - status(`exported ${take.name}.take — ${keys.length} keys`, 'ok'); + status(`exported — ${kept.length} plate drawings, ${N} mouth frames`, 'ok'); } /* ---------- wiring ---------- */ async function runFrames() { try { - status('loading frames…'); const images = await loadFrameSequence(); if (!images.length) { - status(`no frames found in ${el('framedir').value}/ — run extract.sh first`, 'err'); + status(`no frames in ${el('framedir').value}/ — run extract.sh first`, 'err'); return; } const { dense, missing } = await detectAll(images); - state.images = images; state.dense = dense; state.source = 'video'; + state.images = images; state.dense = dense; el('scrub').max = dense.length - 1; state.frame = 0; - rebuild(); + rebuild(true); status(missing.length ? `${images.length} frames · no face on ${missing.length} (held previous)` : `${images.length} frames detected`, missing.length ? 'warn' : 'ok'); - } catch (e) { - status(`${e.message}`, 'err'); - console.error(e); - } + } catch (e) { status(e.message, 'err'); console.error(e); } } function runSynthetic() { state.images = []; state.dense = synthDense(72); - state.source = 'synthetic'; el('scrub').max = 71; state.frame = 0; - rebuild(); - status('synthetic take — no video needed; exercises the whole chain below detection', 'ok'); + rebuild(true); + status('synthetic — exercises everything below detection', 'ok'); } -for (const id of ['verts','minHold','distThresh','velSmooth','smoothWin','exposure','apertureThresh']) { +for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol']) { el(id).addEventListener('input', () => { - el(id + 'v').textContent = id === 'distThresh' ? (+el(id).value / 10).toFixed(1) - : id === 'apertureThresh' ? (+el(id).value / 1000).toFixed(3) - : el(id).value; - rebuild(); + el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol' + ? (+el(id).value / 1000).toFixed(3) : el(id).value; + if (id === 'tol') return; // tol only matters when you ask for a suggestion + rebuild(false); }); el(id + 'v').textContent = el(id).value; } -el('scrub').addEventListener('input', (e) => { state.frame = +e.target.value; draw(); drawTimeline(); }); + +el('scrub').addEventListener('input', (e) => { state.frame = +e.target.value; drawAll(); }); el('btn-frames').onclick = runFrames; el('btn-synth').onclick = runSynthetic; el('btn-export').onclick = exportTake; +el('btn-keepall').onclick = () => { if (state.dense) { rebuild(true); } }; +el('btn-suggest').onclick = () => { + if (!state.dense) return; + const kept = suggestPlateFrames(state.stab.rigid, opts().tol); + state.keep = new Set(kept); + drawAll(); + status(`suggested ${kept.length} drawings at tolerance ${opts().tol.toFixed(3)} — now hand-correct`, 'ok'); +}; el('btn-play').onclick = () => { state.playing = !state.playing; el('btn-play').textContent = state.playing ? 'Stop' : 'Play'; if (state.playing) tick(); }; +// Keyboard is the point: stepping and deleting 37 frames by mouse is miserable. +window.addEventListener('keydown', (e) => { + if (!state.dense || e.target.tagName === 'INPUT') return; + const N = state.dense.length; + if (e.key === 'ArrowRight') { state.frame = Math.min(N - 1, state.frame + 1); drawAll(); } + else if (e.key === 'ArrowLeft') { state.frame = Math.max(0, state.frame - 1); drawAll(); } + else if (e.key === 'Backspace' || e.key === 'Delete' || e.key === 'x') { + state.keep.delete(state.frame === 0 ? -1 : state.frame); drawAll(); + } else if (e.key === 'k' || e.key === ' ') { + if (state.frame !== 0) state.keep.add(state.frame); + drawAll(); + } else return; + el('scrub').value = state.frame; + e.preventDefault(); +}); + let last = 0; function tick(ts = 0) { if (!state.playing) return; - if (ts - last > 1000 / 24) { + if (ts - last > 1000 / +el('playfps').value) { last = ts; state.frame = (state.frame + 1) % state.dense.length; el('scrub').value = state.frame; - draw(); drawTimeline(); + drawPanes(); drawStrip(); } requestAnimationFrame(tick); } @@ -395,9 +447,15 @@ PALETTE.forEach((p) => { sw.className = 'sw'; const inp = document.createElement('input'); inp.type = 'color'; inp.value = p.hex; - inp.oninput = () => { p.hex = inp.value; if (state.dense) { draw(); drawContactSheet(); } }; + inp.oninput = () => { p.hex = inp.value; if (state.dense) drawAll(); }; sw.append(inp, document.createTextNode(p.name)); el('palette').append(sw); }); -status('ready — "Synthetic take" works with no video; "Load frames" reads ./frames/'); +// #synth / #frames autorun, so the tool can be driven headlessly for smoke tests +// and deep-linked. Detection needs WebGL; the synthetic path does not. +if (location.hash === '#synth') runSynthetic(); +else if (location.hash === '#frames') runFrames(); +else status('ready — Load frames, then step with \u2190 \u2192 and delete with X'); + +window.__roto = state; // headless smoke test reads this diff --git a/js/mathutil.js b/js/mathutil.js index e8c6418..310eb8c 100644 --- a/js/mathutil.js +++ b/js/mathutil.js @@ -74,9 +74,12 @@ export function procrustesMean(framesRigid, iters = 3) { return ref; } -function movingAverage(vals, win) { - if (win <= 1) return vals.slice(); - const half = Math.floor(win / 2), out = new Array(vals.length); +// `radius` is in frames either side: 0 is off, 1 averages over 3 frames, 2 over +// 5. Expressed as a radius rather than a window so that "off" is 0 and every +// value is symmetric - an even window would be lopsided in time. +function movingAverage(vals, radius) { + if (radius <= 0) return vals.slice(); + const half = Math.floor(radius), out = new Array(vals.length); for (let i = 0; i < vals.length; i++) { let acc = 0, cnt = 0; for (let j = i - half; j <= i + half; j++) { @@ -95,12 +98,12 @@ export { movingAverage }; // where the low-pass belongs; smoothing the contour would destroy the // performance, which is the entire asset. // Angles are smoothed as (cos, sin) so wrapping cannot produce a spike. -export function smoothTransforms(tfs, win) { - const c = movingAverage(tfs.map((t) => Math.cos(t.theta)), win); - const sn = movingAverage(tfs.map((t) => Math.sin(t.theta)), win); - const s = movingAverage(tfs.map((t) => t.s), win); - const tx = movingAverage(tfs.map((t) => t.tx), win); - const ty = movingAverage(tfs.map((t) => t.ty), win); +export function smoothTransforms(tfs, radius) { + const c = movingAverage(tfs.map((t) => Math.cos(t.theta)), radius); + const sn = movingAverage(tfs.map((t) => Math.sin(t.theta)), radius); + const s = movingAverage(tfs.map((t) => t.s), radius); + const tx = movingAverage(tfs.map((t) => t.tx), radius); + const ty = movingAverage(tfs.map((t) => t.ty), radius); return tfs.map((_, i) => ({ theta: Math.atan2(sn[i], c[i]), s: s[i], tx: tx[i], ty: ty[i], })); diff --git a/js/pipeline.js b/js/pipeline.js index a98065c..90ae12f 100644 --- a/js/pipeline.js +++ b/js/pipeline.js @@ -10,15 +10,19 @@ const pick = (lm, idx) => idx.map((i) => ({ x: lm[i].x, y: lm[i].y })); // Stage 1-3: fit the rigid transform per frame, smooth its parameters, then map // every contour through it into the reference frame. The result is head-local: // translation, roll and depth-scale of the head are gone. -export function stabilize(dense, smoothWin) { +export function stabilize(dense, smoothRadius) { const rigid = dense.map((f) => pick(f, RIGID)); const ref = procrustesMean(rigid); const raw = rigid.map((r) => fitSimilarity(r, ref)); - const tfs = smoothTransforms(raw, smoothWin); + const tfs = smoothTransforms(raw, smoothRadius); return { ref, transforms: tfs, + // Rigid landmarks in IMAGE space: the head-pose signal. Frame removal is + // decided from head motion, not from the mouth, so this has to survive the + // fit rather than being consumed by it. + rigid, // Residual rises with out-of-plane rotation, which no 2D similarity can // remove. High values mean this section wants a different head plate. residual: tfs.map((tf, i) => fitResidual(tf, rigid[i], ref)), @@ -106,3 +110,61 @@ export function activeKey(keys, f) { for (const k of keys) { if (k.f <= f) hit = k; else break; } return hit; } + +// Temporal smoothing of a contour, per vertex, across time. +// +// docs/roto-puppet.md says to smooth the transform and never the contour. That +// was correct while keys were sparse: sampling at velocity minima rejected +// per-frame detector noise for free. With a key on every frame the noise is +// visible as a shimmer along the lip edge, so a bounded exception applies - +// the window must stay SHORTER than the shortest articulation worth keeping. +// At 12fps, mouth movement spans 3-6 frames and detector noise is per-frame, so +// a radius of 1 separates them and a radius of 3 would start eating speech. +// +// `radius` in frames either side: 0 off, 1 = 3-frame average, 2 = 5-frame. +export function smoothContours(rings, radius) { + if (radius <= 0) return rings; + const half = Math.floor(radius), N = rings.length, V = rings[0].length; + const out = []; + for (let t = 0; t < N; t++) { + const frame = []; + for (let v = 0; v < V; v++) { + let sx = 0, sy = 0, c = 0; + for (let j = t - half; j <= t + half; j++) { + const k = Math.min(N - 1, Math.max(0, j)); + sx += rings[k][v].x; sy += rings[k][v].y; c++; + } + frame.push({ x: sx / c, y: sy / c }); + } + out.push(frame); + } + return out; +} + +// Which frames need their own PLATE drawing. +// +// This is frame removal, not keyframe extraction: every frame is a candidate and +// the question is which can be dropped. Walk forward holding the current drawing +// until the head has moved further than `tol` from it, then a new drawing is +// required. The cost being managed is an artist drawing a head, which is why the +// signal is head pose and not the mouth - the mouth is traced and free. +export function suggestPlateFrames(rigid, tol) { + const dist = (a, b) => { + let m = 0; + for (let i = 0; i < a.length; i++) m = Math.max(m, Math.hypot(a[i].x - b[i].x, a[i].y - b[i].y)); + return m; + }; + const keep = [0]; + let anchor = 0; + for (let f = 1; f < rigid.length; f++) { + if (dist(rigid[f], rigid[anchor]) > tol) { keep.push(f); anchor = f; } + } + return keep; +} + +// Nearest kept frame at or before f - the plate that is on screen. +export function heldFrame(kept, f) { + let hit = kept[0]; + for (const k of kept) { if (k <= f) hit = k; else break; } + return hit; +} diff --git a/js/selftest.js b/js/selftest.js index 64e366a..af45d75 100644 --- a/js/selftest.js +++ b/js/selftest.js @@ -91,7 +91,7 @@ export function run() { `s=${got.s.toFixed(6)} th=${got.theta.toFixed(6)}`); // stabilisation: head motion out, mouth motion kept - const stab = stabilize(dense, 1); + const stab = stabilize(dense, 0); const rawSpread = spreadX(dense, 133); const stabSpread = (() => { const xs = stab.eyes.map((e) => e[0].x); diff --git a/probe.html b/probe.html new file mode 100644 index 0000000..06b6493 --- /dev/null +++ b/probe.html @@ -0,0 +1,79 @@ +probe… +
running…
+