import { FaceLandmarker, FilesetResolver } from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/vision_bundle.mjs'; import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js'; import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js'; import { IndexedRaster } from './raster.js'; import { drawRegistered, posterizeInto } from './underlay.js'; import { writeTake } from './take.js'; import { synthDense } from './synth.js'; const RW = 320, RH = 200, ZOOM = 2, THUMB = 92; const PALETTE = [ { name: 'bg', hex: '#12141c' }, { name: 'skin_base', hex: '#b07a5a' }, { name: 'skin_dark', hex: '#7a4f3a' }, { name: 'mouth_dark', hex: '#24161a' }, { name: 'skin_lite', hex: '#d9a884' }, ]; const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, lite: 4 }; const state = { dense: null, images: [], stab: null, xform: null, outer: null, inner: null, plates: null, hidden: null, keep: new Set(), // frames that get their own plate drawing frame: 0, playing: false, faceBox: null, fps: 12, audio: null, // fps comes from manifest.json, never guessed }; const el = (id) => document.getElementById(id); const opts = () => ({ verts: +el('verts').value, smoothWin: +el('smoothWin').value, contourSmooth: +el('contourSmooth').value, apertureThresh: +el('apertureThresh').value / 1000, tol: +el('tol').value / 1000, }); function status(msg, kind = '') { el('status').textContent = msg; el('status').className = kind; } /* ---------- loading ---------- */ const loadImage = (src) => new Promise((r) => { const im = new Image(); im.onload = () => r(im); im.onerror = () => r(null); im.src = src; }); // The extraction rate is read, not assumed. Guessing it would desynchronise // audio from picture, which is the one thing this view exists to show. async function loadManifest() { try { const r = await fetch('./manifest.json', { cache: 'no-store' }); if (!r.ok) return null; return await r.json(); } catch { return null; } } function attachAudio(name) { const a = el('audio'); if (!name) { a.removeAttribute('src'); a.hidden = true; state.audio = null; return; } a.src = './' + name; a.hidden = false; state.audio = a; } async function loadFrameSequence() { const dir = el('framedir').value.replace(/\/$/, ''); const imgs = []; for (let i = 1; i <= 900; i++) { const im = await loadImage(`${dir}/${String(i).padStart(4, '0')}.png`); if (!im) break; imgs.push(im); if (i % 10 === 0) status(`loading frames… ${i}`); } return imgs; } let landmarker = null; async function initLandmarker() { if (landmarker) return landmarker; status('loading MediaPipe wasm…'); const fileset = await FilesetResolver.forVisionTasks( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/wasm'); // GPU is faster but unavailable in some contexts; fall back rather than fail. for (const delegate of ['GPU', 'CPU']) { try { landmarker = await FaceLandmarker.createFromOptions(fileset, { baseOptions: { modelAssetPath: './face_landmarker.task', delegate }, runningMode: 'IMAGE', numFaces: 1, }); status(`landmarker ready (${delegate})`); return landmarker; } catch (e) { if (delegate === 'CPU') throw e; console.warn('GPU delegate failed, falling back to CPU:', e.message); } } } async function detectAll(images) { const lm = await initLandmarker(); const cv = document.createElement('canvas'); const dense = [], missing = []; for (let i = 0; i < images.length; i++) { const im = images[i]; cv.width = im.naturalWidth; cv.height = im.naturalHeight; cv.getContext('2d').drawImage(im, 0, 0); const out = lm.detect(cv); if (out.faceLandmarks && out.faceLandmarks.length) dense.push(out.faceLandmarks[0]); else { missing.push(i); dense.push(dense.length ? dense[dense.length - 1] : null); } if (i % 4 === 0) status(`detecting… ${i + 1}/${images.length}`); } if (dense[0] === null) throw new Error('no face found in the first frame'); return { dense, missing }; } /* ---------- build ---------- */ function makeXform(stab, neutral) { const oval = stab.oval[neutral]; let x0 = Infinity, y0 = Infinity, x1 = -Infinity, y1 = -Infinity; for (const p of oval) { x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y); x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y); } const s = (RH * 0.80) / (y1 - y0); const cx = (x0 + x1) / 2, cy = (y0 + y1) / 2; return (p) => ({ x: (p.x - cx) * s + RW / 2, y: (p.y - cy) * s + RH / 2 }); } function rebuild(resetKeep) { if (!state.dense) return; const o = opts(); const N = state.dense.length; state.stab = stabilize(state.dense, o.smoothWin); const ap = state.stab.aperture; const head = Math.max(1, Math.floor(N / 4)); let neutral = 0; for (let i = 0; i < head; i++) if (ap[i] < ap[neutral]) neutral = i; state.neutral = neutral; state.xform = makeXform(state.stab, neutral); // Mouth is traced, so it costs nothing: a key on EVERY frame. Only the plate, // which a human draws, gets decimated. state.outer = smoothContours( state.stab.outer.map((r) => toRasterRing(r, LIPS_OUTER, o.verts, state.xform)), o.contourSmooth); state.inner = smoothContours( state.stab.inner.map((r) => toRasterRing(r, LIPS_INNER, o.verts, state.xform)), o.contourSmooth); const apMax = Math.max(...ap); state.hidden = ap.map((v) => v / apMax < o.apertureThresh); // Plate outline per frame, so a kept frame shows its own head shape. state.plates = state.stab.oval.map((r) => r.map(state.xform)); if (resetKeep || !state.keep.size) { state.keep = new Set(Array.from({ length: N }, (_, i) => i)); } // Frame 0 must always be kept: something has to be on screen at the start. state.keep.add(0); state.faceBox = faceBoxes(); drawAll(); } // Face bounding box per frame in image space, for legible strip thumbnails. function faceBoxes() { return state.dense.map((lm) => { let x0 = 1, y0 = 1, x1 = 0, y1 = 0; for (const i of FACE_OVAL) { x0 = Math.min(x0, lm[i].x); y0 = Math.min(y0, lm[i].y); x1 = Math.max(x1, lm[i].x); y1 = Math.max(y1, lm[i].y); } const mx = (x1 - x0) * 0.18, my = (y1 - y0) * 0.14; return { x0: x0 - mx, y0: y0 - my, x1: x1 + mx, y1: y1 + my }; }); } /* ---------- render ---------- */ // The plate layer has several representations because its job changes: a flat // shape to judge the mouth against, or a registered photograph to draw over. // Only the latter is any use as reference art, and the generated oval is only a // stand-in until a drawing exists. function renderFrame(f, mode = plateMode()) { const r = new IndexedRaster(RW, RH); const pf = heldFrame(keptSorted(), f); // the plate frame on screen if (mode === 'posterize' && state.images[pf]) { posterizeInto(r, state.images[pf], state.stab.transforms[pf], state.xform, PALETTE.map((p) => p.hex)); } else { r.clear(IDX.bg); if (mode === 'oval' || mode === 'oval+photo') r.fillPoly(state.plates[pf], IDX.base); } r.fillPoly(state.outer[f], IDX.dark); // mouth keeps every frame if (!state.hidden[f]) r.fillPoly(state.inner[f], IDX.mouth); return r; } const plateMode = () => el('plateMode').value; // Photo modes composite under the indexed layer, so the flat shapes stay exactly // as they render while the reference sits behind them. function compositeRender(canvas, f, zoom) { const mode = plateMode(); const pf = heldFrame(keptSorted(), f); const img = state.images[pf]; const showPhoto = img && (mode === 'photo' || mode === 'photo-dim' || mode === 'oval+photo'); canvas.width = RW * zoom; canvas.height = RH * zoom; const g = canvas.getContext('2d'); g.fillStyle = PALETTE[IDX.bg].hex; g.fillRect(0, 0, canvas.width, canvas.height); if (showPhoto) { drawRegistered(g, img, state.stab.transforms[pf], state.xform, zoom, mode === 'photo-dim' ? 0.34 : 1); } const r = renderFrame(f, mode === 'oval+photo' ? 'oval' : (showPhoto ? 'off' : mode)); const img2 = r.toImageData(PALETTE.map((p) => p.hex), zoom); if (showPhoto) { // Keep the photo visible wherever the indexed layer is background. const bg = PALETTE[IDX.bg].hex.replace('#', ''); const br = parseInt(bg.slice(0, 2), 16), bgn = parseInt(bg.slice(2, 4), 16), bb = parseInt(bg.slice(4, 6), 16); const d = img2.data; for (let i = 0; i < d.length; i += 4) { if (d[i] === br && d[i + 1] === bgn && d[i + 2] === bb) d[i + 3] = 0; } const tmp = document.createElement('canvas'); tmp.width = img2.width; tmp.height = img2.height; tmp.getContext('2d').putImageData(img2, 0, 0); g.drawImage(tmp, 0, 0); } else { g.putImageData(img2, 0, 0); } } const keptSorted = () => [...state.keep].sort((a, b) => a - b); function blit(canvas, raster, zoom) { canvas.width = RW * zoom; canvas.height = RH * zoom; canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0); } function drawAll() { drawPanes(); drawStrip(); drawWorksheet(); drawReadout(); } function drawReadout() { const kept = keptSorted(); const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k); el('readout').textContent = `${state.dense.length} frames → ${kept.length} drawings · ` + `holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` + `neutral f${state.neutral} · residual ` + `${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`; } function drawPanes() { const f = state.frame, kept = keptSorted(); const pf = heldFrame(kept, f); el('framelabel').textContent = `f ${f} / ${state.dense.length - 1} · ${(f / state.fps).toFixed(2)}s · plate f${pf}` + (state.keep.has(f) ? ' · KEPT' : ' · held'); const c1 = el('cv-source'), g1 = c1.getContext('2d'); c1.width = RW * ZOOM; c1.height = RH * ZOOM; g1.fillStyle = '#000'; g1.fillRect(0, 0, c1.width, c1.height); const im = state.images[f]; if (im) { const b = state.faceBox[f]; const sx = b.x0 * im.naturalWidth, sy = b.y0 * im.naturalHeight; const sw = (b.x1 - b.x0) * im.naturalWidth, sh = (b.y1 - b.y0) * im.naturalHeight; const s = Math.min(c1.width / sw, c1.height / sh); const dw = sw * s, dh = sh * s, dx = (c1.width - dw) / 2, dy = (c1.height - dh) / 2; g1.drawImage(im, sx, sy, sw, sh, dx, dy, dw, dh); const map = (p) => ({ x: dx + (p.x * im.naturalWidth - sx) * s, y: dy + (p.y * im.naturalHeight - sy) * s }); strokePts(g1, LIPS_OUTER.map((i) => map(state.dense[f][i])), '#4ade80'); strokePts(g1, LIPS_INNER.map((i) => map(state.dense[f][i])), '#f87171'); } else { g1.fillStyle = '#555'; g1.font = '13px system-ui'; g1.fillText('synthetic — no source frames', 14, 24); const sc = (p) => ({ x: p.x * c1.width, y: p.y * c1.height }); strokePts(g1, LIPS_OUTER.map((i) => sc(state.dense[f][i])), '#4ade80'); strokePts(g1, LIPS_INNER.map((i) => sc(state.dense[f][i])), '#f87171'); } const c2 = el('cv-stab'), g2 = c2.getContext('2d'); c2.width = RW * ZOOM; c2.height = RH * ZOOM; g2.fillStyle = '#0d0f16'; g2.fillRect(0, 0, c2.width, c2.height); g2.strokeStyle = '#2a2f3e'; g2.lineWidth = 1; g2.beginPath(); g2.moveTo(c2.width / 2, 0); g2.lineTo(c2.width / 2, c2.height); g2.moveTo(0, c2.height / 2); g2.lineTo(c2.width, c2.height / 2); g2.stroke(); const z = (pts) => pts.map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM })); strokePts(g2, z(state.plates[pf]), '#3b4a63'); strokePts(g2, z(state.outer[f]), '#4ade80'); if (!state.hidden[f]) strokePts(g2, z(state.inner[f]), '#f87171'); compositeRender(el('cv-render'), f, ZOOM); } function strokePts(g, pts, color, lw = 1) { g.strokeStyle = color; g.lineWidth = lw; g.beginPath(); pts.forEach((p, i) => (i ? g.lineTo(p.x, p.y) : g.moveTo(p.x, p.y))); g.closePath(); g.stroke(); } /* ---------- the frame strip: this is the editing surface ---------- */ function drawStrip() { const host = el('strip'); host.innerHTML = ''; const N = state.dense.length; for (let f = 0; f < N; f++) { const cell = document.createElement('div'); cell.className = 'fr' + (state.keep.has(f) ? ' keep' : ' drop') + (f === state.frame ? ' cur' : ''); cell.dataset.f = f; const cv = document.createElement('canvas'); const im = state.images[f]; cv.width = THUMB; cv.height = THUMB; const g = cv.getContext('2d'); g.fillStyle = '#0d0f16'; g.fillRect(0, 0, THUMB, THUMB); if (im) { const b = state.faceBox[f]; const sx = b.x0 * im.naturalWidth, sy = b.y0 * im.naturalHeight; const sw = (b.x1 - b.x0) * im.naturalWidth, sh = (b.y1 - b.y0) * im.naturalHeight; const s = Math.min(THUMB / sw, THUMB / sh); g.drawImage(im, sx, sy, sw, sh, (THUMB - sw * s) / 2, (THUMB - sh * s) / 2, sw * s, sh * s); } else { const r = new IndexedRaster(RW, RH); r.clear(IDX.bg); r.fillPoly(state.plates[f], IDX.base); const tmp = document.createElement('canvas'); blit(tmp, r, 1); g.drawImage(tmp, 0, 0, RW, RH, 0, 0, THUMB, THUMB * (RH / RW)); } const tag = document.createElement('span'); tag.textContent = f; cell.append(cv, tag); cell.onclick = (ev) => { seekTo(f); if (ev.shiftKey) toggle(f); drawAll(); }; cell.ondblclick = () => { toggle(f); drawAll(); }; host.append(cell); } } function toggle(f) { if (f === 0) return; // frame 0 always has a drawing if (state.keep.has(f)) state.keep.delete(f); else state.keep.add(f); } /* ---------- worksheet: the frames a human must draw ---------- */ function drawWorksheet() { const host = el('sheet'); host.innerHTML = ''; const kept = keptSorted(); kept.forEach((f, i) => { const until = (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - 1; const cell = document.createElement('div'); cell.className = 'cell'; // Registered, not raw-cropped: the worksheet frame is in raster space, so a // drawing traced from it is already aligned to the mouth. const cv = document.createElement('canvas'); compositeRender(cv, f, 1); cv.style.width = '176px'; const cap = document.createElement('span'); cap.textContent = `f${f}` + (until > f ? ` → ${until}` : '') + ` (${until - f + 1}f)`; cell.append(cv, cap); cell.onclick = () => { seekTo(f); drawAll(); }; host.append(cell); }); } /* ---------- export ---------- */ function exportTake() { const kept = keptSorted(); const N = state.dense.length; const take = { name: el('takename').value || 'line_01', frames: N, width: RW, height: RH, exposure: 1, fps: state.fps, palette: PALETTE, slot: { x: RW / 2, y: RH / 2 }, parts: [ // Sparse: one plate key per frame a human draws. { name: 'head', kind: 'plate', z: 0, interp: 'hold', keys: kept.map((f, i) => ({ f: i, plate: i, src: f })) }, // Dense: the traced mouth keeps every frame. { name: 'mouth', kind: 'poly', z: 30, color: 'skin_dark', interp: 'hold', keys: state.outer.map((pts, f) => ({ f, src: f, pts })) }, { name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth', keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) }, ], }; const text = writeTake(take) + `\n# plate drawings needed: ${kept.length} of ${N} frames\n` + kept.map((f, i) => { const until = (i + 1 < kept.length ? kept[i + 1] : N) - 1; return `# plate ${i} = source frame ${f}, holds f${f}..${until}`; }).join('\n') + '\n'; const a = document.createElement('a'); a.href = URL.createObjectURL(new Blob([text], { type: 'text/plain' })); a.download = `${take.name}.take`; a.click(); status(`exported — ${kept.length} plate drawings, ${N} mouth frames`, 'ok'); } /* ---------- wiring ---------- */ async function runFrames() { try { const man = await loadManifest(); if (man) { state.fps = man.fps; el('framedir').value = man.dir || 'frames'; attachAudio(man.audio); } else { attachAudio(null); status('no manifest.json — assuming 12fps, no audio. Re-run extract.sh', 'warn'); } const images = await loadFrameSequence(); if (!images.length) { status(`no frames in ${el('framedir').value}/ — run extract.sh first`, 'err'); return; } const { dense, missing } = await detectAll(images); state.images = images; state.dense = dense; el('scrub').max = dense.length - 1; state.frame = 0; rebuild(true); const dur = (dense.length / state.fps).toFixed(2); status(`${images.length} frames · ${state.fps}fps · ${dur}s` + (state.audio ? ' · audio loaded' : ' · no audio') + (missing.length ? ` · no face on ${missing.length} (held previous)` : ''), missing.length ? 'warn' : 'ok'); } catch (e) { status(e.message, 'err'); console.error(e); } } function runSynthetic() { state.images = []; attachAudio(null); state.fps = 12; state.dense = synthDense(72); el('scrub').max = 71; state.frame = 0; rebuild(true); status('synthetic — exercises everything below detection', 'ok'); } for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol']) { el(id).addEventListener('input', () => { el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol' ? (+el(id).value / 1000).toFixed(3) : el(id).value; if (id === 'tol') return; // tol only matters when you ask for a suggestion rebuild(false); }); el(id + 'v').textContent = el(id).value; } function seekTo(f) { state.frame = f; if (state.audio) state.audio.currentTime = f / state.fps; el('scrub').value = f; } el('scrub').addEventListener('input', (e) => { seekTo(+e.target.value); drawAll(); }); el('btn-frames').onclick = runFrames; el('btn-synth').onclick = runSynthetic; el('btn-export').onclick = exportTake; el('plateMode').addEventListener('change', () => { if (state.dense) drawAll(); }); el('btn-saveframe').onclick = () => { if (!state.dense) return; const cv = document.createElement('canvas'); compositeRender(cv, state.frame, 4); // 1280x800: enough to draw on const a = document.createElement('a'); a.href = cv.toDataURL('image/png'); a.download = `${el('takename').value || 'take'}_f${String(state.frame).padStart(4, '0')}.png`; a.click(); status(`saved registered frame f${state.frame} at 4x`, 'ok'); }; el('btn-keepall').onclick = () => { if (state.dense) { rebuild(true); } }; el('btn-suggest').onclick = () => { if (!state.dense) return; const kept = suggestPlateFrames(state.stab.rigid, opts().tol); state.keep = new Set(kept); drawAll(); status(`suggested ${kept.length} drawings at tolerance ${opts().tol.toFixed(3)} — now hand-correct`, 'ok'); }; el('btn-play').onclick = () => { if (!state.dense) return; state.playing = !state.playing; el('btn-play').textContent = state.playing ? 'Stop' : 'Play'; if (state.playing) { if (state.audio) { state.audio.playbackRate = +el('speed').value; // Restart from the top if we are sitting at the end. if (state.frame >= state.dense.length - 1) state.frame = 0; state.audio.currentTime = state.frame / state.fps; state.audio.play().catch((e) => status('audio blocked: ' + e.message, 'warn')); } last = 0; tick(); } else if (state.audio) { state.audio.pause(); } }; el('speed').addEventListener('change', () => { // playbackRate retimes the clock, and the picture follows it for free. if (state.audio) state.audio.playbackRate = +el('speed').value; }); // Keyboard is the point: stepping and deleting 37 frames by mouse is miserable. window.addEventListener('keydown', (e) => { if (!state.dense || e.target.tagName === 'INPUT') return; const N = state.dense.length; if (e.key === 'ArrowRight') { seekTo(Math.min(N - 1, state.frame + 1)); drawAll(); } else if (e.key === 'ArrowLeft') { seekTo(Math.max(0, state.frame - 1)); drawAll(); } else if (e.key === 'Backspace' || e.key === 'Delete' || e.key === 'x') { state.keep.delete(state.frame === 0 ? -1 : state.frame); drawAll(); } else if (e.key === 'b') { const sel = el('plateMode'); sel.selectedIndex = (sel.selectedIndex + 1) % sel.options.length; drawAll(); } else if (e.key === 'k' || e.key === ' ') { if (state.frame !== 0) state.keep.add(state.frame); drawAll(); } else return; e.preventDefault(); }); let last = 0; function tick(ts = 0) { if (!state.playing) return; const N = state.dense.length; let f; if (state.audio) { // Audio is the clock. Deriving the frame from currentTime rather than // counting means a slow render loop drops frames instead of drifting out // of sync, which is the behaviour you want when judging lip sync. f = Math.floor(state.audio.currentTime * state.fps); if (f >= N || state.audio.ended) { state.audio.currentTime = 0; state.audio.play().catch(() => {}); f = 0; } } else { const rate = state.fps * +el('speed').value; if (ts - last < 1000 / rate) { requestAnimationFrame(tick); return; } last = ts; f = (state.frame + 1) % N; } if (f !== state.frame) { state.frame = f; el('scrub').value = f; drawPanes(); drawStrip(); } requestAnimationFrame(tick); } PALETTE.forEach((p) => { const sw = document.createElement('label'); sw.className = 'sw'; const inp = document.createElement('input'); inp.type = 'color'; inp.value = p.hex; inp.oninput = () => { p.hex = inp.value; if (state.dense) drawAll(); }; sw.append(inp, document.createTextNode(p.name)); el('palette').append(sw); }); // #synth / #frames autorun, so the tool can be driven headlessly for smoke tests // and deep-linked. Detection needs WebGL; the synthetic path does not. if (location.hash === '#synth') runSynthetic(); else if (location.hash === '#frames') runFrames(); else status('ready — Load frames, then step with \u2190 \u2192 and delete with X'); window.__roto = state; // headless smoke test reads this