Your Name
941022b69f
MediaPipe has no landmarks inside the lips, so teeth have to come from pixels. Tracing the bright blob would give a new contour every frame with no vertex correspondence - the exact boil docs/roto-puppet.md warns about. So the measurement yields a scalar, not a shape: Otsu within the cavity, scanned from the top for where the bright run stops, giving one line height per frame. The teeth polygon is the inner lip ring clipped to that line, so the silhouette is always the mouth's own shape and cannot disagree with the lips around it, and the only per-frame variable is a single number that smooths trivially. Presence gets hysteresis and minimum dwell, as plate selection does: a teeth block blinking on and off for single frames is worse than one simply absent. Tongue is not implemented. The same scalar approach would apply, gated on redness rather than brightness, but it is not visible in the test footage - the cavity reads dark with a bright upper-teeth band and nothing else. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
671 lines
25 KiB
JavaScript
671 lines
25 KiB
JavaScript
import { FaceLandmarker, FilesetResolver } from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/vision_bundle.mjs';
|
||
import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js';
|
||
import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js';
|
||
import { IndexedRaster } from './raster.js';
|
||
import { drawRegistered, posterizeInto } from './underlay.js';
|
||
import { measureInterior, clipPolyAbove } from './interior.js';
|
||
import { writeTake } from './take.js';
|
||
import { synthDense } from './synth.js';
|
||
|
||
const RW = 320, RH = 200, ZOOM = 2, THUMB = 92;
|
||
|
||
const PALETTE = [
|
||
{ name: 'bg', hex: '#12141c' },
|
||
{ name: 'skin_base', hex: '#b07a5a' },
|
||
{ name: 'skin_dark', hex: '#7a4f3a' },
|
||
{ name: 'mouth_dark', hex: '#24161a' },
|
||
{ name: 'teeth', hex: '#d9cfc2' },
|
||
];
|
||
const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, teeth: 4 };
|
||
|
||
const state = {
|
||
dense: null, images: [], stab: null, xform: null,
|
||
outer: null, inner: null, plates: null, hidden: null,
|
||
keep: new Set(), // frames that get their own plate drawing
|
||
frame: 0, playing: false, faceBox: null,
|
||
fps: 12, audio: null, // fps comes from manifest.json, never guessed
|
||
aspect: 1, // imgW/imgH; converts MediaPipe's anisotropic space
|
||
interior: null, // per-frame teeth measurement from image content
|
||
teeth: null, // resolved per-frame {show, t} after knobs
|
||
};
|
||
|
||
const el = (id) => document.getElementById(id);
|
||
const opts = () => ({
|
||
verts: +el('verts').value,
|
||
teethOn: +el('teethOn').value / 100,
|
||
teethDwell: +el('teethDwell').value,
|
||
smoothWin: +el('smoothWin').value,
|
||
contourSmooth: +el('contourSmooth').value,
|
||
apertureThresh: +el('apertureThresh').value / 1000,
|
||
tol: +el('tol').value / 1000,
|
||
});
|
||
|
||
function status(msg, kind = '') {
|
||
el('status').textContent = msg;
|
||
el('status').className = kind;
|
||
}
|
||
|
||
/* ---------- loading ---------- */
|
||
|
||
const loadImage = (src) => new Promise((r) => {
|
||
const im = new Image();
|
||
im.onload = () => r(im); im.onerror = () => r(null); im.src = src;
|
||
});
|
||
|
||
// The extraction rate is read, not assumed. Guessing it would desynchronise
|
||
// audio from picture, which is the one thing this view exists to show.
|
||
async function loadManifest() {
|
||
try {
|
||
const r = await fetch('./manifest.json', { cache: 'no-store' });
|
||
if (!r.ok) return null;
|
||
return await r.json();
|
||
} catch { return null; }
|
||
}
|
||
|
||
function attachAudio(name) {
|
||
const a = el('audio');
|
||
if (!name) { a.removeAttribute('src'); a.hidden = true; state.audio = null; return; }
|
||
a.src = './' + name;
|
||
a.hidden = false;
|
||
state.audio = a;
|
||
}
|
||
|
||
async function loadFrameSequence() {
|
||
const dir = el('framedir').value.replace(/\/$/, '');
|
||
const imgs = [];
|
||
for (let i = 1; i <= 900; i++) {
|
||
const im = await loadImage(`${dir}/${String(i).padStart(4, '0')}.png`);
|
||
if (!im) break;
|
||
imgs.push(im);
|
||
if (i % 10 === 0) status(`loading frames… ${i}`);
|
||
}
|
||
return imgs;
|
||
}
|
||
|
||
let landmarker = null;
|
||
async function initLandmarker() {
|
||
if (landmarker) return landmarker;
|
||
status('loading MediaPipe wasm…');
|
||
const fileset = await FilesetResolver.forVisionTasks(
|
||
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/wasm');
|
||
// GPU is faster but unavailable in some contexts; fall back rather than fail.
|
||
for (const delegate of ['GPU', 'CPU']) {
|
||
try {
|
||
landmarker = await FaceLandmarker.createFromOptions(fileset, {
|
||
baseOptions: { modelAssetPath: './face_landmarker.task', delegate },
|
||
runningMode: 'IMAGE', numFaces: 1,
|
||
});
|
||
status(`landmarker ready (${delegate})`);
|
||
return landmarker;
|
||
} catch (e) {
|
||
if (delegate === 'CPU') throw e;
|
||
console.warn('GPU delegate failed, falling back to CPU:', e.message);
|
||
}
|
||
}
|
||
}
|
||
|
||
async function detectAll(images) {
|
||
const lm = await initLandmarker();
|
||
const cv = document.createElement('canvas');
|
||
const dense = [], missing = [];
|
||
for (let i = 0; i < images.length; i++) {
|
||
const im = images[i];
|
||
cv.width = im.naturalWidth; cv.height = im.naturalHeight;
|
||
cv.getContext('2d').drawImage(im, 0, 0);
|
||
const out = lm.detect(cv);
|
||
if (out.faceLandmarks && out.faceLandmarks.length) dense.push(out.faceLandmarks[0]);
|
||
else { missing.push(i); dense.push(dense.length ? dense[dense.length - 1] : null); }
|
||
if (i % 4 === 0) status(`detecting… ${i + 1}/${images.length}`);
|
||
}
|
||
if (dense[0] === null) throw new Error('no face found in the first frame');
|
||
return { dense, missing };
|
||
}
|
||
|
||
// Interior measurement is a function of pixels alone, so it runs once with
|
||
// detection and the knobs re-resolve it instantly afterwards.
|
||
function measureAll(images, dense) {
|
||
const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
|
||
return dense.map((lm, i) =>
|
||
measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx));
|
||
}
|
||
|
||
/* ---------- build ---------- */
|
||
|
||
function makeXform(stab, neutral) {
|
||
const oval = stab.oval[neutral];
|
||
let x0 = Infinity, y0 = Infinity, x1 = -Infinity, y1 = -Infinity;
|
||
for (const p of oval) {
|
||
x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y);
|
||
x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y);
|
||
}
|
||
const s = (RH * 0.80) / (y1 - y0);
|
||
const cx = (x0 + x1) / 2, cy = (y0 + y1) / 2;
|
||
return (p) => ({ x: (p.x - cx) * s + RW / 2, y: (p.y - cy) * s + RH / 2 });
|
||
}
|
||
|
||
function rebuild(resetKeep) {
|
||
if (!state.dense) return;
|
||
const o = opts();
|
||
const N = state.dense.length;
|
||
|
||
state.stab = stabilize(state.dense, o.smoothWin, state.aspect);
|
||
|
||
const ap = state.stab.aperture;
|
||
const head = Math.max(1, Math.floor(N / 4));
|
||
let neutral = 0;
|
||
for (let i = 0; i < head; i++) if (ap[i] < ap[neutral]) neutral = i;
|
||
state.neutral = neutral;
|
||
state.xform = makeXform(state.stab, neutral);
|
||
|
||
// Mouth is traced, so it costs nothing: a key on EVERY frame. Only the plate,
|
||
// which a human draws, gets decimated.
|
||
state.outer = smoothContours(
|
||
state.stab.outer.map((r) => toRasterRing(r, LIPS_OUTER, o.verts, state.xform)), o.contourSmooth);
|
||
state.inner = smoothContours(
|
||
state.stab.inner.map((r) => toRasterRing(r, LIPS_INNER, o.verts, state.xform)), o.contourSmooth);
|
||
|
||
const apMax = Math.max(...ap);
|
||
state.hidden = ap.map((v) => v / apMax < o.apertureThresh);
|
||
|
||
state.teeth = resolveTeeth(o);
|
||
|
||
// Plate outline per frame, so a kept frame shows its own head shape.
|
||
state.plates = state.stab.oval.map((r) => r.map(state.xform));
|
||
|
||
if (resetKeep || !state.keep.size) {
|
||
state.keep = new Set(Array.from({ length: N }, (_, i) => i));
|
||
}
|
||
// Frame 0 must always be kept: something has to be on screen at the start.
|
||
state.keep.add(0);
|
||
|
||
state.faceBox = faceBoxes();
|
||
drawAll();
|
||
}
|
||
|
||
// Face bounding box per frame in image space, for legible strip thumbnails.
|
||
function faceBoxes() {
|
||
return state.dense.map((lm) => {
|
||
let x0 = 1, y0 = 1, x1 = 0, y1 = 0;
|
||
for (const i of FACE_OVAL) {
|
||
x0 = Math.min(x0, lm[i].x); y0 = Math.min(y0, lm[i].y);
|
||
x1 = Math.max(x1, lm[i].x); y1 = Math.max(y1, lm[i].y);
|
||
}
|
||
const mx = (x1 - x0) * 0.18, my = (y1 - y0) * 0.14;
|
||
return { x0: x0 - mx, y0: y0 - my, x1: x1 + mx, y1: y1 + my };
|
||
});
|
||
}
|
||
|
||
// Presence gets hysteresis and a minimum dwell, the same treatment plate
|
||
// selection gets: a teeth block that blinks on and off for single frames is
|
||
// worse than one that is simply absent. Appearing needs a clear signal, staying
|
||
// needs only a weak one.
|
||
function resolveTeeth(o) {
|
||
const N = state.dense.length;
|
||
if (!state.interior) return new Array(N).fill({ show: false, t: 0 });
|
||
|
||
const raw = state.interior.map((m, f) => (state.hidden[f] ? 0 : m.coverage));
|
||
const on = o.teethOn, off = o.teethOn * 0.6;
|
||
const shown = new Array(N).fill(false);
|
||
let live = false, since = 0;
|
||
for (let f = 0; f < N; f++) {
|
||
const want = live ? raw[f] > off : raw[f] > on;
|
||
if (want !== live && since >= o.teethDwell) { live = want; since = 0; }
|
||
else since++;
|
||
shown[f] = live && !state.hidden[f];
|
||
}
|
||
|
||
// Smooth the line height only across frames where it is actually shown, so a
|
||
// gap does not drag the band toward zero on either side of it.
|
||
const t = state.interior.map((m) => m.teethT);
|
||
const sm = t.map((_, f) => {
|
||
let acc = 0, c = 0;
|
||
for (let j = f - o.contourSmooth; j <= f + o.contourSmooth; j++) {
|
||
const k = Math.min(N - 1, Math.max(0, j));
|
||
if (!shown[k]) continue;
|
||
acc += t[k]; c++;
|
||
}
|
||
return c ? acc / c : t[f];
|
||
});
|
||
return shown.map((show, f) => ({ show, t: sm[f] }));
|
||
}
|
||
|
||
/* ---------- render ---------- */
|
||
|
||
// The plate layer has several representations because its job changes: a flat
|
||
// shape to judge the mouth against, or a registered photograph to draw over.
|
||
// Only the latter is any use as reference art, and the generated oval is only a
|
||
// stand-in until a drawing exists.
|
||
function renderFrame(f, mode = plateMode()) {
|
||
const r = new IndexedRaster(RW, RH);
|
||
const pf = heldFrame(keptSorted(), f); // the plate frame on screen
|
||
|
||
if (mode === 'posterize' && state.images[pf]) {
|
||
posterizeInto(r, state.images[pf], state.stab.transforms[pf], state.xform,
|
||
PALETTE.map((p) => p.hex));
|
||
} else {
|
||
r.clear(IDX.bg);
|
||
if (mode === 'oval' || mode === 'oval+photo') r.fillPoly(state.plates[pf], IDX.base);
|
||
}
|
||
|
||
r.fillPoly(state.outer[f], IDX.dark); // mouth keeps every frame
|
||
if (!state.hidden[f]) {
|
||
r.fillPoly(state.inner[f], IDX.mouth);
|
||
const te = state.teeth[f];
|
||
if (te.show) {
|
||
// The band is the mouth's own silhouette clipped to a height, so the shape
|
||
// can never disagree with the lips around it.
|
||
const ys = state.inner[f].map((p) => p.y);
|
||
const top = Math.min(...ys), bot = Math.max(...ys);
|
||
const poly = clipPolyAbove(state.inner[f], top + te.t * (bot - top));
|
||
if (poly.length >= 3) r.fillPoly(poly, IDX.teeth);
|
||
}
|
||
}
|
||
return r;
|
||
}
|
||
|
||
const plateMode = () => el('plateMode').value;
|
||
|
||
// Photo modes composite under the indexed layer, so the flat shapes stay exactly
|
||
// as they render while the reference sits behind them.
|
||
function compositeRender(canvas, f, zoom) {
|
||
const mode = plateMode();
|
||
const pf = heldFrame(keptSorted(), f);
|
||
const img = state.images[pf];
|
||
const showPhoto = img && (mode === 'photo' || mode === 'photo-dim' || mode === 'oval+photo');
|
||
|
||
canvas.width = RW * zoom; canvas.height = RH * zoom;
|
||
const g = canvas.getContext('2d');
|
||
g.fillStyle = PALETTE[IDX.bg].hex;
|
||
g.fillRect(0, 0, canvas.width, canvas.height);
|
||
|
||
if (showPhoto) {
|
||
drawRegistered(g, img, state.stab.transforms[pf], state.xform, zoom,
|
||
mode === 'photo-dim' ? 0.34 : 1);
|
||
}
|
||
|
||
const r = renderFrame(f, mode === 'oval+photo' ? 'oval' : (showPhoto ? 'off' : mode));
|
||
const img2 = r.toImageData(PALETTE.map((p) => p.hex), zoom);
|
||
if (showPhoto) {
|
||
// Keep the photo visible wherever the indexed layer is background.
|
||
const bg = PALETTE[IDX.bg].hex.replace('#', '');
|
||
const br = parseInt(bg.slice(0, 2), 16), bgn = parseInt(bg.slice(2, 4), 16), bb = parseInt(bg.slice(4, 6), 16);
|
||
const d = img2.data;
|
||
for (let i = 0; i < d.length; i += 4) {
|
||
if (d[i] === br && d[i + 1] === bgn && d[i + 2] === bb) d[i + 3] = 0;
|
||
}
|
||
const tmp = document.createElement('canvas');
|
||
tmp.width = img2.width; tmp.height = img2.height;
|
||
tmp.getContext('2d').putImageData(img2, 0, 0);
|
||
g.drawImage(tmp, 0, 0);
|
||
} else {
|
||
g.putImageData(img2, 0, 0);
|
||
}
|
||
}
|
||
|
||
const keptSorted = () => [...state.keep].sort((a, b) => a - b);
|
||
|
||
function blit(canvas, raster, zoom) {
|
||
canvas.width = RW * zoom; canvas.height = RH * zoom;
|
||
canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0);
|
||
}
|
||
|
||
function drawAll() {
|
||
drawPanes();
|
||
drawStrip();
|
||
drawWorksheet();
|
||
drawReadout();
|
||
}
|
||
|
||
function drawReadout() {
|
||
const kept = keptSorted();
|
||
const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k);
|
||
const teethFrames = state.teeth ? state.teeth.filter((t) => t.show).length : 0;
|
||
el('readout').textContent =
|
||
`${state.dense.length} frames → ${kept.length} drawings · ` +
|
||
`teeth on ${teethFrames}f · ` +
|
||
`holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` +
|
||
`neutral f${state.neutral} · residual ` +
|
||
`${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`;
|
||
}
|
||
|
||
function drawPanes() {
|
||
const f = state.frame, kept = keptSorted();
|
||
const pf = heldFrame(kept, f);
|
||
el('framelabel').textContent =
|
||
`f ${f} / ${state.dense.length - 1} · ${(f / state.fps).toFixed(2)}s · plate f${pf}` +
|
||
(state.keep.has(f) ? ' · KEPT' : ' · held');
|
||
|
||
const c1 = el('cv-source'), g1 = c1.getContext('2d');
|
||
c1.width = RW * ZOOM; c1.height = RH * ZOOM;
|
||
g1.fillStyle = '#000'; g1.fillRect(0, 0, c1.width, c1.height);
|
||
const im = state.images[f];
|
||
if (im) {
|
||
const b = state.faceBox[f];
|
||
const sx = b.x0 * im.naturalWidth, sy = b.y0 * im.naturalHeight;
|
||
const sw = (b.x1 - b.x0) * im.naturalWidth, sh = (b.y1 - b.y0) * im.naturalHeight;
|
||
const s = Math.min(c1.width / sw, c1.height / sh);
|
||
const dw = sw * s, dh = sh * s, dx = (c1.width - dw) / 2, dy = (c1.height - dh) / 2;
|
||
g1.drawImage(im, sx, sy, sw, sh, dx, dy, dw, dh);
|
||
const map = (p) => ({ x: dx + (p.x * im.naturalWidth - sx) * s, y: dy + (p.y * im.naturalHeight - sy) * s });
|
||
strokePts(g1, LIPS_OUTER.map((i) => map(state.dense[f][i])), '#4ade80');
|
||
strokePts(g1, LIPS_INNER.map((i) => map(state.dense[f][i])), '#f87171');
|
||
} else {
|
||
g1.fillStyle = '#555'; g1.font = '13px system-ui';
|
||
g1.fillText('synthetic — no source frames', 14, 24);
|
||
const sc = (p) => ({ x: p.x * c1.width, y: p.y * c1.height });
|
||
strokePts(g1, LIPS_OUTER.map((i) => sc(state.dense[f][i])), '#4ade80');
|
||
strokePts(g1, LIPS_INNER.map((i) => sc(state.dense[f][i])), '#f87171');
|
||
}
|
||
|
||
const c2 = el('cv-stab'), g2 = c2.getContext('2d');
|
||
c2.width = RW * ZOOM; c2.height = RH * ZOOM;
|
||
g2.fillStyle = '#0d0f16'; g2.fillRect(0, 0, c2.width, c2.height);
|
||
g2.strokeStyle = '#2a2f3e'; g2.lineWidth = 1; g2.beginPath();
|
||
g2.moveTo(c2.width / 2, 0); g2.lineTo(c2.width / 2, c2.height);
|
||
g2.moveTo(0, c2.height / 2); g2.lineTo(c2.width, c2.height / 2); g2.stroke();
|
||
const z = (pts) => pts.map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM }));
|
||
strokePts(g2, z(state.plates[pf]), '#3b4a63');
|
||
strokePts(g2, z(state.outer[f]), '#4ade80');
|
||
if (!state.hidden[f]) strokePts(g2, z(state.inner[f]), '#f87171');
|
||
|
||
compositeRender(el('cv-render'), f, ZOOM);
|
||
}
|
||
|
||
function strokePts(g, pts, color, lw = 1) {
|
||
g.strokeStyle = color; g.lineWidth = lw;
|
||
g.beginPath();
|
||
pts.forEach((p, i) => (i ? g.lineTo(p.x, p.y) : g.moveTo(p.x, p.y)));
|
||
g.closePath(); g.stroke();
|
||
}
|
||
|
||
/* ---------- the frame strip: this is the editing surface ---------- */
|
||
|
||
function drawStrip() {
|
||
const host = el('strip');
|
||
host.innerHTML = '';
|
||
const N = state.dense.length;
|
||
for (let f = 0; f < N; f++) {
|
||
const cell = document.createElement('div');
|
||
cell.className = 'fr' + (state.keep.has(f) ? ' keep' : ' drop') + (f === state.frame ? ' cur' : '');
|
||
cell.dataset.f = f;
|
||
|
||
const cv = document.createElement('canvas');
|
||
const im = state.images[f];
|
||
cv.width = THUMB; cv.height = THUMB;
|
||
const g = cv.getContext('2d');
|
||
g.fillStyle = '#0d0f16'; g.fillRect(0, 0, THUMB, THUMB);
|
||
if (im) {
|
||
const b = state.faceBox[f];
|
||
const sx = b.x0 * im.naturalWidth, sy = b.y0 * im.naturalHeight;
|
||
const sw = (b.x1 - b.x0) * im.naturalWidth, sh = (b.y1 - b.y0) * im.naturalHeight;
|
||
const s = Math.min(THUMB / sw, THUMB / sh);
|
||
g.drawImage(im, sx, sy, sw, sh, (THUMB - sw * s) / 2, (THUMB - sh * s) / 2, sw * s, sh * s);
|
||
} else {
|
||
const r = new IndexedRaster(RW, RH);
|
||
r.clear(IDX.bg); r.fillPoly(state.plates[f], IDX.base);
|
||
const tmp = document.createElement('canvas');
|
||
blit(tmp, r, 1);
|
||
g.drawImage(tmp, 0, 0, RW, RH, 0, 0, THUMB, THUMB * (RH / RW));
|
||
}
|
||
|
||
const tag = document.createElement('span');
|
||
tag.textContent = f;
|
||
cell.append(cv, tag);
|
||
cell.onclick = (ev) => {
|
||
seekTo(f);
|
||
if (ev.shiftKey) toggle(f);
|
||
drawAll();
|
||
};
|
||
cell.ondblclick = () => { toggle(f); drawAll(); };
|
||
host.append(cell);
|
||
}
|
||
}
|
||
|
||
function toggle(f) {
|
||
if (f === 0) return; // frame 0 always has a drawing
|
||
if (state.keep.has(f)) state.keep.delete(f); else state.keep.add(f);
|
||
}
|
||
|
||
/* ---------- worksheet: the frames a human must draw ---------- */
|
||
|
||
function drawWorksheet() {
|
||
const host = el('sheet');
|
||
host.innerHTML = '';
|
||
const kept = keptSorted();
|
||
kept.forEach((f, i) => {
|
||
const until = (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - 1;
|
||
const cell = document.createElement('div');
|
||
cell.className = 'cell';
|
||
// Registered, not raw-cropped: the worksheet frame is in raster space, so a
|
||
// drawing traced from it is already aligned to the mouth.
|
||
const cv = document.createElement('canvas');
|
||
compositeRender(cv, f, 1);
|
||
cv.style.width = '176px';
|
||
const cap = document.createElement('span');
|
||
cap.textContent = `f${f}` + (until > f ? ` → ${until}` : '') + ` (${until - f + 1}f)`;
|
||
cell.append(cv, cap);
|
||
cell.onclick = () => { seekTo(f); drawAll(); };
|
||
host.append(cell);
|
||
});
|
||
}
|
||
|
||
/* ---------- export ---------- */
|
||
|
||
function exportTake() {
|
||
const kept = keptSorted();
|
||
const N = state.dense.length;
|
||
const take = {
|
||
name: el('takename').value || 'line_01',
|
||
frames: N, width: RW, height: RH, exposure: 1, fps: state.fps,
|
||
palette: PALETTE,
|
||
slot: { x: RW / 2, y: RH / 2 },
|
||
parts: [
|
||
// Sparse: one plate key per frame a human draws.
|
||
{ name: 'head', kind: 'plate', z: 0, interp: 'hold',
|
||
keys: kept.map((f, i) => ({ f: i, plate: i, src: f })) },
|
||
// Dense: the traced mouth keeps every frame.
|
||
{ name: 'mouth', kind: 'poly', z: 30, color: 'skin_dark', interp: 'hold',
|
||
keys: state.outer.map((pts, f) => ({ f, src: f, pts })) },
|
||
{ name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
|
||
keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) },
|
||
{ name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in',
|
||
keys: state.inner.map((pts, f) => {
|
||
const te = state.teeth[f];
|
||
if (state.hidden[f] || !te.show) return { f, hidden: true };
|
||
const ys = pts.map((p) => p.y);
|
||
const top = Math.min(...ys), bot = Math.max(...ys);
|
||
return { f, src: f, pts: clipPolyAbove(pts, top + te.t * (bot - top)) };
|
||
}) },
|
||
],
|
||
};
|
||
const text = writeTake(take)
|
||
+ `\n# plate drawings needed: ${kept.length} of ${N} frames\n`
|
||
+ kept.map((f, i) => {
|
||
const until = (i + 1 < kept.length ? kept[i + 1] : N) - 1;
|
||
return `# plate ${i} = source frame ${f}, holds f${f}..${until}`;
|
||
}).join('\n') + '\n';
|
||
const a = document.createElement('a');
|
||
a.href = URL.createObjectURL(new Blob([text], { type: 'text/plain' }));
|
||
a.download = `${take.name}.take`;
|
||
a.click();
|
||
status(`exported — ${kept.length} plate drawings, ${N} mouth frames`, 'ok');
|
||
}
|
||
|
||
/* ---------- wiring ---------- */
|
||
|
||
async function runFrames() {
|
||
try {
|
||
const man = await loadManifest();
|
||
if (man) {
|
||
state.fps = man.fps;
|
||
el('framedir').value = man.dir || 'frames';
|
||
attachAudio(man.audio);
|
||
} else {
|
||
attachAudio(null);
|
||
status('no manifest.json — assuming 12fps, no audio. Re-run extract.sh', 'warn');
|
||
}
|
||
const images = await loadFrameSequence();
|
||
if (!images.length) {
|
||
status(`no frames in ${el('framedir').value}/ — run extract.sh first`, 'err');
|
||
return;
|
||
}
|
||
const { dense, missing } = await detectAll(images);
|
||
state.images = images; state.dense = dense;
|
||
state.aspect = images[0].naturalWidth / images[0].naturalHeight;
|
||
status('measuring mouth interiors…');
|
||
state.interior = measureAll(images, dense);
|
||
el('scrub').max = dense.length - 1;
|
||
state.frame = 0;
|
||
rebuild(true);
|
||
const dur = (dense.length / state.fps).toFixed(2);
|
||
status(`${images.length} frames · ${images[0].naturalWidth}x${images[0].naturalHeight} · ` +
|
||
`${state.fps}fps · ${dur}s` +
|
||
(state.audio ? ' · audio loaded' : ' · no audio') +
|
||
(missing.length ? ` · no face on ${missing.length} (held previous)` : ''),
|
||
missing.length ? 'warn' : 'ok');
|
||
} catch (e) { status(e.message, 'err'); console.error(e); }
|
||
}
|
||
|
||
function runSynthetic() {
|
||
state.images = [];
|
||
attachAudio(null);
|
||
state.fps = 12;
|
||
state.aspect = 1; // synthetic landmarks are generated square
|
||
state.interior = null; // no pixels, so no teeth
|
||
|
||
state.dense = synthDense(72);
|
||
el('scrub').max = 71;
|
||
state.frame = 0;
|
||
rebuild(true);
|
||
status('synthetic — exercises everything below detection', 'ok');
|
||
}
|
||
|
||
for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell']) {
|
||
el(id).addEventListener('input', () => {
|
||
el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol'
|
||
? (+el(id).value / 1000).toFixed(3)
|
||
: id === 'teethOn' ? (+el(id).value / 100).toFixed(2) : el(id).value;
|
||
if (id === 'tol') return; // tol only matters when you ask for a suggestion
|
||
rebuild(false);
|
||
});
|
||
el(id + 'v').textContent = el(id).value;
|
||
}
|
||
|
||
function seekTo(f) {
|
||
state.frame = f;
|
||
if (state.audio) state.audio.currentTime = f / state.fps;
|
||
el('scrub').value = f;
|
||
}
|
||
|
||
el('scrub').addEventListener('input', (e) => { seekTo(+e.target.value); drawAll(); });
|
||
el('btn-frames').onclick = runFrames;
|
||
el('btn-synth').onclick = runSynthetic;
|
||
el('btn-export').onclick = exportTake;
|
||
el('plateMode').addEventListener('change', () => { if (state.dense) drawAll(); });
|
||
el('btn-saveframe').onclick = () => {
|
||
if (!state.dense) return;
|
||
const cv = document.createElement('canvas');
|
||
compositeRender(cv, state.frame, 4); // 1280x800: enough to draw on
|
||
const a = document.createElement('a');
|
||
a.href = cv.toDataURL('image/png');
|
||
a.download = `${el('takename').value || 'take'}_f${String(state.frame).padStart(4, '0')}.png`;
|
||
a.click();
|
||
status(`saved registered frame f${state.frame} at 4x`, 'ok');
|
||
};
|
||
el('btn-keepall').onclick = () => { if (state.dense) { rebuild(true); } };
|
||
el('btn-suggest').onclick = () => {
|
||
if (!state.dense) return;
|
||
const kept = suggestPlateFrames(state.stab.rigid, opts().tol);
|
||
state.keep = new Set(kept);
|
||
drawAll();
|
||
status(`suggested ${kept.length} drawings at tolerance ${opts().tol.toFixed(3)} — now hand-correct`, 'ok');
|
||
};
|
||
el('btn-play').onclick = () => {
|
||
if (!state.dense) return;
|
||
state.playing = !state.playing;
|
||
el('btn-play').textContent = state.playing ? 'Stop' : 'Play';
|
||
if (state.playing) {
|
||
if (state.audio) {
|
||
state.audio.playbackRate = +el('speed').value;
|
||
// Restart from the top if we are sitting at the end.
|
||
if (state.frame >= state.dense.length - 1) state.frame = 0;
|
||
state.audio.currentTime = state.frame / state.fps;
|
||
state.audio.play().catch((e) => status('audio blocked: ' + e.message, 'warn'));
|
||
}
|
||
last = 0;
|
||
tick();
|
||
} else if (state.audio) {
|
||
state.audio.pause();
|
||
}
|
||
};
|
||
|
||
el('speed').addEventListener('change', () => {
|
||
// playbackRate retimes the clock, and the picture follows it for free.
|
||
if (state.audio) state.audio.playbackRate = +el('speed').value;
|
||
});
|
||
|
||
// Keyboard is the point: stepping and deleting 37 frames by mouse is miserable.
|
||
window.addEventListener('keydown', (e) => {
|
||
if (!state.dense || e.target.tagName === 'INPUT') return;
|
||
const N = state.dense.length;
|
||
if (e.key === 'ArrowRight') { seekTo(Math.min(N - 1, state.frame + 1)); drawAll(); }
|
||
else if (e.key === 'ArrowLeft') { seekTo(Math.max(0, state.frame - 1)); drawAll(); }
|
||
else if (e.key === 'Backspace' || e.key === 'Delete' || e.key === 'x') {
|
||
state.keep.delete(state.frame === 0 ? -1 : state.frame); drawAll();
|
||
} else if (e.key === 'b') {
|
||
const sel = el('plateMode');
|
||
sel.selectedIndex = (sel.selectedIndex + 1) % sel.options.length;
|
||
drawAll();
|
||
} else if (e.key === 'k' || e.key === ' ') {
|
||
if (state.frame !== 0) state.keep.add(state.frame);
|
||
drawAll();
|
||
} else return;
|
||
e.preventDefault();
|
||
});
|
||
|
||
let last = 0;
|
||
function tick(ts = 0) {
|
||
if (!state.playing) return;
|
||
const N = state.dense.length;
|
||
let f;
|
||
if (state.audio) {
|
||
// Audio is the clock. Deriving the frame from currentTime rather than
|
||
// counting means a slow render loop drops frames instead of drifting out
|
||
// of sync, which is the behaviour you want when judging lip sync.
|
||
f = Math.floor(state.audio.currentTime * state.fps);
|
||
if (f >= N || state.audio.ended) {
|
||
state.audio.currentTime = 0;
|
||
state.audio.play().catch(() => {});
|
||
f = 0;
|
||
}
|
||
} else {
|
||
const rate = state.fps * +el('speed').value;
|
||
if (ts - last < 1000 / rate) { requestAnimationFrame(tick); return; }
|
||
last = ts;
|
||
f = (state.frame + 1) % N;
|
||
}
|
||
if (f !== state.frame) {
|
||
state.frame = f;
|
||
el('scrub').value = f;
|
||
drawPanes(); drawStrip();
|
||
}
|
||
requestAnimationFrame(tick);
|
||
}
|
||
|
||
PALETTE.forEach((p) => {
|
||
const sw = document.createElement('label');
|
||
sw.className = 'sw';
|
||
const inp = document.createElement('input');
|
||
inp.type = 'color'; inp.value = p.hex;
|
||
inp.oninput = () => { p.hex = inp.value; if (state.dense) drawAll(); };
|
||
sw.append(inp, document.createTextNode(p.name));
|
||
el('palette').append(sw);
|
||
});
|
||
|
||
// #synth / #frames autorun, so the tool can be driven headlessly for smoke tests
|
||
// and deep-linked. Detection needs WebGL; the synthetic path does not.
|
||
if (location.hash === '#synth') runSynthetic();
|
||
else if (location.hash === '#frames') runFrames();
|
||
else status('ready — Load frames, then step with \u2190 \u2192 and delete with X');
|
||
|
||
window.__roto = state; // headless smoke test reads this
|