roto: video -> take file builder with interactive tuning
Analysis half of the pipeline in docs/roto-puppet.md. Stabilises a face out of a clip via a similarity fit on rigid landmarks, reduces the lip contour to a fixed vertex budget, selects sparse keys on velocity minima, and previews the result as flat indexed fills so timing can be judged without an Animator Pro render. - landmarks.js ordered lip/oval rings; slot position is vertex identity - mathutil.js closed-form 2D similarity, Procrustes mean, transform smoothing - pipeline.js stabilise -> subsample -> key-select - raster.js indexed scanline fill, no antialiasing - take.js take-file writer - selftest.js 29 assertions, incl. ring simplicity at every vertex budget Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
commit
7a6bdea157
13 changed files with 1277 additions and 0 deletions
403
js/app.js
Normal file
403
js/app.js
Normal file
|
|
@ -0,0 +1,403 @@
|
|||
import { FaceLandmarker, FilesetResolver } from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/vision_bundle.mjs';
|
||||
import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js';
|
||||
import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js';
|
||||
import { IndexedRaster } from './raster.js';
|
||||
import { writeTake } from './take.js';
|
||||
import { synthDense } from './synth.js';
|
||||
|
||||
const RW = 320, RH = 200, ZOOM = 2;
|
||||
|
||||
const PALETTE = [
|
||||
{ name: 'bg', hex: '#12141c' },
|
||||
{ name: 'skin_base', hex: '#b07a5a' },
|
||||
{ name: 'skin_dark', hex: '#7a4f3a' },
|
||||
{ name: 'mouth_dark', hex: '#24161a' },
|
||||
{ name: 'skin_lite', hex: '#d9a884' },
|
||||
];
|
||||
const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, lite: 4 };
|
||||
|
||||
const state = {
|
||||
dense: null, images: [], stab: null, xform: null,
|
||||
plate: null, neutral: 0, keysOuter: null, keysInner: null,
|
||||
frame: 0, playing: false, source: 'none',
|
||||
};
|
||||
|
||||
const el = (id) => document.getElementById(id);
|
||||
const opts = () => ({
|
||||
verts: +el('verts').value,
|
||||
minHold: +el('minHold').value,
|
||||
distThresh: +el('distThresh').value / 10,
|
||||
velSmooth: +el('velSmooth').value,
|
||||
smoothWin: +el('smoothWin').value,
|
||||
exposure: +el('exposure').value,
|
||||
apertureThresh: +el('apertureThresh').value / 1000,
|
||||
});
|
||||
|
||||
function status(msg, kind = '') {
|
||||
const s = el('status');
|
||||
s.textContent = msg;
|
||||
s.className = kind;
|
||||
}
|
||||
|
||||
/* ---------- frame loading ---------- */
|
||||
|
||||
function loadImage(src) {
|
||||
return new Promise((res) => {
|
||||
const im = new Image();
|
||||
im.onload = () => res(im);
|
||||
im.onerror = () => res(null);
|
||||
im.src = src;
|
||||
});
|
||||
}
|
||||
|
||||
async function loadFrameSequence() {
|
||||
const dir = el('framedir').value.replace(/\/$/, '');
|
||||
const imgs = [];
|
||||
for (let i = 1; i <= 900; i++) {
|
||||
const name = `${dir}/${String(i).padStart(4, '0')}.png`;
|
||||
const im = await loadImage(name);
|
||||
if (!im) break;
|
||||
imgs.push(im);
|
||||
if (i % 10 === 0) status(`loading frames… ${i}`);
|
||||
}
|
||||
return imgs;
|
||||
}
|
||||
|
||||
/* ---------- detection ---------- */
|
||||
|
||||
let landmarker = null;
|
||||
|
||||
async function initLandmarker() {
|
||||
if (landmarker) return landmarker;
|
||||
status('loading MediaPipe wasm…');
|
||||
const fileset = await FilesetResolver.forVisionTasks(
|
||||
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/wasm');
|
||||
landmarker = await FaceLandmarker.createFromOptions(fileset, {
|
||||
baseOptions: { modelAssetPath: './face_landmarker.task', delegate: 'GPU' },
|
||||
runningMode: 'IMAGE',
|
||||
numFaces: 1,
|
||||
});
|
||||
return landmarker;
|
||||
}
|
||||
|
||||
async function detectAll(images) {
|
||||
const lm = await initLandmarker();
|
||||
const cv = document.createElement('canvas');
|
||||
const dense = [];
|
||||
const missing = [];
|
||||
for (let i = 0; i < images.length; i++) {
|
||||
const im = images[i];
|
||||
cv.width = im.naturalWidth; cv.height = im.naturalHeight;
|
||||
cv.getContext('2d').drawImage(im, 0, 0);
|
||||
const out = lm.detect(cv);
|
||||
if (out.faceLandmarks && out.faceLandmarks.length) {
|
||||
dense.push(out.faceLandmarks[0]);
|
||||
} else {
|
||||
// Hold the previous frame rather than dropping it, so frame indices stay
|
||||
// aligned with the source sequence. A gap is reported, not hidden.
|
||||
missing.push(i);
|
||||
dense.push(dense.length ? dense[dense.length - 1] : null);
|
||||
}
|
||||
if (i % 5 === 0) status(`detecting… ${i + 1}/${images.length}`);
|
||||
}
|
||||
if (dense[0] === null) throw new Error('no face found in the first frame');
|
||||
return { dense, missing };
|
||||
}
|
||||
|
||||
/* ---------- calibration ---------- */
|
||||
|
||||
// v1 calibration: fit the reference face oval's bounding box to a target box on
|
||||
// the character raster. Identity retargeting - it makes any clip frame sensibly,
|
||||
// but a real project replaces this with a measured performer->character map.
|
||||
function makeXform(stab) {
|
||||
const oval = stab.oval[state.neutral];
|
||||
let x0 = Infinity, y0 = Infinity, x1 = -Infinity, y1 = -Infinity;
|
||||
for (const p of oval) {
|
||||
x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y);
|
||||
x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y);
|
||||
}
|
||||
const targetH = RH * 0.80;
|
||||
const s = targetH / (y1 - y0);
|
||||
const cx = (x0 + x1) / 2, cy = (y0 + y1) / 2;
|
||||
return (p) => ({ x: (p.x - cx) * s + RW / 2, y: (p.y - cy) * s + RH / 2 });
|
||||
}
|
||||
|
||||
/* ---------- build ---------- */
|
||||
|
||||
function rebuild() {
|
||||
if (!state.dense) return;
|
||||
const o = opts();
|
||||
|
||||
state.stab = stabilize(state.dense, o.smoothWin);
|
||||
|
||||
// Neutral frame = most closed mouth in the first quarter of the shot, which is
|
||||
// where the performer is asked to hold a neutral closed mouth.
|
||||
const ap = state.stab.aperture;
|
||||
const head = Math.max(1, Math.floor(ap.length / 4));
|
||||
let best = 0;
|
||||
for (let i = 0; i < head; i++) if (ap[i] < ap[best]) best = i;
|
||||
state.neutral = best;
|
||||
|
||||
state.xform = makeXform(state.stab);
|
||||
state.plate = state.stab.oval[state.neutral].map(state.xform);
|
||||
|
||||
const outer = state.stab.outer.map((r) => toRasterRing(r, LIPS_OUTER, o.verts, state.xform));
|
||||
const inner = state.stab.inner.map((r) => toRasterRing(r, LIPS_INNER, o.verts, state.xform));
|
||||
|
||||
state.shapesOuter = outer;
|
||||
state.shapesInner = inner;
|
||||
state.keysOuter = selectKeys(outer, o);
|
||||
// The interior is a child of the lip silhouette: it keys on exactly its
|
||||
// parent's frames, never independently, or it swims.
|
||||
state.keysInner = {
|
||||
...state.keysOuter,
|
||||
keys: state.keysOuter.keys.map((k) => ({ ...k })),
|
||||
};
|
||||
|
||||
const apMax = Math.max(...ap);
|
||||
state.hidden = state.keysOuter.keys.map((k) => ap[k.src] / apMax < o.apertureThresh);
|
||||
|
||||
el('readout').textContent =
|
||||
`${state.dense.length} frames · ${state.keysOuter.candidates.length} candidates → ` +
|
||||
`${state.keysOuter.keys.length} keys · ${o.verts} verts · neutral f${state.neutral} · ` +
|
||||
`residual ${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`;
|
||||
|
||||
drawTimeline();
|
||||
drawContactSheet();
|
||||
draw();
|
||||
}
|
||||
|
||||
/* ---------- drawing ---------- */
|
||||
|
||||
function renderFrame(f) {
|
||||
const r = new IndexedRaster(RW, RH);
|
||||
r.clear(IDX.bg);
|
||||
if (state.plate) r.fillPoly(state.plate, IDX.base);
|
||||
|
||||
const ko = activeKey(state.keysOuter.keys, f);
|
||||
const ki = activeKey(state.keysInner.keys, f);
|
||||
if (ko) r.fillPoly(state.shapesOuter[ko.src], IDX.dark);
|
||||
if (ki) {
|
||||
const slot = state.keysOuter.keys.indexOf(ki);
|
||||
if (!state.hidden[slot]) r.fillPoly(state.shapesInner[ki.src], IDX.mouth);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
function blit(canvas, raster, zoom) {
|
||||
canvas.width = RW * zoom; canvas.height = RH * zoom;
|
||||
canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0);
|
||||
}
|
||||
|
||||
function draw() {
|
||||
const f = state.frame;
|
||||
el('framelabel').textContent =
|
||||
`f ${String(f).padStart(3)} / ${state.dense.length - 1} ` +
|
||||
`key ${activeKey(state.keysOuter.keys, f).f}`;
|
||||
|
||||
// pane 1: source with the raw contour overlaid
|
||||
const c1 = el('cv-source'), g1 = c1.getContext('2d');
|
||||
c1.width = RW * ZOOM; c1.height = RH * ZOOM;
|
||||
g1.fillStyle = '#000'; g1.fillRect(0, 0, c1.width, c1.height);
|
||||
const im = state.images[f];
|
||||
if (im) {
|
||||
const s = Math.min(c1.width / im.naturalWidth, c1.height / im.naturalHeight);
|
||||
const w = im.naturalWidth * s, h = im.naturalHeight * s;
|
||||
g1.drawImage(im, (c1.width - w) / 2, (c1.height - h) / 2, w, h);
|
||||
g1.save();
|
||||
g1.translate((c1.width - w) / 2, (c1.height - h) / 2);
|
||||
strokeRing(g1, LIPS_OUTER.map((i) => state.dense[f][i]), w, h, '#4ade80');
|
||||
strokeRing(g1, LIPS_INNER.map((i) => state.dense[f][i]), w, h, '#f87171');
|
||||
g1.restore();
|
||||
} else {
|
||||
g1.fillStyle = '#555'; g1.font = '13px system-ui';
|
||||
g1.fillText('synthetic — no source frames', 14, 24);
|
||||
strokeRing(g1, LIPS_OUTER.map((i) => state.dense[f][i]), c1.width, c1.height, '#4ade80');
|
||||
strokeRing(g1, LIPS_INNER.map((i) => state.dense[f][i]), c1.width, c1.height, '#f87171');
|
||||
}
|
||||
|
||||
// pane 2: stabilised contour against a fixed reference cross.
|
||||
// If stabilisation works, this contour stays put except for mouth motion.
|
||||
const c2 = el('cv-stab'), g2 = c2.getContext('2d');
|
||||
c2.width = RW * ZOOM; c2.height = RH * ZOOM;
|
||||
g2.fillStyle = '#0d0f16'; g2.fillRect(0, 0, c2.width, c2.height);
|
||||
g2.strokeStyle = '#2a2f3e'; g2.lineWidth = 1;
|
||||
g2.beginPath();
|
||||
g2.moveTo(c2.width / 2, 0); g2.lineTo(c2.width / 2, c2.height);
|
||||
g2.moveTo(0, c2.height / 2); g2.lineTo(c2.width, c2.height / 2);
|
||||
g2.stroke();
|
||||
const pl = (pts) => pts.map((p) => { const q = state.xform(p); return { x: q.x * ZOOM, y: q.y * ZOOM }; });
|
||||
strokePts(g2, pl(state.stab.oval[f]), '#3b4a63');
|
||||
strokePts(g2, pl(state.stab.outer[f]), '#4ade80');
|
||||
strokePts(g2, pl(state.stab.inner[f]), '#f87171');
|
||||
const ko = activeKey(state.keysOuter.keys, f);
|
||||
strokePts(g2, state.shapesOuter[ko.src].map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM })), '#fbbf24', 2);
|
||||
|
||||
// pane 3: the flat indexed render
|
||||
blit(el('cv-render'), renderFrame(f), ZOOM);
|
||||
}
|
||||
|
||||
function strokeRing(g, pts, w, h, color) {
|
||||
strokePts(g, pts.map((p) => ({ x: p.x * w, y: p.y * h })), color);
|
||||
}
|
||||
|
||||
function strokePts(g, pts, color, lw = 1) {
|
||||
g.strokeStyle = color; g.lineWidth = lw;
|
||||
g.beginPath();
|
||||
pts.forEach((p, i) => (i ? g.lineTo(p.x, p.y) : g.moveTo(p.x, p.y)));
|
||||
g.closePath(); g.stroke();
|
||||
}
|
||||
|
||||
function drawTimeline() {
|
||||
const c = el('cv-timeline'), g = c.getContext('2d');
|
||||
const N = state.dense.length;
|
||||
const W = Math.max(640, N * 8), H = 90;
|
||||
c.width = W; c.height = H;
|
||||
const px = W / N;
|
||||
g.fillStyle = '#0d0f16'; g.fillRect(0, 0, W, H);
|
||||
|
||||
const vel = state.keysOuter.velocity;
|
||||
const vmax = Math.max(...vel) || 1;
|
||||
g.strokeStyle = '#3b4a63'; g.lineWidth = 1; g.beginPath();
|
||||
vel.forEach((v, i) => {
|
||||
const x = i * px + px / 2, y = H - 18 - (v / vmax) * (H - 34);
|
||||
i ? g.lineTo(x, y) : g.moveTo(x, y);
|
||||
});
|
||||
g.stroke();
|
||||
|
||||
g.fillStyle = '#5b6478';
|
||||
for (const t of state.keysOuter.candidates) g.fillRect(t * px + px / 2 - 0.5, H - 18, 1, 6);
|
||||
|
||||
for (const k of state.keysOuter.keys) {
|
||||
g.fillStyle = '#fbbf24';
|
||||
g.fillRect(k.f * px + px / 2 - 1, 6, 2, H - 24); // f: what renders
|
||||
if (k.src !== k.f) {
|
||||
g.fillStyle = '#f87171';
|
||||
g.fillRect(k.src * px + px / 2 - 0.5, 6, 1, 10); // src: the true extreme
|
||||
}
|
||||
}
|
||||
g.fillStyle = '#e8eaf0';
|
||||
g.fillRect(state.frame * px + px / 2 - 0.5, 0, 1, H);
|
||||
}
|
||||
|
||||
function drawContactSheet() {
|
||||
const host = el('sheet');
|
||||
host.innerHTML = '';
|
||||
state.keysOuter.keys.forEach((k, i) => {
|
||||
const cell = document.createElement('div');
|
||||
cell.className = 'cell';
|
||||
const cv = document.createElement('canvas');
|
||||
blit(cv, renderFrame(k.f), 1);
|
||||
cv.style.width = '104px';
|
||||
const cap = document.createElement('span');
|
||||
cap.textContent = `f${k.f}${k.src !== k.f ? ` ←${k.src}` : ''}${state.hidden[i] ? ' ·closed' : ''}`;
|
||||
cell.append(cv, cap);
|
||||
cell.onclick = () => { state.frame = k.f; draw(); drawTimeline(); };
|
||||
host.append(cell);
|
||||
});
|
||||
}
|
||||
|
||||
/* ---------- export ---------- */
|
||||
|
||||
function exportTake() {
|
||||
const keys = state.keysOuter.keys;
|
||||
const take = {
|
||||
name: el('takename').value || 'line_01',
|
||||
frames: state.dense.length, width: RW, height: RH,
|
||||
exposure: opts().exposure,
|
||||
palette: PALETTE,
|
||||
slot: { x: RW / 2, y: RH / 2 },
|
||||
parts: [
|
||||
{ name: 'head', kind: 'plate', z: 0, interp: 'hold', keys: [{ f: 0, plate: 0 }] },
|
||||
{ name: 'mouth', kind: 'poly', z: 30, color: 'skin_dark', interp: 'hold',
|
||||
keys: keys.map((k) => ({ f: k.f, src: k.src, pts: state.shapesOuter[k.src] })) },
|
||||
{ name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
|
||||
keys: keys.map((k, i) => state.hidden[i]
|
||||
? { f: k.f, hidden: true }
|
||||
: { f: k.f, src: k.src, pts: state.shapesInner[k.src] }) },
|
||||
],
|
||||
};
|
||||
const text = writeTake(take);
|
||||
const a = document.createElement('a');
|
||||
a.href = URL.createObjectURL(new Blob([text], { type: 'text/plain' }));
|
||||
a.download = `${take.name}.take`;
|
||||
a.click();
|
||||
status(`exported ${take.name}.take — ${keys.length} keys`, 'ok');
|
||||
}
|
||||
|
||||
/* ---------- wiring ---------- */
|
||||
|
||||
async function runFrames() {
|
||||
try {
|
||||
status('loading frames…');
|
||||
const images = await loadFrameSequence();
|
||||
if (!images.length) {
|
||||
status(`no frames found in ${el('framedir').value}/ — run extract.sh first`, 'err');
|
||||
return;
|
||||
}
|
||||
const { dense, missing } = await detectAll(images);
|
||||
state.images = images; state.dense = dense; state.source = 'video';
|
||||
el('scrub').max = dense.length - 1;
|
||||
state.frame = 0;
|
||||
rebuild();
|
||||
status(missing.length
|
||||
? `${images.length} frames · no face on ${missing.length} (held previous)`
|
||||
: `${images.length} frames detected`, missing.length ? 'warn' : 'ok');
|
||||
} catch (e) {
|
||||
status(`${e.message}`, 'err');
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
function runSynthetic() {
|
||||
state.images = [];
|
||||
state.dense = synthDense(72);
|
||||
state.source = 'synthetic';
|
||||
el('scrub').max = 71;
|
||||
state.frame = 0;
|
||||
rebuild();
|
||||
status('synthetic take — no video needed; exercises the whole chain below detection', 'ok');
|
||||
}
|
||||
|
||||
for (const id of ['verts','minHold','distThresh','velSmooth','smoothWin','exposure','apertureThresh']) {
|
||||
el(id).addEventListener('input', () => {
|
||||
el(id + 'v').textContent = id === 'distThresh' ? (+el(id).value / 10).toFixed(1)
|
||||
: id === 'apertureThresh' ? (+el(id).value / 1000).toFixed(3)
|
||||
: el(id).value;
|
||||
rebuild();
|
||||
});
|
||||
el(id + 'v').textContent = el(id).value;
|
||||
}
|
||||
el('scrub').addEventListener('input', (e) => { state.frame = +e.target.value; draw(); drawTimeline(); });
|
||||
el('btn-frames').onclick = runFrames;
|
||||
el('btn-synth').onclick = runSynthetic;
|
||||
el('btn-export').onclick = exportTake;
|
||||
el('btn-play').onclick = () => {
|
||||
state.playing = !state.playing;
|
||||
el('btn-play').textContent = state.playing ? 'Stop' : 'Play';
|
||||
if (state.playing) tick();
|
||||
};
|
||||
|
||||
let last = 0;
|
||||
function tick(ts = 0) {
|
||||
if (!state.playing) return;
|
||||
if (ts - last > 1000 / 24) {
|
||||
last = ts;
|
||||
state.frame = (state.frame + 1) % state.dense.length;
|
||||
el('scrub').value = state.frame;
|
||||
draw(); drawTimeline();
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
|
||||
PALETTE.forEach((p) => {
|
||||
const sw = document.createElement('label');
|
||||
sw.className = 'sw';
|
||||
const inp = document.createElement('input');
|
||||
inp.type = 'color'; inp.value = p.hex;
|
||||
inp.oninput = () => { p.hex = inp.value; if (state.dense) { draw(); drawContactSheet(); } };
|
||||
sw.append(inp, document.createTextNode(p.name));
|
||||
el('palette').append(sw);
|
||||
});
|
||||
|
||||
status('ready — "Synthetic take" works with no video; "Load frames" reads ./frames/');
|
||||
Loading…
Add table
Add a link
Reference in a new issue