roto: video -> take file builder with interactive tuning
Analysis half of the pipeline in docs/roto-puppet.md. Stabilises a face out of a clip via a similarity fit on rigid landmarks, reduces the lip contour to a fixed vertex budget, selects sparse keys on velocity minima, and previews the result as flat indexed fills so timing can be judged without an Animator Pro render. - landmarks.js ordered lip/oval rings; slot position is vertex identity - mathutil.js closed-form 2D similarity, Procrustes mean, transform smoothing - pipeline.js stabilise -> subsample -> key-select - raster.js indexed scanline fill, no antialiasing - take.js take-file writer - selftest.js 29 assertions, incl. ring simplicity at every vertex budget Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
commit
7a6bdea157
13 changed files with 1277 additions and 0 deletions
108
js/pipeline.js
Normal file
108
js/pipeline.js
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
// Analysis: dense track -> stabilised head-local contours -> selected keys.
|
||||
// All policy lives here, never in the renderer. See docs/roto-puppet.md,
|
||||
// "The take is the contract".
|
||||
|
||||
import { RIGID, LIPS_OUTER, LIPS_INNER, APERTURE, FACE_OVAL, EYE_INNER, subsampleSlots } from './landmarks.js';
|
||||
import { fitSimilarity, applySimAll, applySim, fitResidual, procrustesMean, smoothTransforms, movingAverage } from './mathutil.js';
|
||||
|
||||
const pick = (lm, idx) => idx.map((i) => ({ x: lm[i].x, y: lm[i].y }));
|
||||
|
||||
// Stage 1-3: fit the rigid transform per frame, smooth its parameters, then map
|
||||
// every contour through it into the reference frame. The result is head-local:
|
||||
// translation, roll and depth-scale of the head are gone.
|
||||
export function stabilize(dense, smoothWin) {
|
||||
const rigid = dense.map((f) => pick(f, RIGID));
|
||||
const ref = procrustesMean(rigid);
|
||||
const raw = rigid.map((r) => fitSimilarity(r, ref));
|
||||
const tfs = smoothTransforms(raw, smoothWin);
|
||||
|
||||
return {
|
||||
ref,
|
||||
transforms: tfs,
|
||||
// Residual rises with out-of-plane rotation, which no 2D similarity can
|
||||
// remove. High values mean this section wants a different head plate.
|
||||
residual: tfs.map((tf, i) => fitResidual(tf, rigid[i], ref)),
|
||||
outer: dense.map((f, i) => applySimAll(tfs[i], pick(f, LIPS_OUTER))),
|
||||
inner: dense.map((f, i) => applySimAll(tfs[i], pick(f, LIPS_INNER))),
|
||||
oval: dense.map((f, i) => applySimAll(tfs[i], pick(f, FACE_OVAL))),
|
||||
eyes: dense.map((f, i) => applySimAll(tfs[i], pick(f, EYE_INNER))),
|
||||
aperture: dense.map((f, i) => {
|
||||
const a = applySimAll(tfs[i], pick(f, APERTURE));
|
||||
return Math.hypot(a[0].x - a[1].x, a[0].y - a[1].y);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
// Stage 4: fixed-index subsample of a stabilised ring, then map from normalised
|
||||
// face space into character raster space.
|
||||
export function toRasterRing(stabRing, ringTable, n, xform) {
|
||||
return subsampleSlots(ringTable.length, n).map((s) => xform(stabRing[s]));
|
||||
}
|
||||
|
||||
// Stage 6: key selection.
|
||||
//
|
||||
// Keys go on velocity MINIMA, not on distance thresholds. A threshold fires at
|
||||
// the frame it was crossed - partway through a transition - so every pose lands
|
||||
// mushy and late. A minimum is where the shape is momentarily parked, which is
|
||||
// the pose a viewer actually reads.
|
||||
//
|
||||
// Minima alone are not enough: during a long hold the velocity wobbles near zero
|
||||
// and produces a key per wobble. So a candidate minimum is only accepted if the
|
||||
// shape has actually moved since the last accepted key (distThresh) and the
|
||||
// minimum hold has elapsed (minHold).
|
||||
export function selectKeys(shapes, opts) {
|
||||
const { minHold, distThresh, velSmooth, exposure } = opts;
|
||||
const N = shapes.length;
|
||||
if (N === 0) return { keys: [], velocity: [], candidates: [] };
|
||||
|
||||
const vel = new Array(N).fill(0);
|
||||
for (let t = 1; t < N; t++) {
|
||||
let acc = 0;
|
||||
for (let i = 0; i < shapes[t].length; i++) {
|
||||
acc += Math.hypot(shapes[t][i].x - shapes[t - 1][i].x, shapes[t][i].y - shapes[t - 1][i].y);
|
||||
}
|
||||
vel[t] = acc / shapes[t].length;
|
||||
}
|
||||
const sv = movingAverage(vel, velSmooth);
|
||||
|
||||
const candidates = [];
|
||||
for (let t = 1; t < N - 1; t++) {
|
||||
if (sv[t] <= sv[t - 1] && sv[t] <= sv[t + 1]) candidates.push(t);
|
||||
}
|
||||
|
||||
const shapeDist = (a, b) => {
|
||||
let acc = 0;
|
||||
for (let i = 0; i < a.length; i++) acc += Math.hypot(a[i].x - b[i].x, a[i].y - b[i].y);
|
||||
return acc / a.length;
|
||||
};
|
||||
|
||||
const accepted = [0];
|
||||
for (const t of candidates) {
|
||||
const last = accepted[accepted.length - 1];
|
||||
if (t - last < minHold) continue;
|
||||
if (shapeDist(shapes[t], shapes[last]) < distThresh) continue;
|
||||
accepted.push(t);
|
||||
}
|
||||
|
||||
// Snap onto the exposure grid. f is what renders; src is provenance.
|
||||
const keys = [];
|
||||
for (const src of accepted) {
|
||||
const f = Math.round(src / exposure) * exposure;
|
||||
const prev = keys[keys.length - 1];
|
||||
if (prev && prev.f === f) {
|
||||
// Two extremes collapsed onto one grid slot: keep the stronger one.
|
||||
if (sv[src] < sv[prev.src]) { prev.src = src; prev.frame = src; }
|
||||
continue;
|
||||
}
|
||||
keys.push({ f, src, frame: src });
|
||||
}
|
||||
return { keys, velocity: sv, candidates };
|
||||
}
|
||||
|
||||
// Resolve which key is live on a given output frame under interp=hold.
|
||||
// "Most recent key at or before f" - lookup, not policy.
|
||||
export function activeKey(keys, f) {
|
||||
let hit = keys[0];
|
||||
for (const k of keys) { if (k.f <= f) hit = k; else break; }
|
||||
return hit;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue