Teeth from mouth-interior image content
MediaPipe has no landmarks inside the lips, so teeth have to come from pixels. Tracing the bright blob would give a new contour every frame with no vertex correspondence - the exact boil docs/roto-puppet.md warns about. So the measurement yields a scalar, not a shape: Otsu within the cavity, scanned from the top for where the bright run stops, giving one line height per frame. The teeth polygon is the inner lip ring clipped to that line, so the silhouette is always the mouth's own shape and cannot disagree with the lips around it, and the only per-frame variable is a single number that smooths trivially. Presence gets hysteresis and minimum dwell, as plate selection does: a teeth block blinking on and off for single frames is worse than one simply absent. Tongue is not implemented. The same scalar approach would apply, gated on redness rather than brightness, but it is not visible in the test footage - the cavity reads dark with a bright upper-teeth band and nothing else. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
29b0bd6690
commit
941022b69f
5 changed files with 242 additions and 5 deletions
84
js/app.js
84
js/app.js
|
|
@ -3,6 +3,7 @@ import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js';
|
|||
import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js';
|
||||
import { IndexedRaster } from './raster.js';
|
||||
import { drawRegistered, posterizeInto } from './underlay.js';
|
||||
import { measureInterior, clipPolyAbove } from './interior.js';
|
||||
import { writeTake } from './take.js';
|
||||
import { synthDense } from './synth.js';
|
||||
|
||||
|
|
@ -13,9 +14,9 @@ const PALETTE = [
|
|||
{ name: 'skin_base', hex: '#b07a5a' },
|
||||
{ name: 'skin_dark', hex: '#7a4f3a' },
|
||||
{ name: 'mouth_dark', hex: '#24161a' },
|
||||
{ name: 'skin_lite', hex: '#d9a884' },
|
||||
{ name: 'teeth', hex: '#d9cfc2' },
|
||||
];
|
||||
const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, lite: 4 };
|
||||
const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, teeth: 4 };
|
||||
|
||||
const state = {
|
||||
dense: null, images: [], stab: null, xform: null,
|
||||
|
|
@ -24,11 +25,15 @@ const state = {
|
|||
frame: 0, playing: false, faceBox: null,
|
||||
fps: 12, audio: null, // fps comes from manifest.json, never guessed
|
||||
aspect: 1, // imgW/imgH; converts MediaPipe's anisotropic space
|
||||
interior: null, // per-frame teeth measurement from image content
|
||||
teeth: null, // resolved per-frame {show, t} after knobs
|
||||
};
|
||||
|
||||
const el = (id) => document.getElementById(id);
|
||||
const opts = () => ({
|
||||
verts: +el('verts').value,
|
||||
teethOn: +el('teethOn').value / 100,
|
||||
teethDwell: +el('teethDwell').value,
|
||||
smoothWin: +el('smoothWin').value,
|
||||
contourSmooth: +el('contourSmooth').value,
|
||||
apertureThresh: +el('apertureThresh').value / 1000,
|
||||
|
|
@ -116,6 +121,14 @@ async function detectAll(images) {
|
|||
return { dense, missing };
|
||||
}
|
||||
|
||||
// Interior measurement is a function of pixels alone, so it runs once with
|
||||
// detection and the knobs re-resolve it instantly afterwards.
|
||||
function measureAll(images, dense) {
|
||||
const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
|
||||
return dense.map((lm, i) =>
|
||||
measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx));
|
||||
}
|
||||
|
||||
/* ---------- build ---------- */
|
||||
|
||||
function makeXform(stab, neutral) {
|
||||
|
|
@ -154,6 +167,8 @@ function rebuild(resetKeep) {
|
|||
const apMax = Math.max(...ap);
|
||||
state.hidden = ap.map((v) => v / apMax < o.apertureThresh);
|
||||
|
||||
state.teeth = resolveTeeth(o);
|
||||
|
||||
// Plate outline per frame, so a kept frame shows its own head shape.
|
||||
state.plates = state.stab.oval.map((r) => r.map(state.xform));
|
||||
|
||||
|
|
@ -180,6 +195,40 @@ function faceBoxes() {
|
|||
});
|
||||
}
|
||||
|
||||
// Presence gets hysteresis and a minimum dwell, the same treatment plate
|
||||
// selection gets: a teeth block that blinks on and off for single frames is
|
||||
// worse than one that is simply absent. Appearing needs a clear signal, staying
|
||||
// needs only a weak one.
|
||||
function resolveTeeth(o) {
|
||||
const N = state.dense.length;
|
||||
if (!state.interior) return new Array(N).fill({ show: false, t: 0 });
|
||||
|
||||
const raw = state.interior.map((m, f) => (state.hidden[f] ? 0 : m.coverage));
|
||||
const on = o.teethOn, off = o.teethOn * 0.6;
|
||||
const shown = new Array(N).fill(false);
|
||||
let live = false, since = 0;
|
||||
for (let f = 0; f < N; f++) {
|
||||
const want = live ? raw[f] > off : raw[f] > on;
|
||||
if (want !== live && since >= o.teethDwell) { live = want; since = 0; }
|
||||
else since++;
|
||||
shown[f] = live && !state.hidden[f];
|
||||
}
|
||||
|
||||
// Smooth the line height only across frames where it is actually shown, so a
|
||||
// gap does not drag the band toward zero on either side of it.
|
||||
const t = state.interior.map((m) => m.teethT);
|
||||
const sm = t.map((_, f) => {
|
||||
let acc = 0, c = 0;
|
||||
for (let j = f - o.contourSmooth; j <= f + o.contourSmooth; j++) {
|
||||
const k = Math.min(N - 1, Math.max(0, j));
|
||||
if (!shown[k]) continue;
|
||||
acc += t[k]; c++;
|
||||
}
|
||||
return c ? acc / c : t[f];
|
||||
});
|
||||
return shown.map((show, f) => ({ show, t: sm[f] }));
|
||||
}
|
||||
|
||||
/* ---------- render ---------- */
|
||||
|
||||
// The plate layer has several representations because its job changes: a flat
|
||||
|
|
@ -199,7 +248,18 @@ function renderFrame(f, mode = plateMode()) {
|
|||
}
|
||||
|
||||
r.fillPoly(state.outer[f], IDX.dark); // mouth keeps every frame
|
||||
if (!state.hidden[f]) r.fillPoly(state.inner[f], IDX.mouth);
|
||||
if (!state.hidden[f]) {
|
||||
r.fillPoly(state.inner[f], IDX.mouth);
|
||||
const te = state.teeth[f];
|
||||
if (te.show) {
|
||||
// The band is the mouth's own silhouette clipped to a height, so the shape
|
||||
// can never disagree with the lips around it.
|
||||
const ys = state.inner[f].map((p) => p.y);
|
||||
const top = Math.min(...ys), bot = Math.max(...ys);
|
||||
const poly = clipPolyAbove(state.inner[f], top + te.t * (bot - top));
|
||||
if (poly.length >= 3) r.fillPoly(poly, IDX.teeth);
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
|
|
@ -259,8 +319,10 @@ function drawAll() {
|
|||
function drawReadout() {
|
||||
const kept = keptSorted();
|
||||
const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k);
|
||||
const teethFrames = state.teeth ? state.teeth.filter((t) => t.show).length : 0;
|
||||
el('readout').textContent =
|
||||
`${state.dense.length} frames → ${kept.length} drawings · ` +
|
||||
`teeth on ${teethFrames}f · ` +
|
||||
`holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` +
|
||||
`neutral f${state.neutral} · residual ` +
|
||||
`${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`;
|
||||
|
|
@ -406,6 +468,14 @@ function exportTake() {
|
|||
keys: state.outer.map((pts, f) => ({ f, src: f, pts })) },
|
||||
{ name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
|
||||
keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) },
|
||||
{ name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in',
|
||||
keys: state.inner.map((pts, f) => {
|
||||
const te = state.teeth[f];
|
||||
if (state.hidden[f] || !te.show) return { f, hidden: true };
|
||||
const ys = pts.map((p) => p.y);
|
||||
const top = Math.min(...ys), bot = Math.max(...ys);
|
||||
return { f, src: f, pts: clipPolyAbove(pts, top + te.t * (bot - top)) };
|
||||
}) },
|
||||
],
|
||||
};
|
||||
const text = writeTake(take)
|
||||
|
|
@ -442,6 +512,8 @@ async function runFrames() {
|
|||
const { dense, missing } = await detectAll(images);
|
||||
state.images = images; state.dense = dense;
|
||||
state.aspect = images[0].naturalWidth / images[0].naturalHeight;
|
||||
status('measuring mouth interiors…');
|
||||
state.interior = measureAll(images, dense);
|
||||
el('scrub').max = dense.length - 1;
|
||||
state.frame = 0;
|
||||
rebuild(true);
|
||||
|
|
@ -459,6 +531,7 @@ function runSynthetic() {
|
|||
attachAudio(null);
|
||||
state.fps = 12;
|
||||
state.aspect = 1; // synthetic landmarks are generated square
|
||||
state.interior = null; // no pixels, so no teeth
|
||||
|
||||
state.dense = synthDense(72);
|
||||
el('scrub').max = 71;
|
||||
|
|
@ -467,10 +540,11 @@ function runSynthetic() {
|
|||
status('synthetic — exercises everything below detection', 'ok');
|
||||
}
|
||||
|
||||
for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol']) {
|
||||
for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell']) {
|
||||
el(id).addEventListener('input', () => {
|
||||
el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol'
|
||||
? (+el(id).value / 1000).toFixed(3) : el(id).value;
|
||||
? (+el(id).value / 1000).toFixed(3)
|
||||
: id === 'teethOn' ? (+el(id).value / 100).toFixed(2) : el(id).value;
|
||||
if (id === 'tol') return; // tol only matters when you ask for a suggestion
|
||||
rebuild(false);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue