Teeth as an extracted blob contour, not a clipped band
The band filled the mouth because a band is the wrong reduction: the bright region is a blob, and reading it as "everything above a line" throws the shape away. Extracting a contour reintroduces the vertex-correspondence problem that made me avoid it, but for a blob there is a way out. Radial sampling from the centroid along N fixed directions makes vertex k always mean "the extent in direction k": correspondence holds by construction, the count is fixed, and temporal smoothing cannot reorder anything. It also yields a star-shaped reduction, which suits flat colour. Tongue rejection, which the band had no way to express: - pixels red relative to their own brightness are dropped (teeth are neutral) - component choice is biased toward the top of the cavity, since area alone picks the tongue when the mouth is wide - separate inner and outer controls: cavity erode pulls the sampled region off the lip edge, blob grow/erode resizes the found blob Also: a knob wired in app.js but missing from index.html threw during wiring and left a blank page with nothing useful in the console - which is exactly what happened to teethDwell in the previous commit. el() now names the missing id, and window.onerror surfaces it in the status line. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
c6daf827d1
commit
9a11eeabc1
7 changed files with 272 additions and 275 deletions
116
js/app.js
116
js/app.js
|
|
@ -3,7 +3,8 @@ import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js';
|
|||
import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js';
|
||||
import { IndexedRaster } from './raster.js';
|
||||
import { drawRegistered, posterizeInto } from './underlay.js';
|
||||
import { measureInterior, clipPolyAbove } from './interior.js';
|
||||
import { extractTeeth } from './interior.js';
|
||||
import { applySim } from './mathutil.js';
|
||||
import { writeTake } from './take.js';
|
||||
import { synthDense } from './synth.js';
|
||||
|
||||
|
|
@ -29,12 +30,23 @@ const state = {
|
|||
teeth: null, // resolved per-frame {show, t} after knobs
|
||||
};
|
||||
|
||||
const el = (id) => document.getElementById(id);
|
||||
const el = (id) => {
|
||||
const n = document.getElementById(id);
|
||||
// A knob present in the code but missing from the markup used to throw during
|
||||
// wiring and leave a blank page with nothing in the console worth reading.
|
||||
if (!n) throw new Error(`missing element #${id} — knob wired in app.js but not in index.html`);
|
||||
return n;
|
||||
};
|
||||
const opts = () => ({
|
||||
verts: +el('verts').value,
|
||||
teethOn: +el('teethOn').value / 100, // minimum Otsu class separation
|
||||
teethDwell: +el('teethDwell').value,
|
||||
teethErode: +el('teethErode').value / 100,
|
||||
teethSmooth: +el('teethSmooth').value,
|
||||
cavityErode: +el('teethErode').value / 100,
|
||||
tongueReject: +el('tongueReject').value / 100,
|
||||
blobGrow: +el('blobGrow').value,
|
||||
topBias: +el('topBias').value / 100,
|
||||
verts: +el('teethVerts').value,
|
||||
smoothWin: +el('smoothWin').value,
|
||||
contourSmooth: +el('contourSmooth').value,
|
||||
apertureThresh: +el('apertureThresh').value / 1000,
|
||||
|
|
@ -127,10 +139,14 @@ async function detectAll(images) {
|
|||
function measureAll(images, dense, o) {
|
||||
const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
|
||||
return dense.map((lm, i) =>
|
||||
measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx,
|
||||
{ minContrast: 0, erode: o.teethErode })); // gate applied later, in resolveTeeth
|
||||
extractTeeth(images[i], LIPS_INNER.map((k) => lm[k]), ctx, o));
|
||||
}
|
||||
|
||||
// Extraction keys on every knob that changes the pixels examined, so the cache
|
||||
// is keyed on exactly those and a change to anything else stays instant.
|
||||
const extractKey = (o) =>
|
||||
[o.cavityErode, o.tongueReject, o.blobGrow, o.topBias, o.verts].join('|');
|
||||
|
||||
/* ---------- build ---------- */
|
||||
|
||||
function makeXform(stab, neutral) {
|
||||
|
|
@ -169,9 +185,9 @@ function rebuild(resetKeep) {
|
|||
const apMax = Math.max(...ap);
|
||||
state.hidden = ap.map((v) => v / apMax < o.apertureThresh);
|
||||
|
||||
if (state.images.length && (!state.interior || state.erodeUsed !== o.teethErode)) {
|
||||
if (state.images.length && state.extractKey !== extractKey(o)) {
|
||||
state.interior = measureAll(state.images, state.dense, o);
|
||||
state.erodeUsed = o.teethErode;
|
||||
state.extractKey = extractKey(o);
|
||||
}
|
||||
state.teeth = resolveTeeth(o);
|
||||
|
||||
|
|
@ -207,13 +223,10 @@ function faceBoxes() {
|
|||
// needs only a weak one.
|
||||
function resolveTeeth(o) {
|
||||
const N = state.dense.length;
|
||||
if (!state.interior) return new Array(N).fill({ show: false, t: 0 });
|
||||
if (!state.interior) return new Array(N).fill({ show: false, pts: null });
|
||||
|
||||
// Gate on the separation between Otsu's two class means, not on how much of
|
||||
// the cavity is bright: coverage is high both when the mouth is full of teeth
|
||||
// and when the region is uniformly dark and Otsu has split noise.
|
||||
const raw = state.interior.map((m, f) =>
|
||||
(state.hidden[f] || m.teethT <= 0 ? 0 : m.contrast));
|
||||
(state.hidden[f] || !m.contour ? 0 : m.contrast));
|
||||
const on = o.teethOn, off = o.teethOn * 0.7;
|
||||
const shown = new Array(N).fill(false);
|
||||
let live = false, since = 0;
|
||||
|
|
@ -221,22 +234,34 @@ function resolveTeeth(o) {
|
|||
const want = live ? raw[f] > off : raw[f] > on;
|
||||
if (want !== live && since >= o.teethDwell) { live = want; since = 0; }
|
||||
else since++;
|
||||
shown[f] = live && !state.hidden[f];
|
||||
shown[f] = live && !state.hidden[f] && !!state.interior[f].contour;
|
||||
}
|
||||
|
||||
// Smooth the line height only across frames where it is actually shown, so a
|
||||
// gap does not drag the band toward zero on either side of it.
|
||||
const t = state.interior.map((m) => m.teethT);
|
||||
const sm = t.map((_, f) => {
|
||||
let acc = 0, c = 0;
|
||||
for (let j = f - o.contourSmooth; j <= f + o.contourSmooth; j++) {
|
||||
// Into raster space through the same chain the lips take, including the
|
||||
// isotropic aspect conversion - a contour in MediaPipe's normalised space is
|
||||
// in the same stretched coordinates the landmarks are.
|
||||
const toRaster = (pts, f) => {
|
||||
const tf = state.stab.transforms[f];
|
||||
return pts.map((p) => state.xform(applySim(tf, { x: p.x * state.aspect, y: p.y })));
|
||||
};
|
||||
const rast = state.interior.map((m, f) => (m.contour ? toRaster(m.contour, f) : null));
|
||||
|
||||
// Radial sampling makes vertex k mean the same direction on every frame, so
|
||||
// smoothing across time is well defined and cannot reorder anything.
|
||||
const sm = rast.map((pts, f) => {
|
||||
if (!pts || !shown[f]) return pts;
|
||||
const acc = pts.map(() => ({ x: 0, y: 0 }));
|
||||
let c = 0;
|
||||
for (let j = f - o.teethSmooth; j <= f + o.teethSmooth; j++) {
|
||||
const k = Math.min(N - 1, Math.max(0, j));
|
||||
if (!shown[k]) continue;
|
||||
acc += t[k]; c++;
|
||||
if (!shown[k] || !rast[k] || rast[k].length !== pts.length) continue;
|
||||
for (let v = 0; v < pts.length; v++) { acc[v].x += rast[k][v].x; acc[v].y += rast[k][v].y; }
|
||||
c++;
|
||||
}
|
||||
return c ? acc / c : t[f];
|
||||
return c ? acc.map((p) => ({ x: p.x / c, y: p.y / c })) : pts;
|
||||
});
|
||||
return shown.map((show, f) => ({ show, t: sm[f] }));
|
||||
|
||||
return shown.map((show, f) => ({ show, pts: sm[f] }));
|
||||
}
|
||||
|
||||
/* ---------- render ---------- */
|
||||
|
|
@ -261,14 +286,7 @@ function renderFrame(f, mode = plateMode()) {
|
|||
if (!state.hidden[f]) {
|
||||
r.fillPoly(state.inner[f], IDX.mouth);
|
||||
const te = state.teeth[f];
|
||||
if (te.show) {
|
||||
// The band is the mouth's own silhouette clipped to a height, so the shape
|
||||
// can never disagree with the lips around it.
|
||||
const ys = state.inner[f].map((p) => p.y);
|
||||
const top = Math.min(...ys), bot = Math.max(...ys);
|
||||
const poly = clipPolyAbove(state.inner[f], top + te.t * (bot - top));
|
||||
if (poly.length >= 3) r.fillPoly(poly, IDX.teeth);
|
||||
}
|
||||
if (te.show && te.pts && te.pts.length >= 3) r.fillPoly(te.pts, IDX.teeth);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
|
@ -388,21 +406,24 @@ function drawPanes() {
|
|||
function drawInteriorDebug(f) {
|
||||
const host = el('cv-teeth');
|
||||
const img = state.images[f];
|
||||
if (!img || state.hidden[f]) { host.innerHTML = ''; el('teethinfo').textContent = state.images.length ? 'mouth closed' : 'no source frames'; return; }
|
||||
if (!img || state.hidden[f]) {
|
||||
host.innerHTML = '';
|
||||
el('teethinfo').textContent = state.images.length ? 'mouth closed' : 'no source frames';
|
||||
return;
|
||||
}
|
||||
const o = opts();
|
||||
const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
|
||||
const m = measureInterior(img, LIPS_INNER.map((k) => state.dense[f][k]), ctx,
|
||||
{ minContrast: 0, erode: o.teethErode }, true);
|
||||
const m = extractTeeth(img, LIPS_INNER.map((k) => state.dense[f][k]), ctx, o, true);
|
||||
host.innerHTML = '';
|
||||
if (m.debug) {
|
||||
m.debug.style.width = '150px';
|
||||
m.debug.style.width = '170px';
|
||||
m.debug.style.imageRendering = 'pixelated';
|
||||
host.append(m.debug);
|
||||
}
|
||||
const te = state.teeth[f];
|
||||
el('teethinfo').textContent =
|
||||
`contrast ${m.contrast.toFixed(3)} (gate ${o.teethOn.toFixed(2)}) · ` +
|
||||
`line ${m.teethT.toFixed(2)} · ${te.show ? 'SHOWN' : 'hidden'}`;
|
||||
`contrast ${m.contrast.toFixed(3)} / gate ${o.teethOn.toFixed(2)} · ` +
|
||||
`area ${m.area}px · ${te.show ? 'SHOWN' : 'hidden'}`;
|
||||
}
|
||||
|
||||
function strokePts(g, pts, color, lw = 1) {
|
||||
|
|
@ -503,13 +524,8 @@ function exportTake() {
|
|||
{ name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
|
||||
keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) },
|
||||
{ name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in',
|
||||
keys: state.inner.map((pts, f) => {
|
||||
const te = state.teeth[f];
|
||||
if (state.hidden[f] || !te.show) return { f, hidden: true };
|
||||
const ys = pts.map((p) => p.y);
|
||||
const top = Math.min(...ys), bot = Math.max(...ys);
|
||||
return { f, src: f, pts: clipPolyAbove(pts, top + te.t * (bot - top)) };
|
||||
}) },
|
||||
keys: state.teeth.map((te, f) =>
|
||||
(te.show && te.pts ? { f, src: f, pts: te.pts } : { f, hidden: true })) },
|
||||
],
|
||||
};
|
||||
const text = writeTake(take)
|
||||
|
|
@ -574,11 +590,14 @@ function runSynthetic() {
|
|||
status('synthetic — exercises everything below detection', 'ok');
|
||||
}
|
||||
|
||||
for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell', 'teethErode']) {
|
||||
for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol',
|
||||
'teethOn', 'teethDwell', 'teethErode', 'tongueReject', 'blobGrow',
|
||||
'topBias', 'teethVerts', 'teethSmooth']) {
|
||||
el(id).addEventListener('input', () => {
|
||||
el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol'
|
||||
? (+el(id).value / 1000).toFixed(3)
|
||||
: id === 'teethOn' || id === 'teethErode' ? (+el(id).value / 100).toFixed(2)
|
||||
: ['teethOn', 'teethErode', 'tongueReject', 'topBias'].includes(id)
|
||||
? (+el(id).value / 100).toFixed(2)
|
||||
: el(id).value;
|
||||
if (id === 'tol') return; // tol only matters when you ask for a suggestion
|
||||
rebuild(false);
|
||||
|
|
@ -699,6 +718,11 @@ PALETTE.forEach((p) => {
|
|||
|
||||
// #synth / #frames autorun, so the tool can be driven headlessly for smoke tests
|
||||
// and deep-linked. Detection needs WebGL; the synthetic path does not.
|
||||
window.addEventListener('error', (e) => {
|
||||
const s = document.getElementById('status');
|
||||
if (s) { s.textContent = e.message; s.className = 'err'; }
|
||||
});
|
||||
|
||||
if (location.hash === '#synth') runSynthetic();
|
||||
else if (location.hash === '#frames') runFrames();
|
||||
else status('ready — Load frames, then step with \u2190 \u2192 and delete with X');
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue