// Mouth interior from image content. // // MediaPipe has no landmarks inside the lips - the inner ring bounds the cavity // and everything within it is just pixels. So teeth have to come from the // picture, and the question is how to do that without reintroducing the boil // that per-frame detection causes. // // The answer is to extract a SCALAR, not a shape. Tracing the bright blob would // give a new contour every frame with no vertex correspondence - exactly the // failure docs/roto-puppet.md warns about. Instead the teeth polygon is the // inner lip ring clipped to a horizontal line, and only that line's height is // measured. The silhouette is therefore always the mouth's own shape (stable by // construction) and the only thing that varies per frame is one number, which // smooths trivially. It is also how the shape is drawn by hand: a band bounded // by the lip. // Otsu's threshold over a luminance histogram. Self-tuning, so exposure changes // between frames do not shift what counts as "bright". function otsu(hist, total) { let sum = 0; for (let i = 0; i < 256; i++) sum += i * hist[i]; let sumB = 0, wB = 0, best = 0, bestVar = -1; for (let t = 0; t < 256; t++) { wB += hist[t]; if (!wB) continue; const wF = total - wB; if (!wF) break; sumB += t * hist[t]; const mB = sumB / wB, mF = (sum - sumB) / wF; const between = wB * wF * (mB - mF) * (mB - mF); if (between > bestVar) { bestVar = between; best = t; } } return best; } const pointInPoly = (pts, x, y) => { let inside = false; for (let i = 0, j = pts.length - 1; i < pts.length; j = i++) { if ((pts[i].y > y) !== (pts[j].y > y) && x < ((pts[j].x - pts[i].x) * (y - pts[i].y)) / (pts[j].y - pts[i].y) + pts[i].x) inside = !inside; } return inside; }; // Measure one frame: how far down the cavity the bright region reaches, as a // fraction of cavity height, plus how much of the cavity is bright at all. export function measureInterior(img, innerNorm, ctx) { let x0 = 1, y0 = 1, x1 = 0, y1 = 0; for (const p of innerNorm) { x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y); x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y); } const W = img.naturalWidth, H = img.naturalHeight; const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H)); const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H)); if (pw < 3 || ph < 3) return { teethT: 0, coverage: 0 }; ctx.canvas.width = pw; ctx.canvas.height = ph; ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph); const data = ctx.getImageData(0, 0, pw, ph).data; const poly = innerNorm.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 })); const hist = new Uint32Array(256); const lum = new Float32Array(pw * ph); const mask = new Uint8Array(pw * ph); let n = 0; for (let y = 0; y < ph; y++) { for (let x = 0; x < pw; x++) { if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue; const o = (y * pw + x) * 4; const l = (0.299 * data[o] + 0.587 * data[o + 1] + 0.114 * data[o + 2]) | 0; const i = y * pw + x; lum[i] = l; mask[i] = 1; hist[l]++; n++; } } if (n < 12) return { teethT: 0, coverage: 0 }; const thr = otsu(hist, n); // Rows are scanned from the top of the cavity; the teeth line is where the // bright run stops. Requiring a run rather than a single row rejects specular // speckle on a wet lower lip. let bright = 0, lastBrightRow = -1, run = 0; for (let y = 0; y < ph; y++) { let rowIn = 0, rowBright = 0; for (let x = 0; x < pw; x++) { const i = y * pw + x; if (!mask[i]) continue; rowIn++; if (lum[i] > thr) { rowBright++; bright++; } } if (rowIn >= 2 && rowBright / rowIn > 0.45) { run++; if (run >= 1) lastBrightRow = y; } else run = 0; } return { teethT: lastBrightRow < 0 ? 0 : (lastBrightRow + 1) / ph, coverage: bright / n, }; } // Sutherland-Hodgman against the half-plane y <= limit. export function clipPolyAbove(pts, limit) { const out = []; for (let i = 0; i < pts.length; i++) { const a = pts[i], b = pts[(i + 1) % pts.length]; const ain = a.y <= limit, bin = b.y <= limit; if (ain) out.push(a); if (ain !== bin) { const t = (limit - a.y) / (b.y - a.y); out.push({ x: a.x + t * (b.x - a.x), y: limit }); } } return out; }