// Mouth interior from image content. // // MediaPipe has no landmarks inside the lips - the inner ring bounds the cavity // and everything within it is just pixels. So teeth have to come from the // picture, and the question is how to do that without reintroducing the boil // that per-frame detection causes. // // The answer is to extract a SCALAR, not a shape. Tracing the bright blob would // give a new contour every frame with no vertex correspondence - exactly the // failure docs/roto-puppet.md warns about. Instead the teeth polygon is the // inner lip ring clipped to a horizontal line, and only that line's height is // measured. The silhouette is therefore always the mouth's own shape (stable by // construction) and the only thing that varies per frame is one number, which // smooths trivially. It is also how the shape is drawn by hand: a band bounded // by the lip. // Otsu's threshold over a luminance histogram. Self-tuning, so exposure changes // between frames do not shift what counts as "bright". export function otsuForTest(h, t) { return otsu(h, t); } function otsu(hist, total) { let sum = 0; for (let i = 0; i < 256; i++) sum += i * hist[i]; let sumB = 0, wB = 0, best = 0, bestVar = -1, bestMB = 0, bestMF = 0; for (let t = 0; t < 256; t++) { wB += hist[t]; if (!wB) continue; const wF = total - wB; if (!wF) break; sumB += t * hist[t]; const mB = sumB / wB, mF = (sum - sumB) / wF; // mB = dark class, mF = bright const between = wB * wF * (mB - mF) * (mB - mF); if (between > bestVar) { bestVar = between; best = t; bestMB = mB; bestMF = mF; } } return { thr: best, mDark: bestMB, mBright: bestMF }; } const pointInPoly = (pts, x, y) => { let inside = false; for (let i = 0, j = pts.length - 1; i < pts.length; j = i++) { if ((pts[i].y > y) !== (pts[j].y > y) && x < ((pts[j].x - pts[i].x) * (y - pts[i].y)) / (pts[j].y - pts[i].y) + pts[i].x) inside = !inside; } return inside; }; // Shrink a ring toward its centroid. // // MediaPipe's inner lip landmarks sit slightly OUTSIDE the actual opening, so // sampling the ring as given includes lip pixels - which are bright, and sit // right at the cavity boundary where they do the most damage. function erode(pts, k) { let cx = 0, cy = 0; for (const p of pts) { cx += p.x; cy += p.y; } cx /= pts.length; cy /= pts.length; return pts.map((p) => ({ x: cx + (p.x - cx) * (1 - k), y: cy + (p.y - cy) * (1 - k) })); } // Measure one frame: how far down the cavity the bright region reaches, as a // fraction of cavity height, plus the contrast that justified calling it bright. // // `wantDebug` returns the sampled crop with the classification drawn on it. // Tuning this blind is miserable; the numbers alone do not say whether the // region being measured is even the right region. export function measureInterior(img, innerNorm, ctx, opts = {}, wantDebug = false) { const minContrast = opts.minContrast ?? 0.14; const rowFrac = opts.rowFrac ?? 0.4; const inner = erode(innerNorm, opts.erode ?? 0.18); const none = { teethT: 0, contrast: 0, coverage: 0, debug: null }; let x0 = 1, y0 = 1, x1 = 0, y1 = 0; for (const p of inner) { x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y); x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y); } const W = img.naturalWidth, H = img.naturalHeight; const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H)); const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H)); if (pw < 4 || ph < 4) return none; ctx.canvas.width = pw; ctx.canvas.height = ph; ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph); const img0 = ctx.getImageData(0, 0, pw, ph); const data = img0.data; const poly = inner.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 })); const hist = new Uint32Array(256); const lum = new Float32Array(pw * ph); const mask = new Uint8Array(pw * ph); let n = 0; for (let y = 0; y < ph; y++) { for (let x = 0; x < pw; x++) { if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue; const o = (y * pw + x) * 4; const l = (0.299 * data[o] + 0.587 * data[o + 1] + 0.114 * data[o + 2]) | 0; const i = y * pw + x; lum[i] = l; mask[i] = 1; hist[l]++; n++; } } if (n < 16) return none; const { thr, mDark, mBright } = otsu(hist, n); // Otsu ALWAYS returns a split, including on a homogeneous region: given a dark // cavity with no teeth it invents a threshold and calls half the pixels // bright. The separation between the two class means is what says whether the // split means anything, so it is the actual gate. const contrast = (mBright - mDark) / 255; if (contrast < minContrast) { return { teethT: 0, contrast, coverage: 0, debug: wantDebug ? debugCanvas(img0, mask, lum, thr, pw, ph, -1) : null }; } // Scan from the top and STOP at the first row that fails. Teeth hang from the // upper lip, so what matters is the contiguous run, not whether some row near // the bottom happens to qualify - tracking the latter is what made the band // fill the whole mouth. let lastRow = -1, started = false, bright = 0; for (let y = 0; y < ph; y++) { let rowIn = 0, rowBright = 0; for (let x = 0; x < pw; x++) { const i = y * pw + x; if (!mask[i]) continue; rowIn++; if (lum[i] > thr) { rowBright++; bright++; } } if (rowIn < 2) continue; const ok = rowBright / rowIn > rowFrac; if (ok) { started = true; lastRow = y; } else if (started) break; } return { teethT: lastRow < 0 ? 0 : (lastRow + 1) / ph, contrast, coverage: bright / n, debug: wantDebug ? debugCanvas(img0, mask, lum, thr, pw, ph, lastRow) : null, }; } // The sampled crop with the classification painted on: sampled region tinted, // pixels above threshold in green, the resolved teeth line in amber. function debugCanvas(img0, mask, lum, thr, pw, ph, lastRow) { const c = document.createElement('canvas'); c.width = pw; c.height = ph; const g = c.getContext('2d'); const out = new ImageData(pw, ph); for (let i = 0; i < pw * ph; i++) { const o = i * 4; const [r, gr, b] = [img0.data[o], img0.data[o + 1], img0.data[o + 2]]; if (!mask[i]) { out.data[o] = r * 0.3; out.data[o + 1] = gr * 0.3; out.data[o + 2] = b * 0.3; } else if (lum[i] > thr) { out.data[o] = 60; out.data[o + 1] = 230; out.data[o + 2] = 120; } else { out.data[o] = r; out.data[o + 1] = gr; out.data[o + 2] = b; } out.data[o + 3] = 255; } g.putImageData(out, 0, 0); if (lastRow >= 0) { g.fillStyle = '#fbbf24'; g.fillRect(0, lastRow, pw, 1); } return c; } // Sutherland-Hodgman against the half-plane y <= limit. export function clipPolyAbove(pts, limit) { const out = []; for (let i = 0; i < pts.length; i++) { const a = pts[i], b = pts[(i + 1) % pts.length]; const ain = a.y <= limit, bin = b.y <= limit; if (ain) out.push(a); if (ain !== bin) { const t = (limit - a.y) / (b.y - a.y); out.push({ x: a.x + t * (b.x - a.x), y: limit }); } } return out; }