// Mouth interior from image content. // // MediaPipe has no landmarks inside the lips - the inner ring bounds the cavity // and everything within it is just pixels. So teeth come from the picture. // // The hazard is vertex correspondence. A traced contour reorders between frames // and boils, which is the failure docs/roto-puppet.md exists to avoid. The way // out for a blob specifically is RADIAL SAMPLING: march outward from the // centroid along N fixed directions and take the last pixel inside. Vertex k is // then always "the blob's extent in direction k" - correspondence holds by // construction, the vertex count is fixed, and the result smooths over time // without any reordering being possible. It also yields a star-shaped // reduction, which is what flat blocks of colour want anyway. export function otsuForTest(h, t) { return otsu(h, t); } // Otsu's threshold plus its two class means. The means matter as much as the // threshold: Otsu ALWAYS returns a split, including on a homogeneous region, so // their separation is the only thing that says the split means anything. function otsu(hist, total) { let sum = 0; for (let i = 0; i < 256; i++) sum += i * hist[i]; let sumB = 0, wB = 0, best = 0, bestVar = -1, bestDark = 0, bestBright = 0; for (let t = 0; t < 256; t++) { wB += hist[t]; if (!wB) continue; const wF = total - wB; if (!wF) break; sumB += t * hist[t]; const mDark = sumB / wB, mBright = (sum - sumB) / wF; const between = wB * wF * (mDark - mBright) * (mDark - mBright); if (between > bestVar) { bestVar = between; best = t; bestDark = mDark; bestBright = mBright; } } return { thr: best, mDark: bestDark, mBright: bestBright }; } const pointInPoly = (pts, x, y) => { let inside = false; for (let i = 0, j = pts.length - 1; i < pts.length; j = i++) { if ((pts[i].y > y) !== (pts[j].y > y) && x < ((pts[j].x - pts[i].x) * (y - pts[i].y)) / (pts[j].y - pts[i].y) + pts[i].x) inside = !inside; } return inside; }; // Shrink or grow a ring about its centroid. MediaPipe's inner lip landmarks sit // slightly OUTSIDE the real opening, so sampling the ring as given includes lip // pixels - bright, and right at the boundary where they do most damage. export function scaleRing(pts, k) { let cx = 0, cy = 0; for (const p of pts) { cx += p.x; cy += p.y; } cx /= pts.length; cy /= pts.length; return pts.map((p) => ({ x: cx + (p.x - cx) * k, y: cy + (p.y - cy) * k })); } /* ---- binary morphology on the candidate mask ---- */ function erodeMask(m, w, h) { const o = new Uint8Array(m.length); for (let y = 1; y < h - 1; y++) for (let x = 1; x < w - 1; x++) { const i = y * w + x; o[i] = m[i] && m[i - 1] && m[i + 1] && m[i - w] && m[i + w] ? 1 : 0; } return o; } function dilateMask(m, w, h) { const o = new Uint8Array(m.length); for (let y = 1; y < h - 1; y++) for (let x = 1; x < w - 1; x++) { const i = y * w + x; o[i] = m[i] || m[i - 1] || m[i + 1] || m[i - w] || m[i + w] ? 1 : 0; } return o; } // Largest 4-connected component, scored with a bias toward the TOP of the // cavity: upper teeth hang from the lip, and the usual false positive is the // tongue sitting lower down. Area alone picks the tongue when the mouth is wide. function bestComponent(mask, w, h, topBias) { const label = new Int32Array(mask.length).fill(-1); const stack = []; let best = null, id = 0; for (let s = 0; s < mask.length; s++) { if (!mask[s] || label[s] >= 0) continue; stack.length = 0; stack.push(s); label[s] = id; const px = []; let sumY = 0; while (stack.length) { const i = stack.pop(); px.push(i); sumY += (i / w) | 0; const x = i % w, y = (i / w) | 0; if (x > 0 && mask[i - 1] && label[i - 1] < 0) { label[i - 1] = id; stack.push(i - 1); } if (x < w - 1 && mask[i + 1] && label[i + 1] < 0) { label[i + 1] = id; stack.push(i + 1); } if (y > 0 && mask[i - w] && label[i - w] < 0) { label[i - w] = id; stack.push(i - w); } if (y < h - 1 && mask[i + w] && label[i + w] < 0) { label[i + w] = id; stack.push(i + w); } } const meanY = sumY / px.length / h; // 0 top, 1 bottom const score = px.length * (1 - topBias * meanY); if (!best || score > best.score) best = { score, px, area: px.length, meanY }; id++; } return best; } // Radial sampling from the centroid: N fixed directions, last pixel inside. function radialContour(mask, w, h, cx, cy, n) { const pts = []; const maxR = Math.hypot(w, h); let prev = 1; for (let k = 0; k < n; k++) { const a = -(k / n) * Math.PI * 2; // slot 0 = +x, 5/20 = top const dx = Math.cos(a), dy = Math.sin(a); let hit = 0; for (let r = 0.5; r < maxR; r += 0.5) { const x = Math.round(cx + dx * r), y = Math.round(cy + dy * r); if (x < 0 || y < 0 || x >= w || y >= h) break; if (mask[y * w + x]) hit = r; else if (hit > 0 && r > hit + 2) break; // tolerate a 2px gap, then stop } // A ray that escapes immediately would collapse the polygon; hold the last // good radius so the shape stays closed rather than spiking to the centre. if (hit <= 0) hit = prev * 0.6; prev = hit; pts.push({ x: cx + dx * hit, y: cy + dy * hit }); } return pts; } /* ---- the extraction ---- */ export function extractTeeth(img, innerNorm, ctx, o, wantDebug = false) { const none = { contour: null, contrast: 0, area: 0, debug: null }; const ring = scaleRing(innerNorm, 1 - (o.cavityErode ?? 0.18)); let x0 = 1, y0 = 1, x1 = 0, y1 = 0; for (const p of ring) { x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y); x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y); } const W = img.naturalWidth, H = img.naturalHeight; const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H)); const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H)); if (pw < 5 || ph < 5) return none; ctx.canvas.width = pw; ctx.canvas.height = ph; ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph); const src = ctx.getImageData(0, 0, pw, ph); const d = src.data; const poly = ring.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 })); const hist = new Uint32Array(256); const lum = new Float32Array(pw * ph); const red = new Float32Array(pw * ph); const inReg = new Uint8Array(pw * ph); let n = 0; for (let y = 0; y < ph; y++) for (let x = 0; x < pw; x++) { if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue; const i = y * pw + x, oo = i * 4; const R = d[oo], G = d[oo + 1], B = d[oo + 2]; lum[i] = (0.299 * R + 0.587 * G + 0.114 * B) | 0; // Tongue is red relative to its own brightness; teeth are near-neutral. red[i] = (R - (G + B) / 2) / 255; inReg[i] = 1; hist[lum[i]]++; n++; } if (n < 24) return none; const { thr, mDark, mBright } = otsu(hist, n); const contrast = (mBright - mDark) / 255; let mask = new Uint8Array(pw * ph); for (let i = 0; i < mask.length; i++) { mask[i] = inReg[i] && lum[i] > thr && red[i] < (o.tongueReject ?? 0.18) ? 1 : 0; } // Open once to despeckle, then apply the signed size adjustment. mask = dilateMask(erodeMask(mask, pw, ph), pw, ph); const grow = o.blobGrow | 0; for (let k = 0; k < Math.abs(grow); k++) { mask = grow < 0 ? erodeMask(mask, pw, ph) : dilateMask(mask, pw, ph); } const comp = bestComponent(mask, pw, ph, o.topBias ?? 0.6); if (!comp || comp.area < (o.minArea ?? 12)) { return { contour: null, contrast, area: comp ? comp.area : 0, debug: wantDebug ? debugCanvas(src, inReg, mask, pw, ph, null) : null }; } const only = new Uint8Array(mask.length); let cx = 0, cy = 0; for (const i of comp.px) { only[i] = 1; cx += i % pw; cy += (i / pw) | 0; } cx /= comp.px.length; cy /= comp.px.length; const local = radialContour(only, pw, ph, cx, cy, o.verts ?? 10); const contour = local.map((p) => ({ x: (p.x + px0) / W, y: (p.y + py0) / H })); return { contour, contrast, area: comp.area, debug: wantDebug ? debugCanvas(src, inReg, only, pw, ph, local) : null, }; } // Sampled region dimmed, kept pixels green, extracted contour in amber. function debugCanvas(src, inReg, mask, pw, ph, local) { const c = document.createElement('canvas'); c.width = pw; c.height = ph; const g = c.getContext('2d'); const out = new ImageData(pw, ph); for (let i = 0; i < pw * ph; i++) { const o = i * 4; const [r, gr, b] = [src.data[o], src.data[o + 1], src.data[o + 2]]; if (!inReg[i]) { out.data[o] = r * 0.25; out.data[o + 1] = gr * 0.25; out.data[o + 2] = b * 0.25; } else if (mask[i]) { out.data[o] = 60; out.data[o + 1] = 230; out.data[o + 2] = 120; } else { out.data[o] = r; out.data[o + 1] = gr; out.data[o + 2] = b; } out.data[o + 3] = 255; } g.putImageData(out, 0, 0); if (local && local.length) { g.strokeStyle = '#fbbf24'; g.lineWidth = 1; g.beginPath(); local.forEach((p, i) => (i ? g.lineTo(p.x, p.y) : g.moveTo(p.x, p.y))); g.closePath(); g.stroke(); } return c; }