arthur/js/interior.js

// Mouth interior from image content.
//
// MediaPipe has no landmarks inside the lips - the inner ring bounds the cavity
// and everything within it is just pixels. So teeth have to come from the
// picture, and the question is how to do that without reintroducing the boil
// that per-frame detection causes.
//
// The answer is to extract a SCALAR, not a shape. Tracing the bright blob would
// give a new contour every frame with no vertex correspondence - exactly the
// failure docs/roto-puppet.md warns about. Instead the teeth polygon is the
// inner lip ring clipped to a horizontal line, and only that line's height is
// measured. The silhouette is therefore always the mouth's own shape (stable by
// construction) and the only thing that varies per frame is one number, which
// smooths trivially. It is also how the shape is drawn by hand: a band bounded
// by the lip.

// Otsu's threshold over a luminance histogram. Self-tuning, so exposure changes
// between frames do not shift what counts as "bright".
function otsu(hist, total) {
  let sum = 0;
  for (let i = 0; i < 256; i++) sum += i * hist[i];
  let sumB = 0, wB = 0, best = 0, bestVar = -1;
  for (let t = 0; t < 256; t++) {
    wB += hist[t];
    if (!wB) continue;
    const wF = total - wB;
    if (!wF) break;
    sumB += t * hist[t];
    const mB = sumB / wB, mF = (sum - sumB) / wF;
    const between = wB * wF * (mB - mF) * (mB - mF);
    if (between > bestVar) { bestVar = between; best = t; }
  }
  return best;
}

const pointInPoly = (pts, x, y) => {
  let inside = false;
  for (let i = 0, j = pts.length - 1; i < pts.length; j = i++) {
    if ((pts[i].y > y) !== (pts[j].y > y) &&
        x < ((pts[j].x - pts[i].x) * (y - pts[i].y)) / (pts[j].y - pts[i].y) + pts[i].x) inside = !inside;
  }
  return inside;
};

// Measure one frame: how far down the cavity the bright region reaches, as a
// fraction of cavity height, plus how much of the cavity is bright at all.
export function measureInterior(img, innerNorm, ctx) {
  let x0 = 1, y0 = 1, x1 = 0, y1 = 0;
  for (const p of innerNorm) {
    x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y);
    x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y);
  }
  const W = img.naturalWidth, H = img.naturalHeight;
  const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H));
  const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H));
  if (pw < 3 || ph < 3) return { teethT: 0, coverage: 0 };

  ctx.canvas.width = pw; ctx.canvas.height = ph;
  ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph);
  const data = ctx.getImageData(0, 0, pw, ph).data;

  const poly = innerNorm.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 }));
  const hist = new Uint32Array(256);
  const lum = new Float32Array(pw * ph);
  const mask = new Uint8Array(pw * ph);
  let n = 0;
  for (let y = 0; y < ph; y++) {
    for (let x = 0; x < pw; x++) {
      if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue;
      const o = (y * pw + x) * 4;
      const l = (0.299 * data[o] + 0.587 * data[o + 1] + 0.114 * data[o + 2]) | 0;
      const i = y * pw + x;
      lum[i] = l; mask[i] = 1; hist[l]++; n++;
    }
  }
  if (n < 12) return { teethT: 0, coverage: 0 };

  const thr = otsu(hist, n);
  // Rows are scanned from the top of the cavity; the teeth line is where the
  // bright run stops. Requiring a run rather than a single row rejects specular
  // speckle on a wet lower lip.
  let bright = 0, lastBrightRow = -1, run = 0;
  for (let y = 0; y < ph; y++) {
    let rowIn = 0, rowBright = 0;
    for (let x = 0; x < pw; x++) {
      const i = y * pw + x;
      if (!mask[i]) continue;
      rowIn++;
      if (lum[i] > thr) { rowBright++; bright++; }
    }
    if (rowIn >= 2 && rowBright / rowIn > 0.45) { run++; if (run >= 1) lastBrightRow = y; }
    else run = 0;
  }
  return {
    teethT: lastBrightRow < 0 ? 0 : (lastBrightRow + 1) / ph,
    coverage: bright / n,
  };
}

// Sutherland-Hodgman against the half-plane y <= limit.
export function clipPolyAbove(pts, limit) {
  const out = [];
  for (let i = 0; i < pts.length; i++) {
    const a = pts[i], b = pts[(i + 1) % pts.length];
    const ain = a.y <= limit, bin = b.y <= limit;
    if (ain) out.push(a);
    if (ain !== bin) {
      const t = (limit - a.y) / (b.y - a.y);
      out.push({ x: a.x + t * (b.x - a.x), y: limit });
    }
  }
  return out;
}