arthur/js/interior.js

// Mouth interior from image content.
//
// MediaPipe has no landmarks inside the lips - the inner ring bounds the cavity
// and everything within it is just pixels. So teeth come from the picture.
//
// The hazard is vertex correspondence. A traced contour reorders between frames
// and boils, which is the failure docs/roto-puppet.md exists to avoid. The way
// out for a blob specifically is RADIAL SAMPLING: march outward from the
// centroid along N fixed directions and take the last pixel inside. Vertex k is
// then always "the blob's extent in direction k" - correspondence holds by
// construction, the vertex count is fixed, and the result smooths over time
// without any reordering being possible. It also yields a star-shaped
// reduction, which is what flat blocks of colour want anyway.

export function otsuForTest(h, t) { return otsu(h, t); }

// Otsu's threshold plus its two class means. The means matter as much as the
// threshold: Otsu ALWAYS returns a split, including on a homogeneous region, so
// their separation is the only thing that says the split means anything.
function otsu(hist, total) {
  let sum = 0;
  for (let i = 0; i < 256; i++) sum += i * hist[i];
  let sumB = 0, wB = 0, best = 0, bestVar = -1, bestDark = 0, bestBright = 0;
  for (let t = 0; t < 256; t++) {
    wB += hist[t];
    if (!wB) continue;
    const wF = total - wB;
    if (!wF) break;
    sumB += t * hist[t];
    const mDark = sumB / wB, mBright = (sum - sumB) / wF;
    const between = wB * wF * (mDark - mBright) * (mDark - mBright);
    if (between > bestVar) { bestVar = between; best = t; bestDark = mDark; bestBright = mBright; }
  }
  return { thr: best, mDark: bestDark, mBright: bestBright };
}

const pointInPoly = (pts, x, y) => {
  let inside = false;
  for (let i = 0, j = pts.length - 1; i < pts.length; j = i++) {
    if ((pts[i].y > y) !== (pts[j].y > y) &&
        x < ((pts[j].x - pts[i].x) * (y - pts[i].y)) / (pts[j].y - pts[i].y) + pts[i].x) inside = !inside;
  }
  return inside;
};

// Shrink or grow a ring about its centroid. MediaPipe's inner lip landmarks sit
// slightly OUTSIDE the real opening, so sampling the ring as given includes lip
// pixels - bright, and right at the boundary where they do most damage.
export function scaleRing(pts, k) {
  let cx = 0, cy = 0;
  for (const p of pts) { cx += p.x; cy += p.y; }
  cx /= pts.length; cy /= pts.length;
  return pts.map((p) => ({ x: cx + (p.x - cx) * k, y: cy + (p.y - cy) * k }));
}

/* ---- binary morphology on the candidate mask ---- */

function erodeMask(m, w, h) {
  const o = new Uint8Array(m.length);
  for (let y = 1; y < h - 1; y++) for (let x = 1; x < w - 1; x++) {
    const i = y * w + x;
    o[i] = m[i] && m[i - 1] && m[i + 1] && m[i - w] && m[i + w] ? 1 : 0;
  }
  return o;
}
function dilateMask(m, w, h) {
  const o = new Uint8Array(m.length);
  for (let y = 1; y < h - 1; y++) for (let x = 1; x < w - 1; x++) {
    const i = y * w + x;
    o[i] = m[i] || m[i - 1] || m[i + 1] || m[i - w] || m[i + w] ? 1 : 0;
  }
  return o;
}

// Largest 4-connected component, scored with a bias toward the TOP of the
// cavity: upper teeth hang from the lip, and the usual false positive is the
// tongue sitting lower down. Area alone picks the tongue when the mouth is wide.
function bestComponent(mask, w, h, topBias) {
  const label = new Int32Array(mask.length).fill(-1);
  const stack = [];
  let best = null, id = 0;
  for (let s = 0; s < mask.length; s++) {
    if (!mask[s] || label[s] >= 0) continue;
    stack.length = 0; stack.push(s);
    label[s] = id;
    const px = [];
    let sumY = 0;
    while (stack.length) {
      const i = stack.pop();
      px.push(i);
      sumY += (i / w) | 0;
      const x = i % w, y = (i / w) | 0;
      if (x > 0 && mask[i - 1] && label[i - 1] < 0) { label[i - 1] = id; stack.push(i - 1); }
      if (x < w - 1 && mask[i + 1] && label[i + 1] < 0) { label[i + 1] = id; stack.push(i + 1); }
      if (y > 0 && mask[i - w] && label[i - w] < 0) { label[i - w] = id; stack.push(i - w); }
      if (y < h - 1 && mask[i + w] && label[i + w] < 0) { label[i + w] = id; stack.push(i + w); }
    }
    const meanY = sumY / px.length / h;            // 0 top, 1 bottom
    const score = px.length * (1 - topBias * meanY);
    if (!best || score > best.score) best = { score, px, area: px.length, meanY };
    id++;
  }
  return best;
}

// Radial sampling from the centroid: N fixed directions, last pixel inside.
function radialContour(mask, w, h, cx, cy, n) {
  const pts = [];
  const maxR = Math.hypot(w, h);
  let prev = 1;
  for (let k = 0; k < n; k++) {
    const a = -(k / n) * Math.PI * 2;             // slot 0 = +x, 5/20 = top
    const dx = Math.cos(a), dy = Math.sin(a);
    let hit = 0;
    for (let r = 0.5; r < maxR; r += 0.5) {
      const x = Math.round(cx + dx * r), y = Math.round(cy + dy * r);
      if (x < 0 || y < 0 || x >= w || y >= h) break;
      if (mask[y * w + x]) hit = r;
      else if (hit > 0 && r > hit + 2) break;      // tolerate a 2px gap, then stop
    }
    // A ray that escapes immediately would collapse the polygon; hold the last
    // good radius so the shape stays closed rather than spiking to the centre.
    if (hit <= 0) hit = prev * 0.6;
    prev = hit;
    pts.push({ x: cx + dx * hit, y: cy + dy * hit });
  }
  return pts;
}

/* ---- the extraction ---- */

export function extractTeeth(img, innerNorm, ctx, o, wantDebug = false) {
  const none = { contour: null, contrast: 0, area: 0, debug: null };
  const ring = scaleRing(innerNorm, 1 - (o.cavityErode ?? 0.18));

  let x0 = 1, y0 = 1, x1 = 0, y1 = 0;
  for (const p of ring) {
    x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y);
    x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y);
  }
  const W = img.naturalWidth, H = img.naturalHeight;
  const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H));
  const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H));
  if (pw < 5 || ph < 5) return none;

  ctx.canvas.width = pw; ctx.canvas.height = ph;
  ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph);
  const src = ctx.getImageData(0, 0, pw, ph);
  const d = src.data;

  const poly = ring.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 }));
  const hist = new Uint32Array(256);
  const lum = new Float32Array(pw * ph);
  const red = new Float32Array(pw * ph);
  const inReg = new Uint8Array(pw * ph);
  let n = 0;
  for (let y = 0; y < ph; y++) for (let x = 0; x < pw; x++) {
    if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue;
    const i = y * pw + x, oo = i * 4;
    const R = d[oo], G = d[oo + 1], B = d[oo + 2];
    lum[i] = (0.299 * R + 0.587 * G + 0.114 * B) | 0;
    // Tongue is red relative to its own brightness; teeth are near-neutral.
    red[i] = (R - (G + B) / 2) / 255;
    inReg[i] = 1; hist[lum[i]]++; n++;
  }
  if (n < 24) return none;

  const { thr, mDark, mBright } = otsu(hist, n);
  const contrast = (mBright - mDark) / 255;

  let mask = new Uint8Array(pw * ph);
  for (let i = 0; i < mask.length; i++) {
    mask[i] = inReg[i] && lum[i] > thr && red[i] < (o.tongueReject ?? 0.18) ? 1 : 0;
  }

  // Open once to despeckle, then apply the signed size adjustment.
  mask = dilateMask(erodeMask(mask, pw, ph), pw, ph);
  const grow = o.blobGrow | 0;
  for (let k = 0; k < Math.abs(grow); k++) {
    mask = grow < 0 ? erodeMask(mask, pw, ph) : dilateMask(mask, pw, ph);
  }

  const comp = bestComponent(mask, pw, ph, o.topBias ?? 0.6);
  if (!comp || comp.area < (o.minArea ?? 12)) {
    return { contour: null, contrast, area: comp ? comp.area : 0,
             debug: wantDebug ? debugCanvas(src, inReg, mask, pw, ph, null) : null };
  }

  const only = new Uint8Array(mask.length);
  let cx = 0, cy = 0;
  for (const i of comp.px) { only[i] = 1; cx += i % pw; cy += (i / pw) | 0; }
  cx /= comp.px.length; cy /= comp.px.length;

  const local = radialContour(only, pw, ph, cx, cy, o.verts ?? 10);
  const contour = local.map((p) => ({ x: (p.x + px0) / W, y: (p.y + py0) / H }));

  return {
    contour, contrast, area: comp.area,
    debug: wantDebug ? debugCanvas(src, inReg, only, pw, ph, local) : null,
  };
}

// Sampled region dimmed, kept pixels green, extracted contour in amber.
function debugCanvas(src, inReg, mask, pw, ph, local) {
  const c = document.createElement('canvas');
  c.width = pw; c.height = ph;
  const g = c.getContext('2d');
  const out = new ImageData(pw, ph);
  for (let i = 0; i < pw * ph; i++) {
    const o = i * 4;
    const [r, gr, b] = [src.data[o], src.data[o + 1], src.data[o + 2]];
    if (!inReg[i]) { out.data[o] = r * 0.25; out.data[o + 1] = gr * 0.25; out.data[o + 2] = b * 0.25; }
    else if (mask[i]) { out.data[o] = 60; out.data[o + 1] = 230; out.data[o + 2] = 120; }
    else { out.data[o] = r; out.data[o + 1] = gr; out.data[o + 2] = b; }
    out.data[o + 3] = 255;
  }
  g.putImageData(out, 0, 0);
  if (local && local.length) {
    g.strokeStyle = '#fbbf24'; g.lineWidth = 1;
    g.beginPath();
    local.forEach((p, i) => (i ? g.lineTo(p.x, p.y) : g.moveTo(p.x, p.y)));
    g.closePath(); g.stroke();
  }
  return c;
}