palette
diff --git a/js/app.js b/js/app.js
index 88be7ea..b569ccf 100644
--- a/js/app.js
+++ b/js/app.js
@@ -32,8 +32,9 @@ const state = {
const el = (id) => document.getElementById(id);
const opts = () => ({
verts: +el('verts').value,
- teethOn: +el('teethOn').value / 100,
+ teethOn: +el('teethOn').value / 100, // minimum Otsu class separation
teethDwell: +el('teethDwell').value,
+ teethErode: +el('teethErode').value / 100,
smoothWin: +el('smoothWin').value,
contourSmooth: +el('contourSmooth').value,
apertureThresh: +el('apertureThresh').value / 1000,
@@ -123,10 +124,11 @@ async function detectAll(images) {
// Interior measurement is a function of pixels alone, so it runs once with
// detection and the knobs re-resolve it instantly afterwards.
-function measureAll(images, dense) {
+function measureAll(images, dense, o) {
const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
return dense.map((lm, i) =>
- measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx));
+ measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx,
+ { minContrast: 0, erode: o.teethErode })); // gate applied later, in resolveTeeth
}
/* ---------- build ---------- */
@@ -167,6 +169,10 @@ function rebuild(resetKeep) {
const apMax = Math.max(...ap);
state.hidden = ap.map((v) => v / apMax < o.apertureThresh);
+ if (state.images.length && (!state.interior || state.erodeUsed !== o.teethErode)) {
+ state.interior = measureAll(state.images, state.dense, o);
+ state.erodeUsed = o.teethErode;
+ }
state.teeth = resolveTeeth(o);
// Plate outline per frame, so a kept frame shows its own head shape.
@@ -203,8 +209,12 @@ function resolveTeeth(o) {
const N = state.dense.length;
if (!state.interior) return new Array(N).fill({ show: false, t: 0 });
- const raw = state.interior.map((m, f) => (state.hidden[f] ? 0 : m.coverage));
- const on = o.teethOn, off = o.teethOn * 0.6;
+ // Gate on the separation between Otsu's two class means, not on how much of
+ // the cavity is bright: coverage is high both when the mouth is full of teeth
+ // and when the region is uniformly dark and Otsu has split noise.
+ const raw = state.interior.map((m, f) =>
+ (state.hidden[f] || m.teethT <= 0 ? 0 : m.contrast));
+ const on = o.teethOn, off = o.teethOn * 0.7;
const shown = new Array(N).fill(false);
let live = false, since = 0;
for (let f = 0; f < N; f++) {
@@ -369,6 +379,30 @@ function drawPanes() {
if (!state.hidden[f]) strokePts(g2, z(state.inner[f]), '#f87171');
compositeRender(el('cv-render'), f, ZOOM);
+ drawInteriorDebug(f);
+}
+
+// What the teeth measurement actually saw: sampled region, pixels above
+// threshold in green, the resolved line in amber. Recomputed for the current
+// frame only, so it costs nothing to keep on screen.
+function drawInteriorDebug(f) {
+ const host = el('cv-teeth');
+ const img = state.images[f];
+ if (!img || state.hidden[f]) { host.innerHTML = ''; el('teethinfo').textContent = state.images.length ? 'mouth closed' : 'no source frames'; return; }
+ const o = opts();
+ const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
+ const m = measureInterior(img, LIPS_INNER.map((k) => state.dense[f][k]), ctx,
+ { minContrast: 0, erode: o.teethErode }, true);
+ host.innerHTML = '';
+ if (m.debug) {
+ m.debug.style.width = '150px';
+ m.debug.style.imageRendering = 'pixelated';
+ host.append(m.debug);
+ }
+ const te = state.teeth[f];
+ el('teethinfo').textContent =
+ `contrast ${m.contrast.toFixed(3)} (gate ${o.teethOn.toFixed(2)}) · ` +
+ `line ${m.teethT.toFixed(2)} · ${te.show ? 'SHOWN' : 'hidden'}`;
}
function strokePts(g, pts, color, lw = 1) {
@@ -513,7 +547,7 @@ async function runFrames() {
state.images = images; state.dense = dense;
state.aspect = images[0].naturalWidth / images[0].naturalHeight;
status('measuring mouth interiors…');
- state.interior = measureAll(images, dense);
+ state.interior = measureAll(images, dense, opts());
el('scrub').max = dense.length - 1;
state.frame = 0;
rebuild(true);
@@ -540,11 +574,12 @@ function runSynthetic() {
status('synthetic — exercises everything below detection', 'ok');
}
-for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell']) {
+for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell', 'teethErode']) {
el(id).addEventListener('input', () => {
el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol'
? (+el(id).value / 1000).toFixed(3)
- : id === 'teethOn' ? (+el(id).value / 100).toFixed(2) : el(id).value;
+ : id === 'teethOn' || id === 'teethErode' ? (+el(id).value / 100).toFixed(2)
+ : el(id).value;
if (id === 'tol') return; // tol only matters when you ask for a suggestion
rebuild(false);
});
diff --git a/js/interior.js b/js/interior.js
index 056b75c..46580e6 100644
--- a/js/interior.js
+++ b/js/interior.js
@@ -16,21 +16,23 @@
// Otsu's threshold over a luminance histogram. Self-tuning, so exposure changes
// between frames do not shift what counts as "bright".
+export function otsuForTest(h, t) { return otsu(h, t); }
+
function otsu(hist, total) {
let sum = 0;
for (let i = 0; i < 256; i++) sum += i * hist[i];
- let sumB = 0, wB = 0, best = 0, bestVar = -1;
+ let sumB = 0, wB = 0, best = 0, bestVar = -1, bestMB = 0, bestMF = 0;
for (let t = 0; t < 256; t++) {
wB += hist[t];
if (!wB) continue;
const wF = total - wB;
if (!wF) break;
sumB += t * hist[t];
- const mB = sumB / wB, mF = (sum - sumB) / wF;
+ const mB = sumB / wB, mF = (sum - sumB) / wF; // mB = dark class, mF = bright
const between = wB * wF * (mB - mF) * (mB - mF);
- if (between > bestVar) { bestVar = between; best = t; }
+ if (between > bestVar) { bestVar = between; best = t; bestMB = mB; bestMF = mF; }
}
- return best;
+ return { thr: best, mDark: bestMB, mBright: bestMF };
}
const pointInPoly = (pts, x, y) => {
@@ -42,24 +44,46 @@ const pointInPoly = (pts, x, y) => {
return inside;
};
+// Shrink a ring toward its centroid.
+//
+// MediaPipe's inner lip landmarks sit slightly OUTSIDE the actual opening, so
+// sampling the ring as given includes lip pixels - which are bright, and sit
+// right at the cavity boundary where they do the most damage.
+function erode(pts, k) {
+ let cx = 0, cy = 0;
+ for (const p of pts) { cx += p.x; cy += p.y; }
+ cx /= pts.length; cy /= pts.length;
+ return pts.map((p) => ({ x: cx + (p.x - cx) * (1 - k), y: cy + (p.y - cy) * (1 - k) }));
+}
+
// Measure one frame: how far down the cavity the bright region reaches, as a
-// fraction of cavity height, plus how much of the cavity is bright at all.
-export function measureInterior(img, innerNorm, ctx) {
+// fraction of cavity height, plus the contrast that justified calling it bright.
+//
+// `wantDebug` returns the sampled crop with the classification drawn on it.
+// Tuning this blind is miserable; the numbers alone do not say whether the
+// region being measured is even the right region.
+export function measureInterior(img, innerNorm, ctx, opts = {}, wantDebug = false) {
+ const minContrast = opts.minContrast ?? 0.14;
+ const rowFrac = opts.rowFrac ?? 0.4;
+ const inner = erode(innerNorm, opts.erode ?? 0.18);
+
+ const none = { teethT: 0, contrast: 0, coverage: 0, debug: null };
let x0 = 1, y0 = 1, x1 = 0, y1 = 0;
- for (const p of innerNorm) {
+ for (const p of inner) {
x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y);
x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y);
}
const W = img.naturalWidth, H = img.naturalHeight;
const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H));
const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H));
- if (pw < 3 || ph < 3) return { teethT: 0, coverage: 0 };
+ if (pw < 4 || ph < 4) return none;
ctx.canvas.width = pw; ctx.canvas.height = ph;
ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph);
- const data = ctx.getImageData(0, 0, pw, ph).data;
+ const img0 = ctx.getImageData(0, 0, pw, ph);
+ const data = img0.data;
- const poly = innerNorm.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 }));
+ const poly = inner.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 }));
const hist = new Uint32Array(256);
const lum = new Float32Array(pw * ph);
const mask = new Uint8Array(pw * ph);
@@ -73,13 +97,24 @@ export function measureInterior(img, innerNorm, ctx) {
lum[i] = l; mask[i] = 1; hist[l]++; n++;
}
}
- if (n < 12) return { teethT: 0, coverage: 0 };
+ if (n < 16) return none;
- const thr = otsu(hist, n);
- // Rows are scanned from the top of the cavity; the teeth line is where the
- // bright run stops. Requiring a run rather than a single row rejects specular
- // speckle on a wet lower lip.
- let bright = 0, lastBrightRow = -1, run = 0;
+ const { thr, mDark, mBright } = otsu(hist, n);
+
+ // Otsu ALWAYS returns a split, including on a homogeneous region: given a dark
+ // cavity with no teeth it invents a threshold and calls half the pixels
+ // bright. The separation between the two class means is what says whether the
+ // split means anything, so it is the actual gate.
+ const contrast = (mBright - mDark) / 255;
+ if (contrast < minContrast) {
+ return { teethT: 0, contrast, coverage: 0, debug: wantDebug ? debugCanvas(img0, mask, lum, thr, pw, ph, -1) : null };
+ }
+
+ // Scan from the top and STOP at the first row that fails. Teeth hang from the
+ // upper lip, so what matters is the contiguous run, not whether some row near
+ // the bottom happens to qualify - tracking the latter is what made the band
+ // fill the whole mouth.
+ let lastRow = -1, started = false, bright = 0;
for (let y = 0; y < ph; y++) {
let rowIn = 0, rowBright = 0;
for (let x = 0; x < pw; x++) {
@@ -88,15 +123,43 @@ export function measureInterior(img, innerNorm, ctx) {
rowIn++;
if (lum[i] > thr) { rowBright++; bright++; }
}
- if (rowIn >= 2 && rowBright / rowIn > 0.45) { run++; if (run >= 1) lastBrightRow = y; }
- else run = 0;
+ if (rowIn < 2) continue;
+ const ok = rowBright / rowIn > rowFrac;
+ if (ok) { started = true; lastRow = y; }
+ else if (started) break;
}
+
return {
- teethT: lastBrightRow < 0 ? 0 : (lastBrightRow + 1) / ph,
+ teethT: lastRow < 0 ? 0 : (lastRow + 1) / ph,
+ contrast,
coverage: bright / n,
+ debug: wantDebug ? debugCanvas(img0, mask, lum, thr, pw, ph, lastRow) : null,
};
}
+// The sampled crop with the classification painted on: sampled region tinted,
+// pixels above threshold in green, the resolved teeth line in amber.
+function debugCanvas(img0, mask, lum, thr, pw, ph, lastRow) {
+ const c = document.createElement('canvas');
+ c.width = pw; c.height = ph;
+ const g = c.getContext('2d');
+ const out = new ImageData(pw, ph);
+ for (let i = 0; i < pw * ph; i++) {
+ const o = i * 4;
+ const [r, gr, b] = [img0.data[o], img0.data[o + 1], img0.data[o + 2]];
+ if (!mask[i]) { out.data[o] = r * 0.3; out.data[o + 1] = gr * 0.3; out.data[o + 2] = b * 0.3; }
+ else if (lum[i] > thr) { out.data[o] = 60; out.data[o + 1] = 230; out.data[o + 2] = 120; }
+ else { out.data[o] = r; out.data[o + 1] = gr; out.data[o + 2] = b; }
+ out.data[o + 3] = 255;
+ }
+ g.putImageData(out, 0, 0);
+ if (lastRow >= 0) {
+ g.fillStyle = '#fbbf24';
+ g.fillRect(0, lastRow, pw, 1);
+ }
+ return c;
+}
+
// Sutherland-Hodgman against the half-plane y <= limit.
export function clipPolyAbove(pts, limit) {
const out = [];
diff --git a/js/selftest.js b/js/selftest.js
index 9b6cd95..ddbeec9 100644
--- a/js/selftest.js
+++ b/js/selftest.js
@@ -12,7 +12,7 @@ import { fitSimilarity, applySim, procrustesMean, smoothTransforms } from './mat
import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js';
import { IndexedRaster, hexToRgb } from './raster.js';
import { writeTake } from './take.js';
-import { clipPolyAbove } from './interior.js';
+import { clipPolyAbove, otsuForTest } from './interior.js';
import { synthDense } from './synth.js';
const results = [];
@@ -203,6 +203,30 @@ export function run() {
Math.min(...ct.map((p) => p.x)) >= 0 && Math.max(...ct.map((p) => p.x)) <= 8);
}
+ // Otsu on a uniform region must report near-zero class separation. It will
+ // still return a threshold - that is what Otsu does - so the separation is the
+ // only thing that distinguishes "found teeth" from "split noise in a dark
+ // mouth", which is what made the band fill the whole cavity.
+ {
+ const flat = new Uint32Array(256); flat[40] = 500;
+ const f = otsuForTest(flat, 500);
+ ok('uniform region yields ~no class separation',
+ Math.abs(f.mBright - f.mDark) / 255 < 0.02, `sep ${((f.mBright - f.mDark) / 255).toFixed(4)}`);
+
+ const noisy = new Uint32Array(256);
+ for (let i = 30; i <= 60; i++) noisy[i] = 20; // dark cavity, some spread
+ const nz = otsuForTest(noisy, 31 * 20);
+ ok('dark-but-noisy region stays below a sane gate',
+ (nz.mBright - nz.mDark) / 255 < 0.14, `sep ${((nz.mBright - nz.mDark) / 255).toFixed(4)}`);
+
+ const teeth = new Uint32Array(256);
+ for (let i = 20; i <= 45; i++) teeth[i] = 40; // cavity
+ for (let i = 180; i <= 220; i++) teeth[i] = 30; // teeth
+ const tt = otsuForTest(teeth, 26 * 40 + 41 * 30);
+ ok('real bright/dark split clears the gate',
+ (tt.mBright - tt.mDark) / 255 > 0.4, `sep ${((tt.mBright - tt.mDark) / 255).toFixed(4)}`);
+ }
+
// take writer round-trip
const take = {
name: 'test', frames: 72, width: 320, height: 200, exposure: 2,