Teeth as an extracted blob contour, not a clipped band
The band filled the mouth because a band is the wrong reduction: the bright region is a blob, and reading it as "everything above a line" throws the shape away. Extracting a contour reintroduces the vertex-correspondence problem that made me avoid it, but for a blob there is a way out. Radial sampling from the centroid along N fixed directions makes vertex k always mean "the extent in direction k": correspondence holds by construction, the count is fixed, and temporal smoothing cannot reorder anything. It also yields a star-shaped reduction, which suits flat colour. Tongue rejection, which the band had no way to express: - pixels red relative to their own brightness are dropped (teeth are neutral) - component choice is biased toward the top of the cavity, since area alone picks the tongue when the mouth is wide - separate inner and outer controls: cavity erode pulls the sampled region off the lip edge, blob grow/erode resizes the found blob Also: a knob wired in app.js but missing from index.html threw during wiring and left a blank page with nothing useful in the console - which is exactly what happened to teethDwell in the previous commit. el() now names the missing id, and window.onerror surfaces it in the status line. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
c6daf827d1
commit
9a11eeabc1
7 changed files with 272 additions and 275 deletions
116
js/app.js
116
js/app.js
|
|
@ -3,7 +3,8 @@ import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js';
|
|||
import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js';
|
||||
import { IndexedRaster } from './raster.js';
|
||||
import { drawRegistered, posterizeInto } from './underlay.js';
|
||||
import { measureInterior, clipPolyAbove } from './interior.js';
|
||||
import { extractTeeth } from './interior.js';
|
||||
import { applySim } from './mathutil.js';
|
||||
import { writeTake } from './take.js';
|
||||
import { synthDense } from './synth.js';
|
||||
|
||||
|
|
@ -29,12 +30,23 @@ const state = {
|
|||
teeth: null, // resolved per-frame {show, t} after knobs
|
||||
};
|
||||
|
||||
const el = (id) => document.getElementById(id);
|
||||
const el = (id) => {
|
||||
const n = document.getElementById(id);
|
||||
// A knob present in the code but missing from the markup used to throw during
|
||||
// wiring and leave a blank page with nothing in the console worth reading.
|
||||
if (!n) throw new Error(`missing element #${id} — knob wired in app.js but not in index.html`);
|
||||
return n;
|
||||
};
|
||||
const opts = () => ({
|
||||
verts: +el('verts').value,
|
||||
teethOn: +el('teethOn').value / 100, // minimum Otsu class separation
|
||||
teethDwell: +el('teethDwell').value,
|
||||
teethErode: +el('teethErode').value / 100,
|
||||
teethSmooth: +el('teethSmooth').value,
|
||||
cavityErode: +el('teethErode').value / 100,
|
||||
tongueReject: +el('tongueReject').value / 100,
|
||||
blobGrow: +el('blobGrow').value,
|
||||
topBias: +el('topBias').value / 100,
|
||||
verts: +el('teethVerts').value,
|
||||
smoothWin: +el('smoothWin').value,
|
||||
contourSmooth: +el('contourSmooth').value,
|
||||
apertureThresh: +el('apertureThresh').value / 1000,
|
||||
|
|
@ -127,10 +139,14 @@ async function detectAll(images) {
|
|||
function measureAll(images, dense, o) {
|
||||
const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
|
||||
return dense.map((lm, i) =>
|
||||
measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx,
|
||||
{ minContrast: 0, erode: o.teethErode })); // gate applied later, in resolveTeeth
|
||||
extractTeeth(images[i], LIPS_INNER.map((k) => lm[k]), ctx, o));
|
||||
}
|
||||
|
||||
// Extraction keys on every knob that changes the pixels examined, so the cache
|
||||
// is keyed on exactly those and a change to anything else stays instant.
|
||||
const extractKey = (o) =>
|
||||
[o.cavityErode, o.tongueReject, o.blobGrow, o.topBias, o.verts].join('|');
|
||||
|
||||
/* ---------- build ---------- */
|
||||
|
||||
function makeXform(stab, neutral) {
|
||||
|
|
@ -169,9 +185,9 @@ function rebuild(resetKeep) {
|
|||
const apMax = Math.max(...ap);
|
||||
state.hidden = ap.map((v) => v / apMax < o.apertureThresh);
|
||||
|
||||
if (state.images.length && (!state.interior || state.erodeUsed !== o.teethErode)) {
|
||||
if (state.images.length && state.extractKey !== extractKey(o)) {
|
||||
state.interior = measureAll(state.images, state.dense, o);
|
||||
state.erodeUsed = o.teethErode;
|
||||
state.extractKey = extractKey(o);
|
||||
}
|
||||
state.teeth = resolveTeeth(o);
|
||||
|
||||
|
|
@ -207,13 +223,10 @@ function faceBoxes() {
|
|||
// needs only a weak one.
|
||||
function resolveTeeth(o) {
|
||||
const N = state.dense.length;
|
||||
if (!state.interior) return new Array(N).fill({ show: false, t: 0 });
|
||||
if (!state.interior) return new Array(N).fill({ show: false, pts: null });
|
||||
|
||||
// Gate on the separation between Otsu's two class means, not on how much of
|
||||
// the cavity is bright: coverage is high both when the mouth is full of teeth
|
||||
// and when the region is uniformly dark and Otsu has split noise.
|
||||
const raw = state.interior.map((m, f) =>
|
||||
(state.hidden[f] || m.teethT <= 0 ? 0 : m.contrast));
|
||||
(state.hidden[f] || !m.contour ? 0 : m.contrast));
|
||||
const on = o.teethOn, off = o.teethOn * 0.7;
|
||||
const shown = new Array(N).fill(false);
|
||||
let live = false, since = 0;
|
||||
|
|
@ -221,22 +234,34 @@ function resolveTeeth(o) {
|
|||
const want = live ? raw[f] > off : raw[f] > on;
|
||||
if (want !== live && since >= o.teethDwell) { live = want; since = 0; }
|
||||
else since++;
|
||||
shown[f] = live && !state.hidden[f];
|
||||
shown[f] = live && !state.hidden[f] && !!state.interior[f].contour;
|
||||
}
|
||||
|
||||
// Smooth the line height only across frames where it is actually shown, so a
|
||||
// gap does not drag the band toward zero on either side of it.
|
||||
const t = state.interior.map((m) => m.teethT);
|
||||
const sm = t.map((_, f) => {
|
||||
let acc = 0, c = 0;
|
||||
for (let j = f - o.contourSmooth; j <= f + o.contourSmooth; j++) {
|
||||
// Into raster space through the same chain the lips take, including the
|
||||
// isotropic aspect conversion - a contour in MediaPipe's normalised space is
|
||||
// in the same stretched coordinates the landmarks are.
|
||||
const toRaster = (pts, f) => {
|
||||
const tf = state.stab.transforms[f];
|
||||
return pts.map((p) => state.xform(applySim(tf, { x: p.x * state.aspect, y: p.y })));
|
||||
};
|
||||
const rast = state.interior.map((m, f) => (m.contour ? toRaster(m.contour, f) : null));
|
||||
|
||||
// Radial sampling makes vertex k mean the same direction on every frame, so
|
||||
// smoothing across time is well defined and cannot reorder anything.
|
||||
const sm = rast.map((pts, f) => {
|
||||
if (!pts || !shown[f]) return pts;
|
||||
const acc = pts.map(() => ({ x: 0, y: 0 }));
|
||||
let c = 0;
|
||||
for (let j = f - o.teethSmooth; j <= f + o.teethSmooth; j++) {
|
||||
const k = Math.min(N - 1, Math.max(0, j));
|
||||
if (!shown[k]) continue;
|
||||
acc += t[k]; c++;
|
||||
if (!shown[k] || !rast[k] || rast[k].length !== pts.length) continue;
|
||||
for (let v = 0; v < pts.length; v++) { acc[v].x += rast[k][v].x; acc[v].y += rast[k][v].y; }
|
||||
c++;
|
||||
}
|
||||
return c ? acc / c : t[f];
|
||||
return c ? acc.map((p) => ({ x: p.x / c, y: p.y / c })) : pts;
|
||||
});
|
||||
return shown.map((show, f) => ({ show, t: sm[f] }));
|
||||
|
||||
return shown.map((show, f) => ({ show, pts: sm[f] }));
|
||||
}
|
||||
|
||||
/* ---------- render ---------- */
|
||||
|
|
@ -261,14 +286,7 @@ function renderFrame(f, mode = plateMode()) {
|
|||
if (!state.hidden[f]) {
|
||||
r.fillPoly(state.inner[f], IDX.mouth);
|
||||
const te = state.teeth[f];
|
||||
if (te.show) {
|
||||
// The band is the mouth's own silhouette clipped to a height, so the shape
|
||||
// can never disagree with the lips around it.
|
||||
const ys = state.inner[f].map((p) => p.y);
|
||||
const top = Math.min(...ys), bot = Math.max(...ys);
|
||||
const poly = clipPolyAbove(state.inner[f], top + te.t * (bot - top));
|
||||
if (poly.length >= 3) r.fillPoly(poly, IDX.teeth);
|
||||
}
|
||||
if (te.show && te.pts && te.pts.length >= 3) r.fillPoly(te.pts, IDX.teeth);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
|
@ -388,21 +406,24 @@ function drawPanes() {
|
|||
function drawInteriorDebug(f) {
|
||||
const host = el('cv-teeth');
|
||||
const img = state.images[f];
|
||||
if (!img || state.hidden[f]) { host.innerHTML = ''; el('teethinfo').textContent = state.images.length ? 'mouth closed' : 'no source frames'; return; }
|
||||
if (!img || state.hidden[f]) {
|
||||
host.innerHTML = '';
|
||||
el('teethinfo').textContent = state.images.length ? 'mouth closed' : 'no source frames';
|
||||
return;
|
||||
}
|
||||
const o = opts();
|
||||
const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
|
||||
const m = measureInterior(img, LIPS_INNER.map((k) => state.dense[f][k]), ctx,
|
||||
{ minContrast: 0, erode: o.teethErode }, true);
|
||||
const m = extractTeeth(img, LIPS_INNER.map((k) => state.dense[f][k]), ctx, o, true);
|
||||
host.innerHTML = '';
|
||||
if (m.debug) {
|
||||
m.debug.style.width = '150px';
|
||||
m.debug.style.width = '170px';
|
||||
m.debug.style.imageRendering = 'pixelated';
|
||||
host.append(m.debug);
|
||||
}
|
||||
const te = state.teeth[f];
|
||||
el('teethinfo').textContent =
|
||||
`contrast ${m.contrast.toFixed(3)} (gate ${o.teethOn.toFixed(2)}) · ` +
|
||||
`line ${m.teethT.toFixed(2)} · ${te.show ? 'SHOWN' : 'hidden'}`;
|
||||
`contrast ${m.contrast.toFixed(3)} / gate ${o.teethOn.toFixed(2)} · ` +
|
||||
`area ${m.area}px · ${te.show ? 'SHOWN' : 'hidden'}`;
|
||||
}
|
||||
|
||||
function strokePts(g, pts, color, lw = 1) {
|
||||
|
|
@ -503,13 +524,8 @@ function exportTake() {
|
|||
{ name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
|
||||
keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) },
|
||||
{ name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in',
|
||||
keys: state.inner.map((pts, f) => {
|
||||
const te = state.teeth[f];
|
||||
if (state.hidden[f] || !te.show) return { f, hidden: true };
|
||||
const ys = pts.map((p) => p.y);
|
||||
const top = Math.min(...ys), bot = Math.max(...ys);
|
||||
return { f, src: f, pts: clipPolyAbove(pts, top + te.t * (bot - top)) };
|
||||
}) },
|
||||
keys: state.teeth.map((te, f) =>
|
||||
(te.show && te.pts ? { f, src: f, pts: te.pts } : { f, hidden: true })) },
|
||||
],
|
||||
};
|
||||
const text = writeTake(take)
|
||||
|
|
@ -574,11 +590,14 @@ function runSynthetic() {
|
|||
status('synthetic — exercises everything below detection', 'ok');
|
||||
}
|
||||
|
||||
for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell', 'teethErode']) {
|
||||
for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol',
|
||||
'teethOn', 'teethDwell', 'teethErode', 'tongueReject', 'blobGrow',
|
||||
'topBias', 'teethVerts', 'teethSmooth']) {
|
||||
el(id).addEventListener('input', () => {
|
||||
el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol'
|
||||
? (+el(id).value / 1000).toFixed(3)
|
||||
: id === 'teethOn' || id === 'teethErode' ? (+el(id).value / 100).toFixed(2)
|
||||
: ['teethOn', 'teethErode', 'tongueReject', 'topBias'].includes(id)
|
||||
? (+el(id).value / 100).toFixed(2)
|
||||
: el(id).value;
|
||||
if (id === 'tol') return; // tol only matters when you ask for a suggestion
|
||||
rebuild(false);
|
||||
|
|
@ -699,6 +718,11 @@ PALETTE.forEach((p) => {
|
|||
|
||||
// #synth / #frames autorun, so the tool can be driven headlessly for smoke tests
|
||||
// and deep-linked. Detection needs WebGL; the synthetic path does not.
|
||||
window.addEventListener('error', (e) => {
|
||||
const s = document.getElementById('status');
|
||||
if (s) { s.textContent = e.message; s.className = 'err'; }
|
||||
});
|
||||
|
||||
if (location.hash === '#synth') runSynthetic();
|
||||
else if (location.hash === '#frames') runFrames();
|
||||
else status('ready — Load frames, then step with \u2190 \u2192 and delete with X');
|
||||
|
|
|
|||
255
js/interior.js
255
js/interior.js
|
|
@ -1,38 +1,37 @@
|
|||
// Mouth interior from image content.
|
||||
//
|
||||
// MediaPipe has no landmarks inside the lips - the inner ring bounds the cavity
|
||||
// and everything within it is just pixels. So teeth have to come from the
|
||||
// picture, and the question is how to do that without reintroducing the boil
|
||||
// that per-frame detection causes.
|
||||
// and everything within it is just pixels. So teeth come from the picture.
|
||||
//
|
||||
// The answer is to extract a SCALAR, not a shape. Tracing the bright blob would
|
||||
// give a new contour every frame with no vertex correspondence - exactly the
|
||||
// failure docs/roto-puppet.md warns about. Instead the teeth polygon is the
|
||||
// inner lip ring clipped to a horizontal line, and only that line's height is
|
||||
// measured. The silhouette is therefore always the mouth's own shape (stable by
|
||||
// construction) and the only thing that varies per frame is one number, which
|
||||
// smooths trivially. It is also how the shape is drawn by hand: a band bounded
|
||||
// by the lip.
|
||||
// The hazard is vertex correspondence. A traced contour reorders between frames
|
||||
// and boils, which is the failure docs/roto-puppet.md exists to avoid. The way
|
||||
// out for a blob specifically is RADIAL SAMPLING: march outward from the
|
||||
// centroid along N fixed directions and take the last pixel inside. Vertex k is
|
||||
// then always "the blob's extent in direction k" - correspondence holds by
|
||||
// construction, the vertex count is fixed, and the result smooths over time
|
||||
// without any reordering being possible. It also yields a star-shaped
|
||||
// reduction, which is what flat blocks of colour want anyway.
|
||||
|
||||
// Otsu's threshold over a luminance histogram. Self-tuning, so exposure changes
|
||||
// between frames do not shift what counts as "bright".
|
||||
export function otsuForTest(h, t) { return otsu(h, t); }
|
||||
|
||||
// Otsu's threshold plus its two class means. The means matter as much as the
|
||||
// threshold: Otsu ALWAYS returns a split, including on a homogeneous region, so
|
||||
// their separation is the only thing that says the split means anything.
|
||||
function otsu(hist, total) {
|
||||
let sum = 0;
|
||||
for (let i = 0; i < 256; i++) sum += i * hist[i];
|
||||
let sumB = 0, wB = 0, best = 0, bestVar = -1, bestMB = 0, bestMF = 0;
|
||||
let sumB = 0, wB = 0, best = 0, bestVar = -1, bestDark = 0, bestBright = 0;
|
||||
for (let t = 0; t < 256; t++) {
|
||||
wB += hist[t];
|
||||
if (!wB) continue;
|
||||
const wF = total - wB;
|
||||
if (!wF) break;
|
||||
sumB += t * hist[t];
|
||||
const mB = sumB / wB, mF = (sum - sumB) / wF; // mB = dark class, mF = bright
|
||||
const between = wB * wF * (mB - mF) * (mB - mF);
|
||||
if (between > bestVar) { bestVar = between; best = t; bestMB = mB; bestMF = mF; }
|
||||
const mDark = sumB / wB, mBright = (sum - sumB) / wF;
|
||||
const between = wB * wF * (mDark - mBright) * (mDark - mBright);
|
||||
if (between > bestVar) { bestVar = between; best = t; bestDark = mDark; bestBright = mBright; }
|
||||
}
|
||||
return { thr: best, mDark: bestMB, mBright: bestMF };
|
||||
return { thr: best, mDark: bestDark, mBright: bestBright };
|
||||
}
|
||||
|
||||
const pointInPoly = (pts, x, y) => {
|
||||
|
|
@ -44,133 +43,183 @@ const pointInPoly = (pts, x, y) => {
|
|||
return inside;
|
||||
};
|
||||
|
||||
// Shrink a ring toward its centroid.
|
||||
//
|
||||
// MediaPipe's inner lip landmarks sit slightly OUTSIDE the actual opening, so
|
||||
// sampling the ring as given includes lip pixels - which are bright, and sit
|
||||
// right at the cavity boundary where they do the most damage.
|
||||
function erode(pts, k) {
|
||||
// Shrink or grow a ring about its centroid. MediaPipe's inner lip landmarks sit
|
||||
// slightly OUTSIDE the real opening, so sampling the ring as given includes lip
|
||||
// pixels - bright, and right at the boundary where they do most damage.
|
||||
export function scaleRing(pts, k) {
|
||||
let cx = 0, cy = 0;
|
||||
for (const p of pts) { cx += p.x; cy += p.y; }
|
||||
cx /= pts.length; cy /= pts.length;
|
||||
return pts.map((p) => ({ x: cx + (p.x - cx) * (1 - k), y: cy + (p.y - cy) * (1 - k) }));
|
||||
return pts.map((p) => ({ x: cx + (p.x - cx) * k, y: cy + (p.y - cy) * k }));
|
||||
}
|
||||
|
||||
// Measure one frame: how far down the cavity the bright region reaches, as a
|
||||
// fraction of cavity height, plus the contrast that justified calling it bright.
|
||||
//
|
||||
// `wantDebug` returns the sampled crop with the classification drawn on it.
|
||||
// Tuning this blind is miserable; the numbers alone do not say whether the
|
||||
// region being measured is even the right region.
|
||||
export function measureInterior(img, innerNorm, ctx, opts = {}, wantDebug = false) {
|
||||
const minContrast = opts.minContrast ?? 0.14;
|
||||
const rowFrac = opts.rowFrac ?? 0.4;
|
||||
const inner = erode(innerNorm, opts.erode ?? 0.18);
|
||||
/* ---- binary morphology on the candidate mask ---- */
|
||||
|
||||
function erodeMask(m, w, h) {
|
||||
const o = new Uint8Array(m.length);
|
||||
for (let y = 1; y < h - 1; y++) for (let x = 1; x < w - 1; x++) {
|
||||
const i = y * w + x;
|
||||
o[i] = m[i] && m[i - 1] && m[i + 1] && m[i - w] && m[i + w] ? 1 : 0;
|
||||
}
|
||||
return o;
|
||||
}
|
||||
function dilateMask(m, w, h) {
|
||||
const o = new Uint8Array(m.length);
|
||||
for (let y = 1; y < h - 1; y++) for (let x = 1; x < w - 1; x++) {
|
||||
const i = y * w + x;
|
||||
o[i] = m[i] || m[i - 1] || m[i + 1] || m[i - w] || m[i + w] ? 1 : 0;
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
// Largest 4-connected component, scored with a bias toward the TOP of the
|
||||
// cavity: upper teeth hang from the lip, and the usual false positive is the
|
||||
// tongue sitting lower down. Area alone picks the tongue when the mouth is wide.
|
||||
function bestComponent(mask, w, h, topBias) {
|
||||
const label = new Int32Array(mask.length).fill(-1);
|
||||
const stack = [];
|
||||
let best = null, id = 0;
|
||||
for (let s = 0; s < mask.length; s++) {
|
||||
if (!mask[s] || label[s] >= 0) continue;
|
||||
stack.length = 0; stack.push(s);
|
||||
label[s] = id;
|
||||
const px = [];
|
||||
let sumY = 0;
|
||||
while (stack.length) {
|
||||
const i = stack.pop();
|
||||
px.push(i);
|
||||
sumY += (i / w) | 0;
|
||||
const x = i % w, y = (i / w) | 0;
|
||||
if (x > 0 && mask[i - 1] && label[i - 1] < 0) { label[i - 1] = id; stack.push(i - 1); }
|
||||
if (x < w - 1 && mask[i + 1] && label[i + 1] < 0) { label[i + 1] = id; stack.push(i + 1); }
|
||||
if (y > 0 && mask[i - w] && label[i - w] < 0) { label[i - w] = id; stack.push(i - w); }
|
||||
if (y < h - 1 && mask[i + w] && label[i + w] < 0) { label[i + w] = id; stack.push(i + w); }
|
||||
}
|
||||
const meanY = sumY / px.length / h; // 0 top, 1 bottom
|
||||
const score = px.length * (1 - topBias * meanY);
|
||||
if (!best || score > best.score) best = { score, px, area: px.length, meanY };
|
||||
id++;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
// Radial sampling from the centroid: N fixed directions, last pixel inside.
|
||||
function radialContour(mask, w, h, cx, cy, n) {
|
||||
const pts = [];
|
||||
const maxR = Math.hypot(w, h);
|
||||
let prev = 1;
|
||||
for (let k = 0; k < n; k++) {
|
||||
const a = -(k / n) * Math.PI * 2; // slot 0 = +x, 5/20 = top
|
||||
const dx = Math.cos(a), dy = Math.sin(a);
|
||||
let hit = 0;
|
||||
for (let r = 0.5; r < maxR; r += 0.5) {
|
||||
const x = Math.round(cx + dx * r), y = Math.round(cy + dy * r);
|
||||
if (x < 0 || y < 0 || x >= w || y >= h) break;
|
||||
if (mask[y * w + x]) hit = r;
|
||||
else if (hit > 0 && r > hit + 2) break; // tolerate a 2px gap, then stop
|
||||
}
|
||||
// A ray that escapes immediately would collapse the polygon; hold the last
|
||||
// good radius so the shape stays closed rather than spiking to the centre.
|
||||
if (hit <= 0) hit = prev * 0.6;
|
||||
prev = hit;
|
||||
pts.push({ x: cx + dx * hit, y: cy + dy * hit });
|
||||
}
|
||||
return pts;
|
||||
}
|
||||
|
||||
/* ---- the extraction ---- */
|
||||
|
||||
export function extractTeeth(img, innerNorm, ctx, o, wantDebug = false) {
|
||||
const none = { contour: null, contrast: 0, area: 0, debug: null };
|
||||
const ring = scaleRing(innerNorm, 1 - (o.cavityErode ?? 0.18));
|
||||
|
||||
const none = { teethT: 0, contrast: 0, coverage: 0, debug: null };
|
||||
let x0 = 1, y0 = 1, x1 = 0, y1 = 0;
|
||||
for (const p of inner) {
|
||||
for (const p of ring) {
|
||||
x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y);
|
||||
x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y);
|
||||
}
|
||||
const W = img.naturalWidth, H = img.naturalHeight;
|
||||
const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H));
|
||||
const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H));
|
||||
if (pw < 4 || ph < 4) return none;
|
||||
if (pw < 5 || ph < 5) return none;
|
||||
|
||||
ctx.canvas.width = pw; ctx.canvas.height = ph;
|
||||
ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph);
|
||||
const img0 = ctx.getImageData(0, 0, pw, ph);
|
||||
const data = img0.data;
|
||||
const src = ctx.getImageData(0, 0, pw, ph);
|
||||
const d = src.data;
|
||||
|
||||
const poly = inner.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 }));
|
||||
const poly = ring.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 }));
|
||||
const hist = new Uint32Array(256);
|
||||
const lum = new Float32Array(pw * ph);
|
||||
const mask = new Uint8Array(pw * ph);
|
||||
const red = new Float32Array(pw * ph);
|
||||
const inReg = new Uint8Array(pw * ph);
|
||||
let n = 0;
|
||||
for (let y = 0; y < ph; y++) {
|
||||
for (let x = 0; x < pw; x++) {
|
||||
if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue;
|
||||
const o = (y * pw + x) * 4;
|
||||
const l = (0.299 * data[o] + 0.587 * data[o + 1] + 0.114 * data[o + 2]) | 0;
|
||||
const i = y * pw + x;
|
||||
lum[i] = l; mask[i] = 1; hist[l]++; n++;
|
||||
}
|
||||
for (let y = 0; y < ph; y++) for (let x = 0; x < pw; x++) {
|
||||
if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue;
|
||||
const i = y * pw + x, oo = i * 4;
|
||||
const R = d[oo], G = d[oo + 1], B = d[oo + 2];
|
||||
lum[i] = (0.299 * R + 0.587 * G + 0.114 * B) | 0;
|
||||
// Tongue is red relative to its own brightness; teeth are near-neutral.
|
||||
red[i] = (R - (G + B) / 2) / 255;
|
||||
inReg[i] = 1; hist[lum[i]]++; n++;
|
||||
}
|
||||
if (n < 16) return none;
|
||||
if (n < 24) return none;
|
||||
|
||||
const { thr, mDark, mBright } = otsu(hist, n);
|
||||
|
||||
// Otsu ALWAYS returns a split, including on a homogeneous region: given a dark
|
||||
// cavity with no teeth it invents a threshold and calls half the pixels
|
||||
// bright. The separation between the two class means is what says whether the
|
||||
// split means anything, so it is the actual gate.
|
||||
const contrast = (mBright - mDark) / 255;
|
||||
if (contrast < minContrast) {
|
||||
return { teethT: 0, contrast, coverage: 0, debug: wantDebug ? debugCanvas(img0, mask, lum, thr, pw, ph, -1) : null };
|
||||
|
||||
let mask = new Uint8Array(pw * ph);
|
||||
for (let i = 0; i < mask.length; i++) {
|
||||
mask[i] = inReg[i] && lum[i] > thr && red[i] < (o.tongueReject ?? 0.18) ? 1 : 0;
|
||||
}
|
||||
|
||||
// Scan from the top and STOP at the first row that fails. Teeth hang from the
|
||||
// upper lip, so what matters is the contiguous run, not whether some row near
|
||||
// the bottom happens to qualify - tracking the latter is what made the band
|
||||
// fill the whole mouth.
|
||||
let lastRow = -1, started = false, bright = 0;
|
||||
for (let y = 0; y < ph; y++) {
|
||||
let rowIn = 0, rowBright = 0;
|
||||
for (let x = 0; x < pw; x++) {
|
||||
const i = y * pw + x;
|
||||
if (!mask[i]) continue;
|
||||
rowIn++;
|
||||
if (lum[i] > thr) { rowBright++; bright++; }
|
||||
}
|
||||
if (rowIn < 2) continue;
|
||||
const ok = rowBright / rowIn > rowFrac;
|
||||
if (ok) { started = true; lastRow = y; }
|
||||
else if (started) break;
|
||||
// Open once to despeckle, then apply the signed size adjustment.
|
||||
mask = dilateMask(erodeMask(mask, pw, ph), pw, ph);
|
||||
const grow = o.blobGrow | 0;
|
||||
for (let k = 0; k < Math.abs(grow); k++) {
|
||||
mask = grow < 0 ? erodeMask(mask, pw, ph) : dilateMask(mask, pw, ph);
|
||||
}
|
||||
|
||||
const comp = bestComponent(mask, pw, ph, o.topBias ?? 0.6);
|
||||
if (!comp || comp.area < (o.minArea ?? 12)) {
|
||||
return { contour: null, contrast, area: comp ? comp.area : 0,
|
||||
debug: wantDebug ? debugCanvas(src, inReg, mask, pw, ph, null) : null };
|
||||
}
|
||||
|
||||
const only = new Uint8Array(mask.length);
|
||||
let cx = 0, cy = 0;
|
||||
for (const i of comp.px) { only[i] = 1; cx += i % pw; cy += (i / pw) | 0; }
|
||||
cx /= comp.px.length; cy /= comp.px.length;
|
||||
|
||||
const local = radialContour(only, pw, ph, cx, cy, o.verts ?? 10);
|
||||
const contour = local.map((p) => ({ x: (p.x + px0) / W, y: (p.y + py0) / H }));
|
||||
|
||||
return {
|
||||
teethT: lastRow < 0 ? 0 : (lastRow + 1) / ph,
|
||||
contrast,
|
||||
coverage: bright / n,
|
||||
debug: wantDebug ? debugCanvas(img0, mask, lum, thr, pw, ph, lastRow) : null,
|
||||
contour, contrast, area: comp.area,
|
||||
debug: wantDebug ? debugCanvas(src, inReg, only, pw, ph, local) : null,
|
||||
};
|
||||
}
|
||||
|
||||
// The sampled crop with the classification painted on: sampled region tinted,
|
||||
// pixels above threshold in green, the resolved teeth line in amber.
|
||||
function debugCanvas(img0, mask, lum, thr, pw, ph, lastRow) {
|
||||
// Sampled region dimmed, kept pixels green, extracted contour in amber.
|
||||
function debugCanvas(src, inReg, mask, pw, ph, local) {
|
||||
const c = document.createElement('canvas');
|
||||
c.width = pw; c.height = ph;
|
||||
const g = c.getContext('2d');
|
||||
const out = new ImageData(pw, ph);
|
||||
for (let i = 0; i < pw * ph; i++) {
|
||||
const o = i * 4;
|
||||
const [r, gr, b] = [img0.data[o], img0.data[o + 1], img0.data[o + 2]];
|
||||
if (!mask[i]) { out.data[o] = r * 0.3; out.data[o + 1] = gr * 0.3; out.data[o + 2] = b * 0.3; }
|
||||
else if (lum[i] > thr) { out.data[o] = 60; out.data[o + 1] = 230; out.data[o + 2] = 120; }
|
||||
const [r, gr, b] = [src.data[o], src.data[o + 1], src.data[o + 2]];
|
||||
if (!inReg[i]) { out.data[o] = r * 0.25; out.data[o + 1] = gr * 0.25; out.data[o + 2] = b * 0.25; }
|
||||
else if (mask[i]) { out.data[o] = 60; out.data[o + 1] = 230; out.data[o + 2] = 120; }
|
||||
else { out.data[o] = r; out.data[o + 1] = gr; out.data[o + 2] = b; }
|
||||
out.data[o + 3] = 255;
|
||||
}
|
||||
g.putImageData(out, 0, 0);
|
||||
if (lastRow >= 0) {
|
||||
g.fillStyle = '#fbbf24';
|
||||
g.fillRect(0, lastRow, pw, 1);
|
||||
if (local && local.length) {
|
||||
g.strokeStyle = '#fbbf24'; g.lineWidth = 1;
|
||||
g.beginPath();
|
||||
local.forEach((p, i) => (i ? g.lineTo(p.x, p.y) : g.moveTo(p.x, p.y)));
|
||||
g.closePath(); g.stroke();
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
// Sutherland-Hodgman against the half-plane y <= limit.
|
||||
export function clipPolyAbove(pts, limit) {
|
||||
const out = [];
|
||||
for (let i = 0; i < pts.length; i++) {
|
||||
const a = pts[i], b = pts[(i + 1) % pts.length];
|
||||
const ain = a.y <= limit, bin = b.y <= limit;
|
||||
if (ain) out.push(a);
|
||||
if (ain !== bin) {
|
||||
const t = (limit - a.y) / (b.y - a.y);
|
||||
out.push({ x: a.x + t * (b.x - a.x), y: limit });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import { fitSimilarity, applySim, procrustesMean, smoothTransforms } from './mat
|
|||
import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js';
|
||||
import { IndexedRaster, hexToRgb } from './raster.js';
|
||||
import { writeTake } from './take.js';
|
||||
import { clipPolyAbove, otsuForTest } from './interior.js';
|
||||
import { otsuForTest, scaleRing } from './interior.js';
|
||||
import { synthDense } from './synth.js';
|
||||
|
||||
const results = [];
|
||||
|
|
@ -182,25 +182,17 @@ export function run() {
|
|||
ok('palette expansion introduces no intermediate colours',
|
||||
[...seen].every((c) => allowed.has(c)), `${seen.size} distinct colours`);
|
||||
|
||||
// teeth band: the inner ring clipped to a height. The point of doing it this
|
||||
// way is that the silhouette is the mouth's own shape, so it can never
|
||||
// disagree with the lips - assert that rather than the pixel measurement.
|
||||
// scaleRing is what pulls the sampled region in from MediaPipe's inner lip
|
||||
// landmarks, which sit slightly outside the real opening.
|
||||
{
|
||||
const ring = [{ x: 0, y: 0 }, { x: 10, y: 0 }, { x: 10, y: 10 }, { x: 0, y: 10 }];
|
||||
const half = clipPolyAbove(ring, 5);
|
||||
ok('clip at mid height halves the box',
|
||||
half.length === 4 && Math.max(...half.map((p) => p.y)) === 5,
|
||||
`${half.length} pts, maxY ${Math.max(...half.map((p) => p.y))}`);
|
||||
ok('clip above everything keeps the ring', clipPolyAbove(ring, 99).length === 4);
|
||||
ok('clip below everything empties it', clipPolyAbove(ring, -1).length === 0);
|
||||
const w = clipPolyAbove(ring, 5);
|
||||
ok('clipped band keeps the ring width',
|
||||
Math.min(...w.map((p) => p.x)) === 0 && Math.max(...w.map((p) => p.x)) === 10);
|
||||
// a non-convex ring must not gain or lose x-extent from clipping
|
||||
const tri = [{ x: 0, y: 0 }, { x: 8, y: 2 }, { x: 4, y: 9 }];
|
||||
const ct = clipPolyAbove(tri, 4);
|
||||
ok('clip of a non-convex-ish ring stays within its x range',
|
||||
Math.min(...ct.map((p) => p.x)) >= 0 && Math.max(...ct.map((p) => p.x)) <= 8);
|
||||
const small = scaleRing(ring, 0.5);
|
||||
const w = Math.max(...small.map((p) => p.x)) - Math.min(...small.map((p) => p.x));
|
||||
ok('scaleRing(0.5) halves the extent', Math.abs(w - 5) < 1e-9, `width ${w}`);
|
||||
const same = scaleRing(ring, 1);
|
||||
ok('scaleRing(1) is identity', same.every((p, i) => Math.abs(p.x - ring[i].x) < 1e-9));
|
||||
let cx = 0; for (const p of small) cx += p.x;
|
||||
ok('scaleRing keeps the centroid', Math.abs(cx / 4 - 5) < 1e-9);
|
||||
}
|
||||
|
||||
// Otsu on a uniform region must report near-zero class separation. It will
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue