From 29b0bd6690e1362d934c9e3e2649aa06418bf874 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 24 Sep 2026 15:03:12 -0400 Subject: [PATCH 01/10] Fix horizontal stretch from MediaPipe's anisotropic normalised space MediaPipe normalises x by image WIDTH and y by image HEIGHT, so for a 1080x1920 clip one unit of x is 1080px and one unit of y is 1920px. makeXform applied a single scale to both, stretching everything horizontally by H/W - 1.78x on this footage. The photo underlay looked equally squashed because frameAffine divided x by imgW, matching the equally wrong vector shapes rather than disagreeing with them. Fixed at ingest: landmarks convert to an isotropic space whose unit is one image height (x *= W/H), so equal numbers mean equal pixels everywhere downstream. Pixel mapping follows - both axes divide by imgH. This also silently fixes head roll. fitSimilarity was fitting a rotation in a sheared space, so the "similarity" it recovered was not one, and stabilisation of rolled heads was subtly wrong. selftest: a shape circular in pixel space must stay circular in raster space, checked at 1080x1920, 1920x1080 and 640x640. Fails at ratio 1.78 without the conversion. Co-Authored-By: Claude Opus 5 --- js/app.js | 9 +++++++-- js/pipeline.js | 25 +++++++++++++++++-------- js/selftest.js | 34 ++++++++++++++++++++++++++++++++++ js/underlay.js | 10 ++++++---- 4 files changed, 64 insertions(+), 14 deletions(-) diff --git a/js/app.js b/js/app.js index d4f2528..6e6fc61 100644 --- a/js/app.js +++ b/js/app.js @@ -23,6 +23,7 @@ const state = { keep: new Set(), // frames that get their own plate drawing frame: 0, playing: false, faceBox: null, fps: 12, audio: null, // fps comes from manifest.json, never guessed + aspect: 1, // imgW/imgH; converts MediaPipe's anisotropic space }; const el = (id) => document.getElementById(id); @@ -134,7 +135,7 @@ function rebuild(resetKeep) { const o = opts(); const N = state.dense.length; - state.stab = stabilize(state.dense, o.smoothWin); + state.stab = stabilize(state.dense, o.smoothWin, state.aspect); const ap = state.stab.aperture; const head = Math.max(1, Math.floor(N / 4)); @@ -440,11 +441,13 @@ async function runFrames() { } const { dense, missing } = await detectAll(images); state.images = images; state.dense = dense; + state.aspect = images[0].naturalWidth / images[0].naturalHeight; el('scrub').max = dense.length - 1; state.frame = 0; rebuild(true); const dur = (dense.length / state.fps).toFixed(2); - status(`${images.length} frames · ${state.fps}fps · ${dur}s` + + status(`${images.length} frames · ${images[0].naturalWidth}x${images[0].naturalHeight} · ` + + `${state.fps}fps · ${dur}s` + (state.audio ? ' · audio loaded' : ' · no audio') + (missing.length ? ` · no face on ${missing.length} (held previous)` : ''), missing.length ? 'warn' : 'ok'); @@ -455,6 +458,8 @@ function runSynthetic() { state.images = []; attachAudio(null); state.fps = 12; + state.aspect = 1; // synthetic landmarks are generated square + state.dense = synthDense(72); el('scrub').max = 71; state.frame = 0; diff --git a/js/pipeline.js b/js/pipeline.js index 90ae12f..d75ea99 100644 --- a/js/pipeline.js +++ b/js/pipeline.js @@ -5,13 +5,22 @@ import { RIGID, LIPS_OUTER, LIPS_INNER, APERTURE, FACE_OVAL, EYE_INNER, subsampleSlots } from './landmarks.js'; import { fitSimilarity, applySimAll, applySim, fitResidual, procrustesMean, smoothTransforms, movingAverage } from './mathutil.js'; -const pick = (lm, idx) => idx.map((i) => ({ x: lm[i].x, y: lm[i].y })); +// MediaPipe normalises x by image WIDTH and y by image HEIGHT, so its normalised +// space is anisotropic: for a 1080x1920 frame, one unit of x is 1080px and one +// unit of y is 1920px. Treating those as comparable stretches everything +// horizontally by H/W, and worse, makes fitSimilarity fit a "rotation" in a +// sheared space, so head roll comes out subtly wrong as well. +// +// Multiplying x by aspect = W/H converts to an ISOTROPIC space whose unit is one +// image height, so equal numbers mean equal pixels. Everything downstream - +// Procrustes, the similarity fit, the raster transform - depends on that. +const pick = (lm, idx, aspect) => idx.map((i) => ({ x: lm[i].x * aspect, y: lm[i].y })); // Stage 1-3: fit the rigid transform per frame, smooth its parameters, then map // every contour through it into the reference frame. The result is head-local: // translation, roll and depth-scale of the head are gone. -export function stabilize(dense, smoothRadius) { - const rigid = dense.map((f) => pick(f, RIGID)); +export function stabilize(dense, smoothRadius, aspect = 1) { + const rigid = dense.map((f) => pick(f, RIGID, aspect)); const ref = procrustesMean(rigid); const raw = rigid.map((r) => fitSimilarity(r, ref)); const tfs = smoothTransforms(raw, smoothRadius); @@ -26,12 +35,12 @@ export function stabilize(dense, smoothRadius) { // Residual rises with out-of-plane rotation, which no 2D similarity can // remove. High values mean this section wants a different head plate. residual: tfs.map((tf, i) => fitResidual(tf, rigid[i], ref)), - outer: dense.map((f, i) => applySimAll(tfs[i], pick(f, LIPS_OUTER))), - inner: dense.map((f, i) => applySimAll(tfs[i], pick(f, LIPS_INNER))), - oval: dense.map((f, i) => applySimAll(tfs[i], pick(f, FACE_OVAL))), - eyes: dense.map((f, i) => applySimAll(tfs[i], pick(f, EYE_INNER))), + outer: dense.map((f, i) => applySimAll(tfs[i], pick(f, LIPS_OUTER, aspect))), + inner: dense.map((f, i) => applySimAll(tfs[i], pick(f, LIPS_INNER, aspect))), + oval: dense.map((f, i) => applySimAll(tfs[i], pick(f, FACE_OVAL, aspect))), + eyes: dense.map((f, i) => applySimAll(tfs[i], pick(f, EYE_INNER, aspect))), aperture: dense.map((f, i) => { - const a = applySimAll(tfs[i], pick(f, APERTURE)); + const a = applySimAll(tfs[i], pick(f, APERTURE, aspect)); return Math.hypot(a[0].x - a[1].x, a[0].y - a[1].y); }), }; diff --git a/js/selftest.js b/js/selftest.js index cbf616e..0e41def 100644 --- a/js/selftest.js +++ b/js/selftest.js @@ -113,6 +113,40 @@ export function run() { const apRange = Math.max(...stab.aperture) - Math.min(...stab.aperture); ok('stabilisation preserves mouth motion', apRange > 0.05, `aperture range ${apRange.toFixed(4)}`); + // ASPECT: a shape that is circular in PIXEL space must stay circular in raster + // space. MediaPipe normalises x by width and y by height, so for a portrait + // frame equal normalised numbers are unequal pixel distances; feeding those + // straight through stretches everything horizontally by H/W. This asserts the + // isotropic conversion, and fails at ~1.78 for a 1080x1920 clip without it. + for (const [W, H] of [[1080, 1920], [1920, 1080], [640, 640]]) { + const aspect = W / H; + const N = 24, cx = 0.5, cy = 0.5, rPx = 200; + // a true circle of radius rPx, expressed in MediaPipe normalised coords + const circleFrames = []; + for (let t = 0; t < 4; t++) { + const pts = new Array(478).fill(null).map(() => ({ x: 0.5, y: 0.5, z: 0 })); + RIGID.forEach((id, k) => { + const a = (k / RIGID.length) * Math.PI * 2; + pts[id] = { x: cx + (120 * Math.cos(a)) / W, y: cy + (120 * Math.sin(a)) / H, z: 0 }; + }); + LIPS_OUTER.forEach((id, k) => { + const a = -(k / LIPS_OUTER.length) * Math.PI * 2; + pts[id] = { x: cx + (rPx * Math.cos(a)) / W, y: cy + (rPx * Math.sin(a)) / H, z: 0 }; + }); + FACE_OVAL.forEach((id, k) => { + const a = -(k / FACE_OVAL.length) * Math.PI * 2; + pts[id] = { x: cx + (420 * Math.cos(a)) / W, y: cy + (420 * Math.sin(a)) / H, z: 0 }; + }); + circleFrames.push(pts); + } + const st2 = stabilize(circleFrames, 0, aspect); + const ring = toRasterRing(st2.outer[0], LIPS_OUTER, 16, (p) => p); + const xs = ring.map((p) => p.x), ys = ring.map((p) => p.y); + const ratio = (Math.max(...xs) - Math.min(...xs)) / (Math.max(...ys) - Math.min(...ys)); + ok(`circle stays circular at ${W}x${H}`, Math.abs(ratio - 1) < 0.02, + `w/h ratio ${ratio.toFixed(4)}`); + } + // key selection const xf = (p) => ({ x: p.x * 320, y: p.y * 200 }); const shapes = stab.outer.map((r) => toRasterRing(r, LIPS_OUTER, 8, xf)); diff --git a/js/underlay.js b/js/underlay.js index 81ba004..14f8478 100644 --- a/js/underlay.js +++ b/js/underlay.js @@ -10,11 +10,13 @@ import { applySim } from './mathutil.js'; // Compose pixel-space -> raster-space into one affine. // -// MediaPipe normalises x by width and y by height, so normalised space is a -// stretched pixel space and the composition is a general affine rather than a -// similarity. Three mapped points determine it exactly. +// Landmarks are converted to an isotropic space (unit = one image height) before +// fitting, so pixels map in the same way: BOTH axes divide by imgH, not by their +// own dimension. Dividing x by imgW here instead is what stretched the underlay +// horizontally by H/W and made it disagree with nothing - it matched the equally +// wrong vector shapes. export function frameAffine(tf, xform, imgW, imgH) { - const map = (px, py) => xform(applySim(tf, { x: px / imgW, y: py / imgH })); + const map = (px, py) => xform(applySim(tf, { x: px / imgH, y: py / imgH })); const P0 = map(0, 0), P1 = map(imgW, 0), P2 = map(0, imgH); return { a: (P1.x - P0.x) / imgW, b: (P1.y - P0.y) / imgW, From 941022b69f08eaa3d69fe70b60863bea0d561bc9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 24 Sep 2026 15:12:09 -0400 Subject: [PATCH 02/10] Teeth from mouth-interior image content MediaPipe has no landmarks inside the lips, so teeth have to come from pixels. Tracing the bright blob would give a new contour every frame with no vertex correspondence - the exact boil docs/roto-puppet.md warns about. So the measurement yields a scalar, not a shape: Otsu within the cavity, scanned from the top for where the bright run stops, giving one line height per frame. The teeth polygon is the inner lip ring clipped to that line, so the silhouette is always the mouth's own shape and cannot disagree with the lips around it, and the only per-frame variable is a single number that smooths trivially. Presence gets hysteresis and minimum dwell, as plate selection does: a teeth block blinking on and off for single frames is worse than one simply absent. Tongue is not implemented. The same scalar approach would apply, gated on redness rather than brightness, but it is not visible in the test footage - the cavity reads dark with a bright upper-teeth band and nothing else. Co-Authored-By: Claude Opus 5 --- README.md | 23 ++++++++++ index.html | 5 +++ js/app.js | 84 +++++++++++++++++++++++++++++++++--- js/interior.js | 113 +++++++++++++++++++++++++++++++++++++++++++++++++ js/selftest.js | 22 ++++++++++ 5 files changed, 242 insertions(+), 5 deletions(-) create mode 100644 js/interior.js diff --git a/README.md b/README.md index 85b29bc..8c8f043 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,29 @@ is hand-drawn head plates, which this tool does not yet do. | closed-mouth cut | Aperture below which the mouth interior is emitted as `hidden`. | | suggest tolerance | Max head movement before a new plate drawing is required. Affects **Suggest** only. | +## Teeth + +MediaPipe has no landmarks inside the lips: the inner ring bounds the cavity and +everything within it is just pixels. So teeth come from the image — but tracing +the bright blob would produce a new contour every frame with no vertex +correspondence, which is precisely the boil the design exists to avoid. + +So the extraction yields a **scalar, not a shape**. The teeth polygon is the inner +lip ring clipped to a horizontal line, and only that line's height is measured +(Otsu threshold within the cavity, scanned from the top). The silhouette is +therefore always the mouth's own shape — stable by construction — and the only +per-frame variable is one number, which smooths trivially. It is also how the +shape gets drawn by hand: a band bounded by the lip. + +Presence uses hysteresis plus a minimum dwell, the same treatment plate selection +gets, because a teeth block that blinks on and off for single frames is worse +than one that is simply absent. + +**Tongue** would work the same way — a shape filling the lower cavity, gated on a +redness rather than a brightness statistic. Not implemented, because it is not +visible in the test footage: the cavity reads as dark with a bright upper-teeth +band and nothing else. + ## The plate is reference, not art The plate layer has several representations because its job changes. Cycle with diff --git a/index.html b/index.html index 37475e3..8606bc9 100644 --- a/index.html +++ b/index.html @@ -122,11 +122,16 @@ + +
contour avg 0 = off, 1 = ±1 frame. Removes per-frame landmark jitter. Push past 2 and it starts eating articulation.
anchor avg smooths the head transform only — never the contour.
+ teeth threshold bright fraction of the cavity needed before a teeth + band appears; dwell is how many frames a change must persist, so it + cannot blink on and off.
suggest tolerance only affects the Suggest button: max head movement allowed before a new drawing is required.
diff --git a/js/app.js b/js/app.js index 6e6fc61..88be7ea 100644 --- a/js/app.js +++ b/js/app.js @@ -3,6 +3,7 @@ import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js'; import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js'; import { IndexedRaster } from './raster.js'; import { drawRegistered, posterizeInto } from './underlay.js'; +import { measureInterior, clipPolyAbove } from './interior.js'; import { writeTake } from './take.js'; import { synthDense } from './synth.js'; @@ -13,9 +14,9 @@ const PALETTE = [ { name: 'skin_base', hex: '#b07a5a' }, { name: 'skin_dark', hex: '#7a4f3a' }, { name: 'mouth_dark', hex: '#24161a' }, - { name: 'skin_lite', hex: '#d9a884' }, + { name: 'teeth', hex: '#d9cfc2' }, ]; -const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, lite: 4 }; +const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, teeth: 4 }; const state = { dense: null, images: [], stab: null, xform: null, @@ -24,11 +25,15 @@ const state = { frame: 0, playing: false, faceBox: null, fps: 12, audio: null, // fps comes from manifest.json, never guessed aspect: 1, // imgW/imgH; converts MediaPipe's anisotropic space + interior: null, // per-frame teeth measurement from image content + teeth: null, // resolved per-frame {show, t} after knobs }; const el = (id) => document.getElementById(id); const opts = () => ({ verts: +el('verts').value, + teethOn: +el('teethOn').value / 100, + teethDwell: +el('teethDwell').value, smoothWin: +el('smoothWin').value, contourSmooth: +el('contourSmooth').value, apertureThresh: +el('apertureThresh').value / 1000, @@ -116,6 +121,14 @@ async function detectAll(images) { return { dense, missing }; } +// Interior measurement is a function of pixels alone, so it runs once with +// detection and the knobs re-resolve it instantly afterwards. +function measureAll(images, dense) { + const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true }); + return dense.map((lm, i) => + measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx)); +} + /* ---------- build ---------- */ function makeXform(stab, neutral) { @@ -154,6 +167,8 @@ function rebuild(resetKeep) { const apMax = Math.max(...ap); state.hidden = ap.map((v) => v / apMax < o.apertureThresh); + state.teeth = resolveTeeth(o); + // Plate outline per frame, so a kept frame shows its own head shape. state.plates = state.stab.oval.map((r) => r.map(state.xform)); @@ -180,6 +195,40 @@ function faceBoxes() { }); } +// Presence gets hysteresis and a minimum dwell, the same treatment plate +// selection gets: a teeth block that blinks on and off for single frames is +// worse than one that is simply absent. Appearing needs a clear signal, staying +// needs only a weak one. +function resolveTeeth(o) { + const N = state.dense.length; + if (!state.interior) return new Array(N).fill({ show: false, t: 0 }); + + const raw = state.interior.map((m, f) => (state.hidden[f] ? 0 : m.coverage)); + const on = o.teethOn, off = o.teethOn * 0.6; + const shown = new Array(N).fill(false); + let live = false, since = 0; + for (let f = 0; f < N; f++) { + const want = live ? raw[f] > off : raw[f] > on; + if (want !== live && since >= o.teethDwell) { live = want; since = 0; } + else since++; + shown[f] = live && !state.hidden[f]; + } + + // Smooth the line height only across frames where it is actually shown, so a + // gap does not drag the band toward zero on either side of it. + const t = state.interior.map((m) => m.teethT); + const sm = t.map((_, f) => { + let acc = 0, c = 0; + for (let j = f - o.contourSmooth; j <= f + o.contourSmooth; j++) { + const k = Math.min(N - 1, Math.max(0, j)); + if (!shown[k]) continue; + acc += t[k]; c++; + } + return c ? acc / c : t[f]; + }); + return shown.map((show, f) => ({ show, t: sm[f] })); +} + /* ---------- render ---------- */ // The plate layer has several representations because its job changes: a flat @@ -199,7 +248,18 @@ function renderFrame(f, mode = plateMode()) { } r.fillPoly(state.outer[f], IDX.dark); // mouth keeps every frame - if (!state.hidden[f]) r.fillPoly(state.inner[f], IDX.mouth); + if (!state.hidden[f]) { + r.fillPoly(state.inner[f], IDX.mouth); + const te = state.teeth[f]; + if (te.show) { + // The band is the mouth's own silhouette clipped to a height, so the shape + // can never disagree with the lips around it. + const ys = state.inner[f].map((p) => p.y); + const top = Math.min(...ys), bot = Math.max(...ys); + const poly = clipPolyAbove(state.inner[f], top + te.t * (bot - top)); + if (poly.length >= 3) r.fillPoly(poly, IDX.teeth); + } + } return r; } @@ -259,8 +319,10 @@ function drawAll() { function drawReadout() { const kept = keptSorted(); const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k); + const teethFrames = state.teeth ? state.teeth.filter((t) => t.show).length : 0; el('readout').textContent = `${state.dense.length} frames → ${kept.length} drawings · ` + + `teeth on ${teethFrames}f · ` + `holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` + `neutral f${state.neutral} · residual ` + `${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`; @@ -406,6 +468,14 @@ function exportTake() { keys: state.outer.map((pts, f) => ({ f, src: f, pts })) }, { name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth', keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) }, + { name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in', + keys: state.inner.map((pts, f) => { + const te = state.teeth[f]; + if (state.hidden[f] || !te.show) return { f, hidden: true }; + const ys = pts.map((p) => p.y); + const top = Math.min(...ys), bot = Math.max(...ys); + return { f, src: f, pts: clipPolyAbove(pts, top + te.t * (bot - top)) }; + }) }, ], }; const text = writeTake(take) @@ -442,6 +512,8 @@ async function runFrames() { const { dense, missing } = await detectAll(images); state.images = images; state.dense = dense; state.aspect = images[0].naturalWidth / images[0].naturalHeight; + status('measuring mouth interiors…'); + state.interior = measureAll(images, dense); el('scrub').max = dense.length - 1; state.frame = 0; rebuild(true); @@ -459,6 +531,7 @@ function runSynthetic() { attachAudio(null); state.fps = 12; state.aspect = 1; // synthetic landmarks are generated square + state.interior = null; // no pixels, so no teeth state.dense = synthDense(72); el('scrub').max = 71; @@ -467,10 +540,11 @@ function runSynthetic() { status('synthetic — exercises everything below detection', 'ok'); } -for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol']) { +for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell']) { el(id).addEventListener('input', () => { el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol' - ? (+el(id).value / 1000).toFixed(3) : el(id).value; + ? (+el(id).value / 1000).toFixed(3) + : id === 'teethOn' ? (+el(id).value / 100).toFixed(2) : el(id).value; if (id === 'tol') return; // tol only matters when you ask for a suggestion rebuild(false); }); diff --git a/js/interior.js b/js/interior.js new file mode 100644 index 0000000..056b75c --- /dev/null +++ b/js/interior.js @@ -0,0 +1,113 @@ +// Mouth interior from image content. +// +// MediaPipe has no landmarks inside the lips - the inner ring bounds the cavity +// and everything within it is just pixels. So teeth have to come from the +// picture, and the question is how to do that without reintroducing the boil +// that per-frame detection causes. +// +// The answer is to extract a SCALAR, not a shape. Tracing the bright blob would +// give a new contour every frame with no vertex correspondence - exactly the +// failure docs/roto-puppet.md warns about. Instead the teeth polygon is the +// inner lip ring clipped to a horizontal line, and only that line's height is +// measured. The silhouette is therefore always the mouth's own shape (stable by +// construction) and the only thing that varies per frame is one number, which +// smooths trivially. It is also how the shape is drawn by hand: a band bounded +// by the lip. + +// Otsu's threshold over a luminance histogram. Self-tuning, so exposure changes +// between frames do not shift what counts as "bright". +function otsu(hist, total) { + let sum = 0; + for (let i = 0; i < 256; i++) sum += i * hist[i]; + let sumB = 0, wB = 0, best = 0, bestVar = -1; + for (let t = 0; t < 256; t++) { + wB += hist[t]; + if (!wB) continue; + const wF = total - wB; + if (!wF) break; + sumB += t * hist[t]; + const mB = sumB / wB, mF = (sum - sumB) / wF; + const between = wB * wF * (mB - mF) * (mB - mF); + if (between > bestVar) { bestVar = between; best = t; } + } + return best; +} + +const pointInPoly = (pts, x, y) => { + let inside = false; + for (let i = 0, j = pts.length - 1; i < pts.length; j = i++) { + if ((pts[i].y > y) !== (pts[j].y > y) && + x < ((pts[j].x - pts[i].x) * (y - pts[i].y)) / (pts[j].y - pts[i].y) + pts[i].x) inside = !inside; + } + return inside; +}; + +// Measure one frame: how far down the cavity the bright region reaches, as a +// fraction of cavity height, plus how much of the cavity is bright at all. +export function measureInterior(img, innerNorm, ctx) { + let x0 = 1, y0 = 1, x1 = 0, y1 = 0; + for (const p of innerNorm) { + x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y); + x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y); + } + const W = img.naturalWidth, H = img.naturalHeight; + const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H)); + const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H)); + if (pw < 3 || ph < 3) return { teethT: 0, coverage: 0 }; + + ctx.canvas.width = pw; ctx.canvas.height = ph; + ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph); + const data = ctx.getImageData(0, 0, pw, ph).data; + + const poly = innerNorm.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 })); + const hist = new Uint32Array(256); + const lum = new Float32Array(pw * ph); + const mask = new Uint8Array(pw * ph); + let n = 0; + for (let y = 0; y < ph; y++) { + for (let x = 0; x < pw; x++) { + if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue; + const o = (y * pw + x) * 4; + const l = (0.299 * data[o] + 0.587 * data[o + 1] + 0.114 * data[o + 2]) | 0; + const i = y * pw + x; + lum[i] = l; mask[i] = 1; hist[l]++; n++; + } + } + if (n < 12) return { teethT: 0, coverage: 0 }; + + const thr = otsu(hist, n); + // Rows are scanned from the top of the cavity; the teeth line is where the + // bright run stops. Requiring a run rather than a single row rejects specular + // speckle on a wet lower lip. + let bright = 0, lastBrightRow = -1, run = 0; + for (let y = 0; y < ph; y++) { + let rowIn = 0, rowBright = 0; + for (let x = 0; x < pw; x++) { + const i = y * pw + x; + if (!mask[i]) continue; + rowIn++; + if (lum[i] > thr) { rowBright++; bright++; } + } + if (rowIn >= 2 && rowBright / rowIn > 0.45) { run++; if (run >= 1) lastBrightRow = y; } + else run = 0; + } + return { + teethT: lastBrightRow < 0 ? 0 : (lastBrightRow + 1) / ph, + coverage: bright / n, + }; +} + +// Sutherland-Hodgman against the half-plane y <= limit. +export function clipPolyAbove(pts, limit) { + const out = []; + for (let i = 0; i < pts.length; i++) { + const a = pts[i], b = pts[(i + 1) % pts.length]; + const ain = a.y <= limit, bin = b.y <= limit; + if (ain) out.push(a); + if (ain !== bin) { + const t = (limit - a.y) / (b.y - a.y); + out.push({ x: a.x + t * (b.x - a.x), y: limit }); + } + } + return out; +} diff --git a/js/selftest.js b/js/selftest.js index 0e41def..9b6cd95 100644 --- a/js/selftest.js +++ b/js/selftest.js @@ -12,6 +12,7 @@ import { fitSimilarity, applySim, procrustesMean, smoothTransforms } from './mat import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js'; import { IndexedRaster, hexToRgb } from './raster.js'; import { writeTake } from './take.js'; +import { clipPolyAbove } from './interior.js'; import { synthDense } from './synth.js'; const results = []; @@ -181,6 +182,27 @@ export function run() { ok('palette expansion introduces no intermediate colours', [...seen].every((c) => allowed.has(c)), `${seen.size} distinct colours`); + // teeth band: the inner ring clipped to a height. The point of doing it this + // way is that the silhouette is the mouth's own shape, so it can never + // disagree with the lips - assert that rather than the pixel measurement. + { + const ring = [{ x: 0, y: 0 }, { x: 10, y: 0 }, { x: 10, y: 10 }, { x: 0, y: 10 }]; + const half = clipPolyAbove(ring, 5); + ok('clip at mid height halves the box', + half.length === 4 && Math.max(...half.map((p) => p.y)) === 5, + `${half.length} pts, maxY ${Math.max(...half.map((p) => p.y))}`); + ok('clip above everything keeps the ring', clipPolyAbove(ring, 99).length === 4); + ok('clip below everything empties it', clipPolyAbove(ring, -1).length === 0); + const w = clipPolyAbove(ring, 5); + ok('clipped band keeps the ring width', + Math.min(...w.map((p) => p.x)) === 0 && Math.max(...w.map((p) => p.x)) === 10); + // a non-convex ring must not gain or lose x-extent from clipping + const tri = [{ x: 0, y: 0 }, { x: 8, y: 2 }, { x: 4, y: 9 }]; + const ct = clipPolyAbove(tri, 4); + ok('clip of a non-convex-ish ring stays within its x range', + Math.min(...ct.map((p) => p.x)) >= 0 && Math.max(...ct.map((p) => p.x)) <= 8); + } + // take writer round-trip const take = { name: 'test', frames: 72, width: 320, height: 200, exposure: 2, From c6daf827d19b79d803149b80b454fbf73937ab38 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 24 Sep 2026 15:14:49 -0400 Subject: [PATCH 03/10] Fix teeth band filling the whole mouth Three causes, all of them mine: Otsu always returns a split, including on a homogeneous region - given a dark cavity with no teeth it invents a threshold and calls half the pixels bright. The gate is now the separation between the two class means, which is the only thing that says whether the split means anything. Coverage was the wrong signal: it is high both when the mouth is full of teeth and when the region is uniformly dark and Otsu has split noise. The row scan tracked the last qualifying row anywhere rather than where the run from the top stops, so one bright row near the bottom - a lit lower lip inside the ring - pushed the line to full height. It now breaks at the first failing row once the run has started. MediaPipe's inner lip landmarks sit slightly outside the real opening, so the sampled region included lip pixels, which are bright and sit exactly at the boundary where they do most damage. The ring is now eroded toward its centroid before sampling, with the amount exposed as a knob. Adds a diagnostic panel showing the sampled crop, pixels above threshold, and the resolved line, because tuning this from numbers alone does not tell you whether the region being measured is even the right region. Co-Authored-By: Claude Opus 5 --- index.html | 19 +++++++--- js/app.js | 51 +++++++++++++++++++++---- js/interior.js | 101 +++++++++++++++++++++++++++++++++++++++---------- js/selftest.js | 26 ++++++++++++- 4 files changed, 164 insertions(+), 33 deletions(-) diff --git a/index.html b/index.html index 8606bc9..5aa1f76 100644 --- a/index.html +++ b/index.html @@ -122,21 +122,30 @@ - + +
contour avg 0 = off, 1 = ±1 frame. Removes per-frame landmark jitter. Push past 2 and it starts eating articulation.
anchor avg smooths the head transform only — never the contour.
- teeth threshold bright fraction of the cavity needed before a teeth - band appears; dwell is how many frames a change must persist, so it - cannot blink on and off.
+ teeth contrast how far apart the dark and bright halves of the + cavity must be before the split means anything — Otsu always returns + some threshold, so this is what stops it inventing teeth in a dark + mouth. erode shrinks the sampled region inward, away from the lip + edge. dwell is how many frames a change must persist.
suggest tolerance only affects the Suggest button: max head movement allowed before a new drawing is required.
-
+
+

teeth measurement

+
+
+
green = above threshold · amber = resolved line
+
+

palette

diff --git a/js/app.js b/js/app.js index 88be7ea..b569ccf 100644 --- a/js/app.js +++ b/js/app.js @@ -32,8 +32,9 @@ const state = { const el = (id) => document.getElementById(id); const opts = () => ({ verts: +el('verts').value, - teethOn: +el('teethOn').value / 100, + teethOn: +el('teethOn').value / 100, // minimum Otsu class separation teethDwell: +el('teethDwell').value, + teethErode: +el('teethErode').value / 100, smoothWin: +el('smoothWin').value, contourSmooth: +el('contourSmooth').value, apertureThresh: +el('apertureThresh').value / 1000, @@ -123,10 +124,11 @@ async function detectAll(images) { // Interior measurement is a function of pixels alone, so it runs once with // detection and the knobs re-resolve it instantly afterwards. -function measureAll(images, dense) { +function measureAll(images, dense, o) { const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true }); return dense.map((lm, i) => - measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx)); + measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx, + { minContrast: 0, erode: o.teethErode })); // gate applied later, in resolveTeeth } /* ---------- build ---------- */ @@ -167,6 +169,10 @@ function rebuild(resetKeep) { const apMax = Math.max(...ap); state.hidden = ap.map((v) => v / apMax < o.apertureThresh); + if (state.images.length && (!state.interior || state.erodeUsed !== o.teethErode)) { + state.interior = measureAll(state.images, state.dense, o); + state.erodeUsed = o.teethErode; + } state.teeth = resolveTeeth(o); // Plate outline per frame, so a kept frame shows its own head shape. @@ -203,8 +209,12 @@ function resolveTeeth(o) { const N = state.dense.length; if (!state.interior) return new Array(N).fill({ show: false, t: 0 }); - const raw = state.interior.map((m, f) => (state.hidden[f] ? 0 : m.coverage)); - const on = o.teethOn, off = o.teethOn * 0.6; + // Gate on the separation between Otsu's two class means, not on how much of + // the cavity is bright: coverage is high both when the mouth is full of teeth + // and when the region is uniformly dark and Otsu has split noise. + const raw = state.interior.map((m, f) => + (state.hidden[f] || m.teethT <= 0 ? 0 : m.contrast)); + const on = o.teethOn, off = o.teethOn * 0.7; const shown = new Array(N).fill(false); let live = false, since = 0; for (let f = 0; f < N; f++) { @@ -369,6 +379,30 @@ function drawPanes() { if (!state.hidden[f]) strokePts(g2, z(state.inner[f]), '#f87171'); compositeRender(el('cv-render'), f, ZOOM); + drawInteriorDebug(f); +} + +// What the teeth measurement actually saw: sampled region, pixels above +// threshold in green, the resolved line in amber. Recomputed for the current +// frame only, so it costs nothing to keep on screen. +function drawInteriorDebug(f) { + const host = el('cv-teeth'); + const img = state.images[f]; + if (!img || state.hidden[f]) { host.innerHTML = ''; el('teethinfo').textContent = state.images.length ? 'mouth closed' : 'no source frames'; return; } + const o = opts(); + const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true }); + const m = measureInterior(img, LIPS_INNER.map((k) => state.dense[f][k]), ctx, + { minContrast: 0, erode: o.teethErode }, true); + host.innerHTML = ''; + if (m.debug) { + m.debug.style.width = '150px'; + m.debug.style.imageRendering = 'pixelated'; + host.append(m.debug); + } + const te = state.teeth[f]; + el('teethinfo').textContent = + `contrast ${m.contrast.toFixed(3)} (gate ${o.teethOn.toFixed(2)}) · ` + + `line ${m.teethT.toFixed(2)} · ${te.show ? 'SHOWN' : 'hidden'}`; } function strokePts(g, pts, color, lw = 1) { @@ -513,7 +547,7 @@ async function runFrames() { state.images = images; state.dense = dense; state.aspect = images[0].naturalWidth / images[0].naturalHeight; status('measuring mouth interiors…'); - state.interior = measureAll(images, dense); + state.interior = measureAll(images, dense, opts()); el('scrub').max = dense.length - 1; state.frame = 0; rebuild(true); @@ -540,11 +574,12 @@ function runSynthetic() { status('synthetic — exercises everything below detection', 'ok'); } -for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell']) { +for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell', 'teethErode']) { el(id).addEventListener('input', () => { el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol' ? (+el(id).value / 1000).toFixed(3) - : id === 'teethOn' ? (+el(id).value / 100).toFixed(2) : el(id).value; + : id === 'teethOn' || id === 'teethErode' ? (+el(id).value / 100).toFixed(2) + : el(id).value; if (id === 'tol') return; // tol only matters when you ask for a suggestion rebuild(false); }); diff --git a/js/interior.js b/js/interior.js index 056b75c..46580e6 100644 --- a/js/interior.js +++ b/js/interior.js @@ -16,21 +16,23 @@ // Otsu's threshold over a luminance histogram. Self-tuning, so exposure changes // between frames do not shift what counts as "bright". +export function otsuForTest(h, t) { return otsu(h, t); } + function otsu(hist, total) { let sum = 0; for (let i = 0; i < 256; i++) sum += i * hist[i]; - let sumB = 0, wB = 0, best = 0, bestVar = -1; + let sumB = 0, wB = 0, best = 0, bestVar = -1, bestMB = 0, bestMF = 0; for (let t = 0; t < 256; t++) { wB += hist[t]; if (!wB) continue; const wF = total - wB; if (!wF) break; sumB += t * hist[t]; - const mB = sumB / wB, mF = (sum - sumB) / wF; + const mB = sumB / wB, mF = (sum - sumB) / wF; // mB = dark class, mF = bright const between = wB * wF * (mB - mF) * (mB - mF); - if (between > bestVar) { bestVar = between; best = t; } + if (between > bestVar) { bestVar = between; best = t; bestMB = mB; bestMF = mF; } } - return best; + return { thr: best, mDark: bestMB, mBright: bestMF }; } const pointInPoly = (pts, x, y) => { @@ -42,24 +44,46 @@ const pointInPoly = (pts, x, y) => { return inside; }; +// Shrink a ring toward its centroid. +// +// MediaPipe's inner lip landmarks sit slightly OUTSIDE the actual opening, so +// sampling the ring as given includes lip pixels - which are bright, and sit +// right at the cavity boundary where they do the most damage. +function erode(pts, k) { + let cx = 0, cy = 0; + for (const p of pts) { cx += p.x; cy += p.y; } + cx /= pts.length; cy /= pts.length; + return pts.map((p) => ({ x: cx + (p.x - cx) * (1 - k), y: cy + (p.y - cy) * (1 - k) })); +} + // Measure one frame: how far down the cavity the bright region reaches, as a -// fraction of cavity height, plus how much of the cavity is bright at all. -export function measureInterior(img, innerNorm, ctx) { +// fraction of cavity height, plus the contrast that justified calling it bright. +// +// `wantDebug` returns the sampled crop with the classification drawn on it. +// Tuning this blind is miserable; the numbers alone do not say whether the +// region being measured is even the right region. +export function measureInterior(img, innerNorm, ctx, opts = {}, wantDebug = false) { + const minContrast = opts.minContrast ?? 0.14; + const rowFrac = opts.rowFrac ?? 0.4; + const inner = erode(innerNorm, opts.erode ?? 0.18); + + const none = { teethT: 0, contrast: 0, coverage: 0, debug: null }; let x0 = 1, y0 = 1, x1 = 0, y1 = 0; - for (const p of innerNorm) { + for (const p of inner) { x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y); x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y); } const W = img.naturalWidth, H = img.naturalHeight; const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H)); const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H)); - if (pw < 3 || ph < 3) return { teethT: 0, coverage: 0 }; + if (pw < 4 || ph < 4) return none; ctx.canvas.width = pw; ctx.canvas.height = ph; ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph); - const data = ctx.getImageData(0, 0, pw, ph).data; + const img0 = ctx.getImageData(0, 0, pw, ph); + const data = img0.data; - const poly = innerNorm.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 })); + const poly = inner.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 })); const hist = new Uint32Array(256); const lum = new Float32Array(pw * ph); const mask = new Uint8Array(pw * ph); @@ -73,13 +97,24 @@ export function measureInterior(img, innerNorm, ctx) { lum[i] = l; mask[i] = 1; hist[l]++; n++; } } - if (n < 12) return { teethT: 0, coverage: 0 }; + if (n < 16) return none; - const thr = otsu(hist, n); - // Rows are scanned from the top of the cavity; the teeth line is where the - // bright run stops. Requiring a run rather than a single row rejects specular - // speckle on a wet lower lip. - let bright = 0, lastBrightRow = -1, run = 0; + const { thr, mDark, mBright } = otsu(hist, n); + + // Otsu ALWAYS returns a split, including on a homogeneous region: given a dark + // cavity with no teeth it invents a threshold and calls half the pixels + // bright. The separation between the two class means is what says whether the + // split means anything, so it is the actual gate. + const contrast = (mBright - mDark) / 255; + if (contrast < minContrast) { + return { teethT: 0, contrast, coverage: 0, debug: wantDebug ? debugCanvas(img0, mask, lum, thr, pw, ph, -1) : null }; + } + + // Scan from the top and STOP at the first row that fails. Teeth hang from the + // upper lip, so what matters is the contiguous run, not whether some row near + // the bottom happens to qualify - tracking the latter is what made the band + // fill the whole mouth. + let lastRow = -1, started = false, bright = 0; for (let y = 0; y < ph; y++) { let rowIn = 0, rowBright = 0; for (let x = 0; x < pw; x++) { @@ -88,15 +123,43 @@ export function measureInterior(img, innerNorm, ctx) { rowIn++; if (lum[i] > thr) { rowBright++; bright++; } } - if (rowIn >= 2 && rowBright / rowIn > 0.45) { run++; if (run >= 1) lastBrightRow = y; } - else run = 0; + if (rowIn < 2) continue; + const ok = rowBright / rowIn > rowFrac; + if (ok) { started = true; lastRow = y; } + else if (started) break; } + return { - teethT: lastBrightRow < 0 ? 0 : (lastBrightRow + 1) / ph, + teethT: lastRow < 0 ? 0 : (lastRow + 1) / ph, + contrast, coverage: bright / n, + debug: wantDebug ? debugCanvas(img0, mask, lum, thr, pw, ph, lastRow) : null, }; } +// The sampled crop with the classification painted on: sampled region tinted, +// pixels above threshold in green, the resolved teeth line in amber. +function debugCanvas(img0, mask, lum, thr, pw, ph, lastRow) { + const c = document.createElement('canvas'); + c.width = pw; c.height = ph; + const g = c.getContext('2d'); + const out = new ImageData(pw, ph); + for (let i = 0; i < pw * ph; i++) { + const o = i * 4; + const [r, gr, b] = [img0.data[o], img0.data[o + 1], img0.data[o + 2]]; + if (!mask[i]) { out.data[o] = r * 0.3; out.data[o + 1] = gr * 0.3; out.data[o + 2] = b * 0.3; } + else if (lum[i] > thr) { out.data[o] = 60; out.data[o + 1] = 230; out.data[o + 2] = 120; } + else { out.data[o] = r; out.data[o + 1] = gr; out.data[o + 2] = b; } + out.data[o + 3] = 255; + } + g.putImageData(out, 0, 0); + if (lastRow >= 0) { + g.fillStyle = '#fbbf24'; + g.fillRect(0, lastRow, pw, 1); + } + return c; +} + // Sutherland-Hodgman against the half-plane y <= limit. export function clipPolyAbove(pts, limit) { const out = []; diff --git a/js/selftest.js b/js/selftest.js index 9b6cd95..ddbeec9 100644 --- a/js/selftest.js +++ b/js/selftest.js @@ -12,7 +12,7 @@ import { fitSimilarity, applySim, procrustesMean, smoothTransforms } from './mat import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js'; import { IndexedRaster, hexToRgb } from './raster.js'; import { writeTake } from './take.js'; -import { clipPolyAbove } from './interior.js'; +import { clipPolyAbove, otsuForTest } from './interior.js'; import { synthDense } from './synth.js'; const results = []; @@ -203,6 +203,30 @@ export function run() { Math.min(...ct.map((p) => p.x)) >= 0 && Math.max(...ct.map((p) => p.x)) <= 8); } + // Otsu on a uniform region must report near-zero class separation. It will + // still return a threshold - that is what Otsu does - so the separation is the + // only thing that distinguishes "found teeth" from "split noise in a dark + // mouth", which is what made the band fill the whole cavity. + { + const flat = new Uint32Array(256); flat[40] = 500; + const f = otsuForTest(flat, 500); + ok('uniform region yields ~no class separation', + Math.abs(f.mBright - f.mDark) / 255 < 0.02, `sep ${((f.mBright - f.mDark) / 255).toFixed(4)}`); + + const noisy = new Uint32Array(256); + for (let i = 30; i <= 60; i++) noisy[i] = 20; // dark cavity, some spread + const nz = otsuForTest(noisy, 31 * 20); + ok('dark-but-noisy region stays below a sane gate', + (nz.mBright - nz.mDark) / 255 < 0.14, `sep ${((nz.mBright - nz.mDark) / 255).toFixed(4)}`); + + const teeth = new Uint32Array(256); + for (let i = 20; i <= 45; i++) teeth[i] = 40; // cavity + for (let i = 180; i <= 220; i++) teeth[i] = 30; // teeth + const tt = otsuForTest(teeth, 26 * 40 + 41 * 30); + ok('real bright/dark split clears the gate', + (tt.mBright - tt.mDark) / 255 > 0.4, `sep ${((tt.mBright - tt.mDark) / 255).toFixed(4)}`); + } + // take writer round-trip const take = { name: 'test', frames: 72, width: 320, height: 200, exposure: 2, From 9a11eeabc1cdf053dcbe52e5dc8842c5f24ea08a Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 24 Sep 2026 15:24:52 -0400 Subject: [PATCH 04/10] Teeth as an extracted blob contour, not a clipped band The band filled the mouth because a band is the wrong reduction: the bright region is a blob, and reading it as "everything above a line" throws the shape away. Extracting a contour reintroduces the vertex-correspondence problem that made me avoid it, but for a blob there is a way out. Radial sampling from the centroid along N fixed directions makes vertex k always mean "the extent in direction k": correspondence holds by construction, the count is fixed, and temporal smoothing cannot reorder anything. It also yields a star-shaped reduction, which suits flat colour. Tongue rejection, which the band had no way to express: - pixels red relative to their own brightness are dropped (teeth are neutral) - component choice is biased toward the top of the cavity, since area alone picks the tongue when the mouth is wide - separate inner and outer controls: cavity erode pulls the sampled region off the lip edge, blob grow/erode resizes the found blob Also: a knob wired in app.js but missing from index.html threw during wiring and left a blank page with nothing useful in the console - which is exactly what happened to teethDwell in the previous commit. el() now names the missing id, and window.onerror surfaces it in the status line. Co-Authored-By: Claude Opus 5 --- README.md | 40 ++++---- gl.html | 6 -- index.html | 23 +++-- js/app.js | 116 +++++++++++++--------- js/interior.js | 255 +++++++++++++++++++++++++++++-------------------- js/selftest.js | 28 ++---- probe.html | 79 --------------- 7 files changed, 272 insertions(+), 275 deletions(-) delete mode 100644 gl.html delete mode 100644 probe.html diff --git a/README.md b/README.md index 8c8f043..af8dca5 100644 --- a/README.md +++ b/README.md @@ -65,25 +65,33 @@ is hand-drawn head plates, which this tool does not yet do. ## Teeth MediaPipe has no landmarks inside the lips: the inner ring bounds the cavity and -everything within it is just pixels. So teeth come from the image — but tracing -the bright blob would produce a new contour every frame with no vertex -correspondence, which is precisely the boil the design exists to avoid. +everything within it is just pixels. So teeth come from the image. -So the extraction yields a **scalar, not a shape**. The teeth polygon is the inner -lip ring clipped to a horizontal line, and only that line's height is measured -(Otsu threshold within the cavity, scanned from the top). The silhouette is -therefore always the mouth's own shape — stable by construction — and the only -per-frame variable is one number, which smooths trivially. It is also how the -shape gets drawn by hand: a band bounded by the lip. +The hazard is vertex correspondence — a traced contour reorders between frames +and boils. The way out for a blob specifically is **radial sampling**: march +outward from the blob's centroid along N fixed directions and take the last pixel +inside. Vertex *k* is then always "the extent in direction *k*", so correspondence +holds by construction, the vertex count is fixed, and temporal smoothing is well +defined with no reordering possible. It also produces a star-shaped reduction, +which is what flat blocks of colour want. -Presence uses hysteresis plus a minimum dwell, the same treatment plate selection -gets, because a teeth block that blinks on and off for single frames is worse -than one that is simply absent. +The **teeth measurement** panel shows exactly what is sampled: region dimmed, +kept pixels green, extracted contour amber. Tune against that, not the numbers. -**Tongue** would work the same way — a shape filling the lower cavity, gated on a -redness rather than a brightness statistic. Not implemented, because it is not -visible in the test footage: the cavity reads as dark with a bright upper-teeth -band and nothing else. +| Knob | What it does | +| --- | --- | +| teeth contrast | Gate on the separation between the cavity's dark and bright class means. Otsu always returns *some* threshold, so this is what stops it inventing teeth in a dark mouth. | +| cavity erode | Pulls the sampled region in from the lip edge — MediaPipe's inner lip landmarks sit slightly outside the real opening, and lips are bright. | +| blob grow/erode | Resizes the found blob. An open pass always runs first to despeckle. | +| tongue reject | Drops pixels red relative to their own brightness. Teeth are near-neutral; tongue is not. | +| prefer upper | Biases component choice toward the top of the cavity. Area alone picks the tongue when the mouth is wide. | +| teeth vertices | Radial sample count. | +| teeth avg ±f | Temporal average over the contour. | +| teeth dwell | Frames a presence change must persist before it takes effect. | + +**Tongue** as its own part would work the same way, gated on redness instead of +brightness and biased low rather than high. Not implemented: it is not visible in +the test footage, which reads as a dark cavity with a bright upper-teeth band. ## The plate is reference, not art diff --git a/gl.html b/gl.html deleted file mode 100644 index b2b042c..0000000 --- a/gl.html +++ /dev/null @@ -1,6 +0,0 @@ -
?
diff --git a/index.html b/index.html index 5aa1f76..b6c44e3 100644 --- a/index.html +++ b/index.html @@ -123,18 +123,27 @@ - + + + + + +
contour avg 0 = off, 1 = ±1 frame. Removes per-frame landmark jitter. Push past 2 and it starts eating articulation.
anchor avg smooths the head transform only — never the contour.
- teeth contrast how far apart the dark and bright halves of the - cavity must be before the split means anything — Otsu always returns - some threshold, so this is what stops it inventing teeth in a dark - mouth. erode shrinks the sampled region inward, away from the lip - edge. dwell is how many frames a change must persist.
+ teeth contrast gates on how far apart the cavity's dark and bright + halves are — Otsu always returns some threshold, so this is what + stops it inventing teeth in a dark mouth. + cavity erode pulls the sampled region in from the lip edge; + blob grow/erode resizes the found blob itself. + tongue reject drops pixels that are red relative to their own + brightness; prefer upper biases component choice toward the top of + the cavity, where teeth are and the tongue is not. + dwell is how many frames a presence change must persist.
suggest tolerance only affects the Suggest button: max head movement allowed before a new drawing is required.
@@ -143,7 +152,7 @@

teeth measurement

-
green = above threshold · amber = resolved line
+
green = kept pixels · amber = extracted contour

palette

diff --git a/js/app.js b/js/app.js index b569ccf..e15e444 100644 --- a/js/app.js +++ b/js/app.js @@ -3,7 +3,8 @@ import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js'; import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js'; import { IndexedRaster } from './raster.js'; import { drawRegistered, posterizeInto } from './underlay.js'; -import { measureInterior, clipPolyAbove } from './interior.js'; +import { extractTeeth } from './interior.js'; +import { applySim } from './mathutil.js'; import { writeTake } from './take.js'; import { synthDense } from './synth.js'; @@ -29,12 +30,23 @@ const state = { teeth: null, // resolved per-frame {show, t} after knobs }; -const el = (id) => document.getElementById(id); +const el = (id) => { + const n = document.getElementById(id); + // A knob present in the code but missing from the markup used to throw during + // wiring and leave a blank page with nothing in the console worth reading. + if (!n) throw new Error(`missing element #${id} — knob wired in app.js but not in index.html`); + return n; +}; const opts = () => ({ verts: +el('verts').value, teethOn: +el('teethOn').value / 100, // minimum Otsu class separation teethDwell: +el('teethDwell').value, - teethErode: +el('teethErode').value / 100, + teethSmooth: +el('teethSmooth').value, + cavityErode: +el('teethErode').value / 100, + tongueReject: +el('tongueReject').value / 100, + blobGrow: +el('blobGrow').value, + topBias: +el('topBias').value / 100, + verts: +el('teethVerts').value, smoothWin: +el('smoothWin').value, contourSmooth: +el('contourSmooth').value, apertureThresh: +el('apertureThresh').value / 1000, @@ -127,10 +139,14 @@ async function detectAll(images) { function measureAll(images, dense, o) { const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true }); return dense.map((lm, i) => - measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx, - { minContrast: 0, erode: o.teethErode })); // gate applied later, in resolveTeeth + extractTeeth(images[i], LIPS_INNER.map((k) => lm[k]), ctx, o)); } +// Extraction keys on every knob that changes the pixels examined, so the cache +// is keyed on exactly those and a change to anything else stays instant. +const extractKey = (o) => + [o.cavityErode, o.tongueReject, o.blobGrow, o.topBias, o.verts].join('|'); + /* ---------- build ---------- */ function makeXform(stab, neutral) { @@ -169,9 +185,9 @@ function rebuild(resetKeep) { const apMax = Math.max(...ap); state.hidden = ap.map((v) => v / apMax < o.apertureThresh); - if (state.images.length && (!state.interior || state.erodeUsed !== o.teethErode)) { + if (state.images.length && state.extractKey !== extractKey(o)) { state.interior = measureAll(state.images, state.dense, o); - state.erodeUsed = o.teethErode; + state.extractKey = extractKey(o); } state.teeth = resolveTeeth(o); @@ -207,13 +223,10 @@ function faceBoxes() { // needs only a weak one. function resolveTeeth(o) { const N = state.dense.length; - if (!state.interior) return new Array(N).fill({ show: false, t: 0 }); + if (!state.interior) return new Array(N).fill({ show: false, pts: null }); - // Gate on the separation between Otsu's two class means, not on how much of - // the cavity is bright: coverage is high both when the mouth is full of teeth - // and when the region is uniformly dark and Otsu has split noise. const raw = state.interior.map((m, f) => - (state.hidden[f] || m.teethT <= 0 ? 0 : m.contrast)); + (state.hidden[f] || !m.contour ? 0 : m.contrast)); const on = o.teethOn, off = o.teethOn * 0.7; const shown = new Array(N).fill(false); let live = false, since = 0; @@ -221,22 +234,34 @@ function resolveTeeth(o) { const want = live ? raw[f] > off : raw[f] > on; if (want !== live && since >= o.teethDwell) { live = want; since = 0; } else since++; - shown[f] = live && !state.hidden[f]; + shown[f] = live && !state.hidden[f] && !!state.interior[f].contour; } - // Smooth the line height only across frames where it is actually shown, so a - // gap does not drag the band toward zero on either side of it. - const t = state.interior.map((m) => m.teethT); - const sm = t.map((_, f) => { - let acc = 0, c = 0; - for (let j = f - o.contourSmooth; j <= f + o.contourSmooth; j++) { + // Into raster space through the same chain the lips take, including the + // isotropic aspect conversion - a contour in MediaPipe's normalised space is + // in the same stretched coordinates the landmarks are. + const toRaster = (pts, f) => { + const tf = state.stab.transforms[f]; + return pts.map((p) => state.xform(applySim(tf, { x: p.x * state.aspect, y: p.y }))); + }; + const rast = state.interior.map((m, f) => (m.contour ? toRaster(m.contour, f) : null)); + + // Radial sampling makes vertex k mean the same direction on every frame, so + // smoothing across time is well defined and cannot reorder anything. + const sm = rast.map((pts, f) => { + if (!pts || !shown[f]) return pts; + const acc = pts.map(() => ({ x: 0, y: 0 })); + let c = 0; + for (let j = f - o.teethSmooth; j <= f + o.teethSmooth; j++) { const k = Math.min(N - 1, Math.max(0, j)); - if (!shown[k]) continue; - acc += t[k]; c++; + if (!shown[k] || !rast[k] || rast[k].length !== pts.length) continue; + for (let v = 0; v < pts.length; v++) { acc[v].x += rast[k][v].x; acc[v].y += rast[k][v].y; } + c++; } - return c ? acc / c : t[f]; + return c ? acc.map((p) => ({ x: p.x / c, y: p.y / c })) : pts; }); - return shown.map((show, f) => ({ show, t: sm[f] })); + + return shown.map((show, f) => ({ show, pts: sm[f] })); } /* ---------- render ---------- */ @@ -261,14 +286,7 @@ function renderFrame(f, mode = plateMode()) { if (!state.hidden[f]) { r.fillPoly(state.inner[f], IDX.mouth); const te = state.teeth[f]; - if (te.show) { - // The band is the mouth's own silhouette clipped to a height, so the shape - // can never disagree with the lips around it. - const ys = state.inner[f].map((p) => p.y); - const top = Math.min(...ys), bot = Math.max(...ys); - const poly = clipPolyAbove(state.inner[f], top + te.t * (bot - top)); - if (poly.length >= 3) r.fillPoly(poly, IDX.teeth); - } + if (te.show && te.pts && te.pts.length >= 3) r.fillPoly(te.pts, IDX.teeth); } return r; } @@ -388,21 +406,24 @@ function drawPanes() { function drawInteriorDebug(f) { const host = el('cv-teeth'); const img = state.images[f]; - if (!img || state.hidden[f]) { host.innerHTML = ''; el('teethinfo').textContent = state.images.length ? 'mouth closed' : 'no source frames'; return; } + if (!img || state.hidden[f]) { + host.innerHTML = ''; + el('teethinfo').textContent = state.images.length ? 'mouth closed' : 'no source frames'; + return; + } const o = opts(); const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true }); - const m = measureInterior(img, LIPS_INNER.map((k) => state.dense[f][k]), ctx, - { minContrast: 0, erode: o.teethErode }, true); + const m = extractTeeth(img, LIPS_INNER.map((k) => state.dense[f][k]), ctx, o, true); host.innerHTML = ''; if (m.debug) { - m.debug.style.width = '150px'; + m.debug.style.width = '170px'; m.debug.style.imageRendering = 'pixelated'; host.append(m.debug); } const te = state.teeth[f]; el('teethinfo').textContent = - `contrast ${m.contrast.toFixed(3)} (gate ${o.teethOn.toFixed(2)}) · ` + - `line ${m.teethT.toFixed(2)} · ${te.show ? 'SHOWN' : 'hidden'}`; + `contrast ${m.contrast.toFixed(3)} / gate ${o.teethOn.toFixed(2)} · ` + + `area ${m.area}px · ${te.show ? 'SHOWN' : 'hidden'}`; } function strokePts(g, pts, color, lw = 1) { @@ -503,13 +524,8 @@ function exportTake() { { name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth', keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) }, { name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in', - keys: state.inner.map((pts, f) => { - const te = state.teeth[f]; - if (state.hidden[f] || !te.show) return { f, hidden: true }; - const ys = pts.map((p) => p.y); - const top = Math.min(...ys), bot = Math.max(...ys); - return { f, src: f, pts: clipPolyAbove(pts, top + te.t * (bot - top)) }; - }) }, + keys: state.teeth.map((te, f) => + (te.show && te.pts ? { f, src: f, pts: te.pts } : { f, hidden: true })) }, ], }; const text = writeTake(take) @@ -574,11 +590,14 @@ function runSynthetic() { status('synthetic — exercises everything below detection', 'ok'); } -for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell', 'teethErode']) { +for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', + 'teethOn', 'teethDwell', 'teethErode', 'tongueReject', 'blobGrow', + 'topBias', 'teethVerts', 'teethSmooth']) { el(id).addEventListener('input', () => { el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol' ? (+el(id).value / 1000).toFixed(3) - : id === 'teethOn' || id === 'teethErode' ? (+el(id).value / 100).toFixed(2) + : ['teethOn', 'teethErode', 'tongueReject', 'topBias'].includes(id) + ? (+el(id).value / 100).toFixed(2) : el(id).value; if (id === 'tol') return; // tol only matters when you ask for a suggestion rebuild(false); @@ -699,6 +718,11 @@ PALETTE.forEach((p) => { // #synth / #frames autorun, so the tool can be driven headlessly for smoke tests // and deep-linked. Detection needs WebGL; the synthetic path does not. +window.addEventListener('error', (e) => { + const s = document.getElementById('status'); + if (s) { s.textContent = e.message; s.className = 'err'; } +}); + if (location.hash === '#synth') runSynthetic(); else if (location.hash === '#frames') runFrames(); else status('ready — Load frames, then step with \u2190 \u2192 and delete with X'); diff --git a/js/interior.js b/js/interior.js index 46580e6..f3d7053 100644 --- a/js/interior.js +++ b/js/interior.js @@ -1,38 +1,37 @@ // Mouth interior from image content. // // MediaPipe has no landmarks inside the lips - the inner ring bounds the cavity -// and everything within it is just pixels. So teeth have to come from the -// picture, and the question is how to do that without reintroducing the boil -// that per-frame detection causes. +// and everything within it is just pixels. So teeth come from the picture. // -// The answer is to extract a SCALAR, not a shape. Tracing the bright blob would -// give a new contour every frame with no vertex correspondence - exactly the -// failure docs/roto-puppet.md warns about. Instead the teeth polygon is the -// inner lip ring clipped to a horizontal line, and only that line's height is -// measured. The silhouette is therefore always the mouth's own shape (stable by -// construction) and the only thing that varies per frame is one number, which -// smooths trivially. It is also how the shape is drawn by hand: a band bounded -// by the lip. +// The hazard is vertex correspondence. A traced contour reorders between frames +// and boils, which is the failure docs/roto-puppet.md exists to avoid. The way +// out for a blob specifically is RADIAL SAMPLING: march outward from the +// centroid along N fixed directions and take the last pixel inside. Vertex k is +// then always "the blob's extent in direction k" - correspondence holds by +// construction, the vertex count is fixed, and the result smooths over time +// without any reordering being possible. It also yields a star-shaped +// reduction, which is what flat blocks of colour want anyway. -// Otsu's threshold over a luminance histogram. Self-tuning, so exposure changes -// between frames do not shift what counts as "bright". export function otsuForTest(h, t) { return otsu(h, t); } +// Otsu's threshold plus its two class means. The means matter as much as the +// threshold: Otsu ALWAYS returns a split, including on a homogeneous region, so +// their separation is the only thing that says the split means anything. function otsu(hist, total) { let sum = 0; for (let i = 0; i < 256; i++) sum += i * hist[i]; - let sumB = 0, wB = 0, best = 0, bestVar = -1, bestMB = 0, bestMF = 0; + let sumB = 0, wB = 0, best = 0, bestVar = -1, bestDark = 0, bestBright = 0; for (let t = 0; t < 256; t++) { wB += hist[t]; if (!wB) continue; const wF = total - wB; if (!wF) break; sumB += t * hist[t]; - const mB = sumB / wB, mF = (sum - sumB) / wF; // mB = dark class, mF = bright - const between = wB * wF * (mB - mF) * (mB - mF); - if (between > bestVar) { bestVar = between; best = t; bestMB = mB; bestMF = mF; } + const mDark = sumB / wB, mBright = (sum - sumB) / wF; + const between = wB * wF * (mDark - mBright) * (mDark - mBright); + if (between > bestVar) { bestVar = between; best = t; bestDark = mDark; bestBright = mBright; } } - return { thr: best, mDark: bestMB, mBright: bestMF }; + return { thr: best, mDark: bestDark, mBright: bestBright }; } const pointInPoly = (pts, x, y) => { @@ -44,133 +43,183 @@ const pointInPoly = (pts, x, y) => { return inside; }; -// Shrink a ring toward its centroid. -// -// MediaPipe's inner lip landmarks sit slightly OUTSIDE the actual opening, so -// sampling the ring as given includes lip pixels - which are bright, and sit -// right at the cavity boundary where they do the most damage. -function erode(pts, k) { +// Shrink or grow a ring about its centroid. MediaPipe's inner lip landmarks sit +// slightly OUTSIDE the real opening, so sampling the ring as given includes lip +// pixels - bright, and right at the boundary where they do most damage. +export function scaleRing(pts, k) { let cx = 0, cy = 0; for (const p of pts) { cx += p.x; cy += p.y; } cx /= pts.length; cy /= pts.length; - return pts.map((p) => ({ x: cx + (p.x - cx) * (1 - k), y: cy + (p.y - cy) * (1 - k) })); + return pts.map((p) => ({ x: cx + (p.x - cx) * k, y: cy + (p.y - cy) * k })); } -// Measure one frame: how far down the cavity the bright region reaches, as a -// fraction of cavity height, plus the contrast that justified calling it bright. -// -// `wantDebug` returns the sampled crop with the classification drawn on it. -// Tuning this blind is miserable; the numbers alone do not say whether the -// region being measured is even the right region. -export function measureInterior(img, innerNorm, ctx, opts = {}, wantDebug = false) { - const minContrast = opts.minContrast ?? 0.14; - const rowFrac = opts.rowFrac ?? 0.4; - const inner = erode(innerNorm, opts.erode ?? 0.18); +/* ---- binary morphology on the candidate mask ---- */ + +function erodeMask(m, w, h) { + const o = new Uint8Array(m.length); + for (let y = 1; y < h - 1; y++) for (let x = 1; x < w - 1; x++) { + const i = y * w + x; + o[i] = m[i] && m[i - 1] && m[i + 1] && m[i - w] && m[i + w] ? 1 : 0; + } + return o; +} +function dilateMask(m, w, h) { + const o = new Uint8Array(m.length); + for (let y = 1; y < h - 1; y++) for (let x = 1; x < w - 1; x++) { + const i = y * w + x; + o[i] = m[i] || m[i - 1] || m[i + 1] || m[i - w] || m[i + w] ? 1 : 0; + } + return o; +} + +// Largest 4-connected component, scored with a bias toward the TOP of the +// cavity: upper teeth hang from the lip, and the usual false positive is the +// tongue sitting lower down. Area alone picks the tongue when the mouth is wide. +function bestComponent(mask, w, h, topBias) { + const label = new Int32Array(mask.length).fill(-1); + const stack = []; + let best = null, id = 0; + for (let s = 0; s < mask.length; s++) { + if (!mask[s] || label[s] >= 0) continue; + stack.length = 0; stack.push(s); + label[s] = id; + const px = []; + let sumY = 0; + while (stack.length) { + const i = stack.pop(); + px.push(i); + sumY += (i / w) | 0; + const x = i % w, y = (i / w) | 0; + if (x > 0 && mask[i - 1] && label[i - 1] < 0) { label[i - 1] = id; stack.push(i - 1); } + if (x < w - 1 && mask[i + 1] && label[i + 1] < 0) { label[i + 1] = id; stack.push(i + 1); } + if (y > 0 && mask[i - w] && label[i - w] < 0) { label[i - w] = id; stack.push(i - w); } + if (y < h - 1 && mask[i + w] && label[i + w] < 0) { label[i + w] = id; stack.push(i + w); } + } + const meanY = sumY / px.length / h; // 0 top, 1 bottom + const score = px.length * (1 - topBias * meanY); + if (!best || score > best.score) best = { score, px, area: px.length, meanY }; + id++; + } + return best; +} + +// Radial sampling from the centroid: N fixed directions, last pixel inside. +function radialContour(mask, w, h, cx, cy, n) { + const pts = []; + const maxR = Math.hypot(w, h); + let prev = 1; + for (let k = 0; k < n; k++) { + const a = -(k / n) * Math.PI * 2; // slot 0 = +x, 5/20 = top + const dx = Math.cos(a), dy = Math.sin(a); + let hit = 0; + for (let r = 0.5; r < maxR; r += 0.5) { + const x = Math.round(cx + dx * r), y = Math.round(cy + dy * r); + if (x < 0 || y < 0 || x >= w || y >= h) break; + if (mask[y * w + x]) hit = r; + else if (hit > 0 && r > hit + 2) break; // tolerate a 2px gap, then stop + } + // A ray that escapes immediately would collapse the polygon; hold the last + // good radius so the shape stays closed rather than spiking to the centre. + if (hit <= 0) hit = prev * 0.6; + prev = hit; + pts.push({ x: cx + dx * hit, y: cy + dy * hit }); + } + return pts; +} + +/* ---- the extraction ---- */ + +export function extractTeeth(img, innerNorm, ctx, o, wantDebug = false) { + const none = { contour: null, contrast: 0, area: 0, debug: null }; + const ring = scaleRing(innerNorm, 1 - (o.cavityErode ?? 0.18)); - const none = { teethT: 0, contrast: 0, coverage: 0, debug: null }; let x0 = 1, y0 = 1, x1 = 0, y1 = 0; - for (const p of inner) { + for (const p of ring) { x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y); x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y); } const W = img.naturalWidth, H = img.naturalHeight; const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H)); const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H)); - if (pw < 4 || ph < 4) return none; + if (pw < 5 || ph < 5) return none; ctx.canvas.width = pw; ctx.canvas.height = ph; ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph); - const img0 = ctx.getImageData(0, 0, pw, ph); - const data = img0.data; + const src = ctx.getImageData(0, 0, pw, ph); + const d = src.data; - const poly = inner.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 })); + const poly = ring.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 })); const hist = new Uint32Array(256); const lum = new Float32Array(pw * ph); - const mask = new Uint8Array(pw * ph); + const red = new Float32Array(pw * ph); + const inReg = new Uint8Array(pw * ph); let n = 0; - for (let y = 0; y < ph; y++) { - for (let x = 0; x < pw; x++) { - if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue; - const o = (y * pw + x) * 4; - const l = (0.299 * data[o] + 0.587 * data[o + 1] + 0.114 * data[o + 2]) | 0; - const i = y * pw + x; - lum[i] = l; mask[i] = 1; hist[l]++; n++; - } + for (let y = 0; y < ph; y++) for (let x = 0; x < pw; x++) { + if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue; + const i = y * pw + x, oo = i * 4; + const R = d[oo], G = d[oo + 1], B = d[oo + 2]; + lum[i] = (0.299 * R + 0.587 * G + 0.114 * B) | 0; + // Tongue is red relative to its own brightness; teeth are near-neutral. + red[i] = (R - (G + B) / 2) / 255; + inReg[i] = 1; hist[lum[i]]++; n++; } - if (n < 16) return none; + if (n < 24) return none; const { thr, mDark, mBright } = otsu(hist, n); - - // Otsu ALWAYS returns a split, including on a homogeneous region: given a dark - // cavity with no teeth it invents a threshold and calls half the pixels - // bright. The separation between the two class means is what says whether the - // split means anything, so it is the actual gate. const contrast = (mBright - mDark) / 255; - if (contrast < minContrast) { - return { teethT: 0, contrast, coverage: 0, debug: wantDebug ? debugCanvas(img0, mask, lum, thr, pw, ph, -1) : null }; + + let mask = new Uint8Array(pw * ph); + for (let i = 0; i < mask.length; i++) { + mask[i] = inReg[i] && lum[i] > thr && red[i] < (o.tongueReject ?? 0.18) ? 1 : 0; } - // Scan from the top and STOP at the first row that fails. Teeth hang from the - // upper lip, so what matters is the contiguous run, not whether some row near - // the bottom happens to qualify - tracking the latter is what made the band - // fill the whole mouth. - let lastRow = -1, started = false, bright = 0; - for (let y = 0; y < ph; y++) { - let rowIn = 0, rowBright = 0; - for (let x = 0; x < pw; x++) { - const i = y * pw + x; - if (!mask[i]) continue; - rowIn++; - if (lum[i] > thr) { rowBright++; bright++; } - } - if (rowIn < 2) continue; - const ok = rowBright / rowIn > rowFrac; - if (ok) { started = true; lastRow = y; } - else if (started) break; + // Open once to despeckle, then apply the signed size adjustment. + mask = dilateMask(erodeMask(mask, pw, ph), pw, ph); + const grow = o.blobGrow | 0; + for (let k = 0; k < Math.abs(grow); k++) { + mask = grow < 0 ? erodeMask(mask, pw, ph) : dilateMask(mask, pw, ph); } + const comp = bestComponent(mask, pw, ph, o.topBias ?? 0.6); + if (!comp || comp.area < (o.minArea ?? 12)) { + return { contour: null, contrast, area: comp ? comp.area : 0, + debug: wantDebug ? debugCanvas(src, inReg, mask, pw, ph, null) : null }; + } + + const only = new Uint8Array(mask.length); + let cx = 0, cy = 0; + for (const i of comp.px) { only[i] = 1; cx += i % pw; cy += (i / pw) | 0; } + cx /= comp.px.length; cy /= comp.px.length; + + const local = radialContour(only, pw, ph, cx, cy, o.verts ?? 10); + const contour = local.map((p) => ({ x: (p.x + px0) / W, y: (p.y + py0) / H })); + return { - teethT: lastRow < 0 ? 0 : (lastRow + 1) / ph, - contrast, - coverage: bright / n, - debug: wantDebug ? debugCanvas(img0, mask, lum, thr, pw, ph, lastRow) : null, + contour, contrast, area: comp.area, + debug: wantDebug ? debugCanvas(src, inReg, only, pw, ph, local) : null, }; } -// The sampled crop with the classification painted on: sampled region tinted, -// pixels above threshold in green, the resolved teeth line in amber. -function debugCanvas(img0, mask, lum, thr, pw, ph, lastRow) { +// Sampled region dimmed, kept pixels green, extracted contour in amber. +function debugCanvas(src, inReg, mask, pw, ph, local) { const c = document.createElement('canvas'); c.width = pw; c.height = ph; const g = c.getContext('2d'); const out = new ImageData(pw, ph); for (let i = 0; i < pw * ph; i++) { const o = i * 4; - const [r, gr, b] = [img0.data[o], img0.data[o + 1], img0.data[o + 2]]; - if (!mask[i]) { out.data[o] = r * 0.3; out.data[o + 1] = gr * 0.3; out.data[o + 2] = b * 0.3; } - else if (lum[i] > thr) { out.data[o] = 60; out.data[o + 1] = 230; out.data[o + 2] = 120; } + const [r, gr, b] = [src.data[o], src.data[o + 1], src.data[o + 2]]; + if (!inReg[i]) { out.data[o] = r * 0.25; out.data[o + 1] = gr * 0.25; out.data[o + 2] = b * 0.25; } + else if (mask[i]) { out.data[o] = 60; out.data[o + 1] = 230; out.data[o + 2] = 120; } else { out.data[o] = r; out.data[o + 1] = gr; out.data[o + 2] = b; } out.data[o + 3] = 255; } g.putImageData(out, 0, 0); - if (lastRow >= 0) { - g.fillStyle = '#fbbf24'; - g.fillRect(0, lastRow, pw, 1); + if (local && local.length) { + g.strokeStyle = '#fbbf24'; g.lineWidth = 1; + g.beginPath(); + local.forEach((p, i) => (i ? g.lineTo(p.x, p.y) : g.moveTo(p.x, p.y))); + g.closePath(); g.stroke(); } return c; } - -// Sutherland-Hodgman against the half-plane y <= limit. -export function clipPolyAbove(pts, limit) { - const out = []; - for (let i = 0; i < pts.length; i++) { - const a = pts[i], b = pts[(i + 1) % pts.length]; - const ain = a.y <= limit, bin = b.y <= limit; - if (ain) out.push(a); - if (ain !== bin) { - const t = (limit - a.y) / (b.y - a.y); - out.push({ x: a.x + t * (b.x - a.x), y: limit }); - } - } - return out; -} diff --git a/js/selftest.js b/js/selftest.js index ddbeec9..06b01e0 100644 --- a/js/selftest.js +++ b/js/selftest.js @@ -12,7 +12,7 @@ import { fitSimilarity, applySim, procrustesMean, smoothTransforms } from './mat import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js'; import { IndexedRaster, hexToRgb } from './raster.js'; import { writeTake } from './take.js'; -import { clipPolyAbove, otsuForTest } from './interior.js'; +import { otsuForTest, scaleRing } from './interior.js'; import { synthDense } from './synth.js'; const results = []; @@ -182,25 +182,17 @@ export function run() { ok('palette expansion introduces no intermediate colours', [...seen].every((c) => allowed.has(c)), `${seen.size} distinct colours`); - // teeth band: the inner ring clipped to a height. The point of doing it this - // way is that the silhouette is the mouth's own shape, so it can never - // disagree with the lips - assert that rather than the pixel measurement. + // scaleRing is what pulls the sampled region in from MediaPipe's inner lip + // landmarks, which sit slightly outside the real opening. { const ring = [{ x: 0, y: 0 }, { x: 10, y: 0 }, { x: 10, y: 10 }, { x: 0, y: 10 }]; - const half = clipPolyAbove(ring, 5); - ok('clip at mid height halves the box', - half.length === 4 && Math.max(...half.map((p) => p.y)) === 5, - `${half.length} pts, maxY ${Math.max(...half.map((p) => p.y))}`); - ok('clip above everything keeps the ring', clipPolyAbove(ring, 99).length === 4); - ok('clip below everything empties it', clipPolyAbove(ring, -1).length === 0); - const w = clipPolyAbove(ring, 5); - ok('clipped band keeps the ring width', - Math.min(...w.map((p) => p.x)) === 0 && Math.max(...w.map((p) => p.x)) === 10); - // a non-convex ring must not gain or lose x-extent from clipping - const tri = [{ x: 0, y: 0 }, { x: 8, y: 2 }, { x: 4, y: 9 }]; - const ct = clipPolyAbove(tri, 4); - ok('clip of a non-convex-ish ring stays within its x range', - Math.min(...ct.map((p) => p.x)) >= 0 && Math.max(...ct.map((p) => p.x)) <= 8); + const small = scaleRing(ring, 0.5); + const w = Math.max(...small.map((p) => p.x)) - Math.min(...small.map((p) => p.x)); + ok('scaleRing(0.5) halves the extent', Math.abs(w - 5) < 1e-9, `width ${w}`); + const same = scaleRing(ring, 1); + ok('scaleRing(1) is identity', same.every((p, i) => Math.abs(p.x - ring[i].x) < 1e-9)); + let cx = 0; for (const p of small) cx += p.x; + ok('scaleRing keeps the centroid', Math.abs(cx / 4 - 5) < 1e-9); } // Otsu on a uniform region must report near-zero class separation. It will diff --git a/probe.html b/probe.html deleted file mode 100644 index 06b6493..0000000 --- a/probe.html +++ /dev/null @@ -1,79 +0,0 @@ -probe… -
running…
- From a146d271b6e474a8ff3832c1a2c2a313d0287604 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 24 Sep 2026 15:25:39 -0400 Subject: [PATCH 05/10] Cross-check el() ids against index.html in the selftest A knob wired in app.js but missing from the markup throws during wiring, aborts the module and leaves a blank page - a symptom pointing nowhere near its cause. It has now happened twice, so it gets a check rather than vigilance: the selftest fetches both files and compares the id sets in each direction. Co-Authored-By: Claude Opus 5 --- README.md | 6 +++++- js/selftest.js | 31 +++++++++++++++++++++++++++++++ selftest.html | 3 ++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index af8dca5..6ced56b 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,11 @@ chromium --headless --virtual-time-budget=8000 --dump-dom \ http://127.0.0.1:8777/selftest.html | grep -oE '(PASS|FAIL) [0-9/]+' ``` -Or open `selftest.html`. 29 assertions over the stages below detection. +Or open `selftest.html`. 41 assertions over the stages below detection, plus a +wiring cross-check: every `el('id')` in `app.js` must exist in `index.html`. A +knob wired in one but not the other throws during wiring, which aborts the rest +of the module and leaves a blank page — a symptom that points nowhere near its +cause, and which has happened twice. The ring-simplicity check is the load-bearing one. Because `hold` parts *cut* between poses instead of interpolating, a ring whose vertex order is wrong diff --git a/js/selftest.js b/js/selftest.js index 06b01e0..24cae67 100644 --- a/js/selftest.js +++ b/js/selftest.js @@ -45,6 +45,37 @@ const spreadX = (frames, slot) => { /* ---- the tests ---- */ +// Cross-check every el('id') in app.js against the ids in index.html. +// +// This bug class has bitten twice: a knob wired in app.js but absent from the +// markup throws during wiring, which aborts the rest of the module and leaves a +// blank page. The symptom ("nothing happens") points nowhere near the cause, so +// it is worth an automated check rather than vigilance. +export async function runWiring() { + const out = []; + try { + const [app, html] = await Promise.all([ + fetch('./js/app.js').then((r) => r.text()), + fetch('./index.html').then((r) => r.text()), + ]); + const ids = new Set([...app.matchAll(/\bel\(\s*['"]([\w-]+)['"]\s*\)/g)].map((m) => m[1])); + const list = app.match(/for \(const id of \[([\s\S]*?)\]\)/); + if (list) { + for (const m of list[1].matchAll(/'([\w]+)'/g)) { ids.add(m[1]); ids.add(m[1] + 'v'); } + } + const have = new Set([...html.matchAll(/id="([\w-]+)"/g)].map((m) => m[1])); + const missing = [...ids].filter((i) => !have.has(i)); + out.push({ name: `every el() id exists in index.html (${ids.size} checked)`, + pass: missing.length === 0, detail: missing.join(', ') }); + const unused = [...have].filter((i) => !ids.has(i)); + out.push({ name: 'no orphaned ids in index.html', pass: unused.length === 0, + detail: unused.join(', ') }); + } catch (e) { + out.push({ name: 'wiring check ran', pass: false, detail: e.message }); + } + return out; +} + export function run() { results.length = 0; diff --git a/selftest.html b/selftest.html index 7fa7360..0fa0d31 100644 --- a/selftest.html +++ b/selftest.html @@ -7,10 +7,11 @@

running…