diff --git a/README.md b/README.md
index 85b29bc..8c8f043 100644
--- a/README.md
+++ b/README.md
@@ -62,6 +62,29 @@ is hand-drawn head plates, which this tool does not yet do.
| closed-mouth cut | Aperture below which the mouth interior is emitted as `hidden`. |
| suggest tolerance | Max head movement before a new plate drawing is required. Affects **Suggest** only. |
+## Teeth
+
+MediaPipe has no landmarks inside the lips: the inner ring bounds the cavity and
+everything within it is just pixels. So teeth come from the image — but tracing
+the bright blob would produce a new contour every frame with no vertex
+correspondence, which is precisely the boil the design exists to avoid.
+
+So the extraction yields a **scalar, not a shape**. The teeth polygon is the inner
+lip ring clipped to a horizontal line, and only that line's height is measured
+(Otsu threshold within the cavity, scanned from the top). The silhouette is
+therefore always the mouth's own shape — stable by construction — and the only
+per-frame variable is one number, which smooths trivially. It is also how the
+shape gets drawn by hand: a band bounded by the lip.
+
+Presence uses hysteresis plus a minimum dwell, the same treatment plate selection
+gets, because a teeth block that blinks on and off for single frames is worse
+than one that is simply absent.
+
+**Tongue** would work the same way — a shape filling the lower cavity, gated on a
+redness rather than a brightness statistic. Not implemented, because it is not
+visible in the test footage: the cavity reads as dark with a bright upper-teeth
+band and nothing else.
+
## The plate is reference, not art
The plate layer has several representations because its job changes. Cycle with
diff --git a/index.html b/index.html
index 37475e3..8606bc9 100644
--- a/index.html
+++ b/index.html
@@ -122,11 +122,16 @@
+
+
contour avg 0 = off, 1 = ±1 frame. Removes per-frame landmark
jitter. Push past 2 and it starts eating articulation. anchor avg smooths the head transform only — never the contour.
+ teeth threshold bright fraction of the cavity needed before a teeth
+ band appears; dwell is how many frames a change must persist, so it
+ cannot blink on and off. suggest tolerance only affects the Suggest button: max head movement
allowed before a new drawing is required.
diff --git a/js/app.js b/js/app.js
index 6e6fc61..88be7ea 100644
--- a/js/app.js
+++ b/js/app.js
@@ -3,6 +3,7 @@ import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js';
import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js';
import { IndexedRaster } from './raster.js';
import { drawRegistered, posterizeInto } from './underlay.js';
+import { measureInterior, clipPolyAbove } from './interior.js';
import { writeTake } from './take.js';
import { synthDense } from './synth.js';
@@ -13,9 +14,9 @@ const PALETTE = [
{ name: 'skin_base', hex: '#b07a5a' },
{ name: 'skin_dark', hex: '#7a4f3a' },
{ name: 'mouth_dark', hex: '#24161a' },
- { name: 'skin_lite', hex: '#d9a884' },
+ { name: 'teeth', hex: '#d9cfc2' },
];
-const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, lite: 4 };
+const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, teeth: 4 };
const state = {
dense: null, images: [], stab: null, xform: null,
@@ -24,11 +25,15 @@ const state = {
frame: 0, playing: false, faceBox: null,
fps: 12, audio: null, // fps comes from manifest.json, never guessed
aspect: 1, // imgW/imgH; converts MediaPipe's anisotropic space
+ interior: null, // per-frame teeth measurement from image content
+ teeth: null, // resolved per-frame {show, t} after knobs
};
const el = (id) => document.getElementById(id);
const opts = () => ({
verts: +el('verts').value,
+ teethOn: +el('teethOn').value / 100,
+ teethDwell: +el('teethDwell').value,
smoothWin: +el('smoothWin').value,
contourSmooth: +el('contourSmooth').value,
apertureThresh: +el('apertureThresh').value / 1000,
@@ -116,6 +121,14 @@ async function detectAll(images) {
return { dense, missing };
}
+// Interior measurement is a function of pixels alone, so it runs once with
+// detection and the knobs re-resolve it instantly afterwards.
+function measureAll(images, dense) {
+ const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
+ return dense.map((lm, i) =>
+ measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx));
+}
+
/* ---------- build ---------- */
function makeXform(stab, neutral) {
@@ -154,6 +167,8 @@ function rebuild(resetKeep) {
const apMax = Math.max(...ap);
state.hidden = ap.map((v) => v / apMax < o.apertureThresh);
+ state.teeth = resolveTeeth(o);
+
// Plate outline per frame, so a kept frame shows its own head shape.
state.plates = state.stab.oval.map((r) => r.map(state.xform));
@@ -180,6 +195,40 @@ function faceBoxes() {
});
}
+// Presence gets hysteresis and a minimum dwell, the same treatment plate
+// selection gets: a teeth block that blinks on and off for single frames is
+// worse than one that is simply absent. Appearing needs a clear signal, staying
+// needs only a weak one.
+function resolveTeeth(o) {
+ const N = state.dense.length;
+ if (!state.interior) return new Array(N).fill({ show: false, t: 0 });
+
+ const raw = state.interior.map((m, f) => (state.hidden[f] ? 0 : m.coverage));
+ const on = o.teethOn, off = o.teethOn * 0.6;
+ const shown = new Array(N).fill(false);
+ let live = false, since = 0;
+ for (let f = 0; f < N; f++) {
+ const want = live ? raw[f] > off : raw[f] > on;
+ if (want !== live && since >= o.teethDwell) { live = want; since = 0; }
+ else since++;
+ shown[f] = live && !state.hidden[f];
+ }
+
+ // Smooth the line height only across frames where it is actually shown, so a
+ // gap does not drag the band toward zero on either side of it.
+ const t = state.interior.map((m) => m.teethT);
+ const sm = t.map((_, f) => {
+ let acc = 0, c = 0;
+ for (let j = f - o.contourSmooth; j <= f + o.contourSmooth; j++) {
+ const k = Math.min(N - 1, Math.max(0, j));
+ if (!shown[k]) continue;
+ acc += t[k]; c++;
+ }
+ return c ? acc / c : t[f];
+ });
+ return shown.map((show, f) => ({ show, t: sm[f] }));
+}
+
/* ---------- render ---------- */
// The plate layer has several representations because its job changes: a flat
@@ -199,7 +248,18 @@ function renderFrame(f, mode = plateMode()) {
}
r.fillPoly(state.outer[f], IDX.dark); // mouth keeps every frame
- if (!state.hidden[f]) r.fillPoly(state.inner[f], IDX.mouth);
+ if (!state.hidden[f]) {
+ r.fillPoly(state.inner[f], IDX.mouth);
+ const te = state.teeth[f];
+ if (te.show) {
+ // The band is the mouth's own silhouette clipped to a height, so the shape
+ // can never disagree with the lips around it.
+ const ys = state.inner[f].map((p) => p.y);
+ const top = Math.min(...ys), bot = Math.max(...ys);
+ const poly = clipPolyAbove(state.inner[f], top + te.t * (bot - top));
+ if (poly.length >= 3) r.fillPoly(poly, IDX.teeth);
+ }
+ }
return r;
}
@@ -259,8 +319,10 @@ function drawAll() {
function drawReadout() {
const kept = keptSorted();
const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k);
+ const teethFrames = state.teeth ? state.teeth.filter((t) => t.show).length : 0;
el('readout').textContent =
`${state.dense.length} frames → ${kept.length} drawings · ` +
+ `teeth on ${teethFrames}f · ` +
`holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` +
`neutral f${state.neutral} · residual ` +
`${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`;
@@ -406,6 +468,14 @@ function exportTake() {
keys: state.outer.map((pts, f) => ({ f, src: f, pts })) },
{ name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) },
+ { name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in',
+ keys: state.inner.map((pts, f) => {
+ const te = state.teeth[f];
+ if (state.hidden[f] || !te.show) return { f, hidden: true };
+ const ys = pts.map((p) => p.y);
+ const top = Math.min(...ys), bot = Math.max(...ys);
+ return { f, src: f, pts: clipPolyAbove(pts, top + te.t * (bot - top)) };
+ }) },
],
};
const text = writeTake(take)
@@ -442,6 +512,8 @@ async function runFrames() {
const { dense, missing } = await detectAll(images);
state.images = images; state.dense = dense;
state.aspect = images[0].naturalWidth / images[0].naturalHeight;
+ status('measuring mouth interiors…');
+ state.interior = measureAll(images, dense);
el('scrub').max = dense.length - 1;
state.frame = 0;
rebuild(true);
@@ -459,6 +531,7 @@ function runSynthetic() {
attachAudio(null);
state.fps = 12;
state.aspect = 1; // synthetic landmarks are generated square
+ state.interior = null; // no pixels, so no teeth
state.dense = synthDense(72);
el('scrub').max = 71;
@@ -467,10 +540,11 @@ function runSynthetic() {
status('synthetic — exercises everything below detection', 'ok');
}
-for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol']) {
+for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell']) {
el(id).addEventListener('input', () => {
el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol'
- ? (+el(id).value / 1000).toFixed(3) : el(id).value;
+ ? (+el(id).value / 1000).toFixed(3)
+ : id === 'teethOn' ? (+el(id).value / 100).toFixed(2) : el(id).value;
if (id === 'tol') return; // tol only matters when you ask for a suggestion
rebuild(false);
});
diff --git a/js/interior.js b/js/interior.js
new file mode 100644
index 0000000..056b75c
--- /dev/null
+++ b/js/interior.js
@@ -0,0 +1,113 @@
+// Mouth interior from image content.
+//
+// MediaPipe has no landmarks inside the lips - the inner ring bounds the cavity
+// and everything within it is just pixels. So teeth have to come from the
+// picture, and the question is how to do that without reintroducing the boil
+// that per-frame detection causes.
+//
+// The answer is to extract a SCALAR, not a shape. Tracing the bright blob would
+// give a new contour every frame with no vertex correspondence - exactly the
+// failure docs/roto-puppet.md warns about. Instead the teeth polygon is the
+// inner lip ring clipped to a horizontal line, and only that line's height is
+// measured. The silhouette is therefore always the mouth's own shape (stable by
+// construction) and the only thing that varies per frame is one number, which
+// smooths trivially. It is also how the shape is drawn by hand: a band bounded
+// by the lip.
+
+// Otsu's threshold over a luminance histogram. Self-tuning, so exposure changes
+// between frames do not shift what counts as "bright".
+function otsu(hist, total) {
+ let sum = 0;
+ for (let i = 0; i < 256; i++) sum += i * hist[i];
+ let sumB = 0, wB = 0, best = 0, bestVar = -1;
+ for (let t = 0; t < 256; t++) {
+ wB += hist[t];
+ if (!wB) continue;
+ const wF = total - wB;
+ if (!wF) break;
+ sumB += t * hist[t];
+ const mB = sumB / wB, mF = (sum - sumB) / wF;
+ const between = wB * wF * (mB - mF) * (mB - mF);
+ if (between > bestVar) { bestVar = between; best = t; }
+ }
+ return best;
+}
+
+const pointInPoly = (pts, x, y) => {
+ let inside = false;
+ for (let i = 0, j = pts.length - 1; i < pts.length; j = i++) {
+ if ((pts[i].y > y) !== (pts[j].y > y) &&
+ x < ((pts[j].x - pts[i].x) * (y - pts[i].y)) / (pts[j].y - pts[i].y) + pts[i].x) inside = !inside;
+ }
+ return inside;
+};
+
+// Measure one frame: how far down the cavity the bright region reaches, as a
+// fraction of cavity height, plus how much of the cavity is bright at all.
+export function measureInterior(img, innerNorm, ctx) {
+ let x0 = 1, y0 = 1, x1 = 0, y1 = 0;
+ for (const p of innerNorm) {
+ x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y);
+ x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y);
+ }
+ const W = img.naturalWidth, H = img.naturalHeight;
+ const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H));
+ const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H));
+ if (pw < 3 || ph < 3) return { teethT: 0, coverage: 0 };
+
+ ctx.canvas.width = pw; ctx.canvas.height = ph;
+ ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph);
+ const data = ctx.getImageData(0, 0, pw, ph).data;
+
+ const poly = innerNorm.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 }));
+ const hist = new Uint32Array(256);
+ const lum = new Float32Array(pw * ph);
+ const mask = new Uint8Array(pw * ph);
+ let n = 0;
+ for (let y = 0; y < ph; y++) {
+ for (let x = 0; x < pw; x++) {
+ if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue;
+ const o = (y * pw + x) * 4;
+ const l = (0.299 * data[o] + 0.587 * data[o + 1] + 0.114 * data[o + 2]) | 0;
+ const i = y * pw + x;
+ lum[i] = l; mask[i] = 1; hist[l]++; n++;
+ }
+ }
+ if (n < 12) return { teethT: 0, coverage: 0 };
+
+ const thr = otsu(hist, n);
+ // Rows are scanned from the top of the cavity; the teeth line is where the
+ // bright run stops. Requiring a run rather than a single row rejects specular
+ // speckle on a wet lower lip.
+ let bright = 0, lastBrightRow = -1, run = 0;
+ for (let y = 0; y < ph; y++) {
+ let rowIn = 0, rowBright = 0;
+ for (let x = 0; x < pw; x++) {
+ const i = y * pw + x;
+ if (!mask[i]) continue;
+ rowIn++;
+ if (lum[i] > thr) { rowBright++; bright++; }
+ }
+ if (rowIn >= 2 && rowBright / rowIn > 0.45) { run++; if (run >= 1) lastBrightRow = y; }
+ else run = 0;
+ }
+ return {
+ teethT: lastBrightRow < 0 ? 0 : (lastBrightRow + 1) / ph,
+ coverage: bright / n,
+ };
+}
+
+// Sutherland-Hodgman against the half-plane y <= limit.
+export function clipPolyAbove(pts, limit) {
+ const out = [];
+ for (let i = 0; i < pts.length; i++) {
+ const a = pts[i], b = pts[(i + 1) % pts.length];
+ const ain = a.y <= limit, bin = b.y <= limit;
+ if (ain) out.push(a);
+ if (ain !== bin) {
+ const t = (limit - a.y) / (b.y - a.y);
+ out.push({ x: a.x + t * (b.x - a.x), y: limit });
+ }
+ }
+ return out;
+}
diff --git a/js/selftest.js b/js/selftest.js
index 0e41def..9b6cd95 100644
--- a/js/selftest.js
+++ b/js/selftest.js
@@ -12,6 +12,7 @@ import { fitSimilarity, applySim, procrustesMean, smoothTransforms } from './mat
import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js';
import { IndexedRaster, hexToRgb } from './raster.js';
import { writeTake } from './take.js';
+import { clipPolyAbove } from './interior.js';
import { synthDense } from './synth.js';
const results = [];
@@ -181,6 +182,27 @@ export function run() {
ok('palette expansion introduces no intermediate colours',
[...seen].every((c) => allowed.has(c)), `${seen.size} distinct colours`);
+ // teeth band: the inner ring clipped to a height. The point of doing it this
+ // way is that the silhouette is the mouth's own shape, so it can never
+ // disagree with the lips - assert that rather than the pixel measurement.
+ {
+ const ring = [{ x: 0, y: 0 }, { x: 10, y: 0 }, { x: 10, y: 10 }, { x: 0, y: 10 }];
+ const half = clipPolyAbove(ring, 5);
+ ok('clip at mid height halves the box',
+ half.length === 4 && Math.max(...half.map((p) => p.y)) === 5,
+ `${half.length} pts, maxY ${Math.max(...half.map((p) => p.y))}`);
+ ok('clip above everything keeps the ring', clipPolyAbove(ring, 99).length === 4);
+ ok('clip below everything empties it', clipPolyAbove(ring, -1).length === 0);
+ const w = clipPolyAbove(ring, 5);
+ ok('clipped band keeps the ring width',
+ Math.min(...w.map((p) => p.x)) === 0 && Math.max(...w.map((p) => p.x)) === 10);
+ // a non-convex ring must not gain or lose x-extent from clipping
+ const tri = [{ x: 0, y: 0 }, { x: 8, y: 2 }, { x: 4, y: 9 }];
+ const ct = clipPolyAbove(tri, 4);
+ ok('clip of a non-convex-ish ring stays within its x range',
+ Math.min(...ct.map((p) => p.x)) >= 0 && Math.max(...ct.map((p) => p.x)) <= 8);
+ }
+
// take writer round-trip
const take = {
name: 'test', frames: 72, width: 320, height: 200, exposure: 2,