Teeth from mouth-interior image content
MediaPipe has no landmarks inside the lips, so teeth have to come from pixels. Tracing the bright blob would give a new contour every frame with no vertex correspondence - the exact boil docs/roto-puppet.md warns about. So the measurement yields a scalar, not a shape: Otsu within the cavity, scanned from the top for where the bright run stops, giving one line height per frame. The teeth polygon is the inner lip ring clipped to that line, so the silhouette is always the mouth's own shape and cannot disagree with the lips around it, and the only per-frame variable is a single number that smooths trivially. Presence gets hysteresis and minimum dwell, as plate selection does: a teeth block blinking on and off for single frames is worse than one simply absent. Tongue is not implemented. The same scalar approach would apply, gated on redness rather than brightness, but it is not visible in the test footage - the cavity reads dark with a bright upper-teeth band and nothing else. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
29b0bd6690
commit
941022b69f
5 changed files with 242 additions and 5 deletions
23
README.md
23
README.md
|
|
@ -62,6 +62,29 @@ is hand-drawn head plates, which this tool does not yet do.
|
|||
| closed-mouth cut | Aperture below which the mouth interior is emitted as `hidden`. |
|
||||
| suggest tolerance | Max head movement before a new plate drawing is required. Affects **Suggest** only. |
|
||||
|
||||
## Teeth
|
||||
|
||||
MediaPipe has no landmarks inside the lips: the inner ring bounds the cavity and
|
||||
everything within it is just pixels. So teeth come from the image — but tracing
|
||||
the bright blob would produce a new contour every frame with no vertex
|
||||
correspondence, which is precisely the boil the design exists to avoid.
|
||||
|
||||
So the extraction yields a **scalar, not a shape**. The teeth polygon is the inner
|
||||
lip ring clipped to a horizontal line, and only that line's height is measured
|
||||
(Otsu threshold within the cavity, scanned from the top). The silhouette is
|
||||
therefore always the mouth's own shape — stable by construction — and the only
|
||||
per-frame variable is one number, which smooths trivially. It is also how the
|
||||
shape gets drawn by hand: a band bounded by the lip.
|
||||
|
||||
Presence uses hysteresis plus a minimum dwell, the same treatment plate selection
|
||||
gets, because a teeth block that blinks on and off for single frames is worse
|
||||
than one that is simply absent.
|
||||
|
||||
**Tongue** would work the same way — a shape filling the lower cavity, gated on a
|
||||
redness rather than a brightness statistic. Not implemented, because it is not
|
||||
visible in the test footage: the cavity reads as dark with a bright upper-teeth
|
||||
band and nothing else.
|
||||
|
||||
## The plate is reference, not art
|
||||
|
||||
The plate layer has several representations because its job changes. Cycle with
|
||||
|
|
|
|||
|
|
@ -122,11 +122,16 @@
|
|||
<label class="ctl"><span>contour avg ±f</span><input type="range" id="contourSmooth" min="0" max="4" value="1"><output id="contourSmoothv"></output></label>
|
||||
<label class="ctl"><span>anchor avg ±f</span><input type="range" id="smoothWin" min="0" max="8" value="2"><output id="smoothWinv"></output></label>
|
||||
<label class="ctl"><span>closed-mouth cut</span><input type="range" id="apertureThresh" min="0" max="400" value="120"><output id="apertureThreshv"></output></label>
|
||||
<label class="ctl"><span>teeth threshold</span><input type="range" id="teethOn" min="1" max="60" value="14"><output id="teethOnv"></output></label>
|
||||
<label class="ctl"><span>teeth dwell</span><input type="range" id="teethDwell" min="0" max="6" value="1"><output id="teethDwellv"></output></label>
|
||||
<label class="ctl"><span>suggest tolerance</span><input type="range" id="tol" min="2" max="60" value="14"><output id="tolv"></output></label>
|
||||
<div class="legend">
|
||||
<b>contour avg</b> 0 = off, 1 = ±1 frame. Removes per-frame landmark
|
||||
jitter. Push past 2 and it starts eating articulation.<br>
|
||||
<b>anchor avg</b> smooths the head transform only — never the contour.<br>
|
||||
<b>teeth threshold</b> bright fraction of the cavity needed before a teeth
|
||||
band appears; <b>dwell</b> is how many frames a change must persist, so it
|
||||
cannot blink on and off.<br>
|
||||
<b>suggest tolerance</b> only affects the Suggest button: max head movement
|
||||
allowed before a new drawing is required.
|
||||
</div>
|
||||
|
|
|
|||
84
js/app.js
84
js/app.js
|
|
@ -3,6 +3,7 @@ import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js';
|
|||
import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame } from './pipeline.js';
|
||||
import { IndexedRaster } from './raster.js';
|
||||
import { drawRegistered, posterizeInto } from './underlay.js';
|
||||
import { measureInterior, clipPolyAbove } from './interior.js';
|
||||
import { writeTake } from './take.js';
|
||||
import { synthDense } from './synth.js';
|
||||
|
||||
|
|
@ -13,9 +14,9 @@ const PALETTE = [
|
|||
{ name: 'skin_base', hex: '#b07a5a' },
|
||||
{ name: 'skin_dark', hex: '#7a4f3a' },
|
||||
{ name: 'mouth_dark', hex: '#24161a' },
|
||||
{ name: 'skin_lite', hex: '#d9a884' },
|
||||
{ name: 'teeth', hex: '#d9cfc2' },
|
||||
];
|
||||
const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, lite: 4 };
|
||||
const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, teeth: 4 };
|
||||
|
||||
const state = {
|
||||
dense: null, images: [], stab: null, xform: null,
|
||||
|
|
@ -24,11 +25,15 @@ const state = {
|
|||
frame: 0, playing: false, faceBox: null,
|
||||
fps: 12, audio: null, // fps comes from manifest.json, never guessed
|
||||
aspect: 1, // imgW/imgH; converts MediaPipe's anisotropic space
|
||||
interior: null, // per-frame teeth measurement from image content
|
||||
teeth: null, // resolved per-frame {show, t} after knobs
|
||||
};
|
||||
|
||||
const el = (id) => document.getElementById(id);
|
||||
const opts = () => ({
|
||||
verts: +el('verts').value,
|
||||
teethOn: +el('teethOn').value / 100,
|
||||
teethDwell: +el('teethDwell').value,
|
||||
smoothWin: +el('smoothWin').value,
|
||||
contourSmooth: +el('contourSmooth').value,
|
||||
apertureThresh: +el('apertureThresh').value / 1000,
|
||||
|
|
@ -116,6 +121,14 @@ async function detectAll(images) {
|
|||
return { dense, missing };
|
||||
}
|
||||
|
||||
// Interior measurement is a function of pixels alone, so it runs once with
|
||||
// detection and the knobs re-resolve it instantly afterwards.
|
||||
function measureAll(images, dense) {
|
||||
const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
|
||||
return dense.map((lm, i) =>
|
||||
measureInterior(images[i], LIPS_INNER.map((k) => lm[k]), ctx));
|
||||
}
|
||||
|
||||
/* ---------- build ---------- */
|
||||
|
||||
function makeXform(stab, neutral) {
|
||||
|
|
@ -154,6 +167,8 @@ function rebuild(resetKeep) {
|
|||
const apMax = Math.max(...ap);
|
||||
state.hidden = ap.map((v) => v / apMax < o.apertureThresh);
|
||||
|
||||
state.teeth = resolveTeeth(o);
|
||||
|
||||
// Plate outline per frame, so a kept frame shows its own head shape.
|
||||
state.plates = state.stab.oval.map((r) => r.map(state.xform));
|
||||
|
||||
|
|
@ -180,6 +195,40 @@ function faceBoxes() {
|
|||
});
|
||||
}
|
||||
|
||||
// Presence gets hysteresis and a minimum dwell, the same treatment plate
|
||||
// selection gets: a teeth block that blinks on and off for single frames is
|
||||
// worse than one that is simply absent. Appearing needs a clear signal, staying
|
||||
// needs only a weak one.
|
||||
function resolveTeeth(o) {
|
||||
const N = state.dense.length;
|
||||
if (!state.interior) return new Array(N).fill({ show: false, t: 0 });
|
||||
|
||||
const raw = state.interior.map((m, f) => (state.hidden[f] ? 0 : m.coverage));
|
||||
const on = o.teethOn, off = o.teethOn * 0.6;
|
||||
const shown = new Array(N).fill(false);
|
||||
let live = false, since = 0;
|
||||
for (let f = 0; f < N; f++) {
|
||||
const want = live ? raw[f] > off : raw[f] > on;
|
||||
if (want !== live && since >= o.teethDwell) { live = want; since = 0; }
|
||||
else since++;
|
||||
shown[f] = live && !state.hidden[f];
|
||||
}
|
||||
|
||||
// Smooth the line height only across frames where it is actually shown, so a
|
||||
// gap does not drag the band toward zero on either side of it.
|
||||
const t = state.interior.map((m) => m.teethT);
|
||||
const sm = t.map((_, f) => {
|
||||
let acc = 0, c = 0;
|
||||
for (let j = f - o.contourSmooth; j <= f + o.contourSmooth; j++) {
|
||||
const k = Math.min(N - 1, Math.max(0, j));
|
||||
if (!shown[k]) continue;
|
||||
acc += t[k]; c++;
|
||||
}
|
||||
return c ? acc / c : t[f];
|
||||
});
|
||||
return shown.map((show, f) => ({ show, t: sm[f] }));
|
||||
}
|
||||
|
||||
/* ---------- render ---------- */
|
||||
|
||||
// The plate layer has several representations because its job changes: a flat
|
||||
|
|
@ -199,7 +248,18 @@ function renderFrame(f, mode = plateMode()) {
|
|||
}
|
||||
|
||||
r.fillPoly(state.outer[f], IDX.dark); // mouth keeps every frame
|
||||
if (!state.hidden[f]) r.fillPoly(state.inner[f], IDX.mouth);
|
||||
if (!state.hidden[f]) {
|
||||
r.fillPoly(state.inner[f], IDX.mouth);
|
||||
const te = state.teeth[f];
|
||||
if (te.show) {
|
||||
// The band is the mouth's own silhouette clipped to a height, so the shape
|
||||
// can never disagree with the lips around it.
|
||||
const ys = state.inner[f].map((p) => p.y);
|
||||
const top = Math.min(...ys), bot = Math.max(...ys);
|
||||
const poly = clipPolyAbove(state.inner[f], top + te.t * (bot - top));
|
||||
if (poly.length >= 3) r.fillPoly(poly, IDX.teeth);
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
|
|
@ -259,8 +319,10 @@ function drawAll() {
|
|||
function drawReadout() {
|
||||
const kept = keptSorted();
|
||||
const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k);
|
||||
const teethFrames = state.teeth ? state.teeth.filter((t) => t.show).length : 0;
|
||||
el('readout').textContent =
|
||||
`${state.dense.length} frames → ${kept.length} drawings · ` +
|
||||
`teeth on ${teethFrames}f · ` +
|
||||
`holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` +
|
||||
`neutral f${state.neutral} · residual ` +
|
||||
`${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`;
|
||||
|
|
@ -406,6 +468,14 @@ function exportTake() {
|
|||
keys: state.outer.map((pts, f) => ({ f, src: f, pts })) },
|
||||
{ name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
|
||||
keys: state.inner.map((pts, f) => (state.hidden[f] ? { f, hidden: true } : { f, src: f, pts })) },
|
||||
{ name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in',
|
||||
keys: state.inner.map((pts, f) => {
|
||||
const te = state.teeth[f];
|
||||
if (state.hidden[f] || !te.show) return { f, hidden: true };
|
||||
const ys = pts.map((p) => p.y);
|
||||
const top = Math.min(...ys), bot = Math.max(...ys);
|
||||
return { f, src: f, pts: clipPolyAbove(pts, top + te.t * (bot - top)) };
|
||||
}) },
|
||||
],
|
||||
};
|
||||
const text = writeTake(take)
|
||||
|
|
@ -442,6 +512,8 @@ async function runFrames() {
|
|||
const { dense, missing } = await detectAll(images);
|
||||
state.images = images; state.dense = dense;
|
||||
state.aspect = images[0].naturalWidth / images[0].naturalHeight;
|
||||
status('measuring mouth interiors…');
|
||||
state.interior = measureAll(images, dense);
|
||||
el('scrub').max = dense.length - 1;
|
||||
state.frame = 0;
|
||||
rebuild(true);
|
||||
|
|
@ -459,6 +531,7 @@ function runSynthetic() {
|
|||
attachAudio(null);
|
||||
state.fps = 12;
|
||||
state.aspect = 1; // synthetic landmarks are generated square
|
||||
state.interior = null; // no pixels, so no teeth
|
||||
|
||||
state.dense = synthDense(72);
|
||||
el('scrub').max = 71;
|
||||
|
|
@ -467,10 +540,11 @@ function runSynthetic() {
|
|||
status('synthetic — exercises everything below detection', 'ok');
|
||||
}
|
||||
|
||||
for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol']) {
|
||||
for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol', 'teethOn', 'teethDwell']) {
|
||||
el(id).addEventListener('input', () => {
|
||||
el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol'
|
||||
? (+el(id).value / 1000).toFixed(3) : el(id).value;
|
||||
? (+el(id).value / 1000).toFixed(3)
|
||||
: id === 'teethOn' ? (+el(id).value / 100).toFixed(2) : el(id).value;
|
||||
if (id === 'tol') return; // tol only matters when you ask for a suggestion
|
||||
rebuild(false);
|
||||
});
|
||||
|
|
|
|||
113
js/interior.js
Normal file
113
js/interior.js
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
// Mouth interior from image content.
|
||||
//
|
||||
// MediaPipe has no landmarks inside the lips - the inner ring bounds the cavity
|
||||
// and everything within it is just pixels. So teeth have to come from the
|
||||
// picture, and the question is how to do that without reintroducing the boil
|
||||
// that per-frame detection causes.
|
||||
//
|
||||
// The answer is to extract a SCALAR, not a shape. Tracing the bright blob would
|
||||
// give a new contour every frame with no vertex correspondence - exactly the
|
||||
// failure docs/roto-puppet.md warns about. Instead the teeth polygon is the
|
||||
// inner lip ring clipped to a horizontal line, and only that line's height is
|
||||
// measured. The silhouette is therefore always the mouth's own shape (stable by
|
||||
// construction) and the only thing that varies per frame is one number, which
|
||||
// smooths trivially. It is also how the shape is drawn by hand: a band bounded
|
||||
// by the lip.
|
||||
|
||||
// Otsu's threshold over a luminance histogram. Self-tuning, so exposure changes
|
||||
// between frames do not shift what counts as "bright".
|
||||
function otsu(hist, total) {
|
||||
let sum = 0;
|
||||
for (let i = 0; i < 256; i++) sum += i * hist[i];
|
||||
let sumB = 0, wB = 0, best = 0, bestVar = -1;
|
||||
for (let t = 0; t < 256; t++) {
|
||||
wB += hist[t];
|
||||
if (!wB) continue;
|
||||
const wF = total - wB;
|
||||
if (!wF) break;
|
||||
sumB += t * hist[t];
|
||||
const mB = sumB / wB, mF = (sum - sumB) / wF;
|
||||
const between = wB * wF * (mB - mF) * (mB - mF);
|
||||
if (between > bestVar) { bestVar = between; best = t; }
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
const pointInPoly = (pts, x, y) => {
|
||||
let inside = false;
|
||||
for (let i = 0, j = pts.length - 1; i < pts.length; j = i++) {
|
||||
if ((pts[i].y > y) !== (pts[j].y > y) &&
|
||||
x < ((pts[j].x - pts[i].x) * (y - pts[i].y)) / (pts[j].y - pts[i].y) + pts[i].x) inside = !inside;
|
||||
}
|
||||
return inside;
|
||||
};
|
||||
|
||||
// Measure one frame: how far down the cavity the bright region reaches, as a
|
||||
// fraction of cavity height, plus how much of the cavity is bright at all.
|
||||
export function measureInterior(img, innerNorm, ctx) {
|
||||
let x0 = 1, y0 = 1, x1 = 0, y1 = 0;
|
||||
for (const p of innerNorm) {
|
||||
x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y);
|
||||
x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y);
|
||||
}
|
||||
const W = img.naturalWidth, H = img.naturalHeight;
|
||||
const px0 = Math.max(0, Math.floor(x0 * W)), py0 = Math.max(0, Math.floor(y0 * H));
|
||||
const pw = Math.min(W - px0, Math.ceil((x1 - x0) * W)), ph = Math.min(H - py0, Math.ceil((y1 - y0) * H));
|
||||
if (pw < 3 || ph < 3) return { teethT: 0, coverage: 0 };
|
||||
|
||||
ctx.canvas.width = pw; ctx.canvas.height = ph;
|
||||
ctx.drawImage(img, px0, py0, pw, ph, 0, 0, pw, ph);
|
||||
const data = ctx.getImageData(0, 0, pw, ph).data;
|
||||
|
||||
const poly = innerNorm.map((p) => ({ x: p.x * W - px0, y: p.y * H - py0 }));
|
||||
const hist = new Uint32Array(256);
|
||||
const lum = new Float32Array(pw * ph);
|
||||
const mask = new Uint8Array(pw * ph);
|
||||
let n = 0;
|
||||
for (let y = 0; y < ph; y++) {
|
||||
for (let x = 0; x < pw; x++) {
|
||||
if (!pointInPoly(poly, x + 0.5, y + 0.5)) continue;
|
||||
const o = (y * pw + x) * 4;
|
||||
const l = (0.299 * data[o] + 0.587 * data[o + 1] + 0.114 * data[o + 2]) | 0;
|
||||
const i = y * pw + x;
|
||||
lum[i] = l; mask[i] = 1; hist[l]++; n++;
|
||||
}
|
||||
}
|
||||
if (n < 12) return { teethT: 0, coverage: 0 };
|
||||
|
||||
const thr = otsu(hist, n);
|
||||
// Rows are scanned from the top of the cavity; the teeth line is where the
|
||||
// bright run stops. Requiring a run rather than a single row rejects specular
|
||||
// speckle on a wet lower lip.
|
||||
let bright = 0, lastBrightRow = -1, run = 0;
|
||||
for (let y = 0; y < ph; y++) {
|
||||
let rowIn = 0, rowBright = 0;
|
||||
for (let x = 0; x < pw; x++) {
|
||||
const i = y * pw + x;
|
||||
if (!mask[i]) continue;
|
||||
rowIn++;
|
||||
if (lum[i] > thr) { rowBright++; bright++; }
|
||||
}
|
||||
if (rowIn >= 2 && rowBright / rowIn > 0.45) { run++; if (run >= 1) lastBrightRow = y; }
|
||||
else run = 0;
|
||||
}
|
||||
return {
|
||||
teethT: lastBrightRow < 0 ? 0 : (lastBrightRow + 1) / ph,
|
||||
coverage: bright / n,
|
||||
};
|
||||
}
|
||||
|
||||
// Sutherland-Hodgman against the half-plane y <= limit.
|
||||
export function clipPolyAbove(pts, limit) {
|
||||
const out = [];
|
||||
for (let i = 0; i < pts.length; i++) {
|
||||
const a = pts[i], b = pts[(i + 1) % pts.length];
|
||||
const ain = a.y <= limit, bin = b.y <= limit;
|
||||
if (ain) out.push(a);
|
||||
if (ain !== bin) {
|
||||
const t = (limit - a.y) / (b.y - a.y);
|
||||
out.push({ x: a.x + t * (b.x - a.x), y: limit });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@ import { fitSimilarity, applySim, procrustesMean, smoothTransforms } from './mat
|
|||
import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js';
|
||||
import { IndexedRaster, hexToRgb } from './raster.js';
|
||||
import { writeTake } from './take.js';
|
||||
import { clipPolyAbove } from './interior.js';
|
||||
import { synthDense } from './synth.js';
|
||||
|
||||
const results = [];
|
||||
|
|
@ -181,6 +182,27 @@ export function run() {
|
|||
ok('palette expansion introduces no intermediate colours',
|
||||
[...seen].every((c) => allowed.has(c)), `${seen.size} distinct colours`);
|
||||
|
||||
// teeth band: the inner ring clipped to a height. The point of doing it this
|
||||
// way is that the silhouette is the mouth's own shape, so it can never
|
||||
// disagree with the lips - assert that rather than the pixel measurement.
|
||||
{
|
||||
const ring = [{ x: 0, y: 0 }, { x: 10, y: 0 }, { x: 10, y: 10 }, { x: 0, y: 10 }];
|
||||
const half = clipPolyAbove(ring, 5);
|
||||
ok('clip at mid height halves the box',
|
||||
half.length === 4 && Math.max(...half.map((p) => p.y)) === 5,
|
||||
`${half.length} pts, maxY ${Math.max(...half.map((p) => p.y))}`);
|
||||
ok('clip above everything keeps the ring', clipPolyAbove(ring, 99).length === 4);
|
||||
ok('clip below everything empties it', clipPolyAbove(ring, -1).length === 0);
|
||||
const w = clipPolyAbove(ring, 5);
|
||||
ok('clipped band keeps the ring width',
|
||||
Math.min(...w.map((p) => p.x)) === 0 && Math.max(...w.map((p) => p.x)) === 10);
|
||||
// a non-convex ring must not gain or lose x-extent from clipping
|
||||
const tri = [{ x: 0, y: 0 }, { x: 8, y: 2 }, { x: 4, y: 9 }];
|
||||
const ct = clipPolyAbove(tri, 4);
|
||||
ok('clip of a non-convex-ish ring stays within its x range',
|
||||
Math.min(...ct.map((p) => p.x)) >= 0 && Math.max(...ct.map((p) => p.x)) <= 8);
|
||||
}
|
||||
|
||||
// take writer round-trip
|
||||
const take = {
|
||||
name: 'test', frames: 72, width: 320, height: 200, exposure: 2,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue