arthur/js/app.js

import { FaceLandmarker, FilesetResolver } from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/vision_bundle.mjs';
import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js';
import { stabilize, toRasterRing, smoothContours, suggestPlateFrames, heldFrame, shiftIndex } from './pipeline.js';
import { IndexedRaster } from './raster.js';
import { drawRegistered, posterizeInto } from './underlay.js';
import { extractTeeth } from './interior.js';
import { applySim } from './mathutil.js';
import { writeTake } from './take.js';
import { synthDense } from './synth.js';

const RW = 320, RH = 200, ZOOM = 2, THUMB = 92;

const PALETTE = [
  { name: 'bg',         hex: '#12141c' },
  { name: 'skin_base',  hex: '#b07a5a' },
  { name: 'skin_dark',  hex: '#7a4f3a' },
  { name: 'mouth_dark', hex: '#24161a' },
  { name: 'teeth',      hex: '#d9cfc2' },
];
const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, teeth: 4 };

const state = {
  dense: null, images: [], stab: null, xform: null,
  outer: null, inner: null, plates: null, hidden: null,
  keep: new Set(),          // frames that get their own plate drawing
  frame: 0, playing: false, faceBox: null,
  fps: 12, audio: null,     // fps comes from manifest.json, never guessed
  aspect: 1,                // imgW/imgH; converts MediaPipe's anisotropic space
  lead: 0,                  // performance-track offset in frames
  interior: null,           // per-frame teeth measurement from image content
  teeth: null,              // resolved per-frame {show, t} after knobs
};

const el = (id) => {
  const n = document.getElementById(id);
  // A knob present in the code but missing from the markup used to throw during
  // wiring and leave a blank page with nothing in the console worth reading.
  if (!n) throw new Error(`missing element #${id} — knob wired in app.js but not in index.html`);
  return n;
};
const opts = () => ({
  verts: +el('verts').value,
  lead: +el('lead').value,
  teethOn: +el('teethOn').value / 100,       // minimum Otsu class separation
  teethDwell: +el('teethDwell').value,
  teethSmooth: +el('teethSmooth').value,
  cavityErode: +el('teethErode').value / 100,
  tongueReject: +el('tongueReject').value / 100,
  blobGrow: +el('blobGrow').value,
  topBias: +el('topBias').value / 100,
  teethVerts: +el('teethVerts').value,
  smoothWin: +el('smoothWin').value,
  contourSmooth: +el('contourSmooth').value,
  apertureThresh: +el('apertureThresh').value / 1000,
  tol: +el('tol').value / 1000,
});

function status(msg, kind = '') {
  el('status').textContent = msg;
  el('status').className = kind;
}

/* ---------- loading ---------- */

const loadImage = (src) => new Promise((r) => {
  const im = new Image();
  im.onload = () => r(im); im.onerror = () => r(null); im.src = src;
});

// The extraction rate is read, not assumed. Guessing it would desynchronise
// audio from picture, which is the one thing this view exists to show.
async function loadManifest() {
  try {
    const r = await fetch('./manifest.json', { cache: 'no-store' });
    if (!r.ok) return null;
    return await r.json();
  } catch { return null; }
}

function attachAudio(name) {
  const a = el('audio');
  if (!name) { a.removeAttribute('src'); a.hidden = true; state.audio = null; return; }
  a.src = './' + name;
  a.hidden = false;
  state.audio = a;
}

async function loadFrameSequence() {
  const dir = el('framedir').value.replace(/\/$/, '');
  const imgs = [];
  for (let i = 1; i <= 900; i++) {
    const im = await loadImage(`${dir}/${String(i).padStart(4, '0')}.png`);
    if (!im) break;
    imgs.push(im);
    if (i % 10 === 0) status(`loading frames… ${i}`);
  }
  return imgs;
}

let landmarker = null;
async function initLandmarker() {
  if (landmarker) return landmarker;
  status('loading MediaPipe wasm…');
  const fileset = await FilesetResolver.forVisionTasks(
    'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/wasm');
  // GPU is faster but unavailable in some contexts; fall back rather than fail.
  for (const delegate of ['GPU', 'CPU']) {
    try {
      landmarker = await FaceLandmarker.createFromOptions(fileset, {
        baseOptions: { modelAssetPath: './face_landmarker.task', delegate },
        runningMode: 'IMAGE', numFaces: 1,
      });
      status(`landmarker ready (${delegate})`);
      return landmarker;
    } catch (e) {
      if (delegate === 'CPU') throw e;
      console.warn('GPU delegate failed, falling back to CPU:', e.message);
    }
  }
}

async function detectAll(images) {
  const lm = await initLandmarker();
  const cv = document.createElement('canvas');
  const dense = [], missing = [];
  for (let i = 0; i < images.length; i++) {
    const im = images[i];
    cv.width = im.naturalWidth; cv.height = im.naturalHeight;
    cv.getContext('2d').drawImage(im, 0, 0);
    const out = lm.detect(cv);
    if (out.faceLandmarks && out.faceLandmarks.length) dense.push(out.faceLandmarks[0]);
    else { missing.push(i); dense.push(dense.length ? dense[dense.length - 1] : null); }
    if (i % 4 === 0) status(`detecting… ${i + 1}/${images.length}`);
  }
  if (dense[0] === null) throw new Error('no face found in the first frame');
  return { dense, missing };
}

// Interior measurement is a function of pixels alone, so it runs once with
// detection and the knobs re-resolve it instantly afterwards.
function measureAll(images, dense, o) {
  const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
  return dense.map((lm, i) =>
    extractTeeth(images[i], LIPS_INNER.map((k) => lm[k]), ctx, o));
}

// Extraction keys on every knob that changes the pixels examined, so the cache
// is keyed on exactly those and a change to anything else stays instant.
const extractKey = (o) =>
  [o.cavityErode, o.tongueReject, o.blobGrow, o.topBias, o.teethVerts].join('|');

/* ---------- build ---------- */

function makeXform(stab, neutral) {
  const oval = stab.oval[neutral];
  let x0 = Infinity, y0 = Infinity, x1 = -Infinity, y1 = -Infinity;
  for (const p of oval) {
    x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y);
    x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y);
  }
  const s = (RH * 0.80) / (y1 - y0);
  const cx = (x0 + x1) / 2, cy = (y0 + y1) / 2;
  return (p) => ({ x: (p.x - cx) * s + RW / 2, y: (p.y - cy) * s + RH / 2 });
}

function rebuild(resetKeep) {
  if (!state.dense) return;
  const o = opts();
  state.lead = o.lead;
  const N = state.dense.length;

  state.stab = stabilize(state.dense, o.smoothWin, state.aspect);

  const ap = state.stab.aperture;
  const head = Math.max(1, Math.floor(N / 4));
  let neutral = 0;
  for (let i = 0; i < head; i++) if (ap[i] < ap[neutral]) neutral = i;
  state.neutral = neutral;
  state.xform = makeXform(state.stab, neutral);

  // Mouth is traced, so it costs nothing: a key on EVERY frame. Only the plate,
  // which a human draws, gets decimated.
  state.outer = smoothContours(
    state.stab.outer.map((r) => toRasterRing(r, LIPS_OUTER, o.verts, state.xform)), o.contourSmooth);
  state.inner = smoothContours(
    state.stab.inner.map((r) => toRasterRing(r, LIPS_INNER, o.verts, state.xform)), o.contourSmooth);

  const apMax = Math.max(...ap);
  state.hidden = ap.map((v) => v / apMax < o.apertureThresh);

  if (state.images.length && state.extractKey !== extractKey(o)) {
    state.interior = measureAll(state.images, state.dense, o);
    state.extractKey = extractKey(o);
  }
  state.teeth = resolveTeeth(o);

  // Plate outline per frame, so a kept frame shows its own head shape.
  state.plates = state.stab.oval.map((r) => r.map(state.xform));

  if (resetKeep || !state.keep.size) {
    state.keep = new Set(Array.from({ length: N }, (_, i) => i));
  }
  // Frame 0 must always be kept: something has to be on screen at the start.
  state.keep.add(0);

  state.faceBox = faceBoxes();
  drawAll();
}

// Face bounding box per frame in image space, for legible strip thumbnails.
function faceBoxes() {
  return state.dense.map((lm) => {
    let x0 = 1, y0 = 1, x1 = 0, y1 = 0;
    for (const i of FACE_OVAL) {
      x0 = Math.min(x0, lm[i].x); y0 = Math.min(y0, lm[i].y);
      x1 = Math.max(x1, lm[i].x); y1 = Math.max(y1, lm[i].y);
    }
    const mx = (x1 - x0) * 0.18, my = (y1 - y0) * 0.14;
    return { x0: x0 - mx, y0: y0 - my, x1: x1 + mx, y1: y1 + my };
  });
}

// Presence gets hysteresis and a minimum dwell, the same treatment plate
// selection gets: a teeth block that blinks on and off for single frames is
// worse than one that is simply absent. Appearing needs a clear signal, staying
// needs only a weak one.
function resolveTeeth(o) {
  const N = state.dense.length;
  if (!state.interior) return new Array(N).fill({ show: false, pts: null });

  const raw = state.interior.map((m, f) =>
    (state.hidden[f] || !m.contour ? 0 : m.contrast));
  const on = o.teethOn, off = o.teethOn * 0.7;
  const shown = new Array(N).fill(false);
  let live = false, since = 0;
  for (let f = 0; f < N; f++) {
    const want = live ? raw[f] > off : raw[f] > on;
    if (want !== live && since >= o.teethDwell) { live = want; since = 0; }
    else since++;
    shown[f] = live && !state.hidden[f] && !!state.interior[f].contour;
  }

  // Into raster space through the same chain the lips take, including the
  // isotropic aspect conversion - a contour in MediaPipe's normalised space is
  // in the same stretched coordinates the landmarks are.
  const toRaster = (pts, f) => {
    const tf = state.stab.transforms[f];
    return pts.map((p) => state.xform(applySim(tf, { x: p.x * state.aspect, y: p.y })));
  };
  const rast = state.interior.map((m, f) => (m.contour ? toRaster(m.contour, f) : null));

  // Radial sampling makes vertex k mean the same direction on every frame, so
  // smoothing across time is well defined and cannot reorder anything.
  const sm = rast.map((pts, f) => {
    if (!pts || !shown[f]) return pts;
    const acc = pts.map(() => ({ x: 0, y: 0 }));
    let c = 0;
    for (let j = f - o.teethSmooth; j <= f + o.teethSmooth; j++) {
      const k = Math.min(N - 1, Math.max(0, j));
      if (!shown[k] || !rast[k] || rast[k].length !== pts.length) continue;
      for (let v = 0; v < pts.length; v++) { acc[v].x += rast[k][v].x; acc[v].y += rast[k][v].y; }
      c++;
    }
    return c ? acc.map((p) => ({ x: p.x / c, y: p.y / c })) : pts;
  });

  return shown.map((show, f) => ({ show, pts: sm[f] }));
}

/* ---------- render ---------- */

// The plate layer has several representations because its job changes: a flat
// shape to judge the mouth against, or a registered photograph to draw over.
// Only the latter is any use as reference art, and the generated oval is only a
// stand-in until a drawing exists.
function renderFrame(f, mode = plateMode()) {
  const r = new IndexedRaster(RW, RH);
  const pf = heldFrame(keptSorted(), f);   // the plate frame on screen

  if (mode === 'posterize' && state.images[pf]) {
    posterizeInto(r, state.images[pf], state.stab.transforms[pf], state.xform,
                  PALETTE.map((p) => p.hex));
  } else {
    r.clear(IDX.bg);
    if (mode === 'oval' || mode === 'oval+photo') r.fillPoly(state.plates[pf], IDX.base);
  }

  const mf = leadIndex(f);                 // performance frame, possibly ahead
  r.fillPoly(state.outer[mf], IDX.dark);   // mouth keeps every frame
  if (!state.hidden[mf]) {
    r.fillPoly(state.inner[mf], IDX.mouth);
    const te = state.teeth[mf];
    if (te.show && te.pts && te.pts.length >= 3) r.fillPoly(te.pts, IDX.teeth);
  }
  return r;
}

const plateMode = () => el('plateMode').value;

// Photo modes composite under the indexed layer, so the flat shapes stay exactly
// as they render while the reference sits behind them.
function compositeRender(canvas, f, zoom) {
  const mode = plateMode();
  const pf = heldFrame(keptSorted(), f);
  const img = state.images[pf];
  const showPhoto = img && (mode === 'photo' || mode === 'photo-dim' || mode === 'oval+photo');

  canvas.width = RW * zoom; canvas.height = RH * zoom;
  const g = canvas.getContext('2d');
  g.fillStyle = PALETTE[IDX.bg].hex;
  g.fillRect(0, 0, canvas.width, canvas.height);

  if (showPhoto) {
    drawRegistered(g, img, state.stab.transforms[pf], state.xform, zoom,
                   mode === 'photo-dim' ? 0.34 : 1);
  }

  const r = renderFrame(f, mode === 'oval+photo' ? 'oval' : (showPhoto ? 'off' : mode));
  const img2 = r.toImageData(PALETTE.map((p) => p.hex), zoom);
  if (showPhoto) {
    // Keep the photo visible wherever the indexed layer is background.
    const bg = PALETTE[IDX.bg].hex.replace('#', '');
    const br = parseInt(bg.slice(0, 2), 16), bgn = parseInt(bg.slice(2, 4), 16), bb = parseInt(bg.slice(4, 6), 16);
    const d = img2.data;
    for (let i = 0; i < d.length; i += 4) {
      if (d[i] === br && d[i + 1] === bgn && d[i + 2] === bb) d[i + 3] = 0;
    }
    const tmp = document.createElement('canvas');
    tmp.width = img2.width; tmp.height = img2.height;
    tmp.getContext('2d').putImageData(img2, 0, 0);
    g.drawImage(tmp, 0, 0);
  } else {
    g.putImageData(img2, 0, 0);
  }
}

const keptSorted = () => [...state.keep].sort((a, b) => a - b);

// Performance tracks can lead the clock.
//
// A centred moving average has no phase lag, so smoothing does not literally
// delay anything - but it blurs onsets, and the visually salient moment of a
// mouth opening moves later even though the mean does not. Animators also draw
// mouth shapes one or two frames ahead of the sound as a matter of course, so
// this is the normal control rather than a correction.
//
// Positive lead = the mouth arrives earlier. Only performance parts shift; the
// head stays with the audio, because it is the mouth that should anticipate.
function leadIndex(f) {
  // Reads a cached scalar, not opts(): this runs once per strip thumbnail, and
  // calling opts() here meant ~14 DOM reads x 74 frames on every redraw.
  return shiftIndex(f, state.lead, state.dense.length);
}

function blit(canvas, raster, zoom) {
  canvas.width = RW * zoom; canvas.height = RH * zoom;
  canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0);
}

function drawAll() {
  drawPanes();
  drawStrip();
  drawWorksheet();
  drawReadout();
}

function drawReadout() {
  const kept = keptSorted();
  const runs = kept.map((k, i) => (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - k);
  const lead = state.lead;
  const teethFrames = state.teeth ? state.teeth.filter((t) => t.show).length : 0;
  el('readout').textContent =
    `${state.dense.length} frames → ${kept.length} drawings · ` +
    `teeth on ${teethFrames}f · ` +
    (lead ? `mouth leads ${lead}f (${(lead / state.fps * 1000).toFixed(0)}ms) · ` : '') +
    `holds ${Math.min(...runs)}–${Math.max(...runs)} frames · ` +
    `neutral f${state.neutral} · residual ` +
    `${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`;
}

function drawPanes() {
  const f = state.frame, kept = keptSorted();
  const pf = heldFrame(kept, f);
  // The mouth frame is always shown, not only when shifted, so the number can be
  // watched diverging from f rather than taken on trust.
  const lead = state.lead;
  el('framelabel').textContent =
    `f ${f} / ${state.dense.length - 1} · ${(f / state.fps).toFixed(2)}s · ` +
    `plate f${pf} · mouth f${leadIndex(f)}` +
    (lead ? ` (${lead > 0 ? '+' : ''}${lead} = ${(lead / state.fps * 1000).toFixed(0)}ms)` : '') +
    (state.keep.has(f) ? ' · KEPT' : ' · held');

  const c1 = el('cv-source'), g1 = c1.getContext('2d');
  c1.width = RW * ZOOM; c1.height = RH * ZOOM;
  g1.fillStyle = '#000'; g1.fillRect(0, 0, c1.width, c1.height);
  const im = state.images[f];
  if (im) {
    const b = state.faceBox[f];
    const sx = b.x0 * im.naturalWidth, sy = b.y0 * im.naturalHeight;
    const sw = (b.x1 - b.x0) * im.naturalWidth, sh = (b.y1 - b.y0) * im.naturalHeight;
    const s = Math.min(c1.width / sw, c1.height / sh);
    const dw = sw * s, dh = sh * s, dx = (c1.width - dw) / 2, dy = (c1.height - dh) / 2;
    g1.drawImage(im, sx, sy, sw, sh, dx, dy, dw, dh);
    const map = (p) => ({ x: dx + (p.x * im.naturalWidth - sx) * s, y: dy + (p.y * im.naturalHeight - sy) * s });
    strokePts(g1, LIPS_OUTER.map((i) => map(state.dense[f][i])), '#4ade80');
    strokePts(g1, LIPS_INNER.map((i) => map(state.dense[f][i])), '#f87171');
  } else {
    g1.fillStyle = '#555'; g1.font = '13px system-ui';
    g1.fillText('synthetic — no source frames', 14, 24);
    const sc = (p) => ({ x: p.x * c1.width, y: p.y * c1.height });
    strokePts(g1, LIPS_OUTER.map((i) => sc(state.dense[f][i])), '#4ade80');
    strokePts(g1, LIPS_INNER.map((i) => sc(state.dense[f][i])), '#f87171');
  }

  const c2 = el('cv-stab'), g2 = c2.getContext('2d');
  c2.width = RW * ZOOM; c2.height = RH * ZOOM;
  g2.fillStyle = '#0d0f16'; g2.fillRect(0, 0, c2.width, c2.height);
  g2.strokeStyle = '#2a2f3e'; g2.lineWidth = 1; g2.beginPath();
  g2.moveTo(c2.width / 2, 0); g2.lineTo(c2.width / 2, c2.height);
  g2.moveTo(0, c2.height / 2); g2.lineTo(c2.width, c2.height / 2); g2.stroke();
  const z = (pts) => pts.map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM }));
  const mf = leadIndex(f);
  strokePts(g2, z(state.plates[pf]), '#3b4a63');
  // With a lead set, the unshifted contour is drawn as a ghost so the offset is
  // something you can see rather than something you have to trust.
  if (mf !== f) strokePts(g2, z(state.outer[f]), '#2f6b46');
  strokePts(g2, z(state.outer[mf]), '#4ade80');
  if (!state.hidden[mf]) strokePts(g2, z(state.inner[mf]), '#f87171');

  compositeRender(el('cv-render'), f, ZOOM);
  drawInteriorDebug(f);
}

// What the teeth measurement actually saw: sampled region, pixels above
// threshold in green, the resolved line in amber. Recomputed for the current
// frame only, so it costs nothing to keep on screen.
function drawInteriorDebug(fRaw) {
  const f = leadIndex(fRaw);
  const host = el('cv-teeth');
  const img = state.images[f];
  if (!img || state.hidden[f]) {
    host.innerHTML = '';
    el('teethinfo').textContent = state.images.length ? 'mouth closed' : 'no source frames';
    return;
  }
  const o = opts();
  const ctx = document.createElement('canvas').getContext('2d', { willReadFrequently: true });
  const m = extractTeeth(img, LIPS_INNER.map((k) => state.dense[f][k]), ctx, o, true);
  host.innerHTML = '';
  if (m.debug) {
    m.debug.style.width = '170px';
    m.debug.style.imageRendering = 'pixelated';
    host.append(m.debug);
  }
  const te = state.teeth[f];
  el('teethinfo').textContent =
    `contrast ${m.contrast.toFixed(3)} / gate ${o.teethOn.toFixed(2)} · ` +
    `area ${m.area}px · ${te.show ? 'SHOWN' : 'hidden'}`;
}

function strokePts(g, pts, color, lw = 1) {
  g.strokeStyle = color; g.lineWidth = lw;
  g.beginPath();
  pts.forEach((p, i) => (i ? g.lineTo(p.x, p.y) : g.moveTo(p.x, p.y)));
  g.closePath(); g.stroke();
}

/* ---------- the frame strip: this is the editing surface ---------- */

function drawStrip() {
  const host = el('strip');
  host.innerHTML = '';
  const N = state.dense.length;
  for (let f = 0; f < N; f++) {
    const cell = document.createElement('div');
    cell.className = 'fr' + (state.keep.has(f) ? ' keep' : ' drop') + (f === state.frame ? ' cur' : '');
    cell.dataset.f = f;

    const cv = document.createElement('canvas');
    const im = state.images[f];
    cv.width = THUMB; cv.height = THUMB;
    const g = cv.getContext('2d');
    g.fillStyle = '#0d0f16'; g.fillRect(0, 0, THUMB, THUMB);
    if (im) {
      const b = state.faceBox[f];
      const sx = b.x0 * im.naturalWidth, sy = b.y0 * im.naturalHeight;
      const sw = (b.x1 - b.x0) * im.naturalWidth, sh = (b.y1 - b.y0) * im.naturalHeight;
      const s = Math.min(THUMB / sw, THUMB / sh);
      g.drawImage(im, sx, sy, sw, sh, (THUMB - sw * s) / 2, (THUMB - sh * s) / 2, sw * s, sh * s);
    } else {
      const r = new IndexedRaster(RW, RH);
      r.clear(IDX.bg); r.fillPoly(state.plates[f], IDX.base);
      const tmp = document.createElement('canvas');
      blit(tmp, r, 1);
      g.drawImage(tmp, 0, 0, RW, RH, 0, 0, THUMB, THUMB * (RH / RW));
    }

    const tag = document.createElement('span');
    tag.textContent = f;
    cell.append(cv, tag);
    cell.onclick = (ev) => {
      seekTo(f);
      if (ev.shiftKey) toggle(f);
      drawAll();
    };
    cell.ondblclick = () => { toggle(f); drawAll(); };
    host.append(cell);
  }
}

function toggle(f) {
  if (f === 0) return;                       // frame 0 always has a drawing
  if (state.keep.has(f)) state.keep.delete(f); else state.keep.add(f);
}

/* ---------- worksheet: the frames a human must draw ---------- */

function drawWorksheet() {
  const host = el('sheet');
  host.innerHTML = '';
  const kept = keptSorted();
  kept.forEach((f, i) => {
    const until = (i + 1 < kept.length ? kept[i + 1] : state.dense.length) - 1;
    const cell = document.createElement('div');
    cell.className = 'cell';
    // Registered, not raw-cropped: the worksheet frame is in raster space, so a
    // drawing traced from it is already aligned to the mouth.
    const cv = document.createElement('canvas');
    compositeRender(cv, f, 1);
    cv.style.width = '176px';
    const cap = document.createElement('span');
    cap.textContent = `f${f}` + (until > f ? ` → ${until}` : '') + `  (${until - f + 1}f)`;
    cell.append(cv, cap);
    cell.onclick = () => { seekTo(f); drawAll(); };
    host.append(cell);
  });
}

/* ---------- export ---------- */

function exportTake() {
  const kept = keptSorted();
  const N = state.dense.length;
  const take = {
    name: el('takename').value || 'line_01',
    frames: N, width: RW, height: RH, exposure: 1, fps: state.fps,
    palette: PALETTE,
    slot: { x: RW / 2, y: RH / 2 },
    parts: [
      // Sparse: one plate key per frame a human draws.
      { name: 'head', kind: 'plate', z: 0, interp: 'hold',
        keys: kept.map((f, i) => ({ f: i, plate: i, src: f })) },
      // Dense: the traced mouth keeps every frame. The lead is baked in here -
      // key f carries the pose from source frame f+lead - so the renderer never
      // needs to know about it.
      { name: 'mouth', kind: 'poly', z: 30, color: 'skin_dark', interp: 'hold',
        keys: state.outer.map((_, f) => ({ f, src: leadIndex(f), pts: state.outer[leadIndex(f)] })) },
      { name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
        keys: state.inner.map((_, f) => {
          const m = leadIndex(f);
          return state.hidden[m] ? { f, hidden: true } : { f, src: m, pts: state.inner[m] };
        }) },
      { name: 'teeth', kind: 'poly', z: 32, color: 'teeth', interp: 'hold', parent: 'mouth_in',
        keys: state.teeth.map((_, f) => {
          const m = leadIndex(f), te = state.teeth[m];
          return te.show && te.pts ? { f, src: m, pts: te.pts } : { f, hidden: true };
        }) },
    ],
  };
  const text = writeTake(take)
    + `\n# plate drawings needed: ${kept.length} of ${N} frames\n`
    + kept.map((f, i) => {
        const until = (i + 1 < kept.length ? kept[i + 1] : N) - 1;
        return `# plate ${i} = source frame ${f}, holds f${f}..${until}`;
      }).join('\n') + '\n';
  const a = document.createElement('a');
  a.href = URL.createObjectURL(new Blob([text], { type: 'text/plain' }));
  a.download = `${take.name}.take`;
  a.click();
  status(`exported — ${kept.length} plate drawings, ${N} mouth frames`, 'ok');
}

/* ---------- wiring ---------- */

async function runFrames() {
  try {
    const man = await loadManifest();
    if (man) {
      state.fps = man.fps;
      el('framedir').value = man.dir || 'frames';
      attachAudio(man.audio);
    } else {
      attachAudio(null);
      status('no manifest.json — assuming 12fps, no audio. Re-run extract.sh', 'warn');
    }
    const images = await loadFrameSequence();
    if (!images.length) {
      status(`no frames in ${el('framedir').value}/ — run extract.sh first`, 'err');
      return;
    }
    const { dense, missing } = await detectAll(images);
    state.images = images; state.dense = dense;
    state.aspect = images[0].naturalWidth / images[0].naturalHeight;
    status('measuring mouth interiors…');
    state.interior = measureAll(images, dense, opts());
    el('scrub').max = dense.length - 1;
    state.frame = 0;
    rebuild(true);
    const dur = (dense.length / state.fps).toFixed(2);
    status(`${images.length} frames · ${images[0].naturalWidth}x${images[0].naturalHeight} · ` +
      `${state.fps}fps · ${dur}s` +
      (state.audio ? ' · audio loaded' : ' · no audio') +
      (missing.length ? ` · no face on ${missing.length} (held previous)` : ''),
      missing.length ? 'warn' : 'ok');
  } catch (e) { status(e.message, 'err'); console.error(e); }
}

function runSynthetic() {
  state.images = [];
  attachAudio(null);
  state.fps = 12;
  state.aspect = 1;         // synthetic landmarks are generated square
  state.lead = 0;
  state.interior = null;    // no pixels, so no teeth

  state.dense = synthDense(72);
  el('scrub').max = 71;
  state.frame = 0;
  rebuild(true);
  status('synthetic — exercises everything below detection', 'ok');
}

for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol',
                  'teethOn', 'teethDwell', 'teethErode', 'tongueReject', 'blobGrow',
                  'topBias', 'teethVerts', 'teethSmooth', 'lead']) {
  el(id).addEventListener('input', () => {
    el(id + 'v').textContent = id === 'apertureThresh' || id === 'tol'
      ? (+el(id).value / 1000).toFixed(3)
      : ['teethOn', 'teethErode', 'tongueReject', 'topBias'].includes(id)
        ? (+el(id).value / 100).toFixed(2)
      : id === 'lead' && +el(id).value > 0 ? `+${el(id).value}`
      : el(id).value;
    if (id === 'tol') return;                 // tol only matters when you ask for a suggestion
    rebuild(false);
  });
  el(id + 'v').textContent = el(id).value;
}

function seekTo(f) {
  state.frame = f;
  if (state.audio) state.audio.currentTime = f / state.fps;
  el('scrub').value = f;
}

el('scrub').addEventListener('input', (e) => { seekTo(+e.target.value); drawAll(); });
el('btn-frames').onclick = runFrames;
el('btn-synth').onclick = runSynthetic;
el('btn-export').onclick = exportTake;
el('plateMode').addEventListener('change', () => { if (state.dense) drawAll(); });
el('btn-saveframe').onclick = () => {
  if (!state.dense) return;
  const cv = document.createElement('canvas');
  compositeRender(cv, state.frame, 4);      // 1280x800: enough to draw on
  const a = document.createElement('a');
  a.href = cv.toDataURL('image/png');
  a.download = `${el('takename').value || 'take'}_f${String(state.frame).padStart(4, '0')}.png`;
  a.click();
  status(`saved registered frame f${state.frame} at 4x`, 'ok');
};
el('btn-keepall').onclick = () => { if (state.dense) { rebuild(true); } };
el('btn-suggest').onclick = () => {
  if (!state.dense) return;
  const kept = suggestPlateFrames(state.stab.rigid, opts().tol);
  state.keep = new Set(kept);
  drawAll();
  status(`suggested ${kept.length} drawings at tolerance ${opts().tol.toFixed(3)} — now hand-correct`, 'ok');
};
el('btn-play').onclick = () => {
  if (!state.dense) return;
  state.playing = !state.playing;
  el('btn-play').textContent = state.playing ? 'Stop' : 'Play';
  if (state.playing) {
    if (state.audio) {
      state.audio.playbackRate = +el('speed').value;
      // Restart from the top if we are sitting at the end.
      if (state.frame >= state.dense.length - 1) state.frame = 0;
      state.audio.currentTime = state.frame / state.fps;
      state.audio.play().catch((e) => status('audio blocked: ' + e.message, 'warn'));
    }
    last = 0;
    tick();
  } else if (state.audio) {
    state.audio.pause();
  }
};

el('speed').addEventListener('change', () => {
  // playbackRate retimes the clock, and the picture follows it for free.
  if (state.audio) state.audio.playbackRate = +el('speed').value;
});

// Keyboard is the point: stepping and deleting 37 frames by mouse is miserable.
window.addEventListener('keydown', (e) => {
  if (!state.dense || e.target.tagName === 'INPUT') return;
  const N = state.dense.length;
  if (e.key === 'ArrowRight') { seekTo(Math.min(N - 1, state.frame + 1)); drawAll(); }
  else if (e.key === 'ArrowLeft') { seekTo(Math.max(0, state.frame - 1)); drawAll(); }
  else if (e.key === 'Backspace' || e.key === 'Delete' || e.key === 'x') {
    state.keep.delete(state.frame === 0 ? -1 : state.frame); drawAll();
  } else if (e.key === '[' || e.key === ']') {
    const n = el('lead');
    n.value = Math.max(+n.min, Math.min(+n.max, +n.value + (e.key === ']' ? 1 : -1)));
    state.lead = +n.value;
    el('leadv').textContent = state.lead > 0 ? `+${state.lead}` : String(state.lead);
    drawAll();
  } else if (e.key === 'b') {
    const sel = el('plateMode');
    sel.selectedIndex = (sel.selectedIndex + 1) % sel.options.length;
    drawAll();
  } else if (e.key === 'k' || e.key === ' ') {
    if (state.frame !== 0) state.keep.add(state.frame);
    drawAll();
  } else return;
  e.preventDefault();
});

let last = 0;
function tick(ts = 0) {
  if (!state.playing) return;
  const N = state.dense.length;
  let f;
  if (state.audio) {
    // Audio is the clock. Deriving the frame from currentTime rather than
    // counting means a slow render loop drops frames instead of drifting out
    // of sync, which is the behaviour you want when judging lip sync.
    f = Math.floor(state.audio.currentTime * state.fps);
    if (f >= N || state.audio.ended) {
      state.audio.currentTime = 0;
      state.audio.play().catch(() => {});
      f = 0;
    }
  } else {
    const rate = state.fps * +el('speed').value;
    if (ts - last < 1000 / rate) { requestAnimationFrame(tick); return; }
    last = ts;
    f = (state.frame + 1) % N;
  }
  if (f !== state.frame) {
    state.frame = f;
    el('scrub').value = f;
    drawPanes(); drawStrip();
  }
  requestAnimationFrame(tick);
}

PALETTE.forEach((p) => {
  const sw = document.createElement('label');
  sw.className = 'sw';
  const inp = document.createElement('input');
  inp.type = 'color'; inp.value = p.hex;
  inp.oninput = () => { p.hex = inp.value; if (state.dense) drawAll(); };
  sw.append(inp, document.createTextNode(p.name));
  el('palette').append(sw);
});

// #synth / #frames autorun, so the tool can be driven headlessly for smoke tests
// and deep-linked. Detection needs WebGL; the synthetic path does not.
window.addEventListener('error', (e) => {
  const s = document.getElementById('status');
  if (s) { s.textContent = e.message; s.className = 'err'; }
});

if (location.hash === '#synth') runSynthetic();
else if (location.hash === '#frames') runFrames();
else status('ready — Load frames, then step with \u2190 \u2192 and delete with X');

window.__roto = state;   // headless smoke test reads this