arthur/js/app.js

import { FaceLandmarker, FilesetResolver } from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/vision_bundle.mjs';
import { LIPS_OUTER, LIPS_INNER, FACE_OVAL } from './landmarks.js';
import { stabilize, toRasterRing, selectKeys, activeKey } from './pipeline.js';
import { IndexedRaster } from './raster.js';
import { writeTake } from './take.js';
import { synthDense } from './synth.js';

const RW = 320, RH = 200, ZOOM = 2;

const PALETTE = [
  { name: 'bg',         hex: '#12141c' },
  { name: 'skin_base',  hex: '#b07a5a' },
  { name: 'skin_dark',  hex: '#7a4f3a' },
  { name: 'mouth_dark', hex: '#24161a' },
  { name: 'skin_lite',  hex: '#d9a884' },
];
const IDX = { bg: 0, base: 1, dark: 2, mouth: 3, lite: 4 };

const state = {
  dense: null, images: [], stab: null, xform: null,
  plate: null, neutral: 0, keysOuter: null, keysInner: null,
  frame: 0, playing: false, source: 'none',
};

const el = (id) => document.getElementById(id);
const opts = () => ({
  verts: +el('verts').value,
  minHold: +el('minHold').value,
  distThresh: +el('distThresh').value / 10,
  velSmooth: +el('velSmooth').value,
  smoothWin: +el('smoothWin').value,
  exposure: +el('exposure').value,
  apertureThresh: +el('apertureThresh').value / 1000,
});

function status(msg, kind = '') {
  const s = el('status');
  s.textContent = msg;
  s.className = kind;
}

/* ---------- frame loading ---------- */

function loadImage(src) {
  return new Promise((res) => {
    const im = new Image();
    im.onload = () => res(im);
    im.onerror = () => res(null);
    im.src = src;
  });
}

async function loadFrameSequence() {
  const dir = el('framedir').value.replace(/\/$/, '');
  const imgs = [];
  for (let i = 1; i <= 900; i++) {
    const name = `${dir}/${String(i).padStart(4, '0')}.png`;
    const im = await loadImage(name);
    if (!im) break;
    imgs.push(im);
    if (i % 10 === 0) status(`loading frames… ${i}`);
  }
  return imgs;
}

/* ---------- detection ---------- */

let landmarker = null;

async function initLandmarker() {
  if (landmarker) return landmarker;
  status('loading MediaPipe wasm…');
  const fileset = await FilesetResolver.forVisionTasks(
    'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/wasm');
  landmarker = await FaceLandmarker.createFromOptions(fileset, {
    baseOptions: { modelAssetPath: './face_landmarker.task', delegate: 'GPU' },
    runningMode: 'IMAGE',
    numFaces: 1,
  });
  return landmarker;
}

async function detectAll(images) {
  const lm = await initLandmarker();
  const cv = document.createElement('canvas');
  const dense = [];
  const missing = [];
  for (let i = 0; i < images.length; i++) {
    const im = images[i];
    cv.width = im.naturalWidth; cv.height = im.naturalHeight;
    cv.getContext('2d').drawImage(im, 0, 0);
    const out = lm.detect(cv);
    if (out.faceLandmarks && out.faceLandmarks.length) {
      dense.push(out.faceLandmarks[0]);
    } else {
      // Hold the previous frame rather than dropping it, so frame indices stay
      // aligned with the source sequence. A gap is reported, not hidden.
      missing.push(i);
      dense.push(dense.length ? dense[dense.length - 1] : null);
    }
    if (i % 5 === 0) status(`detecting… ${i + 1}/${images.length}`);
  }
  if (dense[0] === null) throw new Error('no face found in the first frame');
  return { dense, missing };
}

/* ---------- calibration ---------- */

// v1 calibration: fit the reference face oval's bounding box to a target box on
// the character raster. Identity retargeting - it makes any clip frame sensibly,
// but a real project replaces this with a measured performer->character map.
function makeXform(stab) {
  const oval = stab.oval[state.neutral];
  let x0 = Infinity, y0 = Infinity, x1 = -Infinity, y1 = -Infinity;
  for (const p of oval) {
    x0 = Math.min(x0, p.x); y0 = Math.min(y0, p.y);
    x1 = Math.max(x1, p.x); y1 = Math.max(y1, p.y);
  }
  const targetH = RH * 0.80;
  const s = targetH / (y1 - y0);
  const cx = (x0 + x1) / 2, cy = (y0 + y1) / 2;
  return (p) => ({ x: (p.x - cx) * s + RW / 2, y: (p.y - cy) * s + RH / 2 });
}

/* ---------- build ---------- */

function rebuild() {
  if (!state.dense) return;
  const o = opts();

  state.stab = stabilize(state.dense, o.smoothWin);

  // Neutral frame = most closed mouth in the first quarter of the shot, which is
  // where the performer is asked to hold a neutral closed mouth.
  const ap = state.stab.aperture;
  const head = Math.max(1, Math.floor(ap.length / 4));
  let best = 0;
  for (let i = 0; i < head; i++) if (ap[i] < ap[best]) best = i;
  state.neutral = best;

  state.xform = makeXform(state.stab);
  state.plate = state.stab.oval[state.neutral].map(state.xform);

  const outer = state.stab.outer.map((r) => toRasterRing(r, LIPS_OUTER, o.verts, state.xform));
  const inner = state.stab.inner.map((r) => toRasterRing(r, LIPS_INNER, o.verts, state.xform));

  state.shapesOuter = outer;
  state.shapesInner = inner;
  state.keysOuter = selectKeys(outer, o);
  // The interior is a child of the lip silhouette: it keys on exactly its
  // parent's frames, never independently, or it swims.
  state.keysInner = {
    ...state.keysOuter,
    keys: state.keysOuter.keys.map((k) => ({ ...k })),
  };

  const apMax = Math.max(...ap);
  state.hidden = state.keysOuter.keys.map((k) => ap[k.src] / apMax < o.apertureThresh);

  el('readout').textContent =
    `${state.dense.length} frames · ${state.keysOuter.candidates.length} candidates → ` +
    `${state.keysOuter.keys.length} keys · ${o.verts} verts · neutral f${state.neutral} · ` +
    `residual ${(state.stab.residual.reduce((a, b) => a + b, 0) / state.dense.length).toFixed(4)}`;

  drawTimeline();
  drawContactSheet();
  draw();
}

/* ---------- drawing ---------- */

function renderFrame(f) {
  const r = new IndexedRaster(RW, RH);
  r.clear(IDX.bg);
  if (state.plate) r.fillPoly(state.plate, IDX.base);

  const ko = activeKey(state.keysOuter.keys, f);
  const ki = activeKey(state.keysInner.keys, f);
  if (ko) r.fillPoly(state.shapesOuter[ko.src], IDX.dark);
  if (ki) {
    const slot = state.keysOuter.keys.indexOf(ki);
    if (!state.hidden[slot]) r.fillPoly(state.shapesInner[ki.src], IDX.mouth);
  }
  return r;
}

function blit(canvas, raster, zoom) {
  canvas.width = RW * zoom; canvas.height = RH * zoom;
  canvas.getContext('2d').putImageData(raster.toImageData(PALETTE.map((p) => p.hex), zoom), 0, 0);
}

function draw() {
  const f = state.frame;
  el('framelabel').textContent =
    `f ${String(f).padStart(3)} / ${state.dense.length - 1}   ` +
    `key ${activeKey(state.keysOuter.keys, f).f}`;

  // pane 1: source with the raw contour overlaid
  const c1 = el('cv-source'), g1 = c1.getContext('2d');
  c1.width = RW * ZOOM; c1.height = RH * ZOOM;
  g1.fillStyle = '#000'; g1.fillRect(0, 0, c1.width, c1.height);
  const im = state.images[f];
  if (im) {
    const s = Math.min(c1.width / im.naturalWidth, c1.height / im.naturalHeight);
    const w = im.naturalWidth * s, h = im.naturalHeight * s;
    g1.drawImage(im, (c1.width - w) / 2, (c1.height - h) / 2, w, h);
    g1.save();
    g1.translate((c1.width - w) / 2, (c1.height - h) / 2);
    strokeRing(g1, LIPS_OUTER.map((i) => state.dense[f][i]), w, h, '#4ade80');
    strokeRing(g1, LIPS_INNER.map((i) => state.dense[f][i]), w, h, '#f87171');
    g1.restore();
  } else {
    g1.fillStyle = '#555'; g1.font = '13px system-ui';
    g1.fillText('synthetic — no source frames', 14, 24);
    strokeRing(g1, LIPS_OUTER.map((i) => state.dense[f][i]), c1.width, c1.height, '#4ade80');
    strokeRing(g1, LIPS_INNER.map((i) => state.dense[f][i]), c1.width, c1.height, '#f87171');
  }

  // pane 2: stabilised contour against a fixed reference cross.
  // If stabilisation works, this contour stays put except for mouth motion.
  const c2 = el('cv-stab'), g2 = c2.getContext('2d');
  c2.width = RW * ZOOM; c2.height = RH * ZOOM;
  g2.fillStyle = '#0d0f16'; g2.fillRect(0, 0, c2.width, c2.height);
  g2.strokeStyle = '#2a2f3e'; g2.lineWidth = 1;
  g2.beginPath();
  g2.moveTo(c2.width / 2, 0); g2.lineTo(c2.width / 2, c2.height);
  g2.moveTo(0, c2.height / 2); g2.lineTo(c2.width, c2.height / 2);
  g2.stroke();
  const pl = (pts) => pts.map((p) => { const q = state.xform(p); return { x: q.x * ZOOM, y: q.y * ZOOM }; });
  strokePts(g2, pl(state.stab.oval[f]), '#3b4a63');
  strokePts(g2, pl(state.stab.outer[f]), '#4ade80');
  strokePts(g2, pl(state.stab.inner[f]), '#f87171');
  const ko = activeKey(state.keysOuter.keys, f);
  strokePts(g2, state.shapesOuter[ko.src].map((p) => ({ x: p.x * ZOOM, y: p.y * ZOOM })), '#fbbf24', 2);

  // pane 3: the flat indexed render
  blit(el('cv-render'), renderFrame(f), ZOOM);
}

function strokeRing(g, pts, w, h, color) {
  strokePts(g, pts.map((p) => ({ x: p.x * w, y: p.y * h })), color);
}

function strokePts(g, pts, color, lw = 1) {
  g.strokeStyle = color; g.lineWidth = lw;
  g.beginPath();
  pts.forEach((p, i) => (i ? g.lineTo(p.x, p.y) : g.moveTo(p.x, p.y)));
  g.closePath(); g.stroke();
}

function drawTimeline() {
  const c = el('cv-timeline'), g = c.getContext('2d');
  const N = state.dense.length;
  const W = Math.max(640, N * 8), H = 90;
  c.width = W; c.height = H;
  const px = W / N;
  g.fillStyle = '#0d0f16'; g.fillRect(0, 0, W, H);

  const vel = state.keysOuter.velocity;
  const vmax = Math.max(...vel) || 1;
  g.strokeStyle = '#3b4a63'; g.lineWidth = 1; g.beginPath();
  vel.forEach((v, i) => {
    const x = i * px + px / 2, y = H - 18 - (v / vmax) * (H - 34);
    i ? g.lineTo(x, y) : g.moveTo(x, y);
  });
  g.stroke();

  g.fillStyle = '#5b6478';
  for (const t of state.keysOuter.candidates) g.fillRect(t * px + px / 2 - 0.5, H - 18, 1, 6);

  for (const k of state.keysOuter.keys) {
    g.fillStyle = '#fbbf24';
    g.fillRect(k.f * px + px / 2 - 1, 6, 2, H - 24);        // f: what renders
    if (k.src !== k.f) {
      g.fillStyle = '#f87171';
      g.fillRect(k.src * px + px / 2 - 0.5, 6, 1, 10);      // src: the true extreme
    }
  }
  g.fillStyle = '#e8eaf0';
  g.fillRect(state.frame * px + px / 2 - 0.5, 0, 1, H);
}

function drawContactSheet() {
  const host = el('sheet');
  host.innerHTML = '';
  state.keysOuter.keys.forEach((k, i) => {
    const cell = document.createElement('div');
    cell.className = 'cell';
    const cv = document.createElement('canvas');
    blit(cv, renderFrame(k.f), 1);
    cv.style.width = '104px';
    const cap = document.createElement('span');
    cap.textContent = `f${k.f}${k.src !== k.f ? ` ←${k.src}` : ''}${state.hidden[i] ? ' ·closed' : ''}`;
    cell.append(cv, cap);
    cell.onclick = () => { state.frame = k.f; draw(); drawTimeline(); };
    host.append(cell);
  });
}

/* ---------- export ---------- */

function exportTake() {
  const keys = state.keysOuter.keys;
  const take = {
    name: el('takename').value || 'line_01',
    frames: state.dense.length, width: RW, height: RH,
    exposure: opts().exposure,
    palette: PALETTE,
    slot: { x: RW / 2, y: RH / 2 },
    parts: [
      { name: 'head', kind: 'plate', z: 0, interp: 'hold', keys: [{ f: 0, plate: 0 }] },
      { name: 'mouth', kind: 'poly', z: 30, color: 'skin_dark', interp: 'hold',
        keys: keys.map((k) => ({ f: k.f, src: k.src, pts: state.shapesOuter[k.src] })) },
      { name: 'mouth_in', kind: 'poly', z: 31, color: 'mouth_dark', interp: 'hold', parent: 'mouth',
        keys: keys.map((k, i) => state.hidden[i]
          ? { f: k.f, hidden: true }
          : { f: k.f, src: k.src, pts: state.shapesInner[k.src] }) },
    ],
  };
  const text = writeTake(take);
  const a = document.createElement('a');
  a.href = URL.createObjectURL(new Blob([text], { type: 'text/plain' }));
  a.download = `${take.name}.take`;
  a.click();
  status(`exported ${take.name}.take — ${keys.length} keys`, 'ok');
}

/* ---------- wiring ---------- */

async function runFrames() {
  try {
    status('loading frames…');
    const images = await loadFrameSequence();
    if (!images.length) {
      status(`no frames found in ${el('framedir').value}/ — run extract.sh first`, 'err');
      return;
    }
    const { dense, missing } = await detectAll(images);
    state.images = images; state.dense = dense; state.source = 'video';
    el('scrub').max = dense.length - 1;
    state.frame = 0;
    rebuild();
    status(missing.length
      ? `${images.length} frames · no face on ${missing.length} (held previous)`
      : `${images.length} frames detected`, missing.length ? 'warn' : 'ok');
  } catch (e) {
    status(`${e.message}`, 'err');
    console.error(e);
  }
}

function runSynthetic() {
  state.images = [];
  state.dense = synthDense(72);
  state.source = 'synthetic';
  el('scrub').max = 71;
  state.frame = 0;
  rebuild();
  status('synthetic take — no video needed; exercises the whole chain below detection', 'ok');
}

for (const id of ['verts','minHold','distThresh','velSmooth','smoothWin','exposure','apertureThresh']) {
  el(id).addEventListener('input', () => {
    el(id + 'v').textContent = id === 'distThresh' ? (+el(id).value / 10).toFixed(1)
      : id === 'apertureThresh' ? (+el(id).value / 1000).toFixed(3)
      : el(id).value;
    rebuild();
  });
  el(id + 'v').textContent = el(id).value;
}
el('scrub').addEventListener('input', (e) => { state.frame = +e.target.value; draw(); drawTimeline(); });
el('btn-frames').onclick = runFrames;
el('btn-synth').onclick = runSynthetic;
el('btn-export').onclick = exportTake;
el('btn-play').onclick = () => {
  state.playing = !state.playing;
  el('btn-play').textContent = state.playing ? 'Stop' : 'Play';
  if (state.playing) tick();
};

let last = 0;
function tick(ts = 0) {
  if (!state.playing) return;
  if (ts - last > 1000 / 24) {
    last = ts;
    state.frame = (state.frame + 1) % state.dense.length;
    el('scrub').value = state.frame;
    draw(); drawTimeline();
  }
  requestAnimationFrame(tick);
}

PALETTE.forEach((p) => {
  const sw = document.createElement('label');
  sw.className = 'sw';
  const inp = document.createElement('input');
  inp.type = 'color'; inp.value = p.hex;
  inp.oninput = () => { p.hex = inp.value; if (state.dense) { draw(); drawContactSheet(); } };
  sw.append(inp, document.createTextNode(p.name));
  el('palette').append(sw);
});

status('ready — "Synthetic take" works with no video; "Load frames" reads ./frames/');