arthur/js/synth.js

// Synthetic landmark frames, shaped exactly like FaceLandmarker output.
//
// Exists so the whole chain downstream of detection - Procrustes, smoothing,
// stabilisation, key selection, rasterising, take writing - can be exercised and
// verified without a video file. A synthetic face is also the only way to test
// stabilisation against a KNOWN head motion, since real footage gives no ground
// truth to compare against.
import { LIPS_OUTER, LIPS_INNER, FACE_OVAL, RIGID,
         EYE_R_RING, EYE_L_RING, IRIS_A, IRIS_B } from './landmarks.js';

const NUM = 478;

// `swapIris` places the two iris blocks on the opposite eyes. It exists so the
// pairing resolver can be tested against a track it actually disagrees with:
// a resolver checked only against the convention it was written for is checking
// nothing at all.
export function synthDense(nFrames = 72, { swapIris = false } = {}) {
  const frames = [];
  for (let t = 0; t < nFrames; t++) {
    const pts = new Array(NUM);
    for (let i = 0; i < NUM; i++) pts[i] = { x: 0.5, y: 0.5, z: 0 };

    // Known head motion: drift, sway, roll and a slow scale change, plus a
    // little per-frame jitter so transform smoothing has something to remove.
    const ph = t / nFrames;
    const hx = 0.5 + 0.045 * Math.sin(ph * Math.PI * 2) + (Math.random() - 0.5) * 0.002;
    const hy = 0.5 + 0.02 * Math.cos(ph * Math.PI * 3) + (Math.random() - 0.5) * 0.002;
    const roll = 0.18 * Math.sin(ph * Math.PI * 2.5);
    const scale = 1 + 0.06 * Math.sin(ph * Math.PI * 1.5);
    const cr = Math.cos(roll), sr = Math.sin(roll);
    const place = (i, lx, ly) => {
      const sx = lx * scale, sy = ly * scale;
      pts[i] = { x: hx + cr * sx - sr * sy, y: hy + sr * sx + cr * sy, z: 0 };
    };

    // Mouth opens in four sustained beats with holds between, so key selection
    // has genuine extremes and genuine plateaux to find.
    const beat = Math.floor(t / 9) % 4;
    const target = [0.004, 0.05, 0.022, 0.0]; 
    const openAmt = target[beat];
    const wide = 0.10 + (beat === 1 ? 0.012 : beat === 3 ? -0.008 : 0);

    place(RIGID[4],  0.000, -0.050); place(RIGID[5],  0.000, -0.020);
    place(RIGID[6],  0.000,  0.012);

    // Eyes. The corners (RIGID[0..3]) are placed BY the lid rings rather than
    // separately, because they are slots 0 and 8 of those rings: writing them
    // twice is how the mouth grew a bowtie, and a corner that disagrees with
    // its own ring would make the eye self-intersect at some vertex budgets
    // and not others.
    //
    // A blink is ONE frame, which is the honest hard case: at 12fps that is
    // what a real blink costs, and it is exactly the length that reads as a
    // dropped frame rather than as a blink unless `hold` extends it.
    const blink = t > 5 && t % 19 === 0;
    const openness = blink ? 0.05 : 1;

    // Gaze holds and then jumps, the way gaze actually behaves, with a little
    // jitter on top so quantisation has noise to remove and the dwell has
    // something to suppress.
    const LOOK = [[0, 0], [0.16, 0.0], [-0.16, 0.05], [0.0, -0.09]];
    const [gx, gy] = LOOK[Math.floor(t / 11) % LOOK.length];
    const jit = () => (Math.random() - 0.5) * 0.012;

    // Half the corner separation, and the lid half-height at full open.
    const EYE_RX = 0.0235, EYE_RY = 0.011, EYE_Y = -0.044;
    const eye = (ring, cx, dir, iris) => {
      const n = ring.length;
      for (let k = 0; k < n; k++) {
        // dir flips the traversal so each ring runs the direction its real
        // table does: slot 0 outer corner, 4 upper lid, 8 inner, 12 lower.
        const a = dir > 0 ? Math.PI + (k / n) * Math.PI * 2 : -(k / n) * Math.PI * 2;
        place(ring[k], cx + EYE_RX * Math.cos(a),
                       EYE_Y + EYE_RY * openness * Math.sin(a));
      }
      // Iris: centre first, then four ring points, as the refined mesh emits.
      const ix = cx + (gx + jit()) * EYE_RX * 2, iy = EYE_Y + (gy + jit()) * EYE_RX * 2;
      place(iris[0], ix, iy);
      for (let k = 1; k < iris.length; k++) {
        const a = ((k - 1) / (iris.length - 1)) * Math.PI * 2;
        place(iris[k], ix + 0.008 * Math.cos(a), iy + 0.008 * Math.sin(a));
      }
    };
    eye(EYE_R_RING, -0.0515,  1, swapIris ? IRIS_B : IRIS_A);
    eye(EYE_L_RING,  0.0515, -1, swapIris ? IRIS_A : IRIS_B);

    // Lip rings as ellipse arcs, traversed so ring ORDER matches the tables:
    // slot 0 = right corner, 5 = top centre, 10 = left corner, 15 = bottom
    // centre, with y growing downward. Getting this convention wrong swaps two
    // opposite vertices and the ring self-intersects into a bowtie - see the
    // ring-simplicity assertion in selftest.
    const ring = (table, rx, ry, cy) => {
      const n = table.length;
      for (let k = 0; k < n; k++) {
        const a = -(k / n) * Math.PI * 2;
        place(table[k], rx * Math.cos(a), cy + ry * Math.sin(a));
      }
    };
    ring(LIPS_OUTER, wide / 2, 0.012 + openAmt * 0.6, 0.075);
    // APERTURE (13, 14) are slots 5 and 15 of the inner ring, so the ring itself
    // places them at the vertical extremes. Writing them again afterwards is what
    // produced the bowtie; the aperture is simply the inner ring's height.
    ring(LIPS_INNER, wide / 2.6, 0.001 + openAmt, 0.075);

    for (let k = 0; k < FACE_OVAL.length; k++) {
      const a = -Math.PI / 2 + (k / FACE_OVAL.length) * Math.PI * 2;
      place(FACE_OVAL[k], 0.105 * Math.cos(a), 0.145 * Math.sin(a) + 0.01);
    }
    frames.push(pts);
  }
  return frames;
}