Frame removal for plates; mouth keeps every frame

Sparseness was being applied for two different reasons at once. Aesthetic
sparseness is set by the extraction rate; labour sparseness only binds on the
plate, because a human draws each one. The mouth is traced and therefore free,
and in limited animation lip sync is routinely the densest element - on 1s
while heads hold on 2s and 3s.

So: the mouth gets a key on every frame, and the frame strip is now the
editing surface for deciding which frames need their own plate drawing. All
frames start kept; delete the ones you don't want.

- strip of face-cropped thumbnails, keep/drop per frame, keyboard driven
- worksheet panel lists the drawings needed and the range each one holds
- Suggest runs error-tolerance decimation on head pose as a starting point
- export writes sparse plate keys + dense mouth keys, with a hold manifest
- smoothContours: bounded exception to "never smooth the contour", which held
  only while keys were sparse enough to reject detector noise by sampling
- averages are now a RADIUS in frames: 0 is off, 1 is +-1
- GPU delegate falls back to CPU instead of failing
- #synth / #frames autorun for headless smoke tests

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Your Name 2026-09-24 14:51:15 -04:00
parent 7a6bdea157
commit a082208ad5
7 changed files with 489 additions and 265 deletions

79
probe.html Normal file
View file

@ -0,0 +1,79 @@
<!doctype html><html><head><meta charset="utf-8"><title>probe…</title></head>
<body><pre id="o">running…</pre>
<script type="module">
import { FaceLandmarker, FilesetResolver } from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/vision_bundle.mjs';
import { LIPS_OUTER, LIPS_INNER, subsampleSlots } from './js/landmarks.js';
import { stabilize, toRasterRing, selectKeys } from './js/pipeline.js';
const log = [];
const say = (s) => { log.push(s); document.getElementById('o').textContent = log.join('\n'); };
function crosses(a,b,c,d){const o=(p,q,r)=>Math.sign((q.x-p.x)*(r.y-p.y)-(q.y-p.y)*(r.x-p.x));
const o1=o(a,b,c),o2=o(a,b,d),o3=o(c,d,a),o4=o(c,d,b);
return o1!==o2&&o3!==o4&&o1!==0&&o2!==0&&o3!==0&&o4!==0;}
function selfInts(pts){const n=pts.length,h=[];for(let i=0;i<n;i++)for(let j=i+1;j<n;j++){
if((j+1)%n===i||(i+1)%n===j)continue;
if(crosses(pts[i],pts[(i+1)%n],pts[j],pts[(j+1)%n]))h.push([i,j]);}return h;}
const loadImg = (src) => new Promise(r => { const i=new Image(); i.onload=()=>r(i); i.onerror=()=>r(null); i.src=src; });
try {
const imgs=[];
for(let i=1;i<=900;i++){const im=await loadImg(`frames/${String(i).padStart(4,'0')}.png`); if(!im)break; imgs.push(im);}
say(`frames loaded: ${imgs.length} @ ${imgs[0].naturalWidth}x${imgs[0].naturalHeight}`);
const fs = await FilesetResolver.forVisionTasks('https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@1.0.1/wasm');
const lm = await FaceLandmarker.createFromOptions(fs, {
baseOptions:{ modelAssetPath:'./face_landmarker.task', delegate:'CPU' },
runningMode:'IMAGE', numFaces:1 });
say('landmarker ready (CPU delegate)');
const cv=document.createElement('canvas'); const dense=[]; let miss=0;
for(const im of imgs){
cv.width=im.naturalWidth; cv.height=im.naturalHeight;
cv.getContext('2d').drawImage(im,0,0);
const out=lm.detect(cv);
if(out.faceLandmarks?.length) dense.push(out.faceLandmarks[0]);
else { miss++; if(dense.length) dense.push(dense[dense.length-1]); }
}
say(`detected: ${dense.length}/${imgs.length} (no face on ${miss})`);
if(!dense.length) throw new Error('no face detected in any frame');
// THE key check: are LIPS_OUTER / LIPS_INNER correct traversals of real data?
for(const [name,tab] of [['LIPS_OUTER',LIPS_OUTER],['LIPS_INNER',LIPS_INNER]]){
let bad=0, first=null;
for(let n=4;n<=16;n+=2){
const slots=subsampleSlots(tab.length,n);
for(let f=0;f<dense.length;f++){
const h=selfInts(slots.map(s=>dense[f][tab[s]]));
if(h.length){bad++; first=first||`verts=${n} f=${f} edges ${JSON.stringify(h[0])}`;}
}
}
say(`${name}: ${bad===0?'SIMPLE at every budget/frame':`SELF-INTERSECTS ${bad}x first ${first}`}`);
// full 20-ring too
let bad20=0;
for(let f=0;f<dense.length;f++) if(selfInts(tab.map(i=>dense[f][i])).length) bad20++;
say(` full 20-point ring: ${bad20===0?'simple on all frames':`self-intersects on ${bad20} frames`}`);
}
const st = stabilize(dense, 5);
const res = st.residual;
const mean = res.reduce((a,b)=>a+b,0)/res.length;
say(`residual mean ${mean.toFixed(5)} max ${Math.max(...res).toFixed(5)} (high = out-of-plane rotation)`);
const eyeX = st.eyes.map(e=>e[0].x);
const rawX = dense.map(f=>f[133].x);
say(`eye-inner x spread: raw ${(Math.max(...rawX)-Math.min(...rawX)).toFixed(4)} -> stabilised ${(Math.max(...eyeX)-Math.min(...eyeX)).toFixed(4)}`);
const ap=st.aperture;
say(`aperture min ${Math.min(...ap).toFixed(4)} max ${Math.max(...ap).toFixed(4)} range ${(Math.max(...ap)-Math.min(...ap)).toFixed(4)}`);
let nf=0; ap.forEach((v,i)=>{ if(v===Math.min(...ap)) nf=i; });
say(`most-closed frame: ${nf}`);
const xf=p=>({x:p.x*320,y:p.y*200});
const shapes=st.outer.map(r=>toRasterRing(r,LIPS_OUTER,8,xf));
for(const [mh,dt] of [[1,0.6],[2,0.6],[2,1.5],[2,3.0],[3,1.5]]){
const k=selectKeys(shapes,{minHold:mh,distThresh:dt,velSmooth:3,exposure:1});
say(`minHold=${mh} gate=${dt}: ${k.candidates.length} cand -> ${k.keys.length} keys [${k.keys.map(x=>x.f).join(' ')}]`);
}
document.title='PROBE OK';
} catch(e){ say('ERROR: '+e.message+'\n'+e.stack); document.title='PROBE FAIL'; }
</script></body></html>