Audio-clocked playback for judging sync
Lip sync cannot be judged silently. extract.sh now pulls the audio track and writes manifest.json alongside the frames; the page reads the true extraction rate from it rather than assuming one, since a guessed fps desynchronises picture from sound - the one thing this view exists to show. Audio is the clock: frame = floor(currentTime * fps). A slow render loop drops frames instead of drifting, and half/quarter speed work via playbackRate with the picture following for free. Scrubbing, stepping and clicking a thumbnail all seek the audio too. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
a082208ad5
commit
5ac919b9d6
7 changed files with 157 additions and 36 deletions
102
js/app.js
102
js/app.js
|
|
@ -21,6 +21,7 @@ const state = {
|
|||
outer: null, inner: null, plates: null, hidden: null,
|
||||
keep: new Set(), // frames that get their own plate drawing
|
||||
frame: 0, playing: false, faceBox: null,
|
||||
fps: 12, audio: null, // fps comes from manifest.json, never guessed
|
||||
};
|
||||
|
||||
const el = (id) => document.getElementById(id);
|
||||
|
|
@ -44,6 +45,24 @@ const loadImage = (src) => new Promise((r) => {
|
|||
im.onload = () => r(im); im.onerror = () => r(null); im.src = src;
|
||||
});
|
||||
|
||||
// The extraction rate is read, not assumed. Guessing it would desynchronise
|
||||
// audio from picture, which is the one thing this view exists to show.
|
||||
async function loadManifest() {
|
||||
try {
|
||||
const r = await fetch('./manifest.json', { cache: 'no-store' });
|
||||
if (!r.ok) return null;
|
||||
return await r.json();
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
function attachAudio(name) {
|
||||
const a = el('audio');
|
||||
if (!name) { a.removeAttribute('src'); a.hidden = true; state.audio = null; return; }
|
||||
a.src = './' + name;
|
||||
a.hidden = false;
|
||||
state.audio = a;
|
||||
}
|
||||
|
||||
async function loadFrameSequence() {
|
||||
const dir = el('framedir').value.replace(/\/$/, '');
|
||||
const imgs = [];
|
||||
|
|
@ -199,7 +218,7 @@ function drawPanes() {
|
|||
const f = state.frame, kept = keptSorted();
|
||||
const pf = heldFrame(kept, f);
|
||||
el('framelabel').textContent =
|
||||
`f ${f} / ${state.dense.length - 1} · plate from f${pf}` +
|
||||
`f ${f} / ${state.dense.length - 1} · ${(f / state.fps).toFixed(2)}s · plate f${pf}` +
|
||||
(state.keep.has(f) ? ' · KEPT' : ' · held');
|
||||
|
||||
const c1 = el('cv-source'), g1 = c1.getContext('2d');
|
||||
|
|
@ -279,7 +298,7 @@ function drawStrip() {
|
|||
tag.textContent = f;
|
||||
cell.append(cv, tag);
|
||||
cell.onclick = (ev) => {
|
||||
state.frame = f;
|
||||
seekTo(f);
|
||||
if (ev.shiftKey) toggle(f);
|
||||
drawAll();
|
||||
};
|
||||
|
|
@ -318,7 +337,7 @@ function drawWorksheet() {
|
|||
const cap = document.createElement('span');
|
||||
cap.textContent = `f${f}` + (until > f ? ` → ${until}` : '') + ` (${until - f + 1}f)`;
|
||||
cell.append(cv, cap);
|
||||
cell.onclick = () => { state.frame = f; drawAll(); };
|
||||
cell.onclick = () => { seekTo(f); drawAll(); };
|
||||
host.append(cell);
|
||||
});
|
||||
}
|
||||
|
|
@ -330,7 +349,7 @@ function exportTake() {
|
|||
const N = state.dense.length;
|
||||
const take = {
|
||||
name: el('takename').value || 'line_01',
|
||||
frames: N, width: RW, height: RH, exposure: 1,
|
||||
frames: N, width: RW, height: RH, exposure: 1, fps: state.fps,
|
||||
palette: PALETTE,
|
||||
slot: { x: RW / 2, y: RH / 2 },
|
||||
parts: [
|
||||
|
|
@ -361,6 +380,15 @@ function exportTake() {
|
|||
|
||||
async function runFrames() {
|
||||
try {
|
||||
const man = await loadManifest();
|
||||
if (man) {
|
||||
state.fps = man.fps;
|
||||
el('framedir').value = man.dir || 'frames';
|
||||
attachAudio(man.audio);
|
||||
} else {
|
||||
attachAudio(null);
|
||||
status('no manifest.json — assuming 12fps, no audio. Re-run extract.sh', 'warn');
|
||||
}
|
||||
const images = await loadFrameSequence();
|
||||
if (!images.length) {
|
||||
status(`no frames in ${el('framedir').value}/ — run extract.sh first`, 'err');
|
||||
|
|
@ -371,14 +399,18 @@ async function runFrames() {
|
|||
el('scrub').max = dense.length - 1;
|
||||
state.frame = 0;
|
||||
rebuild(true);
|
||||
status(missing.length
|
||||
? `${images.length} frames · no face on ${missing.length} (held previous)`
|
||||
: `${images.length} frames detected`, missing.length ? 'warn' : 'ok');
|
||||
const dur = (dense.length / state.fps).toFixed(2);
|
||||
status(`${images.length} frames · ${state.fps}fps · ${dur}s` +
|
||||
(state.audio ? ' · audio loaded' : ' · no audio') +
|
||||
(missing.length ? ` · no face on ${missing.length} (held previous)` : ''),
|
||||
missing.length ? 'warn' : 'ok');
|
||||
} catch (e) { status(e.message, 'err'); console.error(e); }
|
||||
}
|
||||
|
||||
function runSynthetic() {
|
||||
state.images = [];
|
||||
attachAudio(null);
|
||||
state.fps = 12;
|
||||
state.dense = synthDense(72);
|
||||
el('scrub').max = 71;
|
||||
state.frame = 0;
|
||||
|
|
@ -396,7 +428,13 @@ for (const id of ['verts', 'smoothWin', 'contourSmooth', 'apertureThresh', 'tol'
|
|||
el(id + 'v').textContent = el(id).value;
|
||||
}
|
||||
|
||||
el('scrub').addEventListener('input', (e) => { state.frame = +e.target.value; drawAll(); });
|
||||
function seekTo(f) {
|
||||
state.frame = f;
|
||||
if (state.audio) state.audio.currentTime = f / state.fps;
|
||||
el('scrub').value = f;
|
||||
}
|
||||
|
||||
el('scrub').addEventListener('input', (e) => { seekTo(+e.target.value); drawAll(); });
|
||||
el('btn-frames').onclick = runFrames;
|
||||
el('btn-synth').onclick = runSynthetic;
|
||||
el('btn-export').onclick = exportTake;
|
||||
|
|
@ -409,34 +447,68 @@ el('btn-suggest').onclick = () => {
|
|||
status(`suggested ${kept.length} drawings at tolerance ${opts().tol.toFixed(3)} — now hand-correct`, 'ok');
|
||||
};
|
||||
el('btn-play').onclick = () => {
|
||||
if (!state.dense) return;
|
||||
state.playing = !state.playing;
|
||||
el('btn-play').textContent = state.playing ? 'Stop' : 'Play';
|
||||
if (state.playing) tick();
|
||||
if (state.playing) {
|
||||
if (state.audio) {
|
||||
state.audio.playbackRate = +el('speed').value;
|
||||
// Restart from the top if we are sitting at the end.
|
||||
if (state.frame >= state.dense.length - 1) state.frame = 0;
|
||||
state.audio.currentTime = state.frame / state.fps;
|
||||
state.audio.play().catch((e) => status('audio blocked: ' + e.message, 'warn'));
|
||||
}
|
||||
last = 0;
|
||||
tick();
|
||||
} else if (state.audio) {
|
||||
state.audio.pause();
|
||||
}
|
||||
};
|
||||
|
||||
el('speed').addEventListener('change', () => {
|
||||
// playbackRate retimes the clock, and the picture follows it for free.
|
||||
if (state.audio) state.audio.playbackRate = +el('speed').value;
|
||||
});
|
||||
|
||||
// Keyboard is the point: stepping and deleting 37 frames by mouse is miserable.
|
||||
window.addEventListener('keydown', (e) => {
|
||||
if (!state.dense || e.target.tagName === 'INPUT') return;
|
||||
const N = state.dense.length;
|
||||
if (e.key === 'ArrowRight') { state.frame = Math.min(N - 1, state.frame + 1); drawAll(); }
|
||||
else if (e.key === 'ArrowLeft') { state.frame = Math.max(0, state.frame - 1); drawAll(); }
|
||||
if (e.key === 'ArrowRight') { seekTo(Math.min(N - 1, state.frame + 1)); drawAll(); }
|
||||
else if (e.key === 'ArrowLeft') { seekTo(Math.max(0, state.frame - 1)); drawAll(); }
|
||||
else if (e.key === 'Backspace' || e.key === 'Delete' || e.key === 'x') {
|
||||
state.keep.delete(state.frame === 0 ? -1 : state.frame); drawAll();
|
||||
} else if (e.key === 'k' || e.key === ' ') {
|
||||
if (state.frame !== 0) state.keep.add(state.frame);
|
||||
drawAll();
|
||||
} else return;
|
||||
el('scrub').value = state.frame;
|
||||
e.preventDefault();
|
||||
});
|
||||
|
||||
let last = 0;
|
||||
function tick(ts = 0) {
|
||||
if (!state.playing) return;
|
||||
if (ts - last > 1000 / +el('playfps').value) {
|
||||
const N = state.dense.length;
|
||||
let f;
|
||||
if (state.audio) {
|
||||
// Audio is the clock. Deriving the frame from currentTime rather than
|
||||
// counting means a slow render loop drops frames instead of drifting out
|
||||
// of sync, which is the behaviour you want when judging lip sync.
|
||||
f = Math.floor(state.audio.currentTime * state.fps);
|
||||
if (f >= N || state.audio.ended) {
|
||||
state.audio.currentTime = 0;
|
||||
state.audio.play().catch(() => {});
|
||||
f = 0;
|
||||
}
|
||||
} else {
|
||||
const rate = state.fps * +el('speed').value;
|
||||
if (ts - last < 1000 / rate) { requestAnimationFrame(tick); return; }
|
||||
last = ts;
|
||||
state.frame = (state.frame + 1) % state.dense.length;
|
||||
el('scrub').value = state.frame;
|
||||
f = (state.frame + 1) % N;
|
||||
}
|
||||
if (f !== state.frame) {
|
||||
state.frame = f;
|
||||
el('scrub').value = f;
|
||||
drawPanes(); drawStrip();
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ const r = (v) => Math.round(v);
|
|||
|
||||
export function writeTake(take) {
|
||||
const L = [];
|
||||
L.push(`take name=${take.name} frames=${take.frames} width=${take.width} height=${take.height} exposure=${take.exposure}`);
|
||||
L.push(`take name=${take.name} frames=${take.frames} width=${take.width} height=${take.height}` +
|
||||
` exposure=${take.exposure}${take.fps ? ` fps=${take.fps}` : ''}`);
|
||||
L.push('');
|
||||
take.palette.forEach((p, i) => L.push(`pal ${i} ${p.name.padEnd(11)} ${i}`));
|
||||
L.push('');
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue