fix: thumbnail parser overflow, autostop-killed jobs, unified playhead
Thumbnails failed immediately on any clip past ~2 min. The one-pass select
expression joined frames with a flat a+b+c+... chain, which overflows ffmpeg's
recursive expression parser past ~120 terms ("Error while parsing expression").
Join the same terms as a balanced tree (depth O(log N)) instead — identical
selection, still one decode pass, exact per-tile frames preserved.
- generate_thumbnails management command: runs synchronously and streams each
phase to stdout, so jobs are watchable without tailing app logs
- surface ffmpeg's stderr instead of swallowing it in CalledProcessError
- fly: min_machines_running = 1. Background thumbnail jobs outlive the HTTP
request that triggers them, so scale-to-zero let the proxy autostop the VM
mid-job and leave projects stuck "running"
- timeline: replace the two split playhead bars (pinned lanes vs scrolling
tracks) with one full-height overlay, positioned in JS to track horizontal
scroll and sit above the translucent header
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
20ae2b4316
commit
28aafc047f
5 changed files with 135 additions and 17 deletions
|
|
@ -19,7 +19,8 @@ THUMB_MAX_HEIGHT = 360
|
|||
THUMB_JPEG_QUALITY = 9
|
||||
# ffmpeg's select filter decodes the whole clip on every pass, so each chunk is
|
||||
# a full re-decode. Keep it one pass (one decode) for all but absurdly long clips
|
||||
# — a single select expression with thousands of terms is cheap to evaluate.
|
||||
# — thousands of terms evaluate fine once the select expression is balanced (see
|
||||
# _balanced_sum); the limit is parser depth, not term count.
|
||||
CLIP_SELECT_CHUNK_SIZE = 4096
|
||||
|
||||
|
||||
|
|
@ -103,6 +104,25 @@ def _chunks(items, size):
|
|||
yield i // size, items[i:i + size]
|
||||
|
||||
|
||||
def _balanced_sum(terms):
|
||||
"""Join `terms` with `+` as a *balanced* tree, e.g. ((a+b)+(c+d)).
|
||||
|
||||
ffmpeg's expression parser is recursive-descent: a flat "a+b+c+..." chain
|
||||
nests right-associatively to depth N and overflows the parser past ~120
|
||||
terms ("Error while parsing expression"). Balancing makes the parse depth
|
||||
O(log N), so thousands of terms evaluate fine. `+` is associative, so the
|
||||
selection is identical either way."""
|
||||
nodes = list(terms)
|
||||
if not nodes:
|
||||
return "0"
|
||||
while len(nodes) > 1:
|
||||
nodes = [
|
||||
f"({nodes[i]}+{nodes[i + 1]})" if i + 1 < len(nodes) else nodes[i]
|
||||
for i in range(0, len(nodes), 2)
|
||||
]
|
||||
return nodes[0]
|
||||
|
||||
|
||||
def enqueue_thumbnail_generation(project_id):
|
||||
thread = threading.Thread(
|
||||
target=generate_project_thumbnails,
|
||||
|
|
@ -113,10 +133,16 @@ def enqueue_thumbnail_generation(project_id):
|
|||
thread.start()
|
||||
|
||||
|
||||
def generate_project_thumbnails(project_id):
|
||||
def generate_project_thumbnails(project_id, log=None):
|
||||
"""Build the thumbnail manifest for a project. `log` is an optional
|
||||
callable(str) for progress lines — the management command passes one so the
|
||||
run is watchable on the console instead of only in app logs."""
|
||||
emit = log if log is not None else (lambda msg: logger.info("[thumbs %s] %s", project_id, msg))
|
||||
|
||||
try:
|
||||
project = Project.objects.get(pk=project_id)
|
||||
except Project.DoesNotExist:
|
||||
emit("project does not exist")
|
||||
return
|
||||
|
||||
if not project.clip:
|
||||
|
|
@ -124,6 +150,7 @@ def generate_project_thumbnails(project_id):
|
|||
thumbnail_status="failed",
|
||||
thumbnail_error="project has no clip",
|
||||
)
|
||||
emit("failed: project has no clip")
|
||||
return
|
||||
|
||||
Project.objects.filter(pk=project_id).update(
|
||||
|
|
@ -136,6 +163,7 @@ def generate_project_thumbnails(project_id):
|
|||
try:
|
||||
# pull the clip down to a local temp file so ffmpeg reads it directly,
|
||||
# regardless of whether storage is local disk or R2.
|
||||
emit(f"downloading clip {project.clip.name}")
|
||||
clip_path = work / ("clip" + (Path(project.clip.name).suffix or ".mp4"))
|
||||
with project.clip.open("rb") as src, open(clip_path, "wb") as dst:
|
||||
shutil.copyfileobj(src, dst, length=1024 * 1024)
|
||||
|
|
@ -153,21 +181,33 @@ def generate_project_thumbnails(project_id):
|
|||
if not unique_clip_frames:
|
||||
raise RuntimeError("no clip frames to sample")
|
||||
|
||||
chunks = list(_chunks(unique_clip_frames, CLIP_SELECT_CHUNK_SIZE))
|
||||
emit(f"{len(unique_clip_frames)} frames across {len(clip_items)} clips, "
|
||||
f"{len(chunks)} ffmpeg pass(es)")
|
||||
|
||||
frame_filenames = {}
|
||||
for chunk_index, frames in _chunks(unique_clip_frames, CLIP_SELECT_CHUNK_SIZE):
|
||||
select_expr = "+".join(f"eq(n\\,{frame})" for frame in frames)
|
||||
subprocess.run([
|
||||
for chunk_index, frames in chunks:
|
||||
emit(f" ffmpeg pass {chunk_index + 1}/{len(chunks)} ({len(frames)} frames)…")
|
||||
select_expr = _balanced_sum(f"eq(n\\,{frame})" for frame in frames)
|
||||
proc = subprocess.run([
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-i", str(clip_path),
|
||||
"-vf", f"select='{select_expr}',{scale}",
|
||||
"-vsync", "0",
|
||||
"-q:v", str(THUMB_JPEG_QUALITY),
|
||||
str(work / f"clipframe_{chunk_index:03d}_%06d.jpg"),
|
||||
], check=True, capture_output=True, text=True)
|
||||
], capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
# surface ffmpeg's stderr — CalledProcessError hides it, which is
|
||||
# what turned the last failure into a guessing game.
|
||||
raise RuntimeError(
|
||||
f"ffmpeg exited {proc.returncode}: {proc.stderr.strip() or '(no stderr)'}"
|
||||
)
|
||||
for idx, frame in enumerate(frames, start=1):
|
||||
frame_filenames[frame] = f"clipframe_{chunk_index:03d}_{idx:06d}.jpg"
|
||||
|
||||
# upload each needed tile once, then point the manifest at its public URL
|
||||
emit("uploading tiles…")
|
||||
uploaded = {}
|
||||
clip_thumbnails = {}
|
||||
for clip in clip_items:
|
||||
|
|
@ -198,6 +238,7 @@ def generate_project_thumbnails(project_id):
|
|||
thumbnail_manifest=manifest,
|
||||
thumbnail_error="",
|
||||
)
|
||||
emit(f"ready: {len(uploaded)} tiles uploaded for {len(clip_thumbnails)} clips")
|
||||
except Exception as exc:
|
||||
logger.exception("thumbnail generation failed for project %s", project_id)
|
||||
Project.objects.filter(pk=project_id).update(
|
||||
|
|
@ -205,5 +246,6 @@ def generate_project_thumbnails(project_id):
|
|||
thumbnail_error=str(exc),
|
||||
thumbnail_manifest={},
|
||||
)
|
||||
emit(f"failed: {exc}")
|
||||
finally:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue