fix: thumbnail parser overflow, autostop-killed jobs, unified playhead

Thumbnails failed immediately on any clip past ~2 min. The one-pass select
expression joined frames with a flat a+b+c+... chain, which overflows ffmpeg's
recursive expression parser past ~120 terms ("Error while parsing expression").
Join the same terms as a balanced tree (depth O(log N)) instead — identical
selection, still one decode pass, exact per-tile frames preserved.

- generate_thumbnails management command: runs synchronously and streams each
  phase to stdout, so jobs are watchable without tailing app logs
- surface ffmpeg's stderr instead of swallowing it in CalledProcessError
- fly: min_machines_running = 1. Background thumbnail jobs outlive the HTTP
  request that triggers them, so scale-to-zero let the proxy autostop the VM
  mid-job and leave projects stuck "running"
- timeline: replace the two split playhead bars (pinned lanes vs scrolling
  tracks) with one full-height overlay, positioned in JS to track horizontal
  scroll and sit above the translucent header

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

fly.toml

@@ -16,9 +16,11 @@ primary_region = "iad"
 [http_service]
   internal_port = 8000
   force_https = true
-  auto_stop_machines = "stop"      # scale to zero when idle
+  auto_stop_machines = "stop"      # extra machines can still scale to zero
   auto_start_machines = true       # wake on the next request
-  min_machines_running = 0
+  min_machines_running = 1         # keep one VM always up: background thumbnail
+                                   # jobs outlive any HTTP request, so scale-to-zero
+                                   # would let the proxy reap the VM mid-job.
 
 [[vm]]
   size = "shared-cpu-1x"

scenes/management/commands/generate_thumbnails.py

@@ -0,0 +1,53 @@
+"""Run the thumbnailer synchronously and stream progress to the console.
+
+    python manage.py generate_thumbnails 1 2      # specific projects
+    python manage.py generate_thumbnails --all     # every project with a clip
+    python manage.py generate_thumbnails --failed   # only failed/stuck ones
+
+Unlike the upload path (which spawns a daemon thread and logs), this runs in the
+foreground so you can watch each ffmpeg pass and the final status without tailing
+app logs.
+"""
+from django.core.management.base import BaseCommand, CommandError
+
+from scenes.models import Project
+from scenes.thumbnails import generate_project_thumbnails
+
+
+class Command(BaseCommand):
+    help = "Generate thumbnails for projects, streaming progress to stdout."
+
+    def add_arguments(self, parser):
+        parser.add_argument("project_ids", nargs="*", type=int,
+                            help="Project ids to (re)generate.")
+        parser.add_argument("--all", action="store_true",
+                            help="Generate for every project that has a clip.")
+        parser.add_argument("--failed", action="store_true",
+                            help="Generate for projects whose status is failed/running/pending.")
+
+    def handle(self, *args, **opts):
+        ids = list(opts["project_ids"])
+        if opts["all"]:
+            ids += list(Project.objects.exclude(clip="").values_list("pk", flat=True))
+        if opts["failed"]:
+            ids += list(Project.objects
+                        .filter(thumbnail_status__in=["failed", "running", "pending"])
+                        .values_list("pk", flat=True))
+        ids = sorted(set(ids))
+        if not ids:
+            raise CommandError("no projects selected (pass ids, --all, or --failed)")
+
+        for pk in ids:
+            self.stdout.write(self.style.MIGRATE_HEADING(f"project {pk}"))
+            generate_project_thumbnails(pk, log=lambda msg: self.stdout.write(f"  {msg}"))
+
+        # report the resulting state so a sweep ends with a clear summary
+        self.stdout.write("")
+        for project in Project.objects.filter(pk__in=ids).order_by("pk"):
+            line = f"project {project.pk} [{project.thumbnail_status}] {project.name}"
+            style = (self.style.SUCCESS if project.thumbnail_status == "ready"
+                     else self.style.ERROR if project.thumbnail_status == "failed"
+                     else self.style.WARNING)
+            self.stdout.write(style(line))
+            if project.thumbnail_error:
+                self.stdout.write(f"    error: {project.thumbnail_error[:500]}")

scenes/thumbnails.py

@@ -19,7 +19,8 @@ THUMB_MAX_HEIGHT = 360
 THUMB_JPEG_QUALITY = 9
 # ffmpeg's select filter decodes the whole clip on every pass, so each chunk is
 # a full re-decode. Keep it one pass (one decode) for all but absurdly long clips
-# — a single select expression with thousands of terms is cheap to evaluate.
+# — thousands of terms evaluate fine once the select expression is balanced (see
+# _balanced_sum); the limit is parser depth, not term count.
 CLIP_SELECT_CHUNK_SIZE = 4096
 
 
@@ -103,6 +104,25 @@ def _chunks(items, size):
         yield i // size, items[i:i + size]
 
 
+def _balanced_sum(terms):
+    """Join `terms` with `+` as a *balanced* tree, e.g. ((a+b)+(c+d)).
+
+    ffmpeg's expression parser is recursive-descent: a flat "a+b+c+..." chain
+    nests right-associatively to depth N and overflows the parser past ~120
+    terms ("Error while parsing expression"). Balancing makes the parse depth
+    O(log N), so thousands of terms evaluate fine. `+` is associative, so the
+    selection is identical either way."""
+    nodes = list(terms)
+    if not nodes:
+        return "0"
+    while len(nodes) > 1:
+        nodes = [
+            f"({nodes[i]}+{nodes[i + 1]})" if i + 1 < len(nodes) else nodes[i]
+            for i in range(0, len(nodes), 2)
+        ]
+    return nodes[0]
+
+
 def enqueue_thumbnail_generation(project_id):
     thread = threading.Thread(
         target=generate_project_thumbnails,
@@ -113,10 +133,16 @@ def enqueue_thumbnail_generation(project_id):
     thread.start()
 
 
-def generate_project_thumbnails(project_id):
+def generate_project_thumbnails(project_id, log=None):
+    """Build the thumbnail manifest for a project. `log` is an optional
+    callable(str) for progress lines — the management command passes one so the
+    run is watchable on the console instead of only in app logs."""
+    emit = log if log is not None else (lambda msg: logger.info("[thumbs %s] %s", project_id, msg))
+
     try:
         project = Project.objects.get(pk=project_id)
     except Project.DoesNotExist:
+        emit("project does not exist")
         return
 
     if not project.clip:
@@ -124,6 +150,7 @@ def generate_project_thumbnails(project_id):
             thumbnail_status="failed",
             thumbnail_error="project has no clip",
         )
+        emit("failed: project has no clip")
         return
 
     Project.objects.filter(pk=project_id).update(
@@ -136,6 +163,7 @@ def generate_project_thumbnails(project_id):
     try:
         # pull the clip down to a local temp file so ffmpeg reads it directly,
         # regardless of whether storage is local disk or R2.
+        emit(f"downloading clip {project.clip.name}")
         clip_path = work / ("clip" + (Path(project.clip.name).suffix or ".mp4"))
         with project.clip.open("rb") as src, open(clip_path, "wb") as dst:
             shutil.copyfileobj(src, dst, length=1024 * 1024)
@@ -153,21 +181,33 @@ def generate_project_thumbnails(project_id):
         if not unique_clip_frames:
             raise RuntimeError("no clip frames to sample")
 
+        chunks = list(_chunks(unique_clip_frames, CLIP_SELECT_CHUNK_SIZE))
+        emit(f"{len(unique_clip_frames)} frames across {len(clip_items)} clips, "
+             f"{len(chunks)} ffmpeg pass(es)")
+
         frame_filenames = {}
-        for chunk_index, frames in _chunks(unique_clip_frames, CLIP_SELECT_CHUNK_SIZE):
-            select_expr = "+".join(f"eq(n\\,{frame})" for frame in frames)
-            subprocess.run([
+        for chunk_index, frames in chunks:
+            emit(f"  ffmpeg pass {chunk_index + 1}/{len(chunks)} ({len(frames)} frames)…")
+            select_expr = _balanced_sum(f"eq(n\\,{frame})" for frame in frames)
+            proc = subprocess.run([
                 "ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
                 "-i", str(clip_path),
                 "-vf", f"select='{select_expr}',{scale}",
                 "-vsync", "0",
                 "-q:v", str(THUMB_JPEG_QUALITY),
                 str(work / f"clipframe_{chunk_index:03d}_%06d.jpg"),
-            ], check=True, capture_output=True, text=True)
+            ], capture_output=True, text=True)
+            if proc.returncode != 0:
+                # surface ffmpeg's stderr — CalledProcessError hides it, which is
+                # what turned the last failure into a guessing game.
+                raise RuntimeError(
+                    f"ffmpeg exited {proc.returncode}: {proc.stderr.strip() or '(no stderr)'}"
+                )
             for idx, frame in enumerate(frames, start=1):
                 frame_filenames[frame] = f"clipframe_{chunk_index:03d}_{idx:06d}.jpg"
 
         # upload each needed tile once, then point the manifest at its public URL
+        emit("uploading tiles…")
         uploaded = {}
         clip_thumbnails = {}
         for clip in clip_items:
@@ -198,6 +238,7 @@ def generate_project_thumbnails(project_id):
             thumbnail_manifest=manifest,
             thumbnail_error="",
         )
+        emit(f"ready: {len(uploaded)} tiles uploaded for {len(clip_thumbnails)} clips")
     except Exception as exc:
         logger.exception("thumbnail generation failed for project %s", project_id)
         Project.objects.filter(pk=project_id).update(
@@ -205,5 +246,6 @@ def generate_project_thumbnails(project_id):
             thumbnail_error=str(exc),
             thumbnail_manifest={},
         )
+        emit(f"failed: {exc}")
     finally:
         shutil.rmtree(work, ignore_errors=True)

tl/resources/public/css/app.css

@@ -364,8 +364,11 @@ html.dark .timeline-head {
                1px 1px 0 var(--paper), -1px -1px 0 var(--paper);
 }
 
-.playhead { position: absolute; top: 0; bottom: 0; width: 2px; background: var(--ink);
-            z-index: 10; pointer-events: none; }
+/* one full-height playhead spanning both the pinned lanes and the scrolling
+   tracks; positioned in JS (see position-playhead!) so it tracks horizontal
+   scroll while sitting above the translucent header. */
+.timeline-playhead { position: absolute; top: 0; bottom: 0; width: 2px; background: var(--ink);
+                     z-index: 14; pointer-events: none; }
 .playhead-handle { position: absolute; top: 0; left: -5px; width: 12px; height: 10px;
                    background: var(--ink); border-radius: 0; }
 
@@ -379,7 +382,6 @@ html.dark .timeline-head {
 .ann-bar-label { position: absolute; top: 1px; font-family: var(--chicago); font-size: 9px;
                  line-height: 16px; color: var(--ink); white-space: nowrap;
                  pointer-events: none; }
-.lane-playhead { bottom: 0; }
 
 /* --- mobile: stack the top region vertically (video over annotation) ---- */
 @media (max-width: 720px) {

tl/src/tl/views.cljs

@@ -12,6 +12,7 @@
 (defonce video-el  (atom nil))
 (defonce raf       (atom nil))
 (defonce scroll-el (atom nil))
+(defonce playhead-el (atom nil))   ; the single full-height playhead overlay
 (defonce play      (atom nil))   ; {:ctx :segs :fps :idx} while playing, else nil
 (defonce active-insert! (atom nil))   ; the live content-editor's (insert! link) fn
 (declare commit-link!)
@@ -71,6 +72,23 @@
     (let [x (px playhead fps zoom)]
       (set! (.-scrollLeft el) (max 0 (- x (/ (.-clientWidth el) 2)))))))
 
+;; The playhead is a single full-height overlay on `.timeline` (spanning the
+;; pinned annotation lanes AND the scrolling tracks as one line) rather than a
+;; bar per region. It lives outside the horizontal scroller, so we track the
+;; scroller's scrollLeft/offsetLeft by hand and hide it once it slips under the
+;; track-name gutter or past the right edge.
+(defn- position-playhead! [fps zoom playhead]
+  (when-let [ph @playhead-el]
+    (if-let [sc @scroll-el]
+      (let [x      (- (px playhead fps zoom) (.-scrollLeft sc))
+            gutter (.-offsetLeft sc)
+            view   (.-clientWidth sc)]
+        (if (and (>= x 0) (<= x view))
+          (do (set! (.. ph -style -left) (str (+ gutter x) "px"))
+              (set! (.. ph -style -display) "block"))
+          (set! (.. ph -style -display) "none")))
+      (set! (.. ph -style -display) "none"))))
+
 (defn- seg-at
   "Index of the segment whose LOCAL range contains `local` (0 if none)."
   [segs local]
@@ -271,7 +289,10 @@
             tracks-h (* row-h (count tracks))
             track-y  (into {} (map-indexed (fn [i t] [(:id t) i]) tracks))]
         (when playing? (r/after-render #(follow! fps zoom playhead)))
+        (r/after-render #(position-playhead! fps zoom playhead))
         [:div.timeline {:class (when-not @tracks-open? "tracks-collapsed")}
+         [:div.playhead.timeline-playhead {:ref (fn [n] (reset! playhead-el n))}
+          [:div.playhead-handle]]
          [:div.timeline-head
           [:div.gutter-lanes {:style {:height lane-h}}
            [:button.track-toggle {:title (if @tracks-open? "Collapse track names" "Show track names")
@@ -280,8 +301,6 @@
           [:div.lanes-hscroll {:ref (fn [n] (reset! lanes-scroll n))}
            [:div.ann-lanes {:style {:height lane-h :width width}
                             :on-mouse-down #(scrub! @content fps zoom %)}
-            [:div.playhead.lane-playhead {:style {:left (px playhead fps zoom)}}
-             [:div.playhead-handle]]
             ;; annotation bars — solid for saved, dashed for the in-progress draft
             (for [[i a] (map-indexed vector anns)
                   [j [lo hi]] (map-indexed vector (:bars a))]
@@ -310,13 +329,13 @@
              [:div.gutter-label {:style {:height row-h :line-height (str (dec row-h) "px")}}
               (:name t)])]
           [:div.hscroll {:ref (fn [n] (reset! scroll-el n))
-                         :on-scroll #(when @lanes-scroll
-                                       (set! (.-scrollLeft @lanes-scroll)
-                                             (.. % -target -scrollLeft)))}
+                         :on-scroll #(let [sl (.. % -target -scrollLeft)]
+                                       (when @lanes-scroll
+                                         (set! (.-scrollLeft @lanes-scroll) sl))
+                                       (position-playhead! fps zoom playhead))}
            [:div.content.track-content {:ref (fn [n] (reset! content n))
                                         :on-mouse-down #(scrub! @content fps zoom %)
                                         :style {:width width :height tracks-h}}
-            [:div.playhead {:style {:left (px playhead fps zoom)}}]
             (for [[i t] (map-indexed vector tracks)]
               ^{:key (str "lane-" (:id t))}
               [:div.track-lane {:style {:top (* i row-h) :width width :height row-h}}])