Add video upload, extraction progress, and reusable analysis sources

This commit is contained in:
Olive Vaughn 2026-09-28 09:38:49 -04:00
parent 690de21fa4
commit 686f897401
24 changed files with 927 additions and 137 deletions

View file

@ -15,6 +15,7 @@ addressing that answers questions about work not yet done.
"""
import hashlib
import os
import tempfile
from pathlib import Path
from django.conf import settings
@ -60,6 +61,31 @@ def write(data: bytes) -> tuple[str, int]:
return digest, len(data)
def write_stream(chunks) -> tuple[str, int]:
"""Store an uploaded file without reading the whole video into memory."""
root = Path(settings.BLOB_ROOT)
root.mkdir(parents=True, exist_ok=True)
digest = hashlib.sha256()
size = 0
with tempfile.NamedTemporaryFile(dir=root, prefix="upload-", delete=False) as out:
temporary = Path(out.name)
try:
for chunk in chunks:
digest.update(chunk)
size += len(chunk)
out.write(chunk)
except BaseException:
temporary.unlink(missing_ok=True)
raise
dest = path_for(digest.hexdigest())
dest.parent.mkdir(parents=True, exist_ok=True)
if dest.exists():
temporary.unlink()
else:
os.replace(temporary, dest)
return digest.hexdigest(), size
def adopt(source: Path) -> tuple[str, int]:
"""Store a file already on disk, by hard link where the filesystem allows it.

176
clips/extraction.py Normal file
View file

@ -0,0 +1,176 @@
"""Upload a video once, then decode it into the existing footage model."""
import hashlib
import json
import subprocess
import tempfile
import threading
import time
from fractions import Fraction
from pathlib import Path
from django.db import close_old_connections, transaction
from . import blobs
from .models import Blob, Extraction, Footage, FootageFrame
_active = set()
_lock = threading.Lock()
TIMEOUT = 3600
def _command(args):
result = subprocess.run(args, capture_output=True, text=True, timeout=TIMEOUT)
if result.returncode:
raise ValueError((result.stderr or result.stdout or "media tool failed")[-1200:])
return result.stdout
def _decode_frames(job, source_path, frames_dir, facts, root):
"""Decode one frame per source frame and publish ffmpeg's live frame count."""
progress_path = root / "frames.progress"
log_path = root / "frames.log"
total = facts.get("reported_frames") or round(facts["duration"] * facts["fps"])
args = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
"-stats_period", "0.25", "-progress", str(progress_path),
"-i", str(source_path), "-fps_mode", "passthrough",
str(frames_dir / "%04d.png")]
with open(log_path, "wb") as log:
proc = subprocess.Popen(args, stdout=log, stderr=subprocess.STDOUT)
deadline = time.monotonic() + TIMEOUT
try:
while proc.poll() is None:
if time.monotonic() >= deadline:
raise TimeoutError("video frame extraction timed out")
if progress_path.exists():
lines = progress_path.read_text(errors="replace").splitlines()
count = next((int(line[6:].strip()) for line in reversed(lines)
if line.startswith("frame=") and
line[6:].strip().isdigit()), 0)
if count and total:
progress = min(59, int(60 * count / total))
if progress > job.progress:
job.progress = progress
job.save(update_fields=["progress", "updated"])
time.sleep(0.2)
finally:
if proc.poll() is None:
proc.kill()
proc.wait()
if proc.returncode:
raise ValueError(log_path.read_text(errors="replace")[-1200:] or
"video frame extraction failed")
def probe(path):
data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams",
"-show_format", "-of", "json", str(path)]))
video = next((s for s in data.get("streams", []) if s.get("codec_type") == "video"), None)
if not video:
raise ValueError("the uploaded file has no video stream")
nominal = Fraction(video.get("r_frame_rate") or "0")
average = Fraction(video.get("avg_frame_rate") or "0")
if nominal <= 0 or average <= 0:
raise ValueError("the video's frame rate is unknown")
vfr = abs(float(nominal / average) - 1) > 0.001
if vfr:
raise ValueError("variable-frame-rate video needs timestamp-aware playback")
frames = video.get("nb_frames")
duration = float(data.get("format", {}).get("duration") or 0)
if ((frames and frames.isdigit() and int(frames) > 900)
or (duration > 0 and duration * float(average) > 901)):
raise ValueError("video is longer than the 900-frame footage limit")
return {"fps": float(average), "nominal_fps": float(nominal),
"width": int(video["width"]), "height": int(video["height"]),
"duration": duration,
"reported_frames": int(frames) if frames and frames.isdigit() else None,
"has_audio": any(s.get("codec_type") == "audio" for s in data.get("streams", [])),
"vfr": False}
def extraction_key(source, settings):
text = json.dumps({"scheme": 1, "source": source.blob_id, "settings": settings},
sort_keys=True, separators=(",", ":"))
return "sha256:" + hashlib.sha256(text.encode()).hexdigest()
def _register(job, frames, audio_path, facts):
width, height = blobs.png_size(frames[0])
frame_blobs = []
for index, path in enumerate(frames):
if blobs.png_size(path) != (width, height):
raise ValueError(f"decoded frame {index + 1} has different dimensions")
digest, size = blobs.adopt(path)
frame_blobs.append((index, digest, size))
audio_digest, audio_size = blobs.adopt(audio_path)
h = hashlib.sha256()
h.update(f"arthur-footage-1/{facts['fps']}/{len(frames)}/{width}x{height}\n".encode())
for _, digest, _ in frame_blobs:
h.update(digest.encode())
h.update(audio_digest.encode())
with transaction.atomic():
audio_blob, _ = Blob.objects.get_or_create(
digest=audio_digest, defaults={"size": audio_size, "media_type": "audio/wav"})
footage, created = Footage.objects.get_or_create(
digest=h.hexdigest(),
defaults={"label": job.source.filename[:200], "source": job.source.filename[:200],
"fps": facts["fps"],
"frames": len(frames), "width": width,
"height": height, "audio": audio_blob})
if created:
rows = []
for index, digest, size in frame_blobs:
blob, _ = Blob.objects.get_or_create(
digest=digest, defaults={"size": size, "media_type": "image/png"})
rows.append(FootageFrame(footage=footage, index=index, blob=blob))
FootageFrame.objects.bulk_create(rows)
return footage
def run(key):
close_old_connections()
try:
job = Extraction.objects.select_related("source", "source__blob").get(key=key)
job.state, job.progress, job.error = "running", 0, ""
job.save(update_fields=["state", "progress", "error", "updated"])
facts = job.source.probe
source_path = blobs.path_for(job.source.blob_id)
with tempfile.TemporaryDirectory(prefix="arthur-extract-") as directory:
root = Path(directory)
frames_dir = root / "frames"
frames_dir.mkdir()
_decode_frames(job, source_path, frames_dir, facts, root)
frames = sorted(frames_dir.glob("*.png"))
expected = facts.get("reported_frames")
if not frames or len(frames) > 900 or (expected and len(frames) != expected):
raise ValueError(f"decoded {len(frames)} frames; expected {expected or '1–900'}")
job.progress = 60
job.save(update_fields=["progress", "updated"])
audio_path = root / "audio.wav"
if facts["has_audio"]:
_command(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
"-i", str(source_path), "-vn", "-ac", "1", "-ar", "44100",
str(audio_path)])
else:
_command(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
"-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono",
"-t", str(len(frames) / facts["fps"]), "-c:a", "pcm_s16le",
str(audio_path)])
footage = _register(job, frames, audio_path, facts)
job.footage, job.state, job.progress = footage, "done", 100
job.save(update_fields=["footage", "state", "progress", "updated"])
except Exception as exc:
Extraction.objects.filter(key=key).update(state="failed", error=str(exc)[:2000])
finally:
with _lock:
_active.discard(key)
close_old_connections()
def enqueue(key):
with _lock:
if key in _active:
return
_active.add(key)
threading.Thread(target=run, args=(key,), daemon=True,
name=f"arthur-extract-{key[7:15]}").start()

View file

@ -0,0 +1,22 @@
# Generated by Django 5.2.17 on 2026-09-28 13:10
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('clips', '0001_initial'),
]
operations = [
migrations.RemoveField(
model_name='analysis',
name='artifact',
),
migrations.AddField(
model_name='analysis',
name='source_blocks',
field=models.ManyToManyField(blank=True, help_text='pixel-dependent landmarks, detection mask and mouth crops', related_name='source_for', to='clips.block'),
),
]

View file

@ -0,0 +1,39 @@
# Generated by Django 5.2.17 on 2026-09-28 13:23
import django.db.models.deletion
import uuid
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('clips', '0002_remove_analysis_artifact_analysis_source_blocks'),
]
operations = [
migrations.CreateModel(
name='Source',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('filename', models.CharField(max_length=255)),
('probe', models.JSONField(default=dict)),
('created', models.DateTimeField(auto_now_add=True)),
('blob', models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='video_source', to='clips.blob')),
],
),
migrations.CreateModel(
name='Extraction',
fields=[
('key', models.CharField(max_length=71, primary_key=True, serialize=False)),
('settings', models.JSONField(default=dict)),
('state', models.CharField(default='queued', max_length=16)),
('progress', models.PositiveIntegerField(default=0)),
('error', models.TextField(blank=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('footage', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='extractions', to='clips.footage')),
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='extractions', to='clips.source')),
],
),
]

View file

@ -40,6 +40,33 @@ class Blob(models.Model):
return f"{self.digest[:12]}… {self.size}B {self.media_type}"
class Source(models.Model):
"""An uploaded video, identified by its byte digest."""
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
blob = models.OneToOneField(Blob, on_delete=models.PROTECT, related_name="video_source")
filename = models.CharField(max_length=255)
probe = models.JSONField(default=dict)
created = models.DateTimeField(auto_now_add=True)
class Extraction(models.Model):
"""One requested decode of a source into immutable footage."""
key = models.CharField(primary_key=True, max_length=71)
source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="extractions")
settings = models.JSONField(default=dict)
state = models.CharField(max_length=16, default="queued")
progress = models.PositiveIntegerField(default=0)
error = models.TextField(blank=True)
footage = models.ForeignKey(
"Footage", null=True, blank=True, on_delete=models.SET_NULL,
related_name="extractions",
)
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
class Footage(models.Model):
"""Tier 3: the frames and audio of one extraction, immutable.
@ -108,9 +135,9 @@ class Analysis(models.Model):
footage = models.ForeignKey(
Footage, null=True, blank=True, on_delete=models.SET_NULL, related_name="analyses"
)
artifact = models.ForeignKey(
Blob, null=True, blank=True, on_delete=models.SET_NULL, related_name="analysis_for",
help_text="the dense landmark track, once bake A is uploaded",
source_blocks = models.ManyToManyField(
"Block", blank=True, related_name="source_for",
help_text="pixel-dependent landmarks, detection mask and mouth crops",
)
created = models.DateTimeField(auto_now_add=True)

View file

@ -18,15 +18,20 @@ manifest that makes the frames the backend's to serve.
"""
import hashlib
import json
import shutil
import struct
import subprocess
import tempfile
import zlib
from pathlib import Path
from unittest import skipUnless
from unittest.mock import Mock, patch
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase, override_settings
from clips import blobs
from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision
from clips import blobs, extraction
from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision, Source
BLOB_DIR = tempfile.mkdtemp(prefix="arthur-test-blobs-")
@ -215,6 +220,29 @@ class Tier2Tests(TestCase):
# duplicate it on disk.
self.assertEqual(1, Blob.objects.filter(block_data_for__isnull=False).distinct().count())
def test_an_analysis_reopens_its_three_source_blocks(self):
analysis = self.register_analysis()
keys = []
for role in ("source/dense", "source/detected", "source/crops"):
descriptor = block_descriptor(analysis, role=role)
key = key_for(descriptor)
self.assertEqual(201, self.post("/api/blocks", {
"key": key, "descriptor": descriptor, "data": "AA==",
}).status_code)
keys.append(key)
response = self.client.put(
f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys}),
content_type="application/json")
self.assertEqual(200, response.status_code, response.content)
self.assertEqual(set(keys), set(self.client.get(
f"/api/analyses/{analysis}").json()["source_blocks"]))
self.assertEqual(200, self.client.put(
f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys}),
content_type="application/json").status_code)
self.assertEqual(400, self.client.put(
f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys[:2]}),
content_type="application/json").status_code)
@override_settings(BLOB_ROOT=BLOB_DIR)
class DocumentTests(TestCase):
@ -476,3 +504,70 @@ class PageTests(TestCase):
self.assertNotEqual("unknown", report["version"])
self.assertTrue(report["model"].startswith("sha256:"))
self.assertIn("+", report["version"])
@skipUnless(shutil.which("ffmpeg") and shutil.which("ffprobe"), "ffmpeg is required")
@override_settings(BLOB_ROOT=BLOB_DIR)
class UploadTests(TestCase):
def test_frame_decode_reports_live_progress(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
frames = root / "frames"
frames.mkdir()
job = Mock(progress=0)
class FakeProcess:
returncode = 0
calls = 0
def poll(self):
self.calls += 1
if self.calls == 1:
(root / "frames.progress").write_text("frame=2\nprogress=continue\n")
return None
return 0
def wait(self):
return 0
with patch("clips.extraction.subprocess.Popen", return_value=FakeProcess()), \
patch("clips.extraction.time.sleep"):
extraction._decode_frames(job, root / "source.mp4", frames,
{"reported_frames": 4, "duration": 1, "fps": 4},
root)
self.assertEqual(30, job.progress)
job.save.assert_called_once_with(update_fields=["progress", "updated"])
def test_uploaded_video_extracts_to_reopenable_footage(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "four-frames.mp4"
subprocess.run([
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
"-f", "lavfi", "-i", "color=c=red:s=64x48:r=4:d=1",
"-c:v", "mpeg4", str(path),
], check=True, capture_output=True)
payload = path.read_bytes()
uploaded = self.client.post("/api/sources", {
"file": SimpleUploadedFile("four-frames.mp4", payload, content_type="video/mp4")})
self.assertEqual(201, uploaded.status_code, uploaded.content)
source_id = uploaded.json()["id"]
self.assertEqual(4, uploaded.json()["probe"]["reported_frames"])
self.assertEqual(1, Source.objects.count())
again = self.client.post("/api/sources", {
"file": SimpleUploadedFile("same-video.mp4", payload, content_type="video/mp4")})
self.assertEqual(200, again.status_code, again.content)
self.assertEqual(source_id, again.json()["id"])
with patch("clips.extraction.enqueue", side_effect=extraction.run):
queued = self.client.post("/api/extractions", json.dumps({
"source": source_id, "settings": {},
}), content_type="application/json")
self.assertIn(queued.status_code, (200, 202), queued.content)
job = self.client.get(f"/api/extractions/{queued.json()['key']}").json()
self.assertEqual("done", job["state"], job)
footage = self.client.get(f"/api/footage/{job['footage']}").json()
self.assertEqual((4, 64, 48), (footage["frames"], footage["width"], footage["height"]))
self.assertEqual(4, len(footage["urls"]))
self.assertEqual(200, self.client.get(footage["urls"][0]).status_code)
self.assertEqual(200, self.client.get(footage["audio"]).status_code)

View file

@ -17,6 +17,9 @@ from . import views
urlpatterns = [
path("detector", views.detector),
path("sources", views.sources),
path("extractions", views.extractions),
path("extractions/<str:key>", views.extraction_detail),
path("footage", views.footage_list),
path("footage/<uuid:footage_id>", views.footage_detail),
path("projects", views.projects),
@ -24,6 +27,7 @@ urlpatterns = [
path("projects/<uuid:project_id>/leaves/<path:leaf_path>", views.leaf_detail),
path("projects/<uuid:project_id>/revisions", views.revisions),
path("analyses", views.analyses),
path("analyses/<str:key>", views.analysis_detail),
path("blocks", views.blocks),
path("blocks/missing", views.blocks_missing),
path("blocks/<str:key>", views.block_detail),

View file

@ -27,15 +27,17 @@ import hashlib
import json
from functools import lru_cache
from pathlib import Path
from uuid import UUID
from django.conf import settings
from django.core.exceptions import ValidationError
from django.db import transaction
from django.http import FileResponse, HttpResponse, JsonResponse
from django.shortcuts import render
from django.views.decorators.http import require_http_methods
from . import blobs
from .models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision
from . import blobs, extraction
from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Leaf, Project, Revision, Source
KEY_LENGTH = 71 # "sha256:" + 64 hex
@ -155,8 +157,71 @@ def detector(request):
# THE MANIFEST NOW CARRIES URLS. It used to carry a directory and the loader built
# `frames/0001.png` itself, which quietly made the frame layout a shared secret
# between a shell script and a ClojureScript namespace. The server names every
# frame instead, so the frames can move into the blob store — or later be uploaded
# from the browser by wasm ffmpeg — without the client learning anything new.
# frame instead, so uploaded video and command-line bundles produce the same
# footage response without the client knowing where either stored its frames.
@require_http_methods(["GET", "POST"])
def sources(request):
if request.method == "GET":
return JsonResponse({"sources": [
{"id": str(row.id), "filename": row.filename, "probe": row.probe}
for row in Source.objects.order_by("-created")[:100]
]})
upload = request.FILES.get("file")
if upload is None:
return JsonResponse({"error": "upload a video as the file field"}, status=400)
try:
digest, size = blobs.write_stream(upload.chunks())
facts = extraction.probe(blobs.path_for(digest))
blob, _ = Blob.objects.get_or_create(
digest=digest, defaults={"size": size,
"media_type": upload.content_type or "video/mp4"})
row, created = Source.objects.get_or_create(
blob=blob, defaults={"filename": Path(upload.name).name[:255], "probe": facts})
return JsonResponse({"id": str(row.id), "digest": digest,
"filename": row.filename, "probe": row.probe,
"created": created}, status=201 if created else 200)
except (ValueError, OSError) as exc:
return JsonResponse({"error": str(exc)}, status=400)
def _extraction_json(row):
return {"key": row.key, "source": str(row.source_id), "state": row.state,
"progress": row.progress, "error": row.error,
"footage": str(row.footage_id) if row.footage_id else None}
@require_http_methods(["POST"])
def extractions(request):
try:
data = _body(request)
source_id = data.get("source")
if not source_id:
raise Bad("an extraction needs a source id")
try:
source = Source.objects.get(id=UUID(str(source_id)))
except (ValueError, ValidationError, Source.DoesNotExist):
raise Bad("no such source", status=404)
settings = data.get("settings") or {}
if settings != {}:
raise Bad("extraction currently keeps the source frame rate; settings must be empty")
key = extraction.extraction_key(source, settings)
row, _ = Extraction.objects.get_or_create(
key=key, defaults={"source": source, "settings": settings})
if row.state != "done":
extraction.enqueue(key)
return JsonResponse(_extraction_json(row), status=202 if row.state != "done" else 200)
except Bad as exc:
return _error(exc)
@require_http_methods(["GET"])
def extraction_detail(request, key):
try:
return JsonResponse(_extraction_json(Extraction.objects.get(key=key)))
except Extraction.DoesNotExist:
return JsonResponse({"error": "no such extraction"}, status=404)
def _footage_json(footage: Footage, urls=True):
@ -254,6 +319,39 @@ def analyses(request):
return _error(exc)
@require_http_methods(["GET", "PUT"])
def analysis_detail(request, key):
try:
row = Analysis.objects.get(key=key)
except Analysis.DoesNotExist:
return JsonResponse({"error": "no such analysis"}, status=404)
if request.method == "GET":
return JsonResponse({
"key": row.key, "descriptor": row.descriptor,
"detector": row.detector, "version": row.version,
"footage": str(row.footage_id) if row.footage_id else None,
"source_blocks": sorted(row.source_blocks.values_list("key", flat=True)),
})
try:
keys = _body(request).get("source_blocks")
roles = {"source/dense", "source/detected", "source/crops"}
if not isinstance(keys, list) or len(keys) != len(roles) or len(set(keys)) != len(roles):
raise Bad("an analysis needs one block for each source role")
blocks = list(Block.objects.filter(key__in=keys))
if (len(blocks) != len(roles) or {b.role for b in blocks} != roles
or any(b.analysis_id != key for b in blocks)):
raise Bad("source blocks must have distinct source roles and name this analysis")
with transaction.atomic():
row = Analysis.objects.select_for_update().get(key=key)
current = set(row.source_blocks.values_list("key", flat=True))
if current and current != set(keys):
raise Bad("the source blocks of an analysis are immutable", status=409)
row.source_blocks.set(blocks)
return JsonResponse({"key": key, "source_blocks": sorted(keys)})
except Bad as exc:
return _error(exc)
@require_http_methods(["POST"])
def blocks_missing(request):
"""Which of these keys the server does not have.