diff --git a/README.md b/README.md index d374cc7..8e19db1 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,11 @@ modern conveniences belong in the workflow, not the output. See ## ClojureScript port -The active port has reached [step 9 of the port plan](docs/port-plan.md): it plays -the synthetic take, loads real footage with mouth, eyes, brows and pixel-derived -teeth, and now has a Django backend that persists the document. The step 8 data -model represents persistent feature IDs, eye pairs and feature-level observation -gaps; its controls are still pending. +The active port plays the synthetic take, accepts video uploads, extracts their +frames and audio, analyzes real footage for mouth, eyes, brows and pixel-derived +teeth, and saves the project with reusable analysis data. The step 8 data model +represents persistent feature IDs, eye pairs and feature-level observation gaps; +its controls are still pending. See the [port plan](docs/port-plan.md). ```sh mise install # both halves @@ -29,9 +29,30 @@ mise exec -- python manage.py migrate ./do start # Django + frontend watcher ``` -See [frontend/README.md](frontend/README.md) for the **load frames** and **save** -workflows. Anything under "## Run" and below describes the older JS prototype, -which still runs separately on port 8777. +In the app, upload a video, choose its footage, click **load frames**, then +**save**. Opening that project on another client reuses its saved landmarks and +mouth crops without detecting source frames again. The upload path derives its +footage response from database records; it does not create or consume a +`manifest.json` file. See [frontend/README.md](frontend/README.md) for details. +Anything under "## Run" and below describes the older JS prototype, which still +runs separately on port 8777. + +### Deploy to Fly.io + +The app uses a persistent Fly volume for its SQLite document database and +content-addressed footage blobs. Create the app once, set its Django secret, and +deploy from the repository root: + +```sh +fly apps create arthur --org personal +fly volumes create data --region iad --size 1 +fly secrets set DJANGO_SECRET_KEY="$(openssl rand -hex 32)" --app arthur +fly deploy --app arthur +``` + +The deployed app is at . The container builds the +ClojureScript frontend, collects static assets, and runs database migrations on +startup. ### How it is stored @@ -41,8 +62,8 @@ Three tiers, cut by mutability and size — the full argument is in | Tier | What | Where | | --- | --- | --- | | 1 **authored** | the scene: nodes, channels, features, time maps | the database, as independently addressed leaves. Kilobytes | -| 2 **derived** | the dense channel blocks | `var/blobs`, content-addressed by a hash over every input — including the detector version | -| 3 **source** | frames and audio | the same blob store, by the hash of their bytes | +| 2 **derived** | detected landmarks, raw mouth crops, and dense channel blocks | `var/blobs`, addressed by analysis and block inputs, including the detector version | +| 3 **source** | uploaded video, extracted frames, and audio | the same blob store, by the hash of their bytes | Only tier 1 is the document. Tier 2 is a pure function of tiers 1 and 3, so a saved project names its blocks rather than carrying them, and a knob change gives diff --git a/clips/blobs.py b/clips/blobs.py index c9e0f3e..7469e66 100644 --- a/clips/blobs.py +++ b/clips/blobs.py @@ -15,6 +15,7 @@ addressing that answers questions about work not yet done. """ import hashlib import os +import tempfile from pathlib import Path from django.conf import settings @@ -60,6 +61,31 @@ def write(data: bytes) -> tuple[str, int]: return digest, len(data) +def write_stream(chunks) -> tuple[str, int]: + """Store an uploaded file without reading the whole video into memory.""" + root = Path(settings.BLOB_ROOT) + root.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + size = 0 + with tempfile.NamedTemporaryFile(dir=root, prefix="upload-", delete=False) as out: + temporary = Path(out.name) + try: + for chunk in chunks: + digest.update(chunk) + size += len(chunk) + out.write(chunk) + except BaseException: + temporary.unlink(missing_ok=True) + raise + dest = path_for(digest.hexdigest()) + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.exists(): + temporary.unlink() + else: + os.replace(temporary, dest) + return digest.hexdigest(), size + + def adopt(source: Path) -> tuple[str, int]: """Store a file already on disk, by hard link where the filesystem allows it. diff --git a/clips/extraction.py b/clips/extraction.py new file mode 100644 index 0000000..8019103 --- /dev/null +++ b/clips/extraction.py @@ -0,0 +1,176 @@ +"""Upload a video once, then decode it into the existing footage model.""" + +import hashlib +import json +import subprocess +import tempfile +import threading +import time +from fractions import Fraction +from pathlib import Path + +from django.db import close_old_connections, transaction + +from . import blobs +from .models import Blob, Extraction, Footage, FootageFrame + +_active = set() +_lock = threading.Lock() +TIMEOUT = 3600 + + +def _command(args): + result = subprocess.run(args, capture_output=True, text=True, timeout=TIMEOUT) + if result.returncode: + raise ValueError((result.stderr or result.stdout or "media tool failed")[-1200:]) + return result.stdout + + +def _decode_frames(job, source_path, frames_dir, facts, root): + """Decode one frame per source frame and publish ffmpeg's live frame count.""" + progress_path = root / "frames.progress" + log_path = root / "frames.log" + total = facts.get("reported_frames") or round(facts["duration"] * facts["fps"]) + args = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-stats_period", "0.25", "-progress", str(progress_path), + "-i", str(source_path), "-fps_mode", "passthrough", + str(frames_dir / "%04d.png")] + with open(log_path, "wb") as log: + proc = subprocess.Popen(args, stdout=log, stderr=subprocess.STDOUT) + deadline = time.monotonic() + TIMEOUT + try: + while proc.poll() is None: + if time.monotonic() >= deadline: + raise TimeoutError("video frame extraction timed out") + if progress_path.exists(): + lines = progress_path.read_text(errors="replace").splitlines() + count = next((int(line[6:].strip()) for line in reversed(lines) + if line.startswith("frame=") and + line[6:].strip().isdigit()), 0) + if count and total: + progress = min(59, int(60 * count / total)) + if progress > job.progress: + job.progress = progress + job.save(update_fields=["progress", "updated"]) + time.sleep(0.2) + finally: + if proc.poll() is None: + proc.kill() + proc.wait() + if proc.returncode: + raise ValueError(log_path.read_text(errors="replace")[-1200:] or + "video frame extraction failed") + + +def probe(path): + data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams", + "-show_format", "-of", "json", str(path)])) + video = next((s for s in data.get("streams", []) if s.get("codec_type") == "video"), None) + if not video: + raise ValueError("the uploaded file has no video stream") + nominal = Fraction(video.get("r_frame_rate") or "0") + average = Fraction(video.get("avg_frame_rate") or "0") + if nominal <= 0 or average <= 0: + raise ValueError("the video's frame rate is unknown") + vfr = abs(float(nominal / average) - 1) > 0.001 + if vfr: + raise ValueError("variable-frame-rate video needs timestamp-aware playback") + frames = video.get("nb_frames") + duration = float(data.get("format", {}).get("duration") or 0) + if ((frames and frames.isdigit() and int(frames) > 900) + or (duration > 0 and duration * float(average) > 901)): + raise ValueError("video is longer than the 900-frame footage limit") + return {"fps": float(average), "nominal_fps": float(nominal), + "width": int(video["width"]), "height": int(video["height"]), + "duration": duration, + "reported_frames": int(frames) if frames and frames.isdigit() else None, + "has_audio": any(s.get("codec_type") == "audio" for s in data.get("streams", [])), + "vfr": False} + + +def extraction_key(source, settings): + text = json.dumps({"scheme": 1, "source": source.blob_id, "settings": settings}, + sort_keys=True, separators=(",", ":")) + return "sha256:" + hashlib.sha256(text.encode()).hexdigest() + + +def _register(job, frames, audio_path, facts): + width, height = blobs.png_size(frames[0]) + frame_blobs = [] + for index, path in enumerate(frames): + if blobs.png_size(path) != (width, height): + raise ValueError(f"decoded frame {index + 1} has different dimensions") + digest, size = blobs.adopt(path) + frame_blobs.append((index, digest, size)) + audio_digest, audio_size = blobs.adopt(audio_path) + h = hashlib.sha256() + h.update(f"arthur-footage-1/{facts['fps']}/{len(frames)}/{width}x{height}\n".encode()) + for _, digest, _ in frame_blobs: + h.update(digest.encode()) + h.update(audio_digest.encode()) + with transaction.atomic(): + audio_blob, _ = Blob.objects.get_or_create( + digest=audio_digest, defaults={"size": audio_size, "media_type": "audio/wav"}) + footage, created = Footage.objects.get_or_create( + digest=h.hexdigest(), + defaults={"label": job.source.filename[:200], "source": job.source.filename[:200], + "fps": facts["fps"], + "frames": len(frames), "width": width, + "height": height, "audio": audio_blob}) + if created: + rows = [] + for index, digest, size in frame_blobs: + blob, _ = Blob.objects.get_or_create( + digest=digest, defaults={"size": size, "media_type": "image/png"}) + rows.append(FootageFrame(footage=footage, index=index, blob=blob)) + FootageFrame.objects.bulk_create(rows) + return footage + + +def run(key): + close_old_connections() + try: + job = Extraction.objects.select_related("source", "source__blob").get(key=key) + job.state, job.progress, job.error = "running", 0, "" + job.save(update_fields=["state", "progress", "error", "updated"]) + facts = job.source.probe + source_path = blobs.path_for(job.source.blob_id) + with tempfile.TemporaryDirectory(prefix="arthur-extract-") as directory: + root = Path(directory) + frames_dir = root / "frames" + frames_dir.mkdir() + _decode_frames(job, source_path, frames_dir, facts, root) + frames = sorted(frames_dir.glob("*.png")) + expected = facts.get("reported_frames") + if not frames or len(frames) > 900 or (expected and len(frames) != expected): + raise ValueError(f"decoded {len(frames)} frames; expected {expected or '1–900'}") + job.progress = 60 + job.save(update_fields=["progress", "updated"]) + audio_path = root / "audio.wav" + if facts["has_audio"]: + _command(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-i", str(source_path), "-vn", "-ac", "1", "-ar", "44100", + str(audio_path)]) + else: + _command(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono", + "-t", str(len(frames) / facts["fps"]), "-c:a", "pcm_s16le", + str(audio_path)]) + footage = _register(job, frames, audio_path, facts) + job.footage, job.state, job.progress = footage, "done", 100 + job.save(update_fields=["footage", "state", "progress", "updated"]) + except Exception as exc: + Extraction.objects.filter(key=key).update(state="failed", error=str(exc)[:2000]) + finally: + with _lock: + _active.discard(key) + close_old_connections() + + +def enqueue(key): + with _lock: + if key in _active: + return + _active.add(key) + threading.Thread(target=run, args=(key,), daemon=True, + name=f"arthur-extract-{key[7:15]}").start() diff --git a/clips/migrations/0002_remove_analysis_artifact_analysis_source_blocks.py b/clips/migrations/0002_remove_analysis_artifact_analysis_source_blocks.py new file mode 100644 index 0000000..a4f3396 --- /dev/null +++ b/clips/migrations/0002_remove_analysis_artifact_analysis_source_blocks.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.17 on 2026-09-28 13:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('clips', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='analysis', + name='artifact', + ), + migrations.AddField( + model_name='analysis', + name='source_blocks', + field=models.ManyToManyField(blank=True, help_text='pixel-dependent landmarks, detection mask and mouth crops', related_name='source_for', to='clips.block'), + ), + ] diff --git a/clips/migrations/0003_source_extraction.py b/clips/migrations/0003_source_extraction.py new file mode 100644 index 0000000..2bb2f51 --- /dev/null +++ b/clips/migrations/0003_source_extraction.py @@ -0,0 +1,39 @@ +# Generated by Django 5.2.17 on 2026-09-28 13:23 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('clips', '0002_remove_analysis_artifact_analysis_source_blocks'), + ] + + operations = [ + migrations.CreateModel( + name='Source', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('filename', models.CharField(max_length=255)), + ('probe', models.JSONField(default=dict)), + ('created', models.DateTimeField(auto_now_add=True)), + ('blob', models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='video_source', to='clips.blob')), + ], + ), + migrations.CreateModel( + name='Extraction', + fields=[ + ('key', models.CharField(max_length=71, primary_key=True, serialize=False)), + ('settings', models.JSONField(default=dict)), + ('state', models.CharField(default='queued', max_length=16)), + ('progress', models.PositiveIntegerField(default=0)), + ('error', models.TextField(blank=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('footage', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='extractions', to='clips.footage')), + ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='extractions', to='clips.source')), + ], + ), + ] diff --git a/clips/models.py b/clips/models.py index e083808..daacd4c 100644 --- a/clips/models.py +++ b/clips/models.py @@ -40,6 +40,33 @@ class Blob(models.Model): return f"{self.digest[:12]}… {self.size}B {self.media_type}" +class Source(models.Model): + """An uploaded video, identified by its byte digest.""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + blob = models.OneToOneField(Blob, on_delete=models.PROTECT, related_name="video_source") + filename = models.CharField(max_length=255) + probe = models.JSONField(default=dict) + created = models.DateTimeField(auto_now_add=True) + + +class Extraction(models.Model): + """One requested decode of a source into immutable footage.""" + + key = models.CharField(primary_key=True, max_length=71) + source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="extractions") + settings = models.JSONField(default=dict) + state = models.CharField(max_length=16, default="queued") + progress = models.PositiveIntegerField(default=0) + error = models.TextField(blank=True) + footage = models.ForeignKey( + "Footage", null=True, blank=True, on_delete=models.SET_NULL, + related_name="extractions", + ) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + class Footage(models.Model): """Tier 3: the frames and audio of one extraction, immutable. @@ -108,9 +135,9 @@ class Analysis(models.Model): footage = models.ForeignKey( Footage, null=True, blank=True, on_delete=models.SET_NULL, related_name="analyses" ) - artifact = models.ForeignKey( - Blob, null=True, blank=True, on_delete=models.SET_NULL, related_name="analysis_for", - help_text="the dense landmark track, once bake A is uploaded", + source_blocks = models.ManyToManyField( + "Block", blank=True, related_name="source_for", + help_text="pixel-dependent landmarks, detection mask and mouth crops", ) created = models.DateTimeField(auto_now_add=True) diff --git a/clips/tests/test_api.py b/clips/tests/test_api.py index c7e902e..9954a4d 100644 --- a/clips/tests/test_api.py +++ b/clips/tests/test_api.py @@ -18,15 +18,20 @@ manifest that makes the frames the backend's to serve. """ import hashlib import json +import shutil import struct +import subprocess import tempfile import zlib from pathlib import Path +from unittest import skipUnless +from unittest.mock import Mock, patch +from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase, override_settings -from clips import blobs -from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision +from clips import blobs, extraction +from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision, Source BLOB_DIR = tempfile.mkdtemp(prefix="arthur-test-blobs-") @@ -215,6 +220,29 @@ class Tier2Tests(TestCase): # duplicate it on disk. self.assertEqual(1, Blob.objects.filter(block_data_for__isnull=False).distinct().count()) + def test_an_analysis_reopens_its_three_source_blocks(self): + analysis = self.register_analysis() + keys = [] + for role in ("source/dense", "source/detected", "source/crops"): + descriptor = block_descriptor(analysis, role=role) + key = key_for(descriptor) + self.assertEqual(201, self.post("/api/blocks", { + "key": key, "descriptor": descriptor, "data": "AA==", + }).status_code) + keys.append(key) + response = self.client.put( + f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys}), + content_type="application/json") + self.assertEqual(200, response.status_code, response.content) + self.assertEqual(set(keys), set(self.client.get( + f"/api/analyses/{analysis}").json()["source_blocks"])) + self.assertEqual(200, self.client.put( + f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys}), + content_type="application/json").status_code) + self.assertEqual(400, self.client.put( + f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys[:2]}), + content_type="application/json").status_code) + @override_settings(BLOB_ROOT=BLOB_DIR) class DocumentTests(TestCase): @@ -476,3 +504,70 @@ class PageTests(TestCase): self.assertNotEqual("unknown", report["version"]) self.assertTrue(report["model"].startswith("sha256:")) self.assertIn("+", report["version"]) + + +@skipUnless(shutil.which("ffmpeg") and shutil.which("ffprobe"), "ffmpeg is required") +@override_settings(BLOB_ROOT=BLOB_DIR) +class UploadTests(TestCase): + def test_frame_decode_reports_live_progress(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + frames = root / "frames" + frames.mkdir() + job = Mock(progress=0) + + class FakeProcess: + returncode = 0 + calls = 0 + + def poll(self): + self.calls += 1 + if self.calls == 1: + (root / "frames.progress").write_text("frame=2\nprogress=continue\n") + return None + return 0 + + def wait(self): + return 0 + + with patch("clips.extraction.subprocess.Popen", return_value=FakeProcess()), \ + patch("clips.extraction.time.sleep"): + extraction._decode_frames(job, root / "source.mp4", frames, + {"reported_frames": 4, "duration": 1, "fps": 4}, + root) + self.assertEqual(30, job.progress) + job.save.assert_called_once_with(update_fields=["progress", "updated"]) + + def test_uploaded_video_extracts_to_reopenable_footage(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "four-frames.mp4" + subprocess.run([ + "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "color=c=red:s=64x48:r=4:d=1", + "-c:v", "mpeg4", str(path), + ], check=True, capture_output=True) + payload = path.read_bytes() + + uploaded = self.client.post("/api/sources", { + "file": SimpleUploadedFile("four-frames.mp4", payload, content_type="video/mp4")}) + self.assertEqual(201, uploaded.status_code, uploaded.content) + source_id = uploaded.json()["id"] + self.assertEqual(4, uploaded.json()["probe"]["reported_frames"]) + self.assertEqual(1, Source.objects.count()) + again = self.client.post("/api/sources", { + "file": SimpleUploadedFile("same-video.mp4", payload, content_type="video/mp4")}) + self.assertEqual(200, again.status_code, again.content) + self.assertEqual(source_id, again.json()["id"]) + + with patch("clips.extraction.enqueue", side_effect=extraction.run): + queued = self.client.post("/api/extractions", json.dumps({ + "source": source_id, "settings": {}, + }), content_type="application/json") + self.assertIn(queued.status_code, (200, 202), queued.content) + job = self.client.get(f"/api/extractions/{queued.json()['key']}").json() + self.assertEqual("done", job["state"], job) + footage = self.client.get(f"/api/footage/{job['footage']}").json() + self.assertEqual((4, 64, 48), (footage["frames"], footage["width"], footage["height"])) + self.assertEqual(4, len(footage["urls"])) + self.assertEqual(200, self.client.get(footage["urls"][0]).status_code) + self.assertEqual(200, self.client.get(footage["audio"]).status_code) diff --git a/clips/urls.py b/clips/urls.py index 4874732..4b4042f 100644 --- a/clips/urls.py +++ b/clips/urls.py @@ -17,6 +17,9 @@ from . import views urlpatterns = [ path("detector", views.detector), + path("sources", views.sources), + path("extractions", views.extractions), + path("extractions/", views.extraction_detail), path("footage", views.footage_list), path("footage/", views.footage_detail), path("projects", views.projects), @@ -24,6 +27,7 @@ urlpatterns = [ path("projects//leaves/", views.leaf_detail), path("projects//revisions", views.revisions), path("analyses", views.analyses), + path("analyses/", views.analysis_detail), path("blocks", views.blocks), path("blocks/missing", views.blocks_missing), path("blocks/", views.block_detail), diff --git a/clips/views.py b/clips/views.py index d2c0355..51de35d 100644 --- a/clips/views.py +++ b/clips/views.py @@ -27,15 +27,17 @@ import hashlib import json from functools import lru_cache from pathlib import Path +from uuid import UUID from django.conf import settings +from django.core.exceptions import ValidationError from django.db import transaction from django.http import FileResponse, HttpResponse, JsonResponse from django.shortcuts import render from django.views.decorators.http import require_http_methods -from . import blobs -from .models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision +from . import blobs, extraction +from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Leaf, Project, Revision, Source KEY_LENGTH = 71 # "sha256:" + 64 hex @@ -155,8 +157,71 @@ def detector(request): # THE MANIFEST NOW CARRIES URLS. It used to carry a directory and the loader built # `frames/0001.png` itself, which quietly made the frame layout a shared secret # between a shell script and a ClojureScript namespace. The server names every -# frame instead, so the frames can move into the blob store — or later be uploaded -# from the browser by wasm ffmpeg — without the client learning anything new. +# frame instead, so uploaded video and command-line bundles produce the same +# footage response without the client knowing where either stored its frames. + + +@require_http_methods(["GET", "POST"]) +def sources(request): + if request.method == "GET": + return JsonResponse({"sources": [ + {"id": str(row.id), "filename": row.filename, "probe": row.probe} + for row in Source.objects.order_by("-created")[:100] + ]}) + upload = request.FILES.get("file") + if upload is None: + return JsonResponse({"error": "upload a video as the file field"}, status=400) + try: + digest, size = blobs.write_stream(upload.chunks()) + facts = extraction.probe(blobs.path_for(digest)) + blob, _ = Blob.objects.get_or_create( + digest=digest, defaults={"size": size, + "media_type": upload.content_type or "video/mp4"}) + row, created = Source.objects.get_or_create( + blob=blob, defaults={"filename": Path(upload.name).name[:255], "probe": facts}) + return JsonResponse({"id": str(row.id), "digest": digest, + "filename": row.filename, "probe": row.probe, + "created": created}, status=201 if created else 200) + except (ValueError, OSError) as exc: + return JsonResponse({"error": str(exc)}, status=400) + + +def _extraction_json(row): + return {"key": row.key, "source": str(row.source_id), "state": row.state, + "progress": row.progress, "error": row.error, + "footage": str(row.footage_id) if row.footage_id else None} + + +@require_http_methods(["POST"]) +def extractions(request): + try: + data = _body(request) + source_id = data.get("source") + if not source_id: + raise Bad("an extraction needs a source id") + try: + source = Source.objects.get(id=UUID(str(source_id))) + except (ValueError, ValidationError, Source.DoesNotExist): + raise Bad("no such source", status=404) + settings = data.get("settings") or {} + if settings != {}: + raise Bad("extraction currently keeps the source frame rate; settings must be empty") + key = extraction.extraction_key(source, settings) + row, _ = Extraction.objects.get_or_create( + key=key, defaults={"source": source, "settings": settings}) + if row.state != "done": + extraction.enqueue(key) + return JsonResponse(_extraction_json(row), status=202 if row.state != "done" else 200) + except Bad as exc: + return _error(exc) + + +@require_http_methods(["GET"]) +def extraction_detail(request, key): + try: + return JsonResponse(_extraction_json(Extraction.objects.get(key=key))) + except Extraction.DoesNotExist: + return JsonResponse({"error": "no such extraction"}, status=404) def _footage_json(footage: Footage, urls=True): @@ -254,6 +319,39 @@ def analyses(request): return _error(exc) +@require_http_methods(["GET", "PUT"]) +def analysis_detail(request, key): + try: + row = Analysis.objects.get(key=key) + except Analysis.DoesNotExist: + return JsonResponse({"error": "no such analysis"}, status=404) + if request.method == "GET": + return JsonResponse({ + "key": row.key, "descriptor": row.descriptor, + "detector": row.detector, "version": row.version, + "footage": str(row.footage_id) if row.footage_id else None, + "source_blocks": sorted(row.source_blocks.values_list("key", flat=True)), + }) + try: + keys = _body(request).get("source_blocks") + roles = {"source/dense", "source/detected", "source/crops"} + if not isinstance(keys, list) or len(keys) != len(roles) or len(set(keys)) != len(roles): + raise Bad("an analysis needs one block for each source role") + blocks = list(Block.objects.filter(key__in=keys)) + if (len(blocks) != len(roles) or {b.role for b in blocks} != roles + or any(b.analysis_id != key for b in blocks)): + raise Bad("source blocks must have distinct source roles and name this analysis") + with transaction.atomic(): + row = Analysis.objects.select_for_update().get(key=key) + current = set(row.source_blocks.values_list("key", flat=True)) + if current and current != set(keys): + raise Bad("the source blocks of an analysis are immutable", status=409) + row.source_blocks.set(blocks) + return JsonResponse({"key": key, "source_blocks": sorted(keys)}) + except Bad as exc: + return _error(exc) + + @require_http_methods(["POST"]) def blocks_missing(request): """Which of these keys the server does not have. diff --git a/docs/architecture.md b/docs/architecture.md index f3d0b63..2620267 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -570,10 +570,15 @@ addressing that answers questions about work not yet done. ``` POST /api/analyses {key, descriptor} idempotent +GET /api/analyses/ metadata + source block keys +PUT /api/analyses/ link dense landmarks, mask, crops POST /api/blocks/missing {keys} -> {missing} POST /api/blocks {key, descriptor, data, state} GET /api/blocks/ GET /api/footage/ the manifest, with a URL per frame +POST /api/sources multipart video upload +POST /api/extractions idempotent decode job +GET /api/extractions/ job state and footage id ``` **The server verifies, rather than trusting a name it was handed.** It recomputes @@ -596,9 +601,9 @@ separately. fetched `manifest.json` and built `frames/0001.png` itself, which made the frame layout a shared secret between a shell script and a ClojureScript namespace. The manifest now carries a URL per frame, so the frames can live in the blob store — -or, when wasm-ffmpeg extraction arrives, be uploaded into the same store by the -app — and the client learns nothing new when that happens. The producer changes; -the shape does not. +or be produced by the app's video upload and server-side ffmpeg extraction. The +upload path needs no `manifest.json` file: the server builds the footage response +from the extracted frame records. The producer changes; the shape does not. **The document stores what a block IS, not what it holds.** A block's element type is in its own descriptor, which is the only place it is written down: an diff --git a/frontend/README.md b/frontend/README.md index 3fa1f21..80783f1 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -41,8 +41,9 @@ things fail in ways that read like the code being broken and are not. mise exec -- python manage.py test clips # from the REPO ROOT ``` -Thirty-one tests over the API: the blob store, key verification, the load/save -round trip, the conditional write, and the footage manifest. The two groups worth +Tests cover the API: the blob store, key verification, the load/save round trip, +the conditional write, source analysis blocks, and video upload and extraction. +The two groups worth reading are the ones that make the tier split a property of the system rather than a convention in ClojureScript — the server recomputes every tier-2 key it is handed, and refuses a block whose analysis does not declare a detector version. @@ -111,10 +112,16 @@ The demo scene itself is `src/arthur/demo/scene.edn`. Both the synthetic take and real footage use `src/arthur/flow/take.cljs` for the measurement order and `src/arthur/flow/freeze.cljs` for the landmark-to-channel conversion. -### Real footage (port steps 6–7, served by the backend since step 9) +### Real footage -Two commands from the repo root, then pick the take in the app and click -**load frames**: +Choose a video in the **footage** file input. The server probes it, extracts one +PNG per source frame and WAV audio, then makes the resulting footage selectable. +Click **load frames** to detect and freeze it. Extraction progress is currently +read from `/api/extractions/`; a future WebSocket can push the same job state. +The uploaded bytes, extraction job, and decoded footage have separate records, so +the same uploaded video can be reopened without decoding it again. + +The command-line route is also available for an existing extracted bundle: ```sh ./extract.sh /path/to/clip.mov # decode to frames + audio + manifest @@ -161,6 +168,9 @@ stays 320×200 regardless of the footage dimensions. Real footage starts at the source picture rate. The **picture fps** buttons sample the frozen roto at lower rates while the source track, duration and audio clock stay unchanged. Picking frames to trace into cels is a separate future editing step. +**save** also stores the detection mask, dense landmarks and raw RGBA mouth crops +as three analysis blocks. **open** restores these without running MediaPipe or +loading source PNGs. The frozen shapes remain separate channel blocks. For known occlusion intervals, an extracted manifest may add `"feature-absence": {"eye-r": [[10, 14]]}`. Frame numbers are one-based and @@ -184,13 +194,14 @@ runs the old JS tool on 8777, and the two are meant to run side by side. ## Saving -**save** and **open** in the transport. A save is three requests, in an order that +**save** and **open** in the transport. A save has three ordered stages: is the tier split: 1. the **analysis** record, so every block stored afterwards can name the detector version that produced it. The server refuses a block whose analysis it does not know. -2. ask which **blocks** are missing, and upload only those. +2. ask which **blocks** are missing, upload the source analysis blocks and frozen + channel blocks, then link the source blocks to the analysis. 3. the **document** — tier 1, as leaves. The server refuses a clip that names blocks it does not hold, so a saved document cannot load into a blank stage somewhere else. diff --git a/frontend/src/arthur/domain/wire.cljs b/frontend/src/arthur/domain/wire.cljs index dc10d57..272c673 100644 --- a/frontend/src/arthur/domain/wire.cljs +++ b/frontend/src/arthur/domain/wire.cljs @@ -97,6 +97,7 @@ (case type "int16" (js/Int16Array. (.-buffer u8)) "float32" (js/Float32Array. (.-buffer u8)) + "float64" (js/Float64Array. (.-buffer u8)) "uint8" u8 - (throw (ex-info "a block's element type is \"int16\", \"float32\" or \"uint8\"" + (throw (ex-info "a block's element type is \"int16\", \"float32\", \"float64\" or \"uint8\"" {:type type}))))) diff --git a/frontend/src/arthur/events/footage.cljs b/frontend/src/arthur/events/footage.cljs index bc1d025..58a935c 100644 --- a/frontend/src/arthur/events/footage.cljs +++ b/frontend/src/arthur/events/footage.cljs @@ -9,16 +9,18 @@ [arthur.flow.detect :as detect] [arthur.flow.ingest :as ingest] [arthur.flow.measure.interior :as interior] + [arthur.flow.source :as source] [arthur.flow.take :as take] [arthur.footage.store :as store] [arthur.domain.landmarks :as lm] + [arthur.fx.http :as http] [re-frame.core :as rf])) (defn- detect-frames! [manifest model] (let [canvas (.createElement js/document "canvas") ctx (.getContext canvas "2d") raw (atom []) - interiors (atom []) + crops (atom []) dims (atom nil) total (:frames manifest)] (js/Promise. @@ -27,7 +29,7 @@ (if (= i total) (try (resolve (assoc (detect/fill-gaps @raw) - :dimensions @dims :interior @interiors)) + :dimensions @dims :crops @crops)) (catch :default error (reject error))) (-> (ingest/image! (ingest/frame-url manifest i)) (.then @@ -43,13 +45,11 @@ (let [face (detect/detect! model canvas) ring (when face (mapv #(nth face %) lm/LIPS-INNER)) box (when ring (interior/crop ring wh)) - measured (if box - (interior/measure take/knobs box - (.getImageData ctx (:x box) (:y box) - (:w box) (:h box))) - {:contour nil :contrast 0 :area 0 :debug nil})] + pixels (when box + (.-data (.getImageData ctx (:x box) (:y box) + (:w box) (:h box))))] (swap! raw conj face) - (swap! interiors conj measured)) + (swap! crops conj (when box {:box box :data pixels}))) (when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total)) (rf/dispatch [::progress (str "detecting " (inc i) "/" total)])) ;; Let the status and the transport paint between sync @@ -59,43 +59,67 @@ (next-frame 0)))))) (defn- build-clip [manifest detector - {:keys [dense detected dimensions interior missing first-real]}] + {:keys [dense detected dimensions crops missing first-real] :as source-inputs}] (let [[w h] dimensions + interior (source/measure-crops take/knobs crops) frozen (take/footage manifest {:dense dense :detected detected :dimensions dimensions :interior interior :presence (:presence manifest) :detector detector}) - built (:clip frozen)] + built (:clip frozen) + source-blocks (source/pack (:id (:analysis built)) source-inputs)] (assoc (select-keys built [:fps :width :height]) :frames (clip/frames built) :display-fps (:fps built) :clip built :store (:store frozen) + :source-blocks source-blocks :source-inputs source-inputs ;; No cache-buster. The audio is a blob named by the hash of its own ;; bytes, so re-extracting gives it a different URL rather than ;; overwriting this one — which is what the `?v=` here used to work ;; around. :audio (ingest/audio-url manifest) :label (or (:label manifest) (:source manifest) "footage") + :footage-id (:id manifest) :cid (or (:id manifest) "footage") :summary (str (:frames manifest) " frames · " w "×" h " · " (:fps manifest) " fps" (when (pos? missing) (str " · " missing " without a face" (when (pos? first-real) - (str " (first found on " (inc first-real) ")")))))))) + (str " (first found on " (inc first-real) ")")))))))) + +(defn- cached-source! [manifest detector] + (let [key (:id (take/analysis-for manifest detector))] + (-> (http/GET (str "/api/analyses/" key)) + (.then (fn [^js analysis] + (let [keys (array-seq (.-source_blocks analysis))] + (when (= (count keys) (count source/roles)) + (-> (js/Promise.all + (into-array (map #(http/GET (str "/api/blocks/" %)) keys))) + (.then (fn [blocks] + (source/unpack blocks + [(:width manifest) (:height manifest)])))))))) + (.catch (fn [error] + (if (= 404 (:status (ex-data error))) nil (throw error))))))) (rf/reg-fx ::begin! (fn [footage-id] (-> (js/Promise.all #js [(ingest/manifest! footage-id) (ingest/detector!)]) (.then (fn [[manifest detector]] - (rf/dispatch [::progress "loading MediaPipe…"]) - (-> (detect/landmarker!) - (.then (fn [model] - (rf/dispatch [::progress "loading frames…"]) - (-> (detect-frames! manifest model) - (.then (fn [track] - (build-clip manifest detector track))))))))) + (rf/dispatch [::progress "looking for saved analysis…"]) + (-> (cached-source! manifest detector) + (.then (fn [track] + (if track + (do (rf/dispatch [::progress "reusing saved analysis…"]) + (build-clip manifest detector track)) + (do (rf/dispatch [::progress "loading MediaPipe…"]) + (-> (detect/landmarker!) + (.then (fn [model] + (rf/dispatch [::progress "loading frames…"]) + (-> (detect-frames! manifest model) + (.then (fn [fresh] + (build-clip manifest detector fresh)))))))))))))) (.then (fn [entry] (let [id (store/install! entry)] (rf/dispatch [::loaded id (:summary entry)])))) @@ -111,6 +135,47 @@ (.catch (fn [error] (rf/dispatch [::failed (or (ex-message error) (str error))])))))) +(defn- poll-extraction! [key] + (-> (http/GET (str "/api/extractions/" key)) + (.then (fn [^js job] + (case (.-state job) + "done" (rf/dispatch [::uploaded (.-footage job)]) + "failed" (rf/dispatch [::failed (.-error job)]) + (do (rf/dispatch [::progress + (str "extracting " (.-progress job) "%")]) + (js/setTimeout #(poll-extraction! key) 800))))) + (.catch (fn [error] + (rf/dispatch [::failed (or (ex-message error) (str error))]))))) + +(rf/reg-fx + ::upload! + (fn [file] + (let [form (js/FormData.)] + (.append form "file" file) + (-> (http/POST-form "/api/sources" form) + (.then (fn [^js source] + (rf/dispatch [::progress "queued for extraction…"]) + (http/POST "/api/extractions" #js {:source (.-id source) + :settings #js {}}))) + (.then (fn [^js job] (poll-extraction! (.-key job)))) + (.catch (fn [error] + (rf/dispatch [::failed (or (ex-message error) (str error))]))))))) + +(rf/reg-event-fx + ::upload + (fn [{:keys [db]} [_ file]] + (if (or (nil? file) (get-in db [:footage :loading?])) + {} + {:db (update db :footage merge {:loading? true :status "uploading video…"}) + ::upload! file}))) + +(rf/reg-event-fx + ::uploaded + (fn [{:keys [db]} [_ footage-id]] + {:db (update db :footage merge {:loading? false :chosen footage-id + :status "video extracted — load frames to analyze"}) + :dispatch [::refresh]})) + (rf/reg-event-fx ::refresh (fn [_ _] {::list! nil})) @@ -119,10 +184,9 @@ ::listed (fn [db [_ footage]] (update db :footage merge - {:available (vec footage) - :chosen (or (:chosen (:footage db)) (:id (first footage))) - :status (when (empty? footage) - "no footage ingested — ./extract.sh, then manage.py ingest_bundle")}))) + (cond-> {:available (vec footage) + :chosen (or (:chosen (:footage db)) (:id (first footage)))} + (empty? footage) (assoc :status "upload a video to begin"))))) (rf/reg-event-db ::choose @@ -135,8 +199,7 @@ (cond (get-in db [:footage :loading?]) {} (nil? chosen) - {:db (assoc-in db [:footage :status] - "no footage ingested — ./extract.sh, then manage.py ingest_bundle")} + {:db (assoc-in db [:footage :status] "upload a video to begin")} :else {:db (update db :footage merge {:loading? true :status "reading the manifest…"}) ::pb/pause! nil @@ -150,7 +213,7 @@ ::failed (fn [db [_ message]] (assoc db :footage (assoc (:footage db) - :loading? false :status (str "load failed: " message))))) + :loading? false :status (str "footage failed: " message))))) (rf/reg-event-fx ::loaded diff --git a/frontend/src/arthur/events/project.cljs b/frontend/src/arthur/events/project.cljs index d6a683f..6ee31d4 100644 --- a/frontend/src/arthur/events/project.cljs +++ b/frontend/src/arthur/events/project.cljs @@ -26,6 +26,8 @@ [arthur.events.playback :as pb] [arthur.footage.store :as store] [arthur.flow.address :as address] + [arthur.flow.source :as source] + [arthur.flow.ingest :as ingest] [arthur.fx.http :as http] [re-frame.core :as rf])) @@ -65,16 +67,64 @@ (-> (http/POST "/api/projects" #js {:name name}) (.then (fn [^js created] (.-id created)))))) +(defn- opened-entry! [^js clip-json] + (let [footage-id (.-footage clip-json) + analysis-key (.-analysis clip-json)] + (-> (js/Promise.all + #js [(js/Promise.all + (into-array (map #(http/GET (str "/api/blocks/" %)) + (array-seq (.-blocks clip-json))))) + (if footage-id (ingest/manifest! footage-id) (js/Promise.resolve nil)) + (if analysis-key (http/GET (str "/api/analyses/" analysis-key)) + (js/Promise.resolve nil))]) + (.then (fn [[blocks manifest analysis]] + (let [source-keys (when analysis (array-seq (.-source_blocks ^js analysis)))] + (-> (js/Promise.all + (into-array (map #(http/GET (str "/api/blocks/" %)) source-keys))) + (.then (fn [source-responses] + (let [cid (.-cid clip-json) + loaded (project/load + cid #js {:leaves (.-leaves clip-json) + :blocks blocks}) + built (:clip loaded) + inputs (when (seq source-keys) + (when-not manifest + (throw (ex-info "saved analysis has no footage" + {:analysis analysis-key}))) + (source/unpack source-responses + [(:width manifest) (:height manifest)]))] + (merge (select-keys built [:fps :width :height]) + {:label (str (or (.-name clip-json) cid) " (saved)") + :cid cid :frames (clip/frames built) + :display-fps (:fps built) + :clip built :store (:store loaded) + :footage-id footage-id + :source-inputs inputs + :source-blocks (when inputs + (source/pack analysis-key inputs)) + :audio (if manifest (:audio manifest) + "/static/arthur/audio.wav")}))))))))))) + (rf/reg-fx ::save! (fn [{:keys [id cid label clip]}] (let [analysis (:analysis (:clip clip)) - doc (project/save cid clip)] + doc (project/save cid clip) + source-blocks (:source-blocks clip)] (-> (ensure-project! id label) (.then (fn [pid] (-> (if analysis (http/POST "/api/analyses" (analysis-payload analysis)) (js/Promise.resolve nil)) + (.then (fn [_] + (when (seq source-blocks) + (-> (upload-missing! + #js {:blocks (source/wire-blocks source-blocks)}) + (.then (fn [_] + (http/PUT + (str "/api/analyses/" (:id analysis)) + #js {:source_blocks + (into-array (map :key (vals source-blocks)))}))))))) (.then (fn [_] (upload-missing! doc))) (.then (fn [uploaded] (-> (http/PUT (str "/api/projects/" pid) @@ -82,6 +132,7 @@ :clips #js [#js {:cid cid :name label :analysis (:id analysis) + :footage (:footage-id clip) :leaves (.-leaves doc) :blocks (block-keys doc)}]}) (.then (fn [^js saved] @@ -109,35 +160,13 @@ (let [^js clip-json (first (array-seq (.-clips loaded)))] (when-not clip-json (throw (ex-info "that project has no clips" {}))) - (-> (js/Promise.all - (into-array (map #(http/GET (str "/api/blocks/" %)) - (array-seq (.-blocks clip-json))))) - (.then (fn [blocks] - (let [doc #js {:leaves (.-leaves clip-json) :blocks blocks} - cid (.-cid clip-json) - loaded' (project/load cid doc) - built (:clip loaded') - entry (merge - (select-keys built [:fps :width :height]) - {:label (str (or (.-name clip-json) cid) " (saved)") - :cid cid - :frames (clip/frames built) - :display-fps (:fps built) - :clip built - :store (:store loaded') - ;; The audio is the clip's, and a - ;; document does not carry it: tier 3 - ;; is by hash and the clip names the - ;; analysis, not the sound. Until the - ;; footage id is in the document, the - ;; synthetic take's is the one that - ;; keeps the clock running. - :audio "/static/arthur/audio.wav"})] - (rf/dispatch [::opened - (store/install! entry "project") - (.-id loaded) - (.-name loaded) - (.-seq loaded)])))))))) + (-> (opened-entry! clip-json) + (.then (fn [entry] + (rf/dispatch [::opened + (store/install! entry "project") + (.-id loaded) + (.-name loaded) + (.-seq loaded)]))))))) (.catch (fn [error] (js/console.error error) (rf/dispatch [::failed (or (ex-message error) (str error))])))))) diff --git a/frontend/src/arthur/flow/address.cljs b/frontend/src/arthur/flow/address.cljs index 5822f35..c2af4e1 100644 --- a/frontend/src/arthur/flow/address.cljs +++ b/frontend/src/arthur/flow/address.cljs @@ -137,6 +137,9 @@ budget does not reach the teeth at all (the crop is taken from raw landmarks). It reads like a mistake in both directions and is neither." {"geom" [:anchor-avg :contour-avg :verts] + "source/dense" [] + "source/detected" [] + "source/crops" [] "head-pos" [:anchor-avg] "head-rot" [:anchor-avg] "head-scale" [:anchor-avg] diff --git a/frontend/src/arthur/flow/ingest.cljs b/frontend/src/arthur/flow/ingest.cljs index 8569056..40fad03 100644 --- a/frontend/src/arthur/flow/ingest.cljs +++ b/frontend/src/arthur/flow/ingest.cljs @@ -1,24 +1,7 @@ (ns arthur.flow.ingest - "Read an ingested take. The manifest owns timing, the exact frame count, and — - since step 9 — the URL of every frame. - - WHAT CHANGED, AND WHY IT IS NOT A DETAIL. This used to fetch `/manifest.json` off - the filesystem and then build `frames/0001.png` itself, with shadow-cljs serving - the repo root. So the frame layout was a shared secret between a shell script and - this namespace, and \"where are the frames\" was answered by a directory listing - that nothing could version. - - Now the server names every frame and this asks it. The manifest carries a URL per - frame, so the frames can live in a content-addressed blob store — or, when the - in-browser wasm-ffmpeg extraction docs/architecture.md describes arrives, be - uploaded into the same store by the app itself — and nothing in here learns - anything new. That is the whole point of tier 3 being addressed rather than - located. - - The cache-busting `?v=` that used to hang off every frame URL went with it. It - was there because re-extracting overwrote `frames/0001.png` under the same name; - a blob's name IS the hash of its bytes, so a stale copy is not a thing that can - happen." + "Read footage from the server. Its response gives source timing and one + content-addressed URL per frame. The upload path derives this response from + stored frame records; no `manifest.json` file is part of that path." (:require [arthur.fx.http :as http])) (defn feature-presence @@ -67,8 +50,7 @@ :presence (feature-presence frames (:feature-absence m))))) (defn available! - "Every ingested take the server holds. `manage.py ingest_bundle` is what puts one - there." + "Every extracted take the server holds." [] (-> (http/GET "/api/footage") (.then (fn [json] (:footage (js->clj json :keywordize-keys true)))))) diff --git a/frontend/src/arthur/flow/source.cljs b/frontend/src/arthur/flow/source.cljs new file mode 100644 index 0000000..1c11c6d --- /dev/null +++ b/frontend/src/arthur/flow/source.cljs @@ -0,0 +1,118 @@ +(ns arthur.flow.source + "The pixel-dependent result of analyzing footage, stored as three ordinary + content-addressed blocks. Everything after this boundary can run without PNGs + or MediaPipe. The mouth crops hold raw pixels and an un-eroded lip ring." + (:require [arthur.domain.landmarks :as lm] + [arthur.domain.wire :as wire] + [arthur.flow.address :as address] + [arthur.flow.measure.interior :as interior])) + +(def roles ["source/dense" "source/detected" "source/crops"]) + +(defn measure-crops [params crops] + (mapv (fn [crop] + (if crop + (interior/measure params (:box crop) #js {:data (:data crop)}) + {:contour nil :contrast 0 :area 0 :debug nil})) + crops)) + +(defn- named [role analysis tracks layout data] + (merge (address/block {:role role :analysis analysis :params {} + :tracks tracks :features [] :observation nil + :layout layout}) + {:role role :data data})) + +(defn pack + "Dense landmarks, detection mask and raw RGBA mouth crops -> source blocks." + [analysis {:keys [dense detected crops]}] + (let [frames (count dense) + points (count (first dense))] + (when-not (and (pos? frames) (pos? points) + (= frames (count detected) (count crops)) + (every? #(= points (count %)) dense)) + (throw (ex-info "source tracks must have the same frame and landmark counts" + {:frames frames :points points}))) + (let [landmarks (js/Float64Array. (* frames points 3)) + mask (js/Uint8Array. frames) + lengths (mapv #(if % (.-length ^js (:data %)) 0) crops) + offsets (vec (reductions + 0 lengths)) + pixels (js/Uint8Array. (max 1 (peek offsets))) + boxes (mapv (fn [crop] + (when-let [box (:box crop)] + (mapv box [:x :y :w :h]))) + crops)] + (doseq [f (range frames)] + (aset mask f (if (nth detected f) 1 0)) + (doseq [i (range points)] + (let [{:keys [x y z]} (nth (nth dense f) i) + base (* 3 (+ (* f points) i))] + (aset landmarks base x) + (aset landmarks (inc base) y) + (aset landmarks (+ base 2) z))) + (when-let [crop (nth crops f)] + (.set pixels (:data crop) (nth offsets f)))) + {"source/dense" (named "source/dense" analysis ["landmarks"] + {:type "float64" :frames frames :points points :stride (* 3 points)} + landmarks) + "source/detected" (named "source/detected" analysis ["detected"] + {:type "uint8" :frames frames :stride 1} mask) + "source/crops" (named "source/crops" analysis ["rgba"] + {:type "uint8" :frames frames :boxes boxes :offsets offsets} + pixels)}))) + +(defn wire-blocks [blocks] + (into-array + (map (fn [role] + (let [{:keys [key descriptor data]} (get blocks role)] + #js {:key key :descriptor descriptor :data (wire/base64 data)})) + roles))) + +(defn unpack + "The three block-detail responses -> inputs for measurement and freeze." + [^js responses [width height]] + (let [by-role (into {} + (map (fn [^js response] + (let [descriptor (js->clj (js/JSON.parse (.-descriptor response)) + :keywordize-keys true)] + [(:role descriptor) + {:layout (:layout descriptor) + :data (wire/typed (get-in descriptor [:layout :type]) + (.-data response))}])) + (array-seq responses))) + _ (when-not (= (set roles) (set (keys by-role))) + (throw (ex-info "analysis is missing source blocks" {:roles (keys by-role)}))) + {:keys [layout data]} (get by-role "source/dense") + frames (:frames layout) + points (:points layout) + mask (:data (get by-role "source/detected")) + crop-data (:data (get by-role "source/crops")) + crop-layout (:layout (get by-role "source/crops"))] + (when-not (and (= (* frames points 3) (.-length data)) + (= frames (.-length mask) (:frames crop-layout)) + (= frames (count (:boxes crop-layout))) + (= (inc frames) (count (:offsets crop-layout)))) + (throw (ex-info "source block layouts disagree" {:frames frames :points points}))) + (let [dense (mapv (fn [f] + (mapv (fn [i] + (let [base (* 3 (+ (* f points) i))] + {:x (aget data base) :y (aget data (inc base)) + :z (aget data (+ base 2))})) + (range points))) + (range frames)) + detected (mapv #(pos? (aget mask %)) (range frames)) + crops (mapv + (fn [f] + (when-let [[x y w h] (nth (:boxes crop-layout) f)] + (let [start (nth (:offsets crop-layout) f) + end (nth (:offsets crop-layout) (inc f)) + shape (mapv #(nth (nth dense f) %) lm/LIPS-INNER)] + (when-not (= (* 4 w h) (- end start)) + (throw (ex-info "mouth crop byte count disagrees with its box" + {:frame f :expected (* 4 w h) :actual (- end start)}))) + {:box {:x x :y y :w w :h h :shape shape + :source-width width :source-height height} + :data (js/Uint8ClampedArray. (.-buffer crop-data) start (- end start))}))) + (range frames))] + {:dense dense :detected detected :crops crops :dimensions [width height] + :missing (count (remove true? detected)) + :first-real (first (keep-indexed (fn [i present?] (when present? i)) detected))}))) diff --git a/frontend/src/arthur/flow/take.cljs b/frontend/src/arthur/flow/take.cljs index a0cce80..3b6c953 100644 --- a/frontend/src/arthur/flow/take.cljs +++ b/frontend/src/arthur/flow/take.cljs @@ -15,6 +15,17 @@ (def knobs params/defaults) +(defn analysis-for [manifest detector] + (let [aspect (/ (:width manifest) (:height manifest))] + (address/analysis + (merge {:detector "mediapipe" :version "unknown"} + detector + {:source (:source manifest) + :footage (:footage manifest) + :frames (:frames manifest) + :fps (:fps manifest) + :aspect aspect})))) + (defn measure "Condition the anchor before measuring rings through it." [{:keys [aspect] :as params} {:keys [dense detected interior presence]}] @@ -66,13 +77,7 @@ ;; version string somebody has to remember to bump. See ;; `flow/address`: a model upgrade that silently reused ;; these landmarks is the failure this prevents. - :analysis (address/analysis - (merge {:detector "mediapipe" :version "unknown"} - detector - {:source (:source manifest) - :footage (:footage manifest) - :frames (:frames manifest) - :fps (:fps manifest) - :aspect (/ w h)}))})] + :analysis (analysis-for (assoc manifest :width w :height h) + detector)})] (build params {:dense dense :detected detected :interior interior :presence presence}))) diff --git a/frontend/src/arthur/fx/http.cljs b/frontend/src/arthur/fx/http.cljs index c83cc61..4ad9cce 100644 --- a/frontend/src/arthur/fx/http.cljs +++ b/frontend/src/arthur/fx/http.cljs @@ -34,14 +34,16 @@ (defn request! ([method url] (request! method url nil)) ([method url body] - (-> (js/fetch url - (clj->js (cond-> {:method method - :cache "no-store" - :headers (cond-> {"Accept" "application/json"} - body (assoc "Content-Type" "application/json") - (not= "GET" method) - (assoc "X-CSRFToken" (or (csrf-token) "")))} - body (assoc :body (js/JSON.stringify body))))) + (let [form? (instance? js/FormData body) + headers #js {"Accept" "application/json"} + options #js {:method method :cache "no-store" :headers headers}] + (when (and body (not form?)) + (aset headers "Content-Type" "application/json")) + (when (not= "GET" method) + (aset headers "X-CSRFToken" (or (csrf-token) ""))) + (when body + (aset options "body" (if form? body (js/JSON.stringify body)))) + (-> (js/fetch url options) (.then (fn [response] (-> (.text response) (.then (fn [text] @@ -49,8 +51,9 @@ (try (js/JSON.parse text) (catch :default _ nil)))] (if (.-ok response) parsed - (fail response parsed))))))))))) + (fail response parsed)))))))))))) (defn GET [url] (request! "GET" url)) (defn POST [url body] (request! "POST" url body)) +(defn POST-form [url body] (request! "POST" url body)) (defn PUT [url body] (request! "PUT" url body)) diff --git a/frontend/src/arthur/ui/shell.cljs b/frontend/src/arthur/ui/shell.cljs index d239f74..8a9e0b1 100644 --- a/frontend/src/arthur/ui/shell.cljs +++ b/frontend/src/arthur/ui/shell.cljs @@ -108,10 +108,14 @@ [:button {:class (when (= r picture-fps) "on") :on-click #(rf/dispatch [::pb/set-picture-fps r])} (if (= r fps) "source" (str r))]))]) - ;; The takes the SERVER holds, since step 9. There is no path to type any - ;; more: `./extract.sh` decodes a clip and `manage.py ingest_bundle` registers - ;; it, and from then on the frames are addressed rather than located. + ;; Upload a video or choose existing server footage. Both routes produce the + ;; same immutable frame and audio manifest. [:label.source-path "footage " + [:input {:type "file" :accept "video/*" :disabled loading? + :on-change (fn [event] + (when-let [file (aget (.. event -target -files) 0)] + (rf/dispatch [::footage/upload file]) + (set! (.. event -target -value) "")))}] [:select {:value (or chosen "") :disabled loading? :on-change #(rf/dispatch [::footage/choose (.. % -target -value)])} (if (seq available) @@ -119,7 +123,7 @@ ^{:key id} [:option {:value id} (str label " · " frames "f @" fps)])) - [:option {:value ""} "nothing ingested"])] + [:option {:value ""} "no footage yet"])] [:button {:disabled loading? :on-click #(rf/dispatch [::footage/refresh])} "refresh"]] (when status [:div.load-status status]) @@ -148,5 +152,5 @@ [audio] [transport] [:p.note - "Extract a clip with extract.sh, then load frames. The manifest supplies " - "the frame count, audio and fps."]]) + "Upload a video, choose its footage, then load frames. Save the project to " + "share the analyzed take without detecting frames again."]]) diff --git a/frontend/test/arthur/domain/wire_test.cljs b/frontend/test/arthur/domain/wire_test.cljs index f55ead6..3e3b40b 100644 --- a/frontend/test/arthur/domain/wire_test.cljs +++ b/frontend/test/arthur/domain/wire_test.cljs @@ -88,4 +88,4 @@ (is (= [3 4] (vec (array-seq (wire/typed "int16" (wire/base64 view)))))))) (deftest an-unknown-element-type-is-refused - (is (thrown-with-msg? ExceptionInfo #"int16" (wire/typed "float64" "AA==")))) + (is (thrown-with-msg? ExceptionInfo #"int16" (wire/typed "float16" "AA==")))) diff --git a/frontend/test/arthur/flow/address_test.cljs b/frontend/test/arthur/flow/address_test.cljs index e9a49d4..1460cd4 100644 --- a/frontend/test/arthur/flow/address_test.cljs +++ b/frontend/test/arthur/flow/address_test.cljs @@ -119,9 +119,13 @@ ;; the thing it is derived FROM is the thing the biconditional above asserts. ;; These two assertions are what keep the derivation honest in the only two ways ;; it could rot. - (testing "every role in the table is reachable from some knob" - (is (= (set (keys address/block-knobs)) + (testing "every knob-dependent role is reachable from some knob" + (is (= (into #{} (keep (fn [[role knobs]] (when (seq knobs) role))) + address/block-knobs) (into #{} (mapcat val) address/knob-roles)))) + (testing "source roles have no knob dependencies" + (doseq [role ["source/dense" "source/detected" "source/crops"]] + (is (= [] (get address/block-knobs role))))) (testing "every knob a block declares is a knob the registry defines" ;; The other direction of the same edge: a typo in `block-knobs` would name a ;; setting no slider can move, and `block-descriptor` only checks that the diff --git a/frontend/test/arthur/flow/source_test.cljs b/frontend/test/arthur/flow/source_test.cljs new file mode 100644 index 0000000..d5f3626 --- /dev/null +++ b/frontend/test/arthur/flow/source_test.cljs @@ -0,0 +1,21 @@ +(ns arthur.flow.source-test + (:require [cljs.test :refer [deftest is]] + [arthur.flow.source :as source])) + +(deftest source-blocks-round-trip-without-source-images + (let [face (vec (repeat 478 {:x 0.25 :y 0.5 :z -0.125})) + pixels (js/Uint8ClampedArray. #js [12 24 36 255]) + input {:dense [face face] :detected [true false] + :crops [{:box {:x 3 :y 4 :w 1 :h 1} :data pixels} nil]} + blocks (source/pack "sha256:analysis" input) + out (source/unpack (source/wire-blocks blocks) [100 80])] + (is (= (set source/roles) (set (keys blocks)))) + (is (= (:dense input) (:dense out))) + (is (= (:detected input) (:detected out))) + (is (= [3 4 1 1] + ((juxt :x :y :w :h) (:box (first (:crops out)))))) + (is (= [12 24 36 255] + (vec (array-seq (:data (first (:crops out))))))) + (is (nil? (second (:crops out)))) + (is (= (mapv :key (vals blocks)) + (mapv :key (vals (source/pack "sha256:analysis" out))))))) diff --git a/frontend/test/browser/take.mjs b/frontend/test/browser/take.mjs index 8ef8381..1adf6ea 100644 --- a/frontend/test/browser/take.mjs +++ b/frontend/test/browser/take.mjs @@ -24,7 +24,7 @@ // Writes a PNG per sampled frame into test/browser/out/ so that "it drew // something" can be checked by eye as well as by pixel count. -import { spawn } from 'node:child_process'; +import { spawn, spawnSync } from 'node:child_process'; import { mkdirSync, writeFileSync, rmSync } from 'node:fs'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -356,9 +356,15 @@ async function main() { check(await page.eval(CLICK('save')), 'save is clickable'); const saved = await statusMatching(/saved r\d+/); check(saved !== null, 'the document saves', saved ?? (await page.eval(STATUS))); - // Eleven blocks the first time. The COUNT is not asserted — that is a fact - // about the freeze, not about saving — but that some went up is. - check(/· [1-9]\d* blocks?/.test(saved ?? ''), 'and its tier 2 went with it', saved ?? ''); + // Upload count may be zero when the content-addressed blocks already exist + // on this server. The saved clip must still reference them. + const savedBlocks = await page.eval(`(async () => { + const listed = await fetch('/api/projects').then(r => r.json()); + const project = await fetch('/api/projects/' + listed.projects[0].id).then(r => r.json()); + return project.clips[0].blocks.length; + })()`); + check(savedBlocks > 0, 'the saved clip references its tier 2 blocks', + `${savedBlocks} blocks`); // Again, unchanged. Content addressing means the second save uploads nothing // and rewrites nothing: this is the assertion that the keys are stable across @@ -387,6 +393,33 @@ async function main() { check(!back[0].selectedClip.includes('take'), 'the picture is the reopened document', JSON.stringify(back[0].selectedClip)); + // Exercise the actual file input and FormData path, then wait for the + // extraction job's footage to appear in the server list. + const uploadDir = mkdtempSync(join(tmpdir(), 'arthur-upload-')); + try { + const video = join(uploadDir, 'browser-upload.mp4'); + const encoded = spawnSync('ffmpeg', [ + '-hide_banner', '-loglevel', 'error', '-y', '-f', 'lavfi', + '-i', 'color=c=blue:s=64x48:r=4:d=1', '-c:v', 'mpeg4', video, + ]); + if (encoded.status !== 0) throw new Error(`ffmpeg failed: ${encoded.stderr}`); + const { root } = await page.send('DOM.getDocument'); + const { nodeId } = await page.send('DOM.querySelector', { + nodeId: root.nodeId, selector: 'input[type=file]', + }); + await page.send('DOM.setFileInputFiles', { files: [video], nodeId }); + const extracted = await statusMatching(/video extracted/, 160); + check(extracted !== null, 'the page uploads and extracts a video', + extracted ?? (await page.eval(STATUS))); + const uploadedFootage = await page.eval(`(async () => { + const listed = await fetch('/api/footage').then(r => r.json()); + return listed.footage.some(f => f.label === 'browser-upload.mp4' && f.frames === 4); + })()`); + check(uploadedFootage, 'the uploaded footage is selectable'); + } finally { + rmSync(uploadDir, { recursive: true, force: true }); + } + check(page.logs.length === 0, 'no errors on the console', page.logs.slice(0, 3).join(' | ')); } finally {