diff --git a/.gitignore b/.gitignore index 83b39c3..ab2e70d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,11 @@ +# extract.sh's output. It is TIER 3 — immutable, large, and the backend's to serve +# once `manage.py ingest_bundle` has hashed it into var/blobs — so none of it +# belongs in the repo. `audio.wav` was tracked before step 9 because the synthetic +# take borrowed it for a clock; that copy now lives at static/arthur/audio.wav, +# which is an asset the project owns rather than an extraction that churns. frames/ +/audio.wav +/manifest.json # local extracted takes for comparing source cadences /scratch/ *.task @@ -17,3 +24,15 @@ static/arthur/js/ # mise-managed venv for the Django half .venv/ + +# the Django half's own state: the document database, the content-addressed blob +# store (tiers 2 and 3), and collectstatic's output +db.sqlite3 +/var/ + +# vim swap files +*.swp + +# Python bytecode +__pycache__/ +*.py[cod] diff --git a/README.md b/README.md index 7ab6dcb..01ce7dd 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,38 @@ modern conveniences belong in the workflow, not the output. See ## ClojureScript port -The active port has reached [step 7 of the port plan](docs/port-plan.md): it plays -the synthetic take and can load extracted real footage with mouth, eyes, brows, -and pixel-derived teeth. The step 8 data model now represents persistent feature -IDs, eye pairs and feature-level observation gaps; its controls are still pending. -See [frontend/README.md](frontend/README.md) for setup and the **load frames** -workflow. The rest of this README describes the older JS prototype, which still -runs separately on port 8777. +The active port has reached [step 9 of the port plan](docs/port-plan.md): it plays +the synthetic take, loads real footage with mouth, eyes, brows and pixel-derived +teeth, and now has a Django backend that persists the document. The step 8 data +model represents persistent feature IDs, eye pairs and feature-level observation +gaps; its controls are still pending. + +```sh +mise install # both halves +pip install -r requirements.txt +mise exec -- python manage.py migrate +mise exec -- python manage.py runserver 8778 # then open localhost:8778 +cd frontend && mise exec -- npx shadow-cljs watch app +``` + +See [frontend/README.md](frontend/README.md) for the **load frames** and **save** +workflows. Anything under "## Run" and below describes the older JS prototype, +which still runs separately on port 8777. + +### How it is stored + +Three tiers, cut by mutability and size — the full argument is in +[docs/architecture.md](docs/architecture.md): + +| Tier | What | Where | +| --- | --- | --- | +| 1 **authored** | the scene: nodes, channels, features, time maps | the database, as independently addressed leaves. Kilobytes | +| 2 **derived** | the dense channel blocks | `var/blobs`, content-addressed by a hash over every input — including the detector version | +| 3 **source** | frames and audio | the same blob store, by the hash of their bytes | + +Only tier 1 is the document. Tier 2 is a pure function of tiers 1 and 3, so a +saved project names its blocks rather than carrying them, and a knob change gives +a block a new name rather than overwriting an old one. ## Run @@ -383,3 +408,16 @@ performer→character calibration (currently identity, fitting the face oval to canvas); the override layer; anything on the Animator Pro side. The plate is a face-oval polygon per kept frame — it exists so the mouth has a face to read against, not to look good. + +In the port specifically: the parameter UI and scoped regeneration (the model is +built, the controls are not); automatic per-feature detection, so presence still +comes from the full-face mask plus a manifest annotation; multiplayer, for which +step 9 built the addressing and none of the socket; and in-browser extraction, so +`extract.sh` plus `manage.py ingest_bundle` is still how footage arrives. + +Two smaller things that are known and undecided. `measure/brows` takes no +`presence` where `measure/eyes` does, so an occluded brow affects the freeze mask +but not brow measurement, and occluded landmarks still enter contour smoothing — +asymmetric with the eyes, and it is not settled which way is right. And `open` +takes the most recently updated project and shows its first clip: there is no +project browser, and the runtime store holds one clip at a time. diff --git a/audio.wav b/audio.wav deleted file mode 100644 index f5793b0..0000000 Binary files a/audio.wav and /dev/null differ diff --git a/clips/__init__.py b/clips/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clips/admin.py b/clips/admin.py new file mode 100644 index 0000000..9e73270 --- /dev/null +++ b/clips/admin.py @@ -0,0 +1,63 @@ +"""The admin, which is here for one reason: tier 1 is readable. + +docs/architecture.md's argument against a CRDT is partly this — "the canonical +document moves into an opaque blob, and every server-side thing that reads the +document needs it materialised back out". A leaf is transit-as-JSON in a +JSONField, so it is legible here, and that is a property worth being able to see. +""" +from django.contrib import admin + +from .models import Analysis, Block, Blob, Clip, Footage, FootageFrame, Leaf, Project, Revision + + +@admin.register(Project) +class ProjectAdmin(admin.ModelAdmin): + list_display = ("name", "id", "seq", "updated") + search_fields = ("name", "id") + + +@admin.register(Clip) +class ClipAdmin(admin.ModelAdmin): + list_display = ("cid", "project", "name", "footage", "analysis") + list_filter = ("project",) + + +@admin.register(Leaf) +class LeafAdmin(admin.ModelAdmin): + list_display = ("path", "project", "version", "updated") + list_filter = ("project",) + search_fields = ("path",) + + +@admin.register(Revision) +class RevisionAdmin(admin.ModelAdmin): + list_display = ("project", "seq", "summary", "author", "created") + + +@admin.register(Footage) +class FootageAdmin(admin.ModelAdmin): + list_display = ("label", "source", "fps", "frames", "width", "height", "created") + + +@admin.register(FootageFrame) +class FootageFrameAdmin(admin.ModelAdmin): + list_display = ("footage", "index", "blob") + list_filter = ("footage",) + + +@admin.register(Analysis) +class AnalysisAdmin(admin.ModelAdmin): + list_display = ("key", "detector", "version", "footage", "created") + search_fields = ("key", "detector", "version") + + +@admin.register(Block) +class BlockAdmin(admin.ModelAdmin): + list_display = ("key", "role", "analysis", "data", "state", "created") + list_filter = ("role",) + search_fields = ("key",) + + +@admin.register(Blob) +class BlobAdmin(admin.ModelAdmin): + list_display = ("digest", "media_type", "size", "created") diff --git a/clips/apps.py b/clips/apps.py new file mode 100644 index 0000000..6b51579 --- /dev/null +++ b/clips/apps.py @@ -0,0 +1,14 @@ +from django.apps import AppConfig + + +class ClipsConfig(AppConfig): + """The one app. + + `clips` because the CLIP is the entity the whole tool is about and the one the + prototype had exactly one of and never named — `state` in `js/app.js` is a clip + with its analysis inlined and its palette global. Project, Footage, Analysis, + Block, Leaf and Revision all hang off it. + """ + + default_auto_field = "django.db.models.BigAutoField" + name = "clips" diff --git a/clips/blobs.py b/clips/blobs.py new file mode 100644 index 0000000..c9e0f3e --- /dev/null +++ b/clips/blobs.py @@ -0,0 +1,104 @@ +"""The content-addressed blob store: tiers 2 and 3 on disk. + +One store for both, and docs/architecture.md says why in a sentence: once tier 3 +is decoded by the app rather than by a shell script, frames and audio become "the +same kind of thing as tier 2 — a cache with a hash". So there is one place that +writes bytes, one that reads them, and one URL shape for both. + +TWO KINDS OF HASH, AND THEY ARE NOT THE SAME HASH. A blob is named by the sha256 +of its BYTES: that is what makes identical frames in two extractions one file. A +derived thing — an analysis artifact, a dense block — is named by a sha256 over +its INPUTS, which is what lets the client ask for the block the current settings +want before anything has computed it. So `Block.key` is an input hash and +`Block.data.digest` is a byte hash, and conflating them would break the half of +addressing that answers questions about work not yet done. +""" +import hashlib +import os +from pathlib import Path + +from django.conf import settings + +CHUNK = 1 << 20 + + +def digest_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def digest_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + while chunk := fh.read(CHUNK): + h.update(chunk) + return h.hexdigest() + + +def path_for(digest: str) -> Path: + """Where a blob lives. + + Fanned out two levels, so that a take's worth of frames does not put a hundred + thousand entries in one directory — which is slow on every filesystem and + unusable on some. + """ + if len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise ValueError(f"not a sha256: {digest!r}") + return Path(settings.BLOB_ROOT) / digest[:2] / digest[2:4] / digest + + +def write(data: bytes) -> tuple[str, int]: + """Store bytes, return (digest, size). Writing the same bytes twice is a + no-op, which is what content addressing is for.""" + digest = digest_bytes(data) + dest = path_for(digest) + if not dest.exists(): + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_suffix(".part") + with open(tmp, "wb") as fh: + fh.write(data) + os.replace(tmp, dest) + return digest, len(data) + + +def adopt(source: Path) -> tuple[str, int]: + """Store a file already on disk, by hard link where the filesystem allows it. + + 112MB of PNGs is a normal extraction and copying them into a second place in + the tree for no reason is not. A hard link is exact — the blob is immutable, so + two names for one inode is the whole of what is wanted — and a copy is the + fallback when `extract.sh` wrote to another volume. + """ + digest = digest_file(source) + dest = path_for(digest) + size = source.stat().st_size + if not dest.exists(): + dest.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(source, dest) + except OSError: + tmp = dest.with_suffix(".part") + with open(source, "rb") as src, open(tmp, "wb") as out: + while chunk := src.read(CHUNK): + out.write(chunk) + os.replace(tmp, dest) + return digest, size + + +def read(digest: str) -> bytes: + with open(path_for(digest), "rb") as fh: + return fh.read() + + +def png_size(path: Path) -> tuple[int, int]: + """A PNG's dimensions, out of its IHDR. + + Twenty-four bytes rather than a dependency. The footage's width and height are + manifest data — docs/architecture.md's entity model puts them there — and + Pillow to read two integers out of a header that has held them in the same + place since 1996 is not a trade worth making. + """ + with open(path, "rb") as fh: + head = fh.read(24) + if head[:8] != b"\x89PNG\r\n\x1a\n" or head[12:16] != b"IHDR": + raise ValueError(f"{path} is not a PNG") + return int.from_bytes(head[16:20], "big"), int.from_bytes(head[20:24], "big") diff --git a/clips/management/__init__.py b/clips/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clips/management/commands/__init__.py b/clips/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clips/management/commands/ingest_bundle.py b/clips/management/commands/ingest_bundle.py new file mode 100644 index 0000000..89ca5b5 --- /dev/null +++ b/clips/management/commands/ingest_bundle.py @@ -0,0 +1,120 @@ +"""Register an extracted bundle as tier 3. + + python manage.py ingest_bundle # ./manifest.json + python manage.py ingest_bundle scratch/my-take # that bundle + +WHAT THIS REPLACES. Until step 9 the page fetched `/manifest.json` and then built +`frames/0001.png` itself, with shadow-cljs's `:dev-http` serving the repo root. So +the frame layout was a shared secret between a shell script and a ClojureScript +namespace, and "where the frames are" was answered by a directory listing. + +Now the server names every frame, and the client asks it. The frames go into the +content-addressed blob store — by hard link, so 112MB of PNGs is not copied — and +the manifest the client receives carries a URL per frame. That is the whole of what +makes the frames the backend's to serve, and it is what the in-browser wasm-ffmpeg +extraction docs/architecture.md describes will upload INTO, without the client +learning anything new when it arrives: the same blobs, the same manifest, a +different producer. + +`extract.sh` still does the decoding. It is out of step 9's scope, it works, and it +is the only part of this that needs a terminal. +""" +import json +from pathlib import Path + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from clips import blobs +from clips.models import Blob, Footage, FootageFrame + + +class Command(BaseCommand): + help = "Register an extracted frames+audio+manifest bundle as footage." + + def add_arguments(self, parser): + parser.add_argument( + "bundle", nargs="?", default=".", + help="a directory holding manifest.json, or the manifest itself", + ) + parser.add_argument("--label", default="", help="what to call it in the UI") + + def handle(self, *args, **options): + manifest_path = Path(options["bundle"]) + if manifest_path.is_dir(): + manifest_path = manifest_path / "manifest.json" + if not manifest_path.exists(): + raise CommandError(f"{manifest_path} does not exist — run ./extract.sh first") + + manifest = json.loads(manifest_path.read_text()) + root = manifest_path.parent + frames_dir = root / manifest["dir"] + audio_path = root / manifest["audio"] + count = int(manifest["frames"]) + + pngs = sorted(frames_dir.glob("*.png")) + if len(pngs) != count: + raise CommandError( + f"the manifest says {count} frames and {frames_dir} holds {len(pngs)}; " + "refusing an inaccurate footage" + ) + if not audio_path.exists(): + raise CommandError(f"{audio_path} does not exist") + + width, height = blobs.png_size(pngs[0]) + + self.stdout.write(f"hashing {len(pngs)} frames…") + frame_blobs = [] + for i, png in enumerate(pngs): + digest, size = blobs.adopt(png) + frame_blobs.append((i, digest, size)) + if (i + 1) % 25 == 0 or i + 1 == len(pngs): + self.stdout.write(f" {i + 1}/{len(pngs)}") + + audio_digest, audio_size = blobs.adopt(audio_path) + + # The footage's own identity: every frame in order, plus the audio and the + # rate. Two extractions of one clip at one rate are one footage, so an + # analysis over it is reusable across both. + import hashlib + + h = hashlib.sha256() + h.update(f"arthur-footage-1/{manifest['fps']}/{count}/{width}x{height}\n".encode()) + for _, digest, _ in frame_blobs: + h.update(digest.encode()) + h.update(audio_digest.encode()) + footage_digest = h.hexdigest() + + if existing := Footage.objects.filter(digest=footage_digest).first(): + self.stdout.write(self.style.SUCCESS(f"already ingested: {existing.id}")) + return + + with transaction.atomic(): + audio_blob, _ = Blob.objects.get_or_create( + digest=audio_digest, + defaults={"size": audio_size, "media_type": "audio/wav"}, + ) + footage = Footage.objects.create( + digest=footage_digest, + label=options["label"] or manifest.get("source") or frames_dir.name, + source=manifest.get("source") or "", + fps=float(manifest["fps"]), + frames=count, + width=width, + height=height, + audio=audio_blob, + feature_absence=manifest.get("feature-absence") or {}, + ) + rows = [] + for index, digest, size in frame_blobs: + blob, _ = Blob.objects.get_or_create( + digest=digest, defaults={"size": size, "media_type": "image/png"} + ) + rows.append(FootageFrame(footage=footage, index=index, blob=blob)) + FootageFrame.objects.bulk_create(rows) + + self.stdout.write( + self.style.SUCCESS( + f"{count} frames at {manifest['fps']}fps, {width}x{height} -> footage {footage.id}" + ) + ) diff --git a/clips/migrations/0001_initial.py b/clips/migrations/0001_initial.py new file mode 100644 index 0000000..15e07ba --- /dev/null +++ b/clips/migrations/0001_initial.py @@ -0,0 +1,149 @@ +# Generated by Django 5.2.17 on 2026-09-28 04:44 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Blob', + fields=[ + ('digest', models.CharField(max_length=64, primary_key=True, serialize=False)), + ('media_type', models.CharField(default='application/octet-stream', max_length=100)), + ('size', models.BigIntegerField()), + ('created', models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name='Project', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(default='untitled', max_length=200)), + ('seq', models.PositiveBigIntegerField(default=0)), + ('palette', models.CharField(default='arthur/default', max_length=64)), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ], + options={ + 'ordering': ['-updated'], + }, + ), + migrations.CreateModel( + name='Analysis', + fields=[ + ('key', models.CharField(max_length=71, primary_key=True, serialize=False)), + ('descriptor', models.TextField()), + ('detector', models.CharField(max_length=64)), + ('version', models.CharField(max_length=64)), + ('created', models.DateTimeField(auto_now_add=True)), + ('artifact', models.ForeignKey(blank=True, help_text='the dense landmark track, once bake A is uploaded', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='analysis_for', to='clips.blob')), + ], + options={ + 'verbose_name_plural': 'analyses', + }, + ), + migrations.CreateModel( + name='Block', + fields=[ + ('key', models.CharField(max_length=71, primary_key=True, serialize=False)), + ('descriptor', models.TextField()), + ('role', models.CharField(max_length=32)), + ('created', models.DateTimeField(auto_now_add=True)), + ('analysis', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='blocks', to='clips.analysis')), + ('data', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='block_data_for', to='clips.blob')), + ('state', models.ForeignKey(blank=True, help_text='the per-track absence mask, when the take has one', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='block_state_for', to='clips.blob')), + ], + ), + migrations.CreateModel( + name='Footage', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('digest', models.CharField(max_length=64, unique=True)), + ('label', models.CharField(blank=True, max_length=200)), + ('source', models.CharField(blank=True, max_length=200)), + ('fps', models.FloatField()), + ('frames', models.PositiveIntegerField()), + ('width', models.PositiveIntegerField()), + ('height', models.PositiveIntegerField()), + ('feature_absence', models.JSONField(blank=True, default=dict)), + ('created', models.DateTimeField(auto_now_add=True)), + ('audio', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='audio_for', to='clips.blob')), + ], + options={ + 'ordering': ['-created'], + }, + ), + migrations.AddField( + model_name='analysis', + name='footage', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='analyses', to='clips.footage'), + ), + migrations.CreateModel( + name='Revision', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('seq', models.PositiveBigIntegerField()), + ('author', models.CharField(blank=True, max_length=200)), + ('summary', models.CharField(blank=True, max_length=500)), + ('document', models.JSONField(help_text='every leaf of the project, by path')), + ('created', models.DateTimeField(auto_now_add=True)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions', to='clips.project')), + ], + options={ + 'ordering': ['-seq'], + }, + ), + migrations.CreateModel( + name='FootageFrame', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('index', models.PositiveIntegerField(help_text="0-based; source frame index + 1 is the PNG's name")), + ('blob', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='frame_for', to='clips.blob')), + ('footage', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='frame_set', to='clips.footage')), + ], + options={ + 'ordering': ['index'], + 'constraints': [models.UniqueConstraint(fields=('footage', 'index'), name='one_blob_per_frame')], + }, + ), + migrations.CreateModel( + name='Leaf', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('path', models.CharField(max_length=300)), + ('value', models.JSONField()), + ('version', models.PositiveBigIntegerField(default=1)), + ('updated', models.DateTimeField(auto_now=True)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='leaves', to='clips.project')), + ], + options={ + 'ordering': ['path'], + 'constraints': [models.UniqueConstraint(fields=('project', 'path'), name='one_leaf_per_path')], + }, + ), + migrations.CreateModel( + name='Clip', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('cid', models.SlugField(max_length=64)), + ('name', models.CharField(blank=True, max_length=200)), + ('order', models.IntegerField(default=0)), + ('analysis', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='clips', to='clips.analysis')), + ('blocks', models.ManyToManyField(blank=True, help_text="the tier-2 blocks this clip's channels name", related_name='clips', to='clips.block')), + ('footage', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='clips', to='clips.footage')), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='clips', to='clips.project')), + ], + options={ + 'ordering': ['order', 'cid'], + 'constraints': [models.UniqueConstraint(fields=('project', 'cid'), name='one_cid_per_project')], + }, + ), + ] diff --git a/clips/migrations/__init__.py b/clips/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clips/models.py b/clips/models.py new file mode 100644 index 0000000..e083808 --- /dev/null +++ b/clips/models.py @@ -0,0 +1,261 @@ +"""The entity model, as tables. + +It follows docs/architecture.md's model exactly, and the one thing worth reading +it for is which tier each table is in, because that is what decides whether a row +is a document, a cache entry or a source. + + TIER 1, the document. Project, Clip, Leaf, Revision. Kilobytes, authored, + versioned, and the only tier anything will ever sync. + + TIER 2, derived. Analysis, Block. Content-addressed by a hash over every input + that produced them — including the detector version — so a stale bake is + unreachable rather than wrong, and a collaborator's bake is fetchable by the + same key. + + TIER 3, source. Footage, FootageFrame. Immutable, by hash. + + Blob is under all three of them: bytes, named by the sha256 of themselves. + +WHAT IS DELIBERATELY NOT HERE. `Clip` does not store fps, frames, width or height. +They are in the document — the `timing` and `stage` leaves — and a copy of them in +a column is a copy that comes to disagree with the scene it describes. The columns +`Clip` does have are the ones the SERVER needs to answer a question about a clip +without parsing its leaves: which footage, which analysis, which blocks. +""" +import uuid + +from django.db import models + + +class Blob(models.Model): + """Bytes, named by the sha256 of themselves. The file is on disk under + `BLOB_ROOT`; this row is the index and the size.""" + + digest = models.CharField(primary_key=True, max_length=64) + media_type = models.CharField(max_length=100, default="application/octet-stream") + size = models.BigIntegerField() + created = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f"{self.digest[:12]}… {self.size}B {self.media_type}" + + +class Footage(models.Model): + """Tier 3: the frames and audio of one extraction, immutable. + + `digest` is over the ordered frame digests plus the audio's, so two + extractions of the same clip at the same rate are one footage and the same + analysis can be reused across both. + + `feature_absence` is the manifest annotation step 8 introduced: known + occlusion intervals, one-based and inclusive, expanded into presence tracks by + the loader. An input format, not a control UI. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + digest = models.CharField(max_length=64, unique=True) + label = models.CharField(max_length=200, blank=True) + source = models.CharField(max_length=200, blank=True) + fps = models.FloatField() + frames = models.PositiveIntegerField() + width = models.PositiveIntegerField() + height = models.PositiveIntegerField() + audio = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="audio_for") + feature_absence = models.JSONField(default=dict, blank=True) + created = models.DateTimeField(auto_now_add=True) + + class Meta: + ordering = ["-created"] + + def __str__(self): + return f"{self.label or self.source or self.id} ({self.frames}f @{self.fps})" + + +class FootageFrame(models.Model): + """One source frame. A row rather than an entry in a JSON list, because a + frame is a thing the server serves, and because a blob's references have to be + countable before anything can be collected.""" + + footage = models.ForeignKey(Footage, on_delete=models.CASCADE, related_name="frame_set") + index = models.PositiveIntegerField(help_text="0-based; source frame index + 1 is the PNG's name") + blob = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="frame_for") + + class Meta: + ordering = ["index"] + constraints = [ + models.UniqueConstraint(fields=["footage", "index"], name="one_blob_per_frame"), + ] + + +class Analysis(models.Model): + """Tier 2: one detector, at one version, over one footage. + + `key` is a content address over every input, and `descriptor` is the exact + canonical text that key is the sha256 of — sent by the client and stored, not + recomputed here. `clips/views.py` says why that is the honest arrangement: JS + prints an integral double as `1` and Python as `1.0`, so a scheme where both + sides re-render the numbers breaks on the first one of them. + + `detector` and `version` are columns as well as descriptor fields so that the + question "which model produced this take" is answerable in the admin and in a + query, rather than only by parsing a hash's preimage. + """ + + key = models.CharField(primary_key=True, max_length=71) + descriptor = models.TextField() + detector = models.CharField(max_length=64) + version = models.CharField(max_length=64) + footage = models.ForeignKey( + Footage, null=True, blank=True, on_delete=models.SET_NULL, related_name="analyses" + ) + artifact = models.ForeignKey( + Blob, null=True, blank=True, on_delete=models.SET_NULL, related_name="analysis_for", + help_text="the dense landmark track, once bake A is uploaded", + ) + created = models.DateTimeField(auto_now_add=True) + + class Meta: + verbose_name_plural = "analyses" + + def __str__(self): + return f"{self.detector} {self.version} → {self.key[7:19]}…" + + +class Block(models.Model): + """Tier 2: one dense channel block. + + Two hashes, and they are not the same hash. `key` is over the block's INPUTS, + which is what lets a client ask for the block its current settings want before + anything has computed it. `data.digest` is over the bytes. See clips/blobs.py. + """ + + key = models.CharField(primary_key=True, max_length=71) + descriptor = models.TextField() + role = models.CharField(max_length=32) + analysis = models.ForeignKey( + Analysis, null=True, blank=True, on_delete=models.SET_NULL, related_name="blocks" + ) + data = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="block_data_for") + state = models.ForeignKey( + Blob, null=True, blank=True, on_delete=models.PROTECT, related_name="block_state_for", + help_text="the per-track absence mask, when the take has one", + ) + created = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f"{self.role} {self.key[7:19]}…" + + +class Project(models.Model): + """Tier 1: the document's root. + + `seq` is the monotonic project version docs/architecture.md asks for. Every + write bumps it, and a client that sees `seq > local + 1` refetches — which is + what makes staleness self-healing rather than permanent once there is a + broadcast to miss. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=200, default="untitled") + seq = models.PositiveBigIntegerField(default=0) + palette = models.CharField(max_length=64, default="arthur/default") + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["-updated"] + + def __str__(self): + return f"{self.name} ({self.id})" + + def bump(self): + self.seq += 1 + self.save(update_fields=["seq", "updated"]) + return self.seq + + +class Clip(models.Model): + """Tier 1: the unit of work, and the thing leaf paths are scoped by. + + `cid` is what appears in `clip//...`, so it is the clip's identity as far + as addressing is concerned and it does not change. + """ + + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="clips") + cid = models.SlugField(max_length=64) + name = models.CharField(max_length=200, blank=True) + order = models.IntegerField(default=0) + footage = models.ForeignKey( + Footage, null=True, blank=True, on_delete=models.SET_NULL, related_name="clips" + ) + analysis = models.ForeignKey( + Analysis, null=True, blank=True, on_delete=models.SET_NULL, related_name="clips" + ) + blocks = models.ManyToManyField( + Block, blank=True, related_name="clips", + help_text="the tier-2 blocks this clip's channels name", + ) + + class Meta: + ordering = ["order", "cid"] + constraints = [ + models.UniqueConstraint(fields=["project", "cid"], name="one_cid_per_project"), + ] + + def __str__(self): + return f"{self.cid} of {self.project.name}" + + +class Leaf(models.Model): + """Tier 1: one independently addressed, independently versioned piece of the + document. + + The value is transit-as-JSON in a JSONField, so the column holds JSON rather + than a string containing JSON: the admin can read a leaf, and the field-wise + merge of a channel leaf that docs/architecture.md describes as fifteen lines of + Python is possible over it. `version` is the entity tag a conditional write + compares — RFC 7232, not a bespoke invention. + """ + + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="leaves") + path = models.CharField(max_length=300) + value = models.JSONField() + version = models.PositiveBigIntegerField(default=1) + updated = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["path"] + constraints = [ + models.UniqueConstraint(fields=["project", "path"], name="one_leaf_per_path"), + ] + + @property + def etag(self): + return f'"{self.version}"' + + def __str__(self): + return f"{self.path}@{self.version}" + + +class Revision(models.Model): + """Tier 1: a snapshot of the authored layer, with a user and a summary. + + ON AN EXPLICIT TRIGGER, not on every save. tl snapshots a small annotation + layer; arthur's tier 1 will contain cel polygons, so a snapshot per save bloats + the table — docs/architecture.md's "revisions need a coarser trigger". So this + is written by `POST /api/projects//revisions`, which is a "mark version" + button, and never by a save. + """ + + project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="revisions") + seq = models.PositiveBigIntegerField() + author = models.CharField(max_length=200, blank=True) + summary = models.CharField(max_length=500, blank=True) + document = models.JSONField(help_text="every leaf of the project, by path") + created = models.DateTimeField(auto_now_add=True) + + class Meta: + ordering = ["-seq"] + + def __str__(self): + return f"{self.project.name} r{self.seq}: {self.summary}" diff --git a/frontend/public/index.html b/clips/templates/clips/index.html similarity index 64% rename from frontend/public/index.html rename to clips/templates/clips/index.html index 782f667..2a75220 100644 --- a/frontend/public/index.html +++ b/clips/templates/clips/index.html @@ -1,7 +1,17 @@ - - +{% load static %} +{% comment %} +The host page, served by Django since port-plan step 9. + +It was `frontend/public/index.html`, served by shadow-cljs's `:dev-http`, and that +key is gone. The bundle is unchanged: shadow-cljs writes it into +`static/arthur/js` and staticfiles serves it from there, so `manage.py runserver` +and `shadow-cljs watch app` are the whole dev loop with nothing copying files +between them. + +The CSRF token is rendered so that Django sets its cookie, which is what +`arthur.fx.http` reads to write the `X-CSRFToken` header. Saves are ordinary POSTs +and PUTs with ordinary CSRF protection — no endpoint in this app is exempt. +{% endcomment %} @@ -34,16 +44,17 @@ .picture-rate { display: flex; align-items: center; gap: 6px; margin-top: 7px; font-size: 12px; } .source-path { display: block; margin-top: 8px; font-size: 12px; opacity: .7; } - .source-path input { width: 300px; margin-left: 8px; padding: 3px 5px; - color: var(--fg); background: #1c1f2b; border: 1px solid #2b3040; - font: inherit; } + .source-path select { margin: 0 8px; padding: 3px 5px; + color: var(--fg); background: #1c1f2b; border: 1px solid #2b3040; + font: inherit; max-width: 360px; } .load-status { margin-top: 6px; font-size: 12px; opacity: .75; } .note { opacity: .35; font-size: 12px; max-width: 640px; } + {% csrf_token %}
- - + + diff --git a/clips/tests/__init__.py b/clips/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/clips/tests/test_api.py b/clips/tests/test_api.py new file mode 100644 index 0000000..69b395e --- /dev/null +++ b/clips/tests/test_api.py @@ -0,0 +1,476 @@ +"""What the server guarantees, as opposed to what the client intends. + +The two interesting groups here are the ones that make the tier split a property +of the system rather than a convention in ClojureScript: + + A KEY DESCRIBES ITS BYTES. The server recomputes every tier-2 key it is handed + and refuses a mismatch, so nothing can store a block under a name that is not + the hash of its own descriptor. + + A BLOCK CAN NAME ITS DETECTOR VERSION. Every block names an analysis and every + analysis declares a detector and a version, both enforced here. That chain is + what docs/architecture.md asks for: without it, a model upgrade that silently + reuses old landmarks presents as "the tool got worse" with no event to attach it + to. + +The rest is the load/save round trip, the conditional write, and the footage +manifest that makes the frames the backend's to serve. +""" +import hashlib +import json +import struct +import tempfile +import zlib +from pathlib import Path + +from django.test import TestCase, override_settings + +from clips import blobs +from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision + +BLOB_DIR = tempfile.mkdtemp(prefix="arthur-test-blobs-") + + +def key_for(descriptor: str) -> str: + return "sha256:" + hashlib.sha256(descriptor.encode("utf-8")).hexdigest() + + +def analysis_descriptor(version="1.0.1"): + # Canonical JSON, written the way arthur.domain.canon writes it: sorted keys, + # no spaces, integral doubles with no point. + return ('{"aspect":1,"detector":"mediapipe","frames":48,"fps":30,"scheme":1,' + f'"version":"{version}"}}') + + +def block_descriptor(analysis_key, role="geom", anchor_avg=2): + return (f'{{"analysis":"{analysis_key}","features":["mouth"],' + f'"layout":{{"frames":48,"scale":16384,"stride":16,"tracks":1,"type":"int16"}},' + f'"observation":null,"params":{{"anchor-avg":{anchor_avg}}},' + f'"role":"{role}","scheme":1,"tracks":["outer"]}}') + + +def png(width=4, height=3): + """The smallest valid PNG of a given size, written by hand. + + So that `blobs.png_size` and the ingest path are exercised without Pillow. The + one thing the backend needs from a PNG is its IHDR, and this is a PNG with one. + """ + def chunk(kind, payload): + return (struct.pack(">I", len(payload)) + kind + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)) + + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + raw = b"".join(b"\x00" + b"\x40\x40\x40" * width for _ in range(height)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) + + +@override_settings(BLOB_ROOT=BLOB_DIR) +class BlobStoreTests(TestCase): + def test_the_same_bytes_are_stored_once(self): + a, size = blobs.write(b"the same bytes") + b, _ = blobs.write(b"the same bytes") + self.assertEqual(a, b) + self.assertEqual(size, 14) + self.assertEqual(blobs.read(a), b"the same bytes") + + def test_a_path_that_is_not_a_hash_is_refused(self): + # The blob route takes its digest from the URL, so this is the check that + # stops `/blob/../../etc/passwd` being a path at all. + with self.assertRaises(ValueError): + blobs.path_for("../../etc/passwd") + with self.assertRaises(ValueError): + blobs.path_for("deadbeef") + + def test_a_png_reports_its_own_size(self): + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as fh: + fh.write(png(17, 5)) + self.assertEqual((17, 5), blobs.png_size(Path(fh.name))) + + def test_a_blob_is_served_immutable(self): + digest, size = blobs.write(b"bytes on the wire") + Blob.objects.create(digest=digest, size=size, media_type="application/octet-stream") + response = self.client.get(f"/blob/{digest}") + self.assertEqual(200, response.status_code) + self.assertIn("immutable", response["Cache-Control"]) + self.assertEqual(f'"{digest}"', response["ETag"]) + self.assertEqual(b"bytes on the wire", b"".join(response.streaming_content)) + + def test_an_unknown_blob_is_a_404_and_not_a_traceback(self): + self.assertEqual(404, self.client.get("/blob/" + "0" * 64).status_code) + self.assertEqual(404, self.client.get("/blob/nonsense").status_code) + + +@override_settings(BLOB_ROOT=BLOB_DIR) +class Tier2Tests(TestCase): + def post(self, url, payload): + return self.client.post(url, data=json.dumps(payload), + content_type="application/json") + + def register_analysis(self, version="1.0.1"): + descriptor = analysis_descriptor(version) + key = key_for(descriptor) + response = self.post("/api/analyses", { + "key": key, "descriptor": descriptor, + "detector": "mediapipe", "version": version, + }) + self.assertEqual(201, response.status_code, response.content) + return key + + def test_an_analysis_is_its_own_descriptors_hash(self): + key = self.register_analysis() + row = Analysis.objects.get(key=key) + self.assertEqual("mediapipe", row.detector) + self.assertEqual("1.0.1", row.version) + # Idempotent: the same inputs are the same key are the same row. + again = self.post("/api/analyses", { + "key": key, "descriptor": analysis_descriptor(), "detector": "mediapipe", + "version": "1.0.1", + }) + self.assertEqual(200, again.status_code) + self.assertEqual(1, Analysis.objects.count()) + + def test_a_key_that_is_not_the_hash_of_its_descriptor_is_refused(self): + response = self.post("/api/analyses", { + "key": "sha256:" + "0" * 64, "descriptor": analysis_descriptor(), + }) + self.assertEqual(409, response.status_code) + self.assertIn("not the hash", response.json()["error"]) + self.assertEqual(0, Analysis.objects.count()) + + def test_an_analysis_without_a_detector_version_is_refused(self): + # The rule docs/architecture.md is most insistent about, enforced where a + # client cannot forget it. + descriptor = '{"detector":"mediapipe","frames":48,"scheme":1}' + response = self.post("/api/analyses", { + "key": key_for(descriptor), "descriptor": descriptor, + }) + self.assertEqual(400, response.status_code) + self.assertEqual("version", response.json()["missing"]) + + def test_a_block_is_stored_under_the_hash_of_its_inputs(self): + analysis = self.register_analysis() + descriptor = block_descriptor(analysis) + key = key_for(descriptor) + response = self.post("/api/blocks", { + "key": key, "descriptor": descriptor, + "data": "AAECAwQFBgc=", "state": "AAE=", + }) + self.assertEqual(201, response.status_code, response.content) + row = Block.objects.get(key=key) + self.assertEqual("geom", row.role) + self.assertEqual(analysis, row.analysis_id) + # Two hashes, and they are not the same hash: the key is over the inputs, + # the blob's digest is over the bytes. + self.assertNotEqual(key[7:], row.data.digest) + self.assertEqual(8, row.data.size) + + fetched = self.client.get(f"/api/blocks/{key}").json() + self.assertEqual("AAECAwQFBgc=", fetched["data"]) + self.assertEqual("AAE=", fetched["state"]) + self.assertEqual(descriptor, fetched["descriptor"]) + + def test_a_block_whose_analysis_is_unknown_is_refused(self): + descriptor = block_descriptor("sha256:" + "f" * 64) + response = self.post("/api/blocks", { + "key": key_for(descriptor), "descriptor": descriptor, "data": "AA==", + }) + self.assertEqual(400, response.status_code) + self.assertIn("analysis the server does not know", response.json()["error"]) + + def test_a_block_that_does_not_say_what_its_elements_are_is_refused(self): + analysis = self.register_analysis() + descriptor = ('{"analysis":"%s","layout":{"frames":48},"role":"geom","scheme":1}' + % analysis) + response = self.post("/api/blocks", { + "key": key_for(descriptor), "descriptor": descriptor, "data": "AA==", + }) + self.assertEqual(400, response.status_code) + self.assertIn("valid readings", response.json()["error"]) + + def test_only_the_missing_blocks_are_asked_for(self): + analysis = self.register_analysis() + here = key_for(block_descriptor(analysis)) + self.post("/api/blocks", { + "key": here, "descriptor": block_descriptor(analysis), "data": "AA==", + }) + elsewhere = key_for(block_descriptor(analysis, anchor_avg=3)) + response = self.post("/api/blocks/missing", {"keys": [here, elsewhere]}) + self.assertEqual([elsewhere], response.json()["missing"]) + + def test_a_detector_upgrade_gives_a_block_a_new_name(self): + # The end-to-end statement of the requirement: the same measurements under + # a new model version are a different, additional block, and the old one is + # unreachable from the new document rather than wrong. + old = self.register_analysis("1.0.1") + new_descriptor = analysis_descriptor("1.0.2") + self.post("/api/analyses", {"key": key_for(new_descriptor), + "descriptor": new_descriptor}) + for analysis in (old, key_for(new_descriptor)): + descriptor = block_descriptor(analysis) + self.post("/api/blocks", {"key": key_for(descriptor), + "descriptor": descriptor, "data": "AAEC"}) + self.assertEqual(2, Block.objects.count()) + # One set of bytes, two names: the upgrade renamed the block and did not + # duplicate it on disk. + self.assertEqual(1, Blob.objects.filter(block_data_for__isnull=False).distinct().count()) + + +@override_settings(BLOB_ROOT=BLOB_DIR) +class DocumentTests(TestCase): + """Tier 1: load, save, and the conditional write.""" + + def setUp(self): + self.project = Project.objects.create(name="a project") + descriptor = analysis_descriptor() + self.analysis = key_for(descriptor) + self.client.post("/api/analyses", data=json.dumps( + {"key": self.analysis, "descriptor": descriptor}), + content_type="application/json") + block = block_descriptor(self.analysis) + self.block = key_for(block) + self.client.post("/api/blocks", data=json.dumps( + {"key": self.block, "descriptor": block, "data": "AAECAwQFBgc="}), + content_type="application/json") + + def put(self, url, payload, **headers): + return self.client.put(url, data=json.dumps(payload), + content_type="application/json", **headers) + + def leaves(self): + # Transit-shaped, because that is what a leaf actually holds: a map with a + # cache marker, keyword keys, and a frame-keyed inner map. + return { + "clip/c1/timing": ["^ ", "~:fps", 30, "~:frames", 48], + "clip/c1/node/mouth": ["^ ", "~:id", "~:mouth", "~:z", "a1"], + "clip/c1/channel/mouth/geom.pts": [ + "^ ", "~:animated?", True, "~:dense", + ["^ ", "~:store", self.block, "~:offset", 0, "~:stride", 16], + ], + "clip/c1/channel/mouth-in/vis": [ + "^ ", "~:animated?", True, "~:keys", ["^ ", "~i0", True, "~i12", False], + ], + } + + def save(self, leaves=None, blocks=None): + return self.put(f"/api/projects/{self.project.id}", { + "name": "a project", + "clips": [{"cid": "c1", "name": "take", "analysis": self.analysis, + "leaves": leaves if leaves is not None else self.leaves(), + "blocks": blocks if blocks is not None else [self.block]}], + }) + + def test_a_document_comes_back_exactly(self): + response = self.save() + self.assertEqual(200, response.status_code, response.content) + self.assertEqual(4, len(response.json()["written"])) + + loaded = self.client.get(f"/api/projects/{self.project.id}").json() + self.assertEqual(1, len(loaded["clips"])) + clip = loaded["clips"][0] + self.assertEqual("c1", clip["cid"]) + self.assertEqual([self.block], clip["blocks"]) + self.assertEqual(self.analysis, clip["analysis"]) + # The whole point: byte-identical values, including the integer frame keys + # transit writes as "~i0". A JSON round trip that stringified them would + # come back "0" and the part would hold its first pose forever. + self.assertEqual(self.leaves(), clip["leaves"]) + + def test_an_unchanged_leaf_keeps_its_version(self): + # What makes an entity tag worth having: a save where one channel moved + # invalidates one leaf's etag, not the whole document's. + self.save() + first = {leaf.path: leaf.version for leaf in Leaf.objects.all()} + moved = self.leaves() + moved["clip/c1/channel/mouth-in/vis"] = [ + "^ ", "~:animated?", True, "~:keys", ["^ ", "~i0", False], + ] + response = self.save(moved) + self.assertEqual(["clip/c1/channel/mouth-in/vis"], response.json()["written"]) + self.assertEqual(3, response.json()["unchanged"]) + after = {leaf.path: leaf.version for leaf in Leaf.objects.all()} + self.assertEqual(2, after["clip/c1/channel/mouth-in/vis"]) + self.assertEqual(first["clip/c1/timing"], after["clip/c1/timing"]) + + def test_a_removed_node_removes_its_leaf(self): + self.save() + fewer = {k: v for k, v in self.leaves().items() if k != "clip/c1/node/mouth"} + response = self.save(fewer) + self.assertEqual(["clip/c1/node/mouth"], response.json()["removed"]) + self.assertEqual(3, Leaf.objects.count()) + + def test_a_save_does_not_disturb_another_clip(self): + # A save is not the only way the document changes, so a save that cleared + # what it did not mention would undo a collaborator. + Leaf.objects.create(project=self.project, path="clip/c2/timing", value=["^ "]) + self.save() + self.assertTrue(Leaf.objects.filter(path="clip/c2/timing").exists()) + + def test_a_leaf_addressed_to_another_clip_is_refused(self): + response = self.save({"clip/c9/timing": ["^ "]}) + self.assertEqual(400, response.status_code) + self.assertIn("not addressed to clip", response.json()["error"]) + self.assertEqual(0, Leaf.objects.count()) + + def test_a_document_naming_blocks_the_server_lacks_is_refused(self): + # Referential integrity across the tiers. Saved without this, the document + # loads into a blank stage on any other machine. + response = self.save(blocks=[self.block, "sha256:" + "a" * 64]) + self.assertEqual(409, response.status_code) + self.assertEqual(["sha256:" + "a" * 64], response.json()["missing"]) + self.assertEqual(0, Leaf.objects.count()) + + def test_every_write_bumps_the_projects_version(self): + before = Project.objects.get(id=self.project.id).seq + self.save() + self.assertEqual(before + 1, Project.objects.get(id=self.project.id).seq) + + # --- the conditional write --------------------------------------------- + + def test_a_leaf_write_carries_an_etag(self): + self.save() + url = f"/api/projects/{self.project.id}/leaves/clip/c1/node/mouth" + got = self.client.get(url) + self.assertEqual('"1"', got["ETag"]) + + ok = self.put(url, {"value": ["^ ", "~:id", "~:mouth", "~:z", "a2"]}, + HTTP_IF_MATCH='"1"') + self.assertEqual(200, ok.status_code) + self.assertEqual('"2"', ok["ETag"]) + self.assertEqual(["^ ", "~:id", "~:mouth", "~:z", "a2"], + self.client.get(url).json()["value"]) + + def test_a_stale_write_is_refused_and_says_what_is_there(self): + # 409 with the current value, so the client can offer keep-mine / + # take-theirs. A PUT that replaced unconditionally is the bug where the + # loser's work disappears silently. + self.save() + url = f"/api/projects/{self.project.id}/leaves/clip/c1/node/mouth" + self.put(url, {"value": ["^ ", "~:z", "a2"]}, HTTP_IF_MATCH='"1"') + stale = self.put(url, {"value": ["^ ", "~:z", "a3"]}, HTTP_IF_MATCH='"1"') + self.assertEqual(409, stale.status_code) + self.assertEqual(2, stale.json()["version"]) + self.assertEqual(["^ ", "~:z", "a2"], stale.json()["value"]) + # And the value on the server is the one that won, not the one refused. + self.assertEqual(["^ ", "~:z", "a2"], self.client.get(url).json()["value"]) + + def test_an_unconditional_write_still_works(self): + # Conditional writes are the protocol, not a requirement: the first write + # of a leaf has no etag to match. + url = f"/api/projects/{self.project.id}/leaves/clip/c1/stage" + response = self.put(url, {"value": ["^ ", "~:width", 320]}) + self.assertEqual(200, response.status_code) + self.assertEqual('"1"', response["ETag"]) + + def test_if_match_star_requires_the_leaf_to_exist(self): + url = f"/api/projects/{self.project.id}/leaves/clip/c1/nothing" + self.assertEqual(409, self.put(url, {"value": []}, HTTP_IF_MATCH="*").status_code) + + # --- revisions --------------------------------------------------------- + + def test_a_revision_snapshots_the_authored_layer(self): + self.save() + response = self.client.post( + f"/api/projects/{self.project.id}/revisions", + data=json.dumps({"summary": "first pass", "author": "olive"}), + content_type="application/json", + ) + self.assertEqual(201, response.status_code) + revision = Revision.objects.get() + self.assertEqual(4, len(revision.document)) + self.assertEqual(self.leaves(), revision.document) + # Coarse on purpose: a save does not write one, because tier 1 will hold + # cel polygons and a snapshot per save bloats the table. + self.save() + self.assertEqual(1, Revision.objects.count()) + + +@override_settings(BLOB_ROOT=BLOB_DIR) +class FootageTests(TestCase): + """Tier 3, and the thing that makes the frames the backend's to serve: the + manifest names every frame by URL.""" + + def bundle(self, frames=3, absence=None): + root = Path(tempfile.mkdtemp(prefix="arthur-test-bundle-")) + (root / "frames").mkdir() + for i in range(frames): + (root / "frames" / f"{i + 1:04d}.png").write_bytes(png(8, 6) + bytes([i])) + (root / "audio.wav").write_bytes(b"RIFF....WAVEfmt ") + manifest = {"fps": 12, "frames": frames, "dir": "frames", + "audio": "audio.wav", "source": "IMG_8608.MOV"} + if absence: + manifest["feature-absence"] = absence + (root / "manifest.json").write_text(json.dumps(manifest)) + return root + + def ingest(self, root): + from django.core.management import call_command + from io import StringIO + + call_command("ingest_bundle", str(root), stdout=StringIO()) + return Footage.objects.get() + + def test_a_bundle_becomes_footage_with_a_url_per_frame(self): + footage = self.ingest(self.bundle(frames=3, absence={"eye-r": [[1, 2]]})) + self.assertEqual(3, footage.frames) + self.assertEqual((8, 6), (footage.width, footage.height)) + self.assertEqual(12, footage.fps) + self.assertEqual({"eye-r": [[1, 2]]}, footage.feature_absence) + + manifest = self.client.get(f"/api/footage/{footage.id}").json() + self.assertEqual(3, len(manifest["urls"])) + self.assertTrue(all(url.startswith("/blob/") for url in manifest["urls"])) + self.assertEqual(f"sha256:{footage.digest}", manifest["footage"]) + self.assertTrue(manifest["audio"].startswith("/blob/")) + self.assertEqual({"eye-r": [[1, 2]]}, manifest["feature-absence"]) + # The frames are in order, and each one is fetchable. + first = self.client.get(manifest["urls"][0]) + self.assertEqual(200, first.status_code) + self.assertEqual("image/png", first["Content-Type"]) + + def test_ingesting_the_same_bundle_twice_is_one_footage(self): + root = self.bundle() + self.ingest(root) + self.ingest(root) + self.assertEqual(1, Footage.objects.count()) + + def test_a_bundle_whose_count_disagrees_with_its_frames_is_refused(self): + from django.core.management import call_command + from django.core.management.base import CommandError + from io import StringIO + + root = self.bundle(frames=3) + (root / "frames" / "0003.png").unlink() + with self.assertRaisesMessage(CommandError, "refusing an inaccurate footage"): + call_command("ingest_bundle", str(root), stdout=StringIO()) + + def test_the_footage_list_does_not_carry_every_url(self): + # A list of takes should not be a list of six hundred URLs each. + self.ingest(self.bundle()) + listed = self.client.get("/api/footage").json()["footage"] + self.assertEqual(1, len(listed)) + self.assertNotIn("urls", listed[0]) + + +class PageTests(TestCase): + def test_django_serves_the_page_at_both_urls(self): + for url in ("/", "/index.html"): + response = self.client.get(url) + self.assertEqual(200, response.status_code, url) + body = response.content.decode() + self.assertIn("/static/arthur/js/main.js", body) + self.assertIn("/static/mediapipe/vision_bundle.js", body) + self.assertIn('id="app"', body) + # The token is rendered so Django sets its cookie, which is what the + # save path reads to write the X-CSRFToken header. + self.assertIn("csrfmiddlewaretoken", body) + + def test_the_detector_reports_a_version_derived_from_the_model(self): + # The version is the package version plus the model asset's own hash, + # because a version string in the client is one somebody has to remember to + # bump, and the server is the thing that serves the model. + report = self.client.get("/api/detector").json() + self.assertEqual("mediapipe", report["detector"]) + self.assertNotEqual("unknown", report["version"]) + self.assertTrue(report["model"].startswith("sha256:")) + self.assertIn("+", report["version"]) diff --git a/clips/urls.py b/clips/urls.py new file mode 100644 index 0000000..4874732 --- /dev/null +++ b/clips/urls.py @@ -0,0 +1,30 @@ +"""The API, which is nine endpoints and no framework. + +The shape is RFC 7232 over addressed resources: a leaf is a resource, its version +is an entity tag, and a conditional write answers 409. docs/architecture.md is +explicit that this part is not a bespoke invention — "optimistic concurrency +control over addressed resources with an entity tag" is what HTTP has done for +thirty years — so the plumbing here is deliberately boring. + +WRITES ARE ON HTTP AND STAY THERE. When the websocket arrives it carries presence +and broadcasts, and not writes: auth, idempotency, status codes, retries and +conditional requests all come for free here, and a dropped socket cannot lose a +write. +""" +from django.urls import path + +from . import views + +urlpatterns = [ + path("detector", views.detector), + path("footage", views.footage_list), + path("footage/", views.footage_detail), + path("projects", views.projects), + path("projects/", views.project_detail), + path("projects//leaves/", views.leaf_detail), + path("projects//revisions", views.revisions), + path("analyses", views.analyses), + path("blocks", views.blocks), + path("blocks/missing", views.blocks_missing), + path("blocks/", views.block_detail), +] diff --git a/clips/views.py b/clips/views.py new file mode 100644 index 0000000..d2c0355 --- /dev/null +++ b/clips/views.py @@ -0,0 +1,578 @@ +"""The API's implementation. + +Two things in here are load-bearing and neither is Django. + +THE SERVER VERIFIES EVERY TIER-2 KEY IT IS HANDED. A key is the sha256 of a +canonical descriptor, and this recomputes it and refuses a mismatch. That is what +makes content addressing a property of the system rather than a convention in the +client: nothing can store bytes under a name that does not describe them. + +It hashes THE TEXT IT WAS SENT rather than re-rendering the descriptor from parsed +values, and that is the honest arrangement rather than a shortcut. JS prints an +integral double as `1` and Python prints `1.0`, so a scheme where both sides +re-render the numbers would disagree on the first parameter whose value happens to +be whole — and the failure would be an upload that 409s with nothing wrong. The +bytes are the contract; the schema on top of them is a convention, and the two +fields this file actually reads out of that schema are checked separately. + +AND IT REFUSES A BLOCK WHOSE ANALYSIS IT DOES NOT KNOW. Every block descriptor +names an analysis, and every analysis declares a detector and a VERSION. So the +chain from a stored block to the model version that produced it cannot be broken +by a client that forgot a step — which is the whole point of +docs/architecture.md's insistence that the cache key include the detector version. +A model upgrade that silently reused old landmarks would otherwise present as "the +tool got worse", with no event to attach it to. +""" +import hashlib +import json +from functools import lru_cache +from pathlib import Path + +from django.conf import settings +from django.db import transaction +from django.http import FileResponse, HttpResponse, JsonResponse +from django.shortcuts import render +from django.views.decorators.http import require_http_methods + +from . import blobs +from .models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision + +KEY_LENGTH = 71 # "sha256:" + 64 hex + + +# --------------------------------------------------------------------------- +# helpers + + +def _body(request): + try: + return json.loads(request.body or b"{}") + except json.JSONDecodeError as exc: + raise Bad(f"the request body is not JSON: {exc}") from exc + + +class Bad(Exception): + """A 400 with a message, raised where the problem is noticed.""" + + def __init__(self, message, status=400, **detail): + super().__init__(message) + self.message = message + self.status = status + self.detail = detail + + +def _error(exc: Bad): + return JsonResponse({"error": exc.message, **exc.detail}, status=exc.status) + + +def _check_key(key, descriptor): + """The verification. A key is the sha256 of the descriptor stored beside it.""" + if not isinstance(key, str) or len(key) != KEY_LENGTH or not key.startswith("sha256:"): + raise Bad(f"not a content address: {key!r}") + if not isinstance(descriptor, str) or not descriptor: + raise Bad("a key without its descriptor addresses nothing") + actual = hashlib.sha256(descriptor.encode("utf-8")).hexdigest() + if actual != key[7:]: + raise Bad( + "the key is not the hash of its descriptor", + status=409, + expected=f"sha256:{actual}", + given=key, + ) + try: + return json.loads(descriptor) + except json.JSONDecodeError as exc: + raise Bad(f"the descriptor is not canonical JSON: {exc}") from exc + + +def _blob(b64, media_type="application/octet-stream"): + import base64 + + digest, size = blobs.write(base64.b64decode(b64)) + blob, _ = Blob.objects.get_or_create( + digest=digest, defaults={"size": size, "media_type": media_type} + ) + return blob + + +# --------------------------------------------------------------------------- +# the page + + +def page(request): + """The host page. This replaced `frontend/public/index.html` at step 9, and + `:dev-http` in shadow-cljs.edn went away with it.""" + return render(request, "clips/index.html") + + +# --------------------------------------------------------------------------- +# the detector +# +# WHY THE SERVER ANSWERS THIS. The analysis key has to include the detector +# version, and a version string in the client is a string somebody has to remember +# to bump. The server serves the model, so it can hash the model — and then the +# version is a fact about the bytes that produced the landmarks rather than a +# claim about them. + + +@lru_cache(maxsize=4) +def _model_digest(path: str, mtime: float) -> str: + return blobs.digest_file(Path(path)) + + +def _package_version() -> str: + pkg = Path(settings.BASE_DIR) / "frontend" / "package.json" + try: + deps = json.loads(pkg.read_text())["dependencies"] + return deps["@mediapipe/tasks-vision"].lstrip("^~") + except Exception: + return "unknown" + + +@require_http_methods(["GET"]) +def detector(request): + model = Path(settings.BASE_DIR) / "frontend" / "public" / "mediapipe" / "face_landmarker.task" + if not model.exists(): + # Honest rather than fatal: detection will fail at the MediaPipe boundary + # with a better message than this one could give, and an analysis stamped + # "unknown" is a take somebody can still look at and re-freeze later. + return JsonResponse({"detector": "mediapipe", "version": "unknown", "model": None}) + digest = _model_digest(str(model), model.stat().st_mtime) + return JsonResponse( + { + "detector": "mediapipe", + # The package version AND the model's own hash. Either alone can change + # while the other does not, and both change the landmarks. + "version": f"{_package_version()}+{digest[:16]}", + "model": f"sha256:{digest}", + } + ) + + +# --------------------------------------------------------------------------- +# tier 3: footage +# +# THE MANIFEST NOW CARRIES URLS. It used to carry a directory and the loader built +# `frames/0001.png` itself, which quietly made the frame layout a shared secret +# between a shell script and a ClojureScript namespace. The server names every +# frame instead, so the frames can move into the blob store — or later be uploaded +# from the browser by wasm ffmpeg — without the client learning anything new. + + +def _footage_json(footage: Footage, urls=True): + out = { + "id": str(footage.id), + "label": footage.label or footage.source, + "source": footage.source, + "fps": footage.fps, + "frames": footage.frames, + "width": footage.width, + "height": footage.height, + "footage": f"sha256:{footage.digest}", + "audio": f"/blob/{footage.audio.digest}", + "feature-absence": footage.feature_absence or {}, + } + if urls: + out["urls"] = [f"/blob/{f.blob.digest}" for f in footage.frame_set.select_related("blob")] + return out + + +@require_http_methods(["GET"]) +def footage_list(request): + return JsonResponse( + {"footage": [_footage_json(f, urls=False) for f in Footage.objects.all()]} + ) + + +@require_http_methods(["GET"]) +def footage_detail(request, footage_id): + try: + footage = Footage.objects.select_related("audio").get(id=footage_id) + except Footage.DoesNotExist: + return JsonResponse({"error": "no such footage"}, status=404) + return JsonResponse(_footage_json(footage)) + + +@require_http_methods(["GET"]) +def blob(request, digest): + """Raw bytes, immutable. + + `immutable` is not optimism here, it is the definition: the name IS the hash of + the content, so a cached copy cannot be stale. That is what makes serving 600 + frames out of this cheap enough to do on every load. + """ + try: + row = Blob.objects.get(digest=digest) + path = blobs.path_for(digest) + except (Blob.DoesNotExist, ValueError): + return JsonResponse({"error": "no such blob"}, status=404) + response = FileResponse(open(path, "rb"), content_type=row.media_type) + response["Cache-Control"] = "public, max-age=31536000, immutable" + response["ETag"] = f'"{digest}"' + return response + + +# --------------------------------------------------------------------------- +# tier 2: analyses and blocks + + +@require_http_methods(["POST"]) +def analyses(request): + """Register an analysis artifact's identity. Idempotent: the same inputs are + the same key are the same row.""" + try: + data = _body(request) + key = data.get("key") + descriptor = data.get("descriptor") + parsed = _check_key(key, descriptor) + for field in ("detector", "version"): + if not parsed.get(field): + raise Bad( + f"the descriptor does not declare a {field}: a cache key that " + "omits the detector version lets a model upgrade silently reuse " + "old landmarks", + missing=field, + ) + footage = None + if data.get("footage"): + digest = str(data["footage"]).removeprefix("sha256:") + footage = Footage.objects.filter(digest=digest).first() + if footage is None: + raise Bad("the analysis names footage this server does not have", + footage=data["footage"]) + row, created = Analysis.objects.get_or_create( + key=key, + defaults={ + "descriptor": descriptor, + "detector": parsed["detector"], + "version": str(parsed["version"]), + "footage": footage, + }, + ) + return JsonResponse({"key": row.key, "created": created}, status=201 if created else 200) + except Bad as exc: + return _error(exc) + + +@require_http_methods(["POST"]) +def blocks_missing(request): + """Which of these keys the server does not have. + + The return on content addressing, as one request: a save uploads the blocks + that are new and nothing else, so re-saving a document after a knob-free edit + moves kilobytes. + """ + try: + keys = _body(request).get("keys") or [] + if not isinstance(keys, list): + raise Bad("keys must be a list") + have = set(Block.objects.filter(key__in=keys).values_list("key", flat=True)) + return JsonResponse({"missing": [k for k in keys if k not in have]}) + except Bad as exc: + return _error(exc) + + +@require_http_methods(["POST"]) +def blocks(request): + """Store one dense block: its bytes, its optional absence mask, and the + descriptor its key is the hash of.""" + try: + data = _body(request) + key = data.get("key") + descriptor = data.get("descriptor") + parsed = _check_key(key, descriptor) + role = parsed.get("role") + if not role: + raise Bad("a block's descriptor names its role") + if not parsed.get("layout", {}).get("type"): + raise Bad( + "a block's descriptor must say what its elements are: an Int16Array " + "and a Float32Array over the same bytes are both valid readings and " + "only one of them is the block" + ) + analysis_key = parsed.get("analysis") + analysis = Analysis.objects.filter(key=analysis_key).first() + if analysis is None: + raise Bad( + "this block names an analysis the server does not know; register the " + "analysis first, so that every stored block can name the detector " + "version that produced it", + analysis=analysis_key, + ) + if not data.get("data"): + raise Bad("a block with no bytes") + with transaction.atomic(): + row, created = Block.objects.get_or_create( + key=key, + defaults={ + "descriptor": descriptor, + "role": role, + "analysis": analysis, + "data": _blob(data["data"]), + "state": _blob(data["state"]) if data.get("state") else None, + }, + ) + return JsonResponse({"key": row.key, "created": created}, status=201 if created else 200) + except Bad as exc: + return _error(exc) + + +@require_http_methods(["GET"]) +def block_detail(request, key): + import base64 + + try: + row = Block.objects.select_related("data", "state").get(key=key) + except Block.DoesNotExist: + return JsonResponse({"error": "no such block"}, status=404) + out = { + "key": row.key, + "descriptor": row.descriptor, + "data": base64.b64encode(blobs.read(row.data.digest)).decode("ascii"), + } + if row.state_id: + out["state"] = base64.b64encode(blobs.read(row.state.digest)).decode("ascii") + response = JsonResponse(out) + response["Cache-Control"] = "public, max-age=31536000, immutable" + return response + + +# --------------------------------------------------------------------------- +# tier 1: projects, clips, leaves + + +def _project_json(project: Project): + leaves = list(project.leaves.all()) + clips = [] + for clip in project.clips.all(): + prefix = f"clip/{clip.cid}/" + clips.append( + { + "cid": clip.cid, + "name": clip.name, + "footage": str(clip.footage_id) if clip.footage_id else None, + "analysis": clip.analysis_id, + "blocks": sorted(clip.blocks.values_list("key", flat=True)), + "leaves": {leaf.path: leaf.value for leaf in leaves if leaf.path.startswith(prefix)}, + } + ) + return { + "id": str(project.id), + "name": project.name, + "seq": project.seq, + "palette": project.palette, + "clips": clips, + } + + +@require_http_methods(["GET", "POST"]) +def projects(request): + if request.method == "GET": + return JsonResponse( + { + "projects": [ + {"id": str(p.id), "name": p.name, "seq": p.seq, + "updated": p.updated.isoformat()} + for p in Project.objects.all()[:100] + ] + } + ) + try: + data = _body(request) + project = Project.objects.create(name=data.get("name") or "untitled") + return JsonResponse(_project_json(project), status=201) + except Bad as exc: + return _error(exc) + + +@require_http_methods(["GET", "PUT"]) +def project_detail(request, project_id): + try: + project = Project.objects.get(id=project_id) + except Project.DoesNotExist: + return JsonResponse({"error": "no such project"}, status=404) + if request.method == "GET": + return JsonResponse(_project_json(project)) + try: + return _save(project, _body(request)) + except Bad as exc: + return _error(exc) + + +@transaction.atomic +def _save(project: Project, data): + """A whole-document save: one clip's leaves replace that clip's leaves. + + SCOPED BY CLIP, not by project. A payload that carries clip `a` does not + disturb clip `b`'s leaves, because a save is not the only way the document + changes — a single-leaf conditional write is — and a save that cleared + everything it did not mention would be a save that undoes a collaborator. + + A leaf whose value is unchanged keeps its VERSION. That is what makes the + entity tag mean something: a save of a document where one channel moved + invalidates one leaf's etag, not all four hundred. + """ + if data.get("name"): + project.name = data["name"] + if data.get("palette"): + project.palette = data["palette"] + + written, removed, unchanged = [], [], [] + for spec in data.get("clips") or []: + cid = spec.get("cid") + if not cid: + raise Bad("every clip in a save names its cid") + leaves = spec.get("leaves") or {} + prefix = f"clip/{cid}/" + for path in leaves: + if not path.startswith(prefix): + raise Bad( + f"leaf {path!r} is not addressed to clip {cid!r}", + clip=cid, path=path, + ) + + keys = spec.get("blocks") or [] + have = set(Block.objects.filter(key__in=keys).values_list("key", flat=True)) + if missing := [k for k in keys if k not in have]: + # Referential integrity across the tiers, enforced where it can be: + # a document that names blocks the server does not hold would load + # into a blank stage on any other machine. + raise Bad( + "this clip names tier-2 blocks the server does not have; upload them " + "before saving the document that points at them", + status=409, missing=missing, + ) + + analysis = Analysis.objects.filter(key=spec.get("analysis")).first() + footage = None + if spec.get("footage"): + footage = Footage.objects.filter(id=spec["footage"]).first() + clip, _ = Clip.objects.update_or_create( + project=project, + cid=cid, + defaults={"name": spec.get("name") or "", "analysis": analysis, "footage": footage}, + ) + clip.blocks.set(Block.objects.filter(key__in=keys)) + + existing = {leaf.path: leaf for leaf in project.leaves.filter(path__startswith=prefix)} + for path, value in leaves.items(): + leaf = existing.get(path) + if leaf is None: + Leaf.objects.create(project=project, path=path, value=value) + written.append(path) + elif leaf.value != value: + leaf.value = value + leaf.version += 1 + leaf.save(update_fields=["value", "version", "updated"]) + written.append(path) + else: + unchanged.append(path) + for path, leaf in existing.items(): + if path not in leaves: + leaf.delete() + removed.append(path) + + seq = project.seq + 1 + project.seq = seq + project.save() + return JsonResponse( + { + "id": str(project.id), + "seq": seq, + "written": sorted(written), + "removed": sorted(removed), + "unchanged": len(unchanged), + } + ) + + +@require_http_methods(["GET", "PUT"]) +def leaf_detail(request, project_id, leaf_path): + """One leaf, conditionally. + + `If-Match` and a 409 whose body carries the CURRENT value, so the client can + offer keep-mine / take-theirs. A PUT that replaced unconditionally is the bug + docs/architecture.md calls out in tl: the loser's work disappears silently, and + for a painted cel that is the class of bug that ends trust in a tool. + """ + try: + project = Project.objects.get(id=project_id) + except Project.DoesNotExist: + return JsonResponse({"error": "no such project"}, status=404) + + leaf = project.leaves.filter(path=leaf_path).first() + if request.method == "GET": + if leaf is None: + return JsonResponse({"error": "no such leaf"}, status=404) + response = JsonResponse({"path": leaf.path, "value": leaf.value, "version": leaf.version}) + response["ETag"] = leaf.etag + return response + + try: + data = _body(request) + except Bad as exc: + return _error(exc) + if "value" not in data: + return _error(Bad("a leaf write carries a value")) + + match = request.headers.get("If-Match") + if leaf is None: + # ANY `If-Match` on a leaf that does not exist is a failed precondition, + # `*` included: RFC 7232 gives `*` the meaning "the resource must already + # exist", which is exactly the write a client makes when it believes it is + # editing something. Creating it instead would turn "somebody deleted this + # node" into a silent resurrection. + if match: + return JsonResponse( + {"error": "no such leaf", "path": leaf_path}, status=409 + ) + leaf = Leaf.objects.create(project=project, path=leaf_path, value=data["value"]) + else: + if match and match not in ("*", leaf.etag): + response = JsonResponse( + { + "error": "stale write", + "path": leaf.path, + "version": leaf.version, + "value": leaf.value, + }, + status=409, + ) + response["ETag"] = leaf.etag + return response + leaf.value = data["value"] + leaf.version += 1 + leaf.save(update_fields=["value", "version", "updated"]) + + seq = project.bump() + response = JsonResponse({"path": leaf.path, "version": leaf.version, "seq": seq}) + response["ETag"] = leaf.etag + return response + + +@require_http_methods(["GET", "POST"]) +def revisions(request, project_id): + """Mark a version: one snapshot of the authored layer, with a summary.""" + try: + project = Project.objects.get(id=project_id) + except Project.DoesNotExist: + return JsonResponse({"error": "no such project"}, status=404) + if request.method == "GET": + return JsonResponse( + { + "revisions": [ + {"seq": r.seq, "author": r.author, "summary": r.summary, + "created": r.created.isoformat(), "leaves": len(r.document)} + for r in project.revisions.all()[:100] + ] + } + ) + data = json.loads(request.body or b"{}") + revision = Revision.objects.create( + project=project, + seq=project.seq, + author=data.get("author") or "", + summary=data.get("summary") or "", + document={leaf.path: leaf.value for leaf in project.leaves.all()}, + ) + return JsonResponse({"seq": revision.seq, "leaves": len(revision.document)}, status=201) diff --git a/docs/animation-model.md b/docs/animation-model.md index ec02118..d7a407e 100644 --- a/docs/animation-model.md +++ b/docs/animation-model.md @@ -93,7 +93,18 @@ frames where that feature is occluded and later reappears: :eye-l {:id :eye-l :subject :face-1 :area :eye :nodes [:eye-l :eye-l-in :iris-l :pupil-l] :params {}} :mouth {:id :mouth :subject :face-1 :area :mouth - :nodes [:mouth :mouth-in :teeth] :params {}}} + :nodes [:mouth :mouth-in] :params {}} + ;; The teeth are their OWN feature and not three nodes of the mouth. A feature + ;; carries the params of exactly one area, and the teeth have an `:area :teeth` + ;; of their own — the otsu threshold, the tongue rejection, the radial contour's + ;; vertex budget — which could not be reached if they were part of `:mouth`. + ;; The coupling that made them look like the mouth's is real and is enforced + ;; elsewhere: `:teeth` is STENCILLED by `:mouth-in`, and a node whose stencil drew + ;; nothing is dropped, so an absent mouth takes the teeth with it without either + ;; of them sharing an absence mask. An earlier draft of this block listed them + ;; together; the code is right and this document was wrong. + :teeth {:id :teeth :subject :face-1 :area :teeth + :nodes [:teeth] :params {}}} :groups {:eyes-1 {:id :eyes-1 :kind :eye-pair :subject :face-1 :members [:eye-r :eye-l] :params {}}} diff --git a/docs/architecture.md b/docs/architecture.md index 18c4ab3..b1ffaf2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -527,6 +527,102 @@ becomes addressable as a path; and two people keying different frames of one par merge field-wise with no merge algorithm at all. `selectKeys` returns an array today — change it before anything depends on the order. +### Serving tiers 2 and 3 + +Built at step 9. Above this point the tiers are a rule about what is allowed on +the wire; this is the shape that enforces it. + +**One store for both, named by the sha256 of the bytes.** Once tier 3 is decoded +by the app rather than by a shell script it becomes the same kind of thing as tier +2 — a cache with a hash — so there is one place that writes bytes, one that reads +them, and one URL shape: + +``` +GET /blob/ raw bytes, Cache-Control: immutable +``` + +`immutable` is not optimism there, it is the definition: the name IS the hash of +the content, so a cached copy cannot be stale. That is what makes serving six +hundred frames out of it cheap enough to do on every load. + +**Two kinds of hash, and they are not the same hash.** A blob is named by the hash +of its BYTES, which is what makes an identical frame in two extractions one file. +A derived thing — an analysis artifact, a dense block — is named by a hash over its +INPUTS, which is what lets a client ask for the block the current settings want +*before* anything has computed it, and what makes a stale bake unreachable rather +than wrong. So a `Block` row has both: `key` over the inputs, and a foreign key to +the blob whose digest is over the bytes. Conflating them would break the half of +addressing that answers questions about work not yet done. + +``` +POST /api/analyses {key, descriptor} idempotent +POST /api/blocks/missing {keys} -> {missing} +POST /api/blocks {key, descriptor, data, state} +GET /api/blocks/ +GET /api/footage/ the manifest, with a URL per frame +``` + +**The server verifies, rather than trusting a name it was handed.** It recomputes +`sha256(descriptor)` for every key and refuses a mismatch; it refuses an analysis +whose descriptor does not declare a detector and a version; it refuses a block +whose analysis it does not know; and it refuses a document naming blocks it does +not hold. The chain from a stored block to the model version that produced it +therefore cannot be broken by a client that skipped a step — which is what the +cache-key rule above actually requires, as opposed to recommends. + +It hashes the descriptor TEXT rather than re-rendering it from parsed values, and +that is not a shortcut. JS prints an integral double as `1` and Python prints +`1.0`, so a scheme where both ends re-render the numbers disagrees on the first +parameter whose value happens to be whole — and the failure is an upload that 409s +with nothing wrong. The bytes are the contract; the schema on top of them is a +convention, and the two fields the server reads out of that schema are checked +separately. + +**A manifest names frames; it does not locate them.** Until step 9 the client +fetched `manifest.json` and built `frames/0001.png` itself, which made the frame +layout a shared secret between a shell script and a ClojureScript namespace. The +manifest now carries a URL per frame, so the frames can live in the blob store — +or, when wasm-ffmpeg extraction arrives, be uploaded into the same store by the +app — and the client learns nothing new when that happens. The producer changes; +the shape does not. + +**The document stores what a block IS, not what it holds.** A block's element type +is in its own descriptor, which is the only place it is written down: an +`Int16Array` and a `Float32Array` over the same bytes are both valid readings, and +only one of them is the block. That makes the descriptor load-bearing rather than +documentation, which is the right way round for the thing a key is the hash of. + +### Leaf paths, as built + +The list under **Make the merge unit small instead of clever** is the design; this +is what step 9 implemented, for the subset that exists: + +``` +clip//name clip//subject/ +clip//timing clip//feature/ +clip//stage clip//group/ +clip//source clip//node/ +clip//measured/ clip//channel// +``` + +Two departures from the design above, both because step 8 moved settings. + +`params/:area` is **not** a leaf. That path came from a draft where params were one +blob per clip, and two people tuning teeth and eyes collided on every slider move. +Settings now live on the subject, the feature and the group, and a feature has +exactly one area — so the feature leaf already *is* the area-scoped leaf, and +splitting it again would separate a feature's params from its identity. + +`measured/` is one leaf holding several channels, which contradicts "every +channel gets its own". `:head`'s measured channels are not authored: a freeze +writes them together and a re-freeze replaces them together, and `head-mode` reads +them to write `:channels`. A leaf per measured channel would offer a write nobody +can make. + +A leaf path is "/"-delimited and an id is one segment of it, so a namespaced id — +`:eye-r/iris`, as drawn under **The node, decomposed** — is written `eye-r~iris`, +and `~` is then refused inside a name. That is the whole of the escaping. + ## Collaboration `../tl` already has the model, and it is the right one to copy: diff --git a/docs/port-plan.md b/docs/port-plan.md index 5b8935c..8462fc4 100644 --- a/docs/port-plan.md +++ b/docs/port-plan.md @@ -2,16 +2,20 @@ Self-contained. You should not need any prior conversation to execute this. -**Implementation status (2026-09-27):** steps 0–7 are in the CLJS frontend. -Step 6 reads extracted footage from the manifest, detects landmarks with local -MediaPipe assets at full source cadence, and runs the same freeze path as the -synthetic take. The scene time map can sample the frozen roto at a lower picture -fps without changing source analysis, duration or audio. Step 7 adds dense -eyelids, shared gaze, brows and pixel-derived teeth. Step 8's data model now -has stable feature identity, feature-level presence, explicit eye pairs and -shared parameter definitions. A manifest can now supply known feature absence -intervals through measurement and freeze. Automatic per-feature detection, -parameter editing and scoped regeneration remain. +**Implementation status (2026-09-28):** steps 0–9 are in. Step 6 reads extracted +footage, detects landmarks with local MediaPipe assets at full source cadence, and +runs the same freeze path as the synthetic take. The scene time map can sample the +frozen roto at a lower picture fps without changing source analysis, duration or +audio. Step 7 adds dense eyelids, shared gaze, brows and pixel-derived teeth. +Step 8's data model has stable feature identity, feature-level presence, explicit +eye pairs and shared parameter definitions; a manifest can supply known feature +absence intervals through measurement and freeze. Step 9 adds the Django backend, +the three-tier split, content-addressed tier 2 with the detector version inside +every key, leaf addressing for tier 1, and project load/save that round-trips. + +**Still open.** Step 8's parameter UI and scoped regeneration, and automatic +per-feature detection. Everything under "Out, and do not build it" below, which +step 9 did not touch. ## What arthur is @@ -59,17 +63,40 @@ arthur/ src/arthur/** namespace root stays arthur.* whatever the dir is called test/arthur/** static/arthur/js/ shadow-cljs output, collected by Django staticfiles + static/arthur/audio.wav the synthetic take's clock. NOT extract.sh's output — + that is tier 3 and lives in the blob store + var/blobs/ the content-addressed blob store: tiers 2 and 3. Gitignored docs/ js/ index.html serve.py extract.sh the old tool — see "the oracle" ``` +The namespaces step 9 added, since the list under **Namespaces** in +`docs/architecture.md` predates them: + +``` +domain/sha256.cljs SHA-256, synchronous and pure, byte-compatible with hashlib +domain/canon.cljs the one canonical text for a descriptor, so hashing it means + something +domain/leaf.cljs leaf addressing: the document as path -> value +domain/wire.cljs transit for tier 1, base64 for tier 2 +domain/project.cljs clip <-> the document and blocks that travel +flow/address.cljs tier-2 keys, and the invalidation table they are built from +fx/http.cljs the only namespace that talks to the server +events/project.cljs save and open +``` + `clips` is a naming call, not a constraint — it is the Django app holding Project, Clip, Footage, Analysis, Leaf and Revision. Rename in one line if something fits better. -Dev runs two processes: Django serves the page, `shadow-cljs watch app` rebuilds -into `static/arthur/js`. Set `:output-dir "../static/arthur/js"` in -`shadow-cljs.edn`. +Dev runs two processes and they do not talk to each other: Django serves the page, +`shadow-cljs watch app` rebuilds into `static/arthur/js`, which is already +`:output-dir` in `shadow-cljs.edn`. `:dev-http` is gone. + +```sh +mise exec -- python manage.py runserver 8778 # from the repo root +cd frontend && mise exec -- npx shadow-cljs watch app +``` ## Toolchain @@ -352,10 +379,50 @@ group. Retain source measurements so a setting change can regenerate affected channels without re-detecting footage. Time-varying parameter values and all parameter controls are deferred to the UI pass. -### 9 — backend -Django project, the `clips` app, models for Project/Clip/Footage/Analysis/Leaf, -and project load/save. Round-tripping a project through the server is the proof -the model serialises. +### 9 — backend — DONE +Django project, the `clips` app, models for +Project/Clip/Footage/FootageFrame/Analysis/Block/Leaf/Revision/Blob, and project +load/save. Round-tripping a project through the server is the proof the model +serialises. + +Django was the easy half; the tier split was the work. What it came to: + +**Tier 2 keys are content addresses over every input**, and the detector version +is in every one of them, through the analysis id that each block descriptor names. +`flow/address`'s `block-knobs` is the invalidation table, and it is not trusted: +`address-test` re-freezes the take once per knob and asserts the biconditional — +a block's bytes changed if and only if its key changed. That test found two +things reading the code would not have. `brow-pos` does not depend on +`contour-avg`, because the brow RING is smoothed and the raise is not. And the +first version of the test was itself wrong: a 3% perturbation of `gaze-gain` +moves every sample inside the grid cell `quantize-snap` had already rounded it +into, so the bytes came out identical and the knob looked like an input the block +did not have. + +**The server verifies what it is handed.** It recomputes every key from the +descriptor stored beside it and refuses a mismatch, refuses an analysis that does +not declare a detector version, and refuses a document naming blocks it does not +hold. It hashes the descriptor TEXT rather than re-rendering it from parsed +values, because JS prints an integral double as `1` and Python as `1.0` — a +scheme where both sides re-render breaks on the first parameter whose value +happens to be whole. + +**Tier 3 is served by hash.** `extract.sh` still decodes; `manage.py +ingest_bundle` hashes the result into the blob store, by hard link. The manifest +the client receives now carries a URL per frame, so the frame layout stopped being +a shared secret between a shell script and a ClojureScript namespace. The +cache-busting `?v=` on every frame URL went with it: a blob's name is the hash of +its bytes, so a stale copy is not a thing that can happen. + +**Leaf addressing exists**, with conditional writes and a monotonic project +version, so the sync design has nothing to retrofit. The socket, presence and +broadcasts are still out of scope. + +Two loose ends from step 8 closed on the way. `pack` no longer takes a +`(track, frame)` predicate whose call sites each derived a feature from an index — +every track names the feature it follows, which deleted five hand-maintained +mappings and handed `flow/address` the same list for its observation digest. And +the `presence-check` binding in a `let` nobody read is now an ordinary `doseq`. **Output is not in this plan.** The `.take` writer in `js/take.js` was for driving an Animator Pro render script and it is not where this is going: the diff --git a/frontend/README.md b/frontend/README.md index 8fe9c2e..9717139 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -6,7 +6,9 @@ what order; this file is only how to run it. ## Once ```sh -mise install # from the REPO ROOT: java 21+, node 20, clojure, python +mise install # from the REPO ROOT +pip install -r requirements.txt # the Django half; one dependency +mise exec -- python manage.py migrate # the document database cd frontend && npm install ``` @@ -33,6 +35,18 @@ Run them separately if a compile error is in the way. must be 21+ and node 20.19+. On an nvm node 20.11 shadowing the pinned one, things fail in ways that read like the code being broken and are not. +### And the Django one + +```sh +mise exec -- python manage.py test clips # from the REPO ROOT +``` + +Thirty-one tests over the API: the blob store, key verification, the load/save +round trip, the conditional write, and the footage manifest. The two groups worth +reading are the ones that make the tier split a property of the system rather than +a convention in ClojureScript — the server recomputes every tier-2 key it is +handed, and refuses a block whose analysis does not declare a detector version. + ### And the browser one Step 5's done-criterion is a PICTURE, and no assertion in `cljs.test` can check @@ -40,14 +54,21 @@ one: a take that resolves to the right numbers and draws nothing would pass ever test in `arthur.flow.freeze-test`. A blank canvas under a perfectly correct transport is the bug class unit tests miss, and it has happened here once. -So there is a second suite that drives a real Chrome over CDP. It needs the dev -server up: +Step 9's is a picture too, for a different reason: the ways a document survives a +round trip LOOKING correct are the interesting ones. So the suite now also saves +the take, reopens it, and checks the frames are the same pixels. + +It drives a real Chrome over CDP, and needs both processes up: ```sh +mise exec -- python manage.py runserver 8778 # from the REPO ROOT cd frontend && mise exec -- npx shadow-cljs watch app # in one shell cd frontend && mise exec -- npm run browser # in another ``` +`ARTHUR_URL` overrides the page it drives; it defaults to +`http://localhost:8778/index.html`, which since step 9 is Django's. + No dependencies. Playwright is not installed and CDP needs none — `node --experimental-websocket` has a global `WebSocket` and `--headless=new --remote-debugging-port=N` is the whole of the other side. It @@ -58,13 +79,21 @@ eye as well as by count. ## The app +Two processes, which do not talk to each other: + ```sh +mise exec -- python manage.py runserver 8778 # from the REPO ROOT cd frontend && mise exec -- npx shadow-cljs watch app ``` -Then open **** — with the `/index.html`, not -bare `/`. This shadow-cljs does no directory-index resolution, so `/` is a 404 -whatever the roots are. +Then open ****. Django serves the page from +`clips/templates/clips/index.html`, and staticfiles serves the bundle out of +`static/arthur/js`, where `shadow-cljs` already writes it — so nothing copies files +between the two. + +`/index.html` still works, and that is deliberate: it is the URL the browser suite +has used since step 5, when shadow-cljs's `:dev-http` did no directory-index +resolution and the suite learned to ask for the file. Four built-in clips, on buttons in the transport: @@ -82,29 +111,44 @@ The demo scene itself is `src/arthur/demo/scene.edn`. Both the synthetic take and real footage use `src/arthur/flow/take.cljs` for the measurement order and `src/arthur/flow/freeze.cljs` for the landmark-to-channel conversion. -### Real footage (port steps 6–7) +### Real footage (port steps 6–7, served by the backend since step 9) -From the repo root, extract a clip, then click **load frames** in the CLJS app: +Two commands from the repo root, then pick the take in the app and click +**load frames**: ```sh -./extract.sh /path/to/clip.mov +./extract.sh /path/to/clip.mov # decode to frames + audio + manifest +mise exec -- python manage.py ingest_bundle ``` -This keeps every source frame and writes `frames/0001.png` onward, `audio.wav`, -and `manifest.json` at the repo root. The manifest supplies the exact frame -count, source fps and audio path. Variable frame rate sources are rejected until -the manifest and clock carry per-frame timestamps. +`extract.sh` keeps every source frame and writes `frames/0001.png` onward, +`audio.wav` and `manifest.json`. Variable frame rate sources are rejected until the +manifest and clock carry per-frame timestamps. -To keep multiple takes or compare with a previous extraction, pass a bundle -directory and enter its manifest path in the app: +`ingest_bundle` then hashes all of it into the content-addressed blob store under +`var/blobs` — by hard link, so 112MB of PNGs is not copied — and registers one +`Footage` row. From then on the frames are the backend's: `GET /api/footage/` +answers with a manifest carrying **a URL per frame**, and the app fetches those. + +That replaced a shared secret. Until step 9 the page fetched `/manifest.json` off +the filesystem and built `frames/0001.png` itself, with shadow-cljs serving the +repo root — so the frame layout was agreed between a shell script and a +ClojureScript namespace, and "where are the frames" was answered by a directory +listing. The cache-busting `?v=` that used to hang off every frame URL went with +it: a blob's name is the hash of its bytes, so re-extracting gives a frame a +different URL rather than overwriting one. + +To keep several takes, pass a bundle directory; each ingests separately and both +stay selectable in the app. ```sh ./extract.sh /path/to/clip.mov scratch/my-take -# source manifest field: /scratch/my-take/manifest.json +mise exec -- python manage.py ingest_bundle scratch/my-take ``` -`scratch/` is ignored by Git. The directory contains its own frames, audio and -manifest, so extracting it does not replace another take's files. +`scratch/` is ignored by Git, as are `frames/`, `audio.wav` and `manifest.json` at +the root — all of it is extraction output, and tier 3 does not belong in the repo. + Loading detects one face per frame, measures the mouth, eyes and brows from landmarks and the teeth from source pixels, then freezes them into channels, and adds a button for the footage clip. Detection happens once when you load; @@ -124,13 +168,42 @@ inclusive, matching PNG filenames. The eye remains the same feature when it reappears; the other eye and the mouth continue through the gap. This is an input annotation, with no UI for editing it yet. -MediaPipe's JS, wasm and model are under `public/mediapipe/` and served locally. -No CDN is used by this app. See that directory's README for provenance. +MediaPipe's JS, wasm and model are under `public/mediapipe/`, served by Django's +staticfiles under `/static/mediapipe/`. No CDN is used by this app. See that +directory's README for provenance. + +The server reports what it serves at `GET /api/detector`: the package version plus +the **sha256 of the model asset**, and that string goes inside the content address +of every block a detection produces. Asked rather than assumed, because a version +constant in the client is one somebody has to remember to bump — and +`docs/architecture.md` is explicit that a model upgrade silently reusing old +landmarks presents as "the tool got worse", with no event to attach it to. Port 8778 is deliberately not 8777. `python3 serve.py` from the repo root still runs the old JS tool on 8777, and the two are meant to run side by side. -From step 9 Django serves the page and `:dev-http` goes away. +## Saving + +**save** and **open** in the transport. A save is three requests, in an order that +is the tier split: + +1. the **analysis** record, so every block stored afterwards can name the detector + version that produced it. The server refuses a block whose analysis it does not + know. +2. ask which **blocks** are missing, and upload only those. +3. the **document** — tier 1, as leaves. The server refuses a clip that names + blocks it does not hold, so a saved document cannot load into a blank stage + somewhere else. + +The status line says what happened: `saved r3 · 64 leaves · 8 blocks`. Saving an +unchanged document says `0 leaves · 0 blocks`, which is both halves of the +addressing working at once — an unchanged leaf keeps its version, and a +content-addressed block is already there. + +Two things are deliberately visible as failures. Saving `swarm` is refused, +because its blocks have hand-written names and a document may only name content +addresses. And **open** takes the most recently updated project and shows its first +clip: there is no project browser, and the store holds one clip at a time. ## The oracle, which is finished @@ -152,6 +225,7 @@ and pixel extraction. Its comments encode bugs that actually happened. ``` src/arthur/domain/ pure. No re-frame, no DOM, no flow/. +src/arthur/fx/ the only namespaces that talk to the network src/arthur/flow/ the stages. `(f params inputs) -> output`, no state. src/arthur/synth.cljs the synthetic track. In src/ because the take PLAYS it — it stands in for flow/detect, and a tool that needs a @@ -160,10 +234,14 @@ src/arthur/synth.cljs the synthetic track. In src/ because the take PLAYS it src/arthur/demo.cljs the hand-written scene, read from demo/scene.edn src/arthur/demo/take.cljs the synthetic source for the shared flow/take path src/arthur/ui/canvas.cljs indexed raster blit to the display canvas +test/arthur/support/ machinery shared between suites; not tests itself test/browser/ drives a real Chrome over CDP. Not run by `npm test`. -public/index.html dev host page. Django replaces it at step 9. +public/mediapipe/ vendored wasm and model, served under /static/mediapipe/ ``` +`public/` holds nothing but those assets now. The host page that used to sit beside +them is `clips/templates/clips/index.html`. + ## Two evaluators, on purpose `domain/scene` has both `eval-frame` and `resolver`, and they are not diff --git a/frontend/shadow-cljs.edn b/frontend/shadow-cljs.edn index 977814f..41ed051 100644 --- a/frontend/shadow-cljs.edn +++ b/frontend/shadow-cljs.edn @@ -10,25 +10,32 @@ {:source-paths ["src" "test"] :dependencies [[reagent "1.2.0"] - [re-frame "1.4.3"]] + [re-frame "1.4.3"] + ;; The document's wire format. JSON would do for the shape of tier + ;; 1 but not for its VALUES: channel keys are a map by FRAME + ;; NUMBER and every id is a keyword, and JSON has neither, so a + ;; save would quietly turn `{0 v}` into `{"0" v}` and `:mouth` + ;; into "mouth". Transit is JSON on the wire, so Django stores a + ;; leaf in a JSONField and the admin can still read it. + [com.cognitect/transit-cljs "0.8.280"]] - ;; Dev server for the CLJS half, on a different port from serve.py (8777) so the - ;; old tool and the port can run side by side. js/ remains the source for the - ;; measurement stages not yet ported. + ;; THERE IS NO :dev-http, since step 9. Django serves the page — one template, + ;; out of `clips/templates/` — and shadow-cljs only builds into the staticfiles + ;; tree, which is what `:output-dir` below already did. So the dev loop is two + ;; processes that do not talk to each other: ;; - ;; Two roots: `public` holds the host page, `..` is the repo root so the bundle - ;; at /static/arthur/js/ resolves, and so manifest.json, audio.wav and frames/ - ;; are reachable for real footage. + ;; mise exec -- python manage.py runserver 8778 (from the repo root) + ;; cd frontend && mise exec -- npx shadow-cljs watch app ;; - ;; THE ORDER IS LOAD-BEARING. The repo root has an index.html too — the old - ;; tool's — so with `..` first, /index.html would quietly serve the prototype - ;; instead of the port. That would look like the CLJS build having regressed to - ;; a suspiciously complete tool rather than like a misconfigured server. + ;; What went away with the key was a set of problems rather than a feature. The + ;; two roots it needed — `public` for the host page and `..` for the repo root, IN + ;; THAT ORDER, because the root has the old tool's index.html and serving that one + ;; instead would look like the port having regressed to a suspiciously complete + ;; tool — were a way of reaching frames/, audio.wav and manifest.json off the + ;; filesystem. Those are tier 3, and tier 3 is now the backend's, by hash. ;; - ;; Open /index.html, not /. This shadow-cljs does no directory-index resolution, - ;; so bare / is a 404 whatever the roots are. Django serves the page from step 9 - ;; and this whole key goes away. - :dev-http {8778 ["public" ".."]} + ;; 8778 is still deliberately not 8777, which is still the old JS tool's under + ;; `python3 serve.py`. The two are meant to run side by side. :builds {:app {:target :browser diff --git a/frontend/src/arthur/core.cljs b/frontend/src/arthur/core.cljs index 6ed9e1c..15cf9c4 100644 --- a/frontend/src/arthur/core.cljs +++ b/frontend/src/arthur/core.cljs @@ -4,8 +4,9 @@ port-plan step 3: the hand-written scene plays at 30fps against audio, scrubs, and runs at ½× and ¼×." (:require [arthur.db :as db] - [arthur.events.footage] + [arthur.events.footage :as footage] [arthur.events.playback] + [arthur.events.project] [arthur.subs.playback] [arthur.subs.render] [arthur.ui.player :as player] @@ -26,6 +27,10 @@ (defn init [] (rf/dispatch-sync [::init]) + ;; What the server already holds, asked for once. The list is small — a row per + ;; ingested take — and having it before the first click is what lets the footage + ;; picker be a picker rather than a path to type. + (rf/dispatch [::footage/refresh]) (reset! root (rdc/create-root (js/document.getElementById "app"))) (mount) (player/start!)) diff --git a/frontend/src/arthur/db.cljs b/frontend/src/arthur/db.cljs index 9e28ad7..ce5a591 100644 --- a/frontend/src/arthur/db.cljs +++ b/frontend/src/arthur/db.cljs @@ -22,8 +22,14 @@ frame space, not a rate and not a size — and they sit on the scene map only because there is one clip per scene today. Copying them by hand into this table is how one of them comes to disagree with the scene it describes." - [label scene store] - (merge {:label label :scene scene :store store :audio "/audio.wav" + [label-key label scene store] + (merge {:label label :scene scene :store store + ;; A static asset since step 9, and not the repo root's `audio.wav`. + ;; That file is `extract.sh`'s output — tier 3, which the backend now + ;; serves by hash — and the synthetic take needs a sound of its own so + ;; that the clock has something to run against with no footage ingested. + :audio "/static/arthur/audio.wav" + :cid (name label-key) :display-fps (:fps scene)} (select-keys scene [:fps :frames :width :height]))) @@ -35,10 +41,10 @@ `:head` written as a dense track in one and as framed identity in the other, so the button that switches between them switches a document field and nothing else." - {:demo (clip "demo" demo/scene nil) - :swarm (clip "swarm" @swarm/scene @swarm/store) - :take (clip "take" @take/scene @take/store) - :take-locked (clip "locked" @take/locked @take/store)}) + {:demo (clip :demo "demo" demo/scene nil) + :swarm (clip :swarm "swarm" @swarm/scene @swarm/store) + :take (clip :take "take" @take/scene @take/store) + :take-locked (clip :take-locked "locked" @take/locked @take/store)}) (def default {;; --- the document --- @@ -53,8 +59,16 @@ ;; size — and it is why ui/player no longer hardcodes 320x200. :clip (select-keys (:take scenes) [:fps :frames :width :height :audio :display-fps]) + ;; Which ingested footage to detect, and what the last load said. The list + ;; comes from the server — tier 3 is the backend's since step 9 — so there is + ;; no path to type any more. :footage {:id nil :label nil :loading? false :status nil - :manifest-path "/manifest.json"} + :available [] :chosen nil} + + ;; The document's own identity on the server. `:seq` is the monotonic project + ;; version: a client that sees a delta with `seq > local + 1` refetches, which + ;; is what will make staleness self-healing once there is a broadcast to miss. + :project {:id nil :cid nil :name nil :seq nil :busy? false :status nil} ;; --- transport --- ;; diff --git a/frontend/src/arthur/demo/take.cljs b/frontend/src/arthur/demo/take.cljs index 276cc18..e368eec 100644 --- a/frontend/src/arthur/demo/take.cljs +++ b/frontend/src/arthur/demo/take.cljs @@ -26,7 +26,8 @@ written two ways. That is the claim \"stabilisation is a channel, not a mode\" made checkable by eye: switching between them is a document edit, tier 1, and not one byte of tier 2 differs." - (:require [arthur.flow.freeze :as freeze] + (:require [arthur.flow.address :as address] + [arthur.flow.freeze :as freeze] [arthur.flow.take :as take] [arthur.synth :as synth])) @@ -74,9 +75,16 @@ ;; The head as filmed. `:take-locked` is the same freeze with this one ;; field changed, which is the point. :head :as-filmed - ;; Provenance. A content hash once the analysis is an artifact the - ;; backend stores; until then, honest about what it actually is. - :analysis "synth:mulberry32/seed-1"})) + ;; Provenance, and now a content address. There is no detector here, so + ;; the generator IS the detector and its seed is the source: two synth + ;; takes at different seeds are different analyses, which is the same + ;; statement content addressing makes about two model versions. + :analysis (address/analysis {:detector "synth" + :version "mulberry32" + :seed 1 + :frames frames + :fps fps + :aspect aspect})})) (def frozen (delay (freeze/clip params @measured))) diff --git a/frontend/src/arthur/domain/canon.cljs b/frontend/src/arthur/domain/canon.cljs new file mode 100644 index 0000000..4085b45 --- /dev/null +++ b/frontend/src/arthur/domain/canon.cljs @@ -0,0 +1,77 @@ +(ns arthur.domain.canon + "One canonical text for a map, so that hashing it means something. + + A content address is a hash of a DESCRIPTION of every input, and a description + only addresses anything if the same inputs always write the same bytes. A CLJS + map has no key order, `pr-str` will happily print `{:a 1 :b 2}` in either order + between runs, and JSON has no canonical form of its own. So this is the one + place that decides. + + The text is VALID JSON, deliberately. The server stores it beside the key and + verifies `sha256(descriptor) == key` (clips/views.py), and it also has to read + two fields out of it to enforce that a detector version was declared at all. + Hashing the text the client sent, rather than recomputing it from parsed + values, is what keeps that check free of a cross-language float-formatting + agreement nobody could hold: Python writes `1.0` where JS writes `1`, and a + scheme where both sides re-render the numbers would break on the first integral + double. The bytes are the contract; the schema on top of them is a convention. + + It is also meant to be READ. A stale bake presents as a picture that will not + update, and the descriptor is the only thing that can say which input moved, so + it is short, flat where it can be, and never has a 229-frame mask inlined — + see `arthur.flow.address`, which digests masks before they reach here. + + Three refusals, all of them cases where a canonical text is not possible or + the key would be ambiguous: + + A KEYWORD VALUE. Keys are keywords and become their names, because a key is + a name and nothing else. A keyword VALUE is refused instead of being named, + because then `:mouth` and \"mouth\" would hash alike, and the server would be + reading a field whose type depended on the caller's mood. Callers convert at + the boundary, which is also what makes the stored JSON clean. + + A SET. Unordered, so there is no one text for it. Sort it into a vector at + the call site, where it is obvious which order was meant. + + NaN OR INFINITY. Neither is JSON, and both mean a measurement went wrong + upstream of here — silently addressing it would cache the mistake." + (:require [clojure.string :as str])) + +(defn- number->text [x] + (when-not (js/Number.isFinite x) + (throw (ex-info "a descriptor cannot hold NaN or infinity" {:value x}))) + ;; `(str 1.0)` is "1" and `(str 0.12)` is "0.12": JS prints the shortest decimal + ;; that round-trips, so this is stable without a format string. + (str x)) + +(defn- key->text [k] + (cond + (keyword? k) (subs (str k) 1) ; :a -> "a", :roto/b -> "roto/b" + (string? k) k + :else (throw (ex-info "a descriptor key is a keyword or a string" + {:key k :type (type k)})))) + +(declare write) + +(defn- write-map [m] + (str "{" + (str/join "," (map (fn [[k v]] (str (js/JSON.stringify (key->text k)) ":" (write v))) + (sort-by (comp key->text key) (seq m)))) + "}")) + +(defn write + "The canonical JSON text of a descriptor value." + [v] + (cond + (nil? v) "null" + (true? v) "true" + (false? v) "false" + (number? v) (number->text v) + (string? v) (js/JSON.stringify v) + (map? v) (write-map v) + (set? v) (throw (ex-info "a descriptor cannot hold a set: sort it into a vector where the order is visible" + {:value v})) + (keyword? v) (throw (ex-info "a descriptor cannot hold a keyword VALUE: name it at the call site, so \"mouth\" and :mouth cannot address the same block" + {:value v})) + (sequential? v) (str "[" (str/join "," (map write v)) "]") + :else (throw (ex-info "not a descriptor value" {:value v :type (type v)})))) diff --git a/frontend/src/arthur/domain/leaf.cljs b/frontend/src/arthur/domain/leaf.cljs new file mode 100644 index 0000000..7f13381 --- /dev/null +++ b/frontend/src/arthur/domain/leaf.cljs @@ -0,0 +1,183 @@ +(ns arthur.domain.leaf + "Leaf addressing for tier 1: the document as a map of PATH -> value. + + This is the shape docs/architecture.md's sync design needs, built now so that + there is nothing to retrofit later. Multiplayer is out of this step's scope and + the addressing is not, because the addressing is the part that cannot be added + afterwards: it decides what a write is, and therefore what two people can do at + once. + + clip//name a label + clip//timing fps, frames + clip//stage width, height + clip//source the analysis record this came out of + clip//subject/ a tracked subject and its params + clip//feature/ one feature: area, nodes, params + clip//group/ an eye pair and its shared params + clip//node/ one node: kind, parent, stencil, z, time + clip//channel// one channel + clip//measured/ the measured channels a re-freeze owns + + WHY THESE BOUNDARIES. Last-writer-wins only clobbers when its unit is too big, + so the cut is chosen so that the things people do simultaneously land on + different leaves. Every node has its own leaf, because two people adding nodes + would otherwise collide always. Every channel has its own, because keying the + mouth and keying a brow are the same size of edit as each other and nothing + like the same edit. With fractional `:z` there is no separate draw-order leaf to + contend on, which is the second thing fractional indices buy. + + WHY PARAMS ARE NOT SPLIT BY AREA. docs/architecture.md's list has + `clip/:cid/params/:area`, from a draft where params were one blob per clip and + two people tuning teeth and eyes collided on every slider move. Step 8 moved + settings onto the subject, the feature and the group, and a FEATURE HAS EXACTLY + ONE AREA — so the feature leaf already is the area-scoped leaf, and splitting it + again would only separate a feature's params from the feature's identity. + + WHY `measured` IS ONE LEAF AND CHANNELS ARE NOT. `:head`'s measured channels are + not authored: they are written together by a freeze and replaced together by a + re-freeze, and `head-mode` reads them to write `:channels`. A leaf per measured + channel would offer a write nobody can make. The authored channels beside them + are one leaf each, because a hand writes one at a time. + + A LEAF PATH IS \"/\"-DELIMITED and an id is one segment of it, so a namespaced id + — docs/architecture.md draws one as `:eye-r/iris` — is written `eye-r~iris`. + `~` is then refused inside a name, which is the whole of the escaping and is why + it is one character rather than a scheme." + (:require [arthur.domain.sha256 :as sha] + [clojure.string :as str])) + +;; --------------------------------------------------------------------------- +;; ids and paths + +(defn segment + "An id -> one path segment." + [id] + (let [s (if (keyword? id) (subs (str id) 1) (str id))] + (when (str/includes? s "~") + (throw (ex-info "an id cannot contain ~: it is the namespace separator inside a leaf path" + {:id id}))) + (str/replace s "/" "~"))) + +(defn unsegment + "One path segment -> the id it names." + [s] + (keyword (str/replace s "~" "/"))) + +(defn- prop->path + "A channel's property vector -> one path segment. `[:geom :pts]` is \"geom.pts\" + and `[:vis]` is \"vis\"." + [prop] + (let [parts (map #(subs (str %) 1) prop)] + (doseq [p parts] + (when (or (str/includes? p ".") (str/includes? p "/")) + (throw (ex-info "a channel property cannot contain . or /: both are path punctuation" + {:prop prop})))) + (str/join "." parts))) + +(defn- path->prop [s] + (mapv keyword (str/split s #"\."))) + +;; --------------------------------------------------------------------------- +;; the split + +(def scene-keys + "Every top-level field of a scene, and the reason `leaves` refuses one it does + not know: a field added to the scene without a leaf is a field that saves + silently and comes back missing. The failure is a document that loses something + on every round trip, which is the one bug a persistence layer must not be able + to have. Add the field here and to `leaves` and `scene` in the same commit." + #{:name :frames :fps :analysis :subjects :features :groups :width :height :nodes}) + +(def ^:private node-channel-keys #{:channels :measured}) + +(defn leaves + "One clip's scene -> path -> value. + + A leaf whose value would be empty is OMITTED rather than written as `{}`, and + that is what makes the round trip exact: the demo scene has no `:fps` and the + root node has no `:channels`, and a codec that invented them would hand back a + scene that is not `=` to the one it was given." + [cid scene] + (let [unknown (remove scene-keys (keys scene))] + (when (seq unknown) + (throw (ex-info "the scene has a field with no leaf to save it in; see arthur.domain.leaf/scene-keys" + {:unknown (vec (sort-by str unknown))})))) + (let [at (fn [& parts] (str/join "/" (into ["clip" (segment cid)] parts))) + some-leaf (fn [path v] (when (seq v) {path v}))] + (apply merge + (some-leaf (at "name") (select-keys scene [:name])) + (some-leaf (at "timing") (select-keys scene [:fps :frames])) + (some-leaf (at "stage") (select-keys scene [:width :height])) + (some-leaf (at "source") (:analysis scene)) + (concat + (for [[id v] (:subjects scene)] {(at "subject" (segment id)) v}) + (for [[id v] (:features scene)] {(at "feature" (segment id)) v}) + (for [[id v] (:groups scene)] {(at "group" (segment id)) v}) + (for [[id n] (:nodes scene)] {(at "node" (segment id)) + (apply dissoc n node-channel-keys)}) + (for [[id n] (:nodes scene) + :when (seq (:measured n))] {(at "measured" (segment id)) (:measured n)}) + (for [[id n] (:nodes scene) + [prop ch] (:channels n)] + {(at "channel" (segment id) (prop->path prop)) ch}))))) + +(defn scene + "The inverse of `leaves`, for one clip. Paths belonging to another clip are + ignored, so a project's whole leaf map can be handed straight in." + [cid leaves] + (let [want (segment cid)] + (reduce + (fn [acc [path v]] + (let [[kind a b] (drop 2 (str/split path #"/"))] + (if-not (= want (second (str/split path #"/"))) + acc + (case kind + "name" (merge acc v) + "timing" (merge acc v) + "stage" (merge acc v) + "source" (assoc acc :analysis v) + "subject" (assoc-in acc [:subjects (unsegment a)] v) + "feature" (assoc-in acc [:features (unsegment a)] v) + "group" (assoc-in acc [:groups (unsegment a)] v) + "node" (update-in acc [:nodes (unsegment a)] merge v) + "measured" (assoc-in acc [:nodes (unsegment a) :measured] v) + "channel" (assoc-in acc [:nodes (unsegment a) :channels (path->prop b)] v) + (throw (ex-info "not a leaf path" {:path path})))))) + {} + ;; Sorted, so `node` lands before `channel` and `measured` under one id and + ;; the node map is merged INTO rather than over. `update-in ... merge` makes + ;; the order not matter; sorting makes it not matter for a reason. + (sort-by key leaves)))) + +;; --------------------------------------------------------------------------- + +(defn problems + "Human-readable reasons this leaf map is not a document. Empty means it is one. + + The dense check is the tier discipline, stated where a save can enforce it: a + document that named `\"take/geom\"` would be a document that only means anything + on the machine that produced it, and the whole point of tier 2 being + content-addressed is that it does not have to travel with tier 1 to be found." + [leaves] + (let [nodes (into #{} (keep (fn [path] + (let [[_ cid kind id] (str/split path #"/")] + (when (= "node" kind) [cid id])))) + (keys leaves))] + (vec + (concat + (for [[path v] (sort-by key leaves) + :let [[root cid kind id prop] (str/split path #"/")] + :when (or (not= "clip" root) (nil? cid) + (not (#{"name" "timing" "stage" "source" "subject" "feature" + "group" "node" "channel" "measured"} kind)))] + (str (pr-str path) " is not a leaf path")) + (for [[path _] (sort-by key leaves) + :let [[_ cid kind id] (str/split path #"/")] + :when (and (#{"channel" "measured"} kind) (not (contains? nodes [cid id])))] + (str (pr-str path) " addresses a node with no node leaf")) + (for [[path v] (sort-by key leaves) + :let [[_ _ kind] (str/split path #"/")] + :when (and (= "channel" kind) (:dense v) + (not (sha/key? (:store (:dense v)))))] + (str (pr-str path) " names tier 2 as " (pr-str (:store (:dense v))) + " — a dense channel in a saved document names a content address")))))) diff --git a/frontend/src/arthur/domain/project.cljs b/frontend/src/arthur/domain/project.cljs new file mode 100644 index 0000000..abedd4a --- /dev/null +++ b/frontend/src/arthur/domain/project.cljs @@ -0,0 +1,100 @@ +(ns arthur.domain.project + "A clip <-> the document that travels. The tier split, as a pair of functions. + + `save` takes what `flow/freeze` produced — `{:scene ... :store ...}` — and + returns two things that are allowed on the wire for different reasons: + + :leaves TIER 1. The document. Nodes, channels, subjects, features, groups, + time maps, the analysis record. Kilobytes, and every byte of it + authored or authorable. + + :blocks TIER 2. The dense blocks the document NAMES, each with the + descriptor its key is the hash of. Megabytes, content-addressed, + and not part of the document — a bake inside the shared document is + a system that puts 48KB on the wire per vertex drag. + + ONLY WHAT THE DOCUMENT NAMES TRAVELS. The blocks are selected by walking the + leaves for dense store keys, not by taking the store wholesale, so a store that + has accumulated a block nothing points at does not upload it. That is also the + check that the split is honest: if a channel named a block the store did not + have, `save` would say so here rather than producing a document that loads into + a blank stage somewhere else. + + THE BYTES ARE NOT IN THE DOCUMENT AND THE TYPE IS NOT IN THE BYTES. A block's + element type comes out of its own descriptor, which is the only place it is + written down: an Int16Array and a Float32Array over the same bytes are both + valid readings of them, and only one is the block. That makes the descriptor + load-bearing rather than documentation, which is the right way round for the + thing a key is the hash of." + (:require [arthur.domain.leaf :as leaf] + [arthur.domain.wire :as wire])) + +(defn block-keys + "Every tier-2 key a leaf map names, in a stable order." + [leaves] + (->> (vals leaves) + (keep (comp :store :dense)) + distinct + sort + vec)) + +(defn- block-type + "A block's element type, out of its descriptor." + [descriptor] + (or (get-in (js->clj (js/JSON.parse descriptor)) ["layout" "type"]) + (throw (ex-info "a block's descriptor does not say what its elements are" + {:descriptor descriptor})))) + +(defn save + "One clip -> the JS object a save PUTs, ready for `JSON.stringify`. + + A JS object rather than CLJS data, and `load` takes one back, because this is + the wire boundary and both ends of it should speak the wire: a test can then + round-trip a clip through `JSON.parse(JSON.stringify(...))` and be running the + same conversion the network runs, rather than a CLJS-shaped rehearsal of it. The + one thing a keywordising `js->clj` would quietly break is the leaf paths — + `:clip/c1/node/mouth` is a keyword whose `name` is \"c1/node/mouth\", so the + \"clip/\" would be lost on the way back in. + + Refuses a document `domain/leaf` calls unaddressable, which is where a hand-made + scene with placeholder store keys — `demo/swarm`'s \"swarm/pos\" — stops rather + than being uploaded as a project that means something only on the machine that + made it." + [cid {:keys [scene store]}] + (let [leaves (leaf/leaves cid scene) + ps (leaf/problems leaves)] + (when (seq ps) + (throw (ex-info (str "this clip cannot be saved: " (first ps)) + {:problems ps}))) + (let [out (js-obj)] + (doseq [[path v] leaves] + (aset out path (wire/encode-json v))) + #js {:leaves out + :blocks (into-array + (map (fn [k] + (let [{:keys [data state descriptor]} + (or (get store k) + (throw (ex-info "the document names a block the store does not have" + {:key k})))] + #js {:key k + :descriptor descriptor + :data (wire/base64 data) + :state (when state (wire/base64 state))})) + (block-keys leaves)))}))) + +(defn load + "The parsed response -> `{:scene :store}`, which is what `flow/freeze` returns + and therefore what the player already knows how to play." + [cid ^js doc] + (let [leaves (.-leaves doc) + tier1 (into {} (map (fn [path] [path (wire/decode-json (aget leaves path))])) + (js-keys leaves))] + {:scene (leaf/scene cid tier1) + :store (into {} + (map (fn [^js b] + [(.-key b) + (cond-> {:descriptor (.-descriptor b) + :data (wire/typed (block-type (.-descriptor b)) + (.-data b))} + (.-state b) (assoc :state (wire/bytes-of (.-state b))))])) + (array-seq (or (.-blocks doc) #js [])))})) diff --git a/frontend/src/arthur/domain/sha256.cljs b/frontend/src/arthur/domain/sha256.cljs new file mode 100644 index 0000000..30d7c6f --- /dev/null +++ b/frontend/src/arthur/domain/sha256.cljs @@ -0,0 +1,148 @@ +(ns arthur.domain.sha256 + "SHA-256, synchronous, in pure ClojureScript. + + WHY NOT `crypto.subtle`. It is async, and every caller here is a pure function + in the `(f params inputs) -> output` shape: a content address is computed in the + middle of `flow/freeze`, inside a `let`, and a promise there would turn the + whole stage inside out. `crypto.createHash` exists in node and not in the + browser, which is worse — the tests would be hashing with a different + implementation from the app. + + WHY NOT A DEPENDENCY. It is sixty lines, it never changes, and the thing it has + to agree with is not another JS library: it is Python's `hashlib`. The server + recomputes the key of every block and every analysis it is handed and refuses a + mismatch (see clips/views.py), so a disagreement between the two languages is + not a hash that looks different — it is an upload that 409s with nothing wrong. + `sha256-test` therefore pins the digests that `hashlib` produced, including the + 55/56/63/64 and 119/120-byte cases either side of both padding boundaries, + which is where a hand-written implementation is wrong if it is wrong at all. + + SIGN. JS bitwise operators work on 32-bit SIGNED integers, so `bit-xor` and + `bit-shift-left` hand back negative numbers, and a negative number entering an + addition mod 2^32 is off by 2^32. Every intermediate that feeds an addition is + therefore normalised through `u32`. That is the bug this implementation would + have, and it is invisible on short inputs — `\"abc\"` passes with the sign bug in + place on some rounds — which is the other reason the vectors above are pinned." + (:require [clojure.string :as str])) + +(def ^:private round-k + (js/Uint32Array. + #js [0x428a2f98 0x71374491 0xb5c0fbcf 0xe9b5dba5 0x3956c25b 0x59f111f1 + 0x923f82a4 0xab1c5ed5 0xd807aa98 0x12835b01 0x243185be 0x550c7dc3 + 0x72be5d74 0x80deb1fe 0x9bdc06a7 0xc19bf174 0xe49b69c1 0xefbe4786 + 0x0fc19dc6 0x240ca1cc 0x2de92c6f 0x4a7484aa 0x5cb0a9dc 0x76f988da + 0x983e5152 0xa831c66d 0xb00327c8 0xbf597fc7 0xc6e00bf3 0xd5a79147 + 0x06ca6351 0x14292967 0x27b70a85 0x2e1b2138 0x4d2c6dfc 0x53380d13 + 0x650a7354 0x766a0abb 0x81c2c92e 0x92722c85 0xa2bfe8a1 0xa81a664b + 0xc24b8b70 0xc76c51a3 0xd192e819 0xd6990624 0xf40e3585 0x106aa070 + 0x19a4c116 0x1e376c08 0x2748774c 0x34b0bcb5 0x391c0cb3 0x4ed8aa4a + 0x5b9cca4f 0x682e6ff3 0x748f82ee 0x78a5636f 0x84c87814 0x8cc70208 + 0x90befffa 0xa4506ceb 0xbef9a3f7 0xc67178f2])) + +(defn- u32 [x] (unsigned-bit-shift-right x 0)) + +(defn- rotr [x n] + (u32 (bit-or (unsigned-bit-shift-right x n) (bit-shift-left x (- 32 n))))) + +(defn- pad + "The message, padded: a 0x80 byte, zeros, and the bit length as a big-endian + 64-bit integer. The length is written as two 32-bit halves because a JS number + cannot hold a 64-bit integer and nothing here will ever hash 512MB." + [^js bytes] + (let [n (.-length bytes) + total (* 64 (js/Math.ceil (/ (+ n 9) 64))) + out (js/Uint8Array. total) + bits (* 8 n)] + (.set out bytes) + (aset out n 0x80) + ;; The high half is the bit count above 2^32; exact for any input JS can hold. + (let [hi (js/Math.floor (/ bits 4294967296)) + lo (u32 bits)] + (dotimes [i 4] + (aset out (+ total -8 i) (bit-and 0xff (unsigned-bit-shift-right hi (* 8 (- 3 i))))) + (aset out (+ total -4 i) (bit-and 0xff (unsigned-bit-shift-right lo (* 8 (- 3 i))))))) + out)) + +(defn digest + "SHA-256 of a Uint8Array, as a Uint8Array of 32 bytes." + [^js bytes] + (let [msg (pad bytes) + h (js/Uint32Array. #js [0x6a09e667 0xbb67ae85 0x3c6ef372 0xa54ff53a + 0x510e527f 0x9b05688c 0x1f83d9ab 0x5be0cd19]) + w (js/Uint32Array. 64) + v (js/Uint32Array. 8)] + (dotimes [block (quot (.-length msg) 64)] + (let [base (* 64 block)] + (dotimes [i 16] + (let [o (+ base (* 4 i))] + (aset w i (u32 (bit-or (bit-shift-left (aget msg o) 24) + (bit-shift-left (aget msg (+ o 1)) 16) + (bit-shift-left (aget msg (+ o 2)) 8) + (aget msg (+ o 3))))))) + (dotimes [j 48] + (let [i (+ j 16) + x (aget w (- i 15)) + y (aget w (- i 2)) + s0 (u32 (bit-xor (rotr x 7) (rotr x 18) (unsigned-bit-shift-right x 3))) + s1 (u32 (bit-xor (rotr y 17) (rotr y 19) (unsigned-bit-shift-right y 10)))] + (aset w i (+ (aget w (- i 16)) s0 (aget w (- i 7)) s1)))) + (.set v h) + (dotimes [i 64] + (let [a (aget v 0) b (aget v 1) c (aget v 2) d (aget v 3) + e (aget v 4) f (aget v 5) g (aget v 6) hh (aget v 7) + s1 (u32 (bit-xor (rotr e 6) (rotr e 11) (rotr e 25))) + choice (u32 (bit-xor (bit-and e f) (bit-and (bit-not e) g))) + t1 (+ hh s1 choice (aget round-k i) (aget w i)) + s0 (u32 (bit-xor (rotr a 2) (rotr a 13) (rotr a 22))) + maj (u32 (bit-xor (bit-and a b) (bit-and a c) (bit-and b c))) + t2 (+ s0 maj)] + (aset v 7 g) (aset v 6 f) (aset v 5 e) + (aset v 4 (+ d t1)) + (aset v 3 c) (aset v 2 b) (aset v 1 a) + (aset v 0 (+ t1 t2)))) + (dotimes [i 8] + (aset h i (+ (aget h i) (aget v i)))))) + (let [out (js/Uint8Array. 32)] + (dotimes [i 8] + (dotimes [b 4] + (aset out (+ (* 4 i) b) + (bit-and 0xff (unsigned-bit-shift-right (aget h i) (* 8 (- 3 b))))))) + out))) + +(defn hex + "Lowercase hex of a byte array, which is the form `hashlib.hexdigest()` gives + and therefore the form a key is written in." + [^js bytes] + (str/join (map (fn [i] (.padStart (.toString (aget bytes i) 16) 2 "0")) + (range (.-length bytes))))) + +(defn of-bytes [^js bytes] (hex (digest bytes))) + +(def ^:private utf8 (js/TextEncoder.)) + +(defn of-string + "UTF-8 first, and that is not a detail: a descriptor holds source filenames, so + a clip called \"café.mov\" hashes to what Python's `hashlib` gives for the same + bytes only if the encoding is agreed. `TextEncoder` is UTF-8 by definition." + [s] + (of-bytes (.encode utf8 s))) + +(defn key-of + "The form a tier-2 key is written in everywhere: \"sha256:<64 hex>\". + + PREFIXED, because a bare hex string in a document says nothing about what + produced it, and the first time this changes algorithm every stored key has to + be readable as the old one. It is also what makes a descriptive placeholder + key — the `\"take/geom\"` these replaced — impossible to confuse with an address." + [s] + (str "sha256:" (of-string s))) + +(defn key? + "Does this string name a content address? + + A string test and not a lookup, on purpose: tier 1 must be checkable without + tier 2 in hand, which is the whole point of the split. It is what `domain/leaf` + uses to refuse a document carrying a placeholder key like the \"take/geom\" that + content addressing replaced." + [s] + (boolean (and (string? s) (re-matches #"sha256:[0-9a-f]{64}" s)))) diff --git a/frontend/src/arthur/domain/wire.cljs b/frontend/src/arthur/domain/wire.cljs new file mode 100644 index 0000000..dc10d57 --- /dev/null +++ b/frontend/src/arthur/domain/wire.cljs @@ -0,0 +1,102 @@ +(ns arthur.domain.wire + "The document's wire format, and the bytes' one. + + TRANSIT, not JSON, and the reason is the two things docs/animation-model.md is + most specific about. A channel's keys are a map BY FRAME NUMBER, and JSON has + only string keys, so a save through `JSON.stringify` turns `{0 v, 4 v}` into + `{\"0\" v, \"4\" v}` and every id in the scene from `:mouth` into `\"mouth\"` — a + document that reloads as a subtly different type and fails somewhere downstream + of where it broke. Transit carries integers, keywords and vector keys as + themselves, and its output is still JSON, so the server stores a leaf in a + JSONField and the admin can read it. + + TRANSIT LOSES SORTEDNESS, which is why `domain/channel` says keys are a PLAIN + map and builds the sorted index at read time. Nothing here re-sorts anything: + a codec that returned a sorted map would work locally and stop working after one + round trip, which is the failure the plain-map rule already prevents. + + The bytes are separate and base64, because tier 2 is typed arrays and transit + has nothing to say about them. `channel/dense-at` reads a block as + `{:data :state }` and both sides of the wire must hold + byte-for-byte the same array — a handle that names a sha256 has to name the + bytes you actually hold." + (:require [cognitect.transit :as t])) + +(def ^:private writer (t/writer :json)) +(def ^:private reader (t/reader :json)) + +(defn encode + "A tier-1 value -> the transit-JSON text that goes in a leaf." + [v] + (t/write writer v)) + +(defn decode + "The inverse. Whatever comes back is ordinary CLJS data." + [s] + (t/read reader s)) + +(defn encode-json + "A tier-1 value -> transit as a PARSED JSON value, ready to go in a request body. + + Transit's output is a JSON string, so a leaf could travel as a string and the + server could store it as one. It travels parsed instead, so that the column + holding it is a JSONField holding JSON rather than a JSONField holding a string + that happens to contain JSON. Two things need that: the admin, where a leaf is + either readable or it is a blob, and the field-wise merge of a channel leaf that + docs/architecture.md describes as fifteen lines of Python — which is fifteen + lines over transit's own `[\"^ \", \"~:keys\", ...]` and impossible over an + opaque string." + [v] + (js/JSON.parse (encode v))) + +(defn decode-json + "The inverse of `encode-json`." + [json] + (decode (js/JSON.stringify json))) + +;; --------------------------------------------------------------------------- +;; the bytes + +(def ^:private chunk-size + "Not `chunk`, which is `cljs.core/chunk`. 8192 characters per `apply`." + 8192) + +(defn base64 + "A typed array -> base64 of its bytes. + + Chunked through `String.fromCharCode`: `apply` with a few hundred thousand + arguments overflows the stack, and a 600-frame geometry block is exactly that + size. The failure is a RangeError from inside a save, which points nowhere near + the array that caused it." + [^js block] + (let [bytes (js/Uint8Array. (.-buffer block) (.-byteOffset block) (.-byteLength block)) + parts (js/Array.)] + (loop [i 0] + (when (< i (.-length bytes)) + (.push parts (.apply js/String.fromCharCode nil (.subarray bytes i (+ i chunk-size)))) + (recur (+ i chunk-size)))) + (js/btoa (.join parts "")))) + +(defn bytes-of + "base64 -> a Uint8Array." + [s] + (let [binary (js/atob s) + out (js/Uint8Array. (.-length binary))] + (dotimes [i (.-length binary)] + (aset out i (.charCodeAt binary i))) + out)) + +(defn typed + "base64 -> the typed array a block of this element type is read through. + + The type is a FIELD the block carries rather than something inferred from its + length, because an Int16Array and a Float32Array over the same bytes are both + valid readings and only one of them is the block." + [type s] + (let [u8 (bytes-of s)] + (case type + "int16" (js/Int16Array. (.-buffer u8)) + "float32" (js/Float32Array. (.-buffer u8)) + "uint8" u8 + (throw (ex-info "a block's element type is \"int16\", \"float32\" or \"uint8\"" + {:type type}))))) diff --git a/frontend/src/arthur/events/footage.cljs b/frontend/src/arthur/events/footage.cljs index f3e6633..c1a5d7f 100644 --- a/frontend/src/arthur/events/footage.cljs +++ b/frontend/src/arthur/events/footage.cljs @@ -1,5 +1,9 @@ (ns arthur.events.footage - "Load and freeze extracted footage once, outside the playback loop." + "Load and freeze ingested footage once, outside the playback loop. + + The frames come from the server by URL since step 9 — see `flow/ingest` — and the + detector's identity comes from the server too, because it goes into the content + address of every block this produces." (:require [arthur.events.playback :as pb] [arthur.flow.detect :as detect] [arthur.flow.ingest :as ingest] @@ -15,8 +19,7 @@ raw (atom []) interiors (atom []) dims (atom nil) - total (:frames manifest) - load-id (.now js/Date)] + total (:frames manifest)] (js/Promise. (fn [resolve reject] (letfn [(next-frame [i] @@ -25,7 +28,7 @@ (resolve (assoc (detect/fill-gaps @raw) :dimensions @dims :interior @interiors)) (catch :default error (reject error))) - (-> (ingest/image! (ingest/frame-url manifest i load-id)) + (-> (ingest/image! (ingest/frame-url manifest i)) (.then (fn [image] (let [wh [(.-naturalWidth image) (.-naturalHeight image)]] @@ -55,19 +58,24 @@ (.catch reject))))] (next-frame 0)))))) -(defn- build-clip [manifest {:keys [dense detected dimensions interior missing first-real]}] +(defn- build-clip [manifest detector + {:keys [dense detected dimensions interior missing first-real]}] (let [[w h] dimensions frozen (take/footage manifest {:dense dense :detected detected :dimensions dimensions :interior interior - :presence (:presence manifest)}) + :presence (:presence manifest) + :detector detector}) scene (:scene frozen)] (assoc (select-keys scene [:fps :frames :width :height]) :display-fps (:fps scene) :scene scene :store (:store frozen) - ;; Re-extraction often overwrites audio.wav under the same name. A new - ;; URL makes the element fetch the new sound when this clip is loaded. - :audio (str (ingest/audio-url manifest) "?v=" (.now js/Date)) - :label (or (:source manifest) "footage") + ;; No cache-buster. The audio is a blob named by the hash of its own + ;; bytes, so re-extracting gives it a different URL rather than + ;; overwriting this one — which is what the `?v=` here used to work + ;; around. + :audio (ingest/audio-url manifest) + :label (or (:label manifest) (:source manifest) "footage") + :cid (or (:id manifest) "footage") :summary (str (:frames manifest) " frames · " w "×" h " · " (:fps manifest) " fps" (when (pos? missing) @@ -77,35 +85,61 @@ (rf/reg-fx ::begin! - (fn [path] - (-> (ingest/manifest! path) - (.then (fn [manifest] + (fn [footage-id] + (-> (js/Promise.all #js [(ingest/manifest! footage-id) (ingest/detector!)]) + (.then (fn [[manifest detector]] (rf/dispatch [::progress "loading MediaPipe…"]) (-> (detect/landmarker!) (.then (fn [model] (rf/dispatch [::progress "loading frames…"]) (-> (detect-frames! manifest model) - (.then (fn [track] (build-clip manifest track))))))))) + (.then (fn [track] + (build-clip manifest detector track))))))))) (.then (fn [entry] (let [id (store/install! entry)] (rf/dispatch [::loaded id (:summary entry)])))) (.catch (fn [error] (js/console.error error) - (rf/dispatch [::failed (or (.-message error) (str error))])))))) + (rf/dispatch [::failed (or (ex-message error) (.-message error) (str error))])))))) + +(rf/reg-fx + ::list! + (fn [_] + (-> (ingest/available!) + (.then (fn [footage] (rf/dispatch [::listed footage]))) + (.catch (fn [error] + (rf/dispatch [::failed (or (ex-message error) (str error))])))))) + +(rf/reg-event-fx + ::refresh + (fn [_ _] {::list! nil})) + +(rf/reg-event-db + ::listed + (fn [db [_ footage]] + (update db :footage merge + {:available (vec footage) + :chosen (or (:chosen (:footage db)) (:id (first footage))) + :status (when (empty? footage) + "no footage ingested — ./extract.sh, then manage.py ingest_bundle")}))) + +(rf/reg-event-db + ::choose + (fn [db [_ id]] (assoc-in db [:footage :chosen] id))) (rf/reg-event-fx ::load (fn [{:keys [db]} _] - (if (get-in db [:footage :loading?]) - {} - {:db (assoc db :footage (assoc (:footage db) :loading? true - :status "reading manifest.json…")) - ::pb/pause! nil - ::begin! (get-in db [:footage :manifest-path])}))) - -(rf/reg-event-db - ::set-manifest-path - (fn [db [_ path]] (assoc-in db [:footage :manifest-path] path))) + (let [chosen (get-in db [:footage :chosen])] + (cond + (get-in db [:footage :loading?]) {} + (nil? chosen) + {:db (assoc-in db [:footage :status] + "no footage ingested — ./extract.sh, then manage.py ingest_bundle")} + :else + {:db (update db :footage merge {:loading? true :status "reading the manifest…"}) + ::pb/pause! nil + ::begin! chosen})))) (rf/reg-event-db ::progress diff --git a/frontend/src/arthur/events/project.cljs b/frontend/src/arthur/events/project.cljs new file mode 100644 index 0000000..1ab6262 --- /dev/null +++ b/frontend/src/arthur/events/project.cljs @@ -0,0 +1,195 @@ +(ns arthur.events.project + "Save and open: the document over HTTP. + + THE ORDER OF A SAVE IS THE TIER SPLIT, and it is not an arrangement of + convenience — each step is the precondition for the next one to be checkable: + + 1. the ANALYSIS record, so that every block stored afterwards can name the + detector version that produced it. The server refuses a block whose + analysis it does not know, for exactly that reason. + 2. ask which BLOCKS are missing, and upload only those. A re-save after a + document edit moves kilobytes, which is the whole return on content + addressing. + 3. the DOCUMENT. The server refuses a clip that names blocks it does not hold, + so a saved document cannot load into a blank stage somewhere else. + + Open is the same order backwards: the document, then the blocks it names. It + needs no analysis step, because the document carries the analysis record — a + content address alone would make a take unreadable the first time a detector + upgrade orphaned one, and \"sha256:7f2…\" is not an answer to \"which model + produced this\". + + Nothing here touches app-db except through events. The promise chain lives in an + fx, which is the only thing in this namespace that is not pure." + (:require [arthur.domain.project :as project] + [arthur.events.playback :as pb] + [arthur.footage.store :as store] + [arthur.flow.address :as address] + [arthur.fx.http :as http] + [re-frame.core :as rf])) + +(defn- analysis-payload [analysis] + #js {:key (:id analysis) + :descriptor (address/analysis-descriptor analysis) + :footage (:footage analysis)}) + +(defn- block-keys [^js doc] + (into-array (map #(.-key %) (array-seq (.-blocks doc))))) + +(defn- upload-missing! + "POST the blocks the server said it does not have, and nothing else. + + ONE AT A TIME. `Promise.all` over eleven uploads is the obvious way to write + this and it made sqlite answer \"database is locked\" on a save — which reaches + the page as a 500 with nothing wrong with the request. The backend was fixed too + (WAL, and a busy timeout, in server/settings.py), and this stays sequential + anyway: the uploads are a few kilobytes each, nothing is waiting on them, and a + burst of parallel writes to buy nothing is how the same bug comes back the first + time a take has sixty blocks instead of eleven." + [^js doc] + (-> (http/POST "/api/blocks/missing" #js {:keys (block-keys doc)}) + (.then (fn [^js answer] + (let [missing (set (array-seq (.-missing answer))) + todo (filterv #(contains? missing (.-key ^js %)) + (array-seq (.-blocks doc)))] + (-> (reduce (fn [chain block] + (.then chain (fn [_] (http/POST "/api/blocks" block)))) + (js/Promise.resolve nil) + todo) + (.then (fn [_] (count todo))))))))) + +(defn- ensure-project! [id name] + (if id + (js/Promise.resolve id) + (-> (http/POST "/api/projects" #js {:name name}) + (.then (fn [^js created] (.-id created)))))) + +(rf/reg-fx + ::save! + (fn [{:keys [id cid label clip]}] + (let [analysis (:analysis (:scene clip)) + doc (project/save cid clip)] + (-> (ensure-project! id label) + (.then (fn [pid] + (-> (if analysis + (http/POST "/api/analyses" (analysis-payload analysis)) + (js/Promise.resolve nil)) + (.then (fn [_] (upload-missing! doc))) + (.then (fn [uploaded] + (-> (http/PUT (str "/api/projects/" pid) + #js {:name label + :clips #js [#js {:cid cid + :name label + :analysis (:id analysis) + :leaves (.-leaves doc) + :blocks (block-keys doc)}]}) + (.then (fn [^js saved] + (rf/dispatch [::saved pid cid label + (.-seq saved) + (count (array-seq (.-written saved))) + uploaded]))))))))) + (.catch (fn [error] + (js/console.error error) + (rf/dispatch [::failed (or (ex-message error) (str error))]))))))) + +(rf/reg-fx + ::open! + (fn [id] + (-> (if id + (js/Promise.resolve #js {:id id}) + ;; No id: the most recently updated project, which is what "open" means + ;; when there is no project browser yet. + (-> (http/GET "/api/projects") + (.then (fn [^js listed] + (or (first (array-seq (.-projects listed))) + (throw (ex-info "there is no saved project to open" {}))))))) + (.then (fn [^js row] (http/GET (str "/api/projects/" (.-id row))))) + (.then (fn [^js loaded] + (let [^js clip-json (first (array-seq (.-clips loaded)))] + (when-not clip-json + (throw (ex-info "that project has no clips" {}))) + (-> (js/Promise.all + (into-array (map #(http/GET (str "/api/blocks/" %)) + (array-seq (.-blocks clip-json))))) + (.then (fn [blocks] + (let [doc #js {:leaves (.-leaves clip-json) :blocks blocks} + cid (.-cid clip-json) + clip (project/load cid doc) + scene (:scene clip) + entry (merge + (select-keys scene [:fps :frames :width :height]) + {:label (str (or (.-name clip-json) cid) " (saved)") + :cid cid + :display-fps (:fps scene) + :scene scene + :store (:store clip) + ;; The audio is the clip's, and a + ;; document does not carry it: tier 3 + ;; is by hash and the scene names the + ;; analysis, not the sound. Until the + ;; footage id is in the document, the + ;; synthetic take's is the one that + ;; keeps the clock running. + :audio "/static/arthur/audio.wav"})] + (rf/dispatch [::opened + (store/install! entry "project") + (.-id loaded) + (.-name loaded) + (.-seq loaded)])))))))) + (.catch (fn [error] + (js/console.error error) + (rf/dispatch [::failed (or (ex-message error) (str error))])))))) + +;; --------------------------------------------------------------------------- +;; events + +(rf/reg-event-fx + ::save + (fn [{:keys [db]} _] + (let [id (:scene/current db) + clip (store/entry id)] + (if (or (:busy? (:project db)) (nil? clip)) + {} + {:db (update db :project merge {:busy? true :status "saving…"}) + ::save! {:id (:id (:project db)) + :cid (or (:cid clip) (name id)) + :label (or (:label clip) (name id)) + :clip clip}})))) + +(rf/reg-event-fx + ::open + (fn [{:keys [db]} _] + (if (:busy? (:project db)) + {} + {:db (update db :project merge {:busy? true :status "opening…"}) + ::pb/pause! nil + ::open! (:id (:project db))}))) + +(rf/reg-event-db + ::saved + (fn [db [_ id cid label seq written uploaded]] + (update db :project merge + {:id id :cid cid :name label :seq seq :busy? false + :status (str "saved r" seq " · " written + (if (= 1 written) " leaf" " leaves") + " · " uploaded (if (= 1 uploaded) " block" " blocks"))}))) + +(rf/reg-event-fx + ::opened + (fn [{:keys [db]} [_ clip-id project-id name seq]] + (let [clip (store/entry clip-id)] + {:db (-> db + (assoc :scene/current clip-id + :clip (select-keys clip [:fps :frames :width :height :audio :display-fps])) + (update :project merge + {:id project-id :name name :seq seq :cid (:cid clip) + :busy? false + :status (str "opened " name " r" seq)}) + (assoc-in [:playback :frame] 0) + (assoc-in [:playback :playing?] false)) + ::pb/pause! nil}))) + +(rf/reg-event-db + ::failed + (fn [db [_ message]] + (update db :project merge {:busy? false :status (str "failed: " message)}))) diff --git a/frontend/src/arthur/flow/address.cljs b/frontend/src/arthur/flow/address.cljs new file mode 100644 index 0000000..809380b --- /dev/null +++ b/frontend/src/arthur/flow/address.cljs @@ -0,0 +1,185 @@ +(ns arthur.flow.address + "Tier 2 keys: what a dense block is NAMED, and what that name is made of. + + Until step 9 a block's key was a descriptive string — \"take/geom\", + \"footage/iris-pos\" — and `flow/freeze` said of them: \"they become the blocks' + sha256 when the backend arrives and nothing above here changes, which is the + point of a handle.\" This is that, and nothing above it did change. + + A KEY IS A HASH OVER INPUTS, NOT OVER BYTES. Both are content addressing and + they answer different questions. Hashing the bytes tells you whether two blocks + are identical; hashing the inputs tells you, BEFORE computing anything, which + block the current settings want — which is the question a cache is asked. It is + also what makes a stale bake unreachable rather than wrong: change a knob and + the scene names a key that no longer exists, so the worst case is a re-freeze. + Nothing in the system can serve old landmarks under new settings. + + THE DETECTOR VERSION IS IN IT, and docs/architecture.md is explicit about why: + a model upgrade that silently reuses old landmarks presents as \"the tool got + worse\", with no event to attach it to. It enters through the ANALYSIS id, which + every block descriptor names, so it cannot be in one block's key and missing + from another's. + + WHICH KNOBS. `block-knobs` below is the invalidation table: for each block, the + settings its BYTES depend on. Getting it wrong in either direction is a bug with + a different symptom — too few and a knob silently does nothing until a reload, + too many and every unrelated tweak throws away a good bake — so it is not + trusted. `address-test` re-freezes the take once per knob and asserts the + biconditional: a block's bytes changed if and only if its key changed. That is + what keeps this table honest, because reading it will not. + + Not a namespace with state, and not a registry: every function here is + `(f inputs) -> string`." + (:require [arthur.domain.canon :as canon] + [arthur.domain.sha256 :as sha])) + +(def ^:const scheme + "The addressing scheme's own version, inside every key. + + If the shape of a descriptor changes — a field added, a field's meaning + revised — then keys computed the old way name bytes produced by code that no + longer exists. Bumping this makes every one of them unreachable in one edit, + which is the cheap version of a migration." + 1) + +;; --------------------------------------------------------------------------- +;; the analysis artifact + +(defn analysis-descriptor + "The canonical text naming one analysis artifact: which detector, at which + version, over which source. + + `:fps` and `:aspect` are in here rather than in the block descriptors, and that + is not an arrangement of convenience. Both are properties of the FOOTAGE — the + source cadence and the pixel aspect of the frames it was decoded from — and both + reach tier 2 bytes: aspect through every landmark that is de-anisotropised + before a fit, and fps through every dwell that is specified in seconds + (`condition/quantize-snap`'s gaze and brow cells, `resolve-blink`'s hold). A + block inherits them by naming the analysis, so they cannot be in one block's key + and missing from another's. + + `:source` is in it and `:name` is NOT. A clip's name is a label a human types; + two clips of the same footage under different names are the same analysis and + must share it, which is the whole return on addressing." + [{:keys [detector version source footage frames fps aspect seed]}] + (when-not (and (string? detector) (seq detector) (string? version) (seq version)) + (throw (ex-info "an analysis names its detector and the detector's VERSION: an upgrade that silently reuses old landmarks is the failure content addressing exists to prevent" + {:detector detector :version version}))) + (canon/write (cond-> {:scheme scheme + :detector detector + :version version + :frames frames + :fps fps + :aspect aspect} + source (assoc :source source) + footage (assoc :footage footage) + seed (assoc :seed seed)))) + +(defn analysis + "An analysis record with its `:id` filled in. The record is tier 1 — it says + what produced the clip's channels — and the id is what `:generated :analysis` + carries on every generated channel." + [record] + (assoc record :id (sha/key-of (analysis-descriptor record)))) + +;; --------------------------------------------------------------------------- +;; the observation masks + +(defn- feature-name + "A feature id as a descriptor string. `canon` refuses a keyword value on + purpose — so that \"mouth\" and :mouth cannot address the same block — and this + is the naming it insists happens at the call site. `nil` is the whole face, + which is what a track with no feature follows." + [id] + (if id (subs (str id) 1) "face")) + +(defn observation + "A digest of exactly the absence data one block reads. + + Digested rather than inlined, for two reasons. A descriptor is meant to be READ + — a stale bake presents as a picture that will not update, and the descriptor is + the only thing that can say which input moved — and a 229-frame boolean mask + inlined in it would bury the knobs it sits beside. And it is per-block: the eye + block reads `:eye-r` and `:eye-l`'s presence and no other feature's, so a gap in + one brow does not rewrite the mouth's address for nothing. + + `nil` features mean the track follows the whole face's detection rather than a + feature's presence, which is what the head's three blocks do." + [features {:keys [detected presence]}] + (let [wanted (sort-by str (distinct (keep identity features))) + masks (into {} (map (fn [id] [id (mapv boolean (get presence id))])) + (filter #(contains? presence %) wanted))] + (when (or detected (seq masks)) + (sha/key-of (canon/write {:scheme scheme + :detected (when detected (mapv boolean detected)) + :presence (into {} (map (fn [[id m]] [(feature-name id) m])) + (sort-by (comp str key) masks))}))))) + +;; --------------------------------------------------------------------------- +;; the blocks + +(def block-knobs + "Per block, the settings its BYTES depend on. The invalidation table, and the + thing `address-test` refuses to take on trust. + + Two entries worth reading twice, because both are asymmetries a reasonable + person would call a mistake: + + The EYE block does not depend on `blink-cut`. A blink is a `[:vis]` key on the + eye's interior — tier 1, editable, a handful of transitions — and the lid + geometry underneath it is the same either way. `iris-size` and `pupil-size` are + missing for the same reason: both land on framed channels, not in a block. + + The TEETH block depends on `aperture-cut`, which nothing else in the table does. + `condition/interior` will not smooth a contour on a frame the teeth are not + shown on, and whether they are shown starts with the mouth being open — so the + mouth's threshold reaches the pixel geometry, while the mouth's own vertex + budget does not reach the teeth at all (the crop is taken from raw landmarks). + It reads like a mistake in both directions and is neither." + {"geom" [:anchor-avg :contour-avg :verts] + "head-pos" [:anchor-avg] + "head-rot" [:anchor-avg] + "head-scale" [:anchor-avg] + "eyes" [:anchor-avg :contour-avg :eye-verts :lash-weight] + "iris-pos" [:anchor-avg :contour-avg :gaze-gain :gaze-step] + "brows" [:anchor-avg :contour-avg :brow-verts :brow-gain :brow-step :brow-weight] + ;; `brow-pos` and not `contour-avg`: the ring is smoothed and the RAISE is not. + ;; `condition/brows` takes the end heights straight from measure, medians them + ;; for a rest position and snaps them onto a grid, and never passes them + ;; through `condition/contours`. Asserted, not assumed — the biconditional in + ;; address-test is what found it here. + "brow-pos" [:anchor-avg :brow-gain :brow-step] + "teeth" [:anchor-avg :aperture-cut :blob-grow :cavity-erode :min-area + :teeth-on :teeth-smooth :teeth-verts :tongue-reject :top-bias]}) + +(defn block-descriptor + "The canonical text naming one dense block. + + `:tracks` is in it and so is `:layout`: two blocks over the same inputs that + pack a different number of tracks, or the same tracks in another order, are + different bytes at the same offsets, and a reader that trusted the key would + hand the left eye's geometry to the right one." + [{:keys [role analysis params features tracks layout observation]}] + (let [knobs (or (get block-knobs role) + (throw (ex-info "no invalidation table for this block role: add it to block-knobs in the same commit as the block, or its key cannot change when its bytes do" + {:role role :roles (sort (keys block-knobs))}))) + missing (remove #(contains? params %) knobs)] + (when (seq missing) + (throw (ex-info "a knob this block's bytes depend on was not passed to the freeze" + {:role role :missing (vec missing)}))) + (canon/write {:scheme scheme + :role role + :analysis analysis + :params (select-keys params knobs) + :features (mapv feature-name features) + :tracks (vec tracks) + :layout layout + :observation observation}))) + +(defn block + "`{:key :descriptor}` for one dense block. The descriptor travels with the bytes + — see `arthur.domain.leaf` and clips/views.py — because the server verifies + `sha256(descriptor) == key` on upload rather than trusting a name it was handed." + [spec] + (let [text (block-descriptor spec)] + {:key (sha/key-of text) :descriptor text})) diff --git a/frontend/src/arthur/flow/detect.cljs b/frontend/src/arthur/flow/detect.cljs index a2986dd..50cccf0 100644 --- a/frontend/src/arthur/flow/detect.cljs +++ b/frontend/src/arthur/flow/detect.cljs @@ -6,7 +6,13 @@ (defn landmarker! "Initialize once, using the vendored wasm and the local model. CPU also works - in browsers where a GPU delegate initializes but fails on its first frame." + in browsers where a GPU delegate initializes but fails on its first frame. + + Under `/static/` since step 9: the assets still live in `frontend/public/mediapipe` + — 26MB of wasm and model that has no business being copied into a second place in + the tree — and Django's staticfiles serves that directory under the `mediapipe/` + prefix. Still no CDN, which is the property that matters: the only thing in this + tool that would silently require a network is the one thing that must not." [] (if-let [model @instance] (js/Promise.resolve model) @@ -14,12 +20,12 @@ (if-let [vision (aget js/window "Vision")] (let [resolver (aget vision "FilesetResolver") landmarker (aget vision "FaceLandmarker") - ready (-> (.call (aget resolver "forVisionTasks") resolver "/mediapipe/wasm") + ready (-> (.call (aget resolver "forVisionTasks") resolver "/static/mediapipe/wasm") (.then (fn [fileset] (.call (aget landmarker "createFromOptions") landmarker fileset #js {:baseOptions - #js {:modelAssetPath "/mediapipe/face_landmarker.task" + #js {:modelAssetPath "/static/mediapipe/face_landmarker.task" :delegate "CPU"} :runningMode "IMAGE" :numFaces 1}))) diff --git a/frontend/src/arthur/flow/freeze.cljs b/frontend/src/arthur/flow/freeze.cljs index cd093a3..0ac8a9b 100644 --- a/frontend/src/arthur/flow/freeze.cljs +++ b/frontend/src/arthur/flow/freeze.cljs @@ -34,7 +34,8 @@ decisions about the mouth cavity, blink and teeth without thinning geometry." (:require [arthur.domain.channel :as ch] [arthur.domain.geom :as geom] - [arthur.domain.ring :as ring])) + [arthur.domain.ring :as ring] + [arthur.flow.address :as address])) ;; --------------------------------------------------------------------------- ;; fixed point @@ -74,22 +75,49 @@ a performance asset rather than a cost. `demo/swarm` holds the same layout and is the load test for it. - `:ctor` makes the array — a function and not a type, because `new` is not - something a value can carry — `:scale` is the fixed-point scale or nil, and - `:absent` an optional (track, frame) predicate. The state is per track, so one - occluded eye can be absent while its partner still has a value. A full-face - miss marks every track absent. + `:type` names the array — \"int16\" or \"float32\" — rather than handing over a + constructor, because the type is also a field in the block's descriptor and the + two must not be able to disagree. `:scale` is the fixed-point scale or nil. + + ABSENCE IS PER TRACK, and `:features` is what says whose. Each track names the + feature it follows, so one occluded eye can be absent while its partner still + has a value; `nil` means the track follows the whole face's detection and no + feature, which is what the head's blocks do. `absent?` is then asked + `(absent? feature f)` and never about a track index. + + An earlier shape passed a `(track, frame)` predicate instead, and each call site + derived a feature from an index — `(if (< i 2) :eye-r :eye-l)` — so the + predicate and the vector of tracks beside it had to agree BY HAND, in five + places, with a left/right swap for a failure mode. docs/port-plan.md warns about + that swap twice: every part is still roughly where it belongs, so it survives + inspection. Naming the feature per track deletes the derivation, and it hands + `flow/address` the same list for the block's observation digest, so the key and + the mask cannot disagree either. + + `:missing` is an additional per-track predicate for absence that is not a + feature's: the teeth have no contour on a frame no contour could be extracted + from, which is a different fact from the teeth being occluded. Written with `dotimes` and `aset` rather than as a fold, and that is the exception rather than the rule in this codebase: the destination is a typed array, so there is nothing to accumulate into and a collection idiom here would allocate a seq per frame to throw away." - [{:keys [ctor scale absent]} tracks] + [{:keys [type scale features absent? missing]} tracks] + (when-not (= (count tracks) (count features)) + (throw (ex-info "every track of a block names the feature it follows" + {:tracks (count tracks) :features (count features)}))) (let [n (count tracks) nf (count (first tracks)) stride (count (first (first tracks))) + ctor (case type + "int16" #(js/Int16Array. %) + "float32" #(js/Float32Array. %) + (throw (ex-info "a block's element type is \"int16\" or \"float32\"" + {:type type}))) data (ctor (* n nf stride)) - state (when absent (js/Uint8Array. (* n nf)))] + gone? (fn [i f] (or (and absent? (absent? (nth features i) f)) + (and missing (missing i f)))) + state (when (or absent? missing) (js/Uint8Array. (* n nf)))] (dotimes [i n] (let [track (vec (nth tracks i)) base (* i nf stride)] @@ -109,16 +137,55 @@ :track i :frame f :component k}))) q) v)))) - (when (and state (absent i f)) + (when (and state (gone? i f)) (aset state (+ (* i nf) f) ch/absent-bit)))))) - {:data data :state state :stride stride :frames nf :scale scale + {:data data :state state :stride stride :frames nf :scale scale :type type + :features features :offsets (mapv #(* % nf stride) (range n))})) +(defn- block + "Pack the tracks and NAME the result: `pack`'s block plus the `:key` it is + stored under and the `:descriptor` that key is the hash of. + + Addressing happens HERE, beside the packing, rather than at the call sites, + because a block referenced under one key and stored under another is a handle + into somebody else's array — the failure the key exists to make impossible. + + `spec` is the block's identity for `flow/address`: its role, the tracks by name, + and the analysis and settings its bytes came out of. `obs` is the absence data, + which the descriptor digests down to one line." + [{:keys [role analysis params tracks] :as spec} opts obs values] + (let [features (:features opts) + blk (pack opts values) + named (address/block + {:role role :analysis analysis :params params :tracks tracks + :features features + :observation (address/observation features obs) + :layout {:type (:type blk) :scale (:scale blk) + :stride (:stride blk) :frames (:frames blk) + :tracks (count tracks)}})] + (merge blk named))) + +(defn- stored + "Blocks -> the tier-2 store they go in: key -> what is kept under it. + + THE DESCRIPTOR TRAVELS WITH THE BYTES. It is not in the document — tier 1 stays + the authored layer and a descriptor is derived — and it is not thrown away + either, because the server verifies `sha256(descriptor) == key` on upload and + will not take a name on trust. So it rides in tier 2, where a cache entry + knowing what produced it is the ordinary arrangement. `channel/dense-at` reads + `:data` and `:state` and ignores the rest." + [& blocks] + (into {} (map (juxt :key #(select-keys % [:data :state :descriptor]))) blocks)) + (defn- dense - "Track i of a packed block, as a DENSE channel definition." - [store-key blk i generated] + "Track i of a packed block, as a DENSE channel definition. + + The block names its own key, so a channel cannot be pointed at one block and + stored under another's address." + [blk i generated] {:animated? true :interp :hold - :dense (cond-> {:store store-key + :dense (cond-> {:store (:key blk) :offset (nth (:offsets blk) i) :stride (:stride blk) :frames (:frames blk)} @@ -361,43 +428,44 @@ "Freeze eyes and brows into their own dense blocks and scene nodes. This owns only representation: the landmark correspondence, blink and pose choices have already been settled by measure and condition." - [name absent {:keys [eye-verts brow-verts analysis contour-avg anchor-avg]} + [absent? obs {:keys [eye-verts brow-verts analysis contour-avg anchor-avg] :as params} {:keys [eyes brows]}] - (let [eye-k (str name "/eyes") - iris-k (str name "/iris-pos") - brow-k (str name "/brows") - brow-pos-k (str name "/brow-pos") - provenance (fn [by extra] - {:by by :analysis analysis + (let [provenance (fn [by extra] + {:by by :analysis (:id analysis) :params (merge {:anchor-avg anchor-avg :contour-avg contour-avg} extra)}) - eye-block (pack {:ctor #(js/Int16Array. %) :scale geom-scale - :absent (when absent (fn [i f] - (absent (if (< i 2) :eye-r :eye-l) f)))} - (mapv #(rings->flat % eye-verts) - [(:lash-r eyes) (:lid-r eyes) - (:lash-l eyes) (:lid-l eyes)])) - iris-block (pack {:ctor #(js/Float32Array. %) - :absent (when absent (fn [i f] - (absent (if (zero? i) :eye-r :eye-l) f)))} - [(:iris-r eyes) (:iris-l eyes)]) - brow-block (pack {:ctor #(js/Int16Array. %) :scale geom-scale - :absent (when absent (fn [i f] - (absent (if (zero? i) :brow-r :brow-l) f)))} - (mapv #(rings->flat % brow-verts) - [(:ring-r brows) (:ring-l brows)])) - brow-pos-block (pack {:ctor #(js/Float32Array. %) - :absent (when absent (fn [i f] - (absent (if (zero? i) :brow-r :brow-l) f)))} + named (fn [role tracks features type values] + (block {:role role :analysis (:id analysis) :params params + :tracks tracks} + {:type type :features features :absent? absent? + :scale (when (= "int16" type) geom-scale)} + obs values)) + ;; Each block's tracks, named, in the order they are packed — and the + ;; feature each one follows, in the same order. The two vectors are read + ;; together on purpose: this is the mapping `pack` cannot check for itself, + ;; and `each-dense-track-follows-its-own-features-presence` is what pins it. + eye-block (named "eyes" + ["lash-r" "lid-r" "lash-l" "lid-l"] + [:eye-r :eye-r :eye-l :eye-l] + "int16" + (mapv #(rings->flat % eye-verts) + [(:lash-r eyes) (:lid-r eyes) + (:lash-l eyes) (:lid-l eyes)])) + iris-block (named "iris-pos" ["iris-r" "iris-l"] [:eye-r :eye-l] "float32" + [(:iris-r eyes) (:iris-l eyes)]) + brow-block (named "brows" ["ring-r" "ring-l"] [:brow-r :brow-l] "int16" + (mapv #(rings->flat % brow-verts) + [(:ring-r brows) (:ring-l brows)])) + brow-pos-block (named "brow-pos" ["pos-r" "pos-l"] [:brow-r :brow-l] "float32" [(:pos-r brows) (:pos-l brows)]) eye-node (fn [id z track] {:id id :name (clojure.core/name id) :kind :poly :parent :head :z z - :channels {[:geom :pts] (dense eye-k eye-block track + :channels {[:geom :pts] (dense eye-block track (provenance :roto/eyelid {:verts eye-verts})) [:style :color] (ch/framed :skin-dark)}}) inner-node (fn [id parent z track shut] {:id id :name (clojure.core/name id) :kind :poly :parent parent :z z - :channels {[:geom :pts] (dense eye-k eye-block track + :channels {[:geom :pts] (dense eye-block track (provenance :roto/eye-opening {:verts eye-verts})) [:style :color] (ch/framed :eye-white) @@ -406,7 +474,7 @@ iris-node (fn [id parent track radius] {:id id :name (clojure.core/name id) :kind :disc :parent parent :z "a1" :stencil parent - :channels {[:xform :pos] (dense iris-k iris-block track + :channels {[:xform :pos] (dense iris-block track (provenance :roto/gaze nil)) [:geom :radius] (ch/framed radius) [:style :color] (ch/framed :iris)}}) @@ -417,9 +485,9 @@ [:style :color] (ch/framed :pupil)}}) brow-node (fn [id z track] {:id id :name (clojure.core/name id) :kind :poly :parent :head :z z - :channels {[:geom :pts] (dense brow-k brow-block track + :channels {[:geom :pts] (dense brow-block track (provenance :roto/brow {:verts brow-verts})) - [:xform :pos] (dense brow-pos-k brow-pos-block track + [:xform :pos] (dense brow-pos-block track (provenance :roto/brow-raise nil)) [:style :color] (ch/framed :brow)}})] {:nodes {:eye-r (eye-node :eye-r "a2" 0) @@ -432,28 +500,29 @@ :pupil-l (pupil-node :pupil-l :iris-l) :brow-r (brow-node :brow-r "a4" 0) :brow-l (brow-node :brow-l "a5" 1)} - :store {eye-k (select-keys eye-block [:data :state]) - iris-k (select-keys iris-block [:data :state]) - brow-k (select-keys brow-block [:data :state]) - brow-pos-k (select-keys brow-pos-block [:data :state])}})) + :store (stored eye-block iris-block brow-block brow-pos-block)})) (defn- interior-part "Freeze the pixel-derived radial contour under the mouth cavity. Missing contours use the dense block's absence bit; contrast decides editable :vis." - [name {:keys [analysis teeth-verts cavity-erode tongue-reject blob-grow - top-bias teeth-on teeth-smooth]} absent-feature + [{:keys [analysis teeth-verts cavity-erode tongue-reject blob-grow + top-bias teeth-on teeth-smooth] :as params} absent? obs {:keys [contours shown]}] - (let [key (str name "/teeth") - absent (fn [_ f] (or (nil? (nth contours f)) - (and absent-feature (absent-feature :teeth f)))) - empty-points (vec (repeat (* 2 teeth-verts) 0)) + (let [empty-points (vec (repeat (* 2 teeth-verts) 0)) values (mapv (fn [ring] (if ring (into [] (mapcat (juxt :x :y)) ring) empty-points)) contours) - block (pack {:ctor #(js/Int16Array. %) :scale geom-scale :absent absent} - [values]) - generated {:by :pixels/teeth :analysis analysis + blk (block {:role "teeth" :analysis (:id analysis) :params params + :tracks ["contour"]} + {:type "int16" :scale geom-scale + :features [:teeth] :absent? absent? + ;; Not the feature's absence: a frame no contour could be + ;; extracted from has no teeth to draw whether or not the + ;; teeth were occluded, and the two reasons are different facts. + :missing (fn [_ f] (nil? (nth contours f)))} + obs [values]) + generated {:by :pixels/teeth :analysis (:id analysis) :params {:cavity-erode cavity-erode :tongue-reject tongue-reject :blob-grow blob-grow :top-bias top-bias :teeth-verts teeth-verts @@ -461,10 +530,10 @@ {:nodes {:teeth {:id :teeth :name "teeth" :kind :poly :parent :mouth-in :z "a1" :stencil :mouth-in - :channels {[:geom :pts] (dense key block 0 generated) + :channels {[:geom :pts] (dense blk 0 generated) [:style :color] (ch/framed :teeth) [:vis] (keyed-visibility shown generated)}}} - :store {key (select-keys block [:data :state])}})) + :store (stored blk)})) ;; --------------------------------------------------------------------------- ;; the clip @@ -474,7 +543,9 @@ blocks they read. `(f params inputs)`, no state. params - :name names the clip and its store keys + :name labels the clip. It is NOT in any key: two clips of the same + footage under different names are the same analysis and the + same blocks, and sharing them is the return on addressing. :fps the clip's rate. FRAMES is not a parameter — it is `(count outer)`, because a freeze that could disagree with its own input about the length of the take would. @@ -489,7 +560,11 @@ interior is not present :head :locked | :as-filmed | :per-plate :kept frames, for :per-plate only - :analysis which analysis artifact these measurements came from + :analysis the analysis record these measurements came from — detector, + VERSION, source, source cadence and pixel aspect. Its `:id` is + a content address over all of that, every block's key is a hash + over that id, and `flow/address` says why the version being in + there is the one field that must not be forgotten. :anchor-avg :contour-avg the stage-4 knobs. Freeze does not use them; it RECORDS them, because `:generated` is what lets the UI offer a re-freeze at @@ -537,29 +612,39 @@ (when (not= nf (count track)) (throw (ex-info "feature presence track must match the clip" {:feature id :frames nf :actual (count track)})))) - absent (when (or detected presence) + ;; Asked `(absent? feature f)`, where a nil feature is the whole face. A + ;; feature with no presence track is present whenever a face was found. + absent? (when (or detected presence) (fn [id f] (or (and detected (not (nth detected f true))) (and (contains? presence id) (not (nth (get presence id) f)))))) - head-absent (when detected (fn [_ f] (not (nth detected f true)))) - geom-k (str name "/geom") - head-k #(str name "/head-" %) + obs {:detected detected :presence presence} prov (fn [by extra] - {:by by :analysis analysis + {:by by :analysis (:id analysis) :params (merge {:anchor-avg anchor-avg} extra)}) - rings (pack {:ctor #(js/Int16Array. %) :scale geom-scale - :absent (when absent (fn [_ f] (absent :mouth f)))} - [(rings->flat outer verts) (rings->flat inner verts)]) + rings (block {:role "geom" :analysis (:id analysis) :params params + :tracks ["outer" "inner"]} + {:type "int16" :scale geom-scale + :features [:mouth :mouth] :absent? absent?} + obs + [(rings->flat outer verts) (rings->flat inner verts)]) ;; The anchor, inverted and split into its three components. Three blocks ;; and not one: they are three channels, they have three strides, and a ;; single block would need a per-component offset table to say so. inv (mapv invert transforms) - xf (fn [f] (pack {:ctor #(js/Float32Array. %) :absent head-absent} - [(mapv f inv)])) - pos (xf (fn [t] [(:tx t) (:ty t)])) - rot (xf (fn [t] [(:theta t)])) - scale (xf (fn [t] [(:s t) (:s t)])) + xf (fn [role f] + ;; The head follows DETECTION and no feature's presence: an + ;; occluded eye does not mean the head was not there. That is + ;; what the nil feature says. + (block {:role role :analysis (:id analysis) :params params + :tracks [role]} + {:type "float32" :features [nil] :absent? absent?} + {:detected detected} + [(mapv f inv)])) + pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)])) + rot (xf "head-rot" (fn [t] [(:theta t)])) + scale (xf "head-scale" (fn [t] [(:s t) (:s t)])) ;; Which knobs each channel records is not decoration, it is the ;; invalidation table written down where a re-freeze can read it. The ;; anchor depends on `anchor avg` alone. The rings depend on it and on @@ -568,11 +653,16 @@ ;; measure reports the inner ring's own height and nothing smooths it. anchor-prov (prov :anchor/similarity nil) roto (fn [by] (prov by {:verts verts :contour-avg contour-avg})) - features (when (and eyes brows) (feature-parts name absent params inputs)) - interior (when teeth (interior-part name params absent teeth)) + features (when (and eyes brows) (feature-parts absent? obs params inputs)) + interior (when teeth (interior-part params absent? obs teeth)) scene {:name name :frames nf :fps fps + ;; Tier 1 says which analysis its channels came out of, in full. + ;; The id alone would make the document unreadable the first time + ;; a detector upgrade orphaned a block: "sha256:7f2…" is not an + ;; answer to "which model produced this take". + :analysis analysis ;; A gap changes channel state, never these IDs or pair links. :subjects {:face-1 {:id :face-1 :params {}}} :features (merge @@ -610,9 +700,9 @@ :head {:id :head :name "head" :kind :group :parent :face :z "a1" - :measured {[:xform :pos] (dense (head-k "pos") pos 0 anchor-prov) - [:xform :rot] (dense (head-k "rot") rot 0 anchor-prov) - [:xform :scale] (dense (head-k "scale") scale 0 anchor-prov)}} + :measured {[:xform :pos] (dense pos 0 anchor-prov) + [:xform :rot] (dense rot 0 anchor-prov) + [:xform :scale] (dense scale 0 anchor-prov)}} ;; The outer lip ring is the dark band OUTSIDE the interior, and ;; that three-layer structure — dark ring, pale interior, teeth @@ -620,28 +710,27 @@ ;; as a blob. So it keeps every frame and is never hidden. :mouth {:id :mouth :name "mouth" :kind :poly :parent :head :z "a1" - :channels {[:geom :pts] (dense geom-k rings 0 (roto :roto/lips-outer)) + :channels {[:geom :pts] (dense rings 0 (roto :roto/lips-outer)) [:style :color] (ch/framed :skin-dark)}} :mouth-in {:id :mouth-in :name "mouth interior" :kind :poly :parent :mouth :z "a2" - :channels {[:geom :pts] (dense geom-k rings 1 (roto :roto/lips-inner)) + :channels {[:geom :pts] (dense rings 1 (roto :roto/lips-inner)) [:style :color] (ch/framed :mouth-dark) [:vis] (visibility params inputs (prov :roto/mouth-aperture {:aperture-cut aperture-cut}))}}} (:nodes features) (:nodes interior))} - presence-check (doseq [id (keys presence)] - (when-not (contains? (:features scene) id) - (throw (ex-info "presence track names no feature in this scene" - {:feature id :features (keys (:features scene))})))) - ;; Tier 2, behind a handle. The keys are descriptive because there is no - ;; hashing yet; they become the blocks' sha256 when the backend arrives - ;; and nothing above here changes, which is the point of a handle. - store (merge (into {} (map (fn [[k blk]] [k (select-keys blk [:data :state])])) - {geom-k rings - (head-k "pos") pos, (head-k "rot") rot, (head-k "scale") scale}) + ;; Tier 2, behind a handle, and now behind a content address: every key is + ;; a sha256 over the analysis, the settings and the absence data that + ;; produced the bytes under it. Nothing above this line changed when they + ;; stopped being "take/geom", which is the point of a handle. + store (merge (stored rings pos rot scale) (:store features) (:store interior))] + (doseq [id (keys presence)] + (when-not (contains? (:features scene) id) + (throw (ex-info "presence track names no feature in this scene" + {:feature id :features (keys (:features scene))})))) {:store store :scene (head-mode {:mode head :kept kept} {:scene scene :store store})})) diff --git a/frontend/src/arthur/flow/ingest.cljs b/frontend/src/arthur/flow/ingest.cljs index c657309..8569056 100644 --- a/frontend/src/arthur/flow/ingest.cljs +++ b/frontend/src/arthur/flow/ingest.cljs @@ -1,6 +1,25 @@ (ns arthur.flow.ingest - "Read a pre-extracted take. The manifest owns timing and the exact frame count." - (:require [clojure.string :as str])) + "Read an ingested take. The manifest owns timing, the exact frame count, and — + since step 9 — the URL of every frame. + + WHAT CHANGED, AND WHY IT IS NOT A DETAIL. This used to fetch `/manifest.json` off + the filesystem and then build `frames/0001.png` itself, with shadow-cljs serving + the repo root. So the frame layout was a shared secret between a shell script and + this namespace, and \"where are the frames\" was answered by a directory listing + that nothing could version. + + Now the server names every frame and this asks it. The manifest carries a URL per + frame, so the frames can live in a content-addressed blob store — or, when the + in-browser wasm-ffmpeg extraction docs/architecture.md describes arrives, be + uploaded into the same store by the app itself — and nothing in here learns + anything new. That is the whole point of tier 3 being addressed rather than + located. + + The cache-busting `?v=` that used to hang off every frame URL went with it. It + was there because re-extracting overwrote `frames/0001.png` under the same name; + a blob's name IS the hash of its bytes, so a stale copy is not a thing that can + happen." + (:require [arthur.fx.http :as http])) (defn feature-presence "Expand one-based, inclusive absence intervals from a manifest into boolean @@ -30,36 +49,53 @@ (defn- valid-manifest [m] (let [fps (js/Number (:fps m)) - frames (js/Number (:frames m))] + frames (js/Number (:frames m)) + urls (:urls m)] (when-not (and (js/Number.isFinite fps) (pos? fps) (js/Number.isInteger frames) (<= 1 frames 900) - (string? (:dir m)) (seq (:dir m)) - (string? (:audio m)) (seq (:audio m))) - (throw (ex-info "manifest.json needs fps, frames (1–900), dir and audio" {:manifest m}))) - (assoc m :fps fps :frames frames + (string? (:audio m)) (seq (:audio m)) + (sequential? urls) (every? string? urls)) + (throw (ex-info "a footage manifest needs fps, frames (1–900), audio and a url per frame" + {:manifest (dissoc m :urls)}))) + (when-not (= frames (count urls)) + ;; The count is the manifest's and the URLs are the manifest's, so a + ;; disagreement between them is the server contradicting itself — and it + ;; would present as a take that is silently short. + (throw (ex-info "the manifest's frame count and its list of frames disagree" + {:frames frames :urls (count urls)}))) + (assoc m :fps fps :frames frames :urls (vec urls) :presence (feature-presence frames (:feature-absence m))))) +(defn available! + "Every ingested take the server holds. `manage.py ingest_bundle` is what puts one + there." + [] + (-> (http/GET "/api/footage") + (.then (fn [json] (:footage (js->clj json :keywordize-keys true)))))) + (defn manifest! - [path] - (-> (js/fetch (str "/" (str/replace path #"^/+" "")) #js {:cache "no-store"}) - (.then (fn [response] - (when-not (.-ok response) - (throw (ex-info "manifest.json was not found; run extract.sh first" - {:status (.-status response)}))) - (.json response))) + "One take's manifest, including a URL per frame." + [id] + (-> (http/GET (str "/api/footage/" id)) (.then (fn [json] (valid-manifest (js->clj json :keywordize-keys true)))))) -(defn- asset-url [path] - ;; Manifest paths are relative to the extraction root, served at / in dev. - (str "/" (str/replace path #"^/+" ""))) +(defn detector! + "Who is about to do the detecting, as the server understands it: the MediaPipe + package version and the hash of the model asset it serves. + + ASKED RATHER THAN ASSUMED, because this string ends up inside every block's + content address, and a version constant in the client is one somebody has to + remember to bump. The server serves the model, so it can hash it — and then the + version is a fact about the bytes that produced the landmarks." + [] + (-> (http/GET "/api/detector") + (.then (fn [json] (js->clj json :keywordize-keys true))))) (defn audio-url [manifest] - (asset-url (:audio manifest))) + (:audio manifest)) -(defn frame-url [manifest i load-id] - (str (asset-url (str (str/replace (:dir manifest) #"/+$" "") - "/" (.padStart (str (inc i)) 4 "0") ".png")) - "?v=" load-id)) +(defn frame-url [manifest i] + (nth (:urls manifest) i)) (defn image! [src] (js/Promise. diff --git a/frontend/src/arthur/flow/take.cljs b/frontend/src/arthur/flow/take.cljs index 8fb75b7..a0cce80 100644 --- a/frontend/src/arthur/flow/take.cljs +++ b/frontend/src/arthur/flow/take.cljs @@ -1,6 +1,7 @@ (ns arthur.flow.take "The shared landmark-to-channel path for synthetic and detected takes." - (:require [arthur.flow.condition :as condition] + (:require [arthur.flow.address :as address] + [arthur.flow.condition :as condition] [arthur.flow.condition.brows :as condition-brows] [arthur.flow.condition.eyes :as condition-eyes] [arthur.flow.condition.interior :as condition-interior] @@ -53,12 +54,25 @@ "A real manifest and its detected landmarks through the same measurement and freeze path as the synthetic take. The source cadence stays in :fps; picture sampling is a root time map applied only after this artifact exists." - [manifest {:keys [dense detected dimensions interior presence]}] + [manifest {:keys [dense detected dimensions interior presence detector]}] (let [[w h] dimensions params (merge knobs - {:name "footage" :fps (:fps manifest) :aspect (/ w h) + {:name (or (:source manifest) "footage") + :fps (:fps manifest) :aspect (/ w h) :stage [320 200] :fit-motion? true :expose 1 :head :as-filmed - :analysis (str "mediapipe:1.0.1/" (:source manifest))})] + ;; The detector's identity comes from the server, which + ;; hashes the model asset it serves rather than trusting a + ;; version string somebody has to remember to bump. See + ;; `flow/address`: a model upgrade that silently reused + ;; these landmarks is the failure this prevents. + :analysis (address/analysis + (merge {:detector "mediapipe" :version "unknown"} + detector + {:source (:source manifest) + :footage (:footage manifest) + :frames (:frames manifest) + :fps (:fps manifest) + :aspect (/ w h)}))})] (build params {:dense dense :detected detected :interior interior :presence presence}))) diff --git a/frontend/src/arthur/footage/store.cljs b/frontend/src/arthur/footage/store.cljs index 21a6cd8..a11b74b 100644 --- a/frontend/src/arthur/footage/store.cljs +++ b/frontend/src/arthur/footage/store.cljs @@ -1,14 +1,28 @@ (ns arthur.footage.store - "Loaded clip artifacts live outside app-db. The db keeps only their id." + "Clips loaded at RUNTIME live outside app-db. The db keeps only their id. + + Two things arrive this way and they are the same kind of thing: footage that has + been detected and frozen, and a project opened from the server. Both are + `{:scene ... :store ...}` — which is what `flow/freeze` returns — plus the clip + facts the transport needs, and both hold typed arrays that have no business being + in a map every mounted subscription compares." (:require [arthur.db :as db])) (defonce ^:private loaded (atom nil)) (defonce ^:private serial (atom 0)) -(defn install! [entry] - (let [id (keyword "footage" (str (swap! serial inc)))] - (reset! loaded (assoc entry :id id)) - id)) +(defn install! + "Hold one loaded clip, and hand back the id app-db will refer to it by. + + ONE AT A TIME, on purpose: a second loaded clip is a second megabyte-scale store + with nothing to evict it, and the timeline that would want several is out of + scope. `kind` only names the id — `:footage/3`, `:project/4` — so that a clip's + origin is legible in the db without a lookup." + ([entry] (install! entry "footage")) + ([entry kind] + (let [id (keyword kind (str (swap! serial inc)))] + (reset! loaded (assoc entry :id id)) + id))) (defn entry [id] (if (= id (:id @loaded)) diff --git a/frontend/src/arthur/fx/http.cljs b/frontend/src/arthur/fx/http.cljs new file mode 100644 index 0000000..d594c37 --- /dev/null +++ b/frontend/src/arthur/fx/http.cljs @@ -0,0 +1,55 @@ +(ns arthur.fx.http + "The one place that talks to the server. + + Everything here returns a promise of a PARSED JS VALUE, not of CLJS data, and + that is deliberate: a leaf is transit, and `domain/project` reads it straight out + of the response object. A keywordising `js->clj` on the way past would turn the + leaf path \"clip/c1/node/mouth\" into a keyword whose name is \"c1/node/mouth\", + losing the prefix — a corruption that only shows up on the way back in. + + CSRF IS NOT EXEMPTED. The page renders `{% csrf_token %}`, so Django sets its + cookie, and every unsafe request carries it back in the header Django looks for. + Nine lines against `@csrf_exempt` on an API that writes the document." + (:require [clojure.string :as str])) + +(defn csrf-token [] + (some (fn [pair] + (let [[k v] (str/split pair #"=" 2)] + (when (= "csrftoken" (str/trim (or k ""))) v))) + (str/split (or (.-cookie js/document) "") #";"))) + +(defn- fail + "Turn a non-2xx into an ex-info carrying what the server said. + + The server's message is the useful one — \"this clip names tier-2 blocks the + server does not have\" — and a status code alone would put the interesting half + of it in a console nobody is watching." + [response body] + (throw (ex-info (or (some-> body .-error) + (str "the server answered " (.-status response))) + {:status (.-status response) + :body (when body (js->clj body :keywordize-keys true))}))) + +(defn request! + ([method url] (request! method url nil)) + ([method url body] + (-> (js/fetch url + (clj->js (cond-> {:method method + :cache "no-store" + :headers (cond-> {"Accept" "application/json"} + body (assoc "Content-Type" "application/json") + (not= "GET" method) + (assoc "X-CSRFToken" (or (csrf-token) "")))} + body (assoc :body (js/JSON.stringify body))))) + (.then (fn [response] + (-> (.text response) + (.then (fn [text] + (let [parsed (when (seq text) + (try (js/JSON.parse text) (catch :default _ nil)))] + (if (.-ok response) + parsed + (fail response parsed))))))))))) + +(defn GET [url] (request! "GET" url)) +(defn POST [url body] (request! "POST" url body)) +(defn PUT [url body] (request! "PUT" url body)) diff --git a/frontend/src/arthur/subs/playback.cljs b/frontend/src/arthur/subs/playback.cljs index 41f7102..c8b176e 100644 --- a/frontend/src/arthur/subs/playback.cljs +++ b/frontend/src/arthur/subs/playback.cljs @@ -19,3 +19,6 @@ (rf/reg-sub ::height (fn [db _] (get-in db [:clip :height]))) (rf/reg-sub ::audio (fn [db _] (get-in db [:clip :audio]))) (rf/reg-sub ::footage (fn [db _] (:footage db))) +;; The document's identity on the server. Not derived and not large — an id, a +;; name, the project version and the last thing a save or an open said. +(rf/reg-sub ::project (fn [db _] (:project db))) diff --git a/frontend/src/arthur/ui/shell.cljs b/frontend/src/arthur/ui/shell.cljs index 924ce42..a38667a 100644 --- a/frontend/src/arthur/ui/shell.cljs +++ b/frontend/src/arthur/ui/shell.cljs @@ -9,6 +9,7 @@ [arthur.db :as db] [arthur.events.footage :as footage] [arthur.events.playback :as pb] + [arthur.events.project :as project] [arthur.subs.playback :as sub] [arthur.subs.render :as render] [arthur.ui.player :as player] @@ -38,7 +39,9 @@ picture-fps @(rf/subscribe [::sub/display-fps]) current @(rf/subscribe [::render/scene-id]) expose @(rf/subscribe [::render/exposure]) - {:keys [id label loading? status manifest-path]} @(rf/subscribe [::sub/footage])] + {:keys [id label loading? status available chosen]} @(rf/subscribe [::sub/footage]) + {project-name :name :keys [busy?] project-status :status + project-seq :seq} @(rf/subscribe [::sub/project])] [:div.transport [:div.row [:button {:on-click #(rf/dispatch [::pb/toggle])} @@ -61,10 +64,15 @@ [:button {:class (when (= id @(rf/subscribe [::render/scene-id])) "on") :on-click #(rf/dispatch [::pb/select-scene id])} (or label "footage")]) - [:button {:disabled loading? + [:button {:disabled (or loading? (nil? chosen)) :on-click #(rf/dispatch [::footage/load])} (if loading? "loading…" "load frames")] [:span.gap] + ;; The document, over HTTP. Two buttons, because the round trip is the proof + ;; the model serialises and a proof nobody can run is not one. + [:button {:disabled busy? :on-click #(rf/dispatch [::project/save])} "save"] + [:button {:disabled busy? :on-click #(rf/dispatch [::project/open])} "open"] + [:span.gap] (doall (for [r db/rates] ^{:key r} @@ -100,11 +108,26 @@ [:button {:class (when (= r picture-fps) "on") :on-click #(rf/dispatch [::pb/set-picture-fps r])} (if (= r fps) "source" (str r))]))]) - [:label.source-path "source manifest " - [:input {:type "text" :value manifest-path :disabled loading? - :on-change #(rf/dispatch [::footage/set-manifest-path - (.. % -target -value)])}]] - (when status [:div.load-status status])])) + ;; The takes the SERVER holds, since step 9. There is no path to type any + ;; more: `./extract.sh` decodes a clip and `manage.py ingest_bundle` registers + ;; it, and from then on the frames are addressed rather than located. + [:label.source-path "footage " + [:select {:value (or chosen "") :disabled loading? + :on-change #(rf/dispatch [::footage/choose (.. % -target -value)])} + (if (seq available) + (doall (for [{:keys [id label frames fps]} available] + ^{:key id} + [:option {:value id} + (str label " · " frames "f @" fps)])) + [:option {:value ""} "nothing ingested"])] + [:button {:disabled loading? + :on-click #(rf/dispatch [::footage/refresh])} "refresh"]] + (when status [:div.load-status status]) + (when (or project-name project-status) + [:div.load-status + (when project-name (str "project " project-name + (when project-seq (str " r" project-seq)) " · ")) + project-status])])) (defn- stage [] ;; The canvas is the STAGE's size, and the stage is the clip's — not a constant diff --git a/frontend/test/arthur/domain/canon_test.cljs b/frontend/test/arthur/domain/canon_test.cljs new file mode 100644 index 0000000..5217742 --- /dev/null +++ b/frontend/test/arthur/domain/canon_test.cljs @@ -0,0 +1,65 @@ +(ns arthur.domain.canon-test + "A content address is only an address if the same inputs always write the same + bytes, so these are the ways that could stop being true." + (:require [cljs.test :refer [deftest is testing]] + [arthur.domain.canon :as canon])) + +(deftest key-order-does-not-change-the-text + ;; The reason this namespace exists. Two maps that are `=` must hash alike, and + ;; CLJS map iteration order is not part of `=`. + (is (= (canon/write {:b 2 :a 1 :c 3}) + (canon/write {:c 3 :a 1 :b 2}) + (canon/write (into {} [[:c 3] [:b 2] [:a 1]])))) + (is (= "{\"a\":1,\"b\":2,\"c\":3}" (canon/write {:a 1 :b 2 :c 3})))) + +(deftest the-text-is-valid-json-because-the-server-reads-two-fields-out-of-it + (let [text (canon/write {:detector "mediapipe" :version "1.0.1" + :params {:anchor-avg 2 :contour-avg 1} + :tracks ["outer" "inner"] :scale 16384}) + back (js->clj (js/JSON.parse text))] + (is (= "mediapipe" (get back "detector"))) + (is (= "1.0.1" (get back "version"))) + (is (= 2 (get-in back ["params" "anchor-avg"]))))) + +(deftest an-integral-double-is-written-without-a-point + ;; JS and Python disagree here — `1` against `1.0` — which is exactly why the + ;; server hashes the text it was sent instead of re-rendering the values. + (is (= "1" (canon/write 1.0))) + (is (= "1" (canon/write 1))) + (is (= "0.12" (canon/write 0.12))) + (is (= "-0.5" (canon/write -0.5))) + ;; And a double that needs all its digits keeps them: shortest round-trip, not + ;; a fixed precision, or two different takes would share a key. + (is (= "0.5625" (canon/write 0.5625))) + (is (= (str (/ 1 3)) (canon/write (/ 1 3))))) + +(deftest a-namespaced-key-keeps-its-namespace + (is (= "{\"roto/lips-outer\":1}" (canon/write {:roto/lips-outer 1})))) + +(deftest strings-are-escaped-by-a-json-writer-and-not-by-hand + (is (= "{\"source\":\"a \\\"quoted\\\" clip.mov\"}" + (canon/write {:source "a \"quoted\" clip.mov"}))) + (is (= "{\"source\":\"café.mov\"}" (canon/write {:source "café.mov"})))) + +(deftest nil-and-the-booleans-are-json-and-not-omitted + ;; Omitting a nil would make {:presence nil} and {} the same address, and those + ;; are a take with no absence data and a take whose absence data was forgotten. + (is (= "{\"a\":null,\"b\":false,\"c\":true}" (canon/write {:a nil :b false :c true}))) + (is (not= (canon/write {:presence nil}) (canon/write {})))) + +(deftest a-keyword-value-is-refused-rather-than-named + ;; Because then :mouth and "mouth" would address the same block, and the field + ;; the server reads would have a type that depended on the caller. + (is (thrown-with-msg? ExceptionInfo #"keyword VALUE" (canon/write {:role :eyes}))) + (is (thrown-with-msg? ExceptionInfo #"keyword VALUE" (canon/write {:roles [:eyes]})))) + +(deftest a-set-is-refused-because-it-has-no-one-text + (is (thrown-with-msg? ExceptionInfo #"sort it into a vector" (canon/write {:kept #{1 2}})))) + +(deftest nan-is-refused-because-it-would-cache-a-measurement-that-went-wrong + (is (thrown-with-msg? ExceptionInfo #"NaN or infinity" (canon/write {:scale js/NaN}))) + (is (thrown-with-msg? ExceptionInfo #"NaN or infinity" (canon/write {:scale js/Infinity})))) + +(deftest nesting-is-ordered-all-the-way-down + (is (= (canon/write {:a {:z 1 :y [{:q 1 :p 2}]}}) + (canon/write {:a {:y [{:p 2 :q 1}] :z 1}})))) diff --git a/frontend/test/arthur/domain/leaf_test.cljs b/frontend/test/arthur/domain/leaf_test.cljs new file mode 100644 index 0000000..d26c0de --- /dev/null +++ b/frontend/test/arthur/domain/leaf_test.cljs @@ -0,0 +1,110 @@ +(ns arthur.domain.leaf-test + "Leaf addressing has one property that matters above every other: nothing is + lost. A persistence layer that drops a field saves a document which comes back + subtly smaller, and the loss is discovered later, by somebody whose work is + already gone. + + So the assertion is exact equality on the real scenes — the frozen take in both + head modes, the hand-written demo, the swarm — rather than on a fixture, and + `scene-keys` makes a field added without a leaf fail loudly instead." + (:require [cljs.test :refer [deftest is testing]] + [arthur.demo :as demo] + [arthur.demo.swarm :as swarm] + [arthur.demo.take :as take] + [arthur.domain.channel :as ch] + [arthur.domain.leaf :as leaf])) + +(deftest every-real-scene-survives-the-split-exactly + (doseq [[label scene] [["the frozen take" @take/scene] + ["the locked take" @take/locked] + ["the hand-written demo" demo/scene] + ["the swarm" @swarm/scene]]] + (testing label + (is (= scene (leaf/scene :c1 (leaf/leaves :c1 scene))))))) + +(deftest the-leaves-are-the-paths-the-sync-design-names + (let [ls (leaf/leaves :c7 @take/scene)] + (is (contains? ls "clip/c7/timing")) + (is (contains? ls "clip/c7/stage")) + (is (contains? ls "clip/c7/source")) + (is (contains? ls "clip/c7/node/mouth")) + (is (contains? ls "clip/c7/channel/mouth/geom.pts")) + (is (contains? ls "clip/c7/channel/mouth-in/vis")) + (is (contains? ls "clip/c7/feature/eye-r")) + (is (contains? ls "clip/c7/group/eyes-1")) + (is (contains? ls "clip/c7/subject/face-1")) + ;; `:head`'s measured channels are written together by a freeze and replaced + ;; together by a re-freeze, so they are one leaf and not three. + (is (contains? ls "clip/c7/measured/head")) + (is (= 3 (count (get ls "clip/c7/measured/head")))))) + +(deftest a-node-and-its-channels-are-different-leaves + ;; The boundary that lets two people key different parts without meeting. A node + ;; leaf carries structure and no geometry. + (let [ls (leaf/leaves :c1 @take/scene) + n (get ls "clip/c1/node/mouth")] + (is (= {:id :mouth :name "mouth" :kind :poly :parent :head :z "a1"} n)) + (is (nil? (:channels n))) + (is (:animated? (get ls "clip/c1/channel/mouth/geom.pts"))))) + +(deftest a-field-with-no-leaf-is-refused-rather-than-dropped + ;; The invariant that keeps the round trip exact as the model grows: a scene + ;; field nobody gave a leaf to would save silently and come back missing. + (is (thrown-with-msg? ExceptionInfo #"no leaf to save it in" + (leaf/leaves :c1 (assoc @take/scene :sequences [])))) + (is (= leaf/scene-keys (set (keys (assoc @take/scene :name "x")))) + "scene-keys has drifted from what a frozen scene actually holds")) + +(deftest an-absent-field-stays-absent + ;; A scene with no fps must not come back with `:fps nil`. `=` is the test, and + ;; the demo scene is the case: it has no analysis record and its root has no + ;; channels. + (let [ls (leaf/leaves :c1 demo/scene)] + (is (not (contains? ls "clip/c1/source"))) + (is (not (contains? (leaf/scene :c1 ls) :analysis))) + (is (not (contains? (get-in (leaf/scene :c1 ls) [:nodes :root]) :channels))))) + +(deftest a-namespaced-id-is-one-path-segment + ;; docs/architecture.md draws a node as `:eye-r/iris`, and a leaf path is + ;; "/"-delimited, so the two have to be reconciled somewhere. + (let [scene {:nodes {:eye-r/iris {:id :eye-r/iris :kind :disc :parent nil :z "a1" + :channels {[:geom :radius] (ch/framed 2)}}}} + ls (leaf/leaves :c1 scene)] + (is (contains? ls "clip/c1/node/eye-r~iris")) + (is (= scene (leaf/scene :c1 ls)))) + ;; `(keyword "a~b")` rather than a literal: ~ is unquote in CLJS source. + (is (thrown-with-msg? ExceptionInfo #"cannot contain ~" + (leaf/segment (keyword "a~b"))))) + +(deftest another-clips-leaves-are-ignored-rather-than-merged + ;; A project's whole leaf map can be handed in for one clip, which is what makes + ;; a two-clip project one fetch. + (let [a (leaf/leaves :a @take/scene) + b (leaf/leaves :b demo/scene)] + (is (= @take/scene (leaf/scene :a (merge a b)))) + (is (= demo/scene (leaf/scene :b (merge a b)))))) + +;; --------------------------------------------------------------------------- +;; what a document may not contain + +(deftest a-placeholder-tier-2-key-is-refused + ;; `demo/swarm` names its blocks "swarm/pos", which is exactly the descriptive + ;; key content addressing replaced: a handle that only means something on the + ;; machine that made it. It is a fine load test and not a document. + (let [ps (leaf/problems (leaf/leaves :c1 @swarm/scene))] + (is (seq ps)) + (is (some #(re-find #"names tier 2 as \"swarm/pos\"" %) ps) (pr-str (first ps))))) + +(deftest a-frozen-clip-has-no-problems + (is (empty? (leaf/problems (leaf/leaves :c1 @take/scene)))) + (is (empty? (leaf/problems (leaf/leaves :c1 @take/locked))))) + +(deftest a-channel-leaf-for-a-node-that-is-not-there-is-named + (let [ls (dissoc (leaf/leaves :c1 @take/scene) "clip/c1/node/mouth")] + (is (some #(re-find #"node with no node leaf" %) (leaf/problems ls))))) + +(deftest a-property-with-path-punctuation-in-it-is-refused + (is (thrown-with-msg? + ExceptionInfo #"cannot contain . or /" + (leaf/leaves :c1 {:nodes {:a {:id :a :kind :poly :parent nil :z "a1" + :channels {[:geom :pts.x] (ch/framed [0 0])}}}})))) diff --git a/frontend/test/arthur/domain/project_test.cljs b/frontend/test/arthur/domain/project_test.cljs new file mode 100644 index 0000000..0a6de19 --- /dev/null +++ b/frontend/test/arthur/domain/project_test.cljs @@ -0,0 +1,146 @@ +(ns arthur.domain.project-test + "The done criterion of port-plan step 9, made mechanical. + + \"Round-tripping a project through the server is the proof the model + serialises\" — and the proof has to be an assertion rather than a look, because + the ways a document survives a round trip LOOKING correct are the interesting + ones: a frame key that came back a string, an absence mask that came back all + zeroes, a dense block read as the wrong element type. Every one of those plays + back as a slightly wrong performance rather than as an error. + + So what is compared is the OPS, frame for frame, through both evaluators, in + every frame order — the same machinery scene-test uses to hold `eval-frame` and + `resolver` to each other, which is the strictest statement available about two + scenes being the same scene. + + This runs the conversion the network runs — `JSON.parse(JSON.stringify(...))` — + and not the network. `clips/tests.py` puts the same document through Django, and + the browser suite drives the real thing end to end; what is asserted here is the + half that does not need a server to be wrong." + (:require [cljs.test :refer [deftest is testing]] + [arthur.demo.take :as take] + [arthur.domain.channel :as ch] + [arthur.domain.project :as project] + [arthur.domain.scene :as scene] + [arthur.flow.freeze :as freeze] + [arthur.support.ops :as ops])) + +(defn- wired + "A clip out and back, over a wire that is really only JSON." + [cid clip] + (project/load cid (js/JSON.parse (js/JSON.stringify (project/save cid clip))))) + +(def ^:private before (delay @take/frozen)) +(def ^:private after (delay (wired :c1 @before))) + +(deftest what-comes-back-is-a-valid-scene + (let [ps (scene/problems (:scene @after))] + (is (empty? ps) (pr-str ps)))) + +(deftest the-document-comes-back-equal + ;; Stronger than it needs to be and worth having: not merely equivalent, EQUAL. + ;; Any drift here is a field the codec is rewriting, and a field that is + ;; rewritten once is rewritten again on every save. + (is (= (:scene @before) (:scene @after)))) + +(deftest every-frame-resolves-to-the-same-ops-before-and-after + ;; The assertion. Both evaluators, both scenes, every frame order — so a block + ;; that came back with its offsets shifted, or a cursor that seeks differently + ;; over a rebuilt key map, has nowhere to hide. + (let [n (:frames (:scene @before)) + paths {"specification" [ops/specified ops/specified] + "playback" [ops/resolved ops/resolved] + "spec vs playback, after" [ops/specified ops/resolved]}] + (doseq [[label [f g]] paths + [order fs] (ops/orders n)] + (let [a (f (:scene @before) (:store @before)) + b (g (:scene @after) (:store @after))] + (testing (str label ", " order) + (doseq [frame fs] + (is (= (a frame) (b frame)) + (str label " disagrees at frame " frame " going " order)))))))) + +(deftest the-blocks-come-back-byte-for-byte + ;; A handle that names a sha256 has to name the bytes you actually hold. + (is (= (set (keys (:store @before))) (set (keys (:store @after))))) + (doseq [[k entry] (:store @before)] + (let [back (get (:store @after) k)] + (is (= (.-constructor (:data entry)) (.-constructor (:data back))) + (str k " came back as a different element type")) + (is (= (vec (array-seq (:data entry))) (vec (array-seq (:data back)))) + (str k " came back with different numbers")) + (is (= (some? (:state entry)) (some? (:state back))) + (str k " gained or lost its state mask"))))) + +(deftest the-locked-take-round-trips-too + ;; The other head mode, because it is the one whose `:head` channels are FRAMED + ;; rather than dense: a codec that only handled dense channels would pass + ;; everything above and lose the locked take's identity transform. + (let [locked {:scene @take/locked :store @take/store} + back (wired :c1 locked) + a (ops/resolved (:scene locked) (:store locked)) + b (ops/resolved (:scene back) (:store back))] + (is (= (:scene locked) (:scene back))) + (doseq [frame (range 0 take/frames 7)] + (is (= (a frame) (b frame)) (str "frame " frame))))) + +;; --------------------------------------------------------------------------- +;; the state masks +;; +;; freeze-test's presence assertions, re-run on the other side of the wire. This +;; is the part most likely to survive looking correct: a mask lost in transit +;; shows up as a part that is drawn on a frame it was not observed on, which is a +;; pose invented out of its neighbours rather than a blank or an error. + +(def ^:private windows + {:eye-r (set (range 10 15)) + :eye-l (set (range 20 25)) + :brow-r (set (range 30 35)) + :brow-l (set (range 40 45))}) + +(def ^:private gappy + (delay (freeze/clip (assoc take/params :name "gappy") + (assoc @take/measured + :presence + (into {} (map (fn [[id gap]] + [id (mapv #(not (contains? gap %)) + (range take/frames))])) + windows))))) + +(deftest an-absence-mask-survives-the-wire + (let [back (wired :c1 @gappy) + at (fn [clip id path f] + (ch/value-at (get-in (:scene clip) [:nodes id :channels path]) + f (:store clip))) + ;; Every dense track of the eye, iris, brow and brow-position blocks, and + ;; the feature whose gap it must follow — the same table + ;; `each-dense-track-follows-its-own-features-presence` pins. + tracks [[:eye-r [:geom :pts] :eye-r] + [:eye-r-in [:geom :pts] :eye-r] + [:eye-l [:geom :pts] :eye-l] + [:eye-l-in [:geom :pts] :eye-l] + [:iris-r [:xform :pos] :eye-r] + [:iris-l [:xform :pos] :eye-l] + [:brow-r [:geom :pts] :brow-r] + [:brow-l [:geom :pts] :brow-l] + [:brow-r [:xform :pos] :brow-r] + [:brow-l [:xform :pos] :brow-l]]] + (doseq [[id path owner] tracks + [feature gap] windows + f gap] + (if (= feature owner) + (is (ch/nothing? (at back id path f)) + (str id " " path " is present at " f " after the round trip, with " + feature " occluded")) + (is (not (ch/nothing? (at back id path f))) + (str id " " path " follows " feature "'s gap at frame " f + " after the round trip")))))) + +(deftest an-occluded-feature-is-still-not-drawn-after-the-wire + ;; Presence is not visibility, on the far side too: the node is dropped from the + ;; frame rather than hidden, and its partner is not. + (let [back (wired :c1 @gappy) + drawn (into #{} (map :node) ((scene/resolver (:scene back) (:store back)) 12))] + (is (not (contains? drawn :eye-r))) + (is (contains? drawn :eye-l)) + (is (contains? drawn :mouth)))) diff --git a/frontend/test/arthur/domain/scene_test.cljs b/frontend/test/arthur/domain/scene_test.cljs index b783a42..95dba73 100644 --- a/frontend/test/arthur/domain/scene_test.cljs +++ b/frontend/test/arthur/domain/scene_test.cljs @@ -13,7 +13,8 @@ [arthur.domain.node :as node] [arthur.domain.palette :as pal] [arthur.domain.raster :as raster] - [arthur.domain.scene :as scene])) + [arthur.domain.scene :as scene] + [arthur.support.ops :as ops])) (defn- poly [id parent z pts color & [extra]] (merge {:id id :kind :poly :parent parent :z z @@ -27,9 +28,7 @@ (defn- ids-at [scene f] (mapv :node (scene/eval-frame scene f))) -(defn- pts-of [op] - (mapv (fn [i] [(aget (:pts op) (* 2 i)) (aget (:pts op) (inc (* 2 i)))]) - (range (:n op)))) +(def ^:private pts-of ops/points) ;; ---- structure ---- @@ -264,22 +263,16 @@ ;; z paths, holds a cursor per channel and reuses one point buffer per node, and ;; every one of those is a way to be subtly wrong on some frames and not others ;; — which presents as a bad take rather than as an error. - (let [s demo/scene - res (scene/resolver s) - n (:frames s) - snapshot (fn [ops] - (mapv (fn [op] - (cond-> (dissoc op :pts :i) - (:pts op) (assoc :points (pts-of op)))) - ops))] - (doseq [[label fs] [["forward" (range n)] - ["backward" (reverse (range n))] - ["random access" [0 71 5 5 40 6 70 1 23 24 25 24 23 0 47 48]] - ["every third" (range 0 n 3)]]] + ;; The frame orders and the snapshot live in `arthur.support.ops`, because the + ;; same comparison is what proves a scene survived the server — see + ;; flow/project-test. + (let [s demo/scene + spec (ops/specified s nil) + fast (ops/resolved s nil)] + (doseq [[label fs] (ops/orders (:frames s))] (testing label (doseq [f fs] - (is (= (snapshot (scene/eval-frame s f)) (snapshot (res f))) - (str label " at frame " f))))))) + (is (= (spec f) (fast f)) (str label " at frame " f))))))) (deftest the-resolver-reuses-one-buffer-per-node ;; At 30fps per-frame allocation is the only thing that will make this stutter, diff --git a/frontend/test/arthur/domain/sha256_test.cljs b/frontend/test/arthur/domain/sha256_test.cljs new file mode 100644 index 0000000..5b7bbe0 --- /dev/null +++ b/frontend/test/arthur/domain/sha256_test.cljs @@ -0,0 +1,51 @@ +(ns arthur.domain.sha256-test + "The digests below came out of Python's `hashlib`, which is the implementation + this one has to agree with: the server recomputes the key of every block and + every analysis it is handed and refuses a mismatch, so a disagreement between + the two languages is an upload that fails with nothing wrong. + + The lengths are chosen, not arbitrary. 55 and 56 bytes are either side of the + point where the length field no longer fits in the first block, and 63/64 and + 119/120 are the block boundaries themselves. A hand-written SHA-256 that is + wrong is almost always wrong exactly there, or wrong about sign — see the + namespace docstring — and a sign bug is invisible on short inputs." + (:require [cljs.test :refer [deftest is testing]] + [arthur.domain.sha256 :as sha])) + +(defn- a [n] (apply str (repeat n "a"))) + +(deftest the-digests-are-the-ones-hashlib-gives + (doseq [[input expect] + [["" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"] + ["abc" "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"] + ["abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"] + [(a 55) "9f4390f8d30c2dd92ec9f095b65e2b9ae9b0a925a5258e241c9f1e910f734318"] + [(a 56) "b35439a4ac6f0948b6d6f9e3c6af0f5f590ce20f1bde7090ef7970686ec6738a"] + [(a 63) "7d3e74a05d7db15bce4ad9ec0658ea98e3f06eeecf16b4c6fff2da457ddc2f34"] + [(a 64) "ffe054fe7ae0cb6dc65c3af9b61d5209f439851db43d0ba5997337df154668eb"] + [(a 119) "31eba51c313a5c08226adf18d4a359cfdfd8d2e816b13f4af952f7ea6584dcfb"] + [(a 120) "2f3d335432c70b580af0e8e1b3674a7c020d683aa5f73aaaedfdc55af904c21c"] + ["arthur" "befa156f0283eb0062beb9b86e16a413e1cf8c5135e5518d5c4fa321ce0c7b6b"]]] + (is (= expect (sha/of-string input)) + (str (count input) " bytes")))) + +(deftest a-non-ascii-descriptor-hashes-as-utf-8 + ;; A descriptor holds source filenames, so this is reachable from a clip called + ;; "café.mov" and not a curiosity. UTF-16 code units would give another answer. + (is (= "3392aa2b9d70af3b1de0c4d4f7bc6fcd02e1df8079263bb7cda7b1b71704a90a" + (sha/of-string "café — naïve ✓")))) + +(deftest raw-bytes-hash-too-because-a-block-is-bytes + (is (= "40aff2e9d2d8922e47afd4648e6967497158785fbd1da870e7110266bf944880" + (sha/of-bytes (js/Uint8Array. (into-array (range 256))))))) + +(deftest a-key-says-which-algorithm-produced-it + (is (= (str "sha256:" (sha/of-string "abc")) (sha/key-of "abc"))) + ;; The point of the prefix: a key cannot be mistaken for the descriptive + ;; placeholder — "take/geom" — that content addressing replaced. + (is (re-matches #"sha256:[0-9a-f]{64}" (sha/key-of "abc")))) + +(deftest one-changed-bit-changes-the-key + ;; The whole property everything above this file relies on, asserted once. + (is (not= (sha/of-string "anchor-avg:2") (sha/of-string "anchor-avg:3")))) diff --git a/frontend/test/arthur/domain/wire_test.cljs b/frontend/test/arthur/domain/wire_test.cljs new file mode 100644 index 0000000..f3592e7 --- /dev/null +++ b/frontend/test/arthur/domain/wire_test.cljs @@ -0,0 +1,91 @@ +(ns arthur.domain.wire-test + "What the wire format has to carry, stated as the things JSON would have lost." + (:require [cljs.test :refer [deftest is testing]] + [arthur.demo.take :as take] + [arthur.domain.channel :as ch] + [arthur.domain.leaf :as leaf] + [arthur.domain.wire :as wire])) + +(defn- round [v] (wire/decode (wire/encode v))) +(defn- round-json [v] (wire/decode-json (wire/encode-json v))) + +(deftest a-frame-key-comes-back-a-number + ;; THE reason this is transit. Keys are a map by FRAME, and `{"0" v}` is not + ;; `{0 v}`: `value-at` would find no key at frame 0 and the part would hold its + ;; first pose forever, on a document that looked fine. + (let [c (ch/keyed {0 true 4 false 12 true})] + (is (= c (round c))) + (is (every? number? (keys (:keys (round c))))) + (is (= true (ch/value-at (round c) 13))))) + +(deftest an-id-comes-back-a-keyword + (is (= {:id :mouth-in :kind :poly :parent :mouth :z "a2"} + (round {:id :mouth-in :kind :poly :parent :mouth :z "a2"}))) + (is (keyword? (:id (round {:id :mouth}))))) + +(deftest a-channels-property-vector-survives-as-a-map-key + ;; `:channels` is keyed by `[:geom :pts]`, and a format with string keys only + ;; would have to invent an encoding for that — which is what leaf paths do for + ;; addressing and what the wire format must NOT have to do for values. + (let [m {[:geom :pts] (ch/framed [0 1]) [:vis] (ch/framed true)}] + (is (= m (round m))) + (is (vector? (first (keys (round m))))))) + +(deftest a-keyed-channel-comes-back-a-plain-map-and-not-a-sorted-one + ;; Transit loses sortedness, which is why `domain/channel` says keys are a PLAIN + ;; map and builds the sorted index at read time. Asserted so that nobody + ;; "improves" the codec into a sorted map that works until the first round trip. + (let [c (round (ch/keyed (into {} (map (juxt identity str)) (range 20))))] + (is (map? (:keys c))) + (is (not (sorted? (:keys c)))) + (is (= (vec (range 20)) (ch/frames c))))) + +(deftest the-numbers-come-back-as-themselves + (is (= {:a 0.5625 :b -1 :c 1e-9 :d 0 :e 16384} + (round {:a 0.5625 :b -1 :c 1e-9 :d 0 :e 16384})))) + +(deftest an-empty-vector-stays-an-empty-vector + ;; `:over` is present and empty by design — port-plan step 2 scope — and + ;; `channel/check-unimplemented!` throws on a NON-empty one, so a codec that + ;; turned `[]` into nil or into `[nil]` would either lose the field or refuse to + ;; play the document back. + (is (= {:over []} (round {:over []}))) + (is (= [] (:over (round (ch/keyed {0 1})))))) + +(deftest a-whole-leaf-map-round-trips-through-parsed-json + ;; What a save actually does: transit, then parsed so the column holds JSON. + (let [ls (leaf/leaves :c1 @take/scene)] + (is (= ls (into {} (map (fn [[p v]] [p (round-json v)])) ls))))) + +;; --------------------------------------------------------------------------- +;; the bytes + +(deftest a-block-comes-back-byte-for-byte + (doseq [[label array] [["int16" (js/Int16Array. #js [0 1 -1 32767 -32768 12345])] + ["float32" (js/Float32Array. #js [0 1.5 -0.25 1e-8])] + ["uint8" (js/Uint8Array. #js [0 1 255])]]] + (testing label + (let [back (wire/typed label (wire/base64 array))] + (is (= (vec (array-seq array)) (vec (array-seq back)))) + (is (= (.-constructor array) (.-constructor back))))))) + +(deftest a-real-sized-block-does-not-overflow-the-stack + ;; `String.fromCharCode.apply` with a few hundred thousand arguments throws a + ;; RangeError from inside a save, pointing nowhere near the array that caused it. + ;; A 600-frame geometry block is that size, so the chunking is load-bearing. + (let [big (js/Int16Array. 600000)] + (dotimes [i 600000] (aset big i (- (mod i 65536) 32768))) + (let [back (wire/typed "int16" (wire/base64 big))] + (is (= 600000 (.-length back))) + (is (= (aget big 599999) (aget back 599999)))))) + +(deftest a-view-into-a-block-encodes-only-its-own-bytes + ;; `dense-at` hands out SUBARRAYS, so a caller can reach this with a view whose + ;; byteOffset is not zero. Encoding the whole underlying buffer would silently + ;; store the neighbouring tracks too. + (let [whole (js/Int16Array. #js [1 2 3 4 5 6]) + view (.subarray whole 2 4)] + (is (= [3 4] (vec (array-seq (wire/typed "int16" (wire/base64 view)))))))) + +(deftest an-unknown-element-type-is-refused + (is (thrown-with-msg? ExceptionInfo #"int16" (wire/typed "float64" "AA==")))) diff --git a/frontend/test/arthur/flow/address_test.cljs b/frontend/test/arthur/flow/address_test.cljs new file mode 100644 index 0000000..0e4327b --- /dev/null +++ b/frontend/test/arthur/flow/address_test.cljs @@ -0,0 +1,217 @@ +(ns arthur.flow.address-test + "Content addressing has one failure mode in each direction, and reading the + table in `flow/address` will catch neither. + + A KNOB MISSING from a block's table gives two different sets of bytes the same + name. The symptom is not an error: it is a slider that appears to do nothing + until something else forces a reload, and then does everything at once. That is + the bug docs/architecture.md describes as \"the tool got worse\" with no event to + attach it to. + + A KNOB TOO MANY throws away a good bake on an unrelated tweak. Invisible while + everything is fast, and the reason baking exists once it is not. + + So the table is not trusted. `every-knob-that-moves-a-block-renames-it` freezes + the same synthetic take once per knob and asserts the biconditional per block: + the bytes changed if and only if the key changed. It is the same shape of + assertion as `each-dense-track-follows-its-own-features-presence` in + freeze-test, for the same reason — a hand-maintained mapping needs a test that + fails when the hand is wrong." + (:require [cljs.test :refer [deftest is testing]] + [arthur.domain.params :as params] + [arthur.domain.sha256 :as sha] + [arthur.flow.address :as address] + [arthur.flow.freeze :as freeze] + [arthur.flow.take :as take] + [arthur.synth :as synth])) + +;; Short on purpose: the property is about which inputs reach which block, and it +;; holds at any length. Twenty-four freezes of the 229-frame take would be a +;; minute of test time to assert nothing extra. +(def ^:private frames 48) +(def ^:private fps 30) + +(def ^:private dense-track (delay (synth/synth-dense frames {:seed 3}))) + +(defn- params-at [overrides] + (merge take/knobs + {:name "addr" :fps fps :aspect 1 :stage [320 200] + :expose 1 :head :as-filmed} + overrides)) + +(defn- freeze-at + "Measure AND freeze at these settings, which is what a re-freeze does: several + of the knobs act in stage 4, so a fixture that only re-froze would hold most of + them still and pass whatever the table said." + [overrides] + (let [p (params-at overrides) + p (assoc p :analysis (address/analysis + (merge {:detector "synth" :version "mulberry32" + :seed 3 :frames frames :fps (:fps p) + :aspect (:aspect p)} + (:detector overrides))))] + (freeze/clip p (take/measure p {:dense @dense-track})))) + +(defn- bytes-of [{:keys [data state]}] + (str (sha/of-bytes (js/Uint8Array. (.-buffer data))) + "/" (if state (sha/of-bytes state) "-"))) + +(defn- by-role + "role -> {:key :bytes}. The role comes out of the block's own descriptor, which + is how a block is identified across two freezes that renamed it." + [clip] + (into {} + (map (fn [[k entry]] + [(get (js->clj (js/JSON.parse (:descriptor entry))) "role") + {:key k :bytes (bytes-of entry)}])) + (:store clip))) + +(def ^:private base (delay (by-role (freeze-at {})))) + +;; The knobs whose effect is upstream of `flow/take/measure`: they are read by +;; `measure/interior`, which runs over SOURCE PIXELS in events/footage before any +;; of this. A synthetic fixture has no pixels to move, so the biconditional cannot +;; be asserted for them here and `the-teeth-table-is-asserted-at-the-descriptor` +;; asserts the half that is assertable instead. +(def ^:private pixel-knobs + #{:cavity-erode :tongue-reject :blob-grow :top-bias :teeth-verts :min-area + :teeth-on :teeth-smooth}) + +(defn- bump + "A different, still valid value for a knob — and a LARGE difference, which is + not laziness about picking one. + + Several of these knobs feed `condition/quantize-snap`, which rounds onto a grid: + gaze and brow cells, the blink hold. A 3% change to `gaze-gain` moves every + sample inside the cell it was already in, so the bytes come out identical and + the biconditional reports the knob as an input the block does not have — which + is true of that perturbation and false of the knob. A 75% change crosses cells. + The first version of this test asserted the fixture's resolution rather than the + invalidation table." + [id] + (let [{:keys [type default] must-even? :even?} (get params/definitions id)] + (cond + (and (= :integer type) must-even?) (+ default 2) + (= :integer type) (+ default 1) + (zero? default) 0.5 + :else (* default 1.75)))) + +(deftest every-knob-that-moves-a-block-renames-it + (doseq [id (sort (remove pixel-knobs (keys params/definitions)))] + (let [moved (by-role (freeze-at {id (bump id)}))] + (testing (str id " " (get params/defaults id) " -> " (bump id)) + (is (= (set (keys @base)) (set (keys moved))) + "a knob changed which blocks exist at all") + (doseq [role (sort (keys @base))] + (let [a (get @base role) + b (get moved role)] + (is (= (not= (:bytes a) (:bytes b)) + (not= (:key a) (:key b))) + (str role ": bytes " (if (= (:bytes a) (:bytes b)) "same" "differ") + " but key " (if (= (:key a) (:key b)) "same" "differs") + " — " (if (= (:key a) (:key b)) + (str "add " id " to block-knobs for " (pr-str role)) + (str "remove " id " from block-knobs for " (pr-str role))))))))))) + +(deftest the-teeth-table-is-asserted-at-the-descriptor + ;; Every knob the teeth block declares must reach its key. The other half — that + ;; each one also moves its bytes — needs real pixels, because the crop, the otsu + ;; threshold and the radial contour all happen in `measure/interior` above the + ;; stage this fixture starts at. Stated rather than quietly skipped. + (let [spec {:role "teeth" :analysis "sha256:0" :params params/defaults + :tracks ["contour"] :features [:teeth] + :observation nil + :layout {:type "int16" :scale 16384 :stride 20 :frames 48 :tracks 1}} + base (:key (address/block spec))] + (doseq [id (get address/block-knobs "teeth")] + (is (not= base (:key (address/block (assoc-in spec [:params id] (bump id))))) + (str id " does not reach the teeth block's key"))))) + +(deftest a-block-whose-role-has-no-table-is-refused + ;; The table cannot be forgotten for a new block: without an entry there is no + ;; answer to "which knobs rename this", and a key that never changes is worse + ;; than no key at all. + (is (thrown-with-msg? + ExceptionInfo #"no invalidation table" + (address/block {:role "plate" :analysis "sha256:0" :params {} :tracks [] + :features [] :observation nil :layout {}})))) + +(deftest a-knob-the-freeze-was-not-handed-is-refused + (is (thrown-with-msg? + ExceptionInfo #"knob this block's bytes depend on" + (address/block {:role "geom" :analysis "sha256:0" + :params {:anchor-avg 2} :tracks ["outer"] + :features [:mouth] :observation nil :layout {}})))) + +;; --------------------------------------------------------------------------- +;; the detector version + +(deftest the-detector-version-reaches-every-block + ;; THE requirement, and the reason a key is a hash over inputs rather than over + ;; bytes: after a model upgrade the old blocks must be UNREACHABLE, not merely + ;; different. Every block, because a version in one key and not another is the + ;; same bug with a smaller blast radius. + (let [upgraded (by-role (freeze-at {:detector {:version "1.0.2"}}))] + (is (= (set (keys @base)) (set (keys upgraded)))) + (doseq [role (sort (keys @base))] + (is (not= (:key (get @base role)) (:key (get upgraded role))) + (str role " survived a detector upgrade under its old name")) + ;; And nothing was recomputed: same numbers, new name. That is what makes + ;; this a cache-key change and not a re-analysis. + (is (= (:bytes (get @base role)) (:bytes (get upgraded role))))))) + +(deftest an-analysis-without-a-version-is-refused + (is (thrown-with-msg? ExceptionInfo #"detector's VERSION" + (address/analysis {:detector "mediapipe" :frames 1}))) + (is (thrown-with-msg? ExceptionInfo #"detector's VERSION" + (address/analysis {:version "1.0.1" :frames 1})))) + +(deftest the-analysis-id-is-a-hash-of-its-own-descriptor + (let [a (address/analysis {:detector "synth" :version "mulberry32" :seed 1 + :frames 229 :fps 30 :aspect 1})] + (is (= (:id a) (sha/key-of (address/analysis-descriptor a)))) + (is (sha/key? (:id a))))) + +;; --------------------------------------------------------------------------- +;; the rest of what a key is made of + +(deftest the-same-inputs-give-the-same-keys + ;; Twice, independently — not the same clip read twice. Map order, `delay`s and + ;; `Math.round` are all places a second run could differ, and a key that is not + ;; reproducible addresses nothing. + (is (= (into {} (map (juxt key (comp :key val))) (by-role (freeze-at {}))) + (into {} (map (juxt key (comp :key val))) (by-role (freeze-at {})))))) + +(deftest a-key-is-the-hash-of-the-descriptor-stored-beside-it + ;; What the server checks on upload, checked here too, because if it can only + ;; fail on the wire it fails as a 409 with nothing wrong. + (doseq [[k entry] (:store (freeze-at {}))] + (is (sha/key? k)) + (is (= k (sha/key-of (:descriptor entry))) + (str "the descriptor stored under " k " is not what that key hashes")))) + +(deftest a-presence-gap-renames-only-the-blocks-that-read-it + ;; The observation digest is per block. A brow occlusion must not rename the + ;; mouth's block: that would be correct and useless, throwing away every bake in + ;; the clip on one feature's gap. + (let [gap (set (range 10 15)) + with (by-role (freeze/clip + (assoc (params-at {}) :analysis + (address/analysis {:detector "synth" :version "mulberry32" + :seed 3 :frames frames :fps fps :aspect 1})) + (assoc (take/measure (params-at {}) {:dense @dense-track}) + :presence {:brow-r (mapv #(not (contains? gap %)) + (range frames))})))] + (is (not= (:key (get @base "brows")) (:key (get with "brows")))) + (is (not= (:key (get @base "brow-pos")) (:key (get with "brow-pos")))) + (doseq [role ["geom" "eyes" "iris-pos" "head-pos" "head-rot" "head-scale"]] + (is (= (:key (get @base role)) (:key (get with role))) + (str role " was renamed by a gap in a brow"))))) + +(deftest the-clips-name-and-stage-are-not-in-any-key + ;; Tier 1 facts. Two clips of one take under two names, at two stage sizes, are + ;; the same analysis over the same bytes, and sharing them is the return on + ;; addressing. A name in the key would make every rename a re-freeze. + (let [other (by-role (freeze-at {:name "another take" :stage [640 480]}))] + (doseq [role (sort (keys @base))] + (is (= (:key (get @base role)) (:key (get other role))) role)))) diff --git a/frontend/test/arthur/flow/freeze_test.cljs b/frontend/test/arthur/flow/freeze_test.cljs index 7ab1ab2..1db91e0 100644 --- a/frontend/test/arthur/flow/freeze_test.cljs +++ b/frontend/test/arthur/flow/freeze_test.cljs @@ -33,6 +33,11 @@ (defn- node [id] (get-in @scene* [:nodes id])) (defn- chan [id path] (get-in (node id) [:channels path])) +(defn- block-of + "The typed array a dense channel reads, through its own handle." + [ch] + (:data (get @store (:store (:dense ch))))) + (defn- pts-at "The mouth's [:geom :pts] at frame f, as a flat CLJS vector." [id f] @@ -127,8 +132,11 @@ (doseq [path [[:xform :pos] [:xform :rot] [:xform :scale]]] (is (nil? (:scale (:dense (get-in (node :head) [:measured path])))) (str path " should be plain Float32"))) - (is (instance? js/Int16Array (:data (get @store "take/geom")))) - (is (instance? js/Float32Array (:data (get @store "take/head-pos"))))) + ;; Reached through the channel's own handle rather than by naming a key. A key + ;; is a hash now, so a test that wrote one out would be asserting a digest. + (is (instance? js/Int16Array (block-of (chan :mouth [:geom :pts])))) + (is (instance? js/Float32Array + (block-of (get-in (node :head) [:measured [:xform :pos]]))))) (deftest a-value-past-the-block-s-range-is-refused-rather-than-saturated ;; Saturating reads as articulation flattening off at the extremes — a bad @@ -305,10 +313,17 @@ ;; asking for a different stage moves and rescales the same geometry rather than ;; re-measuring anything. (let [big (freeze/clip (assoc take/params :stage [640 480] :name "big") - @take/measured)] + @take/measured) + key-of (fn [sc] (:store (:dense (get-in sc [:nodes :mouth :channels [:geom :pts]]))))] (is (= [640 480] [(:width (:scene big)) (:height (:scene big))])) - (is (= (vec (array-seq (:data (get @store "take/geom")))) - (vec (array-seq (:data (get (:store big) "big/geom"))))) + ;; Stronger than it was, and for free: the stage is not an input to tier 2, so + ;; the two clips do not merely hold equal bytes — they name the SAME BLOCK, and + ;; a stage change cannot invalidate a bake. The clip's name is not an input + ;; either, which is why "big" and "take" still agree. + (is (= (key-of @scene*) (key-of (:scene big))) + "a different stage is a different document over the same tier 2") + (is (= (vec (array-seq (:data (get @store (key-of @scene*))))) + (vec (array-seq (:data (get (:store big) (key-of (:scene big))))))) "the geometry is the same numbers at either stage size") (is (not= (:value (get-in (:scene big) [:nodes :face :channels [:xform :scale]])) (:value (chan :face [:xform :scale])))))) diff --git a/frontend/test/arthur/support/ops.cljs b/frontend/test/arthur/support/ops.cljs new file mode 100644 index 0000000..6f5cf06 --- /dev/null +++ b/frontend/test/arthur/support/ops.cljs @@ -0,0 +1,59 @@ +(ns arthur.support.ops + "Comparing two evaluations of a scene, frame for frame. + + `domain/scene` has two evaluators on purpose — `eval-frame` is the + specification and `resolver` is what playback uses — and scene-test's central + assertion is that they agree in forward, backward and random frame order. + Step 9 needs the same comparison for a different question: that a scene which + has been through the server produces the same ops as the one that went in. + + Shared rather than copied, because the interesting part is not the equality — + it is the FRAME ORDERS. The resolver holds a cursor per channel and reuses one + point buffer per node, so it can agree on a forward pass and disagree on a + scrub, and a copy of this list that forgot 'backward' would test the easy half. + + `snapshot` is what makes ops comparable at all: a resolved op carries `:pts` as + a VIEW into a reused buffer, so two ops from different frames can be `=` while + naming the same array, and holding one and then asking for the next frame + changes what the first one says. Reading the points out is what pins the frame." + (:require [arthur.domain.palette :as pal] + [arthur.domain.scene :as scene])) + +(defn points + "An op's points as a vector of [x y], read out of its buffer." + [op] + (mapv (fn [i] [(aget (:pts op) (* 2 i)) (aget (:pts op) (inc (* 2 i)))]) + (range (:n op)))) + +(defn snapshot + "Ops -> comparable data. `:i` goes too: it is the draw-order index, and it is a + function of the scene rather than of the frame." + [ops] + (mapv (fn [op] + (cond-> (dissoc op :pts :i) + (:pts op) (assoc :points (points op)))) + ops)) + +(defn orders + "The frame orders any two evaluators have to agree in, over `n` frames. + + Random access is a fixed list rather than a shuffle: a failure that only + reproduces one run in five is worse than no test." + [n] + [["forward" (range n)] + ["backward" (reverse (range n))] + ["random access" (filterv #(< % n) [0 71 5 5 40 6 70 1 23 24 25 24 23 0 47 48])] + ["every third" (range 0 n 3)]]) + +(defn specified + "(fn [f] -> snapshot) through `eval-frame`, the specification." + ([scene store] (specified scene store pal/index-of)) + ([scene store palette] + (fn [f] (snapshot (scene/eval-frame scene f store palette))))) + +(defn resolved + "(fn [f] -> snapshot) through `resolver`, the playback path." + ([scene store] (resolved scene store pal/index-of)) + ([scene store palette] + (let [res (scene/resolver scene store palette)] + (fn [f] (snapshot (res f)))))) diff --git a/frontend/test/browser/take.mjs b/frontend/test/browser/take.mjs index a595ffb..6337ff1 100644 --- a/frontend/test/browser/take.mjs +++ b/frontend/test/browser/take.mjs @@ -1,5 +1,7 @@ -// Drives a real Chrome at the running dev server and checks that the frozen -// take is a MOVING MOUTH on a canvas. +// Drives a real Chrome at the running server and checks two things that no +// assertion in cljs.test can: that the frozen take is a MOVING MOUTH on a canvas, +// and that a document which has been through the server comes back as the same +// picture. // // This exists because port-plan step 5 is the first step whose done-criterion is // a picture, and a picture cannot be asserted from cljs.test. A take that @@ -11,10 +13,13 @@ // nothing: `node --experimental-websocket` has a global WebSocket and // `--headless=new --remote-debugging-port=N` is the whole of the other side. // -// cd frontend -// mise exec -- npx shadow-cljs compile app -// mise exec -- npx shadow-cljs watch app # or `server`, for :dev-http -// mise exec -- node --experimental-websocket test/browser/take.mjs +// Since step 9 the page is Django's, so the suite needs the backend up rather than +// shadow-cljs's `:dev-http`, which is gone. ARTHUR_URL is unchanged because the +// port is unchanged — 8778 was never 8777, which is still the old JS tool's. +// +// mise exec -- python manage.py runserver 8778 # from the REPO ROOT +// cd frontend && mise exec -- npx shadow-cljs watch app +// cd frontend && mise exec -- node --experimental-websocket test/browser/take.mjs // // Writes a PNG per sampled frame into test/browser/out/ so that "it drew // something" can be checked by eye as well as by pixel count. @@ -198,6 +203,11 @@ const SEEK = (f) => `(() => { return el.value; })()`; +// Everything the page has to say about loading, saving and opening. Read off the +// page rather than out of app-db, for the same reason the playhead is: what the +// page SHOWS is what a person would check. +const STATUS = `[...document.querySelectorAll('.load-status')].map((d) => d.textContent).join(' | ')`; + const CLICK = (label) => `(() => { const b = [...document.querySelectorAll('.transport button')] .find((b) => b.textContent.trim() === ${JSON.stringify(label)}); @@ -312,6 +322,71 @@ async function main() { `${new Set(during.map((p) => p.hash)).size} distinct of ${during.length}`); await page.shot('take-playing'); + // --- it round-trips through the server --- + // + // THE DONE CRITERION of port-plan step 9, end to end: tier 1 over HTTP, tier 2 + // as content-addressed blocks, and the same frames on the far side. The ops are + // compared frame for frame in arthur.domain.project-test, which is the strict + // version of this; what only a browser can check is that the whole path — the + // CSRF header, the block upload, the leaf write, the reload, the typed arrays + // rebuilt out of base64 — draws the same pixels at the end of it. + async function statusMatching(pattern, tries = 120) { + for (let i = 0; i < tries; i++) { + const text = await page.eval(STATUS); + if (pattern.test(text)) return text; + await sleep(250); + } + return null; + } + async function sample(frames) { + const out = []; + for (const f of frames) { + await page.eval(SEEK(f)); + await sleep(120); + out.push(await page.eval(PROBE)); + } + return out; + } + + const FRAMES = [0, 10, 28, 80, 160]; + check(await page.eval(CLICK('take')), 'back to the take, for the round trip'); + await sleep(150); + const sent = await sample(FRAMES); + + check(await page.eval(CLICK('save')), 'save is clickable'); + const saved = await statusMatching(/saved r\d+/); + check(saved !== null, 'the document saves', saved ?? (await page.eval(STATUS))); + // Eleven blocks the first time. The COUNT is not asserted — that is a fact + // about the freeze, not about saving — but that some went up is. + check(/· [1-9]\d* blocks?/.test(saved ?? ''), 'and its tier 2 went with it', saved ?? ''); + + // Again, unchanged. Content addressing means the second save uploads nothing + // and rewrites nothing: this is the assertion that the keys are stable across + // two independent freezes of the same take, and that an unchanged leaf keeps + // its version rather than being rewritten. + check(await page.eval(CLICK('save')), 'save is clickable again'); + const resaved = await statusMatching(/saved r\d+ · 0 leaves · 0 blocks/); + check(resaved !== null, 'saving an unchanged document writes nothing', + resaved ?? (await page.eval(STATUS))); + + check(await page.eval(CLICK('open')), 'open is clickable'); + const opened = await statusMatching(/opened /); + check(opened !== null, 'the project opens', opened ?? (await page.eval(STATUS))); + + const back = await sample(FRAMES); + await page.shot('take-round-trip'); + check(back.every((p) => p.drawn > 200), 'the reopened take draws', + back.map((p) => p.drawn).join(',')); + check(FRAMES.every((f, i) => sent[i].hash === back[i].hash), + 'every sampled frame is the same picture after the round trip', + FRAMES.filter((f, i) => sent[i].hash !== back[i].hash).join(',') || 'all identical'); + check(back[1].toneSet.includes(MOUTH_DARK), + 'and the open mouth still has an interior on the far side'); + // The reopened clip is not one of the built-ins: this is the document that came + // back from the server, not the one that was in the page all along. + check(!back[0].scene.includes('take'), 'the picture is the reopened document', + JSON.stringify(back[0].scene)); + check(page.logs.length === 0, 'no errors on the console', page.logs.slice(0, 3).join(' | ')); } finally { diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..7d4bba0 --- /dev/null +++ b/manage.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +"""Django's command line, at the repo root. + +At the root rather than in a subdirectory so that every `python manage.py` works +with no `cd`, and so that the two halves of the repo — this and `frontend/` — are +peers. `mise install` from here sets up both. +""" +import os +import sys + + +def main(): + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Django is not importable. `mise install` from the repo root creates " + "the venv; then `pip install -r requirements.txt`." + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/manifest.json b/manifest.json deleted file mode 100644 index b67b3e5..0000000 --- a/manifest.json +++ /dev/null @@ -1 +0,0 @@ -{"fps":24,"frames":105,"dir":"frames","audio":"audio.wav","source":"ScreenRecording_09-24-2026 16-26-57_1.mov"} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5bc4225 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +# The backend's whole dependency list, and it is meant to stay this short. +# +# No DRF: the API is nine endpoints over JSON, and a document whose values are +# transit is one Django ORM JSONField and a `json.loads` — a serializer layer +# would be a second description of a shape that already has one in +# arthur.domain.leaf. No Pillow either; the one thing the backend needs from a PNG +# is its dimensions, which is a 24-byte header read in clips/blobs.py. +# +# Channels arrives with the websocket consumers, which are out of step 9's scope. +Django>=5.0,<6.0 diff --git a/server/__init__.py b/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/asgi.py b/server/asgi.py new file mode 100644 index 0000000..3f4b49e --- /dev/null +++ b/server/asgi.py @@ -0,0 +1,12 @@ +"""ASGI entry point. + +ASGI and not only WSGI because the collaboration design in docs/architecture.md +puts presence and document deltas on a websocket. Those consumers are out of step +9's scope; this is the half of their setup that costs nothing now. +""" +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server.settings") +application = get_asgi_application() diff --git a/server/settings.py b/server/settings.py new file mode 100644 index 0000000..5cf59ac --- /dev/null +++ b/server/settings.py @@ -0,0 +1,122 @@ +"""Settings for arthur's backend. + +It serves three things and they are three different kinds of thing, which is most +of what there is to know about this file: + + THE PAGE. One template, which replaced `frontend/public/index.html` at step 9. + It pulls the shadow-cljs bundle out of staticfiles, so `manage.py runserver` and + `shadow-cljs watch app` are the whole dev loop with nothing copying files + between them. + + THE DOCUMENT (tier 1). Small, authored, in the database. + + THE BLOBS (tiers 2 and 3). Large, immutable, content-addressed, on disk under + `var/blobs`. Never in the database, and never in a migration. +""" +from pathlib import Path + +BASE_DIR = Path(__file__).resolve().parent.parent + +# Dev default. The deployment that needs a real one will set it, and until there +# is a deployment, a checked-in placeholder that says so beats a checked-in secret +# that does not. +SECRET_KEY = "dev-only-not-a-secret-arthur" +DEBUG = True +ALLOWED_HOSTS = ["localhost", "127.0.0.1", "[::1]"] + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "clips", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", +] + +ROOT_URLCONF = "server.urls" +WSGI_APPLICATION = "server.wsgi.application" +ASGI_APPLICATION = "server.asgi.application" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +# WAL and a real busy timeout, because a save is a BURST of writes: one analysis, +# then a block per dense channel, then the leaves. Under the default rollback +# journal and a 5-second timeout, the block uploads of one save fail with +# "database is locked" — which surfaces in the page as a 500 with nothing wrong, +# and in the browser suite as a document that will not save. WAL lets readers and +# one writer proceed at once, and the timeout makes the writers queue instead of +# giving up. The client serialises its uploads as well (see events/project), so +# this is the belt to that braces. +# +# The ceiling this has is the one docs/architecture.md already names for tl's +# process-local ROOMS: one writer. It is a single-worker arrangement, and moving +# off it is a Postgres URL rather than a change to anything above this line. +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", + "OPTIONS": { + "timeout": 20, + "init_command": "PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;", + }, + } +} + +AUTH_PASSWORD_VALIDATORS = [] +LANGUAGE_CODE = "en-gb" +TIME_ZONE = "UTC" +USE_I18N = True +USE_TZ = True + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +# --- static ----------------------------------------------------------------- +# +# Two roots, and the second is the interesting one. `static/` is where +# shadow-cljs writes the bundle (`:output-dir "../static/arthur/js"`), so the +# build lands straight in the staticfiles tree. The vendored MediaPipe is served +# under the prefix `mediapipe/` from where it already lives in `frontend/public/` +# — 26MB of wasm and model that has no business being copied into a second place +# in the tree, and that `collectstatic` picks up from there. +STATIC_URL = "/static/" +STATICFILES_DIRS = [ + BASE_DIR / "static", + ("mediapipe", BASE_DIR / "frontend" / "public" / "mediapipe"), +] +STATIC_ROOT = BASE_DIR / "var" / "static" + +# --- blobs ------------------------------------------------------------------ +# +# Tiers 2 and 3 live here, named by the sha256 of their own bytes, fanned out two +# levels so no directory holds a hundred thousand entries. Deliberately NOT under +# `static/`: a blob is served by a view that can set immutable cache headers and +# refuse a path that is not a hash, and `collectstatic` has no business walking +# gigabytes of frames. +BLOB_ROOT = BASE_DIR / "var" / "blobs" + +# The port the browser suite expects by default, and not 8777, which is still the +# old JS tool's under `serve.py`. Both are meant to run side by side. +DEV_PORT = 8778 diff --git a/server/urls.py b/server/urls.py new file mode 100644 index 0000000..a134005 --- /dev/null +++ b/server/urls.py @@ -0,0 +1,16 @@ +from django.contrib import admin +from django.urls import include, path + +from clips import views + +urlpatterns = [ + # `/index.html` as well as `/`, because that is the URL the browser suite has + # used since step 5 — shadow-cljs's dev server did no directory-index + # resolution, so the suite learned to ask for the file. Keeping both means + # ARTHUR_URL can stay pointed at the same place and only the port moves. + path("", views.page, name="page"), + path("index.html", views.page), + path("api/", include("clips.urls")), + path("blob/", views.blob, name="blob"), + path("admin/", admin.site.urls), +] diff --git a/server/wsgi.py b/server/wsgi.py new file mode 100644 index 0000000..fb6e225 --- /dev/null +++ b/server/wsgi.py @@ -0,0 +1,6 @@ +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server.settings") +application = get_wsgi_application() diff --git a/static/arthur/audio.wav b/static/arthur/audio.wav new file mode 100644 index 0000000..1478629 Binary files /dev/null and b/static/arthur/audio.wav differ