diff --git a/.claude/worktrees/tracing-layers b/.claude/worktrees/tracing-layers new file mode 160000 index 0000000..f4dd047 --- /dev/null +++ b/.claude/worktrees/tracing-layers @@ -0,0 +1 @@ +Subproject commit f4dd04764204506fc275180364d0366d693036f4 diff --git a/.dockerignore b/.dockerignore index ef0e9d8..dca80db 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,5 @@ audio.wav manifest.json *.take *.tflite +.claude +.venv* diff --git a/Dockerfile b/Dockerfile index 805d9a6..6e6d717 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,4 +40,4 @@ RUN python manage.py collectstatic --noinput \ USER app EXPOSE 8000 -CMD ["sh", "-c", "python manage.py migrate --noinput && exec gunicorn server.wsgi:application --bind 0.0.0.0:8000 --workers 1 --threads 4 --timeout 120"] +CMD ["sh", "-c", "python manage.py migrate --noinput && exec daphne --bind 0.0.0.0 --port 8000 server.asgi:application"] diff --git a/README.md b/README.md index 6f178d2..7b70ce8 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ mise exec -- python manage.py migrate ./do start # Django + frontend watcher ``` -In the app, upload a video, choose its footage, click **load frames**, then -**save**. Opening that project on another client reuses its saved landmarks and +In the app, drop a video on the media pool — it uploads, extracts and runs +detection — then **save**. Opening that project on another client reuses its saved landmarks and mouth crops without detecting source frames again. The upload path derives its footage response from database records; it does not create or consume a `manifest.json` file. See [frontend/README.md](frontend/README.md) for details. diff --git a/clips/admin.py b/clips/admin.py index 9e73270..8b7add7 100644 --- a/clips/admin.py +++ b/clips/admin.py @@ -18,7 +18,7 @@ class ProjectAdmin(admin.ModelAdmin): @admin.register(Clip) class ClipAdmin(admin.ModelAdmin): - list_display = ("cid", "project", "name", "footage", "analysis") + list_display = ("cid", "project", "name") list_filter = ("project",) diff --git a/clips/consumers.py b/clips/consumers.py new file mode 100644 index 0000000..98de12f --- /dev/null +++ b/clips/consumers.py @@ -0,0 +1,87 @@ +"""One socket per open project, and it is tl's, nearly line for line. + +Two things ride it. DELTAS, which the server sends after a write commits — the +socket is read-only for the document, and a dropped socket cannot lose a write. +PRESENCE, which peers gossip between themselves: all the server does is hand out +a connection id and stamp the sender's identity onto every message, so nobody can +post as somebody else. +""" +import json +import uuid + +from asgiref.sync import async_to_sync +from channels.generic.websocket import AsyncWebsocketConsumer +from channels.layers import get_channel_layer + +# Who is connected, per project: {group: {cid: presence}}. A cache of what has +# already been relayed, so a joiner gets the room in one message. Process-local, +# like the in-memory channel layer this runs on. +ROOMS = {} + + +def group(project_id): + return f"project_{project_id}" + + +def broadcast(project_id, delta, kind="delta"): + """Send a committed write to everyone in the project's room. `access` says + only that who may write has changed, and each client asks for itself.""" + async_to_sync(get_channel_layer().group_send)( + group(project_id), {"type": "project.delta", "delta": {"kind": kind, **delta}}, + ) + + +class ProjectConsumer(AsyncWebsocketConsumer): + RELAYED = ("state",) + + @property + def room(self): + return ROOMS.setdefault(self.group, {}) + + async def connect(self): + self.group = group(self.scope["url_route"]["kwargs"]["project_id"]) + self.cid = uuid.uuid4().hex[:12] + user = self.scope.get("user") + self.username = user.get_username() if user and user.is_authenticated else None + await self.channel_layer.group_add(self.group, self.channel_name) + await self.accept() + + me = {"cid": self.cid, "user": self.username} + others = list(self.room.values()) + self.room[self.cid] = me + await self.send(text_data=json.dumps({"kind": "welcome", **me})) + await self.send(text_data=json.dumps({"kind": "roster", "peers": others})) + await self._relay({"kind": "join"}) + + async def disconnect(self, code): + if hasattr(self, "cid"): + self.room.pop(self.cid, None) + if not self.room: + ROOMS.pop(self.group, None) + await self._relay({"kind": "leave"}) + await self.channel_layer.group_discard(self.group, self.channel_name) + + async def receive(self, text_data=None, bytes_data=None): + try: + msg = json.loads(text_data or "{}") + except ValueError: + return + if not isinstance(msg, dict) or msg.get("kind") not in self.RELAYED: + return + if self.cid in self.room: + self.room[self.cid].update( + {k: v for k, v in msg.items() if k not in ("kind", "cid", "user")} + ) + await self._relay(msg) + + async def _relay(self, msg): + await self.channel_layer.group_send( + self.group, + {"type": "peer.msg", "msg": {**msg, "cid": self.cid, "user": self.username}}, + ) + + async def peer_msg(self, event): + await self.send(text_data=json.dumps(event["msg"])) + + async def project_delta(self, event): + await self.send(text_data=json.dumps(event["delta"])) diff --git a/clips/extraction.py b/clips/extraction.py index ff31bb6..4c4c5d0 100644 --- a/clips/extraction.py +++ b/clips/extraction.py @@ -159,6 +159,108 @@ def _extract_stills(job, proxy_path, frames_dir, frames, root): MAX_RATE = 120 # a capture rate; past this the container is describing something else +# How many packet timestamps `_measured_rate` reads, and the fewest intervals it +# will draw a conclusion from. 300 is a flat cost on a long take and still a +# wide enough sample for a median; below 8 intervals there is not enough of a +# stream to outvote one odd timestamp, so the metadata is left to speak. +RATE_SAMPLE = 300 +RATE_MINIMUM = 8 +# How far a declared rate may sit from the measured one and still be taken as +# what the stream is: 2% covers 30 against 30000/1001 and nothing like 120 +# against 30. +RATE_TOLERANCE = 0.02 + + +def probe_image(path): + """An uploaded still's pixel size as (width, height), refusing anything that + is not one picture.""" + data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams", + "-of", "json", str(path)])) + video = [s for s in data.get("streams", []) if s.get("codec_type") == "video"] + if len(video) != 1 or not (video[0].get("width") and video[0].get("height")): + raise ValueError("the uploaded file is not an image") + return int(video[0]["width"]), int(video[0]["height"]) + + +def probe_audio(path): + """The length of an uploaded sound in seconds, refusing a file with no audio.""" + data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams", + "-show_format", "-of", "json", str(path)])) + if not any(s.get("codec_type") == "audio" for s in data.get("streams", [])): + raise ValueError("the uploaded file has no audio stream") + duration = float(data.get("format", {}).get("duration") or 0) + if duration <= 0: + raise ValueError("the sound's length is unknown") + return duration + + +def _measured_rate(path): + """The rate the stream's own packet timestamps imply, or None. + + THE CONTAINER'S SUMMARY OF ITSELF IS NOT EVIDENCE, and this is the function + that goes and looks. An iPhone's `r_frame_rate` is 120 on footage whose + timestamps are 1/30s apart, which is the difference between 323 frames and + 1293 — four times the encode, four times the tracing stills, four times the + blobs, for 970 frames that are copies of their neighbours. + + It reads TIMESTAMPS, not frames: `-show_entries packet=pts_time` demuxes + without decoding, so this costs a file read and no pixels. The times are + SORTED before differencing because a stream with B-frames arrives in decode + order — an HEVC clip's first packets come out 0, 0.133, 0.067, 0.033 — and + differencing that order measures the reordering rather than the rate. + + THE MEDIAN INTERVAL, which is what makes this safe on genuinely variable + input. It answers "how far apart are two frames normally", so a take held on + one frame for a second still reports the rate of the parts that move, and + choosing it keeps every distinct frame — the property `probe` used to reach + for by taking the nominal rate. Only the last few intervals of the sample are + unreliable (a frame whose turn comes after the window is missing from it), and + a median does not care. + + Returning None is the honest answer for a clip too short to sample, and this + also swallows a probe that fails outright: the rate the metadata declares is + the documented fallback, so an optimisation must not be able to refuse an + upload that would otherwise have been accepted. + """ + try: + text = _command(["ffprobe", "-v", "error", "-select_streams", "v:0", + "-show_entries", "packet=pts_time", "-of", "json", + "-read_intervals", f"%+#{RATE_SAMPLE}", str(path)]) + packets = json.loads(text).get("packets") or [] + times = sorted(float(packet["pts_time"]) for packet in packets + if (packet.get("pts_time") or "N/A") != "N/A") + except (ValueError, OSError): + return None + intervals = sorted(b - a for a, b in zip(times, times[1:]) if b > a) + if len(intervals) < RATE_MINIMUM: + return None + median = intervals[len(intervals) // 2] + return 1.0 / median if median > 0 else None + + +def _choose_rate(nominal, average, measured): + """The rate to resample onto, as an exact Fraction. + + A DECLARED RATE IS PREFERRED WHEN IT AGREES WITH THE TIMESTAMPS, because it is + the exact rational the stream was authored at — 30000/1001 is not a float, and + `limit_denominator` on a measured 29.97 is a guess at a number the container + already states. So the measured rate is used to CHOOSE between what the + container declares, and only stands in itself when neither declaration + describes the stream. + """ + candidates = [rate for rate in (nominal, average) if 0 < rate <= MAX_RATE] + if measured: + agreeing = [rate for rate in candidates + if abs(float(rate) - measured) <= RATE_TOLERANCE * measured] + if agreeing: + return min(agreeing, key=lambda rate: abs(float(rate) - measured)) + from_timestamps = Fraction(measured).limit_denominator(1001) + if 0 < from_timestamps <= MAX_RATE: + return from_timestamps + # Nothing to go on but the metadata, and nominal first keeps the rate that + # drops no distinct frame. An unusable pair falls through to the refusal + # below, which names the rate the file claimed rather than one of these. + return nominal if 0 < nominal <= MAX_RATE else average def probe(path): @@ -178,12 +280,21 @@ def probe(path): of itself disagreed with the container's own contents, so the guard rejected CFR video for being variable. - THE RATE IS THE NOMINAL ONE. `r_frame_rate` is the rate every timestamp in the - stream can be expressed at, which is the rate that keeps every distinct source - frame; resampling to the average would drop some. Duration is preserved either - way — ffmpeg's CFR conversion is driven by timestamps, so the audio stays in - sync at any rate — so this trades a possible duplicated frame against a - certainly lost one. + THE RATE IS MEASURED AND THE DECLARATIONS ARE VOTED ON, which is the same + distrust applied to the one number that still comes from here. This used to + take `r_frame_rate` outright — the rate every timestamp in the stream can be + expressed at, and so the rate that keeps every distinct source frame. The + trouble is that it is not a claim about frames at all: the file above declares + 120 and holds 30, and resampling it up cost four times the encode, four times + the tracing stills and four times the blobs for 970 duplicated frames. So + `_measured_rate` reads the timestamps, `_choose_rate` keeps whichever declared + rate they bear out, and the nominal rate is believed when it is true rather + than because it is nominal. + + Duration is preserved either way — ffmpeg's CFR conversion is driven by + timestamps, so the audio stays in sync at any rate — and the median interval + keeps the no-distinct-frame-dropped property that taking the nominal rate was + reaching for. See `_measured_rate`. """ data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams", "-show_format", "-of", "json", str(path)])) @@ -194,19 +305,25 @@ def probe(path): average = Fraction(video.get("avg_frame_rate") or "0") if nominal <= 0 and average <= 0: raise ValueError("the video's frame rate is unknown") - rate = nominal if 0 < nominal <= MAX_RATE else average + measured = _measured_rate(path) + rate = _choose_rate(nominal, average, measured) if not 0 < rate <= MAX_RATE: raise ValueError(f"the video reports a frame rate of {float(rate):g}, which is " "not a rate footage can be measured at") duration = float(data.get("format", {}).get("duration") or 0) - if duration > 0 and duration * float(rate) > 901: - raise ValueError("video is longer than the 900-frame footage limit") + # if duration > 0 and duration * float(rate) > 901: + # raise ValueError("video is longer than the 900-frame footage limit") frames = video.get("nb_frames") return {"fps": float(rate), # The exact rate, for ffmpeg. 30000/1001 is not a float, and handing # `-r` a rounded one is how a long take drifts out of sync. "rate": f"{rate.numerator}/{rate.denominator}", "nominal_fps": float(nominal), "average_fps": float(average), + # What the timestamps said, and null when there were too few to ask. + # Recorded because it is the input to a decision this file used not to + # make, and the one number that explains a chosen rate matching + # neither declaration. + "measured_fps": measured, "width": int(video["width"]), "height": int(video["height"]), "duration": duration, # KEPT, AND NO LONGER TRUSTED AS A COUNT. See the docstring: this is @@ -326,8 +443,8 @@ def run(key): proxy_facts = probe(proxy_path) _refuse_a_shifted_timeline(proxy_path) frames = count_frames(proxy_path) - if not 1 <= frames <= 900: - raise ValueError(f"the proxy holds {frames} frames; the limit is 1–900") + #if not 1 <= frames <= 900: + # raise ValueError(f"the proxy holds {frames} frames; the limit is 1–900") # CHECKED AS A DURATION, not as a frame count. The page's clock is # `frame = floor(audio.currentTime * fps)`, so what must not drift is # how long the picture lasts against how long the audio lasts — and diff --git a/clips/migrations/0007_symbols_not_timelines.py b/clips/migrations/0007_symbols_not_timelines.py new file mode 100644 index 0000000..1f726c8 --- /dev/null +++ b/clips/migrations/0007_symbols_not_timelines.py @@ -0,0 +1,75 @@ +"""Schema 2: a document holds symbols, not timelines, and no symbol is reserved. + +Three renames, each in the stored transit and nowhere else: + + clip//timeline/... -> clip//symbol/... + a node leaf's :kind :symbol -> :kind :instance + a feature leaf's :timeline key -> :symbol + +A leaf value is transit's map form, ["^ ", k1, v1, k2, v2, ...]. Only TOP-LEVEL +pairs are rewritten, and only literal ones: transit caches a repeated keyword as +"^N", and a rename that met a cache reference where it expected the keyword would +be guessing. Every saved leaf at the time of writing had these as literals; if one +does not, the migration stops rather than writing a document that decodes to +something else. + +Renaming a cached keyword in place is safe because the cache is positional: the +literal keeps its slot, so any later "^N" that referred to it now refers to the +new name, which is what it meant. +""" + +import re + +from django.db import migrations, models + +PATH = re.compile(r"^(clip/[^/]+/)timeline(/|$)") + + +def _rename_pair(value, key, old, new, path): + if not (isinstance(value, list) and value[:1] == ["^ "]): + return value + out = list(value) + for i in range(1, len(out) - 1, 2): + if out[i] != key: + continue + if old is None: + out[i] = new + elif out[i + 1] == old: + out[i + 1] = new + elif isinstance(out[i + 1], str) and out[i + 1].startswith("^") and out[i + 1] != "^ ": + raise RuntimeError(f"leaf {path!r} has a cached {key} value; migrate it by hand") + return out + + +def forwards(apps, schema_editor): + Leaf = apps.get_model("clips", "Leaf") + Project = apps.get_model("clips", "Project") + for leaf in Leaf.objects.all(): + path = PATH.sub(r"\1symbol\2", leaf.path) + value = leaf.value + parts = path.split("/") + if len(parts) == 6 and parts[2] == "symbol" and parts[4] == "node": + value = _rename_pair(value, "~:kind", "~:symbol", "~:instance", leaf.path) + if len(parts) == 4 and parts[2] == "feature": + value = _rename_pair(value, "~:timeline", None, "~:symbol", leaf.path) + if path != leaf.path or value != leaf.value: + leaf.path = path + leaf.value = value + leaf.version += 1 + leaf.save(update_fields=["path", "value", "version"]) + Project.objects.update(schema_version=2) + + +class Migration(migrations.Migration): + dependencies = [ + ("clips", "0006_project_schema_version"), + ] + + operations = [ + migrations.AlterField( + model_name="project", + name="schema_version", + field=models.PositiveIntegerField(default=2), + ), + migrations.RunPython(forwards, migrations.RunPython.noop), + ] diff --git a/clips/migrations/0008_owners_editors_leaf_seq.py b/clips/migrations/0008_owners_editors_leaf_seq.py new file mode 100644 index 0000000..66aaa89 --- /dev/null +++ b/clips/migrations/0008_owners_editors_leaf_seq.py @@ -0,0 +1,39 @@ +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +def orphans(apps, schema_editor): + # Every project has an owner, and none of the ones saved before owners did. + apps.get_model("clips", "Project").objects.all().delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ("clips", "0007_symbols_not_timelines"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.RunPython(orphans, migrations.RunPython.noop), + migrations.AddField( + model_name="leaf", + name="seq", + field=models.PositiveBigIntegerField( + default=0, help_text="the project seq of the write that last changed it"), + ), + migrations.AddField( + model_name="project", + name="editors", + field=models.ManyToManyField(blank=True, related_name="shared_projects", + to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name="project", + name="owner", + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, + related_name="projects", to=settings.AUTH_USER_MODEL), + preserve_default=False, + ), + ] diff --git a/clips/migrations/0009_revision_blocks.py b/clips/migrations/0009_revision_blocks.py new file mode 100644 index 0000000..3bd1303 --- /dev/null +++ b/clips/migrations/0009_revision_blocks.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.17 on 2026-09-30 01:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('clips', '0008_owners_editors_leaf_seq'), + ] + + operations = [ + migrations.AddField( + model_name='revision', + name='blocks', + field=models.JSONField(default=dict, help_text="each clip's tier-2 block keys, by cid, so a restore can name them"), + ), + ] diff --git a/clips/migrations/0010_sounds.py b/clips/migrations/0010_sounds.py new file mode 100644 index 0000000..1f26395 --- /dev/null +++ b/clips/migrations/0010_sounds.py @@ -0,0 +1,25 @@ +# Generated by Django 5.2.17 on 2026-09-30 07:14 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('clips', '0009_revision_blocks'), + ] + + operations = [ + migrations.CreateModel( + name='Sound', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('filename', models.CharField(max_length=255)), + ('duration', models.FloatField(help_text='seconds, as ffprobe reports it')), + ('created', models.DateTimeField(auto_now_add=True)), + ('blob', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='sound_for', to='clips.blob')), + ], + ), + ] diff --git a/clips/migrations/0011_occurrence_schema.py b/clips/migrations/0011_occurrence_schema.py new file mode 100644 index 0000000..cf9ad2e --- /dev/null +++ b/clips/migrations/0011_occurrence_schema.py @@ -0,0 +1,13 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [("clips", "0010_sounds")] + + operations = [ + migrations.AlterField( + model_name="project", + name="schema_version", + field=models.PositiveIntegerField(default=3), + ), + ] diff --git a/clips/migrations/0012_sound_label.py b/clips/migrations/0012_sound_label.py new file mode 100644 index 0000000..7baee07 --- /dev/null +++ b/clips/migrations/0012_sound_label.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.17 on 2026-10-01 04:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('clips', '0011_occurrence_schema'), + ] + + operations = [ + migrations.AddField( + model_name='sound', + name='label', + field=models.CharField(blank=True, help_text='what a person called it; the filename when empty. Separate from `filename` because the name on disk is a fact about the upload and renaming must not rewrite it', max_length=200), + ), + ] diff --git a/clips/migrations/0013_palette_track.py b/clips/migrations/0013_palette_track.py new file mode 100644 index 0000000..ab00fc7 --- /dev/null +++ b/clips/migrations/0013_palette_track.py @@ -0,0 +1,45 @@ +"""Schema 4: split animated symbol palettes from the authoring palette. + +Before schema 4 a symbol's ``:palette`` leaf value was always a channel. It now +names the static palette used when that symbol is the viewed root, while the +old channel is retained as ``:palette-channel`` compatibility data. New edits +use a real lane symbol referenced by ``:palette-track``. +""" + +from django.db import migrations, models + + +def forwards(apps, schema_editor): + Leaf = apps.get_model("clips", "Leaf") + Project = apps.get_model("clips", "Project") + for leaf in Leaf.objects.filter(path__contains="/symbol/"): + parts = leaf.path.split("/") + if len(parts) != 4 or parts[2] != "symbol": + continue + value = leaf.value + if not (isinstance(value, list) and value[:1] == ["^ "]): + continue + out = list(value) + changed = False + for i in range(1, len(out) - 1, 2): + if out[i] == "~:palette" and isinstance(out[i + 1], list): + out[i] = "~:palette-channel" + changed = True + if changed: + leaf.value = out + leaf.version += 1 + leaf.save(update_fields=["value", "version"]) + Project.objects.update(schema_version=4) + + +class Migration(migrations.Migration): + dependencies = [("clips", "0012_sound_label")] + + operations = [ + migrations.AlterField( + model_name="project", + name="schema_version", + field=models.PositiveIntegerField(default=4), + ), + migrations.RunPython(forwards, migrations.RunPython.noop), + ] diff --git a/clips/migrations/0014_multiple_analyses.py b/clips/migrations/0014_multiple_analyses.py new file mode 100644 index 0000000..97fe72b --- /dev/null +++ b/clips/migrations/0014_multiple_analyses.py @@ -0,0 +1,15 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [("clips", "0013_palette_track")] + + operations = [ + migrations.RemoveField(model_name="clip", name="analysis"), + migrations.RemoveField(model_name="clip", name="footage"), + migrations.AlterField( + model_name="project", + name="schema_version", + field=models.PositiveIntegerField(default=5), + ), + ] diff --git a/clips/migrations/0015_tracing_images.py b/clips/migrations/0015_tracing_images.py new file mode 100644 index 0000000..ab3aa3c --- /dev/null +++ b/clips/migrations/0015_tracing_images.py @@ -0,0 +1,48 @@ +"""Schema 6: tracing is a symbol. + +A face's `:head :trace` is gone: its footage is a placement of a `:type :trace` +symbol under the head, the trace keys are that placement's `:time :holds`, and the +head follows them with `:reads`. Nothing is converted. Every project is marked 6, +and one that still carries a `:trace` is refused when it is opened, by name and +with what to do about it; the rest open as they did. + +Images are stills to trace over, stored like sounds. +""" + +import uuid + +from django.db import migrations, models +import django.db.models.deletion + + +def forwards(apps, schema_editor): + apps.get_model("clips", "Project").objects.update(schema_version=6) + + +class Migration(migrations.Migration): + dependencies = [("clips", "0014_multiple_analyses")] + + operations = [ + migrations.CreateModel( + name="Image", + fields=[ + ("id", models.UUIDField(default=uuid.uuid4, editable=False, + primary_key=True, serialize=False)), + ("filename", models.CharField(max_length=255)), + ("label", models.CharField( + blank=True, max_length=200, + help_text="what a person called it; the filename when empty")), + ("width", models.PositiveIntegerField(help_text="pixels, as ffprobe reports them")), + ("height", models.PositiveIntegerField()), + ("created", models.DateTimeField(auto_now_add=True)), + ("blob", models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, + related_name="image_for", to="clips.blob")), + ], + ), + migrations.AlterField( + model_name="project", + name="schema_version", + field=models.PositiveIntegerField(default=6), + ), + migrations.RunPython(forwards, migrations.RunPython.noop), + ] diff --git a/clips/migrations/0016_an_anchor_is_a_peg.py b/clips/migrations/0016_an_anchor_is_a_peg.py new file mode 100644 index 0000000..208d603 --- /dev/null +++ b/clips/migrations/0016_an_anchor_is_a_peg.py @@ -0,0 +1,42 @@ +"""Schema 7: an anchor is a peg. + +`[:xform :anchor]` is gone from the transform. `T(a)·M·T(-a)` is a transform +conjugated by a translation — "do M in a frame shifted by a" — and a parent +already is a shifted frame, so an anchor was a peg written inline: one that could +not be selected, keyed, shared between nodes, or placed above a measured channel. +A pivot nobody chose is now derived from what the node draws, per drag, and stored +nowhere; a pivot to keep is a peg, an ordinary `:group` parent. + +Nothing is converted, as in schema 6. Every project is marked 7, and one that +still carries an anchor is refused when it is opened, by name and with what to do +about it — `node/problems` in the frontend. + +Not converted rather than not worth converting. Dropping an anchor is in fact +pixel-exact wherever rotation and scale are the identity, since the anchor +cancels out of the composition there — and that is everywhere a freeze, a drop or +a new drawing wrote one. It is NOT exact on anything a hand has since turned or +scaled, where the composed translation is `a + p - M·a`, and it cannot be made +exact at all where `pos` is dense, because tier 2 is content-addressed and not +rewritable here. A conversion would therefore be silent and right for most nodes +and silent and wrong for exactly the ones somebody had hand-placed, which is the +worse failure: a refusal names the document and says what to do. +""" + +from django.db import migrations, models + + +def forwards(apps, schema_editor): + apps.get_model("clips", "Project").objects.update(schema_version=7) + + +class Migration(migrations.Migration): + dependencies = [("clips", "0015_tracing_images")] + + operations = [ + migrations.AlterField( + model_name="project", + name="schema_version", + field=models.PositiveIntegerField(default=7), + ), + migrations.RunPython(forwards, migrations.RunPython.noop), + ] diff --git a/clips/migrations/0017_a_node_has_a_pivot.py b/clips/migrations/0017_a_node_has_a_pivot.py new file mode 100644 index 0000000..2e71d6e --- /dev/null +++ b/clips/migrations/0017_a_node_has_a_pivot.py @@ -0,0 +1,45 @@ +"""Schema 8: a node has a pivot. + +`[:xform :pivot]` is back in the transform, as the point rotation and scale are +composed about: `local = T(pos)·T(piv)·R·K·S·T(-piv)`. Schema 7 deleted it, on +the argument that an anchor is a peg — true as algebra, and not true as a feature. +A peg is a node, and a turn about a point that is not the turning node's own +origin still has to solve for a position to hold that point still; that solution +is an arc in the angle while a position channel tweens along the chord, so it is +right on the frame it is written and wrong on every frame between two keys. A +drawing escaped it, since its origin is the middle of what it draws. A symbol +instance could not: its origin is its symbol's, which is the top-left corner of +the stage, so one keyed turn of an instance swung its drawing round that corner +on an orbit the size of the stage. + +CONVERTED, unlike 6 and 7, because adding this one is exact. A schema-7 node has +no pivot; an absent pivot reads as [0 0]; and T(pos)·T(0)·M·T(-0) is T(pos)·M to +the last bit of the mantissa. Every stored document therefore composes to exactly +the matrices it composed to before, dense tier-2 transforms included, so there is +nothing to guess at and no node a conversion could silently move. The version is +restamped and nothing else is touched. + +What a converted document does NOT get is a pivot somebody chose: nodes placed +before this carry none, so they still turn about their own origin until the first +turn or scale writes one — `gesture/with-pivot`, from the middle of what the node +draws at that moment — or until the cross is dragged (ctrl/cmd-drag on the stage). +""" + +from django.db import migrations, models + + +def forwards(apps, schema_editor): + apps.get_model("clips", "Project").objects.update(schema_version=8) + + +class Migration(migrations.Migration): + dependencies = [("clips", "0016_an_anchor_is_a_peg")] + + operations = [ + migrations.AlterField( + model_name="project", + name="schema_version", + field=models.PositiveIntegerField(default=8), + ), + migrations.RunPython(forwards, migrations.RunPython.noop), + ] diff --git a/clips/models.py b/clips/models.py index b120873..c8cc44c 100644 --- a/clips/models.py +++ b/clips/models.py @@ -24,7 +24,9 @@ without parsing its leaves: which footage, which analysis, which blocks. """ import uuid +from django.conf import settings from django.db import models +from django.utils import timezone class Blob(models.Model): @@ -50,6 +52,39 @@ class Source(models.Model): created = models.DateTimeField(auto_now_add=True) +class Sound(models.Model): + """An uploaded sound file — mp3, wav, whatever the browser can decode — kept + as uploaded. Not footage: it has no frames and nothing measures it, so it + skips extraction and an audio node plays its bytes directly.""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + blob = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="sound_for") + filename = models.CharField(max_length=255) + label = models.CharField( + max_length=200, blank=True, + help_text="what a person called it; the filename when empty. Separate " + "from `filename` because the name on disk is a fact about the " + "upload and renaming must not rewrite it", + ) + duration = models.FloatField(help_text="seconds, as ffprobe reports it") + created = models.DateTimeField(auto_now_add=True) + + +class Image(models.Model): + """An uploaded still — a drawing, a photo, a model sheet — kept as uploaded, to + be traced over. Never part of the picture: a document names its blob as a + tracing symbol's `:media`, and the page draws it over the stage.""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + blob = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="image_for") + filename = models.CharField(max_length=255) + label = models.CharField(max_length=200, blank=True, + help_text="what a person called it; the filename when empty") + width = models.PositiveIntegerField(help_text="pixels, as ffprobe reports them") + height = models.PositiveIntegerField() + created = models.DateTimeField(auto_now_add=True) + + class Extraction(models.Model): """One requested decode of a source into immutable footage.""" @@ -201,12 +236,21 @@ class Project(models.Model): `schema_version` identifies the stored document format. `seq` counts writes to this particular project; it is not a format version. Every write bumps - `seq`, and a client that sees `seq > local + 1` refetches once broadcasts exist. + `seq`, and a client that sees `seq > local + 1` refetches. + + ANYONE WITH THE LINK CAN VIEW; the owner and the editors can write. Every + project has an owner. """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + owner = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="projects", + ) + editors = models.ManyToManyField( + settings.AUTH_USER_MODEL, blank=True, related_name="shared_projects", + ) name = models.CharField(max_length=200, default="untitled") - schema_version = models.PositiveIntegerField(default=1) + schema_version = models.PositiveIntegerField(default=8) seq = models.PositiveBigIntegerField(default=0) palette = models.CharField(max_length=64, default="arthur/default") created = models.DateTimeField(auto_now_add=True) @@ -219,10 +263,19 @@ class Project(models.Model): return f"{self.name} ({self.id})" def bump(self): - self.seq += 1 - self.save(update_fields=["seq", "updated"]) + """The next seq, taken with an UPDATE so that inside a transaction it is + also the write lock: two concurrent saves cannot both get the same one.""" + Project.objects.filter(id=self.id).update( + seq=models.F("seq") + 1, updated=timezone.now() + ) + self.refresh_from_db(fields=["seq", "updated"]) return self.seq + def can_edit(self, user): + return user.is_authenticated and ( + user.id == self.owner_id or self.editors.filter(id=user.id).exists() + ) + class Clip(models.Model): """Tier 1: the unit of work, and the thing leaf paths are scoped by. @@ -235,12 +288,6 @@ class Clip(models.Model): cid = models.SlugField(max_length=64) name = models.CharField(max_length=200, blank=True) order = models.IntegerField(default=0) - footage = models.ForeignKey( - Footage, null=True, blank=True, on_delete=models.SET_NULL, related_name="clips" - ) - analysis = models.ForeignKey( - Analysis, null=True, blank=True, on_delete=models.SET_NULL, related_name="clips" - ) blocks = models.ManyToManyField( Block, blank=True, related_name="clips", help_text="the tier-2 blocks this clip's channels name", @@ -271,6 +318,9 @@ class Leaf(models.Model): path = models.CharField(max_length=300) value = models.JSONField() version = models.PositiveBigIntegerField(default=1) + seq = models.PositiveBigIntegerField( + default=0, help_text="the project seq of the write that last changed it", + ) updated = models.DateTimeField(auto_now=True) class Meta: @@ -288,7 +338,9 @@ class Leaf(models.Model): class Revision(models.Model): - """Tier 1: a snapshot of the authored layer, with a user and a summary. + """Tier 1: a snapshot of the authored layer, with a user and a summary — a + named snapshot, which is how a person marks a version now that every edit + saves itself. ON AN EXPLICIT TRIGGER, not on every save. tl snapshots a small annotation layer; arthur's tier 1 will contain cel polygons, so a snapshot per save bloats @@ -302,6 +354,9 @@ class Revision(models.Model): author = models.CharField(max_length=200, blank=True) summary = models.CharField(max_length=500, blank=True) document = models.JSONField(help_text="every leaf of the project, by path") + blocks = models.JSONField( + default=dict, help_text="each clip's tier-2 block keys, by cid, so a restore can name them", + ) created = models.DateTimeField(auto_now_add=True) class Meta: diff --git a/clips/routing.py b/clips/routing.py new file mode 100644 index 0000000..b94cd1b --- /dev/null +++ b/clips/routing.py @@ -0,0 +1,7 @@ +from django.urls import path + +from .consumers import ProjectConsumer + +websocket_urlpatterns = [ + path("ws/projects/", ProjectConsumer.as_asgi()), +] diff --git a/clips/templates/clips/index.html b/clips/templates/clips/index.html index e08393d..d73c62e 100644 --- a/clips/templates/clips/index.html +++ b/clips/templates/clips/index.html @@ -2,11 +2,14 @@ {% comment %} The host page, served by Django since port-plan step 9. -It was `frontend/public/index.html`, served by shadow-cljs's `:dev-http`, and that -key is gone. The bundle is unchanged: shadow-cljs writes it into -`static/arthur/js` and staticfiles serves it from there, so `manage.py runserver` -and `shadow-cljs watch app` are the whole dev loop with nothing copying files -between them. +It carries no styles of its own any more. They are `static/arthur/app.css`, which +staticfiles serves from the same tree as the bundle — the page grew a five-pane +application chrome and "the styles" stopped being a thing you read in passing on +the way to the markup. + +The bundle is unchanged: shadow-cljs writes it into `static/arthur/js` and +staticfiles serves it from there, so `manage.py runserver` and `shadow-cljs watch +app` are the whole dev loop with nothing copying files between them. The CSRF token is rendered so that Django sets its cookie, which is what `arthur.fx.http` reads to write the `X-CSRFToken` header. Saves are ordinary POSTs @@ -17,72 +20,12 @@ and PUTs with ordinary CSRF protection — no endpoint in this app is exempt. arthur - + {% csrf_token %}
- + diff --git a/clips/tests/test_api.py b/clips/tests/test_api.py index 08554ad..706c89a 100644 --- a/clips/tests/test_api.py +++ b/clips/tests/test_api.py @@ -34,7 +34,7 @@ from django.core.management import call_command from django.test import TestCase, override_settings from clips import blobs, extraction -from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision, Source +from clips.models import Analysis, Block, Blob, Clip, Footage, Image, Leaf, Project, Revision, Sound, Source BLOB_DIR = tempfile.mkdtemp(prefix="arthur-test-blobs-") @@ -361,7 +361,10 @@ class DocumentTests(TestCase): """Tier 1: load, save, and the conditional write.""" def setUp(self): - self.project = Project.objects.create(name="a project") + from django.contrib.auth import get_user_model + owner = get_user_model().objects.create_user("owner", password="password1") + self.client.force_login(owner) + self.project = Project.objects.create(name="a project", owner=owner) descriptor = analysis_descriptor() self.analysis = key_for(descriptor) self.client.post("/api/analyses", data=json.dumps( @@ -382,37 +385,73 @@ class DocumentTests(TestCase): # cache marker, keyword keys, and a frame-keyed inner map. return { "clip/c1/timing": ["^ ", "~:fps", 30], - "clip/c1/timeline/main": ["^ ", "~:frames", 48], - "clip/c1/timeline/main/node/mouth": ["^ ", "~:id", "~:mouth", "~:z", "a1"], - "clip/c1/timeline/main/channel/mouth/geom.pts": [ + "clip/c1/symbol/main": ["^ ", "~:frames", 48], + "clip/c1/symbol/main/node/mouth": ["^ ", "~:id", "~:mouth", "~:z", "a1"], + "clip/c1/symbol/main/channel/mouth/geom.pts": [ "^ ", "~:animated?", True, "~:dense", ["^ ", "~:store", self.block, "~:offset", 0, "~:stride", 16], ], - "clip/c1/timeline/main/channel/mouth-in/vis": [ + "clip/c1/symbol/main/channel/mouth-in/vis": [ "^ ", "~:animated?", True, "~:keys", ["^ ", "~i0", True, "~i12", False], ], } - def save(self, leaves=None, blocks=None): + def save(self, leaves=None, blocks=None, analyses=None): return self.put(f"/api/projects/{self.project.id}", { "name": "a project", - "clips": [{"cid": "c1", "name": "take", "analysis": self.analysis, + "clips": [{"cid": "c1", "name": "take", + "analyses": [self.analysis] if analyses is None else analyses, "leaves": leaves if leaves is not None else self.leaves(), "blocks": blocks if blocks is not None else [self.block]}], }) + def test_a_clip_declares_the_registered_analyses_its_blocks_name(self): + undeclared = self.save(analyses=[]) + self.assertEqual(409, undeclared.status_code) + self.assertIn("every block", undeclared.json()["error"]) + + unknown = "sha256:" + "f" * 64 + missing = self.save(analyses=[self.analysis, unknown]) + self.assertEqual(409, missing.status_code) + self.assertEqual([unknown], missing.json()["missing"]) + + def test_every_saved_symbol_is_listed_across_projects(self): + leaves = self.leaves() + leaves["clip/c1/symbol/sym~face"] = ["^ ", "~:name", "face", "~:frames", 12] + leaves["clip/c1/symbol/sym~face/node/mark"] = ["^ ", "~:id", "~:mark", "~:z", "a1"] + self.assertEqual(200, self.save(leaves).status_code) + rows = self.client.get("/api/symbols").json()["symbols"] + self.assertEqual( + [("face", "sym~face", 12), ("main", "main", 48)], + [(r["name"], r["symbol"], r["frames"]) for r in rows]) + self.assertEqual({str(self.project.id)}, {r["project"] for r in rows}) + self.assertEqual({"c1"}, {r["cid"] for r in rows}) + + def test_saved_palettes_are_listed_as_assets(self): + leaves = self.leaves() + leaves["clip/c1/palette/night"] = [ + "^ ", "~:id", "~:night", "~:name", "Moonlit", + "~:slots", ["~#list", [["^ ", "~:hex", "#001122"]]], + ] + self.assertEqual(200, self.save(leaves).status_code) + rows = self.client.get("/api/symbols").json()["palettes"] + self.assertEqual( + [("Moonlit", "night", "c1")], + [(r["name"], r["palette"], r["cid"]) for r in rows], + ) + def test_a_document_comes_back_exactly(self): response = self.save() self.assertEqual(200, response.status_code, response.content) self.assertEqual(5, len(response.json()["written"])) loaded = self.client.get(f"/api/projects/{self.project.id}").json() - self.assertEqual(1, loaded["schema_version"]) + self.assertEqual(7, loaded["schema_version"]) self.assertEqual(1, len(loaded["clips"])) clip = loaded["clips"][0] self.assertEqual("c1", clip["cid"]) self.assertEqual([self.block], clip["blocks"]) - self.assertEqual(self.analysis, clip["analysis"]) + self.assertNotIn("analysis", clip) # The whole point: byte-identical values, including the integer frame keys # transit writes as "~i0". A JSON round trip that stringified them would # come back "0" and the part would hold its first pose forever. @@ -424,22 +463,22 @@ class DocumentTests(TestCase): self.save() first = {leaf.path: leaf.version for leaf in Leaf.objects.all()} moved = self.leaves() - moved["clip/c1/timeline/main/channel/mouth-in/vis"] = [ + moved["clip/c1/symbol/main/channel/mouth-in/vis"] = [ "^ ", "~:animated?", True, "~:keys", ["^ ", "~i0", False], ] response = self.save(moved) - self.assertEqual(["clip/c1/timeline/main/channel/mouth-in/vis"], response.json()["written"]) + self.assertEqual(["clip/c1/symbol/main/channel/mouth-in/vis"], response.json()["written"]) self.assertEqual(4, response.json()["unchanged"]) after = {leaf.path: leaf.version for leaf in Leaf.objects.all()} - self.assertEqual(2, after["clip/c1/timeline/main/channel/mouth-in/vis"]) + self.assertEqual(2, after["clip/c1/symbol/main/channel/mouth-in/vis"]) self.assertEqual(first["clip/c1/timing"], after["clip/c1/timing"]) def test_a_removed_node_removes_its_leaf(self): self.save() fewer = {k: v for k, v in self.leaves().items() - if k != "clip/c1/timeline/main/node/mouth"} + if k != "clip/c1/symbol/main/node/mouth"} response = self.save(fewer) - self.assertEqual(["clip/c1/timeline/main/node/mouth"], response.json()["removed"]) + self.assertEqual(["clip/c1/symbol/main/node/mouth"], response.json()["removed"]) self.assertEqual(4, Leaf.objects.count()) def test_a_save_does_not_disturb_another_clip(self): @@ -472,7 +511,7 @@ class DocumentTests(TestCase): def test_a_leaf_write_carries_an_etag(self): self.save() - url = f"/api/projects/{self.project.id}/leaves/clip/c1/timeline/main/node/mouth" + url = f"/api/projects/{self.project.id}/leaves/clip/c1/symbol/main/node/mouth" got = self.client.get(url) self.assertEqual('"1"', got["ETag"]) @@ -488,7 +527,7 @@ class DocumentTests(TestCase): # take-theirs. A PUT that replaced unconditionally is the bug where the # loser's work disappears silently. self.save() - url = f"/api/projects/{self.project.id}/leaves/clip/c1/timeline/main/node/mouth" + url = f"/api/projects/{self.project.id}/leaves/clip/c1/symbol/main/node/mouth" self.put(url, {"value": ["^ ", "~:z", "a2"]}, HTTP_IF_MATCH='"1"') stale = self.put(url, {"value": ["^ ", "~:z", "a3"]}, HTTP_IF_MATCH='"1"') self.assertEqual(409, stale.status_code) @@ -511,6 +550,61 @@ class DocumentTests(TestCase): # --- revisions --------------------------------------------------------- + def patch(self, base, leaves, removed=()): + return self.put(f"/api/projects/{self.project.id}", { + "base": base, + "clips": [{"cid": "c1", "analyses": [self.analysis], "leaves": leaves, + "removed": list(removed), "blocks": [self.block]}], + }) + + def test_a_patch_leaves_what_it_does_not_name_alone(self): + seq = self.save().json()["seq"] + response = self.patch(seq, {"clip/c1/timing": ["^ ", "~:fps", 24]}, + removed=["clip/c1/symbol/main/node/mouth"]) + self.assertEqual(200, response.status_code, response.content) + self.assertEqual(["clip/c1/timing"], response.json()["written"]) + self.assertEqual(4, Leaf.objects.count()) + + def test_two_people_on_different_leaves_both_land(self): + seq = self.save().json()["seq"] + self.assertEqual(200, self.patch(seq, {"clip/c1/timing": ["^ ", "~:fps", 24]}).status_code) + # The second saver has not caught up, and touched a different leaf. + response = self.patch(seq, {"clip/c1/symbol/main": ["^ ", "~:frames", 12]}) + self.assertEqual(200, response.status_code, response.content) + leaves = self.client.get(f"/api/projects/{self.project.id}").json()["clips"][0]["leaves"] + self.assertEqual(["^ ", "~:fps", 24], leaves["clip/c1/timing"]) + self.assertEqual(["^ ", "~:frames", 12], leaves["clip/c1/symbol/main"]) + + def test_two_people_on_one_leaf_is_a_conflict_that_writes_nothing(self): + seq = self.save().json()["seq"] + self.patch(seq, {"clip/c1/timing": ["^ ", "~:fps", 24]}) + response = self.patch(seq, {"clip/c1/timing": ["^ ", "~:fps", 12], + "clip/c1/symbol/main": ["^ ", "~:frames", 12]}) + self.assertEqual(409, response.status_code) + self.assertEqual({"clip/c1/timing": ["^ ", "~:fps", 24]}, response.json()["conflicts"]) + self.assertEqual(seq + 1, Project.objects.get(id=self.project.id).seq) + self.assertEqual(["^ ", "~:frames", 48], + Leaf.objects.get(path="clip/c1/symbol/main").value) + # Caught up to their seq, the same write is ordinary. + self.assertEqual(200, self.patch(seq + 1, {"clip/c1/timing": ["^ ", "~:fps", 12]}).status_code) + + def test_a_named_snapshot_restores_as_an_ordinary_write(self): + self.save() + snap = self.client.post(f"/api/projects/{self.project.id}/revisions", + data=json.dumps({"summary": "before the big change"}), + content_type="application/json").json() + moved = self.leaves() + moved["clip/c1/timing"] = ["^ ", "~:fps", 12] + del moved["clip/c1/symbol/main/node/mouth"] + self.save(moved) + listed = self.client.get(f"/api/projects/{self.project.id}/revisions").json()["revisions"] + self.assertEqual(["before the big change"], [r["summary"] for r in listed]) + restored = self.client.post( + f"/api/projects/{self.project.id}/revisions/{snap['id']}/restore").json() + self.assertEqual(2, restored["changed"]) + leaves = self.client.get(f"/api/projects/{self.project.id}").json()["clips"][0]["leaves"] + self.assertEqual(self.leaves(), leaves) + def test_a_revision_snapshots_the_authored_layer(self): self.save() response = self.client.post( @@ -587,6 +681,64 @@ class FootageTests(TestCase): with self.assertRaisesMessage(CommandError, "refusing an inaccurate footage"): call_command("ingest_bundle", str(root), stdout=StringIO()) + def test_footage_can_be_renamed_and_falls_back_when_cleared(self): + """A LABEL IS THE ONE FIELD A CLIENT MAY WRITE ON FOOTAGE. The rest is a + description of bytes that are content-addressed and immutable, so a + rename that could reach `frames` or `digest` would let the pool's name + for a clip contradict the clip.""" + footage = self.ingest(self.bundle()) + self.assertEqual("IMG_8608.MOV", self.client.get( + f"/api/footage/{footage.id}").json()["label"]) + + renamed = self.client.patch(f"/api/footage/{footage.id}", + json.dumps({"label": " the long take "}), + content_type="application/json") + self.assertEqual(200, renamed.status_code, renamed.content) + self.assertEqual("the long take", renamed.json()["label"]) + # On the row, so every project listing this footage sees the new name. + footage.refresh_from_db() + self.assertEqual("the long take", footage.label) + self.assertEqual("the long take", + self.client.get("/api/footage").json()["footage"][0]["label"]) + + # Cleared gives back the name it was ingested under rather than nothing. + cleared = self.client.patch(f"/api/footage/{footage.id}", + json.dumps({"label": ""}), + content_type="application/json") + self.assertEqual("IMG_8608.MOV", cleared.json()["label"]) + self.assertEqual(3, Footage.objects.get().frames) + + def test_a_rename_that_names_no_label_is_refused(self): + footage = self.ingest(self.bundle()) + refused = self.client.patch(f"/api/footage/{footage.id}", + json.dumps({"frames": 900}), + content_type="application/json") + self.assertEqual(400, refused.status_code) + self.assertEqual("a rename needs a label", refused.json()["error"]) + self.assertEqual(3, Footage.objects.get().frames) + + def test_a_sound_is_renamed_without_losing_the_name_it_arrived_as(self): + """`filename` is a fact about the upload and `label` is what a person + called it, which is why renaming does not write over the first one.""" + digest, size = blobs.write_stream([b"RIFF....WAVEfmt "]) + blob = Blob.objects.create(digest=digest, size=size, media_type="audio/wav") + sound = Sound.objects.create(blob=blob, filename="rec0012.wav", duration=2.5) + + renamed = self.client.patch(f"/api/sounds/{sound.id}", + json.dumps({"label": "arthur, line 4"}), + content_type="application/json") + self.assertEqual(200, renamed.status_code, renamed.content) + self.assertEqual("arthur, line 4", renamed.json()["label"]) + self.assertEqual("rec0012.wav", renamed.json()["filename"]) + sound.refresh_from_db() + self.assertEqual("rec0012.wav", sound.filename) + self.assertEqual("arthur, line 4", sound.label) + + self.client.patch(f"/api/sounds/{sound.id}", json.dumps({"label": " "}), + content_type="application/json") + self.assertEqual("rec0012.wav", + self.client.get(f"/api/sounds/{sound.id}").json()["label"]) + def test_the_footage_list_does_not_carry_every_url(self): # A list of takes should not be a list of six hundred URLs each. self.ingest(self.bundle()) @@ -651,6 +803,62 @@ class UploadTests(TestCase): self.assertEqual(27, job.progress) job.save.assert_called_once_with(update_fields=["progress", "updated"]) + def test_an_uploaded_mp3_is_a_sound_and_not_footage(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "tone.mp3" + subprocess.run([ + "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "sine=frequency=440:duration=1.5", str(path), + ], check=True, capture_output=True) + payload = path.read_bytes() + + uploaded = self.client.post("/api/sounds", { + "file": SimpleUploadedFile("tone.mp3", payload, content_type="audio/mpeg")}) + self.assertEqual(201, uploaded.status_code, uploaded.content) + sound = uploaded.json() + self.assertEqual("tone.mp3", sound["label"]) + self.assertAlmostEqual(1.5, sound["duration"], delta=0.1) + self.assertEqual(payload, b"".join(self.client.get(sound["audio"]).streaming_content)) + self.assertEqual(sound, self.client.get(f"/api/sounds/{sound['id']}").json()) + self.assertEqual([sound], self.client.get("/api/sounds").json()["sounds"]) + self.assertEqual(0, Source.objects.count()) + + again = self.client.post("/api/sounds", { + "file": SimpleUploadedFile("again.mp3", payload, content_type="audio/mpeg")}) + self.assertEqual(200, again.status_code) + self.assertEqual(1, Sound.objects.count()) + + def test_an_uploaded_still_is_an_image_named_by_its_bytes(self): + payload = png(17, 5) + uploaded = self.client.post("/api/images", { + "file": SimpleUploadedFile("sheet.png", payload, content_type="image/png")}) + self.assertEqual(201, uploaded.status_code, uploaded.content) + image = uploaded.json() + self.assertEqual(("sheet.png", 17, 5), (image["label"], image["width"], image["height"])) + self.assertEqual(f"/blob/{image['digest']}", image["url"]) + self.assertEqual(payload, b"".join(self.client.get(image["url"]).streaming_content)) + self.assertEqual([image], self.client.get("/api/images").json()["images"]) + + again = self.client.post("/api/images", { + "file": SimpleUploadedFile("again.png", payload, content_type="image/png")}) + self.assertEqual(200, again.status_code) + self.assertEqual(1, Image.objects.count()) + + renamed = self.client.patch(f"/api/images/{image['id']}", + json.dumps({"label": "model sheet"}), + content_type="application/json") + self.assertEqual("model sheet", renamed.json()["label"]) + + def test_a_file_that_is_not_a_picture_is_not_an_image(self): + refused = self.client.post("/api/images", { + "file": SimpleUploadedFile("notes.txt", b"not a picture", content_type="text/plain")}) + self.assertEqual(400, refused.status_code) + + def test_a_file_without_audio_is_not_a_sound(self): + refused = self.client.post("/api/sounds", { + "file": SimpleUploadedFile("notes.txt", b"not audio", content_type="text/plain")}) + self.assertEqual(400, refused.status_code) + def test_uploaded_video_extracts_to_reopenable_footage(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "four-frames.mp4" @@ -700,6 +908,37 @@ class UploadTests(TestCase): self.assertEqual("image/jpeg", still["Content-Type"]) self.assertEqual(200, self.client.get(footage["audio"]).status_code) + def test_re_uploading_a_source_re_reads_its_facts(self): + # A SOURCE ROW HOLDS A READING, NOT A DECISION. The facts are a pure + # function of bytes that are themselves this row's identity, so the row + # cannot be the place a reading goes to be preserved: `probe` got better + # at phone footage — it stopped believing a declared 120 over timestamps + # 1/30s apart — and a stored reading that nothing can replace would have + # left every already-uploaded source resampling to four times the frames + # with no way to correct it short of deleting the row. + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "four-frames.mp4" + subprocess.run([ + "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "color=c=red:s=64x48:r=4:d=1", + "-c:v", "mpeg4", str(path), + ], check=True, capture_output=True) + payload = path.read_bytes() + + first = self.client.post("/api/sources", { + "file": SimpleUploadedFile("four-frames.mp4", payload, content_type="video/mp4")}) + self.assertEqual(201, first.status_code, first.content) + self.assertEqual(4.0, first.json()["probe"]["fps"]) + + better = dict(first.json()["probe"], fps=12.0, rate="12/1", measured_fps=12.0) + with patch("clips.extraction.probe", return_value=better): + again = self.client.post("/api/sources", { + "file": SimpleUploadedFile("same.mp4", payload, content_type="video/mp4")}) + self.assertEqual(200, again.status_code, again.content) + self.assertFalse(again.json()["created"], "the same bytes are the same source") + self.assertEqual("12/1", again.json()["probe"]["rate"]) + self.assertEqual(12.0, Source.objects.get(id=first.json()["id"]).probe["fps"]) + def test_the_proxy_is_re_encoded_rather_than_the_upload_re_served(self): # The footage's identity is the proxy's digest, and the proxy is produced # by one ffmpeg invocation whatever the upload was. If the upload were @@ -747,6 +986,73 @@ class UploadTests(TestCase): self.assertTrue(facts["vfr"], "the disagreement is still recorded, just not fatal") self.assertTrue(facts["has_audio"]) + def test_a_declared_rate_the_timestamps_do_not_bear_out_is_not_resampled_to(self): + # THE FOUR-TIMES. An iPhone container declares `r_frame_rate` 120 over a + # stream whose frames are 1/30s apart, and taking the declaration at its + # word turned an 11-second clip into 1293 proxy frames instead of 323: + # four times the encode, four times the tracing stills, four times the + # blobs and the rows, for 970 frames that are copies of their neighbours. + # The timestamps are the evidence and they say 30. + streams = json.dumps({"streams": [ + {"codec_type": "video", "r_frame_rate": "120/1", + "avg_frame_rate": "96900/3233", "nb_frames": "323", + "width": 1920, "height": 1440}, + {"codec_type": "audio"}], + "format": {"duration": "10.775"}}) + # IN DECODE ORDER, which is how an HEVC stream really arrives — the first + # packets of the fixture this was found on come out 0, 0.133, 0.067, + # 0.033. Differencing that order unsorted measures the reordering delay + # and not the rate, so the fixture keeps the hazard in it. + shuffled = [0, 4, 2, 1, 3, 8, 6, 5, 7, 12, 10, 9, 11] + packets = json.dumps({"packets": [{"pts_time": f"{i / 30:.6f}"} for i in shuffled]}) + with patch("clips.extraction._command", side_effect=[streams, packets]): + facts = extraction.probe(Path("phone.mov")) + self.assertEqual("96900/3233", facts["rate"], "resampled to the declared 120") + self.assertAlmostEqual(30.0, facts["measured_fps"], places=2) + + def test_a_genuine_high_rate_capture_is_still_taken_at_its_own_rate(self): + # The other half of the same decision, and the one that would be easy to + # break: a real 120fps capture must not be dragged down to anything. Its + # declaration and its timestamps agree, so the declaration — the exact + # rational the stream was authored at — is what is used. + streams = json.dumps({"streams": [ + {"codec_type": "video", "r_frame_rate": "120/1", "avg_frame_rate": "120/1", + "width": 640, "height": 480}], + "format": {"duration": "2"}}) + packets = json.dumps({"packets": [{"pts_time": f"{i / 120:.6f}"} for i in range(13)]}) + with patch("clips.extraction._command", side_effect=[streams, packets]): + facts = extraction.probe(Path("slowmo.mov")) + self.assertEqual("120/1", facts["rate"]) + + def test_too_few_timestamps_to_measure_leaves_the_declaration_standing(self): + # A clip with nine-ish frames cannot outvote one odd timestamp, so the + # measurement declines to have an opinion and the nominal rate — the one + # that drops no distinct frame — is used exactly as it was before. + streams = json.dumps({"streams": [ + {"codec_type": "video", "r_frame_rate": "30/1", "avg_frame_rate": "24/1", + "width": 640, "height": 480}], + "format": {"duration": "0.1"}}) + packets = json.dumps({"packets": [{"pts_time": f"{i / 30:.6f}"} for i in range(3)]}) + with patch("clips.extraction._command", side_effect=[streams, packets]): + facts = extraction.probe(Path("tiny.mov")) + self.assertEqual("30/1", facts["rate"]) + self.assertIsNone(facts["measured_fps"]) + + def test_a_rate_measurement_that_fails_outright_cannot_refuse_an_upload(self): + # The measurement is an optimisation. If ffprobe cannot read the packets + # of a file whose streams it just read happily, the upload still has to be + # accepted on its metadata — an optimisation that can reject work is worse + # than no optimisation. + streams = json.dumps({"streams": [ + {"codec_type": "video", "r_frame_rate": "25/1", "avg_frame_rate": "25/1", + "width": 640, "height": 480}], + "format": {"duration": "4"}}) + with patch("clips.extraction._command", + side_effect=[streams, ValueError("ffprobe fell over")]): + facts = extraction.probe(Path("awkward.mov")) + self.assertEqual("25/1", facts["rate"]) + self.assertIsNone(facts["measured_fps"]) + def test_the_proxy_rate_is_exact_rather_than_a_rounded_float(self): # 30000/1001 is not a float. Handing ffmpeg's -r a rounded one is how a # long take drifts out of sync with its own audio. @@ -814,3 +1120,99 @@ class UploadTests(TestCase): digest="e" * 64, fps=12, frames=3, width=8, height=6, audio=blob) manifest = self.client.get(f"/api/footage/{footage.id}").json() self.assertIsNone(manifest["video"]) + + +@override_settings(BLOB_ROOT=BLOB_DIR) +class OwnershipTests(TestCase): + """Anyone with the link reads; the owner and the editors write.""" + + def setUp(self): + from django.contrib.auth import get_user_model + User = get_user_model() + self.ann = User.objects.create_user("ann", password="password1") + self.bob = User.objects.create_user("bob", password="password1") + self.project = Project.objects.create(name="ann's", owner=self.ann) + + def write(self): + return self.client.put(f"/api/projects/{self.project.id}", + data=json.dumps({"name": "renamed", "clips": []}), + content_type="application/json") + + def test_anyone_with_the_link_can_read_and_nobody_else_can_write(self): + loaded = self.client.get(f"/api/projects/{self.project.id}").json() + self.assertEqual(("ann", False), (loaded["owner"], loaded["can_edit"])) + self.assertEqual(403, self.write().status_code) + self.client.login(username="bob", password="password1") + self.assertEqual(403, self.write().status_code) + + def test_the_owner_names_an_editor_who_can_then_write(self): + self.client.login(username="bob", password="password1") + self.assertEqual(403, self.client.post( + f"/api/projects/{self.project.id}/editors", data=json.dumps({"username": "bob"}), + content_type="application/json").status_code) + self.client.login(username="ann", password="password1") + self.assertEqual(200, self.write().status_code) + self.assertEqual(["bob"], self.client.post( + f"/api/projects/{self.project.id}/editors", data=json.dumps({"username": "bob"}), + content_type="application/json").json()["editors"]) + self.client.login(username="bob", password="password1") + self.assertTrue(self.client.get(f"/api/projects/{self.project.id}").json()["can_edit"]) + self.assertEqual(200, self.write().status_code) + self.client.login(username="ann", password="password1") + self.client.delete(f"/api/projects/{self.project.id}/editors/bob") + self.client.login(username="bob", password="password1") + self.assertEqual(403, self.write().status_code) + + def test_a_project_is_made_by_somebody_signed_in_and_is_theirs(self): + self.assertEqual(403, self.client.post("/api/projects", data=json.dumps({"name": "x"}), + content_type="application/json").status_code) + self.client.post("/api/signup", data=json.dumps( + {"username": "cat", "password": "password1"}), content_type="application/json") + self.assertEqual("cat", self.client.get("/api/me").json()["username"]) + mine = self.client.post("/api/projects", data=json.dumps({"name": "y"}), + content_type="application/json").json() + self.assertEqual(("cat", True), (mine["owner"], mine["can_edit"])) + listed = {p["name"] for p in self.client.get("/api/projects").json()["projects"]} + self.assertEqual({"y"}, listed) + self.client.logout() + self.assertEqual([], self.client.get("/api/projects").json()["projects"]) + + def test_a_project_has_an_address(self): + response = self.client.get(f"/p/{self.project.id}") + self.assertEqual(200, response.status_code) + # The slug is the name, for people; the id is what finds it. + self.assertEqual(200, self.client.get(f"/p/{self.project.id}/anything-at-all").status_code) + self.assertContains(response, 'id="app"') + + +class SocketTests(TestCase): + """A committed write reaches everyone in the room; presence is stamped.""" + + def test_a_save_is_broadcast_to_the_room(self): + from asgiref.sync import async_to_sync, sync_to_async + from channels.testing import WebsocketCommunicator + from clips.consumers import broadcast + from server.asgi import application + + from django.contrib.auth import get_user_model + project = Project.objects.create( + name="shared", owner=get_user_model().objects.create_user("host")) + + async def scenario(): + peer = WebsocketCommunicator(application, f"/ws/projects/{project.id}", + headers=[(b"origin", b"http://localhost")]) + connected, _ = await peer.connect() + self.assertTrue(connected) + self.assertEqual("welcome", (await peer.receive_json_from())["kind"]) + self.assertEqual([], (await peer.receive_json_from())["peers"]) + self.assertEqual("join", (await peer.receive_json_from())["kind"]) + # The server stamps who sent it; a claimed name is overwritten. + await peer.send_json_to({"kind": "state", "frame": 12, "user": "forged"}) + state = await peer.receive_json_from() + self.assertEqual((12, None), (state["frame"], state["user"])) + await sync_to_async(broadcast)(project.id, {"seq": 1, "clips": []}) + delta = await peer.receive_json_from() + self.assertEqual(("delta", 1), (delta["kind"], delta["seq"])) + await peer.disconnect() + + async_to_sync(scenario)() diff --git a/clips/urls.py b/clips/urls.py index 4b4042f..ab52fb7 100644 --- a/clips/urls.py +++ b/clips/urls.py @@ -16,16 +16,28 @@ from django.urls import path from . import views urlpatterns = [ + path("me", views.me), + path("login", views.login), + path("signup", views.signup), + path("logout", views.logout), path("detector", views.detector), path("sources", views.sources), + path("sounds", views.sounds), + path("sounds/", views.sound_detail), + path("images", views.images), + path("images/", views.image_detail), path("extractions", views.extractions), path("extractions/", views.extraction_detail), path("footage", views.footage_list), path("footage/", views.footage_detail), path("projects", views.projects), + path("symbols", views.symbols), path("projects/", views.project_detail), path("projects//leaves/", views.leaf_detail), path("projects//revisions", views.revisions), + path("projects//revisions//restore", views.restore), + path("projects//editors", views.editors), + path("projects//editors/", views.editors), path("analyses", views.analyses), path("analyses/", views.analysis_detail), path("blocks", views.blocks), diff --git a/clips/views.py b/clips/views.py index ce8ce79..597a286 100644 --- a/clips/views.py +++ b/clips/views.py @@ -32,14 +32,18 @@ from pathlib import Path from uuid import UUID from django.conf import settings +from django.contrib.auth import authenticate, get_user_model +from django.contrib.auth import login as auth_login, logout as auth_logout from django.core.exceptions import ValidationError from django.db import transaction +from django.db.models import Q from django.http import FileResponse, HttpResponse, JsonResponse from django.shortcuts import render from django.views.decorators.http import require_http_methods from . import blobs, extraction -from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Leaf, Project, Revision, Source +from .consumers import broadcast +from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Image, Leaf, Project, Revision, Sound, Source KEY_LENGTH = 71 # "sha256:" + 64 hex @@ -119,10 +123,27 @@ def _crop_blob(chunks): # the page -def page(request): +def _asset_version(relative): + """A static file's modification time, for its URL. + + The stylesheet and the bundle are served with `Last-Modified` and nothing + else, so a browser is free to keep a stale copy on heuristic freshness — and + new JavaScript over an old stylesheet renders a pane the stylesheet has never + heard of as bare elements. A version in the URL makes each edit a new URL.""" + for root in settings.STATICFILES_DIRS: + path = Path(root) / relative + if path.exists(): + return str(int(path.stat().st_mtime)) + return "0" + + +def page(request, project_id=None, slug=None): """The host page. This replaced `frontend/public/index.html` at step 9, and `:dev-http` in shadow-cljs.edn went away with it.""" - return render(request, "clips/index.html") + return render(request, "clips/index.html", { + "css_version": _asset_version("arthur/app.css"), + "js_version": _asset_version("arthur/js/main.js"), + }) # --------------------------------------------------------------------------- @@ -197,6 +218,17 @@ def sources(request): "media_type": upload.content_type or "video/mp4"}) row, created = Source.objects.get_or_create( blob=blob, defaults={"filename": Path(upload.name).name[:255], "probe": facts}) + if not created and row.probe != facts: + # THE FACTS ARE RE-READ, NOT REMEMBERED. They are a pure function of + # the bytes, and the bytes are this row's identity — so a + # disagreement means the server reads the file differently now from + # whenever it first saw it, and the fresh reading is the one to keep. + # Storing the first reading forever pins a source to a rate the code + # no longer believes in, and makes it unfixable without deleting the + # row: `extraction.probe` got better at phone footage and every + # already-uploaded source would have gone on being wrong. + row.probe = facts + row.save(update_fields=["probe"]) return JsonResponse({"id": str(row.id), "digest": digest, "filename": row.filename, "probe": row.probe, "created": created}, status=201 if created else 200) @@ -204,6 +236,112 @@ def sources(request): return JsonResponse({"error": str(exc)}, status=400) +def _sound_json(row): + return {"id": str(row.id), "label": row.label or row.filename, + "filename": row.filename, "duration": row.duration, + "audio": f"/blob/{row.blob_id}"} + + +def _relabel(request, row): + """PATCH one asset's display name. + + A LABEL IS THE ONLY FIELD EITHER ROW LETS A CLIENT WRITE, and the body is + read for that key alone. Footage is content-addressed and its frame count, + rate and digest are facts about the bytes; an endpoint that merged whatever + it was sent would let a rename quietly contradict them. Blank clears it, + which puts the row back to the name it was uploaded under rather than + leaving it nameless. + """ + data = _body(request) + if "label" not in data: + raise Bad("a rename needs a label") + label = str(data["label"] or "").strip()[:200] + if label != row.label: + row.label = label + row.save(update_fields=["label"]) + return row + + +@require_http_methods(["GET", "POST"]) +def sounds(request): + if request.method == "GET": + return JsonResponse({"sounds": [_sound_json(row) + for row in Sound.objects.order_by("-created")]}) + upload = request.FILES.get("file") + if upload is None: + return JsonResponse({"error": "upload a sound as the file field"}, status=400) + try: + digest, size = blobs.write_stream(upload.chunks()) + duration = extraction.probe_audio(blobs.path_for(digest)) + blob, _ = Blob.objects.get_or_create( + digest=digest, defaults={"size": size, + "media_type": upload.content_type or "audio/mpeg"}) + row, created = Sound.objects.get_or_create( + blob=blob, defaults={"filename": Path(upload.name).name[:255], + "duration": duration}) + return JsonResponse(_sound_json(row), status=201 if created else 200) + except (ValueError, OSError) as exc: + return JsonResponse({"error": str(exc)}, status=400) + + +@require_http_methods(["GET", "PATCH"]) +def sound_detail(request, sound_id): + try: + row = Sound.objects.get(id=sound_id) + except Sound.DoesNotExist: + return JsonResponse({"error": "no such sound"}, status=404) + try: + if request.method == "PATCH": + row = _relabel(request, row) + except Bad as exc: + return _error(exc) + return JsonResponse(_sound_json(row)) + + +def _image_json(row): + return {"id": str(row.id), "label": row.label or row.filename, + "filename": row.filename, "width": row.width, "height": row.height, + "digest": row.blob_id, "url": f"/blob/{row.blob_id}"} + + +@require_http_methods(["GET", "POST"]) +def images(request): + """Stills to trace over. The document names one by its blob digest, so an + image is the same picture in every project that uses it.""" + if request.method == "GET": + return JsonResponse({"images": [_image_json(row) + for row in Image.objects.order_by("-created")]}) + upload = request.FILES.get("file") + if upload is None: + return JsonResponse({"error": "upload an image as the file field"}, status=400) + try: + digest, size = blobs.write_stream(upload.chunks()) + width, height = extraction.probe_image(blobs.path_for(digest)) + blob, _ = Blob.objects.get_or_create( + digest=digest, defaults={"size": size, + "media_type": upload.content_type or "image/png"}) + row, created = Image.objects.get_or_create( + blob=blob, defaults={"filename": Path(upload.name).name[:255], + "width": width, "height": height}) + return JsonResponse(_image_json(row), status=201 if created else 200) + except (ValueError, OSError) as exc: + return JsonResponse({"error": str(exc)}, status=400) + + +@require_http_methods(["GET", "PATCH"]) +def image_detail(request, image_id): + try: + row = Image.objects.get(id=image_id) + except Image.DoesNotExist: + return JsonResponse({"error": "no such image"}, status=404) + try: + if request.method == "PATCH": + row = _relabel(request, row) + except Bad as exc: + return _error(exc) + return JsonResponse(_image_json(row)) + + def _extraction_json(row): return {"key": row.key, "source": str(row.source_id), "state": row.state, "progress": row.progress, "error": row.error, @@ -273,12 +411,70 @@ def footage_list(request): ) +_SYMBOL_LEAF = re.compile(r"^clip/([^/]+)/symbol/([^/]+)$") +_PALETTE_LEAF = re.compile(r"^clip/([^/]+)/palette/([^/]+)$") + + +def _transit_fields(value, *keys): + """Top-level fields of a transit map leaf, by keyword name. A leaf's own + facts are a small flat map, so no key repeats and transit's cache never + stands in for one; anything else reads as absent.""" + if not (isinstance(value, list) and value[:1] == ["^ "]): + return {} + pairs = dict(zip(value[1::2], value[2::2])) + return {k: pairs.get(f"~:{k}") for k in keys} + + @require_http_methods(["GET"]) +def symbols(request): + """Every symbol in every saved project, for the pool's all-assets folder. + + Read off the leaf PATHS rather than by loading documents: a symbol's own leaf + is `clip//symbol/`, so listing them is one query and no decoding + beyond the name and length its value carries.""" + rows = [] + for leaf in Leaf.objects.filter(path__contains="/symbol/").select_related("project"): + m = _SYMBOL_LEAF.match(leaf.path) + if not m: + continue + fields = _transit_fields(leaf.value, "name", "frames") + rows.append({ + "project": str(leaf.project_id), + "project_name": leaf.project.name, + "cid": m.group(1), + "symbol": m.group(2), + "name": fields.get("name") or m.group(2).replace("~", "/"), + "frames": fields.get("frames"), + }) + rows.sort(key=lambda r: (r["project_name"], r["project"], r["name"])) + palettes = [] + for leaf in Leaf.objects.filter(path__contains="/palette/").select_related("project"): + m = _PALETTE_LEAF.match(leaf.path) + if not m: + continue + fields = _transit_fields(leaf.value, "name") + palettes.append({ + "project": str(leaf.project_id), + "project_name": leaf.project.name, + "cid": m.group(1), + "palette": m.group(2), + "name": fields.get("name") or m.group(2).replace("~", "/"), + }) + palettes.sort(key=lambda r: (r["project_name"], r["project"], r["name"])) + return JsonResponse({"symbols": rows, "palettes": palettes}) + + +@require_http_methods(["GET", "PATCH"]) def footage_detail(request, footage_id): try: footage = Footage.objects.select_related("audio", "video", "stream").get(id=footage_id) except Footage.DoesNotExist: return JsonResponse({"error": "no such footage"}, status=404) + try: + if request.method == "PATCH": + footage = _relabel(request, footage) + except Bad as exc: + return _error(exc) return JsonResponse(_footage_json(footage)) @@ -564,7 +760,11 @@ def block_detail(request, key): # tier 1: projects, clips, leaves -def _project_json(project: Project): +def _who(user): + return {"username": user.get_username() if user.is_authenticated else None} + + +def _project_json(project: Project, user): leaves = list(project.leaves.all()) clips = [] for clip in project.clips.all(): @@ -573,8 +773,6 @@ def _project_json(project: Project): { "cid": clip.cid, "name": clip.name, - "footage": str(clip.footage_id) if clip.footage_id else None, - "analysis": clip.analysis_id, "blocks": sorted(clip.blocks.values_list("key", flat=True)), "leaves": {leaf.path: leaf.value for leaf in leaves if leaf.path.startswith(prefix)}, } @@ -585,27 +783,103 @@ def _project_json(project: Project): "schema_version": project.schema_version, "seq": project.seq, "palette": project.palette, + "owner": project.owner.get_username(), + "editors": sorted(project.editors.values_list("username", flat=True)), + "can_edit": project.can_edit(user), "clips": clips, } +def _project(project_id): + try: + return Project.objects.select_related("owner").get(id=project_id) + except Project.DoesNotExist: + raise Bad("no such project", status=404) + + +def _writable(request, project_id): + project = _project(project_id) + if not project.can_edit(request.user): + raise Bad("only the owner and the editors can change this project; " + "save a copy instead", status=403) + return project + + +# --------------------------------------------------------------------------- +# who you are +# +# Django's session cookie, and the page's CSRF cookie on every write. Nothing +# here that a signed-in admin does not already have; the API gains a way in that +# is not the admin's login page. + + +@require_http_methods(["GET"]) +def me(request): + return JsonResponse(_who(request.user)) + + +@require_http_methods(["POST"]) +def login(request): + data = json.loads(request.body or b"{}") + user = authenticate(request, username=data.get("username"), password=data.get("password")) + if user is None: + return JsonResponse({"error": "wrong username or password"}, status=400) + auth_login(request, user) + return JsonResponse(_who(user)) + + +@require_http_methods(["POST"]) +def signup(request): + data = json.loads(request.body or b"{}") + username = (data.get("username") or "").strip() + password = data.get("password") or "" + if not username or len(password) < 8: + return JsonResponse({"error": "a username, and a password of 8 or more"}, status=400) + User = get_user_model() + if User.objects.filter(username__iexact=username).exists(): + return JsonResponse({"error": "that username is taken"}, status=409) + user = User.objects.create_user(username=username, password=password) + auth_login(request, user) + return JsonResponse(_who(user), status=201) + + +@require_http_methods(["POST"]) +def logout(request): + auth_logout(request) + return JsonResponse(_who(request.user)) + + +# --------------------------------------------------------------------------- +# tier 1: projects, clips, leaves + + @require_http_methods(["GET", "POST"]) def projects(request): + """GET lists what you own and are an editor of — nothing, signed out; POST + makes one, owned by you. Every project has an owner, so making one needs you + signed in.""" if request.method == "GET": + if not request.user.is_authenticated: + return JsonResponse({"projects": []}) + visible = Q(owner=request.user) | Q(editors=request.user) return JsonResponse( { "projects": [ {"id": str(p.id), "name": p.name, "schema_version": p.schema_version, "seq": p.seq, + "owner": p.owner.get_username(), "updated": p.updated.isoformat()} - for p in Project.objects.all()[:100] + for p in Project.objects.filter(visible).distinct() + .select_related("owner")[:100] ] } ) + if not request.user.is_authenticated: + return JsonResponse({"error": "sign in to make a project"}, status=403) try: data = _body(request) - project = Project.objects.create(name=data.get("name") or "untitled") - return JsonResponse(_project_json(project), status=201) + project = Project.objects.create(name=data.get("name") or "untitled", owner=request.user) + return JsonResponse(_project_json(project, request.user), status=201) except Bad as exc: return _error(exc) @@ -613,43 +887,74 @@ def projects(request): @require_http_methods(["GET", "PUT"]) def project_detail(request, project_id): try: - project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - return JsonResponse({"error": "no such project"}, status=404) - if request.method == "GET": - return JsonResponse(_project_json(project)) + if request.method == "GET": + return JsonResponse(_project_json(_project(project_id), request.user)) + project = _writable(request, project_id) + return _save(project, _body(request), request.user) + except Bad as exc: + return _error(exc) + + +@require_http_methods(["POST", "DELETE"]) +def editors(request, project_id, username=None): + """The owner names who else can write. POST {username} adds; DELETE + `editors/` removes.""" try: - return _save(project, _body(request)) + project = _project(project_id) + if not (request.user.is_authenticated and request.user.id == project.owner_id): + raise Bad("only the owner can change who edits", status=403) + if request.method == "POST": + username = _body(request).get("username") + user = get_user_model().objects.filter(username__iexact=username or "").first() + if user is None: + raise Bad(f"nobody is called {username!r}", status=404) + if request.method == "POST": + project.editors.add(user) + else: + project.editors.remove(user) + broadcast(project.id, {}, kind="access") + return JsonResponse({"editors": sorted(project.editors.values_list("username", flat=True))}) except Bad as exc: return _error(exc) @transaction.atomic -def _save(project: Project, data): - """A whole-document save: one clip's leaves replace that clip's leaves. +def _save(project: Project, data, user): + """A save: one clip's leaves, written. SCOPED BY CLIP, not by project. A payload that carries clip `a` does not - disturb clip `b`'s leaves, because a save is not the only way the document - changes — a single-leaf conditional write is — and a save that cleared - everything it did not mention would be a save that undoes a collaborator. + disturb clip `b`'s leaves. + + Two shapes. Without `base`, a clip's leaves REPLACE that clip's leaves — the + whole-document save. With `base`, the seq the client last caught up to, the + save is a PATCH: `leaves` are the ones it changed, `removed` the ones it + deleted, and nothing it did not mention is touched. A leaf it names that + somebody else changed after `base`, to something else, is a conflict, and the + whole save answers 409 with their values — last-writer-wins per leaf, with the + loser told rather than silently clobbered. docs/architecture.md, "Make the + merge unit small instead of clever". A leaf whose value is unchanged keeps its VERSION. That is what makes the entity tag mean something: a save of a document where one channel moved invalidates one leaf's etag, not all four hundred. """ + base = data.get("base") + seq = project.bump() if data.get("name"): project.name = data["name"] if data.get("palette"): project.palette = data["palette"] + project.save(update_fields=["name", "palette"]) - written, removed, unchanged = [], [], [] + written, removed, unchanged, conflicts, deltas = [], [], [], {}, [] for spec in data.get("clips") or []: cid = spec.get("cid") if not cid: raise Bad("every clip in a save names its cid") leaves = spec.get("leaves") or {} + gone = spec.get("removed") or [] if base is not None else [] prefix = f"clip/{cid}/" - for path in leaves: + for path in [*leaves, *gone]: if not path.startswith(prefix): raise Bad( f"leaf {path!r} is not addressed to clip {cid!r}", @@ -657,7 +962,19 @@ def _save(project: Project, data): ) keys = spec.get("blocks") or [] - have = set(Block.objects.filter(key__in=keys).values_list("key", flat=True)) + analyses = spec.get("analyses") or [] + if (not isinstance(analyses, list) + or not all(isinstance(key, str) for key in analyses) + or len(analyses) != len(set(analyses))): + raise Bad("a clip's analyses must be a list of distinct analysis ids") + registered = set(Analysis.objects.filter(key__in=analyses) + .values_list("key", flat=True)) + if unknown := [key for key in analyses if key not in registered]: + raise Bad("this clip names analyses the server does not know; register them first", + status=409, missing=unknown) + + block_rows = list(Block.objects.filter(key__in=keys)) + have = {block.key for block in block_rows} if missing := [k for k in keys if k not in have]: # Referential integrity across the tiers, enforced where it can be: # a document that names blocks the server does not hold would load @@ -667,39 +984,60 @@ def _save(project: Project, data): "before saving the document that points at them", status=409, missing=missing, ) + if undeclared := sorted({block.analysis_id for block in block_rows} - registered): + raise Bad("every block in a clip must name one of that clip's analyses", + status=409, missing=undeclared) + + existing = {leaf.path: leaf for leaf in project.leaves.filter(path__startswith=prefix)} + if base is not None: + for path in [*leaves, *gone]: + theirs = existing.get(path) + if theirs and theirs.seq > base and ( + path not in leaves or theirs.value != leaves[path]): + conflicts[path] = theirs.value + if conflicts: + continue - analysis = Analysis.objects.filter(key=spec.get("analysis")).first() - footage = None - if spec.get("footage"): - footage = Footage.objects.filter(id=spec["footage"]).first() clip, _ = Clip.objects.update_or_create( project=project, cid=cid, - defaults={"name": spec.get("name") or "", "analysis": analysis, "footage": footage}, + defaults={"name": spec.get("name") or ""}, ) - clip.blocks.set(Block.objects.filter(key__in=keys)) + blocks = block_rows + if base is None: + clip.blocks.set(blocks) + gone = [path for path in existing if path not in leaves] + else: + clip.blocks.add(*blocks) - existing = {leaf.path: leaf for leaf in project.leaves.filter(path__startswith=prefix)} + changed = {} for path, value in leaves.items(): leaf = existing.get(path) if leaf is None: - Leaf.objects.create(project=project, path=path, value=value) - written.append(path) + Leaf.objects.create(project=project, path=path, value=value, seq=seq) elif leaf.value != value: - leaf.value = value + leaf.value, leaf.seq = value, seq leaf.version += 1 - leaf.save(update_fields=["value", "version", "updated"]) - written.append(path) + leaf.save(update_fields=["value", "version", "seq", "updated"]) else: unchanged.append(path) - for path, leaf in existing.items(): - if path not in leaves: - leaf.delete() - removed.append(path) + continue + changed[path] = value + dropped = [path for path in gone if path in existing] + project.leaves.filter(path__in=dropped).delete() + written += changed + removed += dropped + deltas.append({"cid": cid, "leaves": changed, "removed": dropped, "blocks": keys}) - seq = project.seq + 1 - project.seq = seq - project.save() + if conflicts: + raise Bad( + "somebody else changed these since you last caught up", + status=409, seq=seq - 1, conflicts=conflicts, + ) + by = user.get_username() if user.is_authenticated else None + transaction.on_commit(lambda: broadcast(project.id, { + "seq": seq, "by": by, "name": project.name, "clips": deltas, + })) return JsonResponse( { "id": str(project.id), @@ -722,9 +1060,9 @@ def leaf_detail(request, project_id, leaf_path): for a painted cel that is the class of bug that ends trust in a tool. """ try: - project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - return JsonResponse({"error": "no such project"}, status=404) + project = _project(project_id) if request.method == "GET" else _writable(request, project_id) + except Bad as exc: + return _error(exc) leaf = project.leaves.filter(path=leaf_path).first() if request.method == "GET": @@ -742,35 +1080,45 @@ def leaf_detail(request, project_id, leaf_path): return _error(Bad("a leaf write carries a value")) match = request.headers.get("If-Match") - if leaf is None: - # ANY `If-Match` on a leaf that does not exist is a failed precondition, - # `*` included: RFC 7232 gives `*` the meaning "the resource must already - # exist", which is exactly the write a client makes when it believes it is - # editing something. Creating it instead would turn "somebody deleted this - # node" into a silent resurrection. - if match: - return JsonResponse( - {"error": "no such leaf", "path": leaf_path}, status=409 - ) - leaf = Leaf.objects.create(project=project, path=leaf_path, value=data["value"]) - else: - if match and match not in ("*", leaf.etag): - response = JsonResponse( - { - "error": "stale write", - "path": leaf.path, - "version": leaf.version, - "value": leaf.value, - }, - status=409, - ) - response["ETag"] = leaf.etag - return response - leaf.value = data["value"] - leaf.version += 1 - leaf.save(update_fields=["value", "version", "updated"]) + with transaction.atomic(): + seq = project.bump() + leaf = project.leaves.filter(path=leaf_path).first() + if leaf is None: + # ANY `If-Match` on a leaf that does not exist is a failed precondition, + # `*` included: RFC 7232 gives `*` the meaning "the resource must already + # exist", which is exactly the write a client makes when it believes it + # is editing something. Creating it instead would turn "somebody deleted + # this node" into a silent resurrection. + if match: + transaction.set_rollback(True) + return JsonResponse( + {"error": "no such leaf", "path": leaf_path}, status=409 + ) + leaf = Leaf.objects.create(project=project, path=leaf_path, value=data["value"], seq=seq) + else: + if match and match not in ("*", leaf.etag): + transaction.set_rollback(True) + response = JsonResponse( + { + "error": "stale write", + "path": leaf.path, + "version": leaf.version, + "value": leaf.value, + }, + status=409, + ) + response["ETag"] = leaf.etag + return response + leaf.value, leaf.seq = data["value"], seq + leaf.version += 1 + leaf.save(update_fields=["value", "version", "seq", "updated"]) + cid = leaf_path.split("/")[1] if leaf_path.startswith("clip/") else None + by = request.user.get_username() if request.user.is_authenticated else None + transaction.on_commit(lambda: broadcast(project.id, { + "seq": seq, "by": by, "name": project.name, + "clips": [{"cid": cid, "leaves": {leaf.path: leaf.value}, "removed": [], "blocks": []}], + })) - seq = project.bump() response = JsonResponse({"path": leaf.path, "version": leaf.version, "seq": seq}) response["ETag"] = leaf.etag return response @@ -778,16 +1126,17 @@ def leaf_detail(request, project_id, leaf_path): @require_http_methods(["GET", "POST"]) def revisions(request, project_id): - """Mark a version: one snapshot of the authored layer, with a summary.""" + """Named snapshots: GET lists them, POST {summary} takes one of the document + as it is now.""" try: - project = Project.objects.get(id=project_id) - except Project.DoesNotExist: - return JsonResponse({"error": "no such project"}, status=404) + project = _project(project_id) if request.method == "GET" else _writable(request, project_id) + except Bad as exc: + return _error(exc) if request.method == "GET": return JsonResponse( { "revisions": [ - {"seq": r.seq, "author": r.author, "summary": r.summary, + {"id": r.id, "seq": r.seq, "author": r.author, "summary": r.summary, "created": r.created.isoformat(), "leaves": len(r.document)} for r in project.revisions.all()[:100] ] @@ -797,8 +1146,57 @@ def revisions(request, project_id): revision = Revision.objects.create( project=project, seq=project.seq, - author=data.get("author") or "", - summary=data.get("summary") or "", + author=request.user.get_username(), + summary=(data.get("summary") or "").strip()[:500], document={leaf.path: leaf.value for leaf in project.leaves.all()}, + blocks={clip.cid: sorted(clip.blocks.values_list("key", flat=True)) + for clip in project.clips.all()}, ) - return JsonResponse({"seq": revision.seq, "leaves": len(revision.document)}, status=201) + return JsonResponse({"id": revision.id, "seq": revision.seq, + "leaves": len(revision.document)}, status=201) + + +@require_http_methods(["POST"]) +def restore(request, project_id, revision_id): + """Put a snapshot back: an ordinary write of every leaf that differs, so + everybody in the room receives it the way they receive any other.""" + try: + project = _writable(request, project_id) + revision = project.revisions.filter(id=revision_id).first() + if revision is None: + raise Bad("no such snapshot", status=404) + except Bad as exc: + return _error(exc) + with transaction.atomic(): + seq = project.bump() + existing = {leaf.path: leaf for leaf in project.leaves.all()} + deltas = {} + def delta(path): + cid = path.split("/")[1] + return deltas.setdefault(cid, {"cid": cid, "leaves": {}, "removed": [], + "blocks": revision.blocks.get(cid, [])}) + for path, value in revision.document.items(): + leaf = existing.get(path) + if leaf is None: + Leaf.objects.create(project=project, path=path, value=value, seq=seq) + elif leaf.value != value: + leaf.value, leaf.seq = value, seq + leaf.version += 1 + leaf.save(update_fields=["value", "version", "seq", "updated"]) + else: + continue + delta(path)["leaves"][path] = value + gone = [path for path in existing if path not in revision.document] + project.leaves.filter(path__in=gone).delete() + for path in gone: + delta(path)["removed"].append(path) + for cid, keys in revision.blocks.items(): + clip = project.clips.filter(cid=cid).first() + if clip: + clip.blocks.add(*Block.objects.filter(key__in=keys)) + by = request.user.get_username() + transaction.on_commit(lambda: broadcast(project.id, { + "seq": seq, "by": by, "name": project.name, "clips": list(deltas.values()), + })) + return JsonResponse({"seq": seq, "changed": sum(len(d["leaves"]) + len(d["removed"]) + for d in deltas.values())}) diff --git a/docs/animation-model.md b/docs/animation-model.md index 4c1a2bc..2d1d2b4 100644 --- a/docs/animation-model.md +++ b/docs/animation-model.md @@ -1,5 +1,10 @@ # arthur — the animation model +The revised target for lanes, occurrences, source playback, shared editing, and +multi-view UX is [The Lane Model](lane-model.md). It supersedes conflicting +proposals below. Backward compatibility is not required; this document still +contains descriptions of earlier shapes and planned features. + The data that describes a moving picture: what the primitives are, how they nest, how they change over time, and how rotoscoped and hand-authored work end up being the same thing with one flag between them. @@ -79,6 +84,20 @@ this way. over which the node exists at all. Distinct from a `[:vis]` channel, which blinks an existing node on and off. +**Every node has the same two maps into its parent**, whatever kind it is: + +- **space** — the matrix its transform channels compose to, times a `:pinv` if + it has been moved in from elsewhere; +- **time** — `local = rate · (parent − at)`, from `:time :at` and `:rate`, + identity when absent. `:span` and every key are in the node's **own** frames. + +A move keeps a node's world maps and re-expresses them under its new parent: +the matrix becomes a `:pinv`, the time becomes a new `:at` and `:rate`, and its +channels, keys and span are not touched. Both maps are affine, so any depth of +nesting is one map and every move is one inverse. `node/time-of`, +`node/then-time` and `node/placed-span` are the time half; `clip/move-node` and +`clip/group` are the move. + ### Subjects and tracked features Scene nodes describe drawings, not tracking identity. A scene may also carry a @@ -153,7 +172,6 @@ Every animatable property is a channel, and channels are addressed **by path**: [:xform :rot] {:animated? false :value 0.0} [:xform :scale] {:animated? false :value [1.0 1.0]} [:xform :skew] {:animated? false :value [0.0 0.0]} - [:xform :anchor]{:animated? false :value [0.0 0.0]} [:geom :pts] {:animated? true :interp :hold :dense {...} :generated {...}} [:style :color] {:animated? false :value :skin-dark} [:vis] {:animated? true :interp :hold :keys {0 true, 37 false}}} @@ -216,10 +234,30 @@ combines: ```clojure {:animated? true :interp :hold :dense {...} :generated {...} - :over [{:blend :offset :keys {88 [2 0], 96 [0 0]}} - {:blend :replace :keys {104 [[3 7] [4 7] …]}}]} + :over [{:id :nudge :support [88 98] :op :offset + :values {:animated? true :interp :linear :keys {88 [2 0], 96 [0 0]}}} + {:id :redraw :support [104 105] :op :replace + :values {:animated? false :value [[3 7] [4 7] …]}}]} ``` +A LAYER'S VALUES ARE A CHANNEL, which is what keeps a constant adjustment, a +ramp and a return motion from being three mechanisms: a framed one says the same +thing on every frame it covers, a keyed one moves. They read through `value-at` +and `cursor` like any channel, one reading head each, so the specification and +the playback path share their blending and differ only in how they read — and a +layer's values may not carry layers of their own, which the stack already +orders. + +`:support` is half-open and explicit, `[in out)`. Outside it a layer is inactive +and the base evaluates exactly as it did before, which is the difference between +a bounded correction and inserting boundary keys — the latter alters the +neighbouring segments. And a layer has NO TIME SPACE of its own: its support and +its values' keys are in the frames the base channel's keys are in, the node's +own. A correction on a lane is therefore in lane frames and reaches across the +drawings exposed under it; one on a single occurrence is in that occurrence's +frames and travels with it when the exposure moves. Ownership had already +answered the question, so there is no field to disagree with. + - **`:offset`** adds a delta to the base. "Nudge the mouth two pixels right for ten frames" survives a re-freeze at different parameters, because it was never a position — it was a correction. @@ -229,6 +267,22 @@ This is what `docs/design.md` means by an override layer, and it is why re-freezing is safe: the base is regenerated, the layers are untouched. It is Blender's NLA blending and AE's effect stack at one property. +WHEN THE BASE OUTGROWS A CORRECTION it is a CONFLICT, which is neither a dropped +layer nor an applied one. Turning `:verts` gives the mouth a different number of +points, and an `:offset` is a row of components that has to match: so the +regeneration records `:conflict` on the layer, the layer stays in the document, +the picture is the base meanwhile, and `clip/conflicts` is the list a view +offers to resolve. Deliberately not `problems` — the document loads and saves +fine, it just contains a decision nobody has made yet. A later regeneration +that restores the shape clears the mark. Only `:offset` can conflict; `:replace` +states a whole value and has nothing to agree with. + +A correction is NOT a hand placement. `regenerate-head` leaves the head's +authored channels alone once somebody has placed it by hand, and it compares the +channels WITHOUT their layers to decide: otherwise the first correction anyone +made would stop the head following re-measurement forever, which is the opposite +of what a layer is for. + Layers are what "set it by hand" means for anything measured, and the measured channel does not need to know. A hand-set gaze is an `:over` on `[:xform :pos]` of the iris; a hand-set mouth shape is an `:over` on @@ -269,7 +323,7 @@ to change to allow it. ## Transform: decomposed, never a matrix ```clojure -{:pos [x y] :rot θ :scale [sx sy] :skew [kx ky] :anchor [ax ay]} +{:pos [x y] :rot θ :scale [sx sy] :skew [kx ky]} ``` Stored decomposed for two reasons. Each component has to be independently @@ -279,18 +333,141 @@ entries is meaningless — a rotation tweened through its matrix shears on the w Composition, per node: ``` -local = T(pos) · T(anchor) · R(rot) · K(skew) · S(scale) · T(-anchor) +local = T(pos) · T(piv) · R(rot) · K(skew) · S(scale) · T(-piv) world = world(parent) · pinv · local ``` -`:anchor` is Flash's registration point and Blender's origin: rotation and scale -happen about it, and getting it wrong is why hand-placed parts swing rather than -turn. - `:pinv` is Blender's `parent_inverse`, captured at the moment of parenting so the child does not jump when it acquires a parent. Small, and its absence is the kind of thing that makes a parenting feature feel broken. +### A node has a `:pivot`, and a peg is still a peg + +Rotation and scale happen about the node's **pivot**, `[:xform :pivot]`, a point +in its own coordinates: + +``` +local = T(pos) · T(piv) · R(rot) · K(skew) · S(scale) · T(-piv) + = T(pos + piv - M·piv) · M +``` + +Toon Boom gives every layer and every peg a pivot, Flash gives every instance a +transformation point, After Effects calls it the anchor point. All three store +it, and the reason is one sentence: **a turn has to be a turn on every frame**, +and the only way to keep a point still through an interpolated angle is for the +angle to be composed about that point. + +This was deleted in schema 7 and restored in schema 8, and the argument for +deleting it was *not wrong*, which is why it is worth writing down. It was: + +``` +T(pos) · T(a) · R·K·S · T(-a) ≡ peg at pos+a carrying R·K·S, child at -a +``` + +to the last bit of the mantissa — `node-test` asserts it, still. `T(a)·M·T(-a)` +is `M` conjugated by a translation, which is "do `M` in a frame shifted by `a`", +and a **parent already is a shifted frame**. So an anchor was a peg written +inline, and a peg can be selected, keyed, shared between nodes and put above a +measured channel. Same expressive content, strictly more reach. + +**What that identity does not say is what a node turns about when nobody has +made a peg.** It is an equivalence between a pivot and a peg *that already +exists*; it is silent on the default, and the default is what a person meets. +With no pivot in the composition, a turn about any point that is not the node's +own origin has to be paid for by writing `pos` as well — `gesture/about` solves +for it: + +``` +q = M⁻¹(c − t) the material point under c +p' = c − M'·q +``` + +and that solution is an **arc** in the angle while `pos` interpolates along the +**chord**: + +| | pivot = origin | pivot ≠ origin | +| --- | --- | --- | +| one drag | right | right | +| between two keys | right | **wrong**, by the sagitta of the arc | + +A 360° turn is where that is unmissable: 0° and 360° are the only two frames +where a wrong pivot cannot be seen at all, so the keys look right and every +frame between them is wrong. + +**A drawing escaped it. A symbol instance could not.** `paint/centred` puts a +shape's origin on the middle of what it draws the moment it is drawn, so for a +drawing the pivot *is* the origin, `about` has nothing to do, and a keyed turn is +right between its keys. An instance's origin is its **symbol's**, and a symbol is +drawn on the stage, so its origin is the stage's top-left corner. Measured from +the document this was reported on: a symbol holding six drawn shapes had its +content centred at (99, 127), 161 px from its own origin, on a 320×200 stage. One +instance of it, keyed `rot` 0 → 60 and dragged round by hand, put the drawing at +(115, 116) on frame 0 and (241, 104) on frame 60 — both where they were put — and +at (−88, 121) on frame 30, a stage and a half from either. The answer on offer +was "make a peg first", for wanting to spin a drawing. + +So the pivot is back, with the default and the escape hatch spelled out, because +a stored pivot without either is the field that was deleted: + +| | what | where | +| --- | --- | --- | +| **the default, for a node nobody has pivoted** | the middle of what it draws — `pick/bounds-of`, the same call the selection box comes from, so the cross starts out on the middle of the box | `gesture/pivot` | +| **choosing it, invisibly** | the first turn or scale writes that middle down, in the same edit, with the `pos` that holds the picture still | `gesture/with-pivot` | +| **choosing it, by hand** | ⌃/⌘-drag the cross on the stage: the pivot goes under the pointer and nothing moves | `gesture/repivot`, `::ui/repivot` | +| **a placement** | `clip/place-symbol` stores the middle of what the symbol draws as the instance's pivot, so an instance turns about its drawing from the moment it is dropped | `clip/place-symbol` | +| **putting it back** | ⌖ beside the pivot row in the inspector: back to the middle of what the node draws *now*, moving nothing | `gesture/centred`, `::ui/centre-pivot` | + +**A pivot is a choice, and does not follow the drawing.** Once it is the node's +own, the derived middle is never consulted for it again. This is the half the +old stored anchor got right and the derived pivot got wrong: adding a shape +inside a symbol must not re-aim every keyed spin of every instance of it, and a +pivot that tracked the content did exactly that, silently, with nothing changing +on screen at the moment it happened. The cross is visible and draggable and ⌖ +puts it back, which is what the anchor was missing — it was never the storing +that was wrong. + +**A peg is an ordinary `:group` parent, `nest/peg`, with `:pinv` captured so +nothing moves when it appears.** It is no longer the answer to "this turns about +the wrong point", and it is still the answer to three things a node's own pivot +is not: + +| want | why the node's own pivot is not it | what the peg does | +| --- | --- | --- | +| a pivot **shared** between nodes — an arm and a forearm about one shoulder | two pivots that have to agree frame for frame are not one pivot | one transform, two children hanging off it | +| a **second** transform on one node — a drawing spinning about its middle while the limb swings about the shoulder | a node has one `rot` | stack them, as Harmony does | +| a hand transform over a **measured** one | `gesture/refusal` turns a drag on a measured channel away, because the next regenerate would discard it | the peg's channels are its own, so the hand transform composes outside the measurement, which stays regenerable | + +The pivot of a measured node is *not* in that table: `[:xform :pivot]` is +authored on every node alike, never dense and never regenerated, so a traced +mouth can be told to turn about its own middle without a peg and with nothing a +regenerate will throw away. That is the row that used to be impossible — writing +an anchor under a measured `M` moved the thing it was meant to leave alone, +because the old composition was `T(pos)·M·T(-a)` and `pos` was the measurement's. +The conjugated form has no such problem: `T(a)·M·T(-a)` is the identity at `a` +whatever `M` is. + +`demo/stage` places its seven faces on pegs, and that is now one way of writing +something a pivot says directly: the faces' `:scale` is **keyed** — they pulse — +and the source's middle has to stay on its authored centre throughout, which a +static `pos` cannot do since `T(pos)·S(k(f))` moves that point whenever `k` +changes. `T(center)·S(k(f))·T(-origin)` does, for every `k`, and so does one +instance with its pivot on the middle. The demo is left as it is, pegs and all: +it is a hand-authored scene that renders correctly and `instance-test` asserts +its structure, and a peg carrying a keyed scale is a perfectly good thing to +have written. + +`gesture/about` survives for the one gesture whose pivot belongs to no node: a +**multi-selection** scaling about the middle of its shared box, where every +member has to move to keep the arrangement. Nobody keys that. + +Schema 8 is the first version that **converts** rather than refusing. A schema-7 +node has no pivot, an absent pivot reads as `[0 0]`, and `T(pos)·T(0)·M·T(-0)` is +`T(pos)·M` to the bit — so every stored document composes to exactly the matrices +it did, dense tier-2 transforms included, and the migration only restamps the +version. What a converted document does not get is a pivot anybody chose; its +nodes still turn about their origins until the first turn writes one or the cross +is dragged. + **The similarity fit already produces a decomposition.** `fitSimilarity` returns `{s θ tx ty}`, which drops straight into `[:xform :scale]`, `[:xform :rot]` and `[:xform :pos]` with no conversion. The analysis output and the animation model @@ -379,7 +556,7 @@ selected head placement, so it aligns with the vectors drawn over it. :mouth :mouth-in :teeth :lid-r :lid-l :brow-r :brow-l … ``` -Changing anchor keys edits `:head` and never touches `:face`, so it cannot move +Changing anchor keys edits `:head` and never touches `:place`, so it cannot move something that was placed by hand. A group node is free, and keeping the authored and the measured transform apart is the whole reason the transform is decomposed in the first place. @@ -415,9 +592,11 @@ different rules: *not* to the plate, which is the whole point of it — so the offset genuinely belongs at the node, not the clip. -## Timelines, and why a scene is one +## Symbols, and why a scene is one -A **timeline** is an ordered bag of nodes in its own frame space: +A **symbol** is an ordered bag of nodes in its own frame space. (Earlier drafts +and code called this a *timeline*; that word now means only the UI pane that +shows one.) ```clojure {:frames 91 @@ -427,9 +606,10 @@ A **timeline** is an ordered bag of nodes in its own frame space: That is the whole type, and **everything that holds nodes is one of these**: -- a clip's **scene** is its root timeline, -- a **symbol** in the library is a timeline, -- a node with `:kind :symbol` is an **instance** of one. +- what a document opens on is a symbol, and **no symbol is reserved** — a new + document's is called `main` only because it has to be called something, +- anything placed inside another symbol is a symbol, +- a node with `:kind :instance` is an **instance** of one. An earlier draft of this document had a scene and a `:kind :timeline` symbol as two structures with the same fields and never said they were the same thing. @@ -474,7 +654,9 @@ for all three is the same — **their own**: ### Instances -A node with `:kind :symbol` and `:of :sym/blink` places one. Its own channels +A node with `:kind :instance` and `:source {:symbol :sym/blink}` places one, and +its `:playback` says how time runs inside it — which drawing is used and how it +is played are separate facts, per [the lane model](lane-model.md). Its own channels compose *over* the symbol's, so one definition is placed many times and tinted, offset or retimed at each placement — that is how a three-frame blink is reused at frames 40, 88 and 200 without copying it. @@ -545,28 +727,26 @@ dense geometry. A topology setting cannot be treated as a per-frame gain curve. The op list is the boundary with stage 7 in `docs/architecture.md`: the rasteriser takes ops and knows nothing about nodes, channels or time. -**A photographic underlay is not an op.** The registered source frame that an -animator traces over is a reference, not output, and it may not enter the indexed -buffer — the same rule `docs/architecture.md` already sets for handles and -vertex boxes. It is a `drawImage` at an affine on a separate canvas, which clips -at the canvas edge for free, and the only thing it needs from the model is the -world transform of the node it rides: +**A tracing layer is an op that never reaches the raster.** Footage or a still +to draw over is a symbol with `:type :trace` and a `:media`, placed by an ordinary +instance — so it is moved, scaled, trimmed, held and put in a lane like anything +else — and it resolves to one `:trace` op: `{:kind :trace :node :layer :media +:frame :size :m}`. The raster refuses that kind, the player hands it to a +`drawImage` on a separate canvas over the picture, and `clip/resolver` makes one +only when asked with `:tracing?`, which only the stage does. An export, a +symbol's centre and a thumbnail never ask, so a reference cannot reach the +picture by any path that forgets to filter it. See `docs/tracing-symbol-plan.md`. -```clojure -(world-of resolver :head) ;; -> Float64Array[6] -``` +A face's footage is one of these, placed as `:plate` under `:head` with the +anchor fit itself as its measured transform — the inverse of the head's, over +image height. Its world is `head · fit · 1/H`, so on a frame where the head and +the plate read the same measured frame the two cancel and the photo sits where +the face was filmed; on any other frame it rides the head. Registration is the +ordinary walk, not a matrix built beside it. -Composed with image-pixels-to-local — **both axes divided by `imgH`**, never by -their own dimension — the photo is registered with the shapes by construction, -and an unregistered underlay is merely decorative. The tracing editor chooses -which source frame to show under a cel. That reference choice is independent of -the finished picture fps and does not change the dense analysis track. A cel can -therefore use any useful source frame as its drawing reference, even when that -frame is not one of the displayed picture poses. - -A photo that has to sit *between* two drawn layers is the case that would make it -a `:bitmap` node with an op of its own. Nothing wants that yet: a reference is -either under everything or over everything at low alpha. +Which frame it shows is the placement's: `:time {:holds [...]}` holds it on +chosen frames, and a head with `:reads {:holds-of :plate}` jumps to the same +ones. Whether it is showing at all is the editor's, `[:ui :tracing]`. ### Making it fast in CLJS @@ -619,9 +799,9 @@ Proof that it covers what exists, not just what is wanted: | square pupil | node `:pupil-r`, `:kind :rect`, parent `:iris-r`, stencil `:iris-r` | | brow ring + quantised raise | node `:brow-r`, `[:geom :pts]` dense (the traced ring with height removed), `[:xform :pos]` dense (the quantised raise). **The decomposition design.md insists on is two channels.** | | head plate, kept frames | node `:head`, `:symbol` per instance, keys on `[:symbol]` at kept frames | -| `makeXform` face-oval crop | **gone.** Placement is `[:xform :*]` on `:face`; the stage clips | -| `stabilize` transforms | dense `[:xform :*]` on `:head`, read through its optional `:anchors` map | -| registered underlay | not data — a UI layer riding `(world-of resolver :head)` | +| `makeXform` face-oval crop | **gone.** Placement is `[:xform :*]` on the face's own `:place`; the stage clips | +| `stabilize` transforms | dense `[:xform :*]` on `:head` (the inverse fit) and on its `:plate` (the fit), read where `:reads` and `:time :holds` say | +| registered underlay | the face's `:plate`, an instance of the footage's tracing symbol under `:head`; a `:trace` op the raster never sees | | painted background cel | node per layer, `[:geom :pts]` **framed**, `[:style :color]` framed | | `mouth lead` | `:time {:offset k}` on performance nodes only | | `exposure` | `:time {:expose n}` on the clip root, inherited | @@ -658,28 +838,33 @@ scope does not define resolves to the loud magenta, like any other missing index ### The scope rule -`:palette` on a timeline is a channel like any other: +`:palette-track` points to an ordinary lane symbol. Its clips are instances of +restricted palette symbols: a palette symbol owns no nodes and points at exactly +one project palette. ```clojure -{:frames 91 - :palette {:animated? true :interp :hold :keys {0 :day, 48 :dusk, 72 :night}} - :nodes {...}} +{:id :shot :frames 91 :palette :day :palette-track :shot-palettes :nodes {...}} + +{:id :shot-palettes :type :palette-track :display :lane :frames 91 + :nodes {:day-clip {:kind :instance :source {:symbol :day-palette} ...} + :dusk-clip {:kind :instance :source {:symbol :dusk-palette} ...}}} + +{:id :day-palette :type :palette :palette-ref :day :frames 1 :nodes {}} ``` -**Absent means inherit** from the instancing context. **Present means this -timeline's content is read in that ramp, and it travels with the timeline** — a -symbol authored against `:night` stays night wherever it is placed. That is -lexical scope, and deliberately: a character with their own palette is a -character, not a decoration of whichever scene they were dropped into. +`:palette` is the symbol's authoring/preview palette. It seeds evaluation only +when that symbol is the viewed root; nested symbols do not replace the root's +choice merely because they were authored under another ramp. When absent, the +project default seeds evaluation. -Composition is the same walk as `:time` — down the instance chain, **innermost -set palette wins**. An enclosing timeline's palette therefore applies to -everything inside it that does not set its own, which is adjustment-layer -behaviour with no adjustment layer in it. It is just scope. - -And because it is an ordinary channel, a project switches palette over time with -keys on the root timeline, a child timeline switches on its own, and neither -knows about the other. +Covered clips of the viewed root's palette track override that seed. An +uncovered lane interval is a genuine gap, restoring the authoring palette or +project default. Palette clips use the same trim, roll, slide, claim-time and +undo commands as visual clips; palette code does not duplicate those edits. +Thus palette-track coverage, authoring preview, and project fallback are +separate facts rather than three accidental meanings of one field. There is no +second keyed palette control on symbols or instances: time-varying palette +changes are authored only as clips in the palette lane. ### One index space, partitioned by palette diff --git a/docs/architecture.md b/docs/architecture.md index 90d6536..a6b5341 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -8,6 +8,9 @@ rotoscoping rather than a sketch bolted to the side. `docs/animation-model.md` specifies the data both of them are about — nodes, channels, symbols and time maps — and supersedes this document wherever the two describe the same type. +The newer [Lane Model](lane-model.md) takes precedence for occurrence ownership, +playback semantics, shared editing operations, and multi-view UX. It explicitly +allows replacing the current format without backward compatibility. Nothing here revises an aesthetic decision; several things here split a decision that is currently made in two places at once. @@ -49,10 +52,10 @@ overrides and kept-frame sets are all in clip-frame space, so a clip slides on the timeline without a single stored number changing. Exposure and lead are transforms *within* clip space: -Detection retains every source frame. A chosen picture fps samples the frozen -roto in clip time; it changes neither source-frame count nor the audio clock. -The set of source frames an artist uses as cel tracing references is another -selection, independent of the picture fps. +Detection retains every source frame. Symbols carry their native fps; project +fps selects the output grid without changing source data or audio speed. +See [Time selection](time.md) for boundary sampling and frame units. +Tracing references remain an independent selection. ```clojure (defn pose-frame [clip cf] @@ -87,8 +90,8 @@ it a name is most of the work: `state` in `app.js` is a clip with its analysis inlined and its palette global. Cel keys select where drawings begin and how long they hold. The source frames -shown beneath a cel while tracing are chosen independently, and picture fps -only controls which analyzed pose the finished roto displays at a given time. +shown beneath a cel while tracing are chosen independently. Project fps controls +which native frames can appear on the output grid. ### Two things called "track" @@ -639,10 +642,10 @@ is what step 9 implemented, for the subset that exists: clip//name clip//subject/ clip//timing clip//feature/ clip//stage clip//group/ -clip//source clip//timeline/ -clip//timeline//node/ -clip//timeline//measured/ -clip//timeline//channel// +clip//source clip//symbol/ +clip//symbol//node/ +clip//symbol//measured/ +clip//symbol//channel// ``` Settings live on subject, feature and group leaves. Each feature has one area, so @@ -675,6 +678,45 @@ and `~` is then refused inside a name. That is the whole of the escaping. That maps onto the tiers exactly — the server stores tier 1 and snapshots tier 1, with tiers 2 and 3 as content-addressed blobs beside it. +### As built + +- **Addresses.** `/` is the index of the projects you own or edit. A project + is only ever at `/p//`: the id finds it, the slug is its name and + follows a rename without a history entry. "new" makes the project on the + server first and opens it; a built-in example opened in the editor is saved + at once as a project of its own. There is no bare project. The address, the + title and the socket follow `[:project :id]` through one global interceptor + (`events/collab`). +- **Ownership.** Every project has an owner (`Project.owner`, not nullable) and + `editors`. Anyone with the link reads; the owner and editors write. Making a + project needs you signed in. A reader can make a copy of their own. + `/api/{me,login,signup,logout}`, `/api/projects//editors[/]`. +- **Every edit saves.** No save button. The same interceptor sees + `:paint/revision` move and saves; one request is in flight at a time, and an + edit made meanwhile goes when it lands — so a drag reaches the room as fast + as the round trip allows. A save sends only the leaves that differ from what + was last synced, and a clean document sends nothing. Analyses and blocks + already put on the server are not asked about again. +- **Saves are patches.** With `base` (the seq last caught up to) a save names + only its changed and removed leaves. A named leaf somebody else changed after + `base`, to something else, fails the whole save with 409. +- **The first write wins.** A remote write to a leaf with a local change not + yet sent waits in the entry's `:behind`; the next save, or a 409, puts theirs + on screen over ours and says so. Ours stays in the undo list. +- **The socket** (`/ws/projects/`, channels + daphne) carries presence, + the delta each committed write broadcasts, and `access` when the editor list + changes. A gap in `seq`, a welcome, or a 409 refetches the document. +- **Undo** is per person, recorded in `events/edit` as leaf befores and afters + (`domain/history`), and applied as an ordinary edit. A step undoes only if + every leaf it touched still holds what it left there: somebody else's edit + since refuses it rather than being undone with it. Edits to the same leaves + within a second, each starting where the last left off, are one step. +- **Snapshots** are named revisions (`/api/projects//revisions`), with each + clip's block keys. Restoring one is an ordinary write, broadcast like any. + +Not yet: follow mode, frame/selection in presence, the advisory `:editing` +lease, the durable outbox. + ### Why this model, and not a CRDT The usual reason to reach for Yjs or Automerge is automatic convergence without @@ -780,17 +822,17 @@ collaborator's keying. The fix is addressing, not an algorithm: ``` palette -sequence/:sid +lane/:sid clip/:cid/timing clip rate clip/:cid/subject/:sid tracked subject and settings clip/:cid/feature/:fid tracked feature and settings clip/:cid/group/:gid shared settings for an eye pair -clip/:cid/timeline/:tid frame count, palette -clip/:cid/timeline/:tid/node/:nid one node: parent, stencil, z, time -clip/:cid/timeline/:tid/channel/:nid/:prop -clip/:cid/timeline/:tid/measured/:nid -clip/:cid/timeline/:tid/cel/:nid/:frame -clip/:cid/timeline/:tid/overrides/:nid/:prop +clip/:cid/symbol/:sid frame count, palette +clip/:cid/symbol/:sid/node/:nid one node: parent, stencil, z, time +clip/:cid/symbol/:sid/channel/:nid/:prop +clip/:cid/symbol/:sid/measured/:nid +clip/:cid/symbol/:sid/cel/:nid/:frame +clip/:cid/symbol/:sid/overrides/:nid/:prop ``` Each feature and node has its own leaf, so tuning separate features and adding diff --git a/docs/clipboard-plan.md b/docs/clipboard-plan.md new file mode 100644 index 0000000..5b0b616 --- /dev/null +++ b/docs/clipboard-plan.md @@ -0,0 +1,135 @@ +# Selection and clipboard + +This is the implementation contract for multi-selection, copy, cut, paste, +duplicate, and duplicate unique. It deliberately replaces any incidental older +behavior. The document model is the authority; the stage and timeline are two +views of the same editor state. + +## One selection + +`[:ui :selections]` is the ordered selection set. Its last member is the primary +selection in `[:ui :selection]`, used by the inspector and single-subject tools. +Every member is an occurrence address: + +```clojure +[:node owner-symbol-id node-id row-path] +``` + +The row path distinguishes two occurrences of shared content. Commands that +write the document canonicalize those addresses before acting: + +- invalid and non-node addresses are ignored; +- the same owned node, `[owner-symbol-id node-id]`, is acted on once; +- when one selected row path is below another selected row path, only the + ancestor is a clipboard root. Its ordinary parent-pointer subtree comes with + it, and an instance already displays the symbol it references, so also + materializing the visibly nested selection would duplicate it twice. + +Plain click replaces the selection. Shift-click toggles membership, on both the +stage and timeline. A stage marquee replaces, or with Shift adds to, the same +set. Timeline rows, bars, and cel blocks render membership from that same set; +the primary member gets the inspector/focus treatment. + +Creation targeting is derived rather than stored. The primary (last-selected) +occurrence is the preferred row; the playhead validates it and, when necessary, +walks outward to the nearest valid containing occurrence. A target is singular +even when selection is plural. See `docs/creating-in.md` for the resolver +contract. + +## Clipboard value + +The clipboard is editor state, not document state and not history. Copy records +a detached snapshot of each canonical root and its complete parent-pointer +subtree. It records source occurrence paths and authored node data, but normal +copy deliberately keeps referenced symbol identities. Therefore a pasted +instance is another use of the same symbol. The clipboard survives cutting its +nodes because it contains the node snapshot, not merely their addresses. + +This first implementation is the application's clipboard, not the operating +system clipboard. It is consequently project-local and has no serialization or +cross-project identity collision policy hidden inside it. + +## Paste + +Paste resolves one destination from the primary selection and playhead. An +ordinary occurrence is usable only while the playhead maps through every +enclosing occurrence and lies within its extent. Otherwise resolution walks +outward, with the open symbol as the total fallback. A selected lane is an +insertion surface only while all occurrences enclosing its parent symbol are +valid. Multi-selection supplies one ordered payload, not several destinations; +only its primary member anchors this resolution. + +The earliest finite start among the copied roots is aligned with the playhead +in the destination. All other root starts retain their offset from it. Roots +without a finite span remain timeless; paste does not invent a span for a shape +that was authored for the whole symbol. Parent/child timing, transforms, +channels, corrections, playback, stencil links, and relative root order are +otherwise copied exactly. Every node receives a new identity, and all internal +parent and stencil references are remapped. + +In an ordinary composition overlap is valid. In a lane the pasted finite spans +claim their intervals using the lane's existing overwrite rule: covered cels +are removed, crossing cels are trimmed or split, and the pasted roots do not +overlap one another. Pasting a timeless root into a lane is refused. Validation +is all-or-nothing. + +After paste, the new roots are the selection, in clipboard order, and the last +one is primary. + +## Cut + +Cut first takes exactly the same snapshot as copy, then deletes every canonical +root and its parent-pointer subtree. The clipboard write is editor state; the +whole document deletion is one history transaction. The selection is cleared. +Undo restores the deleted document nodes. It does not roll back the clipboard, +which matches ordinary editor behavior. + +## Duplicate and Duplicate Unique + +Duplicate does not read the insertion target or playhead. It is a local +operation beside the selected material: + +- in an ordinary composition, copies keep the originals' parent, transform, + timing, and span, and are stacked immediately in front; +- for direct children of a lane, the selected temporal envelope is repeated + immediately after itself. Relative timing and gaps inside the selected set are + preserved, and later cels ripple forward by the envelope duration. This is + the lane's useful "duplicate forward" behavior; it is not a second command. + +Selections in several owners are handled per owner in one command. Thus two +lane selections repeat in their respective lanes, while a selected composition +node duplicates in place, all as one history step. + +Normal Duplicate preserves symbol references, just like normal copy/paste. +Duplicate Unique performs the same placement but deep-copies the complete graph +of every referenced symbol. One shared remap table is used for the whole batch, +so two duplicated instances that shared a nested part still share one new copy +with each other, while sharing nothing mutable with the originals. Immutable +media/store blocks may remain shared. + +After either duplicate command, the new roots replace the selection. + +## History, refusal, and stale state + +Cut, paste, duplicate, and duplicate unique each call one domain command and +commit through one `edit/transaction`; each is exactly one undo/redo step no +matter how many nodes or symbols it touches. Copy and selection do not +touch history. A refusal changes no document leaves and creates no history step. + +Undo/redo filters the complete selection set against the restored document and +repairs the primary selection. Clipboard payloads remain snapshots. Paste +validates the resolved destination and all remapped references at commit time, +so a stale selection or a newly impossible symbol cycle refuses rather than +partially editing. + +## Required tests + +Domain tests cover canonical ancestor/descendant selection, subtree ID remaps, +normal shared references, deep unique graph remaps, multi-root relative timing, +composition overlap, lane overwrite, lane forward duplication and ripple, +mixed-owner duplication, cycle refusal, stale targets, and all-or-nothing +failure. Event tests cover copy without history, atomic cut/paste/duplicates, +resulting multi-selection, target fallback at the playhead, +and one undo plus redo of each mutation. Browser tests cover mirrored stage and +timeline selection, Shift-toggle on labels/bars/cels, singular derived creation +targeting, and the keyboard commands. diff --git a/docs/correction-authoring-plan.md b/docs/correction-authoring-plan.md new file mode 100644 index 0000000..9997d81 --- /dev/null +++ b/docs/correction-authoring-plan.md @@ -0,0 +1,234 @@ +# Correction authoring implementation plan + +Written against `2f1c9b9` (2026-09-30), following the lane handoff in +`7a54bfc`. Implemented on `codex/correction-authoring`; this now records the +scope and acceptance criteria of that implementation. +The cel-sheet targeting work described below was removed with the cel-sheet UI +on 2026-10-01; it remains here only as history of that implementation. +Read [lane-handoff.md](lane-handoff.md) and the correction section of +[lane-model.md](lane-model.md) first. Their ownership and document rules remain +the foundation. The choices below settle the first implementation's scope. + +## Outcome + +A person can select a lane or cel, specify a range, and apply Constant +adjustment, Ramp, or Return motion to rotation or position. The result is one +correction layer and one undo step. It works from either timing view. A lane +correction crosses drawing boundaries; a cel correction travels with its cel. +Regeneration preserves the hand work and presents incompatible layers for an +explicit decision. Frames outside the support evaluate exactly as before. + +Finish this vertical slice before adding more property types or gestures. +Numeric range fields and an Apply button are sufficient for this pass. Dragging +a range or manipulating a peak on the stage can later issue the same command. + +## 1. Fix sheet targeting first + +In `ui/timeline.cljs`, `cel-sheet` currently drops each lane row's `:select`. +An occupied cell selects its cel; a gap only seeks, leaving the previous target +selected. Thus clicking lane B's gap after selecting lane A can send an insert +or overwrite to A. + +Carry the row's complete selection address into its column and gap cells. +An occupied cell selects its cel; a gap selects its lane. Make the column header +select the lane too: this is the explicit way to author across drawings. +Preserve full paths, not just `(peek path)`, as view identity. Keep seek and +selection dispatch order deterministic. + +Add a two-lane browser case: select A, click a gap in B, overwrite, assert that +only B changes, and undo once. Add a header-selection assertion. Retain the +existing occupied-cell hold test. Do not redesign sheet rendering in this step. + +## 2. Make stack compatibility consistent + +There is a concrete discrepancy at this HEAD: + +- `channel/problems` uses `stack-conflict`, accounting for prior replacements. +- `channel/conflicts` and `flow/regenerate.cljs`'s `rebased` use + `conflict-with`, comparing an offset directly with the base. + +A two-component base, a covering three-component replacement, then a +three-component offset is valid and evaluates correctly, but the latter paths +can report or mark that offset incompatible. Conversely a replacement can make +an offset incompatible even when it fits the original base. + +Extract one ordered-stack compatibility operation and use it for validation, +conflict discovery, regeneration, and resolution. Keep `conflict-with` if useful +for the narrower question its name/docstring describe. Do not use it alone to +decide whether a stacked layer is applicable. + +Compute compatibility against the values that can actually reach a layer over +its support. Partition at overlapping support boundaries if needed: two adjacent +replacements can jointly cover an offset even though neither covers it alone. +An empty replacement channel does not supply a value and must not erase the +possible input shape. Preserve the evaluator's absence behavior. Explicitly +marked conflicts are skipped, so later layers must be checked against the stack +that actually runs. During regeneration, recompute compatibility in order, using +each preceding layer's resulting active/conflicted state. Preserve IDs, values, +support, and order; update compatibility reasons without dropping hand work. + +Use structural shape reasoning for dense data rather than requiring every block +to be sampled. If unknown shape or missing samples limit what can be proven, +retain the current absence contract and document that limit; do not claim an +unconditional proof of runtime safety from incomplete metadata. + +Tests: covering replacement of a different shape; partial coverage; adjacent +covering replacements; empty replacement; inactive conflicted replacement; +regeneration changing base shape; and the same cases through cursor evaluation. +Assert that an accepted compatible stack is not listed as a conflict, and that +an incompatible regenerated layer remains persisted but is skipped. + +## 3. Pure correction commands + +Add `frontend/src/arthur/domain/correction.cljs`. It owns authoring and resolving +corrections; `channel.cljs` continues to own evaluation and compatibility. +Suggested API (names may follow repository conventions): + +```clojure +(add clip sid node-id channel-path + {:id layer-id :support [a b] :motion :return + :start 0 :peak angle :peak-frame p}) +(remove-layer clip sid node-id channel-path layer-id) +(retry-layer clip sid node-id channel-path layer-id) +``` + +Return `{:clip updated :selection node-id}` or `{:refused reason}`. IDs come +from the event caller (`random-uuid`), never from the pure command. Reject a nil +ID or one already used within that channel stack. Address layers by the full +symbol/node/channel/layer tuple; no global layer registry is needed. + +Resolve the base via `node/channels`, which supplies defaults. A lane with no +explicit rotation channel already has a zero rotation; materialize that channel +with its new `:over`. Preserve every existing base field, generated provenance, +and previous layer. Never route this through a setter that bakes the correction +into base keys. Append to the ordered stack and validate the resulting document. +Do not run correction edits through `lane/finish`, whose extent policy belongs +to cel arrangement. Use `clip/problems` for the candidate document instead. + +First authoring properties: `[:xform :rot]` (scalar radians) and `[:xform :pos]` +(two numeric components). First blend operation: `:offset`. UI labels must say +offset/delta, since a target offset of 20 degrees does not mean an absolute +rotation of 20 degrees. Keep existing `:replace` evaluation and loaded stacks; +there is no new replacement-authoring UI in this slice. + +All command support endpoints are finite integer OWNER frames, `[a b)`, with +`a < b`. Do not ban negative owner frames merely because displayed shot frames +start at zero. Validate all supplied values for finite numbers and exact shape. +Refuse unknown targets, unsupported properties/motions, malformed ranges, and +incompatible stacks with useful messages. Refusal must not mutate store/history. + +Motion construction uses existing channels only: + +| Command | Values | Minimum samples | +| --- | --- | --- | +| Constant adjustment | `(ch/framed delta)` | 1 | +| Ramp | `(ch/keyed {a start, (dec b) end} :linear)` | 2 | +| Return motion | `(ch/keyed {a start, p peak, (dec b) start} :linear)` | 3 | + +For Return, require integer `a < p < b-1`. Default the UI peak to +`a + floor((b-a-1)/2)`; on an even-length range the earlier middle sample wins. +Expose the peak frame so this is visible and adjustable. Never place an endpoint +at `b`: it is outside the selected samples. `[10 13)` with start 0 and peak 0.5 +must yield offsets `0, 0.5, 0` at 10, 11, 12. Support controls the boundary; +there is no need to insert zero keys into the base before/after it. + +## 4. Owner and range UI + +Add a Corrections section to the right pane (`ui/params.cljs`), extracting a +`ui/corrections.cljs` component if that keeps the pane readable. Provide an +explicit target readout (symbol, lane or cel), Rotation/Position, motion choice, +From/Through fields, relevant value fields, peak frame for Return, and Apply. +Offer a selected cel's owning lane as an explicit target choice. Do not silently +promote a cel edit to a lane edit. Shared drawing content is outside this first +UI; it has different sharing consequences. + +For this first pass the range fields explicitly read **owner frames**, with +inclusive From/Through converted to `[from, through+1)`. This is a deliberate +UI scope choice, not a claim that a displayed shot range and owner range are +interchangeable. It lets nested and retimed owners be addressed without an +unproven range conversion. Match the existing zero-based numbering and show the +owner beside the range. The handoff must record that displayed-range dragging +is still outstanding. + +Use an explicit three-sample initial draft in owner coordinates; for a cel, +prefer its span start where it is integral. Show the range, allow adjustment, +and do not extend a cel or shot to make the correction visible. Reset the draft +when the target changes. Rotation is shown in degrees and converted to radians +at the event boundary, following the existing inspector convention. Position +uses x/y inputs in the owner's transform coordinates. + +Draft inputs must not write document state, start history groups, or invoke the +existing inspector `number-input`'s hold/settle behavior. Apply dispatches one +event; success uses `edit/transaction` once and preserves the selected node's +full address. Do not use a layer ID as node selection. Validate again at Apply, +since the target/document may have changed since the draft was opened. + +If adding a “use playhead” convenience, prove its mapping separately. The clock +of a cel's transform is its own node clock, not the drawing source clock selected +by `:playback`. `nest/inside` on the complete cel path enters the source and is +therefore the wrong shortcut. The existing `selection-frame` resolves only the +owning symbol; node/ancestor time conversion remains necessary. Floors, loops, +and nonintegral mappings must never silently snap an authored range. Omit this +convenience rather than expanding the first pass into a new timing system. + +## 5. Conflict actions and regeneration proof + +Show a document-wide list from `clip/conflicts` in the pane, including symbol, +node, property, layer ID, and reason. Keep it accessible even when a different +node is selected. Also list the selected target's layers in stack order with +support, motion values, and status; do not require a new persisted motion label. + +Provide Remove correction and Retry compatibility. Remove is explicit and +undoable; Retry rechecks the complete candidate stack and clears a conflict only +when it is valid. If retry would invalidate a downstream offset, refuse and say +why. Similarly, removing a replacement that makes a later offset invalid must +refuse, rather than commit an invalid document or silently remove more layers. +A retry that changes nothing must not manufacture an undo step. + +These are minimal resolution actions, not topology remapping. Automatic geometry +remapping, reordering layers, editing arbitrary stored vector values, and resolving +removed targets are separate work. Preserve all existing generated-data behavior. + +Exercise the actual `flow/regenerate.cljs` entry points in integration tests: +author a correction, regenerate compatible base data, and confirm the correction +survives with its ID/support/values intact and affects the new base. Then change +topology on a geometry fixture and assert persisted actionable conflicts. The +geometry fixture can use an existing layer directly; geometry authoring is not +required to expose and resolve a conflict already present in a document. + +## 6. Verification and completion + +Use focused tests that establish observable promises: + +- Domain: each motion's sample values; refusal on short ranges/nonfinite values; + default channel materialization; unchanged base and prior layers; duplicate ID. +- Evaluation: compare before/after on every frame outside support, including + neighboring interpolated frames. Cursor/spec agreement in nonmonotonic order. +- Ownership: a lane Return crosses a drawing boundary; a cel correction moves + with the cel and survives split/trim. Neither alters another use of its drawing. +- Sampling: picture-rate/pose selection changes the generated base frame while + the authored correction still reads owner time. Keep HEAD's regression tests. +- Events: one Apply is one undo step; undo/redo restores complete layer data; + refusal leaves clip/history unchanged; stale target refuses; selection survives. +- Persistence: leaf and Transit round trips retain layers, order, IDs, conflicts. +- Browser: both views can select a target and apply the same correction using + actual controls. Assert evaluated results and history, not only a layer count. + Include the two-lane gap-targeting case from step 1. +- Regeneration and conflict actions: use the real flow and test undoable removal, + valid retry, invalid retry, and removal that would break a downstream layer. + +Run the suites documented in `lane-handoff.md`: CLJS tests, lane and take browser +flows, Django tests, and optimized frontend build. Restore the dev app bundle +after the release build. Note that `take.mjs` writes a local project. Report +actual results and any unrun checks; do not copy previous test counts as evidence. + +Suggested commit sequence: sheet targeting; consistent stack compatibility; +pure correction commands; pane/events plus browser proof; updated handoff. +Keep each commit coherent and tested. No schema version bump should be needed: +the layers already have a persisted representation. + +Update `lane-handoff.md` and `lane-model.md` with what shipped, the owner-frame +range UI limitation, conflict actions available, and verified test counts. Done +means a person can author and undo the correction, regenerate its base, and see +either their preserved edit or a useful conflict. A constructor without reachable +controls, or controls without that regeneration proof, does not finish this work. diff --git a/docs/creating-in.md b/docs/creating-in.md new file mode 100644 index 0000000..8d5a8dd --- /dev/null +++ b/docs/creating-in.md @@ -0,0 +1,155 @@ +# Creating in + +Revised 2026-10-02. + +Arthur's creation model is a layer list plus a playhead. The primary active row +is the preferred place for a new thing. The playhead decides whether that row is +present in the current occurrence and supplies the frame at which the thing is +created. + +This is editor behavior, not document structure. A saved clip has symbols, +instances and spans; it does not have timeline rows or a creation target. + +## Selection and the active row + +The selection set names the objects affected by copy, delete and transform. Its +primary member is also the active row, analogous to the active layer in a paint +program. There is no second targeting gesture for a user to maintain. + +- Clicking a row label, its timeline body, or the same occurrence on the stage + makes that occurrence primary. +- Shift/marquee selection may retain several objects, but only the primary row + anchors creation. +- Clicking blank stage or timeline space returns the active row to the symbol in + the current tab. +- Row disclosure is independent. Selecting or creating never implicitly expands + a row. + +The selection does not change merely because the playhead moves. An off-frame +object remains available for copying, deletion and inspection. + +## Resolving the active row at the playhead + +Resolution starts with the structural destination implied by the primary row: + +- A lane row means the lane symbol itself. +- An ordinary symbol-instance row means the symbol placed by that occurrence. +- A non-instance row means the symbol containing that node. +- No row means the symbol in the current tab. + +For an ordinary symbol occurrence, the playhead must be within that occurrence's +extent in the current nested context. Extents are tested after walking all parent +time maps and use the half-open interval `[in, out)`. If the preferred occurrence +is not present, resolution walks outward to the nearest parent whose occurrence +is present. A containing lane is therefore the natural fallback from an inactive +cel. If no nested occurrence is present, the current tab is the destination. + +The preferred row is retained during fallback. Scrubbing back into its extent +makes it the effective destination again. + +A lane differs only in what its row means. It is an insertion surface across its +containing timeline and does not require an existing cel under the playhead. A +cel within the lane is still an ordinary symbol occurrence with an extent. + +For `A -> B -> lane C -> cel D`: + +- over D, with D primary, creation happens inside D; +- past D but while C is available, creation inserts a new cel in C; +- outside C's containing occurrence but inside B, creation happens inside B; +- outside every nested occurrence, creation happens in A, the current tab. + +## What creation does + +Once resolved, every creation command follows the destination kind. + +### Lane destination + +- A new or dropped symbol is instantiated directly in the lane at the playhead. +- Its interval claims that time. Existing cels under the interval are removed or + trimmed by the lane's ordinary claim-time command. +- Beginning a drawing creates a new one-frame drawing symbol in the lane and the + finished shape is a child of that drawing. +- Selecting an existing cel changes the destination from the lane to the symbol + placed by that cel; subsequent symbols and shapes become children there. + +Double-clicking a lane creates an empty cel at the playhead; beginning a drawing +creates a drawing cel there. These are the same lane-creation operation with +different payloads. The pointer chooses the lane, never a second creation time. +The resulting cel is selected, so it immediately becomes the preferred target: +drawing again enters that cel's symbol instead of replacing it. + +Thus no separate "new cel" versus "add inside" mode is needed. Selecting the +lane header says new cel; selecting a cel says add inside. + +### Ordinary symbol destination + +- A new symbol is instantiated as a child at the mapped playhead frame. +- A new shape is authored directly in the symbol at that frame. +- Stage coordinates are transformed through the occurrence into the destination + symbol's local coordinates. + +### Explicit timeline drop + +A timeline drop uses the row and frame under the pointer, not the stored active +row and playhead. Dropping onto a lane therefore always instantiates in that lane +and claims the pointer's interval. A stage drop uses the resolved active row and +the playhead. + +Paste, imported symbols and converted footage obey the same resolver as direct +creation. They must not each reconstruct nesting or extent fallback separately. + +## Multi-selection and paste + +Multi-selection does not create multiple insertion targets. Copy and cut take +the canonical forest of selected roots as one ordered payload; the primary +(last-selected) occurrence alone supplies the preferred row for a later paste. +At paste time that row and the current playhead resolve one effective +destination, and every root in the payload is inserted there in one transaction. + +The earliest finite root start is aligned to the destination frame. Other roots +keep their timing offsets, hierarchy, and clipboard order. In an ordinary symbol +the roots may overlap. In a lane every root must have a finite span and the +payload's root spans must not overlap one another; valid spans claim their times +and trim or remove existing cels as a batch. An invalid member refuses the whole +paste rather than inserting a partial payload. + +After paste, all new roots form the selection and the final root is primary, so +it becomes the preferred row for the next creation. Pasting one payload into +several selected destinations is intentionally not implicit: that would be a +separate distribute command. Duplicate is also distinct from paste—it stays +beside each source in its original owner and does not consult the playhead or +creation target. + +## Palette lanes + +Palette lanes use the same row-and-playhead resolution. Their content filter and +transition command remain palette-specific: only palettes and palette +transitions can be inserted there. This is a type restriction, not a second +targeting model. + +## Implementation boundary + +One pure resolver returns the effective destination: + +```clojure +{:kind :lane | :symbol + :sid destination-symbol + :path effective-occurrence-path + :frame destination-local-frame + :matrix destination-to-current-tab-transform} +``` + +Callers may add the unchanged document as `:clip` or rename `:frame` to `:at`, +but they must not reinterpret the active row. Polygon creation, symbol creation, +stage drops, paste, import and footage conversion all consume this answer. +Explicit timeline drops use the same structural row rule with the row and frame +under the pointer; unlike playhead resolution, an invalid pointer destination is +refused rather than allowed to fall outward. + +The invariants are: + +1. The primary row is the preferred structural destination. +2. The playhead validates occurrences and supplies creation time. +3. Inactive targets fall outward; selection does not follow them. +4. A lane row inserts a cel, while a cel row enters its symbol. +5. Stage creation uses the playhead; timeline drops use pointer time. diff --git a/docs/frame-selection.md b/docs/frame-selection.md new file mode 100644 index 0000000..b875ef8 --- /dev/null +++ b/docs/frame-selection.md @@ -0,0 +1,510 @@ +# Frame selection + +> Since `docs/tracing-symbol-plan.md`: `domain/trace` is gone. Trace keys are +> the face's `:plate` placement's `:time :holds`, the origin is the head's +> `:reads`, and the photo's registration is the plate's own measured channels. +> Where this document names `trace/measured-local` or `trace/prepare`, read the +> plate's or the head's measured channels and `node/hold`. + +Two mechanisms. One vocabulary. An earlier draft of this document claimed they +were one component used twice — because `suggestPlateFrames` in the old +`js/pipeline.js` and the never-built "performance poses" of +[timing-handoff](timing-handoff.md) looked like the same function — and that claim +is wrong. They share how a selection is *read* and how the hand overrides one. +They do not share how frames get chosen, because the two are answering questions +of different shapes. + +[Time selection](time.md) is the floor both stand on: an output frame reads the +latest native frame at or before its time, and nothing rewrites the dense +measurements. That is a *cadence*: an answer with no opinion about content. It +cannot know that the one frame where the eye is fully closed is worth more than +its neighbours, so at 12fps out of 30 it drops that frame two times in three. +This document is how the picture gets an opinion. + +## The one idea + +**A selection is a set of frames chosen out of a dense measurement, and read by +holding the latest one at or before now.** + +The holding half already exists and is already shared: `pose/held-frame` is called +by `node/hold` (a placement's `:time :holds`) and by `pose/source-frame`, which is the two sites agreeing +about reading. The hand half is shared too — see *Three layers* below. Choosing is +what differs. + +| | Plate drawings (tracing) | Performance poses | +| --- | --- | --- | +| the question | which frames does an artist have to draw a head on? | which frames does the picture change a shape on? | +| the cost being managed | a person drawing | a pose looking wrong | +| signal | the measured head's motion | — none; a stored cut | +| the baseline it improves on | drawing on 2s | the cadence, or the exposure grid | +| the shape of the answer | a non-uniform set out of dense | the same grid, nudged | +| lives on | the face's `:plate` `:time :holds` | the instance's `:playback :tracks` | +| hand edit today | `::project/toggle-hold` | `pose/put-cut` / `pose/remove-cut` | +| UI today | `params/layer-section` | **none** | +| proposes today | **nothing** | **nothing** | + +## Why they are not one function + +A plate selection has to be **non-uniform**, and that is the whole reason it +exists. A head still for sixty frames and then whipping across in ten wants two +drawings for the first stretch and eight for the second. Drawing on 2s gives +thirty-five drawings, most of them identical, and no amount of nudging a uniform +grid will produce the distribution that is wanted — the spacing itself is the +answer. That is what the prototype's walk was for, and it is why a cost knob +(`:tolerance`) belongs on this side: the artist is buying drawings. + +A performance selection is **not choosing sparseness at all**. The output rate or +the exposure setting has already chosen it. The question left over is only *which* +native frame each already-decided slot reads, and the failure it fixes is narrow: +a slot landing one or two frames off the closure. Nudging the grid is the right +size of answer, and there is nothing for a tolerance to mean. + +There is a second, harder reason, and it is the one that settles it: + +**A selection cannot put a frame on screen that the output grid never samples.** +At 12fps out of 30, output frame 5 reads native 12 and output frame 6 reads native +15. A closure at native 13 is *between* them. Protecting frame 13 in a set of +kept frames makes it available and makes it the frame held across 13 and 14 in +native space — and at a 12fps output it still never appears, exactly as +[time.md](time.md) says: an event between output frames cannot create an extra +frame in a 12fps output. Only moving what output frame 6 reads can show it. So +the performance side has to act on the grid, not on a set beside it. + +## Three layers, and the middle one is derived + +The trap this is designed around is stated in +[timing-handoff](timing-handoff.md) and is worth repeating because it is the +only hard rule here: + +> Store manual edits separately from generated proposals so changing the rate or +> tolerance retains hand decisions. + +So a selection is: + +```clojure +{:policy {:tolerance 0.02} ; what the proposer was asked for + :keep #{47} ; frames the hand insists on + :drop #{30}} ; frames the hand refuses +``` + +and the effective set is `(proposed ∪ keep) \ drop`, always containing frame 0. + +`:keep` and `:drop` are the document. The proposal is not: it is recomputed from +`:policy` and the dense signal whenever either changes. **Re-suggesting at a new +tolerance must never cost somebody their pinned blink**, and that is the entire +reason the hand decisions are stored as their own two sets rather than as the +resulting frame list. + +This layering is the part that really is shared. On the performance side there is +no `:policy` worth storing — the grid is the policy — but `:keep` and `:drop` mean +exactly what they mean on the plate side, and `select/effective` is the one +implementation for both. A preserve mark *is* a keep. + +### Materialise the result, do not derive it on the render path + +The effective set is written back to where each site already reads it — +the plate's `:time :holds`, or the pose track — so that every existing reader is untouched +and nothing on the per-frame path has to open a dense block. Proposing is a +command, not a subscription. `ch/value-at` allocates per call and says so; that +is fine for a button press over a few hundred frames and would not be fine at +30fps. + +This means the stored frame list is redundant with `policy + keep + drop`. That +is deliberate and it is the cheap direction of the trade: a stale list is +recoverable by pressing Suggest again, and a dense read per node per frame is +not recoverable at all. + +## The plate selection: a non-uniform chooser + +`arthur.domain.select`, built — see *What is revertible* for the one part of it +that is not yet wanted. Pure, no store access, no clip access: the site hands it a +signal it has already read. + +```clojure +(defn propose + "Frames worth keeping out of `n`, given `signal`." + [n signal {:keys [tolerance protect]}]) + +(defn effective + "`proposed` with the hand's decisions applied. Always contains 0." + [proposed keep drop]) +``` + +`propose` is the prototype's walk, generalised off landmarks: + +1. keep frame 0, make it the anchor; +2. settle the protected frames from `:protect` *before* walking; +3. for each later frame, keep it when it is protected, or when + `distance(anchor, f) > tolerance`. Either way it becomes the anchor. + +`distance` is the max absolute difference over components, so a signal of +landmark pairs and a signal of one number both work without the caller saying +which it handed over. Every sample in a signal is the same width, and a signal of +two widths is refused: comparing the prefix two samples happen to share would let +a reader that drops a component read as no movement at all. + +**A protected frame anchors the walk like any other kept frame**, which is why +protection is settled first and is not unioned onto the walk's result. The anchor +is what is on screen; once a protected frame is kept the viewer is looking at it, +so measuring the next frame's drift from a frame no longer displayed is wrong. + +**An absent measurement is `nil`, and converting to that is the reader's job.** A +frame where the face was not found says nothing about the signal: it cannot move +the anchor and it is not a frame worth keeping. `trace/measured-local` already +returns nil there. A protected frame with no measurement is still kept — a plate +frame is a frame somebody draws on whether or not the detector found a face. + +**A non-finite tolerance falls back to nought.** A cleared slider reads as NaN and +every comparison against NaN is false, which taken literally proposes frame 0 +alone and collapses the whole take to one drawing. Nought proposes every frame +that changes, which is merely the baseline back again: wrong in a way somebody can +see and undo. + +### The signal + +The head's measured transform, applied to a fixed reference quad, giving +displacement in stage units — so a tolerance means "the head has moved this far" +and is a number a person can reason about. `trace/measured-local` already builds +that matrix per frame and is private; make it public rather than writing a second +one. Map it over the frames and transform four corners through `node/apply-pt!`. + +The quad's size is a real parameter hiding in the word "fixed": it sets how much +rotation registers against translation. Give it a name and a comment rather than +an inline literal. + +Do not reach for raw landmarks. The prototype used them because it had them lying +around; the transform is what the drawing actually follows, it is already on the +node, and it is three channels instead of a block. + +### The upgrade path, and do not start here + +The greedy walk is order-dependent and slightly suboptimal. The optimal version +is a dynamic program — choose `k` frames minimising held-reconstruction error, +which is textbook segmented least squares and is O(n²k), nothing at n≈300 — and +it keeps extrema *for free*, because an extremum is exactly where a zero-order +hold is most wrong. + +Build the greedy one first anyway. It is proven, it shipped in the prototype, and +having two implementations to compare is how the DP gets tested. Swap it behind +`propose` afterwards, where the signature already permits it. + +Most of `select_test` pins the greedy walk's exact output, deliberately, for that +comparison — so expect to rewrite those expectations when the DP lands, and keep +them as greedy-specific tests rather than deleting them. The assertion that is a +*spec* rather than a pinned vector, and should be written on this side before the +swap, is: + +> reading the signal through the selection, held, never differs from the dense +> measurement by more than `tolerance` + +That is what makes the tolerance number mean something to a person. It is true of +the greedy walk by construction and it is what the DP optimises, so it survives +the swap untouched. + +## The performance selection: a preserve-snap on the grid + +Not built. The rule is ten lines; getting the two halves of it into the same place +is the work. An earlier draft of this section said "about thirty lines" and that +was understated — see *Where it goes* below. + +A grid slot already picks a native frame — `cadence/frame` for the output rate, or +the exposure fold for a deliberate hold at full rate. Write `d(k)` for the native +frame slot `k` defaults to. Slot `k` is the first slot to cover everything in +`(d(k-1), d(k)]`, and the frames strictly inside that interval are the ones the +grid shows to nobody. So: + +> **Slot `k` reads the latest preserved frame in `(d(k-1), d(k)]`, and `d(k)` when +> there is none.** + +That is the whole mechanism. It recovers a dropped frame out of the slot's own gap, +it can never read a frame another slot already showed, and it cannot reach past +`d(k)`. + +**Snap backward only, never forward**, and note which direction that actually is, +because it is easy to get backwards. The closure at native 13 in a 12-from-30 +output is recovered by slot **6** — whose default is 15 — reading 13. It is *not* +recovered by slot 5, whose default is 12, reaching forward to 13: slot 5's instant +is 5/12s = 0.4167s and native 13's is 13/30s = 0.4333s, so that would show the +closure 17ms before the mouth shut. `cadence/frame`'s contract is the latest native +frame at or before the slot's time and `cadence_test` asserts +`selected <= f*native/grid` over every grid and native pair, so reaching forward +breaks a tested invariant as well as the no-lead rule. Reading 13 at slot 6 shows +the closure two native frames late, which is the same lateness every hold already +has. + +**The marks come from a cut that is already stored.** `flow/freeze` computes both +closures and keys them as `[:vis]`, with the thresholding and hysteresis already +decided: + +- the mouth, from the aperture relative to the take's peak — `[:vis]` on + `:mouth-in`, provenance `:roto/mouth-aperture` (`freeze.cljs:473`); +- the eyes, from `condition/resolve-blink` with its cut, dwell and hold — `[:vis]` + keyed per eye part, provenance `:roto/blink` (`freeze.cljs:533`). + +So "preserve the frames the cut says shut" reads what the document already holds. +No new signal, no new dense track, no threshold decided twice. Brows have no +closure and get no marks, which is correct: there is no extreme brow position +worth protecting. + +**Precompute the mark set when the resolver is built**, the way `pose/prepare` and +`trace/prepare` already do (`symbol.cljs:420`, `:470`, `:536`). A `[:vis]` channel +is keys, not dense, so reading it per node per frame would be cheap — but the snap +also needs the marks sorted for a backward lookup, and building that per frame is +the one thing [animation-model.md](animation-model.md) and the render-path note +above both forbid. + +### Where it goes, and why it is not a one-liner + +The rule needs two things that currently live at opposite ends of the resolver: + +- **the slot interval** `(d(k-1), d(k)]`, which needs the output slot index and the + fps ratio. Both exist at `clip.cljs:261`, the single place the grid becomes a + native frame: `(cadence/frame f (or (:grid-fps opts) (:fps clip)) (fps clip sid))`. +- **the marks**, which are per pose group, and so belong where group identity + exists: `symbol/base-channel-frame` (`symbol.cljs:399`), whose `:pose-sampled?` + branch already calls `pose/source-frame` with `(js/Math.floor lf)` as the default + pose. **That default is the snap's seat.** Replacing it leaves an explicit hand + cut winning over a snap, which is correct — manual precedence is absolute — and + costs no new plumbing on the pose side, because per-group choices are already + threaded and already prepared. + +By the time control reaches `base-channel-frame` there is only `lf`, a native local +frame that placement and retime have already been through, so the slot interval +cannot be recovered there. It has to be threaded down from `clip.cljs:261` +alongside the frame. That is the actual work of this step: two namespaces' internal +signatures, not a drop-in. + +**Do not take the shortcut of snapping at `clip.cljs:261` itself.** It is right +there, it needs no threading, and it is wrong: one native frame per output frame +means the *whole picture* reads 13 instead of 15, so the head goes two frames stale +for one output frame to fix the mouth. At 12fps that is a 67ms hitch on a moving +head, and it fights the trace selection, which has its own opinion about which head +frame to show. The snap is per group because the thing being recovered is one +group's closure. + +### Why not an aperture signal + +An earlier draft had this side read the group's aperture as a single-component +signal and hand it to `propose` with `:protect :extrema`. It cannot. The aperture +is the separation of landmarks 13 and 14, at positions 5 and 15 of the 20-slot +`LIPS-INNER` ring, and `freeze/rings->flat` subsamples the ring to the `verts` +budget: those two positions survive only when `verts` is a multiple of four. At +`verts` 6, 10, 14 and 18 — all legal, all even — they are not in the stored data +at all. `ring/subsample-slots` used to claim otherwise and has been corrected. + +`flow/measure/mouth` does compute the exact scalar and `freeze` does throw it away +after thresholding, so storing it was an option. Reading the cut is strictly less +work and decides nothing twice. + +## How the two interact + +They are keyed in **the same frame space**: `clip/resolver` hands an instance's +`:playback :tracks` down into the child it places, and `symbol/base-channel-frame` +reads both the trace and the pose choices at `lf`, the node's local frame inside +that face. A trace frame and a pose cut are the same kind of number. + +What differs is the owner, and that asymmetry is load-bearing: + +- the **trace** is the face's, on its `:head` — every instance of that face shares + it, because it says how the drawings were made; +- the **pose tracks** are the instance's — two placements of one face can be + timed differently. + +### The hazard, which is already written down + +[animation-model.md](animation-model.md) states it for exposure and it is the +same hazard here: + +> a head cutting on odd frames against a mouth cutting on even ones reads as two +> performances + +**It is not a correctness problem.** The mouth is a child of `:head` and its +geometry is stored head-local, so a mouth from frame 17 composed onto a head held +at frame 12 is exactly lip-sync on a held drawing — the decomposition already +decoupled them and nothing is geometrically wrong. The problem is perceptual, and +perceptual problems want a constraint rather than a repair. + +### Nest them, do not couple them + +The two are not peers. One is coarse and expensive — the prototype's own comment +says it: *"The cost being managed is an artist drawing a head, which is why the +signal is head pose and not the mouth — the mouth is traced and free."* The other +is fine and cheap. + +So the rule is a subset, in one direction only: + +**Every kept plate frame is a preserved frame of the performance selection.** + +When the drawing changes, the performance changes with it, so the two can never +cut against each other on neighbouring frames. The mouth stays free to change on +frames where the head does not, which is what shooting a held drawing with a live +mouth *is*. + +This is why the preserve set is a set and not a closure predicate: plate frames +and shut-mouth frames go into the same pile, and the snap does not care which is +which. It needs no new mechanism on either side. + +The reverse is a suggestion and never automatic. A mouth closure is a reasonable +place to want a new drawing, but proposing one spends somebody's afternoon. Offer +it; do not take it. + +It also composes with `:origin` for free. A head on `:continuous` has opted out +of its own selection, so there are no plate frames to preserve and the constraint +is vacuous — which is correct, because a continuously moving head cannot cut +against anything. + +### Between the kept frames is a third shared field + +The head's `:reads` (once `:trace :origin`) is not a tracing setting. It is the answer to *what happens +between kept frames*, and the plate selection has to answer it: + +- `:continuous` — ignore the selection for this purpose and read the frame you + are on, +- `:keys` — jump to each kept frame and hold it, a hold and not a tween, +- `:start` — hold the first forever. + +The performance side does not need the field: a snap picks which frame a slot +reads and the grid does the holding, so `:keys` is the only behaviour there is. +Do not rename `:origin` to match anything: for a head it genuinely means where the +face's origin goes, and a saved field in a shipped UI is not worth churning for a +vocabulary tidy. + +## The modes + +One setting, two states, and **"manual" is not a third state.** + +- **off** — the cadence alone, which is what ships today. The output frame + reads the latest native frame at or before it, and nothing has an opinion. +- **smart frame picking** — on the plate side the proposal is live, and `:policy` + holds the tolerance; on the performance side the snap is active. + +Hand keeps and drops apply in **both** states, which is why they are not a mode: +turning smart picking off must not throw away the frames somebody pinned, and +pinning a frame with smart picking off is a perfectly reasonable thing to want. +Manual precedence is absolute — a drop beats a proposal, always. + +The name on the toggle should be the same word in both sections. "Smart frame +picking" is fine. What it must not be is two different names for the one idea, +which is how these became two features the first time. That the two sections are +now backed by different code is an implementation fact and must not reach the UI. + +## The UI + +One component rendered twice, in `ui/params.cljs`: + +``` +smart frame picking [ off | on ] +tolerance [ ----•------- ] 0.02 (plate section only) + [ Suggest ] +frames 0 12 30 47* 61 (* = kept by hand, strikethrough = dropped) +``` + +The frame strip already exists in miniature — `params/trace-keys` draws the trace +keys as seek buttons (`params.cljs:386`). Lift it into a shared component and +give it three affordances: click to seek, a modifier to pin, a modifier to drop. +A pinned frame and a proposed frame must be visually distinct, because "will this +survive me moving the slider?" is the question the strip exists to answer. + +Render it in two sections: + +- **`tracing · `** (`params.cljs:426`), beside the existing origin row, + with the tolerance slider and Suggest. +- **`performance · `** — new, on an instance's inspector, one per pose + group the placed symbol has. No tolerance: the strip shows the preserved frames + and the grid slots that snapped to them. + +## Order to build it + +1. **`domain/select` with the greedy walk, and its tests.** **Done**, in commit + `02069e8`. Pure, no store, no clip. Note that the extrema part of it is not + wanted by anything below — see *What is revertible*. +2. **The preserve-snap**, pulled forward ahead of the plate side because it is the + half with no UI and nothing proposing today, and because it needs nothing from + the plate side except a fold that can land last. Three pieces: the mark set from + the stored `[:vis]` cuts, built in a prepare step beside `pose/prepare`; the + slot interval threaded from `clip.cljs:261`; the rule itself, seated in + `base-channel-frame`'s default pose. The test that matters is the same synthetic + case `select_test` uses — a one-frame closure at native 13 that a 12-from-30 + grid drops — asserted end to end this time: the resolver shows a shut mouth on + exactly one output frame, and the head's frame does not move while it happens. + Write it first. +3. **The plate signal reader.** `trace/head-signal`: make `trace/measured-local` + public, map it over the frames, four corners through `node/apply-pt!`. Test + that it returns a vector of the right length, that a motionless take proposes + `[0]`, and that a stretch where the face was not found becomes `nil` rather + than a pose, a zero or a gap in the vector. Add the held-reconstruction + invariant from *The upgrade path* here, since this is the first place a real + signal exists to assert it over. +4. **Storage for the plate selection.** `:policy`/`:keep`/`:drop` beside + the plate's `:time :holds`. Extend `leaf/leaves` and the key whitelists in the same + commit — a field without a leaf saves silently and comes back missing, which + is the one bug persistence must not be able to have. Round-trip test. +5. **Re-suggest preserves hand decisions.** Propose at one tolerance, pin a frame, + drop a frame, propose at another, assert both survive. Settle here whether the + hand's `:keep` is also fed to `propose` as `:protect`: under the layering as + written it is not, so a pinned frame does not re-anchor the walk even though it + is on screen, which contradicts the anchor rule above. It is the one live + caller for `propose`'s `:protect` frames. +6. **Fold the plate frames into the mark set**, which is the nesting rule and is + one line once both sides exist. +7. **The shared UI component**, then its two mountings. + +## What is revertible, and where it is + +Commit `02069e8` contains one part that **nothing below asks for**: extrema +detection. Specifically `segments`, `turns`, the `:extrema` branch of `propose`, +the multi-component refusal that exists only to guard it, and three tests — +`a-one-frame-closure-survives-only-because-it-is-protected`, +`extrema-are-turns-worth-more-than-the-tolerance` and +`extrema-are-refused-on-a-multi-component-signal`. + +It has no caller because plate selections take no extrema by design — displacement +is the whole story for a head — and the performance side reads a stored cut +instead of finding extrema in a signal. It is correct, tested and speculative. + +Revert it if the shape above holds. Keep it if either of these turns out to be +wanted: a group whose extreme is not a closure and therefore has no `[:vis]` cut +to read (a mouth at its widest, a head at the top of a nod), or a take whose mouth +never shuts far enough to cross `aperture-cut`, where a peak-relative threshold +marks nothing and an extremum would still find the most closed frame. Neither is +asked for today. The DP in *The upgrade path* gets extrema for free regardless, so +reverting costs nothing that cannot be had again more cheaply. + +## Traps + +- **Do not let Suggest write `:keep`.** The proposal and the hand are different + layers; collapsing them is the bug this whole shape exists to avoid, and it + will look like it works right up until somebody moves the tolerance slider. +- **Do not snap a grid slot forward.** Every hold and every pick is "at or + before". Showing a closure before the mouth shut is a lead, which is a different + control for a different reason. +- **Do not quantise a plate selection onto the output grid.** A kept frame is a + native frame and lands where it lands. [time.md](time.md) is explicit that an + event between output frames appears on the next one; forcing kept frames onto + the grid would re-create the problem the selection exists to solve. The snap is + the opposite operation and is on the other side of the fence: it moves the grid's + pick, never the kept frame. +- **Do not read the aperture pair off a subsampled ring.** Positions 5 and 15 of + `LIPS-INNER` survive only at `verts` divisible by four. +- **Do not thin the plate selection with the mouth's.** Two scopes, two owners, + and the nesting runs one way only: plate frames preserve performance frames, + never the reverse. +- **Do not give the performance side a tolerance.** The grid has already chosen + the sparseness. A second knob there would be a control with nothing to control. +- **A skipped frame, a hidden feature and an absent measurement remain three + different facts.** A selection says nothing about visibility and nothing about + whether a face was found. Note that the preserve marks are *derived from* a + visibility cut, which makes this easy to blur: the cut says the interior is + hidden, the mark says the frame is worth landing on, and one is not the other. + +## Not in scope + +- Automatic *grouping* of related parts beyond the existing `:pose-group`. + Related parts must share one selection — a mouth outline, its interior, the + teeth and the generated visibility reading different frames is the bug that + grouping prevents — and `:pose-group` is where the grouping already lives. +- A per-instance request for a different tolerance than the symbol's. The + resolver threads no such option today and should not grow one until something + needs it. +- Variable frame rate. [time.md](time.md) assumes constant fps and so does this; + a VFR source needs presentation timestamps before any of this means anything. diff --git a/docs/lane-handoff.md b/docs/lane-handoff.md new file mode 100644 index 0000000..ed14944 --- /dev/null +++ b/docs/lane-handoff.md @@ -0,0 +1,247 @@ +# Lane and symbol-clip handoff + +Status (2026-10-01): the timeline is the one timing interface. A lane is a +generic non-overlapping row of symbol clips; it is not a special drawing type. +Dropping a library symbol makes a naturally playing clip, while creating a new +empty symbol makes a one-frame held clip at the playhead. With no destination +lane, either operation creates one. Existing legacy root symbol rows can be +dragged into a lane. Blocks move by mouse; edge drags claim time by trimming +neighbors; Shift-edge drags ripple every later clip; and the center of a shared +cut composes the two edge edits into a rolling edit. Linked audio follows picture +moves while its edges remain independently trimmable. + +The commits beginning at `3d3c1bb` are the argument for the model and are worth +reading before touching what they did — they are the design record, more than +this file is. + + 3d3c1bb An occurrence is a node, with a clock of its own + 9446829 Reuse, duplicate and make unique: deciding what is shared + 26517af A position is an argument, not another command + 94c0a21 A correction is a layer, and a layer's values are a channel + 72b57e3 Regenerate the base, keep the hand work, and say when you cannot + 76106d3 The shot is as long as somebody said it was + 598c186 One word for one thing: it is a cel + +## Read first, in this order + +1. [The Lane Model](lane-model.md) — the design, and the status note under + *Proof obligations* says what is built. It supersedes `animation-model.md`, + `timing-model.md` and `architecture.md` wherever they overlap. +2. `frontend/src/arthur/domain/lane.cljs` — every command, and the reasoning in + its docstrings. +3. `frontend/test/arthur/domain/lane_test.cljs` — what the model is asserted to + do. It is the fastest way to see the shapes. +4. `frontend/src/arthur/domain/channel.cljs`, the correction-layer section. + +## Vocabulary — one word for one thing + +Renamed in `598c186`, after four words had accumulated for one object. Use these +and do not reintroduce the others. + +| word | means | +| --- | --- | +| instance | the `:kind`. The general thing, anywhere in a document | +| clip | an instance in a lane. It can hold one source frame or play a symbol naturally | +| cel | specifically a one-frame source held over a clip's duration; the empty-symbol/drawing creation policy | +| lane | a group with `:layout :sequence` | +| drawing | content authored into a symbol; not a different timeline node type | +| placement | ONLY where a node sits: `nest/placement`, and the transform that puts a face on the stage. Never the node itself | + +`occurrence` and `exposure` are not words for a cel. **`exposure` means something +else and still does**: `:time :expose` is how many frames each step of a subtree +lasts, which is what shooting on twos is — `node/expose`, `clock/exposed-frame`, +`subs/render ::exposure`. Keeping these apart is why the block is called a cel. + +`:layout :sequence` stays as the field, and is the one place two words are kept +on purpose: the layout names the RULE — children follow one another and may not +overlap — and a group carrying it is called a lane. `node/lane?` is where they +meet. + +## Decisions already made — do not re-litigate + +These were each argued out and are load-bearing. Changing one is a design +decision, not a cleanup. + +- **The shot length is authored.** `:frames` is the symbol's window; the + occupied extent of its lanes is a different fact derived from the cels. A + command grows the window only when the caller passes `:extent :grow-symbol`, + and never shrinks it. Blanking the end of a shot leaves empty frames at the + end, because deriving the window from the extent would make deleting the last + drawing silently shorten the film. `lane/finish`. +- **Placing ripples; overwrite is `blank` then non-rippling placement.** + `lane/overwrite-drawing` composes those pieces as one transaction. Insertion + retains its ripple rule; overwrite does not move any surviving cel. +- **A position inside a cel refuses and names `split`.** One command must not + quietly perform two. The UI offers the retry. +- **A correction has no time space of its own.** Its `:support` and its values' + keys are in the frames the base channel's keys are in — the node's. A + correction on a lane is in lane frames and reaches across the drawings under + it; one on a cel travels with that cel. Ownership already answered it. +- **A layer's values are a channel.** Constant, ramp and return motion are one + mechanism. Do not add a second way to say what a value is over time. +- **A conflict is not a `problem`.** A document whose topology outgrew a + correction loads, evaluates and saves; `clip/conflicts` lists the decisions + waiting for a person. `problems` means the document will not load. +- **Refuse rather than guess.** Every command returns `{:clip :selection}` or + `{:refused why}`, never a half-applied edit. Where the model needs a choice + nobody has made, refusing and saying why is the behaviour, not a placeholder. +- **A clip is not a row.** Rows, expansion and selection are editor state. The + document has never known about rows and must not learn — which is what let + the row model change three times in one sitting (blocks, then a + selected-clip portal, then sound lanes under the audio heading) without + touching a single document. +- **A lane is generic.** Drawing creation, library placement, and adopting an + existing root instance all produce the same child instance shape. The only + difference is playback policy: a new empty drawing holds source frame zero; + a dropped library symbol plays at speed one. +- **Lanes are explicit.** A symbol may contain ordinary overlapping children + without a lane. Selecting a lane row opts creation into its claim-time + behavior; selecting a cel enters that cel's source symbol instead. +- **Placement claims time.** Lanes never store overlaps. A new or extended clip + trims, removes, or splits whatever previously owned the claimed interval. + Real compositing overlap uses another lane, where ordering remains explicit. + +## The timeline opens the whole document + +Expanding a lane opens exactly one clip — the selected one — and that portal +opens the lanes and nodes of the symbol it places, recursively, mapped into +the open symbol's ruler. The portal follows the LINEAGE of the selection, so +working on something nested keeps the rows that revealed it open. A held clip +opens too, with its rows marked `:unmapped?`: shown across the hold, with no +keys and no draggable edges, because a frozen clock gives its frames no place +on this ruler. `docs/lane-nesting-notes.md` has the reasoning and what is +still missing. + +Double-clicking a clip opens the symbol it places as a tab, the same as +double-clicking that symbol in the pool. Shift while dragging a clip body +turns the temporal move into a structural one — see the nesting notes for why +that is mostly refused today. + +## Current timeline interaction + +- Creation follows the primary active row and the playhead; the complete rule is + in `docs/creating-in.md`. A lane row creates a new cel and claims its interval, + while selecting a cel creates inside the symbol that cel places. +- Drawing with a lane row active creates a new one-frame drawing cel. Dropping a + library symbol there creates a natural-duration playing cel. Explicit timeline + drops use the row and frame under the pointer. +- Dragging a clip body moves it. A linked audio node follows a picture move; + moving or trimming the audio itself remains independent. +- Dragging a right edge changes its endpoint. Growth consumes adjacent spans + instead of overlapping them. Shift-drag inserts or removes lane time by moving + every later clip by the same delta. +- At a shared boundary, the left and right hit zones trim one side. The center + is a rolling edit: right-edge resize followed by left-edge resize at one frame. +- Split, trim-in, and trim-out are direct buttons and are disabled without an + editable selected span. Movement is a mouse gesture, not a toolbar command. + +## Next steps, in order + +The implemented correction slice and its remaining UI limits are recorded in +[Correction authoring](correction-authoring-plan.md). + +1. **Slip source and retime.** Both have real design questions open and the doc + says to refuse rather than approximate: retime needs a defined warp and + interpolation behaviour, and is not moving keys whose numbers happen to fall + inside a selection. +2. **Deleting reused content.** Reference discovery exists (`node/sources`, + `clip/places`, `clip/contains-symbol?`); the policy does not. +3. **Displayed-range correction gestures.** The first correction panel asks for + explicit owner frames. Dragging a range in a retimed/nested view still needs + a proved mapping; do not make it snap through floors or loops. +4. **Collaboration.** `lane-model.md` is explicit that one leaf per channel does + NOT solve two people editing different keys of the same channel. No conflict + policy exists for that. + +## Mechanisms to reuse — these keep paying out + +- **`:span` is in the node's OWN frames** and `:time` says where they land in + the lane. Moving an edge of a cel is therefore one write to `:span`, with + `:time` and `:playback` untouched. This is why split costs nothing, why the + two halves of a split go on meaning what the one cel meant, why trimming the + front of a playing insert starts it later into its animation instead of + restarting it, and why extending a hold leaves lane keys alone. `lane/local` + and `lane/edged` are the whole geometry; trim, split and blank are all it. +- **`lane/finish`** is the one commit path: it validates, applies the shot-length + policy, and returns the refusal. New commands go through it. +- **`:required-frames` plus the retry event** is the pattern for "this needs a + decision you have not made": the domain reports what it would need, the UI + offers one button. `events/ui/lane-retry`. +- **`lane/lane-frame`** converts a symbol frame to a lane frame, or returns nil + through a stepped or looping lane where there is no single answer. Nil refuses; + it never snaps. +- **`channel/conflict-with`** is the rule for whether one offset fits a base, + used by `conflicts` and regeneration. Validation additionally follows prior + replacement layers, so it cannot approve a stack that throws when read. +- **Generated sampling applies to the base, not the hand correction.** Picture + rate and pose selection may choose an earlier generated frame; correction + support and values still read the node's current authored frame. +- **Correction commands live in `domain/correction.cljs`.** IDs come from the + event caller; the pure command materializes default transform channels, + appends one layer, and validates the complete document. The inspector authors + rotation and position offsets in explicit owner frames. One Apply is one undo + step. `channel/reconcile` is the shared ordered-stack compatibility rule used + by validation, conflict reporting, and regeneration. +- **Two test patterns worth copying.** `the-cursor-agrees-with-the-specification-in-any-frame-order` + holds the optimized cursor to `value-at` in forward, backward and random order + — add a case to it for any new channel shape. And `drawn` in `lane_test` + samples every frame before and after an edit, which is how split and trim are + proved to change nothing: state a claim as "the same picture" rather than as + numbers computed by hand. + +## Known gaps and traps + +- **Audio is a clip in a lane too, and a lane holds one kind.** A sound placed + from the pool lands in a lane and is moved and trimmed by the same commands + as picture. The capability the earlier note asked for is the homogeneity + rule rather than a field: `symbol/lane-problems` refuses a lane holding both + kinds, and `lane/place-symbol` and `lane/adopt` refuse BEFORE claiming time, + because placement claims time and would otherwise have deleted the sound to + make room for the picture and left a valid document behind. Audio nested + inside a placed symbol — a take's own sound — is still shown flattened by + `nest/audio-tracks`; what is in a lane of the open symbol is drawn as a lane + and not flattened twice. +- **`:z` is required on cels and means nothing there.** A lane never has two + cels on one frame, so draw order between them cannot matter. `node/problems` + requires `:z` on every node uniformly, which is its own kind of simplicity — + but the field is noise on a cel. +- **`channel/offset-onto` throws** on a shape mismatch that no regeneration has + recorded as a conflict. That is deliberate — a correction that silently does + not take is the failure the design exists to prevent, and `channel/problems` + catches the authored case — but it is a throw in the read path, so any new + producer of layers must not create a mismatched one. +- **`docs/timing-handoff.md` is a separate, unreconciled thread.** Performance- + pose selection and plate drawings/tracing, instance-specific picture-rate + requests, `pose/put-cut` addressing only `:main`. It predates the lane model + and nobody has squared the two. +- **Slip and retime are still absent.** The timeline action strip now applies + split/trim uniformly to a selected root or lane clip, but source-time slip and + retime still need their own proved semantics before they become controls. +- **`shadow-cljs release app` clobbers the dev bundle.** Both builds write + `../static/arthur/js`, which Django serves, and the optimized build does not + export the `arthur` global — so after a release the browser tests fail with + `ReferenceError: arthur is not defined`. Run `npx shadow-cljs compile app` to + restore it. A running `watch app` does not notice; it rebuilds on the next + source change. + +## Running it + +From `frontend/`: + + npx shadow-cljs compile test && node out/node-tests.js # 469 tests, 9,592 assertions + npx shadow-cljs compile app # the bundle Django serves + npx shadow-cljs release app # then `compile app` again — see above + +The browser tests need the Django dev server up (`mise exec -- python manage.py +runserver 8778` from the repo root) and a compiled dev bundle: + + node --experimental-websocket test/browser/lane.mjs # generic symbol-lane flow + CHROME=/usr/bin/chromium node --experimental-websocket test/browser/take.mjs + +`take.mjs` defaults to a macOS Chrome path, hence `CHROME=`. It writes a real +project to the local server by design; `lane.mjs` never writes to the server. + +From the repo root: `mise exec -- python manage.py test clips` — 56 tests. + +Documents are schema 3. A version 2 document is not read and nothing converts +one; there is no backward compatibility to preserve anywhere in this work. diff --git a/docs/lane-is-a-view-notes.md b/docs/lane-is-a-view-notes.md new file mode 100644 index 0000000..92e2b2c --- /dev/null +++ b/docs/lane-is-a-view-notes.md @@ -0,0 +1,71 @@ +# Implementation notes — "A lane is a view" + +Running log for `docs/lane-is-a-view-plan.md`. `[ ]` not started, `[~]` in +progress, `[x]` done with `npm test` green. + +Baseline at `fb38990`: 475 tests, 9621 assertions, 0 failures. + +## Order of work + +The plan's seven steps, re-grouped — see *Deviation from the plan's order* below. + +- [x] A. Domain: `symbol/children`, `symbol/lane?`, `symbol/overlaps`, + `lane.cljs` → `span.cljs`, the overlap check in `span/finish` + (plan steps 3, 4, and the domain half of 5) +- [x] B. Events: re-base callers and remove lane-node-specific commands; keep + explicit `::new-lane` and cross-lane adoption (plan step 5) +- [x] C. UI: row per symbol, explicit lane creation, and drag handling + (plan steps 1, 2, and the UI half of 5) +- [x] D. Audio: delete `holds-other?`, the mixed-lane refusal, the `in-lane` + filter in `sound-rows` (plan step 6) +- [x] E. Tests: `domain/sequence_test`, `events/lane_test`, `browser/lane.mjs` +- [x] F. Shift-to-reparent still works, untouched (plan step 7) + +Not in this pass — see *Left for a second pass*: tearing out the held cel, the +instance-playback control, drawing a loop's repeats, the audio period guard. + +## Deviation from the plan's order + +The plan's steps 1 and 2 are display work that keys off "a symbol's children", +and step 3 is what MAKES the cels a symbol's children. Until then a cel's +`:parent` is the lane node, so there is nothing for the display to read: step 2 +cannot draw "a symbol's children as blocks" while the children belong to a +group. So the data model moves first (A) and the display follows (C). The +content of each step is unchanged; only the order is. + +The one thing this gives up is the plan's promise that every step leaves the +editor usable — between A and C the timeline draws the new shape with the old +code. `npm test` is green at each step either way. + +## Decisions taken + +1. **Lane mode is `:display :lane` on the SYMBOL** — plan's recommendation 2, + and open question 1 answered "the symbol, not the instance". A symbol placed + twice is drawn as a lane in both places. Added to `symbol/symbol-keys` and to + `leaf/leaves`' `select-keys` so it saves like `:frames`. +2. **A symbol's children are its parent-less nodes** that have a placed span. + The plan's step 6 settles it: "an audio node is already a parent-less child + of a symbol, which is exactly the new shape". Span-less nodes — a shape on + screen for the whole shot — are not in the sequence and are skipped, which is + also what stops the commands destructuring a nil span. +3. **A symbol holds at most one sequence.** It follows from 1 and 2: the + container is the symbol. Two lanes of picture is now two symbols placed in a + third, which is what compositing already was. +4. **The open symbol gets a row of its own in lane mode**, and only then. The + blocks have to sit on a row and the open symbol had none; expanding it turns + its children into ordinary rows. Not a row always, which would shift every + row in the pane for no gain. +5. **Open question 2** — a lane row's edge drag trims the PLACING INSTANCE's + span, via `span/resize-out`, like the handle on every other row. Rippling + the children is what the cel blocks' own edges already do, and giving one + handle two meanings is what the plan refuses elsewhere. +6. **Open question 3** — the lane work first, the held cel after. The plan says + they are independent, and the held cel is joined to a loop control that does + not exist yet; doing it second costs one more pass over `lane_test`'s + fixtures and risks nothing. +7. **Lane creation stays explicit.** A blank document and `new symbol` create + ordinary symbols. The separate `new → lane` command creates and places a + symbol with `:display :lane` in the effective creation target derived from + selection and playhead; the new lane then becomes the primary selection. + +## Notes diff --git a/docs/lane-is-a-view-plan.md b/docs/lane-is-a-view-plan.md new file mode 100644 index 0000000..cc136d6 --- /dev/null +++ b/docs/lane-is-a-view-plan.md @@ -0,0 +1,301 @@ +# A lane is a view + +Plan, 2026-10-01, written at `2dc5735`. It undoes the lane model as a thing in +the document and keeps what it was for. Build on what is there and tear out +half of it. + +## The decision + +> A lane is a view over a symbol with sequential, non-overlapping children. + +Nothing in the document is a lane. There is no lane type, no lane group, no +`:layout :sequence`, no lane commands and no lane validation. The word +survives in exactly two places: the UI, where a symbol can be DRAWN as a lane, +and the drag handling that re-spans a symbol's children while it is being +drawn that way. + +The display model goes back to a row per symbol. A symbol in lane mode draws +its children as blocks on its own single row; expanded, they are rows like +anything else. Everything else is an ordinary row that expands into what it +places. + +## What a lane was, and what each part becomes + +| was | becomes | +| --- | --- | +| a group node with `:layout :sequence` | nothing — the symbol is the container | +| `node/lane?` | a view question: is this symbol drawn in lane mode | +| `symbol/lane-clips nodes lane-id` | the children of a symbol, sorted by `node/placed-span` | +| `symbol/lane-problems` | `symbol/overlaps`, a diagnostic the write path calls | +| `lane/lane-frame` | `clip/source-time` — one clock instead of two | +| `domain/lane.cljs` | re-based onto `domain/span.cljs`: re-spanning a symbol's children | +| `clip/lane-node`, the born-with lane | gone; a symbol is born empty again | +| `::ui/new-lane`, `::ui/adopt-in-lane`, lane renaming | gone, gone, and ordinary node renaming | + +`lane.cljs`'s fourteen commands are not deleted — they are what "endpoint drag +overlap handling" means, and they already do the right arithmetic. What +changes is their subject: every one of them currently takes a host symbol AND +a lane id and asks `lane-clips nodes lane-id`; each takes a symbol and asks +for its children. `extend-hold`, `resize-out`, `resize-in`, `roll`, `blank`, +`place-symbol`, `adopt`, `append-drawing`, `reuse-drawing`, +`duplicate-drawing`, `overwrite-drawing`, `make-unique`. `span/finish` is +already the one commit path and stays exactly as it is. + +Put them in `span.cljs`, which already owns "one write to one node's span" and +`finish`. The sequence operations are the same subject — re-spanning children +— and keeping them apart was a consequence of lanes existing. + +## Children in lane mode never overlap + +This is an invariant, not a condition to check for and report. Placement +claims time: anything placed, moved or grown over occupied time TRIMS the +extents it lands on — trimming the incumbent, removing one wholly covered, or +splitting one it lands inside — so the result has no overlap because the +operation that could have made one did not. That is `blank` followed by a +non-rippling placement, which is what `overwrite-drawing` already composes. + +Enforced at the boundary, which already exists: `span/finish` is the single +commit path for every one of these commands, it validates before it returns, +and it refuses rather than half-applying. So `finish` gains the overlap check +for a symbol in lane mode, and no command can commit one. An overlap that +appears anyway is a bug in a command, not a state to design around. + +Keep the check as a named diagnostic — `symbol/overlaps`, taking a symbol and +returning the pairs — used three ways: + +1. `span/finish` refuses when it would commit one. +2. The test suite asserts no command can produce one: a property over the + commands in the style of `drawn` in `lane_test`, which samples rather than + computing expected numbers by hand. +3. A document that somehow arrives holding one still LOADS — a display hint + must never be able to stop a document loading — and the timeline draws it + visibly wrong with the status line saying so. Not `clip/problems`, which + means the document will not load, and not `clip/conflicts`, which means a + person has a decision to make. This is neither: it is a bug report. + +Toggling lane mode ON for a symbol whose children already overlap is the one +place a person can ask for the impossible. Refuse it and say why, with the +`:required-frames` retry pattern offering to trim them into a sequence — the +domain reports what it would need, the UI offers one button. + +Outside lane mode nothing is enforced, because overlapping children are what +compositing IS. An endpoint drag there is an ordinary span edit that may +overlap; the claim-time rule follows the mode. + +## The two drag intentions + +Unchanged from `docs/lane-nesting-notes.md`, and both kept: + +- **Plain drag** of a clip body is temporal: it moves in time, within its + symbol or into another symbol drawn as a lane, and it REPLACES — trimming, + removing and splitting extents as needed so nothing overlaps. +- **Shift-drag** is structural: the dragged node goes INSIDE the symbol the + clip under the pointer places, through `nest/move-node`, which preserves the + world transform and the root timing. This must keep working for symbols + contained in a lane, which is the case it exists for. + +Overlap cannot distinguish them — dropping on occupied time already means +claiming it — so the modifier says which, and the label by the pointer says it +back. `nest/move-refusal` already answers before the drop. + +## Where lane mode lives + +A symbol is drawn as a lane because somebody said so, not because of what its +children happen to look like at this moment. Deriving it from "the children do +not currently overlap" means a symbol stops being a lane the moment anything +overlaps, and the rules that maintain non-overlap switch off exactly when they +are needed. + +Two options: + +1. **Editor state**, `[:ui :lane-mode #{sid}]`. Purest reading of "a lane is a + view". But the drag rules follow the mode, so an unsaved, per-person toggle + would decide whether dropping a symbol trims its neighbour or composites + over it — the same gesture doing two different things to the document + depending on something the document does not record. +2. **A display hint on the symbol**, e.g. `:display :lane`, saved like any + other field (`clip-keys`, `leaf/leaves`, `leaf/clip` in the same commit). + Still not a type: nothing in evaluation reads it, `symbol/problems` does + not check it, and a symbol with it set behaves identically on the stage. + +Recommended: 2. It is one field, it keeps editing rules reproducible between +people, and it does not make the symbol a different kind of thing. The thing +to hold the line on is that nothing outside the timeline, and the commit +path's overlap check, is allowed to read it. + +## Two things are called loop + +Before any of this, name them apart, in the way the vocabulary table in +`lane-handoff.md` names a cel apart from an exposure. + +- **Loop playback** is the transport repeating the open symbol while it plays. + It is `[:playback :loop?]` in app-db, the ⟳ button in the strip, and it is + EDITOR STATE. The document does not know about it. +- **A looping instance** is a node repeating the symbol it places: a four-frame + tire turning for the hundred and twenty frames the instance is on screen. + It is `:playback {:end :loop}` on the node, and it is in the DOCUMENT. + +The car tire is the second one, and here is its actual status: it already +works in the evaluator and cannot be asked for. `node/placed-frame` does the +modulo, `node/problems` already admits `:end` of `:stop`, `:hold` or `:loop`, +`nest/audio-tracks` already expands a loop into its periods — and nothing in +the UI sets it. It is implemented and unreachable. + +So three things are missing, and they are the work: + +1. **A control.** Where an instance's playback is edited: `:in`, `:speed`, and + what happens at the end. One place, three fields, rather than a loop + checkbox somewhere else. +2. **Drawing the repeats.** `ui/timeline`'s docstring already admits that only + the first pass of a looping instance is drawn, so a tire turning thirty + times shows one turn's keys and then nothing. A looping block should show + its passes — at minimum the period boundaries, so the row says how many + times round it goes. +3. **The audio period guard** below, which a loop needs whether or not holds + become loops. + +## Tear out the held cel + +A held cel is a 1-frame symbol shown for many frames. A 1-frame symbol with +`:end :loop`, lengthened, is the same picture by a different route — and the +second route is a case the model already has, so keeping the first one is +keeping a special case for free. + +**What is already true**, so that nothing has to move: `:span` is on the node, +in its own frames; `:time` (`:at`, `:rate`) is on the node; looping is on the +node, as `:playback :end`. The SYMBOL owns only `:frames`, the authored +window. So looping and span are instance properties already, and this change +is about deleting a mode, not relocating a field. + +It does mean the two changes are joined at one point: making every drawing a +looping instance is not safe until a looping instance can be seen and edited, +or every drawing in the document acquires a property with no control on it. + +**What has to be decided and collapsed:** + +- There are two spellings of looping — `:time :loop?` and `:playback :end + :loop` — and both are read, in `node/placed-frame` and in + `nest/audio-tracks`. Keep one. `:playback {:in :speed :end}` already says + what happens at the ends, so `:end :loop` is the one to keep and + `:time :loop?` is the one to delete. +- `:playback :speed 0` stops being produced. Make it illegal in + `node/problems` rather than legal-but-unused, so a frozen clock has exactly + one spelling: a 1-frame loop. +- `lane/extend-hold` exists only because holds were special — it refuses + anything whose speed is not 0 and then edits a span. Once a hold is a loop, + lengthening one IS `resize-out`, and the command collapses into it. +- `cel` survives as the word for the creation policy — a new empty symbol is + one frame — but stops naming a playback mode. + +**What it buys, and this is the point:** one rule for nesting, which settles +the refusal that blocks shift-to-reparent today. `nest/inside` currently has +no `:time` for a hold, for `:end :hold`, or for a loop, and so refuses all +three. The general rule that covers all of them: **resolve the move with the +destination's map at the CURRENT frame — the affine piece the current frame +falls in.** + +- A loop of length L is affine within the period the current frame is in. + Timing is preserved inside that period and repeats after it, which is what + looping means. +- A 1-frame loop — the ex-hold — has a period of length 1, so the map within + it is trivially invertible and lands the moved node on frame 0, aligned to + the current frame. That is exactly the answer `docs/lane-nesting-notes.md` + argues for from first principles, arrived at here as an instance of the + general rule instead of a special case. +- `:end :hold` is affine in the played part and frozen in the tail, which the + same sentence covers. + +**The trap, which must be handled in the same change.** `nest/audio-tracks` +expands a loop into one walk PER PERIOD: + + periods (range (floor (/ (to-local source lo) length)) + (ceil (/ (to-local source hi) length))) + +Today a held cel is skipped entirely — `(pos? speed)` is the guard, and the +comment says a visual freeze does not emit a sustained audio sample. Turn +every drawing into a 1-frame loop and that guard stops firing: a drawing held +for 120 frames becomes 120 recursive walks, and any sound inside it is emitted +120 times. That is both a wrong mix and a performance cliff on the most common +node in the document. Required with this change: a cheap `symbol/audible?` +precheck so a source with no audio anywhere inside it is never period-expanded, +and a cap or a different formulation for the ones that are. + +Also worth knowing before the change: `ui/timeline`'s own docstring already +says a looping instance draws only its first pass. With every drawing a loop, +that sentence now describes every drawing — harmless, since one pass of a +1-frame symbol is the whole of it, but the docstring should stop sounding like +a limitation. + +**Dropping into a 1-frame symbol.** The window is authored and crops what it +holds, so a 10-frame symbol dropped into a 1-frame drawing shows its frame 0 +and nothing else. That is consistent — `:frames` is the shot length and +`:extent :grow-symbol` is the opt-in — but it is probably not what somebody +dragging means. Offer the growth through the `:required-frames` retry the +model already uses: the command reports what it would need, the UI offers one +button. + +## Order of work + +Each step compiles, passes `npm test`, and leaves the editor usable. + +1. **Row per symbol.** In `ui/timeline.cljs`, delete `portal`, the `::portal` + hint row, the `under?` lineage predicate and the `chosen` argument; emit a + row for a clip whose parent is a lane instead of skipping it. Keep the + `:cels` blocks for the collapsed row, and keep `inside-rows` — including + its `:unmapped?` branch, which is what makes a held drawing's contents + reachable at all. +2. **Lane mode as a hint.** Add the field and the toggle, draw a symbol's + children as blocks when it is set and as rows when it is not, and move the + lane-row drag handling onto it. Both display paths now exist and nothing in + the domain has changed. +3. **Re-base the commands.** Move `lane.cljs` into `span.cljs`, replacing + `(lane-clips nodes lane-id)` with the symbol's children and dropping the + `lane-id` argument. `frontend/test/arthur/domain/lane_test.cljs` is the + proof: its fixtures should change and its assertions should not, and any + assertion that has to change is a behaviour change worth noticing. +4. **Move the invariant.** `symbol/lane-problems` becomes `symbol/overlaps`, + called by `span/finish` for a symbol in lane mode, plus the property test + that no command can produce an overlap. +5. **Delete the rest.** `node/lane?`, `symbol/lane-clips`, `clip/lane-node`, + `::ui/new-lane`, `::ui/adopt-in-lane`, the lane branch of + `::ui/new-symbol`, lane renaming, `aimed-lane`, and the + `:lane?`/`sound-lane?` row flags. Rename what is left so the word does not + appear outside the timeline. +6. **Audio falls out.** An audio node is already a parent-less child of a + symbol, which is exactly the new shape — so the audio-in-lane rules added + in `2dc5735` (`holds-other?`, the mixed-lane refusal, the `in-lane` filter + in `sound-rows`) delete rather than migrate. A symbol drawn as a lane whose + children are sounds is an audio lane, and that is the whole of it. +7. **Shift-to-reparent stays** as it is: `nest/move-node` and + `nest/move-refusal` never knew about lanes. + +## What must not be lost + +All of this was broken at some point today and is now proved; each has a test +to keep. + +- A held clip's contents are reachable from the root timeline, with no keys + and no draggable edges — `source-time` is nil for a hold, and the walk used + to stop there. +- Double-clicking a clip opens its symbol as a tab, and the editor survives + it: `symbol/lineage` must not report a cycle for an id the symbol does not + hold, and opening a symbol must drop a selection pointing into the one being + left. +- Selection waits for pointer-up, so a press does not re-draw the timeline out + from under the gesture it is starting. +- A drop never silently deletes what it lands on. +- A sound is drawn once, not twice. + +## Open questions + +1. **Is lane mode a property of the symbol or of the instance placing it?** A + symbol placed twice would be drawn the same way in both places under the + first reading. That is probably right, and worth saying out loud. +2. **Does a lane row's edge drag trim the placing instance's span, or ripple + the children?** Same handle, two commands; the row is now an instance, so + it has a span of its own for the first time. +3. **Does tearing out the held cel come before or after the lane work?** It + is independent of it — `nest` never knew about lanes — and it is what makes + shift-to-reparent work on the thing people would actually drag onto. Doing + it first means the lane work lands on a model with one playback mode fewer; + doing it after means two changes to `lane_test`'s fixtures instead of one. diff --git a/docs/lane-model.md b/docs/lane-model.md new file mode 100644 index 0000000..0fc11de --- /dev/null +++ b/docs/lane-model.md @@ -0,0 +1,591 @@ +# The Lane Model + +Revised 2026-10-01. Clip ownership, source playback, one-row generic lanes, +direct clip movement and edge editing, correction evaluation, and correction +authoring for rotation and position are implemented. The former cel-sheet +projection was removed: the timeline is the single timing interface. Sections +below that describe a cel sheet are retained as design history and are superseded +by this revision. Retiming commands are not. +See the status note under +[Proof obligations](#proof-obligations-and-implementation-order). + +[Lane and cel handoff](lane-handoff.md) records what is built, the decisions +that are settled, and what to do next. + +This revises the Claude artifact [The Lane Model](https://claude.ai/code/artifact/cd42981d-ed08-493f-94df-b7dd6657f0e6). +Its prose and diagram source were recovered from session +`1c603f71-84eb-498e-aeaf-4c0346f1f513`; the live artifact was not accessible for +reading or editing here. This repository document is the revised design. The +original artifact has not been updated, and edits made there outside the recorded +session may not be represented here. + +For the subjects covered here, this document supersedes the original artifact +and conflicting proposals in `animation-model.md`, `timing-model.md`, and +`architecture.md`. Those documents retain useful detail about the existing system. + +## Goal and compatibility policy + +Arthur is one animation document with several ways to see and edit it: drawing +on the stage, arranging clips, timing cels, editing curves, and generating +motion from footage. Each view exposes relevant facts and invokes shared editing +operations. Switching views must preserve the meaning of the work. + +The user explicitly requires no backward compatibility. Replace obsolete shapes, +APIs, and tests when a better model requires it. Do not retain compatibility +branches, adapters, or migrations solely to preserve the current document format. +A format marker can reject unsupported files clearly; it does not promise to +convert them. This policy does not authorize deleting existing user assets. + +Simplicity means predictable composition, clear ownership, and few independent +rules. Minimizing field count is secondary to representing independent choices. + +## What stays + +- A symbol is the one container for authored scene nodes. A drawing can be a + one-frame symbol; an animation uses the same container over more frames. +- Nodes have stable identities and flat parent references. Shared content is + referenced rather than copied implicitly. +- Animatable properties are addressed by channel paths. Generated and authored + values participate in the same evaluation machinery. +- Authored data, generated blocks, and source media remain separate. Documents + reference immutable blocks; caches and resolver indexes remain derived. +- A pure reference evaluator specifies the result. Playback, seeking, preview, + export, and optimized cursors must agree with it. +- Existence, visibility, and missing measured data remain distinct facts. + +## Content, cels, lanes, and rows + +These have different identities and responsibilities: + +| Concept | Owns | Example | +| --- | --- | --- | +| Content | Reusable nodes and their animation | Drawing `a2`, an animated head, or a sound asset | +| Cel | One use of content, its interval, source playback, and local treatment | `a2` exposed on frames 12–16 | +| Lane | A sequence of cels and shared properties | The girl's drawings and the girl's overall transform | +| View row or column | Presentation and editor state | Timeline row, cel-sheet column, or property curve | + +Use the existing instance/node identity mechanism for cels. A cel +should not acquire a second identity system just because it is shown as a cel. +Lanes group cels; they do not introduce another node-holding content type. +The concrete candidate below uses existing group and instance nodes; its ownership +boundaries are part of the design. It is now the implemented shape, and the field +spellings below are the ones the runtime reads. + +Each cel has a stable ID. Moving it, changing its hold, swapping its source, +or trimming it preserves that ID. Repeating it creates a new cel that may +reference the same content. A split retains the original ID on the left and gives +the right piece a new ID; commands return the resulting selection explicitly. + +Cels are the canonical authored arrangement. A source-at-time channel or +interval index may be compiled from them for evaluation, but is not a second +editable copy of the schedule. This replaces the earlier proposal that every cel +must be represented solely as a source key. Ordinary property animation still +uses lightweight keys; it does not need cel objects. + +A lane has non-overlapping half-open cel intervals `[start,end)` +in its own time space. Uncovered intervals are gaps. Empty lanes are valid. +Compositing and simultaneous sounds are represented by multiple lanes or ordinary +scene composition; an accidental overlap never silently selects a winner. +Transitions, if added, need explicit overlap and mixing semantics. + +Properties can belong to content, one cel, or the lane. For example: + +- Rotate the reusable drawing: all its uses change. +- Rotate one cel: only that cel changes. +- Animate the lane's rotation: whichever drawing is showing follows it. + +A cel can have its own transform, gain, corrections, and source timing +while remaining a block in the same timeline row. Independent treatment never +requires a new row or an otherwise unnecessary wrapper symbol. + +### Concrete candidate: a lane and ordinary instances + +A lane is a group node with `:layout :sequence`. Its cels are ordinary +instance nodes whose `:parent` points to the group. All remain in their symbol's +flat node map. The sequence constraint is document semantics; which rows the UI +expands remains editor state. Ordinary groups retain unconstrained composition. + +This example is a document the runtime accepts, built and evaluated by +`frontend/test/arthur/domain/lane_test.cljs`. Times here are zero-based. Channels +use the existing representation; cel source references and playback have +replaced the `[:source]` channel, which no longer exists. + +```clojure +;; Within :main's :nodes; referenced drawings/animations live in :symbols. +{:girl + {:id :girl :kind :group :layout :sequence :z "b" + :channels {[:xform :rot] + {:animated? true :interp :linear :keys {0 0, 6 30, 12 0}}}} + + :cel-a + {:id :cel-a :kind :instance :parent :girl :z "a" + :time {:at 0 :rate 1} :span [0 4] + :source {:symbol :drawing-a} + :playback {:in 0 :speed 0 :end :stop}} + + :cel-b + {:id :cel-b :kind :instance :parent :girl :z "b" + :time {:at 4 :rate 1} :span [0 4] + :source {:symbol :drawing-b} + :playback {:in 0 :speed 0 :end :stop} + :channels {[:xform :pos] + {:animated? true :interp :hold :keys {0 [0 0], 1 [2 0]}}}} + + :animated-insert + {:id :animated-insert :kind :instance :parent :girl :z "c" + :time {:at 8 :rate 1} :span [0 4] + :source {:symbol :wave} + :playback {:in 3 :speed 1 :end :stop}}} +``` + +The lane's rotation reads lane time. Each cel's channels read cel +time. Its content reads source time. The stills sample frame 0, while the insert +samples source frames 3, 4, 5, and 6. The group transform composes with the +cel transform and then the content's own transform. + +`:span` remains in the node's own coordinates, consistent with ordinary nodes. +The interval in lane time is derived through `:time`; do not also store parent +start/end values. Sequence children require finite intervals and positive +placement rates. Ordering and overlap checks use the mapped intervals, not `:z`. +The current sequence group contains visual symbol clips. Audio remains an +independent root node (and can be linked to picture); if audio lanes are added, +their capability must be explicit rather than inferred per frame. + +A source reference is fixed within a cel. The lane changes content when +another cel becomes active. This is a deliberate revision of the original +diagnosis that making `:of` a channel was necessary to avoid vertical growth: +multiple instances can occupy one row when the view presents their containing +sequence. A lane-level source schedule is therefore derived, not authored twice. + +## Source selection and source playback are independent + +The current implementation makes framed sources play and keyed sources hold. +Retire that rule. Channel storage shape must not determine playback behavior. +Adding or removing a key must not turn a still into an animation or vice versa. + +A cel names content and describes how its source time is sampled. In the +basic case, after mapping lane time into cel time: + +```text +source_time = in_point + speed × cel_time +``` + +A newly created cel starts at local time zero. Moving it preserves this +origin relative to its content. Trimming can narrow its local support without +resetting that origin; split pieces likewise preserve the source and property +values at the cut. Trimming, slipping, and retiming are distinct operations with +explicitly different effects on the interval and the source map. + +| Intent | Source playback | +| --- | --- | +| Hold a drawing | Constant source frame, equivalently speed 0 | +| Play an animated symbol | Advancing source time, normally speed 1 | +| Cut between animations | Several cels, each with its own in-point and speed | +| Mix stills and animation in a lane | Constant and advancing maps in the same sequence | + +The source reference itself is discrete and never numerically interpolated. +Interpolation belongs to properties that support it; a property registry should +declare value types, defaults, and permitted interpolation and correction modes. +Generic key toggles must consult those capabilities rather than assume every +non-boolean value can be tweened. + +Define source bounds and end behavior explicitly: stop contributing outside the +source, hold an endpoint, or loop an explicit range. A still uses a valid constant +frame. A loop uses a nonempty half-open range and a defined modulo rule. Playback +never guesses these policies from whether a channel happens to have keys. + +Audio shares cel arrangement, trimming, gain ownership, and clock mapping. +It does not inherit visual frame-hold semantics: holding one audio sample is not +an audio freeze effect. Validate supported playback policies by media capability. +Actual audio scheduling must follow active cels, including gaps and cuts, +rather than playing every sound reachable through a structural reference. + +## Time spaces and sampling + +Name the relevant space whenever an API accepts a time or range: project, +symbol/lane, cel, or source. Store authored frame coordinates exactly; +avoid cumulative rounding when moving through nested mappings. Quantize at a +declared sampling boundary, not at every traversal step. Audio also needs its +continuous clock/sample space rather than visual frame quantization. + +A hold is an evaluable time map with no unique inverse. A loop can map many +displayed cels to one source time. APIs must distinguish forward sampling +from inverse editing, and expose enough context to resolve a cel or +explicitly refuse an ambiguous operation. Do not report a missing time map merely +because inversion is unavailable. + +Separate invertible placement timing from source sampling. A zero source speed +can mean hold without making the cel's own edit clock non-invertible. +Reparenting through changing transforms or non-invertible timing must either +preserve the full result by an explicit bake or return a reason it cannot; a +matrix captured at one frame does not prove preservation across the animation. + +The source's frame step, generated-pose sampling, and the lane's transform clock +are independent scopes. Drawing on twos must not accidentally step a smooth lane +transform. An explicit whole-subtree stepping operation can exist separately. + +Share the quantization primitive where possible, but retain its units, phase, +rounding policy, and order relative to retiming and lead. The original suggestion +that cel and picture-rate sampling are simply one floor is insufficient: +noninteger grids and source-frame quantization require specified behavior. +Identity timing can be implicit; remove `:time :mode` if it only duplicates that. + +A cel interval is authored. Lane content extent is derived from its +cels, including the explicit end of the last one. A separately authored +container trim/window is legitimate when it intentionally gates children. Do not +conflate that window with occupied extent or infer a final hold from the next key +when no next key exists. A range of frame numbers alone cannot encode visibility +or a missing measurement. + +## Shared editing operations + +Every view issues the same domain commands. A command accepts an explicit target +and edit policy, computes a valid change, and returns the change, resulting +selection, and any refusal reason. A button and a drag must not implement two +versions of cel extension. + +An edit target identifies the symbol, cel path, selected entities or +properties, and the time range with its space. Navigation also distinguishes +editing shared content directly from editing it through a particular cel. +Crossing a source cut must not silently redirect an active drawing edit to a +different symbol: retain the explicit content target until navigation changes it. + +Core commands include new drawing, reuse drawing, duplicate drawing, make unique, +blank range, split, trim, move, extend cel, slip source, retime, and apply a +bounded property edit. Ripple/overwrite policy and the set of affected lanes are +explicit command arguments. Preview consequences before committing a gesture. + +New drawing creates fresh empty content and a cel. Blank range removes +content coverage without inventing a hidden drawing. These are different actions. +Reuse creates another cel pointing at existing content. Duplicate creates +a new content identity. Make unique rebinds the selected cel only. + +Copy semantics must specify nested sharing. A normal content copy duplicates its +owned nodes and channels while preserving references to other reusable symbols. +For a fully independent drawing assembled from nested symbols, provide an +explicit deep-copy operation with ID remapping. Never promise decoupling while +leaving the relevant edited object shared. Immutable media blocks may remain shared. + +Commands are atomic undo transactions, even when they touch several leaves. +Pointer movement and keyboard invocation use explicit begin/preview/commit or +cancel boundaries; a timing heuristic alone must not decide user intent. +Collaboration applies a transaction consistently, validates affected references, +and detects conflicts at the owned data being changed. One leaf per channel does +not solve simultaneous edits to different keys of that same channel; define a +conflict policy rather than claiming that granularity solves all collaboration. + +### Default timing behavior: cel edits preserve lane keys + +Working default from the follow-up discussion: extending a drawing's hold changes +cel timing, leaving lane animation at its authored times. The user raised +keeping keyframes in place as a possibility; this is the proposed predictable +default, not a claim that they selected every timing policy below. + +Ownership supplies the remaining rule: properties attached to a cel +travel with it. Extending its end does not stretch those properties; moving it +changes where their existing local times land. No per-key attachment flag is +needed to recover ownership that the document already expresses. + +For the concrete example, extend `:cel-a` by two lane frames with ripple: + +| Fact | Before | After | +| --- | --- | --- | +| Cel A's lane interval | `[0,4)` | `[0,6)` | +| Cel B's lane interval | `[4,8)` | `[6,10)` | +| Animated insert's lane interval | `[8,12)` | `[10,14)` | +| Girl's rotation peak | Lane frame 6 | Lane frame 6 | +| B's position change | B frame 1, lane frame 5 | B frame 1, lane frame 7 | +| Insert's first source frame | Source frame 3 | Source frame 3 | + +The rotation peak now coincides with a different point in the drawing sequence. +That is the intended consequence of changing cels underneath timed motion. +The position correction stays attached to drawing B's cel. Neither the +background's keys nor audio on another lane moves. + +The command contract for this edit names the symbol and cel, a delta in +lane frames, `:ripple` behavior, and an explicit scope of cel timing. It +extends A's local support by the delta converted through A's placement rate, +and shifts subsequent cel placements by that delta in lane time. It does +not modify any channel's key map, source in-point, or playback speed. Reject a +nonpositive resulting duration. Validate and commit the entire change together. + +The symbol's authored end is another explicit boundary: preview an overflow and +offer to extend the symbol or cancel. A command can request that extension as +part of its transaction; it must not silently truncate later cels or grow +other uses of a shared symbol. In the example, a 12-frame symbol needs an explicit +extension to 14 frames or the edit must be refused without partial changes. + +Retime performance is a separate operation over explicitly selected cels +and channels. It applies the same time transformation to their relevant clocks, +keys, and correction supports. Stretching an interval requires a defined warp +and interpolation behavior; it is not merely moving keys whose frame numbers +happen to lie inside the selection. Until supported, refuse this operation +rather than approximating it with a cel ripple. + +The initial UI should default stage transforms to the lane when drawing in a cel +workflow, so movement usually remains independent of cel timing. The +inspector names the target: lane motion, this cel, or shared drawing. +Changing that scope is explicit. It changes what the edit means, not just which +panel happens to be open. + +## Three-frame rotation and correction layers + +A range says where an edit applies; it does not specify the motion. Offer distinct +commands for a constant adjustment, a ramp, and a return-to-start motion. For UI +frames 10–12, the internal range contains exactly three frame samples after +conversion from the displayed numbering convention. + +- Constant adjustment: the same offset throughout those three samples. +- Ramp: interpolate from the specified start value to the target over the range. +- Return motion: interpolate from the starting value to a peak and back. + +For a return motion sampled on three frames, the values can be `0, angle, 0`. +Outside the selected range, the underlying animation must evaluate exactly as it +did before. A range-scoped correction layer expresses this directly; blindly +inserting boundary keys can alter neighboring segments or destroy existing motion. + +Implement corrections as an ordered stack over the base channel. Each correction +has stable identity, explicit support interval, blend operation, and values in a +named time space. Outside its support it is inactive. `replace` can supply a value +over an absent base; `offset` cannot offset a nonexistent value. Blend capability +depends on property type, and geometry corrections require compatible topology. + +Regeneration replaces the generated base and preserves corrections. If changed +topology or removed targets make a correction incompatible, report a resolvable +conflict instead of silently dropping or misapplying it. Provenance explains +where the base came from; explicit sampling policy determines its playback. + +This is core to the workflow: generate motion, correct it by hand, adjust the +generator, and keep the corrections. It should be proven before adding many views. + +## Other unifications worth keeping + +Pose choices, tracing-frame choices, and ordinary held values should share the +channel evaluator and cursor infrastructure. Preserve their different ownership, +fallback behavior, and sampling scope. A pose choice must address the relevant +content/feature explicitly; switching to another symbol must not accidentally +reuse a track just because both symbols contain a node with the same local name. + +Keep the two animation idioms distinct: keyed geometry modifies one mark over +time; drawing substitution selects content that may have different structure. +Linear geometry interpolation requires compatible vertex correspondence, not +merely two drawings that happen to look related. + +Derived library grouping may collect drawings used by a single lane. This is a +convenience, not ownership or deletion authority. Reference discovery for cycle +validation, copying, and deletion examines all structural references, including +currently inactive cels. Authored folders, favorites, and labels remain +legitimate user data even when the UI could have suggested defaults. + +## UX: location, selection, and controls + +The breadcrumb sits above the timeline and states the editing location, shared +content identity, and cel context when applicable. Show local time and +its project context where a useful mapping exists. Holds and loops need an honest +description instead of a fictitious unique global frame. + +Creation follows the primary active row, resolved at the playhead as specified in +[`creating-in.md`](creating-in.md). The wider selection set still names what copy, +delete and transform affect; it is not a second list of creation destinations. A +shared drawing indicates its reuse and offers Make this cel unique. Names help identify content; +linked-use indicators must rely on IDs, because different drawings can share names. + +| Surface | Primary scope and controls | +| --- | --- | +| Topbar | Project name, save/open/export, project rate and stage size | +| Location bar | Breadcrumb, add lane/content, shared-content context | +| Cel action strip | New drawing, duplicate drawing, hold longer/shorter, blank range | +| Lane header | Lane selection, lock, mute/solo where applicable, onion settings, expansion | +| Stage tools | Drawing and transform modes, active target and scope | +| Inspector | Selected content/cel/lane properties and valid key controls | + +Cel actions have visible contextual buttons, shortcuts, a context menu, and +command-palette entries. These are different entrances to the same commands. +Shortcut names from the original sketch (`N`, `D`, `H`, `B`, `K`) are provisional; +their meanings must match the visible labels and avoid tool conflicts. + +The inspector normally edits values and the timeline normally edits timing, but +this is an organizational default. Numeric duration and in-point controls are +useful inspector edits to the same domain facts. Do not ban a convenient control +just to preserve a visual division. + +Default nesting navigation enters content; expanding a lane reveals properties. +Other views may show hierarchies differently without changing the document. +Tabs can pin explicit locations. Zoom, expansion, onion preferences, and current +selection are editor state rather than animation content. Persistent workspace +preferences can be saved separately. + +## A session, revised + +1. In `main`, create a girl lane and a new drawing. Draw; use New drawing (`N`) + to create the next one with the previous cel ghosted behind it. +2. Use Duplicate drawing (`D`) when the current shapes are the starting point. + Use Reuse drawing for a deliberately linked cel. The UI shows the + difference before an edit can change other uses. +3. Time the performance. Hold longer (`H`) extends the selected cel and + ripples later cels in the explicitly targeted lane. A trim gesture + can use overwrite instead. The preview shows which boundaries will move. +4. Choose a two-frame default cel for newly created drawings, or run a + separate Retime cels command on a selected range. This does not quantize + lane transforms or silently retime already authored cels. +5. Place the background in a lane below. Its source holds one frame throughout + its cel. Key the lane's X position at the beginning and end and choose + linear interpolation. The background slides while the girl's drawings cut. +6. Select three frames on the girl's lane, choose Return motion, and rotate to + the desired peak. A bounded rotation correction affects the girl across any + drawing boundaries in that range. Existing motion survives outside it. +7. Insert a playing animated symbol among the girl's held drawings. Set that + cel's source playback to advance. No lane conversion is required. + +The timeline shows named cel blocks with property marks and optional curve +subrows. The cel sheet shows the same cels by frame and lane. The +graph editor edits the same properties; the stage resolves the same document. +Onion skin is configurable and counts neighboring cel events, skipping gaps +by default; a long hold does not consume the budget. Repeated uses of the same +drawing remain distinct events. Deduplicating identical ghosts is a display option. + +## Proof obligations and implementation order + +The source-channel prototype has been removed: a cel names one symbol +and carries its own playback clock, and `node/problems` rejects the old +`[:source]` channel. What a lane IS lives in `arthur.domain.symbol` beside the +other rules about a node map; `arthur.domain.lane` holds the commands over +one — add lane, place a drawing (new, reused or duplicated), make unique, split, +trim, move, blank and extend hold. Each is one history step, and each refuses rather than +half-applying. The timeline draws a lane's cels as cel blocks on the +lane's own row, and offers Make unique only where the selected cel actually +shares its drawing. + +There is ONE placement function and a position argument, so appending is not a +different operation from inserting: `:end` is a position like any other, the one +where nothing has to move. Placing ripples — cels at or after the +position move later by the new cel's duration — and `:keep` versus +`:grow-symbol` still decides what happens at the shot's end. OVERWRITE is not a +policy argument yet, deliberately: taking frames away from the cel +already there is trimming, and until `trim` exists, placement that would need it +refuses instead of approximating it. A position inside an existing cel +refuses too, and names `split` — one command does not quietly perform two. + +Splitting turned out to cost almost nothing, which is evidence for the +representation rather than for the command. The two pieces keep ONE `:time` and +differ only in `:span`, so the right piece's own frames carry on where the +left's stopped and its source clock, keys and corrections go on meaning what +they meant — a held drawing holds the same frame either side, a playing insert +plays through the cut without a seam, and the test for it samples every frame +before and after and asserts the picture is identical. That falls out of `:span` +being in the node's own coordinates; it is not something split arranges. + +Content copies are shallow by default and keep their references to other +symbols; `:deep? true` is the explicit copy that shares nothing, so the promise +of independence is only made where it is kept. + +THE SHOT LENGTH IS AUTHORED, which is the decision the range commands forced. +`:frames` is the symbol's window — how long the shot IS — and the occupied +extent of its lanes is a different fact derived from the cels. A command +grows the window only when the caller says `:grow-symbol`, and never shrinks it: +blanking the end of a shot leaves a shot with empty frames at the end, because +that is a true statement about what somebody authored, and deriving the window +from the extent would make deleting the last drawing quietly shorten the film. +`finish` keeps the two numbers apart by name now rather than by a `max` that +read like an accident. + +Trim NARROWS one edge and moves nothing else; lengthening is `extend-hold`, +which carries the ripple and shot-length policies because it needs them. +Move is one write to `:time :at` and REFUSES a destination that would overlap, +because moving a drawing and re-timing the ones around it are different +intentions — clear the room with `blank` or `trim` first, which is the +composition. Blank leaves a gap and does not close it; a cel wholly inside +the range goes, one overlapping an end is trimmed to it, and the one spanning +the range is split. Their drawings stay in the library, since a lane does not +own its content. + +All three are the same geometry as `split`: a `:span` is in the cel's own +frames, so moving an edge is one write and `:time` and `:playback` are never +touched. That is why trimming the front of a playing insert starts it later into +its animation instead of restarting it — the difference between trimming and +slipping, and the reason they stay separate commands. + +Correction layers EVALUATE. `channel/problems` used to refuse an `:over` stack +and `value-at`/`cursor` used to throw on one; both now read it, and the +agreement test that holds the optimized cursor to the specification covers +stacked channels in forward, backward and random frame order. A layer's values +are themselves a channel, so a constant adjustment, a ramp and a return motion +are one mechanism; `:support` is half-open and a layer is inactive outside it; +and a layer has no time space of its own, because the node its channel is on +already has one. Nothing had to change in the codec — a channel is one leaf, so +a correction persists inside it — and nothing had to change in validation +plumbing, since `node/problems` already reports every channel's problems. + +Both halves of ownership are under test at lane level: a three-frame correction +on the girl's lane reaches across the drawing boundary beneath it and leaves +every frame outside its support identical, and a correction owned by one +cel travels with that cel when a hold before it grows. + +Regeneration keeps them, which is the obligation the layer design exists to +meet: `rebased` replaces a base and carries its corrections across, and a +correction the new base no longer fits is MARKED rather than dropped or +misapplied — `clip/conflicts` lists those for a view to offer, separately from +`problems`, because a conflict is a decision nobody has made yet and not a +document that will not load. Turning the mouth's `:verts` knob is a real +topology change and is what the test uses. Two latent faults turned up there and +are fixed: `regenerate-head` compared authored channels to measured ones +directly, so the first correction on the head would have stopped it following +re-measurement for good; and an incompatible offset threw in the read path, +which would have taken the stage down on exactly the case the model says to +report. + +Overwrite is `blank` followed by non-rippling placement, composed inside one +transaction; insertion keeps its ripple rule. Still unbuilt: slip source, +retime, and deleting reused content. A lane cannot hold AUDIO cels — `lane-problems` +requires visual ones, though this document says a lane may hold either and +should reject only a mixture. + +`domain/correction.cljs` now produces Constant adjustment, Ramp, and Return +motion layers for rotation and position. The inspector exposes them on a selected +lane or cel using an explicit range in that owner's frames; this deliberately +leaves displayed-range dragging through nested or retimed owners for later. One +Apply is one undo step. Conflicted layers are listed, can be removed, and can be +retried when the complete ordered stack is compatible again. Validation, +conflict reporting, and regeneration share that ordered-stack rule, including +coverage by adjacent replacement layers. Slip source and retime are still not +implemented; a refusal is the current behavior where the model demands an +explicit choice nobody has made yet. +The cel sheet is the same projected cels and selection addresses with its axes +turned: frames down and lanes across, so commands selected there and in the +timeline have identical targets; a gap selects its column's lane rather than +retaining a stale selection from another column. The suite stands at 437 tests and 5,804 +assertions, with `frontend/test/browser/lane.mjs` driving the editor through +create, hold, overflow, undo, reuse, make unique, duplicate, split, insert, +trim, move, blank, correction authoring, and two-lane sheet targeting. Rewrite tests that encode superseded +behavior rather than preserving behavior to keep them green. + +Build small adversarial documents and test their domain operations before +expanding the interface: + +| Scenario | Required invariant | +| --- | --- | +| Same drawing exposed twice, then one made unique | Linked edits affect both before copying and only the selected content after | +| Holds, playing inserts, nonzero in-points, and gaps on one lane | Source behavior is independent of property key count and channel encoding | +| Adjacent cels, final hold, split, trim, ripple, and overwrite | Exact boundaries, stable IDs, deterministic collision handling | +| Extend a hold under lane keys and cel-local corrections | Lane key times remain fixed; later cel corrections travel with their owners; source playback origins survive | +| Ripple beyond the symbol end | Explicit extent policy; refusal leaves the document unchanged; resizing and retiming undo together | +| Girl on twos over a moving background | Drawing cadence does not quantize either lane's continuous properties | +| Three-frame correction crossing a drawing boundary | Exact support, same result outside it, one undo step | +| Nested retiming, holds, loops, and fractional sampling | Explicit time spaces; ambiguous inverse edits cannot silently choose a target | +| Audio inside changing source cels | Only active intervals sound, with correct trim and source timing | +| Regenerate with corrections and a topology change | Compatible edits survive; incompatible ones produce actionable conflicts | +| Reference cycles and deletion of reused content | Inactive references are validated too; no dangling references | +| Save/load and command undo/redo | Identity, source maps, corrections, and evaluation round-trip | +| Timeline and cel-sheet invocation of one command | Identical document changes and selection targets | +| Random forward/backward seeks and export | Reference and optimized evaluation agree, including defaults and absence | +| Concurrent commands on overlapping and disjoint targets | Transactions remain valid; conflicts are explicit and undo preserves others' work | + +Implementation order: cel ownership and playback semantics; shared +commands and validation; correction layers and time-addressing contracts; then +breadcrumb, cel strip, and a cel-sheet projection. Use those two temporal +views plus direct stage editing to prove the model before broadening the UI. + +A new presentation should not require duplicate animation state. A genuinely new +authoring capability may require new domain data. The model is successful when +such additions have a clear owner and compose with existing operations, not when +it can claim that no future feature will ever need another field. diff --git a/docs/lane-nesting-notes.md b/docs/lane-nesting-notes.md new file mode 100644 index 0000000..73d6ec0 --- /dev/null +++ b/docs/lane-nesting-notes.md @@ -0,0 +1,161 @@ +# Lane nesting interaction notes + +Status: design note, 2026-10-01. This records the interaction before more lane +UI is implemented. + +## The capability that must not be lost + +A lane owns temporal placement, but a symbol instance is still a doorway into +another symbol. A drawing accidentally authored at the root must be movable into +an instance in any lane, including another lane, without changing its visible +position or timing. + +That operation already exists as `nest/move-node`. It resolves the source and +destination at the current root frame, transplants the node, and re-expresses +its transform and time under the new parent. The lane UI must expose a target +path for it; it must not replace it with a weaker `:parent` assignment. + +There are therefore two different drag intentions: + +1. **Temporal move:** drag a clip body onto lane space. It remains a clip in a + lane, moves in time, and claims the destination interval by trimming/removing + incumbents. +2. **Structural move:** drag from the clip's grab affordance onto another symbol + instance. The dragged node is transplanted into the target instance's source + symbol with `nest/move-node`, preserving its world transform and root timing. + +These cannot be inferred from overlap alone. Dropping clip A onto time occupied +by clip B already means “A claims that time and trims B.” Structural nesting +therefore needs an explicit grab affordance/mode. Its cursor is `grab` and +`grabbing`; trim edges keep their resize cursors and the ordinary body keeps its +timeline-move behavior. + +Both visible clip blocks and an expanded symbol header are structural drop +targets. This permits moving a root drawing directly into `symbol-3` even when +its lane is collapsed. + +## Compact expansion: one selected-clip portal + +Expanding a lane must not restore row-per-clip vertical growth. Instead, an +expanded lane reveals exactly one clip portal: the currently selected clip in +that lane. + +```text +▾ foreground lane [symbol-1][symbol-2][symbol-3] + ▾ symbol-3 instance/source header and drop target + ▸ body lane nested rows, mapped to the root ruler + ▸ face lane + position nested keyframes mapped to root time +``` + +- Selecting another block in the same lane swaps the portal in place. +- With no selected clip in that lane, expansion shows a compact “select a clip + to inspect” row. It must not follow the playhead during playback; that would + make the timeline restructure itself while playing. +- The portal header represents the selected instance and is the structural drop + target for moving root or sibling content into its source symbol. +- Sub-expanding the portal uses the existing recursive symbol-row walk. Nested + lanes and channels are mapped through the instance clock into the open/root + ruler, as ordinary expanded instances already are. +- The lane's own transform/channel rows remain available separately. They affect + every clip in the lane and are not properties of the selected portal. + +This keeps the cost of inspection constant: an expanded lane adds one selected +symbol branch, not one branch for every temporal clip it contains. + +## Keyframe visibility + +Two levels should be visible without changing editors: + +- The selected clip's instance-level keys (transform, visibility, corrections) + appear as ticks inside that clip block on the lane row. +- Expanding the lane opens the selected clip portal, where source-symbol and + recursively nested keys appear on their own rows, mapped to root time. + +Thus the collapsed lane answers “where does this clip change?” and the expanded +portal answers “which property inside this symbol changes?” The second view is +still the root timeline; entering the symbol is not required merely to see or +edit its keys. + +## Drag targets and feedback + +- Grab onto lane background: move/adopt the instance into that lane. +- Grab onto a symbol clip: structurally transplant into that clip's source + symbol. +- Grab onto the expanded portal header: the same structural transplant, with a + larger and less ambiguous target. +- Grab onto itself or one of its descendants: refuse before drop to prevent a + symbol cycle. +- A structural target receives an inset highlight and the preview stays in that + target. A lane-time target receives the dashed temporal clip preview. +- Successful structural drops expand the target lane and select the moved node + beneath the target portal, so the result is immediately visible. + +## Data model consequence + +No lane-as-symbol type is required. The hierarchy remains: + +```text +symbol -> sequence lane -> instance clip -> source symbol -> its lanes/nodes +``` + +Lane membership owns time partitioning. Symbol instances own composition +nesting. The UI may present the selected instance below its lane, but that is a +derived portal, not another ownership edge and not a duplicated node. + +## Implementation order + +1. ~~Render instance-level key ticks within lane clips.~~ Done: a clip's keys + are on its block, drawn after the blocks so they land on the one they + belong to. +2. ~~Add selected-clip portal expansion to `timeline/rows`.~~ Done, with two + additions the note did not anticipate: + - The portal is chosen by the whole LINEAGE of the selection, not the + selected id. Selecting a shape inside the clip, or the end of its span, + is still working inside that clip, and matching the id alone closed the + portal the moment anything under it was touched. + - A HELD clip opens too. `clip/source-time` is nil for a hold, so the walk + used to stop there and the inside of every drawing was unreachable from + the root timeline. Its rows are now shown across the hold and marked + `:unmapped?`: no keys, and no draggable edges, because no frame inside it + has a place on this ruler. +3. ~~Add the explicit structural affordance.~~ Done as SHIFT on a clip-body + drag rather than a separate grab handle: shift turns a temporal move into a + structural one, the target clip takes an inset highlight, and a label by the + pointer says which of the two is about to happen. +4. Route structural drops through `nest/move-node`. **Wired, and blocked in + the domain.** The gesture asks `nest/move-refusal` on the way past, so the + label says before the drop what the command would say after it. Two + refusals stand in the way of ordinary use: + - *both have to be on screen at this frame.* Inherent, and worth keeping: + the move preserves the world transform and there is no common frame to + preserve it at otherwise. It does mean nesting one clip into another in + the SAME lane can never work — a lane never overlaps itself — so this is + a between-lanes gesture with the playhead somewhere both are showing. + - *a held or looping clip has no clock to move through.* `nest/inside` + returns no `:time` for a hold, and a held one-frame drawing is the most + common thing in a document, so today nesting into one is refused — which + is most of what anybody would try. +5. After the transplant, expand the destination portal and reveal/select the + moved row. `::ui/move-node` already selects the moved node and opens the + rows down to it; the portal follows from the lineage rule in 2. + +## The held destination, unresolved + +A held cel shows ONE source frame for its whole span, so there is no +invertible map from the lane's frames to the drawing's and `move-node` +refuses. But the refusal is stronger than the facts require. Inside a frozen +destination only one frame is ever observed, so: + +- the RATE of any map into it is unobservable — every rate shows frame `in`; +- what IS observable is that the moved node should show, at that one frame, + what it shows now at the current root frame. + +That pins a unique sensible answer — rate 1, aligned so the current frame maps +to the shown frame — and nothing else about the mapping can be seen. If that +argument holds, it is a rule rather than a guess, and it is the difference +between structural nesting working for drawings and not working at all. It +needs its own proof: a drawing authored at the root, nested into a held cel in +another lane, sampled before and after to show the same picture, in the style +of `drawn` in `lane_test`. + diff --git a/docs/multi-face-representation.md b/docs/multi-face-representation.md index 65584c3..bed5863 100644 --- a/docs/multi-face-representation.md +++ b/docs/multi-face-representation.md @@ -8,11 +8,13 @@ symbol instance. Timelines already provide local node names, independent playbac and persistence. No new kind of scene container is needed. ```clojure -:timelines +:symbols {:main {:nodes {:root {:time {:mode :map :expose 2}} :face {:parent :root :channels } - :face-1 {:kind :symbol :of :face-1 :parent :face :z "a0"} - :face-2 {:kind :symbol :of :face-2 :parent :face :z "a1"}}} + :face-1 {:kind :instance :source {:symbol :face-1} + :parent :face :z "a0"} + :face-2 {:kind :instance :source {:symbol :face-2} + :parent :face :z "a1"}}} :face-1 {:nodes {:head {...} :mouth {:parent :head ...} ...}} :face-2 {:nodes {:head {...} :mouth {:parent :head ...} ...}}} diff --git a/docs/one-grid-plan.md b/docs/one-grid-plan.md new file mode 100644 index 0000000..90414ce --- /dev/null +++ b/docs/one-grid-plan.md @@ -0,0 +1,66 @@ +# The editing grid is the symbol's own frames + +## What was wrong + +Every number a person authors — a span, a `:time :at`, a key, a cut — is in the +frame space of the symbol it lives in (`docs/time.md`, and that part is right). +The timeline, though, drew its ruler in OUTPUT frames: `clip/output-frames`, the +transport's length. In a 12fps project holding 30fps symbols those two spaces sit +at a ratio of 2.5, so: + +* a clip at symbol frame 31 was drawn at ruler frame 12.4 — **no clip edge landed + on a frame mark**, because almost none of them can; +* a gesture measured in ruler frames had to be multiplied into the symbol's + frames and rounded (`nest/dragged`), so dragging one ruler frame moved the clip + 2 or 3 symbol frames — **0.8 or 1.2 ruler frames, never the 1 the pointer + said**. That is the jumpiness; +* the preview drew the gesture's own number of ruler frames while the commit + wrote the rounded one, so **the ghost sat somewhere the settled clip did not**; +* before the rounding was added, the fraction went into the document and every + later edge edit on that clip was refused for ever (`span/*` refuses a + fractional edge, as it should). + +One cause, four symptoms. None of them is an edge case to patch. + +## The model + +1. **A symbol's own frames are the only coordinate anything authored lives in.** + Unchanged. +2. **The editor edits in the open symbol's frames.** The ruler, the marks, the + playhead's position on it, every pointer→frame answer, every drop frame and + every drag delta are the open symbol's frames. No multiplication anywhere in + the gesture path, so no rounding and nothing fractional to refuse. +3. **The output grid is playback's alone** — the clock, the audio mix, export, + and the frame the stage draws. Exactly two pure functions cross between them + and nothing else does: + * `clip/shown-frame clip sid f` — which of `sid`'s frames output frame `f` + shows (`cadence/frame`: the latest at or before it). + * `clip/first-output-frame clip sid n` — the output frame that first shows + symbol frame `n`; the inverse, for seeking from the ruler. +4. `nest/inside`, `nest/placement`, `nest/spans` and the gestures take the + subject symbol's OWN frame. They used to take an output frame and multiply it + secretly, which is what made every caller's units a guess. A caller holding + the playhead converts with `clip/shown-frame`, at its own edge, visibly. + +## The gestures + +5. **One pointer→frame function** for the whole timeline, `frame-under`. A drag's + delta is the difference of two of its answers, never a pixel ratio rounded + separately — so the preview and the commit are the same number by + construction. +6. **The junction between two clips is one handle with one meaning**: roll. It + moves the end of the left clip and the start of the right one together, which + is `span/roll`, which is already nothing but `resize-out` then `resize-in`. + The three 4px-wide zones it used to pick between — trim-left, roll, + trim-right, inside twelve pixels — were the "it just picks one" the handle + was accused of. An edge that is not shared still has its own in/out handles. + +## Audio goes with the picture it belongs to + +A take's sound lived inside the take symbol, so placing the take brought it and +placing the FACE the take is made of brought nothing. A symbol now says what it +sounds like — `:audio`, a sound source — and placing one places a linked audio +node beside the clip. Detection sets it on the face it extracts, which is the +automatic link; `::ui/link-audio` sets or clears it by hand, which is the manual +one. `:linked-to` on the audio node already existed and already follows a moved +picture. diff --git a/docs/port-plan.md b/docs/port-plan.md index f9a67e3..338e39d 100644 --- a/docs/port-plan.md +++ b/docs/port-plan.md @@ -146,7 +146,6 @@ Full specification in `docs/animation-model.md`. The subset to build: [:xform :rot] {:animated? false :value 0.0} [:xform :scale] {:animated? false :value [1.0 1.0]} [:xform :skew] {:animated? false :value [0.0 0.0]} - [:xform :anchor] {:animated? false :value [0.0 0.0]} [:geom :pts] {:animated? true :interp :hold :dense {:store "sha256:…" :offset 0 :stride 40 :frames 600} :generated {:by :roto/lips-outer :analysis "sha256:…" @@ -170,15 +169,24 @@ uses to offer a parameter panel instead of raw keys. It lives on the *channel*, not the node, because a node wants a rotoscoped `[:geom :pts]` and a hand-animated `[:xform :pos]` at the same time. -`:skew`, `:span`, `:anchor` and `:over` stay in the shape even though nothing -drives them yet: each is a component of a decomposition or of a composition -order, and adding one later migrates every stored transform. +`:skew`, `:span` and `:over` stay in the shape even though nothing drives them +yet: each is a component of a decomposition or of a composition order, and adding +one later migrates every stored transform. + +`:anchor` was in this list and has since been **deleted**, which is the one place +the reasoning above came out wrong. It is not a component of the decomposition: +`T(a)·M·T(-a)` is `M` conjugated by a translation, and a parent already is a +translated frame, so an anchor is a peg written inline — one that cannot be +selected, keyed, shared, or put above a measured channel. Rotation and scale +happen about the node's own origin; a pivot nobody chose is derived per drag by +`domain/gesture` and a pivot somebody chose is a peg. See +docs/animation-model.md, "There is no `:anchor`, because an anchor is a peg". Transform composition, per node: ``` -local = T(pos) · T(anchor) · R(rot) · K(skew) · S(scale) · T(-anchor) -world = world(parent) · local +local = T(pos) · R(rot) · K(skew) · S(scale) +world = world(parent) · pinv · local ``` ## What the prototype knows that you would otherwise rediscover @@ -333,11 +341,12 @@ per-frame header. numbers: it centres on the face oval's bbox and zooms until the face is 80% of the raster height, so every vertex carries a cropping decision made once from one frame's landmarks. Dropping it is a deletion. Placement becomes `[:xform :*]` on -an authored `:face` node, the stage clips whatever hangs off, and project +an authored `:place` node inside the face, the stage clips whatever hangs off, +and project dimensions stop being tied to the footage. See "What space geometry is in" in `docs/animation-model.md`. -The anchor transform freezes onto `:head`, one level under `:face`, and the +The anchor transform freezes onto `:head`, one level under `:place`, and the normalise on/off/per-plate toggle is which of the three channel shapes that node carries. Always measure and always store factored, whatever the toggle says: smoothing and velocity-minimum key selection both require the split to exist in diff --git a/docs/time.md b/docs/time.md new file mode 100644 index 0000000..c0d9205 --- /dev/null +++ b/docs/time.md @@ -0,0 +1,41 @@ +# Time selection + +Project `:fps` is the playback and export grid. Each symbol has its own native +`:fps` and `:frames`; keys, spans, trace choices and corrections stay in that +native space. A symbol without an explicit rate inherits the document rate; +changing project fps first records that rate so its existing timing stays put. +The untouched symbol in a new document is deliberately different: it has no +authored timing to preserve, so it stays on the project grid and its empty frame +extent is rescaled to keep the same duration. This makes changing fps before +authoring establish the editor's grid instead of preserving the 30fps default. + +An output frame selects the latest native frame at or before its time: +`floor(output-frame * native-fps / output-fps)`. Thus 30fps content in a 12fps +project reads source frames 0, 2, 5, 7, 10… and retains its duration. A partial +last output frame is included. Changing back to 30 restores the original grid. +Nothing rewrites or discards the dense measurements. + +The same boundary selection runs when entering a placed symbol. Placement and +artistic speed are applied before selection; the stored `:time :rate` and +`:playback :speed` never contain a frame-rate conversion. The derived maps used +by timeline rows, picking and editing account for the units of each symbol. +`clip/frames` is a native length; `clip/output-frames` is a transport/export +length. Resolver frame queries return native frames for edits. + +There is one fps control. The old transient picture-fps control and node +sample-fps fields are gone. Existing exposure, trace choices and per-instance +pose tracks remain available: a pose track can hold a chosen closed-mouth frame +without deleting its neighboring measurements. Those choices stay in native +frames when output fps changes. Automatic content-aware frame selection is not +implemented; [frame-selection.md](frame-selection.md) is how it should be. An +event between output frames appears on the next output frame; it cannot create +an extra frame in a 12fps output. + +Audio uses continuous time through the same derived placement maps, without +picture floors or holds. Frame-rate units cancel before Web Audio playbackRate +is set, so only deliberate speed changes affect pitch and duration. Export and +playback use the same output count and resolver. + +Earlier imports with frame-rate conversion baked into stored retimes must be +re-imported. There is no second reader for that representation. Source video +presentation timestamps are still future work; this model assumes constant fps. diff --git a/docs/timing-handoff.md b/docs/timing-handoff.md index 30e15f7..b9fa406 100644 --- a/docs/timing-handoff.md +++ b/docs/timing-handoff.md @@ -70,7 +70,7 @@ handling and the relevant key whitelist if its storage location requires it. - `freeze/performance-nodes` marks generated animated channels with `:pose-sampled?` and local `:pose-group` names. This includes keyed visibility as well as dense geometry. `:generated` remains provenance for regeneration. -- `timeline/channel-frame` already applies explicit pose choices and default +- `symbol/channel-frame` already applies explicit pose choices and default picture sampling to marked channels. Playback and export both use `clip/resolver` with `:picture-fps`; there is no need for a second sampling implementation. Export's pose count is still a rate-based estimate. diff --git a/docs/timing-model.md b/docs/timing-model.md index 608266e..5cf947f 100644 --- a/docs/timing-model.md +++ b/docs/timing-model.md @@ -1,5 +1,11 @@ # Timing model +[Time selection](time.md) defines the current frame-rate representation. + +[The Lane Model](lane-model.md) defines the revised target for occurrence timing, +source playback, sampling scope, and inverse editing. It supersedes conflicting +proposals here; the sections below describe earlier implementation decisions. + The source footage, authored drawings, generated face motion, and stage placement have different frame decisions. They share a clock but do not share one kept-frame list. `timing-handoff.md` records earlier implementation notes. @@ -35,7 +41,7 @@ measurements without losing the anchor choices. A source image used for tracing should be registered with that image's measured stabilizing transform, then the selected head transform, then the authored -`:face` placement. This makes the photo and head-local vectors share the same +`:place` placement the face carries. This makes the photo and head-local vectors share the same orientation and position. Tracing-photo selection is a separate editor address; it does not choose the head anchor. @@ -78,7 +84,8 @@ candidate poses so stage cuts can still select any of them. | --- | --- | --- | | Source frames and timestamps | Footage/analysis | Constant-rate frame indexing exists; variable timestamps remain future work | | Head anchor map | `:head` node | Implemented, stored with the node | -| Tracing cel starts and photo address | Authored cel | Separate future work | +| Trace frames (photo address) and origin | The face's `:plate` `:time :holds`, and its `:head` `:reads` | Implemented, see `docs/tracing-symbol-plan.md` | +| Showing a tracing layer, and its opacity | Editor state, `[:ui :tracing]` | Implemented; a drawing aid, never saved or keyed | | Generated picture-rate proposal and closure protection | Roto clip/symbol | Generated-only picture sampling exists; closure protection remains future work | | Stage pose cuts | Symbol instance | Implemented, stored with the instance | diff --git a/docs/tracing-symbol-plan.md b/docs/tracing-symbol-plan.md new file mode 100644 index 0000000..2e2e7ba --- /dev/null +++ b/docs/tracing-symbol-plan.md @@ -0,0 +1,411 @@ +# Plan: tracing is a symbol + +Status: built, 2026-10-03, on branch `worktree-tracing-layers`. Where the build +differs from the plan below, the build wins: + +- The head's field is `:reads` (`{:holds [...]}` or `{:holds-of :plate}`), not + `:follow`. +- A still is not one frame. It gets as many frames as remain in the symbol it is + dropped into, at that symbol's rate, and is trimmed like any clip. +- An image's `:media` is `{:image }`, served at `/blob/`. The + server's `Image` row exists for the pool's list and labels, not for identity. +- Dropping media that a tracing symbol in the document already shows reuses that + symbol. +- Nothing can be created or dropped inside a tracing symbol (`creation/target`, + `drop-destination-at`, `nest/move-refusal`), and one cannot be opened in a tab. +- Schema 6. Every project is marked 6. One that still carries `:trace` on a + node is refused when opened, with what to do about it, rather than all old + projects being refused. + +No backward compatibility (see `lane-model.md`, "Goal and compatibility policy"). + +## What is wrong today + +Tracing is not a thing in the document. It is three mechanisms that each know +about faces: + +- **`:trace {:frames :origin}` on a face's `:head`.** It decides which measured + frame the head reads (`symbol/base-channel-frame` → `trace/held-frame`), and + the underlay also reads it to decide which photo to show (`trace/photo-frame`). +- **`ui/underlay`**, a painter that walks `trace/shown` → `trace/faces` (a + separate instance walk), looks up each face's subject → analysis → footage, + asks the resolver where `[...path :head]` went, and builds the photo's matrix + by hand: `world(head) · M(p)⁻¹ · 1/imageH` (`trace/photo-matrix`). +- **`[:ui :trace {:faces #{} :opacity}]`**, a per-face switch in editor state, + plus the special "opening a face shows its footage, opening a take does not" + rule (`trace/showing-for`). + +What you cannot do: place footage or a still as a reference where you like, move +it, scale it, turn it, trim it, hold it, put it in a lane, or trace something +that is not a tracked face. The photo is not selectable and has no row. The one +case that works (a face over its own footage) runs on code that nothing else +uses. + +## The model in one paragraph + +A **tracing symbol** is a symbol with `:type :trace`. It has no nodes. It names +media (a footage range or a still image) and has that media's frame count, fps and +pixel size. It is **placed by an ordinary instance**, so lanes, spans, trim, +split, move, playback (`:in :speed :end`), transform, nesting, selection, picking +and gestures all work on it with no new code. It is evaluated to a single +**`:trace` op**, which never reaches the raster or an export. A face's footage is +not a special case. It is one of these instances, placed inside the face as a +child of `:head` and carrying the measured registration transform. The trace keys +become **holds on that instance's time**. The head's "origin" becomes the head +saying **which node's frames it follows**. + +This follows the precedent of `:type :palette` symbols (no nodes, they name an +asset, and they are placed by instances in a lane), so the uniformity rule holds +(see the `model-uniformity` memory). Nothing new holds nodes, and no special +instance kind is added. + +## Data model + +### The tracing symbol + +```clojure +:footage-8625 ; a symbol id like any other +{:id :footage-8625 :name "8625.mov" + :type :trace + :media {:footage #uuid "f8ca…" :range [12 241]} ; or {:image "sha256:…"} + :frames 229 ; the range's length; 1 for a still + :fps 30 ; the footage's own rate, so cadence handles 30→12 for free + :width 1440 :height 1920 ; pixel size = the symbol's own stage + :audio {:footage #uuid "f8ca…"} ; optional, the existing "a symbol says what it sounds like" + :nodes {}} +``` + +- `:media` is the one new symbol key. Add it to `symbol/symbol-keys` and to the + symbol leaf's `select-keys` in `leaf/leaves`. +- The symbol's local space is **pixels of the media**: `[0 w) × [0 h)`. + `clip/center` of a node-less symbol already returns its stage middle, which here + is `[w/2 h/2]`. So `place-symbol` puts the anchor at the image centre with no + new code. +- `symbol/problems`: `:type :trace` needs `:media` with exactly one of + `:footage`/`:image`, needs `(empty? nodes)`, and for footage needs + `(= frames (- end start))`. +- **One tracing symbol per footage range, shared.** Two faces from one take + place the same symbol, and so does a hand-placed reference. Like any symbol it + is reused by reference, and `bring/symbols` copies it like any other. + +### Placing it + +It is an ordinary `:kind :instance` with `:source {:symbol :footage-8625}`: + +- **Start and end** are `:span` (in its own frames) and `:time :at`. You get + these from the existing trim, split, move and roll. +- **Which frame shows** comes from `:playback {:in :speed :end}`. Footage plays + with speed 1, a frozen frame has speed 0, and a still is 1 frame with `:end :hold`. +- **Placement in space** is `[:xform …]`, the same as every node: gestures, + inspector and keys. +- **In a lane, or on its own**: it is a parent-less spanned node, so a `:display + :lane` symbol draws it as a block like any cel. It can also sit as a free node + or under a group. To have a reference with its own tab, wrap it in an ordinary + symbol. + +**Default on creation** (in the drop event, not in `place-symbol`): scale so the +image's height fits the stage height, centred on the drop point. This is a +creation default like "use center anchor when dropping", and nothing updates it +afterwards. + +### Holds: the one new time feature + +`:time {:holds [0 12 30]}` floors a node's local frame to the last hold at or +before it. Before the first hold, the first hold applies. This is `:expose` +generalized from a regular grid to authored frames. It is applied in +`node/local-frame` at the same point as `:expose`, and is inherited in the same +way. It is in the node's **own** frames. On a tracing instance with +`:in 0 :speed 1`, own frames are source frames, so the hold list is the set of +traced frames. + +It is general on purpose. Holding a playing symbol on chosen drawings is the +same feature. It costs about three lines in `local-frame`, one `problems` clause +(sorted, distinct, finite), and the timeline drawing hold frames as marks on the +row. + +`symbol/frame-map` refuses floors (`:expose > 1`). It must also refuse a +non-empty `:holds`, for the same reason. + +## The face: "a child symbol that represents the trace" + +The face symbol after a freeze: + +``` +:place group authored source→stage mapping (image heights → stage px) + :head group measured M(p) :reads — see below + :mouth … parts generated, every frame + :plate instance of :footage-8625 ← the trace + measured channels: fit(p) = M(p)⁻¹ · S(1/imageH) + :time {:holds [0 12 30]} ← the trace keys +``` + +### Registration comes from the parent + +The plate is a child of `:head`. Its own measured channels are the +**stabilizing fit** at frame p: the inverse of the head's measured transform, +with pixels → image heights folded into the scale (it stays a similarity, so it +decomposes into `pos`/`rot`/`scale`). Freeze already computes the fit; the +head's measured channels are its inverse (`freeze/invert`). The plate gets a +second dense block, which is three small channels. + +Because holds are a **time** floor, the plate's channels and the frame its +content shows are read at the **same** held frame q. Its world transform is: + +``` +world(plate) = place · M(p_head) · M(q)⁻¹ · S(1/H) +``` + +That is exactly `trace/photo-matrix`, but now it falls out of the ordinary walk. +It is registered to whatever the head is doing, by construction: + +| head reads (p_head) | plate shows (q) | result | +| --- | --- | --- | +| f (continuous) | f (no holds) | footage where filmed | +| f (continuous) | held key | held photo rides the moving head (today's behaviour) | +| held key (same as plate) | held key | `M(q)·M(q)⁻¹ = I`: photo sits where filmed | +| 0 (start) | f or held | stabilized footage under a still head | + +If a frame has no measurement, the plate's dense channel has nothing there, so +`xform-at` gives nil, so the plate is not placed and no photo shows. That is what +happens today too. + +### Trace keys versus origin: who owns what + +The two decisions are separate and stay separate: + +- **Trace keys** are which footage frames get drawn over (the "plate drawings" + in `frame-selection.md`). They belong to the **plate**, as `:time :holds`. A + hand-placed tracing layer with holds is the *same thing*: the face's plate is + an ordinary tracing placement and nothing more. +- **Origin** is how the head moves between kept frames. `frame-selection.md` + already says `:origin` "is not a tracing setting" but a performance one, so it + belongs to the **head**: + +```clojure +:head {…} ; continuous — reads its own frame +:head {… :reads {:holds [0]}} ; start +:head {… :reads {:holds-of :plate}} ; at keys — reads its measured channels at + ; the frames :plate's holds select +``` + +`{:holds [...]}` is also what a face with no footage uses for "at keys", since it +has no plate to follow. + +`:reads` changes only the **head's own channel reads**, not its children's +frames. The parts must keep running every frame, which is why this cannot be a +`:time` hold on the head. It is today's `traces` branch of +`base-channel-frame` with the hold list read from the named node. A node +reference has precedent (`:stencil`, `:pose-group`). It points from the follower +to the thing followed, so there is still one stored list of frames and nothing to +keep in sync. Validate it in `symbol/problems` the way `:stencil` is validated: +the target exists in the symbol, and it is not the head itself or an ancestor of +the head. + +Rejected alternatives, so they are not re-proposed: + +- **Keys on the head, and the plate reads them.** This is today's direction. It + makes the plate special: a free tracing layer could not have keys that a face's + plate also understands. +- **Hold `:time` on `:head`.** Exposure inherits strictly, so the mouth and eyes + would freeze along with the head. +- **A wrapper "registered footage" symbol holding the fit.** It is correct but + adds a symbol per face. The time-floor holds already put the fit read and the + content read on the same frame, so the wrapper buys nothing. +- **Plate as a sibling of `:head` at identity.** It is only registered when the + head and the photo read the same frame, so it breaks the continuous-plus-holds + and start rows above. + +## Evaluation: an op that is never rendered + +- **`clip/resolver`**, in the instance branch: when the source symbol has + `:type :trace`, it does not recurse. If `(:tracing? opts)` is set, it emits one op: + + ```clojure + {:kind :trace :node [id] :m :media … :frame shown-frame :size [w h]} + ``` + + The frame comes from the same `placed-frame` path as any instance, so playback, + holds and the fps cadence apply. Do not build a child resolver for a trace + symbol. +- **`transform-op`** gets a `:trace` case that composes the matrix. Row paths, + solo filtering and nesting at any depth then work for free. +- **The output guarantee is structural.** `:tracing?` defaults to false. Only the + stage's `::render/resolver` passes true. Export, `clip/center` (a big photo + must not pull a symbol's pivot), thumbnails and the bench never ask for trace + ops. `raster/draw-ops!` keeps throwing on unknown kinds, so a leak fails loudly. +- **`ui/player`** sends picture ops to the raster and `:trace` ops to the + painter. +- **`ui/underlay` becomes `ui/tracing`.** It paints `:trace` ops in draw order + with `drawImage` at `op.m` and the global opacity. The media URL is the footage + manifest's `urls[range-start + frame]`, or the image blob URL. The three + steadiness fixes stay: LRU cache, hold the last still per op `:node`, and read + ahead while playing. Everything that walked faces is deleted. +- **`pick`**: a `:trace` op is hit when the point, mapped through `m⁻¹`, falls in + `[0 w) × [0 h)`. Picture ops are tested first and trace ops only if nothing + drawn is under the pointer, so a full-frame photo does not steal every click. + When the global switch is off, traces are neither painted nor picked. +- **Gestures**: no change. The face's plate is measured, so `gesture/refusal` + already says "place the instance it is in". A hand-placed layer is authored and + moves, turns and scales like anything else. + +## On and off + +All of it is EDITOR STATE, as ed88c5e decided for the per-face switch: showing +a reference is a way of looking at the stage, so it is not an undo step, does not +travel to collaborators, and cannot reach an export. + +```clojure +[:ui :tracing {:on? true :opacity 0.5 :hidden #{[sid node-id] …}}] +``` + +- **One layer** is an entry in `:hidden`, keyed by the symbol the tracing + instance is in and its node id. The symbol id is needed because every face's + plate is called `:plate`. Keyed this way, hiding a face's plate hides it in + every placement of that face, which is what the per-face switch did. Shown is + the default, so opening a face shows its footage with no setup. +- **How it's applied**: `clip/resolver` already knows which symbol and node a + trace op comes from, so the op carries `:layer [sid id]`. The painter and + `pick` skip ops whose layer is hidden. The resolver does not change when you + toggle a layer, so nothing is rebuilt. +- **Global**: `:on?` and `:opacity`, a toggle plus an opacity slider in + `ui/palette/bar` next to the palette controls. When it is off, traces are + neither painted nor picked. +- **Switching one layer on also switches the global setting on.** This applies + from the tracing instance's inspector, from its timeline row, and from the + face/roto section. A layer you just enabled must not stay invisible behind a + switch you forgot. Turning one layer off never touches the global switch. It is + one event (`::ui/show-trace layer on?`) that updates `:hidden` and, when + turning on, sets `:on? true` in the same handler, so the three places cannot + behave differently. +- **`[:vis]`** still works on a tracing instance like on any node: key it to + show a reference only over part of the shot. It is not the on/off switch. +- **Deleted**: `[:ui :trace :faces]`, `::trace-face`, `::trace-faces`, + `trace/showing-for`, `traceable-faces`, `shown`, `faces`, and the "a take shows + nothing by default" rule. + +## UI + +- **Timeline.** A tracing instance is an ordinary row or clip block, styled to + read as reference-only (hatched block, an eye icon in place of the colour + chip). Hold frames are marks on the row. The row's eye button dispatches + `::ui/show-trace`, which replaces the face row's `tl-trace` button. +- **Inspector, tracing instance.** Show the media (footage name and range, or + image), an on/off control (which goes through `::ui/show-trace`), and the hold + list ("hold here" / "remove hold" plus seek buttons, which is `trace-keys` + re-aimed at `:time :holds`). Playback, transform and span use the existing + sections. +- **Inspector, face/roto section.** The same hold controls, aimed at the face's + `:plate`. Origin buttons write `:head :reads`. The section is found by "this + face has a `:plate`", not by `trace/traceable?`. +- **Palette bar.** The global toggle and opacity slider. +- **Making one.** + - Footage: the convert dialog gets a choice between "animate faces" (today's + flow) and "tracing layer" (no detection). The tracing-layer choice makes the + `:type :trace` symbol for the chosen range and places it where the video was + dropped. + - Footage can also be dragged from the pool with a modifier or as a second + drag kind. + - A still image: drop it on the pool and it uploads; drop it on the stage or + timeline and it is placed with `:end :hold` and a span of the host's + remaining frames. + +## Freeze, bring, regenerate + +- **`freeze/subject-part`** emits the `:plate` instance under `:head`, with fit + measured channels and `:time {:holds []}`, and emits one `:type :trace` symbol + for the analysed footage range. `bring/take` already copies every symbol the + take reaches and rewrites `:source :symbol` references, so the tracing symbol + comes along. It gets the footage's `:audio` too, so a dropped tracing layer + can bring its sound through the existing link. +- **`head-mode`** writes `:reads` on the head and `:holds` on the plate, in + place of `:trace`. The `frame-selection.md` plate proposal materializes into the + plate's `:time :holds` in place of `:trace :frames`. +- **Regenerate** replaces both measured blocks (head and plate) and keeps + `:holds` and `:reads`. Check that `regenerate-head`'s measured/authored + comparison handles a second measured node. + +## Server + +- `Image` endpoints mirroring `Sound`: `POST /api/images` stores a `Blob` and + returns `{id, width, height, url}`, and `GET /api/images` lists them. The + migration is a model only. Bump `schema_version` and refuse older documents + clearly. Do not convert them. + +## Deleted outright + +`trace/of`, `prepare`, `held-frame`, `problems`, `toggle-frame`, `photo-frame`, +`measured-local`, `photo-matrix`, `traceable?`, `faces`, `traceable-faces`, +`showing-for`, `shown`, `opacity-default`. `domain/trace.cljs` probably goes +away entirely; if anything is left, it is a hold helper that belongs in `node`. + +Also deleted: `symbol/prepared-traces` and the `traces` arm of +`base-channel-frame`, which becomes the `:reads` lookup. `:trace` on nodes (and +`symbol/problems` reports it as removed). `::project/set-trace`. +`::render/underlay` and `::render/tracing` become one `::render/tracing` that +returns `[:ui :tracing]`. The face lookup in `params/view`. + +Expected effect on code size: net negative. One time-floor clause, one op kind, +one resolver branch, one pick case and a `:reads` lookup replace the face walk, +the hand-built photo matrix, per-face showing state and the underlay's face +bookkeeping. Measure the change and report the number honestly (see the +`cljs-style` memory). + +## Tests + +**Domain (node).** + +- `:holds` in `local-frame`, with eval-frame and resolver agreeing forward, + backward and in random order. +- A trace op appears only with `:tracing?`, and `export/run!` output contains no + `:trace` op. +- `transform-op` composes the matrix through two nesting levels. +- `clip/center` ignores traces, and is `[w/2 h/2]` for a tracing symbol. +- `pick` returns picture ops before traces and inverse-maps through a rotated + layer. +- `symbol/problems` covers `:type :trace`, `:reads` targets and cycles, and + `:holds` shape. +- Registration: port `trace_test`'s photo-matrix assertions onto the walk. + For every row of the registration table, the plate's world transform equals + the expected matrix. At a held key with `:reads {:holds-of :plate}`, it equals + `world(:place) · S(1/H)`. +- Freeze emits the plate and the tracing symbol. Regeneration keeps the holds. +- The `::ui/show-trace` event sets the global switch on when turning a layer on + and leaves it alone when turning one off. + +**Browser** (CDP, see the `arthur-verify-dont-guess` memory): + +- Drop a still, then drag, turn and scale it. +- Toggle a layer on with the global switch off, and confirm both are on and the + photo paints. +- Export a frame and check it has no photo pixels. +- Open a face: the plate is registered at a hold with origin "at keys", and + stabilized with origin "start". + +## Order of work, each step green + +1. `:time :holds` in `node/local-frame`, `problems`, `frame-map`, and timeline + marks. +2. `:type :trace` symbol, `:media`, the `:trace` op behind `:tracing?`, + `transform-op`, the player split, `ui/tracing` painter and `pick`. Footage + media only, placed by hand from a REPL or test document. +3. The face: freeze and bring emit the plate and tracing symbol, `:reads` + replaces `:trace`, delete the old trace and underlay paths, and re-aim the + inspector's face section. +4. On/off: the `:hidden` set and `:layer` on trace ops, the row eye, the + `::ui/show-trace` rule, the global toggle and opacity in the palette bar, and + delete `[:ui :trace :faces]`. +5. Creation: "tracing layer" in the convert dialog, pool drag, then the image + endpoint, pool images and image drops. +6. Docs: `animation-model.md` "A photographic underlay is not an op" becomes "a + trace is an op that never reaches the raster". Update `frame-selection.md` + (where plate keys live) and `lane-model.md` (tracing clips in lanes). + +## Open questions + +1. **"A take shows its faces' footage."** Dropping the old "only in a face's own + tab" default means opening a take shows every face's plate if the global + switch is on. The recommendation is to accept that, since the switch is one + click. +2. **Lane-model audio rule.** `lane-problems` requires visual cels. A tracing cel + is visual for this purpose, but say so explicitly when a lane may hold both + tracing and drawing cels. diff --git a/frontend/README.md b/frontend/README.md index 62a7946..42dc4ba 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -88,18 +88,66 @@ cd frontend && mise exec -- npx shadow-cljs watch app ``` Then open ****. Django serves the page from -`clips/templates/clips/index.html`, and staticfiles serves the bundle out of -`static/arthur/js`, where `shadow-cljs` already writes it — so nothing copies files -between the two. +`clips/templates/clips/index.html`, its styles from `static/arthur/app.css`, and +the bundle out of `static/arthur/js`, where `shadow-cljs` already writes it — so +nothing copies files between the two. + +### The window + +One screen, five panes, no scrolling page. `src/arthur/ui/shell.cljs` is the grid +and nothing else; each pane owns its own subscriptions. + +``` + top the document: its name and last status, export, new / open / save + left media pool — the open document's symbols, and footage on the server + centre the palette strip (16 slots) above the stage + right inspector — the clip, the selected node, the tracked objects + bottom timeline — transport, ruler, a row per node +``` + +**It opens on a blank document**, and **new** makes another one. Nothing is +loaded until it is asked for. + +**Whole documents live under `open ▾`, not in the media pool**, and the split is +load-bearing rather than tidy. Opening a project REPLACES the stage; everything +in the pool is a thing to put ON it. Listing documents beside the symbols inside +one of them makes them read as two kinds of the same thing. The menu lists the +projects the server holds; the built-in scenes are under their own heading, +italic, and are not projects — they are compiled into the bundle and the server +has never heard of them. + +Everything that holds nodes is a **symbol**, and none is special: a new document +has one called `main` because it has to be called something. Which symbol is on +screen is editor state, `[:ui :open]`, not a fact about the document — the stage +draws it, the timeline lists it, the transport plays it and a new shape goes into +it. A document opens on the longest symbol nothing else places. + +Selection lives in app-db under `:ui`, as `[:node ]`, +`[:symbol ]` or `[:subject|:feature|:group ]` — four panes ask what is +selected, and a ratom private to one of them can only be shared by making the +other three require it. + +**Drop a video on the media pool** and it uploads, extracts and goes straight on +into detection. Dragging a symbol out of the pool onto the stage places an +instance of it at the playhead. + +The timeline's rows are the open symbol's nodes, front-most first, with a dot per +keyframe and a bar over the frames the node exists on; a dense channel is hatched +rather than ticked, because one value per frame is a solid block that says less +than the bar does. Opening a row shows its channels; opening an **instance** row +shows the symbol it places, with every frame number mapped back into the open +symbol's frame space — see the namespace docstring in `ui/timeline.cljs`, which +is where that mapping is argued. ### Paint sketch -Click **new polygon**, place at least three vertices on the stage, then click -**finish shape**. Select a shape to drag its vertices. Scrub to another frame and -click **new drawing key** to copy the visible outline there; the previous drawing -holds until that key. The numbered drawing-key buttons jump to editable keys. -The transition control between two drawing keys can switch that gap between a -hold and linear vertex tweening. Other gaps keep their own timing. Tweening works +Pick a tone from the palette strip, click **polygon**, place at least three +vertices on the stage, then click **finish**. Select a shape — on the stage, or by +its timeline row — to drag its vertices. Scrub to another frame and click +**drawing key here** in the inspector to copy the visible outline there; the +previous drawing holds until that key. The numbered key buttons jump to editable +keys. The transition control between two drawing keys can switch that gap between +a hold and linear vertex tweening. Other gaps keep their own timing. Tweening works best when the same vertex keeps the same meaning in every drawing. Paint shapes use the timeline clock directly, so the roto exposure grid does not delay a drawing key or step its @@ -109,7 +157,7 @@ tween. Use the project **save** button to persist the drawings. has used since step 5, when shadow-cljs's `:dev-http` did no directory-index resolution and the suite learned to ask for the file. -Four built-in clips, on buttons in the transport: +Four built-in clips, under **built-in examples** in the open menu: | | | | --- | --- | @@ -125,14 +173,14 @@ The demo scene itself is `src/arthur/demo/scene.edn`. Both the synthetic take and real footage use `src/arthur/flow/take.cljs` for the measurement order and `src/arthur/flow/freeze.cljs` for the landmark-to-channel conversion. -**stage 8625** loads the locally saved `IMG_8625.MOV` project and places its -post-processed timeline twice. The stage layout is +**8625 stage study**, in the open menu, loads the locally saved `IMG_8625.MOV` project and places its +post-processed face symbol twice. The stage layout is `src/arthur/demo/stage_8625.edn`: the right picture and sound start at frame 48, -and the two pictures overlap slightly in stage space. Audio has its own timeline +and the two pictures overlap slightly in stage space. Audio has its own nodes, linked to the picture instances but with independent spans and gain channels. The right sound swells and pans across the stage, then fades out at frame 260 while its picture continues to -frame 280. The button needs that saved 8625 project in the local server database. +frame 280. The row needs that saved 8625 project in the local server database. ### Projects and the EDN fixtures @@ -148,16 +196,17 @@ the same ClojureScript clip. The intended editor creates and changes that in-memory clip directly: a project browser and **new stage** action, timeline instance placement, node and channel editors, then the existing save path. EDN remains useful for checked-in examples -and reproducible studies. The current UI has save and open, but no project -browser, blank-stage action, or authoring controls yet; open chooses the most -recent project. +and reproducible studies. The UI now has the blank-stage action (**new**), a +project browser (`open ▾`), and placement by dragging a symbol out of the media +pool; node and channel editors are still to come — the inspector reports a +channel's shape but has nowhere to change its values. ### Real footage -Choose a video in the **footage** file input. The server probes it, re-encodes it +Drop a video on the media pool, or use its **+** button. The server probes it, re-encodes it to an H.264 proxy and a raw stream of the same coded frames, pulls WAV audio and one tracing JPEG per frame, -then makes the resulting footage selectable. Click **load frames** to detect and -freeze it. Extraction progress is currently read from `/api/extractions/`; a +then makes the resulting footage selectable and runs detection on it. **roto**, in +the pool's header, does the same for footage that is already there. Extraction progress is currently read from `/api/extractions/`; a future WebSocket can push the same job state. The uploaded bytes, extraction job, and decoded footage have separate records, so the same uploaded video can be reopened without decoding it again. @@ -216,14 +265,14 @@ the root — all of it is extraction output, and tier 3 does not belong in the r Loading detects one face per frame, measures the mouth, eyes and brows from landmarks and the teeth from source pixels, then freezes them into channels, -and adds a button for the footage clip. Detection happens once when you load; +and opens the footage clip. Detection happens once when you load; playback only resolves channels and paints. Frames without a detection remain marked absent even though their neighbouring poses are used to condition the track. The scene now records stable subject and feature IDs and explicit eye pairs; dense channels can mark one feature absent while another is observed. Current MediaPipe loading supplies only the full-face detection mask. The stage stays 320×200 regardless of the footage dimensions. Real -footage starts at the source picture rate. The **picture fps** buttons sample the +footage starts at the source picture rate. The **picture** buttons in the inspector sample the frozen roto at lower rates while the source track, duration and audio clock stay unchanged. Picking frames to trace into cels is a separate future editing step. **save** also stores the detection mask, dense landmarks and raw RGBA mouth crops @@ -252,7 +301,7 @@ runs the old JS tool on 8777, and the two are meant to run side by side. ## Saving -**save** and **open** in the transport. A save has three ordered stages: +**new**, **open** and **save** in the top bar. A save has three ordered stages: is the tier split: 1. the **analysis** record, so every block stored afterwards can name the detector @@ -269,10 +318,11 @@ unchanged document says `0 leaves · 0 blocks`, which is both halves of the addressing working at once — an unchanged leaf keeps its version, and a content-addressed block is already there. -Two things are deliberately visible as failures. Saving `swarm` is refused, -because its blocks have hand-written names and a document may only name content -addresses. And **open** takes the most recently updated project and shows its first -clip: there is no project browser, and the store holds one clip at a time. +Saving `swarm` is deliberately visible as a failure: its blocks have +hand-written names and a document may only name content addresses. + +`open ▾` lists every project the server holds, newest first, and shows the first +clip of whichever one is picked — the store holds one clip at a time. ## The oracle, which is finished @@ -313,12 +363,12 @@ them is `clips/templates/clips/index.html`. ## Two evaluators, on purpose -`domain/timeline` has both `eval-frame` and `resolver`, and they are not +`domain/symbol` has both `eval-frame` and `resolver`, and they are not alternatives: -- **`(eval-frame timeline f store)`** is the specification. Allocating, order-free, +- **`(eval-frame symbol f store)`** is the specification. Allocating, order-free, obviously correct. Tests and one-off renders use it. -- **`(resolver timeline store)` -> `(fn [f] ops)`** is what playback uses. It caches +- **`(resolver symbol store)` -> `(fn [f] ops)`** is what playback uses. It caches the topological order and the z paths, holds a cursor per channel and reuses one point buffer per node, so a frame allocates the op maps and nothing else. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a4c5be8..901ccc3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "dependencies": { "@mediapipe/tasks-vision": "1.0.1", + "polygon-clipping": "^0.15.7", "react": "^18.3.1", "react-dom": "^18.3.1" }, @@ -1015,6 +1016,16 @@ "node": ">= 0.10" } }, + "node_modules/polygon-clipping": { + "version": "0.15.7", + "resolved": "https://registry.npmjs.org/polygon-clipping/-/polygon-clipping-0.15.7.tgz", + "integrity": "sha512-nhfdr83ECBg6xtqOAJab1tbksbBAOMUltN60bU+llHVOL0e5Onm1WpAXXWXVB39L8AJFssoIhEVuy/S90MmotA==", + "license": "MIT", + "dependencies": { + "robust-predicates": "^3.0.2", + "splaytree": "^3.1.0" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", @@ -1216,6 +1227,12 @@ "node": ">= 0.8" } }, + "node_modules/robust-predicates": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", + "integrity": "sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA==", + "license": "Unlicense" + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1416,6 +1433,15 @@ "source-map": "^0.5.6" } }, + "node_modules/splaytree": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/splaytree/-/splaytree-3.2.3.tgz", + "integrity": "sha512-7OXrNWzy6CK+r7Ch9OLPBDTKfB6XlWHjX4P0RU5B3IgFuWPeYN0XtRtlexGRjgbQxpfaUve6jTAwBGWuGntz/w==", + "license": "MIT", + "engines": { + "node": ">=18.20 || >=20" + } + }, "node_modules/stream-browserify": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 8cb1e39..fe89f12 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,6 +10,7 @@ }, "dependencies": { "@mediapipe/tasks-vision": "1.0.1", + "polygon-clipping": "^0.15.7", "react": "^18.3.1", "react-dom": "^18.3.1" }, diff --git a/frontend/shadow-cljs.edn b/frontend/shadow-cljs.edn index 41ed051..75b4b8f 100644 --- a/frontend/shadow-cljs.edn +++ b/frontend/shadow-cljs.edn @@ -41,6 +41,7 @@ {:app {:target :browser :output-dir "../static/arthur/js" :asset-path "/static/arthur/js" + :compiler-options {:source-map true} :modules {:main {:init-fn arthur.core/init}}} :test {:target :node-test diff --git a/frontend/src/arthur/audio/mix.cljs b/frontend/src/arthur/audio/mix.cljs index cec1e6f..f03710c 100644 --- a/frontend/src/arthur/audio/mix.cljs +++ b/frontend/src/arthur/audio/mix.cljs @@ -2,16 +2,22 @@ "Render independently placed audio tracks into one stage audio clock. The mix is derived from saved audio track leaves and immutable footage blobs. - The transport still has one audio element, so seeking, rate changes and looping - stay tied to the same clock the picture reads. + The transport still has ONE clock, so seeking, rate changes and looping stay + tied to the same position the picture is drawn from. - THE AUDIO BUFFER IS THE PRODUCT AND THE WAV IS ONE PACKAGING OF IT. Playback - wants a URL an `