diff --git a/clips/extraction.py b/clips/extraction.py index d83df64..3f59ac8 100644 --- a/clips/extraction.py +++ b/clips/extraction.py @@ -161,6 +161,17 @@ def _extract_stills(job, proxy_path, frames_dir, frames, root): MAX_RATE = 120 # a capture rate; past this the container is describing something else +def probe_image(path): + """An uploaded still's pixel size as (width, height), refusing anything that + is not one picture.""" + data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams", + "-of", "json", str(path)])) + video = [s for s in data.get("streams", []) if s.get("codec_type") == "video"] + if len(video) != 1 or not (video[0].get("width") and video[0].get("height")): + raise ValueError("the uploaded file is not an image") + return int(video[0]["width"]), int(video[0]["height"]) + + def probe_audio(path): """The length of an uploaded sound in seconds, refusing a file with no audio.""" data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams", diff --git a/clips/migrations/0015_tracing_images.py b/clips/migrations/0015_tracing_images.py new file mode 100644 index 0000000..ab3aa3c --- /dev/null +++ b/clips/migrations/0015_tracing_images.py @@ -0,0 +1,48 @@ +"""Schema 6: tracing is a symbol. + +A face's `:head :trace` is gone: its footage is a placement of a `:type :trace` +symbol under the head, the trace keys are that placement's `:time :holds`, and the +head follows them with `:reads`. Nothing is converted. Every project is marked 6, +and one that still carries a `:trace` is refused when it is opened, by name and +with what to do about it; the rest open as they did. + +Images are stills to trace over, stored like sounds. +""" + +import uuid + +from django.db import migrations, models +import django.db.models.deletion + + +def forwards(apps, schema_editor): + apps.get_model("clips", "Project").objects.update(schema_version=6) + + +class Migration(migrations.Migration): + dependencies = [("clips", "0014_multiple_analyses")] + + operations = [ + migrations.CreateModel( + name="Image", + fields=[ + ("id", models.UUIDField(default=uuid.uuid4, editable=False, + primary_key=True, serialize=False)), + ("filename", models.CharField(max_length=255)), + ("label", models.CharField( + blank=True, max_length=200, + help_text="what a person called it; the filename when empty")), + ("width", models.PositiveIntegerField(help_text="pixels, as ffprobe reports them")), + ("height", models.PositiveIntegerField()), + ("created", models.DateTimeField(auto_now_add=True)), + ("blob", models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, + related_name="image_for", to="clips.blob")), + ], + ), + migrations.AlterField( + model_name="project", + name="schema_version", + field=models.PositiveIntegerField(default=6), + ), + migrations.RunPython(forwards, migrations.RunPython.noop), + ] diff --git a/clips/models.py b/clips/models.py index 9eeb4b3..1348c15 100644 --- a/clips/models.py +++ b/clips/models.py @@ -70,6 +70,21 @@ class Sound(models.Model): created = models.DateTimeField(auto_now_add=True) +class Image(models.Model): + """An uploaded still — a drawing, a photo, a model sheet — kept as uploaded, to + be traced over. Never part of the picture: a document names its blob as a + tracing symbol's `:media`, and the page draws it over the stage.""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + blob = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="image_for") + filename = models.CharField(max_length=255) + label = models.CharField(max_length=200, blank=True, + help_text="what a person called it; the filename when empty") + width = models.PositiveIntegerField(help_text="pixels, as ffprobe reports them") + height = models.PositiveIntegerField() + created = models.DateTimeField(auto_now_add=True) + + class Extraction(models.Model): """One requested decode of a source into immutable footage.""" @@ -235,7 +250,7 @@ class Project(models.Model): settings.AUTH_USER_MODEL, blank=True, related_name="shared_projects", ) name = models.CharField(max_length=200, default="untitled") - schema_version = models.PositiveIntegerField(default=5) + schema_version = models.PositiveIntegerField(default=6) seq = models.PositiveBigIntegerField(default=0) palette = models.CharField(max_length=64, default="arthur/default") created = models.DateTimeField(auto_now_add=True) diff --git a/clips/tests/test_api.py b/clips/tests/test_api.py index 26e8e73..8e32565 100644 --- a/clips/tests/test_api.py +++ b/clips/tests/test_api.py @@ -34,7 +34,7 @@ from django.core.management import call_command from django.test import TestCase, override_settings from clips import blobs, extraction -from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision, Sound, Source +from clips.models import Analysis, Block, Blob, Clip, Footage, Image, Leaf, Project, Revision, Sound, Source BLOB_DIR = tempfile.mkdtemp(prefix="arthur-test-blobs-") @@ -446,7 +446,7 @@ class DocumentTests(TestCase): self.assertEqual(5, len(response.json()["written"])) loaded = self.client.get(f"/api/projects/{self.project.id}").json() - self.assertEqual(5, loaded["schema_version"]) + self.assertEqual(6, loaded["schema_version"]) self.assertEqual(1, len(loaded["clips"])) clip = loaded["clips"][0] self.assertEqual("c1", clip["cid"]) @@ -828,6 +828,32 @@ class UploadTests(TestCase): self.assertEqual(200, again.status_code) self.assertEqual(1, Sound.objects.count()) + def test_an_uploaded_still_is_an_image_named_by_its_bytes(self): + payload = png(17, 5) + uploaded = self.client.post("/api/images", { + "file": SimpleUploadedFile("sheet.png", payload, content_type="image/png")}) + self.assertEqual(201, uploaded.status_code, uploaded.content) + image = uploaded.json() + self.assertEqual(("sheet.png", 17, 5), (image["label"], image["width"], image["height"])) + self.assertEqual(f"/blob/{image['digest']}", image["url"]) + self.assertEqual(payload, b"".join(self.client.get(image["url"]).streaming_content)) + self.assertEqual([image], self.client.get("/api/images").json()["images"]) + + again = self.client.post("/api/images", { + "file": SimpleUploadedFile("again.png", payload, content_type="image/png")}) + self.assertEqual(200, again.status_code) + self.assertEqual(1, Image.objects.count()) + + renamed = self.client.patch(f"/api/images/{image['id']}", + json.dumps({"label": "model sheet"}), + content_type="application/json") + self.assertEqual("model sheet", renamed.json()["label"]) + + def test_a_file_that_is_not_a_picture_is_not_an_image(self): + refused = self.client.post("/api/images", { + "file": SimpleUploadedFile("notes.txt", b"not a picture", content_type="text/plain")}) + self.assertEqual(400, refused.status_code) + def test_a_file_without_audio_is_not_a_sound(self): refused = self.client.post("/api/sounds", { "file": SimpleUploadedFile("notes.txt", b"not audio", content_type="text/plain")}) diff --git a/clips/urls.py b/clips/urls.py index 19cee07..ab52fb7 100644 --- a/clips/urls.py +++ b/clips/urls.py @@ -24,6 +24,8 @@ urlpatterns = [ path("sources", views.sources), path("sounds", views.sounds), path("sounds/", views.sound_detail), + path("images", views.images), + path("images/", views.image_detail), path("extractions", views.extractions), path("extractions/", views.extraction_detail), path("footage", views.footage_list), diff --git a/clips/views.py b/clips/views.py index e88dff8..0e431eb 100644 --- a/clips/views.py +++ b/clips/views.py @@ -43,7 +43,7 @@ from django.views.decorators.http import require_http_methods from . import blobs, extraction from .consumers import broadcast -from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Leaf, Project, Revision, Sound, Source +from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Image, Leaf, Project, Revision, Sound, Source KEY_LENGTH = 71 # "sha256:" + 64 hex @@ -287,6 +287,50 @@ def sound_detail(request, sound_id): return JsonResponse(_sound_json(row)) +def _image_json(row): + return {"id": str(row.id), "label": row.label or row.filename, + "filename": row.filename, "width": row.width, "height": row.height, + "digest": row.blob_id, "url": f"/blob/{row.blob_id}"} + + +@require_http_methods(["GET", "POST"]) +def images(request): + """Stills to trace over. The document names one by its blob digest, so an + image is the same picture in every project that uses it.""" + if request.method == "GET": + return JsonResponse({"images": [_image_json(row) + for row in Image.objects.order_by("-created")]}) + upload = request.FILES.get("file") + if upload is None: + return JsonResponse({"error": "upload an image as the file field"}, status=400) + try: + digest, size = blobs.write_stream(upload.chunks()) + width, height = extraction.probe_image(blobs.path_for(digest)) + blob, _ = Blob.objects.get_or_create( + digest=digest, defaults={"size": size, + "media_type": upload.content_type or "image/png"}) + row, created = Image.objects.get_or_create( + blob=blob, defaults={"filename": Path(upload.name).name[:255], + "width": width, "height": height}) + return JsonResponse(_image_json(row), status=201 if created else 200) + except (ValueError, OSError) as exc: + return JsonResponse({"error": str(exc)}, status=400) + + +@require_http_methods(["GET", "PATCH"]) +def image_detail(request, image_id): + try: + row = Image.objects.get(id=image_id) + except Image.DoesNotExist: + return JsonResponse({"error": "no such image"}, status=404) + try: + if request.method == "PATCH": + row = _relabel(request, row) + except Bad as exc: + return _error(exc) + return JsonResponse(_image_json(row)) + + def _extraction_json(row): return {"key": row.key, "source": str(row.source_id), "state": row.state, "progress": row.progress, "error": row.error, diff --git a/docs/animation-model.md b/docs/animation-model.md index 0514b97..d8a57a0 100644 --- a/docs/animation-model.md +++ b/docs/animation-model.md @@ -605,28 +605,26 @@ dense geometry. A topology setting cannot be treated as a per-frame gain curve. The op list is the boundary with stage 7 in `docs/architecture.md`: the rasteriser takes ops and knows nothing about nodes, channels or time. -**A photographic underlay is not an op.** The registered source frame that an -animator traces over is a reference, not output, and it may not enter the indexed -buffer — the same rule `docs/architecture.md` already sets for handles and -vertex boxes. It is a `drawImage` at an affine on a separate canvas, which clips -at the canvas edge for free, and the only thing it needs from the model is the -world transform of the node it rides: +**A tracing layer is an op that never reaches the raster.** Footage or a still +to draw over is a symbol with `:type :trace` and a `:media`, placed by an ordinary +instance — so it is moved, scaled, trimmed, held and put in a lane like anything +else — and it resolves to one `:trace` op: `{:kind :trace :node :layer :media +:frame :size :m}`. The raster refuses that kind, the player hands it to a +`drawImage` on a separate canvas over the picture, and `clip/resolver` makes one +only when asked with `:tracing?`, which only the stage does. An export, a +symbol's centre and a thumbnail never ask, so a reference cannot reach the +picture by any path that forgets to filter it. See `docs/tracing-symbol-plan.md`. -```clojure -(world-of resolver :head) ;; -> Float64Array[6] -``` +A face's footage is one of these, placed as `:plate` under `:head` with the +anchor fit itself as its measured transform — the inverse of the head's, over +image height. Its world is `head · fit · 1/H`, so on a frame where the head and +the plate read the same measured frame the two cancel and the photo sits where +the face was filmed; on any other frame it rides the head. Registration is the +ordinary walk, not a matrix built beside it. -Composed with image-pixels-to-local — **both axes divided by `imgH`**, never by -their own dimension — the photo is registered with the shapes by construction, -and an unregistered underlay is merely decorative. The tracing editor chooses -which source frame to show under a cel. That reference choice is independent of -the finished picture fps and does not change the dense analysis track. A cel can -therefore use any useful source frame as its drawing reference, even when that -frame is not one of the displayed picture poses. - -A photo that has to sit *between* two drawn layers is the case that would make it -a `:bitmap` node with an op of its own. Nothing wants that yet: a reference is -either under everything or over everything at low alpha. +Which frame it shows is the placement's: `:time {:holds [...]}` holds it on +chosen frames, and a head with `:reads {:holds-of :plate}` jumps to the same +ones. Whether it is showing at all is the editor's, `[:ui :tracing]`. ### Making it fast in CLJS @@ -680,8 +678,8 @@ Proof that it covers what exists, not just what is wanted: | brow ring + quantised raise | node `:brow-r`, `[:geom :pts]` dense (the traced ring with height removed), `[:xform :pos]` dense (the quantised raise). **The decomposition design.md insists on is two channels.** | | head plate, kept frames | node `:head`, `:symbol` per instance, keys on `[:symbol]` at kept frames | | `makeXform` face-oval crop | **gone.** Placement is `[:xform :*]` on the face's own `:place`; the stage clips | -| `stabilize` transforms | dense `[:xform :*]` on `:head`, read through its optional `:anchors` map | -| registered underlay | not data — a UI layer riding `(world-of resolver :head)` | +| `stabilize` transforms | dense `[:xform :*]` on `:head` (the inverse fit) and on its `:plate` (the fit), read where `:reads` and `:time :holds` say | +| registered underlay | the face's `:plate`, an instance of the footage's tracing symbol under `:head`; a `:trace` op the raster never sees | | painted background cel | node per layer, `[:geom :pts]` **framed**, `[:style :color]` framed | | `mouth lead` | `:time {:offset k}` on performance nodes only | | `exposure` | `:time {:expose n}` on the clip root, inherited | diff --git a/docs/frame-selection.md b/docs/frame-selection.md index 9faf0f5..b875ef8 100644 --- a/docs/frame-selection.md +++ b/docs/frame-selection.md @@ -1,5 +1,11 @@ # Frame selection +> Since `docs/tracing-symbol-plan.md`: `domain/trace` is gone. Trace keys are +> the face's `:plate` placement's `:time :holds`, the origin is the head's +> `:reads`, and the photo's registration is the plate's own measured channels. +> Where this document names `trace/measured-local` or `trace/prepare`, read the +> plate's or the head's measured channels and `node/hold`. + Two mechanisms. One vocabulary. An earlier draft of this document claimed they were one component used twice — because `suggestPlateFrames` in the old `js/pipeline.js` and the never-built "performance poses" of @@ -21,7 +27,7 @@ This document is how the picture gets an opinion. holding the latest one at or before now.** The holding half already exists and is already shared: `pose/held-frame` is called -by `trace/held-frame` and by `pose/source-frame`, which is the two sites agreeing +by `node/hold` (a placement's `:time :holds`) and by `pose/source-frame`, which is the two sites agreeing about reading. The hand half is shared too — see *Three layers* below. Choosing is what differs. @@ -32,9 +38,9 @@ what differs. | signal | the measured head's motion | — none; a stored cut | | the baseline it improves on | drawing on 2s | the cadence, or the exposure grid | | the shape of the answer | a non-uniform set out of dense | the same grid, nudged | -| lives on | the face's `:head` `:trace` | the instance's `:playback :tracks` | -| hand edit today | `trace/toggle-frame` | `pose/put-cut` / `pose/remove-cut` | -| UI today | `params/trace-keys` | **none** | +| lives on | the face's `:plate` `:time :holds` | the instance's `:playback :tracks` | +| hand edit today | `::project/toggle-hold` | `pose/put-cut` / `pose/remove-cut` | +| UI today | `params/layer-section` | **none** | | proposes today | **nothing** | **nothing** | ## Why they are not one function @@ -97,7 +103,7 @@ implementation for both. A preserve mark *is* a keep. ### Materialise the result, do not derive it on the render path The effective set is written back to where each site already reads it — -`:trace :frames`, or the pose track — so that every existing reader is untouched +the plate's `:time :holds`, or the pose track — so that every existing reader is untouched and nothing on the per-frame path has to open a dense block. Proposing is a command, not a subscription. `ch/value-at` allocates per call and says so; that is fine for a button press over a few hundred frames and would not be fine at @@ -349,7 +355,7 @@ against anything. ### Between the kept frames is a third shared field -`:trace :origin` is not a tracing setting. It is the answer to *what happens +The head's `:reads` (once `:trace :origin`) is not a tracing setting. It is the answer to *what happens between kept frames*, and the plate selection has to answer it: - `:continuous` — ignore the selection for this purpose and read the frame you @@ -430,7 +436,7 @@ Render it in two sections: invariant from *The upgrade path* here, since this is the first place a real signal exists to assert it over. 4. **Storage for the plate selection.** `:policy`/`:keep`/`:drop` beside - `:trace :frames`. Extend `leaf/leaves` and the key whitelists in the same + the plate's `:time :holds`. Extend `leaf/leaves` and the key whitelists in the same commit — a field without a leaf saves silently and comes back missing, which is the one bug persistence must not be able to have. Round-trip test. 5. **Re-suggest preserves hand decisions.** Propose at one tolerance, pin a frame, diff --git a/docs/timing-model.md b/docs/timing-model.md index bb45ec0..5cf947f 100644 --- a/docs/timing-model.md +++ b/docs/timing-model.md @@ -84,8 +84,8 @@ candidate poses so stage cuts can still select any of them. | --- | --- | --- | | Source frames and timestamps | Footage/analysis | Constant-rate frame indexing exists; variable timestamps remain future work | | Head anchor map | `:head` node | Implemented, stored with the node | -| Trace frames (photo address) and origin | Face symbol's `:head` `:trace`, written through `:anchors` | Implemented, see `domain/trace` | -| Showing the tracing photo, and its opacity | Face instance's `:underlay` | Implemented; a drawing aid, not keyed | +| Trace frames (photo address) and origin | The face's `:plate` `:time :holds`, and its `:head` `:reads` | Implemented, see `docs/tracing-symbol-plan.md` | +| Showing a tracing layer, and its opacity | Editor state, `[:ui :tracing]` | Implemented; a drawing aid, never saved or keyed | | Generated picture-rate proposal and closure protection | Roto clip/symbol | Generated-only picture sampling exists; closure protection remains future work | | Stage pose cuts | Symbol instance | Implemented, stored with the instance | diff --git a/docs/tracing-symbol-plan.md b/docs/tracing-symbol-plan.md new file mode 100644 index 0000000..2e2e7ba --- /dev/null +++ b/docs/tracing-symbol-plan.md @@ -0,0 +1,411 @@ +# Plan: tracing is a symbol + +Status: built, 2026-10-03, on branch `worktree-tracing-layers`. Where the build +differs from the plan below, the build wins: + +- The head's field is `:reads` (`{:holds [...]}` or `{:holds-of :plate}`), not + `:follow`. +- A still is not one frame. It gets as many frames as remain in the symbol it is + dropped into, at that symbol's rate, and is trimmed like any clip. +- An image's `:media` is `{:image }`, served at `/blob/`. The + server's `Image` row exists for the pool's list and labels, not for identity. +- Dropping media that a tracing symbol in the document already shows reuses that + symbol. +- Nothing can be created or dropped inside a tracing symbol (`creation/target`, + `drop-destination-at`, `nest/move-refusal`), and one cannot be opened in a tab. +- Schema 6. Every project is marked 6. One that still carries `:trace` on a + node is refused when opened, with what to do about it, rather than all old + projects being refused. + +No backward compatibility (see `lane-model.md`, "Goal and compatibility policy"). + +## What is wrong today + +Tracing is not a thing in the document. It is three mechanisms that each know +about faces: + +- **`:trace {:frames :origin}` on a face's `:head`.** It decides which measured + frame the head reads (`symbol/base-channel-frame` → `trace/held-frame`), and + the underlay also reads it to decide which photo to show (`trace/photo-frame`). +- **`ui/underlay`**, a painter that walks `trace/shown` → `trace/faces` (a + separate instance walk), looks up each face's subject → analysis → footage, + asks the resolver where `[...path :head]` went, and builds the photo's matrix + by hand: `world(head) · M(p)⁻¹ · 1/imageH` (`trace/photo-matrix`). +- **`[:ui :trace {:faces #{} :opacity}]`**, a per-face switch in editor state, + plus the special "opening a face shows its footage, opening a take does not" + rule (`trace/showing-for`). + +What you cannot do: place footage or a still as a reference where you like, move +it, scale it, turn it, trim it, hold it, put it in a lane, or trace something +that is not a tracked face. The photo is not selectable and has no row. The one +case that works (a face over its own footage) runs on code that nothing else +uses. + +## The model in one paragraph + +A **tracing symbol** is a symbol with `:type :trace`. It has no nodes. It names +media (a footage range or a still image) and has that media's frame count, fps and +pixel size. It is **placed by an ordinary instance**, so lanes, spans, trim, +split, move, playback (`:in :speed :end`), transform, nesting, selection, picking +and gestures all work on it with no new code. It is evaluated to a single +**`:trace` op**, which never reaches the raster or an export. A face's footage is +not a special case. It is one of these instances, placed inside the face as a +child of `:head` and carrying the measured registration transform. The trace keys +become **holds on that instance's time**. The head's "origin" becomes the head +saying **which node's frames it follows**. + +This follows the precedent of `:type :palette` symbols (no nodes, they name an +asset, and they are placed by instances in a lane), so the uniformity rule holds +(see the `model-uniformity` memory). Nothing new holds nodes, and no special +instance kind is added. + +## Data model + +### The tracing symbol + +```clojure +:footage-8625 ; a symbol id like any other +{:id :footage-8625 :name "8625.mov" + :type :trace + :media {:footage #uuid "f8ca…" :range [12 241]} ; or {:image "sha256:…"} + :frames 229 ; the range's length; 1 for a still + :fps 30 ; the footage's own rate, so cadence handles 30→12 for free + :width 1440 :height 1920 ; pixel size = the symbol's own stage + :audio {:footage #uuid "f8ca…"} ; optional, the existing "a symbol says what it sounds like" + :nodes {}} +``` + +- `:media` is the one new symbol key. Add it to `symbol/symbol-keys` and to the + symbol leaf's `select-keys` in `leaf/leaves`. +- The symbol's local space is **pixels of the media**: `[0 w) × [0 h)`. + `clip/center` of a node-less symbol already returns its stage middle, which here + is `[w/2 h/2]`. So `place-symbol` puts the anchor at the image centre with no + new code. +- `symbol/problems`: `:type :trace` needs `:media` with exactly one of + `:footage`/`:image`, needs `(empty? nodes)`, and for footage needs + `(= frames (- end start))`. +- **One tracing symbol per footage range, shared.** Two faces from one take + place the same symbol, and so does a hand-placed reference. Like any symbol it + is reused by reference, and `bring/symbols` copies it like any other. + +### Placing it + +It is an ordinary `:kind :instance` with `:source {:symbol :footage-8625}`: + +- **Start and end** are `:span` (in its own frames) and `:time :at`. You get + these from the existing trim, split, move and roll. +- **Which frame shows** comes from `:playback {:in :speed :end}`. Footage plays + with speed 1, a frozen frame has speed 0, and a still is 1 frame with `:end :hold`. +- **Placement in space** is `[:xform …]`, the same as every node: gestures, + inspector and keys. +- **In a lane, or on its own**: it is a parent-less spanned node, so a `:display + :lane` symbol draws it as a block like any cel. It can also sit as a free node + or under a group. To have a reference with its own tab, wrap it in an ordinary + symbol. + +**Default on creation** (in the drop event, not in `place-symbol`): scale so the +image's height fits the stage height, centred on the drop point. This is a +creation default like "use center anchor when dropping", and nothing updates it +afterwards. + +### Holds: the one new time feature + +`:time {:holds [0 12 30]}` floors a node's local frame to the last hold at or +before it. Before the first hold, the first hold applies. This is `:expose` +generalized from a regular grid to authored frames. It is applied in +`node/local-frame` at the same point as `:expose`, and is inherited in the same +way. It is in the node's **own** frames. On a tracing instance with +`:in 0 :speed 1`, own frames are source frames, so the hold list is the set of +traced frames. + +It is general on purpose. Holding a playing symbol on chosen drawings is the +same feature. It costs about three lines in `local-frame`, one `problems` clause +(sorted, distinct, finite), and the timeline drawing hold frames as marks on the +row. + +`symbol/frame-map` refuses floors (`:expose > 1`). It must also refuse a +non-empty `:holds`, for the same reason. + +## The face: "a child symbol that represents the trace" + +The face symbol after a freeze: + +``` +:place group authored source→stage mapping (image heights → stage px) + :head group measured M(p) :reads — see below + :mouth … parts generated, every frame + :plate instance of :footage-8625 ← the trace + measured channels: fit(p) = M(p)⁻¹ · S(1/imageH) + :time {:holds [0 12 30]} ← the trace keys +``` + +### Registration comes from the parent + +The plate is a child of `:head`. Its own measured channels are the +**stabilizing fit** at frame p: the inverse of the head's measured transform, +with pixels → image heights folded into the scale (it stays a similarity, so it +decomposes into `pos`/`rot`/`scale`). Freeze already computes the fit; the +head's measured channels are its inverse (`freeze/invert`). The plate gets a +second dense block, which is three small channels. + +Because holds are a **time** floor, the plate's channels and the frame its +content shows are read at the **same** held frame q. Its world transform is: + +``` +world(plate) = place · M(p_head) · M(q)⁻¹ · S(1/H) +``` + +That is exactly `trace/photo-matrix`, but now it falls out of the ordinary walk. +It is registered to whatever the head is doing, by construction: + +| head reads (p_head) | plate shows (q) | result | +| --- | --- | --- | +| f (continuous) | f (no holds) | footage where filmed | +| f (continuous) | held key | held photo rides the moving head (today's behaviour) | +| held key (same as plate) | held key | `M(q)·M(q)⁻¹ = I`: photo sits where filmed | +| 0 (start) | f or held | stabilized footage under a still head | + +If a frame has no measurement, the plate's dense channel has nothing there, so +`xform-at` gives nil, so the plate is not placed and no photo shows. That is what +happens today too. + +### Trace keys versus origin: who owns what + +The two decisions are separate and stay separate: + +- **Trace keys** are which footage frames get drawn over (the "plate drawings" + in `frame-selection.md`). They belong to the **plate**, as `:time :holds`. A + hand-placed tracing layer with holds is the *same thing*: the face's plate is + an ordinary tracing placement and nothing more. +- **Origin** is how the head moves between kept frames. `frame-selection.md` + already says `:origin` "is not a tracing setting" but a performance one, so it + belongs to the **head**: + +```clojure +:head {…} ; continuous — reads its own frame +:head {… :reads {:holds [0]}} ; start +:head {… :reads {:holds-of :plate}} ; at keys — reads its measured channels at + ; the frames :plate's holds select +``` + +`{:holds [...]}` is also what a face with no footage uses for "at keys", since it +has no plate to follow. + +`:reads` changes only the **head's own channel reads**, not its children's +frames. The parts must keep running every frame, which is why this cannot be a +`:time` hold on the head. It is today's `traces` branch of +`base-channel-frame` with the hold list read from the named node. A node +reference has precedent (`:stencil`, `:pose-group`). It points from the follower +to the thing followed, so there is still one stored list of frames and nothing to +keep in sync. Validate it in `symbol/problems` the way `:stencil` is validated: +the target exists in the symbol, and it is not the head itself or an ancestor of +the head. + +Rejected alternatives, so they are not re-proposed: + +- **Keys on the head, and the plate reads them.** This is today's direction. It + makes the plate special: a free tracing layer could not have keys that a face's + plate also understands. +- **Hold `:time` on `:head`.** Exposure inherits strictly, so the mouth and eyes + would freeze along with the head. +- **A wrapper "registered footage" symbol holding the fit.** It is correct but + adds a symbol per face. The time-floor holds already put the fit read and the + content read on the same frame, so the wrapper buys nothing. +- **Plate as a sibling of `:head` at identity.** It is only registered when the + head and the photo read the same frame, so it breaks the continuous-plus-holds + and start rows above. + +## Evaluation: an op that is never rendered + +- **`clip/resolver`**, in the instance branch: when the source symbol has + `:type :trace`, it does not recurse. If `(:tracing? opts)` is set, it emits one op: + + ```clojure + {:kind :trace :node [id] :m :media … :frame shown-frame :size [w h]} + ``` + + The frame comes from the same `placed-frame` path as any instance, so playback, + holds and the fps cadence apply. Do not build a child resolver for a trace + symbol. +- **`transform-op`** gets a `:trace` case that composes the matrix. Row paths, + solo filtering and nesting at any depth then work for free. +- **The output guarantee is structural.** `:tracing?` defaults to false. Only the + stage's `::render/resolver` passes true. Export, `clip/center` (a big photo + must not pull a symbol's pivot), thumbnails and the bench never ask for trace + ops. `raster/draw-ops!` keeps throwing on unknown kinds, so a leak fails loudly. +- **`ui/player`** sends picture ops to the raster and `:trace` ops to the + painter. +- **`ui/underlay` becomes `ui/tracing`.** It paints `:trace` ops in draw order + with `drawImage` at `op.m` and the global opacity. The media URL is the footage + manifest's `urls[range-start + frame]`, or the image blob URL. The three + steadiness fixes stay: LRU cache, hold the last still per op `:node`, and read + ahead while playing. Everything that walked faces is deleted. +- **`pick`**: a `:trace` op is hit when the point, mapped through `m⁻¹`, falls in + `[0 w) × [0 h)`. Picture ops are tested first and trace ops only if nothing + drawn is under the pointer, so a full-frame photo does not steal every click. + When the global switch is off, traces are neither painted nor picked. +- **Gestures**: no change. The face's plate is measured, so `gesture/refusal` + already says "place the instance it is in". A hand-placed layer is authored and + moves, turns and scales like anything else. + +## On and off + +All of it is EDITOR STATE, as ed88c5e decided for the per-face switch: showing +a reference is a way of looking at the stage, so it is not an undo step, does not +travel to collaborators, and cannot reach an export. + +```clojure +[:ui :tracing {:on? true :opacity 0.5 :hidden #{[sid node-id] …}}] +``` + +- **One layer** is an entry in `:hidden`, keyed by the symbol the tracing + instance is in and its node id. The symbol id is needed because every face's + plate is called `:plate`. Keyed this way, hiding a face's plate hides it in + every placement of that face, which is what the per-face switch did. Shown is + the default, so opening a face shows its footage with no setup. +- **How it's applied**: `clip/resolver` already knows which symbol and node a + trace op comes from, so the op carries `:layer [sid id]`. The painter and + `pick` skip ops whose layer is hidden. The resolver does not change when you + toggle a layer, so nothing is rebuilt. +- **Global**: `:on?` and `:opacity`, a toggle plus an opacity slider in + `ui/palette/bar` next to the palette controls. When it is off, traces are + neither painted nor picked. +- **Switching one layer on also switches the global setting on.** This applies + from the tracing instance's inspector, from its timeline row, and from the + face/roto section. A layer you just enabled must not stay invisible behind a + switch you forgot. Turning one layer off never touches the global switch. It is + one event (`::ui/show-trace layer on?`) that updates `:hidden` and, when + turning on, sets `:on? true` in the same handler, so the three places cannot + behave differently. +- **`[:vis]`** still works on a tracing instance like on any node: key it to + show a reference only over part of the shot. It is not the on/off switch. +- **Deleted**: `[:ui :trace :faces]`, `::trace-face`, `::trace-faces`, + `trace/showing-for`, `traceable-faces`, `shown`, `faces`, and the "a take shows + nothing by default" rule. + +## UI + +- **Timeline.** A tracing instance is an ordinary row or clip block, styled to + read as reference-only (hatched block, an eye icon in place of the colour + chip). Hold frames are marks on the row. The row's eye button dispatches + `::ui/show-trace`, which replaces the face row's `tl-trace` button. +- **Inspector, tracing instance.** Show the media (footage name and range, or + image), an on/off control (which goes through `::ui/show-trace`), and the hold + list ("hold here" / "remove hold" plus seek buttons, which is `trace-keys` + re-aimed at `:time :holds`). Playback, transform and span use the existing + sections. +- **Inspector, face/roto section.** The same hold controls, aimed at the face's + `:plate`. Origin buttons write `:head :reads`. The section is found by "this + face has a `:plate`", not by `trace/traceable?`. +- **Palette bar.** The global toggle and opacity slider. +- **Making one.** + - Footage: the convert dialog gets a choice between "animate faces" (today's + flow) and "tracing layer" (no detection). The tracing-layer choice makes the + `:type :trace` symbol for the chosen range and places it where the video was + dropped. + - Footage can also be dragged from the pool with a modifier or as a second + drag kind. + - A still image: drop it on the pool and it uploads; drop it on the stage or + timeline and it is placed with `:end :hold` and a span of the host's + remaining frames. + +## Freeze, bring, regenerate + +- **`freeze/subject-part`** emits the `:plate` instance under `:head`, with fit + measured channels and `:time {:holds []}`, and emits one `:type :trace` symbol + for the analysed footage range. `bring/take` already copies every symbol the + take reaches and rewrites `:source :symbol` references, so the tracing symbol + comes along. It gets the footage's `:audio` too, so a dropped tracing layer + can bring its sound through the existing link. +- **`head-mode`** writes `:reads` on the head and `:holds` on the plate, in + place of `:trace`. The `frame-selection.md` plate proposal materializes into the + plate's `:time :holds` in place of `:trace :frames`. +- **Regenerate** replaces both measured blocks (head and plate) and keeps + `:holds` and `:reads`. Check that `regenerate-head`'s measured/authored + comparison handles a second measured node. + +## Server + +- `Image` endpoints mirroring `Sound`: `POST /api/images` stores a `Blob` and + returns `{id, width, height, url}`, and `GET /api/images` lists them. The + migration is a model only. Bump `schema_version` and refuse older documents + clearly. Do not convert them. + +## Deleted outright + +`trace/of`, `prepare`, `held-frame`, `problems`, `toggle-frame`, `photo-frame`, +`measured-local`, `photo-matrix`, `traceable?`, `faces`, `traceable-faces`, +`showing-for`, `shown`, `opacity-default`. `domain/trace.cljs` probably goes +away entirely; if anything is left, it is a hold helper that belongs in `node`. + +Also deleted: `symbol/prepared-traces` and the `traces` arm of +`base-channel-frame`, which becomes the `:reads` lookup. `:trace` on nodes (and +`symbol/problems` reports it as removed). `::project/set-trace`. +`::render/underlay` and `::render/tracing` become one `::render/tracing` that +returns `[:ui :tracing]`. The face lookup in `params/view`. + +Expected effect on code size: net negative. One time-floor clause, one op kind, +one resolver branch, one pick case and a `:reads` lookup replace the face walk, +the hand-built photo matrix, per-face showing state and the underlay's face +bookkeeping. Measure the change and report the number honestly (see the +`cljs-style` memory). + +## Tests + +**Domain (node).** + +- `:holds` in `local-frame`, with eval-frame and resolver agreeing forward, + backward and in random order. +- A trace op appears only with `:tracing?`, and `export/run!` output contains no + `:trace` op. +- `transform-op` composes the matrix through two nesting levels. +- `clip/center` ignores traces, and is `[w/2 h/2]` for a tracing symbol. +- `pick` returns picture ops before traces and inverse-maps through a rotated + layer. +- `symbol/problems` covers `:type :trace`, `:reads` targets and cycles, and + `:holds` shape. +- Registration: port `trace_test`'s photo-matrix assertions onto the walk. + For every row of the registration table, the plate's world transform equals + the expected matrix. At a held key with `:reads {:holds-of :plate}`, it equals + `world(:place) · S(1/H)`. +- Freeze emits the plate and the tracing symbol. Regeneration keeps the holds. +- The `::ui/show-trace` event sets the global switch on when turning a layer on + and leaves it alone when turning one off. + +**Browser** (CDP, see the `arthur-verify-dont-guess` memory): + +- Drop a still, then drag, turn and scale it. +- Toggle a layer on with the global switch off, and confirm both are on and the + photo paints. +- Export a frame and check it has no photo pixels. +- Open a face: the plate is registered at a hold with origin "at keys", and + stabilized with origin "start". + +## Order of work, each step green + +1. `:time :holds` in `node/local-frame`, `problems`, `frame-map`, and timeline + marks. +2. `:type :trace` symbol, `:media`, the `:trace` op behind `:tracing?`, + `transform-op`, the player split, `ui/tracing` painter and `pick`. Footage + media only, placed by hand from a REPL or test document. +3. The face: freeze and bring emit the plate and tracing symbol, `:reads` + replaces `:trace`, delete the old trace and underlay paths, and re-aim the + inspector's face section. +4. On/off: the `:hidden` set and `:layer` on trace ops, the row eye, the + `::ui/show-trace` rule, the global toggle and opacity in the palette bar, and + delete `[:ui :trace :faces]`. +5. Creation: "tracing layer" in the convert dialog, pool drag, then the image + endpoint, pool images and image drops. +6. Docs: `animation-model.md` "A photographic underlay is not an op" becomes "a + trace is an op that never reaches the raster". Update `frame-selection.md` + (where plate keys live) and `lane-model.md` (tracing clips in lanes). + +## Open questions + +1. **"A take shows its faces' footage."** Dropping the old "only in a face's own + tab" default means opening a take shows every face's plate if the global + switch is on. The recommendation is to accept that, since the switch is one + click. +2. **Lane-model audio rule.** `lane-problems` requires visual cels. A tracing cel + is visual for this purpose, but say so explicitly when a lane may hold both + tracing and drawing cels. diff --git a/frontend/src/arthur/db.cljs b/frontend/src/arthur/db.cljs index b53f3c8..e14d793 100644 --- a/frontend/src/arthur/db.cljs +++ b/frontend/src/arthur/db.cljs @@ -12,7 +12,6 @@ are entirely dense." (:require [arthur.demo :as demo] [arthur.domain.clip :as domain-clip] - [arthur.domain.trace :as trace] [arthur.demo.swarm :as swarm] [arthur.demo.take :as take])) @@ -49,6 +48,18 @@ (defn clip-entry [id] (some-> (get-in clips [id :entry]) deref)) +(def tracing + "How tracing layers show on the stage: all of them or none, how strongly, and + which ones are switched off, as `[symbol-id node-id]` — the symbol a tracing + placement is in and its id, so a face's footage is one layer wherever the face + is placed. + + THE EDITOR'S, NOT THE DOCUMENT'S. Showing a reference is a way of looking at + the stage, like solo and zoom: not an undo step, not sent to collaborators, + and it cannot reach an export. A layer is shown unless it is in `:hidden`, so + footage brought in shows without being found and switched on first." + {:on? true :opacity 0.5 :hidden #{}}) + (def default {;; --- the document --- ;; @@ -165,7 +176,7 @@ :auto-key? false :draft [] :knobs {} - :trace {:faces #{} :opacity trace/opacity-default} + :tracing tracing :expanded #{}}}) (def rates diff --git a/frontend/src/arthur/domain/bring.cljs b/frontend/src/arthur/domain/bring.cljs index 97a18b5..30fddaf 100644 --- a/frontend/src/arthur/domain/bring.cljs +++ b/frontend/src/arthur/domain/bring.cljs @@ -87,7 +87,8 @@ (keys (:subjects frozen))))) (symbol-id label)) scoped (partial scope root) - wanted (into {:main root} (map (fn [subject] [subject (scoped subject)])) + wanted (into {:main root :footage (scoped :footage)} + (map (fn [subject] [subject (scoped subject)])) (keys (:subjects frozen))) {c :clip ids :ids} (symbols clip frozen [:main] wanted) sid (ids :main) diff --git a/frontend/src/arthur/domain/clip.cljs b/frontend/src/arthur/domain/clip.cljs index c0a49e2..65c8957 100644 --- a/frontend/src/arthur/domain/clip.cljs +++ b/frontend/src/arthur/domain/clip.cljs @@ -59,6 +59,12 @@ [clip sid] (get-in clip [:symbols sid])) +(defn trace? + "Is `sym` a tracing symbol: footage or a still to draw over, which is placed and + moved like any symbol and never drawn into the picture? See `trace-op`." + [sym] + (= :trace (:type sym))) + (defn symbol-name "What to call a symbol: its `:name`, or its id when it has none." [clip sid] @@ -199,10 +205,13 @@ "The symbol a document opens on: the longest one nothing else places, ties broken by id. The symbol that contains everything else is the longest of the unplaced ones in every document made so far, and a reserved name is what this - replaces." + replaces. + + Never a tracing symbol: it is footage nobody has placed yet, and as long as + the take it came from, so it would otherwise win." [clip] (first (sort-by (fn [sid] [(- (or (frames clip sid) 0)) (str sid)]) - (unplaced clip)))) + (remove #(trace? (symbol clip %)) (unplaced clip))))) (defn set-root-fps "Set the document/output rate and the root symbol's editing rate together. @@ -266,8 +275,31 @@ (assoc op :cx x :cy y :r (* scale (:r op)))) :rect (let [[x y] (at (:cx op) (:cy op))] (assoc op :cx x :cy y :size (* scale (:size op)))) + :trace (assoc op :m (node/mul! (node/mat) m (:m op))) op))) +(defn- trace-op + "The one op an instance of tracing symbol `sym` makes, showing its frame `frame` + at world `m`, from node `id` of symbol `sid`. + + NOT A PICTURE OP. Nothing indexed can show a photo, so the raster refuses this + kind and the player hands it to `ui/tracing` instead; and the resolver makes one + only when asked with `:tracing?`, which only the stage does. An export, a + symbol's centre and a thumbnail never ask, so a reference cannot reach the + picture by any path that forgets to filter it. + + `:layer` is what the on/off switch for one layer is keyed by: the symbol the + placement is in and its id, so a face's plate is one layer wherever the face + is placed." + [sym frame m sid id] + {:kind :trace + :node id + :layer [sid id] + :media (:media sym) + :frame frame + :size [(:width sym) (:height sym)] + :m (js/Float64Array.from m)}) + (defprotocol IActivePalette (active-palette [this] "The palette selected by this resolver's most recently resolved frame.")) @@ -345,11 +377,13 @@ #(pal/render-index palette @active %) palette) (assoc opts :pose-tracks pose-tracks)) - ;; Each cel owns its source resolver and mutable buffers. + ;; Each cel owns its source resolver and mutable buffers. A + ;; tracing symbol has nothing to resolve: it is one op. children (into {} (for [[id n] nodes :when (= :instance (:kind n)) - child (sort-by str (node/sources n))] + child (sort-by str (node/sources n)) + :when (not (trace? (symbol clip child)))] [[id child] (build child (conj chain sid) (get-in n [:playback :tracks]) false)])) ;; The instances that were on the last frame, and WHICH @@ -383,7 +417,15 @@ shown (when (and m (number? local)) (placed-frame clip sid n local)) frame (:frame shown)] - (if (and frame (<= 0 frame) (< frame length)) + (cond + (not (and frame (<= 0 frame) (< frame length))) + [] + + (trace? (symbol clip (:symbol shown))) + (when (:tracing? opts) + [(trace-op (symbol clip (:symbol shown)) frame m sid id)]) + + :else (do (vswap! entered assoc id (:symbol shown)) (map #(transform-op % m [id]) ((get children [id (:symbol shown)]) @@ -403,8 +445,7 @@ (dec frame)) @active (when (and context? (:palette n)) - (selection-at n frame nil))))) - [])) + (selection-at n frame nil))))))) (when-let [op (get by-id id)] [op])))) ids))))] (reify diff --git a/frontend/src/arthur/domain/creation.cljs b/frontend/src/arthur/domain/creation.cljs index 49660b0..80a56db 100644 --- a/frontend/src/arthur/domain/creation.cljs +++ b/frontend/src/arthur/domain/creation.cljs @@ -50,12 +50,17 @@ but the occurrence chain that reaches the lane must still be valid. The empty path always resolves to the open symbol. + A tracing symbol is never one: it holds a picture to draw over and no nodes, + so selecting a tracing layer creates beside it, as selecting a shape does. + Returns `{:kind :lane|:symbol :sid :path :frame :matrix :time}`." [document store open selection frame] (let [preferred (preferred-path document open selection)] (some (fn [path] (when-let [inside (nest/inside document store open path frame)] - (when-let [sid (:sid inside)] + (when-let [sid (and (:sid inside) + (not (clip/trace? (clip/symbol document (:sid inside)))) + (:sid inside))] (assoc inside :path path :kind (if (symbol/lane? (clip/symbol document sid)) :lane :symbol))))) diff --git a/frontend/src/arthur/domain/leaf.cljs b/frontend/src/arthur/domain/leaf.cljs index e47accf..822294d 100644 --- a/frontend/src/arthur/domain/leaf.cljs +++ b/frontend/src/arthur/domain/leaf.cljs @@ -45,9 +45,10 @@ WHY `measured` IS ONE LEAF AND CHANNELS ARE NOT. `:head`'s measured channels are not authored: they are written together by a freeze and replaced together by a - re-freeze, and `head-mode` exposes them through `:channels`. The optional - `:trace` on the head node chooses which measured frame those channels read. - A leaf per measured + re-freeze, and `head-mode` exposes them through `:channels`. The same is true + of a face's `:plate`, whose measured channels register its footage. Which + measured frame they read is the node's `:reads` and `:time :holds`, which + save with the node. A leaf per measured channel would offer a write nobody can make. The authored channels beside them are one leaf each, because a hand writes one at a time. @@ -149,7 +150,8 @@ (for [[sid sym] (:symbols clip)] {(at "symbol" (segment sid)) (select-keys sym [:name :frames :fps :width :height :palette - :palette-track :palette-channel :type :palette-ref :display])}) + :palette-track :palette-channel :type :palette-ref :display + :media])}) (for [[sid sym] (:symbols clip) [id n] (:nodes sym)] {(at "symbol" (segment sid) "node" (segment id)) diff --git a/frontend/src/arthur/domain/nest.cljs b/frontend/src/arthur/domain/nest.cljs index e8a6cdc..978c931 100644 --- a/frontend/src/arthur/domain/nest.cljs +++ b/frontend/src/arthur/domain/nest.cljs @@ -95,6 +95,23 @@ :matrix (node/mat) :time node/same-time} path)) +(defn own-time + "The time map from symbol `sid`'s frames to the OWN frames of the node at row + path `path` from it, with `sid` showing frame `f`: the frames its keys and + holds are written in. Nil where no affine map exists — through a loop or a + floor above the node, or where it is not on screen. + + It STOPS AT THE NODE, where `inside` goes into what the node places, and it + leaves out the node's own floors: a hold is added on a frame the node's holds + would otherwise floor away." + [clip store sid path f] + (when-let [{inner :sid t :time} (inside clip store sid (pop path) f)] + (let [nodes (:nodes (clip/symbol clip inner)) + n (get nodes (peek path)) + up (symbol/frame-map nodes (:parent n))] + (when (and n t up) + (-> t (node/then-time up) (node/then-time (node/time-of n))))))) + (defn placement "Where the node at row path `path` is, from symbol `sid` showing frame `f`, as a transform would change it: `{:sid :id :frame :parent :world}` — the symbol it @@ -319,6 +336,8 @@ (nil? n) "nothing to move" (or (nil? here) (nil? there)) "both have to be on screen at this frame" (nil? (:sid there)) "only a symbol can take it" + (clip/trace? (clip/symbol clip (:sid there))) + "a tracing layer is a picture to draw over — nothing goes inside it" (span/placement-refusal clip (:sid there) n) (span/placement-refusal clip (:sid there) n) (not (and (:time here) (:time there))) diff --git a/frontend/src/arthur/domain/node.cljs b/frontend/src/arthur/domain/node.cljs index 34d4775..5e48816 100644 --- a/frontend/src/arthur/domain/node.cljs +++ b/frontend/src/arthur/domain/node.cljs @@ -295,16 +295,33 @@ (defn finite-number? [v] (and (number? v) (js/Number.isFinite v))) +(defn hold + "Floor `f` onto the last of `holds` at or before it, and before the first onto + the first. Exposure on authored frames rather than on a grid: `[0 12 30]` shows + frame 12 from 12 until 30. What a tracing layer's held photos and a face's + trace keys are. + + A `reduce` that stops at the first hold past `f`, because this runs per node + per frame and the list is sorted." + [f holds] + (if (seq holds) + (reduce (fn [held h] (if (<= h f) h (reduced held))) (first holds) holds) + f)) + (defn local-frame - "Apply an artistic time map within one frame space: expose, then offset. - Frame-rate selection happens at symbol boundaries in domain/clip." + "Apply an artistic time map within one frame space: expose, hold, then offset. + Frame-rate selection happens at symbol boundaries in domain/clip. + + Holds come after exposure and inherit the same way, strictly: they are a floor + of this node's own frame, and its children are handed the floored frame." [n f] - (let [{:keys [mode at rate offset] ex :expose :or {mode :map at 0 rate 1}} (:time n)] + (let [{:keys [mode at rate offset holds] ex :expose :or {mode :map at 0 rate 1}} (:time n)] (if (= mode :inherit) f (cond-> (* rate (- f at)) - ex (expose ex) - offset (+ offset))))) + ex (expose ex) + (seq holds) (hold holds) + offset (+ offset))))) ;; --------------------------------------------------------------------------- ;; the transform @@ -468,7 +485,11 @@ (apply < (:span n))))) (conj ":span must be a finite, increasing [in out]") (some? (get-in n [:time :in])) - (conj ":time has an :in — an instance's first frame is the start of its own :span")) + (conj ":time has an :in — an instance's first frame is the start of its own :span") + (let [hs (get-in n [:time :holds])] + (and (some? hs) (not (and (vector? hs) (every? finite-number? hs) + (or (empty? hs) (apply < hs)))))) + (conj ":time :holds must be a vector of increasing frames")) (into (when valid (for [[path _] (:channels n) diff --git a/frontend/src/arthur/domain/pick.cljs b/frontend/src/arthur/domain/pick.cljs index 8d0abff..5e53382 100644 --- a/frontend/src/arthur/domain/pick.cljs +++ b/frontend/src/arthur/domain/pick.cljs @@ -41,22 +41,44 @@ (range n)))) (some #(near-segment? x y (px %) (py %) (px (inc %)) (py (inc %))) (range n))))) -(defn- on? [{:keys [kind pts n cx cy r size]} x y] +(defn- trace-corners + "A trace op's image rectangle on the stage, as four [x y] corners." + [{[w h] :size m :m}] + (for [[x y] [[0 0] [w 0] [w h] [0 h]]] + [(+ (* (aget m 0) x) (* (aget m 2) y) (aget m 4)) + (+ (* (aget m 1) x) (* (aget m 3) y) (aget m 5))])) + +(defn- on? [{:keys [kind pts n cx cy r size m]} x y] (case kind :poly (on-poly? pts n x y) :disc (<= (js/Math.hypot (- x cx) (- y cy)) (+ r slop)) :rect (let [h (+ slop (/ size 2))] (and (<= (js/Math.abs (- x cx)) h) (<= (js/Math.abs (- y cy)) h))) + ;; Into the image's own pixels, where the test is a rectangle however the + ;; layer is turned or scaled. + :trace (when-let [inv (node/invert m)] + (let [[w h] size + u (+ (* (aget inv 0) x) (* (aget inv 2) y) (aget inv 4)) + v (+ (* (aget inv 1) x) (* (aget inv 3) y) (aget inv 5))] + (and (<= 0 u) (< u w) (<= 0 v) (< v h)))) false)) (defn hit "The row path of the topmost op in `ops`, in draw order, at stage point `[x y]`, - or nil." - [ops [x y]] - (some #(when (on? % x y) (path-of %)) (rseq (vec ops)))) + or nil. -(defn- op-bounds [{:keys [kind pts n cx cy r size]}] + WHAT IS DRAWN WINS OVER A REFERENCE. A tracing photo usually covers the whole + stage and is painted over the picture, so topmost-first would make every click + land on it; a trace is picked only where nothing drawn is under the pointer." + [ops [x y]] + (let [{traces true drawn false} (group-by #(= :trace (:kind %)) (rseq (vec ops)))] + (some #(when (on? % x y) (path-of %)) (concat drawn traces)))) + +(defn- op-bounds [{:keys [kind pts n cx cy r size] :as op}] (case kind + :trace (let [cs (trace-corners op)] + [(apply min (map first cs)) (apply min (map second cs)) + (apply max (map first cs)) (apply max (map second cs))]) :poly (reduce (fn [b i] (let [x (aget pts (* 2 i)) y (aget pts (inc (* 2 i)))] (if b (let [[x0 y0 x1 y1] b] @@ -121,7 +143,13 @@ (let [grow (fn [[x0 y0 x1 y1 :as b] x y] (if b [(min x0 x) (min y0 y) (max x1 x) (max y1 y)] [x y x y])) at (fn [f p] (ch/value-at (get (node/channels n) p) f store))] - (case (:kind n) + (case (if (some->> (node/source n) (clip/symbol document) clip/trace?) :trace (:kind n)) + ;; A tracing symbol draws nothing to resolve: what it covers is its media's + ;; pixels, in its own coordinates, on every frame it shows one. + :trace + (let [{:keys [width height]} (clip/symbol document (node/source n))] + (fn [f] (when (clip/placed-frame document sid n f) [0 0 width height]))) + :instance ;; ONE RESOLVER PER DRAWING THE LANE CAN SHOW, built once for the reason ;; the single one used to be: a resolver costs the symbol to build and a diff --git a/frontend/src/arthur/domain/pose.cljs b/frontend/src/arthur/domain/pose.cljs index 2b013b7..a6be6f6 100644 --- a/frontend/src/arthur/domain/pose.cljs +++ b/frontend/src/arthur/domain/pose.cljs @@ -65,7 +65,7 @@ "Frames worth landing on, per pose group: `{group -> ascending frames}`, or nil where nothing is marked. - PRECOMPUTED WHEN THE RESOLVER IS BUILT, beside `prepare` and `trace/prepare`, + PRECOMPUTED WHEN THE RESOLVER IS BUILT, beside `prepare`, and this is the reason it is a function rather than a line inside the snap. A `[:vis]` channel is keys and not dense, so reading one per node per frame would be cheap — but the snap needs the marks SORTED for a backward lookup, and diff --git a/frontend/src/arthur/domain/project.cljs b/frontend/src/arthur/domain/project.cljs index 38920a7..7eed9bf 100644 --- a/frontend/src/arthur/domain/project.cljs +++ b/frontend/src/arthur/domain/project.cljs @@ -29,7 +29,7 @@ (:require [arthur.domain.leaf :as leaf] [arthur.domain.wire :as wire])) -(def schema-version 5) +(def schema-version 6) (defn block-keys "Every tier-2 key a leaf map names, in a stable order." diff --git a/frontend/src/arthur/domain/symbol.cljs b/frontend/src/arthur/domain/symbol.cljs index c407451..c523e98 100644 --- a/frontend/src/arthur/domain/symbol.cljs +++ b/frontend/src/arthur/domain/symbol.cljs @@ -56,7 +56,6 @@ (:require [arthur.domain.channel :as ch] [arthur.domain.node :as node] [arthur.domain.pose :as pose] - [arthur.domain.trace :as trace] [arthur.domain.palette :as pal])) ;; --------------------------------------------------------------------------- @@ -137,7 +136,8 @@ (let [n (get nodes id) t (:time n)] (when (and n (not (contains? seen id)) (not (:loop? t)) - (<= (or (:expose t) 1) 1)) + (<= (or (:expose t) 1) 1) + (empty? (:holds t))) (recur (:parent n) (conj seen id) (conj chain n))))))) (defn lane? @@ -457,7 +457,7 @@ (into {} (remove #(= :audio (:kind (val %)))) nodes))) (defn- base-channel-frame - "A trace selects the measured frames its node reads; marked channels read + "`:reads` holds the frames a node's own channels read; marked channels read instance pose choices, and the preserve-snap where nobody has made one. `lf` is this slot's local frame and `plf` the PREVIOUS slot's, both already @@ -466,10 +466,10 @@ snap needs from the grid, and it is why the pair is threaded this far down instead of the snap happening where the grid becomes a native frame: the snap is per pose group, and a group is a fact that only exists here." - [choices marks traces nodes id c lf plf] + [choices marks reads nodes id c lf plf] (cond - (contains? traces id) - (trace/held-frame (get traces id) lf) + (contains? reads id) + (node/hold (js/Math.floor lf) (get reads id)) (:pose-sampled? c) (let [group (or (:pose-group (get nodes id)) id)] @@ -487,12 +487,19 @@ :else (js/Math.floor lf))) -(defn- prepared-traces [nodes] - (into {} - (for [[id n] nodes - :let [p (trace/prepare (:trace n))] - :when p] - [id p]))) +(defn- holds-read + "The frames node `n` reads its OWN channels at, held — not its children's, + which is what makes this different from `:time :holds` — or nil for every + frame. `{:holds [0]}` is a face's head at its start, and `{:holds-of :plate}` + is the head holding wherever its footage holds, which is one list of frames + read by two nodes rather than two lists kept in step." + [nodes n] + (let [{:keys [holds holds-of]} (:reads n) + hs (if holds-of (get-in nodes [holds-of :time :holds]) holds)] + (when (seq hs) hs))) + +(defn- prepared-reads [nodes] + (into {} (keep (fn [[id n]] (some->> (holds-read nodes n) (vector id)))) nodes)) (defn- eval-into "One frame, as a fold over the nodes in topological order. @@ -550,10 +557,10 @@ choices (pose/prepare pose-tracks) marks (when (and snap (snap (:id sym))) (pose/marks nodes (:frames sym) store)) - traces (prepared-traces nodes) + reads (prepared-reads nodes) ord (order nodes)] (eval-into {:read (fn [id path c lf plf] - (ch/value-at c (base-channel-frame choices marks traces + (ch/value-at c (base-channel-frame choices marks reads nodes id c lf plf) lf store)) :palette palette @@ -636,11 +643,11 @@ ;; argument, and any predicate on one will do. ;; ;; One option, and it is the snap's alone. The plate side will never want - ;; one — its proposal is materialised into `:trace :frames` rather than + ;; one — its proposal is materialised into the plate's `:time :holds` rather than ;; computed on the render path — so this is not half of a pair. marks (when (and snap (snap (:id sym))) (pose/marks nodes (:frames sym) store)) - traces (prepared-traces nodes) + reads (prepared-reads nodes) ord (order nodes) rank (draw-rank nodes ord) cursors (into {} @@ -665,7 +672,7 @@ ctx {:read (fn [id path c lf plf] (when-let [cursor (get-in cursors [id path])] (ch/sample! cursor - (base-channel-frame choices marks traces + (base-channel-frame choices marks reads nodes id c lf plf) lf))) :palette palette @@ -708,9 +715,34 @@ `:display` is how the TIMELINE draws the symbol — `:lane` for its clips as blocks on one row — and is saved because two people editing one document must - play by the same editing rules. See `lane?`." + play by the same editing rules. See `lane?`. + + `:media` is what a `:type :trace` symbol shows — `{:footage id :range [in + out]}` or `{:image sha256}` — and such a symbol has no nodes: its frames are + the footage's, its `:fps` the footage's own and its `:width`/`:height` its + pixels. A still shows the same picture on every one of its frames, so how many + it has is only how long it was made to last, and trimming its placement is + how that changes. See `clip/trace-op`." #{:id :name :frames :fps :width :height :nodes :palette :palette-track - :palette-channel :type :palette-ref :display}) + :palette-channel :type :palette-ref :display :media}) + +(defn- media-problems + "Why `sym`, a tracing symbol, does not say what it shows. Empty when it does." + [{:keys [media frames width height nodes]}] + (let [{:keys [footage image range]} media] + (cond-> [] + (seq nodes) + (conj "a tracing symbol cannot contain nodes") + (not (= 1 (count (filter some? [footage image])))) + (conj ":media must name exactly one of :footage or :image") + (and footage (not (and (vector? range) (= 2 (count range)) + (every? integer? range) (apply < range) + (= frames (- (second range) (first range)))))) + (conj "a footage :media needs a [in out) :range as long as the symbol's :frames") + (not (and (integer? frames) (pos? frames))) + (conj "a tracing symbol is a positive whole number of frames") + (not (and (pos? width) (pos? height))) + (conj "a tracing symbol needs its media's pixel :width and :height")))) (defn problems "Human-readable reasons this symbol will not evaluate. Empty means it will. @@ -731,6 +763,7 @@ (cond-> (and (= :palette (:type sym)) (seq nodes)) (conj "a palette symbol cannot contain nodes")) + (into (when (= :trace (:type sym)) (media-problems sym))) (into (for [[id n] nodes :when (not= id (:id n))] (str "node under key " (pr-str id) " has :id " (pr-str (:id n))))) @@ -745,13 +778,30 @@ (into (for [[id n] nodes p (node/problems n)] (str "node " (pr-str id) ": " p))) - ;; A trace re-addresses the node's measurement, regardless of its name. (into (for [[id n] nodes - :when (some? (:trace n)) - p (if (and (integer? (:frames sym)) (seq (:measured n)) - (= (:channels n) (:measured n))) - (trace/problems (:trace n) (:frames sym)) - ["a trace reads the node's own measured channels"])] + :when (some? (:trace n))] + (str "node " (pr-str id) ": :trace is gone — trace keys are the " + "footage placement's :time :holds, and a head follows them " + "with :reads {:holds-of …}; make the face again from its footage"))) + ;; `:reads` names its holds or another node's, the way `:stencil` names + ;; a node: one that is here, and never one whose frame depends on the + ;; reader's own channels being read first. + (into (for [[id n] nodes + :let [{:keys [holds holds-of] :as r} (:reads n)] + :when (some? r) + p (cond + (and holds holds-of) + [":reads names its own :holds or a node's, not both"] + holds-of + (cond + (not (contains? nodes holds-of)) + [(str ":reads :holds-of " (pr-str holds-of) " is not in the symbol")] + (some #{holds-of} (lineage nodes id)) + [":reads :holds-of names the node itself or one above it"]) + :else + (when-not (and (vector? holds) (every? node/finite-number? holds) + (or (empty? holds) (apply < holds))) + [":reads :holds must be a vector of increasing frames"]))] (str "node " (pr-str id) ": " p))) (into (for [k (remove symbol-keys (keys sym))] (str "symbol has a field with no leaf to save it in: " (pr-str k)))) diff --git a/frontend/src/arthur/domain/trace.cljs b/frontend/src/arthur/domain/trace.cljs deleted file mode 100644 index 37e2f51..0000000 --- a/frontend/src/arthur/domain/trace.cljs +++ /dev/null @@ -1,164 +0,0 @@ -(ns arthur.domain.trace - "Tracing a face over its footage: which source frame the photo shows, where the - face's origin goes between those frames, and where the photo sits on stage. - - TWO OWNERS, because they are two different kinds of decision. The trace frames - and the origin are the face's — they say how its drawings were made and how it - moves, so they live on its symbol's `:head`, and every instance of that face - shares them: - - :trace {:frames [0 12 30] :origin :keys} - - Whether the photo is showing, and how strongly, is not the document's at all. - It is a viewing aid, like soloing a row, so it lives in the editor's own state - — `[:ui :trace]`, a set of face symbols switched on and one opacity — and is - never keyed, saved or exported. A face is the same face wherever it is placed, - so one switch shows it in the take it is placed in AND in its own tab, which is - where it is drawn over; nothing has to be switched on twice or per placement. - - THE TRACE IS THE ONE FACT about which measured frame a head reads. `:continuous` - reads the frame it is on, `:keys` jumps to each trace frame's measured head and - holds it — a hold, not a tween — and `:start` holds frame 0's forever. Before - the first trace frame, the first holds. `domain/symbol` reads the head through - `held-frame`, so nothing is derived from the trace and stored beside it. - Normalisation is per face because each face symbol has its own head." - (:require [arthur.domain.channel :as ch] - [arthur.domain.node :as node] - [arthur.domain.pose :as pose])) - -(def origins [:continuous :keys :start]) - -(def ^:const opacity-default - "How strongly a switched-on photo draws until someone moves the slider." - 0.5) - -(defn of - "The head's trace, a head that has none being one that moves freely." - [head] - (merge {:frames [] :origin :continuous} (:trace head))) - -(defn prepare - "Trace `t` ready to be read every frame without allocating, or nil for a head - that reads the frame it is on." - [t] - (let [{:keys [frames origin]} (merge {:frames []} t)] - (case origin - :start {:entries [] :before 0} - :keys {:entries (mapv #(vector % %) frames) :before (or (first frames) 0)} - nil))) - -(defn held-frame - "The measured frame a head with prepared trace `p` reads at its frame `f`." - [p f] - (pose/held-frame (:entries p) f (:before p))) - -(defn problems - "Why `t` is not a trace for a symbol `frames` long. Empty when it is one." - [t frames] - (let [fs (:frames t)] - (cond-> [] - (not (some #{(:origin t)} origins)) - (conj (str ":origin is " (pr-str (:origin t)) ", not one of " (pr-str origins))) - (not (and (vector? fs) (= fs (vec (sort (distinct fs)))) - (every? #(and (integer? %) (<= 0 %) (< % frames)) fs))) - (conj (str ":frames must be distinct frames of the symbol, in order: " (pr-str fs)))))) - -(defn toggle-frame - "A trace frame at `f`, or none there if there was one." - [t f] - (update t :frames - #(vec (sort (if (some #{f} %) (remove #{f} %) (conj % f)))))) - -(defn photo-frame - "Which of the face's own frames the photo shows at frame `f`: the last trace - frame at or before it, held, and before the first the first. With none, every - frame is its own." - [{:keys [frames]} f] - (if (seq frames) - (or (last (take-while #(<= % f) frames)) (first frames)) - f)) - -(defn- measured-local - "The head's own measured transform at frame `p`, or nil where the face was not - found and there is none." - [head store p] - (let [[pos rot scale] (map #(ch/value-at (get (:measured head) %) p store) - [[:xform :pos] [:xform :rot] [:xform :scale]])] - (when-not (some ch/nothing? [pos rot scale]) - (node/local! (node/mat) pos rot scale [0 0] [0 0])))) - -(defn photo-matrix - "From the pixels of source frame `p`'s still, `image-h` pixels tall, to where - the head at world `world` puts them. - - The landmarks were measured in image heights, so a pixel is 1/image-h of one; - the head's measured transform at `p` takes that frame's image space into the - head's; `world` takes the head's to the stage. When the head is showing frame - `p` itself the middle two cancel, and the photo sits exactly where the face - was filmed." - [world head store p image-h] - (when-let [inv (some-> (measured-local head store p) node/invert)] - (let [px (js/Float64Array. #js [(/ 1 image-h) 0 0 (/ 1 image-h) 0 0])] - (node/mul! (node/mat) (node/mul! (node/mat) world inv) px)))) - -(defn traceable? - "Does symbol `sid` have a measured head to trace over?" - [clip sid] - (seq (get-in clip [:symbols sid :nodes :head :measured]))) - -(defn faces - "The instances of faces inside symbol `sid`, at any depth, as `{:path :in - :face}`: the row path from `sid`, the symbol the instance is in, and the face - symbol it places." - [clip sid] - (letfn [(walk [sid path] - (mapcat (fn [[id n]] - (when (= :instance (:kind n)) - ;; Every drawing the lane can show, not only the one it - ;; happens to be on: a face traced in one cel is the same - ;; face when the lane cuts to another. - (let [p (conj path id)] - (mapcat (fn [child] - (cond->> (walk child p) - (traceable? clip child) - (cons {:path p :in sid :face child}))) - (sort-by str (node/sources n)))))) - (sort-by (comp str key) (get-in clip [:symbols sid :nodes]))))] - (vec (walk sid [])))) - -(defn traceable-faces - "Every face that can be traced while symbol `sid` is open, each once: `sid` - itself when it is a face — open in its own tab, to be drawn over — and the - faces placed inside it at any depth." - [clip sid] - (into [] (distinct) - (cond->> (map :face (faces clip sid)) - (traceable? clip sid) (cons sid)))) - -(defn showing-for - "The faces showing their footage once `sid` is the open symbol, given the ones - `on` already showing. - - OPENING A FACE IS ASKING TO DRAW OVER IT: a symbol has measured footage behind - it only because it was traced from that footage, so its own tab starts with the - footage showing rather than with a switch to be found first. A take or a scene - is the picture itself, and a reference drawn over one would read as part of it, - so nothing is switched on for those. Either way it is switched by hand - afterwards, from the inspector's footage section or from a face's own timeline - row." - [clip sid on] - (cond-> (set on) (traceable? clip sid) (conj sid))) - -(defn shown - "The faces whose footage is showing while symbol `sid` is open, as `faces` lists - them — the row path of the face's instance from `sid`, and the face — filtered - to the `showing` set. - - The open symbol itself is in the list, at the EMPTY path, when it is a face: - that is a face open in its own tab to be drawn over, and the one place tracing - matters most. There is no inheritance to work out and no opacity to carry - because the switch is the face's, not a placement's." - [clip sid showing] - (into [] (filter (comp (set showing) :face)) - (cond->> (faces clip sid) - (traceable? clip sid) (cons {:path [] :in sid :face sid})))) diff --git a/frontend/src/arthur/events/footage.cljs b/frontend/src/arthur/events/footage.cljs index 3a7670c..a88f926 100644 --- a/frontend/src/arthur/events/footage.cljs +++ b/frontend/src/arthur/events/footage.cljs @@ -315,8 +315,10 @@ (fn [_] (-> (js/Promise.all #js [(ingest/available!) (.then (http/GET "/api/sounds") - #(:sounds (js->clj % :keywordize-keys true)))]) - (.then (fn [[footage sounds]] (rf/dispatch [::listed footage sounds]))) + #(:sounds (js->clj % :keywordize-keys true))) + (.then (http/GET "/api/images") + #(:images (js->clj % :keywordize-keys true)))]) + (.then (fn [[footage sounds images]] (rf/dispatch [::listed footage sounds images]))) (.catch (fn [error] (rf/dispatch [::failed (or (ex-message error) (str error))])))))) @@ -356,17 +358,34 @@ (.catch (fn [error] (rf/dispatch [::failed (or (ex-message error) (str error))]))))))) +(rf/reg-fx + ::upload-image! + (fn [file] + (let [form (js/FormData.)] + (.append form "file" file) + (-> (http/POST-form "/api/images" form) + (.then (fn [^js image] (rf/dispatch [::uploaded (.-id image) "image imported"]))) + (.catch (fn [error] + (rf/dispatch [::failed (or (ex-message error) (str error))]))))))) + (rf/reg-event-fx ::upload (fn [{:keys [db]} [_ ^js file]] ;; By type, and by name for a browser that leaves the type empty. - (let [sound? (and file (or (string/starts-with? (.-type file) "audio/") - (re-find #"(?i)\.(mp3|wav|aiff?|flac|ogg|m4a|aac)$" (.-name file))))] + (let [kind (when file + (cond + (or (string/starts-with? (.-type file) "audio/") + (re-find #"(?i)\.(mp3|wav|aiff?|flac|ogg|m4a|aac)$" (.-name file))) + :sound + (or (string/starts-with? (.-type file) "image/") + (re-find #"(?i)\.(png|jpe?g|gif|webp)$" (.-name file))) + :image + :else :video))] (if (or (nil? file) (get-in db [:footage :loading?])) {} {:db (update db :footage merge {:loading? true - :status (if sound? "uploading sound…" "uploading video…")}) - (if sound? ::upload-sound! ::upload!) file})))) + :status (str "uploading " (name kind) "…")}) + (case kind :sound ::upload-sound! :image ::upload-image! ::upload!) file})))) (rf/reg-event-fx ::uploaded @@ -386,10 +405,11 @@ (rf/reg-event-db ::listed - (fn [db [_ footage sounds]] + (fn [db [_ footage sounds images]] (update db :footage merge (cond-> {:available (vec footage) :sounds (vec sounds) + :images (vec images) :chosen (or (:chosen (:footage db)) (:id (first footage)))} (empty? footage) (assoc :status "upload a video to begin"))))) @@ -438,12 +458,13 @@ (rf/reg-event-fx ::relabel (fn [{:keys [db]} [_ kind id value]] - ;; `kind` is `:footage` or `:sound`: two resources with one field between + ;; `kind` is `:footage`, `:sound` or `:image`: resources with one field between ;; them, and one event rather than two that differ by a path and a URL. (let [label (string/trim (str value)) [key url] (case kind :footage [:available (str "/api/footage/" id)] :sound [:sounds (str "/api/sounds/" id)] + :image [:images (str "/api/images/" id)] [nil nil])] (if (or (nil? id) (nil? key)) {} @@ -474,7 +495,7 @@ ::ask-convert (fn [db [_ {:keys [frames label] :as footage} frame point target]] (assoc-in db [:ui :convert] - (merge (select-keys footage [:id :label :frames :fps :video]) + (merge (select-keys footage [:id :label :frames :fps :video :width :height]) {:range [0 frames] :name (string/replace (str label) #"\.[^.]*$" "") :host (get-in db [:ui :open]) :frame frame :point point @@ -499,6 +520,25 @@ ::pb/pause! nil ::convert! {:footage-id id :range range :request request}})))) +(rf/reg-event-fx + ::convert-tracing + ;; The same question answered the other way: the chosen frames become a tracing + ;; layer to draw over, with nothing detected and nothing measured. Its frames + ;; are the footage's own and its rate the footage's, so it plays at the speed it + ;; was filmed in a project at any rate. + (fn [{:keys [db]} _] + (let [{:keys [id range fps width height name frame point target] :as request} + (get-in db [:ui :convert])] + (if (or (nil? request) (get-in db [:footage :loading?])) + {} + {:db (update db :ui dissoc :convert) + :dispatch [::ui/drop-tracing + {:name name :type :trace + :media {:footage id :range range} + :frames (- (second range) (first range)) :fps fps + :width width :height height :nodes {}} + frame point target]})))) + (rf/reg-event-fx ::converted ;; A TAKE IS A CLIP LIKE ANY OTHER, placed into whichever symbol the drop names diff --git a/frontend/src/arthur/events/playback.cljs b/frontend/src/arthur/events/playback.cljs index 7515339..59c78c6 100644 --- a/frontend/src/arthur/events/playback.cljs +++ b/frontend/src/arthur/events/playback.cljs @@ -13,7 +13,7 @@ (:require [arthur.audio.mix :as mix] [arthur.clock :as clock] [arthur.domain.clip :as clip] - [arthur.domain.trace :as trace] + [arthur.db :as app-db] [arthur.footage.store :as footage] [re-frame.core :as rf])) @@ -39,11 +39,10 @@ :clip (select-keys entry [:fps :width :height :audio])) (update :ui merge {:open sid :tabs (if sid [sid] []) - ;; From scratch, not merged: the faces switched on were another - ;; document's, and a face id means nothing in this one. - :trace {:faces (trace/showing-for (:clip entry) sid #{}) - :opacity (or (get-in db [:ui :trace :opacity]) - trace/opacity-default)}}) + ;; The layers switched off were another document's, and their + ;; ids mean nothing in this one. Whether tracing shows at all, + ;; and how strongly, is the person's and carries over. + :tracing (assoc (get-in db [:ui :tracing] app-db/tracing) :hidden #{})}) ;; Occurrence addresses belong to the document being left. Creation is ;; derived from the primary selection, so carrying one across documents ;; could otherwise make a coincidentally equal id a nested destination. @@ -212,7 +211,10 @@ ::open-symbol (fn [{:keys [db]} [_ sid]] (let [clip (:clip (footage/entry (:clip/current db)))] - (if (or (nil? (clip/symbol clip sid)) (= sid (get-in db [:ui :open]))) + ;; A tracing symbol has no inside to open: it is footage, which is seen by + ;; placing it. + (if (or (nil? (clip/symbol clip sid)) (= sid (get-in db [:ui :open])) + (clip/trace? (clip/symbol clip sid))) {} {:db (-> db (update-in [:ui :tabs] #(if (some #{sid} %) % (conj (vec %) sid))) @@ -224,7 +226,6 @@ (cond-> ui (= :node (first (:selection ui))) (dissoc :selection :selections)))) - (update-in [:ui :trace :faces] #(trace/showing-for clip sid %)) (assoc-in [:playback :frame] 0) (assoc-in [:playback :playing?] false)) ::pause! nil diff --git a/frontend/src/arthur/events/project.cljs b/frontend/src/arthur/events/project.cljs index c3aee48..34554a1 100644 --- a/frontend/src/arthur/events/project.cljs +++ b/frontend/src/arthur/events/project.cljs @@ -801,11 +801,28 @@ (let [st (:store (store/entry (:clip/current db)))] (edit/edit db #(update-in % [:symbols sid :nodes id] node/toggle-key path frame st))))) -;; A face's trace frames and origin, on its symbol — see `domain/trace`. +;; A hold on node `id`'s own frame `frame`, or none there if there was one: the +;; frames a tracing layer holds its picture on — a face's trace keys are its +;; plate's — and, for any node, `node/hold`. Taking the last one off removes the +;; floor rather than leaving an empty list behind. (rf/reg-event-db - ::set-trace - (fn [db [_ sid value]] - (edit/edit db #(assoc-in % [:symbols sid :nodes :head :trace] value)))) + ::toggle-hold + (fn [db [_ sid id frame]] + (edit/edit db #(update-in % [:symbols sid :nodes id :time] + (fn [t] + (let [hs (get t :holds []) + hs (vec (sort (if (some #{frame} hs) + (remove #{frame} hs) + (conj hs frame))))] + (if (seq hs) (assoc t :holds hs) (dissoc t :holds)))))))) + +;; How a face's head moves between its footage's holds — `symbol`'s `:reads`. +;; Nil reads every frame. +(rf/reg-event-db + ::set-reads + (fn [db [_ sid reads]] + (edit/edit db #(update-in % [:symbols sid :nodes :head] + (fn [n] (if reads (assoc n :reads reads) (dissoc n :reads))))))) (rf/reg-event-db diff --git a/frontend/src/arthur/events/ui.cljs b/frontend/src/arthur/events/ui.cljs index 0a8e829..a3cb9da 100644 --- a/frontend/src/arthur/events/ui.cljs +++ b/frontend/src/arthur/events/ui.cljs @@ -369,30 +369,28 @@ :else #{path}))))) (rf/reg-event-db - ::trace-face - ;; Showing the footage under a face is a viewing aid, like solo: editor state - ;; rather than the document, so it is not an undo step, does not travel to a - ;; collaborator and cannot reach an export. Per FACE and not per placement — a - ;; face is the same face wherever it is placed, and it is the face being traced - ;; — so one switch shows it in the take and in its own tab both. - (fn [db [_ face]] - (update-in db [:ui :trace :faces] - #(if (contains? % face) (disj % face) (conj (set %) face))))) + ::show-trace + ;; One tracing layer on or off — `layer` is `[symbol-id node-id]`, see + ;; `arthur.db/tracing`. Editor state, like solo: not an undo step, not sent to + ;; a collaborator, and it cannot reach an export. + ;; + ;; SWITCHING ONE ON SWITCHES TRACING ON. A layer just asked for must not stay + ;; invisible behind the global switch somebody forgot was off; switching one + ;; off leaves the global switch alone. One event, so the inspector, a timeline + ;; row and a face's section cannot disagree about it. + (fn [db [_ layer on?]] + (update-in db [:ui :tracing] + #(cond-> (update % :hidden (fnil (if on? disj conj) #{}) layer) + on? (assoc :on? true))))) (rf/reg-event-db - ::trace-faces - ;; Every face the open symbol has, from the inspector's footage section: - ;; switched on unless they all already are, which is the one gesture a person - ;; wants when there is exactly one face and when there are five. - (fn [db [_ faces]] - (let [faces (set faces) - on (set (get-in db [:ui :trace :faces]))] - (assoc-in db [:ui :trace :faces] - (if (every? on faces) (reduce disj on faces) (into on faces)))))) + ::tracing-on + ;; Every tracing layer at once, from the bar above the stage. + (fn [db [_ on?]] (assoc-in db [:ui :tracing :on?] (boolean on?)))) (rf/reg-event-db - ::trace-opacity - (fn [db [_ opacity]] (assoc-in db [:ui :trace :opacity] opacity))) + ::tracing-opacity + (fn [db [_ opacity]] (assoc-in db [:ui :tracing :opacity] opacity))) (rf/reg-event-db ::smart-picking @@ -769,6 +767,8 @@ (nest/inside document st from path frame)] (cond (nil? destination) {:refused "what you are dropping into is not on screen at this frame"} + (clip/trace? (clip/symbol document destination)) + {:refused "a tracing layer is a picture to draw over — nothing goes inside it"} (not (integer? at)) {:refused "the drop is not on one frame of that symbol"} :else {:clip document :sid destination :at at :path path :matrix matrix}))) @@ -888,6 +888,60 @@ {:extent :grow-symbol :point point :remainder-id (random-uuid)})))))) +(defn- fitted + "Tracing placement `uuid` in symbol `sid` of `document`, sized so its picture + is as tall as `sid`'s stage, and — dropped on the timeline, with no point to + land on — middled on that stage rather than wherever the picture's own middle + happens to fall. A 1920px still is otherwise six stages tall. + + A DEFAULT, set once, as the anchor is: nothing keeps it fitted afterwards." + [document sid uuid {:keys [height]} point?] + (let [[w h] (clip/stage document sid) + k (/ h height)] + (update-in document [:symbols sid :nodes uuid :channels] + (fn [chs] + (cond-> (assoc chs [:xform :scale] (ch/framed [k k])) + (not point?) + (assoc [:xform :pos] + (ch/framed (mapv - [(/ w 2) (/ h 2)] + (:value (get chs [:xform :anchor])))))))))) + +(rf/reg-event-db + ::drop-tracing + ;; Footage or a still to draw over, as a tracing symbol — see `clip/trace?` — + ;; placed by the same rule as any symbol dropped from the pool. One that shows + ;; the same media already in the document is reused rather than copied: it is + ;; the same picture, and a second symbol would only be a second name for it. + ;; + ;; A STILL ARRIVES WITH NO LENGTH, and is made as long as what is left of the + ;; symbol it lands in, at that symbol's rate: it shows the same picture on every + ;; frame, so its length is only how long it lasts, and trimming is how that + ;; changes afterwards. + (fn [db [_ sym frame point target]] + (let [{document :clip st :store} (store/entry (:clip/current db)) + where (if point + (creation-destination db document st frame) + (drop-destination db document st frame target))] + (if (:refused where) + (-> db (update :ui dissoc :drop) (update :project merge {:status (:refused where)})) + (let [{host :sid at :at document :clip} where + same (some (fn [[id s]] (when (= (:media s) (:media sym)) id)) (:symbols document)) + sid (or same (clip/free-id #(contains? (:symbols document) %) :tracing)) + sym (if same + (clip/symbol document same) + (cond-> (assoc sym :id sid) + (nil? (:frames sym)) (assoc :frames (max 1 (- (clip/frames document host) at)) + :fps (clip/fps document host)))) + uuid (random-uuid) + result (span/place-symbol (assoc-in document [:symbols sid] sym) st host + uuid sid at + {:extent :grow-symbol + :point (destination-point where point) + :remainder-id (random-uuid)})] + (landed db where uuid + (cond-> result + (:clip result) (update :clip fitted host uuid sym (some? point))))))))) + (rf/reg-event-db ::drop-sound ;; A SOUND IS A CLIP TOO. It is placed and then adopted rather than written diff --git a/frontend/src/arthur/flow/address.cljs b/frontend/src/arthur/flow/address.cljs index 1e6146e..4c01a8d 100644 --- a/frontend/src/arthur/flow/address.cljs +++ b/frontend/src/arthur/flow/address.cljs @@ -158,7 +158,13 @@ "head-pos" [:anchor-avg] "head-rot" [:anchor-avg] "head-scale" [:anchor-avg] - "eyes" [:anchor-avg :contour-avg :eye-verts :lash-weight] + ;; The head's fit itself, registering the footage under it — see + ;; `freeze/plate-part`. The image height it divides by is the footage's, which + ;; the analysis already names. + "plate-pos" [:anchor-avg] + "plate-rot" [:anchor-avg] + "plate-scale" [:anchor-avg] + "eyes" [:anchor-avg :contour-avg :eye-verts :lash-weight] "iris-pos" [:anchor-avg :contour-avg :gaze-gain :gaze-step] "brows" [:anchor-avg :contour-avg :brow-verts :brow-gain :brow-step :brow-weight] ;; `brow-pos` and not `contour-avg`: the ring is smoothed and the RAISE is not. diff --git a/frontend/src/arthur/flow/freeze.cljs b/frontend/src/arthur/flow/freeze.cljs index 91d057a..2f2641b 100644 --- a/frontend/src/arthur/flow/freeze.cljs +++ b/frontend/src/arthur/flow/freeze.cljs @@ -40,7 +40,6 @@ [arthur.domain.pick :as pick] [arthur.domain.palette :as pal] [arthur.domain.ring :as ring] - [arthur.domain.trace :as trace] [arthur.flow.address :as address])) ;; --------------------------------------------------------------------------- @@ -250,37 +249,57 @@ :tx (* (- s') (+ (* c tx) (* sn ty))) :ty (* (- s') (+ (* (- sn) tx) (* c ty)))})) -(defn head-mode - "Keep a subject's measured transform dense; optionally trace it. +(defn reads-for + "What a head with face `nodes` reads at, for trace keys `frames` and an + `origin` of `:continuous`, `:keys` or `:start` — see `domain/symbol`'s + `:reads`. At keys it holds where the face's footage holds, when it has footage + to follow; with none it holds at `frames` itself." + [nodes frames origin] + (case origin + :continuous nil + :start {:holds [0]} + :keys (if (contains? nodes :plate) {:holds-of :plate} {:holds (vec frames)}))) - With no trace the head reads measured frame f at frame f (free movement). - `{:frames [12] :origin :keys}` holds the measured transform of source frame - 12, `{:frames [12 42] :origin :keys}` jumps to 42's at 42, and `{:origin - :start}` holds frame 0's. See `domain/trace`. The trace selects position, - rotation and scale together, so the head and registered photo cannot drift - apart. No analysis block or authored face placement changes. +(defn head-mode + "Keep a subject's measured transform dense; optionally hold it at trace keys. + + `{:frames [12 42] :origin :keys}`: the face's footage holds on frames 12 and + 42 — its plate's `:time :holds` — and the head jumps to 12's measured + transform and then 42's, because it reads where the plate holds. `:start` + holds frame 0's forever, and `:continuous` reads every frame. The keys and the + origin are two facts with two owners: the plate's are which frames are drawn + over, and the head's is how it moves between them. No analysis block or + authored face placement changes. ONE SUBJECT AT A TIME when `:subject` is given, and EVERY subject when it is not. Two faces in one shot were filmed together and are posed apart: choosing - frame 12 for the second face must leave the first one running, and it does, - because a trace lives on that subject's own head node and `domain/symbol` - reads it off whatever node carries it." + frame 12 for the second face must leave the first one running." [{:keys [subject] t :trace} {:keys [clip]}] (when (and subject (not (contains? (:subjects clip) subject))) (throw (ex-info "head mode names a subject this clip did not track" {:subject subject :subjects (vec (sort-by str (keys (:subjects clip))))}))) - (let [t (some->> t (merge {:frames []}))] + (let [{:keys [frames origin] :as t} (merge {:frames []} (or t {:origin :continuous}))] (reduce (fn [c sid] - (let [frames (get-in c [:symbols sid :frames])] - (when-let [why (some-> t (trace/problems frames) first)] - (throw (ex-info (str "a head's trace is not one this take can hold: " why) - {:subject sid :trace t :frames frames}))) - (update-in c [:symbols sid :nodes :head] - (fn [n] - (cond-> (assoc n :channels (:measured n)) - t (assoc :trace t) - (nil? t) (dissoc :trace)))))) + (let [nodes (get-in c [:symbols sid :nodes]) + length (get-in c [:symbols sid :frames]) + why (cond + (not (#{:continuous :keys :start} origin)) + (str ":origin is " (pr-str origin) ", not :continuous, :keys or :start") + (not (and (vector? frames) (every? integer? frames) + (or (empty? frames) (apply < frames)) + (every? #(< -1 % length) frames))) + (str "trace keys must be increasing frames of the face: " (pr-str frames))) + _ (when why + (throw (ex-info (str "a head's trace is not one this take can hold: " why) + {:subject sid :trace t :frames length}))) + reads (reads-for nodes frames origin)] + (cond-> (update-in c [:symbols sid :nodes :head] + #(cond-> (assoc % :channels (:measured %)) + reads (assoc :reads reads) + (nil? reads) (dissoc :reads))) + (contains? nodes :plate) + (assoc-in [:symbols sid :nodes :plate :time :holds] (vec frames))))) clip (if subject [subject] (sort-by str (keys (:subjects clip))))))) ;; --------------------------------------------------------------------------- @@ -656,6 +675,53 @@ [:xform :scale] (dense scale 0 prov)} :store (stored pos rot scale)})) +(defn plate-part + "Freeze one subject's footage registration: the anchor fit itself, which is the + inverse of the head's, with the image's pixels taken into image heights. + + This is what puts a face's footage UNDER ITS HEAD and keeps it there. The plate + is a child of `:head`, so its world is `head · fit · 1/H`; on a frame where the + head and the plate read the same measured frame, `head · fit` is the identity + and the footage sits exactly where the face was filmed. On any other frame it + rides the head — held photo on a moving head, or live footage under a head + held at frame 0 — registered either way, because the fit and the photo are read + on the same held frame. Nothing in the evaluator knows it is a face. + + `fit · S(1/H)` is still a similarity — the scale divides by H and nothing else + moves — so it lands on the same three decomposed channels as the head's." + [subject {:keys [analysis anchor-avg footage] :as params} + {:keys [transforms detected presence]}] + (let [h (:height footage) + absent? (when (or detected presence) + (fn [_ f] (and detected (not (nth detected f true))))) + xf (fn [role f] + (block {:role role :analysis (:id analysis) :params params + :tracks [role]} + {:type "float32" :features [subject] :absent? absent?} + {:detected detected} + [(mapv f transforms)])) + pos (xf "plate-pos" (fn [t] [(:tx t) (:ty t)])) + rot (xf "plate-rot" (fn [t] [(:theta t)])) + scale (xf "plate-scale" (fn [t] [(/ (:s t) h) (/ (:s t) h)])) + prov {:by :anchor/similarity :analysis (:id analysis) + :params {:anchor-avg anchor-avg}}] + {:measured {[:xform :pos] (dense pos 0 prov) + [:xform :rot] (dense rot 0 prov) + [:xform :scale] (dense scale 0 prov)} + :store (stored pos rot scale)})) + +(defn footage-symbol + "The tracing symbol for `:footage` — `{:id :range :width :height}`, the + footage measured and which of its frames — or nil when the measurement came + from no footage: a synthetic take has nothing to show. See `clip/trace?`." + [{:keys [analysis fps footage]}] + (when-let [{:keys [id range width height]} footage] + {:id :footage :name (or (:source analysis) "footage") + :type :trace + :media {:footage id :range range} + :frames (- (second range) (first range)) :fps fps :width width :height height + :nodes {}})) + ;; --------------------------------------------------------------------------- ;; the clip @@ -677,6 +743,9 @@ areas (cond-> [:mouth] (and eyes brows) (into [:eye :brow]) teeth (conj :teeth)) parts (mapv #(part subject % params inputs) areas) head (head-part subject params inputs) + ;; THE FACE'S FOOTAGE IS A PLACEMENT LIKE ANY OTHER, of the footage's + ;; tracing symbol, under the head — see `plate-part`. + plate (when (:footage params) (plate-part subject params inputs)) features (cond-> {:mouth [:mouth [:mouth :mouth-in]]} (and eyes brows) (merge {:eye-r [:eye [:eye-r :eye-r-in :iris-r :pupil-r]] @@ -684,9 +753,14 @@ :brow-r [:brow [:brow-r]] :brow-l [:brow [:brow-l]]}) teeth (assoc :teeth [:teeth [:teeth]]))] {:symbol {:id subject :frames (count outer) - :nodes (into {:head {:id :head :name "head" :kind :group :z "a1" - :measured (:measured head)}} - (mapcat :nodes) parts)} + :nodes (cond-> (into {:head {:id :head :name "head" :kind :group :z "a1" + :measured (:measured head)}} + (mapcat :nodes) parts) + plate (assoc :plate {:id :plate :name "footage" :kind :instance + :parent :head :z "a0" + :source {:symbol :footage} + :measured (:measured plate) + :channels (:measured plate)}))} :features (into {} (map (fn [[role [area nodes]]] [(own role) {:id (own role) :subject subject :symbol subject :area area @@ -695,7 +769,7 @@ {(own :eyes) {:id (own :eyes) :kind :eye-pair :subject subject :members [(own :eye-r) (own :eye-l)] :params {}}} {}) - :store (into (:store head) (mapcat :store) parts)})) + :store (into (merge (:store head) (:store plate)) (mapcat :store) parts)})) (defn pivoted "Every node a freeze makes that a hand can transform, pivoting about the @@ -751,8 +825,8 @@ [{:keys [name fps stage expose] :as params} subjects] (when-not (and (map? subjects) (seq subjects) (every? keyword? (keys subjects)) - (not-any? #{:main :root :place} (keys subjects))) - (throw (ex-info "a freeze needs subjects with ids distinct from :main, :root and :place" {}))) + (not-any? #{:main :root :place :footage} (keys subjects))) + (throw (ex-info "a freeze needs subjects with ids distinct from :main, :root, :place and :footage" {}))) (let [ordered (sort-by (comp str key) subjects) parts (mapv (fn [[id inputs]] [id (subject-part params id inputs)]) ordered) placement (face-placement params subjects) @@ -783,9 +857,12 @@ :z (str "a" i) :source {:symbol id}}])) ordered)}} - (map (fn [[id part]] - [id (assoc (place-in (:symbol part) placement) :fps fps)])) - parts)}] + (concat + (map (fn [[id part]] + [id (assoc (place-in (:symbol part) placement) :fps fps)]) + parts) + (when-let [footage (footage-symbol params)] + [[:footage footage]])))}] (doseq [[subject inputs] ordered [id track] (:presence inputs)] (when-not (and (= nf (count track)) diff --git a/frontend/src/arthur/flow/regenerate.cljs b/frontend/src/arthur/flow/regenerate.cljs index 7148df0..9a33497 100644 --- a/frontend/src/arthur/flow/regenerate.cljs +++ b/frontend/src/arthur/flow/regenerate.cljs @@ -2,7 +2,9 @@ "Recompute a changed feature from retained source tracks, then replace only channels owned by that feature. Upload remains project/save's ordinary job." (:require [arthur.domain.channel :as ch] + [arthur.domain.clip :as clip] [arthur.domain.feature :as feature] + [arthur.domain.node :as node] [arthur.domain.params :as params] [arthur.flow.address :as address] [arthur.flow.freeze :as freeze] @@ -125,17 +127,28 @@ still ARE the measurement, and are left alone once somebody has placed the head by hand." [entry params base subject] - (let [baked (freeze/head-part subject params @base) - at [:clip :symbols subject :nodes :head] - old (get-in entry at) - measured (:measured baked)] - (cond-> (-> entry - (assoc-in (conj at :measured) measured) - (update :store merge (:store baked))) - (= (bases (:channels old)) (bases (:measured old))) - (assoc-in (conj at :channels) - (into {} (map (fn [[p c]] [p (rebased (get-in old [:channels p]) c)])) - measured))))) + ;; The face's footage is registered by the same fit, inverted, so it is + ;; re-frozen with the head whenever the face has one — see `freeze/plate-part`. + (let [plate (get-in entry [:clip :symbols subject :nodes :plate]) + footage (some->> plate node/source (clip/symbol (:clip entry))) + params (cond-> params + plate (assoc :footage {:height (:height footage)}))] + (reduce + (fn [entry [id bake]] + (let [baked (bake subject params @base) + at [:clip :symbols subject :nodes id] + old (get-in entry at) + measured (:measured baked)] + (cond-> (-> entry + (assoc-in (conj at :measured) measured) + (update :store merge (:store baked))) + (= (bases (:channels old)) (bases (:measured old))) + (assoc-in (conj at :channels) + (into {} (map (fn [[p c]] [p (rebased (get-in old [:channels p]) c)])) + measured))))) + entry + (cond-> [[:head freeze/head-part]] + plate (conj [:plate freeze/plate-part]))))) (defn change "One scoped static edit. `source-inputs` holds dense landmarks and, when the diff --git a/frontend/src/arthur/flow/take.cljs b/frontend/src/arthur/flow/take.cljs index fc65899..5e0bd03 100644 --- a/frontend/src/arthur/flow/take.cljs +++ b/frontend/src/arthur/flow/take.cljs @@ -134,5 +134,12 @@ ;; `flow/address`: a model upgrade that silently reused ;; these landmarks is the failure this prevents. :analysis (analysis-for (assoc manifest :width w :height h) - detector)})] + detector) + ;; What the faces are traced over: these frames of this + ;; footage, at the pixel size its stills are. See + ;; `freeze/footage-symbol`. + :footage {:id (:id manifest) + :range (or (:range manifest) [0 (:frames manifest)]) + :width (or (:width manifest) w) + :height (or (:height manifest) h)}})] (build params subjects))) diff --git a/frontend/src/arthur/subs/render.cljs b/frontend/src/arthur/subs/render.cljs index 6ea94bc..9366fc3 100644 --- a/frontend/src/arthur/subs/render.cljs +++ b/frontend/src/arthur/subs/render.cljs @@ -13,7 +13,6 @@ [arthur.domain.node :as node] [arthur.domain.palette :as pal] [arthur.domain.symbol :as symbol] - [arthur.domain.trace :as trace] [arthur.footage.store :as footage] [arthur.subs.playback :as playback] [re-frame.core :as rf])) @@ -25,7 +24,7 @@ (rf/reg-sub ::sliding (fn [db _] (get-in db [:ui :sliding]))) (rf/reg-sub ::gesture (fn [db _] (get-in db [:ui :gesture]))) (rf/reg-sub ::solo (fn [db _] (get-in db [:ui :solo (get-in db [:ui :open])]))) -(rf/reg-sub ::tracing (fn [db _] (get-in db [:ui :trace]))) +(rf/reg-sub ::tracing (fn [db _] (get-in db [:ui :tracing]))) ;; The faces with smart frame picking on, as a PREVIEW switch. The setting ;; docs/frame-selection.md specifies is the document's and is not built; this is ;; the editor's own, so it moves what the stage shows and not what exports. @@ -170,7 +169,9 @@ :<- [::smart] (fn [[document sid store palette smart] _] (when (and document (clip/symbol document sid)) - (clip/resolver document sid store palette {:snap smart})))) + ;; The only resolver that asks for tracing layers: the stage shows them and + ;; nothing else may. See `clip/trace-op`. + (clip/resolver document sid store palette {:snap smart :tracing? true})))) (rf/reg-sub ::shown @@ -204,25 +205,3 @@ clip/IActivePalette (active-palette [_] (clip/active-palette resolve))))))) -(rf/reg-sub - ::underlay - :<- [::clip-id] - :<- [::clip] - :<- [::store] - :<- [::open] - :<- [::solo] - :<- [::tracing] - (fn [[id document store open solo {:keys [faces opacity]}] _] - ;; What `ui/underlay` needs to paint the footage of the faces being traced, - ;; besides the resolver that says where they went. A face outside every soloed - ;; row is not on stage, so neither is its footage. - (let [solo (filter #(placed? document open %) solo)] - {:document document :store store - :opacity (or opacity trace/opacity-default) - :traces (cond->> (when document (trace/shown document open faces)) - (seq solo) (filterv (fn [{:keys [path]}] - ;; The open symbol's own face is at the - ;; empty path: it is not under any row, so - ;; soloing a row cannot hide it. - (or (empty? path) - (some #(= % (take (count %) path)) solo)))))}))) diff --git a/frontend/src/arthur/subs/ui.cljs b/frontend/src/arthur/subs/ui.cljs index 8fe9623..4e6399f 100644 --- a/frontend/src/arthur/subs/ui.cljs +++ b/frontend/src/arthur/subs/ui.cljs @@ -68,6 +68,18 @@ (let [{clip :clip st :store} (store/entry clip-id)] (nest/inside clip st open (or path [id]) f))))) +(rf/reg-sub + ::own-time + :<- [::render/clip-id] + :<- [::render/open] + :<- [::render/open-frame] + (fn [[clip-id open f] [_ path]] + ;; Row `path`'s own clock from the open symbol — `nest/own-time` — and the + ;; frame of it the playhead is over. + (let [{clip :clip st :store} (store/entry clip-id)] + (when-let [t (and clip (seq path) (nest/own-time clip st open (vec path) f))] + {:time t :frame (js/Math.floor (* (:rate t) (- f (:at t))))})))) + (rf/reg-sub ::creation-target :<- [::selection] @@ -172,7 +184,10 @@ [_ n] (:nodes sym) :let [f (get-in n [:source :footage])] :when f] - f))] + f)) + ;; And what a tracing layer shows. + used (into used (keep #(get-in % [:media :footage])) + (vals (get-in entry [:clip :symbols])))] (filterv #(contains? used (:id %)) available)))) (rf/reg-sub diff --git a/frontend/src/arthur/ui/convert.cljs b/frontend/src/arthur/ui/convert.cljs index 1a12ee5..6cc6f2a 100644 --- a/frontend/src/arthur/ui/convert.cljs +++ b/frontend/src/arthur/ui/convert.cljs @@ -85,6 +85,10 @@ [:div.convert-actions [:button {:disabled loading? :on-click #(rf/dispatch [::footage/convert-cancel])} "cancel"] + [:button {:disabled (or loading? (< n 1) (empty? name)) + :title "draw over these frames · nothing is detected, and it is never exported" + :on-click #(rf/dispatch [::footage/convert-tracing])} + "tracing layer"] [:button.on {:disabled (or loading? (< n 1) (empty? name)) :on-click #(rf/dispatch [::footage/convert])} (if loading? "working…" "make symbol")]]]])))) diff --git a/frontend/src/arthur/ui/drag.cljs b/frontend/src/arthur/ui/drag.cljs index 1b50f0f..2f12131 100644 --- a/frontend/src/arthur/ui/drag.cljs +++ b/frontend/src/arthur/ui/drag.cljs @@ -52,7 +52,7 @@ out of the pool, and not one that would make a cycle." [] (let [c @carrying] - (and c (#{:symbol :footage :import :sound} (:kind c)) (not (:refused? c))))) + (and c (#{:symbol :footage :import :sound :tracing} (:kind c)) (not (:refused? c))))) (defn row! "Start carrying the timeline row at `path` — a node of kind `node-kind`, to be @@ -118,5 +118,7 @@ :import (rf/dispatch [::project/import c frame point target]) ;; Where it is dropped in time; a sound has no place in space. :sound (rf/dispatch [::ui/drop-sound c frame target]) + ;; A still to trace over, made into a tracing symbol where it lands. + :tracing (rf/dispatch [::ui/drop-tracing (:symbol c) frame point target]) nil)) (done!))) diff --git a/frontend/src/arthur/ui/palette.cljs b/frontend/src/arthur/ui/palette.cljs index 6559d5b..19d1c9e 100644 --- a/frontend/src/arthur/ui/palette.cljs +++ b/frontend/src/arthur/ui/palette.cljs @@ -71,6 +71,19 @@ (when (> (count selections) 1) [:span.dim (str (count selections) " selected")]) [:span {:style {:flex 1}}] + ;; Every tracing layer at once: the reference on or off, and how strongly + ;; it draws. A property of looking at the stage, like the zoom beside it. + (let [{:keys [on? opacity]} @(rf/subscribe [::render/tracing])] + [:<> + [:button {:class (when on? "on") + :title "show tracing layers over the picture · never exported" + :on-click #(rf/dispatch [::ui/tracing-on (not on?)])} + "tracing"] + [:input.trace-opacity + {:type "range" :min 0 :max 1 :step 0.05 :value opacity :disabled (not on?) + :title "how strongly tracing layers draw" + :style {:flex "0 0 64px"} + :on-change #(rf/dispatch [::ui/tracing-opacity (js/parseFloat (.. % -target -value))])}]]) (if (= :polygon tool) [:<> [:span.dim (str (quot (count draft) 2) " points")] diff --git a/frontend/src/arthur/ui/params.cljs b/frontend/src/arthur/ui/params.cljs index 60f5140..0c348ce 100644 --- a/frontend/src/arthur/ui/params.cljs +++ b/frontend/src/arthur/ui/params.cljs @@ -15,7 +15,6 @@ [arthur.domain.params :as params] [arthur.domain.pose :as pose] [arthur.domain.symbol :as symbol] - [arthur.domain.trace :as trace] [arthur.events.history :as history] [arthur.events.paint :as paint-events] [arthur.events.playback :as pb] @@ -468,91 +467,71 @@ (str/join " " (map name channel)) " · " why)]))])]))))) ;; --------------------------------------------------------------------------- -;; the footage showing under the picture +;; tracing layers +;; +;; A tracing layer is a placement of a tracing symbol, and a face's footage is +;; one of them — its `:plate`, under its head — so one section serves both: +;; whether it is showing, and which of its frames it holds on. Showing is the +;; editor's (`arthur.db/tracing`); the holds are the document's, on the +;; placement's own `:time`. A face adds one fact of its own below them: whether +;; its head follows those holds. -(defn- footage-section - "The footage under the explicitly selected face or footage asset, on or off - and how strongly. `here` is exactly those faces. +(defn- layer-section + "Tracing layer `id` of symbol `sid`, at row `path` from the open symbol, under + `title`, with `more` appended." + [title sid id path & more] + (let [n (get-in @(rf/subscribe [::render/clip]) [:symbols sid :nodes id]) + {:keys [on? hidden]} @(rf/subscribe [::render/tracing]) + shown? (and on? (not (contains? hidden [sid id]))) + {:keys [frame time]} @(rf/subscribe [::sub/own-time path]) + holds (get-in n [:time :holds] []) + seek! #(rf/dispatch [::pb/seek (js/Math.round (+ (:at time) (/ % (:rate time))))])] + (into + [section title + [:div.row + [:label.dim {:title "show it over the picture · a reference, never exported"} + [:input {:type "checkbox" :checked (boolean shown?) + :on-change #(rf/dispatch [::ui/show-trace [sid id] (not shown?)])}] + " show"]] + [:div.row {:style {:margin "5px 0"}} + ;; A DEAD BUTTON WITH NO REASON GIVEN is what reads as the feature not + ;; working, so say why there is nothing to hold. + (if (nil? frame) + [:span.dim "move the playhead over this layer to hold a frame"] + [:button {:title "hold the picture on this frame until the next hold" + :on-click #(rf/dispatch [::project/toggle-hold sid id frame])} + (if (some #{frame} holds) (str "remove hold at " frame) (str "hold at " frame))])] + (when (seq holds) + [:div.row + [:span.dim "holds"] + (doall + (for [f holds] + ^{:key f} + [:button {:class (when (= frame f) "on") + :disabled (nil? time) + :on-click #(seek! f)} + (str f)]))])] + more))) - ONE SWITCH FOR THE FACES THAT ARE HERE. A face's footage is the face's, not a - placement's, so there is nothing to inherit and nothing to set twice; with - several faces in a take the box says how many are showing and switches the rest - on, and one face alone is switched from its own timeline row. +(def ^:private origins + "How a face's head moves between its footage's holds, as the head's `:reads`." + [["continuous" nil "the head reads the frame it is on"] + ["at holds" {:holds-of :plate} "the head jumps to each hold and stays there"] + ["start" {:holds [0]} "the head holds frame 0 forever"]]) - This is editor state, but the inspector still obeys selection scope: selecting - an unrelated shape must neither expose nor mutate a face's viewing aid." - [here] - (let [{:keys [faces opacity]} @(rf/subscribe [::render/tracing]) - on (filterv (set faces) here)] - [section "footage" - [:div.row - [:label.dim {:title (str "show the footage these faces were traced from, over the " - "picture \u00b7 a reference, never exported")} - [:input {:type "checkbox" :checked (= (count on) (count here)) - :on-change #(rf/dispatch [::ui/trace-faces here])}] - (str " show" (when (< 1 (count here)) (str " " (count on) "/" (count here))))]] +(defn- face-section + "Face `face`'s footage, at row `path` from the open symbol: its plate as a + tracing layer, and how its head follows the plate's holds." + [face path] + (let [reads (get-in @(rf/subscribe [::render/clip]) [:symbols face :nodes :head :reads])] + [layer-section (str "tracing · " (name face)) face :plate (conj path :plate) [:div.row {:style {:margin-top "5px"}} - [:span.dim "opacity"] - [:input.trace-opacity - {:type "range" :min 0 :max 1 :step 0.05 :title "how strongly the footage draws" - :value (or opacity trace/opacity-default) :disabled (empty? on) - :on-change #(rf/dispatch [::ui/trace-opacity (js/parseFloat (.. % -target -value))])}]]])) - -;; --------------------------------------------------------------------------- -;; tracing a face -;; -;; THE FACE'S OWN FACTS ONLY, and both of them are keyed to the face rather than -;; to an instance of it: which of its frames its drawings were made over, and -;; what its origin does between those frames. See `domain/trace`. -;; -;; `performance-section` below it is the face's too, for a third reason of the -;; same kind: the closure cuts the preserve-snap reads are stored on the face's -;; own nodes, so every placement of one face has the same marks, and a -;; per-placement switch would be a setting with nothing in it. -;; -;; Whether the footage is SHOWING is `footage-section` above, and not part of this -;; section. It is a viewing aid rather than a fact about a face, and it must not -;; come and go with the selection — so it is keyed to the OPEN symbol's faces, -;; which is a different question from the one this section answers. - -(defn- trace-keys - "The face's trace keys and origin. `frame` is the frame of the FACE that the - playhead is over, and `seek!` goes to one of its frames — both of which depend - on whether the face is open in its own tab or placed in what is." - [face frame seek!] - (let [t (trace/of (get-in @(rf/subscribe [::render/clip]) [:symbols face :nodes :head])) - put #(rf/dispatch [::project/set-trace face %]) - key? (boolean (some #{frame} (:frames t)))] - [:<> - [:div.row {:style {:margin "5px 0"}} - ;; A DEAD BUTTON WITH NO REASON GIVEN is what reads as the feature not - ;; working. There is no frame of this face under the playhead, so say that - ;; rather than greying out the one control in the section. - (if (nil? frame) - [:span.dim "move the playhead over this face to key it"] - [:button {:on-click #(put (trace/toggle-frame t frame))} - (if key? (str "remove trace key at " frame) (str "trace key at " frame))])] - (when (seq (:frames t)) - [:div.row - [:span.dim "keys"] - (doall - (for [f (:frames t)] - ^{:key f} - [:button {:class (when (= frame f) "on") - :disabled (nil? seek!) - :on-click #(seek! f)} - (str f)]))]) - [:div.row {:style {:margin-top "5px"}} - [:span.dim "origin"] + [:span.dim "head"] (doall - (for [[o label] (map vector trace/origins ["continuous" "at keys" "start"])] - ^{:key o} - [:button {:class (when (= o (:origin t)) "on") - :title (case o - :continuous "the head reads the frame it is on" - :keys "the head jumps to each trace key and holds it" - :start "the head holds frame 0 forever") - :on-click #(put (assoc t :origin o))} + (for [[label r title] origins] + ^{:key label} + [:button {:class (when (= r reads) "on") :title title + :on-click #(rf/dispatch [::project/set-reads face r])} label]))]])) (defn- performance-section @@ -601,32 +580,6 @@ [:div.row [:span.dim "the stage only \u00b7 an export is unaffected until this is a saved setting"]]])) -(defn- tracing-section - "`face` is the face these facts belong to, `faces` the faces placed inside it - to offer as somewhere to go next, and `path` the row path `faces` are under." - [face faces path] - (let [clip @(rf/subscribe [::render/clip]) - own? (= face @(rf/subscribe [::render/open])) - ;; The face's OWN frame, which is the playhead itself when the face is - ;; the open symbol, and the selected placement's local frame when it is - ;; placed in it. `time` maps the open symbol's frames to that - ;; placement's, so seeking to one of the face's frames is a conversion. - {:keys [frame time]} (when-not own? @(rf/subscribe [::sub/selected-local])) - frame (if own? @(rf/subscribe [::playback/frame]) frame) - seek! (cond own? #(rf/dispatch [::pb/seek %]) - time #(rf/dispatch [::pb/seek (js/Math.round - (+ (:at time) (/ % (:rate time))))]))] - [section (str "tracing · " (name face)) - (when (trace/traceable? clip face) [trace-keys face frame seek!]) - (when (seq faces) - [:div.row {:style {:margin-top "5px"}} - [:span.dim "faces"] - (doall - (for [{p :path in :in f :face} faces] - ^{:key (str p)} - [:button {:on-click #(rf/dispatch [::ui/select [:node in (peek p) (into path p)]])} - (name f)]))])])) - ;; --------------------------------------------------------------------------- ;; a symbol @@ -701,6 +654,11 @@ :group (get-in clip [:groups id :subject]) nil)) +(defn- face? + "Is symbol `sid` a face: does it have a measured head?" + [clip sid] + (boolean (seq (get-in clip [:symbols sid :nodes :head :measured])))) + (defn- face-selection "The one face explicitly named by a face symbol, its placement, or a tracking owner. A shape merely living inside an open face is deliberately not one." @@ -711,7 +669,7 @@ (= :symbol kind) id (#{:subject :feature :group} kind) (subject-of-owner clip selection) (= :node kind) placed)] - (when (and candidate (trace/traceable? clip candidate)) candidate))) + (when (and candidate (face? clip candidate)) candidate))) (defn- footage-faces [clip selection selected-node] (if (= :footage (first selection)) @@ -812,12 +770,11 @@ (get-in clip [:symbols (first node) :type]))) palette-symbol? (and (= :symbol (first selection)) (= :palette (get-in clip [:symbols (second selection) :type]))) - face (or placed (when (trace/traceable? clip open) open)) - faces (when face (trace/faces clip face)) - ;; Where that face sits, as a row path from the open symbol, so the faces - ;; inside it can be selected by their own rows. A selection made on the - ;; stage has no path and names a node directly in the open symbol; the - ;; open symbol itself is at no path at all. + face (or placed (when (face? clip open) open)) + layer? (and placed (clip-domain/trace? (clip-domain/symbol clip placed))) + ;; Where that face or layer sits, as a row path from the open symbol. A + ;; selection made on the stage has no path and names a node directly in + ;; the open symbol; the open symbol itself is at no path at all. path (if placed (or (nth selection 3 nil) [(second node)]) [])] [:section.pane.params [:div.pane-head "inspector"] @@ -829,11 +786,12 @@ (when (and placed (not palette-placement?)) [symbol-section placed "source symbol"]) (when (and node (not palette-placement?)) ^{:key (str (first node) "/" (second node))} [correction-section node]) - (when (and (not palette-placement?) (seq footage-faces)) - [footage-section footage-faces]) - (when (and face (or (trace/traceable? clip face) (seq faces))) - [tracing-section face faces path]) - (when face ^{:key (str "perf/" face)} [performance-section face]) + (when layer? + [layer-section (str "tracing · " (clip-domain/symbol-name clip placed)) + (first node) (second node) path]) + (when (get-in clip [:symbols face :nodes :plate]) + [face-section face path]) + (when (and face (not layer?)) ^{:key (str "perf/" face)} [performance-section face]) (when palette-symbol? [palette-symbol-section (second selection)]) (when (and (= :symbol (first selection)) (not palette-symbol?)) [symbol-section (second selection)]) diff --git a/frontend/src/arthur/ui/player.cljs b/frontend/src/arthur/ui/player.cljs index 480e3f9..d8c587a 100644 --- a/frontend/src/arthur/ui/player.cljs +++ b/frontend/src/arthur/ui/player.cljs @@ -27,7 +27,7 @@ [arthur.subs.playback :as sub] [arthur.subs.render :as render] [arthur.ui.canvas :as canvas] - [arthur.ui.underlay :as underlay] + [arthur.ui.tracing :as tracing] [re-frame.core :as rf] [reagent.ratom :as ratom])) @@ -103,9 +103,9 @@ (some-> @tracker ratom/dispose!) (reset! tracker (ratom/run! - (let [{was :resolver was-u :underlay} @snapshot + (let [{was :resolver was-t :tracing} @snapshot now @(rf/subscribe [::render/shown]) - u @(rf/subscribe [::render/underlay])] + t @(rf/subscribe [::render/tracing])] (reset! snapshot {:resolver now :palette @(rf/subscribe [::render/palette]) @@ -114,22 +114,17 @@ :frames @(rf/subscribe [::render/frames]) :width @(rf/subscribe [::sub/width]) :height @(rf/subscribe [::sub/height]) - :underlay u + :tracing t :frame @(rf/subscribe [::sub/frame]) :playing? @(rf/subscribe [::sub/playing?])}) ;; A new resolver means a new scene or a new palette, and neither ;; moves the playhead — so nothing else would ask for a redraw. ;; - ;; Switching a face's footage on wants the same redraw for the same + ;; Switching tracing on or off wants the same redraw for the same ;; reason, and it needs asking for SEPARATELY: it is a viewing aid ;; in the editor's own state, so it changes what the canvases should - ;; show without touching the frame number OR the resolver. The - ;; document and the store are left out of the comparison because - ;; they are what a new resolver already means. - ;; Comparing the whole map is as cheap as picking fields out of it: - ;; the document and the store it carries are the same OBJECTS unless - ;; the resolver changed too, and that is tested first. - (when-not (and (identical? was now) (= was-u u)) + ;; show without touching the frame number OR the resolver. + (when-not (and (identical? was now) (= was-t t)) (repaint!)))))) (defn set-canvas! [el] @@ -163,7 +158,7 @@ rasterised before the next frame is asked for." [f] (let [{:keys [canvas]} @state - {:keys [resolver palette ramp width height underlay playing?]} @snapshot] + {:keys [resolver palette ramp width height tracing playing?]} @snapshot] (when (and canvas resolver width height) ;; User Timing, so a profile in the DevTools performance panel has named ;; spans in the Timings track instead of a wall of anonymous frames. Three @@ -173,17 +168,20 @@ ;; independent of the footage, so the size the frame is rasterised at comes ;; out of the document like everything else. (let [ras (raster-for width height) - ops (resolver f) + {traces true picture false} (group-by #(= :trace (:kind %)) (resolver f)) + ;; A layer switched off, or all of them, is not on the stage at all: + ;; not painted and not there to be clicked. + traces (when (:on? tracing) + (into [] (remove #(contains? (:hidden tracing) (:layer %))) traces)) active (clip/active-palette resolver) bg (pal/background-index palette active)] - (swap! state assoc :ops ops) + (swap! state assoc :ops (into (vec picture) traces)) (-> ras (raster/clear! bg) - (raster/draw-ops! ops)) + (raster/draw-ops! picture)) (js/performance.mark "arthur/blit:start") (canvas/blit! canvas ras (pal/effective-ramp palette active)) - (underlay/paint! (assoc underlay :width width :playing? playing?) - resolver repaint!)) + (tracing/paint! traces (assoc tracing :width width :playing? playing?) repaint!)) (js/performance.measure "arthur/resolve+draw" "arthur/paint:start" "arthur/blit:start") (js/performance.measure "arthur/paint" "arthur/paint:start") ;; User Timing entries otherwise accumulate forever in the browser's diff --git a/frontend/src/arthur/ui/pool.cljs b/frontend/src/arthur/ui/pool.cljs index 5c93ba0..68e9def 100644 --- a/frontend/src/arthur/ui/pool.cljs +++ b/frontend/src/arthur/ui/pool.cljs @@ -56,6 +56,7 @@ [arthur.subs.ui :as sub] [arthur.ui.canvas :as canvas] [arthur.ui.drag :as drag] + [arthur.ui.tracing :as tracing] [clojure.string :as str] [re-frame.core :as rf] [reagent.core :as r])) @@ -141,6 +142,13 @@ :style {:width 32 :height 20 :max-width 32 :max-height 20}}] [:span.thumb])) +(defn- tracing-thumb + "The first source frame, using the stage's media URL cache." + [media] + (r/with-let [ready (r/atom 0)] + @ready + [picture (tracing/url-of media 0 #(swap! ready inc))])) + ;; --------------------------------------------------------------------------- ;; the row @@ -254,8 +262,12 @@ :title (str label " · " (:frames sym) " frames · " nodes (if (= 1 nodes) " node" " nodes") (when (= sid open) " · open") - " — double-click to open, drag to place") - :thumb [picture (symbol-thumb document sid store palette ramp)] + (if (= :trace (:type sym)) + " — drag to place" + " — double-click to open, drag to place")) + :thumb (if (= :trace (:type sym)) + [tracing-thumb (:media sym)] + [picture (symbol-thumb document sid store palette ramp)]) :sub2 (str (:frames sym) "f · " nodes (if (= 1 nodes) " node" " nodes")) :on? (= selection [:symbol sid]) :open? (= sid open) @@ -287,7 +299,7 @@ (rf/dispatch [::project/rename-symbol sid value]))) :on-click #(rf/dispatch [::ui/select [:symbol sid]])}])) -(defn- footage-row [{:keys [id label frames fps video] :as f} chosen rename] +(defn- footage-row [{:keys [id label frames fps video width height] :as f} chosen rename] ^{:key id} [row (merge {:label label :sub (str frames "f") @@ -304,7 +316,8 @@ :on-click #(rf/dispatch [::footage/choose id])} (carrying (str "footage:" id) #(drag/other! {:kind :footage :id id :label label - :frames frames :fps fps :video video})))]) + :frames frames :fps fps :video video + :width width :height height})))]) (defn- sound-row "An uploaded sound, or with `:footage?` a video's own — which is how a take's @@ -335,6 +348,30 @@ #(drag/other! {:kind :sound :source source :label label :length length :rate rate :frames frames})))])) +(defn- image-row + "An uploaded still, to be traced over: dropped, it becomes a tracing layer — + see `events/ui ::drop-tracing`." + [{:keys [id label width height digest url]} rename] + ^{:key id} + [row (merge {:label label + :sub (str width "×" height) + :title (str label " · " width "×" height + " — drag onto the stage or the timeline to trace over it") + :thumb [:img.thumb {:src url :alt "" + :style {:width 32 :height 20 :object-fit "cover"}}] + :rename (assoc rename + :key [:image id] + :value label + :commit! (fn [value] + ((:begin! rename) nil) + (rf/dispatch [::footage/relabel :image id value])))} + (carrying (str "image:" id) + #(drag/other! {:kind :tracing :label label :frames 1 + :center [(/ width 2) (/ height 2)] + :symbol {:name label :type :trace + :media {:image digest} + :width width :height height :nodes {}}})))]) + (defn- import-row [{:keys [cid symbol name frames]} pid] ^{:key (str cid symbol)} [row (merge {:label name @@ -400,7 +437,7 @@ Still not `:main` being special. The document says which symbol that is by its structure; rename it, place it inside something else, and the pool follows." - [{:keys [document query searching? media sounds chosen rename main palette-choice selection] :as ctx}] + [{:keys [document query searching? media sounds images chosen rename main palette-choice selection] :as ctx}] (let [named? #(hit? query (clip/symbol-name document %)) top (when (and main (named? main)) main) symbol-ids (sort-by str (keys (:symbols document))) @@ -417,10 +454,10 @@ symbol-ids) media (filterv #(hit? query (:label %)) media) sounds (filterv #(hit? query (:label %)) sounds) - ] + images (filterv #(hit? query (:label %)) images)] [sections searching? (+ (if top 1 0) (count rest) (count transitions) - (count media) (count sounds)) + (count media) (count sounds) (count images)) [{:title "project" :searching? searching? :blank "nothing to open yet" :rows (when top [(symbol-row document top ctx)])} @@ -435,28 +472,35 @@ :rows (mapv #(footage-row % chosen rename) media)} {:title "sounds" :searching? searching? :blank "drop an mp3 or wav here" - :rows (mapv #(sound-row % (:fps document) rename) sounds)}]])) + :rows (mapv #(sound-row % (:fps document) rename) sounds)} + {:title "images" :searching? searching? + :blank "drop a png or jpeg here to trace over" + :rows (mapv #(image-row % rename) images)}]])) (defn- all-assets "Everything the server holds. Other projects' symbols stay grouped by project and closed: a server holds many, and a wall of every symbol in every one buries the one you want." - [{:keys [document query searching? rename chosen available all-sounds symbols palettes - project-id]}] + [{:keys [document query searching? rename chosen available all-sounds all-images symbols + palettes project-id]}] (let [media (filterv #(hit? query (:label %)) available) sounds (filterv #(hit? query (:label %)) all-sounds) + images (filterv #(hit? query (:label %)) all-images) others (filterv #(and (not= project-id (:project %)) (hit? query (:name %))) symbols) grouped (sort-by (comp str second key) (group-by (juxt :project :project-name) others)) palettes (filterv #(and (not= project-id (:project %)) (hit? query (:name %))) palettes)] [sections searching? - (+ (count media) (count sounds) (count others) (count palettes)) + (+ (count media) (count sounds) (count images) (count others) (count palettes)) [{:title "media" :searching? searching? :blank "nothing uploaded yet" :rows (mapv #(footage-row % chosen rename) media)} {:title "sounds" :searching? searching? :blank "no sounds uploaded yet" :rows (mapv #(sound-row % (:fps document) rename) sounds)} + {:title "images" :searching? searching? + :blank "no images uploaded yet" + :rows (mapv #(image-row % rename) images)} {:title "palettes" :searching? searching? :blank "no palettes in other saved projects" :rows (mapv import-palette-row palettes)} @@ -481,14 +525,17 @@ one cost of collapsing two folders into two tabs — that a hit could be behind the tab you did not pick — is paid off by a number, counted over the same labels the rows are filtered by." - [{:keys [document query media sounds available all-sounds symbols palettes project-id]}] + [{:keys [document query media sounds images available all-sounds all-images symbols + palettes project-id]}] (let [n (fn [labels] (count (filter #(hit? query %) labels)))] {:project (+ (n (map #(clip/symbol-name document %) (keys (:symbols document)))) (n (map :name (vals (pal/palettes document)))) (n (map :label media)) - (n (map :label sounds))) + (n (map :label sounds)) + (n (map :label images))) :assets (+ (n (map :label available)) (n (map :label all-sounds)) + (n (map :label all-images)) (n (map :name (remove #(= project-id (:project %)) palettes))) (n (map :name (remove #(= project-id (:project %)) symbols))))})) @@ -504,7 +551,8 @@ ;; than a flag per row: exactly one name is ever being edited, and ;; opening a second input has to close the first. editing (r/atom nil)] - (let [{:keys [loading? status available sounds chosen]} @(rf/subscribe [::playback/footage]) + (let [{:keys [loading? status available sounds images uploaded chosen]} + @(rf/subscribe [::playback/footage]) document @(rf/subscribe [::sub/settled-clip]) clip-id @(rf/subscribe [::render/clip-id]) store @(rf/subscribe [::render/store]) @@ -522,12 +570,19 @@ :duration (/ frames fps)}) media) @(rf/subscribe [::sub/project-sounds])) + ;; The stills this document traces over, and those uploaded while it + ;; was open — as `::sub/project-sounds` decides for sounds. + traced (into #{} (keep #(get-in % [:media :image])) (vals (:symbols document))) + own-images (filterv #(or (contains? traced (:digest %)) + (contains? (set uploaded) (:id %))) + images) needle (str/lower-case (str/trim @query)) searching? (boolean (seq needle)) ctx {:document document :clip-id clip-id :store store :palette palette :ramp ramp :selection selection :open open - :media media :sounds own-sounds :chosen chosen + :media media :sounds own-sounds :images own-images :chosen chosen :available (vec available) :all-sounds (vec sounds) + :all-images (vec images) :symbols symbols :palettes palettes :project-id project-id :palette-choice palette-choice :query needle :searching? searching? @@ -553,11 +608,11 @@ [:div.pane-head "media pool" [:span.spacer] - [:button {:title "add a video or a sound" + [:button {:title "add a video, a sound or an image" :disabled loading? :on-click #(.click (js/document.getElementById "pool-file"))} "+"]] - [:input {:id "pool-file" :type "file" :accept "video/*,audio/*" + [:input {:id "pool-file" :type "file" :accept "video/*,audio/*,image/*" :style {:display "none"} :on-change (fn [^js event] (when-let [file (aget (.. event -target -files) 0)] diff --git a/frontend/src/arthur/ui/stage.cljs b/frontend/src/arthur/ui/stage.cljs index bf13415..0a121b0 100644 --- a/frontend/src/arthur/ui/stage.cljs +++ b/frontend/src/arthur/ui/stage.cljs @@ -25,7 +25,7 @@ [arthur.ui.drag :as drag] [arthur.ui.layout :as layout] [arthur.ui.player :as player] - [arthur.ui.underlay :as underlay] + [arthur.ui.tracing :as tracing] [re-frame.core :as rf] [reagent.core :as r])) @@ -501,8 +501,8 @@ :width w :height h :style {:width (str (* zoom w) "px") :height (str (* zoom h) "px")}}] - [:canvas.underlay {:ref #(underlay/set-canvas! %) - :width (* zoom w) :height (* zoom h)}] + [:canvas.tracing {:ref #(tracing/set-canvas! %) + :width (* zoom w) :height (* zoom h)}] [overlay w h zoom]] (when destination-name [:div.stage-target "creating in " [:strong destination-name]])])) diff --git a/frontend/src/arthur/ui/timeline.cljs b/frontend/src/arthur/ui/timeline.cljs index 544e76b..95285ea 100644 --- a/frontend/src/arthur/ui/timeline.cljs +++ b/frontend/src/arthur/ui/timeline.cljs @@ -26,7 +26,6 @@ [arthur.domain.nest :as nest] [arthur.domain.span :as span] [arthur.domain.symbol :as symbol] - [arthur.domain.trace :as trace] [arthur.events.playback :as pb] [arthur.events.ui :as ui] [arthur.footage.store :as store] @@ -720,17 +719,22 @@ {:title "rename lane" :on-click (fn [^js e] (.stopPropagation e) (begin-rename!))} "✎"]) - ;; A face's row is where its own footage is switched on, next to solo - ;; because the two are the same kind of thing: what this row shows, here, - ;; now, and nothing the picture keeps. The inspector's footage section does - ;; all of them at once; this is how one face out of a take is singled out. - (when (contains? (:faces tracing) of) - [:button {:class (str "tl-trace" (when (contains? (:on tracing) of) " on")) - :title "show the footage this face was traced from" - :on-click (fn [^js e] - (.stopPropagation e) - (rf/dispatch [::ui/trace-face of]))} - "T"]) + ;; A tracing layer's row switches it on and off, next to solo because the + ;; two are the same kind of thing: what this row shows, here, now, and + ;; nothing the picture keeps. A face's row switches its footage — its + ;; plate — which is the same switch wherever the face is placed. + (when-let [layer (cond (contains? (:traces tracing) of) [(second select) (nth select 2)] + (contains? (:plated tracing) of) [of :plate])] + (let [{:keys [on? hidden]} (:state tracing) + shown? (and on? (not (contains? hidden layer)))] + [:button {:class (str "tl-trace" (when shown? " on")) + :title (if (contains? (:traces tracing) of) + "show this tracing layer · never exported" + "show the footage this face was traced from") + :on-click (fn [^js e] + (.stopPropagation e) + (rf/dispatch [::ui/show-trace layer (not shown?)]))} + "T"])) (when (= :instance node-kind) [:button {:class (str "tl-solo" (when (contains? solo path) " on")) :title "show only this on the stage (⇧ for more than one)" @@ -965,6 +969,7 @@ ;; mapping to edit through, so the bar selects and does not slide. [:div {:class (str "tl-span" (when dense? " dense") (when (= :ghost kind) " ghost") (when (= :audio node-kind) " sound") + (when (= :trace (get-in clip [:symbols of :type])) " trace") (when unmapped? " unmapped") (when (and select (not unmapped?)) " movable") (when (and select (contains? selected-set select)) " on") @@ -1015,6 +1020,9 @@ ;; unexplained. It wears the pick colour rather than the accent every ;; span and drop already uses — see `--pick` in app.css. :class (str (when ghost? "ghost") + ;; A reference, not the picture: drawn so it cannot be + ;; mistaken for a drawing that will be exported. + (when (= :trace (get-in clip [:symbols source :type])) " trace") (when (and select (contains? selected-set select)) " on") (when (and select (= select selection)) " primary") (when (= (nth select 3 nil) target-path) " target") @@ -1135,11 +1143,14 @@ drop @(rf/subscribe [::sub/drop]) solo (set @(rf/subscribe [::render/solo])) open @(rf/subscribe [::render/open]) - ;; Which rows offer a footage switch, and which of them are switched - ;; on. A set rather than a lookup per row: the answer is the same for - ;; every placement of a face, because the switch is the face's. - tracing {:faces (set (trace/traceable-faces clip open)) - :on (set (:faces @(rf/subscribe [::render/tracing])))} + ;; Which rows offer a tracing switch — placements of a tracing symbol, + ;; and faces that have footage — and what is switched on. Sets rather + ;; than a lookup per row. + tracing {:traces (into #{} (comp (filter #(= :trace (:type (val %)))) (map key)) + (:symbols clip)) + :plated (into #{} (comp (filter #(get-in (val %) [:nodes :plate])) (map key)) + (:symbols clip)) + :state @(rf/subscribe [::render/tracing])} ;; WHERE THE DROP WILL LAND IS WHERE THE PREVIEW GOES, and what is being ;; carried does not come into it. A row under the pointer takes the ;; preview as a block in that row -- the same row `ui/drop-destination` diff --git a/frontend/src/arthur/ui/tracing.cljs b/frontend/src/arthur/ui/tracing.cljs new file mode 100644 index 0000000..710fd7a --- /dev/null +++ b/frontend/src/arthur/ui/tracing.cljs @@ -0,0 +1,154 @@ +(ns arthur.ui.tracing + "Tracing layers — footage or a still to draw over — on their own canvas above + the stage. + + A REFERENCE, NOT OUTPUT. A tracing symbol resolves to a `:trace` op, which the + raster refuses and only the stage's resolver asks for (see `clip/trace-op`), so + it cannot reach an export. It is drawn OVER the picture at one opacity rather + than under it, because the raster clears to an opaque ground. The canvas is the + stage's size on screen, not the raster's, so a 1280px still is not squeezed + through a 320px stage on its way to being seen. + + The op says where the image goes and which of its frames to show, and nothing + else: a face's footage is registered to the face because it is placed inside + it, under the head, so there is nothing about faces in here. + + THREE THINGS KEEP IT STEADY, because a still is decoded asynchronously and the + frame it belongs to is already on screen by the time it arrives: the cache drops + the stills used longest ago rather than emptying itself, a layer keeps showing + the still it last showed until its next one has decoded, and the stills a little + way ahead of the playhead are asked for before they are needed. Each is a + separate cause of the same symptom — the tracing blinking out for a frame or two + — and none of them covers the others: the hold is what survives a seek, the + reading ahead is what keeps playback from being a frame behind for good." + (:require [arthur.flow.ingest :as ingest])) + +(defonce ^:private state (atom {:canvas nil :urls {} :images {} :order [] :last {}})) + +(defn set-canvas! [el] (swap! state assoc :canvas el)) + +(defn- footage-urls + "The footage's still URLs, or nil until its manifest has come back." + [footage-id on-ready] + (let [u (get-in @state [:urls footage-id])] + (when (nil? u) + (swap! state assoc-in [:urls footage-id] :loading) + (-> (ingest/manifest! footage-id) + (.then (fn [m] + (swap! state assoc-in [:urls footage-id] (:urls m)) + (on-ready))) + (.catch (fn [error] + (swap! state assoc-in [:urls footage-id] :failed) + (js/console.error "arthur: no stills for footage" footage-id error))))) + (when (vector? u) u))) + +(defn url-of + "The URL of the still showing frame `frame` of `media`, or nil until it is + known. A footage symbol's frame 0 is the first frame of its range." + [{:keys [footage image range]} frame on-ready] + (cond + image (str "/blob/" image) + footage (some-> (footage-urls footage on-ready) (get (+ (first range) frame))))) + +(def ^:private ^:const kept + "Decoded stills held at once. A 1280px still is about 5MB decoded, and + scrubbing footage with no holds asks for every frame of it." + 48) + +(def ^:private ^:const ahead + "Frames past the playhead whose stills are asked for while the current one is + drawn. A still cannot be decoded in the animation frame that wants it, so + playing footage would otherwise always be a frame or two behind itself." + 8) + +(defn- decoded + "`img` if the still has arrived and can be drawn." + [^js img] + (when (and (.-complete img) (pos? (.-naturalHeight img))) img)) + +(defn- touch! + "Hold `img` under `url` as the most recently used still, dropping the ones used + longest ago once there are more than `kept`. + + LEAST RECENTLY USED, touched when it arrives and again on every frame it is + DRAWN, which is what makes it safe without a list of exceptions: the still on + screen is the newest thing in the cache and cannot be the one dropped — a still + held for a hundred frames included. Read-and-not-drawn does not touch, because + a still read only to be warmed was just inserted anyway." + [url img] + (swap! state + (fn [{:keys [images order] :as s}] + (let [images (assoc images url img) + order (conj (into [] (remove #{url}) order) url) + over (- (count order) kept)] + (if (pos? over) + (assoc s :images (apply dissoc images (subvec order 0 over)) + :order (subvec order over)) + (assoc s :images images :order order)))))) + +(defn- image + "The still at `url`, or nil until it has loaded. Asks for it the first time it + is wanted; `on-ready` paints again once it is there. + + A still that fails is remembered as having failed, so a broken URL is one + console line and one request rather than one of each per animation frame." + [url on-ready] + (when url + (let [held (get-in @state [:images url])] + (cond + (= :failed held) nil + held (decoded held) + :else (let [img (js/Image.)] + (set! (.-onload img) on-ready) + (set! (.-onerror img) + (fn [_] + (swap! state assoc-in [:images url] :failed) + (js/console.error "arthur: a tracing still did not load" url))) + (set! (.-src img) url) + (touch! url img) + nil))))) + +(defn- warm! + "Ask for the stills the next `ahead` frames of footage will want. + + WHILE PLAYING ONLY, and only for a layer whose frame MOVED since it was last + painted. A scrub asks for a different `ahead` frames at every step, and a held + layer shows one still for as long as it is held, so reading ahead through + either would be requests for stills nobody is going to look at." + [{:keys [media frame]} on-ready] + (when (:footage media) + (doseq [p (range (inc frame) (+ frame 1 ahead))] + (image (url-of media p on-ready) on-ready)))) + +(defn paint! + "Draw trace ops `ops`, in draw order, at `opacity`, over a stage `width` wide. + `on-ready` is called when a still or a manifest that was missing arrives, to + paint again." + [ops {:keys [opacity width playing?]} on-ready] + (when-let [^js canvas (:canvas @state)] + (let [ctx (.getContext canvas "2d") + zoom (/ (.-width canvas) width) + was (:last @state)] + (.setTransform ctx 1 0 0 1 0 0) + (.clearRect ctx 0 0 (.-width canvas) (.-height canvas)) + ;; What each layer is showing, forgotten for the layers that are gone. + ;; Before the draw, so switching them all off forgets all of them. + (swap! state update :last select-keys (map :node ops)) + (set! (.-globalAlpha ctx) opacity) + (doseq [{:keys [node frame m] [w h] :size :as op} ops] + ;; THE STILL IT HAS, not nothing. A frame whose still is still decoding + ;; keeps the one before it rather than blinking off. + (let [want (url-of (:media op) frame on-ready) + [img url] (or (when-let [i (image want on-ready)] [i want]) + (let [url (get-in was [node :url])] + (when-let [i (image url on-ready)] [i url])))] + (when img + (touch! url img) + (swap! state assoc-in [:last node] {:url url :frame frame}) + (.setTransform ctx + (* zoom (aget m 0)) (* zoom (aget m 1)) + (* zoom (aget m 2)) (* zoom (aget m 3)) + (* zoom (aget m 4)) (* zoom (aget m 5))) + (.drawImage ctx img 0 0 w h)) + (when (and playing? (not= frame (get-in was [node :frame]))) + (warm! op on-ready))))))) diff --git a/frontend/src/arthur/ui/underlay.cljs b/frontend/src/arthur/ui/underlay.cljs deleted file mode 100644 index 7daf387..0000000 --- a/frontend/src/arthur/ui/underlay.cljs +++ /dev/null @@ -1,182 +0,0 @@ -(ns arthur.ui.underlay - "The footage a face is traced over, on its own canvas above the stage. - - A REFERENCE, NOT OUTPUT. The tracing still never enters the indexed raster, so - it cannot reach an export, and it is drawn OVER the picture at the face's - opacity rather than under it, because the raster clears to an opaque ground. - The canvas is the stage's size on screen, not the raster's, so a 1280px still - is not squeezed through a 320px stage on its way to being seen. - - Painted by `ui/player` straight after each frame, from the same snapshot, so - it moves with the face it registers to — see `domain/trace/photo-matrix`. - - THREE THINGS KEEP IT STEADY, because a still is decoded asynchronously and the - frame it belongs to is already on screen by the time it arrives: the cache drops - the stills used longest ago rather than emptying itself, a face keeps showing - the still it last showed until its next one has decoded, and the stills a little - way ahead of the playhead are asked for before they are needed. Each is a - separate cause of the same symptom — the tracing blinking out for a frame or two - — and none of them covers the others: the hold is what survives a seek, the - reading ahead is what keeps playback from being a frame behind for good." - (:require [arthur.domain.symbol :as symbol] - [arthur.domain.trace :as trace] - [arthur.flow.ingest :as ingest])) - -(defonce ^:private state (atom {:canvas nil :urls {} :images {} :order [] :last {}})) - -(defn set-canvas! [el] (swap! state assoc :canvas el)) - -(defn- urls - "The footage's still URLs, or nil until its manifest has come back." - [footage-id on-ready] - (let [u (get-in @state [:urls footage-id])] - (when (nil? u) - (swap! state assoc-in [:urls footage-id] :loading) - (-> (ingest/manifest! footage-id) - (.then (fn [m] - (swap! state assoc-in [:urls footage-id] (:urls m)) - (on-ready))) - (.catch (fn [error] - (swap! state assoc-in [:urls footage-id] :failed) - (js/console.error "arthur: no stills for footage" footage-id error))))) - (when (vector? u) u))) - -(def ^:private ^:const kept - "Decoded stills held at once. A 1280px still is about 5MB decoded, and - scrubbing a face with no trace keys asks for every frame of the take." - 48) - -(def ^:private ^:const ahead - "Frames past the playhead whose stills are asked for while the current one is - drawn. A still cannot be decoded in the animation frame that wants it, so - playing a continuously traced face would otherwise always be a frame or two - behind its own footage." - 8) - -(defn- decoded - "`img` if the still has arrived and can be drawn." - [^js img] - (when (and (.-complete img) (pos? (.-naturalHeight img))) img)) - -(defn- touch! - "Hold `img` under `url` as the most recently used still, dropping the ones used - longest ago once there are more than `kept`. - - LEAST RECENTLY USED, touched when it arrives and again on every frame it is - DRAWN, which is what makes it safe without a list of exceptions: the still on - screen is the newest thing in the cache and cannot be the one dropped — a still - held at a trace key for a hundred frames included. Read-and-not-drawn does not - touch, because a still read only to be warmed was just inserted anyway, and - reordering the whole list ten times per face per frame in the draw loop is the - kind of allocation `ui/player` exists to avoid. What this replaces emptied the - cache on the frame it filled, so every still on screen had to be fetched and - decoded again: a blink every `kept` frames of a scrub, and with two faces and a - full cache a permanent one." - [url img] - (swap! state - (fn [{:keys [images order] :as s}] - (let [images (assoc images url img) - order (conj (into [] (remove #{url}) order) url) - over (- (count order) kept)] - (if (pos? over) - (assoc s :images (apply dissoc images (subvec order 0 over)) - :order (subvec order over)) - (assoc s :images images :order order)))))) - -(defn- image - "The still at `url`, or nil until it has loaded. Asks for it the first time it - is wanted; `on-ready` paints again once it is there. - - A still that fails is remembered as having failed, so a broken URL is one - console line and one request rather than one of each per animation frame." - [url on-ready] - (when url - (let [held (get-in @state [:images url])] - (cond - (= :failed held) nil - held (decoded held) - :else (let [img (js/Image.)] - (set! (.-onload img) on-ready) - (set! (.-onerror img) - (fn [_] - (swap! state assoc-in [:images url] :failed) - (js/console.error "arthur: a tracing still did not load" url))) - (set! (.-src img) url) - (touch! url img) - nil))))) - -(defn- still - "The URL of the still showing the face's own frame `p`. The manifest is the - whole footage's and `p` is an index into the analysed range, so the range's - start is where the face's frame 0 was filmed." - [us start p] - (get us (+ start p))) - -(defn- warm! - "Ask for the stills the next `ahead` frames will want. Held trace frames - collapse to the one still, so a face traced at keys asks for almost nothing. - - WHILE PLAYING ONLY, because that is the only time the next frame is the one - after this one. A scrub asks for a different `ahead` frames at every step, so - reading ahead through one would be `ahead` requests per step for stills the - pointer has already gone past — and a cache thrashed by its own guesses." - [us start t lf on-ready] - (doseq [p (distinct (map #(trace/photo-frame t %) (range (inc lf) (+ lf 1 ahead))))] - (image (still us start p) on-ready))) - -(defn paint! - "Draw every switched-on face's footage on the frame `resolver` last resolved. It - says where each face's head went and which of its frames it was on, so this - reads the frame rather than resolving it again. `on-ready` is called when a - still or a manifest that was missing arrives, to paint again." - [{:keys [document store traces opacity width playing?]} resolver on-ready] - (when-let [^js canvas (:canvas @state)] - (let [ctx (.getContext canvas "2d") - zoom (/ (.-width canvas) width)] - (.setTransform ctx 1 0 0 1 0 0) - (.clearRect ctx 0 0 (.-width canvas) (.-height canvas)) - ;; What each face is showing, forgotten for the faces switched off. Before - ;; the early exits, so switching them all off forgets all of them. - (swap! state update :last select-keys (map :path traces)) - (when (seq traces) - (let [;; Read once: each face writes only its own entry below, and what it - ;; was showing is what it falls back to. - was (:last @state)] - ;; ONE OPACITY for all of them, set once: how strongly the reference - ;; draws is a property of looking at the stage, not of a face. - (set! (.-globalAlpha ctx) opacity) - (doseq [{:keys [path face]} traces - :let [subject (get-in document [:subjects face]) - analysis (get-in document [:analyses (:analysis subject)]) - us (urls (:footage subject) on-ready)] - :when us] - (let [start (first (get analysis :range [0])) - at (conj path :head) - world (symbol/world-of resolver at) - frame (symbol/frame-of resolver at)] - (when (and world (number? frame)) - (let [head (get-in document [:symbols face :nodes :head]) - t (trace/of head) - lf (js/Math.floor frame) - want (trace/photo-frame t lf) - ;; THE STILL IT HAS, not nothing. A frame whose still is - ;; still decoding keeps the one before it — a reference a - ;; frame stale, registered where that frame's face was, - ;; rather than a face with its footage blinking off. - [img url p] (or (let [u (still us start want)] - (when-let [i (image u on-ready)] [i u want])) - (let [{:keys [url p]} (get was path)] - (when-let [i (image url on-ready)] [i url p]))) - m (when img - (trace/photo-matrix world head store p (.-naturalHeight img)))] - (when m - ;; Drawn, so it is the newest still in the cache and what - ;; this face falls back to while its next one decodes. - (touch! url img) - (swap! state assoc-in [:last path] {:url url :p p}) - (.setTransform ctx - (* zoom (aget m 0)) (* zoom (aget m 1)) - (* zoom (aget m 2)) (* zoom (aget m 3)) - (* zoom (aget m 4)) (* zoom (aget m 5))) - (.drawImage ctx img 0 0)) - (when playing? (warm! us start t lf on-ready))))))))))) diff --git a/frontend/test/arthur/domain/node_test.cljs b/frontend/test/arthur/domain/node_test.cljs index 73d96dc..6245ef5 100644 --- a/frontend/test/arthur/domain/node_test.cljs +++ b/frontend/test/arthur/domain/node_test.cljs @@ -116,6 +116,20 @@ (is (= [0 0 0 3 3 3] (mapv #(node/expose % 3) (range 6)))) (is (= 4 (node/expose 5 2)) "frame 5 at exposure 2 reads frame 4, not 6")) +(deftest a-hold-floors-onto-authored-frames + ;; Exposure on frames somebody chose rather than on a grid: the last hold at or + ;; before the frame, and before the first one the first. + (is (= [0 1 2 3] (mapv #(node/hold % []) (range 4))) "no holds is no floor") + (is (= [12 12 12 12 30 30] (mapv #(node/hold % [12 30]) [0 11 12 29 30 99]))) + (is (= [0 0 12 12 30] (mapv #(node/hold % [0 12 30]) [0 11 12 29 30]))) + (testing "after exposure, before the lead" + (is (= 13 (node/local-frame {:time {:holds [4 12] :expose 2 :offset 1}} 13)) + "13 exposes to 12, holds at 12, and the lead adds 1")) + (testing "refused unless increasing" + (is (seq (node/problems {:id :n :kind :group :z "a" :time {:holds [3 1]}}))) + (is (seq (node/problems {:id :n :kind :group :z "a" :time {:holds '(1 3)}}))) + (is (empty? (node/problems {:id :n :kind :group :z "a" :time {:holds [1 3]}}))))) + (deftest exposure-comes-before-offset-and-the-order-is-visible ;; THE INVARIANT: flooring onto a grid and shifting against the clock do not ;; commute. Shift first and the floor discards it on most frames, so the lead diff --git a/frontend/test/arthur/domain/symbol_test.cljs b/frontend/test/arthur/domain/symbol_test.cljs index 80fb8d6..740821c 100644 --- a/frontend/test/arthur/domain/symbol_test.cljs +++ b/frontend/test/arthur/domain/symbol_test.cljs @@ -167,6 +167,20 @@ x-at #(first (first (pts-of (first (symbol/eval-frame s % nil pal/index-of nil)))))] (is (= [0 0 0 0 4 4 4 4 8 8 8 8] (mapv x-at (range 12))))))) +(deftest holds-are-inherited-like-exposure + (let [walk (ch/keyed (into {} (map (fn [f] [f [f 0]])) (range 12)) :hold) + s (assoc (sc {:id :g :kind :group :z "a1" :time {:holds [2 7]} + :channels {[:xform :pos] walk}} + (poly :p :g "a1" [0 0 1 0 1 1] :skin-base)) + :frames 12) + x-at #(first (first (pts-of (first (symbol/eval-frame s % nil pal/index-of nil)))))] + (is (= [2 2 2 2 2 2 2 7 7 7 7 7] (mapv x-at (range 12))) + "the group reads its held frame, before the first hold included") + (testing "and the playback path agrees" + (let [spec (ops/specified s nil) fast (ops/resolved s nil)] + (doseq [[label fs] (ops/orders 12) f fs] + (is (= (spec f) (fast f)) (str label " at frame " f))))))) + (deftest offset-is-per-node-which-is-the-entire-point-of-mouth-lead ;; Lead applies to performance nodes and NOT to the plate. If it were a clip ;; property the mouth would drag the whole head forward with it. diff --git a/frontend/test/arthur/domain/trace_test.cljs b/frontend/test/arthur/domain/trace_test.cljs deleted file mode 100644 index e0ccbc9..0000000 --- a/frontend/test/arthur/domain/trace_test.cljs +++ /dev/null @@ -1,139 +0,0 @@ -(ns arthur.domain.trace-test - (:require [cljs.test :refer [deftest is]] - [arthur.demo.take :as take] - [arthur.domain.clip :as clip] - [arthur.domain.leaf :as leaf] - [arthur.domain.nest :as nest] - [arthur.domain.node :as node] - [arthur.domain.palette :as pal] - [arthur.domain.symbol :as symbol] - [arthur.domain.trace :as trace] - [arthur.flow.freeze :as freeze])) - -(def ^:private frozen (delay (freeze/head-mode {} @take/frozen))) -(def ^:private store (delay (:store @take/frozen))) - -(defn- traced - "The synthetic take with face-1's trace set to `t`." - [t] - (assoc-in @frozen [:symbols :face-1 :nodes :head :trace] t)) - -(defn- head [c] (get-in c [:symbols :face-1 :nodes :head])) - -(defn- near? [a b] - (every? #(< (js/Math.abs %) 1e-9) (map - a b))) - -(deftest the-origin-picks-the-measured-frame-a-head-reads - (let [at (fn [t fs] (map #(trace/held-frame (trace/prepare t) %) fs))] - (is (nil? (trace/prepare {:frames [4 9] :origin :continuous}))) - (is (nil? (trace/prepare nil))) - (is (= [0 0 0] (at {:frames [4 9] :origin :start} [0 5 50]))) - (is (= [4 4 4 4 9 9] (at {:frames [4 9] :origin :keys} [0 3 4 8 9 50])) - "a jump at each key, and before the first the first") - (is (= [0 0] (at {:frames [] :origin :keys} [0 30]))))) - -(deftest every-origin-is-a-valid-document-that-saves - (doseq [origin trace/origins - :let [c (traced {:frames [0 12 40] :origin origin})]] - (is (empty? (clip/problems c)) (str origin ": " (pr-str (clip/problems c)))) - (is (= c (leaf/clip "t" (leaf/leaves "t" c))) (str origin " round-trips")))) - -(deftest a-trace-the-take-cannot-hold-will-not-save - (doseq [t [{:frames [0] :origin :sideways} {:frames [9 3] :origin :keys} - {:frames [9999] :origin :keys}]] - (is (seq (clip/problems (traced t))) (pr-str t)))) - -(deftest the-photo-holds-each-trace-frame-until-the-next - (let [t {:frames [5 20]}] - (is (= [5 5 5 5 20 20] (map #(trace/photo-frame t %) [0 4 5 19 20 90])))) - (is (= 33 (trace/photo-frame {:frames []} 33)) "no keys: every frame is its own") - (is (= [3 7] (:frames (trace/toggle-frame {:frames [7]} 3)))) - (is (= [] (:frames (trace/toggle-frame {:frames [7]} 7))))) - -(defn- photo-at - "The photo matrix of face-1 alone at frame `f`, the still being 1000px tall." - [c f] - (let [r (symbol/resolver (clip/symbol c :face-1) @store pal/index-of nil) - h (head c)] - (r f) - (vec (array-seq (trace/photo-matrix (symbol/world-of r :head) h @store - (trace/photo-frame (trace/of h) f) 1000))))) - -(defn- filmed - "Where a photo sitting exactly where it was filmed must land: the face's OWN - placement, over image height, and nothing else. - - `:place` carries the source-to-stage mapping now, above `:head` — so a face - opened in its own tab draws at the size it is on the stage, and its photo has - to come with it. That is the whole reason the mapping belongs to the face: the - head still cancels out, which is what these assertions are about, but it - cancels against a placement rather than against nothing." - [c f] - (let [r (symbol/resolver (clip/symbol c :face-1) @store pal/index-of nil)] - (r f) - (vec (array-seq (node/mul! (node/mat) (symbol/world-of r :place) - (js/Float64Array. #js [0.001 0 0 0.001 0 0])))))) - -(deftest the-photo-registers-to-the-face-it-was-filmed-with - (let [a (traced {:frames [] :origin :continuous}) - b (traced {:frames [12] :origin :keys}) - d (traced {:frames [12] :origin :continuous})] - (is (near? (filmed a 30) (photo-at a 30)) - "a head showing the photo's own frame cancels out") - (is (near? (filmed b 30) (photo-at b 30)) - "held at the trace frame, face and photo both stand still") - (is (not (near? (filmed d 30) (photo-at d 30))) - "a continuous head carries the held photo along with it"))) - -(defn- wrapped - "Face-1's take placed, moved, inside a symbol :wrap." - [] - (assoc-in @frozen [:symbols :wrap] - {:id :wrap :frames 200 - :nodes {:m {:id :m :kind :instance :z "a0" - :source {:symbol :main} - :channels {[:xform :pos] {:animated? false :value [30 -10]}}}}})) - -(deftest a-face-switched-on-shows-wherever-it-is-placed - (let [c (wrapped)] - (is (= [{:path [:m :face-1] :in :main :face :face-1}] (trace/shown c :wrap #{:face-1})) - "a face inside a take inside a symbol, at the row path it is at") - (is (= [] (trace/shown c :wrap #{})) "and nothing when it is switched off") - (is (= [{:path [:face-1] :in :main :face :face-1}] (trace/shown c :main #{:face-1})) - "the same switch, one symbol down") - (is (= [{:path [] :in :face-1 :face :face-1}] (trace/shown c :face-1 #{:face-1})) - "the face open in its own tab is at no path at all — it IS the stage") - (is (= [] (trace/shown c :wrap #{:main})) - "a symbol that is not a face has no footage of its own to show"))) - -(deftest the-faces-that-can-be-traced-are-listed-once-each - (is (= [:face-1] (trace/traceable-faces (wrapped) :wrap))) - (is (= [:face-1] (trace/traceable-faces @frozen :main)) "the take it was frozen into") - (is (= [:face-1] (trace/traceable-faces @frozen :face-1)) "itself, open to draw over")) - -(deftest a-face-opened-to-be-drawn-over-starts-with-its-footage-showing - (let [c (wrapped)] - (is (= #{:face-1} (trace/showing-for c :face-1 #{})) "the face's own tab") - (is (= #{} (trace/showing-for c :main #{})) - "and not the take it is placed in, which is the picture itself") - (is (= #{:face-1} (trace/showing-for c :main #{:face-1})) - "one already switched on stays on wherever you go"))) - -(deftest a-take-lists-the-faces-in-it - (is (= [{:path [:face-1] :in :main :face :face-1}] (trace/faces @frozen :main))) - (is (= [{:path [:m :face-1] :in :main :face :face-1}] (trace/faces (wrapped) :wrap))) - (is (= [] (trace/faces @frozen :face-1)))) - -(deftest the-resolver-says-where-a-nested-head-went-on-its-last-frame - ;; The same answer as `nest/placement`, which walks and resolves the path all - ;; over again — the resolver has it already, from drawing the frame. - (let [c (wrapped) - r (clip/resolver c :wrap @store pal/index-of nil) - path [:m :face-1 :head]] - (doseq [f [0 17 60]] - (r f) - (let [pl (nest/placement c @store :wrap path f)] - (is (near? (array-seq (:world pl)) (array-seq (symbol/world-of r path))) (str f)) - (is (= (:frame pl) (js/Math.floor (symbol/frame-of r path))) (str f)))) - (r 500) - (is (nil? (symbol/world-of r path)) "not on the frame, not anywhere"))) diff --git a/frontend/test/arthur/domain/tracing_test.cljs b/frontend/test/arthur/domain/tracing_test.cljs new file mode 100644 index 0000000..ce36076 --- /dev/null +++ b/frontend/test/arthur/domain/tracing_test.cljs @@ -0,0 +1,220 @@ +(ns arthur.domain.tracing-test + "Tracing symbols: footage or a still to draw over, placed like any symbol and + never part of the picture — and a face's footage as one of them, under its head." + (:require [cljs.test :refer [deftest is testing]] + [arthur.demo.take :as take] + [arthur.domain.clip :as clip] + [arthur.domain.creation :as creation] + [arthur.domain.leaf :as leaf] + [arthur.domain.nest :as nest] + [arthur.domain.node :as node] + [arthur.domain.palette :as pal] + [arthur.domain.pick :as pick] + [arthur.domain.symbol :as symbol] + [arthur.events.ui :as ui] + [arthur.footage.store :as store] + [arthur.flow.freeze :as freeze] + [re-frame.core :as rf] + [re-frame.db :as rf-db])) + +(def ^:private image-h 1000) + +(def ^:private frozen + "The synthetic take frozen as though it had come from footage 1000px tall." + (delay (freeze/clip (assoc take/params :footage {:id "f00" :range [0 take/frames] + :width 1500 :height image-h}) + {:face-1 @take/measured}))) + +(defn- traced + "The take with face-1's trace keys and origin set." + [t] + (freeze/head-mode {:trace t} @frozen)) + +(defn- near? [a b] + (every? #(< (js/Math.abs %) 1e-3) (map - a b))) + +(defn- resolver [c sid opts] + (clip/resolver c sid (:store @frozen) pal/index-of opts)) + +(defn- traces-at [r f] + (filterv #(= :trace (:kind %)) (r f))) + +;; --------------------------------------------------------------------------- +;; the document + +(deftest a-freeze-from-footage-makes-a-tracing-symbol-and-places-it-under-the-head + (let [c (traced nil) + plate (get-in c [:symbols :face-1 :nodes :plate])] + (is (clip/trace? (clip/symbol c :footage))) + (is (= {:footage "f00" :range [0 take/frames]} (:media (clip/symbol c :footage)))) + (is (= :head (:parent plate)) "the footage rides the head") + (is (= :footage (node/source plate))) + (is (node/measured? plate) "and is registered by the measurement, not by hand") + (is (empty? (clip/problems c)) (pr-str (clip/problems c))) + (is (= c (leaf/clip "t" (leaf/leaves "t" c))) "it saves and comes back"))) + +(deftest the-origin-writes-the-heads-reads-and-the-keys-the-plates-holds + (is (nil? (get-in (traced {:origin :continuous :frames [12]}) [:symbols :face-1 :nodes :head :reads]))) + (is (= {:holds-of :plate} (get-in (traced {:origin :keys :frames [12]}) [:symbols :face-1 :nodes :head :reads]))) + (is (= {:holds [0]} (get-in (traced {:origin :start}) [:symbols :face-1 :nodes :head :reads]))) + (is (= [12 40] (get-in (traced {:origin :continuous :frames [12 40]}) + [:symbols :face-1 :nodes :plate :time :holds])) + "the keys are the footage's whatever the head does")) + +(deftest a-tracing-symbol-that-does-not-say-what-it-shows-will-not-load + (doseq [[why sym] [["nodes" {:nodes {:x {:id :x :kind :group :z "a"}}}] + ["no media" {:media {}}] + ["both" {:media {:footage "f" :image "i" :range [0 10]}}] + ["short range" {:media {:footage "f" :range [0 9]}}] + ["no size" {:width nil}]]] + (let [c (update-in (traced nil) [:symbols :footage] merge sym)] + (is (seq (clip/problems c)) why)))) + +(deftest reads-must-name-a-node-that-can-be-followed + (doseq [reads [{:holds-of :nowhere} {:holds-of :head} {:holds [3 1]} + {:holds [0] :holds-of :plate}]] + (is (seq (clip/problems (assoc-in (traced nil) [:symbols :face-1 :nodes :head :reads] reads))) + (pr-str reads)))) + +;; --------------------------------------------------------------------------- +;; registration + +(defn- plate-at + "Face-1's footage op at frame `f`, its world matrix as a vector." + [c f] + (let [[op] (traces-at (resolver c :face-1 {:tracing? true}) f)] + (vec (array-seq (:m op))))) + +(defn- filmed + "Where footage sitting exactly where it was filmed lands: the face's own + placement, over image height, and nothing else." + [c f] + (let [r (resolver c :face-1 nil)] + (r f) + (vec (array-seq (node/mul! (node/mat) (symbol/world-of r [:place]) + (js/Float64Array. #js [(/ 1 image-h) 0 0 (/ 1 image-h) 0 0])))))) + +(deftest the-footage-registers-to-the-face-it-was-filmed-with + (let [free (traced {:origin :continuous}) + keyed (traced {:origin :keys :frames [12]}) + ride (traced {:origin :continuous :frames [12]}) + start (traced {:origin :start})] + (is (near? (filmed free 30) (plate-at free 30)) + "a head reading the footage's own frame cancels out") + (is (near? (filmed keyed 30) (plate-at keyed 30)) + "held at the key, face and footage both stand still where it was filmed") + (is (not (near? (filmed ride 30) (plate-at ride 30))) + "a continuous head carries the held footage along with it") + (is (= 12 (:frame (first (traces-at (resolver ride :face-1 {:tracing? true}) 30)))) + "showing the held frame") + (is (not (near? (filmed start 60) (plate-at start 60))) + "footage under a head held at its start is stabilised, not where it was filmed"))) + +;; --------------------------------------------------------------------------- +;; never in the picture + +(deftest only-the-stage-asks-for-tracing + (let [c (traced nil)] + (doseq [f [0 30 100]] + (is (empty? (traces-at (resolver c :main nil) f)) "an export, a centre, a thumbnail") + (is (= 1 (count (traces-at (resolver c :main {:tracing? true}) f))) "the stage")))) + +(deftest a-trace-op-comes-up-through-every-instance-above-it + (let [c (assoc-in (traced nil) [:symbols :wrap] + {:id :wrap :frames 200 + :nodes {:m {:id :m :kind :instance :z "a0" :source {:symbol :main} + :channels {[:xform :pos] {:animated? false :value [30 -10]}}}}}) + inner (first (traces-at (resolver c :main {:tracing? true}) 20)) + outer (first (traces-at (resolver c :wrap {:tracing? true}) 20)) + moved (fn [m] (let [v (vec (array-seq m))] (-> v (update 4 + 30) (update 5 - 10))))] + (is (= [:m :face-1 :plate] (:node outer)) "named by its row path, as anything drawn is") + (is (= [:face-1 :plate] (:layer outer)) "and switched by where it is, not where it is placed") + (is (near? (moved (:m inner)) (vec (array-seq (:m outer))))))) + +;; --------------------------------------------------------------------------- +;; a tracing layer placed by hand + +(def ^:private still + {:id :still :name "ref" :type :trace :media {:image "abc"} + :frames 1 :fps 30 :width 400 :height 200 :nodes {}}) + +(defn- with-still [] + (-> (clip/blank) + (assoc-in [:symbols :still] still) + (clip/place-symbol {} :main :still 0 :layer [160 100]) + (update-in [:symbols :main :nodes :layer] assoc + :playback {:in 0 :speed 0 :end :hold} :span [0 120]))) + +(deftest a-still-is-placed-about-its-middle-and-shows-on-every-frame + (let [c (with-still) + r (clip/resolver c :main {} pal/index-of {:tracing? true})] + (is (empty? (clip/problems c)) (pr-str (clip/problems c))) + (is (= [200 100] (clip/center c {} :still)) "a tracing symbol's middle is its pixels'") + (doseq [f [0 50 119]] + (let [[op] (traces-at r f)] + (is (= 0 (:frame op))) + (is (= [-40 0] (vec (take 2 (drop 4 (array-seq (:m op)))))) + (str "its middle on the drop point, frame " f)))))) + +(deftest nothing-is-created-inside-a-tracing-layer + (let [c (with-still)] + (is (= :main (:sid (creation/target c {} :main [:node :main :layer [:layer]] 10))) + "selecting a tracing layer creates beside it, as selecting a shape does") + (is (= "a tracing layer is a picture to draw over — nothing goes inside it" + (:refused (ui/drop-destination-at c {} :main 10 [:node :main :layer [:layer]])))))) + +(deftest a-click-picks-what-is-drawn-before-a-reference + (let [trace {:kind :trace :node :layer :size [100 100] + :m (js/Float64Array. #js [0 1 -1 0 50 0])} + shape {:kind :rect :node :dot :cx 10 :cy 10 :size 4}] + (is (= [:dot] (pick/hit [shape trace] [10 10])) "drawn wins, though the photo is on top") + (is (= [:layer] (pick/hit [shape trace] [20 50])) "a reference where nothing is drawn") + (is (nil? (pick/hit [shape trace] [60 50])) "outside the turned image is outside it"))) + +(deftest a-held-layer-is-keyed-on-its-own-unfloored-frames + (let [c (assoc-in (traced {:origin :keys :frames [12]}) [:symbols :face-1 :nodes :plate :time :holds] [12]) + t (nest/own-time c (:store @frozen) :face-1 [:plate] 30)] + (is (= 30 (* (:rate t) (- 30 (:at t)))) "frame 30 is 30, though it shows 12"))) + +(deftest a-dropped-still-lasts-the-rest-of-the-symbol-fits-it-and-is-reused + (let [id (store/install! {:clip (clip/blank) :store {}} "drop-tracing") + sym {:name "sheet" :type :trace :media {:image "abc"} + :width 400 :height 800 :nodes {}} + drop! (fn [] + (rf/dispatch-sync [::ui/drop-tracing sym 30 nil nil]) + (let [doc (:clip (store/entry id)) + [_ _ uuid] (get-in @rf-db/app-db [:ui :selection])] + [doc (get-in doc [:symbols :main :nodes uuid])]))] + (reset! rf-db/app-db {:clip/current id :paint/revision 0 + :ui {:open :main} :playback {:frame 30}}) + (let [[doc n] (drop!) + sid (node/source n) + [w h] (clip/stage doc :main) + k (/ h 800)] + (is (empty? (clip/problems doc)) (pr-str (clip/problems doc))) + (is (clip/trace? (clip/symbol doc sid))) + (is (= (- (clip/frames doc :main) 30) (clip/frames doc sid)) + "as long as what is left of the symbol it landed in") + (is (= [30 (clip/frames doc :main)] (node/placed-span n))) + (is (= [k k] (get-in n [:channels [:xform :scale] :value])) "as tall as the stage") + (is (= [(/ w 2) (/ h 2)] (mapv + (get-in n [:channels [:xform :pos] :value]) + (get-in n [:channels [:xform :anchor] :value]))) + "dropped on the timeline, it is middled on the stage") + (let [[doc2 n2] (drop!)] + (is (= sid (node/source n2)) "the same picture is the same symbol") + (is (= 1 (count (filter clip/trace? (vals (:symbols doc2)))))))))) + +;; --------------------------------------------------------------------------- +;; on and off + +(deftest switching-one-layer-on-switches-tracing-on + (reset! rf-db/app-db {:ui {:tracing {:on? false :opacity 0.5 :hidden #{[:face-1 :plate]}}}}) + (rf/dispatch-sync [::ui/show-trace [:face-1 :plate] true]) + (is (= {:on? true :opacity 0.5 :hidden #{}} (get-in @rf-db/app-db [:ui :tracing]))) + (testing "and switching one off leaves the rest alone" + (rf/dispatch-sync [::ui/show-trace [:face-1 :plate] false]) + (is (= {:on? true :opacity 0.5 :hidden #{[:face-1 :plate]}} (get-in @rf-db/app-db [:ui :tracing])))) + (testing "from nothing" + (reset! rf-db/app-db {:ui {:tracing {:on? true}}}) + (rf/dispatch-sync [::ui/show-trace [:s :n] false]) + (is (= #{[:s :n]} (get-in @rf-db/app-db [:ui :tracing :hidden]))))) diff --git a/frontend/test/arthur/flow/freeze_test.cljs b/frontend/test/arthur/flow/freeze_test.cljs index 8906b53..3a6003f 100644 --- a/frontend/test/arthur/flow/freeze_test.cljs +++ b/frontend/test/arthur/flow/freeze_test.cljs @@ -249,8 +249,9 @@ path [[:xform :pos] [:xform :rot] [:xform :scale]]] (is (= :dense (ch/describe (of c path)))) (is (= (of free path) (of c path)) "a trace does not copy measurements")) - (is (nil? (get-in (nodes free) [:head :trace]))) - (is (= {:frames [12] :origin :keys} (get-in (nodes one) [:head :trace]))) + (is (nil? (get-in (nodes free) [:head :reads]))) + (is (= {:holds [12]} (get-in (nodes one) [:head :reads])) + "with no footage to follow, the head holds at the keys itself") (is (= keyed (leaf/clip "head" (leaf/leaves "head" keyed))) "the trace survives the document round trip"))) diff --git a/frontend/test/arthur/flow/multi_face_test.cljs b/frontend/test/arthur/flow/multi_face_test.cljs index 98af740..b17e639 100644 --- a/frontend/test/arthur/flow/multi_face_test.cljs +++ b/frontend/test/arthur/flow/multi_face_test.cljs @@ -77,7 +77,7 @@ (channel after :face-2 :iris-r [:xform :pos]))) (is (= (channel before :face-2 :iris-l [:xform :pos]) (channel after :face-2 :iris-l [:xform :pos]))) - (is (= {:origin :keys :frames [12]} (get-in anchored [:symbols :face-2 :nodes :head :trace]))) + (is (= {:holds [12]} (get-in anchored [:symbols :face-2 :nodes :head :reads]))) (is (empty? (clip/problems anchored))))) (deftest the-second-subject-regenerates-inside-a-composed-stage diff --git a/static/arthur/app.css b/static/arthur/app.css index 6c69f65..27ed27c 100644 --- a/static/arthur/app.css +++ b/static/arthur/app.css @@ -875,7 +875,7 @@ button.share-button:hover, button.share-button.on { filter: brightness(1.1); } .paint-overlay { position: absolute; inset: 0; touch-action: none; } /* The footage traced over: a reference above the picture, never part of it. */ -.underlay { position: absolute; inset: 0; pointer-events: none; } +.tracing { position: absolute; inset: 0; pointer-events: none; } .paint-overlay.drawing { cursor: crosshair; } .paint-overlay circle { cursor: grab; } @@ -1176,6 +1176,12 @@ button.share-button:hover, button.share-button.on { filter: brightness(1.1); } /* A sound's bar, told apart from the picture's at a glance. */ .tl-span.sound { background: #e3efdf; border-color: #a6c49b; } +/* A tracing layer is a reference, never exported: hatched and dashed so it + cannot be read as a drawing. */ +.tl-span.trace, .tl-cel.trace { + background: repeating-linear-gradient(135deg, transparent 0 4px, rgba(0, 0, 0, 0.07) 4px 8px); + border: 1px dashed var(--span-line); +} .pool-item .thumb.sound { display: flex; align-items: center; justify-content: center; color: #d9d9d9; } /* The span the node exists over. Drawn under the keys so a dot on the first