Compare commits
31 commits
082d8561d2
...
ddabfbeaa8
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ddabfbeaa8 | ||
|
|
49ece8dee6 | ||
|
|
a45e89f4e4 | ||
|
|
73ab153b02 | ||
|
|
e22ee600b9 | ||
|
|
3058b9a5f2 | ||
|
|
39ee37db02 | ||
|
|
ffb95543a3 | ||
|
|
a611b86c0d | ||
|
|
65ad67c129 | ||
|
|
131b39bff0 | ||
|
|
44976cbb4b | ||
|
|
83d106bbc5 | ||
|
|
686f897401 | ||
|
|
690de21fa4 | ||
|
|
7249e73e7c | ||
|
|
7738c4e1c8 | ||
|
|
9778b9023b | ||
|
|
27bfe18bee | ||
|
|
9cd5243983 | ||
|
|
b6517f837a | ||
|
|
35ef150b48 | ||
|
|
ccca93e233 | ||
|
|
663b7c367a | ||
|
|
06cf02db83 | ||
|
|
32683efccf | ||
|
|
8a06835895 | ||
|
|
942e2f38ab | ||
|
|
11192d61c6 | ||
|
|
18d6495592 | ||
|
|
eb06be005c |
159 changed files with 41926 additions and 58 deletions
16
.dockerignore
Normal file
16
.dockerignore
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
.git
|
||||
.venv
|
||||
**/node_modules
|
||||
frontend/.shadow-cljs
|
||||
frontend/out
|
||||
frontend/.cpcache
|
||||
frontend/test/browser/out
|
||||
static/arthur/js
|
||||
db.sqlite3
|
||||
var
|
||||
frames
|
||||
scratch
|
||||
audio.wav
|
||||
manifest.json
|
||||
*.take
|
||||
*.tflite
|
||||
35
.gitignore
vendored
35
.gitignore
vendored
|
|
@ -1,5 +1,40 @@
|
|||
# extract.sh's output. It is TIER 3 — immutable, large, and the backend's to serve
|
||||
# once `manage.py ingest_bundle` has hashed it into var/blobs — so none of it
|
||||
# belongs in the repo. `audio.wav` was tracked before step 9 because the synthetic
|
||||
# take borrowed it for a clock; that copy now lives at static/arthur/audio.wav,
|
||||
# which is an asset the project owns rather than an extraction that churns.
|
||||
frames/
|
||||
/audio.wav
|
||||
/manifest.json
|
||||
# local extracted takes for comparing source cadences
|
||||
/scratch/
|
||||
*.task
|
||||
*.take
|
||||
|
||||
*.tflite
|
||||
|
||||
# CLJS build
|
||||
frontend/node_modules/
|
||||
# screenshots from the browser suite; regenerated by `npm run browser`
|
||||
frontend/test/browser/out/
|
||||
frontend/.shadow-cljs/
|
||||
frontend/out/
|
||||
frontend/.cpcache/
|
||||
static/arthur/js/
|
||||
|
||||
# mise-managed venv for the Django half
|
||||
.venv/
|
||||
|
||||
# the Django half's own state: the document database, the content-addressed blob
|
||||
# store (tiers 2 and 3), and collectstatic's output
|
||||
db.sqlite3
|
||||
db.sqlite3-shm
|
||||
db.sqlite3-wal
|
||||
/var/
|
||||
|
||||
# vim swap files
|
||||
*.swp
|
||||
|
||||
# Python bytecode
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
|
|
|||
43
Dockerfile
Normal file
43
Dockerfile
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
FROM node:20-bookworm AS frontend
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl tar \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& curl -fsSL 'https://api.adoptium.net/v3/binary/latest/21/ga/linux/x64/jdk/hotspot/normal/eclipse' -o /tmp/jdk.tar.gz \
|
||||
&& mkdir -p /opt/java \
|
||||
&& tar -xzf /tmp/jdk.tar.gz -C /opt/java --strip-components=1 \
|
||||
&& rm /tmp/jdk.tar.gz
|
||||
|
||||
ENV JAVA_HOME=/opt/java
|
||||
ENV PATH="/opt/java/bin:${PATH}"
|
||||
WORKDIR /app/frontend
|
||||
COPY frontend/package.json frontend/package-lock.json ./
|
||||
RUN npm ci
|
||||
COPY frontend/ ./
|
||||
RUN npx shadow-cljs release app
|
||||
|
||||
FROM python:3.12-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
DJANGO_DEBUG=0 \
|
||||
DJANGO_DB_PATH=/data/db.sqlite3 \
|
||||
DJANGO_BLOB_ROOT=/data/blobs
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ffmpeg \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& useradd --create-home --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY . ./
|
||||
COPY --from=frontend /app/static/arthur/js/ ./static/arthur/js/
|
||||
RUN python manage.py collectstatic --noinput \
|
||||
&& mkdir -p /data \
|
||||
&& chown -R app:app /app /data
|
||||
|
||||
USER app
|
||||
EXPOSE 8000
|
||||
CMD ["sh", "-c", "python manage.py migrate --noinput && exec gunicorn server.wsgi:application --bind 0.0.0.0:8000 --workers 1 --threads 4 --timeout 120"]
|
||||
123
README.md
123
README.md
|
|
@ -14,6 +14,62 @@ Animator Pro, where this started, but they are why the output looks right —
|
|||
modern conveniences belong in the workflow, not the output. See
|
||||
[docs/design.md](docs/design.md).
|
||||
|
||||
## ClojureScript port
|
||||
|
||||
The active port plays the synthetic take, accepts video uploads, transcodes them
|
||||
to an H.264 proxy and decodable stream plus audio and tracing stills, analyzes real footage
|
||||
for mouth, eyes, brows and pixel-derived teeth, and saves the project with
|
||||
reusable analysis data. The step 8 data model
|
||||
represents persistent feature IDs, eye pairs and feature-level observation gaps;
|
||||
its controls are still pending. See the [port plan](docs/port-plan.md).
|
||||
|
||||
```sh
|
||||
mise install # both halves
|
||||
pip install -r requirements.txt
|
||||
mise exec -- python manage.py migrate
|
||||
./do start # Django + frontend watcher
|
||||
```
|
||||
|
||||
In the app, upload a video, choose its footage, click **load frames**, then
|
||||
**save**. Opening that project on another client reuses its saved landmarks and
|
||||
mouth crops without detecting source frames again. The upload path derives its
|
||||
footage response from database records; it does not create or consume a
|
||||
`manifest.json` file. See [frontend/README.md](frontend/README.md) for details.
|
||||
Anything under "## Run" and below describes the older JS prototype, which still
|
||||
runs separately on port 8777.
|
||||
|
||||
### Deploy to Fly.io
|
||||
|
||||
The app uses a persistent Fly volume for its SQLite document database and
|
||||
content-addressed footage blobs. Create the app once, set its Django secret, and
|
||||
deploy from the repository root:
|
||||
|
||||
```sh
|
||||
fly apps create arthur --org personal
|
||||
fly volumes create data --region iad --size 1
|
||||
fly secrets set DJANGO_SECRET_KEY="$(openssl rand -hex 32)" --app arthur
|
||||
fly deploy --app arthur
|
||||
```
|
||||
|
||||
The deployed app is at <https://arthur.fly.dev>. The container builds the
|
||||
ClojureScript frontend, collects static assets, and runs database migrations on
|
||||
startup.
|
||||
|
||||
### How it is stored
|
||||
|
||||
Three tiers, cut by mutability and size — the full argument is in
|
||||
[docs/architecture.md](docs/architecture.md):
|
||||
|
||||
| Tier | What | Where |
|
||||
| --- | --- | --- |
|
||||
| 1 **authored** | the scene: nodes, channels, features, time maps | the database, as independently addressed leaves. Kilobytes |
|
||||
| 2 **derived** | detected landmarks, raw mouth crops, and dense channel blocks | `var/blobs`, addressed by analysis and block inputs, including the detector version |
|
||||
| 3 **source** | the uploaded video, H.264 proxy and elementary stream, tracing stills, and audio | the same blob store, by the hash of their bytes |
|
||||
|
||||
Only tier 1 is the document. Tier 2 is a pure function of tiers 1 and 3, so a
|
||||
saved project names its blocks rather than carrying them, and a knob change gives
|
||||
a block a new name rather than overwriting an old one.
|
||||
|
||||
## Run
|
||||
|
||||
```sh
|
||||
|
|
@ -33,27 +89,35 @@ wasm, which is fetched from a CDN on first use.
|
|||
|
||||
For real footage:
|
||||
|
||||
```sh
|
||||
./extract.sh /path/to/clip.mov 12 # -> frames/*.png, audio.wav, manifest.json
|
||||
```
|
||||
Upload it in the app. `./extract.sh` still writes the old PNG-sequence bundle and
|
||||
`ingest_bundle` still registers it, but footage ingested that way has no decodable
|
||||
stream and the loader will say so — the measured pixels come out of the video now.
|
||||
|
||||
then **Load frames**. MediaPipe's wasm is fetched from jsdelivr on first use;
|
||||
`face_landmarker.task` is local.
|
||||
MediaPipe's wasm and `face_landmarker.task` are both local; nothing in detection
|
||||
touches the network.
|
||||
|
||||
Frames are pre-extracted rather than decoded in the page because browser video
|
||||
seeking is approximate and `requestVideoFrameCallback` only delivers frames at
|
||||
playback speed — neither gives a deterministic per-frame pass.
|
||||
Detection reads the H.264 elementary stream with WebCodecs, one coded frame at a
|
||||
time. The proxy has no B-frames, so decode order is frame order. Each decoded
|
||||
frame reaches MediaPipe in VIDEO running mode at its footage timestamp. The
|
||||
decoder and detector advance together, with a pause between frames so progress
|
||||
can paint. Saved analyses reuse their stored crop pixels and measure them with
|
||||
the same pauses.
|
||||
|
||||
`manifest.json` records the true extraction rate. The page reads it rather than
|
||||
assuming, because a guessed fps desynchronises audio from picture — and sync is
|
||||
the one thing this view exists to show.
|
||||
This is what replaced the PNG sequence, which was 112MB for 7.6 seconds and would
|
||||
be 1.1GB at the 900-frame limit. The proxy is 6MB, and the landmarks barely
|
||||
notice: detected off decoded H.264 rather than off the PNGs, they moved at most
|
||||
0.0033 of frame width.
|
||||
|
||||
**Exposure** decides how often the picture gets a new drawing: rip at 24 and
|
||||
render `on 2s` for 12, `on 3s` for 8. The dense track and the audio are
|
||||
untouched, so it is a dropdown rather than a re-rip, and the export emits keys
|
||||
only on the grid instead of the same pose twice. Everything rides the same grid
|
||||
— mouth, eyes, teeth, plate — because a head cutting on the odd frames while the
|
||||
mouth cuts on the even ones reads as two performances laid over each other.
|
||||
The server's footage manifest records the proxy's frame rate and frame count;
|
||||
the page reads that rate because a guessed fps desynchronises audio from picture.
|
||||
Choosing a lower picture rate happens after analysis.
|
||||
|
||||
**Picture fps** decides how often the finished roto gets a new pose. Analyze all
|
||||
source frames, then sample those frozen poses at 12, 24 or the source rate while
|
||||
keeping the original duration and audio. **Exposure** can hold a drawing across
|
||||
more than one picture slot. The tracing editor chooses source frames for cel
|
||||
references separately. Shared timing is the useful default for mouth, eyes,
|
||||
teeth and plate so their changes read as one performance.
|
||||
|
||||
**Audio is the playback clock**: `frame = floor(audio.currentTime * fps)`. A slow
|
||||
render loop therefore drops frames instead of drifting, and ½x / ¼x work by
|
||||
|
|
@ -314,13 +378,13 @@ the tool a person made by hand; everything else regenerates. They are not in the
|
|||
## Two kinds of sparseness
|
||||
|
||||
Sparseness has two unrelated causes, and conflating them was the original design
|
||||
error here. **Aesthetic** sparseness is set by the extraction rate — pick 12fps and
|
||||
you have already chosen your timing. **Labour** sparseness is a human drawing
|
||||
each one, and it binds only on the plate.
|
||||
error here. **Aesthetic** sparseness is chosen from the full analyzed source
|
||||
track at rendering time. **Labour** sparseness is a human drawing each cel and
|
||||
selecting which source frames to use as tracing references.
|
||||
|
||||
Aesthetic sparseness is the **exposure** control, not the extraction rate —
|
||||
making it a render-time grid means auditioning 12 against 24 costs a dropdown
|
||||
instead of a re-rip and a full re-detection.
|
||||
Aesthetic sparseness is the **picture fps** control, with exposure available for
|
||||
longer holds. Both happen after analysis, so auditioning 12 against 24 needs no
|
||||
re-extraction or re-detection.
|
||||
|
||||
So the mouth keeps **every** frame: it is traced, and therefore free. In limited
|
||||
animation lip sync is routinely the densest element, on 1s, while heads hold on
|
||||
|
|
@ -373,3 +437,16 @@ performer→character calibration (currently identity, fitting the face oval to
|
|||
canvas); the override layer; anything on the Animator Pro side. The plate is a
|
||||
face-oval polygon per kept frame — it exists so the mouth has a face to read
|
||||
against, not to look good.
|
||||
|
||||
In the port specifically: the parameter UI and scoped regeneration (the model is
|
||||
built, the controls are not); automatic per-feature detection, so presence still
|
||||
comes from the full-face mask plus a manifest annotation; multiplayer, for which
|
||||
step 9 built the addressing and none of the socket; and in-browser extraction, so
|
||||
`extract.sh` plus `manage.py ingest_bundle` is still how footage arrives.
|
||||
|
||||
Two smaller things that are known and undecided. `measure/brows` takes no
|
||||
`presence` where `measure/eyes` does, so an occluded brow affects the freeze mask
|
||||
but not brow measurement, and occluded landmarks still enter contour smoothing —
|
||||
asymmetric with the eyes, and it is not settled which way is right. And `open`
|
||||
takes the most recently updated project and shows its first clip: there is no
|
||||
project browser, and the runtime store holds one clip at a time.
|
||||
|
|
|
|||
BIN
audio.wav
BIN
audio.wav
Binary file not shown.
0
clips/__init__.py
Normal file
0
clips/__init__.py
Normal file
63
clips/admin.py
Normal file
63
clips/admin.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
"""The admin, which is here for one reason: tier 1 is readable.
|
||||
|
||||
docs/architecture.md's argument against a CRDT is partly this — "the canonical
|
||||
document moves into an opaque blob, and every server-side thing that reads the
|
||||
document needs it materialised back out". A leaf is transit-as-JSON in a
|
||||
JSONField, so it is legible here, and that is a property worth being able to see.
|
||||
"""
|
||||
from django.contrib import admin
|
||||
|
||||
from .models import Analysis, Block, Blob, Clip, Footage, FootageFrame, Leaf, Project, Revision
|
||||
|
||||
|
||||
@admin.register(Project)
|
||||
class ProjectAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "id", "seq", "updated")
|
||||
search_fields = ("name", "id")
|
||||
|
||||
|
||||
@admin.register(Clip)
|
||||
class ClipAdmin(admin.ModelAdmin):
|
||||
list_display = ("cid", "project", "name", "footage", "analysis")
|
||||
list_filter = ("project",)
|
||||
|
||||
|
||||
@admin.register(Leaf)
|
||||
class LeafAdmin(admin.ModelAdmin):
|
||||
list_display = ("path", "project", "version", "updated")
|
||||
list_filter = ("project",)
|
||||
search_fields = ("path",)
|
||||
|
||||
|
||||
@admin.register(Revision)
|
||||
class RevisionAdmin(admin.ModelAdmin):
|
||||
list_display = ("project", "seq", "summary", "author", "created")
|
||||
|
||||
|
||||
@admin.register(Footage)
|
||||
class FootageAdmin(admin.ModelAdmin):
|
||||
list_display = ("label", "source", "fps", "frames", "width", "height", "created")
|
||||
|
||||
|
||||
@admin.register(FootageFrame)
|
||||
class FootageFrameAdmin(admin.ModelAdmin):
|
||||
list_display = ("footage", "index", "blob")
|
||||
list_filter = ("footage",)
|
||||
|
||||
|
||||
@admin.register(Analysis)
|
||||
class AnalysisAdmin(admin.ModelAdmin):
|
||||
list_display = ("key", "detector", "version", "footage", "created")
|
||||
search_fields = ("key", "detector", "version")
|
||||
|
||||
|
||||
@admin.register(Block)
|
||||
class BlockAdmin(admin.ModelAdmin):
|
||||
list_display = ("key", "role", "analysis", "data", "state", "created")
|
||||
list_filter = ("role",)
|
||||
search_fields = ("key",)
|
||||
|
||||
|
||||
@admin.register(Blob)
|
||||
class BlobAdmin(admin.ModelAdmin):
|
||||
list_display = ("digest", "media_type", "size", "created")
|
||||
14
clips/apps.py
Normal file
14
clips/apps.py
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ClipsConfig(AppConfig):
|
||||
"""The one app.
|
||||
|
||||
`clips` because the CLIP is the entity the whole tool is about and the one the
|
||||
prototype had exactly one of and never named — `state` in `js/app.js` is a clip
|
||||
with its analysis inlined and its palette global. Project, Footage, Analysis,
|
||||
Block, Leaf and Revision all hang off it.
|
||||
"""
|
||||
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "clips"
|
||||
146
clips/blobs.py
Normal file
146
clips/blobs.py
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
"""The content-addressed blob store: tiers 2 and 3 on disk.
|
||||
|
||||
One store for both, and docs/architecture.md says why in a sentence: once tier 3
|
||||
is decoded by the app rather than by a shell script, frames and audio become "the
|
||||
same kind of thing as tier 2 — a cache with a hash". So there is one place that
|
||||
writes bytes, one that reads them, and one URL shape for both.
|
||||
|
||||
TWO KINDS OF HASH, AND THEY ARE NOT THE SAME HASH. A blob is named by the sha256
|
||||
of its BYTES: that is what makes identical frames in two extractions one file. A
|
||||
derived thing — an analysis artifact, a dense block — is named by a sha256 over
|
||||
its INPUTS, which is what lets the client ask for the block the current settings
|
||||
want before anything has computed it. So `Block.key` is an input hash and
|
||||
`Block.data.digest` is a byte hash, and conflating them would break the half of
|
||||
addressing that answers questions about work not yet done.
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
CHUNK = 1 << 20
|
||||
CROP_MEDIA_TYPE = "application/zlib"
|
||||
|
||||
|
||||
def digest_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def digest_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
while chunk := fh.read(CHUNK):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def path_for(digest: str) -> Path:
|
||||
"""Where a blob lives.
|
||||
|
||||
Fanned out two levels, so that a take's worth of frames does not put a hundred
|
||||
thousand entries in one directory — which is slow on every filesystem and
|
||||
unusable on some.
|
||||
"""
|
||||
if len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest):
|
||||
raise ValueError(f"not a sha256: {digest!r}")
|
||||
return Path(settings.BLOB_ROOT) / digest[:2] / digest[2:4] / digest
|
||||
|
||||
|
||||
def write(data: bytes) -> tuple[str, int]:
|
||||
"""Store bytes, return (digest, size). Writing the same bytes twice is a
|
||||
no-op, which is what content addressing is for."""
|
||||
digest = digest_bytes(data)
|
||||
dest = path_for(digest)
|
||||
if not dest.exists():
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = dest.with_suffix(".part")
|
||||
with open(tmp, "wb") as fh:
|
||||
fh.write(data)
|
||||
os.replace(tmp, dest)
|
||||
return digest, len(data)
|
||||
|
||||
|
||||
def write_stream(chunks) -> tuple[str, int]:
|
||||
"""Store an uploaded file without reading the whole video into memory."""
|
||||
root = Path(settings.BLOB_ROOT)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with tempfile.NamedTemporaryFile(dir=root, prefix="upload-", delete=False) as out:
|
||||
temporary = Path(out.name)
|
||||
try:
|
||||
for chunk in chunks:
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
out.write(chunk)
|
||||
except BaseException:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
dest = path_for(digest.hexdigest())
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dest.exists():
|
||||
temporary.unlink()
|
||||
else:
|
||||
os.replace(temporary, dest)
|
||||
return digest.hexdigest(), size
|
||||
|
||||
|
||||
def write_compressed_stream(chunks) -> tuple[str, int]:
|
||||
"""Store a losslessly compressed stream; the digest names stored bytes."""
|
||||
compressor = zlib.compressobj()
|
||||
|
||||
def compressed():
|
||||
for chunk in chunks:
|
||||
if part := compressor.compress(chunk):
|
||||
yield part
|
||||
if part := compressor.flush():
|
||||
yield part
|
||||
|
||||
return write_stream(compressed())
|
||||
|
||||
|
||||
def adopt(source: Path) -> tuple[str, int]:
|
||||
"""Store a file already on disk, by hard link where the filesystem allows it.
|
||||
|
||||
112MB of PNGs is a normal extraction and copying them into a second place in
|
||||
the tree for no reason is not. A hard link is exact — the blob is immutable, so
|
||||
two names for one inode is the whole of what is wanted — and a copy is the
|
||||
fallback when `extract.sh` wrote to another volume.
|
||||
"""
|
||||
digest = digest_file(source)
|
||||
dest = path_for(digest)
|
||||
size = source.stat().st_size
|
||||
if not dest.exists():
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
os.link(source, dest)
|
||||
except OSError:
|
||||
tmp = dest.with_suffix(".part")
|
||||
with open(source, "rb") as src, open(tmp, "wb") as out:
|
||||
while chunk := src.read(CHUNK):
|
||||
out.write(chunk)
|
||||
os.replace(tmp, dest)
|
||||
return digest, size
|
||||
|
||||
|
||||
def read(digest: str) -> bytes:
|
||||
with open(path_for(digest), "rb") as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
def png_size(path: Path) -> tuple[int, int]:
|
||||
"""A PNG's dimensions, out of its IHDR.
|
||||
|
||||
Twenty-four bytes rather than a dependency. The footage's width and height are
|
||||
manifest data — docs/architecture.md's entity model puts them there — and
|
||||
Pillow to read two integers out of a header that has held them in the same
|
||||
place since 1996 is not a trade worth making.
|
||||
"""
|
||||
with open(path, "rb") as fh:
|
||||
head = fh.read(24)
|
||||
if head[:8] != b"\x89PNG\r\n\x1a\n" or head[12:16] != b"IHDR":
|
||||
raise ValueError(f"{path} is not a PNG")
|
||||
return int.from_bytes(head[16:20], "big"), int.from_bytes(head[20:24], "big")
|
||||
384
clips/extraction.py
Normal file
384
clips/extraction.py
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
"""Upload a video once, then turn it into the two things the app actually reads.
|
||||
|
||||
WHAT CHANGED AND WHY. This used to decode one PNG per source frame and store every
|
||||
one of them. A 7.6-second 1440x1920 take is 112MB that way, and the 900-frame limit
|
||||
is 1.1GB — for pixels whose only consumer was a canvas that MediaPipe then read
|
||||
once. The page now detects from the video itself (see `frontend/src/arthur/flow/
|
||||
ingest.cljs`), so this produces:
|
||||
|
||||
THE PROXY. One browser-safe H.264/yuv420p MP4, CFR, `+faststart`. The same take
|
||||
is 6MB. This is the analysis source, and it is re-encoded RATHER THAN KEPT AS
|
||||
UPLOADED even when the upload is already H.264, for two reasons that are both
|
||||
about not guessing: an iPhone's HEVC is not decodable in every browser, and the
|
||||
footage's identity is the digest of this file — one produced by one ffmpeg
|
||||
invocation, not one that depends on which branch the source happened to take.
|
||||
|
||||
THE TRACING STILLS. One JPEG per frame, long edge capped, for the tracing editor
|
||||
to draw over. Reference images; nothing measures them. They are not in the
|
||||
footage digest — see `models.Footage`.
|
||||
|
||||
The proxy is probed after it is written rather than before. `width`, `height` and
|
||||
`frames` are properties of the file the browser will decode, and taking them from
|
||||
the source instead is how a scaler or a dropped frame becomes a silent one-frame
|
||||
offset between the landmarks and the audio.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from fractions import Fraction
|
||||
from pathlib import Path
|
||||
|
||||
from django.db import close_old_connections, transaction
|
||||
|
||||
from . import blobs
|
||||
from .models import Blob, Extraction, Footage, FootageFrame
|
||||
|
||||
_active = set()
|
||||
_lock = threading.Lock()
|
||||
TIMEOUT = 3600
|
||||
|
||||
# Visually lossless enough that landmarks do not move: measured against the same
|
||||
# frames as PNGs, IMAGE-mode landmarks shifted at most 0.0033 of frame width.
|
||||
PROXY_CRF = "18"
|
||||
# The long edge of a tracing still. The proxy keeps full resolution because the
|
||||
# detector reads it; a still only has to be good enough to draw a cel over.
|
||||
TRACING_EDGE = 1280
|
||||
TRACING_QUALITY = "4"
|
||||
|
||||
|
||||
def _command(args):
|
||||
result = subprocess.run(args, capture_output=True, text=True, timeout=TIMEOUT)
|
||||
if result.returncode:
|
||||
raise ValueError((result.stderr or result.stdout or "media tool failed")[-1200:])
|
||||
return result.stdout
|
||||
|
||||
|
||||
def _run_with_progress(job, args, root, name, total, span):
|
||||
"""Run one ffmpeg and publish its live frame count as `span` of the job.
|
||||
|
||||
ffmpeg's `-progress` file is the only honest source for this: parsing its
|
||||
stderr means parsing a format that is explicitly not an interface, and a
|
||||
spinner that is not attached to frames is a spinner that lies on a long take.
|
||||
"""
|
||||
progress_path = root / f"{name}.progress"
|
||||
log_path = root / f"{name}.log"
|
||||
first, last = span
|
||||
args = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-stats_period", "0.25", "-progress", str(progress_path)] + args
|
||||
with open(log_path, "wb") as log:
|
||||
proc = subprocess.Popen(args, stdout=log, stderr=subprocess.STDOUT)
|
||||
deadline = time.monotonic() + TIMEOUT
|
||||
try:
|
||||
while proc.poll() is None:
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError(f"{name} timed out")
|
||||
if progress_path.exists():
|
||||
lines = progress_path.read_text(errors="replace").splitlines()
|
||||
count = next((int(line[6:].strip()) for line in reversed(lines)
|
||||
if line.startswith("frame=") and
|
||||
line[6:].strip().isdigit()), 0)
|
||||
if count and total:
|
||||
reached = first + int((last - first) * min(1.0, count / total))
|
||||
if reached > job.progress:
|
||||
job.progress = reached
|
||||
job.save(update_fields=["progress", "updated"])
|
||||
time.sleep(0.2)
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
if proc.returncode:
|
||||
raise ValueError(log_path.read_text(errors="replace")[-1200:] or f"{name} failed")
|
||||
|
||||
|
||||
def _encode_proxy(job, source_path, proxy_path, facts, root):
|
||||
"""The uploaded video -> one H.264 file every browser can decode and seek."""
|
||||
total = facts.get("reported_frames") or round(facts["duration"] * facts["fps"])
|
||||
_run_with_progress(
|
||||
job,
|
||||
["-i", str(source_path), "-an",
|
||||
# Constant frame rate at the rate `probe` chose. This RESAMPLES rather
|
||||
# than asserts: the upload is allowed to be variable, and this is the
|
||||
# step that makes the thing the page measures not be.
|
||||
"-fps_mode", "cfr", "-r", facts.get("rate") or str(facts["fps"]),
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", PROXY_CRF,
|
||||
# NO B-FRAMES, AND THIS IS THE LOAD-BEARING FLAG. It is what makes
|
||||
# decode order presentation order, so the page can treat access unit k
|
||||
# of the elementary stream as frame k without demuxing a container or
|
||||
# consulting a timestamp. With them x264 has a
|
||||
# two-frame reordering delay, ffmpeg compensates by writing an edit list
|
||||
# (`elst` media_time 1024 at timebase 1/15360 — exactly two frames), and
|
||||
# the browser then lives on two timelines at once: `currentTime` obeys the
|
||||
# edit list and the `mediaTime` reported by requestVideoFrameCallback does
|
||||
# not. Seek to frame 0 and the browser correctly hands back a frame whose
|
||||
# mediaTime says 2. Software decoding hides it; hardware decoding does
|
||||
# not, which is the worst possible way for it to be wrong. Without
|
||||
# B-frames DTS equals PTS, no edit list is written, and the two timelines
|
||||
# are the same one. It also makes decode order presentation order, should
|
||||
# this ever be fed to a WebCodecs VideoDecoder.
|
||||
"-bf", "0",
|
||||
# yuv420p and an even frame size are what makes this playable everywhere
|
||||
# rather than only in the browser that happened to be tested.
|
||||
"-pix_fmt", "yuv420p", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2",
|
||||
"-movflags", "+faststart", str(proxy_path)],
|
||||
root, "proxy", total, (0, 55))
|
||||
|
||||
|
||||
def _elementary_stream(proxy_path, out_path):
|
||||
"""The proxy's video, unwrapped into a raw Annex-B H.264 stream.
|
||||
|
||||
A STREAM COPY, not a second encode: the same coded frames as the MP4, with
|
||||
the container's length-prefixed NAL units rewritten as start-code-delimited
|
||||
ones. It costs a file read and nothing else.
|
||||
|
||||
This exists because the page decodes with WebCodecs, and `VideoDecoder` takes
|
||||
demuxed chunks rather than a container. Handing it Annex-B means the client
|
||||
needs no demuxer: NAL start codes are findable in a loop, and because the
|
||||
proxy is encoded with no B-frames, decode order is presentation order — so
|
||||
access unit k IS frame k, with no container timing to consult and no clock to
|
||||
reconcile. That is the whole reason this file is worth the bytes it costs.
|
||||
"""
|
||||
_command(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-i", str(proxy_path), "-an", "-c:v", "copy",
|
||||
"-bsf:v", "h264_mp4toannexb", "-f", "h264", str(out_path)])
|
||||
|
||||
|
||||
def _extract_stills(job, proxy_path, frames_dir, frames, root):
|
||||
"""The proxy -> one tracing JPEG per frame, long edge capped."""
|
||||
_run_with_progress(
|
||||
job,
|
||||
["-i", str(proxy_path), "-fps_mode", "passthrough",
|
||||
"-vf", f"scale='if(gt(iw,ih),min({TRACING_EDGE},iw),-2)':"
|
||||
f"'if(gt(iw,ih),-2,min({TRACING_EDGE},ih))'",
|
||||
"-q:v", TRACING_QUALITY, str(frames_dir / "%04d.jpg")],
|
||||
root, "stills", frames, (55, 85))
|
||||
|
||||
|
||||
MAX_RATE = 120 # a capture rate; past this the container is describing something else
|
||||
|
||||
|
||||
def probe(path):
|
||||
"""What the upload is, as far as choosing a proxy rate goes.
|
||||
|
||||
IT NO LONGER REFUSES VARIABLE-FRAME-RATE INPUT, and the reason is the proxy.
|
||||
That refusal was written when the page measured the source's own frames, where
|
||||
a wandering frame duration really does break `frame = floor(t * fps)`. Nothing
|
||||
measures the source now: ffmpeg resamples it onto a constant rate, and the
|
||||
proxy — constant by construction, and re-probed after it is written — is the
|
||||
only timeline anything downstream sees.
|
||||
|
||||
Keeping the check would have been worse than useless, because the thing it
|
||||
tested is not reliable. Ordinary iPhone footage, shot straight from the camera
|
||||
app, reports `avg_frame_rate` 8670/299 and `nb_frames` 289 on a stream whose
|
||||
decoded timestamps are 280 frames exactly 1/30s apart. The container's summary
|
||||
of itself disagreed with the container's own contents, so the guard rejected
|
||||
CFR video for being variable.
|
||||
|
||||
THE RATE IS THE NOMINAL ONE. `r_frame_rate` is the rate every timestamp in the
|
||||
stream can be expressed at, which is the rate that keeps every distinct source
|
||||
frame; resampling to the average would drop some. Duration is preserved either
|
||||
way — ffmpeg's CFR conversion is driven by timestamps, so the audio stays in
|
||||
sync at any rate — so this trades a possible duplicated frame against a
|
||||
certainly lost one.
|
||||
"""
|
||||
data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams",
|
||||
"-show_format", "-of", "json", str(path)]))
|
||||
video = next((s for s in data.get("streams", []) if s.get("codec_type") == "video"), None)
|
||||
if not video:
|
||||
raise ValueError("the uploaded file has no video stream")
|
||||
nominal = Fraction(video.get("r_frame_rate") or "0")
|
||||
average = Fraction(video.get("avg_frame_rate") or "0")
|
||||
if nominal <= 0 and average <= 0:
|
||||
raise ValueError("the video's frame rate is unknown")
|
||||
rate = nominal if 0 < nominal <= MAX_RATE else average
|
||||
if not 0 < rate <= MAX_RATE:
|
||||
raise ValueError(f"the video reports a frame rate of {float(rate):g}, which is "
|
||||
"not a rate footage can be measured at")
|
||||
duration = float(data.get("format", {}).get("duration") or 0)
|
||||
if duration > 0 and duration * float(rate) > 901:
|
||||
raise ValueError("video is longer than the 900-frame footage limit")
|
||||
frames = video.get("nb_frames")
|
||||
return {"fps": float(rate),
|
||||
# The exact rate, for ffmpeg. 30000/1001 is not a float, and handing
|
||||
# `-r` a rounded one is how a long take drifts out of sync.
|
||||
"rate": f"{rate.numerator}/{rate.denominator}",
|
||||
"nominal_fps": float(nominal), "average_fps": float(average),
|
||||
"width": int(video["width"]), "height": int(video["height"]),
|
||||
"duration": duration,
|
||||
# KEPT, AND NO LONGER TRUSTED AS A COUNT. See the docstring: this is
|
||||
# the container's claim about itself, it is wrong on ordinary phone
|
||||
# footage, and `run` checks the proxy's DURATION instead.
|
||||
"reported_frames": int(frames) if frames and frames.isdigit() else None,
|
||||
"has_audio": any(s.get("codec_type") == "audio" for s in data.get("streams", [])),
|
||||
"vfr": nominal != average}
|
||||
|
||||
|
||||
def _refuse_a_shifted_timeline(path):
|
||||
"""The proxy must put frame `i` at `i / fps` on BOTH of the browser's clocks.
|
||||
|
||||
Asserted rather than assumed, because the failure is silent and the symptom is
|
||||
unrecognisable. An encoder delay makes ffmpeg write an edit list, `currentTime`
|
||||
then obeys it while `requestVideoFrameCallback`'s `mediaTime` does not, and the
|
||||
page's frame walk is uniformly off by the delay — on hardware decoding only. It
|
||||
cost two wrong diagnoses to find, so it does not get to come back silently if
|
||||
somebody changes an encoder flag.
|
||||
"""
|
||||
data = json.loads(_command(["ffprobe", "-v", "error", "-select_streams", "v:0",
|
||||
"-show_streams", "-of", "json", str(path)]))
|
||||
stream = data["streams"][0]
|
||||
if int(stream.get("has_b_frames") or 0):
|
||||
raise ValueError(
|
||||
"the proxy was encoded with B-frames, whose reordering delay makes the "
|
||||
"browser's seek clock and its frame-timestamp clock disagree")
|
||||
if float(stream.get("start_time") or 0) != 0:
|
||||
raise ValueError(f"the proxy starts at {stream['start_time']}s rather than 0")
|
||||
|
||||
|
||||
def count_frames(path):
|
||||
"""How many frames a file really holds, counted rather than reported.
|
||||
|
||||
`nb_frames` is a container's claim. This is the decoder's answer, and it is
|
||||
what the page will get when it walks the proxy — so a disagreement between the
|
||||
two has to be settled before the count reaches a manifest, not after it has
|
||||
become a one-frame audio offset nobody can find.
|
||||
"""
|
||||
text = _command(["ffprobe", "-v", "error", "-select_streams", "v:0",
|
||||
"-count_frames", "-show_entries", "stream=nb_read_frames",
|
||||
"-of", "default=nokey=1:noprint_wrappers=1", str(path)])
|
||||
counted = text.strip()
|
||||
if not counted.isdigit():
|
||||
raise ValueError("could not count the proxy's frames")
|
||||
return int(counted)
|
||||
|
||||
|
||||
def extraction_key(source, settings):
|
||||
# Scheme 3: the extraction now also produces the elementary stream the page
|
||||
# decodes, so a job run under scheme 2 did not make everything this one does.
|
||||
text = json.dumps({"scheme": 3, "source": source.blob_id, "settings": settings},
|
||||
sort_keys=True, separators=(",", ":"))
|
||||
return "sha256:" + hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
def _register(job, proxy_path, stream_path, stills, audio_path, facts):
|
||||
proxy_digest, proxy_size = blobs.adopt(proxy_path)
|
||||
stream_digest, stream_size = blobs.adopt(stream_path)
|
||||
audio_digest, audio_size = blobs.adopt(audio_path)
|
||||
still_blobs = [(index, *blobs.adopt(path)) for index, path in enumerate(stills)]
|
||||
width, height, fps, frames = facts["width"], facts["height"], facts["fps"], facts["frames"]
|
||||
|
||||
# The footage's own identity: the bytes the page will measure, the audio it
|
||||
# will clock against, and the rate that ties them together. Scheme 2 — scheme
|
||||
# 1 hashed a PNG per frame, and those footages name pixels this no longer has.
|
||||
h = hashlib.sha256()
|
||||
h.update(f"arthur-footage-2/{fps}/{frames}/{width}x{height}\n".encode())
|
||||
h.update(proxy_digest.encode())
|
||||
h.update(audio_digest.encode())
|
||||
|
||||
with transaction.atomic():
|
||||
proxy_blob, _ = Blob.objects.get_or_create(
|
||||
digest=proxy_digest, defaults={"size": proxy_size, "media_type": "video/mp4"})
|
||||
stream_blob, _ = Blob.objects.get_or_create(
|
||||
digest=stream_digest, defaults={"size": stream_size, "media_type": "video/h264"})
|
||||
audio_blob, _ = Blob.objects.get_or_create(
|
||||
digest=audio_digest, defaults={"size": audio_size, "media_type": "audio/wav"})
|
||||
footage, created = Footage.objects.get_or_create(
|
||||
digest=h.hexdigest(),
|
||||
defaults={"label": job.source.filename[:200], "source": job.source.filename[:200],
|
||||
"fps": fps, "frames": frames, "width": width, "height": height,
|
||||
"audio": audio_blob, "video": proxy_blob, "stream": stream_blob})
|
||||
if not created and not footage.stream_id:
|
||||
# The same footage by identity, extracted before the elementary
|
||||
# stream existed. Its digest is over the proxy and the audio, which
|
||||
# have not changed — so this is the same footage gaining a file it
|
||||
# was always entitled to, not a different one.
|
||||
footage.stream = stream_blob
|
||||
if not footage.video_id:
|
||||
footage.video = proxy_blob
|
||||
footage.save(update_fields=["stream", "video"])
|
||||
if created:
|
||||
rows = []
|
||||
for index, digest, size in still_blobs:
|
||||
blob, _ = Blob.objects.get_or_create(
|
||||
digest=digest, defaults={"size": size, "media_type": "image/jpeg"})
|
||||
rows.append(FootageFrame(footage=footage, index=index, blob=blob))
|
||||
FootageFrame.objects.bulk_create(rows)
|
||||
return footage
|
||||
|
||||
|
||||
def run(key):
|
||||
close_old_connections()
|
||||
try:
|
||||
job = Extraction.objects.select_related("source", "source__blob").get(key=key)
|
||||
job.state, job.progress, job.error = "running", 0, ""
|
||||
job.save(update_fields=["state", "progress", "error", "updated"])
|
||||
facts = job.source.probe
|
||||
source_path = blobs.path_for(job.source.blob_id)
|
||||
with tempfile.TemporaryDirectory(prefix="arthur-extract-") as directory:
|
||||
root = Path(directory)
|
||||
proxy_path = root / "proxy.mp4"
|
||||
_encode_proxy(job, source_path, proxy_path, facts, root)
|
||||
|
||||
# Everything downstream describes the PROXY, not the upload.
|
||||
proxy_facts = probe(proxy_path)
|
||||
_refuse_a_shifted_timeline(proxy_path)
|
||||
frames = count_frames(proxy_path)
|
||||
if not 1 <= frames <= 900:
|
||||
raise ValueError(f"the proxy holds {frames} frames; the limit is 1–900")
|
||||
# CHECKED AS A DURATION, not as a frame count. The page's clock is
|
||||
# `frame = floor(audio.currentTime * fps)`, so what must not drift is
|
||||
# how long the picture lasts against how long the audio lasts — and
|
||||
# the source's own frame count is a number this has already caught
|
||||
# lying. A resample to a constant rate legitimately changes the count
|
||||
# and must not change the duration.
|
||||
drift = abs(frames / proxy_facts["fps"] - facts["duration"])
|
||||
if facts["duration"] > 0 and drift > 0.5:
|
||||
raise ValueError(
|
||||
f"the proxy runs {frames / proxy_facts['fps']:.2f}s and the upload "
|
||||
f"runs {facts['duration']:.2f}s; refusing footage whose picture and "
|
||||
"audio would drift")
|
||||
proxy_facts["frames"] = frames
|
||||
|
||||
stream_path = root / "proxy.h264"
|
||||
_elementary_stream(proxy_path, stream_path)
|
||||
|
||||
frames_dir = root / "stills"
|
||||
frames_dir.mkdir()
|
||||
_extract_stills(job, proxy_path, frames_dir, frames, root)
|
||||
stills = sorted(frames_dir.glob("*.jpg"))
|
||||
if len(stills) != frames:
|
||||
raise ValueError(f"wrote {len(stills)} tracing stills for {frames} frames")
|
||||
|
||||
job.progress = 85
|
||||
job.save(update_fields=["progress", "updated"])
|
||||
audio_path = root / "audio.wav"
|
||||
if facts["has_audio"]:
|
||||
_command(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-i", str(source_path), "-vn", "-ac", "1", "-ar", "44100",
|
||||
str(audio_path)])
|
||||
else:
|
||||
_command(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono",
|
||||
"-t", str(frames / proxy_facts["fps"]), "-c:a", "pcm_s16le",
|
||||
str(audio_path)])
|
||||
footage = _register(job, proxy_path, stream_path, stills, audio_path, proxy_facts)
|
||||
job.footage, job.state, job.progress = footage, "done", 100
|
||||
job.save(update_fields=["footage", "state", "progress", "updated"])
|
||||
except Exception as exc:
|
||||
Extraction.objects.filter(key=key).update(state="failed", error=str(exc)[:2000])
|
||||
finally:
|
||||
with _lock:
|
||||
_active.discard(key)
|
||||
close_old_connections()
|
||||
|
||||
|
||||
def enqueue(key):
|
||||
with _lock:
|
||||
if key in _active:
|
||||
return
|
||||
_active.add(key)
|
||||
threading.Thread(target=run, args=(key,), daemon=True,
|
||||
name=f"arthur-extract-{key[7:15]}").start()
|
||||
0
clips/management/__init__.py
Normal file
0
clips/management/__init__.py
Normal file
0
clips/management/commands/__init__.py
Normal file
0
clips/management/commands/__init__.py
Normal file
57
clips/management/commands/compress_crop_blocks.py
Normal file
57
clips/management/commands/compress_crop_blocks.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
"""Compress existing raw mouth crop blocks without changing their public bytes."""
|
||||
|
||||
import hashlib
|
||||
import zlib
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.db import transaction
|
||||
from django.db.models.deletion import ProtectedError
|
||||
|
||||
from clips import blobs
|
||||
from clips.models import Blob, Block
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Compress existing source/crops blobs and remove unreferenced raw copies"
|
||||
|
||||
def handle(self, *args, **options):
|
||||
converted = 0
|
||||
before = after = 0
|
||||
for block in Block.objects.filter(role="source/crops").select_related("data"):
|
||||
old = block.data
|
||||
if old.media_type == blobs.CROP_MEDIA_TYPE:
|
||||
continue
|
||||
old_digest = old.digest
|
||||
with open(blobs.path_for(old_digest), "rb") as source:
|
||||
digest, size = blobs.write_compressed_stream(
|
||||
iter(lambda: source.read(blobs.CHUNK), b"")
|
||||
)
|
||||
check = hashlib.sha256()
|
||||
decompressor = zlib.decompressobj()
|
||||
with open(blobs.path_for(digest), "rb") as compressed:
|
||||
while chunk := compressed.read(blobs.CHUNK):
|
||||
check.update(decompressor.decompress(chunk))
|
||||
check.update(decompressor.flush())
|
||||
if not decompressor.eof or check.hexdigest() != old_digest:
|
||||
raise CommandError(f"crop compression failed verification: {block.key}")
|
||||
with transaction.atomic():
|
||||
new, _ = Blob.objects.get_or_create(
|
||||
digest=digest,
|
||||
defaults={"size": size, "media_type": blobs.CROP_MEDIA_TYPE},
|
||||
)
|
||||
changed = Block.objects.filter(key=block.key, data=old).update(data=new)
|
||||
if not changed:
|
||||
continue
|
||||
converted += 1
|
||||
before += old.size
|
||||
after += size
|
||||
if old_digest != new.digest:
|
||||
try:
|
||||
old.delete()
|
||||
except ProtectedError:
|
||||
pass
|
||||
else:
|
||||
blobs.path_for(old_digest).unlink(missing_ok=True)
|
||||
self.stdout.write(
|
||||
f"Compressed {converted} crop blocks: {before:,} -> {after:,} bytes"
|
||||
)
|
||||
120
clips/management/commands/ingest_bundle.py
Normal file
120
clips/management/commands/ingest_bundle.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
"""Register an extracted bundle as tier 3.
|
||||
|
||||
python manage.py ingest_bundle # ./manifest.json
|
||||
python manage.py ingest_bundle scratch/my-take # that bundle
|
||||
|
||||
WHAT THIS REPLACES. Until step 9 the page fetched `/manifest.json` and then built
|
||||
`frames/0001.png` itself, with shadow-cljs's `:dev-http` serving the repo root. So
|
||||
the frame layout was a shared secret between a shell script and a ClojureScript
|
||||
namespace, and "where the frames are" was answered by a directory listing.
|
||||
|
||||
Now the server names every frame, and the client asks it. The frames go into the
|
||||
content-addressed blob store — by hard link, so 112MB of PNGs is not copied — and
|
||||
the manifest the client receives carries a URL per frame. That is the whole of what
|
||||
makes the frames the backend's to serve, and it is what the in-browser wasm-ffmpeg
|
||||
extraction docs/architecture.md describes will upload INTO, without the client
|
||||
learning anything new when it arrives: the same blobs, the same manifest, a
|
||||
different producer.
|
||||
|
||||
`extract.sh` still does the decoding. It is out of step 9's scope, it works, and it
|
||||
is the only part of this that needs a terminal.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.db import transaction
|
||||
|
||||
from clips import blobs
|
||||
from clips.models import Blob, Footage, FootageFrame
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Register an extracted frames+audio+manifest bundle as footage."
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"bundle", nargs="?", default=".",
|
||||
help="a directory holding manifest.json, or the manifest itself",
|
||||
)
|
||||
parser.add_argument("--label", default="", help="what to call it in the UI")
|
||||
|
||||
def handle(self, *args, **options):
|
||||
manifest_path = Path(options["bundle"])
|
||||
if manifest_path.is_dir():
|
||||
manifest_path = manifest_path / "manifest.json"
|
||||
if not manifest_path.exists():
|
||||
raise CommandError(f"{manifest_path} does not exist — run ./extract.sh first")
|
||||
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
root = manifest_path.parent
|
||||
frames_dir = root / manifest["dir"]
|
||||
audio_path = root / manifest["audio"]
|
||||
count = int(manifest["frames"])
|
||||
|
||||
pngs = sorted(frames_dir.glob("*.png"))
|
||||
if len(pngs) != count:
|
||||
raise CommandError(
|
||||
f"the manifest says {count} frames and {frames_dir} holds {len(pngs)}; "
|
||||
"refusing an inaccurate footage"
|
||||
)
|
||||
if not audio_path.exists():
|
||||
raise CommandError(f"{audio_path} does not exist")
|
||||
|
||||
width, height = blobs.png_size(pngs[0])
|
||||
|
||||
self.stdout.write(f"hashing {len(pngs)} frames…")
|
||||
frame_blobs = []
|
||||
for i, png in enumerate(pngs):
|
||||
digest, size = blobs.adopt(png)
|
||||
frame_blobs.append((i, digest, size))
|
||||
if (i + 1) % 25 == 0 or i + 1 == len(pngs):
|
||||
self.stdout.write(f" {i + 1}/{len(pngs)}")
|
||||
|
||||
audio_digest, audio_size = blobs.adopt(audio_path)
|
||||
|
||||
# The footage's own identity: every frame in order, plus the audio and the
|
||||
# rate. Two extractions of one clip at one rate are one footage, so an
|
||||
# analysis over it is reusable across both.
|
||||
import hashlib
|
||||
|
||||
h = hashlib.sha256()
|
||||
h.update(f"arthur-footage-1/{manifest['fps']}/{count}/{width}x{height}\n".encode())
|
||||
for _, digest, _ in frame_blobs:
|
||||
h.update(digest.encode())
|
||||
h.update(audio_digest.encode())
|
||||
footage_digest = h.hexdigest()
|
||||
|
||||
if existing := Footage.objects.filter(digest=footage_digest).first():
|
||||
self.stdout.write(self.style.SUCCESS(f"already ingested: {existing.id}"))
|
||||
return
|
||||
|
||||
with transaction.atomic():
|
||||
audio_blob, _ = Blob.objects.get_or_create(
|
||||
digest=audio_digest,
|
||||
defaults={"size": audio_size, "media_type": "audio/wav"},
|
||||
)
|
||||
footage = Footage.objects.create(
|
||||
digest=footage_digest,
|
||||
label=options["label"] or manifest.get("source") or frames_dir.name,
|
||||
source=manifest.get("source") or "",
|
||||
fps=float(manifest["fps"]),
|
||||
frames=count,
|
||||
width=width,
|
||||
height=height,
|
||||
audio=audio_blob,
|
||||
feature_absence=manifest.get("feature-absence") or {},
|
||||
)
|
||||
rows = []
|
||||
for index, digest, size in frame_blobs:
|
||||
blob, _ = Blob.objects.get_or_create(
|
||||
digest=digest, defaults={"size": size, "media_type": "image/png"}
|
||||
)
|
||||
rows.append(FootageFrame(footage=footage, index=index, blob=blob))
|
||||
FootageFrame.objects.bulk_create(rows)
|
||||
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
f"{count} frames at {manifest['fps']}fps, {width}x{height} -> footage {footage.id}"
|
||||
)
|
||||
)
|
||||
149
clips/migrations/0001_initial.py
Normal file
149
clips/migrations/0001_initial.py
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
# Generated by Django 5.2.17 on 2026-09-28 04:44
|
||||
|
||||
import django.db.models.deletion
|
||||
import uuid
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='Blob',
|
||||
fields=[
|
||||
('digest', models.CharField(max_length=64, primary_key=True, serialize=False)),
|
||||
('media_type', models.CharField(default='application/octet-stream', max_length=100)),
|
||||
('size', models.BigIntegerField()),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Project',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('name', models.CharField(default='untitled', max_length=200)),
|
||||
('seq', models.PositiveBigIntegerField(default=0)),
|
||||
('palette', models.CharField(default='arthur/default', max_length=64)),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('updated', models.DateTimeField(auto_now=True)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-updated'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Analysis',
|
||||
fields=[
|
||||
('key', models.CharField(max_length=71, primary_key=True, serialize=False)),
|
||||
('descriptor', models.TextField()),
|
||||
('detector', models.CharField(max_length=64)),
|
||||
('version', models.CharField(max_length=64)),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('artifact', models.ForeignKey(blank=True, help_text='the dense landmark track, once bake A is uploaded', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='analysis_for', to='clips.blob')),
|
||||
],
|
||||
options={
|
||||
'verbose_name_plural': 'analyses',
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Block',
|
||||
fields=[
|
||||
('key', models.CharField(max_length=71, primary_key=True, serialize=False)),
|
||||
('descriptor', models.TextField()),
|
||||
('role', models.CharField(max_length=32)),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('analysis', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='blocks', to='clips.analysis')),
|
||||
('data', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='block_data_for', to='clips.blob')),
|
||||
('state', models.ForeignKey(blank=True, help_text='the per-track absence mask, when the take has one', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='block_state_for', to='clips.blob')),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Footage',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('digest', models.CharField(max_length=64, unique=True)),
|
||||
('label', models.CharField(blank=True, max_length=200)),
|
||||
('source', models.CharField(blank=True, max_length=200)),
|
||||
('fps', models.FloatField()),
|
||||
('frames', models.PositiveIntegerField()),
|
||||
('width', models.PositiveIntegerField()),
|
||||
('height', models.PositiveIntegerField()),
|
||||
('feature_absence', models.JSONField(blank=True, default=dict)),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('audio', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='audio_for', to='clips.blob')),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-created'],
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysis',
|
||||
name='footage',
|
||||
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='analyses', to='clips.footage'),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Revision',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('seq', models.PositiveBigIntegerField()),
|
||||
('author', models.CharField(blank=True, max_length=200)),
|
||||
('summary', models.CharField(blank=True, max_length=500)),
|
||||
('document', models.JSONField(help_text='every leaf of the project, by path')),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions', to='clips.project')),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-seq'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='FootageFrame',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('index', models.PositiveIntegerField(help_text="0-based; source frame index + 1 is the PNG's name")),
|
||||
('blob', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='frame_for', to='clips.blob')),
|
||||
('footage', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='frame_set', to='clips.footage')),
|
||||
],
|
||||
options={
|
||||
'ordering': ['index'],
|
||||
'constraints': [models.UniqueConstraint(fields=('footage', 'index'), name='one_blob_per_frame')],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Leaf',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('path', models.CharField(max_length=300)),
|
||||
('value', models.JSONField()),
|
||||
('version', models.PositiveBigIntegerField(default=1)),
|
||||
('updated', models.DateTimeField(auto_now=True)),
|
||||
('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='leaves', to='clips.project')),
|
||||
],
|
||||
options={
|
||||
'ordering': ['path'],
|
||||
'constraints': [models.UniqueConstraint(fields=('project', 'path'), name='one_leaf_per_path')],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Clip',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('cid', models.SlugField(max_length=64)),
|
||||
('name', models.CharField(blank=True, max_length=200)),
|
||||
('order', models.IntegerField(default=0)),
|
||||
('analysis', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='clips', to='clips.analysis')),
|
||||
('blocks', models.ManyToManyField(blank=True, help_text="the tier-2 blocks this clip's channels name", related_name='clips', to='clips.block')),
|
||||
('footage', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='clips', to='clips.footage')),
|
||||
('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='clips', to='clips.project')),
|
||||
],
|
||||
options={
|
||||
'ordering': ['order', 'cid'],
|
||||
'constraints': [models.UniqueConstraint(fields=('project', 'cid'), name='one_cid_per_project')],
|
||||
},
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# Generated by Django 5.2.17 on 2026-09-28 13:10
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('clips', '0001_initial'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='analysis',
|
||||
name='artifact',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='analysis',
|
||||
name='source_blocks',
|
||||
field=models.ManyToManyField(blank=True, help_text='pixel-dependent landmarks, detection mask and mouth crops', related_name='source_for', to='clips.block'),
|
||||
),
|
||||
]
|
||||
39
clips/migrations/0003_source_extraction.py
Normal file
39
clips/migrations/0003_source_extraction.py
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
# Generated by Django 5.2.17 on 2026-09-28 13:23
|
||||
|
||||
import django.db.models.deletion
|
||||
import uuid
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('clips', '0002_remove_analysis_artifact_analysis_source_blocks'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='Source',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('filename', models.CharField(max_length=255)),
|
||||
('probe', models.JSONField(default=dict)),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('blob', models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='video_source', to='clips.blob')),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='Extraction',
|
||||
fields=[
|
||||
('key', models.CharField(max_length=71, primary_key=True, serialize=False)),
|
||||
('settings', models.JSONField(default=dict)),
|
||||
('state', models.CharField(default='queued', max_length=16)),
|
||||
('progress', models.PositiveIntegerField(default=0)),
|
||||
('error', models.TextField(blank=True)),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('updated', models.DateTimeField(auto_now=True)),
|
||||
('footage', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='extractions', to='clips.footage')),
|
||||
('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='extractions', to='clips.source')),
|
||||
],
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
# Generated by Django 5.2.17 on 2026-09-28 15:10
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('clips', '0003_source_extraction'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='footage',
|
||||
name='video',
|
||||
field=models.ForeignKey(blank=True, help_text='the browser-safe proxy the page detects from; null on pre-proxy footage', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='video_for', to='clips.blob'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='footageframe',
|
||||
name='index',
|
||||
field=models.PositiveIntegerField(help_text="0-based; source frame index + 1 is the JPEG's name"),
|
||||
),
|
||||
]
|
||||
24
clips/migrations/0005_footage_stream_alter_footage_video.py
Normal file
24
clips/migrations/0005_footage_stream_alter_footage_video.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# Generated by Django 5.2.17 on 2026-09-28 17:11
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('clips', '0004_footage_video_alter_footageframe_index'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='footage',
|
||||
name='stream',
|
||||
field=models.ForeignKey(blank=True, help_text="the proxy's video as raw Annex-B H.264: what the page DECODES, one access unit per frame; null on footage extracted before it", null=True, on_delete=django.db.models.deletion.PROTECT, related_name='stream_for', to='clips.blob'),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='footage',
|
||||
name='video',
|
||||
field=models.ForeignKey(blank=True, help_text='the browser-safe proxy, playable and seekable', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='video_for', to='clips.blob'),
|
||||
),
|
||||
]
|
||||
15
clips/migrations/0006_project_schema_version.py
Normal file
15
clips/migrations/0006_project_schema_version.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("clips", "0005_footage_stream_alter_footage_video"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name="project",
|
||||
name="schema_version",
|
||||
field=models.PositiveIntegerField(default=1),
|
||||
),
|
||||
]
|
||||
0
clips/migrations/__init__.py
Normal file
0
clips/migrations/__init__.py
Normal file
311
clips/models.py
Normal file
311
clips/models.py
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
"""The entity model, as tables.
|
||||
|
||||
It follows docs/architecture.md's model exactly, and the one thing worth reading
|
||||
it for is which tier each table is in, because that is what decides whether a row
|
||||
is a document, a cache entry or a source.
|
||||
|
||||
TIER 1, the document. Project, Clip, Leaf, Revision. Kilobytes, authored,
|
||||
versioned, and the only tier anything will ever sync.
|
||||
|
||||
TIER 2, derived. Analysis, Block. Content-addressed by a hash over every input
|
||||
that produced them — including the detector version — so a stale bake is
|
||||
unreachable rather than wrong, and a collaborator's bake is fetchable by the
|
||||
same key.
|
||||
|
||||
TIER 3, source. Footage, FootageFrame. Immutable, by hash.
|
||||
|
||||
Blob is under all three of them: bytes, named by the sha256 of themselves.
|
||||
|
||||
WHAT IS DELIBERATELY NOT HERE. `Clip` does not store fps, frames, width or height.
|
||||
They are in the document — the `timing` and `stage` leaves — and a copy of them in
|
||||
a column is a copy that comes to disagree with the scene it describes. The columns
|
||||
`Clip` does have are the ones the SERVER needs to answer a question about a clip
|
||||
without parsing its leaves: which footage, which analysis, which blocks.
|
||||
"""
|
||||
import uuid
|
||||
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Blob(models.Model):
|
||||
"""Bytes, named by the sha256 of themselves. The file is on disk under
|
||||
`BLOB_ROOT`; this row is the index and the size."""
|
||||
|
||||
digest = models.CharField(primary_key=True, max_length=64)
|
||||
media_type = models.CharField(max_length=100, default="application/octet-stream")
|
||||
size = models.BigIntegerField()
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.digest[:12]}… {self.size}B {self.media_type}"
|
||||
|
||||
|
||||
class Source(models.Model):
|
||||
"""An uploaded video, identified by its byte digest."""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
blob = models.OneToOneField(Blob, on_delete=models.PROTECT, related_name="video_source")
|
||||
filename = models.CharField(max_length=255)
|
||||
probe = models.JSONField(default=dict)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
|
||||
class Extraction(models.Model):
|
||||
"""One requested decode of a source into immutable footage."""
|
||||
|
||||
key = models.CharField(primary_key=True, max_length=71)
|
||||
source = models.ForeignKey(Source, on_delete=models.CASCADE, related_name="extractions")
|
||||
settings = models.JSONField(default=dict)
|
||||
state = models.CharField(max_length=16, default="queued")
|
||||
progress = models.PositiveIntegerField(default=0)
|
||||
error = models.TextField(blank=True)
|
||||
footage = models.ForeignKey(
|
||||
"Footage", null=True, blank=True, on_delete=models.SET_NULL,
|
||||
related_name="extractions",
|
||||
)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
updated = models.DateTimeField(auto_now=True)
|
||||
|
||||
|
||||
class Footage(models.Model):
|
||||
"""Tier 3: the frames and audio of one extraction, immutable.
|
||||
|
||||
`digest` is over the PROXY VIDEO's digest plus the audio's and the rate, so
|
||||
two extractions of the same clip at the same settings are one footage and the
|
||||
same analysis can be reused across both.
|
||||
|
||||
THE PROXY IS THE ANALYSIS SOURCE AND THE FRAMES ARE NOT. `video` is one
|
||||
browser-safe H.264 file, and it is what the page seeks through to detect
|
||||
landmarks. `frame_set` is a JPEG per frame at tracing size: reference stills
|
||||
for the tracing editor, never the thing measured. The two are not
|
||||
interchangeable, and which one carries the pixels an analysis was computed
|
||||
from is the difference between a 6MB take and a 1.1GB one.
|
||||
|
||||
So the frame JPEGs are deliberately NOT in `digest`. They are a rendering of
|
||||
this footage for a human to trace over; re-rendering them at another size does
|
||||
not make it different footage, and putting them in the identity would throw
|
||||
away every analysis when the tracing size changed.
|
||||
|
||||
`feature_absence` is the manifest annotation step 8 introduced: known
|
||||
occlusion intervals, one-based and inclusive, expanded into presence tracks by
|
||||
the loader. An input format, not a control UI.
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
digest = models.CharField(max_length=64, unique=True)
|
||||
label = models.CharField(max_length=200, blank=True)
|
||||
source = models.CharField(max_length=200, blank=True)
|
||||
fps = models.FloatField()
|
||||
frames = models.PositiveIntegerField()
|
||||
width = models.PositiveIntegerField()
|
||||
height = models.PositiveIntegerField()
|
||||
audio = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="audio_for")
|
||||
video = models.ForeignKey(
|
||||
Blob, null=True, blank=True, on_delete=models.PROTECT, related_name="video_for",
|
||||
help_text="the browser-safe proxy, playable and seekable",
|
||||
)
|
||||
stream = models.ForeignKey(
|
||||
Blob, null=True, blank=True, on_delete=models.PROTECT, related_name="stream_for",
|
||||
help_text="the proxy's video as raw Annex-B H.264: what the page DECODES, "
|
||||
"one access unit per frame; null on footage extracted before it",
|
||||
)
|
||||
feature_absence = models.JSONField(default=dict, blank=True)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-created"]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.label or self.source or self.id} ({self.frames}f @{self.fps})"
|
||||
|
||||
|
||||
class FootageFrame(models.Model):
|
||||
"""One tracing still. A row rather than an entry in a JSON list, because a
|
||||
frame is a thing the server serves, and because a blob's references have to be
|
||||
countable before anything can be collected.
|
||||
|
||||
A REFERENCE IMAGE, NOT A MEASUREMENT INPUT. See `Footage.video`."""
|
||||
|
||||
footage = models.ForeignKey(Footage, on_delete=models.CASCADE, related_name="frame_set")
|
||||
index = models.PositiveIntegerField(help_text="0-based; source frame index + 1 is the JPEG's name")
|
||||
blob = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="frame_for")
|
||||
|
||||
class Meta:
|
||||
ordering = ["index"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(fields=["footage", "index"], name="one_blob_per_frame"),
|
||||
]
|
||||
|
||||
|
||||
class Analysis(models.Model):
|
||||
"""Tier 2: one detector, at one version, over one footage.
|
||||
|
||||
`key` is a content address over every input, and `descriptor` is the exact
|
||||
canonical text that key is the sha256 of — sent by the client and stored, not
|
||||
recomputed here. `clips/views.py` says why that is the honest arrangement: JS
|
||||
prints an integral double as `1` and Python as `1.0`, so a scheme where both
|
||||
sides re-render the numbers breaks on the first one of them.
|
||||
|
||||
`detector` and `version` are columns as well as descriptor fields so that the
|
||||
question "which model produced this take" is answerable in the admin and in a
|
||||
query, rather than only by parsing a hash's preimage.
|
||||
"""
|
||||
|
||||
key = models.CharField(primary_key=True, max_length=71)
|
||||
descriptor = models.TextField()
|
||||
detector = models.CharField(max_length=64)
|
||||
version = models.CharField(max_length=64)
|
||||
footage = models.ForeignKey(
|
||||
Footage, null=True, blank=True, on_delete=models.SET_NULL, related_name="analyses"
|
||||
)
|
||||
source_blocks = models.ManyToManyField(
|
||||
"Block", blank=True, related_name="source_for",
|
||||
help_text="pixel-dependent landmarks, detection mask and mouth crops",
|
||||
)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
verbose_name_plural = "analyses"
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.detector} {self.version} → {self.key[7:19]}…"
|
||||
|
||||
|
||||
class Block(models.Model):
|
||||
"""Tier 2: one dense channel block.
|
||||
|
||||
Two hashes, and they are not the same hash. `key` is over the block's INPUTS,
|
||||
which is what lets a client ask for the block its current settings want before
|
||||
anything has computed it. `data.digest` is over the bytes. See clips/blobs.py.
|
||||
"""
|
||||
|
||||
key = models.CharField(primary_key=True, max_length=71)
|
||||
descriptor = models.TextField()
|
||||
role = models.CharField(max_length=32)
|
||||
analysis = models.ForeignKey(
|
||||
Analysis, null=True, blank=True, on_delete=models.SET_NULL, related_name="blocks"
|
||||
)
|
||||
data = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="block_data_for")
|
||||
state = models.ForeignKey(
|
||||
Blob, null=True, blank=True, on_delete=models.PROTECT, related_name="block_state_for",
|
||||
help_text="the per-track absence mask, when the take has one",
|
||||
)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.role} {self.key[7:19]}…"
|
||||
|
||||
|
||||
class Project(models.Model):
|
||||
"""Tier 1: the document's root.
|
||||
|
||||
`schema_version` identifies the stored document format. `seq` counts writes
|
||||
to this particular project; it is not a format version. Every write bumps
|
||||
`seq`, and a client that sees `seq > local + 1` refetches once broadcasts exist.
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
name = models.CharField(max_length=200, default="untitled")
|
||||
schema_version = models.PositiveIntegerField(default=1)
|
||||
seq = models.PositiveBigIntegerField(default=0)
|
||||
palette = models.CharField(max_length=64, default="arthur/default")
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
updated = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-updated"]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.name} ({self.id})"
|
||||
|
||||
def bump(self):
|
||||
self.seq += 1
|
||||
self.save(update_fields=["seq", "updated"])
|
||||
return self.seq
|
||||
|
||||
|
||||
class Clip(models.Model):
|
||||
"""Tier 1: the unit of work, and the thing leaf paths are scoped by.
|
||||
|
||||
`cid` is what appears in `clip/<cid>/...`, so it is the clip's identity as far
|
||||
as addressing is concerned and it does not change.
|
||||
"""
|
||||
|
||||
project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="clips")
|
||||
cid = models.SlugField(max_length=64)
|
||||
name = models.CharField(max_length=200, blank=True)
|
||||
order = models.IntegerField(default=0)
|
||||
footage = models.ForeignKey(
|
||||
Footage, null=True, blank=True, on_delete=models.SET_NULL, related_name="clips"
|
||||
)
|
||||
analysis = models.ForeignKey(
|
||||
Analysis, null=True, blank=True, on_delete=models.SET_NULL, related_name="clips"
|
||||
)
|
||||
blocks = models.ManyToManyField(
|
||||
Block, blank=True, related_name="clips",
|
||||
help_text="the tier-2 blocks this clip's channels name",
|
||||
)
|
||||
|
||||
class Meta:
|
||||
ordering = ["order", "cid"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(fields=["project", "cid"], name="one_cid_per_project"),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.cid} of {self.project.name}"
|
||||
|
||||
|
||||
class Leaf(models.Model):
|
||||
"""Tier 1: one independently addressed, independently versioned piece of the
|
||||
document.
|
||||
|
||||
The value is transit-as-JSON in a JSONField, so the column holds JSON rather
|
||||
than a string containing JSON: the admin can read a leaf, and the field-wise
|
||||
merge of a channel leaf that docs/architecture.md describes as fifteen lines of
|
||||
Python is possible over it. `version` is the entity tag a conditional write
|
||||
compares — RFC 7232, not a bespoke invention.
|
||||
"""
|
||||
|
||||
project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="leaves")
|
||||
path = models.CharField(max_length=300)
|
||||
value = models.JSONField()
|
||||
version = models.PositiveBigIntegerField(default=1)
|
||||
updated = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["path"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(fields=["project", "path"], name="one_leaf_per_path"),
|
||||
]
|
||||
|
||||
@property
|
||||
def etag(self):
|
||||
return f'"{self.version}"'
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.path}@{self.version}"
|
||||
|
||||
|
||||
class Revision(models.Model):
|
||||
"""Tier 1: a snapshot of the authored layer, with a user and a summary.
|
||||
|
||||
ON AN EXPLICIT TRIGGER, not on every save. tl snapshots a small annotation
|
||||
layer; arthur's tier 1 will contain cel polygons, so a snapshot per save bloats
|
||||
the table — docs/architecture.md's "revisions need a coarser trigger". So this
|
||||
is written by `POST /api/projects/<id>/revisions`, which is a "mark version"
|
||||
button, and never by a save.
|
||||
"""
|
||||
|
||||
project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="revisions")
|
||||
seq = models.PositiveBigIntegerField()
|
||||
author = models.CharField(max_length=200, blank=True)
|
||||
summary = models.CharField(max_length=500, blank=True)
|
||||
document = models.JSONField(help_text="every leaf of the project, by path")
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-seq"]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.project.name} r{self.seq}: {self.summary}"
|
||||
88
clips/templates/clips/index.html
Normal file
88
clips/templates/clips/index.html
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
{% load static %}<!doctype html>
|
||||
{% comment %}
|
||||
The host page, served by Django since port-plan step 9.
|
||||
|
||||
It was `frontend/public/index.html`, served by shadow-cljs's `:dev-http`, and that
|
||||
key is gone. The bundle is unchanged: shadow-cljs writes it into
|
||||
`static/arthur/js` and staticfiles serves it from there, so `manage.py runserver`
|
||||
and `shadow-cljs watch app` are the whole dev loop with nothing copying files
|
||||
between them.
|
||||
|
||||
The CSRF token is rendered so that Django sets its cookie, which is what
|
||||
`arthur.fx.http` reads to write the `X-CSRFToken` header. Saves are ordinary POSTs
|
||||
and PUTs with ordinary CSRF protection — no endpoint in this app is exempt.
|
||||
{% endcomment %}
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>arthur</title>
|
||||
<style>
|
||||
:root { color-scheme: dark; --bg: #12141c; --fg: #c9c3b4; }
|
||||
html, body { margin: 0; height: 100%; background: var(--bg); color: var(--fg); }
|
||||
body { font: 14px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; }
|
||||
main { padding: 24px; }
|
||||
/* The preview is nearest-neighbour everywhere. A browser that smooths the
|
||||
upscale would misrepresent the look the tool exists to judge. */
|
||||
canvas { image-rendering: pixelated; }
|
||||
h1 { font-size: 14px; font-weight: normal; opacity: .5; margin: 0 0 12px; }
|
||||
.stage { display: block; background: #12141c; }
|
||||
.stage-wrap { position: relative; width: fit-content; }
|
||||
.paint-overlay { position: absolute; inset: 0; touch-action: none; }
|
||||
.paint-overlay circle { cursor: grab; }
|
||||
.paint-tools { width: 640px; margin-top: 9px; font-size: 12px; }
|
||||
.paint-tools .row { display: flex; flex-wrap: wrap; align-items: center; gap: 6px; margin: 4px 0; }
|
||||
.paint-tools select { color: var(--fg); background: #1c1f2b; border: 1px solid #2b3040; font: inherit; }
|
||||
.paint-tools .hint { color: #d0ba86; opacity: .8; }
|
||||
audio { display: none; }
|
||||
.transport { margin-top: 12px; width: 640px; }
|
||||
.transport .row { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; }
|
||||
.transport .gap { flex: 1; }
|
||||
button {
|
||||
font: inherit; color: var(--fg); background: #1c1f2b;
|
||||
border: 1px solid #2b3040; padding: 3px 10px; cursor: pointer;
|
||||
}
|
||||
button:hover { background: #242836; }
|
||||
button:disabled { opacity: .45; cursor: wait; }
|
||||
button.on { background: #3a4258; border-color: #556080; }
|
||||
.scrub { width: 100%; margin: 10px 0 6px; }
|
||||
.readout { display: flex; gap: 18px; opacity: .55; font-size: 12px; }
|
||||
.readout .warn { color: #d98f5a; opacity: 1; }
|
||||
.picture-rate { display: flex; align-items: center; gap: 6px; margin-top: 7px;
|
||||
font-size: 12px; }
|
||||
.source-path { display: block; margin-top: 8px; font-size: 12px; opacity: .7; }
|
||||
.source-path select { margin: 0 8px; padding: 3px 5px;
|
||||
color: var(--fg); background: #1c1f2b; border: 1px solid #2b3040;
|
||||
font: inherit; max-width: 360px; }
|
||||
.load-status { margin-top: 6px; font-size: 12px; opacity: .75; }
|
||||
.export { width: 640px; margin-top: 14px; padding-top: 12px;
|
||||
border-top: 1px solid #2b3040; font-size: 12px; }
|
||||
.export .row { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; }
|
||||
.export .gap { flex: 1; }
|
||||
.export select { margin-left: 6px; padding: 3px 5px; color: var(--fg);
|
||||
background: #1c1f2b; border: 1px solid #2b3040; font: inherit; }
|
||||
.export .readout { margin-top: 7px; }
|
||||
.export .note { margin: 7px 0 0; }
|
||||
.controls { width: 640px; margin-top: 18px; padding-top: 12px;
|
||||
border-top: 1px solid #2b3040; font-size: 12px; }
|
||||
.controls select { margin-left: 8px; padding: 3px 5px; color: var(--fg);
|
||||
background: #1c1f2b; border: 1px solid #2b3040; font: inherit; }
|
||||
.shared-note { margin-top: 6px; color: #d0ba86; }
|
||||
.control-list { display: grid; grid-template-columns: 1fr 1fr; gap: 6px 16px;
|
||||
margin-top: 10px; }
|
||||
.control-row { display: grid; grid-template-columns: 115px 1fr 42px;
|
||||
align-items: center; gap: 6px; }
|
||||
.control-row input { width: 100%; }
|
||||
.control-row output { text-align: right; }
|
||||
.regeneration-debug { padding: 8px; margin-top: 10px; background: #1c1f2b;
|
||||
white-space: pre-wrap; color: #d0ba86; }
|
||||
.note { opacity: .35; font-size: 12px; max-width: 640px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
{% csrf_token %}
|
||||
<div id="app"></div>
|
||||
<script src="{% static 'mediapipe/vision_bundle.js' %}"></script>
|
||||
<script src="{% static 'arthur/js/main.js' %}"></script>
|
||||
</body>
|
||||
</html>
|
||||
0
clips/tests/__init__.py
Normal file
0
clips/tests/__init__.py
Normal file
816
clips/tests/test_api.py
Normal file
816
clips/tests/test_api.py
Normal file
|
|
@ -0,0 +1,816 @@
|
|||
"""What the server guarantees, as opposed to what the client intends.
|
||||
|
||||
The two interesting groups here are the ones that make the tier split a property
|
||||
of the system rather than a convention in ClojureScript:
|
||||
|
||||
A KEY DESCRIBES ITS BYTES. The server recomputes every tier-2 key it is handed
|
||||
and refuses a mismatch, so nothing can store a block under a name that is not
|
||||
the hash of its own descriptor.
|
||||
|
||||
A BLOCK CAN NAME ITS DETECTOR VERSION. Every block names an analysis and every
|
||||
analysis declares a detector and a version, both enforced here. That chain is
|
||||
what docs/architecture.md asks for: without it, a model upgrade that silently
|
||||
reuses old landmarks presents as "the tool got worse" with no event to attach it
|
||||
to.
|
||||
|
||||
The rest is the load/save round trip, the conditional write, and the footage
|
||||
manifest that makes the frames the backend's to serve.
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import zlib
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
from unittest import skipUnless
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.core.management import call_command
|
||||
from django.test import TestCase, override_settings
|
||||
|
||||
from clips import blobs, extraction
|
||||
from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision, Source
|
||||
|
||||
BLOB_DIR = tempfile.mkdtemp(prefix="arthur-test-blobs-")
|
||||
|
||||
|
||||
def key_for(descriptor: str) -> str:
|
||||
return "sha256:" + hashlib.sha256(descriptor.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def analysis_descriptor(version="1.0.1"):
|
||||
# Canonical JSON, written the way arthur.domain.canon writes it: sorted keys,
|
||||
# no spaces, integral doubles with no point.
|
||||
return ('{"aspect":1,"detector":"mediapipe","frames":48,"fps":30,"scheme":1,'
|
||||
f'"version":"{version}"}}')
|
||||
|
||||
|
||||
def block_descriptor(analysis_key, role="geom", anchor_avg=2):
|
||||
return (f'{{"analysis":"{analysis_key}","features":["mouth"],'
|
||||
f'"layout":{{"frames":48,"scale":16384,"stride":16,"tracks":1,"type":"int16"}},'
|
||||
f'"observation":null,"params":{{"anchor-avg":{anchor_avg}}},'
|
||||
f'"role":"{role}","scheme":1,"tracks":["outer"]}}')
|
||||
|
||||
|
||||
def png(width=4, height=3):
|
||||
"""The smallest valid PNG of a given size, written by hand.
|
||||
|
||||
So that `blobs.png_size` and the ingest path are exercised without Pillow. The
|
||||
one thing the backend needs from a PNG is its IHDR, and this is a PNG with one.
|
||||
"""
|
||||
def chunk(kind, payload):
|
||||
return (struct.pack(">I", len(payload)) + kind + payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF))
|
||||
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
raw = b"".join(b"\x00" + b"\x40\x40\x40" * width for _ in range(height))
|
||||
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr)
|
||||
+ chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b""))
|
||||
|
||||
|
||||
@override_settings(BLOB_ROOT=BLOB_DIR)
|
||||
class BlobStoreTests(TestCase):
|
||||
def test_the_same_bytes_are_stored_once(self):
|
||||
a, size = blobs.write(b"the same bytes")
|
||||
b, _ = blobs.write(b"the same bytes")
|
||||
self.assertEqual(a, b)
|
||||
self.assertEqual(size, 14)
|
||||
self.assertEqual(blobs.read(a), b"the same bytes")
|
||||
|
||||
def test_a_path_that_is_not_a_hash_is_refused(self):
|
||||
# The blob route takes its digest from the URL, so this is the check that
|
||||
# stops `/blob/../../etc/passwd` being a path at all.
|
||||
with self.assertRaises(ValueError):
|
||||
blobs.path_for("../../etc/passwd")
|
||||
with self.assertRaises(ValueError):
|
||||
blobs.path_for("deadbeef")
|
||||
|
||||
def test_a_png_reports_its_own_size(self):
|
||||
with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as fh:
|
||||
fh.write(png(17, 5))
|
||||
self.assertEqual((17, 5), blobs.png_size(Path(fh.name)))
|
||||
|
||||
def test_a_blob_is_served_immutable(self):
|
||||
digest, size = blobs.write(b"bytes on the wire")
|
||||
Blob.objects.create(digest=digest, size=size, media_type="application/octet-stream")
|
||||
response = self.client.get(f"/blob/{digest}")
|
||||
self.assertEqual(200, response.status_code)
|
||||
self.assertIn("immutable", response["Cache-Control"])
|
||||
self.assertEqual(f'"{digest}"', response["ETag"])
|
||||
self.assertEqual(b"bytes on the wire", b"".join(response.streaming_content))
|
||||
|
||||
def test_an_unknown_blob_is_a_404_and_not_a_traceback(self):
|
||||
self.assertEqual(404, self.client.get("/blob/" + "0" * 64).status_code)
|
||||
self.assertEqual(404, self.client.get("/blob/nonsense").status_code)
|
||||
|
||||
def test_a_blob_serves_byte_ranges(self):
|
||||
# NOT AN OPTIMISATION. A <video> that is handed 200 with no Accept-Ranges
|
||||
# reports an empty `seekable`, every currentTime write is a no-op, and the
|
||||
# detector then measures frame one over and over without anything raising.
|
||||
# Django's FileResponse does no Range handling, so this is the whole of
|
||||
# what makes the analysis source seekable.
|
||||
digest, _ = blobs.write(b"0123456789")
|
||||
Blob.objects.create(digest=digest, size=10, media_type="video/mp4")
|
||||
|
||||
whole = self.client.get(f"/blob/{digest}")
|
||||
self.assertEqual(200, whole.status_code)
|
||||
self.assertEqual("bytes", whole["Accept-Ranges"])
|
||||
|
||||
part = self.client.get(f"/blob/{digest}", headers={"range": "bytes=2-5"})
|
||||
self.assertEqual(206, part.status_code)
|
||||
self.assertEqual("bytes 2-5/10", part["Content-Range"])
|
||||
self.assertEqual("4", part["Content-Length"])
|
||||
self.assertEqual(b"2345", b"".join(part.streaming_content))
|
||||
|
||||
# An open end, which is what a media element actually sends first.
|
||||
tail = self.client.get(f"/blob/{digest}", headers={"range": "bytes=7-"})
|
||||
self.assertEqual(206, tail.status_code)
|
||||
self.assertEqual("bytes 7-9/10", tail["Content-Range"])
|
||||
self.assertEqual(b"789", b"".join(tail.streaming_content))
|
||||
|
||||
# A suffix range asks a different question: the LAST n bytes.
|
||||
suffix = self.client.get(f"/blob/{digest}", headers={"range": "bytes=-3"})
|
||||
self.assertEqual(206, suffix.status_code)
|
||||
self.assertEqual("bytes 7-9/10", suffix["Content-Range"])
|
||||
|
||||
# Past the end is a 416 with the real length, so the client can recover.
|
||||
over = self.client.get(f"/blob/{digest}", headers={"range": "bytes=50-60"})
|
||||
self.assertEqual(416, over.status_code)
|
||||
self.assertEqual("bytes */10", over["Content-Range"])
|
||||
|
||||
# Unparsable is not an error: RFC 9110 says ignore it and send it all.
|
||||
junk = self.client.get(f"/blob/{digest}", headers={"range": "furlongs=1-2"})
|
||||
self.assertEqual(200, junk.status_code)
|
||||
self.assertEqual(b"0123456789", b"".join(junk.streaming_content))
|
||||
|
||||
|
||||
@override_settings(BLOB_ROOT=BLOB_DIR)
|
||||
class Tier2Tests(TestCase):
|
||||
def post(self, url, payload):
|
||||
return self.client.post(url, data=json.dumps(payload),
|
||||
content_type="application/json")
|
||||
|
||||
def register_analysis(self, version="1.0.1"):
|
||||
descriptor = analysis_descriptor(version)
|
||||
key = key_for(descriptor)
|
||||
response = self.post("/api/analyses", {
|
||||
"key": key, "descriptor": descriptor,
|
||||
"detector": "mediapipe", "version": version,
|
||||
})
|
||||
self.assertEqual(201, response.status_code, response.content)
|
||||
return key
|
||||
|
||||
def test_an_analysis_is_its_own_descriptors_hash(self):
|
||||
key = self.register_analysis()
|
||||
row = Analysis.objects.get(key=key)
|
||||
self.assertEqual("mediapipe", row.detector)
|
||||
self.assertEqual("1.0.1", row.version)
|
||||
# Idempotent: the same inputs are the same key are the same row.
|
||||
again = self.post("/api/analyses", {
|
||||
"key": key, "descriptor": analysis_descriptor(), "detector": "mediapipe",
|
||||
"version": "1.0.1",
|
||||
})
|
||||
self.assertEqual(200, again.status_code)
|
||||
self.assertEqual(1, Analysis.objects.count())
|
||||
|
||||
def test_a_key_that_is_not_the_hash_of_its_descriptor_is_refused(self):
|
||||
response = self.post("/api/analyses", {
|
||||
"key": "sha256:" + "0" * 64, "descriptor": analysis_descriptor(),
|
||||
})
|
||||
self.assertEqual(409, response.status_code)
|
||||
self.assertIn("not the hash", response.json()["error"])
|
||||
self.assertEqual(0, Analysis.objects.count())
|
||||
|
||||
def test_an_analysis_without_a_detector_version_is_refused(self):
|
||||
# The rule docs/architecture.md is most insistent about, enforced where a
|
||||
# client cannot forget it.
|
||||
descriptor = '{"detector":"mediapipe","frames":48,"scheme":1}'
|
||||
response = self.post("/api/analyses", {
|
||||
"key": key_for(descriptor), "descriptor": descriptor,
|
||||
})
|
||||
self.assertEqual(400, response.status_code)
|
||||
self.assertEqual("version", response.json()["missing"])
|
||||
|
||||
def test_a_block_is_stored_under_the_hash_of_its_inputs(self):
|
||||
analysis = self.register_analysis()
|
||||
descriptor = block_descriptor(analysis)
|
||||
key = key_for(descriptor)
|
||||
response = self.post("/api/blocks", {
|
||||
"key": key, "descriptor": descriptor,
|
||||
"data": "AAECAwQFBgc=", "state": "AAE=",
|
||||
})
|
||||
self.assertEqual(201, response.status_code, response.content)
|
||||
row = Block.objects.get(key=key)
|
||||
self.assertEqual("geom", row.role)
|
||||
self.assertEqual(analysis, row.analysis_id)
|
||||
# Two hashes, and they are not the same hash: the key is over the inputs,
|
||||
# the blob's digest is over the bytes.
|
||||
self.assertNotEqual(key[7:], row.data.digest)
|
||||
self.assertEqual(8, row.data.size)
|
||||
|
||||
fetched = self.client.get(f"/api/blocks/{key}").json()
|
||||
self.assertEqual("AAECAwQFBgc=", fetched["data"])
|
||||
self.assertEqual("AAE=", fetched["state"])
|
||||
self.assertEqual(descriptor, fetched["descriptor"])
|
||||
|
||||
@override_settings(DATA_UPLOAD_MAX_MEMORY_SIZE=1024, FILE_UPLOAD_MAX_MEMORY_SIZE=1024)
|
||||
def test_large_block_upload_streams_past_json_body_limit(self):
|
||||
analysis = self.register_analysis()
|
||||
descriptor = block_descriptor(analysis, role="source/crops")
|
||||
key = key_for(descriptor)
|
||||
payload = bytes(range(256)) * 16
|
||||
response = self.client.post("/api/blocks", {
|
||||
"key": key,
|
||||
"descriptor": descriptor,
|
||||
"data": SimpleUploadedFile("block.bin", payload),
|
||||
"state": SimpleUploadedFile("state.bin", b"\x00\x01"),
|
||||
})
|
||||
self.assertEqual(201, response.status_code, response.content)
|
||||
row = Block.objects.get(key=key)
|
||||
self.assertEqual(blobs.CROP_MEDIA_TYPE, row.data.media_type)
|
||||
self.assertLess(row.data.size, len(payload))
|
||||
self.assertEqual(payload, zlib.decompress(blobs.read(row.data_id)))
|
||||
self.assertEqual(base64.b64encode(payload).decode(),
|
||||
self.client.get(f"/api/blocks/{key}").json()["data"])
|
||||
self.assertEqual(b"\x00\x01", blobs.read(row.state_id))
|
||||
|
||||
def test_existing_raw_crop_block_is_compressed_without_changing_its_key_or_read(self):
|
||||
analysis = self.register_analysis()
|
||||
descriptor = block_descriptor(analysis, role="source/crops")
|
||||
key = key_for(descriptor)
|
||||
payload = b"raw crop pixels" * 100
|
||||
digest, size = blobs.write(payload)
|
||||
old = Blob.objects.create(digest=digest, size=size)
|
||||
Block.objects.create(key=key, descriptor=descriptor, role="source/crops",
|
||||
analysis_id=analysis, data=old)
|
||||
|
||||
call_command("compress_crop_blocks", stdout=StringIO())
|
||||
row = Block.objects.select_related("data").get(key=key)
|
||||
self.assertEqual(blobs.CROP_MEDIA_TYPE, row.data.media_type)
|
||||
self.assertEqual(base64.b64encode(payload).decode(),
|
||||
self.client.get(f"/api/blocks/{key}").json()["data"])
|
||||
self.assertFalse(blobs.path_for(digest).exists())
|
||||
compressed_digest = row.data_id
|
||||
call_command("compress_crop_blocks", stdout=StringIO())
|
||||
self.assertEqual(compressed_digest, Block.objects.get(key=key).data_id)
|
||||
|
||||
def test_a_block_whose_analysis_is_unknown_is_refused(self):
|
||||
descriptor = block_descriptor("sha256:" + "f" * 64)
|
||||
response = self.post("/api/blocks", {
|
||||
"key": key_for(descriptor), "descriptor": descriptor, "data": "AA==",
|
||||
})
|
||||
self.assertEqual(400, response.status_code)
|
||||
self.assertIn("analysis the server does not know", response.json()["error"])
|
||||
|
||||
def test_a_block_that_does_not_say_what_its_elements_are_is_refused(self):
|
||||
analysis = self.register_analysis()
|
||||
descriptor = ('{"analysis":"%s","layout":{"frames":48},"role":"geom","scheme":1}'
|
||||
% analysis)
|
||||
response = self.post("/api/blocks", {
|
||||
"key": key_for(descriptor), "descriptor": descriptor, "data": "AA==",
|
||||
})
|
||||
self.assertEqual(400, response.status_code)
|
||||
self.assertIn("valid readings", response.json()["error"])
|
||||
|
||||
def test_only_the_missing_blocks_are_asked_for(self):
|
||||
analysis = self.register_analysis()
|
||||
here = key_for(block_descriptor(analysis))
|
||||
self.post("/api/blocks", {
|
||||
"key": here, "descriptor": block_descriptor(analysis), "data": "AA==",
|
||||
})
|
||||
elsewhere = key_for(block_descriptor(analysis, anchor_avg=3))
|
||||
response = self.post("/api/blocks/missing", {"keys": [here, elsewhere]})
|
||||
self.assertEqual([elsewhere], response.json()["missing"])
|
||||
|
||||
def test_a_detector_upgrade_gives_a_block_a_new_name(self):
|
||||
# The end-to-end statement of the requirement: the same measurements under
|
||||
# a new model version are a different, additional block, and the old one is
|
||||
# unreachable from the new document rather than wrong.
|
||||
old = self.register_analysis("1.0.1")
|
||||
new_descriptor = analysis_descriptor("1.0.2")
|
||||
self.post("/api/analyses", {"key": key_for(new_descriptor),
|
||||
"descriptor": new_descriptor})
|
||||
for analysis in (old, key_for(new_descriptor)):
|
||||
descriptor = block_descriptor(analysis)
|
||||
self.post("/api/blocks", {"key": key_for(descriptor),
|
||||
"descriptor": descriptor, "data": "AAEC"})
|
||||
self.assertEqual(2, Block.objects.count())
|
||||
# One set of bytes, two names: the upgrade renamed the block and did not
|
||||
# duplicate it on disk.
|
||||
self.assertEqual(1, Blob.objects.filter(block_data_for__isnull=False).distinct().count())
|
||||
|
||||
def test_an_analysis_reopens_its_three_source_blocks(self):
|
||||
analysis = self.register_analysis()
|
||||
keys = []
|
||||
for role in ("source/dense", "source/detected", "source/crops"):
|
||||
descriptor = block_descriptor(analysis, role=role)
|
||||
key = key_for(descriptor)
|
||||
self.assertEqual(201, self.post("/api/blocks", {
|
||||
"key": key, "descriptor": descriptor, "data": "AA==",
|
||||
}).status_code)
|
||||
keys.append(key)
|
||||
response = self.client.put(
|
||||
f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys}),
|
||||
content_type="application/json")
|
||||
self.assertEqual(200, response.status_code, response.content)
|
||||
self.assertEqual(set(keys), set(self.client.get(
|
||||
f"/api/analyses/{analysis}").json()["source_blocks"]))
|
||||
self.assertEqual(200, self.client.put(
|
||||
f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys}),
|
||||
content_type="application/json").status_code)
|
||||
self.assertEqual(400, self.client.put(
|
||||
f"/api/analyses/{analysis}", json.dumps({"source_blocks": keys[:2]}),
|
||||
content_type="application/json").status_code)
|
||||
|
||||
def test_source_roles_are_complete_and_unique_per_subject(self):
|
||||
analysis = self.register_analysis()
|
||||
keys = []
|
||||
for subject in ("face-1", "face-2"):
|
||||
for role in ("source/dense", "source/detected", "source/crops"):
|
||||
desc = json.loads(block_descriptor(analysis, role=role))
|
||||
desc["features"] = [subject]
|
||||
descriptor = json.dumps(desc, sort_keys=True, separators=(",", ":"))
|
||||
key = key_for(descriptor)
|
||||
self.assertEqual(201, self.post("/api/blocks", {
|
||||
"key": key, "descriptor": descriptor, "data": "AA==",
|
||||
}).status_code)
|
||||
keys.append(key)
|
||||
|
||||
def put(keys):
|
||||
return self.client.put(f"/api/analyses/{analysis}",
|
||||
json.dumps({"source_blocks": keys}),
|
||||
content_type="application/json")
|
||||
|
||||
self.assertEqual(400, put(keys[:-1]).status_code)
|
||||
self.assertEqual(400, put(keys + keys[:1]).status_code)
|
||||
self.assertEqual(200, put(keys).status_code)
|
||||
self.assertEqual(200, put(list(reversed(keys))).status_code)
|
||||
self.assertEqual(409, put(keys[:3]).status_code)
|
||||
self.assertEqual(set(keys), set(self.client.get(
|
||||
f"/api/analyses/{analysis}").json()["source_blocks"]))
|
||||
|
||||
|
||||
|
||||
@override_settings(BLOB_ROOT=BLOB_DIR)
|
||||
class DocumentTests(TestCase):
|
||||
"""Tier 1: load, save, and the conditional write."""
|
||||
|
||||
def setUp(self):
|
||||
self.project = Project.objects.create(name="a project")
|
||||
descriptor = analysis_descriptor()
|
||||
self.analysis = key_for(descriptor)
|
||||
self.client.post("/api/analyses", data=json.dumps(
|
||||
{"key": self.analysis, "descriptor": descriptor}),
|
||||
content_type="application/json")
|
||||
block = block_descriptor(self.analysis)
|
||||
self.block = key_for(block)
|
||||
self.client.post("/api/blocks", data=json.dumps(
|
||||
{"key": self.block, "descriptor": block, "data": "AAECAwQFBgc="}),
|
||||
content_type="application/json")
|
||||
|
||||
def put(self, url, payload, **headers):
|
||||
return self.client.put(url, data=json.dumps(payload),
|
||||
content_type="application/json", **headers)
|
||||
|
||||
def leaves(self):
|
||||
# Transit-shaped, because that is what a leaf actually holds: a map with a
|
||||
# cache marker, keyword keys, and a frame-keyed inner map.
|
||||
return {
|
||||
"clip/c1/timing": ["^ ", "~:fps", 30],
|
||||
"clip/c1/timeline/main": ["^ ", "~:frames", 48],
|
||||
"clip/c1/timeline/main/node/mouth": ["^ ", "~:id", "~:mouth", "~:z", "a1"],
|
||||
"clip/c1/timeline/main/channel/mouth/geom.pts": [
|
||||
"^ ", "~:animated?", True, "~:dense",
|
||||
["^ ", "~:store", self.block, "~:offset", 0, "~:stride", 16],
|
||||
],
|
||||
"clip/c1/timeline/main/channel/mouth-in/vis": [
|
||||
"^ ", "~:animated?", True, "~:keys", ["^ ", "~i0", True, "~i12", False],
|
||||
],
|
||||
}
|
||||
|
||||
def save(self, leaves=None, blocks=None):
|
||||
return self.put(f"/api/projects/{self.project.id}", {
|
||||
"name": "a project",
|
||||
"clips": [{"cid": "c1", "name": "take", "analysis": self.analysis,
|
||||
"leaves": leaves if leaves is not None else self.leaves(),
|
||||
"blocks": blocks if blocks is not None else [self.block]}],
|
||||
})
|
||||
|
||||
def test_a_document_comes_back_exactly(self):
|
||||
response = self.save()
|
||||
self.assertEqual(200, response.status_code, response.content)
|
||||
self.assertEqual(5, len(response.json()["written"]))
|
||||
|
||||
loaded = self.client.get(f"/api/projects/{self.project.id}").json()
|
||||
self.assertEqual(1, loaded["schema_version"])
|
||||
self.assertEqual(1, len(loaded["clips"]))
|
||||
clip = loaded["clips"][0]
|
||||
self.assertEqual("c1", clip["cid"])
|
||||
self.assertEqual([self.block], clip["blocks"])
|
||||
self.assertEqual(self.analysis, clip["analysis"])
|
||||
# The whole point: byte-identical values, including the integer frame keys
|
||||
# transit writes as "~i0". A JSON round trip that stringified them would
|
||||
# come back "0" and the part would hold its first pose forever.
|
||||
self.assertEqual(self.leaves(), clip["leaves"])
|
||||
|
||||
def test_an_unchanged_leaf_keeps_its_version(self):
|
||||
# What makes an entity tag worth having: a save where one channel moved
|
||||
# invalidates one leaf's etag, not the whole document's.
|
||||
self.save()
|
||||
first = {leaf.path: leaf.version for leaf in Leaf.objects.all()}
|
||||
moved = self.leaves()
|
||||
moved["clip/c1/timeline/main/channel/mouth-in/vis"] = [
|
||||
"^ ", "~:animated?", True, "~:keys", ["^ ", "~i0", False],
|
||||
]
|
||||
response = self.save(moved)
|
||||
self.assertEqual(["clip/c1/timeline/main/channel/mouth-in/vis"], response.json()["written"])
|
||||
self.assertEqual(4, response.json()["unchanged"])
|
||||
after = {leaf.path: leaf.version for leaf in Leaf.objects.all()}
|
||||
self.assertEqual(2, after["clip/c1/timeline/main/channel/mouth-in/vis"])
|
||||
self.assertEqual(first["clip/c1/timing"], after["clip/c1/timing"])
|
||||
|
||||
def test_a_removed_node_removes_its_leaf(self):
|
||||
self.save()
|
||||
fewer = {k: v for k, v in self.leaves().items()
|
||||
if k != "clip/c1/timeline/main/node/mouth"}
|
||||
response = self.save(fewer)
|
||||
self.assertEqual(["clip/c1/timeline/main/node/mouth"], response.json()["removed"])
|
||||
self.assertEqual(4, Leaf.objects.count())
|
||||
|
||||
def test_a_save_does_not_disturb_another_clip(self):
|
||||
# A save is not the only way the document changes, so a save that cleared
|
||||
# what it did not mention would undo a collaborator.
|
||||
Leaf.objects.create(project=self.project, path="clip/c2/timing", value=["^ "])
|
||||
self.save()
|
||||
self.assertTrue(Leaf.objects.filter(path="clip/c2/timing").exists())
|
||||
|
||||
def test_a_leaf_addressed_to_another_clip_is_refused(self):
|
||||
response = self.save({"clip/c9/timing": ["^ "]})
|
||||
self.assertEqual(400, response.status_code)
|
||||
self.assertIn("not addressed to clip", response.json()["error"])
|
||||
self.assertEqual(0, Leaf.objects.count())
|
||||
|
||||
def test_a_document_naming_blocks_the_server_lacks_is_refused(self):
|
||||
# Referential integrity across the tiers. Saved without this, the document
|
||||
# loads into a blank stage on any other machine.
|
||||
response = self.save(blocks=[self.block, "sha256:" + "a" * 64])
|
||||
self.assertEqual(409, response.status_code)
|
||||
self.assertEqual(["sha256:" + "a" * 64], response.json()["missing"])
|
||||
self.assertEqual(0, Leaf.objects.count())
|
||||
|
||||
def test_every_write_bumps_the_projects_version(self):
|
||||
before = Project.objects.get(id=self.project.id).seq
|
||||
self.save()
|
||||
self.assertEqual(before + 1, Project.objects.get(id=self.project.id).seq)
|
||||
|
||||
# --- the conditional write ---------------------------------------------
|
||||
|
||||
def test_a_leaf_write_carries_an_etag(self):
|
||||
self.save()
|
||||
url = f"/api/projects/{self.project.id}/leaves/clip/c1/timeline/main/node/mouth"
|
||||
got = self.client.get(url)
|
||||
self.assertEqual('"1"', got["ETag"])
|
||||
|
||||
ok = self.put(url, {"value": ["^ ", "~:id", "~:mouth", "~:z", "a2"]},
|
||||
HTTP_IF_MATCH='"1"')
|
||||
self.assertEqual(200, ok.status_code)
|
||||
self.assertEqual('"2"', ok["ETag"])
|
||||
self.assertEqual(["^ ", "~:id", "~:mouth", "~:z", "a2"],
|
||||
self.client.get(url).json()["value"])
|
||||
|
||||
def test_a_stale_write_is_refused_and_says_what_is_there(self):
|
||||
# 409 with the current value, so the client can offer keep-mine /
|
||||
# take-theirs. A PUT that replaced unconditionally is the bug where the
|
||||
# loser's work disappears silently.
|
||||
self.save()
|
||||
url = f"/api/projects/{self.project.id}/leaves/clip/c1/timeline/main/node/mouth"
|
||||
self.put(url, {"value": ["^ ", "~:z", "a2"]}, HTTP_IF_MATCH='"1"')
|
||||
stale = self.put(url, {"value": ["^ ", "~:z", "a3"]}, HTTP_IF_MATCH='"1"')
|
||||
self.assertEqual(409, stale.status_code)
|
||||
self.assertEqual(2, stale.json()["version"])
|
||||
self.assertEqual(["^ ", "~:z", "a2"], stale.json()["value"])
|
||||
# And the value on the server is the one that won, not the one refused.
|
||||
self.assertEqual(["^ ", "~:z", "a2"], self.client.get(url).json()["value"])
|
||||
|
||||
def test_an_unconditional_write_still_works(self):
|
||||
# Conditional writes are the protocol, not a requirement: the first write
|
||||
# of a leaf has no etag to match.
|
||||
url = f"/api/projects/{self.project.id}/leaves/clip/c1/stage"
|
||||
response = self.put(url, {"value": ["^ ", "~:width", 320]})
|
||||
self.assertEqual(200, response.status_code)
|
||||
self.assertEqual('"1"', response["ETag"])
|
||||
|
||||
def test_if_match_star_requires_the_leaf_to_exist(self):
|
||||
url = f"/api/projects/{self.project.id}/leaves/clip/c1/nothing"
|
||||
self.assertEqual(409, self.put(url, {"value": []}, HTTP_IF_MATCH="*").status_code)
|
||||
|
||||
# --- revisions ---------------------------------------------------------
|
||||
|
||||
def test_a_revision_snapshots_the_authored_layer(self):
|
||||
self.save()
|
||||
response = self.client.post(
|
||||
f"/api/projects/{self.project.id}/revisions",
|
||||
data=json.dumps({"summary": "first pass", "author": "olive"}),
|
||||
content_type="application/json",
|
||||
)
|
||||
self.assertEqual(201, response.status_code)
|
||||
revision = Revision.objects.get()
|
||||
self.assertEqual(5, len(revision.document))
|
||||
self.assertEqual(self.leaves(), revision.document)
|
||||
# Coarse on purpose: a save does not write one, because tier 1 will hold
|
||||
# cel polygons and a snapshot per save bloats the table.
|
||||
self.save()
|
||||
self.assertEqual(1, Revision.objects.count())
|
||||
|
||||
|
||||
@override_settings(BLOB_ROOT=BLOB_DIR)
|
||||
class FootageTests(TestCase):
|
||||
"""Tier 3, and the thing that makes the frames the backend's to serve: the
|
||||
manifest names every frame by URL."""
|
||||
|
||||
def bundle(self, frames=3, absence=None):
|
||||
root = Path(tempfile.mkdtemp(prefix="arthur-test-bundle-"))
|
||||
(root / "frames").mkdir()
|
||||
for i in range(frames):
|
||||
(root / "frames" / f"{i + 1:04d}.png").write_bytes(png(8, 6) + bytes([i]))
|
||||
(root / "audio.wav").write_bytes(b"RIFF....WAVEfmt ")
|
||||
manifest = {"fps": 12, "frames": frames, "dir": "frames",
|
||||
"audio": "audio.wav", "source": "IMG_8608.MOV"}
|
||||
if absence:
|
||||
manifest["feature-absence"] = absence
|
||||
(root / "manifest.json").write_text(json.dumps(manifest))
|
||||
return root
|
||||
|
||||
def ingest(self, root):
|
||||
from django.core.management import call_command
|
||||
from io import StringIO
|
||||
|
||||
call_command("ingest_bundle", str(root), stdout=StringIO())
|
||||
return Footage.objects.get()
|
||||
|
||||
def test_a_bundle_becomes_footage_with_a_url_per_frame(self):
|
||||
footage = self.ingest(self.bundle(frames=3, absence={"eye-r": [[1, 2]]}))
|
||||
self.assertEqual(3, footage.frames)
|
||||
self.assertEqual((8, 6), (footage.width, footage.height))
|
||||
self.assertEqual(12, footage.fps)
|
||||
self.assertEqual({"eye-r": [[1, 2]]}, footage.feature_absence)
|
||||
|
||||
manifest = self.client.get(f"/api/footage/{footage.id}").json()
|
||||
self.assertEqual(3, len(manifest["urls"]))
|
||||
self.assertTrue(all(url.startswith("/blob/") for url in manifest["urls"]))
|
||||
self.assertEqual(f"sha256:{footage.digest}", manifest["footage"])
|
||||
self.assertTrue(manifest["audio"].startswith("/blob/"))
|
||||
self.assertEqual({"eye-r": [[1, 2]]}, manifest["feature-absence"])
|
||||
# The frames are in order, and each one is fetchable.
|
||||
first = self.client.get(manifest["urls"][0])
|
||||
self.assertEqual(200, first.status_code)
|
||||
self.assertEqual("image/png", first["Content-Type"])
|
||||
|
||||
def test_ingesting_the_same_bundle_twice_is_one_footage(self):
|
||||
root = self.bundle()
|
||||
self.ingest(root)
|
||||
self.ingest(root)
|
||||
self.assertEqual(1, Footage.objects.count())
|
||||
|
||||
def test_a_bundle_whose_count_disagrees_with_its_frames_is_refused(self):
|
||||
from django.core.management import call_command
|
||||
from django.core.management.base import CommandError
|
||||
from io import StringIO
|
||||
|
||||
root = self.bundle(frames=3)
|
||||
(root / "frames" / "0003.png").unlink()
|
||||
with self.assertRaisesMessage(CommandError, "refusing an inaccurate footage"):
|
||||
call_command("ingest_bundle", str(root), stdout=StringIO())
|
||||
|
||||
def test_the_footage_list_does_not_carry_every_url(self):
|
||||
# A list of takes should not be a list of six hundred URLs each.
|
||||
self.ingest(self.bundle())
|
||||
listed = self.client.get("/api/footage").json()["footage"]
|
||||
self.assertEqual(1, len(listed))
|
||||
self.assertNotIn("urls", listed[0])
|
||||
|
||||
|
||||
class PageTests(TestCase):
|
||||
def test_django_serves_the_page_at_both_urls(self):
|
||||
for url in ("/", "/index.html"):
|
||||
response = self.client.get(url)
|
||||
self.assertEqual(200, response.status_code, url)
|
||||
body = response.content.decode()
|
||||
self.assertIn("/static/arthur/js/main.js", body)
|
||||
self.assertIn("/static/mediapipe/vision_bundle.js", body)
|
||||
self.assertIn('id="app"', body)
|
||||
# The token is rendered so Django sets its cookie, which is what the
|
||||
# save path reads to write the X-CSRFToken header.
|
||||
self.assertIn("csrfmiddlewaretoken", body)
|
||||
|
||||
def test_the_detector_reports_a_version_derived_from_the_model(self):
|
||||
# The version is the package version plus the model asset's own hash,
|
||||
# because a version string in the client is one somebody has to remember to
|
||||
# bump, and the server is the thing that serves the model.
|
||||
report = self.client.get("/api/detector").json()
|
||||
self.assertEqual("mediapipe", report["detector"])
|
||||
self.assertNotEqual("unknown", report["version"])
|
||||
self.assertTrue(report["model"].startswith("sha256:"))
|
||||
self.assertIn("+", report["version"])
|
||||
|
||||
|
||||
@skipUnless(shutil.which("ffmpeg") and shutil.which("ffprobe"), "ffmpeg is required")
|
||||
@override_settings(BLOB_ROOT=BLOB_DIR)
|
||||
class UploadTests(TestCase):
|
||||
def test_an_ffmpeg_stage_reports_live_progress_within_its_own_span(self):
|
||||
# The job's percentage is shared between the encode and the stills, so a
|
||||
# stage reports its own fraction of its own span rather than of the job.
|
||||
# Half of the frames through a stage that owns 0-55 is 27.
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
job = Mock(progress=0)
|
||||
|
||||
class FakeProcess:
|
||||
returncode = 0
|
||||
calls = 0
|
||||
|
||||
def poll(self):
|
||||
self.calls += 1
|
||||
if self.calls == 1:
|
||||
(root / "proxy.progress").write_text("frame=2\nprogress=continue\n")
|
||||
return None
|
||||
return 0
|
||||
|
||||
def wait(self):
|
||||
return 0
|
||||
|
||||
with patch("clips.extraction.subprocess.Popen", return_value=FakeProcess()), \
|
||||
patch("clips.extraction.time.sleep"):
|
||||
extraction._run_with_progress(job, ["-i", "in.mp4", "out.mp4"],
|
||||
root, "proxy", 4, (0, 55))
|
||||
self.assertEqual(27, job.progress)
|
||||
job.save.assert_called_once_with(update_fields=["progress", "updated"])
|
||||
|
||||
def test_uploaded_video_extracts_to_reopenable_footage(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "four-frames.mp4"
|
||||
subprocess.run([
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-f", "lavfi", "-i", "color=c=red:s=64x48:r=4:d=1",
|
||||
"-c:v", "mpeg4", str(path),
|
||||
], check=True, capture_output=True)
|
||||
payload = path.read_bytes()
|
||||
|
||||
uploaded = self.client.post("/api/sources", {
|
||||
"file": SimpleUploadedFile("four-frames.mp4", payload, content_type="video/mp4")})
|
||||
self.assertEqual(201, uploaded.status_code, uploaded.content)
|
||||
source_id = uploaded.json()["id"]
|
||||
self.assertEqual(4, uploaded.json()["probe"]["reported_frames"])
|
||||
self.assertEqual(1, Source.objects.count())
|
||||
again = self.client.post("/api/sources", {
|
||||
"file": SimpleUploadedFile("same-video.mp4", payload, content_type="video/mp4")})
|
||||
self.assertEqual(200, again.status_code, again.content)
|
||||
self.assertEqual(source_id, again.json()["id"])
|
||||
|
||||
with patch("clips.extraction.enqueue", side_effect=extraction.run):
|
||||
queued = self.client.post("/api/extractions", json.dumps({
|
||||
"source": source_id, "settings": {},
|
||||
}), content_type="application/json")
|
||||
self.assertIn(queued.status_code, (200, 202), queued.content)
|
||||
job = self.client.get(f"/api/extractions/{queued.json()['key']}").json()
|
||||
self.assertEqual("done", job["state"], job)
|
||||
footage = self.client.get(f"/api/footage/{job['footage']}").json()
|
||||
self.assertEqual((4, 64, 48), (footage["frames"], footage["width"], footage["height"]))
|
||||
|
||||
# THE PROXY IS THE ANALYSIS SOURCE. The page seeks this URL frame by
|
||||
# frame, so it has to exist, be a video, and answer a Range request —
|
||||
# without the last of those a media element cannot seek it at all.
|
||||
self.assertTrue(footage["video"].startswith("/blob/"), footage)
|
||||
proxy = self.client.get(footage["video"])
|
||||
self.assertEqual(200, proxy.status_code)
|
||||
self.assertEqual("video/mp4", proxy["Content-Type"])
|
||||
self.assertEqual("bytes", proxy["Accept-Ranges"])
|
||||
self.assertEqual(206, self.client.get(footage["video"],
|
||||
headers={"range": "bytes=0-31"}).status_code)
|
||||
|
||||
# And the stills beside it are JPEGs for tracing, one per frame.
|
||||
self.assertEqual(4, len(footage["urls"]))
|
||||
still = self.client.get(footage["urls"][0])
|
||||
self.assertEqual(200, still.status_code)
|
||||
self.assertEqual("image/jpeg", still["Content-Type"])
|
||||
self.assertEqual(200, self.client.get(footage["audio"]).status_code)
|
||||
|
||||
def test_the_proxy_is_re_encoded_rather_than_the_upload_re_served(self):
|
||||
# The footage's identity is the proxy's digest, and the proxy is produced
|
||||
# by one ffmpeg invocation whatever the upload was. If the upload were
|
||||
# passed through when it happened to be playable, identity would depend on
|
||||
# which branch ran — and an HEVC upload would reach a browser that cannot
|
||||
# decode it.
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "already-h264.mp4"
|
||||
subprocess.run([
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-f", "lavfi", "-i", "testsrc=s=64x48:r=4:d=1",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p", str(path),
|
||||
], check=True, capture_output=True)
|
||||
payload = path.read_bytes()
|
||||
|
||||
uploaded = self.client.post("/api/sources", {
|
||||
"file": SimpleUploadedFile("already-h264.mp4", payload, content_type="video/mp4")})
|
||||
with patch("clips.extraction.enqueue", side_effect=extraction.run):
|
||||
queued = self.client.post("/api/extractions", json.dumps({
|
||||
"source": uploaded.json()["id"], "settings": {},
|
||||
}), content_type="application/json")
|
||||
job = self.client.get(f"/api/extractions/{queued.json()['key']}").json()
|
||||
self.assertEqual("done", job["state"], job)
|
||||
|
||||
footage = Footage.objects.get(id=job["footage"])
|
||||
self.assertIsNotNone(footage.video)
|
||||
self.assertNotEqual(Source.objects.get(id=uploaded.json()["id"]).blob_id,
|
||||
footage.video_id)
|
||||
|
||||
def test_a_container_whose_metadata_disagrees_with_itself_is_not_refused(self):
|
||||
# THE REGRESSION. Ordinary iPhone footage, shot straight from the camera
|
||||
# app, reports avg_frame_rate 8670/299 and nb_frames 289 on a stream whose
|
||||
# decoded timestamps are 280 frames exactly 1/30s apart. Refusing that as
|
||||
# "variable-frame-rate" rejected CFR video on the strength of a summary the
|
||||
# container got wrong about its own contents. Nothing measures the source
|
||||
# any more, so the rate is a choice rather than a fact to be verified.
|
||||
report = json.dumps({"streams": [
|
||||
{"codec_type": "video", "r_frame_rate": "30/1", "avg_frame_rate": "8670/299",
|
||||
"nb_frames": "289", "width": 1920, "height": 1440},
|
||||
{"codec_type": "audio"}],
|
||||
"format": {"duration": "9.316667"}})
|
||||
with patch("clips.extraction._command", return_value=report):
|
||||
facts = extraction.probe(Path("phone.mov"))
|
||||
self.assertEqual(30.0, facts["fps"])
|
||||
self.assertTrue(facts["vfr"], "the disagreement is still recorded, just not fatal")
|
||||
self.assertTrue(facts["has_audio"])
|
||||
|
||||
def test_the_proxy_rate_is_exact_rather_than_a_rounded_float(self):
|
||||
# 30000/1001 is not a float. Handing ffmpeg's -r a rounded one is how a
|
||||
# long take drifts out of sync with its own audio.
|
||||
report = json.dumps({"streams": [
|
||||
{"codec_type": "video", "r_frame_rate": "30000/1001",
|
||||
"avg_frame_rate": "30000/1001", "width": 640, "height": 480}],
|
||||
"format": {"duration": "10"}})
|
||||
with patch("clips.extraction._command", return_value=report):
|
||||
facts = extraction.probe(Path("ntsc.mov"))
|
||||
self.assertEqual("30000/1001", facts["rate"])
|
||||
|
||||
def test_a_rate_no_footage_could_have_been_shot_at_is_refused(self):
|
||||
report = json.dumps({"streams": [
|
||||
{"codec_type": "video", "r_frame_rate": "1000/1", "avg_frame_rate": "900/1",
|
||||
"width": 640, "height": 480}],
|
||||
"format": {"duration": "10"}})
|
||||
with patch("clips.extraction._command", return_value=report):
|
||||
with self.assertRaisesMessage(ValueError, "not a rate footage can be measured at"):
|
||||
extraction.probe(Path("nonsense.mov"))
|
||||
|
||||
def test_variable_frame_rate_video_extracts_to_constant_rate_footage(self):
|
||||
# End to end on a genuinely variable file: irregular timestamps in, one
|
||||
# constant-rate proxy out, and the DURATION preserved — which is the thing
|
||||
# that must not move, because the page's clock is the audio.
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
held = root / "held.mp4"
|
||||
wobbly = root / "wobbly.mp4"
|
||||
subprocess.run([
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-f", "lavfi", "-i", "testsrc=s=64x48:r=5:d=2", "-r", "30",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p", str(held)],
|
||||
check=True, capture_output=True)
|
||||
subprocess.run([
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i", str(held),
|
||||
"-vf", "mpdecimate", "-fps_mode", "vfr",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p", str(wobbly)],
|
||||
check=True, capture_output=True)
|
||||
facts = extraction.probe(wobbly)
|
||||
self.assertTrue(facts["vfr"], "the fixture is not actually variable")
|
||||
payload = wobbly.read_bytes()
|
||||
|
||||
uploaded = self.client.post("/api/sources", {
|
||||
"file": SimpleUploadedFile("wobbly.mp4", payload, content_type="video/mp4")})
|
||||
self.assertEqual(201, uploaded.status_code, uploaded.content)
|
||||
with patch("clips.extraction.enqueue", side_effect=extraction.run):
|
||||
queued = self.client.post("/api/extractions", json.dumps({
|
||||
"source": uploaded.json()["id"], "settings": {},
|
||||
}), content_type="application/json")
|
||||
job = self.client.get(f"/api/extractions/{queued.json()['key']}").json()
|
||||
self.assertEqual("done", job["state"], job)
|
||||
|
||||
footage = Footage.objects.get(id=job["footage"])
|
||||
self.assertEqual(facts["fps"], footage.fps)
|
||||
self.assertAlmostEqual(facts["duration"], footage.frames / footage.fps, delta=0.5)
|
||||
self.assertEqual(footage.frames, footage.frame_set.count())
|
||||
|
||||
def test_footage_without_a_proxy_says_so_rather_than_serving_nothing(self):
|
||||
# Footage ingested before the proxy existed. The manifest reports a null
|
||||
# video so the loader can name the fix; it does not omit the field and let
|
||||
# the client discover it somewhere inside MediaPipe.
|
||||
audio, size = blobs.write(b"RIFF....WAVEfmt ")
|
||||
blob = Blob.objects.create(digest=audio, size=size, media_type="audio/wav")
|
||||
footage = Footage.objects.create(
|
||||
digest="e" * 64, fps=12, frames=3, width=8, height=6, audio=blob)
|
||||
manifest = self.client.get(f"/api/footage/{footage.id}").json()
|
||||
self.assertIsNone(manifest["video"])
|
||||
34
clips/urls.py
Normal file
34
clips/urls.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
"""The API, which is nine endpoints and no framework.
|
||||
|
||||
The shape is RFC 7232 over addressed resources: a leaf is a resource, its version
|
||||
is an entity tag, and a conditional write answers 409. docs/architecture.md is
|
||||
explicit that this part is not a bespoke invention — "optimistic concurrency
|
||||
control over addressed resources with an entity tag" is what HTTP has done for
|
||||
thirty years — so the plumbing here is deliberately boring.
|
||||
|
||||
WRITES ARE ON HTTP AND STAY THERE. When the websocket arrives it carries presence
|
||||
and broadcasts, and not writes: auth, idempotency, status codes, retries and
|
||||
conditional requests all come for free here, and a dropped socket cannot lose a
|
||||
write.
|
||||
"""
|
||||
from django.urls import path
|
||||
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
path("detector", views.detector),
|
||||
path("sources", views.sources),
|
||||
path("extractions", views.extractions),
|
||||
path("extractions/<str:key>", views.extraction_detail),
|
||||
path("footage", views.footage_list),
|
||||
path("footage/<uuid:footage_id>", views.footage_detail),
|
||||
path("projects", views.projects),
|
||||
path("projects/<uuid:project_id>", views.project_detail),
|
||||
path("projects/<uuid:project_id>/leaves/<path:leaf_path>", views.leaf_detail),
|
||||
path("projects/<uuid:project_id>/revisions", views.revisions),
|
||||
path("analyses", views.analyses),
|
||||
path("analyses/<str:key>", views.analysis_detail),
|
||||
path("blocks", views.blocks),
|
||||
path("blocks/missing", views.blocks_missing),
|
||||
path("blocks/<str:key>", views.block_detail),
|
||||
]
|
||||
804
clips/views.py
Normal file
804
clips/views.py
Normal file
|
|
@ -0,0 +1,804 @@
|
|||
"""The API's implementation.
|
||||
|
||||
Two things in here are load-bearing and neither is Django.
|
||||
|
||||
THE SERVER VERIFIES EVERY TIER-2 KEY IT IS HANDED. A key is the sha256 of a
|
||||
canonical descriptor, and this recomputes it and refuses a mismatch. That is what
|
||||
makes content addressing a property of the system rather than a convention in the
|
||||
client: nothing can store bytes under a name that does not describe them.
|
||||
|
||||
It hashes THE TEXT IT WAS SENT rather than re-rendering the descriptor from parsed
|
||||
values, and that is the honest arrangement rather than a shortcut. JS prints an
|
||||
integral double as `1` and Python prints `1.0`, so a scheme where both sides
|
||||
re-render the numbers would disagree on the first parameter whose value happens to
|
||||
be whole — and the failure would be an upload that 409s with nothing wrong. The
|
||||
bytes are the contract; the schema on top of them is a convention, and the two
|
||||
fields this file actually reads out of that schema are checked separately.
|
||||
|
||||
AND IT REFUSES A BLOCK WHOSE ANALYSIS IT DOES NOT KNOW. Every block descriptor
|
||||
names an analysis, and every analysis declares a detector and a VERSION. So the
|
||||
chain from a stored block to the model version that produced it cannot be broken
|
||||
by a client that forgot a step — which is the whole point of
|
||||
docs/architecture.md's insistence that the cache key include the detector version.
|
||||
A model upgrade that silently reused old landmarks would otherwise present as "the
|
||||
tool got worse", with no event to attach it to.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import zlib
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.db import transaction
|
||||
from django.http import FileResponse, HttpResponse, JsonResponse
|
||||
from django.shortcuts import render
|
||||
from django.views.decorators.http import require_http_methods
|
||||
|
||||
from . import blobs, extraction
|
||||
from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Leaf, Project, Revision, Source
|
||||
|
||||
KEY_LENGTH = 71 # "sha256:" + 64 hex
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# helpers
|
||||
|
||||
|
||||
def _body(request):
|
||||
try:
|
||||
return json.loads(request.body or b"{}")
|
||||
except json.JSONDecodeError as exc:
|
||||
raise Bad(f"the request body is not JSON: {exc}") from exc
|
||||
|
||||
|
||||
class Bad(Exception):
|
||||
"""A 400 with a message, raised where the problem is noticed."""
|
||||
|
||||
def __init__(self, message, status=400, **detail):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.status = status
|
||||
self.detail = detail
|
||||
|
||||
|
||||
def _error(exc: Bad):
|
||||
return JsonResponse({"error": exc.message, **exc.detail}, status=exc.status)
|
||||
|
||||
|
||||
def _check_key(key, descriptor):
|
||||
"""The verification. A key is the sha256 of the descriptor stored beside it."""
|
||||
if not isinstance(key, str) or len(key) != KEY_LENGTH or not key.startswith("sha256:"):
|
||||
raise Bad(f"not a content address: {key!r}")
|
||||
if not isinstance(descriptor, str) or not descriptor:
|
||||
raise Bad("a key without its descriptor addresses nothing")
|
||||
actual = hashlib.sha256(descriptor.encode("utf-8")).hexdigest()
|
||||
if actual != key[7:]:
|
||||
raise Bad(
|
||||
"the key is not the hash of its descriptor",
|
||||
status=409,
|
||||
expected=f"sha256:{actual}",
|
||||
given=key,
|
||||
)
|
||||
try:
|
||||
return json.loads(descriptor)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise Bad(f"the descriptor is not canonical JSON: {exc}") from exc
|
||||
|
||||
|
||||
def _blob(b64, media_type="application/octet-stream"):
|
||||
import base64
|
||||
|
||||
digest, size = blobs.write(base64.b64decode(b64))
|
||||
blob, _ = Blob.objects.get_or_create(
|
||||
digest=digest, defaults={"size": size, "media_type": media_type}
|
||||
)
|
||||
return blob
|
||||
|
||||
|
||||
def _uploaded_blob(upload, media_type="application/octet-stream"):
|
||||
digest, size = blobs.write_stream(upload.chunks())
|
||||
blob, _ = Blob.objects.get_or_create(
|
||||
digest=digest, defaults={"size": size, "media_type": media_type}
|
||||
)
|
||||
return blob
|
||||
|
||||
|
||||
def _crop_blob(chunks):
|
||||
digest, size = blobs.write_compressed_stream(chunks)
|
||||
blob, _ = Blob.objects.get_or_create(
|
||||
digest=digest, defaults={"size": size, "media_type": blobs.CROP_MEDIA_TYPE}
|
||||
)
|
||||
return blob
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# the page
|
||||
|
||||
|
||||
def page(request):
|
||||
"""The host page. This replaced `frontend/public/index.html` at step 9, and
|
||||
`:dev-http` in shadow-cljs.edn went away with it."""
|
||||
return render(request, "clips/index.html")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# the detector
|
||||
#
|
||||
# WHY THE SERVER ANSWERS THIS. The analysis key has to include the detector
|
||||
# version, and a version string in the client is a string somebody has to remember
|
||||
# to bump. The server serves the model, so it can hash the model — and then the
|
||||
# version is a fact about the bytes that produced the landmarks rather than a
|
||||
# claim about them.
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def _model_digest(path: str, mtime: float) -> str:
|
||||
return blobs.digest_file(Path(path))
|
||||
|
||||
|
||||
def _package_version() -> str:
|
||||
pkg = Path(settings.BASE_DIR) / "frontend" / "package.json"
|
||||
try:
|
||||
deps = json.loads(pkg.read_text())["dependencies"]
|
||||
return deps["@mediapipe/tasks-vision"].lstrip("^~")
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
@require_http_methods(["GET"])
|
||||
def detector(request):
|
||||
model = Path(settings.BASE_DIR) / "frontend" / "public" / "mediapipe" / "face_landmarker.task"
|
||||
if not model.exists():
|
||||
# Honest rather than fatal: detection will fail at the MediaPipe boundary
|
||||
# with a better message than this one could give, and an analysis stamped
|
||||
# "unknown" is a take somebody can still look at and re-freeze later.
|
||||
return JsonResponse({"detector": "mediapipe", "version": "unknown", "model": None})
|
||||
digest = _model_digest(str(model), model.stat().st_mtime)
|
||||
return JsonResponse(
|
||||
{
|
||||
"detector": "mediapipe",
|
||||
# The package version AND the model's own hash. Either alone can change
|
||||
# while the other does not, and both change the landmarks.
|
||||
"version": f"{_package_version()}+{digest[:16]}",
|
||||
"model": f"sha256:{digest}",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tier 3: footage
|
||||
#
|
||||
# THE MANIFEST NOW CARRIES URLS. It used to carry a directory and the loader built
|
||||
# `frames/0001.png` itself, which quietly made the frame layout a shared secret
|
||||
# between a shell script and a ClojureScript namespace. The server names every
|
||||
# frame instead, so uploaded video and command-line bundles produce the same
|
||||
# footage response without the client knowing where either stored its frames.
|
||||
|
||||
|
||||
@require_http_methods(["GET", "POST"])
|
||||
def sources(request):
|
||||
if request.method == "GET":
|
||||
return JsonResponse({"sources": [
|
||||
{"id": str(row.id), "filename": row.filename, "probe": row.probe}
|
||||
for row in Source.objects.order_by("-created")[:100]
|
||||
]})
|
||||
upload = request.FILES.get("file")
|
||||
if upload is None:
|
||||
return JsonResponse({"error": "upload a video as the file field"}, status=400)
|
||||
try:
|
||||
digest, size = blobs.write_stream(upload.chunks())
|
||||
facts = extraction.probe(blobs.path_for(digest))
|
||||
blob, _ = Blob.objects.get_or_create(
|
||||
digest=digest, defaults={"size": size,
|
||||
"media_type": upload.content_type or "video/mp4"})
|
||||
row, created = Source.objects.get_or_create(
|
||||
blob=blob, defaults={"filename": Path(upload.name).name[:255], "probe": facts})
|
||||
return JsonResponse({"id": str(row.id), "digest": digest,
|
||||
"filename": row.filename, "probe": row.probe,
|
||||
"created": created}, status=201 if created else 200)
|
||||
except (ValueError, OSError) as exc:
|
||||
return JsonResponse({"error": str(exc)}, status=400)
|
||||
|
||||
|
||||
def _extraction_json(row):
|
||||
return {"key": row.key, "source": str(row.source_id), "state": row.state,
|
||||
"progress": row.progress, "error": row.error,
|
||||
"footage": str(row.footage_id) if row.footage_id else None}
|
||||
|
||||
|
||||
@require_http_methods(["POST"])
|
||||
def extractions(request):
|
||||
try:
|
||||
data = _body(request)
|
||||
source_id = data.get("source")
|
||||
if not source_id:
|
||||
raise Bad("an extraction needs a source id")
|
||||
try:
|
||||
source = Source.objects.get(id=UUID(str(source_id)))
|
||||
except (ValueError, ValidationError, Source.DoesNotExist):
|
||||
raise Bad("no such source", status=404)
|
||||
settings = data.get("settings") or {}
|
||||
if settings != {}:
|
||||
raise Bad("extraction currently keeps the source frame rate; settings must be empty")
|
||||
key = extraction.extraction_key(source, settings)
|
||||
row, _ = Extraction.objects.get_or_create(
|
||||
key=key, defaults={"source": source, "settings": settings})
|
||||
if row.state != "done":
|
||||
extraction.enqueue(key)
|
||||
return JsonResponse(_extraction_json(row), status=202 if row.state != "done" else 200)
|
||||
except Bad as exc:
|
||||
return _error(exc)
|
||||
|
||||
|
||||
@require_http_methods(["GET"])
|
||||
def extraction_detail(request, key):
|
||||
try:
|
||||
return JsonResponse(_extraction_json(Extraction.objects.get(key=key)))
|
||||
except Extraction.DoesNotExist:
|
||||
return JsonResponse({"error": "no such extraction"}, status=404)
|
||||
|
||||
|
||||
def _footage_json(footage: Footage, urls=True):
|
||||
out = {
|
||||
"id": str(footage.id),
|
||||
"label": footage.label or footage.source,
|
||||
"source": footage.source,
|
||||
"fps": footage.fps,
|
||||
"frames": footage.frames,
|
||||
"width": footage.width,
|
||||
"height": footage.height,
|
||||
"footage": f"sha256:{footage.digest}",
|
||||
"audio": f"/blob/{footage.audio.digest}",
|
||||
# The analysis source. `null` on footage ingested before the proxy
|
||||
# existed, which the loader reports as "re-extract this" rather than
|
||||
# failing somewhere inside MediaPipe.
|
||||
"video": f"/blob/{footage.video.digest}" if footage.video_id else None,
|
||||
# What the page actually decodes: one access unit per frame, no container.
|
||||
"stream": f"/blob/{footage.stream.digest}" if footage.stream_id else None,
|
||||
"feature-absence": footage.feature_absence or {},
|
||||
}
|
||||
if urls:
|
||||
out["urls"] = [f"/blob/{f.blob.digest}" for f in footage.frame_set.select_related("blob")]
|
||||
return out
|
||||
|
||||
|
||||
@require_http_methods(["GET"])
|
||||
def footage_list(request):
|
||||
return JsonResponse(
|
||||
{"footage": [_footage_json(f, urls=False) for f in Footage.objects.all()]}
|
||||
)
|
||||
|
||||
|
||||
@require_http_methods(["GET"])
|
||||
def footage_detail(request, footage_id):
|
||||
try:
|
||||
footage = Footage.objects.select_related("audio", "video", "stream").get(id=footage_id)
|
||||
except Footage.DoesNotExist:
|
||||
return JsonResponse({"error": "no such footage"}, status=404)
|
||||
return JsonResponse(_footage_json(footage))
|
||||
|
||||
|
||||
_RANGE = re.compile(r"^bytes=(\d*)-(\d*)$")
|
||||
|
||||
|
||||
class _Slice:
|
||||
"""A file, readable only up to `remaining` bytes from where it was seeked."""
|
||||
|
||||
def __init__(self, handle, remaining):
|
||||
self.handle, self.remaining = handle, remaining
|
||||
|
||||
def read(self, size=-1):
|
||||
if self.remaining <= 0:
|
||||
return b""
|
||||
if size < 0 or size > self.remaining:
|
||||
size = self.remaining
|
||||
data = self.handle.read(size)
|
||||
self.remaining -= len(data)
|
||||
return data
|
||||
|
||||
def close(self):
|
||||
self.handle.close()
|
||||
|
||||
|
||||
def _byte_range(header, size):
|
||||
"""One `Range` header -> (start, end) inclusive, or None for the whole blob.
|
||||
|
||||
A syntactically broken header is NOT an error: RFC 9110 says an unparsable
|
||||
Range is ignored and the whole representation is sent, which is what a client
|
||||
that meant nothing by it wants. `False` is the third answer — a range that
|
||||
parses and cannot be satisfied — because that one is a 416.
|
||||
"""
|
||||
if not header:
|
||||
return None
|
||||
match = _RANGE.match(header.strip())
|
||||
if not match or match.group(1) == "" and match.group(2) == "":
|
||||
return None
|
||||
first, last = match.group(1), match.group(2)
|
||||
if first == "":
|
||||
# `bytes=-500`: the LAST 500 bytes, which is a different question.
|
||||
length = int(last)
|
||||
if length == 0:
|
||||
return False
|
||||
return (max(0, size - length), size - 1)
|
||||
start = int(first)
|
||||
end = int(last) if last else size - 1
|
||||
end = min(end, size - 1)
|
||||
if start >= size or start > end:
|
||||
return False
|
||||
return (start, end)
|
||||
|
||||
|
||||
@require_http_methods(["GET"])
|
||||
def blob(request, digest):
|
||||
"""Raw bytes, immutable, and serveable a slice at a time.
|
||||
|
||||
`immutable` is not optimism here, it is the definition: the name IS the hash of
|
||||
the content, so a cached copy cannot be stale. That is what makes serving a
|
||||
take's frames out of this cheap enough to do on every load.
|
||||
|
||||
RANGE IS NOT AN OPTIMISATION HERE, IT IS THE FEATURE. Since the analysis source
|
||||
became a video file, a `<video>` element seeks this URL, and a media element
|
||||
that is handed 200OK with no `Accept-Ranges` cannot seek: it reports an empty
|
||||
`seekable` range, every `currentTime` write is a no-op, and detection then runs
|
||||
ninety times over frame one without anything raising. Django's `FileResponse`
|
||||
does not do this for us — there is no Range handling anywhere in it — so the
|
||||
absence of these thirty lines presents as "MediaPipe's video mode is broken".
|
||||
"""
|
||||
try:
|
||||
row = Blob.objects.get(digest=digest)
|
||||
path = blobs.path_for(digest)
|
||||
except (Blob.DoesNotExist, ValueError):
|
||||
return JsonResponse({"error": "no such blob"}, status=404)
|
||||
|
||||
size = path.stat().st_size
|
||||
span = _byte_range(request.headers.get("Range"), size)
|
||||
if span is False:
|
||||
response = HttpResponse(status=416)
|
||||
response["Content-Range"] = f"bytes */{size}"
|
||||
elif span is None:
|
||||
response = FileResponse(open(path, "rb"), content_type=row.media_type)
|
||||
else:
|
||||
start, end = span
|
||||
handle = open(path, "rb")
|
||||
handle.seek(start)
|
||||
response = FileResponse(_Slice(handle, end - start + 1),
|
||||
status=206, content_type=row.media_type)
|
||||
response["Content-Range"] = f"bytes {start}-{end}/{size}"
|
||||
response["Content-Length"] = str(end - start + 1)
|
||||
response["Accept-Ranges"] = "bytes"
|
||||
response["Cache-Control"] = "public, max-age=31536000, immutable"
|
||||
response["ETag"] = f'"{digest}"'
|
||||
return response
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tier 2: analyses and blocks
|
||||
|
||||
|
||||
@require_http_methods(["POST"])
|
||||
def analyses(request):
|
||||
"""Register an analysis artifact's identity. Idempotent: the same inputs are
|
||||
the same key are the same row."""
|
||||
try:
|
||||
data = _body(request)
|
||||
key = data.get("key")
|
||||
descriptor = data.get("descriptor")
|
||||
parsed = _check_key(key, descriptor)
|
||||
for field in ("detector", "version"):
|
||||
if not parsed.get(field):
|
||||
raise Bad(
|
||||
f"the descriptor does not declare a {field}: a cache key that "
|
||||
"omits the detector version lets a model upgrade silently reuse "
|
||||
"old landmarks",
|
||||
missing=field,
|
||||
)
|
||||
footage = None
|
||||
if data.get("footage"):
|
||||
digest = str(data["footage"]).removeprefix("sha256:")
|
||||
footage = Footage.objects.filter(digest=digest).first()
|
||||
if footage is None:
|
||||
raise Bad("the analysis names footage this server does not have",
|
||||
footage=data["footage"])
|
||||
row, created = Analysis.objects.get_or_create(
|
||||
key=key,
|
||||
defaults={
|
||||
"descriptor": descriptor,
|
||||
"detector": parsed["detector"],
|
||||
"version": str(parsed["version"]),
|
||||
"footage": footage,
|
||||
},
|
||||
)
|
||||
return JsonResponse({"key": row.key, "created": created}, status=201 if created else 200)
|
||||
except Bad as exc:
|
||||
return _error(exc)
|
||||
|
||||
|
||||
@require_http_methods(["GET", "PUT"])
|
||||
def analysis_detail(request, key):
|
||||
try:
|
||||
row = Analysis.objects.get(key=key)
|
||||
except Analysis.DoesNotExist:
|
||||
return JsonResponse({"error": "no such analysis"}, status=404)
|
||||
if request.method == "GET":
|
||||
return JsonResponse({
|
||||
"key": row.key, "descriptor": row.descriptor,
|
||||
"detector": row.detector, "version": row.version,
|
||||
"footage": str(row.footage_id) if row.footage_id else None,
|
||||
"source_blocks": sorted(row.source_blocks.values_list("key", flat=True)),
|
||||
})
|
||||
try:
|
||||
keys = _body(request).get("source_blocks")
|
||||
roles = {"source/dense", "source/detected", "source/crops"}
|
||||
if (not isinstance(keys, list) or not keys
|
||||
or not all(isinstance(k, str) for k in keys) or len(set(keys)) != len(keys)):
|
||||
raise Bad("an analysis needs distinct source block keys")
|
||||
blocks = list(Block.objects.filter(key__in=keys))
|
||||
if len(blocks) != len(keys) or any(b.analysis_id != key for b in blocks):
|
||||
raise Bad("source blocks must exist and name this analysis")
|
||||
by_subject = {}
|
||||
for block in blocks:
|
||||
subjects = json.loads(block.descriptor).get("features", [])
|
||||
if (not isinstance(subjects, list) or len(subjects) > 1
|
||||
or any(not isinstance(s, str) or not s for s in subjects)):
|
||||
raise Bad("a source block must name one subject")
|
||||
# Older single-face analyses used an empty feature list.
|
||||
by_subject.setdefault(tuple(subjects), []).append(block.role)
|
||||
if any(len(found) != len(roles) or set(found) != roles
|
||||
for found in by_subject.values()):
|
||||
raise Bad("each subject needs one block for each source role")
|
||||
with transaction.atomic():
|
||||
row = Analysis.objects.select_for_update().get(key=key)
|
||||
current = set(row.source_blocks.values_list("key", flat=True))
|
||||
if current and current != set(keys):
|
||||
raise Bad("the source blocks of an analysis are immutable", status=409)
|
||||
row.source_blocks.set(blocks)
|
||||
return JsonResponse({"key": key, "source_blocks": sorted(keys)})
|
||||
except Bad as exc:
|
||||
return _error(exc)
|
||||
|
||||
|
||||
@require_http_methods(["POST"])
|
||||
def blocks_missing(request):
|
||||
"""Which of these keys the server does not have.
|
||||
|
||||
The return on content addressing, as one request: a save uploads the blocks
|
||||
that are new and nothing else, so re-saving a document after a knob-free edit
|
||||
moves kilobytes.
|
||||
"""
|
||||
try:
|
||||
keys = _body(request).get("keys") or []
|
||||
if not isinstance(keys, list):
|
||||
raise Bad("keys must be a list")
|
||||
have = set(Block.objects.filter(key__in=keys).values_list("key", flat=True))
|
||||
return JsonResponse({"missing": [k for k in keys if k not in have]})
|
||||
except Bad as exc:
|
||||
return _error(exc)
|
||||
|
||||
|
||||
@require_http_methods(["POST"])
|
||||
def blocks(request):
|
||||
"""Store one dense block: its bytes, its optional absence mask, and the
|
||||
descriptor its key is the hash of."""
|
||||
try:
|
||||
multipart = request.content_type == "multipart/form-data"
|
||||
data = request.POST if multipart else _body(request)
|
||||
upload = request.FILES.get("data") if multipart else None
|
||||
state_upload = request.FILES.get("state") if multipart else None
|
||||
key = data.get("key")
|
||||
descriptor = data.get("descriptor")
|
||||
parsed = _check_key(key, descriptor)
|
||||
role = parsed.get("role")
|
||||
if not role:
|
||||
raise Bad("a block's descriptor names its role")
|
||||
if not parsed.get("layout", {}).get("type"):
|
||||
raise Bad(
|
||||
"a block's descriptor must say what its elements are: an Int16Array "
|
||||
"and a Float32Array over the same bytes are both valid readings and "
|
||||
"only one of them is the block"
|
||||
)
|
||||
analysis_key = parsed.get("analysis")
|
||||
analysis = Analysis.objects.filter(key=analysis_key).first()
|
||||
if analysis is None:
|
||||
raise Bad(
|
||||
"this block names an analysis the server does not know; register the "
|
||||
"analysis first, so that every stored block can name the detector "
|
||||
"version that produced it",
|
||||
analysis=analysis_key,
|
||||
)
|
||||
if not (upload and upload.size) and not data.get("data"):
|
||||
raise Bad("a block with no bytes")
|
||||
with transaction.atomic():
|
||||
if role == "source/crops":
|
||||
if upload:
|
||||
data_blob = _crop_blob(upload.chunks())
|
||||
else:
|
||||
import base64
|
||||
|
||||
data_blob = _crop_blob([base64.b64decode(data["data"])])
|
||||
else:
|
||||
data_blob = _uploaded_blob(upload) if upload else _blob(data["data"])
|
||||
row, created = Block.objects.get_or_create(
|
||||
key=key,
|
||||
defaults={
|
||||
"descriptor": descriptor,
|
||||
"role": role,
|
||||
"analysis": analysis,
|
||||
"data": data_blob,
|
||||
"state": (_uploaded_blob(state_upload) if state_upload else
|
||||
_blob(data["state"]) if data.get("state") else None),
|
||||
},
|
||||
)
|
||||
return JsonResponse({"key": row.key, "created": created}, status=201 if created else 200)
|
||||
except Bad as exc:
|
||||
return _error(exc)
|
||||
|
||||
|
||||
@require_http_methods(["GET"])
|
||||
def block_detail(request, key):
|
||||
import base64
|
||||
|
||||
try:
|
||||
row = Block.objects.select_related("data", "state").get(key=key)
|
||||
except Block.DoesNotExist:
|
||||
return JsonResponse({"error": "no such block"}, status=404)
|
||||
data = blobs.read(row.data.digest)
|
||||
if row.data.media_type == blobs.CROP_MEDIA_TYPE:
|
||||
data = zlib.decompress(data)
|
||||
out = {
|
||||
"key": row.key,
|
||||
"descriptor": row.descriptor,
|
||||
"data": base64.b64encode(data).decode("ascii"),
|
||||
}
|
||||
if row.state_id:
|
||||
out["state"] = base64.b64encode(blobs.read(row.state.digest)).decode("ascii")
|
||||
response = JsonResponse(out)
|
||||
response["Cache-Control"] = "public, max-age=31536000, immutable"
|
||||
return response
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tier 1: projects, clips, leaves
|
||||
|
||||
|
||||
def _project_json(project: Project):
|
||||
leaves = list(project.leaves.all())
|
||||
clips = []
|
||||
for clip in project.clips.all():
|
||||
prefix = f"clip/{clip.cid}/"
|
||||
clips.append(
|
||||
{
|
||||
"cid": clip.cid,
|
||||
"name": clip.name,
|
||||
"footage": str(clip.footage_id) if clip.footage_id else None,
|
||||
"analysis": clip.analysis_id,
|
||||
"blocks": sorted(clip.blocks.values_list("key", flat=True)),
|
||||
"leaves": {leaf.path: leaf.value for leaf in leaves if leaf.path.startswith(prefix)},
|
||||
}
|
||||
)
|
||||
return {
|
||||
"id": str(project.id),
|
||||
"name": project.name,
|
||||
"schema_version": project.schema_version,
|
||||
"seq": project.seq,
|
||||
"palette": project.palette,
|
||||
"clips": clips,
|
||||
}
|
||||
|
||||
|
||||
@require_http_methods(["GET", "POST"])
|
||||
def projects(request):
|
||||
if request.method == "GET":
|
||||
return JsonResponse(
|
||||
{
|
||||
"projects": [
|
||||
{"id": str(p.id), "name": p.name,
|
||||
"schema_version": p.schema_version, "seq": p.seq,
|
||||
"updated": p.updated.isoformat()}
|
||||
for p in Project.objects.all()[:100]
|
||||
]
|
||||
}
|
||||
)
|
||||
try:
|
||||
data = _body(request)
|
||||
project = Project.objects.create(name=data.get("name") or "untitled")
|
||||
return JsonResponse(_project_json(project), status=201)
|
||||
except Bad as exc:
|
||||
return _error(exc)
|
||||
|
||||
|
||||
@require_http_methods(["GET", "PUT"])
|
||||
def project_detail(request, project_id):
|
||||
try:
|
||||
project = Project.objects.get(id=project_id)
|
||||
except Project.DoesNotExist:
|
||||
return JsonResponse({"error": "no such project"}, status=404)
|
||||
if request.method == "GET":
|
||||
return JsonResponse(_project_json(project))
|
||||
try:
|
||||
return _save(project, _body(request))
|
||||
except Bad as exc:
|
||||
return _error(exc)
|
||||
|
||||
|
||||
@transaction.atomic
|
||||
def _save(project: Project, data):
|
||||
"""A whole-document save: one clip's leaves replace that clip's leaves.
|
||||
|
||||
SCOPED BY CLIP, not by project. A payload that carries clip `a` does not
|
||||
disturb clip `b`'s leaves, because a save is not the only way the document
|
||||
changes — a single-leaf conditional write is — and a save that cleared
|
||||
everything it did not mention would be a save that undoes a collaborator.
|
||||
|
||||
A leaf whose value is unchanged keeps its VERSION. That is what makes the
|
||||
entity tag mean something: a save of a document where one channel moved
|
||||
invalidates one leaf's etag, not all four hundred.
|
||||
"""
|
||||
if data.get("name"):
|
||||
project.name = data["name"]
|
||||
if data.get("palette"):
|
||||
project.palette = data["palette"]
|
||||
|
||||
written, removed, unchanged = [], [], []
|
||||
for spec in data.get("clips") or []:
|
||||
cid = spec.get("cid")
|
||||
if not cid:
|
||||
raise Bad("every clip in a save names its cid")
|
||||
leaves = spec.get("leaves") or {}
|
||||
prefix = f"clip/{cid}/"
|
||||
for path in leaves:
|
||||
if not path.startswith(prefix):
|
||||
raise Bad(
|
||||
f"leaf {path!r} is not addressed to clip {cid!r}",
|
||||
clip=cid, path=path,
|
||||
)
|
||||
|
||||
keys = spec.get("blocks") or []
|
||||
have = set(Block.objects.filter(key__in=keys).values_list("key", flat=True))
|
||||
if missing := [k for k in keys if k not in have]:
|
||||
# Referential integrity across the tiers, enforced where it can be:
|
||||
# a document that names blocks the server does not hold would load
|
||||
# into a blank stage on any other machine.
|
||||
raise Bad(
|
||||
"this clip names tier-2 blocks the server does not have; upload them "
|
||||
"before saving the document that points at them",
|
||||
status=409, missing=missing,
|
||||
)
|
||||
|
||||
analysis = Analysis.objects.filter(key=spec.get("analysis")).first()
|
||||
footage = None
|
||||
if spec.get("footage"):
|
||||
footage = Footage.objects.filter(id=spec["footage"]).first()
|
||||
clip, _ = Clip.objects.update_or_create(
|
||||
project=project,
|
||||
cid=cid,
|
||||
defaults={"name": spec.get("name") or "", "analysis": analysis, "footage": footage},
|
||||
)
|
||||
clip.blocks.set(Block.objects.filter(key__in=keys))
|
||||
|
||||
existing = {leaf.path: leaf for leaf in project.leaves.filter(path__startswith=prefix)}
|
||||
for path, value in leaves.items():
|
||||
leaf = existing.get(path)
|
||||
if leaf is None:
|
||||
Leaf.objects.create(project=project, path=path, value=value)
|
||||
written.append(path)
|
||||
elif leaf.value != value:
|
||||
leaf.value = value
|
||||
leaf.version += 1
|
||||
leaf.save(update_fields=["value", "version", "updated"])
|
||||
written.append(path)
|
||||
else:
|
||||
unchanged.append(path)
|
||||
for path, leaf in existing.items():
|
||||
if path not in leaves:
|
||||
leaf.delete()
|
||||
removed.append(path)
|
||||
|
||||
seq = project.seq + 1
|
||||
project.seq = seq
|
||||
project.save()
|
||||
return JsonResponse(
|
||||
{
|
||||
"id": str(project.id),
|
||||
"schema_version": project.schema_version,
|
||||
"seq": seq,
|
||||
"written": sorted(written),
|
||||
"removed": sorted(removed),
|
||||
"unchanged": len(unchanged),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@require_http_methods(["GET", "PUT"])
|
||||
def leaf_detail(request, project_id, leaf_path):
|
||||
"""One leaf, conditionally.
|
||||
|
||||
`If-Match` and a 409 whose body carries the CURRENT value, so the client can
|
||||
offer keep-mine / take-theirs. A PUT that replaced unconditionally is the bug
|
||||
docs/architecture.md calls out in tl: the loser's work disappears silently, and
|
||||
for a painted cel that is the class of bug that ends trust in a tool.
|
||||
"""
|
||||
try:
|
||||
project = Project.objects.get(id=project_id)
|
||||
except Project.DoesNotExist:
|
||||
return JsonResponse({"error": "no such project"}, status=404)
|
||||
|
||||
leaf = project.leaves.filter(path=leaf_path).first()
|
||||
if request.method == "GET":
|
||||
if leaf is None:
|
||||
return JsonResponse({"error": "no such leaf"}, status=404)
|
||||
response = JsonResponse({"path": leaf.path, "value": leaf.value, "version": leaf.version})
|
||||
response["ETag"] = leaf.etag
|
||||
return response
|
||||
|
||||
try:
|
||||
data = _body(request)
|
||||
except Bad as exc:
|
||||
return _error(exc)
|
||||
if "value" not in data:
|
||||
return _error(Bad("a leaf write carries a value"))
|
||||
|
||||
match = request.headers.get("If-Match")
|
||||
if leaf is None:
|
||||
# ANY `If-Match` on a leaf that does not exist is a failed precondition,
|
||||
# `*` included: RFC 7232 gives `*` the meaning "the resource must already
|
||||
# exist", which is exactly the write a client makes when it believes it is
|
||||
# editing something. Creating it instead would turn "somebody deleted this
|
||||
# node" into a silent resurrection.
|
||||
if match:
|
||||
return JsonResponse(
|
||||
{"error": "no such leaf", "path": leaf_path}, status=409
|
||||
)
|
||||
leaf = Leaf.objects.create(project=project, path=leaf_path, value=data["value"])
|
||||
else:
|
||||
if match and match not in ("*", leaf.etag):
|
||||
response = JsonResponse(
|
||||
{
|
||||
"error": "stale write",
|
||||
"path": leaf.path,
|
||||
"version": leaf.version,
|
||||
"value": leaf.value,
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
response["ETag"] = leaf.etag
|
||||
return response
|
||||
leaf.value = data["value"]
|
||||
leaf.version += 1
|
||||
leaf.save(update_fields=["value", "version", "updated"])
|
||||
|
||||
seq = project.bump()
|
||||
response = JsonResponse({"path": leaf.path, "version": leaf.version, "seq": seq})
|
||||
response["ETag"] = leaf.etag
|
||||
return response
|
||||
|
||||
|
||||
@require_http_methods(["GET", "POST"])
|
||||
def revisions(request, project_id):
|
||||
"""Mark a version: one snapshot of the authored layer, with a summary."""
|
||||
try:
|
||||
project = Project.objects.get(id=project_id)
|
||||
except Project.DoesNotExist:
|
||||
return JsonResponse({"error": "no such project"}, status=404)
|
||||
if request.method == "GET":
|
||||
return JsonResponse(
|
||||
{
|
||||
"revisions": [
|
||||
{"seq": r.seq, "author": r.author, "summary": r.summary,
|
||||
"created": r.created.isoformat(), "leaves": len(r.document)}
|
||||
for r in project.revisions.all()[:100]
|
||||
]
|
||||
}
|
||||
)
|
||||
data = json.loads(request.body or b"{}")
|
||||
revision = Revision.objects.create(
|
||||
project=project,
|
||||
seq=project.seq,
|
||||
author=data.get("author") or "",
|
||||
summary=data.get("summary") or "",
|
||||
document={leaf.path: leaf.value for leaf in project.leaves.all()},
|
||||
)
|
||||
return JsonResponse({"seq": revision.seq, "leaves": len(revision.document)}, status=201)
|
||||
59
do
Executable file
59
do
Executable file
|
|
@ -0,0 +1,59 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Run the local app and its frontend watcher with one command."""
|
||||
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def start():
|
||||
with socket.socket() as probe:
|
||||
if probe.connect_ex(("127.0.0.1", 8778)) == 0:
|
||||
raise SystemExit("port 8778 is already in use")
|
||||
|
||||
commands = [
|
||||
(["mise", "exec", "--", "python", "manage.py", "runserver", "8778"], ROOT),
|
||||
(["mise", "exec", "--", "npx", "shadow-cljs", "watch", "app"], ROOT / "frontend"),
|
||||
]
|
||||
children = []
|
||||
stopping = False
|
||||
|
||||
def stop(_signal, _frame):
|
||||
nonlocal stopping
|
||||
stopping = True
|
||||
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
try:
|
||||
for command, directory in commands:
|
||||
children.append(subprocess.Popen(command, cwd=directory, start_new_session=True))
|
||||
print("arthur: http://localhost:8778 (Ctrl-C stops both processes)", flush=True)
|
||||
while not stopping:
|
||||
for child in children:
|
||||
if child.poll() is not None:
|
||||
raise SystemExit(f"app process exited with status {child.returncode}")
|
||||
time.sleep(0.25)
|
||||
finally:
|
||||
for child in children:
|
||||
if child.poll() is None:
|
||||
os.killpg(child.pid, signal.SIGTERM)
|
||||
for child in children:
|
||||
try:
|
||||
child.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(child.pid, signal.SIGKILL)
|
||||
child.wait()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if sys.argv[1:] == ["start"]:
|
||||
start()
|
||||
else:
|
||||
raise SystemExit("usage: ./do start")
|
||||
752
docs/animation-model.md
Normal file
752
docs/animation-model.md
Normal file
|
|
@ -0,0 +1,752 @@
|
|||
# arthur — the animation model
|
||||
|
||||
The data that describes a moving picture: what the primitives are, how they
|
||||
nest, how they change over time, and how rotoscoped and hand-authored work end
|
||||
up being the same thing with one flag between them.
|
||||
|
||||
`docs/design.md` is the aesthetic argument. `docs/architecture.md` is where the
|
||||
code goes. This is the type that both of them are about.
|
||||
|
||||
## What this replaces
|
||||
|
||||
`docs/design.md` has a table of five kinds of part — plate, feature, interior,
|
||||
primitive, scalar — each with its own source, vocabulary and interpolation. That
|
||||
table is a good description of **where data comes from** and a bad description of
|
||||
**what data is**, and the current code follows it too literally: eyes, brows,
|
||||
teeth and mouth each get their own build function, their own key shape and their
|
||||
own path through the prototype's writer.
|
||||
|
||||
They are all one thing. A part is a **node** with **channels**, and the five
|
||||
kinds collapse into differences of which channels exist and who filled them in.
|
||||
|
||||
## Prior art, and what each one gets right
|
||||
|
||||
| System | The idea worth taking |
|
||||
| --- | --- |
|
||||
| **Flash / SWF** | A **library of symbols** and a timeline of **instances** at depths. "Framed" content that simply exists on a frame, versus tweened content. `DefineMorphShape` requires matching vertex counts — the fixed-topology rule, arrived at from the other direction. |
|
||||
| **Blender** | Animation is **addressed by path into the data** (`location[0]`), not stored as fields on the object. An Action is a bag of F-Curves. That decoupling is what makes the dope sheet, the graph editor and the NLA three views of one dataset. Also: parenting captures a `parent_inverse` so the child does not jump. |
|
||||
| **After Effects** | Every leaf property is animatable, uniformly. Property groups form a tree. Pre-comps nest arbitrarily and a pre-comp is just a layer. |
|
||||
| **Lottie** | The uniform property shape: `{a: 0, k: <value>}` or `{a: 1, k: [<keys>]}`. One representation for static and animated, which is exactly "framed or keyframed". |
|
||||
| **Grease Pencil** | A 2D layer holds frames at frame numbers, and a frame **holds until the next one**. Hold is the default, not a special case. |
|
||||
|
||||
What none of them get right for this project: colour. All four store RGB on the
|
||||
shape. `docs/design.md` forbids that, so colour is a palette index here and it is
|
||||
a channel like any other.
|
||||
|
||||
## The one idea
|
||||
|
||||
**Analysis is a channel generator.** It does not produce a different kind of
|
||||
data; it produces keys, densely, on the same channels a hand would fill in
|
||||
sparsely. So:
|
||||
|
||||
```
|
||||
footage ──▶ analysis ──▶ FREEZE ──▶ channels on nodes ──▶ evaluate ──▶ raster
|
||||
▲
|
||||
hand authoring ──┘
|
||||
```
|
||||
|
||||
Freezing is not a conversion into a second format. There is one format, and
|
||||
freezing fills it in. That is what makes "the only difference is a special flag"
|
||||
literally true: the flag is provenance on a channel, and nothing in the renderer
|
||||
reads it.
|
||||
|
||||
## Node
|
||||
|
||||
A node is an instance in the scene. The tree is stored **flat, with parent
|
||||
pointers** — never as nested maps.
|
||||
|
||||
```clojure
|
||||
{:id :mouth
|
||||
:name "mouth"
|
||||
:kind :poly ; :poly :disc :rect :group :bitmap :symbol
|
||||
:parent :head ; nil at the root
|
||||
:z "a3" ; fractional index, ordered among all siblings
|
||||
:symbol nil ; or :sym/blink — see Symbols
|
||||
:stencil :mouth-in ; colour-key clip; structural, not a channel
|
||||
:span [0 240] ; in/out in the parent's frame space
|
||||
:pinv [1 0 0 1 0 0] ; parent-inverse, captured when parented
|
||||
:channels {...}}
|
||||
```
|
||||
|
||||
Flat with pointers, for four reasons that all point the same way: any node is
|
||||
addressable without a walk; reparenting is a one-field write rather than a
|
||||
subtree move; an edit to a leaf does not change the identity of its ancestors, so
|
||||
re-frame's structural sharing keeps ancestor subs from invalidating; and it is
|
||||
what lets every node be its own sync leaf. Flash, Blender and AE all store it
|
||||
this way.
|
||||
|
||||
`:span` is Lottie's `ip`/`op` and Flash's `PlaceObject`/`RemoveObject`: the range
|
||||
over which the node exists at all. Distinct from a `[:vis]` channel, which
|
||||
blinks an existing node on and off.
|
||||
|
||||
### Subjects and tracked features
|
||||
|
||||
Scene nodes describe drawings, not tracking identity. A scene may also carry a
|
||||
flat `:features` map. A feature ID stays stable for the whole clip, including
|
||||
frames where that feature is occluded and later reappears:
|
||||
|
||||
```clojure
|
||||
:subjects {:face-1 {:id :face-1}}
|
||||
:features
|
||||
{:eye-r {:id :eye-r :subject :face-1 :area :eye
|
||||
:nodes [:eye-r :eye-r-in :iris-r :pupil-r] :params {}}
|
||||
:eye-l {:id :eye-l :subject :face-1 :area :eye
|
||||
:nodes [:eye-l :eye-l-in :iris-l :pupil-l] :params {}}
|
||||
:mouth {:id :mouth :subject :face-1 :area :mouth
|
||||
:nodes [:mouth :mouth-in] :params {}}
|
||||
;; The teeth are their OWN feature and not three nodes of the mouth. A feature
|
||||
;; carries the params of exactly one area, and the teeth have an `:area :teeth`
|
||||
;; of their own — the otsu threshold, the tongue rejection, the radial contour's
|
||||
;; vertex budget — which could not be reached if they were part of `:mouth`.
|
||||
;; The coupling that made them look like the mouth's is real and is enforced
|
||||
;; elsewhere: `:teeth` is STENCILLED by `:mouth-in`, and a node whose stencil drew
|
||||
;; nothing is dropped, so an absent mouth takes the teeth with it without either
|
||||
;; of them sharing an absence mask. An earlier draft of this block listed them
|
||||
;; together; the code is right and this document was wrong.
|
||||
:teeth {:id :teeth :subject :face-1 :area :teeth
|
||||
:nodes [:teeth] :params {}}}
|
||||
:groups
|
||||
{:eyes-1 {:id :eyes-1 :kind :eye-pair :subject :face-1
|
||||
:members [:eye-r :eye-l] :params {}}}
|
||||
```
|
||||
|
||||
An eye pair is an explicit relationship between one or two eyes of the **same
|
||||
subject**. It may have one member when only one eye has been identified; it does
|
||||
not invent a second eye. Five subjects with nine identified eyes can have four
|
||||
two-member pairs and one one-member pair. Each eye still has its own feature ID
|
||||
and presence track. A group is a settings association, not a scene parent or a
|
||||
tracking ID. Membership lives only on the group, avoiding a second pointer on
|
||||
the feature that could disagree with it.
|
||||
|
||||
Each feature resolves settings from its area's definitions, then its group,
|
||||
then its own `:params`. An eye can therefore inherit a pair setting or override
|
||||
it without changing its partner. Removing it from a pair copies its effective
|
||||
values into the feature first, so the result does not jump. Feature identity
|
||||
and pair membership are clip-wide; a future parameter track can vary values
|
||||
over time without splitting a feature at an observation gap.
|
||||
|
||||
Parameter definitions live in one registry: key, default, applicable area,
|
||||
value constraints and affected areas. The registry supplies the take's defaults
|
||||
today. The parameter UI and regeneration from edited values are later work.
|
||||
|
||||
Dense channel state records whether a measurement exists **for that feature on
|
||||
that frame**. Occlusion means absent data on that frame, not a false `[:vis]`
|
||||
value and not the end of the feature's identity. A full-face detection failure
|
||||
makes all its features absent. A single occluded eye need only make that eye's
|
||||
channels absent. Footage can carry explicit feature absence intervals in its
|
||||
manifest, with one-based inclusive source frame numbers, for example
|
||||
`"feature-absence": {"eye-r": [[10, 14]]}`. The loader expands these into
|
||||
per-frame observation tracks before measurement. Unobserved landmarks may fill
|
||||
rectangular numeric buffers, but they cannot contribute to an eye's contour,
|
||||
blink or shared gaze. When one eye is absent, gaze uses the observed eye.
|
||||
Until a detector supplies feature-level confidence, footage without annotations
|
||||
uses the full-face detection mask as the fallback; it must not claim to detect
|
||||
individual occlusions that it cannot see.
|
||||
|
||||
## Channel
|
||||
|
||||
Every animatable property is a channel, and channels are addressed **by path**:
|
||||
|
||||
```clojure
|
||||
:channels
|
||||
{[:xform :pos] {:animated? false :value [0.0 0.0]}
|
||||
[:xform :rot] {:animated? false :value 0.0}
|
||||
[:xform :scale] {:animated? false :value [1.0 1.0]}
|
||||
[:xform :skew] {:animated? false :value [0.0 0.0]}
|
||||
[:xform :anchor]{:animated? false :value [0.0 0.0]}
|
||||
[:geom :pts] {:animated? true :interp :hold :dense {...} :generated {...}}
|
||||
[:style :color] {:animated? false :value :skin-dark}
|
||||
[:vis] {:animated? true :interp :hold :keys {0 true, 37 false}}}
|
||||
```
|
||||
|
||||
A path is a **vector**, not a string — CLJS maps take vectors as keys natively,
|
||||
so Blender's `data_path` idea arrives with no parsing. The set of valid paths for
|
||||
a node follows from its `:kind`, and that is a spec, not a schema migration.
|
||||
|
||||
Three channel shapes, and the uniformity across them is the point:
|
||||
|
||||
```clojure
|
||||
;; FRAMED — one static thing. No animation, no vertex correspondence to worry
|
||||
;; about. A painted background cel is this.
|
||||
{:animated? false :value v}
|
||||
|
||||
;; KEYED — sparse, authored, in the document. Undoable and syncable.
|
||||
{:animated? true :interp :hold :keys {0 v, 4 v, 12 v}}
|
||||
|
||||
;; DENSE — generated, one value per frame, held in tier 2 as a typed array.
|
||||
{:animated? true :interp :hold
|
||||
:dense {:store "sha256:…" :offset 0 :stride 40 :frames 600}
|
||||
:generated {...}}
|
||||
```
|
||||
|
||||
`:interp` defaults to `:hold`, which `docs/design.md` requires of every cut part.
|
||||
An authored keyed channel may also carry `:segments {8 :linear}`: the key at 8
|
||||
tweens toward the next key, while other gaps use the channel default. The
|
||||
transition belongs to the gap starting at a key, so a shape can cut into one
|
||||
drawing and tween out of it. Per-key easing beyond hold and linear is deferred.
|
||||
|
||||
### Keys are a map by frame, not a list
|
||||
|
||||
Already argued in `docs/architecture.md` for merge reasons; here it also gives
|
||||
"the most recent key at or before `f`" as a `rsubseq` on a sorted map instead of
|
||||
a scan. **Store a plain map** in the document — transit and JSON both lose
|
||||
sortedness — and build the sorted index in the resolver.
|
||||
|
||||
### The flag lives on the channel, not the node
|
||||
|
||||
```clojure
|
||||
:generated {:by :roto/lips-outer
|
||||
:analysis "sha256:…" ; which analysis artifact
|
||||
:params {:verts 8 :contour-avg 1 :aperture-cut 0.004}}
|
||||
```
|
||||
|
||||
Present means the UI offers a parameter panel and a re-freeze button. Absent
|
||||
means the UI offers the keys directly. **The renderer never reads it.**
|
||||
|
||||
It belongs on the channel rather than the node because a node routinely wants
|
||||
both at once: a mouth whose `[:geom :pts]` is rotoscoped and whose `[:xform :pos]`
|
||||
is hand-animated to sit on a plate. Putting the flag on the node would forbid the
|
||||
most useful thing in the model.
|
||||
|
||||
### Channels are layered
|
||||
|
||||
A channel is a base plus optional override layers, and a layer declares how it
|
||||
combines:
|
||||
|
||||
```clojure
|
||||
{:animated? true :interp :hold
|
||||
:dense {...} :generated {...}
|
||||
:over [{:blend :offset :keys {88 [2 0], 96 [0 0]}}
|
||||
{:blend :replace :keys {104 [[3 7] [4 7] …]}}]}
|
||||
```
|
||||
|
||||
- **`:offset`** adds a delta to the base. "Nudge the mouth two pixels right for
|
||||
ten frames" survives a re-freeze at different parameters, because it was never
|
||||
a position — it was a correction.
|
||||
- **`:replace`** wins outright. For the frame where detection simply failed.
|
||||
|
||||
This is what `docs/design.md` means by an override layer, and it is why
|
||||
re-freezing is safe: the base is regenerated, the layers are untouched. It is
|
||||
Blender's NLA blending and AE's effect stack at one property.
|
||||
|
||||
Layers are what "set it by hand" means for anything measured, and the measured
|
||||
channel does not need to know. A hand-set gaze is an `:over` on
|
||||
`[:xform :pos]` of the iris; a hand-set mouth shape is an `:over` on
|
||||
`[:geom :pts]`. Turning the gaze-step or gaze-dwell knob regenerates the base and
|
||||
leaves the correction alone, which is the entire reason a correction is stored as
|
||||
a layer rather than written into the track.
|
||||
|
||||
**A `:replace` layer overrides absence, an `:offset` layer does not.** Sampling a
|
||||
channel is: read the base, then apply the layers — and the base coming back
|
||||
`absent` does not short-circuit that. `:replace` is explicitly for the frame
|
||||
where detection failed, so it has to be able to supply a value where there is
|
||||
none; `:offset` is a delta, and there is nothing to nudge, so an offset over an
|
||||
absent base stays absent. Implemented the obvious way — bail out on absence
|
||||
before reaching the layers — the one case the feature exists for is the one case
|
||||
it would not cover.
|
||||
|
||||
### One signal, two nodes
|
||||
|
||||
Gaze is deliberately **one measurement shared by both eyes**: at this size the
|
||||
per-eye difference is noise, and independent noise reads as wall-eyed
|
||||
immediately, which is the most expensive artefact on a face. But it is stored as
|
||||
`[:xform :pos]` on `:iris-r` and on `:iris-l`, which are two channels on two
|
||||
nodes with two different parents — so the invariant lives in `measure` and
|
||||
nothing in the document enforces it.
|
||||
|
||||
That matters as soon as either one can be overridden by hand, because an `:over`
|
||||
on one iris alone reproduces exactly the artefact the shared measurement exists
|
||||
to prevent. Until drivers exist, **the override is on both or on neither**, and
|
||||
that is a rule the UI has to keep rather than one the data can.
|
||||
|
||||
This is the case that will eventually justify **drivers** — one value, evaluated
|
||||
once, feeding several channels — which is why gaze is named in Deferred as the
|
||||
obvious first one. Nothing here forecloses it: a driver needs a place in the
|
||||
document and a `:driven-by` on a channel, both of which are additive, and an
|
||||
absent key means "not driven". So it stays deferred, and the shape does not have
|
||||
to change to allow it.
|
||||
|
||||
## Transform: decomposed, never a matrix
|
||||
|
||||
```clojure
|
||||
{:pos [x y] :rot θ :scale [sx sy] :skew [kx ky] :anchor [ax ay]}
|
||||
```
|
||||
|
||||
Stored decomposed for two reasons. Each component has to be independently
|
||||
keyframable, which is the entire point of channels. And interpolating matrix
|
||||
entries is meaningless — a rotation tweened through its matrix shears on the way.
|
||||
|
||||
Composition, per node:
|
||||
|
||||
```
|
||||
local = T(pos) · T(anchor) · R(rot) · K(skew) · S(scale) · T(-anchor)
|
||||
world = world(parent) · pinv · local
|
||||
```
|
||||
|
||||
`:anchor` is Flash's registration point and Blender's origin: rotation and scale
|
||||
happen about it, and getting it wrong is why hand-placed parts swing rather than
|
||||
turn.
|
||||
|
||||
`:pinv` is Blender's `parent_inverse`, captured at the moment of parenting so the
|
||||
child does not jump when it acquires a parent. Small, and its absence is the kind
|
||||
of thing that makes a parenting feature feel broken.
|
||||
|
||||
**The similarity fit already produces a decomposition.** `fitSimilarity` returns
|
||||
`{s θ tx ty}`, which drops straight into `[:xform :scale]`, `[:xform :rot]` and
|
||||
`[:xform :pos]` with no conversion. The analysis output and the animation model
|
||||
meet without an adapter, which is a sign the decomposition is the right one.
|
||||
|
||||
## What space geometry is in
|
||||
|
||||
**`[:geom :pts]` is always in the node's own local space, and the transform
|
||||
chain says what that means.** There is no global geometry space and no decision
|
||||
to make about one.
|
||||
|
||||
| Node | Its local space | Why that one |
|
||||
| --- | --- | --- |
|
||||
| a rotoscoped feature | head-local, isotropic, unit = one image height | what the anchor fit already produces; the `xform` to raster is not applied and not stored |
|
||||
| a painted cel | the stage, in pixels, grid-snapped | the artist is placing pixels, so the pixel grid is the thing being authored |
|
||||
| a primitive under a feature | its parent's | the iris is positioned on the lid ring, not on the stage |
|
||||
|
||||
This looks like a small clarification and it removes a whole class of argument.
|
||||
The prototype bakes the framing into the numbers: `toRasterRing` applies
|
||||
`makeXform`, which centres on the face oval's bounding box and zooms until the
|
||||
face is 80% of the raster height, so **every stored vertex carries a cropping
|
||||
decision** that was made once, at analysis time, from one frame's landmarks.
|
||||
Dropping that step is a deletion, not a feature, and after it the framing is
|
||||
simply a transform on a node.
|
||||
|
||||
Grid snapping belongs to the cel and not to the roto, for the same reason: a cel
|
||||
is authored on the grid and a traced contour is not. So it is a property of a
|
||||
node's space rather than a rule about all geometry, and the tension between
|
||||
"integer polygons" and "arbitrary placement" was never real.
|
||||
|
||||
Each dense block therefore carries its own **fixed-point scale** in its header,
|
||||
because a block in image-height units and a block in stage pixels need different
|
||||
ones to fill an `Int16` usefully.
|
||||
|
||||
### There is no camera node
|
||||
|
||||
A camera is a global transform over everything, and nothing here wants one.
|
||||
Placing the face on the stage is a transform on a node, which already exists;
|
||||
what is not on the stage hangs off the edges and the canvas clips it. Every fill
|
||||
in `domain/raster` already clamps rather than assuming it is inside, so drawing
|
||||
past the edge is not a feature to add.
|
||||
|
||||
Project dimensions are therefore **independent of the footage**. A 1440x1920
|
||||
portrait clip composited onto a 320x200 stage is not a problem to solve — the
|
||||
head is placed and scaled where it belongs and the rest of the frame is simply
|
||||
not on stage. The full frame stays *available* for tracing without being
|
||||
*visible*, and those are different requirements.
|
||||
|
||||
## Head motion: free or anchored to measured frames
|
||||
|
||||
`stabilize` produces `{s, θ, tx, ty}` per source frame. Its inverse is stored
|
||||
densely on `:head`'s position, rotation and scale channels. The same measured
|
||||
track serves every placement choice:
|
||||
|
||||
```clojure
|
||||
;; no :anchors — free: read the measured transform at the current frame
|
||||
;; one key — lock to a chosen measured frame throughout
|
||||
:anchors {0 12}
|
||||
;; several keys — cut to another measured head transform at frame 40
|
||||
:anchors {0 12, 40 42}
|
||||
```
|
||||
|
||||
The map is `local change frame -> measured source frame`. A single lock is a
|
||||
one-key map. Position, rotation and scale read the same held source frame. The
|
||||
frame set belongs to head placement, independently of plate drawings and stage
|
||||
pose cuts. No measured block is copied into authored transform keys.
|
||||
|
||||
**Always measure, always store factored.** The fit is computed and the geometry
|
||||
is stored head-local in every mode. Only the frame address used to read the
|
||||
head's measured transform changes. Two things downstream require that split:
|
||||
|
||||
- *Smoothing.* "Smooth the transform, never the contour" only means anything
|
||||
while the two are separate.
|
||||
- *Key selection.* A velocity minimum is "articulation paused" in head-local
|
||||
space and "the head happened to be still" in image space.
|
||||
|
||||
This is a document edit, not a reason to re-analyse. A registered tracing photo
|
||||
will use its own source frame's stabilising transform followed by the same
|
||||
selected head placement, so it aligns with the vectors drawn over it.
|
||||
|
||||
### Two nodes, because two different things want that transform
|
||||
|
||||
```
|
||||
:face group — AUTHORED. where the face sits on the stage, and how big.
|
||||
:head group — MEASURED. dense head motion read at the selected frame.
|
||||
:mouth :mouth-in :teeth :lid-r :lid-l :brow-r :brow-l …
|
||||
```
|
||||
|
||||
Changing anchor keys edits `:head` and never touches `:face`, so it cannot move
|
||||
something that was placed by hand. A group node is free, and keeping the authored
|
||||
and the measured transform apart is the whole reason the transform is decomposed
|
||||
in the first place.
|
||||
|
||||
## Time maps — exposure, lead and symbol timing are one thing
|
||||
|
||||
Every node may map the frame it is evaluated at:
|
||||
|
||||
```clojure
|
||||
:time {:mode :inherit} ; the default, and almost always right
|
||||
:time {:mode :map :expose 2 :offset -1 :rate 1.0 :loop? false}
|
||||
:time {:mode :map :source-fps 30 :sample-fps 12} ; root: lower picture cadence
|
||||
```
|
||||
|
||||
Three features that look unrelated are this one mechanism:
|
||||
|
||||
- **exposure** is `⌊f/n⌋·n`,
|
||||
- **picture fps** quantises source time to a chosen picture grid, then reads the
|
||||
latest source pose at or before that time; source analysis and audio keep their
|
||||
original cadence,
|
||||
- **mouth lead** is `f + k`,
|
||||
- **a symbol instance's timing** is `(f - at)·rate + in`, with optional looping.
|
||||
|
||||
Composed along the nesting chain, outermost first. Two rules follow, and they are
|
||||
different rules:
|
||||
|
||||
- **Exposure inherits strictly.** `docs/design.md` is emphatic that everything
|
||||
rides one grid, because a head cutting on odd frames against a mouth cutting on
|
||||
even ones reads as two performances. The model permits a per-node grid; the
|
||||
default must be `:inherit`, and setting it lower is a deliberate act the UI
|
||||
should make feel like one.
|
||||
- **Offset is per-node by design.** Mouth lead applies to performance nodes and
|
||||
*not* to the plate, which is the whole point of it — so the offset genuinely
|
||||
belongs at the node, not the clip.
|
||||
|
||||
## Timelines, and why a scene is one
|
||||
|
||||
A **timeline** is an ordered bag of nodes in its own frame space:
|
||||
|
||||
```clojure
|
||||
{:frames 91
|
||||
:palette {...} ; see Palettes
|
||||
:nodes {id -> node}}
|
||||
```
|
||||
|
||||
That is the whole type, and **everything that holds nodes is one of these**:
|
||||
|
||||
- a clip's **scene** is its root timeline,
|
||||
- a **symbol** in the library is a timeline,
|
||||
- a node with `:kind :symbol` is an **instance** of one.
|
||||
|
||||
An earlier draft of this document had a scene and a `:kind :timeline` symbol as
|
||||
two structures with the same fields and never said they were the same thing.
|
||||
They are. Flash's `_root` is a MovieClip; After Effects' "a pre-comp is just a
|
||||
layer" is already in the prior-art table above. Collapsing them is what makes
|
||||
nesting arbitrary and free, rather than a feature to be added.
|
||||
|
||||
### Two axes of nesting, and they are different
|
||||
|
||||
This is the distinction the flat-storage rule is about, and conflating the two is
|
||||
why "nested" and "flat with parent pointers" sound contradictory when they are
|
||||
not:
|
||||
|
||||
| Axis | What nests | How it is stored |
|
||||
| --- | --- | --- |
|
||||
| **parent / child** | transform composition within one timeline | **flat, with parent pointers** — never nested maps |
|
||||
| **instance** | a timeline inside another timeline | by reference into the library |
|
||||
|
||||
Each timeline is flat. Timelines nest. Every argument for flat storage —
|
||||
addressability, one-field reparenting, structural sharing, per-node sync leaves —
|
||||
is about the first axis and is untouched by the second.
|
||||
|
||||
The instance boundary is also **the only place the frame space changes.** Within
|
||||
a timeline, `:time` is exposure and lead: a shift inside one space. At an
|
||||
instance it is `(f - at)·rate + in`, into a different one. That is why `:rate` is
|
||||
meaningless on an ordinary node and why sampling one must fail loudly rather than
|
||||
be ignored.
|
||||
|
||||
### What is scoped to a timeline
|
||||
|
||||
Three fields on a node only have meaning relative to a timeline, and the answer
|
||||
for all three is the same — **their own**:
|
||||
|
||||
- **`:z`** orders among siblings; a node cannot interleave with nodes inside a
|
||||
nested instance. The instance occupies one position in its parent's order and
|
||||
its contents sort beneath it, which the z path gives for free by being a
|
||||
vector.
|
||||
- **`:stencil`** names a node in the same timeline. A colour key does not
|
||||
naturally respect a boundary — it is just pixels — so this is a rule rather
|
||||
than a consequence, and it is Flash's rule for masks.
|
||||
- **`:span`** is in the parent node's frame space.
|
||||
|
||||
### Instances
|
||||
|
||||
A node with `:kind :symbol` and `:of :sym/blink` places one. Its own channels
|
||||
compose *over* the symbol's, so one definition is placed many times and tinted,
|
||||
offset or retimed at each placement — that is how a three-frame blink is reused
|
||||
at frames 40, 88 and 200 without copying it.
|
||||
|
||||
This is also where `docs/design.md`'s "closed vocabulary is right for the head"
|
||||
lands: a plate library is a set of `:sym/head-*` timelines, and the strip chooses
|
||||
which is instanced on which frame.
|
||||
|
||||
**Cursors and point buffers are per-instance, not per-node.** Two instances of
|
||||
one symbol sit at different frames in their own space, so they cannot share a
|
||||
reading head over the same channel. The resolver keys its caches by the instance
|
||||
path, not by node id — which is a detail of `Making it fast` below, and the one
|
||||
place symbol nesting is not free.
|
||||
|
||||
### Audio placements and controls
|
||||
|
||||
Sound is placed on a timeline as a separate `:audio` node. It uses the same
|
||||
`:span`, `:time`, and channel representation as a drawn node. A `:linked-to` id
|
||||
records which picture instance it was placed with; it does not force the two
|
||||
spans or source in-points to match.
|
||||
|
||||
```clojure
|
||||
{:id :voice-right :kind :audio :parent :root :z "a4"
|
||||
:linked-to :right
|
||||
:source {:footage "f8cace9e-..."}
|
||||
:span [48 260]
|
||||
:time {:mode :map :at 48 :in 0 :rate 1}
|
||||
:channels {[:audio :gain]
|
||||
{:animated? true :interp :linear
|
||||
:keys {48 0.0, 60 1.0, 245 1.0, 259 0.0} :over []}}}
|
||||
```
|
||||
|
||||
`[:audio :gain]`, `[:audio :pan]`, and `[:audio :rate]` are ordinary scalar
|
||||
channels. They may be framed, keyed, or dense; numeric keyed channels can ramp
|
||||
linearly. The time map sets the placement's base source rate, and
|
||||
`[:audio :rate]` multiplies it. Audio is mixed from the referenced immutable
|
||||
footage when the clip opens. The mix is derived output; the saved document holds
|
||||
the nodes and channel keys, not another audio file. One audio element plays that
|
||||
mix and remains the clock for both sound and picture.
|
||||
|
||||
This is also the boundary for a future control surface. A control has a stable
|
||||
target, such as a feature's `:verts` setting or an audio node's
|
||||
`[:audio :gain]` channel. The UI and a MIDI binding can address both through the
|
||||
same control interface. Their update costs differ: gain can be keyed over time;
|
||||
changing the number of lip vertices changes topology and must regenerate its
|
||||
dense geometry. A topology setting cannot be treated as a per-frame gain curve.
|
||||
|
||||
## Evaluating a frame
|
||||
|
||||
```clojure
|
||||
(defn eval-frame
|
||||
"Scene at clip frame f -> draw ops in z order. Pure."
|
||||
[scene f] ...)
|
||||
```
|
||||
|
||||
1. Walk nodes in **topological order** by parent depth (cached; recompute only
|
||||
when parentage changes).
|
||||
2. Skip nodes outside `:span`.
|
||||
3. Apply the node's time map to get its own local frame `fn`.
|
||||
4. **Sample** each channel at `fn`: a map lookup for framed, a sorted-index
|
||||
lookup for keyed, an array read for dense. Then apply `:over` layers.
|
||||
5. Compose `world` from the parent's.
|
||||
6. Transform geometry into raster space, writing into a **preallocated buffer**
|
||||
owned by the node.
|
||||
7. Emit `{:kind :poly :pts buf :n 20 :color idx :stencil id}`.
|
||||
8. Sort by resolved `z`.
|
||||
|
||||
The op list is the boundary with stage 7 in `docs/architecture.md`: the
|
||||
rasteriser takes ops and knows nothing about nodes, channels or time.
|
||||
|
||||
**A photographic underlay is not an op.** The registered source frame that an
|
||||
animator traces over is a reference, not output, and it may not enter the indexed
|
||||
buffer — the same rule `docs/architecture.md` already sets for handles and
|
||||
vertex boxes. It is a `drawImage` at an affine on a separate canvas, which clips
|
||||
at the canvas edge for free, and the only thing it needs from the model is the
|
||||
world transform of the node it rides:
|
||||
|
||||
```clojure
|
||||
(world-of resolver :head) ;; -> Float64Array[6]
|
||||
```
|
||||
|
||||
Composed with image-pixels-to-local — **both axes divided by `imgH`**, never by
|
||||
their own dimension — the photo is registered with the shapes by construction,
|
||||
and an unregistered underlay is merely decorative. The tracing editor chooses
|
||||
which source frame to show under a cel. That reference choice is independent of
|
||||
the finished picture fps and does not change the dense analysis track. A cel can
|
||||
therefore use any useful source frame as its drawing reference, even when that
|
||||
frame is not one of the displayed picture poses.
|
||||
|
||||
A photo that has to sit *between* two drawn layers is the case that would make it
|
||||
a `:bitmap` node with an op of its own. Nothing wants that yet: a reference is
|
||||
either under everything or over everything at low alpha.
|
||||
|
||||
### Making it fast in CLJS
|
||||
|
||||
Three things, and only these three matter:
|
||||
|
||||
- **Decomposed and persistent for storage; flat and mutable for evaluation.**
|
||||
Composed transforms are 6-element `Float64Array`s, not maps. Every renderer
|
||||
does this; the storage form and the evaluation form are allowed to differ.
|
||||
- **A cursor per channel.** Playback is sequential, so "most recent key at or
|
||||
before `f`" is an advance of a saved index, O(1) amortised. Binary search only
|
||||
on a seek. This is the difference between a `rsubseq` allocation per channel per
|
||||
frame and none.
|
||||
- **Preallocated point buffers per node.** Fixed topology means the size is known
|
||||
at freeze time, so the vertices — the overwhelming majority of the per-frame
|
||||
bytes — are written into a buffer the node already owns. A frame still
|
||||
allocates its op maps and the sorted op vector; that is a dozen small objects
|
||||
against hundreds of points, and pooling them would buy nothing and cost the
|
||||
ability to pass an op list around as plain data. At 30fps, per-vertex
|
||||
allocation is the thing that will make this stutter.
|
||||
|
||||
Because the buffers are reused, **ops must be consumed before the next frame is
|
||||
asked for.** That is the contract the rAF loop wants anyway: it reads, blits,
|
||||
and dispatches nothing.
|
||||
|
||||
### What is in app-db, and what is not
|
||||
|
||||
| In app-db (tier 1) | In tier 2, behind a handle |
|
||||
| --- | --- |
|
||||
| nodes, parentage, z, spans, stencils | dense channel blocks |
|
||||
| channel definitions, `:interp`, `:generated` | analysis artifacts |
|
||||
| **framed** values, **keyed** keys, `:over` layers | preallocated eval buffers |
|
||||
| library / symbol definitions | composed transform scratch |
|
||||
|
||||
The rule: **anything a human placed is in the document; anything a generator
|
||||
produced is a handle.** Which is the same line `docs/architecture.md` draws for
|
||||
sync and baking, arrived at again from the renderer's side.
|
||||
|
||||
## The current parts, in this model
|
||||
|
||||
Proof that it covers what exists, not just what is wanted:
|
||||
|
||||
| Now | Becomes |
|
||||
| --- | --- |
|
||||
| `mouth` outer ring, every frame | node `:mouth`, `[:geom :pts]` dense, `:generated {:by :roto/lips-outer}` |
|
||||
| `mouth_in`, hidden below aperture | node `:mouth-in`, parent `:mouth`, `[:geom :pts]` dense + `[:vis]` dense |
|
||||
| `teeth` from image content | node `:teeth`, stencil `:mouth-in`, `[:geom :pts]` dense, `:generated {:by :interior/teeth}` |
|
||||
| lid rings | nodes `:lid-r/-l`, `[:geom :pts]` dense |
|
||||
| lash line (`offsetRing`) | not data — a stage-6 parameter on the node, `{:grow px}` |
|
||||
| iris disc | node `:iris-r`, `:kind :disc`, parent `:lid-r`, stencil `:sclera-r`, `[:xform :pos]` dense (quantised at freeze), radius framed |
|
||||
| square pupil | node `:pupil-r`, `:kind :rect`, parent `:iris-r`, stencil `:iris-r` |
|
||||
| brow ring + quantised raise | node `:brow-r`, `[:geom :pts]` dense (the traced ring with height removed), `[:xform :pos]` dense (the quantised raise). **The decomposition design.md insists on is two channels.** |
|
||||
| head plate, kept frames | node `:head`, `:symbol` per instance, keys on `[:symbol]` at kept frames |
|
||||
| `makeXform` face-oval crop | **gone.** Placement is `[:xform :*]` on `:face`; the stage clips |
|
||||
| `stabilize` transforms | dense `[:xform :*]` on `:head`, read through its optional `:anchors` map |
|
||||
| registered underlay | not data — a UI layer riding `(world-of resolver :head)` |
|
||||
| painted background cel | node per layer, `[:geom :pts]` **framed**, `[:style :color]` framed |
|
||||
| `mouth lead` | `:time {:offset k}` on performance nodes only |
|
||||
| `exposure` | `:time {:expose n}` on the clip root, inherited |
|
||||
| picture fps | resolver samples marked generated channels at the picture rate; authored keys keep their own time |
|
||||
| hand correction | an `:over` layer, `:offset` or `:replace` |
|
||||
|
||||
The brow row is the one worth looking at twice. `docs/design.md` argues at length
|
||||
that the traced ring already contains the height, so the quantised raise must be
|
||||
measured *out* and put *back* or the brow moves twice. In this model that is not
|
||||
an argument to remember — it is two channels on one node, and getting it wrong
|
||||
would mean writing the height into both.
|
||||
|
||||
## Palettes
|
||||
|
||||
Three levels, and keeping them apart is what makes a palette swap a
|
||||
**reinterpretation** rather than an edit:
|
||||
|
||||
| Level | Holds | Lives on |
|
||||
| --- | --- | --- |
|
||||
| **tone** | which mark this is — `:skin-dark` | `[:style :color]`, a channel on the node |
|
||||
| **ramp** | what that tone looks like *here* | `:palette`, a channel on the timeline |
|
||||
| **the ramps** | every named palette | the project |
|
||||
|
||||
A node names a **tone**, never a colour and never a ramp. Which ramp the tone is
|
||||
read in is decided by the timeline the node is in. So the same drawing reads day
|
||||
or night without one stored value changing — which is the entire payoff of
|
||||
indexed colour, and is why `docs/design.md` forbids sampled RGB: once a shape
|
||||
holds a measured colour there is nothing left to reinterpret.
|
||||
|
||||
Named palettes are **variants over one tone vocabulary**, not arbitrary colour
|
||||
lists. `:day` and `:night` both define `:skin-dark`; that is what keeps a swap
|
||||
total and keeps `docs/design.md`'s closed vocabulary closed. A tone the ramp in
|
||||
scope does not define resolves to the loud magenta, like any other missing index.
|
||||
|
||||
### The scope rule
|
||||
|
||||
`:palette` on a timeline is a channel like any other:
|
||||
|
||||
```clojure
|
||||
{:frames 91
|
||||
:palette {:animated? true :interp :hold :keys {0 :day, 48 :dusk, 72 :night}}
|
||||
:nodes {...}}
|
||||
```
|
||||
|
||||
**Absent means inherit** from the instancing context. **Present means this
|
||||
timeline's content is read in that ramp, and it travels with the timeline** — a
|
||||
symbol authored against `:night` stays night wherever it is placed. That is
|
||||
lexical scope, and deliberately: a character with their own palette is a
|
||||
character, not a decoration of whichever scene they were dropped into.
|
||||
|
||||
Composition is the same walk as `:time` — down the instance chain, **innermost
|
||||
set palette wins**. An enclosing timeline's palette therefore applies to
|
||||
everything inside it that does not set its own, which is adjustment-layer
|
||||
behaviour with no adjustment layer in it. It is just scope.
|
||||
|
||||
And because it is an ordinary channel, a project switches palette over time with
|
||||
keys on the root timeline, a child timeline switches on its own, and neither
|
||||
knows about the other.
|
||||
|
||||
### One index space, partitioned by palette
|
||||
|
||||
A raster is one `Uint8Array` and an index means one colour in it, so two ramps in
|
||||
one frame cannot both own index 2. The resolution: **the output index space is
|
||||
the concatenation of the named palettes**, and a tone resolves to
|
||||
`palette-base + tone-index`.
|
||||
|
||||
Everything downstream is then unchanged — one buffer, one flat table for
|
||||
`->rgba`, no per-frame palette construction, and an index does not change meaning
|
||||
between frames, so bakes and thumbnails stay valid.
|
||||
|
||||
Two consequences worth stating rather than discovering:
|
||||
|
||||
- **The limit is real and reachable.** 256 indices over a nine-tone vocabulary is
|
||||
twenty-eight palettes. Detect it and say so; do not let it arrive as wrapped
|
||||
colour.
|
||||
- **It makes the stencil sharper.** A stencil is a colour key, so two nodes
|
||||
sharing a tone share a stencil — a genuine weakness of the technique.
|
||||
Partitioning the index space by palette means two nodes in *different* palettes
|
||||
no longer collide at all, and the resolved stencil picks up whichever index the
|
||||
stencil node actually drew in.
|
||||
|
||||
### Where it is resolved
|
||||
|
||||
At the op boundary, and nowhere else. `[:style :color]` holds a keyword all the
|
||||
way through evaluation; the walk carries the palette in scope the same way it
|
||||
carries the parent transform and the local frame; the op carries a resolved
|
||||
index. The rasteriser never sees a tone name and the node never sees an index.
|
||||
|
||||
This also means the palette is a **parameter of evaluation**, not a global. The
|
||||
resolver takes it alongside the store.
|
||||
|
||||
## Format on disk and on the wire
|
||||
|
||||
Tier 1 is EDN/transit: the node tree, channel definitions, framed values, keys,
|
||||
layers, library. Kilobytes, human-readable, diffable, and leaf-addressable for
|
||||
sync.
|
||||
|
||||
Dense blocks are separate content-addressed binaries — `Int16Array` for
|
||||
geometry, `Float32Array` for transforms — with a small header naming the channel
|
||||
path, frame count, stride, and the **fixed-point scale** of the node-local space
|
||||
the block is in. Geometry is stored in the node's own space, not in raster space;
|
||||
see "What space geometry is in".
|
||||
|
||||
**Not Lottie internally**, despite the property shape being borrowed from it.
|
||||
Lottie has no palette-indexed colour, its shapes are bezier with in/out tangents
|
||||
where these are integer polygons, and its interpolation defaults are the opposite
|
||||
of what is wanted. It is a fine thing to write out one day and a bad thing to
|
||||
store.
|
||||
|
||||
Output is deliberately not specified here. The target is encoding video in the
|
||||
browser, which touches the op list and nothing above it — a writer consumes
|
||||
frames, and frames are what stage 7 already produces.
|
||||
|
||||
## Deferred
|
||||
|
||||
- **Per-key easing.** The structure allows it; nothing should use it until a
|
||||
parented transform on a painted cel asks for it.
|
||||
- **More than two channel layers.** The `:over` vector is already a list; a real
|
||||
blend stack with weights is the NLA, and it is not needed to fix a bad frame.
|
||||
- **Skew beyond the field.** `[:xform :skew]` is in the transform and in the
|
||||
composition order from the start, because adding a component to a decomposition
|
||||
later means migrating every stored transform.
|
||||
- **Instance channel overrides on symbols.** Compose-over is specified; only
|
||||
colour and transform need it at first.
|
||||
- **Constraints and drivers.** Blender's other half. A gaze that aims at a null
|
||||
object is the obvious first one, and it is a long way off. Until then the one
|
||||
gaze shared by two iris nodes is a UI rule, not a stored relationship — see
|
||||
"One signal, two nodes".
|
||||
1017
docs/architecture.md
Normal file
1017
docs/architecture.md
Normal file
File diff suppressed because it is too large
Load diff
101
docs/multi-face-representation.md
Normal file
101
docs/multi-face-representation.md
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
# Multi-face representation
|
||||
|
||||
Status (2026-09-29): representation work complete. Reopen it for a concrete
|
||||
requirement, rather than another round of abstract alternatives.
|
||||
|
||||
Implemented: each tracked face has a drawing timeline, placed by an ordinary
|
||||
symbol instance. Timelines already provide local node names, independent playback,
|
||||
and persistence. No new kind of scene container is needed.
|
||||
|
||||
```clojure
|
||||
:timelines
|
||||
{:main {:nodes {:root {:time {:mode :map :expose 2}}
|
||||
:face {:parent :root :channels <source-to-stage placement>}
|
||||
:face-1 {:kind :symbol :of :face-1 :parent :face :z "a0"}
|
||||
:face-2 {:kind :symbol :of :face-2 :parent :face :z "a1"}}}
|
||||
:face-1 {:nodes {:head {...} :mouth {:parent :head ...} ...}}
|
||||
:face-2 {:nodes {:head {...} :mouth {:parent :head ...} ...}}}
|
||||
|
||||
:features
|
||||
{:face-1/mouth {:subject :face-1 :timeline :face-1 :area :mouth
|
||||
:nodes [:mouth :mouth-in]}
|
||||
:face-2/mouth {:subject :face-2 :timeline :face-2 :area :mouth
|
||||
:nodes [:mouth :mouth-in]}}
|
||||
```
|
||||
|
||||
The example omits ordinary ids, frame counts and channel details.
|
||||
|
||||
## What belongs where
|
||||
|
||||
- A **subject** identifies a source track and supplies shared measurement settings.
|
||||
Its timeline has the same id and contains its measured `:head`.
|
||||
- A **feature** owns nodes in an explicitly named timeline. Its clip-level id is
|
||||
qualified when generated; ownership is read from fields, never parsed from ids.
|
||||
- A **node** has a local name. Parents, stencils and pose groups use local names too.
|
||||
- An **instance** places and retimes a drawing. Its pose tracks can hold one face's
|
||||
mouth while the other face continues moving.
|
||||
|
||||
Subject metadata and drawing timelines remain separate facts. Hand-drawn timelines
|
||||
need no subject. Features retain explicit timeline references, so their locations
|
||||
are not inferred from their labels.
|
||||
|
||||
Both filmed faces share one source-to-stage transform. Fitting them independently
|
||||
would stack them at the center. Additional placement uses each instance's ordinary
|
||||
channels. Cross-face draw order is the instances' `:z` order.
|
||||
|
||||
## Consequences
|
||||
|
||||
Regeneration updates the addressed timeline directly. There is no temporary swap
|
||||
into `:main`, no special stage regeneration path, and no renaming of parents or
|
||||
stencils. Composing a stage moves the take's root into the library and preserves
|
||||
its child timelines. Settings appear once per tracked object, since all placements
|
||||
read that same drawing.
|
||||
|
||||
Presence masks inside a subject use local feature names, matching measurement.
|
||||
Freeze qualifies them when building block descriptors. Head and retained-source
|
||||
blocks explicitly name their subject; otherwise two faces with identical detection
|
||||
masks could produce different bytes under the same key. Analysis addresses also
|
||||
include detection capacity and assignment settings, so old single-face detection
|
||||
results cannot satisfy a new multi-face request.
|
||||
|
||||
Validation counts node ownership by `[timeline node]`. The server requires one
|
||||
complete set of retained source roles **per subject**, rather than exactly three
|
||||
blocks for the entire analysis.
|
||||
|
||||
Nested rectangles retain fractional sizes until rasterization. Rounding inside a
|
||||
face timeline discarded small head-local pupils before the source-to-stage scale
|
||||
was applied. This was a real rendering error missed by the earlier proposal's
|
||||
coordinate-only benchmark; the regression now compares all mark extents as well.
|
||||
|
||||
## Verification and limits
|
||||
|
||||
`frontend/test/arthur/flow/multi_face_test.cljs` exercises two distinct subjects,
|
||||
source block separation, detection and feature gaps, independent pose cuts and
|
||||
regeneration, nested stage save/load, and equivalence to a flat single-face scene.
|
||||
Existing geometry, raster, source, and regeneration tests cover the same paths.
|
||||
`clips/tests/test_api.py` checks complete source roles per subject and immutability.
|
||||
The browser suite checks rendering, pupils, playback, save/open, drawing and upload.
|
||||
|
||||
Assignment remains a nearest-centroid heuristic, with a version and distance gate
|
||||
recorded in the analysis. Reordered detections, late arrivals and gaps are tested;
|
||||
identity through crossings or long disappearances is not guaranteed. Assignment
|
||||
happens before measurement, so correcting it requires measuring again.
|
||||
|
||||
This changes the freeze and retained-source contracts. It does not migrate older
|
||||
flat captures; reanalyze their footage to use the new regeneration path. Existing
|
||||
rendering and leaf codecs still understand their node/channel representation.
|
||||
|
||||
## Next steps
|
||||
|
||||
1. Commit the verified checkpoint: 319 frontend tests, 44 API tests, browser
|
||||
checks, and app/test builds passed. Builds reported no warnings.
|
||||
2. Exercise real two-person footage, especially crossings, late arrivals and
|
||||
disappearances. Check assignment before treating the resulting geometry as
|
||||
evidence about the representation.
|
||||
3. Build performance-pose selection and instance-scoped picture rates, reusing
|
||||
the existing held-frame lookup and pose groups.
|
||||
4. Then build plate-drawing selection and independent tracing references.
|
||||
|
||||
The [timing handoff](timing-handoff.md) owns the detailed next implementation
|
||||
sequence. Older flat captures need reanalysis unless a migration is separately
|
||||
undertaken to preserve their authored edits.
|
||||
461
docs/port-plan.md
Normal file
461
docs/port-plan.md
Normal file
|
|
@ -0,0 +1,461 @@
|
|||
# arthur — port plan and handoff
|
||||
|
||||
Self-contained. You should not need any prior conversation to execute this.
|
||||
|
||||
**Implementation status (2026-09-29):** steps 0–9 are in. Step 6 reads extracted
|
||||
footage, detects landmarks with local MediaPipe assets at full source cadence, and
|
||||
runs the same freeze path as the synthetic take. The scene time map can sample the
|
||||
frozen roto at a lower picture fps without changing source analysis, duration or
|
||||
audio. Step 7 adds dense eyelids, shared gaze, brows and pixel-derived teeth.
|
||||
Step 8's data model has stable feature identity, feature-level presence, explicit
|
||||
eye pairs and shared parameter definitions; a manifest can supply known feature
|
||||
absence intervals through measurement and freeze. Step 9 adds the Django backend,
|
||||
the three-tier split, content-addressed tier 2 with the detector version inside
|
||||
every key, leaf addressing for tier 1, and project load/save that round-trips.
|
||||
|
||||
Step 8 now has parameter controls and scoped regeneration from retained source.
|
||||
Multi-face representation is complete: each tracked subject has a drawing
|
||||
timeline, placed by an ordinary symbol instance. See
|
||||
[multi-face representation](multi-face-representation.md) for the implemented
|
||||
model, verification and compatibility limits.
|
||||
|
||||
**Next, in order:** commit the verified checkpoint; exercise real two-person
|
||||
footage, including crossings and disappearances; build performance-pose
|
||||
Suggest/Keep/Drop and instance-scoped picture rates; then add plate-drawing
|
||||
selection and independent tracing references. The
|
||||
[timing handoff](timing-handoff.md) records current code and implementation order.
|
||||
Reopen the representation only for a concrete requirement it cannot express.
|
||||
|
||||
**Still open:** real-footage identity validation, automatic per-feature detection,
|
||||
the timing and tracing work above, and time-varying parameter settings. Older flat
|
||||
captures need reanalysis for the new regeneration path; no migration is included.
|
||||
The step descriptions below retain the original port scope; this status and the
|
||||
linked handoffs describe subsequent work.
|
||||
|
||||
## What arthur is
|
||||
|
||||
A tool that turns live-action video into 2D animation that reads as
|
||||
hand-authored: flat polygons, a tiny indexed palette, hard edges, 320×200, no
|
||||
antialiasing, motion carried by silhouette. It tracks a face out of a clip,
|
||||
reduces the lip contour to a handful of vertices, derives teeth from image
|
||||
content, and renders flat indexed fills.
|
||||
|
||||
It currently works, as vanilla JS ES modules with no build step. `python3
|
||||
serve.py`, open `127.0.0.1:8777`. **Synthetic take** exercises everything below
|
||||
detection with no video needed.
|
||||
|
||||
This plan converts it to ClojureScript + re-frame, restructured around one
|
||||
uniform animation data model, and adds a Django backend for persistence and
|
||||
(later) collaboration.
|
||||
|
||||
## Status of the existing documents
|
||||
|
||||
| File | What it is | Authority |
|
||||
| --- | --- | --- |
|
||||
| `js/**` | the working tool, ~4,800 lines | **authoritative.** The comments encode bugs that actually happened. |
|
||||
| `docs/animation-model.md` | the target data model: nodes, channels, symbols, time maps | build to this |
|
||||
| `docs/architecture.md` | module layout, stages, sync and baking design | build to this; much of it is future scope |
|
||||
| `docs/design.md`, `README.md` | prior synthesis by an earlier agent | useful, **not authoritative**. Revise freely. Do not treat its aesthetic claims as settled requirements. |
|
||||
|
||||
Where a document and the code disagree, the code wins, and the invariant list
|
||||
below is lifted from the code for exactly that reason.
|
||||
|
||||
## Target repo layout
|
||||
|
||||
Both halves live here. Django at the root, because `manage.py` at the root is the
|
||||
convention and keeps every `python manage.py` invocation working with no `cd`.
|
||||
|
||||
```
|
||||
arthur/
|
||||
mise.toml toolchain for both halves
|
||||
manage.py
|
||||
requirements.txt
|
||||
server/ Django project: settings, urls, asgi, wsgi
|
||||
clips/ Django app: models, views, consumers, routing, migrations
|
||||
frontend/ the CLJS app
|
||||
shadow-cljs.edn
|
||||
package.json
|
||||
src/arthur/** namespace root stays arthur.* whatever the dir is called
|
||||
test/arthur/**
|
||||
static/arthur/js/ shadow-cljs output, collected by Django staticfiles
|
||||
static/arthur/audio.wav the synthetic take's clock. NOT extract.sh's output —
|
||||
that is tier 3 and lives in the blob store
|
||||
var/blobs/ the content-addressed blob store: tiers 2 and 3. Gitignored
|
||||
docs/
|
||||
js/ index.html serve.py extract.sh the old tool — see "the oracle"
|
||||
```
|
||||
|
||||
The namespaces step 9 added, since the list under **Namespaces** in
|
||||
`docs/architecture.md` predates them:
|
||||
|
||||
```
|
||||
domain/sha256.cljs SHA-256, synchronous and pure, byte-compatible with hashlib
|
||||
domain/canon.cljs the one canonical text for a descriptor, so hashing it means
|
||||
something
|
||||
domain/leaf.cljs leaf addressing: the document as path -> value
|
||||
domain/wire.cljs transit for tier 1, base64 for tier 2
|
||||
domain/project.cljs clip <-> the document and blocks that travel
|
||||
flow/address.cljs tier-2 keys, and the invalidation table they are built from
|
||||
fx/http.cljs the only namespace that talks to the server
|
||||
events/project.cljs save and open
|
||||
```
|
||||
|
||||
`clips` is a naming call, not a constraint — it is the Django app holding
|
||||
Project, Clip, Footage, Analysis, Leaf and Revision. Rename in one line if
|
||||
something fits better.
|
||||
|
||||
Dev runs two processes and they do not talk to each other: Django serves the page,
|
||||
`shadow-cljs watch app` rebuilds into `static/arthur/js`, which is already
|
||||
`:output-dir` in `shadow-cljs.edn`. `:dev-http` is gone.
|
||||
|
||||
```sh
|
||||
mise exec -- python manage.py runserver 8778 # from the repo root
|
||||
cd frontend && mise exec -- npx shadow-cljs watch app
|
||||
```
|
||||
|
||||
## Toolchain
|
||||
|
||||
`mise install` from the repo root. `mise.toml` pins java 21+, node 20, clojure,
|
||||
python 3.12, and creates `.venv`.
|
||||
|
||||
Verified to resolve cleanly: `reagent 1.2.0`, `re-frame 1.4.3`, current
|
||||
shadow-cljs.
|
||||
|
||||
## Scope
|
||||
|
||||
**In:** analysis → keyframes → playback. The pure numeric core, the animation
|
||||
data model, a player, the measurement stages, and freezing measurements into
|
||||
channels.
|
||||
|
||||
**Out, and do not build it:** paint and cels; `suggest` (it only decides which
|
||||
frames get a hand-drawn cel, so it has no job until drawing exists); the timeline
|
||||
and sequences; symbols and the plate library; multiplayer; the override layer.
|
||||
Each is designed for in `docs/architecture.md` and `docs/animation-model.md`.
|
||||
Leave the `:over` field present and empty; leave `:symbol` out entirely.
|
||||
|
||||
## The data model
|
||||
|
||||
Full specification in `docs/animation-model.md`. The subset to build:
|
||||
|
||||
```clojure
|
||||
;; The scene is a flat map of id -> node. Parent pointers, never nested maps.
|
||||
{:id :mouth :kind :poly :parent :head :z "a3" :stencil nil :span [0 240]
|
||||
:time {:mode :inherit} ; or {:mode :map :expose 2 :offset -1 :rate 1.0}
|
||||
:channels
|
||||
{[:xform :pos] {:animated? false :value [0.0 0.0]}
|
||||
[:xform :rot] {:animated? false :value 0.0}
|
||||
[:xform :scale] {:animated? false :value [1.0 1.0]}
|
||||
[:xform :skew] {:animated? false :value [0.0 0.0]}
|
||||
[:xform :anchor] {:animated? false :value [0.0 0.0]}
|
||||
[:geom :pts] {:animated? true :interp :hold
|
||||
:dense {:store "sha256:…" :offset 0 :stride 40 :frames 600}
|
||||
:generated {:by :roto/lips-outer :analysis "sha256:…"
|
||||
:params {:verts 8 :contour-avg 1}}
|
||||
:over []}
|
||||
[:style :color] {:animated? false :value :skin-dark}
|
||||
[:vis] {:animated? false :value true}}}
|
||||
```
|
||||
|
||||
Three channel shapes, one accessor `(value-at channel f)`:
|
||||
|
||||
- `{:animated? false :value v}` — static. A thing that simply exists.
|
||||
- `{:animated? true :interp :hold :keys {0 v, 4 v}}` — sparse, authored, in the
|
||||
document. **Keys are a map by frame, never a vector.** Store a plain map
|
||||
(transit loses sortedness) and build the sorted index in the resolver.
|
||||
- `{:animated? true :interp :hold :dense {...}}` — generated, one value per
|
||||
frame, in a typed array outside app-db.
|
||||
|
||||
`:generated` is provenance and **the renderer never reads it.** It is what the UI
|
||||
uses to offer a parameter panel instead of raw keys. It lives on the *channel*,
|
||||
not the node, because a node wants a rotoscoped `[:geom :pts]` and a
|
||||
hand-animated `[:xform :pos]` at the same time.
|
||||
|
||||
`:skew`, `:span`, `:anchor` and `:over` stay in the shape even though nothing
|
||||
drives them yet: each is a component of a decomposition or of a composition
|
||||
order, and adding one later migrates every stored transform.
|
||||
|
||||
Transform composition, per node:
|
||||
|
||||
```
|
||||
local = T(pos) · T(anchor) · R(rot) · K(skew) · S(scale) · T(-anchor)
|
||||
world = world(parent) · local
|
||||
```
|
||||
|
||||
## What the prototype knows that you would otherwise rediscover
|
||||
|
||||
**The JS is a prototype.** Its conclusions about what looks right are provisional
|
||||
and you may revisit any of them; several contradict each other already. But a few
|
||||
things in it are not taste — they are facts about MediaPipe, about the maths, or
|
||||
about what an operation means — and those cost real time to rediscover.
|
||||
|
||||
### Mechanical. Getting these wrong produces wrong output, not a different look.
|
||||
|
||||
1. **MediaPipe's normalised space is anisotropic.** It divides x by image *width*
|
||||
and y by *height*, so equal numbers do not mean equal pixels. Multiply x by
|
||||
`aspect = W/H` before any fit, or a "similarity" fitted in that space is not
|
||||
one and head roll comes out subtly wrong. When mapping pixels for an underlay,
|
||||
**both** axes divide by `imgH`.
|
||||
2. **MediaPipe's left/right naming is viewer-relative in some places and
|
||||
subject-relative in others.** Any left/right pairing read off a table is a coin
|
||||
flip, and a swap looks *almost* right — each eye still has an iris roughly
|
||||
where it belongs — so it survives inspection. Resolve it from geometry.
|
||||
3. **Ring tables are ordered traversals**, and slot position is the vertex's
|
||||
identity. That is what makes temporal correspondence possible at all, whatever
|
||||
you decide the shapes should look like. `subsampleSlots` returns ring
|
||||
*positions*, not landmark ids.
|
||||
4. **A wrongly-ordered ring self-intersects, and it is invisible at odd vertex
|
||||
budgets and obvious at even ones.** If you keep ordered rings, assert
|
||||
simplicity in a test; no amount of looking will catch it reliably.
|
||||
5. **Scaling a ring to thicken it collapses when the ring is degenerate** — a shut
|
||||
eyelid scaled by 1.1 is still shut, so the lash line vanishes on exactly the
|
||||
frames where it is the whole drawing. A fixed radial offset does not. Maths,
|
||||
not taste.
|
||||
6. **A fractional centre for a small integer-sized shape changes its size.**
|
||||
Round the origin, not the extents, or a 3px mark is 3px on one frame and 4px on
|
||||
the next.
|
||||
7. **Order of operations on time:** flooring onto a grid and shifting against the
|
||||
clock do not commute. Shift first and the floor discards it on most frames.
|
||||
|
||||
### Choices the prototype made. Revisit freely; here is what each was for.
|
||||
|
||||
| Choice | Its stated reason | How you would learn it was wrong |
|
||||
| --- | --- | --- |
|
||||
| similarity (4 DOF), not affine | extra DOF absorbs out-of-plane head rotation as shear and smears it into the mouth | the residual readout stops responding to head turn |
|
||||
| reference is the Procrustes mean over the shot, not frame 0 | no single frame's idiosyncrasies get baked into every other | one frame's detection error biases the whole take |
|
||||
| smooth the transform, not the contour | sparse keys at velocity minima rejected detector noise for free | it was already broken by a "bounded exception" once keys went dense, so it was never a law |
|
||||
| gaze measured against the eye's corner midpoint | measured against the lid, every blink drags the origin down and fakes a glance at the floor | gaze correlates with blinks |
|
||||
| one gaze shared by both eyes | at this size the per-eye difference is noise, and independent noise reads as wall-eyed | a wink or a real vergence is lost |
|
||||
| hold, never interpolate | a tweened mouth reads as puppet software | motion looks stepped rather than snappy |
|
||||
| palette indices, never sampled RGB | sampling colour produces a pixel-art filter irrecoverably | — |
|
||||
|
||||
These are where to look first if the output is wrong. They are also where to look
|
||||
first if you want to change the look.
|
||||
|
||||
## Conventions
|
||||
|
||||
- `domain/*` may not require `flow/*`; neither may require `re-frame`.
|
||||
- Every flow function is `(f params inputs) -> output`. No state, no db, no atoms.
|
||||
- Nothing below `subs/` calls `subscribe`.
|
||||
- Every analysis function that reads pixels takes a `debug?` flag and returns its
|
||||
intermediate masks alongside its result, the way
|
||||
`interior.js/extractTeeth(..., wantDebug)` already does.
|
||||
- Port the invariant comments across verbatim. They are the most valuable text in
|
||||
the repo.
|
||||
|
||||
## The oracle
|
||||
|
||||
**Keep `js/`, `index.html` and `serve.py` in the tree through step 5.** They cost
|
||||
nothing, `serve.py` still runs the old tool, and they are the numeric oracle:
|
||||
run both implementations on the same synthetic track and diff.
|
||||
`fit-similarity` and `procrustes-mean` should agree to **1e-9**; a larger gap is a
|
||||
port bug, not float noise.
|
||||
|
||||
**Parity proves the port is faithful, not that the answer is right.** The JS is a
|
||||
prototype, so keep the two kinds of test apart: a *parity* test pins behaviour
|
||||
while you move it, and is deleted once the move is done; a *correctness* test
|
||||
asserts something you have decided you want, and stays. Conflating them bakes the
|
||||
prototype's mistakes into the rewrite and makes them permanent. Delete them in one commit once the CLJS player renders
|
||||
the synthetic take correctly.
|
||||
|
||||
**Do not port the debug views** (`drawPanes`, `drawInteriorDebug`,
|
||||
`drawEyeOverlay`, `drawGazeDebug` in `js/app.js`). The knowledge in them is not
|
||||
the canvas calls — it is *which things you must see to tune teeth*: the source
|
||||
crop, the in-region mask, the surviving mask, and the local contour. That contract
|
||||
already exists as `extractTeeth(..., wantDebug)` returning
|
||||
`debugCanvas(src, inReg, mask, pw, ph, local)`. **Port the payload, skip the
|
||||
drawing.** Redrawing it is ten lines whenever it is wanted.
|
||||
|
||||
## Steps
|
||||
|
||||
Each step ends somewhere runnable. Do not proceed past a step whose "done" does
|
||||
not hold.
|
||||
|
||||
### 0 — scaffold and the oracle
|
||||
`mise install`. Create `frontend/` with shadow-cljs, reagent, re-frame. Port
|
||||
`synth.js` (the synthetic landmark generator, including its `swapIris` flag) and
|
||||
the numeric assertions from `selftest.js` to `cljs.test`.
|
||||
|
||||
**Done:** the suite runs and fails informatively.
|
||||
|
||||
### 1 — the pure bottom
|
||||
Port verbatim: `landmarks.js` → `domain/landmarks`, `mathutil.js` → `domain/geom`,
|
||||
ring helpers → `domain/ring`, `raster.js` → `domain/raster`, the palette →
|
||||
`domain/palette`.
|
||||
|
||||
**Done:** tests pass, including ring simplicity and the swapped-iris vote. Numeric
|
||||
agreement with the JS to 1e-9. Nothing renders.
|
||||
|
||||
### 2 — the data model, with no analysis in it
|
||||
`domain/channel` (`value-at` across all three shapes, plus a per-channel cursor),
|
||||
`domain/node` (transform composition), `domain/scene` (topological order by parent
|
||||
depth, `eval-frame` → draw ops in z order).
|
||||
|
||||
Hand-write a scene in EDN — a rectangle parented to a group whose
|
||||
`[:xform :pos]` is keyed on four frames — and render it through `domain/raster`
|
||||
into a canvas.
|
||||
|
||||
This is deliberately before any analysis. **The data model has never been
|
||||
validated; find out here**, with fifty lines to throw away, rather than after
|
||||
porting nine hundred lines of measurement into a shape that does not work.
|
||||
|
||||
**Done:** something moves on screen.
|
||||
|
||||
### 3 — the player
|
||||
`clock` (audio-clocked: `frame = ⌊currentTime · fps⌋`, so a slow loop drops frames
|
||||
instead of drifting; ½× and ¼× come free from `playbackRate`), the rAF loop, a
|
||||
`::resolver` sub, and transport UI.
|
||||
|
||||
The loop reads and blits and **dispatches nothing**. The sub yields a resolver
|
||||
closure; the loop applies it at the playhead. The playhead itself lives in app-db
|
||||
like everything else — with layer-2 extractors and layer-3 computations, a
|
||||
playhead tick does not invalidate the expensive stages.
|
||||
|
||||
**Done:** the hand-written scene plays at 30fps against audio, scrubs, and runs at
|
||||
½× and ¼×.
|
||||
|
||||
### 4 — measure: anchor and mouth
|
||||
Port `stabilize` and the lip rings out of `pipeline.js`. Split **condition**
|
||||
(`smoothTransforms`, `smoothContours`) into its own stage so the two smoothing
|
||||
knobs do not re-run measurement.
|
||||
|
||||
**Done:** measured numbers match the JS on the synthetic track. Note that
|
||||
parity here is on `stabilize`'s output, not on `toRasterRing`'s — the framing
|
||||
step is being deleted, not ported.
|
||||
|
||||
### 5 — freeze
|
||||
The new module, and the heart of this work: measurements → channels. A dense
|
||||
`[:geom :pts]` block per node — `Int16Array[frames × verts × 2]` in the node's
|
||||
own local space, with the block's fixed-point scale in its header — plus
|
||||
`:generated`. Fixed topology is what makes this a rectangular array with no
|
||||
per-frame header.
|
||||
|
||||
**Do not port `makeXform`.** The prototype bakes the framing into the stored
|
||||
numbers: it centres on the face oval's bbox and zooms until the face is 80% of
|
||||
the raster height, so every vertex carries a cropping decision made once from one
|
||||
frame's landmarks. Dropping it is a deletion. Placement becomes `[:xform :*]` on
|
||||
an authored `:face` node, the stage clips whatever hangs off, and project
|
||||
dimensions stop being tied to the footage. See "What space geometry is in" in
|
||||
`docs/animation-model.md`.
|
||||
|
||||
The anchor transform freezes onto `:head`, one level under `:face`, and the
|
||||
normalise on/off/per-plate toggle is which of the three channel shapes that node
|
||||
carries. Always measure and always store factored, whatever the toggle says:
|
||||
smoothing and velocity-minimum key selection both require the split to exist in
|
||||
storage.
|
||||
|
||||
**Done:** the synthetic take plays back as a moving mouth. Full vertical slice.
|
||||
|
||||
### 6 — detect
|
||||
MediaPipe interop behind one namespace; real frames, real audio, real fps from
|
||||
the manifest. **Vendor the wasm** rather than fetching from jsdelivr — it is
|
||||
currently the only thing in the tool that silently requires a network.
|
||||
|
||||
Decode every source frame for analysis. A lower output picture fps is a time map
|
||||
over frozen channels, not a reduced detection track. Selecting source frames to
|
||||
trace into cels is independent again and remains outside this port's paint scope.
|
||||
|
||||
**Done:** real footage plays back as a rotoscoped mouth.
|
||||
|
||||
### 7 — the rest of measure
|
||||
Eyes (openness, gaze, iris pairing vote, blink resolution with its `hold`), brows
|
||||
(raise and tilt at both ends, both correspondence votes), interior (otsu,
|
||||
morphology, components, radial contour). Each keeps its `debug?` payload.
|
||||
|
||||
Two things fall out of the model instead of being written: the brow's
|
||||
measure-the-height-out-and-put-it-back is `[:geom :pts]` plus `[:xform :pos]`, two
|
||||
channels on one node; and the iris is a `:disc` node parented to the lid ring and
|
||||
stencilled by the sclera.
|
||||
|
||||
**Done:** the same face parts are measured and rendered through the CLJS scene,
|
||||
minus paint. The fixed pixel thresholds remain provisional; step 8 exposes their
|
||||
parameters for tuning without changing the source track or picture timing.
|
||||
|
||||
### 8 — knobs — DONE for static settings and scoped regeneration
|
||||
Build the parameter model before its UI. Define each parameter once with its
|
||||
default, validation, applicable area and regeneration dependencies. Store values
|
||||
by stable subject and feature ID. Represent an eye pair as one group with one or
|
||||
two eye member IDs from the same subject; a profile view with one identified eye
|
||||
needs no invented partner. Each eye may override a pair value. Removing an eye
|
||||
from a pair materialises its effective values so playback does not change. Keep the
|
||||
existing frozen channels as renderer input; settings and provenance do not enter
|
||||
the render path.
|
||||
|
||||
Carry feature-level presence through freeze and dense channel state. The same
|
||||
feature ID covers every observed run across occlusion; a missing measurement
|
||||
has no channel value on that frame. Full-face detection is the fallback mask
|
||||
until there is a feature-level detector or authored presence data. A shared gaze
|
||||
measurement may still feed two independently identified eyes. A small manifest
|
||||
annotation can supply feature absence intervals now: the loader expands them
|
||||
before measurement, so invalid eye landmarks are ignored and gaze uses the
|
||||
visible eye. This is an input format, not a control UI or an automatic detector.
|
||||
|
||||
Use leaf-addressable settings under the clip, subject, feature and optional
|
||||
group. Retain source measurements so a setting change can regenerate affected
|
||||
channels without re-detecting footage. Static parameter controls and scoped
|
||||
regeneration are implemented for takes and composed stages. Time-varying
|
||||
parameter values remain deferred.
|
||||
|
||||
### 9 — backend — DONE
|
||||
Django project, the `clips` app, models for
|
||||
Project/Clip/Footage/FootageFrame/Analysis/Block/Leaf/Revision/Blob, and project
|
||||
load/save. Round-tripping a project through the server is the proof the model
|
||||
serialises.
|
||||
|
||||
Django was the easy half; the tier split was the work. What it came to:
|
||||
|
||||
**Tier 2 keys are content addresses over every input**, and the detector version
|
||||
is in every one of them, through the analysis id that each block descriptor names.
|
||||
`flow/address`'s `block-knobs` is the invalidation table, and it is not trusted:
|
||||
`address-test` re-freezes the take once per knob and asserts the biconditional —
|
||||
a block's bytes changed if and only if its key changed. That test found two
|
||||
things reading the code would not have. `brow-pos` does not depend on
|
||||
`contour-avg`, because the brow RING is smoothed and the raise is not. And the
|
||||
first version of the test was itself wrong: a 3% perturbation of `gaze-gain`
|
||||
moves every sample inside the grid cell `quantize-snap` had already rounded it
|
||||
into, so the bytes came out identical and the knob looked like an input the block
|
||||
did not have.
|
||||
|
||||
**The server verifies what it is handed.** It recomputes every key from the
|
||||
descriptor stored beside it and refuses a mismatch, refuses an analysis that does
|
||||
not declare a detector version, and refuses a document naming blocks it does not
|
||||
hold. It hashes the descriptor TEXT rather than re-rendering it from parsed
|
||||
values, because JS prints an integral double as `1` and Python as `1.0` — a
|
||||
scheme where both sides re-render breaks on the first parameter whose value
|
||||
happens to be whole.
|
||||
|
||||
**Tier 3 is served by hash.** `extract.sh` still decodes; `manage.py
|
||||
ingest_bundle` hashes the result into the blob store, by hard link. The manifest
|
||||
the client receives now carries a URL per frame, so the frame layout stopped being
|
||||
a shared secret between a shell script and a ClojureScript namespace. The
|
||||
cache-busting `?v=` on every frame URL went with it: a blob's name is the hash of
|
||||
its bytes, so a stale copy is not a thing that can happen.
|
||||
|
||||
**Leaf addressing exists**, with conditional writes and a monotonic project
|
||||
version, so the sync design has nothing to retrofit. The socket, presence and
|
||||
broadcasts are still out of scope.
|
||||
|
||||
Two loose ends from step 8 closed on the way. `pack` no longer takes a
|
||||
`(track, frame)` predicate whose call sites each derived a feature from an index —
|
||||
every track names the feature it follows, which deleted five hand-maintained
|
||||
mappings and handed `flow/address` the same list for its observation digest. And
|
||||
the `presence-check` binding in a `let` nobody read is now an ordinary `doseq`.
|
||||
|
||||
**Output is not in this plan.** The `.take` writer in `js/take.js` was for
|
||||
driving an Animator Pro render script and it is not where this is going: the
|
||||
target is encoding video in the browser, and that is a separate piece of design
|
||||
nobody should pre-empt by porting the old one.
|
||||
|
||||
## Two things to not foreclose
|
||||
|
||||
Feature controls now handle more than one face; editing presence remains future work.
|
||||
The underlying identity, occlusion and group association model begins in step 8:
|
||||
|
||||
- **Presence is not visibility.** An occluded feature has *no value* on a frame,
|
||||
which is different from a part being hidden. Dense blocks carry a per-track,
|
||||
per-frame absence mask; `[:vis]` remains the sole hiding mechanism.
|
||||
- **Params carry stable identity.** A subject and its features keep their IDs
|
||||
across observation gaps. A run of visible frames is not a new identity.
|
||||
|
||||
The current identity tracker uses nearest-centroid assignment. Validate it on
|
||||
real crossings and disappearances before choosing a more elaborate policy; the
|
||||
iris and brow correspondence code offers whole-take voting as one option.
|
||||
128
docs/timing-handoff.md
Normal file
128
docs/timing-handoff.md
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
# Timing and frame-selection handoff
|
||||
|
||||
Status (2026-09-29): the multi-face representation is complete. Each face has a
|
||||
local drawing timeline and an ordinary symbol instance. Keep that model; the next
|
||||
feature is performance-pose selection, followed by plate drawings and tracing.
|
||||
See [multi-face representation](multi-face-representation.md) for verification
|
||||
and compatibility limits.
|
||||
|
||||
## Next steps, in order
|
||||
|
||||
1. **Commit the verified checkpoint.** Representation, scoped regeneration,
|
||||
nested stage composition and source persistence are implemented and tested.
|
||||
2. **Exercise real two-person footage.** Include crossings, late arrivals and
|
||||
disappearances. Assignment is still a nearest-centroid heuristic; inspect
|
||||
whether identities, landmarks and mouth crops stay together. Correcting an
|
||||
assignment requires measuring again. Do not redesign the representation to
|
||||
compensate for an assignment failure.
|
||||
3. **Build performance-pose selection.** Propose frames from a target picture
|
||||
rate, allow explicit Keep/Drop edits, and apply requests per instance. Reuse
|
||||
the existing held-frame lookup and generated pose groups. Keep authored keys
|
||||
and audio timing intact.
|
||||
4. **Then build plate drawings and tracing.** Suggest drawing frames from head
|
||||
displacement, allow manual choices, and give each cel an independently
|
||||
selectable tracing reference.
|
||||
|
||||
Older flat captures need reanalysis for the new regeneration path. Migrating their
|
||||
existing authored edits is separate work; it is not implemented by this checkpoint.
|
||||
|
||||
## Timing decisions
|
||||
|
||||
Keep the dense analyzed frames. Generated motion holds the most recent selected
|
||||
source pose; removing a selected pose never deletes source data or shortens the
|
||||
clip. Store edits in the animation's local frame space, so moving an instance
|
||||
does not move its edits. Authored keys follow intentional instance retiming but
|
||||
must not be quantized by a picture-rate request. Clip FPS and audio duration stay
|
||||
fixed.
|
||||
|
||||
There are two selections with different owners, sharing held-frame lookup:
|
||||
|
||||
- **Performance poses:** propose a kept-frame list from the target picture rate,
|
||||
then apply explicit keep/drop edits. A parent instance may request a lower
|
||||
rate. Mouth outline, interior, teeth and visibility read the same selected
|
||||
source frame; likewise each eye's coupled parts. Use group overrides when
|
||||
needed, rather than a setting on every channel. Head motion currently has its
|
||||
own anchor selection; do not silently put it under mouth timing.
|
||||
- **Plate drawings:** start with frame 0, walk measured rigid head poses, and
|
||||
suggest a frame when maximum landmark displacement from the last kept pose
|
||||
exceeds tolerance. Let the artist add/remove frames. A removed drawing stays
|
||||
stored so it can reappear if restored. This selection does not thin the mouth.
|
||||
|
||||
A target rate is approximate. Pin a useful closed-mouth pose at its actual frame,
|
||||
even if that produces more changes than the target. Do not show a future pose
|
||||
early to fit a grid. Manual drop wins over an automatic suggestion; make removal
|
||||
of the only closed pose in a beat visible in the UI. Skip missing detections when
|
||||
suggesting a replacement. Keep a frame-zero selection and hold the last selection
|
||||
through the end. A skipped pose (hold), `[:vis] false` (hidden), and an absent
|
||||
measurement remain different facts.
|
||||
|
||||
Store manual edits separately from generated proposals so changing the rate or
|
||||
tolerance retains hand decisions. Selection edits change the document, not dense
|
||||
blocks or analysis addresses. Verify save/open for every new field; extend leaf
|
||||
handling and the relevant key whitelist if its storage location requires it.
|
||||
|
||||
## Current code: reuse these mechanisms
|
||||
|
||||
- `domain/pose.cljs` already has `prepare`, `held-frame` and `source-frame`.
|
||||
Instance `:playback :tracks` map local change frames to held source frames,
|
||||
keyed by pose group. Reuse this lookup; frame suggestion and Keep/Drop policy
|
||||
are the missing layer. An explicit cut is not itself a complete selection UI.
|
||||
- `freeze/performance-nodes` marks generated animated channels with
|
||||
`:pose-sampled?` and local `:pose-group` names. This includes keyed visibility
|
||||
as well as dense geometry. `:generated` remains provenance for regeneration.
|
||||
- `timeline/channel-frame` already applies explicit pose choices and default
|
||||
picture sampling to marked channels. Playback and export both use
|
||||
`clip/resolver` with `:picture-fps`; there is no need for a second sampling
|
||||
implementation. Export's pose count is still a rate-based estimate.
|
||||
- The picture-rate option is currently passed through the resolver tree
|
||||
unchanged. Instance-specific parent requests are still to be implemented.
|
||||
Instance offset/rate must apply before selecting the local source pose.
|
||||
- `pose/put-cut` and `remove-cut` currently address instances in `:main`.
|
||||
A take's face instances are there, but a composed stage nests them inside a
|
||||
shared source timeline. Make the editing scope explicit when adding nested
|
||||
controls. A request on one outer placement must not rewrite the shared
|
||||
drawing's playback settings for every placement.
|
||||
- Generic root `:time :expose` still retimes descendants, and frozen takes still
|
||||
store it. Paint nodes are rootless to escape it. When the selection path
|
||||
replaces take picture cadence, remove that redundant quantization from the
|
||||
take default; preserve intentional generic time maps. Moving exposure to
|
||||
`:head` would still retime authored children.
|
||||
- `freeze/head-mode` supports `:free` and `:anchored`. It keeps measured channels
|
||||
dense and writes optional per-subject `:anchors` maps; it does **not** implement
|
||||
`:per-plate` mode or materialize transform keys from `:kept`. Plate selection
|
||||
should reuse held measured-frame addresses where appropriate, without
|
||||
rerunning analysis or copying the measurements.
|
||||
- `:over` hand corrections are currently refused by the channel reader. Their
|
||||
future application belongs after generated pose selection.
|
||||
|
||||
## Performance-pose implementation sequence
|
||||
|
||||
1. Add pure proposal and Keep/Drop policy around the existing held-frame lookup.
|
||||
Cover frame zero, nondivisible rates, manual precedence, missing poses and a
|
||||
protected mouth closure. Preserve all source frames.
|
||||
2. Feed instance requests and group selections into the existing channel read
|
||||
path. Cover two faces, two differently timed placements of one source, nested
|
||||
instances, coupled visibility/geometry, and authored keys at their normal time.
|
||||
Use this same path for preview and export; keep audio duration unchanged.
|
||||
3. Wire the performance strip's Suggest/Keep/Drop controls and persistence.
|
||||
Replace the export pose estimate with the actual selection count. Retire the
|
||||
take's redundant root exposure only when this path replaces its behavior.
|
||||
|
||||
## Plate drawings and tracing, afterward
|
||||
|
||||
The old suggestion algorithm is `js/pipeline.js:suggestPlateFrames`; the strip,
|
||||
worksheet and tracing photo are in `js/app.js`. Port the useful policy over the
|
||||
measured head poses and reuse held-frame lookup for the resulting drawing set.
|
||||
|
||||
Give a cel an editor-only source-frame reference, defaulting to its plate frame
|
||||
but independently changeable. It may point to a frame omitted from either rendered
|
||||
selection. Register the photo using that source frame's measured transform. The
|
||||
old prototype coupled photo and cel addresses; independent tracing is new work.
|
||||
|
||||
The old iris socket lock, gaze origin, CLJS head anchors and registration pivot
|
||||
are separate settings. Clarify what an "origin-lock" request means before adding
|
||||
that control.
|
||||
|
||||
Keep the UI to two scopes: **performance poses** and **plate drawings**, each with
|
||||
Suggest/Keep/Drop. Tracing reference and lock controls live with the cel or feature
|
||||
they affect. No general keyframe framework is needed for this work.
|
||||
86
docs/timing-model.md
Normal file
86
docs/timing-model.md
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
# Timing model
|
||||
|
||||
The source footage, authored drawings, generated face motion, and stage placement
|
||||
have different frame decisions. They share a clock but do not share one kept-frame
|
||||
list. `timing-handoff.md` records earlier implementation notes.
|
||||
|
||||
## Frame spaces
|
||||
|
||||
- A source frame addresses a decoded image and its measured face data. Keep the
|
||||
source cadence and, for variable-rate video, its presentation timestamp.
|
||||
- A timeline frame addresses authored keys in the clip or symbol's local space.
|
||||
- A stage frame is mapped through the symbol instance's offset and rate before
|
||||
local frame decisions are read. Moving a placement does not rewrite its keys.
|
||||
|
||||
The analyzed source poses remain dense. A lower picture rate or a skipped pose
|
||||
never removes source data or shortens audio.
|
||||
|
||||
## Head placement
|
||||
|
||||
Analysis fits each source frame's rigid landmarks into one common head-local
|
||||
space. Its inverse is the measured head transform, stored densely on `:head`.
|
||||
The head node has one optional anchor map:
|
||||
|
||||
```clojure
|
||||
;; no :anchors free movement: read measured frame f at f
|
||||
:anchors {0 12} ; one lock: use frame 12's transform throughout
|
||||
:anchors {0 12, 40 42} ; keyed locks: switch to frame 42 at local frame 40
|
||||
```
|
||||
|
||||
A key is `(local change frame -> measured source frame)`. Its value holds to the
|
||||
next key. The map chooses position, rotation and scale together. Frame zero must
|
||||
have a key when the map exists. The dense transform blocks remain intact, so
|
||||
editing anchors is a small document change and re-freezing can replace the
|
||||
measurements without losing the anchor choices.
|
||||
|
||||
A source image used for tracing should be registered with that image's measured
|
||||
stabilizing transform, then the selected head transform, then the authored
|
||||
`:face` placement. This makes the photo and head-local vectors share the same
|
||||
orientation and position. Tracing-photo selection is a separate editor address;
|
||||
it does not choose the head anchor.
|
||||
|
||||
The prototype stabilizes into the shot's mean rigid pose and uses an early
|
||||
closed-mouth frame for raster framing. Those are internal analysis and framing
|
||||
choices. The authored head-anchor map above controls which measured head pose is
|
||||
shown over each range. It is independent of plate drawing starts.
|
||||
|
||||
## Performance poses
|
||||
|
||||
Generated mouth, eye, and brow channels can be sampled at a lower picture rate
|
||||
without retiming authored keys. The normal rule picks the latest available pose
|
||||
at or before a picture-grid time. A future performance policy may add important
|
||||
closed-mouth poses and store manual keeps/drops separately from the rate's
|
||||
proposal. Related parts should share a selected pose by default: a mouth outline,
|
||||
interior, teeth and generated visibility must not disagree about its frame.
|
||||
|
||||
## Stage placement
|
||||
|
||||
A symbol placement has optional pose-cut tracks, separate from head anchors:
|
||||
|
||||
```clojure
|
||||
:playback {:tracks {:mouth {0 12, 8 27}
|
||||
:eye-r {0 0, 4 6}}}
|
||||
```
|
||||
|
||||
These maps are also `(local change frame -> source pose frame)`. They select which
|
||||
baked/generated shape pose appears on that placement. Before the first explicit
|
||||
cut, normal generated motion continues. Cuts hold, without interpolation, until
|
||||
the next cut. A `[:node id]` track can override one shape in a shared group.
|
||||
Authored cels, transforms, and audio remain on their normal local time.
|
||||
|
||||
The current implementation reads retained frozen channels. A separate resolved
|
||||
geometry bake is not implemented; when added, it must preserve addressable
|
||||
candidate poses so stage cuts can still select any of them.
|
||||
|
||||
## Ownership
|
||||
|
||||
| Choice | Owner | Current state |
|
||||
| --- | --- | --- |
|
||||
| Source frames and timestamps | Footage/analysis | Constant-rate frame indexing exists; variable timestamps remain future work |
|
||||
| Head anchor map | `:head` node | Implemented, stored with the node |
|
||||
| Tracing cel starts and photo address | Authored cel | Separate future work |
|
||||
| Generated picture-rate proposal and closure protection | Roto clip/symbol | Generated-only picture sampling exists; closure protection remains future work |
|
||||
| Stage pose cuts | Symbol instance | Implemented, stored with the instance |
|
||||
|
||||
Preview and export use the same resolver for generated picture sampling and stage
|
||||
cuts. Export still emits every timeline frame at the clip's audio rate.
|
||||
93
extract.sh
93
extract.sh
|
|
@ -7,32 +7,77 @@
|
|||
# pass. A PNG sequence is exact, instantly seekable, and reproducible.
|
||||
#
|
||||
# Audio comes out alongside because the page uses it as the PLAYBACK CLOCK -
|
||||
# frame = floor(audio.currentTime * fps) - so picture and sound cannot drift
|
||||
# apart no matter how long the shot is or how slow the render loop runs.
|
||||
# frame = floor(audio.currentTime * fps). Detection sees every decoded source
|
||||
# frame; a lower drawing rate is a later playback choice, never an extraction
|
||||
# choice. This script accepts CFR footage because frame-index timing needs a
|
||||
# single rate. VFR needs per-frame timestamps in the manifest first.
|
||||
set -euo pipefail
|
||||
|
||||
src="${1:?usage: ./extract.sh CLIP [FPS] [OUTDIR]}"
|
||||
fps="${2:-12}"
|
||||
out="${3:-frames}"
|
||||
|
||||
rm -rf "$out"; mkdir -p "$out"
|
||||
ffmpeg -hide_banner -loglevel warning -i "$src" -vf "fps=$fps" "$out/%04d.png"
|
||||
count=$(ls -1 "$out" | wc -l)
|
||||
|
||||
# Mono is enough for judging sync and halves the file. Absent audio is not fatal.
|
||||
if ffprobe -v error -select_streams a:0 -show_entries stream=codec_type \
|
||||
-of csv=p=0 "$src" 2>/dev/null | grep -q audio; then
|
||||
ffmpeg -hide_banner -loglevel warning -y -i "$src" -vn -ac 1 -ar 44100 audio.wav
|
||||
audio='"audio.wav"'
|
||||
echo "audio -> audio.wav"
|
||||
else
|
||||
audio='null'
|
||||
echo "no audio stream"
|
||||
src="${1:?usage: ./extract.sh CLIP [BUNDLE_DIR]}"
|
||||
bundle="${2:-.}"
|
||||
if [[ "$bundle" =~ ^[0-9]+([.][0-9]+)?$ ]]; then
|
||||
echo "The FPS argument was removed: extraction always keeps the source rate. Use a directory as argument 2." >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ "$bundle" = /* || "$bundle" = *..* ]]; then
|
||||
echo "BUNDLE_DIR must be a relative directory inside this repo" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# The page must know the true extraction rate: if it guessed, audio and picture
|
||||
# would drift. Source of truth lives here, next to the frames it describes.
|
||||
printf '{"fps":%s,"frames":%s,"dir":"%s","audio":%s,"source":"%s"}\n' \
|
||||
"$fps" "$count" "$out" "$audio" "$(basename "$src")" > manifest.json
|
||||
probe=$(ffprobe -v error -select_streams v:0 \
|
||||
-show_entries stream=r_frame_rate,avg_frame_rate,nb_frames \
|
||||
-of json "$src")
|
||||
fps=$(python3 -c '
|
||||
import json, sys
|
||||
from fractions import Fraction
|
||||
streams = json.load(sys.stdin).get("streams", [])
|
||||
if not streams:
|
||||
raise SystemExit("no video stream in source")
|
||||
s = streams[0]
|
||||
nominal = Fraction(s["r_frame_rate"])
|
||||
average = Fraction(s["avg_frame_rate"])
|
||||
if nominal <= 0 or average <= 0 or abs(float(nominal / average) - 1) > 0.001:
|
||||
raise SystemExit("variable-frame-rate source needs timestamp-aware playback; refusing to guess its fps")
|
||||
print(float(average))
|
||||
' <<< "$probe")
|
||||
|
||||
echo "$count frames at ${fps}fps -> $out/ (manifest.json written)"
|
||||
if [[ "$bundle" = "." ]]; then
|
||||
dir="frames"; audio_path="audio.wav"; manifest_path="manifest.json"
|
||||
else
|
||||
dir="${bundle%/}/frames"
|
||||
audio_path="${bundle%/}/audio.wav"
|
||||
manifest_path="${bundle%/}/manifest.json"
|
||||
fi
|
||||
|
||||
rm -rf "$dir"; mkdir -p "$dir"
|
||||
ffmpeg -hide_banner -loglevel warning -i "$src" -fps_mode passthrough "$dir/%04d.png"
|
||||
count=$(find "$dir" -maxdepth 1 -name '*.png' -type f | wc -l | tr -d ' ')
|
||||
|
||||
expected=$(python3 -c 'import json,sys; print(json.load(sys.stdin)["streams"][0].get("nb_frames", ""))' <<< "$probe")
|
||||
if [[ "$expected" =~ ^[0-9]+$ && "$count" != "$expected" ]]; then
|
||||
echo "decoded $count frames but source reports $expected; refusing an inaccurate manifest" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Mono is enough for judging sync and halves the file. A silent clock lets a
|
||||
# mute source use the same audio-driven transport.
|
||||
if ffprobe -v error -select_streams a:0 -show_entries stream=codec_type \
|
||||
-of csv=p=0 "$src" 2>/dev/null | grep -q audio; then
|
||||
ffmpeg -hide_banner -loglevel warning -y -i "$src" -vn -ac 1 -ar 44100 "$audio_path"
|
||||
else
|
||||
duration=$(python3 -c 'import sys; print(int(sys.argv[1]) / float(sys.argv[2]))' "$count" "$fps")
|
||||
ffmpeg -hide_banner -loglevel warning -y -f lavfi -i anullsrc=r=44100:cl=mono \
|
||||
-t "$duration" -c:a pcm_s16le "$audio_path"
|
||||
fi
|
||||
|
||||
# JSON escaping belongs to a JSON writer, especially for source filenames.
|
||||
python3 - "$fps" "$count" "$dir" "$audio_path" "$src" "$manifest_path" <<'PY'
|
||||
import json, os, sys
|
||||
fps, count, frames, audio, source, path = sys.argv[1:]
|
||||
with open(path, "w") as out:
|
||||
json.dump({"fps": float(fps), "frames": int(count), "dir": frames,
|
||||
"audio": audio, "source": os.path.basename(source)}, out)
|
||||
out.write("\n")
|
||||
PY
|
||||
|
||||
echo "$count source frames at ${fps}fps -> $dir/ ($manifest_path written)"
|
||||
|
|
|
|||
38
fly.toml
Normal file
38
fly.toml
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
app = "arthur"
|
||||
primary_region = "iad"
|
||||
|
||||
[build]
|
||||
dockerfile = "Dockerfile"
|
||||
|
||||
[env]
|
||||
DJANGO_DEBUG = "0"
|
||||
DJANGO_ALLOWED_HOSTS = ".fly.dev"
|
||||
DJANGO_CSRF_TRUSTED = "https://arthur.fly.dev"
|
||||
DJANGO_DB_PATH = "/data/db.sqlite3"
|
||||
DJANGO_BLOB_ROOT = "/data/blobs"
|
||||
|
||||
[[mounts]]
|
||||
source = "data"
|
||||
destination = "/data"
|
||||
initial_size = "1gb"
|
||||
|
||||
[http_service]
|
||||
internal_port = 8000
|
||||
force_https = true
|
||||
auto_stop_machines = "stop"
|
||||
auto_start_machines = true
|
||||
min_machines_running = 1
|
||||
processes = ["app"]
|
||||
|
||||
[[http_service.checks]]
|
||||
interval = "30s"
|
||||
timeout = "5s"
|
||||
grace_period = "60s"
|
||||
method = "GET"
|
||||
path = "/"
|
||||
headers = { Host = "arthur.fly.dev" }
|
||||
|
||||
[[vm]]
|
||||
cpu_kind = "shared"
|
||||
cpus = 1
|
||||
memory = "1gb"
|
||||
334
frontend/README.md
Normal file
334
frontend/README.md
Normal file
|
|
@ -0,0 +1,334 @@
|
|||
# frontend
|
||||
|
||||
The ClojureScript half. See `docs/port-plan.md` for what is being built and in
|
||||
what order; this file is only how to run it.
|
||||
|
||||
## Once
|
||||
|
||||
```sh
|
||||
mise install # from the REPO ROOT
|
||||
pip install -r requirements.txt # the Django half; one dependency
|
||||
mise exec -- python manage.py migrate # the document database
|
||||
cd frontend && npm install
|
||||
```
|
||||
|
||||
`java` must be 21+. On an older JDK shadow-cljs fails with "CompilerOptions has
|
||||
been compiled by a more recent version of the Java Runtime", which reads like a
|
||||
shadow-cljs bug and is not one. `mise install` is what prevents it.
|
||||
|
||||
## The tests
|
||||
|
||||
```sh
|
||||
cd frontend && mise exec -- npm test
|
||||
```
|
||||
|
||||
Two things: compile the `:test` build, run it under node.
|
||||
|
||||
```
|
||||
shadow-cljs compile test
|
||||
node out/node-tests.js
|
||||
```
|
||||
|
||||
Run them separately if a compile error is in the way.
|
||||
|
||||
**Run them through `mise`**, or make sure `mise`'s node is first on PATH. `java`
|
||||
must be 21+ and node 20.19+. On an nvm node 20.11 shadowing the pinned one,
|
||||
things fail in ways that read like the code being broken and are not.
|
||||
|
||||
### And the Django one
|
||||
|
||||
```sh
|
||||
mise exec -- python manage.py test clips # from the REPO ROOT
|
||||
```
|
||||
|
||||
Tests cover the API: the blob store, key verification, the load/save round trip,
|
||||
the conditional write, source analysis blocks, and video upload and extraction.
|
||||
The two groups worth
|
||||
reading are the ones that make the tier split a property of the system rather than
|
||||
a convention in ClojureScript — the server recomputes every tier-2 key it is
|
||||
handed, and refuses a block whose analysis does not declare a detector version.
|
||||
|
||||
### And the browser one
|
||||
|
||||
Step 5's done-criterion is a PICTURE, and no assertion in `cljs.test` can check
|
||||
one: a take that resolves to the right numbers and draws nothing would pass every
|
||||
test in `arthur.flow.freeze-test`. A blank canvas under a perfectly correct
|
||||
transport is the bug class unit tests miss, and it has happened here once.
|
||||
|
||||
Step 9's is a picture too, for a different reason: the ways a document survives a
|
||||
round trip LOOKING correct are the interesting ones. So the suite now also saves
|
||||
the take, reopens it, and checks the frames are the same pixels.
|
||||
|
||||
It drives a real Chrome over CDP, and needs both processes up:
|
||||
|
||||
```sh
|
||||
mise exec -- python manage.py runserver 8778 # from the REPO ROOT
|
||||
cd frontend && mise exec -- npx shadow-cljs watch app # in one shell
|
||||
cd frontend && mise exec -- npm run browser # in another
|
||||
```
|
||||
|
||||
`ARTHUR_URL` overrides the page it drives; it defaults to
|
||||
`http://localhost:8778/index.html`, which since step 9 is Django's.
|
||||
|
||||
No dependencies. Playwright is not installed and CDP needs none —
|
||||
`node --experimental-websocket` has a global `WebSocket` and
|
||||
`--headless=new --remote-debugging-port=N` is the whole of the other side. It
|
||||
reads the canvas's own pixels rather than a screenshot, because the CSS scales
|
||||
the stage up by 2 and a screenshot is four pixels per raster pixel; it writes
|
||||
PNGs into `test/browser/out/` anyway, so "it drew something" can be checked by
|
||||
eye as well as by count.
|
||||
|
||||
## The app
|
||||
|
||||
Two processes, which do not talk to each other:
|
||||
|
||||
```sh
|
||||
mise exec -- python manage.py runserver 8778 # from the REPO ROOT
|
||||
cd frontend && mise exec -- npx shadow-cljs watch app
|
||||
```
|
||||
|
||||
Then open **<http://localhost:8778/>**. Django serves the page from
|
||||
`clips/templates/clips/index.html`, and staticfiles serves the bundle out of
|
||||
`static/arthur/js`, where `shadow-cljs` already writes it — so nothing copies files
|
||||
between the two.
|
||||
|
||||
### Paint sketch
|
||||
|
||||
Click **new polygon**, place at least three vertices on the stage, then click
|
||||
**finish shape**. Select a shape to drag its vertices. Scrub to another frame and
|
||||
click **new drawing key** to copy the visible outline there; the previous drawing
|
||||
holds until that key. The numbered drawing-key buttons jump to editable keys.
|
||||
The transition control between two drawing keys can switch that gap between a
|
||||
hold and linear vertex tweening. Other gaps keep their own timing. Tweening works
|
||||
best when the same vertex
|
||||
keeps the same meaning in every drawing. Paint shapes use the timeline clock
|
||||
directly, so the roto exposure grid does not delay a drawing key or step its
|
||||
tween. Use the project **save** button to persist the drawings.
|
||||
|
||||
`/index.html` still works, and that is deliberate: it is the URL the browser suite
|
||||
has used since step 5, when shadow-cljs's `:dev-http` did no directory-index
|
||||
resolution and the suite learned to ask for the file.
|
||||
|
||||
Four built-in clips, on buttons in the transport:
|
||||
|
||||
| | |
|
||||
| --- | --- |
|
||||
| `take` | the synthetic take, head **as filmed**. Step 5's deliverable: a moving mouth, frozen into dense channels, with no video file anywhere. |
|
||||
| `locked` | the same freeze, head **locked**. The same blocks — `:head`'s channels are written as framed identity instead of as a dense track, and nothing in tier 2 differs. |
|
||||
| `demo` | the hand-written scene from step 2. Not a face: the smallest scene that exercises every mechanism the model claims to have, so that each one is visible when it breaks. |
|
||||
| `swarm` | a hundred and twenty dense nodes. Not useful; it is the load test. |
|
||||
|
||||
`take` and `locked` are the pair worth looking at together, because switching
|
||||
between them is the whole of what "stabilisation is a channel, not a mode" means.
|
||||
|
||||
The demo scene itself is `src/arthur/demo/scene.edn`. Both the synthetic take
|
||||
and real footage use `src/arthur/flow/take.cljs` for the measurement order and
|
||||
`src/arthur/flow/freeze.cljs` for the landmark-to-channel conversion.
|
||||
|
||||
**stage 8625** loads the locally saved `IMG_8625.MOV` project and places its
|
||||
post-processed timeline twice. The stage layout is
|
||||
`src/arthur/demo/stage_8625.edn`: the right picture and sound start at frame 48,
|
||||
and the two pictures overlap slightly in stage space. Audio has its own timeline
|
||||
nodes, linked to the picture instances but with independent spans and gain
|
||||
channels. The right sound swells and pans across the stage, then fades out at
|
||||
frame 260 while its picture continues to
|
||||
frame 280. The button needs that saved 8625 project in the local server database.
|
||||
|
||||
### Projects and the EDN fixtures
|
||||
|
||||
The EDN files under `demo/` are authored examples compiled into the frontend.
|
||||
They seed a clip in memory; the server does not read EDN. Clicking **save** on a
|
||||
clip without a project id creates a project through `POST /api/projects`, uploads
|
||||
any missing content-addressed blocks, then writes the clip's addressed leaves
|
||||
through `PUT /api/projects/<id>`. Each leaf value is Transit JSON inside the
|
||||
request's ordinary JSON envelope. Python stores those values in JSON columns and
|
||||
does not need an EDN parser. **open** reads the leaves and blocks and rebuilds
|
||||
the same ClojureScript clip.
|
||||
|
||||
The intended editor creates and changes that in-memory clip directly: a project
|
||||
browser and **new stage** action, timeline instance placement, node and channel
|
||||
editors, then the existing save path. EDN remains useful for checked-in examples
|
||||
and reproducible studies. The current UI has save and open, but no project
|
||||
browser, blank-stage action, or authoring controls yet; open chooses the most
|
||||
recent project.
|
||||
|
||||
### Real footage
|
||||
|
||||
Choose a video in the **footage** file input. The server probes it, re-encodes it
|
||||
to an H.264 proxy and a raw stream of the same coded frames, pulls WAV audio and one tracing JPEG per frame,
|
||||
then makes the resulting footage selectable. Click **load frames** to detect and
|
||||
freeze it. Extraction progress is currently read from `/api/extractions/<key>`; a
|
||||
future WebSocket can push the same job state. The uploaded bytes, extraction job,
|
||||
and decoded footage have separate records, so the same uploaded video can be
|
||||
reopened without decoding it again.
|
||||
|
||||
**The proxy is what gets measured, and the stills are not.** `flow/ingest` cuts
|
||||
its raw H.264 stream into coded frames and decodes them in order with WebCodecs.
|
||||
The proxy has no B-frames, so decode order matches frame order. `flow/detect`
|
||||
hands each decoded frame to MediaPipe in **VIDEO** running mode at
|
||||
`i * 1000 / fps` milliseconds. That timestamp has to increase
|
||||
strictly and has to be real footage time: video mode is a tracker, it reads the
|
||||
gap between timestamps as motion, and a repeat leaves the graph in an error state
|
||||
that every later call re-throws. The JPEGs beside the proxy are reference images
|
||||
for the tracing editor and nothing measures them, so they are not in the footage
|
||||
digest.
|
||||
|
||||
It is re-encoded even when the upload is already H.264, for two reasons: an
|
||||
iPhone's HEVC is not decodable in every browser, and the footage's identity is the
|
||||
proxy's digest — one produced by one ffmpeg invocation, not one that depends on
|
||||
which branch the source happened to take. Its raw stream is copied from that
|
||||
proxy without another encode.
|
||||
|
||||
The command-line route is also available for an existing extracted bundle:
|
||||
|
||||
```sh
|
||||
./extract.sh /path/to/clip.mov # decode to frames + audio + manifest
|
||||
mise exec -- python manage.py ingest_bundle
|
||||
```
|
||||
|
||||
`extract.sh` keeps every source frame and writes `frames/0001.png` onward,
|
||||
`audio.wav` and `manifest.json`. Variable frame rate sources are rejected until the
|
||||
manifest and clock carry per-frame timestamps.
|
||||
|
||||
`ingest_bundle` then hashes all of it into the content-addressed blob store under
|
||||
`var/blobs` — by hard link, so 112MB of PNGs is not copied — and registers one
|
||||
`Footage` row. From then on the frames are the backend's: `GET /api/footage/<id>`
|
||||
answers with a manifest carrying **a URL per frame**, and the app fetches those.
|
||||
|
||||
That replaced a shared secret. Until step 9 the page fetched `/manifest.json` off
|
||||
the filesystem and built `frames/0001.png` itself, with shadow-cljs serving the
|
||||
repo root — so the frame layout was agreed between a shell script and a
|
||||
ClojureScript namespace, and "where are the frames" was answered by a directory
|
||||
listing. The cache-busting `?v=` that used to hang off every frame URL went with
|
||||
it: a blob's name is the hash of its bytes, so re-extracting gives a frame a
|
||||
different URL rather than overwriting one.
|
||||
|
||||
To keep several takes, pass a bundle directory; each ingests separately and both
|
||||
stay selectable in the app.
|
||||
|
||||
```sh
|
||||
./extract.sh /path/to/clip.mov scratch/my-take
|
||||
mise exec -- python manage.py ingest_bundle scratch/my-take
|
||||
```
|
||||
|
||||
`scratch/` is ignored by Git, as are `frames/`, `audio.wav` and `manifest.json` at
|
||||
the root — all of it is extraction output, and tier 3 does not belong in the repo.
|
||||
|
||||
Loading detects one face per frame, measures the mouth, eyes and brows from
|
||||
landmarks and the teeth from source pixels, then freezes them into channels,
|
||||
and adds a button for the footage clip. Detection happens once when you load;
|
||||
playback only resolves channels and paints. Frames without a detection remain
|
||||
marked absent even though their neighbouring poses are used to condition the
|
||||
track. The scene now records stable subject and feature IDs and explicit eye
|
||||
pairs; dense channels can mark one feature absent while another is observed.
|
||||
Current MediaPipe loading supplies only the full-face detection mask. The stage
|
||||
stays 320×200 regardless of the footage dimensions. Real
|
||||
footage starts at the source picture rate. The **picture fps** buttons sample the
|
||||
frozen roto at lower rates while the source track, duration and audio clock stay
|
||||
unchanged. Picking frames to trace into cels is a separate future editing step.
|
||||
**save** also stores the detection mask, dense landmarks and raw RGBA mouth crops
|
||||
as three analysis blocks. **open** restores these without running MediaPipe or
|
||||
loading source PNGs. The frozen shapes remain separate channel blocks.
|
||||
|
||||
For known occlusion intervals, an extracted manifest may add
|
||||
`"feature-absence": {"eye-r": [[10, 14]]}`. Frame numbers are one-based and
|
||||
inclusive, matching PNG filenames. The eye remains the same feature when it
|
||||
reappears; the other eye and the mouth continue through the gap. This is an
|
||||
input annotation, with no UI for editing it yet.
|
||||
|
||||
MediaPipe's JS, wasm and model are under `public/mediapipe/`, served by Django's
|
||||
staticfiles under `/static/mediapipe/`. No CDN is used by this app. See that
|
||||
directory's README for provenance.
|
||||
|
||||
The server reports what it serves at `GET /api/detector`: the package version plus
|
||||
the **sha256 of the model asset**, and that string goes inside the content address
|
||||
of every block a detection produces. Asked rather than assumed, because a version
|
||||
constant in the client is one somebody has to remember to bump — and
|
||||
`docs/architecture.md` is explicit that a model upgrade silently reusing old
|
||||
landmarks presents as "the tool got worse", with no event to attach it to.
|
||||
|
||||
Port 8778 is deliberately not 8777. `python3 serve.py` from the repo root still
|
||||
runs the old JS tool on 8777, and the two are meant to run side by side.
|
||||
|
||||
## Saving
|
||||
|
||||
**save** and **open** in the transport. A save has three ordered stages:
|
||||
is the tier split:
|
||||
|
||||
1. the **analysis** record, so every block stored afterwards can name the detector
|
||||
version that produced it. The server refuses a block whose analysis it does not
|
||||
know.
|
||||
2. ask which **blocks** are missing, upload the source analysis blocks and frozen
|
||||
channel blocks, then link the source blocks to the analysis.
|
||||
3. the **document** — tier 1, as leaves. The server refuses a clip that names
|
||||
blocks it does not hold, so a saved document cannot load into a blank stage
|
||||
somewhere else.
|
||||
|
||||
The status line says what happened: `saved r3 · 64 leaves · 8 blocks`. Saving an
|
||||
unchanged document says `0 leaves · 0 blocks`, which is both halves of the
|
||||
addressing working at once — an unchanged leaf keeps its version, and a
|
||||
content-addressed block is already there.
|
||||
|
||||
Two things are deliberately visible as failures. Saving `swarm` is refused,
|
||||
because its blocks have hand-written names and a document may only name content
|
||||
addresses. And **open** takes the most recently updated project and shows its first
|
||||
clip: there is no project browser, and the store holds one clip at a time.
|
||||
|
||||
## The oracle, which is finished
|
||||
|
||||
`js/` was the numeric oracle through step 4: `test/parity/` ran both
|
||||
implementations on the same synthetic track and diffed `fit-similarity`,
|
||||
`procrustes-mean`, the raster and `stabilize` to 1e-9.
|
||||
|
||||
**It was deleted at step 5, on purpose.** Parity proves the port is FAITHFUL, not
|
||||
that the answer is RIGHT. The JS is a prototype and several of its conclusions
|
||||
contradict each other; a parity test pins behaviour while code moves, and keeping
|
||||
it afterwards would bake the prototype's mistakes into the rewrite and make them
|
||||
permanent. `docs/port-plan.md` says to delete it in one commit once the CLJS
|
||||
player renders the synthetic take, and that is what happened.
|
||||
|
||||
`js/` itself stays as the reference for the MediaPipe setup, face measurements
|
||||
and pixel extraction. Its comments encode bugs that actually happened.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
src/arthur/domain/ pure. No re-frame, no DOM, no flow/.
|
||||
src/arthur/fx/ the only namespaces that talk to the network
|
||||
src/arthur/flow/ the stages. `(f params inputs) -> output`, no state.
|
||||
src/arthur/synth.cljs the synthetic track. In src/ because the take PLAYS it —
|
||||
it stands in for flow/detect, and a tool that needs a
|
||||
video file before it shows you anything is one you
|
||||
cannot debug.
|
||||
src/arthur/demo.cljs the hand-written scene, read from demo/scene.edn
|
||||
src/arthur/demo/take.cljs the synthetic source for the shared flow/take path
|
||||
src/arthur/ui/canvas.cljs indexed raster blit to the display canvas
|
||||
test/arthur/support/ machinery shared between suites; not tests itself
|
||||
test/browser/ drives a real Chrome over CDP. Not run by `npm test`.
|
||||
public/mediapipe/ vendored wasm and model, served under /static/mediapipe/
|
||||
```
|
||||
|
||||
`public/` holds nothing but those assets now. The host page that used to sit beside
|
||||
them is `clips/templates/clips/index.html`.
|
||||
|
||||
## Two evaluators, on purpose
|
||||
|
||||
`domain/timeline` has both `eval-frame` and `resolver`, and they are not
|
||||
alternatives:
|
||||
|
||||
- **`(eval-frame timeline f store)`** is the specification. Allocating, order-free,
|
||||
obviously correct. Tests and one-off renders use it.
|
||||
- **`(resolver timeline store)` -> `(fn [f] ops)`** is what playback uses. It caches
|
||||
the topological order and the z paths, holds a cursor per channel and reuses
|
||||
one point buffer per node, so a frame allocates the op maps and nothing else.
|
||||
|
||||
Both run the same walk, parameterised by how a channel is read and where its
|
||||
points are written — two independent implementations of frame evaluation would
|
||||
drift, and the drift would look like a rendering bug rather than like two
|
||||
functions disagreeing. What differs between them is exactly the part that can be
|
||||
wrong, and `scene-test` asserts they agree frame for frame in forward, backward
|
||||
and random order.
|
||||
|
||||
Because the resolver reuses its buffers, **ops must be rasterised before the
|
||||
next frame is asked for.** That is the contract the rAF loop wants anyway: it
|
||||
reads, blits, and dispatches nothing.
|
||||
1631
frontend/package-lock.json
generated
Normal file
1631
frontend/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
19
frontend/package.json
Normal file
19
frontend/package.json
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"name": "arthur-frontend",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"scripts": {
|
||||
"watch": "shadow-cljs watch app",
|
||||
"release": "shadow-cljs release app",
|
||||
"test": "shadow-cljs compile test && node out/node-tests.js",
|
||||
"browser": "node --experimental-websocket test/browser/take.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mediapipe/tasks-vision": "1.0.1",
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"shadow-cljs": "^2.28.21"
|
||||
}
|
||||
}
|
||||
218
frontend/public/mediapipe/LICENSE
Normal file
218
frontend/public/mediapipe/LICENSE
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
===========================================================================
|
||||
For files under tasks/cc/text/language_detector/custom_ops/utils/utf/
|
||||
===========================================================================
|
||||
/*
|
||||
* The authors of this software are Rob Pike and Ken Thompson.
|
||||
* Copyright (c) 2002 by Lucent Technologies.
|
||||
* Permission to use, copy, modify, and distribute this software for any
|
||||
* purpose without fee is hereby granted, provided that this entire notice
|
||||
* is included in all copies of any software which is or includes a copy
|
||||
* or modification of this software and in all copies of the supporting
|
||||
* documentation for such software.
|
||||
* THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED
|
||||
* WARRANTY. IN PARTICULAR, NEITHER THE AUTHORS NOR LUCENT TECHNOLOGIES MAKE ANY
|
||||
* REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY
|
||||
* OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE.
|
||||
*/
|
||||
17
frontend/public/mediapipe/README.md
Normal file
17
frontend/public/mediapipe/README.md
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# Local MediaPipe assets
|
||||
|
||||
`vision_bundle.js` and the four wasm loader/binary files under `wasm/` come
|
||||
from `@mediapipe/tasks-vision` **1.0.1**, pinned in `frontend/package.json`.
|
||||
`FilesetResolver.forVisionTasks` uses the SIMD pair where supported and the
|
||||
no-SIMD pair elsewhere. The package and these files are Apache-2.0; see
|
||||
[LICENSE](LICENSE).
|
||||
|
||||
`face_landmarker.task` is the official [Face Landmarker model](https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task).
|
||||
Its SHA-256 is
|
||||
`64184e229b263107bc2b804c6625db1341ff2bb731874b0bcc2fe6544e0bc9ff`.
|
||||
|
||||
These are served from `/mediapipe/` so detection needs no CDN at runtime. The
|
||||
browser bundle is loaded as a script before the CLJS app because Shadow CLJS
|
||||
cannot parse the package's CommonJS bundle (its dynamic `import()` is unsupported
|
||||
by the current compiler). `flow/detect.cljs` is the sole call site for its
|
||||
`Vision` global.
|
||||
BIN
frontend/public/mediapipe/face_landmarker.task
Normal file
BIN
frontend/public/mediapipe/face_landmarker.task
Normal file
Binary file not shown.
2
frontend/public/mediapipe/vision_bundle.js
Normal file
2
frontend/public/mediapipe/vision_bundle.js
Normal file
File diff suppressed because one or more lines are too long
8840
frontend/public/mediapipe/wasm/vision_wasm_internal.js
Normal file
8840
frontend/public/mediapipe/wasm/vision_wasm_internal.js
Normal file
File diff suppressed because it is too large
Load diff
BIN
frontend/public/mediapipe/wasm/vision_wasm_internal.wasm
Normal file
BIN
frontend/public/mediapipe/wasm/vision_wasm_internal.wasm
Normal file
Binary file not shown.
8831
frontend/public/mediapipe/wasm/vision_wasm_nosimd_internal.js
Normal file
8831
frontend/public/mediapipe/wasm/vision_wasm_nosimd_internal.js
Normal file
File diff suppressed because it is too large
Load diff
BIN
frontend/public/mediapipe/wasm/vision_wasm_nosimd_internal.wasm
Normal file
BIN
frontend/public/mediapipe/wasm/vision_wasm_nosimd_internal.wasm
Normal file
Binary file not shown.
48
frontend/shadow-cljs.edn
Normal file
48
frontend/shadow-cljs.edn
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
;; Two builds and no more:
|
||||
;;
|
||||
;; app the tool. Output goes straight into the Django staticfiles tree, so
|
||||
;; `python manage.py runserver` and `shadow-cljs watch app` are the whole
|
||||
;; dev loop with nothing copying files between them.
|
||||
;; test :node-test, because everything below `ui/` and `fx/` is pure and has
|
||||
;; no business needing a browser to be asserted about. The canvas-facing
|
||||
;; parts get asserted through domain/raster's byte buffer instead, which
|
||||
;; is what the JS selftest already did.
|
||||
{:source-paths ["src" "test"]
|
||||
|
||||
:dependencies [[reagent "1.2.0"]
|
||||
[re-frame "1.4.3"]
|
||||
;; The document's wire format. JSON would do for the shape of tier
|
||||
;; 1 but not for its VALUES: channel keys are a map by FRAME
|
||||
;; NUMBER and every id is a keyword, and JSON has neither, so a
|
||||
;; save would quietly turn `{0 v}` into `{"0" v}` and `:mouth`
|
||||
;; into "mouth". Transit is JSON on the wire, so Django stores a
|
||||
;; leaf in a JSONField and the admin can still read it.
|
||||
[com.cognitect/transit-cljs "0.8.280"]]
|
||||
|
||||
;; THERE IS NO :dev-http, since step 9. Django serves the page — one template,
|
||||
;; out of `clips/templates/` — and shadow-cljs only builds into the staticfiles
|
||||
;; tree, which is what `:output-dir` below already did. So the dev loop is two
|
||||
;; processes that do not talk to each other:
|
||||
;;
|
||||
;; mise exec -- python manage.py runserver 8778 (from the repo root)
|
||||
;; cd frontend && mise exec -- npx shadow-cljs watch app
|
||||
;;
|
||||
;; What went away with the key was a set of problems rather than a feature. The
|
||||
;; two roots it needed — `public` for the host page and `..` for the repo root, IN
|
||||
;; THAT ORDER, because the root has the old tool's index.html and serving that one
|
||||
;; instead would look like the port having regressed to a suspiciously complete
|
||||
;; tool — were a way of reaching frames/, audio.wav and manifest.json off the
|
||||
;; filesystem. Those are tier 3, and tier 3 is now the backend's, by hash.
|
||||
;;
|
||||
;; 8778 is still deliberately not 8777, which is still the old JS tool's under
|
||||
;; `python3 serve.py`. The two are meant to run side by side.
|
||||
|
||||
:builds
|
||||
{:app {:target :browser
|
||||
:output-dir "../static/arthur/js"
|
||||
:asset-path "/static/arthur/js"
|
||||
:modules {:main {:init-fn arthur.core/init}}}
|
||||
|
||||
:test {:target :node-test
|
||||
:output-to "out/node-tests.js"
|
||||
:ns-regexp "-test$"}}}
|
||||
170
frontend/src/arthur/audio/mix.cljs
Normal file
170
frontend/src/arthur/audio/mix.cljs
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
(ns arthur.audio.mix
|
||||
"Render independently placed audio tracks into one stage audio clock.
|
||||
|
||||
The mix is derived from saved audio track leaves and immutable footage blobs.
|
||||
The transport still has one audio element, so seeking, rate changes and looping
|
||||
stay tied to the same clock the picture reads.
|
||||
|
||||
THE AUDIO BUFFER IS THE PRODUCT AND THE WAV IS ONE PACKAGING OF IT. Playback
|
||||
wants a URL an `<audio>` element can hold; an export wants the samples, either
|
||||
as WAV bytes to put in an archive or as the `AudioBuffer` a muxer takes as an
|
||||
audio track. So `buffer!` renders and the two wrappers below it package, rather
|
||||
than the render being spelled once per consumer."
|
||||
(:require [arthur.domain.channel :as ch]
|
||||
[arthur.domain.clip :as clip]
|
||||
[arthur.domain.node :as node]))
|
||||
|
||||
(defn wav-bytes
|
||||
"An `AudioBuffer` -> the bytes of a 16-bit PCM WAV.
|
||||
|
||||
PEAK-NORMALISED ONLY IF IT WOULD CLIP. A mix of several tracks can sum past
|
||||
1.0, and 16-bit PCM has nowhere to put that, so the alternative to scaling is
|
||||
audible clipping on exactly the loudest moment. Below the threshold nothing is
|
||||
touched, so a single-track mix is the footage's own audio sample for sample."
|
||||
[^js buffer]
|
||||
(let [channels (.-numberOfChannels buffer)
|
||||
frames (.-length buffer)
|
||||
rate (.-sampleRate buffer)
|
||||
bytes (js/ArrayBuffer. (+ 44 (* frames channels 2)))
|
||||
view (js/DataView. bytes)
|
||||
samples (mapv #(.getChannelData buffer %) (range channels))
|
||||
peak (reduce max 0
|
||||
(for [channel samples i (range frames)]
|
||||
(js/Math.abs (aget channel i))))
|
||||
level (if (> peak 0.98) (/ 0.98 peak) 1)]
|
||||
(doseq [[offset word] [[0 "RIFF"] [8 "WAVE"] [12 "fmt "] [36 "data"]]]
|
||||
(dotimes [i 4] (.setUint8 view (+ offset i) (.charCodeAt word i))))
|
||||
(.setUint32 view 4 (- (.-byteLength bytes) 8) true)
|
||||
(.setUint32 view 16 16 true)
|
||||
(.setUint16 view 20 1 true)
|
||||
(.setUint16 view 22 channels true)
|
||||
(.setUint32 view 24 rate true)
|
||||
(.setUint32 view 28 (* rate channels 2) true)
|
||||
(.setUint16 view 32 (* channels 2) true)
|
||||
(.setUint16 view 34 16 true)
|
||||
(.setUint32 view 40 (* frames channels 2) true)
|
||||
(dotimes [i frames]
|
||||
(dotimes [c channels]
|
||||
(let [sample (* level (aget (get samples c) i))]
|
||||
(.setInt16 view (+ 44 (* (+ (* i channels) c) 2))
|
||||
(js/Math.round (* 32767 (max -1 (min 1 sample)))) true))))
|
||||
(js/Uint8Array. bytes)))
|
||||
|
||||
(defn- wav-url [^js buffer]
|
||||
(js/URL.createObjectURL
|
||||
(js/Blob. #js [(wav-bytes buffer)] #js {:type "audio/wav"})))
|
||||
|
||||
(defn- source! [footage-id]
|
||||
(-> (js/fetch (str "/api/footage/" footage-id))
|
||||
(.then (fn [response]
|
||||
(when-not (.-ok response)
|
||||
(throw (ex-info "audio track's footage is missing"
|
||||
{:footage footage-id :status (.-status response)})))
|
||||
(.json response)))
|
||||
(.then (fn [^js manifest]
|
||||
(-> (js/fetch (.-audio manifest))
|
||||
(.then (fn [response]
|
||||
(when-not (.-ok response)
|
||||
(throw (ex-info "audio track's blob is missing"
|
||||
{:footage footage-id :status (.-status response)})))
|
||||
(.arrayBuffer response)))
|
||||
(.then (fn [bytes]
|
||||
(let [decoder (js/OfflineAudioContext. 1 1 44100)]
|
||||
(-> (.decodeAudioData decoder bytes)
|
||||
(.then (fn [buffer]
|
||||
[footage-id {:buffer buffer
|
||||
:fps (.-fps manifest)}])))))))))))
|
||||
|
||||
(defn- automate! [^js param channel start end fps factor default store]
|
||||
(let [channel (or channel (ch/framed default))]
|
||||
(.setValueAtTime param (* factor (ch/value-at channel start store)) (/ start fps))
|
||||
(cond
|
||||
(:dense channel)
|
||||
(doseq [f (range (inc start) end)]
|
||||
(.setValueAtTime param (* factor (ch/value-at channel f store)) (/ f fps)))
|
||||
|
||||
(:animated? channel)
|
||||
(doseq [[f v] (sort-by key (:keys channel))
|
||||
:when (and (> f start) (< f end))]
|
||||
(if (= :linear (:interp channel))
|
||||
(.linearRampToValueAtTime param (* factor v) (/ f fps))
|
||||
(.setValueAtTime param (* factor v) (/ f fps)))))))
|
||||
|
||||
(defn tracks-of
|
||||
"The audio nodes of one of the clip's timelines.
|
||||
|
||||
A timeline parameter rather than always the root, because a symbol is a
|
||||
timeline and may carry its own sound. `:main` is the clip's own, which is what
|
||||
playback mixes."
|
||||
[document tid]
|
||||
(filter #(= :audio (:kind %)) (vals (:nodes (clip/timeline document tid)))))
|
||||
|
||||
(defn- render! [document tid sources store]
|
||||
(let [fps (:fps document)
|
||||
frames (:frames (clip/timeline document tid))
|
||||
tracks (tracks-of document tid)
|
||||
output (js/OfflineAudioContext.
|
||||
2 (js/Math.ceil (* (/ frames fps) 44100)) 44100)]
|
||||
(doseq [track tracks]
|
||||
(let [[start end] (or (:span track) [0 frames])
|
||||
start (max 0 start)
|
||||
end (min frames end)
|
||||
{:keys [buffer fps]} (get sources (get-in track [:source :footage]))
|
||||
sound (.createBufferSource output)
|
||||
gain (.createGain output)
|
||||
pan (.createStereoPanner output)]
|
||||
(when (< start end)
|
||||
(set! (.-buffer sound) buffer)
|
||||
(set! (.-loop sound) (boolean (get-in track [:time :loop?])))
|
||||
(automate! (.-playbackRate sound)
|
||||
(get-in track [:channels [:audio :rate]])
|
||||
start end (:fps document) (or (get-in track [:time :rate]) 1) 1 store)
|
||||
(automate! (.-gain gain)
|
||||
(get-in track [:channels [:audio :gain]])
|
||||
start end (:fps document) 1 1 store)
|
||||
(automate! (.-pan pan)
|
||||
(get-in track [:channels [:audio :pan]])
|
||||
start end (:fps document) 1 0 store)
|
||||
(.connect sound gain)
|
||||
(.connect gain pan)
|
||||
(.connect pan (.-destination output))
|
||||
(.start sound (/ start (:fps document)) (/ (node/local-frame track start) fps))
|
||||
(.stop sound (/ end (:fps document))))))
|
||||
(.startRendering output)))
|
||||
|
||||
(defn buffer!
|
||||
"Promise of the `AudioBuffer` one timeline's audio tracks mix down to, or nil
|
||||
when it has none.
|
||||
|
||||
The raw product. `mix!` packages it as a WAV URL for the transport and
|
||||
`export/frames` packages it as WAV bytes in an archive; a muxer would take it as
|
||||
it is, which is why this is the function the others are written in terms of."
|
||||
([document tid] (buffer! document tid nil))
|
||||
([document tid store]
|
||||
(let [tracks (tracks-of document tid)]
|
||||
(if (empty? tracks)
|
||||
(js/Promise.resolve nil)
|
||||
(-> (js/Promise.all
|
||||
(into-array (map source! (distinct (map #(get-in % [:source :footage]) tracks)))))
|
||||
(.then (fn [pairs] (render! document tid (into {} (array-seq pairs)) store))))))))
|
||||
|
||||
(defn decode!
|
||||
"Promise of the `AudioBuffer` behind a URL. What a clip whose audio is a plain
|
||||
file rather than placed tracks exports."
|
||||
[url]
|
||||
(-> (js/fetch url)
|
||||
(.then (fn [^js response]
|
||||
(when-not (.-ok response)
|
||||
(throw (ex-info "the clip's audio did not load"
|
||||
{:url url :status (.-status response)})))
|
||||
(.arrayBuffer response)))
|
||||
(.then (fn [bytes]
|
||||
(.decodeAudioData (js/OfflineAudioContext. 1 1 44100) bytes)))))
|
||||
|
||||
(defn mix!
|
||||
"Promise of a mixed WAV URL, or the original URL for a clip without audio
|
||||
tracks. Each track can be trimmed and faded independently of its linked picture."
|
||||
([document fallback-url] (mix! document fallback-url nil))
|
||||
([document fallback-url store]
|
||||
(-> (buffer! document clip/root-id store)
|
||||
(.then (fn [buffer] (if buffer (wav-url buffer) fallback-url))))))
|
||||
104
frontend/src/arthur/clock.cljs
Normal file
104
frontend/src/arthur/clock.cljs
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
(ns arthur.clock
|
||||
"The audio clock. Lives OUTSIDE app-db, deliberately.
|
||||
|
||||
THE FRAME IS DERIVED FROM THE AUDIO, never counted:
|
||||
|
||||
frame = ⌊currentTime · fps⌋
|
||||
|
||||
A loop that counted frames and hoped to keep up would drift, and drift against
|
||||
a voice is the one artefact that cannot be fixed downstream — a lip-sync tool
|
||||
whose sync wanders is not a lip-sync tool. Deriving instead means a slow frame
|
||||
DROPS the frames it missed and the next one lands where the audio already is.
|
||||
The failure mode becomes a visible stutter rather than an invisible slide, and
|
||||
those are very different bugs to own.
|
||||
|
||||
½× and ¼× are `playbackRate` and nothing else. The audio slows, `currentTime`
|
||||
advances proportionally, and the derived frame follows — so slow motion cannot
|
||||
desync by construction. Implementing rate as a multiplier on a counted frame
|
||||
would give the picture a rate and the sound another.
|
||||
|
||||
It is outside app-db because the audio element is the source of truth and
|
||||
copying it into the db every frame would make the db a lagging mirror of
|
||||
something authoritative elsewhere. What DOES belong in the db is the playhead
|
||||
as a piece of document state — see events/playback — and that is written from
|
||||
here, not read by here."
|
||||
(:require [arthur.domain.node :as node]))
|
||||
|
||||
(defonce ^:private el (atom nil))
|
||||
|
||||
(defn attach!
|
||||
"Hand the clock its audio element. Idempotent."
|
||||
[audio-el]
|
||||
(reset! el audio-el))
|
||||
|
||||
(defn element [] @el)
|
||||
|
||||
(defn- clamp [f frames]
|
||||
(-> f (max 0) (min (dec frames))))
|
||||
|
||||
(defn frame
|
||||
"The clip frame the audio is currently on."
|
||||
[fps frames]
|
||||
(if-let [a @el]
|
||||
(clamp (js/Math.floor (* (.-currentTime a) fps)) frames)
|
||||
0))
|
||||
|
||||
(defn playing? []
|
||||
(boolean (when-let [a @el] (and (not (.-paused a)) (not (.-ended a))))))
|
||||
|
||||
(defn rate []
|
||||
(if-let [a @el] (.-playbackRate a) 1.0))
|
||||
|
||||
(defn set-rate! [r]
|
||||
(when-let [a @el] (set! (.-playbackRate a) r)))
|
||||
|
||||
(defn play! []
|
||||
(when-let [a @el]
|
||||
;; Returns a promise that rejects if the browser has not seen a gesture yet.
|
||||
;; Swallowed: the transport button IS the gesture, so this can only fire on a
|
||||
;; programmatic play, where a console error is noise rather than news.
|
||||
(some-> (.play a) (.catch (fn [_])))))
|
||||
|
||||
(defn pause! []
|
||||
(when-let [a @el] (.pause a)))
|
||||
|
||||
(defn seek!
|
||||
"Put the audio at the start of frame f. Seeking to the frame's start rather
|
||||
than its middle keeps `frame` idempotent: seek to f, read back f."
|
||||
[fps frames f]
|
||||
(when-let [a @el]
|
||||
(set! (.-currentTime a) (/ (clamp f frames) fps))))
|
||||
|
||||
(defn set-loop!
|
||||
"Wrap at the end instead of stopping. The frame stays derived — `currentTime`
|
||||
simply returns to zero — so nothing about the sync changes, which is the point
|
||||
of not counting frames.
|
||||
|
||||
It earns its place at 2x and 4x, where the whole clip is gone in under four
|
||||
seconds and a profile wants more than that to look at."
|
||||
[on?]
|
||||
(when-let [a @el] (set! (.-loop a) (boolean on?))))
|
||||
|
||||
(defn set-muted! [on?]
|
||||
(when-let [a @el] (set! (.-muted a) (boolean on?))))
|
||||
|
||||
(defn duration-frames
|
||||
"How many frames the audio actually covers, which need not be the clip's
|
||||
length. Reported rather than assumed: a clip longer than its audio is a
|
||||
legitimate thing to be told about, not a thing to silently truncate."
|
||||
[fps]
|
||||
(when-let [a @el]
|
||||
(let [d (.-duration a)]
|
||||
(when (and d (js/isFinite d)) (js/Math.ceil (* d fps))))))
|
||||
|
||||
(defn exposed-frame
|
||||
"The frame a clip-level exposure grid holds `f` back onto. The player shows it
|
||||
as a readout so that `exposure 2` is visibly doing something at the transport
|
||||
rather than only inside the scene."
|
||||
[f expose]
|
||||
(node/expose f expose))
|
||||
|
||||
(defn picture-frame
|
||||
"The source pose displayed at f after picture-rate sampling and exposure."
|
||||
[f source-fps picture-fps expose]
|
||||
(node/expose (node/sample-frame f source-fps picture-fps) expose))
|
||||
37
frontend/src/arthur/core.cljs
Normal file
37
frontend/src/arthur/core.cljs
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
(ns arthur.core
|
||||
"The app's entry point.
|
||||
|
||||
port-plan step 3: the hand-written scene plays at 30fps against audio, scrubs,
|
||||
and runs at ½× and ¼×."
|
||||
(:require [arthur.db :as db]
|
||||
[arthur.events.footage :as footage]
|
||||
[arthur.events.playback]
|
||||
[arthur.events.paint]
|
||||
[arthur.events.project]
|
||||
[arthur.subs.playback]
|
||||
[arthur.subs.render]
|
||||
[arthur.ui.player :as player]
|
||||
[arthur.ui.shell :as shell]
|
||||
[re-frame.core :as rf]
|
||||
[reagent.dom.client :as rdc]))
|
||||
|
||||
(defonce root (atom nil))
|
||||
|
||||
(rf/reg-event-db ::init (fn [_ _] db/default))
|
||||
|
||||
(defn ^:dev/after-load mount []
|
||||
;; A hot reload changes the scene or the rasteriser and not the playhead, so
|
||||
;; the loop would otherwise sit on an unchanged frame number and never redraw.
|
||||
(rf/clear-subscription-cache!)
|
||||
(player/refresh-subs!)
|
||||
(rdc/render @root [shell/view]))
|
||||
|
||||
(defn init []
|
||||
(rf/dispatch-sync [::init])
|
||||
;; What the server already holds, asked for once. The list is small — a row per
|
||||
;; ingested take — and having it before the first click is what lets the footage
|
||||
;; picker be a picker rather than a path to type.
|
||||
(rf/dispatch [::footage/refresh])
|
||||
(reset! root (rdc/create-root (js/document.getElementById "app")))
|
||||
(mount)
|
||||
(player/start!))
|
||||
121
frontend/src/arthur/db.cljs
Normal file
121
frontend/src/arthur/db.cljs
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
(ns arthur.db
|
||||
"app-db: authored data and ids. Nothing derived, and nothing large.
|
||||
|
||||
That sounds like hygiene and it is the precondition for two things that are
|
||||
otherwise unbuildable — spec validation on every event, which is only
|
||||
affordable over authored data, and cheap writes, since every edit `assoc`es
|
||||
into this map and every mounted layer-2 sub compares the result.
|
||||
|
||||
So the clip is here (it is a document — a human placed every node) and dense
|
||||
channel blocks are not; they live behind a handle in `store`. The hand-written
|
||||
demo clip has no dense blocks and its store is empty; the swarm and the take
|
||||
are entirely dense."
|
||||
(:require [arthur.demo :as demo]
|
||||
[arthur.domain.clip :as domain-clip]
|
||||
[arthur.demo.swarm :as swarm]
|
||||
[arthur.demo.take :as take]))
|
||||
|
||||
(defn- entry
|
||||
"A clip plus what the transport and the stage read off it.
|
||||
|
||||
Read OFF the clip rather than written again beside it: copying a number by hand
|
||||
into this table is how it comes to disagree with the document it describes.
|
||||
`:frames` comes from the ROOT TIMELINE and `:fps` from the clip, which is the
|
||||
split `arthur.domain.clip` exists to make — a timeline is a frame space, a clip
|
||||
is a rate — and an earlier version of this docstring noted that they sat on one
|
||||
map \"only because there is one clip per scene today\". They do not any more."
|
||||
[label-key label clip store]
|
||||
(merge {:label label :clip clip :store store
|
||||
;; A static asset since step 9, and not the repo root's `audio.wav`.
|
||||
;; That file is `extract.sh`'s output — tier 3, which the backend now
|
||||
;; serves by hash — and the synthetic take needs a sound of its own so
|
||||
;; that the clock has something to run against with no footage ingested.
|
||||
:audio "/static/arthur/audio.wav"
|
||||
:cid (name label-key)
|
||||
:display-fps (:fps clip)
|
||||
:frames (domain-clip/frames clip)}
|
||||
(select-keys clip [:fps :width :height])))
|
||||
|
||||
(def clips
|
||||
"The hand-made clips, selectable from the transport.
|
||||
|
||||
`:swarm` is the load test: a hundred and twenty nodes, entirely dense. The two
|
||||
takes are step 5's deliverable and they are ONE freeze — the same blocks, with
|
||||
`:head` written as a dense track in one and as framed identity in the other, so
|
||||
the button that switches between them switches a document field and nothing
|
||||
else."
|
||||
{:demo {:label "demo" :entry (delay (entry :demo "demo" demo/clip nil))}
|
||||
:swarm {:label "swarm" :entry (delay (entry :swarm "swarm" @swarm/clip @swarm/store))}
|
||||
:take {:label "take" :entry (delay (entry :take "take" @take/clip @take/store))}
|
||||
:take-locked {:label "locked" :entry (delay (entry :take-locked "locked" @take/locked @take/store))}})
|
||||
|
||||
(defn clip-entry [id]
|
||||
(some-> (get-in clips [id :entry]) deref))
|
||||
|
||||
(def default
|
||||
{;; --- the document ---
|
||||
:clip/current :take
|
||||
:paint/revision 0
|
||||
:palette :arthur/default ; a NAME; the ramp itself is project data
|
||||
|
||||
;; --- the clip ---
|
||||
;;
|
||||
;; Including the STAGE DIMENSIONS, which are the project's and not the
|
||||
;; footage's. That is what deleting `makeXform` buys — the framing became a
|
||||
;; transform on a node, so nothing downstream of the freeze knows the frame
|
||||
;; size — and it is why ui/player no longer hardcodes 320x200.
|
||||
:clip (select-keys (clip-entry :take) [:fps :frames :width :height :audio :display-fps])
|
||||
|
||||
;; Which ingested footage to detect, and what the last load said. The list
|
||||
;; comes from the server — tier 3 is the backend's since step 9 — so there is
|
||||
;; no path to type any more.
|
||||
:footage {:id nil :label nil :loading? false :status nil
|
||||
:available [] :chosen nil}
|
||||
|
||||
;; The document's own identity on the server. `:seq` is the monotonic project
|
||||
;; version: a client that sees a delta with `seq > local + 1` refetches, which
|
||||
;; is what will make staleness self-healing once there is a broadcast to miss.
|
||||
:project {:id nil :cid nil :name nil :seq nil :busy? false :status nil}
|
||||
|
||||
;; --- transport ---
|
||||
;;
|
||||
;; The playhead is in app-db like everything else. An earlier draft of
|
||||
;; docs/architecture.md put it in a standalone ratom to dodge an invalidation
|
||||
;; storm that does not happen: with layer-2 extractors and layer-3
|
||||
;; computations, a tick re-runs one cheap extractor per mounted sub, each
|
||||
;; returning the same value for every subtree the tick did not touch, and
|
||||
;; therefore notifying nobody. ::resolver does not re-run.
|
||||
;;
|
||||
;; Two reasons it belongs here rather than outside: a seek in the event log is
|
||||
;; how scrubbing becomes inspectable in re-frame-10x, and a collaborator's
|
||||
;; playhead is a feature — putting it outside app-db puts it outside the
|
||||
;; machinery that would share it.
|
||||
;; --- export ---
|
||||
;;
|
||||
;; The REQUEST and its progress, never the frames. Which timeline to write and
|
||||
;; at what integer zoom is authored state like anything else; the megabytes the
|
||||
;; render produces are handed straight to a download and never enter the db.
|
||||
;; `:isolate` is the placement to render alone, or nil for the whole timeline.
|
||||
:export {:timeline :main :isolate nil :zoom 4 :busy? false :done 0 :total 0
|
||||
:status nil}
|
||||
|
||||
:playback {:frame 0
|
||||
:playing? false
|
||||
:rate 1.0
|
||||
;; Both for profiling: loop so a run at 4x lasts longer than the
|
||||
;; clip, mute so sitting in one does not require enduring it.
|
||||
:loop? false
|
||||
:muted? false}})
|
||||
|
||||
(def rates
|
||||
"The transport's rates — all of them `playbackRate` on the audio element, so
|
||||
the picture cannot drift from the sound at any of them.
|
||||
|
||||
2x and 4x are there to be profiled at rather than watched. A 30fps clip at 2x
|
||||
wants sixty clip frames a second against a 60Hz display, so every animation
|
||||
frame has to paint a new one: it is the point where the loop stops having
|
||||
slack. Past that the clock starts dropping frames rather than falling behind,
|
||||
which is the whole reason the frame is derived from the audio instead of
|
||||
counted — and the transport reports the drop rate so that it is visible rather
|
||||
than merely survivable."
|
||||
[0.25 0.5 1.0 2.0 4.0])
|
||||
28
frontend/src/arthur/demo.cljs
Normal file
28
frontend/src/arthur/demo.cljs
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
(ns arthur.demo
|
||||
"The hand-written clip from port-plan step 2, and nothing else.
|
||||
|
||||
The EDN is a resource rather than a literal in this file so that the test and
|
||||
the page read the SAME bytes. If the scene were written twice, the one the test
|
||||
validates would not be the one that renders, and the model would be validated
|
||||
against a scene nobody ever looked at."
|
||||
(:require [arthur.domain.clip :as domain-clip]
|
||||
[arthur.domain.timeline :as timeline]
|
||||
[cljs.reader :as reader]
|
||||
[shadow.resource :as rc]))
|
||||
|
||||
(def source (rc/inline "arthur/demo/scene.edn"))
|
||||
|
||||
(def clip (reader/read-string source))
|
||||
|
||||
(def timeline
|
||||
"The clip's root timeline: what an evaluator takes. `clip` is the document."
|
||||
(domain-clip/root clip))
|
||||
|
||||
(def fps (:fps clip))
|
||||
(def frames (domain-clip/frames clip))
|
||||
|
||||
(defn ops-at
|
||||
"Draw ops for one frame, via the specification path. The page uses
|
||||
`timeline/resolver` instead; this is here for the REPL."
|
||||
[f]
|
||||
(timeline/eval-frame timeline f))
|
||||
102
frontend/src/arthur/demo/scene.edn
Normal file
102
frontend/src/arthur/demo/scene.edn
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
;; A scene written by hand, before any analysis exists.
|
||||
;;
|
||||
;; port-plan step 2 is deliberately ahead of measurement: the data model has
|
||||
;; never been validated, and it is worth finding out here, with fifty lines to
|
||||
;; throw away, rather than after nine hundred lines of measurement have been
|
||||
;; ported into a shape that does not work.
|
||||
;;
|
||||
;; So this is not a demo of a face. It is the smallest scene that exercises every
|
||||
;; mechanism the model claims to have, chosen so that each one is visible when it
|
||||
;; breaks:
|
||||
;;
|
||||
;; exposure inherited from the clip root the motion steps on 2s
|
||||
;; a keyed [:xform :pos], sparse, held the card jumps between 4 poses
|
||||
;; transform composition through a group the eye rides the card
|
||||
;; rotation about an anchor the card turns, it does not swing
|
||||
;; a stencil as a colour key the iris cannot leave the card
|
||||
;; a stencil chain nor can the pupil
|
||||
;; a keyed [:vis] the bar blinks off and back
|
||||
;; a :span the bar does not exist at either end
|
||||
;; fractional z among siblings the bar is behind, the pupil in front
|
||||
;;
|
||||
;; Everything is in 320x200 raster space, which is what [:geom :pts] holds.
|
||||
{:name "step-2 demo"
|
||||
;; 229 frames at 30fps is 7.63s, which covers audio.wav (7.601s) with a frame to
|
||||
;; spare. fps belongs to the CLIP rather than to the timeline — a timeline has a
|
||||
;; frame space, not a rate — and it is here only because there is one clip.
|
||||
:fps 30
|
||||
;; The STAGE, in pixels. The project's dimensions, not the footage's — which is
|
||||
;; what makes `makeXform` deletable: placement is a transform on a node and the
|
||||
;; stage clips whatever hangs off. Here everything is authored in stage pixels
|
||||
;; already, because a hand-written scene is a painted one.
|
||||
:width 320
|
||||
:height 200
|
||||
|
||||
:timelines
|
||||
{:main
|
||||
{:id :main
|
||||
:frames 229
|
||||
:nodes
|
||||
{;; The clip root. EXPOSURE LIVES HERE and is inherited, because
|
||||
;; docs/design.md is emphatic that everything rides one grid: a head cutting on
|
||||
;; odd frames against a mouth cutting on even ones reads as two performances.
|
||||
;; Setting it lower on a child is possible and is meant to feel deliberate.
|
||||
:root
|
||||
{:id :root :name "clip" :kind :group :parent nil :z "a1"
|
||||
:time {:mode :map :expose 2}}
|
||||
|
||||
;; A bar, behind everything, purely to assert that :vis and :span are different
|
||||
;; questions. It stops existing outside [6 66) — nothing to hide, nothing to
|
||||
;; hold — and inside that range it is switched off between frames 76 and 153.
|
||||
:bar
|
||||
{:id :bar :name "bar" :kind :poly :parent :root :z "a0"
|
||||
:span [19 210]
|
||||
:channels
|
||||
{[:vis] {:animated? true :interp :hold :keys {0 true, 76 false, 153 true} :over []}
|
||||
[:geom :pts] {:animated? false :value [20 168 300 168 300 176 20 176]}
|
||||
[:style :color] {:animated? false :value :brow}}}
|
||||
|
||||
;; The group the plan asks for: four sparse keys on [:xform :pos], held. At
|
||||
;; exposure 2 the card reads its pose from an even frame, so a key landing on
|
||||
;; an odd frame would be seen on the even frame after it — which is the whole
|
||||
;; reason exposure is applied before anything else and not folded into keys.
|
||||
:swing
|
||||
{:id :swing :name "swing" :kind :group :parent :root :z "a1"
|
||||
:channels
|
||||
{[:xform :pos] {:animated? true :interp :hold
|
||||
:keys {0 [90.0 100.0], 57 [200.0 70.0], 114 [230.0 140.0], 171 [110.0 150.0]}
|
||||
:over []}
|
||||
[:xform :rot] {:animated? true :interp :hold
|
||||
:keys {0 0.0, 57 0.35, 114 0.0, 171 -0.35}
|
||||
:over []}}}
|
||||
|
||||
;; The rectangle. Its points are centred on the origin and its :anchor is the
|
||||
;; origin too, so :swing's rotation TURNS it rather than swinging it round a
|
||||
;; corner — which is the failure mode :anchor exists to prevent.
|
||||
:card
|
||||
{:id :card :name "card" :kind :poly :parent :swing :z "a1"
|
||||
:channels
|
||||
{[:geom :pts] {:animated? false :value [-44 -30 44 -30 44 30 -44 30]}
|
||||
[:style :color] {:animated? false :value :skin-base}}}
|
||||
|
||||
;; A disc stencilled by the card. The stencil is a COLOUR KEY, not a node
|
||||
;; reference — it is the take format's clip= — so the iris is written only over
|
||||
;; pixels that currently hold the card's index. Push the radius up and it is
|
||||
;; cropped by the card's edge rather than spilling, at any position, with no
|
||||
;; clamp anywhere.
|
||||
:iris
|
||||
{:id :iris :name "iris" :kind :disc :parent :card :stencil :card :z "a2"
|
||||
:channels
|
||||
{[:xform :pos] {:animated? false :value [14.0 -6.0]}
|
||||
[:geom :radius] {:animated? false :value 13.0}
|
||||
[:style :color] {:animated? false :value :iris}}}
|
||||
|
||||
;; The pupil is a SQUARE, and three pixels of it. A circle of radius 1.5 is not
|
||||
;; a circle, it is a plus sign with the corners gnawed off, and it changes shape
|
||||
;; as it moves. Stencilled by the iris, which is itself already cropped by the
|
||||
;; card, so the clip composes without the chain being expressed anywhere.
|
||||
:pupil
|
||||
{:id :pupil :name "pupil" :kind :rect :parent :iris :stencil :iris :z "a3"
|
||||
:channels
|
||||
{[:geom :size] {:animated? false :value 5.0}
|
||||
[:style :color] {:animated? false :value :pupil}}}}}}}
|
||||
73
frontend/src/arthur/demo/stage.cljs
Normal file
73
frontend/src/arthur/demo/stage.cljs
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
(ns arthur.demo.stage
|
||||
"A saved take placed seven times on a stage. The EDN is the authored layout."
|
||||
(:require [arthur.domain.channel :as ch]
|
||||
[cljs.reader :as reader]
|
||||
[shadow.resource :as rc]))
|
||||
|
||||
(def layout (reader/read-string (rc/inline "arthur/demo/stage_8625.edn")))
|
||||
|
||||
(defn- position-track [center anchor drift phase frames]
|
||||
(let [base (mapv - center anchor)
|
||||
[dx dy] drift
|
||||
wave (fn [f period] (js/Math.sin (* 2 js/Math.PI (/ (+ f phase) period))))
|
||||
x0 (wave 0 96)
|
||||
y0 (wave 0 132)]
|
||||
(ch/keyed
|
||||
(into {}
|
||||
(for [f (conj (vec (range 0 frames 20)) (dec frames))]
|
||||
[f [(+ (first base) (* dx (- (wave f 96) x0)))
|
||||
(+ (second base) (* dy (- (wave f 132) y0)))]]))
|
||||
:linear)))
|
||||
|
||||
(defn compose
|
||||
"The authored layout plus a source clip -> the composed stage document.
|
||||
|
||||
A PLACEMENT IS KEYED BY ITS :uuid, not by the authored id. The authored id
|
||||
(`:left`, `:voice-right`) is a handle for reading the EDN and for the
|
||||
`:linked-to` written there; it does not appear in the document this returns.
|
||||
What replaces it is an identity that means one placement and nothing else: seven
|
||||
instances of one symbol are seven different things to name — to export on their
|
||||
own, to link a voice to, to point at later — and an id like `:left` is a
|
||||
description of where a thing sits, which is exactly what changes when the stage
|
||||
is re-arranged. `:name` carries the label for a human and `:of` carries the
|
||||
symbol, so the node still says what it is and which drawing it plays."
|
||||
[source]
|
||||
(let [{:keys [name width height frames symbol instances audio scale]} layout
|
||||
default-anchor (or (:anchor layout)
|
||||
[(/ (:width source) 2) (/ (:height source) 2)])
|
||||
original (get-in source [:timelines :main])
|
||||
;; Authored id -> uuid, so the `:linked-to` in the EDN resolves to the
|
||||
;; identity the document uses. Built before either pass because the audio
|
||||
;; nodes refer to the instances.
|
||||
by-id (into {} (map (juxt :id :uuid)) (concat instances audio))
|
||||
uuid-of (fn [what id]
|
||||
(or (get by-id id)
|
||||
(throw (ex-info "the stage layout names a placement that is not there"
|
||||
{:in what :id id
|
||||
:known (vec (sort-by str (keys by-id)))}))))
|
||||
nodes (into
|
||||
{:root {:id :root :name "stage" :kind :group :z "a1"}}
|
||||
(map (fn [{:keys [uuid name z span at in center anchor drift phase]}]
|
||||
(let [anchor (or anchor default-anchor)]
|
||||
[uuid {:id uuid :name name :kind :symbol :of symbol
|
||||
:parent :root :z z :span span
|
||||
:time {:mode :map :at at :in in :rate 1}
|
||||
:channels {[:xform :pos] (if drift
|
||||
(position-track center anchor drift phase frames)
|
||||
(ch/framed (mapv - center anchor)))
|
||||
[:xform :anchor] {:animated? false :value anchor}
|
||||
[:xform :scale] scale}}]))
|
||||
instances))
|
||||
nodes (into nodes
|
||||
(map (fn [{:keys [uuid linked-to z source span at in gain pan]}]
|
||||
[uuid {:id uuid :kind :audio :parent :root :z z
|
||||
:linked-to (uuid-of uuid linked-to)
|
||||
:source source :span span
|
||||
:time {:mode :map :at at :in in :rate 1}
|
||||
:channels (cond-> {[:audio :gain] gain}
|
||||
pan (assoc [:audio :pan] pan))}])
|
||||
audio))]
|
||||
(assoc source :name name :width width :height height
|
||||
:timelines (assoc (:timelines source)
|
||||
:main {:id :main :frames frames :nodes nodes}
|
||||
symbol (assoc original :id symbol)))))
|
||||
77
frontend/src/arthur/demo/stage_8625.edn
Normal file
77
frontend/src/arthur/demo/stage_8625.edn
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
;; A local stage sketch. The source is the saved, post-processed IMG_8625.MOV
|
||||
;; clip in the project store; its dense channel blocks are shared by all seven
|
||||
;; instances. Centers, drift and timing are authored in stage pixels and frames.
|
||||
{:source-project "4379f900-bdd2-409b-acf6-32081f8ce01f"
|
||||
:source-cid "f8cace9e-4ad3-4796-973c-c62eeebe3d01"
|
||||
:symbol :sym/face-8625
|
||||
:name "8625 stage study"
|
||||
:width 320 :height 200 :frames 280
|
||||
;; Each :center below places the source clip's center on the stage. A symbol
|
||||
;; can author :anchor to override that default for an off-center drawing.
|
||||
;; Each placement reads this pulse in its own local time, so the staggered
|
||||
;; entrances start their growth at different moments on the master timeline.
|
||||
:scale {:animated? true :interp :linear
|
||||
:keys {0 [0.4 0.4], 12 [0.56 0.56], 24 [0.48 0.48],
|
||||
48 [0.52 0.52], 72 [0.48 0.48], 96 [0.52 0.52],
|
||||
120 [0.48 0.48], 144 [0.52 0.52], 168 [0.48 0.48],
|
||||
192 [0.52 0.52], 216 [0.48 0.48], 240 [0.52 0.52],
|
||||
279 [0.48 0.48]}
|
||||
:over []}
|
||||
;; Audio placements are ordinary timeline nodes with channel parameters.
|
||||
;; :linked-to is an editorial link; their spans and time maps are independent.
|
||||
:audio
|
||||
[{:id :voice-left :uuid #uuid "eeaa49c3-1238-469f-bf54-44929e379f6b"
|
||||
:linked-to :left :z "a3"
|
||||
:source {:footage "f8cace9e-4ad3-4796-973c-c62eeebe3d01"}
|
||||
:span [0 280] :at 0 :in 0
|
||||
:gain {:animated? false :value 1.0}}
|
||||
{:id :voice-right :uuid #uuid "468239dd-0e3a-4e5c-ac6f-1858430a0355"
|
||||
:linked-to :right :z "a4"
|
||||
:source {:footage "f8cace9e-4ad3-4796-973c-c62eeebe3d01"}
|
||||
:span [48 260] :at 48 :in 0
|
||||
:gain {:animated? true :interp :linear
|
||||
:keys {48 0.0, 60 1.0, 90 0.35, 115 0.9, 145 0.45,
|
||||
170 1.0, 195 0.4, 220 0.85, 245 1.0, 259 0.0}
|
||||
:over []}
|
||||
:pan {:animated? true :interp :linear
|
||||
:keys {48 -0.8, 90 -0.8, 130 0.7, 175 0.7, 220 -0.65, 259 0.65}
|
||||
:over []}}]
|
||||
;;
|
||||
;; EVERY PLACEMENT CARRIES A :uuid, and it is authored here rather than generated
|
||||
;; in `compose`. The uuid is the node's identity in the composed document — it is
|
||||
;; the key in the timeline's node map — so generating one per load would give the
|
||||
;; same stage a different document on every load, and nothing that refers to a
|
||||
;; placement (`:linked-to` above, an export target in the UI, a comment in a
|
||||
;; review) could survive a reload. The `:id` beside it stays as the AUTHORING
|
||||
;; handle: it is what the reader of this file uses to see which placement is
|
||||
;; which, and what the `:linked-to` above names, and `compose` resolves it to the
|
||||
;; uuid. Nothing downstream of `compose` sees the authored id.
|
||||
:instances
|
||||
[{:id :left :uuid #uuid "ee7321c8-faf1-46d7-8029-37771898accb"
|
||||
:name "8625 left" :z "a1"
|
||||
:span [0 280] :at 0 :in 0
|
||||
:center [40 40] :drift [3 2] :phase 0}
|
||||
{:id :right :uuid #uuid "1aa0da78-b4ed-4bb6-8d70-b09a3ec5e2c3"
|
||||
:name "8625 right" :z "a2"
|
||||
:span [48 280] :at 48 :in 0
|
||||
:center [120 40] :drift [-3 2] :phase 17}
|
||||
{:id :top-third :uuid #uuid "23bb697d-eba7-4af6-a86c-606c50107088"
|
||||
:name "8625 top third" :z "a5"
|
||||
:span [24 280] :at 24 :in 0
|
||||
:center [200 40] :drift [2 -3] :phase 31}
|
||||
{:id :top-fourth :uuid #uuid "f4f0241d-026e-4e50-9bea-a4ccde896d8a"
|
||||
:name "8625 top fourth" :z "a6"
|
||||
:span [72 280] :at 72 :in 0
|
||||
:center [280 40] :drift [-2 -2] :phase 49}
|
||||
{:id :bottom-left :uuid #uuid "8f594d72-a97f-4a32-82fd-08d1670a2218"
|
||||
:name "8625 bottom left" :z "a7"
|
||||
:span [96 280] :at 96 :in 0
|
||||
:center [70 135] :drift [3 -2] :phase 63}
|
||||
{:id :bottom-middle :uuid #uuid "63f3fb32-9e94-4d68-a1c2-12e6de2d04b5"
|
||||
:name "8625 bottom middle" :z "a8"
|
||||
:span [120 280] :at 120 :in 0
|
||||
:center [160 135] :drift [-2 3] :phase 81}
|
||||
{:id :bottom-right :uuid #uuid "fa338701-cb21-4d45-89f1-a5e706f045ec"
|
||||
:name "8625 bottom right" :z "a9"
|
||||
:span [144 280] :at 144 :in 0
|
||||
:center [250 135] :drift [2 2] :phase 107}]}
|
||||
168
frontend/src/arthur/demo/swarm.cljs
Normal file
168
frontend/src/arthur/demo/swarm.cljs
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
(ns arthur.demo.swarm
|
||||
"A hundred and twenty shapes, orbiting, spinning, pulsing and blinking.
|
||||
|
||||
Not useful. It is here because it is the first thing to exercise the DENSE
|
||||
channel path end to end — typed-array blocks behind a store handle, one value
|
||||
per frame, read through a cursor — which until now had tests and no traffic.
|
||||
Step 5 writes exactly this shape out of the freeze module, so it is worth
|
||||
knowing the resolver can carry it at rate before anything depends on that.
|
||||
|
||||
Everything is generated from deterministic trigonometry rather than from a
|
||||
random seed: the same scene every load, so a stutter or a wrong pose is
|
||||
reproducible instead of being a thing that happened once.
|
||||
|
||||
Layout of each block is the rectangular one freeze produces — node-major,
|
||||
frame-minor, no per-frame header and no indirection:
|
||||
|
||||
offset(node i) = i · frames · stride
|
||||
value(i, f) = data[offset(i) + f · stride]"
|
||||
(:require [arthur.domain.channel :as ch]
|
||||
[arthur.domain.palette :as pal]))
|
||||
|
||||
(def frames 229)
|
||||
(def fps 30)
|
||||
(def n-orbits 6)
|
||||
(def n-shapes 120)
|
||||
|
||||
(def ^:private TAU (* 2 js/Math.PI))
|
||||
|
||||
;; Every tone except the background, so the swarm uses the whole ramp.
|
||||
(def ^:private tones
|
||||
(vec (remove #{:bg} (map :name pal/entries))))
|
||||
|
||||
(defn- regular-poly
|
||||
"A closed n-gon about the origin, flat in [x0 y0 x1 y1 …] — the same layout a
|
||||
dense block holds, which is the point of geometry being flat everywhere."
|
||||
[n radius phase]
|
||||
(vec (mapcat (fn [k]
|
||||
(let [a (+ phase (/ (* TAU k) n))]
|
||||
[(* radius (js/Math.cos a))
|
||||
(* radius (js/Math.sin a))]))
|
||||
(range n))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the dense blocks
|
||||
|
||||
(defn- fill-block!
|
||||
"Write one node's frames into a node-major block."
|
||||
[^js data i stride f->vals]
|
||||
(let [base (* i frames stride)]
|
||||
(dotimes [f frames]
|
||||
(let [vs (f->vals f)
|
||||
o (+ base (* f stride))]
|
||||
(dotimes [k stride]
|
||||
(aset data (+ o k) (nth vs k)))))))
|
||||
|
||||
(defn- orbit-blocks []
|
||||
(let [pos (js/Float32Array. (* n-orbits frames 2))
|
||||
rot (js/Float32Array. (* n-orbits frames 1))]
|
||||
(dotimes [i n-orbits]
|
||||
(let [ph (/ (* TAU i) n-orbits)
|
||||
;; Lissajous, so the six orbits drift in and out of phase with each
|
||||
;; other instead of marching in step.
|
||||
wx (+ 0.011 (* 0.004 (mod i 3)))
|
||||
wy (+ 0.017 (* 0.003 (mod i 4)))
|
||||
spin (* 0.008 (if (even? i) 1 -1) (inc (mod i 3)))]
|
||||
(fill-block! pos i 2
|
||||
(fn [f] [(+ 160 (* 104 (js/Math.sin (+ (* f wx) ph))))
|
||||
(+ 100 (* 64 (js/Math.sin (+ (* f wy) (* 1.7 ph)))))]))
|
||||
(fill-block! rot i 1 (fn [f] [(* f spin)]))))
|
||||
{"swarm/orbit-pos" {:data pos :state nil}
|
||||
"swarm/orbit-rot" {:data rot :state nil}}))
|
||||
|
||||
(defn- shape-blocks []
|
||||
(let [pos (js/Float32Array. (* n-shapes frames 2))
|
||||
rot (js/Float32Array. (* n-shapes frames 1))
|
||||
scale (js/Float32Array. (* n-shapes frames 2))
|
||||
;; The state mask: a handful of shapes wink out entirely for a stretch.
|
||||
;; ABSENT, not hidden — this is the mask meaning "there is no value on
|
||||
;; this frame", which is what an occluded subject will mean at step 6.
|
||||
state (js/Uint8Array. (* n-shapes frames))]
|
||||
(dotimes [i n-shapes]
|
||||
(let [ph (/ (* TAU i) n-shapes)
|
||||
ring (+ 18 (* 26 (js/Math.abs (js/Math.sin (* 2.3 ph)))))
|
||||
wob (+ 0.03 (* 0.02 (mod i 5)))
|
||||
spin (* (if (zero? (mod i 3)) -1 1) (+ 0.02 (* 0.011 (mod i 7))))
|
||||
pulse (+ 0.05 (* 0.013 (mod i 6)))]
|
||||
(fill-block! pos i 2
|
||||
(fn [f]
|
||||
;; Orbit position plus a small independent wobble, so no
|
||||
;; two neighbours trace the same path.
|
||||
(let [a (+ ph (* f 0.014 (if (even? i) 1 -1)))]
|
||||
[(+ (* ring (js/Math.cos a)) (* 5 (js/Math.sin (* f wob))))
|
||||
(+ (* ring (js/Math.sin a)) (* 5 (js/Math.cos (+ 1.1 (* f wob)))))])))
|
||||
(fill-block! rot i 1 (fn [f] [(+ ph (* f spin))]))
|
||||
(fill-block! scale i 2
|
||||
(fn [f]
|
||||
(let [s (+ 1.0 (* 0.45 (js/Math.sin (+ ph (* f pulse)))))]
|
||||
[s s])))
|
||||
;; Every eleventh shape is absent for a window that moves with i.
|
||||
(when (zero? (mod i 11))
|
||||
(let [from (mod (* i 9) frames)
|
||||
to (min frames (+ from 34))]
|
||||
(doseq [f (range from to)]
|
||||
(aset state (+ (* i frames) f) ch/absent-bit))))))
|
||||
{"swarm/pos" {:data pos :state state}
|
||||
"swarm/rot" {:data rot :state nil}
|
||||
"swarm/scale" {:data scale :state nil}}))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the nodes
|
||||
|
||||
(defn- dense [store i stride]
|
||||
{:animated? true :interp :hold
|
||||
:dense {:store store :offset (* i frames stride) :stride stride :frames frames}
|
||||
;; Provenance, which nothing in the renderer reads. Here it is honest about
|
||||
;; where these numbers came from, the same way :roto/lips-outer will be.
|
||||
:generated {:by :demo/swarm}
|
||||
:over []})
|
||||
|
||||
(defn- orbit-node [i]
|
||||
{:id (keyword (str "orbit-" i)) :kind :group :parent :root
|
||||
:z (str "b" i)
|
||||
:channels {[:xform :pos] (dense "swarm/orbit-pos" i 2)
|
||||
[:xform :rot] (dense "swarm/orbit-rot" i 1)}})
|
||||
|
||||
(defn- shape-node [i]
|
||||
(let [orbit (keyword (str "orbit-" (mod i n-orbits)))
|
||||
tone (nth tones (mod i (count tones)))
|
||||
kind (case (mod i 7) 5 :disc 6 :rect :poly)
|
||||
verts (+ 3 (mod i 10))
|
||||
size (+ 3.5 (* 0.9 (mod i 8)))
|
||||
base {:id (keyword (str "s-" i)) :kind kind :parent orbit
|
||||
;; Fractional index among siblings. Zero-padded so the strings
|
||||
;; sort the way the numbers do — "c9" would otherwise land after
|
||||
;; "c10", which is the classic way a z order goes subtly wrong.
|
||||
:z (str "c" (.padStart (str i) 4 "0"))
|
||||
:channels {[:xform :pos] (dense "swarm/pos" i 2)
|
||||
[:xform :rot] (dense "swarm/rot" i 1)
|
||||
[:xform :scale] (dense "swarm/scale" i 2)
|
||||
[:style :color] (ch/framed tone)}}]
|
||||
(update base :channels merge
|
||||
(case kind
|
||||
:poly {[:geom :pts] (ch/framed (regular-poly verts size (* 0.3 i)))}
|
||||
:disc {[:geom :radius] (ch/framed (* 0.75 size))}
|
||||
:rect {[:geom :size] (ch/framed (js/Math.round size))}))))
|
||||
|
||||
(def store
|
||||
(delay (merge (orbit-blocks) (shape-blocks))))
|
||||
|
||||
(def clip
|
||||
(delay
|
||||
{:name "swarm"
|
||||
:fps fps
|
||||
:width 320
|
||||
:height 200
|
||||
:timelines
|
||||
{:main
|
||||
{:id :main
|
||||
:frames frames
|
||||
:nodes
|
||||
(into {:root {:id :root :kind :group :parent nil :z "a1"
|
||||
;; On 2s, like everything else. A hundred and twenty shapes
|
||||
;; cutting on one grid reads as animation; the same shapes on
|
||||
;; their own grids read as a screensaver, which is the whole
|
||||
;; argument for exposure inheriting strictly.
|
||||
:time {:mode :map :expose 2}}}
|
||||
(concat (map (juxt :id identity) (map orbit-node (range n-orbits)))
|
||||
(map (juxt :id identity) (map shape-node (range n-shapes)))))}}}))
|
||||
98
frontend/src/arthur/demo/take.cljs
Normal file
98
frontend/src/arthur/demo/take.cljs
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
(ns arthur.demo.take
|
||||
"The synthetic take: the whole vertical slice, with no video file in it.
|
||||
|
||||
This is port-plan step 5's deliverable. `flow/take` now composes the shared
|
||||
measurement path for both this generator and real footage —
|
||||
|
||||
synth ──▶ measure/anchor ──▶ condition/anchor
|
||||
│ │
|
||||
└──▶ mouth, eyes, brows ◀─┘
|
||||
│
|
||||
condition/parts
|
||||
│
|
||||
FREEZE ──▶ channels on nodes
|
||||
│
|
||||
timeline/resolver ──▶ raster
|
||||
|
||||
— and the order of that diagram is the whole argument for the stage split. The
|
||||
anchor fit is knob-free. Conditioning smooths its four parameters. The rings are
|
||||
then measured THROUGH the conditioned transform, so `anchor avg` does re-run the
|
||||
ring mapping — a few hundred frames of twenty points, free — and does not re-run
|
||||
anything that reads a source pixel, because that part takes the landmarks and
|
||||
the frames and never the transform.
|
||||
|
||||
TWO CLIPS, ONE STORE. `:take` reads each measured head transform in time;
|
||||
`:take-locked` holds the measured transform from frame zero. They share the
|
||||
same dense blocks; only the head node's anchor map differs."
|
||||
(:require [arthur.flow.address :as address]
|
||||
[arthur.flow.freeze :as freeze]
|
||||
[arthur.flow.take :as take]
|
||||
[arthur.synth :as synth]))
|
||||
|
||||
(def frames 229)
|
||||
(def fps 30)
|
||||
|
||||
(def ^:private stage
|
||||
;; The project's dimensions, and NOT the footage's. This is what deleting
|
||||
;; `makeXform` buys: the head is placed and scaled on the stage by a transform
|
||||
;; on a node, so a 1440x1920 portrait clip and a 320x200 stage are not a
|
||||
;; conflict to resolve. Whatever hangs off the edge is clipped.
|
||||
[320 200])
|
||||
|
||||
(def ^:private aspect
|
||||
;; The synth writes x and y in the SAME unit, so its normalised space is already
|
||||
;; isotropic and the anisotropy correction is the identity here. Real footage
|
||||
;; passes W/H from the manifest at step 6. Worth knowing while reading anything
|
||||
;; here: aspect 1 is the one setting at which a port that dropped the
|
||||
;; anisotropy correction entirely would still look right, which is why
|
||||
;; anchor-test exercises 0.5625 and this does not.
|
||||
1)
|
||||
|
||||
(def analysis
|
||||
"Stage 2's output, synthesised. A seeded generator, so a wrong pose is
|
||||
reproducible rather than something that happened once."
|
||||
(delay (synth/synth-dense frames {:seed 1})))
|
||||
|
||||
(def measured
|
||||
"Stages 3 and 4, in the order the stage split requires."
|
||||
(delay (take/measure (assoc take/knobs :aspect aspect :fps fps)
|
||||
{:dense @analysis})))
|
||||
|
||||
(def params
|
||||
"What the freeze was handed. Public because it is the honest way to re-freeze at
|
||||
other settings — a test that built its own copy would be asserting about a clip
|
||||
nobody looks at."
|
||||
(merge take/knobs
|
||||
{:name "take"
|
||||
:fps fps
|
||||
:stage stage
|
||||
;; On 2s. docs/design.md is emphatic that everything rides ONE grid: a
|
||||
;; head cutting on odd frames against a mouth cutting on even ones reads
|
||||
;; as two performances, so exposure lives on the clip root and inherits.
|
||||
:expose 2
|
||||
;; The head as filmed. `:take-locked` is the same freeze with this one
|
||||
;; field changed, which is the point.
|
||||
:head :free
|
||||
;; Provenance, and now a content address. There is no detector here, so
|
||||
;; the generator IS the detector and its seed is the source: two synth
|
||||
;; takes at different seeds are different analyses, which is the same
|
||||
;; statement content addressing makes about two model versions.
|
||||
:analysis (address/analysis {:detector "synth"
|
||||
:version "mulberry32"
|
||||
:seed 1
|
||||
:frames frames
|
||||
:fps fps
|
||||
:aspect aspect})}))
|
||||
|
||||
(def subject :face-1)
|
||||
|
||||
(def frozen
|
||||
(delay (freeze/clip params {subject @measured})))
|
||||
|
||||
(def store (delay (:store @frozen)))
|
||||
|
||||
(def clip (delay (:clip @frozen)))
|
||||
|
||||
(def locked
|
||||
"The same blocks, with `:head` held at measured frame zero."
|
||||
(delay (freeze/head-mode {:mode :anchored :anchors {0 0}} @frozen)))
|
||||
77
frontend/src/arthur/domain/canon.cljs
Normal file
77
frontend/src/arthur/domain/canon.cljs
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
(ns arthur.domain.canon
|
||||
"One canonical text for a map, so that hashing it means something.
|
||||
|
||||
A content address is a hash of a DESCRIPTION of every input, and a description
|
||||
only addresses anything if the same inputs always write the same bytes. A CLJS
|
||||
map has no key order, `pr-str` will happily print `{:a 1 :b 2}` in either order
|
||||
between runs, and JSON has no canonical form of its own. So this is the one
|
||||
place that decides.
|
||||
|
||||
The text is VALID JSON, deliberately. The server stores it beside the key and
|
||||
verifies `sha256(descriptor) == key` (clips/views.py), and it also has to read
|
||||
two fields out of it to enforce that a detector version was declared at all.
|
||||
Hashing the text the client sent, rather than recomputing it from parsed
|
||||
values, is what keeps that check free of a cross-language float-formatting
|
||||
agreement nobody could hold: Python writes `1.0` where JS writes `1`, and a
|
||||
scheme where both sides re-render the numbers would break on the first integral
|
||||
double. The bytes are the contract; the schema on top of them is a convention.
|
||||
|
||||
It is also meant to be READ. A stale bake presents as a picture that will not
|
||||
update, and the descriptor is the only thing that can say which input moved, so
|
||||
it is short, flat where it can be, and never has a 229-frame mask inlined —
|
||||
see `arthur.flow.address`, which digests masks before they reach here.
|
||||
|
||||
Three refusals, all of them cases where a canonical text is not possible or
|
||||
the key would be ambiguous:
|
||||
|
||||
A KEYWORD VALUE. Keys are keywords and become their names, because a key is
|
||||
a name and nothing else. A keyword VALUE is refused instead of being named,
|
||||
because then `:mouth` and \"mouth\" would hash alike, and the server would be
|
||||
reading a field whose type depended on the caller's mood. Callers convert at
|
||||
the boundary, which is also what makes the stored JSON clean.
|
||||
|
||||
A SET. Unordered, so there is no one text for it. Sort it into a vector at
|
||||
the call site, where it is obvious which order was meant.
|
||||
|
||||
NaN OR INFINITY. Neither is JSON, and both mean a measurement went wrong
|
||||
upstream of here — silently addressing it would cache the mistake."
|
||||
(:require [clojure.string :as str]))
|
||||
|
||||
(defn- number->text [x]
|
||||
(when-not (js/Number.isFinite x)
|
||||
(throw (ex-info "a descriptor cannot hold NaN or infinity" {:value x})))
|
||||
;; `(str 1.0)` is "1" and `(str 0.12)` is "0.12": JS prints the shortest decimal
|
||||
;; that round-trips, so this is stable without a format string.
|
||||
(str x))
|
||||
|
||||
(defn- key->text [k]
|
||||
(cond
|
||||
(keyword? k) (subs (str k) 1) ; :a -> "a", :roto/b -> "roto/b"
|
||||
(string? k) k
|
||||
:else (throw (ex-info "a descriptor key is a keyword or a string"
|
||||
{:key k :type (type k)}))))
|
||||
|
||||
(declare write)
|
||||
|
||||
(defn- write-map [m]
|
||||
(str "{"
|
||||
(str/join "," (map (fn [[k v]] (str (js/JSON.stringify (key->text k)) ":" (write v)))
|
||||
(sort-by (comp key->text key) (seq m))))
|
||||
"}"))
|
||||
|
||||
(defn write
|
||||
"The canonical JSON text of a descriptor value."
|
||||
[v]
|
||||
(cond
|
||||
(nil? v) "null"
|
||||
(true? v) "true"
|
||||
(false? v) "false"
|
||||
(number? v) (number->text v)
|
||||
(string? v) (js/JSON.stringify v)
|
||||
(map? v) (write-map v)
|
||||
(set? v) (throw (ex-info "a descriptor cannot hold a set: sort it into a vector where the order is visible"
|
||||
{:value v}))
|
||||
(keyword? v) (throw (ex-info "a descriptor cannot hold a keyword VALUE: name it at the call site, so \"mouth\" and :mouth cannot address the same block"
|
||||
{:value v}))
|
||||
(sequential? v) (str "[" (str/join "," (map write v)) "]")
|
||||
:else (throw (ex-info "not a descriptor value" {:value v :type (type v)}))))
|
||||
358
frontend/src/arthur/domain/channel.cljs
Normal file
358
frontend/src/arthur/domain/channel.cljs
Normal file
|
|
@ -0,0 +1,358 @@
|
|||
(ns arthur.domain.channel
|
||||
"A channel is one animatable property, sampled at a frame.
|
||||
|
||||
Three shapes, and the uniformity across them is the entire point of the model
|
||||
— analysis does not produce a different kind of data, it produces keys densely
|
||||
on the same channels a hand fills in sparsely:
|
||||
|
||||
FRAMED {:animated? false :value v}
|
||||
A thing that simply exists. A painted background cel is this.
|
||||
|
||||
KEYED {:animated? true :interp :hold :keys {0 v, 4 v, 12 v}}
|
||||
Sparse, authored, in the document. Undoable and syncable.
|
||||
|
||||
DENSE {:animated? true :interp :hold
|
||||
:dense {:store \"sha256:…\" :offset 0 :stride 40 :frames 600}
|
||||
:generated {...}}
|
||||
Generated, one value per frame, in a typed array outside app-db.
|
||||
|
||||
`value-at` reads all three and is the specification. `cursor`/`sample!` is the
|
||||
fast path for playback and must agree with it exactly; scene-test asserts that
|
||||
across forward, backward and random frame order, because a cursor that drifts
|
||||
is a bug you would see as the wrong pose rather than as an error.
|
||||
|
||||
KEYS ARE A MAP BY FRAME, NEVER A VECTOR, and the map stored in the document is
|
||||
a PLAIN map — transit and JSON both lose sortedness, so the sorted index is
|
||||
built here at read time and never persisted.
|
||||
|
||||
:generated is provenance. NOTHING IN HERE READS IT, and nothing downstream may:
|
||||
it exists so the UI can offer a parameter panel instead of raw keys. It lives
|
||||
on the channel rather than the node because a mouth wants a rotoscoped
|
||||
[:geom :pts] and a hand-animated [:xform :pos] at the same time, and putting
|
||||
the flag on the node would forbid the most useful thing in the model.")
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the state mask
|
||||
;;
|
||||
;; PRESENCE IS NOT VISIBILITY, and the distinction is free now and expensive to
|
||||
;; retrofit. A part that is hidden EXISTS and is not drawn, which is `[:vis]`, a
|
||||
;; channel like any other. A subject that is occluded has NO VALUE on that frame
|
||||
;; — there is nothing to hide and nothing to fall back on — and that is this.
|
||||
;;
|
||||
;; The mask carries ABSENCE ONLY. An earlier draft gave it a hidden bit as well,
|
||||
;; per docs/architecture.md's "hidden flag + palette index", and that bit was
|
||||
;; simply a dense `[:vis]` wearing a different hat: two mechanisms for one
|
||||
;; question, which is how you end up with a part that is hidden by one and shown
|
||||
;; by the other. Hiding is a channel; absence is a state. Remaining bits are
|
||||
;; reserved.
|
||||
|
||||
(def ^:const present 0)
|
||||
(def ^:const absent-bit 1)
|
||||
|
||||
(def absent
|
||||
"Sampled value for a frame the subject was not on.
|
||||
|
||||
Distinct from a part being switched off, which is `[:vis]` being false, and
|
||||
distinct from a part having no keys. The identity tracker, when it arrives,
|
||||
needs somewhere to say \"not on screen\" without inventing a pose."
|
||||
::absent)
|
||||
|
||||
(defn nothing?
|
||||
"True when there is no value to draw with."
|
||||
[v]
|
||||
(identical? v absent))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; constructors, for hand-written scenes and tests
|
||||
|
||||
(defn framed [v] {:animated? false :value v})
|
||||
|
||||
(defn keyed
|
||||
([ks] (keyed ks :hold))
|
||||
([ks interp] {:animated? true :interp interp :keys ks :over []}))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(defn component
|
||||
"Component i of a multi-component channel value.
|
||||
|
||||
An authored value is a CLJS vector; a value read out of a dense block is a
|
||||
typed-array view over the block, because copying it would allocate per node
|
||||
per frame. Both have to read the same way here or every consumer downstream
|
||||
grows the same two-way branch."
|
||||
[v i]
|
||||
(if (vector? v) (-nth v i) (aget v i)))
|
||||
|
||||
(defn frames
|
||||
"Sorted vector of the frames a keyed channel has keys on, or nil. Built here
|
||||
and cached by `cursor`; `value-at` rebuilds it, which is why `value-at` is the
|
||||
specification and not the playback path."
|
||||
[ch]
|
||||
(when-let [ks (:keys ch)]
|
||||
(vec (sort (keys ks)))))
|
||||
|
||||
(defn- check-unimplemented!
|
||||
"An override layer must fail LOUDLY rather than be ignored.
|
||||
|
||||
Silently dropping an :over layer would present as a hand
|
||||
correction that did not take — a correction the user made once, watched fail,
|
||||
and has no reason to trust again. Nothing can produce one yet, so this can
|
||||
only fire on a data shape that has run ahead of the code."
|
||||
[ch]
|
||||
(when (seq (:over ch))
|
||||
(throw (ex-info "channel has :over layers and the override layer is not built (port-plan step 2 scope)"
|
||||
{:over (:over ch) :channel (dissoc ch :dense)}))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; dense blocks
|
||||
|
||||
(defn- absent-at?
|
||||
"Is the subject absent on frame f of this block's slice?
|
||||
|
||||
INDEXED THE WAY THE DATA IS. A block is node-major — offset(node i) =
|
||||
i·frames·stride — so a block holding several nodes holds several mask regions,
|
||||
and the one belonging to this channel starts at offset/stride. Indexing the
|
||||
mask by f alone reads the FIRST node's absence for every node in the block,
|
||||
which is not a subtly wrong pose: it is every part in the block vanishing on
|
||||
the frames where one of them was occluded."
|
||||
[state offset stride f]
|
||||
(and (some? state)
|
||||
(pos? (bit-and (aget state (+ (quot offset stride) f)) absent-bit))))
|
||||
|
||||
(defn dense-at
|
||||
"Read frame f out of a dense block.
|
||||
|
||||
`store` is {store-key -> {:data <typed array> :state <Uint8Array or nil>}},
|
||||
tier 2, behind a handle and never in app-db.
|
||||
|
||||
The frame is CLAMPED into the block. A time map with an offset deliberately
|
||||
reads the future — mouth lead is the whole reason `:offset` exists — so the
|
||||
last frame of a leading track is asked for a frame past the end on every one
|
||||
of the last `lead` frames. Clamping there is what the JS `shiftIndex` does and
|
||||
it is the right answer: the track holds its final pose. Returning nothing
|
||||
instead would blank the mouth at the end of every take.
|
||||
|
||||
stride 1 yields a number; anything wider yields a SUBARRAY VIEW over the
|
||||
block, not a copy. Fixed topology is what makes that possible — the frame's
|
||||
data is a rectangular slice at a known offset with no per-frame header.
|
||||
|
||||
FIXED POINT. `:scale` in the block header means the stored integers are the
|
||||
value times that scale, so a block of geometry in image-height units fills an
|
||||
Int16 usefully and a block of stage pixels — which wants a different scale
|
||||
entirely — fills one too. It is in the header rather than agreed by convention
|
||||
for exactly that reason, and it is why the block in memory is byte for byte the
|
||||
block on the wire: a handle that names a sha256 has to name the bytes you
|
||||
actually hold.
|
||||
|
||||
Decoding costs the view. `out` is a stride-sized destination the caller owns —
|
||||
`cursor` allocates one per channel — because a copy per node per frame is the
|
||||
allocation this whole model is arranged to avoid; passing nil allocates, which
|
||||
is what `value-at`, the specification, does."
|
||||
([blk f st] (dense-at blk f st nil))
|
||||
([{:keys [store offset stride scale] nf :frames} f st out]
|
||||
(let [{:keys [data state]} (get st store)]
|
||||
(when (nil? data)
|
||||
(throw (ex-info "dense channel's store key is not in the store"
|
||||
{:store store :have (vec (sort (map str (keys st))))})))
|
||||
(let [f (-> f (max 0) (min (dec nf)))]
|
||||
(if (absent-at? state offset stride f)
|
||||
absent
|
||||
(let [o (+ offset (* f stride))]
|
||||
(cond
|
||||
(= 1 stride) (let [v (aget data o)] (if scale (/ v scale) v))
|
||||
(nil? scale) (.subarray data o (+ o stride))
|
||||
:else (let [dst (or out (js/Float64Array. stride))]
|
||||
(dotimes [k stride]
|
||||
(aset dst k (/ (aget data (+ o k)) scale)))
|
||||
dst))))))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the specification
|
||||
|
||||
(defn segment-interp
|
||||
"How the key at `left` leads to the next key. A channel default remains useful
|
||||
for uniform tracks; :segments overrides only the gaps an artist chose."
|
||||
[ch left]
|
||||
(get (:segments ch) left (:interp ch)))
|
||||
|
||||
(defn- interpolate [ch f left right]
|
||||
(let [a (get (:keys ch) left)]
|
||||
(if (and (= :linear (segment-interp ch left)) right (>= f left) (> right left))
|
||||
(let [b (get (:keys ch) right)
|
||||
t (/ (- f left) (- right left))]
|
||||
(if (vector? a)
|
||||
(mapv (fn [x y] (+ x (* t (- y x)))) a b)
|
||||
(+ a (* t (- b a)))))
|
||||
a)))
|
||||
|
||||
(defn- keyed-at
|
||||
"The most recent key at or before f, CLAMPED to the first key below it.
|
||||
|
||||
Hold is the default and clamping at the low end is the JS `activeKey`'s
|
||||
behaviour, kept: a channel's first key is the pose the part starts in, so a
|
||||
frame before it reads that pose rather than having no value. This is not the
|
||||
same question as presence — a part with no value at all is `absent`, which is
|
||||
a state bit, not an empty key map."
|
||||
[ch f]
|
||||
(let [fr (sort (keys (:keys ch)))
|
||||
left (or (last (take-while #(<= % f) fr)) (first fr))
|
||||
right (first (drop-while #(<= % f) fr))]
|
||||
(interpolate ch f left right)))
|
||||
|
||||
(defn value-at
|
||||
"Sample a channel at frame f. THE SPECIFICATION — correct, allocating, and
|
||||
O(n) in the keys. `cursor`/`sample!` is what playback uses."
|
||||
([ch f] (value-at ch f nil))
|
||||
([ch f store]
|
||||
(check-unimplemented! ch)
|
||||
(cond
|
||||
(not (:animated? ch)) (:value ch)
|
||||
(:dense ch) (dense-at (:dense ch) f store)
|
||||
(:keys ch) (let [ks (:keys ch)]
|
||||
(if (empty? ks) absent (keyed-at ch f)))
|
||||
:else
|
||||
(throw (ex-info "animated channel has neither :keys nor :dense" {:channel ch})))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the playback path
|
||||
;;
|
||||
;; Playback is SEQUENTIAL, so "the most recent key at or before f" is an advance
|
||||
;; of a saved index rather than a search. The difference at 30fps is a `sort` and
|
||||
;; a `take-while` allocation per channel per frame against none, which is the
|
||||
;; difference between the model being usable and being a demo.
|
||||
|
||||
(defn- bsearch
|
||||
"Largest index i with ks[i] <= f, or 0 when f precedes every key (hold clamps
|
||||
low, see keyed-at)."
|
||||
[ks f]
|
||||
(loop [lo 0, hi (dec (count ks)), best 0]
|
||||
(if (> lo hi)
|
||||
best
|
||||
(let [mid (bit-shift-right (+ lo hi) 1)]
|
||||
(if (<= (nth ks mid) f)
|
||||
(recur (inc mid) hi mid)
|
||||
(recur lo (dec mid) best))))))
|
||||
|
||||
(deftype Cursor [ch ks store buf ^:mutable i]
|
||||
Object
|
||||
(toString [_] (str "#Cursor{" (pr-str (if ks :keyed (if (:dense ch) :dense :framed))) " i=" i "}")))
|
||||
|
||||
(defn cursor
|
||||
"A reading head on one channel. Build once per channel per resolver, then
|
||||
`sample!` it per frame. Holds the sorted key index, which is why the index is
|
||||
built here and not in the document — and the decode buffer a fixed-point block
|
||||
needs, for the same reason the resolver owns one point buffer per node.
|
||||
|
||||
Only a wide fixed-point block gets a buffer: a stride-1 block decodes to a
|
||||
number and a block with no `:scale` is handed back as a view."
|
||||
([ch] (cursor ch nil))
|
||||
([ch store]
|
||||
(check-unimplemented! ch)
|
||||
(let [d (:dense ch)]
|
||||
(->Cursor ch
|
||||
(when (and (:animated? ch) (not d) (seq (:keys ch))) (frames ch))
|
||||
store
|
||||
(when (and d (:scale d) (> (:stride d) 1))
|
||||
(js/Float64Array. (:stride d)))
|
||||
0))))
|
||||
|
||||
(defn sample!
|
||||
"Value of the cursor's channel at f. O(1) when f is at or one key past where
|
||||
the cursor already sits — the playback case — and O(log n) otherwise, which is
|
||||
a seek. Advancing and seeking are deliberately different costs: a scrub can
|
||||
afford a binary search and a frame cannot."
|
||||
[^Cursor cur f]
|
||||
(let [ch (.-ch cur)
|
||||
ks (.-ks cur)]
|
||||
(cond
|
||||
(not (:animated? ch)) (:value ch)
|
||||
(:dense ch) (dense-at (:dense ch) f (.-store cur) (.-buf cur))
|
||||
(nil? ks) absent ; animated with an empty key map
|
||||
:else
|
||||
(let [n (count ks)
|
||||
i (.-i cur)
|
||||
last (dec n)
|
||||
i' (cond
|
||||
;; still inside the key the cursor sits on
|
||||
(and (<= (nth ks i) f)
|
||||
(or (= i last) (> (nth ks (inc i)) f)))
|
||||
i
|
||||
;; the next one — one frame of playback crossed one key
|
||||
(and (< i last)
|
||||
(<= (nth ks (inc i)) f)
|
||||
(or (= (inc i) last) (> (nth ks (+ i 2)) f)))
|
||||
(inc i)
|
||||
|
||||
:else (bsearch ks f))]
|
||||
(set! (.-i cur) i')
|
||||
(interpolate ch f (nth ks i') (when (< i' last) (nth ks (inc i'))))))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(defn describe
|
||||
"Which of the three shapes, for error messages and the parameter panel."
|
||||
[ch]
|
||||
(cond
|
||||
(not (:animated? ch)) :framed
|
||||
(:dense ch) :dense
|
||||
:else :keyed))
|
||||
|
||||
(defn problems
|
||||
"Human-readable reasons this map is not a channel. Empty means it is one."
|
||||
[ch]
|
||||
(let [values (when (map? (:keys ch)) (vals (:keys ch)))
|
||||
first-value (first values)
|
||||
linear-values? (or (every? number? values)
|
||||
(and (vector? first-value)
|
||||
(pos? (count first-value))
|
||||
(every? (fn [v] (and (vector? v)
|
||||
(= (count v) (count first-value))
|
||||
(every? number? v)))
|
||||
values)))
|
||||
linear? (or (= :linear (:interp ch))
|
||||
(some #{:linear} (vals (:segments ch))))]
|
||||
(cond-> []
|
||||
(not (map? ch))
|
||||
(conj "not a map")
|
||||
|
||||
(and (map? ch) (not (contains? ch :animated?)))
|
||||
(conj ":animated? is required — the flag is what makes framed and keyed one type")
|
||||
|
||||
(and (map? ch) (:animated? ch) (not (or (:keys ch) (:dense ch))))
|
||||
(conj "animated but has neither :keys nor :dense")
|
||||
|
||||
(and (map? ch) (:animated? ch) (:keys ch) (:dense ch))
|
||||
(conj "has both :keys and :dense; a channel is one shape at a time")
|
||||
|
||||
(and (map? ch) (:keys ch) (not (map? (:keys ch))))
|
||||
(conj (str ":keys is a " (if (vector? (:keys ch)) "vector" "non-map")
|
||||
" — keys are a MAP by frame, so a merge can be per-key"))
|
||||
|
||||
(and (map? ch) (:keys ch) (map? (:keys ch)) (not (every? number? (keys (:keys ch)))))
|
||||
(conj ":keys has a non-numeric frame")
|
||||
|
||||
(and (map? ch) (:animated? ch) (not (#{:hold :linear nil} (:interp ch))))
|
||||
(conj (str ":interp " (:interp ch) " is not implemented"))
|
||||
|
||||
(and (map? ch) (contains? ch :segments)
|
||||
(or (not (map? (:segments ch)))
|
||||
(not (:keys ch))
|
||||
(not (every? (set (keys (:keys ch))) (keys (:segments ch))))
|
||||
(not (every? #{:hold :linear} (vals (:segments ch))))))
|
||||
(conj ":segments must map existing key frames to :hold or :linear")
|
||||
|
||||
(and (map? ch) linear?
|
||||
(or (:dense ch) (not linear-values?)))
|
||||
(conj ":linear interpolation needs numeric keys of one shape")
|
||||
|
||||
(and (map? ch) (seq (:over ch)))
|
||||
(conj ":over layers are not implemented (port-plan step 2 scope)")
|
||||
|
||||
;; A scale of zero divides every value in the block by zero, and a negative
|
||||
;; one mirrors the geometry. Both are authored-data bugs that present as a
|
||||
;; part drawn nowhere or inside out, not as an error.
|
||||
(and (map? ch) (:dense ch) (contains? (:dense ch) :scale)
|
||||
(not (and (number? (:scale (:dense ch))) (pos? (:scale (:dense ch))))))
|
||||
(conj (str ":dense :scale is " (pr-str (:scale (:dense ch)))
|
||||
" — a fixed-point scale is a positive number the stored integers"
|
||||
" were multiplied by")))))
|
||||
204
frontend/src/arthur/domain/clip.cljs
Normal file
204
frontend/src/arthur/domain/clip.cljs
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
(ns arthur.domain.clip
|
||||
"A CLIP: the unit of work, and a library of timelines.
|
||||
|
||||
{:name \"take\"
|
||||
:fps 30
|
||||
:width 320 :height 200
|
||||
:analysis {...}
|
||||
:subjects {...} :features {...} :groups {...}
|
||||
:timelines {:main {:id :main :frames 229 :nodes {...}}}}
|
||||
|
||||
Every field here is a fact about the clip and NOT about a bag of nodes, which is
|
||||
the cut this namespace exists to make. Before it, one map carried both: `:fps`,
|
||||
the stage dimensions, the analysis record and the tracking identities sat beside
|
||||
`:nodes`, and `arthur.db` said of it — correctly — that they \"sit on the scene
|
||||
map only because there is one clip per scene today\". The cost of leaving them
|
||||
together was not untidiness. It was that a SYMBOL had nowhere to live: a library
|
||||
timeline is a bag of nodes with a frame space and nothing else, so under the old
|
||||
shape it would have had to be a clip with seven meaningless fields, or a second
|
||||
structure with the same `:nodes` key that every walk had to be taught about.
|
||||
|
||||
Now there is one node-holding type — `arthur.domain.timeline` — and a clip holds
|
||||
a MAP of them. A `:kind :symbol` instance names a timeline in `:timelines`,
|
||||
and the clip resolver gives each placement its own reading heads.
|
||||
|
||||
THE ROOT TIMELINE HAS A RESERVED ID, `:main`, rather than the clip carrying a
|
||||
pointer to it. A pointer is a field that can be wrong — it can name a timeline
|
||||
that is not there, and then every reader needs a fallback — where a reserved name
|
||||
can only be absent, which `problems` reports once. Flash reserves `_root` the
|
||||
same way and for the same reason. Nothing else about `:main` is special: it is an
|
||||
ordinary entry in the map, and a symbol is another one.
|
||||
|
||||
WHY :fps IS HERE AND :frames IS NOT. A rate is how fast the whole clip plays
|
||||
against its audio, and a nested timeline cannot have one of its own — retiming an
|
||||
instance is `:rate` on its `:time` map, which is a factor and not a rate. A
|
||||
frame COUNT is a property of a frame space, so every timeline has its own."
|
||||
(:require [arthur.domain.feature :as feature]
|
||||
[arthur.domain.node :as node]
|
||||
[arthur.domain.palette :as pal]
|
||||
[arthur.domain.pose :as pose]
|
||||
[arthur.domain.timeline :as timeline]))
|
||||
|
||||
(def ^:const root-id
|
||||
"The reserved id of the timeline a clip plays. See the namespace docstring."
|
||||
:main)
|
||||
|
||||
(def clip-keys
|
||||
"Every top-level field of a clip, and the reason `arthur.domain.leaf` refuses
|
||||
one it does not know: a field added to the clip without a leaf to save it in is
|
||||
a field that saves silently and comes back missing. The failure is a document
|
||||
that loses something on every round trip, which is the one bug a persistence
|
||||
layer must not be able to have. Add the field here and to `leaf/leaves` and
|
||||
`leaf/clip` in the same commit."
|
||||
#{:name :fps :analysis :subjects :features :groups :width :height :timelines})
|
||||
|
||||
(defn timeline
|
||||
"One of the clip's timelines, by id."
|
||||
[clip id]
|
||||
(get-in clip [:timelines id]))
|
||||
|
||||
(defn root
|
||||
"The timeline the clip plays."
|
||||
[clip]
|
||||
(timeline clip root-id))
|
||||
|
||||
(defn frames
|
||||
"The clip's length, which is its root timeline's frame space and is not written
|
||||
down twice. Reading it off the root is what stops the two from disagreeing."
|
||||
[clip]
|
||||
(:frames (root clip)))
|
||||
|
||||
(defn update-timeline
|
||||
"Apply f to one timeline in place."
|
||||
[clip id f & args]
|
||||
(apply update-in clip [:timelines id] f args))
|
||||
|
||||
(defn update-root [clip f & args]
|
||||
(apply update-timeline clip root-id f args))
|
||||
|
||||
(defn nodes
|
||||
"The root timeline's nodes. A convenience for the many callers that mean the
|
||||
root and would otherwise spell it out; anything that could mean a symbol says
|
||||
which timeline instead."
|
||||
[clip]
|
||||
(:nodes (root clip)))
|
||||
|
||||
(defn- transform-op
|
||||
"Put a symbol's already resolved mark into its instance's parent space."
|
||||
[op m path]
|
||||
(let [at (fn [x y] [(+ (* (aget m 0) x) (* (aget m 2) y) (aget m 4))
|
||||
(+ (* (aget m 1) x) (* (aget m 3) y) (aget m 5))])
|
||||
scale (node/mean-scale m)
|
||||
op (assoc op :node (conj path (:node op)))]
|
||||
(case (:kind op)
|
||||
:poly (let [out (js/Float64Array. (.-length (:pts op)))]
|
||||
(dotimes [i (:n op)]
|
||||
(let [[x y] (at (aget (:pts op) (* 2 i))
|
||||
(aget (:pts op) (inc (* 2 i))))]
|
||||
(aset out (* 2 i) x)
|
||||
(aset out (inc (* 2 i)) y)))
|
||||
(assoc op :pts out))
|
||||
:disc (let [[x y] (at (:cx op) (:cy op))]
|
||||
(assoc op :cx x :cy y :r (* scale (:r op))))
|
||||
:rect (let [[x y] (at (:cx op) (:cy op))]
|
||||
(assoc op :cx x :cy y :size (* scale (:size op))))
|
||||
op)))
|
||||
|
||||
(defn resolver
|
||||
"Resolve a clip, including each library timeline placed by a symbol instance.
|
||||
|
||||
Each instance owns its own timeline resolver, so two offsets never share a
|
||||
channel cursor or point buffer. The returned ops must be drawn before the next
|
||||
frame, as with timeline/resolver.
|
||||
|
||||
`root` is which timeline to resolve AS the root, and it defaults to the clip's.
|
||||
Passing a symbol's id is the whole of \"render that symbol\": a library timeline
|
||||
and the clip's own are the same type, so a symbol resolves by being rooted
|
||||
rather than by a second code path — which is the return on collapsing the two
|
||||
into `domain/timeline`. Its frame space is its own `:frames`, and nested symbols
|
||||
inside it still resolve, because this is the function that knows how to do that."
|
||||
([clip store] (resolver clip store pal/index-of root-id))
|
||||
([clip store palette] (resolver clip store palette root-id))
|
||||
([clip store palette root] (resolver clip store palette root nil))
|
||||
([clip store palette root {:keys [picture-fps] :as opts}]
|
||||
(letfn [(build [tid chain pose-tracks]
|
||||
(when (some #{tid} chain)
|
||||
(throw (ex-info "symbol timeline cycle" {:chain (conj chain tid)})))
|
||||
(let [tl (or (timeline clip tid)
|
||||
(throw (ex-info "symbol names a missing timeline" {:timeline tid})))
|
||||
nodes (:nodes tl)
|
||||
rank (timeline/draw-rank nodes (timeline/order nodes))
|
||||
ids (sort-by rank (keys nodes))
|
||||
own (timeline/resolver tl store palette pose-tracks
|
||||
(assoc opts :source-fps (:fps clip)))
|
||||
children (into {}
|
||||
(for [[id n] nodes :when (= :symbol (:kind n))]
|
||||
[id (build (:of n) (conj chain tid)
|
||||
(get-in n [:playback :tracks]))]))]
|
||||
(fn [f]
|
||||
(let [by-id (into {} (map (juxt :node identity)) (own f))]
|
||||
(into []
|
||||
(mapcat
|
||||
(fn [id]
|
||||
(let [n (get nodes id)]
|
||||
(if (= :symbol (:kind n))
|
||||
(let [m (timeline/world-of own id)
|
||||
local (timeline/frame-of own id)
|
||||
target (timeline clip (:of n))
|
||||
length (:frames target)
|
||||
frame (when (and m (number? local))
|
||||
(if (get-in n [:time :loop?])
|
||||
(mod local length)
|
||||
local))]
|
||||
(if (and frame (<= 0 frame) (< frame length))
|
||||
(map #(transform-op % m [id]) ((get children id) frame))
|
||||
[]))
|
||||
(when-let [op (get by-id id)] [op]))))
|
||||
ids))))))]
|
||||
(build root [] nil))))
|
||||
|
||||
(defn problems
|
||||
"Human-readable reasons this clip will not evaluate or save."
|
||||
[clip]
|
||||
(vec
|
||||
(concat
|
||||
(for [k (remove clip-keys (keys clip))]
|
||||
(str "clip has a field with no leaf to save it in: " (pr-str k)))
|
||||
(when-not (map? (:timelines clip))
|
||||
[":timelines must be a map of id -> timeline"])
|
||||
(when (and (map? (:timelines clip)) (nil? (root clip)))
|
||||
[(str "no " (pr-str root-id) " timeline — a clip plays the one with the reserved id")])
|
||||
(when-not (or (nil? (:fps clip)) (and (number? (:fps clip)) (pos? (:fps clip))))
|
||||
[(str ":fps is " (pr-str (:fps clip)) " — a rate is a positive number")])
|
||||
(for [[id tl] (:timelines clip)
|
||||
:when (not= id (:id tl))]
|
||||
(str "timeline under key " (pr-str id) " has :id " (pr-str (:id tl))))
|
||||
(for [[id tl] (:timelines clip)
|
||||
p (timeline/problems tl)]
|
||||
(str "timeline " (pr-str id) ": " p))
|
||||
(for [[tid tl] (:timelines clip)
|
||||
[id n] (:nodes tl)
|
||||
:when (and (= :symbol (:kind n))
|
||||
(not (contains? (:timelines clip) (:of n))))]
|
||||
(str "timeline " (pr-str tid) " symbol " (pr-str id)
|
||||
" names missing timeline " (pr-str (:of n))))
|
||||
(for [[tid tl] (:timelines clip)
|
||||
[id n] (:nodes tl)
|
||||
:when (= :symbol (:kind n))
|
||||
:let [target (get-in clip [:timelines (:of n)])
|
||||
active (filter (fn [node]
|
||||
(some :pose-sampled? (vals (:channels node))))
|
||||
(vals (:nodes target)))
|
||||
groups (set (concat
|
||||
(map #(or (:pose-group %) (:id %)) active)
|
||||
(map #(vector :node (:id %)) active)))]
|
||||
p (pose/problems (get-in n [:playback :tracks])
|
||||
(:frames target) groups)]
|
||||
(str "timeline " (pr-str tid) " symbol " (pr-str id) ": " p))
|
||||
(for [[tid tl] (:timelines clip)
|
||||
[id n] (:nodes tl)
|
||||
:when (and (= :audio (:kind n)) (:linked-to n)
|
||||
(not (contains? (:nodes tl) (:linked-to n))))]
|
||||
(str "timeline " (pr-str tid) " audio " (pr-str id)
|
||||
" links to missing node " (pr-str (:linked-to n))))
|
||||
(feature/problems clip))))
|
||||
47
frontend/src/arthur/domain/crc32.cljs
Normal file
47
frontend/src/arthur/domain/crc32.cljs
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
(ns arthur.domain.crc32
|
||||
"CRC-32, as PNG chunks and ZIP entries both define it.
|
||||
|
||||
ONE implementation for both, and that is not premature sharing: a PNG chunk's
|
||||
trailing checksum and a ZIP local header's `crc-32` field are the same function
|
||||
of the same bytes — IEEE 802.3, reflected, with an initial and final complement
|
||||
— down to the polynomial. Two copies would be two chances to get the table
|
||||
wrong in a way that reads as \"the file is corrupt\" rather than as \"these two
|
||||
functions disagree\".
|
||||
|
||||
It lives beside `domain/sha256` for the same reason that one does: a digest is
|
||||
a pure function of bytes with no DOM in it, so every assertion about it runs
|
||||
under node.")
|
||||
|
||||
(def ^:private table
|
||||
;; The standard 256-entry table, built once. The bit-twiddling loop IS the
|
||||
;; definition of the polynomial and there is no collection idiom hiding in it:
|
||||
;; each entry is eight dependent shifts of one accumulator.
|
||||
(let [t (js/Uint32Array. 256)]
|
||||
(dotimes [n 256]
|
||||
(aset t n (loop [c n k 0]
|
||||
(if (= k 8)
|
||||
c
|
||||
(recur (if (odd? c)
|
||||
(bit-xor 0xedb88320 (unsigned-bit-shift-right c 1))
|
||||
(unsigned-bit-shift-right c 1))
|
||||
(inc k))))))
|
||||
t))
|
||||
|
||||
(defn of
|
||||
"CRC-32 of a byte array, or of the half-open range [from to) of one, as an
|
||||
unsigned 32-bit number.
|
||||
|
||||
`loop` over the bytes rather than a reduce over a `range`: this walks the whole
|
||||
of every PNG written, which at 1920x1200 is seven megabytes a frame, and a seq
|
||||
cell per byte is the allocation the rest of this codebase is arranged to
|
||||
avoid."
|
||||
([bytes] (of bytes 0 (.-length bytes)))
|
||||
([bytes from to]
|
||||
(-> (loop [c 0xffffffff i from]
|
||||
(if (>= i to)
|
||||
c
|
||||
(recur (bit-xor (aget table (bit-and (bit-xor c (aget bytes i)) 0xff))
|
||||
(unsigned-bit-shift-right c 8))
|
||||
(inc i))))
|
||||
(bit-xor 0xffffffff)
|
||||
(unsigned-bit-shift-right 0))))
|
||||
98
frontend/src/arthur/domain/feature.cljs
Normal file
98
frontend/src/arthur/domain/feature.cljs
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
(ns arthur.domain.feature
|
||||
"Tracked subjects, feature ownership, and eye-pair settings.
|
||||
Features name their timeline explicitly; node ids are local to that timeline."
|
||||
(:require [arthur.domain.params :as params]))
|
||||
|
||||
(defn owned
|
||||
"A deterministic clip-level feature or group id. Nodes keep local names."
|
||||
[subject role]
|
||||
(keyword (subs (str subject) 1) (name role)))
|
||||
|
||||
(defn group-for [clip feature-id]
|
||||
(first (filter (fn [[_ group]] (some #{feature-id} (:members group)))
|
||||
(:groups clip))))
|
||||
|
||||
(defn effective-params
|
||||
"Resolve static settings for one feature. A future parameter channel can
|
||||
replace a scalar at this boundary without changing feature or pair identity."
|
||||
[clip feature-id]
|
||||
(let [{:keys [subject area params] :as feature} (get-in clip [:features feature-id])
|
||||
[_ group] (group-for clip feature-id)]
|
||||
(when-not feature
|
||||
(throw (ex-info "unknown feature" {:feature feature-id})))
|
||||
(merge (params/for-area :subject)
|
||||
(get-in clip [:subjects subject :params])
|
||||
(params/for-area area)
|
||||
(:params group)
|
||||
params)))
|
||||
|
||||
(defn remove-from-pair
|
||||
"Keep the eye's current settings when its association is removed. Empty pairs
|
||||
are removed; a one-eye pair remains valid and can acquire a partner later."
|
||||
[clip feature-id]
|
||||
(if-let [[group-id group] (group-for clip feature-id)]
|
||||
(let [area (get-in clip [:features feature-id :area])
|
||||
values (select-keys (effective-params clip feature-id)
|
||||
(keys (params/for-area area)))
|
||||
members (vec (remove #{feature-id} (:members group)))]
|
||||
(-> clip
|
||||
(assoc-in [:features feature-id :params] values)
|
||||
(update :groups (fn [groups]
|
||||
(if (seq members)
|
||||
(assoc-in groups [group-id :members] members)
|
||||
(dissoc groups group-id))))))
|
||||
clip))
|
||||
|
||||
(defn problems
|
||||
"Check tracked identities and timeline-local node ownership."
|
||||
[clip]
|
||||
(let [subjects (:subjects clip)
|
||||
features (:features clip)
|
||||
groups (:groups clip)
|
||||
memberships (mapcat (comp :members val) groups)
|
||||
node-owners (for [[_ f] features n (:nodes f)]
|
||||
[(:timeline f) n])]
|
||||
(vec
|
||||
(concat
|
||||
(for [[id s] subjects :when (not= id (:id s))]
|
||||
(str "subject " (pr-str id) " has a different :id"))
|
||||
(for [[id s] subjects
|
||||
:when (not (params/valid-settings? :subject (or (:params s) {})))]
|
||||
(str "subject " (pr-str id) " has invalid settings"))
|
||||
(for [[id _] subjects
|
||||
:when (not (seq (get-in clip [:timelines id :nodes :head :measured])))]
|
||||
(str "subject " (pr-str id) " has no measured head in its timeline"))
|
||||
(for [[id f] features :when (not= id (:id f))]
|
||||
(str "feature " (pr-str id) " has a different :id"))
|
||||
(for [[id f] features :when (not (contains? subjects (:subject f)))]
|
||||
(str "feature " (pr-str id) " has no subject"))
|
||||
(for [[id f] features :when (not (contains? (disj params/areas :subject) (:area f)))]
|
||||
(str "feature " (pr-str id) " has an unknown area"))
|
||||
(for [[id f] features
|
||||
:when (not (params/valid-settings? (:area f) (or (:params f) {})))]
|
||||
(str "feature " (pr-str id) " has invalid settings for " (pr-str (:area f))))
|
||||
(for [[id f] features
|
||||
:when (not (contains? (:timelines clip) (:timeline f)))]
|
||||
(str "feature " (pr-str id) " names a missing timeline"))
|
||||
(for [[id f] features node-id (:nodes f)
|
||||
:let [owned-nodes (get-in clip [:timelines (:timeline f) :nodes])]
|
||||
:when (not (contains? owned-nodes node-id))]
|
||||
(str "feature " (pr-str id) " refers to missing node " (pr-str node-id)))
|
||||
(for [[id n] (frequencies node-owners) :when (> n 1)]
|
||||
(str "node " (pr-str id) " belongs to more than one feature"))
|
||||
(for [[id g] groups :when (not= id (:id g))]
|
||||
(str "group " (pr-str id) " has a different :id"))
|
||||
(for [[id g] groups
|
||||
:when (not (and (= :eye-pair (:kind g))
|
||||
(<= 1 (count (:members g)) 2)
|
||||
(= (count (:members g)) (count (distinct (:members g))))))]
|
||||
(str "group " (pr-str id) " must be an eye pair of one or two distinct eyes"))
|
||||
(for [[id g] groups
|
||||
:when (not (params/valid-settings? :eye (or (:params g) {})))]
|
||||
(str "group " (pr-str id) " has invalid eye settings"))
|
||||
(for [[id g] groups member (:members g)
|
||||
:let [f (get features member)]
|
||||
:when (not (and f (= :eye (:area f)) (= (:subject g) (:subject f))))]
|
||||
(str "group " (pr-str id) " has an eye from another subject or an unknown feature"))
|
||||
(for [[id n] (frequencies memberships) :when (> n 1)]
|
||||
(str "feature " (pr-str id) " belongs to more than one group"))))))
|
||||
136
frontend/src/arthur/domain/geom.cljs
Normal file
136
frontend/src/arthur/domain/geom.cljs
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
(ns arthur.domain.geom
|
||||
"2D similarity transforms and temporal smoothing.
|
||||
|
||||
A transform is {:s :theta :tx :ty}; a point is {:x :y}. Both stay maps at this
|
||||
layer: this is the numeric oracle the JS is diffed against, and a faithful
|
||||
port is worth more here than a fast one. The dense typed-array
|
||||
representations appear at the freeze boundary, not below it.")
|
||||
|
||||
(defn centroid
|
||||
"Mean of a point set."
|
||||
[pts]
|
||||
(let [n (count pts)]
|
||||
{:x (/ (transduce (map :x) + 0.0 pts) n)
|
||||
:y (/ (transduce (map :y) + 0.0 pts) n)}))
|
||||
|
||||
(defn fit-similarity
|
||||
"Least-squares similarity (translation + rotation + uniform scale, 4 DOF)
|
||||
mapping P onto Q. Closed form; no iteration.
|
||||
|
||||
Deliberately NOT affine or homography: the extra degrees of freedom absorb
|
||||
out-of-plane head rotation as shear/perspective and smear it into the mouth.
|
||||
Four DOF removes exactly translation, roll and depth-scale, and leaves yaw and
|
||||
pitch as a measurable residual."
|
||||
[P Q]
|
||||
(let [n (count P)
|
||||
cp (centroid P)
|
||||
cq (centroid Q)
|
||||
;; Dot, cross and squared norm of the centred configurations, in one
|
||||
;; pass. Reduced in input order, so the floating-point result is bit for
|
||||
;; bit what an index loop would give and the 1e-9 parity against the JS
|
||||
;; holds.
|
||||
[a b norm]
|
||||
(reduce (fn [[a b norm] [p q]]
|
||||
(let [px (- (:x p) (:x cp)) py (- (:y p) (:y cp))
|
||||
qx (- (:x q) (:x cq)) qy (- (:y q) (:y cq))]
|
||||
[(+ a (+ (* px qx) (* py qy))) ; dot
|
||||
(+ b (- (* px qy) (* py qx))) ; cross
|
||||
(+ norm (+ (* px px) (* py py)))]))
|
||||
[0.0 0.0 0.0]
|
||||
(map vector P Q))
|
||||
pcx (:x cp) pcy (:y cp)
|
||||
qcx (:x cq) qcy (:y cq)
|
||||
theta (js/Math.atan2 b a)
|
||||
;; A degenerate configuration has nothing to recover a scale from. Fall
|
||||
;; back to 1 rather than dividing by zero: one bad detection frame would
|
||||
;; otherwise poison the Procrustes mean and therefore every frame.
|
||||
s (if (> norm 1e-12) (/ (js/Math.hypot a b) norm) 1)
|
||||
c (js/Math.cos theta)
|
||||
sn (js/Math.sin theta)]
|
||||
{:s s
|
||||
:theta theta
|
||||
:tx (- qcx (* s (- (* c pcx) (* sn pcy))))
|
||||
:ty (- qcy (* s (+ (* sn pcx) (* c pcy))))}))
|
||||
|
||||
(defn apply-sim [tf p]
|
||||
(let [c (js/Math.cos (:theta tf))
|
||||
sn (js/Math.sin (:theta tf))]
|
||||
{:x (+ (* (:s tf) (- (* c (:x p)) (* sn (:y p)))) (:tx tf))
|
||||
:y (+ (* (:s tf) (+ (* sn (:x p)) (* c (:y p)))) (:ty tf))}))
|
||||
|
||||
(defn apply-sim-all [tf pts]
|
||||
(mapv #(apply-sim tf %) pts))
|
||||
|
||||
(defn fit-residual
|
||||
"Residual RMS after the fit, in the units of Q. Rises with out-of-plane
|
||||
rotation, so it is the signal for \"this section is not stabilisable\"."
|
||||
[tf P Q]
|
||||
(let [sq (fn [d] (* d d))]
|
||||
(js/Math.sqrt
|
||||
(/ (transduce (map (fn [[p q]]
|
||||
(let [m (apply-sim tf p)]
|
||||
(+ (sq (- (:x m) (:x q)))
|
||||
(sq (- (:y m) (:y q)))))))
|
||||
+ 0.0 (map vector P Q))
|
||||
(count P)))))
|
||||
|
||||
(defn procrustes-mean
|
||||
"Generalised Procrustes: the reference is the MEAN rigid configuration over the
|
||||
shot, not frame zero, so no single frame's idiosyncrasies get baked into every
|
||||
other frame. Three passes is plenty."
|
||||
([frames-rigid] (procrustes-mean frames-rigid 3))
|
||||
([frames-rigid iters]
|
||||
(let [n (count frames-rigid)
|
||||
;; One pass: fit every frame onto the current reference, sum the
|
||||
;; aligned configurations, divide. Iterative refinement, so the whole
|
||||
;; thing is `iterate` taken `iters` deep — which is what the algorithm
|
||||
;; actually says, rather than a counter that happens to stop.
|
||||
refine (fn [ref]
|
||||
(->> frames-rigid
|
||||
(reduce (fn [acc rig]
|
||||
(let [moved (apply-sim-all (fit-similarity rig ref) rig)]
|
||||
(mapv (fn [a m] {:x (+ (:x a) (:x m))
|
||||
:y (+ (:y a) (:y m))})
|
||||
acc moved)))
|
||||
(mapv (constantly {:x 0.0 :y 0.0}) ref))
|
||||
(mapv (fn [p] {:x (/ (:x p) n) :y (/ (:y p) n)}))))]
|
||||
(-> (iterate refine (mapv (fn [p] {:x (:x p) :y (:y p)}) (first frames-rigid)))
|
||||
(nth iters)))))
|
||||
|
||||
(defn moving-average
|
||||
"`radius` is in frames either side: 0 is off, 1 averages over 3 frames, 2 over
|
||||
5. Expressed as a radius rather than a window so that \"off\" is 0 and every
|
||||
value is symmetric - an even window would be lopsided in time."
|
||||
[vals radius]
|
||||
(if (<= radius 0)
|
||||
(vec vals)
|
||||
(let [v (vec vals)
|
||||
n (count v)
|
||||
half (js/Math.floor radius)]
|
||||
(mapv (fn [i]
|
||||
;; Clamped at the ends rather than shortened, so every output is an
|
||||
;; average of the same COUNT of samples and the first frame is not
|
||||
;; noisier than the rest.
|
||||
(let [lo (- i half) hi (+ i half)]
|
||||
(/ (reduce + (map (fn [j] (nth v (min (dec n) (max 0 j))))
|
||||
(range lo (inc hi))))
|
||||
(inc (- hi lo)))))
|
||||
(range n)))))
|
||||
|
||||
(defn smooth-transforms
|
||||
"Smooth the four transform parameters, NEVER the contour. Landmark jitter of a
|
||||
pixel is smeared into the mouth by the inverse transform, so the transform is
|
||||
where the low-pass belongs; smoothing the contour would destroy the
|
||||
performance, which is the entire asset.
|
||||
Angles are smoothed as (cos, sin) so wrapping cannot produce a spike."
|
||||
[tfs radius]
|
||||
(let [c (moving-average (map #(js/Math.cos (:theta %)) tfs) radius)
|
||||
sn (moving-average (map #(js/Math.sin (:theta %)) tfs) radius)
|
||||
s (moving-average (map :s tfs) radius)
|
||||
tx (moving-average (map :tx tfs) radius)
|
||||
ty (moving-average (map :ty tfs) radius)]
|
||||
(mapv (fn [i] {:theta (js/Math.atan2 (nth sn i) (nth c i))
|
||||
:s (nth s i)
|
||||
:tx (nth tx i)
|
||||
:ty (nth ty i)})
|
||||
(range (count tfs)))))
|
||||
115
frontend/src/arthur/domain/landmarks.cljs
Normal file
115
frontend/src/arthur/domain/landmarks.cljs
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
(ns arthur.domain.landmarks
|
||||
"MediaPipe FaceLandmarker index tables.
|
||||
|
||||
Ring vectors are ORDERED traversals, not raw connection sets: vertex position
|
||||
within a ring is the vertex's identity, and every downstream stage depends on
|
||||
that ordering being stable. See docs/design.md, \"Fixed topology\".
|
||||
|
||||
Tables only. The operations over a ring — subsample, offset, simplicity —
|
||||
live in arthur.domain.ring, because they are about ordered traversals in
|
||||
general and know nothing about faces.")
|
||||
|
||||
;; Rigid landmarks for the similarity fit. Eye corners, nose bridge, nose tip.
|
||||
;; Nothing here may be a feature that moves under performance: including the
|
||||
;; mouth or brows bleeds performance into the stabilization.
|
||||
(def RIGID [33 133 362 263 168 6 1])
|
||||
|
||||
;; Outer lip ring, clockwise from the right corner over the top.
|
||||
;; index 0 = right corner, 5 = top centre, 10 = left corner, 15 = bottom centre.
|
||||
(def LIPS-OUTER
|
||||
[61 185 40 39 37 0 267 269 270 409
|
||||
291 375 321 405 314 17 84 181 91 146])
|
||||
|
||||
;; Inner lip ring, same orientation and the same four cardinal positions.
|
||||
(def LIPS-INNER
|
||||
[78 191 80 81 82 13 312 311 310 415
|
||||
308 324 318 402 317 14 87 178 88 95])
|
||||
|
||||
;; Inner upper / lower lip centres. Their separation is the aperture signal that
|
||||
;; decides whether the mouth interior is present at all.
|
||||
(def APERTURE [13 14])
|
||||
|
||||
;; Face oval, used only to derive the placeholder plate in v1.
|
||||
(def FACE-OVAL
|
||||
[10 338 297 332 284 251 389 356 454 323 361 288
|
||||
397 365 379 378 400 377 152 148 176 149 150 136
|
||||
172 58 132 93 234 127 162 21 54 103 67 109])
|
||||
|
||||
;; Eye corners, for the calibration box and for reporting fit residual.
|
||||
(def EYE-INNER [133 362])
|
||||
|
||||
;; ---- eyes ----
|
||||
;;
|
||||
;; Eyelid rings, under the same contract as the lip rings: ORDERED traversals
|
||||
;; where slot position IS vertex identity. Both eyes start at the OUTER corner
|
||||
;; and go over the UPPER lid first, so slot k means the same anatomy on both
|
||||
;; sides. On a 16-slot ring that puts the four cardinals exactly on the four
|
||||
;; quarter slots - 0 outer corner, 4 upper lid centre, 8 inner corner, 12 lower
|
||||
;; lid centre - so every even vertex budget lands on real landmarks.
|
||||
;;
|
||||
;; The two rings traverse opposite directions on screen, because they are
|
||||
;; mirrored anatomy described the same way. Nothing downstream cares: an
|
||||
;; even-odd fill has no winding, and ring SIMPLICITY is what is asserted.
|
||||
(def EYE-R-RING
|
||||
[33 246 161 160 159 158 157 173
|
||||
133 155 154 153 145 144 163 7])
|
||||
|
||||
(def EYE-L-RING
|
||||
[263 466 388 387 386 385 384 398
|
||||
362 382 381 380 374 373 390 249])
|
||||
|
||||
;; Outer, inner corner per eye. All four are also in RIGID, and that is the
|
||||
;; point: the eye's reference frame is built only from landmarks that do not
|
||||
;; move under performance, so a blink cannot be mistaken for a change of gaze.
|
||||
(def EYE-R-CORNERS [33 133])
|
||||
(def EYE-L-CORNERS [263 362])
|
||||
|
||||
;; Upper and lower lid centres. Their separation over the corner distance is the
|
||||
;; openness signal that decides whether the eye is shut - the same shape of
|
||||
;; measurement as APERTURE is for the mouth, but normalised, so one threshold
|
||||
;; carries across takes and faces.
|
||||
(def EYE-R-LIDS [159 145])
|
||||
(def EYE-L-LIDS [386 374])
|
||||
|
||||
;; The two iris blocks the refined mesh appends: centre first, then four ring
|
||||
;; points. WHICH BLOCK BELONGS TO WHICH EYE IS NOT DECLARED HERE - MediaPipe's
|
||||
;; own "left"/"right" is viewer-relative in some docs and subject-relative in
|
||||
;; others, and a swap looks almost right, so it would survive an eyeball and
|
||||
;; then read as a permanently wall-eyed character. flow/measure/eyes resolves it
|
||||
;; from the geometry instead.
|
||||
(def IRIS-A [468 469 470 471 472])
|
||||
(def IRIS-B [473 474 475 476 477])
|
||||
|
||||
;; ---- brows ----
|
||||
;;
|
||||
;; Each brow is two five-point chains, an upper edge and a lower edge, which
|
||||
;; close into a ten-point ring: out along one edge from the outer end to the
|
||||
;; inner, back along the other.
|
||||
;;
|
||||
;; WHICH EDGE IS UPPER IS DELIBERATELY NOT DECLARED, and unlike the iris it does
|
||||
;; not need to be. Swapping them traverses the same ring the other way round,
|
||||
;; and an even-odd fill has no winding, so the shape is identical either way.
|
||||
;; What the ring guarantees instead is that the two ENDS land on fixed slots:
|
||||
;; 0 and 9 are one end, 4 and 5 the other. Averaging a pair therefore gives the
|
||||
;; brow's height at that end whichever edge is on top, which is all the raise
|
||||
;; and tilt measurement needs.
|
||||
;;
|
||||
;; Which end is the OUTER one is resolved from geometry in flow/measure/brows,
|
||||
;; because getting it backwards mirrors the tilt - inner-up "worried" would
|
||||
;; render as outer-up - and that is an expression error, not a glitch, so it
|
||||
;; would read as a directed performance choice rather than as a bug.
|
||||
(def BROW-A-RING
|
||||
[70 63 105 66 107
|
||||
55 65 52 53 46])
|
||||
|
||||
(def BROW-B-RING
|
||||
[300 293 334 296 336
|
||||
285 295 282 283 276])
|
||||
|
||||
;; The slots at each end of a brow ring, as pairs to average.
|
||||
(def BROW-END-0 [0 9])
|
||||
(def BROW-END-1 [4 5])
|
||||
|
||||
;; The number of landmarks the refined mesh emits: 468 face + 10 iris. Dense
|
||||
;; frames are this long whether or not the iris blocks carry anything.
|
||||
(def NUM-LANDMARKS 478)
|
||||
250
frontend/src/arthur/domain/leaf.cljs
Normal file
250
frontend/src/arthur/domain/leaf.cljs
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
(ns arthur.domain.leaf
|
||||
"Leaf addressing for tier 1: the document as a map of PATH -> value.
|
||||
|
||||
This is the shape docs/architecture.md's sync design needs, built now so that
|
||||
there is nothing to retrofit later. Multiplayer is out of this step's scope and
|
||||
the addressing is not, because the addressing is the part that cannot be added
|
||||
afterwards: it decides what a write is, and therefore what two people can do at
|
||||
once.
|
||||
|
||||
clip/<cid>/name a label
|
||||
clip/<cid>/timing fps
|
||||
clip/<cid>/stage width, height
|
||||
clip/<cid>/source the analysis record this came out of
|
||||
clip/<cid>/subject/<sid> a tracked subject and its params
|
||||
clip/<cid>/feature/<fid> one feature: area, nodes, params
|
||||
clip/<cid>/group/<gid> an eye pair and its shared params
|
||||
clip/<cid>/timeline/<tid> frames, and a palette one day
|
||||
clip/<cid>/timeline/<tid>/node/<nid> kind, parent, stencil, z, time
|
||||
clip/<cid>/timeline/<tid>/channel/<nid>/<prop>
|
||||
clip/<cid>/timeline/<tid>/measured/<nid> the channels a re-freeze owns
|
||||
|
||||
WHY NODES SIT UNDER A TIMELINE. A clip holds a library of timelines. Its root
|
||||
and each symbol have their own nodes, so the timeline id is a path segment.
|
||||
The root is `main`, and a symbol's nodes use the same path shape.
|
||||
|
||||
`:frames` MOVED OFF `timing` onto the timeline. A timeline is a frame space and a
|
||||
clip is a rate, so `timing` holds `:fps` alone. Both used to be in one leaf, which
|
||||
is how a nested timeline's length would have had nowhere to go.
|
||||
|
||||
WHY THESE BOUNDARIES. Last-writer-wins only clobbers when its unit is too big,
|
||||
so the cut is chosen so that the things people do simultaneously land on
|
||||
different leaves. Every node has its own leaf, because two people adding nodes
|
||||
would otherwise collide always. Every channel has its own, because keying the
|
||||
mouth and keying a brow are the same size of edit as each other and nothing
|
||||
like the same edit. With fractional `:z` there is no separate draw-order leaf to
|
||||
contend on, which is the second thing fractional indices buy.
|
||||
|
||||
WHY PARAMS ARE NOT SPLIT BY AREA. docs/architecture.md's list has
|
||||
`clip/:cid/params/:area`, from a draft where params were one blob per clip and
|
||||
two people tuning teeth and eyes collided on every slider move. Step 8 moved
|
||||
settings onto the subject, the feature and the group, and a FEATURE HAS EXACTLY
|
||||
ONE AREA — so the feature leaf already is the area-scoped leaf, and splitting it
|
||||
again would only separate a feature's params from the feature's identity.
|
||||
|
||||
WHY `measured` IS ONE LEAF AND CHANNELS ARE NOT. `:head`'s measured channels are
|
||||
not authored: they are written together by a freeze and replaced together by a
|
||||
re-freeze, and `head-mode` exposes them through `:channels`. The optional
|
||||
`:anchors` map on the head node chooses which measured frame those channels
|
||||
read. A leaf per measured
|
||||
channel would offer a write nobody can make. The authored channels beside them
|
||||
are one leaf each, because a hand writes one at a time.
|
||||
|
||||
A LEAF PATH IS \"/\"-DELIMITED and an id is one segment of it, so a namespaced id
|
||||
— docs/architecture.md draws one as `:eye-r/iris` — is written `eye-r~iris`.
|
||||
`~` is then refused inside a name, which is the whole of the escaping and is why
|
||||
it is one character rather than a scheme."
|
||||
(:require [arthur.domain.clip :as clip]
|
||||
[arthur.domain.sha256 :as sha]
|
||||
[arthur.domain.timeline :as timeline]
|
||||
[clojure.string :as str]))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; ids and paths
|
||||
|
||||
(defn segment
|
||||
"An id -> one path segment."
|
||||
[id]
|
||||
(let [s (if (keyword? id) (subs (str id) 1) (str id))]
|
||||
(when (str/includes? s "~")
|
||||
(throw (ex-info "an id cannot contain ~: it is the namespace separator inside a leaf path"
|
||||
{:id id})))
|
||||
(str/replace s "/" "~")))
|
||||
|
||||
(def ^:private uuid-segment
|
||||
"Canonical UUID form: 8-4-4-4-12 hex digits, and nothing else.
|
||||
|
||||
A PLACEMENT'S ID IS A UUID — see `demo/stage/compose` for why — and a leaf path
|
||||
is text, so reading one back has to decide which ids are uuids and which are
|
||||
keywords. It decides by SHAPE, which is a judgement worth stating: a keyword
|
||||
that happened to be thirty-six characters of hex in exactly this grouping would
|
||||
come back a uuid. Nothing names a node that by hand, and the alternative — a
|
||||
sigil on every segment — would change the shape of every path in every leaf to
|
||||
disambiguate a case that does not arise. `^` and `$` are the load-bearing part;
|
||||
without them a longer id CONTAINING a uuid would match."
|
||||
#"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
|
||||
|
||||
(defn unsegment
|
||||
"One path segment -> the id it names: a uuid when it is shaped like one, a
|
||||
keyword otherwise."
|
||||
[s]
|
||||
(let [s (str/replace s "~" "/")]
|
||||
(if (re-find uuid-segment s)
|
||||
(uuid s)
|
||||
(keyword s))))
|
||||
|
||||
(defn- prop->path
|
||||
"A channel's property vector -> one path segment. `[:geom :pts]` is \"geom.pts\"
|
||||
and `[:vis]` is \"vis\"."
|
||||
[prop]
|
||||
(let [parts (map #(subs (str %) 1) prop)]
|
||||
(doseq [p parts]
|
||||
(when (or (str/includes? p ".") (str/includes? p "/"))
|
||||
(throw (ex-info "a channel property cannot contain . or /: both are path punctuation"
|
||||
{:prop prop}))))
|
||||
(str/join "." parts)))
|
||||
|
||||
(defn- path->prop [s]
|
||||
(mapv keyword (str/split s #"\.")))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the split
|
||||
|
||||
(def ^:private node-channel-keys #{:channels :measured})
|
||||
|
||||
(defn leaves
|
||||
"One clip -> path -> value.
|
||||
|
||||
A leaf whose value would be empty is OMITTED rather than written as `{}`, and
|
||||
that is what makes the round trip exact: the demo clip has no `:fps` and its root
|
||||
node has no `:channels`, and a codec that invented them would hand back a clip
|
||||
that is not `=` to the one it was given."
|
||||
[cid clip]
|
||||
(let [unknown (remove clip/clip-keys (keys clip))]
|
||||
(when (seq unknown)
|
||||
(throw (ex-info "the clip has a field with no leaf to save it in; see arthur.domain.clip/clip-keys"
|
||||
{:unknown (vec (sort-by str unknown))}))))
|
||||
(doseq [[id tl] (:timelines clip)]
|
||||
(let [unknown (remove timeline/timeline-keys (keys tl))]
|
||||
(when (seq unknown)
|
||||
(throw (ex-info "a timeline has a field with no leaf to save it in; see arthur.domain.timeline/timeline-keys"
|
||||
{:timeline id :unknown (vec (sort-by str unknown))})))))
|
||||
(let [at (fn [& parts] (str/join "/" (into ["clip" (segment cid)] parts)))
|
||||
some-leaf (fn [path v] (when (seq v) {path v}))]
|
||||
(apply merge
|
||||
(some-leaf (at "name") (select-keys clip [:name]))
|
||||
(some-leaf (at "timing") (select-keys clip [:fps]))
|
||||
(some-leaf (at "stage") (select-keys clip [:width :height]))
|
||||
(some-leaf (at "source") (:analysis clip))
|
||||
(concat
|
||||
(for [[id v] (:subjects clip)] {(at "subject" (segment id)) v})
|
||||
(for [[id v] (:features clip)] {(at "feature" (segment id)) v})
|
||||
(for [[id v] (:groups clip)] {(at "group" (segment id)) v})
|
||||
;; The timeline's own facts. `:id` is the path segment, so writing it
|
||||
;; into the value as well would be the one field a rename could
|
||||
;; disagree with itself about; `clip` puts it back.
|
||||
(for [[tid tl] (:timelines clip)]
|
||||
{(at "timeline" (segment tid))
|
||||
(select-keys tl [:frames :palette])})
|
||||
(for [[tid tl] (:timelines clip)
|
||||
[id n] (:nodes tl)]
|
||||
{(at "timeline" (segment tid) "node" (segment id))
|
||||
(apply dissoc n node-channel-keys)})
|
||||
(for [[tid tl] (:timelines clip)
|
||||
[id n] (:nodes tl)
|
||||
:when (seq (:measured n))]
|
||||
{(at "timeline" (segment tid) "measured" (segment id)) (:measured n)})
|
||||
(for [[tid tl] (:timelines clip)
|
||||
[id n] (:nodes tl)
|
||||
[prop ch] (:channels n)]
|
||||
{(at "timeline" (segment tid) "channel" (segment id) (prop->path prop)) ch})))))
|
||||
|
||||
(defn clip
|
||||
"The inverse of `leaves`, for one clip. Paths belonging to another clip are
|
||||
ignored, so a project's whole leaf map can be handed straight in.
|
||||
|
||||
A timeline's `:id` is restored from its path segment rather than read out of the
|
||||
value, which is why `leaves` does not write it: a segment and a field that both
|
||||
claim to be the id are two places for one fact."
|
||||
[cid leaves]
|
||||
(let [want (segment cid)]
|
||||
(reduce
|
||||
(fn [acc [path v]]
|
||||
(let [[_ found kind a b c] (str/split path #"/")]
|
||||
(if-not (= want found)
|
||||
acc
|
||||
(if (= "timeline" kind)
|
||||
(let [tid (unsegment a)
|
||||
acc (assoc-in acc [:timelines tid :id] tid)]
|
||||
(case b
|
||||
nil (update-in acc [:timelines tid] merge v)
|
||||
"node" (update-in acc [:timelines tid :nodes (unsegment c)] merge v)
|
||||
"measured" (assoc-in acc [:timelines tid :nodes (unsegment c) :measured] v)
|
||||
"channel" (assoc-in acc [:timelines tid :nodes (unsegment c)
|
||||
:channels (path->prop (nth (str/split path #"/") 6))]
|
||||
v)
|
||||
(throw (ex-info "not a leaf path" {:path path}))))
|
||||
(case kind
|
||||
"name" (merge acc v)
|
||||
"timing" (merge acc v)
|
||||
"stage" (merge acc v)
|
||||
"source" (assoc acc :analysis v)
|
||||
"subject" (assoc-in acc [:subjects (unsegment a)] v)
|
||||
"feature" (assoc-in acc [:features (unsegment a)] v)
|
||||
"group" (assoc-in acc [:groups (unsegment a)] v)
|
||||
(throw (ex-info "not a leaf path" {:path path})))))))
|
||||
{}
|
||||
;; Sorted, so `node` lands before `channel` and `measured` under one id and
|
||||
;; the node map is merged INTO rather than over. `update-in ... merge` makes
|
||||
;; the order not matter; sorting makes it not matter for a reason.
|
||||
(sort-by key leaves))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(defn problems
|
||||
"Human-readable reasons this leaf map is not a document. Empty means it is one.
|
||||
|
||||
The dense check is the tier discipline, stated where a save can enforce it: a
|
||||
document that named `\"take/geom\"` would be a document that only means anything
|
||||
on the machine that produced it, and the whole point of tier 2 being
|
||||
content-addressed is that it does not have to travel with tier 1 to be found."
|
||||
[leaves]
|
||||
(let [parts (into {} (map (juxt identity #(vec (str/split % #"/")))) (keys leaves))
|
||||
;; A node leaf, by (clip, timeline, node). Under a timeline id, because a
|
||||
;; symbol and the root may both hold a `:mouth` and a channel of one is not
|
||||
;; a channel of the other.
|
||||
nodes (into #{} (keep (fn [[_ p]]
|
||||
(when (and (= 6 (count p)) (= "timeline" (nth p 2))
|
||||
(= "node" (nth p 4)))
|
||||
[(nth p 1) (nth p 3) (nth p 5)])))
|
||||
parts)
|
||||
;; Which segment index holds the kind, and what shapes are legal.
|
||||
legal? (fn [p]
|
||||
(and (= "clip" (first p)) (second p)
|
||||
(if (= "timeline" (nth p 2 nil))
|
||||
(case (count p)
|
||||
4 true ; the timeline itself
|
||||
6 (#{"node" "measured"} (nth p 4))
|
||||
7 (= "channel" (nth p 4))
|
||||
false)
|
||||
(case (count p)
|
||||
;; The clip's own facts carry no id.
|
||||
3 (#{"name" "timing" "stage" "source"} (nth p 2))
|
||||
4 (#{"subject" "feature" "group"} (nth p 2))
|
||||
false))))]
|
||||
(vec
|
||||
(concat
|
||||
(for [[path p] (sort-by key parts)
|
||||
:when (not (legal? p))]
|
||||
(str (pr-str path) " is not a leaf path"))
|
||||
(for [[path p] (sort-by key parts)
|
||||
:when (and (legal? p) (= "timeline" (nth p 2 nil)) (>= (count p) 6)
|
||||
(#{"channel" "measured"} (nth p 4))
|
||||
(not (contains? nodes [(nth p 1) (nth p 3) (nth p 5)])))]
|
||||
(str (pr-str path) " addresses a node with no node leaf"))
|
||||
(for [[path p] (sort-by key parts)
|
||||
:let [v (get leaves path)]
|
||||
:when (and (legal? p) (= "timeline" (nth p 2 nil)) (= 7 (count p))
|
||||
(:dense v) (not (sha/key? (:store (:dense v)))))]
|
||||
(str (pr-str path) " names tier 2 as " (pr-str (:store (:dense v)))
|
||||
" — a dense channel in a saved document names a content address"))))))
|
||||
285
frontend/src/arthur/domain/node.cljs
Normal file
285
frontend/src/arthur/domain/node.cljs
Normal file
|
|
@ -0,0 +1,285 @@
|
|||
(ns arthur.domain.node
|
||||
"A node is an instance in the scene: what kind of mark it is, who it hangs off,
|
||||
what clips it, where it sits in draw order, and a bag of channels.
|
||||
|
||||
The tree is stored FLAT, WITH PARENT POINTERS, never as nested maps. Four
|
||||
reasons that all point the same way: any node is addressable without a walk;
|
||||
reparenting is a one-field write rather than a subtree move; an edit to a leaf
|
||||
does not change the identity of its ancestors, so re-frame's structural sharing
|
||||
keeps ancestor subs from invalidating; and it is what lets every node be its own
|
||||
sync leaf. Flash, Blender and After Effects all store it this way.
|
||||
|
||||
Transforms are DECOMPOSED for storage and FLAT AND MUTABLE for evaluation, and
|
||||
the two forms are allowed to differ. Decomposed because each component has to
|
||||
be independently keyframable — that is what channels are for — and because
|
||||
interpolating matrix entries is meaningless: a rotation tweened through its
|
||||
matrix shears on the way. Flat Float64Array for evaluation because at 30fps
|
||||
per-frame allocation is the only thing that will make this stutter."
|
||||
(:require [arthur.domain.channel :as ch]))
|
||||
|
||||
(def kinds
|
||||
"`:bitmap` is in the vocabulary and not implemented; it is
|
||||
here so that a scene that names one fails as \"not implemented\" rather than as
|
||||
\"not a kind\"."
|
||||
#{:poly :disc :rect :group :bitmap :symbol :audio})
|
||||
|
||||
(def implemented-kinds #{:poly :disc :rect :group :symbol :audio})
|
||||
|
||||
(def xform-paths
|
||||
"In composition order, which is also the order they have to be sampled in.
|
||||
|
||||
:skew and :anchor are in here although nothing drives either yet. A
|
||||
decomposition is not extensible after the fact: adding a component later means
|
||||
migrating every stored transform, so both are in the shape and in the
|
||||
composition order from the start."
|
||||
[[:xform :pos] [:xform :rot] [:xform :scale] [:xform :skew] [:xform :anchor]])
|
||||
|
||||
(def valid-paths
|
||||
"The set of valid channel paths follows from the node's :kind, and that is a
|
||||
SPEC rather than a schema migration — a node does not grow or lose fields, it
|
||||
simply has no `[:geom :radius]` unless it is a disc.
|
||||
|
||||
Written out per kind rather than derived from a table shared with the
|
||||
renderer. What a kind may CARRY and what the renderer READS off it coincide
|
||||
today and are not the same question, and tying them together would make a
|
||||
change to this spec silently change what gets drawn."
|
||||
(let [base (into #{[:vis]} xform-paths)]
|
||||
{:group base
|
||||
:symbol base
|
||||
:audio (into base [[:audio :gain] [:audio :pan] [:audio :rate]])
|
||||
:poly (into base [[:geom :pts] [:style :color]])
|
||||
;; A disc's radius is framed in practice — iris size is a knob, not a
|
||||
;; performance — but it is a channel like any other so it can be keyed.
|
||||
:disc (into base [[:geom :radius] [:style :color]])
|
||||
;; :size, not a radius: the pupil is a SQUARE, an exactly size x size
|
||||
;; block. See raster/fill-rect!.
|
||||
:rect (into base [[:geom :size] [:style :color]])}))
|
||||
|
||||
(def defaults
|
||||
"The identity transform, as channels. A node's channel map is merged over this,
|
||||
so a hand-written scene says only what it means to say."
|
||||
{[:xform :pos] (ch/framed [0.0 0.0])
|
||||
[:xform :rot] (ch/framed 0.0)
|
||||
[:xform :scale] (ch/framed [1.0 1.0])
|
||||
[:xform :skew] (ch/framed [0.0 0.0])
|
||||
[:xform :anchor] (ch/framed [0.0 0.0])
|
||||
[:vis] (ch/framed true)})
|
||||
|
||||
(defn channels
|
||||
"The node's channels with the transform defaults filled in."
|
||||
[n]
|
||||
(merge defaults (:channels n)))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; time maps
|
||||
;;
|
||||
;; Exposure, mouth lead and a symbol instance's timing are ONE mechanism, and
|
||||
;; seeing that is what keeps them from being three implementations that disagree
|
||||
;; at the edges.
|
||||
|
||||
(defn expose
|
||||
"Hold a frame back onto an exposure grid: 1 = on 1s, 2 = on 2s. Frame 5 at
|
||||
exposure 2 reads the pose from frame 4.
|
||||
|
||||
FLOOR, NEVER ROUND. Rounding would let an output frame read a pose from the
|
||||
FUTURE, which is a lead — a separate control, applied after this one, for a
|
||||
separate reason."
|
||||
[f n]
|
||||
(if (and n (> n 1)) (* (js/Math.floor (/ f n)) n) f))
|
||||
|
||||
(defn sample-frame
|
||||
"Pick a source frame for a lower picture rate without changing clip time.
|
||||
|
||||
The input is already an integer source frame from the audio clock. Its time is
|
||||
f/source-fps. Quantise that time to the picture grid, then read the latest
|
||||
source frame at or before it. The result is always an integer and never from
|
||||
the future, including when the rates do not divide (30 source → 24 picture)."
|
||||
[f source-fps picture-fps]
|
||||
(if (and source-fps picture-fps
|
||||
(pos? source-fps) (pos? picture-fps)
|
||||
(< picture-fps source-fps))
|
||||
(min f (js/Math.floor
|
||||
(* (js/Math.floor (/ (* f picture-fps) source-fps))
|
||||
(/ source-fps picture-fps))))
|
||||
f))
|
||||
|
||||
(defn local-frame
|
||||
"Apply a node's time map to the frame it was handed by its parent.
|
||||
|
||||
ORDER IS LOAD-BEARING: expose first, then offset. Flooring onto a grid and
|
||||
shifting against the clock do not commute — shift first and the floor discards
|
||||
it on most frames, so the lead slider reads as doing nothing at exposures above
|
||||
1, which is indistinguishable from the slider being unwired.
|
||||
|
||||
Composed along the parent chain, outermost first, by timeline/eval-frame. Two
|
||||
rules fall out and they are different rules: exposure INHERITS STRICTLY,
|
||||
because a head cutting on odd frames against a mouth cutting on even ones reads
|
||||
as two performances; offset is PER-NODE by design, because mouth lead applies
|
||||
to performance nodes and not to the plate, which is the entire point of it."
|
||||
[n f]
|
||||
(let [{:keys [mode offset rate at in source-fps sample-fps]
|
||||
ex :expose :or {mode :inherit}} (:time n)]
|
||||
(if (= mode :inherit)
|
||||
f
|
||||
(do
|
||||
(when (and (not (#{:symbol :audio} (:kind n))) rate (not= rate 1.0) (not= rate 1))
|
||||
(throw (ex-info "time map :rate belongs to a symbol or audio instance"
|
||||
{:node (:id n) :time (:time n)})))
|
||||
(when (and sample-fps (not (and source-fps (pos? source-fps))))
|
||||
(throw (ex-info "picture sampling needs a positive source fps"
|
||||
{:node (:id n) :time (:time n)})))
|
||||
(cond-> (if (#{:symbol :audio} (:kind n))
|
||||
(+ (or in 0) (* (or rate 1) (- f (or at 0))))
|
||||
f)
|
||||
sample-fps (sample-frame source-fps sample-fps)
|
||||
ex (expose ex)
|
||||
offset (+ offset))))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the transform
|
||||
;;
|
||||
;; A 2x3 affine as a 6-element Float64Array [a b c d e f], the canvas convention:
|
||||
;;
|
||||
;; | a c e | x' = a·x + c·y + e
|
||||
;; | b d f | y' = b·x + d·y + f
|
||||
;; | 0 0 1 |
|
||||
|
||||
(defn mat [] (js/Float64Array. #js [1 0 0 1 0 0]))
|
||||
|
||||
(defn set-identity! [^js m]
|
||||
(aset m 0 1) (aset m 1 0) (aset m 2 0) (aset m 3 1) (aset m 4 0) (aset m 5 0)
|
||||
m)
|
||||
|
||||
(defn mul!
|
||||
"dest := m · n. Reads both fully before writing, so dest may alias either."
|
||||
[^js dest ^js m ^js n]
|
||||
(let [a (+ (* (aget m 0) (aget n 0)) (* (aget m 2) (aget n 1)))
|
||||
b (+ (* (aget m 1) (aget n 0)) (* (aget m 3) (aget n 1)))
|
||||
c (+ (* (aget m 0) (aget n 2)) (* (aget m 2) (aget n 3)))
|
||||
d (+ (* (aget m 1) (aget n 2)) (* (aget m 3) (aget n 3)))
|
||||
e (+ (* (aget m 0) (aget n 4)) (* (aget m 2) (aget n 5)) (aget m 4))
|
||||
f (+ (* (aget m 1) (aget n 4)) (* (aget m 3) (aget n 5)) (aget m 5))]
|
||||
(aset dest 0 a) (aset dest 1 b) (aset dest 2 c)
|
||||
(aset dest 3 d) (aset dest 4 e) (aset dest 5 f)
|
||||
dest))
|
||||
|
||||
(defn local!
|
||||
"dest := T(pos) · T(anchor) · R(rot) · K(skew) · S(scale) · T(-anchor)
|
||||
|
||||
Written out closed-form rather than as five matrix products, because this runs
|
||||
per node per frame and the five products would each allocate. The derivation,
|
||||
so the constants are checkable rather than trusted:
|
||||
|
||||
R·K·S = | c -s | · | 1 kx | · | sx 0 |
|
||||
| s c | | ky 1 | | 0 sy |
|
||||
|
||||
K·S = | sx kx·sy |
|
||||
| ky·sx sy |
|
||||
|
||||
R·K·S = | sx(c - s·ky) sy(c·kx - s) |
|
||||
| sx(s + c·ky) sy(s·kx + c) |
|
||||
|
||||
and the translation is anchor + pos - M·anchor, which is what makes rotation
|
||||
and scale happen ABOUT the anchor. :anchor is Flash's registration point and
|
||||
Blender's origin, and getting it wrong is why hand-placed parts swing rather
|
||||
than turn.
|
||||
|
||||
:skew is stored as shear FACTORS, not angles — kx is x gained per unit y — so
|
||||
that the identity is 0 and a decomposition round-trips without a tangent."
|
||||
[^js dest pos rot scale skew anchor]
|
||||
(let [c (js/Math.cos rot)
|
||||
s (js/Math.sin rot)
|
||||
sx (ch/component scale 0)
|
||||
sy (ch/component scale 1)
|
||||
kx (ch/component skew 0)
|
||||
ky (ch/component skew 1)
|
||||
ax (ch/component anchor 0)
|
||||
ay (ch/component anchor 1)
|
||||
a (* sx (- c (* s ky)))
|
||||
b (* sx (+ s (* c ky)))
|
||||
cc (* sy (- (* c kx) s))
|
||||
d (* sy (+ (* s kx) c))]
|
||||
(aset dest 0 a)
|
||||
(aset dest 1 b)
|
||||
(aset dest 2 cc)
|
||||
(aset dest 3 d)
|
||||
(aset dest 4 (+ ax (ch/component pos 0) (- (+ (* a ax) (* cc ay)))))
|
||||
(aset dest 5 (+ ay (ch/component pos 1) (- (+ (* b ax) (* d ay)))))
|
||||
dest))
|
||||
|
||||
(defn pinv
|
||||
"The parent-inverse, captured at the moment of parenting so the child does not
|
||||
jump when it acquires a parent. Blender's `parent_inverse`. Small, and its
|
||||
absence is the kind of thing that makes a parenting feature feel broken."
|
||||
[n]
|
||||
(if-let [p (:pinv n)]
|
||||
(js/Float64Array.from (clj->js p))
|
||||
nil))
|
||||
|
||||
(defn world!
|
||||
"dest := parent · pinv · local. `parent` is nil at the root, `pinv-m` nil until
|
||||
something is reparented. `scratch` is a 6-element Float64Array the caller owns;
|
||||
it is an argument rather than an allocation because this runs per node per
|
||||
frame."
|
||||
[^js dest ^js parent ^js pinv-m ^js local ^js scratch]
|
||||
(cond
|
||||
(and parent pinv-m) (mul! dest parent (mul! scratch pinv-m local))
|
||||
parent (mul! dest parent local)
|
||||
pinv-m (mul! dest pinv-m local)
|
||||
:else (doto dest (.set local))))
|
||||
|
||||
(defn apply-pt!
|
||||
"out[2i], out[2i+1] := m · (x, y)."
|
||||
[^js out i ^js m x y]
|
||||
(aset out (* 2 i) (+ (* (aget m 0) x) (* (aget m 2) y) (aget m 4)))
|
||||
(aset out (inc (* 2 i)) (+ (* (aget m 1) x) (* (aget m 3) y) (aget m 5)))
|
||||
out)
|
||||
|
||||
(defn mean-scale
|
||||
"The geometric-mean scale of a transform, sqrt|det|.
|
||||
|
||||
A disc under a non-uniform transform is an ellipse and this rasteriser has no
|
||||
ellipse — the iris is a disc because at 320x200 it is a few pixels across, and
|
||||
a few-pixel ellipse is not a shape, it is a stair. So a disc's radius takes the
|
||||
mean scale. For a similarity, which is the only transform the anchor fit
|
||||
produces, this is exact."
|
||||
[^js m]
|
||||
(js/Math.sqrt (js/Math.abs (- (* (aget m 0) (aget m 3))
|
||||
(* (aget m 1) (aget m 2))))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(defn problems
|
||||
"Human-readable reasons this map is not a usable node. Empty means it is one.
|
||||
|
||||
Worth having at all because app-db holds only authored data, which is what
|
||||
makes validating every event affordable; this is the per-node half of that."
|
||||
[n]
|
||||
(let [k (:kind n)
|
||||
valid (get valid-paths k)]
|
||||
(-> []
|
||||
(cond->
|
||||
(nil? (:id n)) (conj "no :id")
|
||||
(not (contains? kinds k)) (conj (str ":kind " (pr-str k) " is not one of " (pr-str kinds)))
|
||||
(and (contains? kinds k)
|
||||
(not (contains? implemented-kinds k)))
|
||||
(conj (str ":kind " k " is in the vocabulary but not implemented"))
|
||||
|
||||
(and (= k :symbol) (nil? (:of n))) (conj "a symbol instance needs :of")
|
||||
(and (= k :audio) (nil? (get-in n [:source :footage])))
|
||||
(conj "an audio instance needs :source :footage")
|
||||
(and (#{:symbol :audio} k) (some? (get-in n [:time :rate]))
|
||||
(not (pos? (get-in n [:time :rate]))))
|
||||
(conj "an instance's :rate must be positive")
|
||||
(nil? (:z n)) (conj "no :z — draw order is authored per scene, not implied by the tree")
|
||||
(and (:span n) (not= 2 (count (:span n))))
|
||||
(conj ":span must be [in out]"))
|
||||
|
||||
(into (when valid
|
||||
(for [[path _] (:channels n)
|
||||
:when (not (contains? valid path))]
|
||||
(str "channel " (pr-str path) " is not valid on a " k " node"))))
|
||||
|
||||
(into (for [[path c] (:channels n)
|
||||
p (ch/problems c)]
|
||||
(str "channel " (pr-str path) ": " p))))))
|
||||
57
frontend/src/arthur/domain/paint.cljs
Normal file
57
frontend/src/arthur/domain/paint.cljs
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
(ns arthur.domain.paint
|
||||
"Small authored polygon operations. Paint nodes read timeline frames directly;
|
||||
the roto root's exposure and picture sampling must not quantise a hand edit."
|
||||
(:require [arthur.domain.channel :as channel]))
|
||||
|
||||
(def geometry [:geom :pts])
|
||||
|
||||
(defn shapes [clip]
|
||||
(->> (get-in clip [:timelines :main :nodes])
|
||||
(filter (fn [[_ node]] (:paint? node)))
|
||||
(sort-by (comp :z val))
|
||||
vec))
|
||||
|
||||
(defn active-frame [ch frame]
|
||||
(let [frames (sort (keys (:keys ch)))]
|
||||
(or (last (take-while #(<= % frame) frames)) (first frames))))
|
||||
|
||||
(defn new-shape [clip id frame points color]
|
||||
(let [end (get-in clip [:timelines :main :frames])
|
||||
z (str "z" (js/Date.now) "-" (name id))]
|
||||
(if (and (<= 0 frame) (< frame end) (>= (count points) 6)
|
||||
(even? (count points)))
|
||||
(assoc-in clip [:timelines :main :nodes id]
|
||||
{:id id :name (str "shape " (inc (count (shapes clip))))
|
||||
:kind :poly :paint? true :parent nil :z z
|
||||
:span [frame end]
|
||||
:channels {geometry (channel/keyed {frame points})
|
||||
[:style :color] (channel/framed color)}})
|
||||
clip)))
|
||||
|
||||
(defn add-key [clip id frame]
|
||||
(let [path [:timelines :main :nodes id]
|
||||
node (get-in clip path)
|
||||
ch (get-in node [:channels geometry])
|
||||
[start end] (:span node)]
|
||||
(if (and (:paint? node) (<= start frame) (< frame end) ch)
|
||||
(assoc-in clip (into path [:channels geometry :keys frame])
|
||||
(vec (channel/value-at ch frame)))
|
||||
clip)))
|
||||
|
||||
(defn set-vertex [clip id key-frame vertex [x y]]
|
||||
(let [path [:timelines :main :nodes id :channels geometry :keys key-frame]
|
||||
points (get-in clip path)
|
||||
i (* 2 vertex)]
|
||||
(if (and points (< (inc i) (count points)))
|
||||
(assoc-in clip path (-> points (assoc i x) (assoc (inc i) y)))
|
||||
clip)))
|
||||
|
||||
(defn set-segment-interp [clip id key-frame interp]
|
||||
(let [node (get-in clip [:timelines :main :nodes id])
|
||||
keys (get-in node [:channels geometry :keys])]
|
||||
(if (and (:paint? node) (contains? keys key-frame)
|
||||
(some #(< key-frame %) (clojure.core/keys keys))
|
||||
(#{:hold :linear} interp))
|
||||
(assoc-in clip [:timelines :main :nodes id :channels geometry
|
||||
:segments key-frame] interp)
|
||||
clip)))
|
||||
50
frontend/src/arthur/domain/palette.cljs
Normal file
50
frontend/src/arthur/domain/palette.cljs
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
(ns arthur.domain.palette
|
||||
"The indexed palette.
|
||||
|
||||
THE RULE, and it is a rule rather than a default: a part carries a palette
|
||||
INDEX, never a sampled RGB value. Sampling colour off the footage produces a
|
||||
pixel-art filter, and it does so irrecoverably — once a shape holds a measured
|
||||
colour there is no way back to an authored one, because the information that it
|
||||
was ever a choice is gone. Every `[:style :color]` channel holds one of the
|
||||
keywords below.
|
||||
|
||||
Entries are ordered, and the order IS the index the raster writes. Inserting in
|
||||
the middle renumbers every stored index, so new tones append.")
|
||||
|
||||
(def entries
|
||||
[{:name :bg :hex "#12141c"}
|
||||
{:name :skin-base :hex "#b07a5a"}
|
||||
{:name :skin-dark :hex "#7a4f3a"}
|
||||
{:name :mouth-dark :hex "#24161a"}
|
||||
{:name :teeth :hex "#d9cfc2"}
|
||||
;; Sclera is not white, and that is authored, not measured. A true white at
|
||||
;; 320x200 next to a warm skin ramp reads as a hole punched in the face; the
|
||||
;; eye sits in a socket, in shadow, so it is a dimmer and cooler tone than the
|
||||
;; teeth, which catch the light.
|
||||
{:name :eye-white :hex "#c9c3b4"}
|
||||
;; Three tones for the eye - sclera, iris, pupil - which is the "two or three
|
||||
;; tones per part" budget, spent where it buys the most: an eye with no tonal
|
||||
;; step inside it reads as a hole.
|
||||
{:name :iris :hex "#4a5468"}
|
||||
{:name :pupil :hex "#171a22"}
|
||||
;; Brows get their own entry rather than sharing skin-dark with the lash line.
|
||||
;; They are hair, not shadow: when hair plates exist they want to match those,
|
||||
;; and tying them to the lash means you cannot change one without the other.
|
||||
{:name :brow :hex "#3a2a22"}])
|
||||
|
||||
(def hexes (mapv :hex entries))
|
||||
|
||||
(def index-of
|
||||
"Palette keyword -> the index the raster writes. Derived, so the vector above
|
||||
is the single place an ordering is declared."
|
||||
(into {} (map-indexed (fn [i e] [(:name e) i]) entries)))
|
||||
|
||||
(defn hex->rgb [hex]
|
||||
(let [s (.replace hex "#" "")]
|
||||
[(js/parseInt (.slice s 0 2) 16)
|
||||
(js/parseInt (.slice s 2 4) 16)
|
||||
(js/parseInt (.slice s 4 6) 16)]))
|
||||
|
||||
(def rgb
|
||||
"Index -> [r g b], precomputed."
|
||||
(mapv hex->rgb hexes))
|
||||
75
frontend/src/arthur/domain/params.cljs
Normal file
75
frontend/src/arthur/domain/params.cljs
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
(ns arthur.domain.params
|
||||
"Definitions for generated settings: defaults, scope and value constraints, once.
|
||||
|
||||
WHAT IS NOT HERE. A knob's INVALIDATION — which stored bytes stop being valid
|
||||
when it moves — is `arthur.flow.address/block-knobs`, and it is there rather
|
||||
than here for two reasons that are not about layering.
|
||||
|
||||
It is per BLOCK and this registry is per AREA, and the difference is not
|
||||
granularity, it is disagreement. `:aperture-cut` is a mouth setting that reaches
|
||||
the TEETH block's bytes and does not reach the mouth's, because it gates the
|
||||
contour smoothing and the mouth's own geometry is smoothed either way.
|
||||
`:blink-cut` is an eye setting that reaches no block at all, because a blink is
|
||||
`[:vis]` keys in tier 1. An `:affects #{:eye}` on that entry read as documentation
|
||||
and was wrong in both directions.
|
||||
|
||||
And `block-knobs` is TESTED — `address-test` re-freezes the take once per knob
|
||||
and asserts the biconditional — while a field here could only be believed. An
|
||||
earlier version of this namespace carried `:affects` and an `affected-areas`
|
||||
reading it, and the only thing that ever called it was a test asserting it
|
||||
returned what it was written as. Two tables where one is checked and one is not
|
||||
is worse than one table, because the unchecked one is the one a parameter UI
|
||||
would reach for first. `address/invalidates` is the derived inverse, and it
|
||||
cannot drift from the thing that is asserted.")
|
||||
|
||||
(def definitions
|
||||
{:anchor-avg {:area :subject :default 2 :type :integer :min 0}
|
||||
:contour-avg {:area :subject :default 1 :type :integer :min 0}
|
||||
:verts {:area :mouth :default 8 :type :integer :min 4 :even? true}
|
||||
:aperture-cut {:area :mouth :default 0.12 :type :number :min 0 :max 1}
|
||||
:eye-verts {:area :eye :default 8 :type :integer :min 4 :even? true}
|
||||
:blink-cut {:area :eye :default 0.13 :type :number :min 0}
|
||||
:gaze-gain {:area :eye :default 1 :type :number :min 0}
|
||||
:gaze-step {:area :eye :default 0.08 :type :number :min 0}
|
||||
:iris-size {:area :eye :default 0.42 :type :number :min 0}
|
||||
:lash-weight {:area :eye :default 0.06 :type :number :min 0}
|
||||
:pupil-size {:area :eye :default 0.15 :type :number :min 0}
|
||||
:brow-verts {:area :brow :default 6 :type :integer :min 4 :even? true}
|
||||
:brow-gain {:area :brow :default 1 :type :number :min 0}
|
||||
:brow-step {:area :brow :default 0.08 :type :number :min 0}
|
||||
:brow-weight {:area :brow :default 0.05 :type :number :min 0}
|
||||
:cavity-erode {:area :teeth :default 0.18 :type :number :min 0}
|
||||
:tongue-reject {:area :teeth :default 0.18 :type :number :min 0}
|
||||
:blob-grow {:area :teeth :default 0 :type :integer}
|
||||
:top-bias {:area :teeth :default 0.6 :type :number}
|
||||
:teeth-verts {:area :teeth :default 10 :type :integer :min 4}
|
||||
:min-area {:area :teeth :default 12 :type :integer :min 0}
|
||||
:teeth-on {:area :teeth :default 0.12 :type :number :min 0}
|
||||
:teeth-smooth {:area :teeth :default 1 :type :integer :min 0}})
|
||||
|
||||
(def defaults
|
||||
(into {} (map (fn [[id spec]] [id (:default spec)])) definitions))
|
||||
|
||||
(def areas #{:subject :mouth :eye :brow :teeth})
|
||||
|
||||
(defn for-area [wanted-area]
|
||||
(into {} (keep (fn [[id {:keys [area default]}]]
|
||||
(when (= wanted-area area) [id default])))
|
||||
definitions))
|
||||
|
||||
(defn valid-value? [id value]
|
||||
(when-let [{:keys [type min max] must-even? :even?} (get definitions id)]
|
||||
(and (case type
|
||||
:integer (integer? value)
|
||||
:number (number? value)
|
||||
false)
|
||||
(or (nil? min) (<= min value))
|
||||
(or (nil? max) (<= value max))
|
||||
(or (not must-even?) (and (integer? value) (even? value))))))
|
||||
|
||||
(defn valid-settings? [area settings]
|
||||
(and (map? settings)
|
||||
(every? (fn [[id value]]
|
||||
(and (= area (get-in definitions [id :area]))
|
||||
(valid-value? id value)))
|
||||
settings)))
|
||||
149
frontend/src/arthur/domain/png.cljs
Normal file
149
frontend/src/arthur/domain/png.cljs
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
(ns arthur.domain.png
|
||||
"An indexed raster -> one PNG, at integer zoom.
|
||||
|
||||
THE EXPORT'S MASTER FORMAT, and the reasons are all about not resampling.
|
||||
Everything above `domain/raster` exists to put hard-edged flat fills into a
|
||||
byte buffer; a lossy encoder would put chroma fringes on exactly the edges the
|
||||
whole idiom is made of, and a fractional scale would put grey on them. So the
|
||||
picture leaves the tool as PNG, and it leaves it at an INTEGER zoom — a pixel
|
||||
becomes a block of identical pixels and nothing is interpolated.
|
||||
|
||||
TRUECOLOUR, NOT PALETTED, and that is a deliberate loss. Colour type 3 would be
|
||||
the faithful shape — the buffer IS palette indices and a PLTE chunk is the ramp
|
||||
— and it would be a third of the bytes into deflate. But the entire purpose of
|
||||
this file is to be imported by a program we cannot test against here, and
|
||||
type 2 is the type every reader on earth handles. Faithfulness that depends on
|
||||
someone else's PNG decoder being complete is not faithfulness. The pixels are
|
||||
identical either way; only the file is bigger, and deflate takes most of that
|
||||
back because the art is flat.
|
||||
|
||||
`CompressionStream` does the deflating, which is why `encoder` hands back a
|
||||
promise. It is the zlib-wrapped variety — RFC 1950, which is what an IDAT
|
||||
requires — and getting that wrong is a one-word difference from `deflate-raw`
|
||||
and a file no reader will open.
|
||||
|
||||
No DOM. A canvas `toBlob` would be shorter and would put this namespace out of
|
||||
reach of node, where the rest of the rasteriser is asserted about; it would also
|
||||
hand the encoding decisions to the browser, and the point of this file is that
|
||||
they are decisions."
|
||||
;; A `chunk` is the format's own word for its one structural unit, and chunked
|
||||
;; seqs never come up in here, so core's loses the name rather than ours.
|
||||
(:refer-clojure :exclude [chunk])
|
||||
(:require [arthur.domain.crc32 :as crc32]))
|
||||
|
||||
(def ^:private signature
|
||||
(js/Uint8Array. #js [0x89 0x50 0x4e 0x47 0x0d 0x0a 0x1a 0x0a]))
|
||||
|
||||
(defn- u32! [^js bytes at n]
|
||||
(aset bytes at (bit-and (unsigned-bit-shift-right n 24) 0xff))
|
||||
(aset bytes (+ at 1) (bit-and (unsigned-bit-shift-right n 16) 0xff))
|
||||
(aset bytes (+ at 2) (bit-and (unsigned-bit-shift-right n 8) 0xff))
|
||||
(aset bytes (+ at 3) (bit-and n 0xff)))
|
||||
|
||||
(defn chunk
|
||||
"One PNG chunk: length, type, payload, CRC over type and payload.
|
||||
|
||||
Big-endian throughout, which is the format's and not the machine's — the same
|
||||
reason `domain/raster/->rgba` has to ask which way round the machine is and this
|
||||
does not."
|
||||
[tag ^js payload]
|
||||
(let [n (.-length payload)
|
||||
out (js/Uint8Array. (+ n 12))]
|
||||
(u32! out 0 n)
|
||||
(dotimes [i 4] (aset out (+ 4 i) (.charCodeAt tag i)))
|
||||
(.set out payload 8)
|
||||
(u32! out (+ 8 n) (crc32/of out 4 (+ 8 n)))
|
||||
out))
|
||||
|
||||
(defn- ihdr [w h]
|
||||
(let [p (js/Uint8Array. 13)]
|
||||
(u32! p 0 w)
|
||||
(u32! p 4 h)
|
||||
(aset p 8 8) ; bit depth
|
||||
(aset p 9 2) ; colour type 2: truecolour RGB
|
||||
(aset p 10 0) ; deflate, the only compression PNG has
|
||||
(aset p 11 0) ; adaptive filtering, the only method
|
||||
(aset p 12 0) ; no interlace
|
||||
p))
|
||||
|
||||
(defn- deflate!
|
||||
"Promise of the zlib stream of `bytes`.
|
||||
|
||||
`CompressionStream` rather than a deflate implementation: it is in every browser
|
||||
this tool runs in and in node, so the one thing here that would be hundreds of
|
||||
lines is none of them."
|
||||
[^js bytes]
|
||||
;; `.stream` first: `pipeThrough` is a ReadableStream's method, not a Blob's.
|
||||
(-> (js/Response. (.pipeThrough (.stream (js/Blob. #js [bytes]))
|
||||
(js/CompressionStream. "deflate")))
|
||||
(.arrayBuffer)
|
||||
(.then #(js/Uint8Array. %))))
|
||||
|
||||
(defn- concat!
|
||||
[parts]
|
||||
(let [out (js/Uint8Array. (transduce (map #(.-length ^js %)) + 0 parts))]
|
||||
(reduce (fn [at ^js part] (.set out part at) (+ at (.-length part))) 0 parts)
|
||||
out))
|
||||
|
||||
(defn encoder
|
||||
"(fn [raster ramp] -> promise of PNG bytes), for one stage size and one zoom.
|
||||
|
||||
Built once per export rather than per frame, in the shape `timeline/resolver`
|
||||
already uses: everything that does not change frame to frame is held here. What
|
||||
that buys is the scanline scratch, which at zoom 6 is seven megabytes — a
|
||||
per-frame allocation of that size is the one thing that would make a long export
|
||||
thrash, and it is the same buffer every frame because the stage is.
|
||||
|
||||
FILTER TYPE 2 (Up) ON EVERY ROW, including the first, where PNG defines the
|
||||
prior row as zeros and Up therefore degenerates to None. It is chosen for the
|
||||
zoom: at zoom 4 three of every four output rows are byte-identical to the one
|
||||
above, so Up turns them into runs of zeros and deflate takes them to almost
|
||||
nothing. Paletted or not, that is where the size of an upscaled flat-fill frame
|
||||
goes."
|
||||
[w h zoom]
|
||||
(let [zoom (max 1 (js/Math.floor zoom))
|
||||
out-w (* w zoom)
|
||||
out-h (* h zoom)
|
||||
stride (* out-w 3)
|
||||
;; One filter byte per output row, then the filtered row.
|
||||
raw (js/Uint8Array. (* out-h (inc stride)))
|
||||
;; The row as it actually is, kept because Up filters against the
|
||||
;; UNFILTERED row above, not against the stored bytes.
|
||||
cur (js/Uint8Array. stride)
|
||||
prev (js/Uint8Array. stride)
|
||||
head (concat! [signature (chunk "IHDR" (ihdr out-w out-h))])
|
||||
tail (chunk "IEND" (js/Uint8Array. 0))]
|
||||
(fn [{:keys [buf] :as _raster} ramp]
|
||||
;; The ramp is read as a flat byte table for the same reason ->rgba reads
|
||||
;; one: `nth` into a vector of vectors is four protocol dispatches a pixel,
|
||||
;; and this walks every pixel of every frame.
|
||||
(let [p8 (js/Uint8Array. (* 256 3))]
|
||||
(dotimes [i 256]
|
||||
(let [c (or (nth ramp i nil) [255 0 255])]
|
||||
(aset p8 (* i 3) (nth c 0))
|
||||
(aset p8 (+ 1 (* i 3)) (nth c 1))
|
||||
(aset p8 (+ 2 (* i 3)) (nth c 2))))
|
||||
(.fill prev 0)
|
||||
(dotimes [y h]
|
||||
(let [srow (* y w)]
|
||||
;; Expand one SOURCE row through the ramp once, repeating each pixel
|
||||
;; `zoom` times across.
|
||||
(dotimes [x w]
|
||||
(let [p (* 3 (aget buf (+ srow x)))
|
||||
r (aget p8 p) g (aget p8 (+ p 1)) b (aget p8 (+ p 2))]
|
||||
(dotimes [k zoom]
|
||||
(let [o (* 3 (+ (* x zoom) k))]
|
||||
(aset cur o r)
|
||||
(aset cur (+ o 1) g)
|
||||
(aset cur (+ o 2) b)))))
|
||||
;; …and emit it `zoom` times down. The second and later copies filter
|
||||
;; to all zeros, which is the whole point of Up here.
|
||||
(dotimes [k zoom]
|
||||
(let [at (* (+ (* y zoom) k) (inc stride))]
|
||||
(aset raw at 2)
|
||||
(dotimes [i stride]
|
||||
(aset raw (+ at 1 i)
|
||||
(bit-and (- (aget cur i) (aget prev i)) 0xff)))
|
||||
(.set prev cur)))))
|
||||
(-> (deflate! raw)
|
||||
(.then (fn [z] (concat! [head (chunk "IDAT" z) tail]))))))))
|
||||
94
frontend/src/arthur/domain/pose.cljs
Normal file
94
frontend/src/arthur/domain/pose.cljs
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
(ns arthur.domain.pose
|
||||
"An instance's explicit, held choices of source pose for each shape group.
|
||||
|
||||
A track is {local-frame -> source-frame}. The key is when the cut happens;
|
||||
the value is the frozen pose to read. Skipped source frames remain available.")
|
||||
|
||||
(defn prepare
|
||||
"Sort exposure tracks once when building a resolver."
|
||||
[tracks]
|
||||
(into {}
|
||||
(map (fn [[group entries]]
|
||||
[group (vec (sort-by first entries))]))
|
||||
tracks))
|
||||
|
||||
(defn held-frame
|
||||
"Last value keyed at or before f, or default before the first key."
|
||||
[entries f default-frame]
|
||||
(loop [lo 0 hi (dec (count entries)) hit nil]
|
||||
(if (> lo hi)
|
||||
(if (some? hit) (second (nth entries hit)) default-frame)
|
||||
(let [mid (bit-shift-right (+ lo hi) 1)]
|
||||
(if (<= (first (nth entries mid)) f)
|
||||
(recur (inc mid) hi mid)
|
||||
(recur lo (dec mid) hit))))))
|
||||
|
||||
(defn source-frame
|
||||
"Read an explicit cut if one has happened; otherwise read the default pose.
|
||||
This keeps the existing motion before the first edited cut."
|
||||
[prepared group f default-frame]
|
||||
(if-let [entries (get prepared group)]
|
||||
(held-frame entries f default-frame)
|
||||
default-frame))
|
||||
|
||||
(defn put-cut
|
||||
"Set one held pose on a symbol instance. Earlier motion stays untouched."
|
||||
[clip instance group at source]
|
||||
(let [node (get-in clip [:timelines :main :nodes instance])
|
||||
symbol (get-in clip [:timelines (:of node)])
|
||||
length (:frames symbol)
|
||||
active (filter (fn [n] (some :pose-sampled? (vals (:channels n))))
|
||||
(vals (:nodes symbol)))
|
||||
groups (set (map #(or (:pose-group %) (:id %)) active))
|
||||
ids (set (map :id active))]
|
||||
(when-not (and (= :symbol (:kind node))
|
||||
(or (contains? groups group)
|
||||
(and (vector? group) (= 2 (count group))
|
||||
(= :node (first group))
|
||||
(contains? ids (second group))))
|
||||
(integer? at) (<= 0 at) (< at length)
|
||||
(integer? source) (<= 0 source) (< source length))
|
||||
(throw (ex-info "invalid stage pose cut"
|
||||
{:instance instance :group group :at at :source source})))
|
||||
(update-in clip [:timelines :main :nodes instance :playback :tracks group]
|
||||
#(assoc (or % {}) at source))))
|
||||
|
||||
(defn remove-cut
|
||||
"Remove a cut; an empty track again follows the normal generated motion."
|
||||
[clip instance group at]
|
||||
(let [path [:timelines :main :nodes instance :playback :tracks group]]
|
||||
(if-let [entries (get-in clip path)]
|
||||
(if-let [remaining (not-empty (dissoc entries at))]
|
||||
(assoc-in clip path remaining)
|
||||
(update-in clip [:timelines :main :nodes instance :playback :tracks]
|
||||
dissoc group))
|
||||
clip)))
|
||||
|
||||
(defn problems
|
||||
"Errors in one symbol instance's exposure tracks."
|
||||
[tracks source-frames groups]
|
||||
(cond
|
||||
(nil? tracks) []
|
||||
(not (map? tracks)) [":playback :tracks must be a map"]
|
||||
:else
|
||||
(vec
|
||||
(mapcat
|
||||
(fn [[group entries]]
|
||||
(cond
|
||||
(not (contains? groups group))
|
||||
[(str "pose track " (pr-str group) " names no generated shape")]
|
||||
|
||||
(not (map? entries))
|
||||
[(str "pose track " (pr-str group) " must map local frames to source frames")]
|
||||
|
||||
:else
|
||||
(concat
|
||||
(when-not (every? #(and (integer? %) (<= 0 %)
|
||||
(or (nil? source-frames) (< % source-frames)))
|
||||
(keys entries))
|
||||
[(str "pose track " (pr-str group) " has an invalid change frame")])
|
||||
(when-not (every? #(and (integer? %) (<= 0 %)
|
||||
(or (nil? source-frames) (< % source-frames)))
|
||||
(vals entries))
|
||||
[(str "pose track " (pr-str group) " names a pose outside the source")]))))
|
||||
tracks))))
|
||||
101
frontend/src/arthur/domain/project.cljs
Normal file
101
frontend/src/arthur/domain/project.cljs
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
(ns arthur.domain.project
|
||||
"A clip <-> the document that travels. The tier split, as a pair of functions.
|
||||
|
||||
`save` takes what `flow/freeze` produced — `{:clip ... :store ...}` — and
|
||||
returns two things that are allowed on the wire for different reasons:
|
||||
|
||||
:leaves TIER 1. The document. Timelines, nodes, channels, subjects,
|
||||
features, groups, time maps, the analysis record. Kilobytes, and
|
||||
every byte of it authored or authorable.
|
||||
|
||||
:blocks TIER 2. The dense blocks the document NAMES, each with the
|
||||
descriptor its key is the hash of. Megabytes, content-addressed,
|
||||
and not part of the document — a bake inside the shared document is
|
||||
a system that puts 48KB on the wire per vertex drag.
|
||||
|
||||
ONLY WHAT THE DOCUMENT NAMES TRAVELS. The blocks are selected by walking the
|
||||
leaves for dense store keys, not by taking the store wholesale, so a store that
|
||||
has accumulated a block nothing points at does not upload it. That is also the
|
||||
check that the split is honest: if a channel named a block the store did not
|
||||
have, `save` would say so here rather than producing a document that loads into
|
||||
a blank stage somewhere else.
|
||||
|
||||
THE BYTES ARE NOT IN THE DOCUMENT AND THE TYPE IS NOT IN THE BYTES. A block's
|
||||
element type comes out of its own descriptor, which is the only place it is
|
||||
written down: an Int16Array and a Float32Array over the same bytes are both
|
||||
valid readings of them, and only one is the block. That makes the descriptor
|
||||
load-bearing rather than documentation, which is the right way round for the
|
||||
thing a key is the hash of."
|
||||
(:require [arthur.domain.leaf :as leaf]
|
||||
[arthur.domain.wire :as wire]))
|
||||
|
||||
(defn block-keys
|
||||
"Every tier-2 key a leaf map names, in a stable order."
|
||||
[leaves]
|
||||
(->> (vals leaves)
|
||||
(keep (comp :store :dense))
|
||||
distinct
|
||||
sort
|
||||
vec))
|
||||
|
||||
(defn- block-type
|
||||
"A block's element type, out of its descriptor."
|
||||
[descriptor]
|
||||
(or (get-in (js->clj (js/JSON.parse descriptor)) ["layout" "type"])
|
||||
(throw (ex-info "a block's descriptor does not say what its elements are"
|
||||
{:descriptor descriptor}))))
|
||||
|
||||
(defn save
|
||||
"One clip -> the JS object a save PUTs, ready for `JSON.stringify`.
|
||||
|
||||
A JS object rather than CLJS data, and `load` takes one back, because this is
|
||||
the wire boundary and both ends of it should speak the wire: a test can then
|
||||
round-trip a clip through `JSON.parse(JSON.stringify(...))` and be running the
|
||||
same conversion the network runs, rather than a CLJS-shaped rehearsal of it. The
|
||||
one thing a keywordising `js->clj` would quietly break is the leaf paths —
|
||||
`:clip/c1/timeline/main/node/mouth` is a keyword whose `name` is
|
||||
\"c1/timeline/main/node/mouth\", so the
|
||||
\"clip/\" would be lost on the way back in.
|
||||
|
||||
Refuses a document `domain/leaf` calls unaddressable, which is where a hand-made
|
||||
clip with placeholder store keys — `demo/swarm`'s \"swarm/pos\" — stops rather
|
||||
than being uploaded as a project that means something only on the machine that
|
||||
made it."
|
||||
[cid {:keys [clip store]}]
|
||||
(let [leaves (leaf/leaves cid clip)
|
||||
ps (leaf/problems leaves)]
|
||||
(when (seq ps)
|
||||
(throw (ex-info (str "this clip cannot be saved: " (first ps))
|
||||
{:problems ps})))
|
||||
(let [out (js-obj)]
|
||||
(doseq [[path v] leaves]
|
||||
(aset out path (wire/encode-json v)))
|
||||
#js {:leaves out
|
||||
:blocks (into-array
|
||||
(map (fn [k]
|
||||
(let [{:keys [data state descriptor]}
|
||||
(or (get store k)
|
||||
(throw (ex-info "the document names a block the store does not have"
|
||||
{:key k})))]
|
||||
#js {:key k
|
||||
:descriptor descriptor
|
||||
:data (wire/base64 data)
|
||||
:state (when state (wire/base64 state))}))
|
||||
(block-keys leaves)))})))
|
||||
|
||||
(defn load
|
||||
"The parsed response -> `{:clip :store}`, which is what `flow/freeze` returns
|
||||
and therefore what the player already knows how to play."
|
||||
[cid ^js doc]
|
||||
(let [leaves (.-leaves doc)
|
||||
tier1 (into {} (map (fn [path] [path (wire/decode-json (aget leaves path))]))
|
||||
(js-keys leaves))]
|
||||
{:clip (leaf/clip cid tier1)
|
||||
:store (into {}
|
||||
(map (fn [^js b]
|
||||
[(.-key b)
|
||||
(cond-> {:descriptor (.-descriptor b)
|
||||
:data (wire/typed (block-type (.-descriptor b))
|
||||
(.-data b))}
|
||||
(.-state b) (assoc :state (wire/bytes-of (.-state b))))]))
|
||||
(array-seq (or (.-blocks doc) #js [])))}))
|
||||
241
frontend/src/arthur/domain/raster.cljs
Normal file
241
frontend/src/arthur/domain/raster.cljs
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
(ns arthur.domain.raster
|
||||
"Indexed flat-fill rasteriser.
|
||||
|
||||
Canvas2D antialiases path fills, and antialiasing is exactly what the target
|
||||
idiom does not have: Animator Pro fills polygons into a 256-colour indexed
|
||||
raster with hard edges (csd_render_poly). A preview that antialiases would
|
||||
misrepresent the look it exists to judge, so this writes palette indices into
|
||||
a byte buffer with an even-odd scanline fill and expands to RGBA only at the
|
||||
very end.
|
||||
|
||||
A raster is {:w :h :buf} where :buf is a Uint8Array, and the fill functions
|
||||
MUTATE it and return it. That is deliberate and it is the one place in domain/
|
||||
that mutates: a persistent 64000-entry vector rebuilt per draw op per frame is
|
||||
not a rasteriser. The mutation is confined — a raster is created, filled and
|
||||
blitted inside one frame, and never stored in app-db.
|
||||
|
||||
No DOM here. `->rgba` returns plain bytes; wrapping them in an ImageData is
|
||||
ui/canvas's job, which is also what lets every assertion below run in node.")
|
||||
|
||||
(defn make [w h]
|
||||
{:w w :h h :buf (js/Uint8Array. (* w h))})
|
||||
|
||||
(defn clear! [{:keys [buf] :as r} index]
|
||||
(.fill buf index)
|
||||
r)
|
||||
|
||||
(defn fill-poly-buf!
|
||||
"Even-odd scanline fill of a polygon held FLAT in `pts` as [x0 y0 x1 y1 …],
|
||||
using the first `n` points. Samples at pixel centres (y + 0.5), so a polygon
|
||||
edge landing exactly on a pixel boundary resolves consistently.
|
||||
|
||||
Flat and preallocated because this is the per-frame path: fixed topology means
|
||||
a node's vertex count is known at freeze time, so timeline/resolver hands the same
|
||||
buffer back every frame and a frame allocates nothing. At 30fps per-frame
|
||||
allocation is the only thing that will make this stutter.
|
||||
|
||||
`pts` may be a CLJS vector or any typed array; scanline crossings are collected
|
||||
into a plain JS array and sorted in place."
|
||||
[{:keys [w h buf] :as r} pts n index]
|
||||
(when (>= n 3)
|
||||
(let [px (fn [i] (if (vector? pts) (-nth pts (* 2 i)) (aget pts (* 2 i))))
|
||||
py (fn [i] (if (vector? pts) (-nth pts (inc (* 2 i))) (aget pts (inc (* 2 i)))))
|
||||
xs (array)
|
||||
ys (map py (range n))
|
||||
y0 (max 0 (js/Math.ceil (- (reduce min ys) 0.5)))
|
||||
y1 (min (dec h) (inc (js/Math.floor (- (reduce max ys) 0.5))))]
|
||||
;; `dotimes` over the span rather than a hand-rolled index: bounded
|
||||
;; iteration with no accumulator is what it is for, and it compiles to the
|
||||
;; same JS for-loop the recur did.
|
||||
(dotimes [dy (inc (- y1 y0))]
|
||||
(let [y (+ y0 dy)
|
||||
sy (+ y 0.5)]
|
||||
(set! (.-length xs) 0)
|
||||
(dotimes [i n]
|
||||
(let [j (mod (inc i) n)
|
||||
ay (py i) by (py j)]
|
||||
;; A horizontal edge contributes no crossing, and dividing by its
|
||||
;; zero height would emit Infinity.
|
||||
(when (not= ay by)
|
||||
(let [lo (min ay by) hi (max ay by)]
|
||||
;; Half-open in y: >= lo and < hi. A vertex shared by two edges
|
||||
;; is counted exactly once, so the parity cannot flip at a
|
||||
;; corner and leak a whole scanline.
|
||||
(when (and (>= sy lo) (< sy hi))
|
||||
(.push xs (+ (px i) (* (/ (- sy ay) (- by ay))
|
||||
(- (px j) (px i))))))))))
|
||||
(when (>= (.-length xs) 2)
|
||||
(.sort xs (fn [a b] (- a b)))
|
||||
;; Crossings pair up left to right: inside a span, outside the next.
|
||||
;; Iterated as PAIRS rather than as a stepped index, but over the
|
||||
;; array directly — `partition 2` over an `array-seq` says the same
|
||||
;; thing and allocates two seqs per scanline, which is some five
|
||||
;; thousand throwaway objects a frame in the hottest loop here.
|
||||
(dotimes [k (quot (.-length xs) 2)]
|
||||
(let [xa (aget xs (* 2 k))
|
||||
xb (aget xs (inc (* 2 k)))
|
||||
x-from (max 0 (js/Math.ceil (- xa 0.5)))
|
||||
x-to (min (dec w) (js/Math.floor (- xb 0.5)))
|
||||
row (* y w)]
|
||||
(dotimes [dx (inc (- x-to x-from))]
|
||||
(aset buf (+ row x-from dx) index)))))))))
|
||||
r)
|
||||
|
||||
(defn fill-poly!
|
||||
"`fill-poly-buf!` over a seq of {:x :y} points.
|
||||
|
||||
The map form is what the analysis stages and the paint tool speak, and what the
|
||||
JS oracle is diffed against; the flat form is what evaluation produces. ONE
|
||||
scanline implementation serves both, because two would drift and the drift
|
||||
would read as a rendering bug rather than as two functions disagreeing."
|
||||
[r pts index]
|
||||
(fill-poly-buf! r (into-array (mapcat (juxt :x :y) pts)) (count pts) index))
|
||||
|
||||
(defn fill-disc!
|
||||
"`over` is an optional stencil: when given, only pixels that currently hold
|
||||
that index are written. The indexed buffer is its own clip mask, which is
|
||||
how Animator Pro would do it - and it is what keeps the iris inside the
|
||||
eye. A disc clipped by the sclera cannot spill past the lid at any gaze or
|
||||
any radius, including mid-blink when the opening is a two-pixel sliver, so
|
||||
the lid crops the iris for free instead of the gaze range needing a
|
||||
clamp that would flatten the performance at the extremes."
|
||||
([r cx cy rad index] (fill-disc! r cx cy rad index nil))
|
||||
([{:keys [w h buf] :as r} cx cy rad index over]
|
||||
(let [rr (* rad rad)
|
||||
y0 (max 0 (js/Math.floor (- cy rad)))
|
||||
y1 (min (dec h) (js/Math.ceil (+ cy rad)))
|
||||
x0 (max 0 (js/Math.floor (- cx rad)))
|
||||
x1 (min (dec w) (js/Math.ceil (+ cx rad)))]
|
||||
(dotimes [iy (inc (- y1 y0))]
|
||||
(dotimes [ix (inc (- x1 x0))]
|
||||
(let [x (+ x0 ix)
|
||||
y (+ y0 iy)
|
||||
dx (- (+ x 0.5) cx)
|
||||
dy (- (+ y 0.5) cy)]
|
||||
(when (<= (+ (* dx dx) (* dy dy)) rr)
|
||||
(let [o (+ (* y w) x)]
|
||||
(when (or (nil? over) (= (aget buf o) over))
|
||||
(aset buf o index)))))))
|
||||
r)))
|
||||
|
||||
(defn fill-rect!
|
||||
"An exactly size x size block of pixels, snapped to the pixel grid, with the
|
||||
same optional stencil as fill-disc!.
|
||||
|
||||
The pupil is a SQUARE because at 320x200 it is three pixels across, and a
|
||||
circle of radius 1.5 is not a circle - it is a plus sign with the corners
|
||||
gnawed off, and it changes shape as it moves. A square that size is a
|
||||
deliberate mark that stays the same mark wherever it lands, which is the
|
||||
whole argument for flat shapes at this resolution.
|
||||
|
||||
The top-left is rounded rather than the centre, so the block is size x size
|
||||
on every frame. Round the extents instead and a fractional centre gives you
|
||||
three pixels on one frame and four on the next, which reads as the pupil
|
||||
breathing."
|
||||
([r cx cy size index] (fill-rect! r cx cy size index nil))
|
||||
([{:keys [w h buf] :as r} cx cy size index over]
|
||||
(let [size (js/Math.round size)]
|
||||
(when (>= size 1)
|
||||
(let [x0 (js/Math.round (- cx (/ size 2)))
|
||||
y0 (js/Math.round (- cy (/ size 2)))
|
||||
ya (max 0 y0) yb (min h (+ y0 size))
|
||||
xa (max 0 x0) xb (min w (+ x0 size))]
|
||||
(dotimes [iy (- yb ya)]
|
||||
(dotimes [ix (- xb xa)]
|
||||
(let [o (+ (* (+ ya iy) w) xa ix)]
|
||||
(when (or (nil? over) (= (aget buf o) over))
|
||||
(aset buf o index))))))))
|
||||
r))
|
||||
|
||||
(def ^:private little-endian?
|
||||
(let [b (js/ArrayBuffer. 4)]
|
||||
(aset (js/Uint32Array. b) 0 1)
|
||||
(= 1 (aget (js/Uint8Array. b) 0))))
|
||||
|
||||
;; The palette arrives as a CLJS vector of [r g b] vectors, which is the right
|
||||
;; shape to author and the wrong shape to read 64,000 times a frame: a `nth` into
|
||||
;; a vector of vectors is four protocol dispatches per pixel, and that measured at
|
||||
;; 3.16ms per frame against 0.11ms for the same work off typed arrays. So it is
|
||||
;; flattened once and cached by IDENTITY of the source vector — palettes are
|
||||
;; values and a swap replaces the whole thing, so identity is exactly the right
|
||||
;; test and there is no invalidation to get wrong.
|
||||
(defonce ^:private flat-cache (atom nil))
|
||||
|
||||
(defn- flatten-palette [palette-rgb]
|
||||
(let [cached @flat-cache]
|
||||
(if (and cached (identical? palette-rgb (:src cached)))
|
||||
cached
|
||||
(let [p8 (js/Uint8Array. (* 256 3))
|
||||
p32 (js/Uint32Array. 256)]
|
||||
(dotimes [i 256]
|
||||
;; An index with no palette entry comes out MAGENTA rather than
|
||||
;; transparent or black: writing an index the palette does not have is
|
||||
;; a bug, and it should be impossible to miss.
|
||||
(let [c (or (nth palette-rgb i nil) [255 0 255])
|
||||
r (nth c 0) g (nth c 1) b (nth c 2)]
|
||||
(aset p8 (* i 3) r)
|
||||
(aset p8 (+ 1 (* i 3)) g)
|
||||
(aset p8 (+ 2 (* i 3)) b)
|
||||
(aset p32 i (if little-endian?
|
||||
(bit-or (bit-shift-left 255 24) (bit-shift-left b 16)
|
||||
(bit-shift-left g 8) r)
|
||||
(bit-or (bit-shift-left r 24) (bit-shift-left g 16)
|
||||
(bit-shift-left b 8) 255)))))
|
||||
(reset! flat-cache {:src palette-rgb :p8 p8 :p32 p32})))))
|
||||
|
||||
(defn ->rgba
|
||||
"Expand indices through the palette at integer zoom. Nearest-neighbour by
|
||||
construction, so no filtering softens the result.
|
||||
|
||||
Returns {:width :height :data} with :data a Uint8ClampedArray, ready to hand to
|
||||
an ImageData.
|
||||
|
||||
`dest` is an optional Uint8ClampedArray to write into instead of allocating
|
||||
one. At 320x200 the buffer is 256KB, and allocating and discarding that thirty
|
||||
times a second is exactly the per-frame allocation the model is arranged to
|
||||
avoid; ui/canvas passes the live ImageData's own array.
|
||||
|
||||
At zoom 1 on a little-endian machine this writes ONE 32-bit word per pixel
|
||||
through a Uint32Array view of the same buffer, which is the whole of the inner
|
||||
loop. Every other case walks bytes. Both paths read the same flattened palette
|
||||
and raster-test asserts they agree with a naive reference pixel for pixel,
|
||||
because a fast path that is subtly wrong about colour would look like a palette
|
||||
bug rather than like an optimisation."
|
||||
([r palette-rgb] (->rgba r palette-rgb 1 nil))
|
||||
([r palette-rgb zoom] (->rgba r palette-rgb zoom nil))
|
||||
([{:keys [w h buf]} palette-rgb zoom dest]
|
||||
(let [W (* w zoom)
|
||||
H (* h zoom)
|
||||
d (or dest (js/Uint8ClampedArray. (* W H 4)))
|
||||
{:keys [p8 p32]} (flatten-palette palette-rgb)]
|
||||
(if (and (= 1 zoom) little-endian? (zero? (mod (.-byteOffset d) 4)))
|
||||
(let [v (js/Uint32Array. (.-buffer d) (.-byteOffset d) (* w h))]
|
||||
(dotimes [i (* w h)]
|
||||
(aset v i (aget p32 (aget buf i)))))
|
||||
(dotimes [y H]
|
||||
(let [srow (* (js/Math.floor (/ y zoom)) w)]
|
||||
(dotimes [x W]
|
||||
(let [p (* 3 (aget buf (+ srow (js/Math.floor (/ x zoom)))))
|
||||
o (* (+ (* y W) x) 4)]
|
||||
(aset d o (aget p8 p))
|
||||
(aset d (+ o 1) (aget p8 (+ p 1)))
|
||||
(aset d (+ o 2) (aget p8 (+ p 2)))
|
||||
(aset d (+ o 3) 255))))))
|
||||
{:width W :height H :data d})))
|
||||
|
||||
(defn draw-ops!
|
||||
"Paint a list of draw ops, in the order given, into the raster. Stage 7.
|
||||
|
||||
This is the boundary the whole model is arranged around: an op carries raster
|
||||
space points and a PALETTE INDEX, and the rasteriser knows nothing about nodes,
|
||||
channels, time maps or provenance. Everything above here can be rearranged
|
||||
without touching a scanline, and a painted cel and a rotoscoped mouth arrive
|
||||
here indistinguishable from each other, which is the point."
|
||||
[r ops]
|
||||
(doseq [{:keys [kind pts n color stencil cx cy size] :as op} ops]
|
||||
(case kind
|
||||
:poly (fill-poly-buf! r pts n color)
|
||||
:disc (fill-disc! r cx cy (:r op) color stencil)
|
||||
:rect (fill-rect! r cx cy size color stencil)
|
||||
(throw (ex-info "draw op kind is not rasterisable" {:op (dissoc op :pts)}))))
|
||||
r)
|
||||
96
frontend/src/arthur/domain/ring.cljs
Normal file
96
frontend/src/arthur/domain/ring.cljs
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
(ns arthur.domain.ring
|
||||
"Operations on an ordered ring of points.
|
||||
|
||||
A ring here is a closed traversal: a vector of points where slot k means the
|
||||
same thing on every frame of a shot. That is what makes temporal
|
||||
correspondence possible at all, so nothing in here is allowed to reorder,
|
||||
insert or adaptively decimate — every function is index-preserving or returns
|
||||
slot positions.")
|
||||
|
||||
(defn subsample-slots
|
||||
"Pick `n` slots from a ring of `len` by even spacing. Returns RING POSITIONS,
|
||||
not landmark ids: positions are the vertex identity downstream, and mapping ids
|
||||
back to positions with indexOf would silently pick the wrong slot if a table
|
||||
ever repeated an id.
|
||||
|
||||
For even n this naturally lands on the cardinal positions (corners and lip
|
||||
centres) of a 20-point ring. Fixed indices, never adaptive decimation: the
|
||||
vertex at slot k means the same thing on every frame of the shot."
|
||||
[len n]
|
||||
(mapv (fn [k] (mod (js/Math.round (/ (* k len) n)) len)) (range n)))
|
||||
|
||||
(defn subsample-ring
|
||||
"`subsample-slots` applied to a table, yielding landmark ids."
|
||||
[ring n]
|
||||
(mapv #(nth ring %) (subsample-slots (count ring) n)))
|
||||
|
||||
(defn offset-ring
|
||||
"Push a ring outward from its centroid by a FIXED distance, not by a scale
|
||||
factor.
|
||||
|
||||
Scaling collapses with the shape: a shut eyelid scaled by 1.1 is still a shut
|
||||
eyelid, so the lash line - the only thing left to draw when the eye is closed
|
||||
- would vanish exactly on the frames where it is the whole drawing. A fixed
|
||||
radial offset gives a band of roughly constant thickness that survives the
|
||||
ring going degenerate, and it keeps a star-shaped ring simple, which
|
||||
docs/design.md requires of every cut part."
|
||||
[pts d]
|
||||
(if (or (nil? d) (zero? d))
|
||||
pts
|
||||
(let [n (count pts)
|
||||
cx (/ (reduce + (map :x pts)) n)
|
||||
cy (/ (reduce + (map :y pts)) n)]
|
||||
(mapv (fn [p]
|
||||
(let [dx (- (:x p) cx)
|
||||
dy (- (:y p) cy)
|
||||
m (js/Math.hypot dx dy)]
|
||||
;; A vertex sitting exactly on the centroid has no outward
|
||||
;; direction. Leave it where it is rather than emitting NaN and
|
||||
;; poisoning the whole ring.
|
||||
(if (< m 1e-9)
|
||||
{:x (:x p) :y (:y p)}
|
||||
{:x (+ (:x p) (* (/ dx m) d))
|
||||
:y (+ (:y p) (* (/ dy m) d))})))
|
||||
pts))))
|
||||
|
||||
(defn- orient
|
||||
"Sign of the cross product (p->q) x (p->r): which side of pq the point r is on."
|
||||
[p q r]
|
||||
(js/Math.sign (- (* (- (:x q) (:x p)) (- (:y r) (:y p)))
|
||||
(* (- (:y q) (:y p)) (- (:x r) (:x p))))))
|
||||
|
||||
(defn segments-cross?
|
||||
"True when ab and cd cross properly. Collinear and touching cases are
|
||||
deliberately NOT crossings: adjacent ring edges share an endpoint, and a
|
||||
degenerate ring — the shut eyelid — has collinear ones. Reporting those would
|
||||
make the simplicity assertion fire on exactly the shapes it has to allow."
|
||||
[a b c d]
|
||||
(let [o1 (orient a b c) o2 (orient a b d)
|
||||
o3 (orient c d a) o4 (orient c d b)]
|
||||
(and (not= o1 o2) (not= o3 o4)
|
||||
(not (zero? o1)) (not (zero? o2))
|
||||
(not (zero? o3)) (not (zero? o4)))))
|
||||
|
||||
(defn self-intersections
|
||||
"Every pair of non-adjacent edges of the closed ring that cross, as [i j].
|
||||
|
||||
This exists because \"fixed topology\" is load-bearing: because hold parts CUT
|
||||
between poses rather than interpolating, a ring whose vertex order is wrong
|
||||
self-intersects and renders as blocks meeting at corners. It is invisible at
|
||||
odd vertex counts and obvious at even ones, so it needs an assertion rather
|
||||
than an eyeball."
|
||||
[pts]
|
||||
(let [n (count pts)
|
||||
at (fn [i] (nth pts (mod i n)))]
|
||||
(vec
|
||||
(for [i (range n)
|
||||
j (range (inc i) n)
|
||||
:when (and (not= (mod (inc j) n) i)
|
||||
(not= (mod (inc i) n) j))
|
||||
:when (segments-cross? (at i) (at (inc i)) (at j) (at (inc j)))]
|
||||
[i j]))))
|
||||
|
||||
(defn simple?
|
||||
"True when no pair of non-adjacent edges crosses."
|
||||
[pts]
|
||||
(empty? (self-intersections pts)))
|
||||
148
frontend/src/arthur/domain/sha256.cljs
Normal file
148
frontend/src/arthur/domain/sha256.cljs
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
(ns arthur.domain.sha256
|
||||
"SHA-256, synchronous, in pure ClojureScript.
|
||||
|
||||
WHY NOT `crypto.subtle`. It is async, and every caller here is a pure function
|
||||
in the `(f params inputs) -> output` shape: a content address is computed in the
|
||||
middle of `flow/freeze`, inside a `let`, and a promise there would turn the
|
||||
whole stage inside out. `crypto.createHash` exists in node and not in the
|
||||
browser, which is worse — the tests would be hashing with a different
|
||||
implementation from the app.
|
||||
|
||||
WHY NOT A DEPENDENCY. It is sixty lines, it never changes, and the thing it has
|
||||
to agree with is not another JS library: it is Python's `hashlib`. The server
|
||||
recomputes the key of every block and every analysis it is handed and refuses a
|
||||
mismatch (see clips/views.py), so a disagreement between the two languages is
|
||||
not a hash that looks different — it is an upload that 409s with nothing wrong.
|
||||
`sha256-test` therefore pins the digests that `hashlib` produced, including the
|
||||
55/56/63/64 and 119/120-byte cases either side of both padding boundaries,
|
||||
which is where a hand-written implementation is wrong if it is wrong at all.
|
||||
|
||||
SIGN. JS bitwise operators work on 32-bit SIGNED integers, so `bit-xor` and
|
||||
`bit-shift-left` hand back negative numbers, and a negative number entering an
|
||||
addition mod 2^32 is off by 2^32. Every intermediate that feeds an addition is
|
||||
therefore normalised through `u32`. That is the bug this implementation would
|
||||
have, and it is invisible on short inputs — `\"abc\"` passes with the sign bug in
|
||||
place on some rounds — which is the other reason the vectors above are pinned."
|
||||
(:require [clojure.string :as str]))
|
||||
|
||||
(def ^:private round-k
|
||||
(js/Uint32Array.
|
||||
#js [0x428a2f98 0x71374491 0xb5c0fbcf 0xe9b5dba5 0x3956c25b 0x59f111f1
|
||||
0x923f82a4 0xab1c5ed5 0xd807aa98 0x12835b01 0x243185be 0x550c7dc3
|
||||
0x72be5d74 0x80deb1fe 0x9bdc06a7 0xc19bf174 0xe49b69c1 0xefbe4786
|
||||
0x0fc19dc6 0x240ca1cc 0x2de92c6f 0x4a7484aa 0x5cb0a9dc 0x76f988da
|
||||
0x983e5152 0xa831c66d 0xb00327c8 0xbf597fc7 0xc6e00bf3 0xd5a79147
|
||||
0x06ca6351 0x14292967 0x27b70a85 0x2e1b2138 0x4d2c6dfc 0x53380d13
|
||||
0x650a7354 0x766a0abb 0x81c2c92e 0x92722c85 0xa2bfe8a1 0xa81a664b
|
||||
0xc24b8b70 0xc76c51a3 0xd192e819 0xd6990624 0xf40e3585 0x106aa070
|
||||
0x19a4c116 0x1e376c08 0x2748774c 0x34b0bcb5 0x391c0cb3 0x4ed8aa4a
|
||||
0x5b9cca4f 0x682e6ff3 0x748f82ee 0x78a5636f 0x84c87814 0x8cc70208
|
||||
0x90befffa 0xa4506ceb 0xbef9a3f7 0xc67178f2]))
|
||||
|
||||
(defn- u32 [x] (unsigned-bit-shift-right x 0))
|
||||
|
||||
(defn- rotr [x n]
|
||||
(u32 (bit-or (unsigned-bit-shift-right x n) (bit-shift-left x (- 32 n)))))
|
||||
|
||||
(defn- pad
|
||||
"The message, padded: a 0x80 byte, zeros, and the bit length as a big-endian
|
||||
64-bit integer. The length is written as two 32-bit halves because a JS number
|
||||
cannot hold a 64-bit integer and nothing here will ever hash 512MB."
|
||||
[^js bytes]
|
||||
(let [n (.-length bytes)
|
||||
total (* 64 (js/Math.ceil (/ (+ n 9) 64)))
|
||||
out (js/Uint8Array. total)
|
||||
bits (* 8 n)]
|
||||
(.set out bytes)
|
||||
(aset out n 0x80)
|
||||
;; The high half is the bit count above 2^32; exact for any input JS can hold.
|
||||
(let [hi (js/Math.floor (/ bits 4294967296))
|
||||
lo (u32 bits)]
|
||||
(dotimes [i 4]
|
||||
(aset out (+ total -8 i) (bit-and 0xff (unsigned-bit-shift-right hi (* 8 (- 3 i)))))
|
||||
(aset out (+ total -4 i) (bit-and 0xff (unsigned-bit-shift-right lo (* 8 (- 3 i)))))))
|
||||
out))
|
||||
|
||||
(defn digest
|
||||
"SHA-256 of a Uint8Array, as a Uint8Array of 32 bytes."
|
||||
[^js bytes]
|
||||
(let [msg (pad bytes)
|
||||
h (js/Uint32Array. #js [0x6a09e667 0xbb67ae85 0x3c6ef372 0xa54ff53a
|
||||
0x510e527f 0x9b05688c 0x1f83d9ab 0x5be0cd19])
|
||||
w (js/Uint32Array. 64)
|
||||
v (js/Uint32Array. 8)]
|
||||
(dotimes [block (quot (.-length msg) 64)]
|
||||
(let [base (* 64 block)]
|
||||
(dotimes [i 16]
|
||||
(let [o (+ base (* 4 i))]
|
||||
(aset w i (u32 (bit-or (bit-shift-left (aget msg o) 24)
|
||||
(bit-shift-left (aget msg (+ o 1)) 16)
|
||||
(bit-shift-left (aget msg (+ o 2)) 8)
|
||||
(aget msg (+ o 3)))))))
|
||||
(dotimes [j 48]
|
||||
(let [i (+ j 16)
|
||||
x (aget w (- i 15))
|
||||
y (aget w (- i 2))
|
||||
s0 (u32 (bit-xor (rotr x 7) (rotr x 18) (unsigned-bit-shift-right x 3)))
|
||||
s1 (u32 (bit-xor (rotr y 17) (rotr y 19) (unsigned-bit-shift-right y 10)))]
|
||||
(aset w i (+ (aget w (- i 16)) s0 (aget w (- i 7)) s1))))
|
||||
(.set v h)
|
||||
(dotimes [i 64]
|
||||
(let [a (aget v 0) b (aget v 1) c (aget v 2) d (aget v 3)
|
||||
e (aget v 4) f (aget v 5) g (aget v 6) hh (aget v 7)
|
||||
s1 (u32 (bit-xor (rotr e 6) (rotr e 11) (rotr e 25)))
|
||||
choice (u32 (bit-xor (bit-and e f) (bit-and (bit-not e) g)))
|
||||
t1 (+ hh s1 choice (aget round-k i) (aget w i))
|
||||
s0 (u32 (bit-xor (rotr a 2) (rotr a 13) (rotr a 22)))
|
||||
maj (u32 (bit-xor (bit-and a b) (bit-and a c) (bit-and b c)))
|
||||
t2 (+ s0 maj)]
|
||||
(aset v 7 g) (aset v 6 f) (aset v 5 e)
|
||||
(aset v 4 (+ d t1))
|
||||
(aset v 3 c) (aset v 2 b) (aset v 1 a)
|
||||
(aset v 0 (+ t1 t2))))
|
||||
(dotimes [i 8]
|
||||
(aset h i (+ (aget h i) (aget v i))))))
|
||||
(let [out (js/Uint8Array. 32)]
|
||||
(dotimes [i 8]
|
||||
(dotimes [b 4]
|
||||
(aset out (+ (* 4 i) b)
|
||||
(bit-and 0xff (unsigned-bit-shift-right (aget h i) (* 8 (- 3 b)))))))
|
||||
out)))
|
||||
|
||||
(defn hex
|
||||
"Lowercase hex of a byte array, which is the form `hashlib.hexdigest()` gives
|
||||
and therefore the form a key is written in."
|
||||
[^js bytes]
|
||||
(str/join (map (fn [i] (.padStart (.toString (aget bytes i) 16) 2 "0"))
|
||||
(range (.-length bytes)))))
|
||||
|
||||
(defn of-bytes [^js bytes] (hex (digest bytes)))
|
||||
|
||||
(def ^:private utf8 (js/TextEncoder.))
|
||||
|
||||
(defn of-string
|
||||
"UTF-8 first, and that is not a detail: a descriptor holds source filenames, so
|
||||
a clip called \"café.mov\" hashes to what Python's `hashlib` gives for the same
|
||||
bytes only if the encoding is agreed. `TextEncoder` is UTF-8 by definition."
|
||||
[s]
|
||||
(of-bytes (.encode utf8 s)))
|
||||
|
||||
(defn key-of
|
||||
"The form a tier-2 key is written in everywhere: \"sha256:<64 hex>\".
|
||||
|
||||
PREFIXED, because a bare hex string in a document says nothing about what
|
||||
produced it, and the first time this changes algorithm every stored key has to
|
||||
be readable as the old one. It is also what makes a descriptive placeholder
|
||||
key — the `\"take/geom\"` these replaced — impossible to confuse with an address."
|
||||
[s]
|
||||
(str "sha256:" (of-string s)))
|
||||
|
||||
(defn key?
|
||||
"Does this string name a content address?
|
||||
|
||||
A string test and not a lookup, on purpose: tier 1 must be checkable without
|
||||
tier 2 in hand, which is the whole point of the split. It is what `domain/leaf`
|
||||
uses to refuse a document carrying a placeholder key like the \"take/geom\" that
|
||||
content addressing replaced."
|
||||
[s]
|
||||
(boolean (and (string? s) (re-matches #"sha256:[0-9a-f]{64}" s))))
|
||||
573
frontend/src/arthur/domain/timeline.cljs
Normal file
573
frontend/src/arthur/domain/timeline.cljs
Normal file
|
|
@ -0,0 +1,573 @@
|
|||
(ns arthur.domain.timeline
|
||||
"A TIMELINE: an ordered bag of nodes in its own frame space, and the two ways to
|
||||
evaluate it at a frame.
|
||||
|
||||
{:id :main :frames 229 :nodes {id -> node} :palette nil}
|
||||
|
||||
That is the whole type, and EVERYTHING THAT HOLDS NODES IS ONE OF THESE. A
|
||||
clip's root timeline is one; a symbol in the library is one; a `:kind :symbol`
|
||||
node is an INSTANCE of one. An earlier arrangement had the clip's node tree and
|
||||
a library symbol as two structures with the same fields and never said they were
|
||||
the same thing — the clip map carried `:fps`, `:width`, `:height`, `:analysis`
|
||||
and the tracking identities alongside `:nodes`, so a symbol had nowhere to live
|
||||
that was not a clip with seven meaningless fields. Flash's `_root` is a
|
||||
MovieClip and After Effects' pre-comp is just a layer; collapsing them is what
|
||||
makes nesting arbitrary and free rather than a feature to be added.
|
||||
|
||||
The clip-level facts are in `arthur.domain.clip`. A timeline has a FRAME SPACE,
|
||||
not a rate and not a size: `:fps` is the clip's, because a rate is a fact about
|
||||
how fast the whole thing plays, and a nested timeline cannot have its own.
|
||||
|
||||
TWO AXES OF NESTING, and conflating them is why \"nested\" and \"flat with parent
|
||||
pointers\" sound contradictory when they are not. Parent/child is transform
|
||||
composition WITHIN one timeline and is stored flat with pointers. Instance is a
|
||||
timeline inside another timeline and is stored by reference into the library.
|
||||
Each timeline is flat; timelines nest. Every argument for flat storage —
|
||||
addressability, one-field reparenting, structural sharing, per-node sync leaves —
|
||||
is about the first axis and is untouched by the second.
|
||||
|
||||
Two ways to evaluate one at a frame:
|
||||
|
||||
(eval-frame tl f store) THE SPECIFICATION. Allocating, order-free,
|
||||
obviously correct. Use it in tests and for a
|
||||
one-off render.
|
||||
|
||||
(resolver tl store) -> (fn [f] ops). What playback uses. Caches the
|
||||
topological order and the z paths, holds one
|
||||
CURSOR per channel and one PREALLOCATED point
|
||||
buffer per node, so a frame allocates the op
|
||||
maps and nothing else.
|
||||
|
||||
Both run the same walk — `eval-into` below — parameterised by how a channel is
|
||||
read and where points are written. That is deliberate: two independent
|
||||
implementations of frame evaluation would drift, and the drift would look like
|
||||
a rendering bug rather than like two functions disagreeing. What differs
|
||||
between them is exactly the part that can be wrong, and timeline-test asserts
|
||||
they agree frame for frame in forward, backward and random order.
|
||||
|
||||
The output is a list of DRAW OPS, and it is the boundary with the rasteriser:
|
||||
ops carry palette indices and raster-space points, and the rasteriser knows
|
||||
nothing about nodes, channels or time.
|
||||
|
||||
Geometry is stored FLAT — [x0 y0 x1 y1 …] — in authored channels as well as
|
||||
dense ones. A dense block is a rectangular Int16Array and an authored ring is a
|
||||
vector of numbers, and they read the same way, which is what makes freezing
|
||||
fill in the same channel rather than convert into a second format."
|
||||
(:require [arthur.domain.channel :as ch]
|
||||
[arthur.domain.node :as node]
|
||||
[arthur.domain.pose :as pose]
|
||||
[arthur.domain.palette :as pal]))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; structure: depth, topological order, draw order
|
||||
|
||||
(defn lineage
|
||||
"The node's id and every ancestor's, nearest first and root last.
|
||||
|
||||
One walk, shared by `depth` and `z-path`, which otherwise duplicate it.
|
||||
|
||||
A cycle is caught by LENGTH rather than by a `seen` set: a chain that does not
|
||||
repeat cannot be longer than the number of nodes, so one step past that is
|
||||
proof of a loop and needs no bookkeeping. Caught rather than hung — a cycle is
|
||||
reachable from one bad `:node/set-parent`, and a hung tab is a far worse
|
||||
diagnostic than a stack trace naming the nodes."
|
||||
[nodes id]
|
||||
(let [up (fn [i]
|
||||
(when-let [p (:parent (get nodes i))]
|
||||
(if (contains? nodes p)
|
||||
p
|
||||
(throw (ex-info "node's :parent is not in the timeline"
|
||||
{:node i :parent p})))))
|
||||
chain (into [] (comp (take-while some?) (take (inc (count nodes))))
|
||||
(iterate up id))]
|
||||
(when (> (count chain) (count nodes))
|
||||
(throw (ex-info "parent cycle in timeline" {:node id :chain chain})))
|
||||
chain))
|
||||
|
||||
(defn depth
|
||||
"Number of ancestors."
|
||||
[nodes id]
|
||||
(dec (count (lineage nodes id))))
|
||||
|
||||
(defn order
|
||||
"Node ids in topological order: every node after its parent.
|
||||
|
||||
Sorting by parent depth is enough — it does not need Kahn's algorithm, because
|
||||
the only edge is parent, and a node's depth is by definition greater than its
|
||||
parent's. Ties are broken by id so the order is deterministic across runs,
|
||||
which matters because the draw-order sort below falls back on this position."
|
||||
[nodes]
|
||||
(vec (sort-by (juxt #(depth nodes %) str) (keys nodes))))
|
||||
|
||||
(defn z-path
|
||||
"The node's z index and every ancestor's, root first.
|
||||
|
||||
Draw order is depth-first by sibling z, so the key that sorts it is the chain
|
||||
of z values from the root. A parent's path is a PREFIX of its child's, which is
|
||||
why a parent draws before its children without that being a special case.
|
||||
|
||||
`:z` values are fractional-index STRINGS (\"a1\", \"a3\") and compare
|
||||
lexicographically, so a node can always be inserted between two siblings
|
||||
without renumbering either."
|
||||
[nodes id]
|
||||
(mapv #(:z (get nodes %)) (rseq (lineage nodes id))))
|
||||
|
||||
(defn- z-lex
|
||||
"Lexicographic compare of two z paths, a prefix sorting first.
|
||||
|
||||
`compare` on vectors will not do: it compares COUNT first, so a deep
|
||||
descendant of \"a1\" would sort after a shallow \"a2\" and a painted cel would
|
||||
jump in front of the head that carries it.
|
||||
|
||||
`map` over two collections stops at the shorter and `first` short-circuits at
|
||||
the first difference, so this walks no further than it has to."
|
||||
[a b]
|
||||
(or (first (remove zero? (map compare a b)))
|
||||
(- (count a) (count b))))
|
||||
|
||||
(defn draw-rank
|
||||
"id -> its position in draw order.
|
||||
|
||||
Computed ONCE. Draw order is a function of the z paths, which are structural —
|
||||
they change when the timeline changes and never because the playhead moved — so
|
||||
sorting ops by z on every frame was re-deriving a constant thirty times a
|
||||
second. Here it is derived when the timeline is, and a frame sorts small integers.
|
||||
|
||||
`sort-by` is stable and `ord` is topological, so nodes sharing a z path keep
|
||||
parent-before-child order without a tiebreak field on every op."
|
||||
[nodes ord]
|
||||
(let [paths (into {} (map (juxt identity #(z-path nodes %))) ord)]
|
||||
(into {} (map-indexed (fn [i id] [id i])) (sort-by paths z-lex ord))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; colour
|
||||
|
||||
(defn colour-index
|
||||
"Tone keyword -> the index the raster writes, in a given palette.
|
||||
|
||||
`palette` is a map of tone -> index. It is a PARAMETER, not a global: a tone
|
||||
names which mark this is, and which ramp it is read in belongs to the timeline
|
||||
the node sits in, so resolution cannot reach for one ambient answer. Today
|
||||
there is one palette and it is passed in anyway; when timelines carry a
|
||||
`:palette` channel, the walk carries the palette in scope exactly as it already
|
||||
carries the parent transform and the local frame.
|
||||
|
||||
An unknown tone resolves to 255, which the palette expansion renders MAGENTA.
|
||||
Loud rather than fatal, and the same choice raster/->rgba already makes:
|
||||
naming a colour the ramp does not have is a bug in authored data, and it should
|
||||
be impossible to miss and should not take the frame down."
|
||||
[palette k]
|
||||
(cond
|
||||
(number? k) k
|
||||
(nil? k) 255
|
||||
:else (get palette k 255)))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the walk
|
||||
|
||||
(defn- in-span?
|
||||
"`:span` is Lottie's ip/op and Flash's PlaceObject/RemoveObject: the range over
|
||||
which the node EXISTS, tested in the PARENT's frame space and therefore before
|
||||
the node's own time map runs. Distinct from `[:vis]`, which blinks an existing
|
||||
node on and off. Half-open, so two adjacent spans do not both own a frame."
|
||||
[n f]
|
||||
(if-let [[in out] (:span n)]
|
||||
(and (>= f in) (< f out))
|
||||
true))
|
||||
|
||||
(defn- finish
|
||||
"Resolve stencils, then sort into draw order.
|
||||
|
||||
A stencil is a COLOUR KEY, not a node reference: it is the take format's
|
||||
`clip=`, and the indexed buffer being its own clip mask is what keeps the iris
|
||||
inside the eye at any gaze and any radius without a per-part mask. So the
|
||||
stencil node's own colour is looked up here, after the walk, because the
|
||||
stencil may sit anywhere in the order. Two nodes sharing a palette entry share
|
||||
a stencil, which is inherent to the technique rather than a defect in it.
|
||||
|
||||
A node stencilled by something that drew NOTHING is DROPPED, not drawn
|
||||
unclipped: unclipped would be an iris floating over the cheek on exactly the
|
||||
frames where the eye is missing."
|
||||
[rank ops]
|
||||
(let [by-id (into {} (map (juxt :node :color)) ops)]
|
||||
(->> ops
|
||||
(keep (fn [op]
|
||||
(if-let [s (:stencil op)]
|
||||
(when-let [idx (get by-id s)]
|
||||
(assoc op :stencil idx))
|
||||
op)))
|
||||
(sort-by (comp rank :node))
|
||||
vec)))
|
||||
|
||||
(defn- n-points
|
||||
"Points in a flat [x0 y0 x1 y1 …] value, authored vector or dense view alike."
|
||||
[pts]
|
||||
(quot (if (vector? pts) (count pts) (.-length pts)) 2))
|
||||
|
||||
(defn- xform-at
|
||||
"The five transform components at the node's local frame, or nil when any of
|
||||
them has no value on it."
|
||||
[rd]
|
||||
(let [pos (rd [:xform :pos])
|
||||
rot (rd [:xform :rot])
|
||||
scl (rd [:xform :scale])
|
||||
skw (rd [:xform :skew])
|
||||
anc (rd [:xform :anchor])]
|
||||
(when-not (or (ch/nothing? pos) (ch/nothing? rot) (ch/nothing? scl)
|
||||
(ch/nothing? skw) (ch/nothing? anc))
|
||||
[pos rot scl skw anc])))
|
||||
|
||||
(defn- visible?
|
||||
"Is the node switched on this frame?
|
||||
|
||||
`[:vis]` IS A BOOLEAN, and this insists on it rather than testing truthiness,
|
||||
because the two obvious implementations are both wrong about a DENSE `[:vis]`.
|
||||
A dense block yields 0 or 1, and 0 is TRUTHY in CLJS — so `(if v …)` shows a
|
||||
hidden frame, and `(true? v)` hides every frame. Neither reads as an error.
|
||||
|
||||
docs/animation-model.md's parts table says `:mouth-in` carries `[:vis]` dense;
|
||||
flow/freeze writes it KEYED, because a threshold crossing is a handful of
|
||||
transitions and hold is the default, and because a human has to be able to fix
|
||||
one frame of it. When something does want a dense one it will land here loudly
|
||||
instead of blanking the timeline.
|
||||
|
||||
Absence is not a boolean and is not an error: a subject that is not on the
|
||||
frame has nothing to show."
|
||||
[id v]
|
||||
(cond
|
||||
(true? v) true
|
||||
(false? v) false
|
||||
(ch/nothing? v) false
|
||||
:else (throw (ex-info "[:vis] must sample to a boolean"
|
||||
{:node id :value v}))))
|
||||
|
||||
(defn- place
|
||||
"Where a node sits this frame, as {:m world :f local-frame :rd reader}, or nil
|
||||
when it is not on the frame at all.
|
||||
|
||||
Three gates, and nil from any of them removes the node's DESCENDANTS too,
|
||||
which is why this is one answer rather than three flags: a node outside its
|
||||
span does not exist, a switched-off feature takes its parts with it, and a node
|
||||
with no transform gives its children nowhere to be.
|
||||
|
||||
A missing [:geom :pts] is deliberately NOT one of them — that is `emit`'s
|
||||
business. An absent mouth outline has nothing to draw, but the head it hangs
|
||||
off is still exactly where it was, and that asymmetry is the whole reason
|
||||
presence is tracked per channel rather than per node."
|
||||
[{:keys [read mat-for pinv-for scratch]} n parent f]
|
||||
(let [pf (if parent (:f parent) f)]
|
||||
(when (in-span? n pf)
|
||||
(let [id (:id n)
|
||||
chs (node/channels n)
|
||||
lf (node/local-frame n pf)
|
||||
rd (fn [path] (read id path (get chs path) lf))]
|
||||
(when (visible? id (rd [:vis]))
|
||||
(when-let [[pos rot scl skw anc] (xform-at rd)]
|
||||
;; dest aliases `local` here, which mul! allows: it reads both
|
||||
;; operands fully before writing either.
|
||||
(let [m (node/local! (mat-for id) pos rot scl skw anc)]
|
||||
{:m (node/world! m (:m parent) (pinv-for id) m scratch)
|
||||
:f lf
|
||||
:rd rd})))))))
|
||||
|
||||
(defn- emit
|
||||
"Emit geometry in the timeline's space. Rect sizes stay fractional until
|
||||
rasterization, so enclosing symbol transforms can still scale them."
|
||||
[{:keys [palette buf-for]} n {:keys [m rd]} base]
|
||||
(let [colour #(colour-index palette (rd [:style :color]))]
|
||||
(case (:kind n)
|
||||
:group nil
|
||||
:symbol nil
|
||||
:audio nil
|
||||
|
||||
:poly
|
||||
(let [pts (rd [:geom :pts])]
|
||||
(when-not (ch/nothing? pts)
|
||||
(let [np (n-points pts)
|
||||
out (buf-for (:id n) np)]
|
||||
(dotimes [k np]
|
||||
(node/apply-pt! out k m
|
||||
(ch/component pts (* 2 k))
|
||||
(ch/component pts (inc (* 2 k)))))
|
||||
(assoc base :kind :poly :pts out :n np :color (colour)))))
|
||||
|
||||
:disc
|
||||
(let [rad (rd [:geom :radius])]
|
||||
(when-not (ch/nothing? rad)
|
||||
(assoc base :kind :disc
|
||||
:cx (aget m 4) :cy (aget m 5)
|
||||
:r (* rad (node/mean-scale m))
|
||||
:color (colour))))
|
||||
|
||||
:rect
|
||||
(let [size (rd [:geom :size])]
|
||||
(when-not (ch/nothing? size)
|
||||
(assoc base :kind :rect
|
||||
:cx (aget m 4) :cy (aget m 5)
|
||||
:size (* size (node/mean-scale m))
|
||||
:color (colour))))
|
||||
|
||||
(throw (ex-info "node kind is not implemented"
|
||||
{:node (:id n) :kind (:kind n)})))))
|
||||
|
||||
(defn- nodes-of
|
||||
"The timeline's node map, REFUSING a map that has none.
|
||||
|
||||
A clip and a timeline both have an `:id` and both are maps, so handing a CLIP to
|
||||
an evaluator is the one mistake this type split makes easy — and the result is
|
||||
not an error, it is `(:nodes clip)` being nil and a frame resolving to no ops at
|
||||
all. That reads as a black stage, or, in a benchmark, as \"0 nodes\" and a
|
||||
flattering number. It happened once while the split was being made, which is why
|
||||
this is a guard and not a comment."
|
||||
[tl]
|
||||
(let [nodes (:nodes tl)]
|
||||
(when-not (map? nodes)
|
||||
(throw (ex-info (str "not a timeline: :nodes is " (pr-str nodes)
|
||||
" — a clip is not a timeline, its `:timelines` hold them")
|
||||
{:keys (vec (sort-by str (keys tl)))})))
|
||||
nodes))
|
||||
|
||||
(defn- channel-frame
|
||||
"Anchors select measured frames; marked channels read instance pose choices."
|
||||
[choices anchors nodes source-fps picture-fps id c lf]
|
||||
(cond
|
||||
(contains? anchors id)
|
||||
(pose/held-frame (get anchors id) lf lf)
|
||||
|
||||
(:pose-sampled? c)
|
||||
(pose/source-frame choices
|
||||
(if (contains? choices [:node id])
|
||||
[:node id]
|
||||
(or (:pose-group (get nodes id)) id))
|
||||
lf
|
||||
(node/sample-frame lf source-fps picture-fps))
|
||||
|
||||
:else lf))
|
||||
|
||||
(defn- prepared-anchors [nodes]
|
||||
(into {}
|
||||
(for [[id n] nodes :when (seq (:anchors n))]
|
||||
[id (vec (sort-by first (:anchors n)))])))
|
||||
|
||||
(defn- eval-into
|
||||
"One frame, as a fold over the nodes in topological order.
|
||||
|
||||
`ctx` carries how a channel is read and where its points are written:
|
||||
|
||||
:read (fn [id path channel local-frame] -> v)
|
||||
:palette tone -> index, the ramp in scope
|
||||
:mat-for (fn [id] -> Float64Array) the node's world transform
|
||||
:pinv-for (fn [id] -> Float64Array|nil) its parent-inverse
|
||||
:buf-for (fn [id n-points] -> Float64Array)
|
||||
:scratch one spare 6-element matrix"
|
||||
[ctx nodes ord rank f]
|
||||
(-> (reduce
|
||||
(fn [{:keys [placed ops] :as acc} id]
|
||||
(let [n (get nodes id)
|
||||
pid (:parent n)
|
||||
parent (when pid (get placed pid))]
|
||||
;; A node whose parent was dropped is dropped with it, and so is
|
||||
;; everything under it. Topological order is what makes that one
|
||||
;; lookup instead of a subtree walk.
|
||||
(if (and pid (nil? parent))
|
||||
acc
|
||||
(if-let [p (place ctx n parent f)]
|
||||
(let [_ (when-let [on-place (:on-place ctx)] (on-place id p))
|
||||
op (emit ctx n p {:node id :stencil (:stencil n)})]
|
||||
(cond-> (update acc :placed assoc id p)
|
||||
op (update :ops conj op)))
|
||||
acc))))
|
||||
{:placed {} :ops []}
|
||||
ord)
|
||||
:ops
|
||||
(->> (finish rank))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the specification
|
||||
|
||||
(defn eval-frame
|
||||
"Timeline at frame f -> draw ops in z order. Pure, and allocates freely.
|
||||
|
||||
`f` is in THIS timeline's frame space. At the clip's root that is clip frames;
|
||||
inside an instance it is the instance's own space, and the instance boundary is
|
||||
the only place the space changes.
|
||||
|
||||
This is the definition of what a frame means. `resolver` is what plays it."
|
||||
([tl f] (eval-frame tl f nil pal/index-of))
|
||||
([tl f store] (eval-frame tl f store pal/index-of))
|
||||
([tl f store palette] (eval-frame tl f store palette nil nil))
|
||||
([tl f store palette pose-tracks opts]
|
||||
(let [nodes (nodes-of tl)
|
||||
choices (pose/prepare pose-tracks)
|
||||
anchors (prepared-anchors nodes)
|
||||
{:keys [source-fps picture-fps]} opts
|
||||
ord (order nodes)]
|
||||
(eval-into {:read (fn [id path c lf]
|
||||
(ch/value-at c (channel-frame choices anchors nodes
|
||||
source-fps picture-fps id c lf)
|
||||
store))
|
||||
:palette palette
|
||||
:mat-for (fn [_id] (node/mat))
|
||||
:pinv-for (fn [id] (node/pinv (get nodes id)))
|
||||
:buf-for (fn [_id n] (js/Float64Array. (* 2 n)))
|
||||
:scratch (node/mat)}
|
||||
nodes ord (draw-rank nodes ord) f))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the playback path
|
||||
|
||||
(defn- point-capacity
|
||||
"How many points the widest value of a [:geom :pts] channel holds.
|
||||
|
||||
FIXED TOPOLOGY is what makes this a number at all: every key of a part carries
|
||||
the same vertex count with the same vertex meanings, so the buffer can be
|
||||
allocated once. A variable vertex count would force a per-frame offset table
|
||||
and a scan, which is why the aesthetic constraint is a performance asset rather
|
||||
than a cost."
|
||||
[c]
|
||||
(quot (cond
|
||||
(:dense c) (:stride (:dense c))
|
||||
(:animated? c) (transduce (map #(if (vector? %) (count %) (.-length %)))
|
||||
max 0 (vals (:keys c)))
|
||||
:else (let [v (:value c)] (if (vector? v) (count v) (.-length v))))
|
||||
2))
|
||||
|
||||
(defprotocol IResolver
|
||||
(world-of [this id]
|
||||
"The node's world transform AS OF THE LAST FRAME RESOLVED, or nil if it was
|
||||
not placed on that frame.
|
||||
|
||||
The matrices are the ones evaluation mutates in place, so this is a read of
|
||||
live state rather than a snapshot — which is exactly what the caller wants.
|
||||
A registered photo underlay has to ride the same transform the vectors went
|
||||
through or it is merely decorative, and it paints immediately after the frame
|
||||
it belongs to, so \"as of the last frame\" is the only answer that can be
|
||||
correct.")
|
||||
(frame-of [this id] "The placed node's local frame on the last resolve."))
|
||||
|
||||
(defn resolver
|
||||
"(fn [f] -> ops). Holds everything that does not change per frame.
|
||||
|
||||
The point buffers are REUSED between frames, so a caller must consume the ops
|
||||
before asking for the next frame. That is the contract the rAF loop wants
|
||||
anyway — it reads, blits, and dispatches nothing — and it is what makes a frame
|
||||
cost a lookup and a blit rather than an allocation per vertex.
|
||||
|
||||
The op maps themselves are allocated fresh, and deliberately: there are a dozen
|
||||
of them per frame against hundreds of points, so pooling them would buy
|
||||
nothing and cost the ability to hand an op list around as plain data."
|
||||
([tl] (resolver tl nil pal/index-of nil nil))
|
||||
([tl store] (resolver tl store pal/index-of nil nil))
|
||||
([tl store palette] (resolver tl store palette nil nil))
|
||||
([tl store palette pose-tracks] (resolver tl store palette pose-tracks nil))
|
||||
([tl store palette pose-tracks {:keys [source-fps picture-fps]}]
|
||||
(let [nodes (nodes-of tl)
|
||||
choices (pose/prepare pose-tracks)
|
||||
anchors (prepared-anchors nodes)
|
||||
ord (order nodes)
|
||||
rank (draw-rank nodes ord)
|
||||
cursors (into {}
|
||||
(map (fn [id]
|
||||
[id (into {} (map (fn [[p c]] [p (ch/cursor c store)]))
|
||||
(node/channels (get nodes id)))]))
|
||||
ord)
|
||||
mats (into {} (map (fn [id] [id (node/mat)])) ord)
|
||||
pinvs (into {} (keep (fn [id] (when-let [p (node/pinv (get nodes id))] [id p]))) ord)
|
||||
bufs (into {}
|
||||
(keep (fn [id]
|
||||
(when-let [c (get-in nodes [id :channels [:geom :pts]])]
|
||||
[id (js/Float64Array. (* 2 (point-capacity c)))])))
|
||||
ord)
|
||||
scratch (node/mat)
|
||||
;; Every call to mat-for is a placement: eval-into reaches it only after
|
||||
;; the span, visibility and transform gates have all passed. So wrapping
|
||||
;; it is how the resolver learns which nodes exist this frame without
|
||||
;; eval-into having to report it — and it covers groups, which are
|
||||
;; placed but emit no op, and which are exactly what an underlay rides.
|
||||
placed (volatile! {})
|
||||
ctx {:read (fn [id path c lf]
|
||||
(ch/sample! (get-in cursors [id path])
|
||||
(channel-frame choices anchors nodes
|
||||
source-fps picture-fps id c lf)))
|
||||
:palette palette
|
||||
:mat-for (fn [id] (get mats id))
|
||||
:on-place (fn [id p] (vswap! placed assoc id p))
|
||||
:pinv-for (fn [id] (get pinvs id))
|
||||
:buf-for (fn [id _n] (get bufs id))
|
||||
:scratch scratch}
|
||||
step (fn [f]
|
||||
(vreset! placed {})
|
||||
(eval-into ctx nodes ord rank f))]
|
||||
(reify
|
||||
IFn
|
||||
(-invoke [_ f] (step f))
|
||||
IResolver
|
||||
(world-of [_ id] (:m (get @placed id)))
|
||||
(frame-of [_ id] (:f (get @placed id)))))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(def timeline-keys
|
||||
"Every field a timeline may carry, and the reason `arthur.domain.leaf` refuses
|
||||
one it does not know: a field added without a leaf to save it in is a field that
|
||||
saves silently and comes back missing.
|
||||
|
||||
`:palette` is in the vocabulary and nothing writes one yet. A timeline is where
|
||||
a ramp belongs — `domain/timeline` takes the palette as a PARAMETER rather than
|
||||
reaching for a global precisely so that a nested timeline can carry its own —
|
||||
and leaving the field out would make the first one a migration instead of a
|
||||
write."
|
||||
#{:id :frames :nodes :palette})
|
||||
|
||||
(defn problems
|
||||
"Human-readable reasons this timeline will not evaluate. Empty means it will.
|
||||
|
||||
Node structure only. The tracking identities — subjects, features, groups — are
|
||||
the CLIP's and are checked by `arthur.domain.clip/problems`, which is not a
|
||||
layering nicety: a feature names nodes, and a library symbol's nodes are not
|
||||
the ones a face was tracked into.
|
||||
|
||||
Total by construction — it reports a cycle rather than looping on one — because
|
||||
its whole job is to be safe to run over authored data before that data is
|
||||
trusted."
|
||||
[tl]
|
||||
(let [nodes (:nodes tl)]
|
||||
(if-not (map? nodes)
|
||||
[":nodes must be a map of id -> node"]
|
||||
(-> []
|
||||
(into (for [[id n] nodes
|
||||
:when (not= id (:id n))]
|
||||
(str "node under key " (pr-str id) " has :id " (pr-str (:id n)))))
|
||||
(into (for [[id n] nodes
|
||||
:when (and (:parent n) (not (contains? nodes (:parent n))))]
|
||||
(str "node " (pr-str id) " has :parent " (pr-str (:parent n))
|
||||
" which is not in the timeline")))
|
||||
(into (for [[id n] nodes
|
||||
:when (and (:stencil n) (not (contains? nodes (:stencil n))))]
|
||||
(str "node " (pr-str id) " has :stencil " (pr-str (:stencil n))
|
||||
" which is not in the timeline")))
|
||||
(into (for [[id n] nodes
|
||||
p (node/problems n)]
|
||||
(str "node " (pr-str id) ": " p)))
|
||||
;; Anchors re-address the node's measurement, regardless of its name.
|
||||
(into (for [[id n] nodes
|
||||
:let [anchors (:anchors n)]
|
||||
:when (some? anchors)
|
||||
:when (not (and (map? anchors) (contains? anchors 0)
|
||||
(integer? (:frames tl))
|
||||
(every? #(and (integer? %) (<= 0 %)
|
||||
(< % (:frames tl)))
|
||||
(concat (keys anchors) (vals anchors)))
|
||||
(seq (:measured n))
|
||||
(= (:channels n) (:measured n))))]
|
||||
(str "node " (pr-str id)
|
||||
": :anchors must start at frame 0, name valid measured frames, and read that node's own measured channels")))
|
||||
(into (for [k (remove timeline-keys (keys tl))]
|
||||
(str "timeline has a field with no leaf to save it in: " (pr-str k))))
|
||||
(into (when-not (or (nil? (:frames tl)) (and (integer? (:frames tl)) (pos? (:frames tl))))
|
||||
[(str ":frames is " (pr-str (:frames tl))
|
||||
" — a timeline is a frame SPACE, so its length is a positive integer")]))
|
||||
(into (try
|
||||
(doall (map #(depth nodes %) (keys nodes)))
|
||||
nil
|
||||
(catch :default e [(ex-message e)])))))))
|
||||
103
frontend/src/arthur/domain/wire.cljs
Normal file
103
frontend/src/arthur/domain/wire.cljs
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
(ns arthur.domain.wire
|
||||
"The document's wire format, and the bytes' one.
|
||||
|
||||
TRANSIT, not JSON, and the reason is the two things docs/animation-model.md is
|
||||
most specific about. A channel's keys are a map BY FRAME NUMBER, and JSON has
|
||||
only string keys, so a save through `JSON.stringify` turns `{0 v, 4 v}` into
|
||||
`{\"0\" v, \"4\" v}` and every id in the scene from `:mouth` into `\"mouth\"` — a
|
||||
document that reloads as a subtly different type and fails somewhere downstream
|
||||
of where it broke. Transit carries integers, keywords and vector keys as
|
||||
themselves, and its output is still JSON, so the server stores a leaf in a
|
||||
JSONField and the admin can read it.
|
||||
|
||||
TRANSIT LOSES SORTEDNESS, which is why `domain/channel` says keys are a PLAIN
|
||||
map and builds the sorted index at read time. Nothing here re-sorts anything:
|
||||
a codec that returned a sorted map would work locally and stop working after one
|
||||
round trip, which is the failure the plain-map rule already prevents.
|
||||
|
||||
The bytes are separate from transit: JSON block reads carry base64, while
|
||||
uploads send binary file parts. `channel/dense-at` reads a block as
|
||||
`{:data <typed array> :state <Uint8Array>}` and both sides of the wire must hold
|
||||
byte-for-byte the same array — a handle that names a sha256 has to name the
|
||||
bytes you actually hold."
|
||||
(:require [cognitect.transit :as t]))
|
||||
|
||||
(def ^:private writer (t/writer :json))
|
||||
(def ^:private reader (t/reader :json))
|
||||
|
||||
(defn encode
|
||||
"A tier-1 value -> the transit-JSON text that goes in a leaf."
|
||||
[v]
|
||||
(t/write writer v))
|
||||
|
||||
(defn decode
|
||||
"The inverse. Whatever comes back is ordinary CLJS data."
|
||||
[s]
|
||||
(t/read reader s))
|
||||
|
||||
(defn encode-json
|
||||
"A tier-1 value -> transit as a PARSED JSON value, ready to go in a request body.
|
||||
|
||||
Transit's output is a JSON string, so a leaf could travel as a string and the
|
||||
server could store it as one. It travels parsed instead, so that the column
|
||||
holding it is a JSONField holding JSON rather than a JSONField holding a string
|
||||
that happens to contain JSON. Two things need that: the admin, where a leaf is
|
||||
either readable or it is a blob, and the field-wise merge of a channel leaf that
|
||||
docs/architecture.md describes as fifteen lines of Python — which is fifteen
|
||||
lines over transit's own `[\"^ \", \"~:keys\", ...]` and impossible over an
|
||||
opaque string."
|
||||
[v]
|
||||
(js/JSON.parse (encode v)))
|
||||
|
||||
(defn decode-json
|
||||
"The inverse of `encode-json`."
|
||||
[json]
|
||||
(decode (js/JSON.stringify json)))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the bytes
|
||||
|
||||
(def ^:private chunk-size
|
||||
"Not `chunk`, which is `cljs.core/chunk`. 8192 characters per `apply`."
|
||||
8192)
|
||||
|
||||
(defn base64
|
||||
"A typed array -> base64 of its bytes.
|
||||
|
||||
Chunked through `String.fromCharCode`: `apply` with a few hundred thousand
|
||||
arguments overflows the stack, and a 600-frame geometry block is exactly that
|
||||
size. The failure is a RangeError from inside a save, which points nowhere near
|
||||
the array that caused it."
|
||||
[^js block]
|
||||
(let [bytes (js/Uint8Array. (.-buffer block) (.-byteOffset block) (.-byteLength block))
|
||||
parts (js/Array.)]
|
||||
(loop [i 0]
|
||||
(when (< i (.-length bytes))
|
||||
(.push parts (.apply js/String.fromCharCode nil (.subarray bytes i (+ i chunk-size))))
|
||||
(recur (+ i chunk-size))))
|
||||
(js/btoa (.join parts ""))))
|
||||
|
||||
(defn bytes-of
|
||||
"base64 -> a Uint8Array."
|
||||
[s]
|
||||
(let [binary (js/atob s)
|
||||
out (js/Uint8Array. (.-length binary))]
|
||||
(dotimes [i (.-length binary)]
|
||||
(aset out i (.charCodeAt binary i)))
|
||||
out))
|
||||
|
||||
(defn typed
|
||||
"base64 -> the typed array a block of this element type is read through.
|
||||
|
||||
The type is a FIELD the block carries rather than something inferred from its
|
||||
length, because an Int16Array and a Float32Array over the same bytes are both
|
||||
valid readings and only one of them is the block."
|
||||
[type s]
|
||||
(let [u8 (bytes-of s)]
|
||||
(case type
|
||||
"int16" (js/Int16Array. (.-buffer u8))
|
||||
"float32" (js/Float32Array. (.-buffer u8))
|
||||
"float64" (js/Float64Array. (.-buffer u8))
|
||||
"uint8" u8
|
||||
(throw (ex-info "a block's element type is \"int16\", \"float32\", \"float64\" or \"uint8\""
|
||||
{:type type})))))
|
||||
149
frontend/src/arthur/domain/zip.cljs
Normal file
149
frontend/src/arthur/domain/zip.cljs
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
(ns arthur.domain.zip
|
||||
"A ZIP with STORED entries, for shipping a frame sequence and its audio as one
|
||||
file.
|
||||
|
||||
STORED — compression method 0, the bytes verbatim — because every entry going
|
||||
into it is already deflated (a PNG's IDAT) or is PCM that the user is about to
|
||||
hand a codec (the WAV). Deflating a deflated stream buys nothing and costs a
|
||||
pass over every byte of a long export, and it is also what lets this namespace
|
||||
be about the container alone: an archive with no compressor in it is a handful
|
||||
of little-endian headers, and there is no dependency to vendor.
|
||||
|
||||
WHY A ZIP AND NOT A DIRECTORY. `showDirectoryPicker` would write the numbered
|
||||
sequence straight to disk, which is closer to what an NLE wants, and it does not
|
||||
exist in Firefox — which is a browser this tool is already known to behave
|
||||
differently in (see `flow/ingest` on seeking). One archive downloads the same
|
||||
way everywhere and pairs the picture with the sound it has to stay in sync with,
|
||||
so the two cannot be separated on the way to the cutting room.
|
||||
|
||||
NO ZIP64. The offsets and sizes here are 32-bit, so this refuses an archive at
|
||||
4GB rather than writing one whose central directory silently wraps. A 900-frame
|
||||
export at 1920x1200 is tens of megabytes, so the limit is not in the way; a
|
||||
limit that corrupts instead of refusing would be."
|
||||
(:require [arthur.domain.crc32 :as crc32]))
|
||||
|
||||
(def ^:private limit
|
||||
"The largest archive this writer will produce. Past it the format needs Zip64
|
||||
and every offset below would have to be 64-bit."
|
||||
0xffffffff)
|
||||
|
||||
(defn- bytes-of [^String s]
|
||||
;; ASCII by construction — `0001.png`, `audio.wav` — and asserted rather than
|
||||
;; assumed, because a non-ASCII name would need the UTF-8 general-purpose flag
|
||||
;; and would otherwise arrive mojibake'd in the archive.
|
||||
(let [out (js/Uint8Array. (.-length s))]
|
||||
(dotimes [i (.-length s)]
|
||||
(let [c (.charCodeAt s i)]
|
||||
(when (> c 127)
|
||||
(throw (ex-info "a zip entry name must be ASCII" {:name s})))
|
||||
(aset out i c)))
|
||||
out))
|
||||
|
||||
(defn- u16! [^js b at n]
|
||||
(aset b at (bit-and n 0xff))
|
||||
(aset b (+ at 1) (bit-and (unsigned-bit-shift-right n 8) 0xff)))
|
||||
|
||||
(defn- u32! [^js b at n]
|
||||
(u16! b at (bit-and n 0xffff))
|
||||
(u16! b (+ at 2) (unsigned-bit-shift-right n 16)))
|
||||
|
||||
(defn dos-time
|
||||
"A `js/Date` as the two 16-bit fields ZIP inherited from MS-DOS: [date time].
|
||||
|
||||
Seconds have one bit less than they need, so they land on even values, and the
|
||||
year is an offset from 1980. Both are the format's, not an approximation — a
|
||||
date before 1980 is not representable and is clamped rather than wrapped into a
|
||||
plausible-looking wrong one."
|
||||
[^js d]
|
||||
[(bit-or (bit-shift-left (max 0 (- (.getFullYear d) 1980)) 9)
|
||||
(bit-shift-left (inc (.getMonth d)) 5)
|
||||
(.getDate d))
|
||||
(bit-or (bit-shift-left (.getHours d) 11)
|
||||
(bit-shift-left (.getMinutes d) 5)
|
||||
(quot (.getSeconds d) 2))])
|
||||
|
||||
(defn archive
|
||||
"Entries -> the parts of one ZIP, ready for a `js/Blob`.
|
||||
|
||||
Each entry is `{:name \"0001.png\" :data <Uint8Array>}`. Returns a vector of
|
||||
byte arrays rather than one buffer: the payloads are already in memory and a
|
||||
long export is tens of megabytes, so the archive REFERS to them instead of
|
||||
copying every one into a second buffer of the same size. `js/Blob` takes the
|
||||
parts as they are.
|
||||
|
||||
`at` is the modification time stamped on every entry. Passed in rather than read
|
||||
from the clock so that the same frames produce the same archive, byte for byte,
|
||||
which is what makes it assertable."
|
||||
([entries] (archive entries (js/Date.)))
|
||||
([entries ^js at]
|
||||
(let [[date time] (dos-time at)
|
||||
;; One pass, because a central directory entry needs the local header's
|
||||
;; OFFSET and therefore the running total, and the CRC is wanted in both
|
||||
;; places. Building the two lists separately would mean either computing
|
||||
;; every CRC twice or keeping a parallel vector of them.
|
||||
{:keys [parts central offset]}
|
||||
(reduce
|
||||
(fn [{:keys [parts central offset]} {:keys [name data]}]
|
||||
(let [nm (bytes-of name)
|
||||
n (.-length nm)
|
||||
size (.-length ^js data)
|
||||
crc (crc32/of data)
|
||||
local (js/Uint8Array. (+ 30 n))
|
||||
dir (js/Uint8Array. (+ 46 n))]
|
||||
(u32! local 0 0x04034b50) ; local file header
|
||||
(u16! local 4 10) ; version needed: 1.0 is enough to store
|
||||
(u16! local 6 0) ; no flags; the name is ASCII
|
||||
(u16! local 8 0) ; method 0: stored
|
||||
(u16! local 10 time)
|
||||
(u16! local 12 date)
|
||||
(u32! local 14 crc)
|
||||
(u32! local 18 size) ; compressed size — the same, stored
|
||||
(u32! local 22 size)
|
||||
(u16! local 26 n)
|
||||
(u16! local 28 0) ; no extra field
|
||||
(.set local nm 30)
|
||||
|
||||
(u32! dir 0 0x02014b50) ; central directory header
|
||||
(u16! dir 4 10) ; made by
|
||||
(u16! dir 6 10) ; version needed
|
||||
(u16! dir 8 0)
|
||||
(u16! dir 10 0)
|
||||
(u16! dir 12 time)
|
||||
(u16! dir 14 date)
|
||||
(u32! dir 16 crc)
|
||||
(u32! dir 20 size)
|
||||
(u32! dir 24 size)
|
||||
(u16! dir 28 n)
|
||||
(u16! dir 30 0) ; extra
|
||||
(u16! dir 32 0) ; comment
|
||||
(u16! dir 34 0) ; disk number
|
||||
(u16! dir 36 0) ; internal attributes
|
||||
(u32! dir 38 0) ; external attributes
|
||||
(u32! dir 42 offset)
|
||||
(.set dir nm 46)
|
||||
|
||||
(when (> (+ offset (.-length local) size) limit)
|
||||
(throw (ex-info "this export is too big for a zip without Zip64"
|
||||
{:bytes (+ offset (.-length local) size)})))
|
||||
{:parts (conj parts local data)
|
||||
:central (conj central dir)
|
||||
:offset (+ offset (.-length local) size)}))
|
||||
{:parts [] :central [] :offset 0}
|
||||
entries)
|
||||
dir-size (transduce (map #(.-length ^js %)) + 0 central)
|
||||
end (js/Uint8Array. 22)]
|
||||
(u32! end 0 0x06054b50) ; end of central directory
|
||||
(u16! end 4 0) ; this disk
|
||||
(u16! end 6 0) ; the disk the directory starts on
|
||||
(u16! end 8 (count central))
|
||||
(u16! end 10 (count central))
|
||||
(u32! end 12 dir-size)
|
||||
(u32! end 16 offset)
|
||||
(u16! end 20 0) ; no archive comment
|
||||
(-> (into parts central) (conj end) vec))))
|
||||
|
||||
(defn blob
|
||||
"The archive as one `js/Blob`, which is what a download wants."
|
||||
([entries] (blob entries (js/Date.)))
|
||||
([entries at]
|
||||
(js/Blob. (into-array (archive entries at)) #js {:type "application/zip"})))
|
||||
216
frontend/src/arthur/events/export.cljs
Normal file
216
frontend/src/arthur/events/export.cljs
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
(ns arthur.events.export
|
||||
"Export, as intents and one effect.
|
||||
|
||||
The walk is not an event and must not become one: it is a promise chain that
|
||||
runs for as long as the timeline is long, and re-frame events are the wrong unit
|
||||
for something with a middle. So `::start` collects what the render needs out of
|
||||
the db and hands it to an fx, and the fx dispatches progress back — the same
|
||||
arrangement `events/project`'s save uses, and for the same reason.
|
||||
|
||||
WHAT GOES IN THE DB IS THE REQUEST AND THE PROGRESS, never the frames. A
|
||||
megabyte of PNG in app-db would be compared by every mounted subscription on
|
||||
every tick."
|
||||
(:require [arthur.domain.palette :as pal]
|
||||
[arthur.export :as export]
|
||||
[arthur.export.frames :as frames]
|
||||
[arthur.footage.store :as store]
|
||||
[clojure.string :as str]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
(def zooms
|
||||
"The integer zooms offered. 320x200 times these is 320x200 up to 1920x1200.
|
||||
|
||||
INTEGERS ONLY, and the list is short for that reason rather than for tidiness:
|
||||
a stage at a non-integer scale has to invent pixels, and there is nothing in
|
||||
this tool downstream of `domain/raster` that is allowed to. 6 is here because
|
||||
1920 wide is what a delivery timeline usually is; the 1200 height that comes
|
||||
with it is 16:10 and is the project's aspect, not a mistake to letterbox away."
|
||||
[1 2 3 4 6])
|
||||
|
||||
(defn- stem
|
||||
"A filesystem-safe name for the artefact: the clip's label and the target's.
|
||||
|
||||
The target is in the name because the clip, each symbol in its library and each
|
||||
placement on its stage are all exportable, and they would otherwise land in the
|
||||
downloads folder as the same file. It is the target's LABEL rather than its id
|
||||
because a placement's id is a uuid, and `arthur-8f594d72-a97f-....zip` names
|
||||
nothing to the person who has to find it again."
|
||||
[label target]
|
||||
(-> (str (or label "arthur") "-" (or target "main"))
|
||||
(str/replace #"[^A-Za-z0-9._-]+" "-")
|
||||
(str/replace #"^-+|-+$" "")
|
||||
(str/lower-case)))
|
||||
|
||||
(defn target-value
|
||||
"An export target as a `<select>` option value.
|
||||
|
||||
Two kinds, told apart by a leading letter: `t:<timeline>` is a whole timeline,
|
||||
`n:<timeline>:<node>` is one placement inside one. The parts are joined with `:`
|
||||
because neither a timeline id nor a uuid contains one.
|
||||
|
||||
IT CARRIES THE NAMESPACE. `(name :sym/face-8625)` is \"face-8625\", and a value
|
||||
written that way cannot be read back: `keyword` on it gives `:face-8625`, which
|
||||
is not a key in `:timelines`, so the plan silently becomes nil and the export
|
||||
throws \"there is no such timeline\" from inside re-frame's `:do-fx`. That
|
||||
presented as the tab locking up rather than as an error — see `::run!` below for
|
||||
the other half of why — and it is the reason this is a named pair of functions
|
||||
with a test rather than `name` and `keyword` at the two ends of a select."
|
||||
[{:keys [timeline isolate]}]
|
||||
(let [tl (subs (str (or timeline :main)) 1)]
|
||||
(if isolate (str "n:" tl ":" isolate) (str "t:" tl))))
|
||||
|
||||
(defn target-id
|
||||
"The inverse of `target-value`. `keyword` splits on the `/` itself, so a
|
||||
namespaced timeline id survives; a placement comes back a uuid, which is what
|
||||
the node map is keyed by."
|
||||
[v]
|
||||
(let [[kind tl node] (str/split v #":")]
|
||||
(cond-> {:timeline (keyword tl)}
|
||||
(= "n" kind) (assoc :isolate (uuid node)))))
|
||||
|
||||
(defn targets
|
||||
"Everything an export can be pointed at, in the order the picker lists them.
|
||||
|
||||
THREE KINDS, and the distinction is the point. `:main` is the clip. A symbol
|
||||
timeline is the DRAWING — one file however many times it is placed, in its own
|
||||
frame space. A placement is that drawing WHERE IT SITS: the stage's length and
|
||||
rate, with the other placements removed, which is why seven instances of one
|
||||
symbol are seven different exports rather than seven copies of one.
|
||||
|
||||
Placements are ordered and labelled by `:name`, never by id: a uuid sorts at
|
||||
random and means nothing to read."
|
||||
[clip]
|
||||
(let [libs (cons :main (sort-by str (remove #{:main} (keys (:timelines clip)))))
|
||||
placements (->> (get-in clip [:timelines :main :nodes])
|
||||
(filter (comp #{:symbol} :kind val))
|
||||
(sort-by (fn [[id n]] [(or (:name n) "") (str id)])))]
|
||||
(into (mapv (fn [tid]
|
||||
{:timeline tid
|
||||
:label (if (= :main tid) "main (the clip)" (name tid))})
|
||||
libs)
|
||||
(mapv (fn [[id n]]
|
||||
{:timeline :main :isolate id
|
||||
:label (or (:name n) (str id))})
|
||||
placements))))
|
||||
|
||||
(defn- label-of
|
||||
"The label of the target `db` currently points at, for the filename."
|
||||
[clip {:keys [timeline isolate]}]
|
||||
(:label (or (first (filter #(and (= timeline (:timeline %))
|
||||
(= isolate (:isolate %)))
|
||||
(targets clip)))
|
||||
{:label (some-> timeline name)})))
|
||||
|
||||
(rf/reg-sub ::state (fn [db _] (:export db)))
|
||||
|
||||
(rf/reg-sub
|
||||
::targets
|
||||
(fn [db _]
|
||||
(targets (:clip (store/entry (:clip/current db))))))
|
||||
|
||||
(rf/reg-sub
|
||||
::plan
|
||||
(fn [db _]
|
||||
(let [{:keys [clip]} (store/entry (:clip/current db))
|
||||
{:keys [timeline zoom isolate]} (:export db)]
|
||||
(export/plan {:clip clip :timeline timeline :zoom zoom :isolate isolate
|
||||
:picture-fps (get-in db [:clip :display-fps])}))))
|
||||
|
||||
(rf/reg-event-db
|
||||
::set-target
|
||||
;; Both keys always, so switching from a placement back to a whole timeline
|
||||
;; clears the isolate rather than leaving it to filter the new target.
|
||||
(fn [db [_ {:keys [timeline isolate]}]]
|
||||
(update db :export merge {:timeline (or timeline :main) :isolate isolate})))
|
||||
|
||||
(rf/reg-event-db
|
||||
::set-zoom
|
||||
(fn [db [_ z]] (assoc-in db [:export :zoom] z)))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::start
|
||||
(fn [{:keys [db]} _]
|
||||
(if (get-in db [:export :busy?])
|
||||
{}
|
||||
(let [id (:clip/current db)
|
||||
entry (store/entry id)
|
||||
{:keys [timeline zoom isolate]} (:export db)]
|
||||
{:db (update db :export merge {:busy? true :done 0
|
||||
:total (:frames (export/plan
|
||||
{:clip (:clip entry)
|
||||
:timeline timeline
|
||||
:isolate isolate
|
||||
:zoom zoom}))
|
||||
:status "rendering…"})
|
||||
::run! {:clip (:clip entry)
|
||||
:timeline timeline
|
||||
:isolate isolate
|
||||
:store (:store entry)
|
||||
;; The same palette and ramp the preview resolves and blits
|
||||
;; through. Read here rather than in the fx so that the effect
|
||||
;; takes data and nothing else.
|
||||
:palette (get {:arthur/default pal/index-of}
|
||||
(:palette db) pal/index-of)
|
||||
:ramp (get {:arthur/default pal/rgb} (:palette db) pal/rgb)
|
||||
:zoom zoom
|
||||
:picture-fps (get-in db [:clip :display-fps])
|
||||
:audio-url (:audio entry)
|
||||
:name (stem (:label entry)
|
||||
(label-of (:clip entry)
|
||||
{:timeline timeline :isolate isolate}))}}))))
|
||||
|
||||
(rf/reg-event-db
|
||||
::progress
|
||||
(fn [db [_ done total]]
|
||||
(update db :export merge {:done done :total total})))
|
||||
|
||||
(rf/reg-event-db
|
||||
::done
|
||||
(fn [db [_ filename bytes]]
|
||||
(update db :export merge
|
||||
{:busy? false
|
||||
:status (str "wrote " filename " · "
|
||||
(.toFixed (/ bytes 1048576) 1) " MB")})))
|
||||
|
||||
(rf/reg-event-db
|
||||
::failed
|
||||
(fn [db [_ message]]
|
||||
(update db :export merge {:busy? false :status (str "export failed: " message)})))
|
||||
|
||||
(defn- download!
|
||||
"Hand the browser a blob as a file.
|
||||
|
||||
The object URL is revoked on a timeout rather than immediately: the click starts
|
||||
the download asynchronously and revoking in the same turn cancels it in some
|
||||
browsers, which presents as the button doing nothing at all."
|
||||
[filename ^js blob]
|
||||
(let [url (js/URL.createObjectURL blob)
|
||||
a (.createElement js/document "a")]
|
||||
(set! (.-href a) url)
|
||||
(set! (.-download a) filename)
|
||||
(.appendChild (.-body js/document) a)
|
||||
(.click a)
|
||||
(.removeChild (.-body js/document) a)
|
||||
(js/setTimeout #(js/URL.revokeObjectURL url) 30000)))
|
||||
|
||||
(rf/reg-fx
|
||||
::run!
|
||||
(fn [spec]
|
||||
;; THE CALL IS GUARDED because `export/run!` validates its request BEFORE it
|
||||
;; returns a promise, so a bad timeline id throws synchronously — here, inside
|
||||
;; re-frame's `:do-fx` interceptor. An uncaught throw there never reaches the
|
||||
;; `.catch` below, so `::failed` never dispatches and `:busy?` stays true: the
|
||||
;; button sits disabled on \"rendering…\" and the readout on \"frame 0 /\"
|
||||
;; forever. That reads as the tab having locked up, which is the worst way for
|
||||
;; an export to fail — there is nothing to see and nothing in the status line.
|
||||
;; Turning the throw into a rejection gives every failure one path to the user.
|
||||
(-> (try (export/run! spec
|
||||
(frames/exporter)
|
||||
(fn [done total] (rf/dispatch [::progress done total])))
|
||||
(catch :default e (js/Promise.reject e)))
|
||||
(.then (fn [{:keys [filename ^js blob]}]
|
||||
(download! filename blob)
|
||||
(rf/dispatch [::done filename (.-size blob)])))
|
||||
(.catch (fn [error]
|
||||
(js/console.error error)
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))]))))))
|
||||
394
frontend/src/arthur/events/footage.cljs
Normal file
394
frontend/src/arthur/events/footage.cljs
Normal file
|
|
@ -0,0 +1,394 @@
|
|||
(ns arthur.events.footage
|
||||
"Load and freeze ingested footage once, outside the playback loop.
|
||||
|
||||
The frames come from the server by URL since step 9 — see `flow/ingest` — and the
|
||||
detector's identity comes from the server too, because it goes into the content
|
||||
address of every block this produces."
|
||||
(:require [arthur.domain.clip :as clip]
|
||||
[arthur.events.playback :as pb]
|
||||
[arthur.flow.detect :as detect]
|
||||
[arthur.flow.ingest :as ingest]
|
||||
[arthur.flow.measure.interior :as interior]
|
||||
[arthur.flow.source :as source]
|
||||
[arthur.flow.take :as take]
|
||||
[arthur.footage.store :as store]
|
||||
[arthur.domain.landmarks :as lm]
|
||||
[arthur.fx.http :as http]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
(defonce ^:private clock (atom 0))
|
||||
|
||||
(defn- mark!
|
||||
"Where the time goes, on the console, one line per stage.
|
||||
|
||||
Kept rather than removed after it earned its keep. Progress only paints every
|
||||
fourth frame, so a stall near the end looks identical whether the decoder has
|
||||
stopped delivering or the work after it is holding the main thread — and those
|
||||
two were confused for each other three times before this printed the answer:
|
||||
decoding was finished, and `measure-crops` was ten seconds of synchronous
|
||||
arithmetic with nothing able to repaint."
|
||||
[label]
|
||||
(let [now (js/Date.now)
|
||||
since (- now @clock)]
|
||||
(reset! clock now)
|
||||
(js/console.log (str "arthur ⏱ " label " +" since "ms"))))
|
||||
|
||||
(defn- detect-frames!
|
||||
"Decode the proxy once, in order, and measure every frame as it goes.
|
||||
|
||||
ONCE AND FORWARD IS A REQUIREMENT, NOT A STYLE. MediaPipe's video mode is a
|
||||
tracker whose input stream refuses a timestamp that does not advance, and the
|
||||
error it raises is terminal for the landmarker — so there is no re-reading a
|
||||
frame and no second pass. That used to be a constraint the frame walk had to be
|
||||
careful about; with a decoder it is simply what decoding is.
|
||||
|
||||
The work happens inside `decode!`'s callback, and the promise it returns is the
|
||||
backpressure: the decoder does not run ahead of the detector, so a 900-frame
|
||||
take does not hold 900 decoded frames at 1440x1920 in memory."
|
||||
[manifest model]
|
||||
(let [[w h] [(:width manifest) (:height manifest)]
|
||||
canvas (.createElement js/document "canvas")
|
||||
ctx (.getContext canvas "2d" #js {:willReadFrequently true})
|
||||
fps (:fps manifest)
|
||||
raw (atom [])
|
||||
crops (atom [])
|
||||
inner (atom [])
|
||||
total (:frames manifest)]
|
||||
(set! (.-width canvas) w)
|
||||
(set! (.-height canvas) h)
|
||||
(rf/dispatch [::progress "loading the video…"])
|
||||
(-> (ingest/stream! (ingest/stream-url manifest) total)
|
||||
(.then
|
||||
(fn [stream]
|
||||
(ingest/decode!
|
||||
stream fps w h
|
||||
(fn [i frame]
|
||||
(.drawImage ctx frame 0 0)
|
||||
;; EVERY FACE ON THIS FRAME, each with its own mouth crop taken
|
||||
;; while the frame's pixels are still on the canvas. Which of these
|
||||
;; detections belongs to which subject is not decided here — the
|
||||
;; answer needs the whole take — so all three vectors stay in
|
||||
;; DETECTION ORDER and `detect/tracks` re-keys them afterwards.
|
||||
(let [faces (detect/detect! model canvas (ingest/frame-ms fps i))
|
||||
boxes (mapv (fn [face]
|
||||
(interior/crop (mapv #(nth face %) lm/LIPS-INNER)
|
||||
[w h]))
|
||||
faces)
|
||||
frame-crops (mapv (fn [box]
|
||||
(when box
|
||||
{:box box
|
||||
:data (.-data (.getImageData
|
||||
ctx (:x box) (:y box)
|
||||
(:w box) (:h box)))}))
|
||||
boxes)]
|
||||
(swap! raw conj faces)
|
||||
(swap! crops conj frame-crops)
|
||||
;; MEASURED HERE, NOT IN A SECOND PASS. It is the same work
|
||||
;; either way, but done after the fact it is ten seconds of
|
||||
;; synchronous arithmetic with the main thread held and the
|
||||
;; frame counter frozen on its last value — which reads as the
|
||||
;; decoder hanging, and was diagnosed as that twice.
|
||||
(swap! inner conj (mapv #(source/measure-crop take/knobs %)
|
||||
frame-crops)))
|
||||
(when (or (zero? i) (zero? (mod (inc i) 4)) (= (inc i) total))
|
||||
(rf/dispatch [::progress (str "detecting " (inc i) "/" total)]))
|
||||
;; Yield, so the status and the transport paint between synchronous
|
||||
;; MediaPipe calls. `decode!` waits on this before feeding more.
|
||||
(js/Promise. (fn [done] (js/setTimeout done 0)))))))
|
||||
(.then (fn [_]
|
||||
(mark! (str "DECODE FINISHED — " (count @raw) " frames"))
|
||||
(let [slots (detect/tracks @raw)
|
||||
subjects (into {}
|
||||
(map (fn [[id track]]
|
||||
[id (assoc (detect/fill-gaps
|
||||
(detect/pick @raw track))
|
||||
:crops (detect/pick @crops track)
|
||||
:interior (detect/pick @inner track)
|
||||
:interior-settings take/knobs)]))
|
||||
slots)]
|
||||
(mark! "fill-gaps")
|
||||
{:subjects subjects
|
||||
:dimensions [w h]
|
||||
;; Frames where NOBODY was found, which is a fact about the
|
||||
;; footage. A frame one of two faces is missing from is a gap
|
||||
;; in that subject's own detection mask and is reported there.
|
||||
:missing (count (remove seq @raw))
|
||||
:first-real (first (keep-indexed (fn [i faces] (when (seq faces) i))
|
||||
@raw))}))))))
|
||||
|
||||
(defn presence-for
|
||||
"Select a subject's masks and express them in its local feature names.
|
||||
Unqualified manifest ids refer to the first face, as in single-face footage."
|
||||
[manifest subject]
|
||||
(not-empty
|
||||
(into {} (keep (fn [[id mask]]
|
||||
(when (= (or (namespace id) "face-1") (subs (str subject) 1))
|
||||
[(keyword (name id)) mask])))
|
||||
(:presence manifest))))
|
||||
|
||||
(defn- build-clip [manifest detector
|
||||
{:keys [subjects dimensions missing first-real] :as source-inputs}]
|
||||
(let [[w h] dimensions
|
||||
_ (mark! "build-clip: start")
|
||||
with-presence (into {}
|
||||
(map (fn [[id inputs]]
|
||||
[id (assoc inputs :presence (presence-for manifest id))]))
|
||||
subjects)
|
||||
frozen (take/footage manifest {:dimensions dimensions
|
||||
:subjects with-presence
|
||||
:detector detector})
|
||||
_ (mark! "build-clip: freeze")
|
||||
built (:clip frozen)
|
||||
source-blocks (source/pack-subjects (:id (:analysis built)) subjects)
|
||||
_ (mark! "build-clip: pack source blocks")]
|
||||
(assoc (select-keys built [:fps :width :height])
|
||||
:frames (clip/frames built)
|
||||
:display-fps (:fps built)
|
||||
:clip built :store (:store frozen)
|
||||
:source-blocks source-blocks
|
||||
:source-inputs (assoc source-inputs :subjects with-presence)
|
||||
;; No cache-buster. The audio is a blob named by the hash of its own
|
||||
;; bytes, so re-extracting gives it a different URL rather than
|
||||
;; overwriting this one — which is what the `?v=` here used to work
|
||||
;; around.
|
||||
:audio (ingest/audio-url manifest)
|
||||
:label (or (:label manifest) (:source manifest) "footage")
|
||||
:footage-id (:id manifest)
|
||||
:cid (or (:id manifest) "footage")
|
||||
:summary (str (:frames manifest) " frames · " w "×" h " · "
|
||||
(:fps manifest) " fps"
|
||||
(when (> (count subjects) 1)
|
||||
(str " · " (count subjects) " faces"))
|
||||
(when (pos? missing)
|
||||
(str " · " missing " without a face"
|
||||
(when (pos? first-real)
|
||||
(str " (first found on " (inc first-real) ")"))))))))
|
||||
|
||||
(defn- with-default-interior!
|
||||
"Backfill one subject's retained interior measurements at the default knobs."
|
||||
[analysis subject track]
|
||||
(let [frames (count (:crops track))
|
||||
key (source/interior-key analysis subject take/knobs frames)]
|
||||
(-> (http/GET (str "/api/blocks/" key))
|
||||
(.then (fn [block]
|
||||
(assoc track :interior (source/unpack-interior block take/knobs frames)
|
||||
:interior-settings take/knobs
|
||||
:interior-key key)))
|
||||
(.catch (fn [error]
|
||||
(if (= 404 (:status (ex-data error))) track (throw error)))))))
|
||||
|
||||
(defn saved-source!
|
||||
"Restore retained source tracks by analysis id, for regeneration after open.
|
||||
|
||||
An analysis holds one set of blocks per tracked subject, so the count is a
|
||||
multiple of `source/roles` rather than equal to it; `source/unpack` reads each
|
||||
block's own descriptor to find out whose it is."
|
||||
[key dimensions]
|
||||
(-> (http/GET (str "/api/analyses/" key))
|
||||
(.then (fn [^js analysis]
|
||||
(let [keys (array-seq (.-source_blocks analysis))]
|
||||
(when (and (seq keys) (zero? (mod (count keys) (count source/roles))))
|
||||
(-> (js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %)) keys)))
|
||||
(.then (fn [blocks] (source/unpack blocks dimensions)))
|
||||
(.then (fn [{:keys [subjects]}]
|
||||
(-> (js/Promise.all
|
||||
(into-array
|
||||
(map (fn [[id track]]
|
||||
(.then (with-default-interior! key id track)
|
||||
(fn [filled] [id filled])))
|
||||
subjects)))
|
||||
(.then (fn [pairs]
|
||||
{:subjects (into {} (array-seq pairs))
|
||||
:dimensions dimensions}))))))))))
|
||||
(.catch (fn [error]
|
||||
(if (= 404 (:status (ex-data error))) nil (throw error))))))
|
||||
|
||||
(defn- cached-source! [manifest detector]
|
||||
(saved-source! (:id (take/analysis-for manifest detector))
|
||||
[(:width manifest) (:height manifest)]))
|
||||
|
||||
(defn measure-one!
|
||||
"One subject's crops, ONE PER EVENT-LOOP TURN."
|
||||
[settings track on-step]
|
||||
(if (or (:interior track) (not (:crops track)))
|
||||
(js/Promise.resolve track)
|
||||
(let [crops (:crops track)
|
||||
total (count crops)
|
||||
interior (atom [])]
|
||||
(js/Promise.
|
||||
(fn [resolve reject]
|
||||
(letfn [(step [i]
|
||||
(if (= i total)
|
||||
(resolve (assoc track :interior @interior
|
||||
:interior-settings settings))
|
||||
(try
|
||||
(swap! interior conj (source/measure-crop settings (nth crops i)))
|
||||
(when on-step (on-step (inc i) total))
|
||||
(js/setTimeout #(step (inc i)) 0)
|
||||
(catch :default error (reject error)))))]
|
||||
(step 0)))))))
|
||||
|
||||
(defn measure-crops!
|
||||
"Measure every retained crop's interior, one subject after another and one
|
||||
crop per event-loop turn.
|
||||
|
||||
Done in a tight loop instead, it is ten seconds of synchronous arithmetic with
|
||||
the main thread held and the frame counter frozen on its last value — which
|
||||
reads as the decoder hanging, and was diagnosed as that twice.
|
||||
|
||||
Two callers, and the only difference between them is who is waiting: opening an
|
||||
older analysis that predates the interior block backfills it behind a progress
|
||||
line, and a knob drag that needs the pixels measured at new settings does the
|
||||
same work with nobody watching, so it passes no `on-step`. The settings ride
|
||||
back on each track, because a measurement and the knobs it was taken at are one
|
||||
fact — `source/pack` will not address an interior block without them."
|
||||
[settings track on-step]
|
||||
(reduce (fn [chain [id one]]
|
||||
(.then chain
|
||||
(fn [acc]
|
||||
(.then (measure-one! settings one on-step)
|
||||
(fn [measured] (assoc-in acc [:subjects id] measured))))))
|
||||
(js/Promise.resolve track)
|
||||
(:subjects track)))
|
||||
|
||||
(rf/reg-fx
|
||||
::begin!
|
||||
(fn [footage-id]
|
||||
(-> (js/Promise.all #js [(ingest/manifest! footage-id) (ingest/detector!)])
|
||||
(.then (fn [[manifest detector]]
|
||||
(reset! clock (js/Date.now))
|
||||
(rf/dispatch [::progress "looking for saved analysis…"])
|
||||
(-> (cached-source! manifest detector)
|
||||
(.then (fn [track]
|
||||
(if track
|
||||
(do (rf/dispatch [::progress "reusing saved analysis…"])
|
||||
(-> (measure-crops!
|
||||
take/knobs track
|
||||
(fn [done total]
|
||||
(when (or (= 1 done) (zero? (mod done 4))
|
||||
(= done total))
|
||||
(rf/dispatch
|
||||
[::progress (str "measuring " done "/" total)]))))
|
||||
(.then (fn [measured]
|
||||
(build-clip manifest detector measured)))))
|
||||
(do (rf/dispatch [::progress "loading MediaPipe…"])
|
||||
(-> (detect/landmarker!)
|
||||
(.then (fn [model]
|
||||
(mark! "MediaPipe ready")
|
||||
(rf/dispatch [::progress "opening the video…"])
|
||||
(-> (detect-frames! manifest model)
|
||||
(.then (fn [fresh]
|
||||
(build-clip manifest detector fresh))))))))))))))
|
||||
(.then (fn [entry]
|
||||
(mark! "build-clip: done")
|
||||
(let [id (store/install! entry)]
|
||||
(rf/dispatch [::loaded id (:summary entry)]))))
|
||||
(.catch (fn [error]
|
||||
(js/console.error error)
|
||||
;; A run that ended badly may have ended on a MediaPipe graph
|
||||
;; error, and a landmarker that has hit one throws the same error
|
||||
;; for the rest of the page's life. Retrying has to get a new one.
|
||||
(detect/discard!)
|
||||
(rf/dispatch [::failed (or (ex-message error) (.-message error) (str error))]))))))
|
||||
|
||||
(rf/reg-fx
|
||||
::list!
|
||||
(fn [_]
|
||||
(-> (ingest/available!)
|
||||
(.then (fn [footage] (rf/dispatch [::listed footage])))
|
||||
(.catch (fn [error]
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))]))))))
|
||||
|
||||
(defn- poll-extraction! [key]
|
||||
(-> (http/GET (str "/api/extractions/" key))
|
||||
(.then (fn [^js job]
|
||||
(case (.-state job)
|
||||
"done" (rf/dispatch [::uploaded (.-footage job)])
|
||||
"failed" (rf/dispatch [::failed (.-error job)])
|
||||
(do (rf/dispatch [::progress
|
||||
(str "extracting " (.-progress job) "%")])
|
||||
(js/setTimeout #(poll-extraction! key) 800)))))
|
||||
(.catch (fn [error]
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))])))))
|
||||
|
||||
(rf/reg-fx
|
||||
::upload!
|
||||
(fn [file]
|
||||
(let [form (js/FormData.)]
|
||||
(.append form "file" file)
|
||||
(-> (http/POST-form "/api/sources" form)
|
||||
(.then (fn [^js source]
|
||||
(rf/dispatch [::progress "queued for extraction…"])
|
||||
(http/POST "/api/extractions" #js {:source (.-id source)
|
||||
:settings #js {}})))
|
||||
(.then (fn [^js job] (poll-extraction! (.-key job))))
|
||||
(.catch (fn [error]
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))])))))))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::upload
|
||||
(fn [{:keys [db]} [_ file]]
|
||||
(if (or (nil? file) (get-in db [:footage :loading?]))
|
||||
{}
|
||||
{:db (update db :footage merge {:loading? true :status "uploading video…"})
|
||||
::upload! file})))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::uploaded
|
||||
(fn [{:keys [db]} [_ footage-id]]
|
||||
{:db (update db :footage merge {:loading? false :chosen footage-id
|
||||
:status "video extracted — load frames to analyze"})
|
||||
:dispatch [::refresh]}))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::refresh
|
||||
(fn [_ _] {::list! nil}))
|
||||
|
||||
(rf/reg-event-db
|
||||
::listed
|
||||
(fn [db [_ footage]]
|
||||
(update db :footage merge
|
||||
(cond-> {:available (vec footage)
|
||||
:chosen (or (:chosen (:footage db)) (:id (first footage)))}
|
||||
(empty? footage) (assoc :status "upload a video to begin")))))
|
||||
|
||||
(rf/reg-event-db
|
||||
::choose
|
||||
(fn [db [_ id]] (assoc-in db [:footage :chosen] id)))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::load
|
||||
(fn [{:keys [db]} _]
|
||||
(let [chosen (get-in db [:footage :chosen])]
|
||||
(cond
|
||||
(get-in db [:footage :loading?]) {}
|
||||
(nil? chosen)
|
||||
{:db (assoc-in db [:footage :status] "upload a video to begin")}
|
||||
:else
|
||||
{:db (update db :footage merge {:loading? true :status "reading the manifest…"})
|
||||
::pb/pause! nil
|
||||
::begin! chosen}))))
|
||||
|
||||
(rf/reg-event-db
|
||||
::progress
|
||||
(fn [db [_ message]] (assoc-in db [:footage :status] message)))
|
||||
|
||||
(rf/reg-event-db
|
||||
::failed
|
||||
(fn [db [_ message]]
|
||||
(assoc db :footage (assoc (:footage db)
|
||||
:loading? false :status (str "footage failed: " message)))))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::loaded
|
||||
(fn [{:keys [db]} [_ id summary]]
|
||||
(let [clip (store/entry id)]
|
||||
{:db (-> db
|
||||
(assoc :clip/current id
|
||||
:clip (select-keys clip [:fps :frames :width :height :audio :display-fps])
|
||||
:footage (assoc (:footage db) :id id :label (:label clip)
|
||||
:loading? false :status summary))
|
||||
(assoc-in [:playback :frame] 0)
|
||||
(assoc-in [:playback :playing?] false))
|
||||
::pb/pause! nil})))
|
||||
33
frontend/src/arthur/events/paint.cljs
Normal file
33
frontend/src/arthur/events/paint.cljs
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
(ns arthur.events.paint
|
||||
(:require [arthur.domain.paint :as paint]
|
||||
[arthur.footage.store :as store]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
(defn- edit [db f]
|
||||
(let [id (store/edit-clip! (:clip/current db) f)]
|
||||
(if id
|
||||
(-> db
|
||||
(assoc :clip/current id)
|
||||
(update :paint/revision (fnil inc 0))
|
||||
(update :project merge {:status "paint edited · unsaved"}))
|
||||
db)))
|
||||
|
||||
(rf/reg-event-db
|
||||
::new-shape
|
||||
(fn [db [_ id points color]]
|
||||
(edit db #(paint/new-shape % id (get-in db [:playback :frame]) points color))))
|
||||
|
||||
(rf/reg-event-db
|
||||
::add-key
|
||||
(fn [db [_ id]]
|
||||
(edit db #(paint/add-key % id (get-in db [:playback :frame])))))
|
||||
|
||||
(rf/reg-event-db
|
||||
::set-vertex
|
||||
(fn [db [_ id key-frame vertex point]]
|
||||
(edit db #(paint/set-vertex % id key-frame vertex point))))
|
||||
|
||||
(rf/reg-event-db
|
||||
::set-segment-interp
|
||||
(fn [db [_ id key-frame interp]]
|
||||
(edit db #(paint/set-segment-interp % id key-frame interp))))
|
||||
112
frontend/src/arthur/events/playback.cljs
Normal file
112
frontend/src/arthur/events/playback.cljs
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
(ns arthur.events.playback
|
||||
"Transport events.
|
||||
|
||||
Named for intent rather than for the field they happen to set: `::toggle` is
|
||||
not `set-playing?`, because what the button means is \"start or stop\", and the
|
||||
resulting boolean is a consequence.
|
||||
|
||||
NO GLOBAL INTERCEPTORS ON ::tick. At 30fps a spec-validating `after` or
|
||||
`std-interceptors/debug`'s `clojure.data/diff` would be thirty full-db
|
||||
traversals a second, which is the one genuinely expensive thing you can do to
|
||||
a small app-db. If global interceptors are added later they are added to a
|
||||
chain these events are excluded from, not to `reg-global-interceptor`."
|
||||
(:require [arthur.clock :as clock]
|
||||
[arthur.footage.store :as footage]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
(defn- fps [db] (get-in db [:clip :fps]))
|
||||
(defn- frames [db] (get-in db [:clip :frames]))
|
||||
|
||||
(rf/reg-event-db
|
||||
::tick
|
||||
(fn [db [_ f]]
|
||||
;; Written from the rAF loop when the DERIVED frame changes — not every
|
||||
;; animation frame, and never as the thing the blit waits on. The picture is
|
||||
;; painted from the clock directly; this only brings the document's idea of
|
||||
;; the playhead up to date so the readout and the scrubber agree with it.
|
||||
(if (= f (get-in db [:playback :frame]))
|
||||
db
|
||||
(assoc-in db [:playback :frame] f))))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::play
|
||||
(fn [{:keys [db]} _]
|
||||
{:db (assoc-in db [:playback :playing?] true)
|
||||
::play! nil}))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::pause
|
||||
(fn [{:keys [db]} _]
|
||||
{:db (assoc-in db [:playback :playing?] false)
|
||||
::pause! nil}))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::toggle
|
||||
(fn [{:keys [db]} _]
|
||||
(if (get-in db [:playback :playing?])
|
||||
{:db (assoc-in db [:playback :playing?] false) ::pause! nil}
|
||||
{:db (assoc-in db [:playback :playing?] true) ::play! nil})))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::seek
|
||||
(fn [{:keys [db]} [_ f]]
|
||||
(let [f (-> f (max 0) (min (dec (frames db))))]
|
||||
{:db (assoc-in db [:playback :frame] f)
|
||||
::seek! [(fps db) (frames db) f]})))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::step
|
||||
(fn [{:keys [db]} [_ delta]]
|
||||
{:fx [[:dispatch [::seek (+ (get-in db [:playback :frame]) delta)]]]}))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::set-rate
|
||||
(fn [{:keys [db]} [_ r]]
|
||||
{:db (assoc-in db [:playback :rate] r)
|
||||
::rate! r}))
|
||||
|
||||
(rf/reg-event-db
|
||||
::set-picture-fps
|
||||
(fn [db [_ target]]
|
||||
(if (and (number? target) (pos? target) (<= target (fps db)))
|
||||
(assoc-in db [:clip :display-fps] target)
|
||||
db)))
|
||||
|
||||
;; --- effects: every DOM touch on the audio element is one of these ---
|
||||
|
||||
(rf/reg-fx ::play! (fn [_] (clock/play!)))
|
||||
(rf/reg-fx ::pause! (fn [_] (clock/pause!)))
|
||||
(rf/reg-fx ::rate! (fn [r] (clock/set-rate! r)))
|
||||
(rf/reg-fx ::seek! (fn [[fps frames f]] (clock/seek! fps frames f)))
|
||||
(rf/reg-fx ::loop! (fn [on?] (clock/set-loop! on?)))
|
||||
(rf/reg-fx ::mute! (fn [on?] (clock/set-muted! on?)))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::toggle-loop
|
||||
(fn [{:keys [db]} _]
|
||||
(let [on? (not (get-in db [:playback :loop?]))]
|
||||
{:db (assoc-in db [:playback :loop?] on?) ::loop! on?})))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::toggle-mute
|
||||
(fn [{:keys [db]} _]
|
||||
(let [on? (not (get-in db [:playback :muted?]))]
|
||||
{:db (assoc-in db [:playback :muted?] on?) ::mute! on?})))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::select-clip
|
||||
(fn [{:keys [db]} [_ id]]
|
||||
;; Changing the clip changes the resolver, the frame count and the rate all
|
||||
;; at once, so the playhead goes home rather than being left pointing at a
|
||||
;; frame the new clip may not have.
|
||||
(let [{:keys [fps frames] :as clip} (footage/entry id)]
|
||||
{:db (-> db
|
||||
(assoc :clip/current id)
|
||||
;; The stage travels with the clip: two clips may be different
|
||||
;; sizes, and the raster the loop paints into is the clip's, not
|
||||
;; the app's.
|
||||
(assoc :clip (select-keys clip [:fps :frames :width :height :audio :display-fps]))
|
||||
(assoc-in [:playback :frame] 0)
|
||||
(assoc-in [:playback :playing?] false))
|
||||
::pause! nil
|
||||
::seek! [fps frames 0]})))
|
||||
419
frontend/src/arthur/events/project.cljs
Normal file
419
frontend/src/arthur/events/project.cljs
Normal file
|
|
@ -0,0 +1,419 @@
|
|||
(ns arthur.events.project
|
||||
"Save and open: the document over HTTP.
|
||||
|
||||
THE ORDER OF A SAVE IS THE TIER SPLIT, and it is not an arrangement of
|
||||
convenience — each step is the precondition for the next one to be checkable:
|
||||
|
||||
1. the ANALYSIS record, so that every block stored afterwards can name the
|
||||
detector version that produced it. The server refuses a block whose
|
||||
analysis it does not know, for exactly that reason.
|
||||
2. ask which BLOCKS are missing, and upload only those. A re-save after a
|
||||
document edit moves kilobytes, which is the whole return on content
|
||||
addressing.
|
||||
3. the DOCUMENT. The server refuses a clip that names blocks it does not hold,
|
||||
so a saved document cannot load into a blank stage somewhere else.
|
||||
|
||||
Open is the same order backwards: the document, then the blocks it names. It
|
||||
needs no analysis step, because the document carries the analysis record — a
|
||||
content address alone would make a take unreadable the first time a detector
|
||||
upgrade orphaned one, and \"sha256:7f2…\" is not an answer to \"which model
|
||||
produced this\".
|
||||
|
||||
Nothing here touches app-db except through events. The promise chain lives in an
|
||||
fx, which is the only thing in this namespace that is not pure."
|
||||
(:require [arthur.domain.clip :as clip]
|
||||
[arthur.audio.mix :as mix]
|
||||
[arthur.demo.stage :as stage]
|
||||
[arthur.domain.feature :as feature]
|
||||
[arthur.domain.project :as project]
|
||||
[arthur.domain.wire :as wire]
|
||||
[arthur.events.footage :as footage]
|
||||
[arthur.events.playback :as pb]
|
||||
[arthur.footage.store :as store]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.ingest :as ingest]
|
||||
[arthur.flow.regenerate :as regenerate]
|
||||
[arthur.flow.source :as source]
|
||||
[arthur.flow.take :as take]
|
||||
[arthur.fx.http :as http]
|
||||
[arthur.synth :as synth]
|
||||
[re-frame.core :as rf]))
|
||||
|
||||
(defn- analysis-payload [analysis]
|
||||
#js {:key (:id analysis)
|
||||
:descriptor (address/analysis-descriptor analysis)
|
||||
:footage (:footage analysis)})
|
||||
|
||||
(defn- block-keys [^js doc]
|
||||
(into-array (map #(.-key %) (array-seq (.-blocks doc)))))
|
||||
|
||||
(defn- block-bytes [value]
|
||||
(if (string? value)
|
||||
(wire/bytes-of value)
|
||||
(js/Uint8Array. (.-buffer value) (.-byteOffset value) (.-byteLength value))))
|
||||
|
||||
(defn- block-form [^js block]
|
||||
(let [form (js/FormData.)]
|
||||
(.append form "key" (.-key block))
|
||||
(.append form "descriptor" (.-descriptor block))
|
||||
(.append form "data" (js/Blob. #js [(block-bytes (.-data block))]) "block.bin")
|
||||
(when-let [state (.-state block)]
|
||||
(.append form "state" (js/Blob. #js [(block-bytes state)]) "state.bin"))
|
||||
form))
|
||||
|
||||
(defn- upload-missing!
|
||||
"POST the blocks the server said it does not have, and nothing else.
|
||||
|
||||
ONE AT A TIME. `Promise.all` over eleven uploads is the obvious way to write
|
||||
this and it made sqlite answer \"database is locked\" on a save — which reaches
|
||||
the page as a 500 with nothing wrong with the request. The backend was fixed too
|
||||
(WAL, and a busy timeout, in server/settings.py), and this stays sequential
|
||||
anyway: most uploads are small, and a burst of parallel writes to buy nothing
|
||||
is how the same bug comes back the first time a take has sixty blocks instead
|
||||
of eleven."
|
||||
[^js doc]
|
||||
(-> (http/POST "/api/blocks/missing" #js {:keys (block-keys doc)})
|
||||
(.then (fn [^js answer]
|
||||
(let [missing (set (array-seq (.-missing answer)))
|
||||
todo (filterv #(contains? missing (.-key ^js %))
|
||||
(array-seq (.-blocks doc)))]
|
||||
(-> (reduce (fn [chain block]
|
||||
(.then chain
|
||||
(fn [_]
|
||||
(http/POST-form "/api/blocks" (block-form block)))))
|
||||
(js/Promise.resolve nil)
|
||||
todo)
|
||||
(.then (fn [_] (count todo)))))))))
|
||||
|
||||
(defn- ensure-project! [id name]
|
||||
(if id
|
||||
(js/Promise.resolve id)
|
||||
(-> (http/POST "/api/projects" #js {:name name})
|
||||
(.then (fn [^js created] (.-id created))))))
|
||||
|
||||
(defn- opened-entry! [^js clip-json]
|
||||
(let [footage-id (.-footage clip-json)]
|
||||
(-> (js/Promise.all
|
||||
#js [(js/Promise.all
|
||||
(into-array (map #(http/GET (str "/api/blocks/" %))
|
||||
(array-seq (.-blocks clip-json)))))
|
||||
(if footage-id
|
||||
(http/GET (str "/api/footage/" footage-id))
|
||||
(js/Promise.resolve nil))])
|
||||
(.then (fn [[blocks ^js footage]]
|
||||
(let [cid (.-cid clip-json)
|
||||
loaded (project/load
|
||||
cid #js {:leaves (.-leaves clip-json)
|
||||
:blocks blocks})
|
||||
built (:clip loaded)]
|
||||
(let [entry (merge (select-keys built [:fps :width :height])
|
||||
{:label (str (or (.-name clip-json) cid) " (saved)")
|
||||
:cid cid :frames (clip/frames built)
|
||||
:display-fps (:fps built)
|
||||
:clip built :store (:store loaded)
|
||||
:footage-id footage-id
|
||||
:audio (if footage (.-audio footage)
|
||||
"/static/arthur/audio.wav")})]
|
||||
(-> (mix/mix! built (:audio entry) (:store entry))
|
||||
(.then (fn [audio] (assoc entry :audio audio)))))))))))
|
||||
|
||||
(rf/reg-fx
|
||||
::save!
|
||||
(fn [{:keys [id cid label clip]}]
|
||||
(let [analysis (:analysis (:clip clip))
|
||||
doc (project/save cid clip)
|
||||
source-blocks (:source-blocks clip)]
|
||||
(-> (ensure-project! id label)
|
||||
(.then (fn [pid]
|
||||
(-> (if analysis
|
||||
(http/POST "/api/analyses" (analysis-payload analysis))
|
||||
(js/Promise.resolve nil))
|
||||
(.then (fn [_]
|
||||
(when (seq source-blocks)
|
||||
(-> (upload-missing!
|
||||
#js {:blocks (source/upload-blocks source-blocks)})
|
||||
(.then (fn [_]
|
||||
(http/PUT
|
||||
(str "/api/analyses/" (:id analysis))
|
||||
;; One set per tracked subject, in
|
||||
;; the order `source/unpack` does
|
||||
;; not depend on.
|
||||
#js {:source_blocks
|
||||
(into-array
|
||||
(source/block-keys source-blocks))})))))))
|
||||
(.then (fn [_] (upload-missing! doc)))
|
||||
(.then (fn [uploaded]
|
||||
(-> (http/PUT (str "/api/projects/" pid)
|
||||
#js {:name label
|
||||
:clips #js [#js {:cid cid
|
||||
:name label
|
||||
:analysis (:id analysis)
|
||||
:footage (:footage-id clip)
|
||||
:leaves (.-leaves doc)
|
||||
:blocks (block-keys doc)}]})
|
||||
(.then (fn [^js saved]
|
||||
(rf/dispatch [::saved pid cid label
|
||||
(.-seq saved)
|
||||
(count (array-seq (.-written saved)))
|
||||
uploaded])))))))))
|
||||
(.catch (fn [error]
|
||||
(js/console.error error)
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))])))))))
|
||||
|
||||
(rf/reg-fx
|
||||
::open!
|
||||
(fn [id]
|
||||
(-> (if id
|
||||
(js/Promise.resolve #js {:id id})
|
||||
;; No id: the most recently updated project, which is what "open" means
|
||||
;; when there is no project browser yet.
|
||||
(-> (http/GET "/api/projects")
|
||||
(.then (fn [^js listed]
|
||||
(or (first (array-seq (.-projects listed)))
|
||||
(throw (ex-info "there is no saved project to open" {})))))))
|
||||
(.then (fn [^js row] (http/GET (str "/api/projects/" (.-id row)))))
|
||||
(.then (fn [^js loaded]
|
||||
(let [^js clip-json (first (array-seq (.-clips loaded)))]
|
||||
(when-not clip-json
|
||||
(throw (ex-info "that project has no clips" {})))
|
||||
(-> (opened-entry! clip-json)
|
||||
(.then (fn [entry]
|
||||
(rf/dispatch [::opened
|
||||
(store/install! entry "project")
|
||||
(.-id loaded)
|
||||
(.-name loaded)
|
||||
(.-seq loaded)])))))))
|
||||
(.catch (fn [error]
|
||||
(js/console.error error)
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))]))))))
|
||||
|
||||
(rf/reg-fx
|
||||
::stage!
|
||||
(fn [_]
|
||||
(-> (http/GET (str "/api/projects/" (:source-project stage/layout)))
|
||||
(.then (fn [^js saved]
|
||||
(or (first (filter #(= (:source-cid stage/layout) (.-cid ^js %))
|
||||
(array-seq (.-clips saved))))
|
||||
(throw (ex-info "the saved 8625 clip is missing" {})))))
|
||||
(.then opened-entry!)
|
||||
(.then (fn [entry]
|
||||
(let [built (stage/compose (:clip entry))
|
||||
entry (assoc entry :clip built :label (:name built)
|
||||
:cid "stage-8625" :frames (clip/frames built)
|
||||
:width (:width built) :height (:height built))]
|
||||
(-> (mix/mix! built (:audio entry) (:store entry))
|
||||
(.then (fn [audio]
|
||||
(rf/dispatch
|
||||
[::stage-opened
|
||||
(store/install! (assoc entry :audio audio) "stage")])))))))
|
||||
(.catch (fn [error]
|
||||
(js/console.error error)
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))]))))))
|
||||
|
||||
(defonce ^:private retained-source (atom nil))
|
||||
(defonce ^:private retained-interior (atom nil))
|
||||
|
||||
(defn- retained-interior! [analysis subject settings inputs]
|
||||
(let [frames (count (:crops inputs))
|
||||
block-key (source/interior-key analysis subject settings frames)]
|
||||
(-> (http/GET (str "/api/blocks/" block-key))
|
||||
(.then (fn [block]
|
||||
(assoc inputs
|
||||
:interior (source/unpack-interior block settings frames)
|
||||
:interior-key block-key)))
|
||||
(.catch (fn [error]
|
||||
(if (= 404 (:status (ex-data error)))
|
||||
(-> (footage/measure-one! settings (dissoc inputs :interior) nil)
|
||||
(.then (fn [measured]
|
||||
(let [{:keys [key descriptor data]}
|
||||
(source/interior-block analysis subject settings
|
||||
(:interior measured))]
|
||||
(-> (upload-missing!
|
||||
#js {:blocks #js [#js {:key key
|
||||
:descriptor descriptor
|
||||
:data data}]})
|
||||
(.then (fn [_]
|
||||
(assoc measured :interior-key block-key))))))))
|
||||
(throw error)))))))
|
||||
|
||||
(defn- source-for! [entry]
|
||||
(if-let [inputs (:source-inputs entry)]
|
||||
(js/Promise.resolve inputs)
|
||||
(let [analysis (get-in entry [:clip :analysis])]
|
||||
(if (= (:id analysis) (:id @retained-source))
|
||||
(:promise @retained-source)
|
||||
(let [promise
|
||||
(if (= "synth" (:detector analysis))
|
||||
;; The synthetic take tracks one face and regenerating it reads
|
||||
;; that face's landmarks, so it arrives in the same shape real
|
||||
;; footage does rather than in a flat one only this branch uses.
|
||||
(js/Promise.resolve
|
||||
{:subjects
|
||||
{:face-1 {:dense (synth/synth-dense (:frames analysis)
|
||||
{:seed (:seed analysis)})}}})
|
||||
(-> (ingest/manifest! (:footage-id entry))
|
||||
(.then (fn [manifest]
|
||||
(-> (footage/saved-source! (:id analysis)
|
||||
[(:width manifest)
|
||||
(:height manifest)])
|
||||
(.then (fn [inputs]
|
||||
(when-not inputs
|
||||
(throw (ex-info "saved analysis has no source blocks" {})))
|
||||
(update inputs :subjects
|
||||
(fn [subjects]
|
||||
(into {}
|
||||
(map (fn [[id one]]
|
||||
[id (assoc one :presence
|
||||
(footage/presence-for
|
||||
manifest id))]))
|
||||
subjects))))))))))]
|
||||
(do
|
||||
(reset! retained-source {:id (:id analysis) :promise promise})
|
||||
promise))))))
|
||||
|
||||
(defn- inputs-for-edit!
|
||||
"Bring the EDITED SUBJECT's retained pixel measurements up to the settings this
|
||||
edit needs. Only that subject's: a knob dragged on the second face does not
|
||||
re-measure the first face's mouth."
|
||||
[entry edit inputs]
|
||||
(let [{clip :changed fids :features subject :subject} (regenerate/plan (:clip entry) edit)
|
||||
teeth (first (filter #(= :teeth (get-in clip [:features % :area])) fids))
|
||||
one (get-in inputs [:subjects subject])]
|
||||
(if (and teeth (:crops one))
|
||||
(let [settings (merge take/knobs (feature/effective-params clip teeth))
|
||||
analysis (get-in clip [:analysis :id])
|
||||
frames (count (:crops one))
|
||||
key (source/interior-key analysis subject settings frames)
|
||||
done (fn [measured] (assoc-in inputs [:subjects subject] measured))]
|
||||
(if (and (:interior one)
|
||||
(or (= (:interior-key one) key)
|
||||
(and (nil? (:interior-key one))
|
||||
(= key (source/interior-key analysis subject take/knobs frames)))))
|
||||
(js/Promise.resolve inputs)
|
||||
(.then (if (= key (:key @retained-interior))
|
||||
(:promise @retained-interior)
|
||||
(let [promise (retained-interior! analysis subject settings one)]
|
||||
(reset! retained-interior {:key key :promise promise})
|
||||
promise))
|
||||
done)))
|
||||
(js/Promise.resolve inputs))))
|
||||
|
||||
(rf/reg-fx
|
||||
::preview-settings!
|
||||
(fn [{:keys [id entry edit request]}]
|
||||
(-> (source-for! entry)
|
||||
(.then (fn [inputs] (inputs-for-edit! entry edit inputs)))
|
||||
(.then (fn [inputs]
|
||||
(regenerate/change (assoc entry :source-inputs inputs) edit)))
|
||||
(.then (fn [changed]
|
||||
(rf/dispatch [::settings-previewed id request changed])))
|
||||
(.catch (fn [error]
|
||||
(js/console.error error)
|
||||
(rf/dispatch [::failed (or (ex-message error) (str error))]))))))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::preview-settings
|
||||
(fn [{:keys [db]} [_ edit]]
|
||||
(let [id (:clip/current db)
|
||||
entry (store/entry id)]
|
||||
(if (or (get-in db [:project :busy?]) (nil? (:analysis (:clip entry))))
|
||||
{}
|
||||
(let [plan (regenerate/plan (:clip entry) edit)
|
||||
report (select-keys plan [:features :roles])
|
||||
request (inc (or (:preview-request db) 0))]
|
||||
(js/console.info "arthur regeneration" (clj->js (assoc report :edit edit)))
|
||||
{:db (-> db
|
||||
(assoc :preview-request request)
|
||||
(update :project merge {:status "previewing…"})
|
||||
(assoc :regeneration (assoc report :edit edit)))
|
||||
::preview-settings! {:id id :entry entry :edit edit
|
||||
:request request}})))))
|
||||
|
||||
(rf/reg-sub ::regeneration (fn [db _] (:regeneration db)))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::settings-previewed
|
||||
(fn [{:keys [db]} [_ previous request entry]]
|
||||
(if (and (= previous (:clip/current db))
|
||||
(= request (:preview-request db)))
|
||||
(let [id (store/install! entry "edited")]
|
||||
{:db (-> db
|
||||
(assoc :clip/current id)
|
||||
(update :project merge {:status "preview · unsaved"}))})
|
||||
{})))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; events
|
||||
|
||||
(rf/reg-event-fx
|
||||
::save
|
||||
(fn [{:keys [db]} _]
|
||||
(let [id (:clip/current db)
|
||||
clip (store/entry id)]
|
||||
(if (or (:busy? (:project db)) (nil? clip))
|
||||
{}
|
||||
{:db (update db :project merge {:busy? true :status "saving…"})
|
||||
::save! {:id (:id (:project db))
|
||||
:cid (or (:cid clip) (name id))
|
||||
:label (or (:label clip) (name id))
|
||||
:clip clip}}))))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::open
|
||||
(fn [{:keys [db]} _]
|
||||
(if (:busy? (:project db))
|
||||
{}
|
||||
{:db (update db :project merge {:busy? true :status "opening…"})
|
||||
::pb/pause! nil
|
||||
::open! (:id (:project db))})))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::load-stage
|
||||
(fn [{:keys [db]} _]
|
||||
(if (:busy? (:project db))
|
||||
{}
|
||||
{:db (update db :project merge {:busy? true :status "loading 8625 stage…"})
|
||||
::pb/pause! nil
|
||||
::stage! nil})))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::stage-opened
|
||||
(fn [{:keys [db]} [_ clip-id]]
|
||||
(let [entry (store/entry clip-id)]
|
||||
{:db (-> db
|
||||
(assoc :clip/current clip-id
|
||||
:clip (select-keys entry [:fps :frames :width :height :audio :display-fps]))
|
||||
(assoc :project {:id nil :cid nil :name nil :seq nil
|
||||
:busy? false :status "loaded 8625 stage study"})
|
||||
(assoc-in [:playback :frame] 0)
|
||||
(assoc-in [:playback :playing?] false))
|
||||
::pb/seek! [(:fps entry) (:frames entry) 0]})))
|
||||
|
||||
(rf/reg-event-db
|
||||
::saved
|
||||
(fn [db [_ id cid label seq written uploaded]]
|
||||
(update db :project merge
|
||||
{:id id :cid cid :name label :seq seq :busy? false
|
||||
:status (str "saved r" seq " · " written
|
||||
(if (= 1 written) " leaf" " leaves")
|
||||
" · " uploaded (if (= 1 uploaded) " block" " blocks"))})))
|
||||
|
||||
(rf/reg-event-fx
|
||||
::opened
|
||||
(fn [{:keys [db]} [_ clip-id project-id name seq]]
|
||||
(let [clip (store/entry clip-id)]
|
||||
{:db (-> db
|
||||
(assoc :clip/current clip-id
|
||||
:clip (select-keys clip [:fps :frames :width :height :audio :display-fps]))
|
||||
(update :project merge
|
||||
{:id project-id :name name :seq seq :cid (:cid clip)
|
||||
:busy? false
|
||||
:status (str "opened " name " r" seq)})
|
||||
(assoc-in [:playback :frame] 0)
|
||||
(assoc-in [:playback :playing?] false))
|
||||
::pb/pause! nil})))
|
||||
|
||||
(rf/reg-event-db
|
||||
::failed
|
||||
(fn [db [_ message]]
|
||||
(update db :project merge {:busy? false :status (str "failed: " message)})))
|
||||
227
frontend/src/arthur/export.cljs
Normal file
227
frontend/src/arthur/export.cljs
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
(ns arthur.export
|
||||
"Export a TIMELINE: one frame walk, and a sink that decides what comes out.
|
||||
|
||||
THE SINK IS A PROTOCOL because there is more than one right answer to \"a
|
||||
video file\" and they disagree about the thing this project cares most about.
|
||||
A PNG sequence is bit-exact — the file holds the bytes `raster/draw-ops!`
|
||||
produced, expanded through the ramp and nothing else. A muxed MP4 is one file
|
||||
that plays anywhere, and every codec a browser can reach either subsamples
|
||||
chroma (which puts fringes on precisely the hard flat-colour edges the whole
|
||||
idiom is made of) or is a codec an NLE will not open. Those are different
|
||||
trades for different jobs, not a better and a worse, so both should be
|
||||
reachable and neither should be the other's special case.
|
||||
|
||||
What is genuinely shared is everything above the sink, and it is most of the
|
||||
work: rooting the resolver at the chosen timeline, generated-channel picture
|
||||
sampling,
|
||||
the raster, the frame loop, the audio mix, the progress reporting and the
|
||||
yielding that lets the page paint. So `run!` owns all of that and calls three
|
||||
methods.
|
||||
|
||||
TWO RULES THE WALK ENFORCES, both about sync:
|
||||
|
||||
Every frame of the timeline's frame space is emitted, at the CLIP's rate. A
|
||||
lower picture rate holds a pose across several frames — it never drops them —
|
||||
so the exported duration matches the audio no matter what the picture rate is.
|
||||
Decimating instead is how an export silently runs short and the sound slides
|
||||
off the picture, which is the one artefact this tool exists to prevent.
|
||||
|
||||
The zoom is an INTEGER. A pixel becomes a block of identical pixels. Anything
|
||||
else resamples, and `domain/png` and `ui/canvas` both have the longer argument
|
||||
for why that is not allowed to happen here."
|
||||
;; `run!` is the verb this namespace is about, and nothing here folds a
|
||||
;; side-effect over a seq, so core's loses the name rather than ours.
|
||||
(:refer-clojure :exclude [run!])
|
||||
(:require [arthur.audio.mix :as mix]
|
||||
[arthur.domain.clip :as clip]
|
||||
[arthur.domain.raster :as raster]))
|
||||
|
||||
(defprotocol Exporter
|
||||
"A sink for a rendered timeline. Implementations live under `arthur.export.*`.
|
||||
|
||||
Called in this order, once, per export: `begin!`, then `frame!` for every frame
|
||||
in order from 0, then `finish!`. Any of them may return a promise and the walk
|
||||
waits for it, which is what keeps a slow encoder from being fed faster than it
|
||||
drains and what gives the page a chance to paint between frames.
|
||||
|
||||
`Exporter` rather than `IExporter`, which is what `domain/timeline`'s
|
||||
`IResolver` would suggest, because it names a role a thing plays rather than a
|
||||
capability a value has."
|
||||
|
||||
(begin! [this spec]
|
||||
"Prepare to receive frames.
|
||||
|
||||
`spec` carries everything constant for the export:
|
||||
|
||||
:name a filesystem-safe stem for the artefact
|
||||
:width stage width in raster pixels, before zoom
|
||||
:height stage height, before zoom
|
||||
:zoom integer pixel multiplier
|
||||
:fps frames per second of the finished file — the CLIP's rate
|
||||
:frames how many frames will arrive
|
||||
:ramp index -> [r g b], the palette to expand through
|
||||
:audio an AudioBuffer, or nil when the timeline has no sound
|
||||
|
||||
The ramp and the audio are here rather than on `frame!` because neither
|
||||
changes across an export, and a muxer has to declare its tracks before it
|
||||
will accept a sample.")
|
||||
|
||||
(frame! [this i raster]
|
||||
"Take frame `i`, an indexed `domain/raster`.
|
||||
|
||||
THE RASTER IS REUSED and must be consumed before this returns (or before the
|
||||
promise it returns settles). The walk hands back the same buffer every frame,
|
||||
for the same reason `timeline/resolver` reuses its point buffers: a 900-frame
|
||||
export that allocates a stage per frame is a tab that swaps. A sink that wants
|
||||
to keep pixels has to copy or encode them here.")
|
||||
|
||||
(finish! [this]
|
||||
"Close the artefact. Promise of `{:filename :blob}`."))
|
||||
|
||||
(defn kin
|
||||
"The ids to keep when isolating `id` in `nodes`.
|
||||
|
||||
Four things, and each for its own reason:
|
||||
|
||||
the node itself;
|
||||
everything ABOVE it, because a placement's transform is relative to its
|
||||
parent and dropping the chain would move the thing being isolated;
|
||||
everything BELOW it, because a group instance is its children;
|
||||
any audio track `:linked-to` it, because the link is the statement that this
|
||||
sound belongs to that placement, and a face exported without its voice is
|
||||
not the thing that was asked for.
|
||||
|
||||
Siblings go. That is the whole point: what comes out is one placement, where it
|
||||
sits, on the timeline it sits on."
|
||||
[nodes id]
|
||||
(let [up (loop [i id acc #{}]
|
||||
(if (or (nil? i) (contains? acc i))
|
||||
acc
|
||||
(recur (:parent (get nodes i)) (conj acc i))))
|
||||
down (loop [edge #{id} acc #{}]
|
||||
(if (empty? edge)
|
||||
acc
|
||||
(let [acc' (into acc edge)]
|
||||
(recur (set (for [[k n] nodes
|
||||
:when (and (contains? edge (:parent n))
|
||||
(not (contains? acc' k)))]
|
||||
k))
|
||||
acc'))))
|
||||
kept (into up down)]
|
||||
(into kept
|
||||
(for [[k n] nodes
|
||||
:when (and (= :audio (:kind n)) (contains? kept (:linked-to n)))]
|
||||
k))))
|
||||
|
||||
(defn isolate
|
||||
"The timeline with only `id` and its kin kept. `nil` leaves it alone.
|
||||
|
||||
The FRAME SPACE IS UNTOUCHED, which is what makes this different from exporting
|
||||
the symbol a placement plays. Rooting at `:sym/face-8625` renders the drawing in
|
||||
its own time, identically for all seven placements. Isolating one placement
|
||||
renders the STAGE — its length, its rate, the placement's span, drift and scale
|
||||
— with the other six removed. The first is the drawing; the second is that face
|
||||
on the stage, and they are different deliverables."
|
||||
[tl id]
|
||||
(if (and id (get-in tl [:nodes id]))
|
||||
(update tl :nodes select-keys (kin (:nodes tl) id))
|
||||
tl))
|
||||
|
||||
(defn- yield!
|
||||
"Hand the event loop a turn between frames.
|
||||
|
||||
`setTimeout 0` and not a resolved promise: a promise continuation is a
|
||||
microtask, so a chain of them runs to completion without the browser ever
|
||||
painting, and the progress readout would jump from 0 to done. `flow/ingest`'s
|
||||
decode loop pauses for the same reason."
|
||||
[]
|
||||
(js/Promise. (fn [resolve] (js/setTimeout resolve 0))))
|
||||
|
||||
(defn audio!
|
||||
"Promise of the AudioBuffer to export alongside the picture, or nil.
|
||||
|
||||
A timeline's own placed audio tracks win. Failing that, the ROOT timeline — and
|
||||
only the root — falls back to the clip's audio file, which is where a take's
|
||||
sound lives before anyone has placed a track. A symbol exports silence rather
|
||||
than the whole clip's soundtrack, because a symbol's frame space is its own and
|
||||
the clip's audio is not a fact about it."
|
||||
[clip-doc tid store fallback-url]
|
||||
(-> (mix/buffer! clip-doc tid store)
|
||||
(.then (fn [buffer]
|
||||
(cond
|
||||
buffer buffer
|
||||
(and (= tid clip/root-id) fallback-url) (mix/decode! fallback-url)
|
||||
:else nil)))))
|
||||
|
||||
(defn plan
|
||||
"What an export of `tid` will produce, without producing any of it.
|
||||
|
||||
Separate from `run!` so the UI can show the size and length it is about to
|
||||
commit to, and so the arithmetic is assertable without a sink."
|
||||
[{:keys [clip timeline zoom picture-fps] isolate-id :isolate}]
|
||||
(let [tl (some-> (clip/timeline clip timeline) (isolate isolate-id))
|
||||
zoom (max 1 (js/Math.floor (or zoom 1)))]
|
||||
(when tl
|
||||
{:frames (:frames tl)
|
||||
:fps (:fps clip)
|
||||
:zoom zoom
|
||||
:width (* (:width clip) zoom)
|
||||
:height (* (:height clip) zoom)
|
||||
:seconds (/ (:frames tl) (:fps clip))
|
||||
;; The unedited picture-grid count. A per-instance pose track can add or
|
||||
;; remove changes, so this is only the grid's nominal count.
|
||||
:poses (if (and picture-fps (< picture-fps (:fps clip)))
|
||||
(js/Math.ceil (* (/ (:frames tl) (:fps clip)) picture-fps))
|
||||
(:frames tl))})))
|
||||
|
||||
(defn run!
|
||||
"Render `timeline` into `exporter`. Promise of `{:filename :blob}`.
|
||||
|
||||
`on-progress` is called with `[done total]` as frames complete, and is where a
|
||||
UI hangs its readout."
|
||||
[{:keys [clip timeline store palette ramp zoom picture-fps name audio-url]
|
||||
isolate-id :isolate}
|
||||
exporter on-progress]
|
||||
(let [tl (some-> (clip/timeline clip timeline) (isolate isolate-id))]
|
||||
(when-not tl
|
||||
(throw (ex-info "there is no such timeline to export"
|
||||
{:timeline timeline
|
||||
:timelines (vec (sort-by str (keys (:timelines clip))))})))
|
||||
(let [{:keys [frames fps zoom]} (plan {:clip clip :timeline timeline :zoom zoom
|
||||
:isolate isolate-id})
|
||||
;; Rooted at the chosen timeline, so exporting a symbol is exporting a
|
||||
;; clip whose root that symbol is. Nested symbols inside it still
|
||||
;; resolve — clip/resolver is the function that knows how.
|
||||
doc (assoc-in clip [:timelines timeline] tl)
|
||||
resolve-frame (clip/resolver doc store palette timeline
|
||||
{:picture-fps picture-fps})
|
||||
ras (raster/make (:width clip) (:height clip))
|
||||
bg (get palette :bg 0)]
|
||||
(-> (audio! doc timeline store audio-url)
|
||||
(.then (fn [audio]
|
||||
(js/Promise.resolve
|
||||
(begin! exporter {:name name :width (:width clip)
|
||||
:height (:height clip) :zoom zoom
|
||||
:fps fps :frames frames :ramp ramp
|
||||
:audio audio}))))
|
||||
(.then (fn [_]
|
||||
;; A fold over the frames as a promise CHAIN rather than a
|
||||
;; doseq: each frame has to wait for the last one's sink to
|
||||
;; drain, and `reduce` building that chain is the shape that
|
||||
;; says so. Nothing here is concurrent on purpose — an encoder
|
||||
;; fed from two places at once is not fast, it is wrong.
|
||||
(reduce
|
||||
(fn [chain i]
|
||||
(.then chain
|
||||
(fn [_]
|
||||
(-> ras
|
||||
(raster/clear! bg)
|
||||
(raster/draw-ops! (resolve-frame i)))
|
||||
(-> (js/Promise.resolve (frame! exporter i ras))
|
||||
(.then (fn [_]
|
||||
(when on-progress
|
||||
(on-progress (inc i) frames))
|
||||
(yield!)))))))
|
||||
(js/Promise.resolve)
|
||||
(range frames))))
|
||||
(.then (fn [_] (finish! exporter)))))))
|
||||
77
frontend/src/arthur/export/frames.cljs
Normal file
77
frontend/src/arthur/export/frames.cljs
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
(ns arthur.export.frames
|
||||
"An `Exporter` that writes a numbered PNG sequence and its WAV into one zip.
|
||||
|
||||
THE MASTER FORMAT. Every other export is a re-interpretation of this one: the
|
||||
PNGs hold exactly the bytes `raster/draw-ops!` wrote, expanded through the ramp
|
||||
at an integer zoom, so nothing between the scanline fill and the file resamples,
|
||||
subsamples or smooths. `domain/png` has the argument for why that matters here
|
||||
more than it would in most tools.
|
||||
|
||||
IT IS ALSO SMALLER THAN IT SOUNDS, which is worth saying because \"lossless
|
||||
frame sequence\" reads as gigabytes. That intuition comes from photographic
|
||||
frames — the 1440x1920 source stills this project stopped storing were 112MB for
|
||||
7.6 seconds. This is nine palette colours of flat fill at 320x200, upscaled by
|
||||
an integer: a frame's entropy is on the order of kilobytes, and the zoom is
|
||||
nearly free because a duplicated scanline filters to zeros. The finished master
|
||||
of a take runs comparable to the lossy PROXY of the footage it came from.
|
||||
|
||||
ONE ARCHIVE, PICTURE AND SOUND TOGETHER, rather than two downloads. They have
|
||||
to stay in sync all the way to the cutting room, and a second `<a download>`
|
||||
click is also the one a browser is most likely to block."
|
||||
(:require [arthur.audio.mix :as mix]
|
||||
[arthur.domain.png :as png]
|
||||
[arthur.domain.zip :as zip]
|
||||
[arthur.export :as export]))
|
||||
|
||||
(defn- pad
|
||||
"Frame numbers are ONE-BASED and zero-padded to a fixed width, because that is
|
||||
what an NLE's image-sequence importer looks for: a common stem, a fixed-width
|
||||
counter, one extension. Width comes from the frame count, so a 900-frame export
|
||||
is `0001`..`0900` and nothing sorts `10` before `9`."
|
||||
[i width]
|
||||
(let [s (str i)]
|
||||
(str (.repeat "0" (max 0 (- width (.-length s)))) s)))
|
||||
|
||||
(defn exporter
|
||||
"A frame-sequence `Exporter`.
|
||||
|
||||
`at` is the timestamp stamped on every zip entry, defaulting to now. It is a
|
||||
parameter so that the same frames produce the same archive byte for byte, which
|
||||
is what makes `export.frames-test` able to assert on one."
|
||||
([] (exporter (js/Date.)))
|
||||
([at]
|
||||
(let [state (atom nil)]
|
||||
(reify export/Exporter
|
||||
(begin! [_ {:keys [name width height zoom fps frames ramp audio]}]
|
||||
(reset! state
|
||||
{:name name
|
||||
:ramp ramp
|
||||
:fps fps
|
||||
:frames frames
|
||||
;; Held once. At zoom 6 the scratch inside it is seven
|
||||
;; megabytes, which is not a thing to allocate per frame.
|
||||
:encode (png/encoder width height zoom)
|
||||
:digits (max 4 (.-length (str frames)))
|
||||
:entries (cond-> []
|
||||
audio (conj {:name (str name "/audio.wav")
|
||||
:data (mix/wav-bytes audio)}))})
|
||||
nil)
|
||||
|
||||
(frame! [_ i raster]
|
||||
(let [{:keys [encode ramp name digits]} @state]
|
||||
;; Encoded HERE, inside the frame's turn, because the walk reuses the
|
||||
;; raster: keeping a reference to it and encoding later would encode
|
||||
;; the last frame N times, and every frame would be a valid PNG of the
|
||||
;; wrong picture.
|
||||
(-> (encode raster ramp)
|
||||
(.then (fn [bytes]
|
||||
(swap! state update :entries conj
|
||||
{:name (str name "/" (pad (inc i) digits) ".png")
|
||||
:data bytes})
|
||||
nil)))))
|
||||
|
||||
(finish! [_]
|
||||
(let [{:keys [name entries]} @state]
|
||||
(js/Promise.resolve
|
||||
{:filename (str name ".zip")
|
||||
:blob (zip/blob entries at)})))))))
|
||||
251
frontend/src/arthur/flow/address.cljs
Normal file
251
frontend/src/arthur/flow/address.cljs
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
(ns arthur.flow.address
|
||||
"Tier 2 keys: what a dense block is NAMED, and what that name is made of.
|
||||
|
||||
Until step 9 a block's key was a descriptive string — \"take/geom\",
|
||||
\"footage/iris-pos\" — and `flow/freeze` said of them: \"they become the blocks'
|
||||
sha256 when the backend arrives and nothing above here changes, which is the
|
||||
point of a handle.\" This is that, and nothing above it did change.
|
||||
|
||||
A KEY IS A HASH OVER INPUTS, NOT OVER BYTES. Both are content addressing and
|
||||
they answer different questions. Hashing the bytes tells you whether two blocks
|
||||
are identical; hashing the inputs tells you, BEFORE computing anything, which
|
||||
block the current settings want — which is the question a cache is asked. It is
|
||||
also what makes a stale bake unreachable rather than wrong: change a knob and
|
||||
the scene names a key that no longer exists, so the worst case is a re-freeze.
|
||||
Nothing in the system can serve old landmarks under new settings.
|
||||
|
||||
THE DETECTOR VERSION IS IN IT, and docs/architecture.md is explicit about why:
|
||||
a model upgrade that silently reuses old landmarks presents as \"the tool got
|
||||
worse\", with no event to attach it to. It enters through the ANALYSIS id, which
|
||||
every block descriptor names, so it cannot be in one block's key and missing
|
||||
from another's.
|
||||
|
||||
WHICH KNOBS. `block-knobs` below is the invalidation table: for each block, the
|
||||
settings its BYTES depend on. Getting it wrong in either direction is a bug with
|
||||
a different symptom — too few and a knob silently does nothing until a reload,
|
||||
too many and every unrelated tweak throws away a good bake — so it is not
|
||||
trusted. `address-test` re-freezes the take once per knob and asserts the
|
||||
biconditional: a block's bytes changed if and only if its key changed. That is
|
||||
what keeps this table honest, because reading it will not.
|
||||
|
||||
Not a namespace with state, and not a registry: every function here is
|
||||
`(f inputs) -> string`."
|
||||
(:require [arthur.domain.canon :as canon]
|
||||
[arthur.domain.sha256 :as sha]))
|
||||
|
||||
(def ^:const scheme
|
||||
"The addressing scheme's own version, inside every key.
|
||||
|
||||
If the shape of a descriptor changes — a field added, a field's meaning
|
||||
revised — then keys computed the old way name bytes produced by code that no
|
||||
longer exists. Bumping this makes every one of them unreachable in one edit,
|
||||
which is the cheap version of a migration."
|
||||
1)
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the analysis artifact
|
||||
|
||||
(defn analysis-descriptor
|
||||
"The canonical text naming one analysis artifact: which detector, at which
|
||||
version, over which source.
|
||||
|
||||
`:fps` and `:aspect` are in here rather than in the block descriptors, and that
|
||||
is not an arrangement of convenience. Both are properties of the FOOTAGE — the
|
||||
source cadence and the pixel aspect of the frames it was decoded from — and both
|
||||
reach tier 2 bytes: aspect through every landmark that is de-anisotropised
|
||||
before a fit, and fps through every dwell that is specified in seconds
|
||||
(`condition/quantize-snap`'s gaze and brow cells, `resolve-blink`'s hold). A
|
||||
block inherits them by naming the analysis, so they cannot be in one block's key
|
||||
and missing from another's.
|
||||
|
||||
`:source` is in it and `:name` is NOT. A clip's name is a label a human types;
|
||||
two clips of the same footage under different names are the same analysis and
|
||||
must share it, which is the whole return on addressing.
|
||||
|
||||
`:mode` is how the detector was RUN, and it belongs here for the same reason the
|
||||
version does. MediaPipe's video mode is a tracker and its image mode is not:
|
||||
over the same frames and the same model they disagree by up to 0.013 of frame
|
||||
width, which is a visible difference on a mouth. Optional, because the synthetic
|
||||
take has no running mode to declare and an absent field is how the other
|
||||
optional inputs already say \"not applicable\"."
|
||||
[{:keys [detector version source footage frames fps aspect seed mode tracking]}]
|
||||
(when-not (and (string? detector) (seq detector) (string? version) (seq version))
|
||||
(throw (ex-info "an analysis names its detector and the detector's VERSION: an upgrade that silently reuses old landmarks is the failure content addressing exists to prevent"
|
||||
{:detector detector :version version})))
|
||||
(canon/write (cond-> {:scheme scheme
|
||||
:detector detector
|
||||
:version version
|
||||
:frames frames
|
||||
:fps fps
|
||||
:aspect aspect}
|
||||
source (assoc :source source)
|
||||
footage (assoc :footage footage)
|
||||
seed (assoc :seed seed)
|
||||
mode (assoc :mode mode)
|
||||
tracking (assoc :tracking tracking))))
|
||||
|
||||
(defn analysis
|
||||
"An analysis record with its `:id` filled in. The record is tier 1 — it says
|
||||
what produced the clip's channels — and the id is what `:generated :analysis`
|
||||
carries on every generated channel."
|
||||
[record]
|
||||
(assoc record :id (sha/key-of (analysis-descriptor record))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the observation masks
|
||||
|
||||
(defn- feature-name
|
||||
"An explicit subject or feature id as a descriptor string."
|
||||
[id]
|
||||
(subs (str id) 1))
|
||||
|
||||
(defn observation
|
||||
"A digest of exactly the absence data one block reads.
|
||||
|
||||
Digested rather than inlined, for two reasons. A descriptor is meant to be READ
|
||||
— a stale bake presents as a picture that will not update, and the descriptor is
|
||||
the only thing that can say which input moved — and a 229-frame boolean mask
|
||||
inlined in it would bury the knobs it sits beside. And it is per-block: the eye
|
||||
block reads `:eye-r` and `:eye-l`'s presence and no other feature's, so a gap in
|
||||
one brow does not rewrite the mouth's address for nothing.
|
||||
|
||||
Head tracks name their subject and follow its detection mask."
|
||||
[features {:keys [detected presence]}]
|
||||
(let [wanted (sort-by str (distinct features))
|
||||
masks (into {} (map (fn [id] [id (mapv boolean (get presence id))]))
|
||||
(filter #(contains? presence %) wanted))]
|
||||
(when (or detected (seq masks))
|
||||
(sha/key-of (canon/write {:scheme scheme
|
||||
:detected (when detected (mapv boolean detected))
|
||||
:presence (into {} (map (fn [[id m]] [(feature-name id) m]))
|
||||
(sort-by (comp str key) masks))})))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the blocks
|
||||
|
||||
(def block-knobs
|
||||
"Per block, the settings its BYTES depend on. The invalidation table, and the
|
||||
thing `address-test` refuses to take on trust.
|
||||
|
||||
Two entries worth reading twice, because both are asymmetries a reasonable
|
||||
person would call a mistake:
|
||||
|
||||
The EYE block does not depend on `blink-cut`. A blink is a `[:vis]` key on the
|
||||
eye's interior — tier 1, editable, a handful of transitions — and the lid
|
||||
geometry underneath it is the same either way. `iris-size` and `pupil-size` are
|
||||
missing for the same reason: both land on framed channels, not in a block.
|
||||
|
||||
The TEETH block depends on `aperture-cut`, which nothing else in the table does.
|
||||
`condition/interior` will not smooth a contour on a frame the teeth are not
|
||||
shown on, and whether they are shown starts with the mouth being open — so the
|
||||
mouth's threshold reaches the pixel geometry, while the mouth's own vertex
|
||||
budget does not reach the teeth at all (the crop is taken from raw landmarks).
|
||||
It reads like a mistake in both directions and is neither."
|
||||
{"geom" [:anchor-avg :contour-avg :verts]
|
||||
"source/dense" []
|
||||
"source/detected" []
|
||||
"source/crops" []
|
||||
"source/interior" [:blob-grow :cavity-erode :min-area :teeth-verts
|
||||
:tongue-reject :top-bias]
|
||||
"head-pos" [:anchor-avg]
|
||||
"head-rot" [:anchor-avg]
|
||||
"head-scale" [:anchor-avg]
|
||||
"eyes" [:anchor-avg :contour-avg :eye-verts :lash-weight]
|
||||
"iris-pos" [:anchor-avg :contour-avg :gaze-gain :gaze-step]
|
||||
"brows" [:anchor-avg :contour-avg :brow-verts :brow-gain :brow-step :brow-weight]
|
||||
;; `brow-pos` and not `contour-avg`: the ring is smoothed and the RAISE is not.
|
||||
;; `condition/brows` takes the end heights straight from measure, medians them
|
||||
;; for a rest position and snaps them onto a grid, and never passes them
|
||||
;; through `condition/contours`. Asserted, not assumed — the biconditional in
|
||||
;; address-test is what found it here.
|
||||
"brow-pos" [:anchor-avg :brow-gain :brow-step]
|
||||
"teeth" [:anchor-avg :aperture-cut :blob-grow :cavity-erode :min-area
|
||||
:teeth-on :teeth-smooth :teeth-verts :tongue-reject :top-bias]})
|
||||
|
||||
(def knob-roles
|
||||
"knob -> the roles whose bytes it moves. The inverse of `block-knobs`, DERIVED.
|
||||
|
||||
This is what a parameter UI wants — \"what stops being valid if I drag this\" —
|
||||
and deriving it is the whole point: `block-knobs` is the table `address-test`
|
||||
asserts by biconditional, so an answer computed from it cannot drift from an
|
||||
answer that is checked, and an answer written down beside it could.
|
||||
|
||||
A knob absent from this map invalidates NO BLOCK, and that is a real answer
|
||||
rather than a gap. `:blink-cut`, `:iris-size` and `:pupil-size` are all absent,
|
||||
because a blink is `[:vis]` keys and the two sizes are framed channels: tier 1,
|
||||
editable, and rewritten by a re-freeze without a byte of tier 2 moving."
|
||||
(reduce (fn [m [role knobs]]
|
||||
(reduce (fn [m knob] (update m knob (fnil conj #{}) role)) m knobs))
|
||||
{}
|
||||
block-knobs))
|
||||
|
||||
(defn invalidates
|
||||
"The roles one knob's bytes depend on, or an empty set."
|
||||
[knob]
|
||||
(get knob-roles knob #{}))
|
||||
|
||||
(def area-roles
|
||||
"Feature area -> the block roles `freeze/part` freezes for it.
|
||||
|
||||
The other half of a question neither table answers alone. `block-knobs` says
|
||||
which knobs reach a ROLE's bytes; this says which roles a FEATURE owns; and what
|
||||
a regeneration actually asks is which knobs reach one feature."
|
||||
{:mouth ["geom"]
|
||||
:eye ["eyes" "iris-pos"]
|
||||
:brow ["brows" "brow-pos"]
|
||||
:teeth ["teeth"]})
|
||||
|
||||
(def framed-knobs
|
||||
"Feature area -> the knobs its TIER 1 channels read.
|
||||
|
||||
What `block-knobs` cannot answer and deliberately does not: a framed radius and a
|
||||
keyed `[:vis]` hold no bytes, so no block key moves when they move — and a
|
||||
re-freeze still has to happen or the knob does nothing at all. This is the half
|
||||
that used to live nowhere, and a regeneration had to guess at with a per-knob
|
||||
special case. `regenerate-test` asserts the biconditional over the union, the
|
||||
same way `address-test` does for `block-knobs`, so it is checked and not believed.
|
||||
|
||||
`:aperture-cut` is here AND in the teeth block: it gates `mouth-in`'s visibility
|
||||
in tier 1 and the interior contour's smoothing in tier 2. One knob, two features,
|
||||
two routes — which is exactly why this cannot be a per-area list of its own."
|
||||
{:mouth #{:aperture-cut}
|
||||
:eye #{:blink-cut :iris-size :pupil-size}
|
||||
:brow #{}
|
||||
:teeth #{}})
|
||||
|
||||
(defn area-knobs
|
||||
"Every knob one feature area's frozen output depends on, across both tiers."
|
||||
[area]
|
||||
(into (get framed-knobs area #{}) (mapcat block-knobs) (get area-roles area)))
|
||||
|
||||
(defn block-descriptor
|
||||
"The canonical text naming one dense block.
|
||||
|
||||
`:tracks` is in it and so is `:layout`: two blocks over the same inputs that
|
||||
pack a different number of tracks, or the same tracks in another order, are
|
||||
different bytes at the same offsets, and a reader that trusted the key would
|
||||
hand the left eye's geometry to the right one."
|
||||
[{:keys [role analysis params features tracks layout observation]}]
|
||||
(let [knobs (or (get block-knobs role)
|
||||
(throw (ex-info "no invalidation table for this block role: add it to block-knobs in the same commit as the block, or its key cannot change when its bytes do"
|
||||
{:role role :roles (sort (keys block-knobs))})))
|
||||
missing (remove #(contains? params %) knobs)]
|
||||
(when (seq missing)
|
||||
(throw (ex-info "a knob this block's bytes depend on was not passed to the freeze"
|
||||
{:role role :missing (vec missing)})))
|
||||
(canon/write {:scheme scheme
|
||||
:role role
|
||||
:analysis analysis
|
||||
:params (select-keys params knobs)
|
||||
:features (mapv feature-name features)
|
||||
:tracks (vec tracks)
|
||||
:layout layout
|
||||
:observation observation})))
|
||||
|
||||
(defn block
|
||||
"`{:key :descriptor}` for one dense block. The descriptor travels with the bytes
|
||||
— see `arthur.domain.leaf` and clips/views.py — because the server verifies
|
||||
`sha256(descriptor) == key` on upload rather than trusting a name it was handed."
|
||||
[spec]
|
||||
(let [text (block-descriptor spec)]
|
||||
{:key (sha/key-of text) :descriptor text}))
|
||||
89
frontend/src/arthur/flow/condition.cljs
Normal file
89
frontend/src/arthur/flow/condition.cljs
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
(ns arthur.flow.condition
|
||||
"Stage 4: reusable temporal conditioning of measured signals.
|
||||
|
||||
It is a stage of its own for exactly one reason. `anchor avg` and `contour avg`
|
||||
are knobs and the rest of measure is not, so dragging either must not re-run the
|
||||
interior extraction — the one part of measure that reads a source pixel, and the
|
||||
only part that costs seconds.
|
||||
|
||||
`anchor` smooths four transform parameters and `contours` smooths a ring track
|
||||
per vertex. Median rest positions, grid dwell and blink holds also operate on
|
||||
measurements without reading footage pixels or depending on a scene node."
|
||||
(:require [arthur.domain.geom :as geom]))
|
||||
|
||||
(defn anchor
|
||||
"Smooth the anchor fit's four parameters. `arthur.domain.geom/smooth-transforms`
|
||||
says why it is the transform and not the contour.
|
||||
|
||||
Returns the measured map with `:transforms` replaced, so the anchor keeps
|
||||
travelling as one value and nothing downstream has to know whether it has been
|
||||
conditioned yet. `:residual` is deliberately left alone: it is the residual of
|
||||
the FIT, and it is not a function of this knob."
|
||||
[{:keys [anchor-avg]} anchored]
|
||||
(update anchored :transforms geom/smooth-transforms anchor-avg))
|
||||
|
||||
(defn contours
|
||||
"Temporal smoothing of a ring track, per vertex, across time.
|
||||
|
||||
docs/design.md says to smooth the transform and never the contour. That was
|
||||
correct while keys were sparse: sampling at velocity minima rejected per-frame
|
||||
detector noise for free. With a key on every frame the noise is visible as a
|
||||
shimmer along the lip edge, so a bounded exception applies - the window must
|
||||
stay SHORTER than the shortest articulation worth keeping. At 12fps, mouth
|
||||
movement spans 3-6 frames and detector noise is per-frame, so a radius of 1
|
||||
separates them and a radius of 3 would start eating speech.
|
||||
|
||||
`contour-avg` is in frames either side: 0 off, 1 = 3-frame average, 2 = 5-frame.
|
||||
It is the same clamped window `geom/moving-average` gives the transform
|
||||
parameters — reused rather than re-derived, so \"radius 2\" cannot come to mean
|
||||
two different things at the two knobs."
|
||||
[{:keys [contour-avg]} rings]
|
||||
(if (<= contour-avg 0)
|
||||
(vec rings)
|
||||
(let [rings (vec rings)
|
||||
axis (fn [v k] (geom/moving-average (map #(k (nth % v)) rings) contour-avg))
|
||||
;; Transposed once into a per-vertex pair of series, because the
|
||||
;; smoothing is along time and the storage is along vertices.
|
||||
axes (mapv (fn [v] [(axis v :x) (axis v :y)])
|
||||
(range (count (first rings))))]
|
||||
(mapv (fn [t] (mapv (fn [[xs ys]] {:x (nth xs t) :y (nth ys t)}) axes))
|
||||
(range (count rings))))))
|
||||
|
||||
(defn median-point
|
||||
"A rest position per axis, resistant to a few extreme poses."
|
||||
[points]
|
||||
(let [median (fn [xs]
|
||||
(let [v (vec (sort xs)) n (count v)]
|
||||
(if (odd? n) (nth v (quot n 2))
|
||||
(/ (+ (nth v (dec (quot n 2))) (nth v (quot n 2))) 2))))]
|
||||
{:x (median (map :x points)) :y (median (map :y points))}))
|
||||
|
||||
(defn quantize-snap
|
||||
"Snap a two-axis signal to a grid, accepting a new cell after its dwell."
|
||||
[points step dwell]
|
||||
(let [snap (fn [v] (* (js/Math.round (/ v step)) step))]
|
||||
(if (not (pos? step))
|
||||
(vec points)
|
||||
(let [q (mapv (fn [{:keys [x y]}] {:x (snap x) :y (snap y)}) points)]
|
||||
(loop [remaining q live (first q) pending (first q) run 0 out []]
|
||||
(if-let [p (first remaining)]
|
||||
(let [same? (= p pending)
|
||||
pending (if same? pending p)
|
||||
run (if same? (inc run) 1)
|
||||
live (if (and (> run dwell) (not= pending live)) pending live)]
|
||||
(recur (rest remaining) live pending run (conj out live)))
|
||||
out))))))
|
||||
|
||||
(defn resolve-blink
|
||||
"Hysteresis, dwell and minimum shut hold. The hold is specified in source
|
||||
frames and is applied before a lower picture fps samples the result."
|
||||
[openness {:keys [cut dwell hold]}]
|
||||
(loop [readings openness live false run 0 held 0 out []]
|
||||
(if-let [v (first readings)]
|
||||
(let [reading (< v (if live (* cut 1.35) cut))
|
||||
run (if (= reading live) 0 (inc run))
|
||||
change? (and (> run dwell) (or (not live) (>= held hold)))
|
||||
live (if change? reading live)
|
||||
held (if change? 1 (inc held))]
|
||||
(recur (rest readings) live (if change? 0 run) held (conj out live)))
|
||||
out)))
|
||||
46
frontend/src/arthur/flow/condition/brows.cljs
Normal file
46
frontend/src/arthur/flow/condition/brows.cljs
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
(ns arthur.flow.condition.brows
|
||||
"Condition measured brow shape and end heights in head-local space."
|
||||
(:require [arthur.domain.ring :as ring]
|
||||
[arthur.flow.condition :as condition]))
|
||||
|
||||
(defn- distance [a b]
|
||||
(js/Math.hypot (- (:x a) (:x b)) (- (:y a) (:y b))))
|
||||
|
||||
(defn- clamp01 [x] (max 0 (min 1 x)))
|
||||
|
||||
(defn- conditioned-side [rings raises corners
|
||||
{:keys [fps contour-avg brow-gain brow-step brow-weight]}]
|
||||
(let [fps (or fps 30)
|
||||
rings (condition/contours {:contour-avg contour-avg} rings)
|
||||
w (/ (reduce + (map (fn [[a b]] (distance a b)) corners))
|
||||
(count corners))
|
||||
rest (condition/median-point raises)
|
||||
px (mapv (fn [{:keys [x y]}]
|
||||
{:x (* (- x (:x rest)) brow-gain w)
|
||||
:y (* (- y (:y rest)) brow-gain w)}) raises)
|
||||
cells (condition/quantize-snap px (* brow-step w)
|
||||
(js/Math.round (* fps 0.08)))]
|
||||
(let [frames (mapv (fn [shape [outer inner] measured snapped]
|
||||
(let [d-outer (- (:x measured) (:x snapped))
|
||||
d-inner (- (:y measured) (:y snapped))
|
||||
move (/ (+ d-outer d-inner) 2)
|
||||
span (- (:x inner) (:x outer))
|
||||
warped (mapv (fn [p]
|
||||
(let [t (if (< (abs span) 1e-9) 0
|
||||
(clamp01 (/ (- (:x p) (:x outer)) span)))]
|
||||
(update p :y + (+ (- d-outer move)
|
||||
(* (- d-inner d-outer) t)))))
|
||||
shape)]
|
||||
{:ring (ring/offset-ring warped (* brow-weight w))
|
||||
:pos [0 move]}))
|
||||
rings corners px cells)]
|
||||
{:rings (mapv :ring frames) :positions (mapv :pos frames)})))
|
||||
|
||||
(defn apply-defaults
|
||||
[params {:keys [ring-r ring-l raise-r raise-l] :as measured}
|
||||
{:keys [corners-r corners-l]}]
|
||||
(let [r (conditioned-side ring-r raise-r corners-r params)
|
||||
l (conditioned-side ring-l raise-l corners-l params)]
|
||||
(assoc measured
|
||||
:ring-r (:rings r) :ring-l (:rings l)
|
||||
:pos-r (:positions r) :pos-l (:positions l))))
|
||||
79
frontend/src/arthur/flow/condition/eyes.cljs
Normal file
79
frontend/src/arthur/flow/condition/eyes.cljs
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
(ns arthur.flow.condition.eyes
|
||||
"Condition measured eyes while preserving every source frame. Lid geometry
|
||||
stays dense; blink and shared gaze are decisions over those measurements."
|
||||
(:require [arthur.domain.ring :as ring]
|
||||
[arthur.flow.condition :as condition]))
|
||||
|
||||
(defn- distance [a b]
|
||||
(js/Math.hypot (- (:x a) (:x b)) (- (:y a) (:y b))))
|
||||
|
||||
(defn- socket [lid]
|
||||
(let [a (first lid) b (nth lid (quot (count lid) 2))]
|
||||
{:x (/ (+ (:x a) (:x b)) 2)
|
||||
:y (/ (+ (:y a) (:y b)) 2)
|
||||
:w (distance a b)}))
|
||||
|
||||
(defn- hold-observed
|
||||
"Give temporal filters real samples at every index. The freeze mask still
|
||||
marks the gap absent; these held values are only numeric placeholders."
|
||||
[values observed fallback]
|
||||
(let [values (vec values)
|
||||
seed (or (first (keep-indexed (fn [i v]
|
||||
(when (and (nth observed i) (some? v)) v))
|
||||
values))
|
||||
fallback (first values))]
|
||||
(loop [f 0 last-value seed out []]
|
||||
(if (= f (count values))
|
||||
out
|
||||
(let [v (nth values f)
|
||||
next-value (if (and (nth observed f) (some? v)) v last-value)]
|
||||
(recur (inc f) next-value (conj out next-value)))))))
|
||||
|
||||
(defn- mean-width [lids observed]
|
||||
(let [valid (seq (keep-indexed (fn [i lid] (when (nth observed i) lid)) lids))
|
||||
rings (or valid lids)]
|
||||
(/ (reduce + (map (comp :w socket) rings)) (count rings))))
|
||||
|
||||
(defn apply-defaults
|
||||
[{:keys [fps contour-avg blink-cut gaze-gain gaze-step iris-size
|
||||
lash-weight pupil-size]} {:keys [lid-r lid-l open-r open-l gaze
|
||||
observed-r observed-l gaze-observed]
|
||||
:as measured}]
|
||||
(let [fps (or fps 30)
|
||||
observed-r (or observed-r (vec (repeat (count lid-r) true)))
|
||||
observed-l (or observed-l (vec (repeat (count lid-l) true)))
|
||||
gaze-observed (or gaze-observed (vec (repeat (count gaze) true)))
|
||||
right (condition/contours {:contour-avg contour-avg}
|
||||
(hold-observed lid-r observed-r nil))
|
||||
left (condition/contours {:contour-avg contour-avg}
|
||||
(hold-observed lid-l observed-l nil))
|
||||
w-r (mean-width right observed-r)
|
||||
w-l (mean-width left observed-l)
|
||||
w (/ (+ w-r w-l) 2)
|
||||
valid-gaze (seq (keep-indexed (fn [i point]
|
||||
(when (nth gaze-observed i) point)) gaze))
|
||||
gaze (hold-observed gaze gaze-observed {:x 0 :y 0})
|
||||
rest (if valid-gaze (condition/median-point valid-gaze) {:x 0 :y 0})
|
||||
px (mapv (fn [{:keys [x y]}]
|
||||
{:x (* (- x (:x rest)) gaze-gain w)
|
||||
:y (* (- y (:y rest)) gaze-gain w)}) gaze)
|
||||
cells (condition/quantize-snap px (* gaze-step w)
|
||||
(js/Math.round (* fps 0.08)))
|
||||
centres (fn [lids]
|
||||
(mapv (fn [lid shift]
|
||||
(let [{:keys [x y]} (socket lid)]
|
||||
[ (+ x (:x shift)) (+ y (:y shift)) ]))
|
||||
lids cells))
|
||||
blink {:cut blink-cut :dwell 0 :hold (max 2 (js/Math.ceil (* fps 0.1)))}]
|
||||
(assoc measured
|
||||
:lid-r right :lid-l left
|
||||
:lash-r (mapv #(ring/offset-ring % (* lash-weight w-r)) right)
|
||||
:lash-l (mapv #(ring/offset-ring % (* lash-weight w-l)) left)
|
||||
:shut-r (condition/resolve-blink
|
||||
(hold-observed open-r observed-r nil) blink)
|
||||
:shut-l (condition/resolve-blink
|
||||
(hold-observed open-l observed-l nil) blink)
|
||||
:iris-r (centres right) :iris-l (centres left)
|
||||
:radius-r (* 0.5 iris-size w-r)
|
||||
:radius-l (* 0.5 iris-size w-l)
|
||||
:pupil-size (* pupil-size w))))
|
||||
44
frontend/src/arthur/flow/condition/interior.cljs
Normal file
44
frontend/src/arthur/flow/condition/interior.cljs
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
(ns arthur.flow.condition.interior
|
||||
"Teeth presence and temporal contour conditioning. The contrast measurement
|
||||
remains available for diagnosis; visibility is an editable freeze decision."
|
||||
(:require [arthur.domain.geom :as geom]))
|
||||
|
||||
(defn apply-defaults
|
||||
[{:keys [fps teeth-on teeth-dwell teeth-smooth aperture-cut]}
|
||||
measures aperture]
|
||||
(let [peak (reduce max aperture)
|
||||
dwell (or teeth-dwell (js/Math.round (* (or fps 30) 0.08)))
|
||||
raw (mapv (fn [m a]
|
||||
(if (and (:contour m) (pos? peak)
|
||||
(>= (/ a peak) aperture-cut))
|
||||
(:contrast m) 0))
|
||||
measures aperture)
|
||||
shown (loop [f 0 live false since 0 out []]
|
||||
(if (= f (count raw))
|
||||
out
|
||||
(let [want (> (nth raw f) (if live (* teeth-on 0.7) teeth-on))
|
||||
change? (and (not= want live) (>= since dwell))
|
||||
live (if change? want live)
|
||||
since (if change? 0 (inc since))]
|
||||
(recur (inc f) live since
|
||||
(conj out (and live (some? (:contour (nth measures f)))))))))
|
||||
contours (mapv :contour measures)
|
||||
smoothed (mapv (fn [f points]
|
||||
(if (or (nil? points) (not (nth shown f))
|
||||
(zero? teeth-smooth))
|
||||
points
|
||||
(let [near (keep (fn [j]
|
||||
(let [k (max 0 (min (dec (count contours)) j))]
|
||||
(when (and (nth shown k)
|
||||
(= (count points)
|
||||
(count (nth contours k))))
|
||||
(nth contours k))))
|
||||
(range (- f teeth-smooth)
|
||||
(inc (+ f teeth-smooth))))]
|
||||
(if (seq near)
|
||||
(mapv geom/centroid (apply map vector near))
|
||||
points))))
|
||||
(range (count contours)) contours)]
|
||||
{:contours smoothed :shown shown
|
||||
:contrast (mapv :contrast measures)
|
||||
:debug (mapv :debug measures)}))
|
||||
205
frontend/src/arthur/flow/detect.cljs
Normal file
205
frontend/src/arthur/flow/detect.cljs
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
(ns arthur.flow.detect
|
||||
"The only MediaPipe boundary. Landmarks leave here as ordinary CLJS values.")
|
||||
|
||||
(defonce ^:private instance (atom nil))
|
||||
(defonce ^:private pending (atom nil))
|
||||
|
||||
(def settings
|
||||
"Detection and assignment inputs, also included in the analysis address."
|
||||
{:max-faces 4 :assignment "nearest-centroid-v1" :gate 0.2})
|
||||
|
||||
(defn discard!
|
||||
"Throw away the cached landmarker so the next run builds a fresh one.
|
||||
|
||||
Because a MediaPipe graph error is PERMANENT for the instance that hit it. A
|
||||
timestamp that did not advance leaves the graph in an error state, and every
|
||||
later `detectForVideo` on that landmarker re-throws it — so a memoised instance
|
||||
turns one bad run into a tool that is broken until the tab is reloaded. Called
|
||||
from the failure path, not from the happy one: a landmarker costs 26MB of wasm
|
||||
and a model parse, and that is worth keeping for a run that ended cleanly."
|
||||
[]
|
||||
(when-let [model @instance]
|
||||
(try (.call (aget model "close") model) (catch :default _ nil)))
|
||||
(reset! instance nil)
|
||||
(reset! pending nil))
|
||||
|
||||
(defn landmarker!
|
||||
"Initialize once, using the vendored wasm and the local model. CPU also works
|
||||
in browsers where a GPU delegate initializes but fails on its first frame.
|
||||
|
||||
Under `/static/` since step 9: the assets still live in `frontend/public/mediapipe`
|
||||
— 26MB of wasm and model that has no business being copied into a second place in
|
||||
the tree — and Django's staticfiles serves that directory under the `mediapipe/`
|
||||
prefix. Still no CDN, which is the property that matters: the only thing in this
|
||||
tool that would silently require a network is the one thing that must not."
|
||||
[]
|
||||
(if-let [model @instance]
|
||||
(js/Promise.resolve model)
|
||||
(or @pending
|
||||
(if-let [vision (aget js/window "Vision")]
|
||||
(let [resolver (aget vision "FilesetResolver")
|
||||
landmarker (aget vision "FaceLandmarker")
|
||||
ready (-> (.call (aget resolver "forVisionTasks") resolver "/static/mediapipe/wasm")
|
||||
(.then (fn [fileset]
|
||||
(.call (aget landmarker "createFromOptions")
|
||||
landmarker fileset
|
||||
#js {:baseOptions
|
||||
#js {:modelAssetPath "/static/mediapipe/face_landmarker.task"
|
||||
:delegate "CPU"}
|
||||
:runningMode "VIDEO"
|
||||
:numFaces (:max-faces settings)})))
|
||||
(.then (fn [model]
|
||||
(reset! instance model)
|
||||
model))
|
||||
(.catch (fn [e]
|
||||
(reset! pending nil)
|
||||
(throw e))))]
|
||||
(reset! pending ready)
|
||||
ready)
|
||||
(js/Promise.reject (js/Error. "local MediaPipe script did not load"))))))
|
||||
|
||||
(defn detect!
|
||||
"Detect one already-drawn canvas frame at its own time in the take. An empty
|
||||
vector means no face was detected.
|
||||
|
||||
`at-ms` IS THE FRAME'S REAL PRESENTATION TIME, and all three words are
|
||||
load-bearing. In VIDEO mode the graph is a tracker: it runs face DETECTION only
|
||||
when it has lost the face, and otherwise follows the previous frame's region,
|
||||
using the gap between timestamps as the motion it has to account for. So:
|
||||
|
||||
IT MUST INCREASE, STRICTLY. MediaPipe's input streams reject a timestamp that
|
||||
does not advance — `Packet timestamp mismatch on a calculator receiving from
|
||||
stream \"norm_rect\"` — and that error is not recoverable: the graph is left in an
|
||||
error state and every later call on this landmarker throws the same thing. One
|
||||
repeated frame kills the run, so the caller walks frames forward exactly once.
|
||||
|
||||
IT MUST BE MILLISECONDS OF FOOTAGE, not a frame counter. Feeding `i` instead of
|
||||
`i * 1000 / fps` still runs — and measured over the same 91 frames it moved
|
||||
landmarks six times further from the per-frame answer (0.079 of frame width
|
||||
against 0.013), because a tracker told that every frame is 1ms apart expects a
|
||||
face that has barely moved."
|
||||
[model canvas at-ms]
|
||||
(let [faces (aget (.call (aget model "detectForVideo") model canvas at-ms)
|
||||
"faceLandmarks")]
|
||||
(mapv (fn [face] (mapv (fn [p] {:x (.-x p) :y (.-y p) :z (.-z p)}) face))
|
||||
(array-seq faces))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; which face is which
|
||||
;;
|
||||
;; MediaPipe hands back a LIST, and a list has an order rather than an identity.
|
||||
;; Nothing in the API promises that slot 0 is the same person on frame 41 as on
|
||||
;; frame 40 — and when two faces cross, or one is briefly lost and re-detected,
|
||||
;; it is not. Left unassigned, the two faces' geometry would swap mid-shot inside
|
||||
;; one dense block, which reads as both heads snapping and is invisible in any
|
||||
;; per-frame assertion.
|
||||
;;
|
||||
;; So identity is assigned HERE, once, by nearest centroid to each track's last
|
||||
;; known position. Greedy and cheap: a handful of faces, one pass, and the
|
||||
;; ordering it produces is the `:subjects` map every later stage keys on.
|
||||
|
||||
(defn- centroid [face]
|
||||
(let [n (count face)]
|
||||
{:x (/ (reduce + (map :x face)) n)
|
||||
:y (/ (reduce + (map :y face)) n)}))
|
||||
|
||||
(defn- distance [a b]
|
||||
(let [dx (- (:x a) (:x b)) dy (- (:y a) (:y b))]
|
||||
(js/Math.sqrt (+ (* dx dx) (* dy dy)))))
|
||||
|
||||
(defn assign
|
||||
"Per-frame lists of faces -> one vector per TRACK of `detection index or nil`.
|
||||
|
||||
INDICES AND NOT FACES, because a detection is more than its landmarks: the
|
||||
mouth crop and the pixel measurement taken beside it on the same frame have to
|
||||
follow the same face, and handing back the index is what lets one assignment
|
||||
re-key all three. Whoever holds the per-frame lists does the lookup.
|
||||
|
||||
A track is claimed by the unclaimed detection nearest its last known centroid,
|
||||
nearest pair first, so a frame where MediaPipe swaps its slot order does not
|
||||
swap the tracks. A detection that matches no existing track within `gate`
|
||||
starts a new one — which is how a second person walking into the shot on frame
|
||||
200 gets their own subject instead of stealing the first one's.
|
||||
|
||||
`gate` is in normalised image units: two centroids closer than this on
|
||||
consecutive frames are one face moving, and further apart are not."
|
||||
([frames] (assign frames (:gate settings)))
|
||||
([frames gate]
|
||||
(let [step (fn [[rows last] faces]
|
||||
(let [cs (mapv centroid faces)
|
||||
pairs (sort-by :d
|
||||
(for [[t at] (map-indexed vector last)
|
||||
:when at
|
||||
[d c] (map-indexed vector cs)
|
||||
:let [gap (distance at c)]
|
||||
:when (< gap gate)]
|
||||
{:d gap :track t :face d}))
|
||||
claim (reduce (fn [{:keys [by-track taken] :as acc}
|
||||
{:keys [track face]}]
|
||||
(if (or (contains? by-track track)
|
||||
(contains? taken face))
|
||||
acc
|
||||
{:by-track (assoc by-track track face)
|
||||
:taken (conj taken face)}))
|
||||
{:by-track {} :taken #{}}
|
||||
pairs)
|
||||
opened (map-indexed (fn [i d] [(+ (count last) i) d])
|
||||
(remove (:taken claim) (range (count faces))))
|
||||
by-track (into (:by-track claim) opened)
|
||||
width (+ (count last) (count opened))]
|
||||
[(conj rows (mapv by-track (range width)))
|
||||
(mapv (fn [t] (if-let [d (get by-track t)]
|
||||
(nth cs d)
|
||||
(nth last t nil)))
|
||||
(range width))]))
|
||||
[rows _] (reduce step [[] []] frames)
|
||||
width (reduce max 0 (map count rows))]
|
||||
;; Transposed to column-major: one vector per subject, padded to the final
|
||||
;; width so a track that opened late still spans the whole take.
|
||||
(mapv (fn [t] (mapv (fn [row] (nth row t nil)) rows))
|
||||
(range width)))))
|
||||
|
||||
(defn fill-gaps
|
||||
"Keep the detection mask while supplying real poses for measurement. A leading
|
||||
gap uses the first observed face; later gaps hold the previous observed pose.
|
||||
Freeze uses the mask to mark those frames absent in the channel blocks.
|
||||
|
||||
ONE TRACK, which is one subject: the mask says when THIS face was on screen,
|
||||
and two faces in a shot have two of them. A frame where the second person has
|
||||
not walked in yet is a frame their track is absent on, which is the same fact
|
||||
as a frame nobody was found on and needs no second mechanism."
|
||||
[raw]
|
||||
(let [first-real (first (keep-indexed (fn [i frame] (when frame i)) raw))]
|
||||
(when-not first-real
|
||||
(throw (ex-info "no face found in any frame — check framing and light" {})))
|
||||
{:dense (loop [i 0 last-face (nth raw first-real) out []]
|
||||
(if (= i (count raw))
|
||||
out
|
||||
(let [face (or (nth raw i) last-face)]
|
||||
(recur (inc i) face (conj out face)))))
|
||||
:detected (mapv some? raw)
|
||||
:missing (count (remove some? raw))
|
||||
:first-real first-real}))
|
||||
|
||||
(defn subject-id
|
||||
"Track index -> the subject that owns it. `:face-1` is the first face found,
|
||||
which is also the id a single-face take has always used."
|
||||
[i]
|
||||
(keyword (str "face-" (inc i))))
|
||||
|
||||
(defn tracks
|
||||
"Per-frame detection lists -> `{:face-1 <slots>, …}`, one entry per tracked
|
||||
face, each a per-frame detection index or nil."
|
||||
[frames]
|
||||
(let [assigned (assign frames)]
|
||||
(when (empty? assigned)
|
||||
(throw (ex-info "no face found in any frame — check framing and light" {})))
|
||||
(into {} (map-indexed (fn [i slots] [(subject-id i) slots])) assigned)))
|
||||
|
||||
(defn pick
|
||||
"One track's slots applied to anything measured PER DETECTION — the landmarks,
|
||||
the mouth crop, the pixel measurement taken beside it. All three were recorded
|
||||
in detection order on a frame where nobody yet knew whose face was whose, and
|
||||
this is the single lookup that turns any of them into one subject's track."
|
||||
[per-frame slots]
|
||||
(mapv (fn [row slot] (when slot (nth row slot))) per-frame slots))
|
||||
735
frontend/src/arthur/flow/freeze.cljs
Normal file
735
frontend/src/arthur/flow/freeze.cljs
Normal file
|
|
@ -0,0 +1,735 @@
|
|||
(ns arthur.flow.freeze
|
||||
"Stage 5, the freeze: measurements become CHANNELS.
|
||||
|
||||
This is the hinge the whole model turns on. Freezing is NOT a conversion into a
|
||||
second format — there is one format, and freezing fills it in. That is what
|
||||
makes \"the only difference between rotoscoped and hand-authored is a flag\"
|
||||
literally true: what comes out of here is the same `:channels` map a hand fills
|
||||
in sparsely, and the flag is `:generated`, which nothing in the renderer reads.
|
||||
|
||||
Three conversions happen here and nowhere else.
|
||||
|
||||
MAPS BECOME FLAT. `flow/measure/*` speaks {:x :y}, because it is the numeric
|
||||
oracle and a faithful port was worth more there than a fast one. A channel
|
||||
value is FLAT — [x0 y0 x1 y1 …] — in authored vectors and dense blocks alike.
|
||||
`rings->flat` is the only place that crossing is made.
|
||||
|
||||
FLOATS BECOME FIXED POINT. Geometry goes into an Int16 block with the scale in
|
||||
its header; see `geom-scale`.
|
||||
|
||||
A SIMILARITY BECOMES THREE CHANNELS. `{s θ tx ty}` IS `[:xform :scale]`,
|
||||
`[:xform :rot]` and `[:xform :pos]`, so the anchor drops onto `:head` with no
|
||||
adapter — which is the sign the decomposition is the right one.
|
||||
|
||||
`makeXform` IS NOT HERE AND IS NOT COMING. The prototype centres on the face
|
||||
oval's bbox and zooms until the face is 80% of the raster height, so every
|
||||
stored vertex carries a cropping decision made once, at analysis time, from one
|
||||
frame's landmarks. Here the geometry stays in the node's own local space and the
|
||||
framing is `[:xform :*]` on an authored `:face` node, which the stage clips.
|
||||
Project dimensions are therefore independent of the footage — see
|
||||
`face-placement`, and \"What space geometry is in\" in docs/animation-model.md.
|
||||
|
||||
Not `flow/key`. Traced lips, lids and brows keep every source frame; only a
|
||||
plate, which a human draws, is worth decimating. Sparse visibility keys capture
|
||||
decisions about the mouth cavity, blink and teeth without thinning geometry."
|
||||
(:require [arthur.domain.channel :as ch]
|
||||
[arthur.domain.clip :as clip]
|
||||
[arthur.domain.feature :as feature]
|
||||
[arthur.domain.geom :as geom]
|
||||
[arthur.domain.ring :as ring]
|
||||
[arthur.flow.address :as address]))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; fixed point
|
||||
|
||||
(def ^:const geom-scale
|
||||
"Q14: the stored integer is the value times 16384.
|
||||
|
||||
Geometry is head-local and its unit is ONE IMAGE HEIGHT, so 16384 covers ±2
|
||||
image heights in an Int16 and quantises to 1/16384 = 6.1e-5 of an image height.
|
||||
Against a face placed so that its 0.29-image-height oval fills most of a 200px
|
||||
stage — around 850 stage pixels per image height — that is 0.05px, two orders
|
||||
below anything the rasteriser can express.
|
||||
|
||||
A power of two, so the decode is a floating-point exact division and freezing
|
||||
the same numbers twice cannot drift.
|
||||
|
||||
It is a CONSTANT here and a FIELD in the block header, and the difference
|
||||
matters: a painted cel's geometry is in stage pixels, where ±2 would be absurd
|
||||
and 1/16384 of a pixel is waste. Each block says what its own space needs."
|
||||
16384)
|
||||
|
||||
(def ^:private ^:const int16-max 32767)
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; blocks
|
||||
|
||||
(defn- pack
|
||||
"Tracks -> one dense block, NODE-MAJOR and FRAME-MINOR:
|
||||
|
||||
offset(track i) = i · frames · stride
|
||||
value(i, f) = data[offset(i) + f · stride]
|
||||
|
||||
No per-frame header and no indirection, which FIXED TOPOLOGY is what buys: every
|
||||
frame of a part carries the same component count with the same meanings, so a
|
||||
frame is a rectangular slice at an arithmetic offset. A variable vertex count
|
||||
would force an offset table and a scan per frame, so the aesthetic constraint is
|
||||
a performance asset rather than a cost. `demo/swarm` holds the same layout and
|
||||
is the load test for it.
|
||||
|
||||
`:type` names the array — \"int16\" or \"float32\" — rather than handing over a
|
||||
constructor, because the type is also a field in the block's descriptor and the
|
||||
two must not be able to disagree. `:scale` is the fixed-point scale or nil.
|
||||
|
||||
ABSENCE IS PER TRACK, and `:features` is what says whose. Each track names the
|
||||
feature it follows, so one occluded eye can be absent while its partner still
|
||||
has a value; a subject id means the track follows whole-face detection and no
|
||||
feature, which is what the head's blocks do. `absent?` is then asked
|
||||
`(absent? feature f)` and never about a track index.
|
||||
|
||||
An earlier shape passed a `(track, frame)` predicate instead, and each call site
|
||||
derived a feature from an index — `(if (< i 2) :eye-r :eye-l)` — so the
|
||||
predicate and the vector of tracks beside it had to agree BY HAND, in five
|
||||
places, with a left/right swap for a failure mode. docs/port-plan.md warns about
|
||||
that swap twice: every part is still roughly where it belongs, so it survives
|
||||
inspection. Naming the feature per track deletes the derivation, and it hands
|
||||
`flow/address` the same list for the block's observation digest, so the key and
|
||||
the mask cannot disagree either.
|
||||
|
||||
`:missing` is an additional per-track predicate for absence that is not a
|
||||
feature's: the teeth have no contour on a frame no contour could be extracted
|
||||
from, which is a different fact from the teeth being occluded.
|
||||
|
||||
Written with `dotimes` and `aset` rather than as a fold, and that is the
|
||||
exception rather than the rule in this codebase: the destination is a typed
|
||||
array, so there is nothing to accumulate into and a collection idiom here would
|
||||
allocate a seq per frame to throw away."
|
||||
[{:keys [type scale features absent? missing]} tracks]
|
||||
(when-not (= (count tracks) (count features))
|
||||
(throw (ex-info "every track of a block names the feature it follows"
|
||||
{:tracks (count tracks) :features (count features)})))
|
||||
(let [n (count tracks)
|
||||
nf (count (first tracks))
|
||||
stride (count (first (first tracks)))
|
||||
ctor (case type
|
||||
"int16" #(js/Int16Array. %)
|
||||
"float32" #(js/Float32Array. %)
|
||||
(throw (ex-info "a block's element type is \"int16\" or \"float32\""
|
||||
{:type type})))
|
||||
data (ctor (* n nf stride))
|
||||
gone? (fn [i f] (or (and absent? (absent? (nth features i) f))
|
||||
(and missing (missing i f))))
|
||||
state (when (or absent? missing) (js/Uint8Array. (* n nf)))]
|
||||
(dotimes [i n]
|
||||
(let [track (vec (nth tracks i))
|
||||
base (* i nf stride)]
|
||||
(dotimes [f nf]
|
||||
(let [vs (nth track f)
|
||||
o (+ base (* f stride))]
|
||||
(dotimes [k stride]
|
||||
(let [v (nth vs k)]
|
||||
(aset data (+ o k)
|
||||
(if scale
|
||||
(let [q (js/Math.round (* v scale))]
|
||||
;; Saturating would read as articulation flattening off
|
||||
;; at the extremes — a bad detection, not a bad scale.
|
||||
(when (> (abs q) int16-max)
|
||||
(throw (ex-info "value does not fit the block's fixed point"
|
||||
{:value v :scale scale :quantised q
|
||||
:track i :frame f :component k})))
|
||||
q)
|
||||
v))))
|
||||
(when (and state (gone? i f))
|
||||
(aset state (+ (* i nf) f) ch/absent-bit))))))
|
||||
{:data data :state state :stride stride :frames nf :scale scale :type type
|
||||
:features features
|
||||
:offsets (mapv #(* % nf stride) (range n))}))
|
||||
|
||||
(defn- block
|
||||
"Pack the tracks and NAME the result: `pack`'s block plus the `:key` it is
|
||||
stored under and the `:descriptor` that key is the hash of.
|
||||
|
||||
Addressing happens HERE, beside the packing, rather than at the call sites,
|
||||
because a block referenced under one key and stored under another is a handle
|
||||
into somebody else's array — the failure the key exists to make impossible.
|
||||
|
||||
`spec` is the block's identity for `flow/address`: its role, the tracks by name,
|
||||
and the analysis and settings its bytes came out of. `obs` is the absence data,
|
||||
which the descriptor digests down to one line."
|
||||
[{:keys [role analysis params tracks] :as spec} opts obs values]
|
||||
(let [features (:features opts)
|
||||
blk (pack opts values)
|
||||
named (address/block
|
||||
{:role role :analysis analysis :params params :tracks tracks
|
||||
:features features
|
||||
:observation (address/observation features obs)
|
||||
:layout {:type (:type blk) :scale (:scale blk)
|
||||
:stride (:stride blk) :frames (:frames blk)
|
||||
:tracks (count tracks)}})]
|
||||
(merge blk named)))
|
||||
|
||||
(defn- stored
|
||||
"Blocks -> the tier-2 store they go in: key -> what is kept under it.
|
||||
|
||||
THE DESCRIPTOR TRAVELS WITH THE BYTES. It is not in the document — tier 1 stays
|
||||
the authored layer and a descriptor is derived — and it is not thrown away
|
||||
either, because the server verifies `sha256(descriptor) == key` on upload and
|
||||
will not take a name on trust. So it rides in tier 2, where a cache entry
|
||||
knowing what produced it is the ordinary arrangement. `channel/dense-at` reads
|
||||
`:data` and `:state` and ignores the rest."
|
||||
[& blocks]
|
||||
(into {} (map (juxt :key #(select-keys % [:data :state :descriptor]))) blocks))
|
||||
|
||||
(defn- dense
|
||||
"Track i of a packed block, as a DENSE channel definition.
|
||||
|
||||
The block names its own key, so a channel cannot be pointed at one block and
|
||||
stored under another's address."
|
||||
[blk i generated]
|
||||
{:animated? true :interp :hold
|
||||
:dense (cond-> {:store (:key blk)
|
||||
:offset (nth (:offsets blk) i)
|
||||
:stride (:stride blk)
|
||||
:frames (:frames blk)}
|
||||
(:scale blk) (assoc :scale (:scale blk)))
|
||||
:generated generated
|
||||
:over []})
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; geometry
|
||||
|
||||
(defn rings->flat
|
||||
"Ring track -> one flat [x0 y0 x1 y1 …] per frame, at the vertex budget.
|
||||
|
||||
The vertex budget is a fixed-index SUBSAMPLE, never adaptive decimation: slot k
|
||||
means the same anatomy on every frame of the shot, and that is what makes
|
||||
temporal correspondence possible at all. It is applied here, after stage 4's
|
||||
contour average, and the two commute because both are per-slot — which is the
|
||||
whole reason the vertex knob can sit downstream of the smoothing knob instead
|
||||
of alongside it. `mouth-test` asserts that rather than leaving it to look
|
||||
obvious.
|
||||
|
||||
An ODD budget is refused. It lands off the cardinal slots — the corners and the
|
||||
lip centres — and a wrongly-ordered ring self-intersects INVISIBLY at odd vertex
|
||||
counts and obviously at even ones, so an odd budget is the one setting at which
|
||||
the simplicity assertion stops protecting anything."
|
||||
[rings verts]
|
||||
(let [len (count (first rings))]
|
||||
(when-not (and (integer? verts) (even? verts) (>= verts 4) (<= verts len))
|
||||
(throw (ex-info "vertex budget must be even and between 4 and the ring's slot count"
|
||||
{:verts verts :slots len})))
|
||||
(let [slots (ring/subsample-slots len verts)]
|
||||
(mapv (fn [r]
|
||||
(into [] (mapcat (fn [s] (let [p (nth r s)] [(:x p) (:y p)]))) slots))
|
||||
rings))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the anchor, onto :head
|
||||
|
||||
(defn invert
|
||||
"The inverse of a similarity, STILL FACTORED.
|
||||
|
||||
The anchor fit maps each frame onto the shot's mean pose, so it is what takes
|
||||
the head's motion OUT; geometry is stored in the space it produces. Putting the
|
||||
head's motion back — the \"as filmed\" mode — is therefore the fit's inverse, and
|
||||
it has to stay a {s θ t} rather than becoming a matrix, because the three
|
||||
components land on three independently keyframable channels and interpolating
|
||||
matrix entries is meaningless.
|
||||
|
||||
p ↦ s·R(θ)·p + t inverts to q ↦ (1/s)·R(-θ)·(q - t)"
|
||||
[{:keys [s theta tx ty]}]
|
||||
(let [s' (/ 1.0 s)
|
||||
c (js/Math.cos theta)
|
||||
sn (js/Math.sin theta)]
|
||||
{:s s'
|
||||
:theta (- theta)
|
||||
:tx (* (- s') (+ (* c tx) (* sn ty)))
|
||||
:ty (* (- s') (+ (* (- sn) tx) (* c ty)))}))
|
||||
|
||||
(def ^:private head-modes #{:free :anchored})
|
||||
|
||||
(defn head-mode
|
||||
"Keep a subject's measured transform dense; optionally hold chosen source
|
||||
frames.
|
||||
|
||||
A nil anchor map reads measured frame f at frame f (free movement).
|
||||
`{0 12}` locks to the measured transform of source frame 12. `{0 12, 40 42}`
|
||||
cuts to source frame 42 at local frame 40. The same map selects position,
|
||||
rotation and scale, so the head and registered photo cannot drift apart.
|
||||
No analysis block or authored face placement changes.
|
||||
|
||||
ONE SUBJECT AT A TIME when `:subject` is given, and EVERY subject when it is
|
||||
not. Two faces in one shot were filmed together and are posed apart: choosing
|
||||
frame 12 for the second face must leave the first one running, and it does,
|
||||
because an anchor map lives on that subject's own head node and
|
||||
`domain/timeline` reads anchors off whatever node carries them."
|
||||
[{:keys [subject mode anchors]} {:keys [clip]}]
|
||||
(when-not (contains? head-modes mode)
|
||||
(throw (ex-info "head mode must be free or anchored"
|
||||
{:mode mode :modes head-modes})))
|
||||
(when (and (= mode :free) (some? anchors))
|
||||
(throw (ex-info "free head motion has no anchors" {:anchors anchors})))
|
||||
(when (and subject (not (contains? (:subjects clip) subject)))
|
||||
(throw (ex-info "head mode names a subject this clip did not track"
|
||||
{:subject subject :subjects (vec (sort-by str (keys (:subjects clip))))})))
|
||||
(reduce
|
||||
(fn [c sid]
|
||||
(let [frames (get-in c [:timelines sid :frames])]
|
||||
(when (and (= mode :anchored)
|
||||
(not (and (map? anchors) (contains? anchors 0)
|
||||
(every? #(and (integer? %) (<= 0 %) (< % frames))
|
||||
(concat (keys anchors) (vals anchors))))))
|
||||
(throw (ex-info "anchored head needs a frame-zero key and valid source frames"
|
||||
{:subject sid :anchors anchors :frames frames})))
|
||||
(update-in c [:timelines sid :nodes :head]
|
||||
(fn [n]
|
||||
(cond-> (assoc n :channels (:measured n))
|
||||
(= mode :anchored) (assoc :anchors anchors)
|
||||
(= mode :free) (dissoc :anchors))))))
|
||||
clip (if subject [subject] (sort-by str (keys (:subjects clip))))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the aperture, onto [:vis] of :mouth-in
|
||||
|
||||
(defn visibility
|
||||
"The aperture track -> `[:vis]` keys on the mouth interior.
|
||||
|
||||
`flow/measure/mouth` reports the aperture and deliberately does not threshold
|
||||
it: the measurement is the inner ring's own height and the threshold is a
|
||||
policy, which is stage 5. Relative to the take's PEAK aperture, not absolute, so
|
||||
one number works across faces and framings — the prototype's
|
||||
`ap.map(v => v / apMax < apertureThresh)`, which lives in its `app.js` and not
|
||||
in its `pipeline.js`, and is easy to miss when porting from the latter.
|
||||
|
||||
KEYED, not dense, although docs/animation-model.md's parts table says dense. A
|
||||
threshold crossing is a handful of transitions over a take, hold is the default,
|
||||
and keys are what a human can correct — \"this frame's mouth should be shut\" is
|
||||
the single most likely hand edit on a lip-sync take, and a dense block in tier 2
|
||||
is the one shape that cannot receive it. `:generated` still rides along, which is
|
||||
the point of provenance being on the channel rather than implied by its shape.
|
||||
|
||||
A key on frame 0 always, because the first key is the pose the part starts in."
|
||||
[{:keys [aperture-cut]} {:keys [aperture]} generated]
|
||||
(let [peak (reduce max aperture)
|
||||
;; A take with no mouth at all has no peak to be a fraction of. Present
|
||||
;; rather than absent: an all-zero aperture is a shut mouth, and the
|
||||
;; interior of a shut mouth is simply not drawn.
|
||||
shown (mapv (fn [v] (and (pos? peak) (>= (/ v peak) aperture-cut))) aperture)]
|
||||
(assoc (ch/keyed (into {} (keep (fn [f]
|
||||
(when (or (zero? f)
|
||||
(not= (nth shown f) (nth shown (dec f))))
|
||||
[f (nth shown f)])))
|
||||
(range (count shown))))
|
||||
:generated generated)))
|
||||
|
||||
(defn- keyed-visibility [values generated]
|
||||
(assoc (ch/keyed (into {} (keep (fn [f]
|
||||
(when (or (zero? f)
|
||||
(not= (nth values f) (nth values (dec f))))
|
||||
[f (nth values f)])))
|
||||
(range (count values))))
|
||||
:generated generated))
|
||||
|
||||
(def ^:private pose-groups
|
||||
{:mouth :mouth :mouth-in :mouth :teeth :mouth
|
||||
:eye-r :eye-r :eye-r-in :eye-r :iris-r :eye-r :pupil-r :eye-r
|
||||
:eye-l :eye-l :eye-l-in :eye-l :iris-l :eye-l :pupil-l :eye-l
|
||||
:brow-r :brow-r :brow-l :brow-l})
|
||||
|
||||
(defn- performance-nodes
|
||||
"Mark channels that read the containing instance's pose choices."
|
||||
[nodes]
|
||||
(into {}
|
||||
(map (fn [[id n]]
|
||||
[id (if-let [group (get pose-groups id)]
|
||||
(-> n
|
||||
(assoc :pose-group group)
|
||||
(update :channels
|
||||
(fn [channels]
|
||||
(into {}
|
||||
(map (fn [[path ch]]
|
||||
[path (cond-> ch
|
||||
(and (:generated ch) (:animated? ch))
|
||||
(assoc :pose-sampled? true))]))
|
||||
channels))))
|
||||
n)]))
|
||||
nodes))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the face, onto the stage
|
||||
|
||||
(defn- motion-points
|
||||
"Every source-space point one subject's drawn features visit over the shot,
|
||||
including raised brows and a moving mouth."
|
||||
[{:keys [rigid transforms outer eyes brows detected]}]
|
||||
(mapcat (fn [i]
|
||||
(when (or (nil? detected) (nth detected i))
|
||||
(let [local (concat (nth outer i)
|
||||
(when eyes (concat (nth (:lash-r eyes) i)
|
||||
(nth (:lash-l eyes) i)))
|
||||
(when brows (concat (nth (:ring-r brows) i)
|
||||
(nth (:ring-l brows) i))))]
|
||||
(concat (nth rigid i)
|
||||
(geom/apply-sim-all (invert (nth transforms i))
|
||||
local)))))
|
||||
(range (count outer))))
|
||||
|
||||
(defn- fit-points
|
||||
"Source-space points -> the placement that puts their bounding box on stage."
|
||||
[[w h] points]
|
||||
(let [xs (map :x points)
|
||||
ys (map :y points)
|
||||
x0 (reduce min xs)
|
||||
x1 (reduce max xs)
|
||||
y0 (reduce min ys)
|
||||
y1 (reduce max ys)
|
||||
cx (/ (+ x0 x1) 2)
|
||||
cy (/ (+ y0 y1) 2)
|
||||
k (min (/ (* 0.8 w) (max 1e-9 (- x1 x0)))
|
||||
(/ (* 0.8 h) (max 1e-9 (- y1 y0))))]
|
||||
{[:xform :anchor] (ch/framed [cx cy])
|
||||
[:xform :scale] (ch/framed [k k])
|
||||
[:xform :pos] (ch/framed [(- (/ w 2) cx) (- (/ h 2) cy)])}))
|
||||
|
||||
(defn face-placement
|
||||
"The face's transform on the stage, as FRAMED channels.
|
||||
|
||||
AUTHORED, and that is the whole difference from `makeXform`. What comes back is
|
||||
a DEFAULT — the placement a human would otherwise have to make from scratch on
|
||||
first open — and from then on it is an ordinary hand-placed transform on an
|
||||
ordinary node. `makeXform` made the same decision and then baked it into every
|
||||
vertex, where nothing could ever revise it.
|
||||
|
||||
The synthetic take uses the reference rigid configuration for its default.
|
||||
Real footage can request `:fit-motion?`: its default fits the observed mouth,
|
||||
eyes and brows in the stage across the shot.
|
||||
Both are ordinary editable transforms on :face, never baked into the geometry.
|
||||
The face oval is not measured, because its only consumers in the prototype were
|
||||
the old baked framing transform and the placeholder plate outline.
|
||||
|
||||
For the synthetic default, two numbers:
|
||||
|
||||
SCALE is stage pixels per image height, set so the reference's eye-corner span
|
||||
is 40% of the stage width. Landmark-free — it is the rigid configuration's own
|
||||
bounding box — and it is a fraction of the STAGE, so a 1440x1920 portrait clip
|
||||
composited onto a 320x200 stage is not a problem to solve.
|
||||
|
||||
ANCHOR is the reference centroid, and this is where `:anchor` earns its place.
|
||||
MediaPipe's normalised space has its origin at the image's TOP-LEFT CORNER, so
|
||||
head-local geometry is not centred on anything; the registration point of the
|
||||
face is the head's own centre, and rotation and scale have to happen about that
|
||||
rather than about a corner of the footage. Getting that wrong is why hand-placed
|
||||
parts swing rather than turn.
|
||||
|
||||
POSITION puts the anchor a QUARTER of the way down the stage, because the rigid
|
||||
landmarks are eyes and nose — the upper middle of a face — so a quarter down
|
||||
leaves the jaw and the mouth on the stage. Whatever hangs off is clipped, which
|
||||
is not a feature to add: every fill in `domain/raster` clamps already.
|
||||
|
||||
All subjects share one source-to-stage mapping on the :face group. Each
|
||||
instance can then be placed independently with ordinary transform channels."
|
||||
[{:keys [stage fit-motion?]} subjects]
|
||||
(let [[w h] stage
|
||||
inputs (vals subjects)]
|
||||
(if fit-motion?
|
||||
(fit-points stage (mapcat motion-points inputs))
|
||||
(let [ref (mapcat :ref inputs)
|
||||
c (geom/centroid ref)
|
||||
span (- (reduce max (map :x ref)) (reduce min (map :x ref)))
|
||||
k (/ (* 0.4 w) span)]
|
||||
{[:xform :anchor] (ch/framed [(:x c) (:y c)])
|
||||
[:xform :scale] (ch/framed [k k])
|
||||
[:xform :pos] (ch/framed [(- (/ w 2) (:x c))
|
||||
(- (* 0.25 h) (:y c))])}))))
|
||||
|
||||
(defn- mouth-part [subject absent? obs
|
||||
{:keys [analysis verts anchor-avg contour-avg aperture-cut] :as params}
|
||||
{:keys [outer inner] :as inputs}]
|
||||
(let [own (partial feature/owned subject)
|
||||
mouth (own :mouth)
|
||||
rings (block {:role "geom" :analysis (:id analysis) :params params
|
||||
:tracks ["outer" "inner"]}
|
||||
{:type "int16" :scale geom-scale
|
||||
:features [mouth mouth] :absent? absent?}
|
||||
obs [(rings->flat outer verts) (rings->flat inner verts)])
|
||||
prov (fn [by]
|
||||
{:by by :analysis (:id analysis)
|
||||
:params {:anchor-avg anchor-avg :contour-avg contour-avg
|
||||
:verts verts}})]
|
||||
{:nodes {:mouth
|
||||
{:id :mouth :name "mouth" :kind :poly :parent :head :z "a1"
|
||||
:channels {[:geom :pts] (dense rings 0 (prov :roto/lips-outer))
|
||||
[:style :color] (ch/framed :skin-dark)}}
|
||||
:mouth-in
|
||||
{:id :mouth-in :name "mouth interior" :kind :poly
|
||||
:parent :mouth :z "a2"
|
||||
:channels {[:geom :pts] (dense rings 1 (prov :roto/lips-inner))
|
||||
[:style :color] (ch/framed :mouth-dark)
|
||||
[:vis] (visibility params inputs
|
||||
{:by :roto/mouth-aperture
|
||||
:analysis (:id analysis)
|
||||
:params {:anchor-avg anchor-avg
|
||||
:aperture-cut aperture-cut}})}}}
|
||||
:store (stored rings)}))
|
||||
|
||||
(defn- feature-parts
|
||||
"Freeze eyes and brows into their own dense blocks and scene nodes. This owns
|
||||
only representation: the landmark correspondence, blink and pose choices have
|
||||
already been settled by measure and condition."
|
||||
[subject absent? obs
|
||||
{:keys [eye-verts brow-verts analysis contour-avg anchor-avg] :as params}
|
||||
{:keys [eyes brows]}]
|
||||
(let [own (partial feature/owned subject)
|
||||
provenance (fn [by extra]
|
||||
{:by by :analysis (:id analysis)
|
||||
:params (merge {:anchor-avg anchor-avg :contour-avg contour-avg}
|
||||
extra)})
|
||||
named (fn [role tracks features type values]
|
||||
(block {:role role :analysis (:id analysis) :params params
|
||||
:tracks tracks}
|
||||
{:type type :features features :absent? absent?
|
||||
:scale (when (= "int16" type) geom-scale)}
|
||||
obs values))
|
||||
;; Each block's tracks, named, in the order they are packed — and the
|
||||
;; feature each one follows, in the same order. The two vectors are read
|
||||
;; together on purpose: this is the mapping `pack` cannot check for itself,
|
||||
;; and `each-dense-track-follows-its-own-features-presence` is what pins it.
|
||||
eye-block (when eyes (named "eyes"
|
||||
["lash-r" "lid-r" "lash-l" "lid-l"]
|
||||
[(own :eye-r) (own :eye-r) (own :eye-l) (own :eye-l)]
|
||||
"int16"
|
||||
(mapv #(rings->flat % eye-verts)
|
||||
[(:lash-r eyes) (:lid-r eyes)
|
||||
(:lash-l eyes) (:lid-l eyes)])))
|
||||
iris-block (when eyes
|
||||
(named "iris-pos" ["iris-r" "iris-l"]
|
||||
[(own :eye-r) (own :eye-l)] "float32"
|
||||
[(:iris-r eyes) (:iris-l eyes)]))
|
||||
brow-block (when brows
|
||||
(named "brows" ["ring-r" "ring-l"]
|
||||
[(own :brow-r) (own :brow-l)] "int16"
|
||||
(mapv #(rings->flat % brow-verts)
|
||||
[(:ring-r brows) (:ring-l brows)])))
|
||||
brow-pos-block (when brows
|
||||
(named "brow-pos" ["pos-r" "pos-l"]
|
||||
[(own :brow-r) (own :brow-l)] "float32"
|
||||
[(:pos-r brows) (:pos-l brows)]))
|
||||
eye-node (fn [id z track]
|
||||
{:id id :name (clojure.core/name id) :kind :poly :parent :head :z z
|
||||
:channels {[:geom :pts] (dense eye-block track
|
||||
(provenance :roto/eyelid {:verts eye-verts}))
|
||||
[:style :color] (ch/framed :skin-dark)}})
|
||||
inner-node (fn [id parent z track shut]
|
||||
{:id id :name (clojure.core/name id) :kind :poly :parent parent :z z
|
||||
:channels {[:geom :pts] (dense eye-block track
|
||||
(provenance :roto/eye-opening
|
||||
{:verts eye-verts}))
|
||||
[:style :color] (ch/framed :eye-white)
|
||||
[:vis] (keyed-visibility (mapv not shut)
|
||||
(provenance :roto/blink nil))}})
|
||||
iris-node (fn [id parent track radius]
|
||||
{:id id :name (clojure.core/name id) :kind :disc :parent parent :z "a1"
|
||||
:stencil parent
|
||||
:channels {[:xform :pos] (dense iris-block track
|
||||
(provenance :roto/gaze nil))
|
||||
[:geom :radius] (assoc (ch/framed radius)
|
||||
:generated
|
||||
(provenance :roto/iris-size
|
||||
{:iris-size (:iris-size params)}))
|
||||
[:style :color] (ch/framed :iris)}})
|
||||
pupil-node (fn [id parent]
|
||||
{:id id :name (clojure.core/name id) :kind :rect :parent parent :z "a1"
|
||||
:stencil parent
|
||||
:channels {[:geom :size] (assoc (ch/framed (:pupil-size eyes))
|
||||
:generated
|
||||
(provenance :roto/pupil-size
|
||||
{:pupil-size (:pupil-size params)}))
|
||||
[:style :color] (ch/framed :pupil)}})
|
||||
brow-node (fn [id z track]
|
||||
{:id id :name (clojure.core/name id) :kind :poly :parent :head :z z
|
||||
:channels {[:geom :pts] (dense brow-block track
|
||||
(provenance :roto/brow {:verts brow-verts}))
|
||||
[:xform :pos] (dense brow-pos-block track
|
||||
(provenance :roto/brow-raise nil))
|
||||
[:style :color] (ch/framed :brow)}})]
|
||||
{:nodes (merge
|
||||
(when eyes
|
||||
{:eye-r (eye-node :eye-r "a2" 0)
|
||||
:eye-r-in (inner-node :eye-r-in :eye-r "a1" 1 (:shut-r eyes))
|
||||
:iris-r (iris-node :iris-r :eye-r-in 0 (:radius-r eyes))
|
||||
:pupil-r (pupil-node :pupil-r :iris-r)
|
||||
:eye-l (eye-node :eye-l "a3" 2)
|
||||
:eye-l-in (inner-node :eye-l-in :eye-l "a1" 3 (:shut-l eyes))
|
||||
:iris-l (iris-node :iris-l :eye-l-in 1 (:radius-l eyes))
|
||||
:pupil-l (pupil-node :pupil-l :iris-l)})
|
||||
(when brows
|
||||
{:brow-r (brow-node :brow-r "a4" 0)
|
||||
:brow-l (brow-node :brow-l "a5" 1)}))
|
||||
:store (apply stored (remove nil?
|
||||
[eye-block iris-block brow-block brow-pos-block]))}))
|
||||
|
||||
(defn- interior-part
|
||||
"Freeze the pixel-derived radial contour under the mouth cavity. Missing
|
||||
contours use the dense block's absence bit; contrast decides editable :vis."
|
||||
[subject {:keys [analysis teeth-verts cavity-erode tongue-reject blob-grow
|
||||
top-bias teeth-on teeth-smooth] :as params} absent? obs
|
||||
{:keys [contours shown]}]
|
||||
(let [own (partial feature/owned subject)
|
||||
empty-points (vec (repeat (* 2 teeth-verts) 0))
|
||||
values (mapv (fn [ring]
|
||||
(if ring
|
||||
(into [] (mapcat (juxt :x :y)) ring)
|
||||
empty-points)) contours)
|
||||
blk (block {:role "teeth" :analysis (:id analysis) :params params
|
||||
:tracks ["contour"]}
|
||||
{:type "int16" :scale geom-scale
|
||||
:features [(own :teeth)] :absent? absent?
|
||||
;; Not the feature's absence: a frame no contour could be
|
||||
;; extracted from has no teeth to draw whether or not the
|
||||
;; teeth were occluded, and the two reasons are different facts.
|
||||
:missing (fn [_ f] (nil? (nth contours f)))}
|
||||
obs [values])
|
||||
generated {:by :pixels/teeth :analysis (:id analysis)
|
||||
:params {:cavity-erode cavity-erode
|
||||
:tongue-reject tongue-reject :blob-grow blob-grow
|
||||
:top-bias top-bias :teeth-verts teeth-verts
|
||||
:teeth-on teeth-on :teeth-smooth teeth-smooth}}]
|
||||
{:nodes {:teeth
|
||||
{:id :teeth :name "teeth" :kind :poly
|
||||
:parent :mouth-in :z "a1"
|
||||
:stencil :mouth-in
|
||||
:channels {[:geom :pts] (dense blk 0 generated)
|
||||
[:style :color] (ch/framed :teeth)
|
||||
[:vis] (keyed-visibility shown generated)}}}
|
||||
:store (stored blk)}))
|
||||
|
||||
(defn part
|
||||
"One feature type: local nodes and blocks addressed by subject and feature."
|
||||
[subject area params {:keys [detected presence] :as measured}]
|
||||
(let [presence (into {} (map (fn [[role mask]] [(feature/owned subject role) mask])) presence)
|
||||
absent? (when (or detected presence)
|
||||
(fn [id f]
|
||||
(or (and detected (not (nth detected f true)))
|
||||
(and (contains? presence id)
|
||||
(not (nth (get presence id) f))))))
|
||||
obs {:detected detected :presence presence}]
|
||||
(update (case area
|
||||
:mouth (mouth-part subject absent? obs params measured)
|
||||
:eye (feature-parts subject absent? obs params (select-keys measured [:eyes]))
|
||||
:brow (feature-parts subject absent? obs params (select-keys measured [:brows]))
|
||||
:teeth (interior-part subject params absent? obs (:teeth measured))
|
||||
(throw (ex-info "unknown frozen feature type" {:area area})))
|
||||
:nodes performance-nodes)))
|
||||
|
||||
(defn head-part
|
||||
"Freeze one subject's measured head transform from its conditioned anchor.
|
||||
|
||||
THE THREE BLOCKS NAME THE SUBJECT, and that is not decoration. A block's key is
|
||||
a hash over its descriptor, and the head follows DETECTION rather than any
|
||||
feature's presence — so with `nil` in the feature slot, two faces tracked in one
|
||||
analysis, both detected on every frame, produced byte-for-byte different
|
||||
transforms under one identical key, and the second freeze's block silently
|
||||
replaced the first's. The subject is the feature the head follows."
|
||||
[subject {:keys [analysis anchor-avg] :as params} {:keys [transforms detected presence]}]
|
||||
(let [absent? (when (or detected presence)
|
||||
(fn [_ f] (and detected (not (nth detected f true)))))
|
||||
inv (mapv invert transforms)
|
||||
xf (fn [role f]
|
||||
(block {:role role :analysis (:id analysis) :params params
|
||||
:tracks [role]}
|
||||
{:type "float32" :features [subject] :absent? absent?}
|
||||
{:detected detected}
|
||||
[(mapv f inv)]))
|
||||
pos (xf "head-pos" (fn [t] [(:tx t) (:ty t)]))
|
||||
rot (xf "head-rot" (fn [t] [(:theta t)]))
|
||||
scale (xf "head-scale" (fn [t] [(:s t) (:s t)]))
|
||||
prov {:by :anchor/similarity :analysis (:id analysis)
|
||||
:params {:anchor-avg anchor-avg}}]
|
||||
{:measured {[:xform :pos] (dense pos 0 prov)
|
||||
[:xform :rot] (dense rot 0 prov)
|
||||
[:xform :scale] (dense scale 0 prov)}
|
||||
:store (stored pos rot scale)}))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; the clip
|
||||
|
||||
(defn- subject-part
|
||||
"A subject's drawing, metadata and blocks. Node names are timeline-local."
|
||||
[params subject {:keys [outer eyes brows teeth] :as inputs}]
|
||||
(let [own (partial feature/owned subject)
|
||||
areas (cond-> [:mouth] (and eyes brows) (into [:eye :brow]) teeth (conj :teeth))
|
||||
parts (mapv #(part subject % params inputs) areas)
|
||||
head (head-part subject params inputs)
|
||||
features (cond-> {:mouth [:mouth [:mouth :mouth-in]]}
|
||||
(and eyes brows)
|
||||
(merge {:eye-r [:eye [:eye-r :eye-r-in :iris-r :pupil-r]]
|
||||
:eye-l [:eye [:eye-l :eye-l-in :iris-l :pupil-l]]
|
||||
:brow-r [:brow [:brow-r]] :brow-l [:brow [:brow-l]]})
|
||||
teeth (assoc :teeth [:teeth [:teeth]]))]
|
||||
{:timeline {:id subject :frames (count outer)
|
||||
:nodes (into {:head {:id :head :name "head" :kind :group :z "a1"
|
||||
:measured (:measured head)}}
|
||||
(mapcat :nodes) parts)}
|
||||
:features (into {} (map (fn [[role [area nodes]]]
|
||||
[(own role) {:id (own role) :subject subject
|
||||
:timeline subject :area area
|
||||
:nodes nodes :params {}}])) features)
|
||||
:groups (if (and eyes brows)
|
||||
{(own :eyes) {:id (own :eyes) :kind :eye-pair :subject subject
|
||||
:members [(own :eye-r) (own :eye-l)] :params {}}}
|
||||
{})
|
||||
:store (into (:store head) (mapcat :store) parts)}))
|
||||
|
||||
(defn clip
|
||||
"Subject-id -> conditioned measurements becomes a library of face timelines.
|
||||
|
||||
:main holds exposure and a shared source-to-stage placement. Each subject is
|
||||
placed by an ordinary symbol instance, so pose choices and transforms have
|
||||
their existing instance scope. Features name local nodes in that subject's
|
||||
timeline; block descriptors still name globally distinct features.
|
||||
|
||||
Subjects share a source frame space. :head and :anchors may be overridden
|
||||
per subject; all other freeze settings come from params."
|
||||
[{:keys [name fps stage expose head anchors] :as params} subjects]
|
||||
(when-not (and (map? subjects) (seq subjects)
|
||||
(every? keyword? (keys subjects))
|
||||
(not-any? #{:main :root :face} (keys subjects)))
|
||||
(throw (ex-info "a freeze needs subjects with ids distinct from :main, :root and :face" {})))
|
||||
(let [ordered (sort-by (comp str key) subjects)
|
||||
parts (mapv (fn [[id inputs]] [id (subject-part params id inputs)]) ordered)
|
||||
lengths (distinct (map #(get-in % [1 :timeline :frames]) parts))
|
||||
_ (when-not (and (= 1 (count lengths)) (pos? (first lengths)))
|
||||
(throw (ex-info "subjects need the same positive frame count"
|
||||
{:frames (vec lengths)})))
|
||||
nf (first lengths)
|
||||
merged (fn [k] (into {} (mapcat (comp k second)) parts))
|
||||
built {:name name :fps fps :analysis (:analysis params)
|
||||
:width (first stage) :height (second stage)
|
||||
:subjects (into {} (map (fn [[id _]] [id {:id id :params {}}])) ordered)
|
||||
:features (merged :features) :groups (merged :groups)
|
||||
:timelines
|
||||
(into {clip/root-id
|
||||
{:id clip/root-id :frames nf
|
||||
:nodes (into {:root {:id :root :name "clip" :kind :group :z "a1"
|
||||
:time {:mode :map :expose expose}}
|
||||
:face {:id :face :name "source placement" :kind :group
|
||||
:parent :root :z "a1"
|
||||
:channels (face-placement params subjects)}}
|
||||
(map-indexed
|
||||
(fn [i [id _]]
|
||||
[id {:id id :kind :symbol :of id :parent :face
|
||||
:z (str "a" i)}]))
|
||||
ordered)}}
|
||||
(map (fn [[id part]] [id (:timeline part)])) parts)}]
|
||||
(doseq [[subject inputs] ordered
|
||||
[id track] (:presence inputs)]
|
||||
(when-not (and (= nf (count track))
|
||||
(= subject (get-in built [:features (feature/owned subject id) :subject])))
|
||||
(throw (ex-info "presence must name this subject's feature and span the take"
|
||||
{:subject subject :feature id :frames nf :actual (count track)}))))
|
||||
{:store (merged :store)
|
||||
:clip (reduce (fn [c [subject inputs]]
|
||||
(head-mode {:subject subject :mode (or (:head inputs) head)
|
||||
:anchors (get inputs :anchors anchors)}
|
||||
{:clip c}))
|
||||
built ordered)}))
|
||||
304
frontend/src/arthur/flow/ingest.cljs
Normal file
304
frontend/src/arthur/flow/ingest.cljs
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
(ns arthur.flow.ingest
|
||||
"Read footage from the server: source timing, one video to measure, and one
|
||||
content-addressed URL per tracing still.
|
||||
|
||||
THE MEASURED PIXELS COME OUT OF A VIDEO NOW, not out of a PNG per frame. The old
|
||||
arrangement stored 112MB for a 7.6-second take and 1.1GB at the 900-frame limit;
|
||||
the same footage is a 6MB H.264 proxy. What that cost was the property a PNG
|
||||
sequence gave for free — that asking for frame 12 gets frame 12 — and `decode!`
|
||||
below is how it is bought back."
|
||||
(:require [arthur.fx.http :as http]
|
||||
[clojure.string :as str]))
|
||||
|
||||
(defn feature-presence
|
||||
"Expand one-based, inclusive absence intervals from a manifest into boolean
|
||||
observation tracks. Unknown features are rejected by freeze, where the scene
|
||||
knows its feature IDs."
|
||||
[frames absence]
|
||||
(when (some? absence)
|
||||
(when-not (map? absence)
|
||||
(throw (ex-info "feature-absence must be a map of feature IDs to intervals"
|
||||
{:feature-absence absence})))
|
||||
(into {}
|
||||
(map (fn [[id intervals]]
|
||||
(when-not (and (keyword? id) (sequential? intervals)
|
||||
(every? (fn [span]
|
||||
(and (vector? span) (= 2 (count span))
|
||||
(every? integer? span)
|
||||
(<= 1 (first span) (second span) frames)))
|
||||
intervals))
|
||||
(throw (ex-info "feature-absence intervals must be [first last] source frames"
|
||||
{:feature id :intervals intervals :frames frames})))
|
||||
[id (mapv (fn [frame]
|
||||
(not-any? (fn [[first-frame last-frame]]
|
||||
(<= first-frame frame last-frame))
|
||||
intervals))
|
||||
(range 1 (inc frames)))]))
|
||||
absence)))
|
||||
|
||||
(defn- valid-manifest [m]
|
||||
(let [fps (js/Number (:fps m))
|
||||
frames (js/Number (:frames m))
|
||||
urls (:urls m)]
|
||||
(when-not (and (js/Number.isFinite fps) (pos? fps)
|
||||
(js/Number.isInteger frames) (<= 1 frames 900)
|
||||
(string? (:audio m)) (seq (:audio m))
|
||||
(sequential? urls) (every? string? urls))
|
||||
(throw (ex-info "a footage manifest needs fps, frames (1–900), audio and a url per frame"
|
||||
{:manifest (dissoc m :urls)})))
|
||||
;; Named as its own failure rather than folded into the check above, because
|
||||
;; it has a specific cause and a specific fix: this footage was extracted
|
||||
;; before the proxy existed, and its frames were stored as PNGs that the
|
||||
;; measurement path no longer reads.
|
||||
(when-not (and (string? (:stream m)) (seq (:stream m)))
|
||||
(throw (ex-info "this footage has no decodable stream — re-extract it from its source"
|
||||
{:footage (:id m)})))
|
||||
(when-not (= frames (count urls))
|
||||
;; The count is the manifest's and the URLs are the manifest's, so a
|
||||
;; disagreement between them is the server contradicting itself — and it
|
||||
;; would present as a take that is silently short.
|
||||
(throw (ex-info "the manifest's frame count and its list of frames disagree"
|
||||
{:frames frames :urls (count urls)})))
|
||||
(assoc m :fps fps :frames frames :urls (vec urls)
|
||||
:presence (feature-presence frames (:feature-absence m)))))
|
||||
|
||||
(defn available!
|
||||
"Every extracted take the server holds."
|
||||
[]
|
||||
(-> (http/GET "/api/footage")
|
||||
(.then (fn [json] (:footage (js->clj json :keywordize-keys true))))))
|
||||
|
||||
(defn manifest!
|
||||
"One take's manifest, including a URL per frame."
|
||||
[id]
|
||||
(-> (http/GET (str "/api/footage/" id))
|
||||
(.then (fn [json] (valid-manifest (js->clj json :keywordize-keys true))))))
|
||||
|
||||
(defn detector!
|
||||
"Who is about to do the detecting, as the server understands it: the MediaPipe
|
||||
package version and the hash of the model asset it serves.
|
||||
|
||||
ASKED RATHER THAN ASSUMED, because this string ends up inside every block's
|
||||
content address, and a version constant in the client is one somebody has to
|
||||
remember to bump. The server serves the model, so it can hash it — and then the
|
||||
version is a fact about the bytes that produced the landmarks."
|
||||
[]
|
||||
(-> (http/GET "/api/detector")
|
||||
(.then (fn [json] (js->clj json :keywordize-keys true)))))
|
||||
|
||||
(defn audio-url [manifest]
|
||||
(:audio manifest))
|
||||
|
||||
(defn video-url [manifest]
|
||||
(:video manifest))
|
||||
|
||||
(defn stream-url [manifest]
|
||||
(:stream manifest))
|
||||
|
||||
(defn frame-url
|
||||
"The tracing still for one frame. A reference image for drawing over — the
|
||||
landmarks and the mouth crops come from the video, not from these."
|
||||
[manifest i]
|
||||
(nth (:urls manifest) i))
|
||||
|
||||
(defn image! [src]
|
||||
(js/Promise.
|
||||
(fn [resolve reject]
|
||||
(let [image (js/Image.)]
|
||||
(set! (.-onload image) #(resolve image))
|
||||
(set! (.-onerror image) #(reject (ex-info (str "frame did not load: " src)
|
||||
{:src src})))
|
||||
(set! (.-src image) src)))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; decoding the proxy
|
||||
;;
|
||||
;; WEBCODECS, NOT SEEKING, and the seeking is worth a paragraph because three
|
||||
;; separate failures came out of it. A `<video>` cannot be asked for frame 12: it
|
||||
;; can be asked for a TIME, and which frame that lands on is up to the engine.
|
||||
;; Measured on a video whose every frame carries its own index in its pixels,
|
||||
;; Firefox 156 returned the wrong frame for 4 of 40 seeks aimed at the middle of
|
||||
;; each frame, and 15 of 40 aimed at the start — and `requestVideoFrameCallback`,
|
||||
;; the thing that is supposed to say which frame arrived, reported a different
|
||||
;; frame from the one actually on screen 27 times out of 40. There is no way to
|
||||
;; verify a seek when the verification is the part that is wrong.
|
||||
;;
|
||||
;; So the page decodes instead. `VideoDecoder` takes coded chunks and returns
|
||||
;; exactly one frame per chunk, in order — measured 120 of 120 in order in both
|
||||
;; Firefox and Chrome. The server hands us the proxy's video as a raw Annex-B
|
||||
;; stream, which needs no demuxer: NAL start codes are findable in a loop. And
|
||||
;; because the proxy is encoded with no B-frames, decode order is presentation
|
||||
;; order, so ACCESS UNIT k IS FRAME k. No timestamps are interpreted, no clock is
|
||||
;; reconciled, and nothing here can be off by one.
|
||||
|
||||
(defn frame-ms
|
||||
"When source frame `i` happens, in milliseconds of footage.
|
||||
|
||||
What `flow/detect` hands MediaPipe as the frame's timestamp. Real elapsed time
|
||||
rather than the frame number, because the tracker reads the gap between
|
||||
timestamps as motion — see `detect/detect!` — and strictly increasing, which its
|
||||
input stream requires."
|
||||
[fps i]
|
||||
(/ (* i 1000) fps))
|
||||
|
||||
(defn access-units
|
||||
"An Annex-B H.264 stream -> one `{:from :to :key?}` per coded frame.
|
||||
|
||||
A NAL unit starts at a three- or four-byte start code, and an access unit is the
|
||||
parameter sets and SEI leading up to and including one coded slice. So: a new
|
||||
unit begins at the first non-slice NAL after a slice. Types 1 and 5 are the
|
||||
slice types — 5 is an IDR, which is what makes a chunk a keyframe.
|
||||
|
||||
Twenty-five lines instead of a demuxer, and the reason it is only twenty-five is
|
||||
that the stream was produced for this: constant rate, no B-frames, one slice per
|
||||
frame."
|
||||
[^js bytes]
|
||||
(let [n (.-length bytes)
|
||||
starts (loop [i 0 out (transient [])]
|
||||
(if (>= i (- n 3))
|
||||
(persistent! out)
|
||||
(let [a (aget bytes i) b (aget bytes (inc i)) c (aget bytes (+ i 2))]
|
||||
(cond
|
||||
(and (zero? a) (zero? b) (= 1 c))
|
||||
(recur (inc i) (conj! out i))
|
||||
|
||||
(and (zero? a) (zero? b) (zero? c) (= 1 (aget bytes (+ i 3))))
|
||||
(recur (+ i 2) (conj! out i))
|
||||
|
||||
:else (recur (inc i) out)))))]
|
||||
(loop [ks 0 current nil saw-slice? false out []]
|
||||
(if (= ks (count starts))
|
||||
(if current (conj out current) out)
|
||||
(let [start (nth starts ks)
|
||||
end (if (< (inc ks) (count starts)) (nth starts (inc ks)) n)
|
||||
header (aget bytes (+ start (if (= 1 (aget bytes (+ start 2))) 3 4)))
|
||||
kind (bit-and header 0x1f)
|
||||
slice? (or (= 1 kind) (= 5 kind))
|
||||
[out current saw-slice?] (if (and slice? saw-slice?)
|
||||
[(conj out current) nil false]
|
||||
[out current saw-slice?])
|
||||
current (or current {:from start :to end :key? false})]
|
||||
(recur (inc ks)
|
||||
(assoc current :to end :key? (or (:key? current) (= 5 kind)))
|
||||
(or saw-slice? slice?)
|
||||
out))))))
|
||||
|
||||
(defn stream!
|
||||
"Fetch the elementary stream and cut it into one chunk per frame."
|
||||
[src frames]
|
||||
(-> (js/fetch src)
|
||||
(.then (fn [^js response]
|
||||
(when-not (.-ok response)
|
||||
(throw (ex-info (str "the footage's video stream did not load: "
|
||||
(.-status response))
|
||||
{:src src})))
|
||||
(.arrayBuffer response)))
|
||||
(.then (fn [buffer]
|
||||
(let [units (access-units (js/Uint8Array. buffer))]
|
||||
(when-not (= (count units) frames)
|
||||
(throw (ex-info (str "the video stream holds " (count units)
|
||||
" coded frames and the manifest says " frames)
|
||||
{:units (count units) :frames frames})))
|
||||
{:bytes (js/Uint8Array. buffer) :units (vec units)})))))
|
||||
|
||||
(def ^:private decode-lookahead
|
||||
"How many chunks may be in the decoder at once.
|
||||
|
||||
Backpressure, and not a tuning knob to leave at infinity: a 900-frame take at
|
||||
1440x1920 is gigabytes of decoded frames, so feeding the whole stream in and
|
||||
letting the output callback keep up is how the tab dies. Small enough to bound
|
||||
that, and more than one so the decoder is never idle waiting on us."
|
||||
4)
|
||||
|
||||
(defn decode!
|
||||
"Decode every frame in order, calling `(on-frame i frame)` for each.
|
||||
|
||||
`on-frame` runs while the frame is alive and must not retain it — this closes it
|
||||
as soon as the call returns, because a `VideoFrame` holds decoder memory and the
|
||||
decoder stalls when it runs out. It may return a promise, which the walk waits
|
||||
for before feeding more; that is what lets a synchronous MediaPipe call and a
|
||||
repaint happen between frames without the decoder running ahead.
|
||||
|
||||
Resolves when the last frame has been handed over."
|
||||
[{:keys [bytes units]} fps width height on-frame]
|
||||
(js/Promise.
|
||||
(fn [resolve reject]
|
||||
(if-not (exists? js/VideoDecoder)
|
||||
(reject (ex-info (str "this browser has no WebCodecs VideoDecoder, which is "
|
||||
"what reads footage a frame at a time")
|
||||
{}))
|
||||
(let [total (count units)
|
||||
next-in (volatile! 0)
|
||||
done-out (volatile! 0)
|
||||
failed (volatile! false)
|
||||
;; `taken` is claimed as a frame ARRIVES. `done-out` counts frames
|
||||
;; FINISHED, and is what backpressure and completion read. They are
|
||||
;; two different numbers whenever `on-frame` yields, which it does —
|
||||
;; reading `done-out` as the index let frames 0 and 1 both claim 0,
|
||||
;; call the detector twice at timestamp 0, and get the run killed by
|
||||
;; `Packet timestamp mismatch ... expected 1 but received 0`.
|
||||
taken (volatile! 0)
|
||||
;; And the work is CHAINED rather than started, because the detector
|
||||
;; is a tracker fed one frame at a time in order. Two overlapping
|
||||
;; `detect!` calls are not slow, they are wrong.
|
||||
chain (volatile! (js/Promise.resolve))
|
||||
decoder (volatile! nil)
|
||||
fail! (fn [error]
|
||||
(when-not @failed
|
||||
(vreset! failed true)
|
||||
(when-let [^js d @decoder]
|
||||
(try (when-not (= "closed" (.-state d)) (.close d))
|
||||
(catch :default _ nil)))
|
||||
(reject error)))]
|
||||
(letfn [(feed! []
|
||||
(while (and (not @failed)
|
||||
(< @next-in total)
|
||||
(< (- @next-in @done-out) decode-lookahead))
|
||||
(let [i @next-in
|
||||
{:keys [from to key?]} (nth units i)]
|
||||
(vreset! next-in (inc i))
|
||||
(.decode ^js @decoder
|
||||
(js/EncodedVideoChunk.
|
||||
#js {:type (if key? "key" "delta")
|
||||
:timestamp (js/Math.round (/ (* i 1e6) fps))
|
||||
:duration (js/Math.round (/ 1e6 fps))
|
||||
:data (.subarray bytes from to)})))))]
|
||||
(vreset!
|
||||
decoder
|
||||
(js/VideoDecoder.
|
||||
#js {:output
|
||||
(fn [^js frame]
|
||||
(if @failed
|
||||
(.close frame)
|
||||
(let [i @taken]
|
||||
(vreset! taken (inc i))
|
||||
(vreset!
|
||||
chain
|
||||
(.then
|
||||
@chain
|
||||
(fn []
|
||||
(if @failed
|
||||
(do (try (.close frame) (catch :default _ nil)) nil)
|
||||
(-> (js/Promise.resolve
|
||||
(try (on-frame i frame)
|
||||
(catch :default error (js/Promise.reject error))))
|
||||
(.then (fn [_]
|
||||
(try (.close frame) (catch :default _ nil))
|
||||
(vreset! done-out (inc i))
|
||||
(if (= (inc i) total)
|
||||
(do (.close ^js @decoder)
|
||||
(resolve total))
|
||||
(feed!))))
|
||||
(.catch (fn [error]
|
||||
(try (.close frame) (catch :default _ nil))
|
||||
(fail! error)
|
||||
nil))))))))))
|
||||
:error (fn [^js e]
|
||||
(fail! (ex-info (str "the browser could not decode this "
|
||||
"footage: " (.-message e))
|
||||
{})))}))
|
||||
(.configure ^js @decoder
|
||||
#js {:codec "avc1.640028"
|
||||
:codedWidth width :codedHeight height
|
||||
:optimizeForLatency true})
|
||||
(feed!)))))))
|
||||
65
frontend/src/arthur/flow/measure/anchor.cljs
Normal file
65
frontend/src/arthur/flow/measure/anchor.cljs
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
(ns arthur.flow.measure.anchor
|
||||
"Stage 3, the anchor: the rigid transform per frame, and the space every other
|
||||
measurement is taken in.
|
||||
|
||||
The fit is knob-free, deliberately. Smoothing its four parameters is stage 4 —
|
||||
`arthur.flow.condition` — because `anchor avg` is a knob and the rest of measure
|
||||
is not, and because a residual that moved when a smoothing slider moved would
|
||||
report the footage as unstabilisable on account of a setting.
|
||||
|
||||
`makeXform` is NOT here, and is not being ported. It centres on the face oval's
|
||||
bounding box and zooms until the face is 80% of the raster height, so every
|
||||
vertex it touches carries a cropping decision made once, at analysis time, from
|
||||
one frame's landmarks. Geometry is stored in the node's own local space and the
|
||||
framing is a transform on a node; see \"What space geometry is in\" in
|
||||
docs/animation-model.md. So the face oval is not measured here either — its only
|
||||
consumers in the prototype were that transform and the placeholder plate
|
||||
outline, and the plate outline belongs to painting."
|
||||
(:require [arthur.domain.geom :as geom]
|
||||
[arthur.domain.landmarks :as lm]))
|
||||
|
||||
(defn pick
|
||||
"Landmarks `idx` out of one dense `frame`, converted to an ISOTROPIC space.
|
||||
|
||||
MediaPipe normalises x by image WIDTH and y by image HEIGHT, so its normalised
|
||||
space is anisotropic: for a 1080x1920 frame, one unit of x is 1080px and one
|
||||
unit of y is 1920px. Treating those as comparable stretches everything
|
||||
horizontally by H/W, and worse, makes fit-similarity fit a \"rotation\" in a
|
||||
sheared space, so head roll comes out subtly wrong as well.
|
||||
|
||||
Multiplying x by aspect = W/H converts to an isotropic space whose unit is one
|
||||
image height, so equal numbers mean equal pixels. Everything downstream -
|
||||
Procrustes, the similarity fit, the raster transform - depends on that."
|
||||
[frame idx aspect]
|
||||
(mapv (fn [i] (let [p (nth frame i)] {:x (* (:x p) aspect) :y (:y p)})) idx))
|
||||
|
||||
(defn residuals
|
||||
"RMS misfit per frame, in the isotropic space's units — one image height.
|
||||
|
||||
Taken against the transforms it is HANDED rather than against a fit of its own,
|
||||
which is what let the parity diff assert on exactly what the prototype handed
|
||||
it while that diff existed. The prototype took the residual against the
|
||||
SMOOTHED transforms, which folds the smoothing
|
||||
error into a number whose whole job is to say whether the footage is
|
||||
stabilisable at all; `fit` takes it against the raw fit instead."
|
||||
[ref rigid tfs]
|
||||
(mapv (fn [rig tf] (geom/fit-residual tf rig ref)) rigid tfs))
|
||||
|
||||
(defn fit
|
||||
"Dense landmarks -> the rigid fit of every frame onto the shot's mean pose.
|
||||
|
||||
The reference is the Procrustes MEAN configuration over the shot, not frame
|
||||
zero, so no single frame's idiosyncrasies get baked into every other frame."
|
||||
[{:keys [aspect]} {:keys [dense]}]
|
||||
(let [rigid (mapv #(pick % lm/RIGID aspect) dense)
|
||||
ref (geom/procrustes-mean rigid)
|
||||
tfs (mapv #(geom/fit-similarity % ref) rigid)]
|
||||
{:ref ref
|
||||
;; Rigid landmarks in IMAGE space: the head-pose signal. Frame removal is
|
||||
;; decided from head motion, not from the mouth, so this has to survive the
|
||||
;; fit rather than being consumed by it.
|
||||
:rigid rigid
|
||||
:transforms tfs
|
||||
;; Residual rises with out-of-plane rotation, which no 2D similarity can
|
||||
;; remove. High values mean this section wants a different head plate.
|
||||
:residual (residuals ref rigid tfs)}))
|
||||
62
frontend/src/arthur/flow/measure/brows.cljs
Normal file
62
frontend/src/arthur/flow/measure/brows.cljs
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
(ns arthur.flow.measure.brows
|
||||
"Head-local brow rings and outer/inner raise signals, measured against the
|
||||
rigid eye corners so blinking cannot masquerade as an eyebrow movement."
|
||||
(:require [arthur.domain.geom :as geom]
|
||||
[arthur.domain.landmarks :as lm]
|
||||
[arthur.flow.measure.anchor :as anchor]))
|
||||
|
||||
(defn- midpoint [a b]
|
||||
{:x (/ (+ (:x a) (:x b)) 2) :y (/ (+ (:y a) (:y b)) 2)})
|
||||
|
||||
(defn- distance [a b]
|
||||
(js/Math.hypot (- (:x a) (:x b)) (- (:y a) (:y b))))
|
||||
|
||||
(defn- local-track [dense transforms aspect table]
|
||||
(mapv (fn [frame tf]
|
||||
(geom/apply-sim-all tf (anchor/pick frame table aspect)))
|
||||
dense transforms))
|
||||
|
||||
(defn measure
|
||||
[{:keys [aspect debug?]} {:keys [dense transforms]}]
|
||||
(let [track (partial local-track dense transforms aspect)
|
||||
a (track lm/BROW-A-RING)
|
||||
b (track lm/BROW-B-RING)
|
||||
corners-r (track lm/EYE-R-CORNERS)
|
||||
corners-l (track lm/EYE-L-CORNERS)
|
||||
centre-x (fn [ring] (:x (geom/centroid ring)))
|
||||
side-votes (reduce +
|
||||
(map (fn [ar [ro ri] [lo li]]
|
||||
(if (< (abs (- (centre-x ar) (:x (midpoint ro ri))))
|
||||
(abs (- (centre-x ar) (:x (midpoint lo li)))))
|
||||
1 -1))
|
||||
a corners-r corners-l))
|
||||
a-right? (pos? side-votes)
|
||||
right (if a-right? a b)
|
||||
left (if a-right? b a)
|
||||
outer-votes (reduce +
|
||||
(mapcat (fn [rings corners]
|
||||
(map (fn [ring [outer inner]]
|
||||
(if (< (distance (first ring) outer)
|
||||
(distance (first ring) inner))
|
||||
1 -1))
|
||||
rings corners))
|
||||
[right left] [corners-r corners-l]))
|
||||
outer-at-zero? (pos? outer-votes)
|
||||
end-outer (if outer-at-zero? lm/BROW-END-0 lm/BROW-END-1)
|
||||
end-inner (if outer-at-zero? lm/BROW-END-1 lm/BROW-END-0)
|
||||
signal (fn [rings corners]
|
||||
(mapv (fn [ring [outer inner]]
|
||||
(let [c (midpoint outer inner)
|
||||
w (max 1e-9 (distance outer inner))
|
||||
at (fn [[i j]] (/ (+ (:y (nth ring i))
|
||||
(:y (nth ring j))) 2))]
|
||||
{:x (/ (- (:y c) (at end-outer)) w)
|
||||
:y (/ (- (:y c) (at end-inner)) w)}))
|
||||
rings corners))]
|
||||
{:ring-r right :ring-l left
|
||||
:raise-r (signal right corners-r)
|
||||
:raise-l (signal left corners-l)
|
||||
:outer-at-zero? outer-at-zero? :a-right? a-right?
|
||||
:debug (when debug? {:side-votes side-votes :outer-votes outer-votes
|
||||
:raise-r (signal right corners-r)
|
||||
:raise-l (signal left corners-l)})}))
|
||||
97
frontend/src/arthur/flow/measure/eyes.cljs
Normal file
97
frontend/src/arthur/flow/measure/eyes.cljs
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
(ns arthur.flow.measure.eyes
|
||||
"Head-local eyelid and iris measurements. No blink threshold or drawn gaze
|
||||
grid belongs here; those decisions are made after measurement."
|
||||
(:require [arthur.domain.geom :as geom]
|
||||
[arthur.domain.landmarks :as lm]
|
||||
[arthur.flow.measure.anchor :as anchor]))
|
||||
|
||||
(defn- midpoint [a b]
|
||||
{:x (/ (+ (:x a) (:x b)) 2) :y (/ (+ (:y a) (:y b)) 2)})
|
||||
|
||||
(defn- distance [a b]
|
||||
(js/Math.hypot (- (:x a) (:x b)) (- (:y a) (:y b))))
|
||||
|
||||
(defn- local-track [dense transforms aspect table]
|
||||
(mapv (fn [frame tf]
|
||||
(geom/apply-sim-all tf (anchor/pick frame table aspect)))
|
||||
dense transforms))
|
||||
|
||||
(defn- observed-track [frames detected presence feature-id]
|
||||
(let [part (get presence feature-id)]
|
||||
(mapv (fn [f]
|
||||
(and (or (nil? detected) (nth detected f))
|
||||
(or (nil? part) (nth part f))))
|
||||
(range frames))))
|
||||
|
||||
(defn- iris-pair [iris-a iris-b corners-r corners-l observed-r observed-l]
|
||||
(when iris-a
|
||||
;; An occluded eye may still get a plausible iris from MediaPipe. It gets no
|
||||
;; vote. Either eye can establish the pairing when its partner is absent.
|
||||
(let [votes (reduce +
|
||||
(for [f (range (count iris-a))
|
||||
[seen corners sign] [[(nth observed-r f) (nth corners-r f) 1]
|
||||
[(nth observed-l f) (nth corners-l f) -1]]
|
||||
:when seen]
|
||||
(let [a (first (nth iris-a f))
|
||||
b (first (nth iris-b f))
|
||||
[outer inner] corners]
|
||||
(if (< (distance a (midpoint outer inner))
|
||||
(distance b (midpoint outer inner)))
|
||||
sign (- sign)))))]
|
||||
(if (pos? votes) {:right :a :left :b} {:right :b :left :a}))))
|
||||
|
||||
(defn measure
|
||||
"All rings and iris coordinates use the anchor's head-local image-height unit.
|
||||
Eye openness and gaze use each eye's rigid corner width as their unit. Iris
|
||||
block identity is voted from observed eyes over the whole take."
|
||||
[{:keys [aspect debug?]} {:keys [dense transforms detected presence]}]
|
||||
(let [track (partial local-track dense transforms aspect)
|
||||
observed-r (observed-track (count dense) detected presence :eye-r)
|
||||
observed-l (observed-track (count dense) detected presence :eye-l)
|
||||
lid-r (track lm/EYE-R-RING)
|
||||
lid-l (track lm/EYE-L-RING)
|
||||
corners-r (track lm/EYE-R-CORNERS)
|
||||
corners-l (track lm/EYE-L-CORNERS)
|
||||
lids-r (track lm/EYE-R-LIDS)
|
||||
lids-l (track lm/EYE-L-LIDS)
|
||||
has-iris? (every? #(> (count %) (last lm/IRIS-B)) dense)
|
||||
iris-a (when has-iris? (track lm/IRIS-A))
|
||||
iris-b (when has-iris? (track lm/IRIS-B))
|
||||
pairing (iris-pair iris-a iris-b corners-r corners-l observed-r observed-l)
|
||||
iris-for (fn [side f]
|
||||
(first (nth (if (= (get pairing side) :a) iris-a iris-b) f)))
|
||||
openness (fn [corners lids]
|
||||
(mapv (fn [[a b] [up down]]
|
||||
(/ (distance up down) (max 1e-9 (distance a b))))
|
||||
corners lids))
|
||||
gaze (if pairing
|
||||
(mapv (fn [f]
|
||||
(let [one (fn [side corners]
|
||||
(let [[a b] (nth corners f)
|
||||
c (midpoint a b)
|
||||
w (max 1e-9 (distance a b))
|
||||
iris (iris-for side f)]
|
||||
{:x (/ (- (:x iris) (:x c)) w)
|
||||
:y (/ (- (:y iris) (:y c)) w)}))
|
||||
r (when (nth observed-r f) (one :right corners-r))
|
||||
l (when (nth observed-l f) (one :left corners-l))]
|
||||
(cond
|
||||
(and r l) (midpoint r l)
|
||||
r r
|
||||
l l
|
||||
:else nil)))
|
||||
(range (count dense)))
|
||||
(vec (repeat (count dense) {:x 0 :y 0})))]
|
||||
{:lid-r lid-r :lid-l lid-l
|
||||
:corners-r corners-r :corners-l corners-l
|
||||
:open-r (openness corners-r lids-r)
|
||||
:open-l (openness corners-l lids-l)
|
||||
:observed-r observed-r :observed-l observed-l
|
||||
:gaze-observed (if pairing (mapv #(or %1 %2) observed-r observed-l)
|
||||
(vec (repeat (count dense) true)))
|
||||
:gaze gaze :has-iris? (boolean pairing)
|
||||
:iris-pair pairing
|
||||
:debug (when debug? {:open-r (openness corners-r lids-r)
|
||||
:open-l (openness corners-l lids-l)
|
||||
:observed-r observed-r :observed-l observed-l
|
||||
:gaze gaze :iris-pair pairing})}))
|
||||
207
frontend/src/arthur/flow/measure/interior.cljs
Normal file
207
frontend/src/arthur/flow/measure/interior.cljs
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
(ns arthur.flow.measure.interior
|
||||
"Pixel measurement inside the inner lip ring. The image crop is supplied by
|
||||
ingestion; this namespace owns Otsu, morphology, component choice and radial
|
||||
contour correspondence, and never reads a DOM element."
|
||||
(:require [arthur.domain.geom :as geom]))
|
||||
|
||||
(defn- scaled-ring [ring amount]
|
||||
(let [{:keys [x y]} (geom/centroid ring)]
|
||||
(mapv (fn [p] {:x (+ x (* amount (- (:x p) x)))
|
||||
:y (+ y (* amount (- (:y p) y)))}) ring)))
|
||||
|
||||
(defn crop
|
||||
"A clamped source-pixel box for the raw inner-lip ring. The box is independent
|
||||
of teeth settings so its pixels can be reused when those settings change. nil
|
||||
means too small for a useful contrast measurement."
|
||||
[ring [width height]]
|
||||
(let [x0 (max 0 (js/Math.floor (* width (reduce min (map :x ring)))))
|
||||
y0 (max 0 (js/Math.floor (* height (reduce min (map :y ring)))))
|
||||
x1 (min width (js/Math.ceil (* width (reduce max (map :x ring)))))
|
||||
y1 (min height (js/Math.ceil (* height (reduce max (map :y ring)))))
|
||||
w (- x1 x0) h (- y1 y0)]
|
||||
(when (and (>= w 5) (>= h 5))
|
||||
{:x x0 :y y0 :w w :h h :shape ring
|
||||
:source-width width :source-height height})))
|
||||
|
||||
(defn- point-in-poly? [points x y]
|
||||
(let [n (count points)]
|
||||
(loop [i 0 j (dec n) inside? false]
|
||||
(if (= i n)
|
||||
inside?
|
||||
(let [a (nth points i) b (nth points j)
|
||||
cross? (and (not= (> (:y a) y) (> (:y b) y))
|
||||
(< x (+ (:x a)
|
||||
(/ (* (- (:x b) (:x a)) (- y (:y a)))
|
||||
(- (:y b) (:y a))))))]
|
||||
(recur (inc i) i (if cross? (not inside?) inside?)))))))
|
||||
|
||||
(defn- otsu [hist total]
|
||||
(let [sum (reduce + (map-indexed * hist))]
|
||||
(loop [t 0 weight 0 sum-dark 0 best-var -1 best {:thr 0 :dark 0 :bright 0}]
|
||||
(if (= t 256)
|
||||
best
|
||||
(let [weight (+ weight (aget hist t))
|
||||
sum-dark (+ sum-dark (* t (aget hist t)))
|
||||
remain (- total weight)]
|
||||
(if (zero? remain)
|
||||
best
|
||||
(let [dark (if (pos? weight) (/ sum-dark weight) 0)
|
||||
bright (/ (- sum sum-dark) remain)
|
||||
delta (- dark bright)
|
||||
variance (* weight remain delta delta)]
|
||||
(recur (inc t) weight sum-dark
|
||||
(max best-var variance)
|
||||
(if (and (pos? weight) (> variance best-var))
|
||||
{:thr t :dark dark :bright bright}
|
||||
best)))))))))
|
||||
|
||||
(defn- morph [mask w h dilate?]
|
||||
(let [out (js/Uint8Array. (.-length mask))]
|
||||
(doseq [y (range 1 (dec h)) x (range 1 (dec w))]
|
||||
(let [i (+ (* y w) x)
|
||||
values [(aget mask i) (aget mask (dec i)) (aget mask (inc i))
|
||||
(aget mask (- i w)) (aget mask (+ i w))]]
|
||||
(aset out i (if (if dilate? (some pos? values) (every? pos? values)) 1 0))))
|
||||
out))
|
||||
|
||||
(defn- best-component [mask w h top-bias]
|
||||
(let [labels (js/Int32Array. (.-length mask))
|
||||
stack (array)]
|
||||
(.fill labels -1)
|
||||
(loop [seed 0 label 0 best nil]
|
||||
(if (= seed (.-length mask))
|
||||
best
|
||||
(if (or (zero? (aget mask seed)) (>= (aget labels seed) 0))
|
||||
(recur (inc seed) label best)
|
||||
(do
|
||||
(set! (.-length stack) 0)
|
||||
(.push stack seed)
|
||||
(aset labels seed label)
|
||||
(let [{:keys [pixels sum-y]}
|
||||
(loop [pixels [] sum-y 0]
|
||||
(if (zero? (.-length stack))
|
||||
{:pixels pixels :sum-y sum-y}
|
||||
(let [i (.pop stack)
|
||||
x (mod i w) y (quot i w)
|
||||
neighbours (cond-> []
|
||||
(> x 0) (conj (dec i))
|
||||
(< x (dec w)) (conj (inc i))
|
||||
(> y 0) (conj (- i w))
|
||||
(< y (dec h)) (conj (+ i w)))]
|
||||
(doseq [j neighbours]
|
||||
(when (and (pos? (aget mask j)) (neg? (aget labels j)))
|
||||
(aset labels j label)
|
||||
(.push stack j)))
|
||||
(recur (conj pixels i) (+ sum-y y)))))
|
||||
area (count pixels)
|
||||
mean-y (/ sum-y area h)
|
||||
score (* area (- 1 (* top-bias mean-y)))
|
||||
winner {:pixels pixels :area area :score score :mean-y mean-y}]
|
||||
(recur (inc seed) (inc label)
|
||||
(if (or (nil? best) (> score (:score best))) winner best)))))))))
|
||||
|
||||
(defn- radial-contour [mask w h cx cy vertices]
|
||||
(let [maximum (js/Math.hypot w h)]
|
||||
(loop [k 0 previous 1 out []]
|
||||
(if (= k vertices)
|
||||
out
|
||||
(let [angle (- (* (/ k vertices) js/Math.PI 2))
|
||||
dx (js/Math.cos angle) dy (js/Math.sin angle)
|
||||
hit (loop [radius 0.5 last-hit 0]
|
||||
(if (>= radius maximum)
|
||||
last-hit
|
||||
(let [x (js/Math.round (+ cx (* dx radius)))
|
||||
y (js/Math.round (+ cy (* dy radius)))]
|
||||
(if (or (< x 0) (< y 0) (>= x w) (>= y h))
|
||||
last-hit
|
||||
(if (pos? (aget mask (+ (* y w) x)))
|
||||
(recur (+ radius 0.5) radius)
|
||||
(if (and (pos? last-hit) (> radius (+ last-hit 2)))
|
||||
last-hit
|
||||
(recur (+ radius 0.5) last-hit)))))))
|
||||
radius (if (pos? hit) hit (* previous 0.6))]
|
||||
(recur (inc k) radius
|
||||
(conj out {:x (+ cx (* dx radius))
|
||||
:y (+ cy (* dy radius))})))))))
|
||||
|
||||
(defn measure
|
||||
"Measure one cropped RGBA ImageData at `box`. Returns normalized source image
|
||||
coordinates and optional intermediate masks for a diagnostic view."
|
||||
[{:keys [cavity-erode tongue-reject blob-grow top-bias teeth-verts min-area
|
||||
debug?]}
|
||||
{:keys [x y w h shape source-width source-height] :as box}
|
||||
image-data]
|
||||
(let [pixels (.-data image-data)
|
||||
polygon (mapv (fn [p] {:x (- (* (:x p) source-width) x)
|
||||
:y (- (* (:y p) source-height) y)})
|
||||
(scaled-ring shape (- 1 cavity-erode)))
|
||||
hist (js/Uint32Array. 256)
|
||||
luminance (js/Uint8Array. (* w h))
|
||||
redness (js/Float32Array. (* w h))
|
||||
region (js/Uint8Array. (* w h))
|
||||
samples (volatile! 0)]
|
||||
(doseq [row (range h) col (range w)]
|
||||
(when (point-in-poly? polygon (+ col 0.5) (+ row 0.5))
|
||||
(let [i (+ (* row w) col) o (* i 4)
|
||||
r (aget pixels o) g (aget pixels (inc o)) b (aget pixels (+ o 2))
|
||||
lum (js/Math.floor (+ (* 0.299 r) (* 0.587 g) (* 0.114 b)))]
|
||||
(aset luminance i lum)
|
||||
(aset redness i (/ (- r (/ (+ g b) 2)) 255))
|
||||
(aset region i 1)
|
||||
(aset hist lum (inc (aget hist lum)))
|
||||
(vswap! samples inc))))
|
||||
(if (< @samples 24)
|
||||
{:contour nil :contrast 0 :area 0 :debug (when debug? {:box box :region region})}
|
||||
(let [{:keys [thr dark bright]} (otsu hist @samples)
|
||||
contrast (/ (- bright dark) 255)
|
||||
;; Warm light shifts even the pale teeth toward red. An absolute
|
||||
;; tongue-rejection cut can exclude every pixel in a real mouth.
|
||||
;; Calibrate the cut to the least-red bright quarter of THIS cavity,
|
||||
;; while retaining the authored cut as a floor for neutral footage.
|
||||
bright-red (->> (range (* w h))
|
||||
(keep (fn [i]
|
||||
(when (and (pos? (aget region i))
|
||||
(> (aget luminance i) bright))
|
||||
(aget redness i))))
|
||||
sort vec)
|
||||
red-cut (if (seq bright-red)
|
||||
(max tongue-reject
|
||||
(+ 0.015 (nth bright-red
|
||||
(js/Math.floor (* 0.25 (dec (count bright-red)))))))
|
||||
tongue-reject)
|
||||
candidate (js/Uint8Array. (* w h))]
|
||||
(dotimes [i (* w h)]
|
||||
(when (and (pos? (aget region i)) (> (aget luminance i) bright)
|
||||
(< (aget redness i) red-cut))
|
||||
(aset candidate i 1)))
|
||||
(let [opened (morph (morph candidate w h false) w h true)
|
||||
adjusted (loop [remaining (abs blob-grow) mask opened]
|
||||
(if (zero? remaining) mask
|
||||
(recur (dec remaining) (morph mask w h (pos? blob-grow)))))
|
||||
component (best-component adjusted w h top-bias)
|
||||
area (or (:area component) 0)
|
||||
selected (js/Uint8Array. (* w h))]
|
||||
(when component
|
||||
(doseq [i (:pixels component)] (aset selected i 1)))
|
||||
(let [contour (when (>= area min-area)
|
||||
(let [cx (/ (reduce + (map #(mod % w) (:pixels component))) area)
|
||||
cy (/ (reduce + (map #(quot % w) (:pixels component))) area)]
|
||||
(mapv (fn [p] {:x (/ (+ (:x p) x) source-width)
|
||||
:y (/ (+ (:y p) y) source-height)})
|
||||
(radial-contour selected w h cx cy teeth-verts))))]
|
||||
{:contour contour :contrast contrast :area area
|
||||
:debug (when debug? {:box box :region region :luminance luminance
|
||||
:redness redness :candidate candidate
|
||||
:opened opened :selected selected
|
||||
:threshold thr :dark dark :bright bright
|
||||
:red-cut red-cut})}))))))
|
||||
|
||||
(defn head-local
|
||||
"Map a pixel-derived contour through the same conditioned similarity as the
|
||||
lip rings. Both coordinates are normalized by image height first."
|
||||
[measure transform aspect]
|
||||
(if-let [contour (:contour measure)]
|
||||
(assoc measure :contour
|
||||
(geom/apply-sim-all transform
|
||||
(mapv (fn [p] (update p :x * aspect)) contour)))
|
||||
measure))
|
||||
39
frontend/src/arthur/flow/measure/mouth.cljs
Normal file
39
frontend/src/arthur/flow/measure/mouth.cljs
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
(ns arthur.flow.measure.mouth
|
||||
"Stage 3, the mouth: the lip rings with the head's motion taken out, and the
|
||||
aperture that decides whether there is an interior at all.
|
||||
|
||||
Head-local means the anchor's space, so `pick` is read from
|
||||
`arthur.flow.measure.anchor` rather than written a second time. A ring measured
|
||||
in a different space from the fit that placed it is not a failure anyone would
|
||||
see — it is a mouth that is quietly the wrong width.
|
||||
|
||||
The rings keep every slot of their table. A vertex budget is a stage-5 knob, and
|
||||
subsampling is a per-slot pick while stage 4's contour average is a per-slot
|
||||
average over time, so the two commute: smooth-then-subsample and
|
||||
subsample-then-smooth are the same numbers. That is what lets the vertex knob
|
||||
sit downstream of the smoothing knob instead of alongside it, and mouth-test
|
||||
asserts it rather than leaving it to look obvious.
|
||||
|
||||
Turning the aperture into `[:vis]` on `:mouth-in` is stage 5. This reports the
|
||||
measurement, not the decision."
|
||||
(:require [arthur.domain.geom :as geom]
|
||||
[arthur.domain.landmarks :as lm]
|
||||
[arthur.flow.measure.anchor :as anchor]))
|
||||
|
||||
(defn measure
|
||||
"Dense landmarks plus the anchor's transforms -> head-local lip rings.
|
||||
|
||||
`transforms` is whatever the caller has: the raw fit, or — normally — stage 4's
|
||||
conditioned one. Which it is belongs to the caller, because \"smooth the
|
||||
transform, never the contour\" only means anything while the two are separate."
|
||||
[{:keys [aspect]} {:keys [dense transforms]}]
|
||||
(let [local (fn [table]
|
||||
(mapv (fn [tf frame] (geom/apply-sim-all tf (anchor/pick frame table aspect)))
|
||||
transforms dense))]
|
||||
{:outer (local lm/LIPS-OUTER)
|
||||
:inner (local lm/LIPS-INNER)
|
||||
;; Not a separate measurement: APERTURE is slots 5 and 15 of LIPS_INNER, so
|
||||
;; this is the inner ring's own height read off as a scalar. Writing those
|
||||
;; two landmarks a second time is what gave the synthetic mouth a bowtie.
|
||||
:aperture (mapv (fn [[a b]] (js/Math.hypot (- (:x a) (:x b)) (- (:y a) (:y b))))
|
||||
(local lm/APERTURE))}))
|
||||
141
frontend/src/arthur/flow/regenerate.cljs
Normal file
141
frontend/src/arthur/flow/regenerate.cljs
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
(ns arthur.flow.regenerate
|
||||
"Recompute a changed feature from retained source tracks, then replace only
|
||||
channels owned by that feature. Upload remains project/save's ordinary job."
|
||||
(:require [arthur.domain.feature :as feature]
|
||||
[arthur.domain.params :as params]
|
||||
[arthur.flow.address :as address]
|
||||
[arthur.flow.freeze :as freeze]
|
||||
[arthur.flow.take :as take]))
|
||||
|
||||
(defn- settings
|
||||
"One feature's MEASUREMENT inputs. The teeth read the mouth's aperture cut
|
||||
because `condition/interior` will not smooth a contour on a frame the mouth is
|
||||
shut on: an input edge between two features, and the only one there is."
|
||||
[clip fid]
|
||||
(let [f (get-in clip [:features fid])
|
||||
mouth (first (for [[id peer] (:features clip)
|
||||
:when (and (= :mouth (:area peer))
|
||||
(= (:subject f) (:subject peer)))] id))]
|
||||
(cond-> (feature/effective-params clip fid)
|
||||
(and (= :teeth (:area f)) mouth)
|
||||
(assoc :aperture-cut (:aperture-cut (feature/effective-params clip mouth))))))
|
||||
|
||||
(defn- reads
|
||||
"The knob values one feature's frozen channels actually depend on.
|
||||
|
||||
Narrower than its measurement inputs, and that difference IS the dirty-set
|
||||
calculation. `:contour-avg` is a subject setting every feature inherits and the
|
||||
teeth block does not read, so inheriting a knob and being stale because of it are
|
||||
not the same thing. `address/area-knobs` is what knows which is which, over the
|
||||
table `address-test` asserts by biconditional — so there is no second per-knob
|
||||
list here to drift away from the one that is checked."
|
||||
[clip fid]
|
||||
(select-keys (settings clip fid)
|
||||
(address/area-knobs (get-in clip [:features fid :area]))))
|
||||
|
||||
(defn- replace-feature [entry fragment fid]
|
||||
(let [paths (for [id (get-in entry [:clip :features fid :nodes])
|
||||
[prop channel] (get-in fragment [:nodes id :channels])
|
||||
:when (:generated channel)]
|
||||
[id prop channel])]
|
||||
(-> (reduce (fn [entry [id prop channel]]
|
||||
(let [at [:clip :timelines (get-in entry [:clip :features fid :timeline])
|
||||
:nodes id :channels prop]
|
||||
old (get-in entry at)]
|
||||
(assoc-in entry at
|
||||
(cond-> channel
|
||||
(contains? old :over) (assoc :over (:over old))))))
|
||||
entry paths)
|
||||
(update :store merge (:store fragment)))))
|
||||
|
||||
(defn plan
|
||||
"The changed document, the feature IDs an edit dirties, and the subject they
|
||||
belong to. Also reports tier-2 block roles from the address table for the
|
||||
debug UI."
|
||||
[clip {:keys [scope id knob value]}]
|
||||
(let [area (get-in params/definitions [knob :area])
|
||||
collection (case scope
|
||||
:subject :subjects :feature :features :group :groups
|
||||
(throw (ex-info "unknown setting scope" {:scope scope})))
|
||||
owner (get-in clip [collection id])]
|
||||
(when-not (and owner (params/valid-value? knob value)
|
||||
(case scope
|
||||
:subject (= area :subject)
|
||||
:feature (= area (:area owner))
|
||||
:group (and (= area :eye) (= :eye-pair (:kind owner)))))
|
||||
(throw (ex-info "invalid scoped setting" {:scope scope :id id
|
||||
:knob knob :value value})))
|
||||
(let [changed (assoc-in clip [collection id :params knob] value)
|
||||
subject (if (= scope :subject) id (:subject owner))
|
||||
;; Every feature of the subject is a candidate, not just the edited
|
||||
;; object's own members, because a knob can reach a feature it does not
|
||||
;; belong to: `:aperture-cut` is a mouth setting that the TEETH read.
|
||||
;; `reads` is what narrows this back down, and it is the only thing
|
||||
;; that does — no per-knob cases here, in either direction.
|
||||
candidates (sort-by str (for [[fid f] (:features clip)
|
||||
:when (= subject (:subject f))] fid))]
|
||||
{:changed changed
|
||||
:subject subject
|
||||
:features (vec (filter #(not= (reads clip %) (reads changed %)) candidates))
|
||||
:roles (address/invalidates knob)})))
|
||||
|
||||
(defn- regenerate-feature
|
||||
"One dirty feature, re-measured through the shared anchor and re-frozen. A brow
|
||||
reads the eye corners and the teeth read mouth aperture; `take/measure-part`
|
||||
owns those input edges, and neither one is a request to freeze the other
|
||||
feature."
|
||||
[base-params base source-inputs entry fid]
|
||||
(let [{:keys [area subject]} (get-in entry [:clip :features fid])
|
||||
params (merge base-params (settings (:clip entry) fid))]
|
||||
(when (and (= :teeth area) (nil? (:interior source-inputs)))
|
||||
(throw (ex-info "teeth regeneration needs retained pixel measurements"
|
||||
{:feature fid})))
|
||||
(replace-feature entry
|
||||
(freeze/part subject area params
|
||||
(take/measure-part area params source-inputs @base))
|
||||
fid)))
|
||||
|
||||
(defn- regenerate-head
|
||||
"Re-freeze ONE SUBJECT's head transform, which is a different job from a
|
||||
feature's: its only input is that subject's conditioned anchor, and it owns no
|
||||
channels to replace. The authored `:channels` follow the measurement while they
|
||||
still ARE the measurement, and are left alone once somebody has placed the head
|
||||
by hand."
|
||||
[entry params base subject]
|
||||
(let [baked (freeze/head-part subject params @base)
|
||||
at [:clip :timelines subject :nodes :head]
|
||||
old (get-in entry at)
|
||||
measured (:measured baked)]
|
||||
(cond-> (-> entry
|
||||
(assoc-in (conj at :measured) measured)
|
||||
(update :store merge (:store baked)))
|
||||
(= (:channels old) (:measured old))
|
||||
(assoc-in (conj at :channels) measured))))
|
||||
|
||||
(defn change
|
||||
"One scoped static edit. `source-inputs` holds dense landmarks and, when the
|
||||
teeth are dirty, retained pixel measurements. No IO or app-db here."
|
||||
[{:keys [clip source-inputs] :as entry} edit]
|
||||
(let [{:keys [changed subject features]} (plan clip edit)
|
||||
;; THE EDITED SUBJECT'S OWN LANDMARKS. Retained source is per subject —
|
||||
;; one video, one dense track per tracked face — so re-measuring the
|
||||
;; second face through the first one's anchor is the mistake this lookup
|
||||
;; exists to prevent.
|
||||
source-inputs (get-in source-inputs [:subjects subject])
|
||||
_ (when-not (:dense source-inputs)
|
||||
(throw (ex-info "regeneration needs retained source landmarks"
|
||||
{:subject subject})))
|
||||
base-params (merge take/knobs
|
||||
{:fps (:fps changed)
|
||||
:aspect (get-in changed [:analysis :aspect])
|
||||
:analysis (:analysis changed)})
|
||||
;; One conditioned anchor for the whole edit, and it is the SUBJECT's.
|
||||
;; `:anchor-avg` is a subject setting, so the shared upstream measurement
|
||||
;; is not read off whichever dirty feature happened to sort first — and
|
||||
;; the head below does not need a feature to exist at all.
|
||||
anchor-params (merge base-params (params/for-area :subject)
|
||||
(get-in changed [:subjects subject :params]))
|
||||
base (delay (take/anchor-base anchor-params source-inputs))]
|
||||
(cond-> (reduce (partial regenerate-feature base-params base source-inputs)
|
||||
(assoc entry :clip changed) features)
|
||||
(= :anchor-avg (:knob edit)) (regenerate-head anchor-params base subject))))
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue