noe/arthur
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support multiple regeneratable footage analyses

Browse source
This commit is contained in:
Your Name 2026-10-03 01:24:45 -04:00
parent 5bcf22e458
commit 15deaea19e
20 changed files with 333 additions and 168 deletions
Download patch file Download diff file Expand all files Collapse all files

2
clips/admin.py
View file

@ -18,7 +18,7 @@ class ProjectAdmin(admin.ModelAdmin):
@admin.register(Clip)
class ClipAdmin(admin.ModelAdmin):
list_display = ("cid", "project", "name", "footage", "analysis")
list_display = ("cid", "project", "name")
list_filter = ("project",)

15
clips/migrations/0014_multiple_analyses.py Normal file
View file

@ -0,0 +1,15 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [("clips", "0013_palette_track")]
operations = [
migrations.RemoveField(model_name="clip", name="analysis"),
migrations.RemoveField(model_name="clip", name="footage"),
migrations.AlterField(
model_name="project",
name="schema_version",
field=models.PositiveIntegerField(default=5),
),
]

8
clips/models.py
View file

@ -235,7 +235,7 @@ class Project(models.Model):
settings.AUTH_USER_MODEL, blank=True, related_name="shared_projects",
)
name = models.CharField(max_length=200, default="untitled")
schema_version = models.PositiveIntegerField(default=4)
schema_version = models.PositiveIntegerField(default=5)
seq = models.PositiveBigIntegerField(default=0)
palette = models.CharField(max_length=64, default="arthur/default")
created = models.DateTimeField(auto_now_add=True)
@ -273,12 +273,6 @@ class Clip(models.Model):
cid = models.SlugField(max_length=64)
name = models.CharField(max_length=200, blank=True)
order = models.IntegerField(default=0)
footage = models.ForeignKey(
Footage, null=True, blank=True, on_delete=models.SET_NULL, related_name="clips"
)
analysis = models.ForeignKey(
Analysis, null=True, blank=True, on_delete=models.SET_NULL, related_name="clips"
)
blocks = models.ManyToManyField(
Block, blank=True, related_name="clips",
help_text="the tier-2 blocks this clip's channels name",

21
clips/tests/test_api.py
View file

@ -396,14 +396,25 @@ class DocumentTests(TestCase):
],
}
def save(self, leaves=None, blocks=None):
def save(self, leaves=None, blocks=None, analyses=None):
return self.put(f"/api/projects/{self.project.id}", {
"name": "a project",
"clips": [{"cid": "c1", "name": "take", "analysis": self.analysis,
"clips": [{"cid": "c1", "name": "take",
"analyses": [self.analysis] if analyses is None else analyses,
"leaves": leaves if leaves is not None else self.leaves(),
"blocks": blocks if blocks is not None else [self.block]}],
})
def test_a_clip_declares_the_registered_analyses_its_blocks_name(self):
undeclared = self.save(analyses=[])
self.assertEqual(409, undeclared.status_code)
self.assertIn("every block", undeclared.json()["error"])
unknown = "sha256:" + "f" * 64
missing = self.save(analyses=[self.analysis, unknown])
self.assertEqual(409, missing.status_code)
self.assertEqual([unknown], missing.json()["missing"])
def test_every_saved_symbol_is_listed_across_projects(self):
leaves = self.leaves()
leaves["clip/c1/symbol/sym~face"] = ["^ ", "~:name", "face", "~:frames", 12]
@ -435,12 +446,12 @@ class DocumentTests(TestCase):
self.assertEqual(5, len(response.json()["written"]))
loaded = self.client.get(f"/api/projects/{self.project.id}").json()
self.assertEqual(4, loaded["schema_version"])
self.assertEqual(5, loaded["schema_version"])
self.assertEqual(1, len(loaded["clips"]))
clip = loaded["clips"][0]
self.assertEqual("c1", clip["cid"])
self.assertEqual([self.block], clip["blocks"])
self.assertEqual(self.analysis, clip["analysis"])
self.assertNotIn("analysis", clip)
# The whole point: byte-identical values, including the integer frame keys
# transit writes as "~i0". A JSON round trip that stringified them would
# come back "0" and the part would hold its first pose forever.
@ -542,7 +553,7 @@ class DocumentTests(TestCase):
def patch(self, base, leaves, removed=()):
return self.put(f"/api/projects/{self.project.id}", {
"base": base,
"clips": [{"cid": "c1", "analysis": self.analysis, "leaves": leaves,
"clips": [{"cid": "c1", "analyses": [self.analysis], "leaves": leaves,
"removed": list(removed), "blocks": [self.block]}],
})

27
clips/views.py
View file

@ -718,8 +718,6 @@ def _project_json(project: Project, user):
{
"cid": clip.cid,
"name": clip.name,
"footage": str(clip.footage_id) if clip.footage_id else None,
"analysis": clip.analysis_id,
"blocks": sorted(clip.blocks.values_list("key", flat=True)),
"leaves": {leaf.path: leaf.value for leaf in leaves if leaf.path.startswith(prefix)},
}
@ -909,7 +907,19 @@ def _save(project: Project, data, user):
)
keys = spec.get("blocks") or []
have = set(Block.objects.filter(key__in=keys).values_list("key", flat=True))
analyses = spec.get("analyses") or []
if (not isinstance(analyses, list)
or not all(isinstance(key, str) for key in analyses)
or len(analyses) != len(set(analyses))):
raise Bad("a clip's analyses must be a list of distinct analysis ids")
registered = set(Analysis.objects.filter(key__in=analyses)
.values_list("key", flat=True))
if unknown := [key for key in analyses if key not in registered]:
raise Bad("this clip names analyses the server does not know; register them first",
status=409, missing=unknown)
block_rows = list(Block.objects.filter(key__in=keys))
have = {block.key for block in block_rows}
if missing := [k for k in keys if k not in have]:
# Referential integrity across the tiers, enforced where it can be:
# a document that names blocks the server does not hold would load
@ -919,6 +929,9 @@ def _save(project: Project, data, user):
"before saving the document that points at them",
status=409, missing=missing,
)
if undeclared := sorted({block.analysis_id for block in block_rows} - registered):
raise Bad("every block in a clip must name one of that clip's analyses",
status=409, missing=undeclared)
existing = {leaf.path: leaf for leaf in project.leaves.filter(path__startswith=prefix)}
if base is not None:
@ -930,16 +943,12 @@ def _save(project: Project, data, user):
if conflicts:
continue
analysis = Analysis.objects.filter(key=spec.get("analysis")).first()
footage = None
if spec.get("footage"):
footage = Footage.objects.filter(id=spec["footage"]).first()
clip, _ = Clip.objects.update_or_create(
project=project,
cid=cid,
defaults={"name": spec.get("name") or "", "analysis": analysis, "footage": footage},
defaults={"name": spec.get("name") or ""},
)
blocks = Block.objects.filter(key__in=keys)
blocks = block_rows
if base is None:
clip.blocks.set(blocks)
gone = [path for path in existing if path not in leaves]

73
frontend/src/arthur/domain/bring.cljs
View file

@ -11,6 +11,7 @@
so the events that fetch them are only fetching."
(:refer-clojure :exclude [take])
(:require [arthur.domain.clip :as clip]
[arthur.domain.feature :as feature]
[arthur.domain.node :as node]
[clojure.string :as string]))
@ -59,8 +60,8 @@
(keyword (if (seq slug) slug "symbol"))))
(defn take
"Put `frozen`, a take, into `clip` as ONE symbol called `label`. Returns
`{:clip :sid :tracked?}`.
"Put `frozen`, a take, into `clip` as ONE symbol called `label`. Returns the
changed clip, the imported symbol id and the old-to-scoped subject ids.
`frozen` is what `flow/freeze/clip` makes: a `:main` that places one symbol per
tracked face. `:main` becomes the named symbol — it is what holds the faces in
@ -75,18 +76,22 @@
symbols. `nest/audio-tracks` collapses simultaneous copies carrying the same
`:media-link`; placing those faces at different times still schedules each one.
The tracking identities, and the analysis they were measured by, come along
only when `clip` has no analysis of its own and no face had to be renamed. A
document holds one analysis, and regeneration finds a face's symbol by its
subject id, so either condition failing means the take comes in as drawings
that play but cannot be re-tuned — `:tracked? false` says so."
The unique imported symbol id scopes every subject id. Thus two takes may both
arrive with detector subject `:face-1` without colliding in the document."
[clip frozen label footage-id range]
(let [{c :clip ids :ids} (symbols clip frozen [:main] {:main (symbol-id label)})
(let [scope (fn [root subject] (keyword (str (name root) "." (name subject))))
root (clip/free-id
(fn [candidate]
(or (contains? (:symbols clip) candidate)
(some #(contains? (:symbols clip) (scope candidate %))
(keys (:subjects frozen)))))
(symbol-id label))
scoped (partial scope root)
wanted (into {:main root} (map (fn [subject] [subject (scoped subject)]))
(keys (:subjects frozen)))
{c :clip ids :ids} (symbols clip frozen [:main] wanted)
sid (ids :main)
faces (mapv ids (sort-by str (keys (:subjects frozen))))
;; Older/plain imported clips have no subject table. They are still a
;; take, so their wrapper remains the only honest owner of the sound.
sound-hosts (if (seq faces) faces [sid])
sound {:id :sound :name "sound" :kind :audio :parent nil
:z "z-sound" :source {:footage footage-id}
;; All automatic copies name one recording. The audio walk
@ -96,18 +101,42 @@
;; Source frame `start` plays on the symbol's 0.
:span range
:time {:mode :map :at (- (first range)) :rate 1}}
tracked? (and (nil? (:analysis clip))
(every? #(= % (ids %)) (keys (:subjects frozen))))]
feature-ids (into {}
(map (fn [[id f]]
[id (feature/owned (ids (:subject f)) (keyword (name id)))]))
(:features frozen))
subjects (into {}
(map (fn [[id subject]]
(let [new-id (ids id)]
[new-id (assoc subject :id new-id
:footage footage-id)])))
(:subjects frozen))
features (into {}
(map (fn [[id f]]
(let [new-id (feature-ids id)]
[new-id (-> f
(assoc :id new-id
:subject (ids (:subject f))
:symbol (ids (:symbol f))))])))
(:features frozen))
groups (into {}
(map (fn [[id g]]
(let [new-subject (ids (:subject g))
new-id (feature/owned new-subject (keyword (name id)))]
[new-id (-> g
(assoc :id new-id :subject new-subject)
(update :members #(mapv feature-ids %)))])))
(:groups frozen))]
{:sid sid
:tracked? tracked?
:clip (cond-> (-> (reduce (fn [document face]
(assoc-in document [:symbols face :nodes :sound] sound))
c sound-hosts)
(assoc-in [:symbols sid :name] (str label)))
tracked? (-> (assoc :analysis (:analysis frozen))
(update :subjects merge (:subjects frozen))
(update :features merge (:features frozen))
(update :groups merge (:groups frozen))))}))
:subject-ids (select-keys ids (keys (:subjects frozen)))
:clip (-> (reduce (fn [document face]
(assoc-in document [:symbols face :nodes :sound] sound))
c faces)
(assoc-in [:symbols sid :name] (str label))
(update :analyses merge (:analyses frozen))
(update :subjects merge subjects)
(update :features merge features)
(update :groups merge groups))}))
(defn placed

16
frontend/src/arthur/domain/clip.cljs
View file

@ -4,7 +4,7 @@
{:name \"take\"
:fps 30
:width 320 :height 200
:analysis {...}
:analyses {analysis-id {...}}
:subjects {...} :features {...} :groups {...}
:symbols {:main {:id :main :frames 229 :nodes {...}}}}
@ -51,7 +51,7 @@
that loses something on every round trip, which is the one bug a persistence
layer must not be able to have. Add the field here and to `leaf/leaves` and
`leaf/clip` in the same commit."
#{:name :fps :analysis :subjects :features :groups :width :height :symbols
#{:name :fps :analyses :subjects :features :groups :width :height :symbols
:palettes :default-palette})
(defn symbol
@ -574,6 +574,18 @@
(str "clip has a field with no leaf to save it in: " (pr-str k)))
(when-not (map? (:symbols clip))
[":symbols must be a map of id -> symbol"])
(when (and (contains? clip :analyses) (not (map? (:analyses clip))))
[":analyses must be a map of analysis id -> analysis"])
(for [[id analysis] (:analyses clip)
:when (not= id (:id analysis))]
(str "analysis under key " (pr-str id) " has :id " (pr-str (:id analysis))))
(for [[id subject] (:subjects clip)
:when (not (contains? (:analyses clip) (:analysis subject)))]
(str "subject " (pr-str id) " names missing analysis "
(pr-str (:analysis subject))))
(for [[id subject] (:subjects clip)
:when (not (keyword? (:source-subject subject)))]
(str "subject " (pr-str id) " has no source subject"))
(when (and (contains? clip :palettes) (not (map? (:palettes clip))))
[":palettes must be a map of id -> palette"])
(when (and (contains? clip :default-palette) (map? (:palettes clip))

6
frontend/src/arthur/domain/leaf.cljs
View file

@ -137,7 +137,7 @@
(some-leaf (at "timing") (select-keys clip [:fps]))
(some-leaf (at "stage") (select-keys clip [:width :height]))
(some-leaf (at "palette-default") (select-keys clip [:default-palette]))
(some-leaf (at "source") (:analysis clip))
(some-leaf (at "analyses") (:analyses clip))
(concat
(for [[id v] (:subjects clip)] {(at "subject" (segment id)) v})
(for [[id v] (:features clip)] {(at "feature" (segment id)) v})
@ -194,7 +194,7 @@
"name" (merge acc v)
"timing" (merge acc v)
"stage" (merge acc v)
"source" (assoc acc :analysis v)
"analyses" (assoc acc :analyses v)
"palette-default" (merge acc v)
"palette" (assoc-in acc [:palettes (unsegment a)] v)
"subject" (assoc-in acc [:subjects (unsegment a)] v)
@ -237,7 +237,7 @@
false)
(case (count p)
;; The clip's own facts carry no id.
3 (#{"name" "timing" "stage" "source" "palette-default"} (nth p 2))
3 (#{"name" "timing" "stage" "analyses" "palette-default"} (nth p 2))
4 (#{"subject" "feature" "group" "palette"} (nth p 2))
false))))]
(vec

4
frontend/src/arthur/domain/project.cljs
View file

@ -29,9 +29,7 @@
(:require [arthur.domain.leaf :as leaf]
[arthur.domain.wire :as wire]))
(def schema-version
"4 separates a symbol's static authoring palette from its palette track."
4)
(def schema-version 5)
(defn block-keys
"Every tier-2 key a leaf map names, in a stable order."

28
frontend/src/arthur/events/footage.cljs
View file

@ -151,7 +151,8 @@
:detector detector})
_ (mark! "build-clip: freeze")
built (:clip frozen)
source-blocks (source/pack-subjects (:id (:analysis built)) subjects)
analysis-id (-> built :analyses keys first)
source-blocks (source/pack-subjects analysis-id subjects)
_ (mark! "build-clip: pack source blocks")]
(assoc (select-keys built [:fps :width :height])
@ -504,7 +505,7 @@
(let [entry (store/entry (:clip/current db))
uuid (random-uuid)
fps (get-in db [:clip :fps])
{:keys [clip sid tracked?]}
{:keys [clip sid subject-ids]}
(bring/take (:clip entry) (:clip built) name footage-id range)
st (merge (:store entry) (:store built))
imported-frames (clip/output-frames clip sid)
@ -525,9 +526,22 @@
(update :footage merge {:loading? false :status why}))}
(let [db (edit/edit-entry
db
#(cond-> (assoc % :clip (:clip result) :store st)
tracked? (merge (select-keys built [:footage-id :source-blocks
:source-inputs]))))]
#(let [analysis-id (-> built :clip :analyses keys first)
remap-subjects
(fn [inputs]
(update inputs :subjects
(fn [subjects]
(into {} (map (fn [[old new]] [new (get subjects old)]))
subject-ids))))]
(-> (assoc % :clip (:clip result) :store st)
(update-in [:sources analysis-id]
(fn [source]
{:footage-id (:footage-id built)
:source-blocks (:source-blocks built)
:source-inputs
(update (remap-subjects (:source-inputs built))
:subjects merge
(get-in source [:source-inputs :subjects]))})))))]
{:db (-> db
(update :ui dissoc :convert)
(ui/selected
@ -536,7 +550,5 @@
{:loading? false
:status (str "made " name " · " imported-frames " frames at " fps " fps"
(when (not= fps source-fps)
(str " · sampled from " source-fps " fps"))
(when-not tracked?
" · as drawings: this project already tracks other footage"))}))
(str " · sampled from " source-fps " fps")))}))
:dispatch [::pb/refresh-clock]})))))

159
frontend/src/arthur/events/project.cljs
View file

@ -102,15 +102,10 @@
(.then (fn [^js created] (.-id created))))))
(defn- opened-entry! [^js clip-json]
(let [footage-id (.-footage clip-json)]
(-> (js/Promise.all
#js [(js/Promise.all
(into-array (map #(http/GET (str "/api/blocks/" %))
(array-seq (.-blocks clip-json)))))
(if footage-id
(http/GET (str "/api/footage/" footage-id))
(js/Promise.resolve nil))])
(.then (fn [[blocks ^js footage]]
(-> (js/Promise.all
(into-array (map #(http/GET (str "/api/blocks/" %))
(array-seq (.-blocks clip-json)))))
(.then (fn [blocks]
(let [cid (.-cid clip-json)
loaded (project/load
cid #js {:leaves (.-leaves clip-json)
@ -129,11 +124,9 @@
;; write lands on what does not.
:synced (project/tier1 (.-leaves clip-json))
:clip built :store (:store loaded)
:footage-id footage-id
:audio (if footage (.-audio footage)
"/static/arthur/audio.wav")})]
:audio "/static/arthur/audio.wav"})]
(-> (mix/mix! built (clip/opens-on built) (:audio entry) (:store entry))
(.then (fn [audio] (assoc entry :audio audio)))))))))))
(.then (fn [audio] (assoc entry :audio audio))))))))))
(defn- saved-clip!
"Promise of clip `cid` of saved project `pid`, as `{:clip :store}`: its
@ -240,46 +233,60 @@
:removed (into-array (remove #(contains? local %) (keys synced)))})
#js {:leaves (.-leaves doc)}))
(defonce ^:private on-server
;; Analyses and block keys this page has already put on the server. Content
;; addressed, so once there they are there: a save of a moved vertex asks for
;; none of it again, and is one request.
(defonce ^:private uploaded-blocks
;; Block keys this page has already put on the server. Content addressed, so
;; once there they are there: a save of a moved vertex asks for none again.
(atom #{}))
(defonce ^:private linked-analyses (atom #{}))
(defn- upload-new! [^js doc]
(let [keys (array-seq (block-keys doc))]
(if (every? @on-server keys)
(if (every? @uploaded-blocks keys)
(js/Promise.resolve 0)
(.then (upload-missing! doc) (fn [n] (swap! on-server into keys) n)))))
(.then (upload-missing! doc) (fn [n] (swap! uploaded-blocks into keys) n)))))
(defn- analyses-of [entry]
(vals (get-in entry [:clip :analyses])))
(defn- upload-sources! [sources]
(reduce
(fn [chain [analysis-id {:keys [source-blocks]}]]
(.then chain
(fn [_]
(when (and (seq source-blocks)
(not (@linked-analyses analysis-id)))
(-> (upload-missing! #js {:blocks (source/upload-blocks source-blocks)})
(.then #(http/PUT
(str "/api/analyses/" analysis-id)
#js {:source_blocks
(into-array (source/block-keys source-blocks))}))
(.then (fn [answer]
(swap! linked-analyses conj analysis-id)
answer)))))))
(js/Promise.resolve nil)
sources))
(rf/reg-fx
::save!
(fn [{:keys [id cid label clip base]}]
(let [analysis (:analysis (:clip clip))
doc (project/save cid clip)
local (leaf/leaves cid (:clip clip))
base (when (and id (:synced clip)) base)
source-blocks (:source-blocks clip)]
(let [entry clip
analyses (analyses-of entry)
doc (project/save cid entry)
local (leaf/leaves cid (:clip entry))
base (when (and id (:synced entry)) base)
sources (:sources entry)]
(-> (ensure-project! id label)
(.then (fn [pid]
(-> (if (and analysis (not (@on-server (:id analysis))))
(http/POST "/api/analyses" (analysis-payload analysis))
(js/Promise.resolve nil))
(-> (reduce (fn [chain one]
(.then chain
#(http/POST "/api/analyses"
(analysis-payload one))))
(js/Promise.resolve nil)
analyses)
(.then (fn [_]
(when (and (seq source-blocks) (not (@on-server (:id analysis))))
(-> (upload-missing!
#js {:blocks (source/upload-blocks source-blocks)})
(.then (fn [_]
(http/PUT
(str "/api/analyses/" (:id analysis))
;; One set per tracked subject, in
;; the order `source/unpack` does
;; not depend on.
#js {:source_blocks
(into-array
(source/block-keys source-blocks))})))))))
(upload-sources! sources)))
(.then (fn [_]
(when analysis (swap! on-server conj (:id analysis)))
(upload-new! doc)))
(.then (fn [uploaded]
(-> (http/PUT (str "/api/projects/" pid)
@ -288,11 +295,11 @@
:clips #js [(js/Object.assign
#js {:cid cid
:name label
:analysis (:id analysis)
:footage (:footage-id clip)
:analyses (into-array
(keys (get-in entry [:clip :analyses])))
:blocks (block-keys doc)}
(clip-payload doc base local
(:synced clip)))]})
(:synced entry)))]})
(.then (fn [^js saved]
(rf/dispatch [::saved pid cid label
(.-seq saved)
@ -439,39 +446,38 @@
(assoc measured :interior-key block-key))))))))
(throw error)))))))
(defn- source-for! [entry]
(if-let [inputs (:source-inputs entry)]
(js/Promise.resolve inputs)
(let [analysis (get-in entry [:clip :analysis])]
(if (= (:id analysis) (:id @retained-source))
(defn- source-for! [entry edit]
(let [subject (:subject (regenerate/plan (:clip entry) edit))
subject-record (get-in entry [:clip :subjects subject])
analysis-id (:analysis subject-record)
analysis (get-in entry [:clip :analyses analysis-id])
source-subject (:source-subject subject-record)
local (get-in entry [:sources analysis-id :source-inputs :subjects subject])]
(if local
(js/Promise.resolve {:subjects {subject local}})
(if (= [analysis-id subject] (:key @retained-source))
(:promise @retained-source)
(let [promise
(if (= "synth" (:detector analysis))
;; The synthetic take tracks one face and regenerating it reads
;; that face's landmarks, so it arrives in the same shape real
;; footage does rather than in a flat one only this branch uses.
(js/Promise.resolve
{:subjects
{:face-1 {:dense (synth/synth-dense (:frames analysis)
{:seed (:seed analysis)})}}})
(-> (ingest/manifest! (:footage-id entry))
(.then (fn [manifest]
(-> (footage/saved-source! (:id analysis)
[(:width manifest)
(:height manifest)])
(.then (fn [inputs]
(when-not inputs
(throw (ex-info "saved analysis has no source blocks" {})))
(update inputs :subjects
(fn [subjects]
(into {}
(map (fn [[id one]]
[id (assoc one :presence
(footage/presence-for
manifest id))]))
subjects))))))))))]
{subject {:dense (synth/synth-dense (:frames analysis)
{:seed (:seed analysis)})}}})
(.then (ingest/manifest! (:footage subject-record))
(fn [manifest]
(.then (footage/saved-source!
analysis-id [(:width manifest) (:height manifest)])
(fn [inputs]
(when-not inputs
(throw (ex-info "saved analysis has no source blocks" {})))
{:subjects
{subject
(assoc (get-in inputs [:subjects source-subject])
:presence
(footage/presence-for manifest
source-subject))}})))))]
(do
(reset! retained-source {:id (:id analysis) :promise promise})
(reset! retained-source {:key [analysis-id subject] :promise promise})
promise))))))
(defn- inputs-for-edit!
@ -484,18 +490,19 @@
one (get-in inputs [:subjects subject])]
(if (and teeth (:crops one))
(let [settings (merge take/knobs (feature/effective-params clip teeth))
analysis (get-in clip [:analysis :id])
analysis (get-in clip [:subjects subject :analysis])
source-subject (get-in clip [:subjects subject :source-subject])
frames (count (:crops one))
key (source/interior-key analysis subject settings frames)
key (source/interior-key analysis source-subject settings frames)
done (fn [measured] (assoc-in inputs [:subjects subject] measured))]
(if (and (:interior one)
(or (= (:interior-key one) key)
(and (nil? (:interior-key one))
(= key (source/interior-key analysis subject take/knobs frames)))))
(= key (source/interior-key analysis source-subject take/knobs frames)))))
(js/Promise.resolve inputs)
(.then (if (= key (:key @retained-interior))
(:promise @retained-interior)
(let [promise (retained-interior! analysis subject settings one)]
(let [promise (retained-interior! analysis source-subject settings one)]
(reset! retained-interior {:key key :promise promise})
promise))
done)))
@ -504,7 +511,7 @@
(rf/reg-fx
::preview-settings!
(fn [{:keys [id entry edit request]}]
(-> (source-for! entry)
(-> (source-for! entry edit)
(.then (fn [inputs] (inputs-for-edit! entry edit inputs)))
(.then (fn [inputs]
(regenerate/change (assoc entry :source-inputs inputs) edit)))
@ -519,7 +526,7 @@
(fn [{:keys [db]} [_ edit]]
(let [id (:clip/current db)
entry (store/entry id)]
(if (or (get-in db [:project :busy?]) (nil? (:analysis (:clip entry))))
(if (or (get-in db [:project :busy?]) (empty? (:analyses (:clip entry))))
{}
(let [plan (regenerate/plan (:clip entry) edit)
report (select-keys plan [:features :roles])

8
frontend/src/arthur/flow/freeze.cljs
View file

@ -762,11 +762,15 @@
{:frames (vec lengths)})))
nf (first lengths)
merged (fn [k] (into {} (mapcat (comp k second)) parts))
built {:name name :fps fps :analysis (:analysis params)
analysis (:analysis params)
built {:name name :fps fps :analyses {(:id analysis) analysis}
:width (first stage) :height (second stage)
:palettes {pal/default-id pal/default-palette}
:default-palette pal/default-id
:subjects (into {} (map (fn [[id _]] [id {:id id :params {}}])) ordered)
:subjects (into {} (map (fn [[id _]]
[id {:id id :params {}
:analysis (:id analysis)
:source-subject id}])) ordered)
:features (merged :features) :groups (merged :groups)
:symbols
(into {:main

9
frontend/src/arthur/flow/regenerate.cljs
View file

@ -142,6 +142,11 @@
teeth are dirty, retained pixel measurements. No IO or app-db here."
[{:keys [clip source-inputs] :as entry} edit]
(let [{:keys [changed subject features]} (plan clip edit)
analysis-id (get-in changed [:subjects subject :analysis])
analysis (get-in changed [:analyses analysis-id])
_ (when-not analysis
(throw (ex-info "subject names a missing analysis"
{:subject subject :analysis analysis-id})))
;; THE EDITED SUBJECT'S OWN LANDMARKS. Retained source is per subject —
;; one video, one dense track per tracked face — so re-measuring the
;; second face through the first one's anchor is the mistake this lookup
@ -152,8 +157,8 @@
{:subject subject})))
base-params (merge take/knobs
{:fps (or (get-in changed [:symbols subject :fps]) (:fps changed))
:aspect (get-in changed [:analysis :aspect])
:analysis (:analysis changed)})
:aspect (:aspect analysis)
:analysis analysis})
;; One conditioned anchor for the whole edit, and it is the SUBJECT's.
;; `:anchor-avg` is a subject setting, so the shared upstream measurement
;; is not read off whichever dirty feature happened to sort first — and

1
frontend/src/arthur/subs/render.cljs
View file

@ -212,7 +212,6 @@
;; row is not on stage, so neither is its footage.
(let [solo (filter #(placed? document open %) solo)]
{:document document :store store
:footage-id (:footage-id (footage/entry id))
:opacity (or opacity trace/opacity-default)
:traces (cond->> (when document (trace/shown document open faces))
(seq solo) (filterv (fn [{:keys [path]}]

3
frontend/src/arthur/subs/ui.cljs
View file

@ -156,7 +156,8 @@
;; than kept beside it, so a sound placed from footage makes that footage the
;; project's without anything else being told.
(let [entry (store/entry id)
used (into (set (keep identity [(:footage-id entry)])) uploaded)
used (into (set uploaded)
(keep :footage (vals (get-in entry [:clip :subjects]))))
used (into used (for [[_ sym] (get-in entry [:clip :symbols])
[_ n] (:nodes sym)
:let [f (get-in n [:source :footage])]

21
frontend/src/arthur/ui/underlay.cljs
View file

@ -129,7 +129,7 @@
says where each face's head went and which of its frames it was on, so this
reads the frame rather than resolving it again. `on-ready` is called when a
still or a manifest that was missing arrives, to paint again."
[{:keys [document store footage-id traces opacity width playing?]} resolver on-ready]
[{:keys [document store traces opacity width playing?]} resolver on-ready]
(when-let [^js canvas (:canvas @state)]
(let [ctx (.getContext canvas "2d")
zoom (/ (.-width canvas) width)]
@ -138,17 +138,20 @@
;; What each face is showing, forgotten for the faces switched off. Before
;; the early exits, so switching them all off forgets all of them.
(swap! state update :last select-keys (map :path traces))
(when-let [us (and (seq traces) footage-id (urls footage-id on-ready))]
(let [start (first (get-in document [:analysis :range] [0]))
;; Read once: each face writes only its own entry below, and what it
;; was showing is what it falls back to. A face that has been off
;; stage for a few frames still has the still it went away with.
was (:last @state)]
(when (seq traces)
(let [;; Read once: each face writes only its own entry below, and what it
;; was showing is what it falls back to.
was (:last @state)]
;; ONE OPACITY for all of them, set once: how strongly the reference
;; draws is a property of looking at the stage, not of a face.
(set! (.-globalAlpha ctx) opacity)
(doseq [{:keys [path face]} traces]
(let [at (conj path :head)
(doseq [{:keys [path face]} traces
:let [subject (get-in document [:subjects face])
analysis (get-in document [:analyses (:analysis subject)])
us (urls (:footage subject) on-ready)]
:when us]
(let [start (first (get analysis :range [0]))
at (conj path :head)
world (symbol/world-of resolver at)
frame (symbol/frame-of resolver at)]
(when (and world (number? frame))

31
frontend/test/arthur/domain/bring_test.cljs
View file

@ -26,3 +26,34 @@
(vals (get-in clip [:symbols :take :nodes]))))))
"and the copy's instance follows its renamed symbol")
(is (empty? (clip/problems clip)))))
(defn- tracked-take [analysis-id]
{:analyses {analysis-id {:id analysis-id :detector "test" :version "1"}}
:subjects {:face-1 {:id :face-1 :analysis analysis-id
:source-subject :face-1 :params {}}}
:features {:face-1/mouth {:id :face-1/mouth :subject :face-1
:symbol :face-1 :area :mouth :nodes [:mouth] :params {}}}
:groups {}
:symbols {:main {:id :main :fps 30 :frames 10
:nodes {:face-1 {:id :face-1 :kind :instance :z "a1"
:source {:symbol :face-1}}}}
:face-1 {:id :face-1 :fps 30 :frames 10
:nodes {:head {:id :head :kind :group :z "a1"
:measured {[:xform :pos]
{:animated? false :value [0 0]}}}
:mouth {:id :mouth :kind :poly :z "a2"}}}}})
(deftest every-import-is-tracked-under-its-unique-symbol-scope
(let [first (bring/take (clip/blank) (tracked-take "analysis-a")
"close up" "footage-a" [0 10])
second (bring/take (:clip first) (tracked-take "analysis-b")
"close up" "footage-b" [0 10])
c (:clip second)]
(is (= #{:close-up.face-1 :close-up-2.face-1} (set (keys (:subjects c)))))
(is (= #{"analysis-a" "analysis-b"} (set (keys (:analyses c)))))
(is (= "analysis-a" (get-in c [:subjects :close-up.face-1 :analysis])))
(is (= "analysis-b" (get-in c [:subjects :close-up-2.face-1 :analysis])))
(is (= :face-1 (get-in c [:subjects :close-up-2.face-1 :source-subject])))
(is (= #{:close-up.face-1/mouth :close-up-2.face-1/mouth}
(set (keys (:features c)))))
(is (empty? (clip/problems c)))))

30
frontend/test/arthur/domain/cadence_test.cljs
View file

@ -90,8 +90,17 @@
(is (or (zero? f) (< (clip/shown-frame doc :main (dec f)) n))))))))
(deftest imported-footage-uses-selection-for-picture-and-real-speed-for-audio
(let [{doc :clip sid :sid} (bring/take (clip/set-fps (clip/blank) 12)
footage "take" "video" [15 75])
(let [tracked (-> footage
(assoc :analyses {"a" {:id "a"}}
:subjects {:face-1 {:id :face-1 :analysis "a"
:source-subject :face-1}}
:features {} :groups {}
:symbols {:main {:id :main :fps 30 :frames 60
:nodes {:face-1 {:id :face-1 :kind :instance
:z "a" :source {:symbol :face-1}}}}
:face-1 (get-in footage [:symbols :main])}))
{doc :clip sid :sid} (bring/take (clip/set-fps (clip/blank) 12)
tracked "take" "video" [15 75])
;; A host authored at 12, with a 30fps source starting half a second in.
doc (-> doc (assoc-in [:symbols :main :fps] 12)
(clip/place-symbol store :main sid 6 :insert nil))
@ -99,21 +108,16 @@
draw (clip/resolver doc :main store pal/index-of nil)
[sound] (nest/audio-tracks doc :main)]
(is (= 60 (clip/frames doc sid)))
(is (= 1 (get-in doc [:symbols sid :nodes :sound :time :rate])))
(is (= 1 (get-in doc [:symbols :take.face-1 :nodes :sound :time :rate])))
(is (= [0 24] (:span n)))
(is (empty? (draw 5)))
(is (= 8 (:size (first (draw 9)))))
(is (= 7 (:frame (nest/inside doc store :main [:insert :mark] 9))))
(is (= 7 (:frame (nest/inside doc store :main [:insert :face-1 :mark] 9))))
(is (= [-4 -4 4 4] ((pick/bounds-of doc store :main n) 3)))
(is (= [6 30] (node/placed-span sound)))
(is (= 15 (node/local-frame sound 6)))
(is (= 1 (* (get-in sound [:time :rate]) (/ (:fps doc) (:fps sound)))))
(is (= [6 30] (:span (first (timeline/rows doc :main #{[:insert]})))))
(testing "an off-grid closed mouth can win without modifying the dense source"
(let [pinned (pose/put-cut doc :main :insert :mouth 7 6)
r (clip/resolver pinned :main store pal/index-of nil)]
(is (= 7 (:size (first (r 9)))))
(is (= (get-in doc [:symbols sid]) (get-in pinned [:symbols sid])))))
(testing "a deliberate half speed still retimes audio"
(let [slow (assoc-in doc [:symbols :main :nodes :insert :playback :speed] 0.5)
[track] (nest/audio-tracks slow :main)]
@ -121,7 +125,9 @@
(deftest generated-faces-own-their-footage-sound
(let [face (fn [id] {:id id :fps 30 :frames 20 :nodes {}})
frozen {:fps 30 :subjects {:face-1 {} :face-2 {}}
frozen {:fps 30 :analyses {"a" {:id "a"}}
:subjects {:face-1 {:id :face-1 :analysis "a" :source-subject :face-1}
:face-2 {:id :face-2 :analysis "a" :source-subject :face-2}}
:symbols {:main {:id :main :fps 30 :frames 20
:nodes {:face-1 {:id :face-1 :kind :instance :z "a1"
:source {:symbol :face-1}}
@ -130,11 +136,11 @@
:face-1 (face :face-1)
:face-2 (face :face-2)}}
{doc :clip take :sid} (bring/take (clip/blank) frozen "take" "video" [3 13])
alone (clip/place-symbol doc nil :main :face-1 4 :placed-face nil)]
alone (clip/place-symbol doc nil :main :take.face-1 4 :placed-face nil)]
(is (nil? (get-in doc [:symbols take :nodes :sound]))
"the wrapper does not own a sound the face would lose")
(is (= {:footage "video"}
(get-in doc [:symbols :face-1 :nodes :sound :source])))
(get-in doc [:symbols :take.face-1 :nodes :sound :source])))
(is (= 1 (count (nest/audio-tracks doc take)))
"the same take recording is not mixed once per detected face")
(is (= 1 (count (nest/audio-tracks alone :main)))

2
frontend/test/arthur/domain/leaf_test.cljs
View file

@ -45,7 +45,7 @@
(let [ls (leaf/leaves :c7 @take/clip)]
(is (contains? ls "clip/c7/timing"))
(is (contains? ls "clip/c7/stage"))
(is (contains? ls "clip/c7/source"))
(is (contains? ls "clip/c7/analyses"))
;; A TIMELINE ID IS A SEGMENT, which is what lets a symbol's nodes be
;; addressed by the same path shape as the clip's own. `main` is the root.
(is (contains? ls "clip/c7/symbol/main"))

37
frontend/test/arthur/flow/regenerate_test.cljs
View file

@ -2,6 +2,7 @@
(:require [cljs.test :refer [deftest is testing]]
[clojure.walk :as walk]
[arthur.demo.stage :as stage]
[arthur.domain.bring :as bring]
[arthur.domain.channel :as ch]
[arthur.domain.clip :as clip]
[arthur.domain.params :as params]
@ -29,13 +30,41 @@
(merge take/knobs
{:name "regen" :fps 30 :aspect 1 :stage [320 200]
:expose 1 :head :free
:analysis (get-in @initial [:clip :analysis])}
:analysis (-> @initial :clip :analyses vals first)}
change)
{:face-1 @inputs}))
(defn- channel [entry node path]
(get-in entry [:clip :symbols :face-1 :nodes node :channels path]))
(deftest a-second-import-regenerates-through-its-own-analysis
(let [built (fn [seed]
(let [input {:dense (synth/synth-dense frames {:seed seed})}
analysis (address/analysis
{:detector "synth" :version "mulberry32"
:seed seed :frames frames :fps 30 :aspect 1})]
[(take/build (merge take/knobs
{:name "take" :fps 30 :aspect 1 :stage [320 200]
:expose 1 :head :free :analysis analysis})
{:face-1 input})
input]))
[a input-a] (built 3)
[b input-b] (built 7)
first (bring/take (clip/blank) (:clip a) "shot" "footage-a" [0 frames])
second (bring/take (:clip first) (:clip b) "shot" "footage-b" [0 frames])
entry {:clip (:clip second) :store (merge (:store a) (:store b))
:source-inputs {:subjects {:shot.face-1 input-a
:shot-2.face-1 input-b}}}
fid :shot-2.face-1/mouth
changed (regenerate/change entry
{:scope :feature :id fid :knob :verts :value 10})
analysis-id (get-in changed [:clip :subjects :shot-2.face-1 :analysis])
generated (get-in changed [:clip :symbols :shot-2.face-1 :nodes :mouth
:channels [:geom :pts] :generated])]
(is (= (get-in b [:clip :subjects :face-1 :analysis]) analysis-id))
(is (= analysis-id (:analysis generated)))
(is (= 10 (get-in generated [:params :verts])))))
(deftest eye-rebuild-is-confined-to-the-edited-feature
(let [before @initial
after (regenerate/change before
@ -170,7 +199,7 @@
(defn- part-at [area overrides]
(let [p (merge take/knobs
{:fps 30 :aspect 1 :analysis (get-in @initial [:clip :analysis])}
{:fps 30 :aspect 1 :analysis (-> @initial :clip :analyses vals first)}
overrides)
inputs (assoc @inputs :interior @interior-track)]
(freeze/part :face-1 area p (take/measure-part area p inputs (take/anchor-base p inputs)))))
@ -212,8 +241,8 @@
;; document that cannot be previewed fails in the suite rather than as a slider
;; that silently does nothing in the browser.
(let [entry (staged)]
(is (some? (:analysis (:clip entry)))
"the composed stage keeps the analysis the edit needs")
(is (seq (:analyses (:clip entry)))
"the composed stage keeps the analyses its edits need")
(is (some? (:source-inputs entry)))
(testing "and the features still say which timeline they live in"
(is (every? #(= :face-1 (:symbol %))