diff --git a/clips/admin.py b/clips/admin.py index 9e73270..8b7add7 100644 --- a/clips/admin.py +++ b/clips/admin.py @@ -18,7 +18,7 @@ class ProjectAdmin(admin.ModelAdmin): @admin.register(Clip) class ClipAdmin(admin.ModelAdmin): - list_display = ("cid", "project", "name", "footage", "analysis") + list_display = ("cid", "project", "name") list_filter = ("project",) diff --git a/clips/migrations/0014_multiple_analyses.py b/clips/migrations/0014_multiple_analyses.py new file mode 100644 index 0000000..97fe72b --- /dev/null +++ b/clips/migrations/0014_multiple_analyses.py @@ -0,0 +1,15 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [("clips", "0013_palette_track")] + + operations = [ + migrations.RemoveField(model_name="clip", name="analysis"), + migrations.RemoveField(model_name="clip", name="footage"), + migrations.AlterField( + model_name="project", + name="schema_version", + field=models.PositiveIntegerField(default=5), + ), + ] diff --git a/clips/models.py b/clips/models.py index f304b71..9eeb4b3 100644 --- a/clips/models.py +++ b/clips/models.py @@ -235,7 +235,7 @@ class Project(models.Model): settings.AUTH_USER_MODEL, blank=True, related_name="shared_projects", ) name = models.CharField(max_length=200, default="untitled") - schema_version = models.PositiveIntegerField(default=4) + schema_version = models.PositiveIntegerField(default=5) seq = models.PositiveBigIntegerField(default=0) palette = models.CharField(max_length=64, default="arthur/default") created = models.DateTimeField(auto_now_add=True) @@ -273,12 +273,6 @@ class Clip(models.Model): cid = models.SlugField(max_length=64) name = models.CharField(max_length=200, blank=True) order = models.IntegerField(default=0) - footage = models.ForeignKey( - Footage, null=True, blank=True, on_delete=models.SET_NULL, related_name="clips" - ) - analysis = models.ForeignKey( - Analysis, null=True, blank=True, on_delete=models.SET_NULL, related_name="clips" - ) blocks = models.ManyToManyField( Block, blank=True, related_name="clips", help_text="the tier-2 blocks this clip's channels name", diff --git a/clips/tests/test_api.py b/clips/tests/test_api.py index 68f474f..26e8e73 100644 --- a/clips/tests/test_api.py +++ b/clips/tests/test_api.py @@ -396,14 +396,25 @@ class DocumentTests(TestCase): ], } - def save(self, leaves=None, blocks=None): + def save(self, leaves=None, blocks=None, analyses=None): return self.put(f"/api/projects/{self.project.id}", { "name": "a project", - "clips": [{"cid": "c1", "name": "take", "analysis": self.analysis, + "clips": [{"cid": "c1", "name": "take", + "analyses": [self.analysis] if analyses is None else analyses, "leaves": leaves if leaves is not None else self.leaves(), "blocks": blocks if blocks is not None else [self.block]}], }) + def test_a_clip_declares_the_registered_analyses_its_blocks_name(self): + undeclared = self.save(analyses=[]) + self.assertEqual(409, undeclared.status_code) + self.assertIn("every block", undeclared.json()["error"]) + + unknown = "sha256:" + "f" * 64 + missing = self.save(analyses=[self.analysis, unknown]) + self.assertEqual(409, missing.status_code) + self.assertEqual([unknown], missing.json()["missing"]) + def test_every_saved_symbol_is_listed_across_projects(self): leaves = self.leaves() leaves["clip/c1/symbol/sym~face"] = ["^ ", "~:name", "face", "~:frames", 12] @@ -435,12 +446,12 @@ class DocumentTests(TestCase): self.assertEqual(5, len(response.json()["written"])) loaded = self.client.get(f"/api/projects/{self.project.id}").json() - self.assertEqual(4, loaded["schema_version"]) + self.assertEqual(5, loaded["schema_version"]) self.assertEqual(1, len(loaded["clips"])) clip = loaded["clips"][0] self.assertEqual("c1", clip["cid"]) self.assertEqual([self.block], clip["blocks"]) - self.assertEqual(self.analysis, clip["analysis"]) + self.assertNotIn("analysis", clip) # The whole point: byte-identical values, including the integer frame keys # transit writes as "~i0". A JSON round trip that stringified them would # come back "0" and the part would hold its first pose forever. @@ -542,7 +553,7 @@ class DocumentTests(TestCase): def patch(self, base, leaves, removed=()): return self.put(f"/api/projects/{self.project.id}", { "base": base, - "clips": [{"cid": "c1", "analysis": self.analysis, "leaves": leaves, + "clips": [{"cid": "c1", "analyses": [self.analysis], "leaves": leaves, "removed": list(removed), "blocks": [self.block]}], }) diff --git a/clips/views.py b/clips/views.py index 4092a0e..e88dff8 100644 --- a/clips/views.py +++ b/clips/views.py @@ -718,8 +718,6 @@ def _project_json(project: Project, user): { "cid": clip.cid, "name": clip.name, - "footage": str(clip.footage_id) if clip.footage_id else None, - "analysis": clip.analysis_id, "blocks": sorted(clip.blocks.values_list("key", flat=True)), "leaves": {leaf.path: leaf.value for leaf in leaves if leaf.path.startswith(prefix)}, } @@ -909,7 +907,19 @@ def _save(project: Project, data, user): ) keys = spec.get("blocks") or [] - have = set(Block.objects.filter(key__in=keys).values_list("key", flat=True)) + analyses = spec.get("analyses") or [] + if (not isinstance(analyses, list) + or not all(isinstance(key, str) for key in analyses) + or len(analyses) != len(set(analyses))): + raise Bad("a clip's analyses must be a list of distinct analysis ids") + registered = set(Analysis.objects.filter(key__in=analyses) + .values_list("key", flat=True)) + if unknown := [key for key in analyses if key not in registered]: + raise Bad("this clip names analyses the server does not know; register them first", + status=409, missing=unknown) + + block_rows = list(Block.objects.filter(key__in=keys)) + have = {block.key for block in block_rows} if missing := [k for k in keys if k not in have]: # Referential integrity across the tiers, enforced where it can be: # a document that names blocks the server does not hold would load @@ -919,6 +929,9 @@ def _save(project: Project, data, user): "before saving the document that points at them", status=409, missing=missing, ) + if undeclared := sorted({block.analysis_id for block in block_rows} - registered): + raise Bad("every block in a clip must name one of that clip's analyses", + status=409, missing=undeclared) existing = {leaf.path: leaf for leaf in project.leaves.filter(path__startswith=prefix)} if base is not None: @@ -930,16 +943,12 @@ def _save(project: Project, data, user): if conflicts: continue - analysis = Analysis.objects.filter(key=spec.get("analysis")).first() - footage = None - if spec.get("footage"): - footage = Footage.objects.filter(id=spec["footage"]).first() clip, _ = Clip.objects.update_or_create( project=project, cid=cid, - defaults={"name": spec.get("name") or "", "analysis": analysis, "footage": footage}, + defaults={"name": spec.get("name") or ""}, ) - blocks = Block.objects.filter(key__in=keys) + blocks = block_rows if base is None: clip.blocks.set(blocks) gone = [path for path in existing if path not in leaves] diff --git a/frontend/src/arthur/domain/bring.cljs b/frontend/src/arthur/domain/bring.cljs index 166ca51..97a18b5 100644 --- a/frontend/src/arthur/domain/bring.cljs +++ b/frontend/src/arthur/domain/bring.cljs @@ -11,6 +11,7 @@ so the events that fetch them are only fetching." (:refer-clojure :exclude [take]) (:require [arthur.domain.clip :as clip] + [arthur.domain.feature :as feature] [arthur.domain.node :as node] [clojure.string :as string])) @@ -59,8 +60,8 @@ (keyword (if (seq slug) slug "symbol")))) (defn take - "Put `frozen`, a take, into `clip` as ONE symbol called `label`. Returns - `{:clip :sid :tracked?}`. + "Put `frozen`, a take, into `clip` as ONE symbol called `label`. Returns the + changed clip, the imported symbol id and the old-to-scoped subject ids. `frozen` is what `flow/freeze/clip` makes: a `:main` that places one symbol per tracked face. `:main` becomes the named symbol — it is what holds the faces in @@ -75,18 +76,22 @@ symbols. `nest/audio-tracks` collapses simultaneous copies carrying the same `:media-link`; placing those faces at different times still schedules each one. - The tracking identities, and the analysis they were measured by, come along - only when `clip` has no analysis of its own and no face had to be renamed. A - document holds one analysis, and regeneration finds a face's symbol by its - subject id, so either condition failing means the take comes in as drawings - that play but cannot be re-tuned — `:tracked? false` says so." + The unique imported symbol id scopes every subject id. Thus two takes may both + arrive with detector subject `:face-1` without colliding in the document." [clip frozen label footage-id range] - (let [{c :clip ids :ids} (symbols clip frozen [:main] {:main (symbol-id label)}) + (let [scope (fn [root subject] (keyword (str (name root) "." (name subject)))) + root (clip/free-id + (fn [candidate] + (or (contains? (:symbols clip) candidate) + (some #(contains? (:symbols clip) (scope candidate %)) + (keys (:subjects frozen))))) + (symbol-id label)) + scoped (partial scope root) + wanted (into {:main root} (map (fn [subject] [subject (scoped subject)])) + (keys (:subjects frozen))) + {c :clip ids :ids} (symbols clip frozen [:main] wanted) sid (ids :main) faces (mapv ids (sort-by str (keys (:subjects frozen)))) - ;; Older/plain imported clips have no subject table. They are still a - ;; take, so their wrapper remains the only honest owner of the sound. - sound-hosts (if (seq faces) faces [sid]) sound {:id :sound :name "sound" :kind :audio :parent nil :z "z-sound" :source {:footage footage-id} ;; All automatic copies name one recording. The audio walk @@ -96,18 +101,42 @@ ;; Source frame `start` plays on the symbol's 0. :span range :time {:mode :map :at (- (first range)) :rate 1}} - tracked? (and (nil? (:analysis clip)) - (every? #(= % (ids %)) (keys (:subjects frozen))))] + feature-ids (into {} + (map (fn [[id f]] + [id (feature/owned (ids (:subject f)) (keyword (name id)))])) + (:features frozen)) + subjects (into {} + (map (fn [[id subject]] + (let [new-id (ids id)] + [new-id (assoc subject :id new-id + :footage footage-id)]))) + (:subjects frozen)) + features (into {} + (map (fn [[id f]] + (let [new-id (feature-ids id)] + [new-id (-> f + (assoc :id new-id + :subject (ids (:subject f)) + :symbol (ids (:symbol f))))]))) + (:features frozen)) + groups (into {} + (map (fn [[id g]] + (let [new-subject (ids (:subject g)) + new-id (feature/owned new-subject (keyword (name id)))] + [new-id (-> g + (assoc :id new-id :subject new-subject) + (update :members #(mapv feature-ids %)))]))) + (:groups frozen))] {:sid sid - :tracked? tracked? - :clip (cond-> (-> (reduce (fn [document face] - (assoc-in document [:symbols face :nodes :sound] sound)) - c sound-hosts) - (assoc-in [:symbols sid :name] (str label))) - tracked? (-> (assoc :analysis (:analysis frozen)) - (update :subjects merge (:subjects frozen)) - (update :features merge (:features frozen)) - (update :groups merge (:groups frozen))))})) + :subject-ids (select-keys ids (keys (:subjects frozen))) + :clip (-> (reduce (fn [document face] + (assoc-in document [:symbols face :nodes :sound] sound)) + c faces) + (assoc-in [:symbols sid :name] (str label)) + (update :analyses merge (:analyses frozen)) + (update :subjects merge subjects) + (update :features merge features) + (update :groups merge groups))})) (defn placed diff --git a/frontend/src/arthur/domain/clip.cljs b/frontend/src/arthur/domain/clip.cljs index 6dd8cfe..1bc6b18 100644 --- a/frontend/src/arthur/domain/clip.cljs +++ b/frontend/src/arthur/domain/clip.cljs @@ -4,7 +4,7 @@ {:name \"take\" :fps 30 :width 320 :height 200 - :analysis {...} + :analyses {analysis-id {...}} :subjects {...} :features {...} :groups {...} :symbols {:main {:id :main :frames 229 :nodes {...}}}} @@ -51,7 +51,7 @@ that loses something on every round trip, which is the one bug a persistence layer must not be able to have. Add the field here and to `leaf/leaves` and `leaf/clip` in the same commit." - #{:name :fps :analysis :subjects :features :groups :width :height :symbols + #{:name :fps :analyses :subjects :features :groups :width :height :symbols :palettes :default-palette}) (defn symbol @@ -574,6 +574,18 @@ (str "clip has a field with no leaf to save it in: " (pr-str k))) (when-not (map? (:symbols clip)) [":symbols must be a map of id -> symbol"]) + (when (and (contains? clip :analyses) (not (map? (:analyses clip)))) + [":analyses must be a map of analysis id -> analysis"]) + (for [[id analysis] (:analyses clip) + :when (not= id (:id analysis))] + (str "analysis under key " (pr-str id) " has :id " (pr-str (:id analysis)))) + (for [[id subject] (:subjects clip) + :when (not (contains? (:analyses clip) (:analysis subject)))] + (str "subject " (pr-str id) " names missing analysis " + (pr-str (:analysis subject)))) + (for [[id subject] (:subjects clip) + :when (not (keyword? (:source-subject subject)))] + (str "subject " (pr-str id) " has no source subject")) (when (and (contains? clip :palettes) (not (map? (:palettes clip)))) [":palettes must be a map of id -> palette"]) (when (and (contains? clip :default-palette) (map? (:palettes clip)) diff --git a/frontend/src/arthur/domain/leaf.cljs b/frontend/src/arthur/domain/leaf.cljs index 714d12c..e47accf 100644 --- a/frontend/src/arthur/domain/leaf.cljs +++ b/frontend/src/arthur/domain/leaf.cljs @@ -137,7 +137,7 @@ (some-leaf (at "timing") (select-keys clip [:fps])) (some-leaf (at "stage") (select-keys clip [:width :height])) (some-leaf (at "palette-default") (select-keys clip [:default-palette])) - (some-leaf (at "source") (:analysis clip)) + (some-leaf (at "analyses") (:analyses clip)) (concat (for [[id v] (:subjects clip)] {(at "subject" (segment id)) v}) (for [[id v] (:features clip)] {(at "feature" (segment id)) v}) @@ -194,7 +194,7 @@ "name" (merge acc v) "timing" (merge acc v) "stage" (merge acc v) - "source" (assoc acc :analysis v) + "analyses" (assoc acc :analyses v) "palette-default" (merge acc v) "palette" (assoc-in acc [:palettes (unsegment a)] v) "subject" (assoc-in acc [:subjects (unsegment a)] v) @@ -237,7 +237,7 @@ false) (case (count p) ;; The clip's own facts carry no id. - 3 (#{"name" "timing" "stage" "source" "palette-default"} (nth p 2)) + 3 (#{"name" "timing" "stage" "analyses" "palette-default"} (nth p 2)) 4 (#{"subject" "feature" "group" "palette"} (nth p 2)) false))))] (vec diff --git a/frontend/src/arthur/domain/project.cljs b/frontend/src/arthur/domain/project.cljs index 5c5afc1..38920a7 100644 --- a/frontend/src/arthur/domain/project.cljs +++ b/frontend/src/arthur/domain/project.cljs @@ -29,9 +29,7 @@ (:require [arthur.domain.leaf :as leaf] [arthur.domain.wire :as wire])) -(def schema-version - "4 separates a symbol's static authoring palette from its palette track." - 4) +(def schema-version 5) (defn block-keys "Every tier-2 key a leaf map names, in a stable order." diff --git a/frontend/src/arthur/events/footage.cljs b/frontend/src/arthur/events/footage.cljs index bc2e80b..7c75e21 100644 --- a/frontend/src/arthur/events/footage.cljs +++ b/frontend/src/arthur/events/footage.cljs @@ -151,7 +151,8 @@ :detector detector}) _ (mark! "build-clip: freeze") built (:clip frozen) - source-blocks (source/pack-subjects (:id (:analysis built)) subjects) + analysis-id (-> built :analyses keys first) + source-blocks (source/pack-subjects analysis-id subjects) _ (mark! "build-clip: pack source blocks")] (assoc (select-keys built [:fps :width :height]) @@ -504,7 +505,7 @@ (let [entry (store/entry (:clip/current db)) uuid (random-uuid) fps (get-in db [:clip :fps]) - {:keys [clip sid tracked?]} + {:keys [clip sid subject-ids]} (bring/take (:clip entry) (:clip built) name footage-id range) st (merge (:store entry) (:store built)) imported-frames (clip/output-frames clip sid) @@ -525,9 +526,22 @@ (update :footage merge {:loading? false :status why}))} (let [db (edit/edit-entry db - #(cond-> (assoc % :clip (:clip result) :store st) - tracked? (merge (select-keys built [:footage-id :source-blocks - :source-inputs]))))] + #(let [analysis-id (-> built :clip :analyses keys first) + remap-subjects + (fn [inputs] + (update inputs :subjects + (fn [subjects] + (into {} (map (fn [[old new]] [new (get subjects old)])) + subject-ids))))] + (-> (assoc % :clip (:clip result) :store st) + (update-in [:sources analysis-id] + (fn [source] + {:footage-id (:footage-id built) + :source-blocks (:source-blocks built) + :source-inputs + (update (remap-subjects (:source-inputs built)) + :subjects merge + (get-in source [:source-inputs :subjects]))})))))] {:db (-> db (update :ui dissoc :convert) (ui/selected @@ -536,7 +550,5 @@ {:loading? false :status (str "made " name " · " imported-frames " frames at " fps " fps" (when (not= fps source-fps) - (str " · sampled from " source-fps " fps")) - (when-not tracked? - " · as drawings: this project already tracks other footage"))})) + (str " · sampled from " source-fps " fps")))})) :dispatch [::pb/refresh-clock]}))))) diff --git a/frontend/src/arthur/events/project.cljs b/frontend/src/arthur/events/project.cljs index 295ccfe..ffd743b 100644 --- a/frontend/src/arthur/events/project.cljs +++ b/frontend/src/arthur/events/project.cljs @@ -102,15 +102,10 @@ (.then (fn [^js created] (.-id created)))))) (defn- opened-entry! [^js clip-json] - (let [footage-id (.-footage clip-json)] - (-> (js/Promise.all - #js [(js/Promise.all - (into-array (map #(http/GET (str "/api/blocks/" %)) - (array-seq (.-blocks clip-json))))) - (if footage-id - (http/GET (str "/api/footage/" footage-id)) - (js/Promise.resolve nil))]) - (.then (fn [[blocks ^js footage]] + (-> (js/Promise.all + (into-array (map #(http/GET (str "/api/blocks/" %)) + (array-seq (.-blocks clip-json))))) + (.then (fn [blocks] (let [cid (.-cid clip-json) loaded (project/load cid #js {:leaves (.-leaves clip-json) @@ -129,11 +124,9 @@ ;; write lands on what does not. :synced (project/tier1 (.-leaves clip-json)) :clip built :store (:store loaded) - :footage-id footage-id - :audio (if footage (.-audio footage) - "/static/arthur/audio.wav")})] + :audio "/static/arthur/audio.wav"})] (-> (mix/mix! built (clip/opens-on built) (:audio entry) (:store entry)) - (.then (fn [audio] (assoc entry :audio audio))))))))))) + (.then (fn [audio] (assoc entry :audio audio)))))))))) (defn- saved-clip! "Promise of clip `cid` of saved project `pid`, as `{:clip :store}`: its @@ -240,46 +233,60 @@ :removed (into-array (remove #(contains? local %) (keys synced)))}) #js {:leaves (.-leaves doc)})) -(defonce ^:private on-server - ;; Analyses and block keys this page has already put on the server. Content - ;; addressed, so once there they are there: a save of a moved vertex asks for - ;; none of it again, and is one request. +(defonce ^:private uploaded-blocks + ;; Block keys this page has already put on the server. Content addressed, so + ;; once there they are there: a save of a moved vertex asks for none again. (atom #{})) +(defonce ^:private linked-analyses (atom #{})) + (defn- upload-new! [^js doc] (let [keys (array-seq (block-keys doc))] - (if (every? @on-server keys) + (if (every? @uploaded-blocks keys) (js/Promise.resolve 0) - (.then (upload-missing! doc) (fn [n] (swap! on-server into keys) n))))) + (.then (upload-missing! doc) (fn [n] (swap! uploaded-blocks into keys) n))))) + +(defn- analyses-of [entry] + (vals (get-in entry [:clip :analyses]))) + +(defn- upload-sources! [sources] + (reduce + (fn [chain [analysis-id {:keys [source-blocks]}]] + (.then chain + (fn [_] + (when (and (seq source-blocks) + (not (@linked-analyses analysis-id))) + (-> (upload-missing! #js {:blocks (source/upload-blocks source-blocks)}) + (.then #(http/PUT + (str "/api/analyses/" analysis-id) + #js {:source_blocks + (into-array (source/block-keys source-blocks))})) + (.then (fn [answer] + (swap! linked-analyses conj analysis-id) + answer))))))) + (js/Promise.resolve nil) + sources)) (rf/reg-fx ::save! (fn [{:keys [id cid label clip base]}] - (let [analysis (:analysis (:clip clip)) - doc (project/save cid clip) - local (leaf/leaves cid (:clip clip)) - base (when (and id (:synced clip)) base) - source-blocks (:source-blocks clip)] + (let [entry clip + analyses (analyses-of entry) + doc (project/save cid entry) + local (leaf/leaves cid (:clip entry)) + base (when (and id (:synced entry)) base) + sources (:sources entry)] (-> (ensure-project! id label) (.then (fn [pid] - (-> (if (and analysis (not (@on-server (:id analysis)))) - (http/POST "/api/analyses" (analysis-payload analysis)) - (js/Promise.resolve nil)) + (-> (reduce (fn [chain one] + (.then chain + #(http/POST "/api/analyses" + (analysis-payload one)))) + (js/Promise.resolve nil) + analyses) (.then (fn [_] - (when (and (seq source-blocks) (not (@on-server (:id analysis)))) - (-> (upload-missing! - #js {:blocks (source/upload-blocks source-blocks)}) - (.then (fn [_] - (http/PUT - (str "/api/analyses/" (:id analysis)) - ;; One set per tracked subject, in - ;; the order `source/unpack` does - ;; not depend on. - #js {:source_blocks - (into-array - (source/block-keys source-blocks))}))))))) + (upload-sources! sources))) (.then (fn [_] - (when analysis (swap! on-server conj (:id analysis))) (upload-new! doc))) (.then (fn [uploaded] (-> (http/PUT (str "/api/projects/" pid) @@ -288,11 +295,11 @@ :clips #js [(js/Object.assign #js {:cid cid :name label - :analysis (:id analysis) - :footage (:footage-id clip) + :analyses (into-array + (keys (get-in entry [:clip :analyses]))) :blocks (block-keys doc)} (clip-payload doc base local - (:synced clip)))]}) + (:synced entry)))]}) (.then (fn [^js saved] (rf/dispatch [::saved pid cid label (.-seq saved) @@ -439,39 +446,38 @@ (assoc measured :interior-key block-key)))))))) (throw error))))))) -(defn- source-for! [entry] - (if-let [inputs (:source-inputs entry)] - (js/Promise.resolve inputs) - (let [analysis (get-in entry [:clip :analysis])] - (if (= (:id analysis) (:id @retained-source)) +(defn- source-for! [entry edit] + (let [subject (:subject (regenerate/plan (:clip entry) edit)) + subject-record (get-in entry [:clip :subjects subject]) + analysis-id (:analysis subject-record) + analysis (get-in entry [:clip :analyses analysis-id]) + source-subject (:source-subject subject-record) + local (get-in entry [:sources analysis-id :source-inputs :subjects subject])] + (if local + (js/Promise.resolve {:subjects {subject local}}) + (if (= [analysis-id subject] (:key @retained-source)) (:promise @retained-source) (let [promise (if (= "synth" (:detector analysis)) - ;; The synthetic take tracks one face and regenerating it reads - ;; that face's landmarks, so it arrives in the same shape real - ;; footage does rather than in a flat one only this branch uses. (js/Promise.resolve {:subjects - {:face-1 {:dense (synth/synth-dense (:frames analysis) - {:seed (:seed analysis)})}}}) - (-> (ingest/manifest! (:footage-id entry)) - (.then (fn [manifest] - (-> (footage/saved-source! (:id analysis) - [(:width manifest) - (:height manifest)]) - (.then (fn [inputs] - (when-not inputs - (throw (ex-info "saved analysis has no source blocks" {}))) - (update inputs :subjects - (fn [subjects] - (into {} - (map (fn [[id one]] - [id (assoc one :presence - (footage/presence-for - manifest id))])) - subjects))))))))))] + {subject {:dense (synth/synth-dense (:frames analysis) + {:seed (:seed analysis)})}}}) + (.then (ingest/manifest! (:footage subject-record)) + (fn [manifest] + (.then (footage/saved-source! + analysis-id [(:width manifest) (:height manifest)]) + (fn [inputs] + (when-not inputs + (throw (ex-info "saved analysis has no source blocks" {}))) + {:subjects + {subject + (assoc (get-in inputs [:subjects source-subject]) + :presence + (footage/presence-for manifest + source-subject))}})))))] (do - (reset! retained-source {:id (:id analysis) :promise promise}) + (reset! retained-source {:key [analysis-id subject] :promise promise}) promise)))))) (defn- inputs-for-edit! @@ -484,18 +490,19 @@ one (get-in inputs [:subjects subject])] (if (and teeth (:crops one)) (let [settings (merge take/knobs (feature/effective-params clip teeth)) - analysis (get-in clip [:analysis :id]) + analysis (get-in clip [:subjects subject :analysis]) + source-subject (get-in clip [:subjects subject :source-subject]) frames (count (:crops one)) - key (source/interior-key analysis subject settings frames) + key (source/interior-key analysis source-subject settings frames) done (fn [measured] (assoc-in inputs [:subjects subject] measured))] (if (and (:interior one) (or (= (:interior-key one) key) (and (nil? (:interior-key one)) - (= key (source/interior-key analysis subject take/knobs frames))))) + (= key (source/interior-key analysis source-subject take/knobs frames))))) (js/Promise.resolve inputs) (.then (if (= key (:key @retained-interior)) (:promise @retained-interior) - (let [promise (retained-interior! analysis subject settings one)] + (let [promise (retained-interior! analysis source-subject settings one)] (reset! retained-interior {:key key :promise promise}) promise)) done))) @@ -504,7 +511,7 @@ (rf/reg-fx ::preview-settings! (fn [{:keys [id entry edit request]}] - (-> (source-for! entry) + (-> (source-for! entry edit) (.then (fn [inputs] (inputs-for-edit! entry edit inputs))) (.then (fn [inputs] (regenerate/change (assoc entry :source-inputs inputs) edit))) @@ -519,7 +526,7 @@ (fn [{:keys [db]} [_ edit]] (let [id (:clip/current db) entry (store/entry id)] - (if (or (get-in db [:project :busy?]) (nil? (:analysis (:clip entry)))) + (if (or (get-in db [:project :busy?]) (empty? (:analyses (:clip entry)))) {} (let [plan (regenerate/plan (:clip entry) edit) report (select-keys plan [:features :roles]) diff --git a/frontend/src/arthur/flow/freeze.cljs b/frontend/src/arthur/flow/freeze.cljs index c83efa5..91d057a 100644 --- a/frontend/src/arthur/flow/freeze.cljs +++ b/frontend/src/arthur/flow/freeze.cljs @@ -762,11 +762,15 @@ {:frames (vec lengths)}))) nf (first lengths) merged (fn [k] (into {} (mapcat (comp k second)) parts)) - built {:name name :fps fps :analysis (:analysis params) + analysis (:analysis params) + built {:name name :fps fps :analyses {(:id analysis) analysis} :width (first stage) :height (second stage) :palettes {pal/default-id pal/default-palette} :default-palette pal/default-id - :subjects (into {} (map (fn [[id _]] [id {:id id :params {}}])) ordered) + :subjects (into {} (map (fn [[id _]] + [id {:id id :params {} + :analysis (:id analysis) + :source-subject id}])) ordered) :features (merged :features) :groups (merged :groups) :symbols (into {:main diff --git a/frontend/src/arthur/flow/regenerate.cljs b/frontend/src/arthur/flow/regenerate.cljs index e228d3a..7148df0 100644 --- a/frontend/src/arthur/flow/regenerate.cljs +++ b/frontend/src/arthur/flow/regenerate.cljs @@ -142,6 +142,11 @@ teeth are dirty, retained pixel measurements. No IO or app-db here." [{:keys [clip source-inputs] :as entry} edit] (let [{:keys [changed subject features]} (plan clip edit) + analysis-id (get-in changed [:subjects subject :analysis]) + analysis (get-in changed [:analyses analysis-id]) + _ (when-not analysis + (throw (ex-info "subject names a missing analysis" + {:subject subject :analysis analysis-id}))) ;; THE EDITED SUBJECT'S OWN LANDMARKS. Retained source is per subject — ;; one video, one dense track per tracked face — so re-measuring the ;; second face through the first one's anchor is the mistake this lookup @@ -152,8 +157,8 @@ {:subject subject}))) base-params (merge take/knobs {:fps (or (get-in changed [:symbols subject :fps]) (:fps changed)) - :aspect (get-in changed [:analysis :aspect]) - :analysis (:analysis changed)}) + :aspect (:aspect analysis) + :analysis analysis}) ;; One conditioned anchor for the whole edit, and it is the SUBJECT's. ;; `:anchor-avg` is a subject setting, so the shared upstream measurement ;; is not read off whichever dirty feature happened to sort first — and diff --git a/frontend/src/arthur/subs/render.cljs b/frontend/src/arthur/subs/render.cljs index d225b09..8b09c31 100644 --- a/frontend/src/arthur/subs/render.cljs +++ b/frontend/src/arthur/subs/render.cljs @@ -212,7 +212,6 @@ ;; row is not on stage, so neither is its footage. (let [solo (filter #(placed? document open %) solo)] {:document document :store store - :footage-id (:footage-id (footage/entry id)) :opacity (or opacity trace/opacity-default) :traces (cond->> (when document (trace/shown document open faces)) (seq solo) (filterv (fn [{:keys [path]}] diff --git a/frontend/src/arthur/subs/ui.cljs b/frontend/src/arthur/subs/ui.cljs index 3ffc7b7..763b876 100644 --- a/frontend/src/arthur/subs/ui.cljs +++ b/frontend/src/arthur/subs/ui.cljs @@ -156,7 +156,8 @@ ;; than kept beside it, so a sound placed from footage makes that footage the ;; project's without anything else being told. (let [entry (store/entry id) - used (into (set (keep identity [(:footage-id entry)])) uploaded) + used (into (set uploaded) + (keep :footage (vals (get-in entry [:clip :subjects])))) used (into used (for [[_ sym] (get-in entry [:clip :symbols]) [_ n] (:nodes sym) :let [f (get-in n [:source :footage])] diff --git a/frontend/src/arthur/ui/underlay.cljs b/frontend/src/arthur/ui/underlay.cljs index 73c6525..7daf387 100644 --- a/frontend/src/arthur/ui/underlay.cljs +++ b/frontend/src/arthur/ui/underlay.cljs @@ -129,7 +129,7 @@ says where each face's head went and which of its frames it was on, so this reads the frame rather than resolving it again. `on-ready` is called when a still or a manifest that was missing arrives, to paint again." - [{:keys [document store footage-id traces opacity width playing?]} resolver on-ready] + [{:keys [document store traces opacity width playing?]} resolver on-ready] (when-let [^js canvas (:canvas @state)] (let [ctx (.getContext canvas "2d") zoom (/ (.-width canvas) width)] @@ -138,17 +138,20 @@ ;; What each face is showing, forgotten for the faces switched off. Before ;; the early exits, so switching them all off forgets all of them. (swap! state update :last select-keys (map :path traces)) - (when-let [us (and (seq traces) footage-id (urls footage-id on-ready))] - (let [start (first (get-in document [:analysis :range] [0])) - ;; Read once: each face writes only its own entry below, and what it - ;; was showing is what it falls back to. A face that has been off - ;; stage for a few frames still has the still it went away with. - was (:last @state)] + (when (seq traces) + (let [;; Read once: each face writes only its own entry below, and what it + ;; was showing is what it falls back to. + was (:last @state)] ;; ONE OPACITY for all of them, set once: how strongly the reference ;; draws is a property of looking at the stage, not of a face. (set! (.-globalAlpha ctx) opacity) - (doseq [{:keys [path face]} traces] - (let [at (conj path :head) + (doseq [{:keys [path face]} traces + :let [subject (get-in document [:subjects face]) + analysis (get-in document [:analyses (:analysis subject)]) + us (urls (:footage subject) on-ready)] + :when us] + (let [start (first (get analysis :range [0])) + at (conj path :head) world (symbol/world-of resolver at) frame (symbol/frame-of resolver at)] (when (and world (number? frame)) diff --git a/frontend/test/arthur/domain/bring_test.cljs b/frontend/test/arthur/domain/bring_test.cljs index 50cb809..468d754 100644 --- a/frontend/test/arthur/domain/bring_test.cljs +++ b/frontend/test/arthur/domain/bring_test.cljs @@ -26,3 +26,34 @@ (vals (get-in clip [:symbols :take :nodes])))))) "and the copy's instance follows its renamed symbol") (is (empty? (clip/problems clip))))) + +(defn- tracked-take [analysis-id] + {:analyses {analysis-id {:id analysis-id :detector "test" :version "1"}} + :subjects {:face-1 {:id :face-1 :analysis analysis-id + :source-subject :face-1 :params {}}} + :features {:face-1/mouth {:id :face-1/mouth :subject :face-1 + :symbol :face-1 :area :mouth :nodes [:mouth] :params {}}} + :groups {} + :symbols {:main {:id :main :fps 30 :frames 10 + :nodes {:face-1 {:id :face-1 :kind :instance :z "a1" + :source {:symbol :face-1}}}} + :face-1 {:id :face-1 :fps 30 :frames 10 + :nodes {:head {:id :head :kind :group :z "a1" + :measured {[:xform :pos] + {:animated? false :value [0 0]}}} + :mouth {:id :mouth :kind :poly :z "a2"}}}}}) + +(deftest every-import-is-tracked-under-its-unique-symbol-scope + (let [first (bring/take (clip/blank) (tracked-take "analysis-a") + "close up" "footage-a" [0 10]) + second (bring/take (:clip first) (tracked-take "analysis-b") + "close up" "footage-b" [0 10]) + c (:clip second)] + (is (= #{:close-up.face-1 :close-up-2.face-1} (set (keys (:subjects c))))) + (is (= #{"analysis-a" "analysis-b"} (set (keys (:analyses c))))) + (is (= "analysis-a" (get-in c [:subjects :close-up.face-1 :analysis]))) + (is (= "analysis-b" (get-in c [:subjects :close-up-2.face-1 :analysis]))) + (is (= :face-1 (get-in c [:subjects :close-up-2.face-1 :source-subject]))) + (is (= #{:close-up.face-1/mouth :close-up-2.face-1/mouth} + (set (keys (:features c))))) + (is (empty? (clip/problems c))))) diff --git a/frontend/test/arthur/domain/cadence_test.cljs b/frontend/test/arthur/domain/cadence_test.cljs index c32002b..9e8c30a 100644 --- a/frontend/test/arthur/domain/cadence_test.cljs +++ b/frontend/test/arthur/domain/cadence_test.cljs @@ -90,8 +90,17 @@ (is (or (zero? f) (< (clip/shown-frame doc :main (dec f)) n)))))))) (deftest imported-footage-uses-selection-for-picture-and-real-speed-for-audio - (let [{doc :clip sid :sid} (bring/take (clip/set-fps (clip/blank) 12) - footage "take" "video" [15 75]) + (let [tracked (-> footage + (assoc :analyses {"a" {:id "a"}} + :subjects {:face-1 {:id :face-1 :analysis "a" + :source-subject :face-1}} + :features {} :groups {} + :symbols {:main {:id :main :fps 30 :frames 60 + :nodes {:face-1 {:id :face-1 :kind :instance + :z "a" :source {:symbol :face-1}}}} + :face-1 (get-in footage [:symbols :main])})) + {doc :clip sid :sid} (bring/take (clip/set-fps (clip/blank) 12) + tracked "take" "video" [15 75]) ;; A host authored at 12, with a 30fps source starting half a second in. doc (-> doc (assoc-in [:symbols :main :fps] 12) (clip/place-symbol store :main sid 6 :insert nil)) @@ -99,21 +108,16 @@ draw (clip/resolver doc :main store pal/index-of nil) [sound] (nest/audio-tracks doc :main)] (is (= 60 (clip/frames doc sid))) - (is (= 1 (get-in doc [:symbols sid :nodes :sound :time :rate]))) + (is (= 1 (get-in doc [:symbols :take.face-1 :nodes :sound :time :rate]))) (is (= [0 24] (:span n))) (is (empty? (draw 5))) (is (= 8 (:size (first (draw 9))))) - (is (= 7 (:frame (nest/inside doc store :main [:insert :mark] 9)))) + (is (= 7 (:frame (nest/inside doc store :main [:insert :face-1 :mark] 9)))) (is (= [-4 -4 4 4] ((pick/bounds-of doc store :main n) 3))) (is (= [6 30] (node/placed-span sound))) (is (= 15 (node/local-frame sound 6))) (is (= 1 (* (get-in sound [:time :rate]) (/ (:fps doc) (:fps sound))))) (is (= [6 30] (:span (first (timeline/rows doc :main #{[:insert]}))))) - (testing "an off-grid closed mouth can win without modifying the dense source" - (let [pinned (pose/put-cut doc :main :insert :mouth 7 6) - r (clip/resolver pinned :main store pal/index-of nil)] - (is (= 7 (:size (first (r 9))))) - (is (= (get-in doc [:symbols sid]) (get-in pinned [:symbols sid]))))) (testing "a deliberate half speed still retimes audio" (let [slow (assoc-in doc [:symbols :main :nodes :insert :playback :speed] 0.5) [track] (nest/audio-tracks slow :main)] @@ -121,7 +125,9 @@ (deftest generated-faces-own-their-footage-sound (let [face (fn [id] {:id id :fps 30 :frames 20 :nodes {}}) - frozen {:fps 30 :subjects {:face-1 {} :face-2 {}} + frozen {:fps 30 :analyses {"a" {:id "a"}} + :subjects {:face-1 {:id :face-1 :analysis "a" :source-subject :face-1} + :face-2 {:id :face-2 :analysis "a" :source-subject :face-2}} :symbols {:main {:id :main :fps 30 :frames 20 :nodes {:face-1 {:id :face-1 :kind :instance :z "a1" :source {:symbol :face-1}} @@ -130,11 +136,11 @@ :face-1 (face :face-1) :face-2 (face :face-2)}} {doc :clip take :sid} (bring/take (clip/blank) frozen "take" "video" [3 13]) - alone (clip/place-symbol doc nil :main :face-1 4 :placed-face nil)] + alone (clip/place-symbol doc nil :main :take.face-1 4 :placed-face nil)] (is (nil? (get-in doc [:symbols take :nodes :sound])) "the wrapper does not own a sound the face would lose") (is (= {:footage "video"} - (get-in doc [:symbols :face-1 :nodes :sound :source]))) + (get-in doc [:symbols :take.face-1 :nodes :sound :source]))) (is (= 1 (count (nest/audio-tracks doc take))) "the same take recording is not mixed once per detected face") (is (= 1 (count (nest/audio-tracks alone :main))) diff --git a/frontend/test/arthur/domain/leaf_test.cljs b/frontend/test/arthur/domain/leaf_test.cljs index cae7576..0978d65 100644 --- a/frontend/test/arthur/domain/leaf_test.cljs +++ b/frontend/test/arthur/domain/leaf_test.cljs @@ -45,7 +45,7 @@ (let [ls (leaf/leaves :c7 @take/clip)] (is (contains? ls "clip/c7/timing")) (is (contains? ls "clip/c7/stage")) - (is (contains? ls "clip/c7/source")) + (is (contains? ls "clip/c7/analyses")) ;; A TIMELINE ID IS A SEGMENT, which is what lets a symbol's nodes be ;; addressed by the same path shape as the clip's own. `main` is the root. (is (contains? ls "clip/c7/symbol/main")) diff --git a/frontend/test/arthur/flow/regenerate_test.cljs b/frontend/test/arthur/flow/regenerate_test.cljs index 232e7a6..12b9ab0 100644 --- a/frontend/test/arthur/flow/regenerate_test.cljs +++ b/frontend/test/arthur/flow/regenerate_test.cljs @@ -2,6 +2,7 @@ (:require [cljs.test :refer [deftest is testing]] [clojure.walk :as walk] [arthur.demo.stage :as stage] + [arthur.domain.bring :as bring] [arthur.domain.channel :as ch] [arthur.domain.clip :as clip] [arthur.domain.params :as params] @@ -29,13 +30,41 @@ (merge take/knobs {:name "regen" :fps 30 :aspect 1 :stage [320 200] :expose 1 :head :free - :analysis (get-in @initial [:clip :analysis])} + :analysis (-> @initial :clip :analyses vals first)} change) {:face-1 @inputs})) (defn- channel [entry node path] (get-in entry [:clip :symbols :face-1 :nodes node :channels path])) +(deftest a-second-import-regenerates-through-its-own-analysis + (let [built (fn [seed] + (let [input {:dense (synth/synth-dense frames {:seed seed})} + analysis (address/analysis + {:detector "synth" :version "mulberry32" + :seed seed :frames frames :fps 30 :aspect 1})] + [(take/build (merge take/knobs + {:name "take" :fps 30 :aspect 1 :stage [320 200] + :expose 1 :head :free :analysis analysis}) + {:face-1 input}) + input])) + [a input-a] (built 3) + [b input-b] (built 7) + first (bring/take (clip/blank) (:clip a) "shot" "footage-a" [0 frames]) + second (bring/take (:clip first) (:clip b) "shot" "footage-b" [0 frames]) + entry {:clip (:clip second) :store (merge (:store a) (:store b)) + :source-inputs {:subjects {:shot.face-1 input-a + :shot-2.face-1 input-b}}} + fid :shot-2.face-1/mouth + changed (regenerate/change entry + {:scope :feature :id fid :knob :verts :value 10}) + analysis-id (get-in changed [:clip :subjects :shot-2.face-1 :analysis]) + generated (get-in changed [:clip :symbols :shot-2.face-1 :nodes :mouth + :channels [:geom :pts] :generated])] + (is (= (get-in b [:clip :subjects :face-1 :analysis]) analysis-id)) + (is (= analysis-id (:analysis generated))) + (is (= 10 (get-in generated [:params :verts]))))) + (deftest eye-rebuild-is-confined-to-the-edited-feature (let [before @initial after (regenerate/change before @@ -170,7 +199,7 @@ (defn- part-at [area overrides] (let [p (merge take/knobs - {:fps 30 :aspect 1 :analysis (get-in @initial [:clip :analysis])} + {:fps 30 :aspect 1 :analysis (-> @initial :clip :analyses vals first)} overrides) inputs (assoc @inputs :interior @interior-track)] (freeze/part :face-1 area p (take/measure-part area p inputs (take/anchor-base p inputs))))) @@ -212,8 +241,8 @@ ;; document that cannot be previewed fails in the suite rather than as a slider ;; that silently does nothing in the browser. (let [entry (staged)] - (is (some? (:analysis (:clip entry))) - "the composed stage keeps the analysis the edit needs") + (is (seq (:analyses (:clip entry))) + "the composed stage keeps the analyses its edits need") (is (some? (:source-inputs entry))) (testing "and the features still say which timeline they live in" (is (every? #(= :face-1 (:symbol %))