From ae03b61dcaf7b93f1bfadbc48da00cad2cbbf15f Mon Sep 17 00:00:00 2001 From: Olive Vaughn Date: Wed, 30 Sep 2026 03:31:19 -0400 Subject: [PATCH] sound stuff --- clips/extraction.py | 12 ++ clips/migrations/0010_sounds.py | 25 ++++ clips/models.py | 12 ++ clips/tests/test_api.py | 32 ++++- clips/urls.py | 2 + clips/views.py | 37 +++++- frontend/src/arthur/audio/mix.cljs | 53 +++++---- frontend/src/arthur/domain/clip.cljs | 21 ++++ frontend/src/arthur/domain/node.cljs | 21 +++- frontend/src/arthur/domain/symbol.cljs | 8 +- frontend/src/arthur/events/footage.cljs | 45 ++++--- frontend/src/arthur/events/ui.cljs | 22 +++- frontend/src/arthur/subs/render.cljs | 11 ++ frontend/src/arthur/subs/ui.cljs | 16 +++ frontend/src/arthur/ui/drag.cljs | 22 ++-- frontend/src/arthur/ui/params.cljs | 2 +- frontend/src/arthur/ui/pool.cljs | 41 ++++++- frontend/src/arthur/ui/shell.cljs | 35 ++++-- frontend/src/arthur/ui/timeline.cljs | 130 ++++++++++++++++----- frontend/test/arthur/domain/nest_test.cljs | 23 ++++ frontend/test/arthur/domain/node_test.cljs | 12 +- static/arthur/app.css | 10 ++ 22 files changed, 487 insertions(+), 105 deletions(-) create mode 100644 clips/migrations/0010_sounds.py diff --git a/clips/extraction.py b/clips/extraction.py index ff31bb6..21d07a6 100644 --- a/clips/extraction.py +++ b/clips/extraction.py @@ -161,6 +161,18 @@ def _extract_stills(job, proxy_path, frames_dir, frames, root): MAX_RATE = 120 # a capture rate; past this the container is describing something else +def probe_audio(path): + """The length of an uploaded sound in seconds, refusing a file with no audio.""" + data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams", + "-show_format", "-of", "json", str(path)])) + if not any(s.get("codec_type") == "audio" for s in data.get("streams", [])): + raise ValueError("the uploaded file has no audio stream") + duration = float(data.get("format", {}).get("duration") or 0) + if duration <= 0: + raise ValueError("the sound's length is unknown") + return duration + + def probe(path): """What the upload is, as far as choosing a proxy rate goes. diff --git a/clips/migrations/0010_sounds.py b/clips/migrations/0010_sounds.py new file mode 100644 index 0000000..1f26395 --- /dev/null +++ b/clips/migrations/0010_sounds.py @@ -0,0 +1,25 @@ +# Generated by Django 5.2.17 on 2026-09-30 07:14 + +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('clips', '0009_revision_blocks'), + ] + + operations = [ + migrations.CreateModel( + name='Sound', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('filename', models.CharField(max_length=255)), + ('duration', models.FloatField(help_text='seconds, as ffprobe reports it')), + ('created', models.DateTimeField(auto_now_add=True)), + ('blob', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='sound_for', to='clips.blob')), + ], + ), + ] diff --git a/clips/models.py b/clips/models.py index 47f1990..b7b4ea1 100644 --- a/clips/models.py +++ b/clips/models.py @@ -52,6 +52,18 @@ class Source(models.Model): created = models.DateTimeField(auto_now_add=True) +class Sound(models.Model): + """An uploaded sound file — mp3, wav, whatever the browser can decode — kept + as uploaded. Not footage: it has no frames and nothing measures it, so it + skips extraction and an audio node plays its bytes directly.""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + blob = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="sound_for") + filename = models.CharField(max_length=255) + duration = models.FloatField(help_text="seconds, as ffprobe reports it") + created = models.DateTimeField(auto_now_add=True) + + class Extraction(models.Model): """One requested decode of a source into immutable footage.""" diff --git a/clips/tests/test_api.py b/clips/tests/test_api.py index 5db496c..757acd4 100644 --- a/clips/tests/test_api.py +++ b/clips/tests/test_api.py @@ -34,7 +34,7 @@ from django.core.management import call_command from django.test import TestCase, override_settings from clips import blobs, extraction -from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision, Source +from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision, Sound, Source BLOB_DIR = tempfile.mkdtemp(prefix="arthur-test-blobs-") @@ -721,6 +721,36 @@ class UploadTests(TestCase): self.assertEqual(27, job.progress) job.save.assert_called_once_with(update_fields=["progress", "updated"]) + def test_an_uploaded_mp3_is_a_sound_and_not_footage(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "tone.mp3" + subprocess.run([ + "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "sine=frequency=440:duration=1.5", str(path), + ], check=True, capture_output=True) + payload = path.read_bytes() + + uploaded = self.client.post("/api/sounds", { + "file": SimpleUploadedFile("tone.mp3", payload, content_type="audio/mpeg")}) + self.assertEqual(201, uploaded.status_code, uploaded.content) + sound = uploaded.json() + self.assertEqual("tone.mp3", sound["label"]) + self.assertAlmostEqual(1.5, sound["duration"], delta=0.1) + self.assertEqual(payload, b"".join(self.client.get(sound["audio"]).streaming_content)) + self.assertEqual(sound, self.client.get(f"/api/sounds/{sound['id']}").json()) + self.assertEqual([sound], self.client.get("/api/sounds").json()["sounds"]) + self.assertEqual(0, Source.objects.count()) + + again = self.client.post("/api/sounds", { + "file": SimpleUploadedFile("again.mp3", payload, content_type="audio/mpeg")}) + self.assertEqual(200, again.status_code) + self.assertEqual(1, Sound.objects.count()) + + def test_a_file_without_audio_is_not_a_sound(self): + refused = self.client.post("/api/sounds", { + "file": SimpleUploadedFile("notes.txt", b"not audio", content_type="text/plain")}) + self.assertEqual(400, refused.status_code) + def test_uploaded_video_extracts_to_reopenable_footage(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "four-frames.mp4" diff --git a/clips/urls.py b/clips/urls.py index b21032b..19cee07 100644 --- a/clips/urls.py +++ b/clips/urls.py @@ -22,6 +22,8 @@ urlpatterns = [ path("logout", views.logout), path("detector", views.detector), path("sources", views.sources), + path("sounds", views.sounds), + path("sounds/", views.sound_detail), path("extractions", views.extractions), path("extractions/", views.extraction_detail), path("footage", views.footage_list), diff --git a/clips/views.py b/clips/views.py index 2b91348..44edd2d 100644 --- a/clips/views.py +++ b/clips/views.py @@ -43,7 +43,7 @@ from django.views.decorators.http import require_http_methods from . import blobs, extraction from .consumers import broadcast -from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Leaf, Project, Revision, Source +from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Leaf, Project, Revision, Sound, Source KEY_LENGTH = 71 # "sha256:" + 64 hex @@ -225,6 +225,41 @@ def sources(request): return JsonResponse({"error": str(exc)}, status=400) +def _sound_json(row): + return {"id": str(row.id), "label": row.filename, "duration": row.duration, + "audio": f"/blob/{row.blob_id}"} + + +@require_http_methods(["GET", "POST"]) +def sounds(request): + if request.method == "GET": + return JsonResponse({"sounds": [_sound_json(row) + for row in Sound.objects.order_by("-created")]}) + upload = request.FILES.get("file") + if upload is None: + return JsonResponse({"error": "upload a sound as the file field"}, status=400) + try: + digest, size = blobs.write_stream(upload.chunks()) + duration = extraction.probe_audio(blobs.path_for(digest)) + blob, _ = Blob.objects.get_or_create( + digest=digest, defaults={"size": size, + "media_type": upload.content_type or "audio/mpeg"}) + row, created = Sound.objects.get_or_create( + blob=blob, defaults={"filename": Path(upload.name).name[:255], + "duration": duration}) + return JsonResponse(_sound_json(row), status=201 if created else 200) + except (ValueError, OSError) as exc: + return JsonResponse({"error": str(exc)}, status=400) + + +@require_http_methods(["GET"]) +def sound_detail(request, sound_id): + try: + return JsonResponse(_sound_json(Sound.objects.get(id=sound_id))) + except Sound.DoesNotExist: + return JsonResponse({"error": "no such sound"}, status=404) + + def _extraction_json(row): return {"key": row.key, "source": str(row.source_id), "state": row.state, "progress": row.progress, "error": row.error, diff --git a/frontend/src/arthur/audio/mix.cljs b/frontend/src/arthur/audio/mix.cljs index 06b2bcd..adec0d7 100644 --- a/frontend/src/arthur/audio/mix.cljs +++ b/frontend/src/arthur/audio/mix.cljs @@ -55,26 +55,36 @@ (js/URL.createObjectURL (js/Blob. #js [(wav-bytes buffer)] #js {:type "audio/wav"}))) -(defn- source! [footage-id] - (-> (js/fetch (str "/api/footage/" footage-id)) +(defn- fetch-ok! [url what] + (-> (js/fetch url) (.then (fn [response] (when-not (.-ok response) - (throw (ex-info "audio track's footage is missing" - {:footage footage-id :status (.-status response)}))) - (.json response))) - (.then (fn [^js manifest] - (-> (js/fetch (.-audio manifest)) - (.then (fn [response] - (when-not (.-ok response) - (throw (ex-info "audio track's blob is missing" - {:footage footage-id :status (.-status response)}))) - (.arrayBuffer response))) - (.then (fn [bytes] - (let [decoder (js/OfflineAudioContext. 1 1 44100)] - (-> (.decodeAudioData decoder bytes) - (.then (fn [buffer] - [footage-id {:buffer buffer - :fps (.-fps manifest)}]))))))))))) + (throw (ex-info (str "audio track's " what " is missing") + {:url url :status (.-status response)}))) + response)))) + +(defn- decode-bytes! [bytes] + (.decodeAudioData (js/OfflineAudioContext. 1 1 44100) bytes)) + +(defn- source! + "Promise of `[source {:buffer :fps}]` for an audio node's `:source`. Footage + counts its frames at its own rate; a sound file has no frames of its own, so + its `:fps` is nil and it counts in the document's." + [{:keys [footage sound] :as source}] + (if sound + (-> (fetch-ok! (str "/api/sounds/" sound) "sound") + (.then #(.json %)) + (.then #(fetch-ok! (.-audio %) "blob")) + (.then #(.arrayBuffer %)) + (.then decode-bytes!) + (.then (fn [buffer] [source {:buffer buffer}]))) + (-> (fetch-ok! (str "/api/footage/" footage) "footage") + (.then #(.json %)) + (.then (fn [^js manifest] + (-> (fetch-ok! (.-audio manifest) "blob") + (.then #(.arrayBuffer %)) + (.then decode-bytes!) + (.then (fn [buffer] [source {:buffer buffer :fps (.-fps manifest)}])))))))) (defn- automate! [^js param channel start end fps factor default store] (let [channel (or channel (ch/framed default))] @@ -107,7 +117,7 @@ (let [[start end] (or (node/placed-span track) [0 frames]) start (max 0 start) end (min frames end) - {:keys [buffer fps]} (get sources (get-in track [:source :footage])) + {:keys [buffer fps] :or {fps (:fps document)}} (get sources (:source track)) sound (.createBufferSource output) gain (.createGain output) pan (.createStereoPanner output)] @@ -143,7 +153,7 @@ (if (empty? tracks) (js/Promise.resolve nil) (-> (js/Promise.all - (into-array (map source! (distinct (map #(get-in % [:source :footage]) tracks))))) + (into-array (map source! (distinct (map :source tracks))))) (.then (fn [pairs] (render! document sid (into {} (array-seq pairs)) store)))))))) (defn decode! @@ -156,8 +166,7 @@ (throw (ex-info "the clip's audio did not load" {:url url :status (.-status response)}))) (.arrayBuffer response))) - (.then (fn [bytes] - (.decodeAudioData (js/OfflineAudioContext. 1 1 44100) bytes))))) + (.then decode-bytes!))) (defn mix! "Promise of a mixed WAV URL for symbol `sid`, or the original URL when it has diff --git a/frontend/src/arthur/domain/clip.cljs b/frontend/src/arthur/domain/clip.cljs index 4bd6518..d4089c5 100644 --- a/frontend/src/arthur/domain/clip.cljs +++ b/frontend/src/arthur/domain/clip.cljs @@ -307,6 +307,27 @@ :value (if point (mapv - point middle) [0 0])} [:xform :anchor] {:animated? false :value middle}}}))))) +(defn place-sound + "An audio node playing `source` — `{:sound id}`, an uploaded file, or + `{:footage id}`, a video's own sound — inside symbol `host` from `frame` of it. + Timed as an instance is: `:at frame`, its span `length` of its OWN frames, and + `rate` of those to one of `host`'s, which is what keeps a 30fps video's sound + its own length in a 12fps project. Nothing else: a sound is not in the picture." + [clip host source label length rate frame uuid] + (let [end (frames clip host)] + (if (or (nil? end) (nil? frame) (neg? frame) (>= frame end)) + clip + (update-symbol + clip host assoc-in [:nodes uuid] + {:id uuid + :name label + :kind :audio + :parent nil + :z (str "z" (js/Date.now) "-sound") + :source source + :span [0 (max 1 length)] + :time {:mode :map :at frame :rate rate}})))) + (defn fresh-id "The first `:symbol-N` the clip does not already hold. Readable because an id shows up in saved leaf paths, and deterministic because this namespace is pure." diff --git a/frontend/src/arthur/domain/node.cljs b/frontend/src/arthur/domain/node.cljs index 0e40b74..5ee917a 100644 --- a/frontend/src/arthur/domain/node.cljs +++ b/frontend/src/arthur/domain/node.cljs @@ -46,7 +46,9 @@ (let [base (into #{[:vis]} xform-paths)] {:group base :instance base - :audio (into base [[:audio :gain] [:audio :pan] [:audio :rate]]) + ;; A sound is not in the picture: no transform, no visibility. After + ;; Effects' audio-only layer has no Transform group for the same reason. + :audio #{[:audio :gain] [:audio :pan] [:audio :rate]} :poly (into base [[:geom :pts] [:style :color]]) ;; A disc's radius is framed in practice — iris size is a knob, not a ;; performance — but it is a channel like any other so it can be keyed. @@ -65,10 +67,19 @@ [:xform :anchor] (ch/framed [0.0 0.0]) [:vis] (ch/framed true)}) +(def audio-defaults + "Unity gain, centred, at its own speed." + {[:audio :gain] (ch/framed 1.0) + [:audio :pan] (ch/framed 0.0) + [:audio :rate] (ch/framed 1.0)}) + +(defn defaults-of [n] + (if (= :audio (:kind n)) audio-defaults defaults)) + (defn channels - "The node's channels with the transform defaults filled in." + "The node's channels with its kind's defaults filled in." [n] - (merge defaults (:channels n))) + (merge (defaults-of n) (:channels n))) (defn set-channel "Write `v` into channel `path`: a key on the node's own frame `f` when the @@ -342,8 +353,8 @@ (conj (str ":kind " k " is in the vocabulary but not implemented")) (and (= k :instance) (nil? (:of n))) (conj "an instance needs :of") - (and (= k :audio) (nil? (get-in n [:source :footage]))) - (conj "an audio instance needs :source :footage") + (and (= k :audio) (not (some (:source n) [:footage :sound]))) + (conj "an audio node needs a :source :footage or :sound") (and (some? (get-in n [:time :rate])) (not (pos? (get-in n [:time :rate])))) (conj ":time :rate must be positive") diff --git a/frontend/src/arthur/domain/symbol.cljs b/frontend/src/arthur/domain/symbol.cljs index dbb0fc2..162b72a 100644 --- a/frontend/src/arthur/domain/symbol.cljs +++ b/frontend/src/arthur/domain/symbol.cljs @@ -302,7 +302,6 @@ (case (:kind n) :group nil :instance nil - :audio nil :poly (let [pts (rd [:geom :pts])] @@ -342,14 +341,17 @@ not an error, it is `(:nodes clip)` being nil and a frame resolving to no ops at all. That reads as a black stage, or, in a benchmark, as \"0 nodes\" and a flattering number. It happened once while the split was being made, which is why - this is a guard and not a comment." + this is a guard and not a comment. + + Sounds are left out: they are heard, not drawn, and have no transform to + evaluate. `audio/mix` is what plays them." [sym] (let [nodes (:nodes sym)] (when-not (map? nodes) (throw (ex-info (str "not a symbol: :nodes is " (pr-str nodes) " — a clip is not a symbol, its `:symbols` hold them") {:keys (vec (sort-by str (keys sym)))}))) - nodes)) + (into {} (remove #(= :audio (:kind (val %)))) nodes))) (defn- channel-frame "A trace selects the measured frames its node reads; marked channels read diff --git a/frontend/src/arthur/events/footage.cljs b/frontend/src/arthur/events/footage.cljs index 217eb92..d353d29 100644 --- a/frontend/src/arthur/events/footage.cljs +++ b/frontend/src/arthur/events/footage.cljs @@ -310,8 +310,10 @@ (rf/reg-fx ::list! (fn [_] - (-> (ingest/available!) - (.then (fn [footage] (rf/dispatch [::listed footage]))) + (-> (js/Promise.all #js [(ingest/available!) + (.then (http/GET "/api/sounds") + #(:sounds (js->clj % :keywordize-keys true)))]) + (.then (fn [[footage sounds]] (rf/dispatch [::listed footage sounds]))) (.catch (fn [error] (rf/dispatch [::failed (or (ex-message error) (str error))])))))) @@ -341,24 +343,38 @@ (.catch (fn [error] (rf/dispatch [::failed (or (ex-message error) (str error))]))))))) +(rf/reg-fx + ::upload-sound! + (fn [file] + (let [form (js/FormData.)] + (.append form "file" file) + (-> (http/POST-form "/api/sounds" form) + (.then (fn [^js sound] (rf/dispatch [::uploaded (.-id sound) "sound imported"]))) + (.catch (fn [error] + (rf/dispatch [::failed (or (ex-message error) (str error))]))))))) + (rf/reg-event-fx ::upload - (fn [{:keys [db]} [_ file]] - (if (or (nil? file) (get-in db [:footage :loading?])) - {} - {:db (update db :footage merge {:loading? true :status "uploading video…"}) - ::upload! file}))) + (fn [{:keys [db]} [_ ^js file]] + ;; By type, and by name for a browser that leaves the type empty. + (let [sound? (and file (or (string/starts-with? (.-type file) "audio/") + (re-find #"(?i)\.(mp3|wav|aiff?|flac|ogg|m4a|aac)$" (.-name file))))] + (if (or (nil? file) (get-in db [:footage :loading?])) + {} + {:db (update db :footage merge {:loading? true + :status (if sound? "uploading sound…" "uploading video…")}) + (if sound? ::upload-sound! ::upload!) file})))) (rf/reg-event-fx ::uploaded - (fn [{:keys [db]} [_ footage-id]] + (fn [{:keys [db]} [_ id status]] ;; Into the pool, and no further. An upload is media for this project; turning - ;; it into a symbol is a separate decision — which frames, what name — made by - ;; dropping it where it should go. + ;; it into a symbol or a sound is a separate decision — which frames, what + ;; name, where — made by dropping it where it should go. {:db (update db :footage #(-> % - (merge {:loading? false :chosen footage-id - :status "video extracted"}) - (update :uploaded (fnil conj #{}) footage-id))) + (merge {:loading? false :status (or status "video extracted")}) + (cond-> (not status) (assoc :chosen id)) + (update :uploaded (fnil conj #{}) id))) :dispatch [::refresh]})) (rf/reg-event-fx @@ -367,9 +383,10 @@ (rf/reg-event-db ::listed - (fn [db [_ footage]] + (fn [db [_ footage sounds]] (update db :footage merge (cond-> {:available (vec footage) + :sounds (vec sounds) :chosen (or (:chosen (:footage db)) (:id (first footage)))} (empty? footage) (assoc :status "upload a video to begin"))))) diff --git a/frontend/src/arthur/events/ui.cljs b/frontend/src/arthur/events/ui.cljs index 5ca0bc8..e59f872 100644 --- a/frontend/src/arthur/events/ui.cljs +++ b/frontend/src/arthur/events/ui.cljs @@ -16,13 +16,16 @@ (rf/reg-event-db ::select ;; The rows above a selection are opened, so one made deep on the stage is - ;; seen in the timeline. + ;; seen in the timeline. Not a sound's: its row is always in the audio section, + ;; and opening the placement it is heard through would bury it. (fn [db [_ selection]] - (let [[kind _ _ path] selection] + (let [[kind sid id path] selection + sound? (= :audio (get-in (store/entry (:clip/current db)) + [:clip :symbols sid :nodes id :kind]))] (cond-> (-> db (assoc-in [:ui :selection] selection) (update :ui dissoc :points)) - (and (= :node kind) path) + (and (= :node kind) path (not sound?)) (update-in [:ui :expanded] (fnil into #{}) (rest (reductions conj [] (pop path)))))))) (rf/reg-event-db @@ -174,6 +177,16 @@ host sid frame uuid point)) (assoc-in [:ui :selection] [:node host uuid [uuid]]))))) +(rf/reg-event-db + ::drop-sound + (fn [db [_ {:keys [source label length rate]} frame]] + (let [uuid (random-uuid) + host (get-in db [:ui :open])] + (-> db + (update :ui dissoc :drop) + (edit/edit #(clip/place-sound % host source label length rate frame uuid)) + (assoc-in [:ui :selection] [:node host uuid [uuid]]))))) + ;; --------------------------------------------------------------------------- ;; moving rows between symbols ;; @@ -195,7 +208,8 @@ (-> db (edit/edit (constantly (:clip r))) (assoc-in [:ui :selection] [:node (:sid r) (:id r) (conj to (:id r))]) - (update-in [:ui :expanded] into (rest (reductions conj [] to)))))))) + (cond-> (not= :audio (get-in r [:clip :symbols (:sid r) :nodes (:id r) :kind])) + (update-in [:ui :expanded] into (rest (reductions conj [] to))))))))) (rf/reg-event-db ::sliding diff --git a/frontend/src/arthur/subs/render.cljs b/frontend/src/arthur/subs/render.cljs index 17bad41..0454273 100644 --- a/frontend/src/arthur/subs/render.cljs +++ b/frontend/src/arthur/subs/render.cljs @@ -43,6 +43,17 @@ (when c (gesture/apply-values c sid id frame values))) c)))) +(rf/reg-sub + ::sounds + :<- [::clip-id] + :<- [::paint-revision] + :<- [::open] + (fn [[id _ sid] _] + ;; What the open symbol plays, off the document as saved rather than `::clip`, + ;; so a bar being slid does not re-mix on every frame of the drag. + (let [c (:clip (footage/entry id))] + [sid (when (clip/symbol c sid) (nest/audio-tracks c sid))]))) + (rf/reg-sub ::symbol :<- [::clip] diff --git a/frontend/src/arthur/subs/ui.cljs b/frontend/src/arthur/subs/ui.cljs index f451c3f..a00825a 100644 --- a/frontend/src/arthur/subs/ui.cljs +++ b/frontend/src/arthur/subs/ui.cljs @@ -86,3 +86,19 @@ :when f] f))] (filterv #(contains? used (:id %)) available)))) + +(rf/reg-sub + ::project-sounds + :<- [::render/clip-id] + :<- [::render/paint-revision] + :<- [::playback/footage] + (fn [[id _ {:keys [sounds uploaded]}] _] + ;; As `::project-footage`: the sounds this document plays, and those uploaded + ;; while it was open. + (let [used (into (set uploaded) + (for [[_ sym] (get-in (store/entry id) [:clip :symbols]) + [_ n] (:nodes sym) + :let [s (get-in n [:source :sound])] + :when s] + s))] + (filterv #(contains? used (:id %)) sounds)))) diff --git a/frontend/src/arthur/ui/drag.cljs b/frontend/src/arthur/ui/drag.cljs index 130f93c..690dfe3 100644 --- a/frontend/src/arthur/ui/drag.cljs +++ b/frontend/src/arthur/ui/drag.cljs @@ -52,19 +52,24 @@ out of the pool, and not one that would make a cycle." [] (let [c @carrying] - (and c (#{:symbol :footage :import} (:kind c)) (not (:refused? c))))) + (and c (#{:symbol :footage :import :sound} (:kind c)) (not (:refused? c))))) (defn row! - "Start carrying the timeline row at `path` — a node, to be moved into another - symbol or grouped with another node." - [path] - (reset! carrying {:kind :row :path path})) + "Start carrying the timeline row at `path` — a node of kind `node-kind`, to be + moved into another symbol or grouped with another node." + [path node-kind] + (reset! carrying {:kind :row :path path :node-kind node-kind})) (defn row "The path of the row being carried, or nil when it is not a row." [] (let [c @carrying] (when (= :row (:kind c)) (:path c)))) +(defn row-kind + "The node kind of the row being carried." + [] + (:node-kind @carrying)) + (defn other! "Start carrying something that is not yet in the document: `:kind` says what, and the rest is what a preview can show of it before it is fetched." @@ -81,9 +86,10 @@ "Say where the drag would land, for the previews. `point` is nil over the timeline." [where frame point] - (when-let [{:keys [label frames]} @carrying] + (when-let [{:keys [kind label frames]} @carrying] (rf/dispatch [::ui/drop-hover {:where where :frame frame :point point - :label label :frames frames}]))) + :label label :frames frames + :sound? (= :sound kind)}]))) (defn pos-for "Where the preview goes so the symbol's middle is under `point` — what @@ -102,5 +108,7 @@ ;; the symbol they become is called. :footage (rf/dispatch [::footage/ask-convert c frame point]) :import (rf/dispatch [::project/import c frame point]) + ;; Where it is dropped in time; a sound has no place in space. + :sound (rf/dispatch [::ui/drop-sound c frame]) nil)) (done!)) diff --git a/frontend/src/arthur/ui/params.cljs b/frontend/src/arthur/ui/params.cljs index f5b0daf..b644da3 100644 --- a/frontend/src/arthur/ui/params.cljs +++ b/frontend/src/arthur/ui/params.cljs @@ -229,7 +229,7 @@ ^{:key (str path)} [:<> [:dt (str/join " " (map name path))] - (if (and (contains? node/defaults path) (not (:dense ch))) + (if (and (contains? (node/defaults-of n) path) (not (:dense ch))) [channel-control sid id path ch frame] [:dd (channel-state ch)])]))])])) diff --git a/frontend/src/arthur/ui/pool.cljs b/frontend/src/arthur/ui/pool.cljs index d792856..d9fc04e 100644 --- a/frontend/src/arthur/ui/pool.cljs +++ b/frontend/src/arthur/ui/pool.cljs @@ -13,6 +13,9 @@ one of another's, and `footage:` for video. The stage and the timeline are where they are dropped. + A SOUND — mp3, wav — is `sound:`, and is dropped on the timeline, where it + becomes an audio node in the open symbol from the frame it lands on. + THE WHOLE PANE IS THE DROP TARGET for a file. A video dropped anywhere in it uploads and lands in THIS PROJECT's media, and goes no further: which frames of it become a symbol, and what that symbol is called, is asked when it is dropped @@ -78,6 +81,21 @@ #(drag/other! {:kind :footage :id id :label label :frames frames :fps fps :video video})))]) +(defn- sound-row + "An uploaded sound, or with `:footage?` a video's own — which is how a take's + sound goes back into its symbol after it was deleted there." + [{:keys [id label duration footage? fps]} project-fps] + (let [frames (js/Math.ceil (* duration project-fps)) + [source length rate] (if footage? + [{:footage id} (js/Math.round (* duration fps)) (/ fps project-fps)] + [{:sound id} frames 1])] + [item (merge {:label label + :sub (str (.toFixed duration 1) "s · " frames "f") + :thumb [:span.thumb.sound "♪"]} + (carrying (str "sound:" id) + #(drag/other! {:kind :sound :source source :label label + :length length :rate rate :frames frames})))])) + (defn- folder [title & children] (into [:details.pool-folder {:open true} [:summary title]] children)) @@ -90,6 +108,12 @@ selection @(rf/subscribe [::sub/selection]) open @(rf/subscribe [::render/open]) media @(rf/subscribe [::sub/project-footage]) + ;; The project's videos' sounds first, then its uploaded ones. + sounds (into (mapv (fn [{:keys [id label frames fps]}] + {:id id :label label :footage? true :fps fps + :duration (/ frames fps)}) + media) + @(rf/subscribe [::sub/project-sounds])) {:keys [chosen]} @(rf/subscribe [::playback/footage])] (folder "this project" (group "symbols" @@ -109,10 +133,15 @@ (group "media" (if (empty? media) [:div.dim "drop a video here"] - (doall (for [f media] ^{:key (:id f)} [footage-row f chosen]))))))) + (doall (for [f media] ^{:key (:id f)} [footage-row f chosen])))) + (group "sounds" + (if (empty? sounds) + [:div.dim "drop an mp3 or wav here"] + (doall (for [s sounds] ^{:key (:id s)} [sound-row s (:fps document)]))))))) (defn- all-assets [] - (let [{:keys [available chosen]} @(rf/subscribe [::playback/footage]) + (let [{:keys [available chosen sounds]} @(rf/subscribe [::playback/footage]) + fps (:fps @(rf/subscribe [::render/clip])) {:keys [symbols]} @(rf/subscribe [::project/assets]) {:keys [id]} @(rf/subscribe [::playback/project])] (folder "all assets" @@ -120,6 +149,10 @@ (if (empty? available) [:div.dim "nothing uploaded yet"] (doall (for [f available] ^{:key (:id f)} [footage-row f chosen])))) + (group "sounds" + (if (empty? sounds) + [:div.dim "no sounds uploaded yet"] + (doall (for [s sounds] ^{:key (:id s)} [sound-row s fps])))) (group "symbols" (let [others (remove #(= id (:project %)) symbols)] (if (empty? others) @@ -165,11 +198,11 @@ [:div.pane-head "media pool" [:span.spacer] - [:button {:title "add a video" + [:button {:title "add a video or a sound" :disabled loading? :on-click #(.click (js/document.getElementById "pool-file"))} "+"]] - [:input {:id "pool-file" :type "file" :accept "video/*" + [:input {:id "pool-file" :type "file" :accept "video/*,audio/*" :style {:display "none"} :on-change (fn [^js event] (when-let [file (aget (.. event -target -files) 0)] diff --git a/frontend/src/arthur/ui/shell.cljs b/frontend/src/arthur/ui/shell.cljs index 49b69a9..70a0608 100644 --- a/frontend/src/arthur/ui/shell.cljs +++ b/frontend/src/arthur/ui/shell.cljs @@ -9,6 +9,7 @@ [arthur.ui.convert :as convert] [arthur.events.playback :as pb] [arthur.subs.playback :as playback] + [arthur.subs.render :as render] [arthur.ui.palette :as palette] [arthur.ui.params :as params] [arthur.ui.pool :as pool] @@ -16,19 +17,31 @@ [arthur.ui.tabs :as tabs] [arthur.ui.timeline :as timeline] [arthur.ui.topbar :as topbar] - [re-frame.core :as rf])) + [re-frame.core :as rf] + [reagent.core :as r])) (defn- audio [] - [:audio - {:ref #(when % (clock/attach! %)) - :src @(rf/subscribe [::playback/audio]) - :preload "auto" - ;; Transport state follows the ELEMENT, not the other way round: the audio is - ;; the clock, so anything that can change its state — the end of the file, the - ;; OS media keys, a browser autoplay block — has to be able to correct the - ;; document rather than be contradicted by it. - :on-play #(rf/dispatch [::pb/play]) - :on-pause #(rf/dispatch [::pb/pause])}]) + ;; An edit to what the open symbol plays — a sound dropped, moved, turned + ;; down, undone — re-mixes the clock. Opening a symbol mixes on its own, so + ;; only a change under the same one counts. + (r/with-let [heard (atom nil) + remix (r/track! (fn [] + (let [[sid :as now] @(rf/subscribe [::render/sounds]) + [was :as before] @heard] + (reset! heard now) + (when (and before (= sid was) (not= now before)) + (rf/dispatch [::pb/refresh-clock])))))] + [:audio + {:ref #(when % (clock/attach! %)) + :src @(rf/subscribe [::playback/audio]) + :preload "auto" + ;; Transport state follows the ELEMENT, not the other way round: the audio is + ;; the clock, so anything that can change its state — the end of the file, the + ;; OS media keys, a browser autoplay block — has to be able to correct the + ;; document rather than be contradicted by it. + :on-play #(rf/dispatch [::pb/play]) + :on-pause #(rf/dispatch [::pb/pause])}] + (finally (r/dispose! remix)))) (defn view [] [:div.app diff --git a/frontend/src/arthur/ui/timeline.cljs b/frontend/src/arthur/ui/timeline.cljs index 784128c..2261293 100644 --- a/frontend/src/arthur/ui/timeline.cljs +++ b/frontend/src/arthur/ui/timeline.cljs @@ -1,5 +1,6 @@ (ns arthur.ui.timeline - "The bottom pane: the transport, a ruler, and a row per node of the open symbol. + "The bottom pane: the transport, a ruler, and a row per node of the open symbol + — the picture's, then, under an `audio` heading, every sound it plays. `rows` is the whole of the interesting part and it is a PURE function of the clip, the open symbol and the set of open paths. It flattens the document's two @@ -87,7 +88,10 @@ ;; everywhere. `:z` is the lexicographic draw key; ;; the id breaks ties so the order is stable. (sort-by (fn [[id n]] [(or (:z n) "") (str id)])) - reverse)] + reverse + ;; Sounds are listed below the picture, by + ;; `sound-rows`, wherever they are. + (remove #(= :audio (:kind (val %)))))] (into [] (mapcat (fn [[id n]] @@ -133,6 +137,50 @@ (walk sid [] 0 identity) []))) +(defn sound-rows + "Every sound symbol `sid` plays, one row each, below the picture as an + editor's audio tracks are: its own, and those inside what it places at any + depth — each TIED to the placement it is heard through, `:via`, and drawn + where it is heard, cut to that placement's span as the mix cuts it." + [clip sid expanded] + (letfn [(walk [sid path ->open [lo hi] via] + (mapcat + (fn [[id n]] + (let [rpath (conj path id) + self (comp ->open (local->parent n)) + [a b] (mapv ->open (or (node/placed-span + (cond-> n + (and (= :instance (:kind n)) (nil? (:span n))) + (assoc :span [0 (get-in clip [:symbols (:of n) :frames])]))) + [0 (get-in clip [:symbols sid :frames])])) + span [(max lo a) (min hi b)] + open? (contains? expanded rpath)] + (case (:kind n) + :audio (cons {:path rpath + :depth 0 + :label (node-label id n) + :kind :node + :node-kind :audio + :via via + ;; A sound heard through a placement is + ;; that placement's picture's sound: its bar + ;; moves the placement, so the two stay in + ;; sync. Moving the placement moves it too. + :slides (if via (subvec rpath 0 1) rpath) + :select [:node sid id rpath] + :expandable? true + :expanded? open? + :span span + :keys (into [] (comp (mapcat keyed-frames) (map self) (distinct)) + (vals (node/channels n)))} + (when open? (channel-rows n rpath 1 self span))) + :instance (walk (:of n) rpath self span (or via (node-label id n))) + nil))) + (sort-by (fn [[id n]] [(or (:z n) "") (str id)]) (get-in clip [:symbols sid :nodes]))))] + (if (get-in clip [:symbols sid]) + (vec (walk sid [] identity [##-Inf ##Inf] nil)) + []))) + ;; --------------------------------------------------------------------------- ;; geometry ;; @@ -189,19 +237,26 @@ (when (and drop (pos? drop)) (str " · " (.toFixed drop 2) " f/paint")))]])) (defn- takes? - "Whether the row at `target` can take the row being carried: not itself, and - not anything inside it." - [target] + "Whether the row at `target`, a `target-kind` node, can take the row being + carried: not itself, and not anything inside it. A sound goes into a + placement or beside another sound, and only a sound goes beside a sound." + [target target-kind] (when-let [from (drag/row)] - (not= from (subvec target 0 (min (count from) (count target)))))) + (and (not= from (subvec target 0 (min (count from) (count target)))) + (if (= :audio (drag/row-kind)) + (#{:audio :instance} target-kind) + (not= :audio target-kind))))) (defn- zone "Which part of a row the pointer is over: its top edge, to go in front of it; - its bottom edge, to go behind; its middle, to go into it or be grouped with it." - [^js e] + its bottom edge, to go behind; its middle, to go into it or be grouped with it. + Nothing goes into a sound, so its middle is behind it." + [^js e node-kind] (let [box (.getBoundingClientRect (.-currentTarget e)) y (/ (- (.-clientY e) (.-top box)) (max 1 (.-height box)))] - (cond (< y 0.3) :front (> y 0.7) :back :else :into))) + (cond (< y 0.3) :front + (or (> y 0.7) (= :audio node-kind)) :back + :else :into))) (def ^:private reveal "A ref that scrolls the selected row into view, ONE per selection: React calls @@ -210,7 +265,7 @@ on every render after, which would fight a person scrolling away." (memoize (fn [_selection] (fn [el] (some-> el (.scrollIntoView #js {:block "nearest"})))))) -(defn- label-cell [{:keys [path depth label kind node-kind select expandable? expanded? of]} +(defn- label-cell [{:keys [path depth label kind node-kind select expandable? expanded? of via]} selection over solo] (let [node? (= :node kind) [over-path where] @over] @@ -237,21 +292,21 @@ (.stopPropagation e) (.setData (.-dataTransfer e) "text/plain" "row") (set! (.. e -dataTransfer -effectAllowed) "move") - (drag/row! path)) + (drag/row! path node-kind)) :on-drag-end (fn [_] (reset! over nil) (drag/done!)) - :on-drag-enter (fn [^js e] (when (takes? path) (.preventDefault e))) + :on-drag-enter (fn [^js e] (when (takes? path node-kind) (.preventDefault e))) :on-drag-over (fn [^js e] - (when (takes? path) + (when (takes? path node-kind) (.preventDefault e) (.stopPropagation e) (set! (.. e -dataTransfer -dropEffect) "move") - (let [o [path (zone e)]] + (let [o [path (zone e node-kind)]] (when (not= o @over) (reset! over o))))) :on-drop (fn [^js e] (.preventDefault e) (.stopPropagation e) - (let [from (when (takes? path) (drag/row)) - where (zone e)] + (let [from (when (takes? path node-kind) (drag/row)) + where (zone e node-kind)] (reset! over nil) (drag/done!) (when from @@ -267,7 +322,7 @@ (rf/dispatch [::ui/toggle-row path]))} (when expandable? (if expanded? "▾" "▸"))] [:span.name label] - (when node? [:span.kind (str "·" (name node-kind))]) + (when node? [:span.kind (if via (str "· in " via) (str "·" (name node-kind)))]) (when (= :instance node-kind) [:button {:class (str "tl-solo" (when (contains? solo path) " on")) :title "show only this on the stage (⇧ for more than one)" @@ -286,19 +341,19 @@ "`sliding` is the pointer's side of a bar being dragged, `{:path :x :width :df}`. What it looks like mid-drag is `[:ui :sliding]`, which the clip every row and the stage are drawn from already has in it." - [{:keys [path span keys dense? kind select]} frames sliding] + [{:keys [path span keys dense? kind node-kind select slides]} frames sliding] (let [{from :path x0 :x width :width} @sliding slide (fn [^js e] (when (= path from) (let [df (js/Math.round (/ (* frames (- (.-clientX e) x0)) (max 1 width)))] (when (not= df (:df @sliding)) (swap! sliding assoc :df df) - (rf/dispatch [::ui/sliding path df]))))) + (rf/dispatch [::ui/sliding (or slides path) df]))))) done (fn [commit?] (when (= path from) (let [df (:df @sliding)] (reset! sliding nil) - (rf/dispatch (if commit? [::ui/slide path df] [::ui/sliding nil])))))] + (rf/dispatch (if commit? [::ui/slide (or slides path) df] [::ui/sliding nil])))))] [:div.tl-track ;; The track, not the bar, holds the pointer while a bar slides, so the drag ;; goes on when the bar has slid off the ruler and is no longer drawn. @@ -310,6 +365,7 @@ (when-let [[in out] (when span [(max 0 (first span)) (min frames (second span))])] (when (< in out) [:div {:class (str "tl-span" (when dense? " dense") (when (= :ghost kind) " ghost") + (when (= :audio node-kind) " sound") (when select " movable") (when (= path from) " sliding")) :style {:left (edge% in frames) :width (str (* 100 (/ (- out in) (max 1 frames))) "%")} @@ -345,14 +401,24 @@ expanded @(rf/subscribe [::sub/expanded]) drop @(rf/subscribe [::sub/drop]) solo (set @(rf/subscribe [::render/solo])) + open @(rf/subscribe [::render/open]) ;; Where a drag out of the pool would land, as a row of its own at the - ;; top: its own length, starting on the frame it would start on. The - ;; stage's drop shows it too, at the playhead. - visible (cond->> (rows clip @(rf/subscribe [::render/open]) expanded) - drop (cons {:path [::drop] :depth 0 :kind :ghost - :label (str "+ " (:label drop)) - :span [(:frame drop) (+ (:frame drop) (or (:frames drop) 1))] - :keys []})) + ;; top of its section: its own length, starting on the frame it would + ;; start on. The stage's drop shows it too, at the playhead. + ghost (when drop + {:path [::drop] :depth 0 :kind :ghost + :label (str "+ " (:label drop)) + :span [(:frame drop) (+ (:frame drop) (or (:frames drop) 1))] + :keys []}) + picture (cond->> (rows clip open expanded) + (and ghost (not (:sound? drop))) (cons ghost)) + sounds (cond->> (sound-rows clip open expanded) + (and ghost (:sound? drop)) (cons ghost)) + ;; The audio section's heading is a row like the others, so the two + ;; columns stay aligned without measuring anything. + visible (cond-> (vec picture) + (seq sounds) (-> (conj {:path [::sounds] :kind :section :label "audio"}) + (into sounds))) ;; Roughly ten labels, on a round number of frames. step (* 10 (js/Math.ceil (/ frames 100)))] [:section.pane.time @@ -373,7 +439,10 @@ (rf/dispatch [::ui/move-node from []]))))} [:div.tl-corner] (doall (for [row visible] - ^{:key (str (:path row))} [label-cell row selection over solo]))] + (with-meta (if (= :section (:kind row)) + [:div.tl-label.tl-section (:label row)] + [label-cell row selection over solo]) + {:key (str (:path row))})))] [:div.tl-tracks {:on-drag-enter (fn [^js e] (when (drag/accepts?) (.preventDefault e))) :on-drag-over (fn [^js e] @@ -419,6 +488,9 @@ [:div.tl-knob {:style {:left (at% frame frames)}}]] (if (seq visible) (doall (for [row visible] - ^{:key (str (:path row))} [track-cell row frames sliding])) + (with-meta (if (= :section (:kind row)) + [:div.tl-track.tl-section] + [track-cell row frames sliding]) + {:key (str (:path row))}))) [:div.tl-empty "nothing in this symbol"]) [:div.tl-playhead {:style {:left (at% frame frames)}}]]]]))) diff --git a/frontend/test/arthur/domain/nest_test.cljs b/frontend/test/arthur/domain/nest_test.cljs index f713cf7..48c7039 100644 --- a/frontend/test/arthur/domain/nest_test.cljs +++ b/frontend/test/arthur/domain/nest_test.cljs @@ -238,3 +238,26 @@ "and nothing else is renumbered") (is (:refused (nest/restack c :main [:a] [a-uuid :inner] true)) "only among the things it is beside"))) + +(deftest a-sound-dropped-in-a-placed-symbol-is-heard-where-the-placement-puts-it + (let [u #uuid "00000000-0000-4000-8000-0000000000a1" + c (-> (nested) + (clip/place-sound :inner {:sound "tone"} "tone.mp3" 40 1 2 u)) + sound (get-in c [:symbols :inner :nodes u]) + [heard] (nest/audio-tracks c :outer)] + (is (empty? (node/problems sound)) "a sound needs no transform") + (is (= {:sound "tone"} (:source sound))) + (is (= #{[:audio :gain] [:audio :pan] [:audio :rate]} (set (keys (node/channels sound))))) + (is (= [0 8] (:span heard)) + "own frames 0-8: it starts on inner's 2 and inner ends on 10") + (is (= [7 15] (node/placed-span heard)) "inner starts on 5 of outer") + (is (empty? ((clip/resolver c nil pal/index-of :inner) 3)) + "and it draws nothing") + (is (= c (clip/place-sound c :inner {:sound "tone"} "tone.mp3" 40 1 10 (random-uuid))) + "nor lands past the end of its symbol") + (testing "a video's own sound keeps its length at another rate" + (let [v (-> (nested) + (clip/place-sound :outer {:footage "f"} "take" 90 2.5 0 u) + (get-in [:symbols :outer :nodes u]))] + (is (empty? (node/problems v))) + (is (= [0 36] (node/placed-span v)) "90 frames at 30 are 36 at 12"))))) diff --git a/frontend/test/arthur/domain/node_test.cljs b/frontend/test/arthur/domain/node_test.cljs index ca810c8..55dff81 100644 --- a/frontend/test/arthur/domain/node_test.cljs +++ b/frontend/test/arthur/domain/node_test.cljs @@ -157,8 +157,8 @@ (deftest skew-and-anchor-are-in-the-shape-although-nothing-drives-them ;; A decomposition is not extensible after the fact: adding a component later ;; means migrating every stored transform. So both are present from the start, - ;; on every kind. - (doseq [k node/implemented-kinds] + ;; on every kind that is in the picture. + (doseq [k (disj node/implemented-kinds :audio)] (is (contains? (get node/valid-paths k) [:xform :skew]) (str k)) (is (contains? (get node/valid-paths k) [:xform :anchor]) (str k)))) @@ -168,7 +168,13 @@ (is (contains? (:disc node/valid-paths) [:geom :radius])) (is (contains? (:rect node/valid-paths) [:geom :size])) (is (not (contains? (:group node/valid-paths) [:geom :pts])) - "a group is a pure transform node")) + "a group is a pure transform node") + (is (= #{[:audio :gain] [:audio :pan] [:audio :rate]} (:audio node/valid-paths)) + "a sound has no transform")) + +(deftest a-sound-defaults-to-its-own-channels + (is (= (set (keys node/audio-defaults)) + (set (keys (node/channels {:id :s :kind :audio})))))) (deftest problems-names-the-ways-a-node-is-malformed (is (empty? (node/problems {:id :x :kind :group :z "a1"}))) diff --git a/static/arthur/app.css b/static/arthur/app.css index 7bfb424..7c018d5 100644 --- a/static/arthur/app.css +++ b/static/arthur/app.css @@ -684,6 +684,16 @@ button.share-button:hover, button.share-button.on { filter: brightness(1.1); } line-height: var(--ruler); } +/* The heading between the picture's rows and the sounds', as an editor puts + its audio tracks under its video tracks. */ +.tl-label.tl-section, +.tl-track.tl-section { background: var(--chrome); border-top: 1px solid var(--line); } +.tl-label.tl-section { padding-left: 6px; color: var(--dim); font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; } + +/* A sound's bar, told apart from the picture's at a glance. */ +.tl-span.sound { background: #e3efdf; border-color: #a6c49b; } +.pool-item .thumb.sound { display: flex; align-items: center; justify-content: center; color: #d9d9d9; } + /* The span the node exists over. Drawn under the keys so a dot on the first frame of a span is not half-hidden by its own bar. */ .tl-span {