sound stuff

clips/extraction.py

@@ -161,6 +161,18 @@ def _extract_stills(job, proxy_path, frames_dir, frames, root):
 MAX_RATE = 120  # a capture rate; past this the container is describing something else
 
 
+def probe_audio(path):
+    """The length of an uploaded sound in seconds, refusing a file with no audio."""
+    data = json.loads(_command(["ffprobe", "-v", "error", "-show_streams",
+                                "-show_format", "-of", "json", str(path)]))
+    if not any(s.get("codec_type") == "audio" for s in data.get("streams", [])):
+        raise ValueError("the uploaded file has no audio stream")
+    duration = float(data.get("format", {}).get("duration") or 0)
+    if duration <= 0:
+        raise ValueError("the sound's length is unknown")
+    return duration
+
+
 def probe(path):
     """What the upload is, as far as choosing a proxy rate goes.
 

clips/migrations/0010_sounds.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.2.17 on 2026-09-30 07:14
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('clips', '0009_revision_blocks'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Sound',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('filename', models.CharField(max_length=255)),
+                ('duration', models.FloatField(help_text='seconds, as ffprobe reports it')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('blob', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='sound_for', to='clips.blob')),
+            ],
+        ),
+    ]

clips/models.py

@@ -52,6 +52,18 @@ class Source(models.Model):
     created = models.DateTimeField(auto_now_add=True)
 
 
+class Sound(models.Model):
+    """An uploaded sound file — mp3, wav, whatever the browser can decode — kept
+    as uploaded. Not footage: it has no frames and nothing measures it, so it
+    skips extraction and an audio node plays its bytes directly."""
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    blob = models.ForeignKey(Blob, on_delete=models.PROTECT, related_name="sound_for")
+    filename = models.CharField(max_length=255)
+    duration = models.FloatField(help_text="seconds, as ffprobe reports it")
+    created = models.DateTimeField(auto_now_add=True)
+
+
 class Extraction(models.Model):
     """One requested decode of a source into immutable footage."""
 

clips/tests/test_api.py

@@ -34,7 +34,7 @@ from django.core.management import call_command
 from django.test import TestCase, override_settings
 
 from clips import blobs, extraction
-from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision, Source
+from clips.models import Analysis, Block, Blob, Clip, Footage, Leaf, Project, Revision, Sound, Source
 
 BLOB_DIR = tempfile.mkdtemp(prefix="arthur-test-blobs-")
 
@@ -721,6 +721,36 @@ class UploadTests(TestCase):
             self.assertEqual(27, job.progress)
             job.save.assert_called_once_with(update_fields=["progress", "updated"])
 
+    def test_an_uploaded_mp3_is_a_sound_and_not_footage(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "tone.mp3"
+            subprocess.run([
+                "ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
+                "-f", "lavfi", "-i", "sine=frequency=440:duration=1.5", str(path),
+            ], check=True, capture_output=True)
+            payload = path.read_bytes()
+
+        uploaded = self.client.post("/api/sounds", {
+            "file": SimpleUploadedFile("tone.mp3", payload, content_type="audio/mpeg")})
+        self.assertEqual(201, uploaded.status_code, uploaded.content)
+        sound = uploaded.json()
+        self.assertEqual("tone.mp3", sound["label"])
+        self.assertAlmostEqual(1.5, sound["duration"], delta=0.1)
+        self.assertEqual(payload, b"".join(self.client.get(sound["audio"]).streaming_content))
+        self.assertEqual(sound, self.client.get(f"/api/sounds/{sound['id']}").json())
+        self.assertEqual([sound], self.client.get("/api/sounds").json()["sounds"])
+        self.assertEqual(0, Source.objects.count())
+
+        again = self.client.post("/api/sounds", {
+            "file": SimpleUploadedFile("again.mp3", payload, content_type="audio/mpeg")})
+        self.assertEqual(200, again.status_code)
+        self.assertEqual(1, Sound.objects.count())
+
+    def test_a_file_without_audio_is_not_a_sound(self):
+        refused = self.client.post("/api/sounds", {
+            "file": SimpleUploadedFile("notes.txt", b"not audio", content_type="text/plain")})
+        self.assertEqual(400, refused.status_code)
+
     def test_uploaded_video_extracts_to_reopenable_footage(self):
         with tempfile.TemporaryDirectory() as directory:
             path = Path(directory) / "four-frames.mp4"

clips/urls.py

@@ -22,6 +22,8 @@ urlpatterns = [
     path("logout", views.logout),
     path("detector", views.detector),
     path("sources", views.sources),
+    path("sounds", views.sounds),
+    path("sounds/<uuid:sound_id>", views.sound_detail),
     path("extractions", views.extractions),
     path("extractions/<str:key>", views.extraction_detail),
     path("footage", views.footage_list),

clips/views.py

@@ -43,7 +43,7 @@ from django.views.decorators.http import require_http_methods
 
 from . import blobs, extraction
 from .consumers import broadcast
-from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Leaf, Project, Revision, Source
+from .models import Analysis, Block, Blob, Clip, Extraction, Footage, Leaf, Project, Revision, Sound, Source
 
 KEY_LENGTH = 71  # "sha256:" + 64 hex
 
@@ -225,6 +225,41 @@ def sources(request):
         return JsonResponse({"error": str(exc)}, status=400)
 
 
+def _sound_json(row):
+    return {"id": str(row.id), "label": row.filename, "duration": row.duration,
+            "audio": f"/blob/{row.blob_id}"}
+
+
+@require_http_methods(["GET", "POST"])
+def sounds(request):
+    if request.method == "GET":
+        return JsonResponse({"sounds": [_sound_json(row)
+                                        for row in Sound.objects.order_by("-created")]})
+    upload = request.FILES.get("file")
+    if upload is None:
+        return JsonResponse({"error": "upload a sound as the file field"}, status=400)
+    try:
+        digest, size = blobs.write_stream(upload.chunks())
+        duration = extraction.probe_audio(blobs.path_for(digest))
+        blob, _ = Blob.objects.get_or_create(
+            digest=digest, defaults={"size": size,
+                                     "media_type": upload.content_type or "audio/mpeg"})
+        row, created = Sound.objects.get_or_create(
+            blob=blob, defaults={"filename": Path(upload.name).name[:255],
+                                 "duration": duration})
+        return JsonResponse(_sound_json(row), status=201 if created else 200)
+    except (ValueError, OSError) as exc:
+        return JsonResponse({"error": str(exc)}, status=400)
+
+
+@require_http_methods(["GET"])
+def sound_detail(request, sound_id):
+    try:
+        return JsonResponse(_sound_json(Sound.objects.get(id=sound_id)))
+    except Sound.DoesNotExist:
+        return JsonResponse({"error": "no such sound"}, status=404)
+
+
 def _extraction_json(row):
     return {"key": row.key, "source": str(row.source_id), "state": row.state,
             "progress": row.progress, "error": row.error,

frontend/src/arthur/audio/mix.cljs

@@ -55,26 +55,36 @@
   (js/URL.createObjectURL
    (js/Blob. #js [(wav-bytes buffer)] #js {:type "audio/wav"})))
 
-(defn- source! [footage-id]
+(defn- fetch-ok! [url what]
-  (-> (js/fetch (str "/api/footage/" footage-id))
+  (-> (js/fetch url)
       (.then (fn [response]
                (when-not (.-ok response)
-                 (throw (ex-info "audio track's footage is missing"
+                 (throw (ex-info (str "audio track's " what " is missing")
-                                 {:footage footage-id :status (.-status response)})))
+                                 {:url url :status (.-status response)})))
-               (.json response)))
+               response))))
-      (.then (fn [^js manifest]
+
-               (-> (js/fetch (.-audio manifest))
+(defn- decode-bytes! [bytes]
-                   (.then (fn [response]
+  (.decodeAudioData (js/OfflineAudioContext. 1 1 44100) bytes))
-                            (when-not (.-ok response)
+
-                              (throw (ex-info "audio track's blob is missing"
+(defn- source!
-                                              {:footage footage-id :status (.-status response)})))
+  "Promise of `[source {:buffer :fps}]` for an audio node's `:source`. Footage
-                            (.arrayBuffer response)))
+  counts its frames at its own rate; a sound file has no frames of its own, so
-                   (.then (fn [bytes]
+  its `:fps` is nil and it counts in the document's."
-                            (let [decoder (js/OfflineAudioContext. 1 1 44100)]
+  [{:keys [footage sound] :as source}]
-                              (-> (.decodeAudioData decoder bytes)
+  (if sound
-                                  (.then (fn [buffer]
+    (-> (fetch-ok! (str "/api/sounds/" sound) "sound")
-                                           [footage-id {:buffer buffer
+        (.then #(.json %))
-                                                        :fps (.-fps manifest)}])))))))))))
+        (.then #(fetch-ok! (.-audio %) "blob"))
+        (.then #(.arrayBuffer %))
+        (.then decode-bytes!)
+        (.then (fn [buffer] [source {:buffer buffer}])))
+    (-> (fetch-ok! (str "/api/footage/" footage) "footage")
+        (.then #(.json %))
+        (.then (fn [^js manifest]
+                 (-> (fetch-ok! (.-audio manifest) "blob")
+                     (.then #(.arrayBuffer %))
+                     (.then decode-bytes!)
+                     (.then (fn [buffer] [source {:buffer buffer :fps (.-fps manifest)}]))))))))
 
 (defn- automate! [^js param channel start end fps factor default store]
   (let [channel (or channel (ch/framed default))]
@@ -107,7 +117,7 @@
       (let [[start end] (or (node/placed-span track) [0 frames])
             start (max 0 start)
             end (min frames end)
-            {:keys [buffer fps]} (get sources (get-in track [:source :footage]))
+            {:keys [buffer fps] :or {fps (:fps document)}} (get sources (:source track))
             sound (.createBufferSource output)
             gain (.createGain output)
             pan (.createStereoPanner output)]
@@ -143,7 +153,7 @@
      (if (empty? tracks)
        (js/Promise.resolve nil)
        (-> (js/Promise.all
-            (into-array (map source! (distinct (map #(get-in % [:source :footage]) tracks)))))
+            (into-array (map source! (distinct (map :source tracks)))))
            (.then (fn [pairs] (render! document sid (into {} (array-seq pairs)) store))))))))
 
 (defn decode!
@@ -156,8 +166,7 @@
                  (throw (ex-info "the clip's audio did not load"
                                  {:url url :status (.-status response)})))
                (.arrayBuffer response)))
-      (.then (fn [bytes]
+      (.then decode-bytes!)))
-               (.decodeAudioData (js/OfflineAudioContext. 1 1 44100) bytes)))))
 
 (defn mix!
   "Promise of a mixed WAV URL for symbol `sid`, or the original URL when it has

frontend/src/arthur/domain/clip.cljs

@@ -307,6 +307,27 @@
                                        :value (if point (mapv - point middle) [0 0])}
                      [:xform :anchor] {:animated? false :value middle}}})))))
 
+(defn place-sound
+  "An audio node playing `source` — `{:sound id}`, an uploaded file, or
+  `{:footage id}`, a video's own sound — inside symbol `host` from `frame` of it.
+  Timed as an instance is: `:at frame`, its span `length` of its OWN frames, and
+  `rate` of those to one of `host`'s, which is what keeps a 30fps video's sound
+  its own length in a 12fps project. Nothing else: a sound is not in the picture."
+  [clip host source label length rate frame uuid]
+  (let [end (frames clip host)]
+    (if (or (nil? end) (nil? frame) (neg? frame) (>= frame end))
+      clip
+      (update-symbol
+       clip host assoc-in [:nodes uuid]
+       {:id uuid
+        :name label
+        :kind :audio
+        :parent nil
+        :z (str "z" (js/Date.now) "-sound")
+        :source source
+        :span [0 (max 1 length)]
+        :time {:mode :map :at frame :rate rate}}))))
+
 (defn fresh-id
   "The first `:symbol-N` the clip does not already hold. Readable because an id
   shows up in saved leaf paths, and deterministic because this namespace is pure."

frontend/src/arthur/domain/node.cljs

@@ -46,7 +46,9 @@
   (let [base (into #{[:vis]} xform-paths)]
     {:group base
      :instance base
-     :audio (into base [[:audio :gain] [:audio :pan] [:audio :rate]])
+     ;; A sound is not in the picture: no transform, no visibility. After
+     ;; Effects' audio-only layer has no Transform group for the same reason.
+     :audio #{[:audio :gain] [:audio :pan] [:audio :rate]}
      :poly  (into base [[:geom :pts] [:style :color]])
      ;; A disc's radius is framed in practice — iris size is a knob, not a
      ;; performance — but it is a channel like any other so it can be keyed.
@@ -65,10 +67,19 @@
    [:xform :anchor] (ch/framed [0.0 0.0])
    [:vis]           (ch/framed true)})
 
+(def audio-defaults
+  "Unity gain, centred, at its own speed."
+  {[:audio :gain] (ch/framed 1.0)
+   [:audio :pan]  (ch/framed 0.0)
+   [:audio :rate] (ch/framed 1.0)})
+
+(defn defaults-of [n]
+  (if (= :audio (:kind n)) audio-defaults defaults))
+
 (defn channels
-  "The node's channels with the transform defaults filled in."
+  "The node's channels with its kind's defaults filled in."
   [n]
-  (merge defaults (:channels n)))
+  (merge (defaults-of n) (:channels n)))
 
 (defn set-channel
   "Write `v` into channel `path`: a key on the node's own frame `f` when the
@@ -342,8 +353,8 @@
          (conj (str ":kind " k " is in the vocabulary but not implemented"))
 
          (and (= k :instance) (nil? (:of n))) (conj "an instance needs :of")
-         (and (= k :audio) (nil? (get-in n [:source :footage])))
+         (and (= k :audio) (not (some (:source n) [:footage :sound])))
-         (conj "an audio instance needs :source :footage")
+         (conj "an audio node needs a :source :footage or :sound")
          (and (some? (get-in n [:time :rate]))
               (not (pos? (get-in n [:time :rate]))))
          (conj ":time :rate must be positive")

frontend/src/arthur/domain/symbol.cljs

@@ -302,7 +302,6 @@
     (case (:kind n)
       :group nil
       :instance nil
-      :audio nil
 
       :poly
       (let [pts (rd [:geom :pts])]
@@ -342,14 +341,17 @@
   not an error, it is `(:nodes clip)` being nil and a frame resolving to no ops at
   all. That reads as a black stage, or, in a benchmark, as \"0 nodes\" and a
   flattering number. It happened once while the split was being made, which is why
-  this is a guard and not a comment."
+  this is a guard and not a comment.
+
+  Sounds are left out: they are heard, not drawn, and have no transform to
+  evaluate. `audio/mix` is what plays them."
   [sym]
   (let [nodes (:nodes sym)]
     (when-not (map? nodes)
       (throw (ex-info (str "not a symbol: :nodes is " (pr-str nodes)
                            " — a clip is not a symbol, its `:symbols` hold them")
                       {:keys (vec (sort-by str (keys sym)))})))
-    nodes))
+    (into {} (remove #(= :audio (:kind (val %)))) nodes)))
 
 (defn- channel-frame
   "A trace selects the measured frames its node reads; marked channels read

frontend/src/arthur/events/footage.cljs

@@ -310,8 +310,10 @@
 (rf/reg-fx
  ::list!
  (fn [_]
-   (-> (ingest/available!)
+   (-> (js/Promise.all #js [(ingest/available!)
-       (.then (fn [footage] (rf/dispatch [::listed footage])))
+                            (.then (http/GET "/api/sounds")
+                                   #(:sounds (js->clj % :keywordize-keys true)))])
+       (.then (fn [[footage sounds]] (rf/dispatch [::listed footage sounds])))
        (.catch (fn [error]
                  (rf/dispatch [::failed (or (ex-message error) (str error))]))))))
 
@@ -341,24 +343,38 @@
          (.catch (fn [error]
                    (rf/dispatch [::failed (or (ex-message error) (str error))])))))))
 
+(rf/reg-fx
+ ::upload-sound!
+ (fn [file]
+   (let [form (js/FormData.)]
+     (.append form "file" file)
+     (-> (http/POST-form "/api/sounds" form)
+         (.then (fn [^js sound] (rf/dispatch [::uploaded (.-id sound) "sound imported"])))
+         (.catch (fn [error]
+                   (rf/dispatch [::failed (or (ex-message error) (str error))])))))))
+
 (rf/reg-event-fx
  ::upload
- (fn [{:keys [db]} [_ file]]
+ (fn [{:keys [db]} [_ ^js file]]
-   (if (or (nil? file) (get-in db [:footage :loading?]))
+   ;; By type, and by name for a browser that leaves the type empty.
-     {}
+   (let [sound? (and file (or (string/starts-with? (.-type file) "audio/")
-     {:db (update db :footage merge {:loading? true :status "uploading video…"})
+                              (re-find #"(?i)\.(mp3|wav|aiff?|flac|ogg|m4a|aac)$" (.-name file))))]
-      ::upload! file})))
+     (if (or (nil? file) (get-in db [:footage :loading?]))
+       {}
+       {:db (update db :footage merge {:loading? true
+                                       :status (if sound? "uploading sound…" "uploading video…")})
+        (if sound? ::upload-sound! ::upload!) file}))))
 
 (rf/reg-event-fx
  ::uploaded
- (fn [{:keys [db]} [_ footage-id]]
+ (fn [{:keys [db]} [_ id status]]
    ;; Into the pool, and no further. An upload is media for this project; turning
-   ;; it into a symbol is a separate decision — which frames, what name — made by
+   ;; it into a symbol or a sound is a separate decision — which frames, what
-   ;; dropping it where it should go.
+   ;; name, where — made by dropping it where it should go.
    {:db (update db :footage #(-> %
-                                 (merge {:loading? false :chosen footage-id
+                                 (merge {:loading? false :status (or status "video extracted")})
-                                         :status "video extracted"})
+                                 (cond-> (not status) (assoc :chosen id))
-                                 (update :uploaded (fnil conj #{}) footage-id)))
+                                 (update :uploaded (fnil conj #{}) id)))
     :dispatch [::refresh]}))
 
 (rf/reg-event-fx
@@ -367,9 +383,10 @@
 
 (rf/reg-event-db
  ::listed
- (fn [db [_ footage]]
+ (fn [db [_ footage sounds]]
    (update db :footage merge
            (cond-> {:available (vec footage)
+                    :sounds (vec sounds)
                     :chosen (or (:chosen (:footage db)) (:id (first footage)))}
              (empty? footage) (assoc :status "upload a video to begin")))))
 

frontend/src/arthur/events/ui.cljs

@@ -16,13 +16,16 @@
 (rf/reg-event-db
  ::select
  ;; The rows above a selection are opened, so one made deep on the stage is
- ;; seen in the timeline.
+ ;; seen in the timeline. Not a sound's: its row is always in the audio section,
+ ;; and opening the placement it is heard through would bury it.
  (fn [db [_ selection]]
-   (let [[kind _ _ path] selection]
+   (let [[kind sid id path] selection
+         sound? (= :audio (get-in (store/entry (:clip/current db))
+                                  [:clip :symbols sid :nodes id :kind]))]
      (cond-> (-> db
                  (assoc-in [:ui :selection] selection)
                  (update :ui dissoc :points))
-       (and (= :node kind) path)
+       (and (= :node kind) path (not sound?))
        (update-in [:ui :expanded] (fnil into #{}) (rest (reductions conj [] (pop path))))))))
 
 (rf/reg-event-db
@@ -174,6 +177,16 @@
                                    host sid frame uuid point))
          (assoc-in [:ui :selection] [:node host uuid [uuid]])))))
 
+(rf/reg-event-db
+ ::drop-sound
+ (fn [db [_ {:keys [source label length rate]} frame]]
+   (let [uuid (random-uuid)
+         host (get-in db [:ui :open])]
+     (-> db
+         (update :ui dissoc :drop)
+         (edit/edit #(clip/place-sound % host source label length rate frame uuid))
+         (assoc-in [:ui :selection] [:node host uuid [uuid]])))))
+
 ;; ---------------------------------------------------------------------------
 ;; moving rows between symbols
 ;;
@@ -195,7 +208,8 @@
        (-> db
            (edit/edit (constantly (:clip r)))
            (assoc-in [:ui :selection] [:node (:sid r) (:id r) (conj to (:id r))])
-           (update-in [:ui :expanded] into (rest (reductions conj [] to))))))))
+           (cond-> (not= :audio (get-in r [:clip :symbols (:sid r) :nodes (:id r) :kind]))
+             (update-in [:ui :expanded] into (rest (reductions conj [] to)))))))))
 
 (rf/reg-event-db
  ::sliding

frontend/src/arthur/subs/render.cljs

@@ -43,6 +43,17 @@
            (when c (gesture/apply-values c sid id frame values)))
          c))))
 
+(rf/reg-sub
+ ::sounds
+ :<- [::clip-id]
+ :<- [::paint-revision]
+ :<- [::open]
+ (fn [[id _ sid] _]
+   ;; What the open symbol plays, off the document as saved rather than `::clip`,
+   ;; so a bar being slid does not re-mix on every frame of the drag.
+   (let [c (:clip (footage/entry id))]
+     [sid (when (clip/symbol c sid) (nest/audio-tracks c sid))])))
+
 (rf/reg-sub
  ::symbol
  :<- [::clip]

frontend/src/arthur/subs/ui.cljs

@@ -86,3 +86,19 @@
                                 :when f]
                             f))]
      (filterv #(contains? used (:id %)) available))))
+
+(rf/reg-sub
+ ::project-sounds
+ :<- [::render/clip-id]
+ :<- [::render/paint-revision]
+ :<- [::playback/footage]
+ (fn [[id _ {:keys [sounds uploaded]}] _]
+   ;; As `::project-footage`: the sounds this document plays, and those uploaded
+   ;; while it was open.
+   (let [used (into (set uploaded)
+                    (for [[_ sym] (get-in (store/entry id) [:clip :symbols])
+                          [_ n] (:nodes sym)
+                          :let [s (get-in n [:source :sound])]
+                          :when s]
+                      s))]
+     (filterv #(contains? used (:id %)) sounds))))

frontend/src/arthur/ui/drag.cljs

@@ -52,19 +52,24 @@
   out of the pool, and not one that would make a cycle."
   []
   (let [c @carrying]
-    (and c (#{:symbol :footage :import} (:kind c)) (not (:refused? c)))))
+    (and c (#{:symbol :footage :import :sound} (:kind c)) (not (:refused? c)))))
 
 (defn row!
-  "Start carrying the timeline row at `path` — a node, to be moved into another
+  "Start carrying the timeline row at `path` — a node of kind `node-kind`, to be
-  symbol or grouped with another node."
+  moved into another symbol or grouped with another node."
-  [path]
+  [path node-kind]
-  (reset! carrying {:kind :row :path path}))
+  (reset! carrying {:kind :row :path path :node-kind node-kind}))
 
 (defn row
   "The path of the row being carried, or nil when it is not a row."
   []
   (let [c @carrying] (when (= :row (:kind c)) (:path c))))
 
+(defn row-kind
+  "The node kind of the row being carried."
+  []
+  (:node-kind @carrying))
+
 (defn other!
   "Start carrying something that is not yet in the document: `:kind` says what,
   and the rest is what a preview can show of it before it is fetched."
@@ -81,9 +86,10 @@
   "Say where the drag would land, for the previews. `point` is nil over the
   timeline."
   [where frame point]
-  (when-let [{:keys [label frames]} @carrying]
+  (when-let [{:keys [kind label frames]} @carrying]
     (rf/dispatch [::ui/drop-hover {:where where :frame frame :point point
-                                   :label label :frames frames}])))
+                                   :label label :frames frames
+                                   :sound? (= :sound kind)}])))
 
 (defn pos-for
   "Where the preview goes so the symbol's middle is under `point` — what
@@ -102,5 +108,7 @@
       ;; the symbol they become is called.
       :footage (rf/dispatch [::footage/ask-convert c frame point])
       :import  (rf/dispatch [::project/import c frame point])
+      ;; Where it is dropped in time; a sound has no place in space.
+      :sound   (rf/dispatch [::ui/drop-sound c frame])
       nil))
   (done!))

frontend/src/arthur/ui/params.cljs

@@ -229,7 +229,7 @@
            ^{:key (str path)}
            [:<>
             [:dt (str/join " " (map name path))]
-            (if (and (contains? node/defaults path) (not (:dense ch)))
+            (if (and (contains? (node/defaults-of n) path) (not (:dense ch)))
               [channel-control sid id path ch frame]
               [:dd (channel-state ch)])]))])]))
 

frontend/src/arthur/ui/pool.cljs

@@ -13,6 +13,9 @@
   one of another's, and `footage:<id>` for video. The stage and the timeline are
   where they are dropped.
 
+  A SOUND — mp3, wav — is `sound:<id>`, and is dropped on the timeline, where it
+  becomes an audio node in the open symbol from the frame it lands on.
+
   THE WHOLE PANE IS THE DROP TARGET for a file. A video dropped anywhere in it
   uploads and lands in THIS PROJECT's media, and goes no further: which frames of
   it become a symbol, and what that symbol is called, is asked when it is dropped
@@ -78,6 +81,21 @@
                          #(drag/other! {:kind :footage :id id :label label
                                         :frames frames :fps fps :video video})))])
 
+(defn- sound-row
+  "An uploaded sound, or with `:footage?` a video's own — which is how a take's
+  sound goes back into its symbol after it was deleted there."
+  [{:keys [id label duration footage? fps]} project-fps]
+  (let [frames (js/Math.ceil (* duration project-fps))
+        [source length rate] (if footage?
+                               [{:footage id} (js/Math.round (* duration fps)) (/ fps project-fps)]
+                               [{:sound id} frames 1])]
+    [item (merge {:label label
+                  :sub (str (.toFixed duration 1) "s · " frames "f")
+                  :thumb [:span.thumb.sound "♪"]}
+                 (carrying (str "sound:" id)
+                           #(drag/other! {:kind :sound :source source :label label
+                                          :length length :rate rate :frames frames})))]))
+
 (defn- folder [title & children]
   (into [:details.pool-folder {:open true} [:summary title]] children))
 
@@ -90,6 +108,12 @@
         selection @(rf/subscribe [::sub/selection])
         open      @(rf/subscribe [::render/open])
         media     @(rf/subscribe [::sub/project-footage])
+        ;; The project's videos' sounds first, then its uploaded ones.
+        sounds    (into (mapv (fn [{:keys [id label frames fps]}]
+                                {:id id :label label :footage? true :fps fps
+                                 :duration (/ frames fps)})
+                              media)
+                        @(rf/subscribe [::sub/project-sounds]))
         {:keys [chosen]} @(rf/subscribe [::playback/footage])]
     (folder "this project"
             (group "symbols"
@@ -109,10 +133,15 @@
             (group "media"
                    (if (empty? media)
                      [:div.dim "drop a video here"]
-                     (doall (for [f media] ^{:key (:id f)} [footage-row f chosen])))))))
+                     (doall (for [f media] ^{:key (:id f)} [footage-row f chosen]))))
+            (group "sounds"
+                   (if (empty? sounds)
+                     [:div.dim "drop an mp3 or wav here"]
+                     (doall (for [s sounds] ^{:key (:id s)} [sound-row s (:fps document)])))))))
 
 (defn- all-assets []
-  (let [{:keys [available chosen]} @(rf/subscribe [::playback/footage])
+  (let [{:keys [available chosen sounds]} @(rf/subscribe [::playback/footage])
+        fps (:fps @(rf/subscribe [::render/clip]))
         {:keys [symbols]} @(rf/subscribe [::project/assets])
         {:keys [id]} @(rf/subscribe [::playback/project])]
     (folder "all assets"
@@ -120,6 +149,10 @@
                    (if (empty? available)
                      [:div.dim "nothing uploaded yet"]
                      (doall (for [f available] ^{:key (:id f)} [footage-row f chosen]))))
+            (group "sounds"
+                   (if (empty? sounds)
+                     [:div.dim "no sounds uploaded yet"]
+                     (doall (for [s sounds] ^{:key (:id s)} [sound-row s fps]))))
             (group "symbols"
                    (let [others (remove #(= id (:project %)) symbols)]
                      (if (empty? others)
@@ -165,11 +198,11 @@
        [:div.pane-head
         "media pool"
         [:span.spacer]
-        [:button {:title "add a video"
+        [:button {:title "add a video or a sound"
                   :disabled loading?
                   :on-click #(.click (js/document.getElementById "pool-file"))}
          "+"]]
-       [:input {:id "pool-file" :type "file" :accept "video/*"
+       [:input {:id "pool-file" :type "file" :accept "video/*,audio/*"
                 :style {:display "none"}
                 :on-change (fn [^js event]
                              (when-let [file (aget (.. event -target -files) 0)]

frontend/src/arthur/ui/shell.cljs

@@ -9,6 +9,7 @@
             [arthur.ui.convert :as convert]
             [arthur.events.playback :as pb]
             [arthur.subs.playback :as playback]
+            [arthur.subs.render :as render]
             [arthur.ui.palette :as palette]
             [arthur.ui.params :as params]
             [arthur.ui.pool :as pool]
@@ -16,19 +17,31 @@
             [arthur.ui.tabs :as tabs]
             [arthur.ui.timeline :as timeline]
             [arthur.ui.topbar :as topbar]
-            [re-frame.core :as rf]))
+            [re-frame.core :as rf]
+            [reagent.core :as r]))
 
 (defn- audio []
-  [:audio
+  ;; An edit to what the open symbol plays — a sound dropped, moved, turned
-   {:ref     #(when % (clock/attach! %))
+  ;; down, undone — re-mixes the clock. Opening a symbol mixes on its own, so
-    :src     @(rf/subscribe [::playback/audio])
+  ;; only a change under the same one counts.
-    :preload "auto"
+  (r/with-let [heard (atom nil)
-    ;; Transport state follows the ELEMENT, not the other way round: the audio is
+               remix (r/track! (fn []
-    ;; the clock, so anything that can change its state — the end of the file, the
+                             (let [[sid :as now] @(rf/subscribe [::render/sounds])
-    ;; OS media keys, a browser autoplay block — has to be able to correct the
+                                   [was :as before] @heard]
-    ;; document rather than be contradicted by it.
+                               (reset! heard now)
-    :on-play  #(rf/dispatch [::pb/play])
+                               (when (and before (= sid was) (not= now before))
-    :on-pause #(rf/dispatch [::pb/pause])}])
+                                 (rf/dispatch [::pb/refresh-clock])))))]
+    [:audio
+     {:ref     #(when % (clock/attach! %))
+      :src     @(rf/subscribe [::playback/audio])
+      :preload "auto"
+      ;; Transport state follows the ELEMENT, not the other way round: the audio is
+      ;; the clock, so anything that can change its state — the end of the file, the
+      ;; OS media keys, a browser autoplay block — has to be able to correct the
+      ;; document rather than be contradicted by it.
+      :on-play  #(rf/dispatch [::pb/play])
+      :on-pause #(rf/dispatch [::pb/pause])}]
+    (finally (r/dispose! remix))))
 
 (defn view []
   [:div.app

frontend/src/arthur/ui/timeline.cljs

@@ -1,5 +1,6 @@
 (ns arthur.ui.timeline
-  "The bottom pane: the transport, a ruler, and a row per node of the open symbol.
+  "The bottom pane: the transport, a ruler, and a row per node of the open symbol
+  — the picture's, then, under an `audio` heading, every sound it plays.
 
   `rows` is the whole of the interesting part and it is a PURE function of the
   clip, the open symbol and the set of open paths. It flattens the document's two
@@ -87,7 +88,10 @@
                                ;; everywhere. `:z` is the lexicographic draw key;
                                ;; the id breaks ties so the order is stable.
                                (sort-by (fn [[id n]] [(or (:z n) "") (str id)]))
-                               reverse)]
+                               reverse
+                               ;; Sounds are listed below the picture, by
+                               ;; `sound-rows`, wherever they are.
+                               (remove #(= :audio (:kind (val %)))))]
               (into []
                     (mapcat
                      (fn [[id n]]
@@ -133,6 +137,50 @@
       (walk sid [] 0 identity)
       [])))
 
+(defn sound-rows
+  "Every sound symbol `sid` plays, one row each, below the picture as an
+  editor's audio tracks are: its own, and those inside what it places at any
+  depth — each TIED to the placement it is heard through, `:via`, and drawn
+  where it is heard, cut to that placement's span as the mix cuts it."
+  [clip sid expanded]
+  (letfn [(walk [sid path ->open [lo hi] via]
+            (mapcat
+             (fn [[id n]]
+               (let [rpath (conj path id)
+                     self  (comp ->open (local->parent n))
+                     [a b] (mapv ->open (or (node/placed-span
+                                             (cond-> n
+                                               (and (= :instance (:kind n)) (nil? (:span n)))
+                                               (assoc :span [0 (get-in clip [:symbols (:of n) :frames])])))
+                                            [0 (get-in clip [:symbols sid :frames])]))
+                     span  [(max lo a) (min hi b)]
+                     open? (contains? expanded rpath)]
+                 (case (:kind n)
+                   :audio    (cons {:path rpath
+                                    :depth 0
+                                    :label (node-label id n)
+                                    :kind :node
+                                    :node-kind :audio
+                                    :via via
+                                    ;; A sound heard through a placement is
+                                    ;; that placement's picture's sound: its bar
+                                    ;; moves the placement, so the two stay in
+                                    ;; sync. Moving the placement moves it too.
+                                    :slides (if via (subvec rpath 0 1) rpath)
+                                    :select [:node sid id rpath]
+                                    :expandable? true
+                                    :expanded? open?
+                                    :span span
+                                    :keys (into [] (comp (mapcat keyed-frames) (map self) (distinct))
+                                                (vals (node/channels n)))}
+                                   (when open? (channel-rows n rpath 1 self span)))
+                   :instance (walk (:of n) rpath self span (or via (node-label id n)))
+                   nil)))
+             (sort-by (fn [[id n]] [(or (:z n) "") (str id)]) (get-in clip [:symbols sid :nodes]))))]
+    (if (get-in clip [:symbols sid])
+      (vec (walk sid [] identity [##-Inf ##Inf] nil))
+      [])))
+
 ;; ---------------------------------------------------------------------------
 ;; geometry
 ;;
@@ -189,19 +237,26 @@
            (when (and drop (pos? drop)) (str " · " (.toFixed drop 2) " f/paint")))]]))
 
 (defn- takes?
-  "Whether the row at `target` can take the row being carried: not itself, and
+  "Whether the row at `target`, a `target-kind` node, can take the row being
-  not anything inside it."
+  carried: not itself, and not anything inside it. A sound goes into a
-  [target]
+  placement or beside another sound, and only a sound goes beside a sound."
+  [target target-kind]
   (when-let [from (drag/row)]
-    (not= from (subvec target 0 (min (count from) (count target))))))
+    (and (not= from (subvec target 0 (min (count from) (count target))))
+         (if (= :audio (drag/row-kind))
+           (#{:audio :instance} target-kind)
+           (not= :audio target-kind)))))
 
 (defn- zone
   "Which part of a row the pointer is over: its top edge, to go in front of it;
-  its bottom edge, to go behind; its middle, to go into it or be grouped with it."
+  its bottom edge, to go behind; its middle, to go into it or be grouped with it.
-  [^js e]
+  Nothing goes into a sound, so its middle is behind it."
+  [^js e node-kind]
   (let [box (.getBoundingClientRect (.-currentTarget e))
         y   (/ (- (.-clientY e) (.-top box)) (max 1 (.-height box)))]
-    (cond (< y 0.3) :front (> y 0.7) :back :else :into)))
+    (cond (< y 0.3) :front
+          (or (> y 0.7) (= :audio node-kind)) :back
+          :else :into)))
 
 (def ^:private reveal
   "A ref that scrolls the selected row into view, ONE per selection: React calls
@@ -210,7 +265,7 @@
   on every render after, which would fight a person scrolling away."
   (memoize (fn [_selection] (fn [el] (some-> el (.scrollIntoView #js {:block "nearest"}))))))
 
-(defn- label-cell [{:keys [path depth label kind node-kind select expandable? expanded? of]}
+(defn- label-cell [{:keys [path depth label kind node-kind select expandable? expanded? of via]}
                    selection over solo]
   (let [node? (= :node kind)
         [over-path where] @over]
@@ -237,21 +292,21 @@
                                            (.stopPropagation e)
                                            (.setData (.-dataTransfer e) "text/plain" "row")
                                            (set! (.. e -dataTransfer -effectAllowed) "move")
-                                           (drag/row! path))
+                                           (drag/row! path node-kind))
                           :on-drag-end (fn [_] (reset! over nil) (drag/done!))
-                          :on-drag-enter (fn [^js e] (when (takes? path) (.preventDefault e)))
+                          :on-drag-enter (fn [^js e] (when (takes? path node-kind) (.preventDefault e)))
                           :on-drag-over (fn [^js e]
-                                          (when (takes? path)
+                                          (when (takes? path node-kind)
                                             (.preventDefault e)
                                             (.stopPropagation e)
                                             (set! (.. e -dataTransfer -dropEffect) "move")
-                                            (let [o [path (zone e)]]
+                                            (let [o [path (zone e node-kind)]]
                                               (when (not= o @over) (reset! over o)))))
                           :on-drop (fn [^js e]
                                      (.preventDefault e)
                                      (.stopPropagation e)
-                                     (let [from  (when (takes? path) (drag/row))
+                                     (let [from  (when (takes? path node-kind) (drag/row))
-                                           where (zone e)]
+                                           where (zone e node-kind)]
                                        (reset! over nil)
                                        (drag/done!)
                                        (when from
@@ -267,7 +322,7 @@
                    (rf/dispatch [::ui/toggle-row path]))}
       (when expandable? (if expanded? "▾" "▸"))]
      [:span.name label]
-     (when node? [:span.kind (str "·" (name node-kind))])
+     (when node? [:span.kind (if via (str "· in " via) (str "·" (name node-kind)))])
      (when (= :instance node-kind)
        [:button {:class    (str "tl-solo" (when (contains? solo path) " on"))
                  :title    "show only this on the stage (⇧ for more than one)"
@@ -286,19 +341,19 @@
   "`sliding` is the pointer's side of a bar being dragged, `{:path :x :width
   :df}`. What it looks like mid-drag is `[:ui :sliding]`, which the clip every
   row and the stage are drawn from already has in it."
-  [{:keys [path span keys dense? kind select]} frames sliding]
+  [{:keys [path span keys dense? kind node-kind select slides]} frames sliding]
   (let [{from :path x0 :x width :width} @sliding
         slide   (fn [^js e]
                   (when (= path from)
                     (let [df (js/Math.round (/ (* frames (- (.-clientX e) x0)) (max 1 width)))]
                       (when (not= df (:df @sliding))
                         (swap! sliding assoc :df df)
-                        (rf/dispatch [::ui/sliding path df])))))
+                        (rf/dispatch [::ui/sliding (or slides path) df])))))
         done    (fn [commit?]
                   (when (= path from)
                     (let [df (:df @sliding)]
                       (reset! sliding nil)
-                      (rf/dispatch (if commit? [::ui/slide path df] [::ui/sliding nil])))))]
+                      (rf/dispatch (if commit? [::ui/slide (or slides path) df] [::ui/sliding nil])))))]
     [:div.tl-track
      ;; The track, not the bar, holds the pointer while a bar slides, so the drag
      ;; goes on when the bar has slid off the ruler and is no longer drawn.
@@ -310,6 +365,7 @@
      (when-let [[in out] (when span [(max 0 (first span)) (min frames (second span))])]
        (when (< in out)
          [:div {:class (str "tl-span" (when dense? " dense") (when (= :ghost kind) " ghost")
+                            (when (= :audio node-kind) " sound")
                             (when select " movable") (when (= path from) " sliding"))
                 :style {:left (edge% in frames)
                         :width (str (* 100 (/ (- out in) (max 1 frames))) "%")}
@@ -345,14 +401,24 @@
           expanded  @(rf/subscribe [::sub/expanded])
           drop      @(rf/subscribe [::sub/drop])
           solo      (set @(rf/subscribe [::render/solo]))
+          open      @(rf/subscribe [::render/open])
           ;; Where a drag out of the pool would land, as a row of its own at the
-          ;; top: its own length, starting on the frame it would start on. The
+          ;; top of its section: its own length, starting on the frame it would
-          ;; stage's drop shows it too, at the playhead.
+          ;; start on. The stage's drop shows it too, at the playhead.
-          visible   (cond->> (rows clip @(rf/subscribe [::render/open]) expanded)
+          ghost     (when drop
-                      drop (cons {:path [::drop] :depth 0 :kind :ghost
+                      {:path [::drop] :depth 0 :kind :ghost
-                                  :label (str "+ " (:label drop))
+                       :label (str "+ " (:label drop))
-                                  :span [(:frame drop) (+ (:frame drop) (or (:frames drop) 1))]
+                       :span [(:frame drop) (+ (:frame drop) (or (:frames drop) 1))]
-                                  :keys []}))
+                       :keys []})
+          picture   (cond->> (rows clip open expanded)
+                      (and ghost (not (:sound? drop))) (cons ghost))
+          sounds    (cond->> (sound-rows clip open expanded)
+                      (and ghost (:sound? drop)) (cons ghost))
+          ;; The audio section's heading is a row like the others, so the two
+          ;; columns stay aligned without measuring anything.
+          visible   (cond-> (vec picture)
+                      (seq sounds) (-> (conj {:path [::sounds] :kind :section :label "audio"})
+                                       (into sounds)))
           ;; Roughly ten labels, on a round number of frames.
           step      (* 10 (js/Math.ceil (/ frames 100)))]
       [:section.pane.time
@@ -373,7 +439,10 @@
                          (rf/dispatch [::ui/move-node from []]))))}
          [:div.tl-corner]
          (doall (for [row visible]
-                  ^{:key (str (:path row))} [label-cell row selection over solo]))]
+                  (with-meta (if (= :section (:kind row))
+                               [:div.tl-label.tl-section (:label row)]
+                               [label-cell row selection over solo])
+                    {:key (str (:path row))})))]
         [:div.tl-tracks
          {:on-drag-enter (fn [^js e] (when (drag/accepts?) (.preventDefault e)))
           :on-drag-over (fn [^js e]
@@ -419,6 +488,9 @@
           [:div.tl-knob {:style {:left (at% frame frames)}}]]
          (if (seq visible)
            (doall (for [row visible]
-                    ^{:key (str (:path row))} [track-cell row frames sliding]))
+                    (with-meta (if (= :section (:kind row))
+                                 [:div.tl-track.tl-section]
+                                 [track-cell row frames sliding])
+                      {:key (str (:path row))})))
            [:div.tl-empty "nothing in this symbol"])
          [:div.tl-playhead {:style {:left (at% frame frames)}}]]]])))

frontend/test/arthur/domain/nest_test.cljs

@@ -238,3 +238,26 @@
         "and nothing else is renumbered")
     (is (:refused (nest/restack c :main [:a] [a-uuid :inner] true))
         "only among the things it is beside")))
+
+(deftest a-sound-dropped-in-a-placed-symbol-is-heard-where-the-placement-puts-it
+  (let [u #uuid "00000000-0000-4000-8000-0000000000a1"
+        c (-> (nested)
+              (clip/place-sound :inner {:sound "tone"} "tone.mp3" 40 1 2 u))
+        sound (get-in c [:symbols :inner :nodes u])
+        [heard] (nest/audio-tracks c :outer)]
+    (is (empty? (node/problems sound)) "a sound needs no transform")
+    (is (= {:sound "tone"} (:source sound)))
+    (is (= #{[:audio :gain] [:audio :pan] [:audio :rate]} (set (keys (node/channels sound)))))
+    (is (= [0 8] (:span heard))
+        "own frames 0-8: it starts on inner's 2 and inner ends on 10")
+    (is (= [7 15] (node/placed-span heard)) "inner starts on 5 of outer")
+    (is (empty? ((clip/resolver c nil pal/index-of :inner) 3))
+        "and it draws nothing")
+    (is (= c (clip/place-sound c :inner {:sound "tone"} "tone.mp3" 40 1 10 (random-uuid)))
+        "nor lands past the end of its symbol")
+    (testing "a video's own sound keeps its length at another rate"
+      (let [v (-> (nested)
+                  (clip/place-sound :outer {:footage "f"} "take" 90 2.5 0 u)
+                  (get-in [:symbols :outer :nodes u]))]
+        (is (empty? (node/problems v)))
+        (is (= [0 36] (node/placed-span v)) "90 frames at 30 are 36 at 12")))))

frontend/test/arthur/domain/node_test.cljs

@@ -157,8 +157,8 @@
 (deftest skew-and-anchor-are-in-the-shape-although-nothing-drives-them
   ;; A decomposition is not extensible after the fact: adding a component later
   ;; means migrating every stored transform. So both are present from the start,
-  ;; on every kind.
+  ;; on every kind that is in the picture.
-  (doseq [k node/implemented-kinds]
+  (doseq [k (disj node/implemented-kinds :audio)]
     (is (contains? (get node/valid-paths k) [:xform :skew]) (str k))
     (is (contains? (get node/valid-paths k) [:xform :anchor]) (str k))))
 
@@ -168,7 +168,13 @@
   (is (contains? (:disc node/valid-paths) [:geom :radius]))
   (is (contains? (:rect node/valid-paths) [:geom :size]))
   (is (not (contains? (:group node/valid-paths) [:geom :pts]))
-      "a group is a pure transform node"))
+      "a group is a pure transform node")
+  (is (= #{[:audio :gain] [:audio :pan] [:audio :rate]} (:audio node/valid-paths))
+      "a sound has no transform"))
+
+(deftest a-sound-defaults-to-its-own-channels
+  (is (= (set (keys node/audio-defaults))
+         (set (keys (node/channels {:id :s :kind :audio}))))))
 
 (deftest problems-names-the-ways-a-node-is-malformed
   (is (empty? (node/problems {:id :x :kind :group :z "a1"})))

static/arthur/app.css

@@ -684,6 +684,16 @@ button.share-button:hover, button.share-button.on { filter: brightness(1.1); }
   line-height: var(--ruler);
 }
 
+/* The heading between the picture's rows and the sounds', as an editor puts
+   its audio tracks under its video tracks. */
+.tl-label.tl-section,
+.tl-track.tl-section { background: var(--chrome); border-top: 1px solid var(--line); }
+.tl-label.tl-section { padding-left: 6px; color: var(--dim); font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; }
+
+/* A sound's bar, told apart from the picture's at a glance. */
+.tl-span.sound { background: #e3efdf; border-color: #a6c49b; }
+.pool-item .thumb.sound { display: flex; align-items: center; justify-content: center; color: #d9d9d9; }
+
 /* The span the node exists over. Drawn under the keys so a dot on the first
    frame of a span is not half-hidden by its own bar. */
 .tl-span {