Media · Capture and devices: recorder, camera, mic meter, device choice and 3D model viewer

ScreenRecorderControls holds a recording in one pill: Record, then a live
dot and the time it has run beside pause or resume and stop, with mic and
camera toggles. CameraPreview shows host frames in the camera's shape, or
the camera off. MicLevelMeter lights segments up to the level, amber and
red near the top, with tones shared from data_display. DeviceSelector
picks a microphone, camera or speaker through Select, which may now be
empty while shut. ModelViewer shows the host's frame for an orbit that a
drag, the arrows, Command-scroll, + and - and a reset move; the orbit is
seeded from the host so fast drags add up. The gallery draws a stone
torus per orbit and frees each replaced frame.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

examples/gallery/pages/media/audio.rs

@@ -8,7 +8,7 @@ use gpui::{App, Entity, IntoElement, ParentElement, SharedString, Styled, Window
 
 use crate::{
     probe::probe,
-    ui::{keep, noise, section},
+    ui::{change, keep, noise, section},
 };
 
 const COVER: &str = concat!(
@@ -139,14 +139,6 @@ fn state(window: &mut Window, cx: &mut App) -> Entity<Listening> {
     )
 }
 
-/// Changes the demo's listening and redraws.
-fn change(state: &Entity<Listening>, cx: &mut App, edit: impl FnOnce(&mut Listening)) {
-    state.update(cx, |now, cx| {
-        edit(now);
-        cx.notify();
-    });
-}
-
 fn controls(id: &'static str, state: &Entity<Listening>, now: &Listening) -> MediaControls {
     let (play, back, forward, shuffle, repeat) = (
         state.clone(),

examples/gallery/pages/media/devices.rs

@@ -0,0 +1,244 @@
+use std::{path::Path, sync::Arc};
+
+use ely_gpui_component::{
+    media::{
+        CameraPreview, Device, DeviceKind, DeviceSelector, MicLevelMeter, ModelViewer, Orbit,
+        Recording, ScreenRecorderControls,
+    },
+    theme::{ActiveTheme, TextSize},
+};
+use gpui::{
+    App, ImageSource, IntoElement, ParentElement, RenderImage, SharedString, Styled, Window, div,
+    px,
+};
+
+use super::sculpt;
+use crate::{
+    probe::probe,
+    ui::{change, keep, section},
+};
+
+const OLIVE: &str = concat!(
+    env!("CARGO_MANIFEST_DIR"),
+    "/examples/gallery/assets/atrium-olive.jpg"
+);
+/// The model's frame, in pixels: twice the viewer's points.
+const FRAME: (u32, u32) = (960, 640);
+
+/// The demo recording and what it takes in.
+#[derive(Clone)]
+struct Recorder {
+    state: Recording,
+    mic: bool,
+    camera: bool,
+}
+
+pub fn recorder(window: &mut Window, cx: &mut App) -> impl IntoElement + use<> {
+    let state = keep(
+        "media-recorder",
+        || Recorder {
+            state: Recording::Idle,
+            mic: true,
+            camera: false,
+        },
+        window,
+        cx,
+    );
+    let now = state.read(cx).clone();
+    let (record, pause, resume, stop, mic, camera) = (
+        state.clone(),
+        state.clone(),
+        state.clone(),
+        state.clone(),
+        state.clone(),
+        state.clone(),
+    );
+    let controls = ScreenRecorderControls::new("media-recorder", now.state)
+        .on_record(move |_, cx| {
+            let since = cx.background_executor().now();
+            change(&record, cx, |it| it.state = Recording::Running { since })
+        })
+        .on_pause(move |_, cx| {
+            let at = cx.background_executor().now();
+            change(&pause, cx, |it| {
+                if let Recording::Running { since } = it.state {
+                    it.state = Recording::Paused { since, at };
+                }
+            })
+        })
+        .on_resume(move |_, cx| {
+            let now = cx.background_executor().now();
+            change(&resume, cx, |it| {
+                if let Recording::Paused { since, at } = it.state {
+                    it.state = Recording::Running {
+                        since: since + (now - at),
+                    };
+                }
+            })
+        })
+        .on_stop(move |_, cx| change(&stop, cx, |it| it.state = Recording::Idle))
+        .mic(now.mic, move |on, _, cx| change(&mic, cx, |it| it.mic = on))
+        .camera(now.camera, move |on, _, cx| {
+            change(&camera, cx, |it| it.camera = on)
+        });
+    section(
+        "ScreenRecorderControls",
+        "Record, then pause, resume or stop. While it runs, a live dot and the time it has run; the mic and camera toggles choose what it takes.",
+        cx,
+    )
+    .child(probe("media-recorder", div().flex().child(controls)))
+}
+
+pub fn camera(cx: &mut App) -> impl IntoElement + use<> {
+    let preview = |key: &'static str| div().w(px(300.)).child(CameraPreview::new(key, 1.5));
+    section(
+        "CameraPreview",
+        "What a camera sees, in its own shape, from frames the host hands in. With no frame, the camera is off.",
+        cx,
+    )
+    .child(probe(
+        "media-camera",
+        div()
+            .flex()
+            .flex_wrap()
+            .gap_4()
+            .child(
+                div().w(px(300.)).child(
+                    CameraPreview::new("media-camera-on", 1.5)
+                        .frame(Path::new(OLIVE))
+                        .name("Studio camera"),
+                ),
+            )
+            .child(preview("media-camera-off")),
+    ))
+}
+
+pub fn mic(cx: &mut App) -> impl IntoElement + use<> {
+    let theme = cx.theme();
+    let (small, muted) = (theme.text_size(TextSize::Sm), theme.colors.fg_muted);
+    let row = |label: &'static str, level: f32| {
+        div()
+            .flex()
+            .items_center()
+            .gap_3()
+            .child(
+                div()
+                    .w(px(80.))
+                    .text_size(small)
+                    .text_color(muted)
+                    .child(label),
+            )
+            .child(div().flex_1().child(MicLevelMeter::new(level)))
+    };
+    section(
+        "MicLevelMeter",
+        "A microphone's level as it comes in: amber near the top, red at the top.",
+        cx,
+    )
+    .child(probe(
+        "media-mic",
+        div()
+            .w(px(360.))
+            .flex()
+            .flex_col()
+            .gap_3()
+            .child(row("Quiet", 0.3))
+            .child(row("Speaking", 0.84))
+            .child(row("Too loud", 0.98)),
+    ))
+}
+
+fn device(key: &str, name: &str) -> Device {
+    Device {
+        key: SharedString::from(key.to_string()),
+        name: SharedString::from(name.to_string()),
+    }
+}
+
+pub fn selectors(window: &mut Window, cx: &mut App) -> impl IntoElement + use<> {
+    let chosen = keep(
+        "media-devices",
+        || {
+            (
+                SharedString::from("mic-desk"),
+                SharedString::from("cam-studio"),
+            )
+        },
+        window,
+        cx,
+    );
+    let (mic_now, cam_now) = chosen.read(cx).clone();
+    let (mic, cam) = (chosen.clone(), chosen);
+    let mics = [
+        device("mic-desk", "Desk microphone"),
+        device("mic-headset", "Headset"),
+    ];
+    section(
+        "DeviceSelector",
+        "A microphone, a camera or a speaker to use, each with its kind's icon. With none attached, it says so and stays shut.",
+        cx,
+    )
+    .child(probe(
+        "media-devices",
+        div()
+            .w(px(280.))
+            .flex()
+            .flex_col()
+            .gap_3()
+            .child(
+                DeviceSelector::new("media-mics", DeviceKind::Microphone, mics)
+                    .chosen(mic_now)
+                    .on_change(move |key, _, cx| change(&mic, cx, |it| it.0 = key.clone())),
+            )
+            .child(
+                DeviceSelector::new("media-cams", DeviceKind::Camera, [device("cam-studio", "Studio camera")])
+                    .chosen(cam_now)
+                    .on_change(move |key, _, cx| change(&cam, cx, |it| it.1 = key.clone())),
+            )
+            .child(DeviceSelector::new("media-speakers", DeviceKind::Speaker, [])),
+    ))
+}
+
+/// The demo model: its orbit and the frame drawn for it.
+struct Model {
+    orbit: Orbit,
+    frame: Arc<RenderImage>,
+}
+
+fn drawn(orbit: Orbit) -> Arc<RenderImage> {
+    Arc::new(RenderImage::new(vec![sculpt::draw(
+        orbit, FRAME.0, FRAME.1,
+    )]))
+}
+
+pub fn model(window: &mut Window, cx: &mut App) -> impl IntoElement + use<> {
+    let state = keep(
+        "media-model",
+        || Model {
+            orbit: Orbit::START,
+            frame: drawn(Orbit::START),
+        },
+        window,
+        cx,
+    );
+    let (orbit, frame) = {
+        let now = state.read(cx);
+        (now.orbit, now.frame.clone())
+    };
+    let viewer = ModelViewer::new("media-model", 1.5, orbit)
+        .frame(ImageSource::Render(frame))
+        .on_orbit(move |orbit, window, cx| {
+            let old = state.update(cx, |model, cx| {
+                model.orbit = orbit;
+                cx.notify();
+                std::mem::replace(&mut model.frame, drawn(orbit))
+            });
+            cx.drop_image(old, Some(window));
+        });
+    section(
+        "3DModelViewer",
+        "A model the host renders, a frame for each view. A drag or the arrows turn around it; Command-scroll, + and - bring it nearer; 0 or the reset button returns.",
+        cx,
+    )
+    .child(probe("media-model", div().w(px(480.)).child(viewer)))
+}

examples/gallery/pages/media/mod.rs

@@ -1,6 +1,8 @@
 mod audio;
+mod devices;
 mod edit;
 mod images;
+mod sculpt;
 mod video;
 
 use gpui::{AnyElement, App, IntoElement, ParentElement, Window, div};
@@ -89,6 +91,21 @@ const SCRIPT: &[Step] = &[
     Step::DownAt("media-transport", 180.0, 20.0),
     Step::UpAt("media-transport", 180.0, 20.0),
     Step::Wait(300),
+    Step::DownAt("media-recorder", 45.0, 18.0),
+    Step::UpAt("media-recorder", 45.0, 18.0),
+    Step::Wait(1500),
+    Step::Shot("recording"),
+    Step::DownAt("media-devices", 140.0, 18.0),
+    Step::UpAt("media-devices", 140.0, 18.0),
+    Step::Wait(300),
+    Step::Shot("devices"),
+    Step::Key("escape"),
+    Step::DownAt("media-model", 240.0, 160.0),
+    Step::DragTo("media-model", 360.0, 200.0),
+    Step::UpAt("media-model", 360.0, 200.0),
+    Step::Key("="),
+    Step::Wait(300),
+    Step::Shot("orbited"),
 ];
 
 fn render(window: &mut Window, cx: &mut App) -> AnyElement {
@@ -109,5 +126,10 @@ fn render(window: &mut Window, cx: &mut App) -> AnyElement {
         .child(audio::speed(window, cx))
         .child(audio::transport(window, cx))
         .child(audio::playlist(window, cx))
+        .child(devices::recorder(window, cx))
+        .child(devices::camera(cx))
+        .child(devices::mic(cx))
+        .child(devices::selectors(window, cx))
+        .child(devices::model(window, cx))
         .into_any_element()
 }

examples/gallery/pages/media/sculpt.rs

@@ -0,0 +1,135 @@
+use std::f32::consts::TAU;
+
+use ely_gpui_component::media::Orbit;
+use image::{Frame, RgbaImage};
+
+type Vec3 = [f32; 3];
+
+/// The ring's radius, its tube's, and the steps around each.
+const RING: f32 = 1.0;
+const TUBE: f32 = 0.38;
+const AROUND: usize = 56;
+const ACROSS: usize = 24;
+/// Half the camera's field of view, and how far it sits when the torus just fits.
+const HALF_VIEW: f32 = 0.35;
+const FIT: f32 = 1.55 / 0.35;
+/// A matte stone, lit from the upper left.
+const STONE: Vec3 = [196.0, 186.0, 172.0];
+
+fn sub(a: Vec3, b: Vec3) -> Vec3 {
+    [a[0] - b[0], a[1] - b[1], a[2] - b[2]]
+}
+
+fn dot(a: Vec3, b: Vec3) -> f32 {
+    a[0] * b[0] + a[1] * b[1] + a[2] * b[2]
+}
+
+fn cross(a: Vec3, b: Vec3) -> Vec3 {
+    [
+        a[1] * b[2] - a[2] * b[1],
+        a[2] * b[0] - a[0] * b[2],
+        a[0] * b[1] - a[1] * b[0],
+    ]
+}
+
+fn unit(a: Vec3) -> Vec3 {
+    let length = dot(a, a).sqrt();
+    [a[0] / length, a[1] / length, a[2] / length]
+}
+
+/// A point on the torus, `u` around the ring and `v` around the tube.
+fn point(u: f32, v: f32) -> Vec3 {
+    let reach = RING + TUBE * v.cos();
+    [reach * u.cos(), TUBE * v.sin(), reach * u.sin()]
+}
+
+/// The torus as seen from `orbit`, `width` by `height` pixels in BGRA on a clear ground.
+pub fn draw(orbit: Orbit, width: u32, height: u32) -> Frame {
+    let away = FIT * orbit.distance;
+    let eye = [
+        away * orbit.pitch.cos() * orbit.yaw.sin(),
+        away * orbit.pitch.sin(),
+        away * orbit.pitch.cos() * orbit.yaw.cos(),
+    ];
+    let ahead = unit(sub([0.0; 3], eye));
+    let right = unit(cross(ahead, [0.0, 1.0, 0.0]));
+    let up = cross(right, ahead);
+    let light = unit([
+        -right[0] + 1.6 * up[0] - ahead[0],
+        -right[1] + 1.6 * up[1] - ahead[1],
+        -right[2] + 1.6 * up[2] - ahead[2],
+    ]);
+    let focal = height as f32 / 2.0 / HALF_VIEW.tan();
+    let project = |p: Vec3| {
+        let seen = sub(p, eye);
+        let depth = dot(seen, ahead);
+        (
+            width as f32 / 2.0 + dot(seen, right) / depth * focal,
+            height as f32 / 2.0 - dot(seen, up) / depth * focal,
+            depth,
+        )
+    };
+    let step = |ix: usize, count: usize| ix as f32 / count as f32 * TAU;
+    let mut faces = Vec::with_capacity(AROUND * ACROSS);
+    for i in 0..AROUND {
+        for j in 0..ACROSS {
+            let corners = [
+                point(step(i, AROUND), step(j, ACROSS)),
+                point(step(i + 1, AROUND), step(j, ACROSS)),
+                point(step(i + 1, AROUND), step(j + 1, ACROSS)),
+                point(step(i, AROUND), step(j + 1, ACROSS)),
+            ];
+            let normal = unit(cross(
+                sub(corners[3], corners[0]),
+                sub(corners[1], corners[0]),
+            ));
+            if dot(normal, sub(eye, corners[0])) <= 0.0 {
+                continue;
+            }
+            let lit = 0.42 + 0.58 * dot(normal, light).max(0.0);
+            let shade = STONE.map(|channel| (channel * lit).min(255.0) as u8);
+            let placed = corners.map(project);
+            let depth = placed.iter().map(|corner| corner.2).sum::<f32>();
+            faces.push((depth, placed.map(|(x, y, _)| (x, y)), shade));
+        }
+    }
+    faces.sort_by(|a, b| b.0.total_cmp(&a.0));
+    let mut pixels = RgbaImage::new(width, height);
+    for (_, [a, b, c, d], [red, green, blue]) in faces {
+        for triangle in [[a, b, c], [a, c, d]] {
+            fill(&mut pixels, triangle, [blue, green, red, 255]);
+        }
+    }
+    Frame::new(pixels)
+}
+
+/// Fills a triangle given either way round.
+fn fill(pixels: &mut RgbaImage, [a, b, c]: [(f32, f32); 3], color: [u8; 4]) {
+    let edge = |p: (f32, f32), q: (f32, f32), x: f32, y: f32| {
+        (q.0 - p.0) * (y - p.1) - (q.1 - p.1) * (x - p.0)
+    };
+    let area = edge(a, b, c.0, c.1);
+    if area.abs() < f32::EPSILON {
+        return;
+    }
+    let (width, height) = (pixels.width() as f32, pixels.height() as f32);
+    let low_x = a.0.min(b.0).min(c.0).max(0.0) as u32;
+    let high_x = a.0.max(b.0).max(c.0).min(width - 1.0).max(0.0) as u32;
+    let low_y = a.1.min(b.1).min(c.1).max(0.0) as u32;
+    let high_y = a.1.max(b.1).max(c.1).min(height - 1.0).max(0.0) as u32;
+    for y in low_y..=high_y {
+        for x in low_x..=high_x {
+            let (x_at, y_at) = (x as f32 + 0.5, y as f32 + 0.5);
+            let inside = [
+                edge(a, b, x_at, y_at),
+                edge(b, c, x_at, y_at),
+                edge(c, a, x_at, y_at),
+            ]
+            .iter()
+            .all(|side| side * area >= 0.0);
+            if inside {
+                pixels.put_pixel(x, y, image::Rgba(color));
+            }
+        }
+    }
+}

examples/gallery/ui.rs

@@ -87,6 +87,14 @@ pub fn keep<T: 'static>(
     window.use_keyed_state(key, cx, move |_, _| init())
 }
 
+/// Edits a demo's state in place and redraws.
+pub fn change<T: 'static>(state: &Entity<T>, cx: &mut App, edit: impl FnOnce(&mut T)) {
+    state.update(cx, |state, cx| {
+        edit(state);
+        cx.notify();
+    });
+}
+
 /// Replaces a demo's state and redraws.
 pub fn set<T: 'static>(state: &Entity<T>, value: T, cx: &mut App) {
     state.update(cx, |state, cx| {