T25d Generative: sound and motion

AudioGenerationPlayer shows a generated sound's prompt, model and
length, a bar while it is made, then chat::AudioMessage to play it;
Regenerate and Download wait for it. TTSVoicePicker lists voices with a
sample to play, the chosen one washed with a check, tags dropping below
when narrow. VideoGenerationTimeline lays shots along their time under
a ruler, with posters, placeholders, a playhead and Extend.

AudioMessage's waveform is one Tab stop where Left and Right seek, and
it gives way at 280px. debug::ruler serves the timeline's ruler. A mask
shown to read no longer follows the pointer.
This commit is contained in:
ZacharyZhang-NY committed 2026-09-26 13:16:42 -04:00
1 parent 6bfd53baee
commit 2266b4e367
16 files changed
+1198 -19

No files matched your search

+20
View File
@@ -0,0 +1,20 @@
<!-- @license lucide-static v1.48.0 - ISC -->
<svg
class="lucide lucide-audio-lines"
xmlns="http://www.w3.org/2000/svg"
width="24"
height="24"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
>
<path d="M2 10v3" />
<path d="M6 6v11" />
<path d="M10 3v18" />
<path d="M14 8v7" />
<path d="M18 5v13" />
<path d="M22 10v3" />
</svg>

After

Width:  |  Height:  |  Size: 431 B

+4
View File
@@ -1,6 +1,7 @@
mod canvas;
mod panel;
mod results;
mod sound;
use gpui::{AnyElement, App, IntoElement, ParentElement, Window, div};
@@ -43,5 +44,8 @@ fn render(window: &mut Window, cx: &mut App) -> AnyElement {
.child(results::results(window, cx))
.child(results::variations(window, cx))
.child(canvas::canvas(window, cx))
.child(sound::audio(window, cx))
.child(sound::voice(window, cx))
.child(sound::video(window, cx))
.into_any_element()
}
+189
View File
@@ -0,0 +1,189 @@
use std::time::Duration;
use ely_gpui_component::generative::{
AudioGenerationPlayer, Outcome, Shot, TTSVoicePicker, VideoGenerationTimeline, Voice,
};
use gpui::{App, IntoElement, ParentElement, SharedString, Styled, Window, div, px};
use crate::{
probe::probe,
ui::{keep, section, set},
};
/// A waveform for the demo: a swell and a fall, with a beat.
fn peaks() -> Vec<f32> {
(0..56)
.map(|ix| {
let t = ix as f32 / 55.0;
let swell = (t * std::f32::consts::PI).sin();
let beat = 0.6 + 0.4 * ((ix * 7 % 11) as f32 / 10.0);
(0.15 + 0.8 * swell * beat).min(1.0)
})
.collect()
}
pub fn audio(window: &mut Window, cx: &mut App) -> impl IntoElement + use<> {
let player = keep("gen-audio", || (Duration::from_secs(12), false), window, cx);
let (played, playing) = *player.read(cx);
let (toggled, sought) = (player.clone(), player);
let length = Duration::from_secs(32);
section(
"AudioGenerationPlayer",
"A generated sound: its prompt, model and length, a bar while it is made, then a player whose waveform lights as it plays. A press on the waveform seeks, and so do Left and Right once it has focus.",
cx,
)
.child(
div()
.flex()
.flex_wrap()
.items_start()
.gap_4()
.child(
probe(
"gen-audio",
div().w(px(440.)).child(
AudioGenerationPlayer::new("gen-audio", "rain on a limestone courtyard, far thunder, a slow piano", "Orchid Audio", peaks(), length)
.played(played, playing)
.on_toggle(move |play, _, cx| {
let at = toggled.read(cx).0;
set(&toggled, (at, play), cx)
})
.on_seek(move |share, _, cx| {
let playing = sought.read(cx).1;
set(&sought, (length.mul_f32(share), playing), cx)
})
.on_regenerate(|_, _| log::info!("gallery: regenerate"))
.on_download(|_, _| log::info!("gallery: download")),
),
),
)
.child(
div().w(px(440.)).child(
AudioGenerationPlayer::new("gen-audio-making", "a cello under a high window, morning", "Orchid Audio", peaks(), Duration::from_secs(45))
.making(0.45)
.on_regenerate(|_, _| log::info!("gallery: regenerate"))
.on_download(|_, _| log::info!("gallery: download")),
),
),
)
}
fn voices() -> Vec<Voice> {
let voice = |key: &str, name: &str, about: &str, tags: &[&str]| Voice {
key: key.to_string().into(),
name: name.to_string().into(),
about: about.to_string().into(),
tags: tags
.iter()
.map(|tag| SharedString::from(tag.to_string()))
.collect(),
};
vec![
voice(
"iris",
"Iris",
"Warm and unhurried, for long reads",
&["English", "US"],
),
voice(
"theo",
"Theo",
"Bright and quick, for product tours",
&["English", "UK"],
),
voice(
"mei",
"Mei",
"Calm, with clear diction",
&["Mandarin", "English"],
),
voice(
"ines",
"Inés",
"Soft, close to the microphone",
&["Spanish"],
),
]
}
pub fn voice(window: &mut Window, cx: &mut App) -> impl IntoElement + use<> {
let chosen = keep("gen-voice", || SharedString::from("iris"), window, cx);
let sample = keep("gen-voice-sample", || None::<SharedString>, window, cx);
let (now, sounding) = (chosen.read(cx).clone(), sample.read(cx).clone());
let picker = TTSVoicePicker::new("gen-voices", voices())
.selected(now)
.on_select(move |key, _, cx| set(&chosen, key.clone(), cx))
.on_play(move |key, _, cx| set(&sample, key, cx));
let picker = match sounding {
Some(key) => picker.playing(key),
None => picker,
};
section(
"TTSVoicePicker",
"Voices to speak with: play a sample, then choose one by its row. The chosen voice takes the active wash and a check; tags drop below the name when narrow.",
cx,
)
.child(probe("gen-voices", div().w(px(460.)).child(picker)))
}
fn shots() -> Vec<Shot> {
let shot = |key: &str, prompt: &str, seconds: u64, outcome| Shot {
key: key.to_string().into(),
prompt: prompt.to_string().into(),
length: Duration::from_secs(seconds),
outcome,
};
vec![
shot(
"s1",
"a slow pan across a white atrium",
5,
Outcome::Done(asset!("atrium.jpg").into()),
),
shot(
"s2",
"light moves over the stair",
4,
Outcome::Done(asset!("atrium-stair.jpg").into()),
),
shot("s3", "dusk over the dunes", 6, Outcome::Pending(Some(0.35))),
shot(
"s4",
"grass bends in the wind",
4,
Outcome::Failed("timed out".into()),
),
]
}
pub fn video(window: &mut Window, cx: &mut App) -> impl IntoElement + use<> {
let list = keep("gen-shots", shots, window, cx);
let chosen = keep("gen-shot", || SharedString::from("s1"), window, cx);
let (now, pick) = (list.read(cx).clone(), chosen.read(cx).clone());
section(
"VideoGenerationTimeline",
"A video made shot by shot along its time: each shot as wide as it runs, its poster once made, a placeholder while it comes. Choose a shot to read it; Extend asks for the next.",
cx,
)
.child(
probe(
"gen-timeline",
div().w(px(560.)).child(
VideoGenerationTimeline::new("gen-timeline", now, Duration::from_millis(3_200))
.selected(pick)
.on_select(move |key, _, cx| set(&chosen, key.clone(), cx))
.on_extend(move |_, cx| {
let mut next = list.read(cx).clone();
let key = format!("s{}", next.len() + 1);
next.push(Shot {
key: key.into(),
prompt: "the camera rises over the dunes".into(),
length: Duration::from_secs(5),
outcome: Outcome::Pending(Some(0.0)),
});
set(&list, next, cx)
}),
),
),
)
}
+30 -4
View File
@@ -3,7 +3,7 @@ use std::{rc::Rc, time::Duration};
use gpui::{
App, ElementId, InteractiveElement, IntoElement, MouseButton, ObjectFit, ParentElement,
RenderOnce, SharedString, StatefulInteractiveElement, Styled, Window, div, prelude::*,
relative,
relative, transparent_black,
};
use super::media::shaped;
@@ -11,13 +11,16 @@ use crate::{
buttons::{ButtonVariant, IconButton},
documents::source,
forms::OnFlag,
primitives::{Icon, IconName, Image},
primitives::{FocusRing, Icon, IconName, Image, tab_stop},
theme::{ActiveTheme, ContainerSize, ControlSize, IconSize, Radius, TextSize},
typography::{DurationStyle, format::duration, tabular},
};
type OnSeek = Rc<dyn Fn(f32, &mut Window, &mut App)>;
/// The share of a clip Left and Right move on its waveform.
const STEP: f32 = 0.05;
/// A clip's time as it plays: what played and the length, in clock digits.
fn clock(played: Duration, length: Duration) -> String {
format!(
@@ -27,7 +30,7 @@ fn clock(played: Duration, length: Duration) -> String {
)
}
/// A voice note or a song: play and pause, a waveform lit as far as it has played, and the time; a press on the waveform seeks there. The host plays it.
/// A voice note or a song: play and pause, a waveform lit as far as it has played, and the time; a press on the waveform seeks there, and so do Left and Right once it has focus. The host plays it.
#[derive(IntoElement)]
pub struct AudioMessage {
id: ElementId,
@@ -80,7 +83,9 @@ impl AudioMessage {
}
impl RenderOnce for AudioMessage {
fn render(self, _: &mut Window, cx: &mut App) -> impl IntoElement {
fn render(self, window: &mut Window, cx: &mut App) -> impl IntoElement {
let focus = (self.on_seek.as_ref())
.map(|_| tab_stop((self.id.clone(), "wave").into(), true, window, cx));
let theme = cx.theme();
let colors = theme.colors.clone();
let share = if self.length.is_zero() {
@@ -121,11 +126,31 @@ impl RenderOnce for AudioMessage {
}))
.child(
div()
.id((self.id.clone(), "wave"))
.flex_1()
.min_w_0()
.h(tall)
.flex()
.items_center()
.gap_px()
.rounded(theme.radius(Radius::Sm))
.border_1()
.border_color(transparent_black())
.when_some(focus.zip(self.on_seek.clone()), |wave, (focus, seek)| {
wave.track_focus(&focus).focus_ring(cx).on_key_down(
move |event, window, cx| {
let step = match event.keystroke.key.as_str() {
"left" => -STEP,
"right" => STEP,
_ => return,
};
cx.stop_propagation();
let next = (share + step).clamp(0.0, 1.0);
log::info!("audio message: seek to {next:.2}");
seek(next, window, cx)
},
)
})
.children(self.peaks.iter().enumerate().map(|(ix, peak)| {
let seek = self.on_seek.clone();
let lit = (ix as f32 + 0.5) / count as f32 <= share;
@@ -156,6 +181,7 @@ impl RenderOnce for AudioMessage {
.child(
tabular(
div()
.flex_none()
.pr_2()
.text_size(theme.text_size(TextSize::Xs))
.text_color(colors.fg_muted),
+1
View File
@@ -19,6 +19,7 @@ pub use hex::HexViewer;
pub use network::{NetworkInspector, Request};
pub use session::{Breakpoint, BreakpointList, DebugCommand, DebugState, DebugToolbar};
pub use stack::{CallStack, StackFrame, Thread, ThreadList, ThreadState};
pub(crate) use timeline::ruler;
pub use timeline::{Span, TimelineProfiler, zoomed};
pub use variables::{ValueKind, Variable, VariablesPanel, Watch, WatchPanel};
+1 -1
View File
@@ -35,7 +35,7 @@ fn tick_label(time: f64, step: f64) -> String {
const TICKS: usize = 6;
/// The ruler's step and ticks: as many as leave `room` for each label across `width`, each with room before the end.
fn ruler(range: (f64, f64), width: f32, room: f32) -> (f64, Vec<f64>) {
pub(crate) fn ruler(range: (f64, f64), width: f32, room: f32) -> (f64, Vec<f64>) {
let length = range.1 - range.0;
let across = move |span: f64| (span / length) as f32 * width;
let step = (2..=TICKS)
+221
View File
@@ -0,0 +1,221 @@
use std::{rc::Rc, time::Duration};
use gpui::{
App, ElementId, IntoElement, ParentElement, RenderOnce, SharedString, Styled, Window, div,
};
use crate::{
buttons::IconButton,
chat::AudioMessage,
forms::{OnFlag, Run},
motion::ProgressBar,
primitives::{Icon, IconName},
theme::{ActiveTheme, ControlSize, IconSize, Radius, TextSize},
typography::{
Ellipsis,
format::{DurationStyle, duration, percent},
tabular,
},
};
type OnSeek = Rc<dyn Fn(f32, &mut Window, &mut App)>;
/// A generated sound: its prompt, its model and length, a bar while it is made, then a player with its waveform. Regenerate and Download wait for it. The host plays it.
#[derive(IntoElement)]
pub struct AudioGenerationPlayer {
id: ElementId,
prompt: SharedString,
model: SharedString,
peaks: Vec<f32>,
length: Duration,
making: Option<f32>,
played: Duration,
playing: bool,
on_toggle: Option<OnFlag>,
on_seek: Option<OnSeek>,
on_regenerate: Option<Run>,
on_download: Option<Run>,
}
impl AudioGenerationPlayer {
/// `peaks` are the waveform's heights, 0 to 1, left to right.
pub fn new(
id: impl Into<ElementId>,
prompt: impl Into<SharedString>,
model: impl Into<SharedString>,
peaks: impl Into<Vec<f32>>,
length: Duration,
) -> Self {
let peaks: Vec<f32> = peaks.into();
assert!(
!peaks.is_empty() && peaks.iter().all(|peak| (0.0..=1.0).contains(peak)),
"a sound's peaks lie within 0 to 1"
);
Self {
id: id.into(),
prompt: prompt.into(),
model: model.into(),
peaks,
length,
making: None,
played: Duration::ZERO,
playing: false,
on_toggle: None,
on_seek: None,
on_regenerate: None,
on_download: None,
}
}
/// While it is made, the share done.
pub fn making(mut self, share: f32) -> Self {
assert!((0.0..=1.0).contains(&share), "a sound made to {share} of 1");
self.making = Some(share);
self
}
/// How far it has played, and whether it plays now.
pub fn played(mut self, played: Duration, playing: bool) -> Self {
self.played = played;
self.playing = playing;
self
}
/// Gets true to play, false to pause.
pub fn on_toggle(mut self, handler: impl Fn(bool, &mut Window, &mut App) + 'static) -> Self {
self.on_toggle = Some(Rc::new(handler));
self
}
/// Gets the share of the sound to seek to.
pub fn on_seek(mut self, handler: impl Fn(f32, &mut Window, &mut App) + 'static) -> Self {
self.on_seek = Some(Rc::new(handler));
self
}
pub fn on_regenerate(mut self, handler: impl Fn(&mut Window, &mut App) + 'static) -> Self {
self.on_regenerate = Some(Rc::new(handler));
self
}
pub fn on_download(mut self, handler: impl Fn(&mut Window, &mut App) + 'static) -> Self {
self.on_download = Some(Rc::new(handler));
self
}
}
impl RenderOnce for AudioGenerationPlayer {
fn render(self, _: &mut Window, cx: &mut App) -> impl IntoElement {
let theme = cx.theme();
let colors = theme.colors.clone();
let making = self.making;
let action = |key: &'static str, icon: IconName, tip: &'static str, run: Option<Run>| {
run.map(|run| {
IconButton::new((self.id.clone(), key), icon)
.size(ControlSize::Sm)
.tooltip(tip)
.disabled(making.is_some())
.on_click(move |_, window, cx| {
log::info!("audio generation: {key}");
run(window, cx)
})
})
};
let body = match making {
Some(share) => div()
.flex()
.flex_col()
.gap_2()
.child(ProgressBar::new((self.id.clone(), "making"), share))
.child(
tabular(div())
.text_size(theme.text_size(TextSize::Xs))
.text_color(colors.fg_muted)
.child(format!("Making · {}", percent(f64::from(share), 0, false))),
)
.into_any_element(),
None => {
let player =
AudioMessage::new((self.id.clone(), "player"), self.peaks, self.length)
.played(self.played, self.playing);
let player = match self.on_toggle {
Some(toggle) => {
player.on_toggle(move |play, window, cx| toggle(play, window, cx))
}
None => player,
};
match self.on_seek {
Some(seek) => player.on_seek(move |share, window, cx| seek(share, window, cx)),
None => player,
}
.into_any_element()
}
};
div()
.w_full()
.flex()
.flex_col()
.gap_3()
.p_4()
.rounded(theme.radius(Radius::Lg))
.border_1()
.border_color(colors.border)
.bg(colors.surface)
.child(
div()
.flex()
.items_start()
.gap_3()
.child(
div().flex_none().pt_0p5().child(
Icon::new(IconName::AudioLines)
.size(IconSize::Sm)
.color(colors.fg_muted),
),
)
.child(
div()
.flex_1()
.min_w_0()
.flex()
.flex_col()
.gap_0p5()
.child(
div()
.text_size(theme.text_size(TextSize::Sm))
.text_color(colors.fg)
.child(Ellipsis::new(self.prompt)),
)
.child(
div()
.text_size(theme.text_size(TextSize::Xs))
.text_color(colors.fg_muted)
.child(Ellipsis::new(format!(
"{} · {}",
self.model,
duration(self.length.as_secs(), DurationStyle::Clock)
))),
),
)
.child(
div()
.flex_none()
.flex()
.gap_1()
.children(action(
"regenerate",
IconName::RefreshCw,
"Regenerate",
self.on_regenerate,
))
.children(action(
"download",
IconName::Download,
"Download",
self.on_download,
)),
),
)
.child(body)
}
}
+13 -11
View File
@@ -329,19 +329,21 @@ impl RenderOnce for InpaintCanvas {
.size_full(),
)
.children(ring)
.on_mouse_move(move |event, _, cx| {
hover.update(cx, |pointer, cx| {
*pointer = Some(event.position);
cx.notify();
})
})
.on_hover(move |hovered, _, cx| {
if !hovered {
leave.update(cx, |pointer, cx| {
*pointer = None;
.when(drawn, |area| {
area.on_mouse_move(move |event, _, cx| {
hover.update(cx, |pointer, cx| {
*pointer = Some(event.position);
cx.notify();
})
}
})
.on_hover(move |hovered, _, cx| {
if !hovered {
leave.update(cx, |pointer, cx| {
*pointer = None;
cx.notify();
})
}
})
})
.on_key_down(move |event, window, cx| {
let Some(keys) = &keys else {
+6
View File
@@ -1,3 +1,4 @@
mod audio;
mod brush;
mod compare;
mod grid;
@@ -10,7 +11,10 @@ mod style;
#[cfg(all(test, feature = "test-support"))]
mod tests;
mod variation;
mod video;
mod voice;
pub use audio::AudioGenerationPlayer;
pub use brush::MaskBrush;
pub use compare::{ABCompareView, Verdict};
pub use grid::{GenerationGrid, Outcome};
@@ -21,3 +25,5 @@ pub use ratio::AspectRatioPicker;
pub use seed::SeedInput;
pub use style::{StylePreset, StylePresetPicker};
pub use variation::VariationPicker;
pub use video::{Shot, VideoGenerationTimeline};
pub use voice::{TTSVoicePicker, Voice};
+1
View File
@@ -5,6 +5,7 @@ use crate::{forms, primitives::FocusNext, theme::Theme};
mod canvas;
mod results;
mod settings;
mod sound;
fn settle(cx: &mut VisualTestContext) {
cx.run_until_parked();
+187
View File
@@ -0,0 +1,187 @@
use std::{cell::RefCell, rc::Rc, time::Duration};
use gpui::{
Context, IntoElement, Modifiers, ParentElement, Render, Styled, TestAppContext, Window, div,
point, px,
};
use super::{press, settle, setup, tab};
use crate::generative::{
AudioGenerationPlayer, Outcome, Shot, TTSVoicePicker, VideoGenerationTimeline, Voice,
};
type Heard = Rc<RefCell<Vec<String>>>;
/// A made sound a quarter played and one still being made, and what they heard.
struct Players(Heard);
impl Render for Players {
fn render(&mut self, _: &mut Window, _: &mut Context<Self>) -> impl IntoElement {
let (sought, made, making) = (self.0.clone(), self.0.clone(), self.0.clone());
let length = Duration::from_secs(40);
div()
.w(px(420.0))
.child(
AudioGenerationPlayer::new("made", "rain", "Orchid", vec![0.5; 8], length)
.played(Duration::from_secs(10), false)
.on_toggle(|_, _, _| {})
.on_seek(move |share, _, _| {
sought.borrow_mut().push(format!("seek {share:.2}"))
})
.on_regenerate(move |_, _| made.borrow_mut().push("regenerate made".into())),
)
.child(
AudioGenerationPlayer::new("making", "cello", "Orchid", vec![0.5; 8], length)
.making(0.4)
.on_regenerate(move |_, _| {
making.borrow_mut().push("regenerate making".into())
}),
)
}
}
#[gpui::test]
fn the_waveform_seeks_with_keys_and_a_sound_being_made_waits(cx: &mut TestAppContext) {
setup(cx);
let heard = Heard::default();
let store = heard.clone();
let (_, cx) = cx.add_window_view(|_, _| Players(store));
cx.update(|window, _| window.activate_window());
settle(cx);
tab(1, cx);
press("enter", cx);
tab(2, cx);
press("right", cx);
tab(1, cx);
press("enter", cx);
let heard = heard.borrow().clone();
assert_eq!(
heard,
["regenerate made", "seek 0.30", "regenerate made"],
"Tab passes the waiting button"
);
}
/// Two voices, the first chosen, and what they heard.
struct Voices(Heard);
impl Render for Voices {
fn render(&mut self, _: &mut Window, _: &mut Context<Self>) -> impl IntoElement {
let (chosen, played) = (self.0.clone(), self.0.clone());
let voice = |key: &str| Voice {
key: key.to_string().into(),
name: key.to_string().into(),
about: "a voice".into(),
tags: vec!["English".into()],
};
div().w(px(420.0)).child(
TTSVoicePicker::new("voices", [voice("iris"), voice("theo")])
.selected("iris")
.on_select(move |key, _, _| chosen.borrow_mut().push(format!("choose {key}")))
.on_play(move |key, _, _| played.borrow_mut().push(format!("play {key:?}"))),
)
}
}
#[gpui::test]
fn a_sample_plays_without_choosing_its_voice(cx: &mut TestAppContext) {
setup(cx);
let heard = Heard::default();
let store = heard.clone();
let (_, cx) = cx.add_window_view(|_, _| Voices(store));
cx.update(|window, _| window.activate_window());
settle(cx);
tab(3, cx);
press("enter", cx);
cx.simulate_click(point(px(23.0), px(83.0)), Modifiers::none());
settle(cx);
assert_eq!(
*heard.borrow(),
["choose theo", "play Some(\"theo\")"],
"a sample's press stays off its row"
);
}
/// Two shots, the first chosen, and what they heard.
struct Timeline(Heard);
impl Render for Timeline {
fn render(&mut self, _: &mut Window, _: &mut Context<Self>) -> impl IntoElement {
let (chosen, extended) = (self.0.clone(), self.0.clone());
let shot = |key: &str, outcome| Shot {
key: key.to_string().into(),
prompt: "a pan".into(),
length: Duration::from_secs(4),
outcome,
};
div().w(px(420.0)).child(
VideoGenerationTimeline::new(
"timeline",
[
shot("a", Outcome::Done("missing.jpg".into())),
shot("b", Outcome::Pending(None)),
],
Duration::ZERO,
)
.selected("a")
.on_select(move |key, _, _| chosen.borrow_mut().push(format!("choose {key}")))
.on_extend(move |_, _| extended.borrow_mut().push("extend".into())),
)
}
}
#[gpui::test]
fn a_shot_is_chosen_once_and_extend_asks_for_the_next(cx: &mut TestAppContext) {
setup(cx);
let heard = Heard::default();
let store = heard.clone();
let (_, cx) = cx.add_window_view(|_, _| Timeline(store));
cx.update(|window, _| window.activate_window());
settle(cx);
for _ in 0..3 {
tab(1, cx);
press("enter", cx);
}
assert_eq!(*heard.borrow(), ["choose b", "extend"]);
}
/// A voice list and a timeline shown to read, with nothing to take a choice.
struct Shown;
impl Render for Shown {
fn render(&mut self, _: &mut Window, _: &mut Context<Self>) -> impl IntoElement {
let voice = Voice {
key: "iris".into(),
name: "Iris".into(),
about: "a voice".into(),
tags: Vec::new(),
};
let shot = Shot {
key: "a".into(),
prompt: "a pan".into(),
length: Duration::from_secs(4),
outcome: Outcome::Pending(None),
};
div()
.w(px(420.0))
.child(TTSVoicePicker::new("voices", [voice]).selected("iris"))
.child(VideoGenerationTimeline::new("timeline", [shot], Duration::ZERO).selected("a"))
}
}
#[gpui::test]
fn lists_shown_to_read_take_no_focus(cx: &mut TestAppContext) {
setup(cx);
let (_, cx) = cx.add_window_view(|_, _| Shown);
cx.update(|window, _| window.activate_window());
settle(cx);
tab(1, cx);
assert!(cx.update(|window, cx| window.focused(cx).is_none()));
}
#[test]
#[should_panic(expected = "a sound's peaks lie within 0 to 1")]
fn a_peak_past_full_fails_loud() {
let _ =
AudioGenerationPlayer::new("sound", "rain", "Orchid", vec![1.5], Duration::from_secs(1));
}
+288
View File
@@ -0,0 +1,288 @@
use std::{rc::Rc, time::Duration};
use gpui::{
App, ElementId, InteractiveElement, IntoElement, MouseButton, ParentElement, RenderOnce,
SharedString, StatefulInteractiveElement, Styled, Window, div, prelude::*, relative,
transparent_black,
};
use super::Outcome;
use crate::{
buttons::{Button, ButtonVariant},
debug::ruler,
documents::source,
forms::{OnValue, Run},
layout::on_axis,
motion::Skeleton,
primitives::{Icon, IconName, Image, tab_stop},
theme::{ActiveTheme, ControlSize, IconSize, TextSize},
typography::{
Caption,
format::{DurationStyle, duration, percent},
tabular,
},
};
/// A shot of a generated video: its key, its prompt, how long it runs, and how it came out.
#[derive(Clone, Debug, PartialEq)]
pub struct Shot {
pub key: SharedString,
pub prompt: SharedString,
pub length: Duration,
pub outcome: Outcome,
}
/// A video made shot by shot, laid along its time: a ruler, each shot as wide as it runs with its poster, a breathing placeholder while it comes or a mark if it failed, and the playhead. A press or Enter chooses a shot and a line under names it; Extend asks for the next. It scrolls sideways when long.
#[derive(IntoElement)]
pub struct VideoGenerationTimeline {
id: ElementId,
shots: Vec<Shot>,
at: Duration,
selected: Option<SharedString>,
on_select: Option<OnValue>,
on_extend: Option<Run>,
}
impl VideoGenerationTimeline {
/// `at` is the playhead's time.
pub fn new(
id: impl Into<ElementId>,
shots: impl IntoIterator<Item = Shot>,
at: Duration,
) -> Self {
let shots: Vec<Shot> = shots.into_iter().collect();
for shot in &shots {
assert!(!shot.length.is_zero(), "shot {} runs no time", shot.key);
if let Outcome::Pending(Some(share)) = shot.outcome {
assert!(
(0.0..=1.0).contains(&share),
"shot {} at {share} of 1",
shot.key
);
}
}
Self {
id: id.into(),
shots,
at,
selected: None,
on_select: None,
on_extend: None,
}
}
pub fn selected(mut self, key: impl Into<SharedString>) -> Self {
self.selected = Some(key.into());
self
}
pub fn on_select(
mut self,
handler: impl Fn(&SharedString, &mut Window, &mut App) + 'static,
) -> Self {
self.on_select = Some(Rc::new(handler));
self
}
/// Asks for one more shot after the last.
pub fn on_extend(mut self, handler: impl Fn(&mut Window, &mut App) + 'static) -> Self {
self.on_extend = Some(Rc::new(handler));
self
}
}
impl RenderOnce for VideoGenerationTimeline {
fn render(self, window: &mut Window, cx: &mut App) -> impl IntoElement {
let named = (self.selected.as_ref())
.is_none_or(|key| self.shots.iter().any(|shot| &shot.key == key));
assert!(named, "the chosen shot is not listed");
let pickable = self.on_select.is_some();
let focuses: Vec<_> = self
.shots
.iter()
.map(|shot| {
let focus = tab_stop(
(self.id.clone(), format!("focus-{}", shot.key)).into(),
pickable,
window,
cx,
);
let focused = focus.is_focused(window);
(focus, focused)
})
.collect();
let theme = cx.theme();
let colors = theme.colors.clone();
let sizes = theme.generative();
let rem = window.rem_size();
let total: f64 = self
.shots
.iter()
.map(|shot| shot.length.as_secs_f64())
.sum();
let second = sizes.second;
let span = second * total as f32;
let (_, ticks) = ruler(
(0.0, total.max(1.0)),
f32::from(span.to_pixels(rem)),
f32::from(theme.chart().label.to_pixels(rem)),
);
let caption = (self.selected.as_ref())
.and_then(|key| self.shots.iter().position(|shot| &shot.key == key))
.map(|ix| {
let shot = &self.shots[ix];
let state = match &shot.outcome {
Outcome::Pending(_) => "coming".to_string(),
Outcome::Done(_) => duration(shot.length.as_secs(), DurationStyle::Compact),
Outcome::Failed(reason) => format!("failed, {reason}"),
};
format!("Shot {} · {state} · {}", ix + 1, shot.prompt)
});
let ruler_row = div()
.relative()
.h(theme.text_size(TextSize::Xs) * 1.6)
.w(span)
.border_b_1()
.border_color(colors.border)
.text_size(theme.text_size(TextSize::Xs))
.text_color(colors.fg_subtle)
.children(ticks.into_iter().map(|time| {
tabular(div())
.absolute()
.top_0()
.left(relative((time / total.max(1.0)) as f32))
.pl_1()
.border_l_1()
.border_color(colors.border)
.child(duration(time as u64, DurationStyle::Clock))
}));
let tiles = self.shots.into_iter().zip(focuses).enumerate().map(
|(ix, (shot, (focus, focused)))| {
let chosen = self.selected.as_ref() == Some(&shot.key);
let (pick, key) = (self.on_select.clone(), shot.key.clone());
let face = match &shot.outcome {
Outcome::Done(poster) => {
Image::new((self.id.clone(), format!("poster-{ix}")), source(poster))
.size_full()
.into_any_element()
}
Outcome::Pending(share) => div()
.relative()
.size_full()
.child(
Skeleton::new((self.id.clone(), format!("pending-{ix}"))).size_full(),
)
.children(share.map(|share| {
div()
.absolute()
.inset_0()
.flex()
.items_center()
.justify_center()
.text_size(theme.text_size(TextSize::Xs))
.text_color(colors.fg_muted)
.child(tabular(div()).child(percent(f64::from(share), 0, false)))
}))
.into_any_element(),
Outcome::Failed(_) => div()
.size_full()
.flex()
.items_center()
.justify_center()
.bg(colors.sunken)
.child(
Icon::new(IconName::CircleAlert)
.size(IconSize::Sm)
.color(colors.danger),
)
.into_any_element(),
};
div()
.id((self.id.clone(), format!("shot-{ix}")))
.flex_none()
.w(second * shot.length.as_secs_f32())
.h(sizes.shot)
.p_0p5()
.border_1()
.border_color(if focused {
colors.focus
} else if chosen {
colors.accent
} else {
transparent_black()
})
.when_some(pick, |tile, pick| {
tile.track_focus(&focus)
.cursor_pointer()
.on_mouse_down(MouseButton::Left, |_, window, _| {
window.prevent_default()
})
.on_click(move |_, window, cx| {
if chosen {
return;
}
log::info!("video timeline: chose {key}");
pick(&key, window, cx)
})
})
.child(
div()
.relative()
.size_full()
.overflow_hidden()
.bg(colors.sunken)
.child(face),
)
},
);
let playhead = (self.at.as_secs_f64() / total.max(1.0)).clamp(0.0, 1.0) as f32;
let extend = self.on_extend.map(|extend| {
Button::new((self.id.clone(), "extend"), "Extend")
.variant(ButtonVariant::Secondary)
.size(ControlSize::Sm)
.icon(IconName::Plus)
.on_click(move |_, window, cx| {
log::info!("video timeline: extend");
extend(window, cx)
})
});
div()
.w_full()
.flex()
.flex_col()
.gap_2()
.child(
on_axis(div().id((self.id.clone(), "sideways")))
.w_full()
.overflow_x_scroll()
.child(
div()
.flex()
.items_end()
.gap_3()
.pb_1()
.child(
div()
.relative()
.flex_none()
.flex()
.flex_col()
.gap_1()
.child(ruler_row)
.child(div().flex().children(tiles))
.child(
div()
.absolute()
.top_0()
.bottom_0()
.left(relative(playhead))
.border_l_1()
.border_color(colors.accent),
),
)
.children(extend.map(|extend| div().flex_none().child(extend))),
),
)
.children(caption.map(Caption::new))
}
}
+227
View File
@@ -0,0 +1,227 @@
use std::rc::Rc;
use gpui::{
App, ElementId, FontWeight, InteractiveElement, IntoElement, MouseButton, ParentElement,
RenderOnce, SharedString, StatefulInteractiveElement, Styled, Window, div, prelude::*,
transparent_black,
};
use crate::{
buttons::{ButtonVariant, IconButton},
data_display::Badge,
forms::OnValue,
primitives::{Icon, IconName, tab_stop},
theme::{ActiveTheme, ControlSize, IconSize, Radius, TextSize},
typography::Ellipsis,
};
/// A voice to speak with: its key, its name, a line on how it sounds, and tags such as its language.
#[derive(Clone, Debug, PartialEq)]
pub struct Voice {
pub key: SharedString,
pub name: SharedString,
pub about: SharedString,
pub tags: Vec<SharedString>,
}
type OnPlay = Rc<dyn Fn(Option<SharedString>, &mut Window, &mut App)>;
/// Voices to speak with, a row each: a sample to play, the name and how it sounds, and its tags, which drop below when narrow. A press or Enter chooses a row; the chosen one takes the active wash and a check. The host plays the samples.
#[derive(IntoElement)]
pub struct TTSVoicePicker {
id: ElementId,
voices: Vec<Voice>,
selected: Option<SharedString>,
playing: Option<SharedString>,
on_select: Option<OnValue>,
on_play: Option<OnPlay>,
}
impl TTSVoicePicker {
pub fn new(id: impl Into<ElementId>, voices: impl IntoIterator<Item = Voice>) -> Self {
Self {
id: id.into(),
voices: voices.into_iter().collect(),
selected: None,
playing: None,
on_select: None,
on_play: None,
}
}
pub fn selected(mut self, key: impl Into<SharedString>) -> Self {
self.selected = Some(key.into());
self
}
/// The voice whose sample plays now.
pub fn playing(mut self, key: impl Into<SharedString>) -> Self {
self.playing = Some(key.into());
self
}
pub fn on_select(
mut self,
handler: impl Fn(&SharedString, &mut Window, &mut App) + 'static,
) -> Self {
self.on_select = Some(Rc::new(handler));
self
}
/// Gets the voice whose sample to play, or none to stop.
pub fn on_play(
mut self,
handler: impl Fn(Option<SharedString>, &mut Window, &mut App) + 'static,
) -> Self {
self.on_play = Some(Rc::new(handler));
self
}
}
impl RenderOnce for TTSVoicePicker {
fn render(self, window: &mut Window, cx: &mut App) -> impl IntoElement {
let known = |key: &Option<SharedString>| {
key.as_ref()
.is_none_or(|key| self.voices.iter().any(|voice| &voice.key == key))
};
assert!(
known(&self.selected) && known(&self.playing),
"a voice named is not listed"
);
let pickable = self.on_select.is_some();
let focuses: Vec<_> = self
.voices
.iter()
.map(|voice| {
let focus = tab_stop(
(self.id.clone(), format!("focus-{}", voice.key)).into(),
pickable,
window,
cx,
);
let focused = focus.is_focused(window);
(focus, focused)
})
.collect();
let theme = cx.theme();
let colors = theme.colors.clone();
let label = theme.label_width();
let rows = self
.voices
.into_iter()
.zip(focuses)
.map(|(voice, (focus, focused))| {
let chosen = self.selected.as_ref() == Some(&voice.key);
let sounding = self.playing.as_ref() == Some(&voice.key);
let (pick, play, key) = (
self.on_select.clone(),
self.on_play.clone(),
voice.key.clone(),
);
let sample = play.map(|play| {
let key = key.clone();
IconButton::new(
(self.id.clone(), format!("play-{}", voice.key)),
if sounding {
IconName::CircleStop
} else {
IconName::Play
},
)
.variant(ButtonVariant::Ghost)
.size(ControlSize::Sm)
.tooltip(if sounding {
"Stop the sample"
} else {
"Play a sample"
})
.on_click(move |_, window, cx| {
cx.stop_propagation();
let next = (!sounding).then(|| key.clone());
log::info!("voice picker: sample {next:?}");
play(next, window, cx)
})
});
div()
.id((self.id.clone(), format!("voice-{}", voice.key)))
.w_full()
.flex()
.flex_wrap()
.items_center()
.gap_x_3()
.gap_y_1()
.px_2()
.py_2()
.rounded(theme.radius(Radius::Md))
.border_1()
.border_color(if focused {
colors.focus
} else {
transparent_black()
})
.when(chosen, |row| row.bg(colors.active))
.when_some(pick, |row, pick| {
row.track_focus(&focus)
.when(!chosen, |row| row.hover(|style| style.bg(colors.hover)))
.cursor_pointer()
.on_mouse_down(MouseButton::Left, |_, window, _| {
window.prevent_default()
})
.on_click(move |_, window, cx| {
if chosen {
return;
}
log::info!("voice picker: chose {key}");
pick(&key, window, cx)
})
})
.child(
div()
.flex_1()
.min_w(label)
.flex()
.items_center()
.gap_2()
.children(sample.map(|sample| div().flex_none().child(sample)))
.child(
div()
.flex_1()
.min_w_0()
.flex()
.flex_col()
.child(
div()
.text_size(theme.text_size(TextSize::Sm))
.font_weight(FontWeight::MEDIUM)
.text_color(colors.fg)
.child(Ellipsis::new(voice.name)),
)
.child(
div()
.text_size(theme.text_size(TextSize::Xs))
.text_color(colors.fg_muted)
.child(Ellipsis::new(voice.about)),
),
)
.when(chosen, |name| {
name.child(
div().flex_none().child(
Icon::new(IconName::Check)
.size(IconSize::Sm)
.color(colors.accent),
),
)
}),
)
.child(
div()
.flex_none()
.flex()
.flex_wrap()
.gap_1()
.children(voice.tags.into_iter().map(Badge::new)),
)
});
div().w_full().flex().flex_col().gap_0p5().children(rows)
}
}
+1
View File
@@ -56,6 +56,7 @@ icons! {
ArrowUpRight => "arrow-up-right",
ArrowUp => "arrow-up",
AtSign => "at-sign",
AudioLines => "audio-lines",
Ban => "ban",
Battery => "battery",
Bell => "bell",
+6
View File
@@ -7,12 +7,18 @@ use super::{Theme, tokens::px_to_rems};
pub struct GenerativeSizes {
/// A style preset's picture, square.
pub preset: Rems,
/// A second of video along a timeline.
pub second: Rems,
/// A shot's height on a timeline.
pub shot: Rems,
}
impl Theme {
pub fn generative(&self) -> GenerativeSizes {
GenerativeSizes {
preset: px_to_rems(88.0),
second: px_to_rems(24.0),
shot: px_to_rems(56.0),
}
}
}
+3 -3
View File
@@ -232,9 +232,9 @@
## T25d Generative · Sound and motion
- [ ] AudioGenerationPlayer
- [ ] VideoGenerationTimeline
- [ ] TTSVoicePicker
- [x] AudioGenerationPlayer
- [x] VideoGenerationTimeline
- [x] TTSVoicePicker
## T25e Generative · Models