diff --git a/assets/icons/audio-lines.svg b/assets/icons/audio-lines.svg new file mode 100644 index 0000000..4188d22 --- /dev/null +++ b/assets/icons/audio-lines.svg @@ -0,0 +1,20 @@ + + + + + + + + + diff --git a/examples/gallery/pages/generative/mod.rs b/examples/gallery/pages/generative/mod.rs index a2ded88..99073a1 100644 --- a/examples/gallery/pages/generative/mod.rs +++ b/examples/gallery/pages/generative/mod.rs @@ -1,6 +1,7 @@ mod canvas; mod panel; mod results; +mod sound; use gpui::{AnyElement, App, IntoElement, ParentElement, Window, div}; @@ -43,5 +44,8 @@ fn render(window: &mut Window, cx: &mut App) -> AnyElement { .child(results::results(window, cx)) .child(results::variations(window, cx)) .child(canvas::canvas(window, cx)) + .child(sound::audio(window, cx)) + .child(sound::voice(window, cx)) + .child(sound::video(window, cx)) .into_any_element() } diff --git a/examples/gallery/pages/generative/sound.rs b/examples/gallery/pages/generative/sound.rs new file mode 100644 index 0000000..fb4501a --- /dev/null +++ b/examples/gallery/pages/generative/sound.rs @@ -0,0 +1,189 @@ +use std::time::Duration; + +use ely_gpui_component::generative::{ + AudioGenerationPlayer, Outcome, Shot, TTSVoicePicker, VideoGenerationTimeline, Voice, +}; +use gpui::{App, IntoElement, ParentElement, SharedString, Styled, Window, div, px}; + +use crate::{ + probe::probe, + ui::{keep, section, set}, +}; + +/// A waveform for the demo: a swell and a fall, with a beat. +fn peaks() -> Vec { + (0..56) + .map(|ix| { + let t = ix as f32 / 55.0; + let swell = (t * std::f32::consts::PI).sin(); + let beat = 0.6 + 0.4 * ((ix * 7 % 11) as f32 / 10.0); + (0.15 + 0.8 * swell * beat).min(1.0) + }) + .collect() +} + +pub fn audio(window: &mut Window, cx: &mut App) -> impl IntoElement + use<> { + let player = keep("gen-audio", || (Duration::from_secs(12), false), window, cx); + let (played, playing) = *player.read(cx); + let (toggled, sought) = (player.clone(), player); + let length = Duration::from_secs(32); + section( + "AudioGenerationPlayer", + "A generated sound: its prompt, model and length, a bar while it is made, then a player whose waveform lights as it plays. A press on the waveform seeks, and so do Left and Right once it has focus.", + cx, + ) + .child( + div() + .flex() + .flex_wrap() + .items_start() + .gap_4() + .child( + probe( + "gen-audio", + div().w(px(440.)).child( + AudioGenerationPlayer::new("gen-audio", "rain on a limestone courtyard, far thunder, a slow piano", "Orchid Audio", peaks(), length) + .played(played, playing) + .on_toggle(move |play, _, cx| { + let at = toggled.read(cx).0; + set(&toggled, (at, play), cx) + }) + .on_seek(move |share, _, cx| { + let playing = sought.read(cx).1; + set(&sought, (length.mul_f32(share), playing), cx) + }) + .on_regenerate(|_, _| log::info!("gallery: regenerate")) + .on_download(|_, _| log::info!("gallery: download")), + ), + ), + ) + .child( + div().w(px(440.)).child( + AudioGenerationPlayer::new("gen-audio-making", "a cello under a high window, morning", "Orchid Audio", peaks(), Duration::from_secs(45)) + .making(0.45) + .on_regenerate(|_, _| log::info!("gallery: regenerate")) + .on_download(|_, _| log::info!("gallery: download")), + ), + ), + ) +} + +fn voices() -> Vec { + let voice = |key: &str, name: &str, about: &str, tags: &[&str]| Voice { + key: key.to_string().into(), + name: name.to_string().into(), + about: about.to_string().into(), + tags: tags + .iter() + .map(|tag| SharedString::from(tag.to_string())) + .collect(), + }; + vec![ + voice( + "iris", + "Iris", + "Warm and unhurried, for long reads", + &["English", "US"], + ), + voice( + "theo", + "Theo", + "Bright and quick, for product tours", + &["English", "UK"], + ), + voice( + "mei", + "Mei", + "Calm, with clear diction", + &["Mandarin", "English"], + ), + voice( + "ines", + "Inés", + "Soft, close to the microphone", + &["Spanish"], + ), + ] +} + +pub fn voice(window: &mut Window, cx: &mut App) -> impl IntoElement + use<> { + let chosen = keep("gen-voice", || SharedString::from("iris"), window, cx); + let sample = keep("gen-voice-sample", || None::, window, cx); + let (now, sounding) = (chosen.read(cx).clone(), sample.read(cx).clone()); + let picker = TTSVoicePicker::new("gen-voices", voices()) + .selected(now) + .on_select(move |key, _, cx| set(&chosen, key.clone(), cx)) + .on_play(move |key, _, cx| set(&sample, key, cx)); + let picker = match sounding { + Some(key) => picker.playing(key), + None => picker, + }; + section( + "TTSVoicePicker", + "Voices to speak with: play a sample, then choose one by its row. The chosen voice takes the active wash and a check; tags drop below the name when narrow.", + cx, + ) + .child(probe("gen-voices", div().w(px(460.)).child(picker))) +} + +fn shots() -> Vec { + let shot = |key: &str, prompt: &str, seconds: u64, outcome| Shot { + key: key.to_string().into(), + prompt: prompt.to_string().into(), + length: Duration::from_secs(seconds), + outcome, + }; + vec![ + shot( + "s1", + "a slow pan across a white atrium", + 5, + Outcome::Done(asset!("atrium.jpg").into()), + ), + shot( + "s2", + "light moves over the stair", + 4, + Outcome::Done(asset!("atrium-stair.jpg").into()), + ), + shot("s3", "dusk over the dunes", 6, Outcome::Pending(Some(0.35))), + shot( + "s4", + "grass bends in the wind", + 4, + Outcome::Failed("timed out".into()), + ), + ] +} + +pub fn video(window: &mut Window, cx: &mut App) -> impl IntoElement + use<> { + let list = keep("gen-shots", shots, window, cx); + let chosen = keep("gen-shot", || SharedString::from("s1"), window, cx); + let (now, pick) = (list.read(cx).clone(), chosen.read(cx).clone()); + section( + "VideoGenerationTimeline", + "A video made shot by shot along its time: each shot as wide as it runs, its poster once made, a placeholder while it comes. Choose a shot to read it; Extend asks for the next.", + cx, + ) + .child( + probe( + "gen-timeline", + div().w(px(560.)).child( + VideoGenerationTimeline::new("gen-timeline", now, Duration::from_millis(3_200)) + .selected(pick) + .on_select(move |key, _, cx| set(&chosen, key.clone(), cx)) + .on_extend(move |_, cx| { + let mut next = list.read(cx).clone(); + let key = format!("s{}", next.len() + 1); + next.push(Shot { + key: key.into(), + prompt: "the camera rises over the dunes".into(), + length: Duration::from_secs(5), + outcome: Outcome::Pending(Some(0.0)), + }); + set(&list, next, cx) + }), + ), + ), + ) +} diff --git a/src/chat/play.rs b/src/chat/play.rs index de933de..93f5c6a 100644 --- a/src/chat/play.rs +++ b/src/chat/play.rs @@ -3,7 +3,7 @@ use std::{rc::Rc, time::Duration}; use gpui::{ App, ElementId, InteractiveElement, IntoElement, MouseButton, ObjectFit, ParentElement, RenderOnce, SharedString, StatefulInteractiveElement, Styled, Window, div, prelude::*, - relative, + relative, transparent_black, }; use super::media::shaped; @@ -11,13 +11,16 @@ use crate::{ buttons::{ButtonVariant, IconButton}, documents::source, forms::OnFlag, - primitives::{Icon, IconName, Image}, + primitives::{FocusRing, Icon, IconName, Image, tab_stop}, theme::{ActiveTheme, ContainerSize, ControlSize, IconSize, Radius, TextSize}, typography::{DurationStyle, format::duration, tabular}, }; type OnSeek = Rc; +/// The share of a clip Left and Right move on its waveform. +const STEP: f32 = 0.05; + /// A clip's time as it plays: what played and the length, in clock digits. fn clock(played: Duration, length: Duration) -> String { format!( @@ -27,7 +30,7 @@ fn clock(played: Duration, length: Duration) -> String { ) } -/// A voice note or a song: play and pause, a waveform lit as far as it has played, and the time; a press on the waveform seeks there. The host plays it. +/// A voice note or a song: play and pause, a waveform lit as far as it has played, and the time; a press on the waveform seeks there, and so do Left and Right once it has focus. The host plays it. #[derive(IntoElement)] pub struct AudioMessage { id: ElementId, @@ -80,7 +83,9 @@ impl AudioMessage { } impl RenderOnce for AudioMessage { - fn render(self, _: &mut Window, cx: &mut App) -> impl IntoElement { + fn render(self, window: &mut Window, cx: &mut App) -> impl IntoElement { + let focus = (self.on_seek.as_ref()) + .map(|_| tab_stop((self.id.clone(), "wave").into(), true, window, cx)); let theme = cx.theme(); let colors = theme.colors.clone(); let share = if self.length.is_zero() { @@ -121,11 +126,31 @@ impl RenderOnce for AudioMessage { })) .child( div() + .id((self.id.clone(), "wave")) .flex_1() + .min_w_0() .h(tall) .flex() .items_center() .gap_px() + .rounded(theme.radius(Radius::Sm)) + .border_1() + .border_color(transparent_black()) + .when_some(focus.zip(self.on_seek.clone()), |wave, (focus, seek)| { + wave.track_focus(&focus).focus_ring(cx).on_key_down( + move |event, window, cx| { + let step = match event.keystroke.key.as_str() { + "left" => -STEP, + "right" => STEP, + _ => return, + }; + cx.stop_propagation(); + let next = (share + step).clamp(0.0, 1.0); + log::info!("audio message: seek to {next:.2}"); + seek(next, window, cx) + }, + ) + }) .children(self.peaks.iter().enumerate().map(|(ix, peak)| { let seek = self.on_seek.clone(); let lit = (ix as f32 + 0.5) / count as f32 <= share; @@ -156,6 +181,7 @@ impl RenderOnce for AudioMessage { .child( tabular( div() + .flex_none() .pr_2() .text_size(theme.text_size(TextSize::Xs)) .text_color(colors.fg_muted), diff --git a/src/debug/mod.rs b/src/debug/mod.rs index 673d0f1..f12cc8b 100644 --- a/src/debug/mod.rs +++ b/src/debug/mod.rs @@ -19,6 +19,7 @@ pub use hex::HexViewer; pub use network::{NetworkInspector, Request}; pub use session::{Breakpoint, BreakpointList, DebugCommand, DebugState, DebugToolbar}; pub use stack::{CallStack, StackFrame, Thread, ThreadList, ThreadState}; +pub(crate) use timeline::ruler; pub use timeline::{Span, TimelineProfiler, zoomed}; pub use variables::{ValueKind, Variable, VariablesPanel, Watch, WatchPanel}; diff --git a/src/debug/timeline.rs b/src/debug/timeline.rs index a4f35b0..847d804 100644 --- a/src/debug/timeline.rs +++ b/src/debug/timeline.rs @@ -35,7 +35,7 @@ fn tick_label(time: f64, step: f64) -> String { const TICKS: usize = 6; /// The ruler's step and ticks: as many as leave `room` for each label across `width`, each with room before the end. -fn ruler(range: (f64, f64), width: f32, room: f32) -> (f64, Vec) { +pub(crate) fn ruler(range: (f64, f64), width: f32, room: f32) -> (f64, Vec) { let length = range.1 - range.0; let across = move |span: f64| (span / length) as f32 * width; let step = (2..=TICKS) diff --git a/src/generative/audio.rs b/src/generative/audio.rs new file mode 100644 index 0000000..e0d5b3d --- /dev/null +++ b/src/generative/audio.rs @@ -0,0 +1,221 @@ +use std::{rc::Rc, time::Duration}; + +use gpui::{ + App, ElementId, IntoElement, ParentElement, RenderOnce, SharedString, Styled, Window, div, +}; + +use crate::{ + buttons::IconButton, + chat::AudioMessage, + forms::{OnFlag, Run}, + motion::ProgressBar, + primitives::{Icon, IconName}, + theme::{ActiveTheme, ControlSize, IconSize, Radius, TextSize}, + typography::{ + Ellipsis, + format::{DurationStyle, duration, percent}, + tabular, + }, +}; + +type OnSeek = Rc; + +/// A generated sound: its prompt, its model and length, a bar while it is made, then a player with its waveform. Regenerate and Download wait for it. The host plays it. +#[derive(IntoElement)] +pub struct AudioGenerationPlayer { + id: ElementId, + prompt: SharedString, + model: SharedString, + peaks: Vec, + length: Duration, + making: Option, + played: Duration, + playing: bool, + on_toggle: Option, + on_seek: Option, + on_regenerate: Option, + on_download: Option, +} + +impl AudioGenerationPlayer { + /// `peaks` are the waveform's heights, 0 to 1, left to right. + pub fn new( + id: impl Into, + prompt: impl Into, + model: impl Into, + peaks: impl Into>, + length: Duration, + ) -> Self { + let peaks: Vec = peaks.into(); + assert!( + !peaks.is_empty() && peaks.iter().all(|peak| (0.0..=1.0).contains(peak)), + "a sound's peaks lie within 0 to 1" + ); + Self { + id: id.into(), + prompt: prompt.into(), + model: model.into(), + peaks, + length, + making: None, + played: Duration::ZERO, + playing: false, + on_toggle: None, + on_seek: None, + on_regenerate: None, + on_download: None, + } + } + + /// While it is made, the share done. + pub fn making(mut self, share: f32) -> Self { + assert!((0.0..=1.0).contains(&share), "a sound made to {share} of 1"); + self.making = Some(share); + self + } + + /// How far it has played, and whether it plays now. + pub fn played(mut self, played: Duration, playing: bool) -> Self { + self.played = played; + self.playing = playing; + self + } + + /// Gets true to play, false to pause. + pub fn on_toggle(mut self, handler: impl Fn(bool, &mut Window, &mut App) + 'static) -> Self { + self.on_toggle = Some(Rc::new(handler)); + self + } + + /// Gets the share of the sound to seek to. + pub fn on_seek(mut self, handler: impl Fn(f32, &mut Window, &mut App) + 'static) -> Self { + self.on_seek = Some(Rc::new(handler)); + self + } + + pub fn on_regenerate(mut self, handler: impl Fn(&mut Window, &mut App) + 'static) -> Self { + self.on_regenerate = Some(Rc::new(handler)); + self + } + + pub fn on_download(mut self, handler: impl Fn(&mut Window, &mut App) + 'static) -> Self { + self.on_download = Some(Rc::new(handler)); + self + } +} + +impl RenderOnce for AudioGenerationPlayer { + fn render(self, _: &mut Window, cx: &mut App) -> impl IntoElement { + let theme = cx.theme(); + let colors = theme.colors.clone(); + let making = self.making; + let action = |key: &'static str, icon: IconName, tip: &'static str, run: Option| { + run.map(|run| { + IconButton::new((self.id.clone(), key), icon) + .size(ControlSize::Sm) + .tooltip(tip) + .disabled(making.is_some()) + .on_click(move |_, window, cx| { + log::info!("audio generation: {key}"); + run(window, cx) + }) + }) + }; + let body = match making { + Some(share) => div() + .flex() + .flex_col() + .gap_2() + .child(ProgressBar::new((self.id.clone(), "making"), share)) + .child( + tabular(div()) + .text_size(theme.text_size(TextSize::Xs)) + .text_color(colors.fg_muted) + .child(format!("Making · {}", percent(f64::from(share), 0, false))), + ) + .into_any_element(), + None => { + let player = + AudioMessage::new((self.id.clone(), "player"), self.peaks, self.length) + .played(self.played, self.playing); + let player = match self.on_toggle { + Some(toggle) => { + player.on_toggle(move |play, window, cx| toggle(play, window, cx)) + } + None => player, + }; + match self.on_seek { + Some(seek) => player.on_seek(move |share, window, cx| seek(share, window, cx)), + None => player, + } + .into_any_element() + } + }; + div() + .w_full() + .flex() + .flex_col() + .gap_3() + .p_4() + .rounded(theme.radius(Radius::Lg)) + .border_1() + .border_color(colors.border) + .bg(colors.surface) + .child( + div() + .flex() + .items_start() + .gap_3() + .child( + div().flex_none().pt_0p5().child( + Icon::new(IconName::AudioLines) + .size(IconSize::Sm) + .color(colors.fg_muted), + ), + ) + .child( + div() + .flex_1() + .min_w_0() + .flex() + .flex_col() + .gap_0p5() + .child( + div() + .text_size(theme.text_size(TextSize::Sm)) + .text_color(colors.fg) + .child(Ellipsis::new(self.prompt)), + ) + .child( + div() + .text_size(theme.text_size(TextSize::Xs)) + .text_color(colors.fg_muted) + .child(Ellipsis::new(format!( + "{} · {}", + self.model, + duration(self.length.as_secs(), DurationStyle::Clock) + ))), + ), + ) + .child( + div() + .flex_none() + .flex() + .gap_1() + .children(action( + "regenerate", + IconName::RefreshCw, + "Regenerate", + self.on_regenerate, + )) + .children(action( + "download", + IconName::Download, + "Download", + self.on_download, + )), + ), + ) + .child(body) + } +} diff --git a/src/generative/mask.rs b/src/generative/mask.rs index 236b58b..081d128 100644 --- a/src/generative/mask.rs +++ b/src/generative/mask.rs @@ -329,19 +329,21 @@ impl RenderOnce for InpaintCanvas { .size_full(), ) .children(ring) - .on_mouse_move(move |event, _, cx| { - hover.update(cx, |pointer, cx| { - *pointer = Some(event.position); - cx.notify(); - }) - }) - .on_hover(move |hovered, _, cx| { - if !hovered { - leave.update(cx, |pointer, cx| { - *pointer = None; + .when(drawn, |area| { + area.on_mouse_move(move |event, _, cx| { + hover.update(cx, |pointer, cx| { + *pointer = Some(event.position); cx.notify(); }) - } + }) + .on_hover(move |hovered, _, cx| { + if !hovered { + leave.update(cx, |pointer, cx| { + *pointer = None; + cx.notify(); + }) + } + }) }) .on_key_down(move |event, window, cx| { let Some(keys) = &keys else { diff --git a/src/generative/mod.rs b/src/generative/mod.rs index 5d53bbf..c34d44d 100644 --- a/src/generative/mod.rs +++ b/src/generative/mod.rs @@ -1,3 +1,4 @@ +mod audio; mod brush; mod compare; mod grid; @@ -10,7 +11,10 @@ mod style; #[cfg(all(test, feature = "test-support"))] mod tests; mod variation; +mod video; +mod voice; +pub use audio::AudioGenerationPlayer; pub use brush::MaskBrush; pub use compare::{ABCompareView, Verdict}; pub use grid::{GenerationGrid, Outcome}; @@ -21,3 +25,5 @@ pub use ratio::AspectRatioPicker; pub use seed::SeedInput; pub use style::{StylePreset, StylePresetPicker}; pub use variation::VariationPicker; +pub use video::{Shot, VideoGenerationTimeline}; +pub use voice::{TTSVoicePicker, Voice}; diff --git a/src/generative/tests/mod.rs b/src/generative/tests/mod.rs index c891dde..8985404 100644 --- a/src/generative/tests/mod.rs +++ b/src/generative/tests/mod.rs @@ -5,6 +5,7 @@ use crate::{forms, primitives::FocusNext, theme::Theme}; mod canvas; mod results; mod settings; +mod sound; fn settle(cx: &mut VisualTestContext) { cx.run_until_parked(); diff --git a/src/generative/tests/sound.rs b/src/generative/tests/sound.rs new file mode 100644 index 0000000..e105f05 --- /dev/null +++ b/src/generative/tests/sound.rs @@ -0,0 +1,187 @@ +use std::{cell::RefCell, rc::Rc, time::Duration}; + +use gpui::{ + Context, IntoElement, Modifiers, ParentElement, Render, Styled, TestAppContext, Window, div, + point, px, +}; + +use super::{press, settle, setup, tab}; +use crate::generative::{ + AudioGenerationPlayer, Outcome, Shot, TTSVoicePicker, VideoGenerationTimeline, Voice, +}; + +type Heard = Rc>>; + +/// A made sound a quarter played and one still being made, and what they heard. +struct Players(Heard); + +impl Render for Players { + fn render(&mut self, _: &mut Window, _: &mut Context) -> impl IntoElement { + let (sought, made, making) = (self.0.clone(), self.0.clone(), self.0.clone()); + let length = Duration::from_secs(40); + div() + .w(px(420.0)) + .child( + AudioGenerationPlayer::new("made", "rain", "Orchid", vec![0.5; 8], length) + .played(Duration::from_secs(10), false) + .on_toggle(|_, _, _| {}) + .on_seek(move |share, _, _| { + sought.borrow_mut().push(format!("seek {share:.2}")) + }) + .on_regenerate(move |_, _| made.borrow_mut().push("regenerate made".into())), + ) + .child( + AudioGenerationPlayer::new("making", "cello", "Orchid", vec![0.5; 8], length) + .making(0.4) + .on_regenerate(move |_, _| { + making.borrow_mut().push("regenerate making".into()) + }), + ) + } +} + +#[gpui::test] +fn the_waveform_seeks_with_keys_and_a_sound_being_made_waits(cx: &mut TestAppContext) { + setup(cx); + let heard = Heard::default(); + let store = heard.clone(); + let (_, cx) = cx.add_window_view(|_, _| Players(store)); + cx.update(|window, _| window.activate_window()); + settle(cx); + tab(1, cx); + press("enter", cx); + tab(2, cx); + press("right", cx); + tab(1, cx); + press("enter", cx); + let heard = heard.borrow().clone(); + assert_eq!( + heard, + ["regenerate made", "seek 0.30", "regenerate made"], + "Tab passes the waiting button" + ); +} + +/// Two voices, the first chosen, and what they heard. +struct Voices(Heard); + +impl Render for Voices { + fn render(&mut self, _: &mut Window, _: &mut Context) -> impl IntoElement { + let (chosen, played) = (self.0.clone(), self.0.clone()); + let voice = |key: &str| Voice { + key: key.to_string().into(), + name: key.to_string().into(), + about: "a voice".into(), + tags: vec!["English".into()], + }; + div().w(px(420.0)).child( + TTSVoicePicker::new("voices", [voice("iris"), voice("theo")]) + .selected("iris") + .on_select(move |key, _, _| chosen.borrow_mut().push(format!("choose {key}"))) + .on_play(move |key, _, _| played.borrow_mut().push(format!("play {key:?}"))), + ) + } +} + +#[gpui::test] +fn a_sample_plays_without_choosing_its_voice(cx: &mut TestAppContext) { + setup(cx); + let heard = Heard::default(); + let store = heard.clone(); + let (_, cx) = cx.add_window_view(|_, _| Voices(store)); + cx.update(|window, _| window.activate_window()); + settle(cx); + tab(3, cx); + press("enter", cx); + cx.simulate_click(point(px(23.0), px(83.0)), Modifiers::none()); + settle(cx); + assert_eq!( + *heard.borrow(), + ["choose theo", "play Some(\"theo\")"], + "a sample's press stays off its row" + ); +} + +/// Two shots, the first chosen, and what they heard. +struct Timeline(Heard); + +impl Render for Timeline { + fn render(&mut self, _: &mut Window, _: &mut Context) -> impl IntoElement { + let (chosen, extended) = (self.0.clone(), self.0.clone()); + let shot = |key: &str, outcome| Shot { + key: key.to_string().into(), + prompt: "a pan".into(), + length: Duration::from_secs(4), + outcome, + }; + div().w(px(420.0)).child( + VideoGenerationTimeline::new( + "timeline", + [ + shot("a", Outcome::Done("missing.jpg".into())), + shot("b", Outcome::Pending(None)), + ], + Duration::ZERO, + ) + .selected("a") + .on_select(move |key, _, _| chosen.borrow_mut().push(format!("choose {key}"))) + .on_extend(move |_, _| extended.borrow_mut().push("extend".into())), + ) + } +} + +#[gpui::test] +fn a_shot_is_chosen_once_and_extend_asks_for_the_next(cx: &mut TestAppContext) { + setup(cx); + let heard = Heard::default(); + let store = heard.clone(); + let (_, cx) = cx.add_window_view(|_, _| Timeline(store)); + cx.update(|window, _| window.activate_window()); + settle(cx); + for _ in 0..3 { + tab(1, cx); + press("enter", cx); + } + assert_eq!(*heard.borrow(), ["choose b", "extend"]); +} + +/// A voice list and a timeline shown to read, with nothing to take a choice. +struct Shown; + +impl Render for Shown { + fn render(&mut self, _: &mut Window, _: &mut Context) -> impl IntoElement { + let voice = Voice { + key: "iris".into(), + name: "Iris".into(), + about: "a voice".into(), + tags: Vec::new(), + }; + let shot = Shot { + key: "a".into(), + prompt: "a pan".into(), + length: Duration::from_secs(4), + outcome: Outcome::Pending(None), + }; + div() + .w(px(420.0)) + .child(TTSVoicePicker::new("voices", [voice]).selected("iris")) + .child(VideoGenerationTimeline::new("timeline", [shot], Duration::ZERO).selected("a")) + } +} + +#[gpui::test] +fn lists_shown_to_read_take_no_focus(cx: &mut TestAppContext) { + setup(cx); + let (_, cx) = cx.add_window_view(|_, _| Shown); + cx.update(|window, _| window.activate_window()); + settle(cx); + tab(1, cx); + assert!(cx.update(|window, cx| window.focused(cx).is_none())); +} + +#[test] +#[should_panic(expected = "a sound's peaks lie within 0 to 1")] +fn a_peak_past_full_fails_loud() { + let _ = + AudioGenerationPlayer::new("sound", "rain", "Orchid", vec![1.5], Duration::from_secs(1)); +} diff --git a/src/generative/video.rs b/src/generative/video.rs new file mode 100644 index 0000000..10a7a89 --- /dev/null +++ b/src/generative/video.rs @@ -0,0 +1,288 @@ +use std::{rc::Rc, time::Duration}; + +use gpui::{ + App, ElementId, InteractiveElement, IntoElement, MouseButton, ParentElement, RenderOnce, + SharedString, StatefulInteractiveElement, Styled, Window, div, prelude::*, relative, + transparent_black, +}; + +use super::Outcome; +use crate::{ + buttons::{Button, ButtonVariant}, + debug::ruler, + documents::source, + forms::{OnValue, Run}, + layout::on_axis, + motion::Skeleton, + primitives::{Icon, IconName, Image, tab_stop}, + theme::{ActiveTheme, ControlSize, IconSize, TextSize}, + typography::{ + Caption, + format::{DurationStyle, duration, percent}, + tabular, + }, +}; + +/// A shot of a generated video: its key, its prompt, how long it runs, and how it came out. +#[derive(Clone, Debug, PartialEq)] +pub struct Shot { + pub key: SharedString, + pub prompt: SharedString, + pub length: Duration, + pub outcome: Outcome, +} + +/// A video made shot by shot, laid along its time: a ruler, each shot as wide as it runs with its poster, a breathing placeholder while it comes or a mark if it failed, and the playhead. A press or Enter chooses a shot and a line under names it; Extend asks for the next. It scrolls sideways when long. +#[derive(IntoElement)] +pub struct VideoGenerationTimeline { + id: ElementId, + shots: Vec, + at: Duration, + selected: Option, + on_select: Option, + on_extend: Option, +} + +impl VideoGenerationTimeline { + /// `at` is the playhead's time. + pub fn new( + id: impl Into, + shots: impl IntoIterator, + at: Duration, + ) -> Self { + let shots: Vec = shots.into_iter().collect(); + for shot in &shots { + assert!(!shot.length.is_zero(), "shot {} runs no time", shot.key); + if let Outcome::Pending(Some(share)) = shot.outcome { + assert!( + (0.0..=1.0).contains(&share), + "shot {} at {share} of 1", + shot.key + ); + } + } + Self { + id: id.into(), + shots, + at, + selected: None, + on_select: None, + on_extend: None, + } + } + + pub fn selected(mut self, key: impl Into) -> Self { + self.selected = Some(key.into()); + self + } + + pub fn on_select( + mut self, + handler: impl Fn(&SharedString, &mut Window, &mut App) + 'static, + ) -> Self { + self.on_select = Some(Rc::new(handler)); + self + } + + /// Asks for one more shot after the last. + pub fn on_extend(mut self, handler: impl Fn(&mut Window, &mut App) + 'static) -> Self { + self.on_extend = Some(Rc::new(handler)); + self + } +} + +impl RenderOnce for VideoGenerationTimeline { + fn render(self, window: &mut Window, cx: &mut App) -> impl IntoElement { + let named = (self.selected.as_ref()) + .is_none_or(|key| self.shots.iter().any(|shot| &shot.key == key)); + assert!(named, "the chosen shot is not listed"); + let pickable = self.on_select.is_some(); + let focuses: Vec<_> = self + .shots + .iter() + .map(|shot| { + let focus = tab_stop( + (self.id.clone(), format!("focus-{}", shot.key)).into(), + pickable, + window, + cx, + ); + let focused = focus.is_focused(window); + (focus, focused) + }) + .collect(); + let theme = cx.theme(); + let colors = theme.colors.clone(); + let sizes = theme.generative(); + let rem = window.rem_size(); + let total: f64 = self + .shots + .iter() + .map(|shot| shot.length.as_secs_f64()) + .sum(); + let second = sizes.second; + let span = second * total as f32; + let (_, ticks) = ruler( + (0.0, total.max(1.0)), + f32::from(span.to_pixels(rem)), + f32::from(theme.chart().label.to_pixels(rem)), + ); + let caption = (self.selected.as_ref()) + .and_then(|key| self.shots.iter().position(|shot| &shot.key == key)) + .map(|ix| { + let shot = &self.shots[ix]; + let state = match &shot.outcome { + Outcome::Pending(_) => "coming".to_string(), + Outcome::Done(_) => duration(shot.length.as_secs(), DurationStyle::Compact), + Outcome::Failed(reason) => format!("failed, {reason}"), + }; + format!("Shot {} · {state} · {}", ix + 1, shot.prompt) + }); + let ruler_row = div() + .relative() + .h(theme.text_size(TextSize::Xs) * 1.6) + .w(span) + .border_b_1() + .border_color(colors.border) + .text_size(theme.text_size(TextSize::Xs)) + .text_color(colors.fg_subtle) + .children(ticks.into_iter().map(|time| { + tabular(div()) + .absolute() + .top_0() + .left(relative((time / total.max(1.0)) as f32)) + .pl_1() + .border_l_1() + .border_color(colors.border) + .child(duration(time as u64, DurationStyle::Clock)) + })); + let tiles = self.shots.into_iter().zip(focuses).enumerate().map( + |(ix, (shot, (focus, focused)))| { + let chosen = self.selected.as_ref() == Some(&shot.key); + let (pick, key) = (self.on_select.clone(), shot.key.clone()); + let face = match &shot.outcome { + Outcome::Done(poster) => { + Image::new((self.id.clone(), format!("poster-{ix}")), source(poster)) + .size_full() + .into_any_element() + } + Outcome::Pending(share) => div() + .relative() + .size_full() + .child( + Skeleton::new((self.id.clone(), format!("pending-{ix}"))).size_full(), + ) + .children(share.map(|share| { + div() + .absolute() + .inset_0() + .flex() + .items_center() + .justify_center() + .text_size(theme.text_size(TextSize::Xs)) + .text_color(colors.fg_muted) + .child(tabular(div()).child(percent(f64::from(share), 0, false))) + })) + .into_any_element(), + Outcome::Failed(_) => div() + .size_full() + .flex() + .items_center() + .justify_center() + .bg(colors.sunken) + .child( + Icon::new(IconName::CircleAlert) + .size(IconSize::Sm) + .color(colors.danger), + ) + .into_any_element(), + }; + div() + .id((self.id.clone(), format!("shot-{ix}"))) + .flex_none() + .w(second * shot.length.as_secs_f32()) + .h(sizes.shot) + .p_0p5() + .border_1() + .border_color(if focused { + colors.focus + } else if chosen { + colors.accent + } else { + transparent_black() + }) + .when_some(pick, |tile, pick| { + tile.track_focus(&focus) + .cursor_pointer() + .on_mouse_down(MouseButton::Left, |_, window, _| { + window.prevent_default() + }) + .on_click(move |_, window, cx| { + if chosen { + return; + } + log::info!("video timeline: chose {key}"); + pick(&key, window, cx) + }) + }) + .child( + div() + .relative() + .size_full() + .overflow_hidden() + .bg(colors.sunken) + .child(face), + ) + }, + ); + let playhead = (self.at.as_secs_f64() / total.max(1.0)).clamp(0.0, 1.0) as f32; + let extend = self.on_extend.map(|extend| { + Button::new((self.id.clone(), "extend"), "Extend") + .variant(ButtonVariant::Secondary) + .size(ControlSize::Sm) + .icon(IconName::Plus) + .on_click(move |_, window, cx| { + log::info!("video timeline: extend"); + extend(window, cx) + }) + }); + div() + .w_full() + .flex() + .flex_col() + .gap_2() + .child( + on_axis(div().id((self.id.clone(), "sideways"))) + .w_full() + .overflow_x_scroll() + .child( + div() + .flex() + .items_end() + .gap_3() + .pb_1() + .child( + div() + .relative() + .flex_none() + .flex() + .flex_col() + .gap_1() + .child(ruler_row) + .child(div().flex().children(tiles)) + .child( + div() + .absolute() + .top_0() + .bottom_0() + .left(relative(playhead)) + .border_l_1() + .border_color(colors.accent), + ), + ) + .children(extend.map(|extend| div().flex_none().child(extend))), + ), + ) + .children(caption.map(Caption::new)) + } +} diff --git a/src/generative/voice.rs b/src/generative/voice.rs new file mode 100644 index 0000000..2a7d2e7 --- /dev/null +++ b/src/generative/voice.rs @@ -0,0 +1,227 @@ +use std::rc::Rc; + +use gpui::{ + App, ElementId, FontWeight, InteractiveElement, IntoElement, MouseButton, ParentElement, + RenderOnce, SharedString, StatefulInteractiveElement, Styled, Window, div, prelude::*, + transparent_black, +}; + +use crate::{ + buttons::{ButtonVariant, IconButton}, + data_display::Badge, + forms::OnValue, + primitives::{Icon, IconName, tab_stop}, + theme::{ActiveTheme, ControlSize, IconSize, Radius, TextSize}, + typography::Ellipsis, +}; + +/// A voice to speak with: its key, its name, a line on how it sounds, and tags such as its language. +#[derive(Clone, Debug, PartialEq)] +pub struct Voice { + pub key: SharedString, + pub name: SharedString, + pub about: SharedString, + pub tags: Vec, +} + +type OnPlay = Rc, &mut Window, &mut App)>; + +/// Voices to speak with, a row each: a sample to play, the name and how it sounds, and its tags, which drop below when narrow. A press or Enter chooses a row; the chosen one takes the active wash and a check. The host plays the samples. +#[derive(IntoElement)] +pub struct TTSVoicePicker { + id: ElementId, + voices: Vec, + selected: Option, + playing: Option, + on_select: Option, + on_play: Option, +} + +impl TTSVoicePicker { + pub fn new(id: impl Into, voices: impl IntoIterator) -> Self { + Self { + id: id.into(), + voices: voices.into_iter().collect(), + selected: None, + playing: None, + on_select: None, + on_play: None, + } + } + + pub fn selected(mut self, key: impl Into) -> Self { + self.selected = Some(key.into()); + self + } + + /// The voice whose sample plays now. + pub fn playing(mut self, key: impl Into) -> Self { + self.playing = Some(key.into()); + self + } + + pub fn on_select( + mut self, + handler: impl Fn(&SharedString, &mut Window, &mut App) + 'static, + ) -> Self { + self.on_select = Some(Rc::new(handler)); + self + } + + /// Gets the voice whose sample to play, or none to stop. + pub fn on_play( + mut self, + handler: impl Fn(Option, &mut Window, &mut App) + 'static, + ) -> Self { + self.on_play = Some(Rc::new(handler)); + self + } +} + +impl RenderOnce for TTSVoicePicker { + fn render(self, window: &mut Window, cx: &mut App) -> impl IntoElement { + let known = |key: &Option| { + key.as_ref() + .is_none_or(|key| self.voices.iter().any(|voice| &voice.key == key)) + }; + assert!( + known(&self.selected) && known(&self.playing), + "a voice named is not listed" + ); + let pickable = self.on_select.is_some(); + let focuses: Vec<_> = self + .voices + .iter() + .map(|voice| { + let focus = tab_stop( + (self.id.clone(), format!("focus-{}", voice.key)).into(), + pickable, + window, + cx, + ); + let focused = focus.is_focused(window); + (focus, focused) + }) + .collect(); + let theme = cx.theme(); + let colors = theme.colors.clone(); + let label = theme.label_width(); + let rows = self + .voices + .into_iter() + .zip(focuses) + .map(|(voice, (focus, focused))| { + let chosen = self.selected.as_ref() == Some(&voice.key); + let sounding = self.playing.as_ref() == Some(&voice.key); + let (pick, play, key) = ( + self.on_select.clone(), + self.on_play.clone(), + voice.key.clone(), + ); + let sample = play.map(|play| { + let key = key.clone(); + IconButton::new( + (self.id.clone(), format!("play-{}", voice.key)), + if sounding { + IconName::CircleStop + } else { + IconName::Play + }, + ) + .variant(ButtonVariant::Ghost) + .size(ControlSize::Sm) + .tooltip(if sounding { + "Stop the sample" + } else { + "Play a sample" + }) + .on_click(move |_, window, cx| { + cx.stop_propagation(); + let next = (!sounding).then(|| key.clone()); + log::info!("voice picker: sample {next:?}"); + play(next, window, cx) + }) + }); + div() + .id((self.id.clone(), format!("voice-{}", voice.key))) + .w_full() + .flex() + .flex_wrap() + .items_center() + .gap_x_3() + .gap_y_1() + .px_2() + .py_2() + .rounded(theme.radius(Radius::Md)) + .border_1() + .border_color(if focused { + colors.focus + } else { + transparent_black() + }) + .when(chosen, |row| row.bg(colors.active)) + .when_some(pick, |row, pick| { + row.track_focus(&focus) + .when(!chosen, |row| row.hover(|style| style.bg(colors.hover))) + .cursor_pointer() + .on_mouse_down(MouseButton::Left, |_, window, _| { + window.prevent_default() + }) + .on_click(move |_, window, cx| { + if chosen { + return; + } + log::info!("voice picker: chose {key}"); + pick(&key, window, cx) + }) + }) + .child( + div() + .flex_1() + .min_w(label) + .flex() + .items_center() + .gap_2() + .children(sample.map(|sample| div().flex_none().child(sample))) + .child( + div() + .flex_1() + .min_w_0() + .flex() + .flex_col() + .child( + div() + .text_size(theme.text_size(TextSize::Sm)) + .font_weight(FontWeight::MEDIUM) + .text_color(colors.fg) + .child(Ellipsis::new(voice.name)), + ) + .child( + div() + .text_size(theme.text_size(TextSize::Xs)) + .text_color(colors.fg_muted) + .child(Ellipsis::new(voice.about)), + ), + ) + .when(chosen, |name| { + name.child( + div().flex_none().child( + Icon::new(IconName::Check) + .size(IconSize::Sm) + .color(colors.accent), + ), + ) + }), + ) + .child( + div() + .flex_none() + .flex() + .flex_wrap() + .gap_1() + .children(voice.tags.into_iter().map(Badge::new)), + ) + }); + div().w_full().flex().flex_col().gap_0p5().children(rows) + } +} diff --git a/src/primitives/icon.rs b/src/primitives/icon.rs index c239b6f..52c9dd4 100644 --- a/src/primitives/icon.rs +++ b/src/primitives/icon.rs @@ -56,6 +56,7 @@ icons! { ArrowUpRight => "arrow-up-right", ArrowUp => "arrow-up", AtSign => "at-sign", + AudioLines => "audio-lines", Ban => "ban", Battery => "battery", Bell => "bell", diff --git a/src/theme/generative.rs b/src/theme/generative.rs index 648098d..55203a6 100644 --- a/src/theme/generative.rs +++ b/src/theme/generative.rs @@ -7,12 +7,18 @@ use super::{Theme, tokens::px_to_rems}; pub struct GenerativeSizes { /// A style preset's picture, square. pub preset: Rems, + /// A second of video along a timeline. + pub second: Rems, + /// A shot's height on a timeline. + pub shot: Rems, } impl Theme { pub fn generative(&self) -> GenerativeSizes { GenerativeSizes { preset: px_to_rems(88.0), + second: px_to_rems(24.0), + shot: px_to_rems(56.0), } } } diff --git a/tasks/ch21-30.md b/tasks/ch21-30.md index 76a161e..420a0e3 100644 --- a/tasks/ch21-30.md +++ b/tasks/ch21-30.md @@ -232,9 +232,9 @@ ## T25d Generative · Sound and motion -- [ ] AudioGenerationPlayer -- [ ] VideoGenerationTimeline -- [ ] TTSVoicePicker +- [x] AudioGenerationPlayer +- [x] VideoGenerationTimeline +- [x] TTSVoicePicker ## T25e Generative · Models