From 7178174e202fe9468aafb2c009f8e40047651476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Fri, 8 May 2026 13:54:56 -0400 Subject: [PATCH] Add Servo scroll snapshots --- crates/ely_app/src/services/servo_sidecar.rs | 15 +- crates/ely_app/src/shell/mod.rs | 3 + crates/ely_app/src/shell/web_surface.rs | 407 +++++++----------- crates/ely_app/src/shell/web_surface_frame.rs | 79 ++++ .../ely_app/src/shell/web_surface_geometry.rs | 95 ++++ crates/ely_app/src/shell/web_surface_view.rs | 165 +++++++ .../src/bin/ely_servo_sidecar.rs | 96 ++++- crates/ely_servo_host/src/host.rs | 9 + crates/ely_servo_host/src/lib.rs | 2 +- crates/ely_servo_host/src/runtime.rs | 29 +- crates/ely_servo_host/tests/sidecar.rs | 96 ++++- crates/ely_servo_host/tests/software_host.rs | 10 +- 12 files changed, 738 insertions(+), 268 deletions(-) create mode 100644 crates/ely_app/src/shell/web_surface_frame.rs create mode 100644 crates/ely_app/src/shell/web_surface_geometry.rs create mode 100644 crates/ely_app/src/shell/web_surface_view.rs diff --git a/crates/ely_app/src/services/servo_sidecar.rs b/crates/ely_app/src/services/servo_sidecar.rs index 1302f6b..37ad235 100644 --- a/crates/ely_app/src/services/servo_sidecar.rs +++ b/crates/ely_app/src/services/servo_sidecar.rs @@ -74,6 +74,10 @@ impl ServoSidecarClient { .arg(request.width.to_string()) .arg("--height") .arg(request.height.to_string()) + .arg("--scroll-x") + .arg(request.scroll_x.to_string()) + .arg("--scroll-y") + .arg(request.scroll_y.to_string()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() @@ -103,12 +107,21 @@ pub struct SidecarSnapshotRequest { url: UrlText, width: u32, height: u32, + scroll_x: i32, + scroll_y: i32, } impl SidecarSnapshotRequest { #[must_use] pub fn new(url: UrlText, width: u32, height: u32) -> Self { - Self { url, width, height } + Self { url, width, height, scroll_x: 0, scroll_y: 0 } + } + + #[must_use] + pub fn with_scroll_offset(mut self, scroll_x: i32, scroll_y: i32) -> Self { + self.scroll_x = scroll_x; + self.scroll_y = scroll_y; + self } } diff --git a/crates/ely_app/src/shell/mod.rs b/crates/ely_app/src/shell/mod.rs index b24cef8..9ea0242 100644 --- a/crates/ely_app/src/shell/mod.rs +++ b/crates/ely_app/src/shell/mod.rs @@ -15,7 +15,10 @@ mod splits; mod tab_groups; mod tab_lifecycle; mod web_surface; +mod web_surface_frame; +mod web_surface_geometry; mod web_surface_image; +mod web_surface_view; use ely_browser_core::{BrowserCore, InitialBrowserConfig}; use ely_domain::{ diff --git a/crates/ely_app/src/shell/web_surface.rs b/crates/ely_app/src/shell/web_surface.rs index e8bac9f..5c5c5fa 100644 --- a/crates/ely_app/src/shell/web_surface.rs +++ b/crates/ely_app/src/shell/web_surface.rs @@ -1,25 +1,25 @@ -use std::{collections::BTreeMap, sync::Arc}; +use std::collections::BTreeMap; use ely_domain::{BrowserTab, TabId}; -use gpui::{ - AnyElement, App, Bounds, Context, Entity, ImageSource, IntoElement, ObjectFit, ParentElement, - Pixels, RenderImage, Styled, StyledImage, Window, canvas, div, img, prelude::FluentBuilder, px, - rgb, -}; -use gpui_component::StyledExt; -use image::{ImageBuffer, Rgba}; -use thiserror::Error; +use gpui::{AnyElement, Bounds, Context, Pixels}; use crate::services::servo_sidecar::{ ServoSidecarClient, ServoSidecarError, SidecarSnapshot, SidecarSnapshotRequest, }; -use super::{ElyShell, web_surface_image::renderable_image_buffer}; -use ely_design_system::{colors, spacing}; +use super::{ + ElyShell, + web_surface_frame::WebSurfaceFrame, + web_surface_geometry::{WebSurfaceScrollDelta, WebSurfaceScrollOffset, WebSurfaceSize}, + web_surface_view::{ + render_failed_web_surface, render_loading_web_surface, render_ready_web_surface, + }, +}; pub(super) struct WebSurfaceStore { client: WebSurfaceClient, pending_viewport_sizes: BTreeMap, + scroll_offsets: BTreeMap, viewport_sizes: BTreeMap, states: BTreeMap, } @@ -29,6 +29,7 @@ impl WebSurfaceStore { Self { client: WebSurfaceClient::new(), pending_viewport_sizes: BTreeMap::new(), + scroll_offsets: BTreeMap::new(), viewport_sizes: BTreeMap::new(), states: BTreeMap::new(), } @@ -45,10 +46,11 @@ impl WebSurfaceStore { let size = self.viewport_sizes.get(tab.id()).copied()?; let requested_url = tab.url().as_str().to_string(); + let scroll_offset = self.scroll_offset_for(tab.id(), requested_url.as_str()); if self.is_loading_requested_url(tab.id(), requested_url.as_str()) { return None; } - if self.has_current_state(tab.id(), &requested_url, size) { + if self.has_current_state(tab.id(), &requested_url, size, scroll_offset) { return None; } @@ -57,7 +59,12 @@ impl WebSurfaceStore { WebSurfaceClient::Unavailable(message) => { self.states.insert( tab.id().clone(), - WebSurfaceState::Failed { requested_url, size, message: message.clone() }, + WebSurfaceState::Failed { + requested_url, + size, + scroll_offset, + message: message.clone(), + }, ); return None; } @@ -68,6 +75,7 @@ impl WebSurfaceStore { WebSurfaceState::Loading { requested_url: requested_url.clone(), size, + scroll_offset, previous_frame: self.previous_ready_frame(tab.id(), requested_url.as_str()), }, ); @@ -76,43 +84,72 @@ impl WebSurfaceStore { tab_id: tab.id().clone(), requested_url, size, + scroll_offset, client, snapshot_request: SidecarSnapshotRequest::new( tab.url().clone(), size.width, size.height, - ), + ) + .with_scroll_offset(scroll_offset.x(), scroll_offset.y()), }) } - fn has_current_state(&self, tab_id: &TabId, requested_url: &str, size: WebSurfaceSize) -> bool { + fn has_current_state( + &self, + tab_id: &TabId, + requested_url: &str, + size: WebSurfaceSize, + scroll_offset: WebSurfaceScrollOffset, + ) -> bool { match self.states.get(tab_id) { Some(WebSurfaceState::Loading { requested_url: current_url, size: current_size, + scroll_offset: current_scroll_offset, .. - }) => current_url == requested_url && *current_size == size, + }) => { + current_url == requested_url + && *current_size == size + && *current_scroll_offset == scroll_offset + } Some(WebSurfaceState::Ready(frame)) => { - frame.requested_url == requested_url && frame.size() == size + frame.requested_url == requested_url + && frame.size() == size + && frame.scroll_offset() == scroll_offset } Some(WebSurfaceState::Failed { requested_url: current_url, size: current_size, + scroll_offset: current_scroll_offset, .. - }) => current_url == requested_url && *current_size == size, + }) => { + current_url == requested_url + && *current_size == size + && *current_scroll_offset == scroll_offset + } None => false, } } - fn is_loading(&self, tab_id: &TabId, requested_url: &str, size: WebSurfaceSize) -> bool { + fn is_loading( + &self, + tab_id: &TabId, + requested_url: &str, + size: WebSurfaceSize, + scroll_offset: WebSurfaceScrollOffset, + ) -> bool { matches!( self.states.get(tab_id), Some(WebSurfaceState::Loading { requested_url: current_url, size: current_size, + scroll_offset: current_scroll_offset, .. }) - if current_url == requested_url && *current_size == size + if current_url == requested_url + && *current_size == size + && *current_scroll_offset == scroll_offset ) } @@ -140,6 +177,33 @@ impl WebSurfaceStore { self.states.insert(tab_id, state); } + fn record_scroll_delta( + &mut self, + tab_id: &TabId, + requested_url: &str, + delta: gpui::Point, + ) -> bool { + let Some(delta) = WebSurfaceScrollDelta::from_point(delta) else { + return false; + }; + + let state = self + .scroll_offsets + .entry(tab_id.clone()) + .or_insert_with(|| WebSurfaceScrollState::new(requested_url.to_string())); + if state.requested_url != requested_url { + *state = WebSurfaceScrollState::new(requested_url.to_string()); + } + + let next_offset = state.offset.scrolled_by(delta); + if next_offset == state.offset { + return false; + } + + state.offset = next_offset; + true + } + fn record_viewport_size(&mut self, tab_id: &TabId, bounds: Bounds) -> bool { let Some(size) = WebSurfaceSize::from_bounds(bounds) else { return false; @@ -165,6 +229,25 @@ impl WebSurfaceStore { self.viewport_sizes.insert(tab_id.clone(), size); true } + + fn scroll_offset_for(&self, tab_id: &TabId, requested_url: &str) -> WebSurfaceScrollOffset { + self.scroll_offsets + .get(tab_id) + .filter(|state| state.requested_url == requested_url) + .map(|state| state.offset) + .unwrap_or_default() + } +} + +struct WebSurfaceScrollState { + requested_url: String, + offset: WebSurfaceScrollOffset, +} + +impl WebSurfaceScrollState { + fn new(requested_url: String) -> Self { + Self { requested_url, offset: WebSurfaceScrollOffset::default() } + } } enum WebSurfaceClient { @@ -185,85 +268,25 @@ struct WebSurfaceRequest { tab_id: TabId, requested_url: String, size: WebSurfaceSize, + scroll_offset: WebSurfaceScrollOffset, client: ServoSidecarClient, snapshot_request: SidecarSnapshotRequest, } enum WebSurfaceState { - Loading { requested_url: String, size: WebSurfaceSize, previous_frame: Option }, - Ready(WebSurfaceFrame), - Failed { requested_url: String, size: WebSurfaceSize, message: String }, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct WebSurfaceSize { - width: u32, - height: u32, -} - -impl WebSurfaceSize { - fn from_bounds(bounds: Bounds) -> Option { - Some(Self { - width: viewport_dimension(bounds.size.width)?, - height: viewport_dimension(bounds.size.height)?, - }) - } -} - -#[derive(Clone)] -struct WebSurfaceFrame { - requested_url: String, - loaded_url: Option, - title: Option, - width: u32, - height: u32, - image: Arc, -} - -impl WebSurfaceFrame { - fn from_snapshot( + Loading { requested_url: String, - snapshot: SidecarSnapshot, - ) -> Result { - let width = snapshot.width(); - let height = snapshot.height(); - let loaded_url = snapshot.loaded_url().map(str::to_string); - let title = snapshot.title().map(str::to_string); - let rgba_bytes = snapshot.into_rgba_bytes(); - - let Some(buffer) = ImageBuffer::, _>::from_raw(width, height, rgba_bytes) else { - return Err(WebSurfaceError::InvalidFrameBuffer { width, height }); - }; - - let image_buffer = renderable_image_buffer(buffer); - - Ok(Self { - requested_url, - loaded_url, - title, - width, - height, - image: Arc::new(RenderImage::new([image::Frame::new(image_buffer)])), - }) - } - - fn title_label(&self) -> String { - self.title.clone().unwrap_or_else(|| self.requested_url.clone()) - } - - fn url_label(&self) -> &str { - self.loaded_url.as_deref().unwrap_or(self.requested_url.as_str()) - } - - fn size(&self) -> WebSurfaceSize { - WebSurfaceSize { width: self.width, height: self.height } - } -} - -#[derive(Debug, Error)] -enum WebSurfaceError { - #[error("invalid servo frame buffer for {width}x{height}")] - InvalidFrameBuffer { width: u32, height: u32 }, + size: WebSurfaceSize, + scroll_offset: WebSurfaceScrollOffset, + previous_frame: Option, + }, + Ready(WebSurfaceFrame), + Failed { + requested_url: String, + size: WebSurfaceSize, + scroll_offset: WebSurfaceScrollOffset, + message: String, + }, } impl ElyShell { @@ -296,7 +319,14 @@ impl ElyShell { return; }; - let WebSurfaceRequest { tab_id, requested_url, size, client, snapshot_request } = request; + let WebSurfaceRequest { + tab_id, + requested_url, + size, + scroll_offset, + client, + snapshot_request, + } = request; cx.spawn(async move |shell, cx| { let result = cx .background_executor() @@ -304,7 +334,13 @@ impl ElyShell { .await; _ = shell.update(cx, |shell, cx| { - shell.handle_external_web_frame_result(tab_id, requested_url, size, result); + shell.handle_external_web_frame_result( + tab_id, + requested_url, + size, + scroll_offset, + result, + ); cx.notify(); }); }) @@ -316,27 +352,37 @@ impl ElyShell { tab_id: TabId, requested_url: String, size: WebSurfaceSize, + scroll_offset: WebSurfaceScrollOffset, result: Result, ) { - if !self.web_surfaces.is_loading(&tab_id, requested_url.as_str(), size) { + if !self.web_surfaces.is_loading(&tab_id, requested_url.as_str(), size, scroll_offset) { return; } let state = match result { - Ok(snapshot) => match WebSurfaceFrame::from_snapshot(requested_url.clone(), snapshot) { - Ok(frame) => WebSurfaceState::Ready(frame), - Err(error) => { - WebSurfaceState::Failed { requested_url, size, message: error.to_string() } + Ok(snapshot) => { + match WebSurfaceFrame::from_snapshot(requested_url.clone(), scroll_offset, snapshot) + { + Ok(frame) => WebSurfaceState::Ready(frame), + Err(error) => WebSurfaceState::Failed { + requested_url, + size, + scroll_offset, + message: error.to_string(), + }, } - }, - Err(error) => { - WebSurfaceState::Failed { requested_url, size, message: error.to_string() } } + Err(error) => WebSurfaceState::Failed { + requested_url, + size, + scroll_offset, + message: error.to_string(), + }, }; self.web_surfaces.finish(tab_id, state); } - fn record_external_web_viewport( + pub(super) fn record_external_web_viewport( &mut self, tab_id: TabId, bounds: Bounds, @@ -346,151 +392,18 @@ impl ElyShell { cx.notify(); } } -} -fn render_ready_web_surface( - frame: &WebSurfaceFrame, - tab: &BrowserTab, - state_entity: Entity, -) -> AnyElement { - render_web_surface( - tab, - state_entity, - frame.title_label(), - frame.url_label().to_string(), - Some(format!("{}x{}", frame.width, frame.height)), - img(ImageSource::Render(frame.image.clone())).size_full().object_fit(ObjectFit::Contain), - ) -} - -fn render_web_surface_header(title: String, url: String, detail: Option) -> AnyElement { - div() - .h(px(34.0)) - .px_3() - .gap_3() - .flex() - .items_center() - .border_b_1() - .border_color(rgb(colors::HAIRLINE)) - .bg(rgb(colors::CANVAS_SOFT)) - .child( - div() - .min_w_0() - .flex_1() - .truncate() - .text_sm() - .font_semibold() - .text_color(rgb(colors::INK)) - .child(title), - ) - .when_some(detail, |this, detail| { - this.child(div().text_xs().text_color(rgb(colors::MUTED)).child(detail)) - }) - .child( - div().max_w(px(420.0)).truncate().text_xs().text_color(rgb(colors::MUTED)).child(url), - ) - .into_any_element() -} - -fn render_loading_web_surface(tab: &BrowserTab, state_entity: Entity) -> AnyElement { - render_web_surface( - tab, - state_entity, - tab.title().to_string(), - tab.url().as_str().to_string(), - Some("Rendering".to_string()), - centered_status(tab.title(), tab.url().as_str(), "Rendering page with Servo", colors::BODY), - ) -} - -fn render_failed_web_surface( - tab: &BrowserTab, - message: &str, - state_entity: Entity, -) -> AnyElement { - render_web_surface( - tab, - state_entity, - tab.title().to_string(), - tab.url().as_str().to_string(), - Some("Render failed".to_string()), - centered_status(tab.title(), tab.url().as_str(), message, colors::ERROR), - ) -} - -fn centered_status(title: &str, url: &str, detail: &str, detail_color: u32) -> impl IntoElement { - div() - .size_full() - .p_8() - .flex() - .flex_col() - .items_center() - .justify_center() - .gap_2() - .bg(rgb(colors::SURFACE_CARD)) - .child(div().text_size(px(26.0)).text_color(rgb(colors::INK)).child(title.to_string())) - .child(div().text_sm().text_color(rgb(colors::MUTED)).child(url.to_string())) - .child(div().text_sm().text_color(rgb(detail_color)).child(detail.to_string())) -} - -fn render_web_surface( - tab: &BrowserTab, - state_entity: Entity, - title: String, - url: String, - detail: Option, - content: impl IntoElement, -) -> AnyElement { - div() - .flex_1() - .h_full() - .p(px(spacing::SM)) - .bg(rgb(colors::CANVAS_SOFT)) - .child( - div() - .size_full() - .overflow_hidden() - .rounded_md() - .border_1() - .border_color(rgb(colors::HAIRLINE)) - .bg(rgb(colors::SURFACE_CARD)) - .flex() - .flex_col() - .child(render_web_surface_header(title, url, detail)) - .child( - div() - .relative() - .flex_1() - .min_h_0() - .overflow_hidden() - .bg(rgb(colors::SURFACE_CARD)) - .child(content) - .child(render_viewport_tracker(tab.id().clone(), state_entity)), - ), - ) - .into_any_element() -} - -fn render_viewport_tracker(tab_id: TabId, state_entity: Entity) -> impl IntoElement { - canvas( - move |bounds, _window: &mut Window, cx: &mut App| { - state_entity.update(cx, |shell, cx| { - shell.record_external_web_viewport(tab_id, bounds, cx); - }); - }, - |_, _, _, _| {}, - ) - .absolute() - .size_full() -} - -fn viewport_dimension(pixels: Pixels) -> Option { - let value = f32::from(pixels.round()); - if !value.is_finite() || value < 1.0 || value > u32::MAX as f32 { - return None; + pub(super) fn scroll_external_web_viewport( + &mut self, + tab_id: TabId, + requested_url: String, + delta: gpui::Point, + cx: &mut Context, + ) { + if self.web_surfaces.record_scroll_delta(&tab_id, requested_url.as_str(), delta) { + cx.notify(); + } } - - Some(value as u32) } pub(super) fn is_external_web_url(url: &str) -> bool { diff --git a/crates/ely_app/src/shell/web_surface_frame.rs b/crates/ely_app/src/shell/web_surface_frame.rs new file mode 100644 index 0000000..4709023 --- /dev/null +++ b/crates/ely_app/src/shell/web_surface_frame.rs @@ -0,0 +1,79 @@ +use std::sync::Arc; + +use gpui::RenderImage; +use image::{ImageBuffer, Rgba}; +use thiserror::Error; + +use crate::services::servo_sidecar::SidecarSnapshot; + +use super::{ + web_surface_geometry::{WebSurfaceScrollOffset, WebSurfaceSize}, + web_surface_image::renderable_image_buffer, +}; + +#[derive(Clone)] +pub(super) struct WebSurfaceFrame { + pub(super) requested_url: String, + loaded_url: Option, + title: Option, + width: u32, + height: u32, + scroll_offset: WebSurfaceScrollOffset, + pub(super) image: Arc, +} + +impl WebSurfaceFrame { + pub(super) fn from_snapshot( + requested_url: String, + scroll_offset: WebSurfaceScrollOffset, + snapshot: SidecarSnapshot, + ) -> Result { + let width = snapshot.width(); + let height = snapshot.height(); + let loaded_url = snapshot.loaded_url().map(str::to_string); + let title = snapshot.title().map(str::to_string); + let rgba_bytes = snapshot.into_rgba_bytes(); + + let Some(buffer) = ImageBuffer::, _>::from_raw(width, height, rgba_bytes) else { + return Err(WebSurfaceError::InvalidFrameBuffer { width, height }); + }; + + let image_buffer = renderable_image_buffer(buffer); + + Ok(Self { + requested_url, + loaded_url, + title, + width, + height, + scroll_offset, + image: Arc::new(RenderImage::new([image::Frame::new(image_buffer)])), + }) + } + + pub(super) fn title_label(&self) -> String { + self.title.clone().unwrap_or_else(|| self.requested_url.clone()) + } + + pub(super) fn url_label(&self) -> &str { + self.loaded_url.as_deref().unwrap_or(self.requested_url.as_str()) + } + + pub(super) fn detail_label(&self) -> String { + self.scroll_offset.detail_label(self.size()) + } + + pub(super) fn size(&self) -> WebSurfaceSize { + WebSurfaceSize { width: self.width, height: self.height } + } + + pub(super) fn scroll_offset(&self) -> WebSurfaceScrollOffset { + self.scroll_offset + } +} + +#[derive(Debug, Error)] +pub(super) enum WebSurfaceError { + #[error("invalid servo frame buffer for {width}x{height}")] + InvalidFrameBuffer { width: u32, height: u32 }, +} diff --git a/crates/ely_app/src/shell/web_surface_geometry.rs b/crates/ely_app/src/shell/web_surface_geometry.rs new file mode 100644 index 0000000..7592b09 --- /dev/null +++ b/crates/ely_app/src/shell/web_surface_geometry.rs @@ -0,0 +1,95 @@ +use gpui::{Bounds, Pixels, Point}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct WebSurfaceSize { + pub(super) width: u32, + pub(super) height: u32, +} + +impl WebSurfaceSize { + pub(super) fn from_bounds(bounds: Bounds) -> Option { + Some(Self { + width: viewport_dimension(bounds.size.width)?, + height: viewport_dimension(bounds.size.height)?, + }) + } +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(super) struct WebSurfaceScrollOffset { + x: i32, + y: i32, +} + +impl WebSurfaceScrollOffset { + pub(super) fn scrolled_by(self, delta: WebSurfaceScrollDelta) -> Self { + Self { + x: positive_scroll_component(self.x, delta.x), + y: positive_scroll_component(self.y, delta.y), + } + } + + pub(super) fn detail_label(self, size: WebSurfaceSize) -> String { + match (self.x, self.y) { + (0, 0) => format!("{}x{}", size.width, size.height), + (0, y) => format!("{}x{} y={y}", size.width, size.height), + (x, y) => format!("{}x{} x={x} y={y}", size.width, size.height), + } + } + + pub(super) fn x(self) -> i32 { + self.x + } + + pub(super) fn y(self) -> i32 { + self.y + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct WebSurfaceScrollDelta { + x: i32, + y: i32, +} + +impl WebSurfaceScrollDelta { + pub(super) fn from_point(delta: Point) -> Option { + let x = scroll_dimension(delta.x)?; + let y = scroll_dimension(delta.y)?; + if x == 0 && y == 0 { + return None; + } + + Some(Self { x, y }) + } +} + +fn viewport_dimension(pixels: Pixels) -> Option { + let value = f32::from(pixels.round()); + if !value.is_finite() || value < 1.0 || value > u32::MAX as f32 { + return None; + } + + Some(value as u32) +} + +fn scroll_dimension(pixels: Pixels) -> Option { + let value = f32::from(pixels.round()); + if !value.is_finite() { + return None; + } + if value > i32::MAX as f32 { + return Some(i32::MAX); + } + if value < i32::MIN as f32 { + return Some(i32::MIN); + } + + Some(value as i32) +} + +fn positive_scroll_component(current: i32, delta: i32) -> i32 { + let value = i64::from(current) + i64::from(delta); + let clamped = value.clamp(0, i64::from(i32::MAX)); + clamped as i32 +} diff --git a/crates/ely_app/src/shell/web_surface_view.rs b/crates/ely_app/src/shell/web_surface_view.rs new file mode 100644 index 0000000..efcbc8f --- /dev/null +++ b/crates/ely_app/src/shell/web_surface_view.rs @@ -0,0 +1,165 @@ +use ely_domain::{BrowserTab, TabId}; +use gpui::{ + AnyElement, App, Entity, ImageSource, InteractiveElement, IntoElement, ObjectFit, + ParentElement, Styled, StyledImage, Window, canvas, div, img, prelude::FluentBuilder, px, rgb, +}; +use gpui_component::StyledExt; + +use super::{ElyShell, web_surface_frame::WebSurfaceFrame}; +use ely_design_system::{colors, spacing}; + +pub(super) fn render_ready_web_surface( + frame: &WebSurfaceFrame, + tab: &BrowserTab, + state_entity: Entity, +) -> AnyElement { + render_web_surface( + tab, + state_entity, + frame.title_label(), + frame.url_label().to_string(), + Some(frame.detail_label()), + img(ImageSource::Render(frame.image.clone())).size_full().object_fit(ObjectFit::Contain), + ) +} + +pub(super) fn render_loading_web_surface( + tab: &BrowserTab, + state_entity: Entity, +) -> AnyElement { + render_web_surface( + tab, + state_entity, + tab.title().to_string(), + tab.url().as_str().to_string(), + Some("Rendering".to_string()), + centered_status(tab.title(), tab.url().as_str(), "Rendering page with Servo", colors::BODY), + ) +} + +pub(super) fn render_failed_web_surface( + tab: &BrowserTab, + message: &str, + state_entity: Entity, +) -> AnyElement { + render_web_surface( + tab, + state_entity, + tab.title().to_string(), + tab.url().as_str().to_string(), + Some("Render failed".to_string()), + centered_status(tab.title(), tab.url().as_str(), message, colors::ERROR), + ) +} + +fn render_web_surface_header(title: String, url: String, detail: Option) -> AnyElement { + div() + .h(px(34.0)) + .px_3() + .gap_3() + .flex() + .items_center() + .border_b_1() + .border_color(rgb(colors::HAIRLINE)) + .bg(rgb(colors::CANVAS_SOFT)) + .child( + div() + .min_w_0() + .flex_1() + .truncate() + .text_sm() + .font_semibold() + .text_color(rgb(colors::INK)) + .child(title), + ) + .when_some(detail, |this, detail| { + this.child(div().text_xs().text_color(rgb(colors::MUTED)).child(detail)) + }) + .child( + div().max_w(px(420.0)).truncate().text_xs().text_color(rgb(colors::MUTED)).child(url), + ) + .into_any_element() +} + +fn centered_status(title: &str, url: &str, detail: &str, detail_color: u32) -> impl IntoElement { + div() + .size_full() + .p_8() + .flex() + .flex_col() + .items_center() + .justify_center() + .gap_2() + .bg(rgb(colors::SURFACE_CARD)) + .child(div().text_size(px(26.0)).text_color(rgb(colors::INK)).child(title.to_string())) + .child(div().text_sm().text_color(rgb(colors::MUTED)).child(url.to_string())) + .child(div().text_sm().text_color(rgb(detail_color)).child(detail.to_string())) +} + +fn render_web_surface( + tab: &BrowserTab, + state_entity: Entity, + title: String, + url: String, + detail: Option, + content: impl IntoElement, +) -> AnyElement { + let scroll_tab_id = tab.id().clone(); + let scroll_url = tab.url().as_str().to_string(); + let scroll_entity = state_entity.clone(); + let tracker_entity = state_entity; + + div() + .flex_1() + .h_full() + .p(px(spacing::SM)) + .bg(rgb(colors::CANVAS_SOFT)) + .child( + div() + .size_full() + .overflow_hidden() + .rounded_md() + .border_1() + .border_color(rgb(colors::HAIRLINE)) + .bg(rgb(colors::SURFACE_CARD)) + .flex() + .flex_col() + .child(render_web_surface_header(title, url, detail)) + .child( + div() + .relative() + .flex_1() + .min_h_0() + .overflow_hidden() + .bg(rgb(colors::SURFACE_CARD)) + .on_scroll_wheel(move |event, window, cx| { + let delta = event.delta.pixel_delta(window.line_height()); + scroll_entity.update(cx, |shell, cx| { + shell.scroll_external_web_viewport( + scroll_tab_id.clone(), + scroll_url.clone(), + delta, + cx, + ); + }); + cx.stop_propagation(); + }) + .child(content) + .child(render_viewport_tracker(tab.id().clone(), tracker_entity)), + ), + ) + .into_any_element() +} + +fn render_viewport_tracker(tab_id: TabId, state_entity: Entity) -> impl IntoElement { + canvas( + move |bounds, _window: &mut Window, cx: &mut App| { + state_entity.update(cx, |shell, cx| { + shell.record_external_web_viewport(tab_id, bounds, cx); + }); + }, + |_, _, _, _| {}, + ) + .absolute() + .size_full() +} diff --git a/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs b/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs index 447e547..58c8bdd 100644 --- a/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs +++ b/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs @@ -8,7 +8,7 @@ use std::{ use ely_domain::{ProfileId, TabId, UrlText}; use ely_servo_host::{ - NavigationRequest, RenderedFrame, ServoHost, ServoHostError, ServoSurfaceSize, + NavigationRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState, }; use serde::Serialize; @@ -17,6 +17,7 @@ use thiserror::Error; const WAIT_ITERATIONS: usize = 5_000; const WAIT_INTERVAL: Duration = Duration::from_millis(2); const RENDER_TIMEOUT: Duration = Duration::from_secs(20); +const SCROLL_SETTLE_TIMEOUT: Duration = Duration::from_millis(700); fn main() -> Result<(), SidecarError> { match parse_command(env::args())? { @@ -33,6 +34,8 @@ struct SnapshotArgs { rgba_out: PathBuf, width: u32, height: u32, + scroll_x: i32, + scroll_y: i32, } #[derive(Debug, Error)] @@ -52,7 +55,7 @@ enum SidecarError { #[error("unknown argument: {value}")] UnknownArgument { value: String }, - #[error("{name} must be a positive integer: {value}")] + #[error("{name} must be an integer: {value}")] InvalidInteger { name: &'static str, value: String, @@ -101,6 +104,8 @@ fn parse_snapshot_args( let mut rgba_out = None; let mut width = None; let mut height = None; + let mut scroll_x = 0; + let mut scroll_y = 0; while let Some(name) = args.next() { match name.as_str() { @@ -114,6 +119,14 @@ fn parse_snapshot_args( "--height" => { height = Some(parse_dimension("--height", next_argument(&mut args, "--height")?)?) } + "--scroll-x" => { + scroll_x = + parse_scroll_delta("--scroll-x", next_argument(&mut args, "--scroll-x")?)? + } + "--scroll-y" => { + scroll_y = + parse_scroll_delta("--scroll-y", next_argument(&mut args, "--scroll-y")?)? + } _ => return Err(SidecarError::UnknownArgument { value: name }), } } @@ -123,6 +136,8 @@ fn parse_snapshot_args( rgba_out: rgba_out.ok_or(SidecarError::MissingRequiredArgument { name: "--rgba-out" })?, width: width.ok_or(SidecarError::MissingRequiredArgument { name: "--width" })?, height: height.ok_or(SidecarError::MissingRequiredArgument { name: "--height" })?, + scroll_x, + scroll_y, }) } @@ -146,6 +161,10 @@ fn parse_dimension(name: &'static str, value: String) -> Result Result { + value.parse::().map_err(|source| SidecarError::InvalidInteger { name, value, source }) +} + fn parse_output_path(value: String) -> Result { if value.trim().is_empty() { return Err(SidecarError::EmptyRgbaOutputPath); @@ -167,16 +186,45 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> { })?; let snapshot = wait_for_frame(&mut host, &webview_id, args.url.as_str())?; + let (snapshot, scroll_changed_frame) = + apply_scroll_if_requested(&mut host, &webview_id, &args, snapshot)?; let frame = host.last_rendered_frame()?; std::fs::write(&args.rgba_out, frame.rgba_bytes())?; serde_json::to_writer( std::io::stdout().lock(), - &SnapshotReport::new(args.url.as_str(), &args.rgba_out, &snapshot, &frame), + &SnapshotReport::new( + args.url.as_str(), + &args.rgba_out, + &snapshot, + &frame, + args.scroll_x, + args.scroll_y, + scroll_changed_frame, + ), )?; Ok(()) } +fn apply_scroll_if_requested( + host: &mut SoftwareServoHost, + webview_id: &ely_domain::WebViewId, + args: &SnapshotArgs, + snapshot: WebViewSnapshot, +) -> Result<(WebViewSnapshot, bool), SidecarError> { + if args.scroll_x == 0 && args.scroll_y == 0 { + return Ok((snapshot, false)); + } + + let previous_frame_hash = host.last_rendered_frame()?.sample_hash(); + host.scroll(ScrollRequest { + webview_id: webview_id.clone(), + delta_x: args.scroll_x, + delta_y: args.scroll_y, + })?; + wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash) +} + fn wait_for_frame( host: &mut SoftwareServoHost, webview_id: &ely_domain::WebViewId, @@ -210,6 +258,39 @@ fn wait_for_frame( }) } +fn wait_for_changed_or_settled_frame( + host: &mut SoftwareServoHost, + webview_id: &ely_domain::WebViewId, + previous_frame_hash: u64, +) -> Result<(WebViewSnapshot, bool), SidecarError> { + let started_at = Instant::now(); + let mut latest_snapshot = host.snapshot(webview_id)?; + + for _ in 0..WAIT_ITERATIONS { + if started_at.elapsed() >= SCROLL_SETTLE_TIMEOUT { + break; + } + + host.tick(); + let snapshot = host.snapshot(webview_id)?; + if snapshot.has_pending_frame() { + host.paint(webview_id)?; + } + + latest_snapshot = host.snapshot(webview_id)?; + let changed_frame = host.last_rendered_frame().is_ok_and(|frame| { + frame.non_white_pixel_count() > 0 && frame.sample_hash() != previous_frame_hash + }); + if changed_frame { + return Ok((latest_snapshot, true)); + } + + thread::sleep(WAIT_INTERVAL); + } + + Ok((latest_snapshot, false)) +} + #[derive(Serialize)] struct SnapshotReport { requested_url: String, @@ -224,6 +305,9 @@ struct SnapshotReport { non_white_pixel_count: u64, content_pixel_count: u64, sample_hash: u64, + scroll_x: i32, + scroll_y: i32, + scroll_changed_frame: bool, } impl SnapshotReport { @@ -232,6 +316,9 @@ impl SnapshotReport { rgba_path: &std::path::Path, snapshot: &WebViewSnapshot, frame: &RenderedFrame, + scroll_x: i32, + scroll_y: i32, + scroll_changed_frame: bool, ) -> Self { Self { requested_url: requested_url.to_string(), @@ -246,6 +333,9 @@ impl SnapshotReport { non_white_pixel_count: frame.non_white_pixel_count(), content_pixel_count: frame.content_pixel_count(), sample_hash: frame.sample_hash(), + scroll_x, + scroll_y, + scroll_changed_frame, } } } diff --git a/crates/ely_servo_host/src/host.rs b/crates/ely_servo_host/src/host.rs index 65abaf6..1b8c7a4 100644 --- a/crates/ely_servo_host/src/host.rs +++ b/crates/ely_servo_host/src/host.rs @@ -213,6 +213,13 @@ pub struct NavigationRequest { pub url: UrlText, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ScrollRequest { + pub webview_id: WebViewId, + pub delta_x: i32, + pub delta_y: i32, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct PermissionRequest { pub webview_id: WebViewId, @@ -237,6 +244,8 @@ pub trait ServoHost { fn navigate(&mut self, request: NavigationRequest) -> Result<(), ServoHostError>; + fn scroll(&mut self, request: ScrollRequest) -> Result<(), ServoHostError>; + fn set_permission( &mut self, request: PermissionRequest, diff --git a/crates/ely_servo_host/src/lib.rs b/crates/ely_servo_host/src/lib.rs index 8f607e7..53f8aa1 100644 --- a/crates/ely_servo_host/src/lib.rs +++ b/crates/ely_servo_host/src/lib.rs @@ -6,7 +6,7 @@ mod runtime; pub use error::ServoHostError; pub use host::{ NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, RenderedFrameSummary, - ServoHost, WebViewSnapshot, WebViewState, + ScrollRequest, ServoHost, WebViewSnapshot, WebViewState, }; #[cfg(feature = "servo-engine")] pub use runtime::{ServoSurfaceSize, SoftwareServoHost}; diff --git a/crates/ely_servo_host/src/runtime.rs b/crates/ely_servo_host/src/runtime.rs index 497bba1..66d5a1f 100644 --- a/crates/ely_servo_host/src/runtime.rs +++ b/crates/ely_servo_host/src/runtime.rs @@ -11,14 +11,15 @@ use std::{ use dpi::PhysicalSize; use ely_domain::{ProfileId, TabId, WebViewId}; use servo::{ - DeviceIntPoint, DeviceIntRect, DeviceIntSize, EventLoopWaker, LoadStatus, RenderingContext, - Servo, ServoBuilder, WebView, WebViewBuilder, WebViewDelegate, + DeviceIntPoint, DeviceIntRect, DeviceIntSize, DevicePoint, DeviceVector2D, EventLoopWaker, + LoadStatus, RenderingContext, Scroll, Servo, ServoBuilder, WebView, WebViewBuilder, + WebViewDelegate, WebViewPoint, WebViewVector, }; use url::Url; use crate::{ - NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, ServoHost, - ServoHostError, WebViewSnapshot, WebViewState, + NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, ScrollRequest, + ServoHost, ServoHostError, WebViewSnapshot, WebViewState, }; static SERVO_RUNTIME_STARTED: AtomicBool = AtomicBool::new(false); @@ -142,6 +143,26 @@ impl ServoHost for SoftwareServoHost { Ok(()) } + fn scroll(&mut self, request: ScrollRequest) -> Result<(), ServoHostError> { + let webview = self + .webviews + .get(&request.webview_id) + .ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?; + + if request.delta_x == 0 && request.delta_y == 0 { + return Ok(()); + } + + webview.webview.notify_scroll_event( + Scroll::Delta(WebViewVector::Device(DeviceVector2D::new( + request.delta_x as f32, + request.delta_y as f32, + ))), + WebViewPoint::Device(DevicePoint::zero()), + ); + Ok(()) + } + fn set_permission( &mut self, request: PermissionRequest, diff --git a/crates/ely_servo_host/tests/sidecar.rs b/crates/ely_servo_host/tests/sidecar.rs index 61fcd11..e9b3bf5 100644 --- a/crates/ely_servo_host/tests/sidecar.rs +++ b/crates/ely_servo_host/tests/sidecar.rs @@ -20,6 +20,10 @@ const PRD_SITE_COMPATIBILITY_SIZES: &[FrameSize] = &[ FrameSize { width: 934, height: 657 }, FrameSize { width: 1614, height: 980 }, ]; +const SERVO_SCROLL_SITE: PrdSiteCompatibilityCase = + PrdSiteCompatibilityCase { url: "https://servo.org", title_fragment: "Servo" }; +const SERVO_SCROLL_SIZE: FrameSize = FrameSize { width: 934, height: 657 }; +const SERVO_SCROLL_OFFSET: ScrollOffset = ScrollOffset { x: 0, y: 480 }; struct PrdSiteCompatibilityCase { url: &'static str, @@ -32,38 +36,81 @@ struct FrameSize { height: u64, } +#[derive(Clone, Copy)] +struct ScrollOffset { + x: i64, + y: i64, +} + +impl ScrollOffset { + const ZERO: Self = Self { x: 0, y: 0 }; +} + #[test] fn sidecar_snapshots_prd_sites_to_rgba_files() -> Result<(), Box> { for case in PRD_SITE_COMPATIBILITY_CASES { for size in PRD_SITE_COMPATIBILITY_SIZES { - snapshot_prd_site(case, *size)?; + snapshot_prd_site(case, *size, ScrollOffset::ZERO)?; } } Ok(()) } +#[test] +fn sidecar_scrolls_prd_site_with_servo_input() -> Result<(), Box> { + let initial_report = + snapshot_prd_site(&SERVO_SCROLL_SITE, SERVO_SCROLL_SIZE, ScrollOffset::ZERO)?; + let scrolled_report = + snapshot_prd_site(&SERVO_SCROLL_SITE, SERVO_SCROLL_SIZE, SERVO_SCROLL_OFFSET)?; + + assert_eq!(report_field_as_i64(&scrolled_report, "scroll_x")?, SERVO_SCROLL_OFFSET.x); + assert_eq!(report_field_as_i64(&scrolled_report, "scroll_y")?, SERVO_SCROLL_OFFSET.y); + assert_eq!( + report_field_as_u64(&scrolled_report, "width")?, + SERVO_SCROLL_SIZE.width, + "{}", + SERVO_SCROLL_SITE.url + ); + assert!( + report_field_as_bool(&scrolled_report, "scroll_changed_frame")?, + "{}", + SERVO_SCROLL_SITE.url + ); + assert_ne!( + report_field_as_u64(&initial_report, "sample_hash")?, + report_field_as_u64(&scrolled_report, "sample_hash")?, + "{}", + SERVO_SCROLL_SITE.url + ); + + Ok(()) +} + fn snapshot_prd_site( case: &PrdSiteCompatibilityCase, size: FrameSize, -) -> Result<(), Box> { + scroll_offset: ScrollOffset, +) -> Result> { let site_name = case .url .chars() .map(|character| if character.is_ascii_alphanumeric() { character } else { '-' }) .collect::(); let output_path = std::env::temp_dir().join(format!( - "ely-servo-sidecar-{}-{site_name}-{}x{}.rgba", + "ely-servo-sidecar-{}-{site_name}-{}x{}-{}-{}.rgba", std::process::id(), size.width, - size.height + size.height, + scroll_offset.x, + scroll_offset.y )); if output_path.exists() { std::fs::remove_file(&output_path)?; } - let output = run_sidecar_snapshot(case.url, &output_path, size)?; + let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset)?; assert!( output.status.success(), @@ -97,15 +144,17 @@ fn snapshot_prd_site( assert_eq!(std::fs::metadata(&output_path)?.len(), size.width * size.height * 4); std::fs::remove_file(&output_path)?; - Ok(()) + Ok(report) } fn run_sidecar_snapshot( site_url: &str, output_path: &std::path::Path, size: FrameSize, + scroll_offset: ScrollOffset, ) -> Result> { - let mut child = Command::new(env!("CARGO_BIN_EXE_ely_servo_sidecar")) + let mut command = Command::new(env!("CARGO_BIN_EXE_ely_servo_sidecar")); + command .arg("snapshot") .arg("--url") .arg(site_url) @@ -114,10 +163,15 @@ fn run_sidecar_snapshot( .arg("--width") .arg(size.width.to_string()) .arg("--height") - .arg(size.height.to_string()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; + .arg(size.height.to_string()); + if scroll_offset.x != 0 { + command.arg("--scroll-x").arg(scroll_offset.x.to_string()); + } + if scroll_offset.y != 0 { + command.arg("--scroll-y").arg(scroll_offset.y.to_string()); + } + + let mut child = command.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; let started_at = Instant::now(); loop { @@ -162,6 +216,26 @@ fn assert_report_text_contains( Ok(()) } +fn report_field_as_bool( + report: &serde_json::Value, + field: &'static str, +) -> Result> { + report + .get(field) + .and_then(serde_json::Value::as_bool) + .ok_or_else(|| format!("missing boolean report field: {field}").into()) +} + +fn report_field_as_i64( + report: &serde_json::Value, + field: &'static str, +) -> Result> { + report + .get(field) + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| format!("missing signed report field: {field}").into()) +} + fn report_field_as_u64( report: &serde_json::Value, field: &'static str, diff --git a/crates/ely_servo_host/tests/software_host.rs b/crates/ely_servo_host/tests/software_host.rs index 4db3065..5657e49 100644 --- a/crates/ely_servo_host/tests/software_host.rs +++ b/crates/ely_servo_host/tests/software_host.rs @@ -4,7 +4,8 @@ use std::{error::Error, thread, time::Duration}; use ely_domain::{ProfileId, TabId, UrlText}; use ely_servo_host::{ - NavigationRequest, ServoHost, ServoHostError, ServoSurfaceSize, SoftwareServoHost, WebViewState, + NavigationRequest, ScrollRequest, ServoHost, ServoHostError, ServoSurfaceSize, + SoftwareServoHost, WebViewState, }; const MINIMUM_CONTENT_PIXELS: u64 = 1_000; @@ -69,6 +70,13 @@ fn manages_real_servo_webview_lifecycle() -> Result<(), Box> { previous_frame_hash = Some(host.last_rendered_frame()?.sample_hash()); } + let previous_frame_hash = host.last_rendered_frame()?.sample_hash(); + host.scroll(ScrollRequest { webview_id: webview_id.clone(), delta_x: 0, delta_y: 480 })?; + let snapshot = wait_for_rendered_webview(&mut host, &webview_id, Some(previous_frame_hash))?; + assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}"); + assert_rendered_frame_has_content(&host, "https://servo.org scrolled", MINIMUM_CONTENT_PIXELS)?; + assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash); + assert!(matches!( SoftwareServoHost::new(ServoSurfaceSize::new(640, 480)), Err(ServoHostError::RuntimeAlreadyStarted)