From b786073826eb3f6579b1417a02fe0ac683796b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Fri, 8 May 2026 14:23:12 -0400 Subject: [PATCH] Bridge Servo click input --- crates/ely_app/src/services/servo_sidecar.rs | 29 ++- crates/ely_app/src/shell/mod.rs | 1 + crates/ely_app/src/shell/web_surface.rs | 186 +++++++++++------- crates/ely_app/src/shell/web_surface_frame.rs | 15 +- .../ely_app/src/shell/web_surface_geometry.rs | 40 ++++ crates/ely_app/src/shell/web_surface_state.rs | 68 +++++++ crates/ely_app/src/shell/web_surface_view.rs | 67 +++++-- .../src/bin/ely_servo_sidecar.rs | 94 +++++++-- crates/ely_servo_host/src/host.rs | 9 + crates/ely_servo_host/src/lib.rs | 4 +- crates/ely_servo_host/src/runtime.rs | 30 ++- crates/ely_servo_host/tests/sidecar.rs | 75 ++++++- crates/ely_servo_host/tests/software_host.rs | 16 +- 13 files changed, 506 insertions(+), 128 deletions(-) create mode 100644 crates/ely_app/src/shell/web_surface_state.rs diff --git a/crates/ely_app/src/services/servo_sidecar.rs b/crates/ely_app/src/services/servo_sidecar.rs index 37ad235..29d7680 100644 --- a/crates/ely_app/src/services/servo_sidecar.rs +++ b/crates/ely_app/src/services/servo_sidecar.rs @@ -64,7 +64,8 @@ impl ServoSidecarClient { request: &SidecarSnapshotRequest, rgba_path: &Path, ) -> Result { - let mut child = Command::new(&self.binary_path) + let mut command = Command::new(&self.binary_path); + command .arg("snapshot") .arg("--url") .arg(request.url.as_str()) @@ -77,7 +78,16 @@ impl ServoSidecarClient { .arg("--scroll-x") .arg(request.scroll_x.to_string()) .arg("--scroll-y") - .arg(request.scroll_y.to_string()) + .arg(request.scroll_y.to_string()); + if let Some(click_point) = request.click_point { + command + .arg("--click-x") + .arg(click_point.x.to_string()) + .arg("--click-y") + .arg(click_point.y.to_string()); + } + + let mut child = command .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() @@ -109,12 +119,13 @@ pub struct SidecarSnapshotRequest { height: u32, scroll_x: i32, scroll_y: i32, + click_point: Option, } impl SidecarSnapshotRequest { #[must_use] pub fn new(url: UrlText, width: u32, height: u32) -> Self { - Self { url, width, height, scroll_x: 0, scroll_y: 0 } + Self { url, width, height, scroll_x: 0, scroll_y: 0, click_point: None } } #[must_use] @@ -123,6 +134,18 @@ impl SidecarSnapshotRequest { self.scroll_y = scroll_y; self } + + #[must_use] + pub fn with_click_point(mut self, x: u32, y: u32) -> Self { + self.click_point = Some(SidecarClickPoint { x, y }); + self + } +} + +#[derive(Clone, Copy, Debug)] +struct SidecarClickPoint { + x: u32, + y: u32, } #[derive(Clone, Debug)] diff --git a/crates/ely_app/src/shell/mod.rs b/crates/ely_app/src/shell/mod.rs index 9ea0242..906c0ba 100644 --- a/crates/ely_app/src/shell/mod.rs +++ b/crates/ely_app/src/shell/mod.rs @@ -18,6 +18,7 @@ mod web_surface; mod web_surface_frame; mod web_surface_geometry; mod web_surface_image; +mod web_surface_state; mod web_surface_view; use ely_browser_core::{BrowserCore, InitialBrowserConfig}; diff --git a/crates/ely_app/src/shell/web_surface.rs b/crates/ely_app/src/shell/web_surface.rs index 5c5c5fa..accf7d9 100644 --- a/crates/ely_app/src/shell/web_surface.rs +++ b/crates/ely_app/src/shell/web_surface.rs @@ -1,16 +1,20 @@ use std::collections::BTreeMap; use ely_domain::{BrowserTab, TabId}; -use gpui::{AnyElement, Bounds, Context, Pixels}; +use gpui::{AnyElement, Bounds, Context, Pixels, Point}; -use crate::services::servo_sidecar::{ - ServoSidecarClient, ServoSidecarError, SidecarSnapshot, SidecarSnapshotRequest, -}; +use crate::services::servo_sidecar::{ServoSidecarError, SidecarSnapshot, SidecarSnapshotRequest}; use super::{ ElyShell, web_surface_frame::WebSurfaceFrame, - web_surface_geometry::{WebSurfaceScrollDelta, WebSurfaceScrollOffset, WebSurfaceSize}, + web_surface_geometry::{ + WebSurfaceClickPoint, WebSurfaceScrollDelta, WebSurfaceScrollOffset, WebSurfaceSize, + }, + web_surface_state::{ + WebSurfaceClickState, WebSurfaceClient, WebSurfaceRequest, WebSurfaceScrollState, + WebSurfaceState, + }, web_surface_view::{ render_failed_web_surface, render_loading_web_surface, render_ready_web_surface, }, @@ -19,7 +23,9 @@ use super::{ pub(super) struct WebSurfaceStore { client: WebSurfaceClient, pending_viewport_sizes: BTreeMap, + click_points: BTreeMap, scroll_offsets: BTreeMap, + viewport_bounds: BTreeMap>, viewport_sizes: BTreeMap, states: BTreeMap, } @@ -29,7 +35,9 @@ impl WebSurfaceStore { Self { client: WebSurfaceClient::new(), pending_viewport_sizes: BTreeMap::new(), + click_points: BTreeMap::new(), scroll_offsets: BTreeMap::new(), + viewport_bounds: BTreeMap::new(), viewport_sizes: BTreeMap::new(), states: BTreeMap::new(), } @@ -47,10 +55,11 @@ impl WebSurfaceStore { let size = self.viewport_sizes.get(tab.id()).copied()?; let requested_url = tab.url().as_str().to_string(); let scroll_offset = self.scroll_offset_for(tab.id(), requested_url.as_str()); + let click_point = self.click_point_for(tab.id(), requested_url.as_str(), scroll_offset); if self.is_loading_requested_url(tab.id(), requested_url.as_str()) { return None; } - if self.has_current_state(tab.id(), &requested_url, size, scroll_offset) { + if self.has_current_state(tab.id(), &requested_url, size, scroll_offset, click_point) { return None; } @@ -63,6 +72,7 @@ impl WebSurfaceStore { requested_url, size, scroll_offset, + click_point, message: message.clone(), }, ); @@ -76,22 +86,26 @@ impl WebSurfaceStore { requested_url: requested_url.clone(), size, scroll_offset, + click_point, previous_frame: self.previous_ready_frame(tab.id(), requested_url.as_str()), }, ); + let mut snapshot_request = + SidecarSnapshotRequest::new(tab.url().clone(), size.width, size.height) + .with_scroll_offset(scroll_offset.x(), scroll_offset.y()); + if let Some(click_point) = click_point { + snapshot_request = snapshot_request.with_click_point(click_point.x(), click_point.y()); + } + Some(WebSurfaceRequest { tab_id: tab.id().clone(), requested_url, size, scroll_offset, + click_point, client, - snapshot_request: SidecarSnapshotRequest::new( - tab.url().clone(), - size.width, - size.height, - ) - .with_scroll_offset(scroll_offset.x(), scroll_offset.y()), + snapshot_request, }) } @@ -101,32 +115,38 @@ impl WebSurfaceStore { requested_url: &str, size: WebSurfaceSize, scroll_offset: WebSurfaceScrollOffset, + click_point: Option, ) -> bool { match self.states.get(tab_id) { Some(WebSurfaceState::Loading { requested_url: current_url, size: current_size, scroll_offset: current_scroll_offset, + click_point: current_click_point, .. }) => { current_url == requested_url && *current_size == size && *current_scroll_offset == scroll_offset + && *current_click_point == click_point } Some(WebSurfaceState::Ready(frame)) => { frame.requested_url == requested_url && frame.size() == size && frame.scroll_offset() == scroll_offset + && frame.click_point() == click_point } Some(WebSurfaceState::Failed { requested_url: current_url, size: current_size, scroll_offset: current_scroll_offset, + click_point: current_click_point, .. }) => { current_url == requested_url && *current_size == size && *current_scroll_offset == scroll_offset + && *current_click_point == click_point } None => false, } @@ -138,6 +158,7 @@ impl WebSurfaceStore { requested_url: &str, size: WebSurfaceSize, scroll_offset: WebSurfaceScrollOffset, + click_point: Option, ) -> bool { matches!( self.states.get(tab_id), @@ -145,11 +166,13 @@ impl WebSurfaceStore { requested_url: current_url, size: current_size, scroll_offset: current_scroll_offset, + click_point: current_click_point, .. }) if current_url == requested_url && *current_size == size && *current_scroll_offset == scroll_offset + && *current_click_point == click_point ) } @@ -201,6 +224,7 @@ impl WebSurfaceStore { } state.offset = next_offset; + self.click_points.remove(tab_id); true } @@ -208,6 +232,7 @@ impl WebSurfaceStore { let Some(size) = WebSurfaceSize::from_bounds(bounds) else { return false; }; + self.viewport_bounds.insert(tab_id.clone(), bounds); let Some(current_size) = self.viewport_sizes.get(tab_id).copied() else { self.viewport_sizes.insert(tab_id.clone(), size); @@ -230,6 +255,32 @@ impl WebSurfaceStore { true } + fn record_click_point( + &mut self, + tab_id: &TabId, + requested_url: &str, + position: Point, + ) -> bool { + let Some(bounds) = self.viewport_bounds.get(tab_id).copied() else { + return false; + }; + let Some(point) = WebSurfaceClickPoint::from_window_position(bounds, position) else { + return false; + }; + + let state = WebSurfaceClickState { + requested_url: requested_url.to_string(), + scroll_offset: self.scroll_offset_for(tab_id, requested_url), + point, + }; + if self.click_points.get(tab_id) == Some(&state) { + return false; + } + + self.click_points.insert(tab_id.clone(), state); + true + } + fn scroll_offset_for(&self, tab_id: &TabId, requested_url: &str) -> WebSurfaceScrollOffset { self.scroll_offsets .get(tab_id) @@ -237,56 +288,20 @@ impl WebSurfaceStore { .map(|state| state.offset) .unwrap_or_default() } -} -struct WebSurfaceScrollState { - requested_url: String, - offset: WebSurfaceScrollOffset, -} - -impl WebSurfaceScrollState { - fn new(requested_url: String) -> Self { - Self { requested_url, offset: WebSurfaceScrollOffset::default() } - } -} - -enum WebSurfaceClient { - Ready(ServoSidecarClient), - Unavailable(String), -} - -impl WebSurfaceClient { - fn new() -> Self { - match ServoSidecarClient::new() { - Ok(client) => Self::Ready(client), - Err(error) => Self::Unavailable(error.to_string()), - } - } -} - -struct WebSurfaceRequest { - tab_id: TabId, - requested_url: String, - size: WebSurfaceSize, - scroll_offset: WebSurfaceScrollOffset, - client: ServoSidecarClient, - snapshot_request: SidecarSnapshotRequest, -} - -enum WebSurfaceState { - Loading { - requested_url: String, - size: WebSurfaceSize, + fn click_point_for( + &self, + tab_id: &TabId, + requested_url: &str, scroll_offset: WebSurfaceScrollOffset, - previous_frame: Option, - }, - Ready(WebSurfaceFrame), - Failed { - requested_url: String, - size: WebSurfaceSize, - scroll_offset: WebSurfaceScrollOffset, - message: String, - }, + ) -> Option { + self.click_points + .get(tab_id) + .filter(|state| { + state.requested_url == requested_url && state.scroll_offset == scroll_offset + }) + .map(|state| state.point) + } } impl ElyShell { @@ -324,6 +339,7 @@ impl ElyShell { requested_url, size, scroll_offset, + click_point, client, snapshot_request, } = request; @@ -339,6 +355,7 @@ impl ElyShell { requested_url, size, scroll_offset, + click_point, result, ); cx.notify(); @@ -353,29 +370,40 @@ impl ElyShell { requested_url: String, size: WebSurfaceSize, scroll_offset: WebSurfaceScrollOffset, + click_point: Option, result: Result, ) { - if !self.web_surfaces.is_loading(&tab_id, requested_url.as_str(), size, scroll_offset) { + if !self.web_surfaces.is_loading( + &tab_id, + requested_url.as_str(), + size, + scroll_offset, + click_point, + ) { return; } let state = match result { - Ok(snapshot) => { - match WebSurfaceFrame::from_snapshot(requested_url.clone(), scroll_offset, snapshot) - { - Ok(frame) => WebSurfaceState::Ready(frame), - Err(error) => WebSurfaceState::Failed { - requested_url, - size, - scroll_offset, - message: error.to_string(), - }, - } - } + Ok(snapshot) => match WebSurfaceFrame::from_snapshot( + requested_url.clone(), + scroll_offset, + click_point, + snapshot, + ) { + Ok(frame) => WebSurfaceState::Ready(frame), + Err(error) => WebSurfaceState::Failed { + requested_url, + size, + scroll_offset, + click_point, + message: error.to_string(), + }, + }, Err(error) => WebSurfaceState::Failed { requested_url, size, scroll_offset, + click_point, message: error.to_string(), }, }; @@ -404,6 +432,18 @@ impl ElyShell { cx.notify(); } } + + pub(super) fn click_external_web_viewport( + &mut self, + tab_id: TabId, + requested_url: String, + position: Point, + cx: &mut Context, + ) { + if self.web_surfaces.record_click_point(&tab_id, requested_url.as_str(), position) { + cx.notify(); + } + } } pub(super) fn is_external_web_url(url: &str) -> bool { diff --git a/crates/ely_app/src/shell/web_surface_frame.rs b/crates/ely_app/src/shell/web_surface_frame.rs index 4709023..5b04fa3 100644 --- a/crates/ely_app/src/shell/web_surface_frame.rs +++ b/crates/ely_app/src/shell/web_surface_frame.rs @@ -7,7 +7,7 @@ use thiserror::Error; use crate::services::servo_sidecar::SidecarSnapshot; use super::{ - web_surface_geometry::{WebSurfaceScrollOffset, WebSurfaceSize}, + web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize}, web_surface_image::renderable_image_buffer, }; @@ -19,6 +19,7 @@ pub(super) struct WebSurfaceFrame { width: u32, height: u32, scroll_offset: WebSurfaceScrollOffset, + click_point: Option, pub(super) image: Arc, } @@ -26,6 +27,7 @@ impl WebSurfaceFrame { pub(super) fn from_snapshot( requested_url: String, scroll_offset: WebSurfaceScrollOffset, + click_point: Option, snapshot: SidecarSnapshot, ) -> Result { let width = snapshot.width(); @@ -47,6 +49,7 @@ impl WebSurfaceFrame { width, height, scroll_offset, + click_point, image: Arc::new(RenderImage::new([image::Frame::new(image_buffer)])), }) } @@ -60,7 +63,11 @@ impl WebSurfaceFrame { } pub(super) fn detail_label(&self) -> String { - self.scroll_offset.detail_label(self.size()) + let detail = self.scroll_offset.detail_label(self.size()); + match self.click_point { + Some(click_point) => format!("{detail} {}", click_point.detail_label()), + None => detail, + } } pub(super) fn size(&self) -> WebSurfaceSize { @@ -70,6 +77,10 @@ impl WebSurfaceFrame { pub(super) fn scroll_offset(&self) -> WebSurfaceScrollOffset { self.scroll_offset } + + pub(super) fn click_point(&self) -> Option { + self.click_point + } } #[derive(Debug, Error)] diff --git a/crates/ely_app/src/shell/web_surface_geometry.rs b/crates/ely_app/src/shell/web_surface_geometry.rs index 7592b09..8cb7842 100644 --- a/crates/ely_app/src/shell/web_surface_geometry.rs +++ b/crates/ely_app/src/shell/web_surface_geometry.rs @@ -15,6 +15,36 @@ impl WebSurfaceSize { } } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct WebSurfaceClickPoint { + x: u32, + y: u32, +} + +impl WebSurfaceClickPoint { + pub(super) fn from_window_position( + bounds: Bounds, + position: Point, + ) -> Option { + Some(Self { + x: click_coordinate(position.x, bounds.origin.x, bounds.size.width)?, + y: click_coordinate(position.y, bounds.origin.y, bounds.size.height)?, + }) + } + + pub(super) fn detail_label(self) -> String { + format!("click={},{}", self.x, self.y) + } + + pub(super) fn x(self) -> u32 { + self.x + } + + pub(super) fn y(self) -> u32 { + self.y + } +} + #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub(super) struct WebSurfaceScrollOffset { x: i32, @@ -88,6 +118,16 @@ fn scroll_dimension(pixels: Pixels) -> Option { Some(value as i32) } +fn click_coordinate(position: Pixels, origin: Pixels, size: Pixels) -> Option { + let relative = f32::from(position) - f32::from(origin); + let size = f32::from(size); + if !relative.is_finite() || !size.is_finite() || relative < 0.0 || relative >= size { + return None; + } + + Some(relative.floor() as u32) +} + fn positive_scroll_component(current: i32, delta: i32) -> i32 { let value = i64::from(current) + i64::from(delta); let clamped = value.clamp(0, i64::from(i32::MAX)); diff --git a/crates/ely_app/src/shell/web_surface_state.rs b/crates/ely_app/src/shell/web_surface_state.rs new file mode 100644 index 0000000..fc69acd --- /dev/null +++ b/crates/ely_app/src/shell/web_surface_state.rs @@ -0,0 +1,68 @@ +use ely_domain::TabId; + +use crate::services::servo_sidecar::{ServoSidecarClient, SidecarSnapshotRequest}; + +use super::{ + web_surface_frame::WebSurfaceFrame, + web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize}, +}; + +pub(super) struct WebSurfaceScrollState { + pub(super) requested_url: String, + pub(super) offset: WebSurfaceScrollOffset, +} + +impl WebSurfaceScrollState { + pub(super) fn new(requested_url: String) -> Self { + Self { requested_url, offset: WebSurfaceScrollOffset::default() } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct WebSurfaceClickState { + pub(super) requested_url: String, + pub(super) scroll_offset: WebSurfaceScrollOffset, + pub(super) point: WebSurfaceClickPoint, +} + +pub(super) enum WebSurfaceClient { + Ready(ServoSidecarClient), + Unavailable(String), +} + +impl WebSurfaceClient { + pub(super) fn new() -> Self { + match ServoSidecarClient::new() { + Ok(client) => Self::Ready(client), + Err(error) => Self::Unavailable(error.to_string()), + } + } +} + +pub(super) struct WebSurfaceRequest { + pub(super) tab_id: TabId, + pub(super) requested_url: String, + pub(super) size: WebSurfaceSize, + pub(super) scroll_offset: WebSurfaceScrollOffset, + pub(super) click_point: Option, + pub(super) client: ServoSidecarClient, + pub(super) snapshot_request: SidecarSnapshotRequest, +} + +pub(super) enum WebSurfaceState { + Loading { + requested_url: String, + size: WebSurfaceSize, + scroll_offset: WebSurfaceScrollOffset, + click_point: Option, + previous_frame: Option, + }, + Ready(WebSurfaceFrame), + Failed { + requested_url: String, + size: WebSurfaceSize, + scroll_offset: WebSurfaceScrollOffset, + click_point: Option, + message: String, + }, +} diff --git a/crates/ely_app/src/shell/web_surface_view.rs b/crates/ely_app/src/shell/web_surface_view.rs index efcbc8f..227aab3 100644 --- a/crates/ely_app/src/shell/web_surface_view.rs +++ b/crates/ely_app/src/shell/web_surface_view.rs @@ -1,6 +1,6 @@ use ely_domain::{BrowserTab, TabId}; use gpui::{ - AnyElement, App, Entity, ImageSource, InteractiveElement, IntoElement, ObjectFit, + AnyElement, App, Entity, ImageSource, InteractiveElement, IntoElement, MouseButton, ObjectFit, ParentElement, Styled, StyledImage, Window, canvas, div, img, prelude::FluentBuilder, px, rgb, }; use gpui_component::StyledExt; @@ -104,9 +104,9 @@ fn render_web_surface( detail: Option, content: impl IntoElement, ) -> AnyElement { - let scroll_tab_id = tab.id().clone(); - let scroll_url = tab.url().as_str().to_string(); - let scroll_entity = state_entity.clone(); + let input_tab_id = tab.id().clone(); + let input_url = tab.url().as_str().to_string(); + let input_entity = state_entity.clone(); let tracker_entity = state_entity; div() @@ -132,25 +132,58 @@ fn render_web_surface( .min_h_0() .overflow_hidden() .bg(rgb(colors::SURFACE_CARD)) - .on_scroll_wheel(move |event, window, cx| { - let delta = event.delta.pixel_delta(window.line_height()); - scroll_entity.update(cx, |shell, cx| { - shell.scroll_external_web_viewport( - scroll_tab_id.clone(), - scroll_url.clone(), - delta, - cx, - ); - }); - cx.stop_propagation(); - }) .child(content) - .child(render_viewport_tracker(tab.id().clone(), tracker_entity)), + .child(render_viewport_tracker(tab.id().clone(), tracker_entity)) + .child(render_input_overlay(input_tab_id, input_url, input_entity)), ), ) .into_any_element() } +fn render_input_overlay( + tab_id: TabId, + url: String, + state_entity: Entity, +) -> impl IntoElement { + let click_tab_id = tab_id.clone(); + let click_url = url.clone(); + let click_entity = state_entity.clone(); + let scroll_tab_id = tab_id; + let scroll_url = url; + let scroll_entity = state_entity; + + div() + .absolute() + .size_full() + .occlude() + .capture_any_mouse_up(move |event, _window, cx| { + if event.button != MouseButton::Left { + return; + } + click_entity.update(cx, |shell, cx| { + shell.click_external_web_viewport( + click_tab_id.clone(), + click_url.clone(), + event.position, + cx, + ); + }); + cx.stop_propagation(); + }) + .on_scroll_wheel(move |event, window, cx| { + let delta = event.delta.pixel_delta(window.line_height()); + scroll_entity.update(cx, |shell, cx| { + shell.scroll_external_web_viewport( + scroll_tab_id.clone(), + scroll_url.clone(), + delta, + cx, + ); + }); + cx.stop_propagation(); + }) +} + fn render_viewport_tracker(tab_id: TabId, state_entity: Entity) -> impl IntoElement { canvas( move |bounds, _window: &mut Window, cx: &mut App| { diff --git a/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs b/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs index 58c8bdd..1ec829e 100644 --- a/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs +++ b/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs @@ -8,8 +8,8 @@ use std::{ use ely_domain::{ProfileId, TabId, UrlText}; use ely_servo_host::{ - NavigationRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, ServoSurfaceSize, - SoftwareServoHost, WebViewSnapshot, WebViewState, + MouseClickRequest, NavigationRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, + ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState, }; use serde::Serialize; use thiserror::Error; @@ -17,7 +17,7 @@ use thiserror::Error; const WAIT_ITERATIONS: usize = 5_000; const WAIT_INTERVAL: Duration = Duration::from_millis(2); const RENDER_TIMEOUT: Duration = Duration::from_secs(20); -const SCROLL_SETTLE_TIMEOUT: Duration = Duration::from_millis(700); +const INPUT_SETTLE_TIMEOUT: Duration = Duration::from_millis(700); fn main() -> Result<(), SidecarError> { match parse_command(env::args())? { @@ -36,6 +36,13 @@ struct SnapshotArgs { height: u32, scroll_x: i32, scroll_y: i32, + click_point: Option, +} + +#[derive(Clone, Copy)] +struct ClickPoint { + x: u32, + y: u32, } #[derive(Debug, Error)] @@ -66,6 +73,9 @@ enum SidecarError { #[error("{name} must be greater than zero")] ZeroDimension { name: &'static str }, + #[error("--click-x and --click-y must be provided together")] + IncompleteClickPoint, + #[error("rgba output path is empty")] EmptyRgbaOutputPath, @@ -106,6 +116,8 @@ fn parse_snapshot_args( let mut height = None; let mut scroll_x = 0; let mut scroll_y = 0; + let mut click_x = None; + let mut click_y = None; while let Some(name) = args.next() { match name.as_str() { @@ -127,10 +139,28 @@ fn parse_snapshot_args( scroll_y = parse_scroll_delta("--scroll-y", next_argument(&mut args, "--scroll-y")?)? } + "--click-x" => { + click_x = Some(parse_click_coordinate( + "--click-x", + next_argument(&mut args, "--click-x")?, + )?) + } + "--click-y" => { + click_y = Some(parse_click_coordinate( + "--click-y", + next_argument(&mut args, "--click-y")?, + )?) + } _ => return Err(SidecarError::UnknownArgument { value: name }), } } + let click_point = match (click_x, click_y) { + (Some(x), Some(y)) => Some(ClickPoint { x, y }), + (None, None) => None, + _ => return Err(SidecarError::IncompleteClickPoint), + }; + Ok(SnapshotArgs { url: url.ok_or(SidecarError::MissingRequiredArgument { name: "--url" })?, rgba_out: rgba_out.ok_or(SidecarError::MissingRequiredArgument { name: "--rgba-out" })?, @@ -138,6 +168,7 @@ fn parse_snapshot_args( height: height.ok_or(SidecarError::MissingRequiredArgument { name: "--height" })?, scroll_x, scroll_y, + click_point, }) } @@ -165,6 +196,10 @@ fn parse_scroll_delta(name: &'static str, value: String) -> Result().map_err(|source| SidecarError::InvalidInteger { name, value, source }) } +fn parse_click_coordinate(name: &'static str, value: String) -> Result { + value.parse::().map_err(|source| SidecarError::InvalidInteger { name, value, source }) +} + fn parse_output_path(value: String) -> Result { if value.trim().is_empty() { return Err(SidecarError::EmptyRgbaOutputPath); @@ -188,20 +223,14 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> { let snapshot = wait_for_frame(&mut host, &webview_id, args.url.as_str())?; let (snapshot, scroll_changed_frame) = apply_scroll_if_requested(&mut host, &webview_id, &args, snapshot)?; + let (snapshot, click_changed_frame) = + apply_click_if_requested(&mut host, &webview_id, &args, snapshot)?; let frame = host.last_rendered_frame()?; std::fs::write(&args.rgba_out, frame.rgba_bytes())?; serde_json::to_writer( std::io::stdout().lock(), - &SnapshotReport::new( - args.url.as_str(), - &args.rgba_out, - &snapshot, - &frame, - args.scroll_x, - args.scroll_y, - scroll_changed_frame, - ), + &SnapshotReport::new(&args, &snapshot, &frame, scroll_changed_frame, click_changed_frame), )?; Ok(()) } @@ -225,6 +254,25 @@ fn apply_scroll_if_requested( wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash) } +fn apply_click_if_requested( + host: &mut SoftwareServoHost, + webview_id: &ely_domain::WebViewId, + args: &SnapshotArgs, + snapshot: WebViewSnapshot, +) -> Result<(WebViewSnapshot, bool), SidecarError> { + let Some(click_point) = args.click_point else { + return Ok((snapshot, false)); + }; + + let previous_frame_hash = host.last_rendered_frame()?.sample_hash(); + host.click(MouseClickRequest { + webview_id: webview_id.clone(), + x: click_point.x, + y: click_point.y, + })?; + wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash) +} + fn wait_for_frame( host: &mut SoftwareServoHost, webview_id: &ely_domain::WebViewId, @@ -267,7 +315,7 @@ fn wait_for_changed_or_settled_frame( let mut latest_snapshot = host.snapshot(webview_id)?; for _ in 0..WAIT_ITERATIONS { - if started_at.elapsed() >= SCROLL_SETTLE_TIMEOUT { + if started_at.elapsed() >= INPUT_SETTLE_TIMEOUT { break; } @@ -308,23 +356,24 @@ struct SnapshotReport { scroll_x: i32, scroll_y: i32, scroll_changed_frame: bool, + click_x: Option, + click_y: Option, + click_changed_frame: bool, } impl SnapshotReport { fn new( - requested_url: &str, - rgba_path: &std::path::Path, + args: &SnapshotArgs, snapshot: &WebViewSnapshot, frame: &RenderedFrame, - scroll_x: i32, - scroll_y: i32, scroll_changed_frame: bool, + click_changed_frame: bool, ) -> Self { Self { - requested_url: requested_url.to_string(), + requested_url: args.url.as_str().to_string(), loaded_url: snapshot.url().map(str::to_string), title: snapshot.title().map(str::to_string), - rgba_path: rgba_path.display().to_string(), + rgba_path: args.rgba_out.display().to_string(), state: state_label(snapshot.state()), width: frame.width(), height: frame.height(), @@ -333,9 +382,12 @@ impl SnapshotReport { non_white_pixel_count: frame.non_white_pixel_count(), content_pixel_count: frame.content_pixel_count(), sample_hash: frame.sample_hash(), - scroll_x, - scroll_y, + scroll_x: args.scroll_x, + scroll_y: args.scroll_y, scroll_changed_frame, + click_x: args.click_point.map(|point| point.x), + click_y: args.click_point.map(|point| point.y), + click_changed_frame, } } } diff --git a/crates/ely_servo_host/src/host.rs b/crates/ely_servo_host/src/host.rs index 1b8c7a4..011fc88 100644 --- a/crates/ely_servo_host/src/host.rs +++ b/crates/ely_servo_host/src/host.rs @@ -220,6 +220,13 @@ pub struct ScrollRequest { pub delta_y: i32, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MouseClickRequest { + pub webview_id: WebViewId, + pub x: u32, + pub y: u32, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct PermissionRequest { pub webview_id: WebViewId, @@ -246,6 +253,8 @@ pub trait ServoHost { fn scroll(&mut self, request: ScrollRequest) -> Result<(), ServoHostError>; + fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError>; + fn set_permission( &mut self, request: PermissionRequest, diff --git a/crates/ely_servo_host/src/lib.rs b/crates/ely_servo_host/src/lib.rs index 53f8aa1..54b75ce 100644 --- a/crates/ely_servo_host/src/lib.rs +++ b/crates/ely_servo_host/src/lib.rs @@ -5,8 +5,8 @@ mod runtime; pub use error::ServoHostError; pub use host::{ - NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, RenderedFrameSummary, - ScrollRequest, ServoHost, WebViewSnapshot, WebViewState, + MouseClickRequest, NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, + RenderedFrameSummary, ScrollRequest, ServoHost, WebViewSnapshot, WebViewState, }; #[cfg(feature = "servo-engine")] pub use runtime::{ServoSurfaceSize, SoftwareServoHost}; diff --git a/crates/ely_servo_host/src/runtime.rs b/crates/ely_servo_host/src/runtime.rs index 66d5a1f..e884c5f 100644 --- a/crates/ely_servo_host/src/runtime.rs +++ b/crates/ely_servo_host/src/runtime.rs @@ -12,14 +12,15 @@ use dpi::PhysicalSize; use ely_domain::{ProfileId, TabId, WebViewId}; use servo::{ DeviceIntPoint, DeviceIntRect, DeviceIntSize, DevicePoint, DeviceVector2D, EventLoopWaker, - LoadStatus, RenderingContext, Scroll, Servo, ServoBuilder, WebView, WebViewBuilder, - WebViewDelegate, WebViewPoint, WebViewVector, + InputEvent, LoadStatus, MouseButton, MouseButtonAction, MouseButtonEvent, MouseMoveEvent, + RenderingContext, Scroll, Servo, ServoBuilder, WebView, WebViewBuilder, WebViewDelegate, + WebViewPoint, WebViewVector, }; use url::Url; use crate::{ - NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, ScrollRequest, - ServoHost, ServoHostError, WebViewSnapshot, WebViewState, + MouseClickRequest, NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, + ScrollRequest, ServoHost, ServoHostError, WebViewSnapshot, WebViewState, }; static SERVO_RUNTIME_STARTED: AtomicBool = AtomicBool::new(false); @@ -163,6 +164,27 @@ impl ServoHost for SoftwareServoHost { Ok(()) } + fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError> { + let webview = self + .webviews + .get(&request.webview_id) + .ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?; + + let point = WebViewPoint::Device(DevicePoint::new(request.x as f32, request.y as f32)); + webview.webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point))); + webview.webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new( + MouseButtonAction::Down, + MouseButton::Left, + point, + ))); + webview.webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new( + MouseButtonAction::Up, + MouseButton::Left, + point, + ))); + Ok(()) + } + fn set_permission( &mut self, request: PermissionRequest, diff --git a/crates/ely_servo_host/tests/sidecar.rs b/crates/ely_servo_host/tests/sidecar.rs index e9b3bf5..36e9444 100644 --- a/crates/ely_servo_host/tests/sidecar.rs +++ b/crates/ely_servo_host/tests/sidecar.rs @@ -24,6 +24,9 @@ const SERVO_SCROLL_SITE: PrdSiteCompatibilityCase = PrdSiteCompatibilityCase { url: "https://servo.org", title_fragment: "Servo" }; const SERVO_SCROLL_SIZE: FrameSize = FrameSize { width: 934, height: 657 }; const SERVO_SCROLL_OFFSET: ScrollOffset = ScrollOffset { x: 0, y: 480 }; +const SERVO_CLICK_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E"; +const SERVO_CLICK_SIZE: FrameSize = FrameSize { width: 640, height: 480 }; +const SERVO_CLICK_POINT: ClickPoint = ClickPoint { x: 160, y: 120 }; struct PrdSiteCompatibilityCase { url: &'static str, @@ -46,6 +49,12 @@ impl ScrollOffset { const ZERO: Self = Self { x: 0, y: 0 }; } +#[derive(Clone, Copy)] +struct ClickPoint { + x: u64, + y: u64, +} + #[test] fn sidecar_snapshots_prd_sites_to_rgba_files() -> Result<(), Box> { for case in PRD_SITE_COMPATIBILITY_CASES { @@ -87,6 +96,22 @@ fn sidecar_scrolls_prd_site_with_servo_input() -> Result<(), Box> { Ok(()) } +#[test] +fn sidecar_clicks_page_with_servo_mouse_input() -> Result<(), Box> { + let initial_report = snapshot_click_probe(None)?; + let clicked_report = snapshot_click_probe(Some(SERVO_CLICK_POINT))?; + + assert_eq!(report_field_as_u64(&clicked_report, "click_x")?, SERVO_CLICK_POINT.x); + assert_eq!(report_field_as_u64(&clicked_report, "click_y")?, SERVO_CLICK_POINT.y); + assert!(report_field_as_bool(&clicked_report, "click_changed_frame")?); + assert_ne!( + report_field_as_u64(&initial_report, "sample_hash")?, + report_field_as_u64(&clicked_report, "sample_hash")? + ); + + Ok(()) +} + fn snapshot_prd_site( case: &PrdSiteCompatibilityCase, size: FrameSize, @@ -110,7 +135,7 @@ fn snapshot_prd_site( std::fs::remove_file(&output_path)?; } - let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset)?; + let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset, None)?; assert!( output.status.success(), @@ -147,11 +172,55 @@ fn snapshot_prd_site( Ok(report) } +fn snapshot_click_probe( + click_point: Option, +) -> Result> { + let output_path = std::env::temp_dir().join(format!( + "ely-servo-sidecar-{}-click-{}x{}.rgba", + std::process::id(), + SERVO_CLICK_SIZE.width, + SERVO_CLICK_SIZE.height + )); + + if output_path.exists() { + std::fs::remove_file(&output_path)?; + } + + let output = run_sidecar_snapshot( + SERVO_CLICK_URL, + &output_path, + SERVO_CLICK_SIZE, + ScrollOffset::ZERO, + click_point, + )?; + + assert!( + output.status.success(), + "click probe\nstatus: {:?}\nstdout: {}\nstderr: {}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let report: serde_json::Value = serde_json::from_slice(&output.stdout)?; + assert_eq!(report_field_as_u64(&report, "width")?, SERVO_CLICK_SIZE.width); + assert_eq!(report_field_as_u64(&report, "height")?, SERVO_CLICK_SIZE.height); + assert!(report_field_as_u64(&report, "content_pixel_count")? > 0); + assert_eq!( + std::fs::metadata(&output_path)?.len(), + SERVO_CLICK_SIZE.width * SERVO_CLICK_SIZE.height * 4 + ); + + std::fs::remove_file(&output_path)?; + Ok(report) +} + fn run_sidecar_snapshot( site_url: &str, output_path: &std::path::Path, size: FrameSize, scroll_offset: ScrollOffset, + click_point: Option, ) -> Result> { let mut command = Command::new(env!("CARGO_BIN_EXE_ely_servo_sidecar")); command @@ -170,6 +239,10 @@ fn run_sidecar_snapshot( if scroll_offset.y != 0 { command.arg("--scroll-y").arg(scroll_offset.y.to_string()); } + if let Some(click_point) = click_point { + command.arg("--click-x").arg(click_point.x.to_string()); + command.arg("--click-y").arg(click_point.y.to_string()); + } let mut child = command.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; diff --git a/crates/ely_servo_host/tests/software_host.rs b/crates/ely_servo_host/tests/software_host.rs index 5657e49..22e1150 100644 --- a/crates/ely_servo_host/tests/software_host.rs +++ b/crates/ely_servo_host/tests/software_host.rs @@ -4,8 +4,8 @@ use std::{error::Error, thread, time::Duration}; use ely_domain::{ProfileId, TabId, UrlText}; use ely_servo_host::{ - NavigationRequest, ScrollRequest, ServoHost, ServoHostError, ServoSurfaceSize, - SoftwareServoHost, WebViewState, + MouseClickRequest, NavigationRequest, ScrollRequest, ServoHost, ServoHostError, + ServoSurfaceSize, SoftwareServoHost, WebViewState, }; const MINIMUM_CONTENT_PIXELS: u64 = 1_000; @@ -13,6 +13,7 @@ const PRD_SITE_COMPATIBILITY_CASES: &[PrdSiteCompatibilityCase] = &[ PrdSiteCompatibilityCase { url: "https://example.com", title_fragment: "Example Domain" }, PrdSiteCompatibilityCase { url: "https://servo.org", title_fragment: "Servo" }, ]; +const CLICK_PROBE_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E"; struct PrdSiteCompatibilityCase { url: &'static str, @@ -33,9 +34,7 @@ fn manages_real_servo_webview_lifecycle() -> Result<(), Box> { assert_eq!(snapshot.profile_id(), &profile_id); assert_eq!(snapshot.state(), &WebViewState::Created); - let url = UrlText::parse( - "data:text/html,%3Ctitle%3EELY%20Host%3C%2Ftitle%3E%3Cmain%3EReady%3C%2Fmain%3E", - )?; + let url = UrlText::parse(CLICK_PROBE_URL)?; host.navigate(NavigationRequest { webview_id: webview_id.clone(), tab_id, url })?; @@ -47,6 +46,13 @@ fn manages_real_servo_webview_lifecycle() -> Result<(), Box> { ); assert_rendered_frame_has_content(&host, "data:text/html", 1)?; + let previous_frame_hash = host.last_rendered_frame()?.sample_hash(); + host.click(MouseClickRequest { webview_id: webview_id.clone(), x: 160, y: 120 })?; + let snapshot = wait_for_rendered_webview(&mut host, &webview_id, Some(previous_frame_hash))?; + assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}"); + assert_rendered_frame_has_content(&host, "data:text/html clicked", 1)?; + assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash); + let mut previous_frame_hash = Some(host.last_rendered_frame()?.sample_hash()); for site in PRD_SITE_COMPATIBILITY_CASES { let tab_id = TabId::new();