Bridge Servo click input

crates/ely_servo_host/src/bin/ely_servo_sidecar.rs

@@ -8,8 +8,8 @@ use std::{
 
 use ely_domain::{ProfileId, TabId, UrlText};
 use ely_servo_host::{
-    NavigationRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, ServoSurfaceSize,
-    SoftwareServoHost, WebViewSnapshot, WebViewState,
+    MouseClickRequest, NavigationRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError,
+    ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState,
 };
 use serde::Serialize;
 use thiserror::Error;
@@ -17,7 +17,7 @@ use thiserror::Error;
 const WAIT_ITERATIONS: usize = 5_000;
 const WAIT_INTERVAL: Duration = Duration::from_millis(2);
 const RENDER_TIMEOUT: Duration = Duration::from_secs(20);
-const SCROLL_SETTLE_TIMEOUT: Duration = Duration::from_millis(700);
+const INPUT_SETTLE_TIMEOUT: Duration = Duration::from_millis(700);
 
 fn main() -> Result<(), SidecarError> {
     match parse_command(env::args())? {
@@ -36,6 +36,13 @@ struct SnapshotArgs {
     height: u32,
     scroll_x: i32,
     scroll_y: i32,
+    click_point: Option<ClickPoint>,
+}
+
+#[derive(Clone, Copy)]
+struct ClickPoint {
+    x: u32,
+    y: u32,
 }
 
 #[derive(Debug, Error)]
@@ -66,6 +73,9 @@ enum SidecarError {
     #[error("{name} must be greater than zero")]
     ZeroDimension { name: &'static str },
 
+    #[error("--click-x and --click-y must be provided together")]
+    IncompleteClickPoint,
+
     #[error("rgba output path is empty")]
     EmptyRgbaOutputPath,
 
@@ -106,6 +116,8 @@ fn parse_snapshot_args(
     let mut height = None;
     let mut scroll_x = 0;
     let mut scroll_y = 0;
+    let mut click_x = None;
+    let mut click_y = None;
 
     while let Some(name) = args.next() {
         match name.as_str() {
@@ -127,10 +139,28 @@ fn parse_snapshot_args(
                 scroll_y =
                     parse_scroll_delta("--scroll-y", next_argument(&mut args, "--scroll-y")?)?
             }
+            "--click-x" => {
+                click_x = Some(parse_click_coordinate(
+                    "--click-x",
+                    next_argument(&mut args, "--click-x")?,
+                )?)
+            }
+            "--click-y" => {
+                click_y = Some(parse_click_coordinate(
+                    "--click-y",
+                    next_argument(&mut args, "--click-y")?,
+                )?)
+            }
             _ => return Err(SidecarError::UnknownArgument { value: name }),
         }
     }
 
+    let click_point = match (click_x, click_y) {
+        (Some(x), Some(y)) => Some(ClickPoint { x, y }),
+        (None, None) => None,
+        _ => return Err(SidecarError::IncompleteClickPoint),
+    };
+
     Ok(SnapshotArgs {
         url: url.ok_or(SidecarError::MissingRequiredArgument { name: "--url" })?,
         rgba_out: rgba_out.ok_or(SidecarError::MissingRequiredArgument { name: "--rgba-out" })?,
@@ -138,6 +168,7 @@ fn parse_snapshot_args(
         height: height.ok_or(SidecarError::MissingRequiredArgument { name: "--height" })?,
         scroll_x,
         scroll_y,
+        click_point,
     })
 }
 
@@ -165,6 +196,10 @@ fn parse_scroll_delta(name: &'static str, value: String) -> Result<i32, SidecarE
     value.parse::<i32>().map_err(|source| SidecarError::InvalidInteger { name, value, source })
 }
 
+fn parse_click_coordinate(name: &'static str, value: String) -> Result<u32, SidecarError> {
+    value.parse::<u32>().map_err(|source| SidecarError::InvalidInteger { name, value, source })
+}
+
 fn parse_output_path(value: String) -> Result<PathBuf, SidecarError> {
     if value.trim().is_empty() {
         return Err(SidecarError::EmptyRgbaOutputPath);
@@ -188,20 +223,14 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> {
     let snapshot = wait_for_frame(&mut host, &webview_id, args.url.as_str())?;
     let (snapshot, scroll_changed_frame) =
         apply_scroll_if_requested(&mut host, &webview_id, &args, snapshot)?;
+    let (snapshot, click_changed_frame) =
+        apply_click_if_requested(&mut host, &webview_id, &args, snapshot)?;
     let frame = host.last_rendered_frame()?;
     std::fs::write(&args.rgba_out, frame.rgba_bytes())?;
 
     serde_json::to_writer(
         std::io::stdout().lock(),
-        &SnapshotReport::new(
-            args.url.as_str(),
-            &args.rgba_out,
-            &snapshot,
-            &frame,
-            args.scroll_x,
-            args.scroll_y,
-            scroll_changed_frame,
-        ),
+        &SnapshotReport::new(&args, &snapshot, &frame, scroll_changed_frame, click_changed_frame),
     )?;
     Ok(())
 }
@@ -225,6 +254,25 @@ fn apply_scroll_if_requested(
     wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash)
 }
 
+fn apply_click_if_requested(
+    host: &mut SoftwareServoHost,
+    webview_id: &ely_domain::WebViewId,
+    args: &SnapshotArgs,
+    snapshot: WebViewSnapshot,
+) -> Result<(WebViewSnapshot, bool), SidecarError> {
+    let Some(click_point) = args.click_point else {
+        return Ok((snapshot, false));
+    };
+
+    let previous_frame_hash = host.last_rendered_frame()?.sample_hash();
+    host.click(MouseClickRequest {
+        webview_id: webview_id.clone(),
+        x: click_point.x,
+        y: click_point.y,
+    })?;
+    wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash)
+}
+
 fn wait_for_frame(
     host: &mut SoftwareServoHost,
     webview_id: &ely_domain::WebViewId,
@@ -267,7 +315,7 @@ fn wait_for_changed_or_settled_frame(
     let mut latest_snapshot = host.snapshot(webview_id)?;
 
     for _ in 0..WAIT_ITERATIONS {
-        if started_at.elapsed() >= SCROLL_SETTLE_TIMEOUT {
+        if started_at.elapsed() >= INPUT_SETTLE_TIMEOUT {
             break;
         }
 
@@ -308,23 +356,24 @@ struct SnapshotReport {
     scroll_x: i32,
     scroll_y: i32,
     scroll_changed_frame: bool,
+    click_x: Option<u32>,
+    click_y: Option<u32>,
+    click_changed_frame: bool,
 }
 
 impl SnapshotReport {
     fn new(
-        requested_url: &str,
-        rgba_path: &std::path::Path,
+        args: &SnapshotArgs,
         snapshot: &WebViewSnapshot,
         frame: &RenderedFrame,
-        scroll_x: i32,
-        scroll_y: i32,
         scroll_changed_frame: bool,
+        click_changed_frame: bool,
     ) -> Self {
         Self {
-            requested_url: requested_url.to_string(),
+            requested_url: args.url.as_str().to_string(),
             loaded_url: snapshot.url().map(str::to_string),
             title: snapshot.title().map(str::to_string),
-            rgba_path: rgba_path.display().to_string(),
+            rgba_path: args.rgba_out.display().to_string(),
             state: state_label(snapshot.state()),
             width: frame.width(),
             height: frame.height(),
@@ -333,9 +382,12 @@ impl SnapshotReport {
             non_white_pixel_count: frame.non_white_pixel_count(),
             content_pixel_count: frame.content_pixel_count(),
             sample_hash: frame.sample_hash(),
-            scroll_x,
-            scroll_y,
+            scroll_x: args.scroll_x,
+            scroll_y: args.scroll_y,
             scroll_changed_frame,
+            click_x: args.click_point.map(|point| point.x),
+            click_y: args.click_point.map(|point| point.y),
+            click_changed_frame,
         }
     }
 }

crates/ely_servo_host/src/host.rs

@@ -220,6 +220,13 @@ pub struct ScrollRequest {
     pub delta_y: i32,
 }
 
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct MouseClickRequest {
+    pub webview_id: WebViewId,
+    pub x: u32,
+    pub y: u32,
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct PermissionRequest {
     pub webview_id: WebViewId,
@@ -246,6 +253,8 @@ pub trait ServoHost {
 
     fn scroll(&mut self, request: ScrollRequest) -> Result<(), ServoHostError>;
 
+    fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError>;
+
     fn set_permission(
         &mut self,
         request: PermissionRequest,

crates/ely_servo_host/src/lib.rs

@@ -5,8 +5,8 @@ mod runtime;
 
 pub use error::ServoHostError;
 pub use host::{
-    NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, RenderedFrameSummary,
-    ScrollRequest, ServoHost, WebViewSnapshot, WebViewState,
+    MouseClickRequest, NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame,
+    RenderedFrameSummary, ScrollRequest, ServoHost, WebViewSnapshot, WebViewState,
 };
 #[cfg(feature = "servo-engine")]
 pub use runtime::{ServoSurfaceSize, SoftwareServoHost};

crates/ely_servo_host/src/runtime.rs

@@ -12,14 +12,15 @@ use dpi::PhysicalSize;
 use ely_domain::{ProfileId, TabId, WebViewId};
 use servo::{
     DeviceIntPoint, DeviceIntRect, DeviceIntSize, DevicePoint, DeviceVector2D, EventLoopWaker,
-    LoadStatus, RenderingContext, Scroll, Servo, ServoBuilder, WebView, WebViewBuilder,
-    WebViewDelegate, WebViewPoint, WebViewVector,
+    InputEvent, LoadStatus, MouseButton, MouseButtonAction, MouseButtonEvent, MouseMoveEvent,
+    RenderingContext, Scroll, Servo, ServoBuilder, WebView, WebViewBuilder, WebViewDelegate,
+    WebViewPoint, WebViewVector,
 };
 use url::Url;
 
 use crate::{
-    NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, ScrollRequest,
-    ServoHost, ServoHostError, WebViewSnapshot, WebViewState,
+    MouseClickRequest, NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame,
+    ScrollRequest, ServoHost, ServoHostError, WebViewSnapshot, WebViewState,
 };
 
 static SERVO_RUNTIME_STARTED: AtomicBool = AtomicBool::new(false);
@@ -163,6 +164,27 @@ impl ServoHost for SoftwareServoHost {
         Ok(())
     }
 
+    fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError> {
+        let webview = self
+            .webviews
+            .get(&request.webview_id)
+            .ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?;
+
+        let point = WebViewPoint::Device(DevicePoint::new(request.x as f32, request.y as f32));
+        webview.webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point)));
+        webview.webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
+            MouseButtonAction::Down,
+            MouseButton::Left,
+            point,
+        )));
+        webview.webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
+            MouseButtonAction::Up,
+            MouseButton::Left,
+            point,
+        )));
+        Ok(())
+    }
+
     fn set_permission(
         &mut self,
         request: PermissionRequest,

crates/ely_servo_host/tests/sidecar.rs

@@ -24,6 +24,9 @@ const SERVO_SCROLL_SITE: PrdSiteCompatibilityCase =
     PrdSiteCompatibilityCase { url: "https://servo.org", title_fragment: "Servo" };
 const SERVO_SCROLL_SIZE: FrameSize = FrameSize { width: 934, height: 657 };
 const SERVO_SCROLL_OFFSET: ScrollOffset = ScrollOffset { x: 0, y: 480 };
+const SERVO_CLICK_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E";
+const SERVO_CLICK_SIZE: FrameSize = FrameSize { width: 640, height: 480 };
+const SERVO_CLICK_POINT: ClickPoint = ClickPoint { x: 160, y: 120 };
 
 struct PrdSiteCompatibilityCase {
     url: &'static str,
@@ -46,6 +49,12 @@ impl ScrollOffset {
     const ZERO: Self = Self { x: 0, y: 0 };
 }
 
+#[derive(Clone, Copy)]
+struct ClickPoint {
+    x: u64,
+    y: u64,
+}
+
 #[test]
 fn sidecar_snapshots_prd_sites_to_rgba_files() -> Result<(), Box<dyn Error>> {
     for case in PRD_SITE_COMPATIBILITY_CASES {
@@ -87,6 +96,22 @@ fn sidecar_scrolls_prd_site_with_servo_input() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+#[test]
+fn sidecar_clicks_page_with_servo_mouse_input() -> Result<(), Box<dyn Error>> {
+    let initial_report = snapshot_click_probe(None)?;
+    let clicked_report = snapshot_click_probe(Some(SERVO_CLICK_POINT))?;
+
+    assert_eq!(report_field_as_u64(&clicked_report, "click_x")?, SERVO_CLICK_POINT.x);
+    assert_eq!(report_field_as_u64(&clicked_report, "click_y")?, SERVO_CLICK_POINT.y);
+    assert!(report_field_as_bool(&clicked_report, "click_changed_frame")?);
+    assert_ne!(
+        report_field_as_u64(&initial_report, "sample_hash")?,
+        report_field_as_u64(&clicked_report, "sample_hash")?
+    );
+
+    Ok(())
+}
+
 fn snapshot_prd_site(
     case: &PrdSiteCompatibilityCase,
     size: FrameSize,
@@ -110,7 +135,7 @@ fn snapshot_prd_site(
         std::fs::remove_file(&output_path)?;
     }
 
-    let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset)?;
+    let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset, None)?;
 
     assert!(
         output.status.success(),
@@ -147,11 +172,55 @@ fn snapshot_prd_site(
     Ok(report)
 }
 
+fn snapshot_click_probe(
+    click_point: Option<ClickPoint>,
+) -> Result<serde_json::Value, Box<dyn Error>> {
+    let output_path = std::env::temp_dir().join(format!(
+        "ely-servo-sidecar-{}-click-{}x{}.rgba",
+        std::process::id(),
+        SERVO_CLICK_SIZE.width,
+        SERVO_CLICK_SIZE.height
+    ));
+
+    if output_path.exists() {
+        std::fs::remove_file(&output_path)?;
+    }
+
+    let output = run_sidecar_snapshot(
+        SERVO_CLICK_URL,
+        &output_path,
+        SERVO_CLICK_SIZE,
+        ScrollOffset::ZERO,
+        click_point,
+    )?;
+
+    assert!(
+        output.status.success(),
+        "click probe\nstatus: {:?}\nstdout: {}\nstderr: {}",
+        output.status.code(),
+        String::from_utf8_lossy(&output.stdout),
+        String::from_utf8_lossy(&output.stderr)
+    );
+
+    let report: serde_json::Value = serde_json::from_slice(&output.stdout)?;
+    assert_eq!(report_field_as_u64(&report, "width")?, SERVO_CLICK_SIZE.width);
+    assert_eq!(report_field_as_u64(&report, "height")?, SERVO_CLICK_SIZE.height);
+    assert!(report_field_as_u64(&report, "content_pixel_count")? > 0);
+    assert_eq!(
+        std::fs::metadata(&output_path)?.len(),
+        SERVO_CLICK_SIZE.width * SERVO_CLICK_SIZE.height * 4
+    );
+
+    std::fs::remove_file(&output_path)?;
+    Ok(report)
+}
+
 fn run_sidecar_snapshot(
     site_url: &str,
     output_path: &std::path::Path,
     size: FrameSize,
     scroll_offset: ScrollOffset,
+    click_point: Option<ClickPoint>,
 ) -> Result<Output, Box<dyn Error>> {
     let mut command = Command::new(env!("CARGO_BIN_EXE_ely_servo_sidecar"));
     command
@@ -170,6 +239,10 @@ fn run_sidecar_snapshot(
     if scroll_offset.y != 0 {
         command.arg("--scroll-y").arg(scroll_offset.y.to_string());
     }
+    if let Some(click_point) = click_point {
+        command.arg("--click-x").arg(click_point.x.to_string());
+        command.arg("--click-y").arg(click_point.y.to_string());
+    }
 
     let mut child = command.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?;
 

crates/ely_servo_host/tests/software_host.rs

@@ -4,8 +4,8 @@ use std::{error::Error, thread, time::Duration};
 
 use ely_domain::{ProfileId, TabId, UrlText};
 use ely_servo_host::{
-    NavigationRequest, ScrollRequest, ServoHost, ServoHostError, ServoSurfaceSize,
-    SoftwareServoHost, WebViewState,
+    MouseClickRequest, NavigationRequest, ScrollRequest, ServoHost, ServoHostError,
+    ServoSurfaceSize, SoftwareServoHost, WebViewState,
 };
 
 const MINIMUM_CONTENT_PIXELS: u64 = 1_000;
@@ -13,6 +13,7 @@ const PRD_SITE_COMPATIBILITY_CASES: &[PrdSiteCompatibilityCase] = &[
     PrdSiteCompatibilityCase { url: "https://example.com", title_fragment: "Example Domain" },
     PrdSiteCompatibilityCase { url: "https://servo.org", title_fragment: "Servo" },
 ];
+const CLICK_PROBE_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E";
 
 struct PrdSiteCompatibilityCase {
     url: &'static str,
@@ -33,9 +34,7 @@ fn manages_real_servo_webview_lifecycle() -> Result<(), Box<dyn Error>> {
     assert_eq!(snapshot.profile_id(), &profile_id);
     assert_eq!(snapshot.state(), &WebViewState::Created);
 
-    let url = UrlText::parse(
-        "data:text/html,%3Ctitle%3EELY%20Host%3C%2Ftitle%3E%3Cmain%3EReady%3C%2Fmain%3E",
-    )?;
+    let url = UrlText::parse(CLICK_PROBE_URL)?;
 
     host.navigate(NavigationRequest { webview_id: webview_id.clone(), tab_id, url })?;
 
@@ -47,6 +46,13 @@ fn manages_real_servo_webview_lifecycle() -> Result<(), Box<dyn Error>> {
     );
     assert_rendered_frame_has_content(&host, "data:text/html", 1)?;
 
+    let previous_frame_hash = host.last_rendered_frame()?.sample_hash();
+    host.click(MouseClickRequest { webview_id: webview_id.clone(), x: 160, y: 120 })?;
+    let snapshot = wait_for_rendered_webview(&mut host, &webview_id, Some(previous_frame_hash))?;
+    assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}");
+    assert_rendered_frame_has_content(&host, "data:text/html clicked", 1)?;
+    assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash);
+
     let mut previous_frame_hash = Some(host.last_rendered_frame()?.sample_hash());
     for site in PRD_SITE_COMPATIBILITY_CASES {
         let tab_id = TabId::new();