Bridge Servo touch tap input

crates/ely_servo_host/src/bin/ely_servo_sidecar.rs

@@ -9,7 +9,8 @@ use std::{
 use ely_domain::{ProfileId, TabId, UrlText};
 use ely_servo_host::{
     KeyboardTextRequest, MouseClickRequest, NavigationRequest, RenderedFrame, ScrollRequest,
-    ServoHost, ServoHostError, ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState,
+    ServoHost, ServoHostError, ServoSurfaceSize, SoftwareServoHost, TouchTapRequest,
+    WebViewSnapshot, WebViewState,
 };
 use serde::Serialize;
 use thiserror::Error;
@@ -37,6 +38,7 @@ struct SnapshotArgs {
     scroll_x: i32,
     scroll_y: i32,
     click_point: Option<ClickPoint>,
+    touch_point: Option<ClickPoint>,
     typed_text: Option<String>,
 }
 
@@ -77,6 +79,9 @@ enum SidecarError {
     #[error("--click-x and --click-y must be provided together")]
     IncompleteClickPoint,
 
+    #[error("--touch-x and --touch-y must be provided together")]
+    IncompleteTouchPoint,
+
     #[error("rgba output path is empty")]
     EmptyRgbaOutputPath,
 
@@ -119,6 +124,8 @@ fn parse_snapshot_args(
     let mut scroll_y = 0;
     let mut click_x = None;
     let mut click_y = None;
+    let mut touch_x = None;
+    let mut touch_y = None;
     let mut typed_text = None;
 
     while let Some(name) = args.next() {
@@ -153,6 +160,18 @@ fn parse_snapshot_args(
                     next_argument(&mut args, "--click-y")?,
                 )?)
             }
+            "--touch-x" => {
+                touch_x = Some(parse_click_coordinate(
+                    "--touch-x",
+                    next_argument(&mut args, "--touch-x")?,
+                )?)
+            }
+            "--touch-y" => {
+                touch_y = Some(parse_click_coordinate(
+                    "--touch-y",
+                    next_argument(&mut args, "--touch-y")?,
+                )?)
+            }
             "--type-text" => typed_text = Some(next_argument(&mut args, "--type-text")?),
             _ => return Err(SidecarError::UnknownArgument { value: name }),
         }
@@ -163,6 +182,11 @@ fn parse_snapshot_args(
         (None, None) => None,
         _ => return Err(SidecarError::IncompleteClickPoint),
     };
+    let touch_point = match (touch_x, touch_y) {
+        (Some(x), Some(y)) => Some(ClickPoint { x, y }),
+        (None, None) => None,
+        _ => return Err(SidecarError::IncompleteTouchPoint),
+    };
 
     Ok(SnapshotArgs {
         url: url.ok_or(SidecarError::MissingRequiredArgument { name: "--url" })?,
@@ -172,6 +196,7 @@ fn parse_snapshot_args(
         scroll_x,
         scroll_y,
         click_point,
+        touch_point,
         typed_text,
     })
 }
@@ -229,6 +254,8 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> {
         apply_scroll_if_requested(&mut host, &webview_id, &args, snapshot)?;
     let (snapshot, click_changed_frame) =
         apply_click_if_requested(&mut host, &webview_id, &args, snapshot)?;
+    let (snapshot, touch_changed_frame) =
+        apply_touch_if_requested(&mut host, &webview_id, &args, snapshot)?;
     let (snapshot, text_changed_frame) =
         apply_text_if_requested(&mut host, &webview_id, &args, snapshot)?;
     let frame = host.last_rendered_frame()?;
@@ -242,6 +269,7 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> {
             &frame,
             scroll_changed_frame,
             click_changed_frame,
+            touch_changed_frame,
             text_changed_frame,
         ),
     )?;
@@ -286,6 +314,25 @@ fn apply_click_if_requested(
     wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash)
 }
 
+fn apply_touch_if_requested(
+    host: &mut SoftwareServoHost,
+    webview_id: &ely_domain::WebViewId,
+    args: &SnapshotArgs,
+    snapshot: WebViewSnapshot,
+) -> Result<(WebViewSnapshot, bool), SidecarError> {
+    let Some(touch_point) = args.touch_point else {
+        return Ok((snapshot, false));
+    };
+
+    let previous_frame_hash = host.last_rendered_frame()?.sample_hash();
+    host.touch_tap(TouchTapRequest {
+        webview_id: webview_id.clone(),
+        x: touch_point.x,
+        y: touch_point.y,
+    })?;
+    wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash)
+}
+
 fn apply_text_if_requested(
     host: &mut SoftwareServoHost,
     webview_id: &ely_domain::WebViewId,
@@ -390,6 +437,9 @@ struct SnapshotReport {
     click_x: Option<u32>,
     click_y: Option<u32>,
     click_changed_frame: bool,
+    touch_x: Option<u32>,
+    touch_y: Option<u32>,
+    touch_changed_frame: bool,
     typed_text_byte_count: usize,
     text_changed_frame: bool,
 }
@@ -401,6 +451,7 @@ impl SnapshotReport {
         frame: &RenderedFrame,
         scroll_changed_frame: bool,
         click_changed_frame: bool,
+        touch_changed_frame: bool,
         text_changed_frame: bool,
     ) -> Self {
         Self {
@@ -422,6 +473,9 @@ impl SnapshotReport {
             click_x: args.click_point.map(|point| point.x),
             click_y: args.click_point.map(|point| point.y),
             click_changed_frame,
+            touch_x: args.touch_point.map(|point| point.x),
+            touch_y: args.touch_point.map(|point| point.y),
+            touch_changed_frame,
             typed_text_byte_count: args.typed_text.as_ref().map_or(0, String::len),
             text_changed_frame,
         }

crates/ely_servo_host/src/host.rs

@@ -227,6 +227,13 @@ pub struct MouseClickRequest {
     pub y: u32,
 }
 
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct TouchTapRequest {
+    pub webview_id: WebViewId,
+    pub x: u32,
+    pub y: u32,
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct KeyboardTextRequest {
     pub webview_id: WebViewId,
@@ -261,6 +268,8 @@ pub trait ServoHost {
 
     fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError>;
 
+    fn touch_tap(&mut self, request: TouchTapRequest) -> Result<(), ServoHostError>;
+
     fn type_text(&mut self, request: KeyboardTextRequest) -> Result<(), ServoHostError>;
 
     fn set_permission(

crates/ely_servo_host/src/lib.rs

@@ -9,7 +9,7 @@ pub use error::ServoHostError;
 pub use host::{
     KeyboardTextRequest, MouseClickRequest, NavigationRequest, PermissionDecision,
     PermissionRequest, RenderedFrame, RenderedFrameSummary, ScrollRequest, ServoHost,
-    WebViewSnapshot, WebViewState,
+    TouchTapRequest, WebViewSnapshot, WebViewState,
 };
 #[cfg(feature = "servo-engine")]
 pub use runtime::{ServoSurfaceSize, SoftwareServoHost};

crates/ely_servo_host/src/runtime.rs

@@ -14,14 +14,15 @@ use servo::{
     DeviceIntPoint, DeviceIntRect, DeviceIntSize, DevicePoint, DeviceVector2D, EventLoopWaker,
     InputEvent, Key, KeyState, KeyboardEvent, LoadStatus, Location, Modifiers, MouseButton,
     MouseButtonAction, MouseButtonEvent, MouseMoveEvent, RenderingContext, Scroll, Servo,
-    ServoBuilder, WebView, WebViewBuilder, WebViewDelegate, WebViewPoint, WebViewVector,
+    ServoBuilder, TouchEvent, TouchEventType, TouchId, WebView, WebViewBuilder, WebViewDelegate,
+    WebViewPoint, WebViewVector,
 };
 use url::Url;
 
 use crate::{
     KeyboardTextRequest, MouseClickRequest, NavigationRequest, PermissionDecision,
-    PermissionRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, WebViewSnapshot,
+    PermissionRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, TouchTapRequest,
-    WebViewState, keyboard::keyboard_code_for_character,
+    WebViewSnapshot, WebViewState, keyboard::keyboard_code_for_character,
 };
 
 static SERVO_RUNTIME_STARTED: AtomicBool = AtomicBool::new(false);
@@ -186,6 +187,22 @@ impl ServoHost for SoftwareServoHost {
         Ok(())
     }
 
+    fn touch_tap(&mut self, request: TouchTapRequest) -> Result<(), ServoHostError> {
+        let webview = self
+            .webviews
+            .get(&request.webview_id)
+            .ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?;
+
+        let point = WebViewPoint::Device(DevicePoint::new(request.x as f32, request.y as f32));
+        let touch_id = TouchId(1);
+        for event_type in [TouchEventType::Down, TouchEventType::Up] {
+            webview.webview.notify_input_event(InputEvent::Touch(TouchEvent::new(
+                event_type, touch_id, point,
+            )));
+        }
+        Ok(())
+    }
+
     fn type_text(&mut self, request: KeyboardTextRequest) -> Result<(), ServoHostError> {
         let webview = self
             .webviews

crates/ely_servo_host/tests/sidecar.rs

@@ -27,6 +27,9 @@ const SERVO_SCROLL_OFFSET: ScrollOffset = ScrollOffset { x: 0, y: 480 };
 const SERVO_CLICK_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E";
 const SERVO_CLICK_SIZE: FrameSize = FrameSize { width: 640, height: 480 };
 const SERVO_CLICK_POINT: ClickPoint = ClickPoint { x: 160, y: 120 };
+const SERVO_TOUCH_URL: &str = "data:text/html,%3C%21doctype%20html%3E%3Ctitle%3ETouch%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3Btouch-action%3Amanipulation%3B%7D%3C%2Fstyle%3E%3Cbutton%20ontouchstart%3D%22document.body.dataset.touch%3D%27start%27%3B%22%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Touched%27%3Bthis.textContent%3D%27Touched%27%3B%22%3ETap%3C%2Fbutton%3E";
+const SERVO_TOUCH_SIZE: FrameSize = FrameSize { width: 640, height: 480 };
+const SERVO_TOUCH_POINT: ClickPoint = ClickPoint { x: 160, y: 120 };
 const SERVO_TEXT_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EText%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3Bfont%3A28px%20sans-serif%3B%7Dinput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A260px%3Bheight%3A70px%3Bfont%3A28px%20sans-serif%3B%7Doutput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A180px%3Bfont%3A32px%20sans-serif%3B%7D%3C%2Fstyle%3E%3Cinput%20id%3Dq%20autofocus%20oninput%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.getElementById%28%27out%27%29.textContent%3Dthis.value%3B%22%3E%3Coutput%20id%3Dout%3Eempty%3C%2Foutput%3E";
 const SERVO_TEXT_SIZE: FrameSize = FrameSize { width: 640, height: 480 };
 const SERVO_TEXT_POINT: ClickPoint = ClickPoint { x: 160, y: 120 };
@@ -116,6 +119,22 @@ fn sidecar_clicks_page_with_servo_mouse_input() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+#[test]
+fn sidecar_touches_page_with_servo_touch_input() -> Result<(), Box<dyn Error>> {
+    let initial_report = snapshot_touch_probe(None)?;
+    let touched_report = snapshot_touch_probe(Some(SERVO_TOUCH_POINT))?;
+
+    assert_eq!(report_field_as_u64(&touched_report, "touch_x")?, SERVO_TOUCH_POINT.x);
+    assert_eq!(report_field_as_u64(&touched_report, "touch_y")?, SERVO_TOUCH_POINT.y);
+    assert!(report_field_as_bool(&touched_report, "touch_changed_frame")?);
+    assert_ne!(
+        report_field_as_u64(&initial_report, "sample_hash")?,
+        report_field_as_u64(&touched_report, "sample_hash")?
+    );
+
+    Ok(())
+}
+
 #[test]
 fn sidecar_types_text_with_servo_keyboard_input() -> Result<(), Box<dyn Error>> {
     let initial_report = snapshot_text_probe(None)?;
@@ -157,7 +176,8 @@ fn snapshot_prd_site(
         std::fs::remove_file(&output_path)?;
     }
 
-    let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset, None, None)?;
+    let output =
+        run_sidecar_snapshot(case.url, &output_path, size, scroll_offset, None, None, None)?;
 
     assert!(
         output.status.success(),
@@ -215,6 +235,7 @@ fn snapshot_click_probe(
         ScrollOffset::ZERO,
         click_point,
         None,
+        None,
     )?;
 
     assert!(
@@ -238,6 +259,51 @@ fn snapshot_click_probe(
     Ok(report)
 }
 
+fn snapshot_touch_probe(
+    touch_point: Option<ClickPoint>,
+) -> Result<serde_json::Value, Box<dyn Error>> {
+    let output_path = std::env::temp_dir().join(format!(
+        "ely-servo-sidecar-{}-touch-{}x{}.rgba",
+        std::process::id(),
+        SERVO_TOUCH_SIZE.width,
+        SERVO_TOUCH_SIZE.height
+    ));
+
+    if output_path.exists() {
+        std::fs::remove_file(&output_path)?;
+    }
+
+    let output = run_sidecar_snapshot(
+        SERVO_TOUCH_URL,
+        &output_path,
+        SERVO_TOUCH_SIZE,
+        ScrollOffset::ZERO,
+        None,
+        touch_point,
+        None,
+    )?;
+
+    assert!(
+        output.status.success(),
+        "touch probe\nstatus: {:?}\nstdout: {}\nstderr: {}",
+        output.status.code(),
+        String::from_utf8_lossy(&output.stdout),
+        String::from_utf8_lossy(&output.stderr)
+    );
+
+    let report: serde_json::Value = serde_json::from_slice(&output.stdout)?;
+    assert_eq!(report_field_as_u64(&report, "width")?, SERVO_TOUCH_SIZE.width);
+    assert_eq!(report_field_as_u64(&report, "height")?, SERVO_TOUCH_SIZE.height);
+    assert!(report_field_as_u64(&report, "content_pixel_count")? > 0);
+    assert_eq!(
+        std::fs::metadata(&output_path)?.len(),
+        SERVO_TOUCH_SIZE.width * SERVO_TOUCH_SIZE.height * 4
+    );
+
+    std::fs::remove_file(&output_path)?;
+    Ok(report)
+}
+
 fn snapshot_text_probe(typed_text: Option<&str>) -> Result<serde_json::Value, Box<dyn Error>> {
     let output_path = std::env::temp_dir().join(format!(
         "ely-servo-sidecar-{}-text-{}x{}.rgba",
@@ -257,6 +323,7 @@ fn snapshot_text_probe(typed_text: Option<&str>) -> Result<serde_json::Value, Bo
         SERVO_TEXT_SIZE,
         ScrollOffset::ZERO,
         click_point,
+        None,
         typed_text,
     )?;
 
@@ -287,6 +354,7 @@ fn run_sidecar_snapshot(
     size: FrameSize,
     scroll_offset: ScrollOffset,
     click_point: Option<ClickPoint>,
+    touch_point: Option<ClickPoint>,
     typed_text: Option<&str>,
 ) -> Result<Output, Box<dyn Error>> {
     let mut command = Command::new(env!("CARGO_BIN_EXE_ely_servo_sidecar"));
@@ -310,6 +378,10 @@ fn run_sidecar_snapshot(
         command.arg("--click-x").arg(click_point.x.to_string());
         command.arg("--click-y").arg(click_point.y.to_string());
     }
+    if let Some(touch_point) = touch_point {
+        command.arg("--touch-x").arg(touch_point.x.to_string());
+        command.arg("--touch-y").arg(touch_point.y.to_string());
+    }
     if let Some(typed_text) = typed_text {
         command.arg("--type-text").arg(typed_text);
     }

crates/ely_servo_host/tests/software_host.rs

@@ -5,7 +5,7 @@ use std::{error::Error, thread, time::Duration};
 use ely_domain::{ProfileId, TabId, UrlText};
 use ely_servo_host::{
     KeyboardTextRequest, MouseClickRequest, NavigationRequest, ScrollRequest, ServoHost,
-    ServoHostError, ServoSurfaceSize, SoftwareServoHost, WebViewState,
+    ServoHostError, ServoSurfaceSize, SoftwareServoHost, TouchTapRequest, WebViewState,
 };
 
 const MINIMUM_CONTENT_PIXELS: u64 = 1_000;
@@ -14,6 +14,7 @@ const PRD_SITE_COMPATIBILITY_CASES: &[PrdSiteCompatibilityCase] = &[
     PrdSiteCompatibilityCase { url: "https://servo.org", title_fragment: "Servo" },
 ];
 const CLICK_PROBE_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E";
+const TOUCH_PROBE_URL: &str = "data:text/html,%3C%21doctype%20html%3E%3Ctitle%3ETouch%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3Btouch-action%3Amanipulation%3B%7D%3C%2Fstyle%3E%3Cbutton%20ontouchstart%3D%22document.body.dataset.touch%3D%27start%27%3B%22%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Touched%27%3Bthis.textContent%3D%27Touched%27%3B%22%3ETap%3C%2Fbutton%3E";
 const TEXT_PROBE_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EText%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3Bfont%3A28px%20sans-serif%3B%7Dinput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A260px%3Bheight%3A70px%3Bfont%3A28px%20sans-serif%3B%7Doutput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A180px%3Bfont%3A32px%20sans-serif%3B%7D%3C%2Fstyle%3E%3Cinput%20id%3Dq%20autofocus%20oninput%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.getElementById%28%27out%27%29.textContent%3Dthis.value%3B%22%3E%3Coutput%20id%3Dout%3Eempty%3C%2Foutput%3E";
 const TEXT_PROBE_VALUE: &str = "ely42";
 
@@ -55,6 +56,20 @@ fn manages_real_servo_webview_lifecycle() -> Result<(), Box<dyn Error>> {
     assert_rendered_frame_has_content(&host, "data:text/html clicked", 1)?;
     assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash);
 
+    let tab_id = TabId::new();
+    let url = UrlText::parse(TOUCH_PROBE_URL)?;
+    host.navigate(NavigationRequest { webview_id: webview_id.clone(), tab_id, url })?;
+    let snapshot = wait_for_rendered_webview(&mut host, &webview_id, None)?;
+    assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}");
+    assert_rendered_frame_has_content(&host, "data:text/html touch", 1)?;
+
+    let previous_frame_hash = host.last_rendered_frame()?.sample_hash();
+    host.touch_tap(TouchTapRequest { webview_id: webview_id.clone(), x: 160, y: 120 })?;
+    let snapshot = wait_for_rendered_webview(&mut host, &webview_id, Some(previous_frame_hash))?;
+    assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}");
+    assert_rendered_frame_has_content(&host, "data:text/html touched", 1)?;
+    assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash);
+
     let tab_id = TabId::new();
     let url = UrlText::parse(TEXT_PROBE_URL)?;
     host.navigate(NavigationRequest { webview_id: webview_id.clone(), tab_id, url })?;