diff --git a/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs b/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs index 8f3e49b..5e5da82 100644 --- a/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs +++ b/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs @@ -9,7 +9,8 @@ use std::{ use ely_domain::{ProfileId, TabId, UrlText}; use ely_servo_host::{ KeyboardTextRequest, MouseClickRequest, NavigationRequest, RenderedFrame, ScrollRequest, - ServoHost, ServoHostError, ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState, + ServoHost, ServoHostError, ServoSurfaceSize, SoftwareServoHost, TouchTapRequest, + WebViewSnapshot, WebViewState, }; use serde::Serialize; use thiserror::Error; @@ -37,6 +38,7 @@ struct SnapshotArgs { scroll_x: i32, scroll_y: i32, click_point: Option, + touch_point: Option, typed_text: Option, } @@ -77,6 +79,9 @@ enum SidecarError { #[error("--click-x and --click-y must be provided together")] IncompleteClickPoint, + #[error("--touch-x and --touch-y must be provided together")] + IncompleteTouchPoint, + #[error("rgba output path is empty")] EmptyRgbaOutputPath, @@ -119,6 +124,8 @@ fn parse_snapshot_args( let mut scroll_y = 0; let mut click_x = None; let mut click_y = None; + let mut touch_x = None; + let mut touch_y = None; let mut typed_text = None; while let Some(name) = args.next() { @@ -153,6 +160,18 @@ fn parse_snapshot_args( next_argument(&mut args, "--click-y")?, )?) } + "--touch-x" => { + touch_x = Some(parse_click_coordinate( + "--touch-x", + next_argument(&mut args, "--touch-x")?, + )?) + } + "--touch-y" => { + touch_y = Some(parse_click_coordinate( + "--touch-y", + next_argument(&mut args, "--touch-y")?, + )?) + } "--type-text" => typed_text = Some(next_argument(&mut args, "--type-text")?), _ => return Err(SidecarError::UnknownArgument { value: name }), } @@ -163,6 +182,11 @@ fn parse_snapshot_args( (None, None) => None, _ => return Err(SidecarError::IncompleteClickPoint), }; + let touch_point = match (touch_x, touch_y) { + (Some(x), Some(y)) => Some(ClickPoint { x, y }), + (None, None) => None, + _ => return Err(SidecarError::IncompleteTouchPoint), + }; Ok(SnapshotArgs { url: url.ok_or(SidecarError::MissingRequiredArgument { name: "--url" })?, @@ -172,6 +196,7 @@ fn parse_snapshot_args( scroll_x, scroll_y, click_point, + touch_point, typed_text, }) } @@ -229,6 +254,8 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> { apply_scroll_if_requested(&mut host, &webview_id, &args, snapshot)?; let (snapshot, click_changed_frame) = apply_click_if_requested(&mut host, &webview_id, &args, snapshot)?; + let (snapshot, touch_changed_frame) = + apply_touch_if_requested(&mut host, &webview_id, &args, snapshot)?; let (snapshot, text_changed_frame) = apply_text_if_requested(&mut host, &webview_id, &args, snapshot)?; let frame = host.last_rendered_frame()?; @@ -242,6 +269,7 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> { &frame, scroll_changed_frame, click_changed_frame, + touch_changed_frame, text_changed_frame, ), )?; @@ -286,6 +314,25 @@ fn apply_click_if_requested( wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash) } +fn apply_touch_if_requested( + host: &mut SoftwareServoHost, + webview_id: &ely_domain::WebViewId, + args: &SnapshotArgs, + snapshot: WebViewSnapshot, +) -> Result<(WebViewSnapshot, bool), SidecarError> { + let Some(touch_point) = args.touch_point else { + return Ok((snapshot, false)); + }; + + let previous_frame_hash = host.last_rendered_frame()?.sample_hash(); + host.touch_tap(TouchTapRequest { + webview_id: webview_id.clone(), + x: touch_point.x, + y: touch_point.y, + })?; + wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash) +} + fn apply_text_if_requested( host: &mut SoftwareServoHost, webview_id: &ely_domain::WebViewId, @@ -390,6 +437,9 @@ struct SnapshotReport { click_x: Option, click_y: Option, click_changed_frame: bool, + touch_x: Option, + touch_y: Option, + touch_changed_frame: bool, typed_text_byte_count: usize, text_changed_frame: bool, } @@ -401,6 +451,7 @@ impl SnapshotReport { frame: &RenderedFrame, scroll_changed_frame: bool, click_changed_frame: bool, + touch_changed_frame: bool, text_changed_frame: bool, ) -> Self { Self { @@ -422,6 +473,9 @@ impl SnapshotReport { click_x: args.click_point.map(|point| point.x), click_y: args.click_point.map(|point| point.y), click_changed_frame, + touch_x: args.touch_point.map(|point| point.x), + touch_y: args.touch_point.map(|point| point.y), + touch_changed_frame, typed_text_byte_count: args.typed_text.as_ref().map_or(0, String::len), text_changed_frame, } diff --git a/crates/ely_servo_host/src/host.rs b/crates/ely_servo_host/src/host.rs index ac14290..89174cf 100644 --- a/crates/ely_servo_host/src/host.rs +++ b/crates/ely_servo_host/src/host.rs @@ -227,6 +227,13 @@ pub struct MouseClickRequest { pub y: u32, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TouchTapRequest { + pub webview_id: WebViewId, + pub x: u32, + pub y: u32, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct KeyboardTextRequest { pub webview_id: WebViewId, @@ -261,6 +268,8 @@ pub trait ServoHost { fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError>; + fn touch_tap(&mut self, request: TouchTapRequest) -> Result<(), ServoHostError>; + fn type_text(&mut self, request: KeyboardTextRequest) -> Result<(), ServoHostError>; fn set_permission( diff --git a/crates/ely_servo_host/src/lib.rs b/crates/ely_servo_host/src/lib.rs index 182383e..e418e96 100644 --- a/crates/ely_servo_host/src/lib.rs +++ b/crates/ely_servo_host/src/lib.rs @@ -9,7 +9,7 @@ pub use error::ServoHostError; pub use host::{ KeyboardTextRequest, MouseClickRequest, NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, RenderedFrameSummary, ScrollRequest, ServoHost, - WebViewSnapshot, WebViewState, + TouchTapRequest, WebViewSnapshot, WebViewState, }; #[cfg(feature = "servo-engine")] pub use runtime::{ServoSurfaceSize, SoftwareServoHost}; diff --git a/crates/ely_servo_host/src/runtime.rs b/crates/ely_servo_host/src/runtime.rs index 9f81590..e392b0a 100644 --- a/crates/ely_servo_host/src/runtime.rs +++ b/crates/ely_servo_host/src/runtime.rs @@ -14,14 +14,15 @@ use servo::{ DeviceIntPoint, DeviceIntRect, DeviceIntSize, DevicePoint, DeviceVector2D, EventLoopWaker, InputEvent, Key, KeyState, KeyboardEvent, LoadStatus, Location, Modifiers, MouseButton, MouseButtonAction, MouseButtonEvent, MouseMoveEvent, RenderingContext, Scroll, Servo, - ServoBuilder, WebView, WebViewBuilder, WebViewDelegate, WebViewPoint, WebViewVector, + ServoBuilder, TouchEvent, TouchEventType, TouchId, WebView, WebViewBuilder, WebViewDelegate, + WebViewPoint, WebViewVector, }; use url::Url; use crate::{ KeyboardTextRequest, MouseClickRequest, NavigationRequest, PermissionDecision, - PermissionRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, WebViewSnapshot, - WebViewState, keyboard::keyboard_code_for_character, + PermissionRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, TouchTapRequest, + WebViewSnapshot, WebViewState, keyboard::keyboard_code_for_character, }; static SERVO_RUNTIME_STARTED: AtomicBool = AtomicBool::new(false); @@ -186,6 +187,22 @@ impl ServoHost for SoftwareServoHost { Ok(()) } + fn touch_tap(&mut self, request: TouchTapRequest) -> Result<(), ServoHostError> { + let webview = self + .webviews + .get(&request.webview_id) + .ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?; + + let point = WebViewPoint::Device(DevicePoint::new(request.x as f32, request.y as f32)); + let touch_id = TouchId(1); + for event_type in [TouchEventType::Down, TouchEventType::Up] { + webview.webview.notify_input_event(InputEvent::Touch(TouchEvent::new( + event_type, touch_id, point, + ))); + } + Ok(()) + } + fn type_text(&mut self, request: KeyboardTextRequest) -> Result<(), ServoHostError> { let webview = self .webviews diff --git a/crates/ely_servo_host/tests/sidecar.rs b/crates/ely_servo_host/tests/sidecar.rs index 0e30b8f..c9d4e21 100644 --- a/crates/ely_servo_host/tests/sidecar.rs +++ b/crates/ely_servo_host/tests/sidecar.rs @@ -27,6 +27,9 @@ const SERVO_SCROLL_OFFSET: ScrollOffset = ScrollOffset { x: 0, y: 480 }; const SERVO_CLICK_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E"; const SERVO_CLICK_SIZE: FrameSize = FrameSize { width: 640, height: 480 }; const SERVO_CLICK_POINT: ClickPoint = ClickPoint { x: 160, y: 120 }; +const SERVO_TOUCH_URL: &str = "data:text/html,%3C%21doctype%20html%3E%3Ctitle%3ETouch%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3Btouch-action%3Amanipulation%3B%7D%3C%2Fstyle%3E%3Cbutton%20ontouchstart%3D%22document.body.dataset.touch%3D%27start%27%3B%22%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Touched%27%3Bthis.textContent%3D%27Touched%27%3B%22%3ETap%3C%2Fbutton%3E"; +const SERVO_TOUCH_SIZE: FrameSize = FrameSize { width: 640, height: 480 }; +const SERVO_TOUCH_POINT: ClickPoint = ClickPoint { x: 160, y: 120 }; const SERVO_TEXT_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EText%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3Bfont%3A28px%20sans-serif%3B%7Dinput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A260px%3Bheight%3A70px%3Bfont%3A28px%20sans-serif%3B%7Doutput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A180px%3Bfont%3A32px%20sans-serif%3B%7D%3C%2Fstyle%3E%3Cinput%20id%3Dq%20autofocus%20oninput%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.getElementById%28%27out%27%29.textContent%3Dthis.value%3B%22%3E%3Coutput%20id%3Dout%3Eempty%3C%2Foutput%3E"; const SERVO_TEXT_SIZE: FrameSize = FrameSize { width: 640, height: 480 }; const SERVO_TEXT_POINT: ClickPoint = ClickPoint { x: 160, y: 120 }; @@ -116,6 +119,22 @@ fn sidecar_clicks_page_with_servo_mouse_input() -> Result<(), Box> { Ok(()) } +#[test] +fn sidecar_touches_page_with_servo_touch_input() -> Result<(), Box> { + let initial_report = snapshot_touch_probe(None)?; + let touched_report = snapshot_touch_probe(Some(SERVO_TOUCH_POINT))?; + + assert_eq!(report_field_as_u64(&touched_report, "touch_x")?, SERVO_TOUCH_POINT.x); + assert_eq!(report_field_as_u64(&touched_report, "touch_y")?, SERVO_TOUCH_POINT.y); + assert!(report_field_as_bool(&touched_report, "touch_changed_frame")?); + assert_ne!( + report_field_as_u64(&initial_report, "sample_hash")?, + report_field_as_u64(&touched_report, "sample_hash")? + ); + + Ok(()) +} + #[test] fn sidecar_types_text_with_servo_keyboard_input() -> Result<(), Box> { let initial_report = snapshot_text_probe(None)?; @@ -157,7 +176,8 @@ fn snapshot_prd_site( std::fs::remove_file(&output_path)?; } - let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset, None, None)?; + let output = + run_sidecar_snapshot(case.url, &output_path, size, scroll_offset, None, None, None)?; assert!( output.status.success(), @@ -215,6 +235,7 @@ fn snapshot_click_probe( ScrollOffset::ZERO, click_point, None, + None, )?; assert!( @@ -238,6 +259,51 @@ fn snapshot_click_probe( Ok(report) } +fn snapshot_touch_probe( + touch_point: Option, +) -> Result> { + let output_path = std::env::temp_dir().join(format!( + "ely-servo-sidecar-{}-touch-{}x{}.rgba", + std::process::id(), + SERVO_TOUCH_SIZE.width, + SERVO_TOUCH_SIZE.height + )); + + if output_path.exists() { + std::fs::remove_file(&output_path)?; + } + + let output = run_sidecar_snapshot( + SERVO_TOUCH_URL, + &output_path, + SERVO_TOUCH_SIZE, + ScrollOffset::ZERO, + None, + touch_point, + None, + )?; + + assert!( + output.status.success(), + "touch probe\nstatus: {:?}\nstdout: {}\nstderr: {}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let report: serde_json::Value = serde_json::from_slice(&output.stdout)?; + assert_eq!(report_field_as_u64(&report, "width")?, SERVO_TOUCH_SIZE.width); + assert_eq!(report_field_as_u64(&report, "height")?, SERVO_TOUCH_SIZE.height); + assert!(report_field_as_u64(&report, "content_pixel_count")? > 0); + assert_eq!( + std::fs::metadata(&output_path)?.len(), + SERVO_TOUCH_SIZE.width * SERVO_TOUCH_SIZE.height * 4 + ); + + std::fs::remove_file(&output_path)?; + Ok(report) +} + fn snapshot_text_probe(typed_text: Option<&str>) -> Result> { let output_path = std::env::temp_dir().join(format!( "ely-servo-sidecar-{}-text-{}x{}.rgba", @@ -257,6 +323,7 @@ fn snapshot_text_probe(typed_text: Option<&str>) -> Result, + touch_point: Option, typed_text: Option<&str>, ) -> Result> { let mut command = Command::new(env!("CARGO_BIN_EXE_ely_servo_sidecar")); @@ -310,6 +378,10 @@ fn run_sidecar_snapshot( command.arg("--click-x").arg(click_point.x.to_string()); command.arg("--click-y").arg(click_point.y.to_string()); } + if let Some(touch_point) = touch_point { + command.arg("--touch-x").arg(touch_point.x.to_string()); + command.arg("--touch-y").arg(touch_point.y.to_string()); + } if let Some(typed_text) = typed_text { command.arg("--type-text").arg(typed_text); } diff --git a/crates/ely_servo_host/tests/software_host.rs b/crates/ely_servo_host/tests/software_host.rs index 53c7b72..2c812d5 100644 --- a/crates/ely_servo_host/tests/software_host.rs +++ b/crates/ely_servo_host/tests/software_host.rs @@ -5,7 +5,7 @@ use std::{error::Error, thread, time::Duration}; use ely_domain::{ProfileId, TabId, UrlText}; use ely_servo_host::{ KeyboardTextRequest, MouseClickRequest, NavigationRequest, ScrollRequest, ServoHost, - ServoHostError, ServoSurfaceSize, SoftwareServoHost, WebViewState, + ServoHostError, ServoSurfaceSize, SoftwareServoHost, TouchTapRequest, WebViewState, }; const MINIMUM_CONTENT_PIXELS: u64 = 1_000; @@ -14,6 +14,7 @@ const PRD_SITE_COMPATIBILITY_CASES: &[PrdSiteCompatibilityCase] = &[ PrdSiteCompatibilityCase { url: "https://servo.org", title_fragment: "Servo" }, ]; const CLICK_PROBE_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E"; +const TOUCH_PROBE_URL: &str = "data:text/html,%3C%21doctype%20html%3E%3Ctitle%3ETouch%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3Btouch-action%3Amanipulation%3B%7D%3C%2Fstyle%3E%3Cbutton%20ontouchstart%3D%22document.body.dataset.touch%3D%27start%27%3B%22%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Touched%27%3Bthis.textContent%3D%27Touched%27%3B%22%3ETap%3C%2Fbutton%3E"; const TEXT_PROBE_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EText%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3Bfont%3A28px%20sans-serif%3B%7Dinput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A260px%3Bheight%3A70px%3Bfont%3A28px%20sans-serif%3B%7Doutput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A180px%3Bfont%3A32px%20sans-serif%3B%7D%3C%2Fstyle%3E%3Cinput%20id%3Dq%20autofocus%20oninput%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.getElementById%28%27out%27%29.textContent%3Dthis.value%3B%22%3E%3Coutput%20id%3Dout%3Eempty%3C%2Foutput%3E"; const TEXT_PROBE_VALUE: &str = "ely42"; @@ -55,6 +56,20 @@ fn manages_real_servo_webview_lifecycle() -> Result<(), Box> { assert_rendered_frame_has_content(&host, "data:text/html clicked", 1)?; assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash); + let tab_id = TabId::new(); + let url = UrlText::parse(TOUCH_PROBE_URL)?; + host.navigate(NavigationRequest { webview_id: webview_id.clone(), tab_id, url })?; + let snapshot = wait_for_rendered_webview(&mut host, &webview_id, None)?; + assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}"); + assert_rendered_frame_has_content(&host, "data:text/html touch", 1)?; + + let previous_frame_hash = host.last_rendered_frame()?.sample_hash(); + host.touch_tap(TouchTapRequest { webview_id: webview_id.clone(), x: 160, y: 120 })?; + let snapshot = wait_for_rendered_webview(&mut host, &webview_id, Some(previous_frame_hash))?; + assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}"); + assert_rendered_frame_has_content(&host, "data:text/html touched", 1)?; + assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash); + let tab_id = TabId::new(); let url = UrlText::parse(TEXT_PROBE_URL)?; host.navigate(NavigationRequest { webview_id: webview_id.clone(), tab_id, url })?;