diff --git a/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs b/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs index 1ec829e..8f3e49b 100644 --- a/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs +++ b/crates/ely_servo_host/src/bin/ely_servo_sidecar.rs @@ -8,8 +8,8 @@ use std::{ use ely_domain::{ProfileId, TabId, UrlText}; use ely_servo_host::{ - MouseClickRequest, NavigationRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, - ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState, + KeyboardTextRequest, MouseClickRequest, NavigationRequest, RenderedFrame, ScrollRequest, + ServoHost, ServoHostError, ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState, }; use serde::Serialize; use thiserror::Error; @@ -37,6 +37,7 @@ struct SnapshotArgs { scroll_x: i32, scroll_y: i32, click_point: Option, + typed_text: Option, } #[derive(Clone, Copy)] @@ -118,6 +119,7 @@ fn parse_snapshot_args( let mut scroll_y = 0; let mut click_x = None; let mut click_y = None; + let mut typed_text = None; while let Some(name) = args.next() { match name.as_str() { @@ -151,6 +153,7 @@ fn parse_snapshot_args( next_argument(&mut args, "--click-y")?, )?) } + "--type-text" => typed_text = Some(next_argument(&mut args, "--type-text")?), _ => return Err(SidecarError::UnknownArgument { value: name }), } } @@ -169,6 +172,7 @@ fn parse_snapshot_args( scroll_x, scroll_y, click_point, + typed_text, }) } @@ -225,12 +229,21 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> { apply_scroll_if_requested(&mut host, &webview_id, &args, snapshot)?; let (snapshot, click_changed_frame) = apply_click_if_requested(&mut host, &webview_id, &args, snapshot)?; + let (snapshot, text_changed_frame) = + apply_text_if_requested(&mut host, &webview_id, &args, snapshot)?; let frame = host.last_rendered_frame()?; std::fs::write(&args.rgba_out, frame.rgba_bytes())?; serde_json::to_writer( std::io::stdout().lock(), - &SnapshotReport::new(&args, &snapshot, &frame, scroll_changed_frame, click_changed_frame), + &SnapshotReport::new( + &args, + &snapshot, + &frame, + scroll_changed_frame, + click_changed_frame, + text_changed_frame, + ), )?; Ok(()) } @@ -273,6 +286,24 @@ fn apply_click_if_requested( wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash) } +fn apply_text_if_requested( + host: &mut SoftwareServoHost, + webview_id: &ely_domain::WebViewId, + args: &SnapshotArgs, + snapshot: WebViewSnapshot, +) -> Result<(WebViewSnapshot, bool), SidecarError> { + let Some(typed_text) = args.typed_text.as_ref() else { + return Ok((snapshot, false)); + }; + + let previous_frame_hash = host.last_rendered_frame()?.sample_hash(); + host.type_text(KeyboardTextRequest { + webview_id: webview_id.clone(), + text: typed_text.clone(), + })?; + wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash) +} + fn wait_for_frame( host: &mut SoftwareServoHost, webview_id: &ely_domain::WebViewId, @@ -359,6 +390,8 @@ struct SnapshotReport { click_x: Option, click_y: Option, click_changed_frame: bool, + typed_text_byte_count: usize, + text_changed_frame: bool, } impl SnapshotReport { @@ -368,6 +401,7 @@ impl SnapshotReport { frame: &RenderedFrame, scroll_changed_frame: bool, click_changed_frame: bool, + text_changed_frame: bool, ) -> Self { Self { requested_url: args.url.as_str().to_string(), @@ -388,6 +422,8 @@ impl SnapshotReport { click_x: args.click_point.map(|point| point.x), click_y: args.click_point.map(|point| point.y), click_changed_frame, + typed_text_byte_count: args.typed_text.as_ref().map_or(0, String::len), + text_changed_frame, } } } diff --git a/crates/ely_servo_host/src/host.rs b/crates/ely_servo_host/src/host.rs index 011fc88..ac14290 100644 --- a/crates/ely_servo_host/src/host.rs +++ b/crates/ely_servo_host/src/host.rs @@ -227,6 +227,12 @@ pub struct MouseClickRequest { pub y: u32, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct KeyboardTextRequest { + pub webview_id: WebViewId, + pub text: String, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct PermissionRequest { pub webview_id: WebViewId, @@ -255,6 +261,8 @@ pub trait ServoHost { fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError>; + fn type_text(&mut self, request: KeyboardTextRequest) -> Result<(), ServoHostError>; + fn set_permission( &mut self, request: PermissionRequest, diff --git a/crates/ely_servo_host/src/keyboard.rs b/crates/ely_servo_host/src/keyboard.rs new file mode 100644 index 0000000..627f744 --- /dev/null +++ b/crates/ely_servo_host/src/keyboard.rs @@ -0,0 +1,55 @@ +use servo::Code; + +pub(crate) fn keyboard_code_for_character(character: char) -> Code { + match character { + 'a' | 'A' => Code::KeyA, + 'b' | 'B' => Code::KeyB, + 'c' | 'C' => Code::KeyC, + 'd' | 'D' => Code::KeyD, + 'e' | 'E' => Code::KeyE, + 'f' | 'F' => Code::KeyF, + 'g' | 'G' => Code::KeyG, + 'h' | 'H' => Code::KeyH, + 'i' | 'I' => Code::KeyI, + 'j' | 'J' => Code::KeyJ, + 'k' | 'K' => Code::KeyK, + 'l' | 'L' => Code::KeyL, + 'm' | 'M' => Code::KeyM, + 'n' | 'N' => Code::KeyN, + 'o' | 'O' => Code::KeyO, + 'p' | 'P' => Code::KeyP, + 'q' | 'Q' => Code::KeyQ, + 'r' | 'R' => Code::KeyR, + 's' | 'S' => Code::KeyS, + 't' | 'T' => Code::KeyT, + 'u' | 'U' => Code::KeyU, + 'v' | 'V' => Code::KeyV, + 'w' | 'W' => Code::KeyW, + 'x' | 'X' => Code::KeyX, + 'y' | 'Y' => Code::KeyY, + 'z' | 'Z' => Code::KeyZ, + '0' => Code::Digit0, + '1' => Code::Digit1, + '2' => Code::Digit2, + '3' => Code::Digit3, + '4' => Code::Digit4, + '5' => Code::Digit5, + '6' => Code::Digit6, + '7' => Code::Digit7, + '8' => Code::Digit8, + '9' => Code::Digit9, + ' ' => Code::Space, + '-' | '_' => Code::Minus, + '=' | '+' => Code::Equal, + ',' | '<' => Code::Comma, + '.' | '>' => Code::Period, + '/' | '?' => Code::Slash, + ';' | ':' => Code::Semicolon, + '\'' | '"' => Code::Quote, + '[' | '{' => Code::BracketLeft, + ']' | '}' => Code::BracketRight, + '\\' | '|' => Code::Backslash, + '`' | '~' => Code::Backquote, + _ => Code::Unidentified, + } +} diff --git a/crates/ely_servo_host/src/lib.rs b/crates/ely_servo_host/src/lib.rs index 54b75ce..182383e 100644 --- a/crates/ely_servo_host/src/lib.rs +++ b/crates/ely_servo_host/src/lib.rs @@ -1,12 +1,15 @@ mod error; mod host; #[cfg(feature = "servo-engine")] +mod keyboard; +#[cfg(feature = "servo-engine")] mod runtime; pub use error::ServoHostError; pub use host::{ - MouseClickRequest, NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, - RenderedFrameSummary, ScrollRequest, ServoHost, WebViewSnapshot, WebViewState, + KeyboardTextRequest, MouseClickRequest, NavigationRequest, PermissionDecision, + PermissionRequest, RenderedFrame, RenderedFrameSummary, ScrollRequest, ServoHost, + WebViewSnapshot, WebViewState, }; #[cfg(feature = "servo-engine")] pub use runtime::{ServoSurfaceSize, SoftwareServoHost}; diff --git a/crates/ely_servo_host/src/runtime.rs b/crates/ely_servo_host/src/runtime.rs index e884c5f..9f81590 100644 --- a/crates/ely_servo_host/src/runtime.rs +++ b/crates/ely_servo_host/src/runtime.rs @@ -12,15 +12,16 @@ use dpi::PhysicalSize; use ely_domain::{ProfileId, TabId, WebViewId}; use servo::{ DeviceIntPoint, DeviceIntRect, DeviceIntSize, DevicePoint, DeviceVector2D, EventLoopWaker, - InputEvent, LoadStatus, MouseButton, MouseButtonAction, MouseButtonEvent, MouseMoveEvent, - RenderingContext, Scroll, Servo, ServoBuilder, WebView, WebViewBuilder, WebViewDelegate, - WebViewPoint, WebViewVector, + InputEvent, Key, KeyState, KeyboardEvent, LoadStatus, Location, Modifiers, MouseButton, + MouseButtonAction, MouseButtonEvent, MouseMoveEvent, RenderingContext, Scroll, Servo, + ServoBuilder, WebView, WebViewBuilder, WebViewDelegate, WebViewPoint, WebViewVector, }; use url::Url; use crate::{ - MouseClickRequest, NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame, - ScrollRequest, ServoHost, ServoHostError, WebViewSnapshot, WebViewState, + KeyboardTextRequest, MouseClickRequest, NavigationRequest, PermissionDecision, + PermissionRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, WebViewSnapshot, + WebViewState, keyboard::keyboard_code_for_character, }; static SERVO_RUNTIME_STARTED: AtomicBool = AtomicBool::new(false); @@ -185,6 +186,41 @@ impl ServoHost for SoftwareServoHost { Ok(()) } + fn type_text(&mut self, request: KeyboardTextRequest) -> Result<(), ServoHostError> { + let webview = self + .webviews + .get(&request.webview_id) + .ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?; + + for character in request.text.chars() { + let key = Key::Character(character.to_string()); + let code = keyboard_code_for_character(character); + webview.webview.notify_input_event(InputEvent::Keyboard( + KeyboardEvent::new_without_event( + KeyState::Down, + key.clone(), + code, + Location::Standard, + Modifiers::empty(), + false, + false, + ), + )); + webview.webview.notify_input_event(InputEvent::Keyboard( + KeyboardEvent::new_without_event( + KeyState::Up, + key, + code, + Location::Standard, + Modifiers::empty(), + false, + false, + ), + )); + } + Ok(()) + } + fn set_permission( &mut self, request: PermissionRequest, diff --git a/crates/ely_servo_host/tests/sidecar.rs b/crates/ely_servo_host/tests/sidecar.rs index 36e9444..0e30b8f 100644 --- a/crates/ely_servo_host/tests/sidecar.rs +++ b/crates/ely_servo_host/tests/sidecar.rs @@ -27,6 +27,10 @@ const SERVO_SCROLL_OFFSET: ScrollOffset = ScrollOffset { x: 0, y: 480 }; const SERVO_CLICK_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E"; const SERVO_CLICK_SIZE: FrameSize = FrameSize { width: 640, height: 480 }; const SERVO_CLICK_POINT: ClickPoint = ClickPoint { x: 160, y: 120 }; +const SERVO_TEXT_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EText%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3Bfont%3A28px%20sans-serif%3B%7Dinput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A260px%3Bheight%3A70px%3Bfont%3A28px%20sans-serif%3B%7Doutput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A180px%3Bfont%3A32px%20sans-serif%3B%7D%3C%2Fstyle%3E%3Cinput%20id%3Dq%20autofocus%20oninput%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.getElementById%28%27out%27%29.textContent%3Dthis.value%3B%22%3E%3Coutput%20id%3Dout%3Eempty%3C%2Foutput%3E"; +const SERVO_TEXT_SIZE: FrameSize = FrameSize { width: 640, height: 480 }; +const SERVO_TEXT_POINT: ClickPoint = ClickPoint { x: 160, y: 120 }; +const SERVO_TEXT_VALUE: &str = "ely42"; struct PrdSiteCompatibilityCase { url: &'static str, @@ -112,6 +116,24 @@ fn sidecar_clicks_page_with_servo_mouse_input() -> Result<(), Box> { Ok(()) } +#[test] +fn sidecar_types_text_with_servo_keyboard_input() -> Result<(), Box> { + let initial_report = snapshot_text_probe(None)?; + let typed_report = snapshot_text_probe(Some(SERVO_TEXT_VALUE))?; + + assert_eq!( + report_field_as_u64(&typed_report, "typed_text_byte_count")?, + SERVO_TEXT_VALUE.len() as u64 + ); + assert!(report_field_as_bool(&typed_report, "text_changed_frame")?); + assert_ne!( + report_field_as_u64(&initial_report, "sample_hash")?, + report_field_as_u64(&typed_report, "sample_hash")? + ); + + Ok(()) +} + fn snapshot_prd_site( case: &PrdSiteCompatibilityCase, size: FrameSize, @@ -135,7 +157,7 @@ fn snapshot_prd_site( std::fs::remove_file(&output_path)?; } - let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset, None)?; + let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset, None, None)?; assert!( output.status.success(), @@ -192,6 +214,7 @@ fn snapshot_click_probe( SERVO_CLICK_SIZE, ScrollOffset::ZERO, click_point, + None, )?; assert!( @@ -215,12 +238,56 @@ fn snapshot_click_probe( Ok(report) } +fn snapshot_text_probe(typed_text: Option<&str>) -> Result> { + let output_path = std::env::temp_dir().join(format!( + "ely-servo-sidecar-{}-text-{}x{}.rgba", + std::process::id(), + SERVO_TEXT_SIZE.width, + SERVO_TEXT_SIZE.height + )); + + if output_path.exists() { + std::fs::remove_file(&output_path)?; + } + + let click_point = typed_text.map(|_| SERVO_TEXT_POINT); + let output = run_sidecar_snapshot( + SERVO_TEXT_URL, + &output_path, + SERVO_TEXT_SIZE, + ScrollOffset::ZERO, + click_point, + typed_text, + )?; + + assert!( + output.status.success(), + "text probe\nstatus: {:?}\nstdout: {}\nstderr: {}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let report: serde_json::Value = serde_json::from_slice(&output.stdout)?; + assert_eq!(report_field_as_u64(&report, "width")?, SERVO_TEXT_SIZE.width); + assert_eq!(report_field_as_u64(&report, "height")?, SERVO_TEXT_SIZE.height); + assert!(report_field_as_u64(&report, "content_pixel_count")? > 0); + assert_eq!( + std::fs::metadata(&output_path)?.len(), + SERVO_TEXT_SIZE.width * SERVO_TEXT_SIZE.height * 4 + ); + + std::fs::remove_file(&output_path)?; + Ok(report) +} + fn run_sidecar_snapshot( site_url: &str, output_path: &std::path::Path, size: FrameSize, scroll_offset: ScrollOffset, click_point: Option, + typed_text: Option<&str>, ) -> Result> { let mut command = Command::new(env!("CARGO_BIN_EXE_ely_servo_sidecar")); command @@ -243,6 +310,9 @@ fn run_sidecar_snapshot( command.arg("--click-x").arg(click_point.x.to_string()); command.arg("--click-y").arg(click_point.y.to_string()); } + if let Some(typed_text) = typed_text { + command.arg("--type-text").arg(typed_text); + } let mut child = command.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; diff --git a/crates/ely_servo_host/tests/software_host.rs b/crates/ely_servo_host/tests/software_host.rs index 22e1150..53c7b72 100644 --- a/crates/ely_servo_host/tests/software_host.rs +++ b/crates/ely_servo_host/tests/software_host.rs @@ -4,8 +4,8 @@ use std::{error::Error, thread, time::Duration}; use ely_domain::{ProfileId, TabId, UrlText}; use ely_servo_host::{ - MouseClickRequest, NavigationRequest, ScrollRequest, ServoHost, ServoHostError, - ServoSurfaceSize, SoftwareServoHost, WebViewState, + KeyboardTextRequest, MouseClickRequest, NavigationRequest, ScrollRequest, ServoHost, + ServoHostError, ServoSurfaceSize, SoftwareServoHost, WebViewState, }; const MINIMUM_CONTENT_PIXELS: u64 = 1_000; @@ -14,6 +14,8 @@ const PRD_SITE_COMPATIBILITY_CASES: &[PrdSiteCompatibilityCase] = &[ PrdSiteCompatibilityCase { url: "https://servo.org", title_fragment: "Servo" }, ]; const CLICK_PROBE_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E"; +const TEXT_PROBE_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EText%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3Bfont%3A28px%20sans-serif%3B%7Dinput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A260px%3Bheight%3A70px%3Bfont%3A28px%20sans-serif%3B%7Doutput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A180px%3Bfont%3A32px%20sans-serif%3B%7D%3C%2Fstyle%3E%3Cinput%20id%3Dq%20autofocus%20oninput%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.getElementById%28%27out%27%29.textContent%3Dthis.value%3B%22%3E%3Coutput%20id%3Dout%3Eempty%3C%2Foutput%3E"; +const TEXT_PROBE_VALUE: &str = "ely42"; struct PrdSiteCompatibilityCase { url: &'static str, @@ -53,6 +55,24 @@ fn manages_real_servo_webview_lifecycle() -> Result<(), Box> { assert_rendered_frame_has_content(&host, "data:text/html clicked", 1)?; assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash); + let tab_id = TabId::new(); + let url = UrlText::parse(TEXT_PROBE_URL)?; + host.navigate(NavigationRequest { webview_id: webview_id.clone(), tab_id, url })?; + let snapshot = wait_for_rendered_webview(&mut host, &webview_id, None)?; + assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}"); + assert_rendered_frame_has_content(&host, "data:text/html input", 1)?; + + let previous_frame_hash = host.last_rendered_frame()?.sample_hash(); + host.click(MouseClickRequest { webview_id: webview_id.clone(), x: 160, y: 120 })?; + host.type_text(KeyboardTextRequest { + webview_id: webview_id.clone(), + text: TEXT_PROBE_VALUE.to_string(), + })?; + let snapshot = wait_for_rendered_webview(&mut host, &webview_id, Some(previous_frame_hash))?; + assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}"); + assert_rendered_frame_has_content(&host, "data:text/html typed", 1)?; + assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash); + let mut previous_frame_hash = Some(host.last_rendered_frame()?.sample_hash()); for site in PRD_SITE_COMPATIBILITY_CASES { let tab_id = TabId::new();