ZacharyZhang-NY/ELY-Browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bridge Servo keyboard text input

Browse Source
This commit is contained in:
ZacharyZhang-NY
2026-05-08 14:36:48 -04:00
parent b786073826
commit 292824dc10
7 changed files with 241 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ely_servo_host/src/bin/ely_servo_sidecar.rs
+39 -3
View File
@@ -8,8 +8,8 @@ use std::{
use ely_domain::{ProfileId, TabId, UrlText};
use ely_servo_host::{
MouseClickRequest, NavigationRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError,
ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState,
KeyboardTextRequest, MouseClickRequest, NavigationRequest, RenderedFrame, ScrollRequest,
ServoHost, ServoHostError, ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState,
};
use serde::Serialize;
use thiserror::Error;
@@ -37,6 +37,7 @@ struct SnapshotArgs {
scroll_x: i32,
scroll_y: i32,
click_point: Option<ClickPoint>,
typed_text: Option<String>,
}
#[derive(Clone, Copy)]
@@ -118,6 +119,7 @@ fn parse_snapshot_args(
let mut scroll_y = 0;
let mut click_x = None;
let mut click_y = None;
let mut typed_text = None;
while let Some(name) = args.next() {
match name.as_str() {
@@ -151,6 +153,7 @@ fn parse_snapshot_args(
next_argument(&mut args, "--click-y")?,
)?)
}
"--type-text" => typed_text = Some(next_argument(&mut args, "--type-text")?),
_ => return Err(SidecarError::UnknownArgument { value: name }),
}
}
@@ -169,6 +172,7 @@ fn parse_snapshot_args(
scroll_x,
scroll_y,
click_point,
typed_text,
})
}
@@ -225,12 +229,21 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> {
apply_scroll_if_requested(&mut host, &webview_id, &args, snapshot)?;
let (snapshot, click_changed_frame) =
apply_click_if_requested(&mut host, &webview_id, &args, snapshot)?;
let (snapshot, text_changed_frame) =
apply_text_if_requested(&mut host, &webview_id, &args, snapshot)?;
let frame = host.last_rendered_frame()?;
std::fs::write(&args.rgba_out, frame.rgba_bytes())?;
serde_json::to_writer(
std::io::stdout().lock(),
&SnapshotReport::new(&args, &snapshot, &frame, scroll_changed_frame, click_changed_frame),
&SnapshotReport::new(
&args,
&snapshot,
&frame,
scroll_changed_frame,
click_changed_frame,
text_changed_frame,
),
)?;
Ok(())
}
@@ -273,6 +286,24 @@ fn apply_click_if_requested(
wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash)
}
fn apply_text_if_requested(
host: &mut SoftwareServoHost,
webview_id: &ely_domain::WebViewId,
args: &SnapshotArgs,
snapshot: WebViewSnapshot,
) -> Result<(WebViewSnapshot, bool), SidecarError> {
let Some(typed_text) = args.typed_text.as_ref() else {
return Ok((snapshot, false));
};
let previous_frame_hash = host.last_rendered_frame()?.sample_hash();
host.type_text(KeyboardTextRequest {
webview_id: webview_id.clone(),
text: typed_text.clone(),
})?;
wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash)
}
fn wait_for_frame(
host: &mut SoftwareServoHost,
webview_id: &ely_domain::WebViewId,
@@ -359,6 +390,8 @@ struct SnapshotReport {
click_x: Option<u32>,
click_y: Option<u32>,
click_changed_frame: bool,
typed_text_byte_count: usize,
text_changed_frame: bool,
}
impl SnapshotReport {
@@ -368,6 +401,7 @@ impl SnapshotReport {
frame: &RenderedFrame,
scroll_changed_frame: bool,
click_changed_frame: bool,
text_changed_frame: bool,
) -> Self {
Self {
requested_url: args.url.as_str().to_string(),
@@ -388,6 +422,8 @@ impl SnapshotReport {
click_x: args.click_point.map(|point| point.x),
click_y: args.click_point.map(|point| point.y),
click_changed_frame,
typed_text_byte_count: args.typed_text.as_ref().map_or(0, String::len),
text_changed_frame,
}
}
}
crates/ely_servo_host/src/host.rs
+8
View File
@@ -227,6 +227,12 @@ pub struct MouseClickRequest {
pub y: u32,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct KeyboardTextRequest {
pub webview_id: WebViewId,
pub text: String,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PermissionRequest {
pub webview_id: WebViewId,
@@ -255,6 +261,8 @@ pub trait ServoHost {
fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError>;
fn type_text(&mut self, request: KeyboardTextRequest) -> Result<(), ServoHostError>;
fn set_permission(
&mut self,
request: PermissionRequest,
crates/ely_servo_host/src/keyboard.rs
+55
View File
@@ -0,0 +1,55 @@
use servo::Code;
pub(crate) fn keyboard_code_for_character(character: char) -> Code {
match character {
'a' | 'A' => Code::KeyA,
'b' | 'B' => Code::KeyB,
'c' | 'C' => Code::KeyC,
'd' | 'D' => Code::KeyD,
'e' | 'E' => Code::KeyE,
'f' | 'F' => Code::KeyF,
'g' | 'G' => Code::KeyG,
'h' | 'H' => Code::KeyH,
'i' | 'I' => Code::KeyI,
'j' | 'J' => Code::KeyJ,
'k' | 'K' => Code::KeyK,
'l' | 'L' => Code::KeyL,
'm' | 'M' => Code::KeyM,
'n' | 'N' => Code::KeyN,
'o' | 'O' => Code::KeyO,
'p' | 'P' => Code::KeyP,
'q' | 'Q' => Code::KeyQ,
'r' | 'R' => Code::KeyR,
's' | 'S' => Code::KeyS,
't' | 'T' => Code::KeyT,
'u' | 'U' => Code::KeyU,
'v' | 'V' => Code::KeyV,
'w' | 'W' => Code::KeyW,
'x' | 'X' => Code::KeyX,
'y' | 'Y' => Code::KeyY,
'z' | 'Z' => Code::KeyZ,
'0' => Code::Digit0,
'1' => Code::Digit1,
'2' => Code::Digit2,
'3' => Code::Digit3,
'4' => Code::Digit4,
'5' => Code::Digit5,
'6' => Code::Digit6,
'7' => Code::Digit7,
'8' => Code::Digit8,
'9' => Code::Digit9,
' ' => Code::Space,
'-' | '_' => Code::Minus,
'=' | '+' => Code::Equal,
',' | '<' => Code::Comma,
'.' | '>' => Code::Period,
'/' | '?' => Code::Slash,
';' | ':' => Code::Semicolon,
'\'' | '"' => Code::Quote,
'[' | '{' => Code::BracketLeft,
']' | '}' => Code::BracketRight,
'\\' | '|' => Code::Backslash,
'`' | '~' => Code::Backquote,
_ => Code::Unidentified,
}
}
crates/ely_servo_host/src/lib.rs
+5 -2
View File
@@ -1,12 +1,15 @@
mod error;
mod host;
#[cfg(feature = "servo-engine")]
mod keyboard;
#[cfg(feature = "servo-engine")]
mod runtime;
pub use error::ServoHostError;
pub use host::{
MouseClickRequest, NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame,
RenderedFrameSummary, ScrollRequest, ServoHost, WebViewSnapshot, WebViewState,
KeyboardTextRequest, MouseClickRequest, NavigationRequest, PermissionDecision,
PermissionRequest, RenderedFrame, RenderedFrameSummary, ScrollRequest, ServoHost,
WebViewSnapshot, WebViewState,
};
#[cfg(feature = "servo-engine")]
pub use runtime::{ServoSurfaceSize, SoftwareServoHost};
crates/ely_servo_host/src/runtime.rs
+41 -5
View File
@@ -12,15 +12,16 @@ use dpi::PhysicalSize;
use ely_domain::{ProfileId, TabId, WebViewId};
use servo::{
DeviceIntPoint, DeviceIntRect, DeviceIntSize, DevicePoint, DeviceVector2D, EventLoopWaker,
InputEvent, LoadStatus, MouseButton, MouseButtonAction, MouseButtonEvent, MouseMoveEvent,
RenderingContext, Scroll, Servo, ServoBuilder, WebView, WebViewBuilder, WebViewDelegate,
WebViewPoint, WebViewVector,
InputEvent, Key, KeyState, KeyboardEvent, LoadStatus, Location, Modifiers, MouseButton,
MouseButtonAction, MouseButtonEvent, MouseMoveEvent, RenderingContext, Scroll, Servo,
ServoBuilder, WebView, WebViewBuilder, WebViewDelegate, WebViewPoint, WebViewVector,
};
use url::Url;
use crate::{
MouseClickRequest, NavigationRequest, PermissionDecision, PermissionRequest, RenderedFrame,
ScrollRequest, ServoHost, ServoHostError, WebViewSnapshot, WebViewState,
KeyboardTextRequest, MouseClickRequest, NavigationRequest, PermissionDecision,
PermissionRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, WebViewSnapshot,
WebViewState, keyboard::keyboard_code_for_character,
};
static SERVO_RUNTIME_STARTED: AtomicBool = AtomicBool::new(false);
@@ -185,6 +186,41 @@ impl ServoHost for SoftwareServoHost {
Ok(())
}
fn type_text(&mut self, request: KeyboardTextRequest) -> Result<(), ServoHostError> {
let webview = self
.webviews
.get(&request.webview_id)
.ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?;
for character in request.text.chars() {
let key = Key::Character(character.to_string());
let code = keyboard_code_for_character(character);
webview.webview.notify_input_event(InputEvent::Keyboard(
KeyboardEvent::new_without_event(
KeyState::Down,
key.clone(),
code,
Location::Standard,
Modifiers::empty(),
false,
false,
),
));
webview.webview.notify_input_event(InputEvent::Keyboard(
KeyboardEvent::new_without_event(
KeyState::Up,
key,
code,
Location::Standard,
Modifiers::empty(),
false,
false,
),
));
}
Ok(())
}
fn set_permission(
&mut self,
request: PermissionRequest,
crates/ely_servo_host/tests/sidecar.rs
+71 -1
View File
@@ -27,6 +27,10 @@ const SERVO_SCROLL_OFFSET: ScrollOffset = ScrollOffset { x: 0, y: 480 };
const SERVO_CLICK_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E";
const SERVO_CLICK_SIZE: FrameSize = FrameSize { width: 640, height: 480 };
const SERVO_CLICK_POINT: ClickPoint = ClickPoint { x: 160, y: 120 };
const SERVO_TEXT_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EText%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3Bfont%3A28px%20sans-serif%3B%7Dinput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A260px%3Bheight%3A70px%3Bfont%3A28px%20sans-serif%3B%7Doutput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A180px%3Bfont%3A32px%20sans-serif%3B%7D%3C%2Fstyle%3E%3Cinput%20id%3Dq%20autofocus%20oninput%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.getElementById%28%27out%27%29.textContent%3Dthis.value%3B%22%3E%3Coutput%20id%3Dout%3Eempty%3C%2Foutput%3E";
const SERVO_TEXT_SIZE: FrameSize = FrameSize { width: 640, height: 480 };
const SERVO_TEXT_POINT: ClickPoint = ClickPoint { x: 160, y: 120 };
const SERVO_TEXT_VALUE: &str = "ely42";
struct PrdSiteCompatibilityCase {
url: &'static str,
@@ -112,6 +116,24 @@ fn sidecar_clicks_page_with_servo_mouse_input() -> Result<(), Box<dyn Error>> {
Ok(())
}
#[test]
fn sidecar_types_text_with_servo_keyboard_input() -> Result<(), Box<dyn Error>> {
let initial_report = snapshot_text_probe(None)?;
let typed_report = snapshot_text_probe(Some(SERVO_TEXT_VALUE))?;
assert_eq!(
report_field_as_u64(&typed_report, "typed_text_byte_count")?,
SERVO_TEXT_VALUE.len() as u64
);
assert!(report_field_as_bool(&typed_report, "text_changed_frame")?);
assert_ne!(
report_field_as_u64(&initial_report, "sample_hash")?,
report_field_as_u64(&typed_report, "sample_hash")?
);
Ok(())
}
fn snapshot_prd_site(
case: &PrdSiteCompatibilityCase,
size: FrameSize,
@@ -135,7 +157,7 @@ fn snapshot_prd_site(
std::fs::remove_file(&output_path)?;
}
let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset, None)?;
let output = run_sidecar_snapshot(case.url, &output_path, size, scroll_offset, None, None)?;
assert!(
output.status.success(),
@@ -192,6 +214,7 @@ fn snapshot_click_probe(
SERVO_CLICK_SIZE,
ScrollOffset::ZERO,
click_point,
None,
)?;
assert!(
@@ -215,12 +238,56 @@ fn snapshot_click_probe(
Ok(report)
}
fn snapshot_text_probe(typed_text: Option<&str>) -> Result<serde_json::Value, Box<dyn Error>> {
let output_path = std::env::temp_dir().join(format!(
"ely-servo-sidecar-{}-text-{}x{}.rgba",
std::process::id(),
SERVO_TEXT_SIZE.width,
SERVO_TEXT_SIZE.height
));
if output_path.exists() {
std::fs::remove_file(&output_path)?;
}
let click_point = typed_text.map(|_| SERVO_TEXT_POINT);
let output = run_sidecar_snapshot(
SERVO_TEXT_URL,
&output_path,
SERVO_TEXT_SIZE,
ScrollOffset::ZERO,
click_point,
typed_text,
)?;
assert!(
output.status.success(),
"text probe\nstatus: {:?}\nstdout: {}\nstderr: {}",
output.status.code(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let report: serde_json::Value = serde_json::from_slice(&output.stdout)?;
assert_eq!(report_field_as_u64(&report, "width")?, SERVO_TEXT_SIZE.width);
assert_eq!(report_field_as_u64(&report, "height")?, SERVO_TEXT_SIZE.height);
assert!(report_field_as_u64(&report, "content_pixel_count")? > 0);
assert_eq!(
std::fs::metadata(&output_path)?.len(),
SERVO_TEXT_SIZE.width * SERVO_TEXT_SIZE.height * 4
);
std::fs::remove_file(&output_path)?;
Ok(report)
}
fn run_sidecar_snapshot(
site_url: &str,
output_path: &std::path::Path,
size: FrameSize,
scroll_offset: ScrollOffset,
click_point: Option<ClickPoint>,
typed_text: Option<&str>,
) -> Result<Output, Box<dyn Error>> {
let mut command = Command::new(env!("CARGO_BIN_EXE_ely_servo_sidecar"));
command
@@ -243,6 +310,9 @@ fn run_sidecar_snapshot(
command.arg("--click-x").arg(click_point.x.to_string());
command.arg("--click-y").arg(click_point.y.to_string());
}
if let Some(typed_text) = typed_text {
command.arg("--type-text").arg(typed_text);
}
let mut child = command.stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?;
crates/ely_servo_host/tests/software_host.rs
+22 -2
View File
@@ -4,8 +4,8 @@ use std::{error::Error, thread, time::Duration};
use ely_domain::{ProfileId, TabId, UrlText};
use ely_servo_host::{
MouseClickRequest, NavigationRequest, ScrollRequest, ServoHost, ServoHostError,
ServoSurfaceSize, SoftwareServoHost, WebViewState,
KeyboardTextRequest, MouseClickRequest, NavigationRequest, ScrollRequest, ServoHost,
ServoHostError, ServoSurfaceSize, SoftwareServoHost, WebViewState,
};
const MINIMUM_CONTENT_PIXELS: u64 = 1_000;
@@ -14,6 +14,8 @@ const PRD_SITE_COMPATIBILITY_CASES: &[PrdSiteCompatibilityCase] = &[
PrdSiteCompatibilityCase { url: "https://servo.org", title_fragment: "Servo" },
];
const CLICK_PROBE_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EClick%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3B%7Dbutton%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A220px%3Bheight%3A90px%3Bfont%3A28px%20sans-serif%3Bbackground%3A%23ffffff%3Bcolor%3A%23111111%3B%7D%3C%2Fstyle%3E%3Cbutton%20onclick%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.title%3D%27Clicked%27%3Bthis.textContent%3D%27Clicked%27%3B%22%3ETap%3C%2Fbutton%3E";
const TEXT_PROBE_URL: &str = "data:text/html,%3C!doctype%20html%3E%3Ctitle%3EText%20Probe%3C%2Ftitle%3E%3Cstyle%3Ebody%7Bmargin%3A0%3Bbackground%3A%23f7f7f7%3Bfont%3A28px%20sans-serif%3B%7Dinput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A80px%3Bwidth%3A260px%3Bheight%3A70px%3Bfont%3A28px%20sans-serif%3B%7Doutput%7Bposition%3Aabsolute%3Bleft%3A80px%3Btop%3A180px%3Bfont%3A32px%20sans-serif%3B%7D%3C%2Fstyle%3E%3Cinput%20id%3Dq%20autofocus%20oninput%3D%22document.body.style.background%3D%27%230039ff%27%3Bdocument.getElementById%28%27out%27%29.textContent%3Dthis.value%3B%22%3E%3Coutput%20id%3Dout%3Eempty%3C%2Foutput%3E";
const TEXT_PROBE_VALUE: &str = "ely42";
struct PrdSiteCompatibilityCase {
url: &'static str,
@@ -53,6 +55,24 @@ fn manages_real_servo_webview_lifecycle() -> Result<(), Box<dyn Error>> {
assert_rendered_frame_has_content(&host, "data:text/html clicked", 1)?;
assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash);
let tab_id = TabId::new();
let url = UrlText::parse(TEXT_PROBE_URL)?;
host.navigate(NavigationRequest { webview_id: webview_id.clone(), tab_id, url })?;
let snapshot = wait_for_rendered_webview(&mut host, &webview_id, None)?;
assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}");
assert_rendered_frame_has_content(&host, "data:text/html input", 1)?;
let previous_frame_hash = host.last_rendered_frame()?.sample_hash();
host.click(MouseClickRequest { webview_id: webview_id.clone(), x: 160, y: 120 })?;
host.type_text(KeyboardTextRequest {
webview_id: webview_id.clone(),
text: TEXT_PROBE_VALUE.to_string(),
})?;
let snapshot = wait_for_rendered_webview(&mut host, &webview_id, Some(previous_frame_hash))?;
assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}");
assert_rendered_frame_has_content(&host, "data:text/html typed", 1)?;
assert_ne!(host.last_rendered_frame()?.sample_hash(), previous_frame_hash);
let mut previous_frame_hash = Some(host.last_rendered_frame()?.sample_hash());
for site in PRD_SITE_COMPATIBILITY_CASES {
let tab_id = TabId::new();