ZacharyZhang-NY/ELY-Browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bridge Servo mouse drag input

Browse Source
This commit is contained in:
ZacharyZhang-NY
2026-05-08 15:15:38 -04:00
parent 7a09967d19
commit 0ea43e8b81
9 changed files with 681 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ely_servo_host/src/bin/ely_servo_sidecar.rs
+83 -81
View File
@@ -8,13 +8,17 @@ use std::{
use ely_domain::{ProfileId, TabId, UrlText};
use ely_servo_host::{
KeyboardTextRequest, MouseClickRequest, NavigationRequest, RenderedFrame, ScrollRequest,
KeyboardTextRequest, MouseClickRequest, MouseDragRequest, NavigationRequest, ScrollRequest,
ServoHost, ServoHostError, ServoSurfaceSize, SoftwareServoHost, TouchTapRequest,
WebViewSnapshot, WebViewState,
};
use serde::Serialize;
use thiserror::Error;
#[path = "ely_servo_sidecar/report.rs"]
mod report;
use report::{SnapshotInputChanges, SnapshotReport};
const WAIT_ITERATIONS: usize = 5_000;
const WAIT_INTERVAL: Duration = Duration::from_millis(2);
const RENDER_TIMEOUT: Duration = Duration::from_secs(20);
@@ -38,6 +42,7 @@ struct SnapshotArgs {
scroll_x: i32,
scroll_y: i32,
click_point: Option<ClickPoint>,
drag_points: Option<DragPoints>,
touch_point: Option<ClickPoint>,
typed_text: Option<String>,
}
@@ -48,6 +53,12 @@ struct ClickPoint {
y: u32,
}
#[derive(Clone, Copy)]
struct DragPoints {
from: ClickPoint,
to: ClickPoint,
}
#[derive(Debug, Error)]
enum SidecarError {
#[error("missing sidecar command")]
@@ -79,6 +90,9 @@ enum SidecarError {
#[error("--click-x and --click-y must be provided together")]
IncompleteClickPoint,
#[error("--drag-from-x, --drag-from-y, --drag-to-x, and --drag-to-y must be provided together")]
IncompleteDragPoints,
#[error("--touch-x and --touch-y must be provided together")]
IncompleteTouchPoint,
@@ -124,6 +138,10 @@ fn parse_snapshot_args(
let mut scroll_y = 0;
let mut click_x = None;
let mut click_y = None;
let mut drag_from_x = None;
let mut drag_from_y = None;
let mut drag_to_x = None;
let mut drag_to_y = None;
let mut touch_x = None;
let mut touch_y = None;
let mut typed_text = None;
@@ -160,6 +178,30 @@ fn parse_snapshot_args(
next_argument(&mut args, "--click-y")?,
)?)
}
"--drag-from-x" => {
drag_from_x = Some(parse_click_coordinate(
"--drag-from-x",
next_argument(&mut args, "--drag-from-x")?,
)?)
}
"--drag-from-y" => {
drag_from_y = Some(parse_click_coordinate(
"--drag-from-y",
next_argument(&mut args, "--drag-from-y")?,
)?)
}
"--drag-to-x" => {
drag_to_x = Some(parse_click_coordinate(
"--drag-to-x",
next_argument(&mut args, "--drag-to-x")?,
)?)
}
"--drag-to-y" => {
drag_to_y = Some(parse_click_coordinate(
"--drag-to-y",
next_argument(&mut args, "--drag-to-y")?,
)?)
}
"--touch-x" => {
touch_x = Some(parse_click_coordinate(
"--touch-x",
@@ -182,6 +224,14 @@ fn parse_snapshot_args(
(None, None) => None,
_ => return Err(SidecarError::IncompleteClickPoint),
};
let drag_points = match (drag_from_x, drag_from_y, drag_to_x, drag_to_y) {
(Some(from_x), Some(from_y), Some(to_x), Some(to_y)) => Some(DragPoints {
from: ClickPoint { x: from_x, y: from_y },
to: ClickPoint { x: to_x, y: to_y },
}),
(None, None, None, None) => None,
_ => return Err(SidecarError::IncompleteDragPoints),
};
let touch_point = match (touch_x, touch_y) {
(Some(x), Some(y)) => Some(ClickPoint { x, y }),
(None, None) => None,
@@ -196,6 +246,7 @@ fn parse_snapshot_args(
scroll_x,
scroll_y,
click_point,
drag_points,
touch_point,
typed_text,
})
@@ -254,6 +305,8 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> {
apply_scroll_if_requested(&mut host, &webview_id, &args, snapshot)?;
let (snapshot, click_changed_frame) =
apply_click_if_requested(&mut host, &webview_id, &args, snapshot)?;
let (snapshot, drag_changed_frame) =
apply_drag_if_requested(&mut host, &webview_id, &args, snapshot)?;
let (snapshot, touch_changed_frame) =
apply_touch_if_requested(&mut host, &webview_id, &args, snapshot)?;
let (snapshot, text_changed_frame) =
@@ -267,10 +320,13 @@ fn run_snapshot(args: SnapshotArgs) -> Result<(), SidecarError> {
&args,
&snapshot,
&frame,
scroll_changed_frame,
click_changed_frame,
touch_changed_frame,
text_changed_frame,
SnapshotInputChanges {
scroll: scroll_changed_frame,
click: click_changed_frame,
drag: drag_changed_frame,
touch: touch_changed_frame,
text: text_changed_frame,
},
),
)?;
Ok(())
@@ -314,6 +370,27 @@ fn apply_click_if_requested(
wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash)
}
fn apply_drag_if_requested(
host: &mut SoftwareServoHost,
webview_id: &ely_domain::WebViewId,
args: &SnapshotArgs,
snapshot: WebViewSnapshot,
) -> Result<(WebViewSnapshot, bool), SidecarError> {
let Some(drag_points) = args.drag_points else {
return Ok((snapshot, false));
};
let previous_frame_hash = host.last_rendered_frame()?.sample_hash();
host.drag(MouseDragRequest {
webview_id: webview_id.clone(),
from_x: drag_points.from.x,
from_y: drag_points.from.y,
to_x: drag_points.to.x,
to_y: drag_points.to.y,
})?;
wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash)
}
fn apply_touch_if_requested(
host: &mut SoftwareServoHost,
webview_id: &ely_domain::WebViewId,
@@ -416,78 +493,3 @@ fn wait_for_changed_or_settled_frame(
Ok((latest_snapshot, false))
}
#[derive(Serialize)]
struct SnapshotReport {
requested_url: String,
loaded_url: Option<String>,
title: Option<String>,
rgba_path: String,
state: &'static str,
width: u32,
height: u32,
rgba_byte_count: usize,
opaque_pixel_count: u64,
non_white_pixel_count: u64,
content_pixel_count: u64,
sample_hash: u64,
scroll_x: i32,
scroll_y: i32,
scroll_changed_frame: bool,
click_x: Option<u32>,
click_y: Option<u32>,
click_changed_frame: bool,
touch_x: Option<u32>,
touch_y: Option<u32>,
touch_changed_frame: bool,
typed_text_byte_count: usize,
text_changed_frame: bool,
}
impl SnapshotReport {
fn new(
args: &SnapshotArgs,
snapshot: &WebViewSnapshot,
frame: &RenderedFrame,
scroll_changed_frame: bool,
click_changed_frame: bool,
touch_changed_frame: bool,
text_changed_frame: bool,
) -> Self {
Self {
requested_url: args.url.as_str().to_string(),
loaded_url: snapshot.url().map(str::to_string),
title: snapshot.title().map(str::to_string),
rgba_path: args.rgba_out.display().to_string(),
state: state_label(snapshot.state()),
width: frame.width(),
height: frame.height(),
rgba_byte_count: frame.rgba_bytes().len(),
opaque_pixel_count: frame.opaque_pixel_count(),
non_white_pixel_count: frame.non_white_pixel_count(),
content_pixel_count: frame.content_pixel_count(),
sample_hash: frame.sample_hash(),
scroll_x: args.scroll_x,
scroll_y: args.scroll_y,
scroll_changed_frame,
click_x: args.click_point.map(|point| point.x),
click_y: args.click_point.map(|point| point.y),
click_changed_frame,
touch_x: args.touch_point.map(|point| point.x),
touch_y: args.touch_point.map(|point| point.y),
touch_changed_frame,
typed_text_byte_count: args.typed_text.as_ref().map_or(0, String::len),
text_changed_frame,
}
}
}
fn state_label(state: &WebViewState) -> &'static str {
match state {
WebViewState::Created => "created",
WebViewState::Loading => "loading",
WebViewState::Complete => "complete",
WebViewState::Sleeping => "sleeping",
WebViewState::Crashed => "crashed",
}
}
crates/ely_servo_host/src/bin/ely_servo_sidecar/report.rs
+94
View File
@@ -0,0 +1,94 @@
use ely_servo_host::{RenderedFrame, WebViewSnapshot, WebViewState};
use serde::Serialize;
use super::SnapshotArgs;
pub(super) struct SnapshotInputChanges {
pub(super) scroll: bool,
pub(super) click: bool,
pub(super) drag: bool,
pub(super) touch: bool,
pub(super) text: bool,
}
#[derive(Serialize)]
pub(super) struct SnapshotReport {
requested_url: String,
loaded_url: Option<String>,
title: Option<String>,
rgba_path: String,
state: &'static str,
width: u32,
height: u32,
rgba_byte_count: usize,
opaque_pixel_count: u64,
non_white_pixel_count: u64,
content_pixel_count: u64,
sample_hash: u64,
scroll_x: i32,
scroll_y: i32,
scroll_changed_frame: bool,
click_x: Option<u32>,
click_y: Option<u32>,
click_changed_frame: bool,
drag_from_x: Option<u32>,
drag_from_y: Option<u32>,
drag_to_x: Option<u32>,
drag_to_y: Option<u32>,
drag_changed_frame: bool,
touch_x: Option<u32>,
touch_y: Option<u32>,
touch_changed_frame: bool,
typed_text_byte_count: usize,
text_changed_frame: bool,
}
impl SnapshotReport {
pub(super) fn new(
args: &SnapshotArgs,
snapshot: &WebViewSnapshot,
frame: &RenderedFrame,
changes: SnapshotInputChanges,
) -> Self {
Self {
requested_url: args.url.as_str().to_string(),
loaded_url: snapshot.url().map(str::to_string),
title: snapshot.title().map(str::to_string),
rgba_path: args.rgba_out.display().to_string(),
state: state_label(snapshot.state()),
width: frame.width(),
height: frame.height(),
rgba_byte_count: frame.rgba_bytes().len(),
opaque_pixel_count: frame.opaque_pixel_count(),
non_white_pixel_count: frame.non_white_pixel_count(),
content_pixel_count: frame.content_pixel_count(),
sample_hash: frame.sample_hash(),
scroll_x: args.scroll_x,
scroll_y: args.scroll_y,
scroll_changed_frame: changes.scroll,
click_x: args.click_point.map(|point| point.x),
click_y: args.click_point.map(|point| point.y),
click_changed_frame: changes.click,
drag_from_x: args.drag_points.map(|points| points.from.x),
drag_from_y: args.drag_points.map(|points| points.from.y),
drag_to_x: args.drag_points.map(|points| points.to.x),
drag_to_y: args.drag_points.map(|points| points.to.y),
drag_changed_frame: changes.drag,
touch_x: args.touch_point.map(|point| point.x),
touch_y: args.touch_point.map(|point| point.y),
touch_changed_frame: changes.touch,
typed_text_byte_count: args.typed_text.as_ref().map_or(0, String::len),
text_changed_frame: changes.text,
}
}
}
fn state_label(state: &WebViewState) -> &'static str {
match state {
WebViewState::Created => "created",
WebViewState::Loading => "loading",
WebViewState::Complete => "complete",
WebViewState::Sleeping => "sleeping",
WebViewState::Crashed => "crashed",
}
}
crates/ely_servo_host/src/host.rs
+11
View File
@@ -227,6 +227,15 @@ pub struct MouseClickRequest {
pub y: u32,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct MouseDragRequest {
pub webview_id: WebViewId,
pub from_x: u32,
pub from_y: u32,
pub to_x: u32,
pub to_y: u32,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TouchTapRequest {
pub webview_id: WebViewId,
@@ -268,6 +277,8 @@ pub trait ServoHost {
fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError>;
fn drag(&mut self, request: MouseDragRequest) -> Result<(), ServoHostError>;
fn touch_tap(&mut self, request: TouchTapRequest) -> Result<(), ServoHostError>;
fn type_text(&mut self, request: KeyboardTextRequest) -> Result<(), ServoHostError>;
crates/ely_servo_host/src/lib.rs
+5 -3
View File
@@ -4,12 +4,14 @@ mod host;
mod keyboard;
#[cfg(feature = "servo-engine")]
mod runtime;
#[cfg(feature = "servo-engine")]
mod runtime_input;
pub use error::ServoHostError;
pub use host::{
KeyboardTextRequest, MouseClickRequest, NavigationRequest, PermissionDecision,
PermissionRequest, RenderedFrame, RenderedFrameSummary, ScrollRequest, ServoHost,
TouchTapRequest, WebViewSnapshot, WebViewState,
KeyboardTextRequest, MouseClickRequest, MouseDragRequest, NavigationRequest,
PermissionDecision, PermissionRequest, RenderedFrame, RenderedFrameSummary, ScrollRequest,
ServoHost, TouchTapRequest, WebViewSnapshot, WebViewState,
};
#[cfg(feature = "servo-engine")]
pub use runtime::{ServoSurfaceSize, SoftwareServoHost};
crates/ely_servo_host/src/runtime.rs
+25 -52
View File
@@ -12,17 +12,16 @@ use dpi::PhysicalSize;
use ely_domain::{ProfileId, TabId, WebViewId};
use servo::{
DeviceIntPoint, DeviceIntRect, DeviceIntSize, DevicePoint, DeviceVector2D, EventLoopWaker,
InputEvent, Key, KeyState, KeyboardEvent, LoadStatus, Location, Modifiers, MouseButton,
MouseButtonAction, MouseButtonEvent, MouseMoveEvent, RenderingContext, Scroll, Servo,
ServoBuilder, TouchEvent, TouchEventType, TouchId, WebView, WebViewBuilder, WebViewDelegate,
WebViewPoint, WebViewVector,
LoadStatus, RenderingContext, Scroll, Servo, ServoBuilder, WebView, WebViewBuilder,
WebViewDelegate, WebViewPoint, WebViewVector,
};
use url::Url;
use crate::{
KeyboardTextRequest, MouseClickRequest, NavigationRequest, PermissionDecision,
PermissionRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError, TouchTapRequest,
WebViewSnapshot, WebViewState, keyboard::keyboard_code_for_character,
KeyboardTextRequest, MouseClickRequest, MouseDragRequest, NavigationRequest,
PermissionDecision, PermissionRequest, RenderedFrame, ScrollRequest, ServoHost, ServoHostError,
TouchTapRequest, WebViewSnapshot, WebViewState,
runtime_input::{send_keyboard_text, send_mouse_click, send_mouse_drag, send_touch_tap},
};
static SERVO_RUNTIME_STARTED: AtomicBool = AtomicBool::new(false);
@@ -172,18 +171,23 @@ impl ServoHost for SoftwareServoHost {
.get(&request.webview_id)
.ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?;
let point = WebViewPoint::Device(DevicePoint::new(request.x as f32, request.y as f32));
webview.webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point)));
webview.webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Down,
MouseButton::Left,
point,
)));
webview.webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Up,
MouseButton::Left,
point,
)));
send_mouse_click(&webview.webview, request.x, request.y);
Ok(())
}
fn drag(&mut self, request: MouseDragRequest) -> Result<(), ServoHostError> {
let webview = self
.webviews
.get(&request.webview_id)
.ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?;
send_mouse_drag(
&webview.webview,
request.from_x,
request.from_y,
request.to_x,
request.to_y,
);
Ok(())
}
@@ -193,13 +197,7 @@ impl ServoHost for SoftwareServoHost {
.get(&request.webview_id)
.ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?;
let point = WebViewPoint::Device(DevicePoint::new(request.x as f32, request.y as f32));
let touch_id = TouchId(1);
for event_type in [TouchEventType::Down, TouchEventType::Up] {
webview.webview.notify_input_event(InputEvent::Touch(TouchEvent::new(
event_type, touch_id, point,
)));
}
send_touch_tap(&webview.webview, request.x, request.y);
Ok(())
}
@@ -209,32 +207,7 @@ impl ServoHost for SoftwareServoHost {
.get(&request.webview_id)
.ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?;
for character in request.text.chars() {
let key = Key::Character(character.to_string());
let code = keyboard_code_for_character(character);
webview.webview.notify_input_event(InputEvent::Keyboard(
KeyboardEvent::new_without_event(
KeyState::Down,
key.clone(),
code,
Location::Standard,
Modifiers::empty(),
false,
false,
),
));
webview.webview.notify_input_event(InputEvent::Keyboard(
KeyboardEvent::new_without_event(
KeyState::Up,
key,
code,
Location::Standard,
Modifiers::empty(),
false,
false,
),
));
}
send_keyboard_text(&webview.webview, &request.text);
Ok(())
}
crates/ely_servo_host/src/runtime_input.rs
+68
View File
@@ -0,0 +1,68 @@
use servo::{
DevicePoint, InputEvent, Key, KeyState, KeyboardEvent, Location, Modifiers, MouseButton,
MouseButtonAction, MouseButtonEvent, MouseMoveEvent, TouchEvent, TouchEventType, TouchId,
WebView, WebViewPoint,
};
use crate::keyboard::keyboard_code_for_character;
pub(super) fn send_mouse_click(webview: &WebView, x: u32, y: u32) {
let point = point(x, y);
webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point)));
send_mouse_button(webview, MouseButtonAction::Down, point);
send_mouse_button(webview, MouseButtonAction::Up, point);
}
pub(super) fn send_mouse_drag(webview: &WebView, from_x: u32, from_y: u32, to_x: u32, to_y: u32) {
let from = point(from_x, from_y);
let to = point(to_x, to_y);
webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(from)));
send_mouse_button(webview, MouseButtonAction::Down, from);
webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(to)));
send_mouse_button(webview, MouseButtonAction::Up, to);
}
pub(super) fn send_touch_tap(webview: &WebView, x: u32, y: u32) {
let point = point(x, y);
let touch_id = TouchId(1);
for event_type in [TouchEventType::Down, TouchEventType::Up] {
webview.notify_input_event(InputEvent::Touch(TouchEvent::new(event_type, touch_id, point)));
}
}
pub(super) fn send_keyboard_text(webview: &WebView, text: &str) {
for character in text.chars() {
let key = Key::Character(character.to_string());
let code = keyboard_code_for_character(character);
webview.notify_input_event(InputEvent::Keyboard(KeyboardEvent::new_without_event(
KeyState::Down,
key.clone(),
code,
Location::Standard,
Modifiers::empty(),
false,
false,
)));
webview.notify_input_event(InputEvent::Keyboard(KeyboardEvent::new_without_event(
KeyState::Up,
key,
code,
Location::Standard,
Modifiers::empty(),
false,
false,
)));
}
}
fn send_mouse_button(webview: &WebView, action: MouseButtonAction, point: WebViewPoint) {
webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
action,
MouseButton::Left,
point,
)));
}
fn point(x: u32, y: u32) -> WebViewPoint {
WebViewPoint::Device(DevicePoint::new(x as f32, y as f32))
}