Route web surface scrolls to Servo hit point

This commit is contained in:
2026-05-12 23:29:40 -04:00
parent 6f279bf3e6
commit 0db1caad69
15 changed files with 220 additions and 126 deletions
+9 -7
View File
@@ -69,7 +69,6 @@ impl ServoLiveClient {
}) })
} }
pub fn ensure( pub fn ensure(
&mut self, &mut self,
request: ServoLiveEnsureRequest, request: ServoLiveEnsureRequest,
@@ -84,6 +83,8 @@ impl ServoLiveClient {
device_pixel_ratio: request.device_pixel_ratio, device_pixel_ratio: request.device_pixel_ratio,
scroll_delta_x: request.scroll_delta_x, scroll_delta_x: request.scroll_delta_x,
scroll_delta_y: request.scroll_delta_y, scroll_delta_y: request.scroll_delta_y,
scroll_point_x: request.scroll_point_x,
scroll_point_y: request.scroll_point_y,
click_x: request.click_x, click_x: request.click_x,
click_y: request.click_y, click_y: request.click_y,
hover_x: request.hover_x, hover_x: request.hover_x,
@@ -136,9 +137,8 @@ impl ServoLiveClient {
// upper limit is `width * height * 4` (RGBA8); `0` is the // upper limit is `width * height * 4` (RGBA8); `0` is the
// explicit "hardware path active, sample the IOSurface // explicit "hardware path active, sample the IOSurface
// instead" signal — anything else is a protocol violation. // instead" signal — anything else is a protocol violation.
let pixel_byte_count = (report.width as u64) let pixel_byte_count =
.saturating_mul(report.height as u64) (report.width as u64).saturating_mul(report.height as u64).saturating_mul(4);
.saturating_mul(4);
let advertised = report.rgba_byte_count as u64; let advertised = report.rgba_byte_count as u64;
if advertised != 0 && advertised != pixel_byte_count { if advertised != 0 && advertised != pixel_byte_count {
return Err(ServoLiveError::FrameBudgetExceeded { return Err(ServoLiveError::FrameBudgetExceeded {
@@ -157,9 +157,7 @@ impl ServoLiveClient {
// fs::read, no temp file. // fs::read, no temp file.
let mut rgba_bytes = vec![0u8; report.rgba_byte_count]; let mut rgba_bytes = vec![0u8; report.rgba_byte_count];
if report.rgba_byte_count > 0 { if report.rgba_byte_count > 0 {
self.stdout self.stdout.read_exact(&mut rgba_bytes).map_err(ServoLiveError::FrameRead)?;
.read_exact(&mut rgba_bytes)
.map_err(ServoLiveError::FrameRead)?;
} }
let mut frame = ServoLiveFrame::from_parts(report, rgba_bytes); let mut frame = ServoLiveFrame::from_parts(report, rgba_bytes);
@@ -220,6 +218,8 @@ pub(crate) struct ServoLiveEnsureRequest {
pub(crate) device_pixel_ratio: f32, pub(crate) device_pixel_ratio: f32,
pub(crate) scroll_delta_x: i32, pub(crate) scroll_delta_x: i32,
pub(crate) scroll_delta_y: i32, pub(crate) scroll_delta_y: i32,
pub(crate) scroll_point_x: Option<u32>,
pub(crate) scroll_point_y: Option<u32>,
pub(crate) click_x: Option<u32>, pub(crate) click_x: Option<u32>,
pub(crate) click_y: Option<u32>, pub(crate) click_y: Option<u32>,
pub(crate) hover_x: Option<u32>, pub(crate) hover_x: Option<u32>,
@@ -410,6 +410,8 @@ enum LiveRequest {
device_pixel_ratio: f32, device_pixel_ratio: f32,
scroll_delta_x: i32, scroll_delta_x: i32,
scroll_delta_y: i32, scroll_delta_y: i32,
scroll_point_x: Option<u32>,
scroll_point_y: Option<u32>,
click_x: Option<u32>, click_x: Option<u32>,
click_y: Option<u32>, click_y: Option<u32>,
hover_x: Option<u32>, hover_x: Option<u32>,
+30 -16
View File
@@ -27,11 +27,7 @@ pub(super) struct WebSurfaceStore {
impl WebSurfaceStore { impl WebSurfaceStore {
pub(super) fn new() -> Self { pub(super) fn new() -> Self {
Self { Self { runtime: WebSurfaceRuntime::new(), surfaces: BTreeMap::new(), keyboard_focus: None }
runtime: WebSurfaceRuntime::new(),
surfaces: BTreeMap::new(),
keyboard_focus: None,
}
} }
pub(super) fn state(&self, tab_id: &TabId) -> Option<&WebSurfaceState> { pub(super) fn state(&self, tab_id: &TabId) -> Option<&WebSurfaceState> {
@@ -102,11 +98,21 @@ impl WebSurfaceStore {
tab_id: &TabId, tab_id: &TabId,
requested_url: &str, requested_url: &str,
delta: Point<Pixels>, delta: Point<Pixels>,
position: Point<Pixels>,
scale_factor: f32, scale_factor: f32,
) -> WebSurfaceInputOutcome { ) -> WebSurfaceInputOutcome {
let Some(delta) = WebSurfaceScrollDelta::from_point(delta, scale_factor) else { let Some(delta) = WebSurfaceScrollDelta::from_point(delta, scale_factor) else {
return WebSurfaceInputOutcome::DroppedZeroDelta; return WebSurfaceInputOutcome::DroppedZeroDelta;
}; };
let Some(bounds) = self.surfaces.get(tab_id).and_then(|surface| surface.viewport_bounds)
else {
return WebSurfaceInputOutcome::DroppedNoViewportBounds;
};
let Some(point) =
WebSurfaceClickPoint::from_window_position(bounds, position, scale_factor)
else {
return WebSurfaceInputOutcome::DroppedOutOfBounds;
};
let surface = self.surface_mut(tab_id); let surface = self.surface_mut(tab_id);
let scroll = surface let scroll = surface
@@ -121,6 +127,7 @@ impl WebSurfaceStore {
Some(current) => current.combined_with(delta), Some(current) => current.combined_with(delta),
None => delta, None => delta,
}); });
surface.pending_scroll_point = Some(point);
// Drop any buffered click — its viewport coordinates were // Drop any buffered click — its viewport coordinates were
// captured against the pre-scroll page, so applying it after // captured against the pre-scroll page, so applying it after
// the scroll would land on the wrong DOM element. Keep // the scroll would land on the wrong DOM element. Keep
@@ -170,12 +177,14 @@ impl WebSurfaceStore {
position: Point<Pixels>, position: Point<Pixels>,
scale_factor: f32, scale_factor: f32,
) -> WebSurfaceInputOutcome { ) -> WebSurfaceInputOutcome {
let surface = self.surfaces.get_mut(tab_id).filter(|surface| surface.viewport_bounds.is_some()); let surface =
self.surfaces.get_mut(tab_id).filter(|surface| surface.viewport_bounds.is_some());
let Some(surface) = surface else { let Some(surface) = surface else {
return WebSurfaceInputOutcome::DroppedNoViewportBounds; return WebSurfaceInputOutcome::DroppedNoViewportBounds;
}; };
let bounds = surface.viewport_bounds.expect("viewport_bounds checked above"); let bounds = surface.viewport_bounds.expect("viewport_bounds checked above");
let Some(point) = WebSurfaceClickPoint::from_window_position(bounds, position, scale_factor) let Some(point) =
WebSurfaceClickPoint::from_window_position(bounds, position, scale_factor)
else { else {
return WebSurfaceInputOutcome::DroppedOutOfBounds; return WebSurfaceInputOutcome::DroppedOutOfBounds;
}; };
@@ -190,12 +199,12 @@ impl WebSurfaceStore {
position: Point<Pixels>, position: Point<Pixels>,
scale_factor: f32, scale_factor: f32,
) -> WebSurfaceInputOutcome { ) -> WebSurfaceInputOutcome {
let Some(bounds) = let Some(bounds) = self.surfaces.get(tab_id).and_then(|surface| surface.viewport_bounds)
self.surfaces.get(tab_id).and_then(|surface| surface.viewport_bounds)
else { else {
return WebSurfaceInputOutcome::DroppedNoViewportBounds; return WebSurfaceInputOutcome::DroppedNoViewportBounds;
}; };
let Some(point) = WebSurfaceClickPoint::from_window_position(bounds, position, scale_factor) let Some(point) =
WebSurfaceClickPoint::from_window_position(bounds, position, scale_factor)
else { else {
return WebSurfaceInputOutcome::DroppedOutOfBounds; return WebSurfaceInputOutcome::DroppedOutOfBounds;
}; };
@@ -206,11 +215,8 @@ impl WebSurfaceStore {
.map(|surface| surface.scroll_offset_for(requested_url)) .map(|surface| surface.scroll_offset_for(requested_url))
.unwrap_or_default(); .unwrap_or_default();
let state = WebSurfaceClickState { let state =
requested_url: requested_url.to_string(), WebSurfaceClickState { requested_url: requested_url.to_string(), scroll_offset, point };
scroll_offset,
point,
};
self.keyboard_focus = Some(WebSurfaceKeyboardFocusState { self.keyboard_focus = Some(WebSurfaceKeyboardFocusState {
tab_id: tab_id.clone(), tab_id: tab_id.clone(),
requested_url: requested_url.to_string(), requested_url: requested_url.to_string(),
@@ -273,6 +279,7 @@ impl WebSurfaceStore {
let surface = self.surface_mut(tab_id); let surface = self.surface_mut(tab_id);
let scroll_offset = surface.scroll_offset_for(requested_url); let scroll_offset = surface.scroll_offset_for(requested_url);
let scroll_delta = surface.pending_scroll_delta.take(); let scroll_delta = surface.pending_scroll_delta.take();
let scroll_point = surface.pending_scroll_point.take();
let click_point = surface let click_point = surface
.click_point .click_point
.take() .take()
@@ -287,7 +294,14 @@ impl WebSurfaceStore {
.map(|state| state.text); .map(|state| state.text);
let hover_point = surface.hover_point.take(); let hover_point = surface.hover_point.take();
WebSurfacePendingInput { scroll_offset, scroll_delta, click_point, hover_point, typed_text } WebSurfacePendingInput {
scroll_offset,
scroll_delta,
scroll_point,
click_point,
hover_point,
typed_text,
}
} }
fn previous_ready_frame( fn previous_ready_frame(
@@ -67,6 +67,7 @@ impl ElyShell {
tab_id: TabId, tab_id: TabId,
requested_url: String, requested_url: String,
delta: Point<Pixels>, delta: Point<Pixels>,
position: Point<Pixels>,
scale_factor: f32, scale_factor: f32,
cx: &mut Context<Self>, cx: &mut Context<Self>,
) { ) {
@@ -74,6 +75,7 @@ impl ElyShell {
&tab_id, &tab_id,
requested_url.as_str(), requested_url.as_str(),
delta, delta,
position,
scale_factor, scale_factor,
) == WebSurfaceInputOutcome::Applied ) == WebSurfaceInputOutcome::Applied
{ {
@@ -45,6 +45,8 @@ impl WebSurfaceRuntime {
if active_scope != &scope { if active_scope != &scope {
return Err(active_scope.error_for(&scope)); return Err(active_scope.error_for(&scope));
} }
let (scroll_delta_x, scroll_delta_y, scroll_point_x, scroll_point_y) =
scroll_wire_fields(input.scroll_delta, input.scroll_point)?;
let session = sessions.entry(tab.id().clone()).or_insert_with(WebSurfaceSession::default); let session = sessions.entry(tab.id().clone()).or_insert_with(WebSurfaceSession::default);
let next_scroll_offset = input.scroll_offset; let next_scroll_offset = input.scroll_offset;
@@ -58,8 +60,10 @@ impl WebSurfaceRuntime {
height: size.height, height: size.height,
page_zoom_percent: zoom_percent, page_zoom_percent: zoom_percent,
device_pixel_ratio: size.device_pixel_ratio_f32(), device_pixel_ratio: size.device_pixel_ratio_f32(),
scroll_delta_x: input.scroll_delta.map_or(0, |delta| delta.x()), scroll_delta_x,
scroll_delta_y: input.scroll_delta.map_or(0, |delta| delta.y()), scroll_delta_y,
scroll_point_x,
scroll_point_y,
click_x: input.click_point.map(|point| point.x()), click_x: input.click_point.map(|point| point.x()),
click_y: input.click_point.map(|point| point.y()), click_y: input.click_point.map(|point| point.y()),
hover_x: input.hover_point.map(|point| point.x()), hover_x: input.hover_point.map(|point| point.x()),
@@ -227,6 +231,20 @@ fn config_dir_for_scope(
} }
} }
fn scroll_wire_fields(
delta: Option<super::web_surface_geometry::WebSurfaceScrollDelta>,
point: Option<super::web_surface_geometry::WebSurfaceClickPoint>,
) -> Result<(i32, i32, Option<u32>, Option<u32>), String> {
match delta {
Some(delta) => {
let point = point
.ok_or_else(|| "Servo scroll input is missing a viewport point".to_string())?;
Ok((delta.x(), delta.y(), Some(point.x()), Some(point.y())))
}
None => Ok((0, 0, None, None)),
}
}
impl From<&WebSurfaceSitePermission> for ServoLiveSitePermission { impl From<&WebSurfaceSitePermission> for ServoLiveSitePermission {
fn from(permission: &WebSurfaceSitePermission) -> Self { fn from(permission: &WebSurfaceSitePermission) -> Self {
Self::new( Self::new(
@@ -45,6 +45,7 @@ pub(super) struct WebSurfaceTextInputState {
pub(super) struct WebSurfacePendingInput { pub(super) struct WebSurfacePendingInput {
pub(super) scroll_offset: WebSurfaceScrollOffset, pub(super) scroll_offset: WebSurfaceScrollOffset,
pub(super) scroll_delta: Option<WebSurfaceScrollDelta>, pub(super) scroll_delta: Option<WebSurfaceScrollDelta>,
pub(super) scroll_point: Option<WebSurfaceClickPoint>,
pub(super) click_point: Option<WebSurfaceClickPoint>, pub(super) click_point: Option<WebSurfaceClickPoint>,
pub(super) hover_point: Option<WebSurfaceClickPoint>, pub(super) hover_point: Option<WebSurfaceClickPoint>,
pub(super) typed_text: Option<String>, pub(super) typed_text: Option<String>,
@@ -116,6 +117,7 @@ pub(super) struct PerTabSurface {
pub(super) hover_point: Option<WebSurfaceClickPoint>, pub(super) hover_point: Option<WebSurfaceClickPoint>,
pub(super) click_point: Option<WebSurfaceClickState>, pub(super) click_point: Option<WebSurfaceClickState>,
pub(super) pending_scroll_delta: Option<WebSurfaceScrollDelta>, pub(super) pending_scroll_delta: Option<WebSurfaceScrollDelta>,
pub(super) pending_scroll_point: Option<WebSurfaceClickPoint>,
pub(super) scroll_offset: Option<WebSurfaceScrollState>, pub(super) scroll_offset: Option<WebSurfaceScrollState>,
pub(super) typed_text: Option<WebSurfaceTextInputState>, pub(super) typed_text: Option<WebSurfaceTextInputState>,
pub(super) state: Option<WebSurfaceState>, pub(super) state: Option<WebSurfaceState>,
@@ -130,6 +132,7 @@ impl PerTabSurface {
hover_point: None, hover_point: None,
click_point: None, click_point: None,
pending_scroll_delta: None, pending_scroll_delta: None,
pending_scroll_point: None,
scroll_offset: None, scroll_offset: None,
typed_text: None, typed_text: None,
state: None, state: None,
+27 -11
View File
@@ -42,12 +42,14 @@ fn scroll_delta_enters_pending_input_after_wheel() -> Result<(), Box<dyn Error>>
tab.id(), tab.id(),
tab.url().as_str(), tab.url().as_str(),
point(px(0.0), px(140.0)), point(px(0.0), px(140.0)),
point(px(320.0), px(240.0)),
1.0, 1.0,
)); ));
assert_applied(store.record_scroll_delta( assert_applied(store.record_scroll_delta(
tab.id(), tab.id(),
tab.url().as_str(), tab.url().as_str(),
point(px(0.0), px(60.0)), point(px(0.0), px(60.0)),
point(px(300.0), px(220.0)),
1.0, 1.0,
)); ));
@@ -55,6 +57,7 @@ fn scroll_delta_enters_pending_input_after_wheel() -> Result<(), Box<dyn Error>>
assert_eq!(input.scroll_offset.y(), 200); assert_eq!(input.scroll_offset.y(), 200);
assert_eq!(input.scroll_delta.map(|delta| (delta.x(), delta.y())), Some((0, 200))); assert_eq!(input.scroll_delta.map(|delta| (delta.x(), delta.y())), Some((0, 200)));
assert_eq!(input.scroll_point.map(|point| (point.x(), point.y())), Some((300, 220)));
Ok(()) Ok(())
} }
@@ -91,7 +94,13 @@ fn scroll_after_click_keeps_keyboard_focus_and_typed_text() -> Result<(), Box<dy
assert_applied(store.record_click_point(tab.id(), url, point(px(160.0), px(120.0)), 1.0)); assert_applied(store.record_click_point(tab.id(), url, point(px(160.0), px(120.0)), 1.0));
assert_applied(store.record_typed_text(tab.id(), url, "h")); assert_applied(store.record_typed_text(tab.id(), url, "h"));
assert_applied(store.record_scroll_delta(tab.id(), url, point(px(0.0), px(140.0)), 1.0)); assert_applied(store.record_scroll_delta(
tab.id(),
url,
point(px(0.0), px(140.0)),
point(px(160.0), px(120.0)),
1.0,
));
assert_eq!( assert_eq!(
store.record_typed_text(tab.id(), url, "i"), store.record_typed_text(tab.id(), url, "i"),
@@ -134,7 +143,13 @@ fn retina_scale_factor_doubles_every_input_coordinate() -> Result<(), Box<dyn Er
assert_applied(store.record_viewport_size(tab.id(), web_bounds(), 2.0)); assert_applied(store.record_viewport_size(tab.id(), web_bounds(), 2.0));
assert_applied(store.record_click_point(tab.id(), url, point(px(160.0), px(120.0)), 2.0)); assert_applied(store.record_click_point(tab.id(), url, point(px(160.0), px(120.0)), 2.0));
assert_applied(store.record_typed_text(tab.id(), url, "h")); assert_applied(store.record_typed_text(tab.id(), url, "h"));
assert_applied(store.record_scroll_delta(tab.id(), url, point(px(0.0), px(140.0)), 2.0)); assert_applied(store.record_scroll_delta(
tab.id(),
url,
point(px(0.0), px(140.0)),
point(px(160.0), px(120.0)),
2.0,
));
let input = store.take_pending_input(tab.id(), url); let input = store.take_pending_input(tab.id(), url);
@@ -145,8 +160,7 @@ fn retina_scale_factor_doubles_every_input_coordinate() -> Result<(), Box<dyn Er
); );
assert_eq!(input.scroll_offset.y(), 280, "scroll offset accumulates in device px"); assert_eq!(input.scroll_offset.y(), 280, "scroll offset accumulates in device px");
assert_eq!( assert_eq!(
input.click_point, input.click_point, None,
None,
"scroll drops the buffered click — its viewport coords are stale", "scroll drops the buffered click — its viewport coords are stale",
); );
Ok(()) Ok(())
@@ -181,12 +195,7 @@ fn click_before_viewport_measured_reports_no_viewport_bounds() -> Result<(), Box
let tab = web_tab("https://example.com/form")?; let tab = web_tab("https://example.com/form")?;
assert_eq!( assert_eq!(
store.record_click_point( store.record_click_point(tab.id(), tab.url().as_str(), point(px(160.0), px(120.0)), 1.0,),
tab.id(),
tab.url().as_str(),
point(px(160.0), px(120.0)),
1.0,
),
WebSurfaceInputOutcome::DroppedNoViewportBounds, WebSurfaceInputOutcome::DroppedNoViewportBounds,
); );
Ok(()) Ok(())
@@ -206,6 +215,7 @@ fn zero_wheel_delta_reports_zero_delta() -> Result<(), Box<dyn Error>> {
tab.id(), tab.id(),
tab.url().as_str(), tab.url().as_str(),
point(px(0.0), px(0.0)), point(px(0.0), px(0.0)),
point(px(160.0), px(120.0)),
1.0, 1.0,
), ),
WebSurfaceInputOutcome::DroppedZeroDelta, WebSurfaceInputOutcome::DroppedZeroDelta,
@@ -276,7 +286,13 @@ fn zero_wheel_delta_must_not_erase_buffered_click() -> Result<(), Box<dyn Error>
assert_applied(store.record_click_point(tab.id(), url, point(px(160.0), px(120.0)), 1.0)); assert_applied(store.record_click_point(tab.id(), url, point(px(160.0), px(120.0)), 1.0));
assert_eq!( assert_eq!(
store.record_scroll_delta(tab.id(), url, point(px(0.0), px(0.0)), 1.0), store.record_scroll_delta(
tab.id(),
url,
point(px(0.0), px(0.0)),
point(px(160.0), px(120.0)),
1.0,
),
WebSurfaceInputOutcome::DroppedZeroDelta, WebSurfaceInputOutcome::DroppedZeroDelta,
); );
+3 -12
View File
@@ -72,12 +72,7 @@ fn error_page(message: &str) -> impl IntoElement {
.text_color(rgb(colors::INK)) .text_color(rgb(colors::INK))
.child("Page unavailable"), .child("Page unavailable"),
) )
.child( .child(div().text_size(px(14.0)).text_color(rgb(colors::INK_3)).child(message.to_string()))
div()
.text_size(px(14.0))
.text_color(rgb(colors::INK_3))
.child(message.to_string()),
)
} }
fn render_web_surface( fn render_web_surface(
@@ -95,12 +90,7 @@ fn render_web_surface(
.size_full() .size_full()
.min_w_0() .min_w_0()
.overflow_hidden() .overflow_hidden()
.child( .child(div().absolute().inset_0().child(content))
div()
.absolute()
.inset_0()
.child(content),
)
.child(render_viewport_tracker(tab.id().clone(), tracker_entity)) .child(render_viewport_tracker(tab.id().clone(), tracker_entity))
.child(render_input_overlay(input_tab_id, input_url, input_entity)) .child(render_input_overlay(input_tab_id, input_url, input_entity))
.into_any_element() .into_any_element()
@@ -169,6 +159,7 @@ fn render_input_overlay(
scroll_tab_id.clone(), scroll_tab_id.clone(),
scroll_url.clone(), scroll_url.clone(),
delta, delta,
event.position,
scale_factor, scale_factor,
cx, cx,
); );
@@ -150,6 +150,8 @@ fn apply_scroll_if_requested(
webview_id: webview_id.clone(), webview_id: webview_id.clone(),
delta_x: args.scroll_x, delta_x: args.scroll_x,
delta_y: args.scroll_y, delta_y: args.scroll_y,
point_x: 0,
point_y: 0,
})?; })?;
wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash) wait_for_changed_or_settled_frame(host, webview_id, previous_frame_hash)
} }
@@ -14,10 +14,10 @@ use ely_servo_host::{
}; };
use super::args::LiveArgs; use super::args::LiveArgs;
pub(super) use super::live_protocol::LiveSidecarError;
use super::live_protocol::{ use super::live_protocol::{
LiveFrameReport, LiveOutcome, LiveRequest, LiveSitePermission, PartialFrameTimings, LiveFrameReport, LiveOutcome, LiveRequest, LiveSitePermission, PartialFrameTimings,
}; };
pub(super) use super::live_protocol::LiveSidecarError;
use super::perf::{FramePerfAggregator, FramePerfSummary, FrameStageTimings, elapsed_ns}; use super::perf::{FramePerfAggregator, FramePerfSummary, FrameStageTimings, elapsed_ns};
/// Per-`Ensure` budget for Servo to paint after input dispatch. /// Per-`Ensure` budget for Servo to paint after input dispatch.
@@ -65,13 +65,7 @@ pub(super) fn run_live(args: LiveArgs) -> Result<(), LiveSidecarError> {
), ),
Err(error) => Err(LiveSidecarError::Json(error)), Err(error) => Err(LiveSidecarError::Json(error)),
}; };
write_outcome( write_outcome(&mut stdout, &mut perf, &mut pending_summary, outcome, frame_started_at)?;
&mut stdout,
&mut perf,
&mut pending_summary,
outcome,
frame_started_at,
)?;
} }
Ok(()) Ok(())
@@ -102,6 +96,8 @@ fn handle_request(
device_pixel_ratio, device_pixel_ratio,
scroll_delta_x, scroll_delta_x,
scroll_delta_y, scroll_delta_y,
scroll_point_x,
scroll_point_y,
click_x, click_x,
click_y, click_y,
hover_x, hover_x,
@@ -115,14 +111,7 @@ fn handle_request(
let session = let session =
ensure_session(host, sessions, tab_id.clone(), &tab, &profile, width, height)?; ensure_session(host, sessions, tab_id.clone(), &tab, &profile, width, height)?;
if apply_layout( if apply_layout(host, session, width, height, page_zoom_percent, device_pixel_ratio)? {
host,
session,
width,
height,
page_zoom_percent,
device_pixel_ratio,
)? {
session.awaiting_visible_frame = true; session.awaiting_visible_frame = true;
} }
apply_permissions(host, session, &profile, site_permissions)?; apply_permissions(host, session, &profile, site_permissions)?;
@@ -141,17 +130,18 @@ fn handle_request(
// the gate skip blank loading frames again. // the gate skip blank loading frames again.
session.ever_visible_frame = false; session.ever_visible_frame = false;
} }
if apply_input( let input = LiveInput {
host,
session,
scroll_delta_x, scroll_delta_x,
scroll_delta_y, scroll_delta_y,
scroll_point_x,
scroll_point_y,
click_x, click_x,
click_y, click_y,
hover_x, hover_x,
hover_y, hover_y,
typed_text, typed_text,
)? { };
if apply_input(host, session, input)? {
// Tell poll_frame to actually wait for Servo to paint // Tell poll_frame to actually wait for Servo to paint
// a response to this input. The visible-content gate // a response to this input. The visible-content gate
// is bypassed on the hardware path inside poll_frame, // is bypassed on the hardware path inside poll_frame,
@@ -162,7 +152,13 @@ fn handle_request(
} }
let webview_id = session.webview_id.clone(); let webview_id = session.webview_id.clone();
let mut outcome = poll_frame(host, session, rendering_context_kind)?; let mut outcome = poll_frame(host, session, rendering_context_kind)?;
populate_surface_fields(host, &webview_id, &tab_id, published_surface_ids, &mut outcome); populate_surface_fields(
host,
&webview_id,
&tab_id,
published_surface_ids,
&mut outcome,
);
Ok(outcome) Ok(outcome)
} }
LiveRequest::Poll { tab_id } => { LiveRequest::Poll { tab_id } => {
@@ -171,7 +167,13 @@ fn handle_request(
}; };
let webview_id = session.webview_id.clone(); let webview_id = session.webview_id.clone();
let mut outcome = poll_frame(host, session, rendering_context_kind)?; let mut outcome = poll_frame(host, session, rendering_context_kind)?;
populate_surface_fields(host, &webview_id, &tab_id, published_surface_ids, &mut outcome); populate_surface_fields(
host,
&webview_id,
&tab_id,
published_surface_ids,
&mut outcome,
);
Ok(outcome) Ok(outcome)
} }
} }
@@ -255,19 +257,19 @@ fn write_outcome(
// header so the client knows nothing follows). At 1080p × 60 fps // header so the client knows nothing follows). At 1080p × 60 fps
// that's 8 MB × 60 = ~480 MB/s of pipe traffic eliminated. // that's 8 MB × 60 = ~480 MB/s of pipe traffic eliminated.
let drop_rgba_payload = outcome.response.current_surface_id.is_some(); let drop_rgba_payload = outcome.response.current_surface_id.is_some();
if drop_rgba_payload { if drop_rgba_payload
if let Some(report) = outcome.response.frame.as_mut() { && let Some(report) = outcome.response.frame.as_mut()
{
report.rgba_byte_count = 0; report.rgba_byte_count = 0;
} }
}
let write_started_at = Instant::now(); let write_started_at = Instant::now();
serde_json::to_writer(&mut *stdout, &outcome.response)?; serde_json::to_writer(&mut *stdout, &outcome.response)?;
stdout.write_all(b"\n")?; stdout.write_all(b"\n")?;
if !drop_rgba_payload { if !drop_rgba_payload
if let Some(frame) = outcome.frame.as_ref() { && let Some(frame) = outcome.frame.as_ref()
{
stdout.write_all(frame.rgba_bytes())?; stdout.write_all(frame.rgba_bytes())?;
} }
}
stdout.flush()?; stdout.flush()?;
if frame_present { if frame_present {
let write_ns = elapsed_ns(write_started_at); let write_ns = elapsed_ns(write_started_at);
@@ -390,37 +392,34 @@ fn apply_permissions(
fn apply_input( fn apply_input(
host: &mut SoftwareServoHost, host: &mut SoftwareServoHost,
session: &mut LiveSession, session: &mut LiveSession,
scroll_delta_x: i32, input: LiveInput,
scroll_delta_y: i32,
click_x: Option<u32>,
click_y: Option<u32>,
hover_x: Option<u32>,
hover_y: Option<u32>,
typed_text: Option<String>,
) -> Result<bool, LiveSidecarError> { ) -> Result<bool, LiveSidecarError> {
let mut changed = false; let mut changed = false;
if scroll_delta_x != 0 || scroll_delta_y != 0 { if input.scroll_delta_x != 0 || input.scroll_delta_y != 0 {
let (point_x, point_y) = input.scroll_point()?;
host.scroll(ScrollRequest { host.scroll(ScrollRequest {
webview_id: session.webview_id.clone(), webview_id: session.webview_id.clone(),
delta_x: scroll_delta_x, delta_x: input.scroll_delta_x,
delta_y: scroll_delta_y, delta_y: input.scroll_delta_y,
point_x,
point_y,
})?; })?;
session.scroll_x = positive_scroll_component(session.scroll_x, scroll_delta_x); session.scroll_x = positive_scroll_component(session.scroll_x, input.scroll_delta_x);
session.scroll_y = positive_scroll_component(session.scroll_y, scroll_delta_y); session.scroll_y = positive_scroll_component(session.scroll_y, input.scroll_delta_y);
changed = true; changed = true;
} }
if let (Some(x), Some(y)) = (hover_x, hover_y) { if let (Some(x), Some(y)) = (input.hover_x, input.hover_y) {
host.hover(MouseHoverRequest { webview_id: session.webview_id.clone(), x, y })?; host.hover(MouseHoverRequest { webview_id: session.webview_id.clone(), x, y })?;
changed = true; changed = true;
} }
if let (Some(x), Some(y)) = (click_x, click_y) { if let (Some(x), Some(y)) = (input.click_x, input.click_y) {
host.click(MouseClickRequest { webview_id: session.webview_id.clone(), x, y })?; host.click(MouseClickRequest { webview_id: session.webview_id.clone(), x, y })?;
changed = true; changed = true;
} }
if let Some(text) = typed_text { if let Some(text) = input.typed_text {
host.type_text(KeyboardTextRequest { webview_id: session.webview_id.clone(), text })?; host.type_text(KeyboardTextRequest { webview_id: session.webview_id.clone(), text })?;
changed = true; changed = true;
} }
@@ -428,6 +427,28 @@ fn apply_input(
Ok(changed) Ok(changed)
} }
struct LiveInput {
scroll_delta_x: i32,
scroll_delta_y: i32,
scroll_point_x: Option<u32>,
scroll_point_y: Option<u32>,
click_x: Option<u32>,
click_y: Option<u32>,
hover_x: Option<u32>,
hover_y: Option<u32>,
typed_text: Option<String>,
}
impl LiveInput {
fn scroll_point(&self) -> Result<(u32, u32), LiveSidecarError> {
let point = match (self.scroll_point_x, self.scroll_point_y) {
(Some(x), Some(y)) => (x, y),
_ => return Err(LiveSidecarError::IncompleteScrollPoint),
};
Ok(point)
}
}
fn poll_frame( fn poll_frame(
host: &mut SoftwareServoHost, host: &mut SoftwareServoHost,
session: &mut LiveSession, session: &mut LiveSession,
@@ -469,8 +490,7 @@ fn poll_frame(
let has_visible_content = match rendering_context_kind { let has_visible_content = match rendering_context_kind {
RenderingContextKind::Software => { RenderingContextKind::Software => {
session.ever_visible_frame session.ever_visible_frame
|| (frame.non_white_pixel_count() > 0 || (frame.non_white_pixel_count() > 0 && frame.content_pixel_count() > 0)
&& frame.content_pixel_count() > 0)
} }
#[cfg(feature = "hardware-render")] #[cfg(feature = "hardware-render")]
RenderingContextKind::Hardware => true, RenderingContextKind::Hardware => true,
@@ -3,7 +3,9 @@
use std::io; use std::io;
use ely_servo_host::{IOSurfaceHandle, RenderedFrame, ServoHostError, WebViewSnapshot, WebViewState}; use ely_servo_host::{
IOSurfaceHandle, RenderedFrame, ServoHostError, WebViewSnapshot, WebViewState,
};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use thiserror::Error; use thiserror::Error;
@@ -29,6 +31,8 @@ pub(super) enum LiveRequest {
device_pixel_ratio: f32, device_pixel_ratio: f32,
scroll_delta_x: i32, scroll_delta_x: i32,
scroll_delta_y: i32, scroll_delta_y: i32,
scroll_point_x: Option<u32>,
scroll_point_y: Option<u32>,
click_x: Option<u32>, click_x: Option<u32>,
click_y: Option<u32>, click_y: Option<u32>,
#[serde(default)] #[serde(default)]
@@ -195,6 +199,9 @@ pub(super) enum LiveSidecarError {
#[error("live session is unavailable after creation")] #[error("live session is unavailable after creation")]
SessionUnavailable, SessionUnavailable,
#[error("scroll input requires both scroll_point_x and scroll_point_y")]
IncompleteScrollPoint,
#[error(transparent)] #[error(transparent)]
Domain(#[from] ely_domain::DomainError), Domain(#[from] ely_domain::DomainError),
@@ -262,17 +262,19 @@ mod tests {
} }
#[test] #[test]
fn aggregator_emits_summary_after_window_size_records() { fn aggregator_emits_summary_after_window_size_records() -> Result<(), &'static str> {
let mut aggregator = let mut aggregator =
FramePerfAggregator::new("software", FramePerfAggregator::DEFAULT_WINDOW_SIZE); FramePerfAggregator::new("software", FramePerfAggregator::DEFAULT_WINDOW_SIZE);
for index in 0..(FramePerfAggregator::DEFAULT_WINDOW_SIZE - 1) { for index in 0..(FramePerfAggregator::DEFAULT_WINDOW_SIZE - 1) {
let result = aggregator.record(constant_timing()); let result = aggregator.record(constant_timing());
assert!(result.is_none(), "should not flush at frame {index}"); assert!(result.is_none(), "should not flush at frame {index}");
} }
let summary = aggregator.record(constant_timing()); let summary = aggregator
let summary = summary.expect("aggregator must flush at window boundary"); .record(constant_timing())
.ok_or("aggregator must flush at window boundary")?;
assert_eq!(summary.window, FramePerfAggregator::DEFAULT_WINDOW_SIZE); assert_eq!(summary.window, FramePerfAggregator::DEFAULT_WINDOW_SIZE);
assert_eq!(summary.context, "software"); assert_eq!(summary.context, "software");
Ok(())
} }
#[test] #[test]
@@ -286,7 +288,7 @@ mod tests {
} }
#[test] #[test]
fn aggregator_percentiles_track_increasing_paint_durations() { fn aggregator_percentiles_track_increasing_paint_durations() -> Result<(), &'static str> {
let mut aggregator = FramePerfAggregator::new("software", 4); let mut aggregator = FramePerfAggregator::new("software", 4);
let paint_durations_us = [10u64, 100, 1_000, 10_000]; let paint_durations_us = [10u64, 100, 1_000, 10_000];
let mut summary = None; let mut summary = None;
@@ -298,11 +300,12 @@ mod tests {
Duration::from_micros(paint_us + 2), Duration::from_micros(paint_us + 2),
)); ));
} }
let summary = summary.expect("4-frame window must flush"); let summary = summary.ok_or("4-frame window must flush")?;
assert!( assert!(
summary.paint_p50_us < summary.paint_p99_us, summary.paint_p50_us < summary.paint_p99_us,
"p99 must dominate p50 for increasing samples: {summary:?}" "p99 must dominate p50 for increasing samples: {summary:?}"
); );
Ok(())
} }
fn constant_timing() -> FrameStageTimings { fn constant_timing() -> FrameStageTimings {
+2
View File
@@ -220,6 +220,8 @@ pub struct ScrollRequest {
pub webview_id: WebViewId, pub webview_id: WebViewId,
pub delta_x: i32, pub delta_x: i32,
pub delta_y: i32, pub delta_y: i32,
pub point_x: u32,
pub point_y: u32,
} }
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
+11 -18
View File
@@ -16,17 +16,15 @@ use dpi::PhysicalSize;
use ely_domain::{ProfileId, TabId, WebViewId}; use ely_domain::{ProfileId, TabId, WebViewId};
use euclid::Scale; use euclid::Scale;
use servo::{ use servo::{
DeviceIndependentPixel, DeviceIntPoint, DeviceIntRect, DeviceIntSize, DevicePixel, DeviceIndependentPixel, DeviceIntPoint, DeviceIntRect, DeviceIntSize, DevicePixel, DevicePoint,
DevicePoint, DeviceVector2D, Opts, RenderingContext, Scroll, Servo, ServoBuilder, DeviceVector2D, Opts, RenderingContext, Scroll, Servo, ServoBuilder, WebViewBuilder,
WebViewBuilder, WebViewPoint, WebViewVector, WebViewPoint, WebViewVector,
}; };
/// Wrap an `f32` scale factor in Servo's typed `Scale<f32, DeviceIndependentPixel, /// Wrap an `f32` scale factor in Servo's typed `Scale<f32, DeviceIndependentPixel,
/// DevicePixel>`. The clamp guards against `NaN`/`inf` reaching Servo's /// DevicePixel>`. The clamp guards against `NaN`/`inf` reaching Servo's
/// layout (which assumes a positive finite scale). /// layout (which assumes a positive finite scale).
fn hidpi_scale_from_factor( fn hidpi_scale_from_factor(scale_factor: f32) -> Scale<f32, DeviceIndependentPixel, DevicePixel> {
scale_factor: f32,
) -> Scale<f32, DeviceIndependentPixel, DevicePixel> {
let safe = if scale_factor.is_finite() && scale_factor > 0.0 { let safe = if scale_factor.is_finite() && scale_factor > 0.0 {
scale_factor.clamp(0.5, 5.0) scale_factor.clamp(0.5, 5.0)
} else { } else {
@@ -37,10 +35,10 @@ fn hidpi_scale_from_factor(
use url::Url; use url::Url;
use crate::{ use crate::{
HidpiScaleRequest, KeyboardTextRequest, MouseClickRequest, MouseDragRequest, HidpiScaleRequest, KeyboardTextRequest, MouseClickRequest, MouseDragRequest, MouseHoverRequest,
MouseHoverRequest, NavigationRequest, PageZoomRequest, PermissionDecision, PermissionRequest, NavigationRequest, PageZoomRequest, PermissionDecision, PermissionRequest, RenderedFrame,
RenderedFrame, ResizeRequest, ScreenshotRequest, ScrollRequest, ServoHost, ServoHostError, ResizeRequest, ScreenshotRequest, ScrollRequest, ServoHost, ServoHostError, TouchTapRequest,
TouchTapRequest, WebViewSnapshot, WebViewState, WebViewSnapshot, WebViewState,
runtime_input::{ runtime_input::{
send_keyboard_text, send_mouse_click, send_mouse_drag, send_mouse_hover, send_touch_tap, send_keyboard_text, send_mouse_click, send_mouse_drag, send_mouse_hover, send_touch_tap,
}, },
@@ -265,7 +263,7 @@ impl ServoHost for SoftwareServoHost {
request.delta_x as f32, request.delta_x as f32,
request.delta_y as f32, request.delta_y as f32,
))), ))),
WebViewPoint::Device(DevicePoint::zero()), WebViewPoint::Device(DevicePoint::new(request.point_x as f32, request.point_y as f32)),
); );
Ok(()) Ok(())
} }
@@ -544,9 +542,7 @@ impl SoftwareServoHost {
crate::HardwareOffscreenContext::new(size.physical()) crate::HardwareOffscreenContext::new(size.physical())
.map_err(|_| ServoHostError::RenderingContextUnavailable)?, .map_err(|_| ServoHostError::RenderingContextUnavailable)?,
); );
hardware hardware.make_current().map_err(|_| ServoHostError::RenderingContextNotCurrent)?;
.make_current()
.map_err(|_| ServoHostError::RenderingContextNotCurrent)?;
Ok(RenderingContextHandles { Ok(RenderingContextHandles {
rendering_context: hardware.clone(), rendering_context: hardware.clone(),
hardware_context: Some(hardware), hardware_context: Some(hardware),
@@ -576,10 +572,7 @@ impl SoftwareServoHost {
/// Re-asserting per dispatch keeps the invariant on the dispatch /// Re-asserting per dispatch keeps the invariant on the dispatch
/// path instead of spread across creation, navigation, and /// path instead of spread across creation, navigation, and
/// tab-switching. /// tab-switching.
fn webview_for_input( fn webview_for_input(&self, webview_id: &WebViewId) -> Result<&HostWebView, ServoHostError> {
&self,
webview_id: &WebViewId,
) -> Result<&HostWebView, ServoHostError> {
let webview = self.webview(webview_id)?; let webview = self.webview(webview_id)?;
webview.webview.show(); webview.webview.show();
webview.webview.focus(); webview.webview.focus();
+19 -4
View File
@@ -332,6 +332,11 @@ fn build_ensure(
) -> String { ) -> String {
let hover_x = if include_hover { Some(256u32) } else { None }; let hover_x = if include_hover { Some(256u32) } else { None };
let hover_y = if include_hover { Some(256u32) } else { None }; let hover_y = if include_hover { Some(256u32) } else { None };
let scroll_point = if scroll_dx != 0 || scroll_dy != 0 {
Some((256u32, 256u32))
} else {
None
};
let hover_x_json = match hover_x { let hover_x_json = match hover_x {
Some(value) => format!("{value}"), Some(value) => format!("{value}"),
None => "null".to_string(), None => "null".to_string(),
@@ -340,8 +345,16 @@ fn build_ensure(
Some(value) => format!("{value}"), Some(value) => format!("{value}"),
None => "null".to_string(), None => "null".to_string(),
}; };
let scroll_point_x_json = match scroll_point {
Some((x, _)) => format!("{x}"),
None => "null".to_string(),
};
let scroll_point_y_json = match scroll_point {
Some((_, y)) => format!("{y}"),
None => "null".to_string(),
};
format!( format!(
r#"{{"type":"ensure","tab_id":"{tab}","profile_id":"{profile}","url":{url},"width":{w},"height":{h},"page_zoom_percent":100,"scroll_delta_x":{dx},"scroll_delta_y":{dy},"click_x":null,"click_y":null,"hover_x":{hx},"hover_y":{hy},"typed_text":null,"site_permissions":[]}}"#, r#"{{"type":"ensure","tab_id":"{tab}","profile_id":"{profile}","url":{url},"width":{w},"height":{h},"page_zoom_percent":100,"scroll_delta_x":{dx},"scroll_delta_y":{dy},"scroll_point_x":{sx},"scroll_point_y":{sy},"click_x":null,"click_y":null,"hover_x":{hx},"hover_y":{hy},"typed_text":null,"site_permissions":[]}}"#,
tab = tab.as_str(), tab = tab.as_str(),
profile = profile_id.as_str(), profile = profile_id.as_str(),
url = serde_json::to_string(url).unwrap_or_else(|_| "\"\"".to_string()), url = serde_json::to_string(url).unwrap_or_else(|_| "\"\"".to_string()),
@@ -349,6 +362,8 @@ fn build_ensure(
h = VIEWPORT_HEIGHT, h = VIEWPORT_HEIGHT,
dx = scroll_dx, dx = scroll_dx,
dy = scroll_dy, dy = scroll_dy,
sx = scroll_point_x_json,
sy = scroll_point_y_json,
hx = hover_x_json, hx = hover_x_json,
hy = hover_y_json, hy = hover_y_json,
) )
@@ -391,12 +406,12 @@ fn read_response_with_bytes(
} }
let response: LiveResponse = serde_json::from_str(json_line.trim_end())?; let response: LiveResponse = serde_json::from_str(json_line.trim_end())?;
let mut rgba = Vec::new(); let mut rgba = Vec::new();
if let Some(frame) = response.frame.as_ref() { if let Some(frame) = response.frame.as_ref()
if frame.rgba_byte_count > 0 { && frame.rgba_byte_count > 0
{
rgba.resize(frame.rgba_byte_count, 0); rgba.resize(frame.rgba_byte_count, 0);
reader.read_exact(&mut rgba)?; reader.read_exact(&mut rgba)?;
} }
}
Ok((response, rgba)) Ok((response, rgba))
} }
+7 -1
View File
@@ -226,7 +226,13 @@ fn exercise_real_servo_webview_lifecycle() -> Result<(), Box<dyn Error>> {
} }
let previous_frame_hash = host.last_rendered_frame()?.sample_hash(); let previous_frame_hash = host.last_rendered_frame()?.sample_hash();
host.scroll(ScrollRequest { webview_id: webview_id.clone(), delta_x: 0, delta_y: 480 })?; host.scroll(ScrollRequest {
webview_id: webview_id.clone(),
delta_x: 0,
delta_y: 480,
point_x: 0,
point_y: 0,
})?;
let snapshot = wait_for_rendered_webview(&mut host, &webview_id, Some(previous_frame_hash))?; let snapshot = wait_for_rendered_webview(&mut host, &webview_id, Some(previous_frame_hash))?;
assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}"); assert_eq!(snapshot.state(), &WebViewState::Complete, "snapshot: {snapshot:?}");
assert_rendered_frame_has_content(&host, "https://servo.org scrolled", MINIMUM_CONTENT_PIXELS)?; assert_rendered_frame_has_content(&host, "https://servo.org scrolled", MINIMUM_CONTENT_PIXELS)?;