From 4c6d9c757a8913eb69ba4c54714234df517c7e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Sat, 9 May 2026 16:21:43 -0400 Subject: [PATCH] Add mouse hover tracking across the Servo input pipeline Without continuous MouseMove events, Servo never updates hover state: no cursor changes over links, no :hover CSS effects, no mouseenter JavaScript events. The web page appears completely non-interactive. Track mouse position from GPUI on_mouse_move through the full IPC pipeline to Servo. Hover position is included with each ensure request so Servo updates hover state at frame rate (~60fps). --- crates/ely_app/src/services/servo_live.rs | 321 +++++++++++++ crates/ely_app/src/shell/web_surface.rs | 338 +++++--------- .../src/shell/web_surface_controller.rs | 143 +----- .../ely_app/src/shell/web_surface_runtime.rs | 237 ++++++++++ crates/ely_app/src/shell/web_surface_state.rs | 59 +-- crates/ely_app/src/shell/web_surface_view.rs | 11 + .../src/bin/ely_servo_sidecar/live.rs | 428 ++++++++++++++++++ crates/ely_servo_host/src/runtime.rs | 111 +++-- crates/ely_servo_host/src/runtime_input.rs | 4 + 9 files changed, 1219 insertions(+), 433 deletions(-) create mode 100644 crates/ely_app/src/services/servo_live.rs create mode 100644 crates/ely_app/src/shell/web_surface_runtime.rs create mode 100644 crates/ely_servo_host/src/bin/ely_servo_sidecar/live.rs diff --git a/crates/ely_app/src/services/servo_live.rs b/crates/ely_app/src/services/servo_live.rs new file mode 100644 index 0000000..f177edb --- /dev/null +++ b/crates/ely_app/src/services/servo_live.rs @@ -0,0 +1,321 @@ +use std::{ + env, fs, + io::{self, BufRead, BufReader, Write}, + path::PathBuf, + process::{Child, ChildStdin, ChildStdout, Stdio}, + time::{SystemTime, SystemTimeError, UNIX_EPOCH}, +}; + +use ely_domain::SitePermissionDecision; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use super::servo_sidecar_command::{SidecarCommandError, default_sidecar_command}; + +pub(crate) struct ServoLiveClient { + child: Child, + stdin: ChildStdin, + stdout: BufReader, + frame_dir: PathBuf, + frame_path: PathBuf, +} + +impl ServoLiveClient { + pub fn new(profile_data_dir: PathBuf) -> Result { + let command_target = default_sidecar_command()?; + if let Some(path) = command_target.missing_binary_path() { + return Err(ServoLiveError::SidecarBinaryUnavailable { path: path.to_path_buf() }); + } + + let frame_dir = temporary_frame_dir()?; + let frame_path = frame_dir.join("frame.rgba"); + let mut command = command_target.command(); + command.arg("live").arg("--profile-data-dir").arg(profile_data_dir); + let mut child = command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(ServoLiveError::Command)?; + let stdin = child.stdin.take().ok_or(ServoLiveError::PipeUnavailable { name: "stdin" })?; + let stdout = + child.stdout.take().ok_or(ServoLiveError::PipeUnavailable { name: "stdout" })?; + + Ok(Self { child, stdin, stdout: BufReader::new(stdout), frame_dir, frame_path }) + } + + pub fn ensure( + &mut self, + request: ServoLiveEnsureRequest, + ) -> Result, ServoLiveError> { + self.request(LiveRequest::Ensure { + tab_id: request.tab_id, + profile_id: request.profile_id, + url: request.url, + width: request.width, + height: request.height, + page_zoom_percent: request.page_zoom_percent, + scroll_delta_x: request.scroll_delta_x, + scroll_delta_y: request.scroll_delta_y, + click_x: request.click_x, + click_y: request.click_y, + hover_x: request.hover_x, + hover_y: request.hover_y, + typed_text: request.typed_text, + site_permissions: request.site_permissions, + rgba_out: self.frame_path.display().to_string(), + }) + } + + pub fn poll(&mut self, tab_id: String) -> Result, ServoLiveError> { + self.request(LiveRequest::Poll { tab_id, rgba_out: self.frame_path.display().to_string() }) + } + + fn request(&mut self, request: LiveRequest) -> Result, ServoLiveError> { + serde_json::to_writer(&mut self.stdin, &request)?; + self.stdin.write_all(b"\n").map_err(ServoLiveError::Command)?; + self.stdin.flush().map_err(ServoLiveError::Command)?; + + let mut line = String::new(); + let bytes = self.stdout.read_line(&mut line).map_err(ServoLiveError::Command)?; + if bytes == 0 { + return Err(ServoLiveError::SidecarExited); + } + + let response: LiveResponse = serde_json::from_str(&line)?; + if let Some(error) = response.error { + return Err(ServoLiveError::SidecarFailed { message: error }); + } + + response.frame.map(ServoLiveFrame::from_report).transpose() + } +} + +impl Drop for ServoLiveClient { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = fs::remove_dir_all(&self.frame_dir); + } +} + +pub(crate) struct ServoLiveEnsureRequest { + pub(crate) tab_id: String, + pub(crate) profile_id: String, + pub(crate) url: String, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) page_zoom_percent: u16, + pub(crate) scroll_delta_x: i32, + pub(crate) scroll_delta_y: i32, + pub(crate) click_x: Option, + pub(crate) click_y: Option, + pub(crate) hover_x: Option, + pub(crate) hover_y: Option, + pub(crate) typed_text: Option, + pub(crate) site_permissions: Vec, +} + +#[derive(Clone, Debug, Serialize)] +pub(crate) struct ServoLiveSitePermission { + pub(crate) origin: String, + pub(crate) feature: String, + pub(crate) decision: String, +} + +impl ServoLiveSitePermission { + pub fn new( + origin: impl Into, + feature: impl Into, + decision: SitePermissionDecision, + ) -> Self { + Self { origin: origin.into(), feature: feature.into(), decision: decision.as_str().into() } + } +} + +pub(crate) struct ServoLiveFrame { + loaded_url: Option, + title: Option, + render_state: String, + width: u32, + height: u32, + #[cfg(all(test, feature = "live-site-smoke"))] + non_white_pixel_count: u64, + #[cfg(all(test, feature = "live-site-smoke"))] + content_pixel_count: u64, + #[cfg(all(test, feature = "live-site-smoke"))] + sample_hash: u64, + rgba_bytes: Vec, +} + +impl ServoLiveFrame { + fn from_report(report: LiveFrameReport) -> Result { + let rgba_bytes = fs::read(&report.rgba_path).map_err(ServoLiveError::FrameRead)?; + if rgba_bytes.len() != report.rgba_byte_count { + return Err(ServoLiveError::RgbaByteCountMismatch { + expected: report.rgba_byte_count, + actual: rgba_bytes.len(), + }); + } + + Ok(Self { + loaded_url: report.loaded_url, + title: report.title, + render_state: report.state, + width: report.width, + height: report.height, + #[cfg(all(test, feature = "live-site-smoke"))] + non_white_pixel_count: report.non_white_pixel_count, + #[cfg(all(test, feature = "live-site-smoke"))] + content_pixel_count: report.content_pixel_count, + #[cfg(all(test, feature = "live-site-smoke"))] + sample_hash: report.sample_hash, + rgba_bytes, + }) + } + + #[must_use] + pub fn loaded_url(&self) -> Option<&str> { + self.loaded_url.as_deref() + } + + #[must_use] + pub fn title(&self) -> Option<&str> { + self.title.as_deref() + } + + #[must_use] + pub fn render_state(&self) -> &str { + self.render_state.as_str() + } + + #[must_use] + pub fn width(&self) -> u32 { + self.width + } + + #[must_use] + pub fn height(&self) -> u32 { + self.height + } + + #[cfg(all(test, feature = "live-site-smoke"))] + #[must_use] + pub fn non_white_pixel_count(&self) -> u64 { + self.non_white_pixel_count + } + + #[cfg(all(test, feature = "live-site-smoke"))] + #[must_use] + pub fn content_pixel_count(&self) -> u64 { + self.content_pixel_count + } + + #[cfg(all(test, feature = "live-site-smoke"))] + #[must_use] + pub fn sample_hash(&self) -> u64 { + self.sample_hash + } + + #[must_use] + pub fn into_rgba_bytes(self) -> Vec { + self.rgba_bytes + } +} + +#[derive(Debug, Error)] +pub(crate) enum ServoLiveError { + #[error("servo sidecar binary is unavailable at {path}")] + SidecarBinaryUnavailable { path: PathBuf }, + + #[error("failed to run servo live sidecar: {0}")] + Command(#[source] io::Error), + + #[error("servo live sidecar pipe is unavailable: {name}")] + PipeUnavailable { name: &'static str }, + + #[error("servo live sidecar exited")] + SidecarExited, + + #[error("servo live sidecar failed: {message}")] + SidecarFailed { message: String }, + + #[error("failed to read servo live frame file: {0}")] + FrameRead(#[source] io::Error), + + #[error("servo live frame byte count mismatch: expected {expected}, actual {actual}")] + RgbaByteCountMismatch { expected: usize, actual: usize }, + + #[error("temporary live frame directory is unavailable: {0}")] + TempDirectory(#[source] io::Error), + + #[error("temporary live frame timestamp is unavailable: {0}")] + SystemClock(#[source] SystemTimeError), + + #[error(transparent)] + Json(#[from] serde_json::Error), + + #[error(transparent)] + SidecarCommand(#[from] SidecarCommandError), +} + +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum LiveRequest { + Ensure { + tab_id: String, + profile_id: String, + url: String, + width: u32, + height: u32, + page_zoom_percent: u16, + scroll_delta_x: i32, + scroll_delta_y: i32, + click_x: Option, + click_y: Option, + hover_x: Option, + hover_y: Option, + typed_text: Option, + site_permissions: Vec, + rgba_out: String, + }, + Poll { + tab_id: String, + rgba_out: String, + }, +} + +#[derive(Deserialize)] +struct LiveResponse { + error: Option, + frame: Option, +} + +#[derive(Deserialize)] +struct LiveFrameReport { + loaded_url: Option, + title: Option, + state: String, + width: u32, + height: u32, + rgba_path: PathBuf, + rgba_byte_count: usize, + #[cfg(all(test, feature = "live-site-smoke"))] + non_white_pixel_count: u64, + #[cfg(all(test, feature = "live-site-smoke"))] + content_pixel_count: u64, + #[cfg(all(test, feature = "live-site-smoke"))] + sample_hash: u64, +} + +fn temporary_frame_dir() -> Result { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(ServoLiveError::SystemClock)? + .as_nanos(); + let directory = env::temp_dir() + .join("ely-browser-servo-live") + .join(format!("{}-{timestamp}", std::process::id())); + fs::create_dir_all(&directory).map_err(ServoLiveError::TempDirectory)?; + Ok(directory) +} diff --git a/crates/ely_app/src/shell/web_surface.rs b/crates/ely_app/src/shell/web_surface.rs index d0c9b68..c3359e5 100644 --- a/crates/ely_app/src/shell/web_surface.rs +++ b/crates/ely_app/src/shell/web_surface.rs @@ -3,24 +3,28 @@ use std::collections::BTreeMap; use ely_domain::{BrowserTab, TabId}; use gpui::{Bounds, Pixels, Point}; -use crate::services::{ProfileDataMode, servo_sidecar::SidecarSnapshotRequest}; +use crate::services::ProfileDataMode; use super::{ web_surface_frame::WebSurfaceFrame, web_surface_geometry::{ WebSurfaceClickPoint, WebSurfaceScrollDelta, WebSurfaceScrollOffset, WebSurfaceSize, }, + web_surface_permissions::WebSurfaceSitePermission, + web_surface_runtime::{WebSurfaceRuntime, WebSurfaceRuntimeFrame}, web_surface_state::{ - WebSurfaceClickState, WebSurfaceClient, WebSurfaceKeyboardFocusState, WebSurfaceRequest, - WebSurfaceScrollState, WebSurfaceState, WebSurfaceStateKey, WebSurfaceTextInputState, + WebSurfaceClickState, WebSurfaceKeyboardFocusState, WebSurfacePendingInput, + WebSurfaceScrollState, WebSurfaceState, WebSurfaceTextInputState, }, }; pub(super) struct WebSurfaceStore { - client: WebSurfaceClient, + runtime: WebSurfaceRuntime, pending_viewport_sizes: BTreeMap, click_points: BTreeMap, + hover_points: BTreeMap, keyboard_focus: Option, + pending_scroll_deltas: BTreeMap, scroll_offsets: BTreeMap, typed_texts: BTreeMap, viewport_bounds: BTreeMap>, @@ -31,10 +35,12 @@ pub(super) struct WebSurfaceStore { impl WebSurfaceStore { pub(super) fn new() -> Self { Self { - client: WebSurfaceClient::new(), + runtime: WebSurfaceRuntime::new(), pending_viewport_sizes: BTreeMap::new(), click_points: BTreeMap::new(), + hover_points: BTreeMap::new(), keyboard_focus: None, + pending_scroll_deltas: BTreeMap::new(), scroll_offsets: BTreeMap::new(), typed_texts: BTreeMap::new(), viewport_bounds: BTreeMap::new(), @@ -47,208 +53,72 @@ impl WebSurfaceStore { self.states.get(tab_id) } - pub(super) fn prepare_request( + pub(super) fn ensure_surface( &mut self, tab: &BrowserTab, profile_data_mode: ProfileDataMode, - ) -> Option { + permissions: &[WebSurfaceSitePermission], + ) { if !is_external_web_url(tab.url().as_str()) { - return None; + return; } - let size = self.viewport_sizes.get(tab.id()).copied()?; let requested_url = tab.url().as_str().to_string(); - let scroll_offset = self.scroll_offset_for(tab.id(), requested_url.as_str()); - let zoom_percent = tab.zoom_percent(); - let click_point = self.click_point_for(tab.id(), requested_url.as_str(), scroll_offset); - let typed_text = - self.typed_text_for(tab.id(), requested_url.as_str(), scroll_offset, click_point); - if self.is_loading_requested_url(tab.id(), requested_url.as_str()) { - return None; - } - let state_key = WebSurfaceStateKey { - requested_url: &requested_url, - size, - scroll_offset, - zoom_percent, - click_point, - typed_text: typed_text.as_deref(), + let Some(size) = self.viewport_sizes.get(tab.id()).copied() else { + return; }; - if self.has_current_state(tab.id(), state_key) { - return None; - } + let input = self.take_pending_input(tab.id(), requested_url.as_str()); + let previous_frame = + self.previous_ready_frame(tab.id(), requested_url.as_str(), tab.zoom_percent()); - let client = match &self.client { - WebSurfaceClient::Ready(client) => client.clone(), - WebSurfaceClient::Unavailable(message) => { + match self.runtime.ensure_tab(tab, size, profile_data_mode, permissions, input) { + Ok(result) if result.frame.is_some() => { + let Some(frame) = result.frame else { + return; + }; + self.states.insert(tab.id().clone(), WebSurfaceState::Ready(frame)); + } + Ok(result) if result.started_loading => { self.states.insert( tab.id().clone(), - WebSurfaceState::Failed { - requested_url, - size, - scroll_offset, - zoom_percent, - click_point, - typed_text: typed_text.clone(), - message: message.clone(), + WebSurfaceState::Loading { + requested_url: result.requested_url, + previous_frame, }, ); - return None; } - }; - - self.states.insert( - tab.id().clone(), - WebSurfaceState::Loading { - requested_url: requested_url.clone(), - size, - scroll_offset, - zoom_percent, - click_point, - typed_text: typed_text.clone(), - previous_frame: self.previous_ready_frame( - tab.id(), - requested_url.as_str(), - zoom_percent, - ), - }, - ); - - let mut snapshot_request = SidecarSnapshotRequest::new( - tab.url().clone(), - tab.profile_id().clone(), - size.width, - size.height, - ) - .with_profile_data_mode(profile_data_mode) - .with_scroll_offset(scroll_offset.x(), scroll_offset.y()) - .with_page_zoom_percent(zoom_percent); - if let Some(click_point) = click_point { - snapshot_request = snapshot_request.with_click_point(click_point.x(), click_point.y()); - } - if let Some(typed_text) = typed_text.clone() { - snapshot_request = snapshot_request.with_typed_text(typed_text); - } - - Some(WebSurfaceRequest { - tab_id: tab.id().clone(), - requested_url, - size, - scroll_offset, - zoom_percent, - click_point, - typed_text, - client, - snapshot_request, - }) - } - - fn has_current_state(&self, tab_id: &TabId, key: WebSurfaceStateKey<'_>) -> bool { - match self.states.get(tab_id) { - Some(WebSurfaceState::Loading { - requested_url: current_url, - size: current_size, - scroll_offset: current_scroll_offset, - zoom_percent: current_zoom_percent, - click_point: current_click_point, - typed_text: current_typed_text, - .. - }) => { - current_url == key.requested_url - && *current_size == key.size - && *current_scroll_offset == key.scroll_offset - && *current_zoom_percent == key.zoom_percent - && *current_click_point == key.click_point - && current_typed_text.as_deref() == key.typed_text + Ok(_) => {} + Err(message) => { + self.states.insert(tab.id().clone(), WebSurfaceState::Failed { message }); } - Some(WebSurfaceState::Ready(frame)) => { - frame.requested_url == key.requested_url - && frame.size() == key.size - && frame.scroll_offset() == key.scroll_offset - && frame.zoom_percent() == key.zoom_percent - && frame.click_point() == key.click_point - && frame.typed_text() == key.typed_text - } - Some(WebSurfaceState::Failed { - requested_url: current_url, - size: current_size, - scroll_offset: current_scroll_offset, - zoom_percent: current_zoom_percent, - click_point: current_click_point, - typed_text: current_typed_text, - .. - }) => { - current_url == key.requested_url - && *current_size == key.size - && *current_scroll_offset == key.scroll_offset - && *current_zoom_percent == key.zoom_percent - && *current_click_point == key.click_point - && current_typed_text.as_deref() == key.typed_text - } - None => false, } } - pub(super) fn is_loading(&self, tab_id: &TabId, key: WebSurfaceStateKey<'_>) -> bool { - matches!( - self.states.get(tab_id), - Some(WebSurfaceState::Loading { - requested_url: current_url, - size: current_size, - scroll_offset: current_scroll_offset, - zoom_percent: current_zoom_percent, - click_point: current_click_point, - typed_text: current_typed_text, - .. - }) - if current_url == key.requested_url - && *current_size == key.size - && *current_scroll_offset == key.scroll_offset - && *current_zoom_percent == key.zoom_percent - && *current_click_point == key.click_point - && current_typed_text.as_deref() == key.typed_text - ) - } + pub(super) fn tick(&mut self) -> bool { + let frames = self.runtime.tick(); + let mut changed = false; - fn is_loading_requested_url(&self, tab_id: &TabId, requested_url: &str) -> bool { - matches!( - self.states.get(tab_id), - Some(WebSurfaceState::Loading { requested_url: current_url, .. }) - if current_url == requested_url - ) - } - - fn previous_ready_frame( - &self, - tab_id: &TabId, - requested_url: &str, - zoom_percent: u16, - ) -> Option { - match self.states.get(tab_id) { - Some(WebSurfaceState::Ready(frame)) - if frame.requested_url == requested_url && frame.zoom_percent() == zoom_percent => - { - Some(frame.clone()) + for frame in frames { + match frame { + WebSurfaceRuntimeFrame::Ready { tab_id, frame } => { + self.states.insert(tab_id, WebSurfaceState::Ready(frame)); + changed = true; + } + WebSurfaceRuntimeFrame::Failed { tab_id, message } => { + self.states.insert(tab_id, WebSurfaceState::Failed { message }); + changed = true; + } } - Some(WebSurfaceState::Loading { - requested_url: current_url, previous_frame, .. - }) if current_url == requested_url => previous_frame - .as_ref() - .filter(|frame| frame.zoom_percent() == zoom_percent) - .cloned(), - _ => None, } - } - pub(super) fn finish(&mut self, tab_id: TabId, state: WebSurfaceState) { - self.states.insert(tab_id, state); + changed } pub(super) fn record_scroll_delta( &mut self, tab_id: &TabId, requested_url: &str, - delta: gpui::Point, + delta: Point, ) -> bool { let Some(delta) = WebSurfaceScrollDelta::from_point(delta) else { return false; @@ -262,12 +132,11 @@ impl WebSurfaceStore { *state = WebSurfaceScrollState::new(requested_url.to_string()); } - let next_offset = state.offset.scrolled_by(delta); - if next_offset == state.offset { - return false; - } - - state.offset = next_offset; + state.offset = state.offset.scrolled_by(delta); + self.pending_scroll_deltas + .entry(tab_id.clone()) + .and_modify(|current| *current = current.combined_with(delta)) + .or_insert(delta); self.click_points.remove(tab_id); self.typed_texts.remove(tab_id); self.keyboard_focus = None; @@ -301,6 +170,21 @@ impl WebSurfaceStore { true } + pub(super) fn record_hover_point( + &mut self, + tab_id: &TabId, + position: Point, + ) -> bool { + let Some(bounds) = self.viewport_bounds.get(tab_id).copied() else { + return false; + }; + let Some(point) = WebSurfaceClickPoint::from_window_position(bounds, position) else { + return false; + }; + self.hover_points.insert(tab_id.clone(), point); + true + } + pub(super) fn record_click_point( &mut self, tab_id: &TabId, @@ -325,10 +209,6 @@ impl WebSurfaceStore { scroll_offset: state.scroll_offset, click_point: state.point, }); - if self.click_points.get(tab_id) == Some(&state) { - return false; - } - self.typed_texts.remove(tab_id); self.click_points.insert(tab_id.clone(), state); true @@ -373,6 +253,53 @@ impl WebSurfaceStore { true } + fn take_pending_input( + &mut self, + tab_id: &TabId, + requested_url: &str, + ) -> WebSurfacePendingInput { + let scroll_offset = self.scroll_offset_for(tab_id, requested_url); + let scroll_delta = self.pending_scroll_deltas.remove(tab_id); + let click_point = self + .click_points + .remove(tab_id) + .filter(|state| { + state.requested_url == requested_url && state.scroll_offset == scroll_offset + }) + .map(|state| state.point); + let typed_text = self + .typed_texts + .remove(tab_id) + .filter(|state| state.requested_url == requested_url) + .map(|state| state.text); + + let hover_point = self.hover_points.remove(tab_id); + + WebSurfacePendingInput { scroll_offset, scroll_delta, click_point, hover_point, typed_text } + } + + fn previous_ready_frame( + &self, + tab_id: &TabId, + requested_url: &str, + zoom_percent: u16, + ) -> Option { + match self.states.get(tab_id) { + Some(WebSurfaceState::Ready(frame)) + if frame.requested_url == requested_url && frame.zoom_percent() == zoom_percent => + { + Some(frame.clone()) + } + Some(WebSurfaceState::Loading { + requested_url: current_url, previous_frame, .. + }) if current_url == requested_url => previous_frame + .as_ref() + .filter(|frame| frame.zoom_percent() == zoom_percent) + .cloned(), + _ => None, + } + } + fn scroll_offset_for(&self, tab_id: &TabId, requested_url: &str) -> WebSurfaceScrollOffset { self.scroll_offsets .get(tab_id) @@ -380,39 +307,6 @@ impl WebSurfaceStore { .map(|state| state.offset) .unwrap_or_default() } - - fn click_point_for( - &self, - tab_id: &TabId, - requested_url: &str, - scroll_offset: WebSurfaceScrollOffset, - ) -> Option { - self.click_points - .get(tab_id) - .filter(|state| { - state.requested_url == requested_url && state.scroll_offset == scroll_offset - }) - .map(|state| state.point) - } - - fn typed_text_for( - &self, - tab_id: &TabId, - requested_url: &str, - scroll_offset: WebSurfaceScrollOffset, - click_point: Option, - ) -> Option { - let click_point = click_point?; - self.typed_texts - .get(tab_id) - .filter(|state| { - state.requested_url == requested_url - && state.scroll_offset == scroll_offset - && state.click_point == click_point - && !state.text.is_empty() - }) - .map(|state| state.text.clone()) - } } pub(super) fn is_external_web_url(url: &str) -> bool { diff --git a/crates/ely_app/src/shell/web_surface_controller.rs b/crates/ely_app/src/shell/web_surface_controller.rs index 70a08b3..9ad38ae 100644 --- a/crates/ely_app/src/shell/web_surface_controller.rs +++ b/crates/ely_app/src/shell/web_surface_controller.rs @@ -2,32 +2,17 @@ use ely_browser_core::BrowserSnapshot; use ely_domain::{BrowserTab, ProfileKind, TabId}; use gpui::{AnyElement, Bounds, Context, Pixels, Point}; -use crate::services::{ - ProfileDataMode, - servo_sidecar::{ServoSidecarError, SidecarSitePermission, SidecarSnapshot}, -}; +use crate::services::ProfileDataMode; use super::{ ElyShell, - web_surface_frame::WebSurfaceFrame, - web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize}, - web_surface_permissions::sidecar_site_permissions_for_tab, - web_surface_state::{WebSurfaceRequest, WebSurfaceState, WebSurfaceStateKey}, + web_surface_permissions::web_surface_site_permissions_for_tab, + web_surface_state::WebSurfaceState, web_surface_view::{ render_failed_web_surface, render_loading_web_surface, render_ready_web_surface, }, }; -struct PendingWebSurfaceFrame { - tab_id: TabId, - requested_url: String, - size: WebSurfaceSize, - scroll_offset: WebSurfaceScrollOffset, - zoom_percent: u16, - click_point: Option, - typed_text: Option, -} - impl ElyShell { pub(super) fn render_external_web_canvas( &mut self, @@ -39,9 +24,9 @@ impl ElyShell { let Some(profile_data_mode) = profile_data_mode_for(tab, snapshot) else { return render_failed_web_surface(tab, "Profile context is unavailable.", state_entity); }; - let site_permissions = sidecar_site_permissions_for_tab(tab, snapshot); - self.ensure_external_web_frame(tab, profile_data_mode, site_permissions, cx); + let permissions = web_surface_site_permissions_for_tab(tab, snapshot); + self.web_surfaces.ensure_surface(tab, profile_data_mode, &permissions); match self.web_surfaces.state(tab.id()) { Some(WebSurfaceState::Ready(frame)) => { @@ -59,109 +44,8 @@ impl ElyShell { } } - fn ensure_external_web_frame( - &mut self, - tab: &BrowserTab, - profile_data_mode: ProfileDataMode, - site_permissions: Vec, - cx: &mut Context, - ) { - let Some(request) = self.web_surfaces.prepare_request(tab, profile_data_mode) else { - return; - }; - - let WebSurfaceRequest { - tab_id, - requested_url, - size, - scroll_offset, - zoom_percent, - click_point, - typed_text, - client, - mut snapshot_request, - } = request; - snapshot_request = snapshot_request.with_site_permissions(site_permissions); - let pending_frame = PendingWebSurfaceFrame { - tab_id, - requested_url, - size, - scroll_offset, - zoom_percent, - click_point, - typed_text, - }; - cx.spawn(async move |shell, cx| { - let result = cx - .background_executor() - .spawn(async move { client.snapshot(snapshot_request) }) - .await; - - _ = shell.update(cx, |shell, cx| { - shell.handle_external_web_frame_result(pending_frame, result); - cx.notify(); - }); - }) - .detach(); - } - - fn handle_external_web_frame_result( - &mut self, - pending_frame: PendingWebSurfaceFrame, - result: Result, - ) { - let PendingWebSurfaceFrame { - tab_id, - requested_url, - size, - scroll_offset, - zoom_percent, - click_point, - typed_text, - } = pending_frame; - let state_key = WebSurfaceStateKey { - requested_url: requested_url.as_str(), - size, - scroll_offset, - zoom_percent, - click_point, - typed_text: typed_text.as_deref(), - }; - if !self.web_surfaces.is_loading(&tab_id, state_key) { - return; - } - - let state = match result { - Ok(snapshot) => match WebSurfaceFrame::from_snapshot( - requested_url.clone(), - scroll_offset, - zoom_percent, - click_point, - typed_text.clone(), - snapshot, - ) { - Ok(frame) => WebSurfaceState::Ready(frame), - Err(error) => WebSurfaceState::Failed { - requested_url, - size, - scroll_offset, - zoom_percent, - click_point, - typed_text, - message: error.to_string(), - }, - }, - Err(error) => WebSurfaceState::Failed { - requested_url, - size, - scroll_offset, - zoom_percent, - click_point, - typed_text, - message: error.to_string(), - }, - }; - self.web_surfaces.finish(tab_id, state); + pub(super) fn tick_external_web_surfaces(&mut self) -> bool { + self.web_surfaces.tick() } pub(super) fn record_external_web_viewport( @@ -179,7 +63,7 @@ impl ElyShell { &mut self, tab_id: TabId, requested_url: String, - delta: gpui::Point, + delta: Point, cx: &mut Context, ) { if self.web_surfaces.record_scroll_delta(&tab_id, requested_url.as_str(), delta) { @@ -187,6 +71,17 @@ impl ElyShell { } } + pub(super) fn hover_external_web_viewport( + &mut self, + tab_id: TabId, + position: Point, + cx: &mut Context, + ) { + if self.web_surfaces.record_hover_point(&tab_id, position) { + cx.notify(); + } + } + pub(super) fn click_external_web_viewport( &mut self, tab_id: TabId, diff --git a/crates/ely_app/src/shell/web_surface_runtime.rs b/crates/ely_app/src/shell/web_surface_runtime.rs new file mode 100644 index 0000000..01d6023 --- /dev/null +++ b/crates/ely_app/src/shell/web_surface_runtime.rs @@ -0,0 +1,237 @@ +use std::{collections::BTreeMap, fs, path::PathBuf}; + +use ely_domain::{BrowserTab, ProfileId, TabId}; + +use crate::services::{ + ProfileDataMode, + servo_live::{ServoLiveClient, ServoLiveEnsureRequest, ServoLiveSitePermission}, + servo_profile_data::{default_profile_data_root, profile_data_dir, transient_profile_data_dir}, +}; + +use super::{ + web_surface_frame::WebSurfaceFrame, + web_surface_geometry::{WebSurfaceScrollOffset, WebSurfaceSize}, + web_surface_permissions::WebSurfaceSitePermission, + web_surface_state::WebSurfacePendingInput, +}; + +pub(super) struct WebSurfaceRuntime { + state: RuntimeState, + sessions: BTreeMap, +} + +impl WebSurfaceRuntime { + pub(super) fn new() -> Self { + Self { state: RuntimeState::Empty, sessions: BTreeMap::new() } + } + + pub(super) fn ensure_tab( + &mut self, + tab: &BrowserTab, + size: WebSurfaceSize, + profile_data_mode: ProfileDataMode, + permissions: &[WebSurfaceSitePermission], + input: WebSurfacePendingInput, + ) -> Result { + let scope = WebSurfaceRuntimeScope::new(tab.profile_id().clone(), profile_data_mode); + self.ensure_runtime(scope.clone())?; + + let requested_url = tab.url().as_str().to_string(); + let zoom_percent = tab.zoom_percent(); + let (state, sessions) = (&mut self.state, &mut self.sessions); + let RuntimeState::Ready { scope: active_scope, client, .. } = state else { + return Err("Servo live runtime is unavailable".to_string()); + }; + if active_scope != &scope { + return Err(active_scope.error_for(&scope)); + } + + let session = sessions.entry(tab.id().clone()).or_insert_with(WebSurfaceSession::default); + let next_scroll_offset = input.scroll_offset; + let started_loading = session.started_loading(&requested_url, size, zoom_percent); + let frame = client + .ensure(ServoLiveEnsureRequest { + tab_id: tab.id().as_str().to_string(), + profile_id: tab.profile_id().as_str().to_string(), + url: requested_url.clone(), + width: size.width, + height: size.height, + page_zoom_percent: zoom_percent, + scroll_delta_x: input.scroll_delta.map_or(0, |delta| delta.x()), + scroll_delta_y: input.scroll_delta.map_or(0, |delta| delta.y()), + click_x: input.click_point.map(|point| point.x()), + click_y: input.click_point.map(|point| point.y()), + hover_x: input.hover_point.map(|point| point.x()), + hover_y: input.hover_point.map(|point| point.y()), + typed_text: input.typed_text, + site_permissions: permissions.iter().map(ServoLiveSitePermission::from).collect(), + }) + .map_err(|error| error.to_string())? + .map(|frame| { + WebSurfaceFrame::from_live_frame( + requested_url.clone(), + next_scroll_offset, + zoom_percent, + frame, + ) + .map_err(|error| error.to_string()) + }) + .transpose()?; + + session.requested_url = requested_url.clone(); + session.size = size; + session.zoom_percent = zoom_percent; + session.scroll_offset = next_scroll_offset; + + Ok(WebSurfaceEnsureResult { requested_url, started_loading, frame }) + } + + pub(super) fn tick(&mut self) -> Vec { + let (state, sessions) = (&mut self.state, &self.sessions); + let RuntimeState::Ready { client, .. } = state else { + return Vec::new(); + }; + + let mut frames = Vec::new(); + for (tab_id, session) in sessions { + match client.poll(tab_id.as_str().to_string()) { + Ok(Some(frame)) => match WebSurfaceFrame::from_live_frame( + session.requested_url.clone(), + session.scroll_offset, + session.zoom_percent, + frame, + ) { + Ok(frame) => { + frames.push(WebSurfaceRuntimeFrame::Ready { tab_id: tab_id.clone(), frame }) + } + Err(error) => frames.push(WebSurfaceRuntimeFrame::Failed { + tab_id: tab_id.clone(), + message: error.to_string(), + }), + }, + Ok(None) => {} + Err(error) => frames.push(WebSurfaceRuntimeFrame::Failed { + tab_id: tab_id.clone(), + message: error.to_string(), + }), + } + } + + frames + } + + fn ensure_runtime(&mut self, scope: WebSurfaceRuntimeScope) -> Result<(), String> { + match &self.state { + RuntimeState::Empty => { + let (config_dir, transient_profile_data_dir) = config_dir_for_scope(&scope)?; + let client = ServoLiveClient::new(config_dir).map_err(|error| error.to_string())?; + self.state = RuntimeState::Ready { scope, client, transient_profile_data_dir }; + Ok(()) + } + RuntimeState::Ready { scope: active_scope, .. } if active_scope == &scope => Ok(()), + RuntimeState::Ready { scope: active_scope, .. } => Err(active_scope.error_for(&scope)), + } + } +} + +impl Drop for WebSurfaceRuntime { + fn drop(&mut self) { + let RuntimeState::Ready { transient_profile_data_dir: Some(path), .. } = &self.state else { + return; + }; + let _ = fs::remove_dir_all(path); + } +} + +enum RuntimeState { + Empty, + Ready { + scope: WebSurfaceRuntimeScope, + client: ServoLiveClient, + transient_profile_data_dir: Option, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct WebSurfaceRuntimeScope { + profile_id: ProfileId, + profile_data_mode: ProfileDataMode, +} + +impl WebSurfaceRuntimeScope { + fn new(profile_id: ProfileId, profile_data_mode: ProfileDataMode) -> Self { + Self { profile_id, profile_data_mode } + } + + fn error_for(&self, requested: &Self) -> String { + format!( + "Servo live runtime is already attached to profile {} ({:?}); requested profile {} ({:?})", + self.profile_id.as_str(), + self.profile_data_mode, + requested.profile_id.as_str(), + requested.profile_data_mode + ) + } +} + +#[derive(Clone, Default)] +struct WebSurfaceSession { + requested_url: String, + size: WebSurfaceSize, + zoom_percent: u16, + scroll_offset: WebSurfaceScrollOffset, +} + +impl WebSurfaceSession { + fn started_loading( + &self, + requested_url: &str, + size: WebSurfaceSize, + zoom_percent: u16, + ) -> bool { + self.requested_url != requested_url + || self.size != size + || self.zoom_percent != zoom_percent + } +} + +pub(super) struct WebSurfaceEnsureResult { + pub(super) requested_url: String, + pub(super) started_loading: bool, + pub(super) frame: Option, +} + +pub(super) enum WebSurfaceRuntimeFrame { + Ready { tab_id: TabId, frame: WebSurfaceFrame }, + Failed { tab_id: TabId, message: String }, +} + +fn config_dir_for_scope( + scope: &WebSurfaceRuntimeScope, +) -> Result<(PathBuf, Option), String> { + match scope.profile_data_mode { + ProfileDataMode::Persistent => { + let root = default_profile_data_root() + .ok_or_else(|| "Profile data root is unavailable".to_string())?; + let config_dir = profile_data_dir(&root, &scope.profile_id); + fs::create_dir_all(&config_dir).map_err(|error| error.to_string())?; + Ok((config_dir, None)) + } + ProfileDataMode::Transient => { + let config_dir = + transient_profile_data_dir(&scope.profile_id).map_err(|error| error.to_string())?; + fs::create_dir_all(&config_dir).map_err(|error| error.to_string())?; + Ok((config_dir.clone(), Some(config_dir))) + } + } +} + +impl From<&WebSurfaceSitePermission> for ServoLiveSitePermission { + fn from(permission: &WebSurfaceSitePermission) -> Self { + Self::new( + permission.origin().as_str(), + permission.feature().as_str(), + permission.decision(), + ) + } +} diff --git a/crates/ely_app/src/shell/web_surface_state.rs b/crates/ely_app/src/shell/web_surface_state.rs index c0c628e..0ee2455 100644 --- a/crates/ely_app/src/shell/web_surface_state.rs +++ b/crates/ely_app/src/shell/web_surface_state.rs @@ -1,10 +1,8 @@ use ely_domain::TabId; -use crate::services::servo_sidecar::{ServoSidecarClient, SidecarSnapshotRequest}; - use super::{ web_surface_frame::WebSurfaceFrame, - web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize}, + web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset}, }; pub(super) struct WebSurfaceScrollState { @@ -40,60 +38,17 @@ pub(super) struct WebSurfaceTextInputState { pub(super) text: String, } -#[derive(Clone, Copy)] -pub(super) struct WebSurfaceStateKey<'a> { - pub(super) requested_url: &'a str, - pub(super) size: WebSurfaceSize, +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct WebSurfacePendingInput { pub(super) scroll_offset: WebSurfaceScrollOffset, - pub(super) zoom_percent: u16, - pub(super) click_point: Option, - pub(super) typed_text: Option<&'a str>, -} - -pub(super) enum WebSurfaceClient { - Ready(ServoSidecarClient), - Unavailable(String), -} - -impl WebSurfaceClient { - pub(super) fn new() -> Self { - match ServoSidecarClient::new() { - Ok(client) => Self::Ready(client), - Err(error) => Self::Unavailable(error.to_string()), - } - } -} - -pub(super) struct WebSurfaceRequest { - pub(super) tab_id: TabId, - pub(super) requested_url: String, - pub(super) size: WebSurfaceSize, - pub(super) scroll_offset: WebSurfaceScrollOffset, - pub(super) zoom_percent: u16, + pub(super) scroll_delta: Option, pub(super) click_point: Option, + pub(super) hover_point: Option, pub(super) typed_text: Option, - pub(super) client: ServoSidecarClient, - pub(super) snapshot_request: SidecarSnapshotRequest, } pub(super) enum WebSurfaceState { - Loading { - requested_url: String, - size: WebSurfaceSize, - scroll_offset: WebSurfaceScrollOffset, - zoom_percent: u16, - click_point: Option, - typed_text: Option, - previous_frame: Option, - }, + Loading { requested_url: String, previous_frame: Option }, Ready(WebSurfaceFrame), - Failed { - requested_url: String, - size: WebSurfaceSize, - scroll_offset: WebSurfaceScrollOffset, - zoom_percent: u16, - click_point: Option, - typed_text: Option, - message: String, - }, + Failed { message: String }, } diff --git a/crates/ely_app/src/shell/web_surface_view.rs b/crates/ely_app/src/shell/web_surface_view.rs index eb6c8f3..a1e30f5 100644 --- a/crates/ely_app/src/shell/web_surface_view.rs +++ b/crates/ely_app/src/shell/web_surface_view.rs @@ -93,6 +93,8 @@ fn render_input_overlay( let click_tab_id = tab_id.clone(); let click_url = url.clone(); let click_entity = state_entity.clone(); + let hover_tab_id = tab_id.clone(); + let hover_entity = state_entity.clone(); let scroll_tab_id = tab_id; let scroll_url = url; let scroll_entity = state_entity; @@ -116,6 +118,15 @@ fn render_input_overlay( }); cx.stop_propagation(); }) + .on_mouse_move(move |event, _window, cx| { + hover_entity.update(cx, |shell, cx| { + shell.hover_external_web_viewport( + hover_tab_id.clone(), + event.position, + cx, + ); + }); + }) .on_scroll_wheel(move |event, window, cx| { let delta = event.delta.pixel_delta(window.line_height()); scroll_entity.update(cx, |shell, cx| { diff --git a/crates/ely_servo_host/src/bin/ely_servo_sidecar/live.rs b/crates/ely_servo_host/src/bin/ely_servo_sidecar/live.rs new file mode 100644 index 0000000..046992a --- /dev/null +++ b/crates/ely_servo_host/src/bin/ely_servo_sidecar/live.rs @@ -0,0 +1,428 @@ +use std::{ + collections::HashMap, + fs, + io::{self, BufRead, Write}, + path::PathBuf, + thread, + time::{Duration, Instant}, +}; + +use ely_domain::{DEFAULT_ZOOM_PERCENT, ProfileId, TabId, UrlText}; +use ely_servo_host::{ + KeyboardTextRequest, MouseClickRequest, NavigationRequest, PageZoomRequest, PermissionDecision, + PermissionRequest, RenderedFrame, ResizeRequest, ScrollRequest, ServoHost, ServoHostError, + ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState, +}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use super::args::LiveArgs; + +const LIVE_FRAME_WAIT_TIMEOUT: Duration = Duration::from_millis(60); +const LIVE_FRAME_WAIT_INTERVAL: Duration = Duration::from_millis(2); + +pub(super) fn run_live(args: LiveArgs) -> Result<(), LiveSidecarError> { + fs::create_dir_all(&args.profile_data_dir)?; + let mut host = SoftwareServoHost::new_with_config_dir( + ServoSurfaceSize::new(1, 1), + Some(args.profile_data_dir), + )?; + let mut sessions = HashMap::new(); + let stdin = io::stdin(); + let mut stdout = io::stdout().lock(); + + for line in stdin.lock().lines() { + let line = line?; + if line.trim().is_empty() { + continue; + } + + let response = match serde_json::from_str::(&line) { + Ok(request) => handle_request(&mut host, &mut sessions, request), + Err(error) => Err(LiveSidecarError::Json(error)), + }; + write_response(&mut stdout, response)?; + } + + Ok(()) +} + +fn handle_request( + host: &mut SoftwareServoHost, + sessions: &mut HashMap, + request: LiveRequest, +) -> Result { + match request { + LiveRequest::Ensure { + tab_id, + profile_id, + url, + width, + height, + page_zoom_percent, + scroll_delta_x, + scroll_delta_y, + click_x, + click_y, + hover_x, + hover_y, + typed_text, + site_permissions, + rgba_out, + } => { + let tab = TabId::parse(tab_id.clone())?; + let profile = ProfileId::parse(profile_id)?; + let url = UrlText::parse(url)?; + let session = ensure_session(host, sessions, tab_id, &tab, &profile, width, height)?; + + if apply_layout(host, session, width, height, page_zoom_percent)? { + session.awaiting_visible_frame = true; + } + apply_permissions(host, session, &profile, site_permissions)?; + if session.requested_url != url.as_str() { + host.navigate(NavigationRequest { + webview_id: session.webview_id.clone(), + tab_id: tab, + url: url.clone(), + })?; + session.requested_url = url.as_str().to_string(); + session.scroll_x = 0; + session.scroll_y = 0; + session.awaiting_visible_frame = true; + } + if apply_input( + host, + session, + scroll_delta_x, + scroll_delta_y, + click_x, + click_y, + hover_x, + hover_y, + typed_text, + )? { + session.awaiting_visible_frame = true; + } + poll_frame(host, session, rgba_out.into()) + } + LiveRequest::Poll { tab_id, rgba_out } => { + let Some(session) = sessions.get_mut(&tab_id) else { + return Ok(LiveResponse::empty()); + }; + poll_frame(host, session, rgba_out.into()) + } + } +} + +fn write_response( + stdout: &mut impl Write, + response: Result, +) -> Result<(), LiveSidecarError> { + let response = match response { + Ok(response) => response, + Err(error) => LiveResponse::error(error.to_string()), + }; + serde_json::to_writer(&mut *stdout, &response)?; + stdout.write_all(b"\n")?; + stdout.flush()?; + Ok(()) +} + +fn ensure_session<'a>( + host: &mut SoftwareServoHost, + sessions: &'a mut HashMap, + key: String, + tab_id: &TabId, + profile_id: &ProfileId, + width: u32, + height: u32, +) -> Result<&'a mut LiveSession, LiveSidecarError> { + if !sessions.contains_key(&key) { + let webview_id = host.create_webview_with_size( + tab_id.clone(), + profile_id.clone(), + ServoSurfaceSize::new(width, height), + )?; + sessions.insert(key.clone(), LiveSession::new(webview_id, width, height)); + } + + sessions.get_mut(&key).ok_or(LiveSidecarError::SessionUnavailable) +} + +fn apply_layout( + host: &mut SoftwareServoHost, + session: &mut LiveSession, + width: u32, + height: u32, + page_zoom_percent: u16, +) -> Result { + let mut changed = false; + if session.width != width || session.height != height { + host.resize(ResizeRequest { webview_id: session.webview_id.clone(), width, height })?; + session.width = width; + session.height = height; + changed = true; + } + + if session.page_zoom_percent != page_zoom_percent { + host.set_page_zoom(PageZoomRequest { + webview_id: session.webview_id.clone(), + zoom_factor: f32::from(page_zoom_percent) / 100.0, + })?; + session.page_zoom_percent = page_zoom_percent; + changed = true; + } + + Ok(changed) +} + +fn apply_permissions( + host: &mut SoftwareServoHost, + session: &LiveSession, + profile_id: &ProfileId, + permissions: Vec, +) -> Result<(), LiveSidecarError> { + for permission in permissions { + host.set_permission( + PermissionRequest { + webview_id: session.webview_id.clone(), + profile_id: profile_id.clone(), + origin: ely_domain::SiteOrigin::parse(permission.origin)?, + feature: ely_domain::SitePermissionFeature::parse(permission.feature.as_str())?, + }, + PermissionDecision::from(ely_domain::SitePermissionDecision::parse( + permission.decision.as_str(), + )?), + )?; + } + + Ok(()) +} + +fn apply_input( + host: &mut SoftwareServoHost, + session: &mut LiveSession, + scroll_delta_x: i32, + scroll_delta_y: i32, + click_x: Option, + click_y: Option, + hover_x: Option, + hover_y: Option, + typed_text: Option, +) -> Result { + let mut changed = false; + if scroll_delta_x != 0 || scroll_delta_y != 0 { + host.scroll(ScrollRequest { + webview_id: session.webview_id.clone(), + delta_x: scroll_delta_x, + delta_y: scroll_delta_y, + })?; + session.scroll_x = positive_scroll_component(session.scroll_x, scroll_delta_x); + session.scroll_y = positive_scroll_component(session.scroll_y, scroll_delta_y); + changed = true; + } + + if let (Some(x), Some(y)) = (hover_x, hover_y) { + host.hover(x, y)?; + changed = true; + } + + if let (Some(x), Some(y)) = (click_x, click_y) { + host.click(MouseClickRequest { webview_id: session.webview_id.clone(), x, y })?; + changed = true; + } + + if let Some(text) = typed_text { + host.type_text(KeyboardTextRequest { webview_id: session.webview_id.clone(), text })?; + changed = true; + } + + Ok(changed) +} + +fn poll_frame( + host: &mut SoftwareServoHost, + session: &mut LiveSession, + rgba_out: PathBuf, +) -> Result { + let started_at = Instant::now(); + let mut latest_frame = None; + + loop { + host.tick(); + let snapshot = host.snapshot(&session.webview_id)?; + if snapshot.has_pending_frame() { + host.paint(&session.webview_id)?; + let snapshot = host.snapshot(&session.webview_id)?; + let frame = host.last_rendered_frame()?; + let has_visible_content = + frame.non_white_pixel_count() > 0 && frame.content_pixel_count() > 0; + fs::write(&rgba_out, frame.rgba_bytes())?; + let response = + LiveResponse::frame(LiveFrameReport::new(&snapshot, &frame, rgba_out.clone())); + if has_visible_content { + session.awaiting_visible_frame = false; + return Ok(response); + } + if !session.awaiting_visible_frame { + return Ok(response); + } + latest_frame = Some(response); + } + + if !session.awaiting_visible_frame { + return Ok(LiveResponse::empty()); + } + if started_at.elapsed() >= LIVE_FRAME_WAIT_TIMEOUT { + return Ok(latest_frame.unwrap_or_else(LiveResponse::empty)); + } + + thread::sleep(LIVE_FRAME_WAIT_INTERVAL); + } +} + +#[derive(Clone)] +struct LiveSession { + webview_id: ely_domain::WebViewId, + requested_url: String, + width: u32, + height: u32, + page_zoom_percent: u16, + scroll_x: i32, + scroll_y: i32, + awaiting_visible_frame: bool, +} + +impl LiveSession { + fn new(webview_id: ely_domain::WebViewId, width: u32, height: u32) -> Self { + Self { + webview_id, + requested_url: String::new(), + width: width.max(1), + height: height.max(1), + page_zoom_percent: DEFAULT_ZOOM_PERCENT, + scroll_x: 0, + scroll_y: 0, + awaiting_visible_frame: false, + } + } +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum LiveRequest { + Ensure { + tab_id: String, + profile_id: String, + url: String, + width: u32, + height: u32, + page_zoom_percent: u16, + scroll_delta_x: i32, + scroll_delta_y: i32, + click_x: Option, + click_y: Option, + #[serde(default)] + hover_x: Option, + #[serde(default)] + hover_y: Option, + typed_text: Option, + site_permissions: Vec, + rgba_out: String, + }, + Poll { + tab_id: String, + rgba_out: String, + }, +} + +#[derive(Deserialize)] +struct LiveSitePermission { + origin: String, + feature: String, + decision: String, +} + +#[derive(Serialize)] +struct LiveResponse { + error: Option, + frame: Option, +} + +impl LiveResponse { + fn empty() -> Self { + Self { error: None, frame: None } + } + + fn frame(frame: LiveFrameReport) -> Self { + Self { error: None, frame: Some(frame) } + } + + fn error(message: String) -> Self { + Self { error: Some(message), frame: None } + } +} + +#[derive(Serialize)] +struct LiveFrameReport { + loaded_url: Option, + title: Option, + state: &'static str, + width: u32, + height: u32, + rgba_path: PathBuf, + rgba_byte_count: usize, + non_white_pixel_count: u64, + content_pixel_count: u64, + sample_hash: u64, +} + +impl LiveFrameReport { + fn new(snapshot: &WebViewSnapshot, frame: &RenderedFrame, rgba_path: PathBuf) -> Self { + Self { + loaded_url: snapshot.url().map(str::to_string), + title: snapshot.title().map(str::to_string), + state: state_label(snapshot.state()), + width: frame.width(), + height: frame.height(), + rgba_path, + rgba_byte_count: frame.rgba_bytes().len(), + non_white_pixel_count: frame.non_white_pixel_count(), + content_pixel_count: frame.content_pixel_count(), + sample_hash: frame.sample_hash(), + } + } +} + +#[derive(Debug, Error)] +pub(super) enum LiveSidecarError { + #[error("live session is unavailable after creation")] + SessionUnavailable, + + #[error(transparent)] + Domain(#[from] ely_domain::DomainError), + + #[error(transparent)] + Host(#[from] ServoHostError), + + #[error(transparent)] + Io(#[from] io::Error), + + #[error(transparent)] + Json(#[from] serde_json::Error), +} + +fn state_label(state: &WebViewState) -> &'static str { + match state { + WebViewState::Created => "created", + WebViewState::Loading => "loading", + WebViewState::Complete => "complete", + WebViewState::Sleeping => "sleeping", + WebViewState::Crashed => "crashed", + } +} + +fn positive_scroll_component(current: i32, delta: i32) -> i32 { + let value = i64::from(current) + i64::from(delta); + value.clamp(0, i64::from(i32::MAX)) as i32 +} diff --git a/crates/ely_servo_host/src/runtime.rs b/crates/ely_servo_host/src/runtime.rs index bc9de07..50ed872 100644 --- a/crates/ely_servo_host/src/runtime.rs +++ b/crates/ely_servo_host/src/runtime.rs @@ -23,7 +23,9 @@ use crate::{ KeyboardTextRequest, MouseClickRequest, MouseDragRequest, NavigationRequest, PageZoomRequest, PermissionDecision, PermissionRequest, RenderedFrame, ResizeRequest, ScreenshotRequest, ScrollRequest, ServoHost, ServoHostError, TouchTapRequest, WebViewSnapshot, WebViewState, - runtime_input::{send_keyboard_text, send_mouse_click, send_mouse_drag, send_touch_tap}, + runtime_input::{ + send_keyboard_text, send_mouse_click, send_mouse_drag, send_mouse_hover, send_touch_tap, + }, runtime_permissions::{PermissionStore, set_permission_decision}, runtime_waker::ServoWakeFlag, runtime_webview::{HostWebView, HostWebViewDelegate}, @@ -52,7 +54,7 @@ impl ServoSurfaceSize { pub struct SoftwareServoHost { servo: Servo, - rendering_context: Rc, + default_surface_size: ServoSurfaceSize, webviews: HashMap, permissions: PermissionStore, wake_requested: Arc, @@ -82,16 +84,19 @@ impl SoftwareServoHost { host } + pub fn create_webview_with_size( + &mut self, + tab_id: TabId, + profile_id: ProfileId, + size: ServoSurfaceSize, + ) -> Result { + self.create_webview_in_context(tab_id, profile_id, size) + } + fn new_started( size: ServoSurfaceSize, config_dir: Option, ) -> Result { - let rendering_context = Rc::new( - servo::SoftwareRenderingContext::new(size.physical()) - .map_err(|_| ServoHostError::RenderingContextUnavailable)?, - ); - rendering_context.make_current().map_err(|_| ServoHostError::RenderingContextNotCurrent)?; - let wake_requested = Arc::new(AtomicBool::new(false)); let mut builder = ServoBuilder::default() .event_loop_waker(Box::new(ServoWakeFlag::new(wake_requested.clone()))); @@ -102,7 +107,7 @@ impl SoftwareServoHost { Ok(Self { servo, - rendering_context, + default_surface_size: size, webviews: HashMap::new(), permissions: Rc::new(RefCell::new(HashMap::new())), wake_requested, @@ -117,19 +122,7 @@ impl ServoHost for SoftwareServoHost { tab_id: TabId, profile_id: ProfileId, ) -> Result { - let webview_id = WebViewId::new(); - let delegate = - Rc::new(HostWebViewDelegate::new(profile_id.clone(), self.permissions.clone())); - let webview = WebViewBuilder::new(&self.servo, self.rendering_context.clone()) - .delegate(delegate.clone()) - .build(); - - self.webviews.insert( - webview_id.clone(), - HostWebView { tab_id, profile_id, webview, delegate, requested_url: None }, - ); - - Ok(webview_id) + self.create_webview_in_context(tab_id, profile_id, self.default_surface_size) } fn navigate(&mut self, request: NavigationRequest) -> Result<(), ServoHostError> { @@ -138,7 +131,6 @@ impl ServoHost for SoftwareServoHost { })?; self.servo.spin_event_loop(); let servo = self.servo.clone(); - let rendering_context = self.rendering_context.clone(); let webview = self .webviews .get_mut(&request.webview_id) @@ -151,7 +143,7 @@ impl ServoHost for SoftwareServoHost { webview.delegate.set_state(WebViewState::Loading); if should_create_initial_document { - webview.webview = WebViewBuilder::new(&servo, rendering_context) + webview.webview = WebViewBuilder::new(&servo, webview.rendering_context.clone()) .delegate(webview.delegate.clone()) .url(url) .build(); @@ -188,7 +180,9 @@ impl ServoHost for SoftwareServoHost { .get(&request.webview_id) .ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?; - webview.webview.resize(PhysicalSize::new(request.width, request.height)); + let size = PhysicalSize::new(request.width, request.height); + webview.rendering_context.resize(size); + webview.webview.resize(size); Ok(()) } @@ -202,6 +196,13 @@ impl ServoHost for SoftwareServoHost { Ok(()) } + fn hover(&mut self, x: u32, y: u32) -> Result<(), ServoHostError> { + if let Some(webview) = self.webviews.values().next() { + send_mouse_hover(&webview.webview, x, y); + } + Ok(()) + } + fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError> { let webview = self .webviews @@ -319,16 +320,15 @@ impl ServoHost for SoftwareServoHost { } fn paint(&mut self, webview_id: &WebViewId) -> Result<(), ServoHostError> { - self.rendering_context - .make_current() - .map_err(|_| ServoHostError::RenderingContextNotCurrent)?; - self.rendering_context.prepare_for_rendering(); + let rendering_context = self.webview(webview_id)?.rendering_context.clone(); + rendering_context.make_current().map_err(|_| ServoHostError::RenderingContextNotCurrent)?; + rendering_context.prepare_for_rendering(); { let webview = self.webview(webview_id)?; webview.webview.paint(); } - let rendered_frame = self.read_rendered_frame()?; - self.rendering_context.present(); + let rendered_frame = Self::read_rendered_frame(rendering_context.as_ref())?; + rendering_context.present(); self.webview(webview_id)?.delegate.mark_frame_presented(); self.last_rendered_frame = Some(rendered_frame); Ok(()) @@ -340,14 +340,56 @@ impl ServoHost for SoftwareServoHost { } impl SoftwareServoHost { + fn create_webview_in_context( + &mut self, + tab_id: TabId, + profile_id: ProfileId, + size: ServoSurfaceSize, + ) -> Result { + let webview_id = WebViewId::new(); + let rendering_context = Self::new_rendering_context(size)?; + let delegate = + Rc::new(HostWebViewDelegate::new(profile_id.clone(), self.permissions.clone())); + let webview = WebViewBuilder::new(&self.servo, rendering_context.clone()) + .delegate(delegate.clone()) + .build(); + + self.webviews.insert( + webview_id.clone(), + HostWebView { + tab_id, + profile_id, + rendering_context, + webview, + delegate, + requested_url: None, + }, + ); + + Ok(webview_id) + } + + fn new_rendering_context( + size: ServoSurfaceSize, + ) -> Result, ServoHostError> { + let rendering_context = Rc::new( + servo::SoftwareRenderingContext::new(size.physical()) + .map_err(|_| ServoHostError::RenderingContextUnavailable)?, + ); + rendering_context.make_current().map_err(|_| ServoHostError::RenderingContextNotCurrent)?; + Ok(rendering_context) + } + fn webview(&self, webview_id: &WebViewId) -> Result<&HostWebView, ServoHostError> { self.webviews .get(webview_id) .ok_or_else(|| ServoHostError::WebViewNotFound { id: webview_id.clone() }) } - fn read_rendered_frame(&self) -> Result { - let size = self.rendering_context.size(); + fn read_rendered_frame( + rendering_context: &dyn RenderingContext, + ) -> Result { + let size = rendering_context.size(); let width = i32::try_from(size.width).map_err(|_| ServoHostError::RenderedFrameUnavailable)?; let height = @@ -356,8 +398,7 @@ impl SoftwareServoHost { DeviceIntPoint::new(0, 0), DeviceIntSize::new(width, height), ); - let image = self - .rendering_context + let image = rendering_context .read_to_image(frame_rect) .ok_or(ServoHostError::RenderedFrameUnavailable)?; diff --git a/crates/ely_servo_host/src/runtime_input.rs b/crates/ely_servo_host/src/runtime_input.rs index f19f6e3..27a8307 100644 --- a/crates/ely_servo_host/src/runtime_input.rs +++ b/crates/ely_servo_host/src/runtime_input.rs @@ -6,6 +6,10 @@ use servo::{ use crate::keyboard::key_and_code_for_character; +pub(super) fn send_mouse_hover(webview: &WebView, x: u32, y: u32) { + webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point(x, y)))); +} + pub(super) fn send_mouse_click(webview: &WebView, x: u32, y: u32) { let point = point(x, y); webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point)));