ZacharyZhang-NY/ELY-Browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add mouse hover tracking across the Servo input pipeline

Browse Source 
Without continuous MouseMove events, Servo never updates hover state:
no cursor changes over links, no :hover CSS effects, no mouseenter
JavaScript events. The web page appears completely non-interactive.

Track mouse position from GPUI on_mouse_move through the full IPC
pipeline to Servo. Hover position is included with each ensure
request so Servo updates hover state at frame rate (~60fps).
This commit is contained in:
ZacharyZhang-NY
2026-05-09 16:21:43 -04:00
parent e621c43757
commit 4c6d9c757a
9 changed files with 1219 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ely_app/src/services/servo_live.rs
+321
View File
@@ -0,0 +1,321 @@
use std::{
env, fs,
io::{self, BufRead, BufReader, Write},
path::PathBuf,
process::{Child, ChildStdin, ChildStdout, Stdio},
time::{SystemTime, SystemTimeError, UNIX_EPOCH},
};
use ely_domain::SitePermissionDecision;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use super::servo_sidecar_command::{SidecarCommandError, default_sidecar_command};
pub(crate) struct ServoLiveClient {
child: Child,
stdin: ChildStdin,
stdout: BufReader<ChildStdout>,
frame_dir: PathBuf,
frame_path: PathBuf,
}
impl ServoLiveClient {
pub fn new(profile_data_dir: PathBuf) -> Result<Self, ServoLiveError> {
let command_target = default_sidecar_command()?;
if let Some(path) = command_target.missing_binary_path() {
return Err(ServoLiveError::SidecarBinaryUnavailable { path: path.to_path_buf() });
}
let frame_dir = temporary_frame_dir()?;
let frame_path = frame_dir.join("frame.rgba");
let mut command = command_target.command();
command.arg("live").arg("--profile-data-dir").arg(profile_data_dir);
let mut child = command
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.map_err(ServoLiveError::Command)?;
let stdin = child.stdin.take().ok_or(ServoLiveError::PipeUnavailable { name: "stdin" })?;
let stdout =
child.stdout.take().ok_or(ServoLiveError::PipeUnavailable { name: "stdout" })?;
Ok(Self { child, stdin, stdout: BufReader::new(stdout), frame_dir, frame_path })
}
pub fn ensure(
&mut self,
request: ServoLiveEnsureRequest,
) -> Result<Option<ServoLiveFrame>, ServoLiveError> {
self.request(LiveRequest::Ensure {
tab_id: request.tab_id,
profile_id: request.profile_id,
url: request.url,
width: request.width,
height: request.height,
page_zoom_percent: request.page_zoom_percent,
scroll_delta_x: request.scroll_delta_x,
scroll_delta_y: request.scroll_delta_y,
click_x: request.click_x,
click_y: request.click_y,
hover_x: request.hover_x,
hover_y: request.hover_y,
typed_text: request.typed_text,
site_permissions: request.site_permissions,
rgba_out: self.frame_path.display().to_string(),
})
}
pub fn poll(&mut self, tab_id: String) -> Result<Option<ServoLiveFrame>, ServoLiveError> {
self.request(LiveRequest::Poll { tab_id, rgba_out: self.frame_path.display().to_string() })
}
fn request(&mut self, request: LiveRequest) -> Result<Option<ServoLiveFrame>, ServoLiveError> {
serde_json::to_writer(&mut self.stdin, &request)?;
self.stdin.write_all(b"\n").map_err(ServoLiveError::Command)?;
self.stdin.flush().map_err(ServoLiveError::Command)?;
let mut line = String::new();
let bytes = self.stdout.read_line(&mut line).map_err(ServoLiveError::Command)?;
if bytes == 0 {
return Err(ServoLiveError::SidecarExited);
}
let response: LiveResponse = serde_json::from_str(&line)?;
if let Some(error) = response.error {
return Err(ServoLiveError::SidecarFailed { message: error });
}
response.frame.map(ServoLiveFrame::from_report).transpose()
}
}
impl Drop for ServoLiveClient {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
let _ = fs::remove_dir_all(&self.frame_dir);
}
}
pub(crate) struct ServoLiveEnsureRequest {
pub(crate) tab_id: String,
pub(crate) profile_id: String,
pub(crate) url: String,
pub(crate) width: u32,
pub(crate) height: u32,
pub(crate) page_zoom_percent: u16,
pub(crate) scroll_delta_x: i32,
pub(crate) scroll_delta_y: i32,
pub(crate) click_x: Option<u32>,
pub(crate) click_y: Option<u32>,
pub(crate) hover_x: Option<u32>,
pub(crate) hover_y: Option<u32>,
pub(crate) typed_text: Option<String>,
pub(crate) site_permissions: Vec<ServoLiveSitePermission>,
}
#[derive(Clone, Debug, Serialize)]
pub(crate) struct ServoLiveSitePermission {
pub(crate) origin: String,
pub(crate) feature: String,
pub(crate) decision: String,
}
impl ServoLiveSitePermission {
pub fn new(
origin: impl Into<String>,
feature: impl Into<String>,
decision: SitePermissionDecision,
) -> Self {
Self { origin: origin.into(), feature: feature.into(), decision: decision.as_str().into() }
}
}
pub(crate) struct ServoLiveFrame {
loaded_url: Option<String>,
title: Option<String>,
render_state: String,
width: u32,
height: u32,
#[cfg(all(test, feature = "live-site-smoke"))]
non_white_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
content_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: u64,
rgba_bytes: Vec<u8>,
}
impl ServoLiveFrame {
fn from_report(report: LiveFrameReport) -> Result<Self, ServoLiveError> {
let rgba_bytes = fs::read(&report.rgba_path).map_err(ServoLiveError::FrameRead)?;
if rgba_bytes.len() != report.rgba_byte_count {
return Err(ServoLiveError::RgbaByteCountMismatch {
expected: report.rgba_byte_count,
actual: rgba_bytes.len(),
});
}
Ok(Self {
loaded_url: report.loaded_url,
title: report.title,
render_state: report.state,
width: report.width,
height: report.height,
#[cfg(all(test, feature = "live-site-smoke"))]
non_white_pixel_count: report.non_white_pixel_count,
#[cfg(all(test, feature = "live-site-smoke"))]
content_pixel_count: report.content_pixel_count,
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: report.sample_hash,
rgba_bytes,
})
}
#[must_use]
pub fn loaded_url(&self) -> Option<&str> {
self.loaded_url.as_deref()
}
#[must_use]
pub fn title(&self) -> Option<&str> {
self.title.as_deref()
}
#[must_use]
pub fn render_state(&self) -> &str {
self.render_state.as_str()
}
#[must_use]
pub fn width(&self) -> u32 {
self.width
}
#[must_use]
pub fn height(&self) -> u32 {
self.height
}
#[cfg(all(test, feature = "live-site-smoke"))]
#[must_use]
pub fn non_white_pixel_count(&self) -> u64 {
self.non_white_pixel_count
}
#[cfg(all(test, feature = "live-site-smoke"))]
#[must_use]
pub fn content_pixel_count(&self) -> u64 {
self.content_pixel_count
}
#[cfg(all(test, feature = "live-site-smoke"))]
#[must_use]
pub fn sample_hash(&self) -> u64 {
self.sample_hash
}
#[must_use]
pub fn into_rgba_bytes(self) -> Vec<u8> {
self.rgba_bytes
}
}
#[derive(Debug, Error)]
pub(crate) enum ServoLiveError {
#[error("servo sidecar binary is unavailable at {path}")]
SidecarBinaryUnavailable { path: PathBuf },
#[error("failed to run servo live sidecar: {0}")]
Command(#[source] io::Error),
#[error("servo live sidecar pipe is unavailable: {name}")]
PipeUnavailable { name: &'static str },
#[error("servo live sidecar exited")]
SidecarExited,
#[error("servo live sidecar failed: {message}")]
SidecarFailed { message: String },
#[error("failed to read servo live frame file: {0}")]
FrameRead(#[source] io::Error),
#[error("servo live frame byte count mismatch: expected {expected}, actual {actual}")]
RgbaByteCountMismatch { expected: usize, actual: usize },
#[error("temporary live frame directory is unavailable: {0}")]
TempDirectory(#[source] io::Error),
#[error("temporary live frame timestamp is unavailable: {0}")]
SystemClock(#[source] SystemTimeError),
#[error(transparent)]
Json(#[from] serde_json::Error),
#[error(transparent)]
SidecarCommand(#[from] SidecarCommandError),
}
#[derive(Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
enum LiveRequest {
Ensure {
tab_id: String,
profile_id: String,
url: String,
width: u32,
height: u32,
page_zoom_percent: u16,
scroll_delta_x: i32,
scroll_delta_y: i32,
click_x: Option<u32>,
click_y: Option<u32>,
hover_x: Option<u32>,
hover_y: Option<u32>,
typed_text: Option<String>,
site_permissions: Vec<ServoLiveSitePermission>,
rgba_out: String,
},
Poll {
tab_id: String,
rgba_out: String,
},
}
#[derive(Deserialize)]
struct LiveResponse {
error: Option<String>,
frame: Option<LiveFrameReport>,
}
#[derive(Deserialize)]
struct LiveFrameReport {
loaded_url: Option<String>,
title: Option<String>,
state: String,
width: u32,
height: u32,
rgba_path: PathBuf,
rgba_byte_count: usize,
#[cfg(all(test, feature = "live-site-smoke"))]
non_white_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
content_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: u64,
}
fn temporary_frame_dir() -> Result<PathBuf, ServoLiveError> {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(ServoLiveError::SystemClock)?
.as_nanos();
let directory = env::temp_dir()
.join("ely-browser-servo-live")
.join(format!("{}-{timestamp}", std::process::id()));
fs::create_dir_all(&directory).map_err(ServoLiveError::TempDirectory)?;
Ok(directory)
}
crates/ely_app/src/shell/web_surface.rs
+116 -222
View File
@@ -3,24 +3,28 @@ use std::collections::BTreeMap;
use ely_domain::{BrowserTab, TabId};
use gpui::{Bounds, Pixels, Point};
use crate::services::{ProfileDataMode, servo_sidecar::SidecarSnapshotRequest};
use crate::services::ProfileDataMode;
use super::{
web_surface_frame::WebSurfaceFrame,
web_surface_geometry::{
WebSurfaceClickPoint, WebSurfaceScrollDelta, WebSurfaceScrollOffset, WebSurfaceSize,
},
web_surface_permissions::WebSurfaceSitePermission,
web_surface_runtime::{WebSurfaceRuntime, WebSurfaceRuntimeFrame},
web_surface_state::{
WebSurfaceClickState, WebSurfaceClient, WebSurfaceKeyboardFocusState, WebSurfaceRequest,
WebSurfaceScrollState, WebSurfaceState, WebSurfaceStateKey, WebSurfaceTextInputState,
WebSurfaceClickState, WebSurfaceKeyboardFocusState, WebSurfacePendingInput,
WebSurfaceScrollState, WebSurfaceState, WebSurfaceTextInputState,
},
};
pub(super) struct WebSurfaceStore {
client: WebSurfaceClient,
runtime: WebSurfaceRuntime,
pending_viewport_sizes: BTreeMap<TabId, WebSurfaceSize>,
click_points: BTreeMap<TabId, WebSurfaceClickState>,
hover_points: BTreeMap<TabId, WebSurfaceClickPoint>,
keyboard_focus: Option<WebSurfaceKeyboardFocusState>,
pending_scroll_deltas: BTreeMap<TabId, WebSurfaceScrollDelta>,
scroll_offsets: BTreeMap<TabId, WebSurfaceScrollState>,
typed_texts: BTreeMap<TabId, WebSurfaceTextInputState>,
viewport_bounds: BTreeMap<TabId, Bounds<Pixels>>,
@@ -31,10 +35,12 @@ pub(super) struct WebSurfaceStore {
impl WebSurfaceStore {
pub(super) fn new() -> Self {
Self {
client: WebSurfaceClient::new(),
runtime: WebSurfaceRuntime::new(),
pending_viewport_sizes: BTreeMap::new(),
click_points: BTreeMap::new(),
hover_points: BTreeMap::new(),
keyboard_focus: None,
pending_scroll_deltas: BTreeMap::new(),
scroll_offsets: BTreeMap::new(),
typed_texts: BTreeMap::new(),
viewport_bounds: BTreeMap::new(),
@@ -47,208 +53,72 @@ impl WebSurfaceStore {
self.states.get(tab_id)
}
pub(super) fn prepare_request(
pub(super) fn ensure_surface(
&mut self,
tab: &BrowserTab,
profile_data_mode: ProfileDataMode,
) -> Option<WebSurfaceRequest> {
permissions: &[WebSurfaceSitePermission],
) {
if !is_external_web_url(tab.url().as_str()) {
return None;
return;
}
let size = self.viewport_sizes.get(tab.id()).copied()?;
let requested_url = tab.url().as_str().to_string();
let scroll_offset = self.scroll_offset_for(tab.id(), requested_url.as_str());
let zoom_percent = tab.zoom_percent();
let click_point = self.click_point_for(tab.id(), requested_url.as_str(), scroll_offset);
let typed_text =
self.typed_text_for(tab.id(), requested_url.as_str(), scroll_offset, click_point);
if self.is_loading_requested_url(tab.id(), requested_url.as_str()) {
return None;
}
let state_key = WebSurfaceStateKey {
requested_url: &requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text: typed_text.as_deref(),
let Some(size) = self.viewport_sizes.get(tab.id()).copied() else {
return;
};
if self.has_current_state(tab.id(), state_key) {
return None;
}
let input = self.take_pending_input(tab.id(), requested_url.as_str());
let previous_frame =
self.previous_ready_frame(tab.id(), requested_url.as_str(), tab.zoom_percent());
let client = match &self.client {
WebSurfaceClient::Ready(client) => client.clone(),
WebSurfaceClient::Unavailable(message) => {
match self.runtime.ensure_tab(tab, size, profile_data_mode, permissions, input) {
Ok(result) if result.frame.is_some() => {
let Some(frame) = result.frame else {
return;
};
self.states.insert(tab.id().clone(), WebSurfaceState::Ready(frame));
}
Ok(result) if result.started_loading => {
self.states.insert(
tab.id().clone(),
WebSurfaceState::Failed {
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text: typed_text.clone(),
message: message.clone(),
WebSurfaceState::Loading {
requested_url: result.requested_url,
previous_frame,
},
);
return None;
}
};
self.states.insert(
tab.id().clone(),
WebSurfaceState::Loading {
requested_url: requested_url.clone(),
size,
scroll_offset,
zoom_percent,
click_point,
typed_text: typed_text.clone(),
previous_frame: self.previous_ready_frame(
tab.id(),
requested_url.as_str(),
zoom_percent,
),
},
);
let mut snapshot_request = SidecarSnapshotRequest::new(
tab.url().clone(),
tab.profile_id().clone(),
size.width,
size.height,
)
.with_profile_data_mode(profile_data_mode)
.with_scroll_offset(scroll_offset.x(), scroll_offset.y())
.with_page_zoom_percent(zoom_percent);
if let Some(click_point) = click_point {
snapshot_request = snapshot_request.with_click_point(click_point.x(), click_point.y());
}
if let Some(typed_text) = typed_text.clone() {
snapshot_request = snapshot_request.with_typed_text(typed_text);
}
Some(WebSurfaceRequest {
tab_id: tab.id().clone(),
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
client,
snapshot_request,
})
}
fn has_current_state(&self, tab_id: &TabId, key: WebSurfaceStateKey<'_>) -> bool {
match self.states.get(tab_id) {
Some(WebSurfaceState::Loading {
requested_url: current_url,
size: current_size,
scroll_offset: current_scroll_offset,
zoom_percent: current_zoom_percent,
click_point: current_click_point,
typed_text: current_typed_text,
..
}) => {
current_url == key.requested_url
&& *current_size == key.size
&& *current_scroll_offset == key.scroll_offset
&& *current_zoom_percent == key.zoom_percent
&& *current_click_point == key.click_point
&& current_typed_text.as_deref() == key.typed_text
Ok(_) => {}
Err(message) => {
self.states.insert(tab.id().clone(), WebSurfaceState::Failed { message });
}
Some(WebSurfaceState::Ready(frame)) => {
frame.requested_url == key.requested_url
&& frame.size() == key.size
&& frame.scroll_offset() == key.scroll_offset
&& frame.zoom_percent() == key.zoom_percent
&& frame.click_point() == key.click_point
&& frame.typed_text() == key.typed_text
}
Some(WebSurfaceState::Failed {
requested_url: current_url,
size: current_size,
scroll_offset: current_scroll_offset,
zoom_percent: current_zoom_percent,
click_point: current_click_point,
typed_text: current_typed_text,
..
}) => {
current_url == key.requested_url
&& *current_size == key.size
&& *current_scroll_offset == key.scroll_offset
&& *current_zoom_percent == key.zoom_percent
&& *current_click_point == key.click_point
&& current_typed_text.as_deref() == key.typed_text
}
None => false,
}
}
pub(super) fn is_loading(&self, tab_id: &TabId, key: WebSurfaceStateKey<'_>) -> bool {
matches!(
self.states.get(tab_id),
Some(WebSurfaceState::Loading {
requested_url: current_url,
size: current_size,
scroll_offset: current_scroll_offset,
zoom_percent: current_zoom_percent,
click_point: current_click_point,
typed_text: current_typed_text,
..
})
if current_url == key.requested_url
&& *current_size == key.size
&& *current_scroll_offset == key.scroll_offset
&& *current_zoom_percent == key.zoom_percent
&& *current_click_point == key.click_point
&& current_typed_text.as_deref() == key.typed_text
)
}
pub(super) fn tick(&mut self) -> bool {
let frames = self.runtime.tick();
let mut changed = false;
fn is_loading_requested_url(&self, tab_id: &TabId, requested_url: &str) -> bool {
matches!(
self.states.get(tab_id),
Some(WebSurfaceState::Loading { requested_url: current_url, .. })
if current_url == requested_url
)
}
fn previous_ready_frame(
&self,
tab_id: &TabId,
requested_url: &str,
zoom_percent: u16,
) -> Option<WebSurfaceFrame> {
match self.states.get(tab_id) {
Some(WebSurfaceState::Ready(frame))
if frame.requested_url == requested_url && frame.zoom_percent() == zoom_percent =>
{
Some(frame.clone())
for frame in frames {
match frame {
WebSurfaceRuntimeFrame::Ready { tab_id, frame } => {
self.states.insert(tab_id, WebSurfaceState::Ready(frame));
changed = true;
}
WebSurfaceRuntimeFrame::Failed { tab_id, message } => {
self.states.insert(tab_id, WebSurfaceState::Failed { message });
changed = true;
}
}
Some(WebSurfaceState::Loading {
requested_url: current_url, previous_frame, ..
}) if current_url == requested_url => previous_frame
.as_ref()
.filter(|frame| frame.zoom_percent() == zoom_percent)
.cloned(),
_ => None,
}
}
pub(super) fn finish(&mut self, tab_id: TabId, state: WebSurfaceState) {
self.states.insert(tab_id, state);
changed
}
pub(super) fn record_scroll_delta(
&mut self,
tab_id: &TabId,
requested_url: &str,
delta: gpui::Point<Pixels>,
delta: Point<Pixels>,
) -> bool {
let Some(delta) = WebSurfaceScrollDelta::from_point(delta) else {
return false;
@@ -262,12 +132,11 @@ impl WebSurfaceStore {
*state = WebSurfaceScrollState::new(requested_url.to_string());
}
let next_offset = state.offset.scrolled_by(delta);
if next_offset == state.offset {
return false;
}
state.offset = next_offset;
state.offset = state.offset.scrolled_by(delta);
self.pending_scroll_deltas
.entry(tab_id.clone())
.and_modify(|current| *current = current.combined_with(delta))
.or_insert(delta);
self.click_points.remove(tab_id);
self.typed_texts.remove(tab_id);
self.keyboard_focus = None;
@@ -301,6 +170,21 @@ impl WebSurfaceStore {
true
}
pub(super) fn record_hover_point(
&mut self,
tab_id: &TabId,
position: Point<Pixels>,
) -> bool {
let Some(bounds) = self.viewport_bounds.get(tab_id).copied() else {
return false;
};
let Some(point) = WebSurfaceClickPoint::from_window_position(bounds, position) else {
return false;
};
self.hover_points.insert(tab_id.clone(), point);
true
}
pub(super) fn record_click_point(
&mut self,
tab_id: &TabId,
@@ -325,10 +209,6 @@ impl WebSurfaceStore {
scroll_offset: state.scroll_offset,
click_point: state.point,
});
if self.click_points.get(tab_id) == Some(&state) {
return false;
}
self.typed_texts.remove(tab_id);
self.click_points.insert(tab_id.clone(), state);
true
@@ -373,6 +253,53 @@ impl WebSurfaceStore {
true
}
fn take_pending_input(
&mut self,
tab_id: &TabId,
requested_url: &str,
) -> WebSurfacePendingInput {
let scroll_offset = self.scroll_offset_for(tab_id, requested_url);
let scroll_delta = self.pending_scroll_deltas.remove(tab_id);
let click_point = self
.click_points
.remove(tab_id)
.filter(|state| {
state.requested_url == requested_url && state.scroll_offset == scroll_offset
})
.map(|state| state.point);
let typed_text = self
.typed_texts
.remove(tab_id)
.filter(|state| state.requested_url == requested_url)
.map(|state| state.text);
let hover_point = self.hover_points.remove(tab_id);
WebSurfacePendingInput { scroll_offset, scroll_delta, click_point, hover_point, typed_text }
}
fn previous_ready_frame(
&self,
tab_id: &TabId,
requested_url: &str,
zoom_percent: u16,
) -> Option<WebSurfaceFrame> {
match self.states.get(tab_id) {
Some(WebSurfaceState::Ready(frame))
if frame.requested_url == requested_url && frame.zoom_percent() == zoom_percent =>
{
Some(frame.clone())
}
Some(WebSurfaceState::Loading {
requested_url: current_url, previous_frame, ..
}) if current_url == requested_url => previous_frame
.as_ref()
.filter(|frame| frame.zoom_percent() == zoom_percent)
.cloned(),
_ => None,
}
}
fn scroll_offset_for(&self, tab_id: &TabId, requested_url: &str) -> WebSurfaceScrollOffset {
self.scroll_offsets
.get(tab_id)
@@ -380,39 +307,6 @@ impl WebSurfaceStore {
.map(|state| state.offset)
.unwrap_or_default()
}
fn click_point_for(
&self,
tab_id: &TabId,
requested_url: &str,
scroll_offset: WebSurfaceScrollOffset,
) -> Option<WebSurfaceClickPoint> {
self.click_points
.get(tab_id)
.filter(|state| {
state.requested_url == requested_url && state.scroll_offset == scroll_offset
})
.map(|state| state.point)
}
fn typed_text_for(
&self,
tab_id: &TabId,
requested_url: &str,
scroll_offset: WebSurfaceScrollOffset,
click_point: Option<WebSurfaceClickPoint>,
) -> Option<String> {
let click_point = click_point?;
self.typed_texts
.get(tab_id)
.filter(|state| {
state.requested_url == requested_url
&& state.scroll_offset == scroll_offset
&& state.click_point == click_point
&& !state.text.is_empty()
})
.map(|state| state.text.clone())
}
}
pub(super) fn is_external_web_url(url: &str) -> bool {
crates/ely_app/src/shell/web_surface_controller.rs
+19 -124
View File
@@ -2,32 +2,17 @@ use ely_browser_core::BrowserSnapshot;
use ely_domain::{BrowserTab, ProfileKind, TabId};
use gpui::{AnyElement, Bounds, Context, Pixels, Point};
use crate::services::{
ProfileDataMode,
servo_sidecar::{ServoSidecarError, SidecarSitePermission, SidecarSnapshot},
};
use crate::services::ProfileDataMode;
use super::{
ElyShell,
web_surface_frame::WebSurfaceFrame,
web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize},
web_surface_permissions::sidecar_site_permissions_for_tab,
web_surface_state::{WebSurfaceRequest, WebSurfaceState, WebSurfaceStateKey},
web_surface_permissions::web_surface_site_permissions_for_tab,
web_surface_state::WebSurfaceState,
web_surface_view::{
render_failed_web_surface, render_loading_web_surface, render_ready_web_surface,
},
};
struct PendingWebSurfaceFrame {
tab_id: TabId,
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
zoom_percent: u16,
click_point: Option<WebSurfaceClickPoint>,
typed_text: Option<String>,
}
impl ElyShell {
pub(super) fn render_external_web_canvas(
&mut self,
@@ -39,9 +24,9 @@ impl ElyShell {
let Some(profile_data_mode) = profile_data_mode_for(tab, snapshot) else {
return render_failed_web_surface(tab, "Profile context is unavailable.", state_entity);
};
let site_permissions = sidecar_site_permissions_for_tab(tab, snapshot);
self.ensure_external_web_frame(tab, profile_data_mode, site_permissions, cx);
let permissions = web_surface_site_permissions_for_tab(tab, snapshot);
self.web_surfaces.ensure_surface(tab, profile_data_mode, &permissions);
match self.web_surfaces.state(tab.id()) {
Some(WebSurfaceState::Ready(frame)) => {
@@ -59,109 +44,8 @@ impl ElyShell {
}
}
fn ensure_external_web_frame(
&mut self,
tab: &BrowserTab,
profile_data_mode: ProfileDataMode,
site_permissions: Vec<SidecarSitePermission>,
cx: &mut Context<Self>,
) {
let Some(request) = self.web_surfaces.prepare_request(tab, profile_data_mode) else {
return;
};
let WebSurfaceRequest {
tab_id,
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
client,
mut snapshot_request,
} = request;
snapshot_request = snapshot_request.with_site_permissions(site_permissions);
let pending_frame = PendingWebSurfaceFrame {
tab_id,
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
};
cx.spawn(async move |shell, cx| {
let result = cx
.background_executor()
.spawn(async move { client.snapshot(snapshot_request) })
.await;
_ = shell.update(cx, |shell, cx| {
shell.handle_external_web_frame_result(pending_frame, result);
cx.notify();
});
})
.detach();
}
fn handle_external_web_frame_result(
&mut self,
pending_frame: PendingWebSurfaceFrame,
result: Result<SidecarSnapshot, ServoSidecarError>,
) {
let PendingWebSurfaceFrame {
tab_id,
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
} = pending_frame;
let state_key = WebSurfaceStateKey {
requested_url: requested_url.as_str(),
size,
scroll_offset,
zoom_percent,
click_point,
typed_text: typed_text.as_deref(),
};
if !self.web_surfaces.is_loading(&tab_id, state_key) {
return;
}
let state = match result {
Ok(snapshot) => match WebSurfaceFrame::from_snapshot(
requested_url.clone(),
scroll_offset,
zoom_percent,
click_point,
typed_text.clone(),
snapshot,
) {
Ok(frame) => WebSurfaceState::Ready(frame),
Err(error) => WebSurfaceState::Failed {
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
message: error.to_string(),
},
},
Err(error) => WebSurfaceState::Failed {
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
message: error.to_string(),
},
};
self.web_surfaces.finish(tab_id, state);
pub(super) fn tick_external_web_surfaces(&mut self) -> bool {
self.web_surfaces.tick()
}
pub(super) fn record_external_web_viewport(
@@ -179,7 +63,7 @@ impl ElyShell {
&mut self,
tab_id: TabId,
requested_url: String,
delta: gpui::Point<Pixels>,
delta: Point<Pixels>,
cx: &mut Context<Self>,
) {
if self.web_surfaces.record_scroll_delta(&tab_id, requested_url.as_str(), delta) {
@@ -187,6 +71,17 @@ impl ElyShell {
}
}
pub(super) fn hover_external_web_viewport(
&mut self,
tab_id: TabId,
position: Point<Pixels>,
cx: &mut Context<Self>,
) {
if self.web_surfaces.record_hover_point(&tab_id, position) {
cx.notify();
}
}
pub(super) fn click_external_web_viewport(
&mut self,
tab_id: TabId,
crates/ely_app/src/shell/web_surface_runtime.rs
+237
View File
@@ -0,0 +1,237 @@
use std::{collections::BTreeMap, fs, path::PathBuf};
use ely_domain::{BrowserTab, ProfileId, TabId};
use crate::services::{
ProfileDataMode,
servo_live::{ServoLiveClient, ServoLiveEnsureRequest, ServoLiveSitePermission},
servo_profile_data::{default_profile_data_root, profile_data_dir, transient_profile_data_dir},
};
use super::{
web_surface_frame::WebSurfaceFrame,
web_surface_geometry::{WebSurfaceScrollOffset, WebSurfaceSize},
web_surface_permissions::WebSurfaceSitePermission,
web_surface_state::WebSurfacePendingInput,
};
pub(super) struct WebSurfaceRuntime {
state: RuntimeState,
sessions: BTreeMap<TabId, WebSurfaceSession>,
}
impl WebSurfaceRuntime {
pub(super) fn new() -> Self {
Self { state: RuntimeState::Empty, sessions: BTreeMap::new() }
}
pub(super) fn ensure_tab(
&mut self,
tab: &BrowserTab,
size: WebSurfaceSize,
profile_data_mode: ProfileDataMode,
permissions: &[WebSurfaceSitePermission],
input: WebSurfacePendingInput,
) -> Result<WebSurfaceEnsureResult, String> {
let scope = WebSurfaceRuntimeScope::new(tab.profile_id().clone(), profile_data_mode);
self.ensure_runtime(scope.clone())?;
let requested_url = tab.url().as_str().to_string();
let zoom_percent = tab.zoom_percent();
let (state, sessions) = (&mut self.state, &mut self.sessions);
let RuntimeState::Ready { scope: active_scope, client, .. } = state else {
return Err("Servo live runtime is unavailable".to_string());
};
if active_scope != &scope {
return Err(active_scope.error_for(&scope));
}
let session = sessions.entry(tab.id().clone()).or_insert_with(WebSurfaceSession::default);
let next_scroll_offset = input.scroll_offset;
let started_loading = session.started_loading(&requested_url, size, zoom_percent);
let frame = client
.ensure(ServoLiveEnsureRequest {
tab_id: tab.id().as_str().to_string(),
profile_id: tab.profile_id().as_str().to_string(),
url: requested_url.clone(),
width: size.width,
height: size.height,
page_zoom_percent: zoom_percent,
scroll_delta_x: input.scroll_delta.map_or(0, |delta| delta.x()),
scroll_delta_y: input.scroll_delta.map_or(0, |delta| delta.y()),
click_x: input.click_point.map(|point| point.x()),
click_y: input.click_point.map(|point| point.y()),
hover_x: input.hover_point.map(|point| point.x()),
hover_y: input.hover_point.map(|point| point.y()),
typed_text: input.typed_text,
site_permissions: permissions.iter().map(ServoLiveSitePermission::from).collect(),
})
.map_err(|error| error.to_string())?
.map(|frame| {
WebSurfaceFrame::from_live_frame(
requested_url.clone(),
next_scroll_offset,
zoom_percent,
frame,
)
.map_err(|error| error.to_string())
})
.transpose()?;
session.requested_url = requested_url.clone();
session.size = size;
session.zoom_percent = zoom_percent;
session.scroll_offset = next_scroll_offset;
Ok(WebSurfaceEnsureResult { requested_url, started_loading, frame })
}
pub(super) fn tick(&mut self) -> Vec<WebSurfaceRuntimeFrame> {
let (state, sessions) = (&mut self.state, &self.sessions);
let RuntimeState::Ready { client, .. } = state else {
return Vec::new();
};
let mut frames = Vec::new();
for (tab_id, session) in sessions {
match client.poll(tab_id.as_str().to_string()) {
Ok(Some(frame)) => match WebSurfaceFrame::from_live_frame(
session.requested_url.clone(),
session.scroll_offset,
session.zoom_percent,
frame,
) {
Ok(frame) => {
frames.push(WebSurfaceRuntimeFrame::Ready { tab_id: tab_id.clone(), frame })
}
Err(error) => frames.push(WebSurfaceRuntimeFrame::Failed {
tab_id: tab_id.clone(),
message: error.to_string(),
}),
},
Ok(None) => {}
Err(error) => frames.push(WebSurfaceRuntimeFrame::Failed {
tab_id: tab_id.clone(),
message: error.to_string(),
}),
}
}
frames
}
fn ensure_runtime(&mut self, scope: WebSurfaceRuntimeScope) -> Result<(), String> {
match &self.state {
RuntimeState::Empty => {
let (config_dir, transient_profile_data_dir) = config_dir_for_scope(&scope)?;
let client = ServoLiveClient::new(config_dir).map_err(|error| error.to_string())?;
self.state = RuntimeState::Ready { scope, client, transient_profile_data_dir };
Ok(())
}
RuntimeState::Ready { scope: active_scope, .. } if active_scope == &scope => Ok(()),
RuntimeState::Ready { scope: active_scope, .. } => Err(active_scope.error_for(&scope)),
}
}
}
impl Drop for WebSurfaceRuntime {
fn drop(&mut self) {
let RuntimeState::Ready { transient_profile_data_dir: Some(path), .. } = &self.state else {
return;
};
let _ = fs::remove_dir_all(path);
}
}
enum RuntimeState {
Empty,
Ready {
scope: WebSurfaceRuntimeScope,
client: ServoLiveClient,
transient_profile_data_dir: Option<PathBuf>,
},
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct WebSurfaceRuntimeScope {
profile_id: ProfileId,
profile_data_mode: ProfileDataMode,
}
impl WebSurfaceRuntimeScope {
fn new(profile_id: ProfileId, profile_data_mode: ProfileDataMode) -> Self {
Self { profile_id, profile_data_mode }
}
fn error_for(&self, requested: &Self) -> String {
format!(
"Servo live runtime is already attached to profile {} ({:?}); requested profile {} ({:?})",
self.profile_id.as_str(),
self.profile_data_mode,
requested.profile_id.as_str(),
requested.profile_data_mode
)
}
}
#[derive(Clone, Default)]
struct WebSurfaceSession {
requested_url: String,
size: WebSurfaceSize,
zoom_percent: u16,
scroll_offset: WebSurfaceScrollOffset,
}
impl WebSurfaceSession {
fn started_loading(
&self,
requested_url: &str,
size: WebSurfaceSize,
zoom_percent: u16,
) -> bool {
self.requested_url != requested_url
|| self.size != size
|| self.zoom_percent != zoom_percent
}
}
pub(super) struct WebSurfaceEnsureResult {
pub(super) requested_url: String,
pub(super) started_loading: bool,
pub(super) frame: Option<WebSurfaceFrame>,
}
pub(super) enum WebSurfaceRuntimeFrame {
Ready { tab_id: TabId, frame: WebSurfaceFrame },
Failed { tab_id: TabId, message: String },
}
fn config_dir_for_scope(
scope: &WebSurfaceRuntimeScope,
) -> Result<(PathBuf, Option<PathBuf>), String> {
match scope.profile_data_mode {
ProfileDataMode::Persistent => {
let root = default_profile_data_root()
.ok_or_else(|| "Profile data root is unavailable".to_string())?;
let config_dir = profile_data_dir(&root, &scope.profile_id);
fs::create_dir_all(&config_dir).map_err(|error| error.to_string())?;
Ok((config_dir, None))
}
ProfileDataMode::Transient => {
let config_dir =
transient_profile_data_dir(&scope.profile_id).map_err(|error| error.to_string())?;
fs::create_dir_all(&config_dir).map_err(|error| error.to_string())?;
Ok((config_dir.clone(), Some(config_dir)))
}
}
}
impl From<&WebSurfaceSitePermission> for ServoLiveSitePermission {
fn from(permission: &WebSurfaceSitePermission) -> Self {
Self::new(
permission.origin().as_str(),
permission.feature().as_str(),
permission.decision(),
)
}
}
crates/ely_app/src/shell/web_surface_state.rs
+7 -52
View File
@@ -1,10 +1,8 @@
use ely_domain::TabId;
use crate::services::servo_sidecar::{ServoSidecarClient, SidecarSnapshotRequest};
use super::{
web_surface_frame::WebSurfaceFrame,
web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize},
web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset},
};
pub(super) struct WebSurfaceScrollState {
@@ -40,60 +38,17 @@ pub(super) struct WebSurfaceTextInputState {
pub(super) text: String,
}
#[derive(Clone, Copy)]
pub(super) struct WebSurfaceStateKey<'a> {
pub(super) requested_url: &'a str,
pub(super) size: WebSurfaceSize,
#[derive(Clone, Debug, Eq, PartialEq)]
pub(super) struct WebSurfacePendingInput {
pub(super) scroll_offset: WebSurfaceScrollOffset,
pub(super) zoom_percent: u16,
pub(super) click_point: Option<WebSurfaceClickPoint>,
pub(super) typed_text: Option<&'a str>,
}
pub(super) enum WebSurfaceClient {
Ready(ServoSidecarClient),
Unavailable(String),
}
impl WebSurfaceClient {
pub(super) fn new() -> Self {
match ServoSidecarClient::new() {
Ok(client) => Self::Ready(client),
Err(error) => Self::Unavailable(error.to_string()),
}
}
}
pub(super) struct WebSurfaceRequest {
pub(super) tab_id: TabId,
pub(super) requested_url: String,
pub(super) size: WebSurfaceSize,
pub(super) scroll_offset: WebSurfaceScrollOffset,
pub(super) zoom_percent: u16,
pub(super) scroll_delta: Option<super::web_surface_geometry::WebSurfaceScrollDelta>,
pub(super) click_point: Option<WebSurfaceClickPoint>,
pub(super) hover_point: Option<WebSurfaceClickPoint>,
pub(super) typed_text: Option<String>,
pub(super) client: ServoSidecarClient,
pub(super) snapshot_request: SidecarSnapshotRequest,
}
pub(super) enum WebSurfaceState {
Loading {
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
zoom_percent: u16,
click_point: Option<WebSurfaceClickPoint>,
typed_text: Option<String>,
previous_frame: Option<WebSurfaceFrame>,
},
Loading { requested_url: String, previous_frame: Option<WebSurfaceFrame> },
Ready(WebSurfaceFrame),
Failed {
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
zoom_percent: u16,
click_point: Option<WebSurfaceClickPoint>,
typed_text: Option<String>,
message: String,
},
Failed { message: String },
}
crates/ely_app/src/shell/web_surface_view.rs
+11
View File
@@ -93,6 +93,8 @@ fn render_input_overlay(
let click_tab_id = tab_id.clone();
let click_url = url.clone();
let click_entity = state_entity.clone();
let hover_tab_id = tab_id.clone();
let hover_entity = state_entity.clone();
let scroll_tab_id = tab_id;
let scroll_url = url;
let scroll_entity = state_entity;
@@ -116,6 +118,15 @@ fn render_input_overlay(
});
cx.stop_propagation();
})
.on_mouse_move(move |event, _window, cx| {
hover_entity.update(cx, |shell, cx| {
shell.hover_external_web_viewport(
hover_tab_id.clone(),
event.position,
cx,
);
});
})
.on_scroll_wheel(move |event, window, cx| {
let delta = event.delta.pixel_delta(window.line_height());
scroll_entity.update(cx, |shell, cx| {