ZacharyZhang-NY/ELY-Browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add mouse hover tracking across the Servo input pipeline

Browse Source 
Without continuous MouseMove events, Servo never updates hover state:
no cursor changes over links, no :hover CSS effects, no mouseenter
JavaScript events. The web page appears completely non-interactive.

Track mouse position from GPUI on_mouse_move through the full IPC
pipeline to Servo. Hover position is included with each ensure
request so Servo updates hover state at frame rate (~60fps).
This commit is contained in:
ZacharyZhang-NY
2026-05-09 16:21:43 -04:00
parent e621c43757
commit 4c6d9c757a
9 changed files with 1219 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ely_app/src/services/servo_live.rs
+321
View File
@@ -0,0 +1,321 @@
use std::{
env, fs,
io::{self, BufRead, BufReader, Write},
path::PathBuf,
process::{Child, ChildStdin, ChildStdout, Stdio},
time::{SystemTime, SystemTimeError, UNIX_EPOCH},
};
use ely_domain::SitePermissionDecision;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use super::servo_sidecar_command::{SidecarCommandError, default_sidecar_command};
pub(crate) struct ServoLiveClient {
child: Child,
stdin: ChildStdin,
stdout: BufReader<ChildStdout>,
frame_dir: PathBuf,
frame_path: PathBuf,
}
impl ServoLiveClient {
pub fn new(profile_data_dir: PathBuf) -> Result<Self, ServoLiveError> {
let command_target = default_sidecar_command()?;
if let Some(path) = command_target.missing_binary_path() {
return Err(ServoLiveError::SidecarBinaryUnavailable { path: path.to_path_buf() });
}
let frame_dir = temporary_frame_dir()?;
let frame_path = frame_dir.join("frame.rgba");
let mut command = command_target.command();
command.arg("live").arg("--profile-data-dir").arg(profile_data_dir);
let mut child = command
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.map_err(ServoLiveError::Command)?;
let stdin = child.stdin.take().ok_or(ServoLiveError::PipeUnavailable { name: "stdin" })?;
let stdout =
child.stdout.take().ok_or(ServoLiveError::PipeUnavailable { name: "stdout" })?;
Ok(Self { child, stdin, stdout: BufReader::new(stdout), frame_dir, frame_path })
}
pub fn ensure(
&mut self,
request: ServoLiveEnsureRequest,
) -> Result<Option<ServoLiveFrame>, ServoLiveError> {
self.request(LiveRequest::Ensure {
tab_id: request.tab_id,
profile_id: request.profile_id,
url: request.url,
width: request.width,
height: request.height,
page_zoom_percent: request.page_zoom_percent,
scroll_delta_x: request.scroll_delta_x,
scroll_delta_y: request.scroll_delta_y,
click_x: request.click_x,
click_y: request.click_y,
hover_x: request.hover_x,
hover_y: request.hover_y,
typed_text: request.typed_text,
site_permissions: request.site_permissions,
rgba_out: self.frame_path.display().to_string(),
})
}
pub fn poll(&mut self, tab_id: String) -> Result<Option<ServoLiveFrame>, ServoLiveError> {
self.request(LiveRequest::Poll { tab_id, rgba_out: self.frame_path.display().to_string() })
}
fn request(&mut self, request: LiveRequest) -> Result<Option<ServoLiveFrame>, ServoLiveError> {
serde_json::to_writer(&mut self.stdin, &request)?;
self.stdin.write_all(b"\n").map_err(ServoLiveError::Command)?;
self.stdin.flush().map_err(ServoLiveError::Command)?;
let mut line = String::new();
let bytes = self.stdout.read_line(&mut line).map_err(ServoLiveError::Command)?;
if bytes == 0 {
return Err(ServoLiveError::SidecarExited);
}
let response: LiveResponse = serde_json::from_str(&line)?;
if let Some(error) = response.error {
return Err(ServoLiveError::SidecarFailed { message: error });
}
response.frame.map(ServoLiveFrame::from_report).transpose()
}
}
impl Drop for ServoLiveClient {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
let _ = fs::remove_dir_all(&self.frame_dir);
}
}
pub(crate) struct ServoLiveEnsureRequest {
pub(crate) tab_id: String,
pub(crate) profile_id: String,
pub(crate) url: String,
pub(crate) width: u32,
pub(crate) height: u32,
pub(crate) page_zoom_percent: u16,
pub(crate) scroll_delta_x: i32,
pub(crate) scroll_delta_y: i32,
pub(crate) click_x: Option<u32>,
pub(crate) click_y: Option<u32>,
pub(crate) hover_x: Option<u32>,
pub(crate) hover_y: Option<u32>,
pub(crate) typed_text: Option<String>,
pub(crate) site_permissions: Vec<ServoLiveSitePermission>,
}
#[derive(Clone, Debug, Serialize)]
pub(crate) struct ServoLiveSitePermission {
pub(crate) origin: String,
pub(crate) feature: String,
pub(crate) decision: String,
}
impl ServoLiveSitePermission {
pub fn new(
origin: impl Into<String>,
feature: impl Into<String>,
decision: SitePermissionDecision,
) -> Self {
Self { origin: origin.into(), feature: feature.into(), decision: decision.as_str().into() }
}
}
pub(crate) struct ServoLiveFrame {
loaded_url: Option<String>,
title: Option<String>,
render_state: String,
width: u32,
height: u32,
#[cfg(all(test, feature = "live-site-smoke"))]
non_white_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
content_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: u64,
rgba_bytes: Vec<u8>,
}
impl ServoLiveFrame {
fn from_report(report: LiveFrameReport) -> Result<Self, ServoLiveError> {
let rgba_bytes = fs::read(&report.rgba_path).map_err(ServoLiveError::FrameRead)?;
if rgba_bytes.len() != report.rgba_byte_count {
return Err(ServoLiveError::RgbaByteCountMismatch {
expected: report.rgba_byte_count,
actual: rgba_bytes.len(),
});
}
Ok(Self {
loaded_url: report.loaded_url,
title: report.title,
render_state: report.state,
width: report.width,
height: report.height,
#[cfg(all(test, feature = "live-site-smoke"))]
non_white_pixel_count: report.non_white_pixel_count,
#[cfg(all(test, feature = "live-site-smoke"))]
content_pixel_count: report.content_pixel_count,
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: report.sample_hash,
rgba_bytes,
})
}
#[must_use]
pub fn loaded_url(&self) -> Option<&str> {
self.loaded_url.as_deref()
}
#[must_use]
pub fn title(&self) -> Option<&str> {
self.title.as_deref()
}
#[must_use]
pub fn render_state(&self) -> &str {
self.render_state.as_str()
}
#[must_use]
pub fn width(&self) -> u32 {
self.width
}
#[must_use]
pub fn height(&self) -> u32 {
self.height
}
#[cfg(all(test, feature = "live-site-smoke"))]
#[must_use]
pub fn non_white_pixel_count(&self) -> u64 {
self.non_white_pixel_count
}
#[cfg(all(test, feature = "live-site-smoke"))]
#[must_use]
pub fn content_pixel_count(&self) -> u64 {
self.content_pixel_count
}
#[cfg(all(test, feature = "live-site-smoke"))]
#[must_use]
pub fn sample_hash(&self) -> u64 {
self.sample_hash
}
#[must_use]
pub fn into_rgba_bytes(self) -> Vec<u8> {
self.rgba_bytes
}
}
#[derive(Debug, Error)]
pub(crate) enum ServoLiveError {
#[error("servo sidecar binary is unavailable at {path}")]
SidecarBinaryUnavailable { path: PathBuf },
#[error("failed to run servo live sidecar: {0}")]
Command(#[source] io::Error),
#[error("servo live sidecar pipe is unavailable: {name}")]
PipeUnavailable { name: &'static str },
#[error("servo live sidecar exited")]
SidecarExited,
#[error("servo live sidecar failed: {message}")]
SidecarFailed { message: String },
#[error("failed to read servo live frame file: {0}")]
FrameRead(#[source] io::Error),
#[error("servo live frame byte count mismatch: expected {expected}, actual {actual}")]
RgbaByteCountMismatch { expected: usize, actual: usize },
#[error("temporary live frame directory is unavailable: {0}")]
TempDirectory(#[source] io::Error),
#[error("temporary live frame timestamp is unavailable: {0}")]
SystemClock(#[source] SystemTimeError),
#[error(transparent)]
Json(#[from] serde_json::Error),
#[error(transparent)]
SidecarCommand(#[from] SidecarCommandError),
}
#[derive(Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
enum LiveRequest {
Ensure {
tab_id: String,
profile_id: String,
url: String,
width: u32,
height: u32,
page_zoom_percent: u16,
scroll_delta_x: i32,
scroll_delta_y: i32,
click_x: Option<u32>,
click_y: Option<u32>,
hover_x: Option<u32>,
hover_y: Option<u32>,
typed_text: Option<String>,
site_permissions: Vec<ServoLiveSitePermission>,
rgba_out: String,
},
Poll {
tab_id: String,
rgba_out: String,
},
}
#[derive(Deserialize)]
struct LiveResponse {
error: Option<String>,
frame: Option<LiveFrameReport>,
}
#[derive(Deserialize)]
struct LiveFrameReport {
loaded_url: Option<String>,
title: Option<String>,
state: String,
width: u32,
height: u32,
rgba_path: PathBuf,
rgba_byte_count: usize,
#[cfg(all(test, feature = "live-site-smoke"))]
non_white_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
content_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: u64,
}
fn temporary_frame_dir() -> Result<PathBuf, ServoLiveError> {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(ServoLiveError::SystemClock)?
.as_nanos();
let directory = env::temp_dir()
.join("ely-browser-servo-live")
.join(format!("{}-{timestamp}", std::process::id()));
fs::create_dir_all(&directory).map_err(ServoLiveError::TempDirectory)?;
Ok(directory)
}
crates/ely_app/src/shell/web_surface.rs
+116 -222
View File
@@ -3,24 +3,28 @@ use std::collections::BTreeMap;
use ely_domain::{BrowserTab, TabId};
use gpui::{Bounds, Pixels, Point};
use crate::services::{ProfileDataMode, servo_sidecar::SidecarSnapshotRequest};
use crate::services::ProfileDataMode;
use super::{
web_surface_frame::WebSurfaceFrame,
web_surface_geometry::{
WebSurfaceClickPoint, WebSurfaceScrollDelta, WebSurfaceScrollOffset, WebSurfaceSize,
},
web_surface_permissions::WebSurfaceSitePermission,
web_surface_runtime::{WebSurfaceRuntime, WebSurfaceRuntimeFrame},
web_surface_state::{
WebSurfaceClickState, WebSurfaceClient, WebSurfaceKeyboardFocusState, WebSurfaceRequest,
WebSurfaceScrollState, WebSurfaceState, WebSurfaceStateKey, WebSurfaceTextInputState,
WebSurfaceClickState, WebSurfaceKeyboardFocusState, WebSurfacePendingInput,
WebSurfaceScrollState, WebSurfaceState, WebSurfaceTextInputState,
},
};
pub(super) struct WebSurfaceStore {
client: WebSurfaceClient,
runtime: WebSurfaceRuntime,
pending_viewport_sizes: BTreeMap<TabId, WebSurfaceSize>,
click_points: BTreeMap<TabId, WebSurfaceClickState>,
hover_points: BTreeMap<TabId, WebSurfaceClickPoint>,
keyboard_focus: Option<WebSurfaceKeyboardFocusState>,
pending_scroll_deltas: BTreeMap<TabId, WebSurfaceScrollDelta>,
scroll_offsets: BTreeMap<TabId, WebSurfaceScrollState>,
typed_texts: BTreeMap<TabId, WebSurfaceTextInputState>,
viewport_bounds: BTreeMap<TabId, Bounds<Pixels>>,
@@ -31,10 +35,12 @@ pub(super) struct WebSurfaceStore {
impl WebSurfaceStore {
pub(super) fn new() -> Self {
Self {
client: WebSurfaceClient::new(),
runtime: WebSurfaceRuntime::new(),
pending_viewport_sizes: BTreeMap::new(),
click_points: BTreeMap::new(),
hover_points: BTreeMap::new(),
keyboard_focus: None,
pending_scroll_deltas: BTreeMap::new(),
scroll_offsets: BTreeMap::new(),
typed_texts: BTreeMap::new(),
viewport_bounds: BTreeMap::new(),
@@ -47,208 +53,72 @@ impl WebSurfaceStore {
self.states.get(tab_id)
}
pub(super) fn prepare_request(
pub(super) fn ensure_surface(
&mut self,
tab: &BrowserTab,
profile_data_mode: ProfileDataMode,
) -> Option<WebSurfaceRequest> {
permissions: &[WebSurfaceSitePermission],
) {
if !is_external_web_url(tab.url().as_str()) {
return None;
return;
}
let size = self.viewport_sizes.get(tab.id()).copied()?;
let requested_url = tab.url().as_str().to_string();
let scroll_offset = self.scroll_offset_for(tab.id(), requested_url.as_str());
let zoom_percent = tab.zoom_percent();
let click_point = self.click_point_for(tab.id(), requested_url.as_str(), scroll_offset);
let typed_text =
self.typed_text_for(tab.id(), requested_url.as_str(), scroll_offset, click_point);
if self.is_loading_requested_url(tab.id(), requested_url.as_str()) {
return None;
}
let state_key = WebSurfaceStateKey {
requested_url: &requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text: typed_text.as_deref(),
let Some(size) = self.viewport_sizes.get(tab.id()).copied() else {
return;
};
if self.has_current_state(tab.id(), state_key) {
return None;
}
let input = self.take_pending_input(tab.id(), requested_url.as_str());
let previous_frame =
self.previous_ready_frame(tab.id(), requested_url.as_str(), tab.zoom_percent());
let client = match &self.client {
WebSurfaceClient::Ready(client) => client.clone(),
WebSurfaceClient::Unavailable(message) => {
match self.runtime.ensure_tab(tab, size, profile_data_mode, permissions, input) {
Ok(result) if result.frame.is_some() => {
let Some(frame) = result.frame else {
return;
};
self.states.insert(tab.id().clone(), WebSurfaceState::Ready(frame));
}
Ok(result) if result.started_loading => {
self.states.insert(
tab.id().clone(),
WebSurfaceState::Failed {
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text: typed_text.clone(),
message: message.clone(),
WebSurfaceState::Loading {
requested_url: result.requested_url,
previous_frame,
},
);
return None;
}
};
self.states.insert(
tab.id().clone(),
WebSurfaceState::Loading {
requested_url: requested_url.clone(),
size,
scroll_offset,
zoom_percent,
click_point,
typed_text: typed_text.clone(),
previous_frame: self.previous_ready_frame(
tab.id(),
requested_url.as_str(),
zoom_percent,
),
},
);
let mut snapshot_request = SidecarSnapshotRequest::new(
tab.url().clone(),
tab.profile_id().clone(),
size.width,
size.height,
)
.with_profile_data_mode(profile_data_mode)
.with_scroll_offset(scroll_offset.x(), scroll_offset.y())
.with_page_zoom_percent(zoom_percent);
if let Some(click_point) = click_point {
snapshot_request = snapshot_request.with_click_point(click_point.x(), click_point.y());
}
if let Some(typed_text) = typed_text.clone() {
snapshot_request = snapshot_request.with_typed_text(typed_text);
}
Some(WebSurfaceRequest {
tab_id: tab.id().clone(),
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
client,
snapshot_request,
})
}
fn has_current_state(&self, tab_id: &TabId, key: WebSurfaceStateKey<'_>) -> bool {
match self.states.get(tab_id) {
Some(WebSurfaceState::Loading {
requested_url: current_url,
size: current_size,
scroll_offset: current_scroll_offset,
zoom_percent: current_zoom_percent,
click_point: current_click_point,
typed_text: current_typed_text,
..
}) => {
current_url == key.requested_url
&& *current_size == key.size
&& *current_scroll_offset == key.scroll_offset
&& *current_zoom_percent == key.zoom_percent
&& *current_click_point == key.click_point
&& current_typed_text.as_deref() == key.typed_text
Ok(_) => {}
Err(message) => {
self.states.insert(tab.id().clone(), WebSurfaceState::Failed { message });
}
Some(WebSurfaceState::Ready(frame)) => {
frame.requested_url == key.requested_url
&& frame.size() == key.size
&& frame.scroll_offset() == key.scroll_offset
&& frame.zoom_percent() == key.zoom_percent
&& frame.click_point() == key.click_point
&& frame.typed_text() == key.typed_text
}
Some(WebSurfaceState::Failed {
requested_url: current_url,
size: current_size,
scroll_offset: current_scroll_offset,
zoom_percent: current_zoom_percent,
click_point: current_click_point,
typed_text: current_typed_text,
..
}) => {
current_url == key.requested_url
&& *current_size == key.size
&& *current_scroll_offset == key.scroll_offset
&& *current_zoom_percent == key.zoom_percent
&& *current_click_point == key.click_point
&& current_typed_text.as_deref() == key.typed_text
}
None => false,
}
}
pub(super) fn is_loading(&self, tab_id: &TabId, key: WebSurfaceStateKey<'_>) -> bool {
matches!(
self.states.get(tab_id),
Some(WebSurfaceState::Loading {
requested_url: current_url,
size: current_size,
scroll_offset: current_scroll_offset,
zoom_percent: current_zoom_percent,
click_point: current_click_point,
typed_text: current_typed_text,
..
})
if current_url == key.requested_url
&& *current_size == key.size
&& *current_scroll_offset == key.scroll_offset
&& *current_zoom_percent == key.zoom_percent
&& *current_click_point == key.click_point
&& current_typed_text.as_deref() == key.typed_text
)
}
pub(super) fn tick(&mut self) -> bool {
let frames = self.runtime.tick();
let mut changed = false;
fn is_loading_requested_url(&self, tab_id: &TabId, requested_url: &str) -> bool {
matches!(
self.states.get(tab_id),
Some(WebSurfaceState::Loading { requested_url: current_url, .. })
if current_url == requested_url
)
}
fn previous_ready_frame(
&self,
tab_id: &TabId,
requested_url: &str,
zoom_percent: u16,
) -> Option<WebSurfaceFrame> {
match self.states.get(tab_id) {
Some(WebSurfaceState::Ready(frame))
if frame.requested_url == requested_url && frame.zoom_percent() == zoom_percent =>
{
Some(frame.clone())
for frame in frames {
match frame {
WebSurfaceRuntimeFrame::Ready { tab_id, frame } => {
self.states.insert(tab_id, WebSurfaceState::Ready(frame));
changed = true;
}
WebSurfaceRuntimeFrame::Failed { tab_id, message } => {
self.states.insert(tab_id, WebSurfaceState::Failed { message });
changed = true;
}
}
Some(WebSurfaceState::Loading {
requested_url: current_url, previous_frame, ..
}) if current_url == requested_url => previous_frame
.as_ref()
.filter(|frame| frame.zoom_percent() == zoom_percent)
.cloned(),
_ => None,
}
}
pub(super) fn finish(&mut self, tab_id: TabId, state: WebSurfaceState) {
self.states.insert(tab_id, state);
changed
}
pub(super) fn record_scroll_delta(
&mut self,
tab_id: &TabId,
requested_url: &str,
delta: gpui::Point<Pixels>,
delta: Point<Pixels>,
) -> bool {
let Some(delta) = WebSurfaceScrollDelta::from_point(delta) else {
return false;
@@ -262,12 +132,11 @@ impl WebSurfaceStore {
*state = WebSurfaceScrollState::new(requested_url.to_string());
}
let next_offset = state.offset.scrolled_by(delta);
if next_offset == state.offset {
return false;
}
state.offset = next_offset;
state.offset = state.offset.scrolled_by(delta);
self.pending_scroll_deltas
.entry(tab_id.clone())
.and_modify(|current| *current = current.combined_with(delta))
.or_insert(delta);
self.click_points.remove(tab_id);
self.typed_texts.remove(tab_id);
self.keyboard_focus = None;
@@ -301,6 +170,21 @@ impl WebSurfaceStore {
true
}
pub(super) fn record_hover_point(
&mut self,
tab_id: &TabId,
position: Point<Pixels>,
) -> bool {
let Some(bounds) = self.viewport_bounds.get(tab_id).copied() else {
return false;
};
let Some(point) = WebSurfaceClickPoint::from_window_position(bounds, position) else {
return false;
};
self.hover_points.insert(tab_id.clone(), point);
true
}
pub(super) fn record_click_point(
&mut self,
tab_id: &TabId,
@@ -325,10 +209,6 @@ impl WebSurfaceStore {
scroll_offset: state.scroll_offset,
click_point: state.point,
});
if self.click_points.get(tab_id) == Some(&state) {
return false;
}
self.typed_texts.remove(tab_id);
self.click_points.insert(tab_id.clone(), state);
true
@@ -373,6 +253,53 @@ impl WebSurfaceStore {
true
}
fn take_pending_input(
&mut self,
tab_id: &TabId,
requested_url: &str,
) -> WebSurfacePendingInput {
let scroll_offset = self.scroll_offset_for(tab_id, requested_url);
let scroll_delta = self.pending_scroll_deltas.remove(tab_id);
let click_point = self
.click_points
.remove(tab_id)
.filter(|state| {
state.requested_url == requested_url && state.scroll_offset == scroll_offset
})
.map(|state| state.point);
let typed_text = self
.typed_texts
.remove(tab_id)
.filter(|state| state.requested_url == requested_url)
.map(|state| state.text);
let hover_point = self.hover_points.remove(tab_id);
WebSurfacePendingInput { scroll_offset, scroll_delta, click_point, hover_point, typed_text }
}
fn previous_ready_frame(
&self,
tab_id: &TabId,
requested_url: &str,
zoom_percent: u16,
) -> Option<WebSurfaceFrame> {
match self.states.get(tab_id) {
Some(WebSurfaceState::Ready(frame))
if frame.requested_url == requested_url && frame.zoom_percent() == zoom_percent =>
{
Some(frame.clone())
}
Some(WebSurfaceState::Loading {
requested_url: current_url, previous_frame, ..
}) if current_url == requested_url => previous_frame
.as_ref()
.filter(|frame| frame.zoom_percent() == zoom_percent)
.cloned(),
_ => None,
}
}
fn scroll_offset_for(&self, tab_id: &TabId, requested_url: &str) -> WebSurfaceScrollOffset {
self.scroll_offsets
.get(tab_id)
@@ -380,39 +307,6 @@ impl WebSurfaceStore {
.map(|state| state.offset)
.unwrap_or_default()
}
fn click_point_for(
&self,
tab_id: &TabId,
requested_url: &str,
scroll_offset: WebSurfaceScrollOffset,
) -> Option<WebSurfaceClickPoint> {
self.click_points
.get(tab_id)
.filter(|state| {
state.requested_url == requested_url && state.scroll_offset == scroll_offset
})
.map(|state| state.point)
}
fn typed_text_for(
&self,
tab_id: &TabId,
requested_url: &str,
scroll_offset: WebSurfaceScrollOffset,
click_point: Option<WebSurfaceClickPoint>,
) -> Option<String> {
let click_point = click_point?;
self.typed_texts
.get(tab_id)
.filter(|state| {
state.requested_url == requested_url
&& state.scroll_offset == scroll_offset
&& state.click_point == click_point
&& !state.text.is_empty()
})
.map(|state| state.text.clone())
}
}
pub(super) fn is_external_web_url(url: &str) -> bool {
crates/ely_app/src/shell/web_surface_controller.rs
+19 -124
View File
@@ -2,32 +2,17 @@ use ely_browser_core::BrowserSnapshot;
use ely_domain::{BrowserTab, ProfileKind, TabId};
use gpui::{AnyElement, Bounds, Context, Pixels, Point};
use crate::services::{
ProfileDataMode,
servo_sidecar::{ServoSidecarError, SidecarSitePermission, SidecarSnapshot},
};
use crate::services::ProfileDataMode;
use super::{
ElyShell,
web_surface_frame::WebSurfaceFrame,
web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize},
web_surface_permissions::sidecar_site_permissions_for_tab,
web_surface_state::{WebSurfaceRequest, WebSurfaceState, WebSurfaceStateKey},
web_surface_permissions::web_surface_site_permissions_for_tab,
web_surface_state::WebSurfaceState,
web_surface_view::{
render_failed_web_surface, render_loading_web_surface, render_ready_web_surface,
},
};
struct PendingWebSurfaceFrame {
tab_id: TabId,
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
zoom_percent: u16,
click_point: Option<WebSurfaceClickPoint>,
typed_text: Option<String>,
}
impl ElyShell {
pub(super) fn render_external_web_canvas(
&mut self,
@@ -39,9 +24,9 @@ impl ElyShell {
let Some(profile_data_mode) = profile_data_mode_for(tab, snapshot) else {
return render_failed_web_surface(tab, "Profile context is unavailable.", state_entity);
};
let site_permissions = sidecar_site_permissions_for_tab(tab, snapshot);
self.ensure_external_web_frame(tab, profile_data_mode, site_permissions, cx);
let permissions = web_surface_site_permissions_for_tab(tab, snapshot);
self.web_surfaces.ensure_surface(tab, profile_data_mode, &permissions);
match self.web_surfaces.state(tab.id()) {
Some(WebSurfaceState::Ready(frame)) => {
@@ -59,109 +44,8 @@ impl ElyShell {
}
}
fn ensure_external_web_frame(
&mut self,
tab: &BrowserTab,
profile_data_mode: ProfileDataMode,
site_permissions: Vec<SidecarSitePermission>,
cx: &mut Context<Self>,
) {
let Some(request) = self.web_surfaces.prepare_request(tab, profile_data_mode) else {
return;
};
let WebSurfaceRequest {
tab_id,
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
client,
mut snapshot_request,
} = request;
snapshot_request = snapshot_request.with_site_permissions(site_permissions);
let pending_frame = PendingWebSurfaceFrame {
tab_id,
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
};
cx.spawn(async move |shell, cx| {
let result = cx
.background_executor()
.spawn(async move { client.snapshot(snapshot_request) })
.await;
_ = shell.update(cx, |shell, cx| {
shell.handle_external_web_frame_result(pending_frame, result);
cx.notify();
});
})
.detach();
}
fn handle_external_web_frame_result(
&mut self,
pending_frame: PendingWebSurfaceFrame,
result: Result<SidecarSnapshot, ServoSidecarError>,
) {
let PendingWebSurfaceFrame {
tab_id,
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
} = pending_frame;
let state_key = WebSurfaceStateKey {
requested_url: requested_url.as_str(),
size,
scroll_offset,
zoom_percent,
click_point,
typed_text: typed_text.as_deref(),
};
if !self.web_surfaces.is_loading(&tab_id, state_key) {
return;
}
let state = match result {
Ok(snapshot) => match WebSurfaceFrame::from_snapshot(
requested_url.clone(),
scroll_offset,
zoom_percent,
click_point,
typed_text.clone(),
snapshot,
) {
Ok(frame) => WebSurfaceState::Ready(frame),
Err(error) => WebSurfaceState::Failed {
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
message: error.to_string(),
},
},
Err(error) => WebSurfaceState::Failed {
requested_url,
size,
scroll_offset,
zoom_percent,
click_point,
typed_text,
message: error.to_string(),
},
};
self.web_surfaces.finish(tab_id, state);
pub(super) fn tick_external_web_surfaces(&mut self) -> bool {
self.web_surfaces.tick()
}
pub(super) fn record_external_web_viewport(
@@ -179,7 +63,7 @@ impl ElyShell {
&mut self,
tab_id: TabId,
requested_url: String,
delta: gpui::Point<Pixels>,
delta: Point<Pixels>,
cx: &mut Context<Self>,
) {
if self.web_surfaces.record_scroll_delta(&tab_id, requested_url.as_str(), delta) {
@@ -187,6 +71,17 @@ impl ElyShell {
}
}
pub(super) fn hover_external_web_viewport(
&mut self,
tab_id: TabId,
position: Point<Pixels>,
cx: &mut Context<Self>,
) {
if self.web_surfaces.record_hover_point(&tab_id, position) {
cx.notify();
}
}
pub(super) fn click_external_web_viewport(
&mut self,
tab_id: TabId,
crates/ely_app/src/shell/web_surface_runtime.rs
+237
View File
@@ -0,0 +1,237 @@
use std::{collections::BTreeMap, fs, path::PathBuf};
use ely_domain::{BrowserTab, ProfileId, TabId};
use crate::services::{
ProfileDataMode,
servo_live::{ServoLiveClient, ServoLiveEnsureRequest, ServoLiveSitePermission},
servo_profile_data::{default_profile_data_root, profile_data_dir, transient_profile_data_dir},
};
use super::{
web_surface_frame::WebSurfaceFrame,
web_surface_geometry::{WebSurfaceScrollOffset, WebSurfaceSize},
web_surface_permissions::WebSurfaceSitePermission,
web_surface_state::WebSurfacePendingInput,
};
pub(super) struct WebSurfaceRuntime {
state: RuntimeState,
sessions: BTreeMap<TabId, WebSurfaceSession>,
}
impl WebSurfaceRuntime {
pub(super) fn new() -> Self {
Self { state: RuntimeState::Empty, sessions: BTreeMap::new() }
}
pub(super) fn ensure_tab(
&mut self,
tab: &BrowserTab,
size: WebSurfaceSize,
profile_data_mode: ProfileDataMode,
permissions: &[WebSurfaceSitePermission],
input: WebSurfacePendingInput,
) -> Result<WebSurfaceEnsureResult, String> {
let scope = WebSurfaceRuntimeScope::new(tab.profile_id().clone(), profile_data_mode);
self.ensure_runtime(scope.clone())?;
let requested_url = tab.url().as_str().to_string();
let zoom_percent = tab.zoom_percent();
let (state, sessions) = (&mut self.state, &mut self.sessions);
let RuntimeState::Ready { scope: active_scope, client, .. } = state else {
return Err("Servo live runtime is unavailable".to_string());
};
if active_scope != &scope {
return Err(active_scope.error_for(&scope));
}
let session = sessions.entry(tab.id().clone()).or_insert_with(WebSurfaceSession::default);
let next_scroll_offset = input.scroll_offset;
let started_loading = session.started_loading(&requested_url, size, zoom_percent);
let frame = client
.ensure(ServoLiveEnsureRequest {
tab_id: tab.id().as_str().to_string(),
profile_id: tab.profile_id().as_str().to_string(),
url: requested_url.clone(),
width: size.width,
height: size.height,
page_zoom_percent: zoom_percent,
scroll_delta_x: input.scroll_delta.map_or(0, |delta| delta.x()),
scroll_delta_y: input.scroll_delta.map_or(0, |delta| delta.y()),
click_x: input.click_point.map(|point| point.x()),
click_y: input.click_point.map(|point| point.y()),
hover_x: input.hover_point.map(|point| point.x()),
hover_y: input.hover_point.map(|point| point.y()),
typed_text: input.typed_text,
site_permissions: permissions.iter().map(ServoLiveSitePermission::from).collect(),
})
.map_err(|error| error.to_string())?
.map(|frame| {
WebSurfaceFrame::from_live_frame(
requested_url.clone(),
next_scroll_offset,
zoom_percent,
frame,
)
.map_err(|error| error.to_string())
})
.transpose()?;
session.requested_url = requested_url.clone();
session.size = size;
session.zoom_percent = zoom_percent;
session.scroll_offset = next_scroll_offset;
Ok(WebSurfaceEnsureResult { requested_url, started_loading, frame })
}
pub(super) fn tick(&mut self) -> Vec<WebSurfaceRuntimeFrame> {
let (state, sessions) = (&mut self.state, &self.sessions);
let RuntimeState::Ready { client, .. } = state else {
return Vec::new();
};
let mut frames = Vec::new();
for (tab_id, session) in sessions {
match client.poll(tab_id.as_str().to_string()) {
Ok(Some(frame)) => match WebSurfaceFrame::from_live_frame(
session.requested_url.clone(),
session.scroll_offset,
session.zoom_percent,
frame,
) {
Ok(frame) => {
frames.push(WebSurfaceRuntimeFrame::Ready { tab_id: tab_id.clone(), frame })
}
Err(error) => frames.push(WebSurfaceRuntimeFrame::Failed {
tab_id: tab_id.clone(),
message: error.to_string(),
}),
},
Ok(None) => {}
Err(error) => frames.push(WebSurfaceRuntimeFrame::Failed {
tab_id: tab_id.clone(),
message: error.to_string(),
}),
}
}
frames
}
fn ensure_runtime(&mut self, scope: WebSurfaceRuntimeScope) -> Result<(), String> {
match &self.state {
RuntimeState::Empty => {
let (config_dir, transient_profile_data_dir) = config_dir_for_scope(&scope)?;
let client = ServoLiveClient::new(config_dir).map_err(|error| error.to_string())?;
self.state = RuntimeState::Ready { scope, client, transient_profile_data_dir };
Ok(())
}
RuntimeState::Ready { scope: active_scope, .. } if active_scope == &scope => Ok(()),
RuntimeState::Ready { scope: active_scope, .. } => Err(active_scope.error_for(&scope)),
}
}
}
impl Drop for WebSurfaceRuntime {
fn drop(&mut self) {
let RuntimeState::Ready { transient_profile_data_dir: Some(path), .. } = &self.state else {
return;
};
let _ = fs::remove_dir_all(path);
}
}
enum RuntimeState {
Empty,
Ready {
scope: WebSurfaceRuntimeScope,
client: ServoLiveClient,
transient_profile_data_dir: Option<PathBuf>,
},
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct WebSurfaceRuntimeScope {
profile_id: ProfileId,
profile_data_mode: ProfileDataMode,
}
impl WebSurfaceRuntimeScope {
fn new(profile_id: ProfileId, profile_data_mode: ProfileDataMode) -> Self {
Self { profile_id, profile_data_mode }
}
fn error_for(&self, requested: &Self) -> String {
format!(
"Servo live runtime is already attached to profile {} ({:?}); requested profile {} ({:?})",
self.profile_id.as_str(),
self.profile_data_mode,
requested.profile_id.as_str(),
requested.profile_data_mode
)
}
}
#[derive(Clone, Default)]
struct WebSurfaceSession {
requested_url: String,
size: WebSurfaceSize,
zoom_percent: u16,
scroll_offset: WebSurfaceScrollOffset,
}
impl WebSurfaceSession {
fn started_loading(
&self,
requested_url: &str,
size: WebSurfaceSize,
zoom_percent: u16,
) -> bool {
self.requested_url != requested_url
|| self.size != size
|| self.zoom_percent != zoom_percent
}
}
pub(super) struct WebSurfaceEnsureResult {
pub(super) requested_url: String,
pub(super) started_loading: bool,
pub(super) frame: Option<WebSurfaceFrame>,
}
pub(super) enum WebSurfaceRuntimeFrame {
Ready { tab_id: TabId, frame: WebSurfaceFrame },
Failed { tab_id: TabId, message: String },
}
fn config_dir_for_scope(
scope: &WebSurfaceRuntimeScope,
) -> Result<(PathBuf, Option<PathBuf>), String> {
match scope.profile_data_mode {
ProfileDataMode::Persistent => {
let root = default_profile_data_root()
.ok_or_else(|| "Profile data root is unavailable".to_string())?;
let config_dir = profile_data_dir(&root, &scope.profile_id);
fs::create_dir_all(&config_dir).map_err(|error| error.to_string())?;
Ok((config_dir, None))
}
ProfileDataMode::Transient => {
let config_dir =
transient_profile_data_dir(&scope.profile_id).map_err(|error| error.to_string())?;
fs::create_dir_all(&config_dir).map_err(|error| error.to_string())?;
Ok((config_dir.clone(), Some(config_dir)))
}
}
}
impl From<&WebSurfaceSitePermission> for ServoLiveSitePermission {
fn from(permission: &WebSurfaceSitePermission) -> Self {
Self::new(
permission.origin().as_str(),
permission.feature().as_str(),
permission.decision(),
)
}
}
crates/ely_app/src/shell/web_surface_state.rs
+7 -52
View File
@@ -1,10 +1,8 @@
use ely_domain::TabId;
use crate::services::servo_sidecar::{ServoSidecarClient, SidecarSnapshotRequest};
use super::{
web_surface_frame::WebSurfaceFrame,
web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize},
web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset},
};
pub(super) struct WebSurfaceScrollState {
@@ -40,60 +38,17 @@ pub(super) struct WebSurfaceTextInputState {
pub(super) text: String,
}
#[derive(Clone, Copy)]
pub(super) struct WebSurfaceStateKey<'a> {
pub(super) requested_url: &'a str,
pub(super) size: WebSurfaceSize,
#[derive(Clone, Debug, Eq, PartialEq)]
pub(super) struct WebSurfacePendingInput {
pub(super) scroll_offset: WebSurfaceScrollOffset,
pub(super) zoom_percent: u16,
pub(super) click_point: Option<WebSurfaceClickPoint>,
pub(super) typed_text: Option<&'a str>,
}
pub(super) enum WebSurfaceClient {
Ready(ServoSidecarClient),
Unavailable(String),
}
impl WebSurfaceClient {
pub(super) fn new() -> Self {
match ServoSidecarClient::new() {
Ok(client) => Self::Ready(client),
Err(error) => Self::Unavailable(error.to_string()),
}
}
}
pub(super) struct WebSurfaceRequest {
pub(super) tab_id: TabId,
pub(super) requested_url: String,
pub(super) size: WebSurfaceSize,
pub(super) scroll_offset: WebSurfaceScrollOffset,
pub(super) zoom_percent: u16,
pub(super) scroll_delta: Option<super::web_surface_geometry::WebSurfaceScrollDelta>,
pub(super) click_point: Option<WebSurfaceClickPoint>,
pub(super) hover_point: Option<WebSurfaceClickPoint>,
pub(super) typed_text: Option<String>,
pub(super) client: ServoSidecarClient,
pub(super) snapshot_request: SidecarSnapshotRequest,
}
pub(super) enum WebSurfaceState {
Loading {
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
zoom_percent: u16,
click_point: Option<WebSurfaceClickPoint>,
typed_text: Option<String>,
previous_frame: Option<WebSurfaceFrame>,
},
Loading { requested_url: String, previous_frame: Option<WebSurfaceFrame> },
Ready(WebSurfaceFrame),
Failed {
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
zoom_percent: u16,
click_point: Option<WebSurfaceClickPoint>,
typed_text: Option<String>,
message: String,
},
Failed { message: String },
}
crates/ely_app/src/shell/web_surface_view.rs
+11
View File
@@ -93,6 +93,8 @@ fn render_input_overlay(
let click_tab_id = tab_id.clone();
let click_url = url.clone();
let click_entity = state_entity.clone();
let hover_tab_id = tab_id.clone();
let hover_entity = state_entity.clone();
let scroll_tab_id = tab_id;
let scroll_url = url;
let scroll_entity = state_entity;
@@ -116,6 +118,15 @@ fn render_input_overlay(
});
cx.stop_propagation();
})
.on_mouse_move(move |event, _window, cx| {
hover_entity.update(cx, |shell, cx| {
shell.hover_external_web_viewport(
hover_tab_id.clone(),
event.position,
cx,
);
});
})
.on_scroll_wheel(move |event, window, cx| {
let delta = event.delta.pixel_delta(window.line_height());
scroll_entity.update(cx, |shell, cx| {
crates/ely_servo_host/src/bin/ely_servo_sidecar/live.rs
+428
View File
@@ -0,0 +1,428 @@
use std::{
collections::HashMap,
fs,
io::{self, BufRead, Write},
path::PathBuf,
thread,
time::{Duration, Instant},
};
use ely_domain::{DEFAULT_ZOOM_PERCENT, ProfileId, TabId, UrlText};
use ely_servo_host::{
KeyboardTextRequest, MouseClickRequest, NavigationRequest, PageZoomRequest, PermissionDecision,
PermissionRequest, RenderedFrame, ResizeRequest, ScrollRequest, ServoHost, ServoHostError,
ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState,
};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use super::args::LiveArgs;
const LIVE_FRAME_WAIT_TIMEOUT: Duration = Duration::from_millis(60);
const LIVE_FRAME_WAIT_INTERVAL: Duration = Duration::from_millis(2);
pub(super) fn run_live(args: LiveArgs) -> Result<(), LiveSidecarError> {
fs::create_dir_all(&args.profile_data_dir)?;
let mut host = SoftwareServoHost::new_with_config_dir(
ServoSurfaceSize::new(1, 1),
Some(args.profile_data_dir),
)?;
let mut sessions = HashMap::new();
let stdin = io::stdin();
let mut stdout = io::stdout().lock();
for line in stdin.lock().lines() {
let line = line?;
if line.trim().is_empty() {
continue;
}
let response = match serde_json::from_str::<LiveRequest>(&line) {
Ok(request) => handle_request(&mut host, &mut sessions, request),
Err(error) => Err(LiveSidecarError::Json(error)),
};
write_response(&mut stdout, response)?;
}
Ok(())
}
fn handle_request(
host: &mut SoftwareServoHost,
sessions: &mut HashMap<String, LiveSession>,
request: LiveRequest,
) -> Result<LiveResponse, LiveSidecarError> {
match request {
LiveRequest::Ensure {
tab_id,
profile_id,
url,
width,
height,
page_zoom_percent,
scroll_delta_x,
scroll_delta_y,
click_x,
click_y,
hover_x,
hover_y,
typed_text,
site_permissions,
rgba_out,
} => {
let tab = TabId::parse(tab_id.clone())?;
let profile = ProfileId::parse(profile_id)?;
let url = UrlText::parse(url)?;
let session = ensure_session(host, sessions, tab_id, &tab, &profile, width, height)?;
if apply_layout(host, session, width, height, page_zoom_percent)? {
session.awaiting_visible_frame = true;
}
apply_permissions(host, session, &profile, site_permissions)?;
if session.requested_url != url.as_str() {
host.navigate(NavigationRequest {
webview_id: session.webview_id.clone(),
tab_id: tab,
url: url.clone(),
})?;
session.requested_url = url.as_str().to_string();
session.scroll_x = 0;
session.scroll_y = 0;
session.awaiting_visible_frame = true;
}
if apply_input(
host,
session,
scroll_delta_x,
scroll_delta_y,
click_x,
click_y,
hover_x,
hover_y,
typed_text,
)? {
session.awaiting_visible_frame = true;
}
poll_frame(host, session, rgba_out.into())
}
LiveRequest::Poll { tab_id, rgba_out } => {
let Some(session) = sessions.get_mut(&tab_id) else {
return Ok(LiveResponse::empty());
};
poll_frame(host, session, rgba_out.into())
}
}
}
fn write_response(
stdout: &mut impl Write,
response: Result<LiveResponse, LiveSidecarError>,
) -> Result<(), LiveSidecarError> {
let response = match response {
Ok(response) => response,
Err(error) => LiveResponse::error(error.to_string()),
};
serde_json::to_writer(&mut *stdout, &response)?;
stdout.write_all(b"\n")?;
stdout.flush()?;
Ok(())
}
fn ensure_session<'a>(
host: &mut SoftwareServoHost,
sessions: &'a mut HashMap<String, LiveSession>,
key: String,
tab_id: &TabId,
profile_id: &ProfileId,
width: u32,
height: u32,
) -> Result<&'a mut LiveSession, LiveSidecarError> {
if !sessions.contains_key(&key) {
let webview_id = host.create_webview_with_size(
tab_id.clone(),
profile_id.clone(),
ServoSurfaceSize::new(width, height),
)?;
sessions.insert(key.clone(), LiveSession::new(webview_id, width, height));
}
sessions.get_mut(&key).ok_or(LiveSidecarError::SessionUnavailable)
}
fn apply_layout(
host: &mut SoftwareServoHost,
session: &mut LiveSession,
width: u32,
height: u32,
page_zoom_percent: u16,
) -> Result<bool, LiveSidecarError> {
let mut changed = false;
if session.width != width || session.height != height {
host.resize(ResizeRequest { webview_id: session.webview_id.clone(), width, height })?;
session.width = width;
session.height = height;
changed = true;
}
if session.page_zoom_percent != page_zoom_percent {
host.set_page_zoom(PageZoomRequest {
webview_id: session.webview_id.clone(),
zoom_factor: f32::from(page_zoom_percent) / 100.0,
})?;
session.page_zoom_percent = page_zoom_percent;
changed = true;
}
Ok(changed)
}
fn apply_permissions(
host: &mut SoftwareServoHost,
session: &LiveSession,
profile_id: &ProfileId,
permissions: Vec<LiveSitePermission>,
) -> Result<(), LiveSidecarError> {
for permission in permissions {
host.set_permission(
PermissionRequest {
webview_id: session.webview_id.clone(),
profile_id: profile_id.clone(),
origin: ely_domain::SiteOrigin::parse(permission.origin)?,
feature: ely_domain::SitePermissionFeature::parse(permission.feature.as_str())?,
},
PermissionDecision::from(ely_domain::SitePermissionDecision::parse(
permission.decision.as_str(),
)?),
)?;
}
Ok(())
}
fn apply_input(
host: &mut SoftwareServoHost,
session: &mut LiveSession,
scroll_delta_x: i32,
scroll_delta_y: i32,
click_x: Option<u32>,
click_y: Option<u32>,
hover_x: Option<u32>,
hover_y: Option<u32>,
typed_text: Option<String>,
) -> Result<bool, LiveSidecarError> {
let mut changed = false;
if scroll_delta_x != 0 || scroll_delta_y != 0 {
host.scroll(ScrollRequest {
webview_id: session.webview_id.clone(),
delta_x: scroll_delta_x,
delta_y: scroll_delta_y,
})?;
session.scroll_x = positive_scroll_component(session.scroll_x, scroll_delta_x);
session.scroll_y = positive_scroll_component(session.scroll_y, scroll_delta_y);
changed = true;
}
if let (Some(x), Some(y)) = (hover_x, hover_y) {
host.hover(x, y)?;
changed = true;
}
if let (Some(x), Some(y)) = (click_x, click_y) {
host.click(MouseClickRequest { webview_id: session.webview_id.clone(), x, y })?;
changed = true;
}
if let Some(text) = typed_text {
host.type_text(KeyboardTextRequest { webview_id: session.webview_id.clone(), text })?;
changed = true;
}
Ok(changed)
}
fn poll_frame(
host: &mut SoftwareServoHost,
session: &mut LiveSession,
rgba_out: PathBuf,
) -> Result<LiveResponse, LiveSidecarError> {
let started_at = Instant::now();
let mut latest_frame = None;
loop {
host.tick();
let snapshot = host.snapshot(&session.webview_id)?;
if snapshot.has_pending_frame() {
host.paint(&session.webview_id)?;
let snapshot = host.snapshot(&session.webview_id)?;
let frame = host.last_rendered_frame()?;
let has_visible_content =
frame.non_white_pixel_count() > 0 && frame.content_pixel_count() > 0;
fs::write(&rgba_out, frame.rgba_bytes())?;
let response =
LiveResponse::frame(LiveFrameReport::new(&snapshot, &frame, rgba_out.clone()));
if has_visible_content {
session.awaiting_visible_frame = false;
return Ok(response);
}
if !session.awaiting_visible_frame {
return Ok(response);
}
latest_frame = Some(response);
}
if !session.awaiting_visible_frame {
return Ok(LiveResponse::empty());
}
if started_at.elapsed() >= LIVE_FRAME_WAIT_TIMEOUT {
return Ok(latest_frame.unwrap_or_else(LiveResponse::empty));
}
thread::sleep(LIVE_FRAME_WAIT_INTERVAL);
}
}
#[derive(Clone)]
struct LiveSession {
webview_id: ely_domain::WebViewId,
requested_url: String,
width: u32,
height: u32,
page_zoom_percent: u16,
scroll_x: i32,
scroll_y: i32,
awaiting_visible_frame: bool,
}
impl LiveSession {
fn new(webview_id: ely_domain::WebViewId, width: u32, height: u32) -> Self {
Self {
webview_id,
requested_url: String::new(),
width: width.max(1),
height: height.max(1),
page_zoom_percent: DEFAULT_ZOOM_PERCENT,
scroll_x: 0,
scroll_y: 0,
awaiting_visible_frame: false,
}
}
}
#[derive(Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
enum LiveRequest {
Ensure {
tab_id: String,
profile_id: String,
url: String,
width: u32,
height: u32,
page_zoom_percent: u16,
scroll_delta_x: i32,
scroll_delta_y: i32,
click_x: Option<u32>,
click_y: Option<u32>,
#[serde(default)]
hover_x: Option<u32>,
#[serde(default)]
hover_y: Option<u32>,
typed_text: Option<String>,
site_permissions: Vec<LiveSitePermission>,
rgba_out: String,
},
Poll {
tab_id: String,
rgba_out: String,
},
}
#[derive(Deserialize)]
struct LiveSitePermission {
origin: String,
feature: String,
decision: String,
}
#[derive(Serialize)]
struct LiveResponse {
error: Option<String>,
frame: Option<LiveFrameReport>,
}
impl LiveResponse {
fn empty() -> Self {
Self { error: None, frame: None }
}
fn frame(frame: LiveFrameReport) -> Self {
Self { error: None, frame: Some(frame) }
}
fn error(message: String) -> Self {
Self { error: Some(message), frame: None }
}
}
#[derive(Serialize)]
struct LiveFrameReport {
loaded_url: Option<String>,
title: Option<String>,
state: &'static str,
width: u32,
height: u32,
rgba_path: PathBuf,
rgba_byte_count: usize,
non_white_pixel_count: u64,
content_pixel_count: u64,
sample_hash: u64,
}
impl LiveFrameReport {
fn new(snapshot: &WebViewSnapshot, frame: &RenderedFrame, rgba_path: PathBuf) -> Self {
Self {
loaded_url: snapshot.url().map(str::to_string),
title: snapshot.title().map(str::to_string),
state: state_label(snapshot.state()),
width: frame.width(),
height: frame.height(),
rgba_path,
rgba_byte_count: frame.rgba_bytes().len(),
non_white_pixel_count: frame.non_white_pixel_count(),
content_pixel_count: frame.content_pixel_count(),
sample_hash: frame.sample_hash(),
}
}
}
#[derive(Debug, Error)]
pub(super) enum LiveSidecarError {
#[error("live session is unavailable after creation")]
SessionUnavailable,
#[error(transparent)]
Domain(#[from] ely_domain::DomainError),
#[error(transparent)]
Host(#[from] ServoHostError),
#[error(transparent)]
Io(#[from] io::Error),
#[error(transparent)]
Json(#[from] serde_json::Error),
}
fn state_label(state: &WebViewState) -> &'static str {
match state {
WebViewState::Created => "created",
WebViewState::Loading => "loading",
WebViewState::Complete => "complete",
WebViewState::Sleeping => "sleeping",
WebViewState::Crashed => "crashed",
}
}
fn positive_scroll_component(current: i32, delta: i32) -> i32 {
let value = i64::from(current) + i64::from(delta);
value.clamp(0, i64::from(i32::MAX)) as i32
}
crates/ely_servo_host/src/runtime.rs
+76 -35
View File
@@ -23,7 +23,9 @@ use crate::{
KeyboardTextRequest, MouseClickRequest, MouseDragRequest, NavigationRequest, PageZoomRequest,
PermissionDecision, PermissionRequest, RenderedFrame, ResizeRequest, ScreenshotRequest,
ScrollRequest, ServoHost, ServoHostError, TouchTapRequest, WebViewSnapshot, WebViewState,
runtime_input::{send_keyboard_text, send_mouse_click, send_mouse_drag, send_touch_tap},
runtime_input::{
send_keyboard_text, send_mouse_click, send_mouse_drag, send_mouse_hover, send_touch_tap,
},
runtime_permissions::{PermissionStore, set_permission_decision},
runtime_waker::ServoWakeFlag,
runtime_webview::{HostWebView, HostWebViewDelegate},
@@ -52,7 +54,7 @@ impl ServoSurfaceSize {
pub struct SoftwareServoHost {
servo: Servo,
rendering_context: Rc<dyn RenderingContext>,
default_surface_size: ServoSurfaceSize,
webviews: HashMap<WebViewId, HostWebView>,
permissions: PermissionStore,
wake_requested: Arc<AtomicBool>,
@@ -82,16 +84,19 @@ impl SoftwareServoHost {
host
}
pub fn create_webview_with_size(
&mut self,
tab_id: TabId,
profile_id: ProfileId,
size: ServoSurfaceSize,
) -> Result<WebViewId, ServoHostError> {
self.create_webview_in_context(tab_id, profile_id, size)
}
fn new_started(
size: ServoSurfaceSize,
config_dir: Option<PathBuf>,
) -> Result<Self, ServoHostError> {
let rendering_context = Rc::new(
servo::SoftwareRenderingContext::new(size.physical())
.map_err(|_| ServoHostError::RenderingContextUnavailable)?,
);
rendering_context.make_current().map_err(|_| ServoHostError::RenderingContextNotCurrent)?;
let wake_requested = Arc::new(AtomicBool::new(false));
let mut builder = ServoBuilder::default()
.event_loop_waker(Box::new(ServoWakeFlag::new(wake_requested.clone())));
@@ -102,7 +107,7 @@ impl SoftwareServoHost {
Ok(Self {
servo,
rendering_context,
default_surface_size: size,
webviews: HashMap::new(),
permissions: Rc::new(RefCell::new(HashMap::new())),
wake_requested,
@@ -117,19 +122,7 @@ impl ServoHost for SoftwareServoHost {
tab_id: TabId,
profile_id: ProfileId,
) -> Result<WebViewId, ServoHostError> {
let webview_id = WebViewId::new();
let delegate =
Rc::new(HostWebViewDelegate::new(profile_id.clone(), self.permissions.clone()));
let webview = WebViewBuilder::new(&self.servo, self.rendering_context.clone())
.delegate(delegate.clone())
.build();
self.webviews.insert(
webview_id.clone(),
HostWebView { tab_id, profile_id, webview, delegate, requested_url: None },
);
Ok(webview_id)
self.create_webview_in_context(tab_id, profile_id, self.default_surface_size)
}
fn navigate(&mut self, request: NavigationRequest) -> Result<(), ServoHostError> {
@@ -138,7 +131,6 @@ impl ServoHost for SoftwareServoHost {
})?;
self.servo.spin_event_loop();
let servo = self.servo.clone();
let rendering_context = self.rendering_context.clone();
let webview = self
.webviews
.get_mut(&request.webview_id)
@@ -151,7 +143,7 @@ impl ServoHost for SoftwareServoHost {
webview.delegate.set_state(WebViewState::Loading);
if should_create_initial_document {
webview.webview = WebViewBuilder::new(&servo, rendering_context)
webview.webview = WebViewBuilder::new(&servo, webview.rendering_context.clone())
.delegate(webview.delegate.clone())
.url(url)
.build();
@@ -188,7 +180,9 @@ impl ServoHost for SoftwareServoHost {
.get(&request.webview_id)
.ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?;
webview.webview.resize(PhysicalSize::new(request.width, request.height));
let size = PhysicalSize::new(request.width, request.height);
webview.rendering_context.resize(size);
webview.webview.resize(size);
Ok(())
}
@@ -202,6 +196,13 @@ impl ServoHost for SoftwareServoHost {
Ok(())
}
fn hover(&mut self, x: u32, y: u32) -> Result<(), ServoHostError> {
if let Some(webview) = self.webviews.values().next() {
send_mouse_hover(&webview.webview, x, y);
}
Ok(())
}
fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError> {
let webview = self
.webviews
@@ -319,16 +320,15 @@ impl ServoHost for SoftwareServoHost {
}
fn paint(&mut self, webview_id: &WebViewId) -> Result<(), ServoHostError> {
self.rendering_context
.make_current()
.map_err(|_| ServoHostError::RenderingContextNotCurrent)?;
self.rendering_context.prepare_for_rendering();
let rendering_context = self.webview(webview_id)?.rendering_context.clone();
rendering_context.make_current().map_err(|_| ServoHostError::RenderingContextNotCurrent)?;
rendering_context.prepare_for_rendering();
{
let webview = self.webview(webview_id)?;
webview.webview.paint();
}
let rendered_frame = self.read_rendered_frame()?;
self.rendering_context.present();
let rendered_frame = Self::read_rendered_frame(rendering_context.as_ref())?;
rendering_context.present();
self.webview(webview_id)?.delegate.mark_frame_presented();
self.last_rendered_frame = Some(rendered_frame);
Ok(())
@@ -340,14 +340,56 @@ impl ServoHost for SoftwareServoHost {
}
impl SoftwareServoHost {
fn create_webview_in_context(
&mut self,
tab_id: TabId,
profile_id: ProfileId,
size: ServoSurfaceSize,
) -> Result<WebViewId, ServoHostError> {
let webview_id = WebViewId::new();
let rendering_context = Self::new_rendering_context(size)?;
let delegate =
Rc::new(HostWebViewDelegate::new(profile_id.clone(), self.permissions.clone()));
let webview = WebViewBuilder::new(&self.servo, rendering_context.clone())
.delegate(delegate.clone())
.build();
self.webviews.insert(
webview_id.clone(),
HostWebView {
tab_id,
profile_id,
rendering_context,
webview,
delegate,
requested_url: None,
},
);
Ok(webview_id)
}
fn new_rendering_context(
size: ServoSurfaceSize,
) -> Result<Rc<dyn RenderingContext>, ServoHostError> {
let rendering_context = Rc::new(
servo::SoftwareRenderingContext::new(size.physical())
.map_err(|_| ServoHostError::RenderingContextUnavailable)?,
);
rendering_context.make_current().map_err(|_| ServoHostError::RenderingContextNotCurrent)?;
Ok(rendering_context)
}
fn webview(&self, webview_id: &WebViewId) -> Result<&HostWebView, ServoHostError> {
self.webviews
.get(webview_id)
.ok_or_else(|| ServoHostError::WebViewNotFound { id: webview_id.clone() })
}
fn read_rendered_frame(&self) -> Result<RenderedFrame, ServoHostError> {
let size = self.rendering_context.size();
fn read_rendered_frame(
rendering_context: &dyn RenderingContext,
) -> Result<RenderedFrame, ServoHostError> {
let size = rendering_context.size();
let width =
i32::try_from(size.width).map_err(|_| ServoHostError::RenderedFrameUnavailable)?;
let height =
@@ -356,8 +398,7 @@ impl SoftwareServoHost {
DeviceIntPoint::new(0, 0),
DeviceIntSize::new(width, height),
);
let image = self
.rendering_context
let image = rendering_context
.read_to_image(frame_rect)
.ok_or(ServoHostError::RenderedFrameUnavailable)?;
crates/ely_servo_host/src/runtime_input.rs
+4
View File
@@ -6,6 +6,10 @@ use servo::{
use crate::keyboard::key_and_code_for_character;
pub(super) fn send_mouse_hover(webview: &WebView, x: u32, y: u32) {
webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point(x, y))));
}
pub(super) fn send_mouse_click(webview: &WebView, x: u32, y: u32) {
let point = point(x, y);
webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point)));