ZacharyZhang-NY/ELY-Browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bridge Servo click input

Browse Source
This commit is contained in:
ZacharyZhang-NY
2026-05-08 14:23:12 -04:00
parent 7178174e20
commit b786073826
13 changed files with 506 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ely_app/src/services/servo_sidecar.rs
+26 -3
View File
@@ -64,7 +64,8 @@ impl ServoSidecarClient {
request: &SidecarSnapshotRequest,
rgba_path: &Path,
) -> Result<Output, ServoSidecarError> {
let mut child = Command::new(&self.binary_path)
let mut command = Command::new(&self.binary_path);
command
.arg("snapshot")
.arg("--url")
.arg(request.url.as_str())
@@ -77,7 +78,16 @@ impl ServoSidecarClient {
.arg("--scroll-x")
.arg(request.scroll_x.to_string())
.arg("--scroll-y")
.arg(request.scroll_y.to_string())
.arg(request.scroll_y.to_string());
if let Some(click_point) = request.click_point {
command
.arg("--click-x")
.arg(click_point.x.to_string())
.arg("--click-y")
.arg(click_point.y.to_string());
}
let mut child = command
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
@@ -109,12 +119,13 @@ pub struct SidecarSnapshotRequest {
height: u32,
scroll_x: i32,
scroll_y: i32,
click_point: Option<SidecarClickPoint>,
}
impl SidecarSnapshotRequest {
#[must_use]
pub fn new(url: UrlText, width: u32, height: u32) -> Self {
Self { url, width, height, scroll_x: 0, scroll_y: 0 }
Self { url, width, height, scroll_x: 0, scroll_y: 0, click_point: None }
}
#[must_use]
@@ -123,6 +134,18 @@ impl SidecarSnapshotRequest {
self.scroll_y = scroll_y;
self
}
#[must_use]
pub fn with_click_point(mut self, x: u32, y: u32) -> Self {
self.click_point = Some(SidecarClickPoint { x, y });
self
}
}
#[derive(Clone, Copy, Debug)]
struct SidecarClickPoint {
x: u32,
y: u32,
}
#[derive(Clone, Debug)]
crates/ely_app/src/shell/mod.rs
+1
View File
@@ -18,6 +18,7 @@ mod web_surface;
mod web_surface_frame;
mod web_surface_geometry;
mod web_surface_image;
mod web_surface_state;
mod web_surface_view;
use ely_browser_core::{BrowserCore, InitialBrowserConfig};
crates/ely_app/src/shell/web_surface.rs
+113 -73
View File
@@ -1,16 +1,20 @@
use std::collections::BTreeMap;
use ely_domain::{BrowserTab, TabId};
use gpui::{AnyElement, Bounds, Context, Pixels};
use gpui::{AnyElement, Bounds, Context, Pixels, Point};
use crate::services::servo_sidecar::{
ServoSidecarClient, ServoSidecarError, SidecarSnapshot, SidecarSnapshotRequest,
};
use crate::services::servo_sidecar::{ServoSidecarError, SidecarSnapshot, SidecarSnapshotRequest};
use super::{
ElyShell,
web_surface_frame::WebSurfaceFrame,
web_surface_geometry::{WebSurfaceScrollDelta, WebSurfaceScrollOffset, WebSurfaceSize},
web_surface_geometry::{
WebSurfaceClickPoint, WebSurfaceScrollDelta, WebSurfaceScrollOffset, WebSurfaceSize,
},
web_surface_state::{
WebSurfaceClickState, WebSurfaceClient, WebSurfaceRequest, WebSurfaceScrollState,
WebSurfaceState,
},
web_surface_view::{
render_failed_web_surface, render_loading_web_surface, render_ready_web_surface,
},
@@ -19,7 +23,9 @@ use super::{
pub(super) struct WebSurfaceStore {
client: WebSurfaceClient,
pending_viewport_sizes: BTreeMap<TabId, WebSurfaceSize>,
click_points: BTreeMap<TabId, WebSurfaceClickState>,
scroll_offsets: BTreeMap<TabId, WebSurfaceScrollState>,
viewport_bounds: BTreeMap<TabId, Bounds<Pixels>>,
viewport_sizes: BTreeMap<TabId, WebSurfaceSize>,
states: BTreeMap<TabId, WebSurfaceState>,
}
@@ -29,7 +35,9 @@ impl WebSurfaceStore {
Self {
client: WebSurfaceClient::new(),
pending_viewport_sizes: BTreeMap::new(),
click_points: BTreeMap::new(),
scroll_offsets: BTreeMap::new(),
viewport_bounds: BTreeMap::new(),
viewport_sizes: BTreeMap::new(),
states: BTreeMap::new(),
}
@@ -47,10 +55,11 @@ impl WebSurfaceStore {
let size = self.viewport_sizes.get(tab.id()).copied()?;
let requested_url = tab.url().as_str().to_string();
let scroll_offset = self.scroll_offset_for(tab.id(), requested_url.as_str());
let click_point = self.click_point_for(tab.id(), requested_url.as_str(), scroll_offset);
if self.is_loading_requested_url(tab.id(), requested_url.as_str()) {
return None;
}
if self.has_current_state(tab.id(), &requested_url, size, scroll_offset) {
if self.has_current_state(tab.id(), &requested_url, size, scroll_offset, click_point) {
return None;
}
@@ -63,6 +72,7 @@ impl WebSurfaceStore {
requested_url,
size,
scroll_offset,
click_point,
message: message.clone(),
},
);
@@ -76,22 +86,26 @@ impl WebSurfaceStore {
requested_url: requested_url.clone(),
size,
scroll_offset,
click_point,
previous_frame: self.previous_ready_frame(tab.id(), requested_url.as_str()),
},
);
let mut snapshot_request =
SidecarSnapshotRequest::new(tab.url().clone(), size.width, size.height)
.with_scroll_offset(scroll_offset.x(), scroll_offset.y());
if let Some(click_point) = click_point {
snapshot_request = snapshot_request.with_click_point(click_point.x(), click_point.y());
}
Some(WebSurfaceRequest {
tab_id: tab.id().clone(),
requested_url,
size,
scroll_offset,
click_point,
client,
snapshot_request: SidecarSnapshotRequest::new(
tab.url().clone(),
size.width,
size.height,
)
.with_scroll_offset(scroll_offset.x(), scroll_offset.y()),
snapshot_request,
})
}
@@ -101,32 +115,38 @@ impl WebSurfaceStore {
requested_url: &str,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
click_point: Option<WebSurfaceClickPoint>,
) -> bool {
match self.states.get(tab_id) {
Some(WebSurfaceState::Loading {
requested_url: current_url,
size: current_size,
scroll_offset: current_scroll_offset,
click_point: current_click_point,
..
}) => {
current_url == requested_url
&& *current_size == size
&& *current_scroll_offset == scroll_offset
&& *current_click_point == click_point
}
Some(WebSurfaceState::Ready(frame)) => {
frame.requested_url == requested_url
&& frame.size() == size
&& frame.scroll_offset() == scroll_offset
&& frame.click_point() == click_point
}
Some(WebSurfaceState::Failed {
requested_url: current_url,
size: current_size,
scroll_offset: current_scroll_offset,
click_point: current_click_point,
..
}) => {
current_url == requested_url
&& *current_size == size
&& *current_scroll_offset == scroll_offset
&& *current_click_point == click_point
}
None => false,
}
@@ -138,6 +158,7 @@ impl WebSurfaceStore {
requested_url: &str,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
click_point: Option<WebSurfaceClickPoint>,
) -> bool {
matches!(
self.states.get(tab_id),
@@ -145,11 +166,13 @@ impl WebSurfaceStore {
requested_url: current_url,
size: current_size,
scroll_offset: current_scroll_offset,
click_point: current_click_point,
..
})
if current_url == requested_url
&& *current_size == size
&& *current_scroll_offset == scroll_offset
&& *current_click_point == click_point
)
}
@@ -201,6 +224,7 @@ impl WebSurfaceStore {
}
state.offset = next_offset;
self.click_points.remove(tab_id);
true
}
@@ -208,6 +232,7 @@ impl WebSurfaceStore {
let Some(size) = WebSurfaceSize::from_bounds(bounds) else {
return false;
};
self.viewport_bounds.insert(tab_id.clone(), bounds);
let Some(current_size) = self.viewport_sizes.get(tab_id).copied() else {
self.viewport_sizes.insert(tab_id.clone(), size);
@@ -230,6 +255,32 @@ impl WebSurfaceStore {
true
}
fn record_click_point(
&mut self,
tab_id: &TabId,
requested_url: &str,
position: Point<Pixels>,
) -> bool {
let Some(bounds) = self.viewport_bounds.get(tab_id).copied() else {
return false;
};
let Some(point) = WebSurfaceClickPoint::from_window_position(bounds, position) else {
return false;
};
let state = WebSurfaceClickState {
requested_url: requested_url.to_string(),
scroll_offset: self.scroll_offset_for(tab_id, requested_url),
point,
};
if self.click_points.get(tab_id) == Some(&state) {
return false;
}
self.click_points.insert(tab_id.clone(), state);
true
}
fn scroll_offset_for(&self, tab_id: &TabId, requested_url: &str) -> WebSurfaceScrollOffset {
self.scroll_offsets
.get(tab_id)
@@ -237,56 +288,20 @@ impl WebSurfaceStore {
.map(|state| state.offset)
.unwrap_or_default()
}
}
struct WebSurfaceScrollState {
requested_url: String,
offset: WebSurfaceScrollOffset,
}
impl WebSurfaceScrollState {
fn new(requested_url: String) -> Self {
Self { requested_url, offset: WebSurfaceScrollOffset::default() }
}
}
enum WebSurfaceClient {
Ready(ServoSidecarClient),
Unavailable(String),
}
impl WebSurfaceClient {
fn new() -> Self {
match ServoSidecarClient::new() {
Ok(client) => Self::Ready(client),
Err(error) => Self::Unavailable(error.to_string()),
}
}
}
struct WebSurfaceRequest {
tab_id: TabId,
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
client: ServoSidecarClient,
snapshot_request: SidecarSnapshotRequest,
}
enum WebSurfaceState {
Loading {
requested_url: String,
size: WebSurfaceSize,
fn click_point_for(
&self,
tab_id: &TabId,
requested_url: &str,
scroll_offset: WebSurfaceScrollOffset,
previous_frame: Option<WebSurfaceFrame>,
},
Ready(WebSurfaceFrame),
Failed {
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
message: String,
},
) -> Option<WebSurfaceClickPoint> {
self.click_points
.get(tab_id)
.filter(|state| {
state.requested_url == requested_url && state.scroll_offset == scroll_offset
})
.map(|state| state.point)
}
}
impl ElyShell {
@@ -324,6 +339,7 @@ impl ElyShell {
requested_url,
size,
scroll_offset,
click_point,
client,
snapshot_request,
} = request;
@@ -339,6 +355,7 @@ impl ElyShell {
requested_url,
size,
scroll_offset,
click_point,
result,
);
cx.notify();
@@ -353,29 +370,40 @@ impl ElyShell {
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
click_point: Option<WebSurfaceClickPoint>,
result: Result<SidecarSnapshot, ServoSidecarError>,
) {
if !self.web_surfaces.is_loading(&tab_id, requested_url.as_str(), size, scroll_offset) {
if !self.web_surfaces.is_loading(
&tab_id,
requested_url.as_str(),
size,
scroll_offset,
click_point,
) {
return;
}
let state = match result {
Ok(snapshot) => {
match WebSurfaceFrame::from_snapshot(requested_url.clone(), scroll_offset, snapshot)
{
Ok(frame) => WebSurfaceState::Ready(frame),
Err(error) => WebSurfaceState::Failed {
requested_url,
size,
scroll_offset,
message: error.to_string(),
},
}
}
Ok(snapshot) => match WebSurfaceFrame::from_snapshot(
requested_url.clone(),
scroll_offset,
click_point,
snapshot,
) {
Ok(frame) => WebSurfaceState::Ready(frame),
Err(error) => WebSurfaceState::Failed {
requested_url,
size,
scroll_offset,
click_point,
message: error.to_string(),
},
},
Err(error) => WebSurfaceState::Failed {
requested_url,
size,
scroll_offset,
click_point,
message: error.to_string(),
},
};
@@ -404,6 +432,18 @@ impl ElyShell {
cx.notify();
}
}
pub(super) fn click_external_web_viewport(
&mut self,
tab_id: TabId,
requested_url: String,
position: Point<Pixels>,
cx: &mut Context<Self>,
) {
if self.web_surfaces.record_click_point(&tab_id, requested_url.as_str(), position) {
cx.notify();
}
}
}
pub(super) fn is_external_web_url(url: &str) -> bool {
crates/ely_app/src/shell/web_surface_frame.rs
+13 -2
View File
@@ -7,7 +7,7 @@ use thiserror::Error;
use crate::services::servo_sidecar::SidecarSnapshot;
use super::{
web_surface_geometry::{WebSurfaceScrollOffset, WebSurfaceSize},
web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize},
web_surface_image::renderable_image_buffer,
};
@@ -19,6 +19,7 @@ pub(super) struct WebSurfaceFrame {
width: u32,
height: u32,
scroll_offset: WebSurfaceScrollOffset,
click_point: Option<WebSurfaceClickPoint>,
pub(super) image: Arc<RenderImage>,
}
@@ -26,6 +27,7 @@ impl WebSurfaceFrame {
pub(super) fn from_snapshot(
requested_url: String,
scroll_offset: WebSurfaceScrollOffset,
click_point: Option<WebSurfaceClickPoint>,
snapshot: SidecarSnapshot,
) -> Result<Self, WebSurfaceError> {
let width = snapshot.width();
@@ -47,6 +49,7 @@ impl WebSurfaceFrame {
width,
height,
scroll_offset,
click_point,
image: Arc::new(RenderImage::new([image::Frame::new(image_buffer)])),
})
}
@@ -60,7 +63,11 @@ impl WebSurfaceFrame {
}
pub(super) fn detail_label(&self) -> String {
self.scroll_offset.detail_label(self.size())
let detail = self.scroll_offset.detail_label(self.size());
match self.click_point {
Some(click_point) => format!("{detail} {}", click_point.detail_label()),
None => detail,
}
}
pub(super) fn size(&self) -> WebSurfaceSize {
@@ -70,6 +77,10 @@ impl WebSurfaceFrame {
pub(super) fn scroll_offset(&self) -> WebSurfaceScrollOffset {
self.scroll_offset
}
pub(super) fn click_point(&self) -> Option<WebSurfaceClickPoint> {
self.click_point
}
}
#[derive(Debug, Error)]
crates/ely_app/src/shell/web_surface_geometry.rs
+40
View File
@@ -15,6 +15,36 @@ impl WebSurfaceSize {
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) struct WebSurfaceClickPoint {
x: u32,
y: u32,
}
impl WebSurfaceClickPoint {
pub(super) fn from_window_position(
bounds: Bounds<Pixels>,
position: Point<Pixels>,
) -> Option<Self> {
Some(Self {
x: click_coordinate(position.x, bounds.origin.x, bounds.size.width)?,
y: click_coordinate(position.y, bounds.origin.y, bounds.size.height)?,
})
}
pub(super) fn detail_label(self) -> String {
format!("click={},{}", self.x, self.y)
}
pub(super) fn x(self) -> u32 {
self.x
}
pub(super) fn y(self) -> u32 {
self.y
}
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub(super) struct WebSurfaceScrollOffset {
x: i32,
@@ -88,6 +118,16 @@ fn scroll_dimension(pixels: Pixels) -> Option<i32> {
Some(value as i32)
}
fn click_coordinate(position: Pixels, origin: Pixels, size: Pixels) -> Option<u32> {
let relative = f32::from(position) - f32::from(origin);
let size = f32::from(size);
if !relative.is_finite() || !size.is_finite() || relative < 0.0 || relative >= size {
return None;
}
Some(relative.floor() as u32)
}
fn positive_scroll_component(current: i32, delta: i32) -> i32 {
let value = i64::from(current) + i64::from(delta);
let clamped = value.clamp(0, i64::from(i32::MAX));
crates/ely_app/src/shell/web_surface_state.rs
+68
View File
@@ -0,0 +1,68 @@
use ely_domain::TabId;
use crate::services::servo_sidecar::{ServoSidecarClient, SidecarSnapshotRequest};
use super::{
web_surface_frame::WebSurfaceFrame,
web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize},
};
pub(super) struct WebSurfaceScrollState {
pub(super) requested_url: String,
pub(super) offset: WebSurfaceScrollOffset,
}
impl WebSurfaceScrollState {
pub(super) fn new(requested_url: String) -> Self {
Self { requested_url, offset: WebSurfaceScrollOffset::default() }
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub(super) struct WebSurfaceClickState {
pub(super) requested_url: String,
pub(super) scroll_offset: WebSurfaceScrollOffset,
pub(super) point: WebSurfaceClickPoint,
}
pub(super) enum WebSurfaceClient {
Ready(ServoSidecarClient),
Unavailable(String),
}
impl WebSurfaceClient {
pub(super) fn new() -> Self {
match ServoSidecarClient::new() {
Ok(client) => Self::Ready(client),
Err(error) => Self::Unavailable(error.to_string()),
}
}
}
pub(super) struct WebSurfaceRequest {
pub(super) tab_id: TabId,
pub(super) requested_url: String,
pub(super) size: WebSurfaceSize,
pub(super) scroll_offset: WebSurfaceScrollOffset,
pub(super) click_point: Option<WebSurfaceClickPoint>,
pub(super) client: ServoSidecarClient,
pub(super) snapshot_request: SidecarSnapshotRequest,
}
pub(super) enum WebSurfaceState {
Loading {
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
click_point: Option<WebSurfaceClickPoint>,
previous_frame: Option<WebSurfaceFrame>,
},
Ready(WebSurfaceFrame),
Failed {
requested_url: String,
size: WebSurfaceSize,
scroll_offset: WebSurfaceScrollOffset,
click_point: Option<WebSurfaceClickPoint>,
message: String,
},
}
crates/ely_app/src/shell/web_surface_view.rs
+50 -17
View File
@@ -1,6 +1,6 @@
use ely_domain::{BrowserTab, TabId};
use gpui::{
AnyElement, App, Entity, ImageSource, InteractiveElement, IntoElement, ObjectFit,
AnyElement, App, Entity, ImageSource, InteractiveElement, IntoElement, MouseButton, ObjectFit,
ParentElement, Styled, StyledImage, Window, canvas, div, img, prelude::FluentBuilder, px, rgb,
};
use gpui_component::StyledExt;
@@ -104,9 +104,9 @@ fn render_web_surface(
detail: Option<String>,
content: impl IntoElement,
) -> AnyElement {
let scroll_tab_id = tab.id().clone();
let scroll_url = tab.url().as_str().to_string();
let scroll_entity = state_entity.clone();
let input_tab_id = tab.id().clone();
let input_url = tab.url().as_str().to_string();
let input_entity = state_entity.clone();
let tracker_entity = state_entity;
div()
@@ -132,25 +132,58 @@ fn render_web_surface(
.min_h_0()
.overflow_hidden()
.bg(rgb(colors::SURFACE_CARD))
.on_scroll_wheel(move |event, window, cx| {
let delta = event.delta.pixel_delta(window.line_height());
scroll_entity.update(cx, |shell, cx| {
shell.scroll_external_web_viewport(
scroll_tab_id.clone(),
scroll_url.clone(),
delta,
cx,
);
});
cx.stop_propagation();
})
.child(content)
.child(render_viewport_tracker(tab.id().clone(), tracker_entity)),
.child(render_viewport_tracker(tab.id().clone(), tracker_entity))
.child(render_input_overlay(input_tab_id, input_url, input_entity)),
),
)
.into_any_element()
}
fn render_input_overlay(
tab_id: TabId,
url: String,
state_entity: Entity<ElyShell>,
) -> impl IntoElement {
let click_tab_id = tab_id.clone();
let click_url = url.clone();
let click_entity = state_entity.clone();
let scroll_tab_id = tab_id;
let scroll_url = url;
let scroll_entity = state_entity;
div()
.absolute()
.size_full()
.occlude()
.capture_any_mouse_up(move |event, _window, cx| {
if event.button != MouseButton::Left {
return;
}
click_entity.update(cx, |shell, cx| {
shell.click_external_web_viewport(
click_tab_id.clone(),
click_url.clone(),
event.position,
cx,
);
});
cx.stop_propagation();
})
.on_scroll_wheel(move |event, window, cx| {
let delta = event.delta.pixel_delta(window.line_height());
scroll_entity.update(cx, |shell, cx| {
shell.scroll_external_web_viewport(
scroll_tab_id.clone(),
scroll_url.clone(),
delta,
cx,
);
});
cx.stop_propagation();
})
}
fn render_viewport_tracker(tab_id: TabId, state_entity: Entity<ElyShell>) -> impl IntoElement {
canvas(
move |bounds, _window: &mut Window, cx: &mut App| {