Add mouse hover tracking across the Servo input pipeline

Without continuous MouseMove events, Servo never updates hover state:
no cursor changes over links, no :hover CSS effects, no mouseenter
JavaScript events. The web page appears completely non-interactive.

Track mouse position from GPUI on_mouse_move through the full IPC
pipeline to Servo. Hover position is included with each ensure
request so Servo updates hover state at frame rate (~60fps).

crates/ely_app/src/services/servo_live.rs

@@ -0,0 +1,321 @@
+use std::{
+    env, fs,
+    io::{self, BufRead, BufReader, Write},
+    path::PathBuf,
+    process::{Child, ChildStdin, ChildStdout, Stdio},
+    time::{SystemTime, SystemTimeError, UNIX_EPOCH},
+};
+
+use ely_domain::SitePermissionDecision;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use super::servo_sidecar_command::{SidecarCommandError, default_sidecar_command};
+
+pub(crate) struct ServoLiveClient {
+    child: Child,
+    stdin: ChildStdin,
+    stdout: BufReader<ChildStdout>,
+    frame_dir: PathBuf,
+    frame_path: PathBuf,
+}
+
+impl ServoLiveClient {
+    pub fn new(profile_data_dir: PathBuf) -> Result<Self, ServoLiveError> {
+        let command_target = default_sidecar_command()?;
+        if let Some(path) = command_target.missing_binary_path() {
+            return Err(ServoLiveError::SidecarBinaryUnavailable { path: path.to_path_buf() });
+        }
+
+        let frame_dir = temporary_frame_dir()?;
+        let frame_path = frame_dir.join("frame.rgba");
+        let mut command = command_target.command();
+        command.arg("live").arg("--profile-data-dir").arg(profile_data_dir);
+        let mut child = command
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::null())
+            .spawn()
+            .map_err(ServoLiveError::Command)?;
+        let stdin = child.stdin.take().ok_or(ServoLiveError::PipeUnavailable { name: "stdin" })?;
+        let stdout =
+            child.stdout.take().ok_or(ServoLiveError::PipeUnavailable { name: "stdout" })?;
+
+        Ok(Self { child, stdin, stdout: BufReader::new(stdout), frame_dir, frame_path })
+    }
+
+    pub fn ensure(
+        &mut self,
+        request: ServoLiveEnsureRequest,
+    ) -> Result<Option<ServoLiveFrame>, ServoLiveError> {
+        self.request(LiveRequest::Ensure {
+            tab_id: request.tab_id,
+            profile_id: request.profile_id,
+            url: request.url,
+            width: request.width,
+            height: request.height,
+            page_zoom_percent: request.page_zoom_percent,
+            scroll_delta_x: request.scroll_delta_x,
+            scroll_delta_y: request.scroll_delta_y,
+            click_x: request.click_x,
+            click_y: request.click_y,
+            hover_x: request.hover_x,
+            hover_y: request.hover_y,
+            typed_text: request.typed_text,
+            site_permissions: request.site_permissions,
+            rgba_out: self.frame_path.display().to_string(),
+        })
+    }
+
+    pub fn poll(&mut self, tab_id: String) -> Result<Option<ServoLiveFrame>, ServoLiveError> {
+        self.request(LiveRequest::Poll { tab_id, rgba_out: self.frame_path.display().to_string() })
+    }
+
+    fn request(&mut self, request: LiveRequest) -> Result<Option<ServoLiveFrame>, ServoLiveError> {
+        serde_json::to_writer(&mut self.stdin, &request)?;
+        self.stdin.write_all(b"\n").map_err(ServoLiveError::Command)?;
+        self.stdin.flush().map_err(ServoLiveError::Command)?;
+
+        let mut line = String::new();
+        let bytes = self.stdout.read_line(&mut line).map_err(ServoLiveError::Command)?;
+        if bytes == 0 {
+            return Err(ServoLiveError::SidecarExited);
+        }
+
+        let response: LiveResponse = serde_json::from_str(&line)?;
+        if let Some(error) = response.error {
+            return Err(ServoLiveError::SidecarFailed { message: error });
+        }
+
+        response.frame.map(ServoLiveFrame::from_report).transpose()
+    }
+}
+
+impl Drop for ServoLiveClient {
+    fn drop(&mut self) {
+        let _ = self.child.kill();
+        let _ = self.child.wait();
+        let _ = fs::remove_dir_all(&self.frame_dir);
+    }
+}
+
+pub(crate) struct ServoLiveEnsureRequest {
+    pub(crate) tab_id: String,
+    pub(crate) profile_id: String,
+    pub(crate) url: String,
+    pub(crate) width: u32,
+    pub(crate) height: u32,
+    pub(crate) page_zoom_percent: u16,
+    pub(crate) scroll_delta_x: i32,
+    pub(crate) scroll_delta_y: i32,
+    pub(crate) click_x: Option<u32>,
+    pub(crate) click_y: Option<u32>,
+    pub(crate) hover_x: Option<u32>,
+    pub(crate) hover_y: Option<u32>,
+    pub(crate) typed_text: Option<String>,
+    pub(crate) site_permissions: Vec<ServoLiveSitePermission>,
+}
+
+#[derive(Clone, Debug, Serialize)]
+pub(crate) struct ServoLiveSitePermission {
+    pub(crate) origin: String,
+    pub(crate) feature: String,
+    pub(crate) decision: String,
+}
+
+impl ServoLiveSitePermission {
+    pub fn new(
+        origin: impl Into<String>,
+        feature: impl Into<String>,
+        decision: SitePermissionDecision,
+    ) -> Self {
+        Self { origin: origin.into(), feature: feature.into(), decision: decision.as_str().into() }
+    }
+}
+
+pub(crate) struct ServoLiveFrame {
+    loaded_url: Option<String>,
+    title: Option<String>,
+    render_state: String,
+    width: u32,
+    height: u32,
+    #[cfg(all(test, feature = "live-site-smoke"))]
+    non_white_pixel_count: u64,
+    #[cfg(all(test, feature = "live-site-smoke"))]
+    content_pixel_count: u64,
+    #[cfg(all(test, feature = "live-site-smoke"))]
+    sample_hash: u64,
+    rgba_bytes: Vec<u8>,
+}
+
+impl ServoLiveFrame {
+    fn from_report(report: LiveFrameReport) -> Result<Self, ServoLiveError> {
+        let rgba_bytes = fs::read(&report.rgba_path).map_err(ServoLiveError::FrameRead)?;
+        if rgba_bytes.len() != report.rgba_byte_count {
+            return Err(ServoLiveError::RgbaByteCountMismatch {
+                expected: report.rgba_byte_count,
+                actual: rgba_bytes.len(),
+            });
+        }
+
+        Ok(Self {
+            loaded_url: report.loaded_url,
+            title: report.title,
+            render_state: report.state,
+            width: report.width,
+            height: report.height,
+            #[cfg(all(test, feature = "live-site-smoke"))]
+            non_white_pixel_count: report.non_white_pixel_count,
+            #[cfg(all(test, feature = "live-site-smoke"))]
+            content_pixel_count: report.content_pixel_count,
+            #[cfg(all(test, feature = "live-site-smoke"))]
+            sample_hash: report.sample_hash,
+            rgba_bytes,
+        })
+    }
+
+    #[must_use]
+    pub fn loaded_url(&self) -> Option<&str> {
+        self.loaded_url.as_deref()
+    }
+
+    #[must_use]
+    pub fn title(&self) -> Option<&str> {
+        self.title.as_deref()
+    }
+
+    #[must_use]
+    pub fn render_state(&self) -> &str {
+        self.render_state.as_str()
+    }
+
+    #[must_use]
+    pub fn width(&self) -> u32 {
+        self.width
+    }
+
+    #[must_use]
+    pub fn height(&self) -> u32 {
+        self.height
+    }
+
+    #[cfg(all(test, feature = "live-site-smoke"))]
+    #[must_use]
+    pub fn non_white_pixel_count(&self) -> u64 {
+        self.non_white_pixel_count
+    }
+
+    #[cfg(all(test, feature = "live-site-smoke"))]
+    #[must_use]
+    pub fn content_pixel_count(&self) -> u64 {
+        self.content_pixel_count
+    }
+
+    #[cfg(all(test, feature = "live-site-smoke"))]
+    #[must_use]
+    pub fn sample_hash(&self) -> u64 {
+        self.sample_hash
+    }
+
+    #[must_use]
+    pub fn into_rgba_bytes(self) -> Vec<u8> {
+        self.rgba_bytes
+    }
+}
+
+#[derive(Debug, Error)]
+pub(crate) enum ServoLiveError {
+    #[error("servo sidecar binary is unavailable at {path}")]
+    SidecarBinaryUnavailable { path: PathBuf },
+
+    #[error("failed to run servo live sidecar: {0}")]
+    Command(#[source] io::Error),
+
+    #[error("servo live sidecar pipe is unavailable: {name}")]
+    PipeUnavailable { name: &'static str },
+
+    #[error("servo live sidecar exited")]
+    SidecarExited,
+
+    #[error("servo live sidecar failed: {message}")]
+    SidecarFailed { message: String },
+
+    #[error("failed to read servo live frame file: {0}")]
+    FrameRead(#[source] io::Error),
+
+    #[error("servo live frame byte count mismatch: expected {expected}, actual {actual}")]
+    RgbaByteCountMismatch { expected: usize, actual: usize },
+
+    #[error("temporary live frame directory is unavailable: {0}")]
+    TempDirectory(#[source] io::Error),
+
+    #[error("temporary live frame timestamp is unavailable: {0}")]
+    SystemClock(#[source] SystemTimeError),
+
+    #[error(transparent)]
+    Json(#[from] serde_json::Error),
+
+    #[error(transparent)]
+    SidecarCommand(#[from] SidecarCommandError),
+}
+
+#[derive(Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum LiveRequest {
+    Ensure {
+        tab_id: String,
+        profile_id: String,
+        url: String,
+        width: u32,
+        height: u32,
+        page_zoom_percent: u16,
+        scroll_delta_x: i32,
+        scroll_delta_y: i32,
+        click_x: Option<u32>,
+        click_y: Option<u32>,
+        hover_x: Option<u32>,
+        hover_y: Option<u32>,
+        typed_text: Option<String>,
+        site_permissions: Vec<ServoLiveSitePermission>,
+        rgba_out: String,
+    },
+    Poll {
+        tab_id: String,
+        rgba_out: String,
+    },
+}
+
+#[derive(Deserialize)]
+struct LiveResponse {
+    error: Option<String>,
+    frame: Option<LiveFrameReport>,
+}
+
+#[derive(Deserialize)]
+struct LiveFrameReport {
+    loaded_url: Option<String>,
+    title: Option<String>,
+    state: String,
+    width: u32,
+    height: u32,
+    rgba_path: PathBuf,
+    rgba_byte_count: usize,
+    #[cfg(all(test, feature = "live-site-smoke"))]
+    non_white_pixel_count: u64,
+    #[cfg(all(test, feature = "live-site-smoke"))]
+    content_pixel_count: u64,
+    #[cfg(all(test, feature = "live-site-smoke"))]
+    sample_hash: u64,
+}
+
+fn temporary_frame_dir() -> Result<PathBuf, ServoLiveError> {
+    let timestamp = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_err(ServoLiveError::SystemClock)?
+        .as_nanos();
+    let directory = env::temp_dir()
+        .join("ely-browser-servo-live")
+        .join(format!("{}-{timestamp}", std::process::id()));
+    fs::create_dir_all(&directory).map_err(ServoLiveError::TempDirectory)?;
+    Ok(directory)
+}

crates/ely_app/src/shell/web_surface.rs

@@ -3,24 +3,28 @@ use std::collections::BTreeMap;
 use ely_domain::{BrowserTab, TabId};
 use gpui::{Bounds, Pixels, Point};
 
-use crate::services::{ProfileDataMode, servo_sidecar::SidecarSnapshotRequest};
+use crate::services::ProfileDataMode;
 
 use super::{
     web_surface_frame::WebSurfaceFrame,
     web_surface_geometry::{
         WebSurfaceClickPoint, WebSurfaceScrollDelta, WebSurfaceScrollOffset, WebSurfaceSize,
     },
+    web_surface_permissions::WebSurfaceSitePermission,
+    web_surface_runtime::{WebSurfaceRuntime, WebSurfaceRuntimeFrame},
     web_surface_state::{
-        WebSurfaceClickState, WebSurfaceClient, WebSurfaceKeyboardFocusState, WebSurfaceRequest,
+        WebSurfaceClickState, WebSurfaceKeyboardFocusState, WebSurfacePendingInput,
-        WebSurfaceScrollState, WebSurfaceState, WebSurfaceStateKey, WebSurfaceTextInputState,
+        WebSurfaceScrollState, WebSurfaceState, WebSurfaceTextInputState,
     },
 };
 
 pub(super) struct WebSurfaceStore {
-    client: WebSurfaceClient,
+    runtime: WebSurfaceRuntime,
     pending_viewport_sizes: BTreeMap<TabId, WebSurfaceSize>,
     click_points: BTreeMap<TabId, WebSurfaceClickState>,
+    hover_points: BTreeMap<TabId, WebSurfaceClickPoint>,
     keyboard_focus: Option<WebSurfaceKeyboardFocusState>,
+    pending_scroll_deltas: BTreeMap<TabId, WebSurfaceScrollDelta>,
     scroll_offsets: BTreeMap<TabId, WebSurfaceScrollState>,
     typed_texts: BTreeMap<TabId, WebSurfaceTextInputState>,
     viewport_bounds: BTreeMap<TabId, Bounds<Pixels>>,
@@ -31,10 +35,12 @@ pub(super) struct WebSurfaceStore {
 impl WebSurfaceStore {
     pub(super) fn new() -> Self {
         Self {
-            client: WebSurfaceClient::new(),
+            runtime: WebSurfaceRuntime::new(),
             pending_viewport_sizes: BTreeMap::new(),
             click_points: BTreeMap::new(),
+            hover_points: BTreeMap::new(),
             keyboard_focus: None,
+            pending_scroll_deltas: BTreeMap::new(),
             scroll_offsets: BTreeMap::new(),
             typed_texts: BTreeMap::new(),
             viewport_bounds: BTreeMap::new(),
@@ -47,208 +53,72 @@ impl WebSurfaceStore {
         self.states.get(tab_id)
     }
 
-    pub(super) fn prepare_request(
+    pub(super) fn ensure_surface(
         &mut self,
         tab: &BrowserTab,
         profile_data_mode: ProfileDataMode,
-    ) -> Option<WebSurfaceRequest> {
+        permissions: &[WebSurfaceSitePermission],
+    ) {
         if !is_external_web_url(tab.url().as_str()) {
-            return None;
+            return;
         }
 
-        let size = self.viewport_sizes.get(tab.id()).copied()?;
         let requested_url = tab.url().as_str().to_string();
-        let scroll_offset = self.scroll_offset_for(tab.id(), requested_url.as_str());
+        let Some(size) = self.viewport_sizes.get(tab.id()).copied() else {
-        let zoom_percent = tab.zoom_percent();
+            return;
-        let click_point = self.click_point_for(tab.id(), requested_url.as_str(), scroll_offset);
-        let typed_text =
-            self.typed_text_for(tab.id(), requested_url.as_str(), scroll_offset, click_point);
-        if self.is_loading_requested_url(tab.id(), requested_url.as_str()) {
-            return None;
-        }
-        let state_key = WebSurfaceStateKey {
-            requested_url: &requested_url,
-            size,
-            scroll_offset,
-            zoom_percent,
-            click_point,
-            typed_text: typed_text.as_deref(),
         };
-        if self.has_current_state(tab.id(), state_key) {
+        let input = self.take_pending_input(tab.id(), requested_url.as_str());
-            return None;
+        let previous_frame =
-        }
+            self.previous_ready_frame(tab.id(), requested_url.as_str(), tab.zoom_percent());
 
-        let client = match &self.client {
+        match self.runtime.ensure_tab(tab, size, profile_data_mode, permissions, input) {
-            WebSurfaceClient::Ready(client) => client.clone(),
+            Ok(result) if result.frame.is_some() => {
-            WebSurfaceClient::Unavailable(message) => {
+                let Some(frame) = result.frame else {
-                self.states.insert(
+                    return;
-                    tab.id().clone(),
-                    WebSurfaceState::Failed {
-                        requested_url,
-                        size,
-                        scroll_offset,
-                        zoom_percent,
-                        click_point,
-                        typed_text: typed_text.clone(),
-                        message: message.clone(),
-                    },
-                );
-                return None;
-            }
                 };
-
+                self.states.insert(tab.id().clone(), WebSurfaceState::Ready(frame));
+            }
+            Ok(result) if result.started_loading => {
                 self.states.insert(
                     tab.id().clone(),
                     WebSurfaceState::Loading {
-                requested_url: requested_url.clone(),
+                        requested_url: result.requested_url,
-                size,
+                        previous_frame,
-                scroll_offset,
-                zoom_percent,
-                click_point,
-                typed_text: typed_text.clone(),
-                previous_frame: self.previous_ready_frame(
-                    tab.id(),
-                    requested_url.as_str(),
-                    zoom_percent,
-                ),
                     },
                 );
-
-        let mut snapshot_request = SidecarSnapshotRequest::new(
-            tab.url().clone(),
-            tab.profile_id().clone(),
-            size.width,
-            size.height,
-        )
-        .with_profile_data_mode(profile_data_mode)
-        .with_scroll_offset(scroll_offset.x(), scroll_offset.y())
-        .with_page_zoom_percent(zoom_percent);
-        if let Some(click_point) = click_point {
-            snapshot_request = snapshot_request.with_click_point(click_point.x(), click_point.y());
             }
-        if let Some(typed_text) = typed_text.clone() {
+            Ok(_) => {}
-            snapshot_request = snapshot_request.with_typed_text(typed_text);
+            Err(message) => {
+                self.states.insert(tab.id().clone(), WebSurfaceState::Failed { message });
             }
-
-        Some(WebSurfaceRequest {
-            tab_id: tab.id().clone(),
-            requested_url,
-            size,
-            scroll_offset,
-            zoom_percent,
-            click_point,
-            typed_text,
-            client,
-            snapshot_request,
-        })
-    }
-
-    fn has_current_state(&self, tab_id: &TabId, key: WebSurfaceStateKey<'_>) -> bool {
-        match self.states.get(tab_id) {
-            Some(WebSurfaceState::Loading {
-                requested_url: current_url,
-                size: current_size,
-                scroll_offset: current_scroll_offset,
-                zoom_percent: current_zoom_percent,
-                click_point: current_click_point,
-                typed_text: current_typed_text,
-                ..
-            }) => {
-                current_url == key.requested_url
-                    && *current_size == key.size
-                    && *current_scroll_offset == key.scroll_offset
-                    && *current_zoom_percent == key.zoom_percent
-                    && *current_click_point == key.click_point
-                    && current_typed_text.as_deref() == key.typed_text
-            }
-            Some(WebSurfaceState::Ready(frame)) => {
-                frame.requested_url == key.requested_url
-                    && frame.size() == key.size
-                    && frame.scroll_offset() == key.scroll_offset
-                    && frame.zoom_percent() == key.zoom_percent
-                    && frame.click_point() == key.click_point
-                    && frame.typed_text() == key.typed_text
-            }
-            Some(WebSurfaceState::Failed {
-                requested_url: current_url,
-                size: current_size,
-                scroll_offset: current_scroll_offset,
-                zoom_percent: current_zoom_percent,
-                click_point: current_click_point,
-                typed_text: current_typed_text,
-                ..
-            }) => {
-                current_url == key.requested_url
-                    && *current_size == key.size
-                    && *current_scroll_offset == key.scroll_offset
-                    && *current_zoom_percent == key.zoom_percent
-                    && *current_click_point == key.click_point
-                    && current_typed_text.as_deref() == key.typed_text
-            }
-            None => false,
         }
     }
 
-    pub(super) fn is_loading(&self, tab_id: &TabId, key: WebSurfaceStateKey<'_>) -> bool {
+    pub(super) fn tick(&mut self) -> bool {
-        matches!(
+        let frames = self.runtime.tick();
-            self.states.get(tab_id),
+        let mut changed = false;
-            Some(WebSurfaceState::Loading {
-                requested_url: current_url,
-                size: current_size,
-                scroll_offset: current_scroll_offset,
-                zoom_percent: current_zoom_percent,
-                click_point: current_click_point,
-                typed_text: current_typed_text,
-                ..
-            })
-                if current_url == key.requested_url
-                    && *current_size == key.size
-                    && *current_scroll_offset == key.scroll_offset
-                    && *current_zoom_percent == key.zoom_percent
-                    && *current_click_point == key.click_point
-                    && current_typed_text.as_deref() == key.typed_text
-        )
-    }
 
-    fn is_loading_requested_url(&self, tab_id: &TabId, requested_url: &str) -> bool {
+        for frame in frames {
-        matches!(
+            match frame {
-            self.states.get(tab_id),
+                WebSurfaceRuntimeFrame::Ready { tab_id, frame } => {
-            Some(WebSurfaceState::Loading { requested_url: current_url, .. })
+                    self.states.insert(tab_id, WebSurfaceState::Ready(frame));
-                if current_url == requested_url
+                    changed = true;
-        )
                 }
-
+                WebSurfaceRuntimeFrame::Failed { tab_id, message } => {
-    fn previous_ready_frame(
+                    self.states.insert(tab_id, WebSurfaceState::Failed { message });
-        &self,
+                    changed = true;
-        tab_id: &TabId,
-        requested_url: &str,
-        zoom_percent: u16,
-    ) -> Option<WebSurfaceFrame> {
-        match self.states.get(tab_id) {
-            Some(WebSurfaceState::Ready(frame))
-                if frame.requested_url == requested_url && frame.zoom_percent() == zoom_percent =>
-            {
-                Some(frame.clone())
                 }
-            Some(WebSurfaceState::Loading {
-                requested_url: current_url, previous_frame, ..
-            }) if current_url == requested_url => previous_frame
-                .as_ref()
-                .filter(|frame| frame.zoom_percent() == zoom_percent)
-                .cloned(),
-            _ => None,
             }
         }
 
-    pub(super) fn finish(&mut self, tab_id: TabId, state: WebSurfaceState) {
+        changed
-        self.states.insert(tab_id, state);
     }
 
     pub(super) fn record_scroll_delta(
         &mut self,
         tab_id: &TabId,
         requested_url: &str,
-        delta: gpui::Point<Pixels>,
+        delta: Point<Pixels>,
     ) -> bool {
         let Some(delta) = WebSurfaceScrollDelta::from_point(delta) else {
             return false;
@@ -262,12 +132,11 @@ impl WebSurfaceStore {
             *state = WebSurfaceScrollState::new(requested_url.to_string());
         }
 
-        let next_offset = state.offset.scrolled_by(delta);
+        state.offset = state.offset.scrolled_by(delta);
-        if next_offset == state.offset {
+        self.pending_scroll_deltas
-            return false;
+            .entry(tab_id.clone())
-        }
+            .and_modify(|current| *current = current.combined_with(delta))
-
+            .or_insert(delta);
-        state.offset = next_offset;
         self.click_points.remove(tab_id);
         self.typed_texts.remove(tab_id);
         self.keyboard_focus = None;
@@ -301,6 +170,21 @@ impl WebSurfaceStore {
         true
     }
 
+    pub(super) fn record_hover_point(
+        &mut self,
+        tab_id: &TabId,
+        position: Point<Pixels>,
+    ) -> bool {
+        let Some(bounds) = self.viewport_bounds.get(tab_id).copied() else {
+            return false;
+        };
+        let Some(point) = WebSurfaceClickPoint::from_window_position(bounds, position) else {
+            return false;
+        };
+        self.hover_points.insert(tab_id.clone(), point);
+        true
+    }
+
     pub(super) fn record_click_point(
         &mut self,
         tab_id: &TabId,
@@ -325,10 +209,6 @@ impl WebSurfaceStore {
             scroll_offset: state.scroll_offset,
             click_point: state.point,
         });
-        if self.click_points.get(tab_id) == Some(&state) {
-            return false;
-        }
-
         self.typed_texts.remove(tab_id);
         self.click_points.insert(tab_id.clone(), state);
         true
@@ -373,6 +253,53 @@ impl WebSurfaceStore {
         true
     }
 
+    fn take_pending_input(
+        &mut self,
+        tab_id: &TabId,
+        requested_url: &str,
+    ) -> WebSurfacePendingInput {
+        let scroll_offset = self.scroll_offset_for(tab_id, requested_url);
+        let scroll_delta = self.pending_scroll_deltas.remove(tab_id);
+        let click_point = self
+            .click_points
+            .remove(tab_id)
+            .filter(|state| {
+                state.requested_url == requested_url && state.scroll_offset == scroll_offset
+            })
+            .map(|state| state.point);
+        let typed_text = self
+            .typed_texts
+            .remove(tab_id)
+            .filter(|state| state.requested_url == requested_url)
+            .map(|state| state.text);
+
+        let hover_point = self.hover_points.remove(tab_id);
+
+        WebSurfacePendingInput { scroll_offset, scroll_delta, click_point, hover_point, typed_text }
+    }
+
+    fn previous_ready_frame(
+        &self,
+        tab_id: &TabId,
+        requested_url: &str,
+        zoom_percent: u16,
+    ) -> Option<WebSurfaceFrame> {
+        match self.states.get(tab_id) {
+            Some(WebSurfaceState::Ready(frame))
+                if frame.requested_url == requested_url && frame.zoom_percent() == zoom_percent =>
+            {
+                Some(frame.clone())
+            }
+            Some(WebSurfaceState::Loading {
+                requested_url: current_url, previous_frame, ..
+            }) if current_url == requested_url => previous_frame
+                .as_ref()
+                .filter(|frame| frame.zoom_percent() == zoom_percent)
+                .cloned(),
+            _ => None,
+        }
+    }
+
     fn scroll_offset_for(&self, tab_id: &TabId, requested_url: &str) -> WebSurfaceScrollOffset {
         self.scroll_offsets
             .get(tab_id)
@@ -380,39 +307,6 @@ impl WebSurfaceStore {
             .map(|state| state.offset)
             .unwrap_or_default()
     }
-
-    fn click_point_for(
-        &self,
-        tab_id: &TabId,
-        requested_url: &str,
-        scroll_offset: WebSurfaceScrollOffset,
-    ) -> Option<WebSurfaceClickPoint> {
-        self.click_points
-            .get(tab_id)
-            .filter(|state| {
-                state.requested_url == requested_url && state.scroll_offset == scroll_offset
-            })
-            .map(|state| state.point)
-    }
-
-    fn typed_text_for(
-        &self,
-        tab_id: &TabId,
-        requested_url: &str,
-        scroll_offset: WebSurfaceScrollOffset,
-        click_point: Option<WebSurfaceClickPoint>,
-    ) -> Option<String> {
-        let click_point = click_point?;
-        self.typed_texts
-            .get(tab_id)
-            .filter(|state| {
-                state.requested_url == requested_url
-                    && state.scroll_offset == scroll_offset
-                    && state.click_point == click_point
-                    && !state.text.is_empty()
-            })
-            .map(|state| state.text.clone())
-    }
 }
 
 pub(super) fn is_external_web_url(url: &str) -> bool {

crates/ely_app/src/shell/web_surface_controller.rs

@@ -2,32 +2,17 @@ use ely_browser_core::BrowserSnapshot;
 use ely_domain::{BrowserTab, ProfileKind, TabId};
 use gpui::{AnyElement, Bounds, Context, Pixels, Point};
 
-use crate::services::{
+use crate::services::ProfileDataMode;
-    ProfileDataMode,
-    servo_sidecar::{ServoSidecarError, SidecarSitePermission, SidecarSnapshot},
-};
 
 use super::{
     ElyShell,
-    web_surface_frame::WebSurfaceFrame,
+    web_surface_permissions::web_surface_site_permissions_for_tab,
-    web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize},
+    web_surface_state::WebSurfaceState,
-    web_surface_permissions::sidecar_site_permissions_for_tab,
-    web_surface_state::{WebSurfaceRequest, WebSurfaceState, WebSurfaceStateKey},
     web_surface_view::{
         render_failed_web_surface, render_loading_web_surface, render_ready_web_surface,
     },
 };
 
-struct PendingWebSurfaceFrame {
-    tab_id: TabId,
-    requested_url: String,
-    size: WebSurfaceSize,
-    scroll_offset: WebSurfaceScrollOffset,
-    zoom_percent: u16,
-    click_point: Option<WebSurfaceClickPoint>,
-    typed_text: Option<String>,
-}
-
 impl ElyShell {
     pub(super) fn render_external_web_canvas(
         &mut self,
@@ -39,9 +24,9 @@ impl ElyShell {
         let Some(profile_data_mode) = profile_data_mode_for(tab, snapshot) else {
             return render_failed_web_surface(tab, "Profile context is unavailable.", state_entity);
         };
-        let site_permissions = sidecar_site_permissions_for_tab(tab, snapshot);
 
-        self.ensure_external_web_frame(tab, profile_data_mode, site_permissions, cx);
+        let permissions = web_surface_site_permissions_for_tab(tab, snapshot);
+        self.web_surfaces.ensure_surface(tab, profile_data_mode, &permissions);
 
         match self.web_surfaces.state(tab.id()) {
             Some(WebSurfaceState::Ready(frame)) => {
@@ -59,109 +44,8 @@ impl ElyShell {
         }
     }
 
-    fn ensure_external_web_frame(
+    pub(super) fn tick_external_web_surfaces(&mut self) -> bool {
-        &mut self,
+        self.web_surfaces.tick()
-        tab: &BrowserTab,
-        profile_data_mode: ProfileDataMode,
-        site_permissions: Vec<SidecarSitePermission>,
-        cx: &mut Context<Self>,
-    ) {
-        let Some(request) = self.web_surfaces.prepare_request(tab, profile_data_mode) else {
-            return;
-        };
-
-        let WebSurfaceRequest {
-            tab_id,
-            requested_url,
-            size,
-            scroll_offset,
-            zoom_percent,
-            click_point,
-            typed_text,
-            client,
-            mut snapshot_request,
-        } = request;
-        snapshot_request = snapshot_request.with_site_permissions(site_permissions);
-        let pending_frame = PendingWebSurfaceFrame {
-            tab_id,
-            requested_url,
-            size,
-            scroll_offset,
-            zoom_percent,
-            click_point,
-            typed_text,
-        };
-        cx.spawn(async move |shell, cx| {
-            let result = cx
-                .background_executor()
-                .spawn(async move { client.snapshot(snapshot_request) })
-                .await;
-
-            _ = shell.update(cx, |shell, cx| {
-                shell.handle_external_web_frame_result(pending_frame, result);
-                cx.notify();
-            });
-        })
-        .detach();
-    }
-
-    fn handle_external_web_frame_result(
-        &mut self,
-        pending_frame: PendingWebSurfaceFrame,
-        result: Result<SidecarSnapshot, ServoSidecarError>,
-    ) {
-        let PendingWebSurfaceFrame {
-            tab_id,
-            requested_url,
-            size,
-            scroll_offset,
-            zoom_percent,
-            click_point,
-            typed_text,
-        } = pending_frame;
-        let state_key = WebSurfaceStateKey {
-            requested_url: requested_url.as_str(),
-            size,
-            scroll_offset,
-            zoom_percent,
-            click_point,
-            typed_text: typed_text.as_deref(),
-        };
-        if !self.web_surfaces.is_loading(&tab_id, state_key) {
-            return;
-        }
-
-        let state = match result {
-            Ok(snapshot) => match WebSurfaceFrame::from_snapshot(
-                requested_url.clone(),
-                scroll_offset,
-                zoom_percent,
-                click_point,
-                typed_text.clone(),
-                snapshot,
-            ) {
-                Ok(frame) => WebSurfaceState::Ready(frame),
-                Err(error) => WebSurfaceState::Failed {
-                    requested_url,
-                    size,
-                    scroll_offset,
-                    zoom_percent,
-                    click_point,
-                    typed_text,
-                    message: error.to_string(),
-                },
-            },
-            Err(error) => WebSurfaceState::Failed {
-                requested_url,
-                size,
-                scroll_offset,
-                zoom_percent,
-                click_point,
-                typed_text,
-                message: error.to_string(),
-            },
-        };
-        self.web_surfaces.finish(tab_id, state);
     }
 
     pub(super) fn record_external_web_viewport(
@@ -179,7 +63,7 @@ impl ElyShell {
         &mut self,
         tab_id: TabId,
         requested_url: String,
-        delta: gpui::Point<Pixels>,
+        delta: Point<Pixels>,
         cx: &mut Context<Self>,
     ) {
         if self.web_surfaces.record_scroll_delta(&tab_id, requested_url.as_str(), delta) {
@@ -187,6 +71,17 @@ impl ElyShell {
         }
     }
 
+    pub(super) fn hover_external_web_viewport(
+        &mut self,
+        tab_id: TabId,
+        position: Point<Pixels>,
+        cx: &mut Context<Self>,
+    ) {
+        if self.web_surfaces.record_hover_point(&tab_id, position) {
+            cx.notify();
+        }
+    }
+
     pub(super) fn click_external_web_viewport(
         &mut self,
         tab_id: TabId,

crates/ely_app/src/shell/web_surface_runtime.rs

@@ -0,0 +1,237 @@
+use std::{collections::BTreeMap, fs, path::PathBuf};
+
+use ely_domain::{BrowserTab, ProfileId, TabId};
+
+use crate::services::{
+    ProfileDataMode,
+    servo_live::{ServoLiveClient, ServoLiveEnsureRequest, ServoLiveSitePermission},
+    servo_profile_data::{default_profile_data_root, profile_data_dir, transient_profile_data_dir},
+};
+
+use super::{
+    web_surface_frame::WebSurfaceFrame,
+    web_surface_geometry::{WebSurfaceScrollOffset, WebSurfaceSize},
+    web_surface_permissions::WebSurfaceSitePermission,
+    web_surface_state::WebSurfacePendingInput,
+};
+
+pub(super) struct WebSurfaceRuntime {
+    state: RuntimeState,
+    sessions: BTreeMap<TabId, WebSurfaceSession>,
+}
+
+impl WebSurfaceRuntime {
+    pub(super) fn new() -> Self {
+        Self { state: RuntimeState::Empty, sessions: BTreeMap::new() }
+    }
+
+    pub(super) fn ensure_tab(
+        &mut self,
+        tab: &BrowserTab,
+        size: WebSurfaceSize,
+        profile_data_mode: ProfileDataMode,
+        permissions: &[WebSurfaceSitePermission],
+        input: WebSurfacePendingInput,
+    ) -> Result<WebSurfaceEnsureResult, String> {
+        let scope = WebSurfaceRuntimeScope::new(tab.profile_id().clone(), profile_data_mode);
+        self.ensure_runtime(scope.clone())?;
+
+        let requested_url = tab.url().as_str().to_string();
+        let zoom_percent = tab.zoom_percent();
+        let (state, sessions) = (&mut self.state, &mut self.sessions);
+        let RuntimeState::Ready { scope: active_scope, client, .. } = state else {
+            return Err("Servo live runtime is unavailable".to_string());
+        };
+        if active_scope != &scope {
+            return Err(active_scope.error_for(&scope));
+        }
+
+        let session = sessions.entry(tab.id().clone()).or_insert_with(WebSurfaceSession::default);
+        let next_scroll_offset = input.scroll_offset;
+        let started_loading = session.started_loading(&requested_url, size, zoom_percent);
+        let frame = client
+            .ensure(ServoLiveEnsureRequest {
+                tab_id: tab.id().as_str().to_string(),
+                profile_id: tab.profile_id().as_str().to_string(),
+                url: requested_url.clone(),
+                width: size.width,
+                height: size.height,
+                page_zoom_percent: zoom_percent,
+                scroll_delta_x: input.scroll_delta.map_or(0, |delta| delta.x()),
+                scroll_delta_y: input.scroll_delta.map_or(0, |delta| delta.y()),
+                click_x: input.click_point.map(|point| point.x()),
+                click_y: input.click_point.map(|point| point.y()),
+                hover_x: input.hover_point.map(|point| point.x()),
+                hover_y: input.hover_point.map(|point| point.y()),
+                typed_text: input.typed_text,
+                site_permissions: permissions.iter().map(ServoLiveSitePermission::from).collect(),
+            })
+            .map_err(|error| error.to_string())?
+            .map(|frame| {
+                WebSurfaceFrame::from_live_frame(
+                    requested_url.clone(),
+                    next_scroll_offset,
+                    zoom_percent,
+                    frame,
+                )
+                .map_err(|error| error.to_string())
+            })
+            .transpose()?;
+
+        session.requested_url = requested_url.clone();
+        session.size = size;
+        session.zoom_percent = zoom_percent;
+        session.scroll_offset = next_scroll_offset;
+
+        Ok(WebSurfaceEnsureResult { requested_url, started_loading, frame })
+    }
+
+    pub(super) fn tick(&mut self) -> Vec<WebSurfaceRuntimeFrame> {
+        let (state, sessions) = (&mut self.state, &self.sessions);
+        let RuntimeState::Ready { client, .. } = state else {
+            return Vec::new();
+        };
+
+        let mut frames = Vec::new();
+        for (tab_id, session) in sessions {
+            match client.poll(tab_id.as_str().to_string()) {
+                Ok(Some(frame)) => match WebSurfaceFrame::from_live_frame(
+                    session.requested_url.clone(),
+                    session.scroll_offset,
+                    session.zoom_percent,
+                    frame,
+                ) {
+                    Ok(frame) => {
+                        frames.push(WebSurfaceRuntimeFrame::Ready { tab_id: tab_id.clone(), frame })
+                    }
+                    Err(error) => frames.push(WebSurfaceRuntimeFrame::Failed {
+                        tab_id: tab_id.clone(),
+                        message: error.to_string(),
+                    }),
+                },
+                Ok(None) => {}
+                Err(error) => frames.push(WebSurfaceRuntimeFrame::Failed {
+                    tab_id: tab_id.clone(),
+                    message: error.to_string(),
+                }),
+            }
+        }
+
+        frames
+    }
+
+    fn ensure_runtime(&mut self, scope: WebSurfaceRuntimeScope) -> Result<(), String> {
+        match &self.state {
+            RuntimeState::Empty => {
+                let (config_dir, transient_profile_data_dir) = config_dir_for_scope(&scope)?;
+                let client = ServoLiveClient::new(config_dir).map_err(|error| error.to_string())?;
+                self.state = RuntimeState::Ready { scope, client, transient_profile_data_dir };
+                Ok(())
+            }
+            RuntimeState::Ready { scope: active_scope, .. } if active_scope == &scope => Ok(()),
+            RuntimeState::Ready { scope: active_scope, .. } => Err(active_scope.error_for(&scope)),
+        }
+    }
+}
+
+impl Drop for WebSurfaceRuntime {
+    fn drop(&mut self) {
+        let RuntimeState::Ready { transient_profile_data_dir: Some(path), .. } = &self.state else {
+            return;
+        };
+        let _ = fs::remove_dir_all(path);
+    }
+}
+
+enum RuntimeState {
+    Empty,
+    Ready {
+        scope: WebSurfaceRuntimeScope,
+        client: ServoLiveClient,
+        transient_profile_data_dir: Option<PathBuf>,
+    },
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+struct WebSurfaceRuntimeScope {
+    profile_id: ProfileId,
+    profile_data_mode: ProfileDataMode,
+}
+
+impl WebSurfaceRuntimeScope {
+    fn new(profile_id: ProfileId, profile_data_mode: ProfileDataMode) -> Self {
+        Self { profile_id, profile_data_mode }
+    }
+
+    fn error_for(&self, requested: &Self) -> String {
+        format!(
+            "Servo live runtime is already attached to profile {} ({:?}); requested profile {} ({:?})",
+            self.profile_id.as_str(),
+            self.profile_data_mode,
+            requested.profile_id.as_str(),
+            requested.profile_data_mode
+        )
+    }
+}
+
+#[derive(Clone, Default)]
+struct WebSurfaceSession {
+    requested_url: String,
+    size: WebSurfaceSize,
+    zoom_percent: u16,
+    scroll_offset: WebSurfaceScrollOffset,
+}
+
+impl WebSurfaceSession {
+    fn started_loading(
+        &self,
+        requested_url: &str,
+        size: WebSurfaceSize,
+        zoom_percent: u16,
+    ) -> bool {
+        self.requested_url != requested_url
+            || self.size != size
+            || self.zoom_percent != zoom_percent
+    }
+}
+
+pub(super) struct WebSurfaceEnsureResult {
+    pub(super) requested_url: String,
+    pub(super) started_loading: bool,
+    pub(super) frame: Option<WebSurfaceFrame>,
+}
+
+pub(super) enum WebSurfaceRuntimeFrame {
+    Ready { tab_id: TabId, frame: WebSurfaceFrame },
+    Failed { tab_id: TabId, message: String },
+}
+
+fn config_dir_for_scope(
+    scope: &WebSurfaceRuntimeScope,
+) -> Result<(PathBuf, Option<PathBuf>), String> {
+    match scope.profile_data_mode {
+        ProfileDataMode::Persistent => {
+            let root = default_profile_data_root()
+                .ok_or_else(|| "Profile data root is unavailable".to_string())?;
+            let config_dir = profile_data_dir(&root, &scope.profile_id);
+            fs::create_dir_all(&config_dir).map_err(|error| error.to_string())?;
+            Ok((config_dir, None))
+        }
+        ProfileDataMode::Transient => {
+            let config_dir =
+                transient_profile_data_dir(&scope.profile_id).map_err(|error| error.to_string())?;
+            fs::create_dir_all(&config_dir).map_err(|error| error.to_string())?;
+            Ok((config_dir.clone(), Some(config_dir)))
+        }
+    }
+}
+
+impl From<&WebSurfaceSitePermission> for ServoLiveSitePermission {
+    fn from(permission: &WebSurfaceSitePermission) -> Self {
+        Self::new(
+            permission.origin().as_str(),
+            permission.feature().as_str(),
+            permission.decision(),
+        )
+    }
+}

crates/ely_app/src/shell/web_surface_state.rs

@@ -1,10 +1,8 @@
 use ely_domain::TabId;
 
-use crate::services::servo_sidecar::{ServoSidecarClient, SidecarSnapshotRequest};
-
 use super::{
     web_surface_frame::WebSurfaceFrame,
-    web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset, WebSurfaceSize},
+    web_surface_geometry::{WebSurfaceClickPoint, WebSurfaceScrollOffset},
 };
 
 pub(super) struct WebSurfaceScrollState {
@@ -40,60 +38,17 @@ pub(super) struct WebSurfaceTextInputState {
     pub(super) text: String,
 }
 
-#[derive(Clone, Copy)]
+#[derive(Clone, Debug, Eq, PartialEq)]
-pub(super) struct WebSurfaceStateKey<'a> {
+pub(super) struct WebSurfacePendingInput {
-    pub(super) requested_url: &'a str,
-    pub(super) size: WebSurfaceSize,
     pub(super) scroll_offset: WebSurfaceScrollOffset,
-    pub(super) zoom_percent: u16,
+    pub(super) scroll_delta: Option<super::web_surface_geometry::WebSurfaceScrollDelta>,
-    pub(super) click_point: Option<WebSurfaceClickPoint>,
-    pub(super) typed_text: Option<&'a str>,
-}
-
-pub(super) enum WebSurfaceClient {
-    Ready(ServoSidecarClient),
-    Unavailable(String),
-}
-
-impl WebSurfaceClient {
-    pub(super) fn new() -> Self {
-        match ServoSidecarClient::new() {
-            Ok(client) => Self::Ready(client),
-            Err(error) => Self::Unavailable(error.to_string()),
-        }
-    }
-}
-
-pub(super) struct WebSurfaceRequest {
-    pub(super) tab_id: TabId,
-    pub(super) requested_url: String,
-    pub(super) size: WebSurfaceSize,
-    pub(super) scroll_offset: WebSurfaceScrollOffset,
-    pub(super) zoom_percent: u16,
     pub(super) click_point: Option<WebSurfaceClickPoint>,
+    pub(super) hover_point: Option<WebSurfaceClickPoint>,
     pub(super) typed_text: Option<String>,
-    pub(super) client: ServoSidecarClient,
-    pub(super) snapshot_request: SidecarSnapshotRequest,
 }
 
 pub(super) enum WebSurfaceState {
-    Loading {
+    Loading { requested_url: String, previous_frame: Option<WebSurfaceFrame> },
-        requested_url: String,
-        size: WebSurfaceSize,
-        scroll_offset: WebSurfaceScrollOffset,
-        zoom_percent: u16,
-        click_point: Option<WebSurfaceClickPoint>,
-        typed_text: Option<String>,
-        previous_frame: Option<WebSurfaceFrame>,
-    },
     Ready(WebSurfaceFrame),
-    Failed {
+    Failed { message: String },
-        requested_url: String,
-        size: WebSurfaceSize,
-        scroll_offset: WebSurfaceScrollOffset,
-        zoom_percent: u16,
-        click_point: Option<WebSurfaceClickPoint>,
-        typed_text: Option<String>,
-        message: String,
-    },
 }

crates/ely_app/src/shell/web_surface_view.rs

@@ -93,6 +93,8 @@ fn render_input_overlay(
     let click_tab_id = tab_id.clone();
     let click_url = url.clone();
     let click_entity = state_entity.clone();
+    let hover_tab_id = tab_id.clone();
+    let hover_entity = state_entity.clone();
     let scroll_tab_id = tab_id;
     let scroll_url = url;
     let scroll_entity = state_entity;
@@ -116,6 +118,15 @@ fn render_input_overlay(
             });
             cx.stop_propagation();
         })
+        .on_mouse_move(move |event, _window, cx| {
+            hover_entity.update(cx, |shell, cx| {
+                shell.hover_external_web_viewport(
+                    hover_tab_id.clone(),
+                    event.position,
+                    cx,
+                );
+            });
+        })
         .on_scroll_wheel(move |event, window, cx| {
             let delta = event.delta.pixel_delta(window.line_height());
             scroll_entity.update(cx, |shell, cx| {

crates/ely_servo_host/src/bin/ely_servo_sidecar/live.rs

@@ -0,0 +1,428 @@
+use std::{
+    collections::HashMap,
+    fs,
+    io::{self, BufRead, Write},
+    path::PathBuf,
+    thread,
+    time::{Duration, Instant},
+};
+
+use ely_domain::{DEFAULT_ZOOM_PERCENT, ProfileId, TabId, UrlText};
+use ely_servo_host::{
+    KeyboardTextRequest, MouseClickRequest, NavigationRequest, PageZoomRequest, PermissionDecision,
+    PermissionRequest, RenderedFrame, ResizeRequest, ScrollRequest, ServoHost, ServoHostError,
+    ServoSurfaceSize, SoftwareServoHost, WebViewSnapshot, WebViewState,
+};
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use super::args::LiveArgs;
+
+const LIVE_FRAME_WAIT_TIMEOUT: Duration = Duration::from_millis(60);
+const LIVE_FRAME_WAIT_INTERVAL: Duration = Duration::from_millis(2);
+
+pub(super) fn run_live(args: LiveArgs) -> Result<(), LiveSidecarError> {
+    fs::create_dir_all(&args.profile_data_dir)?;
+    let mut host = SoftwareServoHost::new_with_config_dir(
+        ServoSurfaceSize::new(1, 1),
+        Some(args.profile_data_dir),
+    )?;
+    let mut sessions = HashMap::new();
+    let stdin = io::stdin();
+    let mut stdout = io::stdout().lock();
+
+    for line in stdin.lock().lines() {
+        let line = line?;
+        if line.trim().is_empty() {
+            continue;
+        }
+
+        let response = match serde_json::from_str::<LiveRequest>(&line) {
+            Ok(request) => handle_request(&mut host, &mut sessions, request),
+            Err(error) => Err(LiveSidecarError::Json(error)),
+        };
+        write_response(&mut stdout, response)?;
+    }
+
+    Ok(())
+}
+
+fn handle_request(
+    host: &mut SoftwareServoHost,
+    sessions: &mut HashMap<String, LiveSession>,
+    request: LiveRequest,
+) -> Result<LiveResponse, LiveSidecarError> {
+    match request {
+        LiveRequest::Ensure {
+            tab_id,
+            profile_id,
+            url,
+            width,
+            height,
+            page_zoom_percent,
+            scroll_delta_x,
+            scroll_delta_y,
+            click_x,
+            click_y,
+            hover_x,
+            hover_y,
+            typed_text,
+            site_permissions,
+            rgba_out,
+        } => {
+            let tab = TabId::parse(tab_id.clone())?;
+            let profile = ProfileId::parse(profile_id)?;
+            let url = UrlText::parse(url)?;
+            let session = ensure_session(host, sessions, tab_id, &tab, &profile, width, height)?;
+
+            if apply_layout(host, session, width, height, page_zoom_percent)? {
+                session.awaiting_visible_frame = true;
+            }
+            apply_permissions(host, session, &profile, site_permissions)?;
+            if session.requested_url != url.as_str() {
+                host.navigate(NavigationRequest {
+                    webview_id: session.webview_id.clone(),
+                    tab_id: tab,
+                    url: url.clone(),
+                })?;
+                session.requested_url = url.as_str().to_string();
+                session.scroll_x = 0;
+                session.scroll_y = 0;
+                session.awaiting_visible_frame = true;
+            }
+            if apply_input(
+                host,
+                session,
+                scroll_delta_x,
+                scroll_delta_y,
+                click_x,
+                click_y,
+                hover_x,
+                hover_y,
+                typed_text,
+            )? {
+                session.awaiting_visible_frame = true;
+            }
+            poll_frame(host, session, rgba_out.into())
+        }
+        LiveRequest::Poll { tab_id, rgba_out } => {
+            let Some(session) = sessions.get_mut(&tab_id) else {
+                return Ok(LiveResponse::empty());
+            };
+            poll_frame(host, session, rgba_out.into())
+        }
+    }
+}
+
+fn write_response(
+    stdout: &mut impl Write,
+    response: Result<LiveResponse, LiveSidecarError>,
+) -> Result<(), LiveSidecarError> {
+    let response = match response {
+        Ok(response) => response,
+        Err(error) => LiveResponse::error(error.to_string()),
+    };
+    serde_json::to_writer(&mut *stdout, &response)?;
+    stdout.write_all(b"\n")?;
+    stdout.flush()?;
+    Ok(())
+}
+
+fn ensure_session<'a>(
+    host: &mut SoftwareServoHost,
+    sessions: &'a mut HashMap<String, LiveSession>,
+    key: String,
+    tab_id: &TabId,
+    profile_id: &ProfileId,
+    width: u32,
+    height: u32,
+) -> Result<&'a mut LiveSession, LiveSidecarError> {
+    if !sessions.contains_key(&key) {
+        let webview_id = host.create_webview_with_size(
+            tab_id.clone(),
+            profile_id.clone(),
+            ServoSurfaceSize::new(width, height),
+        )?;
+        sessions.insert(key.clone(), LiveSession::new(webview_id, width, height));
+    }
+
+    sessions.get_mut(&key).ok_or(LiveSidecarError::SessionUnavailable)
+}
+
+fn apply_layout(
+    host: &mut SoftwareServoHost,
+    session: &mut LiveSession,
+    width: u32,
+    height: u32,
+    page_zoom_percent: u16,
+) -> Result<bool, LiveSidecarError> {
+    let mut changed = false;
+    if session.width != width || session.height != height {
+        host.resize(ResizeRequest { webview_id: session.webview_id.clone(), width, height })?;
+        session.width = width;
+        session.height = height;
+        changed = true;
+    }
+
+    if session.page_zoom_percent != page_zoom_percent {
+        host.set_page_zoom(PageZoomRequest {
+            webview_id: session.webview_id.clone(),
+            zoom_factor: f32::from(page_zoom_percent) / 100.0,
+        })?;
+        session.page_zoom_percent = page_zoom_percent;
+        changed = true;
+    }
+
+    Ok(changed)
+}
+
+fn apply_permissions(
+    host: &mut SoftwareServoHost,
+    session: &LiveSession,
+    profile_id: &ProfileId,
+    permissions: Vec<LiveSitePermission>,
+) -> Result<(), LiveSidecarError> {
+    for permission in permissions {
+        host.set_permission(
+            PermissionRequest {
+                webview_id: session.webview_id.clone(),
+                profile_id: profile_id.clone(),
+                origin: ely_domain::SiteOrigin::parse(permission.origin)?,
+                feature: ely_domain::SitePermissionFeature::parse(permission.feature.as_str())?,
+            },
+            PermissionDecision::from(ely_domain::SitePermissionDecision::parse(
+                permission.decision.as_str(),
+            )?),
+        )?;
+    }
+
+    Ok(())
+}
+
+fn apply_input(
+    host: &mut SoftwareServoHost,
+    session: &mut LiveSession,
+    scroll_delta_x: i32,
+    scroll_delta_y: i32,
+    click_x: Option<u32>,
+    click_y: Option<u32>,
+    hover_x: Option<u32>,
+    hover_y: Option<u32>,
+    typed_text: Option<String>,
+) -> Result<bool, LiveSidecarError> {
+    let mut changed = false;
+    if scroll_delta_x != 0 || scroll_delta_y != 0 {
+        host.scroll(ScrollRequest {
+            webview_id: session.webview_id.clone(),
+            delta_x: scroll_delta_x,
+            delta_y: scroll_delta_y,
+        })?;
+        session.scroll_x = positive_scroll_component(session.scroll_x, scroll_delta_x);
+        session.scroll_y = positive_scroll_component(session.scroll_y, scroll_delta_y);
+        changed = true;
+    }
+
+    if let (Some(x), Some(y)) = (hover_x, hover_y) {
+        host.hover(x, y)?;
+        changed = true;
+    }
+
+    if let (Some(x), Some(y)) = (click_x, click_y) {
+        host.click(MouseClickRequest { webview_id: session.webview_id.clone(), x, y })?;
+        changed = true;
+    }
+
+    if let Some(text) = typed_text {
+        host.type_text(KeyboardTextRequest { webview_id: session.webview_id.clone(), text })?;
+        changed = true;
+    }
+
+    Ok(changed)
+}
+
+fn poll_frame(
+    host: &mut SoftwareServoHost,
+    session: &mut LiveSession,
+    rgba_out: PathBuf,
+) -> Result<LiveResponse, LiveSidecarError> {
+    let started_at = Instant::now();
+    let mut latest_frame = None;
+
+    loop {
+        host.tick();
+        let snapshot = host.snapshot(&session.webview_id)?;
+        if snapshot.has_pending_frame() {
+            host.paint(&session.webview_id)?;
+            let snapshot = host.snapshot(&session.webview_id)?;
+            let frame = host.last_rendered_frame()?;
+            let has_visible_content =
+                frame.non_white_pixel_count() > 0 && frame.content_pixel_count() > 0;
+            fs::write(&rgba_out, frame.rgba_bytes())?;
+            let response =
+                LiveResponse::frame(LiveFrameReport::new(&snapshot, &frame, rgba_out.clone()));
+            if has_visible_content {
+                session.awaiting_visible_frame = false;
+                return Ok(response);
+            }
+            if !session.awaiting_visible_frame {
+                return Ok(response);
+            }
+            latest_frame = Some(response);
+        }
+
+        if !session.awaiting_visible_frame {
+            return Ok(LiveResponse::empty());
+        }
+        if started_at.elapsed() >= LIVE_FRAME_WAIT_TIMEOUT {
+            return Ok(latest_frame.unwrap_or_else(LiveResponse::empty));
+        }
+
+        thread::sleep(LIVE_FRAME_WAIT_INTERVAL);
+    }
+}
+
+#[derive(Clone)]
+struct LiveSession {
+    webview_id: ely_domain::WebViewId,
+    requested_url: String,
+    width: u32,
+    height: u32,
+    page_zoom_percent: u16,
+    scroll_x: i32,
+    scroll_y: i32,
+    awaiting_visible_frame: bool,
+}
+
+impl LiveSession {
+    fn new(webview_id: ely_domain::WebViewId, width: u32, height: u32) -> Self {
+        Self {
+            webview_id,
+            requested_url: String::new(),
+            width: width.max(1),
+            height: height.max(1),
+            page_zoom_percent: DEFAULT_ZOOM_PERCENT,
+            scroll_x: 0,
+            scroll_y: 0,
+            awaiting_visible_frame: false,
+        }
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum LiveRequest {
+    Ensure {
+        tab_id: String,
+        profile_id: String,
+        url: String,
+        width: u32,
+        height: u32,
+        page_zoom_percent: u16,
+        scroll_delta_x: i32,
+        scroll_delta_y: i32,
+        click_x: Option<u32>,
+        click_y: Option<u32>,
+        #[serde(default)]
+        hover_x: Option<u32>,
+        #[serde(default)]
+        hover_y: Option<u32>,
+        typed_text: Option<String>,
+        site_permissions: Vec<LiveSitePermission>,
+        rgba_out: String,
+    },
+    Poll {
+        tab_id: String,
+        rgba_out: String,
+    },
+}
+
+#[derive(Deserialize)]
+struct LiveSitePermission {
+    origin: String,
+    feature: String,
+    decision: String,
+}
+
+#[derive(Serialize)]
+struct LiveResponse {
+    error: Option<String>,
+    frame: Option<LiveFrameReport>,
+}
+
+impl LiveResponse {
+    fn empty() -> Self {
+        Self { error: None, frame: None }
+    }
+
+    fn frame(frame: LiveFrameReport) -> Self {
+        Self { error: None, frame: Some(frame) }
+    }
+
+    fn error(message: String) -> Self {
+        Self { error: Some(message), frame: None }
+    }
+}
+
+#[derive(Serialize)]
+struct LiveFrameReport {
+    loaded_url: Option<String>,
+    title: Option<String>,
+    state: &'static str,
+    width: u32,
+    height: u32,
+    rgba_path: PathBuf,
+    rgba_byte_count: usize,
+    non_white_pixel_count: u64,
+    content_pixel_count: u64,
+    sample_hash: u64,
+}
+
+impl LiveFrameReport {
+    fn new(snapshot: &WebViewSnapshot, frame: &RenderedFrame, rgba_path: PathBuf) -> Self {
+        Self {
+            loaded_url: snapshot.url().map(str::to_string),
+            title: snapshot.title().map(str::to_string),
+            state: state_label(snapshot.state()),
+            width: frame.width(),
+            height: frame.height(),
+            rgba_path,
+            rgba_byte_count: frame.rgba_bytes().len(),
+            non_white_pixel_count: frame.non_white_pixel_count(),
+            content_pixel_count: frame.content_pixel_count(),
+            sample_hash: frame.sample_hash(),
+        }
+    }
+}
+
+#[derive(Debug, Error)]
+pub(super) enum LiveSidecarError {
+    #[error("live session is unavailable after creation")]
+    SessionUnavailable,
+
+    #[error(transparent)]
+    Domain(#[from] ely_domain::DomainError),
+
+    #[error(transparent)]
+    Host(#[from] ServoHostError),
+
+    #[error(transparent)]
+    Io(#[from] io::Error),
+
+    #[error(transparent)]
+    Json(#[from] serde_json::Error),
+}
+
+fn state_label(state: &WebViewState) -> &'static str {
+    match state {
+        WebViewState::Created => "created",
+        WebViewState::Loading => "loading",
+        WebViewState::Complete => "complete",
+        WebViewState::Sleeping => "sleeping",
+        WebViewState::Crashed => "crashed",
+    }
+}
+
+fn positive_scroll_component(current: i32, delta: i32) -> i32 {
+    let value = i64::from(current) + i64::from(delta);
+    value.clamp(0, i64::from(i32::MAX)) as i32
+}

crates/ely_servo_host/src/runtime.rs

@@ -23,7 +23,9 @@ use crate::{
     KeyboardTextRequest, MouseClickRequest, MouseDragRequest, NavigationRequest, PageZoomRequest,
     PermissionDecision, PermissionRequest, RenderedFrame, ResizeRequest, ScreenshotRequest,
     ScrollRequest, ServoHost, ServoHostError, TouchTapRequest, WebViewSnapshot, WebViewState,
-    runtime_input::{send_keyboard_text, send_mouse_click, send_mouse_drag, send_touch_tap},
+    runtime_input::{
+        send_keyboard_text, send_mouse_click, send_mouse_drag, send_mouse_hover, send_touch_tap,
+    },
     runtime_permissions::{PermissionStore, set_permission_decision},
     runtime_waker::ServoWakeFlag,
     runtime_webview::{HostWebView, HostWebViewDelegate},
@@ -52,7 +54,7 @@ impl ServoSurfaceSize {
 
 pub struct SoftwareServoHost {
     servo: Servo,
-    rendering_context: Rc<dyn RenderingContext>,
+    default_surface_size: ServoSurfaceSize,
     webviews: HashMap<WebViewId, HostWebView>,
     permissions: PermissionStore,
     wake_requested: Arc<AtomicBool>,
@@ -82,16 +84,19 @@ impl SoftwareServoHost {
         host
     }
 
+    pub fn create_webview_with_size(
+        &mut self,
+        tab_id: TabId,
+        profile_id: ProfileId,
+        size: ServoSurfaceSize,
+    ) -> Result<WebViewId, ServoHostError> {
+        self.create_webview_in_context(tab_id, profile_id, size)
+    }
+
     fn new_started(
         size: ServoSurfaceSize,
         config_dir: Option<PathBuf>,
     ) -> Result<Self, ServoHostError> {
-        let rendering_context = Rc::new(
-            servo::SoftwareRenderingContext::new(size.physical())
-                .map_err(|_| ServoHostError::RenderingContextUnavailable)?,
-        );
-        rendering_context.make_current().map_err(|_| ServoHostError::RenderingContextNotCurrent)?;
-
         let wake_requested = Arc::new(AtomicBool::new(false));
         let mut builder = ServoBuilder::default()
             .event_loop_waker(Box::new(ServoWakeFlag::new(wake_requested.clone())));
@@ -102,7 +107,7 @@ impl SoftwareServoHost {
 
         Ok(Self {
             servo,
-            rendering_context,
+            default_surface_size: size,
             webviews: HashMap::new(),
             permissions: Rc::new(RefCell::new(HashMap::new())),
             wake_requested,
@@ -117,19 +122,7 @@ impl ServoHost for SoftwareServoHost {
         tab_id: TabId,
         profile_id: ProfileId,
     ) -> Result<WebViewId, ServoHostError> {
-        let webview_id = WebViewId::new();
+        self.create_webview_in_context(tab_id, profile_id, self.default_surface_size)
-        let delegate =
-            Rc::new(HostWebViewDelegate::new(profile_id.clone(), self.permissions.clone()));
-        let webview = WebViewBuilder::new(&self.servo, self.rendering_context.clone())
-            .delegate(delegate.clone())
-            .build();
-
-        self.webviews.insert(
-            webview_id.clone(),
-            HostWebView { tab_id, profile_id, webview, delegate, requested_url: None },
-        );
-
-        Ok(webview_id)
     }
 
     fn navigate(&mut self, request: NavigationRequest) -> Result<(), ServoHostError> {
@@ -138,7 +131,6 @@ impl ServoHost for SoftwareServoHost {
         })?;
         self.servo.spin_event_loop();
         let servo = self.servo.clone();
-        let rendering_context = self.rendering_context.clone();
         let webview = self
             .webviews
             .get_mut(&request.webview_id)
@@ -151,7 +143,7 @@ impl ServoHost for SoftwareServoHost {
 
         webview.delegate.set_state(WebViewState::Loading);
         if should_create_initial_document {
-            webview.webview = WebViewBuilder::new(&servo, rendering_context)
+            webview.webview = WebViewBuilder::new(&servo, webview.rendering_context.clone())
                 .delegate(webview.delegate.clone())
                 .url(url)
                 .build();
@@ -188,7 +180,9 @@ impl ServoHost for SoftwareServoHost {
             .get(&request.webview_id)
             .ok_or_else(|| ServoHostError::WebViewNotFound { id: request.webview_id.clone() })?;
 
-        webview.webview.resize(PhysicalSize::new(request.width, request.height));
+        let size = PhysicalSize::new(request.width, request.height);
+        webview.rendering_context.resize(size);
+        webview.webview.resize(size);
         Ok(())
     }
 
@@ -202,6 +196,13 @@ impl ServoHost for SoftwareServoHost {
         Ok(())
     }
 
+    fn hover(&mut self, x: u32, y: u32) -> Result<(), ServoHostError> {
+        if let Some(webview) = self.webviews.values().next() {
+            send_mouse_hover(&webview.webview, x, y);
+        }
+        Ok(())
+    }
+
     fn click(&mut self, request: MouseClickRequest) -> Result<(), ServoHostError> {
         let webview = self
             .webviews
@@ -319,16 +320,15 @@ impl ServoHost for SoftwareServoHost {
     }
 
     fn paint(&mut self, webview_id: &WebViewId) -> Result<(), ServoHostError> {
-        self.rendering_context
+        let rendering_context = self.webview(webview_id)?.rendering_context.clone();
-            .make_current()
+        rendering_context.make_current().map_err(|_| ServoHostError::RenderingContextNotCurrent)?;
-            .map_err(|_| ServoHostError::RenderingContextNotCurrent)?;
+        rendering_context.prepare_for_rendering();
-        self.rendering_context.prepare_for_rendering();
         {
             let webview = self.webview(webview_id)?;
             webview.webview.paint();
         }
-        let rendered_frame = self.read_rendered_frame()?;
+        let rendered_frame = Self::read_rendered_frame(rendering_context.as_ref())?;
-        self.rendering_context.present();
+        rendering_context.present();
         self.webview(webview_id)?.delegate.mark_frame_presented();
         self.last_rendered_frame = Some(rendered_frame);
         Ok(())
@@ -340,14 +340,56 @@ impl ServoHost for SoftwareServoHost {
 }
 
 impl SoftwareServoHost {
+    fn create_webview_in_context(
+        &mut self,
+        tab_id: TabId,
+        profile_id: ProfileId,
+        size: ServoSurfaceSize,
+    ) -> Result<WebViewId, ServoHostError> {
+        let webview_id = WebViewId::new();
+        let rendering_context = Self::new_rendering_context(size)?;
+        let delegate =
+            Rc::new(HostWebViewDelegate::new(profile_id.clone(), self.permissions.clone()));
+        let webview = WebViewBuilder::new(&self.servo, rendering_context.clone())
+            .delegate(delegate.clone())
+            .build();
+
+        self.webviews.insert(
+            webview_id.clone(),
+            HostWebView {
+                tab_id,
+                profile_id,
+                rendering_context,
+                webview,
+                delegate,
+                requested_url: None,
+            },
+        );
+
+        Ok(webview_id)
+    }
+
+    fn new_rendering_context(
+        size: ServoSurfaceSize,
+    ) -> Result<Rc<dyn RenderingContext>, ServoHostError> {
+        let rendering_context = Rc::new(
+            servo::SoftwareRenderingContext::new(size.physical())
+                .map_err(|_| ServoHostError::RenderingContextUnavailable)?,
+        );
+        rendering_context.make_current().map_err(|_| ServoHostError::RenderingContextNotCurrent)?;
+        Ok(rendering_context)
+    }
+
     fn webview(&self, webview_id: &WebViewId) -> Result<&HostWebView, ServoHostError> {
         self.webviews
             .get(webview_id)
             .ok_or_else(|| ServoHostError::WebViewNotFound { id: webview_id.clone() })
     }
 
-    fn read_rendered_frame(&self) -> Result<RenderedFrame, ServoHostError> {
+    fn read_rendered_frame(
-        let size = self.rendering_context.size();
+        rendering_context: &dyn RenderingContext,
+    ) -> Result<RenderedFrame, ServoHostError> {
+        let size = rendering_context.size();
         let width =
             i32::try_from(size.width).map_err(|_| ServoHostError::RenderedFrameUnavailable)?;
         let height =
@@ -356,8 +398,7 @@ impl SoftwareServoHost {
             DeviceIntPoint::new(0, 0),
             DeviceIntSize::new(width, height),
         );
-        let image = self
+        let image = rendering_context
-            .rendering_context
             .read_to_image(frame_rect)
             .ok_or(ServoHostError::RenderedFrameUnavailable)?;
 

crates/ely_servo_host/src/runtime_input.rs

@@ -6,6 +6,10 @@ use servo::{
 
 use crate::keyboard::key_and_code_for_character;
 
+pub(super) fn send_mouse_hover(webview: &WebView, x: u32, y: u32) {
+    webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point(x, y))));
+}
+
 pub(super) fn send_mouse_click(webview: &WebView, x: u32, y: u32) {
     let point = point(x, y);
     webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point)));