ZacharyZhang-NY/ELY-Browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(servo): isolate profiles with hardware sidecars

Browse Source
This commit is contained in:
ZacharyZhang-NY
2026-07-09 20:27:22 -04:00
parent 78dc86b18e
commit c28ec2bee8
92 changed files with 10306 additions and 1485 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ely_app/src/shell/web_surface_runtime.rs
+234 -185
View File
@@ -1,16 +1,16 @@
use std::{
collections::BTreeMap,
fs,
path::PathBuf,
thread::JoinHandle,
time::{Duration, Instant},
};
use ely_domain::{BrowserTab, TabId};
use gpui::NativeSurfaceHandle;
use crate::services::{
ProfileDataMode,
servo_live::{ServoLiveClient, ServoLiveEnsureRequest, ServoLiveSitePermission},
servo_profile_data::cleanup_stale_transient_profile_data_dirs,
};
use super::{
@@ -25,44 +25,54 @@ use super::{
scroll_wire_fields,
},
web_surface_state::WebSurfacePendingInput,
web_surface_worker::{LiveRuntimeClient, LiveRuntimeWorker, WorkerResponse},
web_surface_worker::{LiveRuntimeClient, LiveRuntimeWorker, RequestGeneration, WorkerResponse},
};
use cleanup::{ScopedWorker, shutdown_scoped_worker};
pub(super) use super::web_surface_runtime_session::{
WebSurfaceEnsureResult, WebSurfaceRuntimeFrame, WebSurfaceUrlChange, WebSurfaceUrlChangeKind,
};
pub(super) struct WebSurfaceRuntime {
worker: Option<ScopedWorker>,
direct_client: Option<ScopedDirectClient>,
pending_direct_responses: Vec<WorkerResponse>,
workers: BTreeMap<WebSurfaceRuntimeScope, ScopedWorker>,
sessions: BTreeMap<TabId, WebSurfaceSession>,
retry_state: BTreeMap<WebSurfaceRuntimeScope, ScopeRetryState>,
retired_workers: Vec<JoinHandle<Result<(), String>>>,
transient_cleanup_error: Option<String>,
client_factory: LiveRuntimeClientFactory,
last_generation: u64,
}
const SIDECAR_RESTART_BASE_DELAY: Duration = Duration::from_millis(250);
const SIDECAR_RESTART_MAX_DELAY: Duration = Duration::from_secs(5);
impl WebSurfaceRuntime {
pub(super) fn new() -> Self {
let transient_cleanup_error =
cleanup_stale_transient_profile_data_dirs().err().map(|error| error.to_string());
Self {
worker: None,
direct_client: None,
pending_direct_responses: Vec::new(),
workers: BTreeMap::new(),
sessions: BTreeMap::new(),
retry_state: BTreeMap::new(),
retired_workers: Vec::new(),
transient_cleanup_error,
client_factory: new_servo_live_client,
last_generation: 0,
}
}
#[cfg(test)]
pub(super) fn new_with_client_factory(client_factory: LiveRuntimeClientFactory) -> Self {
Self {
worker: None,
direct_client: None,
pending_direct_responses: Vec::new(),
workers: BTreeMap::new(),
sessions: BTreeMap::new(),
retry_state: BTreeMap::new(),
retired_workers: Vec::new(),
transient_cleanup_error: None,
client_factory,
last_generation: 0,
}
}
#[cfg(test)]
pub(super) fn ensure_tab(
&mut self,
tab: &BrowserTab,
@@ -70,48 +80,11 @@ impl WebSurfaceRuntime {
profile_data_mode: ProfileDataMode,
permissions: &[WebSurfaceSitePermission],
input: WebSurfacePendingInput,
) -> Result<WebSurfaceEnsureResult, String> {
self.ensure_tab_inner(tab, size, None, profile_data_mode, permissions, input)
}
pub(super) fn ensure_tab_with_native_surface(
&mut self,
tab: &BrowserTab,
size: WebSurfaceSize,
native_surface: NativeSurfaceHandle,
profile_data_mode: ProfileDataMode,
permissions: &[WebSurfaceSitePermission],
input: WebSurfacePendingInput,
) -> Result<WebSurfaceEnsureResult, String> {
self.ensure_tab_inner(
tab,
size,
Some(native_surface),
profile_data_mode,
permissions,
input,
)
}
fn ensure_tab_inner(
&mut self,
tab: &BrowserTab,
size: WebSurfaceSize,
native_surface: Option<NativeSurfaceHandle>,
profile_data_mode: ProfileDataMode,
permissions: &[WebSurfaceSitePermission],
input: WebSurfacePendingInput,
) -> Result<WebSurfaceEnsureResult, String> {
let scope = WebSurfaceRuntimeScope::new(tab.profile_id().clone(), profile_data_mode);
let use_direct_client = native_surface.is_some();
if use_direct_client {
self.ensure_direct_client(scope.clone())?;
} else {
self.ensure_worker(scope.clone())?;
}
self.prepare_tab_scope(tab.id(), tab.profile_id(), profile_data_mode)?;
let requested_url = tab.url().as_str().to_string();
let tab_id_string = tab.id().as_str().to_string();
let zoom_percent = tab.zoom_percent();
let enqueued_at = input.enqueued_at;
let input_kind = pending_input_kind(&input);
@@ -119,6 +92,7 @@ impl WebSurfaceRuntime {
scroll_wire_fields(input.scroll_delta, input.scroll_point)?;
let user_navigation_input = input_requests_history_navigation(&input);
let next_scroll_offset = input.scroll_offset;
let generation = self.next_request_generation();
let submitted_at = Instant::now();
let started_loading = {
@@ -134,6 +108,7 @@ impl WebSurfaceRuntime {
session.size = size;
session.zoom_percent = zoom_percent;
session.scroll_offset = next_scroll_offset;
session.generation = Some(generation);
session.cadence.note_ensure(input_kind, started_loading, submitted_at);
started_loading
};
@@ -146,7 +121,6 @@ impl WebSurfaceRuntime {
height: size.height,
page_zoom_percent: zoom_percent,
device_pixel_ratio: size.device_pixel_ratio_f32(),
native_surface,
scroll_delta_x,
scroll_delta_y,
scroll_point_x,
@@ -159,15 +133,10 @@ impl WebSurfaceRuntime {
site_permissions: permissions.iter().map(ServoLiveSitePermission::from).collect(),
};
if use_direct_client {
let response = self.ensure_direct(request, tab_id_string.clone())?;
self.pending_direct_responses.extend(response);
} else {
let Some(scoped) = self.worker.as_ref() else {
return Err("Servo worker was created but is no longer registered".to_string());
};
scoped.worker.submit_ensure(request);
}
let Some(scoped) = self.workers.get(&scope) else {
return Err("Servo sidecar worker was created but is no longer registered".to_string());
};
scoped.worker.submit_ensure(generation, request);
if let Some(session) = self.sessions.get_mut(tab.id()) {
session.cadence.note_poll_submitted(submitted_at);
}
@@ -177,42 +146,45 @@ impl WebSurfaceRuntime {
}
pub(super) fn tick(&mut self, visible_tab_ids: &[TabId]) -> Vec<WebSurfaceRuntimeFrame> {
self.reap_retired_workers(false);
let mut frames = Vec::new();
let now = Instant::now();
let mut responses = std::mem::take(&mut self.pending_direct_responses);
responses.extend(
self.worker.as_ref().map(|scoped| scoped.worker.drain_responses()).unwrap_or_default(),
);
let runtime_unavailable = self.collect_responses(responses, now, &mut frames);
if runtime_unavailable {
self.remove_worker();
self.remove_direct_client();
let scopes = self.workers.keys().cloned().collect::<Vec<_>>();
let mut unavailable_scopes = Vec::new();
for scope in scopes {
let responses = self
.workers
.get(&scope)
.map(|scoped| scoped.worker.drain_responses())
.unwrap_or_default();
if self.collect_responses(&scope, responses, now, &mut frames) {
unavailable_scopes.push(scope);
}
}
for scope in unavailable_scopes {
self.invalidate_scope(&scope, now);
}
let poll_now = Instant::now();
let mut direct_polls = Vec::new();
for tab_id in visible_tab_ids {
let Some(session) = self.sessions.get_mut(tab_id) else {
let Some(scope) = self.sessions.get(tab_id).and_then(|session| {
session.cadence.should_poll(poll_now).then(|| session.scope.clone())
}) else {
continue;
};
if !session.cadence.should_poll(poll_now) {
if !self.workers.contains_key(&scope) {
continue;
}
if let Some(scoped) = self.worker.as_ref() {
let _ = scoped.worker.submit_poll(tab_id.as_str().to_string());
let generation = self.next_request_generation();
let submitted = self.workers.get(&scope).is_some_and(|scoped| {
scoped.worker.submit_poll(generation, tab_id.as_str().to_string())
});
if let Some(session) = self.sessions.get_mut(tab_id) {
if submitted {
session.generation = Some(generation);
}
session.cadence.note_poll_submitted(poll_now);
} else if self.direct_client.is_some() {
direct_polls.push(tab_id.as_str().to_string());
session.cadence.note_poll_submitted(poll_now);
}
}
if !direct_polls.is_empty() {
let (responses, runtime_unavailable) = self.poll_direct(direct_polls);
let unavailable_from_responses =
self.collect_responses(responses, Instant::now(), &mut frames);
if runtime_unavailable || unavailable_from_responses {
self.remove_direct_client();
}
}
@@ -227,95 +199,104 @@ impl WebSurfaceRuntime {
visible_tab_ids
.iter()
.filter_map(|tab_id| self.sessions.get(tab_id))
.filter(|_| self.worker.is_some() || self.direct_client.is_some())
.filter(|session| self.workers.contains_key(&session.scope))
.map(|session| session.cadence.next_poll_delay(now))
.min()
}
pub(super) fn close_tab(&mut self, tab_id: &TabId) {
if self.sessions.remove(tab_id).is_none() {
let Some(session) = self.sessions.remove(tab_id) else {
return;
}
let direct_result = self
.direct_client
.as_mut()
.map(|scoped| scoped.client.close(tab_id.as_str().to_string()));
if direct_result.as_ref().is_some_and(|result| {
result.as_ref().is_err_and(|error| error.is_runtime_unavailable())
}) {
self.remove_direct_client();
}
if let Some(scoped) = self.worker.as_ref() {
};
let has_remaining_session =
self.sessions.values().any(|candidate| candidate.scope == session.scope);
if session.scope.is_transient() && !has_remaining_session {
self.remove_worker(&session.scope);
} else if let Some(scoped) = self.workers.get(&session.scope) {
scoped.worker.submit_close(tab_id.as_str().to_string());
}
}
pub(super) fn has_session(
&self,
tab_id: &TabId,
profile_id: &ely_domain::ProfileId,
profile_data_mode: ProfileDataMode,
) -> bool {
self.sessions.get(tab_id).is_some_and(|session| {
session.scope == WebSurfaceRuntimeScope::new(profile_id.clone(), profile_data_mode)
&& self.workers.contains_key(&session.scope)
})
}
pub(super) fn prepare_tab_scope(
&mut self,
tab_id: &TabId,
profile_id: &ely_domain::ProfileId,
profile_data_mode: ProfileDataMode,
) -> Result<(), String> {
let scope = WebSurfaceRuntimeScope::new(profile_id.clone(), profile_data_mode);
self.detach_tab_from_previous_scope(tab_id, &scope);
self.ensure_worker(scope.clone())?;
session_for_scope(&mut self.sessions, tab_id, scope);
Ok(())
}
fn ensure_worker(&mut self, scope: WebSurfaceRuntimeScope) -> Result<(), String> {
if self.worker.is_some() {
self.reap_retired_workers(false);
if self.workers.contains_key(&scope) {
return Ok(());
}
if scope.is_transient() && self.transient_cleanup_error.is_some() {
match cleanup_stale_transient_profile_data_dirs() {
Ok(()) => self.transient_cleanup_error = None,
Err(error) => {
let message = error.to_string();
self.transient_cleanup_error = Some(message.clone());
return Err(message);
}
}
}
let now = Instant::now();
if let Some(retry) = self.retry_state.get(&scope)
&& now < retry.retry_after
{
return Err("Servo sidecar restart is cooling down".to_string());
}
let (config_dir, transient_profile_data_dir) = config_dir_for_scope(&scope)?;
let client_factory = self.client_factory;
let worker = LiveRuntimeWorker::new(move || client_factory(config_dir))?;
self.worker = Some(ScopedWorker { worker, transient_profile_data_dir });
Ok(())
}
fn ensure_direct_client(&mut self, scope: WebSurfaceRuntimeScope) -> Result<(), String> {
if self.direct_client.is_some() {
return Ok(());
}
let (config_dir, transient_profile_data_dir) = config_dir_for_scope(&scope)?;
let client = (self.client_factory)(config_dir)?;
self.direct_client = Some(ScopedDirectClient { client, transient_profile_data_dir });
Ok(())
}
fn ensure_direct(
&mut self,
request: ServoLiveEnsureRequest,
tab_id: String,
) -> Result<Option<WorkerResponse>, String> {
let Some(scoped) = self.direct_client.as_mut() else {
return Err("Servo client was created but is no longer registered".to_string());
};
match scoped.client.ensure(request) {
Ok(Some(frame)) => Ok(Some(WorkerResponse::Frame { tab_id, frame })),
Ok(None) => Ok(None),
let worker = match LiveRuntimeWorker::new(move || client_factory(config_dir)) {
Ok(worker) => worker,
Err(error) => {
let message = error.to_string();
if error.is_runtime_unavailable() {
self.remove_direct_client();
}
Err(message)
self.note_scope_failure(&scope, now);
return Err(error);
}
}
};
self.workers.insert(scope, ScopedWorker { worker, transient_profile_data_dir });
Ok(())
}
fn poll_direct(&mut self, tab_ids: Vec<String>) -> (Vec<WorkerResponse>, bool) {
let Some(scoped) = self.direct_client.as_mut() else {
return (Vec::new(), false);
fn detach_tab_from_previous_scope(&mut self, tab_id: &TabId, scope: &WebSurfaceRuntimeScope) {
let Some(previous_scope) = self.sessions.get(tab_id).map(|session| session.scope.clone())
else {
return;
};
let mut responses = Vec::new();
let mut runtime_unavailable = false;
for tab_id in tab_ids {
match scoped.client.poll(tab_id.clone()) {
Ok(Some(frame)) => responses.push(WorkerResponse::Frame { tab_id, frame }),
Ok(None) => {}
Err(error) if error.is_runtime_unavailable() => {
runtime_unavailable = true;
responses.push(WorkerResponse::RuntimeUnavailable);
}
Err(error) => {
responses.push(WorkerResponse::Failed { tab_id, message: error.to_string() })
}
}
if &previous_scope == scope {
return;
}
self.sessions.remove(tab_id);
let has_remaining_session =
self.sessions.values().any(|candidate| candidate.scope == previous_scope);
if previous_scope.is_transient() && !has_remaining_session {
self.remove_worker(&previous_scope);
} else if let Some(scoped) = self.workers.get(&previous_scope) {
scoped.worker.submit_close(tab_id.as_str().to_string());
}
(responses, runtime_unavailable)
}
fn collect_responses(
&mut self,
scope: &WebSurfaceRuntimeScope,
responses: Vec<WorkerResponse>,
now: Instant,
frames: &mut Vec<WebSurfaceRuntimeFrame>,
@@ -323,18 +304,23 @@ impl WebSurfaceRuntime {
let mut runtime_unavailable = false;
for response in responses {
match response {
WorkerResponse::Frame { tab_id, frame } => {
WorkerResponse::Frame { generation, tab_id, frame } => {
let Some(tab_id_obj) = self.lookup_session_tab_id(&tab_id) else {
continue;
};
let session = match self.sessions.get_mut(&tab_id_obj) {
Some(session) => session,
None => continue,
if !self.sessions.get(&tab_id_obj).is_some_and(|session| {
&session.scope == scope && session.generation == Some(generation)
}) {
continue;
}
self.note_scope_success(scope);
let Some(session) = self.sessions.get_mut(&tab_id_obj) else {
continue;
};
let requested_url = session.requested_url.clone();
let scroll_offset = session.scroll_offset;
let zoom_percent = session.zoom_percent;
session.cadence.note_frame(frame.render_state(), now);
session.cadence.note_frame(frame.render_state(), frame.pixels_changed(), now);
match WebSurfaceFrame::from_live_frame(
requested_url.clone(),
scroll_offset,
@@ -356,15 +342,17 @@ impl WebSurfaceRuntime {
}),
}
}
WorkerResponse::Failed { tab_id, message } => {
WorkerResponse::Failed { generation, tab_id, message } => {
let Some(tab_id_obj) = self.lookup_session_tab_id(&tab_id) else {
continue;
};
frames.push(WebSurfaceRuntimeFrame::Failed { tab_id: tab_id_obj, message });
}
WorkerResponse::RuntimeUnavailable => {
runtime_unavailable = true;
if self.sessions.get(&tab_id_obj).is_some_and(|session| {
&session.scope == scope && session.generation == Some(generation)
}) {
frames.push(WebSurfaceRuntimeFrame::Failed { tab_id: tab_id_obj, message });
}
}
WorkerResponse::RuntimeUnavailable => runtime_unavailable = true,
}
}
runtime_unavailable
@@ -374,9 +362,15 @@ impl WebSurfaceRuntime {
self.sessions.keys().find(|key| key.as_str() == tab_id).cloned()
}
fn next_request_generation(&mut self) -> RequestGeneration {
assert!(self.last_generation < u64::MAX, "web surface request generation exhausted");
self.last_generation += 1;
RequestGeneration::new(self.last_generation)
}
#[cfg(test)]
pub(super) fn client_count_for_test(&self) -> usize {
usize::from(self.worker.is_some()) + usize::from(self.direct_client.is_some())
self.workers.len()
}
#[cfg(test)]
@@ -385,39 +379,72 @@ impl WebSurfaceRuntime {
}
#[cfg(test)]
pub(super) fn flush_for_test(&self) {
if let Some(scoped) = self.worker.as_ref() {
pub(super) fn flush_for_test(&mut self) {
for scoped in self.workers.values() {
scoped.worker.wait_until_idle();
}
self.reap_retired_workers(true);
}
fn remove_worker(&mut self) {
let Some(scoped) = self.worker.take() else {
fn remove_worker(&mut self, scope: &WebSurfaceRuntimeScope) {
let Some(scoped) = self.workers.remove(scope) else {
return;
};
let ScopedWorker { worker, transient_profile_data_dir } = scoped;
drop(worker);
if let Some(path) = transient_profile_data_dir {
let _ = fs::remove_dir_all(path);
if scoped.transient_profile_data_dir.is_some() {
match std::thread::Builder::new()
.name("ely-servo-profile-cleanup".to_string())
.spawn(move || shutdown_scoped_worker(scoped))
{
Ok(handle) => self.retired_workers.push(handle),
Err(error) => self.transient_cleanup_error = Some(error.to_string()),
}
} else {
let _ = shutdown_scoped_worker(scoped);
}
}
fn remove_direct_client(&mut self) {
let Some(scoped) = self.direct_client.take() else {
return;
};
let ScopedDirectClient { client, transient_profile_data_dir } = scoped;
drop(client);
if let Some(path) = transient_profile_data_dir {
let _ = fs::remove_dir_all(path);
fn reap_retired_workers(&mut self, wait_for_all: bool) {
let mut pending = Vec::new();
for handle in self.retired_workers.drain(..) {
if !wait_for_all && !handle.is_finished() {
pending.push(handle);
continue;
}
match handle.join() {
Ok(Ok(())) => {}
Ok(Err(error)) => self.transient_cleanup_error = Some(error),
Err(_) => {
self.transient_cleanup_error =
Some("Servo profile cleanup thread panicked".to_string());
}
}
}
self.retired_workers = pending;
}
fn invalidate_scope(&mut self, scope: &WebSurfaceRuntimeScope, now: Instant) {
self.remove_worker(scope);
self.sessions.retain(|_, session| &session.scope != scope);
self.note_scope_failure(scope, now);
}
fn note_scope_failure(&mut self, scope: &WebSurfaceRuntimeScope, now: Instant) {
let retry = ScopeRetryState::after_failure(self.retry_state.get(scope), now);
self.retry_state.insert(scope.clone(), retry);
}
fn note_scope_success(&mut self, scope: &WebSurfaceRuntimeScope) {
self.retry_state.remove(scope);
}
}
impl Drop for WebSurfaceRuntime {
fn drop(&mut self) {
self.remove_worker();
self.remove_direct_client();
let scopes = self.workers.keys().cloned().collect::<Vec<_>>();
for scope in scopes {
self.remove_worker(&scope);
}
self.reap_retired_workers(true);
}
}
@@ -430,16 +457,38 @@ fn new_servo_live_client(config_dir: PathBuf) -> Result<Box<dyn LiveRuntimeClien
.map_err(|error| error.to_string())
}
struct ScopedWorker {
worker: LiveRuntimeWorker,
transient_profile_data_dir: Option<PathBuf>,
#[derive(Clone, Copy, Debug)]
struct ScopeRetryState {
failure_count: u32,
retry_after: Instant,
}
struct ScopedDirectClient {
client: Box<dyn LiveRuntimeClient>,
transient_profile_data_dir: Option<PathBuf>,
impl ScopeRetryState {
fn after_failure(previous: Option<&Self>, now: Instant) -> Self {
let failure_count = previous.map_or(1, |state| state.failure_count.saturating_add(1));
let shift = failure_count.saturating_sub(1).min(31);
let multiplier = 1_u32 << shift;
let delay =
SIDECAR_RESTART_BASE_DELAY.saturating_mul(multiplier).min(SIDECAR_RESTART_MAX_DELAY);
Self { failure_count, retry_after: now + delay }
}
}
#[path = "web_surface_runtime_cleanup.rs"]
mod cleanup;
#[cfg(test)]
#[path = "web_surface_runtime_backpressure_tests.rs"]
mod backpressure_tests;
#[cfg(test)]
#[path = "web_surface_runtime_cleanup_tests.rs"]
mod cleanup_tests;
#[cfg(test)]
#[path = "web_surface_runtime_generation_tests.rs"]
mod generation_tests;
#[cfg(test)]
#[path = "web_surface_runtime_retry_tests.rs"]
mod retry_tests;
#[cfg(test)]
#[path = "web_surface_runtime_tests.rs"]
mod tests;