ZacharyZhang-NY/ELY-Browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Servo IOSurface orientation and resize identity

Browse Source
This commit is contained in:
ZacharyZhang-NY
2026-05-13 10:21:50 -04:00
parent 9212b0be24
commit f4c650c4d8
6 changed files with 277 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ely_app/src/services/iosurface_metal.rs
+51 -24
View File
@@ -37,7 +37,13 @@ use thiserror::Error;
/// Constructed lazily by the renderer-side client on the first
/// hardware-path frame.
pub(crate) struct IOSurfaceCache {
pixel_buffers: HashMap<u64, CVPixelBuffer>,
pixel_buffers: HashMap<u64, CachedPixelBuffer>,
}
struct CachedPixelBuffer {
pixel_buffer: CVPixelBuffer,
width: u32,
height: u32,
}
#[derive(Debug, Error)]
@@ -54,21 +60,15 @@ impl IOSurfaceCache {
}
/// Import an IOSurface published by the sidecar's
/// `surface_handle` field. Idempotent on `surface_id`: a second
/// call with the same id immediately deallocates the duplicate
/// mach port without re-importing. The sender's T10.3 dedup means
/// the duplicate path should never fire in practice — it's here
/// so a misbehaving sidecar can't quietly leak ports.
/// `surface_handle` field. Idempotent on `surface_id` plus pixel
/// dimensions: duplicate handles for the same sized IOSurface are
/// discarded, while a resized IOSurface that reuses the same
/// `surface_id` replaces the cached pixel buffer.
pub fn import(
&mut self,
mach_port_name: u32,
surface_id: u64,
) -> Result<(), SurfaceImportError> {
if self.pixel_buffers.contains_key(&surface_id) {
deallocate_mach_port(mach_port_name);
return Ok(());
}
let Some(iosurface) = objc2_io_surface::IOSurfaceRef::lookup_from_mach_port(mach_port_name)
else {
return Err(SurfaceImportError::LookupFailed { port: mach_port_name });
@@ -91,8 +91,19 @@ impl IOSurfaceCache {
let pixel_buffer = CVPixelBuffer::from_io_surface(&io_surface_view, None)
.map_err(|status| SurfaceImportError::PixelBufferBuildFailed { status })?;
let width = pixel_buffer.get_width() as u32;
let height = pixel_buffer.get_height() as u32;
self.pixel_buffers.insert(surface_id, pixel_buffer);
if self
.pixel_buffers
.get(&surface_id)
.is_some_and(|cached| cached.width == width && cached.height == height)
{
deallocate_mach_port(mach_port_name);
return Ok(());
}
self.pixel_buffers.insert(surface_id, CachedPixelBuffer { pixel_buffer, width, height });
deallocate_mach_port(mach_port_name);
Ok(())
}
@@ -104,7 +115,7 @@ impl IOSurfaceCache {
/// atomic increment) so the caller can hand it to GPUI's
/// `surface(...)` element without holding a borrow on the cache.
pub fn pixel_buffer_for(&self, surface_id: u64) -> Option<CVPixelBuffer> {
self.pixel_buffers.get(&surface_id).cloned()
self.pixel_buffers.get(&surface_id).map(|cached| cached.pixel_buffer.clone())
}
#[cfg(test)]
@@ -156,9 +167,6 @@ mod tests {
};
use std::os::raw::c_void;
const TEST_WIDTH: u32 = 64;
const TEST_HEIGHT: u32 = 48;
/// Build a CPU-backed IOSurface from scratch, the same way
/// surfman's macOS backend does.
///
@@ -167,13 +175,13 @@ mod tests {
///
/// The pointer-casts mirror
/// `surfman::platform::macos::system::surface::create_io_surface`.
fn build_local_iosurface() -> Result<CFRetained<IOSurfaceRef>, String> {
fn build_local_iosurface(width: u32, height: u32) -> Result<CFRetained<IOSurfaceRef>, String> {
let pixel_format: i32 = i32::from_be_bytes(*b"BGRA");
let bytes_per_element: i32 = 4;
let bytes_per_row: i32 = (TEST_WIDTH as i32) * bytes_per_element;
let bytes_per_row: i32 = (width as i32) * bytes_per_element;
let width_num = CFNumber::new_i32(TEST_WIDTH as i32);
let height_num = CFNumber::new_i32(TEST_HEIGHT as i32);
let width_num = CFNumber::new_i32(width as i32);
let height_num = CFNumber::new_i32(height as i32);
let bpe_num = CFNumber::new_i32(bytes_per_element);
let bpr_num = CFNumber::new_i32(bytes_per_row);
let pf_num = CFNumber::new_i32(pixel_format);
@@ -207,7 +215,7 @@ mod tests {
#[test]
fn imports_local_iosurface_into_pixel_buffer() -> Result<(), String> {
let mut cache = IOSurfaceCache::new();
let iosurface = build_local_iosurface()?;
let iosurface = build_local_iosurface(64, 48)?;
let mach_port = iosurface.create_mach_port();
assert!(mach_port != 0, "IOSurfaceCreateMachPort must yield a real port");
let surface_id: u64 = 0xDEAD_BEEFu64;
@@ -219,12 +227,12 @@ mod tests {
.ok_or_else(|| "imported pixel buffer was missing".to_string())?;
assert_eq!(
pixel_buffer.get_width() as u32,
TEST_WIDTH,
64,
"CVPixelBuffer width must match the source IOSurface",
);
assert_eq!(
pixel_buffer.get_height() as u32,
TEST_HEIGHT,
48,
"CVPixelBuffer height must match the source IOSurface",
);
assert_eq!(cache.cached_surface_count(), 1);
@@ -234,7 +242,7 @@ mod tests {
#[test]
fn second_import_with_same_surface_id_is_idempotent() -> Result<(), String> {
let mut cache = IOSurfaceCache::new();
let iosurface = build_local_iosurface()?;
let iosurface = build_local_iosurface(64, 48)?;
let port_a = iosurface.create_mach_port();
let port_b = iosurface.create_mach_port();
assert!(port_a != 0 && port_b != 0 && port_a != port_b);
@@ -246,4 +254,23 @@ mod tests {
assert_eq!(cache.cached_surface_count(), 1);
Ok(())
}
#[test]
fn same_surface_id_with_changed_dimensions_replaces_pixel_buffer() -> Result<(), String> {
let mut cache = IOSurfaceCache::new();
let initial = build_local_iosurface(64, 48)?;
let resized = build_local_iosurface(96, 72)?;
let surface_id = 0xBBBB_BBBB;
cache.import(initial.create_mach_port(), surface_id).map_err(|error| error.to_string())?;
cache.import(resized.create_mach_port(), surface_id).map_err(|error| error.to_string())?;
let pixel_buffer = cache
.pixel_buffer_for(surface_id)
.ok_or_else(|| "resized pixel buffer was missing".to_string())?;
assert_eq!(pixel_buffer.get_width() as u32, 96);
assert_eq!(pixel_buffer.get_height() as u32, 72);
assert_eq!(cache.cached_surface_count(), 1);
Ok(())
}
}
crates/ely_app/src/shell/web_surface_live_site_tests.rs
+142
View File
@@ -28,6 +28,8 @@ use super::WebSurfaceStore;
const LIVE_SURFACE_WIDTH: u32 = 934;
const LIVE_SURFACE_HEIGHT: u32 = 657;
const RESIZED_LIVE_SURFACE_WIDTH: u32 = LIVE_SURFACE_WIDTH + 12;
const RESIZED_LIVE_SURFACE_HEIGHT: u32 = LIVE_SURFACE_HEIGHT + 20;
const MINIMUM_CONTENT_PIXELS: u64 = 1_000;
const LIVE_SITE_RENDER_ATTEMPTS: usize = 3;
const LIVE_SITE_WAIT_TIMEOUT: Duration = Duration::from_secs(20);
@@ -64,6 +66,13 @@ fn web_surface_scrolls_prd_site_down_and_up() -> Result<(), Box<dyn Error>> {
})
}
#[test]
fn web_surface_resizes_prd_site_without_failed_state() -> Result<(), Box<dyn Error>> {
run_isolated_live_site_test("web_surface_resizes_prd_site_without_failed_state", || {
assert_web_surface_resizes_prd_site()
})
}
fn run_isolated_live_site_test(
test_name: &str,
test: impl FnOnce() -> Result<(), Box<dyn Error>>,
@@ -166,6 +175,54 @@ fn assert_web_surface_scrolls_prd_site() -> Result<(), Box<dyn Error>> {
Ok(())
}
fn assert_web_surface_resizes_prd_site() -> Result<(), Box<dyn Error>> {
let mut store = WebSurfaceStore::new();
let profile_id = ProfileId::new();
let case = PRD_TOP_SITE_CASES
.iter()
.find(|case| case.url == "https://servo.org/")
.ok_or("missing servo.org live-site case")?;
let tab = web_tab(profile_id, case.url)?;
assert_eq!(
store.record_viewport_size(tab.id(), live_surface_bounds(), 1.0),
WebSurfaceInputOutcome::Applied,
"{}",
case.url,
);
store.ensure_surface(&tab, ProfileDataMode::Transient, &[]);
let _ = wait_for_ready_frame_at_size(
&mut store,
tab.id(),
case,
LIVE_SURFACE_WIDTH,
LIVE_SURFACE_HEIGHT,
)?;
assert_eq!(
store.record_viewport_size(tab.id(), resized_live_surface_bounds(), 1.0),
WebSurfaceInputOutcome::Buffered,
"{}",
case.url,
);
assert_eq!(
store.record_viewport_size(tab.id(), resized_live_surface_bounds(), 1.0),
WebSurfaceInputOutcome::Applied,
"{}",
case.url,
);
store.ensure_surface(&tab, ProfileDataMode::Transient, &[]);
wait_for_ready_frame_at_size(
&mut store,
tab.id(),
case,
RESIZED_LIVE_SURFACE_WIDTH,
RESIZED_LIVE_SURFACE_HEIGHT,
)?;
store.close_surface(tab.id());
Ok(())
}
fn render_web_surface_frame(
store: &mut WebSurfaceStore,
profile_id: &ProfileId,
@@ -283,6 +340,49 @@ fn wait_for_ready_frame_at_scroll(
}
}
fn wait_for_ready_frame_at_size(
store: &mut WebSurfaceStore,
tab_id: &TabId,
case: &LiveSiteCase,
expected_width: u32,
expected_height: u32,
) -> Result<WebSurfaceFrame, String> {
let started_at = Instant::now();
let mut last_error = None;
loop {
if started_at.elapsed() >= LIVE_SITE_WAIT_TIMEOUT {
return Err(last_error.unwrap_or_else(|| {
format!("timed out rendering {} at {expected_width}x{expected_height}", case.url)
}));
}
store.tick(std::slice::from_ref(tab_id));
match store.state(tab_id) {
Some(WebSurfaceState::Ready(frame))
if frame.size().width == expected_width
&& frame.size().height == expected_height =>
{
if let Err(error) =
validate_prd_frame_at_size(frame, case, expected_width, expected_height)
{
last_error = Some(error);
thread::sleep(LIVE_SITE_WAIT_INTERVAL);
continue;
}
return Ok(frame.clone());
}
Some(WebSurfaceState::Ready(_)) => {}
Some(WebSurfaceState::Failed { message, .. }) => {
return Err(format!("{} failed: {message}", case.url));
}
Some(WebSurfaceState::Loading { .. }) | None => {}
}
thread::sleep(LIVE_SITE_WAIT_INTERVAL);
}
}
fn validate_prd_frame(
frame: &WebSurfaceFrame,
case: &LiveSiteCase,
@@ -330,6 +430,41 @@ fn validate_prd_frame(
require(frame.sample_hash() > 0, case.url.to_string())
}
fn validate_prd_frame_at_size(
frame: &WebSurfaceFrame,
case: &LiveSiteCase,
expected_width: u32,
expected_height: u32,
) -> Result<(), String> {
require(
frame.size()
== WebSurfaceSize {
width: expected_width,
height: expected_height,
device_pixel_ratio_percent: 100,
},
format!("{} size: {:?}", case.url, frame.size()),
)?;
require_render_state_is_open(frame.render_state(), case.url)?;
require(
frame.url_label().contains(normalized_url(case.url)),
format!("url: {}", frame.url_label()),
)?;
require(
frame.title_label().contains(case.title_fragment),
format!("title: {}", frame.title_label()),
)?;
if frame.has_hardware_surface() {
return Ok(());
}
require(frame.non_white_pixel_count() > 0, case.url.to_string())?;
require(
frame.content_pixel_count() >= MINIMUM_CONTENT_PIXELS,
format!("{} content pixels: {}", case.url, frame.content_pixel_count()),
)?;
require(frame.sample_hash() > 0, case.url.to_string())
}
fn log_prd_frame(label: &str, frame: &WebSurfaceFrame, case: &LiveSiteCase) {
eprintln!(
"prd-live-site {label} url={} loaded={} title={} state={} size={}x{} content_pixels={} non_white_pixels={} sample_hash={}",
@@ -360,6 +495,13 @@ fn live_surface_bounds() -> Bounds<gpui::Pixels> {
)
}
fn resized_live_surface_bounds() -> Bounds<gpui::Pixels> {
Bounds::new(
point(px(0.0), px(0.0)),
size(px(RESIZED_LIVE_SURFACE_WIDTH as f32), px(RESIZED_LIVE_SURFACE_HEIGHT as f32)),
)
}
fn live_scroll_point() -> gpui::Point<gpui::Pixels> {
point(px(LIVE_SITE_SCROLL_POINT_X), px(LIVE_SITE_SCROLL_POINT_Y))
}
crates/ely_servo_host/src/bin/ely_servo_sidecar/live.rs
+6 -5
View File
@@ -8,9 +8,10 @@ use std::{
use ely_domain::{DEFAULT_ZOOM_PERCENT, ProfileId, TabId, UrlText};
use ely_servo_host::{
KeyboardTextRequest, MouseClickRequest, MouseHoverRequest, NavigationRequest, PageZoomRequest,
PermissionDecision, PermissionRequest, RenderingContextKind, ResizeRequest, ScrollRequest,
ServoHost, ServoSurfaceSize, SoftwareServoHost,
IOSurfaceIdentity, KeyboardTextRequest, MouseClickRequest, MouseHoverRequest,
NavigationRequest, PageZoomRequest, PermissionDecision, PermissionRequest,
RenderingContextKind, ResizeRequest, ScrollRequest, ServoHost, ServoSurfaceSize,
SoftwareServoHost,
};
use super::args::LiveArgs;
@@ -47,7 +48,7 @@ pub(super) fn run_live(args: LiveArgs) -> Result<(), LiveSidecarError> {
let mut perf =
FramePerfAggregator::new(context_label, FramePerfAggregator::DEFAULT_WINDOW_SIZE);
let mut pending_summary: Option<FramePerfSummary> = None;
let mut published_surface_ids: HashMap<String, HashSet<u64>> = HashMap::new();
let mut published_surface_ids: HashMap<String, HashSet<IOSurfaceIdentity>> = HashMap::new();
let stdin = io::stdin();
let mut stdout = io::stdout().lock();
@@ -91,7 +92,7 @@ const fn rendering_context_label(kind: RenderingContextKind) -> &'static str {
fn handle_request(
host: &mut SoftwareServoHost,
sessions: &mut HashMap<String, LiveSession>,
published_surface_ids: &mut HashMap<String, HashSet<u64>>,
published_surface_ids: &mut HashMap<String, HashSet<IOSurfaceIdentity>>,
rendering_context_kind: RenderingContextKind,
request: LiveRequest,
) -> Result<LiveOutcome, LiveSidecarError> {
crates/ely_servo_host/src/bin/ely_servo_sidecar/live_output.rs
+63 -18
View File
@@ -29,7 +29,7 @@ pub(super) fn populate_surface_fields(
host: &SoftwareServoHost,
webview_id: &ely_domain::WebViewId,
tab_id: &str,
published_surface_ids: &mut HashMap<String, HashSet<u64>>,
published_surface_ids: &mut HashMap<String, HashSet<IOSurfaceIdentity>>,
outcome: &mut LiveOutcome,
) {
if outcome.response.frame.is_none() {
@@ -40,6 +40,7 @@ pub(super) fn populate_surface_fields(
let Ok(Some(identity)) = host.peek_iosurface_identity(webview_id) else {
return;
};
align_report_to_surface_identity(outcome, identity);
let handle = if surface_has_been_published(published_surface_ids, tab_id, identity) {
None
} else {
@@ -57,13 +58,11 @@ pub(super) fn populate_surface_fields(
#[cfg(any(test, all(feature = "hardware-render", target_os = "macos")))]
fn surface_has_been_published(
published_surface_ids: &HashMap<String, HashSet<u64>>,
published_surface_ids: &HashMap<String, HashSet<IOSurfaceIdentity>>,
tab_id: &str,
identity: IOSurfaceIdentity,
) -> bool {
published_surface_ids
.get(tab_id)
.is_some_and(|published| published.contains(&identity.surface_id))
published_surface_ids.get(tab_id).is_some_and(|published| published.contains(&identity))
}
#[cfg(any(test, all(feature = "hardware-render", target_os = "macos")))]
@@ -75,7 +74,7 @@ struct SurfacePublication {
#[cfg(any(test, all(feature = "hardware-render", target_os = "macos")))]
fn surface_publication_for(
published_surface_ids: &mut HashMap<String, HashSet<u64>>,
published_surface_ids: &mut HashMap<String, HashSet<IOSurfaceIdentity>>,
tab_id: &str,
identity: IOSurfaceIdentity,
handle: Option<IOSurfaceHandle>,
@@ -91,7 +90,10 @@ fn surface_publication_for(
return SurfacePublication { current_surface_id: None, surface_handle: None };
};
published_surface_ids.entry(tab_id.to_string()).or_default().insert(handle.surface_id);
published_surface_ids
.entry(tab_id.to_string())
.or_default()
.insert(IOSurfaceIdentity::from_handle(handle));
SurfacePublication { current_surface_id: Some(handle.surface_id), surface_handle: Some(handle) }
}
@@ -102,6 +104,14 @@ fn handle_matches_identity(handle: IOSurfaceHandle, identity: IOSurfaceIdentity)
&& handle.height == identity.height
}
#[cfg(any(test, all(feature = "hardware-render", target_os = "macos")))]
fn align_report_to_surface_identity(outcome: &mut LiveOutcome, identity: IOSurfaceIdentity) {
if let Some(frame) = outcome.response.frame.as_mut() {
frame.width = identity.width;
frame.height = identity.height;
}
}
/// Serialise the response then stream the optional raw RGBA frame on
/// the same stdout pipe. The client reads the JSON line, takes
/// `rgba_byte_count` from the report, then reads that many bytes
@@ -163,11 +173,7 @@ pub(super) fn write_outcome(
#[cfg(test)]
mod tests {
use std::{
collections::{HashMap, HashSet},
error::Error,
time::Instant,
};
use std::{collections::HashMap, error::Error, time::Instant};
use ely_servo_host::{IOSurfaceHandle, IOSurfaceIdentity};
@@ -175,7 +181,7 @@ mod tests {
live_protocol::{LiveFrameReport, LiveOutcome, PartialFrameTimings},
perf::FramePerfAggregator,
};
use super::{surface_publication_for, write_outcome};
use super::{align_report_to_surface_identity, surface_publication_for, write_outcome};
#[test]
fn unpublished_surface_without_handle_leaves_selector_empty() {
@@ -197,12 +203,15 @@ mod tests {
assert_eq!(publication.current_surface_id, Some(7));
assert_eq!(publication.surface_handle, Some(handle));
assert!(published.get("tab-1").is_some_and(|ids| ids.contains(&7)));
assert!(published.get("tab-1").is_some_and(|ids| ids.contains(&identity(7, 800, 600))));
}
#[test]
fn published_surface_reuses_selector_without_republishing_handle() {
let mut published = HashMap::from([("tab-1".to_string(), HashSet::from([7]))]);
let mut published = HashMap::new();
let handle = handle(7, 800, 600);
let _ =
surface_publication_for(&mut published, "tab-1", identity(7, 800, 600), Some(handle));
let publication =
surface_publication_for(&mut published, "tab-1", identity(7, 800, 600), None);
@@ -210,6 +219,36 @@ mod tests {
assert!(publication.surface_handle.is_none());
}
#[test]
fn same_surface_id_with_changed_dimensions_republishes_handle() {
let mut published = HashMap::new();
let initial = handle(7, 800, 600);
let resized = handle(7, 1024, 768);
let _ =
surface_publication_for(&mut published, "tab-1", identity(7, 800, 600), Some(initial));
let publication =
surface_publication_for(&mut published, "tab-1", identity(7, 1024, 768), Some(resized));
assert_eq!(publication.current_surface_id, Some(7));
assert_eq!(publication.surface_handle, Some(resized));
}
#[test]
fn hardware_report_uses_surface_identity_dimensions() -> Result<(), Box<dyn Error>> {
let mut outcome = LiveOutcome::from_report(
report_with_size(2180, 1586),
PartialFrameTimings { paint_ns: 1_000, encode_ns: 2_000 },
);
align_report_to_surface_identity(&mut outcome, identity(7, 2168, 1566));
let report = outcome.response.frame.ok_or("report must remain present")?;
assert_eq!(report.width, 2168);
assert_eq!(report.height, 1566);
Ok(())
}
#[test]
fn mismatched_handle_leaves_surface_unpublished() {
let mut published = HashMap::new();
@@ -263,13 +302,19 @@ mod tests {
}
fn report_with_byte_count(rgba_byte_count: usize) -> LiveFrameReport {
let mut report = report_with_size(2, 2);
report.rgba_byte_count = rgba_byte_count;
report
}
fn report_with_size(width: u32, height: u32) -> LiveFrameReport {
LiveFrameReport {
loaded_url: Some("https://example.com/".to_string()),
title: Some("Example".to_string()),
state: "complete",
width: 2,
height: 2,
rgba_byte_count,
width,
height,
rgba_byte_count: 0,
non_white_pixel_count: 0,
content_pixel_count: 0,
sample_hash: 0,
crates/ely_servo_host/src/iosurface_handle.rs
+12 -5
View File
@@ -16,10 +16,11 @@
/// a Metal texture without copying pixels.
///
/// `surface_id` is the stable surfman `SurfaceID` (a pointer-shaped
/// `usize` widened to `u64` for the wire). It lets the receiver dedup:
/// when two consecutive frames carry the same `surface_id` the
/// imported `MTLTexture` is reused without re-importing. `width` and
/// `height` are reported in surface pixels (post-DPR).
/// `usize` widened to `u64` for the wire). Together with `width` and
/// `height` it lets the receiver dedup imported IOSurfaces. The pixel
/// dimensions are part of the identity because a resize can reuse the
/// same surfman id for a newly-sized IOSurface. `width` and `height`
/// are reported in surface pixels (post-DPR).
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[cfg_attr(feature = "servo-engine", derive(serde::Serialize, serde::Deserialize))]
pub struct IOSurfaceHandle {
@@ -33,9 +34,15 @@ pub struct IOSurfaceHandle {
/// "same surface as last frame" from "resize/swap rotated to a new
/// surface" without minting a fresh mach port (mach ports are a scarce
/// kernel resource and `IOSurfaceCreateMachPort` is not cheap).
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct IOSurfaceIdentity {
pub surface_id: u64,
pub width: u32,
pub height: u32,
}
impl IOSurfaceIdentity {
pub fn from_handle(handle: IOSurfaceHandle) -> Self {
Self { surface_id: handle.surface_id, width: handle.width, height: handle.height }
}
}