Present Servo BGRA hardware surfaces

This commit is contained in:
2026-05-13 00:22:00 -04:00
parent bf1ebfb6fe
commit 05a7a0d67f
154 changed files with 84157 additions and 115 deletions
+22 -20
View File
@@ -3,10 +3,9 @@
//!
//! `T10.4` originally imported the IOSurface into an `MTLTexture`
//! directly. GPUI 0.2.2 exposes `Window::paint_surface` /
//! `elements::surface::Surface` for `CVPixelBuffer`, and that public
//! path is wired for NV12 video frames. Servo's hardware renderer
//! publishes BGRA IOSurfaces, so this cache stays as verified
//! cross-process plumbing until the presenter accepts BGRA surfaces.
//! `elements::surface::Surface` for `CVPixelBuffer`; the local GPUI
//! patch adds a BGRA fragment pipeline for Servo's hardware
//! IOSurfaces, so this cache is the renderer-side handoff point.
//!
//! Lifetime contract:
//!
@@ -161,11 +160,14 @@ mod tests {
const TEST_HEIGHT: u32 = 48;
/// Build a CPU-backed IOSurface from scratch, the same way
/// surfman's macOS backend does — BGRA8 (four-cc '32BGRA'), width
/// + height + bytes_per_element + bytes_per_row in a Core
/// Foundation properties dictionary. The pointer-casts mirror
/// surfman's macOS backend does.
///
/// BGRA8 (four-cc '32BGRA'), width + height + bytes_per_element
/// + bytes_per_row live in a Core Foundation properties dictionary.
///
/// The pointer-casts mirror
/// `surfman::platform::macos::system::surface::create_io_surface`.
fn build_local_iosurface() -> CFRetained<IOSurfaceRef> {
fn build_local_iosurface() -> Result<CFRetained<IOSurfaceRef>, String> {
let pixel_format: i32 = i32::from_be_bytes(*b"BGRA");
let bytes_per_element: i32 = 4;
let bytes_per_row: i32 = (TEST_WIDTH as i32) * bytes_per_element;
@@ -196,27 +198,25 @@ mod tests {
&kCFTypeDictionaryKeyCallBacks,
&kCFTypeDictionaryValueCallBacks,
)
.expect("CFDictionaryCreate must succeed for the properties dict");
.ok_or_else(|| "CFDictionaryCreate returned null".to_string())?;
IOSurfaceRef::new(&properties)
.expect("IOSurfaceCreate must succeed for a well-formed properties dict")
.ok_or_else(|| "IOSurfaceCreate returned null".to_string())
}
}
#[test]
fn imports_local_iosurface_into_pixel_buffer() {
fn imports_local_iosurface_into_pixel_buffer() -> Result<(), String> {
let mut cache = IOSurfaceCache::new();
let iosurface = build_local_iosurface();
let iosurface = build_local_iosurface()?;
let mach_port = iosurface.create_mach_port();
assert!(mach_port != 0, "IOSurfaceCreateMachPort must yield a real port");
let surface_id: u64 = 0xDEAD_BEEFu64;
cache
.import(mach_port, surface_id)
.expect("local IOSurface must round-trip into a CVPixelBuffer");
cache.import(mach_port, surface_id).map_err(|error| error.to_string())?;
let pixel_buffer = cache
.pixel_buffer_for(surface_id)
.expect("imported pixel buffer must be retrievable by surface_id");
.ok_or_else(|| "imported pixel buffer was missing".to_string())?;
assert_eq!(
pixel_buffer.get_width() as u32,
TEST_WIDTH,
@@ -228,20 +228,22 @@ mod tests {
"CVPixelBuffer height must match the source IOSurface",
);
assert_eq!(cache.cached_surface_count(), 1);
Ok(())
}
#[test]
fn second_import_with_same_surface_id_is_idempotent() {
fn second_import_with_same_surface_id_is_idempotent() -> Result<(), String> {
let mut cache = IOSurfaceCache::new();
let iosurface = build_local_iosurface();
let iosurface = build_local_iosurface()?;
let port_a = iosurface.create_mach_port();
let port_b = iosurface.create_mach_port();
assert!(port_a != 0 && port_b != 0 && port_a != port_b);
cache.import(port_a, 0xAAAA_AAAA).expect("first import");
cache.import(port_a, 0xAAAA_AAAA).map_err(|error| error.to_string())?;
// Same surface_id → defensive dedup path; port_b is deallocated
// without minting a duplicate CVPixelBuffer.
cache.import(port_b, 0xAAAA_AAAA).expect("duplicate import is idempotent");
cache.import(port_b, 0xAAAA_AAAA).map_err(|error| error.to_string())?;
assert_eq!(cache.cached_surface_count(), 1);
Ok(())
}
}
+37 -30
View File
@@ -10,7 +10,7 @@ use std::{
/// (default — bit-identical to pre-flag builds) and `hardware` (real
/// GPU adapter via the vendored `HardwareOffscreenContext`; requires
/// the sidecar binary to be compiled with the `hardware-render`
/// feature and a GPUI BGRA surface presenter). Anything else is
/// feature and the local GPUI BGRA surface presenter). Anything else is
/// silently dropped and the sidecar defaults to software so a typo'd
/// value never blocks the browser from starting; the sidecar's own
/// arg parser still errors loudly on an unrecognised value when set
@@ -143,8 +143,8 @@ impl ServoLiveClient {
// header so a buggy or hostile sidecar can't park us on
// `read_exact` for an arbitrarily-sized buffer. The honest
// upper limit is `width * height * 4` (RGBA8); `0` is the
// explicit "hardware path active, sample the IOSurface
// instead" signal anything else is a protocol violation.
// explicit "hardware path active, sample the IOSurface"
// signal; any other byte count is a protocol violation.
let pixel_byte_count =
(report.width as u64).saturating_mul(report.height as u64).saturating_mul(4);
let advertised = report.rgba_byte_count as u64;
@@ -157,12 +157,10 @@ impl ServoLiveClient {
});
}
// Raw frame bytes follow the JSON header on the same pipe
// ONLY when the sidecar didn't drop the payload for the
// hardware path. `read_exact` drains BufReader's buffer first
// Raw frame bytes follow the JSON header on the same pipe for
// software frames. `read_exact` drains BufReader's buffer first
// (the line read never crosses the `\n` boundary) and then
// pulls the rest straight from the child's stdout — no
// fs::read, no temp file.
// pulls the rest straight from the child's stdout.
let mut rgba_bytes = vec![0u8; report.rgba_byte_count];
if report.rgba_byte_count > 0 {
self.stdout.read_exact(&mut rgba_bytes).map_err(ServoLiveError::FrameRead)?;
@@ -189,10 +187,8 @@ impl Drop for ServoLiveClient {
#[cfg(target_os = "macos")]
impl ServoLiveClient {
/// Convert the sidecar's `surface_handle` into a `CVPixelBuffer`
/// in the local cache. Failures are logged but don't error the
/// request — the renderer falls back to the existing software
/// `Arc<RenderImage>` path when no pixel buffer is available, so
/// the user always sees a frame.
/// in the local cache. A later frame with a missing pixel buffer
/// becomes a web-surface error instead of a blank ready frame.
fn import_iosurface_handle(&mut self, handle: &LiveSurfaceHandle) {
match self.iosurface_cache.import(handle.mach_port_name, handle.surface_id) {
Ok(()) => tracing::info!(
@@ -266,10 +262,8 @@ pub(crate) struct ServoLiveFrame {
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: u64,
rgba_bytes: Vec<u8>,
/// Hardware-path companion: the imported IOSurface published by
/// the sidecar. GPUI 0.2.2 presents `surface(...)` through its
/// NV12 video path, so the current BGRA Servo surface stays as
/// observability plumbing until a BGRA presenter lands.
/// Hardware-path surface: the imported IOSurface published by the
/// sidecar, wrapped as a CVPixelBuffer for GPUI's `surface(...)`.
#[cfg(target_os = "macos")]
pixel_buffer: Option<CVPixelBuffer>,
}
@@ -295,9 +289,7 @@ impl ServoLiveFrame {
}
/// Returns the imported `CVPixelBuffer` matching the frame's
/// current hardware surface, if any. The renderer keeps this as
/// wire-path evidence while GPUI's public `surface(...)` element
/// remains NV12-only.
/// current hardware surface.
#[cfg(target_os = "macos")]
#[must_use]
pub fn pixel_buffer(&self) -> Option<&CVPixelBuffer> {
@@ -371,6 +363,29 @@ impl ServoLiveFrame {
pixel_buffer: None,
}
}
#[cfg(all(test, target_os = "macos"))]
pub(crate) fn for_test_with_pixel_buffer(
width: u32,
height: u32,
pixel_buffer: CVPixelBuffer,
) -> Self {
Self {
loaded_url: Some("https://example.com/".to_string()),
title: Some("Example".to_string()),
render_state: "complete".to_string(),
width,
height,
#[cfg(all(test, feature = "live-site-smoke"))]
non_white_pixel_count: 0,
#[cfg(all(test, feature = "live-site-smoke"))]
content_pixel_count: 0,
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: 0,
rgba_bytes: Vec::new(),
pixel_buffer: Some(pixel_buffer),
}
}
}
#[derive(Debug, Error)]
@@ -417,13 +432,6 @@ fn rendering_context_from_env() -> Option<&'static str> {
let raw = env::var(RENDERING_CONTEXT_ENV).ok()?;
match rendering_context_selection(raw.as_str()) {
RenderingContextSelection::Forward(value) => Some(value),
RenderingContextSelection::HoldHardware => {
tracing::warn!(
target: "ely::servo::iosurface",
"hardware rendering context requested; GPUI 0.2.2 surface presenter accepts NV12 CVPixelBuffers; Servo publishes BGRA IOSurfaces; using software rendering context",
);
None
}
RenderingContextSelection::Ignore => None,
}
}
@@ -431,14 +439,13 @@ fn rendering_context_from_env() -> Option<&'static str> {
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RenderingContextSelection {
Forward(&'static str),
HoldHardware,
Ignore,
}
fn rendering_context_selection(raw: &str) -> RenderingContextSelection {
match raw.to_lowercase().as_str() {
"software" => RenderingContextSelection::Forward("software"),
"hardware" => RenderingContextSelection::HoldHardware,
"hardware" => RenderingContextSelection::Forward("hardware"),
_ => RenderingContextSelection::Ignore,
}
}
@@ -448,10 +455,10 @@ mod tests {
use super::{RenderingContextSelection, rendering_context_selection};
#[test]
fn hardware_env_is_held_until_gpui_can_present_bgra_surfaces() {
fn hardware_env_forwards_to_the_sidecar() {
assert_eq!(
rendering_context_selection("hardware"),
RenderingContextSelection::HoldHardware
RenderingContextSelection::Forward("hardware")
);
}
@@ -7,6 +7,9 @@ use std::{
use thiserror::Error;
const SIDECAR_PATH_ENV: &str = "ELY_SERVO_SIDECAR";
const RENDERING_CONTEXT_ENV: &str = "ELY_SERVO_RENDERING_CONTEXT";
const SOFTWARE_SIDECAR_FEATURES: &str = "servo-engine";
const HARDWARE_SIDECAR_FEATURES: &str = "servo-engine,hardware-render";
#[derive(Clone, Debug)]
pub(super) enum SidecarCommandTarget {
@@ -28,7 +31,7 @@ impl SidecarCommandTarget {
.arg("-p")
.arg("ely_servo_host")
.arg("--features")
.arg("servo-engine")
.arg(sidecar_features_from_env())
.arg("--bin")
.arg("ely_servo_sidecar")
.arg("--");
@@ -64,13 +67,17 @@ pub(super) fn default_sidecar_command() -> Result<SidecarCommandTarget, SidecarC
SidecarCommandError::CurrentExecutableDirectoryUnavailable { path: current_exe.clone() }
})?;
let adjacent_sidecar = exe_dir.join(sidecar_binary_name());
if adjacent_sidecar.is_file() {
let workspace_manifest = workspace_manifest_path();
let prefer_cargo_hardware_sidecar = hardware_rendering_context_requested()
&& workspace_manifest.as_ref().is_some_and(|path| path.is_file());
if adjacent_sidecar.is_file() && !prefer_cargo_hardware_sidecar {
return Ok(SidecarCommandTarget::Binary(adjacent_sidecar));
}
if let Some(manifest_path) = workspace_manifest_path() {
if let Some(manifest_path) = workspace_manifest {
if let Some(target_sidecar) = workspace_target_sidecar_path(&manifest_path)
&& target_sidecar.is_file()
&& !prefer_cargo_hardware_sidecar
{
return Ok(SidecarCommandTarget::Binary(target_sidecar));
}
@@ -86,6 +93,30 @@ fn workspace_manifest_path() -> Option<PathBuf> {
option_env!("ELY_WORKSPACE_MANIFEST").map(PathBuf::from)
}
fn hardware_rendering_context_requested() -> bool {
env::var(RENDERING_CONTEXT_ENV).ok().as_deref().is_some_and(rendering_context_requests_hardware)
}
fn sidecar_features_from_env() -> &'static str {
env::var(RENDERING_CONTEXT_ENV)
.ok()
.as_deref()
.map(sidecar_features_for_rendering_context)
.unwrap_or(SOFTWARE_SIDECAR_FEATURES)
}
fn sidecar_features_for_rendering_context(raw: &str) -> &'static str {
if rendering_context_requests_hardware(raw) {
HARDWARE_SIDECAR_FEATURES
} else {
SOFTWARE_SIDECAR_FEATURES
}
}
fn rendering_context_requests_hardware(raw: &str) -> bool {
raw.eq_ignore_ascii_case("hardware")
}
fn workspace_target_sidecar_path(manifest_path: &Path) -> Option<PathBuf> {
let profile = if cfg!(debug_assertions) { "debug" } else { "release" };
Some(manifest_path.parent()?.join("target").join(profile).join(sidecar_binary_name()))
@@ -94,3 +125,23 @@ fn workspace_target_sidecar_path(manifest_path: &Path) -> Option<PathBuf> {
fn sidecar_binary_name() -> String {
format!("ely_servo_sidecar{}", env::consts::EXE_SUFFIX)
}
#[cfg(test)]
mod tests {
use super::{
HARDWARE_SIDECAR_FEATURES, SOFTWARE_SIDECAR_FEATURES,
sidecar_features_for_rendering_context,
};
#[test]
fn hardware_rendering_context_enables_hardware_sidecar_feature() {
assert_eq!(sidecar_features_for_rendering_context("hardware"), HARDWARE_SIDECAR_FEATURES);
assert_eq!(sidecar_features_for_rendering_context("HARDWARE"), HARDWARE_SIDECAR_FEATURES);
}
#[test]
fn software_and_unknown_contexts_use_software_sidecar_feature() {
assert_eq!(sidecar_features_for_rendering_context("software"), SOFTWARE_SIDECAR_FEATURES);
assert_eq!(sidecar_features_for_rendering_context("garbage"), SOFTWARE_SIDECAR_FEATURES);
}
}
+4 -4
View File
@@ -177,12 +177,12 @@ impl WebSurfaceStore {
position: Point<Pixels>,
scale_factor: f32,
) -> WebSurfaceInputOutcome {
let surface =
self.surfaces.get_mut(tab_id).filter(|surface| surface.viewport_bounds.is_some());
let Some(surface) = surface else {
let Some(surface) = self.surfaces.get_mut(tab_id) else {
return WebSurfaceInputOutcome::DroppedNoViewportBounds;
};
let Some(bounds) = surface.viewport_bounds else {
return WebSurfaceInputOutcome::DroppedNoViewportBounds;
};
let bounds = surface.viewport_bounds.expect("viewport_bounds checked above");
let Some(point) =
WebSurfaceClickPoint::from_window_position(bounds, position, scale_factor)
else {
+33 -28
View File
@@ -57,14 +57,12 @@ pub(super) struct WebSurfaceFrame {
content_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: u64,
/// Software-path image. Current GPUI builds require this for every
/// ready web frame because BGRA IOSurface presentation is still
/// held at the protocol boundary.
/// Software-path image built from RGBA bytes when the sidecar runs
/// without hardware surface publication.
pub(super) image: Option<Arc<RenderImage>>,
/// Hardware-path companion imported from the sidecar. GPUI 0.2.2's
/// public `surface(...)` presenter accepts NV12 video buffers, and
/// Servo publishes BGRA IOSurfaces; this remains observability
/// state until a BGRA presenter is available.
/// Hardware-path surface imported from the sidecar's IOSurface.
/// GPUI is patched locally to present BGRA CVPixelBuffers through
/// `surface(...)`, so hardware frames can skip the RGBA pipe.
#[cfg(target_os = "macos")]
pub(super) pixel_buffer: Option<CVPixelBuffer>,
}
@@ -102,23 +100,32 @@ impl WebSurfaceFrame {
}
fn from_parts(parts: WebSurfaceFrameParts) -> Result<Self, WebSurfaceError> {
if parts.rgba_bytes.is_empty() {
#[cfg(target_os = "macos")]
let has_pixel_buffer = parts.pixel_buffer.is_some();
#[cfg(not(target_os = "macos"))]
let has_pixel_buffer = false;
if parts.rgba_bytes.is_empty() && !has_pixel_buffer {
return Err(WebSurfaceError::MissingRenderablePayload);
}
// Servo's `read_pixels(gl::RGBA, gl::UNSIGNED_BYTE)` writes
// R-G-B-A in memory order. GPUI's `RenderImage` is documented
// as "in BGRA format" and uploads via
// `MTLPixelFormat::BGRA8Unorm`, which reads B-G-R-A. Hand the bytes across
// unchanged and the Metal sampler treats R as B (and vice
// versa) — every coloured pixel renders with R and B swapped.
// Swap once here so the rest of the pipeline (dedup hash,
// image buffer, GPU upload) all operate on the same BGRA
// representation.
let mut bytes = parts.rgba_bytes;
swap_red_blue_in_place(&mut bytes);
let bytes_hash = rgba_hash(&bytes);
let image = Some(resolve_render_image(parts.width, parts.height, bytes, bytes_hash)?);
let image = if parts.rgba_bytes.is_empty() {
None
} else {
// Servo's `read_pixels(gl::RGBA, gl::UNSIGNED_BYTE)` writes
// R-G-B-A in memory order. GPUI's `RenderImage` is documented
// as "in BGRA format" and uploads via
// `MTLPixelFormat::BGRA8Unorm`, which reads B-G-R-A. Hand the bytes across
// unchanged and the Metal sampler treats R as B (and vice
// versa) — every coloured pixel renders with R and B swapped.
// Swap once here so the rest of the pipeline (dedup hash,
// image buffer, GPU upload) all operate on the same BGRA
// representation.
let mut bytes = parts.rgba_bytes;
swap_red_blue_in_place(&mut bytes);
let bytes_hash = rgba_hash(&bytes);
Some(resolve_render_image(parts.width, parts.height, bytes, bytes_hash)?)
};
Ok(Self {
requested_url: parts.requested_url,
@@ -240,9 +247,7 @@ struct WebSurfaceFrameParts {
pub(super) enum WebSurfaceError {
#[error("invalid servo frame buffer for {width}x{height}")]
InvalidFrameBuffer { width: u32, height: u32 },
#[error(
"servo live frame did not include renderable pixels; BGRA IOSurface presentation is unavailable in GPUI 0.2.2"
)]
#[error("servo live frame did not include a software image or hardware IOSurface")]
MissingRenderablePayload,
}
@@ -269,10 +274,10 @@ fn resolve_render_image(
) -> Result<Arc<RenderImage>, WebSurfaceError> {
LAST_FRAME_IMAGE.with(|cache| -> Result<Arc<RenderImage>, WebSurfaceError> {
let mut cache = cache.borrow_mut();
if let Some((cached_hash, cached_image)) = cache.as_ref() {
if *cached_hash == bytes_hash {
return Ok(cached_image.clone());
}
if let Some((cached_hash, cached_image)) = cache.as_ref()
&& *cached_hash == bytes_hash
{
return Ok(cached_image.clone());
}
let image_buffer = ImageBuffer::<Rgba<u8>, _>::from_raw(width, height, rgba_bytes)
+31 -4
View File
@@ -384,7 +384,7 @@ fn live_frame_swaps_red_and_blue_bytes_for_gpui_bgra() -> Result<(), Box<dyn Err
}
#[test]
fn empty_live_frame_payload_is_rejected() {
fn empty_live_frame_payload_is_rejected() -> Result<(), String> {
use crate::services::servo_live::ServoLiveFrame;
use crate::shell::web_surface_frame::WebSurfaceFrame;
use crate::shell::web_surface_geometry::WebSurfaceScrollOffset;
@@ -396,13 +396,40 @@ fn empty_live_frame_payload_is_rejected() {
ServoLiveFrame::for_test(1, 1, Vec::new()),
);
let Err(error) = result else {
panic!("empty Servo frame payload must be rejected before it reaches Ready state");
let error = match result {
Ok(_) => return Err("empty Servo frame payload reached Ready state".to_string()),
Err(error) => error,
};
assert_eq!(
error.to_string(),
"servo live frame did not include renderable pixels; BGRA IOSurface presentation is unavailable in GPUI 0.2.2",
"servo live frame did not include a software image or hardware IOSurface",
);
Ok(())
}
#[cfg(target_os = "macos")]
#[test]
fn hardware_live_frame_with_pixel_buffer_skips_software_image() -> Result<(), String> {
use core_video::pixel_buffer::{CVPixelBuffer, kCVPixelFormatType_32BGRA};
use crate::services::servo_live::ServoLiveFrame;
use crate::shell::web_surface_frame::WebSurfaceFrame;
use crate::shell::web_surface_geometry::WebSurfaceScrollOffset;
let pixel_buffer = CVPixelBuffer::new(kCVPixelFormatType_32BGRA, 1, 1, None)
.map_err(|status| format!("CVPixelBufferCreate returned status {status}"))?;
let live = ServoLiveFrame::for_test_with_pixel_buffer(1, 1, pixel_buffer);
let frame = WebSurfaceFrame::from_live_frame(
"https://example.com/".to_string(),
WebSurfaceScrollOffset::default(),
100,
live,
)
.map_err(|error| error.to_string())?;
assert!(frame.image.is_none(), "hardware frame should use the CVPixelBuffer surface path");
assert!(frame.pixel_buffer.is_some(), "hardware frame should carry the imported CVPixelBuffer");
Ok(())
}
fn web_bounds() -> Bounds<gpui::Pixels> {
+10 -17
View File
@@ -1,7 +1,7 @@
use ely_domain::{BrowserTab, TabId};
use gpui::{
AnyElement, App, Entity, ImageSource, InteractiveElement, IntoElement, MouseButton, ObjectFit,
ParentElement, Styled, StyledImage, Window, canvas, div, img, px, rgb,
ParentElement, Styled, StyledImage, Window, canvas, div, img, px, rgb, surface,
};
use super::{ElyShell, web_surface_frame::WebSurfaceFrame};
@@ -12,22 +12,15 @@ pub(super) fn render_ready_web_surface(
tab: &BrowserTab,
state_entity: Entity<ElyShell>,
) -> AnyElement {
// T14: the `gpui::surface(...)` hardware path is held.
//
// GPUI 0.2.2's Blade Metal renderer hard-asserts that any
// CVPixelBuffer handed to `surface(...)` is NV12 YUV
// (kCVPixelFormatType_420YpCbCr8BiPlanarFullRange). See
// ~/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/
// gpui-0.2.2/src/platform/blade/blade_renderer.rs:832
// for the assert. Our sidecar produces BGRA IOSurfaces, so
// calling `surface()` with that buffer panics the renderer on
// the first frame.
//
// We keep `services::iosurface_metal` and
// `WebSurfaceFrame::pixel_buffer` intact so the wire-side
// IOSurfaceHandle import path stays exercised; once GPUI gains a
// BGRA-capable Surface element this branch can come back. Until
// then every frame must go through `img()` below.
#[cfg(target_os = "macos")]
if let Some(pixel_buffer) = frame.pixel_buffer.as_ref() {
return render_web_surface(
tab,
state_entity,
surface(pixel_buffer.clone()).size_full().object_fit(ObjectFit::Fill),
);
}
if let Some(image) = frame.image.as_ref() {
return render_web_surface(
tab,