ZacharyZhang-NY/ELY-Browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

T10.5: render IOSurface via gpui::surface(CVPixelBuffer)

Browse Source
This commit is contained in:
ZacharyZhang-NY
2026-05-10 23:37:00 -04:00
parent c4a6ea3c57
commit a447d52262
6 changed files with 177 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ely_app/src/services/iosurface_metal.rs
+91 -126
View File
@@ -1,65 +1,58 @@
//! macOS-only import of cross-process IOSurface handles into Metal
//! textures.
//! macOS-only import of cross-process IOSurface handles into
//! `CVPixelBuffer`s suitable for GPUI's `Surface` element.
//!
//! `T10.4`: the sidecar publishes an [`crate::services::servo_live`]
//! `surface_handle` carrying a `mach_port_name` + stable `surface_id`.
//! The receiver builds an `MTLTexture` from that IOSurface exactly
//! once per `surface_id` and caches it. Every frame after the first
//! reads `current_surface_id` and samples the cached texture — zero
//! pixel copy, zero re-import.
//! `T10.4` originally imported the IOSurface into an `MTLTexture`
//! directly, but GPUI 0.2.2 already speaks `CVPixelBuffer` end-to-end
//! through `Window::paint_surface` / `elements::surface::Surface`. Its
//! internal Blade Metal renderer takes care of building the Metal
//! texture, so a parallel MTLTexture cache here would be wasted work.
//! The cache now hands the renderer the CVPixelBuffer GPUI already
//! knows how to render.
//!
//! Lifetime contract:
//!
//! * `IOSurfaceCreateMachPort` (sidecar side) gives the receiver a
//! send right whose refcount is 1 in our task. Once we've called
//! `IOSurfaceLookupFromMachPort` to materialise the
//! `IOSurfaceRef`, the mach port has done its job.
//! * Metal's `newTextureWithDescriptor:iosurface:plane:` retains the
//! IOSurface for the texture's lifetime. We `mach_port_deallocate`
//! immediately afterwards so the receiver process doesn't
//! accumulate idle mach send rights.
//! * Dropping `MetalSurfaceImporter` releases every cached
//! `MTLTexture`, which in turn releases each retained IOSurface.
//! The sidecar still holds its own retain via surfman, so the
//! IOSurface itself outlives our cache for as long as the sidecar
//! keeps painting.
//! send right whose refcount is 1 in our task. After we resolve
//! the surface and wrap it in a CVPixelBuffer, the mach port has
//! done its job.
//! * `CVPixelBufferCreateWithIOSurface` retains the IOSurface for
//! the pixel buffer's lifetime. We `mach_port_deallocate`
//! immediately so the receiver process doesn't accumulate idle
//! mach send rights.
//! * Dropping `IOSurfaceCache` releases every cached
//! `CVPixelBuffer`, which in turn releases each retained
//! IOSurface. The sidecar still holds its own retain via surfman,
//! so the IOSurface itself outlives our cache for as long as the
//! sidecar keeps painting.
#![cfg(target_os = "macos")]
use std::collections::HashMap;
use objc2::rc::Retained;
use objc2::runtime::ProtocolObject;
use objc2_foundation::NSUInteger;
use objc2_io_surface::IOSurfaceRef;
use objc2_metal::{
MTLCreateSystemDefaultDevice, MTLDevice, MTLPixelFormat, MTLTexture, MTLTextureDescriptor,
MTLTextureUsage,
};
use core_foundation::base::TCFType as _;
use core_video::pixel_buffer::CVPixelBuffer;
#[allow(deprecated)]
use io_surface::IOSurface;
use thiserror::Error;
/// Owner of the system Metal device + cache of imported textures
/// keyed by IOSurface identity. Constructed once per renderer process
/// when the first hardware-path tab requests an upload.
pub(crate) struct MetalSurfaceImporter {
device: Retained<ProtocolObject<dyn MTLDevice>>,
textures: HashMap<u64, Retained<ProtocolObject<dyn MTLTexture>>>,
/// Cache of imported `CVPixelBuffer`s keyed by IOSurface identity.
/// Constructed lazily by the renderer-side client on the first
/// hardware-path frame.
pub(crate) struct IOSurfaceCache {
pixel_buffers: HashMap<u64, CVPixelBuffer>,
}
#[derive(Debug, Error)]
pub(crate) enum SurfaceImportError {
#[error("system has no default Metal device — hardware path unavailable")]
NoMetalDevice,
#[error("IOSurfaceLookupFromMachPort returned null for port 0x{port:x}")]
LookupFailed { port: u32 },
#[error("MTLDevice rejected the IOSurface (size {width}x{height})")]
TextureBuildFailed { width: u32, height: u32 },
#[error("CVPixelBufferCreateWithIOSurface returned status {status}")]
PixelBufferBuildFailed { status: i32 },
}
impl MetalSurfaceImporter {
pub fn new() -> Result<Self, SurfaceImportError> {
let device = MTLCreateSystemDefaultDevice().ok_or(SurfaceImportError::NoMetalDevice)?;
Ok(Self { device, textures: HashMap::new() })
impl IOSurfaceCache {
pub fn new() -> Self {
Self { pixel_buffers: HashMap::new() }
}
/// Import an IOSurface published by the sidecar's
@@ -72,71 +65,59 @@ impl MetalSurfaceImporter {
&mut self,
mach_port_name: u32,
surface_id: u64,
width: u32,
height: u32,
) -> Result<(), SurfaceImportError> {
if self.textures.contains_key(&surface_id) {
if self.pixel_buffers.contains_key(&surface_id) {
deallocate_mach_port(mach_port_name);
return Ok(());
}
let Some(iosurface) = IOSurfaceRef::lookup_from_mach_port(mach_port_name) else {
// Lookup failed → port is invalid; nothing to deallocate.
let Some(iosurface) = objc2_io_surface::IOSurfaceRef::lookup_from_mach_port(mach_port_name)
else {
return Err(SurfaceImportError::LookupFailed { port: mach_port_name });
};
let descriptor = MTLTextureDescriptor::new();
// surfman's macOS surface backs IOSurface with
// kCVPixelFormatType_32BGRA — match it so Metal samples the
// correct channel order. `setUsage(ShaderRead)` is the minimum
// Metal needs to expose the texture to a fragment shader sampler.
// The setters are unsafe because they cross the FFI boundary
// without descriptor validation; we know our values are sound.
#[expect(unsafe_code)]
unsafe {
descriptor.setPixelFormat(MTLPixelFormat::BGRA8Unorm);
descriptor.setWidth(width as NSUInteger);
descriptor.setHeight(height as NSUInteger);
descriptor.setUsage(MTLTextureUsage::ShaderRead);
}
// Both objc2-io-surface and the legacy `io_surface` crate wrap
// the same C `__IOSurface` pointer. CVPixelBufferCreateWithIOSurface
// (via core-video) expects the legacy crate's wrapper. Reach for
// the raw pointer and let TCFType CFRetain it independently so
// both Rust handles can drop without double-freeing.
let raw_ptr: *const std::ffi::c_void =
(&*iosurface) as *const objc2_io_surface::IOSurfaceRef as *const std::ffi::c_void;
#[allow(deprecated)]
let io_surface_view: IOSurface = {
#[expect(unsafe_code)]
unsafe {
IOSurface::wrap_under_get_rule(raw_ptr as io_surface::IOSurfaceRef)
}
};
let texture = self
.device
.newTextureWithDescriptor_iosurface_plane(&descriptor, &iosurface, 0)
.ok_or(SurfaceImportError::TextureBuildFailed { width, height })?;
let pixel_buffer = CVPixelBuffer::from_io_surface(&io_surface_view, None)
.map_err(|status| SurfaceImportError::PixelBufferBuildFailed { status })?;
self.textures.insert(surface_id, texture);
self.pixel_buffers.insert(surface_id, pixel_buffer);
deallocate_mach_port(mach_port_name);
Ok(())
}
/// Look up an already-imported texture by `surface_id`. The
/// receiver's per-frame `current_surface_id` field selects which
/// of the chain's rotating front/back surfaces to sample.
/// `#[allow(dead_code)]` until T10.5 wires the renderer; the
/// import side is exercised by the live perf bench right now.
#[allow(dead_code)]
pub fn texture_for(
&self,
surface_id: u64,
) -> Option<&Retained<ProtocolObject<dyn MTLTexture>>> {
self.textures.get(&surface_id)
/// Look up an already-imported pixel buffer by `surface_id`. The
/// receiver's per-frame `current_surface_id` selects which of the
/// swap chain's rotating front/back surfaces to sample. Returns a
/// clone (CVPixelBuffer is reference-counted; cloning is a cheap
/// atomic increment) so the caller can hand it to GPUI's
/// `surface(...)` element without holding a borrow on the cache.
pub fn pixel_buffer_for(&self, surface_id: u64) -> Option<CVPixelBuffer> {
self.pixel_buffers.get(&surface_id).cloned()
}
/// Reports how many distinct surfaces have been imported. Used
/// by tests and the live perf bench to assert that dedup is
/// keeping the cache small (one per swap-chain surface).
#[cfg(test)]
pub fn cached_surface_count(&self) -> usize {
self.textures.len()
self.pixel_buffers.len()
}
}
/// Release one send right against the mach port we received. The
/// IOSurface itself stays alive because the `MTLTexture` (or the
/// sidecar's surfman) still retain it. A `KERN_INVALID_NAME` failure
/// here means the port already drained — survivable, log and move
/// on.
/// IOSurface itself stays alive because the `CVPixelBuffer` (or the
/// sidecar's surfman) still retain it.
fn deallocate_mach_port(port: u32) {
#[expect(unsafe_code)]
let result = unsafe { mach_port_deallocate(mach_task_self_, port) };
@@ -160,13 +141,13 @@ unsafe extern "C" {
/// Releases one send right against `name` within `task`. We only
/// ever call this with our own task; the IOSurface keeps its
/// retain via the MTLTexture so this just frees our port slot.
/// retain via the CVPixelBuffer so this just frees our port slot.
fn mach_port_deallocate(task: u32, name: u32) -> i32;
}
#[cfg(test)]
mod tests {
use super::{MetalSurfaceImporter, SurfaceImportError};
use super::IOSurfaceCache;
use objc2_core_foundation::{
CFDictionary, CFIndex, CFNumber, CFRetained, CFString, kCFAllocatorDefault,
kCFTypeDictionaryKeyCallBacks, kCFTypeDictionaryValueCallBacks,
@@ -175,7 +156,6 @@ mod tests {
IOSurfaceRef, kIOSurfaceBytesPerElement, kIOSurfaceBytesPerRow, kIOSurfaceHeight,
kIOSurfacePixelFormat, kIOSurfaceWidth,
};
use objc2_metal::MTLTexture as _;
use std::os::raw::c_void;
const TEST_WIDTH: u32 = 64;
@@ -185,10 +165,7 @@ mod tests {
/// surfman's macOS backend does — BGRA8 (four-cc '32BGRA'), width
/// + height + bytes_per_element + bytes_per_row in a Core
/// Foundation properties dictionary. The pointer-casts mirror
/// `surfman::platform::macos::system::surface::create_io_surface`;
/// the dict has CFString keys and CFNumber values and is built
/// with `kCFTypeDictionaryKeyCallBacks` / `kCFTypeDictionaryValueCallBacks`
/// so CF retains its entries.
/// `surfman::platform::macos::system::surface::create_io_surface`.
fn build_local_iosurface() -> CFRetained<IOSurfaceRef> {
let pixel_format: i32 = i32::from_be_bytes(*b"BGRA");
let bytes_per_element: i32 = 4;
@@ -227,55 +204,43 @@ mod tests {
}
#[test]
fn imports_local_iosurface_into_mtl_texture() {
let mut importer = match MetalSurfaceImporter::new() {
Ok(importer) => importer,
Err(SurfaceImportError::NoMetalDevice) => {
eprintln!(
"no Metal device on this host — \
acceptable in headless / no-GPU CI; skipping"
);
return;
}
Err(error) => panic!("unexpected importer error: {error:?}"),
};
fn imports_local_iosurface_into_pixel_buffer() {
let mut cache = IOSurfaceCache::new();
let iosurface = build_local_iosurface();
let mach_port = iosurface.create_mach_port();
assert!(mach_port != 0, "IOSurfaceCreateMachPort must yield a real port");
let surface_id: u64 = 0xDEAD_BEEFu64;
importer
.import(mach_port, surface_id, TEST_WIDTH, TEST_HEIGHT)
.expect("local IOSurface must round-trip through MTLDevice");
cache.import(mach_port, surface_id).expect("local IOSurface must round-trip into a CVPixelBuffer");
let texture = importer
.texture_for(surface_id)
.expect("imported texture must be retrievable by surface_id");
assert_eq!(texture.width(), TEST_WIDTH as usize, "MTLTexture width must match");
assert_eq!(texture.height(), TEST_HEIGHT as usize, "MTLTexture height must match");
assert_eq!(importer.cached_surface_count(), 1);
let pixel_buffer = cache
.pixel_buffer_for(surface_id)
.expect("imported pixel buffer must be retrievable by surface_id");
assert_eq!(
pixel_buffer.get_width() as u32,
TEST_WIDTH,
"CVPixelBuffer width must match the source IOSurface",
);
assert_eq!(
pixel_buffer.get_height() as u32,
TEST_HEIGHT,
"CVPixelBuffer height must match the source IOSurface",
);
assert_eq!(cache.cached_surface_count(), 1);
}
#[test]
fn second_import_with_same_surface_id_is_idempotent() {
let mut importer = match MetalSurfaceImporter::new() {
Ok(importer) => importer,
Err(SurfaceImportError::NoMetalDevice) => return,
Err(error) => panic!("unexpected importer error: {error:?}"),
};
let mut cache = IOSurfaceCache::new();
let iosurface = build_local_iosurface();
let port_a = iosurface.create_mach_port();
let port_b = iosurface.create_mach_port();
assert!(port_a != 0 && port_b != 0 && port_a != port_b);
importer.import(port_a, 0xAAAA_AAAA, TEST_WIDTH, TEST_HEIGHT).expect("first import");
cache.import(port_a, 0xAAAA_AAAA).expect("first import");
// Same surface_id → defensive dedup path; port_b is deallocated
// without minting a duplicate MTLTexture.
importer
.import(port_b, 0xAAAA_AAAA, TEST_WIDTH, TEST_HEIGHT)
.expect("duplicate import is idempotent");
assert_eq!(importer.cached_surface_count(), 1);
// without minting a duplicate CVPixelBuffer.
cache.import(port_b, 0xAAAA_AAAA).expect("duplicate import is idempotent");
assert_eq!(cache.cached_surface_count(), 1);
}
}
crates/ely_app/src/services/servo_live.rs
+43 -39
View File
@@ -23,18 +23,19 @@ use thiserror::Error;
use super::servo_sidecar_command::{SidecarCommandError, default_sidecar_command};
#[cfg(target_os = "macos")]
use super::iosurface_metal::MetalSurfaceImporter;
use super::iosurface_metal::IOSurfaceCache;
#[cfg(target_os = "macos")]
use core_video::pixel_buffer::CVPixelBuffer;
pub(crate) struct ServoLiveClient {
child: Child,
stdin: ChildStdin,
stdout: BufReader<ChildStdout>,
/// Cache of imported Metal textures keyed by surface_id. Built
/// Cache of imported `CVPixelBuffer`s keyed by surface_id. Built
/// lazily on the first `surface_handle` the sidecar publishes —
/// software-path tabs never trigger construction, so machines
/// without a Metal device aren't penalised.
/// software-path tabs never trigger construction.
#[cfg(target_os = "macos")]
metal_importer: Option<MetalSurfaceImporter>,
iosurface_cache: IOSurfaceCache,
}
impl ServoLiveClient {
@@ -64,10 +65,11 @@ impl ServoLiveClient {
stdin,
stdout: BufReader::new(stdout),
#[cfg(target_os = "macos")]
metal_importer: None,
iosurface_cache: IOSurfaceCache::new(),
})
}
pub fn ensure(
&mut self,
request: ServoLiveEnsureRequest,
@@ -154,7 +156,14 @@ impl ServoLiveClient {
.read_exact(&mut rgba_bytes)
.map_err(ServoLiveError::FrameRead)?;
Ok(Some(ServoLiveFrame::from_parts(report, rgba_bytes)))
let mut frame = ServoLiveFrame::from_parts(report, rgba_bytes);
#[cfg(target_os = "macos")]
if let Some(surface_id) = response.current_surface_id {
frame.pixel_buffer = self.iosurface_cache.pixel_buffer_for(surface_id);
}
Ok(Some(frame))
}
}
@@ -167,49 +176,25 @@ impl Drop for ServoLiveClient {
#[cfg(target_os = "macos")]
impl ServoLiveClient {
/// Run a freshly-arrived `surface_handle` through the Metal
/// importer. Lazily constructs the importer on first call so
/// software-only sessions never touch the GPU. Failures are
/// logged but don't error the request — T10.5 will fall back to
/// the existing software RGBA path if `texture_for` returns
/// `None`, so the user still sees a frame.
/// Convert the sidecar's `surface_handle` into a `CVPixelBuffer`
/// in the local cache. Failures are logged but don't error the
/// request — the renderer falls back to the existing software
/// `Arc<RenderImage>` path when no pixel buffer is available, so
/// the user always sees a frame.
fn import_iosurface_handle(&mut self, handle: &LiveSurfaceHandle) {
if self.metal_importer.is_none() {
match MetalSurfaceImporter::new() {
Ok(importer) => {
self.metal_importer = Some(importer);
}
Err(error) => {
tracing::warn!(
target: "ely::servo::iosurface",
error = %error,
"no Metal device for IOSurface import; staying on software path",
);
return;
}
}
}
let Some(importer) = self.metal_importer.as_mut() else {
return;
};
match importer.import(
handle.mach_port_name,
handle.surface_id,
handle.width,
handle.height,
) {
match self.iosurface_cache.import(handle.mach_port_name, handle.surface_id) {
Ok(()) => tracing::info!(
target: "ely::servo::iosurface",
surface_id = handle.surface_id,
width = handle.width,
height = handle.height,
"imported IOSurface into Metal texture cache",
"imported IOSurface into CVPixelBuffer cache",
),
Err(error) => tracing::warn!(
target: "ely::servo::iosurface",
error = %error,
surface_id = handle.surface_id,
"IOSurface→MTLTexture import failed; subsequent samples will miss",
"IOSurface→CVPixelBuffer import failed; subsequent samples will miss",
),
}
}
@@ -262,6 +247,12 @@ pub(crate) struct ServoLiveFrame {
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: u64,
rgba_bytes: Vec<u8>,
/// Hardware-path companion: when present, the renderer can hand
/// the underlying IOSurface straight to GPUI's Metal pipeline via
/// `gpui::surface(...)` and skip the RGBA upload entirely. Always
/// `None` on the software path and on non-macOS hosts.
#[cfg(target_os = "macos")]
pixel_buffer: Option<CVPixelBuffer>,
}
impl ServoLiveFrame {
@@ -279,9 +270,20 @@ impl ServoLiveFrame {
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: report.sample_hash,
rgba_bytes,
#[cfg(target_os = "macos")]
pixel_buffer: None,
}
}
/// Returns the imported `CVPixelBuffer` matching the frame's
/// current hardware surface, if any. The renderer hands this to
/// `gpui::surface(...)` to skip the RGBA→texture upload path.
#[cfg(target_os = "macos")]
#[must_use]
pub fn pixel_buffer(&self) -> Option<&CVPixelBuffer> {
self.pixel_buffer.as_ref()
}
#[must_use]
pub fn loaded_url(&self) -> Option<&str> {
self.loaded_url.as_deref()
@@ -345,6 +347,8 @@ impl ServoLiveFrame {
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: 0,
rgba_bytes,
#[cfg(target_os = "macos")]
pixel_buffer: None,
}
}
}