ZacharyZhang-NY/ELY-Browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Present Servo BGRA hardware surfaces

Browse Source
This commit is contained in:
ZacharyZhang-NY
2026-05-13 00:22:00 -04:00
parent bf1ebfb6fe
commit 05a7a0d67f
154 changed files with 84157 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ely_app/src/shell/web_surface.rs
+4 -4
View File
@@ -177,12 +177,12 @@ impl WebSurfaceStore {
position: Point<Pixels>,
scale_factor: f32,
) -> WebSurfaceInputOutcome {
let surface =
self.surfaces.get_mut(tab_id).filter(|surface| surface.viewport_bounds.is_some());
let Some(surface) = surface else {
let Some(surface) = self.surfaces.get_mut(tab_id) else {
return WebSurfaceInputOutcome::DroppedNoViewportBounds;
};
let Some(bounds) = surface.viewport_bounds else {
return WebSurfaceInputOutcome::DroppedNoViewportBounds;
};
let bounds = surface.viewport_bounds.expect("viewport_bounds checked above");
let Some(point) =
WebSurfaceClickPoint::from_window_position(bounds, position, scale_factor)
else {
crates/ely_app/src/shell/web_surface_frame.rs
+33 -28
View File
@@ -57,14 +57,12 @@ pub(super) struct WebSurfaceFrame {
content_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: u64,
/// Software-path image. Current GPUI builds require this for every
/// ready web frame because BGRA IOSurface presentation is still
/// held at the protocol boundary.
/// Software-path image built from RGBA bytes when the sidecar runs
/// without hardware surface publication.
pub(super) image: Option<Arc<RenderImage>>,
/// Hardware-path companion imported from the sidecar. GPUI 0.2.2's
/// public `surface(...)` presenter accepts NV12 video buffers, and
/// Servo publishes BGRA IOSurfaces; this remains observability
/// state until a BGRA presenter is available.
/// Hardware-path surface imported from the sidecar's IOSurface.
/// GPUI is patched locally to present BGRA CVPixelBuffers through
/// `surface(...)`, so hardware frames can skip the RGBA pipe.
#[cfg(target_os = "macos")]
pub(super) pixel_buffer: Option<CVPixelBuffer>,
}
@@ -102,23 +100,32 @@ impl WebSurfaceFrame {
}
fn from_parts(parts: WebSurfaceFrameParts) -> Result<Self, WebSurfaceError> {
if parts.rgba_bytes.is_empty() {
#[cfg(target_os = "macos")]
let has_pixel_buffer = parts.pixel_buffer.is_some();
#[cfg(not(target_os = "macos"))]
let has_pixel_buffer = false;
if parts.rgba_bytes.is_empty() && !has_pixel_buffer {
return Err(WebSurfaceError::MissingRenderablePayload);
}
// Servo's `read_pixels(gl::RGBA, gl::UNSIGNED_BYTE)` writes
// R-G-B-A in memory order. GPUI's `RenderImage` is documented
// as "in BGRA format" and uploads via
// `MTLPixelFormat::BGRA8Unorm`, which reads B-G-R-A. Hand the bytes across
// unchanged and the Metal sampler treats R as B (and vice
// versa) — every coloured pixel renders with R and B swapped.
// Swap once here so the rest of the pipeline (dedup hash,
// image buffer, GPU upload) all operate on the same BGRA
// representation.
let mut bytes = parts.rgba_bytes;
swap_red_blue_in_place(&mut bytes);
let bytes_hash = rgba_hash(&bytes);
let image = Some(resolve_render_image(parts.width, parts.height, bytes, bytes_hash)?);
let image = if parts.rgba_bytes.is_empty() {
None
} else {
// Servo's `read_pixels(gl::RGBA, gl::UNSIGNED_BYTE)` writes
// R-G-B-A in memory order. GPUI's `RenderImage` is documented
// as "in BGRA format" and uploads via
// `MTLPixelFormat::BGRA8Unorm`, which reads B-G-R-A. Hand the bytes across
// unchanged and the Metal sampler treats R as B (and vice
// versa) — every coloured pixel renders with R and B swapped.
// Swap once here so the rest of the pipeline (dedup hash,
// image buffer, GPU upload) all operate on the same BGRA
// representation.
let mut bytes = parts.rgba_bytes;
swap_red_blue_in_place(&mut bytes);
let bytes_hash = rgba_hash(&bytes);
Some(resolve_render_image(parts.width, parts.height, bytes, bytes_hash)?)
};
Ok(Self {
requested_url: parts.requested_url,
@@ -240,9 +247,7 @@ struct WebSurfaceFrameParts {
pub(super) enum WebSurfaceError {
#[error("invalid servo frame buffer for {width}x{height}")]
InvalidFrameBuffer { width: u32, height: u32 },
#[error(
"servo live frame did not include renderable pixels; BGRA IOSurface presentation is unavailable in GPUI 0.2.2"
)]
#[error("servo live frame did not include a software image or hardware IOSurface")]
MissingRenderablePayload,
}
@@ -269,10 +274,10 @@ fn resolve_render_image(
) -> Result<Arc<RenderImage>, WebSurfaceError> {
LAST_FRAME_IMAGE.with(|cache| -> Result<Arc<RenderImage>, WebSurfaceError> {
let mut cache = cache.borrow_mut();
if let Some((cached_hash, cached_image)) = cache.as_ref() {
if *cached_hash == bytes_hash {
return Ok(cached_image.clone());
}
if let Some((cached_hash, cached_image)) = cache.as_ref()
&& *cached_hash == bytes_hash
{
return Ok(cached_image.clone());
}
let image_buffer = ImageBuffer::<Rgba<u8>, _>::from_raw(width, height, rgba_bytes)
crates/ely_app/src/shell/web_surface_tests.rs
+31 -4
View File
@@ -384,7 +384,7 @@ fn live_frame_swaps_red_and_blue_bytes_for_gpui_bgra() -> Result<(), Box<dyn Err
}
#[test]
fn empty_live_frame_payload_is_rejected() {
fn empty_live_frame_payload_is_rejected() -> Result<(), String> {
use crate::services::servo_live::ServoLiveFrame;
use crate::shell::web_surface_frame::WebSurfaceFrame;
use crate::shell::web_surface_geometry::WebSurfaceScrollOffset;
@@ -396,13 +396,40 @@ fn empty_live_frame_payload_is_rejected() {
ServoLiveFrame::for_test(1, 1, Vec::new()),
);
let Err(error) = result else {
panic!("empty Servo frame payload must be rejected before it reaches Ready state");
let error = match result {
Ok(_) => return Err("empty Servo frame payload reached Ready state".to_string()),
Err(error) => error,
};
assert_eq!(
error.to_string(),
"servo live frame did not include renderable pixels; BGRA IOSurface presentation is unavailable in GPUI 0.2.2",
"servo live frame did not include a software image or hardware IOSurface",
);
Ok(())
}
#[cfg(target_os = "macos")]
#[test]
fn hardware_live_frame_with_pixel_buffer_skips_software_image() -> Result<(), String> {
use core_video::pixel_buffer::{CVPixelBuffer, kCVPixelFormatType_32BGRA};
use crate::services::servo_live::ServoLiveFrame;
use crate::shell::web_surface_frame::WebSurfaceFrame;
use crate::shell::web_surface_geometry::WebSurfaceScrollOffset;
let pixel_buffer = CVPixelBuffer::new(kCVPixelFormatType_32BGRA, 1, 1, None)
.map_err(|status| format!("CVPixelBufferCreate returned status {status}"))?;
let live = ServoLiveFrame::for_test_with_pixel_buffer(1, 1, pixel_buffer);
let frame = WebSurfaceFrame::from_live_frame(
"https://example.com/".to_string(),
WebSurfaceScrollOffset::default(),
100,
live,
)
.map_err(|error| error.to_string())?;
assert!(frame.image.is_none(), "hardware frame should use the CVPixelBuffer surface path");
assert!(frame.pixel_buffer.is_some(), "hardware frame should carry the imported CVPixelBuffer");
Ok(())
}
fn web_bounds() -> Bounds<gpui::Pixels> {
crates/ely_app/src/shell/web_surface_view.rs
+10 -17
View File
@@ -1,7 +1,7 @@
use ely_domain::{BrowserTab, TabId};
use gpui::{
AnyElement, App, Entity, ImageSource, InteractiveElement, IntoElement, MouseButton, ObjectFit,
ParentElement, Styled, StyledImage, Window, canvas, div, img, px, rgb,
ParentElement, Styled, StyledImage, Window, canvas, div, img, px, rgb, surface,
};
use super::{ElyShell, web_surface_frame::WebSurfaceFrame};
@@ -12,22 +12,15 @@ pub(super) fn render_ready_web_surface(
tab: &BrowserTab,
state_entity: Entity<ElyShell>,
) -> AnyElement {
// T14: the `gpui::surface(...)` hardware path is held.
//
// GPUI 0.2.2's Blade Metal renderer hard-asserts that any
// CVPixelBuffer handed to `surface(...)` is NV12 YUV
// (kCVPixelFormatType_420YpCbCr8BiPlanarFullRange). See
// ~/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/
// gpui-0.2.2/src/platform/blade/blade_renderer.rs:832
// for the assert. Our sidecar produces BGRA IOSurfaces, so
// calling `surface()` with that buffer panics the renderer on
// the first frame.
//
// We keep `services::iosurface_metal` and
// `WebSurfaceFrame::pixel_buffer` intact so the wire-side
// IOSurfaceHandle import path stays exercised; once GPUI gains a
// BGRA-capable Surface element this branch can come back. Until
// then every frame must go through `img()` below.
#[cfg(target_os = "macos")]
if let Some(pixel_buffer) = frame.pixel_buffer.as_ref() {
return render_web_surface(
tab,
state_entity,
surface(pixel_buffer.clone()).size_full().object_fit(ObjectFit::Fill),
);
}
if let Some(image) = frame.image.as_ref() {
return render_web_surface(
tab,