Hold hardware live mode until BGRA presentation

This commit is contained in:
2026-05-12 23:42:52 -04:00
parent 316bf6f9a4
commit bf1ebfb6fe
5 changed files with 136 additions and 70 deletions
+40 -43
View File
@@ -57,15 +57,14 @@ pub(super) struct WebSurfaceFrame {
content_pixel_count: u64,
#[cfg(all(test, feature = "live-site-smoke"))]
sample_hash: u64,
/// Software-path image. `None` whenever the sidecar took the
/// hardware shortcut and dropped the RGBA payload from the
/// wire — the IOSurface in `pixel_buffer` is the source of truth
/// for that frame.
/// Software-path image. Current GPUI builds require this for every
/// ready web frame because BGRA IOSurface presentation is still
/// held at the protocol boundary.
pub(super) image: Option<Arc<RenderImage>>,
/// Hardware-path companion: when present, the view samples the
/// IOSurface through GPUI's Metal pipeline via `gpui::surface(...)`
/// instead of uploading the RGBA bytes again. Always `None` on
/// the software path; `image` is the source of truth there.
/// Hardware-path companion imported from the sidecar. GPUI 0.2.2's
/// public `surface(...)` presenter accepts NV12 video buffers, and
/// Servo publishes BGRA IOSurfaces; this remains observability
/// state until a BGRA presenter is available.
#[cfg(target_os = "macos")]
pub(super) pixel_buffer: Option<CVPixelBuffer>,
}
@@ -103,27 +102,23 @@ impl WebSurfaceFrame {
}
fn from_parts(parts: WebSurfaceFrameParts) -> Result<Self, WebSurfaceError> {
// Hardware path frames arrive with `rgba_bytes` empty — the
// sidecar dropped the 8 MB payload from the wire and the
// receiver samples the IOSurface directly. In that case the
// RGBA hash + LAST_FRAME_IMAGE dedup are skipped entirely.
let image = if parts.rgba_bytes.is_empty() {
None
} else {
// Servo's `read_pixels(gl::RGBA, gl::UNSIGNED_BYTE)`
// writes R-G-B-A in memory order. GPUI's `RenderImage` is
// documented as "in BGRA format" and uploads via
// `MTLPixelFormat::BGRA8Unorm`, which reads B-G-R-A. Hand
// the bytes across unchanged and the Metal sampler treats
// R as B (and vice versa) — every coloured pixel renders
// with R and B swapped. Swap once here so the rest of the
// pipeline (dedup hash, image buffer, GPU upload) all
// operate on the same BGRA representation.
let mut bytes = parts.rgba_bytes;
swap_red_blue_in_place(&mut bytes);
let bytes_hash = rgba_hash(&bytes);
Some(resolve_render_image(parts.width, parts.height, bytes, bytes_hash)?)
};
if parts.rgba_bytes.is_empty() {
return Err(WebSurfaceError::MissingRenderablePayload);
}
// Servo's `read_pixels(gl::RGBA, gl::UNSIGNED_BYTE)` writes
// R-G-B-A in memory order. GPUI's `RenderImage` is documented
// as "in BGRA format" and uploads via
// `MTLPixelFormat::BGRA8Unorm`, which reads B-G-R-A. Hand the bytes across
// unchanged and the Metal sampler treats R as B (and vice
// versa) — every coloured pixel renders with R and B swapped.
// Swap once here so the rest of the pipeline (dedup hash,
// image buffer, GPU upload) all operate on the same BGRA
// representation.
let mut bytes = parts.rgba_bytes;
swap_red_blue_in_place(&mut bytes);
let bytes_hash = rgba_hash(&bytes);
let image = Some(resolve_render_image(parts.width, parts.height, bytes, bytes_hash)?);
Ok(Self {
requested_url: parts.requested_url,
@@ -245,6 +240,10 @@ struct WebSurfaceFrameParts {
pub(super) enum WebSurfaceError {
#[error("invalid servo frame buffer for {width}x{height}")]
InvalidFrameBuffer { width: u32, height: u32 },
#[error(
"servo live frame did not include renderable pixels; BGRA IOSurface presentation is unavailable in GPUI 0.2.2"
)]
MissingRenderablePayload,
}
/// Swap byte 0 and byte 2 of every 4-byte pixel, converting Servo's
@@ -268,20 +267,18 @@ fn resolve_render_image(
rgba_bytes: Vec<u8>,
bytes_hash: u64,
) -> Result<Arc<RenderImage>, WebSurfaceError> {
LAST_FRAME_IMAGE.with(
|cache| -> Result<Arc<RenderImage>, WebSurfaceError> {
let mut cache = cache.borrow_mut();
if let Some((cached_hash, cached_image)) = cache.as_ref() {
if *cached_hash == bytes_hash {
return Ok(cached_image.clone());
}
LAST_FRAME_IMAGE.with(|cache| -> Result<Arc<RenderImage>, WebSurfaceError> {
let mut cache = cache.borrow_mut();
if let Some((cached_hash, cached_image)) = cache.as_ref() {
if *cached_hash == bytes_hash {
return Ok(cached_image.clone());
}
}
let image_buffer = ImageBuffer::<Rgba<u8>, _>::from_raw(width, height, rgba_bytes)
.ok_or(WebSurfaceError::InvalidFrameBuffer { width, height })?;
let new_image = Arc::new(RenderImage::new([image::Frame::new(image_buffer)]));
*cache = Some((bytes_hash, new_image.clone()));
Ok(new_image)
},
)
let image_buffer = ImageBuffer::<Rgba<u8>, _>::from_raw(width, height, rgba_bytes)
.ok_or(WebSurfaceError::InvalidFrameBuffer { width, height })?;
let new_image = Arc::new(RenderImage::new([image::Frame::new(image_buffer)]));
*cache = Some((bytes_hash, new_image.clone()));
Ok(new_image)
})
}
@@ -383,6 +383,28 @@ fn live_frame_swaps_red_and_blue_bytes_for_gpui_bgra() -> Result<(), Box<dyn Err
Ok(())
}
#[test]
fn empty_live_frame_payload_is_rejected() {
use crate::services::servo_live::ServoLiveFrame;
use crate::shell::web_surface_frame::WebSurfaceFrame;
use crate::shell::web_surface_geometry::WebSurfaceScrollOffset;
let result = WebSurfaceFrame::from_live_frame(
"https://example.com/".to_string(),
WebSurfaceScrollOffset::default(),
100,
ServoLiveFrame::for_test(1, 1, Vec::new()),
);
let Err(error) = result else {
panic!("empty Servo frame payload must be rejected before it reaches Ready state");
};
assert_eq!(
error.to_string(),
"servo live frame did not include renderable pixels; BGRA IOSurface presentation is unavailable in GPUI 0.2.2",
);
}
fn web_bounds() -> Bounds<gpui::Pixels> {
Bounds::new(point(px(0.0), px(0.0)), size(px(640.0), px(480.0)))
}
+6 -5
View File
@@ -12,7 +12,7 @@ pub(super) fn render_ready_web_surface(
tab: &BrowserTab,
state_entity: Entity<ElyShell>,
) -> AnyElement {
// T14: the `gpui::surface(...)` hardware path is disabled.
// T14: the `gpui::surface(...)` hardware path is held.
//
// GPUI 0.2.2's Blade Metal renderer hard-asserts that any
// CVPixelBuffer handed to `surface(...)` is NV12 YUV
@@ -35,10 +35,11 @@ pub(super) fn render_ready_web_surface(
img(ImageSource::Render(image.clone())).size_full().object_fit(ObjectFit::Fill),
);
}
// Both image variants empty: the sidecar should always publish
// RGBA while the hardware path is disabled, but a blank canvas is
// the honest user-facing fallback if it ever does not.
render_web_surface(tab, state_entity, div().size_full())
render_web_surface(
tab,
state_entity,
error_page("Web surface frame did not include renderable pixels."),
)
}
pub(super) fn render_loading_web_surface(