Verify PRD sites through app sidecar

2026-05-08 19:43:54 -04:00
parent e80e52bc7e
commit a5907efeb0
4 changed files with 247 additions and 44 deletions
@@ -13,6 +13,9 @@ use thiserror::Error;
 const SIDECAR_BINARY_TIMEOUT: Duration = Duration::from_secs(35);
 const SIDECAR_CARGO_TIMEOUT: Duration = Duration::from_secs(180);
 const SIDECAR_POLL_INTERVAL: Duration = Duration::from_millis(20);
+const SIDECAR_RETRY_INTERVAL: Duration = Duration::from_millis(250);
+const SIDECAR_NAVIGATION_ATTEMPTS: usize = 3;
+const SIDECAR_INTERACTION_ATTEMPTS: usize = 1;
 const SIDECAR_PATH_ENV: &str = "ELY_SERVO_SIDECAR";

 #[derive(Clone, Debug)]
@@ -33,8 +36,27 @@ impl ServoSidecarClient {
            return Err(ServoSidecarError::SidecarBinaryUnavailable { path: path.to_path_buf() });
        }

+        let mut attempt = 0;
+        loop {
+            match self.snapshot_once(&request) {
+                Ok(snapshot) => return Ok(snapshot),
+                Err(error) => {
+                    attempt += 1;
+                    if attempt >= request.max_attempts() {
+                        return Err(error);
+                    }
+                    thread::sleep(SIDECAR_RETRY_INTERVAL);
+                }
+            }
+        }
+    }
+
+    fn snapshot_once(
+        &self,
+        request: &SidecarSnapshotRequest,
+    ) -> Result<SidecarSnapshot, ServoSidecarError> {
        let rgba_path = temporary_rgba_path()?;
-        let output = match self.run_snapshot_command(&request, &rgba_path) {
+        let output = match self.run_snapshot_command(request, &rgba_path) {
            Ok(output) => output,
            Err(error) => {
                remove_temporary_file(&rgba_path)?;
@@ -204,6 +226,14 @@ impl SidecarSnapshotRequest {
    pub(crate) fn typed_text_for_test(&self) -> Option<&str> {
        self.typed_text.as_deref()
    }
+
+    fn max_attempts(&self) -> usize {
+        if self.click_point.is_some() || self.typed_text.is_some() {
+            return SIDECAR_INTERACTION_ATTEMPTS;
+        }
+
+        SIDECAR_NAVIGATION_ATTEMPTS
+    }
 }

 #[derive(Clone, Copy, Debug)]
@@ -437,44 +467,5 @@ fn expected_rgba_byte_count(width: u32, height: u32) -> Result<usize, ServoSidec
 }

 #[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn accepts_loading_report_with_visible_content() -> Result<(), ServoSidecarError> {
-        let snapshot = SidecarSnapshot::from_report(report_with_state("loading"), visible_frame())?;
-
-        assert_eq!(snapshot.loaded_url(), Some("https://example.com/"));
-        assert_eq!(snapshot.title(), Some("Example Domain"));
-        assert_eq!(snapshot.width(), 2);
-        assert_eq!(snapshot.height(), 1);
-        Ok(())
-    }
-
-    #[test]
-    fn rejects_created_report_with_visible_content() {
-        let result = SidecarSnapshot::from_report(report_with_state("created"), visible_frame());
-
-        assert!(
-            matches!(result, Err(ServoSidecarError::IncompleteRender { state }) if state == "created")
-        );
-    }
-
-    fn report_with_state(state: &str) -> SidecarReport {
-        SidecarReport {
-            requested_url: "https://example.com".to_string(),
-            loaded_url: Some("https://example.com/".to_string()),
-            title: Some("Example Domain".to_string()),
-            state: state.to_string(),
-            width: 2,
-            height: 1,
-            rgba_byte_count: 8,
-            non_white_pixel_count: 1,
-            content_pixel_count: 1,
-        }
-    }
-
-    fn visible_frame() -> Vec<u8> {
-        vec![0, 0, 0, 255, 255, 255, 255, 255]
-    }
-}
+#[path = "servo_sidecar_tests.rs"]
+mod tests;
@@ -0,0 +1,205 @@
+use std::error::Error;
+
+use super::*;
+
+#[cfg(feature = "live-site-smoke")]
+const LIVE_SITE_WIDTH: u32 = 934;
+#[cfg(feature = "live-site-smoke")]
+const LIVE_SITE_HEIGHT: u32 = 657;
+#[cfg(feature = "live-site-smoke")]
+const PRD_TOP_SITE_CASES: &[LiveSiteCase] = &[
+    LiveSiteCase { url: "https://github.com", title_fragment: "GitHub" },
+    LiveSiteCase { url: "https://example.com", title_fragment: "Example Domain" },
+    LiveSiteCase { url: "https://servo.org/", title_fragment: "Servo" },
+];
+#[cfg(feature = "live-site-smoke")]
+const PRD_REFERENCE_SITE_CASES: &[LiveSiteCase] = &[
+    LiveSiteCase {
+        url: "https://blog.google/products-and-platforms/products/chrome/new-chrome-productivity-features/",
+        title_fragment: "Chrome",
+    },
+    LiveSiteCase {
+        url: "https://www.microsoft.com/en-us/edge/features/vertical-tabs",
+        title_fragment: "Microsoft Edge",
+    },
+    LiveSiteCase {
+        url: "https://resources.arc.net/hc/en-us/articles/19230755904151-Favorites-Top-Tabs-Across-Every-Space",
+        title_fragment: "Favorites",
+    },
+    LiveSiteCase {
+        url: "https://resources.arc.net/hc/en-us/articles/19228855311127-Auto-Archive-Clean-as-you-go",
+        title_fragment: "Auto Archive",
+    },
+    LiveSiteCase { url: "https://vivaldi.com/features/workspaces/", title_fragment: "Workspaces" },
+    LiveSiteCase {
+        url: "https://help.vivaldi.com/desktop/tabs/tab-tiling/",
+        title_fragment: "Tab Tiling",
+    },
+    LiveSiteCase { url: "https://www.gpui.rs/", title_fragment: "gpui" },
+    LiveSiteCase { url: "https://docs.rs/gpui/latest/gpui/", title_fragment: "gpui" },
+    LiveSiteCase { url: "https://zed.dev/blog/videogame", title_fragment: "Leveraging Rust" },
+    LiveSiteCase {
+        url: "https://github.com/longbridge/gpui-component/",
+        title_fragment: "gpui-component",
+    },
+    LiveSiteCase {
+        url: "https://github.com/zed-industries/awesome-gpui/",
+        title_fragment: "awesome-gpui",
+    },
+    LiveSiteCase { url: "https://servo.org/", title_fragment: "Servo" },
+    LiveSiteCase {
+        url: "https://servo.org/blog/2026/04/13/servo-0.1.0-release/",
+        title_fragment: "Servo",
+    },
+    LiveSiteCase { url: "https://developers.cloudflare.com/d1/", title_fragment: "Cloudflare" },
+    LiveSiteCase {
+        url: "https://developers.cloudflare.com/workers/platform/storage-options/",
+        title_fragment: "Cloudflare",
+    },
+    LiveSiteCase {
+        url: "https://developers.cloudflare.com/kv/concepts/how-kv-works/",
+        title_fragment: "Cloudflare",
+    },
+    LiveSiteCase { url: "https://better-auth.com/blog/1-5", title_fragment: "Better Auth" },
+    LiveSiteCase {
+        url: "https://developers.cloudflare.com/d1/platform/limits/",
+        title_fragment: "Cloudflare",
+    },
+    LiveSiteCase {
+        url: "https://component-model.bytecodealliance.org/",
+        title_fragment: "WebAssembly Component Model",
+    },
+    LiveSiteCase {
+        url: "https://docs.wasmtime.dev/api/wasmtime/component/index.html",
+        title_fragment: "wasmtime",
+    },
+    LiveSiteCase { url: "https://docs.wasmtime.dev/security.html", title_fragment: "Wasmtime" },
+];
+
+#[cfg(feature = "live-site-smoke")]
+struct LiveSiteCase {
+    url: &'static str,
+    title_fragment: &'static str,
+}
+
+#[test]
+fn accepts_loading_report_with_visible_content() -> Result<(), ServoSidecarError> {
+    let snapshot = SidecarSnapshot::from_report(report_with_state("loading"), visible_frame())?;
+
+    assert_eq!(snapshot.loaded_url(), Some("https://example.com/"));
+    assert_eq!(snapshot.title(), Some("Example Domain"));
+    assert_eq!(snapshot.width(), 2);
+    assert_eq!(snapshot.height(), 1);
+    Ok(())
+}
+
+#[test]
+fn rejects_created_report_with_visible_content() {
+    let result = SidecarSnapshot::from_report(report_with_state("created"), visible_frame());
+
+    assert!(
+        matches!(result, Err(ServoSidecarError::IncompleteRender { state }) if state == "created")
+    );
+}
+
+#[test]
+fn retries_navigation_snapshots() -> Result<(), Box<dyn Error>> {
+    let request = SidecarSnapshotRequest::new(UrlText::parse("https://example.com")?, 2, 1);
+
+    assert_eq!(request.max_attempts(), SIDECAR_NAVIGATION_ATTEMPTS);
+    Ok(())
+}
+
+#[test]
+fn keeps_page_interactions_single_attempt() -> Result<(), Box<dyn Error>> {
+    let request = SidecarSnapshotRequest::new(UrlText::parse("https://example.com")?, 2, 1)
+        .with_click_point(1, 1)
+        .with_typed_text("ely".to_string());
+
+    assert_eq!(request.max_attempts(), SIDECAR_INTERACTION_ATTEMPTS);
+    Ok(())
+}
+
+#[cfg(feature = "live-site-smoke")]
+#[test]
+fn desktop_sidecar_opens_prd_top_sites() -> Result<(), Box<dyn Error>> {
+    assert_live_sites_render(PRD_TOP_SITE_CASES)
+}
+
+#[cfg(feature = "live-site-smoke")]
+#[test]
+fn desktop_sidecar_opens_prd_reference_sites() -> Result<(), Box<dyn Error>> {
+    assert_live_sites_render(PRD_REFERENCE_SITE_CASES)
+}
+
+#[cfg(feature = "live-site-smoke")]
+fn assert_live_sites_render(cases: &[LiveSiteCase]) -> Result<(), Box<dyn Error>> {
+    let client = ServoSidecarClient::new()?;
+    for case in cases {
+        let request = SidecarSnapshotRequest::new(
+            UrlText::parse(case.url)?,
+            LIVE_SITE_WIDTH,
+            LIVE_SITE_HEIGHT,
+        );
+        let snapshot = client.snapshot(request)?;
+
+        assert_eq!(snapshot.width(), LIVE_SITE_WIDTH, "{}", case.url);
+        assert_eq!(snapshot.height(), LIVE_SITE_HEIGHT, "{}", case.url);
+        assert_loaded_url_contains(&snapshot, case.url)?;
+        assert_title_contains(&snapshot, case.title_fragment)?;
+
+        let rgba_bytes = snapshot.into_rgba_bytes();
+        assert_eq!(
+            rgba_bytes.len(),
+            expected_rgba_byte_count(LIVE_SITE_WIDTH, LIVE_SITE_HEIGHT)?,
+            "{}",
+            case.url
+        );
+        assert!(non_white_pixel_count(&rgba_bytes) > 0, "{}", case.url);
+    }
+    Ok(())
+}
+
+#[cfg(feature = "live-site-smoke")]
+fn assert_loaded_url_contains(
+    snapshot: &SidecarSnapshot,
+    fragment: &str,
+) -> Result<(), Box<dyn Error>> {
+    let loaded_url =
+        snapshot.loaded_url().ok_or_else(|| format!("missing loaded URL for {fragment}"))?;
+    assert!(loaded_url.contains(fragment), "loaded_url: {loaded_url}");
+    Ok(())
+}
+
+#[cfg(feature = "live-site-smoke")]
+fn assert_title_contains(snapshot: &SidecarSnapshot, fragment: &str) -> Result<(), Box<dyn Error>> {
+    let title = snapshot.title().ok_or_else(|| format!("missing title containing {fragment}"))?;
+    assert!(title.contains(fragment), "title: {title}");
+    Ok(())
+}
+
+#[cfg(feature = "live-site-smoke")]
+fn non_white_pixel_count(rgba_bytes: &[u8]) -> usize {
+    rgba_bytes
+        .chunks_exact(4)
+        .filter(|pixel| pixel[3] > 0 && (pixel[0] != 255 || pixel[1] != 255 || pixel[2] != 255))
+        .count()
+}
+
+fn report_with_state(state: &str) -> SidecarReport {
+    SidecarReport {
+        requested_url: "https://example.com".to_string(),
+        loaded_url: Some("https://example.com/".to_string()),
+        title: Some("Example Domain".to_string()),
+        state: state.to_string(),
+        width: 2,
+        height: 1,
+        rgba_byte_count: 8,
+        non_white_pixel_count: 1,
+        content_pixel_count: 1,
+    }
+}
+
+fn visible_frame() -> Vec<u8> {
+    vec![0, 0, 0, 255, 255, 255, 255, 255]
+}