fix(browser): execute social actions through cdp

This commit is contained in:
ZacharyZhang-NY committed 2026-04-22 16:14:09 -04:00
1 parent fb6d2a0d77
commit f107415f4f
8 files changed
+777 -217

No files matched your search

+2 -1
View File
@@ -5,6 +5,7 @@ import { join, resolve, sep } from "node:path";
import log from "electron-log/main.js";
import type { BrowserAction, BrowserActionResult, QueuedAction } from "./browser/browser-types";
import { getPlatformRules, attachWindowSecurity, assertAllowedUrl, waitForLoad, runAction } from "./browser/browser-action-executor";
import { hasAuthenticatedCookie } from "./browser/browser-action-auth";
export type { BrowserAction, BrowserActionResult } from "./browser/browser-types";
@@ -242,7 +243,7 @@ export class BrowserManager {
}
}),
);
const hasCookies = cookieSets.some((cookies) => cookies.length > 0);
const hasCookies = hasAuthenticatedCookie(cookieSets, rules.authenticatedCookieNames);
if (!hasCookies) {
return false;
}
+49
View File
@@ -0,0 +1,49 @@
import type { Session } from "electron";
import { browserNotLoggedIn } from "./browser-action-errors";
import { getPlatformRules } from "./browser-url-helpers";
type CookieLike = {
name?: string;
};
export async function assertAuthenticatedSession(
session: Pick<Session, "cookies">,
platform: string,
actionType: string,
): Promise<void> {
const rules = getPlatformRules(platform);
const cookieSets = await Promise.all(
rules.cookieOrigins.map((origin) => session.cookies.get({ url: origin }).catch(() => [])),
);
if (!hasAuthenticatedCookie(cookieSets, rules.authenticatedCookieNames)) {
throw browserNotLoggedIn(platform, actionType);
}
}
export function hasAuthenticatedCookie(
cookieSets: readonly (readonly CookieLike[])[],
authenticatedCookieNames?: readonly string[],
): boolean {
if (!authenticatedCookieNames || authenticatedCookieNames.length === 0) {
return cookieSets.some((cookies) => cookies.length > 0);
}
const expectedNames = new Set(authenticatedCookieNames.map((name) => name.toLowerCase()));
return cookieSets.some((cookies) =>
cookies.some((cookie) => cookie.name && expectedNames.has(cookie.name.toLowerCase())),
);
}
export async function assertAuthenticatedPage(
getUrl: () => string,
platform: string,
actionType: string,
): Promise<void> {
const finalUrl = getUrl().toLowerCase();
const rules = getPlatformRules(platform);
const bouncedToLogin = rules.unauthenticatedUrlMarkers.some((marker) =>
finalUrl.includes(marker.toLowerCase()),
);
if (bouncedToLogin) {
throw browserNotLoggedIn(platform, actionType);
}
}
+251
View File
@@ -0,0 +1,251 @@
import type { WebContents } from "electron";
import { browserNotLoggedIn, browserSelectorDrift, browserVerificationFailed } from "./browser-action-errors";
import { isAppError } from "../errors/index.js";
type RuntimeEvaluateResult = {
result?: { value?: unknown };
exceptionDetails?: { text?: string };
};
type ElementProbe = {
selector: string;
x: number;
y: number;
text: string;
};
const POLL_INTERVAL_MS = 250;
export class BrowserCdpClient {
private attachedByClient = false;
private retryCount = 0;
constructor(
private readonly webContents: WebContents,
private readonly platform: string,
private readonly actionType: string,
) {}
async attach(): Promise<void> {
if (this.webContents.debugger.isAttached()) return;
this.webContents.debugger.attach("1.3");
this.attachedByClient = true;
}
async dispose(): Promise<void> {
if (this.attachedByClient && this.webContents.debugger.isAttached()) {
this.webContents.debugger.detach();
}
}
getRetryCount(): number {
return this.retryCount;
}
async evaluate<T>(expression: string): Promise<T> {
await this.attach();
const response = await this.webContents.debugger.sendCommand("Runtime.evaluate", {
expression,
awaitPromise: true,
returnByValue: true,
}) as RuntimeEvaluateResult;
if (response.exceptionDetails) {
throw browserVerificationFailed(
this.platform,
this.actionType,
response.exceptionDetails.text ?? "runtime evaluation failed",
);
}
return response.result?.value as T;
}
async clickFirst(candidates: readonly string[], selectorKey: string, timeoutMs = 10_000): Promise<string> {
const element = await this.waitForElement(candidates, selectorKey, timeoutMs);
await this.webContents.debugger.sendCommand("Input.dispatchMouseEvent", {
type: "mouseMoved",
x: element.x,
y: element.y,
button: "none",
});
await this.webContents.debugger.sendCommand("Input.dispatchMouseEvent", {
type: "mousePressed",
x: element.x,
y: element.y,
button: "left",
clickCount: 1,
});
await this.webContents.debugger.sendCommand("Input.dispatchMouseEvent", {
type: "mouseReleased",
x: element.x,
y: element.y,
button: "left",
clickCount: 1,
});
return element.selector;
}
async clickOptional(candidates: readonly string[], selectorKey: string, timeoutMs = 1_500): Promise<string | null> {
try {
return await this.clickFirst(candidates, selectorKey, timeoutMs);
} catch (error) {
if (!isAppError(error) || error.code !== "browser.selector_drift") throw error;
return null;
}
}
async waitForFirst(candidates: readonly string[], selectorKey: string, timeoutMs = 10_000): Promise<string> {
return (await this.waitForElement(candidates, selectorKey, timeoutMs)).selector;
}
async waitForAuthenticatedSurface(
candidates: readonly string[],
authenticatedCookieNames?: readonly string[],
timeoutMs = 5_000,
): Promise<string> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const selector = await this.evaluate<string | null>(
buildAuthenticatedSurfaceExpression(candidates, authenticatedCookieNames ?? []),
);
if (selector) return selector;
this.retryCount += 1;
await delay(POLL_INTERVAL_MS);
}
throw browserNotLoggedIn(this.platform, this.actionType);
}
async focusAndType(candidates: readonly string[], text: string, selectorKey: string): Promise<string> {
const selector = await this.clickFirst(candidates, selectorKey);
await this.webContents.debugger.sendCommand("Input.insertText", { text });
return selector;
}
async waitForText(
candidates: readonly string[],
expectedText: string,
timeoutMs = 12_000,
): Promise<void> {
const fingerprint = expectedText.trim().slice(0, 40);
if (fingerprint.length === 0) {
throw browserVerificationFailed(this.platform, this.actionType, "empty text fingerprint");
}
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const matched = await this.evaluate<boolean>(buildTextMatchExpression(candidates, fingerprint));
if (matched) return;
this.retryCount += 1;
await delay(POLL_INTERVAL_MS);
}
throw browserVerificationFailed(this.platform, this.actionType, `text not found after submit: ${fingerprint}`);
}
async waitForAnyText(candidates: readonly string[], timeoutMs = 10_000): Promise<string[]> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const texts = await this.evaluate<string[]>(buildVisibleTextsExpression(candidates));
if (texts.length > 0) return texts;
this.retryCount += 1;
await delay(POLL_INTERVAL_MS);
}
throw browserSelectorDrift(this.platform, this.actionType, "content_items");
}
private async waitForElement(
candidates: readonly string[],
selectorKey: string,
timeoutMs: number,
): Promise<ElementProbe> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const element = await this.evaluate<ElementProbe | null>(buildElementProbeExpression(candidates));
if (element) return element;
this.retryCount += 1;
await delay(POLL_INTERVAL_MS);
}
throw browserSelectorDrift(this.platform, this.actionType, selectorKey);
}
}
function buildElementProbeExpression(candidates: readonly string[]): string {
return `(() => {
const candidates = ${JSON.stringify(candidates)};
for (const selector of candidates) {
const elements = Array.from(document.querySelectorAll(selector));
for (const element of elements) {
const rect = element.getBoundingClientRect();
const style = window.getComputedStyle(element);
const disabled = element.disabled === true || element.getAttribute("aria-disabled") === "true";
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || disabled) continue;
element.scrollIntoView({ block: "center", inline: "center" });
const nextRect = element.getBoundingClientRect();
return {
selector,
x: nextRect.left + nextRect.width / 2,
y: nextRect.top + nextRect.height / 2,
text: (element.innerText || element.textContent || "").slice(0, 200)
};
}
}
return null;
})()`;
}
function buildTextMatchExpression(candidates: readonly string[], fingerprint: string): string {
return `(() => {
const candidates = ${JSON.stringify(candidates)};
const fingerprint = ${JSON.stringify(fingerprint)};
for (const selector of candidates) {
for (const element of Array.from(document.querySelectorAll(selector))) {
const text = element.innerText || element.textContent || "";
if (text.includes(fingerprint)) return true;
}
}
return false;
})()`;
}
function buildVisibleTextsExpression(candidates: readonly string[]): string {
return `(() => {
const out = [];
const candidates = ${JSON.stringify(candidates)};
for (const selector of candidates) {
for (const element of Array.from(document.querySelectorAll(selector))) {
const rect = element.getBoundingClientRect();
if (rect.width <= 0 || rect.height <= 0) continue;
const text = (element.innerText || element.textContent || "").trim();
if (text.length > 0) out.push(text.slice(0, 300));
if (out.length >= 20) return out;
}
}
return out;
})()`;
}
function buildAuthenticatedSurfaceExpression(
candidates: readonly string[],
authenticatedCookieNames: readonly string[],
): string {
return `(() => {
const candidates = ${JSON.stringify(candidates)};
for (const selector of candidates) {
for (const element of Array.from(document.querySelectorAll(selector))) {
const rect = element.getBoundingClientRect();
const style = window.getComputedStyle(element);
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue;
return selector;
}
}
const cookie = document.cookie || "";
const authenticatedCookieNames = ${JSON.stringify(authenticatedCookieNames.map((name) => name.toLowerCase()))};
if (authenticatedCookieNames.length === 0 && cookie.trim().length > 0) return "document.cookie";
const cookieNames = new Set(cookie.split(";").map((part) => part.split("=")[0]?.trim().toLowerCase()).filter(Boolean));
for (const name of authenticatedCookieNames) {
if (cookieNames.has(name)) return "document.cookie:" + name;
}
return null;
})()`;
}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
+33
View File
@@ -0,0 +1,33 @@
import { RecoverableError, UpstreamError, UserInputError } from "../errors/index.js";
export function browserNotLoggedIn(platform: string, actionType: string): UserInputError {
return new UserInputError({
code: "browser.not_logged_in",
message: `${platform} session is not authenticated for ${actionType}.`,
context: { platform, actionType },
});
}
export function browserSelectorDrift(
platform: string,
actionType: string,
selectorKey: string,
): UpstreamError {
return new UpstreamError({
code: "browser.selector_drift",
message: `${platform} selector set did not expose ${selectorKey} for ${actionType}.`,
context: { platform, actionType, selectorKey },
});
}
export function browserVerificationFailed(
platform: string,
actionType: string,
expectation: string,
): RecoverableError {
return new RecoverableError({
code: "browser.verification_failed",
message: `${platform} ${actionType} did not pass verification: ${expectation}.`,
context: { platform, actionType, expectation },
});
}
+231 -216
View File
@@ -3,25 +3,86 @@ import { existsSync, writeFileSync } from "node:fs";
import type { NativeImage } from "electron";
import log from "electron-log/main.js";
import type { BrowserAction, BrowserActionResult } from "./browser-types";
import { assertAllowedUrl, attachWindowSecurity, getComposeUrl, getFeedUrl, getSearchUrl } from "./browser-url-helpers";
import {
assertAllowedUrl,
attachWindowSecurity,
getComposeUrl,
getFeedUrl,
getPlatformRules,
getSearchUrl,
} from "./browser-url-helpers";
import { BrowserCdpClient } from "./browser-action-cdp";
import { assertAuthenticatedPage, assertAuthenticatedSession } from "./browser-action-auth";
import { getPlatformSelectors } from "./platform-selectors";
import type { PlatformSelectors } from "./platform-selectors";
import { browserVerificationFailed } from "./browser-action-errors";
import { isAppError } from "../errors/index.js";
export { getPlatformRules, assertAllowedUrl, attachWindowSecurity } from "./browser-url-helpers";
type BrowserActionOutcome =
| "success"
| "logged_out"
| "selector_drift"
| "network"
| "rate_limited"
| "cancelled"
| "verification_failed";
interface BrowserActionTelemetry {
platform: string;
action_type: string;
selector_hit: string | null;
selector_hits: string[];
retries: number;
duration_ms: number;
final_outcome: BrowserActionOutcome;
}
export function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export function waitForLoad(win: BrowserWindow, timeoutMs = 15000): Promise<void> {
return new Promise((resolve) => {
if (win.webContents.isLoading()) {
const timer = setTimeout(resolve, timeoutMs);
win.webContents.once("did-finish-load", () => {
clearTimeout(timer);
resolve();
});
} else {
export function waitForLoad(
win: BrowserWindow,
timeoutMs = 15000,
context: { platform: string; actionType: string } = { platform: "browser", actionType: "load" },
): Promise<void> {
return new Promise((resolve, reject) => {
if (!win.webContents.isLoading()) {
resolve();
return;
}
const cleanup = () => {
clearTimeout(timer);
win.webContents.off("did-finish-load", onFinish);
win.webContents.off("did-fail-load", onFail);
win.off("closed", onClosed);
};
const fail = (expectation: string) => {
cleanup();
reject(browserVerificationFailed(context.platform, context.actionType, expectation));
};
const onFinish = () => {
cleanup();
resolve();
};
const onFail = (
_event: Electron.Event,
errorCode: number,
errorDescription: string,
_validatedUrl: string,
isMainFrame: boolean,
) => {
if (isMainFrame) fail(`page load failed: ${errorDescription} (${errorCode})`);
};
const onClosed = () => fail("window closed during page load");
const timer = setTimeout(() => fail(`page load exceeded ${timeoutMs}ms`), timeoutMs);
win.webContents.once("did-finish-load", onFinish);
win.webContents.on("did-fail-load", onFail);
win.once("closed", onClosed);
});
}
@@ -35,6 +96,8 @@ export async function runAction(
const payload = action.payload;
const screenshotPath = `${screenshotDir}/${action.id}.png`;
const platform = action.platform;
const startedAt = Date.now();
const selectorHits: string[] = [];
log.info(`[browser] Executing ${action.actionType} for ${platform}/${action.socialAccountId}`);
@@ -54,233 +117,106 @@ export async function runAction(
attachWindowSecurity(win, platform);
win.setMenuBarVisibility(false);
const cdp = new BrowserCdpClient(win.webContents, platform, action.actionType);
const selectors = getPlatformSelectors(platform);
const assertNotCancelled = () => {
if (cancelledActionIds.has(action.id)) {
throw new Error("BROWSER_ACTION_CANCELLED");
}
};
const remember = (selector: string | null) => {
if (selector) selectorHits.push(selector);
};
try {
assertNotCancelled();
await assertAuthenticatedSession(session, platform, action.actionType);
let summary = "Action completed";
const assertNotCancelled = () => {
if (cancelledActionIds.has(action.id)) {
throw new Error("BROWSER_ACTION_CANCELLED");
}
};
switch (action.actionType) {
case "navigate": {
assertNotCancelled();
const url = String(payload.url || "https://google.com");
assertAllowedUrl(platform, url, "navigate");
await win.loadURL(url);
await waitForLoad(win);
assertNotCancelled();
const title = await win.webContents.executeJavaScript("document.title");
const url = String(payload.url || getFeedUrl(platform));
await loadActionUrl(win, platform, action.actionType, url);
remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType));
const title = await cdp.evaluate<string>("document.title");
summary = `Navigated to ${title}`;
break;
}
case "post": {
const content = requirePayloadText(payload.content, "content");
await loadActionUrl(win, platform, action.actionType, getComposeUrl(platform));
remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType));
remember(await cdp.clickOptional(selectors.composerOpen, "composer_open"));
remember(await cdp.focusAndType(selectors.editor, content, "editor"));
assertNotCancelled();
const content = String(payload.content || "");
const postUrl = getComposeUrl(platform);
await win.loadURL(postUrl);
await waitForLoad(win);
await delay(2000);
assertNotCancelled();
if (platform === "twitter") {
// Use clipboard-based approach since Twitter's React editor ignores execCommand
await win.webContents.executeJavaScript(`
(async () => {
const editor = document.querySelector('[data-testid="tweetTextarea_0"], [role="textbox"][contenteditable="true"]');
if (!editor) return;
editor.focus();
const dt = new DataTransfer();
dt.setData('text/plain', ${JSON.stringify(content)});
const pasteEvent = new ClipboardEvent('paste', { clipboardData: dt, bubbles: true, cancelable: true });
editor.dispatchEvent(pasteEvent);
await new Promise(r => setTimeout(r, 500));
if (!editor.textContent || editor.textContent.trim().length === 0) {
editor.textContent = ${JSON.stringify(content)};
editor.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: ${JSON.stringify(content)} }));
}
})()
`);
await delay(1500);
await win.webContents.executeJavaScript(`
(() => {
const btn = document.querySelector('[data-testid="tweetButton"], [data-testid="tweetButtonInline"]');
if (btn && !btn.disabled) btn.click();
})()
`);
await delay(3000);
} else if (platform === "linkedin") {
await win.webContents.executeJavaScript(`
(() => {
const btn = document.querySelector('button.share-box-feed-entry__trigger, button.artdeco-button--muted');
if (btn) btn.click();
})()
`);
await delay(2000);
await win.webContents.executeJavaScript(`
(async () => {
const editor = document.querySelector('[role="textbox"][contenteditable="true"], .ql-editor[contenteditable="true"]');
if (editor) {
editor.focus();
const dt = new DataTransfer();
dt.setData('text/plain', ${JSON.stringify(content)});
const pasteEvent = new ClipboardEvent('paste', { clipboardData: dt, bubbles: true, cancelable: true });
editor.dispatchEvent(pasteEvent);
await new Promise(r => setTimeout(r, 500));
if (!editor.textContent || editor.textContent.trim().length === 0) {
editor.textContent = ${JSON.stringify(content)};
editor.dispatchEvent(new InputEvent('input', { bubbles: true }));
}
}
})()
`);
await delay(1500);
await win.webContents.executeJavaScript(`
(() => {
const btn = document.querySelector('button.share-actions__primary-action, button.artdeco-button--primary[type="submit"]');
if (btn && !btn.disabled) btn.click();
})()
`);
await delay(3000);
}
remember(await cdp.clickFirst(selectors.submit, "submit"));
await cdp.waitForText(selectors.feedItems, content);
summary = `Posted to ${platform}: "${content.slice(0, 50)}..."`;
break;
}
case "reply": {
const targetUrl = requirePayloadText(payload.targetUrl, "targetUrl");
const replyContent = requirePayloadText(payload.content, "content");
await loadActionUrl(win, platform, action.actionType, targetUrl);
remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType));
remember(await cdp.focusAndType(selectors.replyEditor, replyContent, "reply_editor"));
assertNotCancelled();
const targetUrl = String(payload.targetUrl || "");
assertAllowedUrl(platform, targetUrl, "reply");
const replyContent = String(payload.content || "");
await win.loadURL(targetUrl);
await waitForLoad(win);
await delay(2000);
assertNotCancelled();
await win.webContents.executeJavaScript(`
(async () => {
const editor = document.querySelector('[data-testid="tweetTextarea_0"], [role="textbox"][contenteditable="true"]');
if (editor) {
editor.focus();
const dt = new DataTransfer();
dt.setData('text/plain', ${JSON.stringify(replyContent)});
const pasteEvent = new ClipboardEvent('paste', { clipboardData: dt, bubbles: true, cancelable: true });
editor.dispatchEvent(pasteEvent);
await new Promise(r => setTimeout(r, 500));
if (!editor.textContent || editor.textContent.trim().length === 0) {
editor.textContent = ${JSON.stringify(replyContent)};
editor.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: ${JSON.stringify(replyContent)} }));
}
}
})()
`);
await delay(1500);
await win.webContents.executeJavaScript(`
(() => {
const btn = document.querySelector('[data-testid="tweetButton"], [data-testid="tweetButtonInline"], button[type="submit"]');
if (btn && !btn.disabled) btn.click();
})()
`);
await delay(3000);
remember(await cdp.clickFirst(selectors.replySubmit, "reply_submit"));
await cdp.waitForText(selectors.feedItems, replyContent);
summary = `Replied to ${targetUrl.slice(0, 50)}`;
break;
}
case "like": {
assertNotCancelled();
const targetUrl = String(payload.targetUrl || "");
assertAllowedUrl(platform, targetUrl, "like");
await win.loadURL(targetUrl);
await waitForLoad(win);
await delay(2000);
assertNotCancelled();
await win.webContents.executeJavaScript(`
(() => {
const btn = document.querySelector('[data-testid="like"], button[aria-label*="Like"]');
if (btn) btn.click();
})()
`);
await delay(2000);
const targetUrl = requirePayloadText(payload.targetUrl, "targetUrl");
await loadActionUrl(win, platform, action.actionType, targetUrl);
remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType));
remember(await cdp.clickFirst(selectors.likeButton, "like_button"));
await cdp.waitForFirst(selectors.likedButton, "liked_state", 5_000);
summary = `Liked post at ${targetUrl.slice(0, 50)}`;
break;
}
case "follow": {
assertNotCancelled();
const targetUrl = String(payload.targetUrl || payload.profileUrl || "");
assertAllowedUrl(platform, targetUrl, "follow");
await win.loadURL(targetUrl);
await waitForLoad(win);
await delay(2000);
assertNotCancelled();
await win.webContents.executeJavaScript(`
(() => {
const btn = document.querySelector('[data-testid*="follow"], button[aria-label*="Follow"]');
if (btn && !btn.textContent.includes('Following')) btn.click();
})()
`);
await delay(2000);
if (!targetUrl) throw browserVerificationFailed(platform, action.actionType, "missing targetUrl/profileUrl");
await loadActionUrl(win, platform, action.actionType, targetUrl);
remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType));
remember(await cdp.clickFirst(selectors.followButton, "follow_button"));
await cdp.waitForFirst(selectors.followingButton, "following_state", 5_000);
summary = `Followed user at ${targetUrl.slice(0, 50)}`;
break;
}
case "browse_feed": {
assertNotCancelled();
const feedUrl = String(payload.url || getFeedUrl(platform));
assertAllowedUrl(platform, feedUrl, "browse_feed");
const maxPosts = Number(payload.maxPosts || 10);
await win.loadURL(feedUrl);
await waitForLoad(win);
await delay(3000);
assertNotCancelled();
for (let i = 0; i < 3; i++) {
await win.webContents.executeJavaScript("window.scrollBy(0, window.innerHeight)");
await delay(1500);
await loadActionUrl(win, platform, action.actionType, feedUrl);
remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType));
for (let i = 0; i < 3; i += 1) {
await cdp.evaluate<void>("window.scrollBy(0, window.innerHeight)");
}
const posts: string[] = await win.webContents.executeJavaScript(`
(() => {
const articles = document.querySelectorAll('article, [data-testid="tweet"], .feed-shared-update-v2');
return Array.from(articles).slice(0, ${maxPosts}).map(el => el.innerText.slice(0, 300));
})()
`);
summary = `Browsed feed — found ${posts.length} posts`;
const posts = await cdp.waitForAnyText(selectors.feedItems);
summary = `Browsed feed — found ${Math.min(posts.length, maxPosts)} posts`;
break;
}
case "search": {
assertNotCancelled();
const query = String(payload.query || "");
const searchUrl = getSearchUrl(platform, query);
assertAllowedUrl(platform, searchUrl, "search");
await win.loadURL(searchUrl);
await waitForLoad(win);
await delay(3000);
assertNotCancelled();
const results: string[] = await win.webContents.executeJavaScript(`
(() => {
const items = document.querySelectorAll('article, [data-testid="tweet"], .search-result');
return Array.from(items).slice(0, 10).map(el => el.innerText.slice(0, 200));
})()
`);
const query = requirePayloadText(payload.query, "query");
await loadActionUrl(win, platform, action.actionType, getSearchUrl(platform, query));
remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType));
const results = await cdp.waitForAnyText(selectors.searchItems);
summary = `Search "${query.slice(0, 30)}" — ${results.length} results`;
break;
}
case "screenshot": {
assertNotCancelled();
const url = String(payload.url || "about:blank");
if (url !== "about:blank") {
assertAllowedUrl(platform, url, "screenshot");
await win.loadURL(url);
await waitForLoad(win);
await delay(2000);
}
assertNotCancelled();
const title = await win.webContents.executeJavaScript("document.title");
const url = String(payload.url || getFeedUrl(platform));
await loadActionUrl(win, platform, action.actionType, url);
remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType));
const title = await cdp.evaluate<string>("document.title");
summary = `Screenshot captured: ${title}`;
break;
}
@@ -289,18 +225,18 @@ export async function runAction(
throw new Error(`Unsupported browser action type: ${action.actionType}`);
}
try {
const image: NativeImage = await win.webContents.capturePage();
if (!image.isEmpty()) {
writeFileSync(screenshotPath, image.toPNG());
}
} catch (err) {
log.warn(`[browser] Screenshot failed: ${err}`);
}
await delay(1000);
if (!win.isDestroyed()) win.close();
assertNotCancelled();
await captureScreenshot(win, screenshotPath, action.id, "success");
await closeWindow(win);
logActionTelemetry({
platform,
action_type: action.actionType,
selector_hit: selectorHits.at(-1) ?? null,
selector_hits: selectorHits,
retries: cdp.getRetryCount(),
duration_ms: Date.now() - startedAt,
final_outcome: "success",
});
return {
success: true,
cancelled: false,
@@ -309,22 +245,25 @@ export async function runAction(
error: null,
};
} catch (error) {
const wasCancelled = cancelledActionIds.has(action.id) || (error instanceof Error && error.message === "BROWSER_ACTION_CANCELLED");
try {
const image: NativeImage = await win.webContents.capturePage();
if (!image.isEmpty()) writeFileSync(screenshotPath, image.toPNG());
} catch (screenshotError) {
log.warn(
`[browser] failure screenshot capture failed for action ${action.id}: ${screenshotError instanceof Error ? screenshotError.message : String(screenshotError)}`,
);
}
const wasCancelled = cancelledActionIds.has(action.id)
|| (error instanceof Error && error.message === "BROWSER_ACTION_CANCELLED");
await captureScreenshot(win, screenshotPath, action.id, "failure");
await closeWindow(win);
if (!win.isDestroyed()) win.close();
const msg = wasCancelled ? "Action cancelled." : error instanceof Error ? error.message : String(error);
const msg = wasCancelled ? "Action cancelled." : describeBrowserActionError(error);
const finalOutcome = classifyBrowserOutcome(error, wasCancelled);
if (!wasCancelled) {
log.error(`[browser] Action ${action.actionType} failed: ${msg}`);
}
logActionTelemetry({
platform,
action_type: action.actionType,
selector_hit: selectorHits.at(-1) ?? null,
selector_hits: selectorHits,
retries: cdp.getRetryCount(),
duration_ms: Date.now() - startedAt,
final_outcome: finalOutcome,
});
return {
success: false,
cancelled: wasCancelled,
@@ -333,7 +272,83 @@ export async function runAction(
error: msg,
};
} finally {
await cdp.dispose().catch(() => undefined);
activeActionWindows.delete(action.id);
cancelledActionIds.delete(action.id);
}
}
async function confirmAuthenticatedPage(
win: BrowserWindow,
cdp: BrowserCdpClient,
selectors: PlatformSelectors,
platform: string,
actionType: string,
): Promise<string> {
await assertAuthenticatedPage(() => win.webContents.getURL(), platform, actionType);
return cdp.waitForAuthenticatedSurface(
selectors.authenticated,
getPlatformRules(platform).authenticatedCookieNames,
);
}
async function loadActionUrl(
win: BrowserWindow,
platform: string,
actionType: string,
url: string,
): Promise<void> {
assertAllowedUrl(platform, url, actionType);
await win.loadURL(url);
await waitForLoad(win, 15000, { platform, actionType });
}
function requirePayloadText(value: unknown, key: string): string {
const text = typeof value === "string" ? value.trim() : "";
if (text.length === 0) {
throw new Error(`Browser action payload missing ${key}.`);
}
return text;
}
async function captureScreenshot(
win: BrowserWindow,
screenshotPath: string,
actionId: string,
phase: "success" | "failure",
): Promise<void> {
try {
const image: NativeImage = await win.webContents.capturePage();
if (!image.isEmpty()) {
writeFileSync(screenshotPath, image.toPNG());
}
} catch (err) {
log.warn(`[browser] ${phase} screenshot failed for action ${actionId}: ${err instanceof Error ? err.message : String(err)}`);
}
}
async function closeWindow(win: BrowserWindow): Promise<void> {
if (win.isDestroyed()) return;
await delay(250);
if (!win.isDestroyed()) win.close();
}
function describeBrowserActionError(error: unknown): string {
if (isAppError(error)) return `[${error.code}] ${error.message}`;
return error instanceof Error ? error.message : String(error);
}
function classifyBrowserOutcome(error: unknown, wasCancelled: boolean): BrowserActionOutcome {
if (wasCancelled) return "cancelled";
if (isAppError(error)) {
if (error.code === "browser.not_logged_in") return "logged_out";
if (error.code === "browser.selector_drift") return "selector_drift";
if (error.code === "browser.verification_failed") return "verification_failed";
}
if (error instanceof Error && /429|rate[_ -]?limit|too many requests/i.test(error.message)) return "rate_limited";
return "network";
}
function logActionTelemetry(telemetry: BrowserActionTelemetry): void {
log.info("[browser] action telemetry", telemetry);
}
+11
View File
@@ -25,6 +25,7 @@ export interface PlatformRule {
authCheckUrl: string;
allowedHosts: string[];
cookieOrigins: string[];
authenticatedCookieNames?: string[];
unauthenticatedUrlMarkers: string[];
}
@@ -34,6 +35,7 @@ export const PLATFORM_RULES: Record<string, PlatformRule> = {
authCheckUrl: "https://x.com/home",
allowedHosts: ["x.com", "twitter.com"],
cookieOrigins: ["https://x.com", "https://twitter.com"],
authenticatedCookieNames: ["auth_token"],
unauthenticatedUrlMarkers: ["/i/flow/login", "/login"],
},
linkedin: {
@@ -41,6 +43,7 @@ export const PLATFORM_RULES: Record<string, PlatformRule> = {
authCheckUrl: "https://www.linkedin.com/feed/",
allowedHosts: ["linkedin.com"],
cookieOrigins: ["https://www.linkedin.com"],
authenticatedCookieNames: ["li_at"],
unauthenticatedUrlMarkers: ["/login", "/checkpoint"],
},
instagram: {
@@ -48,6 +51,7 @@ export const PLATFORM_RULES: Record<string, PlatformRule> = {
authCheckUrl: "https://www.instagram.com/",
allowedHosts: ["instagram.com"],
cookieOrigins: ["https://www.instagram.com"],
authenticatedCookieNames: ["sessionid", "ds_user_id"],
unauthenticatedUrlMarkers: ["/accounts/login"],
},
facebook: {
@@ -55,6 +59,7 @@ export const PLATFORM_RULES: Record<string, PlatformRule> = {
authCheckUrl: "https://www.facebook.com/",
allowedHosts: ["facebook.com"],
cookieOrigins: ["https://www.facebook.com"],
authenticatedCookieNames: ["c_user", "xs"],
unauthenticatedUrlMarkers: ["/login"],
},
reddit: {
@@ -62,6 +67,7 @@ export const PLATFORM_RULES: Record<string, PlatformRule> = {
authCheckUrl: "https://www.reddit.com/",
allowedHosts: ["reddit.com"],
cookieOrigins: ["https://www.reddit.com"],
authenticatedCookieNames: ["reddit_session", "token_v2"],
unauthenticatedUrlMarkers: ["/login"],
},
youtube: {
@@ -69,6 +75,7 @@ export const PLATFORM_RULES: Record<string, PlatformRule> = {
authCheckUrl: "https://www.youtube.com/",
allowedHosts: ["youtube.com", "google.com"],
cookieOrigins: ["https://www.youtube.com", "https://accounts.google.com"],
authenticatedCookieNames: ["SAPISID", "LOGIN_INFO"],
unauthenticatedUrlMarkers: ["/signin", "/ServiceLogin"],
},
tiktok: {
@@ -76,6 +83,7 @@ export const PLATFORM_RULES: Record<string, PlatformRule> = {
authCheckUrl: "https://www.tiktok.com/foryou",
allowedHosts: ["tiktok.com"],
cookieOrigins: ["https://www.tiktok.com"],
authenticatedCookieNames: ["sessionid", "sessionid_ss"],
unauthenticatedUrlMarkers: ["/login"],
},
bluesky: {
@@ -97,6 +105,7 @@ export const PLATFORM_RULES: Record<string, PlatformRule> = {
authCheckUrl: "https://www.threads.net/",
allowedHosts: ["threads.net"],
cookieOrigins: ["https://www.threads.net"],
authenticatedCookieNames: ["sessionid", "ds_user_id"],
unauthenticatedUrlMarkers: ["/login"],
},
pinterest: {
@@ -104,6 +113,7 @@ export const PLATFORM_RULES: Record<string, PlatformRule> = {
authCheckUrl: "https://www.pinterest.com/",
allowedHosts: ["pinterest.com"],
cookieOrigins: ["https://www.pinterest.com"],
authenticatedCookieNames: ["_pinterest_sess"],
unauthenticatedUrlMarkers: ["/login"],
},
telegram: {
@@ -132,6 +142,7 @@ export const PLATFORM_RULES: Record<string, PlatformRule> = {
authCheckUrl: "https://weibo.com/",
allowedHosts: ["weibo.com", "weibo.cn"],
cookieOrigins: ["https://weibo.com"],
authenticatedCookieNames: ["SUB", "SUBP"],
unauthenticatedUrlMarkers: ["/login", "/signup"],
},
xiaohongshu: {
+135
View File
@@ -0,0 +1,135 @@
export interface PlatformSelectors {
authenticated: readonly string[];
composerOpen: readonly string[];
editor: readonly string[];
submit: readonly string[];
replyEditor: readonly string[];
replySubmit: readonly string[];
likeButton: readonly string[];
likedButton: readonly string[];
followButton: readonly string[];
followingButton: readonly string[];
feedItems: readonly string[];
searchItems: readonly string[];
}
const GENERIC_SELECTORS: PlatformSelectors = {
authenticated: [
"[data-testid='AppTabBar_Profile_Link']",
"a[href*='/in/']",
"img[alt*='profile' i]",
"[aria-label*='profile' i]",
],
composerOpen: [
"button[aria-label*='post' i]",
"button[aria-label*='compose' i]",
"button.share-box-feed-entry__trigger",
],
editor: [
"[role='textbox'][contenteditable='true']",
"textarea",
".ql-editor[contenteditable='true']",
],
submit: [
"button[type='submit']",
"button[aria-label*='post' i]",
"button[aria-label*='send' i]",
],
replyEditor: [
"[role='textbox'][contenteditable='true']",
"textarea",
],
replySubmit: [
"button[type='submit']",
"button[aria-label*='reply' i]",
"button[aria-label*='send' i]",
],
likeButton: [
"button[aria-label*='like' i]",
"[role='button'][aria-label*='like' i]",
],
likedButton: [
"button[aria-pressed='true'][aria-label*='like' i]",
"[data-testid='unlike']",
],
followButton: [
"button[aria-label*='follow' i]",
"[role='button'][aria-label*='follow' i]",
],
followingButton: [
"button[aria-label*='following' i]",
"button[aria-label*='requested' i]",
],
feedItems: ["article", "[role='article']", ".feed-shared-update-v2"],
searchItems: ["article", "[role='article']", ".search-result", ".feed-shared-update-v2"],
};
const PLATFORM_SELECTORS: Record<string, Partial<PlatformSelectors>> = {
twitter: {
authenticated: [
"[data-testid='AppTabBar_Profile_Link']",
"[data-testid='SideNav_AccountSwitcher_Button']",
"a[aria-label*='Profile']",
],
editor: [
"[data-testid='tweetTextarea_0']",
"[role='textbox'][contenteditable='true']",
],
submit: [
"[data-testid='tweetButton']",
"[data-testid='tweetButtonInline']",
],
replyEditor: [
"[data-testid='tweetTextarea_0']",
"[role='textbox'][contenteditable='true']",
],
replySubmit: [
"[data-testid='tweetButton']",
"[data-testid='tweetButtonInline']",
"button[type='submit']",
],
likeButton: ["[data-testid='like']", "button[aria-label*='Like']"],
likedButton: ["[data-testid='unlike']", "button[aria-label*='Unlike']"],
feedItems: ["article", "[data-testid='tweet']"],
searchItems: ["article", "[data-testid='tweet']"],
},
linkedin: {
authenticated: [
"a[href*='/in/']",
"img.global-nav__me-photo",
"button.global-nav__primary-link-me-menu-trigger",
],
composerOpen: [
"button.share-box-feed-entry__trigger",
"button.artdeco-button--muted",
],
editor: [
"[role='textbox'][contenteditable='true']",
".ql-editor[contenteditable='true']",
],
submit: [
"button.share-actions__primary-action",
"button.artdeco-button--primary[type='submit']",
],
feedItems: [".feed-shared-update-v2", "article", ".update-components-text"],
searchItems: [".reusable-search__result-container", ".feed-shared-update-v2", "article"],
},
};
export function getPlatformSelectors(platform: string): PlatformSelectors {
const overrides = PLATFORM_SELECTORS[platform] ?? {};
return {
authenticated: overrides.authenticated ?? GENERIC_SELECTORS.authenticated,
composerOpen: overrides.composerOpen ?? GENERIC_SELECTORS.composerOpen,
editor: overrides.editor ?? GENERIC_SELECTORS.editor,
submit: overrides.submit ?? GENERIC_SELECTORS.submit,
replyEditor: overrides.replyEditor ?? overrides.editor ?? GENERIC_SELECTORS.replyEditor,
replySubmit: overrides.replySubmit ?? overrides.submit ?? GENERIC_SELECTORS.replySubmit,
likeButton: overrides.likeButton ?? GENERIC_SELECTORS.likeButton,
likedButton: overrides.likedButton ?? GENERIC_SELECTORS.likedButton,
followButton: overrides.followButton ?? GENERIC_SELECTORS.followButton,
followingButton: overrides.followingButton ?? GENERIC_SELECTORS.followingButton,
feedItems: overrides.feedItems ?? GENERIC_SELECTORS.feedItems,
searchItems: overrides.searchItems ?? overrides.feedItems ?? GENERIC_SELECTORS.searchItems,
};
}
+65
View File
@@ -0,0 +1,65 @@
import type { Session } from "electron";
import { describe, expect, it } from "vitest";
import {
assertAuthenticatedPage,
assertAuthenticatedSession,
} from "../../src/main/browser/browser-action-auth";
import { getPlatformSelectors } from "../../src/main/browser/platform-selectors";
function fakeSession(cookiesByOrigin: Record<string, unknown[]>): Pick<Session, "cookies"> {
return {
cookies: {
get: async ({ url }) => cookiesByOrigin[url ?? ""] ?? [],
},
} as Pick<Session, "cookies">;
}
describe("browser action runtime guards", () => {
it("fails social actions before execution when the account session has no cookies", async () => {
await expect(
assertAuthenticatedSession(fakeSession({}), "twitter", "post"),
).rejects.toMatchObject({
code: "browser.not_logged_in",
});
});
it("rejects guest cookies for platforms with explicit auth cookies", async () => {
await expect(
assertAuthenticatedSession(
fakeSession({ "https://x.com": [{ name: "guest_id", value: "present" }] }),
"twitter",
"post",
),
).rejects.toMatchObject({
code: "browser.not_logged_in",
});
});
it("accepts a persisted platform cookie before executing actions", async () => {
await expect(
assertAuthenticatedSession(
fakeSession({ "https://x.com": [{ name: "auth_token", value: "present" }] }),
"twitter",
"post",
),
).resolves.toBeUndefined();
});
it("treats login redirects as an unauthenticated page state", async () => {
await expect(
assertAuthenticatedPage(() => "https://x.com/i/flow/login", "twitter", "post"),
).rejects.toMatchObject({
code: "browser.not_logged_in",
});
});
it("keeps platform selectors versioned outside the executor", () => {
const twitter = getPlatformSelectors("twitter");
const linkedin = getPlatformSelectors("linkedin");
expect(twitter.editor).toContain("[data-testid='tweetTextarea_0']");
expect(twitter.submit).toContain("[data-testid='tweetButton']");
expect(linkedin.composerOpen).toContain("button.share-box-feed-entry__trigger");
expect(linkedin.editor).toContain(".ql-editor[contenteditable='true']");
});
});