diff --git a/src/main/browser-manager.ts b/src/main/browser-manager.ts index 0bc698bf..3a282937 100644 --- a/src/main/browser-manager.ts +++ b/src/main/browser-manager.ts @@ -5,6 +5,7 @@ import { join, resolve, sep } from "node:path"; import log from "electron-log/main.js"; import type { BrowserAction, BrowserActionResult, QueuedAction } from "./browser/browser-types"; import { getPlatformRules, attachWindowSecurity, assertAllowedUrl, waitForLoad, runAction } from "./browser/browser-action-executor"; +import { hasAuthenticatedCookie } from "./browser/browser-action-auth"; export type { BrowserAction, BrowserActionResult } from "./browser/browser-types"; @@ -242,7 +243,7 @@ export class BrowserManager { } }), ); - const hasCookies = cookieSets.some((cookies) => cookies.length > 0); + const hasCookies = hasAuthenticatedCookie(cookieSets, rules.authenticatedCookieNames); if (!hasCookies) { return false; } diff --git a/src/main/browser/browser-action-auth.ts b/src/main/browser/browser-action-auth.ts new file mode 100644 index 00000000..81aca311 --- /dev/null +++ b/src/main/browser/browser-action-auth.ts @@ -0,0 +1,49 @@ +import type { Session } from "electron"; +import { browserNotLoggedIn } from "./browser-action-errors"; +import { getPlatformRules } from "./browser-url-helpers"; + +type CookieLike = { + name?: string; +}; + +export async function assertAuthenticatedSession( + session: Pick, + platform: string, + actionType: string, +): Promise { + const rules = getPlatformRules(platform); + const cookieSets = await Promise.all( + rules.cookieOrigins.map((origin) => session.cookies.get({ url: origin }).catch(() => [])), + ); + if (!hasAuthenticatedCookie(cookieSets, rules.authenticatedCookieNames)) { + throw browserNotLoggedIn(platform, actionType); + } +} + +export function hasAuthenticatedCookie( + cookieSets: readonly (readonly CookieLike[])[], + authenticatedCookieNames?: readonly string[], +): boolean { + if (!authenticatedCookieNames || authenticatedCookieNames.length === 0) { + return cookieSets.some((cookies) => cookies.length > 0); + } + const expectedNames = new Set(authenticatedCookieNames.map((name) => name.toLowerCase())); + return cookieSets.some((cookies) => + cookies.some((cookie) => cookie.name && expectedNames.has(cookie.name.toLowerCase())), + ); +} + +export async function assertAuthenticatedPage( + getUrl: () => string, + platform: string, + actionType: string, +): Promise { + const finalUrl = getUrl().toLowerCase(); + const rules = getPlatformRules(platform); + const bouncedToLogin = rules.unauthenticatedUrlMarkers.some((marker) => + finalUrl.includes(marker.toLowerCase()), + ); + if (bouncedToLogin) { + throw browserNotLoggedIn(platform, actionType); + } +} diff --git a/src/main/browser/browser-action-cdp.ts b/src/main/browser/browser-action-cdp.ts new file mode 100644 index 00000000..57721abf --- /dev/null +++ b/src/main/browser/browser-action-cdp.ts @@ -0,0 +1,251 @@ +import type { WebContents } from "electron"; +import { browserNotLoggedIn, browserSelectorDrift, browserVerificationFailed } from "./browser-action-errors"; +import { isAppError } from "../errors/index.js"; + +type RuntimeEvaluateResult = { + result?: { value?: unknown }; + exceptionDetails?: { text?: string }; +}; + +type ElementProbe = { + selector: string; + x: number; + y: number; + text: string; +}; + +const POLL_INTERVAL_MS = 250; + +export class BrowserCdpClient { + private attachedByClient = false; + private retryCount = 0; + + constructor( + private readonly webContents: WebContents, + private readonly platform: string, + private readonly actionType: string, + ) {} + + async attach(): Promise { + if (this.webContents.debugger.isAttached()) return; + this.webContents.debugger.attach("1.3"); + this.attachedByClient = true; + } + + async dispose(): Promise { + if (this.attachedByClient && this.webContents.debugger.isAttached()) { + this.webContents.debugger.detach(); + } + } + + getRetryCount(): number { + return this.retryCount; + } + + async evaluate(expression: string): Promise { + await this.attach(); + const response = await this.webContents.debugger.sendCommand("Runtime.evaluate", { + expression, + awaitPromise: true, + returnByValue: true, + }) as RuntimeEvaluateResult; + if (response.exceptionDetails) { + throw browserVerificationFailed( + this.platform, + this.actionType, + response.exceptionDetails.text ?? "runtime evaluation failed", + ); + } + return response.result?.value as T; + } + + async clickFirst(candidates: readonly string[], selectorKey: string, timeoutMs = 10_000): Promise { + const element = await this.waitForElement(candidates, selectorKey, timeoutMs); + await this.webContents.debugger.sendCommand("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: element.x, + y: element.y, + button: "none", + }); + await this.webContents.debugger.sendCommand("Input.dispatchMouseEvent", { + type: "mousePressed", + x: element.x, + y: element.y, + button: "left", + clickCount: 1, + }); + await this.webContents.debugger.sendCommand("Input.dispatchMouseEvent", { + type: "mouseReleased", + x: element.x, + y: element.y, + button: "left", + clickCount: 1, + }); + return element.selector; + } + + async clickOptional(candidates: readonly string[], selectorKey: string, timeoutMs = 1_500): Promise { + try { + return await this.clickFirst(candidates, selectorKey, timeoutMs); + } catch (error) { + if (!isAppError(error) || error.code !== "browser.selector_drift") throw error; + return null; + } + } + + async waitForFirst(candidates: readonly string[], selectorKey: string, timeoutMs = 10_000): Promise { + return (await this.waitForElement(candidates, selectorKey, timeoutMs)).selector; + } + + async waitForAuthenticatedSurface( + candidates: readonly string[], + authenticatedCookieNames?: readonly string[], + timeoutMs = 5_000, + ): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const selector = await this.evaluate( + buildAuthenticatedSurfaceExpression(candidates, authenticatedCookieNames ?? []), + ); + if (selector) return selector; + this.retryCount += 1; + await delay(POLL_INTERVAL_MS); + } + throw browserNotLoggedIn(this.platform, this.actionType); + } + + async focusAndType(candidates: readonly string[], text: string, selectorKey: string): Promise { + const selector = await this.clickFirst(candidates, selectorKey); + await this.webContents.debugger.sendCommand("Input.insertText", { text }); + return selector; + } + + async waitForText( + candidates: readonly string[], + expectedText: string, + timeoutMs = 12_000, + ): Promise { + const fingerprint = expectedText.trim().slice(0, 40); + if (fingerprint.length === 0) { + throw browserVerificationFailed(this.platform, this.actionType, "empty text fingerprint"); + } + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const matched = await this.evaluate(buildTextMatchExpression(candidates, fingerprint)); + if (matched) return; + this.retryCount += 1; + await delay(POLL_INTERVAL_MS); + } + throw browserVerificationFailed(this.platform, this.actionType, `text not found after submit: ${fingerprint}`); + } + + async waitForAnyText(candidates: readonly string[], timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const texts = await this.evaluate(buildVisibleTextsExpression(candidates)); + if (texts.length > 0) return texts; + this.retryCount += 1; + await delay(POLL_INTERVAL_MS); + } + throw browserSelectorDrift(this.platform, this.actionType, "content_items"); + } + + private async waitForElement( + candidates: readonly string[], + selectorKey: string, + timeoutMs: number, + ): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const element = await this.evaluate(buildElementProbeExpression(candidates)); + if (element) return element; + this.retryCount += 1; + await delay(POLL_INTERVAL_MS); + } + throw browserSelectorDrift(this.platform, this.actionType, selectorKey); + } +} + +function buildElementProbeExpression(candidates: readonly string[]): string { + return `(() => { + const candidates = ${JSON.stringify(candidates)}; + for (const selector of candidates) { + const elements = Array.from(document.querySelectorAll(selector)); + for (const element of elements) { + const rect = element.getBoundingClientRect(); + const style = window.getComputedStyle(element); + const disabled = element.disabled === true || element.getAttribute("aria-disabled") === "true"; + if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || disabled) continue; + element.scrollIntoView({ block: "center", inline: "center" }); + const nextRect = element.getBoundingClientRect(); + return { + selector, + x: nextRect.left + nextRect.width / 2, + y: nextRect.top + nextRect.height / 2, + text: (element.innerText || element.textContent || "").slice(0, 200) + }; + } + } + return null; + })()`; +} + +function buildTextMatchExpression(candidates: readonly string[], fingerprint: string): string { + return `(() => { + const candidates = ${JSON.stringify(candidates)}; + const fingerprint = ${JSON.stringify(fingerprint)}; + for (const selector of candidates) { + for (const element of Array.from(document.querySelectorAll(selector))) { + const text = element.innerText || element.textContent || ""; + if (text.includes(fingerprint)) return true; + } + } + return false; + })()`; +} + +function buildVisibleTextsExpression(candidates: readonly string[]): string { + return `(() => { + const out = []; + const candidates = ${JSON.stringify(candidates)}; + for (const selector of candidates) { + for (const element of Array.from(document.querySelectorAll(selector))) { + const rect = element.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) continue; + const text = (element.innerText || element.textContent || "").trim(); + if (text.length > 0) out.push(text.slice(0, 300)); + if (out.length >= 20) return out; + } + } + return out; + })()`; +} + +function buildAuthenticatedSurfaceExpression( + candidates: readonly string[], + authenticatedCookieNames: readonly string[], +): string { + return `(() => { + const candidates = ${JSON.stringify(candidates)}; + for (const selector of candidates) { + for (const element of Array.from(document.querySelectorAll(selector))) { + const rect = element.getBoundingClientRect(); + const style = window.getComputedStyle(element); + if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue; + return selector; + } + } + const cookie = document.cookie || ""; + const authenticatedCookieNames = ${JSON.stringify(authenticatedCookieNames.map((name) => name.toLowerCase()))}; + if (authenticatedCookieNames.length === 0 && cookie.trim().length > 0) return "document.cookie"; + const cookieNames = new Set(cookie.split(";").map((part) => part.split("=")[0]?.trim().toLowerCase()).filter(Boolean)); + for (const name of authenticatedCookieNames) { + if (cookieNames.has(name)) return "document.cookie:" + name; + } + return null; + })()`; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/main/browser/browser-action-errors.ts b/src/main/browser/browser-action-errors.ts new file mode 100644 index 00000000..224f0709 --- /dev/null +++ b/src/main/browser/browser-action-errors.ts @@ -0,0 +1,33 @@ +import { RecoverableError, UpstreamError, UserInputError } from "../errors/index.js"; + +export function browserNotLoggedIn(platform: string, actionType: string): UserInputError { + return new UserInputError({ + code: "browser.not_logged_in", + message: `${platform} session is not authenticated for ${actionType}.`, + context: { platform, actionType }, + }); +} + +export function browserSelectorDrift( + platform: string, + actionType: string, + selectorKey: string, +): UpstreamError { + return new UpstreamError({ + code: "browser.selector_drift", + message: `${platform} selector set did not expose ${selectorKey} for ${actionType}.`, + context: { platform, actionType, selectorKey }, + }); +} + +export function browserVerificationFailed( + platform: string, + actionType: string, + expectation: string, +): RecoverableError { + return new RecoverableError({ + code: "browser.verification_failed", + message: `${platform} ${actionType} did not pass verification: ${expectation}.`, + context: { platform, actionType, expectation }, + }); +} diff --git a/src/main/browser/browser-action-executor.ts b/src/main/browser/browser-action-executor.ts index 8363c182..24914756 100644 --- a/src/main/browser/browser-action-executor.ts +++ b/src/main/browser/browser-action-executor.ts @@ -3,25 +3,86 @@ import { existsSync, writeFileSync } from "node:fs"; import type { NativeImage } from "electron"; import log from "electron-log/main.js"; import type { BrowserAction, BrowserActionResult } from "./browser-types"; -import { assertAllowedUrl, attachWindowSecurity, getComposeUrl, getFeedUrl, getSearchUrl } from "./browser-url-helpers"; +import { + assertAllowedUrl, + attachWindowSecurity, + getComposeUrl, + getFeedUrl, + getPlatformRules, + getSearchUrl, +} from "./browser-url-helpers"; +import { BrowserCdpClient } from "./browser-action-cdp"; +import { assertAuthenticatedPage, assertAuthenticatedSession } from "./browser-action-auth"; +import { getPlatformSelectors } from "./platform-selectors"; +import type { PlatformSelectors } from "./platform-selectors"; +import { browserVerificationFailed } from "./browser-action-errors"; +import { isAppError } from "../errors/index.js"; export { getPlatformRules, assertAllowedUrl, attachWindowSecurity } from "./browser-url-helpers"; +type BrowserActionOutcome = + | "success" + | "logged_out" + | "selector_drift" + | "network" + | "rate_limited" + | "cancelled" + | "verification_failed"; + +interface BrowserActionTelemetry { + platform: string; + action_type: string; + selector_hit: string | null; + selector_hits: string[]; + retries: number; + duration_ms: number; + final_outcome: BrowserActionOutcome; +} + export function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -export function waitForLoad(win: BrowserWindow, timeoutMs = 15000): Promise { - return new Promise((resolve) => { - if (win.webContents.isLoading()) { - const timer = setTimeout(resolve, timeoutMs); - win.webContents.once("did-finish-load", () => { - clearTimeout(timer); - resolve(); - }); - } else { +export function waitForLoad( + win: BrowserWindow, + timeoutMs = 15000, + context: { platform: string; actionType: string } = { platform: "browser", actionType: "load" }, +): Promise { + return new Promise((resolve, reject) => { + if (!win.webContents.isLoading()) { resolve(); + return; } + + const cleanup = () => { + clearTimeout(timer); + win.webContents.off("did-finish-load", onFinish); + win.webContents.off("did-fail-load", onFail); + win.off("closed", onClosed); + }; + const fail = (expectation: string) => { + cleanup(); + reject(browserVerificationFailed(context.platform, context.actionType, expectation)); + }; + const onFinish = () => { + cleanup(); + resolve(); + }; + const onFail = ( + _event: Electron.Event, + errorCode: number, + errorDescription: string, + _validatedUrl: string, + isMainFrame: boolean, + ) => { + if (isMainFrame) fail(`page load failed: ${errorDescription} (${errorCode})`); + }; + const onClosed = () => fail("window closed during page load"); + const timer = setTimeout(() => fail(`page load exceeded ${timeoutMs}ms`), timeoutMs); + + win.webContents.once("did-finish-load", onFinish); + win.webContents.on("did-fail-load", onFail); + win.once("closed", onClosed); }); } @@ -35,6 +96,8 @@ export async function runAction( const payload = action.payload; const screenshotPath = `${screenshotDir}/${action.id}.png`; const platform = action.platform; + const startedAt = Date.now(); + const selectorHits: string[] = []; log.info(`[browser] Executing ${action.actionType} for ${platform}/${action.socialAccountId}`); @@ -54,233 +117,106 @@ export async function runAction( attachWindowSecurity(win, platform); win.setMenuBarVisibility(false); + const cdp = new BrowserCdpClient(win.webContents, platform, action.actionType); + const selectors = getPlatformSelectors(platform); + const assertNotCancelled = () => { + if (cancelledActionIds.has(action.id)) { + throw new Error("BROWSER_ACTION_CANCELLED"); + } + }; + const remember = (selector: string | null) => { + if (selector) selectorHits.push(selector); + }; + try { + assertNotCancelled(); + await assertAuthenticatedSession(session, platform, action.actionType); let summary = "Action completed"; - const assertNotCancelled = () => { - if (cancelledActionIds.has(action.id)) { - throw new Error("BROWSER_ACTION_CANCELLED"); - } - }; switch (action.actionType) { case "navigate": { - assertNotCancelled(); - const url = String(payload.url || "https://google.com"); - assertAllowedUrl(platform, url, "navigate"); - await win.loadURL(url); - await waitForLoad(win); - assertNotCancelled(); - const title = await win.webContents.executeJavaScript("document.title"); + const url = String(payload.url || getFeedUrl(platform)); + await loadActionUrl(win, platform, action.actionType, url); + remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType)); + const title = await cdp.evaluate("document.title"); summary = `Navigated to ${title}`; break; } case "post": { + const content = requirePayloadText(payload.content, "content"); + await loadActionUrl(win, platform, action.actionType, getComposeUrl(platform)); + remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType)); + remember(await cdp.clickOptional(selectors.composerOpen, "composer_open")); + remember(await cdp.focusAndType(selectors.editor, content, "editor")); assertNotCancelled(); - const content = String(payload.content || ""); - const postUrl = getComposeUrl(platform); - await win.loadURL(postUrl); - await waitForLoad(win); - await delay(2000); - assertNotCancelled(); - - if (platform === "twitter") { - // Use clipboard-based approach since Twitter's React editor ignores execCommand - await win.webContents.executeJavaScript(` - (async () => { - const editor = document.querySelector('[data-testid="tweetTextarea_0"], [role="textbox"][contenteditable="true"]'); - if (!editor) return; - editor.focus(); - const dt = new DataTransfer(); - dt.setData('text/plain', ${JSON.stringify(content)}); - const pasteEvent = new ClipboardEvent('paste', { clipboardData: dt, bubbles: true, cancelable: true }); - editor.dispatchEvent(pasteEvent); - await new Promise(r => setTimeout(r, 500)); - if (!editor.textContent || editor.textContent.trim().length === 0) { - editor.textContent = ${JSON.stringify(content)}; - editor.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: ${JSON.stringify(content)} })); - } - })() - `); - await delay(1500); - await win.webContents.executeJavaScript(` - (() => { - const btn = document.querySelector('[data-testid="tweetButton"], [data-testid="tweetButtonInline"]'); - if (btn && !btn.disabled) btn.click(); - })() - `); - await delay(3000); - } else if (platform === "linkedin") { - await win.webContents.executeJavaScript(` - (() => { - const btn = document.querySelector('button.share-box-feed-entry__trigger, button.artdeco-button--muted'); - if (btn) btn.click(); - })() - `); - await delay(2000); - await win.webContents.executeJavaScript(` - (async () => { - const editor = document.querySelector('[role="textbox"][contenteditable="true"], .ql-editor[contenteditable="true"]'); - if (editor) { - editor.focus(); - const dt = new DataTransfer(); - dt.setData('text/plain', ${JSON.stringify(content)}); - const pasteEvent = new ClipboardEvent('paste', { clipboardData: dt, bubbles: true, cancelable: true }); - editor.dispatchEvent(pasteEvent); - await new Promise(r => setTimeout(r, 500)); - if (!editor.textContent || editor.textContent.trim().length === 0) { - editor.textContent = ${JSON.stringify(content)}; - editor.dispatchEvent(new InputEvent('input', { bubbles: true })); - } - } - })() - `); - await delay(1500); - await win.webContents.executeJavaScript(` - (() => { - const btn = document.querySelector('button.share-actions__primary-action, button.artdeco-button--primary[type="submit"]'); - if (btn && !btn.disabled) btn.click(); - })() - `); - await delay(3000); - } + remember(await cdp.clickFirst(selectors.submit, "submit")); + await cdp.waitForText(selectors.feedItems, content); summary = `Posted to ${platform}: "${content.slice(0, 50)}..."`; break; } case "reply": { + const targetUrl = requirePayloadText(payload.targetUrl, "targetUrl"); + const replyContent = requirePayloadText(payload.content, "content"); + await loadActionUrl(win, platform, action.actionType, targetUrl); + remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType)); + remember(await cdp.focusAndType(selectors.replyEditor, replyContent, "reply_editor")); assertNotCancelled(); - const targetUrl = String(payload.targetUrl || ""); - assertAllowedUrl(platform, targetUrl, "reply"); - const replyContent = String(payload.content || ""); - await win.loadURL(targetUrl); - await waitForLoad(win); - await delay(2000); - assertNotCancelled(); - - await win.webContents.executeJavaScript(` - (async () => { - const editor = document.querySelector('[data-testid="tweetTextarea_0"], [role="textbox"][contenteditable="true"]'); - if (editor) { - editor.focus(); - const dt = new DataTransfer(); - dt.setData('text/plain', ${JSON.stringify(replyContent)}); - const pasteEvent = new ClipboardEvent('paste', { clipboardData: dt, bubbles: true, cancelable: true }); - editor.dispatchEvent(pasteEvent); - await new Promise(r => setTimeout(r, 500)); - if (!editor.textContent || editor.textContent.trim().length === 0) { - editor.textContent = ${JSON.stringify(replyContent)}; - editor.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: ${JSON.stringify(replyContent)} })); - } - } - })() - `); - await delay(1500); - await win.webContents.executeJavaScript(` - (() => { - const btn = document.querySelector('[data-testid="tweetButton"], [data-testid="tweetButtonInline"], button[type="submit"]'); - if (btn && !btn.disabled) btn.click(); - })() - `); - await delay(3000); + remember(await cdp.clickFirst(selectors.replySubmit, "reply_submit")); + await cdp.waitForText(selectors.feedItems, replyContent); summary = `Replied to ${targetUrl.slice(0, 50)}`; break; } case "like": { - assertNotCancelled(); - const targetUrl = String(payload.targetUrl || ""); - assertAllowedUrl(platform, targetUrl, "like"); - await win.loadURL(targetUrl); - await waitForLoad(win); - await delay(2000); - assertNotCancelled(); - await win.webContents.executeJavaScript(` - (() => { - const btn = document.querySelector('[data-testid="like"], button[aria-label*="Like"]'); - if (btn) btn.click(); - })() - `); - await delay(2000); + const targetUrl = requirePayloadText(payload.targetUrl, "targetUrl"); + await loadActionUrl(win, platform, action.actionType, targetUrl); + remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType)); + remember(await cdp.clickFirst(selectors.likeButton, "like_button")); + await cdp.waitForFirst(selectors.likedButton, "liked_state", 5_000); summary = `Liked post at ${targetUrl.slice(0, 50)}`; break; } case "follow": { - assertNotCancelled(); const targetUrl = String(payload.targetUrl || payload.profileUrl || ""); - assertAllowedUrl(platform, targetUrl, "follow"); - await win.loadURL(targetUrl); - await waitForLoad(win); - await delay(2000); - assertNotCancelled(); - await win.webContents.executeJavaScript(` - (() => { - const btn = document.querySelector('[data-testid*="follow"], button[aria-label*="Follow"]'); - if (btn && !btn.textContent.includes('Following')) btn.click(); - })() - `); - await delay(2000); + if (!targetUrl) throw browserVerificationFailed(platform, action.actionType, "missing targetUrl/profileUrl"); + await loadActionUrl(win, platform, action.actionType, targetUrl); + remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType)); + remember(await cdp.clickFirst(selectors.followButton, "follow_button")); + await cdp.waitForFirst(selectors.followingButton, "following_state", 5_000); summary = `Followed user at ${targetUrl.slice(0, 50)}`; break; } case "browse_feed": { - assertNotCancelled(); const feedUrl = String(payload.url || getFeedUrl(platform)); - assertAllowedUrl(platform, feedUrl, "browse_feed"); const maxPosts = Number(payload.maxPosts || 10); - await win.loadURL(feedUrl); - await waitForLoad(win); - await delay(3000); - assertNotCancelled(); - - for (let i = 0; i < 3; i++) { - await win.webContents.executeJavaScript("window.scrollBy(0, window.innerHeight)"); - await delay(1500); + await loadActionUrl(win, platform, action.actionType, feedUrl); + remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType)); + for (let i = 0; i < 3; i += 1) { + await cdp.evaluate("window.scrollBy(0, window.innerHeight)"); } - - const posts: string[] = await win.webContents.executeJavaScript(` - (() => { - const articles = document.querySelectorAll('article, [data-testid="tweet"], .feed-shared-update-v2'); - return Array.from(articles).slice(0, ${maxPosts}).map(el => el.innerText.slice(0, 300)); - })() - `); - summary = `Browsed feed — found ${posts.length} posts`; + const posts = await cdp.waitForAnyText(selectors.feedItems); + summary = `Browsed feed — found ${Math.min(posts.length, maxPosts)} posts`; break; } case "search": { - assertNotCancelled(); - const query = String(payload.query || ""); - const searchUrl = getSearchUrl(platform, query); - assertAllowedUrl(platform, searchUrl, "search"); - await win.loadURL(searchUrl); - await waitForLoad(win); - await delay(3000); - assertNotCancelled(); - - const results: string[] = await win.webContents.executeJavaScript(` - (() => { - const items = document.querySelectorAll('article, [data-testid="tweet"], .search-result'); - return Array.from(items).slice(0, 10).map(el => el.innerText.slice(0, 200)); - })() - `); + const query = requirePayloadText(payload.query, "query"); + await loadActionUrl(win, platform, action.actionType, getSearchUrl(platform, query)); + remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType)); + const results = await cdp.waitForAnyText(selectors.searchItems); summary = `Search "${query.slice(0, 30)}" — ${results.length} results`; break; } case "screenshot": { - assertNotCancelled(); - const url = String(payload.url || "about:blank"); - if (url !== "about:blank") { - assertAllowedUrl(platform, url, "screenshot"); - await win.loadURL(url); - await waitForLoad(win); - await delay(2000); - } - assertNotCancelled(); - const title = await win.webContents.executeJavaScript("document.title"); + const url = String(payload.url || getFeedUrl(platform)); + await loadActionUrl(win, platform, action.actionType, url); + remember(await confirmAuthenticatedPage(win, cdp, selectors, platform, action.actionType)); + const title = await cdp.evaluate("document.title"); summary = `Screenshot captured: ${title}`; break; } @@ -289,18 +225,18 @@ export async function runAction( throw new Error(`Unsupported browser action type: ${action.actionType}`); } - try { - const image: NativeImage = await win.webContents.capturePage(); - if (!image.isEmpty()) { - writeFileSync(screenshotPath, image.toPNG()); - } - } catch (err) { - log.warn(`[browser] Screenshot failed: ${err}`); - } - - await delay(1000); - if (!win.isDestroyed()) win.close(); - + assertNotCancelled(); + await captureScreenshot(win, screenshotPath, action.id, "success"); + await closeWindow(win); + logActionTelemetry({ + platform, + action_type: action.actionType, + selector_hit: selectorHits.at(-1) ?? null, + selector_hits: selectorHits, + retries: cdp.getRetryCount(), + duration_ms: Date.now() - startedAt, + final_outcome: "success", + }); return { success: true, cancelled: false, @@ -309,22 +245,25 @@ export async function runAction( error: null, }; } catch (error) { - const wasCancelled = cancelledActionIds.has(action.id) || (error instanceof Error && error.message === "BROWSER_ACTION_CANCELLED"); - try { - const image: NativeImage = await win.webContents.capturePage(); - if (!image.isEmpty()) writeFileSync(screenshotPath, image.toPNG()); - } catch (screenshotError) { - log.warn( - `[browser] failure screenshot capture failed for action ${action.id}: ${screenshotError instanceof Error ? screenshotError.message : String(screenshotError)}`, - ); - } + const wasCancelled = cancelledActionIds.has(action.id) + || (error instanceof Error && error.message === "BROWSER_ACTION_CANCELLED"); + await captureScreenshot(win, screenshotPath, action.id, "failure"); + await closeWindow(win); - if (!win.isDestroyed()) win.close(); - - const msg = wasCancelled ? "Action cancelled." : error instanceof Error ? error.message : String(error); + const msg = wasCancelled ? "Action cancelled." : describeBrowserActionError(error); + const finalOutcome = classifyBrowserOutcome(error, wasCancelled); if (!wasCancelled) { log.error(`[browser] Action ${action.actionType} failed: ${msg}`); } + logActionTelemetry({ + platform, + action_type: action.actionType, + selector_hit: selectorHits.at(-1) ?? null, + selector_hits: selectorHits, + retries: cdp.getRetryCount(), + duration_ms: Date.now() - startedAt, + final_outcome: finalOutcome, + }); return { success: false, cancelled: wasCancelled, @@ -333,7 +272,83 @@ export async function runAction( error: msg, }; } finally { + await cdp.dispose().catch(() => undefined); activeActionWindows.delete(action.id); cancelledActionIds.delete(action.id); } } + +async function confirmAuthenticatedPage( + win: BrowserWindow, + cdp: BrowserCdpClient, + selectors: PlatformSelectors, + platform: string, + actionType: string, +): Promise { + await assertAuthenticatedPage(() => win.webContents.getURL(), platform, actionType); + return cdp.waitForAuthenticatedSurface( + selectors.authenticated, + getPlatformRules(platform).authenticatedCookieNames, + ); +} + +async function loadActionUrl( + win: BrowserWindow, + platform: string, + actionType: string, + url: string, +): Promise { + assertAllowedUrl(platform, url, actionType); + await win.loadURL(url); + await waitForLoad(win, 15000, { platform, actionType }); +} + +function requirePayloadText(value: unknown, key: string): string { + const text = typeof value === "string" ? value.trim() : ""; + if (text.length === 0) { + throw new Error(`Browser action payload missing ${key}.`); + } + return text; +} + +async function captureScreenshot( + win: BrowserWindow, + screenshotPath: string, + actionId: string, + phase: "success" | "failure", +): Promise { + try { + const image: NativeImage = await win.webContents.capturePage(); + if (!image.isEmpty()) { + writeFileSync(screenshotPath, image.toPNG()); + } + } catch (err) { + log.warn(`[browser] ${phase} screenshot failed for action ${actionId}: ${err instanceof Error ? err.message : String(err)}`); + } +} + +async function closeWindow(win: BrowserWindow): Promise { + if (win.isDestroyed()) return; + await delay(250); + if (!win.isDestroyed()) win.close(); +} + +function describeBrowserActionError(error: unknown): string { + if (isAppError(error)) return `[${error.code}] ${error.message}`; + return error instanceof Error ? error.message : String(error); +} + +function classifyBrowserOutcome(error: unknown, wasCancelled: boolean): BrowserActionOutcome { + if (wasCancelled) return "cancelled"; + if (isAppError(error)) { + if (error.code === "browser.not_logged_in") return "logged_out"; + if (error.code === "browser.selector_drift") return "selector_drift"; + if (error.code === "browser.verification_failed") return "verification_failed"; + } + if (error instanceof Error && /429|rate[_ -]?limit|too many requests/i.test(error.message)) return "rate_limited"; + return "network"; +} + +function logActionTelemetry(telemetry: BrowserActionTelemetry): void { + log.info("[browser] action telemetry", telemetry); +} diff --git a/src/main/browser/browser-types.ts b/src/main/browser/browser-types.ts index 2f620e15..233ab0a0 100644 --- a/src/main/browser/browser-types.ts +++ b/src/main/browser/browser-types.ts @@ -25,6 +25,7 @@ export interface PlatformRule { authCheckUrl: string; allowedHosts: string[]; cookieOrigins: string[]; + authenticatedCookieNames?: string[]; unauthenticatedUrlMarkers: string[]; } @@ -34,6 +35,7 @@ export const PLATFORM_RULES: Record = { authCheckUrl: "https://x.com/home", allowedHosts: ["x.com", "twitter.com"], cookieOrigins: ["https://x.com", "https://twitter.com"], + authenticatedCookieNames: ["auth_token"], unauthenticatedUrlMarkers: ["/i/flow/login", "/login"], }, linkedin: { @@ -41,6 +43,7 @@ export const PLATFORM_RULES: Record = { authCheckUrl: "https://www.linkedin.com/feed/", allowedHosts: ["linkedin.com"], cookieOrigins: ["https://www.linkedin.com"], + authenticatedCookieNames: ["li_at"], unauthenticatedUrlMarkers: ["/login", "/checkpoint"], }, instagram: { @@ -48,6 +51,7 @@ export const PLATFORM_RULES: Record = { authCheckUrl: "https://www.instagram.com/", allowedHosts: ["instagram.com"], cookieOrigins: ["https://www.instagram.com"], + authenticatedCookieNames: ["sessionid", "ds_user_id"], unauthenticatedUrlMarkers: ["/accounts/login"], }, facebook: { @@ -55,6 +59,7 @@ export const PLATFORM_RULES: Record = { authCheckUrl: "https://www.facebook.com/", allowedHosts: ["facebook.com"], cookieOrigins: ["https://www.facebook.com"], + authenticatedCookieNames: ["c_user", "xs"], unauthenticatedUrlMarkers: ["/login"], }, reddit: { @@ -62,6 +67,7 @@ export const PLATFORM_RULES: Record = { authCheckUrl: "https://www.reddit.com/", allowedHosts: ["reddit.com"], cookieOrigins: ["https://www.reddit.com"], + authenticatedCookieNames: ["reddit_session", "token_v2"], unauthenticatedUrlMarkers: ["/login"], }, youtube: { @@ -69,6 +75,7 @@ export const PLATFORM_RULES: Record = { authCheckUrl: "https://www.youtube.com/", allowedHosts: ["youtube.com", "google.com"], cookieOrigins: ["https://www.youtube.com", "https://accounts.google.com"], + authenticatedCookieNames: ["SAPISID", "LOGIN_INFO"], unauthenticatedUrlMarkers: ["/signin", "/ServiceLogin"], }, tiktok: { @@ -76,6 +83,7 @@ export const PLATFORM_RULES: Record = { authCheckUrl: "https://www.tiktok.com/foryou", allowedHosts: ["tiktok.com"], cookieOrigins: ["https://www.tiktok.com"], + authenticatedCookieNames: ["sessionid", "sessionid_ss"], unauthenticatedUrlMarkers: ["/login"], }, bluesky: { @@ -97,6 +105,7 @@ export const PLATFORM_RULES: Record = { authCheckUrl: "https://www.threads.net/", allowedHosts: ["threads.net"], cookieOrigins: ["https://www.threads.net"], + authenticatedCookieNames: ["sessionid", "ds_user_id"], unauthenticatedUrlMarkers: ["/login"], }, pinterest: { @@ -104,6 +113,7 @@ export const PLATFORM_RULES: Record = { authCheckUrl: "https://www.pinterest.com/", allowedHosts: ["pinterest.com"], cookieOrigins: ["https://www.pinterest.com"], + authenticatedCookieNames: ["_pinterest_sess"], unauthenticatedUrlMarkers: ["/login"], }, telegram: { @@ -132,6 +142,7 @@ export const PLATFORM_RULES: Record = { authCheckUrl: "https://weibo.com/", allowedHosts: ["weibo.com", "weibo.cn"], cookieOrigins: ["https://weibo.com"], + authenticatedCookieNames: ["SUB", "SUBP"], unauthenticatedUrlMarkers: ["/login", "/signup"], }, xiaohongshu: { diff --git a/src/main/browser/platform-selectors.ts b/src/main/browser/platform-selectors.ts new file mode 100644 index 00000000..4c5a4207 --- /dev/null +++ b/src/main/browser/platform-selectors.ts @@ -0,0 +1,135 @@ +export interface PlatformSelectors { + authenticated: readonly string[]; + composerOpen: readonly string[]; + editor: readonly string[]; + submit: readonly string[]; + replyEditor: readonly string[]; + replySubmit: readonly string[]; + likeButton: readonly string[]; + likedButton: readonly string[]; + followButton: readonly string[]; + followingButton: readonly string[]; + feedItems: readonly string[]; + searchItems: readonly string[]; +} + +const GENERIC_SELECTORS: PlatformSelectors = { + authenticated: [ + "[data-testid='AppTabBar_Profile_Link']", + "a[href*='/in/']", + "img[alt*='profile' i]", + "[aria-label*='profile' i]", + ], + composerOpen: [ + "button[aria-label*='post' i]", + "button[aria-label*='compose' i]", + "button.share-box-feed-entry__trigger", + ], + editor: [ + "[role='textbox'][contenteditable='true']", + "textarea", + ".ql-editor[contenteditable='true']", + ], + submit: [ + "button[type='submit']", + "button[aria-label*='post' i]", + "button[aria-label*='send' i]", + ], + replyEditor: [ + "[role='textbox'][contenteditable='true']", + "textarea", + ], + replySubmit: [ + "button[type='submit']", + "button[aria-label*='reply' i]", + "button[aria-label*='send' i]", + ], + likeButton: [ + "button[aria-label*='like' i]", + "[role='button'][aria-label*='like' i]", + ], + likedButton: [ + "button[aria-pressed='true'][aria-label*='like' i]", + "[data-testid='unlike']", + ], + followButton: [ + "button[aria-label*='follow' i]", + "[role='button'][aria-label*='follow' i]", + ], + followingButton: [ + "button[aria-label*='following' i]", + "button[aria-label*='requested' i]", + ], + feedItems: ["article", "[role='article']", ".feed-shared-update-v2"], + searchItems: ["article", "[role='article']", ".search-result", ".feed-shared-update-v2"], +}; + +const PLATFORM_SELECTORS: Record> = { + twitter: { + authenticated: [ + "[data-testid='AppTabBar_Profile_Link']", + "[data-testid='SideNav_AccountSwitcher_Button']", + "a[aria-label*='Profile']", + ], + editor: [ + "[data-testid='tweetTextarea_0']", + "[role='textbox'][contenteditable='true']", + ], + submit: [ + "[data-testid='tweetButton']", + "[data-testid='tweetButtonInline']", + ], + replyEditor: [ + "[data-testid='tweetTextarea_0']", + "[role='textbox'][contenteditable='true']", + ], + replySubmit: [ + "[data-testid='tweetButton']", + "[data-testid='tweetButtonInline']", + "button[type='submit']", + ], + likeButton: ["[data-testid='like']", "button[aria-label*='Like']"], + likedButton: ["[data-testid='unlike']", "button[aria-label*='Unlike']"], + feedItems: ["article", "[data-testid='tweet']"], + searchItems: ["article", "[data-testid='tweet']"], + }, + linkedin: { + authenticated: [ + "a[href*='/in/']", + "img.global-nav__me-photo", + "button.global-nav__primary-link-me-menu-trigger", + ], + composerOpen: [ + "button.share-box-feed-entry__trigger", + "button.artdeco-button--muted", + ], + editor: [ + "[role='textbox'][contenteditable='true']", + ".ql-editor[contenteditable='true']", + ], + submit: [ + "button.share-actions__primary-action", + "button.artdeco-button--primary[type='submit']", + ], + feedItems: [".feed-shared-update-v2", "article", ".update-components-text"], + searchItems: [".reusable-search__result-container", ".feed-shared-update-v2", "article"], + }, +}; + +export function getPlatformSelectors(platform: string): PlatformSelectors { + const overrides = PLATFORM_SELECTORS[platform] ?? {}; + return { + authenticated: overrides.authenticated ?? GENERIC_SELECTORS.authenticated, + composerOpen: overrides.composerOpen ?? GENERIC_SELECTORS.composerOpen, + editor: overrides.editor ?? GENERIC_SELECTORS.editor, + submit: overrides.submit ?? GENERIC_SELECTORS.submit, + replyEditor: overrides.replyEditor ?? overrides.editor ?? GENERIC_SELECTORS.replyEditor, + replySubmit: overrides.replySubmit ?? overrides.submit ?? GENERIC_SELECTORS.replySubmit, + likeButton: overrides.likeButton ?? GENERIC_SELECTORS.likeButton, + likedButton: overrides.likedButton ?? GENERIC_SELECTORS.likedButton, + followButton: overrides.followButton ?? GENERIC_SELECTORS.followButton, + followingButton: overrides.followingButton ?? GENERIC_SELECTORS.followingButton, + feedItems: overrides.feedItems ?? GENERIC_SELECTORS.feedItems, + searchItems: overrides.searchItems ?? overrides.feedItems ?? GENERIC_SELECTORS.searchItems, + }; +} diff --git a/tests/unit/browser-action-runtime.test.ts b/tests/unit/browser-action-runtime.test.ts new file mode 100644 index 00000000..c9904a08 --- /dev/null +++ b/tests/unit/browser-action-runtime.test.ts @@ -0,0 +1,65 @@ +import type { Session } from "electron"; +import { describe, expect, it } from "vitest"; +import { + assertAuthenticatedPage, + assertAuthenticatedSession, +} from "../../src/main/browser/browser-action-auth"; +import { getPlatformSelectors } from "../../src/main/browser/platform-selectors"; + +function fakeSession(cookiesByOrigin: Record): Pick { + return { + cookies: { + get: async ({ url }) => cookiesByOrigin[url ?? ""] ?? [], + }, + } as Pick; +} + +describe("browser action runtime guards", () => { + it("fails social actions before execution when the account session has no cookies", async () => { + await expect( + assertAuthenticatedSession(fakeSession({}), "twitter", "post"), + ).rejects.toMatchObject({ + code: "browser.not_logged_in", + }); + }); + + it("rejects guest cookies for platforms with explicit auth cookies", async () => { + await expect( + assertAuthenticatedSession( + fakeSession({ "https://x.com": [{ name: "guest_id", value: "present" }] }), + "twitter", + "post", + ), + ).rejects.toMatchObject({ + code: "browser.not_logged_in", + }); + }); + + it("accepts a persisted platform cookie before executing actions", async () => { + await expect( + assertAuthenticatedSession( + fakeSession({ "https://x.com": [{ name: "auth_token", value: "present" }] }), + "twitter", + "post", + ), + ).resolves.toBeUndefined(); + }); + + it("treats login redirects as an unauthenticated page state", async () => { + await expect( + assertAuthenticatedPage(() => "https://x.com/i/flow/login", "twitter", "post"), + ).rejects.toMatchObject({ + code: "browser.not_logged_in", + }); + }); + + it("keeps platform selectors versioned outside the executor", () => { + const twitter = getPlatformSelectors("twitter"); + const linkedin = getPlatformSelectors("linkedin"); + + expect(twitter.editor).toContain("[data-testid='tweetTextarea_0']"); + expect(twitter.submit).toContain("[data-testid='tweetButton']"); + expect(linkedin.composerOpen).toContain("button.share-box-feed-entry__trigger"); + expect(linkedin.editor).toContain(".ql-editor[contenteditable='true']"); + }); +});