From 7a9059fbea2d1402f76ef6f1ae31fa5cf2dc3777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Thu, 8 Oct 2026 20:09:03 -0400 Subject: [PATCH] Q124: the file scan covers the whole file and says what it did and did not check Co-Authored-By: Claude Opus 5.5 --- src/main/app/startup-recovery.ts | 3 +- src/main/services/files/file-object-store.ts | 97 +++++++++++++++----- src/renderer/src/i18n/zh.ts | 7 +- src/renderer/src/pages/FilesPage.tsx | 2 +- src/renderer/src/pages/files/FileVisuals.tsx | 17 ++++ src/renderer/src/pages/files/FilesGrid.tsx | 6 +- src/renderer/src/pages/files/FilesTable.tsx | 4 +- tests/unit/file-scan.test.ts | 43 +++++++++ tests/unit/workspace-file-versions.test.ts | 27 +++++- 9 files changed, 175 insertions(+), 31 deletions(-) create mode 100644 tests/unit/file-scan.test.ts diff --git a/src/main/app/startup-recovery.ts b/src/main/app/startup-recovery.ts index 3c0a1721..260a717f 100644 --- a/src/main/app/startup-recovery.ts +++ b/src/main/app/startup-recovery.ts @@ -1,6 +1,6 @@ import type { DatabaseSync } from "node:sqlite"; import { AuditService } from "../services/audit-service"; -import { preserveRecordedFiles } from "../services/files/file-object-store"; +import { preserveRecordedFiles, rescanLegacyFiles } from "../services/files/file-object-store"; const INTERRUPTED_MESSAGE = "Interrupted by application shutdown before completion."; @@ -16,6 +16,7 @@ const INTERRUPTED_MESSAGE = "Interrupted by application shutdown before completi */ export function recoverInterruptedWork(db: DatabaseSync): void { preserveRecordedFiles(db); + rescanLegacyFiles(db); const now = new Date().toISOString(); const audit = new AuditService(db); const runs = db.prepare("select id, org_id, task_id from task_runs where state = 'running'") diff --git a/src/main/services/files/file-object-store.ts b/src/main/services/files/file-object-store.ts index db64bab9..d281664c 100644 --- a/src/main/services/files/file-object-store.ts +++ b/src/main/services/files/file-object-store.ts @@ -19,14 +19,24 @@ interface TextScanPattern { block: boolean; } +const SCAN_WINDOW_BYTES = 1024 * 1024; +const SCAN_OVERLAP_BYTES = 4096; +const SCAN_CONTEXT_BYTES = 64; +/** Zip (office documents too), PDF, gzip, 7z, RAR, bzip2, xz, PNG, JPEG and GIF. */ +const CONTAINER_SIGNATURES = [ + [0x50, 0x4b, 0x03, 0x04], [0x50, 0x4b, 0x05, 0x06], [0x25, 0x50, 0x44, 0x46], [0x1f, 0x8b], [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c], + [0x52, 0x61, 0x72, 0x21], [0x42, 0x5a, 0x68], [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00], [0x89, 0x50, 0x4e, 0x47], [0xff, 0xd8, 0xff], [0x47, 0x49, 0x46, 0x38], +]; + +// Every quantifier is bounded, so no match is longer than the window overlap and none is cut in two. const TEXT_SCAN_PATTERNS: TextScanPattern[] = [ { label: "dlp:private-key", pattern: /-----BEGIN (?:RSA |DSA |EC |OPENSSH |PRIVATE )?PRIVATE KEY-----/i, block: true }, { label: "dlp:aws-access-key", pattern: /\bAKIA[0-9A-Z]{16}\b/, block: true }, - { label: "dlp:github-token", pattern: /\bgh[pousr]_[A-Za-z0-9_]{30,}\b/, block: true }, - { label: "dlp:provider-secret", pattern: /\bsk-[A-Za-z0-9]{32,}\b/, block: true }, - { label: "dlp:credential", pattern: /\b(?:api[_-]?key|secret|token|password)\s*[:=]\s*["']?[A-Za-z0-9_\-./+=]{20,}/i, block: true }, + { label: "dlp:github-token", pattern: /\bgh[pousr]_[A-Za-z0-9_]{30}/, block: true }, + { label: "dlp:provider-secret", pattern: /\bsk-[A-Za-z0-9]{32}/, block: true }, + { label: "dlp:credential", pattern: /\b(?:api[_-]?key|secret|token|password)[ \t]{0,16}[:=][ \t]{0,16}["']?[A-Za-z0-9_\-./+=]{20}/i, block: true }, { label: "dlp:ssn", pattern: /\b\d{3}-\d{2}-\d{4}\b/, block: true }, - { label: "dlp:email", pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i, block: false }, + { label: "dlp:email", pattern: /\b[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,24}\b/i, block: false }, ]; export function fileKind(path: string): string { @@ -40,23 +50,35 @@ export function fileKind(path: string): string { return ext; } +/** Pattern checks for secrets and the antivirus test string over the whole file; not an antivirus engine, and compressed contents are not opened. */ export function scanFileContent(name: string, bytes: Buffer): FileScanResult { - const textSample = sampleText(bytes); const labels = new Set(scanLabels(name, bytes.length)); let blocked = false; - if (bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) { - labels.add("av:eicar"); - blocked = true; - } - for (const scan of TEXT_SCAN_PATTERNS) { - if (!scan.pattern.test(textSample)) continue; - labels.add(scan.label); - if (scan.block) blocked = true; - } - if (containsPaymentCard(textSample)) { - labels.add("dlp:payment-card"); - blocked = true; + // Overlapping windows cover every byte with bounded memory; the overlap is longer than any pattern match. + for (let start = 0; start === 0 || start < bytes.length; start += SCAN_WINDOW_BYTES) { + // A window owns the matches that start in its own MiB; context on both sides keeps word boundaries real at the cuts. + const ownEnd = Math.min(bytes.length, start + SCAN_WINDOW_BYTES); + const lead = bytes.subarray(Math.max(0, start - SCAN_CONTEXT_BYTES), start).toString("utf8"); + const own = bytes.subarray(start, ownEnd).toString("utf8"); + const tail = bytes.subarray(ownEnd, Math.min(bytes.length, ownEnd + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES)).toString("utf8"); + const text = lead + own + tail; + const range = { from: lead.length, until: lead.length + own.length }; + if (bytes.subarray(start, ownEnd + SCAN_OVERLAP_BYTES).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) { + labels.add("av:eicar"); + blocked = true; + } + for (const scan of TEXT_SCAN_PATTERNS) { + if (!matchesWithin(scan.pattern, text, range)) continue; + labels.add(scan.label); + if (scan.block) blocked = true; + } + if (containsPaymentCard(text, range)) { + labels.add("dlp:payment-card"); + blocked = true; + } } + if (isEncodedContainer(bytes)) labels.add("scan:contents-not-inspected"); + labels.add("scan:whole-file"); labels.add(blocked ? "classification:restricted" : labels.has("dlp:email") ? "classification:confidential" : "classification:internal"); return { state: blocked ? FILE_SCAN_STATE.BLOCKED : FILE_SCAN_STATE.APPROVED, @@ -209,6 +231,22 @@ function keepCopy(workspaceRoot: string, source: string, sha256: string, bytes: return copy; } +/** Files scanned before whole-file scanning are scanned again when their bytes are still the recorded ones; the rest are marked as sampled. */ +export function rescanLegacyFiles(db: DatabaseSync): void { + const rows = db.prepare("select id, name, sha256, storage_path, scan_labels_json from file_objects where scan_labels_json not like '%scan:whole-file%'") + .all() as Array<{ id: string; name: string; sha256: string; storage_path: string; scan_labels_json: string }>; + for (const row of rows) { + const bytes = existsSync(row.storage_path) ? readFileSync(row.storage_path) : null; + if (bytes && createHash("sha256").update(bytes).digest("hex") === row.sha256) { + const scan = scanFileContent(row.name, bytes); + db.prepare("update file_objects set scan_state = ?, scan_labels_json = ?, scanned_at = ? where id = ?").run(scan.state, JSON.stringify(scan.labels), scan.scannedAt, row.id); + continue; + } + const labels = [...new Set([...(JSON.parse(row.scan_labels_json) as string[]), "scan:legacy-sample"])].sort(); + db.prepare("update file_objects set scan_labels_json = ? where id = ?").run(JSON.stringify(labels), row.id); + } +} + /** Ids recorded before copies existed still point at the live file; those whose bytes are unchanged get their copy now. */ export function preserveRecordedFiles(db: DatabaseSync): void { const workspaces = db.prepare("select org_id, local_path from workspaces").all() as Array<{ org_id: string; local_path: string }>; @@ -283,13 +321,28 @@ function scanLabels(name: string, sizeBytes: number): string[] { return [...labels].sort(); } -function sampleText(bytes: Buffer): string { - return bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString("utf8"); +/** Archives, office documents, PDFs and images keep their contents compressed or encoded, so a byte scan cannot see inside them. */ +function isEncodedContainer(bytes: Buffer): boolean { + const head = bytes.subarray(0, 12); + const starts = (signature: number[]): boolean => signature.every((byte, index) => head[index] === byte); + return CONTAINER_SIGNATURES.some(starts) || (starts([0x52, 0x49, 0x46, 0x46]) && head.subarray(8, 12).toString("latin1") === "WEBP"); } -function containsPaymentCard(text: string): boolean { - const matches = text.match(/\b(?:\d[ -]?){13,19}\b/g) ?? []; - return matches.some((match) => luhnValid(match.replace(/[ -]/g, ""))); +/** Whether a match starts in the range; text around it still counts as context for boundaries. */ +function matchesWithin(pattern: RegExp, text: string, range: { from: number; until: number }): boolean { + const search = new RegExp(pattern.source, `${pattern.flags}g`); + search.lastIndex = range.from; + const match = search.exec(text); + return match !== null && match.index < range.until; +} + +function containsPaymentCard(text: string, range: { from: number; until: number }): boolean { + const search = /\b(?:\d[ -]?){13,19}\b/g; + search.lastIndex = range.from; + for (let match = search.exec(text); match && match.index < range.until; match = search.exec(text)) { + if (luhnValid(match[0].replace(/[ -]/g, ""))) return true; + } + return false; } function luhnValid(value: string): boolean { diff --git a/src/renderer/src/i18n/zh.ts b/src/renderer/src/i18n/zh.ts index 36e8b4a4..03bb4d4d 100644 --- a/src/renderer/src/i18n/zh.ts +++ b/src/renderer/src/i18n/zh.ts @@ -171,7 +171,12 @@ export const zh: Record = { "Catalog": "目录", "Check": "检查", "Choose": "选择", - "Clean": "正常", + "No findings": "未发现问题", + "Partly scanned": "部分扫描", + "Scanned before whole-file checks existed: only its first MiB was checked.": "在全文件检查之前扫描：只检查了前 1 MiB。", + "Contents not inspected": "内容未检查", + "Checked for secrets and the antivirus test string across the whole file; no antivirus engine ran.": "已在整个文件中检查密钥和杀毒测试串；未运行杀毒引擎。", + "Its compressed or encoded contents were not opened.": "其压缩或编码的内容未被打开检查。", "Close": "关闭", "Close agent details": "关闭智能体详情", "Cloud": "云端", diff --git a/src/renderer/src/pages/FilesPage.tsx b/src/renderer/src/pages/FilesPage.tsx index cb7c5ed7..c7896e1c 100644 --- a/src/renderer/src/pages/FilesPage.tsx +++ b/src/renderer/src/pages/FilesPage.tsx @@ -143,7 +143,7 @@ function FilesFilter({ }) { const options: ReadonlyArray<{ id: FileScanState | "all"; label: string }> = [ { id: "all", label: t("All files") }, - { id: FILE_SCAN_STATE.APPROVED, label: t("Clean") }, + { id: FILE_SCAN_STATE.APPROVED, label: t("No findings") }, { id: FILE_SCAN_STATE.BLOCKED, label: t("Blocked") }, ]; return ( diff --git a/src/renderer/src/pages/files/FileVisuals.tsx b/src/renderer/src/pages/files/FileVisuals.tsx index 5f0cfb9b..2301f03b 100644 --- a/src/renderer/src/pages/files/FileVisuals.tsx +++ b/src/renderer/src/pages/files/FileVisuals.tsx @@ -1,6 +1,7 @@ import { Icon, Pill } from "../../shared/ui"; import type { FileObjectRecord } from "../../../../shared/domain"; import { FILE_COLOR_MAP, parseScanLabels } from "./file-helpers"; +import { t } from "../../i18n/i18n"; export function FileIcon({ kind, big = false }: { kind: string; big?: boolean }) { const map = FILE_COLOR_MAP[kind] ?? FILE_COLOR_MAP.doc; @@ -45,3 +46,19 @@ export function ScanLabelStrip({ file }: { file: FileObjectRecord }) { ); } + +/** What the scan found, worded for what it checked: secret patterns and the antivirus test string, never a full antivirus pass. */ +export function ScanPill({ file }: { file: FileObjectRecord }) { + if (file.scanState !== "approved") return {t("Blocked")}; + const labels = parseScanLabels(file); + if (labels.includes("scan:legacy-sample")) { + return {t("Partly scanned")}; + } + const sealed = labels.includes("scan:contents-not-inspected"); + const scope = t("Checked for secrets and the antivirus test string across the whole file; no antivirus engine ran."); + return ( + + {sealed ? t("Contents not inspected") : t("No findings")} + + ); +} diff --git a/src/renderer/src/pages/files/FilesGrid.tsx b/src/renderer/src/pages/files/FilesGrid.tsx index 1d0bf5bc..a95a79c5 100644 --- a/src/renderer/src/pages/files/FilesGrid.tsx +++ b/src/renderer/src/pages/files/FilesGrid.tsx @@ -1,4 +1,4 @@ -import { Avatar, Icon, Pill } from "../../shared/ui"; +import { Avatar, Icon } from "../../shared/ui"; import type { AgentRecord, EmployeeRecord, @@ -6,7 +6,7 @@ import type { FileObjectRecord, } from "../../../../shared/domain"; import { formatRelative } from "../types"; -import { FileIcon, ScanLabelStrip } from "./FileVisuals"; +import { FileIcon, ScanLabelStrip, ScanPill } from "./FileVisuals"; import { fileGrantCount, isAgentOwner, resolveOwnerName, shortChecksum, shortObjectRef } from "./file-helpers"; import { t } from "../../i18n/i18n"; @@ -92,7 +92,7 @@ export function PinnedCard({ sha:{shortChecksum(file.sha256)} · obj:{shortObjectRef(file.storagePath)}
- {file.scanState === "approved" ? t("Clean") : t("Blocked")} + {grantCount}
diff --git a/src/renderer/src/pages/files/FilesTable.tsx b/src/renderer/src/pages/files/FilesTable.tsx index e3fe9c78..ef5666ac 100644 --- a/src/renderer/src/pages/files/FilesTable.tsx +++ b/src/renderer/src/pages/files/FilesTable.tsx @@ -7,7 +7,7 @@ import type { FileObjectRecord, } from "../../../../shared/domain"; import { formatBytes, formatRelative } from "../types"; -import { FileIcon } from "./FileVisuals"; +import { FileIcon, ScanPill } from "./FileVisuals"; import { fileGrantCount, isAgentOwner, resolveOwnerName, shortChecksum, shortObjectRef } from "./file-helpers"; import { t } from "../../i18n/i18n"; @@ -89,7 +89,7 @@ function FileRow({ {owner}
- {canOpen ? t("Clean") : t("Blocked")} + {grantCount}
{formatRelative(file.updatedAt ?? file.createdAt)}
diff --git a/tests/unit/file-scan.test.ts b/tests/unit/file-scan.test.ts new file mode 100644 index 00000000..5d75e0b9 --- /dev/null +++ b/tests/unit/file-scan.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "vitest"; +import { scanFileContent } from "../../src/main/services/files/file-object-store"; + +const MIB = 1024 * 1024; + +describe("the file scan", () => { + test("checks the whole file, not just its first MiB, including a secret that straddles a window edge", () => { + const padding = "x".repeat(3 * MIB); + expect(scanFileContent("notes.txt", Buffer.from(`${padding}\nAKIAABCDEFGHIJKLMNOP\n`)).state).toBe("blocked"); + const straddling = Buffer.concat([Buffer.alloc(MIB - 10, "y"), Buffer.from("-----BEGIN RSA PRIVATE KEY-----")]); + expect(scanFileContent("key.txt", straddling).labels).toContain("dlp:private-key"); + expect(scanFileContent("eicar.txt", Buffer.from(`${padding}X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*`)).labels).toContain("av:eicar"); + }); + + test("no pattern match can stretch past the window overlap", () => { + const edge = Buffer.concat([Buffer.alloc(MIB - 20, "z"), Buffer.from(`password:${"\n".repeat(5000)}${"A".repeat(32)}`)]); + // Whitespace between a key name and its value is bounded; a value 5,000 lines away is not that key's value. + expect(scanFileContent("far.txt", edge).labels).not.toContain("dlp:credential"); + const long = Buffer.concat([Buffer.alloc(MIB - 10, "z"), Buffer.from(` ghp_${"a".repeat(9000)} `)]); + expect(scanFileContent("token.txt", long).labels).toContain("dlp:github-token"); + expect(scanFileContent("plan.md", Buffer.from("# Plan")).labels).toContain("scan:whole-file"); + }); + + test("a window cut does not invent a word boundary", () => { + const inWord = Buffer.concat([Buffer.alloc(MIB - 4, "z"), Buffer.from("NOT_AKIAABCDEFGHIJKLMNOP\n")]); + expect(scanFileContent("a.txt", inWord).state).toBe("approved"); + const alone = Buffer.concat([Buffer.alloc(MIB - 1, "z"), Buffer.from(" AKIAABCDEFGHIJKLMNOP\n")]); + expect(scanFileContent("b.txt", alone).state).toBe("blocked"); + // The far edge too: a longer word whose first 20 characters end exactly at a window's last byte is not a key. + const atTail = Buffer.concat([Buffer.alloc(MIB + 4096 - 20, "z"), Buffer.from(" AKIAABCDEFGHIJKLMNOPZ\n")]); + expect(scanFileContent("c.txt", atTail).state).toBe("approved"); + const atOwnEnd = Buffer.concat([Buffer.alloc(MIB - 20, "z"), Buffer.from(" AKIAABCDEFGHIJKLMNOPZ\n")]); + expect(scanFileContent("d.txt", atOwnEnd).state).toBe("approved"); + }); + + test("a container whose contents are compressed or encoded is labelled as not inspected inside", () => { + expect(scanFileContent("report.docx", Buffer.from("PK\u0003\u0004rest")).labels).toContain("scan:contents-not-inspected"); + expect(scanFileContent("deck.pdf", Buffer.from("%PDF-1.7 ...")).labels).toContain("scan:contents-not-inspected"); + expect(scanFileContent("archive.gz", Buffer.from([0x1f, 0x8b, 0x08, 0x00])).labels).toContain("scan:contents-not-inspected"); + expect(scanFileContent("photo.png", Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a])).labels).toContain("scan:contents-not-inspected"); + expect(scanFileContent("plan.md", Buffer.from("# Plan")).labels).not.toContain("scan:contents-not-inspected"); + }); +}); diff --git a/tests/unit/workspace-file-versions.test.ts b/tests/unit/workspace-file-versions.test.ts index 122620b2..a8fb0f1f 100644 --- a/tests/unit/workspace-file-versions.test.ts +++ b/tests/unit/workspace-file-versions.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path"; import { describe, expect, test } from "vitest"; import { DatabaseConnection } from "../../src/main/database/connection"; import { OrganizationService } from "../../src/main/services/organization-service"; -import { preserveRecordedFiles, recordWorkspaceFile } from "../../src/main/services/files/file-object-store"; +import { preserveRecordedFiles, recordWorkspaceFile, rescanLegacyFiles } from "../../src/main/services/files/file-object-store"; import { ChatService } from "../../src/main/services/chat-service"; import { readAttachmentTool } from "../../src/main/services/agent-attachment-tools"; @@ -81,3 +81,28 @@ describe("opening a recorded file", () => { connection.close(); }); }); + +describe("files scanned before whole-file scanning", () => { + test("are scanned again when their bytes are intact, and marked as sampled when they are not", () => { + const connection = new DatabaseConnection(join(tmpdir(), `ac-rescan-${randomUUID()}.db`)); + let orgId = ""; + connection.transaction(() => { + orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" }); + }); + const path = join(tmpdir(), `ac-rescan-${randomUUID()}.txt`); + const content = `${"x".repeat(2 * 1024 * 1024)}\nAKIAABCDEFGHIJKLMNOP\n`; + writeFileSync(path, content); + const insert = connection.db.prepare(` + insert into file_objects(id, org_id, name, kind, size_bytes, sha256, storage_path, source_path, owner_type, owner_id, pinned, scan_state, scan_labels_json, created_at) + values (?, ?, 'notes.txt', 'txt', 1, ?, ?, ?, 'agent', null, 0, 'approved', '["classification:internal"]', ?) + `); + insert.run("fil_intact", orgId, createHash("sha256").update(content).digest("hex"), path, path, new Date().toISOString()); + insert.run("fil_gone", orgId, "0".repeat(64), join(tmpdir(), "missing.txt"), join(tmpdir(), "missing.txt"), new Date().toISOString()); + rescanLegacyFiles(connection.db); + const row = (id: string) => connection.db.prepare("select scan_state, scan_labels_json from file_objects where id = ?").get(id) as { scan_state: string; scan_labels_json: string }; + expect(row("fil_intact").scan_state).toBe("blocked"); + expect(JSON.parse(row("fil_intact").scan_labels_json)).toContain("scan:whole-file"); + expect(row("fil_gone")).toEqual({ scan_state: "approved", scan_labels_json: '["classification:internal","scan:legacy-sample"]' }); + connection.close(); + }); +});