Q124: the file scan covers the whole file and says what it did and did not check

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main/app/startup-recovery.ts

@@ -1,6 +1,6 @@
 import type { DatabaseSync } from "node:sqlite";
 import { AuditService } from "../services/audit-service";
-import { preserveRecordedFiles } from "../services/files/file-object-store";
+import { preserveRecordedFiles, rescanLegacyFiles } from "../services/files/file-object-store";
 
 const INTERRUPTED_MESSAGE = "Interrupted by application shutdown before completion.";
 
@@ -16,6 +16,7 @@ const INTERRUPTED_MESSAGE = "Interrupted by application shutdown before completi
  */
 export function recoverInterruptedWork(db: DatabaseSync): void {
   preserveRecordedFiles(db);
+  rescanLegacyFiles(db);
   const now = new Date().toISOString();
   const audit = new AuditService(db);
   const runs = db.prepare("select id, org_id, task_id from task_runs where state = 'running'")

src/main/services/files/file-object-store.ts

@@ -19,14 +19,24 @@ interface TextScanPattern {
   block: boolean;
 }
 
+const SCAN_WINDOW_BYTES = 1024 * 1024;
+const SCAN_OVERLAP_BYTES = 4096;
+const SCAN_CONTEXT_BYTES = 64;
+/** Zip (office documents too), PDF, gzip, 7z, RAR, bzip2, xz, PNG, JPEG and GIF. */
+const CONTAINER_SIGNATURES = [
+  [0x50, 0x4b, 0x03, 0x04], [0x50, 0x4b, 0x05, 0x06], [0x25, 0x50, 0x44, 0x46], [0x1f, 0x8b], [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c],
+  [0x52, 0x61, 0x72, 0x21], [0x42, 0x5a, 0x68], [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00], [0x89, 0x50, 0x4e, 0x47], [0xff, 0xd8, 0xff], [0x47, 0x49, 0x46, 0x38],
+];
+
+// Every quantifier is bounded, so no match is longer than the window overlap and none is cut in two.
 const TEXT_SCAN_PATTERNS: TextScanPattern[] = [
   { label: "dlp:private-key", pattern: /-----BEGIN (?:RSA |DSA |EC |OPENSSH |PRIVATE )?PRIVATE KEY-----/i, block: true },
   { label: "dlp:aws-access-key", pattern: /\bAKIA[0-9A-Z]{16}\b/, block: true },
-  { label: "dlp:github-token", pattern: /\bgh[pousr]_[A-Za-z0-9_]{30,}\b/, block: true },
-  { label: "dlp:provider-secret", pattern: /\bsk-[A-Za-z0-9]{32,}\b/, block: true },
-  { label: "dlp:credential", pattern: /\b(?:api[_-]?key|secret|token|password)\s*[:=]\s*["']?[A-Za-z0-9_\-./+=]{20,}/i, block: true },
+  { label: "dlp:github-token", pattern: /\bgh[pousr]_[A-Za-z0-9_]{30}/, block: true },
+  { label: "dlp:provider-secret", pattern: /\bsk-[A-Za-z0-9]{32}/, block: true },
+  { label: "dlp:credential", pattern: /\b(?:api[_-]?key|secret|token|password)[ \t]{0,16}[:=][ \t]{0,16}["']?[A-Za-z0-9_\-./+=]{20}/i, block: true },
   { label: "dlp:ssn", pattern: /\b\d{3}-\d{2}-\d{4}\b/, block: true },
-  { label: "dlp:email", pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i, block: false },
+  { label: "dlp:email", pattern: /\b[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,24}\b/i, block: false },
 ];
 
 export function fileKind(path: string): string {
@@ -40,23 +50,35 @@ export function fileKind(path: string): string {
   return ext;
 }
 
+/** Pattern checks for secrets and the antivirus test string over the whole file; not an antivirus engine, and compressed contents are not opened. */
 export function scanFileContent(name: string, bytes: Buffer): FileScanResult {
-  const textSample = sampleText(bytes);
   const labels = new Set(scanLabels(name, bytes.length));
   let blocked = false;
-  if (bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) {
-    labels.add("av:eicar");
-    blocked = true;
-  }
-  for (const scan of TEXT_SCAN_PATTERNS) {
-    if (!scan.pattern.test(textSample)) continue;
-    labels.add(scan.label);
-    if (scan.block) blocked = true;
-  }
-  if (containsPaymentCard(textSample)) {
-    labels.add("dlp:payment-card");
-    blocked = true;
+  // Overlapping windows cover every byte with bounded memory; the overlap is longer than any pattern match.
+  for (let start = 0; start === 0 || start < bytes.length; start += SCAN_WINDOW_BYTES) {
+    // A window owns the matches that start in its own MiB; context on both sides keeps word boundaries real at the cuts.
+    const ownEnd = Math.min(bytes.length, start + SCAN_WINDOW_BYTES);
+    const lead = bytes.subarray(Math.max(0, start - SCAN_CONTEXT_BYTES), start).toString("utf8");
+    const own = bytes.subarray(start, ownEnd).toString("utf8");
+    const tail = bytes.subarray(ownEnd, Math.min(bytes.length, ownEnd + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES)).toString("utf8");
+    const text = lead + own + tail;
+    const range = { from: lead.length, until: lead.length + own.length };
+    if (bytes.subarray(start, ownEnd + SCAN_OVERLAP_BYTES).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) {
+      labels.add("av:eicar");
+      blocked = true;
+    }
+    for (const scan of TEXT_SCAN_PATTERNS) {
+      if (!matchesWithin(scan.pattern, text, range)) continue;
+      labels.add(scan.label);
+      if (scan.block) blocked = true;
+    }
+    if (containsPaymentCard(text, range)) {
+      labels.add("dlp:payment-card");
+      blocked = true;
+    }
   }
+  if (isEncodedContainer(bytes)) labels.add("scan:contents-not-inspected");
+  labels.add("scan:whole-file");
   labels.add(blocked ? "classification:restricted" : labels.has("dlp:email") ? "classification:confidential" : "classification:internal");
   return {
     state: blocked ? FILE_SCAN_STATE.BLOCKED : FILE_SCAN_STATE.APPROVED,
@@ -209,6 +231,22 @@ function keepCopy(workspaceRoot: string, source: string, sha256: string, bytes:
   return copy;
 }
 
+/** Files scanned before whole-file scanning are scanned again when their bytes are still the recorded ones; the rest are marked as sampled. */
+export function rescanLegacyFiles(db: DatabaseSync): void {
+  const rows = db.prepare("select id, name, sha256, storage_path, scan_labels_json from file_objects where scan_labels_json not like '%scan:whole-file%'")
+    .all() as Array<{ id: string; name: string; sha256: string; storage_path: string; scan_labels_json: string }>;
+  for (const row of rows) {
+    const bytes = existsSync(row.storage_path) ? readFileSync(row.storage_path) : null;
+    if (bytes && createHash("sha256").update(bytes).digest("hex") === row.sha256) {
+      const scan = scanFileContent(row.name, bytes);
+      db.prepare("update file_objects set scan_state = ?, scan_labels_json = ?, scanned_at = ? where id = ?").run(scan.state, JSON.stringify(scan.labels), scan.scannedAt, row.id);
+      continue;
+    }
+    const labels = [...new Set([...(JSON.parse(row.scan_labels_json) as string[]), "scan:legacy-sample"])].sort();
+    db.prepare("update file_objects set scan_labels_json = ? where id = ?").run(JSON.stringify(labels), row.id);
+  }
+}
+
 /** Ids recorded before copies existed still point at the live file; those whose bytes are unchanged get their copy now. */
 export function preserveRecordedFiles(db: DatabaseSync): void {
   const workspaces = db.prepare("select org_id, local_path from workspaces").all() as Array<{ org_id: string; local_path: string }>;
@@ -283,13 +321,28 @@ function scanLabels(name: string, sizeBytes: number): string[] {
   return [...labels].sort();
 }
 
-function sampleText(bytes: Buffer): string {
-  return bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString("utf8");
+/** Archives, office documents, PDFs and images keep their contents compressed or encoded, so a byte scan cannot see inside them. */
+function isEncodedContainer(bytes: Buffer): boolean {
+  const head = bytes.subarray(0, 12);
+  const starts = (signature: number[]): boolean => signature.every((byte, index) => head[index] === byte);
+  return CONTAINER_SIGNATURES.some(starts) || (starts([0x52, 0x49, 0x46, 0x46]) && head.subarray(8, 12).toString("latin1") === "WEBP");
 }
 
-function containsPaymentCard(text: string): boolean {
-  const matches = text.match(/\b(?:\d[ -]?){13,19}\b/g) ?? [];
-  return matches.some((match) => luhnValid(match.replace(/[ -]/g, "")));
+/** Whether a match starts in the range; text around it still counts as context for boundaries. */
+function matchesWithin(pattern: RegExp, text: string, range: { from: number; until: number }): boolean {
+  const search = new RegExp(pattern.source, `${pattern.flags}g`);
+  search.lastIndex = range.from;
+  const match = search.exec(text);
+  return match !== null && match.index < range.until;
+}
+
+function containsPaymentCard(text: string, range: { from: number; until: number }): boolean {
+  const search = /\b(?:\d[ -]?){13,19}\b/g;
+  search.lastIndex = range.from;
+  for (let match = search.exec(text); match && match.index < range.until; match = search.exec(text)) {
+    if (luhnValid(match[0].replace(/[ -]/g, ""))) return true;
+  }
+  return false;
 }
 
 function luhnValid(value: string): boolean {

src/renderer/src/i18n/zh.ts

@@ -171,7 +171,12 @@ export const zh: Record<string, string> = {
   "Catalog": "目录",
   "Check": "检查",
   "Choose": "选择",
-  "Clean": "正常",
+  "No findings": "未发现问题",
+  "Partly scanned": "部分扫描",
+  "Scanned before whole-file checks existed: only its first MiB was checked.": "在全文件检查之前扫描：只检查了前 1 MiB。",
+  "Contents not inspected": "内容未检查",
+  "Checked for secrets and the antivirus test string across the whole file; no antivirus engine ran.": "已在整个文件中检查密钥和杀毒测试串；未运行杀毒引擎。",
+  "Its compressed or encoded contents were not opened.": "其压缩或编码的内容未被打开检查。",
   "Close": "关闭",
   "Close agent details": "关闭智能体详情",
   "Cloud": "云端",

src/renderer/src/pages/FilesPage.tsx

@@ -143,7 +143,7 @@ function FilesFilter({
 }) {
   const options: ReadonlyArray<{ id: FileScanState | "all"; label: string }> = [
     { id: "all", label: t("All files") },
-    { id: FILE_SCAN_STATE.APPROVED, label: t("Clean") },
+    { id: FILE_SCAN_STATE.APPROVED, label: t("No findings") },
     { id: FILE_SCAN_STATE.BLOCKED, label: t("Blocked") },
   ];
   return (

src/renderer/src/pages/files/FileVisuals.tsx

@@ -1,6 +1,7 @@
 import { Icon, Pill } from "../../shared/ui";
 import type { FileObjectRecord } from "../../../../shared/domain";
 import { FILE_COLOR_MAP, parseScanLabels } from "./file-helpers";
+import { t } from "../../i18n/i18n";
 
 export function FileIcon({ kind, big = false }: { kind: string; big?: boolean }) {
   const map = FILE_COLOR_MAP[kind] ?? FILE_COLOR_MAP.doc;
@@ -45,3 +46,19 @@ export function ScanLabelStrip({ file }: { file: FileObjectRecord }) {
     </div>
   );
 }
+
+/** What the scan found, worded for what it checked: secret patterns and the antivirus test string, never a full antivirus pass. */
+export function ScanPill({ file }: { file: FileObjectRecord }) {
+  if (file.scanState !== "approved") return <Pill tone="red">{t("Blocked")}</Pill>;
+  const labels = parseScanLabels(file);
+  if (labels.includes("scan:legacy-sample")) {
+    return <span title={t("Scanned before whole-file checks existed: only its first MiB was checked.")}><Pill tone="amber">{t("Partly scanned")}</Pill></span>;
+  }
+  const sealed = labels.includes("scan:contents-not-inspected");
+  const scope = t("Checked for secrets and the antivirus test string across the whole file; no antivirus engine ran.");
+  return (
+    <span title={sealed ? `${scope} ${t("Its compressed or encoded contents were not opened.")}` : scope}>
+      <Pill tone={sealed ? "amber" : "green"}>{sealed ? t("Contents not inspected") : t("No findings")}</Pill>
+    </span>
+  );
+}

src/renderer/src/pages/files/FilesGrid.tsx

@@ -1,4 +1,4 @@
-import { Avatar, Icon, Pill } from "../../shared/ui";
+import { Avatar, Icon } from "../../shared/ui";
 import type {
   AgentRecord,
   EmployeeRecord,
@@ -6,7 +6,7 @@ import type {
   FileObjectRecord,
 } from "../../../../shared/domain";
 import { formatRelative } from "../types";
-import { FileIcon, ScanLabelStrip } from "./FileVisuals";
+import { FileIcon, ScanLabelStrip, ScanPill } from "./FileVisuals";
 import { fileGrantCount, isAgentOwner, resolveOwnerName, shortChecksum, shortObjectRef } from "./file-helpers";
 import { t } from "../../i18n/i18n";
 
@@ -92,7 +92,7 @@ export function PinnedCard({
         sha:{shortChecksum(file.sha256)} · obj:{shortObjectRef(file.storagePath)}
       </div>
       <div className="row" style={{ gap: 6 }}>
-        <Pill tone={file.scanState === "approved" ? "green" : "red"}>{file.scanState === "approved" ? t("Clean") : t("Blocked")}</Pill>
+        <ScanPill file={file} />
         <span className="mono" style={{ color: "var(--text-3)", fontSize: 12 }}>{grantCount}</span>
       </div>
       <ScanLabelStrip file={file} />

src/renderer/src/pages/files/FilesTable.tsx

@@ -7,7 +7,7 @@ import type {
   FileObjectRecord,
 } from "../../../../shared/domain";
 import { formatBytes, formatRelative } from "../types";
-import { FileIcon } from "./FileVisuals";
+import { FileIcon, ScanPill } from "./FileVisuals";
 import { fileGrantCount, isAgentOwner, resolveOwnerName, shortChecksum, shortObjectRef } from "./file-helpers";
 import { t } from "../../i18n/i18n";
 
@@ -89,7 +89,7 @@ function FileRow({
         <span style={{ color: "var(--text-2)", overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>{owner}</span>
       </div>
       <div className="row" style={{ gap: 6, minWidth: 0 }}>
-        <Pill tone={canOpen ? "green" : "red"}>{canOpen ? t("Clean") : t("Blocked")}</Pill>
+        <ScanPill file={file} />
         <span className="mono" style={{ color: "var(--text-3)", fontSize: 12 }}>{grantCount}</span>
       </div>
       <div style={{ color: "var(--text-3)", fontSize: 12 }}>{formatRelative(file.updatedAt ?? file.createdAt)}</div>

tests/unit/file-scan.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, test } from "vitest";
+import { scanFileContent } from "../../src/main/services/files/file-object-store";
+
+const MIB = 1024 * 1024;
+
+describe("the file scan", () => {
+  test("checks the whole file, not just its first MiB, including a secret that straddles a window edge", () => {
+    const padding = "x".repeat(3 * MIB);
+    expect(scanFileContent("notes.txt", Buffer.from(`${padding}\nAKIAABCDEFGHIJKLMNOP\n`)).state).toBe("blocked");
+    const straddling = Buffer.concat([Buffer.alloc(MIB - 10, "y"), Buffer.from("-----BEGIN RSA PRIVATE KEY-----")]);
+    expect(scanFileContent("key.txt", straddling).labels).toContain("dlp:private-key");
+    expect(scanFileContent("eicar.txt", Buffer.from(`${padding}X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*`)).labels).toContain("av:eicar");
+  });
+
+  test("no pattern match can stretch past the window overlap", () => {
+    const edge = Buffer.concat([Buffer.alloc(MIB - 20, "z"), Buffer.from(`password:${"\n".repeat(5000)}${"A".repeat(32)}`)]);
+    // Whitespace between a key name and its value is bounded; a value 5,000 lines away is not that key's value.
+    expect(scanFileContent("far.txt", edge).labels).not.toContain("dlp:credential");
+    const long = Buffer.concat([Buffer.alloc(MIB - 10, "z"), Buffer.from(` ghp_${"a".repeat(9000)} `)]);
+    expect(scanFileContent("token.txt", long).labels).toContain("dlp:github-token");
+    expect(scanFileContent("plan.md", Buffer.from("# Plan")).labels).toContain("scan:whole-file");
+  });
+
+  test("a window cut does not invent a word boundary", () => {
+    const inWord = Buffer.concat([Buffer.alloc(MIB - 4, "z"), Buffer.from("NOT_AKIAABCDEFGHIJKLMNOP\n")]);
+    expect(scanFileContent("a.txt", inWord).state).toBe("approved");
+    const alone = Buffer.concat([Buffer.alloc(MIB - 1, "z"), Buffer.from(" AKIAABCDEFGHIJKLMNOP\n")]);
+    expect(scanFileContent("b.txt", alone).state).toBe("blocked");
+    // The far edge too: a longer word whose first 20 characters end exactly at a window's last byte is not a key.
+    const atTail = Buffer.concat([Buffer.alloc(MIB + 4096 - 20, "z"), Buffer.from(" AKIAABCDEFGHIJKLMNOPZ\n")]);
+    expect(scanFileContent("c.txt", atTail).state).toBe("approved");
+    const atOwnEnd = Buffer.concat([Buffer.alloc(MIB - 20, "z"), Buffer.from(" AKIAABCDEFGHIJKLMNOPZ\n")]);
+    expect(scanFileContent("d.txt", atOwnEnd).state).toBe("approved");
+  });
+
+  test("a container whose contents are compressed or encoded is labelled as not inspected inside", () => {
+    expect(scanFileContent("report.docx", Buffer.from("PK\u0003\u0004rest")).labels).toContain("scan:contents-not-inspected");
+    expect(scanFileContent("deck.pdf", Buffer.from("%PDF-1.7 ...")).labels).toContain("scan:contents-not-inspected");
+    expect(scanFileContent("archive.gz", Buffer.from([0x1f, 0x8b, 0x08, 0x00])).labels).toContain("scan:contents-not-inspected");
+    expect(scanFileContent("photo.png", Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a])).labels).toContain("scan:contents-not-inspected");
+    expect(scanFileContent("plan.md", Buffer.from("# Plan")).labels).not.toContain("scan:contents-not-inspected");
+  });
+});

tests/unit/workspace-file-versions.test.ts

@@ -5,7 +5,7 @@ import { join } from "node:path";
 import { describe, expect, test } from "vitest";
 import { DatabaseConnection } from "../../src/main/database/connection";
 import { OrganizationService } from "../../src/main/services/organization-service";
-import { preserveRecordedFiles, recordWorkspaceFile } from "../../src/main/services/files/file-object-store";
+import { preserveRecordedFiles, recordWorkspaceFile, rescanLegacyFiles } from "../../src/main/services/files/file-object-store";
 import { ChatService } from "../../src/main/services/chat-service";
 import { readAttachmentTool } from "../../src/main/services/agent-attachment-tools";
 
@@ -81,3 +81,28 @@ describe("opening a recorded file", () => {
     connection.close();
   });
 });
+
+describe("files scanned before whole-file scanning", () => {
+  test("are scanned again when their bytes are intact, and marked as sampled when they are not", () => {
+    const connection = new DatabaseConnection(join(tmpdir(), `ac-rescan-${randomUUID()}.db`));
+    let orgId = "";
+    connection.transaction(() => {
+      orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Dana", mission: "m", goal: "g" });
+    });
+    const path = join(tmpdir(), `ac-rescan-${randomUUID()}.txt`);
+    const content = `${"x".repeat(2 * 1024 * 1024)}\nAKIAABCDEFGHIJKLMNOP\n`;
+    writeFileSync(path, content);
+    const insert = connection.db.prepare(`
+      insert into file_objects(id, org_id, name, kind, size_bytes, sha256, storage_path, source_path, owner_type, owner_id, pinned, scan_state, scan_labels_json, created_at)
+      values (?, ?, 'notes.txt', 'txt', 1, ?, ?, ?, 'agent', null, 0, 'approved', '["classification:internal"]', ?)
+    `);
+    insert.run("fil_intact", orgId, createHash("sha256").update(content).digest("hex"), path, path, new Date().toISOString());
+    insert.run("fil_gone", orgId, "0".repeat(64), join(tmpdir(), "missing.txt"), join(tmpdir(), "missing.txt"), new Date().toISOString());
+    rescanLegacyFiles(connection.db);
+    const row = (id: string) => connection.db.prepare("select scan_state, scan_labels_json from file_objects where id = ?").get(id) as { scan_state: string; scan_labels_json: string };
+    expect(row("fil_intact").scan_state).toBe("blocked");
+    expect(JSON.parse(row("fil_intact").scan_labels_json)).toContain("scan:whole-file");
+    expect(row("fil_gone")).toEqual({ scan_state: "approved", scan_labels_json: '["classification:internal","scan:legacy-sample"]' });
+    connection.close();
+  });
+});