Q125: imports stream through a bounded window and refuse files over 1 GiB

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/main/services/file-service.ts

@@ -1,5 +1,5 @@
-import { createHash } from "node:crypto";
-import { copyFileSync, mkdirSync, readFileSync, statSync } from "node:fs";
+import { mkdirSync, statSync } from "node:fs";
+import { copyFile } from "node:fs/promises";
 import { basename, join } from "node:path";
 import type { DatabaseSync } from "node:sqlite";
 import { dialog, shell } from "electron";
@@ -7,7 +7,13 @@ import type { ImportFileInput, OpenFileInput } from "../../shared/commands";
 import { FILE_SCAN_STATE } from "../../shared/domain";
 import { createId } from "../../shared/ids";
 import { AuditService } from "./audit-service";
-import { fileKind, readVerifiedFile, scanFileContent, upsertFileObject } from "./files/file-object-store";
+import { fileKind, readVerifiedFile, scanFileAt, upsertFileObject } from "./files/file-object-store";
+
+const MAX_IMPORT_BYTES = 1024 * 1024 * 1024;
+
+function formatBytes(bytes: number): string {
+  return bytes >= 1024 * 1024 ? `${(bytes / (1024 * 1024)).toFixed(1)} MiB` : `${bytes} bytes`;
+}
 
 export class FileService {
   private readonly audit: AuditService;
@@ -30,20 +36,21 @@ export class FileService {
   }
 
   /** A file imported into a direct conversation or restricted channel is readable by that space's members only. */
-  importPath(orgId: string, sourcePath: string, spaceId: string | null): string {
+  async importPath(orgId: string, sourcePath: string, spaceId: string | null): Promise<string> {
     const access = spaceId ? spaceAccess(this.db, orgId, spaceId) : "workspace";
     const id = createId("fil");
     const now = new Date().toISOString();
-    const stats = statSync(sourcePath);
-    if (!stats.isFile()) throw new Error("Selected path is not a file.");
+    const source = statSync(sourcePath);
+    if (!source.isFile()) throw new Error("Selected path is not a file.");
+    if (source.size > MAX_IMPORT_BYTES) throw new Error(`${basename(sourcePath)} is ${formatBytes(source.size)}; files over ${formatBytes(MAX_IMPORT_BYTES)} cannot be imported.`);
     const name = basename(sourcePath);
-    const bytes = readFileSync(sourcePath);
-    const hash = createHash("sha256").update(bytes).digest("hex");
-    const scan = scanFileContent(name, bytes);
     const orgDir = join(this.artifactRoot, orgId);
     mkdirSync(orgDir, { recursive: true });
     const target = join(orgDir, `${id}-${name}`);
-    copyFileSync(sourcePath, target);
+    // The stored copy is what gets hashed and scanned, so the record describes exactly the bytes kept; nothing is held in memory whole.
+    await copyFile(sourcePath, target);
+    const { scan, sha256: hash, size } = await scanFileAt(target, name);
+    console.info(`[files] imported ${name} (${formatBytes(size)})`);
     this.db.exec("begin immediate");
     try {
       upsertFileObject(this.db, {
@@ -51,7 +58,7 @@ export class FileService {
         orgId,
         name,
         kind: fileKind(name),
-        sizeBytes: stats.size,
+        sizeBytes: size,
         sha256: hash,
         storagePath: target,
         sourcePath: target,
@@ -68,7 +75,7 @@ export class FileService {
         actorType: "system",
         actorId: null,
         action: "imported",
-        payload: { sha256: hash, sizeBytes: stats.size, scanState: scan.state, scanLabels: scan.labels, objectRef: target },
+        payload: { sha256: hash, sizeBytes: size, scanState: scan.state, scanLabels: scan.labels, objectRef: target },
       });
       this.db.exec("commit");
     } catch (error) {

src/main/services/files/file-object-store.ts

@@ -1,5 +1,6 @@
 import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { open } from "node:fs/promises";
 import { basename, dirname, extname, isAbsolute, join, relative } from "node:path";
 import type { DatabaseSync } from "node:sqlite";
 import { FILE_SCAN_STATE, type FileScanState } from "../../../shared/domain";
@@ -52,32 +53,65 @@ export function fileKind(path: string): string {
 
 /** Pattern checks for secrets and the antivirus test string over the whole file; not an antivirus engine, and compressed contents are not opened. */
 export function scanFileContent(name: string, bytes: Buffer): FileScanResult {
-  const labels = new Set(scanLabels(name, bytes.length));
-  let blocked = false;
-  // Overlapping windows cover every byte with bounded memory; the overlap is longer than any pattern match.
+  const state = { labels: new Set(scanLabels(name, bytes.length)), blocked: false };
   for (let start = 0; start === 0 || start < bytes.length; start += SCAN_WINDOW_BYTES) {
-    // A window owns the matches that start in its own MiB; context on both sides keeps word boundaries real at the cuts.
-    const ownEnd = Math.min(bytes.length, start + SCAN_WINDOW_BYTES);
-    const lead = bytes.subarray(Math.max(0, start - SCAN_CONTEXT_BYTES), start).toString("utf8");
-    const own = bytes.subarray(start, ownEnd).toString("utf8");
-    const tail = bytes.subarray(ownEnd, Math.min(bytes.length, ownEnd + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES)).toString("utf8");
-    const text = lead + own + tail;
-    const range = { from: lead.length, until: lead.length + own.length };
-    if (bytes.subarray(start, ownEnd + SCAN_OVERLAP_BYTES).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) {
-      labels.add("av:eicar");
-      blocked = true;
-    }
-    for (const scan of TEXT_SCAN_PATTERNS) {
-      if (!matchesWithin(scan.pattern, text, range)) continue;
-      labels.add(scan.label);
-      if (scan.block) blocked = true;
-    }
-    if (containsPaymentCard(text, range)) {
-      labels.add("dlp:payment-card");
-      blocked = true;
-    }
+    scanWindow(state, bytes.length, start, (from, to) => bytes.subarray(from, to));
   }
-  if (isEncodedContainer(bytes)) labels.add("scan:contents-not-inspected");
+  return finishScan(state, bytes.subarray(0, 12));
+}
+
+/** The same scan over a file on disk, one window in memory at a time and without blocking; the hash comes from the same reads, so both describe one set of bytes. */
+export async function scanFileAt(path: string, name: string): Promise<{ scan: FileScanResult; sha256: string; size: number }> {
+  const handle = await open(path, "r");
+  try {
+    const size = (await handle.stat()).size;
+    const state = { labels: new Set(scanLabels(name, size)), blocked: false };
+    const hash = createHash("sha256");
+    let head = Buffer.alloc(0);
+    for (let start = 0; start === 0 || start < size; start += SCAN_WINDOW_BYTES) {
+      const spanStart = Math.max(0, start - SCAN_CONTEXT_BYTES);
+      const spanEnd = Math.min(size, start + SCAN_WINDOW_BYTES + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES);
+      const span = Buffer.alloc(spanEnd - spanStart);
+      const { bytesRead } = await handle.read(span, 0, span.length, spanStart);
+      if (bytesRead !== span.length) throw new Error(`${name} changed size while it was being scanned.`);
+      if (start === 0) head = Buffer.from(span.subarray(0, 12));
+      // Each window's own range follows the last, so the hash sees every byte once, in order.
+      hash.update(span.subarray(start - spanStart, Math.min(size, start + SCAN_WINDOW_BYTES) - spanStart));
+      scanWindow(state, size, start, (from, to) => span.subarray(from - spanStart, to - spanStart));
+    }
+    if ((await handle.stat()).size !== size) throw new Error(`${name} changed size while it was being scanned.`);
+    return { scan: finishScan(state, head), sha256: hash.digest("hex"), size };
+  } finally {
+    await handle.close();
+  }
+}
+
+/** One window owns the matches that start in its own MiB; context on both sides keeps word boundaries real at the cuts. */
+function scanWindow(state: { labels: Set<string>; blocked: boolean }, size: number, start: number, read: (from: number, to: number) => Buffer): void {
+  const ownEnd = Math.min(size, start + SCAN_WINDOW_BYTES);
+  const lead = read(Math.max(0, start - SCAN_CONTEXT_BYTES), start).toString("utf8");
+  const own = read(start, ownEnd).toString("utf8");
+  const tail = read(ownEnd, Math.min(size, ownEnd + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES)).toString("utf8");
+  const text = lead + own + tail;
+  const range = { from: lead.length, until: lead.length + own.length };
+  if (read(start, Math.min(size, ownEnd + SCAN_OVERLAP_BYTES)).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) {
+    state.labels.add("av:eicar");
+    state.blocked = true;
+  }
+  for (const scan of TEXT_SCAN_PATTERNS) {
+    if (!matchesWithin(scan.pattern, text, range)) continue;
+    state.labels.add(scan.label);
+    if (scan.block) state.blocked = true;
+  }
+  if (containsPaymentCard(text, range)) {
+    state.labels.add("dlp:payment-card");
+    state.blocked = true;
+  }
+}
+
+function finishScan(state: { labels: Set<string>; blocked: boolean }, head: Buffer): FileScanResult {
+  const { labels, blocked } = state;
+  if (isEncodedContainer(head)) labels.add("scan:contents-not-inspected");
   labels.add("scan:whole-file");
   labels.add(blocked ? "classification:restricted" : labels.has("dlp:email") ? "classification:confidential" : "classification:internal");
   return {

tests/unit/domain-services.test.ts

@@ -459,7 +459,7 @@ describe("domain services", () => {
     connection.close();
   });
 
-  test("imports files with checksum, scan labels, object refs, and workspace grants", () => {
+  test("imports files with checksum, scan labels, object refs, and workspace grants", async () => {
     const connection = createConnection();
     const organization = new OrganizationService(connection.db);
     const snapshot = new SnapshotService(connection.db);
@@ -483,8 +483,8 @@ describe("domain services", () => {
         goal: "Ship a verified control plane.",
       });
     });
-    const cleanFileId = fileService.importPath(orgId, cleanPath, null);
-    const blockedFileId = fileService.importPath(orgId, secretPath, null);
+    const cleanFileId = await fileService.importPath(orgId, cleanPath, null);
+    const blockedFileId = await fileService.importPath(orgId, secretPath, null);
 
     const state = snapshot.load();
     const cleanFile = state.files.find((file) => file.id === cleanFileId);

tests/unit/file-scan.test.ts

@@ -1,5 +1,9 @@
+import { createHash, randomUUID } from "node:crypto";
+import { appendFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { describe, expect, test } from "vitest";
-import { scanFileContent } from "../../src/main/services/files/file-object-store";
+import { scanFileAt, scanFileContent } from "../../src/main/services/files/file-object-store";
 
 const MIB = 1024 * 1024;
 
@@ -41,3 +45,30 @@ describe("the file scan", () => {
     expect(scanFileContent("plan.md", Buffer.from("# Plan")).labels).not.toContain("scan:contents-not-inspected");
   });
 });
+
+describe("scanning a file on disk", () => {
+  test("streams the same verdict and hash as scanning it in memory", async () => {
+    const path = join(tmpdir(), `ac-scan-${randomUUID()}.bin`);
+    const content = Buffer.concat([Buffer.from("PK\u0003\u0004"), Buffer.alloc(2 * MIB, "x"), Buffer.from(" AKIAABCDEFGHIJKLMNOP ")]);
+    writeFileSync(path, content);
+    const streamed = await scanFileAt(path, "bundle.zip");
+    const inMemory = scanFileContent("bundle.zip", content);
+    expect({ state: streamed.scan.state, labels: streamed.scan.labels }).toEqual({ state: inMemory.state, labels: inMemory.labels });
+    expect(streamed.scan.labels).toEqual(expect.arrayContaining(["dlp:aws-access-key", "scan:contents-not-inspected"]));
+    expect(streamed.sha256).toBe(createHash("sha256").update(content).digest("hex"));
+    expect(streamed.size).toBe(content.length);
+  });
+});
+
+describe("a file that changes while it is scanned", () => {
+  test("is never approved without its new bytes having been scanned", async () => {
+    const path = join(tmpdir(), `ac-grow-${randomUUID()}.txt`);
+    writeFileSync(path, Buffer.alloc(8 * MIB, "x"));
+    const scanning = scanFileAt(path, "grow.txt").then(
+      (result) => (result.scan.state === "blocked" ? "blocked" : "approved"),
+      (error: Error) => error.message,
+    );
+    appendFileSync(path, "\nAKIAABCDEFGHIJKLMNOP\n");
+    expect(["blocked", "grow.txt changed size while it was being scanned."]).toContain(await scanning);
+  });
+});

tests/unit/private-attachments.test.ts

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { mkdirSync, writeFileSync } from "node:fs";
+import { mkdirSync, readdirSync, rmSync, truncateSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { describe, expect, test, vi } from "vitest";
@@ -13,7 +13,7 @@ const { FileService } = await import("../../src/main/services/file-service");
 const { dispatchAgentTool } = await import("../../src/main/services/agent-tools");
 
 describe("files shared in a private conversation", () => {
-  test("a file imported into a direct conversation is readable by its members only", () => {
+  test("a file imported into a direct conversation is readable by its members only", async () => {
     const connection = new DatabaseConnection(join(tmpdir(), `ac-private-${randomUUID()}.db`));
     const organization = new OrganizationService(connection.db);
     const chat = new ChatService(connection.db);
@@ -33,7 +33,7 @@ describe("files shared in a private conversation", () => {
     mkdirSync(source, { recursive: true });
     writeFileSync(join(source, "salary.md"), "Dana's raise: 10%");
     const artifacts = join(tmpdir(), `ac-private-art-${randomUUID()}`);
-    const fileId = new FileService(connection.db, artifacts).importPath(orgId, join(source, "salary.md"), dm);
+    const fileId = await new FileService(connection.db, artifacts).importPath(orgId, join(source, "salary.md"), dm);
 
     const deps = { db: connection.db, organization, chat };
     const as = (agentId: string) => ({ orgId, agent: { id: agentId, modelProviderId: null, modelId: null }, taskProjectId: null, taskGoalId: null, taskId: null, sourceThreadId: null, workspace: null });
@@ -50,4 +50,20 @@ describe("files shared in a private conversation", () => {
       .toEqual([{ type: "agent" }, { type: "employee" }]);
     connection.close();
   });
+
+  test("a file over the import limit is refused before anything is copied", async () => {
+    const connection = new DatabaseConnection(join(tmpdir(), `ac-import-limit-${randomUUID()}.db`));
+    let orgId = "";
+    connection.transaction(() => {
+      orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Yang", mission: "m", goal: "g" });
+    });
+    const huge = join(tmpdir(), `ac-huge-${randomUUID()}.bin`);
+    writeFileSync(huge, "");
+    truncateSync(huge, 1024 * 1024 * 1024 + 1);
+    const artifacts = join(tmpdir(), `ac-import-limit-art-${randomUUID()}`);
+    await expect(new FileService(connection.db, artifacts).importPath(orgId, huge, null)).rejects.toThrow("files over 1024.0 MiB cannot be imported");
+    expect(readdirSync(artifacts, { recursive: true })).toEqual([]);
+    rmSync(huge);
+    connection.close();
+  });
 });