diff --git a/src/main/services/file-service.ts b/src/main/services/file-service.ts index 513e63b0..aa0f7f18 100644 --- a/src/main/services/file-service.ts +++ b/src/main/services/file-service.ts @@ -1,5 +1,5 @@ -import { createHash } from "node:crypto"; -import { copyFileSync, mkdirSync, readFileSync, statSync } from "node:fs"; +import { mkdirSync, statSync } from "node:fs"; +import { copyFile } from "node:fs/promises"; import { basename, join } from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { dialog, shell } from "electron"; @@ -7,7 +7,13 @@ import type { ImportFileInput, OpenFileInput } from "../../shared/commands"; import { FILE_SCAN_STATE } from "../../shared/domain"; import { createId } from "../../shared/ids"; import { AuditService } from "./audit-service"; -import { fileKind, readVerifiedFile, scanFileContent, upsertFileObject } from "./files/file-object-store"; +import { fileKind, readVerifiedFile, scanFileAt, upsertFileObject } from "./files/file-object-store"; + +const MAX_IMPORT_BYTES = 1024 * 1024 * 1024; + +function formatBytes(bytes: number): string { + return bytes >= 1024 * 1024 ? `${(bytes / (1024 * 1024)).toFixed(1)} MiB` : `${bytes} bytes`; +} export class FileService { private readonly audit: AuditService; @@ -30,20 +36,21 @@ export class FileService { } /** A file imported into a direct conversation or restricted channel is readable by that space's members only. */ - importPath(orgId: string, sourcePath: string, spaceId: string | null): string { + async importPath(orgId: string, sourcePath: string, spaceId: string | null): Promise { const access = spaceId ? spaceAccess(this.db, orgId, spaceId) : "workspace"; const id = createId("fil"); const now = new Date().toISOString(); - const stats = statSync(sourcePath); - if (!stats.isFile()) throw new Error("Selected path is not a file."); + const source = statSync(sourcePath); + if (!source.isFile()) throw new Error("Selected path is not a file."); + if (source.size > MAX_IMPORT_BYTES) throw new Error(`${basename(sourcePath)} is ${formatBytes(source.size)}; files over ${formatBytes(MAX_IMPORT_BYTES)} cannot be imported.`); const name = basename(sourcePath); - const bytes = readFileSync(sourcePath); - const hash = createHash("sha256").update(bytes).digest("hex"); - const scan = scanFileContent(name, bytes); const orgDir = join(this.artifactRoot, orgId); mkdirSync(orgDir, { recursive: true }); const target = join(orgDir, `${id}-${name}`); - copyFileSync(sourcePath, target); + // The stored copy is what gets hashed and scanned, so the record describes exactly the bytes kept; nothing is held in memory whole. + await copyFile(sourcePath, target); + const { scan, sha256: hash, size } = await scanFileAt(target, name); + console.info(`[files] imported ${name} (${formatBytes(size)})`); this.db.exec("begin immediate"); try { upsertFileObject(this.db, { @@ -51,7 +58,7 @@ export class FileService { orgId, name, kind: fileKind(name), - sizeBytes: stats.size, + sizeBytes: size, sha256: hash, storagePath: target, sourcePath: target, @@ -68,7 +75,7 @@ export class FileService { actorType: "system", actorId: null, action: "imported", - payload: { sha256: hash, sizeBytes: stats.size, scanState: scan.state, scanLabels: scan.labels, objectRef: target }, + payload: { sha256: hash, sizeBytes: size, scanState: scan.state, scanLabels: scan.labels, objectRef: target }, }); this.db.exec("commit"); } catch (error) { diff --git a/src/main/services/files/file-object-store.ts b/src/main/services/files/file-object-store.ts index d281664c..69717840 100644 --- a/src/main/services/files/file-object-store.ts +++ b/src/main/services/files/file-object-store.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { open } from "node:fs/promises"; import { basename, dirname, extname, isAbsolute, join, relative } from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { FILE_SCAN_STATE, type FileScanState } from "../../../shared/domain"; @@ -52,32 +53,65 @@ export function fileKind(path: string): string { /** Pattern checks for secrets and the antivirus test string over the whole file; not an antivirus engine, and compressed contents are not opened. */ export function scanFileContent(name: string, bytes: Buffer): FileScanResult { - const labels = new Set(scanLabels(name, bytes.length)); - let blocked = false; - // Overlapping windows cover every byte with bounded memory; the overlap is longer than any pattern match. + const state = { labels: new Set(scanLabels(name, bytes.length)), blocked: false }; for (let start = 0; start === 0 || start < bytes.length; start += SCAN_WINDOW_BYTES) { - // A window owns the matches that start in its own MiB; context on both sides keeps word boundaries real at the cuts. - const ownEnd = Math.min(bytes.length, start + SCAN_WINDOW_BYTES); - const lead = bytes.subarray(Math.max(0, start - SCAN_CONTEXT_BYTES), start).toString("utf8"); - const own = bytes.subarray(start, ownEnd).toString("utf8"); - const tail = bytes.subarray(ownEnd, Math.min(bytes.length, ownEnd + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES)).toString("utf8"); - const text = lead + own + tail; - const range = { from: lead.length, until: lead.length + own.length }; - if (bytes.subarray(start, ownEnd + SCAN_OVERLAP_BYTES).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) { - labels.add("av:eicar"); - blocked = true; - } - for (const scan of TEXT_SCAN_PATTERNS) { - if (!matchesWithin(scan.pattern, text, range)) continue; - labels.add(scan.label); - if (scan.block) blocked = true; - } - if (containsPaymentCard(text, range)) { - labels.add("dlp:payment-card"); - blocked = true; - } + scanWindow(state, bytes.length, start, (from, to) => bytes.subarray(from, to)); } - if (isEncodedContainer(bytes)) labels.add("scan:contents-not-inspected"); + return finishScan(state, bytes.subarray(0, 12)); +} + +/** The same scan over a file on disk, one window in memory at a time and without blocking; the hash comes from the same reads, so both describe one set of bytes. */ +export async function scanFileAt(path: string, name: string): Promise<{ scan: FileScanResult; sha256: string; size: number }> { + const handle = await open(path, "r"); + try { + const size = (await handle.stat()).size; + const state = { labels: new Set(scanLabels(name, size)), blocked: false }; + const hash = createHash("sha256"); + let head = Buffer.alloc(0); + for (let start = 0; start === 0 || start < size; start += SCAN_WINDOW_BYTES) { + const spanStart = Math.max(0, start - SCAN_CONTEXT_BYTES); + const spanEnd = Math.min(size, start + SCAN_WINDOW_BYTES + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES); + const span = Buffer.alloc(spanEnd - spanStart); + const { bytesRead } = await handle.read(span, 0, span.length, spanStart); + if (bytesRead !== span.length) throw new Error(`${name} changed size while it was being scanned.`); + if (start === 0) head = Buffer.from(span.subarray(0, 12)); + // Each window's own range follows the last, so the hash sees every byte once, in order. + hash.update(span.subarray(start - spanStart, Math.min(size, start + SCAN_WINDOW_BYTES) - spanStart)); + scanWindow(state, size, start, (from, to) => span.subarray(from - spanStart, to - spanStart)); + } + if ((await handle.stat()).size !== size) throw new Error(`${name} changed size while it was being scanned.`); + return { scan: finishScan(state, head), sha256: hash.digest("hex"), size }; + } finally { + await handle.close(); + } +} + +/** One window owns the matches that start in its own MiB; context on both sides keeps word boundaries real at the cuts. */ +function scanWindow(state: { labels: Set; blocked: boolean }, size: number, start: number, read: (from: number, to: number) => Buffer): void { + const ownEnd = Math.min(size, start + SCAN_WINDOW_BYTES); + const lead = read(Math.max(0, start - SCAN_CONTEXT_BYTES), start).toString("utf8"); + const own = read(start, ownEnd).toString("utf8"); + const tail = read(ownEnd, Math.min(size, ownEnd + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES)).toString("utf8"); + const text = lead + own + tail; + const range = { from: lead.length, until: lead.length + own.length }; + if (read(start, Math.min(size, ownEnd + SCAN_OVERLAP_BYTES)).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) { + state.labels.add("av:eicar"); + state.blocked = true; + } + for (const scan of TEXT_SCAN_PATTERNS) { + if (!matchesWithin(scan.pattern, text, range)) continue; + state.labels.add(scan.label); + if (scan.block) state.blocked = true; + } + if (containsPaymentCard(text, range)) { + state.labels.add("dlp:payment-card"); + state.blocked = true; + } +} + +function finishScan(state: { labels: Set; blocked: boolean }, head: Buffer): FileScanResult { + const { labels, blocked } = state; + if (isEncodedContainer(head)) labels.add("scan:contents-not-inspected"); labels.add("scan:whole-file"); labels.add(blocked ? "classification:restricted" : labels.has("dlp:email") ? "classification:confidential" : "classification:internal"); return { diff --git a/tests/unit/domain-services.test.ts b/tests/unit/domain-services.test.ts index aed2a4ed..dc68e1ce 100644 --- a/tests/unit/domain-services.test.ts +++ b/tests/unit/domain-services.test.ts @@ -459,7 +459,7 @@ describe("domain services", () => { connection.close(); }); - test("imports files with checksum, scan labels, object refs, and workspace grants", () => { + test("imports files with checksum, scan labels, object refs, and workspace grants", async () => { const connection = createConnection(); const organization = new OrganizationService(connection.db); const snapshot = new SnapshotService(connection.db); @@ -483,8 +483,8 @@ describe("domain services", () => { goal: "Ship a verified control plane.", }); }); - const cleanFileId = fileService.importPath(orgId, cleanPath, null); - const blockedFileId = fileService.importPath(orgId, secretPath, null); + const cleanFileId = await fileService.importPath(orgId, cleanPath, null); + const blockedFileId = await fileService.importPath(orgId, secretPath, null); const state = snapshot.load(); const cleanFile = state.files.find((file) => file.id === cleanFileId); diff --git a/tests/unit/file-scan.test.ts b/tests/unit/file-scan.test.ts index 5d75e0b9..5f0cd156 100644 --- a/tests/unit/file-scan.test.ts +++ b/tests/unit/file-scan.test.ts @@ -1,5 +1,9 @@ +import { createHash, randomUUID } from "node:crypto"; +import { appendFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, test } from "vitest"; -import { scanFileContent } from "../../src/main/services/files/file-object-store"; +import { scanFileAt, scanFileContent } from "../../src/main/services/files/file-object-store"; const MIB = 1024 * 1024; @@ -41,3 +45,30 @@ describe("the file scan", () => { expect(scanFileContent("plan.md", Buffer.from("# Plan")).labels).not.toContain("scan:contents-not-inspected"); }); }); + +describe("scanning a file on disk", () => { + test("streams the same verdict and hash as scanning it in memory", async () => { + const path = join(tmpdir(), `ac-scan-${randomUUID()}.bin`); + const content = Buffer.concat([Buffer.from("PK\u0003\u0004"), Buffer.alloc(2 * MIB, "x"), Buffer.from(" AKIAABCDEFGHIJKLMNOP ")]); + writeFileSync(path, content); + const streamed = await scanFileAt(path, "bundle.zip"); + const inMemory = scanFileContent("bundle.zip", content); + expect({ state: streamed.scan.state, labels: streamed.scan.labels }).toEqual({ state: inMemory.state, labels: inMemory.labels }); + expect(streamed.scan.labels).toEqual(expect.arrayContaining(["dlp:aws-access-key", "scan:contents-not-inspected"])); + expect(streamed.sha256).toBe(createHash("sha256").update(content).digest("hex")); + expect(streamed.size).toBe(content.length); + }); +}); + +describe("a file that changes while it is scanned", () => { + test("is never approved without its new bytes having been scanned", async () => { + const path = join(tmpdir(), `ac-grow-${randomUUID()}.txt`); + writeFileSync(path, Buffer.alloc(8 * MIB, "x")); + const scanning = scanFileAt(path, "grow.txt").then( + (result) => (result.scan.state === "blocked" ? "blocked" : "approved"), + (error: Error) => error.message, + ); + appendFileSync(path, "\nAKIAABCDEFGHIJKLMNOP\n"); + expect(["blocked", "grow.txt changed size while it was being scanned."]).toContain(await scanning); + }); +}); diff --git a/tests/unit/private-attachments.test.ts b/tests/unit/private-attachments.test.ts index eb394bda..0df3658e 100644 --- a/tests/unit/private-attachments.test.ts +++ b/tests/unit/private-attachments.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { mkdirSync, readdirSync, rmSync, truncateSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test, vi } from "vitest"; @@ -13,7 +13,7 @@ const { FileService } = await import("../../src/main/services/file-service"); const { dispatchAgentTool } = await import("../../src/main/services/agent-tools"); describe("files shared in a private conversation", () => { - test("a file imported into a direct conversation is readable by its members only", () => { + test("a file imported into a direct conversation is readable by its members only", async () => { const connection = new DatabaseConnection(join(tmpdir(), `ac-private-${randomUUID()}.db`)); const organization = new OrganizationService(connection.db); const chat = new ChatService(connection.db); @@ -33,7 +33,7 @@ describe("files shared in a private conversation", () => { mkdirSync(source, { recursive: true }); writeFileSync(join(source, "salary.md"), "Dana's raise: 10%"); const artifacts = join(tmpdir(), `ac-private-art-${randomUUID()}`); - const fileId = new FileService(connection.db, artifacts).importPath(orgId, join(source, "salary.md"), dm); + const fileId = await new FileService(connection.db, artifacts).importPath(orgId, join(source, "salary.md"), dm); const deps = { db: connection.db, organization, chat }; const as = (agentId: string) => ({ orgId, agent: { id: agentId, modelProviderId: null, modelId: null }, taskProjectId: null, taskGoalId: null, taskId: null, sourceThreadId: null, workspace: null }); @@ -50,4 +50,20 @@ describe("files shared in a private conversation", () => { .toEqual([{ type: "agent" }, { type: "employee" }]); connection.close(); }); + + test("a file over the import limit is refused before anything is copied", async () => { + const connection = new DatabaseConnection(join(tmpdir(), `ac-import-limit-${randomUUID()}.db`)); + let orgId = ""; + connection.transaction(() => { + orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Yang", mission: "m", goal: "g" }); + }); + const huge = join(tmpdir(), `ac-huge-${randomUUID()}.bin`); + writeFileSync(huge, ""); + truncateSync(huge, 1024 * 1024 * 1024 + 1); + const artifacts = join(tmpdir(), `ac-import-limit-art-${randomUUID()}`); + await expect(new FileService(connection.db, artifacts).importPath(orgId, huge, null)).rejects.toThrow("files over 1024.0 MiB cannot be imported"); + expect(readdirSync(artifacts, { recursive: true })).toEqual([]); + rmSync(huge); + connection.close(); + }); });