Q125: imports stream through a bounded window and refuse files over 1 GiB
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a9059fbea
commit
11cdc4af20
5 files changed
+131
-43
No files matched your search
@@ -1,5 +1,5 @@
|
|||||||
import { createHash } from "node:crypto";
|
import { mkdirSync, statSync } from "node:fs";
|
||||||
import { copyFileSync, mkdirSync, readFileSync, statSync } from "node:fs";
|
import { copyFile } from "node:fs/promises";
|
||||||
import { basename, join } from "node:path";
|
import { basename, join } from "node:path";
|
||||||
import type { DatabaseSync } from "node:sqlite";
|
import type { DatabaseSync } from "node:sqlite";
|
||||||
import { dialog, shell } from "electron";
|
import { dialog, shell } from "electron";
|
||||||
@@ -7,7 +7,13 @@ import type { ImportFileInput, OpenFileInput } from "../../shared/commands";
|
|||||||
import { FILE_SCAN_STATE } from "../../shared/domain";
|
import { FILE_SCAN_STATE } from "../../shared/domain";
|
||||||
import { createId } from "../../shared/ids";
|
import { createId } from "../../shared/ids";
|
||||||
import { AuditService } from "./audit-service";
|
import { AuditService } from "./audit-service";
|
||||||
import { fileKind, readVerifiedFile, scanFileContent, upsertFileObject } from "./files/file-object-store";
|
import { fileKind, readVerifiedFile, scanFileAt, upsertFileObject } from "./files/file-object-store";
|
||||||
|
|
||||||
|
const MAX_IMPORT_BYTES = 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
function formatBytes(bytes: number): string {
|
||||||
|
return bytes >= 1024 * 1024 ? `${(bytes / (1024 * 1024)).toFixed(1)} MiB` : `${bytes} bytes`;
|
||||||
|
}
|
||||||
|
|
||||||
export class FileService {
|
export class FileService {
|
||||||
private readonly audit: AuditService;
|
private readonly audit: AuditService;
|
||||||
@@ -30,20 +36,21 @@ export class FileService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** A file imported into a direct conversation or restricted channel is readable by that space's members only. */
|
/** A file imported into a direct conversation or restricted channel is readable by that space's members only. */
|
||||||
importPath(orgId: string, sourcePath: string, spaceId: string | null): string {
|
async importPath(orgId: string, sourcePath: string, spaceId: string | null): Promise<string> {
|
||||||
const access = spaceId ? spaceAccess(this.db, orgId, spaceId) : "workspace";
|
const access = spaceId ? spaceAccess(this.db, orgId, spaceId) : "workspace";
|
||||||
const id = createId("fil");
|
const id = createId("fil");
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const stats = statSync(sourcePath);
|
const source = statSync(sourcePath);
|
||||||
if (!stats.isFile()) throw new Error("Selected path is not a file.");
|
if (!source.isFile()) throw new Error("Selected path is not a file.");
|
||||||
|
if (source.size > MAX_IMPORT_BYTES) throw new Error(`${basename(sourcePath)} is ${formatBytes(source.size)}; files over ${formatBytes(MAX_IMPORT_BYTES)} cannot be imported.`);
|
||||||
const name = basename(sourcePath);
|
const name = basename(sourcePath);
|
||||||
const bytes = readFileSync(sourcePath);
|
|
||||||
const hash = createHash("sha256").update(bytes).digest("hex");
|
|
||||||
const scan = scanFileContent(name, bytes);
|
|
||||||
const orgDir = join(this.artifactRoot, orgId);
|
const orgDir = join(this.artifactRoot, orgId);
|
||||||
mkdirSync(orgDir, { recursive: true });
|
mkdirSync(orgDir, { recursive: true });
|
||||||
const target = join(orgDir, `${id}-${name}`);
|
const target = join(orgDir, `${id}-${name}`);
|
||||||
copyFileSync(sourcePath, target);
|
// The stored copy is what gets hashed and scanned, so the record describes exactly the bytes kept; nothing is held in memory whole.
|
||||||
|
await copyFile(sourcePath, target);
|
||||||
|
const { scan, sha256: hash, size } = await scanFileAt(target, name);
|
||||||
|
console.info(`[files] imported ${name} (${formatBytes(size)})`);
|
||||||
this.db.exec("begin immediate");
|
this.db.exec("begin immediate");
|
||||||
try {
|
try {
|
||||||
upsertFileObject(this.db, {
|
upsertFileObject(this.db, {
|
||||||
@@ -51,7 +58,7 @@ export class FileService {
|
|||||||
orgId,
|
orgId,
|
||||||
name,
|
name,
|
||||||
kind: fileKind(name),
|
kind: fileKind(name),
|
||||||
sizeBytes: stats.size,
|
sizeBytes: size,
|
||||||
sha256: hash,
|
sha256: hash,
|
||||||
storagePath: target,
|
storagePath: target,
|
||||||
sourcePath: target,
|
sourcePath: target,
|
||||||
@@ -68,7 +75,7 @@ export class FileService {
|
|||||||
actorType: "system",
|
actorType: "system",
|
||||||
actorId: null,
|
actorId: null,
|
||||||
action: "imported",
|
action: "imported",
|
||||||
payload: { sha256: hash, sizeBytes: stats.size, scanState: scan.state, scanLabels: scan.labels, objectRef: target },
|
payload: { sha256: hash, sizeBytes: size, scanState: scan.state, scanLabels: scan.labels, objectRef: target },
|
||||||
});
|
});
|
||||||
this.db.exec("commit");
|
this.db.exec("commit");
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { createHash } from "node:crypto";
|
import { createHash } from "node:crypto";
|
||||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||||
|
import { open } from "node:fs/promises";
|
||||||
import { basename, dirname, extname, isAbsolute, join, relative } from "node:path";
|
import { basename, dirname, extname, isAbsolute, join, relative } from "node:path";
|
||||||
import type { DatabaseSync } from "node:sqlite";
|
import type { DatabaseSync } from "node:sqlite";
|
||||||
import { FILE_SCAN_STATE, type FileScanState } from "../../../shared/domain";
|
import { FILE_SCAN_STATE, type FileScanState } from "../../../shared/domain";
|
||||||
@@ -52,32 +53,65 @@ export function fileKind(path: string): string {
|
|||||||
|
|
||||||
/** Pattern checks for secrets and the antivirus test string over the whole file; not an antivirus engine, and compressed contents are not opened. */
|
/** Pattern checks for secrets and the antivirus test string over the whole file; not an antivirus engine, and compressed contents are not opened. */
|
||||||
export function scanFileContent(name: string, bytes: Buffer): FileScanResult {
|
export function scanFileContent(name: string, bytes: Buffer): FileScanResult {
|
||||||
const labels = new Set(scanLabels(name, bytes.length));
|
const state = { labels: new Set(scanLabels(name, bytes.length)), blocked: false };
|
||||||
let blocked = false;
|
|
||||||
// Overlapping windows cover every byte with bounded memory; the overlap is longer than any pattern match.
|
|
||||||
for (let start = 0; start === 0 || start < bytes.length; start += SCAN_WINDOW_BYTES) {
|
for (let start = 0; start === 0 || start < bytes.length; start += SCAN_WINDOW_BYTES) {
|
||||||
// A window owns the matches that start in its own MiB; context on both sides keeps word boundaries real at the cuts.
|
scanWindow(state, bytes.length, start, (from, to) => bytes.subarray(from, to));
|
||||||
const ownEnd = Math.min(bytes.length, start + SCAN_WINDOW_BYTES);
|
|
||||||
const lead = bytes.subarray(Math.max(0, start - SCAN_CONTEXT_BYTES), start).toString("utf8");
|
|
||||||
const own = bytes.subarray(start, ownEnd).toString("utf8");
|
|
||||||
const tail = bytes.subarray(ownEnd, Math.min(bytes.length, ownEnd + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES)).toString("utf8");
|
|
||||||
const text = lead + own + tail;
|
|
||||||
const range = { from: lead.length, until: lead.length + own.length };
|
|
||||||
if (bytes.subarray(start, ownEnd + SCAN_OVERLAP_BYTES).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) {
|
|
||||||
labels.add("av:eicar");
|
|
||||||
blocked = true;
|
|
||||||
}
|
|
||||||
for (const scan of TEXT_SCAN_PATTERNS) {
|
|
||||||
if (!matchesWithin(scan.pattern, text, range)) continue;
|
|
||||||
labels.add(scan.label);
|
|
||||||
if (scan.block) blocked = true;
|
|
||||||
}
|
|
||||||
if (containsPaymentCard(text, range)) {
|
|
||||||
labels.add("dlp:payment-card");
|
|
||||||
blocked = true;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (isEncodedContainer(bytes)) labels.add("scan:contents-not-inspected");
|
return finishScan(state, bytes.subarray(0, 12));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The same scan over a file on disk, one window in memory at a time and without blocking; the hash comes from the same reads, so both describe one set of bytes. */
|
||||||
|
export async function scanFileAt(path: string, name: string): Promise<{ scan: FileScanResult; sha256: string; size: number }> {
|
||||||
|
const handle = await open(path, "r");
|
||||||
|
try {
|
||||||
|
const size = (await handle.stat()).size;
|
||||||
|
const state = { labels: new Set(scanLabels(name, size)), blocked: false };
|
||||||
|
const hash = createHash("sha256");
|
||||||
|
let head = Buffer.alloc(0);
|
||||||
|
for (let start = 0; start === 0 || start < size; start += SCAN_WINDOW_BYTES) {
|
||||||
|
const spanStart = Math.max(0, start - SCAN_CONTEXT_BYTES);
|
||||||
|
const spanEnd = Math.min(size, start + SCAN_WINDOW_BYTES + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES);
|
||||||
|
const span = Buffer.alloc(spanEnd - spanStart);
|
||||||
|
const { bytesRead } = await handle.read(span, 0, span.length, spanStart);
|
||||||
|
if (bytesRead !== span.length) throw new Error(`${name} changed size while it was being scanned.`);
|
||||||
|
if (start === 0) head = Buffer.from(span.subarray(0, 12));
|
||||||
|
// Each window's own range follows the last, so the hash sees every byte once, in order.
|
||||||
|
hash.update(span.subarray(start - spanStart, Math.min(size, start + SCAN_WINDOW_BYTES) - spanStart));
|
||||||
|
scanWindow(state, size, start, (from, to) => span.subarray(from - spanStart, to - spanStart));
|
||||||
|
}
|
||||||
|
if ((await handle.stat()).size !== size) throw new Error(`${name} changed size while it was being scanned.`);
|
||||||
|
return { scan: finishScan(state, head), sha256: hash.digest("hex"), size };
|
||||||
|
} finally {
|
||||||
|
await handle.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One window owns the matches that start in its own MiB; context on both sides keeps word boundaries real at the cuts. */
|
||||||
|
function scanWindow(state: { labels: Set<string>; blocked: boolean }, size: number, start: number, read: (from: number, to: number) => Buffer): void {
|
||||||
|
const ownEnd = Math.min(size, start + SCAN_WINDOW_BYTES);
|
||||||
|
const lead = read(Math.max(0, start - SCAN_CONTEXT_BYTES), start).toString("utf8");
|
||||||
|
const own = read(start, ownEnd).toString("utf8");
|
||||||
|
const tail = read(ownEnd, Math.min(size, ownEnd + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES)).toString("utf8");
|
||||||
|
const text = lead + own + tail;
|
||||||
|
const range = { from: lead.length, until: lead.length + own.length };
|
||||||
|
if (read(start, Math.min(size, ownEnd + SCAN_OVERLAP_BYTES)).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) {
|
||||||
|
state.labels.add("av:eicar");
|
||||||
|
state.blocked = true;
|
||||||
|
}
|
||||||
|
for (const scan of TEXT_SCAN_PATTERNS) {
|
||||||
|
if (!matchesWithin(scan.pattern, text, range)) continue;
|
||||||
|
state.labels.add(scan.label);
|
||||||
|
if (scan.block) state.blocked = true;
|
||||||
|
}
|
||||||
|
if (containsPaymentCard(text, range)) {
|
||||||
|
state.labels.add("dlp:payment-card");
|
||||||
|
state.blocked = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function finishScan(state: { labels: Set<string>; blocked: boolean }, head: Buffer): FileScanResult {
|
||||||
|
const { labels, blocked } = state;
|
||||||
|
if (isEncodedContainer(head)) labels.add("scan:contents-not-inspected");
|
||||||
labels.add("scan:whole-file");
|
labels.add("scan:whole-file");
|
||||||
labels.add(blocked ? "classification:restricted" : labels.has("dlp:email") ? "classification:confidential" : "classification:internal");
|
labels.add(blocked ? "classification:restricted" : labels.has("dlp:email") ? "classification:confidential" : "classification:internal");
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -459,7 +459,7 @@ describe("domain services", () => {
|
|||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
test("imports files with checksum, scan labels, object refs, and workspace grants", () => {
|
test("imports files with checksum, scan labels, object refs, and workspace grants", async () => {
|
||||||
const connection = createConnection();
|
const connection = createConnection();
|
||||||
const organization = new OrganizationService(connection.db);
|
const organization = new OrganizationService(connection.db);
|
||||||
const snapshot = new SnapshotService(connection.db);
|
const snapshot = new SnapshotService(connection.db);
|
||||||
@@ -483,8 +483,8 @@ describe("domain services", () => {
|
|||||||
goal: "Ship a verified control plane.",
|
goal: "Ship a verified control plane.",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
const cleanFileId = fileService.importPath(orgId, cleanPath, null);
|
const cleanFileId = await fileService.importPath(orgId, cleanPath, null);
|
||||||
const blockedFileId = fileService.importPath(orgId, secretPath, null);
|
const blockedFileId = await fileService.importPath(orgId, secretPath, null);
|
||||||
|
|
||||||
const state = snapshot.load();
|
const state = snapshot.load();
|
||||||
const cleanFile = state.files.find((file) => file.id === cleanFileId);
|
const cleanFile = state.files.find((file) => file.id === cleanFileId);
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
|
import { createHash, randomUUID } from "node:crypto";
|
||||||
|
import { appendFileSync, writeFileSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
import { describe, expect, test } from "vitest";
|
import { describe, expect, test } from "vitest";
|
||||||
import { scanFileContent } from "../../src/main/services/files/file-object-store";
|
import { scanFileAt, scanFileContent } from "../../src/main/services/files/file-object-store";
|
||||||
|
|
||||||
const MIB = 1024 * 1024;
|
const MIB = 1024 * 1024;
|
||||||
|
|
||||||
@@ -41,3 +45,30 @@ describe("the file scan", () => {
|
|||||||
expect(scanFileContent("plan.md", Buffer.from("# Plan")).labels).not.toContain("scan:contents-not-inspected");
|
expect(scanFileContent("plan.md", Buffer.from("# Plan")).labels).not.toContain("scan:contents-not-inspected");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("scanning a file on disk", () => {
|
||||||
|
test("streams the same verdict and hash as scanning it in memory", async () => {
|
||||||
|
const path = join(tmpdir(), `ac-scan-${randomUUID()}.bin`);
|
||||||
|
const content = Buffer.concat([Buffer.from("PK\u0003\u0004"), Buffer.alloc(2 * MIB, "x"), Buffer.from(" AKIAABCDEFGHIJKLMNOP ")]);
|
||||||
|
writeFileSync(path, content);
|
||||||
|
const streamed = await scanFileAt(path, "bundle.zip");
|
||||||
|
const inMemory = scanFileContent("bundle.zip", content);
|
||||||
|
expect({ state: streamed.scan.state, labels: streamed.scan.labels }).toEqual({ state: inMemory.state, labels: inMemory.labels });
|
||||||
|
expect(streamed.scan.labels).toEqual(expect.arrayContaining(["dlp:aws-access-key", "scan:contents-not-inspected"]));
|
||||||
|
expect(streamed.sha256).toBe(createHash("sha256").update(content).digest("hex"));
|
||||||
|
expect(streamed.size).toBe(content.length);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("a file that changes while it is scanned", () => {
|
||||||
|
test("is never approved without its new bytes having been scanned", async () => {
|
||||||
|
const path = join(tmpdir(), `ac-grow-${randomUUID()}.txt`);
|
||||||
|
writeFileSync(path, Buffer.alloc(8 * MIB, "x"));
|
||||||
|
const scanning = scanFileAt(path, "grow.txt").then(
|
||||||
|
(result) => (result.scan.state === "blocked" ? "blocked" : "approved"),
|
||||||
|
(error: Error) => error.message,
|
||||||
|
);
|
||||||
|
appendFileSync(path, "\nAKIAABCDEFGHIJKLMNOP\n");
|
||||||
|
expect(["blocked", "grow.txt changed size while it was being scanned."]).toContain(await scanning);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
import { mkdirSync, writeFileSync } from "node:fs";
|
import { mkdirSync, readdirSync, rmSync, truncateSync, writeFileSync } from "node:fs";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { describe, expect, test, vi } from "vitest";
|
import { describe, expect, test, vi } from "vitest";
|
||||||
@@ -13,7 +13,7 @@ const { FileService } = await import("../../src/main/services/file-service");
|
|||||||
const { dispatchAgentTool } = await import("../../src/main/services/agent-tools");
|
const { dispatchAgentTool } = await import("../../src/main/services/agent-tools");
|
||||||
|
|
||||||
describe("files shared in a private conversation", () => {
|
describe("files shared in a private conversation", () => {
|
||||||
test("a file imported into a direct conversation is readable by its members only", () => {
|
test("a file imported into a direct conversation is readable by its members only", async () => {
|
||||||
const connection = new DatabaseConnection(join(tmpdir(), `ac-private-${randomUUID()}.db`));
|
const connection = new DatabaseConnection(join(tmpdir(), `ac-private-${randomUUID()}.db`));
|
||||||
const organization = new OrganizationService(connection.db);
|
const organization = new OrganizationService(connection.db);
|
||||||
const chat = new ChatService(connection.db);
|
const chat = new ChatService(connection.db);
|
||||||
@@ -33,7 +33,7 @@ describe("files shared in a private conversation", () => {
|
|||||||
mkdirSync(source, { recursive: true });
|
mkdirSync(source, { recursive: true });
|
||||||
writeFileSync(join(source, "salary.md"), "Dana's raise: 10%");
|
writeFileSync(join(source, "salary.md"), "Dana's raise: 10%");
|
||||||
const artifacts = join(tmpdir(), `ac-private-art-${randomUUID()}`);
|
const artifacts = join(tmpdir(), `ac-private-art-${randomUUID()}`);
|
||||||
const fileId = new FileService(connection.db, artifacts).importPath(orgId, join(source, "salary.md"), dm);
|
const fileId = await new FileService(connection.db, artifacts).importPath(orgId, join(source, "salary.md"), dm);
|
||||||
|
|
||||||
const deps = { db: connection.db, organization, chat };
|
const deps = { db: connection.db, organization, chat };
|
||||||
const as = (agentId: string) => ({ orgId, agent: { id: agentId, modelProviderId: null, modelId: null }, taskProjectId: null, taskGoalId: null, taskId: null, sourceThreadId: null, workspace: null });
|
const as = (agentId: string) => ({ orgId, agent: { id: agentId, modelProviderId: null, modelId: null }, taskProjectId: null, taskGoalId: null, taskId: null, sourceThreadId: null, workspace: null });
|
||||||
@@ -50,4 +50,20 @@ describe("files shared in a private conversation", () => {
|
|||||||
.toEqual([{ type: "agent" }, { type: "employee" }]);
|
.toEqual([{ type: "agent" }, { type: "employee" }]);
|
||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("a file over the import limit is refused before anything is copied", async () => {
|
||||||
|
const connection = new DatabaseConnection(join(tmpdir(), `ac-import-limit-${randomUUID()}.db`));
|
||||||
|
let orgId = "";
|
||||||
|
connection.transaction(() => {
|
||||||
|
orgId = new OrganizationService(connection.db).createOrganization({ organizationName: "Acme", employeeName: "Yang", mission: "m", goal: "g" });
|
||||||
|
});
|
||||||
|
const huge = join(tmpdir(), `ac-huge-${randomUUID()}.bin`);
|
||||||
|
writeFileSync(huge, "");
|
||||||
|
truncateSync(huge, 1024 * 1024 * 1024 + 1);
|
||||||
|
const artifacts = join(tmpdir(), `ac-import-limit-art-${randomUUID()}`);
|
||||||
|
await expect(new FileService(connection.db, artifacts).importPath(orgId, huge, null)).rejects.toThrow("files over 1024.0 MiB cannot be imported");
|
||||||
|
expect(readdirSync(artifacts, { recursive: true })).toEqual([]);
|
||||||
|
rmSync(huge);
|
||||||
|
connection.close();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
Reference in new issue
Block a user