ZacharyZhang-NY/AgentCompany
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Q124: the file scan covers the whole file and says what it did and did not check

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 20:09:03 -04:00
1 parent 91fa6b5f1e
commit 7a9059fbea
9 files changed
+175 -31
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/main/services/files/file-object-store.ts
+75 -22
View File
@@ -19,14 +19,24 @@ interface TextScanPattern {
block: boolean;
}
const SCAN_WINDOW_BYTES = 1024 * 1024;
const SCAN_OVERLAP_BYTES = 4096;
const SCAN_CONTEXT_BYTES = 64;
/** Zip (office documents too), PDF, gzip, 7z, RAR, bzip2, xz, PNG, JPEG and GIF. */
const CONTAINER_SIGNATURES = [
[0x50, 0x4b, 0x03, 0x04], [0x50, 0x4b, 0x05, 0x06], [0x25, 0x50, 0x44, 0x46], [0x1f, 0x8b], [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c],
[0x52, 0x61, 0x72, 0x21], [0x42, 0x5a, 0x68], [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00], [0x89, 0x50, 0x4e, 0x47], [0xff, 0xd8, 0xff], [0x47, 0x49, 0x46, 0x38],
];
// Every quantifier is bounded, so no match is longer than the window overlap and none is cut in two.
const TEXT_SCAN_PATTERNS: TextScanPattern[] = [
{ label: "dlp:private-key", pattern: /-----BEGIN (?:RSA |DSA |EC |OPENSSH |PRIVATE )?PRIVATE KEY-----/i, block: true },
{ label: "dlp:aws-access-key", pattern: /\bAKIA[0-9A-Z]{16}\b/, block: true },
{ label: "dlp:github-token", pattern: /\bgh[pousr]_[A-Za-z0-9_]{30,}\b/, block: true },
{ label: "dlp:provider-secret", pattern: /\bsk-[A-Za-z0-9]{32,}\b/, block: true },
{ label: "dlp:credential", pattern: /\b(?:api[_-]?key|secret|token|password)\s*[:=]\s*["']?[A-Za-z0-9_\-./+=]{20,}/i, block: true },
{ label: "dlp:github-token", pattern: /\bgh[pousr]_[A-Za-z0-9_]{30}/, block: true },
{ label: "dlp:provider-secret", pattern: /\bsk-[A-Za-z0-9]{32}/, block: true },
{ label: "dlp:credential", pattern: /\b(?:api[_-]?key|secret|token|password)[ \t]{0,16}[:=][ \t]{0,16}["']?[A-Za-z0-9_\-./+=]{20}/i, block: true },
{ label: "dlp:ssn", pattern: /\b\d{3}-\d{2}-\d{4}\b/, block: true },
{ label: "dlp:email", pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i, block: false },
{ label: "dlp:email", pattern: /\b[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,24}\b/i, block: false },
];
export function fileKind(path: string): string {
@@ -40,23 +50,35 @@ export function fileKind(path: string): string {
return ext;
}
/** Pattern checks for secrets and the antivirus test string over the whole file; not an antivirus engine, and compressed contents are not opened. */
export function scanFileContent(name: string, bytes: Buffer): FileScanResult {
const textSample = sampleText(bytes);
const labels = new Set(scanLabels(name, bytes.length));
let blocked = false;
if (bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) {
labels.add("av:eicar");
blocked = true;
}
for (const scan of TEXT_SCAN_PATTERNS) {
if (!scan.pattern.test(textSample)) continue;
labels.add(scan.label);
if (scan.block) blocked = true;
}
if (containsPaymentCard(textSample)) {
labels.add("dlp:payment-card");
blocked = true;
// Overlapping windows cover every byte with bounded memory; the overlap is longer than any pattern match.
for (let start = 0; start === 0 || start < bytes.length; start += SCAN_WINDOW_BYTES) {
// A window owns the matches that start in its own MiB; context on both sides keeps word boundaries real at the cuts.
const ownEnd = Math.min(bytes.length, start + SCAN_WINDOW_BYTES);
const lead = bytes.subarray(Math.max(0, start - SCAN_CONTEXT_BYTES), start).toString("utf8");
const own = bytes.subarray(start, ownEnd).toString("utf8");
const tail = bytes.subarray(ownEnd, Math.min(bytes.length, ownEnd + SCAN_OVERLAP_BYTES + SCAN_CONTEXT_BYTES)).toString("utf8");
const text = lead + own + tail;
const range = { from: lead.length, until: lead.length + own.length };
if (bytes.subarray(start, ownEnd + SCAN_OVERLAP_BYTES).toString("latin1").includes("EICAR-STANDARD-ANTIVIRUS-TEST-FILE")) {
labels.add("av:eicar");
blocked = true;
}
for (const scan of TEXT_SCAN_PATTERNS) {
if (!matchesWithin(scan.pattern, text, range)) continue;
labels.add(scan.label);
if (scan.block) blocked = true;
}
if (containsPaymentCard(text, range)) {
labels.add("dlp:payment-card");
blocked = true;
}
}
if (isEncodedContainer(bytes)) labels.add("scan:contents-not-inspected");
labels.add("scan:whole-file");
labels.add(blocked ? "classification:restricted" : labels.has("dlp:email") ? "classification:confidential" : "classification:internal");
return {
state: blocked ? FILE_SCAN_STATE.BLOCKED : FILE_SCAN_STATE.APPROVED,
@@ -209,6 +231,22 @@ function keepCopy(workspaceRoot: string, source: string, sha256: string, bytes:
return copy;
}
/** Files scanned before whole-file scanning are scanned again when their bytes are still the recorded ones; the rest are marked as sampled. */
export function rescanLegacyFiles(db: DatabaseSync): void {
const rows = db.prepare("select id, name, sha256, storage_path, scan_labels_json from file_objects where scan_labels_json not like '%scan:whole-file%'")
.all() as Array<{ id: string; name: string; sha256: string; storage_path: string; scan_labels_json: string }>;
for (const row of rows) {
const bytes = existsSync(row.storage_path) ? readFileSync(row.storage_path) : null;
if (bytes && createHash("sha256").update(bytes).digest("hex") === row.sha256) {
const scan = scanFileContent(row.name, bytes);
db.prepare("update file_objects set scan_state = ?, scan_labels_json = ?, scanned_at = ? where id = ?").run(scan.state, JSON.stringify(scan.labels), scan.scannedAt, row.id);
continue;
}
const labels = [...new Set([...(JSON.parse(row.scan_labels_json) as string[]), "scan:legacy-sample"])].sort();
db.prepare("update file_objects set scan_labels_json = ? where id = ?").run(JSON.stringify(labels), row.id);
}
}
/** Ids recorded before copies existed still point at the live file; those whose bytes are unchanged get their copy now. */
export function preserveRecordedFiles(db: DatabaseSync): void {
const workspaces = db.prepare("select org_id, local_path from workspaces").all() as Array<{ org_id: string; local_path: string }>;
@@ -283,13 +321,28 @@ function scanLabels(name: string, sizeBytes: number): string[] {
return [...labels].sort();
}
function sampleText(bytes: Buffer): string {
return bytes.subarray(0, Math.min(bytes.length, 1024 * 1024)).toString("utf8");
/** Archives, office documents, PDFs and images keep their contents compressed or encoded, so a byte scan cannot see inside them. */
function isEncodedContainer(bytes: Buffer): boolean {
const head = bytes.subarray(0, 12);
const starts = (signature: number[]): boolean => signature.every((byte, index) => head[index] === byte);
return CONTAINER_SIGNATURES.some(starts) || (starts([0x52, 0x49, 0x46, 0x46]) && head.subarray(8, 12).toString("latin1") === "WEBP");
}
function containsPaymentCard(text: string): boolean {
const matches = text.match(/\b(?:\d[ -]?){13,19}\b/g) ?? [];
return matches.some((match) => luhnValid(match.replace(/[ -]/g, "")));
/** Whether a match starts in the range; text around it still counts as context for boundaries. */
function matchesWithin(pattern: RegExp, text: string, range: { from: number; until: number }): boolean {
const search = new RegExp(pattern.source, `${pattern.flags}g`);
search.lastIndex = range.from;
const match = search.exec(text);
return match !== null && match.index < range.until;
}
function containsPaymentCard(text: string, range: { from: number; until: number }): boolean {
const search = /\b(?:\d[ -]?){13,19}\b/g;
search.lastIndex = range.from;
for (let match = search.exec(text); match && match.index < range.until; match = search.exec(text)) {
if (luhnValid(match[0].replace(/[ -]/g, ""))) return true;
}
return false;
}
function luhnValid(value: string): boolean {