diff --git a/src/main/services/search-service.ts b/src/main/services/search-service.ts index 6e0760e9..45e7424c 100644 --- a/src/main/services/search-service.ts +++ b/src/main/services/search-service.ts @@ -1,6 +1,8 @@ +import { extname } from "node:path"; import type { DatabaseSync } from "node:sqlite"; +import { readVerifiedFile } from "./files/file-object-store"; import type { SearchInput } from "../../shared/commands"; -import { SEARCH_ENTITY_TYPES, type SearchEntityType, type SearchPage } from "../../shared/domain"; +import { FILE_SCAN_STATE, SEARCH_ENTITY_TYPES, type SearchEntityType, type SearchPage } from "../../shared/domain"; interface SearchRow { entity_type: SearchEntityType; @@ -18,6 +20,9 @@ interface SearchDoc { facets: Record; } +const TEXT_EXTENSIONS = new Set(["md", "markdown", "txt", "csv", "tsv", "json", "yaml", "yml", "toml", "ini", "xml", "html", "css", "js", "jsx", "ts", "tsx", "py", "sql", "sh", "log"]); +const FILE_TEXT_LIMIT_BYTES = 2 * 1024 * 1024; + export class SearchService { constructor(private readonly db: DatabaseSync) {} @@ -137,8 +142,8 @@ export class SearchService { private fileDocs(orgId: string): SearchDoc[] { // Superseded versions are kept for records but listed nowhere, so they are not search hits either. - const rows = this.db.prepare("select id, name, kind, sha256, scan_state from file_objects where org_id = ? and superseded_at is null").all(orgId) as Array<{ id: string; name: string; kind: string; sha256: string; scan_state: string }>; - return rows.map((row) => doc("file", row.id, row.name, `${row.kind} ${row.sha256}`, { type: "Files", state: row.scan_state })); + const rows = this.db.prepare("select id, name, kind, sha256, scan_state, storage_path, size_bytes from file_objects where org_id = ? and superseded_at is null").all(orgId) as Array<{ id: string; name: string; kind: string; sha256: string; scan_state: string; storage_path: string; size_bytes: number }>; + return rows.map((row) => doc("file", row.id, row.name, `${fileText(row)} ${row.kind} ${row.sha256}`, { type: "Files", state: row.scan_state })); } private providerDocs(orgId: string): SearchDoc[] { @@ -219,6 +224,18 @@ function parseQuery(query: string): { blank: boolean; match: string | null; cont return { blank: !match && contains.length === 0, match, contains }; } +/** The text of an approved text file, so it can be found by what it says; a blocked file's content stays behind the scan gate, as it does for opening. */ +function fileText(file: { name: string; storage_path: string; sha256: string; size_bytes: number; scan_state: string }): string { + if (file.scan_state !== FILE_SCAN_STATE.APPROVED) return ""; + if (!TEXT_EXTENSIONS.has(extname(file.name).slice(1).toLowerCase()) || file.size_bytes > FILE_TEXT_LIMIT_BYTES) return ""; + try { + return readVerifiedFile({ name: file.name, storagePath: file.storage_path, sha256: file.sha256 }).toString("utf8"); + } catch (error) { + console.warn(`[search] ${file.name}: content not indexed: ${error instanceof Error ? error.message : String(error)}`); + return ""; + } +} + function validEntityTypes(entityTypes: SearchEntityType[]): Set { const allowed = new Set(SEARCH_ENTITY_TYPES); return new Set(entityTypes.filter((type) => allowed.has(type))); diff --git a/tests/unit/search-service.test.ts b/tests/unit/search-service.test.ts index 325d18f9..bbd7e712 100644 --- a/tests/unit/search-service.test.ts +++ b/tests/unit/search-service.test.ts @@ -1,4 +1,6 @@ import { randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; +import { mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "vitest"; @@ -236,4 +238,42 @@ describe("search service", () => { expect(tasks.results.map((hit) => hit.title)).toEqual(["Harbor survey"]); connection.close(); }); + + test("finds a text file by what it says, read from its verified copy", () => { + const connection = createConnection(); + const organization = new OrganizationService(connection.db); + const search = new SearchService(connection.db); + let orgId = ""; + connection.transaction(() => { + orgId = organization.createOrganization({ organizationName: "Content Co", employeeName: "Dana Lee", mission: "m", goal: "g" }); + }); + const employeeId = (connection.db.prepare("select id from employees where org_id = ?").get(orgId) as { id: string }).id; + const folder = join(tmpdir(), `ac-search-files-${randomUUID()}`); + mkdirSync(folder, { recursive: true }); + const stored = (id: string, name: string, text: string, scanState = "approved") => { + const path = join(folder, name); + writeFileSync(path, text); + connection.db.prepare(` + insert into file_objects(id, org_id, name, kind, size_bytes, sha256, storage_path, owner_type, owner_id, created_at, scan_state) + values (?, ?, ?, 'doc', ?, ?, ?, 'employee', ?, ?, ?) + `).run(id, orgId, name, Buffer.byteLength(text), createHash("sha256").update(text).digest("hex"), path, employeeId, new Date().toISOString(), scanState); + return path; + }; + stored("fil_notes", "notes.md", "Quokka migration checklist"); + stored("fil_scan", "scan.pdf", "Quokka migration checklist"); + stored("fil_key", "deploy-key.txt", "-----BEGIN PRIVATE KEY----- walrussecret", "blocked"); + const changed = stored("fil_changed", "changed.txt", "Quokka original words"); + writeFileSync(changed, "Quokka rewritten behind the app's back"); + const ids = (query: string) => search.search({ orgId, employeeId, query, entityTypes: ["file"] }).results.map((hit) => hit.entityId).sort(); + expect(ids("checklist")).toEqual(["fil_notes"]); + // Bytes that no longer match their record are not indexed; the file is still found by name. + expect(ids("rewritten")).toEqual([]); + expect(ids("changed")).toEqual(["fil_changed"]); + // A blocked file's content stays behind the scan gate; its name is still searchable. + expect(ids("walrussecret")).toEqual([]); + const blocked = search.search({ orgId, employeeId, query: "deploy", entityTypes: ["file"] }).results; + expect(blocked.map((hit) => hit.entityId)).toEqual(["fil_key"]); + expect(blocked[0]!.preview).not.toContain("PRIVATE KEY"); + connection.close(); + }); });