From 3353e460674b0f7a0bd6769522cf118ae32a7451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E7=94=B5=E8=8A=BD=E8=A1=A3?= Date: Mon, 20 Apr 2026 16:25:48 -0400 Subject: [PATCH] Backfill existing workspace files into work_products on first boot after observationPath fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1f2f977 hotfix repairs the projector for new runs, but files that already exist on disk before the fix lands stay invisible to the DocumentCenter — a baseline+diff projector reports "changed" only when a file's mtime advances past the run-start baseline, and a quiescent file written before the fix never advances. This commit adds a bootstrap-time one-shot rescan: - New module `bootstrap/backfill-workspace-artifacts.ts` walks the primary/oldest workspace of every company exactly once (gated by `artifact_rescan_v1_done` in the settings table), applies the same registrability filter as `artifact-projector.ts` (skip .git/, node_modules/, .DS_Store, .lock/.log/.map; ignore package-lock.json / pnpm-lock.yaml / yarn.lock; cap at 5 000 files per workspace), and upserts a `work_products` row via `findWorkProductByExternalId` → `createWorkProduct` for any file the projector never registered. - bootstrap-application.ts wires the call after recoverScheduledPosts (DB ready, services running, before the first heartbeat tick), so the rescan completes in the same boot tick that the user opens the DocumentCenter. Idempotency: subsequent boots short-circuit on the settings flag. The post-fix projector handles all future file activity inside each run, so the backfill never needs to run twice. Per-workspace file cap prevents a pathological tree from locking the bootstrap path; overflow files are still picked up by the next run that touches them. No fallback, no mock — files that fail stat are silently skipped (same policy as the projector). typecheck green, 543/543 unit tests, 10/10 eval. The user's 19 existing markdown deliverables under ~/Documents/Github/test/ should appear in the DocumentCenter on the next `pnpm dev`. --- src/main/bootstrap-application.ts | 8 + .../bootstrap/backfill-workspace-artifacts.ts | 226 ++++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 src/main/bootstrap/backfill-workspace-artifacts.ts diff --git a/src/main/bootstrap-application.ts b/src/main/bootstrap-application.ts index c2ceaf03..c4141057 100644 --- a/src/main/bootstrap-application.ts +++ b/src/main/bootstrap-application.ts @@ -17,6 +17,7 @@ import { prepareDatabase } from "./bootstrap/prepare-database"; import { prepareServices } from "./bootstrap/prepare-services"; import { wireApiServer, type CreateApiServerFn } from "./bootstrap/wire-api-server"; import { recoverScheduledPosts } from "./bootstrap/recover-scheduled-posts"; +import { backfillWorkspaceArtifacts } from "./bootstrap/backfill-workspace-artifacts"; import { runStartupHealthChecks } from "./bootstrap/run-startup-health-checks"; import { emitCrashRecoveryNotification, recoverShutdownInterruptedTasks } from "./bootstrap/recover-interrupted-tasks"; @@ -144,6 +145,13 @@ export async function bootstrapApplication(deps: BootstrapApplicationDependencie getBrowserActionService: () => serviceGraph.getBrowserActionService(), }); + // One-time backfill: surface deliverables that cognitive runs wrote + // to disk while the projector was suppressed by the pre-observationPath + // bug (regression of T1 workspace fix, repaired in 2c12c1a). Gated by + // a settings flag so subsequent boots skip the workspace walk; the + // post-fix projector handles new file activity inside each run. + backfillWorkspaceArtifacts({ db, logger: log }); + // Register cleanup for bootstrap-owned resources during quit. // Runs BEFORE service-graph's onBeforeQuit (which is registered later). app.on("before-quit", () => { diff --git a/src/main/bootstrap/backfill-workspace-artifacts.ts b/src/main/bootstrap/backfill-workspace-artifacts.ts new file mode 100644 index 00000000..0bc844f6 --- /dev/null +++ b/src/main/bootstrap/backfill-workspace-artifacts.ts @@ -0,0 +1,226 @@ +import { existsSync, readdirSync, statSync } from "node:fs"; +import { relative, resolve } from "node:path"; +import type { AppDatabase } from "../database"; + +/** + * One-shot rescan that surfaces files written by cognitive runs whose + * artifact projector was suppressed by the pre-`observationPath` bug + * (regression of T1, fixed in commit 2c12c1a). + * + * The post-fix projector observes per-run via mtime baselines. Files + * that already existed on disk before the projector was repaired never + * advance their mtime past the new run's baseline, so they remain + * invisible to subsequent runs — they are real deliverables that the + * DocumentCenter still does not list. This module walks every company + * primary workspace once at startup, applies the same registrability + * filter as `artifact-projector.ts`, and upserts a `work_products` row + * for each file that is missing one. + * + * Idempotent through `findWorkProductByExternalId` — a file that was + * already projected by some earlier run will short-circuit on lookup + * and be skipped. Bounded by `MAX_BACKFILL_FILES_PER_WORKSPACE` so a + * pathological workspace tree cannot lock the bootstrap path. + * + * Gated by a one-time settings flag (`artifact_rescan_v1_done`) so a + * second startup does not re-walk every workspace tree on disk; the + * projector handles all subsequent file activity. + */ + +const SETTINGS_FLAG_KEY = "artifact_rescan_v1_done"; +const MAX_BACKFILL_FILES_PER_WORKSPACE = 5_000; +const SKIP_DIR_NAMES: ReadonlySet = new Set([ + ".git", + "node_modules", + ".next", + "dist", + "build", + "out", + ".cache", + ".pnpm-store", +]); +const IGNORED_PATH_SEGMENTS: readonly string[] = [ + "/.git/", + "/node_modules/", + "/.next/", + "/dist/", + "/build/", + "/out/", + "/.cache/", + "/.pnpm-store/", + "/.DS_Store", +]; +const IGNORED_EXACT_NAMES: ReadonlySet = new Set([ + ".DS_Store", + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + ".gitignore", +]); +const IGNORED_EXTENSIONS: ReadonlySet = new Set([ + ".lock", + ".log", + ".map", + ".min.js", + ".min.css", +]); + +interface Logger { + info: (...args: unknown[]) => void; + warn: (...args: unknown[]) => void; +} + +export interface BackfillWorkspaceArtifactsDeps { + db: AppDatabase; + logger: Logger; +} + +export interface BackfillWorkspaceArtifactsResult { + /** True when the rescan ran (first launch after the fix). */ + ran: boolean; + /** Files inspected across every company workspace. */ + filesScanned: number; + /** Files inserted as work_products (previously missing rows). */ + filesInserted: number; + /** Files skipped because a row already existed (idempotency). */ + filesAlreadyTracked: number; + /** Files skipped by the registrability filter. */ + filesIgnored: number; +} + +export function backfillWorkspaceArtifacts( + deps: BackfillWorkspaceArtifactsDeps, +): BackfillWorkspaceArtifactsResult { + const empty: BackfillWorkspaceArtifactsResult = { + ran: false, + filesScanned: 0, + filesInserted: 0, + filesAlreadyTracked: 0, + filesIgnored: 0, + }; + const flag = deps.db.settings.getSetting(SETTINGS_FLAG_KEY); + if (flag === "true") return empty; + + let filesScanned = 0; + let filesInserted = 0; + let filesAlreadyTracked = 0; + let filesIgnored = 0; + + const snapshot = deps.db.snap.listSnapshot(); + for (const company of snapshot.companies) { + const observation = deps.db.workspacesMod.getCompanyObservationWorkspace(company.id); + if (!observation) continue; + const workspaceRoot = observation.localPath; + if (!workspaceRoot || !existsSync(workspaceRoot)) continue; + let stat; + try { + stat = statSync(workspaceRoot); + } catch { + continue; + } + if (!stat.isDirectory()) continue; + + const stack: string[] = [workspaceRoot]; + let countForThisWorkspace = 0; + while (stack.length > 0) { + if (countForThisWorkspace >= MAX_BACKFILL_FILES_PER_WORKSPACE) { + deps.logger.warn( + `[backfill-workspace-artifacts] hit per-workspace cap (${MAX_BACKFILL_FILES_PER_WORKSPACE}) for company=${company.id} workspace=${observation.id}; remaining files will be picked up by future runs as they are touched`, + ); + break; + } + const dir = stack.pop(); + if (!dir) break; + let entries: { name: string; isDirectory: () => boolean; isSymbolicLink: () => boolean }[]; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + if (entry.isSymbolicLink()) continue; + const abs = resolve(dir, entry.name); + if (entry.isDirectory()) { + if (SKIP_DIR_NAMES.has(entry.name)) continue; + stack.push(abs); + continue; + } + try { + const fileStat = statSync(abs); + if (!fileStat.isFile()) continue; + } catch { + continue; + } + filesScanned += 1; + countForThisWorkspace += 1; + + const rel = relative(workspaceRoot, abs); + if (rel.length === 0) { + filesIgnored += 1; + continue; + } + const probe = `/${rel}/`; + if (IGNORED_PATH_SEGMENTS.some((seg) => probe.includes(seg))) { + filesIgnored += 1; + continue; + } + const lastSlash = rel.lastIndexOf("/"); + const basename = lastSlash >= 0 ? rel.slice(lastSlash + 1) : rel; + if (IGNORED_EXACT_NAMES.has(basename)) { + filesIgnored += 1; + continue; + } + const dot = basename.lastIndexOf("."); + if (dot > 0) { + const ext = basename.slice(dot).toLowerCase(); + if (IGNORED_EXTENSIONS.has(ext)) { + filesIgnored += 1; + continue; + } + const prevDot = basename.lastIndexOf(".", dot - 1); + if (prevDot > 0) { + const compound = basename.slice(prevDot).toLowerCase(); + if (IGNORED_EXTENSIONS.has(compound)) { + filesIgnored += 1; + continue; + } + } + } + + const existing = deps.db.documentsMod.findWorkProductByExternalId( + company.id, + "filesystem", + rel, + observation.id, + ); + if (existing) { + filesAlreadyTracked += 1; + continue; + } + const title = lastSlash >= 0 ? rel.slice(lastSlash + 1) : rel; + deps.db.documentsMod.createWorkProduct({ + companyId: company.id, + taskId: null, + workspaceId: observation.id, + type: "file", + provider: "filesystem", + externalId: rel, + title, + url: null, + }); + filesInserted += 1; + } + } + } + + deps.db.settings.setSetting(SETTINGS_FLAG_KEY, "true"); + deps.logger.info( + `[backfill-workspace-artifacts] scanned=${filesScanned} inserted=${filesInserted} already-tracked=${filesAlreadyTracked} ignored=${filesIgnored}`, + ); + return { + ran: true, + filesScanned, + filesInserted, + filesAlreadyTracked, + filesIgnored, + }; +}