merge: wave 2 executor (01-02) into main

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

paperclip-reference/cli/src/config/data-dir.ts

@@ -0,0 +1,48 @@
+import path from "node:path";
+import {
+  expandHomePrefix,
+  resolveDefaultConfigPath,
+  resolveDefaultContextPath,
+  resolvePaperclipInstanceId,
+} from "./home.js";
+
+export interface DataDirOptionLike {
+  dataDir?: string;
+  config?: string;
+  context?: string;
+  instance?: string;
+}
+
+export interface DataDirCommandSupport {
+  hasConfigOption?: boolean;
+  hasContextOption?: boolean;
+}
+
+export function applyDataDirOverride(
+  options: DataDirOptionLike,
+  support: DataDirCommandSupport = {},
+): string | null {
+  const rawDataDir = options.dataDir?.trim();
+  if (!rawDataDir) return null;
+
+  const resolvedDataDir = path.resolve(expandHomePrefix(rawDataDir));
+  process.env.PAPERCLIP_HOME = resolvedDataDir;
+
+  if (support.hasConfigOption) {
+    const hasConfigOverride = Boolean(options.config?.trim()) || Boolean(process.env.PAPERCLIP_CONFIG?.trim());
+    if (!hasConfigOverride) {
+      const instanceId = resolvePaperclipInstanceId(options.instance);
+      process.env.PAPERCLIP_INSTANCE_ID = instanceId;
+      process.env.PAPERCLIP_CONFIG = resolveDefaultConfigPath(instanceId);
+    }
+  }
+
+  if (support.hasContextOption) {
+    const hasContextOverride = Boolean(options.context?.trim()) || Boolean(process.env.PAPERCLIP_CONTEXT?.trim());
+    if (!hasContextOverride) {
+      process.env.PAPERCLIP_CONTEXT = resolveDefaultContextPath();
+    }
+  }
+
+  return resolvedDataDir;
+}

paperclip-reference/cli/src/config/env.ts

@@ -0,0 +1,125 @@
+import fs from "node:fs";
+import path from "node:path";
+import { randomBytes } from "node:crypto";
+import { config as loadDotenv, parse as parseEnvFileContents } from "dotenv";
+import { resolveConfigPath } from "./store.js";
+
+const JWT_SECRET_ENV_KEY = "PAPERCLIP_AGENT_JWT_SECRET";
+function resolveEnvFilePath(configPath?: string) {
+  return path.resolve(path.dirname(resolveConfigPath(configPath)), ".env");
+}
+const loadedEnvFiles = new Set<string>();
+
+function isNonEmpty(value: unknown): value is string {
+  return typeof value === "string" && value.trim().length > 0;
+}
+
+function parseEnvFile(contents: string) {
+  try {
+    return parseEnvFileContents(contents);
+  } catch {
+    return {};
+  }
+}
+
+function formatEnvValue(value: string): string {
+  if (/^[A-Za-z0-9_./:@-]+$/.test(value)) {
+    return value;
+  }
+  return JSON.stringify(value);
+}
+
+function renderEnvFile(entries: Record<string, string>) {
+  const lines = [
+    "# Paperclip environment variables",
+    "# Generated by Paperclip CLI commands",
+    ...Object.entries(entries).map(([key, value]) => `${key}=${formatEnvValue(value)}`),
+    "",
+  ];
+  return lines.join("\n");
+}
+
+export function resolvePaperclipEnvFile(configPath?: string): string {
+  return resolveEnvFilePath(configPath);
+}
+
+export function resolveAgentJwtEnvFile(configPath?: string): string {
+  return resolveEnvFilePath(configPath);
+}
+
+export function loadPaperclipEnvFile(configPath?: string): void {
+  loadAgentJwtEnvFile(resolveEnvFilePath(configPath));
+}
+
+export function loadAgentJwtEnvFile(filePath = resolveEnvFilePath()): void {
+  if (loadedEnvFiles.has(filePath)) return;
+
+  if (!fs.existsSync(filePath)) return;
+  loadedEnvFiles.add(filePath);
+  loadDotenv({ path: filePath, override: false, quiet: true });
+}
+
+export function readAgentJwtSecretFromEnv(configPath?: string): string | null {
+  loadAgentJwtEnvFile(resolveEnvFilePath(configPath));
+  const raw = process.env[JWT_SECRET_ENV_KEY];
+  return isNonEmpty(raw) ? raw!.trim() : null;
+}
+
+export function readAgentJwtSecretFromEnvFile(filePath = resolveEnvFilePath()): string | null {
+  if (!fs.existsSync(filePath)) return null;
+
+  const raw = fs.readFileSync(filePath, "utf-8");
+  const values = parseEnvFile(raw);
+  const value = values[JWT_SECRET_ENV_KEY];
+  return isNonEmpty(value) ? value!.trim() : null;
+}
+
+export function ensureAgentJwtSecret(configPath?: string): { secret: string; created: boolean } {
+  const existingEnv = readAgentJwtSecretFromEnv(configPath);
+  if (existingEnv) {
+    return { secret: existingEnv, created: false };
+  }
+
+  const envFilePath = resolveEnvFilePath(configPath);
+  const existingFile = readAgentJwtSecretFromEnvFile(envFilePath);
+  const secret = existingFile ?? randomBytes(32).toString("hex");
+  const created = !existingFile;
+
+  if (!existingFile) {
+    writeAgentJwtEnv(secret, envFilePath);
+  }
+
+  return { secret, created };
+}
+
+export function writeAgentJwtEnv(secret: string, filePath = resolveEnvFilePath()): void {
+  mergePaperclipEnvEntries({ [JWT_SECRET_ENV_KEY]: secret }, filePath);
+}
+
+export function readPaperclipEnvEntries(filePath = resolveEnvFilePath()): Record<string, string> {
+  if (!fs.existsSync(filePath)) return {};
+  return parseEnvFile(fs.readFileSync(filePath, "utf-8"));
+}
+
+export function writePaperclipEnvEntries(entries: Record<string, string>, filePath = resolveEnvFilePath()): void {
+  const dir = path.dirname(filePath);
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(filePath, renderEnvFile(entries), {
+    mode: 0o600,
+  });
+}
+
+export function mergePaperclipEnvEntries(
+  entries: Record<string, string>,
+  filePath = resolveEnvFilePath(),
+): Record<string, string> {
+  const current = readPaperclipEnvEntries(filePath);
+  const next = {
+    ...current,
+    ...Object.fromEntries(
+      Object.entries(entries).filter(([, value]) => typeof value === "string" && value.trim().length > 0),
+    ),
+  };
+  writePaperclipEnvEntries(next, filePath);
+  return next;
+}

paperclip-reference/cli/src/config/home.ts

@@ -0,0 +1,80 @@
+import os from "node:os";
+import path from "node:path";
+
+const DEFAULT_INSTANCE_ID = "default";
+const INSTANCE_ID_RE = /^[a-zA-Z0-9_-]+$/;
+
+export function resolvePaperclipHomeDir(): string {
+  const envHome = process.env.PAPERCLIP_HOME?.trim();
+  if (envHome) return path.resolve(expandHomePrefix(envHome));
+  return path.resolve(os.homedir(), ".paperclip");
+}
+
+export function resolvePaperclipInstanceId(override?: string): string {
+  const raw = override?.trim() || process.env.PAPERCLIP_INSTANCE_ID?.trim() || DEFAULT_INSTANCE_ID;
+  if (!INSTANCE_ID_RE.test(raw)) {
+    throw new Error(
+      `Invalid instance id '${raw}'. Allowed characters: letters, numbers, '_' and '-'.`,
+    );
+  }
+  return raw;
+}
+
+export function resolvePaperclipInstanceRoot(instanceId?: string): string {
+  const id = resolvePaperclipInstanceId(instanceId);
+  return path.resolve(resolvePaperclipHomeDir(), "instances", id);
+}
+
+export function resolveDefaultConfigPath(instanceId?: string): string {
+  return path.resolve(resolvePaperclipInstanceRoot(instanceId), "config.json");
+}
+
+export function resolveDefaultContextPath(): string {
+  return path.resolve(resolvePaperclipHomeDir(), "context.json");
+}
+
+export function resolveDefaultCliAuthPath(): string {
+  return path.resolve(resolvePaperclipHomeDir(), "auth.json");
+}
+
+export function resolveDefaultEmbeddedPostgresDir(instanceId?: string): string {
+  return path.resolve(resolvePaperclipInstanceRoot(instanceId), "db");
+}
+
+export function resolveDefaultLogsDir(instanceId?: string): string {
+  return path.resolve(resolvePaperclipInstanceRoot(instanceId), "logs");
+}
+
+export function resolveDefaultSecretsKeyFilePath(instanceId?: string): string {
+  return path.resolve(resolvePaperclipInstanceRoot(instanceId), "secrets", "master.key");
+}
+
+export function resolveDefaultStorageDir(instanceId?: string): string {
+  return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "storage");
+}
+
+export function resolveDefaultBackupDir(instanceId?: string): string {
+  return path.resolve(resolvePaperclipInstanceRoot(instanceId), "data", "backups");
+}
+
+export function expandHomePrefix(value: string): string {
+  if (value === "~") return os.homedir();
+  if (value.startsWith("~/")) return path.resolve(os.homedir(), value.slice(2));
+  return value;
+}
+
+export function describeLocalInstancePaths(instanceId?: string) {
+  const resolvedInstanceId = resolvePaperclipInstanceId(instanceId);
+  const instanceRoot = resolvePaperclipInstanceRoot(resolvedInstanceId);
+  return {
+    homeDir: resolvePaperclipHomeDir(),
+    instanceId: resolvedInstanceId,
+    instanceRoot,
+    configPath: resolveDefaultConfigPath(resolvedInstanceId),
+    embeddedPostgresDataDir: resolveDefaultEmbeddedPostgresDir(resolvedInstanceId),
+    backupDir: resolveDefaultBackupDir(resolvedInstanceId),
+    logDir: resolveDefaultLogsDir(resolvedInstanceId),
+    secretsKeyFilePath: resolveDefaultSecretsKeyFilePath(resolvedInstanceId),
+    storageDir: resolveDefaultStorageDir(resolvedInstanceId),
+  };
+}

paperclip-reference/cli/src/config/hostnames.ts

@@ -0,0 +1,26 @@
+export function normalizeHostnameInput(raw: string): string {
+  const input = raw.trim();
+  if (!input) {
+    throw new Error("Hostname is required");
+  }
+
+  try {
+    const url = input.includes("://") ? new URL(input) : new URL(`http://${input}`);
+    const hostname = url.hostname.trim().toLowerCase();
+    if (!hostname) throw new Error("Hostname is required");
+    return hostname;
+  } catch {
+    throw new Error(`Invalid hostname: ${raw}`);
+  }
+}
+
+export function parseHostnameCsv(raw: string): string[] {
+  if (!raw.trim()) return [];
+  const unique = new Set<string>();
+  for (const part of raw.split(",")) {
+    const hostname = normalizeHostnameInput(part);
+    unique.add(hostname);
+  }
+  return Array.from(unique);
+}
+

paperclip-reference/cli/src/config/schema.ts

@@ -0,0 +1,28 @@
+export {
+  paperclipConfigSchema,
+  configMetaSchema,
+  llmConfigSchema,
+  databaseBackupConfigSchema,
+  databaseConfigSchema,
+  loggingConfigSchema,
+  serverConfigSchema,
+  authConfigSchema,
+  storageConfigSchema,
+  storageLocalDiskConfigSchema,
+  storageS3ConfigSchema,
+  secretsConfigSchema,
+  secretsLocalEncryptedConfigSchema,
+  type PaperclipConfig,
+  type LlmConfig,
+  type DatabaseBackupConfig,
+  type DatabaseConfig,
+  type LoggingConfig,
+  type ServerConfig,
+  type AuthConfig,
+  type StorageConfig,
+  type StorageLocalDiskConfig,
+  type StorageS3Config,
+  type SecretsConfig,
+  type SecretsLocalEncryptedConfig,
+  type ConfigMeta,
+} from "@paperclipai/shared";

paperclip-reference/cli/src/config/secrets-key.ts

@@ -0,0 +1,48 @@
+import { randomBytes } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import type { PaperclipConfig } from "./schema.js";
+import { resolveRuntimeLikePath } from "../utils/path-resolver.js";
+
+export type EnsureSecretsKeyResult =
+  | { status: "created"; path: string }
+  | { status: "existing"; path: string }
+  | { status: "skipped_env"; path: null }
+  | { status: "skipped_provider"; path: null };
+
+export function ensureLocalSecretsKeyFile(
+  config: Pick<PaperclipConfig, "secrets">,
+  configPath?: string,
+): EnsureSecretsKeyResult {
+  if (config.secrets.provider !== "local_encrypted") {
+    return { status: "skipped_provider", path: null };
+  }
+
+  const envMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY;
+  if (envMasterKey && envMasterKey.trim().length > 0) {
+    return { status: "skipped_env", path: null };
+  }
+
+  const keyFileOverride = process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE;
+  const configuredPath =
+    keyFileOverride && keyFileOverride.trim().length > 0
+      ? keyFileOverride.trim()
+      : config.secrets.localEncrypted.keyFilePath;
+  const keyFilePath = resolveRuntimeLikePath(configuredPath, configPath);
+
+  if (fs.existsSync(keyFilePath)) {
+    return { status: "existing", path: keyFilePath };
+  }
+
+  fs.mkdirSync(path.dirname(keyFilePath), { recursive: true });
+  fs.writeFileSync(keyFilePath, randomBytes(32).toString("base64"), {
+    encoding: "utf8",
+    mode: 0o600,
+  });
+  try {
+    fs.chmodSync(keyFilePath, 0o600);
+  } catch {
+    // best effort
+  }
+  return { status: "created", path: keyFilePath };
+}

paperclip-reference/cli/src/config/store.ts

@@ -0,0 +1,120 @@
+import fs from "node:fs";
+import path from "node:path";
+import { paperclipConfigSchema, type PaperclipConfig } from "./schema.js";
+import {
+  resolveDefaultConfigPath,
+  resolvePaperclipInstanceId,
+} from "./home.js";
+
+const DEFAULT_CONFIG_BASENAME = "config.json";
+
+function findConfigFileFromAncestors(startDir: string): string | null {
+  const absoluteStartDir = path.resolve(startDir);
+  let currentDir = absoluteStartDir;
+
+  while (true) {
+    const candidate = path.resolve(currentDir, ".paperclip", DEFAULT_CONFIG_BASENAME);
+    if (fs.existsSync(candidate)) {
+      return candidate;
+    }
+
+    const nextDir = path.resolve(currentDir, "..");
+    if (nextDir === currentDir) break;
+    currentDir = nextDir;
+  }
+
+  return null;
+}
+
+export function resolveConfigPath(overridePath?: string): string {
+  if (overridePath) return path.resolve(overridePath);
+  if (process.env.PAPERCLIP_CONFIG) return path.resolve(process.env.PAPERCLIP_CONFIG);
+  return findConfigFileFromAncestors(process.cwd()) ?? resolveDefaultConfigPath(resolvePaperclipInstanceId());
+}
+
+function parseJson(filePath: string): unknown {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, "utf-8"));
+  } catch (err) {
+    throw new Error(`Failed to parse JSON at ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
+
+function migrateLegacyConfig(raw: unknown): unknown {
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return raw;
+  const config = { ...(raw as Record<string, unknown>) };
+  const databaseRaw = config.database;
+  if (typeof databaseRaw !== "object" || databaseRaw === null || Array.isArray(databaseRaw)) {
+    return config;
+  }
+
+  const database = { ...(databaseRaw as Record<string, unknown>) };
+  if (database.mode === "pglite") {
+    database.mode = "embedded-postgres";
+
+    if (typeof database.embeddedPostgresDataDir !== "string" && typeof database.pgliteDataDir === "string") {
+      database.embeddedPostgresDataDir = database.pgliteDataDir;
+    }
+    if (
+      typeof database.embeddedPostgresPort !== "number" &&
+      typeof database.pglitePort === "number" &&
+      Number.isFinite(database.pglitePort)
+    ) {
+      database.embeddedPostgresPort = database.pglitePort;
+    }
+  }
+
+  config.database = database;
+  return config;
+}
+
+function formatValidationError(err: unknown): string {
+  const issues = (err as { issues?: Array<{ path?: unknown; message?: unknown }> })?.issues;
+  if (Array.isArray(issues) && issues.length > 0) {
+    return issues
+      .map((issue) => {
+        const pathParts = Array.isArray(issue.path) ? issue.path.map(String) : [];
+        const issuePath = pathParts.length > 0 ? pathParts.join(".") : "config";
+        const message = typeof issue.message === "string" ? issue.message : "Invalid value";
+        return `${issuePath}: ${message}`;
+      })
+      .join("; ");
+  }
+  return err instanceof Error ? err.message : String(err);
+}
+
+export function readConfig(configPath?: string): PaperclipConfig | null {
+  const filePath = resolveConfigPath(configPath);
+  if (!fs.existsSync(filePath)) return null;
+  const raw = parseJson(filePath);
+  const migrated = migrateLegacyConfig(raw);
+  const parsed = paperclipConfigSchema.safeParse(migrated);
+  if (!parsed.success) {
+    throw new Error(`Invalid config at ${filePath}: ${formatValidationError(parsed.error)}`);
+  }
+  return parsed.data;
+}
+
+export function writeConfig(
+  config: PaperclipConfig,
+  configPath?: string,
+): void {
+  const filePath = resolveConfigPath(configPath);
+  const dir = path.dirname(filePath);
+  fs.mkdirSync(dir, { recursive: true });
+
+  // Backup existing config before overwriting
+  if (fs.existsSync(filePath)) {
+    const backupPath = filePath + ".backup";
+    fs.copyFileSync(filePath, backupPath);
+    fs.chmodSync(backupPath, 0o600);
+  }
+
+  fs.writeFileSync(filePath, JSON.stringify(config, null, 2) + "\n", {
+    mode: 0o600,
+  });
+}
+
+export function configExists(configPath?: string): boolean {
+  return fs.existsSync(resolveConfigPath(configPath));
+}