diff --git a/docs/proposals/B1.1-pi-version-upgrade-trace.md b/docs/proposals/B1.1-pi-version-upgrade-trace.md index e1b712e2..11b36008 100644 --- a/docs/proposals/B1.1-pi-version-upgrade-trace.md +++ b/docs/proposals/B1.1-pi-version-upgrade-trace.md @@ -1,12 +1,13 @@ # Proposal B1.1: pi-agent-core 升级到 in-tree reference 的决策记录 > 来源：问题.md B1（pi-mono-reference 与实际依赖 62 patch 漂移） -> 状态：executed（已 commit `35c1229`）+ open（需 eval 监控） +> 状态：executed（已 commit `35c1229`）+ guarded（2026-04-23 runtime contract tests） > 决策时间：2026-04-20 Roundtable > Owner: TBD（建议项目 maintainer） > 关联 commit：`35c1229 Pin pi-agent-core/pi-ai/pi-coding-agent to exact 0.67.68` -> Blocking dependencies for full close-out：B5 LLM eval harness +> Runtime guardrail：`tests/unit/pi-runtime-dependency-contract.test.ts` + `tests/integration/pi-runtime-contract.spec.ts` + `tests/eval/harness.test.ts` > Superseded note（2026-04-22）：in-tree `pi-mono-reference/` snapshot has been removed. Runtime truth now lives in root `package.json` and `pnpm-lock.yaml`. +> Guardrail note（2026-04-23）：runtime truth is now executable: package pins, lockfile entries, installed package versions, exported API surface, Agent factory wiring, coding tools, skill loader, hook bundle, and eval fixture `piAgentCoreVersion` are covered by automated tests. --- @@ -54,9 +55,27 @@ caret range `^0.67.6` 在 pnpm semver 里允许 resolve 到 `>=0.67.6 <0.68.0` 4. **transitive dep 跃迁**：`@mistralai/mistralai` 在该版本范围内做过 1.x → 2.x major bump（pnpm-lock diff 可见），如果使用 Mistral provider 行为可能变化 5. **hook 签名兼容性**：本地 `hook-handlers.ts` / `context-compactor.ts` / `cost-writer.ts` 的 hook 注入接口若有变化，会在运行时才暴露 +## 自动化护栏 + +`pi-mono-reference/` 删除后，源级 review 能力转为本项目内的运行时契约。当前护栏分三层： + +1. `tests/unit/pi-runtime-dependency-contract.test.ts` + - `package.json` 三个 pi runtime 包必须精确 pin 到 `0.67.68` + - `pnpm-lock.yaml` importer / resolution / snapshot 必须指向同一版本 + - `node_modules` 中实际安装包版本必须一致 + - AgentCompany 使用的 pi API surface 必须存在 +2. `tests/integration/pi-runtime-contract.spec.ts` + - `createPiAgent` 必须创建真实 `Agent`，保持 sequential tool execution 和三类 hook + - `createRunScopedCodingTools` 必须输出当前 worker 依赖的 coding-tool surface + - bash spawn hook 必须保留 run-scoped env 注入 + - `loadSkills` 必须能读取 progressive-disclosure `SKILL.md` +3. `tests/eval/harness.test.ts` + - 每个 fixture 的 `meta.piAgentCoreVersion` 必须等于 shipping runtime version + - 真实 fixture 继续穿过 `pi-agent-core` Agent loop replay + ## 监控与回滚计划 -### 监控指标（手动观察直到 B5 落地） +### 监控指标 - 启动主进程后第一个 agent run 是否能跑完一轮 tool call - compactor 触发时是否有 tool_use/tool_result 报错 - 长会话（>50 轮）是否出现非预期 cost / token 偏差 @@ -85,7 +104,7 @@ git commit -m "Revert pi-agent-core pin to 0.67.6 due to behavior regression" ## 完成定义 本 proposal 在以下条件全部满足时关闭： -1. B5 LLM eval harness 落地，能跑一组 agent 行为 fixture -2. 在该 harness 上对比 `0.67.6` ↔ `0.67.68` 的行为 diff -3. 行为 diff 报告附 commit，确认无意外回归 -4. 关闭 B1.1，写入到 changelog +1. `package.json` / `pnpm-lock.yaml` / installed package versions 三者一致 +2. pi exported API surface、Agent factory、coding tools、skill loader、hook bundle 都有自动化契约 +3. eval fixtures 的 `piAgentCoreVersion` 与 shipping runtime version 一致 +4. `pnpm verify` 覆盖 unit / integration / eval / e2e diff --git a/tests/eval/README.md b/tests/eval/README.md index 35294d38..438226af 100644 --- a/tests/eval/README.md +++ b/tests/eval/README.md @@ -180,8 +180,8 @@ Field contracts: - `meta.fixtureId` — stable kebab-case identifier used as the scenario name. - `meta.piAgentCoreVersion` — the `@mariozechner/pi-agent-core` version - used when the fixture was recorded. Drift against the installed version - is allowed but future tooling will surface a warning. + used when the fixture was recorded. It must match the shipping version + in `tests/support/pi-runtime-contract.ts`; drift fails `pnpm test:eval`. - `meta.systemPromptHash` — sha256 of the system prompt text. Changes in this hash are the canonical signal that a fixture is stale. - `turn.turnIndex` — strictly increasing from 0. Gaps or duplicates are @@ -231,6 +231,7 @@ provider tier described in `docs/proposals/B5.1-llm-as-judge-followup.md`. - `scenarios//system.txt` must hash to `meta.systemPromptHash`. - `scenarios//user.txt` must match the first user turn exactly. +- `meta.piAgentCoreVersion` must match the shipping pi runtime version. - `assertions.semanticRubric` must contain at least one semantic dimension. - Structured assistant `toolCall` blocks require matching `toolResult` turns before the next assistant or user turn. diff --git a/tests/eval/harness.test.ts b/tests/eval/harness.test.ts index 260dab23..84498a81 100644 --- a/tests/eval/harness.test.ts +++ b/tests/eval/harness.test.ts @@ -32,6 +32,7 @@ import { buildMatrixPrompt, loadScenarioContract, } from "./scenario-contract.js"; +import { PI_RUNTIME_VERSION } from "../support/pi-runtime-contract"; import type { Assertion, AssistantTurn, @@ -307,6 +308,16 @@ describe("eval SOP coverage matrix", () => { const fixturePaths = await listFixtures(); +describe("eval pi runtime contract", () => { + it("records every fixture against the shipping pi-agent-core version", async () => { + expect(fixturePaths.length).toBeGreaterThan(0); + for (const fixturePath of fixturePaths) { + const fixture = await loadFixture(fixturePath); + expect(fixture.meta.piAgentCoreVersion).toBe(PI_RUNTIME_VERSION); + } + }); +}); + describe.each(fixturePaths)("eval fixture %s", (fixturePath) => { it("passes all assertions", async () => { const fixture = await loadFixture(fixturePath); diff --git a/tests/integration/pi-runtime-contract.spec.ts b/tests/integration/pi-runtime-contract.spec.ts new file mode 100644 index 00000000..97cc34c8 --- /dev/null +++ b/tests/integration/pi-runtime-contract.spec.ts @@ -0,0 +1,144 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Agent } from "@mariozechner/pi-agent-core"; +import type { Model } from "@mariozechner/pi-ai"; +import { loadSkills } from "@mariozechner/pi-coding-agent"; +import { describe, expect, it } from "vitest"; +import { buildAgentHooks } from "../../src/main/agent-runtime/agent-hooks"; +import { + createPiAgent, + resolveThinkingLevel, +} from "../../src/main/worker/runtime/pi-agent-factory"; +import { + createRunScopedBashOptions, + createRunScopedCodingTools, +} from "../../src/main/worker/runtime/run-tools"; +import type { WorkerRpcClient } from "../../src/main/worker/entry/rpc"; + +function createModel(): Model<"openai-responses"> { + return { + id: "contract-model", + name: "Contract Model", + api: "openai-responses", + provider: "openai", + baseUrl: "https://example.invalid", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 8192, + maxTokens: 2048, + }; +} + +function createRpc(): WorkerRpcClient { + return { + invoke: async () => { + throw new Error("Unexpected RPC call in pi runtime contract test."); + }, + } as unknown as WorkerRpcClient; +} + +describe("pi runtime integration contract", () => { + it("builds the AgentCompany pi Agent boundary with sequential execution and hooks", () => { + const agent = createPiAgent({ + runId: "run-contract", + taskSessionId: "session-contract", + systemPrompt: "Contract prompt", + model: createModel(), + tools: [], + priorMessages: [], + thinkingLevel: resolveThinkingLevel("high"), + apiKey: "contract-key", + extraHeaders: { "x-contract": "true" }, + rpc: createRpc(), + }); + + expect(agent).toBeInstanceOf(Agent); + expect(agent.sessionId).toBe("session-contract"); + expect(agent.toolExecution).toBe("sequential"); + expect(agent.state.systemPrompt).toBe("Contract prompt"); + expect(agent.state.thinkingLevel).toBe("high"); + expect(agent.state.tools).toEqual([]); + expect(agent.state.messages).toEqual([]); + expect(typeof agent.beforeToolCall).toBe("function"); + expect(typeof agent.afterToolCall).toBe("function"); + expect(typeof agent.transformContext).toBe("function"); + }); + + it("keeps the coding-tool surface expected by worker runs", () => { + const workspaceDir = mkdtempSync(join(tmpdir(), "agentcompany-pi-tools-")); + try { + const tools = createRunScopedCodingTools(workspaceDir, { AGENTCOMPANY_CONTRACT: "1" }); + expect(tools.map((tool) => tool.name)).toEqual(["read", "bash", "edit", "write"]); + for (const tool of tools) { + expect(typeof tool.execute).toBe("function"); + } + } finally { + rmSync(workspaceDir, { recursive: true, force: true }); + } + }); + + it("preserves run-scoped environment through the bash spawn hook", () => { + const options = createRunScopedBashOptions({ + AGENTCOMPANY_CONTRACT: "1", + PATH: "/contract/bin", + }); + const result = options.spawnHook?.({ + command: "pwd", + cwd: "/tmp", + env: { + HOME: "/home/contract", + PATH: "/base/bin", + }, + }); + + expect(result?.command).toBe("pwd"); + expect(result?.cwd).toBe("/tmp"); + expect(result?.env.HOME).toBe("/home/contract"); + expect(result?.env.PATH).toBe("/contract/bin"); + expect(result?.env.AGENTCOMPANY_CONTRACT).toBe("1"); + }); + + it("loads progressive-disclosure skills through the pinned pi runtime", () => { + const root = mkdtempSync(join(tmpdir(), "agentcompany-pi-skills-")); + const skillDir = join(root, "contract-skill"); + mkdirSync(skillDir); + writeFileSync( + join(skillDir, "SKILL.md"), + [ + "---", + "name: contract-skill", + "description: Contract skill loaded by the pinned pi runtime.", + "---", + "# Contract Skill", + "", + "Use this skill to prove the runtime loader contract.", + ].join("\n"), + "utf8", + ); + + try { + const result = loadSkills({ skillPaths: [skillDir], includeDefaults: false }); + expect(result.diagnostics).toEqual([]); + expect(result.skills).toHaveLength(1); + expect(result.skills[0]?.name).toBe("contract-skill"); + expect(result.skills[0]?.description).toBe("Contract skill loaded by the pinned pi runtime."); + expect(result.skills[0]?.filePath).toBe(join(skillDir, "SKILL.md")); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("keeps the aggregated hook bundle assignable to pi Agent options at runtime", () => { + const hooks = buildAgentHooks({ runId: "run-contract", rpc: createRpc() }); + expect(Object.keys(hooks).sort()).toEqual([ + "afterToolCall", + "beforeToolCall", + "transformContext", + ]); + expect(typeof hooks.beforeToolCall).toBe("function"); + expect(typeof hooks.afterToolCall).toBe("function"); + expect(typeof hooks.transformContext).toBe("function"); + }); +}); diff --git a/tests/support/pi-runtime-contract.ts b/tests/support/pi-runtime-contract.ts new file mode 100644 index 00000000..704e6764 --- /dev/null +++ b/tests/support/pi-runtime-contract.ts @@ -0,0 +1,69 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const PI_RUNTIME_VERSION = "0.67.68"; + +export const PI_RUNTIME_PACKAGES = [ + "@mariozechner/pi-agent-core", + "@mariozechner/pi-ai", + "@mariozechner/pi-coding-agent", +] as const; + +export type PiRuntimePackage = typeof PI_RUNTIME_PACKAGES[number]; + +export type PackageJson = { + dependencies?: Record; + version?: string; +}; + +export function readJsonFile(path: string): T { + return JSON.parse(readFileSync(path, "utf8")) as T; +} + +export function readRootPackage(): PackageJson { + return readJsonFile(join(process.cwd(), "package.json")); +} + +export function readPnpmLockfile(): string { + return readFileSync(join(process.cwd(), "pnpm-lock.yaml"), "utf8"); +} + +export function lockfileHasImporterPin( + lockfile: string, + packageName: PiRuntimePackage, + version: string, +): boolean { + return lockfile.includes([ + ` '${packageName}':`, + ` specifier: ${version}`, + ` version: ${version}(`, + ].join("\n")); +} + +export function lockfileHasPackageResolution( + lockfile: string, + packageName: PiRuntimePackage, + version: string, +): boolean { + return lockfile.includes(` '${packageName}@${version}':\n resolution: {integrity: sha512-`); +} + +export function lockfileHasPackageSnapshot( + lockfile: string, + packageName: PiRuntimePackage, + version: string, +): boolean { + return lockfile.includes(` '${packageName}@${version}(`); +} + +export function readInstalledPackage(packageName: PiRuntimePackage): PackageJson { + const resolvedPath = fileURLToPath(import.meta.resolve(packageName)); + const marker = `/node_modules/${packageName}/`; + const markerIndex = resolvedPath.lastIndexOf(marker); + if (markerIndex < 0) { + throw new Error(`Could not locate package root for ${packageName}: ${resolvedPath}`); + } + const packageRoot = resolvedPath.slice(0, markerIndex + marker.length - 1); + return readJsonFile(join(packageRoot, "package.json")); +} diff --git a/tests/unit/pi-runtime-dependency-contract.test.ts b/tests/unit/pi-runtime-dependency-contract.test.ts index f9145fdb..bf09907d 100644 --- a/tests/unit/pi-runtime-dependency-contract.test.ts +++ b/tests/unit/pi-runtime-dependency-contract.test.ts @@ -1,28 +1,51 @@ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; import { describe, expect, it } from "vitest"; - -type PackageJson = { - dependencies?: Record; -}; - -const PI_RUNTIME_PACKAGES = [ - "@mariozechner/pi-agent-core", - "@mariozechner/pi-ai", - "@mariozechner/pi-coding-agent", -] as const; - -function readRootPackage(): PackageJson { - return JSON.parse(readFileSync(join(process.cwd(), "package.json"), "utf8")) as PackageJson; -} +import { + PI_RUNTIME_PACKAGES, + PI_RUNTIME_VERSION, + lockfileHasImporterPin, + lockfileHasPackageResolution, + lockfileHasPackageSnapshot, + readInstalledPackage, + readPnpmLockfile, + readRootPackage, +} from "../support/pi-runtime-contract"; describe("pi runtime dependency contract", () => { it("keeps the shipping pi runtime on exact npm package pins", () => { const dependencies = readRootPackage().dependencies ?? {}; for (const packageName of PI_RUNTIME_PACKAGES) { - expect(dependencies[packageName]).toBe("0.67.68"); + expect(dependencies[packageName]).toBe(PI_RUNTIME_VERSION); expect(dependencies[packageName]).toMatch(/^\d+\.\d+\.\d+$/); } }); + + it("keeps package.json, pnpm-lock, and installed packages on one pi runtime version", () => { + const lockfile = readPnpmLockfile(); + + for (const packageName of PI_RUNTIME_PACKAGES) { + expect(lockfileHasImporterPin(lockfile, packageName, PI_RUNTIME_VERSION)).toBe(true); + expect(lockfileHasPackageResolution(lockfile, packageName, PI_RUNTIME_VERSION)).toBe(true); + expect(lockfileHasPackageSnapshot(lockfile, packageName, PI_RUNTIME_VERSION)).toBe(true); + expect(readInstalledPackage(packageName).version).toBe(PI_RUNTIME_VERSION); + } + }); + + it("keeps the pi API surface used by AgentCompany available", async () => { + const [agentCore, piAi, piCodingAgent] = await Promise.all([ + import("@mariozechner/pi-agent-core"), + import("@mariozechner/pi-ai"), + import("@mariozechner/pi-coding-agent"), + ]); + + expect(typeof agentCore.Agent).toBe("function"); + expect(typeof piAi.createAssistantMessageEventStream).toBe("function"); + expect(typeof piAi.getEnvApiKey).toBe("function"); + expect(typeof piAi.getModel).toBe("function"); + expect(typeof piAi.getModels).toBe("function"); + expect(typeof piAi.streamSimple).toBe("function"); + expect(typeof piAi.Type.Record).toBe("function"); + expect(typeof piCodingAgent.createCodingTools).toBe("function"); + expect(typeof piCodingAgent.loadSkills).toBe("function"); + }); });