Q131: remove review agents and eval suites whose pass only counted configuration rows
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
074a1fc8e3
commit
fe45cd4d46
20 files changed
+32
-476
No files matched your search
@@ -1,12 +1,6 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { DatabaseConnection } from "../../src/main/database/connection";
|
||||
import { OrganizationService } from "../../src/main/services/organization-service";
|
||||
import { providerAdapters } from "../../src/main/services/provider-adapters";
|
||||
import { SdlcService } from "../../src/main/services/sdlc-service";
|
||||
import { EVAL_SUITE_KEY, PROVIDER_AUTH_TYPE, SDLC_RUN_STATE } from "../../src/shared/domain";
|
||||
import { PROVIDER_AUTH_TYPE } from "../../src/shared/domain";
|
||||
import { PROVIDER_CATALOG } from "../../src/shared/provider-catalog";
|
||||
|
||||
const PROVIDER_SECRET_SAMPLES: Record<string, string> = {
|
||||
@@ -42,19 +36,6 @@ const PROVIDER_SECRET_SAMPLES: Record<string, string> = {
|
||||
"custom-anthropic": JSON.stringify({ baseUrl: "https://llm.example.com", secret: "sk-custom" }),
|
||||
};
|
||||
|
||||
function createConnection(): DatabaseConnection {
|
||||
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-contract-${randomUUID()}.db`));
|
||||
}
|
||||
|
||||
function createOrganization(organization: OrganizationService): string {
|
||||
return organization.createOrganization({
|
||||
organizationName: "Acme Operations",
|
||||
employeeName: "Dana Lee",
|
||||
mission: "Coordinate agents with auditable approvals.",
|
||||
goal: "Ship a verified control plane.",
|
||||
});
|
||||
}
|
||||
|
||||
describe("provider contracts", () => {
|
||||
test("keeps provider catalog and adapters aligned", () => {
|
||||
expect(Object.keys(providerAdapters).sort()).toEqual(PROVIDER_CATALOG.map((provider) => provider.key).sort());
|
||||
@@ -139,33 +120,4 @@ describe("provider contracts", () => {
|
||||
await expect(adapter.refresh({ authProfile: profile })).rejects.toThrow("does not support token refresh");
|
||||
}
|
||||
});
|
||||
|
||||
test("runs the seeded contract eval suite", () => {
|
||||
const connection = createConnection();
|
||||
const organization = new OrganizationService(connection.db);
|
||||
const sdlc = new SdlcService(connection.db);
|
||||
let orgId = "";
|
||||
let suiteId = "";
|
||||
let runId = "";
|
||||
|
||||
connection.transaction(() => {
|
||||
orgId = createOrganization(organization);
|
||||
suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
|
||||
.get(orgId, EVAL_SUITE_KEY.CONTRACT) as { id: string }).id;
|
||||
runId = sdlc.runEvalSuite({ orgId, suiteId });
|
||||
});
|
||||
|
||||
const run = connection.db.prepare("select state, failed_count, result_json from eval_runs where id = ?")
|
||||
.get(runId) as { state: string; failed_count: number; result_json: string };
|
||||
const result = JSON.parse(run.result_json) as { checks: Array<{ key: string; passed: boolean }> };
|
||||
expect(run).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failed_count: 0 });
|
||||
expect(result.checks.map((check) => check.key).sort()).toEqual([
|
||||
"provider_adapter_coverage",
|
||||
"provider_rate_limit_registry",
|
||||
"provider_runtime_contract",
|
||||
"schema_version",
|
||||
]);
|
||||
expect(result.checks.every((check) => check.passed)).toBe(true);
|
||||
connection.close();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user