ZacharyZhang-NY/AgentCompany
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Q131: remove review agents and eval suites whose pass only counted configuration rows

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed 2026-10-08 21:33:03 -04:00
1 parent 074a1fc8e3
commit fe45cd4d46
20 files changed
+32 -476
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/eval/replay-eval.test.ts
+2 -11
View File
@@ -7,9 +7,8 @@ import { AuditService } from "../../src/main/services/audit-service";
import { ChatService } from "../../src/main/services/chat-service";
import { OperationsService } from "../../src/main/services/operations-service";
import { OrganizationService } from "../../src/main/services/organization-service";
import { SdlcService } from "../../src/main/services/sdlc-service";
import { SnapshotService } from "../../src/main/services/snapshot-service";
import { EVAL_SUITE_KEY, SDLC_RUN_STATE, SLO_SIGNAL_KEY } from "../../src/shared/domain";
import { SLO_SIGNAL_KEY } from "../../src/shared/domain";
function createConnection(): DatabaseConnection {
return new DatabaseConnection(join(tmpdir(), `agentcompany-new-replay-${randomUUID()}.db`));
@@ -25,19 +24,17 @@ function createOrganization(organization: OrganizationService): string {
}
describe("replay eval", () => {
test("creates deterministic thread replay evidence and runs replay eval suite", () => {
test("creates deterministic thread replay evidence and records the replay digest", () => {
const connection = createConnection();
const organization = new OrganizationService(connection.db);
const chat = new ChatService(connection.db);
const audit = new AuditService(connection.db);
const operations = new OperationsService(connection.db);
const sdlc = new SdlcService(connection.db);
const snapshot = new SnapshotService(connection.db);
let orgId = "";
let employeeId = "";
let threadId = "";
let replayRunId = "";
let evalRunId = "";
connection.transaction(() => {
orgId = createOrganization(organization);
@@ -68,23 +65,17 @@ describe("replay eval", () => {
durationMs: 12,
detail: { threadId, replayRunId },
});
const suiteId = (connection.db.prepare("select id from eval_suites where org_id = ? and key = ?")
.get(orgId, EVAL_SUITE_KEY.REPLAY) as { id: string }).id;
evalRunId = sdlc.runEvalSuite({ orgId, suiteId });
});
const state = snapshot.load();
const replayRun = state.auditReplayRuns.find((run) => run.id === replayRunId);
const evalRun = state.evalRuns.find((run) => run.id === evalRunId);
if (!replayRun) throw new Error("Audit replay run was not created.");
if (!evalRun) throw new Error("Replay eval run was not created.");
const replay = JSON.parse(replayRun.replayJson) as { events: Array<{ entityType: string; action: string }> };
expect(replayRun.threadId).toBe(threadId);
expect(replayRun.eventDigest).toBe(createHash("sha256").update(replayRun.replayJson).digest("hex"));
expect(replayRun.eventCount).toBe(replay.events.length);
expect(replay.events).toContainEqual(expect.objectContaining({ entityType: "chat_message", action: "sent" }));
expect(replay.events).toContainEqual(expect.objectContaining({ entityType: "approval", action: "requested" }));
expect(evalRun).toMatchObject({ state: SDLC_RUN_STATE.PASSED, failedCount: 0 });
connection.close();
});
});