feat: add live executor observability
This commit is contained in:
liujing
@@ -8,7 +8,7 @@
|
||||
* without running real AI agents.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { describe, it, expect, beforeAll, beforeEach, afterEach } from "vitest";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
@@ -218,6 +218,198 @@ describe("ClaudeAdapter", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Claude stream-json activity derivation (pure unit tests via exported function)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("deriveClaudeActivity (stream-json parsing)", () => {
|
||||
let parseLine: (line: string) => string | undefined;
|
||||
|
||||
beforeAll(async () => {
|
||||
const mod = await import("./workflow-executor.js");
|
||||
parseLine = mod.parseClaudeActivityLine;
|
||||
});
|
||||
|
||||
it("extracts tool_use from a type=assistant record with message.content and file_path", () => {
|
||||
const line = `{"type":"assistant","message":{"content":[{"type":"thinking","thinking":"I'll read the file first"},{"type":"tool_use","name":"Read","input":{"file_path":"src/foo.ts"}}]}}`;
|
||||
expect(parseLine(line)).toBe("Reading src/foo.ts");
|
||||
});
|
||||
|
||||
it("extracts tool_use from a type=assistant record with command", () => {
|
||||
const line = `{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Bash","input":{"command":"npm test"}}]}}`;
|
||||
expect(parseLine(line)).toBe("Running: npm test");
|
||||
});
|
||||
|
||||
it("does NOT surface thinking content in activity summary", () => {
|
||||
const line = `{"type":"assistant","message":{"content":[{"type":"thinking","thinking":"secret reasoning"},{"type":"tool_use","name":"Read","input":{"file_path":"src/ts"}}]}}`;
|
||||
const result = parseLine(line);
|
||||
expect(result).toBe("Reading src/ts");
|
||||
expect(result).not.toContain("secret reasoning");
|
||||
// A line with only thinking should return undefined
|
||||
const thinkingOnly = `{"type":"assistant","message":{"content":[{"type":"thinking","thinking":"hidden"}]}}`;
|
||||
expect(parseLine(thinkingOnly)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("extracts capitalized tool names: Write, Edit, Glob, Grep", () => {
|
||||
expect(parseLine(`{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Write","input":{"file_path":"new.ts"}}]}}`)).toBe("Writing new.ts");
|
||||
expect(parseLine(`{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Edit","input":{"file_path":"edit.ts"}}]}}`)).toBe("Editing edit.ts");
|
||||
expect(parseLine(`{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Glob"}]}}`)).toBe("Searching files");
|
||||
expect(parseLine(`{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Grep","input":{"pattern":"foo"}}]}}`)).toBe("Searching code");
|
||||
});
|
||||
|
||||
it("extracts type=system subtype=init as session", () => {
|
||||
expect(parseLine(`{"type":"system","subtype":"init","session_id":"abc123"}`)).toBe("Creating session");
|
||||
});
|
||||
|
||||
it("extracts is_error result as error", () => {
|
||||
expect(parseLine(`{"type":"result","is_error":true,"result":"Rate limit exceeded","session_id":"abc"}`)).toBe("Error: Rate limit exceeded");
|
||||
});
|
||||
|
||||
it("extracts type=result at top level", () => {
|
||||
expect(parseLine(`{"type":"result","result":"Tool ran without output"}`)).toBe("Result: Tool ran without output");
|
||||
});
|
||||
|
||||
it("extracts type=error at top level", () => {
|
||||
expect(parseLine(`{"type":"error","error":"Permission denied"}`)).toBe("Error: Permission denied");
|
||||
});
|
||||
|
||||
it("returns undefined for non-JSON lines", () => {
|
||||
expect(parseLine("implemented")).toBeUndefined();
|
||||
expect(parseLine("")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns undefined for unknown types", () => {
|
||||
expect(parseLine(`{"type":"ping"}`)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// End-to-end fake Claude stream-json fixture (adapter start and resume)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("ClaudeAdapter stream-json fixture", () => {
|
||||
let tmpDir: string;
|
||||
let previousHome: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "agent-workflow-claude-fixture-"));
|
||||
previousHome = process.env.HOME;
|
||||
process.env.HOME = tmpDir;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (previousHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = previousHome;
|
||||
cleanupDir(tmpDir);
|
||||
});
|
||||
|
||||
/** Create a fake Claude binary that outputs stream-json lines. */
|
||||
function makeFakeClaude(lines: string[], exitCode = 0): string {
|
||||
const scriptPath = path.join(tmpDir, "fake-claude.sh");
|
||||
const escaped = lines.map((l) => l.replace(/'/g, "'\\''")).join("\\n");
|
||||
fs.writeFileSync(scriptPath, `#!/bin/bash
|
||||
printf '${escaped}\\n'
|
||||
exit ${exitCode}
|
||||
`, { mode: 0o755 });
|
||||
return scriptPath;
|
||||
}
|
||||
|
||||
it("start: emits --output-format stream-json, captures session, has tool activity in sidecar", async () => {
|
||||
const argsFile = path.join(tmpDir, "args.txt");
|
||||
const fakeBin = makeFakeClaude([
|
||||
'{"type":"system","subtype":"init","session_id":"fixture-session-456"}',
|
||||
'{"type":"assistant","message":{"content":[{"type":"thinking","thinking":"thinking text hidden"},{"type":"tool_use","name":"Read","input":{"file_path":"src/index.ts"}}]}}',
|
||||
'{"type":"result","result":"ok"}',
|
||||
]);
|
||||
const trackingBin = path.join(tmpDir, "tracking-claude.sh");
|
||||
fs.writeFileSync(trackingBin, `#!/bin/bash
|
||||
printf '%s\\0' "$@" >> "${argsFile}"
|
||||
"${fakeBin}" "$@"
|
||||
`, { mode: 0o755 });
|
||||
|
||||
const adapter = createExecutorAdapter("claude");
|
||||
const logFilePath = path.join(tmpDir, "claude-exec.log");
|
||||
const result = await adapter.start({
|
||||
repoRoot: tmpDir,
|
||||
plan: makeValidPlan(),
|
||||
runDir: tmpDir,
|
||||
cycleIndex: 1,
|
||||
logFilePath,
|
||||
config: { binary: trackingBin },
|
||||
});
|
||||
|
||||
// 1) --output-format stream-json was passed
|
||||
const capturedArgs = fs.existsSync(argsFile) ? fs.readFileSync(argsFile, "utf8") : "";
|
||||
expect(capturedArgs).toContain("--output-format");
|
||||
expect(capturedArgs).toContain("stream-json");
|
||||
|
||||
// 2) Exact session preservation
|
||||
if (result.sessionHandle?.kind === "claude") {
|
||||
expect(result.sessionHandle.sessionId).toBe("fixture-session-456");
|
||||
}
|
||||
|
||||
// 3) Tool activity in the completed sidecar (no thinking text)
|
||||
const { readProgressSidecar } = await import("./workflow-state.js");
|
||||
const sidecar = readProgressSidecar(tmpDir);
|
||||
expect(sidecar).toBeDefined();
|
||||
expect(sidecar!.activitySummary).toContain("Reading");
|
||||
expect(sidecar!.activitySummary).not.toContain("thinking text hidden");
|
||||
expect(sidecar!.phase).toBe("completed");
|
||||
expect(sidecar!.executor).toBe("claude");
|
||||
});
|
||||
|
||||
it("start with error: extracts failure reason and marks sidecar failed", async () => {
|
||||
const fakeBin = makeFakeClaude([
|
||||
'{"type":"system","subtype":"init","session_id":"err-session-789"}',
|
||||
'{"type":"result","is_error":true,"result":"Rate limit exceeded","session_id":"err-session-789"}',
|
||||
], 1);
|
||||
|
||||
const adapter = createExecutorAdapter("claude");
|
||||
const logFilePath = path.join(tmpDir, "claude-error.log");
|
||||
const result = await adapter.start({
|
||||
repoRoot: tmpDir,
|
||||
plan: makeValidPlan(),
|
||||
runDir: tmpDir,
|
||||
cycleIndex: 1,
|
||||
logFilePath,
|
||||
config: { binary: fakeBin },
|
||||
});
|
||||
|
||||
expect(result.failed).toBe(true);
|
||||
expect(result.failureReason).toContain("Rate limit exceeded");
|
||||
|
||||
const { readProgressSidecar } = await import("./workflow-state.js");
|
||||
const sidecar = readProgressSidecar(tmpDir);
|
||||
expect(sidecar).toBeDefined();
|
||||
expect(sidecar!.phase).toBe("failed");
|
||||
});
|
||||
|
||||
it("resume: preserves session ID from the handle", async () => {
|
||||
const fakeBin = makeFakeClaude([
|
||||
'{"type":"system","subtype":"init","session_id":"resumed-session-999"}',
|
||||
'{"type":"result","result":"completed"}',
|
||||
]);
|
||||
|
||||
const adapter = createExecutorAdapter("claude");
|
||||
const logFilePath = path.join(tmpDir, "claude-resume.log");
|
||||
const result = await adapter.resume({
|
||||
repoRoot: tmpDir,
|
||||
plan: makeValidPlan(),
|
||||
acceptedFindings: "Fix the bug",
|
||||
handle: { kind: "claude", sessionId: "resumed-session-999" },
|
||||
runDir: tmpDir,
|
||||
cycleIndex: 2,
|
||||
logFilePath,
|
||||
config: { binary: fakeBin },
|
||||
});
|
||||
|
||||
if (result.sessionHandle?.kind === "claude") {
|
||||
expect(result.sessionHandle.sessionId).toBe("resumed-session-999");
|
||||
}
|
||||
expect(result.failed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Agy adapter tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user