feat: add live executor observability

src/workflow-executor.ts

@@ -7,6 +7,7 @@
  *  - Return ExecutorResult with session handle for future resume
  */
 
+import { randomUUID } from "node:crypto";
 import { execFileSync, spawnSync } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
@@ -22,9 +23,15 @@ import type {
   ExecutorResult,
   ExecutorSessionHandle,
   ExecutorStartOpts,
+  ProgressSidecar,
   ReasonixSessionHandle,
 } from "./workflow-types.js";
-import { nowIso } from "./workflow-state.js";
+import {
+  nowIso,
+  writeProgressSidecar,
+  readProgressSidecar,
+  attemptLogPath,
+} from "./workflow-state.js";
 
 // ---------------------------------------------------------------------------
 // Safety prompt footer (appended to every executor instruction)
@@ -45,6 +52,166 @@ const SAFETY_CONSTRAINTS = `
 - If you encounter an error you cannot resolve, stop and report it clearly.
 `.trimStart();
 
+// ---------------------------------------------------------------------------
+// Progress sidecar helpers
+// ---------------------------------------------------------------------------
+
+function startProgressSidecar(
+  dir: string,
+  executor: "reasonix" | "claude" | "agy",
+  cycleIndex: number,
+  attemptIndex: number,
+  logFilePath: string | undefined,
+  intervalMs = 5000,
+): NodeJS.Timeout {
+  const sidecar: ProgressSidecar = {
+    executor,
+    cycleIndex,
+    attemptIndex,
+    phase: "executing",
+    startedAt: nowIso(),
+    lastActivityAt: nowIso(),
+    logFilePath: logFilePath ? path.relative(dir, logFilePath) : undefined,
+    logBytes: 0,
+    activitySummary: "Starting...",
+    updatedAt: nowIso(),
+  };
+  writeProgressSidecar(dir, sidecar);
+
+  return setInterval(() => {
+    const current = readProgressSidecar(dir);
+    if (!current) return;
+    const now = nowIso();
+    current.updatedAt = now;
+    if (current.logFilePath) {
+      try {
+        current.logBytes = fs.statSync(path.join(dir, current.logFilePath)).size;
+      } catch {
+        // file may not exist yet
+      }
+    }
+    writeProgressSidecar(dir, current);
+  }, intervalMs);
+}
+
+function updateSidecar(dir: string, updates: Partial<ProgressSidecar>): void {
+  const current = readProgressSidecar(dir);
+  if (!current) return;
+  Object.assign(current, updates, { updatedAt: nowIso() });
+  writeProgressSidecar(dir, current);
+}
+
+function stopProgressSidecar(dir: string, timer: NodeJS.Timeout, failed?: boolean): void {
+  clearInterval(timer);
+  const current = readProgressSidecar(dir);
+  if (!current) return;
+  current.phase = failed ? "failed" : "completed";
+  const elapsed = Date.now() - new Date(current.startedAt).getTime();
+  current.executionDurationMs = elapsed;
+  current.updatedAt = nowIso();
+  if (current.logFilePath) {
+    try {
+      current.logBytes = fs.statSync(path.join(dir, current.logFilePath)).size;
+    } catch {
+      // ignore
+    }
+  }
+  writeProgressSidecar(dir, current);
+}
+
+// --------------------------------------------------------------------------
+// Claude stream-json activity derivation
+// ---------------------------------------------------------------------------
+
+function deriveClaudeActivity(line: string): string | undefined {
+  if (!line.trimStart().startsWith("{")) return undefined;
+  try {
+    const parsed = JSON.parse(line) as Record<string, unknown>;
+    const type = String(parsed.type ?? "");
+
+    // Real Claude Code stream-json envelope:
+    //   type=system,  subtype=init,       session_id
+    //   type=assistant, message:{content:[{type:"tool_use",...}]}
+    //   type=result,  is_error?, result,  session_id
+
+    if (type === "assistant") {
+      const message = parsed.message as Record<string, unknown> | undefined;
+      const content = message?.content as unknown[] | undefined;
+      if (!Array.isArray(content) || content.length === 0) return undefined;
+      for (const item of content) {
+        if (typeof item !== "object" || item === null) continue;
+        const block = item as Record<string, unknown>;
+        const blockType = String(block.type ?? "");
+        if (blockType === "thinking") continue; // DO NOT surface thinking
+        if (blockType === "tool_use") {
+          return deriveToolUse(block);
+        }
+      }
+      return undefined;
+    }
+
+    // Direct tool_use at top level (legacy)
+    if (type === "tool_use") {
+      return deriveToolUse(parsed);
+    }
+
+    // System init / result / error at top level
+    switch (type) {
+      case "system": {
+        if (parsed.subtype === "init") return "Creating session";
+        return undefined;
+      }
+      case "result": {
+        // is_error result → error
+        if (parsed.is_error === true) {
+          const err = String(parsed.result ?? "").slice(0, 80);
+          return err ? `Error: ${err}` : "Error occurred";
+        }
+        const result = String(parsed.result ?? "").slice(0, 60);
+        return result ? `Result: ${result}` : "Processing";
+      }
+      case "error": {
+        const err = String(parsed.error ?? "").slice(0, 80);
+        return err ? `Error: ${err}` : "Error occurred";
+      }
+      default: return undefined;
+    }
+  } catch {
+    return undefined;
+  }
+}
+
+/** Derive a concise activity string from a tool_use content block. */
+function deriveToolUse(block: Record<string, unknown>): string | undefined {
+  const name = String(block.name ?? "");
+  const input = block.input as Record<string, unknown> | undefined;
+  const target = (input?.file_path ?? input?.path ?? input?.file ?? input?.filePath ?? "") as string;
+  const lowerName = name.toLowerCase();
+  switch (lowerName) {
+    case "read":
+    case "read_file": return target ? `Reading ${target}` : "Reading file";
+    case "write":
+    case "create": return target ? `Writing ${target}` : "Writing file";
+    case "edit":
+    case "edit_file":
+    case "str_replace_editor": return target ? `Editing ${target}` : "Editing file";
+    case "bash":
+    case "command": {
+      const cmd = String(input?.command ?? "").slice(0, 80);
+      return cmd ? `Running: ${cmd}` : "Running command";
+    }
+    case "glob": return "Searching files";
+    case "grep":
+    case "search": return "Searching code";
+    case "web_fetch":
+    case "web": return "Fetching URL";
+    case "subagent":
+    case "agent":
+    case "dispatch_agent": return "Running sub-agent";
+    default: return `${name}...`;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Prompt builders
 // ---------------------------------------------------------------------------
@@ -147,6 +314,13 @@ function resolveExecutable(config: ExecutorConfig, defaultBin: string): string {
   return config.binary || defaultBin;
 }
 
+function deriveAttemptIndex(logFilePath: string | undefined): number {
+  if (!logFilePath) return 0;
+  const base = path.basename(logFilePath);
+  const match = base.match(/retry-(\d+)\.log$/);
+  return match ? parseInt(match[1], 10) : 0;
+}
+
 // ---------------------------------------------------------------------------
 // Reasonix adapter
 // ---------------------------------------------------------------------------
@@ -291,12 +465,22 @@ class ReasonixAdapter implements ExecutorAdapter {
     const args: string[] = ["run", "--dir", opts.repoRoot, prompt];
 
     const startMs = Date.now();
+    const logFile = opts.logFilePath;
+    const attemptIndex = deriveAttemptIndex(logFile);
+    const sidecarTimer = startProgressSidecar(opts.runDir, "reasonix", opts.cycleIndex, attemptIndex, logFile);
+
     const child = await spawnStreamingChild(bin, args, {
       cwd: opts.repoRoot,
       env: executorEnv(),
       signal: opts.signal,
       processGroup: true,
+      logFilePath: logFile,
+      onProgress: ({ pid, bytesLogged }) => {
+        updateSidecar(opts.runDir, { pid, logBytes: bytesLogged, lastActivityAt: nowIso() });
+      },
     });
+    const reasonixFailed = child.status !== 0 || !!child.error;
+    stopProgressSidecar(opts.runDir, sidecarTimer, reasonixFailed);
     const durationMs = Date.now() - startMs;
 
     const newSession = detectNewSessionFile(beforeSnapshot, snapshotDirs, opts.repoRoot);
@@ -305,7 +489,7 @@ class ReasonixAdapter implements ExecutorAdapter {
       : undefined;
 
     const report = child.stdout + (child.stderr ? `\n[stderr]\n${child.stderr}` : "");
-    const failed = child.status !== 0 || !!child.error;
+    const failed = reasonixFailed;
 
     return {
       exitCode: child.status,
@@ -332,12 +516,22 @@ class ReasonixAdapter implements ExecutorAdapter {
     ];
 
     const startMs = Date.now();
+    const logFile = opts.logFilePath;
+    const attemptIndex = deriveAttemptIndex(logFile);
+    const sidecarTimer = startProgressSidecar(opts.runDir, "reasonix", opts.cycleIndex, attemptIndex, logFile);
+
     const child = await spawnStreamingChild(bin, args, {
       cwd: opts.repoRoot,
       env: executorEnv(),
       signal: opts.signal,
       processGroup: true,
+      logFilePath: logFile,
+      onProgress: ({ pid, bytesLogged }) => {
+        updateSidecar(opts.runDir, { pid, logBytes: bytesLogged, lastActivityAt: nowIso() });
+      },
     });
+    const reasonixFailed = child.status !== 0 || !!child.error;
+    stopProgressSidecar(opts.runDir, sidecarTimer, reasonixFailed);
     const durationMs = Date.now() - startMs;
 
     const report = child.stdout + (child.stderr ? `\n[stderr]\n${child.stderr}` : "");
@@ -346,7 +540,7 @@ class ReasonixAdapter implements ExecutorAdapter {
       report,
       durationMs,
       sessionHandle: handle, // Reasonix handle stays the same file
-      failed: child.status !== 0 || !!child.error,
+      failed: reasonixFailed,
       ...(child.error ? { failureReason: child.error.message } : {}),
     };
   }
@@ -390,6 +584,10 @@ function extractClaudeFailureReason(output: string): string | undefined {
       if (parsed.is_error === true && typeof parsed.result === "string" && parsed.result.trim()) {
         return parsed.result.trim();
       }
+      // Handle stream-json error format: {"type":"error","error":"Rate limit exceeded"}
+      if (parsed.type === "error" && typeof parsed.error === "string" && parsed.error.trim()) {
+        return parsed.error.trim();
+      }
     } catch {
       // Ignore non-JSON output lines.
     }
@@ -397,8 +595,6 @@ function extractClaudeFailureReason(output: string): string | undefined {
   return undefined;
 }
 
-import { randomUUID } from "node:crypto";
-
 class ClaudeAdapter implements ExecutorAdapter {
   readonly kind: ExecutorKind = "claude";
 
@@ -423,19 +619,50 @@ class ClaudeAdapter implements ExecutorAdapter {
       "-p",
       "--safe-mode",
       "--permission-mode", "bypassPermissions",
-      "--output-format", "json",
+      "--output-format", "stream-json",
       "--session-id", sessionId,
       ...(opts.config.model ? ["--model", opts.config.model] : []),
       prompt,
     ];
 
     const startMs = Date.now();
+    const logFile = opts.logFilePath;
+    const attemptIndex = deriveAttemptIndex(logFile);
+    const sidecarTimer = startProgressSidecar(opts.runDir, "claude", opts.cycleIndex, attemptIndex, logFile);
+
+    let lineBuffer = "";
+
     const child = await spawnStreamingChild(bin, args, {
       cwd: opts.repoRoot,
       env: executorEnv(),
       signal: opts.signal,
       processGroup: true,
+      logFilePath: logFile,
+      onProgress: ({ pid, bytesLogged }) => {
+        updateSidecar(opts.runDir, { pid, logBytes: bytesLogged, lastActivityAt: nowIso() });
+      },
+      onStdoutData: (chunk: string) => {
+        lineBuffer += chunk;
+        while (true) {
+          const nl = lineBuffer.indexOf("\n");
+          if (nl < 0) break;
+          const line = lineBuffer.slice(0, nl);
+          lineBuffer = lineBuffer.slice(nl + 1);
+          const activity = deriveClaudeActivity(line);
+          if (activity) {
+            // Only surface tool actions in activitySummary, not result/error
+            const isToolActivity = !activity.startsWith("Result:") && !activity.startsWith("Error:");
+            if (isToolActivity) {
+              updateSidecar(opts.runDir, { activitySummary: activity, lastActivityAt: nowIso() });
+            } else {
+              updateSidecar(opts.runDir, { lastActivityAt: nowIso() });
+            }
+          }
+        }
+      },
     });
+    const claudeFailed = child.status !== 0 || !!child.error;
+    stopProgressSidecar(opts.runDir, sidecarTimer, claudeFailed);
     const durationMs = Date.now() - startMs;
 
     const capturedId = extractClaudeSessionId(child.stdout) ?? sessionId;
@@ -464,19 +691,50 @@ class ClaudeAdapter implements ExecutorAdapter {
       "-p",
       "--safe-mode",
       "--permission-mode", "bypassPermissions",
-      "--output-format", "json",
+      "--output-format", "stream-json",
       "--resume", handle.sessionId,
       ...(opts.config.model ? ["--model", opts.config.model] : []),
       prompt,
     ];
 
     const startMs = Date.now();
+    const logFile = opts.logFilePath;
+    const attemptIndex = deriveAttemptIndex(logFile);
+    const sidecarTimer = startProgressSidecar(opts.runDir, "claude", opts.cycleIndex, attemptIndex, logFile);
+
+    let lineBuffer = "";
+
     const child = await spawnStreamingChild(bin, args, {
       cwd: opts.repoRoot,
       env: executorEnv(),
       signal: opts.signal,
       processGroup: true,
+      logFilePath: logFile,
+      onProgress: ({ pid, bytesLogged }) => {
+        updateSidecar(opts.runDir, { pid, logBytes: bytesLogged, lastActivityAt: nowIso() });
+      },
+      onStdoutData: (chunk: string) => {
+        lineBuffer += chunk;
+        while (true) {
+          const nl = lineBuffer.indexOf("\n");
+          if (nl < 0) break;
+          const line = lineBuffer.slice(0, nl);
+          lineBuffer = lineBuffer.slice(nl + 1);
+          const activity = deriveClaudeActivity(line);
+          if (activity) {
+            // Only surface tool actions in activitySummary, not result/error
+            const isToolActivity = !activity.startsWith("Result:") && !activity.startsWith("Error:");
+            if (isToolActivity) {
+              updateSidecar(opts.runDir, { activitySummary: activity, lastActivityAt: nowIso() });
+            } else {
+              updateSidecar(opts.runDir, { lastActivityAt: nowIso() });
+            }
+          }
+        }
+      },
     });
+    const claudeFailed = child.status !== 0 || !!child.error;
+    stopProgressSidecar(opts.runDir, sidecarTimer, claudeFailed);
     const durationMs = Date.now() - startMs;
 
     const report = child.stdout + (child.stderr ? `\n[stderr]\n${child.stderr}` : "");
@@ -587,26 +845,36 @@ class AgyAdapter implements ExecutorAdapter {
   async start(opts: ExecutorStartOpts): Promise<ExecutorResult> {
     const bin = resolveExecutable(opts.config, "agy");
     const prompt = buildStartPrompt(opts);
-    const logFile = path.join(opts.runDir, `cycle-${opts.cycleIndex}-agy.log`);
+    const agyLogFile = path.join(opts.runDir, `cycle-${opts.cycleIndex}-agy.log`);
     const previousWorkspaceConversationId = getAgyWorkspaceConversationId(opts.repoRoot);
 
     const args: string[] = [
       ...agyCommonArgs(opts.config, opts.timeoutSeconds),
-      "--log-file", logFile,
+      "--log-file", agyLogFile,
       "--print", prompt,
     ];
 
     const startMs = Date.now();
+    const logFile = opts.logFilePath;
+    const attemptIndex = deriveAttemptIndex(logFile);
+    const sidecarTimer = startProgressSidecar(opts.runDir, "agy", opts.cycleIndex, attemptIndex, logFile);
+
     const child = await spawnStreamingChild(bin, args, {
       cwd: opts.repoRoot,
       env: executorEnv(),
       signal: opts.signal,
       processGroup: true,
+      logFilePath: logFile,
+      onProgress: ({ pid, bytesLogged }) => {
+        updateSidecar(opts.runDir, { pid, logBytes: bytesLogged, lastActivityAt: nowIso() });
+      },
     });
+    const agyFailed = child.status !== 0 || !!child.error || !!extractAgyFailureReason(child.stdout, child.stderr);
+    stopProgressSidecar(opts.runDir, sidecarTimer, agyFailed);
     const durationMs = Date.now() - startMs;
 
     const conversationId = extractAgyConversationId(child.stdout)
-      ?? extractAgyConversationIdFromLog(logFile)
+      ?? extractAgyConversationIdFromLog(agyLogFile)
       ?? (() => {
         const currentId = getAgyWorkspaceConversationId(opts.repoRoot);
         return currentId !== previousWorkspaceConversationId ? currentId : undefined;
@@ -634,7 +902,7 @@ class AgyAdapter implements ExecutorAdapter {
     const handle = opts.handle as AgySessionHandle;
     const bin = resolveExecutable(opts.config, "agy");
     const prompt = buildFixPrompt(opts);
-    const logFile = path.join(opts.runDir, `cycle-${opts.cycleIndex}-agy.log`);
+    const agyLogFile = path.join(opts.runDir, `cycle-${opts.cycleIndex}-agy.log`);
     const conversationId = handle.conversationId
       ?? getAgyWorkspaceConversationId(opts.repoRoot);
 
@@ -643,7 +911,7 @@ class AgyAdapter implements ExecutorAdapter {
       args = [
         ...agyCommonArgs(opts.config, opts.timeoutSeconds),
         "--conversation", conversationId,
-        "--log-file", logFile,
+        "--log-file", agyLogFile,
         "--print", prompt,
       ];
     } else {
@@ -651,24 +919,34 @@ class AgyAdapter implements ExecutorAdapter {
       args = [
         ...agyCommonArgs(opts.config, opts.timeoutSeconds),
         "--continue",
-        "--log-file", logFile,
+        "--log-file", agyLogFile,
         "--print", prompt,
       ];
     }
 
     const startMs = Date.now();
+    const logFile = opts.logFilePath;
+    const attemptIndex = deriveAttemptIndex(logFile);
+    const sidecarTimer = startProgressSidecar(opts.runDir, "agy", opts.cycleIndex, attemptIndex, logFile);
+
     const child = await spawnStreamingChild(bin, args, {
       cwd: opts.repoRoot,
       env: executorEnv(),
       signal: opts.signal,
       processGroup: true,
+      logFilePath: logFile,
+      onProgress: ({ pid, bytesLogged }) => {
+        updateSidecar(opts.runDir, { pid, logBytes: bytesLogged, lastActivityAt: nowIso() });
+      },
     });
+    const agyFailed = child.status !== 0 || !!child.error || !!extractAgyFailureReason(child.stdout, child.stderr);
+    stopProgressSidecar(opts.runDir, sidecarTimer, agyFailed);
     const durationMs = Date.now() - startMs;
 
     const report = child.stdout + (child.stderr ? `\n[stderr]\n${child.stderr}` : "");
     const capturedConversationId = conversationId
       ?? extractAgyConversationId(child.stdout)
-      ?? extractAgyConversationIdFromLog(logFile);
+      ?? extractAgyConversationIdFromLog(agyLogFile);
     const updatedHandle: AgySessionHandle = {
       kind: "agy",
       ...(capturedConversationId ? { conversationId: capturedConversationId } : {}),
@@ -723,3 +1001,11 @@ export async function probeExecutor(kind: ExecutorKind, config: ExecutorConfig =
     );
   }
 }
+
+/**
+ * Derive a concise human-readable activity summary from a Claude Code stream-json line.
+ * Exported for testing. Returns undefined when the line does not describe observable activity.
+ */
+export function parseClaudeActivityLine(line: string): string | undefined {
+  return deriveClaudeActivity(line);
+}