2071 lines
71 KiB
TypeScript
2071 lines
71 KiB
TypeScript
/**
|
|
* Workflow engine — implements the workflow CLI subcommands:
|
|
*
|
|
* start — validates plan, acquires lock, starts first execution cycle
|
|
* review — generates a review bundle for the Codex host
|
|
* retry-review — resumes a cleaned scope-gate review
|
|
* decide — receives Codex HostDecisionV1, drives state transitions
|
|
* extend — extends a budget_exhausted run with additional cycles
|
|
* status — prints current run state
|
|
* abort — terminates a run with a reason
|
|
*
|
|
* State machine:
|
|
* executing → awaiting_review → awaiting_host → executing (next cycle)
|
|
* ↘ awaiting_scope_resolution ↗
|
|
* Any active state → completed | needs_human | blocked | budget_exhausted | aborted
|
|
*
|
|
* Outputs AGENT_WORKFLOW_META_JSON: to stderr (or stdout when AGENT_WORKFLOW_META_STDOUT=1).
|
|
*/
|
|
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { execFileSync } from "node:child_process";
|
|
import { resolveWorkflowConfig, validateHostDecision, validateWorkflowPlan, WorkflowConfigError } from "./workflow-config.js";
|
|
import { checkConvergence, decisionSignatures } from "./workflow-convergence.js";
|
|
import type { FindingSignature } from "./workflow-convergence.js";
|
|
import {
|
|
createExecutorAdapter,
|
|
findReasonixSessionForWorkflow,
|
|
probeExecutor,
|
|
validateClaudeSessionForRepository,
|
|
validateClaudeSessionForWorkflow,
|
|
validateReasonixSessionForRepository,
|
|
} from "./workflow-executor.js";
|
|
import {
|
|
getOrCreateTaskBinding,
|
|
readTaskBinding,
|
|
saveTaskExecutorBinding,
|
|
WorkflowTaskBindingError,
|
|
} from "./workflow-task-binding.js";
|
|
import {
|
|
WorkflowLockError,
|
|
acquireLock,
|
|
appendEvent,
|
|
cycleDiffFile,
|
|
cycleDecisionFile,
|
|
cycleExecLogFile,
|
|
cycleHostReviewFile,
|
|
findRunDir,
|
|
generateRunId,
|
|
initWorkflowState,
|
|
acquireLogsFollowLock,
|
|
LogsFollowLockError,
|
|
nowIso,
|
|
readState,
|
|
readProgressSidecar,
|
|
releaseLogsFollowLock,
|
|
releaseLock,
|
|
runDir as makeRunDir,
|
|
writeState,
|
|
workflowsRoot,
|
|
} from "./workflow-state.js";
|
|
import {
|
|
createVcsProvider,
|
|
detectVcs,
|
|
extractBaselineHead,
|
|
VcsError,
|
|
} from "./vcs-provider.js";
|
|
import type {
|
|
CycleRecord,
|
|
ExecutorKind,
|
|
ExecutorResult,
|
|
ExecutorSessionHandle,
|
|
HostDecisionV1,
|
|
HostReviewBundleV1,
|
|
ScopeViolation,
|
|
VcsBaseline,
|
|
VcsKind,
|
|
WorkflowPlanV1,
|
|
WorkflowState,
|
|
WorkflowStatusOutput,
|
|
WorkflowTerminalStatus,
|
|
} from "./workflow-types.js";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Managed Abort Controllers for SIGTERM handling
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const activeAbortControllers = new Set<AbortController>();
|
|
|
|
process.on("SIGTERM", () => {
|
|
for (const ac of activeAbortControllers) {
|
|
ac.abort();
|
|
}
|
|
setTimeout(() => {
|
|
process.exit(143);
|
|
}, 500).unref();
|
|
});
|
|
|
|
export function createManagedController(timeoutSeconds?: number): {
|
|
ac: AbortController;
|
|
markActivity: () => void;
|
|
cleanup: () => void;
|
|
} {
|
|
const ac = new AbortController();
|
|
activeAbortControllers.add(ac);
|
|
let timer: NodeJS.Timeout | undefined;
|
|
|
|
const armTimer = () => {
|
|
if (!timeoutSeconds || ac.signal.aborted) return;
|
|
if (timer) clearTimeout(timer);
|
|
timer = setTimeout(() => {
|
|
ac.abort();
|
|
}, timeoutSeconds * 1000);
|
|
timer.unref();
|
|
};
|
|
|
|
armTimer();
|
|
return {
|
|
ac,
|
|
markActivity: armTimer,
|
|
cleanup: () => {
|
|
if (timer) clearTimeout(timer);
|
|
activeAbortControllers.delete(ac);
|
|
},
|
|
};
|
|
}
|
|
|
|
function writeStateSafety(dir: string, state: WorkflowState): void {
|
|
const diskState = readState(dir);
|
|
if (diskState && ["aborted", "completed", "needs_human", "blocked", "budget_exhausted"].includes(diskState.status)) {
|
|
throw new WorkflowEngineError(`Workflow was terminated externally (status: ${diskState.status}).`);
|
|
}
|
|
writeState(dir, state);
|
|
}
|
|
|
|
export function emitWorkflowMeta(meta: Record<string, unknown>): void {
|
|
const line = `AGENT_WORKFLOW_META_JSON: ${JSON.stringify(meta)}\n`;
|
|
const dest = process.env.AGENT_WORKFLOW_META_STDOUT?.toLowerCase();
|
|
if (dest === "1" || dest === "true" || dest === "stdout") {
|
|
process.stdout.write(line);
|
|
} else {
|
|
process.stderr.write(line);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helper: load state or fail
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function requireState(runId: string, repoRoot?: string): { state: WorkflowState; dir: string } {
|
|
const dir = findRunDir(runId, repoRoot);
|
|
if (!dir) throw new WorkflowEngineError(`Run '${runId}' not found.`);
|
|
const state = readState(dir);
|
|
if (!state) throw new WorkflowEngineError(`State for run '${runId}' is corrupt or missing.`);
|
|
return { state, dir };
|
|
}
|
|
|
|
export class WorkflowEngineError extends Error {
|
|
constructor(
|
|
message: string,
|
|
readonly exitCode: number = 4,
|
|
) {
|
|
super(message);
|
|
this.name = "WorkflowEngineError";
|
|
}
|
|
}
|
|
|
|
function makeScopeViolations(paths: string[]): ScopeViolation[] {
|
|
return [...new Set(paths)].sort().map((filePath, index) => ({
|
|
id: `scope-${index + 1}`,
|
|
path: filePath,
|
|
kind: "out_of_scope",
|
|
}));
|
|
}
|
|
|
|
function scopeViolationPaths(violations: ScopeViolation[] | undefined): string[] {
|
|
return (violations ?? []).map((violation) => violation.path).sort();
|
|
}
|
|
|
|
function sameScopeViolations(a: ScopeViolation[] | undefined, b: ScopeViolation[]): boolean {
|
|
const left = scopeViolationPaths(a);
|
|
const right = scopeViolationPaths(b);
|
|
return left.length === right.length && left.every((value, index) => value === right[index]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Terminal transition helper
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function transitionToTerminal(
|
|
state: WorkflowState,
|
|
dir: string,
|
|
terminal: WorkflowTerminalStatus,
|
|
description: string,
|
|
data?: Record<string, unknown>,
|
|
): void {
|
|
const diskState = readState(dir);
|
|
if (diskState && ["aborted", "completed", "needs_human", "blocked", "budget_exhausted"].includes(diskState.status)) {
|
|
return;
|
|
}
|
|
state.status = terminal;
|
|
state.stopReason = terminal;
|
|
state.stopDescription = description;
|
|
state.enginePid = undefined;
|
|
state.updatedAt = nowIso();
|
|
writeState(dir, state);
|
|
appendEvent(dir, {
|
|
kind: terminal === "aborted" ? "workflow_aborted" : "workflow_stopped",
|
|
runId: state.runId,
|
|
timestamp: state.updatedAt,
|
|
data: { reason: terminal, description, ...data },
|
|
});
|
|
releaseLock(state.repoRoot, state.runId);
|
|
}
|
|
|
|
/**
|
|
* Register an attempt log path in the cycle record BEFORE spawning the executor
|
|
* so that a crash during execution still leaves a discoverable log file.
|
|
*/
|
|
function registerAttemptLog(dir: string, cycleRecord: CycleRecord, logFilePath: string): void {
|
|
const priorLogs = cycleRecord.executorAttemptLogs
|
|
?? (cycleRecord.executorLogFile ? [cycleRecord.executorLogFile] : []);
|
|
const relativeLog = path.relative(dir, logFilePath);
|
|
cycleRecord.executorAttemptLogs = [...priorLogs, relativeLog];
|
|
cycleRecord.executorLogFile = relativeLog;
|
|
}
|
|
|
|
/**
|
|
* Finalize an attempt after the executor finishes: set exit code and report slice.
|
|
* The log path is already registered (by registerAttemptLog), so we never re-derive it.
|
|
*/
|
|
function persistExecutorAttempt(
|
|
dir: string,
|
|
cycleRecord: CycleRecord,
|
|
execResult: ExecutorResult,
|
|
logFilePath?: string,
|
|
): void {
|
|
// If the log file was created and registered before spawning (even if empty),
|
|
// keep it and don't overwrite. Empty logs are valid (e.g. immediate exit).
|
|
if (logFilePath && fs.existsSync(logFilePath)) {
|
|
const relativeLog = path.relative(dir, logFilePath);
|
|
cycleRecord.executorLogFile = relativeLog;
|
|
cycleRecord.executorExitCode = execResult.exitCode ?? undefined;
|
|
cycleRecord.executorReport = execResult.report.slice(0, 2000);
|
|
if (!cycleRecord.executorAttemptLogs?.includes(relativeLog)) {
|
|
cycleRecord.executorAttemptLogs = [...(cycleRecord.executorAttemptLogs ?? []), relativeLog];
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Fallback: write the full report if logFilePath wasn't provided (synchronous adapter or legacy path)
|
|
const attemptIndex = (cycleRecord.executorAttemptLogs?.length ?? 0);
|
|
const execLog = attemptIndex === 0
|
|
? cycleExecLogFile(dir, cycleRecord.cycleIndex)
|
|
: path.join(dir, `cycle-${cycleRecord.cycleIndex}-exec-retry-${attemptIndex}.log`);
|
|
fs.writeFileSync(execLog, execResult.report, "utf8");
|
|
const relativeLog = path.relative(dir, execLog);
|
|
if (!cycleRecord.executorAttemptLogs?.includes(relativeLog)) {
|
|
cycleRecord.executorAttemptLogs = [...(cycleRecord.executorAttemptLogs ?? []), relativeLog];
|
|
}
|
|
cycleRecord.executorLogFile = relativeLog;
|
|
cycleRecord.executorExitCode = execResult.exitCode ?? undefined;
|
|
cycleRecord.executorReport = execResult.report.slice(0, 2000);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// workflow start
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface StartWorkflowOpts {
|
|
planInput: string; // file path or "-" for stdin
|
|
executor?: string;
|
|
vcs?: string;
|
|
maxCycles?: number;
|
|
/** Start a fresh task binding for this Codex thread and repository. */
|
|
newTask?: boolean;
|
|
/** Overrides CODEX_THREAD_ID for programmatic callers and tests. */
|
|
codexThreadId?: string;
|
|
cwd?: string;
|
|
}
|
|
|
|
function validateTaskSessionHandle(
|
|
executor: ExecutorKind,
|
|
handle: ExecutorSessionHandle,
|
|
repoRoot: string,
|
|
): void {
|
|
if (handle.kind !== executor) {
|
|
throw new WorkflowTaskBindingError(`Task binding contains a ${handle.kind} handle for executor ${executor}.`);
|
|
}
|
|
switch (handle.kind) {
|
|
case "reasonix":
|
|
validateReasonixSessionForRepository(handle.sessionFilePath, repoRoot);
|
|
return;
|
|
case "claude":
|
|
validateClaudeSessionForRepository(handle.sessionId, repoRoot);
|
|
return;
|
|
case "agy":
|
|
if (handle.degraded || !handle.conversationId || !/^[0-9a-f-]{36}$/i.test(handle.conversationId)) {
|
|
throw new WorkflowTaskBindingError("Task binding contains an Agy session without a reliable conversation ID.");
|
|
}
|
|
}
|
|
}
|
|
|
|
function persistTaskSession(state: WorkflowState, handle: ExecutorSessionHandle): void {
|
|
if (!state.taskId || !state.codexThreadId) return;
|
|
try {
|
|
saveTaskExecutorBinding({
|
|
taskId: state.taskId,
|
|
codexThreadId: state.codexThreadId,
|
|
repoRoot: state.repoRoot,
|
|
executor: state.executor,
|
|
handle,
|
|
runId: state.runId,
|
|
planTitle: state.plan.title,
|
|
});
|
|
} catch (err) {
|
|
process.stderr.write(`Warning: failed to update task executor binding: ${(err as Error).message}\n`);
|
|
}
|
|
}
|
|
/** Resolve VCS selection from candidates and configs - exported for testing */
|
|
export async function resolveVcsSelection(
|
|
cwd: string,
|
|
explicitVcs?: VcsKind
|
|
): Promise<{ vcsKind: VcsKind; repoRoot: string; configSourceRoot?: string }> {
|
|
// Discover candidates
|
|
const { findVcsCandidates } = await import("./vcs-provider.js");
|
|
const candidates: { git?: string; svn?: string } = findVcsCandidates(cwd);
|
|
|
|
// Load config from candidate roots plus cwd to find defaultVcs
|
|
const configs = new Map<string, { vcs?: VcsKind | "auto"; root: string }>();
|
|
const rootsToTry = new Set<string>();
|
|
if (candidates.git) rootsToTry.add(candidates.git);
|
|
if (candidates.svn) rootsToTry.add(candidates.svn);
|
|
rootsToTry.add(cwd); // always try cwd as well
|
|
|
|
for (const root of rootsToTry) {
|
|
try {
|
|
const config = resolveWorkflowConfig(root, {});
|
|
configs.set(root, { vcs: config.vcs, root });
|
|
} catch (err) {
|
|
if (err instanceof WorkflowConfigError) {
|
|
throw new WorkflowEngineError(`Invalid agent-workflow.json at ${root}: ${err.message}`);
|
|
}
|
|
const errCode = (err as any)?.code;
|
|
if (errCode === "ENOENT") {
|
|
configs.set(root, { root });
|
|
} else {
|
|
throw err;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If explicit VCS provided, use it directly
|
|
if (explicitVcs) {
|
|
const provider = createVcsProvider(explicitVcs);
|
|
try {
|
|
const repoRoot = provider.repoRoot(cwd);
|
|
let configSourceRoot = repoRoot;
|
|
if (explicitVcs === "none") {
|
|
for (const root of rootsToTry) {
|
|
if (fs.existsSync(path.join(root, "agent-workflow.json"))) {
|
|
configSourceRoot = root;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return { vcsKind: explicitVcs, repoRoot, configSourceRoot };
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
throw new WorkflowEngineError(err.message);
|
|
}
|
|
throw new WorkflowEngineError((err as Error).message);
|
|
}
|
|
}
|
|
|
|
// Extract project defaultVcs settings from configs
|
|
const projectDefaults = Array.from(configs.values())
|
|
.map(c => c.vcs)
|
|
.filter((v): v is VcsKind => v !== undefined);
|
|
|
|
// Check for conflicting project defaults
|
|
const uniqueDefaults = new Set(projectDefaults);
|
|
if (uniqueDefaults.size > 1) {
|
|
throw new WorkflowEngineError(
|
|
`Conflicting VCS defaults in project configs: ${Array.from(uniqueDefaults).join(" vs ")}`
|
|
);
|
|
}
|
|
|
|
const projectDefault = projectDefaults[0];
|
|
|
|
if (projectDefault === "none") {
|
|
let configSourceRoot = cwd;
|
|
for (const c of configs.values()) {
|
|
if (c.vcs === "none") {
|
|
configSourceRoot = c.root;
|
|
break;
|
|
}
|
|
}
|
|
return { vcsKind: "none", repoRoot: cwd, configSourceRoot };
|
|
}
|
|
|
|
if (candidates.git && candidates.svn) {
|
|
// Ambiguous: need explicit selection
|
|
if (projectDefault === "git") {
|
|
return { vcsKind: "git", repoRoot: candidates.git, configSourceRoot: candidates.git };
|
|
} else if (projectDefault === "svn") {
|
|
return { vcsKind: "svn", repoRoot: candidates.svn, configSourceRoot: candidates.svn };
|
|
} else {
|
|
throw new WorkflowEngineError(
|
|
"Ambiguous VCS: both Git and SVN detected. " +
|
|
"Use --vcs git or --vcs svn to explicitly select, or set defaultVcs in agent-workflow.json."
|
|
);
|
|
}
|
|
} else if (candidates.git) {
|
|
if (projectDefault === "svn") {
|
|
throw new WorkflowEngineError(`Configured VCS is "svn" but no SVN working copy was detected.`);
|
|
}
|
|
return { vcsKind: "git", repoRoot: candidates.git, configSourceRoot: candidates.git };
|
|
} else if (candidates.svn) {
|
|
if (projectDefault === "git") {
|
|
throw new WorkflowEngineError(`Configured VCS is "git" but no Git repository was detected.`);
|
|
}
|
|
return { vcsKind: "svn", repoRoot: candidates.svn, configSourceRoot: candidates.svn };
|
|
} else {
|
|
// Neither detected
|
|
if (projectDefault === "git" || projectDefault === "svn") {
|
|
throw new WorkflowEngineError(
|
|
`Configured VCS is "${projectDefault}" but no such VCS repository was detected.`
|
|
);
|
|
}
|
|
return { vcsKind: "none", repoRoot: cwd, configSourceRoot: cwd };
|
|
}
|
|
}
|
|
|
|
export async function startWorkflow(opts: StartWorkflowOpts): Promise<{ runId: string }> {
|
|
const cwd = opts.cwd ?? process.cwd();
|
|
|
|
// 1. Load and validate plan first
|
|
let planRaw: string;
|
|
if (opts.planInput === "-") {
|
|
planRaw = fs.readFileSync(0, "utf8");
|
|
} else {
|
|
if (!fs.existsSync(opts.planInput)) {
|
|
throw new WorkflowEngineError(`Plan file not found: ${opts.planInput}`);
|
|
}
|
|
planRaw = fs.readFileSync(opts.planInput, "utf8");
|
|
}
|
|
|
|
let plan: unknown;
|
|
try {
|
|
plan = JSON.parse(planRaw);
|
|
} catch {
|
|
throw new WorkflowEngineError("Plan is not valid JSON.");
|
|
}
|
|
validateWorkflowPlan(plan);
|
|
const typedPlan = plan as WorkflowPlanV1;
|
|
|
|
// 2. Resolve VCS selection using deterministic helper
|
|
const { vcsKind, repoRoot, configSourceRoot } = await resolveVcsSelection(cwd, opts.vcs as VcsKind | undefined);
|
|
|
|
// Reject AGENT_WORKFLOW_DIR/workflow state root that is equal to or nested under target repoRoot in none mode
|
|
if (vcsKind === "none") {
|
|
const absRepoRoot = path.resolve(repoRoot);
|
|
const absStateRoot = path.resolve(workflowsRoot());
|
|
const relative = path.relative(absRepoRoot, absStateRoot);
|
|
const isNestedOrEqual = !relative.startsWith("..") && !path.isAbsolute(relative);
|
|
if (isNestedOrEqual) {
|
|
throw new WorkflowEngineError(
|
|
`Workflow state directory (${absStateRoot}) cannot be equal to or nested under the repository root (${absRepoRoot}) in none VCS mode. ` +
|
|
`Please set the AGENT_WORKFLOW_DIR environment variable to an external location.`,
|
|
4
|
|
);
|
|
}
|
|
}
|
|
|
|
// 3. Resolve full config from configSourceRoot or final repoRoot
|
|
const config = resolveWorkflowConfig(configSourceRoot ?? repoRoot, {
|
|
...(opts.executor ? { executor: opts.executor as import("./workflow-types.js").ExecutorKind } : {}),
|
|
...(opts.vcs ? { vcs: opts.vcs as VcsKind } : {}),
|
|
...(opts.maxCycles !== undefined ? { maxCycles: opts.maxCycles } : {}),
|
|
});
|
|
|
|
const vcsProvider = createVcsProvider(vcsKind);
|
|
const codexThreadId = (opts.codexThreadId ?? process.env.CODEX_THREAD_ID)?.trim();
|
|
if (opts.newTask && !codexThreadId) {
|
|
throw new WorkflowEngineError("--new-task requires CODEX_THREAD_ID to identify the owning Codex session.", 4);
|
|
}
|
|
|
|
let existingTaskBinding;
|
|
let taskSessionHandle: ExecutorSessionHandle | undefined;
|
|
if (codexThreadId && !opts.newTask) {
|
|
try {
|
|
existingTaskBinding = readTaskBinding(repoRoot, codexThreadId);
|
|
taskSessionHandle = existingTaskBinding?.executors[config.executor]?.handle;
|
|
if (taskSessionHandle) validateTaskSessionHandle(config.executor, taskSessionHandle, repoRoot);
|
|
} catch (err) {
|
|
throw new WorkflowEngineError(
|
|
`Cannot reuse the current task's ${config.executor} session: ${(err as Error).message} ` +
|
|
"Start with --new-task only when this is intentionally a new task.",
|
|
4,
|
|
);
|
|
}
|
|
}
|
|
|
|
// 4. Probe executor
|
|
await probeExecutor(config.executor, config.executorConfig);
|
|
|
|
// 5. Require clean working tree (skipped in none mode)
|
|
if (vcsKind !== "none") {
|
|
let isClean: boolean;
|
|
try {
|
|
isClean = vcsProvider.isClean(repoRoot);
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
throw new WorkflowEngineError(err.message);
|
|
}
|
|
throw new WorkflowEngineError((err as Error).message);
|
|
}
|
|
if (!isClean) {
|
|
throw new WorkflowEngineError(
|
|
"Working tree has uncommitted changes. Commit or stash them before starting a workflow.",
|
|
4,
|
|
);
|
|
}
|
|
}
|
|
|
|
// 6. Acquire lock (one active workflow per repo)
|
|
const runId = generateRunId();
|
|
try {
|
|
acquireLock(repoRoot, runId);
|
|
} catch (err) {
|
|
if (err instanceof WorkflowLockError) {
|
|
throw new WorkflowEngineError(err.message, 4);
|
|
}
|
|
throw err;
|
|
}
|
|
|
|
const dir = makeRunDir(repoRoot, runId);
|
|
fs.mkdirSync(dir, { recursive: true });
|
|
|
|
// 7. Capture baseline
|
|
let vcsBaseline: VcsBaseline;
|
|
try {
|
|
vcsBaseline = vcsProvider.captureBaseline(repoRoot, dir);
|
|
} catch (err) {
|
|
try {
|
|
releaseLock(repoRoot, runId);
|
|
} catch {}
|
|
if (err instanceof VcsError) {
|
|
throw new WorkflowEngineError(err.message);
|
|
}
|
|
throw new WorkflowEngineError((err as Error).message);
|
|
}
|
|
|
|
const state = initWorkflowState({
|
|
runId,
|
|
repoRoot,
|
|
vcsBaseline,
|
|
vcsKind,
|
|
executor: config.executor,
|
|
plan: typedPlan,
|
|
maxCycles: config.maxCycles,
|
|
timeoutSeconds: config.timeoutSeconds,
|
|
});
|
|
state.executorConfig = config.executorConfig;
|
|
state.timeoutSeconds = config.timeoutSeconds;
|
|
state.enginePid = process.pid;
|
|
state.status = "executing";
|
|
if (codexThreadId) {
|
|
try {
|
|
const binding = opts.newTask || !existingTaskBinding
|
|
? getOrCreateTaskBinding({
|
|
repoRoot,
|
|
codexThreadId,
|
|
planTitle: typedPlan.title,
|
|
newTask: opts.newTask,
|
|
})
|
|
: existingTaskBinding;
|
|
state.taskId = binding.taskId;
|
|
state.codexThreadId = codexThreadId;
|
|
state.taskSessionReused = !!taskSessionHandle;
|
|
if (taskSessionHandle) state.sessionHandle = taskSessionHandle;
|
|
} catch (err) {
|
|
releaseLock(repoRoot, runId);
|
|
throw new WorkflowEngineError(`Cannot initialize Codex task binding: ${(err as Error).message}`, 4);
|
|
}
|
|
}
|
|
writeState(dir, state);
|
|
appendEvent(dir, {
|
|
kind: "workflow_started",
|
|
runId,
|
|
timestamp: state.createdAt,
|
|
data: {
|
|
executor: config.executor,
|
|
maxCycles: config.maxCycles,
|
|
vcsKind,
|
|
vcsBaseline,
|
|
...(state.taskId ? { taskId: state.taskId, taskSessionReused: state.taskSessionReused } : {}),
|
|
},
|
|
});
|
|
|
|
process.stdout.write(`workflow started: ${runId}\n`);
|
|
process.stdout.write(` vcs: ${vcsKind}\n`);
|
|
process.stdout.write(` executor: ${config.executor}\n`);
|
|
process.stdout.write(` max-cycles: ${config.maxCycles}\n`);
|
|
if (state.taskId) {
|
|
process.stdout.write(` task: ${state.taskId}${state.taskSessionReused ? " (session reused)" : ""}\n`);
|
|
}
|
|
if (vcsBaseline.kind === "git") {
|
|
process.stdout.write(` baseline: ${vcsBaseline.head}\n`);
|
|
} else if (vcsBaseline.kind === "svn") {
|
|
process.stdout.write(` baseline: r${vcsBaseline.revision}\n`);
|
|
} else if (vcsBaseline.kind === "none") {
|
|
process.stdout.write(` baseline: snapshot\n`);
|
|
}
|
|
process.stdout.write(` run dir: ${dir}\n\n`);
|
|
|
|
// 7. Start first execution cycle
|
|
const cycleIndex = 1;
|
|
const cycleRecord: CycleRecord = { cycleIndex, startedAt: nowIso() };
|
|
state.cycles.push(cycleRecord);
|
|
state.currentCycle = cycleIndex;
|
|
state.status = "executing";
|
|
state.updatedAt = nowIso();
|
|
writeState(dir, state);
|
|
|
|
appendEvent(dir, { kind: "cycle_started", runId, timestamp: nowIso(), cycleIndex });
|
|
appendEvent(dir, {
|
|
kind: "executor_started",
|
|
runId,
|
|
timestamp: nowIso(),
|
|
cycleIndex,
|
|
data: { executor: config.executor, sessionMode: taskSessionHandle ? "task_reuse" : "new" },
|
|
});
|
|
if (taskSessionHandle) {
|
|
appendEvent(dir, {
|
|
kind: "executor_session_reused",
|
|
runId,
|
|
timestamp: nowIso(),
|
|
cycleIndex,
|
|
data: { executor: config.executor, taskId: state.taskId },
|
|
});
|
|
}
|
|
|
|
const adapter = createExecutorAdapter(config.executor);
|
|
const { ac, markActivity, cleanup } = createManagedController(state.timeoutSeconds);
|
|
|
|
// Create attempt log file BEFORE spawning the executor
|
|
const execLogPath = path.join(dir, `cycle-${cycleIndex}-exec.log`);
|
|
fs.writeFileSync(execLogPath, "", "utf8");
|
|
registerAttemptLog(dir, cycleRecord, execLogPath);
|
|
writeState(dir, state); // persist so retry can discover the log
|
|
|
|
let execResult;
|
|
try {
|
|
const commonOpts = {
|
|
repoRoot,
|
|
plan: typedPlan,
|
|
runDir: dir,
|
|
cycleIndex,
|
|
logFilePath: execLogPath,
|
|
config: config.executorConfig,
|
|
timeoutSeconds: state.timeoutSeconds,
|
|
signal: ac.signal,
|
|
onActivity: markActivity,
|
|
vcsKind,
|
|
};
|
|
execResult = taskSessionHandle
|
|
? await adapter.resume({
|
|
...commonOpts,
|
|
handle: taskSessionHandle,
|
|
acceptedFindings: "",
|
|
purpose: "task_plan",
|
|
})
|
|
: await adapter.start(commonOpts);
|
|
} catch (err) {
|
|
cleanup();
|
|
transitionToTerminal(state, dir, "blocked", `Executor failed to start: ${(err as Error).message}`);
|
|
throw new WorkflowEngineError(`Executor error: ${(err as Error).message}`);
|
|
}
|
|
cleanup();
|
|
|
|
persistExecutorAttempt(dir, cycleRecord, execResult, execLogPath);
|
|
if (execResult.sessionHandle) {
|
|
state.sessionHandle = execResult.sessionHandle;
|
|
persistTaskSession(state, execResult.sessionHandle);
|
|
if (execResult.sessionHandle.kind === "agy") {
|
|
state.agyDegraded = execResult.sessionHandle.degraded;
|
|
}
|
|
}
|
|
|
|
appendEvent(dir, {
|
|
kind: "executor_completed",
|
|
runId,
|
|
timestamp: nowIso(),
|
|
cycleIndex,
|
|
data: {
|
|
exitCode: execResult.exitCode,
|
|
durationMs: execResult.durationMs,
|
|
failed: execResult.failed,
|
|
...(execResult.failureReason ? { failureReason: execResult.failureReason } : {}),
|
|
},
|
|
});
|
|
|
|
if (execResult.failed) {
|
|
transitionToTerminal(state, dir, "blocked", `Executor exited with failure: ${execResult.failureReason ?? "exit code " + String(execResult.exitCode)}`);
|
|
throw new WorkflowEngineError("Executor failed.");
|
|
}
|
|
|
|
// Transition to awaiting_review
|
|
state.status = "awaiting_review";
|
|
state.enginePid = undefined;
|
|
state.updatedAt = nowIso();
|
|
writeStateSafety(dir, state);
|
|
|
|
process.stdout.write(`cycle ${cycleIndex} execution completed\n`);
|
|
process.stdout.write(`status: awaiting_review\n`);
|
|
process.stdout.write(`next: agent-workflow review --run ${runId}\n`);
|
|
|
|
emitWorkflowMeta({ runId, status: state.status, cycleIndex, executor: config.executor });
|
|
return { runId };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// workflow review
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface ReviewWorkflowOpts {
|
|
runId: string;
|
|
cwd?: string;
|
|
}
|
|
|
|
export async function reviewWorkflow(opts: ReviewWorkflowOpts): Promise<void> {
|
|
const { state, dir } = requireState(opts.runId);
|
|
|
|
if (state.status !== "awaiting_review") {
|
|
throw new WorkflowEngineError(
|
|
`Run '${opts.runId}' is in status '${state.status}', not awaiting_review.`,
|
|
2,
|
|
);
|
|
}
|
|
|
|
const cycleIndex = state.currentCycle;
|
|
const cycleRecord = state.cycles.find((c) => c.cycleIndex === cycleIndex)!;
|
|
|
|
state.enginePid = process.pid;
|
|
writeStateSafety(dir, state);
|
|
|
|
// Get VCS provider
|
|
const vcsKind = state.vcsKind || "git";
|
|
const vcsProvider = createVcsProvider(vcsKind);
|
|
const baseline = state.vcsBaseline || (state.baselineHead ? { kind: "git" as const, head: state.baselineHead } : undefined);
|
|
if (!baseline) {
|
|
throw new WorkflowEngineError("No baseline found in state (corrupt state file).");
|
|
}
|
|
|
|
// 1. Generate diff from baseline
|
|
let diff: string;
|
|
try {
|
|
diff = vcsProvider.diffFromBaseline(state.repoRoot, baseline, dir);
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
transitionToTerminal(state, dir, "blocked", `Failed to generate diff: ${err.message}`);
|
|
throw new WorkflowEngineError(err.message);
|
|
}
|
|
throw err;
|
|
}
|
|
const diffFile = cycleDiffFile(dir, cycleIndex);
|
|
fs.writeFileSync(diffFile, diff, "utf8");
|
|
cycleRecord.diffFile = path.relative(dir, diffFile);
|
|
|
|
// 2. Check for baseline drift
|
|
try {
|
|
vcsProvider.validateBaseline(state.repoRoot, baseline, dir);
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
transitionToTerminal(
|
|
state,
|
|
dir,
|
|
"blocked",
|
|
`Baseline validation failed: ${err.message}`,
|
|
);
|
|
throw new WorkflowEngineError(err.message);
|
|
}
|
|
throw err;
|
|
}
|
|
|
|
// 3. Check out-of-scope files
|
|
let outOfScope: string[];
|
|
try {
|
|
outOfScope = vcsProvider.checkFilesInScope(state.repoRoot, baseline, state.plan.scope, dir);
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
transitionToTerminal(state, dir, "blocked", `Failed to check scope: ${err.message}`);
|
|
throw new WorkflowEngineError(err.message);
|
|
}
|
|
throw err;
|
|
}
|
|
if (outOfScope.length > 0) {
|
|
const scopeViolations = makeScopeViolations(outOfScope);
|
|
const previousCycle = state.cycles.find((cycle) => cycle.cycleIndex === cycleIndex - 1);
|
|
if (previousCycle && sameScopeViolations(previousCycle.scopeViolations, scopeViolations)) {
|
|
transitionToTerminal(
|
|
state,
|
|
dir,
|
|
"needs_human",
|
|
`Scope remediation did not converge: ${outOfScope.sort().join(", ")}`,
|
|
{ cycleIndex, scopeViolations },
|
|
);
|
|
emitWorkflowMeta({ runId: state.runId, status: "needs_human", cycleIndex, scopeViolations });
|
|
throw new WorkflowEngineError("Scope remediation did not converge.", 3);
|
|
}
|
|
|
|
cycleRecord.scopeViolations = scopeViolations;
|
|
state.status = "awaiting_scope_resolution";
|
|
state.enginePid = undefined;
|
|
state.updatedAt = nowIso();
|
|
writeStateSafety(dir, state);
|
|
appendEvent(dir, {
|
|
kind: "scope_violation_detected",
|
|
runId: state.runId,
|
|
timestamp: state.updatedAt,
|
|
cycleIndex,
|
|
data: { scopeViolations },
|
|
});
|
|
|
|
process.stdout.write(`scope violations detected (cycle ${cycleIndex})\n`);
|
|
for (const violation of scopeViolations) {
|
|
process.stdout.write(` ${violation.id}: ${violation.path}\n`);
|
|
}
|
|
process.stdout.write(`next: agent-workflow decide --run ${opts.runId} --input <decision.json|->\n`);
|
|
emitWorkflowMeta({
|
|
runId: state.runId,
|
|
status: state.status,
|
|
cycleIndex,
|
|
scopeViolations,
|
|
});
|
|
return;
|
|
}
|
|
|
|
const generatedAt = nowIso();
|
|
appendEvent(dir, { kind: "review_started", runId: state.runId, timestamp: generatedAt, cycleIndex });
|
|
|
|
// 4. Persist all evidence needed by the Codex host. The CLI does not invoke
|
|
// another reviewer: Codex owns review and final acceptance.
|
|
const hostReviewFile = cycleHostReviewFile(dir, cycleIndex);
|
|
const bundle: HostReviewBundleV1 = {
|
|
version: "1",
|
|
runId: state.runId,
|
|
cycleIndex,
|
|
repoRoot: state.repoRoot,
|
|
...(state.baselineHead ? { baselineHead: state.baselineHead } : {}),
|
|
...(state.vcsBaseline ? { vcsBaseline: state.vcsBaseline } : {}),
|
|
...(state.vcsKind ? { vcsKind: state.vcsKind } : {}),
|
|
generatedAt,
|
|
plan: state.plan,
|
|
diffFile: diffFile,
|
|
...(cycleRecord.executorLogFile
|
|
? { executorLogFile: path.join(dir, cycleRecord.executorLogFile) }
|
|
: {}),
|
|
...(cycleRecord.executorReport ? { executorReport: cycleRecord.executorReport } : {}),
|
|
verificationCommands: state.plan.verificationCommands,
|
|
};
|
|
fs.writeFileSync(hostReviewFile, JSON.stringify(bundle, null, 2), "utf8");
|
|
cycleRecord.hostReviewFile = path.relative(dir, hostReviewFile);
|
|
cycleRecord.hostReviewReadyAt = generatedAt;
|
|
|
|
appendEvent(dir, {
|
|
kind: "review_completed",
|
|
runId: state.runId,
|
|
timestamp: generatedAt,
|
|
cycleIndex,
|
|
data: { hostReviewFile, diffFile },
|
|
});
|
|
|
|
state.status = "awaiting_host";
|
|
state.enginePid = undefined;
|
|
state.updatedAt = generatedAt;
|
|
writeStateSafety(dir, state);
|
|
|
|
process.stdout.write(`Codex review bundle ready (cycle ${cycleIndex})\n`);
|
|
process.stdout.write(` bundle: ${hostReviewFile}\n`);
|
|
process.stdout.write(` diff: ${diffFile}\n`);
|
|
process.stdout.write(`next: agent-workflow decide --run ${opts.runId} --input <decision.json|->\n`);
|
|
emitWorkflowMeta({
|
|
runId: state.runId,
|
|
status: state.status,
|
|
cycleIndex,
|
|
hostReviewFile,
|
|
diffFile,
|
|
});
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// workflow decide
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface DecideWorkflowOpts {
|
|
runId: string;
|
|
decisionInput: string; // file path or "-" for stdin
|
|
cwd?: string;
|
|
}
|
|
|
|
export async function decideWorkflow(opts: DecideWorkflowOpts): Promise<void> {
|
|
const { state, dir } = requireState(opts.runId);
|
|
|
|
const resolvingScope = state.status === "awaiting_scope_resolution";
|
|
if (state.status !== "awaiting_host" && !resolvingScope) {
|
|
throw new WorkflowEngineError(
|
|
`Run '${opts.runId}' is in status '${state.status}', not awaiting_host or awaiting_scope_resolution.`,
|
|
2,
|
|
);
|
|
}
|
|
|
|
// Load decision
|
|
let decisionRaw: string;
|
|
if (opts.decisionInput === "-") {
|
|
decisionRaw = fs.readFileSync(0, "utf8");
|
|
} else {
|
|
if (!fs.existsSync(opts.decisionInput)) {
|
|
throw new WorkflowEngineError(`Decision file not found: ${opts.decisionInput}`);
|
|
}
|
|
decisionRaw = fs.readFileSync(opts.decisionInput, "utf8");
|
|
}
|
|
|
|
let decisionObj: unknown;
|
|
try {
|
|
decisionObj = JSON.parse(decisionRaw);
|
|
} catch {
|
|
throw new WorkflowEngineError("Decision is not valid JSON.", 2);
|
|
}
|
|
validateHostDecision(decisionObj);
|
|
const decision = decisionObj as HostDecisionV1;
|
|
|
|
const cycleIndex = state.currentCycle;
|
|
const cycleRecord = state.cycles.find((c) => c.cycleIndex === cycleIndex)!;
|
|
|
|
if (resolvingScope && decision.outcome === "accept") {
|
|
throw new WorkflowEngineError("Cannot accept while scope violations remain. Use fix or needs_human.", 4);
|
|
}
|
|
if (resolvingScope && decision.outcome === "fix") {
|
|
const expectedIds = new Set((cycleRecord.scopeViolations ?? []).map((violation) => violation.id));
|
|
const submittedIds = decision.findingDecisions.map((finding) => finding.findingId);
|
|
const acceptedIds = new Set(
|
|
decision.findingDecisions
|
|
.filter((finding) => finding.disposition === "accept")
|
|
.map((finding) => finding.findingId),
|
|
);
|
|
const unknownIds = decision.findingDecisions
|
|
.map((finding) => finding.findingId)
|
|
.filter((id) => !expectedIds.has(id));
|
|
const missingIds = [...expectedIds].filter((id) => !acceptedIds.has(id));
|
|
const duplicateIds = submittedIds.filter((id, index) => submittedIds.indexOf(id) !== index);
|
|
if (expectedIds.size === 0 || unknownIds.length > 0 || missingIds.length > 0 || duplicateIds.length > 0) {
|
|
throw new WorkflowEngineError(
|
|
`Scope fix must accept every current violation exactly once. Missing: ${missingIds.join(", ") || "none"}; unknown: ${unknownIds.join(", ") || "none"}; duplicate: ${[...new Set(duplicateIds)].join(", ") || "none"}.`,
|
|
4,
|
|
);
|
|
}
|
|
}
|
|
if (!resolvingScope && decision.outcome === "fix") {
|
|
const accepted = decision.findingDecisions.filter((finding) => finding.disposition === "accept");
|
|
if (accepted.length === 0) {
|
|
throw new WorkflowEngineError("Fix decision must accept at least one Codex finding.", 4);
|
|
}
|
|
const missingSummary = accepted
|
|
.filter((finding) => !finding.summary?.trim())
|
|
.map((finding) => finding.findingId);
|
|
if (missingSummary.length > 0) {
|
|
throw new WorkflowEngineError(
|
|
`Accepted Codex findings require a non-empty summary: ${missingSummary.join(", ")}.`,
|
|
4,
|
|
);
|
|
}
|
|
}
|
|
if (!resolvingScope && decision.outcome === "accept"
|
|
&& decision.findingDecisions.some((finding) => finding.disposition === "accept")) {
|
|
throw new WorkflowEngineError("Accept decision cannot contain findings accepted for fixing.", 4);
|
|
}
|
|
|
|
// Save decision file
|
|
const decFile = cycleDecisionFile(dir, cycleIndex);
|
|
fs.writeFileSync(decFile, JSON.stringify(decision, null, 2), "utf8");
|
|
cycleRecord.decisionFile = path.relative(dir, decFile);
|
|
cycleRecord.decisionOutcome = decision.outcome;
|
|
cycleRecord.completedAt = nowIso();
|
|
|
|
appendEvent(dir, {
|
|
kind: "decision_received",
|
|
runId: state.runId,
|
|
timestamp: nowIso(),
|
|
cycleIndex,
|
|
data: { outcome: decision.outcome, reason: decision.reason },
|
|
});
|
|
|
|
// Handle decision outcomes
|
|
if (decision.outcome === "needs_human") {
|
|
transitionToTerminal(
|
|
state,
|
|
dir,
|
|
"needs_human",
|
|
decision.reason ?? "Codex determined human intervention is required.",
|
|
{ cycleIndex },
|
|
);
|
|
process.stdout.write(`workflow stopped: needs_human\n`);
|
|
process.stdout.write(`reason: ${decision.reason ?? "human intervention required"}\n`);
|
|
emitWorkflowMeta({ runId: state.runId, status: "needs_human", cycleIndex });
|
|
process.exit(3);
|
|
}
|
|
|
|
if (decision.outcome === "accept") {
|
|
// Validate accept conditions
|
|
const acceptError = validateAcceptConditions(state, dir, cycleIndex, decision);
|
|
if (acceptError) {
|
|
throw new WorkflowEngineError(`Cannot accept: ${acceptError}`, 4);
|
|
}
|
|
transitionToTerminal(state, dir, "completed", "Codex accepted — all conditions met.", { cycleIndex });
|
|
process.stdout.write(`workflow completed: accepted\n`);
|
|
emitWorkflowMeta({ runId: state.runId, status: "completed", cycleIndex });
|
|
process.exit(0);
|
|
}
|
|
|
|
// decision.outcome === "fix" — start next cycle
|
|
const nextCycle = cycleIndex + 1;
|
|
if (nextCycle > state.maxCycles) {
|
|
transitionToTerminal(
|
|
state,
|
|
dir,
|
|
"budget_exhausted",
|
|
`Reached maximum cycles (${state.maxCycles}). Further execution not allowed.`,
|
|
);
|
|
process.stdout.write(`workflow stopped: budget_exhausted (max cycles: ${state.maxCycles})\n`);
|
|
emitWorkflowMeta({ runId: state.runId, status: "budget_exhausted", cycleIndex });
|
|
process.exit(1);
|
|
}
|
|
|
|
// Check convergence using finding signatures from all cycles with review data
|
|
const convergenceError = resolvingScope ? undefined : checkWorkflowConvergence(state, dir);
|
|
if (convergenceError) {
|
|
transitionToTerminal(state, dir, "needs_human", convergenceError, { cycleIndex });
|
|
process.stdout.write(`workflow stopped: needs_human (non-convergence detected)\n`);
|
|
process.stdout.write(`reason: ${convergenceError}\n`);
|
|
emitWorkflowMeta({ runId: state.runId, status: "needs_human", cycleIndex });
|
|
process.exit(3);
|
|
}
|
|
|
|
// Build fix prompt from accepted findings
|
|
const acceptedFindings = resolvingScope
|
|
? buildScopeResolutionText(cycleRecord.scopeViolations ?? [])
|
|
: buildAcceptedFindingsText(decision, state, dir, cycleIndex);
|
|
|
|
// Start next cycle
|
|
const nextCycleRecord: CycleRecord = { cycleIndex: nextCycle, startedAt: nowIso() };
|
|
state.cycles.push(nextCycleRecord);
|
|
state.currentCycle = nextCycle;
|
|
state.status = "executing";
|
|
state.updatedAt = nowIso();
|
|
state.enginePid = process.pid;
|
|
writeStateSafety(dir, state);
|
|
|
|
appendEvent(dir, { kind: "cycle_started", runId: state.runId, timestamp: nowIso(), cycleIndex: nextCycle });
|
|
if (resolvingScope) {
|
|
appendEvent(dir, {
|
|
kind: "scope_resolution_started",
|
|
runId: state.runId,
|
|
timestamp: nowIso(),
|
|
cycleIndex: nextCycle,
|
|
data: { scopeViolations: cycleRecord.scopeViolations },
|
|
});
|
|
}
|
|
|
|
await resumeExecutorCycle(state, dir, nextCycleRecord, acceptedFindings);
|
|
}
|
|
|
|
async function resumeExecutorCycle(
|
|
state: WorkflowState,
|
|
dir: string,
|
|
cycleRecord: CycleRecord,
|
|
acceptedFindings: string,
|
|
): Promise<void> {
|
|
const cycleIndex = cycleRecord.cycleIndex;
|
|
if (!state.sessionHandle) {
|
|
transitionToTerminal(state, dir, "blocked", "No session handle available for resume.", { cycleIndex });
|
|
throw new WorkflowEngineError("Cannot resume: no session handle. Workflow blocked.");
|
|
}
|
|
|
|
const adapter = createExecutorAdapter(state.executor);
|
|
const { ac, markActivity, cleanup } = createManagedController(state.timeoutSeconds);
|
|
|
|
// Determine attempt index and create log file before spawning
|
|
const priorLogs = cycleRecord.executorAttemptLogs
|
|
?? (cycleRecord.executorLogFile ? [cycleRecord.executorLogFile] : []);
|
|
const attemptIndex = priorLogs.length;
|
|
const execLogPath = attemptIndex === 0
|
|
? path.join(dir, `cycle-${cycleIndex}-exec.log`)
|
|
: path.join(dir, `cycle-${cycleIndex}-exec-retry-${attemptIndex}.log`);
|
|
fs.writeFileSync(execLogPath, "", "utf8");
|
|
registerAttemptLog(dir, cycleRecord, execLogPath);
|
|
writeState(dir, state); // persist so retry can discover the log
|
|
|
|
let execResult;
|
|
try {
|
|
execResult = await adapter.resume({
|
|
repoRoot: state.repoRoot,
|
|
plan: state.plan,
|
|
acceptedFindings,
|
|
handle: state.sessionHandle,
|
|
runDir: dir,
|
|
cycleIndex,
|
|
logFilePath: execLogPath,
|
|
config: state.executorConfig ?? {},
|
|
timeoutSeconds: state.timeoutSeconds,
|
|
signal: ac.signal,
|
|
onActivity: markActivity,
|
|
vcsKind: state.vcsKind || "git",
|
|
});
|
|
} catch (err) {
|
|
cleanup();
|
|
transitionToTerminal(
|
|
state,
|
|
dir,
|
|
"blocked",
|
|
`Executor resume failed: ${(err as Error).message}`,
|
|
{ cycleIndex },
|
|
);
|
|
throw new WorkflowEngineError(`Executor resume error: ${(err as Error).message}`);
|
|
}
|
|
cleanup();
|
|
|
|
persistExecutorAttempt(dir, cycleRecord, execResult, execLogPath);
|
|
if (execResult.sessionHandle) {
|
|
state.sessionHandle = execResult.sessionHandle;
|
|
persistTaskSession(state, execResult.sessionHandle);
|
|
if (execResult.sessionHandle.kind === "agy") {
|
|
state.agyDegraded = execResult.sessionHandle.degraded;
|
|
}
|
|
}
|
|
|
|
appendEvent(dir, {
|
|
kind: "executor_completed",
|
|
runId: state.runId,
|
|
timestamp: nowIso(),
|
|
cycleIndex,
|
|
data: {
|
|
exitCode: execResult.exitCode,
|
|
durationMs: execResult.durationMs,
|
|
failed: execResult.failed,
|
|
...(execResult.failureReason ? { failureReason: execResult.failureReason } : {}),
|
|
},
|
|
});
|
|
|
|
if (execResult.failed) {
|
|
transitionToTerminal(
|
|
state,
|
|
dir,
|
|
"blocked",
|
|
`Executor failed to resume: ${execResult.failureReason ?? "exit code " + String(execResult.exitCode)}`,
|
|
{ cycleIndex },
|
|
);
|
|
throw new WorkflowEngineError("Executor failed.");
|
|
}
|
|
|
|
state.status = "awaiting_review";
|
|
state.enginePid = undefined;
|
|
state.updatedAt = nowIso();
|
|
writeStateSafety(dir, state);
|
|
|
|
process.stdout.write(`cycle ${cycleIndex} execution completed\n`);
|
|
process.stdout.write(`status: awaiting_review\n`);
|
|
process.stdout.write(`next: agent-workflow review --run ${state.runId}\n`);
|
|
emitWorkflowMeta({ runId: state.runId, status: state.status, cycleIndex });
|
|
}
|
|
|
|
function validateAcceptConditions(state: WorkflowState, dir: string, cycleIndex: number, decision: HostDecisionV1): string | undefined {
|
|
// 1. Codex must have received a persisted review bundle for this cycle.
|
|
const cycle = state.cycles.find((c) => c.cycleIndex === cycleIndex);
|
|
if (!cycle?.hostReviewFile) return "No Codex review bundle for this cycle.";
|
|
if (!fs.existsSync(path.join(dir, cycle.hostReviewFile))) {
|
|
return "Codex review bundle is missing from disk.";
|
|
}
|
|
|
|
// 2. Baseline must not have changed
|
|
const vcsKind = state.vcsKind || "git";
|
|
const vcsProvider = createVcsProvider(vcsKind);
|
|
const baseline = state.vcsBaseline || (state.baselineHead ? { kind: "git" as const, head: state.baselineHead } : undefined);
|
|
if (!baseline) {
|
|
return "No baseline found in state.";
|
|
}
|
|
|
|
try {
|
|
vcsProvider.validateBaseline(state.repoRoot, baseline, dir);
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
return err.message;
|
|
}
|
|
return `Baseline validation failed: ${(err as Error).message}`;
|
|
}
|
|
|
|
// 3. All files must be in scope
|
|
let outOfScope: string[];
|
|
try {
|
|
outOfScope = vcsProvider.checkFilesInScope(state.repoRoot, baseline, state.plan.scope, dir);
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
return `Scope check failed: ${err.message}`;
|
|
}
|
|
return `Scope check failed: ${(err as Error).message}`;
|
|
}
|
|
if (outOfScope.length > 0) {
|
|
return `Out-of-scope files: ${outOfScope.join(", ")}`;
|
|
}
|
|
|
|
// 4. Verification commands must have evidence and succeed
|
|
if (state.plan.verificationCommands && state.plan.verificationCommands.length > 0) {
|
|
if (!decision.verificationEvidence || decision.verificationEvidence.length === 0) {
|
|
return "Accept decision requires verificationEvidence when plan specifies verificationCommands.";
|
|
}
|
|
for (const cmd of state.plan.verificationCommands) {
|
|
if (!cmd || cmd.length === 0) continue;
|
|
try {
|
|
execFileSync(cmd[0], cmd.slice(1), {
|
|
cwd: state.repoRoot,
|
|
encoding: "utf8",
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
timeout: (state.timeoutSeconds ?? 300) * 1000,
|
|
});
|
|
} catch (err: any) {
|
|
return `Verification command failed: ${cmd.join(" ")}\nError: ${err.message}`;
|
|
}
|
|
}
|
|
}
|
|
|
|
return undefined;
|
|
}
|
|
|
|
function checkWorkflowConvergence(state: WorkflowState, dir: string): string | undefined {
|
|
const history: FindingSignature[][] = [];
|
|
for (const cycle of state.cycles) {
|
|
if (!cycle.decisionFile) continue;
|
|
const decisionPath = path.join(dir, cycle.decisionFile);
|
|
if (!fs.existsSync(decisionPath)) continue;
|
|
try {
|
|
const priorDecision = JSON.parse(fs.readFileSync(decisionPath, "utf8")) as HostDecisionV1;
|
|
history.push(decisionSignatures(priorDecision.findingDecisions));
|
|
} catch { /* skip corrupt files */ }
|
|
}
|
|
|
|
const result = checkConvergence(history);
|
|
if (!result.converging) {
|
|
if (result.persistentFindings?.length) {
|
|
return `Findings not converging: the same ${result.persistentFindings.length} issue(s) persist across consecutive cycles. Human review required.`;
|
|
}
|
|
if (result.oscillatingFindings?.length) {
|
|
return `Findings oscillating: ${result.oscillatingFindings.length} issue(s) disappeared and reappeared. Human review required.`;
|
|
}
|
|
return "Findings not converging. Human review required.";
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function buildAcceptedFindingsText(
|
|
decision: HostDecisionV1,
|
|
_state: WorkflowState,
|
|
_dir: string,
|
|
_cycleIndex: number,
|
|
): string {
|
|
const accepted = decision.findingDecisions.filter((d) => d.disposition === "accept");
|
|
|
|
const lines = ["# Codex Review Findings", "", "Fix only the following issues accepted by the Codex host:"];
|
|
for (const d of accepted) {
|
|
lines.push(`\n### Finding ${d.findingId}`);
|
|
if (d.path) lines.push(`- Path: ${d.path}`);
|
|
lines.push(`- Issue: ${d.summary}`);
|
|
if (d.rationale) lines.push(`- Rationale: ${d.rationale}`);
|
|
}
|
|
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function buildScopeResolutionText(violations: ScopeViolation[]): string {
|
|
const lines = [
|
|
"# Scope Remediation",
|
|
"",
|
|
"The frozen scope gate found these exact out-of-scope paths:",
|
|
];
|
|
for (const violation of violations) {
|
|
lines.push(`- ${violation.id}: ${violation.path}`);
|
|
}
|
|
lines.push(
|
|
"",
|
|
"Inspect every listed path. Remove only temporary artifacts you created. If a path contains an intentional change, restore it to the baseline behavior without using git reset, git checkout, or git restore.",
|
|
"Do not modify any other out-of-scope path and do not broaden the plan scope.",
|
|
"Use an OS temporary directory for further investigation. For SQLite files, clean the database plus -wal, -shm, and -journal sidecars.",
|
|
"Before finishing, run git status --porcelain --untracked-files=all and confirm no path outside the allowed scope remains.",
|
|
);
|
|
return lines.join("\n");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// workflow retry-review
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface RetryReviewWorkflowOpts {
|
|
runId: string;
|
|
}
|
|
|
|
export function retryReviewWorkflow(opts: RetryReviewWorkflowOpts): void {
|
|
const { state, dir } = requireState(opts.runId);
|
|
if (state.status !== "awaiting_scope_resolution") {
|
|
throw new WorkflowEngineError(
|
|
`Run '${opts.runId}' cannot retry review from status '${state.status}'. Only scope-resolution runs are eligible.`,
|
|
4,
|
|
);
|
|
}
|
|
|
|
const vcsKind = state.vcsKind || "git";
|
|
const vcsProvider = createVcsProvider(vcsKind);
|
|
const baseline = state.vcsBaseline || (state.baselineHead ? { kind: "git" as const, head: state.baselineHead } : undefined);
|
|
if (!baseline) {
|
|
throw new WorkflowEngineError("No baseline found in state.");
|
|
}
|
|
|
|
try {
|
|
vcsProvider.validateBaseline(state.repoRoot, baseline, dir);
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
throw new WorkflowEngineError(err.message, 4);
|
|
}
|
|
throw err;
|
|
}
|
|
|
|
let outOfScope: string[];
|
|
try {
|
|
outOfScope = vcsProvider.checkFilesInScope(state.repoRoot, baseline, state.plan.scope, dir);
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
throw new WorkflowEngineError(err.message, 4);
|
|
}
|
|
throw err;
|
|
}
|
|
|
|
if (outOfScope.length > 0) {
|
|
throw new WorkflowEngineError(
|
|
`Cannot retry review while out-of-scope files remain: ${outOfScope.sort().join(", ")}.`,
|
|
4,
|
|
);
|
|
}
|
|
|
|
const cycle = state.cycles.find((record) => record.cycleIndex === state.currentCycle);
|
|
if (cycle) {
|
|
delete cycle.scopeViolations;
|
|
delete cycle.hostReviewFile;
|
|
delete cycle.hostReviewReadyAt;
|
|
}
|
|
state.status = "awaiting_review";
|
|
state.enginePid = undefined;
|
|
delete state.stopReason;
|
|
delete state.stopDescription;
|
|
state.updatedAt = nowIso();
|
|
writeState(dir, state);
|
|
appendEvent(dir, {
|
|
kind: "review_retried",
|
|
runId: state.runId,
|
|
timestamp: state.updatedAt,
|
|
cycleIndex: state.currentCycle,
|
|
data: {
|
|
previousStatus: "awaiting_scope_resolution",
|
|
},
|
|
});
|
|
|
|
process.stdout.write(`workflow review retry enabled: ${state.runId}\n`);
|
|
process.stdout.write(`status: awaiting_review\n`);
|
|
process.stdout.write(`next: agent-workflow review --run ${state.runId}\n`);
|
|
emitWorkflowMeta({ runId: state.runId, status: state.status, cycleIndex: state.currentCycle });
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// workflow retry-execute
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function isProcessAlive(pid: number): boolean {
|
|
try {
|
|
process.kill(pid, 0);
|
|
return true;
|
|
} catch (err) {
|
|
return (err as NodeJS.ErrnoException).code === "EPERM";
|
|
}
|
|
}
|
|
|
|
export interface RetryExecuteWorkflowOpts {
|
|
runId: string;
|
|
sessionId?: string;
|
|
}
|
|
|
|
export async function retryExecuteWorkflow(opts: RetryExecuteWorkflowOpts): Promise<void> {
|
|
const { state, dir } = requireState(opts.runId);
|
|
if (opts.sessionId && state.executor !== "claude") {
|
|
throw new WorkflowEngineError("--session can only be used with a Claude executor run.", 4);
|
|
}
|
|
const staleExecution = state.status === "executing"
|
|
&& state.enginePid !== undefined
|
|
&& !isProcessAlive(state.enginePid);
|
|
const retryableFailure = staleExecution || (state.status === "blocked"
|
|
&& (state.stopDescription?.startsWith("Executor exited with failure:") === true
|
|
|| state.stopDescription?.startsWith("Executor failed to resume:") === true
|
|
|| state.stopDescription?.startsWith("Executor resume failed:") === true
|
|
|| state.stopDescription === "No session handle available for resume."));
|
|
if (!retryableFailure) {
|
|
throw new WorkflowEngineError(
|
|
`Run '${opts.runId}' cannot retry execution from status '${state.status}'. Only failed executor runs with a resumable session are eligible.`,
|
|
4,
|
|
);
|
|
}
|
|
|
|
const cycle = state.cycles.find((record) => record.cycleIndex === state.currentCycle);
|
|
const previousCycle = state.cycles.find((record) => record.cycleIndex === state.currentCycle - 1);
|
|
if (!cycle || cycle.diffFile || cycle.hostReviewFile || cycle.decisionFile || cycle.scopeViolations) {
|
|
throw new WorkflowEngineError("Cannot retry execution: the failed cycle already contains review artifacts.", 4);
|
|
}
|
|
const recoveredReasonixSession = !state.sessionHandle && state.executor === "reasonix"
|
|
? findReasonixSessionForWorkflow(
|
|
state.repoRoot,
|
|
state.plan.title,
|
|
previousCycle?.startedAt ?? cycle.startedAt,
|
|
previousCycle?.completedAt,
|
|
)
|
|
: undefined;
|
|
if (!state.sessionHandle && !opts.sessionId && !recoveredReasonixSession) {
|
|
throw new WorkflowEngineError("Cannot retry execution: no executor session handle is available.", 4);
|
|
}
|
|
const recoveringInitialCycle = state.currentCycle === 1;
|
|
if (!recoveringInitialCycle && (!previousCycle?.decisionFile || previousCycle.decisionOutcome !== "fix")) {
|
|
throw new WorkflowEngineError("Cannot retry execution: the preceding fix decision is missing.", 4);
|
|
}
|
|
const resolvingScope = !recoveringInitialCycle && (previousCycle?.scopeViolations?.length ?? 0) > 0;
|
|
|
|
const vcsKind = state.vcsKind || "git";
|
|
const vcsProvider = createVcsProvider(vcsKind);
|
|
const baseline = state.vcsBaseline || (state.baselineHead ? { kind: "git" as const, head: state.baselineHead } : undefined);
|
|
if (!baseline) {
|
|
throw new WorkflowEngineError("No baseline found in state.", 4);
|
|
}
|
|
|
|
try {
|
|
vcsProvider.validateBaseline(state.repoRoot, baseline, dir);
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
throw new WorkflowEngineError(err.message, 4);
|
|
}
|
|
throw err;
|
|
}
|
|
|
|
let outOfScope: string[];
|
|
try {
|
|
outOfScope = vcsProvider.checkFilesInScope(state.repoRoot, baseline, state.plan.scope, dir);
|
|
} catch (err) {
|
|
if (err instanceof VcsError) {
|
|
throw new WorkflowEngineError(err.message, 4);
|
|
}
|
|
throw err;
|
|
}
|
|
const approvedScopePaths = new Set((previousCycle?.scopeViolations ?? []).map((violation) => violation.path));
|
|
const unexpectedOutOfScope = resolvingScope
|
|
? outOfScope.filter((filePath) => !approvedScopePaths.has(filePath))
|
|
: outOfScope;
|
|
if (unexpectedOutOfScope.length > 0) {
|
|
throw new WorkflowEngineError(
|
|
`Cannot retry execution while unexpected out-of-scope files remain: ${unexpectedOutOfScope.sort().join(", ")}.`,
|
|
4,
|
|
);
|
|
}
|
|
|
|
if (opts.sessionId) {
|
|
try {
|
|
validateClaudeSessionForWorkflow(opts.sessionId, state.repoRoot, state.plan.title);
|
|
} catch (err) {
|
|
throw new WorkflowEngineError(`Cannot rebind Claude session: ${(err as Error).message}`, 4);
|
|
}
|
|
}
|
|
|
|
let acceptedFindings: string;
|
|
if (recoveringInitialCycle) {
|
|
acceptedFindings = "# Execution Recovery\n\nContinue the original implementation from the current working tree.";
|
|
} else {
|
|
const decisionPath = path.join(dir, previousCycle!.decisionFile!);
|
|
let decision: HostDecisionV1;
|
|
try {
|
|
const raw = JSON.parse(fs.readFileSync(decisionPath, "utf8")) as unknown;
|
|
validateHostDecision(raw);
|
|
decision = raw;
|
|
} catch (err) {
|
|
throw new WorkflowEngineError(`Cannot retry execution: failed to read preceding decision: ${(err as Error).message}`, 4);
|
|
}
|
|
acceptedFindings = resolvingScope
|
|
? buildScopeResolutionText(previousCycle!.scopeViolations ?? [])
|
|
: buildAcceptedFindingsText(decision, state, dir, previousCycle!.cycleIndex);
|
|
}
|
|
|
|
try {
|
|
acquireLock(state.repoRoot, state.runId);
|
|
} catch (err) {
|
|
if (err instanceof WorkflowLockError) throw new WorkflowEngineError(err.message, 4);
|
|
throw err;
|
|
}
|
|
|
|
const previousSessionId = state.sessionHandle?.kind === "claude"
|
|
? state.sessionHandle.sessionId
|
|
: undefined;
|
|
if (opts.sessionId) {
|
|
state.sessionHandle = { kind: "claude", sessionId: opts.sessionId };
|
|
} else if (recoveredReasonixSession) {
|
|
state.sessionHandle = { kind: "reasonix", sessionFilePath: recoveredReasonixSession };
|
|
}
|
|
|
|
state.status = "executing";
|
|
delete state.stopReason;
|
|
delete state.stopDescription;
|
|
state.enginePid = process.pid;
|
|
state.updatedAt = nowIso();
|
|
writeState(dir, state);
|
|
|
|
if (opts.sessionId) {
|
|
appendEvent(dir, {
|
|
kind: "executor_session_rebound",
|
|
runId: state.runId,
|
|
timestamp: state.updatedAt,
|
|
cycleIndex: state.currentCycle,
|
|
data: { previousSessionId, sessionId: opts.sessionId },
|
|
});
|
|
} else if (recoveredReasonixSession) {
|
|
appendEvent(dir, {
|
|
kind: "executor_session_rebound",
|
|
runId: state.runId,
|
|
timestamp: state.updatedAt,
|
|
cycleIndex: state.currentCycle,
|
|
data: { sessionFilePath: recoveredReasonixSession, recovery: "reasonix_project_session" },
|
|
});
|
|
}
|
|
|
|
// Determine recovery mode: stale_execution trumps initial_continuation
|
|
let recoveryMode: string;
|
|
if (staleExecution) {
|
|
recoveryMode = "stale_execution";
|
|
} else if (recoveringInitialCycle) {
|
|
recoveryMode = "initial_continuation";
|
|
} else {
|
|
recoveryMode = "fix_resume";
|
|
}
|
|
|
|
appendEvent(dir, {
|
|
kind: "executor_retried",
|
|
runId: state.runId,
|
|
timestamp: state.updatedAt,
|
|
cycleIndex: state.currentCycle,
|
|
data: {
|
|
executor: state.executor,
|
|
recoveryMode,
|
|
},
|
|
});
|
|
|
|
const sessionMode = opts.sessionId || recoveredReasonixSession ? "recovered" : "existing";
|
|
process.stdout.write(`retrying executor cycle ${state.currentCycle} with the ${sessionMode} ${state.executor} session\n`);
|
|
await resumeExecutorCycle(state, dir, cycle, acceptedFindings);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// workflow extend
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface ExtendWorkflowOpts {
|
|
runId: string;
|
|
additionalCycles: number;
|
|
}
|
|
|
|
export async function extendWorkflow(opts: ExtendWorkflowOpts): Promise<void> {
|
|
// 1. Validate positive integer
|
|
if (!Number.isInteger(opts.additionalCycles) || opts.additionalCycles < 1) {
|
|
throw new WorkflowEngineError(
|
|
`--additional-cycles must be a positive integer, got: ${opts.additionalCycles}`,
|
|
4,
|
|
);
|
|
}
|
|
|
|
const { state, dir } = requireState(opts.runId);
|
|
|
|
// 2. Must be in budget_exhausted terminal state
|
|
if (state.status !== "budget_exhausted") {
|
|
throw new WorkflowEngineError(
|
|
`Run '${opts.runId}' is in status '${state.status}', not budget_exhausted. ` +
|
|
`Only budget_exhausted runs can be extended.`,
|
|
4,
|
|
);
|
|
}
|
|
|
|
// 3. Must have a valid session handle
|
|
if (!state.sessionHandle) {
|
|
throw new WorkflowEngineError(
|
|
`Run '${opts.runId}' has no session handle. Cannot extend without a resumable session.`,
|
|
4,
|
|
);
|
|
}
|
|
|
|
// 4. The last persisted decision must be 'fix'
|
|
const lastCycle = state.cycles[state.cycles.length - 1];
|
|
if (!lastCycle?.decisionFile || lastCycle.decisionOutcome !== "fix") {
|
|
throw new WorkflowEngineError(
|
|
`Run '${opts.runId}' cannot be extended: the last cycle decision must be 'fix'.`,
|
|
4,
|
|
);
|
|
}
|
|
|
|
// 5. Reacquire the repository lock BEFORE validation
|
|
try {
|
|
acquireLock(state.repoRoot, state.runId);
|
|
} catch (err) {
|
|
if (err instanceof WorkflowLockError) {
|
|
throw new WorkflowEngineError(err.message, 4);
|
|
}
|
|
throw err;
|
|
}
|
|
|
|
// Helper to release lock and throw
|
|
const failAndRelease = (message: string, exitCode = 4): never => {
|
|
releaseLock(state.repoRoot, state.runId);
|
|
throw new WorkflowEngineError(message, exitCode);
|
|
};
|
|
|
|
// 6. Validate baseline and scope under the lock
|
|
const vcsKind = state.vcsKind || "git";
|
|
const vcsProvider = createVcsProvider(vcsKind);
|
|
const baseline = state.vcsBaseline || (state.baselineHead ? { kind: "git" as const, head: state.baselineHead } : undefined);
|
|
if (!baseline) {
|
|
failAndRelease("No baseline found in state.");
|
|
return; // Never reached but helps TypeScript
|
|
}
|
|
|
|
try {
|
|
vcsProvider.validateBaseline(state.repoRoot, baseline, dir);
|
|
} catch (err) {
|
|
const message = err instanceof VcsError ? err.message : (err as Error).message;
|
|
failAndRelease(`Cannot extend: baseline validation failed: ${message}`);
|
|
return;
|
|
}
|
|
|
|
let outOfScope: string[] = [];
|
|
try {
|
|
outOfScope = vcsProvider.checkFilesInScope(state.repoRoot, baseline, state.plan.scope, dir);
|
|
} catch (err) {
|
|
const message = err instanceof VcsError ? err.message : (err as Error).message;
|
|
failAndRelease(`Cannot extend: scope check failed: ${message}`);
|
|
return;
|
|
}
|
|
|
|
if (outOfScope.length > 0) {
|
|
failAndRelease(`Cannot extend: out-of-scope files detected: ${outOfScope.join(", ")}`);
|
|
return;
|
|
}
|
|
|
|
// 7. Check convergence using existing history
|
|
const convergenceError = checkWorkflowConvergence(state, dir);
|
|
if (convergenceError) {
|
|
failAndRelease(`Cannot extend: ${convergenceError}`);
|
|
return;
|
|
}
|
|
|
|
// 8. Load and validate the last decision BEFORE mutating state
|
|
const decisionPath = path.join(dir, lastCycle.decisionFile!);
|
|
let decision: HostDecisionV1;
|
|
let acceptedFindings: string = "";
|
|
try {
|
|
const raw = JSON.parse(fs.readFileSync(decisionPath, "utf8")) as unknown;
|
|
validateHostDecision(raw);
|
|
decision = raw;
|
|
acceptedFindings = buildAcceptedFindingsText(decision, state, dir, lastCycle.cycleIndex);
|
|
} catch (err) {
|
|
failAndRelease(`Cannot extend: failed to read last decision: ${(err as Error).message}`);
|
|
return;
|
|
}
|
|
|
|
// 9. Now atomically mutate state: budget, cycle, status, history
|
|
const oldMaxCycles = state.maxCycles;
|
|
const newMaxCycles = oldMaxCycles + opts.additionalCycles;
|
|
const nextCycle = state.currentCycle + 1;
|
|
const extendedAt = nowIso();
|
|
|
|
state.maxCycles = newMaxCycles;
|
|
state.enginePid = process.pid;
|
|
state.updatedAt = extendedAt;
|
|
state.status = "executing";
|
|
delete state.stopReason;
|
|
delete state.stopDescription;
|
|
|
|
const nextCycleRecord: CycleRecord = { cycleIndex: nextCycle, startedAt: extendedAt };
|
|
state.cycles.push(nextCycleRecord);
|
|
state.currentCycle = nextCycle;
|
|
|
|
// Record extension history in state
|
|
if (!state.extensionHistory) {
|
|
state.extensionHistory = [];
|
|
}
|
|
state.extensionHistory.push({
|
|
extendedAt,
|
|
oldMaxCycles,
|
|
newMaxCycles,
|
|
additionalCycles: opts.additionalCycles,
|
|
resumedCycle: nextCycle,
|
|
});
|
|
|
|
// 10. Persist state and events
|
|
writeState(dir, state);
|
|
appendEvent(dir, {
|
|
kind: "budget_extended",
|
|
runId: state.runId,
|
|
timestamp: extendedAt,
|
|
data: {
|
|
additionalCycles: opts.additionalCycles,
|
|
oldMaxCycles,
|
|
newMaxCycles,
|
|
nextCycle,
|
|
},
|
|
});
|
|
|
|
appendEvent(dir, { kind: "cycle_started", runId: state.runId, timestamp: nowIso(), cycleIndex: nextCycle });
|
|
|
|
process.stdout.write(`workflow extended: ${state.runId}\n`);
|
|
process.stdout.write(` additional cycles: ${opts.additionalCycles}\n`);
|
|
process.stdout.write(` old max cycles: ${oldMaxCycles}\n`);
|
|
process.stdout.write(` new max cycles: ${newMaxCycles}\n`);
|
|
process.stdout.write(` resuming cycle: ${nextCycle}\n\n`);
|
|
|
|
// 11. Resume the executor session immediately (lock released by resumeExecutorCycle on completion)
|
|
try {
|
|
await resumeExecutorCycle(state, dir, nextCycleRecord, acceptedFindings);
|
|
} catch (err) {
|
|
// resumeExecutorCycle already handles errors and releases lock
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// workflow status
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface StatusWorkflowOpts {
|
|
runId: string;
|
|
json?: boolean;
|
|
}
|
|
|
|
export function statusWorkflow(opts: StatusWorkflowOpts): void {
|
|
const { state, dir } = requireState(opts.runId);
|
|
const cycle = state.cycles.find((c) => c.cycleIndex === state.currentCycle);
|
|
|
|
// Read live progress sidecar for real-time observability
|
|
const sidecar = readProgressSidecar(dir);
|
|
const liveLogFile = cycle?.executorLogFile
|
|
? path.join(dir, cycle.executorLogFile)
|
|
: sidecar?.logFilePath
|
|
? path.join(dir, sidecar.logFilePath)
|
|
: undefined;
|
|
let executionElapsed: string | undefined;
|
|
if (sidecar?.executionDurationMs !== undefined) {
|
|
const ms = sidecar.executionDurationMs;
|
|
const secs = Math.floor(ms / 1000);
|
|
if (secs < 60) executionElapsed = `${secs}s`;
|
|
else executionElapsed = `${Math.floor(secs / 60)}m ${secs % 60}s`;
|
|
} else if (sidecar?.startedAt) {
|
|
const ms = Date.now() - new Date(sidecar.startedAt).getTime();
|
|
const secs = Math.floor(ms / 1000);
|
|
if (secs < 60) executionElapsed = `${secs}s`;
|
|
else executionElapsed = `${Math.floor(secs / 60)}m ${secs % 60}s`;
|
|
}
|
|
|
|
const output: WorkflowStatusOutput = {
|
|
runId: state.runId,
|
|
status: state.status,
|
|
executor: state.executor,
|
|
currentCycle: state.currentCycle,
|
|
maxCycles: state.maxCycles,
|
|
remainingCycles: Math.max(0, state.maxCycles - state.currentCycle),
|
|
sessionHandleKind: state.sessionHandle?.kind,
|
|
taskId: state.taskId,
|
|
taskSessionReused: state.taskSessionReused,
|
|
degradedResume: state.agyDegraded,
|
|
runDir: dir,
|
|
diffFile: cycle?.diffFile ? path.join(dir, cycle.diffFile) : undefined,
|
|
executorLogFile: cycle?.executorLogFile ? path.join(dir, cycle.executorLogFile) : undefined,
|
|
hostReviewFile: cycle?.hostReviewFile ? path.join(dir, cycle.hostReviewFile) : undefined,
|
|
scopeViolations: state.status === "awaiting_scope_resolution" ? cycle?.scopeViolations : undefined,
|
|
stopReason: state.stopReason,
|
|
stopDescription: state.stopDescription,
|
|
...(state.baselineHead ? { baselineHead: state.baselineHead } : {}),
|
|
...(state.vcsBaseline ? { vcsBaseline: state.vcsBaseline } : {}),
|
|
...(state.vcsKind ? { vcsKind: state.vcsKind } : {}),
|
|
repoRoot: state.repoRoot,
|
|
createdAt: state.createdAt,
|
|
updatedAt: state.updatedAt,
|
|
liveLogFile,
|
|
executionElapsed,
|
|
lastActivity: sidecar?.lastActivityAt,
|
|
logBytes: sidecar?.logBytes,
|
|
executorPid: sidecar?.pid,
|
|
activitySummary: sidecar?.activitySummary,
|
|
};
|
|
|
|
if (opts.json) {
|
|
process.stdout.write(JSON.stringify(output, null, 2) + "\n");
|
|
return;
|
|
}
|
|
|
|
const lw = 20;
|
|
const p = (label: string, value: string) =>
|
|
` ${(label + ":").padEnd(lw)} ${value}`;
|
|
|
|
const lines = [
|
|
"── agent-workflow ─────────────────────",
|
|
p("Run ID", state.runId),
|
|
p("Status", state.status.toUpperCase()),
|
|
p("Executor", state.executor),
|
|
p("Cycle", `${state.currentCycle} / ${state.maxCycles} (${output.remainingCycles} remaining)`),
|
|
];
|
|
|
|
if (state.sessionHandle) {
|
|
const kind = state.sessionHandle.kind;
|
|
const extra = kind === "agy" && state.agyDegraded ? " [degraded-continue]" : "";
|
|
lines.push(p("Session handle", kind + extra));
|
|
}
|
|
if (state.taskId) {
|
|
lines.push(p("Task", `${state.taskId}${state.taskSessionReused ? " [session reused]" : ""}`));
|
|
}
|
|
|
|
// Live execution fields
|
|
if (output.executorPid) lines.push(p("Executor PID", String(output.executorPid)));
|
|
if (output.executionElapsed) lines.push(p("Elapsed", output.executionElapsed));
|
|
if (output.activitySummary) lines.push(p("Activity", output.activitySummary));
|
|
if (output.lastActivity) lines.push(p("Last activity", output.lastActivity));
|
|
if (output.logBytes !== undefined) lines.push(p("Log bytes", `${output.logBytes}`));
|
|
if (output.liveLogFile) lines.push(p("Live log", output.liveLogFile));
|
|
|
|
if (cycle?.hostReviewFile) lines.push(p("Host review", path.join(dir, cycle.hostReviewFile)));
|
|
if (state.status === "awaiting_scope_resolution" && cycle?.scopeViolations?.length) {
|
|
lines.push(p("Scope violations", String(cycle.scopeViolations.length)));
|
|
for (const violation of cycle.scopeViolations) {
|
|
lines.push(` ${violation.id}: ${violation.path}`);
|
|
}
|
|
}
|
|
if (state.stopReason) lines.push(p("Stop reason", state.stopReason));
|
|
if (state.stopDescription) lines.push(p("Stop detail", state.stopDescription.slice(0, 80)));
|
|
lines.push(p("Repo root", state.repoRoot));
|
|
const baselineHead = extractBaselineHead(state);
|
|
if (baselineHead) {
|
|
lines.push(p("Baseline HEAD", baselineHead.slice(0, 8)));
|
|
} else if (state.vcsBaseline?.kind === "svn") {
|
|
lines.push(p("Baseline SVN", `r${state.vcsBaseline.revision}`));
|
|
}
|
|
if (state.vcsKind) {
|
|
lines.push(p("VCS", state.vcsKind));
|
|
}
|
|
lines.push(p("Created", state.createdAt));
|
|
lines.push(p("Updated", state.updatedAt));
|
|
lines.push("─".repeat(46));
|
|
|
|
process.stdout.write(lines.join("\n") + "\n");
|
|
emitWorkflowMeta(output as unknown as Record<string, unknown>);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// workflow abort
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface AbortWorkflowOpts {
|
|
runId: string;
|
|
reason: string;
|
|
}
|
|
|
|
export function abortWorkflow(opts: AbortWorkflowOpts): void {
|
|
const { state, dir } = requireState(opts.runId);
|
|
|
|
if (["completed", "aborted", "needs_human", "blocked", "budget_exhausted"].includes(state.status)) {
|
|
process.stdout.write(`Run '${opts.runId}' is already in terminal state: ${state.status}\n`);
|
|
return;
|
|
}
|
|
|
|
const pid = state.enginePid;
|
|
transitionToTerminal(state, dir, "aborted", opts.reason, { abortedAt: nowIso() });
|
|
|
|
if (pid) {
|
|
try {
|
|
process.kill(pid, "SIGTERM");
|
|
} catch {
|
|
// Ignore if process is already dead
|
|
}
|
|
}
|
|
|
|
process.stdout.write(`workflow aborted: ${opts.runId}\nreason: ${opts.reason}\n`);
|
|
emitWorkflowMeta({ runId: opts.runId, status: "aborted", reason: opts.reason });
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// workflow logs
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface LogsWorkflowOpts {
|
|
runId: string;
|
|
follow: boolean;
|
|
tail: number;
|
|
repoRoot?: string;
|
|
}
|
|
|
|
export async function logsWorkflow(opts: LogsWorkflowOpts): Promise<void> {
|
|
const { state, dir } = requireState(opts.runId);
|
|
|
|
// Resolve current log path: sidecar is freshest during execution
|
|
const sidecar = readProgressSidecar(dir);
|
|
let logPath: string | undefined;
|
|
if (sidecar?.logFilePath && fs.existsSync(path.join(dir, sidecar.logFilePath))) {
|
|
logPath = path.join(dir, sidecar.logFilePath);
|
|
} else {
|
|
const cycle = state.cycles.find((c) => c.cycleIndex === state.currentCycle);
|
|
if (cycle?.executorLogFile) {
|
|
logPath = path.join(dir, cycle.executorLogFile);
|
|
}
|
|
}
|
|
|
|
// If no log file and state is executing, wait briefly for it to appear
|
|
const isExecuting = state.status === "executing";
|
|
if (!logPath && isExecuting) {
|
|
for (let i = 0; i < 30; i++) {
|
|
await sleepMs(500);
|
|
const sc = readProgressSidecar(dir);
|
|
if (sc?.logFilePath && fs.existsSync(path.join(dir, sc.logFilePath))) {
|
|
logPath = path.join(dir, sc.logFilePath);
|
|
break;
|
|
}
|
|
const updatedState = readState(dir);
|
|
if (updatedState) {
|
|
const uc = updatedState.cycles.find((c) => c.cycleIndex === updatedState.currentCycle);
|
|
if (uc?.executorLogFile) {
|
|
logPath = path.join(dir, uc.executorLogFile);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!logPath || !fs.existsSync(logPath)) {
|
|
process.stdout.write(`No log file found for run '${opts.runId}'.\n`);
|
|
return;
|
|
}
|
|
|
|
let followLockPath: string | undefined;
|
|
if (opts.follow) {
|
|
const codexThreadId = process.env.CODEX_THREAD_ID?.trim();
|
|
if (codexThreadId) {
|
|
try {
|
|
followLockPath = acquireLogsFollowLock(dir, codexThreadId);
|
|
} catch (err) {
|
|
if (err instanceof LogsFollowLockError) {
|
|
throw new WorkflowEngineError(err.message, 4);
|
|
}
|
|
throw err;
|
|
}
|
|
}
|
|
}
|
|
|
|
try {
|
|
// Tail the last N lines
|
|
const lines = readTailLines(logPath, opts.tail);
|
|
for (const line of lines) {
|
|
process.stdout.write(line + "\n");
|
|
}
|
|
|
|
if (!opts.follow) return;
|
|
|
|
// Follow mode: watch for new data and retry rotations
|
|
let currentPath = logPath;
|
|
let lastByteOffset = fs.statSync(currentPath).size;
|
|
|
|
// States that indicate execution has ceased (no more output expected)
|
|
const terminalStates = new Set([
|
|
"completed", "needs_human", "blocked", "budget_exhausted", "aborted",
|
|
"awaiting_review", "awaiting_host", "awaiting_scope_resolution",
|
|
]);
|
|
|
|
while (true) {
|
|
const curState = readState(dir);
|
|
if (!curState) break;
|
|
|
|
// Exit if the workflow has stopped executing
|
|
if (terminalStates.has(curState.status)) {
|
|
drainBytes(currentPath, lastByteOffset);
|
|
return;
|
|
}
|
|
|
|
// Check for log rotation to a newer attempt
|
|
const sc = readProgressSidecar(dir);
|
|
let candidatePath: string | undefined;
|
|
if (sc?.logFilePath && fs.existsSync(path.join(dir, sc.logFilePath))) {
|
|
candidatePath = path.join(dir, sc.logFilePath);
|
|
} else {
|
|
const curCycle = curState.cycles.find((c) => c.cycleIndex === curState.currentCycle);
|
|
if (curCycle?.executorLogFile) {
|
|
candidatePath = path.join(dir, curCycle.executorLogFile);
|
|
}
|
|
}
|
|
|
|
if (candidatePath && candidatePath !== currentPath) {
|
|
drainBytes(currentPath, lastByteOffset);
|
|
currentPath = candidatePath;
|
|
lastByteOffset = fs.statSync(currentPath).size;
|
|
process.stdout.write(`\n--- log rotated to ${path.basename(currentPath)} ---\n`);
|
|
const headLines = readTailLines(currentPath, Math.min(opts.tail, 10));
|
|
for (const line of headLines) {
|
|
process.stdout.write(line + "\n");
|
|
}
|
|
continue;
|
|
}
|
|
|
|
// Drain new bytes since lastByteOffset using byte-accurate reading
|
|
try {
|
|
const stats = fs.statSync(currentPath);
|
|
if (stats.size > lastByteOffset) {
|
|
drainBytes(currentPath, lastByteOffset);
|
|
lastByteOffset = stats.size;
|
|
}
|
|
} catch {
|
|
// File may have been deleted or rotated
|
|
}
|
|
|
|
await sleepMs(500);
|
|
}
|
|
} finally {
|
|
if (followLockPath) releaseLogsFollowLock(followLockPath);
|
|
}
|
|
}
|
|
|
|
function sleepMs(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function readTailLines(filePath: string, n: number): string[] {
|
|
try {
|
|
const content = fs.readFileSync(filePath, "utf8");
|
|
const lines = content.split("\n");
|
|
return lines.slice(Math.max(0, lines.length - n));
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Drain all bytes starting at fromByteOffset using proper byte-level I/O.
|
|
*/
|
|
function drainBytes(filePath: string, fromByteOffset: number): void {
|
|
try {
|
|
const fd = fs.openSync(filePath, "r");
|
|
try {
|
|
const stats = fs.fstatSync(fd);
|
|
const remaining = stats.size - fromByteOffset;
|
|
if (remaining > 0) {
|
|
const buf = Buffer.alloc(remaining);
|
|
fs.readSync(fd, buf, 0, remaining, fromByteOffset);
|
|
process.stdout.write(buf.toString("utf8"));
|
|
}
|
|
} finally {
|
|
fs.closeSync(fd);
|
|
}
|
|
} catch {
|
|
// ignore
|
|
}
|
|
}
|