| import type { |
| AgentRunEvent, |
| AgentRunHeader, |
| AgentRunStore, |
| EmittedAgentRunEvent, |
| RuntimeEvent, |
| RuntimeEventStore, |
| ToolBoundaryProtocol, |
| } from '@maka/core'; |
| import { DurableStoreWriteError, isSessionInlineRun, isTerminalRuntimeEvent } from '@maka/core'; |
| import { ToolLedgerRejectionError } from '@maka/core'; |
| import { Buffer } from 'node:buffer'; |
| import { isDeepStrictEqual } from 'node:util'; |
| import { redactSecrets } from '@maka/core'; |
| import { MODEL_CALL_ATTEMPT_EVENT_TYPE, type ModelCallAttempt } from '@maka/core'; |
| import type { |
| SessionBlockedReason, |
| SessionHeader, |
| SessionStatus, |
| StoredMessage, |
| SystemNoteMessage, |
| TurnRecord, |
| UserMessage, |
| } from '@maka/core'; |
| import type { UserMessageInput } from '@maka/core'; |
| import { resolveEffectiveOrchestration, type EffectiveOrchestration } from '@maka/core'; |
| import type { SessionEvent } from '@maka/core'; |
| import type { AgentBackend, BackendSendInput } from '@maka/core'; |
| import type { RunTraceEvent } from './run-trace.js'; |
| import type { StopSessionInput } from './session-manager.js'; |
| import type { ActiveFullCompactBlock } from './active-full-compact.js'; |
| import type { SemanticCompactBlock } from './semantic-compact.js'; |
| import type { HistoryCompactCheckpoint } from './history-compact-checkpoint.js'; |
| import { projectRuntimeEventsToStoredMessages } from './runtime-event-read-model.js'; |
| import { |
| buildPriorRuntimeContext as buildPriorRuntimeContextProjection, |
| type PriorRuntimeContext, |
| } from './prior-run-context.js'; |
| import { |
| buildStatusPatch, |
| isTerminalRunStatus, |
| normalizeStopSessionSource, |
| statusFromEvent, |
| turnStatusFromEvent, |
| } from './session-projection-helpers.js'; |
| import { |
| buildSyntheticTerminalRuntimeEvent, |
| commitOrCreateTerminalRunFact, |
| } from './terminal-run-commit.js'; |
| import { AiSdkFlow } from './ai-sdk-flow.js'; |
| import type { InvocationContext } from './invocation-context.js'; |
| import { buildInitialUserRuntimeEvent } from './runtime-runner.js'; |
| import type { RuntimeContinuation } from './runtime-resume.js'; |
| import { |
| createRuntimeContinuationStartAdmissionProof, |
| type RuntimeContinuationStartAdmissionProof, |
| } from './runtime-continuation-admission.js'; |
| import { DEFAULT_TOOL_MODE, isToolMode, type ToolMode } from '@maka/core'; |
| import type { |
| ProviderRequestAttemptRecord, |
| ProviderRequestCaptureLedgerRecord, |
| } from './provider-request-telemetry.js'; |
| import { materializeRuntimeEventTranscriptProjection } from './runtime-ledger-repair.js'; |
| |
| export interface AgentRunActiveSession { |
| sessionId: string; |
| backend: AgentBackend; |
| cachedHeader: SessionHeader; |
| activeRuns: Map<string, AgentRun>; |
| turnToRunId: Map<string, string>; |
| } |
| |
| export interface AgentRunHooks { |
| reserveRun( |
| sessionId: string, |
| header: SessionHeader, |
| run: AgentRun, |
| ): Promise<AgentRunActiveSession>; |
| unregisterRun(active: AgentRunActiveSession, run: AgentRun): void | Promise<void>; |
| updateHeader(sessionId: string, patch: Partial<SessionHeader>): Promise<SessionHeader>; |
| updateStatus( |
| sessionId: string, |
| status: SessionStatus, |
| blockedReason?: SessionBlockedReason, |
| ts?: number, |
| ): Promise<void>; |
| appendTurnState( |
| sessionId: string, |
| turnId: string, |
| status: TurnRecord['status'], |
| lineage?: AgentRunLineage, |
| options?: { ts?: number; errorClass?: string; abortSource?: string }, |
| ): Promise<void>; |
| } |
| |
| export type AgentRunLineage = Partial< |
| Pick< |
| UserMessageInput, |
| | 'parentRunId' |
| | 'resumedFromRunId' |
| | 'retriedFromRunId' |
| | 'parentTurnId' |
| | 'retriedFromTurnId' |
| | 'regeneratedFromTurnId' |
| | 'branchOfTurnId' |
| | 'parentSessionId' |
| > |
| >; |
| |
| export type AgentRunDurability = 'best_effort' | 'required'; |
| |
| export interface AgentRunInput { |
| sessionId: string; |
| header: SessionHeader; |
| userInput: UserMessageInput; |
| rootExecutionKind?: AgentRunHeader['rootExecutionKind']; |
| runId?: string; |
| userMessageId?: string; |
| durability?: AgentRunDurability; |
| store: AgentRunSessionStore; |
| runStore?: AgentRunStore; |
| runtimeEventStore?: RuntimeEventStore; |
| repairRunRuntimeLedger?: (sessionId: string, runId: string) => Promise<boolean>; |
| newId: () => string; |
| now: () => number; |
| workspaceIdentity?: string; |
| continuationFailpoint?: (point: RuntimeContinuationFailpoint) => Promise<void>; |
| /** Exact target header already committed inside the durable continuation claim. */ |
| claimedRunHeader?: AgentRunHeader; |
| /** Commits the claimed continuation provider-call T1 after Run creation. */ |
| commitContinuationStart?: (startedAt: number) => Promise<{ startEventId: string; created: true }>; |
| hooks: AgentRunHooks; |
| recordSessionMessages?: boolean; |
| invocationId?: string; |
| /** Pre-resolved snapshot used by continuations; normal turns derive it from header + input. */ |
| effectiveOrchestration?: EffectiveOrchestration; |
| /** Pre-resolved tool protocol used by continuations. */ |
| effectiveToolMode?: ToolMode; |
| /** Set only when this run's backend tool path is guarded by canonical T1. */ |
| toolBoundaryProtocol?: ToolBoundaryProtocol; |
| } |
| |
| export interface AgentRunSessionStore { |
| appendMessage(sessionId: string, message: StoredMessage): Promise<void>; |
| readMessages(sessionId: string): Promise<StoredMessage[]>; |
| } |
| |
| export type RuntimeContinuationFailpoint = |
| | 'after_continuation_claim_committed' |
| | 'after_run_created' |
| | 'after_continuation_start_committed' |
| | 'after_terminal_event_committed' |
| | 'after_terminal_header_committed'; |
| |
| export class ContinuationStartCommitError extends Error { |
| readonly name = 'ContinuationStartCommitError'; |
| |
| constructor(readonly storeCause: unknown) { |
| super( |
| `Continuation start was not durably committed: ${ |
| storeCause instanceof Error ? storeCause.message : String(storeCause) |
| }`, |
| ); |
| } |
| } |
| |
| export interface AgentRunBeginResult { |
| backend: AgentBackend; |
| backendInput: BackendSendInput; |
| initialRuntimeEvent: RuntimeEvent; |
| } |
| |
| export interface AgentRunOperationBeginResult { |
| backend: AgentBackend; |
| runtimeContext: RuntimeEvent[]; |
| startedAt: number; |
| } |
| |
| export interface AgentRunContinuationBeginResult { |
| backend: AgentBackend; |
| startedAt: number; |
| continuationStartAdmission: RuntimeContinuationStartAdmissionProof; |
| } |
| |
| const RUNTIME_PARTIAL_FLUSH_INTERVAL_MS = 80; |
| const RUNTIME_PARTIAL_BATCH_MAX_BYTES = 8 * 1024; |
| |
| export class AgentRun { |
| readonly runId: string; |
| readonly invocationId: string; |
| readonly sessionId: string; |
| readonly turnId: string; |
| readonly toolBoundaryProtocol: ToolBoundaryProtocol | undefined; |
| readonly lineage: AgentRunLineage; |
| readonly effectiveOrchestration: EffectiveOrchestration; |
| readonly toolMode: ToolMode; |
| |
| private header: SessionHeader; |
| private active: AgentRunActiveSession | undefined; |
| private stopped = false; |
| private abortSource: string | undefined; |
| private traceQueue: Promise<void> = Promise.resolve(); |
| private runtimeEventQueue: Promise<void> = Promise.resolve(); |
| private runStoreAvailable = true; |
| private runtimeEventStoreAvailable = true; |
| private runtimeEventStoreFailure: unknown; |
| private runtimePartialStreamKey: string | undefined; |
| private runtimePartialBuffer: RuntimeEvent[] = []; |
| private runtimePartialBufferBytes = 0; |
| private runtimePartialFlushTimer: ReturnType<typeof setTimeout> | undefined; |
| private traceWriteError: string | undefined; |
| private failureClass: string | undefined; |
| private failureMessage: string | undefined; |
| private lastTs = 0; |
| private sawCompletion = false; |
| private finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined; |
| private turnFailed = false; |
| private finalized = false; |
| private terminalRunHeaderCommitted = false; |
| private continuationActive = false; |
| private terminalClaim: |
| | { |
| owner: 'event' | 'stop'; |
| event?: RuntimeEvent; |
| write?: Promise<void>; |
| stopCompleted?: boolean; |
| } |
| | undefined; |
| |
| constructor(private readonly input: AgentRunInput) { |
| if (input.runStore && !input.runtimeEventStore) { |
| throw new Error('RuntimeEventStore is required when AgentRunStore is configured'); |
| } |
| if (input.durability === 'required' && (!input.runStore || !input.runtimeEventStore)) { |
| throw new Error('Required AgentRun durability needs AgentRunStore and RuntimeEventStore'); |
| } |
| this.runId = input.runId ?? input.newId(); |
| this.invocationId = input.invocationId ?? this.runId; |
| this.sessionId = input.sessionId; |
| this.turnId = input.userInput.turnId; |
| this.toolBoundaryProtocol = input.toolBoundaryProtocol; |
| this.header = input.header; |
| this.effectiveOrchestration = |
| input.effectiveOrchestration ?? |
| resolveEffectiveOrchestration( |
| input.header.orchestrationMode, |
| input.userInput.turnOrchestration, |
| ); |
| const requestedToolMode = |
| input.effectiveToolMode ?? input.userInput.toolMode ?? DEFAULT_TOOL_MODE; |
| if (!isToolMode(requestedToolMode)) { |
| throw new Error(`Invalid tool mode: ${String(requestedToolMode)}`); |
| } |
| this.toolMode = requestedToolMode; |
| this.lineage = { |
| ...(input.userInput.parentRunId ? { parentRunId: input.userInput.parentRunId } : {}), |
| ...(input.userInput.resumedFromRunId |
| ? { resumedFromRunId: input.userInput.resumedFromRunId } |
| : {}), |
| ...(input.userInput.retriedFromRunId |
| ? { retriedFromRunId: input.userInput.retriedFromRunId } |
| : {}), |
| ...(input.userInput.parentTurnId ? { parentTurnId: input.userInput.parentTurnId } : {}), |
| ...(input.userInput.retriedFromTurnId |
| ? { retriedFromTurnId: input.userInput.retriedFromTurnId } |
| : {}), |
| ...(input.userInput.regeneratedFromTurnId |
| ? { regeneratedFromTurnId: input.userInput.regeneratedFromTurnId } |
| : {}), |
| ...(input.userInput.branchOfTurnId ? { branchOfTurnId: input.userInput.branchOfTurnId } : {}), |
| ...(input.userInput.parentSessionId |
| ? { parentSessionId: input.userInput.parentSessionId } |
| : {}), |
| }; |
| } |
| |
| stop(source: StopSessionInput['source'] | undefined): boolean { |
| if (this.terminalClaim) return false; |
| this.terminalClaim = { owner: 'stop' }; |
| this.stopped = true; |
| this.abortSource = normalizeStopSessionSource(source); |
| return true; |
| } |
| |
| isStopped(): boolean { |
| return this.stopped; |
| } |
| |
| isSessionInline(): boolean { |
| return isSessionInlineRun({ |
| ...(this.lineage.parentRunId ? { parentRunId: this.lineage.parentRunId } : {}), |
| ...(this.continuationActive ? { continuationSource: true } : {}), |
| }); |
| } |
| |
| hasPendingStop(): boolean { |
| return this.terminalClaim?.owner === 'stop' && this.terminalClaim.stopCompleted !== true; |
| } |
| |
| completeStop(): void { |
| if (this.terminalClaim?.owner === 'stop') this.terminalClaim.stopCompleted = true; |
| } |
| |
| /** |
| * Cash the terminal claim a stop already took. |
| * |
| * `stop()` claims the terminal outcome, but only `finalize()` — reached when |
| * the backend's event stream ends — has ever cashed it. A turn parked on an |
| * unanswered interaction never ends that stream, so the Session projection |
| * read as aborted while the run stayed non-terminal in the ledger forever, |
| * and every later turn dropped it from model context. Cash the claim at the |
| * stop instead. The claim keeps this idempotent: a stream that later |
| * produces its own terminal event finds the claim taken and writes nothing. |
| */ |
| async settleStopTerminal(): Promise<void> { |
| if (this.terminalClaim?.owner !== 'stop' || this.terminalRunHeaderCommitted) return; |
| // Nothing durable is configured, so there is no fact to land. Every other |
| // failure below is real and must reach the stop's caller: a stop that |
| // reports success while the run stays non-terminal is the silent loss this |
| // method exists to prevent. |
| const runStore = this.input.runStore; |
| if (!this.input.runtimeEventStore || !runStore) return; |
| await this.flushRuntimePartialBuffer(true); |
| // The claim only fences writers inside this Run. Another owner — a Host |
| // recovery, a resumed continuation — may have sealed the ledger already, |
| // and a sealed run rejects further appends. Nothing to land in that case: |
| // the fact this method exists to guarantee is already there. This claim's |
| // own reserved event is not foreign — that is a settlement being retried. |
| const claimedEventId = this.terminalClaim.event?.id; |
| const events = await this.loadTurnRuntimeEvents(); |
| if (events.some((event) => isTerminalRuntimeEvent(event) && event.id !== claimedEventId)) |
| return; |
| if (!this.runStoreAvailable) { |
| // The Run-store latch is best-effort history (one busy trace append |
| // sets it) and commitTerminalRun silently skips under it, which here |
| // would turn the stop into a reported success with no terminal fact: |
| // the silent variant of the loss this method exists to prevent. |
| // Probe like the RuntimeEvent read above; a store that answers lifts |
| // the latch, one that cannot fails the settlement loudly so the stop |
| // stays retryable. |
| try { |
| await runStore.readRun(this.sessionId, this.runId); |
| this.runStoreAvailable = true; |
| } catch (error) { |
| throw new Error('AgentRun store is unavailable for stop settlement', { cause: error }); |
| } |
| } |
| const ts = this.lastTs || this.input.now(); |
| const finalStatus = { status: 'aborted' as const }; |
| this.finalStatus ??= finalStatus; |
| this.reserveFinalizationTerminal(finalStatus, ts); |
| const runStoreAvailable = this.runStoreAvailable; |
| try { |
| await this.commitTerminalRun(finalStatus, ts); |
| } catch (error) { |
| // This commit runs ahead of the stream's own finalize, so one failure is |
| // not evidence the store is gone for the rest of the run. Undo the marks |
| // that would turn a single failed attempt into a permanently unwritable |
| // run, and let the caller decide whether to retry. The reserved event |
| // stays, so a retry lands the same terminal fact rather than a new one. |
| this.runStoreAvailable = runStoreAvailable; |
| if (this.terminalClaim) this.terminalClaim.write = undefined; |
| throw error; |
| } |
| } |
| |
| recordRunTrace(event: RunTraceEvent): void { |
| if (!this.input.runStore || !this.runStoreAvailable) return; |
| this.enqueueRunStore('append trace event', async () => { |
| await this.input.runStore?.appendEvent( |
| this.sessionId, |
| this.runId, |
| traceToRunEvent(event, this.runId), |
| ); |
| }); |
| } |
| |
| recordProviderRequestCapture(capture: ProviderRequestCaptureLedgerRecord): Promise<void> { |
| if (!this.input.runStore) return Promise.reject(new Error('AgentRun store is not configured')); |
| return this.enqueueRequiredRunStoreWrite('append provider request capture', async () => { |
| const { |
| schemaVersion, |
| serializedRequest: _serializedRequest, |
| ...data |
| } = capture as ProviderRequestCaptureLedgerRecord & { serializedRequest?: string }; |
| await this.input.runStore?.appendEvent( |
| this.sessionId, |
| this.runId, |
| { |
| type: 'provider_request_captured', |
| id: capture.captureId, |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: capture.turnId, |
| ts: this.input.now(), |
| data: { schemaVersion, ...data }, |
| }, |
| { durable: true }, |
| ); |
| }); |
| } |
| |
| recordProviderRequestAttempt(attempt: ProviderRequestAttemptRecord): void { |
| if (!this.input.runStore) return; |
| this.enqueueBestEffortProviderAttempt('append provider request attempt', async () => { |
| await this.input.runStore?.appendEvent(this.sessionId, this.runId, { |
| type: 'provider_request_attempt_recorded', |
| id: attempt.attemptId, |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: attempt.turnId, |
| ts: attempt.completedAt, |
| data: { ...attempt }, |
| }); |
| }); |
| } |
| |
| /** |
| * Canonical accounting record for one physical provider request (#1679). |
| * |
| * Durable, unlike the diagnostic attempt append above: this is the metering |
| * source of truth, and a record lost to a crashed flush is spend nothing else |
| * can reconstruct. |
| * |
| * It reports the failure as `trace_write_failed` and then rejects, so the |
| * caller can tell whether the authority actually holds the record — the Usage |
| * read model must not be written for a call the authority never committed. |
| * Rejecting here is safe: settlement runs inside the model stream's `pull` |
| * handler, and the seam swallows this so a billed, completed response is |
| * never failed by its own bookkeeping. |
| */ |
| recordModelCallAttempt(attempt: ModelCallAttempt): Promise<void> { |
| if (!this.input.runStore) return Promise.resolve(); |
| return this.enqueueRequiredRunStoreWrite('append model call attempt', async () => { |
| await this.input.runStore?.appendEvent( |
| this.sessionId, |
| this.runId, |
| { |
| type: MODEL_CALL_ATTEMPT_EVENT_TYPE, |
| id: attempt.attemptId, |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: attempt.turnId, |
| ts: attempt.completedAt, |
| data: { ...attempt }, |
| }, |
| { durable: true }, |
| ); |
| }); |
| } |
| |
| recordActiveFullCompactBlock(block: ActiveFullCompactBlock): void { |
| if (!this.input.runStore || !this.runStoreAvailable) return; |
| this.enqueueRunStore('append active full compact block', async () => { |
| await this.input.runStore?.appendEvent(this.sessionId, this.runId, { |
| type: 'active_full_compact_block_recorded', |
| id: this.input.newId(), |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: block.turnId || this.turnId, |
| ts: this.input.now(), |
| data: { |
| blockId: block.blockId, |
| highWaterName: block.highWaterName, |
| highWaterSeq: block.highWaterSeq, |
| boundaryKind: 'activeFullCompact', |
| block, |
| }, |
| }); |
| }); |
| } |
| |
| recordHistoryCompactCheckpoint(checkpoint: HistoryCompactCheckpoint): Promise<void> { |
| if (!this.input.runStore) return Promise.reject(new Error('AgentRun store is not configured')); |
| if (!this.runStoreAvailable) return Promise.reject(new Error('AgentRun store is unavailable')); |
| return this.enqueueRunStore( |
| 'append history compact checkpoint', |
| async () => { |
| await this.input.runStore?.appendEvent(this.sessionId, this.runId, { |
| type: 'history_compact_checkpoint_recorded', |
| id: this.input.newId(), |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: this.turnId, |
| ts: this.input.now(), |
| data: { |
| checkpointId: checkpoint.checkpointId, |
| highWaterName: checkpoint.highWaterName, |
| highWaterSeq: checkpoint.highWaterSeq, |
| boundaryKind: 'historyCompact', |
| checkpoint, |
| }, |
| }); |
| }, |
| { rethrow: true }, |
| ); |
| } |
| |
| /** |
| * Durable read of this run's RuntimeEvent ledger for the mid-turn capacity |
| * invariant: waits for every write enqueued so far, then reads the store, so |
| * a caller-derived coverage prefix can only ever span events that are |
| * already persisted. Rejects when the store is unavailable — coverage must |
| * never be computed over a projection the ledger cannot replay. |
| */ |
| async loadTurnRuntimeEvents(): Promise<RuntimeEvent[]> { |
| const store = this.input.runtimeEventStore; |
| if (!store) { |
| throw new Error('RuntimeEvent store is unavailable for turn runtime events'); |
| } |
| await this.flushRuntimePartialBuffer(false); |
| await this.runtimeEventQueue.catch(() => {}); |
| if (this.runtimeEventStoreAvailable) { |
| return await store.readRuntimeEvents(this.sessionId, this.runId); |
| } |
| // The unavailability latch records that a past write failed, not that |
| // the store cannot answer now. This read is the probe that |
| // disambiguates, the same way recordRuntimeEvents reads the ledger back |
| // after an ambiguous append: a store that answers is available again |
| // and the latch lifts, so a stop retried after one rejected write can |
| // still settle its terminal fact (#2253) instead of failing on stale |
| // history forever. A store that cannot answer keeps rejecting, and |
| // coverage is never computed over a projection the ledger cannot |
| // replay. |
| try { |
| const events = await store.readRuntimeEvents(this.sessionId, this.runId); |
| this.runtimeEventStoreAvailable = true; |
| return events; |
| } catch (error) { |
| throw new Error('RuntimeEvent store is unavailable for turn runtime events', { |
| cause: error, |
| }); |
| } |
| } |
| |
| recordSemanticCompactBlock(block: SemanticCompactBlock): void { |
| if (!this.input.runStore || !this.runStoreAvailable) return; |
| this.enqueueRunStore('append semantic compact block', async () => { |
| await this.input.runStore?.appendEvent(this.sessionId, this.runId, { |
| type: 'semantic_compact_block_recorded', |
| id: this.input.newId(), |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: block.turnId || this.turnId, |
| ts: this.input.now(), |
| data: { |
| blockId: block.blockId, |
| highWaterName: block.highWaterName, |
| highWaterSeq: block.highWaterSeq, |
| boundaryKind: 'semanticCompact', |
| block, |
| }, |
| }); |
| }); |
| } |
| |
| async *execute(): AsyncIterable<SessionEvent> { |
| try { |
| const begin = await this.begin(); |
| const invocationId = begin.initialRuntimeEvent.invocationId; |
| const source = 'desktop' as const; |
| const request: InvocationContext['request'] = { |
| sessionId: this.sessionId, |
| invocationId, |
| runId: this.runId, |
| turnId: this.turnId, |
| orchestration: this.effectiveOrchestration, |
| toolMode: this.toolMode, |
| ...(this.input.userInput.maxSteps !== undefined |
| ? { maxSteps: this.input.userInput.maxSteps } |
| : {}), |
| text: this.input.userInput.text, |
| ...(this.input.userInput.attachments |
| ? { attachments: this.input.userInput.attachments } |
| : {}), |
| ...(this.input.userInput.quotes ? { quotes: this.input.userInput.quotes } : {}), |
| ...(this.input.userInput.inlineReferences |
| ? { inlineReferences: this.input.userInput.inlineReferences } |
| : {}), |
| context: begin.backendInput.context, |
| ...(begin.backendInput.runtimeContext |
| ? { runtimeContext: begin.backendInput.runtimeContext } |
| : {}), |
| initialRuntimeEvent: begin.initialRuntimeEvent, |
| source, |
| lineage: this.lineage, |
| }; |
| const ctx: InvocationContext = { |
| sessionId: this.sessionId, |
| invocationId, |
| runId: this.runId, |
| turnId: this.turnId, |
| source, |
| startedAt: begin.initialRuntimeEvent.ts, |
| request, |
| newId: this.input.newId, |
| now: this.input.now, |
| }; |
| let acceptedSessionEvent: SessionEvent | undefined; |
| const flow = new AiSdkFlow({ |
| backend: begin.backend, |
| drainAfterTerminal: true, |
| onSessionEvent: async (sessionEvent, runtimeEvent) => { |
| await this.acceptMappedEvent(sessionEvent, runtimeEvent); |
| acceptedSessionEvent = sessionEvent; |
| }, |
| }); |
| for await (const _runtimeEvent of flow.run(ctx, { |
| text: begin.backendInput.text, |
| ...(begin.backendInput.toolMode !== undefined |
| ? { toolMode: begin.backendInput.toolMode } |
| : {}), |
| ...(begin.backendInput.maxSteps !== undefined |
| ? { maxSteps: begin.backendInput.maxSteps } |
| : {}), |
| ...(begin.backendInput.attachments ? { attachments: begin.backendInput.attachments } : {}), |
| ...(begin.backendInput.quotes ? { quotes: begin.backendInput.quotes } : {}), |
| context: begin.backendInput.context, |
| ...(begin.backendInput.runtimeContext |
| ? { runtimeContext: begin.backendInput.runtimeContext } |
| : {}), |
| })) { |
| if (acceptedSessionEvent) { |
| yield acceptedSessionEvent; |
| acceptedSessionEvent = undefined; |
| } |
| } |
| } catch (error) { |
| await this.recordFailure(error); |
| throw error; |
| } finally { |
| await this.finalize(); |
| } |
| } |
| |
| async acceptMappedEvent( |
| sessionEvent: SessionEvent, |
| runtimeEvent: RuntimeEvent, |
| options: { requireTerminalWrite?: boolean; allowInteractionResume?: boolean } = {}, |
| ): Promise<void> { |
| const partialStreamKey = runtimePartialCoalescingKey(runtimeEvent); |
| if (!partialStreamKey) await this.flushRuntimePartialBuffer(true); |
| if (isTerminalRuntimeEvent(runtimeEvent)) { |
| await this.recordRuntimeEvents([runtimeEvent], { |
| requireTerminalWrite: options.requireTerminalWrite ?? Boolean(this.input.runtimeEventStore), |
| }); |
| await this.recordSessionEvent(sessionEvent, options); |
| return; |
| } |
| if (this.requiresDurablePersistence() && isInteractionResumeAck(sessionEvent)) { |
| // A hosted continuation may resume execution only after its identity-only |
| // settlement fact is durable. Run status advances next, then Session |
| // status; the queue consumer acknowledges the event only after all three. |
| await this.recordRuntimeEvents([runtimeEvent], { requireDurableWrite: true }); |
| await this.recordSessionEvent(sessionEvent, options); |
| return; |
| } |
| await this.recordSessionEvent(sessionEvent, options); |
| if (sessionEvent.type === 'provider_retry') return; |
| if (partialStreamKey) { |
| await this.recordRuntimePartial(runtimeEvent, partialStreamKey); |
| return; |
| } |
| // ToolRuntime already persisted protocol-tagged tool calls/results through |
| // the atomic RuntimeCommitSink. Re-appending the mapped UI event through |
| // the generic lane would duplicate the fact and violate that boundary. |
| if (isAtomicToolBoundaryProjection(runtimeEvent, this.toolBoundaryProtocol)) return; |
| if (!isNonTerminalErrorRuntimeEvent(runtimeEvent)) { |
| // A steered user message is fail-CLOSED: the backend's delivery ack |
| // waits on this consume, and the provider must never execute a |
| // directive the ledger does not carry. Every other non-terminal event |
| // stays fail-open (a trace gap, not a correctness gap). |
| const steering = |
| runtimeEvent.content?.kind === 'text' && runtimeEvent.content.steering === true; |
| await this.recordRuntimeEvents([runtimeEvent], steering ? { requireDurableWrite: true } : {}); |
| if (this.recordsSessionMessages()) { |
| await materializeRuntimeEventTranscriptProjection( |
| this.input.store, |
| this.sessionId, |
| runtimeEvent, |
| ); |
| } |
| } |
| } |
| |
| async begin(): Promise<AgentRunBeginResult> { |
| await this.createRunRecord(); |
| |
| let initialRuntimeEventId: string; |
| if (this.recordsSessionMessages()) { |
| const userMessageId = this.input.userMessageId ?? this.input.newId(); |
| const userMessageTs = this.input.now(); |
| initialRuntimeEventId = userMessageId; |
| const userMsg: UserMessage = { |
| type: 'user', |
| id: userMessageId, |
| turnId: this.turnId, |
| ts: userMessageTs, |
| text: this.input.userInput.text, |
| ...(this.input.userInput.displayText !== undefined |
| ? { displayText: this.input.userInput.displayText } |
| : {}), |
| ...(this.input.userInput.attachments |
| ? { attachments: this.input.userInput.attachments } |
| : {}), |
| ...(this.input.userInput.quotes ? { quotes: this.input.userInput.quotes } : {}), |
| ...(this.input.userInput.inlineReferences |
| ? { inlineReferences: this.input.userInput.inlineReferences } |
| : {}), |
| ...(this.input.userInput.origin ? { origin: this.input.userInput.origin } : {}), |
| }; |
| await this.input.store.appendMessage(this.sessionId, userMsg); |
| await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage); |
| this.lastTs = userMessageTs; |
| } else { |
| initialRuntimeEventId = this.input.newId(); |
| this.lastTs = this.input.now(); |
| } |
| |
| const initialRuntimeEvent = this.buildInitialRuntimeEvent(initialRuntimeEventId, this.lastTs); |
| await this.recordRuntimeEvents([initialRuntimeEvent], { |
| requireDurableWrite: this.requiresDurablePersistence(), |
| }); |
| |
| if (!this.header.connectionLocked) { |
| this.header = await this.input.hooks.updateHeader(this.sessionId, { connectionLocked: true }); |
| } |
| |
| this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); |
| await this.markRunStarted(this.lastTs); |
| |
| await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, this.lastTs); |
| |
| const priorRuntimeContext = await this.buildPriorRuntimeContext(); |
| const projectionContext = priorRuntimeContext |
| ? projectRuntimeEventsToStoredMessages(priorRuntimeContext.events, { |
| runHeaders: priorRuntimeContext.runs, |
| }).messages |
| : []; |
| |
| return { |
| backend: this.active.backend, |
| backendInput: { |
| turnId: this.turnId, |
| orchestration: this.effectiveOrchestration, |
| toolMode: this.toolMode, |
| ...(this.input.userInput.maxSteps !== undefined |
| ? { maxSteps: this.input.userInput.maxSteps } |
| : {}), |
| text: this.input.userInput.text, |
| ...(this.input.userInput.attachments |
| ? { attachments: this.input.userInput.attachments } |
| : {}), |
| ...(this.input.userInput.quotes ? { quotes: this.input.userInput.quotes } : {}), |
| context: projectionContext, |
| ...(priorRuntimeContext ? { runtimeContext: priorRuntimeContext.events } : {}), |
| }, |
| initialRuntimeEvent, |
| }; |
| } |
| |
| async beginOperation(): Promise<AgentRunOperationBeginResult> { |
| await this.createRunRecord(); |
| |
| const startedAt = this.input.now(); |
| this.lastTs = startedAt; |
| if (this.recordsSessionMessages()) { |
| await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage, { |
| ts: startedAt, |
| }); |
| } |
| |
| if (!this.header.connectionLocked) { |
| this.header = await this.input.hooks.updateHeader(this.sessionId, { connectionLocked: true }); |
| } |
| |
| this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); |
| await this.markRunStarted(startedAt); |
| |
| await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, startedAt); |
| |
| const priorRuntimeContext = await this.buildPriorRuntimeContext(); |
| return { |
| backend: this.active.backend, |
| runtimeContext: priorRuntimeContext?.events ?? [], |
| startedAt, |
| }; |
| } |
| |
| async beginContinuation( |
| continuation: RuntimeContinuation, |
| ): Promise<AgentRunContinuationBeginResult> { |
| if ( |
| continuation.sessionId !== this.sessionId || |
| continuation.runId !== this.runId || |
| continuation.turnId !== this.turnId |
| ) { |
| throw new Error('Runtime continuation identity does not match the target AgentRun'); |
| } |
| |
| this.continuationActive = true; |
| await this.createRunRecord(continuation); |
| await this.input.continuationFailpoint?.('after_run_created'); |
| const startedAt = this.input.now(); |
| this.lastTs = startedAt; |
| if (!this.input.commitContinuationStart) { |
| throw new Error('Runtime continuation requires a durable continuation-start authority'); |
| } |
| let committedStart: { startEventId: string; created: true }; |
| try { |
| committedStart = await this.input.commitContinuationStart(startedAt); |
| } catch (error) { |
| throw new ContinuationStartCommitError(error); |
| } |
| await this.input.continuationFailpoint?.('after_continuation_start_committed'); |
| if (this.recordsSessionMessages()) { |
| await this.input.hooks.appendTurnState(this.sessionId, this.turnId, 'running', this.lineage, { |
| ts: startedAt, |
| }); |
| } |
| |
| if (!this.header.connectionLocked) { |
| this.header = await this.input.hooks.updateHeader(this.sessionId, { connectionLocked: true }); |
| } |
| |
| this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); |
| await this.markRunStarted(startedAt); |
| await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, startedAt); |
| |
| return { |
| backend: this.active.backend, |
| startedAt, |
| continuationStartAdmission: createRuntimeContinuationStartAdmissionProof({ |
| startEventId: committedStart.startEventId, |
| claimId: continuation.claimId ?? '', |
| boundaryDigest: continuation.boundary?.manifestDigest ?? 'sha256:', |
| providerProjectionVersion: continuation.providerProjectionVersion ?? 1, |
| providerReplayDigest: continuation.providerReplayDigest ?? 'sha256:', |
| ...(this.toolBoundaryProtocol ? { toolBoundaryProtocol: this.toolBoundaryProtocol } : {}), |
| target: { |
| sessionId: continuation.sessionId, |
| invocationId: continuation.invocationId, |
| runId: continuation.runId, |
| turnId: continuation.turnId, |
| }, |
| }), |
| }; |
| } |
| |
| private buildInitialRuntimeEvent(id: string, ts: number): RuntimeEvent { |
| return buildInitialUserRuntimeEvent({ |
| id, |
| invocationId: this.invocationId, |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: this.turnId, |
| ts, |
| text: this.input.userInput.text, |
| ...(this.input.userInput.displayText !== undefined |
| ? { displayText: this.input.userInput.displayText } |
| : {}), |
| ...(this.input.userInput.origin !== undefined ? { origin: this.input.userInput.origin } : {}), |
| ...(this.input.userInput.attachments !== undefined |
| ? { attachments: this.input.userInput.attachments } |
| : {}), |
| ...(this.input.userInput.quotes !== undefined ? { quotes: this.input.userInput.quotes } : {}), |
| ...(this.input.userInput.inlineReferences !== undefined |
| ? { inlineReferences: this.input.userInput.inlineReferences } |
| : {}), |
| ...(this.toolBoundaryProtocol ? { toolBoundaryProtocol: this.toolBoundaryProtocol } : {}), |
| }); |
| } |
| |
| async recordStoredSessionEvent(ev: SessionEvent): Promise<void> { |
| if (!this.recordsSessionMessages()) return; |
| if (ev.type === 'token_usage') { |
| await this.input.store.appendMessage(this.sessionId, { ...ev } satisfies StoredMessage); |
| } |
| } |
| |
| async recordSessionEvent( |
| ev: SessionEvent, |
| options: { allowInteractionResume?: boolean } = {}, |
| ): Promise<void> { |
| this.lastTs = ev.ts; |
| const transition = statusFromEvent(ev, options); |
| const terminalSessionEvent = |
| (ev.type === 'complete' || ev.type === 'abort') && !this.turnFailed; |
| const turnStatus = terminalSessionEvent ? turnStatusFromEvent(ev) : undefined; |
| if (terminalSessionEvent) { |
| this.sawCompletion = true; |
| if (ev.type === 'abort' && !this.abortSource) this.abortSource = ev.reason; |
| if (ev.type === 'complete' && ev.stopReason === 'user_stop' && !this.abortSource) |
| this.abortSource = 'user_stop'; |
| this.finalStatus = this.stopped |
| ? { status: 'aborted' } |
| : (transition ?? { status: 'active' }); |
| // A terminal complete event can carry a failure without a preceding |
| // error event. Record it now so finalize preserves the precise class. |
| if ( |
| turnStatus?.status === 'failed' && |
| turnStatus.errorClass && |
| !this.failureClass && |
| !this.stopped |
| ) { |
| this.markRunFailed( |
| turnStatus.errorClass, |
| `turn ended with stopReason=${ev.type === 'complete' ? ev.stopReason : 'unknown'}`, |
| ev.ts, |
| ); |
| } |
| } |
| if (transition && !this.stopped) { |
| const updateSessionStatus = async (): Promise<void> => { |
| if (terminalSessionEvent || ev.type === 'error') { |
| await this.input.hooks |
| .updateStatus(this.sessionId, transition.status, transition.blockedReason, ev.ts) |
| .catch((error) => this.enqueueTraceWriteFailure(error, 'terminal session projection')); |
| return; |
| } |
| await this.input.hooks.updateStatus( |
| this.sessionId, |
| transition.status, |
| transition.blockedReason, |
| ev.ts, |
| ); |
| }; |
| // On resume, advance the Run before the Session so an interrupted pair |
| // remains conservatively waiting rather than advertising false readiness. |
| if (this.requiresDurablePersistence() && isInteractionResumeAck(ev)) { |
| await this.recordStatusFromTransition(ev, transition, ev.ts); |
| await updateSessionStatus(); |
| } else { |
| await updateSessionStatus(); |
| await this.recordStatusFromTransition(ev, transition, ev.ts); |
| } |
| } |
| if (turnStatus && !this.stopped && this.recordsSessionMessages()) { |
| const appendTurnState = this.input.hooks.appendTurnState( |
| this.sessionId, |
| this.turnId, |
| turnStatus.status, |
| this.lineage, |
| { |
| ts: ev.ts, |
| errorClass: turnStatus.errorClass, |
| ...(turnStatus.status === 'aborted' && this.abortSource |
| ? { abortSource: this.abortSource } |
| : {}), |
| }, |
| ); |
| if (terminalSessionEvent || ev.type === 'error') { |
| await appendTurnState.catch((error) => |
| this.enqueueTraceWriteFailure(error, 'terminal session projection'), |
| ); |
| } else { |
| await appendTurnState; |
| } |
| } |
| if (ev.type === 'error') { |
| if (this.stopped) { |
| this.finalStatus = { status: 'aborted' }; |
| } else { |
| this.turnFailed = true; |
| this.finalStatus = transition ?? { status: 'blocked', blockedReason: 'unknown' }; |
| if (this.recordsSessionMessages()) { |
| await this.input.hooks |
| .appendTurnState(this.sessionId, this.turnId, 'failed', this.lineage, { |
| ts: ev.ts, |
| errorClass: ev.reason ?? ev.code ?? 'unknown', |
| }) |
| .catch((error) => this.enqueueTraceWriteFailure(error, 'terminal session projection')); |
| } |
| this.markRunFailed(ev.reason ?? ev.code ?? 'unknown', ev.message, ev.ts); |
| } |
| } |
| } |
| |
| async recordRuntimeEvents( |
| events: readonly RuntimeEvent[], |
| options: { requireTerminalWrite?: boolean; requireDurableWrite?: boolean } = {}, |
| ): Promise<void> { |
| if (events.length === 0) return; |
| for (const event of events) { |
| const terminal = isTerminalRuntimeEvent(event); |
| const eventForStore = terminal ? this.reserveTerminalEvent(event) : event; |
| if (!eventForStore) continue; |
| if (!this.input.runtimeEventStore || !this.runtimeEventStoreAvailable) { |
| if (this.input.runtimeEventStore?.durability === 'canonical') { |
| throw ( |
| this.runtimeEventStoreFailure ?? |
| new Error('canonical RuntimeEvent store is unavailable') |
| ); |
| } |
| if (terminal && options.requireTerminalWrite) { |
| throw new Error('terminal RuntimeEvent store is unavailable'); |
| } |
| if (options.requireDurableWrite && this.input.runtimeEventStore) { |
| // The store exists but earlier writes failed: a durability-required |
| // event (steering) must not silently skip the ledger. |
| throw new Error('RuntimeEvent store is unavailable for a durability-required event'); |
| } |
| continue; |
| } |
| const write = this.enqueueRuntimeEventStore( |
| 'append runtime event', |
| async () => { |
| await this.input.runtimeEventStore?.appendRuntimeEvent( |
| this.sessionId, |
| this.runId, |
| eventForStore, |
| { durable: terminal || options.requireDurableWrite === true }, |
| ); |
| }, |
| { |
| rethrow: |
| terminal || |
| options.requireTerminalWrite || |
| options.requireDurableWrite || |
| this.input.runtimeEventStore.durability === 'canonical', |
| }, |
| ); |
| if (terminal && this.terminalClaim) this.terminalClaim.write = write; |
| if (options.requireDurableWrite && !terminal) { |
| // An append error is AMBIGUOUS: the bytes may have landed before the |
| // failure (e.g. a close error after the write). For a |
| // durability-required event the caller settles a delivery lease on |
| // this outcome, so a false "not durable" would redeliver a message |
| // the ledger already owns. Read the ledger back to disambiguate: |
| // present ⇒ durable (continue on the ack path); absent or read-back |
| // also failing ⇒ fail closed (rethrow ⇒ nack). |
| try { |
| await write; |
| } catch (error) { |
| if (error instanceof DurableStoreWriteError) throw error; |
| if (!(await this.eventLandedInLedger(eventForStore.id))) throw error; |
| // The write landed and the ledger answered a fresh read — the |
| // failure was in the reporting, not the store. Lift the |
| // unavailability latch so the rest of the turn (including its |
| // required terminal write) keeps persisting; a genuinely broken |
| // store re-latches on its next write. |
| this.runtimeEventStoreAvailable = true; |
| } |
| continue; |
| } |
| await write; |
| } |
| } |
| |
| private reserveTerminalEvent(event: RuntimeEvent): RuntimeEvent | undefined { |
| if (this.terminalClaim?.event) return undefined; |
| this.terminalClaim ??= { owner: 'event' }; |
| const eventForStore = |
| this.terminalClaim.owner === 'stop' ? this.abortedRuntimeEvent(event) : event; |
| this.terminalClaim.event = eventForStore; |
| return eventForStore; |
| } |
| |
| private abortedRuntimeEvent(event: RuntimeEvent): RuntimeEvent { |
| const { content: _content, ...rest } = event; |
| void _content; |
| return { |
| ...rest, |
| status: 'aborted', |
| actions: { |
| ...event.actions, |
| endInvocation: true, |
| stateDelta: { |
| ...event.actions?.stateDelta, |
| abortSource: this.abortSource ?? 'user_stop', |
| }, |
| }, |
| }; |
| } |
| |
| async recordFailure(error: unknown): Promise<void> { |
| if (this.stopped) { |
| this.finalStatus = { status: 'aborted' }; |
| return; |
| } |
| this.finalStatus = { status: 'blocked', blockedReason: 'unknown' }; |
| if (this.recordsSessionMessages()) { |
| await this.input.hooks |
| .appendTurnState(this.sessionId, this.turnId, 'failed', this.lineage, { |
| errorClass: error instanceof Error ? error.name : 'unknown', |
| }) |
| .catch(() => {}); |
| } |
| this.markRunFailed( |
| error instanceof Error ? error.name : 'unknown', |
| errorMessage(error), |
| this.input.now(), |
| ); |
| } |
| |
| async finalize(): Promise<void> { |
| if (this.finalized) return; |
| this.finalized = true; |
| await this.flushRuntimePartialBuffer(true); |
| const lastTs = this.lastTs || this.input.now(); |
| if (this.stopped) this.finalStatus = { status: 'aborted' }; |
| if (!this.finalStatus) { |
| this.finalStatus = { status: 'blocked', blockedReason: 'unknown' }; |
| this.markRunFailed( |
| 'missing_terminal_event', |
| 'run finalized without a terminal SessionEvent', |
| lastTs, |
| ); |
| } |
| this.reserveFinalizationTerminal(this.finalStatus, lastTs); |
| if (this.active) { |
| await this.input.hooks.unregisterRun(this.active, this); |
| } |
| const nextStatus = |
| this.active && this.active.activeRuns.size > 0 |
| ? { status: 'running' as const } |
| : (this.finalStatus ?? { status: 'active' as const }); |
| try { |
| await this.input.hooks.updateHeader(this.sessionId, { |
| lastUsedAt: lastTs, |
| lastMessageAt: lastTs, |
| hasUnread: true, |
| ...buildStatusPatch(nextStatus.status, lastTs, nextStatus.blockedReason), |
| }); |
| } catch { |
| // The user-visible turn already completed; preserve existing behavior. |
| } |
| if (this.sawCompletion && this.recordsSessionMessages()) { |
| await this.input.store |
| .appendMessage(this.sessionId, { |
| type: 'system_note', |
| id: this.input.newId(), |
| turnId: this.turnId, |
| ts: lastTs, |
| kind: 'session_resume', |
| } satisfies SystemNoteMessage) |
| .catch(() => {}); |
| } |
| await this.finishRun(this.finalStatus, lastTs); |
| } |
| |
| private recordsSessionMessages(): boolean { |
| return this.input.recordSessionMessages !== false; |
| } |
| |
| private async createRunRecord(continuation?: RuntimeContinuation): Promise<void> { |
| if (!this.input.runStore) { |
| if (continuation) throw new Error('Runtime continuation requires a durable run store'); |
| return; |
| } |
| const createdAt = |
| continuation && this.input.claimedRunHeader |
| ? this.input.claimedRunHeader.createdAt |
| : this.input.now(); |
| const computedHeader: AgentRunHeader = { |
| runId: this.runId, |
| invocationId: this.invocationId, |
| sessionId: this.sessionId, |
| turnId: this.turnId, |
| status: 'created', |
| backendKind: this.header.backend, |
| llmConnectionSlug: this.header.llmConnectionSlug, |
| modelId: this.header.model, |
| cwd: this.header.cwd, |
| ...(this.input.workspaceIdentity ? { workspaceIdentity: this.input.workspaceIdentity } : {}), |
| permissionMode: this.header.permissionMode, |
| collaborationMode: this.header.collaborationMode ?? 'agent', |
| orchestrationMode: this.effectiveOrchestration.mode, |
| orchestrationSource: this.effectiveOrchestration.source, |
| agentSwarmAuthorization: this.effectiveOrchestration.agentSwarmAuthorization, |
| toolMode: this.toolMode, |
| createdAt, |
| updatedAt: createdAt, |
| ...this.lineage, |
| ...(continuation |
| ? { |
| continuationSource: |
| continuation.claimId && continuation.boundary |
| ? { |
| protocol: 'continuation_source_v2' as const, |
| claimId: continuation.claimId, |
| boundaryDigest: continuation.boundary.manifestDigest, |
| sourceInvocationId: continuation.sourceInvocationId, |
| sourceRunId: continuation.sourceRunId, |
| sourceTurnId: continuation.sourceTurnId, |
| sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater, |
| sourcePrefixDigest: continuation.boundary.segments.at(-1)!.prefixDigest, |
| replayManifestDigest: continuation.boundary.manifestDigest, |
| } |
| : { |
| sourceInvocationId: continuation.sourceInvocationId, |
| sourceRunId: continuation.sourceRunId, |
| sourceTurnId: continuation.sourceTurnId, |
| sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater, |
| }, |
| } |
| : {}), |
| ...(this.input.userInput.agentId ? { agentId: this.input.userInput.agentId } : {}), |
| ...(this.input.userInput.agentName ? { agentName: this.input.userInput.agentName } : {}), |
| ...(this.input.userInput.origin?.kind === 'automation' |
| ? { automationId: this.input.userInput.origin.automationId } |
| : {}), |
| ...(this.input.userInput.origin?.kind === 'goal' |
| ? { goalId: this.input.userInput.origin.goalId } |
| : {}), |
| ...(this.input.userInput.origin?.kind === 'agent_graph' |
| ? { |
| agentGraphWakeId: this.input.userInput.origin.wakeId, |
| agentGraphWakeAttemptId: this.input.userInput.origin.attemptId, |
| } |
| : {}), |
| ...(this.input.rootExecutionKind ? { rootExecutionKind: this.input.rootExecutionKind } : {}), |
| }; |
| const header = |
| continuation && this.input.claimedRunHeader ? this.input.claimedRunHeader : computedHeader; |
| if ( |
| continuation && |
| this.input.claimedRunHeader && |
| !isDeepStrictEqual(this.input.claimedRunHeader, computedHeader) |
| ) { |
| throw new Error('Claimed continuation target Run header no longer matches execution'); |
| } |
| try { |
| const durable = this.requiresDurablePersistence(); |
| await this.input.runStore.createRun(header, { durable }); |
| await this.input.runStore.appendEvent( |
| this.sessionId, |
| this.runId, |
| { |
| type: 'run_created', |
| id: this.input.newId(), |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: this.turnId, |
| ts: createdAt, |
| data: { |
| textLength: this.input.userInput.text.length, |
| attachmentCount: this.input.userInput.attachments?.length ?? 0, |
| orchestrationMode: this.effectiveOrchestration.mode, |
| orchestrationSource: this.effectiveOrchestration.source, |
| agentSwarmAuthorization: this.effectiveOrchestration.agentSwarmAuthorization, |
| toolMode: this.toolMode, |
| }, |
| }, |
| { durable }, |
| ); |
| } catch (error) { |
| this.runStoreAvailable = false; |
| if (this.requiresDurablePersistence()) throw error; |
| this.enqueueTraceWriteFailure(error); |
| if (continuation) throw error; |
| } |
| } |
| |
| private requiresDurablePersistence(): boolean { |
| return this.input.durability === 'required'; |
| } |
| |
| private async buildPriorRuntimeContext(): Promise<PriorRuntimeContext | undefined> { |
| return await buildPriorRuntimeContextProjection({ |
| sessionId: this.sessionId, |
| currentRunId: this.runId, |
| currentTurnId: this.turnId, |
| parentRunId: this.lineage.parentRunId, |
| resumedFromRunId: this.lineage.resumedFromRunId, |
| agentId: this.input.userInput.agentId, |
| linkedChildSession: this.input.header.subagentParent?.kind === 'subagent', |
| runStore: this.input.runStore, |
| runtimeEventStore: this.input.runtimeEventStore, |
| runStoreAvailable: this.runStoreAvailable, |
| runtimeEventStoreAvailable: this.runtimeEventStoreAvailable, |
| repairRunRuntimeLedger: this.input.repairRunRuntimeLedger, |
| readMessages: () => this.input.store.readMessages(this.sessionId), |
| }); |
| } |
| |
| private async markRunStarted(ts: number): Promise<void> { |
| if (!this.input.runStore || !this.runStoreAvailable) return; |
| const durable = this.requiresDurablePersistence(); |
| const write = this.enqueueRunStore( |
| 'mark run started', |
| async () => { |
| await this.input.runStore?.appendEvent( |
| this.sessionId, |
| this.runId, |
| { |
| type: 'run_started', |
| id: this.input.newId(), |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: this.turnId, |
| ts, |
| }, |
| { durable }, |
| ); |
| await this.input.runStore?.updateRun( |
| this.sessionId, |
| this.runId, |
| { status: 'running', updatedAt: ts }, |
| { durable }, |
| ); |
| }, |
| { rethrow: durable }, |
| ); |
| if (durable) await write; |
| } |
| |
| private async recordStatusFromTransition( |
| ev: SessionEvent, |
| transition: { status: SessionStatus; blockedReason?: SessionBlockedReason }, |
| ts: number, |
| ): Promise<void> { |
| const durable = this.requiresDurablePersistence(); |
| const runStore = this.input.runStore; |
| if (!runStore) { |
| if (durable) { |
| throw new Error('AgentRun store is unavailable for a required status transition'); |
| } |
| return; |
| } |
| const status = |
| transition.status === 'waiting_for_user' |
| ? 'waiting_for_user' |
| : transition.status === 'aborted' |
| ? 'cancelled' |
| : transition.status === 'blocked' |
| ? 'failed' |
| : transition.status === 'active' |
| ? 'completed' |
| : 'running'; |
| if (isTerminalRunStatus(status)) return; |
| const appendAudit = async (): Promise<void> => { |
| await runStore.appendEvent( |
| this.sessionId, |
| this.runId, |
| { |
| type: 'run_status_changed', |
| id: this.input.newId(), |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: this.turnId, |
| ts, |
| data: { |
| sessionStatus: transition.status, |
| ...(transition.blockedReason ? { blockedReason: transition.blockedReason } : {}), |
| }, |
| }, |
| { durable }, |
| ); |
| }; |
| if (durable) { |
| await this.enqueueRequiredRunStoreWrite('record required run status', async () => { |
| await runStore.updateRun( |
| this.sessionId, |
| this.runId, |
| { status, updatedAt: ts }, |
| { durable: true }, |
| ); |
| }); |
| // The audit remains best-effort, but its physical write belongs to this |
| // required transition and must settle before the resume acknowledgement. |
| await this.enqueueRunStore('append run status audit', appendAudit); |
| } else { |
| this.enqueueRunStore('record run status', async () => { |
| await runStore.updateRun(this.sessionId, this.runId, { status, updatedAt: ts }); |
| await appendAudit(); |
| }); |
| } |
| if (ev.type === 'abort') { |
| this.markRunCancelled(ev.reason, ts); |
| } |
| } |
| |
| private markRunFailed(failureClass: string, message: string, ts: number): void { |
| if (!this.input.runStore || !this.runStoreAvailable) return; |
| this.failureClass = failureClass; |
| this.failureMessage = redactTraceString(message); |
| if (this.input.runtimeEventStore) return; |
| this.enqueueRunStore('mark run failed', async () => { |
| await this.input.runStore?.updateRun(this.sessionId, this.runId, { |
| status: 'failed', |
| updatedAt: ts, |
| completedAt: ts, |
| failureClass, |
| failureMessage: this.failureMessage, |
| }); |
| await this.input.runStore?.appendEvent(this.sessionId, this.runId, { |
| type: 'run_failed', |
| id: this.input.newId(), |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: this.turnId, |
| ts, |
| message: redactTraceString(message), |
| data: { failureClass }, |
| }); |
| }); |
| } |
| |
| private markRunCancelled(reason: string | undefined, ts: number): void { |
| if (!this.input.runStore || !this.runStoreAvailable) return; |
| if (this.input.runtimeEventStore) return; |
| this.enqueueRunStore('mark run cancelled', async () => { |
| await this.input.runStore?.updateRun(this.sessionId, this.runId, { |
| status: 'cancelled', |
| updatedAt: ts, |
| completedAt: ts, |
| }); |
| await this.input.runStore?.appendEvent(this.sessionId, this.runId, { |
| type: 'run_cancelled', |
| id: this.input.newId(), |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: this.turnId, |
| ts, |
| ...(reason ? { message: redactTraceString(reason) } : {}), |
| }); |
| }); |
| } |
| |
| private async finishRun( |
| finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, |
| ts: number, |
| ): Promise<void> { |
| await this.traceQueue.catch(() => {}); |
| if (!this.input.runStore || !this.runStoreAvailable) return; |
| const status = this.runStatusForFinalStatus(finalStatus); |
| const isTerminal = status === 'completed' || status === 'failed' || status === 'cancelled'; |
| if (isTerminal && this.input.runtimeEventStore) { |
| await this.commitTerminalRun(finalStatus, ts); |
| return; |
| } |
| await this.enqueueRunStore('finish run', async () => { |
| await this.input.runStore?.updateRun(this.sessionId, this.runId, { |
| status, |
| updatedAt: ts, |
| ...(isTerminal ? { completedAt: ts } : {}), |
| ...(status === 'failed' |
| ? { |
| failureClass: this.failureClass ?? finalStatus?.blockedReason ?? 'unknown', |
| ...(this.failureMessage ? { failureMessage: this.failureMessage } : {}), |
| } |
| : {}), |
| }); |
| await this.input.runStore?.appendEvent(this.sessionId, this.runId, { |
| type: |
| status === 'cancelled' |
| ? 'run_cancelled' |
| : status === 'failed' |
| ? 'run_failed' |
| : status === 'completed' |
| ? 'run_completed' |
| : 'run_status_changed', |
| id: this.input.newId(), |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: this.turnId, |
| ts, |
| ...(status === 'failed' |
| ? { data: { failureClass: this.failureClass ?? finalStatus?.blockedReason ?? 'unknown' } } |
| : status === 'waiting_for_user' |
| ? { |
| data: { |
| sessionStatus: 'waiting_for_user', |
| blockedReason: finalStatus?.blockedReason ?? 'permission_required', |
| }, |
| } |
| : {}), |
| }); |
| }); |
| await this.traceQueue.catch(() => {}); |
| } |
| |
| private runStatusForFinalStatus( |
| finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, |
| ): AgentRunHeader['status'] { |
| if (this.stopped || finalStatus?.status === 'aborted') return 'cancelled'; |
| if (this.failureClass || finalStatus?.status === 'blocked') return 'failed'; |
| if (finalStatus?.status === 'waiting_for_user') return 'waiting_for_user'; |
| return 'completed'; |
| } |
| |
| private async commitTerminalRun( |
| finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, |
| ts: number, |
| ): Promise<void> { |
| if (this.terminalRunHeaderCommitted) return; |
| const runStore = this.input.runStore; |
| const runtimeEventStore = this.input.runtimeEventStore; |
| if ( |
| !runStore || |
| !this.runStoreAvailable || |
| !runtimeEventStore || |
| !this.runtimeEventStoreAvailable |
| ) |
| return; |
| const fallbackStatus = |
| this.stopped || finalStatus?.status === 'aborted' ? 'cancelled' : 'failed'; |
| const fallbackFailureClass = 'missing_terminal_event'; |
| const fallbackFailureMessage = |
| this.failureMessage ?? 'run finalized without a terminal RuntimeEvent'; |
| try { |
| const terminalClaim = this.terminalClaim; |
| const terminalEvent = terminalClaim?.event; |
| if (!terminalEvent) throw new Error('terminal RuntimeEvent claim is missing'); |
| await terminalClaim.write; |
| // Re-check after the await, not only at entry. Two callers — a stop |
| // settling the claim and the stream's own finalize — can both pass the |
| // entry guard and then queue behind the same write. The claim slot |
| // dedupes the RuntimeEvent, but the run-store projection would append a |
| // second terminal AgentRunEvent for the one run. |
| if (this.terminalRunHeaderCommitted) return; |
| if (this.continuationActive) { |
| await this.input.continuationFailpoint?.('after_terminal_event_committed'); |
| } |
| const commit = commitOrCreateTerminalRunFact({ |
| runStore, |
| runtimeEventStore, |
| newId: this.input.newId, |
| sessionId: this.sessionId, |
| runId: this.runId, |
| turnId: this.turnId, |
| ts, |
| terminalEvent, |
| ...((this.failureClass ?? finalStatus?.blockedReason) |
| ? { failureClass: this.failureClass ?? finalStatus?.blockedReason } |
| : {}), |
| ...(this.failureMessage ? { failureMessage: this.failureMessage } : {}), |
| ...(this.traceWriteError ? { traceWriteError: this.traceWriteError } : {}), |
| ...(this.abortSource || fallbackStatus === 'cancelled' |
| ? { abortSource: this.abortSource ?? 'user_stop' } |
| : {}), |
| fallbackStatus, |
| fallbackInvocationId: this.runId, |
| ...(fallbackStatus === 'failed' ? { fallbackFailureClass, fallbackFailureMessage } : {}), |
| allowHeaderCommitFailure: true, |
| }); |
| if (!terminalClaim.write) { |
| terminalClaim.write = commit.then(() => undefined); |
| void terminalClaim.write.catch(() => {}); |
| } |
| const result = await commit; |
| this.terminalRunHeaderCommitted = result.headerCommitted; |
| if (result.headerCommitted && this.continuationActive) { |
| await this.input.continuationFailpoint?.('after_terminal_header_committed'); |
| } |
| if (result.headerCommitError !== undefined) { |
| await this.enqueueTraceWriteFailure(result.headerCommitError, 'commit terminal run header'); |
| } |
| } catch (error) { |
| this.runStoreAvailable = false; |
| await this.enqueueTraceWriteFailure(error, 'commit terminal run header'); |
| throw error; |
| } |
| await this.traceQueue.catch(() => {}); |
| } |
| |
| private reserveFinalizationTerminal( |
| finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, |
| ts: number, |
| ): void { |
| if (this.terminalClaim?.event) return; |
| const runStatus = this.runStatusForFinalStatus(finalStatus); |
| if (runStatus !== 'completed' && runStatus !== 'failed' && runStatus !== 'cancelled') return; |
| const status = |
| this.terminalClaim?.owner === 'stop' || this.stopped || finalStatus?.status === 'aborted' |
| ? 'cancelled' |
| : 'failed'; |
| const failureClass = 'missing_terminal_event'; |
| const failureMessage = this.failureMessage ?? 'run finalized without a terminal RuntimeEvent'; |
| if (status === 'failed') { |
| this.failureClass = failureClass; |
| this.failureMessage = failureMessage; |
| } |
| this.reserveTerminalEvent( |
| buildSyntheticTerminalRuntimeEvent({ |
| id: this.input.newId(), |
| invocationId: this.invocationId, |
| run: { sessionId: this.sessionId, runId: this.runId, turnId: this.turnId }, |
| status, |
| ts, |
| ...(status === 'failed' ? { failureClass, message: failureMessage } : {}), |
| ...(status === 'cancelled' ? { abortSource: this.abortSource ?? 'user_stop' } : {}), |
| }), |
| ); |
| } |
| |
| private enqueueRunStore( |
| label: string, |
| operation: () => Promise<void>, |
| options: { rethrow?: boolean } = {}, |
| ): Promise<void> { |
| if (!this.input.runStore || !this.runStoreAvailable) return Promise.resolve(); |
| const next = this.traceQueue.then(operation, operation).catch(async (error) => { |
| this.runStoreAvailable = false; |
| await this.enqueueTraceWriteFailure(error, label); |
| if (options.rethrow) throw error; |
| }); |
| this.traceQueue = next.catch(() => {}); |
| return next; |
| } |
| |
| /** |
| * Each physical provider request gets its own best-effort diagnostic row. |
| * One failed attempt append must not suppress later attempts or poison the |
| * general AgentRun store latch; a required capture independently gates every |
| * provider dispatch. |
| */ |
| private enqueueBestEffortProviderAttempt(label: string, operation: () => Promise<void>): void { |
| const next = this.traceQueue |
| .then(operation, operation) |
| .catch((error) => this.enqueueTraceWriteFailure(error, label)); |
| this.traceQueue = next.catch(() => {}); |
| } |
| |
| /** |
| * Serialize a required Run-store write without consulting the best-effort |
| * latch. A successful required write proves the store is available again; |
| * a failed operation rejects its caller without changing the general latch. |
| */ |
| private enqueueRequiredRunStoreWrite( |
| label: string, |
| operation: () => Promise<void>, |
| ): Promise<void> { |
| const probe = async (): Promise<void> => { |
| await operation(); |
| this.runStoreAvailable = true; |
| }; |
| const next = this.traceQueue.then(probe, probe).catch(async (error) => { |
| await this.enqueueTraceWriteFailure(error, label); |
| throw error; |
| }); |
| this.traceQueue = next.catch(() => {}); |
| return next; |
| } |
| |
| /** |
| * Read-back disambiguation for a failed durability-required append: true |
| * only when the ledger demonstrably contains the event. Any doubt (no |
| * read-back capability, read failure, event absent) reports false so the |
| * caller stays fail-closed. |
| */ |
| private async eventLandedInLedger(eventId: string): Promise<boolean> { |
| const store = this.input.runtimeEventStore; |
| if (!store?.readImmutableRuntimeEvents) return false; |
| try { |
| const events = await store.readImmutableRuntimeEvents(this.sessionId, this.runId); |
| return events.some((event) => event.id === eventId); |
| } catch { |
| return false; |
| } |
| } |
| |
| private enqueueRuntimeEventStore( |
| label: string, |
| operation: () => Promise<void>, |
| options: { rethrow?: boolean } = {}, |
| ): Promise<void> { |
| if (!this.input.runtimeEventStore || !this.runtimeEventStoreAvailable) return Promise.resolve(); |
| const next = this.runtimeEventQueue.then(operation, operation).catch(async (error) => { |
| // A rejection is the ledger refusing one malformed candidate, not the |
| // store going away: it stays healthy and readable, so the latch would |
| // only cost this run the writes it still owes — above all its own |
| // terminal event, which `recordRuntimeEvents` refuses once the store |
| // reads unavailable. That is how a single refused append left a run at |
| // `running` with no terminal event and no visible failure (#2234). The |
| // append still fails the caller (a producer bug must not pass quietly), |
| // but the ledger stays open so the turn can end the way every other |
| // failure ends. |
| // |
| // Only that one class is exempt. A store that went away keeps latching: |
| // nothing this run emits next can land. |
| // |
| // `ToolLedgerCorruptionError` also keeps latching, but be precise about |
| // what that buys, because it is less than it looks. A damaged ledger |
| // refuses TOOL facts only — the health scan sits behind |
| // `isToolLedgerBearingEvent` — so this run's terminal event, which bears |
| // no tool fact, is a write the corrupt store would have taken. The latch |
| // is what keeps it out, and the run ends at `running` with no terminal |
| // fact: #2234's own shape, for the already-damaged population. Held here |
| // deliberately rather than fixed in passing — a run that cannot write its |
| // tool facts should arguably still be allowed to say it ended, but that |
| // is a behaviour change on a path this commit does not otherwise touch. |
| // Tracked in #2313; the corrupt-ledger test pins the current price. |
| if (!(error instanceof ToolLedgerRejectionError)) { |
| this.runtimeEventStoreAvailable = false; |
| this.runtimeEventStoreFailure = error; |
| } |
| await this.enqueueTraceWriteFailure(error, label); |
| if (options.rethrow) throw error; |
| }); |
| this.runtimeEventQueue = next.catch(() => {}); |
| return next; |
| } |
| |
| private async recordRuntimePartial(event: RuntimeEvent, streamKey: string): Promise<void> { |
| const store = this.input.runtimeEventStore; |
| if (!store?.appendRuntimePartialBatch) { |
| await this.recordRuntimeEvents([event]); |
| return; |
| } |
| if (!this.runtimeEventStoreAvailable) { |
| await this.recordRuntimeEvents([event]); |
| return; |
| } |
| if (this.runtimePartialStreamKey !== streamKey) { |
| await this.flushRuntimePartialBuffer(true); |
| // Persist the first chunk synchronously. Besides bounding crash loss, this |
| // captures the immutable anchor before an upstream tool boundary can |
| // commit while later chunks are waiting in the coalescer. |
| await this.recordRuntimeEvents([event]); |
| this.runtimePartialStreamKey = streamKey; |
| return; |
| } |
| this.runtimePartialBuffer.push(event); |
| this.runtimePartialBufferBytes += runtimePartialTextBytes(event); |
| if (this.runtimePartialBufferBytes >= RUNTIME_PARTIAL_BATCH_MAX_BYTES) { |
| await this.flushRuntimePartialBuffer(false); |
| return; |
| } |
| this.scheduleRuntimePartialFlush(); |
| } |
| |
| private scheduleRuntimePartialFlush(): void { |
| if (this.runtimePartialFlushTimer) return; |
| this.runtimePartialFlushTimer = setTimeout(() => { |
| this.runtimePartialFlushTimer = undefined; |
| void this.flushRuntimePartialBuffer(false).catch(() => { |
| // enqueueRuntimeEventStore latches and reports the failure. The next |
| // event or execution boundary observes that latch and fails closed. |
| }); |
| }, RUNTIME_PARTIAL_FLUSH_INTERVAL_MS); |
| } |
| |
| private async flushRuntimePartialBuffer(closeStream: boolean): Promise<void> { |
| const ownedPartialWork = |
| this.runtimePartialStreamKey !== undefined || |
| this.runtimePartialBuffer.length > 0 || |
| this.runtimePartialFlushTimer !== undefined; |
| if (this.runtimePartialFlushTimer) { |
| clearTimeout(this.runtimePartialFlushTimer); |
| this.runtimePartialFlushTimer = undefined; |
| } |
| const events = this.runtimePartialBuffer; |
| this.runtimePartialBuffer = []; |
| this.runtimePartialBufferBytes = 0; |
| if (closeStream) this.runtimePartialStreamKey = undefined; |
| if (events.length === 0) { |
| // A timer flush may already be queued. Waiting here preserves the rule |
| // that an immutable boundary never overtakes prior presentation text. |
| if (closeStream) { |
| await this.runtimeEventQueue; |
| if ( |
| ownedPartialWork && |
| !this.runtimeEventStoreAvailable && |
| this.input.runtimeEventStore?.durability === 'canonical' |
| ) { |
| throw ( |
| this.runtimeEventStoreFailure ?? |
| new Error('canonical RuntimeEvent store is unavailable') |
| ); |
| } |
| } |
| return; |
| } |
| const store = this.input.runtimeEventStore; |
| if (!store?.appendRuntimePartialBatch) { |
| await this.recordRuntimeEvents(events); |
| return; |
| } |
| await this.enqueueRuntimeEventStore( |
| 'append runtime partial batch', |
| async () => { |
| await store.appendRuntimePartialBatch?.(this.sessionId, this.runId, events); |
| }, |
| { rethrow: store.durability === 'canonical' }, |
| ); |
| } |
| |
| private async enqueueTraceWriteFailure( |
| error: unknown, |
| label = 'agent run store write', |
| ): Promise<void> { |
| const message = errorMessage(error); |
| this.traceWriteError ??= `${label}: ${message}`; |
| try { |
| await this.input.runStore?.updateRun(this.sessionId, this.runId, { |
| traceWriteError: this.traceWriteError, |
| updatedAt: this.input.now(), |
| }); |
| } catch { |
| // The terminal header commit retries the in-memory latch. |
| } |
| try { |
| await this.input.runStore?.appendEvent(this.sessionId, this.runId, { |
| type: 'trace_write_failed', |
| id: this.input.newId(), |
| runId: this.runId, |
| sessionId: this.sessionId, |
| turnId: this.turnId, |
| ts: this.input.now(), |
| message, |
| }); |
| } catch { |
| // Diagnostic persistence is best effort; never perturb model/tool execution. |
| } |
| } |
| } |
| |
| function runtimePartialCoalescingKey(event: RuntimeEvent): string | undefined { |
| if (!event.partial || event.status !== undefined || event.actions) return undefined; |
| const content = event.content; |
| if (content?.kind !== 'text' && content?.kind !== 'thinking') return undefined; |
| if (content.kind === 'text' && content.attachments !== undefined) return undefined; |
| if (content.kind === 'thinking' && content.signature !== undefined) return undefined; |
| const providerEventId = event.refs?.providerEventId; |
| if (!providerEventId || Object.keys(event.refs ?? {}).some((key) => key !== 'providerEventId')) { |
| return undefined; |
| } |
| return JSON.stringify([ |
| content.kind, |
| providerEventId, |
| event.sessionId, |
| event.invocationId, |
| event.runId, |
| event.turnId, |
| event.branch ?? null, |
| event.role, |
| event.author, |
| ]); |
| } |
| |
| function runtimePartialTextBytes(event: RuntimeEvent): number { |
| const content = event.content; |
| return content?.kind === 'text' || content?.kind === 'thinking' |
| ? Buffer.byteLength(content.text, 'utf8') |
| : 0; |
| } |
| |
| function traceToRunEvent(event: RunTraceEvent, runId: string): EmittedAgentRunEvent { |
| return { |
| type: event.type, |
| id: event.id, |
| runId, |
| sessionId: event.sessionId, |
| turnId: event.turnId, |
| ts: event.ts, |
| message: redactTraceString(event.message), |
| data: sanitizeTraceData(event.data), |
| }; |
| } |
| |
| function sanitizeTraceData( |
| data: Record<string, unknown> | undefined, |
| ): Record<string, unknown> | undefined { |
| if (!data) return undefined; |
| return Object.fromEntries( |
| Object.entries(data) |
| .filter(([, value]) => value !== undefined) |
| .map(([key, value]) => [key, sanitizeTraceValue(value)]), |
| ); |
| } |
| |
| function sanitizeTraceValue(value: unknown): unknown { |
| if (typeof value === 'string') return redactTraceString(value); |
| if (Array.isArray(value)) return value.slice(0, 50).map(sanitizeTraceValue); |
| if (value && typeof value === 'object') { |
| return Object.fromEntries( |
| Object.entries(value as Record<string, unknown>) |
| .slice(0, 50) |
| .map(([key, nested]) => [key, sanitizeTraceValue(nested)]), |
| ); |
| } |
| return value; |
| } |
| |
| function redactTraceString(value: string): string { |
| const redacted = redactSecrets(value); |
| return redacted.length > 2_000 ? `${redacted.slice(0, 2_000)}...[truncated]` : redacted; |
| } |
| |
| function errorMessage(error: unknown): string { |
| return redactTraceString(error instanceof Error ? error.message : String(error)); |
| } |
| function isInteractionResumeAck(event: SessionEvent): boolean { |
| return ( |
| event.type === 'sandbox_boundary_decision_ack' || event.type === 'user_question_answer_ack' |
| ); |
| } |
| |
| /** |
| * Non-terminal error content never reaches the ledger: the trailing terminal |
| * event carries the failure. Exported so readers can reason about which mapped |
| * RuntimeEvents a projection will ever be asked to read. |
| */ |
| export function isNonTerminalErrorRuntimeEvent(event: RuntimeEvent): boolean { |
| return event.content?.kind === 'error' && !isTerminalRuntimeEvent(event); |
| } |
| |
| function isAtomicToolBoundaryProjection( |
| event: RuntimeEvent, |
| protocol: ToolBoundaryProtocol | undefined, |
| ): boolean { |
| if (!protocol || event.refs?.operationId === undefined) return false; |
| return event.content?.kind === 'function_call' || event.content?.kind === 'function_response'; |
| } |