| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| |
| import { |
| isPartialRuntimeEvent, |
| isTerminalRuntimeEvent, |
| runtimeEventHasModelVisibleContent, |
| TOOL_BOUNDARY_PROTOCOL_V1, |
| type RuntimeEvent, |
| type RuntimeEventFunctionCallContent, |
| type RuntimeEventFunctionResponseContent, |
| } from '@maka/core/runtime-event'; |
| import { |
| buildImmutableRuntimePrefix, |
| continuationStartEventMatchesClaim, |
| invocationMatchesClaimTarget, |
| } from '@maka/core/runtime-boundary'; |
| import type { |
| ContinuationClaimV1, |
| ImmutableRuntimePrefixV1, |
| RuntimeBoundaryCursor, |
| RuntimeBoundaryDigest, |
| } from '@maka/core/runtime-boundary'; |
| import type { RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; |
| import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; |
| import type { ContinuationClaimStateV1 } from '@maka/core/runtime-event-store'; |
| import { isDeepStrictEqual } from 'node:util'; |
| import { |
| buildContinuationReplayPlan, |
| type ContinuationReplayAdmissionRoute, |
| type ContinuationReplayPlanV1, |
| } from './continuation-replay.js'; |
| import { |
| PROVIDER_REPLAY_PROJECTION_VERSION, |
| type RuntimeEventModelReplayItem, |
| } from './model-history.js'; |
| import { resolveRuntimeRecovery, type RuntimeRecoveryResolution } from './recovery-resolver.js'; |
| import { classifyRuntimeEventTerminalFact } from './runtime-event-read-model.js'; |
| import { |
| assertHandoffClaimSource, |
| runtimeHandoffPause, |
| type RuntimeHandoffPause, |
| } from '@maka/core/runtime-handoff'; |
| |
| export type ToolOperationStatus = |
| | 'succeeded' |
| | 'failed' |
| | 'indeterminate' |
| | 'not_dispatched' |
| | 'parked' |
| | 'corruption'; |
| |
| export interface ToolOperation { |
| toolCallId: string; |
| toolName: string; |
| args: unknown; |
| status: ToolOperationStatus; |
| callRuntimeEventId: string; |
| responseRuntimeEventId?: string; |
| responseIsError?: boolean; |
| } |
| |
| export type ResumePlanDisposition = 'safe_replay' | 'blocked'; |
| |
| export type RuntimeContinuationRevalidationCode = |
| | 'continuation_claim_conflict' |
| | 'target_run_conflict' |
| | 'source_identity_changed' |
| | 'source_terminal_changed' |
| | 'source_cwd_changed' |
| | 'source_high_water_changed' |
| | 'source_ledger_identity_changed' |
| | 'source_replay_changed' |
| | 'workspace_identity_changed' |
| | 'background_operation_started' |
| | 'tool_catalog_changed' |
| | 'workspace_checkpoint_changed'; |
| |
| export class RuntimeContinuationRevalidationError extends Error { |
| readonly code: RuntimeContinuationRevalidationCode; |
| |
| constructor(code: RuntimeContinuationRevalidationCode, message: string) { |
| super(message); |
| this.name = 'RuntimeContinuationRevalidationError'; |
| this.code = code; |
| } |
| } |
| |
| export type ResumePlanDiagnosticCode = |
| | 'pending_tool_result' |
| | 'unmatched_tool_result' |
| | 'tool_name_mismatch' |
| | 'runtime_offset_mismatch' |
| | 'pending_permission' |
| | 'workspace_identity_mismatch' |
| | 'background_operation_pending' |
| | 'tool_catalog_mismatch' |
| | 'runtime_ledger_unreadable' |
| | 'terminal_repair_failed' |
| | 'workspace_cwd_mismatch' |
| | 'workspace_location_changed' |
| | 'runtime_ledger_empty' |
| | 'runtime_identity_mismatch' |
| | 'continuation_identity_reused' |
| | 'provider_resume_head_unsupported' |
| | 'provider_resume_boundary_unsupported' |
| | 'provider_replay_non_suffix_gap' |
| | 'provider_replay_unsupported' |
| | 'runtime_lineage_cycle' |
| | 'runtime_lineage_depth_exceeded' |
| | 'runtime_lineage_missing' |
| | 'runtime_lineage_start_mismatch' |
| | 'runtime_lineage_replay_mismatch' |
| | 'runtime_lineage_claim_mismatch' |
| | 'source_prefix_digest_mismatch' |
| | 'workspace_ref_missing' |
| | 'checkpoint_restore_failed' |
| | 'source_run_unreadable' |
| | 'continuation_already_exists' |
| | 'continuation_authority_unavailable' |
| | 'continuation_claim_repair_required' |
| | 'continuation_started_indeterminate' |
| | 'workspace_identity_missing' |
| | 'safety_observation_unavailable' |
| | 'resume_feature_disabled' |
| | 'resume_candidate_missing' |
| | 'tool_not_dispatched' |
| | 'tool_recovery_parked' |
| | 'tool_recovery_corruption' |
| | 'tool_ledger_corruption' |
| | 'duplicate_event_id' |
| | 'semantic_lane_conflict' |
| | 'protocol_marker_invalid'; |
| |
| export type ResumeRejectionReason = |
| | 'runtime_offset_mismatch' |
| | 'dangling_tool_state' |
| | 'pending_permission' |
| | 'workspace_identity_mismatch' |
| | 'background_operation_pending' |
| | 'tool_catalog_mismatch' |
| | 'runtime_ledger_unreadable' |
| | 'terminal_repair_failed' |
| | 'workspace_cwd_mismatch' |
| | 'runtime_ledger_empty' |
| | 'runtime_identity_mismatch' |
| | 'continuation_identity_reused' |
| | 'provider_resume_head_unsupported' |
| | 'provider_resume_boundary_unsupported' |
| | 'provider_replay_non_suffix_gap' |
| | 'provider_replay_unsupported' |
| | 'runtime_lineage_cycle' |
| | 'runtime_lineage_depth_exceeded' |
| | 'runtime_lineage_missing' |
| | 'runtime_lineage_start_mismatch' |
| | 'runtime_lineage_replay_mismatch' |
| | 'runtime_lineage_claim_mismatch' |
| | 'source_prefix_digest_mismatch' |
| | 'workspace_ref_missing' |
| | 'checkpoint_restore_failed' |
| | 'source_run_unreadable' |
| | 'continuation_already_exists' |
| | 'continuation_authority_unavailable' |
| | 'continuation_claim_repair_required' |
| | 'continuation_started_indeterminate' |
| | 'workspace_identity_missing' |
| | 'safety_observation_unavailable' |
| | 'resume_feature_disabled' |
| | 'resume_candidate_missing'; |
| |
| export interface ResumePlanDiagnostic { |
| code: ResumePlanDiagnosticCode; |
| message: string; |
| eventId?: string; |
| toolCallId?: string; |
| toolName?: string; |
| detail?: Record<string, unknown>; |
| } |
| |
| export interface ResumePlan { |
| disposition: ResumePlanDisposition; |
| operations: ToolOperation[]; |
| diagnostics: ResumePlanDiagnostic[]; |
| rejectionReasons: ResumeRejectionReason[]; |
| requiresVerification: boolean; |
| sourceRuntimeEventHighWater: number; |
| directive?: string; |
| runtimeEvents: RuntimeEvent[]; |
| replayRuntimeEvents: RuntimeEvent[]; |
| } |
| |
| export interface BuildResumePlanOptions { |
| expectedRuntimeEventHighWater?: number; |
| } |
| |
| export type RuntimeResumeFailpointId = |
| | 'P0' |
| | 'P1' |
| | 'P2' |
| | 'P3' |
| | 'P4' |
| | 'P5' |
| | 'P6' |
| | 'P7' |
| | 'P8' |
| | 'P9' |
| | 'P10' |
| | 'P11'; |
| |
| export type RuntimeResumeCommittedPrefix = |
| | 'before_function_call' |
| | 'after_function_call' |
| | 'after_function_response' |
| | 'after_terminal_event'; |
| |
| export interface RuntimeResumeFailpointSpec { |
| id: RuntimeResumeFailpointId; |
| boundary: string; |
| /** Last fully committed RuntimeEvent prefix that Phase 0 may inspect. */ |
| committedPrefix: RuntimeResumeCommittedPrefix; |
| } |
| |
| /** |
| * Stable crash-injection catalog owned by the Phase 0 process harness. |
| * Later phases may map these labels to richer boundaries, but this catalog |
| * only reasons about the last fully committed RuntimeEvent prefix. |
| */ |
| export const RUNTIME_RESUME_FAILPOINTS = [ |
| { id: 'P0', boundary: 'before tool preparation (T1)', committedPrefix: 'before_function_call' }, |
| { |
| id: 'P1', |
| boundary: 'function_call committed before prepared journal', |
| committedPrefix: 'after_function_call', |
| }, |
| { |
| id: 'P2', |
| boundary: 'prepared journal committed before implementation', |
| committedPrefix: 'after_function_call', |
| }, |
| { id: 'P3', boundary: 'tool implementation in progress', committedPrefix: 'after_function_call' }, |
| { |
| id: 'P4', |
| boundary: 'side effect completed before outcome transaction (T2)', |
| committedPrefix: 'after_function_call', |
| }, |
| { |
| id: 'P5', |
| boundary: 'function_response committed before outcome journal', |
| committedPrefix: 'after_function_response', |
| }, |
| { |
| id: 'P6', |
| boundary: 'outcome transaction committed before model result delivery', |
| committedPrefix: 'after_function_response', |
| }, |
| { |
| id: 'P7', |
| boundary: 'tool result delivered before the next provider step', |
| committedPrefix: 'after_function_response', |
| }, |
| { |
| id: 'P8', |
| boundary: 'terminal RuntimeEvent commit', |
| committedPrefix: 'after_function_response', |
| }, |
| { id: 'P9', boundary: 'terminal run header commit', committedPrefix: 'after_terminal_event' }, |
| { id: 'P10', boundary: 'recovery decision commit', committedPrefix: 'after_terminal_event' }, |
| { id: 'P11', boundary: 'continuation run creation', committedPrefix: 'after_terminal_event' }, |
| ] as const satisfies readonly RuntimeResumeFailpointSpec[]; |
| |
| export interface ContinuationIdentity { |
| invocationId: string; |
| runId: string; |
| turnId: string; |
| } |
| |
| export interface SafeBoundaryContinuationFacts { |
| handoffPause?: RuntimeHandoffPause; |
| ledgerReadable: boolean; |
| terminalRepairSucceeded: boolean; |
| sourceCwd: string; |
| currentCwd: string; |
| sourceWorkspaceIdentity: string; |
| currentWorkspaceIdentity: string; |
| backgroundOperationsSettled: boolean; |
| availableToolNames: readonly string[]; |
| continuationIdentity: ContinuationIdentity; |
| continuationClaimId?: string; |
| /** User-anchored replay prefix inherited from continuation ancestors. */ |
| priorRuntimeContext?: readonly RuntimeEvent[]; |
| /** Versioned, segment-scoped provider replay built from immutable prefixes. */ |
| continuationReplayPlan?: ContinuationReplayPlanV1; |
| expectedRuntimeEventHighWater?: number; |
| workspaceCheckpoint?: { |
| ref?: string; |
| restored: boolean; |
| runtimeEventHighWater: number; |
| }; |
| } |
| |
| export interface RuntimeContinuation { |
| /** Present only for a sealed physical handoff, never a manual resume. */ |
| handoffRootRunId?: string; |
| handoffRemainingSteps?: number | null; |
| sessionId: string; |
| invocationId: string; |
| runId: string; |
| turnId: string; |
| sourceInvocationId: string; |
| sourceRunId: string; |
| sourceTurnId: string; |
| sourceRuntimeEventHighWater: number; |
| /** Proposed durable claim id; only execution may acquire it. */ |
| claimId?: string; |
| /** Replay events owned by the immediate source run. */ |
| sourceRuntimeContext?: RuntimeEvent[]; |
| /** Full user-anchored provider history, including continuation ancestors. */ |
| runtimeContext: RuntimeEvent[]; |
| /** Composite immutable ledger boundary used to build runtimeContext. */ |
| boundary?: RuntimeBoundaryCursor; |
| /** Identity of the exact provider-facing replay projection. */ |
| providerReplayDigest?: RuntimeBoundaryDigest; |
| providerProjectionVersion?: typeof PROVIDER_REPLAY_PROJECTION_VERSION; |
| safetySnapshot: RuntimeContinuationSafetySnapshot; |
| } |
| |
| export interface RuntimeContinuationSafetySnapshot { |
| workspaceIdentity: string; |
| backgroundOperationsSettled: true; |
| availableToolNames: string[]; |
| workspaceCheckpoint?: { |
| ref: string; |
| runtimeEventHighWater: number; |
| }; |
| } |
| |
| /** Keep logical ownership while independently checking the execution's current configuration. */ |
| export function preserveHandoffOpening( |
| continuation: RuntimeContinuation, |
| computed: RuntimeEventInvocationOpenedContent, |
| source: RuntimeEventInvocationOpenedContent | undefined, |
| ): RuntimeEventInvocationOpenedContent { |
| if (continuation.handoffRootRunId === undefined) return computed; |
| if ( |
| source?.kind !== 'invocation_opened' || |
| !continuation.claimId || |
| !continuation.boundary || |
| !isDeepStrictEqual(source.configuration, computed.configuration) || |
| continuation.handoffRootRunId !== |
| (source.source.kind === 'handoff' ? source.source.rootRunId : continuation.sourceRunId) |
| ) { |
| throw new RuntimeContinuationRevalidationError( |
| 'source_identity_changed', |
| 'Handoff execution cannot preserve its source configuration and logical authority', |
| ); |
| } |
| const { lineage: _computedLineage, ...opening } = computed; |
| return { |
| ...opening, |
| root: source.root, |
| ...(source.lineage ? { lineage: source.lineage } : {}), |
| source: { |
| kind: 'handoff', |
| rootRunId: continuation.handoffRootRunId, |
| sourceInvocationId: continuation.sourceInvocationId, |
| sourceRunId: continuation.sourceRunId, |
| sourceTurnId: continuation.sourceTurnId, |
| sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater, |
| claimId: continuation.claimId, |
| boundaryDigest: continuation.boundary.manifestDigest, |
| }, |
| }; |
| } |
| |
| export interface RuntimeContinuationSafetyObservation { |
| workspaceIdentity: string; |
| /** Current location is diagnostic only and never participates in identity. */ |
| workspacePath?: string; |
| backgroundOperationsSettled: boolean; |
| availableToolNames: readonly string[]; |
| workspaceCheckpoint?: { |
| ref?: string; |
| restored: boolean; |
| runtimeEventHighWater: number; |
| }; |
| } |
| |
| export interface SafeBoundaryContinuationPlan { |
| disposition: 'continue' | 'park'; |
| rejectionReasons: ResumeRejectionReason[]; |
| diagnostics: ResumePlanDiagnostic[]; |
| continuation?: RuntimeContinuation; |
| } |
| |
| export interface RuntimeContinuationPlannerInput { |
| purpose?: 'handoff'; |
| sessionId: string; |
| sourceRunId: string; |
| admissionRoute: ContinuationReplayAdmissionRoute; |
| currentCwd: string; |
| sourceWorkspaceIdentity: string; |
| currentWorkspaceIdentity: string; |
| backgroundOperationsSettled: boolean; |
| availableToolNames: readonly string[]; |
| expectedRuntimeEventHighWater?: number; |
| workspaceCheckpoint?: SafeBoundaryContinuationFacts['workspaceCheckpoint']; |
| } |
| |
| export interface RuntimeContinuationPlannerDeps { |
| readSourceInvocation(sessionId: string, runId: string): Promise<RuntimeInvocationRecord>; |
| readImmutableRuntimePrefix(input: { |
| sessionId: string; |
| runId: string; |
| upToEventSeq?: number; |
| }): Promise<ImmutableRuntimePrefixV1>; |
| readContinuationClaimStateByBoundary?( |
| boundaryDigest: RuntimeBoundaryDigest, |
| ): Promise<ContinuationClaimStateV1 | undefined>; |
| findExistingContinuation?( |
| sessionId: string, |
| sourceRunId: string, |
| sourceRuntimeEventHighWater: number, |
| ): Promise<{ runId: string } | undefined>; |
| newId(): string; |
| } |
| |
| export class RuntimeContinuationPlanner { |
| constructor(private readonly deps: RuntimeContinuationPlannerDeps) {} |
| |
| async plan(input: RuntimeContinuationPlannerInput): Promise<SafeBoundaryContinuationPlan> { |
| return this.planBoundary(input); |
| } |
| |
| /** A candidate is never execution authority and never escapes as a continuation. */ |
| async previewHandoff( |
| input: RuntimeContinuationPlannerInput, |
| seal: RuntimeEvent, |
| ): Promise<SafeBoundaryContinuationPlan> { |
| const { continuation: _continuation, ...assessment } = await this.planBoundary( |
| { ...input, purpose: 'handoff' }, |
| seal, |
| ); |
| return assessment; |
| } |
| |
| private async planBoundary( |
| input: RuntimeContinuationPlannerInput, |
| preview?: RuntimeEvent, |
| ): Promise<SafeBoundaryContinuationPlan> { |
| let sourceInvocation: RuntimeInvocationRecord; |
| try { |
| sourceInvocation = await this.deps.readSourceInvocation(input.sessionId, input.sourceRunId); |
| } catch { |
| return parkedPlan('source_run_unreadable', 'source AgentRun could not be read'); |
| } |
| |
| let prefixes: [ImmutableRuntimePrefixV1, ...ImmutableRuntimePrefixV1[]]; |
| try { |
| prefixes = await this.readLineagePrefixes( |
| input.sessionId, |
| input.sourceRunId, |
| sourceInvocation.opening, |
| input.admissionRoute.invocations, |
| ); |
| } catch (error) { |
| if (error instanceof RuntimeLineageError) { |
| return parkedPlan(error.code, error.message); |
| } |
| return parkedPlan( |
| 'runtime_ledger_unreadable', |
| 'RuntimeEvent ledger could not be read reliably', |
| ); |
| } |
| if (preview) { |
| const source = prefixes.at(-1)!; |
| if (!runtimeHandoffPause(preview) || source.events.some(isTerminalRuntimeEvent)) { |
| return parkedPlan( |
| 'runtime_identity_mismatch', |
| 'Handoff preview requires an unsealed live source', |
| ); |
| } |
| prefixes[prefixes.length - 1] = buildImmutableRuntimePrefix( |
| source.identity, |
| [...source.events, preview].map((event, index) => ({ eventSeq: index + 1, event })), |
| ); |
| } |
| const sourcePrefix = prefixes.at(-1)!; |
| const events = [...sourcePrefix.events]; |
| const pause = events.at(-1) && runtimeHandoffPause(events.at(-1)!); |
| if ((input.purpose === 'handoff') !== Boolean(pause)) { |
| return parkedPlan( |
| 'runtime_identity_mismatch', |
| pause |
| ? 'This source is reserved for its sealed handoff successor' |
| : 'A physical handoff requires a durable pause seal', |
| ); |
| } |
| if ( |
| sourcePrefix.identity.sessionId !== input.sessionId || |
| sourcePrefix.identity.runId !== input.sourceRunId |
| ) { |
| return parkedPlan( |
| 'runtime_identity_mismatch', |
| 'RuntimeEvent ledger does not belong to the requested source run', |
| ); |
| } |
| const replay = buildContinuationReplayPlan({ |
| prefixes, |
| providerProjectionVersion: PROVIDER_REPLAY_PROJECTION_VERSION, |
| admissionRoute: input.admissionRoute, |
| }); |
| if (replay.kind === 'blocked') { |
| const reason = |
| replay.reason === 'provider_replay_non_suffix_gap' |
| ? 'provider_replay_non_suffix_gap' |
| : replay.reason === 'provider_replay_unsupported' |
| ? 'provider_replay_unsupported' |
| : 'runtime_ledger_unreadable'; |
| return parkedPlan( |
| reason, |
| `continuation replay segment ${replay.segmentIndex} is not replayable: ${replay.reason}`, |
| ); |
| } |
| let durableClaimState: ContinuationClaimStateV1 | undefined; |
| try { |
| durableClaimState = await this.deps.readContinuationClaimStateByBoundary?.( |
| replay.plan.boundary.manifestDigest, |
| ); |
| } catch { |
| return parkedPlan( |
| 'continuation_authority_unavailable', |
| 'durable continuation authority is unavailable', |
| ); |
| } |
| if (durableClaimState) { |
| const claim = durableClaimState.claim; |
| if ( |
| claim.boundaryDigest !== replay.plan.boundary.manifestDigest || |
| !isDeepStrictEqual(claim.boundary, replay.plan.boundary) || |
| claim.providerProjectionVersion !== replay.plan.providerProjectionVersion || |
| claim.providerReplayDigest !== replay.plan.providerReplayDigest |
| ) { |
| return parkedPlan( |
| 'continuation_claim_repair_required', |
| 'durable continuation claim does not authenticate the current provider replay', |
| { |
| continuationClaimId: claim.claimId, |
| continuationRunId: claim.target.runId, |
| }, |
| ); |
| } |
| return this.classifyExistingClaim(input.sessionId, durableClaimState); |
| } |
| const existingContinuation = await this.deps.findExistingContinuation?.( |
| input.sessionId, |
| input.sourceRunId, |
| sourcePrefix.position.lastEventSeq, |
| ); |
| if (existingContinuation) { |
| return parkedPlan( |
| 'continuation_already_exists', |
| 'source run already has a continuation child', |
| { continuationRunId: existingContinuation.runId }, |
| ); |
| } |
| |
| return buildSafeBoundaryContinuationPlan(events, { |
| ledgerReadable: true, |
| terminalRepairSucceeded: hasConsistentTerminalBoundary(events), |
| sourceCwd: sourceInvocation.opening.configuration.cwd, |
| currentCwd: input.currentCwd, |
| sourceWorkspaceIdentity: input.sourceWorkspaceIdentity, |
| currentWorkspaceIdentity: input.currentWorkspaceIdentity, |
| backgroundOperationsSettled: input.backgroundOperationsSettled, |
| availableToolNames: input.availableToolNames, |
| // One physical execution attempt, one identity. Run and invocation are |
| // the same value at every mint site so the opening fact can be joined |
| // either way while the two names are still being retired. |
| continuationIdentity: (() => { |
| if (pause) |
| return { |
| invocationId: pause.successorInvocationId, |
| runId: pause.successorRunId, |
| turnId: sourcePrefix.identity.turnId, |
| }; |
| const invocationId = this.deps.newId(); |
| return { invocationId, runId: invocationId, turnId: this.deps.newId() }; |
| })(), |
| continuationClaimId: pause?.claimId ?? this.deps.newId(), |
| ...(pause ? { handoffPause: pause } : {}), |
| continuationReplayPlan: replay.plan, |
| ...(input.expectedRuntimeEventHighWater !== undefined |
| ? { expectedRuntimeEventHighWater: input.expectedRuntimeEventHighWater } |
| : {}), |
| ...(input.workspaceCheckpoint !== undefined |
| ? { workspaceCheckpoint: input.workspaceCheckpoint } |
| : {}), |
| }); |
| } |
| |
| private async classifyExistingClaim( |
| sessionId: string, |
| state: ContinuationClaimStateV1, |
| ): Promise<SafeBoundaryContinuationPlan> { |
| const { claim } = state; |
| const detail = { |
| continuationClaimId: claim.claimId, |
| continuationRunId: claim.target.runId, |
| }; |
| let targetInvocation: RuntimeInvocationRecord; |
| try { |
| targetInvocation = await this.deps.readSourceInvocation(sessionId, claim.target.runId); |
| } catch { |
| return parkedPlan( |
| 'continuation_claim_repair_required', |
| 'durable continuation claim exists but its target Run is missing', |
| detail, |
| ); |
| } |
| if (!invocationMatchesClaimTarget(targetInvocation, claim)) { |
| return parkedPlan( |
| 'continuation_claim_repair_required', |
| 'durable continuation claim target Run identity does not match its claim', |
| detail, |
| ); |
| } |
| let prefix: ImmutableRuntimePrefixV1; |
| try { |
| prefix = await this.deps.readImmutableRuntimePrefix({ |
| sessionId, |
| runId: claim.target.runId, |
| }); |
| } catch { |
| return parkedPlan( |
| 'continuation_claim_repair_required', |
| 'durable continuation claim target has no committed continuation-start', |
| detail, |
| ); |
| } |
| const start = prefix.events[0]?.actions?.continuationStart; |
| if ( |
| !state.startEventId || |
| prefix.events[0]?.id !== state.startEventId || |
| !continuationStartEventMatchesClaim(prefix.events[0], claim, state.startKind) |
| ) { |
| return parkedPlan( |
| 'continuation_claim_repair_required', |
| 'durable continuation claim target is missing a matching continuation-start', |
| detail, |
| ); |
| } |
| const terminalClassification = classifyRuntimeEventTerminalFact( |
| targetInvocation, |
| prefix.events, |
| ); |
| const terminal = prefix.events.find(isTerminalRuntimeEvent); |
| if (terminal && prefix.events.at(-1)?.id !== terminal.id) { |
| return parkedPlan( |
| 'continuation_claim_repair_required', |
| 'continuation target has immutable RuntimeEvents after its terminal fact', |
| detail, |
| ); |
| } |
| if (terminal && !terminalClassification.fact) { |
| return parkedPlan( |
| 'continuation_claim_repair_required', |
| 'continuation target has an invalid or ambiguous terminal fact', |
| detail, |
| ); |
| } |
| if (terminalClassification.fact) { |
| return parkedPlan( |
| 'continuation_already_exists', |
| 'source boundary already has a terminal continuation', |
| detail, |
| ); |
| } |
| if (start) { |
| return parkedPlan( |
| 'continuation_started_indeterminate', |
| 'continuation-start is durable but the target Run is not terminal', |
| detail, |
| ); |
| } |
| return parkedPlan( |
| 'continuation_claim_repair_required', |
| 'continuation claim is incomplete', |
| detail, |
| ); |
| } |
| |
| private async readLineagePrefixes( |
| sessionId: string, |
| sourceRunId: string, |
| sourceOpening: RuntimeEventInvocationOpenedContent, |
| invocations: readonly RuntimeInvocationRecord[], |
| ): Promise<[ImmutableRuntimePrefixV1, ...ImmutableRuntimePrefixV1[]]> { |
| const immediate = await this.deps.readImmutableRuntimePrefix({ |
| sessionId, |
| runId: sourceRunId, |
| }); |
| const segments: ImmutableRuntimePrefixV1[] = [immediate]; |
| const seen = new Set<string>([sourceRunId]); |
| const claimedEdges: Array<{ |
| childRunId: string; |
| childInvocation: RuntimeInvocationRecord; |
| startEvent: RuntimeEvent; |
| startKind: 'runtime_admission' | 'claim_repair'; |
| claimId: string; |
| boundaryDigest: RuntimeBoundaryDigest; |
| providerProjectionVersion: 1 | typeof PROVIDER_REPLAY_PROJECTION_VERSION; |
| providerReplayDigest: RuntimeBoundaryDigest; |
| }> = []; |
| let childInvocation: RuntimeInvocationRecord = { |
| sessionId, |
| invocationId: immediate.identity.invocationId, |
| runId: sourceRunId, |
| turnId: immediate.identity.turnId, |
| openedAt: 0, |
| opening: sourceOpening, |
| }; |
| let childRunId = sourceRunId; |
| let childPrefix = immediate; |
| let depth = 1; |
| while (true) { |
| const opened = childInvocation.opening.source; |
| const current = opened.kind !== 'fresh' ? opened : undefined; |
| const start = childPrefix.events[0]?.actions?.continuationStart; |
| // A migrated opening keeps the lineage edge but names no claim, so only |
| // an edge that names one can be authenticated against a durable claim. |
| const claimed = |
| current?.claimId !== undefined && current.boundaryDigest !== undefined |
| ? { ...current, claimId: current.claimId, boundaryDigest: current.boundaryDigest } |
| : undefined; |
| if (start && !claimed) { |
| throw new RuntimeLineageError( |
| 'runtime_lineage_start_mismatch', |
| `canonical continuation-start cannot be downgraded to legacy lineage for ${childRunId}`, |
| ); |
| } |
| if (claimed) { |
| if ( |
| !start || |
| start.claimId !== claimed.claimId || |
| start.boundaryDigest !== claimed.boundaryDigest || |
| start.immediateSource.sessionId !== sessionId || |
| start.immediateSource.invocationId !== claimed.sourceInvocationId || |
| start.immediateSource.runId !== claimed.sourceRunId || |
| start.immediateSource.turnId !== claimed.sourceTurnId || |
| start.immediateSource.highWater !== claimed.sourceRuntimeEventHighWater |
| ) { |
| throw new RuntimeLineageError( |
| 'runtime_lineage_start_mismatch', |
| `continuation-start does not authenticate lineage edge for ${childRunId}`, |
| ); |
| } |
| claimedEdges.push({ |
| childRunId, |
| childInvocation, |
| startEvent: childPrefix.events[0]!, |
| startKind: start.provenance, |
| claimId: start.claimId, |
| boundaryDigest: claimed.boundaryDigest, |
| providerProjectionVersion: start.providerProjectionVersion, |
| providerReplayDigest: start.providerReplayDigest, |
| }); |
| } |
| if (!current) break; |
| if (seen.has(current.sourceRunId)) { |
| throw new RuntimeLineageError( |
| 'runtime_lineage_cycle', |
| 'continuation source lineage contains a cycle', |
| ); |
| } |
| if (depth >= 64) { |
| throw new RuntimeLineageError( |
| 'runtime_lineage_depth_exceeded', |
| 'continuation source lineage exceeds the maximum depth of 64', |
| ); |
| } |
| seen.add(current.sourceRunId); |
| let invocation: RuntimeInvocationRecord; |
| let prefix: ImmutableRuntimePrefixV1; |
| try { |
| [invocation, prefix] = await Promise.all([ |
| this.deps.readSourceInvocation(sessionId, current.sourceRunId), |
| this.deps.readImmutableRuntimePrefix({ |
| sessionId, |
| runId: current.sourceRunId, |
| upToEventSeq: current.sourceRuntimeEventHighWater, |
| }), |
| ]); |
| } catch { |
| throw new RuntimeLineageError( |
| 'runtime_lineage_missing', |
| `continuation ancestor ${current.sourceRunId} is unavailable`, |
| ); |
| } |
| if ( |
| prefix.identity.sessionId !== sessionId || |
| prefix.identity.invocationId !== current.sourceInvocationId || |
| prefix.identity.runId !== current.sourceRunId || |
| prefix.identity.turnId !== current.sourceTurnId || |
| prefix.position.lastEventSeq !== current.sourceRuntimeEventHighWater |
| ) { |
| throw new RuntimeLineageError( |
| 'runtime_identity_mismatch', |
| `continuation ancestor ${current.sourceRunId} identity does not match its lineage edge`, |
| ); |
| } |
| // The child's continuation-start is what froze the ancestor's prefix, so |
| // it is also the only record that can say the prefix has since changed. |
| if (start && start.immediateSource.prefixDigest !== prefix.prefixDigest) { |
| throw new RuntimeLineageError( |
| 'source_prefix_digest_mismatch', |
| `continuation ancestor ${current.sourceRunId} prefix digest changed`, |
| ); |
| } |
| segments.unshift(prefix); |
| childPrefix = prefix; |
| childInvocation = invocation; |
| childRunId = current.sourceRunId; |
| depth += 1; |
| } |
| for (const edge of claimedEdges) { |
| const childIndex = segments.findIndex((prefix) => prefix.identity.runId === edge.childRunId); |
| if (childIndex <= 0) { |
| throw new RuntimeLineageError( |
| 'runtime_lineage_missing', |
| `continuation lineage edge for ${edge.childRunId} is incomplete`, |
| ); |
| } |
| if (!this.deps.readContinuationClaimStateByBoundary) { |
| throw new RuntimeLineageError( |
| 'continuation_authority_unavailable', |
| `durable continuation authority is unavailable for ${edge.childRunId}`, |
| ); |
| } |
| let state: ContinuationClaimStateV1 | undefined; |
| try { |
| state = await this.deps.readContinuationClaimStateByBoundary(edge.boundaryDigest); |
| } catch { |
| throw new RuntimeLineageError( |
| 'continuation_authority_unavailable', |
| `durable continuation claim could not be read for ${edge.childRunId}`, |
| ); |
| } |
| if ( |
| !state || |
| state.claim.claimId !== edge.claimId || |
| state.claim.boundaryDigest !== edge.boundaryDigest || |
| state.startEventId !== edge.startEvent.id || |
| state.startKind !== edge.startKind || |
| !invocationMatchesClaimTarget(edge.childInvocation, state.claim) || |
| !continuationStartEventMatchesClaim(edge.startEvent, state.claim, state.startKind) |
| ) { |
| throw new RuntimeLineageError( |
| 'runtime_lineage_claim_mismatch', |
| `durable continuation claim does not authenticate ${edge.childRunId}`, |
| ); |
| } |
| try { |
| assertHandoffClaimSource(state.claim, segments[childIndex - 1]!); |
| } catch { |
| throw new RuntimeLineageError( |
| 'runtime_lineage_claim_mismatch', |
| `handoff source does not authenticate ${edge.childRunId}`, |
| ); |
| } |
| if (edge.providerProjectionVersion !== PROVIDER_REPLAY_PROJECTION_VERSION) { |
| throw new RuntimeLineageError( |
| 'runtime_lineage_replay_mismatch', |
| `continuation provider replay version is unsupported for ${edge.childRunId}`, |
| ); |
| } |
| const edgeReplay = buildContinuationReplayPlan({ |
| prefixes: segments.slice(0, childIndex) as [ |
| ImmutableRuntimePrefixV1, |
| ...ImmutableRuntimePrefixV1[], |
| ], |
| providerProjectionVersion: edge.providerProjectionVersion, |
| admissionRoute: { |
| invocations, |
| targetProviderStateIdentity: |
| state.claim.targetOpening.route.provenance === 'runtime' |
| ? state.claim.targetOpening.route.providerStateIdentity |
| : undefined, |
| targetModelId: state.claim.targetOpening.route.modelId, |
| }, |
| }); |
| if ( |
| edgeReplay.kind !== 'replayable' || |
| edgeReplay.plan.boundary.manifestDigest !== edge.boundaryDigest || |
| edgeReplay.plan.providerReplayDigest !== edge.providerReplayDigest |
| ) { |
| throw new RuntimeLineageError( |
| 'runtime_lineage_replay_mismatch', |
| `continuation provider replay changed before ${edge.childRunId}`, |
| ); |
| } |
| } |
| return segments as [ImmutableRuntimePrefixV1, ...ImmutableRuntimePrefixV1[]]; |
| } |
| } |
| |
| class RuntimeLineageError extends Error { |
| constructor( |
| readonly code: |
| | 'runtime_lineage_cycle' |
| | 'runtime_lineage_depth_exceeded' |
| | 'runtime_lineage_missing' |
| | 'runtime_identity_mismatch' |
| | 'runtime_lineage_start_mismatch' |
| | 'runtime_lineage_replay_mismatch' |
| | 'runtime_lineage_claim_mismatch' |
| | 'continuation_authority_unavailable' |
| | 'source_prefix_digest_mismatch', |
| message: string, |
| ) { |
| super(message); |
| this.name = 'RuntimeLineageError'; |
| } |
| } |
| |
| /** |
| * Has this run ended, with the terminal event last where a sealed run must |
| * leave it? There is nothing else to agree with: the events are the run. |
| */ |
| function hasConsistentTerminalBoundary(events: readonly RuntimeEvent[]): boolean { |
| const last = events.at(-1); |
| return last !== undefined && isTerminalRuntimeEvent(last); |
| } |
| |
| export const INDETERMINATE_TOOL_RESULT_DIRECTIVE = [ |
| 'Tool execution was interrupted before a matching committed tool result was found.', |
| 'The side effects may or may not have occurred.', |
| 'Do not retry the tool call immediately.', |
| 'Use read-only inspection tools to verify the current state before deciding the next step.', |
| ].join(' '); |
| |
| export function projectToolOperationsFromRuntimeEvents( |
| events: readonly RuntimeEvent[], |
| ): ToolOperation[] { |
| return projectToolOperations(events, resolveRuntimeRecovery(events)); |
| } |
| |
| function projectToolOperations( |
| events: readonly RuntimeEvent[], |
| recovery: RuntimeRecoveryResolution, |
| ): ToolOperation[] { |
| const callsByEventId = new Map( |
| events.flatMap((event) => |
| event.content?.kind === 'function_call' && event.modelVisibility !== 'hidden' |
| ? [[event.id, event.content] as const] |
| : [], |
| ), |
| ); |
| return recovery.decisions.flatMap((decision) => { |
| if (!decision.callRuntimeEventId) return []; |
| const call = callsByEventId.get(decision.callRuntimeEventId); |
| if (!call) return []; |
| const status: ToolOperationStatus = |
| decision.status === 'completed' |
| ? decision.responseIsError |
| ? 'failed' |
| : 'succeeded' |
| : decision.status === 'definitely_not_dispatched' |
| ? 'not_dispatched' |
| : decision.status; |
| return [ |
| { |
| toolCallId: decision.toolCallId, |
| toolName: call.name, |
| args: call.args, |
| status, |
| callRuntimeEventId: decision.callRuntimeEventId, |
| ...(decision.responseRuntimeEventId |
| ? { responseRuntimeEventId: decision.responseRuntimeEventId } |
| : {}), |
| ...(decision.responseIsError !== undefined |
| ? { responseIsError: decision.responseIsError } |
| : {}), |
| }, |
| ]; |
| }); |
| } |
| |
| export function buildResumePlanFromRuntimeEvents( |
| events: readonly RuntimeEvent[], |
| options: BuildResumePlanOptions = {}, |
| ): ResumePlan { |
| const recovery = resolveRuntimeRecovery(events); |
| const operations = projectToolOperations(events, recovery); |
| const sourceRuntimeEventHighWater = events.length; |
| const diagnostics = collectResumeDiagnostics(events, operations, options, recovery); |
| const rejectionReasons = deriveRejectionReasons(diagnostics); |
| // Hidden nested calls stay out of the model-facing operation projection, but |
| // an unresolved one still represents a possible side effect and must block |
| // automatic replay of the enclosing execution. |
| const requiresVerification = |
| operations.some((operation) => operation.status === 'indeterminate') || |
| hasHiddenIndeterminateOperation(events, recovery); |
| const disposition: ResumePlanDisposition = |
| rejectionReasons.length === 0 && !requiresVerification && !recovery.hasCorruption |
| ? 'safe_replay' |
| : 'blocked'; |
| |
| return { |
| disposition, |
| operations, |
| diagnostics, |
| rejectionReasons, |
| requiresVerification, |
| sourceRuntimeEventHighWater, |
| ...(requiresVerification ? { directive: INDETERMINATE_TOOL_RESULT_DIRECTIVE } : {}), |
| runtimeEvents: [...events], |
| replayRuntimeEvents: buildResumeReplayRuntimeEvents(events), |
| }; |
| } |
| |
| export function buildResumeReplayRuntimeEvents(events: readonly RuntimeEvent[]): RuntimeEvent[] { |
| const pairedCallIds = collectPairedCallIds(events); |
| const replayEvents: RuntimeEvent[] = []; |
| |
| for (const event of events) { |
| if (isPartialRuntimeEvent(event)) continue; |
| if (event.modelVisibility === 'hidden') continue; |
| const content = event.content; |
| if (!content) { |
| replayEvents.push(event); |
| continue; |
| } |
| if (content.kind === 'function_call') { |
| if (pairedCallIds.has(content.id)) replayEvents.push(event); |
| continue; |
| } |
| if (content.kind === 'function_response') { |
| if (pairedCallIds.has(content.id)) replayEvents.push(event); |
| continue; |
| } |
| replayEvents.push(event); |
| } |
| |
| return replayEvents; |
| } |
| |
| export function buildSafeBoundaryContinuationPlan( |
| events: readonly RuntimeEvent[], |
| facts: SafeBoundaryContinuationFacts, |
| ): SafeBoundaryContinuationPlan { |
| const expectedRuntimeEventHighWater = |
| facts.workspaceCheckpoint?.runtimeEventHighWater ?? facts.expectedRuntimeEventHighWater; |
| const compositeReplay = facts.continuationReplayPlan; |
| const legacyReplayPlan = compositeReplay |
| ? undefined |
| : buildResumePlanFromRuntimeEvents(events, { |
| ...(expectedRuntimeEventHighWater !== undefined ? { expectedRuntimeEventHighWater } : {}), |
| }); |
| const phaseOneDiagnostics = collectPendingPermissionDiagnostics(events); |
| const phaseOneRejectionReasons: ResumeRejectionReason[] = []; |
| if (phaseOneDiagnostics.length > 0) phaseOneRejectionReasons.push('pending_permission'); |
| if ( |
| compositeReplay && |
| expectedRuntimeEventHighWater !== undefined && |
| compositeReplay.segments.at(-1)?.boundary.position.lastEventSeq !== |
| expectedRuntimeEventHighWater |
| ) { |
| phaseOneDiagnostics.push({ |
| code: 'runtime_offset_mismatch', |
| message: 'persisted runtime high-water does not match the immutable continuation boundary', |
| detail: { |
| expected: expectedRuntimeEventHighWater, |
| actual: compositeReplay.segments.at(-1)?.boundary.position.lastEventSeq ?? null, |
| }, |
| }); |
| phaseOneRejectionReasons.push('runtime_offset_mismatch'); |
| } |
| const source = events[0]; |
| if (!source) { |
| phaseOneDiagnostics.push({ |
| code: 'runtime_ledger_empty', |
| message: 'safe-boundary continuation requires at least one RuntimeEvent', |
| }); |
| phaseOneRejectionReasons.push('runtime_ledger_empty'); |
| } else { |
| const mismatchedEvent = events.find( |
| (event) => |
| event.sessionId !== source.sessionId || |
| event.invocationId !== source.invocationId || |
| event.runId !== source.runId || |
| event.turnId !== source.turnId, |
| ); |
| if (mismatchedEvent) { |
| phaseOneDiagnostics.push({ |
| code: 'runtime_identity_mismatch', |
| message: 'RuntimeEvent ledger contains more than one source execution identity', |
| eventId: mismatchedEvent.id, |
| }); |
| phaseOneRejectionReasons.push('runtime_identity_mismatch'); |
| } |
| if ( |
| facts.continuationIdentity.invocationId === source.invocationId || |
| facts.continuationIdentity.runId === source.runId || |
| (facts.handoffPause |
| ? facts.continuationIdentity.turnId !== source.turnId || |
| facts.continuationIdentity.runId !== facts.handoffPause.successorRunId || |
| facts.continuationIdentity.invocationId !== facts.handoffPause.successorInvocationId || |
| facts.continuationClaimId !== facts.handoffPause.claimId || |
| !isDeepStrictEqual(events.at(-1)?.actions?.handoffPause, facts.handoffPause) |
| : facts.continuationIdentity.turnId === source.turnId) |
| ) { |
| phaseOneDiagnostics.push({ |
| code: 'continuation_identity_reused', |
| message: 'continuation must use fresh invocation, run, and turn identities', |
| }); |
| phaseOneRejectionReasons.push('continuation_identity_reused'); |
| } |
| } |
| if (!facts.ledgerReadable) { |
| phaseOneDiagnostics.push({ |
| code: 'runtime_ledger_unreadable', |
| message: 'RuntimeEvent ledger could not be read reliably', |
| }); |
| phaseOneRejectionReasons.push('runtime_ledger_unreadable'); |
| } |
| if (!facts.terminalRepairSucceeded) { |
| phaseOneDiagnostics.push({ |
| code: 'terminal_repair_failed', |
| message: 'source run terminal repair did not complete successfully', |
| }); |
| phaseOneRejectionReasons.push('terminal_repair_failed'); |
| } |
| if (normalizeCwd(facts.sourceCwd) !== normalizeCwd(facts.currentCwd)) { |
| phaseOneDiagnostics.push({ |
| code: 'workspace_location_changed', |
| message: 'workspace location differs from the source resume boundary', |
| detail: { sourceCwd: facts.sourceCwd, currentCwd: facts.currentCwd }, |
| }); |
| } |
| if (facts.sourceWorkspaceIdentity !== facts.currentWorkspaceIdentity) { |
| phaseOneDiagnostics.push({ |
| code: 'workspace_identity_mismatch', |
| message: 'current workspace identity differs from the source resume boundary', |
| detail: { |
| sourceWorkspaceIdentity: facts.sourceWorkspaceIdentity, |
| currentWorkspaceIdentity: facts.currentWorkspaceIdentity, |
| }, |
| }); |
| phaseOneRejectionReasons.push('workspace_identity_mismatch'); |
| } |
| if (!facts.backgroundOperationsSettled) { |
| phaseOneDiagnostics.push({ |
| code: 'background_operation_pending', |
| message: 'a background or child operation is not settled', |
| }); |
| phaseOneRejectionReasons.push('background_operation_pending'); |
| } |
| if (facts.workspaceCheckpoint) { |
| if (!facts.workspaceCheckpoint.ref) { |
| phaseOneDiagnostics.push({ |
| code: 'workspace_ref_missing', |
| message: 'workspace checkpoint does not contain a restorable ref', |
| }); |
| phaseOneRejectionReasons.push('workspace_ref_missing'); |
| } else if (!facts.workspaceCheckpoint.restored) { |
| phaseOneDiagnostics.push({ |
| code: 'checkpoint_restore_failed', |
| message: 'workspace checkpoint ref could not be restored', |
| detail: { workspaceRef: facts.workspaceCheckpoint.ref }, |
| }); |
| phaseOneRejectionReasons.push('checkpoint_restore_failed'); |
| } |
| } |
| const sourceReplayRuntimeEvents = |
| compositeReplay?.segments.at(-1)?.replayRuntimeEvents ?? |
| legacyReplayPlan?.replayRuntimeEvents ?? |
| []; |
| const modelRuntimeContext = compositeReplay?.runtimeContext ?? [ |
| ...(facts.priorRuntimeContext ?? []), |
| ...sourceReplayRuntimeEvents, |
| ]; |
| const availableToolNames = new Set(facts.availableToolNames); |
| const unavailableToolNames = [ |
| ...new Set( |
| modelRuntimeContext |
| .flatMap((event) => (event.content?.kind === 'function_call' ? [event.content.name] : [])) |
| .filter((toolName) => !availableToolNames.has(toolName)), |
| ), |
| ].sort(); |
| if (unavailableToolNames.length > 0) { |
| phaseOneDiagnostics.push({ |
| code: 'tool_catalog_mismatch', |
| message: 'one or more tools from the source boundary are unavailable', |
| detail: { unavailableToolNames }, |
| }); |
| phaseOneRejectionReasons.push('tool_catalog_mismatch'); |
| } |
| const firstProviderItem = compositeReplay?.providerItems[0]; |
| const firstLegacyEvent = compositeReplay |
| ? undefined |
| : modelRuntimeContext.find(runtimeEventHasModelVisibleContent); |
| const firstModelVisibleRole = compositeReplay |
| ? providerReplayItemRole(firstProviderItem) |
| : firstLegacyEvent?.role; |
| const firstModelVisibleEventId = compositeReplay |
| ? firstProviderItem?.eventId |
| : firstLegacyEvent?.id; |
| if ( |
| source && |
| !phaseOneRejectionReasons.includes('runtime_identity_mismatch') && |
| firstModelVisibleRole !== 'user' |
| ) { |
| phaseOneDiagnostics.push({ |
| code: 'provider_resume_head_unsupported', |
| message: 'provider replay must start at a user boundary for continuation', |
| ...(firstModelVisibleEventId ? { eventId: firstModelVisibleEventId } : {}), |
| detail: { firstRole: firstModelVisibleRole ?? null }, |
| }); |
| phaseOneRejectionReasons.push('provider_resume_head_unsupported'); |
| } |
| const lastProviderItem = compositeReplay?.providerItems.at(-1); |
| const lastLegacyEvent = compositeReplay |
| ? undefined |
| : findLastModelVisibleEvent(modelRuntimeContext); |
| const lastModelVisibleRole = compositeReplay |
| ? providerReplayItemRole(lastProviderItem) |
| : lastLegacyEvent?.role; |
| const lastModelVisibleEventId = compositeReplay ? lastProviderItem?.eventId : lastLegacyEvent?.id; |
| if ( |
| source && |
| !phaseOneRejectionReasons.includes('runtime_identity_mismatch') && |
| lastModelVisibleRole !== 'user' && |
| lastModelVisibleRole !== 'tool' |
| ) { |
| phaseOneDiagnostics.push({ |
| code: 'provider_resume_boundary_unsupported', |
| message: 'provider replay must end at a user or tool boundary for continuation', |
| ...(lastModelVisibleEventId ? { eventId: lastModelVisibleEventId } : {}), |
| detail: { lastRole: lastModelVisibleRole ?? null }, |
| }); |
| phaseOneRejectionReasons.push('provider_resume_boundary_unsupported'); |
| } |
| if ( |
| (legacyReplayPlan !== undefined && legacyReplayPlan.disposition !== 'safe_replay') || |
| phaseOneRejectionReasons.length > 0 || |
| !source |
| ) { |
| return { |
| disposition: 'park', |
| rejectionReasons: [ |
| ...(legacyReplayPlan?.rejectionReasons ?? []), |
| ...phaseOneRejectionReasons, |
| ], |
| diagnostics: [...(legacyReplayPlan?.diagnostics ?? []), ...phaseOneDiagnostics], |
| }; |
| } |
| |
| return { |
| disposition: 'continue', |
| rejectionReasons: [], |
| diagnostics: phaseOneDiagnostics, |
| continuation: { |
| sessionId: source.sessionId, |
| ...facts.continuationIdentity, |
| sourceInvocationId: source.invocationId, |
| sourceRunId: source.runId, |
| sourceTurnId: source.turnId, |
| sourceRuntimeEventHighWater: |
| compositeReplay?.segments.at(-1)?.boundary.position.lastEventSeq ?? |
| legacyReplayPlan!.sourceRuntimeEventHighWater, |
| ...(compositeReplay && compositeReplay.segments.length > 1 |
| ? { sourceRuntimeContext: [...sourceReplayRuntimeEvents] } |
| : facts.priorRuntimeContext?.length |
| ? { sourceRuntimeContext: legacyReplayPlan!.replayRuntimeEvents } |
| : {}), |
| runtimeContext: [...modelRuntimeContext], |
| ...(facts.handoffPause |
| ? { |
| handoffRootRunId: facts.handoffPause.rootRunId, |
| handoffRemainingSteps: facts.handoffPause.remainingSteps, |
| } |
| : {}), |
| ...(facts.continuationClaimId ? { claimId: facts.continuationClaimId } : {}), |
| ...(compositeReplay |
| ? { |
| boundary: compositeReplay.boundary, |
| providerReplayDigest: compositeReplay.providerReplayDigest, |
| providerProjectionVersion: compositeReplay.providerProjectionVersion, |
| } |
| : {}), |
| safetySnapshot: { |
| workspaceIdentity: facts.currentWorkspaceIdentity, |
| backgroundOperationsSettled: true, |
| availableToolNames: [...new Set(facts.availableToolNames)].sort(), |
| ...(facts.workspaceCheckpoint?.ref |
| ? { |
| workspaceCheckpoint: { |
| ref: facts.workspaceCheckpoint.ref, |
| runtimeEventHighWater: facts.workspaceCheckpoint.runtimeEventHighWater, |
| }, |
| } |
| : {}), |
| }, |
| }, |
| }; |
| } |
| |
| function collectPendingPermissionDiagnostics( |
| events: readonly RuntimeEvent[], |
| ): ResumePlanDiagnostic[] { |
| const pending = new Map<string, RuntimeEvent>(); |
| for (const event of events) { |
| if (isPartialRuntimeEvent(event)) continue; |
| const request = event.actions?.permissionRequest; |
| if (request) pending.set(request.requestId, event); |
| const decision = event.actions?.permissionDecision; |
| if (decision) pending.delete(decision.requestId); |
| const accepted = event.actions?.permissionAnswerAccepted; |
| if (accepted) pending.delete(accepted.requestId); |
| const closed = event.actions?.permissionClosureAccepted; |
| if (closed) pending.delete(closed.requestId); |
| } |
| return [...pending.entries()].map(([requestId, event]) => ({ |
| code: 'pending_permission', |
| message: 'permission request has no committed decision', |
| eventId: event.id, |
| detail: { requestId }, |
| })); |
| } |
| |
| function normalizeCwd(value: string): string { |
| const normalized = value.replaceAll('\\', '/').replace(/\/+$/, ''); |
| return /^[A-Za-z]:\//.test(normalized) ? normalized.toLowerCase() : normalized; |
| } |
| |
| function findLastModelVisibleEvent(events: readonly RuntimeEvent[]): RuntimeEvent | undefined { |
| for (let index = events.length - 1; index >= 0; index -= 1) { |
| const event = events[index]; |
| if (event && runtimeEventHasModelVisibleContent(event)) return event; |
| } |
| return undefined; |
| } |
| |
| function providerReplayItemRole( |
| item: RuntimeEventModelReplayItem | undefined, |
| ): 'user' | 'assistant' | 'system' | 'tool' | undefined { |
| if (!item) return undefined; |
| if (item.kind === 'text') return item.role; |
| if (item.kind === 'tool_result') return 'tool'; |
| return 'assistant'; |
| } |
| |
| function parkedPlan( |
| reason: ResumeRejectionReason & ResumePlanDiagnosticCode, |
| message: string, |
| detail?: Record<string, unknown>, |
| ): SafeBoundaryContinuationPlan { |
| return { |
| disposition: 'park', |
| rejectionReasons: [reason], |
| diagnostics: [{ code: reason, message, ...(detail ? { detail } : {}) }], |
| }; |
| } |
| |
| function collectResumeDiagnostics( |
| events: readonly RuntimeEvent[], |
| operations: readonly ToolOperation[], |
| options: BuildResumePlanOptions, |
| recovery: RuntimeRecoveryResolution, |
| ): ResumePlanDiagnostic[] { |
| const diagnostics: ResumePlanDiagnostic[] = []; |
| const operationsById = new Map(operations.map((operation) => [operation.toolCallId, operation])); |
| if ( |
| options.expectedRuntimeEventHighWater !== undefined && |
| options.expectedRuntimeEventHighWater !== events.length |
| ) { |
| diagnostics.push({ |
| code: 'runtime_offset_mismatch', |
| message: 'RuntimeEvent high-water does not match the expected checkpoint offset', |
| detail: { |
| expectedRuntimeEventHighWater: options.expectedRuntimeEventHighWater, |
| actualRuntimeEventHighWater: events.length, |
| }, |
| }); |
| } |
| |
| for (const operation of operations) { |
| if (operation.status === 'indeterminate') { |
| diagnostics.push({ |
| code: 'pending_tool_result', |
| message: 'function_call has no matching committed function_response', |
| eventId: operation.callRuntimeEventId, |
| toolCallId: operation.toolCallId, |
| toolName: operation.toolName, |
| }); |
| } else if (operation.status === 'not_dispatched') { |
| diagnostics.push({ |
| code: 'tool_not_dispatched', |
| message: 'function_call did not cross the durable tool dispatch boundary', |
| eventId: operation.callRuntimeEventId, |
| toolCallId: operation.toolCallId, |
| toolName: operation.toolName, |
| }); |
| } else if (operation.status === 'corruption') { |
| diagnostics.push({ |
| code: 'tool_recovery_corruption', |
| message: 'tool recovery facts conflict', |
| eventId: operation.callRuntimeEventId, |
| toolCallId: operation.toolCallId, |
| toolName: operation.toolName, |
| }); |
| } else if (operation.status === 'parked') { |
| diagnostics.push({ |
| code: 'tool_recovery_parked', |
| message: 'tool recovery reached a terminal parked decision', |
| eventId: operation.callRuntimeEventId, |
| toolCallId: operation.toolCallId, |
| toolName: operation.toolName, |
| }); |
| } |
| } |
| |
| for (const issue of recovery.issues) { |
| if (issue.code === 'protocol_marker_invalid') { |
| diagnostics.push({ |
| code: issue.code, |
| message: 'runtime protocol marker is only valid on the first canonical event', |
| eventId: issue.eventId, |
| }); |
| } else if (issue.code === 'duplicate_event_id') { |
| diagnostics.push({ |
| code: issue.code, |
| message: 'immutable RuntimeEvent identity is duplicated', |
| eventId: issue.eventId, |
| }); |
| } else if (issue.code === 'semantic_lane_conflict') { |
| diagnostics.push({ |
| code: issue.code, |
| message: 'RuntimeEvent claims more than one authoritative tool semantic lane', |
| eventId: issue.eventId, |
| }); |
| } else { |
| diagnostics.push({ |
| code: 'tool_ledger_corruption', |
| message: `immutable tool ledger is corrupt: ${issue.code}`, |
| eventId: issue.eventId, |
| detail: { issueCode: issue.code }, |
| }); |
| } |
| } |
| |
| for (const decision of recovery.decisions) { |
| if ( |
| decision.status !== 'corruption' || |
| decision.callRuntimeEventId || |
| decision.reason === 'orphan_response' |
| ) |
| continue; |
| diagnostics.push({ |
| code: 'tool_recovery_corruption', |
| message: `tool recovery fact is corrupt: ${decision.reason}`, |
| eventId: decision.dispatchRuntimeEventId ?? decision.responseRuntimeEventId, |
| toolCallId: decision.toolCallId, |
| ...(decision.toolName ? { toolName: decision.toolName } : {}), |
| }); |
| } |
| |
| const eventsById = new Map(events.map((event) => [event.id, event] as const)); |
| for (const decision of recovery.decisions) { |
| if (decision.status !== 'indeterminate' || !decision.callRuntimeEventId) continue; |
| const callEvent = eventsById.get(decision.callRuntimeEventId); |
| if (callEvent?.modelVisibility !== 'hidden') continue; |
| diagnostics.push({ |
| code: 'pending_tool_result', |
| message: 'hidden nested tool call has no matching committed function_response', |
| eventId: decision.callRuntimeEventId, |
| toolCallId: decision.toolCallId, |
| ...(decision.toolName ? { toolName: decision.toolName } : {}), |
| }); |
| } |
| |
| for (const event of events) { |
| if (isPartialRuntimeEvent(event)) continue; |
| if (event.modelVisibility === 'hidden') continue; |
| const content = event.content; |
| if (content?.kind !== 'function_response') continue; |
| const operation = operationsById.get(content.id); |
| if (!operation) { |
| diagnostics.push({ |
| code: 'unmatched_tool_result', |
| message: 'function_response has no prior matching function_call', |
| eventId: event.id, |
| toolCallId: content.id, |
| toolName: content.name, |
| }); |
| continue; |
| } |
| if (operation.toolName !== content.name) { |
| diagnostics.push({ |
| code: 'tool_name_mismatch', |
| message: 'function_response tool name differs from matching function_call', |
| eventId: event.id, |
| toolCallId: content.id, |
| toolName: content.name, |
| detail: { |
| callToolName: operation.toolName, |
| responseToolName: content.name, |
| }, |
| }); |
| } |
| } |
| |
| return diagnostics; |
| } |
| |
| function hasHiddenIndeterminateOperation( |
| events: readonly RuntimeEvent[], |
| recovery: RuntimeRecoveryResolution, |
| ): boolean { |
| const eventsById = new Map(events.map((event) => [event.id, event] as const)); |
| return recovery.decisions.some( |
| (decision) => |
| decision.status === 'indeterminate' && |
| decision.callRuntimeEventId !== undefined && |
| eventsById.get(decision.callRuntimeEventId)?.modelVisibility === 'hidden', |
| ); |
| } |
| |
| function deriveRejectionReasons( |
| diagnostics: readonly ResumePlanDiagnostic[], |
| ): ResumeRejectionReason[] { |
| const reasons = new Set<ResumeRejectionReason>(); |
| for (const diagnostic of diagnostics) { |
| switch (diagnostic.code) { |
| case 'runtime_offset_mismatch': |
| reasons.add('runtime_offset_mismatch'); |
| break; |
| case 'pending_tool_result': |
| case 'tool_not_dispatched': |
| case 'tool_recovery_parked': |
| case 'tool_recovery_corruption': |
| case 'tool_ledger_corruption': |
| case 'duplicate_event_id': |
| case 'semantic_lane_conflict': |
| case 'protocol_marker_invalid': |
| case 'unmatched_tool_result': |
| case 'tool_name_mismatch': |
| reasons.add('dangling_tool_state'); |
| break; |
| } |
| } |
| return [...reasons]; |
| } |
| |
| function collectPairedCallIds(events: readonly RuntimeEvent[]): Set<string> { |
| const calls = new Map<string, RuntimeEventFunctionCallContent>(); |
| const paired = new Set<string>(); |
| |
| for (const event of events) { |
| if (isPartialRuntimeEvent(event)) continue; |
| const content = event.content; |
| if (content?.kind === 'function_call') { |
| calls.set(content.id, content); |
| continue; |
| } |
| if (content?.kind === 'function_response' && hasMatchingCall(calls.get(content.id), content)) { |
| paired.add(content.id); |
| } |
| } |
| |
| return paired; |
| } |
| |
| function hasMatchingCall( |
| call: RuntimeEventFunctionCallContent | undefined, |
| response: RuntimeEventFunctionResponseContent, |
| ): boolean { |
| return call !== undefined && call.name === response.name; |
| } |