blob: 4d443df085f4030918723ce41780c9e1d5da1038 [file]
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import type { WorkHubActionReceipt } from '@maka/core/workhub-action-result';
import type { WorkHubRoutingDecision } from '@maka/core/workhub-routing';
import { createHash, randomUUID } from 'node:crypto';
import { isDeepStrictEqual } from 'node:util';
import type { BackendStopMode } from '@maka/core/backend-types';
import {
readLogicalRuntimeExecution,
readLogicalRuntimeExecutionForRun,
} from '@maka/core/runtime-logical-execution';
import { runtimeHandoffPause } from '@maka/core/runtime-handoff';
import type {
RootExecutionDescriptor,
RuntimeInvocationRecord,
} from '@maka/core/runtime-invocation';
import {
INLINE_REFERENCE_MAX_COUNT,
messageContentDigest,
messageContentsEqual,
normalizeMessageContent,
type AttachmentRef,
type MessageContent,
type SessionEvent,
} from '@maka/core/events';
import {
WORKHUB_COORDINATION_SESSION_ID,
isWorkHubCoordinationSessionId,
type SessionHeader,
} from '@maka/core/session';
import { resolveEffectiveOrchestration } from '@maka/core/orchestration';
import {
decodeSkillInvocationResult,
type SkillInvocationResult,
} from '@maka/core/skill-invocation';
import { agentGraphIdForRootSession } from '@maka/runtime/stream-graph-coordinator';
import {
RuntimeHostedRootConflictError,
RuntimeHostedRootUnavailableError,
RuntimeMessageAuthorityInvariantError,
type RuntimeMessageRunIdentity,
} from '@maka/runtime/message-authority';
import {
isShutdownCancelledInteractionAdmission,
RuntimeInteractionAdmissionRejectedError,
RuntimeInteractionFailStopError,
RuntimeInteractionInvariantError,
} from '@maka/runtime/interaction-authority';
import {
normalizeStopSessionSource,
RuntimeRegenerateTurnError,
type SessionManager,
type StopSessionInput,
} from '@maka/runtime/session-manager';
import { RuntimeOwnerCleanupError } from '@maka/runtime/runtime-kernel';
import {
parseSkillInvocationTokens,
type PreparedSkillInvocationMessage,
} from '@maka/runtime/skill-invocation';
import { skillInvocationInlineReferences } from '@maka/runtime/skill-invocation-receipt';
import {
type RuntimeContinuation,
type SafeBoundaryContinuationPlan,
} from '@maka/runtime/runtime-resume';
import {
authenticateExecutionStoresWriter,
isSessionNotFoundError,
normalizeRootTurnAdmissionPayload,
type ExecutionStoresWriter,
type RootTurnAdmission,
} from '@maka/storage/execution-stores';
import type {
OperationOutcome,
TurnResumePlan,
TurnResumeQueryInput,
TurnResumeStartInput,
TurnSnapshot,
TurnStartInput,
TurnStopInput,
} from '../protocol/index.js';
import type { RuntimeHostResidency } from './host-kernel.js';
import type { HostInteractionCoordinator } from './interaction-coordinator.js';
import {
type HostMessageRootState,
type HostMessagePreparationInput,
type HostMessageRecoveryBatch,
type HostMessageSessionHeader,
type HostMessageStartInput,
type HostMessageStartOutcome,
type HostMessageStopClaim,
type HostMessageStopFence,
HostMessageCoordinator,
type QueueFenceResult,
type RootFollowupBatch,
} from './message-coordinator.js';
import type { ConnectionContext, TurnOperationHandlerMap } from './operation-dispatcher.js';
import { RootAdmissionOwner } from './root-admission-owner.js';
import { type SessionAdmissionLease, SessionAdmissionGate } from './session-admission-gate.js';
import {
type RuntimeSessionForwardedEvent,
SessionContinuityCoordinator,
} from './session-continuity-coordinator.js';
import type {
HostClientCapabilityCoordinator,
SessionBindingPreview,
} from './client-capability-coordinator.js';
import { hostedExecutionRunProfile } from './hosted-execution-tool-profile.js';
import {
runtimeHostExecutionUnavailableReason,
runtimeHostExternalTurnUnavailableReason,
runtimeHostSafeBoundaryContinuationUnavailableReason,
WORKHUB_COORDINATION_EXECUTION_UNAVAILABLE_REASON,
} from './host-session-availability.js';
import type {
HostedExecutionAdmission,
HostedExecutionAdmissionResult,
HostedExecutionAuthority,
HostedExecutionCompletion,
HostedExecutionCompletionObserver,
HostedExecutionIdentity,
HostedExecutionListener,
HostedExecutionObserver,
HostedExecutionPreparation,
HostedExecutionRef,
HostedExecutionSnapshot,
HostedExecutionStopInput,
} from './hosted-execution-authority.js';
import { completedHostedExecutionAdmission } from './hosted-execution-authority.js';
import { HostedExecutionProjectionReader } from './hosted-execution-projection.js';
import {
hostedExecutionMessageOrigin,
prepareHostedExecutionRecovery,
requireHostedExecutionMessageContent,
type HostedExecutionRecoveryPlan,
} from './hosted-execution-recovery.js';
import { HostedExecutionRegistry } from './hosted-execution-registry.js';
import {
HostedExecutionAdmissionRegistry,
type HostedExecutionReservation,
} from './hosted-execution-admission-registry.js';
import { waitForHostedExecutionIdleOrAbort } from './hosted-execution-wait.js';
type RootTerminalInteractionFence = Pick<
HostInteractionCoordinator,
'assertTerminalFence' | 'claimRunClosure'
>;
interface ActiveRootTurn {
sessionId: string;
turnId: string;
runId: string;
userMessageId: string | null;
execution?: HostedExecutionAdmission;
continuation?: RuntimeContinuation;
descriptor: RootExecutionDescriptor;
graphOwnerId?: string;
completionObserver?: HostedExecutionCompletionObserver;
completion: ValueDeferred<HostedExecutionCompletion>;
observedCompletion?: HostedExecutionCompletion;
observationSettled?: Promise<void>;
startSettled: Deferred;
done: Promise<void>;
residency: RuntimeHostResidency;
stopRequested: StopSessionInput | undefined;
messageTransitionCommitted: boolean;
handoff?: { decision: ValueDeferred<'resume' | 'detach'>; arrived: Deferred };
}
/** Epoch-local attachment hold. The existing deployment transaction owns cutover. */
export interface RootHandoffPreparation {
readonly executions: readonly HostedExecutionRef[];
seal(): Promise<boolean>;
residencies(): Promise<readonly RuntimeHostResidency[] | undefined>;
detach(): Promise<void>;
cancel(): void;
}
export type TurnStartOutcome = OperationOutcome<'turn.start'>;
type RootMessageStartOutcome =
| { ok: true; result: TurnSnapshot }
| Extract<TurnStartOutcome, { ok: false }>;
export type RootMessageExecution = Extract<
RootExecutionDescriptor,
{ kind: 'external_message' | 'workhub_coordination' | 'regenerate' }
>;
interface RootMessageStartRequestBase {
readonly sessionId: string;
readonly turnId: string;
readonly archivedMessage: string;
readonly prepareReplayContent?: (
lease: SessionAdmissionLease,
) => Promise<RootMessageContentPreparation>;
}
export type RootMessageStartRequest =
| (RootMessageStartRequestBase & {
readonly execution: Extract<RootMessageExecution, { kind: 'external_message' }>;
readonly content: MessageContent;
readonly turnOrchestration?: TurnStartInput['turnOrchestration'];
})
| (RootMessageStartRequestBase & {
readonly execution: Extract<RootMessageExecution, { kind: 'external_message' }>;
readonly turnOrchestration?: TurnStartInput['turnOrchestration'];
prepareFreshContent(lease: SessionAdmissionLease): Promise<RootMessageContentPreparation>;
})
| (RootMessageStartRequestBase & {
readonly execution: Extract<RootMessageExecution, { kind: 'regenerate' }>;
readonly turnOrchestration?: undefined;
prepareContent(): Promise<MessageContent>;
})
| (RootMessageStartRequestBase & {
readonly execution: Extract<RootMessageExecution, { kind: 'workhub_coordination' }>;
readonly operation?: (turnId: string) => Promise<WorkHubActionReceipt>;
readonly turnOrchestration?: undefined;
prepareFreshContent(lease: SessionAdmissionLease): Promise<RootMessageContentPreparation>;
});
export interface HostWorkHubRoutingDecisionPreparation {
readonly header: SessionHeader;
readonly turnId: string;
readonly content: MessageContent;
readonly inputClosedSignal?: AbortSignal;
}
export type RootMessageContentPreparation =
| {
readonly kind: 'ready';
readonly content: MessageContent;
readonly skillInvocation?: SkillInvocationResult;
readonly commitCapabilityBinding?: () => Promise<
{ readonly ok: true } | { readonly ok: false; readonly message: string }
>;
}
| {
readonly kind: 'rejected';
readonly outcome: RootMessageStartOutcome;
readonly skillInvocation?: SkillInvocationResult;
};
export interface HostedExternalTurnTransitionInput {
readonly sessionId: string;
readonly turnId: string;
readonly inputDigest: `sha256:${string}`;
readonly archivedMessage: string;
prepareContent(lease: SessionAdmissionLease): Promise<RootMessageContentPreparation>;
}
interface RootTurnActivationInput {
readonly sessionId: string;
readonly turnId: string;
readonly content: MessageContent | null;
readonly turnOrchestration?: TurnStartInput['turnOrchestration'];
}
type TurnResumeStartOutcome = OperationOutcome<'turn.resume.start'>;
type TurnResumeStartDisposition =
| TurnStartDisposition
| {
kind: 'parked';
plan: Extract<TurnResumePlan, { disposition: 'parked' }>;
};
type ReconstructedContinuation =
| { disposition: 'ready'; continuation: RuntimeContinuation }
| {
disposition: 'parked';
plan: Extract<TurnResumePlan, { disposition: 'parked' }>;
};
type TurnStartDisposition =
| { kind: 'complete'; outcome: RootMessageStartOutcome }
| { kind: 'await_start'; active: ActiveRootTurn };
type TurnStopOutcome = OperationOutcome<'turn.stop'>;
type TurnStopDisposition =
| { kind: 'complete'; outcome: TurnStopOutcome }
| { kind: 'request_stop'; active: ActiveRootTurn }
| { kind: 'await_terminal'; active: ActiveRootTurn };
interface DeclaredStopFence {
readonly active: ActiveRootTurn;
deliverStop(): Promise<void>;
}
interface Deferred {
readonly promise: Promise<void>;
readonly phase: 'pending' | 'resolved' | 'rejected';
resolve(): void;
reject(error: unknown): void;
}
interface ValueDeferred<T> {
readonly promise: Promise<T>;
resolve(value: T): void;
}
type RootTurnReservation = HostedExecutionReservation;
interface HostSkillInvocationPreparer {
(input: {
sessionId: string;
turnId: string;
text: string;
skillIds: readonly string[];
}): Promise<PreparedSkillInvocationMessage>;
}
interface HostTurnAttachmentValidator {
validateTurnAttachments(
sessionId: string,
attachments: readonly AttachmentRef[],
): Promise<string | undefined>;
}
interface HostAgentGraphEpochAuthority {
currentGraphId(rootSessionId: string): Promise<string>;
beginNextGraphEpoch(rootSessionId: string): Promise<string>;
}
export class RootTurnCoordinator implements HostedExecutionAuthority {
readonly handlers: Pick<TurnOperationHandlerMap, 'turn.resume.query' | 'turn.resume.start'> = {
'turn.resume.query': (input, context) => this.queryTurnResume(input, context),
'turn.resume.start': (input, context) => this.startTurnResume(input, context),
};
readonly #executions = new HostedExecutionRegistry<ActiveRootTurn>();
readonly #admissions: HostedExecutionAdmissionRegistry;
#handoffHold: Deferred | undefined;
#cancelHandoff: (() => void) | undefined;
readonly #recoveryPlansBySession = new Map<string, HostedExecutionRecoveryPlan>();
private readonly stores: ExecutionStoresWriter<'interactive'>;
private readonly executionProjection: HostedExecutionProjectionReader;
private readonly attachmentValidator: HostTurnAttachmentValidator | undefined;
private readonly prepareSkillInvocation: HostSkillInvocationPreparer | undefined;
constructor(
private readonly manager: SessionManager,
stores: ExecutionStoresWriter<'interactive'>,
private readonly sessionAdmission: SessionAdmissionGate,
private readonly rootAdmissionOwner: RootAdmissionOwner,
private readonly interactions: RootTerminalInteractionFence,
private readonly messages: HostMessageCoordinator,
private readonly continuity: SessionContinuityCoordinator,
private readonly acquireRecoveryResidency: () => RuntimeHostResidency,
private readonly requestHostDrain: () => void,
private readonly clientCapabilities: HostClientCapabilityCoordinator | undefined,
private readonly resolveExecutionObserver: () => HostedExecutionObserver,
private readonly assertScheduledTaskRecoveryAdmission?: (
admission: RootTurnAdmission,
state: 'pending_fire_required' | 'run_recorded',
) => Promise<void>,
attachmentValidator?: HostTurnAttachmentValidator,
prepareSkillInvocation?: HostSkillInvocationPreparer,
private readonly agentGraphEpochs?: HostAgentGraphEpochAuthority,
private readonly nameSessionFromRootMessage?: (input: {
sessionId: string;
content: MessageContent;
}) => void,
private readonly directoryHostId?: string,
private readonly prepareWorkHubRoutingDecision?: (
input: HostWorkHubRoutingDecisionPreparation,
) => Promise<WorkHubRoutingDecision>,
) {
this.stores = authenticateExecutionStoresWriter(stores, 'interactive');
this.executionProjection = new HostedExecutionProjectionReader(this.stores);
this.#admissions = new HostedExecutionAdmissionRegistry((sessionId) =>
this.#executions.has(sessionId),
);
this.attachmentValidator = attachmentValidator;
this.prepareSkillInvocation = prepareSkillInvocation;
}
async prepareRecovery(): Promise<void> {
const plans = await prepareHostedExecutionRecovery({
stores: this.stores,
rootAdmissions: this.rootAdmissionOwner,
projection: this.executionProjection,
runtime: this.manager,
...(this.assertScheduledTaskRecoveryAdmission
? { assertScheduledTaskAdmission: this.assertScheduledTaskRecoveryAdmission }
: {}),
});
for (const plan of plans) {
this.#recoveryPlansBySession.set(plan.sessionId, plan);
}
}
async recover(): Promise<void> {
for (const [sessionId, plan] of this.#recoveryPlansBySession) {
if (
plan.rootReplayAdmission &&
!(await this.bindRecoveryCapabilities(sessionId, plan.rootReplayAdmission.execution))
)
continue;
try {
for (const admission of plan.admissions) {
const run = await this.readRunIfPresent(sessionId, admission.runId);
if (!run) continue;
await this.assertRunMatchesDurableExecution(run, admission.turnId, admission.execution);
const snapshot = await this.readCanonicalSnapshot(
sessionId,
admission.turnId,
admission.runId,
run,
);
if (isTerminalSnapshot(snapshot)) {
if (admission.sourceMessages.length > 0) {
await this.messages.materializeMessageHandoffsForRun({
sessionId,
turnId: admission.turnId,
runId: admission.runId,
messageIds: admission.sourceMessages.map((source) => source.messageId),
});
}
} else {
if (plan.rootReplayAdmission === admission) continue;
if (admission.execution.kind !== 'safe_boundary_continuation') {
throw new Error(`Startup recovery left Turn ${admission.turnId} non-terminal`);
}
this.parkContinuationAdmission(admission);
}
}
const admission = plan.rootReplayAdmission;
if (!admission) continue;
// Session recovery may have materialized this Run after the replay plan was prepared.
const existingRun = await this.readRunIfPresent(sessionId, admission.runId);
const pendingHandoff =
existingRun &&
(await readLogicalRuntimeExecution(this.stores.runtimeEventStore, admission, existingRun))
?.pendingHandoff;
if (existingRun && !pendingHandoff) continue;
const input = activationInputForAdmission(admission);
const disposition = await this.sessionAdmission.run(sessionId, async (lease) => {
if (admission.execution.kind === 'safe_boundary_continuation') {
const header = await this.stores.sessionStore.readHeaderSnapshot(sessionId);
if (runtimeHostSafeBoundaryContinuationUnavailableReason(header)) {
this.parkContinuationAdmission(admission);
return undefined;
}
}
const continuation =
admission.execution.kind === 'safe_boundary_continuation' && !pendingHandoff
? await this.reconstructAdmittedContinuation(admission)
: undefined;
if (continuation?.disposition === 'parked') {
if (
continuation.plan.reason === 'safety_check_failed' ||
continuation.plan.reason === 'resume_feature_disabled' ||
continuation.plan.reason === 'continuation_authority_unavailable' ||
continuation.plan.reason === 'safety_observation_unavailable'
) {
this.parkContinuationAdmission(admission);
return undefined;
}
throw new Error(
`Unable to recover admitted Turn ${admission.turnId}: ${continuation.plan.reason}`,
);
}
await this.messages.handoffRootSources({
sessionId,
turnId: admission.turnId,
runId: admission.runId,
messageIds: admission.sourceMessages.map((source) => source.messageId),
});
return this.prepareAdmittedTurn(
input,
admission,
this.acquireRecoveryResidency,
lease,
undefined,
undefined,
undefined,
continuation?.continuation,
);
});
if (!disposition) continue;
const outcome = await this.resolveStartDisposition(input, disposition);
if (!outcome.ok) {
throw new Error(
`Unable to recover admitted Turn ${admission.turnId}: ${outcome.error.code}`,
);
}
} finally {
this.#recoveryPlansBySession.delete(sessionId);
}
}
}
private async bindRecoveryCapabilities(
sessionId: string,
execution: RootExecutionDescriptor,
): Promise<boolean> {
const header = await this.stores.sessionStore.readHeaderSnapshot(sessionId);
if (header.toolProfile !== 'workhub-coordination-v2') return true;
// Older admissions without authenticated provider evidence stay pending.
// Neither registration order nor a tool name can establish Desktop ownership.
if (
execution.kind !== 'workhub_coordination' ||
!execution.capabilityBinding ||
!this.clientCapabilities
)
return false;
return this.clientCapabilities.bindRecoveredSession(
sessionId,
execution.capabilityBinding,
hostedExecutionRunProfile(header.toolProfile)!.toolNames.filter((name) =>
name.startsWith('mcp__'),
),
);
}
async close(): Promise<void> {
this.#cancelHandoff?.();
this.beginDrain();
await this.#admissions.waitForSettledAdmissions();
const errors: unknown[] = [];
while (errors.length === 0) {
const active = [...this.#executions.entries()];
if (active.length === 0) break;
const results = await Promise.allSettled(
active.map(([sessionId, turn]) => this.stopActiveTurn(sessionId, turn)),
);
errors.push(
...results
.filter(
(result): result is PromiseRejectedResult =>
result.status === 'rejected' &&
!isShutdownCancelledInteractionAdmission(result.reason),
)
.map((result) => result.reason),
);
}
if (this.#executions.size !== 0) {
errors.push(new Error('Runtime Host execution composition closed with active Turns'));
}
this.#executions.close();
if (errors.length > 0)
throw new AggregateError(errors, 'Unable to close Runtime Host execution composition');
}
async readSessionHeader(sessionId: string): Promise<HostMessageSessionHeader | null> {
if (isWorkHubCoordinationSessionId(sessionId)) {
const active = this.#executions.get(sessionId);
if (
active?.descriptor.kind === 'workhub_coordination' &&
!active.descriptor.operation &&
!active.stopRequested
) {
const header = await this.stores.sessionStore.readHeaderSnapshot(sessionId);
if (header.toolProfile === 'workhub-coordination-v2') {
return { isArchived: header.isArchived, activeTurnOnly: true };
}
}
return {
isArchived: false,
unavailableReason: WORKHUB_COORDINATION_EXECUTION_UNAVAILABLE_REASON,
};
}
try {
const header = await this.stores.sessionStore.readHeaderSnapshot(sessionId);
if (header.conversationCopy?.state === 'preparing') return null;
return {
isArchived: header.isArchived,
unavailableReason: runtimeHostExternalTurnUnavailableReason(header),
};
} catch (error) {
if (isSessionNotFoundError(error)) return null;
throw error;
}
}
/** Called under Session admission before changing an execution's authority ceiling. */
isSessionExecutionIdle(sessionId: string): boolean {
return (
!this.#recoveryPlansBySession.has(sessionId) && this.readRootState(sessionId).kind === 'idle'
);
}
readRootState(sessionId: string): HostMessageRootState {
const active = this.#executions.get(sessionId);
if (active) {
return {
kind: 'active',
sessionId,
turnId: active.turnId,
runId: active.runId,
};
}
return this.#handoffHold ||
this.#admissions.has(sessionId) ||
this.#recoveryPlansBySession.get(sessionId)?.rootReplayAdmission
? { kind: 'reserved' }
: { kind: 'idle' };
}
/** Returns the newest Host-admitted continuation descended from one root execution. */
async readLatestRootTurnLineage(identity: HostedExecutionRef): Promise<HostedExecutionRef> {
const origin = await this.stores.agentRunStore.readRootTurnAdmission(
identity.sessionId,
identity.turnId,
);
if (!origin || origin.runId !== identity.runId) {
throw new RuntimeMessageAuthorityInvariantError(
`Root execution ${identity.turnId}/${identity.runId} has no durable admission`,
);
}
let latest = origin;
while (true) {
const run = await this.readRunIfPresent(latest.sessionId, latest.runId);
const physical =
run?.terminalEvent && runtimeHandoffPause(run.terminalEvent)
? (await readLogicalRuntimeExecution(this.stores.runtimeEventStore, latest, run))?.tip
: run;
const continuation = await this.stores.agentRunStore.readRootTurnContinuationAdmission(
identity.sessionId,
latest.turnId,
physical?.runId ?? latest.runId,
);
if (!continuation) break;
latest = continuation;
}
return { sessionId: latest.sessionId, turnId: latest.turnId, runId: latest.runId };
}
startHostedExternalTransition(
input: HostedExternalTurnTransitionInput,
context: ConnectionContext,
): Promise<RootMessageStartOutcome> {
return this.startInteractiveRootMessage(
{
sessionId: input.sessionId,
turnId: input.turnId,
execution: { kind: 'external_message', inputDigest: input.inputDigest },
archivedMessage: input.archivedMessage,
prepareFreshContent: input.prepareContent,
prepareReplayContent: input.prepareContent,
},
context,
);
}
private reserveRootTurn(sessionId: string): RootTurnReservation | undefined {
if (this.#handoffHold || this.#recoveryPlansBySession.get(sessionId)?.rootReplayAdmission)
return undefined;
return this.#admissions.reserve(sessionId);
}
private parkContinuationAdmission(admission: RootTurnAdmission): void {
this.#admissions.park(admission);
}
private takeParkedContinuationReservation(
admission: RootTurnAdmission,
): RootTurnReservation | undefined {
if (this.#handoffHold) return undefined;
return this.#admissions.takeParked(admission);
}
private clearParkedContinuationAdmission(admission: RootTurnAdmission): void {
this.#admissions.clearParked(admission);
}
private parkedContinuationAdmission(sessionId: string): RootTurnAdmission | undefined {
return this.#admissions.parked(sessionId);
}
private beginRootAdmission(reservation: RootTurnReservation): boolean {
return this.#admissions.begin(reservation);
}
private releaseRootReservation(reservation: RootTurnReservation): void {
this.#admissions.release(reservation);
}
beginDrain(): void {
this.#admissions.beginDrain();
}
async prepareHandoff(
hostEpoch: string,
signal: AbortSignal,
): Promise<RootHandoffPreparation | undefined> {
if (this.#handoffHold || this.#admissions.isDraining || signal.aborted) return undefined;
const hold = deferred();
this.#handoffHold = hold;
const held = new Map<
ActiveRootTurn,
{
request: NonNullable<ReturnType<SessionManager['requestRunHandoff']>>;
attachment: NonNullable<ActiveRootTurn['handoff']>;
}
>();
let sealed = false;
let detached = false;
const releaseHold = () => {
if (this.#handoffHold === hold) {
this.#handoffHold = undefined;
this.#cancelHandoff = undefined;
}
hold.resolve();
signal.removeEventListener('abort', cancel);
};
const cancel = () => {
if (detached) return;
sealed = false;
for (const [active, { request, attachment }] of held) {
request.cancel();
attachment.arrived.resolve();
attachment.decision.resolve('resume');
if (active.handoff === attachment) active.handoff = undefined;
}
releaseHold();
};
this.#cancelHandoff = cancel;
signal.addEventListener('abort', cancel, { once: true });
try {
await waitForHostedExecutionIdleOrAbort(this.#admissions.waitForSettledAdmissions(), signal);
// Natural completion can hand a queued follow-up to a fresh root while
// gates converge. Bound that convergence; never chase an endless queue.
for (let pass = 0; pass < 3; pass += 1) {
for (const [, active] of this.#executions.entries()) {
if (held.has(active)) continue;
await waitForHostedExecutionIdleOrAbort(active.startSettled.promise, signal);
if (this.#executions.get(active.sessionId) !== active) continue;
const request = this.manager.requestRunHandoff(
active.sessionId,
active.continuation?.runId ?? active.runId,
{
protocol: 'runtime_handoff_pause_v1',
hostEpoch,
handoffId: randomUUID(),
rootRunId: active.runId,
successorRunId: randomUUID(),
successorInvocationId: randomUUID(),
claimId: randomUUID(),
},
signal,
);
if (!request) {
cancel();
return undefined;
}
const attachment = {
decision: valueDeferred<'resume' | 'detach'>(),
arrived: deferred(),
};
active.handoff = attachment;
held.set(active, { request, attachment });
}
for (const [active, item] of held) {
if (await waitForAbortableValue(item.request.ready, signal)) continue;
const snapshot = await this.readCanonicalSnapshot(
active.sessionId,
active.turnId,
active.runId,
);
if (!isTerminalSnapshot(snapshot)) {
cancel();
return undefined;
}
item.attachment.decision.resolve('resume');
if (active.handoff === item.attachment) active.handoff = undefined;
await waitForHostedExecutionIdleOrAbort(active.done, signal);
held.delete(active);
}
if ([...this.#executions.entries()].every(([, active]) => held.has(active))) break;
}
if (
signal.aborted ||
[...this.#executions.entries()].some(([, active]) => !held.has(active))
) {
cancel();
return undefined;
}
return {
executions: [...held.keys()].map(({ sessionId, turnId, runId }) => ({
sessionId,
turnId,
runId,
})),
seal: async () => {
if (signal.aborted || this.#handoffHold !== hold || detached) return false;
if (sealed) return true;
for (const [active, { request }] of held) {
if (this.#executions.get(active.sessionId) !== active || !request.commit()) {
cancel();
return false;
}
}
try {
const results = await Promise.all(
[...held.values()].map(({ request }) => request.sealed),
);
sealed = !signal.aborted && this.#handoffHold === hold && results.every(Boolean);
if (sealed) {
await Promise.all(
[...held].map(([active, { attachment }]) =>
Promise.race([
attachment.arrived.promise,
active.done.then(() => {
throw new Error('Sealed execution ended before its attachment handoff');
}),
]),
),
);
}
} catch (error) {
cancel();
throw error;
}
if (!sealed) cancel();
return sealed;
},
residencies: async () => {
if (!sealed || signal.aborted || this.#handoffHold !== hold || detached) return undefined;
const residencies: RuntimeHostResidency[] = [];
for (const [active, { attachment }] of held) {
if (
this.#executions.get(active.sessionId) !== active ||
active.handoff !== attachment ||
active.stopRequested
)
return undefined;
const messages = await this.sessionAdmission.run(active.sessionId, (lease) =>
this.messages.handoffResidencies(active, lease),
);
if (!messages) return undefined;
residencies.push(active.residency, ...messages);
}
return residencies;
},
detach: async () => {
if (!sealed || signal.aborted || this.#handoffHold !== hold || detached) {
throw new Error('Cannot detach a handoff without all durable seals');
}
detached = true;
for (const { attachment } of held.values()) attachment.decision.resolve('detach');
await Promise.all([...held.keys()].map((active) => active.done));
releaseHold();
},
cancel,
};
} catch (error) {
cancel();
if (signal.aborted) return undefined;
throw error;
}
}
async runExclusiveSessionOperation<T>(
input: {
readonly sessionId: string;
readonly abortSignal: AbortSignal;
readonly stopSource?: Exclude<HostedExecutionStopInput['source'], 'workhub_direct_stop'>;
},
operation: () => Promise<T>,
): Promise<T> {
const { sessionId, abortSignal } = input;
let reservation: RootTurnReservation;
for (;;) {
throwIfAborted(abortSignal);
if (this.#admissions.isDraining) {
throw new Error('Runtime Host root authority is draining.');
}
const available = this.reserveRootTurn(sessionId);
if (available) {
reservation = available;
break;
}
const active = this.#executions.get(sessionId);
const pending = this.#admissions.get(sessionId);
const whenIdle = active?.done ?? pending?.whenIdle.promise ?? this.#handoffHold?.promise;
if (whenIdle) {
await waitForHostedExecutionIdleOrAbort(whenIdle, abortSignal);
}
}
try {
return await this.sessionAdmission.run(sessionId, async () => {
throwIfAborted(abortSignal);
if (
this.#admissions.isDraining ||
this.#admissions.get(sessionId) !== reservation ||
!this.beginRootAdmission(reservation)
) {
throw new Error('Exclusive Session operation lost its root reservation.');
}
return this.#runWithAbortStop(
abortSignal,
() =>
this.deliverRuntimeStopIntent(sessionId, {
...(input.stopSource ? { source: input.stopSource } : {}),
}),
operation,
);
});
} finally {
this.releaseRootReservation(reservation);
}
}
async #runWithAbortStop<T>(
abortSignal: AbortSignal,
requestStop: () => Promise<void>,
operation: () => Promise<T>,
): Promise<T> {
let stopTask: Promise<void> | undefined;
const stop = (): void => {
stopTask ??= requestStop();
void stopTask.catch(() => undefined);
};
abortSignal.addEventListener('abort', stop, { once: true });
if (abortSignal.aborted) stop();
try {
return await operation();
} finally {
abortSignal.removeEventListener('abort', stop);
await stopTask;
}
}
prepare(sessionId: string): HostedExecutionPreparation {
if (isWorkHubCoordinationSessionId(sessionId)) {
return { kind: 'unavailable', reason: WORKHUB_COORDINATION_EXECUTION_UNAVAILABLE_REASON };
}
if (this.#admissions.isDraining) {
return {
kind: 'unavailable',
reason: 'Runtime Host execution authority is draining.',
};
}
if (this.#handoffHold) return { kind: 'busy', whenIdle: this.#handoffHold.promise };
const active = this.#executions.get(sessionId);
if (active) {
return {
kind: 'busy',
whenIdle: active.done,
execution: { sessionId, turnId: active.turnId, runId: active.runId },
};
}
const pending = this.#admissions.get(sessionId);
if (pending) return { kind: 'busy', whenIdle: pending.whenIdle.promise };
const reservation = this.reserveRootTurn(sessionId);
if (!reservation) {
return {
kind: 'unavailable',
reason: 'Runtime Host execution admission is unavailable.',
};
}
let consumed = false;
return {
kind: 'prepared',
admission: Object.freeze({
sessionId,
admit: (input: HostedExecutionAdmission) => {
if (consumed) {
return Promise.reject(new Error('Hosted Execution preparation was already consumed'));
}
if (input.sessionId !== sessionId) {
return Promise.reject(new Error('Hosted Execution preparation changed Session'));
}
consumed = true;
return this.#admit(input, reservation);
},
release: () => {
if (consumed) return;
consumed = true;
this.releaseRootReservation(reservation);
},
}),
};
}
admit(input: HostedExecutionAdmission): Promise<HostedExecutionAdmissionResult> {
return this.#admit(input);
}
#admit(
input: HostedExecutionAdmission,
preparedReservation?: RootTurnReservation,
): Promise<HostedExecutionAdmissionResult> {
if (isWorkHubCoordinationSessionId(input.sessionId)) {
return Promise.reject(
new RuntimeHostedRootUnavailableError(
input.sessionId,
WORKHUB_COORDINATION_EXECUTION_UNAVAILABLE_REASON,
),
);
}
// Admission is active work even while queued behind a Session gate. Its
// residency overlaps the execution residency until admission settles.
const residency = this.acquireRecoveryResidency();
const admission = this.runCommand(async () => {
const activeAtEntry = this.#executions.has(input.sessionId);
let reservation =
preparedReservation ?? (activeAtEntry ? undefined : this.reserveRootTurn(input.sessionId));
if (preparedReservation && this.#admissions.get(input.sessionId) !== preparedReservation) {
throw new RuntimeHostedRootConflictError(
input.sessionId,
'Hosted Execution preparation is no longer current',
);
}
if (!activeAtEntry && !reservation) {
throw new RuntimeHostedRootConflictError(
input.sessionId,
'Session already has a pending root Turn',
);
}
const canonicalInput = {
...input,
content: input.content === null ? null : normalizeMessageContent(input.content),
};
const admissionTask = this.sessionAdmission.run(input.sessionId, async (lease) => {
const existing = await this.stores.agentRunStore.readRootTurnAdmission(
input.sessionId,
input.turnId,
);
if (existing) {
this.rootAdmissionOwner.assertKnownAdmission(existing);
if (
existing.runId !== input.runId ||
existing.userMessageId !== input.userMessageId ||
!isDeepStrictEqual(existing.execution, input.execution) ||
!isDeepStrictEqual(existing.turnOrchestration, input.turnOrchestration) ||
!hostedExecutionContentMatches(existing, canonicalInput.content) ||
existing.sourceMessages.length !== 0
) {
throw new RuntimeMessageAuthorityInvariantError(
'Hosted root execution identity conflicts with its durable admission',
);
}
await runHostedExecutionAdmissionGate(canonicalInput.admitExecution);
return this.prepareAdmittedTurn(
canonicalInput,
existing,
this.acquireRecoveryResidency,
lease,
undefined,
canonicalInput,
reservation,
);
}
reservation ??= this.reserveRootTurn(input.sessionId);
if (!reservation) {
throw new RuntimeHostedRootConflictError(
input.sessionId,
'Session already has an active or pending root Turn',
);
}
let header: SessionHeader;
try {
header = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId);
} catch (error) {
if (isSessionNotFoundError(error)) {
throw new RuntimeHostedRootUnavailableError(
input.sessionId,
'Hosted root execution target Session is unavailable',
{ cause: error },
);
}
throw error;
}
if (header.isArchived) {
throw new RuntimeHostedRootUnavailableError(
input.sessionId,
'Cannot start a hosted root execution in an archived Session',
);
}
const unavailableReason = runtimeHostExecutionUnavailableReason(header, input.execution);
if (unavailableReason) {
throw new RuntimeHostedRootUnavailableError(input.sessionId, unavailableReason);
}
if (this.#executions.has(input.sessionId)) {
throw new RuntimeHostedRootConflictError(
input.sessionId,
'Session already has an active root Turn',
);
}
if (reservation && this.#admissions.get(input.sessionId) !== reservation) {
throw new RuntimeHostedRootConflictError(
input.sessionId,
'Hosted root execution lost its pending reservation',
);
}
await runHostedExecutionAdmissionGate(canonicalInput.admitExecution);
if (!reservation || !this.beginRootAdmission(reservation)) {
throw new RuntimeHostedRootConflictError(
input.sessionId,
'Hosted root execution lost its pending reservation',
);
}
const admitted = await this.rootAdmissionOwner.admitRootTurn({
sessionId: input.sessionId,
turnId: input.turnId,
proposedRunId: input.runId,
proposedUserMessageId: input.userMessageId,
execution: input.execution,
normalizedInput: canonicalInput.content,
...(input.turnOrchestration ? { turnOrchestration: input.turnOrchestration } : {}),
sourceMessages: [],
admittedAt: Date.now(),
});
if (
admitted.admission.runId !== input.runId ||
admitted.admission.userMessageId !== input.userMessageId ||
!isDeepStrictEqual(admitted.admission.execution, input.execution) ||
!hostedExecutionContentMatches(admitted.admission, canonicalInput.content) ||
admitted.admission.sourceMessages.length !== 0
) {
throw new RuntimeMessageAuthorityInvariantError(
'Hosted root execution admission changed identity',
);
}
return this.prepareAdmittedTurn(
canonicalInput,
admitted.admission,
this.acquireRecoveryResidency,
lease,
undefined,
canonicalInput,
reservation,
);
});
const disposition = await admissionTask.finally(() => {
if (reservation) this.releaseRootReservation(reservation);
});
if (disposition.kind === 'complete') {
if (!disposition.outcome.ok) {
if (disposition.outcome.error.code === 'session_busy') {
throw new RuntimeHostedRootConflictError(
input.sessionId,
disposition.outcome.error.message,
);
}
if (disposition.outcome.error.code === 'operation_unavailable') {
throw new RuntimeHostedRootUnavailableError(
input.sessionId,
disposition.outcome.error.message,
);
}
throw new RuntimeMessageAuthorityInvariantError(disposition.outcome.error.message);
}
this.#executions.publish(disposition.outcome.result);
return completedHostedExecutionAdmission(disposition.outcome.result);
}
await disposition.active.startSettled.promise;
const snapshot = await this.readCanonicalSnapshot(input.sessionId, input.turnId, input.runId);
this.#executions.publish(snapshot);
return Object.freeze({
snapshot,
completion: disposition.active.completion.promise,
settled: disposition.active.done,
});
}).catch((error) => {
if (error instanceof HostedRootAdmissionGateError) throw error.cause;
throw error;
});
return admission.finally(() => residency.release());
}
lookup(sessionId: string, turnId: string): Promise<HostedExecutionIdentity | undefined> {
return this.runCommand(async () => {
const admission = await this.stores.agentRunStore.readRootTurnAdmission(sessionId, turnId);
if (!admission) return undefined;
this.rootAdmissionOwner.assertKnownAdmission(admission);
return {
sessionId,
turnId,
runId: admission.runId,
userMessageId: admission.userMessageId,
descriptor: admission.execution,
};
});
}
read(execution: HostedExecutionRef): Promise<HostedExecutionSnapshot> {
return this.runCommand(() =>
this.readCanonicalSnapshot(execution.sessionId, execution.turnId, execution.runId),
);
}
async requestStop(input: HostedExecutionStopInput): Promise<HostedExecutionSnapshot> {
await this.stopRoot(input.execution, {
...(input.source ? { source: input.source } : {}),
...(input.workHubActionId !== undefined ? { workHubActionId: input.workHubActionId } : {}),
...(input.mode ? { mode: input.mode } : {}),
} as StopSessionInput);
return await this.read(input.execution);
}
reconcile(execution: HostedExecutionRef): Promise<HostedExecutionSnapshot> {
return this.read(execution);
}
subscribe(listener: HostedExecutionListener): () => void {
return this.#executions.subscribe(listener);
}
whenIdle(sessionId: string): Promise<void> | undefined {
return (
this.#executions.get(sessionId)?.done ??
this.#admissions.get(sessionId)?.whenIdle.promise ??
this.#handoffHold?.promise
);
}
stopRoot(identity: RuntimeMessageRunIdentity, input: StopSessionInput = {}): Promise<void> {
normalizeStopSessionSource(input.source, input.workHubActionId);
return this.runCommand(async () => {
const declared = await this.sessionAdmission.run(identity.sessionId, (lease) =>
this.declareStopFence(
identity,
() => this.messages.commitStopFence(identity),
lease,
input,
),
);
await declared?.deliverStop();
await declared?.active.startSettled.promise;
const disposition = await this.sessionAdmission.run(identity.sessionId, (lease) =>
this.prepareStopDisposition(identity, () => this.messages.commitStopFence(identity), lease),
);
if (disposition.kind === 'complete') {
if (!disposition.outcome.ok) throwHostedStopError(identity.sessionId, disposition.outcome);
return;
}
if (disposition.kind === 'request_stop') {
await this.deliverRuntimeStopIntent(identity.sessionId, input);
}
await disposition.active.done;
});
}
stopSession(sessionId: string, input: StopSessionInput = {}): Promise<void> {
normalizeStopSessionSource(input.source, input.workHubActionId);
return this.runCommand(async () => {
const declared = await this.sessionAdmission.run(sessionId, (lease) => {
const active = this.#executions.get(sessionId);
if (!active) return undefined;
const identity = {
sessionId,
turnId: active.turnId,
runId: active.runId,
};
return this.declareStopFence(
identity,
() => this.messages.commitStopFence(identity),
lease,
input,
);
});
await declared?.deliverStop();
await declared?.active.startSettled.promise;
const disposition = await this.sessionAdmission.run(sessionId, async (lease) => {
if (!declared) return undefined;
const identity = {
sessionId,
turnId: declared.active.turnId,
runId: declared.active.runId,
};
return this.prepareStopDisposition(
identity,
() => this.messages.commitStopFence(identity),
lease,
);
});
if (!disposition || disposition.kind === 'complete') {
if (disposition && !disposition.outcome.ok) {
throwHostedStopError(sessionId, disposition.outcome);
}
return;
}
if (disposition.kind === 'request_stop') {
await this.deliverRuntimeStopIntent(sessionId, input);
}
await disposition.active.done;
});
}
async stopAgentGraphSupervisor(
sessionId: string,
input: {
expectedGraphId?: string;
source?: 'stop_button' | 'graph_supervisor';
mode?: BackendStopMode;
} = {},
): Promise<void> {
const identity = await this.runCommand(() =>
this.sessionAdmission.run(sessionId, async () => {
const graphId = input.expectedGraphId ?? (await this.resolveCurrentGraphId(sessionId));
const active = this.#executions.get(sessionId);
if (!active?.graphOwnerId) return undefined;
if (active.graphOwnerId !== graphId) {
if (input.expectedGraphId !== undefined) {
throw new RuntimeHostedRootConflictError(
sessionId,
`Agent graph ${input.expectedGraphId} is no longer current`,
);
}
return undefined;
}
return {
sessionId,
turnId: active.turnId,
runId: active.runId,
};
}),
);
if (identity) await this.stopRoot(identity, input);
}
startFromMessage(
input: HostMessageStartInput,
admissionLease: SessionAdmissionLease,
commitAdmission: (
canonicalContent: MessageContent,
skillInvocation: SkillInvocationResult,
) => Promise<void>,
): Promise<HostMessageStartOutcome> {
if (isWorkHubCoordinationSessionId(input.sessionId)) {
return Promise.resolve({ error: WORKHUB_COORDINATION_EXECUTION_UNAVAILABLE_REASON });
}
return this.runCommand(async () => {
const content = normalizeMessageContent(input.content);
if (
input.sourceMessage.disposition !== 'turn_started' ||
(!input.preparedSkillInvocation &&
!messageContentsEqual(input.sourceMessage.content, content))
) {
throw new RuntimeMessageAuthorityInvariantError(
'Idle Message start lost its canonical turn_started source',
);
}
if (this.#executions.has(input.sessionId)) {
throw new RuntimeMessageAuthorityInvariantError(
'Message authority attempted an idle start while a root Turn was active',
);
}
const reservation = this.reserveRootTurn(input.sessionId);
if (!reservation) return { error: 'Another root Turn is being admitted' };
try {
const header = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId);
const unavailableReason = runtimeHostExternalTurnUnavailableReason(header);
if (unavailableReason) return { error: unavailableReason };
const turnId = input.turnId ?? randomUUID();
const runId = input.runId ?? randomUUID();
const skillIds = input.skillIds ?? [];
const hasSkillInvocation =
skillIds.length > 0 || parseSkillInvocationTokens(content.text).length > 0;
const prepared = input.preparedSkillInvocation
? ({
kind: 'ready',
content,
skillInvocation: input.preparedSkillInvocation,
} as const)
: hasSkillInvocation
? await this.prepareHostedSkillInvocationContent(
input.sessionId,
turnId,
content,
skillIds,
input.initiatingConnectionId,
)
: ({
kind: 'ready',
content,
skillInvocation: { loaded: [], failed: [], receipts: [] },
} as const);
if (prepared.kind === 'rejected') {
// Skill resolution is the only rejection a client can act on, so it
// travels back as structured feedback instead of an opaque error.
if (prepared.skillInvocation) return { blocked: prepared.skillInvocation };
return {
error: prepared.outcome.ok
? 'Hosted Skill invocation was rejected'
: prepared.outcome.error.message,
};
}
const skillInvocation = prepared.skillInvocation ?? {
loaded: [],
failed: [],
receipts: [],
};
const canonicalContent = preflightRootMessageContent(
this.validateDirectoryReferences(input.sessionId, prepared.content),
);
if (!canonicalContent.ok)
return { error: 'Prepared message content exceeds durable limits' };
const binding = prepared.commitCapabilityBinding
? await prepared.commitCapabilityBinding()
: await this.clientCapabilities?.bindSession(
input.sessionId,
input.initiatingConnectionId,
);
if (binding && !binding.ok) return { error: binding.message };
if (!this.beginRootAdmission(reservation)) {
return { error: 'Root Turn reservation is no longer current' };
}
await this.prepareFreshAgentGraphEpoch(header, input.turnOrchestration);
await commitAdmission(canonicalContent.content, skillInvocation);
const admitted = await this.rootAdmissionOwner.admitRootTurn({
sessionId: input.sessionId,
turnId,
proposedRunId: runId,
proposedUserMessageId: input.sourceMessage.messageId,
execution: {
kind: 'external_message',
inputDigest:
input.sourceMessage.submittedContentDigest ?? messageContentDigest(content),
},
normalizedInput: canonicalContent.content,
...(input.turnOrchestration ? { turnOrchestration: input.turnOrchestration } : {}),
skillInvocation,
sourceMessages: [
{
...input.sourceMessage,
content: normalizeMessageContent(canonicalContent.content),
skillInvocation,
},
],
admittedAt: Date.now(),
});
if (admitted.kind !== 'admitted') {
throw new RuntimeMessageAuthorityInvariantError(
'Fresh Message root Turn identity already existed',
);
}
await this.messages.handoffRootSources({
sessionId: input.sessionId,
turnId,
runId,
messageIds: [input.sourceMessage.messageId],
});
const disposition = await this.prepareAdmittedTurn(
{
sessionId: input.sessionId,
turnId,
content: canonicalContent.content,
...(input.turnOrchestration ? { turnOrchestration: input.turnOrchestration } : {}),
},
admitted.admission,
this.acquireRecoveryResidency,
admissionLease,
undefined,
undefined,
reservation,
);
if (disposition.kind !== 'await_start') {
throw new RuntimeMessageAuthorityInvariantError(
'Fresh Message root Turn did not reserve execution',
);
}
return {
turnId,
skillInvocation,
};
} finally {
this.releaseRootReservation(reservation);
}
});
}
startRecoveredMessages(
input: HostMessageRecoveryBatch,
admissionLease: SessionAdmissionLease,
): Promise<
{ readonly turnId: string } | { readonly error: string } | { readonly deferred: true }
> {
return this.runCommand(async () => {
if (this.#executions.has(input.sessionId)) {
return { error: 'A root Turn is still active' };
}
const header = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId);
const turnId = input.rootIdentity?.turnId ?? randomUUID();
const previous = this.rootAdmissionOwner.latestAdmission(input.sessionId)?.execution;
let execution: RootExecutionDescriptor = {
kind: isWorkHubCoordinationSessionId(header.id)
? ('workhub_coordination' as const)
: ('external_message' as const),
inputDigest: messageContentDigest(input.submittedContent),
...(previous?.kind === 'workhub_coordination' && previous.capabilityBinding
? { capabilityBinding: previous.capabilityBinding }
: {}),
};
if (!(await this.bindRecoveryCapabilities(input.sessionId, execution)))
return { deferred: true };
execution = await this.prepareFreshWorkHubExecution(header, turnId, input.content, execution);
const unavailableReason = runtimeHostExecutionUnavailableReason(header, execution);
if (unavailableReason) return { error: unavailableReason };
const reservation = this.reserveRootTurn(input.sessionId);
if (!reservation) return { error: 'Another root Turn is being admitted' };
try {
// The recovered Message asked for this mode before the Host stopped;
// admitting without it would run a different Turn than was requested.
const turnOrchestration = input.submittedIntent?.turnOrchestration;
await this.prepareFreshAgentGraphEpoch(header, turnOrchestration);
const admitted = await this.rootAdmissionOwner.admitRootTurn({
sessionId: input.sessionId,
turnId,
proposedRunId: input.rootIdentity?.runId ?? randomUUID(),
proposedUserMessageId: input.sources.length === 1 ? input.sources[0]!.messageId : null,
execution,
normalizedInput: input.content,
...(turnOrchestration ? { turnOrchestration } : {}),
sourceMessages: input.sources,
admittedAt: Date.now(),
});
if (admitted.kind !== 'admitted') {
return { error: 'Recovered Message root identity already existed' };
}
await this.messages.handoffRootSources({
sessionId: input.sessionId,
turnId,
runId: admitted.admission.runId,
messageIds: input.sources.map((source) => source.messageId),
});
const disposition = await this.prepareAdmittedTurn(
{ sessionId: input.sessionId, turnId, content: input.content },
admitted.admission,
this.acquireRecoveryResidency,
admissionLease,
undefined,
undefined,
reservation,
);
if (disposition.kind !== 'await_start') {
return { error: 'Recovered Message root did not reserve execution' };
}
return { turnId };
} catch (error) {
this.#admissions.release(reservation);
throw error;
}
});
}
private async prepareFreshWorkHubExecution(
header: SessionHeader,
turnId: string,
content: MessageContent,
execution: RootExecutionDescriptor,
inputClosedSignal?: AbortSignal,
): Promise<RootExecutionDescriptor> {
if (execution.kind !== 'workhub_coordination' || !this.prepareWorkHubRoutingDecision) {
return execution;
}
return {
...execution,
routingDecision: await this.prepareWorkHubRoutingDecision({
header,
turnId,
content,
...(inputClosedSignal ? { inputClosedSignal } : {}),
}),
};
}
prepareMessage(input: HostMessagePreparationInput): Promise<
| {
readonly kind: 'ready';
readonly content: MessageContent;
readonly skillInvocation: SkillInvocationResult;
}
| {
readonly kind: 'rejected';
readonly error: string;
readonly skillInvocation?: SkillInvocationResult;
}
> {
return this.runCommand(async () => {
const content = normalizeMessageContent(input.content);
if (parseSkillInvocationTokens(content.text).length === 0) {
return {
kind: 'ready',
content: this.validateDirectoryReferences(input.sessionId, content),
skillInvocation: { loaded: [], failed: [], receipts: [] },
};
}
const prepare = async () => {
const prepared = await this.prepareSkillInvocationContent(
input.sessionId,
input.turnId,
content,
[],
);
return prepared.kind === 'ready'
? {
...prepared,
content: this.validateDirectoryReferences(input.sessionId, prepared.content),
}
: prepared;
};
if (input.placement === 'current_turn') return prepare();
const preview = await this.previewCapabilityBinding(input.sessionId, '', prepare);
return preview.ok ? preview.value : { kind: 'rejected', error: preview.message };
});
}
private validateDirectoryReferences(sessionId: string, content: MessageContent): MessageContent {
if (!content.directoryReferences?.length) return content;
if (
!this.directoryHostId ||
content.directoryReferences.some((reference) => reference.hostId !== this.directoryHostId)
) {
throw new RuntimeHostedRootUnavailableError(
sessionId,
'Directory references belong to a different Runtime Host',
);
}
return content;
}
claimStop(
input: Pick<TurnStopInput, 'sessionId' | 'turnId' | 'runId'>,
commitQueueFence: () => QueueFenceResult,
admission: SessionAdmissionLease,
): Promise<HostMessageStopClaim> {
return this.runCommand(async () => {
const disposition = await this.prepareStopDisposition(input, commitQueueFence, admission);
if (disposition.kind === 'complete') {
if (!disposition.outcome.ok) {
throw new RuntimeMessageAuthorityInvariantError(
'Message interrupt no longer matched its admitted root Turn',
);
}
return {
deliverStop: () => Promise.resolve(),
terminal: Promise.resolve(disposition.outcome.result),
};
}
return {
deliverStop: () =>
disposition.kind === 'request_stop'
? this.deliverRuntimeStopIntent(input.sessionId)
: Promise.resolve(),
terminal: disposition.active.done.then(() =>
this.readCanonicalSnapshot(input.sessionId, input.turnId, input.runId),
),
};
});
}
claimStopFence(
input: Pick<TurnStopInput, 'sessionId' | 'turnId' | 'runId'>,
commitQueueFence: () => QueueFenceResult,
admission: SessionAdmissionLease,
): Promise<HostMessageStopFence> {
return this.declareStopFence(input, commitQueueFence, admission).then((declared) => ({
ready: declared?.active.startSettled.promise ?? Promise.resolve(),
deliverStop: declared?.deliverStop ?? (() => Promise.resolve()),
}));
}
async prepareHostedSkillInvocationContent(
sessionId: string,
turnId: string,
content: MessageContent,
skillIds: readonly string[],
connectionId: string,
): Promise<RootMessageContentPreparation> {
const preview = await this.previewCapabilityBinding(sessionId, connectionId, () =>
this.prepareSkillInvocationContent(sessionId, turnId, content, skillIds),
);
if (!preview.ok) {
return { kind: 'rejected', outcome: operationConflict(preview.message) };
}
if (preview.value.kind === 'rejected') {
return {
kind: 'rejected',
outcome: operationConflict(preview.value.error),
skillInvocation: preview.value.skillInvocation,
};
}
return {
kind: 'ready',
content: preview.value.content,
skillInvocation: preview.value.skillInvocation,
commitCapabilityBinding: preview.commit,
};
}
private async prepareSkillInvocationContent(
sessionId: string,
turnId: string,
content: MessageContent,
skillIds: readonly string[],
): Promise<
| {
readonly kind: 'ready';
readonly content: MessageContent;
readonly skillInvocation: SkillInvocationResult;
}
| {
readonly kind: 'rejected';
readonly error: string;
readonly skillInvocation?: SkillInvocationResult;
}
> {
if (!this.prepareSkillInvocation) {
return {
kind: 'rejected',
error: 'Hosted Skill invocation authority is unavailable',
};
}
const prepared = await this.prepareSkillInvocation({
sessionId,
turnId,
text: content.text,
skillIds,
});
let skillInvocation: SkillInvocationResult;
try {
skillInvocation = decodeSkillInvocationResult(prepared.skillInvocation);
} catch {
return {
kind: 'rejected',
error: 'Hosted Skill invocation feedback is invalid',
};
}
return prepared.disposition === 'blocked'
? {
kind: 'rejected',
error: 'Explicit Skill invocation could not be resolved',
skillInvocation,
}
: {
kind: 'ready',
content: composeHostedSkillInvocationContent(content, {
...prepared,
skillInvocation,
}),
skillInvocation,
};
}
startInteractiveRootMessage(
request: RootMessageStartRequest,
context: ConnectionContext,
): Promise<RootMessageStartOutcome> {
if (isWorkHubCoordinationSessionId(request.sessionId)) {
return Promise.resolve(
operationUnavailable(WORKHUB_COORDINATION_EXECUTION_UNAVAILABLE_REASON),
);
}
if (request.execution.kind === 'workhub_coordination') {
return Promise.resolve(
operationUnavailable(WORKHUB_COORDINATION_EXECUTION_UNAVAILABLE_REASON),
);
}
return this.startRootMessage(request, context);
}
/** Dedicated WorkHub authority; the ordinary interactive entry stays closed. */
startWorkHubCoordinationMessage(
request: Extract<RootMessageStartRequest, { execution: { kind: 'workhub_coordination' } }>,
context: ConnectionContext,
): Promise<RootMessageStartOutcome> {
if (!isWorkHubCoordinationSessionId(request.sessionId)) {
return Promise.resolve(
operationUnavailable('WorkHub Coordination execution requires its reserved Session'),
);
}
if ((request.execution.operation === 'action') !== (request.operation !== undefined)) {
return Promise.resolve(
operationUnavailable('Coordination execution mode does not match its runner'),
);
}
return this.startRootMessage(request, context);
}
async runWorkHubCoordinationOperation(
request: Extract<RootMessageStartRequest, { execution: { kind: 'workhub_coordination' } }>,
context: ConnectionContext,
): Promise<
{ ok: true; result: WorkHubActionReceipt } | Extract<RootMessageStartOutcome, { ok: false }>
> {
if (request.execution.operation !== 'action' || !request.operation) {
return operationUnavailable('Coordination action requires a Host operation');
}
let turnId = request.turnId;
let recoveredReceipt: WorkHubActionReceipt | undefined;
// A retry preserves the action identity, but never reopens a terminal Run.
// The existing admission chain and terminal facts identify prior attempts.
for (;;) {
const admission = await this.stores.agentRunStore.readRootTurnAdmission(
request.sessionId,
turnId,
);
if (!admission) break;
if (!rootExecutionMatches(admission.execution, request.execution))
return operationConflict('Coordination action identity belongs to different content');
const run = await this.readRunIfPresent(request.sessionId, admission.runId);
if (!run) break;
const snapshot = await this.readCanonicalSnapshot(
request.sessionId,
turnId,
admission.runId,
run,
);
if (!isTerminalSnapshot(snapshot) || snapshot.status === 'completed') break;
// A crash after the receipt does not erase a completed Host effect.
const events = await this.stores.runtimeEventStore.readImmutableRuntimeEvents(
request.sessionId,
admission.runId,
);
recoveredReceipt ??= events.find((event) => event.actions?.coordination)?.actions
?.coordination;
turnId = `whretry_${createHash('sha256').update(admission.runId).digest('hex').slice(0, 48)}`;
}
const receiptToReplay = recoveredReceipt;
const started = await this.startWorkHubCoordinationMessage(
{
...request,
turnId,
...(receiptToReplay ? { operation: async () => receiptToReplay } : {}),
},
context,
);
if (!started.ok) return started;
const active = this.#executions.get(request.sessionId);
if (active?.turnId === turnId) await active.done;
const snapshot = await this.readCanonicalSnapshot(
request.sessionId,
turnId,
started.result.runId,
);
if (snapshot.status !== 'completed')
return operationUnavailable('Coordination operation did not complete');
const events = await this.stores.runtimeEventStore.readImmutableRuntimeEvents(
request.sessionId,
started.result.runId,
);
const receipt = events.find((event) => event.actions?.coordination)?.actions?.coordination;
return receipt
? { ok: true, result: receipt }
: operationUnavailable('Coordination receipt is unavailable');
}
private coordinationOperation(
request: RootMessageStartRequest,
admission: RootTurnAdmission,
): HostedExecutionAdmission | undefined {
if (
request.execution.kind !== 'workhub_coordination' ||
!('operation' in request) ||
!request.operation
)
return undefined;
const execute = request.operation;
const content = requireHostedExecutionMessageContent(admission);
return {
sessionId: admission.sessionId,
turnId: admission.turnId,
runId: admission.runId,
userMessageId: admission.userMessageId,
execution: admission.execution,
content,
start: ({ runId, userMessageId, onRunStarted }) =>
this.manager.runCoordinationOperation(
admission.sessionId,
{ turnId: admission.turnId, ...content },
{ runId, userMessageId, onRunStarted },
() => execute(admission.turnId),
),
};
}
/** Accepts a model action against the exact live WorkHub root and its durable user input. */
async readActiveWorkHubRequest(turnId: string): Promise<MessageContent | undefined> {
return (await this.readActiveWorkHubRoutingRequest(turnId))?.content;
}
/** Reads both user authority and the advisory routing decision of the exact live WorkHub Turn. */
readActiveWorkHubRoutingRequest(turnId: string): Promise<
| {
readonly content: MessageContent;
readonly runId: string;
readonly decision?: WorkHubRoutingDecision;
}
| undefined
> {
const sessionId = WORKHUB_COORDINATION_SESSION_ID;
return this.sessionAdmission.run(sessionId, async () => {
const active = this.#executions.get(sessionId);
if (!active || active.turnId !== turnId) return undefined;
const [admission, header] = await Promise.all([
this.stores.agentRunStore.readRootTurnAdmission(sessionId, turnId),
this.stores.sessionStore.readHeaderSnapshot(sessionId),
]);
if (
this.#executions.get(sessionId) !== active ||
admission?.runId !== active.runId ||
admission.execution.kind !== 'workhub_coordination' ||
header.toolProfile !== 'workhub-coordination-v2'
)
return undefined;
return {
runId: active.runId,
content: requireHostedExecutionMessageContent(admission),
...(admission.execution.routingDecision
? { decision: admission.execution.routingDecision }
: {}),
};
});
}
private startRootMessage(
request: RootMessageStartRequest,
context: ConnectionContext,
): Promise<RootMessageStartOutcome> {
return this.runCommand(async () => {
await this.awaitTerminalRootCleanup(request.sessionId);
const activeAtEntry = this.#executions.has(request.sessionId);
let reservation = activeAtEntry ? undefined : this.reserveRootTurn(request.sessionId);
if (!activeAtEntry && !reservation) {
return sessionBusy('Session already has a pending root Turn');
}
const admissionTask = this.sessionAdmission.run(request.sessionId, async (lease) => {
const existing = await this.stores.agentRunStore.readRootTurnAdmission(
request.sessionId,
request.turnId,
);
if (existing) {
this.rootAdmissionOwner.assertKnownAdmission(existing);
if (existing.execution.kind !== request.execution.kind) {
return completedStart(
operationConflict('Turn identity belongs to a different execution kind'),
);
}
if (!rootExecutionMatches(existing.execution, request.execution)) {
return completedStart(
operationConflict('Turn identity belongs to a different execution payload'),
);
}
let content: MessageContent;
if (request.prepareReplayContent) {
const prepared = await request.prepareReplayContent(lease);
if (prepared.kind === 'rejected') return completedStart(prepared.outcome);
content = normalizeMessageContent(prepared.content);
} else {
content =
'content' in request
? request.content
: requireHostedExecutionMessageContent(existing);
}
if (
!rootMessageAdmissionMatches(
existing,
request,
content,
context.turnAdmissionAuthorization,
)
) {
return completedStart(
operationConflict('Turn identity was already admitted with a different payload'),
);
}
const existingRun = await this.readRunIfPresent(request.sessionId, existing.runId);
if (existingRun) {
const snapshot = await this.readCanonicalSnapshot(
request.sessionId,
request.turnId,
existing.runId,
existingRun,
);
if (isTerminalSnapshot(snapshot)) {
return completedStart({ ok: true, result: snapshot });
}
}
return this.prepareAdmittedTurn(
activationInputForAdmission(existing),
existing,
context.acquireResidency,
lease,
undefined,
this.coordinationOperation(request, existing),
reservation,
);
}
reservation ??= this.reserveRootTurn(request.sessionId);
if (!reservation) {
return completedStart(sessionBusy('Session already has an active or pending root Turn'));
}
let header: SessionHeader;
try {
header = await this.stores.sessionStore.readHeaderSnapshot(request.sessionId);
} catch (error) {
if (isSessionNotFoundError(error)) {
return completedStart(notFound('Session does not exist'));
}
throw error;
}
if (header.isArchived) {
return completedStart(sessionArchived(request.archivedMessage));
}
const unavailableReason = runtimeHostExecutionUnavailableReason(header, request.execution);
if (unavailableReason) return completedStart(operationUnavailable(unavailableReason));
if (this.#executions.has(request.sessionId)) {
return completedStart(sessionBusy('Session already has an active root Turn'));
}
if (this.#admissions.get(request.sessionId) !== reservation) {
return completedStart(sessionBusy('Root Turn reservation is no longer current'));
}
if (
request.execution.kind === 'regenerate' &&
(await this.manager.listTurns(request.sessionId)).some(
(turn) => turn.turnId === request.turnId,
)
) {
return completedStart(operationConflict('Turn identity already exists'));
}
const prepared = await this.prepareRootMessageContent(request, lease);
if (prepared.kind === 'rejected') return completedStart(prepared.outcome);
const canonicalContent = preflightRootMessageContent(
this.validateDirectoryReferences(request.sessionId, prepared.content),
);
if (!canonicalContent.ok) return completedStart(canonicalContent.outcome);
const attachments = canonicalContent.content.attachments ?? [];
if (attachments.length > 0 && !this.attachmentValidator) {
return completedStart(operationConflict('Hosted attachment authority is unavailable'));
}
const attachmentError = await this.attachmentValidator?.validateTurnAttachments(
request.sessionId,
attachments,
);
if (attachmentError) return completedStart(operationConflict(attachmentError));
const isWorkHubV2 =
request.execution.kind === 'workhub_coordination' &&
header.toolProfile === 'workhub-coordination-v2';
const workHubBinding = isWorkHubV2
? await this.clientCapabilities?.bindSession(
request.sessionId,
context.connectionId,
hostedExecutionRunProfile(header.toolProfile)!.toolNames.filter((name) =>
name.startsWith('mcp__'),
),
)
: undefined;
const binding = isWorkHubV2
? workHubBinding
: request.execution.kind === 'workhub_coordination' &&
header.toolProfile === 'workhub-coordination-v1'
? undefined
: prepared.commitCapabilityBinding
? await prepared.commitCapabilityBinding()
: await this.clientCapabilities?.bindSession(request.sessionId, context.connectionId);
if (binding && !binding.ok) {
return completedStart(operationConflict(binding.message));
}
const capabilityBinding = workHubBinding?.ok ? workHubBinding.capabilityBinding : undefined;
if (isWorkHubV2 && !capabilityBinding) {
return completedStart(
operationUnavailable('WorkHub Desktop capability binding is unavailable'),
);
}
const freshExecution = await this.prepareFreshWorkHubExecution(
header,
request.turnId,
canonicalContent.content,
{
...request.execution,
...(capabilityBinding ? { capabilityBinding } : {}),
},
context.inputClosedSignal,
);
if (!this.beginRootAdmission(reservation)) {
return completedStart(sessionBusy('Root Turn reservation is no longer current'));
}
if (request.execution.kind === 'external_message') {
await this.prepareFreshAgentGraphEpoch(header, request.turnOrchestration);
}
const admitted = await this.rootAdmissionOwner.admitRootTurn({
sessionId: request.sessionId,
turnId: request.turnId,
proposedRunId: randomUUID(),
// The interactive send's operation identity is also its canonical
// user-message identity. Clients can therefore render immediately
// and let the durable transcript replace that row in place. Other
// Turn kinds do not carry a user message and retain their own
// generated admission identity.
proposedUserMessageId:
request.execution.kind === 'external_message' ? request.turnId : randomUUID(),
execution: freshExecution,
normalizedInput: canonicalContent.content,
...(request.turnOrchestration ? { turnOrchestration: request.turnOrchestration } : {}),
...(prepared.skillInvocation ? { skillInvocation: prepared.skillInvocation } : {}),
...(context.turnAdmissionAuthorization
? { authorization: context.turnAdmissionAuthorization }
: {}),
sourceMessages: [],
admittedAt: Date.now(),
});
if (admitted.admission.execution.kind !== request.execution.kind) {
return completedStart(
operationConflict('Turn identity belongs to a different execution kind'),
);
}
if (
!rootMessageAdmissionMatches(
admitted.admission,
request,
canonicalContent.content,
context.turnAdmissionAuthorization,
)
) {
return completedStart(
operationConflict('Turn identity was already admitted with a different payload'),
);
}
return this.prepareAdmittedTurn(
activationInputForAdmission(admitted.admission),
admitted.admission,
context.acquireResidency,
lease,
undefined,
this.coordinationOperation(request, admitted.admission),
reservation,
);
});
const disposition = await admissionTask.finally(() => {
if (reservation) this.releaseRootReservation(reservation);
});
return this.resolveStartDisposition(request, disposition);
});
}
private async awaitTerminalRootCleanup(sessionId: string): Promise<void> {
const active = this.#executions.get(sessionId);
if (!active) return;
let snapshot: TurnSnapshot;
try {
snapshot = await this.readCanonicalSnapshot(sessionId, active.turnId, active.runId);
} catch {
return;
}
if (isTerminalSnapshot(snapshot)) await active.done;
}
private async prepareRootMessageContent(
request: RootMessageStartRequest,
lease: SessionAdmissionLease,
): Promise<RootMessageContentPreparation> {
if ('content' in request) return { kind: 'ready', content: request.content };
if ('prepareFreshContent' in request) return request.prepareFreshContent(lease);
try {
return {
kind: 'ready',
content: normalizeMessageContent(await request.prepareContent()),
};
} catch (error) {
if (error instanceof RuntimeRegenerateTurnError) {
return {
kind: 'rejected',
outcome:
error.code === 'not_found' ? notFound(error.message) : operationConflict(error.message),
};
}
throw error;
}
}
private async queryTurnResume(
input: TurnResumeQueryInput,
context: ConnectionContext,
): Promise<OperationOutcome<'turn.resume.query'>> {
if (isWorkHubCoordinationSessionId(input.sessionId)) {
return operationUnavailable(WORKHUB_COORDINATION_EXECUTION_UNAVAILABLE_REASON);
}
return this.sessionAdmission.run(input.sessionId, async () => {
let header: SessionHeader;
try {
header = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId);
} catch (error) {
if (isSessionNotFoundError(error)) return notFound('Session does not exist');
throw error;
}
if (header.isArchived) {
return sessionArchived('Cannot continue an archived Session');
}
const unavailableReason = runtimeHostSafeBoundaryContinuationUnavailableReason(header);
if (unavailableReason) return operationUnavailable(unavailableReason);
const reservation = this.#admissions.get(input.sessionId);
const parkedAdmission = this.parkedContinuationAdmission(input.sessionId);
if (
this.#executions.has(input.sessionId) ||
(reservation &&
(!parkedAdmission || !parkedContinuationMatchesQuery(parkedAdmission, input)))
) {
return {
ok: true,
result: parkedTurnResumePlan(input.sessionId, 'session_busy'),
};
}
const preview = await this.previewCapabilityBinding(
input.sessionId,
context.connectionId,
() => this.planTurnResume(input),
);
return preview.ok
? { ok: true, result: preview.value }
: {
ok: true,
result: parkedTurnResumePlan(input.sessionId, 'safety_check_failed'),
};
});
}
private async previewCapabilityBinding<T>(
sessionId: string,
initiatingConnectionId: string,
operation: () => Promise<T>,
): Promise<SessionBindingPreview<T>> {
if (this.clientCapabilities) {
return this.clientCapabilities.runWithSessionBindingPreview(
sessionId,
initiatingConnectionId,
operation,
);
}
return {
ok: true,
value: await operation(),
commit: async () => ({ ok: true }),
};
}
private startTurnResume(
input: TurnResumeStartInput,
context: ConnectionContext,
): Promise<TurnResumeStartOutcome> {
if (isWorkHubCoordinationSessionId(input.sessionId)) {
return Promise.resolve(
operationUnavailable(WORKHUB_COORDINATION_EXECUTION_UNAVAILABLE_REASON),
);
}
return this.runCommand(async () => {
const turnInput = continuationTurnInput(input.sessionId, input.turnId);
const activeAtEntry = this.#executions.has(input.sessionId);
let reservation = activeAtEntry ? undefined : this.reserveRootTurn(input.sessionId);
const disposition = await this.sessionAdmission
.run<TurnResumeStartDisposition>(input.sessionId, async (lease) => {
const existing = await this.stores.agentRunStore.readRootTurnAdmission(
input.sessionId,
input.turnId,
);
if (existing) {
this.rootAdmissionOwner.assertKnownAdmission(existing);
if (
existing.execution.kind !== 'safe_boundary_continuation' ||
existing.execution.sourceRunId !== input.sourceRunId ||
existing.execution.sourceRuntimeEventHighWater !== input.sourceRuntimeEventHighWater
) {
return {
kind: 'complete',
outcome: operationConflict(
'Turn identity was already admitted for a different continuation boundary',
),
};
}
const header = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId);
const unavailableReason = runtimeHostSafeBoundaryContinuationUnavailableReason(header);
if (unavailableReason) {
return {
kind: 'complete',
outcome: operationUnavailable(unavailableReason),
};
}
const active = this.#executions.get(input.sessionId);
if (active?.turnId === existing.turnId && active.runId === existing.runId) {
return { kind: 'await_start', active };
}
if (active) {
return {
kind: 'complete',
outcome: sessionBusy('Session already has an active root Turn'),
};
}
const existingRun = await this.readRunIfPresent(input.sessionId, existing.runId);
if (existingRun) {
await this.assertRunMatchesDurableExecution(
existingRun,
existing.turnId,
existing.execution,
);
const snapshot = await this.readCanonicalSnapshot(
input.sessionId,
input.turnId,
existing.runId,
existingRun,
);
if (isTerminalSnapshot(snapshot)) {
this.clearParkedContinuationAdmission(existing);
return {
kind: 'complete',
outcome: { ok: true, result: snapshot },
};
}
const reconstructed = await this.reconstructAdmittedContinuation(existing);
if (reconstructed.disposition === 'parked') {
return { kind: 'parked', plan: reconstructed.plan };
}
throw new RuntimeMessageAuthorityInvariantError(
`Non-terminal continuation Turn ${existing.turnId} became replayable`,
);
}
const preview = await this.previewCapabilityBinding(
input.sessionId,
context.connectionId,
() => this.reconstructAdmittedContinuation(existing),
);
if (!preview.ok) {
return {
kind: 'complete',
outcome: operationConflict(preview.message),
};
}
const reconstructed = preview.value;
if (reconstructed.disposition === 'parked') {
return { kind: 'parked', plan: reconstructed.plan };
}
const binding = await preview.commit();
if (!binding.ok) {
return {
kind: 'complete',
outcome: operationConflict(binding.message),
};
}
reservation ??= this.takeParkedContinuationReservation(existing);
return this.prepareAdmittedTurn(
turnInput,
existing,
context.acquireResidency,
lease,
undefined,
undefined,
reservation,
reconstructed.continuation,
);
}
reservation ??= this.reserveRootTurn(input.sessionId);
if (!reservation) {
return {
kind: 'complete',
outcome: sessionBusy('Session already has an active or pending root Turn'),
};
}
let header: SessionHeader;
try {
header = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId);
} catch (error) {
if (isSessionNotFoundError(error)) {
return {
kind: 'complete',
outcome: notFound('Session does not exist'),
};
}
throw error;
}
if (header.isArchived) {
return {
kind: 'complete',
outcome: sessionArchived('Cannot continue an archived Session'),
};
}
const unavailableReason = runtimeHostSafeBoundaryContinuationUnavailableReason(header);
if (unavailableReason) {
return {
kind: 'complete',
outcome: operationUnavailable(unavailableReason),
};
}
if (this.#executions.has(input.sessionId)) {
return {
kind: 'complete',
outcome: sessionBusy('Session already has an active root Turn'),
};
}
if (this.#admissions.get(input.sessionId) !== reservation) {
return {
kind: 'complete',
outcome: sessionBusy('Root Turn reservation is no longer current'),
};
}
const preview = await this.previewCapabilityBinding(
input.sessionId,
context.connectionId,
() =>
this.manager.planAuthoritativeSafeBoundaryContinuation(input.sessionId, {
sourceRunId: input.sourceRunId,
expectedRuntimeEventHighWater: input.sourceRuntimeEventHighWater,
}),
);
if (!preview.ok) {
return {
kind: 'complete',
outcome: operationConflict(preview.message),
};
}
const plan = preview.value;
const projection = projectTurnResumePlan(input.sessionId, plan);
if (projection.disposition === 'parked') {
return { kind: 'parked', plan: projection };
}
const planned = requirePlannedContinuation(plan);
if (planned.sourceTurnId === input.turnId) {
return {
kind: 'complete',
outcome: operationConflict(
'Continuation Turn identity must differ from its source Turn',
),
};
}
const continuation = { ...planned, turnId: input.turnId };
const execution = continuationExecutionDescriptor(continuation);
const binding = await preview.commit();
if (!binding.ok) {
return {
kind: 'complete',
outcome: operationConflict(binding.message),
};
}
if (!this.beginRootAdmission(reservation)) {
return {
kind: 'complete',
outcome: sessionBusy('Root Turn reservation is no longer current'),
};
}
const admitted = await this.rootAdmissionOwner.admitRootTurn({
sessionId: input.sessionId,
turnId: input.turnId,
proposedRunId: continuation.runId,
proposedUserMessageId: null,
execution,
normalizedInput: null,
sourceMessages: [],
admittedAt: Date.now(),
});
if (admitted.kind === 'conflict') {
return {
kind: 'complete',
outcome: operationConflict(
'This root execution already has a different continuation Turn',
),
};
}
if (
admitted.admission.runId !== continuation.runId ||
!isDeepStrictEqual(admitted.admission.execution, execution)
) {
throw new RuntimeMessageAuthorityInvariantError(
'Safe-boundary continuation admission changed identity',
);
}
return this.prepareAdmittedTurn(
turnInput,
admitted.admission,
context.acquireResidency,
lease,
undefined,
undefined,
reservation,
continuation,
);
})
.finally(() => {
if (reservation) this.releaseRootReservation(reservation);
});
if (disposition.kind === 'parked') {
return { ok: true, result: { kind: 'parked', plan: disposition.plan } };
}
const outcome = await this.resolveStartDisposition(turnInput, disposition);
return outcome.ok ? { ok: true, result: { kind: 'started', turn: outcome.result } } : outcome;
});
}
private async planTurnResume(input: TurnResumeQueryInput): Promise<TurnResumePlan> {
let sourceRunId = input.sourceRunId;
if (sourceRunId) {
const run = await this.readRunIfPresent(input.sessionId, sourceRunId);
if (run?.terminalEvent && runtimeHandoffPause(run.terminalEvent)) {
const logical = await readLogicalRuntimeExecutionForRun(this.stores.runtimeEventStore, run);
// Queries may name the stable public root. The returned plan and the
// subsequent start must still bind the exact physical source/high-water.
sourceRunId = logical?.tip.runId ?? sourceRunId;
}
}
const plan = sourceRunId
? await this.manager.planAuthoritativeSafeBoundaryContinuation(input.sessionId, {
sourceRunId,
...(input.expectedRuntimeEventHighWater !== undefined
? {
expectedRuntimeEventHighWater: input.expectedRuntimeEventHighWater,
}
: {}),
})
: await this.manager.planLatestAuthoritativeSafeBoundaryContinuation(input.sessionId);
return projectTurnResumePlan(input.sessionId, plan);
}
private async reconstructAdmittedContinuation(
admission: RootTurnAdmission,
): Promise<ReconstructedContinuation> {
const execution = admission.execution;
if (execution.kind !== 'safe_boundary_continuation') {
throw new RuntimeMessageAuthorityInvariantError(
'Only safe-boundary continuation admission can reconstruct a continuation',
);
}
const plan = await this.manager.planAuthoritativeSafeBoundaryContinuation(admission.sessionId, {
sourceRunId: execution.sourceRunId,
expectedRuntimeEventHighWater: execution.sourceRuntimeEventHighWater,
});
const projection = projectTurnResumePlan(admission.sessionId, plan);
if (projection.disposition === 'parked') {
return { disposition: 'parked', plan: projection };
}
const planned = requirePlannedContinuation(plan);
if (
planned.sourceInvocationId !== execution.sourceInvocationId ||
planned.sourceRunId !== execution.sourceRunId ||
planned.sourceTurnId !== execution.sourceTurnId ||
planned.sourceRuntimeEventHighWater !== execution.sourceRuntimeEventHighWater ||
planned.boundary?.manifestDigest !== execution.boundaryDigest ||
planned.providerReplayDigest !== execution.providerReplayDigest
) {
throw new RuntimeMessageAuthorityInvariantError(
'Safe-boundary continuation source proof changed after admission',
);
}
if (continuationSafetyDigest(planned) !== execution.safetyDigest) {
return {
disposition: 'parked',
plan: parkedTurnResumePlan(admission.sessionId, 'safety_check_failed'),
};
}
return {
disposition: 'ready',
continuation: {
...planned,
invocationId: execution.targetInvocationId,
runId: admission.runId,
turnId: admission.turnId,
claimId: execution.claimId,
},
};
}
private async declareStopFence(
input: Pick<TurnStopInput, 'sessionId' | 'turnId' | 'runId'>,
commitQueueFence: () => QueueFenceResult,
admission: SessionAdmissionLease,
stopInput: StopSessionInput = {},
): Promise<DeclaredStopFence | undefined> {
const active = this.#executions.get(input.sessionId);
if (!active || active.turnId !== input.turnId || active.runId !== input.runId) {
return undefined;
}
if (active.startSettled.phase === 'rejected') {
return { active, deliverStop: () => Promise.resolve() };
}
const fence = commitQueueFence();
await this.messages.cancelMessages(
input.sessionId,
fence.retracted.map((message) => message.messageId),
);
await this.interactions.claimRunClosure(
{ ...input, runId: active.continuation?.runId ?? active.runId },
'turn_stopped',
admission,
);
const shouldDeliverStop = !active.stopRequested;
active.stopRequested = stopInput;
return {
active,
deliverStop: () =>
shouldDeliverStop
? this.deliverRuntimeStopIntent(input.sessionId, stopInput)
: Promise.resolve(),
};
}
private async prepareStopDisposition(
input: Pick<TurnStopInput, 'sessionId' | 'turnId' | 'runId'>,
commitQueueFence: () => QueueFenceResult,
admissionLease: SessionAdmissionLease,
): Promise<TurnStopDisposition> {
const admission = await this.stores.agentRunStore.readRootTurnAdmission(
input.sessionId,
input.turnId,
);
if (!admission) return { kind: 'complete', outcome: notFound('Turn was not admitted') };
this.rootAdmissionOwner.assertKnownAdmission(admission);
if (admission.runId !== input.runId) {
return {
kind: 'complete',
outcome: operationConflict('Run identity does not match the admitted Turn'),
};
}
const snapshot = await this.readCanonicalSnapshot(input.sessionId, input.turnId, input.runId);
const active = this.#executions.get(input.sessionId);
if (isTerminalSnapshot(snapshot)) {
if (active?.turnId === input.turnId && active.runId === input.runId) {
const fence = commitQueueFence();
await this.messages.cancelMessages(
input.sessionId,
fence.retracted.map((message) => message.messageId),
);
active.stopRequested ??= { source: 'stop_button' };
return { kind: 'await_terminal', active };
}
return { kind: 'complete', outcome: { ok: true, result: snapshot } };
}
const parked = this.parkedContinuationAdmission(input.sessionId);
if (parked?.turnId === input.turnId && parked.runId === input.runId) {
return {
kind: 'complete',
outcome: operationConflict(
'Parked continuation cannot be stopped because no active provider execution exists',
),
};
}
if (!active) {
throw new Error('Admitted non-terminal Turn has no active Runtime Host execution');
}
if (active.turnId !== input.turnId || active.runId !== input.runId) {
return {
kind: 'complete',
outcome: operationConflict('A different root Turn owns the active Session execution'),
};
}
const fence = commitQueueFence();
await this.messages.cancelMessages(
input.sessionId,
fence.retracted.map((message) => message.messageId),
);
await this.interactions.claimRunClosure(
{ ...input, runId: active.continuation?.runId ?? active.runId },
'turn_stopped',
admissionLease,
);
const shouldRequestStop = !active.stopRequested;
active.stopRequested ??= { source: 'stop_button' };
return shouldRequestStop
? { kind: 'request_stop', active }
: { kind: 'await_terminal', active };
}
private async prepareAdmittedTurn(
input: RootTurnActivationInput,
admission: RootTurnAdmission,
acquireResidency: () => RuntimeHostResidency,
admissionLease: SessionAdmissionLease,
replacing?: ActiveRootTurn,
execution?: HostedExecutionAdmission,
rootReservation?: RootTurnReservation,
continuation?: RuntimeContinuation,
): Promise<TurnStartDisposition> {
if (admission.sessionId !== input.sessionId || admission.turnId !== input.turnId) {
throw new Error('Root Turn admission identity does not match its input');
}
const inputMatches =
admission.normalizedInput === null || input.content === null
? admission.normalizedInput === input.content
: messageContentsEqual(admission.normalizedInput, input.content);
if (!inputMatches || !isDeepStrictEqual(admission.turnOrchestration, input.turnOrchestration)) {
throw new RuntimeMessageAuthorityInvariantError(
'Root Turn admission payload does not match its input',
);
}
const session = await this.stores.sessionStore.readHeaderSnapshot(input.sessionId);
const unavailableReason =
admission.execution.kind === 'safe_boundary_continuation'
? runtimeHostSafeBoundaryContinuationUnavailableReason(session)
: runtimeHostExecutionUnavailableReason(session, admission.execution);
if (unavailableReason) {
return completedStart(operationUnavailable(unavailableReason));
}
await this.clientCapabilities?.bindDurableRoot({
sessionId: admission.sessionId,
execution: admission.execution,
});
const { runId } = admission;
const existingRun = await this.readRunIfPresent(input.sessionId, runId);
if (replacing && existingRun) {
throw new RuntimeMessageAuthorityInvariantError(
'Fresh follow-up root Turn unexpectedly had an existing Run',
);
}
if (existingRun) {
const snapshot = await this.readCanonicalSnapshot(
input.sessionId,
input.turnId,
runId,
existingRun,
);
if (isTerminalSnapshot(snapshot)) return completedStart({ ok: true, result: snapshot });
const active = this.#executions.get(input.sessionId);
if (active?.turnId === input.turnId && active.runId === runId) {
return { kind: 'await_start', active };
}
if (active) return completedStart(sessionBusy('Session already has an active root Turn'));
const logical = await readLogicalRuntimeExecution(
this.stores.runtimeEventStore,
admission,
existingRun,
);
if (!logical?.pendingHandoff) {
throw new Error('Admitted non-terminal Turn has no active Runtime Host execution');
}
const plan = await this.manager.planAuthoritativeSafeBoundaryContinuation(input.sessionId, {
sourceRunId: logical.tip.runId,
purpose: 'handoff',
});
if (!plan.continuation) {
this.parkContinuationAdmission(admission);
return completedStart(
operationUnavailable(`Sealed handoff cannot resume: ${plan.rejectionReasons.join(', ')}`),
);
}
continuation = plan.continuation;
}
const active = this.#executions.get(input.sessionId);
const currentReservation = this.#admissions.get(input.sessionId);
if (currentReservation && currentReservation !== rootReservation) {
return completedStart(sessionBusy('Another root Turn is being admitted'));
}
if (rootReservation && currentReservation !== rootReservation) {
return completedStart(sessionBusy('Root Turn reservation is no longer current'));
}
if (replacing && active !== replacing) {
throw new RuntimeMessageAuthorityInvariantError(
'Follow-up root replacement lost the previous active Turn',
);
}
if (active && active !== replacing) {
if (active.turnId !== input.turnId || active.runId !== runId) {
return completedStart(sessionBusy('Session already has an active root Turn'));
}
return { kind: 'await_start', active };
}
const graphOwnerId = await this.resolveGraphOwnerId(session, admission);
if (rootReservation && !this.beginRootAdmission(rootReservation)) {
return completedStart(sessionBusy('Root Turn reservation is no longer current'));
}
const residency = acquireResidency();
const messageIdentity = {
sessionId: input.sessionId,
turnId: input.turnId,
runId,
};
let messageReserved = false;
try {
this.messages.reserveRootTurn(messageIdentity);
messageReserved = true;
if (continuation?.handoffRootRunId !== undefined) {
await this.messages.advanceHandoffRun(messageIdentity, continuation.runId, admissionLease);
}
await this.continuity.holdTerminalPublication(
input.sessionId,
input.turnId,
runId,
admissionLease,
);
} catch (error) {
if (messageReserved) this.messages.abandonRootReservation(messageIdentity);
residency.release();
throw error;
}
const startSettled = deferred();
const completion = valueDeferred<HostedExecutionCompletion>();
const completionObserver = this.resolveExecutionObserver().begin({
sessionId: input.sessionId,
turnId: input.turnId,
runId,
descriptor: admission.execution,
});
const entry: ActiveRootTurn = {
sessionId: input.sessionId,
turnId: input.turnId,
runId,
userMessageId: admission.userMessageId,
...(execution ? { execution } : {}),
...(continuation ? { continuation } : {}),
descriptor: admission.execution,
...(graphOwnerId ? { graphOwnerId } : {}),
...(completionObserver ? { completionObserver } : {}),
completion,
startSettled,
done: Promise.resolve(),
residency,
stopRequested: undefined,
messageTransitionCommitted: false,
};
if (replacing && this.#executions.get(input.sessionId) !== replacing) {
residency.release();
throw new RuntimeMessageAuthorityInvariantError(
'Follow-up root replacement changed during execution reservation',
);
}
if (rootReservation) {
if (this.#admissions.get(input.sessionId) !== rootReservation) {
throw new RuntimeMessageAuthorityInvariantError(
'Root Turn activation lost its committing reservation',
);
}
}
this.#executions.activate(entry, replacing);
entry.done = this.sessionAdmission.detach(() => this.drainTurn(input, entry, startSettled));
void entry.done.catch(() => undefined);
if (rootReservation) {
this.#admissions.activated(rootReservation, entry.done);
}
return { kind: 'await_start', active: entry };
}
private async resolveStartDisposition(
input: Pick<RootTurnActivationInput, 'sessionId' | 'turnId'>,
disposition: TurnStartDisposition,
): Promise<RootMessageStartOutcome> {
if (disposition.kind === 'complete') return disposition.outcome;
await disposition.active.startSettled.promise;
const result = await this.readCanonicalSnapshot(
input.sessionId,
input.turnId,
disposition.active.runId,
);
return {
ok: true,
result,
};
}
/**
* The one root path that carries a user Message. Session naming hangs here
* rather than on the shared run-started hook: a compaction or a continuation
* opens a Run without new words, and neither should name a Session.
*/
private startRootMessageTurn(
input: RootTurnActivationInput,
active: ActiveRootTurn,
content: MessageContent,
messageOrigin: ReturnType<typeof hostedExecutionMessageOrigin>,
onRunStarted: () => Promise<void>,
): AsyncIterable<SessionEvent> {
return this.manager.sendMessage(
input.sessionId,
{
turnId: input.turnId,
...content,
...(active.descriptor.kind === 'regenerate'
? {
parentTurnId: active.descriptor.sourceTurnId,
regeneratedFromTurnId: active.descriptor.sourceTurnId,
}
: {}),
...(input.turnOrchestration ? { turnOrchestration: input.turnOrchestration } : {}),
...(active.descriptor.kind === 'external_message' &&
active.descriptor.maxSteps !== undefined
? { maxSteps: active.descriptor.maxSteps }
: {}),
...(messageOrigin ? { origin: messageOrigin } : {}),
},
{
runId: active.runId,
userMessageId: active.userMessageId,
durability: 'required',
onRunStarted: async (startedRunId) => {
if (startedRunId !== active.runId) {
throw new Error('Runtime started a different Run than the admitted identity');
}
await onRunStarted();
this.nameSessionFromRootMessage?.({ sessionId: input.sessionId, content });
},
},
);
}
private async drainTurn(
input: RootTurnActivationInput,
active: ActiveRootTurn,
startSettled: Deferred,
): Promise<void> {
let terminalTransitionStarted = false;
let detached = false;
try {
const messageOrigin = hostedExecutionMessageOrigin(active.descriptor);
const onRunStarted = async (): Promise<void> => {
await this.manager.commitRevisionVersion(input.sessionId);
await this.continuity.refreshCanonical(input.sessionId);
startSettled.resolve();
};
let stream = active.continuation
? this.manager.resumeSafeBoundaryContinuation(active.continuation, {
onRunStarted,
stopBeforeDispatch: () => active.stopRequested,
})
: active.execution
? active.execution.start({
runId: active.runId,
userMessageId: active.userMessageId,
onRunStarted: async () => {
await onRunStarted();
await active.execution?.onReady?.();
},
})
: active.descriptor.kind === 'context_compact'
? this.manager.compactSession(input.sessionId, {
turnId: input.turnId,
hostedRoot: {
runId: active.runId,
onRunStarted,
},
})
: active.continuation
? this.manager.resumeSafeBoundaryContinuation(active.continuation, {
onRunStarted,
})
: this.startRootMessageTurn(
input,
active,
normalizeMessageContent(requireRootMessageContent(input)),
messageOrigin,
onRunStarted,
);
for (;;) {
for await (const event of stream) {
if (active.execution?.onEvent) {
try {
active.execution.onEvent(event);
} catch {
// Presentation observers do not participate in execution authority.
}
}
if (isRuntimeSessionForwardedEvent(event)) {
await this.continuity.acceptRuntimeEvent(input.sessionId, active.runId, event);
} else if (isInteractionAnswerAck(event)) {
await this.continuity.refreshCanonical(input.sessionId);
} else if (event.type === 'user_question_request' || event.type === 'form_request') {
this.continuity.enqueueCanonicalRefresh(input.sessionId);
}
}
const logical = await readLogicalRuntimeExecution(this.stores.runtimeEventStore, active);
if (!logical?.pendingHandoff) break;
const handoff = active.handoff;
handoff?.arrived.resolve();
if (handoff && (await handoff.decision.promise) === 'detach') {
await this.sessionAdmission.run(input.sessionId, (lease) =>
this.messages.detachHandoffRoot(active, lease),
);
detached = true;
return;
}
await this.sessionAdmission.run(input.sessionId, async (lease) => {
if (this.#executions.get(input.sessionId) !== active) {
throw new RuntimeMessageAuthorityInvariantError(
'Sealed handoff lost its original root owner',
);
}
const plan = await this.manager.planAuthoritativeSafeBoundaryContinuation(
input.sessionId,
{
sourceRunId: logical.tip.runId,
purpose: 'handoff',
},
);
if (!plan.continuation)
throw new Error(`Sealed handoff cannot resume: ${plan.rejectionReasons.join(', ')}`);
await this.messages.advanceHandoffRun(active, plan.continuation.runId, lease);
active.continuation = plan.continuation;
});
stream = this.manager.resumeSafeBoundaryContinuation(active.continuation!, {
onRunStarted,
stopBeforeDispatch: () => active.stopRequested,
});
}
const snapshot = await this.readCanonicalSnapshot(
input.sessionId,
input.turnId,
active.runId,
);
await this.assertCompletedExecutionIdentity(input, active);
if (!isTerminalSnapshot(snapshot)) {
throw new Error('Runtime Turn drained without a canonical terminal fact');
}
if (snapshot.status === 'cancelled' && active.stopRequested) {
startSettled.resolve();
}
this.observeExecutionCompletion(active, { kind: 'terminal', snapshot });
await this.interruptPlanAfterUnsuccessfulTurn(input.sessionId, active, snapshot.status);
await this.materializeAdmittedMessageSources(active);
terminalTransitionStarted = true;
await this.completeTerminalTransition(input.sessionId, active);
} catch (error) {
let containedRunFailure = false;
let executionAuditFailure: unknown;
if (!terminalTransitionStarted) {
try {
const snapshot = await this.readCanonicalSnapshot(
input.sessionId,
input.turnId,
active.runId,
);
if (isTerminalSnapshot(snapshot)) {
try {
await this.assertCompletedExecutionIdentity(input, active);
} catch (auditFailure) {
executionAuditFailure = auditFailure;
}
this.observeExecutionCompletion(active, {
kind: 'terminal',
snapshot,
});
await this.interruptPlanAfterUnsuccessfulTurn(input.sessionId, active, snapshot.status);
await this.materializeAdmittedMessageSources(active);
terminalTransitionStarted = true;
await this.completeTerminalTransition(input.sessionId, active);
containedRunFailure =
executionAuditFailure === undefined &&
startSettled.phase === 'resolved' &&
((!active.stopRequested &&
snapshot.status === 'failed' &&
isContainableRunFailure(error)) ||
(Boolean(active.stopRequested) &&
snapshot.status === 'cancelled' &&
isStoppedInteractionAdmission(error)));
}
} catch {
// Preserve the execution error unless identity audit found a stronger failure.
}
}
if (containedRunFailure) return;
const commandFailure = executionAuditFailure ?? error;
this.observeExecutionCompletion(active, {
kind: 'authority_error',
execution: active,
reason: errorMessage(commandFailure),
});
startSettled.reject(commandFailure);
if (!isShutdownCancelledInteractionAdmission(commandFailure)) this.requestHostDrain();
throw commandFailure;
} finally {
if (detached) {
this.#executions.release(active);
active.residency.release();
} else {
this.observeExecutionCompletion(active, {
kind: 'authority_error',
execution: active,
reason: 'Runtime root Turn ended without a canonical completion.',
});
await active.observationSettled?.catch(() => this.requestHostDrain());
let releaseRootOwnership = active.messageTransitionCommitted;
if (!active.messageTransitionCommitted) {
try {
this.messages.abandonRootReservation({
sessionId: input.sessionId,
turnId: active.turnId,
runId: active.runId,
});
releaseRootOwnership = true;
} catch {
this.requestHostDrain();
}
}
if (releaseRootOwnership) {
this.#executions.release(active);
active.residency.release();
}
active.completion.resolve(active.observedCompletion!);
this.#executions.publish(active);
}
}
}
private async materializeAdmittedMessageSources(active: ActiveRootTurn): Promise<void> {
const admission = await this.stores.agentRunStore.readRootTurnAdmission(
active.sessionId,
active.turnId,
);
if (!admission) return;
await this.messages.materializeMessageHandoffsForRun({
sessionId: active.sessionId,
turnId: active.turnId,
runId: active.runId,
messageIds: admission.sourceMessages.map((source) => source.messageId),
});
}
private observeExecutionCompletion(
active: ActiveRootTurn,
completion: HostedExecutionCompletion,
): void {
if (active.observedCompletion) return;
active.observedCompletion = completion;
const settlement = active.completionObserver?.(completion);
if (settlement) active.observationSettled = Promise.resolve(settlement);
}
private async interruptPlanAfterUnsuccessfulTurn(
sessionId: string,
active: ActiveRootTurn,
status: string,
): Promise<void> {
if (status === 'completed' || !this.manager.hasPlanAuthority()) return;
await this.manager.interruptActivePlanExecution(
sessionId,
status === 'cancelled'
? 'Plan execution was interrupted because the Runtime root Turn was cancelled.'
: 'Plan execution was interrupted because the Runtime root Turn failed.',
`plan_interrupt_${active.runId}`,
);
}
private completeTerminalTransition(sessionId: string, active: ActiveRootTurn): Promise<void> {
return this.sessionAdmission.run(sessionId, async (lease) => {
if (this.#executions.get(sessionId) !== active) {
throw new RuntimeMessageAuthorityInvariantError(
'Terminal root Turn no longer owns the Session',
);
}
const identity = {
sessionId,
turnId: active.turnId,
runId: active.runId,
};
await this.interactions.assertTerminalFence(
{ ...identity, runId: active.continuation?.runId ?? active.runId },
lease,
);
const batch = this.messages.beginTerminalTransition(identity);
await this.continuity.publishTerminalProjection(
sessionId,
active.turnId,
active.runId,
lease,
);
if (batch.sources.length === 0) {
this.messages.completeIdle(batch);
active.messageTransitionCommitted = true;
this.#executions.release(active);
return;
}
await this.startFollowupBatch(batch, active, lease);
});
}
private async startFollowupBatch(
batch: RootFollowupBatch,
previous: ActiveRootTurn,
admissionLease: SessionAdmissionLease,
): Promise<void> {
// A confirmed follow-up must become a durable root even when a Session
// provider is unavailable. Lost and ambiguous connection-local tools are
// omitted because a queued Message belongs to the durable Session.
await this.clientCapabilities?.bindSessionSuccessor(batch.sessionId);
const turnId = randomUUID();
const header = await this.stores.sessionStore.readHeaderSnapshot(batch.sessionId);
await this.prepareFreshAgentGraphEpoch(header);
const execution = await this.prepareFreshWorkHubExecution(header, turnId, batch.content, {
kind:
previous.descriptor.kind === 'workhub_coordination'
? 'workhub_coordination'
: 'external_message',
inputDigest: messageContentDigest(batch.submittedContent),
...(previous.descriptor.kind === 'workhub_coordination' &&
previous.descriptor.capabilityBinding
? { capabilityBinding: previous.descriptor.capabilityBinding }
: {}),
});
const admitted = await this.rootAdmissionOwner.admitRootTurn({
sessionId: batch.sessionId,
turnId,
proposedRunId: randomUUID(),
proposedUserMessageId: batch.sources.length === 1 ? batch.sources[0]!.messageId : null,
execution,
normalizedInput: batch.content,
sourceMessages: batch.sources,
admittedAt: Date.now(),
});
if (admitted.kind !== 'admitted') {
throw new RuntimeMessageAuthorityInvariantError(
'Fresh follow-up root Turn identity already existed',
);
}
await this.messages.handoffRootSources({
sessionId: batch.sessionId,
turnId,
runId: admitted.admission.runId,
messageIds: batch.sources.map((source) => source.messageId),
});
const nextIdentity = {
sessionId: batch.sessionId,
turnId,
runId: admitted.admission.runId,
};
this.messages.commitNextRoot(batch, nextIdentity);
previous.messageTransitionCommitted = true;
if (this.#executions.get(batch.sessionId) !== previous) {
throw new RuntimeMessageAuthorityInvariantError(
'Follow-up transition lost the previous root Turn',
);
}
const disposition = await this.prepareAdmittedTurn(
{
sessionId: batch.sessionId,
turnId,
content: admitted.admission.normalizedInput,
},
admitted.admission,
this.acquireRecoveryResidency,
admissionLease,
previous,
);
if (disposition.kind !== 'await_start') {
throw new RuntimeMessageAuthorityInvariantError(
'Fresh follow-up root Turn did not reserve execution',
);
}
}
private async deliverRuntimeStopIntent(
sessionId: string,
input: StopSessionInput = { source: 'stop_button' },
): Promise<void> {
await this.manager.deliverHostedRootStop(sessionId, input);
}
private async stopActiveTurn(sessionId: string, active: ActiveRootTurn): Promise<void> {
await this.stopRoot({
sessionId,
turnId: active.turnId,
runId: active.runId,
});
}
private async readCanonicalSnapshot(
sessionId: string,
turnId: string,
runId: string,
knownRun?: RuntimeInvocationRecord,
): Promise<TurnSnapshot> {
return this.executionProjection.read({ sessionId, turnId, runId }, knownRun);
}
private async readRunIfPresent(
sessionId: string,
runId: string,
): Promise<RuntimeInvocationRecord | undefined> {
return this.executionProjection.readRunIfPresent(sessionId, runId);
}
private async resolveGraphOwnerId(
session: SessionHeader,
admission: RootTurnAdmission,
): Promise<string | undefined> {
if (admission.execution.kind === 'context_compact') return undefined;
const mode =
admission.execution.kind === 'safe_boundary_continuation'
? ((await this.readRunIfPresent(admission.sessionId, admission.execution.sourceRunId))
?.opening.configuration.orchestrationMode ??
resolveEffectiveOrchestration(session.orchestrationMode, undefined).mode)
: resolveEffectiveOrchestration(session.orchestrationMode, admission.turnOrchestration)
.mode;
if (mode !== 'graph' && mode !== 'swarm') return undefined;
return this.resolveCurrentGraphId(admission.sessionId);
}
private async prepareFreshAgentGraphEpoch(
session: SessionHeader,
turnOrchestration?: TurnStartInput['turnOrchestration'],
): Promise<void> {
const mode = resolveEffectiveOrchestration(session.orchestrationMode, turnOrchestration).mode;
if (mode === 'graph' || mode === 'swarm') {
await this.agentGraphEpochs?.beginNextGraphEpoch(session.id);
}
}
private async resolveCurrentGraphId(rootSessionId: string): Promise<string> {
return (
(await this.agentGraphEpochs?.currentGraphId(rootSessionId)) ??
agentGraphIdForRootSession(rootSessionId)
);
}
private async assertRunMatchesDurableExecution(
run: RuntimeInvocationRecord,
turnId: string,
execution: RootTurnAdmission['execution'],
): Promise<void> {
await this.executionProjection.assertRunIdentityAndContinuation(run, turnId, execution);
}
private async assertCompletedExecutionIdentity(
input: Pick<RootTurnActivationInput, 'sessionId' | 'turnId'>,
active: ActiveRootTurn,
): Promise<void> {
const completedRun = await this.readRunIfPresent(input.sessionId, active.runId);
if (!completedRun) {
throw new RuntimeMessageAuthorityInvariantError(
'Hosted root execution completed without its admitted Run',
);
}
await this.assertRunMatchesDurableExecution(completedRun, input.turnId, active.descriptor);
}
private async runCommand<T>(operation: () => Promise<T>): Promise<T> {
try {
return await operation();
} catch (error) {
if (
!(error instanceof RuntimeHostedRootConflictError) &&
!(error instanceof RuntimeHostedRootUnavailableError) &&
!(error instanceof HostedRootAdmissionGateError) &&
!isShutdownCancelledInteractionAdmission(error)
) {
this.requestHostDrain();
}
throw error;
}
}
}
async function waitForAbortableValue<T>(value: Promise<T>, signal: AbortSignal): Promise<T> {
await waitForHostedExecutionIdleOrAbort(
value.then(() => undefined),
signal,
);
return value;
}
class HostedRootAdmissionGateError extends Error {
readonly name = 'HostedRootAdmissionGateError';
constructor(readonly cause: unknown) {
super('Hosted root execution was rejected before Runtime admission', {
cause,
});
}
}
async function runHostedExecutionAdmissionGate(
gate: HostedExecutionAdmission['admitExecution'],
): Promise<void> {
if (!gate) return;
let admission: 'executing' | 'cancelled';
try {
admission = await gate();
} catch (error) {
throw new HostedRootAdmissionGateError(error);
}
if (admission === 'cancelled') {
throw new HostedRootAdmissionGateError(
new Error('Turn start was cancelled before Runtime admission'),
);
}
}
function throwHostedStopError(
sessionId: string,
outcome: Extract<TurnStopOutcome, { ok: false }>,
): never {
switch (outcome.error.code) {
case 'operation_conflict':
throw new RuntimeHostedRootConflictError(sessionId, outcome.error.message);
default:
throw new RuntimeHostedRootUnavailableError(sessionId, outcome.error.message);
}
}
/** Compare the request identity separately from its Host-selected capability binding. */
function rootExecutionMatches(
stored: RootExecutionDescriptor,
incoming: RootExecutionDescriptor,
): boolean {
if (stored.kind === 'workhub_coordination' && incoming.kind === 'workhub_coordination') {
const {
capabilityBinding: _storedBinding,
routingDecision: _storedDecision,
...storedIdentity
} = stored;
const {
capabilityBinding: _incomingBinding,
routingDecision: _incomingDecision,
actionId,
...incomingIdentity
} = incoming;
if (incoming.routingDecision === undefined) {
return isDeepStrictEqual(storedIdentity, {
...incomingIdentity,
...(stored.actionId === undefined ? {} : { actionId }),
});
}
return isDeepStrictEqual(
{ ...storedIdentity, routingDecision: stored.routingDecision },
{
...incomingIdentity,
...(actionId === undefined ? {} : { actionId }),
routingDecision: incoming.routingDecision,
},
);
}
return isDeepStrictEqual(stored, incoming);
}
function rootMessageAdmissionMatches(
admission: RootTurnAdmission,
request: RootMessageStartRequest,
content: MessageContent,
authorization: ConnectionContext['turnAdmissionAuthorization'],
): boolean {
return (
rootExecutionMatches(admission.execution, request.execution) &&
(request.execution.kind === 'external_message' && request.execution.inputDigest
? true
: messageContentsEqual(requireHostedExecutionMessageContent(admission), content)) &&
isDeepStrictEqual(admission.turnOrchestration, request.turnOrchestration) &&
isDeepStrictEqual(admission.authorization, authorization) &&
admission.sourceMessages.length === 0
);
}
function hostedExecutionContentMatches(
admission: RootTurnAdmission,
content: MessageContent | null,
): boolean {
return admission.normalizedInput === null || content === null
? admission.normalizedInput === content
: messageContentsEqual(admission.normalizedInput, content);
}
function composeHostedSkillInvocationContent(
content: MessageContent,
prepared: Exclude<PreparedSkillInvocationMessage, { disposition: 'blocked' }>,
): MessageContent {
if (prepared.disposition === 'passthrough') return content;
const displayText =
content.displayText ??
(content.text.trim().length > 0
? content.text
: prepared.skillInvocation.loaded.map((skill) => `/skill:${skill.id}`).join(' '));
const skillReferences = skillInvocationInlineReferences(
prepared.skillInvocation.receipts,
displayText,
);
const candidates = [
...(content.inlineReferences ?? []).filter((reference) => reference.kind !== 'skill'),
...skillReferences,
].sort((left, right) => left.start - right.start || right.value.length - left.value.length);
const inlineReferences: NonNullable<MessageContent['inlineReferences']> = [];
let previousEnd = 0;
for (const reference of candidates) {
if (inlineReferences.length === INLINE_REFERENCE_MAX_COUNT) break;
if (reference.start < previousEnd) continue;
inlineReferences.push(reference);
previousEnd = reference.start + reference.value.length;
}
return normalizeMessageContent({
...content,
text: prepared.sendText,
displayText,
inlineReferences,
});
}
function preflightRootMessageContent(
content: MessageContent,
):
| { readonly ok: true; readonly content: MessageContent }
| { readonly ok: false; readonly outcome: RootMessageStartOutcome } {
try {
return {
ok: true,
content: normalizeRootTurnAdmissionPayload(content, []).normalizedInput,
};
} catch {
return {
ok: false,
outcome: operationConflict('Turn content exceeds durable admission limits'),
};
}
}
function continuationTurnInput(sessionId: string, turnId: string): RootTurnActivationInput {
return {
sessionId,
turnId,
content: null,
};
}
function requireRootMessageContent(input: RootTurnActivationInput): MessageContent {
if (input.content === null) {
throw new RuntimeMessageAuthorityInvariantError(
`Continuation Turn ${input.turnId} cannot enter message execution`,
);
}
return input.content;
}
function activationInputForAdmission(admission: RootTurnAdmission): RootTurnActivationInput {
if (admission.normalizedInput === null) {
return continuationTurnInput(admission.sessionId, admission.turnId);
}
return {
sessionId: admission.sessionId,
turnId: admission.turnId,
content: normalizeMessageContent(admission.normalizedInput),
...(admission.turnOrchestration
? { turnOrchestration: { ...admission.turnOrchestration } }
: {}),
...(admission.execution.kind === 'external_message' &&
admission.execution.maxSteps !== undefined
? { maxSteps: admission.execution.maxSteps }
: {}),
};
}
function projectTurnResumePlan(
sessionId: string,
plan: SafeBoundaryContinuationPlan,
): TurnResumePlan {
if (plan.disposition === 'continue') {
const continuation = requirePlannedContinuation(plan);
return {
sessionId,
disposition: 'ready',
sourceRunId: continuation.sourceRunId,
sourceTurnId: continuation.sourceTurnId,
sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater,
};
}
const reasons = new Set(plan.rejectionReasons);
let reason: Extract<TurnResumePlan, { disposition: 'parked' }>['reason'];
if (reasons.has('resume_candidate_missing')) reason = 'resume_candidate_missing';
else if (reasons.has('source_run_unreadable') || reasons.has('runtime_ledger_unreadable')) {
reason = 'source_run_unreadable';
} else if (reasons.has('continuation_already_exists')) {
reason = 'continuation_already_exists';
} else if (reasons.has('continuation_started_indeterminate')) {
reason = 'continuation_started_indeterminate';
} else if (reasons.has('continuation_claim_repair_required')) {
reason = 'continuation_repair_required';
} else if (reasons.has('resume_feature_disabled')) {
reason = 'resume_feature_disabled';
} else if (reasons.has('continuation_authority_unavailable')) {
reason = 'continuation_authority_unavailable';
} else if (reasons.has('safety_observation_unavailable')) {
reason = 'safety_observation_unavailable';
} else {
reason = 'safety_check_failed';
}
return parkedTurnResumePlan(sessionId, reason);
}
function parkedTurnResumePlan(
sessionId: string,
reason: Extract<TurnResumePlan, { disposition: 'parked' }>['reason'],
): Extract<TurnResumePlan, { disposition: 'parked' }> {
return { sessionId, disposition: 'parked', reason };
}
function parkedContinuationMatchesQuery(
admission: RootTurnAdmission,
input: TurnResumeQueryInput,
): boolean {
const execution = admission.execution;
if (execution.kind !== 'safe_boundary_continuation') return false;
return (
(input.sourceRunId === undefined || input.sourceRunId === execution.sourceRunId) &&
(input.expectedRuntimeEventHighWater === undefined ||
input.expectedRuntimeEventHighWater === execution.sourceRuntimeEventHighWater)
);
}
function requirePlannedContinuation(plan: SafeBoundaryContinuationPlan): RuntimeContinuation {
if (plan.disposition !== 'continue' || !plan.continuation) {
throw new RuntimeMessageAuthorityInvariantError(
'Ready continuation plan omitted its Runtime continuation',
);
}
return plan.continuation;
}
function continuationExecutionDescriptor(
continuation: RuntimeContinuation,
): Extract<RootExecutionDescriptor, { kind: 'safe_boundary_continuation' }> {
const boundaryDigest = continuation.boundary?.manifestDigest;
if (!continuation.claimId || !boundaryDigest || !continuation.providerReplayDigest) {
throw new RuntimeMessageAuthorityInvariantError(
'Authoritative continuation plan omitted its durable replay proof',
);
}
return {
kind: 'safe_boundary_continuation',
sourceInvocationId: continuation.sourceInvocationId,
sourceRunId: continuation.sourceRunId,
sourceTurnId: continuation.sourceTurnId,
sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater,
claimId: continuation.claimId,
boundaryDigest,
providerReplayDigest: continuation.providerReplayDigest,
safetyDigest: continuationSafetyDigest(continuation),
targetInvocationId: continuation.invocationId,
};
}
export function continuationSafetyDigest(continuation: RuntimeContinuation): `sha256:${string}` {
const snapshot = continuation.safetySnapshot;
const body = JSON.stringify([
'runtime_continuation_safety_v1',
snapshot.workspaceIdentity,
snapshot.backgroundOperationsSettled,
[...new Set(snapshot.availableToolNames)].sort(),
snapshot.workspaceCheckpoint
? [snapshot.workspaceCheckpoint.ref, snapshot.workspaceCheckpoint.runtimeEventHighWater]
: null,
]);
return `sha256:${createHash('sha256').update(body, 'utf8').digest('hex')}`;
}
function deferred(): Deferred {
let phase: Deferred['phase'] = 'pending';
let resolvePromise!: () => void;
let rejectPromise!: (error: unknown) => void;
const promise = new Promise<void>((resolve, reject) => {
resolvePromise = resolve;
rejectPromise = reject;
});
void promise.catch(() => undefined);
return {
promise,
get phase() {
return phase;
},
resolve: () => {
if (phase !== 'pending') return;
phase = 'resolved';
resolvePromise();
},
reject: (error) => {
if (phase !== 'pending') return;
phase = 'rejected';
rejectPromise(error);
},
};
}
function valueDeferred<T>(): ValueDeferred<T> {
let settled = false;
let resolvePromise!: (value: T) => void;
const promise = new Promise<T>((resolve) => {
resolvePromise = resolve;
});
return {
promise,
resolve(value) {
if (settled) return;
settled = true;
resolvePromise(value);
},
};
}
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function throwIfAborted(signal: AbortSignal): void {
if (signal.aborted)
throw new DOMException('Agent graph supervisor Turn was aborted', 'AbortError');
}
function isTerminalSnapshot(
snapshot: TurnSnapshot,
): snapshot is Extract<TurnSnapshot, { status: 'completed' | 'failed' | 'cancelled' }> {
return (
snapshot.status === 'completed' ||
snapshot.status === 'failed' ||
snapshot.status === 'cancelled'
);
}
function isContainableRunFailure(error: unknown): error is Error {
return (
error instanceof Error &&
!(error instanceof RuntimeOwnerCleanupError) &&
!(error instanceof RuntimeMessageAuthorityInvariantError) &&
!(error instanceof RuntimeInteractionInvariantError) &&
!(error instanceof RuntimeInteractionFailStopError)
);
}
function isStoppedInteractionAdmission(
error: unknown,
): error is RuntimeInteractionAdmissionRejectedError {
return (
error instanceof RuntimeInteractionAdmissionRejectedError &&
error.reason === 'run_closed' &&
error.closureReason === 'turn_stopped'
);
}
// Membership answers one question: forward this event live to subscribers via
// the continuity coordinator instead of letting the canonical refresh carry
// it. Persistence is orthogonal — it happens upstream in the run's own event
// stream, which is why the durable steering_message belongs here.
function isRuntimeSessionForwardedEvent(
event: SessionEvent,
): event is RuntimeSessionForwardedEvent {
return (
event.type === 'text_delta' ||
event.type === 'text_complete' ||
event.type === 'thinking_delta' ||
event.type === 'thinking_complete' ||
event.type === 'tool_start' ||
event.type === 'tool_output_delta' ||
event.type === 'tool_progress' ||
event.type === 'tool_result_preview' ||
event.type === 'tool_result' ||
event.type === 'steering_message' ||
event.type === 'provider_retry'
);
}
function isInteractionAnswerAck(event: SessionEvent): boolean {
return event.type === 'user_question_answer_ack' || event.type === 'form_answer_ack';
}
function completedStart(outcome: RootMessageStartOutcome): TurnStartDisposition {
return { kind: 'complete', outcome };
}
function notFound(message: string) {
return { ok: false, error: { code: 'not_found', message } } as const;
}
function sessionBusy(message: string) {
return { ok: false, error: { code: 'session_busy', message } } as const;
}
function sessionArchived(message: string) {
return { ok: false, error: { code: 'session_archived', message } } as const;
}
function operationUnavailable(message: string) {
return {
ok: false,
error: { code: 'operation_unavailable', message },
} as const;
}
function operationConflict(message: string) {
return { ok: false, error: { code: 'operation_conflict', message } } as const;
}