blob: cd9f4dd16ae02f4c9e1d9c2893c3bbb5003dab90 [file]
import assert from 'node:assert/strict';
import { fork, type ChildProcess } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { appendFile, chmod, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import { connect, type Socket } from 'node:net';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { test } from 'node:test';
import type { AgentRunHeader } from '@maka/core/agent-run';
import type { MessageContent } from '@maka/core/events';
import type { StoredMessage } from '@maka/core/session';
import type { Task } from '@maka/core/task-ledger';
import { isTerminalRuntimeEvent } from '@maka/core/runtime-event';
import type { RuntimeEvent } from '@maka/core/runtime-event';
import {
buildTaskLedgerTools,
buildRecoveredTerminalRuntimeEvent,
classifyTerminalRuntimeLedger,
commitTerminalRunWithRuntimeFact,
FAKE_ASK_USER_QUESTION_PROMPT,
type MakaTool,
type MakaToolContext,
} from '@maka/runtime';
import {
openInteractiveExecutionStoresForRead,
openInteractiveExecutionStoresForWrite,
} from '@maka/storage/execution-stores';
import {
resolveRootControlNamespace,
resolveStorageRoot,
tryAcquireInteractiveRootOwner,
tryAcquireInteractiveRootReader,
type StorageRootCapability,
} from '@maka/storage/root-authority';
import { openInteractiveTaskLedgerStoreForWrite } from '@maka/storage/task-ledger-authority';
import {
connectRuntimeHost,
RuntimeHostOperationError,
RuntimeHostSubscriptionError,
type RuntimeHostConnection,
type RuntimeHostSessionSubscription,
} from '../client/index.js';
import {
decodeHostFrame,
RUNTIME_HOST_PROTOCOL_VERSION,
TASK_LEDGER_PAGE_MAX_ITEMS,
type InteractionPendingSnapshot,
type SubscriptionFrame,
type TaskLedgerQueryResult,
type TaskLedgerRevision,
type TurnMessageSubmitInput,
type TurnSnapshot,
} from '../protocol/index.js';
import { SessionAdmissionGate } from '../server/session-admission-gate.js';
import { HostTaskLedgerCoordinator } from '../server/task-ledger-coordinator.js';
import { FramedTransport } from '../transport/framed-transport.js';
const CURRENT_PROTOCOL = {
min: RUNTIME_HOST_PROTOCOL_VERSION,
max: RUNTIME_HOST_PROTOCOL_VERSION,
} as const;
const PROCESS_TIMEOUT_MS = 10_000;
test('dual UDS Clients query persisted Task Ledger tool-port mutations across Host restart', async () => {
await withExecutionRoot(async (fixture) => {
const initialRunId = randomUUID();
const initialTurnId = randomUUID();
// Exercise the Runtime-facing port before Host startup; Hosted tool composition is separate.
const toolPortProjection = await withOwnedTaskLedgerToolPort(
fixture,
async (coordinator, tools) => {
const context = taskLedgerToolContext(fixture, {
runId: initialRunId,
turnId: initialTurnId,
toolCallId: randomUUID(),
});
const create = requireTaskLedgerTool<TaskCreateInput>(tools, 'task_create');
const createInput = create.parameters.parse({
tasks: Array.from({ length: TASK_LEDGER_PAGE_MAX_ITEMS + 1 }, (_, index) => ({
subject: `Authority acceptance task ${index + 1}`,
})),
});
await create.impl(createInput, context);
const update = requireTaskLedgerTool<TaskUpdateInput>(tools, 'task_update');
const updateInput = update.parameters.parse({ id: 'T1', status: 'in_progress' });
await update.impl(updateInput, {
...context,
toolCallId: randomUUID(),
});
return coordinator.list(fixture.sessionId, {
includeTerminal: true,
includeArchived: false,
classifyResumeTrust: true,
});
},
);
assert.equal(toolPortProjection.length, TASK_LEDGER_PAGE_MAX_ITEMS + 1);
assert.deepEqual(toolPortProjection[0]?.owner, {
actor: 'main_agent',
runId: initialRunId,
turnId: initialTurnId,
});
const host = await fixture.startHost();
const desktop = await connectClient(fixture.root, 'desktop');
const tui = await connectClient(fixture.root, 'tui');
let staleContinuation:
| {
revision: TaskLedgerRevision;
cursor: string;
task: Task;
}
| undefined;
try {
const desktopProjection = await collectTaskLedgerProjection(desktop, fixture.sessionId);
const tuiProjection = await collectTaskLedgerProjection(tui, fixture.sessionId);
assert.deepEqual(
desktopProjection.pages.map((page) => page.tasks.length),
[TASK_LEDGER_PAGE_MAX_ITEMS, 1],
);
assert.deepEqual(tuiProjection, desktopProjection);
assert.deepEqual(desktopProjection.tasks, toolPortProjection);
const byKey = await tui.request('task.ledger.query', {
kind: 'get',
sessionId: fixture.sessionId,
taskRef: 'T1',
});
assert.equal(byKey.kind, 'task');
if (byKey.kind !== 'task') throw new Error('Expected Task Ledger get result');
assert.equal(byKey.sessionId, fixture.sessionId);
assert.deepEqual(byKey.task, desktopProjection.tasks[0]);
assert.equal(byKey.task?.owner?.runId, initialRunId);
assert.equal(byKey.task?.owner?.turnId, initialTurnId);
const firstPage = desktopProjection.pages[0];
assert.ok(firstPage?.nextCursor);
staleContinuation = {
revision: firstPage.revision,
cursor: firstPage.nextCursor,
task: desktopProjection.tasks[1]!,
};
} finally {
await Promise.allSettled([desktop.close(), tui.close()]);
await fixture.stopHost(host);
}
assert.ok(staleContinuation);
const { revision: staleRevision, cursor: staleCursor, task: taskToChange } = staleContinuation;
const successorTurnId = randomUUID();
const changedSubject = `${taskToChange.subject} after authority reacquisition`;
await withOwnedTaskLedgerToolPort(fixture, async (_coordinator, tools) => {
const update = requireTaskLedgerTool<TaskUpdateInput>(tools, 'task_update');
const input = update.parameters.parse({
id: taskToChange.key,
subject: changedSubject,
});
await update.impl(
input,
taskLedgerToolContext(fixture, {
runId: randomUUID(),
turnId: successorTurnId,
toolCallId: randomUUID(),
}),
);
});
const successorHost = await fixture.startHost();
const successor = await connectClient(fixture.root, 'desktop');
try {
const continued = await successor.request('task.ledger.query', {
kind: 'list_continue',
sessionId: fixture.sessionId,
revision: staleRevision,
cursor: staleCursor,
});
assert.equal(continued.kind, 'revision_changed');
if (continued.kind !== 'revision_changed') {
throw new Error('Expected stale Task Ledger continuation to report revision_changed');
}
assert.equal(continued.expected, staleRevision);
assert.notEqual(continued.actual, staleRevision);
const changed = await successor.request('task.ledger.query', {
kind: 'get',
sessionId: fixture.sessionId,
taskRef: taskToChange.key,
});
assert.equal(changed.kind, 'task');
if (changed.kind !== 'task') throw new Error('Expected changed Task Ledger task result');
assert.equal(changed.sessionId, fixture.sessionId);
assert.equal(changed.task?.subject, changedSubject);
assert.equal(changed.revision, continued.actual);
} finally {
await successor.close();
await fixture.stopHost(successorHost);
}
});
});
test('two UDS Clients share one Runtime Policy authority and CAS winner', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const first = await connectClient(fixture.root, 'desktop');
const second = await connectClient(fixture.root, 'tui');
try {
const initial = await first.request('runtime.policy.query', {});
assert.deepEqual(await second.request('runtime.policy.query', {}), initial);
const outcomes = await Promise.all([
first.request('runtime.policy.mutate', {
expectedRevision: initial.revision,
operation: {
kind: 'set_personalization',
value: { displayName: 'Desktop', assistantTone: 'precise' },
},
}),
second.request('runtime.policy.mutate', {
expectedRevision: initial.revision,
operation: {
kind: 'set_memory',
value: { enabled: false, agentReadEnabled: false },
},
}),
]);
assert.deepEqual(outcomes.map((outcome) => outcome.kind).sort(), [
'committed',
'revision_conflict',
]);
assert.deepEqual(
await first.request('runtime.policy.query', {}),
await second.request('runtime.policy.query', {}),
);
} finally {
await Promise.allSettled([first.close(), second.close()]);
await fixture.stopHost(host);
}
});
});
test('two Clients share one execution after the starting Client disconnects', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const first = await connectClient(fixture.root, 'desktop');
const second = await connectClient(fixture.root, 'tui');
const secondSubscription = await second.openSessionSubscription({
sessionId: fixture.sessionId,
});
const secondProbe = new SubscriptionProbe(secondSubscription);
const turnId = randomUUID();
const started = await first.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text: FAKE_ASK_USER_QUESTION_PROMPT },
});
assert.equal(started.turnId, turnId);
await assert.rejects(
() =>
second.startTurn({
sessionId: fixture.sessionId,
turnId: randomUUID(),
content: { text: 'must stay busy' },
}),
operationError('session_busy'),
);
await first.close();
const pending = await waitForPendingInteraction(secondSubscription, secondProbe, started.runId);
assert.equal(pending.sessionId, fixture.sessionId);
assert.equal(pending.turnId, turnId);
assert.equal(pending.runId, started.runId);
const questionRequest = pending.request;
assert.ok(questionRequest.kind === 'question');
assert.deepEqual(
await second.request('interaction.query', {
sessionId: fixture.sessionId,
interactionId: pending.interactionId,
}),
pending,
);
const observed = await second.queryTurn({
sessionId: fixture.sessionId,
turnId,
});
assert.equal(observed.runId, started.runId);
assert.ok(observed.status === 'running' || observed.status === 'waiting_for_user');
const stopped = await second.stopTurn(
{
sessionId: fixture.sessionId,
turnId,
runId: started.runId,
},
PROCESS_TIMEOUT_MS,
);
assert.equal(stopped.status, 'cancelled');
const closed = await second.request('interaction.query', {
sessionId: fixture.sessionId,
interactionId: pending.interactionId,
});
assert.equal(closed.sessionId, fixture.sessionId);
assert.equal(closed.turnId, turnId);
assert.equal(closed.runId, started.runId);
assert.equal(closed.status, 'closed');
assert.equal(closed.outcome.kind, 'closure');
if (closed.outcome.kind === 'closure') assert.equal(closed.outcome.reason, 'turn_stopped');
await assert.rejects(
() =>
second.request('interaction.answer', {
interactionId: pending.interactionId,
answer: {
kind: 'question',
answers: questionRequest.questions.map(() => null),
},
}),
operationError('already_resolved'),
);
const nextTurnId = randomUUID();
const next = await second.startTurn({
sessionId: fixture.sessionId,
turnId: nextTurnId,
content: { text: FAKE_ASK_USER_QUESTION_PROMPT },
});
assert.deepEqual(
await second.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text: FAKE_ASK_USER_QUESTION_PROMPT },
}),
stopped,
);
assert.deepEqual(
await second.stopTurn({
sessionId: fixture.sessionId,
turnId,
runId: started.runId,
}),
stopped,
);
const nextObserved = await second.queryTurn({
sessionId: fixture.sessionId,
turnId: nextTurnId,
});
assert.equal(nextObserved.runId, next.runId);
assert.ok(nextObserved.status === 'running' || nextObserved.status === 'waiting_for_user');
await second.stopTurn(
{
sessionId: fixture.sessionId,
turnId: nextTurnId,
runId: next.runId,
},
PROCESS_TIMEOUT_MS,
);
await secondSubscription.close();
await secondProbe.done;
await second.close();
await fixture.stopHost(host);
const ledger = await fixture.readTurn(turnId);
assert.equal(ledger.runs.length, 1);
assert.equal(ledger.userMessages.length, 1);
assert.equal(ledger.terminalEvents.length, 1);
assert.equal(ledger.classification.kind, 'fact');
if (ledger.classification.kind === 'fact') {
assert.equal(ledger.classification.fact.runStatus, 'cancelled');
assert.notEqual(ledger.classification.fact.failureClass, 'app_restarted');
}
});
});
test('a disconnected Client leaves a durable Interaction that another Client can answer', async () => {
await withExecutionRoot(async (fixture) => {
const firstHost = await fixture.startHost();
const first = await connectClient(fixture.root, 'desktop');
const turnId = randomUUID();
const started = await first.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text: FAKE_ASK_USER_QUESTION_PROMPT },
});
await first.close();
const second = await connectClient(fixture.root, 'tui');
const subscription = await second.openSessionSubscription({ sessionId: fixture.sessionId });
const probe = new SubscriptionProbe(subscription);
const pending = await waitForPendingInteraction(subscription, probe, started.runId);
assert.equal(pending.sessionId, fixture.sessionId);
assert.equal(pending.turnId, turnId);
assert.equal(pending.runId, started.runId);
assert.equal(pending.status, 'pending');
const questionRequest = pending.request;
assert.ok(questionRequest.kind === 'question');
assert.deepEqual(
await second.request('interaction.query', {
sessionId: fixture.sessionId,
interactionId: pending.interactionId,
}),
pending,
);
const answer = {
kind: 'question' as const,
answers: questionRequest.questions.map((question) => question.options[0]?.label ?? null),
};
const winner = await second.request('interaction.answer', {
interactionId: pending.interactionId,
answer,
});
assert.equal(winner.sessionId, fixture.sessionId);
assert.equal(winner.turnId, turnId);
assert.equal(winner.runId, started.runId);
assert.equal(winner.status, 'answered');
assert.equal(winner.outcome.kind, 'question_answer');
assert.deepEqual(winner.outcome.answers, answer.answers);
assert.deepEqual(
await second.request('interaction.query', {
sessionId: fixture.sessionId,
interactionId: pending.interactionId,
}),
winner,
);
assert.deepEqual(
await second.request('interaction.answer', {
interactionId: pending.interactionId,
answer,
}),
winner,
);
const resumed = await probe.waitFor(
(frame) =>
frame.kind === 'subscription.session_projection' &&
frame.snapshot.session.status === 'running' &&
frame.snapshot.rootTurn?.runId === started.runId &&
frame.snapshot.rootTurn.status === 'running' &&
frame.snapshot.interactions.pending.length === 0,
'continuity did not publish the resumed Turn after the question answer',
);
assert.equal(resumed.kind, 'subscription.session_projection');
const completed = await waitForTerminalTurn(second, fixture.sessionId, turnId);
assert.equal(completed.runId, started.runId);
assert.equal(completed.status, 'completed');
await subscription.close();
await probe.done;
await second.close();
await fixture.stopHost(firstHost);
const secondHost = await fixture.startHost();
const observer = await connectClient(fixture.root, 'run');
assert.deepEqual(
await observer.request('interaction.query', {
sessionId: fixture.sessionId,
interactionId: pending.interactionId,
}),
winner,
);
assert.deepEqual(
await observer.request('interaction.answer', {
interactionId: pending.interactionId,
answer,
}),
winner,
);
assert.deepEqual(await observer.queryTurn({ sessionId: fixture.sessionId, turnId }), completed);
await observer.close();
await fixture.stopHost(secondHost);
});
});
test('subscribed Clients share one canonical queue and ordered root handoff', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const desktop = await connectClient(fixture.root, 'desktop');
const tui = await connectClient(fixture.root, 'tui');
const desktopSubscription = await desktop.openSessionSubscription({
sessionId: fixture.sessionId,
});
const tuiSubscription = await tui.openSessionSubscription({ sessionId: fixture.sessionId });
const desktopProbe = new SubscriptionProbe(desktopSubscription);
const tuiProbe = new SubscriptionProbe(tuiSubscription);
for (const subscription of [desktopSubscription, tuiSubscription]) {
assert.equal(subscription.hostEpoch, host.hostEpoch);
assert.equal(subscription.snapshot.rootTurn, null);
assert.equal(subscription.snapshot.projectionRevision, 1);
assert.equal(subscription.snapshot.queue.hostEpoch, host.hostEpoch);
}
const firstTurnId = randomUUID();
const started = await desktop.startTurn({
sessionId: fixture.sessionId,
turnId: firstTurnId,
content: { text: `continuity root ${'x'.repeat(540)}` },
});
for (const probe of [desktopProbe, tuiProbe]) {
const liveDelta = await probe.waitFor(
(frame) =>
frame.kind === 'subscription.session_delta' && frame.delta.turnId === firstTurnId,
'continuity did not publish the live assistant delta',
);
assert.equal(liveDelta.kind, 'subscription.session_delta');
if (liveDelta.kind === 'subscription.session_delta') {
assert.equal(liveDelta.delta.runId, started.runId);
}
}
const followupId = randomUUID();
const followupContent = { text: 'continue after the first root completes' };
const queued = await tui.request('turn.message.submit', {
originHostEpoch: host.hostEpoch,
sessionId: fixture.sessionId,
messageId: followupId,
content: followupContent,
placement: 'next_turn',
});
assert.equal(queued.disposition, 'followup');
for (const probe of [desktopProbe, tuiProbe]) {
const queueProjection = await probe.waitFor(
(frame) =>
frame.kind === 'subscription.session_projection' &&
frame.snapshot.queue.followup.some((entry) => entry.messageId === followupId),
'continuity did not publish the accepted follow-up',
);
assert.equal(queueProjection.kind, 'subscription.session_projection');
}
await desktop.close();
await desktopProbe.waitForFailure('connection_closed');
assert.equal((await tui.status()).connections, 1);
const terminal = await tuiProbe.waitFor(
(frame) =>
frame.kind === 'subscription.session_projection' &&
frame.snapshot.rootTurn?.turnId === firstTurnId &&
frame.snapshot.rootTurn.status === 'completed',
'continuity did not publish the terminal root cut',
);
assert.equal(terminal.kind, 'subscription.session_projection');
const successor = await tuiProbe.waitFor(
(frame) =>
frame.kind === 'subscription.session_projection' &&
frame.snapshot.rootTurn !== null &&
frame.snapshot.rootTurn.turnId !== firstTurnId,
'continuity did not publish the successor root',
);
assert.equal(successor.kind, 'subscription.session_projection');
if (successor.kind !== 'subscription.session_projection' || !successor.snapshot.rootTurn) {
return;
}
assert.equal(successor.snapshot.rootTurn.sessionId, fixture.sessionId);
assert.ok(tuiProbe.indexOf(terminal) < tuiProbe.indexOf(successor));
await tuiSubscription.close();
await tuiProbe.done;
await waitForTerminalTurn(tui, fixture.sessionId, successor.snapshot.rootTurn.turnId);
await tui.close();
await fixture.stopHost(host);
const chain = await fixture.readAdmissionChain();
assert.deepEqual(
chain.map((admission) => admission.turnId),
[firstTurnId, successor.snapshot.rootTurn.turnId],
);
assert.deepEqual(chain[1]?.normalizedInput, followupContent);
});
});
test('concurrent root admission for one Session has a single winner', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const first = await connectClient(fixture.root, 'desktop');
const second = await connectClient(fixture.root, 'tui');
const turnIds = [randomUUID(), randomUUID()] as const;
const outcomes = await Promise.allSettled([
first.startTurn({
sessionId: fixture.sessionId,
turnId: turnIds[0],
content: { text: FAKE_ASK_USER_QUESTION_PROMPT },
}),
second.startTurn({
sessionId: fixture.sessionId,
turnId: turnIds[1],
content: { text: FAKE_ASK_USER_QUESTION_PROMPT },
}),
]);
const winners = outcomes.filter(
(outcome): outcome is PromiseFulfilledResult<TurnSnapshot> => outcome.status === 'fulfilled',
);
const rejected = outcomes.filter(
(outcome): outcome is PromiseRejectedResult => outcome.status === 'rejected',
);
assert.equal(winners.length, 1);
assert.equal(rejected.length, 1);
assert.ok(rejected[0]?.reason instanceof RuntimeHostOperationError);
assert.equal(rejected[0]?.reason.code, 'session_busy');
const winner = winners[0]?.value;
assert.ok(winner);
await first.stopTurn({
sessionId: fixture.sessionId,
turnId: winner.turnId,
runId: winner.runId,
});
await first.close();
await second.close();
await fixture.stopHost(host);
const chain = await fixture.readAdmissionChain();
assert.equal(chain.length, 1);
assert.equal(chain[0]?.turnId, winner.turnId);
assert.equal(chain[0]?.previousRootTurnId, null);
});
});
test('an archived Session rejects a new Turn before durable admission', async () => {
await withExecutionRoot(async (fixture) => {
await fixture.archiveSession();
const host = await fixture.startHost();
const client = await connectClient(fixture.root, 'desktop');
const turnId = randomUUID();
await assert.rejects(
() =>
client.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text: 'must not execute' },
}),
operationError('session_archived'),
);
assert.equal((await client.status()).state, 'ready');
await client.close();
await fixture.stopHost(host);
assert.deepEqual(await fixture.readTurnFootprint(turnId), {
admitted: false,
runCount: 0,
userMessageCount: 0,
});
});
});
test('a killed Host is recovered exactly once before its successor becomes ready', {
skip: process.platform === 'win32' ? 'POSIX process death gate' : false,
}, async () => {
await withExecutionRoot(async (fixture) => {
const firstHost = await fixture.startHost();
const first = await connectClient(fixture.root, 'desktop');
const firstSubscription = await first.openSessionSubscription({
sessionId: fixture.sessionId,
});
const firstProbe = new SubscriptionProbe(firstSubscription);
const turnId = randomUUID();
const started = await first.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text: FAKE_ASK_USER_QUESTION_PROMPT },
});
await firstProbe.waitFor(
(frame) =>
frame.kind === 'subscription.session_projection' &&
frame.snapshot.rootTurn?.runId === started.runId &&
frame.snapshot.rootTurn.status !== 'admitted',
'first Host did not publish the active root projection',
);
const pending = await waitForPendingInteraction(firstSubscription, firstProbe, started.runId);
assert.equal(pending.sessionId, fixture.sessionId);
assert.equal(pending.turnId, turnId);
assert.equal(pending.runId, started.runId);
const questionRequest = pending.request;
assert.ok(questionRequest.kind === 'question');
await fixture.killHost(firstHost);
await first.closed;
await firstProbe.waitForFailure('connection_closed');
const secondHost = await fixture.startHost();
const second = await connectClient(fixture.root, 'tui');
const recoveredSubscription = await second.openSessionSubscription({
sessionId: fixture.sessionId,
});
const recovered = await second.queryTurn({
sessionId: fixture.sessionId,
turnId,
});
assert.equal(recovered.status, 'failed');
if (recovered.status === 'failed') assert.equal(recovered.failureClass, 'app_restarted');
assert.notEqual(recoveredSubscription.hostEpoch, firstSubscription.hostEpoch);
assert.equal(recoveredSubscription.snapshot.projectionRevision, 1);
assert.deepEqual(recoveredSubscription.snapshot.rootTurn, recovered);
assert.equal(recoveredSubscription.snapshot.queue.hostEpoch, recoveredSubscription.hostEpoch);
assert.deepEqual(recoveredSubscription.snapshot.queue.steering, []);
assert.deepEqual(recoveredSubscription.snapshot.queue.followup, []);
const closed = await second.request('interaction.query', {
sessionId: fixture.sessionId,
interactionId: pending.interactionId,
});
assert.equal(closed.sessionId, fixture.sessionId);
assert.equal(closed.turnId, turnId);
assert.equal(closed.runId, started.runId);
assert.equal(closed.status, 'closed');
assert.equal(closed.outcome.kind, 'closure');
if (closed.outcome.kind === 'closure') assert.equal(closed.outcome.reason, 'host_restarted');
await assert.rejects(
() =>
second.request('interaction.answer', {
interactionId: pending.interactionId,
answer: {
kind: 'question',
answers: questionRequest.questions.map(() => null),
},
}),
operationError('already_resolved'),
);
await recoveredSubscription.close();
await second.close();
await fixture.stopHost(secondHost);
const thirdHost = await fixture.startHost();
const third = await connectClient(fixture.root, 'run');
const stable = await third.queryTurn({
sessionId: fixture.sessionId,
turnId,
});
assert.deepEqual(stable, recovered);
assert.equal(stable.runId, started.runId);
assert.deepEqual(
await third.request('interaction.query', {
sessionId: fixture.sessionId,
interactionId: pending.interactionId,
}),
closed,
);
await third.close();
await fixture.stopHost(thirdHost);
const ledger = await fixture.readTurn(turnId);
assert.equal(ledger.terminalEvents.length, 1);
assert.equal(ledger.classification.kind, 'fact');
if (ledger.classification.kind === 'fact') {
assert.equal(ledger.classification.fact.failureClass, 'app_restarted');
}
});
});
test('graceful Host shutdown stops and drains an active Turn before releasing ownership', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const client = await connectClient(fixture.root, 'desktop');
const turnId = randomUUID();
const started = await client.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text: FAKE_ASK_USER_QUESTION_PROMPT },
});
const exit = await fixture.stopHost(host);
assert.deepEqual(exit, { code: 0, signal: null });
await client.closed;
const successor = await fixture.startHost();
const observer = await connectClient(fixture.root, 'tui');
const stable = await observer.queryTurn({
sessionId: fixture.sessionId,
turnId,
});
assert.equal(stable.runId, started.runId);
assert.equal(stable.status, 'cancelled');
await observer.close();
await fixture.stopHost(successor);
const ledger = await fixture.readTurn(turnId);
assert.equal(ledger.terminalEvents.length, 1);
assert.equal(ledger.classification.kind, 'fact');
if (ledger.classification.kind === 'fact') {
assert.equal(ledger.classification.fact.runStatus, 'cancelled');
assert.notEqual(ledger.classification.fact.failureClass, 'app_restarted');
}
});
});
test('a durable admission without a Run resumes before the Host becomes ready', async () => {
await withExecutionRoot(async (fixture) => {
const turnId = randomUUID();
const quotes = quotedContent('recover pending admission');
const { runId } = await fixture.seedAdmission(turnId, quotes);
const host = await fixture.startHost();
const client = await connectClient(fixture.root, 'tui');
const recovered = await client.queryTurn({
sessionId: fixture.sessionId,
turnId,
});
assert.equal(recovered.runId, runId);
assert.ok(recovered.status === 'running' || recovered.status === 'waiting_for_user');
await assert.rejects(
() =>
client.startTurn({
sessionId: fixture.sessionId,
turnId: randomUUID(),
content: { text: 'must remain behind the recovered admission' },
}),
operationError('session_busy'),
);
const stopped = await client.stopTurn(
{
sessionId: fixture.sessionId,
turnId,
runId,
},
PROCESS_TIMEOUT_MS,
);
assert.equal(stopped.status, 'cancelled');
await client.close();
await fixture.stopHost(host);
const ledger = await fixture.readTurn(turnId);
assert.equal(ledger.runs.length, 1);
assert.equal(ledger.userMessages.length, 1);
assert.deepEqual(ledger.userMessages[0]?.quotes, quotes.quotes);
assert.deepEqual(userRuntimeContent(ledger.runtimeEvents)?.quotes, quotes.quotes);
assert.equal(ledger.terminalEvents.length, 1);
assert.equal(ledger.classification.kind, 'fact');
if (ledger.classification.kind === 'fact') {
assert.notEqual(ledger.classification.fact.failureClass, 'app_restarted');
}
});
});
test('startup recovery compares an existing quoted UserMessage canonically', async () => {
await withExecutionRoot(async (fixture) => {
const turnId = randomUUID();
const content = quotedContent('recover existing message');
const { runId, userMessageId } = await fixture.seedRunWithUserMessage(turnId, content);
const host = await fixture.startHost();
const client = await connectClient(fixture.root, 'tui');
const recovered = await client.queryTurn({
sessionId: fixture.sessionId,
turnId,
});
assert.equal(recovered.runId, runId);
assert.equal(recovered.status, 'failed');
await client.close();
await fixture.stopHost(host);
const ledger = await fixture.readTurn(turnId);
assert.equal(ledger.userMessages.length, 1);
assert.equal(ledger.userMessages[0]?.id, userMessageId);
assert.deepEqual(ledger.userMessages[0]?.quotes, content.quotes);
assert.equal(ledger.terminalEvents.length, 1);
});
});
test('startup recovery restores the admitted UserMessage before terminalizing its Run', async () => {
await withExecutionRoot(async (fixture) => {
const turnId = randomUUID();
const { runId, userMessageId } = await fixture.seedRunWithoutUserMessage(
turnId,
'recover the admitted message',
);
const host = await fixture.startHost();
const client = await connectClient(fixture.root, 'tui');
const recovered = await client.queryTurn({
sessionId: fixture.sessionId,
turnId,
});
assert.equal(recovered.runId, runId);
assert.equal(recovered.status, 'failed');
if (recovered.status === 'failed') {
assert.equal(recovered.failureClass, 'app_restarted');
}
await client.close();
await fixture.stopHost(host);
const ledger = await fixture.readTurn(turnId);
assert.equal(ledger.runs.length, 1);
assert.equal(ledger.userMessages.length, 1);
assert.equal(ledger.userMessages[0]?.id, userMessageId);
assert.equal(ledger.terminalEvents.length, 1);
});
});
test('startup recovery canonically closes pending linked child admissions without inventing identity', async () => {
await withExecutionRoot(async (fixture) => {
const initial = await fixture.seedPendingChildAdmission('linked_child_initial');
const resume = await fixture.seedPendingChildAdmission('linked_child_resume');
const retry = await fixture.seedPendingChildAdmission('linked_child_provider_retry');
const graph = await fixture.seedPendingChildAdmission('claimed_agent_graph_intent');
const firstHost = await fixture.startHost();
await fixture.stopHost(firstHost);
const secondHost = await fixture.startHost();
await fixture.stopHost(secondHost);
const reader = await tryAcquireInteractiveRootReader(fixture.capability);
assert.ok(reader);
if (!reader) throw new Error('Unable to acquire recovery result reader');
try {
const stores = await openInteractiveExecutionStoresForRead(reader.lease);
for (const recovered of [initial, resume, retry, graph]) {
const run = await stores.agentRunStore.readRun(recovered.sessionId, recovered.runId);
assert.equal(run.status, 'failed');
assert.equal(run.failureClass, 'app_restarted');
assert.equal(run.agentId, recovered.agentId);
assert.equal(run.agentName, recovered.agentName);
assert.equal(run.workspaceIdentity, undefined);
if (recovered.kind === 'linked_child_resume') {
assert.equal(run.resumedFromRunId, recovered.sourceRunId);
assert.equal(run.retriedFromRunId, undefined);
} else if (recovered.kind === 'linked_child_provider_retry') {
assert.equal(run.retriedFromRunId, recovered.sourceRunId);
assert.equal(run.resumedFromRunId, undefined);
} else {
assert.equal(run.resumedFromRunId, undefined);
assert.equal(run.retriedFromRunId, undefined);
}
const runtimeEvents = await stores.runtimeEventStore.readImmutableRuntimeEvents(
recovered.sessionId,
recovered.runId,
);
const terminal = classifyTerminalRuntimeLedger(run, runtimeEvents);
assert.equal(terminal.kind, 'fact');
if (terminal.kind === 'fact') {
assert.equal(terminal.fact.runStatus, 'failed');
assert.equal(terminal.fact.failureClass, 'app_restarted');
}
const userMessages = (await stores.sessionStore.readMessages(recovered.sessionId)).filter(
(message) => message.type === 'user' && message.turnId === recovered.turnId,
);
assert.equal(userMessages.length, recovered.kind === 'linked_child_provider_retry' ? 0 : 1);
if (recovered.kind !== 'linked_child_provider_retry') {
assert.equal(userMessages[0]?.id, recovered.userMessageId);
}
}
} finally {
await reader.close();
}
});
});
test('startup recovery rejects claimed graph Run lineage drift', async () => {
await withExecutionRoot(async (fixture) => {
await fixture.seedClaimedGraphRunLineageDrift();
await fixture.expectHostStartupFailure();
await fixture.assertOwnerAvailable();
});
});
test('startup recovery repairs a truncated RuntimeEvent tail before terminalizing the Run', async () => {
await withExecutionRoot(async (fixture) => {
const turnId = randomUUID();
const { runId } = await fixture.seedRunWithoutUserMessage(
turnId,
'recover after a partial RuntimeEvent write',
);
const runtimeEventsPath = fixture.runtimeEventsPath(runId);
await writeFile(runtimeEventsPath, '{"id":"truncated"', 'utf8');
const host = await fixture.startHost();
const client = await connectClient(fixture.root, 'tui');
const recovered = await client.queryTurn({
sessionId: fixture.sessionId,
turnId,
});
assert.equal(recovered.status, 'failed');
if (recovered.status === 'failed') {
assert.equal(recovered.failureClass, 'app_restarted');
}
await client.close();
await fixture.stopHost(host);
const bytes = await readFile(runtimeEventsPath, 'utf8');
assert.doesNotMatch(bytes, /truncated/);
assertJsonLines(bytes);
const ledger = await fixture.readTurn(turnId);
assert.equal(ledger.terminalEvents.length, 1);
});
});
test('startup recovery fails closed on a complete malformed RuntimeEvent record', async () => {
await withExecutionRoot(async (fixture) => {
const turnId = randomUUID();
const { runId } = await fixture.seedRunWithoutUserMessage(
turnId,
'do not recover across durable corruption',
);
const runtimeEventsPath = fixture.runtimeEventsPath(runId);
const malformed = '{"id":"malformed"\n';
await writeFile(runtimeEventsPath, malformed, 'utf8');
await fixture.expectHostStartupFailure();
assert.equal(await readFile(runtimeEventsPath, 'utf8'), malformed);
await fixture.assertOwnerAvailable();
});
});
test('startup recovery fails closed on a complete malformed Session record', async () => {
await withExecutionRoot(async (fixture) => {
await fixture.seedRunWithoutUserMessage(
randomUUID(),
'do not rewrite durable Session corruption',
);
const sessionPath = fixture.sessionPath();
const malformed = '{"type":"user"\n';
await appendFile(sessionPath, malformed, 'utf8');
const expected = await readFile(sessionPath, 'utf8');
await fixture.expectHostStartupFailure();
assert.equal(await readFile(sessionPath, 'utf8'), expected);
await fixture.assertOwnerAvailable();
});
});
test('startup recovery fails closed on a complete malformed AgentRun record', async () => {
await withExecutionRoot(async (fixture) => {
const { runId } = await fixture.seedRunWithoutUserMessage(
randomUUID(),
'do not recover across durable AgentRun corruption',
);
const eventsPath = fixture.eventsPath(runId);
const malformed = '{"type":"run_started"\n';
await writeFile(eventsPath, malformed, 'utf8');
await fixture.expectHostStartupFailure();
assert.equal(await readFile(eventsPath, 'utf8'), malformed);
await fixture.assertOwnerAvailable();
});
});
test('a pre-start durability failure rejects turn.start and drains the Host', {
skip:
process.platform === 'win32' || process.getuid?.() === 0 ? 'POSIX file-permission gate' : false,
}, async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const client = await connectClient(fixture.root, 'desktop');
const turnId = randomUUID();
const sessionPath = fixture.sessionPath();
await chmod(sessionPath, 0o400);
try {
await assert.rejects(
() =>
client.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text: 'fail before the durable start barrier' },
}),
operationError('internal_failure'),
);
await client.closed;
await fixture.waitForHostExit(host);
} finally {
await chmod(sessionPath, 0o600);
}
const successor = await fixture.startHost();
const observer = await connectClient(fixture.root, 'tui');
const recovered = await observer.queryTurn({
sessionId: fixture.sessionId,
turnId,
});
assert.equal(recovered.status, 'failed');
await observer.close();
await fixture.stopHost(successor);
const ledger = await fixture.readTurn(turnId);
assert.equal(ledger.runs.length, 1);
assert.equal(ledger.userMessages.length, 1);
assert.equal(ledger.terminalEvents.length, 1);
});
});
test('retry after a discarded turn.start response reuses the durable semantic admission', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const turnId = randomUUID();
const text = 'response loss must not duplicate this Turn';
const dropped = await sendStartWithoutReadingResponse(host.endpoint, {
sessionId: fixture.sessionId,
turnId,
text,
});
const observer = await connectClient(fixture.root, 'tui');
const committed = await waitForTurn(observer, fixture.sessionId, turnId);
dropped.destroy();
const retried = await observer.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text },
});
assert.equal(retried.runId, committed.runId);
await assert.rejects(
() =>
observer.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text: `${text} changed` },
}),
operationError('operation_conflict'),
);
const terminal = await waitForTerminalTurn(observer, fixture.sessionId, turnId);
assert.equal(terminal.status, 'completed');
await observer.close();
await fixture.killHost(host);
const successorHost = await fixture.startHost();
const successorClient = await connectClient(fixture.root, 'run');
assert.deepEqual(
await successorClient.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text },
}),
terminal,
);
const successorTurnId = randomUUID();
await successorClient.startTurn({
sessionId: fixture.sessionId,
turnId: successorTurnId,
content: { text: 'successor must extend the recovered durable tip' },
});
await waitForTerminalTurn(successorClient, fixture.sessionId, successorTurnId);
await successorClient.close();
await fixture.stopHost(successorHost);
const ledger = await fixture.readTurn(turnId);
assert.equal(ledger.runs.length, 1);
assert.equal(ledger.userMessages.length, 1);
assert.equal(ledger.terminalEvents.length, 1);
const chain = await fixture.readAdmissionChain();
assert.deepEqual(
chain.map((admission) => admission.turnId),
[turnId, successorTurnId],
);
assert.equal(chain[1]?.previousRootTurnId, turnId);
});
});
test('a fresh quoted Turn preserves durable and Runtime handoff content', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const client = await connectClient(fixture.root, 'desktop');
const turnId = randomUUID();
const content = quotedContent('fresh quoted turn');
await client.startTurn({ sessionId: fixture.sessionId, turnId, content });
await waitForTerminalTurn(client, fixture.sessionId, turnId);
await client.close();
await fixture.stopHost(host);
const chain = await fixture.readAdmissionChain();
assert.equal(chain.length, 1);
assert.deepEqual(chain[0]?.normalizedInput, content);
const ledger = await fixture.readTurn(turnId);
assert.equal(ledger.userMessages.length, 1);
assert.deepEqual(ledger.userMessages[0]?.quotes, content.quotes);
assert.deepEqual(userRuntimeContent(ledger.runtimeEvents)?.quotes, content.quotes);
});
});
test('same idle Message submit is connection-independent and starts one canonical root', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const first = await connectClient(fixture.root, 'desktop');
const second = await connectClient(fixture.root, 'tui');
const messageId = randomUUID();
const content = {
text: '<context>canonical model input</context>',
displayText: 'canonical display input',
attachments: [attachment('idle-message', 'context.png')],
};
const input = {
originHostEpoch: host.hostEpoch,
sessionId: fixture.sessionId,
messageId,
content,
placement: 'next_turn' as const,
};
const [firstResult, secondResult] = await Promise.all([
first.request('turn.message.submit', input),
second.request('turn.message.submit', input),
]);
assert.deepEqual(secondResult, firstResult);
assert.equal(firstResult.disposition, 'turn_started');
if (firstResult.disposition !== 'turn_started') return;
await waitForTerminalTurn(first, fixture.sessionId, firstResult.turnId);
await first.close();
await second.close();
await fixture.stopHost(host);
const chain = await fixture.readAdmissionChain();
assert.equal(chain.length, 1);
assert.deepEqual(chain[0]?.normalizedInput, content);
assert.deepEqual(chain[0]?.sourceMessages, [
{ messageId, content, placement: 'next_turn', disposition: 'turn_started' },
]);
const ledger = await fixture.readTurn(firstResult.turnId);
assert.equal(ledger.runs.length, 1);
assert.equal(ledger.userMessages.length, 1);
assert.equal(ledger.userMessages[0]?.id, messageId);
assert.equal(ledger.userMessages[0]?.text, content.text);
assert.equal(ledger.userMessages[0]?.displayText, content.displayText);
assert.deepEqual(ledger.userMessages[0]?.attachments, content.attachments);
});
});
test('stale Session operations return not_found across the SQLite-backed UDS Host boundary', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const client = await connectClient(fixture.root, 'desktop');
const staleSessionId = randomUUID();
try {
await assert.rejects(
() =>
client.request('turn.message.submit', {
originHostEpoch: host.hostEpoch,
sessionId: staleSessionId,
messageId: randomUUID(),
content: { text: 'stale submit' },
placement: 'next_turn',
}),
operationError('not_found'),
);
await assert.rejects(
() =>
client.request('turn.interrupt', {
originHostEpoch: host.hostEpoch,
sessionId: staleSessionId,
interruptId: randomUUID(),
turnId: randomUUID(),
runId: randomUUID(),
}),
operationError('not_found'),
);
await assert.rejects(
() =>
client.startTurn({
sessionId: staleSessionId,
turnId: randomUUID(),
content: { text: 'stale start' },
}),
operationError('not_found'),
);
} finally {
await client.close();
await fixture.stopHost(host);
}
});
});
test('steering becomes durable and ordered followups automatically start the next root', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const first = await connectClient(fixture.root, 'desktop');
const second = await connectClient(fixture.root, 'tui');
const firstTurnId = randomUUID();
await first.startTurn({
sessionId: fixture.sessionId,
turnId: firstTurnId,
content: { text: `long-running root ${'x'.repeat(540)}` },
});
const steeringId = randomUUID();
const steeringContent = {
text: '<steer>use the correction</steer>',
displayText: 'use the correction',
attachments: [attachment('steering', 'correction.png')],
};
const followupSources: Array<{ messageId: string; content: MessageContent }> = [
{
messageId: randomUUID(),
content: {
text: '<followup>first queued task</followup>',
displayText: 'first queued task',
attachments: [attachment('followup-first', 'first.png')],
quotes: quoteRefs('followup-first'),
},
},
{
messageId: randomUUID(),
content: {
text: 'second queued task',
quotes: [
{
text: 'second followup excerpt',
sourceTurnId: 'turn-followup-second',
},
],
},
},
];
assert.equal(
(
await second.request('turn.message.submit', {
originHostEpoch: host.hostEpoch,
sessionId: fixture.sessionId,
messageId: steeringId,
content: steeringContent,
placement: 'current_turn',
})
).disposition,
'steering',
);
for (const source of followupSources) {
assert.equal(
(
await second.request('turn.message.submit', {
originHostEpoch: host.hostEpoch,
sessionId: fixture.sessionId,
...source,
placement: 'next_turn',
})
).disposition,
'followup',
);
}
assert.equal(
(await waitForTerminalTurn(first, fixture.sessionId, firstTurnId)).status,
'completed',
);
await waitForDurableMessageConflict(second, {
originHostEpoch: 'previous-host-epoch',
sessionId: fixture.sessionId,
messageId: followupSources[0]!.messageId,
content: { text: 'deliberately different durable identity probe' },
placement: 'next_turn',
});
await first.close();
await second.close();
await fixture.stopHost(host);
const firstLedger = await fixture.readTurn(firstTurnId);
const steeringEvents = firstLedger.runtimeEvents.filter(
(event) =>
event.refs?.providerEventId === steeringId &&
event.content?.kind === 'text' &&
event.content.steering === true,
);
assert.equal(steeringEvents.length, 1);
assert.equal(steeringEvents[0]?.content?.kind, 'text');
if (steeringEvents[0]?.content?.kind === 'text') {
const { kind: _kind, steering: _steering, ...durableContent } = steeringEvents[0].content;
assert.deepEqual(durableContent, steeringContent);
}
const chain = await fixture.readAdmissionChain();
assert.equal(chain.length, 2);
assert.equal(chain[1]?.previousRootTurnId, firstTurnId);
assert.deepEqual(
chain[1]?.sourceMessages,
followupSources.map((source) => ({
...source,
placement: 'next_turn',
disposition: 'followup',
})),
);
assert.deepEqual(chain[1]?.normalizedInput, {
text: `${followupSources[0].content.text}\n\n${followupSources[1].content.text}`,
displayText: `${followupSources[0].content.displayText}\n\n${followupSources[1].content.text}`,
attachments: followupSources[0].content.attachments,
quotes: followupSources.flatMap((source) => source.content.quotes ?? []),
});
const followupTurnId = chain[1]?.turnId;
assert.ok(followupTurnId);
const followupLedger = await fixture.readTurn(followupTurnId);
const expectedQuotes = followupSources.flatMap((source) => source.content.quotes ?? []);
assert.equal(followupLedger.userMessages.length, 1);
assert.deepEqual(followupLedger.userMessages[0]?.quotes, expectedQuotes);
assert.deepEqual(userRuntimeContent(followupLedger.runtimeEvents)?.quotes, expectedQuotes);
});
});
test('explicit retract is durable across connections and prevents successor admission', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const first = await connectClient(fixture.root, 'desktop');
const second = await connectClient(fixture.root, 'tui');
const turnId = randomUUID();
const started = await first.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text: FAKE_ASK_USER_QUESTION_PROMPT },
});
const messageId = randomUUID();
const submitted = await first.request('turn.message.submit', {
originHostEpoch: host.hostEpoch,
sessionId: fixture.sessionId,
messageId,
content: { text: 'withdraw before successor admission' },
placement: 'next_turn',
});
assert.equal(submitted.disposition, 'followup');
const retractInput = {
originHostEpoch: host.hostEpoch,
sessionId: fixture.sessionId,
retractId: randomUUID(),
};
const retracted = await second.request('queue.retract', retractInput);
assert.deepEqual(
retracted.retracted.map((entry) => ({ messageId: entry.messageId, state: entry.state })),
[{ messageId, state: 'retracted' }],
);
await second.close();
const retrying = await connectClient(fixture.root, 'run');
assert.deepEqual(await retrying.request('queue.retract', retractInput), retracted);
const terminal = await first.stopTurn({
sessionId: fixture.sessionId,
turnId,
runId: started.runId,
});
assert.equal(terminal.status, 'cancelled');
await first.close();
await retrying.close();
await fixture.stopHost(host);
const chain = await fixture.readAdmissionChain();
assert.deepEqual(
chain.map((admission) => admission.turnId),
[turnId],
);
});
});
test('interrupt atomically retracts queued followup, stops the exact run, and is idempotent', async () => {
await withExecutionRoot(async (fixture) => {
const host = await fixture.startHost();
const first = await connectClient(fixture.root, 'desktop');
const second = await connectClient(fixture.root, 'tui');
const turnId = randomUUID();
const started = await first.startTurn({
sessionId: fixture.sessionId,
turnId,
content: { text: FAKE_ASK_USER_QUESTION_PROMPT },
});
const followupId = randomUUID();
const followupContent = {
text: '<followup>must be withdrawn</followup>',
displayText: 'must be withdrawn',
attachments: [attachment('interrupt-followup', 'withdraw.png')],
};
await second.request('turn.message.submit', {
originHostEpoch: host.hostEpoch,
sessionId: fixture.sessionId,
messageId: followupId,
content: followupContent,
placement: 'next_turn',
});
const interruptInput = {
originHostEpoch: host.hostEpoch,
sessionId: fixture.sessionId,
interruptId: randomUUID(),
turnId,
runId: started.runId,
};
const [interrupted, concurrentRetry] = await Promise.all([
first.request('turn.interrupt', interruptInput, PROCESS_TIMEOUT_MS),
second.request('turn.interrupt', interruptInput, PROCESS_TIMEOUT_MS),
]);
assert.deepEqual(concurrentRetry, interrupted);
assert.deepEqual(
await second.request('turn.interrupt', interruptInput, PROCESS_TIMEOUT_MS),
interrupted,
);
assert.equal(interrupted.turn.turnId, turnId);
assert.equal(interrupted.turn.runId, started.runId);
assert.equal(interrupted.turn.status, 'cancelled');
assert.equal(interrupted.retracted.length, 1);
assert.ok(interrupted.retracted[0]?.entryId);
assert.deepEqual(interrupted.retracted, [
{
entryId: interrupted.retracted[0]?.entryId,
messageId: followupId,
content: followupContent,
placement: 'next_turn',
state: 'retracted',
},
]);
await first.close();
await second.close();
await fixture.stopHost(host);
const chain = await fixture.readAdmissionChain();
assert.equal(chain.length, 1);
assert.equal(chain[0]?.turnId, turnId);
});
});
test('old-Epoch Message submit returns only exact durable outcomes', async () => {
await withExecutionRoot(async (fixture) => {
const firstHost = await fixture.startHost();
const first = await connectClient(fixture.root, 'desktop');
const rootMessageId = randomUUID();
const rootContent = { text: `durable root ${'x'.repeat(360)}` };
const rootResult = await first.request('turn.message.submit', {
originHostEpoch: firstHost.hostEpoch,
sessionId: fixture.sessionId,
messageId: rootMessageId,
content: rootContent,
placement: 'next_turn',
});
assert.equal(rootResult.disposition, 'turn_started');
if (rootResult.disposition !== 'turn_started') return;
await waitForRunningTurn(first, fixture.sessionId, rootResult.turnId);
const steeringId = randomUUID();
const steeringContent = { text: 'durable steering proof' };
await first.request('turn.message.submit', {
originHostEpoch: firstHost.hostEpoch,
sessionId: fixture.sessionId,
messageId: steeringId,
content: steeringContent,
placement: 'current_turn',
});
await waitForTerminalTurn(first, fixture.sessionId, rootResult.turnId);
await first.close();
await fixture.stopHost(firstHost);
const successorHost = await fixture.startHost();
const successor = await connectClient(fixture.root, 'run');
assert.deepEqual(
await successor.request('turn.message.submit', {
originHostEpoch: firstHost.hostEpoch,
sessionId: fixture.sessionId,
messageId: rootMessageId,
content: rootContent,
placement: 'next_turn',
}),
rootResult,
);
await assert.rejects(
() =>
successor.request('turn.message.submit', {
originHostEpoch: firstHost.hostEpoch,
sessionId: fixture.sessionId,
messageId: steeringId,
content: steeringContent,
placement: 'current_turn',
}),
operationError('outcome_unknown'),
);
await assert.rejects(
() =>
successor.request('turn.message.submit', {
originHostEpoch: firstHost.hostEpoch,
sessionId: fixture.sessionId,
messageId: randomUUID(),
content: { text: 'no durable proof exists' },
placement: 'next_turn',
}),
operationError('outcome_unknown'),
);
await successor.close();
await fixture.stopHost(successorHost);
});
});
interface TaskCreateInput {
tasks: Array<{ subject: string; parent_id?: string }>;
}
interface TaskUpdateInput {
id: string;
status?: 'pending' | 'in_progress' | 'blocked' | 'completed' | 'failed' | 'cancelled';
subject?: string;
blockedReason?: string;
failureReason?: string;
completionEvidence?: string;
explicitReopen?: boolean;
}
type TaskLedgerPage = Extract<TaskLedgerQueryResult, { kind: 'page' }>;
type TaskLedgerTool<Input> = MakaTool<Input, string> & {
parameters: { parse(value: unknown): Input };
};
async function withOwnedTaskLedgerToolPort<T>(
fixture: ExecutionFixture,
run: (coordinator: HostTaskLedgerCoordinator, tools: MakaTool[]) => Promise<T>,
): Promise<T> {
const owner = await tryAcquireInteractiveRootOwner(fixture.capability);
assert.ok(owner);
if (!owner) throw new Error('Unable to acquire the interactive Task Ledger tool port');
try {
const writer = await openInteractiveTaskLedgerStoreForWrite(owner.lease);
const coordinator = new HostTaskLedgerCoordinator(writer, new SessionAdmissionGate());
return await run(coordinator, buildTaskLedgerTools({ store: coordinator }));
} finally {
await owner.close();
}
}
function requireTaskLedgerTool<Input>(
tools: readonly MakaTool[],
name: 'task_create' | 'task_update',
): TaskLedgerTool<Input> {
const tool = tools.find((candidate) => candidate.name === name);
assert.ok(tool, `Expected ${name} Runtime tool`);
return tool as TaskLedgerTool<Input>;
}
function taskLedgerToolContext(
fixture: ExecutionFixture,
identity: Pick<MakaToolContext, 'runId' | 'turnId' | 'toolCallId'>,
): MakaToolContext {
return {
sessionId: fixture.sessionId,
cwd: fixture.root,
...identity,
abortSignal: new AbortController().signal,
emitOutput: () => {},
};
}
async function collectTaskLedgerProjection(
client: RuntimeHostConnection,
sessionId: string,
): Promise<{
revision: TaskLedgerRevision;
pages: TaskLedgerPage[];
tasks: Task[];
}> {
const pages: TaskLedgerPage[] = [];
let result = await client.request('task.ledger.query', {
kind: 'list_start',
sessionId,
});
assert.equal(result.kind, 'page');
if (result.kind !== 'page') throw new Error('Expected initial Task Ledger page');
const revision = result.revision;
while (true) {
assert.equal(result.sessionId, sessionId);
assert.equal(result.revision, revision);
pages.push(result);
if (result.nextCursor === null) break;
result = await client.request('task.ledger.query', {
kind: 'list_continue',
sessionId,
revision,
cursor: result.nextCursor,
});
assert.equal(result.kind, 'page');
if (result.kind !== 'page') {
throw new Error('Task Ledger changed while collecting a stable projection');
}
}
return {
revision,
pages,
tasks: pages.flatMap((page) => page.tasks),
};
}
interface ExecutionHostHandle {
child: ChildProcess;
hostEpoch: string;
endpoint: string;
}
interface TurnLedger {
runs: AgentRunHeader[];
userMessages: Array<Extract<StoredMessage, { type: 'user' }>>;
runtimeEvents: RuntimeEvent[];
terminalEvents: RuntimeEvent[];
classification: ReturnType<typeof classifyTerminalRuntimeLedger>;
}
class ExecutionFixture {
readonly #children = new Set<ChildProcess>();
constructor(
readonly base: string,
readonly root: string,
readonly capability: StorageRootCapability<'interactive'>,
readonly sessionId: string,
) {}
sessionPath(): string {
return join(this.root, 'sessions', this.sessionId, 'session.jsonl');
}
runtimeEventsPath(runId: string): string {
return join(this.root, 'sessions', this.sessionId, 'runs', runId, 'runtime-events.jsonl');
}
eventsPath(runId: string): string {
return join(this.root, 'sessions', this.sessionId, 'runs', runId, 'events.jsonl');
}
async seedPendingChildAdmission(
kind:
| 'linked_child_initial'
| 'linked_child_resume'
| 'linked_child_provider_retry'
| 'claimed_agent_graph_intent',
): Promise<{
kind:
| 'linked_child_initial'
| 'linked_child_resume'
| 'linked_child_provider_retry'
| 'claimed_agent_graph_intent';
sessionId: string;
turnId: string;
runId: string;
sourceRunId: string | undefined;
userMessageId: string | null;
agentId: string;
agentName: string;
}> {
const owner = await tryAcquireInteractiveRootOwner(this.capability);
assert.ok(owner);
if (!owner) throw new Error('Unable to acquire execution root for child admission setup');
try {
const stores = await openInteractiveExecutionStoresForWrite(owner.lease);
const turnId = randomUUID();
const runId = randomUUID();
const sourceRunId =
kind === 'linked_child_resume' || kind === 'linked_child_provider_retry'
? randomUUID()
: undefined;
const agentId = 'local-read';
const agentName = 'Local Read';
const child = await stores.sessionStore.createSubagent({
cwd: this.root,
name: `${agentName} ${kind}`,
backend: 'fake',
llmConnectionSlug: 'fake',
model: 'fake-model',
permissionMode: 'explore',
collaborationMode: 'agent',
orchestrationMode: 'default',
subagentParent: {
kind: 'subagent',
parentSessionId: this.sessionId,
spawnedBy: {
parentRunId: `parent-${kind}`,
parentTurnId: `parent-turn-${kind}`,
toolCallId: `tool-${kind}`,
},
lifecycle: 'foreground',
},
subagentRuntime: {
schemaVersion: 1,
definitionVersion: 1,
agentId,
agentName,
profile: 'local_read',
systemPrompt: 'Read the assigned workspace task.',
toolNames: ['Read', 'Glob', 'Grep'],
categoryPolicy: { read: 'allow' },
permissionCeiling: 'ask',
},
subagentSpawn: {
schemaVersion: 1,
requestFingerprint: (kind === 'linked_child_initial'
? 'a'
: kind === 'linked_child_resume'
? 'b'
: kind === 'linked_child_provider_retry'
? 'c'
: 'd'
).repeat(64),
initialTurnId:
kind === 'linked_child_initial' || kind === 'claimed_agent_graph_intent'
? turnId
: `initial-${kind}`,
initialRunId:
kind === 'linked_child_initial' || kind === 'claimed_agent_graph_intent'
? runId
: sourceRunId!,
},
});
assert.equal(child.created, true);
if (sourceRunId) {
const sourceTs = Date.now();
const sourceRun: AgentRunHeader = {
runId: sourceRunId,
invocationId: sourceRunId,
sessionId: child.header.id,
turnId: `source-turn-${kind}`,
status: 'created',
backendKind: 'fake',
llmConnectionSlug: 'fake',
modelId: 'fake-model',
cwd: this.root,
permissionMode: 'explore',
collaborationMode: 'agent',
createdAt: sourceTs,
updatedAt: sourceTs,
agentId,
agentName,
};
await stores.agentRunStore.createRun(sourceRun, { durable: true });
const sourceTerminal = buildRecoveredTerminalRuntimeEvent({
id: randomUUID(),
run: sourceRun,
status: 'failed',
ts: sourceTs,
failureClass: kind === 'linked_child_provider_retry' ? 'RateLimit' : 'source_failed',
recoveryReason: 'test_source_terminal',
});
await commitTerminalRunWithRuntimeFact({
runStore: stores.agentRunStore,
runtimeEventStore: stores.runtimeEventStore,
newId: randomUUID,
sessionId: child.header.id,
runId: sourceRunId,
turnId: sourceRun.turnId,
status: 'failed',
ts: sourceTs,
terminalEvent: sourceTerminal,
failureClass: kind === 'linked_child_provider_retry' ? 'RateLimit' : 'source_failed',
});
}
const userMessageId = kind === 'linked_child_provider_retry' ? null : randomUUID();
const admitted = await stores.agentRunStore.admitRootTurn({
sessionId: child.header.id,
turnId,
proposedRunId: runId,
proposedUserMessageId: userMessageId,
execution:
kind === 'linked_child_initial'
? { kind, agentId, agentName }
: kind === 'claimed_agent_graph_intent'
? {
kind,
claim: {
schemaVersion: 1,
claimId: `graph_claim_${'a'.repeat(32)}`,
graphId: `graph-${runId}`,
intentId: `graph_intent_${'b'.repeat(32)}`,
intentFingerprint: `sha256:${'c'.repeat(64)}`,
readinessContextFingerprint: `sha256:${'d'.repeat(64)}`,
targetOperatorId: 'local-read',
targetSessionId: child.header.id,
targetTurnId: turnId,
targetRunId: runId,
claimedAt: Date.now(),
},
agentId,
agentName,
}
: { kind, agentId, agentName, sourceRunId: sourceRunId! },
previousRootTurnId: null,
normalizedInput: { text: `pending ${kind}` },
sourceMessages: [],
admittedAt: Date.now(),
});
assert.equal(admitted.kind, 'admitted');
return {
kind,
sessionId: child.header.id,
turnId,
runId,
sourceRunId,
userMessageId,
agentId,
agentName,
};
} finally {
await owner.close();
}
}
async seedClaimedGraphRunLineageDrift(): Promise<void> {
const graph = await this.seedPendingChildAdmission('claimed_agent_graph_intent');
const owner = await tryAcquireInteractiveRootOwner(this.capability);
assert.ok(owner);
if (!owner) throw new Error('Unable to acquire execution root for graph lineage setup');
try {
const stores = await openInteractiveExecutionStoresForWrite(owner.lease);
const ts = Date.now();
await stores.agentRunStore.createRun(
{
runId: graph.runId,
invocationId: graph.runId,
sessionId: graph.sessionId,
turnId: graph.turnId,
status: 'created',
backendKind: 'fake',
llmConnectionSlug: 'fake',
modelId: 'fake-model',
cwd: this.root,
permissionMode: 'explore',
collaborationMode: 'agent',
createdAt: ts,
updatedAt: ts,
resumedFromRunId: randomUUID(),
agentId: graph.agentId,
agentName: graph.agentName,
},
{ durable: true },
);
} finally {
await owner.close();
}
}
seedAdmission(
turnId: string,
content: string | MessageContent,
): Promise<{ runId: string; userMessageId: string }> {
return this.seedTurnState(turnId, content, false, false);
}
async archiveSession(): Promise<void> {
const owner = await tryAcquireInteractiveRootOwner(this.capability);
assert.ok(owner);
if (!owner) throw new Error('Unable to acquire execution root for archive');
try {
const stores = await openInteractiveExecutionStoresForWrite(owner.lease);
await stores.sessionStore.archive(this.sessionId);
} finally {
await owner.close();
}
}
seedRunWithoutUserMessage(
turnId: string,
content: string | MessageContent,
): Promise<{ runId: string; userMessageId: string }> {
return this.seedTurnState(turnId, content, true, false);
}
seedRunWithUserMessage(
turnId: string,
content: MessageContent,
): Promise<{ runId: string; userMessageId: string }> {
return this.seedTurnState(turnId, content, true, true);
}
private async seedTurnState(
turnId: string,
input: string | MessageContent,
createRun: boolean,
createUserMessage: boolean,
): Promise<{ runId: string; userMessageId: string }> {
const owner = await tryAcquireInteractiveRootOwner(this.capability);
assert.ok(owner);
if (!owner) throw new Error('Unable to acquire execution root for admission setup');
try {
const stores = await openInteractiveExecutionStoresForWrite(owner.lease);
const admittedAt = Date.now();
const content = typeof input === 'string' ? { text: input } : input;
const result = await stores.agentRunStore.admitRootTurn({
sessionId: this.sessionId,
turnId,
proposedRunId: randomUUID(),
proposedUserMessageId: randomUUID(),
execution: { kind: 'external_message' },
previousRootTurnId: null,
normalizedInput: content,
sourceMessages: [],
admittedAt,
});
assert.equal(result.kind, 'admitted');
if (createRun) {
await stores.agentRunStore.createRun({
runId: result.admission.runId,
invocationId: result.admission.runId,
sessionId: this.sessionId,
turnId,
status: 'created',
backendKind: 'fake',
llmConnectionSlug: 'fake',
modelId: 'fake-model',
cwd: this.root,
permissionMode: 'ask',
createdAt: admittedAt,
updatedAt: admittedAt,
});
}
assert.ok(result.admission.userMessageId);
if (createUserMessage) {
await stores.sessionStore.appendMessage(this.sessionId, {
type: 'user',
id: result.admission.userMessageId,
turnId,
ts: admittedAt,
...content,
});
}
return {
runId: result.admission.runId,
userMessageId: result.admission.userMessageId,
};
} finally {
await owner.close();
}
}
async startHost(): Promise<ExecutionHostHandle> {
const child = this.spawnHost('inherit');
const ready = await waitForHostReady(child);
return { child, ...ready };
}
async expectHostStartupFailure(): Promise<void> {
const child = this.spawnHost('ignore');
await assert.rejects(() => waitForHostReady(child), /execution Host exited before readiness/);
await withTimeout(waitForExit(child), PROCESS_TIMEOUT_MS, 'failed execution Host did not exit');
this.#children.delete(child);
}
async assertOwnerAvailable(): Promise<void> {
const owner = await tryAcquireInteractiveRootOwner(this.capability);
assert.ok(owner);
await owner?.close();
}
async stopHost(
host: ExecutionHostHandle,
): Promise<{ code: number | null; signal: NodeJS.Signals | null }> {
if (host.child.exitCode === null && host.child.signalCode === null) {
host.child.kill('SIGTERM');
}
const exit = await withTimeout(
waitForExitResult(host.child),
PROCESS_TIMEOUT_MS + 2_000,
'execution Host did not stop',
);
this.#children.delete(host.child);
if (exit.code !== 0 || exit.signal !== null) {
throw new Error(
`execution Host stopped uncleanly: ${exit.code === null ? exit.signal : `code ${exit.code}`}`,
);
}
return exit;
}
async killHost(host: ExecutionHostHandle): Promise<void> {
host.child.kill('SIGKILL');
await withTimeout(
waitForExit(host.child),
PROCESS_TIMEOUT_MS,
'execution Host survived SIGKILL',
);
this.#children.delete(host.child);
}
async waitForHostExit(host: ExecutionHostHandle): Promise<void> {
await withTimeout(
waitForExit(host.child),
PROCESS_TIMEOUT_MS,
'draining execution Host did not exit',
);
this.#children.delete(host.child);
}
async readTurn(turnId: string): Promise<TurnLedger> {
const reader = await acquireReader(this.capability);
try {
const stores = await openInteractiveExecutionStoresForRead(reader.lease);
const admission = await stores.agentRunStore.readRootTurnAdmission(this.sessionId, turnId);
assert.ok(admission);
const runs = (await stores.agentRunStore.listSessionRuns(this.sessionId)).filter(
(candidate) => candidate.turnId === turnId,
);
const run = await stores.agentRunStore.readRun(this.sessionId, admission.runId);
const messages = await stores.sessionStore.readMessages(this.sessionId);
const runtimeEvents = await stores.runtimeEventStore.readImmutableRuntimeEvents(
this.sessionId,
admission.runId,
);
return {
runs,
userMessages: messages.filter(
(message): message is Extract<StoredMessage, { type: 'user' }> =>
message.type === 'user' && message.turnId === turnId,
),
runtimeEvents,
terminalEvents: runtimeEvents.filter(isTerminalRuntimeEvent),
classification: classifyTerminalRuntimeLedger(run, runtimeEvents),
};
} finally {
await reader.close();
}
}
async readAdmissionChain() {
const owner = await tryAcquireInteractiveRootOwner(this.capability);
assert.ok(owner);
if (!owner) throw new Error('Unable to acquire execution root for admission inspection');
try {
const stores = await openInteractiveExecutionStoresForWrite(owner.lease);
return stores.agentRunStore.listRootTurnAdmissionsForRecovery(this.sessionId);
} finally {
await owner.close();
}
}
async readTurnFootprint(turnId: string): Promise<{
admitted: boolean;
runCount: number;
userMessageCount: number;
}> {
const reader = await acquireReader(this.capability);
try {
const stores = await openInteractiveExecutionStoresForRead(reader.lease);
const [admission, runs, messages] = await Promise.all([
stores.agentRunStore.readRootTurnAdmission(this.sessionId, turnId),
stores.agentRunStore.listSessionRuns(this.sessionId),
stores.sessionStore.readMessages(this.sessionId),
]);
return {
admitted: admission !== undefined,
runCount: runs.filter((run) => run.turnId === turnId).length,
userMessageCount: messages.filter(
(message) => message.type === 'user' && message.turnId === turnId,
).length,
};
} finally {
await reader.close();
}
}
async close(): Promise<void> {
for (const child of this.#children) {
if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL');
await withTimeout(waitForExit(child), 1_000, 'cleanup Host did not exit').catch(
() => undefined,
);
}
await rm(join(resolveRootControlNamespace(), this.capability.rootId), {
recursive: true,
force: true,
});
await removePosixEndpointDirectories(this.capability.rootId);
await rm(this.base, { recursive: true, force: true });
}
private spawnHost(stderr: 'inherit' | 'ignore'): ChildProcess {
const child = fork(
new URL('./fixtures/execution-host.js', import.meta.url),
[this.root, this.capability.rootId, '60000'],
{ stdio: ['ignore', 'ignore', stderr, 'ipc'] },
);
this.#children.add(child);
return child;
}
}
async function withExecutionRoot(run: (fixture: ExecutionFixture) => Promise<void>): Promise<void> {
const base = await mkdtemp(join(tmpdir(), 'maka-runtime-host-execution-'));
const root = join(base, 'root');
const capability = await resolveStorageRoot({
path: root,
kind: 'interactive',
});
const owner = await tryAcquireInteractiveRootOwner(capability);
assert.ok(owner);
let sessionId: string;
try {
const stores = await openInteractiveExecutionStoresForWrite(owner.lease);
const session = await stores.sessionStore.create({
cwd: root,
backend: 'fake',
llmConnectionSlug: 'fake',
model: 'fake-model',
permissionMode: 'ask',
});
sessionId = session.id;
} finally {
await owner.close();
}
const fixture = new ExecutionFixture(base, root, capability, sessionId);
try {
await run(fixture);
} finally {
await fixture.close();
}
}
async function connectClient(
rootPath: string,
surface: 'desktop' | 'tui' | 'run',
): Promise<RuntimeHostConnection> {
const result = await connectRuntimeHost({
rootPath,
surface,
protocol: CURRENT_PROTOCOL,
});
assert.equal(result.kind, 'connected');
return result.connection;
}
async function sendStartWithoutReadingResponse(
endpoint: string,
input: { sessionId: string; turnId: string; text: string },
): Promise<FramedTransport> {
const transport = new FramedTransport(await openSocket(endpoint));
await transport.write({
kind: 'hello',
clientInstanceId: randomUUID(),
surface: 'desktop',
protocolMin: CURRENT_PROTOCOL.min,
protocolMax: CURRENT_PROTOCOL.max,
});
const handshake = decodeHostFrame(await transport.read(2_000));
assert.ok('kind' in handshake);
assert.equal(handshake.kind, 'accepted');
await transport.write({
requestId: randomUUID(),
operation: 'turn.start',
input: {
sessionId: input.sessionId,
turnId: input.turnId,
content: { text: input.text },
},
});
return transport;
}
function openSocket(path: string): Promise<Socket> {
return new Promise((resolve, reject) => {
const socket = connect(path);
const onError = (error: Error) => {
socket.off('connect', onConnect);
reject(error);
};
const onConnect = () => {
socket.off('error', onError);
resolve(socket);
};
socket.once('error', onError);
socket.once('connect', onConnect);
});
}
async function waitForTurn(
connection: RuntimeHostConnection,
sessionId: string,
turnId: string,
): Promise<TurnSnapshot> {
const deadline = Date.now() + PROCESS_TIMEOUT_MS;
while (true) {
try {
return await connection.queryTurn({ sessionId, turnId });
} catch (error) {
if (!(error instanceof RuntimeHostOperationError) || error.code !== 'not_found') throw error;
if (Date.now() >= deadline) throw new Error('Turn admission was not observed');
await sleep(20);
}
}
}
class SubscriptionProbe {
readonly frames: SubscriptionFrame[] = [];
readonly done: Promise<void>;
#failure: unknown;
#settled = false;
constructor(subscription: RuntimeHostSessionSubscription) {
this.done = this.#consume(subscription);
}
async waitFor(
predicate: (frame: SubscriptionFrame) => boolean,
message: string,
): Promise<SubscriptionFrame> {
const deadline = Date.now() + PROCESS_TIMEOUT_MS;
while (true) {
const frame = this.frames.find(predicate);
if (frame) return frame;
if (this.#failure) throw this.#failure;
if (this.#settled) throw new Error(`${message}: subscription closed`);
if (Date.now() >= deadline) throw new Error(message);
await sleep(10);
}
}
async waitForFailure(reason: RuntimeHostSubscriptionError['reason']): Promise<void> {
const deadline = Date.now() + PROCESS_TIMEOUT_MS;
while (!this.#failure && !this.#settled && Date.now() < deadline) await sleep(10);
assert.ok(this.#failure instanceof RuntimeHostSubscriptionError);
assert.equal(this.#failure.reason, reason);
}
indexOf(frame: SubscriptionFrame): number {
return this.frames.indexOf(frame);
}
async #consume(subscription: RuntimeHostSessionSubscription): Promise<void> {
try {
for await (const frame of subscription) this.frames.push(frame);
} catch (error) {
this.#failure = error;
} finally {
this.#settled = true;
}
}
}
async function waitForPendingInteraction(
subscription: RuntimeHostSessionSubscription,
probe: SubscriptionProbe,
runId: string,
): Promise<InteractionPendingSnapshot> {
const initial = subscription.snapshot.interactions.pending.find(
(interaction) => interaction.runId === runId,
);
if (initial) return initial;
const frame = await probe.waitFor(
(candidate) =>
candidate.kind === 'subscription.session_projection' &&
candidate.snapshot.interactions.pending.some((interaction) => interaction.runId === runId),
'subscription did not publish the pending Interaction',
);
assert.equal(frame.kind, 'subscription.session_projection');
const pending = frame.snapshot.interactions.pending.find(
(interaction) => interaction.runId === runId,
);
assert.ok(pending);
return pending;
}
async function waitForTerminalTurn(
connection: RuntimeHostConnection,
sessionId: string,
turnId: string,
): Promise<TurnSnapshot> {
const deadline = Date.now() + PROCESS_TIMEOUT_MS;
while (true) {
const snapshot = await connection.queryTurn({ sessionId, turnId });
if (
snapshot.status === 'completed' ||
snapshot.status === 'failed' ||
snapshot.status === 'cancelled'
) {
return snapshot;
}
if (Date.now() >= deadline) throw new Error('Turn did not reach a terminal fact');
await sleep(20);
}
}
async function waitForRunningTurn(
connection: RuntimeHostConnection,
sessionId: string,
turnId: string,
): Promise<TurnSnapshot> {
const deadline = Date.now() + PROCESS_TIMEOUT_MS;
while (true) {
const snapshot = await connection.queryTurn({ sessionId, turnId });
if (snapshot.status === 'running' || snapshot.status === 'waiting_for_user') return snapshot;
if (Date.now() >= deadline) throw new Error('Turn did not become active');
await sleep(20);
}
}
async function waitForDurableMessageConflict(
connection: RuntimeHostConnection,
input: TurnMessageSubmitInput,
): Promise<void> {
const deadline = Date.now() + PROCESS_TIMEOUT_MS;
while (true) {
try {
await connection.request('turn.message.submit', input);
throw new Error('Conflicting durable Message identity was accepted');
} catch (error) {
if (error instanceof RuntimeHostOperationError && error.code === 'operation_conflict') return;
if (!(error instanceof RuntimeHostOperationError) || error.code !== 'outcome_unknown') {
throw error;
}
}
if (Date.now() >= deadline) throw new Error('Durable Message source was not observed');
await sleep(20);
}
}
function operationError(code: RuntimeHostOperationError['code']) {
return (error: unknown): boolean =>
error instanceof RuntimeHostOperationError && error.code === code;
}
function assertJsonLines(bytes: string): void {
for (const line of bytes.split('\n').filter(Boolean)) {
assert.doesNotThrow(() => JSON.parse(line));
}
}
function attachment(id: string, name: string) {
return {
kind: 'image' as const,
name,
mimeType: 'image/png',
bytes: 10,
ref: { kind: 'workspace_file' as const, relativePath: `attachments/${id}.png` },
};
}
function quoteRefs(prefix: string) {
return [
{
text: `${prefix} first excerpt`,
label: 'Assistant',
sourceTurnId: `turn-${prefix}-1`,
},
{
text: `${prefix} second excerpt`,
sourceTurnId: `turn-${prefix}-2`,
},
];
}
function quotedContent(text: string): MessageContent {
return { text, quotes: quoteRefs(text.replaceAll(' ', '-')) };
}
function userRuntimeContent(
events: readonly RuntimeEvent[],
): Extract<NonNullable<RuntimeEvent['content']>, { kind: 'text' }> | undefined {
for (const event of events) {
if (event.role === 'user' && event.content?.kind === 'text') return event.content;
}
return undefined;
}
function waitForHostReady(child: ChildProcess): Promise<{ hostEpoch: string; endpoint: string }> {
return withTimeout(
new Promise((resolve, reject) => {
const cleanup = () => {
child.off('error', onError);
child.off('exit', onExit);
child.off('message', onMessage);
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const onExit = (code: number | null, signal: NodeJS.Signals | null) => {
cleanup();
reject(new Error(`execution Host exited before readiness: ${code ?? signal}`));
};
const onMessage = (message: unknown) => {
if (!isHostReadyMessage(message)) return;
cleanup();
resolve({ hostEpoch: message.hostEpoch, endpoint: message.endpoint });
};
child.once('error', onError);
child.once('exit', onExit);
child.on('message', onMessage);
}),
PROCESS_TIMEOUT_MS,
'execution Host did not become ready',
);
}
function isHostReadyMessage(
value: unknown,
): value is { type: 'ready'; hostEpoch: string; endpoint: string } {
if (!value || typeof value !== 'object') return false;
const message = value as Record<string, unknown>;
return (
message.type === 'ready' &&
typeof message.hostEpoch === 'string' &&
typeof message.endpoint === 'string'
);
}
function waitForExit(child: ChildProcess): Promise<void> {
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
return new Promise((resolve, reject) => {
child.once('error', reject);
child.once('exit', () => resolve());
});
}
function waitForExitResult(
child: ChildProcess,
): Promise<{ code: number | null; signal: NodeJS.Signals | null }> {
if (child.exitCode !== null || child.signalCode !== null) {
return Promise.resolve({ code: child.exitCode, signal: child.signalCode });
}
return new Promise((resolve, reject) => {
const cleanup = () => {
child.off('error', onError);
child.off('exit', onExit);
};
const onError = (error: Error) => {
cleanup();
reject(error);
};
const onExit = (code: number | null, signal: NodeJS.Signals | null) => {
cleanup();
resolve({ code, signal });
};
child.once('error', onError);
child.once('exit', onExit);
});
}
async function acquireReader(capability: StorageRootCapability<'interactive'>) {
const deadline = Date.now() + PROCESS_TIMEOUT_MS;
while (true) {
const reader = await tryAcquireInteractiveRootReader(capability);
if (reader) return reader;
if (Date.now() >= deadline)
throw new Error('Interactive root reader could not acquire the released root');
await sleep(20);
}
}
async function removePosixEndpointDirectories(rootId: string): Promise<void> {
if (process.platform === 'win32' || typeof process.getuid !== 'function') return;
const prefix = `m-${process.getuid()}-${Buffer.from(rootId, 'hex').toString('base64url')}-`;
const entries = await readdir('/tmp', { withFileTypes: true });
await Promise.all(
entries.map(async (entry) => {
if (entry.isDirectory() && entry.name.startsWith(prefix)) {
await rm(join('/tmp', entry.name), { recursive: true, force: true });
}
}),
);
}
function withTimeout<T>(promise: Promise<T>, timeoutMs: number, message: string): Promise<T> {
let timer: NodeJS.Timeout | undefined;
return Promise.race([
promise,
new Promise<T>((_resolve, reject) => {
timer = setTimeout(() => reject(new Error(message)), timeoutMs);
}),
]).finally(() => {
if (timer) clearTimeout(timer);
});
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}