blob: 9cf1fdf9c681953178df0ef0e3d31b75247ee931 [file]
import { z } from 'zod';
import { type CuAction, type CuPoint } from '@maka/core';
import type { CuDispatchEvidence, CuRunResult, CuSemanticAction } from './computer-use-types.js';
export const coordinate = z.tuple([z.number().int().nonnegative(), z.number().int().nonnegative()]);
export const text = z.string().max(8000);
/**
* The sentences the schema's own refinements produce.
*
* A `.refine()` failure carries its message and an empty `issue.path`, so the
* violation describer had no field to name and fell through to the generic
* "the argument shape does not match this action" — throwing away the one
* sentence that said what was missing. They are listed here so the describer
* can pass a message through only when this file wrote it: a message is host
* prose, and anything that did not come from this table could be carrying
* whatever the model or the executor put in it.
*/
export const COMPUTER_USE_REFINEMENT_MESSAGES = {
observeTarget:
'observe requires app or window_id — name the application in `app`, or pass a `window_id` from an earlier observe',
screenshotTarget:
'screenshot requires app or window_id — name the application in `app`, or pass a `window_id` from an earlier observe',
windowMovePosition: 'window_action move requires position',
windowResizeSize: 'window_action resize requires size',
waitOneCondition: 'wait takes one condition, not both — send wait_for_text or wait_for_text_gone',
} as const;
const REFINEMENT_MESSAGE_SET: ReadonlySet<string> = new Set(
Object.values(COMPUTER_USE_REFINEMENT_MESSAGES),
);
/**
* The target hints a model may repeat on an action that already names its
* target through `observation_id`.
*
* The tool schema the model is shown is one flat object: `app` and `window_id`
* are top-level optional parameters described as "exact window_id from
* list_apps or observe", so a model being careful about which window it is
* driving supplies them. This union then rejected them as unrecognized keys.
*
* On a real desktop run that cost six of eleven calls: the model repeated the
* same shape three times, was told only "arguments failed validation", guessed
* a different shape, and abandoned the arithmetic half-finished.
*
* They are accepted and then checked against the observation the action is
* bound to — never trusted as the target. Dispatch still resolves through the
* frame, and a hint that contradicts the frame is reported rather than ignored.
*/
const redundantTargetHints = {
app: z.string().min(1).max(512).optional(),
window_id: z.number().int().positive().optional(),
} as const;
const pointerAction = <
T extends 'left_click' | 'right_click' | 'middle_click' | 'double_click' | 'triple_click',
>(
action: T,
) =>
z
.object({
action: z.literal(action),
observation_id: z.string().min(1).max(256),
coordinate,
text: text.optional(),
})
.strict();
export const computerParams = z.discriminatedUnion('action', [
z
.object({
action: z.literal('list_apps'),
/**
* Narrow the list to what was asked for.
*
* Unfiltered this was 12,933 bytes on a real run — about 3,600 tokens,
* 85% of a three-call turn — spent confirming an app id the prompt had
* already named. Every model tried, from the strongest to the weakest,
* because it is the only bridge from a display name to the app id
* `observe` requires.
*/
app: z.string().min(1).max(512).optional(),
})
.strict(),
z
.object({
action: z.literal('launch_app'),
// The model names an app; everything else about how it is launched stays
// host-controlled. The driver also accepts arbitrary argv and a WebKit
// inspector port, neither of which the model gets to set.
app: z.string().min(1).max(512),
})
.strict(),
z
.object({
action: z.literal('observe'),
app: z.string().min(1).max(512).optional(),
window_id: z.number().int().positive().optional(),
include_screenshot: z.boolean().optional(),
// §5.8. Every observation lists the menu titles; this opens one of them.
// A title rather than a path, because only the top level can be opened —
// a submenu comes with the menu that contains it.
menu: z.string().min(1).max(256).optional(),
// Narrows what is written, never what can be addressed.
query: z.string().min(1).max(256).optional(),
})
.strict()
.refine((input) => input.app !== undefined || input.window_id !== undefined, {
// No mention of approval: whether a call is reviewed is a host pipeline
// the model cannot see, cannot influence, and cannot fix by re-sending.
// What it can fix is the missing argument.
message: COMPUTER_USE_REFINEMENT_MESSAGES.observeTarget,
}),
z
.object({
action: z.literal('click_element'),
observation_id: z.string().min(1).max(256),
element_id: z.string().min(1).max(256),
...redundantTargetHints,
})
.strict(),
z
.object({
action: z.literal('set_value'),
observation_id: z.string().min(1).max(256),
element_id: z.string().min(1).max(256),
value: text,
...redundantTargetHints,
})
.strict(),
z
.object({
action: z.literal('select_text'),
observation_id: z.string().min(1).max(256),
element_id: z.string().min(1).max(256),
text,
...redundantTargetHints,
})
.strict(),
z
.object({
action: z.literal('secondary_action'),
observation_id: z.string().min(1).max(256),
element_id: z.string().min(1).max(256),
text,
...redundantTargetHints,
})
.strict(),
z
.object({
action: z.literal('scroll_element'),
observation_id: z.string().min(1).max(256),
element_id: z.string().min(1).max(256),
scroll_direction: z.enum(['up', 'down', 'left', 'right']),
scroll_amount: z.number().int().min(0).max(100).optional(),
...redundantTargetHints,
})
.strict(),
/**
* Several element actions described the way a person would describe them,
* carried out one at a time by the host.
*
* `element_id` is an index into one snapshot: it is spent the moment anything
* happens, so every action costs the model a round trip to look again. On a
* real seven-application matrix that came to 97 calls for at most five useful
* actions per scenario, and six of seven ran out of time — with a median tool
* call of 734ms. The round trips were the whole cost.
*
* A label is not an index. "Press 7, then multiply, then 8, then equals" is
* true of the calculator before and after each press, so the host can take
* one step, look again with its own eyes, find the next control, and take the
* next — which is what frame binding wants. It exists to stop the MODEL
* acting on a view that has moved on; the host acting on a view it captured
* a moment ago is the case it was built to allow.
*
* Stops at the first step it cannot resolve unambiguously or cannot dispatch,
* and says which one.
*/
z
.object({
action: z.literal('element_sequence'),
observation_id: z.string().min(1).max(256),
steps: z
.array(
z
.object({
label: z.string().min(1).max(256),
role: z.string().min(1).max(64).optional(),
do: z.enum(['click', 'set_value']).optional(),
value: text.optional(),
})
.strict(),
)
.min(1)
.max(12),
...redundantTargetHints,
})
.strict(),
/**
* A window's own geometry, which is not a control on the screen.
*
* `AXPosition` and `AXSize` are settable on nearly every window and writing
* them does not bring the application forward, so this costs none of the
* invariants a drag would. It is separate from the element actions because it
* addresses the window rather than something inside it.
*/
z
.object({
action: z.literal('window_action'),
observation_id: z.string().min(1).max(256),
element_id: z.string().min(1).max(256),
window_action: z.enum(['move', 'resize', 'minimize']),
// Not `coordinate`: that one is non-negative because it addresses a pixel
// inside a window's own screenshot, where there is no such thing as a
// negative offset. A window's position is a place on the desktop, and a
// second display is a real place — measured on this machine, display 2
// sits at (-193, -1080) in the space the observation already reports its
// window bounds in. Refusing a negative here makes half the desktop
// unaddressable.
position: z.tuple([z.number().int(), z.number().int()]).optional(),
size: z.tuple([z.number().int().positive(), z.number().int().positive()]).optional(),
...redundantTargetHints,
})
.strict()
.refine((input) => input.window_action !== 'move' || input.position !== undefined, {
message: COMPUTER_USE_REFINEMENT_MESSAGES.windowMovePosition,
})
.refine((input) => input.window_action !== 'resize' || input.size !== undefined, {
message: COMPUTER_USE_REFINEMENT_MESSAGES.windowResizeSize,
}),
z
.object({
action: z.literal('press_key'),
observation_id: z.string().min(1).max(256),
text,
/**
* Which control the key is for.
*
* Optional, and not a hint: the driver focuses the named element through
* accessibility before posting the key. Without it a key is posted to the
* application and lands on whatever happens to have focus, which is a
* guess the model was already trying to replace — it sent `element_id`
* and was told only that the field was unknown.
*/
element_id: z.string().min(1).max(256).optional(),
...redundantTargetHints,
})
.strict(),
z
.object({
action: z.literal('screenshot'),
app: z.string().min(1).max(512).optional(),
window_id: z.number().int().positive().optional(),
})
.strict()
.refine((input) => input.app !== undefined || input.window_id !== undefined, {
message: COMPUTER_USE_REFINEMENT_MESSAGES.screenshotTarget,
}),
z.object({ action: z.literal('cursor_position') }).strict(),
z
.object({
action: z.literal('mouse_move'),
observation_id: z.string().min(1).max(256),
coordinate,
})
.strict(),
pointerAction('left_click'),
pointerAction('right_click'),
pointerAction('middle_click'),
pointerAction('double_click'),
pointerAction('triple_click'),
z
.object({
action: z.literal('left_mouse_down'),
observation_id: z.string().min(1).max(256),
coordinate,
})
.strict(),
z
.object({
action: z.literal('left_mouse_up'),
observation_id: z.string().min(1).max(256),
coordinate,
})
.strict(),
z
.object({
action: z.literal('left_click_drag'),
observation_id: z.string().min(1).max(256),
start_coordinate: coordinate,
coordinate,
text: text.optional(),
})
.strict(),
z
.object({
action: z.literal('type'),
observation_id: z.string().min(1).max(256),
text,
})
.strict(),
z
.object({
action: z.literal('key'),
observation_id: z.string().min(1).max(256),
text,
})
.strict(),
z
.object({
action: z.literal('hold_key'),
observation_id: z.string().min(1).max(256),
text,
duration: z.number().min(0).max(60).optional(),
})
.strict(),
z
.object({
action: z.literal('scroll'),
observation_id: z.string().min(1).max(256),
coordinate,
scroll_direction: z.enum(['up', 'down', 'left', 'right']).optional(),
scroll_amount: z.number().int().min(0).max(100).optional(),
text: text.optional(),
})
.strict(),
z
.object({
action: z.literal('wait'),
duration: z.number().min(0).max(60).optional(),
// A condition, so the wait can end when the thing happens rather than
// when a guessed number of seconds runs out.
wait_for_text: z.string().min(1).max(256).optional(),
wait_for_text_gone: z.string().min(1).max(256).optional(),
})
.strict()
.refine((input) => !(input.wait_for_text && input.wait_for_text_gone), {
message: COMPUTER_USE_REFINEMENT_MESSAGES.waitOneCondition,
}),
z
.object({
action: z.literal('zoom'),
observation_id: z.string().min(1).max(256),
region: z.tuple([
z.number().int().nonnegative(),
z.number().int().nonnegative(),
z.number().int().nonnegative(),
z.number().int().nonnegative(),
]),
})
.strict(),
]);
export type ComputerParams = z.infer<typeof computerParams>;
/** Every action name this tool accepts, in schema order. */
export function computerActionNames(): string[] {
const names: string[] = [];
for (const option of computerParams.options) {
const shape = option.shape as Record<string, { value?: unknown }>;
const literal = shape.action;
const value = literal?.value ?? (literal as { _def?: { value?: unknown } })?._def?.value;
if (typeof value === 'string') names.push(value);
}
return names;
}
/**
* The argument names one action accepts, or undefined if the action is unknown.
*
* Naming what was rejected leaves the model to guess what to send instead, and
* guessing is what it does: a real run spent twenty of twenty-seven calls
* re-sending shapes that had already been refused, because every refusal said
* what was wrong and none said what was right. The schema already holds the
* answer.
*/
export function computerActionFields(action: unknown): string[] | undefined {
if (typeof action !== 'string') return undefined;
for (const option of computerParams.options) {
const shape = option.shape as Record<string, { value?: unknown }>;
const literal = shape.action;
const value = literal?.value ?? (literal as { _def?: { value?: unknown } })?._def?.value;
if (value === action) return Object.keys(shape).filter((key) => key !== 'action');
}
return undefined;
}
/**
* What was wrong with the arguments, in terms of the argument names only.
*
* Computer Use replaced the generic formatter with a fixed string, because the
* generic one hands the model whatever the error carries and these arguments
* can hold typed text. The cost was a model that could not learn: told only
* "arguments failed validation", it re-sent the identical call three times in a
* row on a real desktop run before guessing at a different shape.
*
* Field names and the schema's own constraints are the model's own input
* vocabulary, not screen content, so they are safe to name. Values never are,
* and none are read here — only `issue.path` and the issue's kind.
*/
export function describeComputerUseArgsViolation(
error: unknown,
args?: unknown,
): string | undefined {
const issues = (error as { issues?: unknown })?.issues;
if (!Array.isArray(issues) || issues.length === 0) return undefined;
const parts: string[] = [];
for (const raw of issues.slice(0, 6)) {
const issue = raw as {
code?: string;
keys?: unknown;
path?: unknown;
expected?: unknown;
message?: unknown;
};
const path = Array.isArray(issue.path) ? issue.path.filter((p) => typeof p === 'string') : [];
const field = path.length > 0 ? path.join('.') : undefined;
// A refinement carries the whole answer in its message and nothing in its
// path, so the field-name branches below cannot see it and the generic
// fallback used to replace it with "the argument shape does not match this
// action" — a model told an `observe` needs one of two named arguments can
// fix the call; a model told the shape is wrong cannot.
//
// Only messages this file wrote are passed through. A message is free
// prose, and one from anywhere else could be quoting the arguments back.
if (typeof issue.message === 'string' && REFINEMENT_MESSAGE_SET.has(issue.message)) {
parts.push(issue.message);
continue;
}
if (issue.code === 'unrecognized_keys' && Array.isArray(issue.keys)) {
const keys = issue.keys.filter((k): k is string => typeof k === 'string');
if (keys.length > 0) {
// Naming the rejected key is half an answer. A model that reached for
// `element_id` on `press_key` was trying to say where the key should
// land, and telling it only that the field is unknown leaves it to
// guess again — which on a real run it did.
const recovery = keys.includes('element_id')
? ' — name the control through the action that acts on it, or click_element it first'
: '';
parts.push(
`this action does not take ${keys.map((k) => `\`${k}\``).join(', ')}${recovery}`,
);
continue;
}
}
if (issue.code === 'invalid_type' && field) {
parts.push(
typeof issue.expected === 'string'
? `\`${field}\` must be ${issue.expected}`
: `\`${field}\` has the wrong type`,
);
continue;
}
if (field) {
parts.push(`\`${field}\` is missing or out of range`);
continue;
}
parts.push('the argument shape does not match this action');
}
const unique = [...new Set(parts)];
if (unique.length === 0) return undefined;
// The correction, not just the complaint. A model told only that four keys
// are unrecognised has no way to know whether it got the whole dialect wrong
// — which on a real run it had, sending every key in camelCase — and it will
// keep sending the same shape. Listing the action's own field names ends
// that in one round trip.
const accepted = computerActionFields((args as { action?: unknown } | undefined)?.action);
const guidance =
accepted && accepted.length > 0
? `. This action takes ${accepted.map((field) => `\`${field}\``).join(', ')}`
: accepted
? '. This action takes no other arguments'
: '';
return `${unique.join('; ')}${guidance}`;
}
const point = (c?: [number, number]): CuPoint | undefined => (c ? { x: c[0], y: c[1] } : undefined);
export function snapshotComputerParams(args: ComputerParams): ComputerParams {
for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(args))) {
if (descriptor.get || descriptor.set) {
throw new Error(`invalid_computer_params: '${key}' must be a plain data property`);
}
}
const cloneTuple = <T extends readonly number[] | undefined>(value: T): T =>
(value ? Object.freeze([...value]) : value) as T;
const source = args as ComputerParams & Record<string, unknown>;
const snapshot = { ...source } as Record<string, unknown>;
if (Object.hasOwn(source, 'coordinate')) {
snapshot.coordinate = cloneTuple(source.coordinate as [number, number] | undefined);
}
if (Object.hasOwn(args, 'start_coordinate')) {
snapshot.start_coordinate = cloneTuple(source.start_coordinate as [number, number] | undefined);
}
if (Object.hasOwn(source, 'region')) {
snapshot.region = cloneTuple(source.region as [number, number, number, number] | undefined);
}
return Object.freeze(snapshot) as ComputerParams;
}
/**
* Map the provider-neutral wire grammar onto the discriminated `CuAction` the
* backend consumes. Throws on a malformed action (missing required field); the
* runtime converts the throw into an error tool-result.
*/
export function adaptToCuAction(args: ComputerParams): CuAction {
const need = (c?: [number, number]): CuPoint => {
const p = point(c);
if (!p) throw new Error(`invalid_coordinate: action '${args.action}' requires coordinate`);
return p;
};
const needText = (value: string | undefined, action: string): string => {
if (typeof value !== 'string' || value.length === 0) {
// Not `invalid_coordinate`: nothing here is about a point on the screen,
// and a model handed a coordinate code for a missing string goes and
// checks its coordinates. The field that is missing is the one named.
throw new Error(
`invalid_arguments: action '${action}' requires text — the characters to type, or the key name, go in \`text\``,
);
}
return value;
};
switch (args.action) {
case 'list_apps':
case 'launch_app':
case 'observe':
case 'click_element':
case 'set_value':
case 'select_text':
case 'secondary_action':
case 'scroll_element':
case 'press_key':
throw new Error(`semantic action '${args.action}' requires the semantic backend`);
case 'screenshot':
return { type: 'screenshot' };
case 'cursor_position':
return { type: 'cursor_position' };
case 'mouse_move':
return { type: 'mouse_move', coordinate: need(args.coordinate) };
case 'left_click':
return { type: 'left_click', coordinate: need(args.coordinate), text: args.text };
case 'right_click':
return { type: 'right_click', coordinate: need(args.coordinate), text: args.text };
case 'middle_click':
return { type: 'middle_click', coordinate: need(args.coordinate), text: args.text };
case 'double_click':
return { type: 'double_click', coordinate: need(args.coordinate), text: args.text };
case 'triple_click':
return { type: 'triple_click', coordinate: need(args.coordinate), text: args.text };
case 'left_mouse_down':
return { type: 'left_mouse_down', coordinate: need(args.coordinate) };
case 'left_mouse_up':
return { type: 'left_mouse_up', coordinate: need(args.coordinate) };
case 'left_click_drag':
return {
type: 'left_click_drag',
startCoordinate: need(args.start_coordinate),
coordinate: need(args.coordinate),
text: args.text,
};
case 'type':
return { type: 'type', text: needText(args.text, args.action) };
case 'key':
return { type: 'key', text: needText(args.text, args.action) };
case 'hold_key':
return {
type: 'hold_key',
text: needText(args.text, args.action),
durationMs: Math.round((args.duration ?? 0) * 1000),
};
case 'scroll':
return {
type: 'scroll',
coordinate: need(args.coordinate),
scrollDirection: args.scroll_direction ?? 'down',
scrollAmount: args.scroll_amount ?? 3,
text: args.text,
};
case 'wait':
return { type: 'wait', durationMs: Math.round((args.duration ?? 0) * 1000) };
case 'zoom': {
if (!args.region) throw new Error("invalid_coordinate: action 'zoom' requires region");
const [x1, y1, x2, y2] = args.region;
return { type: 'zoom', region: { x1, y1, x2, y2 } };
}
default:
// The action name is the one field the model always chooses for itself,
// and the schema already holds the closed set it may choose from. Naming
// it "unknown action" under a coordinate code sent the model looking at
// its coordinates for a word it had misspelled.
throw new Error(
`invalid_arguments: unknown action — this tool takes one of: ${computerActionNames().join(', ')}`,
);
}
}
/**
* Who a summary line is written for.
*
* `host` is the stored/journalled line and keeps everything an operator reading
* a trace back needs. `model` is what goes into the conversation, and it drops
* the fields the model has no move to make about.
*/
export type CuSummaryAudience = 'model' | 'host';
/** Concise summary of an outcome (S16-safe: no screen text here). */
export function summarizeEvidence(
evidence: CuDispatchEvidence | undefined,
audience: CuSummaryAudience = 'model',
): string {
if (!evidence) return '';
const safeToken = (value: string): string | undefined =>
/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/.test(value) ? value : undefined;
const fields: string[] = [];
// `cg_event_pid`, `ax_action`, `skylight_pid`: which macOS mechanism carried
// the action. The model does not choose the route and cannot ask for another
// one, so on the model face this is a token it can only copy back. On a real
// run it was on every line — 23 of them in one scenario — and no call ever
// changed because of it. It stays on the host face, where the question "which
// route did this go out on" is the whole reason the field exists.
if (audience === 'host') {
const path = evidence.path ? safeToken(evidence.path) : undefined;
if (path) fields.push(`path=${path}`);
}
// `effect` stays on both faces. It is the one field here the model acts on:
// `suspected_noop` means a retry of the same thing is a retry of nothing.
if (evidence.effect) fields.push(`effect=${evidence.effect}`);
// Which of several conditions produced the error code, when the host
// authored one. `target_changed` alone covers seven different situations —
// the window moved, the tree changed under the observation, the element left
// the window — and they call for different next moves.
//
// Host face only: what actually reached the model was `dispatch.key:none`
// and its siblings — the executor's own RPC method names, which are neither
// a condition nor a next move. The model-facing sentence for a refusal is
// the executor's `message`, which `summarize` already carries.
//
// Passed through `safeToken`, which is what keeps this from becoming the
// driver's free text: only a bounded identifier survives, never an AX label,
// a window title, or anything else that was on screen.
if (audience === 'host') {
const reason = evidence.reason ? safeToken(evidence.reason) : undefined;
if (reason) fields.push(`reason=${reason}`);
}
return fields.length > 0 ? `; dispatch ${fields.join(', ')}` : '';
}
export type ComputerSummaryAction = {
type: CuAction['type'] | CuSemanticAction['type'];
};
export function summarize(
action: ComputerSummaryAction,
result: CuRunResult,
audience: CuSummaryAudience = 'model',
): string {
const { outcome } = result;
const evidence = summarizeEvidence(outcome.evidence, audience);
if (!outcome.ok) {
// The code alone is not a recovery instruction. `unsupported_action` covers
// a key name the host could not parse, an element that does not offer the
// action, and an action this executor has no method for — three different
// next moves. The sentence beside it says which, and the model was never
// shown it.
//
// It is shown only when the backend declares its diagnostics carry no
// application text (`maka.cu/2` §1.2 makes that a protocol rule). Absent
// means withheld: a backend that says nothing is treated as one that
// cannot promise it.
const detail =
outcome.messageIsAppTextFree === true && outcome.message ? ` — ${outcome.message}` : '';
return (
`maka_computer.${action.type} failed: ${outcome.error}${detail}${evidence}` +
(typeof outcome.completedSubSteps === 'number'
? ` (completed ${outcome.completedSubSteps} sub-steps)`
: '')
);
}
const verified = outcome.verified === undefined ? 'n/a' : String(outcome.verified);
const shot = result.screenshot
? `; screenshot ${result.screenshot.widthPx}x${result.screenshot.heightPx}`
: '';
const pointStr =
action.type === 'cursor_position' && result.resolvedScreenPoint
? `; screen_point=${result.resolvedScreenPoint.x},${result.resolvedScreenPoint.y}`
: '';
// `ok` is what the model reads first, and for a dispatch that provably
// changed nothing it is the wrong first word. The executor already says so —
// `effect: "suspected_noop"` means the action was delivered and the tree
// afterwards was the one from before — but that verdict sat inside the
// evidence clause behind the word `ok`.
//
// Measured on a real run: `cmd+p` came back `ok ... suspected_noop` seven
// times in a row, and the model sent it seven times, then switched to `key`
// and sent it twice more. It was not guessing at the schema; it was believing
// a success it had been handed. `ctrl+f2` did the same four times on another
// model.
const noop = outcome.evidence?.effect === 'suspected_noop';
const verdict = noop ? 'delivered but nothing changed' : 'ok';
// `coordinate-background`, `semantic-background`, `ax`: which tier of the
// executor took the action. There is no argument that asks for a tier, so a
// model reading this can only carry it around. `verified` is the part of the
// same clause it can act on, and that stays on both faces.
const via = audience === 'host' ? ` via ${outcome.tier}` : '';
return (
`maka_computer.${action.type} ${verdict}${via} (verified=${verified})${evidence}${pointStr}${shot}` +
(outcome.verified === false
? ' — dispatch could not be confirmed; re-screenshot before retrying'
: outcome.verified === true && outcome.evidence?.effect === 'confirmed'
? ' — effect confirmed; do not repeat this action'
: '')
);
}