blob: d96b1d5d943a21c906cfa30fe352638fafac1247 [file]
import { z } from 'zod';
import { htmlToMarkdown } from '@jackwener/opencli/utils';
import type { MakaTool } from '@maka/runtime';
import {
type BrowserPageRun,
type TakeoverMode,
browserAutomationAvailable,
withBrowserPage,
} from './session.js';
import { parseNavigable } from './logic.js';
/**
* Generic observe→act browser tools over opencli's numbered-ref model:
* browser_snapshot lists interactive elements as `[N]` refs, the act tools
* (click / type) take a ref and self-verify the match. All six drive the
* conversation's OWN embedded-browser view through BrowserSession.
*
* Permission: every tool is the dedicated `browser` category — Maka's mode
* gates it (block in `explore`, prompt in `ask` AND `execute`), so a browser
* action on the user's logged-in sessions is never silently auto-allowed; the
* visible view plus the user's "allow for this turn" is the safety net. A
* finer origin-keyed grant (remember "allow this site") is a later refinement.
*/
const BROWSER_TOOL_CATEGORY = 'browser' as const;
// Above opencli's internal 30s CDP guard so a slow load surfaces the CDP
// command timeout (which names the navigation) rather than our generic one.
const NAVIGATE_TIMEOUT_MS = 35_000;
const MAX_WAIT_SECONDS = 120;
/** Per-call markdown budget; long pages page through `start`/`next_start_char`. */
const EXTRACT_CHAR_LIMIT = 16_000;
/**
* Page-side ceiling on the raw HTML read. Without it a huge DOM serializes
* fully over CDP and feeds htmlToMarkdown whole — a synchronous conversion no
* timeout can interrupt.
*/
const HTML_CHAR_LIMIT = 2_000_000;
/**
* opencli's target resolver treats ONLY a bare number as a snapshot ref —
* "[12]" falls through to querySelectorAll and fails as an invalid CSS
* selector. browser_snapshot prints refs as "[12]" and teaches that spelling,
* so models echo it. Accept the bracketed form and hand opencli the number;
* anything else passes through as a CSS selector.
*/
export function normalizeElementRef(ref: string): string {
const match = /^\s*\[(\d+)\]\s*$/.exec(ref);
return match ? match[1] : ref;
}
/** One-line note appended to the action that hardened (reloaded) a taken-over page. */
export function takeoverNote(info: { takeoverReloaded: boolean }): string {
return info.takeoverReloaded
? '\n\nNote: attached to the page that was already open; it was reloaded once to apply automation hardening.'
: '';
}
/**
* Shared run path for the browser_* tools. Permission is decided by the engine
* BEFORE impl runs (via the tool's category), so this only guards availability
* and runs the action through BrowserSession with its tool-level timeout and
* the user's stop signal.
*/
async function runBrowserAction<T>(input: {
sessionId: string;
label: string;
abortSignal: AbortSignal;
timeoutMs?: number;
// How this action treats a page the user already had open (see TakeoverMode);
// omit for the pure-observe default, which never reloads.
takeover?: TakeoverMode;
run: BrowserPageRun<T>;
}): Promise<T> {
if (!browserAutomationAvailable()) {
throw new Error('Browser automation is only available inside the desktop app.');
}
return withBrowserPage(input.sessionId, input.label, input.run, {
abort: input.abortSignal,
...(input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs } : {}),
...(input.takeover ? { takeover: input.takeover } : {}),
});
}
export function buildBrowserNavigateTool(): MakaTool<{ url: string }, string> {
return {
name: 'browser_navigate',
displayName: '浏览器导航',
description:
'Open a URL in the conversation\'s embedded browser. Pass a full http:// or https:// URL; other schemes are rejected. ' +
'Returns the URL actually landed on (after redirects) and the page title. Follow with browser_snapshot to see what is on the page.',
parameters: z.object({
url: z.string().min(1).max(4000).describe('Full http:// or https:// URL to open. Other schemes are rejected.'),
}),
categoryHint: BROWSER_TOOL_CATEGORY,
impl: async ({ url: rawUrl }, { sessionId, abortSignal }) => {
// Validate BEFORE goto: opencli's goto sends Page.navigate directly,
// bypassing the view's will-navigate guard, so the scheme check is here.
const url = parseNavigable(rawUrl);
if (!url) {
throw new Error(`Not a navigable URL: ${JSON.stringify(rawUrl)}. Pass a full http:// or https:// URL.`);
}
const result = await runBrowserAction({
sessionId,
label: 'navigate',
abortSignal,
timeoutMs: NAVIGATE_TIMEOUT_MS,
// goto re-commits the document with the stealth script, so this clears a
// pending takeover without an extra reload.
takeover: 'navigate',
run: async (page, info) => {
await page.goto(url, { waitUntil: 'load' });
// Read the document's real location: goto caches the REQUESTED url, so
// getCurrentUrl() would just echo it back and a redirect would never
// be visible.
const landed = await page.evaluate<string>('window.location.href').catch(() => url);
const title = await page.evaluate<string>('document.title').catch(() => '');
return { landed: typeof landed === 'string' && landed ? landed : url, title, info };
},
});
return (
[`Loaded ${result.landed}`, result.title ? `Title: ${result.title}` : undefined].filter(Boolean).join('\n') +
takeoverNote(result.info)
);
},
};
}
export function buildBrowserSnapshotTool(): MakaTool<Record<string, never>, string> {
return {
name: 'browser_snapshot',
displayName: '浏览器快照',
description:
'Observe the current page as a list of interactive elements (links, buttons, inputs), each tagged with a `[N]` ' +
'reference you pass to browser_click / browser_type. This is the primary way to see what is on the page before acting.',
parameters: z.object({}),
categoryHint: BROWSER_TOOL_CATEGORY,
impl: async (_args, { sessionId, abortSignal }) => {
const result = await runBrowserAction({
sessionId,
label: 'snapshot',
abortSignal,
run: async (page, info) => {
const snapshot = await page.snapshot({ interactive: true });
const url = (await page.getCurrentUrl?.()) ?? '';
return { snapshot, url, info };
},
});
const text =
typeof result.snapshot === 'string' ? result.snapshot : JSON.stringify(result.snapshot, null, 2);
return (result.url ? `${result.url}\n\n${text}` : text) + takeoverNote(result.info);
},
};
}
export function buildBrowserClickTool(): MakaTool<{ ref: string }, string> {
return {
name: 'browser_click',
displayName: '浏览器点击',
description:
'Click an element by its browser_snapshot reference (like "[12]") or a CSS selector. ' +
'Reports how many elements matched and the match confidence; re-snapshot if multiple matched.',
parameters: z.object({
ref: z.string().min(1).max(2000).describe('Element reference from browser_snapshot (like "[12]") or a CSS selector.'),
}),
categoryHint: BROWSER_TOOL_CATEGORY,
impl: async ({ ref }, { sessionId, abortSignal }) => {
const result = await runBrowserAction({
sessionId,
label: 'click',
abortSignal,
// A mutating action: harden a taken-over page (reload once) before clicking.
takeover: 'mutate',
run: async (page, info) => ({ outcome: await page.click(normalizeElementRef(ref)), info }),
});
const { matches_n, match_level } = result.outcome;
return (
`Clicked ${ref} (matched ${matches_n} element${matches_n === 1 ? '' : 's'}, ${match_level} match).` +
(matches_n > 1
? ' Multiple matches — verify the right element reacted, or re-snapshot for a tighter ref.'
: '') +
takeoverNote(result.info)
);
},
};
}
export function buildBrowserTypeTool(): MakaTool<{ ref: string; text: string; submit?: boolean }, string> {
return {
name: 'browser_type',
displayName: '浏览器输入',
description:
'Fill text into a field by its browser_snapshot reference (like "[7]") or a CSS selector; replaces the field\'s current content. ' +
'Set submit=true to press Enter after (search boxes, single-field forms). Self-verifies the field now holds the requested text.',
parameters: z.object({
ref: z.string().min(1).max(2000).describe('Element reference from browser_snapshot (like "[7]") or a CSS selector.'),
text: z.string().max(100_000).describe("Text to fill in; replaces the field's current content."),
submit: z
.boolean()
.optional()
.describe('Press Enter after filling (for search boxes and single-field forms). Default false.'),
}),
categoryHint: BROWSER_TOOL_CATEGORY,
impl: async ({ ref, text, submit }, { sessionId, abortSignal }) => {
const result = await runBrowserAction({
sessionId,
label: 'type',
abortSignal,
// A mutating action: harden a taken-over page (reload once) before typing.
takeover: 'mutate',
run: async (page, info) => {
const outcome = await page.fillText(normalizeElementRef(ref), text);
if (submit) await page.pressKey('Enter');
return { outcome, info };
},
});
const { verified, actual, match_level } = result.outcome;
const lines = [
`Filled ${ref} (${match_level} match)${submit ? ', then pressed Enter' : ''}.`,
verified
? 'Verified: the field contains the requested text.'
: `Not verified — the field now contains: ${JSON.stringify(actual)}`,
];
return lines.join('\n') + takeoverNote(result.info);
},
};
}
export function buildBrowserWaitTool(): MakaTool<
{ text?: string; selector?: string; time?: number; timeout?: number },
string
> {
return {
name: 'browser_wait',
displayName: '浏览器等待',
description:
'Wait for the page to be ready: until `text` is visible, until a CSS `selector` matches, or a fixed `time` pause in seconds. ' +
'Provide exactly one of text / selector / time. Prefer text or selector over a blind pause.',
parameters: z.object({
text: z.string().optional().describe('Wait until this text is visible on the page.'),
selector: z.string().optional().describe('Wait until this CSS selector matches an element.'),
time: z
.number()
.optional()
.describe('Fixed pause in seconds (use only when nothing observable signals readiness).'),
timeout: z
.number()
.optional()
.describe('Wait limit in seconds for text/selector waits (defaults: text 30s, selector 10s).'),
}),
categoryHint: BROWSER_TOOL_CATEGORY,
impl: async ({ text, selector, time, timeout }, { sessionId, abortSignal }) => {
const conditions = [text, selector, time].filter((v) => v !== undefined);
if (conditions.length !== 1) {
throw new Error('Provide exactly one of `text`, `selector`, or `time`.');
}
// The checks below are truthy-based; a blank string would slip past the
// count above and turn into a meaningless bare-timeout wait.
for (const [key, value] of [
['text', text],
['selector', selector],
] as const) {
if (value !== undefined && value.trim() === '') {
throw new Error(`\`${key}\` must be a non-empty string.`);
}
}
const requested = Math.min(time ?? timeout ?? (selector ? 10 : 30), MAX_WAIT_SECONDS);
if (requested <= 0) {
throw new Error('`time`/`timeout` must be a positive number of seconds.');
}
const condition = text
? `text ${JSON.stringify(text)}`
: selector
? `selector ${JSON.stringify(selector)}`
: `${requested}s pause`;
const info = await runBrowserAction({
sessionId,
label: 'wait',
abortSignal,
// The page-side wait owns the deadline; give the tool wrapper room past it.
timeoutMs: (requested + 5) * 1000,
run: async (page, pageInfo) => {
if (time !== undefined) {
await page.wait({ time: requested });
return pageInfo;
}
try {
await page.wait({
...(text ? { text } : {}),
...(selector ? { selector } : {}),
timeout: requested,
});
} catch (err) {
// The page-side waiter rejects with a raw in-page exception which
// neither says it was a timeout nor how to recover. Say both.
if (err instanceof Error && /Selector not found|Text not found/.test(err.message)) {
throw new Error(
`Waited ${requested}s but ${condition} never appeared. The page may be structured differently than expected — take a browser_snapshot to see what is actually there before retrying.`,
);
}
throw err;
}
return pageInfo;
},
});
return `Done: ${condition}.` + takeoverNote(info);
},
};
}
export function buildBrowserExtractTool(): MakaTool<{ selector?: string; start?: number }, string> {
return {
name: 'browser_extract',
displayName: '浏览器提取',
description:
'Read the page (or a CSS-selected region) as Markdown for analysis. Omit selector for the whole body. ' +
'Long pages page through `start` — the output names the next_start_char to continue from.',
parameters: z.object({
selector: z.string().optional().describe('CSS selector to extract from; omit for the whole page body.'),
start: z
.number()
.optional()
.describe("Character offset to continue from (use the previous call's next_start_char)."),
}),
categoryHint: BROWSER_TOOL_CATEGORY,
impl: async ({ selector, start: rawStart }, { sessionId, abortSignal }) => {
const start = Math.max(0, Math.floor(rawStart ?? 0));
const result = await runBrowserAction({
sessionId,
label: 'extract',
abortSignal,
run: async (page, info) => {
// The selector is JSON-serialized into the script (never string-
// concatenated), so it cannot inject.
const read = await page.evaluate<{ html: string; truncated: boolean } | null>(
readHtmlJs(JSON.stringify(selector ?? null)),
);
const url = (await page.getCurrentUrl?.()) ?? '';
return { read, url, info };
},
});
if (typeof result.read?.html !== 'string') {
throw new Error(
selector
? `No element matches selector ${JSON.stringify(selector)}.`
: 'The page has no readable body yet — navigate somewhere first.',
);
}
const markdown = htmlToMarkdown(result.read.html);
const chunk = markdown.slice(start, start + EXTRACT_CHAR_LIMIT);
const nextStart = start + chunk.length;
const hasMore = nextStart < markdown.length;
return (
(result.url ? `${result.url}\n\n` : '') +
chunk +
(hasMore
? `\n\n(Content continues — call browser_extract again with start=${nextStart}. next_start_char: ${nextStart})`
: '') +
(result.read.truncated
? "\n\n(The page's HTML was larger than the extraction ceiling; trailing content was dropped before conversion. Use `selector` to target the part you need.)"
: '') +
takeoverNote(result.info)
);
},
};
}
// Runs inside the page. The selector arrives pre-serialized via JSON (never
// string-concatenated into code), so selector content cannot inject script. A
// malformed selector (e.g. a "[12]" ref a model echoed) makes querySelector
// throw a SyntaxError; catch it and return null so the tool surfaces its
// friendly "No element matches selector" message instead of a raw DOMException.
export function readHtmlJs(selectorJson: string): string {
return `(() => {
const selector = ${selectorJson};
let el;
try {
el = selector ? document.querySelector(selector) : document.body;
} catch {
return null;
}
if (!el) return null;
const html = el.outerHTML;
return { html: html.slice(0, ${HTML_CHAR_LIMIT}), truncated: html.length > ${HTML_CHAR_LIMIT} };
})()`;
}
/** The six generic observe→act browser tools, in observe-before-act order. */
export function buildBrowserTools(): MakaTool[] {
return [
buildBrowserNavigateTool(),
buildBrowserSnapshotTool(),
buildBrowserClickTool(),
buildBrowserTypeTool(),
buildBrowserWaitTool(),
buildBrowserExtractTool(),
] as MakaTool[];
}