feat(browser): 完成 browser MCP 第四阶段输入能力

补齐 browser_click 的精细点击参数,并新增 browser_press_key 与 browser_scroll,继续扩展 browser MCP 的真实交互能力。
同步更新单测与浏览器自动化文档,确保 Phase 4 能力完成并通过验证。

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
walker1211andClaude Opus 4.7 committed 2026-04-18 13:59:15 +08:00
1 parent cf2375e169
commit e92a2c11bf
3 files changed
+868 -25

No files matched your search

+414 -20
View File
@@ -43,6 +43,8 @@ const TOOL_DOM_SNAPSHOT = 'browser_get_dom_snapshot';
const TOOL_NAVIGATE = 'browser_navigate';
const TOOL_CLICK = 'browser_click';
const TOOL_TYPE = 'browser_type';
const TOOL_PRESS_KEY = 'browser_press_key';
const TOOL_SCROLL = 'browser_scroll';
const TOOL_TAKE_SCREENSHOT = 'browser_take_screenshot';
const TOOL_WAIT_FOR = 'browser_wait_for';
const TOOL_EVAL = 'browser_eval';
@@ -58,6 +60,8 @@ const TOOL_NAMES = [
TOOL_NAVIGATE,
TOOL_CLICK,
TOOL_TYPE,
TOOL_PRESS_KEY,
TOOL_SCROLL,
TOOL_TAKE_SCREENSHOT,
TOOL_WAIT_FOR,
TOOL_EVAL,
@@ -278,6 +282,24 @@ function getTools() {
type: 'boolean',
description: 'When true, search open shadow roots beneath the selected root.',
},
offsetX: {
type: 'number',
description: "Optional horizontal offset in CSS pixels from the target element's left edge.",
},
offsetY: {
type: 'number',
description: "Optional vertical offset in CSS pixels from the target element's top edge.",
},
button: {
type: 'string',
enum: ['left', 'middle', 'right'],
description: 'Optional mouse button. Defaults to left.',
},
clickCount: {
type: 'integer',
minimum: 1,
description: 'Optional click count. Defaults to 1.',
},
},
required: ['selector'],
additionalProperties: false,
@@ -312,6 +334,82 @@ function getTools() {
additionalProperties: false,
},
},
{
name: TOOL_PRESS_KEY,
description:
'Press a key or key combination in the selected page using real keyboard-style events. Optionally choose a page by index.',
inputSchema: {
type: 'object',
properties: {
pageIndex: {
type: 'integer',
minimum: 0,
description: 'Optional zero-based page index from browser_get_session_info. Defaults to the first page.',
},
key: {
type: 'string',
description: 'Required primary key to press.',
},
modifiers: {
type: 'array',
items: {
type: 'string',
enum: ['Alt', 'Control', 'Meta', 'Shift'],
},
description: 'Optional modifier keys such as Alt, Control, Meta, or Shift.',
},
repeat: {
type: 'integer',
minimum: 1,
description: 'Optional repeat count. Defaults to 1.',
},
},
required: ['key'],
additionalProperties: false,
},
},
{
name: TOOL_SCROLL,
description:
'Scroll the selected page or a matched element. Supports explicit deltas or scrolling an element into view.',
inputSchema: {
type: 'object',
properties: {
pageIndex: {
type: 'integer',
minimum: 0,
description: 'Optional zero-based page index from browser_get_session_info. Defaults to the first page.',
},
selector: {
type: 'string',
description: 'Optional CSS selector for an element-scoped scroll target.',
},
frameSelector: {
type: 'string',
description: 'Optional CSS selector for an iframe whose document should be used as the query root.',
},
pierceShadow: {
type: 'boolean',
description: 'When true, search open shadow roots beneath the selected root.',
},
behavior: {
type: 'string',
enum: ['into-view', 'by-offset'],
description: 'Required scroll behavior.',
},
deltaX: {
type: 'number',
description: 'Optional horizontal scroll delta for by-offset behavior.',
},
deltaY: {
type: 'number',
description: 'Optional vertical scroll delta for by-offset behavior.',
},
},
required: ['behavior'],
additionalProperties: false,
},
},
{
name: TOOL_TAKE_SCREENSHOT,
description:
@@ -795,6 +893,49 @@ function requirePositiveIntegerOrDefault(value, label, fallback) {
return value;
}
function requirePositiveInteger(value, label) {
if (!Number.isInteger(value) || value <= 0) {
throw new Error(`${label} must be a positive integer`);
}
return value;
}
function requireFiniteNumber(value, label) {
if (typeof value !== 'number' || !Number.isFinite(value)) {
throw new Error(`${label} must be a finite number`);
}
return value;
}
function requireEnumString(value, label, allowedValues) {
if (value === undefined) {
return undefined;
}
const normalized = requireNonEmptyString(value, label);
if (!allowedValues.includes(normalized)) {
throw new Error(`${label} must be one of: ${allowedValues.join(', ')}`);
}
return normalized;
}
function requireOptionalStringArray(value, label, allowedValues) {
if (value === undefined) {
return [];
}
if (!Array.isArray(value) || value.some((item) => typeof item !== 'string' || item.trim() === '')) {
throw new Error(`${label} must be an array of non-empty strings`);
}
const normalized = value.map((item) => item.trim());
if (Array.isArray(allowedValues) && allowedValues.length > 0) {
for (const item of normalized) {
if (!allowedValues.includes(item)) {
throw new Error(`${label} must only contain: ${allowedValues.join(', ')}`);
}
}
}
return normalized;
}
function getBrowserEvalMode() {
const raw = String(process.env.CCS_BROWSER_EVAL_MODE || 'readonly').trim();
if (raw === 'disabled' || raw === 'readonly' || raw === 'readwrite') {
@@ -1058,12 +1199,20 @@ async function handleClick(toolArgs) {
const targetIndex = nth ?? 0;
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
const pierceShadow = toolArgs.pierceShadow === true;
const offsetX = toolArgs.offsetX === undefined ? undefined : requireFiniteNumber(toolArgs.offsetX, 'offsetX');
const offsetY = toolArgs.offsetY === undefined ? undefined : requireFiniteNumber(toolArgs.offsetY, 'offsetY');
const button = requireEnumString(toolArgs.button, 'button', ['left', 'middle', 'right']) || 'left';
const clickCount = toolArgs.clickCount === undefined ? 1 : requirePositiveInteger(toolArgs.clickCount, 'clickCount');
const expression = `(() => {
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
const nth = ${nth === undefined ? 'undefined' : String(nth)};
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
const offsetX = ${offsetX === undefined ? 'undefined' : String(offsetX)};
const offsetY = ${offsetY === undefined ? 'undefined' : String(offsetY)};
const button = JSON.parse(${JSON.stringify(JSON.stringify(button))});
const clickCount = ${clickCount};
const visitRoots = (root) => {
const roots = [root];
@@ -1113,52 +1262,104 @@ async function handleClick(toolArgs) {
throw new Error('element is disabled for selector: ' + selector);
}
const style = window.getComputedStyle(element);
const rect = element.getBoundingClientRect();
const initialRect = element.getBoundingClientRect();
if (
style.display === 'none' ||
style.visibility === 'hidden' ||
rect.width <= 0 ||
rect.height <= 0
initialRect.width <= 0 ||
initialRect.height <= 0
) {
throw new Error('element is hidden or not interactable for selector: ' + selector);
}
element.scrollIntoView({ block: 'center', inline: 'center' });
const rect = element.getBoundingClientRect();
const dispatchMouseEvent = (type, init) => {
const resolvedOffsetX = offsetX === undefined ? rect.width / 2 : offsetX;
const resolvedOffsetY = offsetY === undefined ? rect.height / 2 : offsetY;
const clientX = rect.left + resolvedOffsetX;
const clientY = rect.top + resolvedOffsetY;
const buttonCode = button === 'middle' ? 1 : button === 'right' ? 2 : 0;
const buttonsMask = buttonCode === 1 ? 4 : buttonCode === 2 ? 2 : 1;
const dispatchMouseEvent = (type, detail, init) => {
const event = new MouseEvent(type, {
bubbles: true,
cancelable: true,
composed: true,
view: window,
detail: 1,
detail,
clientX,
clientY,
button: buttonCode,
buttons: type === 'mousedown' ? buttonsMask : 0,
...init,
});
return element.dispatchEvent(event);
};
try {
const dispatchResult = {
shouldActivate:
dispatchMouseEvent('mousedown', { button: 0, buttons: 1 }) &&
dispatchMouseEvent('mouseup', { button: 0, buttons: 0 }),
};
if (!dispatchResult.shouldActivate) {
return 'ok';
for (let index = 1; index <= clickCount; index += 1) {
const dispatchResult = {
shouldActivate:
dispatchMouseEvent('mousedown', index, {}) &&
dispatchMouseEvent('mouseup', index, {}),
};
if (!dispatchResult.shouldActivate) {
return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount });
}
if (!element.isConnected) {
return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount });
}
if (button === 'left') {
element.click();
} else if (button === 'right') {
element.dispatchEvent(new MouseEvent('contextmenu', {
bubbles: true,
cancelable: true,
composed: true,
view: window,
detail: index,
clientX,
clientY,
button: 2,
buttons: 0,
}));
} else {
element.dispatchEvent(new MouseEvent('auxclick', {
bubbles: true,
cancelable: true,
composed: true,
view: window,
detail: index,
clientX,
clientY,
button: 1,
buttons: 0,
}));
}
}
if (!element.isConnected) {
return 'ok';
if (button === 'left' && clickCount === 2) {
element.dispatchEvent(new MouseEvent('dblclick', {
bubbles: true,
cancelable: true,
composed: true,
view: window,
detail: 2,
clientX,
clientY,
button: 0,
buttons: 0,
}));
}
} catch (mouseError) {
// Fall through to the native activation path below.
}
element.click();
return 'ok';
return JSON.stringify({ resolvedOffsetX, resolvedOffsetY, button, clickCount });
})()`;
await evaluateExpression(page, expression);
return `pageIndex: ${pageIndex}\nselector: ${selector}\nnth: ${targetIndex}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: clicked`;
const raw = await evaluateExpression(page, expression);
const parsed = JSON.parse(raw);
return `pageIndex: ${pageIndex}\nselector: ${selector}\nnth: ${targetIndex}\noffsetX: ${parsed.resolvedOffsetX}\noffsetY: ${parsed.resolvedOffsetY}\nbutton: ${parsed.button}\nclickCount: ${parsed.clickCount}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: clicked`;
}
async function handleType(toolArgs) {
@@ -1240,6 +1441,183 @@ async function handleType(toolArgs) {
return `pageIndex: ${pageIndex}\nselector: ${selector}\ntypedLength: ${typedLength}\nstatus: typed`;
}
async function handlePressKey(toolArgs) {
const { page, pageIndex } = await getSelectedPage(toolArgs);
const key = requireNonEmptyString(toolArgs.key, 'key');
const modifiers = requireOptionalStringArray(toolArgs.modifiers, 'modifiers', [
'Alt',
'Control',
'Meta',
'Shift',
]);
const repeat = toolArgs.repeat === undefined ? 1 : requirePositiveInteger(toolArgs.repeat, 'repeat');
const modifierMask =
(modifiers.includes('Alt') ? 1 : 0) |
(modifiers.includes('Control') ? 2 : 0) |
(modifiers.includes('Meta') ? 4 : 0) |
(modifiers.includes('Shift') ? 8 : 0);
const specialKeyMap = {
Enter: { code: 'Enter', keyCode: 13, text: '\r' },
Tab: { code: 'Tab', keyCode: 9, text: '' },
Escape: { code: 'Escape', keyCode: 27, text: '' },
ArrowUp: { code: 'ArrowUp', keyCode: 38, text: '' },
ArrowDown: { code: 'ArrowDown', keyCode: 40, text: '' },
ArrowLeft: { code: 'ArrowLeft', keyCode: 37, text: '' },
ArrowRight: { code: 'ArrowRight', keyCode: 39, text: '' },
Backspace: { code: 'Backspace', keyCode: 8, text: '' },
Delete: { code: 'Delete', keyCode: 46, text: '' },
Space: { code: 'Space', keyCode: 32, text: ' ' },
};
const keyDescriptor =
key.length === 1
? {
code: `Key${key.toUpperCase()}`,
keyCode: key.toUpperCase().charCodeAt(0),
text: key,
}
: specialKeyMap[key];
if (!keyDescriptor) {
throw new Error(`unsupported key: ${key}`);
}
const normalizedKey = key;
const normalizedText = keyDescriptor.text;
const code = keyDescriptor.code;
const keyCode = keyDescriptor.keyCode;
for (let index = 0; index < repeat; index += 1) {
await sendCdpCommand(page, 'Input.dispatchKeyEvent', {
type: 'keyDown',
key: normalizedKey,
code,
text: normalizedText,
unmodifiedText: normalizedText,
windowsVirtualKeyCode: keyCode,
nativeVirtualKeyCode: keyCode,
modifiers: modifierMask,
autoRepeat: index > 0,
});
await sendCdpCommand(page, 'Input.dispatchKeyEvent', {
type: 'keyUp',
key: normalizedKey,
code,
windowsVirtualKeyCode: keyCode,
nativeVirtualKeyCode: keyCode,
modifiers: modifierMask,
});
}
const modifierText = modifiers.length > 0 ? modifiers.join(',') : 'none';
return `pageIndex: ${pageIndex}\nkey: ${key}\nmodifiers: ${modifierText}\nrepeat: ${repeat}\nstatus: key-pressed`;
}
async function handleScroll(toolArgs) {
const { page, pageIndex } = await getSelectedPage(toolArgs);
const selector = parseOptionalNonEmptyString(toolArgs.selector);
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
const pierceShadow = toolArgs.pierceShadow === true;
const behavior = requireEnumString(toolArgs.behavior, 'behavior', ['into-view', 'by-offset']);
const deltaX = toolArgs.deltaX === undefined ? 0 : requireFiniteNumber(toolArgs.deltaX, 'deltaX');
const deltaY = toolArgs.deltaY === undefined ? 0 : requireFiniteNumber(toolArgs.deltaY, 'deltaY');
const expression = `(() => {
const selector = ${selector ? `JSON.parse(${JSON.stringify(JSON.stringify(selector))})` : 'undefined'};
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
const behavior = JSON.parse(${JSON.stringify(JSON.stringify(behavior))});
const deltaX = ${deltaX};
const deltaY = ${deltaY};
const visitRoots = (root) => {
const roots = [root];
if (!pierceShadow) {
return roots;
}
const queue = [root];
while (queue.length > 0) {
const current = queue.shift();
const elements = Array.from(current.querySelectorAll('*'));
for (const element of elements) {
if (element.shadowRoot) {
roots.push(element.shadowRoot);
queue.push(element.shadowRoot);
}
}
}
return roots;
};
let root = document;
let scrollWindow = window;
if (frameSelector) {
const frame = document.querySelector(frameSelector);
if (!frame) {
throw new Error('frame not found for selector: ' + frameSelector);
}
const frameDocument = frame.contentDocument;
if (!frameDocument) {
throw new Error('frame document is unavailable for selector: ' + frameSelector);
}
if (!frame.contentWindow) {
throw new Error('frame window is unavailable for selector: ' + frameSelector);
}
root = frameDocument;
scrollWindow = frame.contentWindow;
}
if (!selector) {
if (behavior !== 'by-offset') {
throw new Error('selector is required for behavior: ' + behavior);
}
scrollWindow.scrollBy(deltaX, deltaY);
return JSON.stringify({ scope: 'page', behavior, deltaX, deltaY });
}
const roots = visitRoots(root);
const matches = [];
for (const currentRoot of roots) {
matches.push(...Array.from(currentRoot.querySelectorAll(selector)));
}
const element = matches[0];
if (!element) {
throw new Error('element not found for selector: ' + selector);
}
if (behavior === 'into-view') {
element.scrollIntoView({ block: 'center', inline: 'center' });
return JSON.stringify({ scope: 'element', selector, behavior });
}
if (typeof element.scrollBy === 'function') {
element.scrollBy(deltaX, deltaY);
return JSON.stringify({ scope: 'element', selector, behavior, deltaX, deltaY });
}
throw new Error('element does not support scrollBy for selector: ' + selector);
})()`;
const raw = await evaluateExpression(page, expression);
const parsed = JSON.parse(raw);
const lines = [`pageIndex: ${pageIndex}`];
if (parsed.selector) {
lines.push(`selector: ${parsed.selector}`);
}
lines.push(`behavior: ${parsed.behavior}`);
if (typeof parsed.deltaX === 'number') {
lines.push(`deltaX: ${parsed.deltaX}`);
}
if (typeof parsed.deltaY === 'number') {
lines.push(`deltaY: ${parsed.deltaY}`);
}
if (parsed.scope === 'element') {
const scopedSuffix = formatScopedSelectorSuffix(frameSelector, pierceShadow);
if (scopedSuffix) {
lines.push(scopedSuffix.slice(1));
}
}
lines.push('status: scrolled');
return lines.join('\n');
}
async function handleScreenshot(toolArgs) {
const { page, pageIndex } = await getSelectedPage(toolArgs);
const fullPage = toolArgs.fullPage === true;
@@ -1865,6 +2243,22 @@ async function handleToolCall(message) {
return;
}
if (toolName === TOOL_PRESS_KEY) {
const text = await handlePressKey(toolArgs);
writeResponse(id, {
content: [{ type: 'text', text }],
});
return;
}
if (toolName === TOOL_SCROLL) {
const text = await handleScroll(toolArgs);
writeResponse(id, {
content: [{ type: 'text', text }],
});
return;
}
if (toolName === TOOL_TAKE_SCREENSHOT) {
const text = await handleScreenshot(toolArgs);
writeResponse(id, {