feat(browser): 完成 Phase 9 拖拽与指针动作能力

补齐文件拖拽、元素拖拽与受限 pointer 原语,让 Browser MCP 能覆盖常见上传与页面内拖放交互。
并补足 pageId 路由与回归测试,保持与现有 selected-page 语义一致。

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
walker1211andClaude Opus 4.7 committed 2026-04-19 18:41:18 +08:00
1 parent 492ee5b083
commit 844b5f6c42
2 files changed
+1220 -15

No files matched your search

+492 -10
View File
@@ -59,6 +59,9 @@ const TOOL_SET_DOWNLOAD_BEHAVIOR = 'browser_set_download_behavior';
const TOOL_LIST_DOWNLOADS = 'browser_list_downloads';
const TOOL_CANCEL_DOWNLOAD = 'browser_cancel_download';
const TOOL_SET_FILE_INPUT = 'browser_set_file_input';
const TOOL_DRAG_FILES = 'browser_drag_files';
const TOOL_DRAG_ELEMENT = 'browser_drag_element';
const TOOL_POINTER_ACTION = 'browser_pointer_action';
const TOOL_TAKE_SCREENSHOT = 'browser_take_screenshot';
const TOOL_WAIT_FOR = 'browser_wait_for';
const TOOL_EVAL = 'browser_eval';
@@ -87,6 +90,9 @@ const TOOL_NAMES = [
TOOL_LIST_DOWNLOADS,
TOOL_CANCEL_DOWNLOAD,
TOOL_SET_FILE_INPUT,
TOOL_DRAG_FILES,
TOOL_DRAG_ELEMENT,
TOOL_POINTER_ACTION,
TOOL_TAKE_SCREENSHOT,
TOOL_WAIT_FOR,
TOOL_EVAL,
@@ -113,6 +119,8 @@ const CDP_TIMEOUT_MS = 5000;
const NAVIGATION_POLL_INTERVAL_MS = 100;
const DEFAULT_WAIT_TIMEOUT_MS = 2000;
const DEFAULT_WAIT_POLL_INTERVAL_MS = 100;
const DEFAULT_DRAG_STEPS = 5;
const MAX_POINTER_ACTIONS = 25;
let inputBuffer = Buffer.alloc(0);
let requestCounter = 0;
@@ -624,6 +632,83 @@ function getTools() {
additionalProperties: false,
},
},
{
name: TOOL_DRAG_FILES,
description:
'Drag one or more local files onto a matched page drop target using page-side File and DataTransfer injection.',
inputSchema: {
type: 'object',
properties: {
selector: { type: 'string' },
files: {
type: 'array',
items: { type: 'string' },
},
pageIndex: { type: 'integer', minimum: 0 },
pageId: { type: 'string' },
nth: { type: 'integer', minimum: 0 },
frameSelector: { type: 'string' },
pierceShadow: { type: 'boolean' },
},
required: ['selector', 'files'],
additionalProperties: false,
},
},
{
name: TOOL_DRAG_ELEMENT,
description:
'Drag a matched element to another matched element or explicit coordinates using browser mouse events.',
inputSchema: {
type: 'object',
properties: {
selector: { type: 'string' },
targetSelector: { type: 'string' },
targetX: { type: 'number' },
targetY: { type: 'number' },
pageIndex: { type: 'integer', minimum: 0 },
pageId: { type: 'string' },
nth: { type: 'integer', minimum: 0 },
targetNth: { type: 'integer', minimum: 0 },
frameSelector: { type: 'string' },
pierceShadow: { type: 'boolean' },
steps: { type: 'integer', minimum: 1 },
},
required: ['selector'],
additionalProperties: false,
},
},
{
name: TOOL_POINTER_ACTION,
description: 'Run a limited pointer action sequence using browser mouse events.',
inputSchema: {
type: 'object',
properties: {
pageIndex: { type: 'integer', minimum: 0 },
pageId: { type: 'string' },
actions: {
type: 'array',
items: {
type: 'object',
properties: {
type: { type: 'string', enum: ['move', 'down', 'up', 'pause'] },
selector: { type: 'string' },
nth: { type: 'integer', minimum: 0 },
frameSelector: { type: 'string' },
pierceShadow: { type: 'boolean' },
x: { type: 'number' },
y: { type: 'number' },
button: { type: 'string', enum: ['left', 'middle', 'right'] },
durationMs: { type: 'integer', minimum: 0 },
},
required: ['type'],
additionalProperties: false,
},
},
},
required: ['actions'],
additionalProperties: false,
},
},
{
name: TOOL_TAKE_SCREENSHOT,
description:
@@ -2029,6 +2114,122 @@ function formatFileInputResult({ pageIndex, selector, nth, frameSelector, pierce
return lines.join('\n');
}
function inferMimeType(filePath) {
const ext = path.extname(filePath).toLowerCase();
if (ext === '.pdf') return 'application/pdf';
if (ext === '.png') return 'image/png';
if (ext === '.jpg' || ext === '.jpeg') return 'image/jpeg';
if (ext === '.txt') return 'text/plain';
if (ext === '.json') return 'application/json';
return 'application/octet-stream';
}
function readLocalFilesForDrop(files) {
return validateLocalFiles(files).map((resolvedPath) => {
const buffer = fs.readFileSync(resolvedPath);
return {
path: resolvedPath,
name: path.basename(resolvedPath),
mimeType: inferMimeType(resolvedPath),
size: buffer.length,
base64: buffer.toString('base64'),
};
});
}
function buildDropFilesExpression(selector, nth, frameSelector, pierceShadow, files) {
return `(() => {
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
const nth = ${nth === undefined ? 'undefined' : String(nth)};
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
const filePayloads = JSON.parse(${JSON.stringify(JSON.stringify(files))});
const visitRoots = (root) => {
const roots = [root];
if (!pierceShadow) {
return roots;
}
const queue = [root];
while (queue.length > 0) {
const current = queue.shift();
const elements = Array.from(current.querySelectorAll('*'));
for (const element of elements) {
if (element.shadowRoot) {
roots.push(element.shadowRoot);
queue.push(element.shadowRoot);
}
}
}
return roots;
};
let root = document;
if (frameSelector) {
const frame = document.querySelector(frameSelector);
if (!frame) {
throw new Error('frame not found for selector: ' + frameSelector);
}
const frameDocument = frame.contentDocument;
if (!frameDocument) {
throw new Error('frame document is unavailable for selector: ' + frameSelector);
}
root = frameDocument;
}
const roots = visitRoots(root);
const matches = [];
for (const currentRoot of roots) {
matches.push(...Array.from(currentRoot.querySelectorAll(selector)));
}
const targetIndex = nth ?? 0;
const element = matches[targetIndex];
if (!element) {
throw new Error('element not found for selector: ' + selector);
}
const files = filePayloads.map((file) => {
const binary = atob(file.base64);
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) {
bytes[index] = binary.charCodeAt(index);
}
return new File([bytes], file.name, { type: file.mimeType });
});
const dataTransfer = new DataTransfer();
for (const file of files) {
dataTransfer.items.add(file);
}
element.dispatchEvent(new DragEvent('dragenter', { bubbles: true, cancelable: true, dataTransfer }));
element.dispatchEvent(new DragEvent('dragover', { bubbles: true, cancelable: true, dataTransfer }));
const accepted = element.dispatchEvent(
new DragEvent('drop', { bubbles: true, cancelable: true, dataTransfer })
);
return { accepted };
})()`;
}
function formatDragFilesResult({ pageIndex, selector, nth, frameSelector, pierceShadow, files }) {
const lines = [
`pageIndex: ${pageIndex}`,
`selector: ${selector}`,
`nth: ${nth}`,
`fileCount: ${files.length}`,
];
if (frameSelector) {
lines.push(`frameSelector: ${frameSelector}`);
}
if (pierceShadow) {
lines.push('pierceShadow: true');
}
lines.push('status: files-dropped');
return lines.join('\n');
}
function parseEventCondition(value) {
if (!value || typeof value !== 'object') {
throw new Error('event is required');
@@ -2613,9 +2814,10 @@ async function getElementDiagnostics(page, selector, nth, frameSelector = '', pi
return await getScopedDiagnostics(page, selector, nth, frameSelector, pierceShadow);
}
async function getScrolledElementState(page, selector, frameSelector = '', pierceShadow = false) {
async function getScrolledElementStateAt(page, selector, nth, frameSelector = '', pierceShadow = false) {
const expression = `(() => {
const selector = JSON.parse(${JSON.stringify(JSON.stringify(selector))});
const nth = ${nth === undefined ? 'undefined' : String(nth)};
const frameSelector = ${frameSelector ? `JSON.parse(${JSON.stringify(JSON.stringify(frameSelector))})` : 'undefined'};
const pierceShadow = ${pierceShadow ? 'true' : 'false'};
@@ -2659,7 +2861,8 @@ async function getScrolledElementState(page, selector, frameSelector = '', pierc
for (const currentRoot of roots) {
matches.push(...Array.from(currentRoot.querySelectorAll(selector)));
}
const element = matches[0];
const targetIndex = nth ?? 0;
const element = matches[targetIndex];
if (!element) {
return JSON.stringify({ exists: false });
}
@@ -2713,6 +2916,10 @@ async function getScrolledElementState(page, selector, frameSelector = '', pierc
return JSON.parse(await evaluateExpression(page, expression));
}
async function getScrolledElementState(page, selector, frameSelector = '', pierceShadow = false) {
return await getScrolledElementStateAt(page, selector, undefined, frameSelector, pierceShadow);
}
function formatQueryValue(field, value) {
if (field === 'boundingClientRect') {
return JSON.stringify(value);
@@ -2920,6 +3127,220 @@ async function handleEval(toolArgs) {
return `pageIndex: ${pageIndex}\nmode: ${mode}\nvalue: ${JSON.stringify(value)}`;
}
function getButtonMask(button) {
if (button === 'right') {
return 2;
}
if (button === 'middle') {
return 4;
}
return 1;
}
async function dispatchMousePointerEvent(page, type, x, y, button = 'left', isPressed = false) {
await sendCdpCommand(page, 'Input.dispatchMouseEvent', {
type,
x,
y,
button: type === 'mouseMoved' && !isPressed ? 'none' : button,
buttons: isPressed ? getButtonMask(button) : 0,
pointerType: 'mouse',
clickCount: 1,
});
}
function interpolatePoints(sourcePoint, targetPoint, steps) {
const points = [];
for (let index = 1; index <= steps; index += 1) {
const ratio = index / steps;
points.push({
x: Math.round(sourcePoint.x + (targetPoint.x - sourcePoint.x) * ratio),
y: Math.round(sourcePoint.y + (targetPoint.y - sourcePoint.y) * ratio),
});
}
return points;
}
async function resolveElementCenterPoint(page, selector, nth, frameSelector, pierceShadow, missingLabel) {
const state = await getScrolledElementStateAt(page, selector, nth, frameSelector, pierceShadow);
if (!state.exists) {
throw new Error(missingLabel);
}
if (state.connected !== true) {
throw new Error(`drag coordinates unavailable for selector: ${selector}`);
}
if (state.interactable !== true || !state.centerPoint) {
throw new Error(`drag coordinates unavailable for selector: ${selector}`);
}
return state.centerPoint;
}
async function handlePointerAction(toolArgs) {
const pages = await listPageTargets();
if (pages.length === 0) {
throw new Error('Browser MCP did not find any page targets in the current Chrome session.');
}
const { page, pageIndex } = resolveTargetPage(pages, toolArgs, selectedPageId);
if (!page.webSocketDebuggerUrl) {
throw new Error(`Browser MCP page ${pageIndex} does not expose a websocket debugger URL.`);
}
const actions = Array.isArray(toolArgs.actions) ? toolArgs.actions : null;
if (!actions || actions.length === 0) {
throw new Error('actions must be a non-empty array');
}
if (actions.length > MAX_POINTER_ACTIONS) {
throw new Error(`actions exceeds maximum of ${MAX_POINTER_ACTIONS}`);
}
let pointerX = null;
let pointerY = null;
let pressedButton = null;
for (const action of actions) {
if (!action || typeof action !== 'object') {
throw new Error('each action must be an object');
}
if (!['move', 'down', 'up', 'pause'].includes(action.type)) {
throw new Error(`unsupported pointer action: ${String(action.type || '')}`);
}
if (action.type === 'pause') {
const durationMs = action.durationMs === undefined ? 0 : requireNonNegativeInteger(action.durationMs, 'durationMs');
if (durationMs > 0) {
await sleep(durationMs);
}
continue;
}
if (action.type === 'move') {
let point = null;
if (typeof action.selector === 'string' && action.selector.trim() !== '') {
point = await resolveElementCenterPoint(
page,
requireNonEmptyString(action.selector, 'actions.selector'),
requireOptionalNonNegativeInteger(action.nth, 'actions.nth'),
parseOptionalNonEmptyString(action.frameSelector),
action.pierceShadow === true,
'drag coordinates unavailable'
);
} else if (typeof action.x === 'number' && typeof action.y === 'number') {
point = {
x: requireFiniteNumber(action.x, 'actions.x'),
y: requireFiniteNumber(action.y, 'actions.y'),
};
} else {
throw new Error('drag coordinates unavailable');
}
pointerX = point.x;
pointerY = point.y;
await dispatchMousePointerEvent(page, 'mouseMoved', pointerX, pointerY, pressedButton || 'left', Boolean(pressedButton));
continue;
}
if (pointerX === null || pointerY === null) {
throw new Error('pointer state error');
}
if (action.type === 'down') {
if (pressedButton) {
throw new Error('pointer state error');
}
pressedButton = requireEnumString(action.button, 'button', ['left', 'middle', 'right']) || 'left';
await dispatchMousePointerEvent(page, 'mousePressed', pointerX, pointerY, pressedButton, true);
continue;
}
if (!pressedButton) {
throw new Error('pointer state error');
}
const releaseButton = requireEnumString(action.button, 'button', ['left', 'middle', 'right']) || pressedButton;
await dispatchMousePointerEvent(page, 'mouseReleased', pointerX, pointerY, releaseButton, false);
pressedButton = null;
}
if (pressedButton) {
throw new Error('pointer state error');
}
return `pageIndex: ${pageIndex}\nactionCount: ${actions.length}\nstatus: pointer-actions-completed`;
}
async function handleDragElement(toolArgs) {
const pages = await listPageTargets();
if (pages.length === 0) {
throw new Error('Browser MCP did not find any page targets in the current Chrome session.');
}
const { page, pageIndex } = resolveTargetPage(pages, toolArgs, selectedPageId);
if (!page.webSocketDebuggerUrl) {
throw new Error(`Browser MCP page ${pageIndex} does not expose a websocket debugger URL.`);
}
const selector = requireNonEmptyString(toolArgs.selector, 'selector');
const nth = requireOptionalNonNegativeInteger(toolArgs.nth, 'nth');
const targetNth = requireOptionalNonNegativeInteger(toolArgs.targetNth, 'targetNth');
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
const pierceShadow = toolArgs.pierceShadow === true;
const targetSelector = parseOptionalNonEmptyString(toolArgs.targetSelector);
const hasTargetX = Object.prototype.hasOwnProperty.call(toolArgs, 'targetX');
const hasTargetY = Object.prototype.hasOwnProperty.call(toolArgs, 'targetY');
const hasTargetCoordinates = hasTargetX || hasTargetY;
if (targetSelector && hasTargetCoordinates) {
throw new Error('targetSelector and targetX/targetY cannot be used together');
}
if (!targetSelector && !hasTargetCoordinates) {
throw new Error('targetSelector or targetX/targetY is required');
}
if (hasTargetX !== hasTargetY) {
throw new Error('targetX and targetY must be provided together');
}
const sourcePoint = await resolveElementCenterPoint(
page,
selector,
nth,
frameSelector,
pierceShadow,
'source element not found'
);
const targetPoint = targetSelector
? await resolveElementCenterPoint(
page,
targetSelector,
targetNth,
frameSelector,
pierceShadow,
'target element not found'
)
: {
x: requireFiniteNumber(toolArgs.targetX, 'targetX'),
y: requireFiniteNumber(toolArgs.targetY, 'targetY'),
};
const steps = toolArgs.steps === undefined ? DEFAULT_DRAG_STEPS : requirePositiveInteger(toolArgs.steps, 'steps');
await dispatchMousePointerEvent(page, 'mouseMoved', sourcePoint.x, sourcePoint.y, 'left', false);
await dispatchMousePointerEvent(page, 'mousePressed', sourcePoint.x, sourcePoint.y, 'left', true);
for (const point of interpolatePoints(sourcePoint, targetPoint, steps)) {
await dispatchMousePointerEvent(page, 'mouseMoved', point.x, point.y, 'left', true);
}
await dispatchMousePointerEvent(page, 'mouseReleased', targetPoint.x, targetPoint.y, 'left', false);
return [
`pageIndex: ${pageIndex}`,
`selector: ${selector}`,
`nth: ${nth ?? 0}`,
targetSelector ? `targetSelector: ${targetSelector}` : `targetX: ${targetPoint.x}`,
targetSelector ? `targetNth: ${targetNth ?? 0}` : `targetY: ${targetPoint.y}`,
`steps: ${steps}`,
`${frameSelector ? `frameSelector: ${frameSelector}\n` : ''}${pierceShadow ? 'pierceShadow: true\n' : ''}status: dragged`,
].join('\n');
}
async function handleHover(toolArgs) {
const { page, pageIndex } = await getSelectedPage(toolArgs);
const selector = requireNonEmptyString(toolArgs.selector, 'selector');
@@ -2936,14 +3357,7 @@ async function handleHover(toolArgs) {
throw new Error(`element is hidden or not interactable for selector: ${selector}`);
}
await sendCdpCommand(page, 'Input.dispatchMouseEvent', {
type: 'mouseMoved',
x: state.centerPoint.x,
y: state.centerPoint.y,
button: 'none',
buttons: 0,
pointerType: 'mouse',
});
await dispatchMousePointerEvent(page, 'mouseMoved', state.centerPoint.x, state.centerPoint.y, 'left', false);
return `pageIndex: ${pageIndex}\nselector: ${selector}${formatScopedSelectorSuffix(frameSelector, pierceShadow)}\nstatus: hovered`;
}
@@ -3684,6 +4098,50 @@ async function handleSetFileInput(toolArgs) {
});
}
async function handleDragFiles(toolArgs) {
const pages = await listPageTargets();
if (pages.length === 0) {
throw new Error('Browser MCP did not find any page targets in the current Chrome session.');
}
const { page, pageIndex } = resolveTargetPage(pages, toolArgs, selectedPageId);
const selector = requireNonEmptyString(toolArgs.selector, 'selector');
const files = readLocalFilesForDrop(requireNonEmptyStringArray(toolArgs.files, 'files'));
const nth = toolArgs.nth === undefined ? 0 : requireNonNegativeInteger(toolArgs.nth, 'nth');
const frameSelector = parseOptionalNonEmptyString(toolArgs.frameSelector);
const pierceShadow = toolArgs.pierceShadow === true;
const response = await sendCdpCommand(page, 'Runtime.evaluate', {
expression: buildDropFilesExpression(selector, nth, frameSelector, pierceShadow, files),
returnByValue: true,
awaitPromise: true,
});
if (response?.exceptionDetails) {
throw new Error(response.exceptionDetails.text || 'drop target evaluation failed');
}
const result = response?.result || null;
if (!result) {
throw new Error('Browser MCP received an invalid DevTools evaluation response.');
}
if (result.subtype === 'error') {
throw new Error(result.description || 'drop target evaluation failed');
}
if (result.value?.accepted === false) {
throw new Error('drop target rejected files');
}
return formatDragFilesResult({
pageIndex,
selector,
nth,
frameSelector,
pierceShadow,
files,
});
}
async function handleToolCall(message) {
const id = message.id;
const params = message.params || {};
@@ -3848,6 +4306,30 @@ async function handleToolCall(message) {
return;
}
if (toolName === TOOL_DRAG_FILES) {
const text = await handleDragFiles(toolArgs);
writeResponse(id, {
content: [{ type: 'text', text }],
});
return;
}
if (toolName === TOOL_DRAG_ELEMENT) {
const text = await handleDragElement(toolArgs);
writeResponse(id, {
content: [{ type: 'text', text }],
});
return;
}
if (toolName === TOOL_POINTER_ACTION) {
const text = await handlePointerAction(toolArgs);
writeResponse(id, {
content: [{ type: 'text', text }],
});
return;
}
if (toolName === TOOL_TAKE_SCREENSHOT) {
const text = await handleScreenshot(toolArgs);
writeResponse(id, {