Skip to content

Commit 4a19e18

Browse files
Skn0ttSkn0tt
authored
feat: respond with action and generated locator (microsoft#181)
Closes microsoft#163
1 parent 4d59e06 commit 4a19e18

File tree

10 files changed

+126
-52
lines changed

10 files changed

+126
-52
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
lib/
22
node_modules/
33
test-results/
4+
.vscode/mcp.json

‎src/context.ts

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -207,36 +207,52 @@ class Tab {
207207
await this.page.waitForLoadState('load', { timeout: 5000 }).catch(() => {});
208208
}
209209

210-
async run(callback: (tab: Tab) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
210+
async run(callback: (tab: Tab) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
211+
let actionCode: string | undefined;
211212
try {
212213
if (!options?.noClearFileChooser)
213214
this._fileChooser = undefined;
214215
if (options?.waitForCompletion)
215-
await waitForCompletion(this.page, () => callback(this));
216+
actionCode = await waitForCompletion(this.page, () => callback(this)) ?? undefined;
216217
else
217-
await callback(this);
218+
actionCode = await callback(this) ?? undefined;
218219
} finally {
219220
if (options?.captureSnapshot)
220221
this._snapshot = await PageSnapshot.create(this.page);
221222
}
222-
const tabList = this.context.tabs().length > 1 ? await this.context.listTabs() + '\n\nCurrent tab:' + '\n' : '';
223-
const snapshot = this._snapshot?.text({ status: options?.status, hasFileChooser: !!this._fileChooser }) ?? options?.status ?? '';
223+
224+
const result: string[] = [];
225+
if (options?.status)
226+
result.push(options.status, '');
227+
228+
if (this.context.tabs().length > 1)
229+
result.push(await this.context.listTabs(), '');
230+
231+
if (actionCode)
232+
result.push('- Action: ' + actionCode, '');
233+
234+
if (this._snapshot) {
235+
if (this.context.tabs().length > 1)
236+
result.push('Current tab:');
237+
result.push(this._snapshot.text({ hasFileChooser: !!this._fileChooser }));
238+
}
239+
224240
return {
225241
content: [{
226242
type: 'text',
227-
text: tabList + snapshot,
243+
text: result.join('\n'),
228244
}],
229245
};
230246
}
231247

232-
async runAndWait(callback: (tab: Tab) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
248+
async runAndWait(callback: (tab: Tab) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
233249
return await this.run(callback, {
234250
waitForCompletion: true,
235251
...options,
236252
});
237253
}
238254

239-
async runAndWaitWithSnapshot(callback: (snapshot: PageSnapshot) => Promise<void>, options?: RunOptions): Promise<ToolResult> {
255+
async runAndWaitWithSnapshot(callback: (snapshot: PageSnapshot) => Promise<void | string>, options?: RunOptions): Promise<ToolResult> {
240256
return await this.run(tab => callback(tab.lastSnapshot()), {
241257
captureSnapshot: true,
242258
waitForCompletion: true,
@@ -275,13 +291,9 @@ class PageSnapshot {
275291
return snapshot;
276292
}
277293

278-
text(options?: { status?: string, hasFileChooser?: boolean }): string {
294+
text(options: { hasFileChooser: boolean }): string {
279295
const results: string[] = [];
280-
if (options?.status) {
281-
results.push(options.status);
282-
results.push('');
283-
}
284-
if (options?.hasFileChooser) {
296+
if (options.hasFileChooser) {
285297
results.push('- There is a file chooser visible that requires browser_file_upload to be called');
286298
results.push('');
287299
}
@@ -359,3 +371,7 @@ class PageSnapshot {
359371
return frame.locator(`aria-ref=${ref}`);
360372
}
361373
}
374+
375+
export async function generateLocator(locator: playwright.Locator): Promise<string> {
376+
return (locator as any)._generateLocatorString();
377+
}

‎src/javascript.ts

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/**
2+
* Copyright (c) Microsoft Corporation.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
// adapted from:
18+
// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/utils/isomorphic/stringUtils.ts
19+
// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/server/codegen/javascript.ts
20+
21+
// NOTE: this function should not be used to escape any selectors.
22+
export function escapeWithQuotes(text: string, char: string = '\'') {
23+
const stringified = JSON.stringify(text);
24+
const escapedText = stringified.substring(1, stringified.length - 1).replace(/\\"/g, '"');
25+
if (char === '\'')
26+
return char + escapedText.replace(/[']/g, '\\\'') + char;
27+
if (char === '"')
28+
return char + escapedText.replace(/["]/g, '\\"') + char;
29+
if (char === '`')
30+
return char + escapedText.replace(/[`]/g, '`') + char;
31+
throw new Error('Invalid escape char');
32+
}
33+
34+
export function quote(text: string) {
35+
return escapeWithQuotes(text, '\'');
36+
}
37+
38+
export function formatObject(value: any, indent = ' '): string {
39+
if (typeof value === 'string')
40+
return quote(value);
41+
if (Array.isArray(value))
42+
return `[${value.map(o => formatObject(o)).join(', ')}]`;
43+
if (typeof value === 'object') {
44+
const keys = Object.keys(value).filter(key => value[key] !== undefined).sort();
45+
if (!keys.length)
46+
return '{}';
47+
const tokens: string[] = [];
48+
for (const key of keys)
49+
tokens.push(`${key}: ${formatObject(value[key])}`);
50+
return `{\n${indent}${tokens.join(`,\n${indent}`)}\n}`;
51+
}
52+
return String(value);
53+
}

‎src/tools/keyboard.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ const pressKey: ToolFactory = captureSnapshot => ({
3434
const validatedParams = pressKeySchema.parse(params);
3535
return await context.currentTab().runAndWait(async tab => {
3636
await tab.page.keyboard.press(validatedParams.key);
37+
return `await page.keyboard.press('${validatedParams.key}');`;
3738
}, {
3839
status: `Pressed key ${validatedParams.key}`,
3940
captureSnapshot,

‎src/tools/navigate.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ const navigate: ToolFactory = captureSnapshot => ({
3535
const currentTab = await context.ensureTab();
3636
return await currentTab.run(async tab => {
3737
await tab.navigate(validatedParams.url);
38+
return `await page.goto('${validatedParams.url}');`;
3839
}, {
3940
status: `Navigated to ${validatedParams.url}`,
4041
captureSnapshot,
@@ -54,6 +55,7 @@ const goBack: ToolFactory = snapshot => ({
5455
handle: async context => {
5556
return await context.currentTab().runAndWait(async tab => {
5657
await tab.page.goBack();
58+
return `await page.goBack();`;
5759
}, {
5860
status: 'Navigated back',
5961
captureSnapshot: snapshot,
@@ -73,6 +75,7 @@ const goForward: ToolFactory = snapshot => ({
7375
handle: async context => {
7476
return await context.currentTab().runAndWait(async tab => {
7577
await tab.page.goForward();
78+
return `await page.goForward();`;
7679
}, {
7780
status: 'Navigated forward',
7881
captureSnapshot: snapshot,

‎src/tools/snapshot.ts

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ import zodToJsonSchema from 'zod-to-json-schema';
1919

2020
import type * as playwright from 'playwright';
2121
import type { Tool } from './tool';
22+
import { generateLocator } from '../context';
23+
import * as javascript from '../javascript';
2224

2325
const snapshot: Tool = {
2426
capability: 'core',
@@ -51,7 +53,9 @@ const click: Tool = {
5153
const validatedParams = elementSchema.parse(params);
5254
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
5355
const locator = snapshot.refLocator(validatedParams.ref);
56+
const action = `await page.${await generateLocator(locator)}.click();`;
5457
await locator.click();
58+
return action;
5559
}, {
5660
status: `Clicked "${validatedParams.element}"`,
5761
});
@@ -78,7 +82,9 @@ const drag: Tool = {
7882
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
7983
const startLocator = snapshot.refLocator(validatedParams.startRef);
8084
const endLocator = snapshot.refLocator(validatedParams.endRef);
85+
const action = `await page.${await generateLocator(startLocator)}.dragTo(page.${await generateLocator(endLocator)});`;
8186
await startLocator.dragTo(endLocator);
87+
return action;
8288
}, {
8389
status: `Dragged "${validatedParams.startElement}" to "${validatedParams.endElement}"`,
8490
});
@@ -97,7 +103,9 @@ const hover: Tool = {
97103
const validatedParams = elementSchema.parse(params);
98104
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
99105
const locator = snapshot.refLocator(validatedParams.ref);
106+
const action = `await page.${await generateLocator(locator)}.hover();`;
100107
await locator.hover();
108+
return action;
101109
}, {
102110
status: `Hovered over "${validatedParams.element}"`,
103111
});
@@ -122,12 +130,20 @@ const type: Tool = {
122130
const validatedParams = typeSchema.parse(params);
123131
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
124132
const locator = snapshot.refLocator(validatedParams.ref);
125-
if (validatedParams.slowly)
133+
134+
let action = '';
135+
if (validatedParams.slowly) {
136+
action = `await page.${await generateLocator(locator)}.pressSequentially(${javascript.quote(validatedParams.text)});`;
126137
await locator.pressSequentially(validatedParams.text);
127-
else
138+
} else {
139+
action = `await page.${await generateLocator(locator)}.fill(${javascript.quote(validatedParams.text)});`;
128140
await locator.fill(validatedParams.text);
129-
if (validatedParams.submit)
141+
}
142+
if (validatedParams.submit) {
143+
action += `\nawait page.${await generateLocator(locator)}.press('Enter');`;
130144
await locator.press('Enter');
145+
}
146+
return action;
131147
}, {
132148
status: `Typed "${validatedParams.text}" into "${validatedParams.element}"`,
133149
});
@@ -150,7 +166,9 @@ const selectOption: Tool = {
150166
const validatedParams = selectOptionSchema.parse(params);
151167
return await context.currentTab().runAndWaitWithSnapshot(async snapshot => {
152168
const locator = snapshot.refLocator(validatedParams.ref);
169+
const action = `await page.${await generateLocator(locator)}.selectOption(${javascript.formatObject(validatedParams.values)});`;
153170
await locator.selectOption(validatedParams.values);
171+
return action;
154172
}, {
155173
status: `Selected option in "${validatedParams.element}"`,
156174
});

‎tests/basic.spec.ts

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ test('browser_navigate', async ({ client }) => {
2626
})).toHaveTextContent(`
2727
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
2828
29+
- Action: await page.goto('data:text/html,<html><title>Title</title><body>Hello, world!</body></html>');
30+
2931
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
3032
- Page Title: Title
3133
- Page Snapshot
@@ -50,7 +52,10 @@ test('browser_click', async ({ client }) => {
5052
element: 'Submit button',
5153
ref: 's1e3',
5254
},
53-
})).toHaveTextContent(`Clicked "Submit button"
55+
})).toHaveTextContent(`
56+
Clicked "Submit button"
57+
58+
- Action: await page.getByRole('button', { name: 'Submit' }).click();
5459
5560
- Page URL: data:text/html,<html><title>Title</title><button>Submit</button></html>
5661
- Page Title: Title
@@ -77,7 +82,10 @@ test('browser_select_option', async ({ client }) => {
7782
ref: 's1e3',
7883
values: ['bar'],
7984
},
80-
})).toHaveTextContent(`Selected option in "Select"
85+
})).toHaveTextContent(`
86+
Selected option in "Select"
87+
88+
- Action: await page.getByRole('combobox').selectOption(['bar']);
8189
8290
- Page URL: data:text/html,<html><title>Title</title><select><option value="foo">Foo</option><option value="bar">Bar</option></select></html>
8391
- Page Title: Title
@@ -105,7 +113,10 @@ test('browser_select_option (multiple)', async ({ client }) => {
105113
ref: 's1e3',
106114
values: ['bar', 'baz'],
107115
},
108-
})).toHaveTextContent(`Selected option in "Select"
116+
})).toHaveTextContent(`
117+
Selected option in "Select"
118+
119+
- Action: await page.getByRole('listbox').selectOption(['bar', 'baz']);
109120
110121
- Page URL: data:text/html,<html><title>Title</title><select multiple><option value="foo">Foo</option><option value="bar">Bar</option><option value="baz">Baz</option></select></html>
111122
- Page Title: Title

‎tests/cdp.spec.ts

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,7 @@ test('cdp server', async ({ cdpEndpoint, startClient }) => {
2323
arguments: {
2424
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
2525
},
26-
})).toHaveTextContent(`
27-
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
28-
29-
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
30-
- Page Title: Title
31-
- Page Snapshot
32-
\`\`\`yaml
33-
- text: Hello, world!
34-
\`\`\`
35-
`
36-
);
26+
})).toContainTextContent(`- text: Hello, world!`);
3727
});
3828

3929
test('cdp server reuse tab', async ({ cdpEndpoint, startClient }) => {

‎tests/launch.spec.ts

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,16 +33,7 @@ test('test reopen browser', async ({ client }) => {
3333
arguments: {
3434
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
3535
},
36-
})).toHaveTextContent(`
37-
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
38-
39-
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
40-
- Page Title: Title
41-
- Page Snapshot
42-
\`\`\`yaml
43-
- text: Hello, world!
44-
\`\`\`
45-
`);
36+
})).toContainTextContent(`- text: Hello, world!`);
4637
});
4738

4839
test('executable path', async ({ startClient }) => {

‎tests/pdf.spec.ts

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,17 +36,7 @@ test('save as pdf', async ({ client }) => {
3636
arguments: {
3737
url: 'data:text/html,<html><title>Title</title><body>Hello, world!</body></html>',
3838
},
39-
})).toHaveTextContent(`
40-
Navigated to data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
41-
42-
- Page URL: data:text/html,<html><title>Title</title><body>Hello, world!</body></html>
43-
- Page Title: Title
44-
- Page Snapshot
45-
\`\`\`yaml
46-
- text: Hello, world!
47-
\`\`\`
48-
`
49-
);
39+
})).toContainTextContent(`- text: Hello, world!`);
5040

5141
const response = await client.callTool({
5242
name: 'browser_pdf_save',

0 commit comments

Comments
 (0)