devops: add bots for other browsers/platforms (microsoft#174)

Author: yury-s

.github/workflows/ci.yml

@@ -8,7 +8,11 @@ on:
 
 jobs:
   build-and-test:
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v4
@@ -25,6 +29,10 @@ jobs:
     - name: Playwright install
       run: npx playwright install --with-deps
 
+    - name: Install MS Edge
+      if: ${{ matrix.os == 'macos-latest' }} # MS Edge is not preinstalled on macOS runners.
+      run: npx playwright install msedge
+
     - name: Run linting
       run: npm run lint
 

.github/workflows/publish.yml

@@ -18,7 +18,7 @@ jobs:
       - run: npx playwright install --with-deps
       - run: npm run build
       - run: npm run lint
-      - run: npm run test
+      - run: npm run ctest
       - run: npm publish --provenance
         env:
           NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}

package.json

@@ -19,6 +19,7 @@
     "lint": "eslint .",
     "watch": "tsc --watch",
     "test": "playwright test",
+    "ctest": "playwright test --project=chrome",
     "clean": "rm -rf lib",
     "npm-publish": "npm run clean && npm run build && npm run test && npm publish"
   },

playwright.config.ts

@@ -16,12 +16,20 @@
 
 import { defineConfig } from '@playwright/test';
 
+import type { Project } from '@playwright/test';
+
 export default defineConfig({
   testDir: './tests',
   fullyParallel: true,
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
   workers: process.env.CI ? 1 : undefined,
   reporter: 'list',
-  projects: [{ name: 'default' }],
+  projects: [
+    { name: 'chrome' },
+    { name: 'msedge', use: { mcpBrowser: 'msedge' } },
+    { name: 'chromium', use: { mcpBrowser: 'chromium' } },
+    { name: 'firefox', use: { mcpBrowser: 'firefox' } },
+    { name: 'webkit', use: { mcpBrowser: 'webkit' } },
+  ].filter(Boolean) as Project[],
 });

tests/fixtures.ts

@@ -28,6 +28,10 @@ type Fixtures = {
   startClient: (options?: { args?: string[] }) => Promise<Client>;
   wsEndpoint: string;
   cdpEndpoint: string;
+
+  // Cli options.
+  mcpHeadless: boolean;
+  mcpBrowser: string | undefined;
 };
 
 export const test = baseTest.extend<Fixtures>({
@@ -40,12 +44,16 @@ export const test = baseTest.extend<Fixtures>({
     await use(await startClient({ args: ['--vision'] }));
   },
 
-  startClient: async ({ }, use, testInfo) => {
+  startClient: async ({ mcpHeadless, mcpBrowser }, use, testInfo) => {
     const userDataDir = testInfo.outputPath('user-data-dir');
     let client: StdioClientTransport | undefined;
 
     use(async options => {
-      const args = ['--headless', '--user-data-dir', userDataDir];
+      const args = ['--user-data-dir', userDataDir];
+      if (mcpHeadless)
+        args.push('--headless');
+      if (mcpBrowser)
+        args.push(`--browser=${mcpBrowser}`);
       if (options?.args)
         args.push(...options.args);
       const transport = new StdioClientTransport({
@@ -89,6 +97,12 @@ export const test = baseTest.extend<Fixtures>({
     await use(`http://localhost:${port}`);
     browserProcess.kill();
   },
+
+  mcpHeadless: async ({ headless }, use) => {
+    await use(headless);
+  },
+
+  mcpBrowser: ['chromium', { option: true }],
 });
 
 type Response = Awaited<ReturnType<Client['callTool']>>;

tests/pdf.spec.ts

@@ -30,7 +30,8 @@ test('save as pdf unavailable', async ({ startClient }) => {
   })).toHaveTextContent(/Tool \"browser_pdf_save\" not found/);
 });
 
-test('save as pdf', async ({ client }) => {
+test('save as pdf', async ({ client, mcpBrowser }) => {
+  test.skip(!!mcpBrowser && !['chromium', 'chrome', 'msedge'].includes(mcpBrowser), 'Save as PDF is only supported in Chromium.');
   expect(await client.callTool({
     name: 'browser_navigate',
     arguments: {