test(e2e): isolate E2E tests and remove destructive actions

The E2E suite has a litany of problems, including that it is fairly
complex, doesn't work very well locally and destructively modifies the
workspace, and doesn't isolate tests from each other particularly well.
The tests rely on wrapper scripts to work properly, which makes it
unnecessarily difficult to run E2E tests.

This changeset makes a few changes to the E2E test to help address these
issues The tl;dr is that tests are now self contained without any need
for wrapper scripts and are better isolated. Other notable changes
include:

- Each test now gets its own isolated Verdaccio registry; the Verdaccio
  registry storage is no longer shared between tests and is no longer
  persisted between test invocations.
- The CLI is now published to the (isolated) Verdaccio registry from a
  temporary workspace, which is a copy of the project workspace. We no
  longer publish to Verdaccio from the project workspace. This ensures
  that destructive actions that occur on publish don't alter the
  workspace, and ensures that one test can't modify the registry in a
  way that impacts another test.
- The tests no longer rely on wrapper scripts (which ran `vitest` in a
  subprocess and didn't always work locally (depending on your `PATH`).
- I also converted the tests to TypeScript while I was at it.

Theoretically, these changes mean we can run E2E tests concurrently,
though I haven't done that in this changeset. We could also now merge
the E2E vitest configuration into the primary Vitest config (though I
haven't done that here, either).

Author: ndhoule

e2e/install.e2e.js

@@ -0,0 +1,201 @@
+import http from 'node:http'
+import os from 'node:os'
+import events from 'node:events'
+import { existsSync } from 'node:fs'
+import { execSync } from 'node:child_process'
+import fs from 'node:fs/promises'
+import { platform } from 'node:os'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import execa from 'execa'
+import { runServer } from 'verdaccio'
+import { describe, expect, it } from 'vitest'
+import createDebug from 'debug'
+import picomatch from 'picomatch'
+
+import pkg from '../package.json'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const projectRoot = path.resolve(__dirname, '..')
+const distDir = path.join(projectRoot, 'dist')
+const tempdirPrefix = 'netlify-cli-e2e-test--'
+
+const debug = createDebug('netlify-cli:e2e')
+const isNodeModules = picomatch('**/node_modules/**')
+const isNotNodeModules = (target: string) => !isNodeModules(target)
+
+const itWithMockNpmRegistry = it.extend<{ registry: { address: string; cwd: string } }>({
+  registry: async (
+    // Vitest requires this argument is destructured even if no properties are used
+    // eslint-disable-next-line no-empty-pattern
+    {},
+    use,
+  ) => {
+    try {
+      if (!(await fs.stat(distDir)).isDirectory()) {
+        throw new Error(`found unexpected non-directory at "${distDir}"`)
+      }
+    } catch (err) {
+      throw new Error(
+        '"dist" directory does not exist or is not a directory. The project must be built before running E2E tests.',
+        { cause: err },
+      )
+    }
+
+    const verdaccioStorageDir = await fs.mkdtemp(path.join(os.tmpdir(), `${tempdirPrefix}verdaccio-storage`))
+    const server: http.Server = (await runServer(
+      // @ts-expect-error(ndhoule): Verdaccio's types are incorrect
+      {
+        self_path: __dirname,
+        storage: verdaccioStorageDir,
+        web: { title: 'Test Registry' },
+        max_body_size: '128mb',
+        // Disable user registration
+        max_users: -1,
+        logs: { level: 'fatal' },
+        uplinks: {
+          npmjs: {
+            url: 'https://registry.npmjs.org/',
+            maxage: '1d',
+            cache: true,
+          },
+        },
+        packages: {
+          '@*/*': {
+            access: '$all',
+            publish: 'noone',
+            proxy: 'npmjs',
+          },
+          'netlify-cli': {
+            access: '$all',
+            publish: '$all',
+          },
+          '**': {
+            access: '$all',
+            publish: 'noone',
+            proxy: 'npmjs',
+          },
+        },
+      },
+    )) as http.Server
+
+    await Promise.all([
+      Promise.race([
+        events.once(server, 'listening'),
+        events.once(server, 'error').then(() => {
+          throw new Error('Verdaccio server failed to start')
+        }),
+      ]),
+      server.listen(),
+    ])
+    const address = server.address()
+    if (address === null || typeof address === 'string') {
+      throw new Error('Failed to open Verdaccio server')
+    }
+    const registryURL = new URL(
+      `http://${
+        address.family === 'IPv6' && address.address === '::' ? 'localhost' : address.address
+      }:${address.port.toString()}`,
+    )
+
+    // The CLI publishing process modiifes the workspace, so copy it to a temporary directory. This
+    // lets us avoid contaminating the user's workspace when running these tests locally.
+    const publishWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), `${tempdirPrefix}publish-workspace`))
+    await fs.cp(projectRoot, publishWorkspace, {
+      recursive: true,
+      verbatimSymlinks: true,
+      // At this point, the project is built. As long as we limit the prepublish script to built-
+      // ins, node_modules are not be necessary to publish the package.
+      filter: isNotNodeModules,
+    })
+    await fs.writeFile(
+      path.join(publishWorkspace, '.npmrc'),
+      `//${registryURL.hostname}:${registryURL.port}/:_authToken=dummy`,
+    )
+    await execa('npm', ['publish', `--registry=${registryURL.toString()}`, '--tag=testing'], {
+      cwd: publishWorkspace,
+      stdio: debug.enabled ? 'inherit' : 'ignore',
+      env: {
+        // XXX(ndhoule): Is this necessary? Unsure, doesn't --registry cover this?
+        npm_config_registry: registryURL.toString(),
+      },
+    })
+    await fs.rm(publishWorkspace, { force: true, recursive: true })
+
+    const testWorkspace = await fs.mkdtemp(path.join(os.tmpdir(), tempdirPrefix))
+    await use({
+      address: registryURL.toString(),
+      cwd: testWorkspace,
+    })
+
+    await Promise.all([
+      events.once(server, 'close'),
+      // eslint-disable-next-line @typescript-eslint/no-confusing-void-expression
+      server.close().closeAllConnections(),
+    ])
+    await fs.rm(testWorkspace, { force: true, recursive: true })
+    await fs.rm(verdaccioStorageDir, { force: true, recursive: true })
+  },
+})
+
+const doesPackageManagerExist = (packageManager: string): boolean => {
+  try {
+    execSync(`${packageManager} --version`)
+    return true
+  } catch {
+    return false
+  }
+}
+
+const tests: [packageManager: string, config: { install: [cmd: string, args: string[]]; lockfile: string }][] = [
+  [
+    'npm',
+    {
+      install: ['npm', ['install', 'netlify-cli@testing']],
+      lockfile: 'package-lock.json',
+    },
+  ],
+  [
+    'pnpm',
+    {
+      install: ['pnpm', ['add', 'netlify-cli@testing']],
+      lockfile: 'pnpm-lock.yaml',
+    },
+  ],
+  [
+    'yarn',
+    {
+      install: ['yarn', ['add', 'netlify-cli@testing']],
+      lockfile: 'yarn.lock',
+    },
+  ],
+]
+
+describe.each(tests)('%s → installs the cli and runs the help command without error', (packageManager, config) => {
+  itWithMockNpmRegistry.runIf(doesPackageManagerExist(packageManager))(
+    'installs the cli and runs the help command without error',
+    async ({ registry }) => {
+      const cwd = registry.cwd
+      await execa(...config.install, {
+        cwd,
+        env: { npm_config_registry: registry.address },
+        stdio: debug.enabled ? 'inherit' : 'ignore',
+      })
+
+      expect(
+        existsSync(path.join(cwd, config.lockfile)),
+        `Generated lock file ${config.lockfile} does not exist in ${cwd}`,
+      ).toBe(true)
+
+      const binary = path.resolve(path.join(cwd, `./node_modules/.bin/netlify${platform() === 'win32' ? '.cmd' : ''}`))
+      const { stdout } = await execa(binary, ['help'], { cwd })
+
+      expect(stdout.trim(), `Help command does not start with 'VERSION':\n\n${stdout}`).toMatch(/^VERSION/)
+      expect(stdout, `Help command does not include 'netlify-cli/${pkg.version}':\n\n${stdout}`).toContain(
+        `netlify-cli/${pkg.version}`,
+      )
+      expect(stdout, `Help command does not include '$ netlify [COMMAND]':\n\n${stdout}`).toMatch('$ netlify [COMMAND]')
+    },
+  )
+})