chore: move server e2e tests to system-tests (#16354)

Co-authored-by: Brian Mann <[email protected]>
Co-authored-by: Zach Bloomquist <[email protected]>
Co-authored-by: Zach Bloomquist <[email protected]>

Author: 3 people

.eslintignore

@@ -14,6 +14,21 @@
 **/support/fixtures/projects/**/static/*
 **/support/fixtures/projects/**/*.jsx
 **/support/fixtures/projects/**/fail.js
+
+system-tests/fixtures/*
+!system-tests/projects
+system-tests/projects/**/_fixtures/*
+system-tests/projects/**/static/*
+system-tests/projects/**/*.jsx
+system-tests/projects/**/fail.js
+system-tests/lib/scaffold/plugins/index.js
+system-tests/lib/scaffold/support/index.js
+system-tests/lib/scaffold/support/commands.js
+system-tests/test/support/projects/e2e/cypress/
+system-tests/projects/e2e/cypress/integration/stdout_exit_early_failing_spec.js
+system-tests/projects/e2e/cypress/integration/typescript_syntax_error_spec.ts
+
+
 **/test/fixtures
 **/vendor
 
@@ -23,11 +38,7 @@ cli/types
 packages/example
 
 packages/extension/test/helpers/background.js
-packages/server/lib/scaffold/plugins/index.js
-packages/server/lib/scaffold/support/index.js
-packages/server/lib/scaffold/support/commands.js
-packages/server/test/support/fixtures/projects/e2e/cypress/integration/stdout_exit_early_failing_spec.js
-packages/server/test/support/fixtures/projects/e2e/cypress/integration/typescript_syntax_error_spec.ts
+integration/stdout_exit_early_failing_spec.js
 
 npm/webpack-preprocessor/cypress/tests/e2e/compile-error.js
 npm/webpack-preprocessor/examples/use-babelrc/cypress/integration/spec.js

.gitignore

@@ -40,6 +40,10 @@ packages/server/support
 packages/server/test/support/fixtures/server/imgs
 packages/server/test/support/fixtures/server/libs
 
+# from system-tests
+system-tests/.projects
+system-tests/fixtures/large-img
+
 # from npm/react
 /npm/react/bin/*
 /npm/react/cypress/videos

.vscode/terminals.json

@@ -24,12 +24,20 @@
       "command": "yarn cypress:run --project ../project"
     },
     {
-      "name": "packages/server test-e2e",
+      "name": "cypress open (CT)",
+      "focus": true,
+      "onlySingle": true,
+      "execute": true,
+      "cwd": "[cwd]/packages/server-ct",
+      "command": "yarn cypress:open"
+    },
+    {
+      "name": "system-tests test",
       "focus": true,
       "onlySingle": true,
       "execute": false,
-      "cwd": "[cwd]/packages/server",
-      "command": "yarn test-e2e [fileBasename]"
+      "cwd": "[cwd]/system-tests",
+      "command": "yarn test [fileBasename]"
     },
     {
       "name": "packages/server test-watch",

CONTRIBUTING.md

@@ -396,6 +396,7 @@ By default, top level tasks will execute for all packages. However, most scripts
 | `test-unit`        | Run unit tests                                                   |
 | `test-integration` | Run integration tests                                            |
 | `test-e2e`         | Run end-to-end tests                                             |
+| `test-system`      | Run system tests                                                 |
 | `test-watch`       | Run unit tests and rebuild/rerun on file changes                 |
 
 > Most of the time you will only want to run a task within a specific package; this can be done by providing the package name as a scope to the top level task.
@@ -428,7 +429,6 @@ Each package is responsible for building itself and testing itself and can do so
 | `test`             | Runs all tests once (this usually means running unit tests; via `yarn test-unit`)                                                                        |
 | `test-unit`        | Run all unit tests within the package; `exit 0` if N/A                                                                                                   |
 | `test-integration` | Run all integration tests within the package; `exit 0` if N/A                                                                                            |
-| `test-e2e`         | Run all e2e tests within the package; `exit 0` if N/A                                                                                                    |
 | `test-watch`       | Run all unit tests in the package in watch mode                                                                                                          |
 
 #### Debugging
@@ -486,11 +486,11 @@ This is to ensure that links do not go dead in older versions of Cypress when th
 
 For most packages there are typically unit and integration tests.
 
-Our true e2e tests are in [`packages/server`](packages/server), which test the full stack all together.
+Please refer to each packages' `README.md` which documents how to run tests. It is not feasible to try to run all of the tests together. We run our entire test fleet across over a dozen containers in CI.
 
-Additionally, we test the code by running it against various other example projects in CI. See CI badges and links at the top of this document.
+There are also a set of system tests in [`system-tests`](system-tests) which attempt to test the entire Cypress App as close to real world as possible. See  the [`README`](system-tests/README.md) for more information.
 
-Please refer to each packages' `README.md` which documents how to run tests. It is not feasible to try to run all of the tests together. We run our entire test fleet across over a dozen containers in CI.
+Additionally, we test the code by running it against various other example projects in CI. See CI badges and links at the top of this document.
 
 If you're curious how we manage all of these tests in CI check out our [`circle.yml`](circle.yml) file found in the root `cypress` directory.
 

circle.yml

@@ -112,6 +112,7 @@ commands:
                   mkdir -p /tmp/node_modules_cache
                   mv ~/cypress/node_modules /tmp/node_modules_cache/root_node_modules
                   mv ~/cypress/cli/node_modules /tmp/node_modules_cache/cli_node_modules
+                  mv ~/cypress/system-tests/node_modules /tmp/node_modules_cache/system-tests_node_modules
                   mv ~/cypress/globbed_node_modules /tmp/node_modules_cache/globbed_node_modules
 
   build-and-persist:
@@ -142,6 +143,7 @@ commands:
             if [[ -d "/tmp/node_modules_cache" ]]; then
               mv /tmp/node_modules_cache/root_node_modules ~/cypress/node_modules
               mv /tmp/node_modules_cache/cli_node_modules ~/cypress/cli/node_modules
+              mv /tmp/node_modules_cache/system-tests_node_modules ~/cypress/system-tests/node_modules
               mv /tmp/node_modules_cache/globbed_node_modules ~/cypress/globbed_node_modules
               rm -rf /tmp/node_modules_cache
             fi
@@ -194,6 +196,7 @@ commands:
                 paths:
                   - node_modules
                   - cli/node_modules
+                  - system-tests/node_modules
                   - globbed_node_modules
       - unless:
           condition: <<parameters.is-mac>>
@@ -417,16 +420,17 @@ commands:
           path: ./packages/runner-ct/cypress/videos
       - store-npm-logs
 
-  run-e2e-tests:
+  run-system-tests:
     parameters:
       browser:
         description: browser shortname to target
         type: string
     steps:
       - restore_cached_workspace
       - run:
+          name: Run system tests
           command: |
-            ALL_SPECS=`circleci tests glob "/root/cypress/packages/server/test/e2e/*spec*"`
+            ALL_SPECS=`circleci tests glob "/root/cypress/system-tests/test/*spec*"`
             SPECS=
             for file in $ALL_SPECS; do
               # filter out non_root tests, they have their own stage
@@ -438,7 +442,7 @@ commands:
             done
             SPECS=`echo $SPECS | xargs -n 1 | circleci tests split --split-by=timings`
             echo SPECS=$SPECS
-            yarn workspace @packages/server test $SPECS --browser <<parameters.browser>>
+            yarn workspace @tooling/system-tests test:ci $SPECS --browser <<parameters.browser>>
       - verify-mocha-results
       - store_test_results:
           path: /tmp/cypress
@@ -1080,37 +1084,37 @@ jobs:
           path: /tmp/artifacts
       - store-npm-logs
 
-  server-e2e-tests-chrome:
+  system-tests-chrome:
     <<: *defaults
     resource_class: medium
     parallelism: 8
     steps:
-      - run-e2e-tests:
+      - run-system-tests:
           browser: chrome
 
-  server-e2e-tests-electron:
+  system-tests-electron:
     <<: *defaults
     resource_class: medium
     parallelism: 8
     steps:
-      - run-e2e-tests:
+      - run-system-tests:
           browser: electron
 
-  server-e2e-tests-firefox:
+  system-tests-firefox:
     <<: *defaults
     resource_class: medium
     parallelism: 8
     steps:
-      - run-e2e-tests:
+      - run-system-tests:
           browser: firefox
 
-  server-e2e-tests-non-root:
+  system-tests-non-root:
     <<: *defaults
     resource_class: medium
     steps:
       - restore_cached_workspace
       - run:
-          command: yarn workspace @packages/server test ./test/e2e/non_root*spec* --browser electron
+          command: yarn workspace @tooling/system-tests test:ci "test/non_root*spec*" --browser electron
       - verify-mocha-results
       - store_test_results:
           path: /tmp/cypress
@@ -2007,16 +2011,16 @@ linux-workflow: &linux-workflow
     - server-performance-tests:
         requires:
           - build
-    - server-e2e-tests-chrome:
+    - system-tests-chrome:
         requires:
           - build
-    - server-e2e-tests-electron:
+    - system-tests-electron:
         requires:
           - build
-    - server-e2e-tests-firefox:
+    - system-tests-firefox:
         requires:
           - build
-    - server-e2e-tests-non-root:
+    - system-tests-non-root:
         executor: non-root-docker-user
         requires:
           - build
@@ -2123,10 +2127,10 @@ linux-workflow: &linux-workflow
           - driver-integration-tests-firefox
           - driver-integration-tests-chrome
           - driver-integration-tests-electron
-          - server-e2e-tests-non-root
-          - server-e2e-tests-firefox
-          - server-e2e-tests-electron
-          - server-e2e-tests-chrome
+          - system-tests-non-root
+          - system-tests-firefox
+          - system-tests-electron
+          - system-tests-chrome
           - server-performance-tests
           - server-integration-tests
           - server-unit-tests

lerna.json

@@ -3,7 +3,8 @@
   "packages": [
     "cli",
     "packages/*",
-    "npm/*"
+    "npm/*",
+    "system-tests"
   ],
   "useWorkspaces": true,
   "version": "0.0.0"

npm/webpack-preprocessor/index.ts

@@ -268,6 +268,11 @@ const preprocessor: WebpackPreprocessor = (options: PreprocessorOptions = {}): F
 
       const jsonStats = stats.toJson()
 
+      // these stats are really only useful for debugging
+      if (jsonStats.warnings.length > 0) {
+        debug(`warnings for ${outputPath} %o`, jsonStats.warnings)
+      }
+
       if (stats.hasErrors()) {
         err = new Error('Webpack Compilation Error')
 
@@ -279,17 +284,11 @@ const preprocessor: WebpackPreprocessor = (options: PreprocessorOptions = {}): F
 
         err.message += `\n${errorsToAppend}`
 
-        debug('stats had error(s)')
+        debug('stats had error(s) %o', jsonStats.errors)
 
         return rejectWithErr(err)
       }
 
-      // these stats are really only useful for debugging
-      if (jsonStats.warnings.length > 0) {
-        debug(`warnings for ${outputPath}`)
-        debug(jsonStats.warnings)
-      }
-
       debug('finished bundling', outputPath)
       if (debugStats.enabled) {
         /* eslint-disable-next-line no-console */

npm/webpack-preprocessor/test/e2e/helpers.js

@@ -83,7 +83,7 @@ exports.runTest = async (options = {}) => {
     VIDEO_COMPRESSION_THROTTLE: 120000,
 
     // don't fail our own tests running from forked PR's
-    CYPRESS_INTERNAL_E2E_TESTS: '1',
+    CYPRESS_INTERNAL_SYSTEM_TESTS: '1',
     CYPRESS_ENV: 'test',
   })
 

package.json

@@ -49,12 +49,12 @@
     "test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{electron,extension,https-proxy,launcher,net-stubbing,network,proxy,rewriter,runner,runner-shared,socket}'\"",
     "test-debug": "lerna exec yarn test-debug --ignore \"'@packages/{desktop-gui,driver,root,static,web-config}'\"",
     "pretest-e2e": "yarn ensure-deps",
-    "test-e2e": "lerna exec yarn test-e2e --ignore \"'@packages/{desktop-gui,driver,root,static,web-config}'\"",
     "test-integration": "lerna exec yarn test-integration --ignore \"'@packages/{desktop-gui,driver,root,static,web-config}'\"",
     "test-mocha": "mocha --reporter spec scripts/spec.js",
     "test-mocha-snapshot": "mocha scripts/mocha-snapshot-spec.js",
     "test-npm-package-release-script": "npx lerna exec --scope \"@cypress/*\" -- npx --no-install semantic-release --dry-run",
     "test-s3-api": "node -r ./packages/ts/register scripts/binary/s3-api-demo.ts",
+    "test-system": "yarn workspace @tooling/system-tests test",
     "test-scripts": "mocha -r packages/ts/register --reporter spec 'scripts/unit/**/*spec.js'",
     "test-scripts-watch": "yarn test-scripts --watch --watch-extensions 'ts,js'",
     "pretest-unit": "yarn ensure-deps",
@@ -224,7 +224,8 @@
     "packages": [
       "cli",
       "packages/*",
-      "npm/*"
+      "npm/*",
+      "system-tests"
     ],
     "nohoist": [
       "**/@ffmpeg-installer",

packages/runner/cypress/fixtures/errors/exceptions_spec.js

@@ -1,6 +1,6 @@
 import './setup'
 
-const outsideError = require('../../../../server/test/support/fixtures/projects/todos/throws-error')
+const outsideError = require('@tooling/system-tests/projects/todos/throws-error')
 
 describe('exception failures', () => {
   it('in spec file', () => {