User code runner draft (microsoft#19539)

* Realworld runner draft

* Baseline tsc output instead of just checking exit code

* use latest instead of major minor pin

* Add 7 more test cases + update gitignore

* Update baselines for realworld/user tests

* Rename to user

* Do not commit lockfiles

* Add code to run user tests on CRON

* Add rest of most-dependend packages to user tests

Turns out levelup doesn't have types! So I removed that one.

Author: weswigham

.gitignore

@@ -59,3 +59,11 @@ internal/
 .idea
 yarn.lock
 .parallelperf.*
+tests/cases/user/*/package-lock.json
+tests/cases/user/*/node_modules/
+tests/cases/user/*/**/*.js
+tests/cases/user/*/**/*.js.map
+tests/cases/user/*/**/*.d.ts
+!tests/cases/user/zone.js/
+!tests/cases/user/bignumber.js/
+!tests/cases/user/discord.js/

Jakefile.js

@@ -105,6 +105,7 @@ var harnessCoreSources = [
     "projectsRunner.ts",
     "loggedIO.ts",
     "rwcRunner.ts",
+    "userRunner.ts",
     "test262Runner.ts",
     "./parallel/shared.ts",
     "./parallel/host.ts",

src/harness/runner.ts

@@ -18,6 +18,7 @@
 /// <reference path="fourslashRunner.ts" />
 /// <reference path="projectsRunner.ts" />
 /// <reference path="rwcRunner.ts" />
+/// <reference path="userRunner.ts" />
 /// <reference path="harness.ts" />
 /// <reference path="./parallel/shared.ts" />
 
@@ -59,6 +60,8 @@ function createRunner(kind: TestRunnerKind): RunnerBase {
             return new RWCRunner();
         case "test262":
             return new Test262BaselineRunner();
+        case "user":
+            return new UserCodeRunner();
     }
     ts.Debug.fail(`Unknown runner kind ${kind}`);
 }
@@ -175,6 +178,9 @@ function handleTestConfig() {
                     case "test262":
                         runners.push(new Test262BaselineRunner());
                         break;
+                    case "user":
+                        runners.push(new UserCodeRunner());
+                        break;
                 }
             }
         }
@@ -196,6 +202,11 @@ function handleTestConfig() {
         runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess));
         runners.push(new FourSlashRunner(FourSlashTestType.Server));
         // runners.push(new GeneratedFourslashRunner());
+
+        // CRON-only tests
+        if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser && process.env.TRAVIS_EVENT_TYPE === "cron") {
+            runners.push(new UserCodeRunner());
+        }
     }
     if (runUnitTests === undefined) {
         runUnitTests = runners.length !== 1; // Don't run unit tests when running only one runner if unit tests were not explicitly asked for
@@ -215,8 +226,9 @@ function beginTests() {
     }
 }
 
+let isWorker: boolean;
 function startTestEnvironment() {
-    const isWorker = handleTestConfig();
+    isWorker = handleTestConfig();
     if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) {
         if (isWorker) {
             return Harness.Parallel.Worker.start();

src/harness/runnerbase.ts

@@ -1,7 +1,7 @@
 /// <reference path="harness.ts" />
 
 
-type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262";
+type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user";
 type CompilerTestKind = "conformance" | "compiler";
 type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
 

src/harness/tsconfig.json

@@ -92,6 +92,7 @@
         "projectsRunner.ts",
         "loggedIO.ts",
         "rwcRunner.ts",
+        "userRunner.ts",
         "test262Runner.ts",
         "./parallel/shared.ts",
         "./parallel/host.ts",

src/harness/userRunner.ts

@@ -0,0 +1,51 @@
+/// <reference path="harness.ts"/>
+/// <reference path="runnerbase.ts" />
+class UserCodeRunner extends RunnerBase {
+    private static readonly testDir = "tests/cases/user/";
+    public enumerateTestFiles() {
+        return Harness.IO.getDirectories(UserCodeRunner.testDir);
+    }
+
+    public kind(): TestRunnerKind {
+        return "user";
+    }
+
+    /** Setup the runner's tests so that they are ready to be executed by the harness
+     *  The first test should be a describe/it block that sets up the harness's compiler instance appropriately
+     */
+    public initializeTests(): void {
+        // Read in and evaluate the test list
+        const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
+
+        describe(`${this.kind()} code samples`, () => {
+            for (let i = 0; i < testList.length; i++) {
+                this.runTest(testList[i]);
+            }
+        });
+    }
+
+    private runTest(directoryName: string) {
+        describe(directoryName, () => {
+            const cp = require("child_process");
+            const path = require("path");
+
+            it("should build successfully", () => {
+                const cwd = path.join(__dirname, "../../", UserCodeRunner.testDir, directoryName);
+                const timeout = 600000; // 10 minutes
+                const stdio = isWorker ? "pipe" : "inherit";
+                const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio });
+                if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`);
+                Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => {
+                    const result = cp.spawnSync(`node`, ["../../../../built/local/tsc.js"], { cwd, timeout, shell: true });
+                    return `Exit Code: ${result.status}
+Standard output:
+${result.stdout.toString().replace(/\r\n/g, "\n")}
+
+
+Standard error:
+${result.stderr.toString().replace(/\r\n/g, "\n")}`;
+                });
+            });
+        });
+    }
+}

tests/baselines/reference/user/ajv.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/antd.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/axios.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/bignumber.js.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error: