User code runner draft (microsoft#19539)

* Realworld runner draft

* Baseline tsc output instead of just checking exit code

* use latest instead of major minor pin

* Add 7 more test cases + update gitignore

* Update baselines for realworld/user tests

* Rename to user

* Do not commit lockfiles

* Add code to run user tests on CRON

* Add rest of most-dependend packages to user tests

Turns out levelup doesn't have types! So I removed that one.

Author: weswigham

.gitignore

@@ -59,3 +59,11 @@ internal/
 .idea
 yarn.lock
 .parallelperf.*
+tests/cases/user/*/package-lock.json
+tests/cases/user/*/node_modules/
+tests/cases/user/*/**/*.js
+tests/cases/user/*/**/*.js.map
+tests/cases/user/*/**/*.d.ts
+!tests/cases/user/zone.js/
+!tests/cases/user/bignumber.js/
+!tests/cases/user/discord.js/

Jakefile.js

@@ -105,6 +105,7 @@ var harnessCoreSources = [
     "projectsRunner.ts",
     "loggedIO.ts",
     "rwcRunner.ts",
+    "userRunner.ts",
     "test262Runner.ts",
     "./parallel/shared.ts",
     "./parallel/host.ts",

src/harness/runner.ts

@@ -18,6 +18,7 @@
 /// <reference path="fourslashRunner.ts" />
 /// <reference path="projectsRunner.ts" />
 /// <reference path="rwcRunner.ts" />
+/// <reference path="userRunner.ts" />
 /// <reference path="harness.ts" />
 /// <reference path="./parallel/shared.ts" />
 
@@ -59,6 +60,8 @@ function createRunner(kind: TestRunnerKind): RunnerBase {
             return new RWCRunner();
         case "test262":
             return new Test262BaselineRunner();
+        case "user":
+            return new UserCodeRunner();
     }
     ts.Debug.fail(`Unknown runner kind ${kind}`);
 }
@@ -175,6 +178,9 @@ function handleTestConfig() {
                     case "test262":
                         runners.push(new Test262BaselineRunner());
                         break;
+                    case "user":
+                        runners.push(new UserCodeRunner());
+                        break;
                 }
             }
         }
@@ -196,6 +202,11 @@ function handleTestConfig() {
         runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess));
         runners.push(new FourSlashRunner(FourSlashTestType.Server));
         // runners.push(new GeneratedFourslashRunner());
+
+        // CRON-only tests
+        if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser && process.env.TRAVIS_EVENT_TYPE === "cron") {
+            runners.push(new UserCodeRunner());
+        }
     }
     if (runUnitTests === undefined) {
         runUnitTests = runners.length !== 1; // Don't run unit tests when running only one runner if unit tests were not explicitly asked for
@@ -215,8 +226,9 @@ function beginTests() {
     }
 }
 
+let isWorker: boolean;
 function startTestEnvironment() {
-    const isWorker = handleTestConfig();
+    isWorker = handleTestConfig();
     if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) {
         if (isWorker) {
             return Harness.Parallel.Worker.start();

src/harness/runnerbase.ts

@@ -1,7 +1,7 @@
 /// <reference path="harness.ts" />
 
 
-type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262";
+type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user";
 type CompilerTestKind = "conformance" | "compiler";
 type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
 

src/harness/tsconfig.json

@@ -92,6 +92,7 @@
         "projectsRunner.ts",
         "loggedIO.ts",
         "rwcRunner.ts",
+        "userRunner.ts",
         "test262Runner.ts",
         "./parallel/shared.ts",
         "./parallel/host.ts",

src/harness/userRunner.ts

@@ -0,0 +1,51 @@
+/// <reference path="harness.ts"/>
+/// <reference path="runnerbase.ts" />
+class UserCodeRunner extends RunnerBase {
+    private static readonly testDir = "tests/cases/user/";
+    public enumerateTestFiles() {
+        return Harness.IO.getDirectories(UserCodeRunner.testDir);
+    }
+
+    public kind(): TestRunnerKind {
+        return "user";
+    }
+
+    /** Setup the runner's tests so that they are ready to be executed by the harness
+     *  The first test should be a describe/it block that sets up the harness's compiler instance appropriately
+     */
+    public initializeTests(): void {
+        // Read in and evaluate the test list
+        const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
+
+        describe(`${this.kind()} code samples`, () => {
+            for (let i = 0; i < testList.length; i++) {
+                this.runTest(testList[i]);
+            }
+        });
+    }
+
+    private runTest(directoryName: string) {
+        describe(directoryName, () => {
+            const cp = require("child_process");
+            const path = require("path");
+
+            it("should build successfully", () => {
+                const cwd = path.join(__dirname, "../../", UserCodeRunner.testDir, directoryName);
+                const timeout = 600000; // 10 minutes
+                const stdio = isWorker ? "pipe" : "inherit";
+                const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio });
+                if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`);
+                Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => {
+                    const result = cp.spawnSync(`node`, ["../../../../built/local/tsc.js"], { cwd, timeout, shell: true });
+                    return `Exit Code: ${result.status}
+Standard output:
+${result.stdout.toString().replace(/\r\n/g, "\n")}
+
+
+Standard error:
+${result.stderr.toString().replace(/\r\n/g, "\n")}`;
+                });
+            });
+        });
+    }
+}

tests/baselines/reference/user/ajv.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/antd.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/axios.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/bignumber.js.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/discord.js.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/electron.log

@@ -0,0 +1,13 @@
+Exit Code: 2
+Standard output:
+node_modules/electron/electron.d.ts(5390,13): error TS2430: Interface 'WebviewTag' incorrectly extends interface 'HTMLElement'.
+  Types of property 'addEventListener' are incompatible.
+    Type '{ (event: "load-commit", listener: (event: LoadCommitEvent) => void, useCapture?: boolean | undef...' is not assignable to type '{ <K extends "error" | "waiting" | "progress" | "ended" | "change" | "input" | "select" | "abort"...'.
+      Types of parameters 'listener' and 'listener' are incompatible.
+        Type 'EventListenerOrEventListenerObject' is not assignable to type '(event: LoadCommitEvent) => void'.
+          Type 'EventListenerObject' is not assignable to type '(event: LoadCommitEvent) => void'.
+            Type 'EventListenerObject' provides no match for the signature '(event: LoadCommitEvent): void'.
+
+
+
+Standard error:

tests/baselines/reference/user/eventemitter2.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/eventemitter3.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/firebase.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/github.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/immutable.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/isobject.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/jimp.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/jsonschema.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/keycode.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/leveldown.log

@@ -0,0 +1,20 @@
+Exit Code: 2
+Standard output:
+node_modules/abstract-leveldown/index.d.ts(2,3): error TS7010: 'open', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/abstract-leveldown/index.d.ts(3,3): error TS7010: 'open', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/abstract-leveldown/index.d.ts(5,3): error TS7010: 'close', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/abstract-leveldown/index.d.ts(7,3): error TS7010: 'get', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/abstract-leveldown/index.d.ts(7,26): error TS7006: Parameter 'err' implicitly has an 'any' type.
+node_modules/abstract-leveldown/index.d.ts(8,3): error TS7010: 'get', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/abstract-leveldown/index.d.ts(8,39): error TS7006: Parameter 'err' implicitly has an 'any' type.
+node_modules/abstract-leveldown/index.d.ts(10,3): error TS7010: 'put', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/abstract-leveldown/index.d.ts(11,3): error TS7010: 'put', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/abstract-leveldown/index.d.ts(13,3): error TS7010: 'del', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/abstract-leveldown/index.d.ts(14,3): error TS7010: 'del', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/abstract-leveldown/index.d.ts(17,3): error TS7010: 'batch', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/abstract-leveldown/index.d.ts(18,3): error TS7010: 'batch', which lacks return-type annotation, implicitly has an 'any' return type.
+node_modules/leveldown/leveldown.d.ts(66,3): error TS7010: 'seek', which lacks return-type annotation, implicitly has an 'any' return type.
+
+
+
+Standard error:

tests/baselines/reference/user/localforage.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/log4js.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/mobx.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/moment.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/mqtt.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/parse5.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/portfinder.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/postcss.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/protobufjs.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/redux.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/reselect.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/rxjs.log

@@ -0,0 +1,12 @@
+Exit Code: 2
+Standard output:
+node_modules/rxjs/scheduler/VirtualTimeScheduler.d.ts(22,22): error TS2415: Class 'VirtualAction<T>' incorrectly extends base class 'AsyncAction<T>'.
+  Types of property 'work' are incompatible.
+    Type '(this: VirtualAction<T>, state?: T | undefined) => void' is not assignable to type '(this: AsyncAction<T>, state?: T | undefined) => void'.
+      The 'this' types of each signature are incompatible.
+        Type 'AsyncAction<T>' is not assignable to type 'VirtualAction<T>'.
+          Property 'index' is missing in type 'AsyncAction<T>'.
+
+
+
+Standard error:

tests/baselines/reference/user/should.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/sift.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/soap.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/sugar.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error:

tests/baselines/reference/user/tslint.log

@@ -0,0 +1,6 @@
+Exit Code: 0
+Standard output:
+
+
+
+Standard error: