Automatic perf stats in baselines (#57730)

Author: RyanCavanaugh

src/harness/harnessIO.ts

@@ -776,12 +776,57 @@ export namespace Compiler {
 
         function generateBaseLine(isSymbolBaseline: boolean, skipBaseline?: boolean): string | null {
             let result = "";
+            const perfLines: string[] = [];
+            const prePerformanceValues = getPerformanceBaselineValues();
             const gen = iterateBaseLine(isSymbolBaseline, skipBaseline);
             for (const value of gen) {
                 const [, content] = value;
                 result += content;
             }
-            return result ? (`//// [${header}] ////\r\n\r\n` + result) : null; // eslint-disable-line no-null/no-null
+            const postPerformanceValues = getPerformanceBaselineValues();
+
+            if (!isSymbolBaseline) {
+                const perfStats: [name: string, reportThreshold: number, rounding: number, beforeValue: number, afterValue: number][] = [];
+                perfStats.push(["Strict subtype cache", 1000, 100, prePerformanceValues.strictSubtype, postPerformanceValues.strictSubtype]);
+                perfStats.push(["Subtype cache", 1000, 100, prePerformanceValues.subtype, postPerformanceValues.subtype]);
+                perfStats.push(["Identity cache", 1000, 100, prePerformanceValues.identity, postPerformanceValues.identity]);
+                perfStats.push(["Assignability cache", 1000, 100, prePerformanceValues.assignability, postPerformanceValues.assignability]);
+                perfStats.push(["Type Count", 1000, 100, prePerformanceValues.typeCount, postPerformanceValues.typeCount]);
+                perfStats.push(["Instantiation count", 1500, 500, prePerformanceValues.instantiation, postPerformanceValues.instantiation]);
+                perfStats.push(["Symbol count", 45000, 500, prePerformanceValues.symbol, postPerformanceValues.symbol]);
+
+                if (perfStats.some(([, threshold, , , postValue]) => postValue >= threshold)) {
+                    perfLines.push(`=== Performance Stats ===`);
+                    for (const [name, _, rounding, preValue, postValue] of perfStats) {
+                        const preDisplay = valueToString(preValue, rounding);
+                        if (preDisplay !== "0") {
+                            perfLines.push(`${name}: ${preDisplay} / ${valueToString(postValue, rounding)} (nearest ${rounding})`);
+                        }
+                    }
+                    perfLines.push("");
+                    perfLines.push("");
+                }
+            }
+
+            return result ? (`//// [${header}] ////\r\n\r\n${perfLines.join("\n")}${result}`) : null; // eslint-disable-line no-null/no-null
+        }
+
+        function valueToString(value: number, rounding: number) {
+            return (Math.round(value / rounding) * rounding).toLocaleString("en-US");
+        }
+
+        function getPerformanceBaselineValues() {
+            const checker = program.getTypeChecker();
+            const caches = checker.getRelationCacheSizes();
+            return {
+                strictSubtype: caches.strictSubtype,
+                subtype: caches.subtype,
+                identity: caches.identity,
+                assignability: caches.assignable,
+                typeCount: checker.getTypeCount(),
+                instantiation: checker.getInstantiationCount(),
+                symbol: checker.getSymbolCount(),
+            };
         }
 
         function* iterateBaseLine(isSymbolBaseline: boolean, skipBaseline?: boolean): IterableIterator<[string, string]> {

tests/baselines/reference/APILibCheck.types

@@ -1,5 +1,12 @@
 //// [tests/cases/compiler/APILibCheck.ts] ////
 
+=== Performance Stats ===
+Identity cache: 100 / 100 (nearest 100)
+Assignability cache: 9,300 / 9,300 (nearest 100)
+Type Count: 22,500 / 22,500 (nearest 100)
+Instantiation count: 7,000 / 7,000 (nearest 500)
+Symbol count: 58,500 / 58,500 (nearest 500)
+
 === index.ts ===
 import ts = require("typescript");
 >ts : typeof ts

tests/baselines/reference/awaitedType.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/awaitedType.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 200 / 200 (nearest 100)
+Type Count: 900 / 900 (nearest 100)
+Instantiation count: 26,000 / 26,000 (nearest 500)
+Symbol count: 31,500 / 31,500 (nearest 500)
+
 === awaitedType.ts ===
 type T1 = Awaited<number>;
 >T1 : number

tests/baselines/reference/awaitedTypeStrictNull.types

@@ -1,5 +1,12 @@
 //// [tests/cases/compiler/awaitedTypeStrictNull.ts] ////
 
+=== Performance Stats ===
+Identity cache: 100 / 100 (nearest 100)
+Assignability cache: 200 / 200 (nearest 100)
+Type Count: 700 / 700 (nearest 100)
+Instantiation count: 25,500 / 25,500 (nearest 500)
+Symbol count: 30,500 / 30,500 (nearest 500)
+
 === awaitedTypeStrictNull.ts ===
 type T1 = Awaited<number>;
 >T1 : number

tests/baselines/reference/callsOnComplexSignatures.types

@@ -1,5 +1,12 @@
 //// [tests/cases/compiler/callsOnComplexSignatures.tsx] ////
 
+=== Performance Stats ===
+Subtype cache: 100 / 100 (nearest 100)
+Assignability cache: 2,400 / 2,400 (nearest 100)
+Type Count: 8,200 / 8,200 (nearest 100)
+Instantiation count: 92,500 / 92,500 (nearest 500)
+Symbol count: 68,000 / 68,000 (nearest 500)
+
 === callsOnComplexSignatures.tsx ===
 /// <reference path="react16.d.ts" />
 import React from "react";

tests/baselines/reference/checkJsxChildrenCanBeTupleType.types

@@ -1,5 +1,11 @@
 //// [tests/cases/conformance/jsx/checkJsxChildrenCanBeTupleType.tsx] ////
 
+=== Performance Stats ===
+Assignability cache: 2,200 / 2,200 (nearest 100)
+Type Count: 7,800 / 7,800 (nearest 100)
+Instantiation count: 90,000 / 90,000 (nearest 500)
+Symbol count: 66,500 / 66,500 (nearest 500)
+
 === checkJsxChildrenCanBeTupleType.tsx ===
 /// <reference path="react16.d.ts" />
 

tests/baselines/reference/checkJsxChildrenProperty16.types

@@ -1,5 +1,11 @@
 //// [tests/cases/conformance/jsx/checkJsxChildrenProperty16.tsx] ////
 
+=== Performance Stats ===
+Assignability cache: 2,200 / 2,200 (nearest 100)
+Type Count: 7,600 / 7,600 (nearest 100)
+Instantiation count: 89,500 / 89,500 (nearest 500)
+Symbol count: 66,000 / 66,000 (nearest 500)
+
 === checkJsxChildrenProperty16.tsx ===
 /// <reference path="react16.d.ts" />
 

tests/baselines/reference/checkJsxUnionSFXContextualTypeInferredCorrectly.types

@@ -1,5 +1,11 @@
 //// [tests/cases/conformance/jsx/checkJsxUnionSFXContextualTypeInferredCorrectly.tsx] ////
 
+=== Performance Stats ===
+Assignability cache: 2,200 / 2,200 (nearest 100)
+Type Count: 7,600 / 7,600 (nearest 100)
+Instantiation count: 89,500 / 89,500 (nearest 500)
+Symbol count: 66,500 / 66,500 (nearest 500)
+
 === checkJsxUnionSFXContextualTypeInferredCorrectly.tsx ===
 /// <reference path="react16.d.ts" />
 

tests/baselines/reference/circularBaseConstraint.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/circularBaseConstraint.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 100 / 100 (nearest 100)
+Type Count: 600 / 600 (nearest 100)
+Instantiation count: 2,000 / 2,000 (nearest 500)
+Symbol count: 25,500 / 25,500 (nearest 500)
+
 === circularBaseConstraint.ts ===
 // Repro from #54610
 

tests/baselines/reference/circularlySimplifyingConditionalTypesNoCrash.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/circularlySimplifyingConditionalTypesNoCrash.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 300 / 300 (nearest 100)
+Type Count: 600 / 600 (nearest 100)
+Instantiation count: 2,000 / 2,000 (nearest 500)
+Symbol count: 25,500 / 25,500 (nearest 500)
+
 === circularlySimplifyingConditionalTypesNoCrash.ts ===
 type Omit<T, K extends keyof T> = Pick<T, Exclude<keyof T, K>>;
 >Omit : Omit<T, K>

tests/baselines/reference/complexRecursiveCollections.types

@@ -1,5 +1,12 @@
 //// [tests/cases/compiler/complexRecursiveCollections.ts] ////
 
+=== Performance Stats ===
+Identity cache: 600 / 600 (nearest 100)
+Assignability cache: 7,600 / 7,600 (nearest 100)
+Type Count: 53,300 / 53,300 (nearest 100)
+Instantiation count: 178,000 / 178,000 (nearest 500)
+Symbol count: 134,500 / 134,500 (nearest 500)
+
 === complex.ts ===
 interface Ara<T> { t: T }
 >t : T

tests/baselines/reference/conditionalTypeDiscriminatingLargeUnionRegularTypeFetchingSpeedReasonable.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/conditionalTypeDoesntSpinForever.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 600 / 600 (nearest 100)
+Type Count: 1,400 / 1,600 (nearest 100)
+Instantiation count: 3,500 / 4,000 (nearest 500)
+Symbol count: 28,000 / 28,000 (nearest 500)
+
 === conditionalTypeDoesntSpinForever.ts ===
 // A *self-contained* demonstration of the problem follows...
 // Test this by running `tsc --target es6` on the command-line, rather than through another build tool such as Gulp, Webpack, etc.

tests/baselines/reference/conditionalTypeDoesntSpinForever.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/conditionalTypeVarianceBigArrayConstraintsPerformance.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 1,300 / 1,300 (nearest 100)
+Type Count: 5,900 / 5,900 (nearest 100)
+Instantiation count: 76,500 / 76,500 (nearest 500)
+Symbol count: 67,500 / 67,500 (nearest 500)
+
 === conditionalTypeVarianceBigArrayConstraintsPerformance.ts ===
 /// <reference path="react16.d.ts" />
 

tests/baselines/reference/conditionalTypeVarianceBigArrayConstraintsPerformance.types

@@ -1,5 +1,11 @@
 //// [tests/cases/conformance/types/conditional/conditionalTypes1.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 300 / 300 (nearest 100)
+Type Count: 900 / 900 (nearest 100)
+Instantiation count: 1,500 / 1,500 (nearest 500)
+Symbol count: 26,000 / 26,500 (nearest 500)
+
 === conditionalTypes1.ts ===
 type T00 = Exclude<"a" | "b" | "c" | "d", "a" | "c" | "f">;  // "b" | "d"
 >T00 : "b" | "d"

tests/baselines/reference/conditionalTypes1.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/contextuallyTypedJsxChildren.tsx] ////
 
+=== Performance Stats ===
+Assignability cache: 2,200 / 2,200 (nearest 100)
+Type Count: 7,800 / 7,800 (nearest 100)
+Instantiation count: 90,000 / 90,000 (nearest 500)
+Symbol count: 67,000 / 67,000 (nearest 500)
+
 === contextuallyTypedJsxChildren.tsx ===
 /// <reference path="react16.d.ts" />
 

tests/baselines/reference/contextuallyTypedJsxChildren.types

@@ -1,5 +1,11 @@
 //// [tests/cases/conformance/controlFlow/controlFlowOptionalChain3.tsx] ////
 
+=== Performance Stats ===
+Assignability cache: 2,100 / 2,100 (nearest 100)
+Type Count: 7,600 / 7,600 (nearest 100)
+Instantiation count: 89,500 / 89,500 (nearest 500)
+Symbol count: 66,500 / 66,500 (nearest 500)
+
 === controlFlowOptionalChain3.tsx ===
 /// <reference path="react16.d.ts" />
 

tests/baselines/reference/controlFlowOptionalChain3.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/correlatedUnions.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 500 / 500 (nearest 100)
+Type Count: 1,000 / 1,000 (nearest 100)
+Instantiation count: 1,000 / 1,000 (nearest 500)
+Symbol count: 26,500 / 26,500 (nearest 500)
+
 === correlatedUnions.ts ===
 // Various repros from #30581
 

tests/baselines/reference/correlatedUnions.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/declarationEmitRecursiveConditionalAliasPreserved.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 1,900 / 1,900 (nearest 100)
+Type Count: 14,200 / 14,200 (nearest 100)
+Instantiation count: 57,000 / 57,000 (nearest 500)
+Symbol count: 49,000 / 49,000 (nearest 500)
+
 === input.d.ts ===
 type _BuildPowersOf2LengthArrays<
 >_BuildPowersOf2LengthArrays : _BuildPowersOf2LengthArrays<Length, AccumulatedArray>

tests/baselines/reference/declarationEmitRecursiveConditionalAliasPreserved.types

@@ -1,5 +1,11 @@
 //// [tests/cases/conformance/es6/destructuring/declarationsAndAssignments.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 100 / 100 (nearest 100)
+Type Count: 500 / 1,200 (nearest 100)
+Instantiation count: 500 / 500 (nearest 500)
+Symbol count: 27,000 / 28,000 (nearest 500)
+
 === declarationsAndAssignments.ts ===
 function f0() {
 >f0 : () => void

tests/baselines/reference/declarationsAndAssignments.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/deepComparisons.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 300 / 300 (nearest 100)
+Type Count: 2,000 / 2,100 (nearest 100)
+Instantiation count: 3,500 / 3,500 (nearest 500)
+Symbol count: 26,500 / 27,000 (nearest 500)
+
 === deepComparisons.ts ===
 function f1<T, K1 extends keyof T, K2 extends keyof T[K1]>() {
 >f1 : <T, K1 extends keyof T, K2 extends keyof T[K1]>() => void

tests/baselines/reference/deepComparisons.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/deeplyNestedMappedTypes.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 500 / 600 (nearest 100)
+Type Count: 1,500 / 1,500 (nearest 100)
+Instantiation count: 15,500 / 15,500 (nearest 500)
+Symbol count: 26,000 / 26,000 (nearest 500)
+
 === deeplyNestedMappedTypes.ts ===
 // Simplified repro from #55535
 

tests/baselines/reference/deeplyNestedMappedTypes.types

@@ -1,5 +1,11 @@
 //// [tests/cases/conformance/controlFlow/dependentDestructuredVariables.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 300 / 300 (nearest 100)
+Type Count: 2,200 / 2,300 (nearest 100)
+Instantiation count: 1,500 / 1,500 (nearest 500)
+Symbol count: 34,000 / 34,000 (nearest 500)
+
 === dependentDestructuredVariables.ts ===
 type Action =
 >Action : { kind: 'A'; payload: number; } | { kind: 'B'; payload: string; }

tests/baselines/reference/dependentDestructuredVariables.types

@@ -1,5 +1,11 @@
 //// [tests/cases/conformance/types/members/duplicateNumericIndexers.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 3,600 / 3,600 (nearest 100)
+Type Count: 12,600 / 12,600 (nearest 100)
+Instantiation count: 2,500 / 2,500 (nearest 500)
+Symbol count: 29,000 / 29,000 (nearest 500)
+
 === duplicateNumericIndexers.ts ===
 // it is an error to have duplicate index signatures of the same kind in a type
 

tests/baselines/reference/duplicateNumericIndexers.types

@@ -1,5 +1,9 @@
 //// [tests/cases/compiler/enumLiteralsSubtypeReduction.ts] ////
 
+=== Performance Stats ===
+Type Count: 3,200 / 3,200 (nearest 100)
+Symbol count: 26,500 / 26,500 (nearest 500)
+
 === enumLiteralsSubtypeReduction.ts ===
 enum E {
 >E : E

tests/baselines/reference/enumLiteralsSubtypeReduction.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/errorInfoForRelatedIndexTypesNoConstraintElaboration.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 2,300 / 2,300 (nearest 100)
+Type Count: 10,100 / 10,100 (nearest 100)
+Instantiation count: 229,000 / 229,000 (nearest 500)
+Symbol count: 105,500 / 105,500 (nearest 500)
+
 === errorInfoForRelatedIndexTypesNoConstraintElaboration.ts ===
 /// <reference path="react16.d.ts" />
 

tests/baselines/reference/errorInfoForRelatedIndexTypesNoConstraintElaboration.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/excessiveStackDepthFlatArray.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 200 / 200 (nearest 100)
+Type Count: 900 / 900 (nearest 100)
+Instantiation count: 1,500 / 1,500 (nearest 500)
+Symbol count: 28,000 / 28,000 (nearest 500)
+
 === index.tsx ===
 interface MiddlewareArray<T> extends Array<T> {}
 declare function configureStore(options: { middleware: MiddlewareArray<any> }): void;

tests/baselines/reference/excessiveStackDepthFlatArray.types

@@ -1,5 +1,10 @@
 //// [tests/cases/compiler/excessivelyLargeTupleSpread.ts] ////
 
+=== Performance Stats ===
+Type Count: 33,100 / 33,100 (nearest 100)
+Instantiation count: 49,500 / 49,500 (nearest 500)
+Symbol count: 108,000 / 108,000 (nearest 500)
+
 === excessivelyLargeTupleSpread.ts ===
 // #41771
 

tests/baselines/reference/excessivelyLargeTupleSpread.types

@@ -1,5 +1,9 @@
 //// [tests/cases/conformance/fixSignatureCaching.ts] ////
 
+=== Performance Stats ===
+Type Count: 1,300 / 1,300 (nearest 100)
+Symbol count: 26,500 / 27,000 (nearest 500)
+
 === fixSignatureCaching.ts ===
 // Repro from #10697
 

tests/baselines/reference/fixSignatureCaching.types

@@ -1,5 +1,11 @@
 //// [tests/cases/compiler/flatArrayNoExcessiveStackDepth.ts] ////
 
+=== Performance Stats ===
+Assignability cache: 200 / 200 (nearest 100)
+Type Count: 1,100 / 1,100 (nearest 100)
+Instantiation count: 2,500 / 2,500 (nearest 500)
+Symbol count: 31,000 / 31,000 (nearest 500)
+
 === flatArrayNoExcessiveStackDepth.ts ===
 // Repro from #43493