Merge pull request #21004 from uniqueiniquity/updateATA

Enable typings cache entries to expire and be updated

Author: uniqueiniquity

src/compiler/types.ts

@@ -4118,16 +4118,6 @@ namespace ts {
         [option: string]: string[] | boolean | undefined;
     }
 
-    export interface DiscoverTypingsInfo {
-        fileNames: string[];                            // The file names that belong to the same project.
-        projectRootPath: string;                        // The path to the project root directory
-        safeListPath: string;                           // The path used to retrieve the safe list
-        packageNameToTypingLocation: Map<string>;       // The map of package names to their cached typing locations
-        typeAcquisition: TypeAcquisition;               // Used to customize the type acquisition process
-        compilerOptions: CompilerOptions;               // Used as a source for typing inference
-        unresolvedImports: ReadonlyArray<string>;       // List of unresolved module ids from imports
-    }
-
     export enum ModuleKind {
         None = 0,
         CommonJS = 1,

src/harness/unittests/tsserverProjectSystem.ts

@@ -63,7 +63,7 @@ namespace ts.projectSystem {
             readonly globalTypingsCacheLocation: string,
             throttleLimit: number,
             installTypingHost: server.ServerHost,
-            readonly typesRegistry = createMap<void>(),
+            readonly typesRegistry = createMap<MapLike<string>>(),
             log?: TI.Log) {
             super(installTypingHost, globalTypingsCacheLocation, safeList.path, customTypesMap.path, throttleLimit, log);
         }
@@ -126,6 +126,25 @@ namespace ts.projectSystem {
         return JSON.stringify({ dependencies });
     }
 
+    export function createTypesRegistry(...list: string[]): Map<MapLike<string>> {
+        const versionMap = {
+            "latest": "1.3.0",
+            "ts2.0": "1.0.0",
+            "ts2.1": "1.0.0",
+            "ts2.2": "1.2.0",
+            "ts2.3": "1.3.0",
+            "ts2.4": "1.3.0",
+            "ts2.5": "1.3.0",
+            "ts2.6": "1.3.0",
+            "ts2.7": "1.3.0"
+        };
+        const map = createMap<MapLike<string>>();
+        for (const l of list) {
+            map.set(l, versionMap);
+        }
+        return map;
+    }
+
     export function toExternalFile(fileName: string): protocol.ExternalFile {
         return { fileName };
     }
@@ -6682,12 +6701,18 @@ namespace ts.projectSystem {
                     },
                 })
             };
+            const typingsCachePackageLockJson: FileOrFolder = {
+                path: `${typingsCache}/package-lock.json`,
+                content: JSON.stringify({
+                    dependencies: {
+                    },
+                })
+            };
 
-            const files = [file, packageJsonInCurrentDirectory, packageJsonOfPkgcurrentdirectory, indexOfPkgcurrentdirectory, typingsCachePackageJson];
+            const files = [file, packageJsonInCurrentDirectory, packageJsonOfPkgcurrentdirectory, indexOfPkgcurrentdirectory, typingsCachePackageJson, typingsCachePackageLockJson];
             const host = createServerHost(files, { currentDirectory });
 
-            const typesRegistry = createMap<void>();
-            typesRegistry.set("pkgcurrentdirectory", void 0);
+            const typesRegistry = createTypesRegistry("pkgcurrentdirectory");
             const typingsInstaller = new TestTypingsInstaller(typingsCache, /*throttleLimit*/ 5, host, typesRegistry);
 
             const projectService = createProjectService(host, { typingsInstaller });

src/harness/unittests/typingsInstaller.ts

@@ -405,9 +405,11 @@ interface Array<T> {}`
         ensureFileOrFolder(fileOrDirectory: FileOrFolder, ignoreWatchInvokedWithTriggerAsFileCreate?: boolean) {
             if (isString(fileOrDirectory.content)) {
                 const file = this.toFile(fileOrDirectory);
-                Debug.assert(!this.fs.get(file.path));
-                const baseFolder = this.ensureFolder(getDirectoryPath(file.fullPath));
-                this.addFileOrFolderInFolder(baseFolder, file, ignoreWatchInvokedWithTriggerAsFileCreate);
+                // file may already exist when updating existing type declaration file
+                if (!this.fs.get(file.path)) {
+                    const baseFolder = this.ensureFolder(getDirectoryPath(file.fullPath));
+                    this.addFileOrFolderInFolder(baseFolder, file, ignoreWatchInvokedWithTriggerAsFileCreate);
+                }
             }
             else if (isString(fileOrDirectory.symLink)) {
                 const symLink = this.toSymLink(fileOrDirectory);

src/harness/virtualFileSystemWithWatch.ts

@@ -253,7 +253,7 @@ namespace ts.server {
         private requestMap = createMap<QueuedOperation>(); // Maps operation ID to newest requestQueue entry with that ID
         /** We will lazily request the types registry on the first call to `isKnownTypesPackageName` and store it in `typesRegistryCache`. */
         private requestedRegistry: boolean;
-        private typesRegistryCache: Map<void> | undefined;
+        private typesRegistryCache: Map<MapLike<string>> | undefined;
 
         // This number is essentially arbitrary.  Processing more than one typings request
         // at a time makes sense, but having too many in the pipe results in a hang

src/server/server.ts

@@ -75,7 +75,7 @@ declare namespace ts.server {
     /* @internal */
     export interface TypesRegistryResponse extends TypingInstallerResponse {
         readonly kind: EventTypesRegistry;
-        readonly typesRegistry: MapLike<void>;
+        readonly typesRegistry: MapLike<MapLike<string>>;
     }
 
     export interface PackageInstalledResponse extends ProjectResponse {

src/server/types.ts

@@ -41,15 +41,15 @@ namespace ts.server.typingsInstaller {
     }
 
     interface TypesRegistryFile {
-        entries: MapLike<void>;
+        entries: MapLike<MapLike<string>>;
     }
 
-    function loadTypesRegistryFile(typesRegistryFilePath: string, host: InstallTypingHost, log: Log): Map<void> {
+    function loadTypesRegistryFile(typesRegistryFilePath: string, host: InstallTypingHost, log: Log): Map<MapLike<string>> {
         if (!host.fileExists(typesRegistryFilePath)) {
             if (log.isEnabled()) {
                 log.writeLine(`Types registry file '${typesRegistryFilePath}' does not exist`);
             }
-            return createMap<void>();
+            return createMap<MapLike<string>>();
         }
         try {
             const content = <TypesRegistryFile>JSON.parse(host.readFile(typesRegistryFilePath));
@@ -59,7 +59,7 @@ namespace ts.server.typingsInstaller {
             if (log.isEnabled()) {
                 log.writeLine(`Error when loading types registry file '${typesRegistryFilePath}': ${(<Error>e).message}, ${(<Error>e).stack}`);
             }
-            return createMap<void>();
+            return createMap<MapLike<string>>();
         }
     }
 
@@ -77,7 +77,7 @@ namespace ts.server.typingsInstaller {
     export class NodeTypingsInstaller extends TypingsInstaller {
         private readonly nodeExecSync: ExecSync;
         private readonly npmPath: string;
-        readonly typesRegistry: Map<void>;
+        readonly typesRegistry: Map<MapLike<string>>;
 
         private delayedInitializationError: InitializationFailedResponse | undefined;
 
@@ -141,7 +141,7 @@ namespace ts.server.typingsInstaller {
                         this.closeProject(req);
                         break;
                     case "typesRegistry": {
-                        const typesRegistry: { [key: string]: void } = {};
+                        const typesRegistry: { [key: string]: MapLike<string> } = {};
                         this.typesRegistry.forEach((value, key) => {
                             typesRegistry[key] = value;
                         });

src/server/typingsInstaller/nodeTypingsInstaller.ts

@@ -1,6 +1,7 @@
 /// <reference path="../../compiler/core.ts" />
 /// <reference path="../../compiler/moduleNameResolver.ts" />
 /// <reference path="../../services/jsTyping.ts"/>
+/// <reference path="../../services/semver.ts"/>
 /// <reference path="../types.ts"/>
 /// <reference path="../shared.ts"/>
 
@@ -9,6 +10,10 @@ namespace ts.server.typingsInstaller {
         devDependencies: MapLike<any>;
     }
 
+    interface NpmLock {
+        dependencies: { [packageName: string]: { version: string } };
+    }
+
     export interface Log {
         isEnabled(): boolean;
         writeLine(text: string): void;
@@ -42,7 +47,7 @@ namespace ts.server.typingsInstaller {
     }
 
     export abstract class TypingsInstaller {
-        private readonly packageNameToTypingLocation: Map<string> = createMap<string>();
+        private readonly packageNameToTypingLocation: Map<JsTyping.CachedTyping> = createMap<JsTyping.CachedTyping>();
         private readonly missingTypingsSet: Map<true> = createMap<true>();
         private readonly knownCachesSet: Map<true> = createMap<true>();
         private readonly projectWatchers: Map<FileWatcher[]> = createMap<FileWatcher[]>();
@@ -52,7 +57,7 @@ namespace ts.server.typingsInstaller {
         private installRunCount = 1;
         private inFlightRequestCount = 0;
 
-        abstract readonly typesRegistry: Map<void>;
+        abstract readonly typesRegistry: Map<MapLike<string>>;
 
         constructor(
             protected readonly installTypingHost: InstallTypingHost,
@@ -117,7 +122,8 @@ namespace ts.server.typingsInstaller {
                 this.safeList,
                 this.packageNameToTypingLocation,
                 req.typeAcquisition,
-                req.unresolvedImports);
+                req.unresolvedImports,
+                this.typesRegistry);
 
             if (this.log.isEnabled()) {
                 this.log.writeLine(`Finished typings discovery: ${JSON.stringify(discoverTypingsResult)}`);
@@ -156,23 +162,30 @@ namespace ts.server.typingsInstaller {
             if (this.log.isEnabled()) {
                 this.log.writeLine(`Processing cache location '${cacheLocation}'`);
             }
-            if (this.knownCachesSet.get(cacheLocation)) {
+            if (this.knownCachesSet.has(cacheLocation)) {
                 if (this.log.isEnabled()) {
                     this.log.writeLine(`Cache location was already processed...`);
                 }
                 return;
             }
             const packageJson = combinePaths(cacheLocation, "package.json");
+            const packageLockJson = combinePaths(cacheLocation, "package-lock.json");
             if (this.log.isEnabled()) {
                 this.log.writeLine(`Trying to find '${packageJson}'...`);
             }
-            if (this.installTypingHost.fileExists(packageJson)) {
+            if (this.installTypingHost.fileExists(packageJson) && this.installTypingHost.fileExists(packageLockJson)) {
                 const npmConfig = <NpmConfig>JSON.parse(this.installTypingHost.readFile(packageJson));
+                const npmLock = <NpmLock>JSON.parse(this.installTypingHost.readFile(packageLockJson));
                 if (this.log.isEnabled()) {
                     this.log.writeLine(`Loaded content of '${packageJson}': ${JSON.stringify(npmConfig)}`);
+                    this.log.writeLine(`Loaded content of '${packageLockJson}'`);
                 }
-                if (npmConfig.devDependencies) {
+                if (npmConfig.devDependencies && npmLock.dependencies) {
                     for (const key in npmConfig.devDependencies) {
+                        if (!hasProperty(npmLock.dependencies, key)) {
+                            // if package in package.json but not package-lock.json, skip adding to cache so it is reinstalled on next use
+                            continue;
+                        }
                         // key is @types/<package name>
                         const packageName = getBaseFileName(key);
                         if (!packageName) {
@@ -184,18 +197,23 @@ namespace ts.server.typingsInstaller {
                             continue;
                         }
                         const existingTypingFile = this.packageNameToTypingLocation.get(packageName);
-                        if (existingTypingFile === typingFile) {
-                            continue;
-                        }
                         if (existingTypingFile) {
+                            if (existingTypingFile.typingLocation === typingFile) {
+                                continue;
+                            }
+
                             if (this.log.isEnabled()) {
                                 this.log.writeLine(`New typing for package ${packageName} from '${typingFile}' conflicts with existing typing file '${existingTypingFile}'`);
                             }
                         }
                         if (this.log.isEnabled()) {
                             this.log.writeLine(`Adding entry into typings cache: '${packageName}' => '${typingFile}'`);
                         }
-                        this.packageNameToTypingLocation.set(packageName, typingFile);
+                        const info = getProperty(npmLock.dependencies, key);
+                        const version = info && info.version;
+                        const semver = Semver.parse(version);
+                        const newTyping: JsTyping.CachedTyping = { typingLocation: typingFile, version: semver };
+                        this.packageNameToTypingLocation.set(packageName, newTyping);
                     }
                 }
             }
@@ -211,10 +229,6 @@ namespace ts.server.typingsInstaller {
                     if (this.log.isEnabled()) this.log.writeLine(`'${typing}' is in missingTypingsSet - skipping...`);
                     return false;
                 }
-                if (this.packageNameToTypingLocation.get(typing)) {
-                    if (this.log.isEnabled()) this.log.writeLine(`'${typing}' already has a typing - skipping...`);
-                    return false;
-                }
                 const validationResult = JsTyping.validatePackageName(typing);
                 if (validationResult !== JsTyping.PackageNameValidationResult.Ok) {
                     // add typing name to missing set so we won't process it again
@@ -226,6 +240,10 @@ namespace ts.server.typingsInstaller {
                     if (this.log.isEnabled()) this.log.writeLine(`Entry for package '${typing}' does not exist in local types registry - skipping...`);
                     return false;
                 }
+                if (this.packageNameToTypingLocation.get(typing) && JsTyping.isTypingUpToDate(this.packageNameToTypingLocation.get(typing), this.typesRegistry.get(typing))) {
+                    if (this.log.isEnabled()) this.log.writeLine(`'${typing}' already has an up-to-date typing - skipping...`);
+                    return false;
+                }
                 return true;
             });
         }
@@ -294,9 +312,12 @@ namespace ts.server.typingsInstaller {
                             this.missingTypingsSet.set(packageName, true);
                             continue;
                         }
-                        if (!this.packageNameToTypingLocation.has(packageName)) {
-                            this.packageNameToTypingLocation.set(packageName, typingFile);
-                        }
+
+                        // packageName is guaranteed to exist in typesRegistry by filterTypings
+                        const distTags = this.typesRegistry.get(packageName);
+                        const newVersion = Semver.parse(distTags[`ts${ts.versionMajorMinor}`] || distTags[latestDistTag]);
+                        const newTyping: JsTyping.CachedTyping = { typingLocation: typingFile, version: newVersion };
+                        this.packageNameToTypingLocation.set(packageName, newTyping);
                         installedTypingFiles.push(typingFile);
                     }
                     if (this.log.isEnabled()) {
@@ -390,4 +411,6 @@ namespace ts.server.typingsInstaller {
     export function typingsName(packageName: string): string {
         return `@types/${packageName}@ts${versionMajorMinor}`;
     }
+
+    const latestDistTag = "latest";
 }

src/server/typingsInstaller/typingsInstaller.ts

@@ -4,6 +4,7 @@
 /// <reference path='../compiler/types.ts' />
 /// <reference path='../compiler/core.ts' />
 /// <reference path='../compiler/commandLineParser.ts' />
+/// <reference path='../services/semver.ts' />
 
 /* @internal */
 namespace ts.JsTyping {
@@ -26,6 +27,17 @@ namespace ts.JsTyping {
         typings?: string;
     }
 
+    export interface CachedTyping {
+        typingLocation: string;
+        version: Semver;
+    }
+
+    /* @internal */
+    export function isTypingUpToDate(cachedTyping: JsTyping.CachedTyping, availableTypingVersions: MapLike<string>) {
+        const availableVersion = Semver.parse(getProperty(availableTypingVersions, `ts${ts.versionMajorMinor}`) || getProperty(availableTypingVersions, "latest"));
+        return !availableVersion.greaterThan(cachedTyping.version);
+    }
+
     /* @internal */
     export const nodeCoreModuleList: ReadonlyArray<string> = [
         "buffer", "querystring", "events", "http", "cluster",
@@ -60,7 +72,7 @@ namespace ts.JsTyping {
      * @param fileNames are the file names that belong to the same project
      * @param projectRootPath is the path to the project root directory
      * @param safeListPath is the path used to retrieve the safe list
-     * @param packageNameToTypingLocation is the map of package names to their cached typing locations
+     * @param packageNameToTypingLocation is the map of package names to their cached typing locations and installed versions
      * @param typeAcquisition is used to customize the typing acquisition process
      * @param compilerOptions are used as a source for typing inference
      */
@@ -70,9 +82,10 @@ namespace ts.JsTyping {
         fileNames: string[],
         projectRootPath: Path,
         safeList: SafeList,
-        packageNameToTypingLocation: ReadonlyMap<string>,
+        packageNameToTypingLocation: ReadonlyMap<CachedTyping>,
         typeAcquisition: TypeAcquisition,
-        unresolvedImports: ReadonlyArray<string>):
+        unresolvedImports: ReadonlyArray<string>,
+        typesRegistry: ReadonlyMap<MapLike<string>>):
         { cachedTypingPaths: string[], newTypingNames: string[], filesToWatch: string[] } {
 
         if (!typeAcquisition || !typeAcquisition.enable) {
@@ -122,9 +135,9 @@ namespace ts.JsTyping {
             addInferredTypings(module, "Inferred typings from unresolved imports");
         }
         // Add the cached typing locations for inferred typings that are already installed
-        packageNameToTypingLocation.forEach((typingLocation, name) => {
-            if (inferredTypings.has(name) && inferredTypings.get(name) === undefined) {
-                inferredTypings.set(name, typingLocation);
+        packageNameToTypingLocation.forEach((typing, name) => {
+            if (inferredTypings.has(name) && inferredTypings.get(name) === undefined && isTypingUpToDate(typing, typesRegistry.get(name))) {
+                inferredTypings.set(name, typing.typingLocation);
             }
         });