Enable TS Server plugins on web (#47377)

* Prototype TS plugins on web

This prototype allows service plugins to be loaded on web TSServer

Main changes:

- Adds a new host entryPoint called `importServicePlugin` for overriding how plugins can be loaded. This may be async
- Implement `importServicePlugin` for webServer
- The web server plugin implementation looks for a `browser` field in the plugin's `package.json`
- It then uses `import(...)` to load the plugin (the plugin source must be compiled to support being loaded as a module)

* use default export from plugins

This more or less matches how node plugins expect the plugin module to be an init function

* Allow configure plugin requests against any web servers in partial semantic mode

* Addressing some comments

- Use result value instead of try/catch (`ImportPluginResult`)
- Add awaits
- Add logging

* add tsserverWeb to patch in dynamic import

* Remove eval

We should throw instead when dynamic import is not implemented

* Ensure dynamically imported plugins are loaded in the correct order

* Add tests for async service plugin timing

* Update src/server/editorServices.ts

Co-authored-by: Nathan Shively-Sanders <[email protected]>

* Partial PR feedback

* Rename tsserverWeb to dynamicImportCompat

* Additional PR feedback

Co-authored-by: Ron Buckton <[email protected]>
Co-authored-by: Nathan Shively-Sanders <[email protected]>

Author: 3 people

Gulpfile.js

@@ -214,20 +214,29 @@ task("watch-services").flags = {
     "   --built": "Compile using the built version of the compiler."
 };
 
-const buildServer = () => buildProject("src/tsserver", cmdLineOptions);
+const buildDynamicImportCompat = () => buildProject("src/dynamicImportCompat", cmdLineOptions);
+task("dynamicImportCompat", buildDynamicImportCompat);
+
+const buildServerMain = () => buildProject("src/tsserver", cmdLineOptions);
+const buildServer = series(buildDynamicImportCompat, buildServerMain);
+buildServer.displayName = "buildServer";
 task("tsserver", series(preBuild, buildServer));
 task("tsserver").description = "Builds the language server";
 task("tsserver").flags = {
     "   --built": "Compile using the built version of the compiler."
 };
 
-const cleanServer = () => cleanProject("src/tsserver");
+const cleanDynamicImportCompat = () => cleanProject("src/dynamicImportCompat");
+const cleanServerMain = () => cleanProject("src/tsserver");
+const cleanServer = series(cleanDynamicImportCompat, cleanServerMain);
+cleanServer.displayName = "cleanServer";
 cleanTasks.push(cleanServer);
 task("clean-tsserver", cleanServer);
 task("clean-tsserver").description = "Cleans outputs for the language server";
 
+const watchDynamicImportCompat = () => watchProject("src/dynamicImportCompat", cmdLineOptions);
 const watchServer = () => watchProject("src/tsserver", cmdLineOptions);
-task("watch-tsserver", series(preBuild, parallel(watchLib, watchDiagnostics, watchServer)));
+task("watch-tsserver", series(preBuild, parallel(watchLib, watchDiagnostics, watchDynamicImportCompat, watchServer)));
 task("watch-tsserver").description = "Watch for changes and rebuild the language server only";
 task("watch-tsserver").flags = {
     "   --built": "Compile using the built version of the compiler."
@@ -549,6 +558,7 @@ const produceLKG = async () => {
         "built/local/typescriptServices.js",
         "built/local/typescriptServices.d.ts",
         "built/local/tsserver.js",
+        "built/local/dynamicImportCompat.js",
         "built/local/typescript.js",
         "built/local/typescript.d.ts",
         "built/local/tsserverlibrary.js",

scripts/produceLKG.ts

@@ -62,6 +62,7 @@ async function copyScriptOutputs() {
     await copyWithCopyright("cancellationToken.js");
     await copyWithCopyright("tsc.release.js", "tsc.js");
     await copyWithCopyright("tsserver.js");
+    await copyWithCopyright("dynamicImportCompat.js");
     await copyFromBuiltLocal("tsserverlibrary.js"); // copyright added by build
     await copyFromBuiltLocal("typescript.js"); // copyright added by build
     await copyFromBuiltLocal("typescriptServices.js"); // copyright added by build

src/dynamicImportCompat/dynamicImportCompat.ts

@@ -0,0 +1,3 @@
+namespace ts.server {
+    export const dynamicImport = (id: string) => import(id);
+}

src/dynamicImportCompat/tsconfig.json

@@ -0,0 +1,16 @@
+{
+    "extends": "../tsconfig-library-base",
+    "compilerOptions": {
+        "outDir": "../../built/local",
+        "rootDir": ".",
+        "target": "esnext",
+        "module": "esnext",
+        "lib": ["esnext"],
+        "declaration": false,
+        "sourceMap": true,
+        "tsBuildInfoFile": "../../built/local/dynamicImportCompat.tsbuildinfo"
+    },
+    "files": [
+        "dynamicImportCompat.ts",
+    ]
+}

src/harness/util.ts

@@ -109,4 +109,20 @@ namespace Utils {
             value === undefined ? "undefined" :
             JSON.stringify(value);
     }
+
+    export interface Deferred<T> {
+        resolve: (value: T | PromiseLike<T>) => void;
+        reject: (reason: unknown) => void;
+        promise: Promise<T>;
+    }
+
+    export function defer<T = void>(): Deferred<T> {
+        let resolve!: (value: T | PromiseLike<T>) => void;
+        let reject!: (reason: unknown) => void;
+        const promise = new Promise<T>((_resolve, _reject) => {
+            resolve = _resolve;
+            reject = _reject;
+        });
+        return { resolve, reject, promise };
+    }
 }

src/server/editorServices.ts

@@ -804,6 +804,9 @@ namespace ts.server {
 
         private performanceEventHandler?: PerformanceEventHandler;
 
+        private pendingPluginEnablements?: ESMap<Project, Promise<BeginEnablePluginResult>[]>;
+        private currentPluginEnablementPromise?: Promise<void>;
+
         constructor(opts: ProjectServiceOptions) {
             this.host = opts.host;
             this.logger = opts.logger;
@@ -4063,6 +4066,114 @@ namespace ts.server {
             return false;
         }
 
+        /*@internal*/
+        requestEnablePlugin(project: Project, pluginConfigEntry: PluginImport, searchPaths: string[], pluginConfigOverrides: Map<any> | undefined) {
+            if (!this.host.importServicePlugin && !this.host.require) {
+                this.logger.info("Plugins were requested but not running in environment that supports 'require'. Nothing will be loaded");
+                return;
+            }
+
+            this.logger.info(`Enabling plugin ${pluginConfigEntry.name} from candidate paths: ${searchPaths.join(",")}`);
+            if (!pluginConfigEntry.name || parsePackageName(pluginConfigEntry.name).rest) {
+                this.logger.info(`Skipped loading plugin ${pluginConfigEntry.name || JSON.stringify(pluginConfigEntry)} because only package name is allowed plugin name`);
+                return;
+            }
+
+            // If the host supports dynamic import, begin enabling the plugin asynchronously.
+            if (this.host.importServicePlugin) {
+                const importPromise = project.beginEnablePluginAsync(pluginConfigEntry, searchPaths, pluginConfigOverrides);
+                this.pendingPluginEnablements ??= new Map();
+                let promises = this.pendingPluginEnablements.get(project);
+                if (!promises) this.pendingPluginEnablements.set(project, promises = []);
+                promises.push(importPromise);
+                return;
+            }
+
+            // Otherwise, load the plugin using `require`
+            project.endEnablePlugin(project.beginEnablePluginSync(pluginConfigEntry, searchPaths, pluginConfigOverrides));
+        }
+
+        /* @internal */
+        hasNewPluginEnablementRequests() {
+            return !!this.pendingPluginEnablements;
+        }
+
+        /* @internal */
+        hasPendingPluginEnablements() {
+            return !!this.currentPluginEnablementPromise;
+        }
+
+        /**
+         * Waits for any ongoing plugin enablement requests to complete.
+         */
+        /* @internal */
+        async waitForPendingPlugins() {
+            while (this.currentPluginEnablementPromise) {
+                await this.currentPluginEnablementPromise;
+            }
+        }
+
+        /**
+         * Starts enabling any requested plugins without waiting for the result.
+         */
+        /* @internal */
+        enableRequestedPlugins() {
+            if (this.pendingPluginEnablements) {
+                void this.enableRequestedPluginsAsync();
+            }
+        }
+
+        private async enableRequestedPluginsAsync() {
+            if (this.currentPluginEnablementPromise) {
+                // If we're already enabling plugins, wait for any existing operations to complete
+                await this.waitForPendingPlugins();
+            }
+
+            // Skip if there are no new plugin enablement requests
+            if (!this.pendingPluginEnablements) {
+                return;
+            }
+
+            // Consume the pending plugin enablement requests
+            const entries = arrayFrom(this.pendingPluginEnablements.entries());
+            this.pendingPluginEnablements = undefined;
+
+            // Start processing the requests, keeping track of the promise for the operation so that
+            // project consumers can potentially wait for the plugins to load.
+            this.currentPluginEnablementPromise = this.enableRequestedPluginsWorker(entries);
+            await this.currentPluginEnablementPromise;
+        }
+
+        private async enableRequestedPluginsWorker(pendingPlugins: [Project, Promise<BeginEnablePluginResult>[]][]) {
+            // This should only be called from `enableRequestedPluginsAsync`, which ensures this precondition is met.
+            Debug.assert(this.currentPluginEnablementPromise === undefined);
+
+            // Process all pending plugins, partitioned by project. This way a project with few plugins doesn't need to wait
+            // on a project with many plugins.
+            await Promise.all(map(pendingPlugins, ([project, promises]) => this.enableRequestedPluginsForProjectAsync(project, promises)));
+
+            // Clear the pending operation and notify the client that projects have been updated.
+            this.currentPluginEnablementPromise = undefined;
+            this.sendProjectsUpdatedInBackgroundEvent();
+        }
+
+        private async enableRequestedPluginsForProjectAsync(project: Project, promises: Promise<BeginEnablePluginResult>[]) {
+            // Await all pending plugin imports. This ensures all requested plugin modules are fully loaded
+            // prior to patching the language service, and that any promise rejections are observed.
+            const results = await Promise.all(promises);
+            if (project.isClosed()) {
+                // project is not alive, so don't enable plugins.
+                return;
+            }
+
+            for (const result of results) {
+                project.endEnablePlugin(result);
+            }
+
+            // Plugins may have modified external files, so mark the project as dirty.
+            this.delayUpdateProjectGraph(project);
+        }
+
         configurePlugin(args: protocol.ConfigurePluginRequestArguments) {
             // For any projects that already have the plugin loaded, configure the plugin
             this.forEachEnabledProject(project => project.onPluginConfigurationChanged(args.pluginName, args.configuration));

src/server/project.ts

@@ -102,6 +102,14 @@ namespace ts.server {
 
     export type PluginModuleFactory = (mod: { typescript: typeof ts }) => PluginModule;
 
+    /* @internal */
+    export interface BeginEnablePluginResult {
+        pluginConfigEntry: PluginImport;
+        pluginConfigOverrides: Map<any> | undefined;
+        resolvedModule: PluginModuleFactory | undefined;
+        errorLogs: string[] | undefined;
+    }
+
     /**
      * The project root can be script info - if root is present,
      * or it could be just normalized path if root wasn't present on the host(only for non inferred project)
@@ -134,6 +142,7 @@ namespace ts.server {
         private externalFiles: SortedReadonlyArray<string> | undefined;
         private missingFilesMap: ESMap<Path, FileWatcher> | undefined;
         private generatedFilesMap: GeneratedFileWatcherMap | undefined;
+
         private plugins: PluginModuleWithName[] = [];
 
         /*@internal*/
@@ -245,6 +254,26 @@ namespace ts.server {
             return result.module;
         }
 
+        /*@internal*/
+        public static async importServicePluginAsync(moduleName: string, initialDir: string, host: ServerHost, log: (message: string) => void, logErrors?: (message: string) => void): Promise<{} | undefined> {
+            Debug.assertIsDefined(host.importServicePlugin);
+            const resolvedPath = combinePaths(initialDir, "node_modules");
+            log(`Dynamically importing ${moduleName} from ${initialDir} (resolved to ${resolvedPath})`);
+            let result: ModuleImportResult;
+            try {
+                result = await host.importServicePlugin(resolvedPath, moduleName);
+            }
+            catch (e) {
+                result = { module: undefined, error: e };
+            }
+            if (result.error) {
+                const err = result.error.stack || result.error.message || JSON.stringify(result.error);
+                (logErrors || log)(`Failed to dynamically import module '${moduleName}' from ${resolvedPath}: ${err}`);
+                return undefined;
+            }
+            return result.module;
+        }
+
         /*@internal*/
         readonly currentDirectory: string;
 
@@ -1574,19 +1603,19 @@ namespace ts.server {
             return !!this.program && this.program.isSourceOfProjectReferenceRedirect(fileName);
         }
 
-        protected enableGlobalPlugins(options: CompilerOptions, pluginConfigOverrides: Map<any> | undefined) {
+        protected enableGlobalPlugins(options: CompilerOptions, pluginConfigOverrides: Map<any> | undefined): void {
             const host = this.projectService.host;
 
-            if (!host.require) {
+            if (!host.require && !host.importServicePlugin) {
                 this.projectService.logger.info("Plugins were requested but not running in environment that supports 'require'. Nothing will be loaded");
                 return;
             }
 
             // Search any globally-specified probe paths, then our peer node_modules
             const searchPaths = [
-              ...this.projectService.pluginProbeLocations,
-              // ../../.. to walk from X/node_modules/typescript/lib/tsserver.js to X/node_modules/
-              combinePaths(this.projectService.getExecutingFilePath(), "../../.."),
+                ...this.projectService.pluginProbeLocations,
+                // ../../.. to walk from X/node_modules/typescript/lib/tsserver.js to X/node_modules/
+                combinePaths(this.projectService.getExecutingFilePath(), "../../.."),
             ];
 
             if (this.projectService.globalPlugins) {
@@ -1606,20 +1635,51 @@ namespace ts.server {
             }
         }
 
-        protected enablePlugin(pluginConfigEntry: PluginImport, searchPaths: string[], pluginConfigOverrides: Map<any> | undefined) {
-            this.projectService.logger.info(`Enabling plugin ${pluginConfigEntry.name} from candidate paths: ${searchPaths.join(",")}`);
-            if (!pluginConfigEntry.name || parsePackageName(pluginConfigEntry.name).rest) {
-                this.projectService.logger.info(`Skipped loading plugin ${pluginConfigEntry.name || JSON.stringify(pluginConfigEntry)} because only package name is allowed plugin name`);
-                return;
-            }
+        /**
+         * Performs the initial steps of enabling a plugin by finding and instantiating the module for a plugin synchronously using 'require'.
+         */
+        /*@internal*/
+        beginEnablePluginSync(pluginConfigEntry: PluginImport, searchPaths: string[], pluginConfigOverrides: Map<any> | undefined): BeginEnablePluginResult {
+            Debug.assertIsDefined(this.projectService.host.require);
 
-            const log = (message: string) => this.projectService.logger.info(message);
             let errorLogs: string[] | undefined;
+            const log = (message: string) => this.projectService.logger.info(message);
             const logError = (message: string) => {
-                (errorLogs || (errorLogs = [])).push(message);
+                (errorLogs ??= []).push(message);
             };
             const resolvedModule = firstDefined(searchPaths, searchPath =>
                 Project.resolveModule(pluginConfigEntry.name, searchPath, this.projectService.host, log, logError) as PluginModuleFactory | undefined);
+            return { pluginConfigEntry, pluginConfigOverrides, resolvedModule, errorLogs };
+        }
+
+        /**
+         * Performs the initial steps of enabling a plugin by finding and instantiating the module for a plugin asynchronously using dynamic `import`.
+         */
+        /*@internal*/
+        async beginEnablePluginAsync(pluginConfigEntry: PluginImport, searchPaths: string[], pluginConfigOverrides: Map<any> | undefined): Promise<BeginEnablePluginResult> {
+            Debug.assertIsDefined(this.projectService.host.importServicePlugin);
+
+            let errorLogs: string[] | undefined;
+            const log = (message: string) => this.projectService.logger.info(message);
+            const logError = (message: string) => {
+                (errorLogs ??= []).push(message);
+            };
+
+            let resolvedModule: PluginModuleFactory | undefined;
+            for (const searchPath of searchPaths) {
+                resolvedModule = await Project.importServicePluginAsync(pluginConfigEntry.name, searchPath, this.projectService.host, log, logError) as PluginModuleFactory | undefined;
+                if (resolvedModule !== undefined) {
+                    break;
+                }
+            }
+            return { pluginConfigEntry, pluginConfigOverrides, resolvedModule, errorLogs };
+        }
+
+        /**
+         * Performs the remaining steps of enabling a plugin after its module has been instantiated.
+         */
+        /*@internal*/
+        endEnablePlugin({ pluginConfigEntry, pluginConfigOverrides, resolvedModule, errorLogs }: BeginEnablePluginResult) {
             if (resolvedModule) {
                 const configurationOverride = pluginConfigOverrides && pluginConfigOverrides.get(pluginConfigEntry.name);
                 if (configurationOverride) {
@@ -1632,11 +1692,15 @@ namespace ts.server {
                 this.enableProxy(resolvedModule, pluginConfigEntry);
             }
             else {
-                forEach(errorLogs, log);
+                forEach(errorLogs, message => this.projectService.logger.info(message));
                 this.projectService.logger.info(`Couldn't find ${pluginConfigEntry.name}`);
             }
         }
 
+        protected enablePlugin(pluginConfigEntry: PluginImport, searchPaths: string[], pluginConfigOverrides: Map<any> | undefined): void {
+            this.projectService.requestEnablePlugin(this, pluginConfigEntry, searchPaths, pluginConfigOverrides);
+        }
+
         private enableProxy(pluginModuleFactory: PluginModuleFactory, configEntry: PluginImport) {
             try {
                 if (typeof pluginModuleFactory !== "function") {
@@ -2456,10 +2520,10 @@ namespace ts.server {
         }
 
         /*@internal*/
-        enablePluginsWithOptions(options: CompilerOptions, pluginConfigOverrides: ESMap<string, any> | undefined) {
+        enablePluginsWithOptions(options: CompilerOptions, pluginConfigOverrides: ESMap<string, any> | undefined): void {
             const host = this.projectService.host;
 
-            if (!host.require) {
+            if (!host.require && !host.importServicePlugin) {
                 this.projectService.logger.info("Plugins were requested but not running in environment that supports 'require'. Nothing will be loaded");
                 return;
             }
@@ -2481,7 +2545,7 @@ namespace ts.server {
                 }
             }
 
-            this.enableGlobalPlugins(options, pluginConfigOverrides);
+            return this.enableGlobalPlugins(options, pluginConfigOverrides);
         }
 
         /**