resolve npm imports

Author: mbostock

src/build.ts

@@ -112,7 +112,7 @@ export async function build(
       continue;
     }
     effects.output.write(`${faint("copy")} ${sourcePath} ${faint("→")} `);
-    const contents = rewriteModule(await readFile(sourcePath, "utf-8"), file, importResolver);
+    const contents = await rewriteModule(await readFile(sourcePath, "utf-8"), file, importResolver);
     await effects.writeFile(outputPath, contents);
   }
 

src/javascript.ts

@@ -8,7 +8,8 @@ import {findDeclarations} from "./javascript/declarations.js";
 import {findFeatures} from "./javascript/features.js";
 import {rewriteFetches} from "./javascript/fetches.js";
 import {defaultGlobals} from "./javascript/globals.js";
-import {createImportResolver, findExports, findImports, rewriteImports} from "./javascript/imports.js";
+import {findExports, findImportDeclarations, findImports} from "./javascript/imports.js";
+import {createImportResolver, rewriteImports} from "./javascript/imports.js";
 import {findReferences} from "./javascript/references.js";
 import {syntaxError} from "./javascript/syntaxError.js";
 import {Sourcemap} from "./sourcemap.js";
@@ -35,17 +36,24 @@ export interface Feature {
   name: string;
 }
 
-export interface Transpile {
+export interface BaseTranspile {
   id: string;
   inputs?: string[];
   outputs?: string[];
   inline?: boolean;
-  body: string;
   databases?: DatabaseReference[];
   files?: FileReference[];
   imports?: ImportReference[];
 }
 
+export interface PendingTranspile extends BaseTranspile {
+  body: Promise<string>;
+}
+
+export interface Transpile extends BaseTranspile {
+  body: string;
+}
+
 export interface ParseOptions {
   id: string;
   root: string;
@@ -56,7 +64,7 @@ export interface ParseOptions {
   verbose?: boolean;
 }
 
-export function transpileJavaScript(input: string, options: ParseOptions): Transpile {
+export function transpileJavaScript(input: string, options: ParseOptions): PendingTranspile {
   const {id, root, sourcePath, verbose = true} = options;
   try {
     const node = parseJavaScript(input, options);
@@ -67,25 +75,30 @@ export function transpileJavaScript(input: string, options: ParseOptions): Trans
       .filter((f) => f.type === "FileAttachment")
       .map(({name}) => fileReference(name, sourcePath));
     const inputs = Array.from(new Set<string>(node.references.map((r) => r.name)));
-    const output = new Sourcemap(input);
-    trim(output, input);
-    if (node.expression && !inputs.includes("display") && !inputs.includes("view")) {
-      output.insertLeft(0, "display((\n");
-      output.insertRight(input.length, "\n))");
-      inputs.push("display");
-    }
-    rewriteImports(output, node, sourcePath, createImportResolver(root, "_import"));
-    rewriteFetches(output, node, sourcePath);
+    const implicitDisplay = node.expression && !inputs.includes("display") && !inputs.includes("view");
+    if (implicitDisplay) inputs.push("display");
+    if (findImportDeclarations(node).length > 0) node.async = true;
     return {
       id,
       ...(inputs.length ? {inputs} : null),
       ...(options.inline ? {inline: true} : null),
       ...(node.declarations?.length ? {outputs: node.declarations.map(({name}) => name)} : null),
       ...(databases.length ? {databases: resolveDatabases(databases)} : null),
       ...(files.length ? {files} : null),
-      body: `${node.async ? "async " : ""}(${inputs}) => {
+      body: (async () => {
+        const output = new Sourcemap(input);
+        trim(output, input);
+        if (implicitDisplay) {
+          output.insertLeft(0, "display((\n");
+          output.insertRight(input.length, "\n))");
+        }
+        await rewriteImports(output, node, sourcePath, createImportResolver(root, "_import"));
+        rewriteFetches(output, node, sourcePath);
+        const result = `${node.async ? "async " : ""}(${inputs}) => {
 ${String(output)}${node.declarations?.length ? `\nreturn {${node.declarations.map(({name}) => name)}};` : ""}
-}`,
+}`;
+        return result;
+      })(),
       ...(node.imports.length ? {imports: node.imports} : null)
     };
   } catch (error) {
@@ -104,7 +117,7 @@ ${String(output)}${node.declarations?.length ? `\nreturn {${node.declarations.ma
     if (verbose) console.error(red(`${error.name}: ${message}`));
     return {
       id: `${id}`,
-      body: `() => { throw new SyntaxError(${JSON.stringify(error.message)}); }`
+      body: Promise.resolve(`() => { throw new SyntaxError(${JSON.stringify(error.message)}); }`)
     };
   }
 }

src/javascript/imports.ts

@@ -145,10 +145,11 @@ export function parseLocalImports(root: string, paths: string[]): ImportsAndFetc
 }
 
 /** Rewrites import specifiers in the specified ES module source. */
-export function rewriteModule(input: string, sourcePath: string, resolver: ImportResolver): string {
+export async function rewriteModule(input: string, sourcePath: string, resolver: ImportResolver): Promise<string> {
   const body = Parser.parse(input, parseOptions) as Program;
   const references: Identifier[] = findReferences(body, defaultGlobals);
   const output = new Sourcemap(input);
+  const imports: (ImportDeclaration | ImportExpression | ExportAllDeclaration | ExportNamedDeclaration)[] = [];
 
   simple(body, {
     ImportDeclaration: rewriteImport,
@@ -161,59 +162,87 @@ export function rewriteModule(input: string, sourcePath: string, resolver: Impor
   });
 
   function rewriteImport(node: ImportDeclaration | ImportExpression | ExportAllDeclaration | ExportNamedDeclaration) {
+    imports.push(node);
+  }
+
+  for (const node of imports) {
     if (isStringLiteral(node.source)) {
       output.replaceLeft(
         node.source.start,
         node.source.end,
-        JSON.stringify(resolver(sourcePath, getStringLiteralValue(node.source)))
+        JSON.stringify(await resolver(sourcePath, getStringLiteralValue(node.source)))
       );
     }
   }
 
   return String(output);
 }
 
+export function findImportDeclarations(cell: JavaScriptNode): ImportDeclaration[] {
+  const declarations: ImportDeclaration[] = [];
+
+  simple(cell.body, {
+    ImportDeclaration(node) {
+      if (isStringLiteral(node.source)) {
+        declarations.push(node);
+      }
+    }
+  });
+
+  return declarations;
+}
+
 /**
  * Rewrites import specifiers in the specified JavaScript fenced code block or
  * inline expression. TODO parallelize multiple static imports.
  */
-export function rewriteImports(
+export async function rewriteImports(
   output: Sourcemap,
   cell: JavaScriptNode,
   sourcePath: string,
   resolver: ImportResolver
-): void {
+): Promise<void> {
+  const expressions: ImportExpression[] = [];
+  const declarations: ImportDeclaration[] = [];
+
   simple(cell.body, {
     ImportExpression(node) {
       if (isStringLiteral(node.source)) {
-        output.replaceLeft(
-          node.source.start,
-          node.source.end,
-          JSON.stringify(resolver(sourcePath, getStringLiteralValue(node.source)))
-        );
+        expressions.push(node);
       }
     },
     ImportDeclaration(node) {
       if (isStringLiteral(node.source)) {
-        cell.async = true;
-        output.replaceLeft(
-          node.start,
-          node.end,
-          `const ${
-            node.specifiers.some(isNotNamespaceSpecifier)
-              ? `{${node.specifiers.filter(isNotNamespaceSpecifier).map(rewriteImportSpecifier).join(", ")}}`
-              : node.specifiers.find(isNamespaceSpecifier)?.local.name ?? "{}"
-          } = await import(${JSON.stringify(resolver(sourcePath, getStringLiteralValue(node.source)))});`
-        );
+        declarations.push(node);
       }
     }
   });
+
+  for (const node of expressions) {
+    output.replaceLeft(
+      node.source.start,
+      node.source.end,
+      JSON.stringify(await resolver(sourcePath, getStringLiteralValue(node.source)))
+    );
+  }
+
+  for (const node of declarations) {
+    output.replaceLeft(
+      node.start,
+      node.end,
+      `const ${
+        node.specifiers.some(isNotNamespaceSpecifier)
+          ? `{${node.specifiers.filter(isNotNamespaceSpecifier).map(rewriteImportSpecifier).join(", ")}}`
+          : node.specifiers.find(isNamespaceSpecifier)?.local.name ?? "{}"
+      } = await import(${JSON.stringify(await resolver(sourcePath, getStringLiteralValue(node.source)))});`
+    );
+  }
 }
 
-export type ImportResolver = (path: string, specifier: string) => string;
+export type ImportResolver = (path: string, specifier: string) => Promise<string>;
 
 export function createImportResolver(root: string, base: "." | "_import" = "."): ImportResolver {
-  return (path, specifier) => {
+  return async (path, specifier) => {
     return isLocalImport(specifier, path)
       ? relativeUrl(path, resolvePath(base, path, resolveImportHash(root, path, specifier)))
       : specifier === "npm:@observablehq/runtime"
@@ -233,11 +262,29 @@ export function createImportResolver(root: string, base: "." | "_import" = "."):
       : specifier === "npm:@observablehq/xslx"
       ? resolveBuiltin(base, path, "stdlib/xslx.js") // TODO publish to npm
       : specifier.startsWith("npm:")
-      ? `https://cdn.jsdelivr.net/npm/${specifier.slice("npm:".length)}/+esm`
+      ? await resolveNpmImport(specifier.slice("npm:".length))
       : specifier;
   };
 }
 
+// Like import, don’t fetch the same package more than once to ensure
+// consistency; restart the server if you want to clear the cache.
+const npmCache = new Map<string, Promise<string>>();
+
+export async function resolveNpmImport(specifier: string): Promise<string> {
+  let promise = npmCache.get(specifier);
+  if (promise) return promise;
+  promise = (async () => {
+    const response = await fetch(`https://data.jsdelivr.com/v1/packages/npm/${specifier}/resolved`);
+    if (!response.ok) throw new Error(`unable to resolve npm specifier: ${specifier}`);
+    const body = await response.json();
+    return `https://cdn.jsdelivr.net/npm/${specifier}@${body.version}/+esm`;
+  })();
+  promise.catch(() => npmCache.delete(specifier)); // try again on error
+  npmCache.set(specifier, promise);
+  return promise;
+}
+
 function resolveBuiltin(base: "." | "_import", path: string, specifier: string): string {
   return relativeUrl(join(base === "." ? "_import" : ".", path), join("_observablehq", specifier));
 }

src/markdown.ts

@@ -15,7 +15,8 @@ import {isEnoent} from "./error.js";
 import {fileReference, getLocalPath} from "./files.js";
 import {computeHash} from "./hash.js";
 import {parseInfo} from "./info.js";
-import {type FileReference, type ImportReference, type Transpile, transpileJavaScript} from "./javascript.js";
+import type {FileReference, ImportReference, PendingTranspile, Transpile} from "./javascript.js";
+import {transpileJavaScript} from "./javascript.js";
 import {transpileTag} from "./tag.js";
 import {resolvePath} from "./url.js";
 
@@ -50,7 +51,7 @@ export interface ParseResult {
 
 interface RenderPiece {
   html: string;
-  code: Transpile[];
+  code: PendingTranspile[];
 }
 
 interface ParseContext {
@@ -407,16 +408,13 @@ function toParsePieces(pieces: RenderPiece[]): HtmlPiece[] {
   }));
 }
 
-function toParseCells(pieces: RenderPiece[]): CellPiece[] {
+async function toParseCells(pieces: RenderPiece[]): Promise<CellPiece[]> {
   const cellPieces: CellPiece[] = [];
-  pieces.forEach((piece) =>
-    piece.code.forEach((code) =>
-      cellPieces.push({
-        type: "cell",
-        ...code
-      })
-    )
-  );
+  for (const piece of pieces) {
+    for (const {body, ...rest} of piece.code) {
+      cellPieces.push({type: "cell", body: await body, ...rest});
+    }
+  }
   return cellPieces;
 }
 
@@ -435,14 +433,16 @@ export async function parseMarkdown(source: string, root: string, sourcePath: st
   const context: ParseContext = {files: [], imports: [], pieces: [], startLine: 0, currentLine: 0};
   const tokens = md.parse(parts.content, context);
   const html = md.renderer.render(tokens, md.options, context); // Note: mutates context.pieces, context.files!
+  const pieces = toParsePieces(context.pieces);
+  const cells = await toParseCells(context.pieces);
   return {
     html,
     data: isEmpty(parts.data) ? null : parts.data,
     title: parts.data?.title ?? findTitle(tokens) ?? null,
     files: context.files,
     imports: context.imports,
-    pieces: toParsePieces(context.pieces),
-    cells: toParseCells(context.pieces),
+    pieces,
+    cells,
     hash: await computeMarkdownHash(source, root, sourcePath, context.imports)
   };
 }

src/preview.ts

@@ -99,7 +99,7 @@ export class PreviewServer {
           if (!isEnoent(error)) throw error;
           throw new HttpError(`Not found: ${pathname}`, 404);
         }
-        end(req, res, rewriteModule(js, file, createImportResolver(this.root)), "text/javascript");
+        end(req, res, await rewriteModule(js, file, createImportResolver(this.root)), "text/javascript");
       } else if (pathname.startsWith("/_file/")) {
         const path = pathname.slice("/_file".length);
         const filepath = join(this.root, path);

src/render.ts

@@ -2,7 +2,7 @@ import {parseHTML} from "linkedom";
 import {type Config, type Page, type Section, mergeToc} from "./config.js";
 import {type Html, html} from "./html.js";
 import {type ImportResolver, createImportResolver} from "./javascript/imports.js";
-import {type FileReference, type ImportReference} from "./javascript.js";
+import type {FileReference, ImportReference, Transpile} from "./javascript.js";
 import {addImplicitSpecifiers, addImplicitStylesheets} from "./libraries.js";
 import {type ParseResult, parseMarkdown} from "./markdown.js";
 import {type PageLink, findLink} from "./pager.js";
@@ -22,7 +22,7 @@ export interface RenderOptions extends Config {
 export async function renderPreview(source: string, options: RenderOptions): Promise<Render> {
   const parseResult = await parseMarkdown(source, options.root, options.path);
   return {
-    html: render(parseResult, {...options, preview: true}),
+    html: await render(parseResult, {...options, preview: true}),
     files: parseResult.files,
     imports: parseResult.imports
   };
@@ -31,13 +31,13 @@ export async function renderPreview(source: string, options: RenderOptions): Pro
 export async function renderServerless(source: string, options: RenderOptions): Promise<Render> {
   const parseResult = await parseMarkdown(source, options.root, options.path);
   return {
-    html: render(parseResult, options),
+    html: await render(parseResult, options),
     files: parseResult.files,
     imports: parseResult.imports
   };
 }
 
-export function renderDefineCell(cell): string {
+export function renderDefineCell(cell: Transpile): string {
   const {id, inline, inputs, outputs, files, body, databases} = cell;
   return `define({${Object.entries({id, inline, inputs, outputs, files, databases})
     .filter((arg) => arg[1] !== undefined)
@@ -49,7 +49,7 @@ type RenderInternalOptions =
   | {preview?: false} // serverless
   | {preview: true}; // preview
 
-function render(parseResult: ParseResult, options: RenderOptions & RenderInternalOptions): string {
+async function render(parseResult: ParseResult, options: RenderOptions & RenderInternalOptions): Promise<string> {
   const {root, path, pages, title, preview} = options;
   const toc = mergeToc(parseResult.data?.toc, options.toc);
   const headers = toc.show ? findHeaders(parseResult) : [];
@@ -62,7 +62,7 @@ ${
         .filter((title): title is string => !!title)
         .join(" | ")}</title>\n`
     : ""
-}${renderLinks(parseResult, path, createImportResolver(root, "_import"))}
+}${await renderLinks(parseResult, path, createImportResolver(root, "_import"))}
 <script type="module">${html.unsafe(`
 
 import {${preview ? "open, " : ""}define} from ${JSON.stringify(relativeUrl(path, "/_observablehq/client.js"))};
@@ -150,15 +150,15 @@ function prettyPath(path: string): string {
   return path.replace(/\/index$/, "/") || "/";
 }
 
-function renderLinks(parseResult: ParseResult, path: string, resolver: ImportResolver): Html {
+async function renderLinks(parseResult: ParseResult, path: string, resolver: ImportResolver): Promise<Html> {
   const stylesheets = new Set<string>([relativeUrl(path, "/_observablehq/style.css"), "https://fonts.googleapis.com/css2?family=Source+Serif+Pro:ital,wght@0,400;0,600;0,700;1,400;1,600;1,700&display=swap"]); // prettier-ignore
   const specifiers = new Set<string>(["npm:@observablehq/runtime", "npm:@observablehq/stdlib"]);
   for (const {name} of parseResult.imports) specifiers.add(name);
   const inputs = new Set(parseResult.cells.flatMap((cell) => cell.inputs ?? []));
   addImplicitSpecifiers(specifiers, inputs);
   addImplicitStylesheets(stylesheets, specifiers);
   const preloads = new Set<string>();
-  for (const specifier of specifiers) preloads.add(resolver(path, specifier));
+  for (const specifier of specifiers) preloads.add(await resolver(path, specifier));
   if (parseResult.cells.some((cell) => cell.databases?.length)) preloads.add(relativeUrl(path, "/_observablehq/database.js")); // prettier-ignore
   return html`<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>${
     Array.from(stylesheets).sort().map(renderStylesheet) // <link rel=stylesheet>