resolve npm imports

src/build.ts

@@ -126,7 +126,7 @@ export async function build(
       continue;
     }
     effects.output.write(`${faint("copy")} ${sourcePath} ${faint("→")} `);
-    const contents = rewriteModule(await readFile(sourcePath, "utf-8"), file, importResolver);
+    const contents = await rewriteModule(await readFile(sourcePath, "utf-8"), file, importResolver);
     await effects.writeFile(outputPath, contents);
   }
 

src/client/stdlib/duckdb.js

@@ -31,12 +31,12 @@ import * as duckdb from "npm:@duckdb/duckdb-wasm";
 
 const bundle = await duckdb.selectBundle({
   mvp: {
-    mainModule: "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm",
-    mainWorker: "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js"
+    mainModule: import.meta.resolve("npm:@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm"),
+    mainWorker: import.meta.resolve("npm:@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js")
   },
   eh: {
-    mainModule: "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm/dist/duckdb-eh.wasm",
-    mainWorker: "https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js"
+    mainModule: import.meta.resolve("npm:@duckdb/duckdb-wasm/dist/duckdb-eh.wasm"),
+    mainWorker: import.meta.resolve("npm:@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js")
   }
 });
 

src/client/stdlib/mermaid.js

@@ -1,7 +1,7 @@
 import mer from "npm:mermaid";
 
 let nextId = 0;
-const theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "neutral";
+const theme = matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "neutral";
 mer.initialize({startOnLoad: false, securityLevel: "loose", theme});
 
 export default async function mermaid() {

src/client/stdlib/sqlite.js

@@ -1,9 +1,9 @@
 // https://github.com/sql-js/sql.js/issues/284
 const SQLite = await (async () => {
   const exports = {};
-  const response = await fetch("https://cdn.jsdelivr.net/npm/sql.js/dist/sql-wasm.js");
+  const response = await fetch(import.meta.resolve("npm:sql.js/dist/sql-wasm.js"));
   new Function("exports", await response.text())(exports);
-  return exports.Module({locateFile: (name) => `https://cdn.jsdelivr.net/npm/sql.js/dist/${name}`});
+  return exports.Module({locateFile: (name) => import.meta.resolve("npm:sql.js/dist/") + name});
 })();
 
 export default SQLite;

src/javascript.ts

@@ -8,7 +8,8 @@ import {findDeclarations} from "./javascript/declarations.js";
 import {findFeatures} from "./javascript/features.js";
 import {rewriteFetches} from "./javascript/fetches.js";
 import {defaultGlobals} from "./javascript/globals.js";
-import {createImportResolver, findExports, findImports, rewriteImports} from "./javascript/imports.js";
+import {findExports, findImportDeclarations, findImports} from "./javascript/imports.js";
+import {createImportResolver, rewriteImports} from "./javascript/imports.js";
 import {findReferences} from "./javascript/references.js";
 import {syntaxError} from "./javascript/syntaxError.js";
 import {Sourcemap} from "./sourcemap.js";
@@ -35,17 +36,24 @@ export interface Feature {
   name: string;
 }
 
-export interface Transpile {
+export interface BaseTranspile {
   id: string;
   inputs?: string[];
   outputs?: string[];
   inline?: boolean;
-  body: string;
   databases?: DatabaseReference[];
   files?: FileReference[];
   imports?: ImportReference[];
 }
 
+export interface PendingTranspile extends BaseTranspile {
+  body: () => Promise<string>;
+}
+
+export interface Transpile extends BaseTranspile {
+  body: string;
+}
+
 export interface ParseOptions {
   id: string;
   root: string;
@@ -56,7 +64,7 @@ export interface ParseOptions {
   verbose?: boolean;
 }
 
-export function transpileJavaScript(input: string, options: ParseOptions): Transpile {
+export function transpileJavaScript(input: string, options: ParseOptions): PendingTranspile {
   const {id, root, sourcePath, verbose = true} = options;
   try {
     const node = parseJavaScript(input, options);
@@ -67,44 +75,52 @@ export function transpileJavaScript(input: string, options: ParseOptions): Trans
       .filter((f) => f.type === "FileAttachment")
       .map(({name}) => fileReference(name, sourcePath));
     const inputs = Array.from(new Set<string>(node.references.map((r) => r.name)));
-    const output = new Sourcemap(input);
-    trim(output, input);
-    if (node.expression && !inputs.includes("display") && !inputs.includes("view")) {
-      output.insertLeft(0, "display((\n");
-      output.insertRight(input.length, "\n))");
-      inputs.push("display");
-    }
-    rewriteImports(output, node, sourcePath, createImportResolver(root, "_import"));
-    rewriteFetches(output, node, sourcePath);
+    const implicitDisplay = node.expression && !inputs.includes("display") && !inputs.includes("view");
+    if (implicitDisplay) inputs.push("display");
+    if (findImportDeclarations(node).length > 0) node.async = true;
     return {
       id,
       ...(inputs.length ? {inputs} : null),
       ...(options.inline ? {inline: true} : null),
       ...(node.declarations?.length ? {outputs: node.declarations.map(({name}) => name)} : null),
       ...(databases.length ? {databases: resolveDatabases(databases)} : null),
       ...(files.length ? {files} : null),
-      body: `${node.async ? "async " : ""}(${inputs}) => {
+      body: async () => {
+        const output = new Sourcemap(input);
+        trim(output, input);
+        if (implicitDisplay) {
+          output.insertLeft(0, "display((\n");
+          output.insertRight(input.length, "\n))");
+        }
+        await rewriteImports(output, node, sourcePath, createImportResolver(root, "_import"));
+        rewriteFetches(output, node, sourcePath);
+        const result = `${node.async ? "async " : ""}(${inputs}) => {
 ${String(output)}${node.declarations?.length ? `\nreturn {${node.declarations.map(({name}) => name)}};` : ""}
-}`,
+}`;
+        return result;
+      },
       ...(node.imports.length ? {imports: node.imports} : null)
     };
   } catch (error) {
     if (!(error instanceof SyntaxError)) throw error;
-    let message = error.message;
-    const match = /^(.+)\s\((\d+):(\d+)\)$/.exec(message);
-    if (match) {
-      const line = +match[2] + (options?.sourceLine ?? 0);
-      const column = +match[3] + 1;
-      message = `${match[1]} at line ${line}, column ${column}`;
-    } else if (options?.sourceLine) {
-      message = `${message} at line ${options.sourceLine + 1}`;
-    }
+    const message = error.message;
     // TODO: Consider showing a code snippet along with the error. Also, consider
     // whether we want to show the file name here.
-    if (verbose) console.error(red(`${error.name}: ${message}`));
+    if (verbose) {
+      let warning = error.message;
+      const match = /^(.+)\s\((\d+):(\d+)\)$/.exec(message);
+      if (match) {
+        const line = +match[2] + (options?.sourceLine ?? 0);
+        const column = +match[3] + 1;
+        warning = `${match[1]} at line ${line}, column ${column}`;
+      } else if (options?.sourceLine) {
+        warning = `${message} at line ${options.sourceLine + 1}`;
+      }
+      console.error(red(`${error.name}: ${warning}`));
+    }
     return {
       id: `${id}`,
-      body: `() => { throw new SyntaxError(${JSON.stringify(error.message)}); }`
+      body: async () => `() => { throw new SyntaxError(${JSON.stringify(message)}); }`
     };
   }
 }

src/javascript/imports.ts

@@ -145,10 +145,11 @@ export function parseLocalImports(root: string, paths: string[]): ImportsAndFetc
 }
 
 /** Rewrites import specifiers in the specified ES module source. */
-export function rewriteModule(input: string, sourcePath: string, resolver: ImportResolver): string {
+export async function rewriteModule(input: string, sourcePath: string, resolver: ImportResolver): Promise<string> {
   const body = Parser.parse(input, parseOptions) as Program;
   const references: Identifier[] = findReferences(body, defaultGlobals);
   const output = new Sourcemap(input);
+  const imports: (ImportDeclaration | ImportExpression | ExportAllDeclaration | ExportNamedDeclaration)[] = [];
 
   simple(body, {
     ImportDeclaration: rewriteImport,
@@ -161,59 +162,87 @@ export function rewriteModule(input: string, sourcePath: string, resolver: Impor
   });
 
   function rewriteImport(node: ImportDeclaration | ImportExpression | ExportAllDeclaration | ExportNamedDeclaration) {
+    imports.push(node);
+  }
+
+  for (const node of imports) {
     if (isStringLiteral(node.source)) {
       output.replaceLeft(
         node.source.start,
         node.source.end,
-        JSON.stringify(resolver(sourcePath, getStringLiteralValue(node.source)))
+        JSON.stringify(await resolver(sourcePath, getStringLiteralValue(node.source)))
       );
     }
   }
 
   return String(output);
 }
 
+export function findImportDeclarations(cell: JavaScriptNode): ImportDeclaration[] {
+  const declarations: ImportDeclaration[] = [];
+
+  simple(cell.body, {
+    ImportDeclaration(node) {
+      if (isStringLiteral(node.source)) {
+        declarations.push(node);
+      }
+    }
+  });
+
+  return declarations;
+}
+
 /**
  * Rewrites import specifiers in the specified JavaScript fenced code block or
  * inline expression. TODO parallelize multiple static imports.
  */
-export function rewriteImports(
+export async function rewriteImports(
   output: Sourcemap,
   cell: JavaScriptNode,
   sourcePath: string,
   resolver: ImportResolver
-): void {
+): Promise<void> {
+  const expressions: ImportExpression[] = [];
+  const declarations: ImportDeclaration[] = [];
+
   simple(cell.body, {
     ImportExpression(node) {
       if (isStringLiteral(node.source)) {
-        output.replaceLeft(
-          node.source.start,
-          node.source.end,
-          JSON.stringify(resolver(sourcePath, getStringLiteralValue(node.source)))
-        );
+        expressions.push(node);
       }
     },
     ImportDeclaration(node) {
       if (isStringLiteral(node.source)) {
-        cell.async = true;
-        output.replaceLeft(
-          node.start,
-          node.end,
-          `const ${
-            node.specifiers.some(isNotNamespaceSpecifier)
-              ? `{${node.specifiers.filter(isNotNamespaceSpecifier).map(rewriteImportSpecifier).join(", ")}}`
-              : node.specifiers.find(isNamespaceSpecifier)?.local.name ?? "{}"
-          } = await import(${JSON.stringify(resolver(sourcePath, getStringLiteralValue(node.source)))});`
-        );
+        declarations.push(node);
       }
     }
   });
+
+  for (const node of expressions) {
+    output.replaceLeft(
+      node.source.start,
+      node.source.end,
+      JSON.stringify(await resolver(sourcePath, getStringLiteralValue(node.source)))
+    );
+  }
+
+  for (const node of declarations) {
+    output.replaceLeft(
+      node.start,
+      node.end,
+      `const ${
+        node.specifiers.some(isNotNamespaceSpecifier)
+          ? `{${node.specifiers.filter(isNotNamespaceSpecifier).map(rewriteImportSpecifier).join(", ")}}`
+          : node.specifiers.find(isNamespaceSpecifier)?.local.name ?? "{}"
+      } = await import(${JSON.stringify(await resolver(sourcePath, getStringLiteralValue(node.source)))});`
+    );
+  }
 }
 
-export type ImportResolver = (path: string, specifier: string) => string;
+export type ImportResolver = (path: string, specifier: string) => Promise<string>;
 
 export function createImportResolver(root: string, base: "." | "_import" = "."): ImportResolver {
-  return (path, specifier) => {
+  return async (path, specifier) => {
     return isLocalImport(specifier, path)
       ? relativeUrl(path, resolvePath(base, path, resolveImportHash(root, path, specifier)))
       : specifier === "npm:@observablehq/runtime"
@@ -233,11 +262,53 @@ export function createImportResolver(root: string, base: "." | "_import" = "."):
       : specifier === "npm:@observablehq/xslx"
       ? resolveBuiltin(base, path, "stdlib/xslx.js") // TODO publish to npm
       : specifier.startsWith("npm:")
-      ? `https://cdn.jsdelivr.net/npm/${specifier.slice("npm:".length)}/+esm`
+      ? await resolveNpmImport(specifier.slice("npm:".length))
       : specifier;
   };
 }
 
+// Like import, don’t fetch the same package more than once to ensure
+// consistency; restart the server if you want to clear the cache.
+const npmCache = new Map<string, Promise<string>>();
+
+function parseNpmSpecifier(specifier: string): {name: string; range?: string; path?: string} {
+  const parts = specifier.split("/");
+  const namerange = specifier.startsWith("@") ? [parts.shift()!, parts.shift()!].join("/") : parts.shift()!;
+  const ranged = namerange.indexOf("@", 1);
+  return {
+    name: ranged > 0 ? namerange.slice(0, ranged) : namerange,
+    range: ranged > 0 ? namerange.slice(ranged + 1) : undefined,
+    path: parts.length > 0 ? parts.join("/") : undefined
+  };
+}
+
+function formatNpmSpecifier({name, range, path}: {name: string; range?: string; path?: string}): string {
+  return `${name}${range ? `@${range}` : ""}${path ? `/${path}` : ""}`;
+}
+
+async function resolveNpmVersion(specifier: string): Promise<string> {
+  const {name, range} = parseNpmSpecifier(specifier); // ignore path
+  specifier = formatNpmSpecifier({name, range});
+  let promise = npmCache.get(specifier);
+  if (promise) return promise;
+  promise = (async () => {
+    const search = range ? `?specifier=${range}` : "";
+    const response = await fetch(`https://data.jsdelivr.com/v1/packages/npm/${name}/resolved${search}`);
+    if (!response.ok) throw new Error(`unable to resolve npm specifier: ${name}`);
+    const body = await response.json();
+    return body.version;
+  })();
+  promise.catch(() => npmCache.delete(specifier)); // try again on error
+  npmCache.set(specifier, promise);
+  return promise;
+}
+
+export async function resolveNpmImport(specifier: string): Promise<string> {
+  const {name, path = "+esm"} = parseNpmSpecifier(specifier);
+  const version = await resolveNpmVersion(specifier);
+  return `https://cdn.jsdelivr.net/npm/${name}@${version}/${path}`;
+}
+
 function resolveBuiltin(base: "." | "_import", path: string, specifier: string): string {
   return relativeUrl(join(base === "." ? "_import" : ".", path), join("_observablehq", specifier));
 }

src/libraries.ts

@@ -1,3 +1,5 @@
+import {resolveNpmImport} from "./javascript/imports.js";
+
 export function getImplicitSpecifiers(inputs: Set<string>): Set<string> {
   return addImplicitSpecifiers(new Set(), inputs);
 }
@@ -21,13 +23,13 @@ export function addImplicitSpecifiers(specifiers: Set<string>, inputs: Set<strin
   return specifiers;
 }
 
-export function getImplicitStylesheets(specifiers: Set<string>): Set<string> {
+export async function getImplicitStylesheets(specifiers: Set<string>): Promise<Set<string>> {
   return addImplicitStylesheets(new Set(), specifiers);
 }
 
-export function addImplicitStylesheets(stylesheets: Set<string>, specifiers: Set<string>): typeof stylesheets {
-  if (specifiers.has("npm:katex")) stylesheets.add("https://cdn.jsdelivr.net/npm/katex/dist/katex.min.css");
+export async function addImplicitStylesheets(stylesheets: Set<string>, specifiers: Set<string>): Promise<Set<string>> {
   if (specifiers.has("npm:@observablehq/inputs")) stylesheets.add("https://cdn.jsdelivr.net/gh/observablehq/inputs/src/style.css"); // prettier-ignore
-  if (specifiers.has("npm:leaflet")) stylesheets.add("https://cdn.jsdelivr.net/npm/leaflet/dist/leaflet.css");
+  if (specifiers.has("npm:katex")) stylesheets.add(await resolveNpmImport("katex/dist/katex.min.css"));
+  if (specifiers.has("npm:leaflet")) stylesheets.add(await resolveNpmImport("leaflet/dist/leaflet.css"));
   return stylesheets;
 }