build uses a stale cache (#296)

* build uses a stale cache

* don't crash on it.only

* options object

Author: Fil

src/build.ts

@@ -88,7 +88,7 @@ export async function build(
     let sourcePath = join(root, file);
     const outputPath = join("_file", file);
     if (!existsSync(sourcePath)) {
-      const loader = Loader.find(root, file);
+      const loader = Loader.find(root, file, {useStale: true});
       if (!loader) {
         effects.logger.error("missing referenced file", sourcePath);
         continue;

src/dataloader.ts

@@ -35,6 +35,7 @@ export interface LoaderOptions {
   path: string;
   sourceRoot: string;
   targetPath: string;
+  useStale: boolean;
 }
 
 export abstract class Loader {
@@ -57,10 +58,16 @@ export abstract class Loader {
    */
   readonly targetPath: string;
 
-  constructor({path, sourceRoot, targetPath}: LoaderOptions) {
+  /**
+   * Should the loader use a stale cache. true when building.
+   */
+  readonly useStale?: boolean;
+
+  constructor({path, sourceRoot, targetPath, useStale}: LoaderOptions) {
     this.path = path;
     this.sourceRoot = sourceRoot;
     this.targetPath = targetPath;
+    this.useStale = useStale;
   }
 
   /**
@@ -70,8 +77,8 @@ export abstract class Loader {
    * abort if we find a matching folder or reach the source root; for example,
    * if docs/data exists, we won’t look for a docs/data.zip.
    */
-  static find(sourceRoot: string, targetPath: string): Loader | undefined {
-    const exact = this.findExact(sourceRoot, targetPath);
+  static find(sourceRoot: string, targetPath: string, {useStale = false} = {}): Loader | undefined {
+    const exact = this.findExact(sourceRoot, targetPath, {useStale});
     if (exact) return exact;
     let dir = dirname(targetPath);
     for (let parent: string; true; dir = parent) {
@@ -88,23 +95,25 @@ export abstract class Loader {
           inflatePath: targetPath.slice(archive.length - ext.length + 1),
           path: join(sourceRoot, archive),
           sourceRoot,
-          targetPath
+          targetPath,
+          useStale
         });
       }
-      const archiveLoader = this.findExact(sourceRoot, archive);
+      const archiveLoader = this.findExact(sourceRoot, archive, {useStale});
       if (archiveLoader) {
         return new Extractor({
           preload: async (options) => archiveLoader.load(options),
           inflatePath: targetPath.slice(archive.length - ext.length + 1),
           path: archiveLoader.path,
           sourceRoot,
-          targetPath
+          targetPath,
+          useStale
         });
       }
     }
   }
 
-  private static findExact(sourceRoot: string, targetPath: string): Loader | undefined {
+  private static findExact(sourceRoot: string, targetPath: string, {useStale}): Loader | undefined {
     for (const [ext, [command, ...args]] of Object.entries(languages)) {
       if (!existsSync(join(sourceRoot, targetPath + ext))) continue;
       if (extname(targetPath) === "") {
@@ -117,7 +126,8 @@ export abstract class Loader {
         args: command == null ? args : [...args, path],
         path,
         sourceRoot,
-        targetPath
+        targetPath,
+        useStale
       });
     }
   }
@@ -127,6 +137,7 @@ export abstract class Loader {
    * to the source root; this is within the .observablehq/cache folder within
    * the source root.
    */
+
   async load(effects = defaultEffects): Promise<string> {
     const key = join(this.sourceRoot, this.targetPath);
     let command = runningCommands.get(key);
@@ -137,8 +148,10 @@ export abstract class Loader {
         const loaderStat = await maybeStat(this.path);
         const cacheStat = await maybeStat(cachePath);
         if (!cacheStat) effects.output.write(faint("[missing] "));
-        else if (cacheStat.mtimeMs < loaderStat!.mtimeMs) effects.output.write(faint("[stale] "));
-        else return effects.output.write(faint("[fresh] ")), outputPath;
+        else if (cacheStat.mtimeMs < loaderStat!.mtimeMs) {
+          if (this.useStale) return effects.output.write(faint("[using stale] ")), outputPath;
+          else effects.output.write(faint("[stale] "));
+        } else return effects.output.write(faint("[fresh] ")), outputPath;
         const tempPath = join(this.sourceRoot, ".observablehq", "cache", `${this.targetPath}.${process.pid}`);
         await prepareOutput(tempPath);
         const tempFd = await open(tempPath, "w");

test/dataloaders-test.ts

@@ -1,5 +1,5 @@
 import assert from "node:assert";
-import {readFile} from "node:fs/promises";
+import {readFile, stat, unlink, utimes} from "node:fs/promises";
 import {type LoadEffects, Loader} from "../src/dataloader.js";
 
 const noopEffects: LoadEffects = {
@@ -40,3 +40,56 @@ describe("data loaders are called with the appropriate command", () => {
     assert.strictEqual(await readFile("test/" + out, "utf-8"), "Rscript\n");
   });
 });
+
+describe("data loaders optionally use a stale cache", () => {
+  it("a dataloader can use ", async () => {
+    const out = [] as string[];
+    const outputEffects: LoadEffects = {
+      logger: {log() {}, warn() {}, error() {}},
+      output: {
+        write(a) {
+          out.push(a);
+        }
+      }
+    };
+    const loader = Loader.find("test", "dataloaders/data1.txt")!;
+    // save the loader times.
+    const {atime, mtime} = await stat(loader.path);
+    // set the loader mtime to Dec. 1st, 2023.
+    const time = Date.UTC(2023, 11, 1) / 1000;
+    await utimes(loader.path, atime, time);
+    // remove the cache set by another test (unless we it.only this test).
+    try {
+      await unlink("test/.observablehq/cache/dataloaders/data1.txt");
+    } catch {
+      // ignore;
+    }
+    // populate the cache (missing)
+    await loader.load(outputEffects);
+    // run again (fresh)
+    await loader.load(outputEffects);
+    // touch the loader
+    await utimes(loader.path, atime, Date.now() + 100);
+    // run it with useStale=true (using stale)
+    const loader2 = Loader.find("test", "dataloaders/data1.txt", {useStale: true})!;
+    await loader2.load(outputEffects);
+    // run it with useStale=false (stale)
+    await loader.load(outputEffects);
+    // revert the loader to its original mtime
+    await utimes(loader.path, atime, mtime);
+    assert.deepStrictEqual(
+      // eslint-disable-next-line no-control-regex
+      out.map((l) => l.replaceAll(/\x1b\[[0-9]+m/g, "")),
+      [
+        "load test/dataloaders/data1.txt.js → ",
+        "[missing] ",
+        "load test/dataloaders/data1.txt.js → ",
+        "[fresh] ",
+        "load test/dataloaders/data1.txt.js → ",
+        "[using stale] ",
+        "load test/dataloaders/data1.txt.js → ",
+        "[stale] "
+      ]
+    );
+  });
+});