Refactor to a 2-pass system

Author: hatton

.eslintrc.js

@@ -27,6 +27,6 @@ module.exports = {
     "@typescript-eslint/ban-ts-comment": "off",
     "@typescript-eslint/no-explicit-any": "off",
     "@typescript/explicit-module-boundary-types": "off",
-    "@typescript/no-unsafe-call": "off",
+    "@typescript-eslint/no-unsafe-call": "off",
   },
 };

src/FlatGuidLayoutStrategy.ts

@@ -1,4 +1,5 @@
 import { LayoutStrategy } from "./LayoutStrategy";
+import { NotionPage } from "./NotionPage";
 
 // This strategy creates a flat list of files that have notion-id for file names.
 // Pros: the urls will never change so long as the notion pages are not delete and re-recreated.
@@ -11,19 +12,18 @@ import { LayoutStrategy } from "./LayoutStrategy";
 // the directory/file structure itself is no longer representative of the outline we want.
 export class FlatGuidLayoutStrategy extends LayoutStrategy {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  public newLevel(context: string, _levelLabel: string): string {
+  public newLevel(
+    rootDir: string,
+    context: string,
+    _levelLabel: string
+  ): string {
     // In this strategy, we ignore context and don't create any directories to match the levels.
     // Just return the following for the benefit of logging.
     return context + "/" + _levelLabel;
   }
 
-  public getPathForPage(
-    _context: string,
-    pageId: string,
-    _title: string,
-    extensionWithDot: string
-  ): string {
+  public getPathForPage(page: NotionPage, extensionWithDot: string): string {
     // In this strategy, we don't care about the location or the title
-    return this.rootDirectory + "/" + pageId + extensionWithDot;
+    return this.rootDirectory + "/" + page.pageId + extensionWithDot;
   }
 }

src/HierarchicalNamedLayoutStrategy.ts

@@ -1,6 +1,7 @@
 import * as fs from "fs-extra";
 import sanitize from "sanitize-filename";
 import { LayoutStrategy } from "./LayoutStrategy";
+import { NotionPage } from "./NotionPage";
 
 // This strategy gives us a file tree that mirrors that of notion.
 // Each level in the outline becomes a directory, and each file bears the name of the Notion document.
@@ -13,25 +14,31 @@ import { LayoutStrategy } from "./LayoutStrategy";
 // doesn't buy us much... it would give protection against name changes, but not changes to the outline structure.
 
 export class HierarchicalNamedLayoutStrategy extends LayoutStrategy {
-  public newLevel(context: string, levelLabel: string): string {
+  public newLevel(
+    dirRoot: string,
+    context: string,
+    levelLabel: string
+  ): string {
     const path = context + "/" + sanitize(levelLabel);
-    fs.mkdirSync(path, { recursive: true });
+
+    //console.log("Creating level " + path);
+    fs.mkdirSync(dirRoot + "/" + path, { recursive: true });
     return path;
   }
 
-  public getPathForPage(
-    context: string,
-    pageId: string,
-    title: string,
-    extensionWithDot: string
-  ): string {
+  public getPathForPage(page: NotionPage, extensionWithDot: string): string {
     let path =
-      this.rootDirectory + "/" + context + sanitize(title) + extensionWithDot;
+      this.rootDirectory +
+      "/" +
+      page.context +
+      "/" +
+      sanitize(page.nameOrTitle) +
+      extensionWithDot;
 
     path = path.replace("//", "/");
-    console.log(
-      `getPathForPage(${context}, ${pageId}, ${title}) with  root ${this.rootDirectory} --> ${path}`
-    );
+    // console.log(
+    //   `getPathForPage(${context}, ${pageId}, ${title}) with  root ${this.rootDirectory} --> ${path}`
+    // );
     return path;
   }
 }

src/LayoutStrategy.ts

@@ -1,4 +1,5 @@
 import * as fs from "fs-extra";
+import { NotionPage } from "./NotionPage";
 
 // Here a fuller name would be File Tree Layout Strategy. That is,
 // as we walk the Notion outline and create files, where do we create them, what do we name them, etc.
@@ -20,16 +21,18 @@ export abstract class LayoutStrategy {
     }
   }
 
-  public abstract newLevel(context: string, levelLabel: string): string;
+  public abstract newLevel(
+    rootDir: string,
+    ontext: string,
+    levelLabel: string
+  ): string;
   public abstract getPathForPage(
-    context: string,
-    pageId: string,
-    title: string,
+    page: NotionPage,
     extensionWithDot: string
   ): string;
 
-  public pageWasSeen(context: string, pageId: string, title: string): void {
-    const path = this.getPathForPage(context, pageId, title, ".md");
+  public pageWasSeen(page: NotionPage): void {
+    const path = this.getPathForPage(page, ".md");
     this.existingPagesNotSeenYetInPull =
       this.existingPagesNotSeenYetInPull.filter(p => p !== path);
   }

src/NotionImage.ts

@@ -0,0 +1,101 @@
+import * as fs from "fs-extra";
+import FileType from "file-type";
+import fetch from "node-fetch";
+
+let existingImagesNotSeenYetInPull: string[] = [];
+let imageOutputPath = "not set yet";
+let imagePrefix = "not set yet";
+
+export async function initImageHandling(
+  prefix: string,
+  outputPath: string
+): Promise<void> {
+  // If they gave us a trailing slash, remove it because we add it back later.
+  // Note that it's up to the caller to have a *leading* slash or not.
+  imagePrefix = prefix.replace(/\/$/, "");
+  imageOutputPath = outputPath;
+
+  console.log("initimg " + imagePrefix + ", " + imageOutputPath);
+  // Currently we don't delete the image directory, because if an image
+  // changes, it gets a new id. This way can then prevent downloading
+  // and image after the 1st time. The downside is currently we don't
+  // have the smarts to remove unused images.
+  await fs.mkdir(imageOutputPath, { recursive: true });
+}
+
+async function saveImage(
+  url: string,
+  imageFolderPath: string
+): Promise<string> {
+  const response = await fetch(url);
+  const arrayBuffer = await response.arrayBuffer();
+  const buffer = Buffer.from(arrayBuffer);
+  const fileType = await FileType.fromBuffer(buffer);
+  if (fileType?.ext) {
+    // Since most images come from pasting screenshots, there isn't normally a filename. That's fine, we just make a hash of the url
+    // Images that are stored by notion come to us with a complex url that changes over time, so we pick out the UUID that doesn't change. Example:
+    //    https://s3.us-west-2.amazonaws.com/secure.notion-static.com/d1058f46-4d2f-4292-8388-4ad393383439/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20220516%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20220516T233630Z&X-Amz-Expires=3600&X-Amz-Signature=f215704094fcc884d37073b0b108cf6d1c9da9b7d57a898da38bc30c30b4c4b5&X-Amz-SignedHeaders=host&x-id=GetObject
+
+    let thingToHash = url;
+    const m = /.*secure\.notion-static\.com\/(.*)\//gm.exec(url);
+    if (m && m.length > 1) {
+      thingToHash = m[1];
+    }
+
+    const hash = hashOfString(thingToHash);
+    const outputFileName = `${hash}.${fileType.ext}`;
+    const path = imageFolderPath + "/" + outputFileName;
+    imageWasSeen(path);
+    if (!fs.pathExistsSync(path)) {
+      // // I think that this ok that this is writing async as we continue
+      console.log("Adding image " + path);
+      fs.createWriteStream(path).write(buffer);
+    }
+    return outputFileName;
+  } else {
+    console.error(
+      `Something wrong with the filetype extension on the blob we got from ${url}`
+    );
+    return "error";
+  }
+}
+function hashOfString(s: string) {
+  let hash = 0;
+  for (let i = 0; i < s.length; ++i)
+    hash = Math.imul(31, hash) + s.charCodeAt(i);
+
+  return Math.abs(hash);
+}
+
+// Download the image if we don't have it, give it a good name, and
+// change the src to point to our copy of the image.
+export async function processImageBlock(b: any): Promise<void> {
+  let url = "";
+  if ("file" in b.image) {
+    url = b.image.file.url; // image saved on notion (actually AWS)
+  } else {
+    url = b.image.external.url; // image still pointing somewhere else. I've see this happen when copying a Google Doc into Notion. Notion kep pointing at the google doc.
+  }
+
+  const newPath = imagePrefix + "/" + (await saveImage(url, imageOutputPath));
+
+  // change the src to point to our copy of the image
+  if ("file" in b.image) {
+    b.image.file.url = newPath;
+  } else {
+    b.image.external.url = newPath;
+  }
+}
+
+function imageWasSeen(path: string) {
+  existingImagesNotSeenYetInPull = existingImagesNotSeenYetInPull.filter(
+    p => p !== path
+  );
+}
+
+export async function cleanupOldImages(): Promise<void> {
+  for (const p of existingImagesNotSeenYetInPull) {
+    console.log(`Removing old image: ${p}`);
+    await fs.rm(p);
+  }
+}