Give plugins access to notion url --> docusaurus url conversion

Author: hatton

.vscode/settings.json

@@ -7,6 +7,7 @@
     "callouts",
     "DOCU_NOTION",
     "docu-notion",
+    "docunotion",
     "Greenshot",
     "imgur",
     "kanban",
@@ -17,4 +18,4 @@
     "statusBar.noFolderBackground": "#d649ca",
     "statussBar.prominentBackground": "#d649ca"
   }
-}
+}

src/plugins/internalLinks.ts

@@ -2,6 +2,40 @@ import { IDocuNotionContext, IPlugin } from "./pluginTypes";
 import { error, warning } from "../log";
 import { NotionPage } from "../NotionPage";
 
+// converts a url to a local link, if it is a link to a page in the Notion site
+// only here for plugins, notion won't normally be giving us raw urls (at least not that I've noticed)
+// If it finds a URL but can't find the page it points to, it will return undefined.
+// If it doesn't find a match at all, it returns undefined.
+export function convertInternalUrl(
+  context: IDocuNotionContext,
+  url: string
+): string | undefined {
+  const kGetIDFromNotionURL = /https:\/\/www\.notion\.so\S+-([a-z,0-9]+)+.*/;
+  const match = kGetIDFromNotionURL.exec(url);
+  if (match === null) {
+    warning(
+      `[standardInternalLinkConversion] Could not parse link ${url} as a Notion URL`
+    );
+    return undefined;
+  }
+  const id = match[1];
+  const pages = context.pages;
+  // find the page where pageId matches hrefFromNotion
+  const targetPage = pages.find(p => {
+    return p.matchesLinkId(id);
+  });
+
+  if (!targetPage) {
+    // About this situation. See https://github.com/sillsdev/docu-notion/issues/9
+    warning(
+      `[standardInternalLinkConversion] Could not find the target of this link. Note that links to outline sections are not supported. ${url}. https://github.com/sillsdev/docu-notion/issues/9`
+    );
+    return undefined;
+  }
+  return convertLinkHref(context, targetPage, url);
+}
+
+// handles the whole markdown link, including the label
 function convertInternalLink(
   context: IDocuNotionContext,
   markdownLink: string

src/plugins/pluginTestRun.ts

@@ -1,14 +1,13 @@
 import { Client } from "@notionhq/client";
 import { GetPageResponse } from "@notionhq/client/build/src/api-endpoints";
-import { info } from "console";
 import { NotionToMarkdown } from "notion-to-md";
 import { IDocuNotionContext } from "./pluginTypes";
 import { HierarchicalNamedLayoutStrategy } from "../HierarchicalNamedLayoutStrategy";
-import { error, logDebug, verbose, warning } from "../log";
 import { NotionPage } from "../NotionPage";
 import { getMarkdownFromNotionBlocks } from "../transform";
 import { IDocuNotionConfig } from "../config/configuration";
 import { NotionBlock } from "../types";
+import { convertInternalUrl } from "./internalLinks";
 
 export async function blocksToMarkdown(
   config: IDocuNotionConfig,
@@ -34,6 +33,10 @@ export async function blocksToMarkdown(
         resolve([]);
       });
     },
+    convertNotionLinkToLocalDocusaurusLink: (url: string) => {
+      return convertInternalUrl(docunotionContext, url);
+    },
+
     //TODO might be needed for some tests, e.g. the image transformer...
     directoryContainingMarkdown: "not yet",
     relativeFilePathToFolderContainingPage: "not yet",

src/plugins/pluginTypes.ts

@@ -46,7 +46,12 @@ export type IRegexMarkdownModification = {
   // Based on that regex, the outputPattern will be used to replace the matched text
   replacementPattern?: string;
   // Instead of a pattern, you can use this if you have to ask a server somewhere for help in getting the new markdown
-  getReplacement?(s: string): Promise<string>;
+  getReplacement?(
+    context: IDocuNotionContext,
+    match: RegExpExecArray
+  ): Promise<string>;
+  // normally, anything in code blocks is will be ignored. If you want to make changes inside of code blocks, set this to true.
+  includeCodeBlocks?: boolean;
 
   // If the output is creating things like react elements, you can import their definitions here
   imports?: string[];
@@ -66,6 +71,7 @@ export type IDocuNotionContext = {
   notionToMarkdown: NotionToMarkdown;
   directoryContainingMarkdown: string;
   relativeFilePathToFolderContainingPage: string;
+  convertNotionLinkToLocalDocusaurusLink: (url: string) => string;
   pages: NotionPage[];
   counts: ICounts;
 };

src/pull.ts

@@ -28,6 +28,7 @@ import { Client, isFullBlock } from "@notionhq/client";
 import { exit } from "process";
 import { IDocuNotionConfig, loadConfigAsync } from "./config/configuration";
 import { NotionBlock } from "./types";
+import { convertInternalUrl } from "./plugins/internalLinks";
 
 export type DocuNotionOptions = {
   notionToken: string;
@@ -110,6 +111,8 @@ async function outputPages(
     options: options,
     pages: pages,
     counts: counts, // review will this get copied or pointed to?
+    convertNotionLinkToLocalDocusaurusLink: (url: string) =>
+      convertInternalUrl(context, url),
   };
   for (const page of pages) {
     layoutStrategy.pageWasSeen(page);

src/transform.ts

@@ -56,7 +56,11 @@ export async function getMarkdownFromNotionBlocks(
   //console.log("markdown after link fixes", markdown);
 
   // simple regex-based tweaks. These are usually related to docusaurus
-  const { imports, body } = await doTransformsOnMarkdown(config, markdown);
+  const { imports, body } = await doTransformsOnMarkdown(
+    context,
+    config,
+    markdown
+  );
 
   // console.log("markdown after regex fixes", markdown);
   // console.log("body after regex", body);
@@ -82,6 +86,7 @@ function doNotionBlockTransforms(
 }
 
 async function doTransformsOnMarkdown(
+  context: IDocuNotionContext,
   config: IDocuNotionConfig,
   input: string
 ) {
@@ -109,15 +114,24 @@ async function doTransformsOnMarkdown(
     // regex.exec is stateful, so we don't want to mess up the plugin's use of its own regex, so we clone it.
     // we also add the "g" flag to make sure we get all matches
     const regex = new RegExp(`${codeBlocks.source}|(${mod.regex.source})`, "g");
-    let count = 0;
     while ((match = regex.exec(input)) !== null) {
       if (match[0]) {
         const original = match[0];
-        if (original.startsWith("```") && original.endsWith("```")) {
-          continue; // code block
+        if (
+          original.startsWith("```") &&
+          original.endsWith("```") &&
+          !mod.includeCodeBlocks
+        ) {
+          continue; // code block, and they didn't say to include them
         }
         if (mod.getReplacement) {
-          replacement = await mod.getReplacement(original);
+          // our match here has an extra group, which is an implementation detail
+          // that shouldn't be made visible to the plugin
+          const matchAsThePluginWouldExpectIt = mod.regex.exec(match[0])!;
+          replacement = await mod.getReplacement(
+            context,
+            matchAsThePluginWouldExpectIt
+          );
         } else if (mod.replacementPattern) {
           console.log(`mod.replacementPattern.replace("$1", ${match[2]}`);
           replacement = mod.replacementPattern.replace("$1", match[2]);
@@ -149,7 +163,8 @@ async function doNotionToMarkdown(
     blocks
   );
 
-  let markdown = docunotionContext.notionToMarkdown.toMarkdownString(mdBlocks);
+  const markdown =
+    docunotionContext.notionToMarkdown.toMarkdownString(mdBlocks);
   return markdown;
 }
 
@@ -187,6 +202,7 @@ function doLinkFixes(
     config.plugins.some(plugin => {
       if (!plugin.linkModifier) return false;
       if (plugin.linkModifier.match.exec(originalLinkMarkdown) === null) {
+        verbose(`plugin "${plugin.name}" did not match this url`);
         return false;
       }
       const newMarkdown = plugin.linkModifier.convert(