v8.5 refactor: consolidate TOC file handling in cloud document scripts (#21512)

scripts/concatMdByToc.js

@@ -126,7 +126,7 @@ const main = () => {
   let mergedStr = "";
 
   fileList.forEach((filePath) => {
-    mergedStr += `${handleSingleMd(`.${filePath}`)}\n\n`;
+    mergedStr += `${handleSingleMd(`./${filePath}`)}\n\n`;
   });
 
   const variables = JSON.parse(

scripts/filterCloudDoc.js

@@ -1,5 +1,5 @@
 import {
-  getAllMdList,
+  getAllCloudMdList,
   copySingleFileSync,
   copyFileWithCustomContentSync,
   copyDirectoryWithCustomContentSync,
@@ -25,9 +25,9 @@ const extractFilefromList = (
 };
 
 const main = () => {
-  const filteredLinkList = getAllMdList("TOC-tidb-cloud.md");
+  const allFilePaths = getAllCloudMdList();
 
-  extractFilefromList(filteredLinkList, ".", "./tmp");
+  extractFilefromList(allFilePaths, "./", "./tmp");
   copySingleFileSync("TOC-tidb-cloud.md", "./tmp/TOC.md");
   copyDirectoryWithCustomContentSync(
     "./tidb-cloud/",

scripts/filterCloudInitFiles.js

@@ -0,0 +1,69 @@
+import * as fs from "fs";
+import path from "path";
+import { getAllCloudMdList } from "./utils.js";
+
+const allFilePaths = getAllCloudMdList();
+
+// Set to store filtered file paths
+const filePaths = new Set();
+
+// Filter the file paths
+for (const filePath of allFilePaths) {
+  // Skip external links (starting with http/https)
+  if (filePath.startsWith("http")) {
+    continue;
+  }
+
+  // Skip anchor links (starting with #)
+  if (filePath.startsWith("#")) {
+    continue;
+  }
+
+  // Skip files in tidb-cloud folder
+  if (cleanPath.startsWith("tidb-cloud/")) {
+    continue;
+  }
+
+  filePaths.add(cleanPath);
+}
+
+// Create tmp directory if it doesn't exist
+const tmpDir = "tmp";
+if (!fs.existsSync(tmpDir)) {
+  fs.mkdirSync(tmpDir, { recursive: true });
+}
+
+// Copy files to tmp directory
+let copiedCount = 0;
+let skippedCount = 0;
+
+for (const filePath of filePaths) {
+  const sourcePath = filePath;
+  const targetPath = path.join(tmpDir, filePath);
+
+  // Create target directory if it doesn't exist
+  const targetDir = path.dirname(targetPath);
+  if (!fs.existsSync(targetDir)) {
+    fs.mkdirSync(targetDir, { recursive: true });
+  }
+
+  // Check if source file exists
+  if (fs.existsSync(sourcePath)) {
+    try {
+      fs.copyFileSync(sourcePath, targetPath);
+      console.log(`✓ Copied: ${filePath}`);
+      copiedCount++;
+    } catch (error) {
+      console.error(`✗ Error copying ${filePath}: ${error.message}`);
+    }
+  } else {
+    console.log(`⚠ Skipped (not found): ${filePath}`);
+    skippedCount++;
+  }
+}
+
+console.log(`\nSummary:`);
+console.log(`- Total files referenced: ${filePaths.size}`);
+console.log(`- Files copied: ${copiedCount}`);
+console.log(`- Files skipped: ${skippedCount}`);
+console.log(`- Files copied to: ${tmpDir}/`);

scripts/filterNonCloudDoc.js

@@ -3,6 +3,7 @@ import {
   copySingleFileSync,
   copyFileWithCustomContentSync,
   removeCustomContent,
+  CLOUD_TOC_LIST,
 } from "./utils.js";
 
 const contentHandler = (content = "") => {
@@ -17,7 +18,7 @@ const extractFilefromList = (
   fileList.forEach((filePath = "") => {
     if (
       filePath.includes(`/tidb-cloud/`) ||
-      filePath.includes(`TOC-tidb-cloud.md`)
+      CLOUD_TOC_LIST.some((tocFile) => filePath.includes(tocFile))
     ) {
       return;
     }

scripts/filterUpdateFiles.js

@@ -2,6 +2,7 @@ import * as fs from "fs";
 import path from "path";
 import axios from "axios";
 import { Octokit } from "octokit";
+import { CLOUD_TOC_LIST, getAllCloudMdList } from "./utils.js";
 
 const GH_TOKEN = process.env.GH_TOKEN || "";
 
@@ -92,68 +93,29 @@ const deleteFile = (targetFile) => {
   }
 };
 
-// read toc file and parse the file paths
-const parseTOCFile = (tocPath) => {
-  try {
-    if (!fs.existsSync(tocPath)) {
-      console.log(`TOC file not found: ${tocPath}`);
-      return new Set();
-    }
-
-    const content = fs.readFileSync(tocPath, "utf8");
-    const filePaths = new Set();
-
-    // use regex to match the file paths in markdown links
-    // match [text](path) format
-    const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
-    let match;
-
-    while ((match = linkRegex.exec(content)) !== null) {
-      const link = match[2];
-      // only process links ending with .md
-      if (link.endsWith(".md")) {
-        // remove ./ or / at the beginning to ensure path consistency
-        const normalizedPath = link.replace(/^\.?\//, "");
-        filePaths.add(normalizedPath);
-      }
-    }
-
-    console.log(`Found ${filePaths.size} files in TOC: ${tocPath}`);
-    if (filePaths.size > 0) {
-      console.log(
-        "Files in TOC:",
-        Array.from(filePaths).slice(0, 5).join(", "),
-        filePaths.size > 5 ? `... and ${filePaths.size - 5} more` : ""
-      );
-    }
-    return filePaths;
-  } catch (error) {
-    console.error(`Error parsing TOC file ${tocPath}:`, error);
-    return new Set();
-  }
-};
-
 // get the file list from the toc file
 const getCloudTOCFiles = () => {
-  // check ./tmp/TOC-tidb-cloud.md first
-  const tmpTocPath = "./tmp/TOC-tidb-cloud.md";
-  const localTocPath = "TOC-tidb-cloud.md";
+  const tmpTocFiles = getAllCloudMdList([
+    "./tmp/TOC-tidb-cloud.md",
+    "./tmp/TOC-tidb-cloud-starter.md",
+    "./tmp/TOC-tidb-cloud-essential.md",
+  ]);
+  const tocFiles = getAllCloudMdList(CLOUD_TOC_LIST);
 
-  let tocFiles = parseTOCFile(tmpTocPath);
+  // Convert to Set
+  const tmpTocFilesSet = new Set(tmpTocFiles);
+  const tocFilesSet = new Set(tocFiles);
 
-  // if not found in /tmp, check the current directory
-  if (tocFiles.size === 0) {
-    console.log(`No files found in ${tmpTocPath}, trying ${localTocPath}`);
-    tocFiles = parseTOCFile(localTocPath);
-  }
+  // Use tmpTocFiles if not empty, otherwise use tocFiles
+  const finalTocFiles = tmpTocFilesSet.size > 0 ? tmpTocFilesSet : tocFilesSet;
 
-  if (tocFiles.size === 0) {
+  if (finalTocFiles.size === 0) {
     console.log(
       "Warning: No TOC file found or no files in TOC. All .md files will be processed."
     );
   }
 
-  return tocFiles;
+  return finalTocFiles;
 };
 
 // filter the files in tmp directory by the toc file

scripts/utils.js

@@ -59,13 +59,34 @@ const filterLink = (srcList = []) => {
 };
 
 export const getAllMdList = (tocFile) => {
+  if (!fs.existsSync(tocFile)) {
+    console.log(`TOC file not found: ${tocFile}`);
+    return [];
+  }
+
   const tocFileContent = fs.readFileSync(tocFile);
   const mdAst = generateMdAstFromFile(tocFileContent);
   const linkList = extractLinkNodeFromAst(mdAst);
-  const filteredLinkList = filterLink(linkList);
+  const filteredLinkList = filterLink(linkList).map((link) =>
+    link.replace(/^\.?\//, "")
+  );
   return filteredLinkList;
 };
 
+export const CLOUD_TOC_LIST = [
+  "TOC-tidb-cloud.md",
+  "TOC-tidb-cloud-essential.md",
+  "TOC-tidb-cloud-starter.md",
+];
+
+export const getAllCloudMdList = (tocFiles = CLOUD_TOC_LIST) => {
+  // Get all MD files from multiple TOCs and deduplicate
+  const allFilteredLinkLists = tocFiles.map((tocFile) => getAllMdList(tocFile));
+  const flattenedList = allFilteredLinkLists.flat();
+  const allFilePaths = [...new Set(flattenedList)]; // Deduplicate
+  return allFilePaths;
+};
+
 const checkDestDir = (destPath) => {
   const dir = path.dirname(destPath);