add new options to rest schema script (github#21571)

Author: rachmari

script/rest/update-files.js

@@ -6,45 +6,72 @@
 //
 // [end-readme]
 
-import fs from 'fs'
+import { stat, readFile, writeFile, readdir } from 'fs/promises'
 import path from 'path'
 import program from 'commander'
 import { execSync } from 'child_process'
 import mkdirp from 'mkdirp'
 import rimraf from 'rimraf'
 import getOperations from './utils/get-operations.js'
+import yaml from 'js-yaml'
 
 const tempDocsDir = path.join(process.cwd(), 'openapiTmp')
 const githubRepoDir = path.join(process.cwd(), '../github')
 const dereferencedPath = path.join(process.cwd(), 'lib/rest/static/dereferenced')
-const schemas = fs.readdirSync(dereferencedPath)
 const decoratedPath = path.join(process.cwd(), 'lib/rest/static/decorated')
+const openApiReleasesDir = `${githubRepoDir}/app/api/description/config/releases`
 
 program
   .description('Generate dereferenced OpenAPI and decorated schema files.')
   .option(
     '--decorate-only',
     '⚠️ Only used by a 🤖 to generate decorated schema files from existing dereferenced schema files.'
   )
+  .option(
+    '-v --versions <VERSIONS...>',
+    'A list of undeprecated, published versions to build, separated by a space. Example "ghes-3.1" or "api.github.com github.ae"'
+  )
+  .option('-d --include-deprecated', 'Includes schemas that are marked as `deprecated: true`')
+  .option('-u --include-unpublished', 'Includes schemas that are marked as `published: false`')
   .parse(process.argv)
 
-const decorateOnly = program.opts().decorateOnly
+const { decorateOnly, versions, includeUnpublished, includeDeprecated } = program.opts()
+
+// Check that the github/github repo exists. If the files are only being
+// decorated, the github/github repo isn't needed.
+if (!decorateOnly) {
+  try {
+    await stat(githubRepoDir)
+  } catch (error) {
+    console.log(
+      `🛑 The ${githubRepoDir} does not exist. Make sure you have a local, bootstrapped checkout of github/github at the same level as your github/docs-internal repo before running this script.`
+    )
+    process.exit(1)
+  }
+}
+
+// When the input parameter type is decorate-only, use the local
+// `github/docs-internal` repo to generate a list of schema files.
+// Otherwise, use the `github/github` list of config files
+const referenceSchemaDirectory = decorateOnly ? dereferencedPath : openApiReleasesDir
+// A full list of unpublished, deprecated, and active schemas
+const allSchemas = await getOpenApiSchemas(referenceSchemaDirectory)
+
+await validateInputParameters(allSchemas)
+
+// Format the command supplied to the bundle script in `github/github`
+const commandParameters = await getCommandParameters()
+// Get the list of schemas for this bundle, depending on options
+const schemas = await getSchemas(allSchemas)
 
 main()
 
 async function main() {
   // Generate the dereferenced OpenAPI schema files
   if (!decorateOnly) {
-    if (!fs.existsSync(githubRepoDir)) {
-      console.log(
-        `🛑 The ${githubRepoDir} does not exist. Make sure you have a local, bootstrapped checkout of github/github at the same level as your github/docs-internal repo before running this script.`
-      )
-      process.exit(1)
-    }
-
     await getDereferencedFiles()
   }
-
+  // Decorate the dereferenced files in a format ingestible by docs.github.com
   await decorate()
 
   console.log(
@@ -64,16 +91,17 @@ async function getDereferencedFiles() {
     execSync('git pull', { cwd: githubRepoDir })
   }
 
-  // create a tmp directory to store schema files generated from github/github
+  // Create a tmp directory to store schema files generated from github/github
   rimraf.sync(tempDocsDir)
   await mkdirp(tempDocsDir)
 
   console.log(
     `\n🏃‍♀️🏃🏃‍♀️Running \`bin/openapi bundle\` in branch '${githubBranch}' of your github/github checkout to generate the dereferenced OpenAPI schema files.\n`
   )
   try {
+    console.log(`bundle -o ${tempDocsDir} ${commandParameters}`)
     execSync(
-      `${path.join(githubRepoDir, 'bin/openapi')} bundle -o ${tempDocsDir} --include_unpublished`,
+      `${path.join(githubRepoDir, 'bin/openapi')} bundle -o ${tempDocsDir} ${commandParameters}`,
       { stdio: 'inherit' }
     )
   } catch (error) {
@@ -92,21 +120,22 @@ async function getDereferencedFiles() {
   // property in the dereferenced schema is replaced with the branch
   // name of the `github/github` checkout. A CI test
   // checks the version and fails if it's not a semantic version.
-  schemas.forEach((filename) => {
-    const schema = JSON.parse(fs.readFileSync(path.join(dereferencedPath, filename)))
+  for (const filename of schemas) {
+    const schema = JSON.parse(await readFile(path.join(dereferencedPath, filename)))
+
     schema.info.version = `${githubBranch} !!DEVELOPMENT MODE - DO NOT MERGE!!`
-    fs.writeFileSync(path.join(dereferencedPath, filename), JSON.stringify(schema, null, 2))
-  })
+    await writeFile(path.join(dereferencedPath, filename), JSON.stringify(schema, null, 2))
+  }
 }
 
 async function decorate() {
   console.log('\n🎄 Decorating the OpenAPI schema files in lib/rest/static/dereferenced.\n')
-
-  const dereferencedSchemas = schemas.reduce((acc, filename) => {
-    const schema = JSON.parse(fs.readFileSync(path.join(dereferencedPath, filename)))
+  const dereferencedSchemas = {}
+  for (const filename of schemas) {
+    const schema = JSON.parse(await readFile(path.join(dereferencedPath, filename)))
     const key = filename.replace('.deref.json', '')
-    return { ...acc, [key]: schema }
-  }, {})
+    dereferencedSchemas[key] = schema
+  }
 
   for (const [schemaName, schema] of Object.entries(dereferencedSchemas)) {
     try {
@@ -118,7 +147,7 @@ async function decorate() {
 
       const filename = path.join(decoratedPath, `${schemaName}.json`).replace('.deref', '')
       // write processed operations to disk
-      fs.writeFileSync(filename, JSON.stringify(operations, null, 2))
+      await writeFile(filename, JSON.stringify(operations, null, 2))
 
       console.log('Wrote', path.relative(process.cwd(), filename))
     } catch (error) {
@@ -130,3 +159,92 @@ async function decorate() {
     }
   }
 }
+
+async function validateInputParameters(schemas) {
+  // The `--versions` and `--decorate-only` options cannot be used
+  // with the `--include-deprecated` or `--include-unpublished` options
+  const numberOfOptions = Object.keys(program.opts()).length
+
+  if (numberOfOptions > 1 && (decorateOnly || versions)) {
+    console.log(
+      `🛑 You cannot use the versions and decorate-only options with any other options.\nThe decorate-only switch will decorate all dereferenced schemas files in the docs-internal repo.\nThis script doesn't support generating individual deprecated or unpublished schemas.\nPlease reach out to #docs-engineering if this is a use case that you need.`
+    )
+    process.exit(1)
+  }
+
+  // Validate individual versions provided
+  if (versions) {
+    versions.forEach((version) => {
+      if (
+        schemas.deprecated.includes(`${version}.deref.json`) ||
+        schemas.unpublished.includes(`${version}.deref.json`)
+      ) {
+        console.log(
+          `🛑 This script doesn't support generating individual deprecated or unpublished schemas. Please reach out to #docs-engineering if this is a use case that you need.`
+        )
+        process.exit(1)
+      } else if (!schemas.currentReleases.includes(`${version}.deref.json`)) {
+        console.log(`🛑 The version (${version}) you specified is not valid.`)
+        process.exit(1)
+      }
+    })
+  }
+}
+
+async function getOpenApiSchemas(directory) {
+  const openAPIConfigs = await readdir(directory)
+  const unpublished = []
+  const deprecated = []
+  const currentReleases = []
+
+  for (const file of openAPIConfigs) {
+    const newFileName = `${path.basename(file, 'yaml')}deref.json`
+    const content = await readFile(path.join(directory, file), 'utf8')
+    const yamlContent = yaml.load(content)
+    if (!yamlContent.published) {
+      unpublished.push(newFileName)
+    }
+    if (yamlContent.deprecated) {
+      deprecated.push(newFileName)
+    }
+    if (!yamlContent.deprecated && yamlContent.published) {
+      currentReleases.push(newFileName)
+    }
+  }
+
+  return { currentReleases, unpublished, deprecated }
+}
+
+async function getCommandParameters() {
+  let includeParams = []
+
+  if (versions) {
+    includeParams = versions
+  }
+  if (includeUnpublished) {
+    includeParams.push('--include_unpublished')
+  }
+  if (includeDeprecated) {
+    includeParams.push('--include_deprecated')
+  }
+
+  return includeParams.join(' ')
+}
+
+async function getSchemas(allSchemas) {
+  if (decorateOnly) {
+    const files = await readdir(dereferencedPath)
+    return files
+  } else if (versions) {
+    return versions.map((elem) => `${elem}.deref.json`)
+  } else {
+    const schemas = allSchemas.currentReleases
+    if (includeUnpublished) {
+      schemas.push(...allSchemas.unpublished)
+    }
+    if (includeDeprecated) {
+      schemas.push(...allSchemas.deprecated)
+    }
+    return schemas
+  }
+}