Add sitemap.xml file

Generate a sitemap.xml during build time that is served at `/sitemap.xml`, and
add a reference to it in `robots.txt` so that crawlers can find it.
(See: https://developers.google.com/search/docs/crawling-indexing/sitemaps/build-sitemap#addsitemap)

The sitemap entries' `lastmod` property comes from the last modification time
of the source MDX files for each page. Note that this might not be completely
accurate, as updates to MDX fragments in `/includes` or `/platform-includes`
won't be considered.

Pages generated from our OpenAPI specs are missing `lastmod` properties since
we don't have a way to isolate the timestamp when individual pages change.

Author: mjq

app/sitemap.ts

@@ -0,0 +1,21 @@
+import fs from 'fs';
+import path from 'path';
+
+import type {MetadataRoute} from 'next';
+
+import {getDocsFrontMatter} from 'sentry-docs/mdx';
+
+export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+  const docs = await getDocsFrontMatter();
+  const sitemapEntries = docs.map(doc => ({
+    url: `https://docs.sentry.io/${doc.slug}`,
+    lastModified: doc.lastModified,
+  }));
+  sitemapEntries.unshift({
+    url: 'https://docs.sentry.io/',
+    lastModified: fs.statSync(
+      path.join(process.cwd(), 'src', 'components', 'homeClient.tsx')
+    ).mtime,
+  });
+  return sitemapEntries;
+}

src/mdx.ts

@@ -100,6 +100,7 @@ export function getAllFilesFrontMatter(folder: string = 'docs'): FrontMatter[] {
         ...frontmatter,
         slug: formatSlug(fileName),
         sourcePath: path.join(folder, fileName),
+        lastModified: fs.statSync(file).mtime,
       });
     }
   });
@@ -133,6 +134,7 @@ export function getAllFilesFrontMatter(folder: string = 'docs'): FrontMatter[] {
     const commonFiles = commonFileNames.map(commonFileName => {
       const source = fs.readFileSync(commonFileName, 'utf8');
       const {data: frontmatter} = matter(source);
+      frontmatter.lastModified = fs.statSync(commonFileName).mtime;
       return {commonFileName, frontmatter};
     });
 

static/robots.txt

@@ -1,2 +1,3 @@
+Sitemap: https://docs.sentry.io/sitemap.xml
 User-agent: *
 Disallow: /development-api/