Disallow robot crawlers outside of Prod (#2381)

Author: jonkafton

frontends/main/src/app/robots.ts

@@ -5,22 +5,29 @@ invariant(process.env.NEXT_PUBLIC_ORIGIN, "NEXT_PUBLIC_ORIGIN must be defined")
 const BASE_URL: string = process.env.NEXT_PUBLIC_ORIGIN
 
 export default function robots(): MetadataRoute.Robots {
-  const sitemap =
-    process.env.MITOL_NOINDEX === "false"
-      ? `${BASE_URL}/sitemaps/sitemap-index.xml`
-      : undefined
-  return {
-    rules: {
-      userAgent: "*",
-      allow: "/",
-      disallow: [
-        "/dashboard/",
-        "/learningpaths/",
-        "/onboarding/",
-        "/cart/",
-        "/program_letter/",
-      ],
-    },
-    sitemap,
+  const shouldIndex = process.env.MITOL_NOINDEX === "false"
+
+  if (shouldIndex) {
+    return {
+      rules: {
+        userAgent: "*",
+        allow: "/",
+        disallow: [
+          "/dashboard/",
+          "/learningpaths/",
+          "/onboarding/",
+          "/cart/",
+          "/program_letter/",
+        ],
+      },
+      sitemap: `${BASE_URL}/sitemaps/sitemap-index.xml`,
+    }
+  } else {
+    return {
+      rules: {
+        userAgent: "*",
+        disallow: "/",
+      },
+    }
   }
 }