Adding robots txt for cloudflare pages (#418)

* Adding robots txt for cloudflare pages

* template

Author: laliconfigcat

.github/pull_request_template.md

@@ -13,5 +13,6 @@ What part of the application were affected by the changes? What should be tested
 ### Requirement checklist
 
 - [ ] I have validated my changes on a test/local environment.
+- [ ] I have added my changes to the V1 and V2 documentations.
 - [ ] I have checked the SNYK/Dependabot reports and applied the suggested changes.
 - [ ] (Optional) I have updated outdated packages.

website/package.json

@@ -11,7 +11,7 @@
     "serve": "docusaurus serve",
     "gen-api-docs": "docusaurus gen-api-docs all && node ./api/cleanup-api-docs.js",
     "typecheck": "tsc",
-    "cf": "npm run gen-api-docs && docusaurus build --out-dir build/docs && echo '/ /docs' > build/_redirects"
+    "cf": "npm run gen-api-docs && docusaurus build --out-dir build/docs && echo / /docs > build/_redirects && node testrobots.js"
   },
   "dependencies": {
     "@docusaurus/core": "3.1.1",

website/testrobots.js

@@ -0,0 +1,8 @@
+const fs = require('node:fs');
+const content = 'User-agent: *\nDisallow: /';
+fs.writeFile('build/robots.txt', content, err => {
+  if (err) {
+    console.error(err);
+  } else {
+  }
+});