Merge pull request #1346 from onflow/brian-doyle/update-robots.txt

Update robots.txt

Author: briandoyle81

static/robots.txt

@@ -1,5 +1,37 @@
-# Algolia-Crawler-Verif: 98E1096D4FD67E70
+# algolia: 98E1096D4FD67E70
 
+# Allow search engines
+User-agent: Googlebot
+Allow: /
+
+User-agent: Bingbot
+Allow: /
+
+User-agent: DuckDuckBot
+Allow: /
+
+User-agent: Baiduspider
+Allow: /
+
+# Allow LLM bots
+User-agent: GPTBot
+Allow: /
+
+User-agent: CCBot
+Allow: /
+
+User-agent: anthropic-ai
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+# Block all other bots by default
 User-agent: *
-Allow:
-Sitemap: https://developers.flow.com/sitemap.xml
+Disallow: /
+
+# Sitemap
+Sitemap: https://developers.flow.com/sitemap.xml