Create robots.txt

ncbi · Apr 30, 2020 · 3b78d55 · 3b78d55
1 parent df3bc64
commit 3b78d55
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/robots.txt b/robots.txt
@@ -0,0 +1,6 @@
+# Exclude technical page for inhouse only use from indexing
+
+User-agent: *
+Disallow: /pages/notes
+
+Sitemap: https://ncbi.github.io/cxx-toolkit/sitemap_000.xml