From 3b78d55bd7ab3106fde53607819bec9c25ec3337 Mon Sep 17 00:00:00 2001
From: vserova <serovav@ncbi.nlm.nih.gov>
Date: Wed, 29 Apr 2020 21:34:27 -0400
Subject: [PATCH] Create robots.txt

---
 robots.txt | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 robots.txt

diff --git a/robots.txt b/robots.txt
new file mode 100644
index 00000000..954c6227
--- /dev/null
+++ b/robots.txt
@@ -0,0 +1,6 @@
+# Exclude technical page for inhouse only use from indexing
+
+User-agent: *
+Disallow: /pages/notes
+
+Sitemap: https://ncbi.github.io/cxx-toolkit/sitemap_000.xml