create robots.txt for HTML docs

Author: myk002

conf.py

@@ -256,6 +256,10 @@ def get_version():
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['docs/styles']
 
+# A list of paths that contain extra files not directly related to the
+# documentation.
+html_extra_path = ['robots.txt']
+
 # Custom sidebar templates, maps document names to template names.
 html_sidebars = {
     '**': [

robots.txt

@@ -0,0 +1,5 @@
+User-agent: *
+
+Allow: /en/stable/
+
+Sitemap: https://docs.dfhack.org/sitemap.xml