Add test linking

This adds the ability to link rust-lang/rust tests to reference rule
annotations. Rules now have a link that pops up a list of tests that
are associated with that rule. Additionally, there is a new appendix
that lists the number of rules in each chapter, how many tests are
associated, and various summaries.

This requires a local checkout of rust-lang/rust which is pointed to
by the `SPEC_RUST_ROOT` environment variable.

Author: ehuss

.github/workflows/main.yml

@@ -35,6 +35,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@master
+    - name: Checkout rust-lang/rust
+      uses: actions/checkout@master
+      with:
+        repository: rust-lang/rust
+        path: rust
     - name: Update rustup
       run: rustup self update
     - name: Install Rust
@@ -52,16 +57,17 @@ jobs:
         rustup --version
         rustc -Vv
         mdbook --version
-    - name: Verify the book builds
-      env:
-        SPEC_DENY_WARNINGS: 1
-      run: mdbook build
     - name: Style checks
       working-directory: style-check
       run: cargo run --locked -- ../src
     - name: Style fmt
       working-directory: style-check
       run: cargo fmt --check
+    - name: Verify the book builds
+      env:
+        SPEC_DENY_WARNINGS: 1
+        SPEC_RUST_ROOT: ${{ github.workspace }}/rust
+      run: mdbook build
     - name: Check for broken links
       run: |
         curl -sSLo linkcheck.sh \

README.md

@@ -81,3 +81,7 @@ The published site at <https://doc.rust-lang.org/reference/> (or local docs usin
 ### `SPEC_DENY_WARNINGS`
 
 The `SPEC_DENY_WARNINGS=1` environment variable will turn all warnings generated by `mdbook-spec` to errors. This is used in CI to ensure that there aren't any problems with the book content.
+
+### `SPEC_RUST_ROOT`
+
+The `SPEC_RUST_ROOT` can be used to point to the directory of a checkout of <https://github.com/rust-lang/rust>. This is used by the test-linking feature so that it can find tests linked to reference rules. If this is not set, then the tests won't be linked.

book.toml

@@ -5,6 +5,7 @@ author = "The Rust Project Developers"
 
 [output.html]
 additional-css = ["theme/reference.css"]
+additional-js = ["theme/reference.js"]
 git-repository-url = "https://github.com/rust-lang/reference/"
 edit-url-template = "https://github.com/rust-lang/reference/edit/master/{path}"
 smart-punctuation = true

docs/authoring.md

@@ -99,6 +99,18 @@ When assigning rules to new paragraphs, or when modifying rule names, use the fo
     * Target specific admonitions should typically be named by the least specific target property to which they apply (e.g. if a rule affects all x86 CPUs, the rule name should include `x86` rather than separately listing `i586`, `i686` and `x86_64`, and if a rule applies to all ELF platforms, it should be named `elf` rather than listing every ELF OS).
     * Use an appropriately descriptive, but short, name if the language does not provide one.
 
+#### Test rule annotations
+
+Tests in <https://github.com/rust-lang/rust> can be linked to rules in the reference. The rule will include a link to the tests, and there is also an appendix which tracks how the rules are currently linked.
+
+Tests in the `tests` directory can be annotated with the `//@ reference: x.y.z` header to link it to a rule. The header can be specified multiple times if a single file covers multiple rules.
+
+You *should* when possible make sure every rule has a test associated with it. This is beneficial for reviewers to see the behavior and readers who may want to see examples of particular behaviors. When adding new rules, you should wait until the reference side is approved before submitting a PR to `rust-lang/rust` (to avoid churn if we decide on different names).
+
+Prefixed rule names should not be used in tests. That is, do not use something like `asm.rules` when there are specific rules like `asm.rules.reg-not-input`.
+
+We are not expecting 100% coverage at any time. Although it would be nice, it is unrealistic due to the sequence things are developed, and resources available.
+
 ### Standard library links
 
 You should link to the standard library without specifying a URL in a fashion similar to [rustdoc intra-doc links][intra]. Some examples:

mdbook-spec/Cargo.lock

@@ -19,3 +19,4 @@ regex = "1.9.4"
 semver = "1.0.21"
 serde_json = "1.0.113"
 tempfile = "3.10.1"
+walkdir = "2.5.0"

mdbook-spec/Cargo.toml

@@ -1,7 +1,7 @@
 #![deny(rust_2018_idioms, unused_lifetimes)]
 
 use crate::rules::Rules;
-use anyhow::Result;
+use anyhow::{bail, Context, Result};
 use mdbook::book::{Book, Chapter};
 use mdbook::errors::Error;
 use mdbook::preprocess::{CmdPreprocessor, Preprocessor, PreprocessorContext};
@@ -10,9 +10,11 @@ use once_cell::sync::Lazy;
 use regex::{Captures, Regex};
 use semver::{Version, VersionReq};
 use std::io;
+use std::path::PathBuf;
 
 mod rules;
 mod std_links;
+mod test_links;
 
 /// The Regex for the syntax for blockquotes that have a specific CSS class,
 /// like `> [!WARNING]`.
@@ -47,12 +49,37 @@ pub struct Spec {
     /// Whether or not warnings should be errors (set by SPEC_DENY_WARNINGS
     /// environment variable).
     deny_warnings: bool,
+    /// Path to the rust-lang/rust git repository (set by SPEC_RUST_ROOT
+    /// environment variable).
+    rust_root: Option<PathBuf>,
+    /// The git ref that can be used in a URL to the rust-lang/rust repository.
+    git_ref: String,
 }
 
 impl Spec {
     fn new() -> Result<Spec> {
         let deny_warnings = std::env::var("SPEC_DENY_WARNINGS").as_deref() == Ok("1");
-        Ok(Spec { deny_warnings })
+        let rust_root = std::env::var_os("SPEC_RUST_ROOT").map(PathBuf::from);
+        if deny_warnings && rust_root.is_none() {
+            bail!("SPEC_RUST_ROOT environment variable must be set");
+        }
+        let git_ref = match git_ref(&rust_root) {
+            Ok(s) => s,
+            Err(e) => {
+                if deny_warnings {
+                    eprintln!("error: {e:?}");
+                    std::process::exit(1);
+                } else {
+                    eprintln!("warning: {e:?}");
+                    "master".into()
+                }
+            }
+        };
+        Ok(Spec {
+            deny_warnings,
+            rust_root,
+            git_ref,
+        })
     }
 
     /// Generates link references to all rules on all pages, so you can easily
@@ -115,13 +142,37 @@ fn to_initial_case(s: &str) -> String {
     format!("{first}{rest}")
 }
 
+/// Determines the git ref used for linking to a particular branch/tag in GitHub.
+fn git_ref(rust_root: &Option<PathBuf>) -> Result<String> {
+    let Some(rust_root) = rust_root else {
+        return Ok("master".into());
+    };
+    let channel = std::fs::read_to_string(rust_root.join("src/ci/channel"))
+        .context("failed to read src/ci/channel")?;
+    let git_ref = match channel.trim() {
+        // nightly/beta are branches, not stable references. Should be ok
+        // because we're not expecting those channels to be long-lived.
+        "nightly" => "master".into(),
+        "beta" => "beta".into(),
+        "stable" => {
+            let version = std::fs::read_to_string(rust_root.join("src/version"))
+                .context("|| failed to read src/version")?;
+            version.trim().into()
+        }
+        ch => bail!("unknown channel {ch}"),
+    };
+    Ok(git_ref)
+}
+
 impl Preprocessor for Spec {
     fn name(&self) -> &str {
         "spec"
     }
 
     fn run(&self, _ctx: &PreprocessorContext, mut book: Book) -> Result<Book, Error> {
         let rules = self.collect_rules(&book);
+        let tests = self.collect_tests(&rules);
+        let summary_table = test_links::make_summary_table(&book, &tests, &rules);
 
         book.for_each_mut(|item| {
             let BookItem::Chapter(ch) = item else {
@@ -132,7 +183,10 @@ impl Preprocessor for Spec {
             }
             ch.content = self.admonitions(&ch);
             ch.content = self.auto_link_references(&ch, &rules);
-            ch.content = self.render_rule_definitions(&ch.content);
+            ch.content = self.render_rule_definitions(&ch.content, &tests);
+            if ch.name == "Test summary" {
+                ch.content = ch.content.replace("{{summary-table}}", &summary_table);
+            }
         });
 
         // Final pass will resolve everything as a std link (or error if the

mdbook-spec/src/lib.rs

@@ -1,11 +1,13 @@
 //! Handling for rule identifiers.
 
+use crate::test_links::RuleToTests;
 use crate::Spec;
 use mdbook::book::Book;
 use mdbook::BookItem;
 use once_cell::sync::Lazy;
 use regex::{Captures, Regex};
 use std::collections::{BTreeMap, HashSet};
+use std::fmt::Write;
 use std::path::PathBuf;
 
 /// The Regex for rules like `r[foo]`.
@@ -76,13 +78,35 @@ impl Spec {
 
     /// Converts lines that start with `r[…]` into a "rule" which has special
     /// styling and can be linked to.
-    pub fn render_rule_definitions(&self, content: &str) -> String {
+    pub fn render_rule_definitions(&self, content: &str, tests: &RuleToTests) -> String {
         RULE_RE
             .replace_all(content, |caps: &Captures<'_>| {
                 let rule_id = &caps[1];
+                let mut test_html = String::new();
+                if let Some(tests) = tests.get(rule_id) {
+                    test_html = format!(
+                        "<span class=\"popup-container\">\n\
+                            &nbsp;&nbsp;&nbsp;&nbsp;<a href=\"javascript:void(0)\" onclick=\"spec_toggle_tests('{rule_id}');\">\
+                            Tests</a>\n\
+                            <div id=\"tests-{rule_id}\" class=\"tests-popup popup-hidden\">\n\
+                            Tests with this rule:
+                            <ul>");
+                    for test in tests {
+                        writeln!(
+                            test_html,
+                            "<li><a href=\"https://github.com/rust-lang/rust/blob/{git_ref}/{test_path}\">{test_path}</a></li>",
+                            test_path = test.path,
+                            git_ref = self.git_ref
+                        )
+                        .unwrap();
+                    }
+
+                    test_html.push_str("</ul></div></span>");
+                }
                 format!(
                     "<div class=\"rule\" id=\"r-{rule_id}\">\
                      <a class=\"rule-link\" href=\"#r-{rule_id}\">[{rule_id}]</a>\
+                     {test_html}\
                      </div>\n"
                 )
             })