Rollup merge of rust-lang#111348 - ozkanonur:remove-hardcoded-rustdoc-flags, r=albertlarsan68,oli-obk

new tool `rustdoc-gui-test`

Implements new tool `rustdoc-gui-test` that allows using compiletest headers for `rustdoc-gui` tests.

Author: compiler-errors

Cargo.lock

@@ -4379,6 +4379,15 @@ dependencies = [
  "tracing-tree",
 ]
 
+[[package]]
+name = "rustdoc-gui-test"
+version = "0.1.0"
+dependencies = [
+ "compiletest",
+ "getopts",
+ "walkdir",
+]
+
 [[package]]
 name = "rustdoc-json-types"
 version = "0.1.0"

Cargo.toml

@@ -40,6 +40,7 @@ members = [
   "src/tools/generate-copyright",
   "src/tools/suggest-tests",
   "src/tools/generate-windows-sys",
+  "src/tools/rustdoc-gui-test",
 ]
 
 exclude = [

library/test/src/options.rs

@@ -16,19 +16,21 @@ pub enum ShouldPanic {
 }
 
 /// Whether should console output be colored or not
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Default, Debug)]
 pub enum ColorConfig {
+    #[default]
     AutoColor,
     AlwaysColor,
     NeverColor,
 }
 
 /// Format of the test results output
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
 pub enum OutputFormat {
     /// Verbose output
     Pretty,
     /// Quiet output
+    #[default]
     Terse,
     /// JSON output
     Json,

src/bootstrap/builder.rs

@@ -689,7 +689,8 @@ impl<'a> Builder<'a> {
                 tool::Miri,
                 tool::CargoMiri,
                 llvm::Lld,
-                llvm::CrtBeginEnd
+                llvm::CrtBeginEnd,
+                tool::RustdocGUITest,
             ),
             Kind::Check | Kind::Clippy | Kind::Fix => describe!(
                 check::Std,

src/bootstrap/test.rs

@@ -944,28 +944,6 @@ fn get_browser_ui_test_version(npm: &Path) -> Option<String> {
         .or_else(|| get_browser_ui_test_version_inner(npm, true))
 }
 
-fn compare_browser_ui_test_version(installed_version: &str, src: &Path) {
-    match fs::read_to_string(
-        src.join("src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version"),
-    ) {
-        Ok(v) => {
-            if v.trim() != installed_version {
-                eprintln!(
-                    "⚠️ Installed version of browser-ui-test (`{}`) is different than the \
-                     one used in the CI (`{}`)",
-                    installed_version, v
-                );
-                eprintln!(
-                    "You can install this version using `npm update browser-ui-test` or by using \
-                     `npm install browser-ui-test@{}`",
-                    v,
-                );
-            }
-        }
-        Err(e) => eprintln!("Couldn't find the CI browser-ui-test version: {:?}", e),
-    }
-}
-
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct RustdocGUI {
     pub target: TargetSelection,
@@ -997,94 +975,56 @@ impl Step for RustdocGUI {
     }
 
     fn run(self, builder: &Builder<'_>) {
-        let nodejs = builder.config.nodejs.as_ref().expect("nodejs isn't available");
-        let npm = builder.config.npm.as_ref().expect("npm isn't available");
-
         builder.ensure(compile::Std::new(self.compiler, self.target));
 
-        // The goal here is to check if the necessary packages are installed, and if not, we
-        // panic.
-        match get_browser_ui_test_version(&npm) {
-            Some(version) => {
-                // We also check the version currently used in CI and emit a warning if it's not the
-                // same one.
-                compare_browser_ui_test_version(&version, &builder.build.src);
-            }
-            None => {
-                eprintln!(
-                    "error: rustdoc-gui test suite cannot be run because npm `browser-ui-test` \
-                     dependency is missing",
-                );
-                eprintln!(
-                    "If you want to install the `{0}` dependency, run `npm install {0}`",
-                    "browser-ui-test",
-                );
-                panic!("Cannot run rustdoc-gui tests");
-            }
-        }
+        let mut cmd = builder.tool_cmd(Tool::RustdocGUITest);
 
         let out_dir = builder.test_out(self.target).join("rustdoc-gui");
-
-        // We remove existing folder to be sure there won't be artifacts remaining.
         builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler));
 
-        let src_path = builder.build.src.join("tests/rustdoc-gui/src");
-        // We generate docs for the libraries present in the rustdoc-gui's src folder.
-        for entry in src_path.read_dir().expect("read_dir call failed") {
-            if let Ok(entry) = entry {
-                let path = entry.path();
+        if let Some(src) = builder.config.src.to_str() {
+            cmd.arg("--rust-src").arg(src);
+        }
 
-                if !path.is_dir() {
-                    continue;
-                }
+        if let Some(out_dir) = out_dir.to_str() {
+            cmd.arg("--out-dir").arg(out_dir);
+        }
 
-                let mut cargo = Command::new(&builder.initial_cargo);
-                cargo
-                    .arg("doc")
-                    .arg("--target-dir")
-                    .arg(&out_dir)
-                    .env("RUSTC_BOOTSTRAP", "1")
-                    .env("RUSTDOC", builder.rustdoc(self.compiler))
-                    .env("RUSTC", builder.rustc(self.compiler))
-                    .current_dir(path);
-                // FIXME: implement a `// compile-flags` command or similar
-                //        instead of hard-coding this test
-                if entry.file_name() == "link_to_definition" {
-                    cargo.env("RUSTDOCFLAGS", "-Zunstable-options --generate-link-to-definition");
-                } else if entry.file_name() == "scrape_examples" {
-                    cargo.arg("-Zrustdoc-scrape-examples");
-                } else if entry.file_name() == "extend_css" {
-                    cargo.env("RUSTDOCFLAGS", &format!("--extend-css extra.css"));
-                }
-                builder.run(&mut cargo);
-            }
+        if let Some(initial_cargo) = builder.config.initial_cargo.to_str() {
+            cmd.arg("--initial-cargo").arg(initial_cargo);
         }
 
-        // We now run GUI tests.
-        let mut command = Command::new(&nodejs);
-        command
-            .arg(builder.build.src.join("src/tools/rustdoc-gui/tester.js"))
-            .arg("--jobs")
-            .arg(&builder.jobs().to_string())
-            .arg("--doc-folder")
-            .arg(out_dir.join("doc"))
-            .arg("--tests-folder")
-            .arg(builder.build.src.join("tests/rustdoc-gui"));
+        cmd.arg("--jobs").arg(builder.jobs().to_string());
+
+        cmd.env("RUSTDOC", builder.rustdoc(self.compiler))
+            .env("RUSTC", builder.rustc(self.compiler));
+
         for path in &builder.paths {
             if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) {
                 if !p.ends_with(".goml") {
                     eprintln!("A non-goml file was given: `{}`", path.display());
                     panic!("Cannot run rustdoc-gui tests");
                 }
                 if let Some(name) = path.file_name().and_then(|f| f.to_str()) {
-                    command.arg("--file").arg(name);
+                    cmd.arg("--goml-file").arg(name);
                 }
             }
         }
+
         for test_arg in builder.config.test_args() {
-            command.arg(test_arg);
+            cmd.arg("--test-arg").arg(test_arg);
         }
-        builder.run(&mut command);
+
+        if let Some(ref nodejs) = builder.config.nodejs {
+            cmd.arg("--nodejs").arg(nodejs);
+        }
+
+        if let Some(ref npm) = builder.config.npm {
+            cmd.arg("--npm").arg(npm);
+        }
+
+        let _time = util::timeit(&builder);
+        crate::render_tests::try_run_tests(builder, &mut cmd);
     }
 }
 

src/bootstrap/tool.rs

@@ -302,6 +302,7 @@ bootstrap_tool!(
     GenerateCopyright, "src/tools/generate-copyright", "generate-copyright";
     SuggestTests, "src/tools/suggest-tests", "suggest-tests";
     GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys";
+    RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test";
 );
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]

src/tools/compiletest/Cargo.toml

@@ -3,6 +3,9 @@ name = "compiletest"
 version = "0.0.0"
 edition = "2021"
 
+[lib]
+doctest = false
+
 [dependencies]
 colored = "2"
 diff = "0.1.10"

src/tools/compiletest/src/common.rs

@@ -69,6 +69,12 @@ string_enum! {
     }
 }
 
+impl Default for Mode {
+    fn default() -> Self {
+        Mode::Ui
+    }
+}
+
 impl Mode {
     pub fn disambiguator(self) -> &'static str {
         // Pretty-printing tests could run concurrently, and if they do,
@@ -125,7 +131,7 @@ pub enum PanicStrategy {
 }
 
 /// Configuration for compiletest
-#[derive(Debug, Clone)]
+#[derive(Debug, Default, Clone)]
 pub struct Config {
     /// `true` to overwrite stderr/stdout files instead of complaining about changes in output.
     pub bless: bool,

src/tools/compiletest/src/lib.rs

@@ -0,0 +1,9 @@
+[package]
+name = "rustdoc-gui-test"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+compiletest = { path = "../compiletest" }
+getopts = "0.2"
+walkdir = "2"

src/tools/compiletest/src/main.rs

@@ -0,0 +1,62 @@
+use getopts::Options;
+use std::{env, path::PathBuf};
+
+pub(crate) struct Config {
+    pub(crate) nodejs: PathBuf,
+    pub(crate) npm: PathBuf,
+    pub(crate) rust_src: PathBuf,
+    pub(crate) out_dir: PathBuf,
+    pub(crate) initial_cargo: PathBuf,
+    pub(crate) jobs: String,
+    pub(crate) test_args: Vec<PathBuf>,
+    pub(crate) goml_files: Vec<PathBuf>,
+    pub(crate) rustc: PathBuf,
+    pub(crate) rustdoc: PathBuf,
+    pub(crate) verbose: bool,
+}
+
+impl Config {
+    pub(crate) fn from_args(args: Vec<String>) -> Self {
+        let mut opts = Options::new();
+        opts.reqopt("", "nodejs", "absolute path of nodejs", "PATH")
+            .reqopt("", "npm", "absolute path of npm", "PATH")
+            .reqopt("", "out-dir", "output path of doc compilation", "PATH")
+            .reqopt("", "rust-src", "root source of the rust source", "PATH")
+            .reqopt(
+                "",
+                "initial-cargo",
+                "path to cargo to use for compiling tests/rustdoc-gui/src/*",
+                "PATH",
+            )
+            .reqopt("", "jobs", "jobs arg of browser-ui-test", "JOBS")
+            .optflag("", "verbose", "run tests verbosely, showing all output")
+            .optmulti("", "test-arg", "args for browser-ui-test", "FLAGS")
+            .optmulti("", "goml-file", "goml files for testing with browser-ui-test", "LIST");
+
+        let (argv0, args_) = args.split_first().unwrap();
+        if args.len() == 1 || args[1] == "-h" || args[1] == "--help" {
+            let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
+            println!("{}", opts.usage(&message));
+            std::process::exit(1);
+        }
+
+        let matches = &match opts.parse(args_) {
+            Ok(m) => m,
+            Err(f) => panic!("{:?}", f),
+        };
+
+        Self {
+            nodejs: matches.opt_str("nodejs").map(PathBuf::from).expect("nodejs isn't available"),
+            npm: matches.opt_str("npm").map(PathBuf::from).expect("npm isn't available"),
+            rust_src: matches.opt_str("rust-src").map(PathBuf::from).unwrap(),
+            out_dir: matches.opt_str("out-dir").map(PathBuf::from).unwrap(),
+            initial_cargo: matches.opt_str("initial-cargo").map(PathBuf::from).unwrap(),
+            jobs: matches.opt_str("jobs").unwrap(),
+            goml_files: matches.opt_strs("goml-file").iter().map(PathBuf::from).collect(),
+            test_args: matches.opt_strs("test-arg").iter().map(PathBuf::from).collect(),
+            rustc: env::var("RUSTC").map(PathBuf::from).unwrap(),
+            rustdoc: env::var("RUSTDOC").map(PathBuf::from).unwrap(),
+            verbose: matches.opt_present("verbose"),
+        }
+    }
+}

src/tools/rustdoc-gui-test/Cargo.toml

@@ -0,0 +1,162 @@
+use compiletest::header::TestProps;
+use config::Config;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::Arc;
+use std::{env, fs};
+
+mod config;
+
+fn get_browser_ui_test_version_inner(npm: &Path, global: bool) -> Option<String> {
+    let mut command = Command::new(&npm);
+    command.arg("list").arg("--parseable").arg("--long").arg("--depth=0");
+    if global {
+        command.arg("--global");
+    }
+    let lines = command
+        .output()
+        .map(|output| String::from_utf8_lossy(&output.stdout).into_owned())
+        .unwrap_or(String::new());
+    lines
+        .lines()
+        .find_map(|l| l.split(':').nth(1)?.strip_prefix("browser-ui-test@"))
+        .map(|v| v.to_owned())
+}
+
+fn get_browser_ui_test_version(npm: &Path) -> Option<String> {
+    get_browser_ui_test_version_inner(npm, false)
+        .or_else(|| get_browser_ui_test_version_inner(npm, true))
+}
+
+fn compare_browser_ui_test_version(installed_version: &str, src: &Path) {
+    match fs::read_to_string(
+        src.join("src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version"),
+    ) {
+        Ok(v) => {
+            if v.trim() != installed_version {
+                eprintln!(
+                    "⚠️ Installed version of browser-ui-test (`{}`) is different than the \
+                     one used in the CI (`{}`)",
+                    installed_version, v
+                );
+                eprintln!(
+                    "You can install this version using `npm update browser-ui-test` or by using \
+                     `npm install browser-ui-test@{}`",
+                    v,
+                );
+            }
+        }
+        Err(e) => eprintln!("Couldn't find the CI browser-ui-test version: {:?}", e),
+    }
+}
+
+fn find_librs<P: AsRef<Path>>(path: P) -> Option<PathBuf> {
+    for entry in walkdir::WalkDir::new(path) {
+        let entry = entry.ok()?;
+        if entry.file_type().is_file() && entry.file_name() == "lib.rs" {
+            return Some(entry.path().to_path_buf());
+        }
+    }
+    None
+}
+
+// FIXME: move `bootstrap::util::try_run` into `build_helper` crate
+// and use that one instead of creating this function.
+fn try_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool {
+    let status = match cmd.status() {
+        Ok(status) => status,
+        Err(e) => panic!("failed to execute command: {:?}\nerror: {}", cmd, e),
+    };
+    if !status.success() && print_cmd_on_fail {
+        println!(
+            "\n\ncommand did not execute successfully: {:?}\n\
+             expected success, got: {}\n\n",
+            cmd, status
+        );
+    }
+    status.success()
+}
+
+fn main() {
+    let config = Arc::new(Config::from_args(env::args().collect()));
+
+    // The goal here is to check if the necessary packages are installed, and if not, we
+    // panic.
+    match get_browser_ui_test_version(&config.npm) {
+        Some(version) => {
+            // We also check the version currently used in CI and emit a warning if it's not the
+            // same one.
+            compare_browser_ui_test_version(&version, &config.rust_src);
+        }
+        None => {
+            eprintln!(
+                r#"
+error: rustdoc-gui test suite cannot be run because npm `browser-ui-test` dependency is missing.
+
+If you want to install the `browser-ui-test` dependency, run `npm install browser-ui-test`
+"#,
+            );
+
+            panic!("Cannot run rustdoc-gui tests");
+        }
+    }
+
+    let src_path = config.rust_src.join("tests/rustdoc-gui/src");
+    for entry in src_path.read_dir().expect("read_dir call failed") {
+        if let Ok(entry) = entry {
+            let path = entry.path();
+
+            if !path.is_dir() {
+                continue;
+            }
+
+            let mut cargo = Command::new(&config.initial_cargo);
+            cargo
+                .arg("doc")
+                .arg("--target-dir")
+                .arg(&config.out_dir)
+                .env("RUSTC_BOOTSTRAP", "1")
+                .env("RUSTDOC", &config.rustdoc)
+                .env("RUSTC", &config.rustc)
+                .current_dir(path);
+
+            if let Some(librs) = find_librs(entry.path()) {
+                let compiletest_c = compiletest::common::Config {
+                    edition: None,
+                    mode: compiletest::common::Mode::Rustdoc,
+                    ..Default::default()
+                };
+
+                let test_props = TestProps::from_file(&librs, None, &compiletest_c);
+
+                if !test_props.compile_flags.is_empty() {
+                    cargo.env("RUSTDOCFLAGS", test_props.compile_flags.join(" "));
+                }
+
+                if let Some(flags) = &test_props.run_flags {
+                    cargo.arg(flags);
+                }
+            }
+
+            try_run(&mut cargo, config.verbose);
+        }
+    }
+
+    let mut command = Command::new(&config.nodejs);
+    command
+        .arg(config.rust_src.join("src/tools/rustdoc-gui/tester.js"))
+        .arg("--jobs")
+        .arg(&config.jobs)
+        .arg("--doc-folder")
+        .arg(config.out_dir.join("doc"))
+        .arg("--tests-folder")
+        .arg(config.rust_src.join("tests/rustdoc-gui"));
+
+    for file in &config.goml_files {
+        command.arg("--file").arg(file);
+    }
+
+    command.args(&config.test_args);
+
+    try_run(&mut command, config.verbose);
+}

src/tools/rustdoc-gui-test/src/config.rs

@@ -8,13 +8,13 @@ set-window-size: (600, 800)
 assert-property: ("html", {"scrollTop": "0"})
 
 click: '//a[text() = "barbar"]'
-assert-property: ("html", {"scrollTop": "125"})
+assert-property: ("html", {"scrollTop": "149"})
 click: '//a[text() = "bar"]'
-assert-property: ("html", {"scrollTop": "156"})
+assert-property: ("html", {"scrollTop": "180"})
 click: '//a[text() = "sub_fn"]'
-assert-property: ("html", {"scrollTop": "53"})
+assert-property: ("html", {"scrollTop": "77"})
 
 // We now check that clicking on lines doesn't change the scroll
 // Extra information: the "sub_fn" function header is on line 1.
 click: '//*[@id="6"]'
-assert-property: ("html", {"scrollTop": "53"})
+assert-property: ("html", {"scrollTop": "77"})

src/tools/rustdoc-gui-test/src/main.rs

@@ -1 +1,2 @@
+// compile-flags: --extend-css extra.css
 //! <div class="extend">text in red</div>

tests/rustdoc-gui/source-anchor-scroll.goml

@@ -1,3 +1,4 @@
+// compile-flags: -Zunstable-options --generate-link-to-definition
 pub fn sub_fn() {
     barbar();
 }

tests/rustdoc-gui/src/extend_css/lib.rs

@@ -1,3 +1,4 @@
+// run-flags:-Zrustdoc-scrape-examples
 /// # Examples
 ///
 /// ```

tests/rustdoc-gui/src/link_to_definition/lib.rs

@@ -190,6 +190,7 @@ trigger_files = [
     "src/stage0.json",
     "src/tools/compiletest",
     "src/tools/tidy",
+    "src/tools/rustdoc-gui-test",
 ]
 
 [autolabel."T-infra"]
@@ -640,3 +641,4 @@ style-team = [
 "/src/tools/rustdoc-themes" =                ["rustdoc"]
 "/src/tools/tidy" =                          ["bootstrap"]
 "/src/tools/x" =                             ["bootstrap"]
+"/src/tools/rustdoc-gui-test" =              ["bootstrap", "@ozkanonur"]