Rollup merge of rust-lang#138104 - GuillaumeGomez:simplify-doctest-parsing, r=fmease

Greatly simplify doctest parsing and information extraction

The original process was pretty terrible, as it tried to extract information such as attributes by performing matches over tokens like `#!`, which doesn't work very well considering you can have `#   ! [`, which is valid.

Also, it now does it in one pass: if the parser is happy, then we try to extract information, otherwise we return early.

r? `@fmease`

Author: matthiaskrgr

src/librustdoc/doctest/make.rs

@@ -197,6 +197,7 @@ fn make_test_crate_attrs() {
 assert_eq!(2+2, 4);";
     let expected = "#![allow(unused)]
 #![feature(sick_rad)]
+
 fn main() {
 assert_eq!(2+2, 4);
 }"
@@ -228,8 +229,8 @@ fn make_test_fake_main() {
     let input = "//Ceci n'est pas une `fn main`
 assert_eq!(2+2, 4);";
     let expected = "#![allow(unused)]
-//Ceci n'est pas une `fn main`
 fn main() {
+//Ceci n'est pas une `fn main`
 assert_eq!(2+2, 4);
 }"
     .to_string();
@@ -259,8 +260,8 @@ fn make_test_issues_21299() {
 assert_eq!(2+2, 4);";
 
     let expected = "#![allow(unused)]
-// fn main
 fn main() {
+// fn main
 assert_eq!(2+2, 4);
 }"
     .to_string();
@@ -401,3 +402,76 @@ fn check_split_args() {
     compare("a\n\t \rb", &["a", "b"]);
     compare("a\n\t1 \rb", &["a", "1", "b"]);
 }
+
+#[test]
+fn comment_in_attrs() {
+    // If there is an inline code comment after attributes, we need to ensure that
+    // a backline will be added to prevent generating code "inside" it (and thus generating)
+    // invalid code.
+    let opts = default_global_opts("");
+    let input = "\
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
+#![doc(rust_logo)]
+//! This crate has the Rust(tm) branding on it.";
+    let expected = "\
+#![allow(unused)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
+#![doc(rust_logo)]
+//! This crate has the Rust(tm) branding on it.
+fn main() {
+
+}"
+    .to_string();
+    let (output, len) = make_test(input, None, false, &opts, None);
+    assert_eq!((output, len), (expected, 2));
+
+    // And same, if there is a `main` function provided by the user, we ensure that it's
+    // correctly separated.
+    let input = "\
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
+#![doc(rust_logo)]
+//! This crate has the Rust(tm) branding on it.
+fn main() {}";
+    let expected = "\
+#![allow(unused)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
+#![doc(rust_logo)]
+//! This crate has the Rust(tm) branding on it.
+
+fn main() {}"
+        .to_string();
+    let (output, len) = make_test(input, None, false, &opts, None);
+    assert_eq!((output, len), (expected, 1));
+}
+
+// This test ensures that the only attributes taken into account when we switch between
+// "crate level" content and the rest doesn't include inner attributes span, as it would
+// include part of the item and generate broken code.
+#[test]
+fn inner_attributes() {
+    let opts = default_global_opts("");
+    let input = r#"
+//! A doc comment that applies to the implicit anonymous module of this crate
+
+pub mod outer_module {
+    //!! - Still an inner line doc (but with a bang at the beginning)
+}
+"#;
+    let expected = "#![allow(unused)]
+
+//! A doc comment that applies to the implicit anonymous module of this crate
+
+
+fn main() {
+pub mod outer_module {
+    //!! - Still an inner line doc (but with a bang at the beginning)
+}
+}"
+    .to_string();
+    let (output, len) = make_test(input, None, false, &opts, None);
+    assert_eq!((output, len), (expected, 2));
+}

src/librustdoc/doctest/tests.rs

@@ -58,21 +58,21 @@ $DIR/doctest.rs:
    LL|       |//!
    LL|       |//! doctest with custom main:
    LL|       |//! ```
-   LL|      1|//! fn some_func() {
-   LL|      1|//!     println!("called some_func()");
-   LL|      1|//! }
-   LL|       |//!
-   LL|       |//! #[derive(Debug)]
-   LL|       |//! struct SomeError;
+   LL|       |//! fn some_func() {
+   LL|       |//!     println!("called some_func()");
+   LL|       |//! }
+   LL|      1|//!
+   LL|      1|//! #[derive(Debug)]
+   LL|      1|//! struct SomeError;
    LL|       |//!
    LL|       |//! extern crate doctest_crate;
    LL|       |//!
-   LL|      1|//! fn doctest_main() -> Result<(), SomeError> {
+   LL|       |//! fn doctest_main() -> Result<(), SomeError> {
    LL|      1|//!     some_func();
    LL|      1|//!     doctest_crate::fn_run_in_doctests(2);
    LL|      1|//!     Ok(())
    LL|      1|//! }
-   LL|       |//!
+   LL|      1|//!
    LL|       |//! // this `main` is not shown as covered, as it clashes with all the other
    LL|       |//! // `main` functions that were automatically generated for doctests
    LL|       |//! fn main() -> Result<(), SomeError> {

tests/coverage-run-rustdoc/doctest.coverage

@@ -8,11 +8,11 @@ fn main() {
 
     let should_contain = &[
         "input.rs - foo (line 5)",
-        "input.rs:7:15",
+        "input.rs:8:15",
         "input.rs - bar (line 13)",
-        "input.rs:15:15",
+        "input.rs:16:15",
         "input.rs - bar (line 22)",
-        "input.rs:24:15",
+        "input.rs:25:15",
     ];
     for text in should_contain {
         assert!(output.contains(text), "output doesn't contains {:?}", text);

tests/run-make/rustdoc-error-lines/rmake.rs

@@ -6,7 +6,7 @@ successes:
 
 ---- $DIR/display-output.rs - foo (line 9) stdout ----
 warning: unused variable: `x`
-  --> $DIR/display-output.rs:11:5
+  --> $DIR/display-output.rs:12:5
    |
 LL | let x = 12;
    |     ^ help: if this is intentional, prefix it with an underscore: `_x`
@@ -19,13 +19,13 @@ LL | #![warn(unused)]
    = note: `#[warn(unused_variables)]` implied by `#[warn(unused)]`
 
 warning: unused variable: `x`
-  --> $DIR/display-output.rs:13:8
+  --> $DIR/display-output.rs:14:8
    |
 LL | fn foo(x: &dyn std::fmt::Display) {}
    |        ^ help: if this is intentional, prefix it with an underscore: `_x`
 
 warning: function `foo` is never used
-  --> $DIR/display-output.rs:13:4
+  --> $DIR/display-output.rs:14:4
    |
 LL | fn foo(x: &dyn std::fmt::Display) {}
    |    ^^^

tests/rustdoc-ui/doctest/display-output.stdout

@@ -0,0 +1,23 @@
+//@ check-pass
+//@ compile-flags:--test --test-args=--test-threads=1
+//@ normalize-stdout: "tests/rustdoc-ui/doctest" -> "$$DIR"
+//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
+
+// This test ensures that crate imports are placed outside of the `main` function
+// so they work all the time (even in 2015 edition).
+
+/// ```rust
+/// #![feature(test)]
+///
+/// extern crate test;
+/// use test::Bencher;
+///
+/// #[bench]
+/// fn bench_xor_1000_ints(b: &mut Bencher) {
+///     b.iter(|| {
+///         (0..1000).fold(0, |old, new| old ^ new);
+///     });
+/// }
+/// ```
+///
+pub fn foo() {}

tests/rustdoc-ui/doctest/extern-crate.rs

@@ -0,0 +1,6 @@
+
+running 1 test
+test $DIR/extern-crate.rs - foo (line 9) ... ok
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
+

tests/rustdoc-ui/doctest/extern-crate.stdout

@@ -1 +1 @@
-{"format_version":1,"doctests":[{"file":"$DIR/extract-doctests.rs","line":8,"doctest_attributes":{"original":"ignore (checking attributes)","should_panic":false,"no_run":false,"ignore":"All","rust":true,"test_harness":false,"compile_fail":false,"standalone_crate":false,"error_codes":[],"edition":null,"added_css_classes":[],"unknown":[]},"original_code":"let x = 12;\nlet y = 14;","doctest_code":"#![allow(unused)]\nfn main() {\nlet x = 12;\nlet y = 14;\n}","name":"$DIR/extract-doctests.rs - (line 8)"},{"file":"$DIR/extract-doctests.rs","line":13,"doctest_attributes":{"original":"edition2018,compile_fail","should_panic":false,"no_run":true,"ignore":"None","rust":true,"test_harness":false,"compile_fail":true,"standalone_crate":false,"error_codes":[],"edition":"2018","added_css_classes":[],"unknown":[]},"original_code":"let","doctest_code":"#![allow(unused)]\nfn main() {\nlet\n}","name":"$DIR/extract-doctests.rs - (line 13)"}]}
+{"format_version":1,"doctests":[{"file":"$DIR/extract-doctests.rs","line":8,"doctest_attributes":{"original":"ignore (checking attributes)","should_panic":false,"no_run":false,"ignore":"All","rust":true,"test_harness":false,"compile_fail":false,"standalone_crate":false,"error_codes":[],"edition":null,"added_css_classes":[],"unknown":[]},"original_code":"let x = 12;\nlet y = 14;","doctest_code":"#![allow(unused)]\nfn main() {\nlet x = 12;\nlet y = 14;\n}","name":"$DIR/extract-doctests.rs - (line 8)"},{"file":"$DIR/extract-doctests.rs","line":13,"doctest_attributes":{"original":"edition2018,compile_fail","should_panic":false,"no_run":true,"ignore":"None","rust":true,"test_harness":false,"compile_fail":true,"standalone_crate":false,"error_codes":[],"edition":"2018","added_css_classes":[],"unknown":[]},"original_code":"let","doctest_code":null,"name":"$DIR/extract-doctests.rs - (line 13)"}]}

tests/rustdoc-ui/extract-doctests.stdout

@@ -5,11 +5,11 @@ test remapped_path/remap-path-prefix-invalid-doctest.rs - SomeStruct (line 10) .
 failures:
 
 ---- remapped_path/remap-path-prefix-invalid-doctest.rs - SomeStruct (line 10) stdout ----
-error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found `is`
+error: expected one of `!` or `::`, found `is`
   --> remapped_path/remap-path-prefix-invalid-doctest.rs:11:6
    |
 LL | this is not real code
-   |      ^^ expected one of 8 possible tokens
+   |      ^^ expected one of `!` or `::`
 
 error: aborting due to 1 previous error
 

tests/rustdoc-ui/remap-path-prefix-invalid-doctest.stdout

@@ -24,4 +24,4 @@
 
 //@ matches foo/index.html '//a[@class="test-arrow"][@href="https://www.example.com/?code=%23!%5Ballow(unused)%5D%0Afn+main()+%7B%0A++++println!(%22Hello,+world!%22);%0A%7D&edition=2015"]' ""
 //@ matches foo/index.html '//a[@class="test-arrow"][@href="https://www.example.com/?code=%23!%5Ballow(unused)%5D%0Afn+main()+%7B%0A++++println!(%22Hello,+world!%22);%0A%7D&edition=2015"]' ""
-//@ matches foo/index.html '//a[@class="test-arrow"][@href="https://www.example.com/?code=%23!%5Ballow(unused)%5D%0A%23!%5Bfeature(something)%5D%0A%0Afn+main()+%7B%0A++++println!(%22Hello,+world!%22);%0A%7D&version=nightly&edition=2015"]' ""
+//@ matches foo/index.html '//a[@class="test-arrow"][@href="https://www.example.com/?code=%23!%5Ballow(unused)%5D%0A%23!%5Bfeature(something)%5D%0A%0A%0Afn+main()+%7B%0A++++println!(%22Hello,+world!%22);%0A%7D&version=nightly&edition=2015"]' ""