expand: Stop using nonterminals for passing tokens to attribute and derive macros

Author: petrochenkov

src/librustc_ast/token.rs

@@ -11,7 +11,7 @@ use crate::tokenstream::TokenTree;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
 use rustc_macros::HashStable_Generic;
-use rustc_span::symbol::kw;
+use rustc_span::symbol::{kw, sym};
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::{self, Span, DUMMY_SP};
 use std::borrow::Cow;
@@ -785,6 +785,26 @@ impl Nonterminal {
             NtTT(tt) => tt.span(),
         }
     }
+
+    /// This nonterminal looks like some specific enums from
+    /// `proc-macro-hack` and `procedural-masquerade` crates.
+    /// We need to maintain some special pretty-printing behavior for them due to incorrect
+    /// asserts in old versions of those crates and their wide use in the ecosystem.
+    /// See issue #73345 for more details.
+    /// FIXME: Remove this eventually.
+    pub fn pretty_printing_compatibility_hack(&self) -> bool {
+        if let NtItem(item) = self {
+            let name = item.ident.name;
+            if name == sym::ProceduralMasqueradeDummyType || name == sym::ProcMacroHack {
+                if let ast::ItemKind::Enum(enum_def, _) = &item.kind {
+                    if let [variant] = &*enum_def.variants {
+                        return variant.ident.name == sym::Input;
+                    }
+                }
+            }
+        }
+        false
+    }
 }
 
 impl PartialEq for Nonterminal {

src/librustc_ast_pretty/pprust.rs

@@ -148,9 +148,14 @@ pub fn to_string(f: impl FnOnce(&mut State<'_>)) -> String {
     printer.s.eof()
 }
 
-// This makes comma-separated lists look slightly nicer,
-// and also addresses a specific regression described in issue #63896.
+// This makes printed token streams look slightly nicer,
+// and also addresses some specific regressions described in #63896 and #73345.
 fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
+    if let TokenTree::Token(token) = prev {
+        if let token::DocComment(s) = token.kind {
+            return !s.as_str().starts_with("//");
+        }
+    }
     match tt {
         TokenTree::Token(token) => match token.kind {
             token::Comma => false,
@@ -163,7 +168,14 @@ fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
             },
             _ => true,
         },
-        _ => true,
+        TokenTree::Delimited(_, DelimToken::Bracket, _) => match prev {
+            TokenTree::Token(token) => match token.kind {
+                token::Pound => false,
+                _ => true,
+            },
+            _ => true,
+        },
+        TokenTree::Delimited(..) => true,
     }
 }
 

src/librustc_expand/base.rs

@@ -4,14 +4,14 @@ use crate::module::DirectoryOwnership;
 use rustc_ast::ast::{self, Attribute, NodeId, PatKind};
 use rustc_ast::mut_visit::{self, MutVisitor};
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, FlattenGroup};
-use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
+use rustc_ast::token;
+use rustc_ast::tokenstream::{self, TokenStream};
 use rustc_ast::visit::{AssocCtxt, Visitor};
 use rustc_attr::{self as attr, Deprecation, HasAttrs, Stability};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::{self, Lrc};
 use rustc_errors::{DiagnosticBuilder, ErrorReported};
-use rustc_parse::{self, parser, MACRO_ARGUMENTS};
+use rustc_parse::{self, nt_to_tokenstream, parser, MACRO_ARGUMENTS};
 use rustc_session::{parse::ParseSess, Limit};
 use rustc_span::def_id::DefId;
 use rustc_span::edition::Edition;
@@ -120,10 +120,7 @@ impl Annotatable {
         }
     }
 
-    crate fn into_tokens(self) -> TokenStream {
-        // `Annotatable` can be converted into tokens directly, but we
-        // are packing it into a nonterminal as a piece of AST to make
-        // the produced token stream look nicer in pretty-printed form.
+    crate fn into_tokens(self, sess: &ParseSess) -> TokenStream {
         let nt = match self {
             Annotatable::Item(item) => token::NtItem(item),
             Annotatable::TraitItem(item) | Annotatable::ImplItem(item) => {
@@ -142,7 +139,7 @@ impl Annotatable {
             | Annotatable::StructField(..)
             | Annotatable::Variant(..) => panic!("unexpected annotatable"),
         };
-        TokenTree::token(token::Interpolated(Lrc::new(nt), FlattenGroup::Yes), DUMMY_SP).into()
+        nt_to_tokenstream(&nt, sess, DUMMY_SP)
     }
 
     pub fn expect_item(self) -> P<ast::Item> {

src/librustc_expand/expand.rs

@@ -705,7 +705,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 SyntaxExtensionKind::Attr(expander) => {
                     self.gate_proc_macro_input(&item);
                     self.gate_proc_macro_attr_item(span, &item);
-                    let tokens = item.into_tokens();
+                    let tokens = item.into_tokens(self.cx.parse_sess);
                     let attr_item = attr.unwrap_normal_item();
                     if let MacArgs::Eq(..) = attr_item.args {
                         self.cx.span_err(span, "key-value macro attributes are not supported");

src/librustc_expand/proc_macro.rs

@@ -3,9 +3,10 @@ use crate::proc_macro_server;
 
 use rustc_ast::ast::{self, ItemKind, MetaItemKind, NestedMetaItem};
 use rustc_ast::token::{self, FlattenGroup};
-use rustc_ast::tokenstream::{self, TokenStream};
+use rustc_ast::tokenstream::{TokenStream, TokenTree};
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, ErrorReported};
+use rustc_parse::nt_to_tokenstream;
 use rustc_span::symbol::sym;
 use rustc_span::{Span, DUMMY_SP};
 
@@ -102,8 +103,13 @@ impl MultiItemModifier for ProcMacroDerive {
             }
         }
 
-        let token = token::Interpolated(Lrc::new(token::NtItem(item)), FlattenGroup::Yes);
-        let input = tokenstream::TokenTree::token(token, DUMMY_SP).into();
+        let item = token::NtItem(item);
+        let input = if item.pretty_printing_compatibility_hack() {
+            TokenTree::token(token::Interpolated(Lrc::new(item), FlattenGroup::Yes), DUMMY_SP)
+                .into()
+        } else {
+            nt_to_tokenstream(&item, ecx.parse_sess, DUMMY_SP)
+        };
 
         let server = proc_macro_server::Rustc::new(ecx);
         let stream = match self.client.run(&EXEC_STRATEGY, server, input) {

src/librustc_expand/proc_macro_server.rs

@@ -180,13 +180,17 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
                 tt!(Punct::new('#', false))
             }
 
-            Interpolated(nt, flatten) => {
+            Interpolated(nt, _) => {
                 let stream = nt_to_tokenstream(&nt, sess, span);
                 TokenTree::Group(Group {
                     delimiter: Delimiter::None,
                     stream,
                     span: DelimSpan::from_single(span),
-                    flatten,
+                    flatten: if nt.pretty_printing_compatibility_hack() {
+                        FlattenGroup::Yes
+                    } else {
+                        FlattenGroup::No
+                    },
                 })
             }
 

src/librustc_span/symbol.rs

@@ -401,6 +401,7 @@ symbols! {
         infer_outlives_requirements,
         infer_static_outlives_requirements,
         inline,
+        Input,
         intel,
         into_iter,
         IntoIterator,
@@ -588,6 +589,8 @@ symbols! {
         proc_macro_mod,
         proc_macro_non_items,
         proc_macro_path_invoc,
+        ProceduralMasqueradeDummyType,
+        ProcMacroHack,
         profiler_builtins,
         profiler_runtime,
         ptr_guaranteed_eq,

src/test/run-make/rustc-macro-dep-files/foo.rs

@@ -7,6 +7,6 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(A)]
 pub fn derive(input: TokenStream) -> TokenStream {
     let input = input.to_string();
-    assert!(input.contains("struct A;"));
+    assert!(input.contains("struct A ;"));
     "struct B;".parse().unwrap()
 }

src/test/ui/async-await/issues/issue-60674.stdout

@@ -1,3 +1,3 @@
-async fn f(mut x: u8) { }
-async fn g((mut x, y, mut z): (u8, u8, u8)) { }
-async fn g(mut x: u8, (a, mut b, c): (u8, u8, u8), y: u8) { }
+async fn f(mut x : u8) { }
+async fn g((mut x, y, mut z) : (u8, u8, u8)) { }
+async fn g(mut x : u8, (a, mut b, c) : (u8, u8, u8), y : u8) { }

src/test/ui/proc-macro/auxiliary/attr-cfg.rs

@@ -11,11 +11,9 @@ use proc_macro::TokenStream;
 pub fn attr_cfg(args: TokenStream, input: TokenStream) -> TokenStream {
     let input_str = input.to_string();
 
-    assert_eq!(input_str, "fn outer() -> u8 {
-    #[cfg(foo)]
-    fn inner() -> u8 { 1 }
-    #[cfg(bar)]
-    fn inner() -> u8 { 2 }
+    assert_eq!(input_str, "fn outer() -> u8
+{
+    #[cfg(foo)] fn inner() -> u8 { 1 } #[cfg(bar)] fn inner() -> u8 { 2 }
     inner()
 }");
 

src/test/ui/proc-macro/auxiliary/attr-stmt-expr-rpass.rs

@@ -10,14 +10,14 @@ use proc_macro::TokenStream;
 #[proc_macro_attribute]
 pub fn expect_let(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "let string = \"Hello, world!\";");
+    assert_eq!(item.to_string(), "let string = \"Hello, world!\" ;");
     item
 }
 
 #[proc_macro_attribute]
 pub fn expect_print_stmt(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "println!(\"{}\", string);");
+    assert_eq!(item.to_string(), "println ! (\"{}\", string) ;");
     item
 }
 
@@ -31,7 +31,7 @@ pub fn expect_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
 #[proc_macro_attribute]
 pub fn expect_print_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "println!(\"{}\", string)");
+    assert_eq!(item.to_string(), "println ! (\"{}\", string)");
     item
 }
 

src/test/ui/proc-macro/auxiliary/attr-stmt-expr.rs

@@ -10,14 +10,14 @@ use proc_macro::TokenStream;
 #[proc_macro_attribute]
 pub fn expect_let(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "let string = \"Hello, world!\";");
+    assert_eq!(item.to_string(), "let string = \"Hello, world!\" ;");
     item
 }
 
 #[proc_macro_attribute]
 pub fn expect_print_stmt(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "println!(\"{}\", string);");
+    assert_eq!(item.to_string(), "println ! (\"{}\", string) ;");
     item
 }
 
@@ -31,7 +31,7 @@ pub fn expect_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
 #[proc_macro_attribute]
 pub fn expect_print_expr(attr: TokenStream, item: TokenStream) -> TokenStream {
     assert!(attr.to_string().is_empty());
-    assert_eq!(item.to_string(), "println!(\"{}\", string)");
+    assert_eq!(item.to_string(), "println ! (\"{}\", string)");
     item
 }
 

src/test/ui/proc-macro/auxiliary/derive-a.rs

@@ -10,6 +10,6 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(A)]
 pub fn derive(input: TokenStream) -> TokenStream {
     let input = input.to_string();
-    assert!(input.contains("struct A;"));
+    assert!(input.contains("struct A ;"));
     "".parse().unwrap()
 }

src/test/ui/proc-macro/auxiliary/derive-atob.rs

@@ -10,6 +10,6 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(AToB)]
 pub fn derive(input: TokenStream) -> TokenStream {
     let input = input.to_string();
-    assert_eq!(input, "struct A;");
+    assert_eq!(input, "struct A ;");
     "struct B;".parse().unwrap()
 }

src/test/ui/proc-macro/auxiliary/derive-b-rpass.rs

@@ -10,7 +10,7 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(B, attributes(B, C))]
 pub fn derive(input: TokenStream) -> TokenStream {
     let input = input.to_string();
-    assert!(input.contains("#[B[arbitrary tokens]]"));
+    assert!(input.contains("#[B [arbitrary tokens]]"));
     assert!(input.contains("struct B {"));
     assert!(input.contains("#[C]"));
     "".parse().unwrap()

src/test/ui/proc-macro/auxiliary/derive-ctod.rs

@@ -10,6 +10,6 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(CToD)]
 pub fn derive(input: TokenStream) -> TokenStream {
     let input = input.to_string();
-    assert_eq!(input, "struct C;");
+    assert_eq!(input, "struct C ;");
     "struct D;".parse().unwrap()
 }

src/test/ui/proc-macro/auxiliary/derive-same-struct.rs

@@ -10,12 +10,12 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(AToB)]
 pub fn derive1(input: TokenStream) -> TokenStream {
     println!("input1: {:?}", input.to_string());
-    assert_eq!(input.to_string(), "struct A;");
+    assert_eq!(input.to_string(), "struct A ;");
     "#[derive(BToC)] struct B;".parse().unwrap()
 }
 
 #[proc_macro_derive(BToC)]
 pub fn derive2(input: TokenStream) -> TokenStream {
-    assert_eq!(input.to_string(), "struct B;");
+    assert_eq!(input.to_string(), "struct B ;");
     "struct C;".parse().unwrap()
 }

src/test/ui/proc-macro/auxiliary/derive-union.rs

@@ -12,7 +12,7 @@ pub fn derive(input: TokenStream) -> TokenStream {
     let input = input.to_string();
     assert!(input.contains("#[repr(C)]"));
     assert!(input.contains("union Test {"));
-    assert!(input.contains("a: u8,"));
+    assert!(input.contains("a : u8,"));
     assert!(input.contains("}"));
     "".parse().unwrap()
 }

src/test/ui/proc-macro/auxiliary/double.rs

@@ -1,15 +1,16 @@
 // force-host
 // no-prefer-dynamic
 
+#![feature(proc_macro_quote)]
+
 #![crate_type = "proc-macro"]
 
 extern crate proc_macro;
-
-use proc_macro::TokenStream;
+use proc_macro::*;
 
 // Outputs another copy of the struct.  Useful for testing the tokens
 // seen by the proc_macro.
 #[proc_macro_derive(Double)]
 pub fn derive(input: TokenStream) -> TokenStream {
-    format!("mod foo {{ {} }}", input.to_string()).parse().unwrap()
+    quote!(mod foo { $input })
 }

src/test/ui/proc-macro/auxiliary/expand-with-a-macro.rs

@@ -11,7 +11,7 @@ use proc_macro::TokenStream;
 #[proc_macro_derive(A)]
 pub fn derive(input: TokenStream) -> TokenStream {
     let input = input.to_string();
-    assert!(input.contains("struct A;"));
+    assert!(input.contains("struct A ;"));
     r#"
         impl A {
             fn a(&self) {

src/test/ui/proc-macro/derive-same-struct.stdout

@@ -1 +1 @@
-input1: "struct A;"
+input1: "struct A ;"

src/test/ui/proc-macro/dollar-crate-issue-57089.stdout

@@ -38,8 +38,7 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
         span: #3 bytes(LO..HI),
     },
 ]
-PRINT-ATTR INPUT (DISPLAY): struct A(crate::S);
-PRINT-ATTR RE-COLLECTED (DISPLAY): struct A($crate :: S) ;
+PRINT-ATTR INPUT (DISPLAY): struct A($crate :: S) ;
 PRINT-ATTR INPUT (DEBUG): TokenStream [
     Ident {
         ident: "struct",

src/test/ui/proc-macro/dollar-crate-issue-62325.stdout

@@ -1,5 +1,4 @@
-PRINT-ATTR INPUT (DISPLAY): struct A(identity!(crate :: S));
-PRINT-ATTR RE-COLLECTED (DISPLAY): struct A(identity ! ($crate :: S)) ;
+PRINT-ATTR INPUT (DISPLAY): struct A(identity ! ($crate :: S)) ;
 PRINT-ATTR INPUT (DEBUG): TokenStream [
     Ident {
         ident: "struct",
@@ -54,8 +53,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
         span: #3 bytes(LO..HI),
     },
 ]
-PRINT-ATTR INPUT (DISPLAY): struct B(identity!(::dollar_crate_external :: S));
-PRINT-ATTR RE-COLLECTED (DISPLAY): struct B(identity ! ($crate :: S)) ;
+PRINT-ATTR INPUT (DISPLAY): struct B(identity ! ($crate :: S)) ;
 PRINT-ATTR INPUT (DEBUG): TokenStream [
     Ident {
         ident: "struct",