Attach TokenStream to ast::Ty

Aaron1011 · Aaron1011 · commit 1823dea7df9e · 2020-09-10T17:33:05.000-04:00
A `Ty` does not have outer attributes, so we only capture tokens
when parsing a `macro_rules!` matcher
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
@@ -587,7 +587,7 @@ impl Pat {
             _ => return None,
         };
 
-        Some(P(Ty { kind, id: self.id, span: self.span }))
+        Some(P(Ty { kind, id: self.id, span: self.span, tokens: None }))
     }
 
     /// Walk top-down and call `it` in each place where a pattern occurs
@@ -1169,7 +1169,7 @@ impl Expr {
             _ => return None,
         };
 
-        Some(P(Ty { kind, id: self.id, span: self.span }))
+        Some(P(Ty { kind, id: self.id, span: self.span, tokens: None }))
     }
 
     pub fn precedence(&self) -> ExprPrecedence {
@@ -1867,6 +1867,7 @@ pub struct Ty {
     pub id: NodeId,
     pub kind: TyKind,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -2145,7 +2146,7 @@ impl Param {
     /// Builds a `Param` object from `ExplicitSelf`.
     pub fn from_self(attrs: AttrVec, eself: ExplicitSelf, eself_ident: Ident) -> Param {
         let span = eself.span.to(eself_ident.span);
-        let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span });
+        let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span, tokens: None });
         let param = |mutbl, ty| Param {
             attrs,
             pat: P(Pat {
@@ -2168,6 +2169,7 @@ impl Param {
                     id: DUMMY_NODE_ID,
                     kind: TyKind::Rptr(lt, MutTy { ty: infer_ty, mutbl }),
                     span,
+                    tokens: None,
                 }),
             ),
         }
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
@@ -451,7 +451,7 @@ pub fn noop_visit_ty_constraint<T: MutVisitor>(
 }
 
 pub fn noop_visit_ty<T: MutVisitor>(ty: &mut P<Ty>, vis: &mut T) {
-    let Ty { id, kind, span } = ty.deref_mut();
+    let Ty { id, kind, span, tokens: _ } = ty.deref_mut();
     vis.visit_id(id);
     match kind {
         TyKind::Infer | TyKind::ImplicitSelf | TyKind::Err | TyKind::Never | TyKind::CVarArgs => {}
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
@@ -1106,6 +1106,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                                 id: node_id,
                                 kind: TyKind::ImplTrait(impl_trait_node_id, bounds.clone()),
                                 span: constraint.span,
+                                tokens: None,
                             },
                             itctx,
                         );
diff --git a/compiler/rustc_builtin_macros/src/concat_idents.rs b/compiler/rustc_builtin_macros/src/concat_idents.rs
@@ -61,6 +61,7 @@ pub fn expand_concat_idents<'cx>(
                 id: ast::DUMMY_NODE_ID,
                 kind: ast::TyKind::Path(None, ast::Path::from_ident(self.ident)),
                 span: self.ident.span,
+                tokens: None,
             }))
         }
     }
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
@@ -607,6 +607,7 @@ impl DummyResult {
             id: ast::DUMMY_NODE_ID,
             kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) },
             span: sp,
+            tokens: None,
         })
     }
 }
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
@@ -54,7 +54,7 @@ impl<'a> ExtCtxt<'a> {
     }
 
     pub fn ty(&self, span: Span, kind: ast::TyKind) -> P<ast::Ty> {
-        P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind })
+        P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind, tokens: None })
     }
 
     pub fn ty_path(&self, path: ast::Path) -> P<ast::Ty> {
diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs
@@ -37,7 +37,8 @@ pub fn placeholder(
             tokens: None,
         })
     };
-    let ty = || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span });
+    let ty =
+        || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span, tokens: None });
     let pat =
         || P(ast::Pat { id, kind: ast::PatKind::MacCall(mac_placeholder()), span, tokens: None });
 
@@ -88,9 +89,12 @@ pub fn placeholder(
             kind: ast::PatKind::MacCall(mac_placeholder()),
             tokens: None,
         })),
-        AstFragmentKind::Ty => {
-            AstFragment::Ty(P(ast::Ty { id, span, kind: ast::TyKind::MacCall(mac_placeholder()) }))
-        }
+        AstFragmentKind::Ty => AstFragment::Ty(P(ast::Ty {
+            id,
+            span,
+            kind: ast::TyKind::MacCall(mac_placeholder()),
+            tokens: None,
+        })),
         AstFragmentKind::Stmts => AstFragment::Stmts(smallvec![{
             let mac = P(ast::MacCallStmt {
                 mac: mac_placeholder(),
diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs
@@ -270,6 +270,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
         }
         Nonterminal::NtBlock(ref block) => block.tokens.clone(),
         Nonterminal::NtPat(ref pat) => pat.tokens.clone(),
+        Nonterminal::NtTy(ref ty) => ty.tokens.clone(),
         Nonterminal::NtIdent(ident, is_raw) => {
             Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into())
         }
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -28,7 +28,7 @@ pub(super) fn dummy_arg(ident: Ident) -> Param {
         span: ident.span,
         tokens: None,
     });
-    let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID };
+    let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID, tokens: None };
     Param {
         attrs: AttrVec::default(),
         id: ast::DUMMY_NODE_ID,
@@ -75,7 +75,12 @@ impl RecoverQPath for Ty {
         Some(P(self.clone()))
     }
     fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self {
-        Self { span: path.span, kind: TyKind::Path(qself, path), id: ast::DUMMY_NODE_ID }
+        Self {
+            span: path.span,
+            kind: TyKind::Path(qself, path),
+            id: ast::DUMMY_NODE_ID,
+            tokens: None,
+        }
     }
 }
 
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
@@ -510,7 +510,12 @@ impl<'a> Parser<'a> {
         {
             let span = self.prev_token.span.between(self.token.span);
             self.struct_span_err(span, "missing trait in a trait impl").emit();
-            P(Ty { kind: TyKind::Path(None, err_path(span)), span, id: DUMMY_NODE_ID })
+            P(Ty {
+                kind: TyKind::Path(None, err_path(span)),
+                span,
+                id: DUMMY_NODE_ID,
+                tokens: None,
+            })
         } else {
             self.parse_ty()?
         };
@@ -1046,7 +1051,7 @@ impl<'a> Parser<'a> {
 
         // The user intended that the type be inferred,
         // so treat this as if the user wrote e.g. `const A: _ = expr;`.
-        P(Ty { kind: TyKind::Infer, span: id.span, id: ast::DUMMY_NODE_ID })
+        P(Ty { kind: TyKind::Infer, span: id.span, id: ast::DUMMY_NODE_ID, tokens: None })
     }
 
     /// Parses an enum declaration.
diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs
@@ -141,7 +141,14 @@ impl<'a> Parser<'a> {
                 token::NtExpr(expr)
             }
             NonterminalKind::Literal => token::NtLiteral(self.parse_literal_maybe_minus()?),
-            NonterminalKind::Ty => token::NtTy(self.parse_ty()?),
+            NonterminalKind::Ty => {
+                let (mut ty, tokens) = self.collect_tokens(|this| this.parse_ty())?;
+                // We have an eaten an NtTy, which could already have tokens
+                if ty.tokens.is_none() {
+                    ty.tokens = Some(tokens);
+                }
+                token::NtTy(ty)
+            }
             // this could be handled like a token, since it is one
             NonterminalKind::Ident => {
                 if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
@@ -626,6 +626,6 @@ impl<'a> Parser<'a> {
     }
 
     pub(super) fn mk_ty(&self, span: Span, kind: TyKind) -> P<Ty> {
-        P(Ty { kind, span, id: ast::DUMMY_NODE_ID })
+        P(Ty { kind, span, id: ast::DUMMY_NODE_ID, tokens: None })
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -451,7 +451,7 @@ pub fn noop_visit_ty_constraint<T: MutVisitor>(`
`451`	`451`	`}`
`452`	`452`
`453`	`453`	`pub fn noop_visit_ty<T: MutVisitor>(ty: &mut P<Ty>, vis: &mut T) {`
`454`		`- let Ty { id, kind, span } = ty.deref_mut();`
	`454`	`+ let Ty { id, kind, span, tokens: _ } = ty.deref_mut();`
`455`	`455`	`vis.visit_id(id);`
`456`	`456`	`match kind {`
`457`	`457`	`TyKind::Infer \| TyKind::ImplicitSelf \| TyKind::Err \| TyKind::Never \| TyKind::CVarArgs => {}`
Original file line number	Diff line number	Diff line change
`@@ -61,6 +61,7 @@ pub fn expand_concat_idents<'cx>(`
`61`	`61`	`id: ast::DUMMY_NODE_ID,`
`62`	`62`	`kind: ast::TyKind::Path(None, ast::Path::from_ident(self.ident)),`
`63`	`63`	`span: self.ident.span,`
	`64`	`+ tokens: None,`
`64`	`65`	`}))`
`65`	`66`	`}`
`66`	`67`	`}`
Original file line number	Diff line number	Diff line change
`@@ -607,6 +607,7 @@ impl DummyResult {`
`607`	`607`	`id: ast::DUMMY_NODE_ID,`
`608`	`608`	`kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) },`
`609`	`609`	`span: sp,`
	`610`	`+ tokens: None,`
`610`	`611`	`})`
`611`	`612`	`}`
`612`	`613`	`}`
Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@ impl<'a> ExtCtxt<'a> {`
`54`	`54`	`}`
`55`	`55`
`56`	`56`	`pub fn ty(&self, span: Span, kind: ast::TyKind) -> P<ast::Ty> {`
`57`		`- P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind })`
	`57`	`+ P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind, tokens: None })`
`58`	`58`	`}`
`59`	`59`
`60`	`60`	`pub fn ty_path(&self, path: ast::Path) -> P<ast::Ty> {`
Original file line number	Diff line number	Diff line change
`@@ -270,6 +270,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke`
`270`	`270`	`}`
`271`	`271`	`Nonterminal::NtBlock(ref block) => block.tokens.clone(),`
`272`	`272`	`Nonterminal::NtPat(ref pat) => pat.tokens.clone(),`
	`273`	`+ Nonterminal::NtTy(ref ty) => ty.tokens.clone(),`
`273`	`274`	`Nonterminal::NtIdent(ident, is_raw) => {`
`274`	`275`	`Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into())`
`275`	`276`	`}`