Skip to content

Commit 3b977bb

Browse files
nnethercotennethercote
committed
Add Spacing to mbe::TokenTree::Token.
`tokenstream::TokenTree::Token` already has a `Spacing` field, and this commit adds it to the `mbe` equivalent. This improves output in some tests: some involving `-Ztrace-macros`, and some involving `PRINT-BANG`. Also, tests/ui/macros/macro-first-set.rs has an interesting change to some `stringify!` output. The commit adds a comment explaining what happened here. As usual, the output of `stringify!` is subject to change, and when this test was first added the expected output was "a , b , c , d , e", so it has already changed once. Also, the use of macro repetition within `stringify!` seems pretty unusual. Hopefully this won't cause problems with existing crates.
1 parent d8bdab7 commit 3b977bb

16 files changed

+96
-73
lines changed

‎compiler/rustc_expand/src/mbe.rs

+6-6
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ mod transcribe;
1414

1515
use metavar_expr::MetaVarExpr;
1616
use rustc_ast::token::{Delimiter, NonterminalKind, Token, TokenKind};
17-
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan};
17+
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing};
1818
use rustc_macros::{Decodable, Encodable};
1919
use rustc_span::symbol::Ident;
2020
use rustc_span::Span;
@@ -68,7 +68,7 @@ pub(crate) enum KleeneOp {
6868
/// `MetaVarExpr` are "first-class" token trees. Useful for parsing macros.
6969
#[derive(Debug, PartialEq, Encodable, Decodable)]
7070
enum TokenTree {
71-
Token(Token),
71+
Token(Token, Spacing),
7272
/// A delimited sequence, e.g. `($e:expr)` (RHS) or `{ $e }` (LHS).
7373
Delimited(DelimSpan, DelimSpacing, Delimited),
7474
/// A kleene-style repetition sequence, e.g. `$($e:expr)*` (RHS) or `$($e),*` (LHS).
@@ -90,15 +90,15 @@ impl TokenTree {
9090
/// Returns `true` if the given token tree is a token of the given kind.
9191
fn is_token(&self, expected_kind: &TokenKind) -> bool {
9292
match self {
93-
TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind,
93+
TokenTree::Token(Token { kind: actual_kind, .. }, _) => actual_kind == expected_kind,
9494
_ => false,
9595
}
9696
}
9797

9898
/// Retrieves the `TokenTree`'s span.
9999
fn span(&self) -> Span {
100100
match *self {
101-
TokenTree::Token(Token { span, .. })
101+
TokenTree::Token(Token { span, .. }, _)
102102
| TokenTree::MetaVar(span, _)
103103
| TokenTree::MetaVarDecl(span, _, _) => span,
104104
TokenTree::Delimited(span, ..)
@@ -107,7 +107,7 @@ impl TokenTree {
107107
}
108108
}
109109

110-
fn token(kind: TokenKind, span: Span) -> TokenTree {
111-
TokenTree::Token(Token::new(kind, span))
110+
fn token_alone(kind: TokenKind, span: Span) -> TokenTree {
111+
TokenTree::Token(Token::new(kind, span), Spacing::Alone)
112112
}
113113
}

‎compiler/rustc_expand/src/mbe/macro_check.rs

+4-4
Original file line numberDiff line numberDiff line change
@@ -412,7 +412,7 @@ fn check_nested_occurrences(
412412
match (state, tt) {
413413
(
414414
NestedMacroState::Empty,
415-
&TokenTree::Token(Token { kind: TokenKind::Ident(name, IdentIsRaw::No), .. }),
415+
&TokenTree::Token(Token { kind: TokenKind::Ident(name, IdentIsRaw::No), .. }, _),
416416
) => {
417417
if name == kw::MacroRules {
418418
state = NestedMacroState::MacroRules;
@@ -422,13 +422,13 @@ fn check_nested_occurrences(
422422
}
423423
(
424424
NestedMacroState::MacroRules,
425-
&TokenTree::Token(Token { kind: TokenKind::Not, .. }),
425+
&TokenTree::Token(Token { kind: TokenKind::Not, .. }, _),
426426
) => {
427427
state = NestedMacroState::MacroRulesNot;
428428
}
429429
(
430430
NestedMacroState::MacroRulesNot,
431-
&TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }),
431+
&TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }, _),
432432
) => {
433433
state = NestedMacroState::MacroRulesNotName;
434434
}
@@ -459,7 +459,7 @@ fn check_nested_occurrences(
459459
}
460460
(
461461
NestedMacroState::Macro,
462-
&TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }),
462+
&TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }, _),
463463
) => {
464464
state = NestedMacroState::MacroName;
465465
}

‎compiler/rustc_expand/src/mbe/macro_parser.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ pub(super) fn compute_locs(matcher: &[TokenTree]) -> Vec<MatcherLoc> {
180180
) {
181181
for tt in tts {
182182
match tt {
183-
TokenTree::Token(token) => {
183+
TokenTree::Token(token, _) => {
184184
locs.push(MatcherLoc::Token { token: token.clone() });
185185
}
186186
TokenTree::Delimited(span, _, delimited) => {

‎compiler/rustc_expand/src/mbe/macro_rules.rs

+20-17
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use crate::mbe::transcribe::transcribe;
1111
use ast::token::IdentIsRaw;
1212
use rustc_ast as ast;
1313
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind, TokenKind::*};
14-
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
14+
use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream};
1515
use rustc_ast::{NodeId, DUMMY_NODE_ID};
1616
use rustc_ast_pretty::pprust;
1717
use rustc_attr::{self as attr, TransparencyError};
@@ -402,7 +402,7 @@ pub fn compile_declarative_macro(
402402
mbe::SequenceRepetition {
403403
tts: vec![
404404
mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec),
405-
mbe::TokenTree::token(token::FatArrow, def.span),
405+
mbe::TokenTree::token_alone(token::FatArrow, def.span),
406406
mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec),
407407
],
408408
separator: Some(Token::new(
@@ -417,7 +417,7 @@ pub fn compile_declarative_macro(
417417
mbe::TokenTree::Sequence(
418418
DelimSpan::dummy(),
419419
mbe::SequenceRepetition {
420-
tts: vec![mbe::TokenTree::token(
420+
tts: vec![mbe::TokenTree::token_alone(
421421
if macro_rules { token::Semi } else { token::Comma },
422422
def.span,
423423
)],
@@ -627,10 +627,11 @@ fn is_empty_token_tree(sess: &Session, seq: &mbe::SequenceRepetition) -> bool {
627627
while let Some(tt) = iter.next() {
628628
match tt {
629629
mbe::TokenTree::MetaVarDecl(_, _, Some(NonterminalKind::Vis)) => {}
630-
mbe::TokenTree::Token(t @ Token { kind: DocComment(..), .. }) => {
630+
mbe::TokenTree::Token(t @ Token { kind: DocComment(..), .. }, _) => {
631631
let mut now = t;
632632
while let Some(&mbe::TokenTree::Token(
633633
next @ Token { kind: DocComment(..), .. },
634+
_,
634635
)) = iter.peek()
635636
{
636637
now = next;
@@ -696,10 +697,10 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
696697
match rhs {
697698
mbe::TokenTree::Delimited(.., d) => {
698699
let has_compile_error = d.tts.array_windows::<3>().any(|[ident, bang, args]| {
699-
if let mbe::TokenTree::Token(ident) = ident
700+
if let mbe::TokenTree::Token(ident, _) = ident
700701
&& let TokenKind::Ident(ident, _) = ident.kind
701702
&& ident == sym::compile_error
702-
&& let mbe::TokenTree::Token(bang) = bang
703+
&& let mbe::TokenTree::Token(bang, _) = bang
703704
&& let TokenKind::Not = bang.kind
704705
&& let mbe::TokenTree::Delimited(.., del) = args
705706
&& del.delim != Delimiter::Invisible
@@ -896,7 +897,9 @@ enum TtHandle<'tt> {
896897

897898
impl<'tt> TtHandle<'tt> {
898899
fn from_token(tok: Token) -> Self {
899-
TtHandle::Token(mbe::TokenTree::Token(tok))
900+
// `Spacing::Alone` is pessimistic but changing it has no effect on the
901+
// current test suite.
902+
TtHandle::Token(mbe::TokenTree::Token(tok, Spacing::Alone))
900903
}
901904

902905
fn from_token_kind(kind: TokenKind, span: Span) -> Self {
@@ -925,8 +928,8 @@ impl<'tt> Clone for TtHandle<'tt> {
925928

926929
// This variant *must* contain a `mbe::TokenTree::Token`, and not
927930
// any other variant of `mbe::TokenTree`.
928-
TtHandle::Token(mbe::TokenTree::Token(tok)) => {
929-
TtHandle::Token(mbe::TokenTree::Token(tok.clone()))
931+
TtHandle::Token(mbe::TokenTree::Token(tok, spacing)) => {
932+
TtHandle::Token(mbe::TokenTree::Token(tok.clone(), *spacing))
930933
}
931934

932935
_ => unreachable!(),
@@ -1143,7 +1146,7 @@ fn check_matcher_core<'tt>(
11431146
// whereas macros from an external crate have a dummy id.
11441147
if def.id != DUMMY_NODE_ID
11451148
&& matches!(kind, NonterminalKind::PatParam { inferred: true })
1146-
&& matches!(next_token, TokenTree::Token(token) if token.kind == BinOp(token::BinOpToken::Or))
1149+
&& matches!(next_token, TokenTree::Token(token, _) if token.kind == BinOp(token::BinOpToken::Or))
11471150
{
11481151
// It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
11491152
let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
@@ -1274,7 +1277,7 @@ enum IsInFollow {
12741277
fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
12751278
use mbe::TokenTree;
12761279

1277-
if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok {
1280+
if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }, _) = *tok {
12781281
// closing a token tree can never be matched by any fragment;
12791282
// iow, we always require that `(` and `)` match, etc.
12801283
IsInFollow::Yes
@@ -1293,7 +1296,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
12931296
NonterminalKind::Stmt | NonterminalKind::Expr => {
12941297
const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"];
12951298
match tok {
1296-
TokenTree::Token(token) => match token.kind {
1299+
TokenTree::Token(token, _) => match token.kind {
12971300
FatArrow | Comma | Semi => IsInFollow::Yes,
12981301
_ => IsInFollow::No(TOKENS),
12991302
},
@@ -1303,7 +1306,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13031306
NonterminalKind::PatParam { .. } => {
13041307
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"];
13051308
match tok {
1306-
TokenTree::Token(token) => match token.kind {
1309+
TokenTree::Token(token, _) => match token.kind {
13071310
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
13081311
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
13091312
IsInFollow::Yes
@@ -1316,7 +1319,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13161319
NonterminalKind::PatWithOr => {
13171320
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`if`", "`in`"];
13181321
match tok {
1319-
TokenTree::Token(token) => match token.kind {
1322+
TokenTree::Token(token, _) => match token.kind {
13201323
FatArrow | Comma | Eq => IsInFollow::Yes,
13211324
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
13221325
IsInFollow::Yes
@@ -1332,7 +1335,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13321335
"`where`",
13331336
];
13341337
match tok {
1335-
TokenTree::Token(token) => match token.kind {
1338+
TokenTree::Token(token, _) => match token.kind {
13361339
OpenDelim(Delimiter::Brace)
13371340
| OpenDelim(Delimiter::Bracket)
13381341
| Comma
@@ -1369,7 +1372,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13691372
// Explicitly disallow `priv`, on the off chance it comes back.
13701373
const TOKENS: &[&str] = &["`,`", "an ident", "a type"];
13711374
match tok {
1372-
TokenTree::Token(token) => match token.kind {
1375+
TokenTree::Token(token, _) => match token.kind {
13731376
Comma => IsInFollow::Yes,
13741377
Ident(_, IdentIsRaw::Yes) => IsInFollow::Yes,
13751378
Ident(name, _) if name != kw::Priv => IsInFollow::Yes,
@@ -1395,7 +1398,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13951398

13961399
fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
13971400
match tt {
1398-
mbe::TokenTree::Token(token) => pprust::token_to_string(token).into(),
1401+
mbe::TokenTree::Token(token, _) => pprust::token_to_string(token).into(),
13991402
mbe::TokenTree::MetaVar(_, name) => format!("${name}"),
14001403
mbe::TokenTree::MetaVarDecl(_, name, Some(kind)) => format!("${name}:{kind}"),
14011404
mbe::TokenTree::MetaVarDecl(_, name, None) => format!("${name}:"),

‎compiler/rustc_expand/src/mbe/quoted.rs

+9-6
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ fn parse_tree<'a>(
180180
err.emit();
181181
// Returns early the same read `$` to avoid spanning
182182
// unrelated diagnostics that could be performed afterwards
183-
return TokenTree::token(token::Dollar, span);
183+
return TokenTree::token_alone(token::Dollar, span);
184184
}
185185
Ok(elem) => {
186186
maybe_emit_macro_metavar_expr_feature(
@@ -220,11 +220,14 @@ fn parse_tree<'a>(
220220

221221
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate`
222222
// special metavariable that names the crate of the invocation.
223-
Some(tokenstream::TokenTree::Token(token, _)) if token.is_ident() => {
223+
Some(tokenstream::TokenTree::Token(token, spacing)) if token.is_ident() => {
224224
let (ident, is_raw) = token.ident().unwrap();
225225
let span = ident.span.with_lo(span.lo());
226226
if ident.name == kw::Crate && matches!(is_raw, IdentIsRaw::No) {
227-
TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span)
227+
TokenTree::Token(
228+
Token::new(token::Ident(kw::DollarCrate, is_raw), span),
229+
*spacing,
230+
)
228231
} else {
229232
TokenTree::MetaVar(span, ident)
230233
}
@@ -240,7 +243,7 @@ fn parse_tree<'a>(
240243
} else {
241244
maybe_emit_macro_metavar_expr_feature(features, sess, span);
242245
}
243-
TokenTree::token(token::Dollar, span)
246+
TokenTree::token_alone(token::Dollar, span)
244247
}
245248

246249
// `tree` is followed by some other token. This is an error.
@@ -252,12 +255,12 @@ fn parse_tree<'a>(
252255
}
253256

254257
// There are no more tokens. Just return the `$` we already have.
255-
None => TokenTree::token(token::Dollar, span),
258+
None => TokenTree::token_alone(token::Dollar, span),
256259
}
257260
}
258261

259262
// `tree` is an arbitrary token. Keep it.
260-
tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token.clone()),
263+
tokenstream::TokenTree::Token(token, spacing) => TokenTree::Token(token.clone(), *spacing),
261264

262265
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
263266
// descend into the delimited set and further parse it.

‎compiler/rustc_expand/src/mbe/transcribe.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -315,10 +315,10 @@ pub(super) fn transcribe<'a>(
315315

316316
// Nothing much to do here. Just push the token to the result, being careful to
317317
// preserve syntax context.
318-
mbe::TokenTree::Token(token) => {
318+
mbe::TokenTree::Token(token, spacing) => {
319319
let mut token = token.clone();
320320
mut_visit::visit_token(&mut token, &mut marker);
321-
let tt = TokenTree::Token(token, Spacing::Alone);
321+
let tt = TokenTree::Token(token, *spacing);
322322
result.push(tt);
323323
}
324324

‎tests/ui/infinite/issue-41731-infinite-macro-print.stderr

+6-6
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@ LL | stack!("overflow");
1414
| ^^^^^^^^^^^^^^^^^^
1515
|
1616
= note: expanding `stack! { "overflow" }`
17-
= note: to `print! (stack! ("overflow"));`
18-
= note: expanding `print! { stack! ("overflow") }`
19-
= note: to `{ $crate :: io :: _print($crate :: format_args! (stack! ("overflow"))); }`
17+
= note: to `print!(stack!("overflow"));`
18+
= note: expanding `print! { stack!("overflow") }`
19+
= note: to `{ $crate::io::_print($crate::format_args!(stack!("overflow"))); }`
2020
= note: expanding `stack! { "overflow" }`
21-
= note: to `print! (stack! ("overflow"));`
22-
= note: expanding `print! { stack! ("overflow") }`
23-
= note: to `{ $crate :: io :: _print($crate :: format_args! (stack! ("overflow"))); }`
21+
= note: to `print!(stack!("overflow"));`
22+
= note: expanding `print! { stack!("overflow") }`
23+
= note: to `{ $crate::io::_print($crate::format_args!(stack!("overflow"))); }`
2424

2525
error: format argument must be a string literal
2626
--> $DIR/issue-41731-infinite-macro-print.rs:14:5

‎tests/ui/infinite/issue-41731-infinite-macro-println.stderr

+6-6
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@ LL | stack!("overflow");
1414
| ^^^^^^^^^^^^^^^^^^
1515
|
1616
= note: expanding `stack! { "overflow" }`
17-
= note: to `println! (stack! ("overflow"));`
18-
= note: expanding `println! { stack! ("overflow") }`
19-
= note: to `{ $crate :: io :: _print($crate :: format_args_nl! (stack! ("overflow"))); }`
17+
= note: to `println!(stack!("overflow"));`
18+
= note: expanding `println! { stack!("overflow") }`
19+
= note: to `{ $crate::io::_print($crate::format_args_nl!(stack!("overflow"))); }`
2020
= note: expanding `stack! { "overflow" }`
21-
= note: to `println! (stack! ("overflow"));`
22-
= note: expanding `println! { stack! ("overflow") }`
23-
= note: to `{ $crate :: io :: _print($crate :: format_args_nl! (stack! ("overflow"))); }`
21+
= note: to `println!(stack!("overflow"));`
22+
= note: expanding `println! { stack!("overflow") }`
23+
= note: to `{ $crate::io::_print($crate::format_args_nl!(stack!("overflow"))); }`
2424

2525
error: format argument must be a string literal
2626
--> $DIR/issue-41731-infinite-macro-println.rs:14:5

‎tests/ui/macros/macro-first-set.rs

+8-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,14 @@ macro_rules! foo_26444 {
2626
}
2727

2828
fn test_26444() {
29-
assert_eq!("a, b, c, d, e", foo_26444!(a, b; c; d, e));
29+
// This one may be surprising. The macro definition has `$($beginning,)*`
30+
// and `$(,$end)*`. The two commas in those fragments have no space after
31+
// them, so the first one is marked `JointHidden` (because it's followed by
32+
// non-punctuation) and the second is marked `Joint` (because it's followed
33+
// by puncuation). Then they get duplicated into the output. Then
34+
// `stringify!` does token pretty printing and because they are both joint,
35+
// no space is put after them.
36+
assert_eq!("a,b,c,d,e", foo_26444!(a, b; c; d, e));
3037
assert_eq!("f", foo_26444!(; f ;));
3138
}
3239

‎tests/ui/macros/trace-macro.stderr

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,5 @@ LL | println!("Hello, World!");
55
| ^^^^^^^^^^^^^^^^^^^^^^^^^
66
|
77
= note: expanding `println! { "Hello, World!" }`
8-
= note: to `{ $crate :: io :: _print($crate :: format_args_nl! ("Hello, World!")); }`
8+
= note: to `{ $crate::io::_print($crate::format_args_nl!("Hello, World!")); }`
99

0 commit comments

Comments
 (0)