Skip to content

Commit 52f3cd5

Browse files
fee1-deadfee1-dead
committed
Change token::Ident to use IdentKind 
1 parent 83f35c0 commit 52f3cd5

File tree

25 files changed

+246
-139
lines changed

25 files changed

+246
-139
lines changed

‎compiler/rustc_ast/src/token.rs

+54-27
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,9 @@ impl Lit {
107107
/// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
108108
pub fn from_token(token: &Token) -> Option<Lit> {
109109
match token.uninterpolate().kind {
110-
Ident(name, false) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
110+
Ident(name, IdentKind::Default) if name.is_bool_lit() => {
111+
Some(Lit::new(Bool, name, None))
112+
}
111113
Literal(token_lit) => Some(token_lit),
112114
Interpolated(ref nt)
113115
if let NtExpr(expr) | NtLiteral(expr) = &nt.0
@@ -183,8 +185,8 @@ impl LitKind {
183185
}
184186
}
185187

186-
pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
187-
let ident_token = Token::new(Ident(name, is_raw), span);
188+
pub fn ident_can_begin_expr(name: Symbol, span: Span, kind: IdentKind) -> bool {
189+
let ident_token = Token::new(Ident(name, kind), span);
188190

189191
!ident_token.is_reserved_ident()
190192
|| ident_token.is_path_segment_keyword()
@@ -212,15 +214,37 @@ pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
212214
kw::Static,
213215
]
214216
.contains(&name)
217+
|| kind == IdentKind::Keyword
215218
}
216219

217-
fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool {
218-
let ident_token = Token::new(Ident(name, is_raw), span);
220+
fn ident_can_begin_type(name: Symbol, span: Span, kind: IdentKind) -> bool {
221+
let ident_token = Token::new(Ident(name, kind), span);
219222

220223
!ident_token.is_reserved_ident()
221224
|| ident_token.is_path_segment_keyword()
222225
|| [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn]
223226
.contains(&name)
227+
|| kind == IdentKind::Keyword
228+
}
229+
230+
#[derive(Clone, Copy, PartialEq, Eq, Debug, HashStable_Generic, Encodable, Decodable)]
231+
pub enum IdentKind {
232+
/// The usual identifiers (or, depending on the context, keywords): `v`, `union`, `await`, `loop`.
233+
Default,
234+
/// Raw identifiers: `r#just_an_ident`, `r#loop`.
235+
Raw,
236+
/// Forced keywords: `k#break`, `k#await`, `k#some_new_experimental_keyword`.
237+
Keyword,
238+
}
239+
240+
impl IdentKind {
241+
pub fn prefix(self) -> Option<&'static str> {
242+
match self {
243+
IdentKind::Default => None,
244+
IdentKind::Raw => Some("r#"),
245+
IdentKind::Keyword => Some("k#"),
246+
}
247+
}
224248
}
225249

226250
// SAFETY: due to the `Clone` impl below, all fields of all variants other than
@@ -298,10 +322,7 @@ pub enum TokenKind {
298322
/// Do not forget about `NtIdent` when you want to match on identifiers.
299323
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
300324
/// treat regular and interpolated identifiers in the same way.
301-
Ident(Symbol, /* is_raw */ bool),
302-
303-
/// A `k#ident` keyword
304-
Keyword(Symbol),
325+
Ident(Symbol, IdentKind),
305326

306327
/// Lifetime identifier token.
307328
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
@@ -415,7 +436,13 @@ impl Token {
415436

416437
/// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary.
417438
pub fn from_ast_ident(ident: Ident) -> Self {
418-
Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
439+
Token::new(
440+
Ident(
441+
ident.name,
442+
if ident.is_raw_guess() { IdentKind::Raw } else { IdentKind::Default },
443+
),
444+
ident.span,
445+
)
419446
}
420447

421448
/// For interpolated tokens, returns a span of the fragment to which the interpolated
@@ -442,7 +469,7 @@ impl Token {
442469
| ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | SingleQuote => true,
443470

444471
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
445-
| Keyword(..) | Lifetime(..) | Interpolated(..) | Eof => false,
472+
| Lifetime(..) | Interpolated(..) | Eof => false,
446473
}
447474
}
448475

@@ -571,7 +598,7 @@ impl Token {
571598
pub fn can_begin_literal_maybe_minus(&self) -> bool {
572599
match self.uninterpolate().kind {
573600
Literal(..) | BinOp(Minus) => true,
574-
Ident(name, false) if name.is_bool_lit() => true,
601+
Ident(name, IdentKind::Default) if name.is_bool_lit() => true,
575602
Interpolated(ref nt) => match &nt.0 {
576603
NtLiteral(_) => true,
577604
NtExpr(e) => match &e.kind {
@@ -606,10 +633,10 @@ impl Token {
606633

607634
/// Returns an identifier if this token is an identifier.
608635
#[inline]
609-
pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> {
636+
pub fn ident(&self) -> Option<(Ident, IdentKind)> {
610637
// We avoid using `Token::uninterpolate` here because it's slow.
611638
match &self.kind {
612-
&Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
639+
&Ident(name, kind) => Some((Ident::new(name, self.span), kind)),
613640
Interpolated(nt) => match &nt.0 {
614641
NtIdent(ident, is_raw) => Some((*ident, *is_raw)),
615642
_ => None,
@@ -702,46 +729,46 @@ impl Token {
702729

703730
/// Returns `true` if the token is a given keyword, `kw`.
704731
pub fn is_keyword(&self, kw: Symbol) -> bool {
705-
self.is_non_raw_ident_where(|id| id.name == kw)
732+
self.is_keywordable_ident_where(|id| id.name == kw)
706733
}
707734

708735
/// Returns `true` if the token is a given keyword, `kw` or if `case` is `Insensitive` and this token is an identifier equal to `kw` ignoring the case.
709736
pub fn is_keyword_case(&self, kw: Symbol, case: Case) -> bool {
710737
self.is_keyword(kw)
711738
|| (case == Case::Insensitive
712-
&& self.is_non_raw_ident_where(|id| {
739+
&& self.is_keywordable_ident_where(|id| {
713740
id.name.as_str().to_lowercase() == kw.as_str().to_lowercase()
714741
}))
715742
}
716743

717744
pub fn is_path_segment_keyword(&self) -> bool {
718-
self.is_non_raw_ident_where(Ident::is_path_segment_keyword)
745+
self.is_keywordable_ident_where(Ident::is_path_segment_keyword)
719746
}
720747

721748
/// Returns true for reserved identifiers used internally for elided lifetimes,
722749
/// unnamed method parameters, crate root module, error recovery etc.
723750
pub fn is_special_ident(&self) -> bool {
724-
self.is_non_raw_ident_where(Ident::is_special)
751+
self.is_keywordable_ident_where(Ident::is_special)
725752
}
726753

727754
/// Returns `true` if the token is a keyword used in the language.
728755
pub fn is_used_keyword(&self) -> bool {
729-
self.is_non_raw_ident_where(Ident::is_used_keyword)
756+
self.is_keywordable_ident_where(Ident::is_used_keyword)
730757
}
731758

732759
/// Returns `true` if the token is a keyword reserved for possible future use.
733760
pub fn is_unused_keyword(&self) -> bool {
734-
self.is_non_raw_ident_where(Ident::is_unused_keyword)
761+
self.is_keywordable_ident_where(Ident::is_unused_keyword)
735762
}
736763

737764
/// Returns `true` if the token is either a special identifier or a keyword.
738765
pub fn is_reserved_ident(&self) -> bool {
739-
self.is_non_raw_ident_where(Ident::is_reserved)
766+
self.is_keywordable_ident_where(Ident::is_reserved)
740767
}
741768

742769
/// Returns `true` if the token is the identifier `true` or `false`.
743770
pub fn is_bool_lit(&self) -> bool {
744-
self.is_non_raw_ident_where(|id| id.name.is_bool_lit())
771+
self.is_keywordable_ident_where(|id| id.name.is_bool_lit())
745772
}
746773

747774
pub fn is_numeric_lit(&self) -> bool {
@@ -757,9 +784,9 @@ impl Token {
757784
}
758785

759786
/// Returns `true` if the token is a non-raw identifier for which `pred` holds.
760-
pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
787+
pub fn is_keywordable_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
761788
match self.ident() {
762-
Some((id, false)) => pred(id),
789+
Some((id, IdentKind::Default | IdentKind::Keyword)) => pred(id),
763790
_ => false,
764791
}
765792
}
@@ -810,13 +837,13 @@ impl Token {
810837
_ => return None,
811838
},
812839
SingleQuote => match joint.kind {
813-
Ident(name, false) => Lifetime(Symbol::intern(&format!("'{name}"))),
840+
Ident(name, IdentKind::Default) => Lifetime(Symbol::intern(&format!("'{name}"))),
814841
_ => return None,
815842
},
816843

817844
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
818845
| DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
819-
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | Keyword(..)
846+
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
820847
| Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
821848
};
822849

@@ -840,7 +867,7 @@ pub enum Nonterminal {
840867
NtPat(P<ast::Pat>),
841868
NtExpr(P<ast::Expr>),
842869
NtTy(P<ast::Ty>),
843-
NtIdent(Ident, /* is_raw */ bool),
870+
NtIdent(Ident, IdentKind),
844871
NtLifetime(Ident),
845872
NtLiteral(P<ast::Expr>),
846873
/// Stuff inside brackets for attributes

‎compiler/rustc_ast/src/tokenstream.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
1616
use crate::ast::{AttrStyle, StmtKind};
1717
use crate::ast_traits::{HasAttrs, HasSpan, HasTokens};
18-
use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind};
18+
use crate::token::{self, Delimiter, IdentKind, Nonterminal, Token, TokenKind};
1919
use crate::AttrVec;
2020

2121
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -677,7 +677,7 @@ impl TokenStream {
677677
DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
678678
Delimiter::Bracket,
679679
[
680-
TokenTree::token_alone(token::Ident(sym::doc, false), span),
680+
TokenTree::token_alone(token::Ident(sym::doc, IdentKind::Default), span),
681681
TokenTree::token_alone(token::Eq, span),
682682
TokenTree::token_alone(
683683
TokenKind::lit(token::StrRaw(num_of_hashes), data, None),

‎compiler/rustc_ast_pretty/src/pprust/state.rs

+6-5
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,9 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
268268
fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params: bool);
269269

270270
fn print_ident(&mut self, ident: Ident) {
271-
self.word(IdentPrinter::for_ast_ident(ident, ident.is_raw_guess()).to_string());
271+
self.word(
272+
IdentPrinter::for_ast_ident(ident, ident.is_raw_guess().then(|| "r#")).to_string(),
273+
);
272274
self.ann_post(ident)
273275
}
274276

@@ -715,7 +717,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
715717
token::NtBlock(e) => self.block_to_string(e),
716718
token::NtStmt(e) => self.stmt_to_string(e),
717719
token::NtPat(e) => self.pat_to_string(e),
718-
token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(*e, *is_raw).to_string(),
720+
&token::NtIdent(e, kind) => IdentPrinter::for_ast_ident(e, kind.prefix()).to_string(),
719721
token::NtLifetime(e) => e.to_string(),
720722
token::NtLiteral(e) => self.expr_to_string(e),
721723
token::NtVis(e) => self.vis_to_string(e),
@@ -778,10 +780,9 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
778780
token::Literal(lit) => literal_to_string(lit).into(),
779781

780782
/* Name components */
781-
token::Ident(s, is_raw) => {
782-
IdentPrinter::new(s, is_raw, convert_dollar_crate).to_string().into()
783+
token::Ident(s, kind) => {
784+
IdentPrinter::new(s, kind.prefix(), convert_dollar_crate).to_string().into()
783785
}
784-
token::Keyword(s) => format!("k#{s}").into(),
785786
token::Lifetime(s) => s.to_string().into(),
786787

787788
/* Other */

‎compiler/rustc_builtin_macros/src/asm.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use rustc_ast as ast;
22
use rustc_ast::ptr::P;
3-
use rustc_ast::token::{self, Delimiter};
3+
use rustc_ast::token::{self, Delimiter, IdentKind};
44
use rustc_ast::tokenstream::TokenStream;
55
use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
66
use rustc_errors::PResult;
@@ -416,7 +416,7 @@ fn parse_reg<'a>(
416416
) -> PResult<'a, ast::InlineAsmRegOrRegClass> {
417417
p.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
418418
let result = match p.token.uninterpolate().kind {
419-
token::Ident(name, false) => ast::InlineAsmRegOrRegClass::RegClass(name),
419+
token::Ident(name, IdentKind::Default) => ast::InlineAsmRegOrRegClass::RegClass(name),
420420
token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => {
421421
*explicit_reg = true;
422422
ast::InlineAsmRegOrRegClass::Reg(symbol)

‎compiler/rustc_builtin_macros/src/assert/context.rs

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use rustc_ast::{
22
ptr::P,
33
token,
4-
token::Delimiter,
4+
token::{Delimiter, IdentKind},
55
tokenstream::{DelimSpan, TokenStream, TokenTree},
66
BinOpKind, BorrowKind, DelimArgs, Expr, ExprKind, ItemKind, MacCall, MethodCall, Mutability,
77
Path, PathSegment, Stmt, StructRest, UnOp, UseTree, UseTreeKind, DUMMY_NODE_ID,
@@ -170,7 +170,10 @@ impl<'cx, 'a> Context<'cx, 'a> {
170170
];
171171
let captures = self.capture_decls.iter().flat_map(|cap| {
172172
[
173-
TokenTree::token_joint_hidden(token::Ident(cap.ident.name, false), cap.ident.span),
173+
TokenTree::token_joint_hidden(
174+
token::Ident(cap.ident.name, IdentKind::Default),
175+
cap.ident.span,
176+
),
174177
TokenTree::token_alone(token::Comma, self.span),
175178
]
176179
});

‎compiler/rustc_expand/src/mbe/macro_check.rs

+5-2
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@
107107
use crate::errors;
108108
use crate::mbe::{KleeneToken, TokenTree};
109109

110-
use rustc_ast::token::{Delimiter, Token, TokenKind};
110+
use rustc_ast::token::{Delimiter, IdentKind, Token, TokenKind};
111111
use rustc_ast::{NodeId, DUMMY_NODE_ID};
112112
use rustc_data_structures::fx::FxHashMap;
113113
use rustc_errors::{DiagnosticMessage, MultiSpan};
@@ -409,7 +409,10 @@ fn check_nested_occurrences(
409409
match (state, tt) {
410410
(
411411
NestedMacroState::Empty,
412-
&TokenTree::Token(Token { kind: TokenKind::Ident(name, false), .. }),
412+
&TokenTree::Token(Token {
413+
kind: TokenKind::Ident(name, IdentKind::Default | IdentKind::Keyword),
414+
..
415+
}),
413416
) => {
414417
if name == kw::MacroRules {
415418
state = NestedMacroState::MacroRules;

‎compiler/rustc_expand/src/mbe/macro_rules.rs

+18-5
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ use crate::mbe::macro_parser::{MatchedSeq, MatchedTokenTree, MatcherLoc};
99
use crate::mbe::transcribe::transcribe;
1010

1111
use rustc_ast as ast;
12-
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind, TokenKind::*};
12+
use rustc_ast::token::TokenKind::*;
13+
use rustc_ast::token::{self, Delimiter, IdentKind, NonterminalKind, Token, TokenKind};
1314
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
1415
use rustc_ast::{NodeId, DUMMY_NODE_ID};
1516
use rustc_ast_pretty::pprust;
@@ -1323,7 +1324,11 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13231324
match tok {
13241325
TokenTree::Token(token) => match token.kind {
13251326
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
1326-
Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes,
1327+
Ident(name, IdentKind::Default | IdentKind::Keyword)
1328+
if name == kw::If || name == kw::In =>
1329+
{
1330+
IsInFollow::Yes
1331+
}
13271332
_ => IsInFollow::No(TOKENS),
13281333
},
13291334
_ => IsInFollow::No(TOKENS),
@@ -1334,7 +1339,11 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13341339
match tok {
13351340
TokenTree::Token(token) => match token.kind {
13361341
FatArrow | Comma | Eq => IsInFollow::Yes,
1337-
Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes,
1342+
Ident(name, IdentKind::Default | IdentKind::Keyword)
1343+
if name == kw::If || name == kw::In =>
1344+
{
1345+
IsInFollow::Yes
1346+
}
13381347
_ => IsInFollow::No(TOKENS),
13391348
},
13401349
_ => IsInFollow::No(TOKENS),
@@ -1357,7 +1366,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13571366
| BinOp(token::Shr)
13581367
| Semi
13591368
| BinOp(token::Or) => IsInFollow::Yes,
1360-
Ident(name, false) if name == kw::As || name == kw::Where => {
1369+
Ident(name, IdentKind::Default | IdentKind::Keyword)
1370+
if name == kw::As || name == kw::Where =>
1371+
{
13611372
IsInFollow::Yes
13621373
}
13631374
_ => IsInFollow::No(TOKENS),
@@ -1385,7 +1396,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13851396
match tok {
13861397
TokenTree::Token(token) => match token.kind {
13871398
Comma => IsInFollow::Yes,
1388-
Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes,
1399+
Ident(name, kind) if kind == IdentKind::Raw || name != kw::Priv => {
1400+
IsInFollow::Yes
1401+
}
13891402
_ => {
13901403
if token.can_begin_type() {
13911404
IsInFollow::Yes

‎compiler/rustc_expand/src/mbe/metavar_expr.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use rustc_ast::token::{self, Delimiter};
1+
use rustc_ast::token::{self, Delimiter, IdentKind};
22
use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
33
use rustc_ast::{LitIntType, LitKind};
44
use rustc_ast_pretty::pprust;
@@ -142,7 +142,7 @@ fn parse_ident<'sess>(
142142
if let Some(tt) = iter.next()
143143
&& let TokenTree::Token(token, _) = tt
144144
{
145-
if let Some((elem, false)) = token.ident() {
145+
if let Some((elem, IdentKind::Default)) = token.ident() {
146146
return Ok(elem);
147147
}
148148
let token_str = pprust::token_to_string(token);

0 commit comments

Comments
 (0)