Skip to content

Store tokens alongside more AST expressions #70091

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 8 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions src/librustc_ast/ast.rs
Original file line number Diff line number Diff line change
@@ -492,13 +492,15 @@ pub struct Block {
/// Distinguishes between `unsafe { ... }` and `{ ... }`.
pub rules: BlockCheckMode,
pub span: Span,
pub tokens: Option<TokenStream>,
}

#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
pub struct Pat {
pub id: NodeId,
pub kind: PatKind,
pub span: Span,
pub tokens: Option<TokenStream>,
}

impl Pat {
@@ -534,7 +536,7 @@ impl Pat {
_ => return None,
};

Some(P(Ty { kind, id: self.id, span: self.span }))
Some(P(Ty { kind, id: self.id, span: self.span, tokens: None }))
}

/// Walk top-down and call `it` in each place where a pattern occurs
@@ -875,6 +877,7 @@ pub struct Stmt {
pub id: NodeId,
pub kind: StmtKind,
pub span: Span,
pub tokens: Option<TokenStream>,
}

impl Stmt {
@@ -1011,11 +1014,12 @@ pub struct Expr {
pub kind: ExprKind,
pub span: Span,
pub attrs: AttrVec,
pub tokens: Option<TokenStream>,
}

// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(target_arch = "x86_64")]
rustc_data_structures::static_assert_size!(Expr, 96);
rustc_data_structures::static_assert_size!(Expr, 104);

impl Expr {
/// Returns `true` if this expression would be valid somewhere that expects a value;
@@ -1091,7 +1095,7 @@ impl Expr {
_ => return None,
};

Some(P(Ty { kind, id: self.id, span: self.span }))
Some(P(Ty { kind, id: self.id, span: self.span, tokens: None }))
}

pub fn precedence(&self) -> ExprPrecedence {
@@ -1829,6 +1833,7 @@ pub struct Ty {
pub id: NodeId,
pub kind: TyKind,
pub span: Span,
pub tokens: Option<TokenStream>,
}

#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
@@ -2009,13 +2014,14 @@ impl Param {
/// Builds a `Param` object from `ExplicitSelf`.
pub fn from_self(attrs: AttrVec, eself: ExplicitSelf, eself_ident: Ident) -> Param {
let span = eself.span.to(eself_ident.span);
let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span });
let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span, tokens: None });
let param = |mutbl, ty| Param {
attrs,
pat: P(Pat {
id: DUMMY_NODE_ID,
kind: PatKind::Ident(BindingMode::ByValue(mutbl), eself_ident, None),
span,
tokens: None,
}),
span,
ty,
@@ -2031,6 +2037,7 @@ impl Param {
id: DUMMY_NODE_ID,
kind: TyKind::Rptr(lt, MutTy { ty: infer_ty, mutbl }),
span,
tokens: None,
}),
),
}
18 changes: 12 additions & 6 deletions src/librustc_ast/mut_visit.rs
Original file line number Diff line number Diff line change
@@ -449,7 +449,7 @@ pub fn noop_visit_ty_constraint<T: MutVisitor>(
}

pub fn noop_visit_ty<T: MutVisitor>(ty: &mut P<Ty>, vis: &mut T) {
let Ty { id, kind, span } = ty.deref_mut();
let Ty { id, kind, span, tokens: _ } = ty.deref_mut();
vis.visit_id(id);
match kind {
TyKind::Infer | TyKind::ImplicitSelf | TyKind::Err | TyKind::Never | TyKind::CVarArgs => {}
@@ -867,7 +867,7 @@ pub fn noop_visit_mt<T: MutVisitor>(MutTy { ty, mutbl: _ }: &mut MutTy, vis: &mu
}

pub fn noop_visit_block<T: MutVisitor>(block: &mut P<Block>, vis: &mut T) {
let Block { id, stmts, rules: _, span } = block.deref_mut();
let Block { id, stmts, rules: _, span, tokens: _ } = block.deref_mut();
vis.visit_id(id);
stmts.flat_map_in_place(|stmt| vis.flat_map_stmt(stmt));
vis.visit_span(span);
@@ -1050,7 +1050,7 @@ pub fn noop_flat_map_foreign_item<T: MutVisitor>(
}

pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
let Pat { id, kind, span } = pat.deref_mut();
let Pat { id, kind, span, tokens: _ } = pat.deref_mut();
vis.visit_id(id);
match kind {
PatKind::Wild | PatKind::Rest => {}
@@ -1092,7 +1092,10 @@ pub fn noop_visit_anon_const<T: MutVisitor>(AnonConst { id, value }: &mut AnonCo
vis.visit_expr(value);
}

pub fn noop_visit_expr<T: MutVisitor>(Expr { kind, id, span, attrs }: &mut Expr, vis: &mut T) {
pub fn noop_visit_expr<T: MutVisitor>(
Expr { kind, id, span, attrs, tokens: _ }: &mut Expr,
vis: &mut T,
) {
match kind {
ExprKind::Box(expr) => vis.visit_expr(expr),
ExprKind::Array(exprs) => visit_exprs(exprs, vis),
@@ -1254,12 +1257,15 @@ pub fn noop_filter_map_expr<T: MutVisitor>(mut e: P<Expr>, vis: &mut T) -> Optio
}

pub fn noop_flat_map_stmt<T: MutVisitor>(
Stmt { kind, mut span, mut id }: Stmt,
Stmt { kind, mut span, mut id, tokens: _ }: Stmt,
vis: &mut T,
) -> SmallVec<[Stmt; 1]> {
vis.visit_id(&mut id);
vis.visit_span(&mut span);
noop_flat_map_stmt_kind(kind, vis).into_iter().map(|kind| Stmt { id, kind, span }).collect()
noop_flat_map_stmt_kind(kind, vis)
.into_iter()
.map(|kind| Stmt { id, kind, span, tokens: None })
.collect()
}

pub fn noop_flat_map_stmt_kind<T: MutVisitor>(
8 changes: 4 additions & 4 deletions src/librustc_ast/token.rs
Original file line number Diff line number Diff line change
@@ -728,7 +728,7 @@ impl PartialEq<TokenKind> for Token {
}
}

#[derive(Clone, RustcEncodable, RustcDecodable)]
#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
/// For interpolation during macro expansion.
pub enum Nonterminal {
NtItem(P<ast::Item>),
@@ -749,7 +749,7 @@ pub enum Nonterminal {

// `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(target_arch = "x86_64")]
rustc_data_structures::static_assert_size!(Nonterminal, 40);
rustc_data_structures::static_assert_size!(Nonterminal, 48);

impl Nonterminal {
fn span(&self) -> Span {
@@ -786,7 +786,7 @@ impl PartialEq for Nonterminal {
}
}

impl fmt::Debug for Nonterminal {
/*impl fmt::Debug for Nonterminal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
NtItem(..) => f.pad("NtItem(..)"),
@@ -804,7 +804,7 @@ impl fmt::Debug for Nonterminal {
NtLifetime(..) => f.pad("NtLifetime(..)"),
}
}
}
}*/

impl<CTX> HashStable<CTX> for Nonterminal
where
2 changes: 2 additions & 0 deletions src/librustc_ast_lowering/lib.rs
Original file line number Diff line number Diff line change
@@ -1065,6 +1065,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
id: node_id,
kind: TyKind::ImplTrait(impl_trait_node_id, bounds.clone()),
span: constraint.span,
tokens: None,
},
itctx,
);
@@ -1129,6 +1130,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
kind: ExprKind::Path(qself.clone(), path.clone()),
span: ty.span,
attrs: AttrVec::new(),
tokens: None,
};

let ct = self.with_new_scopes(|this| hir::AnonConst {
1 change: 1 addition & 0 deletions src/librustc_builtin_macros/asm.rs
Original file line number Diff line number Diff line change
@@ -61,6 +61,7 @@ pub fn expand_asm<'cx>(
kind: ast::ExprKind::InlineAsm(P(inline_asm)),
span: cx.with_def_site_ctxt(sp),
attrs: ast::AttrVec::new(),
tokens: None,
}))
}

2 changes: 2 additions & 0 deletions src/librustc_builtin_macros/concat_idents.rs
Original file line number Diff line number Diff line change
@@ -52,6 +52,7 @@ pub fn expand_concat_idents<'cx>(
kind: ast::ExprKind::Path(None, ast::Path::from_ident(self.ident)),
span: self.ident.span,
attrs: ast::AttrVec::new(),
tokens: None,
}))
}

@@ -60,6 +61,7 @@ pub fn expand_concat_idents<'cx>(
id: ast::DUMMY_NODE_ID,
kind: ast::TyKind::Path(None, ast::Path::from_ident(self.ident)),
span: self.ident.span,
tokens: None,
}))
}
}
2 changes: 1 addition & 1 deletion src/librustc_builtin_macros/deriving/debug.rs
Original file line number Diff line number Diff line change
@@ -133,5 +133,5 @@ fn stmt_let_undescore(cx: &mut ExtCtxt<'_>, sp: Span, expr: P<ast::Expr>) -> ast
span: sp,
attrs: ast::AttrVec::new(),
});
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp }
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp, tokens: None }
}
1 change: 1 addition & 0 deletions src/librustc_builtin_macros/deriving/mod.rs
Original file line number Diff line number Diff line change
@@ -74,6 +74,7 @@ fn call_intrinsic(
id: ast::DUMMY_NODE_ID,
rules: ast::BlockCheckMode::Unsafe(ast::CompilerGenerated),
span,
tokens: None,
}))
}

7 changes: 6 additions & 1 deletion src/librustc_expand/base.rs
Original file line number Diff line number Diff line change
@@ -391,6 +391,7 @@ macro_rules! make_stmts_default {
id: ast::DUMMY_NODE_ID,
span: e.span,
kind: ast::StmtKind::Expr(e),
tokens: None
}]
})
};
@@ -542,6 +543,7 @@ impl MacResult for MacEager {
id: ast::DUMMY_NODE_ID,
span: e.span,
kind: PatKind::Lit(e),
tokens: None,
}));
}
}
@@ -582,12 +584,13 @@ impl DummyResult {
kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) },
span: sp,
attrs: ast::AttrVec::new(),
tokens: None,
})
}

/// A plain dummy pattern.
pub fn raw_pat(sp: Span) -> ast::Pat {
ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp }
ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp, tokens: None }
}

/// A plain dummy type.
@@ -596,6 +599,7 @@ impl DummyResult {
id: ast::DUMMY_NODE_ID,
kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) },
span: sp,
tokens: None,
})
}
}
@@ -630,6 +634,7 @@ impl MacResult for DummyResult {
id: ast::DUMMY_NODE_ID,
kind: ast::StmtKind::Expr(DummyResult::raw_expr(self.span, self.is_error)),
span: self.span,
tokens: None
}])
}

46 changes: 37 additions & 9 deletions src/librustc_expand/build.rs
Original file line number Diff line number Diff line change
@@ -53,7 +53,7 @@ impl<'a> ExtCtxt<'a> {
}

pub fn ty(&self, span: Span, kind: ast::TyKind) -> P<ast::Ty> {
P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind })
P(ast::Ty { id: ast::DUMMY_NODE_ID, span, kind, tokens: None })
}

pub fn ty_path(&self, path: ast::Path) -> P<ast::Ty> {
@@ -69,7 +69,13 @@ impl<'a> ExtCtxt<'a> {
pub fn anon_const(&self, span: Span, kind: ast::ExprKind) -> ast::AnonConst {
ast::AnonConst {
id: ast::DUMMY_NODE_ID,
value: P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() }),
value: P(ast::Expr {
id: ast::DUMMY_NODE_ID,
kind,
span,
attrs: AttrVec::new(),
tokens: None,
}),
}
}

@@ -151,7 +157,12 @@ impl<'a> ExtCtxt<'a> {
}

pub fn stmt_expr(&self, expr: P<ast::Expr>) -> ast::Stmt {
ast::Stmt { id: ast::DUMMY_NODE_ID, span: expr.span, kind: ast::StmtKind::Expr(expr) }
ast::Stmt {
id: ast::DUMMY_NODE_ID,
span: expr.span,
kind: ast::StmtKind::Expr(expr),
tokens: None,
}
}

pub fn stmt_let(
@@ -175,7 +186,12 @@ impl<'a> ExtCtxt<'a> {
span: sp,
attrs: AttrVec::new(),
});
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp }
ast::Stmt {
id: ast::DUMMY_NODE_ID,
kind: ast::StmtKind::Local(local),
span: sp,
tokens: None,
}
}

// Generates `let _: Type;`, which is usually used for type assertions.
@@ -188,11 +204,16 @@ impl<'a> ExtCtxt<'a> {
span,
attrs: AttrVec::new(),
});
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span }
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span, tokens: None }
}

pub fn stmt_item(&self, sp: Span, item: P<ast::Item>) -> ast::Stmt {
ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Item(item), span: sp }
ast::Stmt {
id: ast::DUMMY_NODE_ID,
kind: ast::StmtKind::Item(item),
span: sp,
tokens: None,
}
}

pub fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block> {
@@ -202,15 +223,22 @@ impl<'a> ExtCtxt<'a> {
id: ast::DUMMY_NODE_ID,
span: expr.span,
kind: ast::StmtKind::Expr(expr),
tokens: None,
}],
)
}
pub fn block(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Block> {
P(ast::Block { stmts, id: ast::DUMMY_NODE_ID, rules: BlockCheckMode::Default, span })
P(ast::Block {
stmts,
id: ast::DUMMY_NODE_ID,
rules: BlockCheckMode::Default,
span,
tokens: None,
})
}

pub fn expr(&self, span: Span, kind: ast::ExprKind) -> P<ast::Expr> {
P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new() })
P(ast::Expr { id: ast::DUMMY_NODE_ID, kind, span, attrs: AttrVec::new(), tokens: None })
}

pub fn expr_path(&self, path: ast::Path) -> P<ast::Expr> {
@@ -396,7 +424,7 @@ impl<'a> ExtCtxt<'a> {
}

pub fn pat(&self, span: Span, kind: PatKind) -> P<ast::Pat> {
P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span })
P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span, tokens: None })
}
pub fn pat_wild(&self, span: Span) -> P<ast::Pat> {
self.pat(span, PatKind::Wild)
Loading