Skip to content

Commit acb3490

Browse files
ChayimFriedman2ChayimFriedman2
committed
Store token trees in contiguous Vec instead of as a tree
I expected this to be faster (due to less allocations and better cache locality), but benchmarked it is not (neither it is slower). Memory usage, however, drops by ~50mb (of `analysis-stats .`). I guess tt construction is just not hot. This also simplifies using even less memory for token trees by compressing equal span, which I plan to do right after. Some workflows are more easily expressed with a flat tt, while some are better expressed with a tree. With the right helpers, though (which was mostly a matter of trial and error), even the worst workflows become very easy indeed.
1 parent 65bf7f7 commit acb3490

File tree

50 files changed

+2322
-2252
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+2322
-2252
lines changed

‎src/tools/rust-analyzer/Cargo.lock

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1389,6 +1389,7 @@ version = "0.0.0"
13891389
dependencies = [
13901390
"proc-macro-api",
13911391
"proc-macro-srv",
1392+
"tt",
13921393
]
13931394

13941395
[[package]]

‎src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ impl From<CfgAtom> for CfgExpr {
4545

4646
impl CfgExpr {
4747
#[cfg(feature = "tt")]
48-
pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
49-
next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
48+
pub fn parse<S: Copy>(tt: &tt::TopSubtree<S>) -> CfgExpr {
49+
next_cfg_expr(&mut tt.iter()).unwrap_or(CfgExpr::Invalid)
5050
}
5151

5252
/// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@@ -66,19 +66,19 @@ impl CfgExpr {
6666
}
6767

6868
#[cfg(feature = "tt")]
69-
fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
69+
fn next_cfg_expr<S: Copy>(it: &mut tt::iter::TtIter<'_, S>) -> Option<CfgExpr> {
7070
use intern::sym;
71+
use tt::iter::TtElement;
7172

7273
let name = match it.next() {
7374
None => return None,
74-
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
75+
Some(TtElement::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
7576
Some(_) => return Some(CfgExpr::Invalid),
7677
};
7778

78-
// Peek
79-
let ret = match it.as_slice().first() {
80-
Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
81-
match it.as_slice().get(1) {
79+
let ret = match it.peek() {
80+
Some(TtElement::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
81+
match it.remaining().flat_tokens().get(1) {
8282
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
8383
it.next();
8484
it.next();
@@ -87,9 +87,8 @@ fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<C
8787
_ => return Some(CfgExpr::Invalid),
8888
}
8989
}
90-
Some(tt::TokenTree::Subtree(subtree)) => {
90+
Some(TtElement::Subtree(_, mut sub_it)) => {
9191
it.next();
92-
let mut sub_it = subtree.token_trees.iter();
9392
let mut subs = std::iter::from_fn(|| next_cfg_expr(&mut sub_it));
9493
match name {
9594
s if s == sym::all => CfgExpr::All(subs.collect()),
@@ -104,7 +103,7 @@ fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<C
104103
};
105104

106105
// Eat comma separator
107-
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = it.as_slice().first() {
106+
if let Some(TtElement::Leaf(tt::Leaf::Punct(punct))) = it.peek() {
108107
if punct.char == ',' {
109108
it.next();
110109
}

‎src/tools/rust-analyzer/crates/hir-def/src/attr.rs

Lines changed: 30 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
//! A higher level attributes based on TokenTree, with also some shortcuts.
22
3-
use std::{borrow::Cow, hash::Hash, ops, slice};
3+
use std::{borrow::Cow, hash::Hash, ops};
44

55
use base_db::CrateId;
66
use cfg::{CfgExpr, CfgOptions};
@@ -17,6 +17,7 @@ use syntax::{
1717
AstPtr,
1818
};
1919
use triomphe::Arc;
20+
use tt::iter::{TtElement, TtIter};
2021

2122
use crate::{
2223
db::DefDatabase,
@@ -154,15 +155,15 @@ impl Attrs {
154155

155156
pub fn has_doc_hidden(&self) -> bool {
156157
self.by_key(&sym::doc).tt_values().any(|tt| {
157-
tt.delimiter.kind == DelimiterKind::Parenthesis &&
158-
matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::hidden)
158+
tt.top_subtree().delimiter.kind == DelimiterKind::Parenthesis &&
159+
matches!(tt.token_trees().flat_tokens(), [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::hidden)
159160
})
160161
}
161162

162163
pub fn has_doc_notable_trait(&self) -> bool {
163164
self.by_key(&sym::doc).tt_values().any(|tt| {
164-
tt.delimiter.kind == DelimiterKind::Parenthesis &&
165-
matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::notable_trait)
165+
tt.top_subtree().delimiter.kind == DelimiterKind::Parenthesis &&
166+
matches!(tt.token_trees().flat_tokens(), [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::notable_trait)
166167
})
167168
}
168169

@@ -245,8 +246,8 @@ impl From<DocAtom> for DocExpr {
245246
}
246247

247248
impl DocExpr {
248-
fn parse<S>(tt: &tt::Subtree<S>) -> DocExpr {
249-
next_doc_expr(&mut tt.token_trees.iter()).unwrap_or(DocExpr::Invalid)
249+
fn parse<S: Copy>(tt: &tt::TopSubtree<S>) -> DocExpr {
250+
next_doc_expr(tt.iter()).unwrap_or(DocExpr::Invalid)
250251
}
251252

252253
pub fn aliases(&self) -> &[Symbol] {
@@ -260,62 +261,47 @@ impl DocExpr {
260261
}
261262
}
262263

263-
fn next_doc_expr<S>(it: &mut slice::Iter<'_, tt::TokenTree<S>>) -> Option<DocExpr> {
264+
fn next_doc_expr<S: Copy>(mut it: TtIter<'_, S>) -> Option<DocExpr> {
264265
let name = match it.next() {
265266
None => return None,
266-
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
267+
Some(TtElement::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
267268
Some(_) => return Some(DocExpr::Invalid),
268269
};
269270

270271
// Peek
271-
let ret = match it.as_slice().first() {
272-
Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
273-
match it.as_slice().get(1) {
274-
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
272+
let ret = match it.peek() {
273+
Some(TtElement::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
274+
it.next();
275+
match it.next() {
276+
Some(TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
275277
symbol: text,
276278
kind: tt::LitKind::Str,
277279
..
278-
}))) => {
279-
it.next();
280-
it.next();
281-
DocAtom::KeyValue { key: name, value: text.clone() }.into()
282-
}
280+
}))) => DocAtom::KeyValue { key: name, value: text.clone() }.into(),
283281
_ => return Some(DocExpr::Invalid),
284282
}
285283
}
286-
Some(tt::TokenTree::Subtree(subtree)) => {
284+
Some(TtElement::Subtree(_, subtree_iter)) => {
287285
it.next();
288-
let subs = parse_comma_sep(subtree);
286+
let subs = parse_comma_sep(subtree_iter);
289287
match &name {
290288
s if *s == sym::alias => DocExpr::Alias(subs),
291289
_ => DocExpr::Invalid,
292290
}
293291
}
294292
_ => DocAtom::Flag(name).into(),
295293
};
296-
297-
// Eat comma separator
298-
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = it.as_slice().first() {
299-
if punct.char == ',' {
300-
it.next();
301-
}
302-
}
303294
Some(ret)
304295
}
305296

306-
fn parse_comma_sep<S>(subtree: &tt::Subtree<S>) -> Vec<Symbol> {
307-
subtree
308-
.token_trees
309-
.iter()
310-
.filter_map(|tt| match tt {
311-
tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
312-
kind: tt::LitKind::Str,
313-
symbol,
314-
..
315-
})) => Some(symbol.clone()),
316-
_ => None,
317-
})
318-
.collect()
297+
fn parse_comma_sep<S>(iter: TtIter<'_, S>) -> Vec<Symbol> {
298+
iter.filter_map(|tt| match tt {
299+
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
300+
kind: tt::LitKind::Str, symbol, ..
301+
})) => Some(symbol.clone()),
302+
_ => None,
303+
})
304+
.collect()
319305
}
320306

321307
impl AttrsWithOwner {
@@ -563,7 +549,7 @@ pub struct AttrQuery<'attr> {
563549
}
564550

565551
impl<'attr> AttrQuery<'attr> {
566-
pub fn tt_values(self) -> impl Iterator<Item = &'attr crate::tt::Subtree> {
552+
pub fn tt_values(self) -> impl Iterator<Item = &'attr crate::tt::TopSubtree> {
567553
self.attrs().filter_map(|attr| attr.token_tree_value())
568554
}
569555

@@ -596,12 +582,12 @@ impl<'attr> AttrQuery<'attr> {
596582
/// ```
597583
pub fn find_string_value_in_tt(self, key: &'attr Symbol) -> Option<&'attr str> {
598584
self.tt_values().find_map(|tt| {
599-
let name = tt.token_trees.iter()
600-
.skip_while(|tt| !matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { sym, ..} )) if *sym == *key))
585+
let name = tt.iter()
586+
.skip_while(|tt| !matches!(tt, TtElement::Leaf(tt::Leaf::Ident(tt::Ident { sym, ..} )) if *sym == *key))
601587
.nth(2);
602588

603589
match name {
604-
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal{ symbol: text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text.as_str()),
590+
Some(TtElement::Leaf(tt::Leaf::Literal(tt::Literal{ symbol: text, kind: tt::LitKind::Str | tt::LitKind::StrRaw(_) , ..}))) => Some(text.as_str()),
605591
_ => None
606592
}
607593
})

‎src/tools/rust-analyzer/crates/hir-def/src/data.rs

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ use la_arena::{Idx, RawIdx};
1111
use smallvec::SmallVec;
1212
use syntax::{ast, Parse};
1313
use triomphe::Arc;
14+
use tt::iter::TtElement;
1415

1516
use crate::{
1617
db::DefDatabase,
@@ -156,20 +157,21 @@ impl FunctionData {
156157
}
157158
}
158159

159-
fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> {
160+
fn parse_rustc_legacy_const_generics(tt: &crate::tt::TopSubtree) -> Box<[u32]> {
160161
let mut indices = Vec::new();
161-
for args in tt.token_trees.chunks(2) {
162-
match &args[0] {
163-
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => match lit.symbol.as_str().parse() {
162+
let mut iter = tt.iter();
163+
while let (Some(first), second) = (iter.next(), iter.next()) {
164+
match first {
165+
TtElement::Leaf(tt::Leaf::Literal(lit)) => match lit.symbol.as_str().parse() {
164166
Ok(index) => indices.push(index),
165167
Err(_) => break,
166168
},
167169
_ => break,
168170
}
169171

170-
if let Some(comma) = args.get(1) {
172+
if let Some(comma) = second {
171173
match comma {
172-
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.char == ',' => {}
174+
TtElement::Leaf(tt::Leaf::Punct(punct)) if punct.char == ',' => {}
173175
_ => break,
174176
}
175177
}
@@ -267,8 +269,8 @@ impl TraitData {
267269
attrs.by_key(&sym::rustc_skip_array_during_method_dispatch).exists();
268270
let mut skip_boxed_slice_during_method_dispatch = false;
269271
for tt in attrs.by_key(&sym::rustc_skip_during_method_dispatch).tt_values() {
270-
for tt in tt.token_trees.iter() {
271-
if let crate::tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = tt {
272+
for tt in tt.iter() {
273+
if let tt::iter::TtElement::Leaf(tt::Leaf::Ident(ident)) = tt {
272274
skip_array_during_method_dispatch |= ident.sym == sym::array;
273275
skip_boxed_slice_during_method_dispatch |= ident.sym == sym::boxed_slice;
274276
}
@@ -421,7 +423,7 @@ impl Macro2Data {
421423
.by_key(&sym::rustc_builtin_macro)
422424
.tt_values()
423425
.next()
424-
.and_then(|attr| parse_macro_name_and_helper_attrs(&attr.token_trees))
426+
.and_then(parse_macro_name_and_helper_attrs)
425427
.map(|(_, helpers)| helpers);
426428

427429
Arc::new(Macro2Data {

‎src/tools/rust-analyzer/crates/hir-def/src/data/adt.rs

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use intern::sym;
1010
use la_arena::Arena;
1111
use rustc_abi::{Align, Integer, IntegerType, ReprFlags, ReprOptions};
1212
use triomphe::Arc;
13+
use tt::iter::TtElement;
1314

1415
use crate::{
1516
builtin_type::{BuiltinInt, BuiltinUint},
@@ -20,7 +21,7 @@ use crate::{
2021
},
2122
lang_item::LangItem,
2223
nameres::diagnostics::{DefDiagnostic, DefDiagnostics},
23-
tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree},
24+
tt::{Delimiter, DelimiterKind, Leaf, TopSubtree},
2425
type_ref::{TypeRefId, TypesMap},
2526
visibility::RawVisibility,
2627
EnumId, EnumVariantId, LocalFieldId, LocalModuleId, Lookup, StructId, UnionId, VariantId,
@@ -95,8 +96,8 @@ fn repr_from_value(
9596
item_tree.attrs(db, krate, of).by_key(&sym::repr).tt_values().find_map(parse_repr_tt)
9697
}
9798

98-
fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
99-
match tt.delimiter {
99+
fn parse_repr_tt(tt: &TopSubtree) -> Option<ReprOptions> {
100+
match tt.top_subtree().delimiter {
100101
Delimiter { kind: DelimiterKind::Parenthesis, .. } => {}
101102
_ => return None,
102103
}
@@ -106,14 +107,14 @@ fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
106107
let mut max_align: Option<Align> = None;
107108
let mut min_pack: Option<Align> = None;
108109

109-
let mut tts = tt.token_trees.iter().peekable();
110+
let mut tts = tt.iter();
110111
while let Some(tt) = tts.next() {
111-
if let TokenTree::Leaf(Leaf::Ident(ident)) = tt {
112+
if let TtElement::Leaf(Leaf::Ident(ident)) = tt {
112113
flags.insert(match &ident.sym {
113114
s if *s == sym::packed => {
114-
let pack = if let Some(TokenTree::Subtree(tt)) = tts.peek() {
115+
let pack = if let Some(TtElement::Subtree(_, mut tt_iter)) = tts.peek() {
115116
tts.next();
116-
if let Some(TokenTree::Leaf(Leaf::Literal(lit))) = tt.token_trees.first() {
117+
if let Some(TtElement::Leaf(Leaf::Literal(lit))) = tt_iter.next() {
117118
lit.symbol.as_str().parse().unwrap_or_default()
118119
} else {
119120
0
@@ -127,9 +128,9 @@ fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
127128
ReprFlags::empty()
128129
}
129130
s if *s == sym::align => {
130-
if let Some(TokenTree::Subtree(tt)) = tts.peek() {
131+
if let Some(TtElement::Subtree(_, mut tt_iter)) = tts.peek() {
131132
tts.next();
132-
if let Some(TokenTree::Leaf(Leaf::Literal(lit))) = tt.token_trees.first() {
133+
if let Some(TtElement::Leaf(Leaf::Literal(lit))) = tt_iter.next() {
133134
if let Ok(align) = lit.symbol.as_str().parse() {
134135
let align = Align::from_bytes(align).ok();
135136
max_align = max_align.max(align);

‎src/tools/rust-analyzer/crates/hir-def/src/db.rs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use crate::{
2121
item_tree::{AttrOwner, ItemTree, ItemTreeSourceMaps},
2222
lang_item::{self, LangItem, LangItemTarget, LangItems},
2323
nameres::{diagnostics::DefDiagnostics, DefMap},
24+
tt,
2425
type_ref::TypesSourceMap,
2526
visibility::{self, Visibility},
2627
AttrDefId, BlockId, BlockLoc, ConstBlockId, ConstBlockLoc, ConstId, ConstLoc, DefWithBodyId,
@@ -294,14 +295,14 @@ fn crate_supports_no_std(db: &dyn DefDatabase, crate_id: CrateId) -> bool {
294295
// This is a `cfg_attr`; check if it could possibly expand to `no_std`.
295296
// Syntax is: `#[cfg_attr(condition(cfg, style), attr0, attr1, <...>)]`
296297
let tt = match attr.token_tree_value() {
297-
Some(tt) => &tt.token_trees,
298+
Some(tt) => tt.token_trees(),
298299
None => continue,
299300
};
300301

301302
let segments =
302-
tt.split(|tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(p)) if p.char == ','));
303+
tt.split(|tt| matches!(tt, tt::TtElement::Leaf(tt::Leaf::Punct(p)) if p.char == ','));
303304
for output in segments.skip(1) {
304-
match output {
305+
match output.flat_tokens() {
305306
[tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.sym == sym::no_std => {
306307
return true
307308
}

‎src/tools/rust-analyzer/crates/hir-def/src/macro_expansion_tests/mod.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ use crate::{
4040
resolver::HasResolver,
4141
src::HasSource,
4242
test_db::TestDB,
43-
tt::Subtree,
43+
tt::TopSubtree,
4444
AdtId, AsMacroCall, Lookup, ModuleDefId,
4545
};
4646

@@ -313,14 +313,14 @@ struct IdentityWhenValidProcMacroExpander;
313313
impl ProcMacroExpander for IdentityWhenValidProcMacroExpander {
314314
fn expand(
315315
&self,
316-
subtree: &Subtree,
317-
_: Option<&Subtree>,
316+
subtree: &TopSubtree,
317+
_: Option<&TopSubtree>,
318318
_: &base_db::Env,
319319
_: Span,
320320
_: Span,
321321
_: Span,
322322
_: Option<String>,
323-
) -> Result<Subtree, ProcMacroExpansionError> {
323+
) -> Result<TopSubtree, ProcMacroExpansionError> {
324324
let (parse, _) = syntax_bridge::token_tree_to_syntax_node(
325325
subtree,
326326
syntax_bridge::TopEntryPoint::MacroItems,

‎src/tools/rust-analyzer/crates/hir-def/src/nameres/attr_resolution.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,8 @@ pub(super) fn attr_macro_as_call_id(
110110
) -> MacroCallId {
111111
let arg = match macro_attr.input.as_deref() {
112112
Some(AttrInput::TokenTree(tt)) => {
113-
let mut tt = tt.as_ref().clone();
114-
tt.delimiter.kind = tt::DelimiterKind::Invisible;
113+
let mut tt = tt.clone();
114+
tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible;
115115
Some(tt)
116116
}
117117

0 commit comments

Comments
 (0)