Auto merge of rust-lang#14061 - Veykril:generic-tt, r=Veykril

Make tt generic over the span data

This also fixes up our delimiter representation in tt, it is no longer optional (we use invisible delims in the same way as before, that is still incorrectly) and we now store two spans instead of one.

These changes should help with adjusting our token map. Though this will probably break proc-macros in some ways, will need to test that for now.

Author: bors

crates/base-db/src/fixture.rs

@@ -6,7 +6,7 @@ use rustc_hash::FxHashMap;
 use test_utils::{
     extract_range_or_offset, Fixture, RangeOrOffset, CURSOR_MARKER, ESCAPED_CURSOR_MARKER,
 };
-use tt::Subtree;
+use tt::token_id::Subtree;
 use vfs::{file_set::FileSet, VfsPath};
 
 use crate::{
@@ -495,16 +495,15 @@ impl ProcMacroExpander for MirrorProcMacroExpander {
         _: &Env,
     ) -> Result<Subtree, ProcMacroExpansionError> {
         fn traverse(input: &Subtree) -> Subtree {
-            let mut res = Subtree::default();
-            res.delimiter = input.delimiter;
+            let mut token_trees = vec![];
             for tt in input.token_trees.iter().rev() {
                 let tt = match tt {
                     tt::TokenTree::Leaf(leaf) => tt::TokenTree::Leaf(leaf.clone()),
                     tt::TokenTree::Subtree(sub) => tt::TokenTree::Subtree(traverse(sub)),
                 };
-                res.token_trees.push(tt);
+                token_trees.push(tt);
             }
-            res
+            Subtree { delimiter: input.delimiter, token_trees }
         }
         Ok(traverse(input))
     }

crates/base-db/src/input.rs

@@ -12,7 +12,7 @@ use cfg::CfgOptions;
 use rustc_hash::FxHashMap;
 use stdx::hash::{NoHashHashMap, NoHashHashSet};
 use syntax::SmolStr;
-use tt::Subtree;
+use tt::token_id::Subtree;
 use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath};
 
 /// Files are grouped into source roots. A source root is a directory on the

crates/cfg/src/cfg_expr.rs

@@ -66,7 +66,7 @@ impl From<CfgAtom> for CfgExpr {
 }
 
 impl CfgExpr {
-    pub fn parse(tt: &tt::Subtree) -> CfgExpr {
+    pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
         next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
     }
     /// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@@ -85,7 +85,7 @@ impl CfgExpr {
     }
 }
 
-fn next_cfg_expr(it: &mut SliceIter<'_, tt::TokenTree>) -> Option<CfgExpr> {
+fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
     let name = match it.next() {
         None => return None,
         Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),

crates/hir-def/src/adt.rs

@@ -2,6 +2,7 @@
 
 use std::sync::Arc;
 
+use crate::tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
 use base_db::CrateId;
 use either::Either;
 use hir_expand::{
@@ -12,7 +13,6 @@ use intern::Interned;
 use la_arena::{Arena, ArenaMap};
 use rustc_abi::{Integer, IntegerType};
 use syntax::ast::{self, HasName, HasVisibility};
-use tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
 
 use crate::{
     body::{CfgExpander, LowerCtx},
@@ -82,7 +82,7 @@ fn repr_from_value(
 
 fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
     match tt.delimiter {
-        Some(Delimiter { kind: DelimiterKind::Parenthesis, .. }) => {}
+        Delimiter { kind: DelimiterKind::Parenthesis, .. } => {}
         _ => return None,
     }
 

crates/hir-def/src/attr.rs

@@ -16,7 +16,6 @@ use syntax::{
     ast::{self, HasAttrs, IsString},
     AstPtr, AstToken, SmolStr, TextRange, TextSize,
 };
-use tt::Subtree;
 
 use crate::{
     db::DefDatabase,
@@ -234,7 +233,7 @@ impl Attrs {
 
     pub fn has_doc_hidden(&self) -> bool {
         self.by_key("doc").tt_values().any(|tt| {
-            tt.delimiter_kind() == Some(DelimiterKind::Parenthesis) &&
+            tt.delimiter.kind == DelimiterKind::Parenthesis &&
                 matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden")
         })
     }
@@ -628,7 +627,7 @@ pub struct AttrQuery<'attr> {
 }
 
 impl<'attr> AttrQuery<'attr> {
-    pub fn tt_values(self) -> impl Iterator<Item = &'attr Subtree> {
+    pub fn tt_values(self) -> impl Iterator<Item = &'attr crate::tt::Subtree> {
         self.attrs().filter_map(|attr| attr.token_tree_value())
     }
 

crates/hir-def/src/data.rs

@@ -142,7 +142,7 @@ impl FunctionData {
     }
 }
 
-fn parse_rustc_legacy_const_generics(tt: &tt::Subtree) -> Box<[u32]> {
+fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> {
     let mut indices = Vec::new();
     for args in tt.token_trees.chunks(2) {
         match &args[0] {

crates/hir-def/src/lib.rs

@@ -79,6 +79,8 @@ use nameres::DefMap;
 use stdx::impl_from;
 use syntax::ast;
 
+use ::tt::token_id as tt;
+
 use crate::{
     adt::VariantData,
     builtin_type::BuiltinType,
@@ -973,15 +975,19 @@ fn attr_macro_as_call_id(
     def: MacroDefId,
     is_derive: bool,
 ) -> MacroCallId {
-    let mut arg = match macro_attr.input.as_deref() {
-        Some(AttrInput::TokenTree(tt, map)) => (tt.clone(), map.clone()),
-        _ => Default::default(),
+    let arg = match macro_attr.input.as_deref() {
+        Some(AttrInput::TokenTree(tt, map)) => (
+            {
+                let mut tt = tt.clone();
+                tt.delimiter = tt::Delimiter::UNSPECIFIED;
+                tt
+            },
+            map.clone(),
+        ),
+        _ => (tt::Subtree::empty(), Default::default()),
     };
 
-    // The parentheses are always disposed here.
-    arg.0.delimiter = None;
-
-    let res = def.as_lazy_macro(
+    def.as_lazy_macro(
         db.upcast(),
         krate,
         MacroCallKind::Attr {
@@ -990,8 +996,7 @@ fn attr_macro_as_call_id(
             invoc_attr_index: macro_attr.id,
             is_derive,
         },
-    );
-    res
+    )
 }
 intern::impl_internable!(
     crate::type_ref::TypeRef,

crates/hir-def/src/macro_expansion_tests.rs

@@ -30,7 +30,7 @@ use syntax::{
     SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT},
     SyntaxNode, TextRange, T,
 };
-use tt::{Subtree, TokenId};
+use tt::token_id::{Subtree, TokenId};
 
 use crate::{
     db::DefDatabase, macro_id_to_def_id, nameres::ModuleSource, resolver::HasResolver,
@@ -253,9 +253,9 @@ fn extract_id_ranges(ranges: &mut Vec<(TextRange, TokenId)>, map: &TokenMap, tre
     tree.token_trees.iter().for_each(|tree| match tree {
         tt::TokenTree::Leaf(leaf) => {
             let id = match leaf {
-                tt::Leaf::Literal(it) => it.id,
-                tt::Leaf::Punct(it) => it.id,
-                tt::Leaf::Ident(it) => it.id,
+                tt::Leaf::Literal(it) => it.span,
+                tt::Leaf::Punct(it) => it.span,
+                tt::Leaf::Ident(it) => it.span,
             };
             ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id)));
         }

crates/hir-def/src/nameres/collector.rs

@@ -46,6 +46,7 @@ use crate::{
     },
     path::{ImportAlias, ModPath, PathKind},
     per_ns::PerNs,
+    tt,
     visibility::{RawVisibility, Visibility},
     AdtId, AstId, AstIdWithPath, ConstLoc, EnumLoc, EnumVariantId, ExternBlockLoc, FunctionId,
     FunctionLoc, ImplLoc, Intern, ItemContainerId, LocalModuleId, Macro2Id, Macro2Loc,
@@ -83,7 +84,8 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, mut def_map: DefMap, tree_id: T
                 .enumerate()
                 .map(|(idx, it)| {
                     // FIXME: a hacky way to create a Name from string.
-                    let name = tt::Ident { text: it.name.clone(), id: tt::TokenId::unspecified() };
+                    let name =
+                        tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() };
                     (
                         name.as_name(),
                         ProcMacroExpander::new(def_map.krate, base_db::ProcMacroId(idx as u32)),
@@ -451,7 +453,10 @@ impl DefCollector<'_> {
                         directive.module_id,
                         MacroCallKind::Attr {
                             ast_id: ast_id.ast_id,
-                            attr_args: Default::default(),
+                            attr_args: std::sync::Arc::new((
+                                tt::Subtree::empty(),
+                                Default::default(),
+                            )),
                             invoc_attr_index: attr.id,
                             is_derive: false,
                         },
@@ -1947,7 +1952,8 @@ impl ModCollector<'_, '_> {
             let name = match attrs.by_key("rustc_builtin_macro").string_value() {
                 Some(it) => {
                     // FIXME: a hacky way to create a Name from string.
-                    name = tt::Ident { text: it.clone(), id: tt::TokenId::unspecified() }.as_name();
+                    name =
+                        tt::Ident { text: it.clone(), span: tt::TokenId::unspecified() }.as_name();
                     &name
                 }
                 None => {

crates/hir-def/src/nameres/proc_macro.rs

@@ -1,9 +1,9 @@
 //! Nameres-specific procedural macro data and helpers.
 
 use hir_expand::name::{AsName, Name};
-use tt::{Leaf, TokenTree};
 
 use crate::attr::Attrs;
+use crate::tt::{Leaf, TokenTree};
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct ProcMacroDef {

crates/hir-expand/src/attrs.rs

@@ -8,13 +8,13 @@ use intern::Interned;
 use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct};
 use smallvec::{smallvec, SmallVec};
 use syntax::{ast, match_ast, AstNode, SmolStr, SyntaxNode};
-use tt::Subtree;
 
 use crate::{
     db::AstDatabase,
     hygiene::Hygiene,
     mod_path::{ModPath, PathKind},
     name::AsName,
+    tt::{self, Subtree},
     InFile,
 };
 
@@ -117,7 +117,10 @@ impl RawAttrs {
                 let index = attr.id;
                 let attrs =
                     parts.enumerate().take(1 << AttrId::CFG_ATTR_BITS).filter_map(|(idx, attr)| {
-                        let tree = Subtree { delimiter: None, token_trees: attr.to_vec() };
+                        let tree = Subtree {
+                            delimiter: tt::Delimiter::unspecified(),
+                            token_trees: attr.to_vec(),
+                        };
                         // FIXME hygiene
                         let hygiene = Hygiene::new_unhygienic();
                         Attr::from_tt(db, &tree, &hygiene, index.with_cfg_attr(idx))
@@ -266,7 +269,7 @@ impl Attr {
     pub fn parse_path_comma_token_tree(&self) -> Option<impl Iterator<Item = ModPath> + '_> {
         let args = self.token_tree_value()?;
 
-        if args.delimiter_kind() != Some(DelimiterKind::Parenthesis) {
+        if args.delimiter.kind != DelimiterKind::Parenthesis {
             return None;
         }
         let paths = args

crates/hir-expand/src/builtin_attr_macro.rs

@@ -1,6 +1,6 @@
 //! Builtin attributes.
 
-use crate::{db::AstDatabase, name, ExpandResult, MacroCallId, MacroCallKind};
+use crate::{db::AstDatabase, name, tt, ExpandResult, MacroCallId, MacroCallKind};
 
 macro_rules! register_builtin {
     ( $(($name:ident, $variant:ident) => $expand:ident),* ) => {
@@ -97,7 +97,7 @@ fn derive_attr_expand(
     let loc = db.lookup_intern_macro_call(id);
     let derives = match &loc.kind {
         MacroCallKind::Attr { attr_args, is_derive: true, .. } => &attr_args.0,
-        _ => return ExpandResult::ok(Default::default()),
+        _ => return ExpandResult::ok(tt::Subtree::empty()),
     };
     pseudo_derive_attr_expansion(tt, derives)
 }
@@ -110,7 +110,7 @@ pub fn pseudo_derive_attr_expansion(
         tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
             char,
             spacing: tt::Spacing::Alone,
-            id: tt::TokenId::unspecified(),
+            span: tt::TokenId::unspecified(),
         }))
     };
 

crates/hir-expand/src/builtin_derive_macro.rs

@@ -3,11 +3,11 @@
 use base_db::{CrateOrigin, LangCrateOrigin};
 use tracing::debug;
 
+use crate::tt::{self, TokenId};
 use syntax::{
     ast::{self, AstNode, HasGenericParams, HasModuleItem, HasName},
     match_ast,
 };
-use tt::TokenId;
 
 use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId};
 
@@ -92,7 +92,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
     })?;
     let name_token_id =
         token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified);
-    let name_token = tt::Ident { id: name_token_id, text: name.text().into() };
+    let name_token = tt::Ident { span: name_token_id, text: name.text().into() };
     let param_types = params
         .into_iter()
         .flat_map(|param_list| param_list.type_or_const_params())
@@ -101,7 +101,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
                 let ty = param
                     .ty()
                     .map(|ty| mbe::syntax_node_to_token_tree(ty.syntax()).0)
-                    .unwrap_or_default();
+                    .unwrap_or_else(tt::Subtree::empty);
                 Some(ty)
             } else {
                 None
@@ -114,15 +114,15 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
 fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResult<tt::Subtree> {
     let info = match parse_adt(tt) {
         Ok(info) => info,
-        Err(e) => return ExpandResult::only_err(e),
+        Err(e) => return ExpandResult::with_err(tt::Subtree::empty(), e),
     };
     let (params, args): (Vec<_>, Vec<_>) = info
         .param_types
         .into_iter()
         .enumerate()
         .map(|(idx, param_ty)| {
             let ident = tt::Leaf::Ident(tt::Ident {
-                id: tt::TokenId::unspecified(),
+                span: tt::TokenId::unspecified(),
                 text: format!("T{idx}").into(),
             });
             let ident_ = ident.clone();