Auto merge of rust-lang#16450 - Urhengulas:edition-aware-parser, r=Veykril

internal: Prepare parser interface for editions

Author: bors

src/tools/rust-analyzer/Cargo.lock

@@ -316,8 +316,11 @@ impl ProcMacroExpander for IdentityWhenValidProcMacroExpander {
         _: Span,
         _: Span,
     ) -> Result<Subtree, ProcMacroExpansionError> {
-        let (parse, _) =
-            ::mbe::token_tree_to_syntax_node(subtree, ::mbe::TopEntryPoint::MacroItems);
+        let (parse, _) = ::mbe::token_tree_to_syntax_node(
+            subtree,
+            ::mbe::TopEntryPoint::MacroItems,
+            span::Edition::CURRENT,
+        );
         if parse.errors().is_empty() {
             Ok(subtree.clone())
         } else {

src/tools/rust-analyzer/crates/hir-def/src/macro_expansion_tests/mod.rs

@@ -534,8 +534,7 @@ impl DefCollector<'_> {
             Edition::Edition2015 => name![rust_2015],
             Edition::Edition2018 => name![rust_2018],
             Edition::Edition2021 => name![rust_2021],
-            // FIXME: update this when rust_2024 exists
-            Edition::Edition2024 => name![rust_2021],
+            Edition::Edition2024 => name![rust_2024],
         };
 
         let path_kind = match self.def_map.data.edition {

src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs

@@ -32,6 +32,7 @@ tt.workspace = true
 mbe.workspace = true
 limit.workspace = true
 span.workspace = true
+parser.workspace = true
 
 [dev-dependencies]
 expect-test = "1.4.0"

src/tools/rust-analyzer/crates/hir-expand/Cargo.toml

@@ -204,7 +204,11 @@ struct BasicAdtInfo {
 }
 
 fn parse_adt(tt: &tt::Subtree, call_site: Span) -> Result<BasicAdtInfo, ExpandError> {
-    let (parsed, tm) = &mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MacroItems);
+    let (parsed, tm) = &mbe::token_tree_to_syntax_node(
+        tt,
+        mbe::TopEntryPoint::MacroItems,
+        parser::Edition::CURRENT,
+    );
     let macro_items = ast::MacroItems::cast(parsed.syntax_node())
         .ok_or_else(|| ExpandError::other("invalid item definition"))?;
     let item = macro_items.items().next().ok_or_else(|| ExpandError::other("no item found"))?;

src/tools/rust-analyzer/crates/hir-expand/src/builtin_derive_macro.rs

@@ -676,7 +676,7 @@ fn token_tree_to_syntax_node(
         ExpandTo::Type => mbe::TopEntryPoint::Type,
         ExpandTo::Expr => mbe::TopEntryPoint::Expr,
     };
-    mbe::token_tree_to_syntax_node(tt, entry_point)
+    mbe::token_tree_to_syntax_node(tt, entry_point, parser::Edition::CURRENT)
 }
 
 fn check_tt_count(tt: &tt::Subtree) -> Result<(), ExpandResult<()>> {

src/tools/rust-analyzer/crates/hir-expand/src/db.rs

@@ -417,7 +417,11 @@ mod tests {
         expect.assert_eq(&actual);
 
         // the fixed-up tree should be syntactically valid
-        let (parse, _) = mbe::token_tree_to_syntax_node(&tt, ::mbe::TopEntryPoint::MacroItems);
+        let (parse, _) = mbe::token_tree_to_syntax_node(
+            &tt,
+            ::mbe::TopEntryPoint::MacroItems,
+            parser::Edition::CURRENT,
+        );
         assert!(
             parse.errors().is_empty(),
             "parse has syntax errors. parse tree:\n{:#?}",

src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs

@@ -303,6 +303,7 @@ pub mod known {
         rust_2015,
         rust_2018,
         rust_2021,
+        rust_2024,
         v1,
         new_display,
         new_debug,

src/tools/rust-analyzer/crates/hir-expand/src/name.rs

@@ -1157,7 +1157,7 @@ fn iterate_trait_method_candidates(
         {
             // FIXME: this should really be using the edition of the method name's span, in case it
             // comes from a macro
-            if db.crate_graph()[krate].edition < Edition::Edition2021 {
+            if db.crate_graph()[krate].edition < Edition::CURRENT {
                 continue;
             }
         }

src/tools/rust-analyzer/crates/hir-ty/src/method_resolution.rs

@@ -743,9 +743,11 @@ fn match_meta_var(
 ) -> ExpandResult<Option<Fragment>> {
     let fragment = match kind {
         MetaVarKind::Path => {
-            return input.expect_fragment(parser::PrefixEntryPoint::Path).map(|it| {
-                it.map(|it| tt::TokenTree::subtree_or_wrap(it, delim_span)).map(Fragment::Path)
-            });
+            return input
+                .expect_fragment(parser::PrefixEntryPoint::Path, parser::Edition::CURRENT)
+                .map(|it| {
+                    it.map(|it| tt::TokenTree::subtree_or_wrap(it, delim_span)).map(Fragment::Path)
+                });
         }
         MetaVarKind::Ty => parser::PrefixEntryPoint::Ty,
         MetaVarKind::Pat => parser::PrefixEntryPoint::PatTop,
@@ -770,21 +772,23 @@ fn match_meta_var(
                 }
                 _ => {}
             };
-            return input.expect_fragment(parser::PrefixEntryPoint::Expr).map(|tt| {
-                tt.map(|tt| match tt {
-                    tt::TokenTree::Leaf(leaf) => tt::Subtree {
-                        delimiter: tt::Delimiter::invisible_spanned(*leaf.span()),
-                        token_trees: Box::new([leaf.into()]),
-                    },
-                    tt::TokenTree::Subtree(mut s) => {
-                        if s.delimiter.kind == tt::DelimiterKind::Invisible {
-                            s.delimiter.kind = tt::DelimiterKind::Parenthesis;
+            return input
+                .expect_fragment(parser::PrefixEntryPoint::Expr, parser::Edition::CURRENT)
+                .map(|tt| {
+                    tt.map(|tt| match tt {
+                        tt::TokenTree::Leaf(leaf) => tt::Subtree {
+                            delimiter: tt::Delimiter::invisible_spanned(*leaf.span()),
+                            token_trees: Box::new([leaf.into()]),
+                        },
+                        tt::TokenTree::Subtree(mut s) => {
+                            if s.delimiter.kind == tt::DelimiterKind::Invisible {
+                                s.delimiter.kind = tt::DelimiterKind::Parenthesis;
+                            }
+                            s
                         }
-                        s
-                    }
-                })
-                .map(Fragment::Expr)
-            });
+                    })
+                    .map(Fragment::Expr)
+                });
         }
         MetaVarKind::Ident | MetaVarKind::Tt | MetaVarKind::Lifetime | MetaVarKind::Literal => {
             let tt_result = match kind {
@@ -819,7 +823,7 @@ fn match_meta_var(
             return tt_result.map(|it| Some(Fragment::Tokens(it))).into();
         }
     };
-    input.expect_fragment(fragment).map(|it| it.map(Fragment::Tokens))
+    input.expect_fragment(fragment, parser::Edition::CURRENT).map(|it| it.map(Fragment::Tokens))
 }
 
 fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) {

src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs

@@ -119,6 +119,7 @@ where
 pub fn token_tree_to_syntax_node<Ctx>(
     tt: &tt::Subtree<SpanData<Ctx>>,
     entry_point: parser::TopEntryPoint,
+    edition: parser::Edition,
 ) -> (Parse<SyntaxNode>, SpanMap<Ctx>)
 where
     SpanData<Ctx>: Copy + fmt::Debug,
@@ -131,7 +132,7 @@ where
         _ => TokenBuffer::from_subtree(tt),
     };
     let parser_input = to_parser_input(&buffer);
-    let parser_output = entry_point.parse(&parser_input);
+    let parser_output = entry_point.parse(&parser_input, edition);
     let mut tree_sink = TtTreeSink::new(buffer.begin());
     for event in parser_output.iter() {
         match event {
@@ -194,7 +195,8 @@ where
     let mut res = Vec::new();
 
     while iter.peek_n(0).is_some() {
-        let expanded = iter.expect_fragment(parser::PrefixEntryPoint::Expr);
+        let expanded =
+            iter.expect_fragment(parser::PrefixEntryPoint::Expr, parser::Edition::CURRENT);
 
         res.push(match expanded.value {
             None => break,

src/tools/rust-analyzer/crates/mbe/src/syntax_bridge.rs

@@ -140,10 +140,11 @@ impl<'a, S: Copy + fmt::Debug> TtIter<'a, S> {
     pub(crate) fn expect_fragment(
         &mut self,
         entry_point: parser::PrefixEntryPoint,
+        edition: parser::Edition,
     ) -> ExpandResult<Option<tt::TokenTree<S>>> {
         let buffer = tt::buffer::TokenBuffer::from_tokens(self.inner.as_slice());
         let parser_input = to_parser_input(&buffer);
-        let tree_traversal = entry_point.parse(&parser_input);
+        let tree_traversal = entry_point.parse(&parser_input, edition);
         let mut cursor = buffer.begin();
         let mut error = false;
         for step in tree_traversal.iter() {

src/tools/rust-analyzer/crates/mbe/src/tt_iter.rs

@@ -0,0 +1,55 @@
+//! The edition of the Rust language used in a crate.
+// Ideally this would be defined in the span crate, but the dependency chain is all over the place
+// wrt to span, parser and syntax.
+use std::fmt;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum Edition {
+    Edition2015,
+    Edition2018,
+    Edition2021,
+    Edition2024,
+}
+
+impl Edition {
+    pub const CURRENT: Edition = Edition::Edition2021;
+    pub const DEFAULT: Edition = Edition::Edition2015;
+}
+
+#[derive(Debug)]
+pub struct ParseEditionError {
+    invalid_input: String,
+}
+
+impl std::error::Error for ParseEditionError {}
+impl fmt::Display for ParseEditionError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "invalid edition: {:?}", self.invalid_input)
+    }
+}
+
+impl std::str::FromStr for Edition {
+    type Err = ParseEditionError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let res = match s {
+            "2015" => Edition::Edition2015,
+            "2018" => Edition::Edition2018,
+            "2021" => Edition::Edition2021,
+            "2024" => Edition::Edition2024,
+            _ => return Err(ParseEditionError { invalid_input: s.to_owned() }),
+        };
+        Ok(res)
+    }
+}
+
+impl fmt::Display for Edition {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(match self {
+            Edition::Edition2015 => "2015",
+            Edition::Edition2018 => "2018",
+            Edition::Edition2021 => "2021",
+            Edition::Edition2024 => "2024",
+        })
+    }
+}

src/tools/rust-analyzer/crates/parser/src/edition.rs

@@ -26,6 +26,7 @@ extern crate ra_ap_rustc_lexer as rustc_lexer;
 #[cfg(feature = "in-rust-tree")]
 extern crate rustc_lexer;
 
+mod edition;
 mod event;
 mod grammar;
 mod input;
@@ -42,6 +43,7 @@ mod tests;
 pub(crate) use token_set::TokenSet;
 
 pub use crate::{
+    edition::Edition,
     input::Input,
     lexed_str::LexedStr,
     output::{Output, Step},
@@ -86,7 +88,7 @@ pub enum TopEntryPoint {
 }
 
 impl TopEntryPoint {
-    pub fn parse(&self, input: &Input) -> Output {
+    pub fn parse(&self, input: &Input, edition: Edition) -> Output {
         let _p = tracing::span!(tracing::Level::INFO, "TopEntryPoint::parse", ?self).entered();
         let entry_point: fn(&'_ mut parser::Parser<'_>) = match self {
             TopEntryPoint::SourceFile => grammar::entry::top::source_file,
@@ -98,7 +100,7 @@ impl TopEntryPoint {
             TopEntryPoint::MetaItem => grammar::entry::top::meta_item,
             TopEntryPoint::MacroEagerInput => grammar::entry::top::eager_macro_input,
         };
-        let mut p = parser::Parser::new(input);
+        let mut p = parser::Parser::new(input, edition);
         entry_point(&mut p);
         let events = p.finish();
         let res = event::process(events);
@@ -150,7 +152,7 @@ pub enum PrefixEntryPoint {
 }
 
 impl PrefixEntryPoint {
-    pub fn parse(&self, input: &Input) -> Output {
+    pub fn parse(&self, input: &Input, edition: Edition) -> Output {
         let entry_point: fn(&'_ mut parser::Parser<'_>) = match self {
             PrefixEntryPoint::Vis => grammar::entry::prefix::vis,
             PrefixEntryPoint::Block => grammar::entry::prefix::block,
@@ -163,7 +165,7 @@ impl PrefixEntryPoint {
             PrefixEntryPoint::Item => grammar::entry::prefix::item,
             PrefixEntryPoint::MetaItem => grammar::entry::prefix::meta_item,
         };
-        let mut p = parser::Parser::new(input);
+        let mut p = parser::Parser::new(input, edition);
         entry_point(&mut p);
         let events = p.finish();
         event::process(events)
@@ -187,9 +189,9 @@ impl Reparser {
     ///
     /// Tokens must start with `{`, end with `}` and form a valid brace
     /// sequence.
-    pub fn parse(self, tokens: &Input) -> Output {
+    pub fn parse(self, tokens: &Input, edition: Edition) -> Output {
         let Reparser(r) = self;
-        let mut p = parser::Parser::new(tokens);
+        let mut p = parser::Parser::new(tokens, edition);
         r(&mut p);
         let events = p.finish();
         event::process(events)

src/tools/rust-analyzer/crates/parser/src/lib.rs

@@ -8,6 +8,7 @@ use limit::Limit;
 use crate::{
     event::Event,
     input::Input,
+    Edition,
     SyntaxKind::{self, EOF, ERROR, TOMBSTONE},
     TokenSet, T,
 };
@@ -26,13 +27,14 @@ pub(crate) struct Parser<'t> {
     pos: usize,
     events: Vec<Event>,
     steps: Cell<u32>,
+    _edition: Edition,
 }
 
 static PARSER_STEP_LIMIT: Limit = Limit::new(15_000_000);
 
 impl<'t> Parser<'t> {
-    pub(super) fn new(inp: &'t Input) -> Parser<'t> {
-        Parser { inp, pos: 0, events: Vec::new(), steps: Cell::new(0) }
+    pub(super) fn new(inp: &'t Input, edition: Edition) -> Parser<'t> {
+        Parser { inp, pos: 0, events: Vec::new(), steps: Cell::new(0), _edition: edition }
     }
 
     pub(crate) fn finish(self) -> Vec<Event> {

src/tools/rust-analyzer/crates/parser/src/parser.rs

@@ -88,7 +88,7 @@ fn parse_inline_err() {
 fn parse(entry: TopEntryPoint, text: &str) -> (String, bool) {
     let lexed = LexedStr::new(text);
     let input = lexed.to_input();
-    let output = entry.parse(&input);
+    let output = entry.parse(&input, crate::Edition::CURRENT);
 
     let mut buf = String::new();
     let mut errors = Vec::new();

src/tools/rust-analyzer/crates/parser/src/tests.rs

@@ -86,7 +86,7 @@ fn check(entry: PrefixEntryPoint, input: &str, prefix: &str) {
     let input = lexed.to_input();
 
     let mut n_tokens = 0;
-    for step in entry.parse(&input).iter() {
+    for step in entry.parse(&input, crate::Edition::CURRENT).iter() {
         match step {
             Step::Token { n_input_tokens, .. } => n_tokens += n_input_tokens as usize,
             Step::FloatSplit { .. } => n_tokens += 1,

src/tools/rust-analyzer/crates/parser/src/tests/prefix_entries.rs

@@ -14,6 +14,7 @@ la-arena.workspace = true
 salsa.workspace = true
 rustc-hash.workspace = true
 hashbrown.workspace = true
+text-size.workspace = true
 
 # local deps
 vfs.workspace = true