Auto merge of rust-lang#12231 - jonas-schievink:fix-float-macro-panic, r=jonas-schievink

fix: fix "X is not a valid punct" panic with floats in macros

Should fix rust-lang/rust-analyzer#12211

Author: bors

crates/hir-def/src/macro_expansion_tests/proc_macros.rs

@@ -92,3 +92,40 @@ fn foo() {
 }"##]],
     );
 }
+
+#[test]
+fn float_parsing_panic() {
+    // Regression test for https://github.com/rust-lang/rust-analyzer/issues/12211
+    check(
+        r#"
+//- proc_macros: identity
+macro_rules! id {
+    ($($t:tt)*) => {
+        $($t)*
+    };
+}
+
+id! {
+    #[proc_macros::identity]
+    impl Foo for WrapBj {
+        async fn foo(&self) {
+            self.0. id().await;
+        }
+    }
+}
+"#,
+        expect![[r##"
+macro_rules! id {
+    ($($t:tt)*) => {
+        $($t)*
+    };
+}
+
+#[proc_macros::identity] impl Foo for WrapBj {
+    async fn foo(&self ) {
+        self .0.id().await ;
+    }
+}
+"##]],
+    );
+}

crates/mbe/src/syntax_bridge.rs

@@ -243,6 +243,8 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree {
             let char = match token.to_char(conv) {
                 Some(c) => c,
                 None => {
+                    // FIXME: this isn't really correct, `to_char` yields the *first* char of the token,
+                    // and this is relevant when eg. creating 2 `tt::Punct` from a single `::` token
                     panic!("Token from lexer must be single char: token = {:#?}", token);
                 }
             };

crates/parser/src/grammar.rs

@@ -324,7 +324,9 @@ fn name_ref_or_index(p: &mut Parser) {
     );
     let m = p.start();
     if p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) {
-        p.bump_remap(INT_NUMBER);
+        // Ideally we'd remap this to `INT_NUMBER` instead, but that causes the MBE conversion to
+        // lose track of what's a float and what isn't, causing panics.
+        p.bump_remap(FLOAT_NUMBER_PART);
     } else {
         p.bump_any();
     }

crates/parser/test_data/parser/inline/ok/0011_field_expr.rast

@@ -50,7 +50,7 @@ SOURCE_FILE
                       IDENT "x"
               DOT "."
               NAME_REF
-                INT_NUMBER "0"
+                FLOAT_NUMBER_PART "0"
             DOT "."
             WHITESPACE " "
             NAME_REF
@@ -67,10 +67,10 @@ SOURCE_FILE
                       IDENT "x"
               DOT "."
               NAME_REF
-                INT_NUMBER "0"
+                FLOAT_NUMBER_PART "0"
             DOT "."
             NAME_REF
-              INT_NUMBER "1"
+              FLOAT_NUMBER_PART "1"
           SEMICOLON ";"
         WHITESPACE "\n    "
         EXPR_STMT