Auto merge of rust-lang#12241 - jonas-schievink:does-not-float-my-boat, r=jonas-schievink

fix: revert float parsing "fix" to avoid macro-related panics

Reverts rust-lang/rust-analyzer#12149 and the follow-up fixes, while keeping their tests.

rust-lang/rust-analyzer#12149 has caused many unexpected panics related to macros, and the fixes for those are not straightforward and further complicate the MBE token conversion logic, which was already fairly hard to follow before these fixes.

Author: bors

crates/hir-def/src/body/lower.rs

@@ -972,7 +972,7 @@ impl From<ast::LiteralKind> for Literal {
                 }
             }
             LiteralKind::FloatNumber(lit) => {
-                let ty = lit.suffix().and_then(|s| BuiltinFloat::from_suffix(&s));
+                let ty = lit.suffix().and_then(BuiltinFloat::from_suffix);
                 Literal::Float(Default::default(), ty)
             }
             LiteralKind::ByteString(bs) => {

crates/hir-def/src/macro_expansion_tests/mbe.rs

@@ -48,6 +48,8 @@ struct#10 MyTraitMap2#32 {#13
 
 #[test]
 fn token_mapping_floats() {
+    // Regression test for https://github.com/rust-lang/rust-analyzer/issues/12216
+    // (and related issues)
     check(
         r#"
 // +tokenids
@@ -87,9 +89,9 @@ macro_rules! f {#0
 // }
 fn#19 main#20(#21)#21 {#22
     1#23;#24
-    1#26.0;
-    let x#31 =#22 1;
-}
+    1.0#25;#26
+    let#27 x#28 =#29 1#30;#31
+}#22
 
 
 "##]],

crates/hir-def/src/macro_expansion_tests/mbe/meta_syntax.rs

@@ -80,7 +80,7 @@ macro_rules! f3 { ($i:_) => () }
 
 #[test]
 fn test_rustc_issue_57597() {
-    // <https://github.com/rust-lang/rust/blob/master/src/test/ui/macros/issue-57597.rs>
+    // <https://github.com/rust-lang/rust/blob/master/src/test/ui/issues/issue-57597.rs>
     check(
         r#"
 macro_rules! m0 { ($($($i:ident)?)+) => {}; }

crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs

@@ -38,7 +38,6 @@ macro_rules! m {
         let _ = 12E+99_f64;
         let _ = "rust1";
         let _ = -92;
-        let _ = -1.3e4f32;
     }
 }
 fn f() {
@@ -53,7 +52,6 @@ macro_rules! m {
         let _ = 12E+99_f64;
         let _ = "rust1";
         let _ = -92;
-        let _ = -1.3e4f32;
     }
 }
 fn f() {
@@ -62,7 +60,6 @@ fn f() {
     let _ = 12E+99_f64;
     let _ = "rust1";
     let _ = -92;
-    let _ = -1.3e4f32;
 }
 "#]],
     );
@@ -153,55 +150,51 @@ $ = ();
 }
 
 #[test]
-fn float_literal_in_output() {
+fn float_literal_in_tt() {
     check(
         r#"
 macro_rules! constant {
-    ($e:expr ;) => {$e};
+    ($( $ret:expr; )*) => {};
 }
-
-const _: () = constant!(0.0;);
-const _: () = constant!(0.;);
-const _: () = constant!(0e0;);
+macro_rules! float_const_impl {
+    () => ( constant!(0.3; 3.3;); );
+}
+float_const_impl! {}
 "#,
         expect![[r#"
 macro_rules! constant {
-    ($e:expr ;) => {$e};
+    ($( $ret:expr; )*) => {};
 }
-
-const _: () = 0.0;
-const _: () = 0.;
-const _: () = 0e0;
+macro_rules! float_const_impl {
+    () => ( constant!(0.3; 3.3;); );
+}
+constant!(0.3;
+3.3;
+);
 "#]],
     );
 }
 
 #[test]
-fn float_literal_in_tt() {
+fn float_literal_in_output() {
     check(
         r#"
 macro_rules! constant {
-    ($( $ret:expr; )*) => {};
-}
-
-macro_rules! float_const_impl {
-    () => ( constant!(0.3; 3.3;); );
+    ($e:expr ;) => {$e};
 }
 
-float_const_impl! {}
+const _: () = constant!(0.0;);
+const _: () = constant!(0.;);
+const _: () = constant!(0e0;);
 "#,
         expect![[r#"
 macro_rules! constant {
-    ($( $ret:expr; )*) => {};
-}
-
-macro_rules! float_const_impl {
-    () => ( constant!(0.3; 3.3;); );
+    ($e:expr ;) => {$e};
 }
 
-constant!(0.3;
-3.3;
-);
+const _: () = 0.0;
+const _: () = 0.;
+const _: () = 0e0;
 "#]],
     );
 }

crates/hir-def/src/macro_expansion_tests/proc_macros.rs

@@ -104,8 +104,7 @@ macro_rules! id {
         $($t)*
     };
 }
-
-id! {
+id /*+errors*/! {
     #[proc_macros::identity]
     impl Foo for WrapBj {
         async fn foo(&self) {
@@ -120,7 +119,7 @@ macro_rules! id {
         $($t)*
     };
 }
-
+/* parse error: expected SEMICOLON */
 #[proc_macros::identity] impl Foo for WrapBj {
     async fn foo(&self ) {
         self .0.id().await ;

crates/hir-expand/src/builtin_fn_macro.rs

@@ -4,7 +4,10 @@ use base_db::{AnchoredPath, Edition, FileId};
 use cfg::CfgExpr;
 use either::Either;
 use mbe::{parse_exprs_with_sep, parse_to_token_tree};
-use syntax::{ast, SmolStr};
+use syntax::{
+    ast::{self, AstToken},
+    SmolStr,
+};
 
 use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc};
 
@@ -355,7 +358,14 @@ fn unreachable_expand(
 }
 
 fn unquote_str(lit: &tt::Literal) -> Option<String> {
-    let token = ast::make::literal(&lit.to_string()).as_string()?;
+    let lit = ast::make::tokens::literal(&lit.to_string());
+    let token = ast::String::cast(lit)?;
+    token.value().map(|it| it.into_owned())
+}
+
+fn unquote_byte_string(lit: &tt::Literal) -> Option<Vec<u8>> {
+    let lit = ast::make::tokens::literal(&lit.to_string());
+    let token = ast::ByteString::cast(lit)?;
     token.value().map(|it| it.into_owned())
 }
 
@@ -432,16 +442,12 @@ fn concat_bytes_expand(
     for (i, t) in tt.token_trees.iter().enumerate() {
         match t {
             tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
-                let lit = ast::make::literal(&lit.to_string());
-                match lit.kind() {
-                    ast::LiteralKind::ByteString(s) => {
-                        s.value()
-                            .unwrap_or_default()
-                            .into_iter()
-                            .for_each(|x| bytes.push(x.to_string()));
-                    }
-                    ast::LiteralKind::Byte(_) => {
-                        bytes.push(lit.to_string());
+                let token = ast::make::tokens::literal(&lit.to_string());
+                match token.kind() {
+                    syntax::SyntaxKind::BYTE => bytes.push(token.text().to_string()),
+                    syntax::SyntaxKind::BYTE_STRING => {
+                        let components = unquote_byte_string(lit).unwrap_or_else(Vec::new);
+                        components.into_iter().for_each(|x| bytes.push(x.to_string()));
                     }
                     _ => {
                         err.get_or_insert(mbe::ExpandError::UnexpectedToken.into());
@@ -475,10 +481,10 @@ fn concat_bytes_expand_subtree(
     for (ti, tt) in tree.token_trees.iter().enumerate() {
         match tt {
             tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
-                let lit = ast::make::literal(&lit.to_string());
+                let lit = ast::make::tokens::literal(&lit.to_string());
                 match lit.kind() {
-                    ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte(_) => {
-                        bytes.push(lit.to_string());
+                    syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => {
+                        bytes.push(lit.text().to_string())
                     }
                     _ => {
                         return Err(mbe::ExpandError::UnexpectedToken.into());

crates/hir-ty/src/tests/simple.rs

@@ -2733,14 +2733,3 @@ fn f() {
     "#,
     );
 }
-
-#[test]
-fn nested_tuple_index() {
-    check_no_mismatches(
-        r#"
-fn main() {
-    let fld: i32 = ((0,),).0.0;
-}
-"#,
-    );
-}

crates/ide-completion/src/completions/dot.rs

@@ -793,24 +793,4 @@ fn main() {
 ",
         )
     }
-
-    #[test]
-    fn tuple_index_completion() {
-        check(
-            r#"
-struct I;
-impl I {
-    fn i_method(&self) {}
-}
-struct S((), I);
-
-fn f(s: S) {
-    s.1.$0
-}
-"#,
-            expect![[r#"
-                me i_method() fn(&self)
-            "#]],
-        );
-    }
 }

crates/ide-completion/src/completions/postfix.rs

@@ -5,7 +5,7 @@ mod format_like;
 use hir::{Documentation, HasAttrs};
 use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap};
 use syntax::{
-    ast::{self, AstNode, LiteralKind},
+    ast::{self, AstNode, AstToken},
     SyntaxKind::{EXPR_STMT, STMT_LIST},
     TextRange, TextSize,
 };
@@ -194,7 +194,7 @@ pub(crate) fn complete_postfix(acc: &mut Completions, ctx: &CompletionContext) {
     }
 
     if let ast::Expr::Literal(literal) = dot_receiver.clone() {
-        if let LiteralKind::String(literal_text) = literal.kind() {
+        if let Some(literal_text) = ast::String::cast(literal.token()) {
             add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text);
         }
     }

crates/ide-completion/src/context.rs

@@ -1062,7 +1062,7 @@ impl<'a> CompletionContext<'a> {
                     let receiver_is_ambiguous_float_literal = match &receiver {
                         Some(ast::Expr::Literal(l)) => matches! {
                             l.kind(),
-                            ast::LiteralKind::FloatNumber { .. } if l.syntax().last_token().map_or(false, |it| it.kind() == T![.])
+                            ast::LiteralKind::FloatNumber { .. } if l.syntax().last_token().map_or(false, |it| it.text().ends_with('.'))
                         },
                         _ => false,
                     };

crates/ide/src/syntax_highlighting/highlight.rs

@@ -30,15 +30,7 @@ pub(super) fn token(sema: &Semantics<RootDatabase>, token: SyntaxToken) -> Optio
         INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => {
             SymbolKind::Field.into()
         }
-        INT_NUMBER | FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1
-        | FLOAT_NUMBER_START_2 => HlTag::NumericLiteral.into(),
-        DOT if matches!(
-            token.prev_token().map(|n| n.kind()),
-            Some(FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2)
-        ) =>
-        {
-            HlTag::NumericLiteral.into()
-        }
+        INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(),
         BYTE => HlTag::ByteLiteral.into(),
         CHAR => HlTag::CharLiteral.into(),
         IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => {