Merge #11134

11134: internal: tighten up parser API r=matklad a=matklad

It's tempting to expose things like `Expr::parse`, 
but they'll necessary have somewhat ad-hoc semantics. 

Instead, we narrow down the parser's interface strictly 
to what's needed for MBE. For everything else (eg, parsing
imports), the proper way is enclose the input string into 
some context, parse the whole as a file, and then verify 
that the input was parsed as intended.

bors r+
🤖

Co-authored-by: Aleksey Kladov <[email protected]>

Author: bors[bot]

crates/hir/src/attrs.rs

@@ -11,7 +11,7 @@ use hir_def::{
 };
 use hir_expand::{hygiene::Hygiene, MacroDefId};
 use hir_ty::db::HirDatabase;
-use syntax::ast;
+use syntax::{ast, AstNode};
 
 use crate::{
     Adt, AssocItem, Const, ConstParam, Enum, Field, Function, GenericParam, Impl, LifetimeParam,
@@ -147,8 +147,18 @@ fn resolve_doc_path(
         // FIXME
         AttrDefId::MacroDefId(_) => return None,
     };
-    let path = ast::Path::parse(link).ok()?;
-    let modpath = ModPath::from_src(db.upcast(), path, &Hygiene::new_unhygienic())?;
+
+    let modpath = {
+        let ast_path = ast::SourceFile::parse(&format!("type T = {};", link))
+            .syntax_node()
+            .descendants()
+            .find_map(ast::Path::cast)?;
+        if ast_path.to_string() != link {
+            return None;
+        }
+        ModPath::from_src(db.upcast(), ast_path, &Hygiene::new_unhygienic())?
+    };
+
     let resolved = resolver.resolve_module_path_in_items(db.upcast(), &modpath);
     let resolved = if resolved == PerNs::none() {
         resolver.resolve_module_path_in_trait_assoc_items(db.upcast(), &modpath)?

crates/hir_def/src/attr.rs

@@ -714,8 +714,7 @@ impl Attr {
         hygiene: &Hygiene,
         id: AttrId,
     ) -> Option<Attr> {
-        let (parse, _) =
-            mbe::token_tree_to_syntax_node(tt, mbe::ParserEntryPoint::MetaItem).ok()?;
+        let (parse, _) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem).ok()?;
         let ast = ast::Meta::cast(parse.syntax_node())?;
 
         Self::from_src(db, ast, hygiene, id)

crates/hir_expand/src/builtin_derive_macro.rs

@@ -72,7 +72,7 @@ struct BasicAdtInfo {
 }
 
 fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, mbe::ExpandError> {
-    let (parsed, token_map) = mbe::token_tree_to_syntax_node(tt, mbe::ParserEntryPoint::Items)?; // FragmentKind::Items doesn't parse attrs?
+    let (parsed, token_map) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MacroItems)?; // FragmentKind::Items doesn't parse attrs?
     let macro_items = ast::MacroItems::cast(parsed.syntax_node()).ok_or_else(|| {
         debug!("derive node didn't parse");
         mbe::ExpandError::UnexpectedToken

crates/hir_expand/src/db.rs

@@ -497,11 +497,11 @@ fn token_tree_to_syntax_node(
     expand_to: ExpandTo,
 ) -> Result<(Parse<SyntaxNode>, mbe::TokenMap), ExpandError> {
     let entry_point = match expand_to {
-        ExpandTo::Statements => mbe::ParserEntryPoint::Statements,
-        ExpandTo::Items => mbe::ParserEntryPoint::Items,
-        ExpandTo::Pattern => mbe::ParserEntryPoint::Pattern,
-        ExpandTo::Type => mbe::ParserEntryPoint::Type,
-        ExpandTo::Expr => mbe::ParserEntryPoint::Expr,
+        ExpandTo::Statements => mbe::TopEntryPoint::MacroStmts,
+        ExpandTo::Items => mbe::TopEntryPoint::MacroItems,
+        ExpandTo::Pattern => mbe::TopEntryPoint::Pattern,
+        ExpandTo::Type => mbe::TopEntryPoint::Type,
+        ExpandTo::Expr => mbe::TopEntryPoint::Expr,
     };
     mbe::token_tree_to_syntax_node(tt, entry_point)
 }

crates/hir_expand/src/eager.rs

@@ -131,7 +131,7 @@ pub fn expand_eager_macro(
     let arg_file_id = arg_id;
 
     let parsed_args = diagnostic_sink
-        .result(mbe::token_tree_to_syntax_node(&parsed_args, mbe::ParserEntryPoint::Expr))?
+        .result(mbe::token_tree_to_syntax_node(&parsed_args, mbe::TopEntryPoint::Expr))?
         .0;
     let result = eager_macro_recur(
         db,

crates/ide_assists/src/handlers/remove_dbg.rs

@@ -36,9 +36,8 @@ pub(crate) fn remove_dbg(acc: &mut Assists, ctx: &AssistContext) -> Option<()> {
     let input_expressions = input_expressions
         .into_iter()
         .filter_map(|(is_sep, group)| (!is_sep).then(|| group))
-        .map(|mut tokens| ast::Expr::parse(&tokens.join("")))
-        .collect::<Result<Vec<ast::Expr>, _>>()
-        .ok()?;
+        .map(|mut tokens| syntax::hacks::parse_expr_from_str(&tokens.join("")))
+        .collect::<Option<Vec<ast::Expr>>>()?;
 
     let parent = macro_call.syntax().parent()?;
     let (range, text) = match &*input_expressions {

crates/ide_completion/src/completions/attribute.rs

@@ -309,7 +309,7 @@ fn parse_comma_sep_expr(input: ast::TokenTree) -> Option<Vec<ast::Expr>> {
         input_expressions
             .into_iter()
             .filter_map(|(is_sep, group)| (!is_sep).then(|| group))
-            .filter_map(|mut tokens| ast::Expr::parse(&tokens.join("")).ok())
+            .filter_map(|mut tokens| syntax::hacks::parse_expr_from_str(&tokens.join("")))
             .collect::<Vec<ast::Expr>>(),
     )
 }

crates/ide_completion/src/snippet.rs

@@ -212,15 +212,14 @@ fn validate_snippet(
 ) -> Option<(Box<[GreenNode]>, String, Option<Box<str>>)> {
     let mut imports = Vec::with_capacity(requires.len());
     for path in requires.iter() {
-        let path = ast::Path::parse(path).ok()?;
-        let valid_use_path = path.segments().all(|seg| {
-            matches!(seg.kind(), Some(ast::PathSegmentKind::Name(_)))
-                || seg.generic_arg_list().is_none()
-        });
-        if !valid_use_path {
+        let use_path = ast::SourceFile::parse(&format!("use {};", path))
+            .syntax_node()
+            .descendants()
+            .find_map(ast::Path::cast)?;
+        if use_path.syntax().text() != path.as_str() {
             return None;
         }
-        let green = path.syntax().green().into_owned();
+        let green = use_path.syntax().green().into_owned();
         imports.push(green);
     }
     let snippet = snippet.iter().join("\n");

crates/ide_db/src/helpers.rs

@@ -67,7 +67,11 @@ pub fn get_path_at_cursor_in_tt(cursor: &ast::Ident) -> Option<ast::Path> {
         .filter_map(SyntaxElement::into_token)
         .take_while(|tok| tok != cursor);
 
-    ast::Path::parse(&path_tokens.chain(iter::once(cursor.clone())).join("")).ok()
+    syntax::hacks::parse_expr_from_str(&path_tokens.chain(iter::once(cursor.clone())).join(""))
+        .and_then(|expr| match expr {
+            ast::Expr::PathExpr(it) => it.path(),
+            _ => None,
+        })
 }
 
 /// Parses and resolves the path at the cursor position in the given attribute, if it is a derive.
@@ -323,7 +327,12 @@ pub fn parse_tt_as_comma_sep_paths(input: ast::TokenTree) -> Option<Vec<ast::Pat
     let paths = input_expressions
         .into_iter()
         .filter_map(|(is_sep, group)| (!is_sep).then(|| group))
-        .filter_map(|mut tokens| ast::Path::parse(&tokens.join("")).ok())
+        .filter_map(|mut tokens| {
+            syntax::hacks::parse_expr_from_str(&tokens.join("")).and_then(|expr| match expr {
+                ast::Expr::PathExpr(it) => it.path(),
+                _ => None,
+            })
+        })
         .collect();
     Some(paths)
 }

crates/ide_ssr/src/fragments.rs

@@ -0,0 +1,58 @@
+//! When specifying SSR rule, you generally want to map one *kind* of thing to
+//! the same kind of thing: path to path, expression to expression, type to
+//! type.
+//!
+//! The problem is, while this *kind* is generally obvious to the human, the ide
+//! needs to determine it somehow. We do this in a stupid way -- by pasting SSR
+//! rule into different contexts and checking what works.
+
+use syntax::{ast, AstNode, SyntaxNode};
+
+pub(crate) fn ty(s: &str) -> Result<SyntaxNode, ()> {
+    fragment::<ast::Type>("type T = {};", s)
+}
+
+pub(crate) fn item(s: &str) -> Result<SyntaxNode, ()> {
+    fragment::<ast::Item>("{}", s)
+}
+
+pub(crate) fn pat(s: &str) -> Result<SyntaxNode, ()> {
+    fragment::<ast::Pat>("const _: () = {let {} = ();};", s)
+}
+
+pub(crate) fn expr(s: &str) -> Result<SyntaxNode, ()> {
+    fragment::<ast::Expr>("const _: () = {};", s)
+}
+
+pub(crate) fn stmt(s: &str) -> Result<SyntaxNode, ()> {
+    let template = "const _: () = { {}; };";
+    let input = template.replace("{}", s);
+    let parse = syntax::SourceFile::parse(&input);
+    if !parse.errors().is_empty() {
+        return Err(());
+    }
+    let mut node =
+        parse.tree().syntax().descendants().skip(2).find_map(ast::Stmt::cast).ok_or(())?;
+    if !s.ends_with(';') && node.to_string().ends_with(';') {
+        node = node.clone_for_update();
+        node.syntax().last_token().map(|it| it.detach());
+    }
+    if node.to_string() != s {
+        return Err(());
+    }
+    Ok(node.syntax().clone_subtree())
+}
+
+fn fragment<T: AstNode>(template: &str, s: &str) -> Result<SyntaxNode, ()> {
+    let s = s.trim();
+    let input = template.replace("{}", s);
+    let parse = syntax::SourceFile::parse(&input);
+    if !parse.errors().is_empty() {
+        return Err(());
+    }
+    let node = parse.tree().syntax().descendants().find_map(T::cast).ok_or(())?;
+    if node.syntax().text() != s {
+        return Err(());
+    }
+    Ok(node.syntax().clone_subtree())
+}

crates/ide_ssr/src/lib.rs

@@ -71,6 +71,7 @@ mod from_comment;
 mod matching;
 mod nester;
 mod parsing;
+mod fragments;
 mod replacing;
 mod resolving;
 mod search;

crates/ide_ssr/src/parsing.rs

@@ -4,12 +4,12 @@
 //! placeholders, which start with `$`. For replacement templates, this is the final form. For
 //! search patterns, we go further and parse the pattern as each kind of thing that we can match.
 //! e.g. expressions, type references etc.
-
-use crate::errors::bail;
-use crate::{SsrError, SsrPattern, SsrRule};
 use rustc_hash::{FxHashMap, FxHashSet};
 use std::{fmt::Display, str::FromStr};
-use syntax::{ast, AstNode, SmolStr, SyntaxKind, SyntaxNode, T};
+use syntax::{SmolStr, SyntaxKind, SyntaxNode, T};
+
+use crate::errors::bail;
+use crate::{fragments, SsrError, SsrPattern, SsrRule};
 
 #[derive(Debug)]
 pub(crate) struct ParsedRule {
@@ -73,17 +73,16 @@ impl ParsedRule {
             rules: Vec::new(),
         };
 
-        let raw_template_stmt = raw_template.map(ast::Stmt::parse);
-        if let raw_template_expr @ Some(Ok(_)) = raw_template.map(ast::Expr::parse) {
-            builder.try_add(ast::Expr::parse(&raw_pattern), raw_template_expr);
+        let raw_template_stmt = raw_template.map(fragments::stmt);
+        if let raw_template_expr @ Some(Ok(_)) = raw_template.map(fragments::expr) {
+            builder.try_add(fragments::expr(&raw_pattern), raw_template_expr);
         } else {
-            builder.try_add(ast::Expr::parse(&raw_pattern), raw_template_stmt.clone());
+            builder.try_add(fragments::expr(&raw_pattern), raw_template_stmt.clone());
         }
-        builder.try_add(ast::Type::parse(&raw_pattern), raw_template.map(ast::Type::parse));
-        builder.try_add(ast::Item::parse(&raw_pattern), raw_template.map(ast::Item::parse));
-        builder.try_add(ast::Path::parse(&raw_pattern), raw_template.map(ast::Path::parse));
-        builder.try_add(ast::Pat::parse(&raw_pattern), raw_template.map(ast::Pat::parse));
-        builder.try_add(ast::Stmt::parse(&raw_pattern), raw_template_stmt);
+        builder.try_add(fragments::ty(&raw_pattern), raw_template.map(fragments::ty));
+        builder.try_add(fragments::item(&raw_pattern), raw_template.map(fragments::item));
+        builder.try_add(fragments::pat(&raw_pattern), raw_template.map(fragments::pat));
+        builder.try_add(fragments::stmt(&raw_pattern), raw_template_stmt);
         builder.build()
     }
 }
@@ -94,20 +93,20 @@ struct RuleBuilder {
 }
 
 impl RuleBuilder {
-    fn try_add<T: AstNode, T2: AstNode>(
+    fn try_add(
         &mut self,
-        pattern: Result<T, ()>,
-        template: Option<Result<T2, ()>>,
+        pattern: Result<SyntaxNode, ()>,
+        template: Option<Result<SyntaxNode, ()>>,
     ) {
         match (pattern, template) {
             (Ok(pattern), Some(Ok(template))) => self.rules.push(ParsedRule {
                 placeholders_by_stand_in: self.placeholders_by_stand_in.clone(),
-                pattern: pattern.syntax().clone(),
-                template: Some(template.syntax().clone()),
+                pattern,
+                template: Some(template),
             }),
             (Ok(pattern), None) => self.rules.push(ParsedRule {
                 placeholders_by_stand_in: self.placeholders_by_stand_in.clone(),
-                pattern: pattern.syntax().clone(),
+                pattern,
                 template: None,
             }),
             _ => {}

crates/ide_ssr/src/replacing.rs

@@ -1,5 +1,6 @@
 //! Code for applying replacement templates for matches that have previously been found.
 
+use crate::fragments;
 use crate::{resolving::ResolvedRule, Match, SsrMatches};
 use itertools::Itertools;
 use rustc_hash::{FxHashMap, FxHashSet};
@@ -225,12 +226,13 @@ fn token_is_method_call_receiver(token: &SyntaxToken) -> bool {
 
 fn parse_as_kind(code: &str, kind: SyntaxKind) -> Option<SyntaxNode> {
     if ast::Expr::can_cast(kind) {
-        if let Ok(expr) = ast::Expr::parse(code) {
-            return Some(expr.syntax().clone());
+        if let Ok(expr) = fragments::expr(code) {
+            return Some(expr);
         }
-    } else if ast::Item::can_cast(kind) {
-        if let Ok(item) = ast::Item::parse(code) {
-            return Some(item.syntax().clone());
+    }
+    if ast::Item::can_cast(kind) {
+        if let Ok(item) = fragments::item(code) {
+            return Some(item);
         }
     }
     None

crates/ide_ssr/src/tests.rs

@@ -331,6 +331,15 @@ fn ssr_struct_lit() {
     )
 }
 
+#[test]
+fn ssr_struct_def() {
+    assert_ssr_transform(
+        "struct Foo { $f: $t } ==>> struct Foo($t);",
+        r#"struct Foo { field: i32 }"#,
+        expect![[r#"struct Foo(i32);"#]],
+    )
+}
+
 #[test]
 fn ignores_whitespace() {
     assert_matches("1+2", "fn f() -> i32 {1  +  2}", &["1  +  2"]);
@@ -792,6 +801,19 @@ fn replace_type() {
             "struct Result<T, E> {} struct Option<T> {} fn f1() -> Option<Vec<Error>> {foo()}"
         ]],
     );
+    assert_ssr_transform(
+        "dyn Trait<$a> ==>> DynTrait<$a>",
+        r#"
+trait Trait<T> {}
+struct DynTrait<T> {}
+fn f1() -> dyn Trait<Vec<Error>> {foo()}
+"#,
+        expect![[r#"
+trait Trait<T> {}
+struct DynTrait<T> {}
+fn f1() -> DynTrait<Vec<Error>> {foo()}
+"#]],
+    );
 }
 
 #[test]