AST improvements to support conversion to rustc AST

crates/ra_assists/src/handlers/add_impl.rs

@@ -1,5 +1,5 @@
 use ra_syntax::{
-    ast::{self, AstNode, NameOwner, TypeParamsOwner},
+    ast::{self, AstNode, AstToken, NameOwner, TypeParamsOwner},
     TextUnit,
 };
 use stdx::{format_to, SepBy};
@@ -42,7 +42,7 @@ pub(crate) fn add_impl(ctx: AssistCtx) -> Option<Assist> {
         if let Some(type_params) = type_params {
             let lifetime_params = type_params
                 .lifetime_params()
-                .filter_map(|it| it.lifetime_token())
+                .filter_map(|it| it.lifetime())
                 .map(|it| it.text().clone());
             let type_params =
                 type_params.type_params().filter_map(|it| it.name()).map(|it| it.text().clone());

crates/ra_assists/src/handlers/add_new.rs

@@ -1,7 +1,8 @@
 use hir::Adt;
 use ra_syntax::{
     ast::{
-        self, AstNode, NameOwner, StructKind, TypeAscriptionOwner, TypeParamsOwner, VisibilityOwner,
+        self, AstNode, AstToken, NameOwner, StructKind, TypeAscriptionOwner, TypeParamsOwner,
+        VisibilityOwner,
     },
     TextUnit, T,
 };
@@ -105,7 +106,7 @@ fn generate_impl_text(strukt: &ast::StructDef, code: &str) -> String {
     if let Some(type_params) = type_params {
         let lifetime_params = type_params
             .lifetime_params()
-            .filter_map(|it| it.lifetime_token())
+            .filter_map(|it| it.lifetime())
             .map(|it| it.text().clone());
         let type_params =
             type_params.type_params().filter_map(|it| it.name()).map(|it| it.text().clone());

crates/ra_assists/src/handlers/introduce_variable.rs

@@ -1,5 +1,5 @@
 use ra_syntax::{
-    ast::{self, AstNode},
+    ast::{self, AstElement, AstNode},
     SyntaxKind::{
         BLOCK_EXPR, BREAK_EXPR, COMMENT, LAMBDA_EXPR, LOOP_EXPR, MATCH_ARM, PATH_EXPR, RETURN_EXPR,
         WHITESPACE,
@@ -124,7 +124,7 @@ fn anchor_stmt(expr: ast::Expr) -> Option<(SyntaxNode, bool)> {
             }
         }
 
-        if ast::Stmt::cast(node.clone()).is_some() {
+        if ast::Stmt::cast_element(node.clone().into()).is_some() {
             return Some((node, false));
         }
 

crates/ra_assists/src/handlers/merge_imports.rs

@@ -3,7 +3,7 @@ use std::iter::successors;
 use ra_syntax::{
     algo::{neighbor, SyntaxRewriter},
     ast::{self, edit::AstNodeEdit, make},
-    AstNode, Direction, InsertPosition, SyntaxElement, T,
+    AstNode, AstToken, Direction, InsertPosition, SyntaxElement, T,
 };
 
 use crate::{Assist, AssistCtx, AssistId};
@@ -82,7 +82,7 @@ fn try_merge_trees(old: &ast::UseTree, new: &ast::UseTree) -> Option<ast::UseTre
             .filter(|it| it.kind() != T!['{'] && it.kind() != T!['}']),
     );
     let use_tree_list = lhs.use_tree_list()?;
-    let pos = InsertPosition::Before(use_tree_list.r_curly()?.into());
+    let pos = InsertPosition::Before(use_tree_list.r_curly()?.syntax().clone().into());
     let use_tree_list = use_tree_list.insert_children(pos, to_insert);
     Some(lhs.with_use_tree_list(use_tree_list))
 }

crates/ra_fmt/src/lib.rs

@@ -60,10 +60,10 @@ pub fn extract_trivial_expression(block: &ast::BlockExpr) -> Option<ast::Expr> {
     } else {
         // Unwrap `{ continue; }`
         let (stmt,) = block.statements().next_tuple()?;
-        if has_anything_else(stmt.syntax()) {
-            return None;
-        }
         if let ast::Stmt::ExprStmt(expr_stmt) = stmt {
+            if has_anything_else(expr_stmt.syntax()) {
+                return None;
+            }
             let expr = expr_stmt.expr()?;
             match expr.syntax().kind() {
                 CONTINUE_EXPR | BREAK_EXPR | RETURN_EXPR => return Some(expr),

crates/ra_hir_def/src/body/lower.rs

@@ -501,14 +501,17 @@ impl ExprCollector<'_> {
         self.collect_block_items(&block);
         let statements = block
             .statements()
-            .map(|s| match s {
+            .filter_map(|s| match s {
                 ast::Stmt::LetStmt(stmt) => {
                     let pat = self.collect_pat_opt(stmt.pat());
                     let type_ref = stmt.ascribed_type().map(TypeRef::from_ast);
                     let initializer = stmt.initializer().map(|e| self.collect_expr(e));
-                    Statement::Let { pat, type_ref, initializer }
+                    Some(Statement::Let { pat, type_ref, initializer })
                 }
-                ast::Stmt::ExprStmt(stmt) => Statement::Expr(self.collect_expr_opt(stmt.expr())),
+                ast::Stmt::ExprStmt(stmt) => {
+                    Some(Statement::Expr(self.collect_expr_opt(stmt.expr())))
+                }
+                ast::Stmt::ModuleItem(_) => None,
             })
             .collect();
         let tail = block.expr().map(|e| self.collect_expr(e));
@@ -560,6 +563,7 @@ impl ExprCollector<'_> {
                     let ast_id = self.expander.ast_id(&def);
                     (TraitLoc { container, ast_id }.intern(self.db).into(), def.name())
                 }
+                ast::ModuleItem::ExternBlock(_) => continue, // FIXME: collect from extern blocks
                 ast::ModuleItem::ImplDef(_)
                 | ast::ModuleItem::UseItem(_)
                 | ast::ModuleItem::ExternCrateItem(_)
@@ -684,6 +688,7 @@ impl ExprCollector<'_> {
             ast::Pat::BoxPat(_) => Pat::Missing,
             ast::Pat::LiteralPat(_) => Pat::Missing,
             ast::Pat::RangePat(_) => Pat::Missing,
+            ast::Pat::MacroCall(_) => Pat::Missing,
         };
         let ptr = AstPtr::new(&pat);
         self.alloc_pat(pattern, Either::Left(ptr))

crates/ra_hir_def/src/nameres/raw.rs

@@ -266,6 +266,10 @@ impl RawItemsCollector {
                 self.add_macro(current_module, it);
                 return;
             }
+            ast::ModuleItem::ExternBlock(_) => {
+                // FIXME: add extern block
+                return;
+            }
         };
         if let Some(name) = name {
             let name = name.as_name();

crates/ra_hir_def/src/path/lower.rs

@@ -28,7 +28,7 @@ pub(super) fn lower_path(mut path: ast::Path, hygiene: &Hygiene) -> Option<Path>
     loop {
         let segment = path.segment()?;
 
-        if segment.has_colon_colon() {
+        if segment.coloncolon().is_some() {
             kind = PathKind::Abs;
         }
 

crates/ra_hir_def/src/path/lower/lower_use.rs

@@ -34,7 +34,7 @@ pub(crate) fn lower_use_tree(
         let alias = tree.alias().map(|a| {
             a.name().map(|it| it.as_name()).map_or(ImportAlias::Underscore, ImportAlias::Alias)
         });
-        let is_glob = tree.has_star();
+        let is_glob = tree.star().is_some();
         if let Some(ast_path) = tree.path() {
             // Handle self in a path.
             // E.g. `use something::{self, <...>}`

crates/ra_hir_def/src/visibility.rs

@@ -84,6 +84,10 @@ impl RawVisibility {
                 let path = ModPath { kind: PathKind::Super(1), segments: Vec::new() };
                 RawVisibility::Module(path)
             }
+            ast::VisibilityKind::PubSelf => {
+                let path = ModPath { kind: PathKind::Plain, segments: Vec::new() };
+                RawVisibility::Module(path)
+            }
             ast::VisibilityKind::Pub => RawVisibility::Public,
         }
     }

crates/ra_hir_ty/src/tests.rs

@@ -23,7 +23,7 @@ use insta::assert_snapshot;
 use ra_db::{fixture::WithFixture, salsa::Database, FilePosition, SourceDatabase};
 use ra_syntax::{
     algo,
-    ast::{self, AstNode},
+    ast::{self, AstNode, AstToken},
 };
 use stdx::format_to;
 
@@ -101,7 +101,7 @@ fn infer_with_mismatches(content: &str, include_mismatches: bool) -> String {
             let node = src_ptr.value.to_node(&src_ptr.file_syntax(&db));
 
             let (range, text) = if let Some(self_param) = ast::SelfParam::cast(node.clone()) {
-                (self_param.self_kw_token().text_range(), "self".to_string())
+                (self_param.self_kw().unwrap().syntax().text_range(), "self".to_string())
             } else {
                 (src_ptr.value.range(), node.text().to_string().replace("\n", " "))
             };

crates/ra_parser/src/grammar.rs

@@ -245,7 +245,7 @@ fn opt_fn_ret_type(p: &mut Parser) -> bool {
     if p.at(T![->]) {
         let m = p.start();
         p.bump(T![->]);
-        types::type_(p);
+        types::type_no_bounds(p);
         m.complete(p, RET_TYPE);
         true
     } else {

crates/ra_parser/src/lib.rs

@@ -12,6 +12,8 @@
 //!
 //! Tests for this crate live in `ra_syntax` crate.
 
+#![allow(elided_lifetimes_in_paths)]
+
 #[macro_use]
 mod token_set;
 #[macro_use]

crates/ra_parser/src/syntax_kind/generated.rs

@@ -105,6 +105,7 @@ pub enum SyntaxKind {
     DEFAULT_KW,
     EXISTENTIAL_KW,
     UNION_KW,
+    RAW_KW,
     INT_NUMBER,
     FLOAT_NUMBER,
     CHAR,
@@ -257,7 +258,7 @@ impl SyntaxKind {
             | IMPL_KW | IN_KW | LET_KW | LOOP_KW | MACRO_KW | MATCH_KW | MOD_KW | MOVE_KW
             | MUT_KW | PUB_KW | REF_KW | RETURN_KW | SELF_KW | STATIC_KW | STRUCT_KW | SUPER_KW
             | TRAIT_KW | TRUE_KW | TRY_KW | TYPE_KW | UNSAFE_KW | USE_KW | WHERE_KW | WHILE_KW
-            | AUTO_KW | DEFAULT_KW | EXISTENTIAL_KW | UNION_KW => true,
+            | AUTO_KW | DEFAULT_KW | EXISTENTIAL_KW | UNION_KW | RAW_KW => true,
             _ => false,
         }
     }
@@ -650,4 +651,7 @@ macro_rules! T {
     ( union ) => {
         $crate::SyntaxKind::UNION_KW
     };
+    ( raw ) => {
+        $crate::SyntaxKind::RAW_KW
+    };
 }

crates/ra_syntax/src/algo.rs

@@ -316,7 +316,7 @@ impl<'a> SyntaxRewriter<'a> {
     }
 }
 
-impl<'a> ops::AddAssign for SyntaxRewriter<'_> {
+impl ops::AddAssign for SyntaxRewriter<'_> {
     fn add_assign(&mut self, rhs: SyntaxRewriter) {
         assert!(rhs.f.is_none());
         self.replacements.extend(rhs.replacements)

crates/ra_syntax/src/ast.rs

@@ -11,7 +11,10 @@ pub mod make;
 use std::marker::PhantomData;
 
 use crate::{
-    syntax_node::{SyntaxNode, SyntaxNodeChildren, SyntaxToken},
+    syntax_node::{
+        NodeOrToken, SyntaxElement, SyntaxElementChildren, SyntaxNode, SyntaxNodeChildren,
+        SyntaxToken,
+    },
     SmolStr, SyntaxKind,
 };
 
@@ -30,16 +33,24 @@ pub use self::{
 /// conversion itself has zero runtime cost: ast and syntax nodes have exactly
 /// the same representation: a pointer to the tree root and a pointer to the
 /// node itself.
-pub trait AstNode: std::fmt::Display {
+pub trait AstNode: AstElement {
     fn can_cast(kind: SyntaxKind) -> bool
     where
         Self: Sized;
 
-    fn cast(syntax: SyntaxNode) -> Option<Self>
+    fn cast_or_return(syntax: SyntaxNode) -> Result<Self, SyntaxNode>
     where
         Self: Sized;
 
+    fn cast(syntax: SyntaxNode) -> Option<Self>
+    where
+        Self: Sized,
+    {
+        <Self as AstNode>::cast_or_return(syntax).ok()
+    }
+
     fn syntax(&self) -> &SyntaxNode;
+    fn into_syntax(self) -> SyntaxNode;
 }
 
 #[test]
@@ -48,16 +59,51 @@ fn assert_ast_is_object_safe() {
 }
 
 /// Like `AstNode`, but wraps tokens rather than interior nodes.
-pub trait AstToken {
-    fn cast(token: SyntaxToken) -> Option<Self>
+pub trait AstToken: AstElement {
+    fn can_cast(token: SyntaxKind) -> bool
     where
         Self: Sized;
+
+    fn cast_or_return(syntax: SyntaxToken) -> Result<Self, SyntaxToken>
+    where
+        Self: Sized;
+
+    fn cast(syntax: SyntaxToken) -> Option<Self>
+    where
+        Self: Sized,
+    {
+        <Self as AstToken>::cast_or_return(syntax).ok()
+    }
+
     fn syntax(&self) -> &SyntaxToken;
+    fn into_syntax(self) -> SyntaxToken;
+
     fn text(&self) -> &SmolStr {
         self.syntax().text()
     }
 }
 
+/// Like `AstNode`, but wraps either nodes or tokens rather than interior nodes.
+pub trait AstElement: std::fmt::Display {
+    fn can_cast_element(kind: SyntaxKind) -> bool
+    where
+        Self: Sized;
+
+    fn cast_or_return_element(syntax: SyntaxElement) -> Result<Self, SyntaxElement>
+    where
+        Self: Sized;
+
+    fn cast_element(syntax: SyntaxElement) -> Option<Self>
+    where
+        Self: Sized,
+    {
+        <Self as AstElement>::cast_or_return_element(syntax).ok()
+    }
+
+    fn syntax_element(&self) -> NodeOrToken<&SyntaxNode, &SyntaxToken>;
+    fn into_syntax_element(self) -> SyntaxElement;
+}
+
 /// An iterator over `SyntaxNode` children of a particular AST type.
 #[derive(Debug, Clone)]
 pub struct AstChildren<N> {
@@ -86,6 +132,64 @@ fn children<P: AstNode + ?Sized, C: AstNode>(parent: &P) -> AstChildren<C> {
     AstChildren::new(parent.syntax())
 }
 
+/// An iterator over `SyntaxToken` children of a particular AST type.
+#[derive(Debug, Clone)]
+pub struct AstChildTokens<N> {
+    inner: SyntaxElementChildren,
+    ph: PhantomData<N>,
+}
+
+impl<N> AstChildTokens<N> {
+    fn new(parent: &SyntaxNode) -> Self {
+        AstChildTokens { inner: parent.children_with_tokens(), ph: PhantomData }
+    }
+}
+
+impl<N: AstToken> Iterator for AstChildTokens<N> {
+    type Item = N;
+    fn next(&mut self) -> Option<N> {
+        self.inner.by_ref().filter_map(|x| x.into_token()).find_map(N::cast)
+    }
+}
+
+fn child_token_opt<P: AstNode + ?Sized, C: AstToken>(parent: &P) -> Option<C> {
+    child_tokens(parent).next()
+}
+
+fn child_tokens<P: AstNode + ?Sized, C: AstToken>(parent: &P) -> AstChildTokens<C> {
+    AstChildTokens::new(parent.syntax())
+}
+
+/// An iterator over `SyntaxNode` children of a particular AST type.
+#[derive(Debug, Clone)]
+pub struct AstChildElements<N> {
+    inner: SyntaxElementChildren,
+    ph: PhantomData<N>,
+}
+
+impl<N> AstChildElements<N> {
+    fn new(parent: &SyntaxNode) -> Self {
+        AstChildElements { inner: parent.children_with_tokens(), ph: PhantomData }
+    }
+}
+
+impl<N: AstElement> Iterator for AstChildElements<N> {
+    type Item = N;
+    fn next(&mut self) -> Option<N> {
+        self.inner.by_ref().find_map(N::cast_element)
+    }
+}
+
+#[allow(dead_code)]
+fn child_element_opt<P: AstNode + ?Sized, C: AstElement>(parent: &P) -> Option<C> {
+    child_elements(parent).next()
+}
+
+#[allow(dead_code)]
+fn child_elements<P: AstNode + ?Sized, C: AstElement>(parent: &P) -> AstChildElements<C> {
+    AstChildElements::new(parent.syntax())
+}
+
 #[test]
 fn test_doc_comment_none() {
     let file = SourceFile::parse(
@@ -265,7 +369,7 @@ where
     let pred = predicates.next().unwrap();
     let mut bounds = pred.type_bound_list().unwrap().bounds();
 
-    assert_eq!("'a", pred.lifetime_token().unwrap().text());
+    assert_eq!("'a", pred.lifetime().unwrap().text());
 
     assert_bound("'b", bounds.next());
     assert_bound("'c", bounds.next());