Support parser backtracking in the GreenNodeBuilder, part 1

Copy GreenNodeBuilder from rowan with the minimal changes to make it
work as part of our crate.

Author: nickolay

src/builder.rs

@@ -0,0 +1,215 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Concrete syntax (green) tree builder
+//!
+//! Based on the code from rowan:
+//! https://github.com/rust-analyzer/rowan/blob/v0.10.0/src/green/builder.rs
+//!
+//! The deviations are marked with `CHANGED(sqlparser)`.
+
+// CHANGED(sqlparser): parts of the imported code may be unused
+#![allow(dead_code)]
+
+use rowan::{GreenNode, GreenToken, NodeOrToken, SmolStr, SyntaxKind};
+
+// CHANGED(sqlparser): Use HashSet from std instead of FxHashSet to avoid the
+// extra dependency
+use std::collections::HashSet;
+// CHANGED(sqlparser): Redefine `GreenElement`, as it's not public in rowan
+pub type GreenElement = NodeOrToken<GreenNode, GreenToken>;
+
+#[derive(Default, Debug)]
+pub struct NodeCache {
+    nodes: HashSet<GreenNode>,
+    tokens: HashSet<GreenToken>,
+}
+
+impl NodeCache {
+    fn node<I>(&mut self, kind: SyntaxKind, children: I) -> GreenNode
+    where
+        I: IntoIterator<Item = GreenElement>,
+        I::IntoIter: ExactSizeIterator,
+    {
+        let mut node = GreenNode::new(kind, children);
+        // Green nodes are fully immutable, so it's ok to deduplicate them.
+        // This is the same optimization that Roslyn does
+        // https://github.com/KirillOsenkov/Bliki/wiki/Roslyn-Immutable-Trees
+        //
+        // For example, all `#[inline]` in this file share the same green node!
+        // For `libsyntax/parse/parser.rs`, measurements show that deduping saves
+        // 17% of the memory for green nodes!
+        // Future work: make hashing faster by avoiding rehashing of subtrees.
+        if node.children().len() <= 3 {
+            match self.nodes.get(&node) {
+                Some(existing) => node = existing.clone(),
+                None => assert!(self.nodes.insert(node.clone())),
+            }
+        }
+        node
+    }
+
+    fn token(&mut self, kind: SyntaxKind, text: SmolStr) -> GreenToken {
+        let mut token = GreenToken::new(kind, text);
+        match self.tokens.get(&token) {
+            Some(existing) => token = existing.clone(),
+            None => assert!(self.tokens.insert(token.clone())),
+        }
+        token
+    }
+}
+
+#[derive(Debug)]
+enum MaybeOwned<'a, T> {
+    Owned(T),
+    Borrowed(&'a mut T),
+}
+
+impl<T> std::ops::Deref for MaybeOwned<'_, T> {
+    type Target = T;
+    fn deref(&self) -> &T {
+        match self {
+            MaybeOwned::Owned(it) => it,
+            MaybeOwned::Borrowed(it) => *it,
+        }
+    }
+}
+
+impl<T> std::ops::DerefMut for MaybeOwned<'_, T> {
+    fn deref_mut(&mut self) -> &mut T {
+        match self {
+            MaybeOwned::Owned(it) => it,
+            MaybeOwned::Borrowed(it) => *it,
+        }
+    }
+}
+
+impl<T: Default> Default for MaybeOwned<'_, T> {
+    fn default() -> Self {
+        MaybeOwned::Owned(T::default())
+    }
+}
+
+/// A checkpoint for maybe wrapping a node. See `GreenNodeBuilder::checkpoint` for details.
+#[derive(Clone, Copy, Debug)]
+pub struct Checkpoint(usize);
+
+/// A builder for a green tree.
+#[derive(Default, Debug)]
+pub struct GreenNodeBuilder<'cache> {
+    cache: MaybeOwned<'cache, NodeCache>,
+    parents: Vec<(SyntaxKind, usize)>,
+    children: Vec<GreenElement>,
+}
+
+impl GreenNodeBuilder<'_> {
+    /// Creates new builder.
+    pub fn new() -> GreenNodeBuilder<'static> {
+        GreenNodeBuilder::default()
+    }
+
+    /// Reusing `NodeCache` between different `GreenNodeBuilder`s saves memory.
+    /// It allows to structurally share underlying trees.
+    pub fn with_cache(cache: &mut NodeCache) -> GreenNodeBuilder<'_> {
+        GreenNodeBuilder {
+            cache: MaybeOwned::Borrowed(cache),
+            parents: Vec::new(),
+            children: Vec::new(),
+        }
+    }
+
+    /// Adds new token to the current branch.
+    #[inline]
+    pub fn token(&mut self, kind: SyntaxKind, text: SmolStr) {
+        let token = self.cache.token(kind, text);
+        self.children.push(token.into());
+    }
+
+    /// Start new node and make it current.
+    #[inline]
+    pub fn start_node(&mut self, kind: SyntaxKind) {
+        let len = self.children.len();
+        self.parents.push((kind, len));
+    }
+
+    /// Finish current branch and restore previous
+    /// branch as current.
+    #[inline]
+    pub fn finish_node(&mut self) {
+        let (kind, first_child) = self.parents.pop().unwrap();
+        let children = self.children.drain(first_child..);
+        let node = self.cache.node(kind, children);
+        self.children.push(node.into());
+    }
+
+    /// Prepare for maybe wrapping the next node.
+    /// The way wrapping works is that you first of all get a checkpoint,
+    /// then you place all tokens you want to wrap, and then *maybe* call
+    /// `start_node_at`.
+    /// Example:
+    /// ```rust
+    /// # use rowan::{GreenNodeBuilder, SyntaxKind};
+    /// # const PLUS: SyntaxKind = SyntaxKind(0);
+    /// # const OPERATION: SyntaxKind = SyntaxKind(1);
+    /// # struct Parser;
+    /// # impl Parser {
+    /// #     fn peek(&self) -> Option<SyntaxKind> { None }
+    /// #     fn parse_expr(&mut self) {}
+    /// # }
+    /// # let mut builder = GreenNodeBuilder::new();
+    /// # let mut parser = Parser;
+    /// let checkpoint = builder.checkpoint();
+    /// parser.parse_expr();
+    /// if parser.peek() == Some(PLUS) {
+    ///   // 1 + 2 = Add(1, 2)
+    ///   builder.start_node_at(checkpoint, OPERATION);
+    ///   parser.parse_expr();
+    ///   builder.finish_node();
+    /// }
+    /// ```
+    #[inline]
+    pub fn checkpoint(&self) -> Checkpoint {
+        Checkpoint(self.children.len())
+    }
+
+    /// Wrap the previous branch marked by `checkpoint` in a new branch and
+    /// make it current.
+    #[inline]
+    pub fn start_node_at(&mut self, checkpoint: Checkpoint, kind: SyntaxKind) {
+        let Checkpoint(checkpoint) = checkpoint;
+        assert!(
+            checkpoint <= self.children.len(),
+            "checkpoint no longer valid, was finish_node called early?"
+        );
+
+        if let Some(&(_, first_child)) = self.parents.last() {
+            assert!(
+                checkpoint >= first_child,
+                "checkpoint no longer valid, was an unmatched start_node_at called?"
+            );
+        }
+
+        self.parents.push((kind, checkpoint));
+    }
+
+    /// Complete tree building. Make sure that
+    /// `start_node_at` and `finish_node` calls
+    /// are paired!
+    #[inline]
+    pub fn finish(mut self) -> GreenNode {
+        assert_eq!(self.children.len(), 1);
+        match self.children.pop().unwrap() {
+            NodeOrToken::Node(node) => node,
+            NodeOrToken::Token(_) => panic!(),
+        }
+    }
+}

src/lib.rs

@@ -35,6 +35,7 @@
 #![warn(clippy::all)]
 
 pub mod ast;
+mod builder;
 pub mod cst;
 pub mod dialect;
 pub mod parser;

src/parser.rs

@@ -21,6 +21,7 @@ use super::tokenizer::*;
 use std::error::Error;
 use std::fmt;
 
+use crate::builder;
 use crate::{cst, cst::SyntaxKind as SK};
 
 #[derive(Debug, Clone, PartialEq)]
@@ -40,7 +41,7 @@ macro_rules! parser_err {
 pub struct Marker {
     /// position in the token stream (`parser.index`)
     index: usize,
-    builder_checkpoint: rowan::Checkpoint,
+    builder_checkpoint: builder::Checkpoint,
 }
 
 #[derive(PartialEq)]
@@ -82,7 +83,7 @@ pub struct Parser {
     tokens: Vec<Token>,
     /// The index of the first unprocessed token in `self.tokens`
     index: usize,
-    builder: rowan::GreenNodeBuilder<'static>,
+    builder: builder::GreenNodeBuilder<'static>,
 
     // TBD: the parser currently provides an API to move around the token
     // stream without restrictions (`next_token`/`prev_token`), while the
@@ -114,7 +115,7 @@ impl Parser {
         let mut parser = Parser {
             tokens,
             index: 0,
-            builder: rowan::GreenNodeBuilder::new(),
+            builder: builder::GreenNodeBuilder::new(),
             pending: vec![],
         };
         parser.builder.start_node(SK::ROOT.into());