Merge pull request dbt-labs#3497 from fishtown-analytics/experimental-parser-rust

Experimental Parser: Swap python extractor for rust dependency

Author: Nathaniel May

CHANGELOG.md

@@ -25,6 +25,7 @@ Contributors:
 
 ### Under the hood
 
+- Swap experimental parser implementation to use Rust [#3497](https://github.com/fishtown-analytics/dbt/pull/3497)
 - Dispatch the core SQL statement of the new test materialization, to benefit adapter maintainers ([#3465](https://github.com/fishtown-analytics/dbt/pull/3465), [#3461](https://github.com/fishtown-analytics/dbt/pull/3461))
 - Minimal validation of yaml dictionaries prior to partial parsing ([#3246](https://github.com/fishtown-analytics/dbt/issues/3246), [#3460](https://github.com/fishtown-analytics/dbt/pull/3460))
 - Add partial parsing tests and improve partial parsing handling of macros ([#3449](https://github.com/fishtown-analytics/dbt/issues/3449), [#3505](https://github.com/fishtown-analytics/dbt/pull/3505))

core/dbt/parser/models.py

@@ -4,7 +4,7 @@
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleSQLParser
 from dbt.parser.search import FileBlock
-from dbt.tree_sitter_jinja.extractor import extract_from_source
+from dbt_extractor import ExtractionError, py_extract_from_source  # type: ignore
 
 
 class ModelParser(SimpleSQLParser[ParsedModelNode]):
@@ -32,15 +32,14 @@ def render_update(
 
         # if the --use-experimental-parser flag was set
         else:
-
-            # run dbt-jinja extractor (powered by tree-sitter)
-            res = extract_from_source(node.raw_sql)
-
-            # if it doesn't need python jinja, fit the refs, sources, and configs
-            # into the node. Down the line the rest of the node will be updated with
-            # this information. (e.g. depends_on etc.)
-            if not res['python_jinja']:
-
+            try:
+                # run dbt jinja extractor (powered by tree-sitter + rust)
+                # throws an exception if it can't parse the source
+                res = py_extract_from_source(node.raw_sql)
+
+                # since it doesn't need python jinja, fit the refs, sources, and configs
+                # into the node. Down the line the rest of the node will be updated with
+                # this information. (e.g. depends_on etc.)
                 config_calls = []
                 for c in res['configs']:
                     config_calls.append({c[0]: c[1]})
@@ -66,5 +65,7 @@ def render_update(
 
                 self.manifest._parsing_info.static_analysis_parsed_path_count += 1
 
-            else:
+            # exception was thrown by dbt jinja extractor meaning it can't
+            # handle this source. fall back to python jinja rendering.
+            except ExtractionError:
                 super().render_update(node, config)

core/dbt/tree_sitter_jinja/README.md

@@ -63,8 +63,7 @@ def read(fname):
         'networkx>=2.3,<3',
         'packaging~=20.9',
         'sqlparse>=0.2.3,<0.4',
-        'tree-sitter==0.19.0',
-        'tree-sitter-jinja2==0.1.0a1',
+        'dbt-extractor==0.2.0',
         'typing-extensions>=3.7.4,<3.11',
         'werkzeug>=0.15,<3.0',
         # the following are all to match snowflake-connector-python