Update to tree-sitter 0.23 (#70)

Author: hendrikvanantwerpen

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "tree-sitter-elixir"
 description = "Elixir grammar for the tree-sitter parsing library"
-version = "0.2.0"
+version = "0.3.0"
 keywords = ["incremental", "parsing", "elixir"]
 categories = ["parsing", "text-editors"]
 repository = "https://github.com/elixir-lang/tree-sitter-elixir"
@@ -20,7 +20,10 @@ include = [
 path = "bindings/rust/lib.rs"
 
 [dependencies]
-tree-sitter = ">=0.21.0"
+tree-sitter-language = "0.1.0"
+
+[dev-dependencies]
+tree-sitter = "0.23.0"
 
 [build-dependencies]
 cc = "1.0"

Makefile

@@ -1,4 +1,4 @@
-VERSION := 0.2.0
+VERSION := 0.3.0
 
 # Repository
 SRC_DIR := src

binding.gyp

@@ -11,7 +11,7 @@
       "sources": [
         "bindings/node/binding.cc",
         "src/parser.c",
-        # NOTE: if your language has an external scanner, add it here.
+        "src/scanner.c",
       ],
       "cflags_c": [
         "-std=c11",

bindings/go/binding.go

@@ -2,7 +2,7 @@ package tree_sitter_elixir
 
 // #cgo CFLAGS: -std=c11 -fPIC
 // #include "../../src/parser.c"
-// // NOTE: if your language has an external scanner, add it here.
+// #include "../../src/scanner.c"
 import "C"
 
 import "unsafe"

bindings/go/binding_test.go

@@ -3,8 +3,8 @@ package tree_sitter_elixir_test
 import (
 	"testing"
 
-	tree_sitter "github.com/smacker/go-tree-sitter"
-	"github.com/tree-sitter/tree-sitter-elixir"
+	tree_sitter "github.com/tree-sitter/go-tree-sitter"
+	tree_sitter_elixir "github.com/tree-sitter/tree-sitter-elixir/bindings/go"
 )
 
 func TestCanLoadGrammar(t *testing.T) {

bindings/go/go.mod

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+
+const assert = require("node:assert");
+const { test } = require("node:test");
+
+test("can load grammar", () => {
+  const parser = new (require("tree-sitter"))();
+  assert.doesNotThrow(() => parser.setLanguage(require(".")));
+});

bindings/node/binding_test.js

@@ -0,0 +1,11 @@
+from unittest import TestCase
+
+import tree_sitter, tree_sitter_elixir
+
+
+class TestLanguage(TestCase):
+    def test_can_load_grammar(self):
+        try:
+            tree_sitter.Language(tree_sitter_elixir.language())
+        except Exception:
+            self.fail("Error loading Elixir grammar")

bindings/python/tests/test_binding.py

@@ -4,8 +4,8 @@ typedef struct TSLanguage TSLanguage;
 
 TSLanguage *tree_sitter_elixir(void);
 
-static PyObject* _binding_language(PyObject *self, PyObject *args) {
-    return PyLong_FromVoidPtr(tree_sitter_elixir());
+static PyObject* _binding_language(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) {
+    return PyCapsule_New(tree_sitter_elixir(), "tree_sitter.Language", NULL);
 }
 
 static PyMethodDef methods[] = {

bindings/python/tree_sitter_elixir/binding.c

@@ -7,6 +7,9 @@ fn main() {
         .flag_if_supported("-Wno-unused-parameter")
         .flag_if_supported("-Wno-unused-but-set-variable")
         .flag_if_supported("-Wno-trigraphs");
+    #[cfg(target_env = "msvc")]
+    c_config.flag("-utf-8");
+
     let parser_path = src_dir.join("parser.c");
     c_config.file(&parser_path);
     println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());

bindings/rust/build.rs

@@ -1,52 +1,53 @@
-//! This crate provides elixir language support for the [tree-sitter][] parsing library.
+//! This crate provides Elixir language support for the [tree-sitter][] parsing library.
 //!
 //! Typically, you will use the [language][language func] function to add this language to a
 //! tree-sitter [Parser][], and then use the parser to parse some code:
 //!
 //! ```
-//! let code = "";
+//! let code = r#"
+//! "#;
 //! let mut parser = tree_sitter::Parser::new();
-//! parser.set_language(tree_sitter_elixir::language()).expect("Error loading elixir grammar");
+//! let language = tree_sitter_elixir::LANGUAGE;
+//! parser
+//!     .set_language(&language.into())
+//!     .expect("Error loading Elixir parser");
 //! let tree = parser.parse(code, None).unwrap();
+//! assert!(!tree.root_node().has_error());
 //! ```
 //!
 //! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
 //! [language func]: fn.language.html
 //! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
 //! [tree-sitter]: https://tree-sitter.github.io/
 
-use tree_sitter::Language;
+use tree_sitter_language::LanguageFn;
 
 extern "C" {
-    fn tree_sitter_elixir() -> Language;
+    fn tree_sitter_elixir() -> *const ();
 }
 
-/// Get the tree-sitter [Language][] for this grammar.
-///
-/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
-pub fn language() -> Language {
-    unsafe { tree_sitter_elixir() }
-}
+/// The tree-sitter [`LanguageFn`] for this grammar.
+pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_elixir) };
 
 /// The content of the [`node-types.json`][] file for this grammar.
 ///
 /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
-pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
+pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");
 
-// Uncomment these to include any queries that this grammar contains
+// NOTE: uncomment these to include any queries that this grammar contains:
 
-pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
-// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
-// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
-pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
+pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
+pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
+// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
+pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
 
 #[cfg(test)]
 mod tests {
     #[test]
     fn test_can_load_grammar() {
         let mut parser = tree_sitter::Parser::new();
         parser
-            .set_language(super::language())
-            .expect("Error loading elixir language");
+            .set_language(&super::LANGUAGE.into())
+            .expect("Error loading Elixir parser");
     }
 }

bindings/rust/lib.rs

@@ -0,0 +1,12 @@
+import XCTest
+import SwiftTreeSitter
+import TreeSitterElixir
+
+final class TreeSitterElixirTests: XCTestCase {
+    func testCanLoadGrammar() throws {
+        let parser = Parser()
+        let language = Language(language: tree_sitter_elixir())
+        XCTAssertNoThrow(try parser.setLanguage(language),
+                         "Error loading Elixir grammar")
+    }
+}

bindings/swift/TreeSitterElixirTests/TreeSitterElixirTests.swift

@@ -0,0 +1,7 @@
+module github.com/tree-sitter/tree-sitter-elixir
+
+go 1.23
+
+require github.com/tree-sitter/go-tree-sitter v0.23.1
+
+require github.com/mattn/go-pointer v0.0.1 // indirect

go.mod

@@ -0,0 +1,36 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0=
+github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tree-sitter/go-tree-sitter v0.23.1 h1:HCfaE19sKfG7q190xfM1loUZf6wEHa4TDqDEW46s9Lg=
+github.com/tree-sitter/go-tree-sitter v0.23.1/go.mod h1:EvIVhMvvPNvhu9x+ddSPxSnUEU5AnsSwi1LMqXIVE3A=
+github.com/tree-sitter/tree-sitter-c v0.21.5-0.20240818205408-927da1f210eb h1:A8425heRM8mylnv4H58FPUiH+aYivyitre0PzxrfmWs=
+github.com/tree-sitter/tree-sitter-c v0.21.5-0.20240818205408-927da1f210eb/go.mod h1:dOF6gtQiF9UwNh995T5OphYmtIypkjsp3ap7r9AN/iA=
+github.com/tree-sitter/tree-sitter-cpp v0.22.4-0.20240818224355-b1a4e2b25148 h1:AfFPZwtwGN01BW1jDdqBVqscTwetvMpydqYZz57RSlc=
+github.com/tree-sitter/tree-sitter-cpp v0.22.4-0.20240818224355-b1a4e2b25148/go.mod h1:Bh6U3viD57rFXRYIQ+kmiYtr+1Bx0AceypDLJJSyi9s=
+github.com/tree-sitter/tree-sitter-embedded-template v0.21.1-0.20240819044651-ffbf64942c33 h1:TwqSV3qLp3tKSqirGLRHnjFk9Tc2oy57LIl+FQ4GjI4=
+github.com/tree-sitter/tree-sitter-embedded-template v0.21.1-0.20240819044651-ffbf64942c33/go.mod h1:CvCKCt3v04Ufos1zZnNCelBDeCGRpPucaN8QczoUsN4=
+github.com/tree-sitter/tree-sitter-go v0.21.3-0.20240818010209-8c0f0e7a6012 h1:Xvxck3tE5FW7F7bTS97iNM2ADMyCMJztVqn5HYKdJGo=
+github.com/tree-sitter/tree-sitter-go v0.21.3-0.20240818010209-8c0f0e7a6012/go.mod h1:T40D0O1cPvUU/+AmiXVXy1cncYQT6wem4Z0g4SfAYvY=
+github.com/tree-sitter/tree-sitter-html v0.20.5-0.20240818004741-d11201a263d0 h1:c46K6uh5Dz00zJeU9BfjXdb8I+E4RkUdfnWJpQADXFo=
+github.com/tree-sitter/tree-sitter-html v0.20.5-0.20240818004741-d11201a263d0/go.mod h1:hcNt/kOJHcIcuMvouE7LJcYdeFUFbVpBJ6d4wmOA+tU=
+github.com/tree-sitter/tree-sitter-java v0.21.1-0.20240824015150-576d8097e495 h1:jrt4qbJVEFs4H93/ITxygHc6u0TGqAkkate7TQ4wFSA=
+github.com/tree-sitter/tree-sitter-java v0.21.1-0.20240824015150-576d8097e495/go.mod h1:oyaR7fLnRV0hT9z6qwE9GkaeTom/hTDwK3H2idcOJFc=
+github.com/tree-sitter/tree-sitter-javascript v0.21.5-0.20240818005344-15887341e5b5 h1:om4X9AVg3asL8gxNJDcz4e/Wp+VpQj1PY3uJXKr6EOg=
+github.com/tree-sitter/tree-sitter-javascript v0.21.5-0.20240818005344-15887341e5b5/go.mod h1:nNqgPoV/h9uYWk6kYEFdEAhNVOacpfpRW5SFmdaP4tU=
+github.com/tree-sitter/tree-sitter-json v0.21.1-0.20240818005659-bdd69eb8c8a5 h1:pfV3G3k7NCKqKk8THBmyuh2zA33lgYHS3GVrzRR8ry4=
+github.com/tree-sitter/tree-sitter-json v0.21.1-0.20240818005659-bdd69eb8c8a5/go.mod h1:GbMKRjLfk0H+PI7nLi1Sx5lHf5wCpLz9al8tQYSxpEk=
+github.com/tree-sitter/tree-sitter-php v0.22.9-0.20240819002312-a552625b56c1 h1:ZXZMDwE+IhUtGug4Brv6NjJWUU3rfkZBKpemf6RY8/g=
+github.com/tree-sitter/tree-sitter-php v0.22.9-0.20240819002312-a552625b56c1/go.mod h1:UKCLuYnJ312Mei+3cyTmGOHzn0YAnaPRECgJmHtzrqs=
+github.com/tree-sitter/tree-sitter-python v0.21.1-0.20240818005537-55a9b8a4fbfb h1:EXEM82lFM7JjJb6qiKZXkpIDaCcbV2obNn82ghwj9lw=
+github.com/tree-sitter/tree-sitter-python v0.21.1-0.20240818005537-55a9b8a4fbfb/go.mod h1:lXCF1nGG5Dr4J3BTS0ObN4xJCCICiSu/b+Xe/VqMV7g=
+github.com/tree-sitter/tree-sitter-ruby v0.21.1-0.20240818211811-7dbc1e2d0e2d h1:fcYCvoXdcP1uRQYXqJHRy6Hec+uKScQdKVtMwK9JeCI=
+github.com/tree-sitter/tree-sitter-ruby v0.21.1-0.20240818211811-7dbc1e2d0e2d/go.mod h1:T1nShQ4v5AJtozZ8YyAS4uzUtDAJj/iv4YfwXSbUHzg=
+github.com/tree-sitter/tree-sitter-rust v0.21.3-0.20240818005432-2b43eafe6447 h1:o9alBu1J/WjrcTKEthYtXmdkDc5OVXD+PqlvnEZ0Lzc=
+github.com/tree-sitter/tree-sitter-rust v0.21.3-0.20240818005432-2b43eafe6447/go.mod h1:1Oh95COkkTn6Ezp0vcMbvfhRP5gLeqqljR0BYnBzWvc=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

go.sum

@@ -1,10 +1,15 @@
 {
   "name": "tree-sitter-elixir",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Elixir grammar for the tree-sitter parsing library",
   "main": "bindings/node",
   "types": "bindings/node",
-  "keywords": ["parser", "lexer", "elixir", "tree-sitter"],
+  "keywords": [
+    "parser",
+    "lexer",
+    "elixir",
+    "tree-sitter"
+  ],
   "files": [
     "grammar.js",
     "binding.gyp",
@@ -19,7 +24,8 @@
     "url": "https://github.com/elixir-lang/tree-sitter-elixir.git"
   },
   "scripts": {
-    "test": "tree-sitter test",
+    "build": "npx tree-sitter-cli generate --no-bindings",
+    "test": "npx tree-sitter-cli test",
     "format": "prettier --trailing-comma es5 --write grammar.js && clang-format -i src/scanner.c",
     "format-check": "prettier --trailing-comma es5 --check grammar.js && cat src/scanner.c | clang-format src/scanner.c | diff src/scanner.c -",
     "install": "node-gyp-build",
@@ -32,7 +38,7 @@
   "devDependencies": {
     "clang-format": "^1.8.0",
     "prettier": "^2.3.2",
-    "tree-sitter-cli": "^0.22.2",
+    "tree-sitter-cli": "^0.23.0",
     "prebuildify": "^6.0.0"
   },
   "peerDependencies": {

package-lock.json

@@ -22,7 +22,7 @@ readme = "README.md"
 Homepage = "https://github.com/tree-sitter/tree-sitter-elixir"
 
 [project.optional-dependencies]
-core = ["tree-sitter~=0.21"]
+core = ["tree-sitter~=0.23"]
 
 [tool.cibuildwheel]
 build = "cp38-*"

package.json

@@ -36,11 +36,14 @@ def get_tag(self):
             sources=[
                 "bindings/python/tree_sitter_elixir/binding.c",
                 "src/parser.c",
-                # NOTE: if your language uses an external scanner, add it here.
+                "src/scanner.c",
+            ],
+            extra_compile_args=[
+                "-std=c11",
+            ] if system() != "Windows" else [
+                "/std:c11",
+                "/utf-8",
             ],
-            extra_compile_args=(
-                ["-std=c11"] if system() != 'Windows' else []
-            ),
             define_macros=[
                 ("Py_LIMITED_API", "0x03080000"),
                 ("PY_SSIZE_T_CLEAN", None)

pyproject.toml

@@ -1,4 +1,4 @@
-#include <tree_sitter/parser.h>
+#include "tree_sitter/parser.h"
 
 // See references in grammar.externals
 enum TokenType {