chore: Switch back to smacker/go-tree-sitter (#3069)
Finally remove the dougthor42/go-tree-sitter fork, fixing #2630.
Admittedly we could have done this sooner had I figured things
out sooner... but c'est la vie.
Instead of using the BUILD.bazel files in dougthor42/go-tree-sitter,
we basically vendor the build file via http_archive.
This is different than using patches because non-root Bazel modules
can still make use of the BUILD.bazel files we make.
Background:
The reason we migrated to dougthor42/go-tree-sitter in the first
place was to support python 3.12 grammar. smacker/go-tree-sitter
supported for python 3.12, but made a change to their file structure
that Gazelle was unable to handle. Specifically, the
python/binding.go file indirectly requires a c header file found
in a parent directory, and Gazelle doesn't know how to handle that
for `go_repository` (WORKSPACE) and `go_deps.from_file` (bzlmod).
So dougthor42/go-tree-sitter created our own BUILD.bazel files
that included the required filegroups and whatnot, thus negating
the need for Gazelle to generate BUILD.bazel files.
Future Work:
This still doesn't resolve the issues with bumping rules_go and go
seen in #2962, but it does simplify that investigation a bit as
it's just one fewer thing to account for. It also doesn't address
the desire to migrate to the official tree-sitter/go-tree-sitter
repo, but @jbedard found some perf issues with that anyway
(https://github.com/tree-sitter/go-tree-sitter/issues/32).
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c1d3a43..7f02c8b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -67,6 +67,8 @@
* 3.12.11
* 3.14.0b3
* (toolchain) Python 3.13 now references 3.13.5
+* (gazelle) Switched back to smacker/go-tree-sitter, fixing
+ [#2630](https://github.com/bazel-contrib/rules_python/issues/2630)
{#v0-0-0-fixed}
### Fixed
diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel
index 6bbc74b..51352a0 100644
--- a/gazelle/MODULE.bazel
+++ b/gazelle/MODULE.bazel
@@ -21,7 +21,6 @@
go_deps,
"com_github_bazelbuild_buildtools",
"com_github_bmatcuk_doublestar_v4",
- "com_github_dougthor42_go_tree_sitter",
"com_github_emirpasic_gods",
"com_github_ghodss_yaml",
"com_github_stretchr_testify",
@@ -29,6 +28,16 @@
"org_golang_x_sync",
)
+http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+ name = "com_github_smacker_go_tree_sitter",
+ build_file = "//:internal/smacker_BUILD.bazel",
+ integrity = "sha256-4AkDY4Rh5Auu9Kwzhj5XYSirMLlhmd6ClMWo/r0kmu4=",
+ strip_prefix = "go-tree-sitter-dd81d9e9be82a8cac96ed1d50c7389c5f1997c02",
+ url = "https://github.com/smacker/go-tree-sitter/archive/dd81d9e9be82a8cac96ed1d50c7389c5f1997c02.zip",
+)
+
python_stdlib_list = use_extension("//python:extensions.bzl", "python_stdlib_list")
use_repo(
python_stdlib_list,
diff --git a/gazelle/deps.bzl b/gazelle/deps.bzl
index 7253ef8..8c4c055 100644
--- a/gazelle/deps.bzl
+++ b/gazelle/deps.bzl
@@ -113,7 +113,6 @@
sum = "h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=",
version = "v1.1.1",
)
-
go_repository(
name = "com_github_emirpasic_gods",
importpath = "github.com/emirpasic/gods",
@@ -175,18 +174,18 @@
sum = "h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=",
version = "v1.0.0",
)
-
go_repository(
name = "com_github_prometheus_client_model",
importpath = "github.com/prometheus/client_model",
sum = "h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=",
version = "v0.0.0-20190812154241-14fe0d1b01d4",
)
- go_repository(
- name = "com_github_dougthor42_go_tree_sitter",
- importpath = "github.com/dougthor42/go-tree-sitter",
- sum = "h1:b9s96BulIARx0konX36sJ5oZhWvAvjQBBntxp1eUukQ=",
- version = "v0.0.0-20241210060307-2737e1d0de6b",
+ http_archive(
+ name = "com_github_smacker_go_tree_sitter",
+ build_file = Label("//:internal/smacker_BUILD.bazel"),
+ integrity = "sha256-4AkDY4Rh5Auu9Kwzhj5XYSirMLlhmd6ClMWo/r0kmu4=",
+ strip_prefix = "go-tree-sitter-dd81d9e9be82a8cac96ed1d50c7389c5f1997c02",
+ url = "https://github.com/smacker/go-tree-sitter/archive/dd81d9e9be82a8cac96ed1d50c7389c5f1997c02.zip",
)
go_repository(
name = "com_github_stretchr_objx",
diff --git a/gazelle/go.mod b/gazelle/go.mod
index 91d27fd..6f65ffb 100644
--- a/gazelle/go.mod
+++ b/gazelle/go.mod
@@ -7,9 +7,9 @@
github.com/bazelbuild/buildtools v0.0.0-20231103205921-433ea8554e82
github.com/bazelbuild/rules_go v0.41.0
github.com/bmatcuk/doublestar/v4 v4.7.1
- github.com/dougthor42/go-tree-sitter v0.0.0-20241210060307-2737e1d0de6b
github.com/emirpasic/gods v1.18.1
github.com/ghodss/yaml v1.0.0
+ github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82
github.com/stretchr/testify v1.9.0
golang.org/x/sync v0.2.0
gopkg.in/yaml.v2 v2.4.0
diff --git a/gazelle/go.sum b/gazelle/go.sum
index 5acd4a6..0aaa186 100644
--- a/gazelle/go.sum
+++ b/gazelle/go.sum
@@ -6,8 +6,6 @@
github.com/bazelbuild/buildtools v0.0.0-20231103205921-433ea8554e82/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo=
github.com/bazelbuild/rules_go v0.41.0 h1:JzlRxsFNhlX+g4drDRPhIaU5H5LnI978wdMJ0vK4I+k=
github.com/bazelbuild/rules_go v0.41.0/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
-github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
-github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bmatcuk/doublestar/v4 v4.7.1 h1:fdDeAqgT47acgwd9bd9HxJRDmc9UAmPpc+2m0CXv75Q=
github.com/bmatcuk/doublestar/v4 v4.7.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -17,8 +15,6 @@
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dougthor42/go-tree-sitter v0.0.0-20241210060307-2737e1d0de6b h1:b9s96BulIARx0konX36sJ5oZhWvAvjQBBntxp1eUukQ=
-github.com/dougthor42/go-tree-sitter v0.0.0-20241210060307-2737e1d0de6b/go.mod h1:87UkDyPt18bTH/FvinLc/kj587VNYOdRKZT1la4T8Hg=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -47,6 +43,8 @@
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4=
+github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82/go.mod h1:xe4pgH49k4SsmkQq5OT8abwhWmnzkhpgnXeekbx2efw=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0=
diff --git a/gazelle/internal/smacker_BUILD.bazel b/gazelle/internal/smacker_BUILD.bazel
new file mode 100644
index 0000000..3ec9676
--- /dev/null
+++ b/gazelle/internal/smacker_BUILD.bazel
@@ -0,0 +1,80 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+filegroup(
+ name = "common_libs",
+ srcs = [
+ "alloc.h",
+ "api.h",
+ "array.h",
+ ],
+ visibility = [":__subpackages__"],
+)
+
+go_library(
+ name = "go-tree-sitter",
+ srcs = [
+ "alloc.c",
+ "alloc.h",
+ "api.h",
+ "array.h",
+ "atomic.h",
+ "bindings.c",
+ "bindings.go",
+ "bindings.h",
+ "bits.h",
+ "clock.h",
+ "error_costs.h",
+ "get_changed_ranges.c",
+ "get_changed_ranges.h",
+ "host.h",
+ "iter.go",
+ "language.c",
+ "language.h",
+ "length.h",
+ "lexer.c",
+ "lexer.h",
+ "node.c",
+ "parser.c",
+ "parser.h",
+ "point.h",
+ "ptypes.h",
+ "query.c",
+ "reduce_action.h",
+ "reusable_node.h",
+ "stack.c",
+ "stack.h",
+ "subtree.c",
+ "subtree.h",
+ "test_grammar.go",
+ "tree.c",
+ "tree.h",
+ "tree_cursor.c",
+ "tree_cursor.h",
+ "umachine.h",
+ "unicode.h",
+ "urename.h",
+ "utf.h",
+ "utf16.h",
+ "utf8.h",
+ "wasm_store.c",
+ "wasm_store.h",
+ ],
+ cgo = True,
+ importpath = "github.com/smacker/go-tree-sitter",
+ visibility = ["//visibility:public"],
+)
+
+go_library(
+ name = "python",
+ srcs = [
+ "python/binding.go",
+ "python/parser.c",
+ "python/parser.h",
+ "python/scanner.c",
+ ":common_libs",
+ ],
+ cgo = True,
+ importpath = "github.com/smacker/go-tree-sitter/python",
+ visibility = ["//visibility:public"],
+ deps = [":go-tree-sitter"],
+)
diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel
index eb2d72e..8e8216d 100644
--- a/gazelle/python/BUILD.bazel
+++ b/gazelle/python/BUILD.bazel
@@ -39,11 +39,11 @@
"@bazel_gazelle//rule:go_default_library",
"@com_github_bazelbuild_buildtools//build:go_default_library",
"@com_github_bmatcuk_doublestar_v4//:doublestar",
- "@com_github_dougthor42_go_tree_sitter//:go-tree-sitter",
- "@com_github_dougthor42_go_tree_sitter//python",
"@com_github_emirpasic_gods//lists/singlylinkedlist",
"@com_github_emirpasic_gods//sets/treeset",
"@com_github_emirpasic_gods//utils",
+ "@com_github_smacker_go_tree_sitter//:go-tree-sitter",
+ "@com_github_smacker_go_tree_sitter//:python",
"@org_golang_x_sync//errgroup",
],
)
diff --git a/gazelle/python/file_parser.go b/gazelle/python/file_parser.go
index aca925c..31fce02 100644
--- a/gazelle/python/file_parser.go
+++ b/gazelle/python/file_parser.go
@@ -22,8 +22,8 @@
"path/filepath"
"strings"
- sitter "github.com/dougthor42/go-tree-sitter"
- "github.com/dougthor42/go-tree-sitter/python"
+ sitter "github.com/smacker/go-tree-sitter"
+ "github.com/smacker/go-tree-sitter/python"
)
const (
@@ -116,10 +116,6 @@
a, b = b, a
}
if a.Type() == sitterNodeTypeIdentifier && a.Content(p.code) == "__name__" &&
- // at github.com/dougthor42/go-tree-sitter@latest (after v0.0.0-20240422154435-0628b34cbf9c we used)
- // "__main__" is the second child of b. But now, it isn't.
- // we cannot use the latest go-tree-sitter because of the top level reference in scanner.c.
- // https://github.com/dougthor42/go-tree-sitter/blob/04d6b33fe138a98075210f5b770482ded024dc0f/python/scanner.c#L1
b.Type() == sitterNodeTypeString && string(p.code[b.StartByte()+1:b.EndByte()-1]) == "__main__" {
return true
}
