fix(gazelle): migrate to official tree-sitter Go bindings (#4272) Replace the unmaintained `github.com/smacker/go-tree-sitter` dependency with the official `github.com/tree-sitter/go-tree-sitter` and `github.com/tree-sitter/tree-sitter-python` (v0.25.0) modules. This updates the Python grammar so the Gazelle plugin can parse Python 3.14 PEP 758 unparenthesized multi-exception `except` clauses without errors. Because the official bindings include C sources across parent and subdirectory paths that standard `go_repository` Gazelle generation cannot compile directly, fetch both modules via `http_archive` with custom `cc_library` and `go_library` targets in both Bzlmod and WORKSPACE modes. We also remove unneeded go_repository items from `gazelle/deps.bzl`. Generated by Gemini. Tested on our large-scale QuantumAI monorepo. TAG=agy ### Timing I ran the parent commit and this PR on our primary QuantumAI monorepo. This PR might make things a little faster, but I didn't want to wait for 5 or more runs in hyperfine :laughing:. Parent commit: ``` $ git -C ../../oss/bazel-contrib/rules_python rev-parse HEAD b7ec5bd151df5f725ab8f76446fd548ac4fedce1 # agents: require user-visible docs prose and workaround comments (#4271) $ hyperfine --warmup=1 --runs=3 'bazel run --bes_backend= //:gazelle' Benchmark 1: bazel run --bes_backend= //:gazelle Time (mean ± σ): 93.052 s ± 18.023 s [User: 99.884 s, System: 6.302 s] Range (min … max): 82.134 s … 113.855 s 3 runs Warning: Statistical outliers were detected. Consider re-running this benchmark on a quiet system without any interferences from other programs. It might help to use the '--warmup' or '--prepare' options. ``` This PR: ``` $ git -C ../../oss/bazel-contrib/rules_python rev-parse HEAD bc5b12a5e160e7f91f665fc730b9b305dc3471b4 $ hyperfine --warmup=1 --runs=3 'bazel run --bes_backend= //:gazelle' Benchmark 1: bazel run --bes_backend= //:gazelle Time (mean ± σ): 80.338 s ± 2.570 s [User: 80.271 s, System: 3.808 s] Range (min … max): 78.292 s … 83.223 s 3 runs ``` Fixes #3416
diff --git a/gazelle/BUILD.bazel b/gazelle/BUILD.bazel index 3075818..7467648 100644 --- a/gazelle/BUILD.bazel +++ b/gazelle/BUILD.bazel
@@ -6,7 +6,9 @@ # gazelle:prefix github.com/bazel-contrib/rules_python/gazelle # gazelle:exclude bazel-out # gazelle:exclude deps.bzl +# gazelle:exclude internal # gazelle:exclude internal_dev_deps.bzl +# gazelle:resolve go github.com/tree-sitter/tree-sitter-python/bindings/go @com_github_tree_sitter_tree_sitter_python//:go_default_library gazelle( name = "gazelle", ) @@ -36,7 +38,7 @@ "//modules_mapping:distribution", "//python:distribution", "//pythonconfig:distribution", - ], + ] + glob(["internal/**"]), visibility = ["@rules_python//:__pkg__"], )
diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel index b6eed23..407be53 100644 --- a/gazelle/MODULE.bazel +++ b/gazelle/MODULE.bazel
@@ -23,12 +23,29 @@ "com_github_bmatcuk_doublestar_v4", "com_github_emirpasic_gods", "com_github_ghodss_yaml", - "com_github_smacker_go_tree_sitter", + "com_github_mattn_go_pointer", "com_github_stretchr_testify", "in_gopkg_yaml_v2", "org_golang_x_sync", ) +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "com_github_tree_sitter_go_tree_sitter", + build_file = "//:internal/go_tree_sitter_BUILD.bazel", + integrity = "sha256-rAsyh/Kydqt24TBaA6H14obQlkaQWwj1QNec3xza6NM=", + strip_prefix = "go-tree-sitter-c9492002f76ed75037e3fe6d3bbabb54ed3e1ff5", + url = "https://github.com/tree-sitter/go-tree-sitter/archive/c9492002f76ed75037e3fe6d3bbabb54ed3e1ff5.zip", +) + +http_archive( + name = "com_github_tree_sitter_tree_sitter_python", + build_file = "//:internal/tree_sitter_python_BUILD.bazel", + integrity = "sha256-e86IfrLzPpS/dKaWRc9RONQJZyDlT9MmmmEkwGuTxYQ=", + url = "https://github.com/tree-sitter/tree-sitter-python/releases/download/v0.25.0/tree-sitter-python.tar.gz", +) + python_stdlib_list = use_extension("//python:extensions.bzl", "python_stdlib_list") use_repo( python_stdlib_list,
diff --git a/gazelle/deps.bzl b/gazelle/deps.bzl index c57d519..a7868c6 100644 --- a/gazelle/deps.bzl +++ b/gazelle/deps.bzl
@@ -31,11 +31,49 @@ url = "https://files.pythonhosted.org/packages/8c/25/f1540879c8815387980e56f973e54605bd924612399ace31487f7444171c/stdlib_list-0.12.0.tar.gz", ) +def tree_sitter_deps(): + "Fetch tree-sitter dependencies with custom BUILD files" + if "com_github_tree_sitter_go_tree_sitter" not in native.existing_rules(): + http_archive( + name = "com_github_tree_sitter_go_tree_sitter", + build_file = Label("//:internal/go_tree_sitter_BUILD.bazel"), + integrity = "sha256-rAsyh/Kydqt24TBaA6H14obQlkaQWwj1QNec3xza6NM=", + strip_prefix = "go-tree-sitter-c9492002f76ed75037e3fe6d3bbabb54ed3e1ff5", + url = "https://github.com/tree-sitter/go-tree-sitter/archive/c9492002f76ed75037e3fe6d3bbabb54ed3e1ff5.zip", + ) + if "com_github_tree_sitter_tree_sitter_python" not in native.existing_rules(): + http_archive( + name = "com_github_tree_sitter_tree_sitter_python", + build_file = Label("//:internal/tree_sitter_python_BUILD.bazel"), + integrity = "sha256-e86IfrLzPpS/dKaWRc9RONQJZyDlT9MmmmEkwGuTxYQ=", + url = "https://github.com/tree-sitter/tree-sitter-python/releases/download/v0.25.0/tree-sitter-python.tar.gz", + ) + def gazelle_deps(): + tree_sitter_deps() go_deps() python_stdlib_list_deps() def go_deps(): + # Prevent `bazel run //:gazelle_update_repos` from generating `go_repository` + # entries for the repos managed by `tree_sitter_deps()` or for the 11 other + # language grammars required by upstream `go-tree-sitter/go.mod` for its own + # internal unit tests. Without these directives, `gazelle update-repos` sees + # those modules in `go list -m all` and re-adds them here; declaring + # `# gazelle:repository` directives tells `update-repos` to skip them. + # gazelle:repository http_archive name=com_github_tree_sitter_go_tree_sitter + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_c + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_cpp + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_embedded_template + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_go + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_html + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_java + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_javascript + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_json + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_php + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_python + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_ruby + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_rust "Fetch go dependencies" go_repository( name = "com_github_bazelbuild_bazel_gazelle", @@ -111,18 +149,18 @@ version = "v0.6.0", ) go_repository( + name = "com_github_mattn_go_pointer", + importpath = "github.com/mattn/go-pointer", + sum = "h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0=", + version = "v0.0.1", + ) + go_repository( name = "com_github_pmezard_go_difflib", importpath = "github.com/pmezard/go-difflib", sum = "h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=", version = "v1.0.0", ) go_repository( - name = "com_github_smacker_go_tree_sitter", - importpath = "github.com/smacker/go-tree-sitter", - sum = "h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4=", - version = "v0.0.0-20240827094217-dd81d9e9be82", - ) - go_repository( name = "com_github_stretchr_objx", importpath = "github.com/stretchr/objx", sum = "h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=", @@ -131,8 +169,8 @@ go_repository( name = "com_github_stretchr_testify", importpath = "github.com/stretchr/testify", - sum = "h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=", - version = "v1.9.0", + sum = "h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=", + version = "v1.10.0", ) go_repository( name = "in_gopkg_check_v1",
diff --git a/gazelle/go.mod b/gazelle/go.mod index 9ad4951..cdf28f2 100644 --- a/gazelle/go.mod +++ b/gazelle/go.mod
@@ -11,8 +11,9 @@ github.com/bmatcuk/doublestar/v4 v4.9.1 github.com/emirpasic/gods v1.18.1 github.com/ghodss/yaml v1.0.0 - github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 github.com/stretchr/testify v1.9.0 + github.com/tree-sitter/go-tree-sitter v0.24.1-0.20251112183152-c9492002f76e + github.com/tree-sitter/tree-sitter-python v0.25.0 golang.org/x/sync v0.11.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -20,6 +21,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/mattn/go-pointer v0.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/mod v0.23.0 // indirect golang.org/x/sys v0.30.0 // indirect
diff --git a/gazelle/go.sum b/gazelle/go.sum index d5fc2b1..1b141b6 100644 --- a/gazelle/go.sum +++ b/gazelle/go.sum
@@ -14,12 +14,16 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4= -github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82/go.mod h1:xe4pgH49k4SsmkQq5OT8abwhWmnzkhpgnXeekbx2efw= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tree-sitter/go-tree-sitter v0.24.1-0.20251112183152-c9492002f76e h1:B9WMBx89/TkqtmwfP+7OhREoGB6RdgmGk9ErlbiTOFI= +github.com/tree-sitter/go-tree-sitter v0.24.1-0.20251112183152-c9492002f76e/go.mod h1:r77ig7BikoZhHrrsjAnv8RqGti5rtSyvDHPzgTPsUuU= +github.com/tree-sitter/tree-sitter-python v0.25.0 h1:O6XD9v8U1LOcRc3cNj9nM7XufrtEBezE6VrpRrHZDf0= +github.com/tree-sitter/tree-sitter-python v0.25.0/go.mod h1:cpdthSy/Yoa28aJFBscFHlGiU+cnSiSh1kuDVtI8YeM= golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
diff --git a/gazelle/internal/go_tree_sitter_BUILD.bazel b/gazelle/internal/go_tree_sitter_BUILD.bazel new file mode 100644 index 0000000..901e614 --- /dev/null +++ b/gazelle/internal/go_tree_sitter_BUILD.bazel
@@ -0,0 +1,47 @@ +"""Build file for github.com/tree-sitter/go-tree-sitter.""" + +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "tree_sitter_sources", + hdrs = glob([ + "include/**/*.h", + "src/**/*.h", + "src/**/*.c", + ]), + defines = [ + "_DEFAULT_SOURCE", + "_POSIX_C_SOURCE=200112L", + ], + includes = [ + "include", + "src", + ], + visibility = ["//visibility:private"], +) + +go_library( + name = "go-tree-sitter", + srcs = glob( + ["*.go"], + exclude = ["*_test.go"], + ) + [ + "allocator.c", + "allocator.h", + ], + cdeps = [":tree_sitter_sources"], + cgo = True, + copts = [ + "-std=c11", + ], + importpath = "github.com/tree-sitter/go-tree-sitter", + visibility = ["//visibility:public"], + deps = ["@com_github_mattn_go_pointer//:go_default_library"], +) + +alias( + name = "go_default_library", + actual = ":go-tree-sitter", + visibility = ["//visibility:public"], +)
diff --git a/gazelle/internal/tree_sitter_python_BUILD.bazel b/gazelle/internal/tree_sitter_python_BUILD.bazel new file mode 100644 index 0000000..1f6fe1f --- /dev/null +++ b/gazelle/internal/tree_sitter_python_BUILD.bazel
@@ -0,0 +1,36 @@ +"""Build file for github.com/tree-sitter/tree-sitter-python.""" + +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "tree_sitter_python_sources", + hdrs = [ + "src/parser.c", + "src/scanner.c", + "src/tree_sitter/alloc.h", + "src/tree_sitter/array.h", + "src/tree_sitter/parser.h", + ], + includes = ["bindings/go"], + visibility = ["//visibility:private"], +) + +go_library( + name = "go", + srcs = ["bindings/go/binding.go"], + cdeps = [":tree_sitter_python_sources"], + cgo = True, + copts = [ + "-std=c11", + "-fPIC", + ], + importpath = "github.com/tree-sitter/tree-sitter-python/bindings/go", + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":go", + visibility = ["//visibility:public"], +)
diff --git a/gazelle/manifest/BUILD.bazel b/gazelle/manifest/BUILD.bazel index 68f4e1b..fc12c5e 100644 --- a/gazelle/manifest/BUILD.bazel +++ b/gazelle/manifest/BUILD.bazel
@@ -12,8 +12,8 @@ importpath = "github.com/bazel-contrib/rules_python/gazelle/manifest", visibility = ["//visibility:public"], deps = [ - "@com_github_emirpasic_gods//sets/treeset:go_default_library", - "@in_gopkg_yaml_v2//:go_default_library", + "@com_github_emirpasic_gods//sets/treeset", + "@in_gopkg_yaml_v2//:yaml_v2", ], )
diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index 1ffa289..0b9fd76 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel
@@ -38,11 +38,11 @@ "@bazel_gazelle//rule", "@com_github_bazelbuild_buildtools//build", "@com_github_bmatcuk_doublestar_v4//:doublestar", - "@com_github_emirpasic_gods//lists/singlylinkedlist:go_default_library", - "@com_github_emirpasic_gods//sets/treeset:go_default_library", - "@com_github_emirpasic_gods//utils:go_default_library", - "@com_github_smacker_go_tree_sitter//:go_default_library", - "@com_github_smacker_go_tree_sitter//python:go_default_library", + "@com_github_emirpasic_gods//lists/singlylinkedlist", + "@com_github_emirpasic_gods//sets/treeset", + "@com_github_emirpasic_gods//utils", + "@com_github_tree_sitter_go_tree_sitter//:go-tree-sitter", + "@com_github_tree_sitter_tree_sitter_python//:go_default_library", "@org_golang_x_sync//errgroup", ], )
diff --git a/gazelle/python/file_parser.go b/gazelle/python/file_parser.go index e129337..7bc3838 100644 --- a/gazelle/python/file_parser.go +++ b/gazelle/python/file_parser.go
@@ -22,8 +22,8 @@ "path/filepath" "strings" - sitter "github.com/smacker/go-tree-sitter" - "github.com/smacker/go-tree-sitter/python" + sitter "github.com/tree-sitter/go-tree-sitter" + tree_sitter_python "github.com/tree-sitter/tree-sitter-python/bindings/go" ) const ( @@ -47,31 +47,35 @@ } type FileParser struct { - code []byte - relFilepath string - output ParserOutput - inTypeCheckingBlock bool + code []byte + relFilepath string + output ParserOutput + inTypeCheckingBlock bool } func NewFileParser() *FileParser { return &FileParser{} } -// ParseCode instantiates a new tree-sitter Parser and parses the python code, returning -// the tree-sitter RootNode. -// It prints a warning if parsing fails. -func ParseCode(code []byte, path string) (*sitter.Node, error) { +// parseCodeToTree instantiates a new tree-sitter Parser and parses the Python code, +// returning the parsed tree-sitter Tree. Callers are responsible for calling Close() +// on the returned Tree to release its underlying C resources when finished. +// It logs a warning if the syntax tree contains parse errors. +func parseCodeToTree(code []byte, path string) (*sitter.Tree, error) { parser := sitter.NewParser() - parser.SetLanguage(python.GetLanguage()) - - tree, err := parser.ParseCtx(context.Background(), nil, code) - if err != nil { + defer parser.Close() + if err := parser.SetLanguage(sitter.NewLanguage(tree_sitter_python.Language())); err != nil { return nil, err } + tree := parser.ParseCtx(context.Background(), code, nil) + if tree == nil { + return nil, fmt.Errorf("failed to parse %q", path) + } + root := tree.RootNode() if !root.HasError() { - return root, nil + return tree, nil } log.Printf("WARNING: failed to parse %q. The resulting BUILD target may be incorrect.", path) @@ -80,43 +84,54 @@ // failure may be in some part of the code that Gazelle doesn't care about. verbose, envExists := os.LookupEnv("RULES_PYTHON_GAZELLE_VERBOSE") if !envExists || verbose != "1" { - return root, nil + return tree, nil } - for i := 0; i < int(root.ChildCount()); i++ { + for i := uint(0); i < root.ChildCount(); i++ { child := root.Child(i) if child.IsError() { // Example logs: // gazelle: Parse error at {Row:1 Column:0}: // def search_one_more_level[T](): - log.Printf("Parse error at %+v:\n%+v", child.StartPoint(), child.Content(code)) + log.Printf("Parse error at %+v:\n%+v", child.StartPosition(), child.Utf8Text(code)) // Log the internal tree-sitter representation of what was parsed. Eg: // gazelle: The above was parsed as: (ERROR (identifier) (call function: (list (identifier)) arguments: (argument_list))) - log.Printf("The above was parsed as: %v", child.String()) + log.Printf("The above was parsed as: %v", child.ToSexp()) } } - return root, nil + return tree, nil +} + +// ParseCode instantiates a new tree-sitter Parser and parses the python code, returning +// the tree-sitter RootNode. +// It prints a warning if parsing fails. +func ParseCode(code []byte, path string) (*sitter.Node, error) { + tree, err := parseCodeToTree(code, path) + if err != nil { + return nil, err + } + return tree.RootNode(), nil } // parseMain returns true if the python file has an `if __name__ == "__main__":` block, // which is a common idiom for python scripts/binaries. func (p *FileParser) parseMain(ctx context.Context, node *sitter.Node) bool { - for i := 0; i < int(node.ChildCount()); i++ { + for i := uint(0); i < node.ChildCount(); i++ { if err := ctx.Err(); err != nil { return false } child := node.Child(i) - if child.Type() == sitterNodeTypeIfStatement && - child.Child(1).Type() == sitterNodeTypeComparisonOperator && child.Child(1).Child(1).Type() == "==" { + if child.Kind() == sitterNodeTypeIfStatement && + child.Child(1).Kind() == sitterNodeTypeComparisonOperator && child.Child(1).Child(1).Kind() == "==" { statement := child.Child(1) a, b := statement.Child(0), statement.Child(2) // convert "'__main__' == __name__" to "__name__ == '__main__'" - if b.Type() == sitterNodeTypeIdentifier { + if b.Kind() == sitterNodeTypeIdentifier { a, b = b, a } - if a.Type() == sitterNodeTypeIdentifier && a.Content(p.code) == "__name__" && - b.Type() == sitterNodeTypeString && string(p.code[b.StartByte()+1:b.EndByte()-1]) == "__main__" { + if a.Kind() == sitterNodeTypeIdentifier && a.Utf8Text(p.code) == "__name__" && + b.Kind() == sitterNodeTypeString && string(p.code[b.StartByte()+1:b.EndByte()-1]) == "__main__" { return true } } @@ -127,18 +142,18 @@ // parseImportStatement parses a node for an import statement, returning a `Module` and a boolean // representing if the parse was OK or not. func parseImportStatement(node *sitter.Node, code []byte) (Module, bool) { - switch node.Type() { + switch node.Kind() { case sitterNodeTypeDottedName: return Module{ - Name: node.Content(code), - LineNumber: node.StartPoint().Row + 1, + Name: node.Utf8Text(code), + LineNumber: uint32(node.StartPosition().Row) + 1, }, true case sitterNodeTypeAliasedImport: return parseImportStatement(node.Child(0), code) case sitterNodeTypeWildcardImport: return Module{ Name: "*", - LineNumber: node.StartPoint().Row + 1, + LineNumber: uint32(node.StartPosition().Row) + 1, }, true } return Module{}, false @@ -158,8 +173,8 @@ // an import statement. It updates FileParser.output.Modules with the `module` that the // import represents. func (p *FileParser) parseImportStatements(node *sitter.Node) bool { - if node.Type() == sitterNodeTypeImportStatement { - for j := 1; j < int(node.ChildCount()); j++ { + if node.Kind() == sitterNodeTypeImportStatement { + for j := uint(1); j < node.ChildCount(); j++ { m, ok := parseImportStatement(node.Child(j), p.code) if !ok { continue @@ -173,15 +188,15 @@ } p.output.Modules = append(p.output.Modules, m) } - } else if node.Type() == sitterNodeTypeImportFromStatement { - from := node.Child(1).Content(p.code) + } else if node.Kind() == sitterNodeTypeImportFromStatement { + from := node.Child(1).Utf8Text(p.code) from = cleanImportString(from) // If the import is from the current package, we don't need to add it to the modules i.e. from . import Class1. // If the import is from a different relative package i.e. from .package1 import foo, we need to add it to the modules. if from == "." { return true } - for j := 3; j < int(node.ChildCount()); j++ { + for j := uint(3); j < node.ChildCount(); j++ { m, ok := parseImportStatement(node.Child(j), p.code) if !ok { continue @@ -202,8 +217,8 @@ // parseComments parses a node for comments, returning true if the node is a comment. // It updates FileParser.output.Comments with the parsed comment. func (p *FileParser) parseComments(node *sitter.Node) bool { - if node.Type() == sitterNodeTypeComment { - p.output.Comments = append(p.output.Comments, Comment(node.Content(p.code))) + if node.Kind() == sitterNodeTypeComment { + p.output.Comments = append(p.output.Comments, Comment(node.Utf8Text(p.code))) return true } return false @@ -217,23 +232,23 @@ // isTypeCheckingBlock returns true if the given node is an `if TYPE_CHECKING:` block. func (p *FileParser) isTypeCheckingBlock(node *sitter.Node) bool { - if node.Type() != sitterNodeTypeIfStatement || node.ChildCount() < 2 { + if node.Kind() != sitterNodeTypeIfStatement || node.ChildCount() < 2 { return false } condition := node.Child(1) // Handle `if TYPE_CHECKING:` - if condition.Type() == sitterNodeTypeIdentifier && condition.Content(p.code) == "TYPE_CHECKING" { + if condition.Kind() == sitterNodeTypeIdentifier && condition.Utf8Text(p.code) == "TYPE_CHECKING" { return true } // Handle `if typing.TYPE_CHECKING:` - if condition.Type() == "attribute" && condition.ChildCount() >= 3 { + if condition.Kind() == "attribute" && condition.ChildCount() >= 3 { object := condition.Child(0) attr := condition.Child(2) - if object.Type() == sitterNodeTypeIdentifier && object.Content(p.code) == "typing" && - attr.Type() == sitterNodeTypeIdentifier && attr.Content(p.code) == "TYPE_CHECKING" { + if object.Kind() == sitterNodeTypeIdentifier && object.Utf8Text(p.code) == "typing" && + attr.Kind() == sitterNodeTypeIdentifier && attr.Utf8Text(p.code) == "TYPE_CHECKING" { return true } } @@ -252,7 +267,7 @@ p.inTypeCheckingBlock = true } - for i := 0; i < int(node.ChildCount()); i++ { + for i := uint(0); i < node.ChildCount(); i++ { if err := ctx.Err(); err != nil { return } @@ -271,10 +286,12 @@ } func (p *FileParser) Parse(ctx context.Context) (*ParserOutput, error) { - rootNode, err := ParseCode(p.code, p.relFilepath) + tree, err := parseCodeToTree(p.code, p.relFilepath) if err != nil { return nil, err } + defer tree.Close() + rootNode := tree.RootNode() p.output.HasMain = p.parseMain(ctx, rootNode)
diff --git a/gazelle/python/file_parser_test.go b/gazelle/python/file_parser_test.go index 0a6fd1b..81c5b99 100644 --- a/gazelle/python/file_parser_test.go +++ b/gazelle/python/file_parser_test.go
@@ -383,3 +383,38 @@ assert.ElementsMatch(t, expected, output.Modules) }) } + +func TestParsePEP758UnparenthesizedExcept(t *testing.T) { + t.Parallel() + code := []byte(`import os + +try: + pass +except ValueError, OSError, OverflowError: + import sys + +if __name__ == "__main__": + pass +`) + rootNode, err := ParseCode(code, "pep758.py") + assert.NoError(t, err) + assert.False(t, rootNode.HasError(), "expected PEP 758 except clause to parse without error") + + p := NewFileParser() + p.SetCodeAndFile(code, "", "pep758.py") + output, err := p.Parse(context.Background()) + assert.NoError(t, err) + assert.True(t, output.HasMain) + assert.Equal(t, []Module{ + { + Name: "os", + LineNumber: 1, + Filepath: "pep758.py", + }, + { + Name: "sys", + LineNumber: 6, + Filepath: "pep758.py", + }, + }, output.Modules) +}
diff --git a/gazelle/pythonconfig/BUILD.bazel b/gazelle/pythonconfig/BUILD.bazel index a82a7f1..9c1e5eb 100644 --- a/gazelle/pythonconfig/BUILD.bazel +++ b/gazelle/pythonconfig/BUILD.bazel
@@ -11,7 +11,7 @@ deps = [ "//manifest", "@bazel_gazelle//label", - "@com_github_emirpasic_gods//lists/singlylinkedlist:go_default_library", + "@com_github_emirpasic_gods//lists/singlylinkedlist", ], )
diff --git a/news/3416.changed.md b/news/3416.changed.md new file mode 100644 index 0000000..5aab21b --- /dev/null +++ b/news/3416.changed.md
@@ -0,0 +1,4 @@ +(gazelle) Migrated the Python AST parser from `smacker/go-tree-sitter` to the +official `tree-sitter/go-tree-sitter` and `tree-sitter/tree-sitter-python` +packages, adding support for Python 3.14 PEP 758 unparenthesized exception +clauses ([#3416](https://github.com/bazel-contrib/rules_python/issues/3416)).