feat(gazelle): Add "python_test_file_pattern" directive (#1819)

Add the `python_test_file_pattern` directive. This directive allows
users to configure what python files get mapped to the `py_test`
rule.

The default behavior is unchanged: both `test_*` and `*_test.py`
files generate `py_test` targets if the directive is unset.

The directive supports multiple glob patterns, separated by a comma.

Note: The original code used, effectively, `test_*` for one of the
patterns. This code uses `test_*.py` instead. These are equivalent
because of the `.py` extension check prior to pattern matching.

Fixes #1816.

Author: dougthor42

CHANGELOG.md

@@ -14,7 +14,7 @@ go_deps.from_file(go_mod = "//:go.mod")
 use_repo(
     go_deps,
     "com_github_bazelbuild_buildtools",
-    "com_github_bmatcuk_doublestar",
+    "com_github_bmatcuk_doublestar_v4",
     "com_github_emirpasic_gods",
     "com_github_ghodss_yaml",
     "in_gopkg_yaml_v2",

gazelle/MODULE.bazel

@@ -38,10 +38,10 @@ def gazelle_deps():
     )
 
     go_repository(
-        name = "com_github_bmatcuk_doublestar",
-        importpath = "github.com/bmatcuk/doublestar",
-        sum = "h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=",
-        version = "v1.3.4",
+        name = "com_github_bmatcuk_doublestar_v4",
+        importpath = "github.com/bmatcuk/doublestar/v4",
+        sum = "h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=",
+        version = "v4.6.1",
     )
 
     go_repository(

gazelle/README.md

@@ -6,7 +6,7 @@ require (
 	github.com/bazelbuild/bazel-gazelle v0.31.1
 	github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d
 	github.com/bazelbuild/rules_go v0.41.0
-	github.com/bmatcuk/doublestar v1.3.4
+	github.com/bmatcuk/doublestar/v4 v4.6.1
 	github.com/emirpasic/gods v1.18.1
 	github.com/ghodss/yaml v1.0.0
 	gopkg.in/yaml.v2 v2.4.0

gazelle/deps.bzl

@@ -6,8 +6,8 @@ github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d h1:Fl1FfItZp
 github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo=
 github.com/bazelbuild/rules_go v0.41.0 h1:JzlRxsFNhlX+g4drDRPhIaU5H5LnI978wdMJ0vK4I+k=
 github.com/bazelbuild/rules_go v0.41.0/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
-github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
-github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I=
+github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=

gazelle/go.mod

@@ -38,7 +38,7 @@ go_library(
         "@bazel_gazelle//resolve:go_default_library",
         "@bazel_gazelle//rule:go_default_library",
         "@com_github_bazelbuild_buildtools//build:go_default_library",
-        "@com_github_bmatcuk_doublestar//:doublestar",
+        "@com_github_bmatcuk_doublestar_v4//:doublestar",
         "@com_github_emirpasic_gods//lists/singlylinkedlist",
         "@com_github_emirpasic_gods//sets/treeset",
         "@com_github_emirpasic_gods//utils",

gazelle/go.sum

@@ -25,6 +25,7 @@ import (
 
 	"github.com/bazelbuild/bazel-gazelle/config"
 	"github.com/bazelbuild/bazel-gazelle/rule"
+	"github.com/bmatcuk/doublestar/v4"
 
 	"github.com/bazelbuild/rules_python/gazelle/manifest"
 	"github.com/bazelbuild/rules_python/gazelle/pythonconfig"
@@ -65,6 +66,7 @@ func (py *Configurer) KnownDirectives() []string {
 		pythonconfig.TestNamingConvention,
 		pythonconfig.DefaultVisibilty,
 		pythonconfig.Visibility,
+		pythonconfig.TestFilePattern,
 	}
 }
 
@@ -181,6 +183,18 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
 			}
 		case pythonconfig.Visibility:
 			config.AppendVisibility(strings.TrimSpace(d.Value))
+		case pythonconfig.TestFilePattern:
+			value := strings.TrimSpace(d.Value)
+			if value == "" {
+				log.Fatal("directive 'python_test_file_pattern' requires a value")
+			}
+			globStrings := strings.Split(value, ",")
+			for _, g := range globStrings {
+				if !doublestar.ValidatePattern(g) {
+					log.Fatalf("invalid glob pattern '%s'", g)
+				}
+			}
+			config.SetTestFilePattern(globStrings)
 		}
 	}
 

gazelle/python/BUILD.bazel

@@ -28,7 +28,7 @@ import (
 	"github.com/bazelbuild/bazel-gazelle/language"
 	"github.com/bazelbuild/bazel-gazelle/rule"
 	"github.com/bazelbuild/rules_python/gazelle/pythonconfig"
-	"github.com/bmatcuk/doublestar"
+	"github.com/bmatcuk/doublestar/v4"
 	"github.com/emirpasic/gods/lists/singlylinkedlist"
 	"github.com/emirpasic/gods/sets/treeset"
 	godsutils "github.com/emirpasic/gods/utils"
@@ -54,6 +54,17 @@ func GetActualKindName(kind string, args language.GenerateArgs) string {
 	return kind
 }
 
+func matchesAnyGlob(s string, globs []string) bool {
+	// This function assumes that the globs have already been validated. If a glob is
+	// invalid, it's considered a non-match and we move on to the next pattern.
+	for _, g := range globs {
+		if ok, _ := doublestar.Match(g, s); ok {
+			return true
+		}
+	}
+	return false
+}
+
 // GenerateRules extracts build metadata from source files in a directory.
 // GenerateRules is called in each directory where an update is requested
 // in depth-first post-order.
@@ -100,6 +111,8 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 	hasPyTestEntryPointTarget := false
 	hasConftestFile := false
 
+	testFileGlobs := cfg.TestFilePattern()
+
 	for _, f := range args.RegularFiles {
 		if cfg.IgnoresFile(filepath.Base(f)) {
 			continue
@@ -113,7 +126,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 				hasPyTestEntryPointFile = true
 			} else if f == conftestFilename {
 				hasConftestFile = true
-			} else if strings.HasSuffix(f, "_test.py") || strings.HasPrefix(f, "test_") {
+			} else if matchesAnyGlob(f, testFileGlobs) {
 				pyTestFilenames.Add(f)
 			} else {
 				pyLibraryFilenames.Add(f)
@@ -195,7 +208,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 							}
 						}
 						baseName := filepath.Base(path)
-						if strings.HasSuffix(baseName, "_test.py") || strings.HasPrefix(baseName, "test_") {
+						if matchesAnyGlob(baseName, testFileGlobs) {
 							pyTestFilenames.Add(srcPath)
 						} else {
 							pyLibraryFilenames.Add(srcPath)

gazelle/python/configure.go

@@ -0,0 +1 @@
+# gazelle:python_generation_mode file

gazelle/python/generate.go

@@ -0,0 +1,18 @@
+load("@rules_python//python:defs.bzl", "py_test")
+
+# gazelle:python_generation_mode file
+
+py_test(
+    name = "hello_test",
+    srcs = ["hello_test.py"],
+)
+
+py_test(
+    name = "test_goodbye",
+    srcs = ["test_goodbye.py"],
+)
+
+py_test(
+    name = "test_hello",
+    srcs = ["test_hello.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/README.md

@@ -0,0 +1 @@
+# gazelle:python_test_file_pattern *_test.py

gazelle/python/testdata/directive_python_test_file_pattern/WORKSPACE

@@ -0,0 +1,17 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_test_file_pattern *_test.py
+
+py_library(
+    name = "test2_star_test_py",
+    srcs = [
+        "test_goodbye.py",
+        "test_hello.py",
+    ],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "hello_test",
+    srcs = ["hello_test.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test.yaml

@@ -0,0 +1,2 @@
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern test_*.py

gazelle/python/testdata/directive_python_test_file_pattern/test1_unset/BUILD.in

@@ -0,0 +1,20 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern test_*.py
+
+py_library(
+    name = "hello_test",
+    srcs = ["hello_test.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "test_goodbye",
+    srcs = ["test_goodbye.py"],
+)
+
+py_test(
+    name = "test_hello",
+    srcs = ["test_hello.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test1_unset/BUILD.out

@@ -0,0 +1,2 @@
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern foo_*_[A-Z]_test?.py

gazelle/python/testdata/directive_python_test_file_pattern/test1_unset/hello_test.py

@@ -0,0 +1,20 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_generation_mode file
+# gazelle:python_test_file_pattern foo_*_[A-Z]_test?.py
+
+py_library(
+    name = "foo_nota_test0_Z1",
+    srcs = ["foo_nota_test0_Z1.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "foo_helloworld_A_testA",
+    srcs = ["foo_helloworld_A_testA.py"],
+)
+
+py_test(
+    name = "foo_my_filename_B_test1",
+    srcs = ["foo_my_filename_B_test1.py"],
+)

gazelle/python/testdata/directive_python_test_file_pattern/test1_unset/test_goodbye.py

@@ -0,0 +1,3 @@
+# gazelle:python_test_file_pattern *_hello.py,hello_*,unittest_*,*_unittest.py
+
+# Note that "foo_unittest.pyc" and "test_bar" files are ignored.

gazelle/python/testdata/directive_python_test_file_pattern/test1_unset/test_hello.py

@@ -0,0 +1,34 @@
+load("@rules_python//python:defs.bzl", "py_library", "py_test")
+
+# gazelle:python_test_file_pattern *_hello.py,hello_*,unittest_*,*_unittest.py
+
+# Note that "foo_unittest.pyc" and "test_bar" files are ignored.
+
+py_library(
+    name = "test5_multiple_patterns",
+    srcs = [
+        "mylib.py",
+        "mylib2.py",
+    ],
+    visibility = ["//:__subpackages__"],
+)
+
+py_test(
+    name = "foo_hello",
+    srcs = ["foo_hello.py"],
+)
+
+py_test(
+    name = "foo_unittest",
+    srcs = ["foo_unittest.py"],
+)
+
+py_test(
+    name = "hello_foo",
+    srcs = ["hello_foo.py"],
+)
+
+py_test(
+    name = "unittest_foo",
+    srcs = ["unittest_foo.py"],
+)