Make inclusion to the test case list based on matching tags

Use tags to narrow down test cases to those labeled with strings matching
the regex passed as a config parameter.

README.md

@@ -133,6 +133,7 @@ Flags:
   -r, --rate-limit duration                    Limit the request rate to the server to 1 request per specified duration. 0 is the default, and disables rate limiting.
       --read-timeout duration                  timeout for receiving responses during test execution (default 10s)
       --show-failures-only                     shows only the results of failed tests
+  -T, --tag string                             include tests tagged with labels matching this Go regular expression (e.g. to include all tests being tagged with "cookie", use "^cookie$").
   -t, --time                                   show time spent per test
       --wait-delay duration                    Time to wait between retries for all wait operations. (default 1s)
       --wait-for-connection-timeout duration   Http connection timeout, The timeout includes connection time, any redirects, and reading the response body. (default 3s)

cmd/run.go

@@ -33,6 +33,7 @@ func NewRunCommand() *cobra.Command {
 
 	runCmd.Flags().StringP("exclude", "e", "", "exclude tests matching this Go regular expression (e.g. to exclude all tests beginning with \"91\", use \"^91.*\"). \nIf you want more permanent exclusion, check the 'exclude' option in the config file.")
 	runCmd.Flags().StringP("include", "i", "", "include only tests matching this Go regular expression (e.g. to include only tests beginning with \"91\", use \"^91.*\"). \\nIf you want more permanent inclusion, check the 'include' option in the config file.\"")
+	runCmd.Flags().StringP("tag", "T", "", "include tests tagged with labels matching this Go regular expression (e.g. to include all tests being tagged with \"cookie\", use \"^cookie$\").")
 	runCmd.Flags().StringP("dir", "d", ".", "recursively find yaml tests in this directory")
 	runCmd.Flags().StringP("output", "o", "normal", "output type for ftw tests. \"normal\" is the default.")
 	runCmd.Flags().StringP("file", "f", "", "output file path for ftw tests. Prints to standard output by default.")
@@ -65,6 +66,7 @@ func runE(cmd *cobra.Command, _ []string) error {
 	cmd.SilenceUsage = true
 	exclude, _ := cmd.Flags().GetString("exclude")
 	include, _ := cmd.Flags().GetString("include")
+	tag, _ := cmd.Flags().GetString("tag")
 	dir, _ := cmd.Flags().GetString("dir")
 	outputFilename, _ := cmd.Flags().GetString("file")
 	logFilePath, _ := cmd.Flags().GetString("log-file")
@@ -119,6 +121,10 @@ func runE(cmd *cobra.Command, _ []string) error {
 	if exclude != "" {
 		excludeRE = regexp.MustCompile(exclude)
 	}
+	var tagRE *regexp.Regexp
+	if tag != "" {
+		tagRE = regexp.MustCompile(tag)
+	}
 
 	// Add wait4x checkers
 	if waitForHost != "" {
@@ -167,6 +173,7 @@ func runE(cmd *cobra.Command, _ []string) error {
 	currentRun, err := runner.Run(cfg, tests, runner.RunnerConfig{
 		Include:        includeRE,
 		Exclude:        excludeRE,
+		Tag:            tagRE,
 		ShowTime:       showTime,
 		ShowOnlyFailed: showOnlyFailed,
 		ConnectTimeout: connectTimeout,

config/config.go

@@ -28,6 +28,7 @@ func NewDefaultConfig() *FTWConfiguration {
 		MaxMarkerLogLines:   DefaultMaxMarkerLogLines,
 		IncludeTests:        make(map[*FTWRegexp]string),
 		ExcludeTests:        make(map[*FTWRegexp]string),
+		TestTags:            make(map[*FTWRegexp]string),
 	}
 	return cfg
 }

config/config_test.go

@@ -20,6 +20,8 @@ include:
     '^9.*': 'Include all tests starting with 9'
 exclude:
     '^920400-2$': 'Exclude this test'
+tag:
+    '^cookie$': 'Run test tagged with this label'
 testoverride:
   input:
     dest_addr: 'httpbingo.org'

config/types.go

@@ -48,6 +48,8 @@ type FTWConfiguration struct {
 	IncludeTests map[*FTWRegexp]string `koanf:"include"`
 	// ExcludeTests is a list of tests to exclude (same as --exclude)
 	ExcludeTests map[*FTWRegexp]string `koanf:"exclude"`
+	// TestTags is a list of tags matching tests to run (same as --tag)
+	TestTags map[*FTWRegexp]string `koanf:"tag"`
 }
 
 type PlatformOverrides struct {

runner/run.go

@@ -52,6 +52,7 @@ func Run(cfg *config.FTWConfiguration, tests []*test.FTWTest, c RunnerConfig, ou
 		RunnerConfig:   &c,
 		Include:        c.Include,
 		Exclude:        c.Exclude,
+		Tag:            c.Tag,
 		ShowTime:       c.ShowTime,
 		Output:         out,
 		ShowOnlyFailed: c.ShowOnlyFailed,
@@ -84,7 +85,7 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error {
 
 	for _, testCase := range ftwTest.Tests {
 		// if we received a particular test ID, skip until we find it
-		if needToSkipTest(runContext.Include, runContext.Exclude, &testCase) {
+		if needToSkipTest(runContext.Include, runContext.Exclude, runContext.Tag, &testCase) {
 			runContext.Stats.addResultToStats(Skipped, &testCase)
 			continue
 		}
@@ -257,7 +258,7 @@ func markAndFlush(runContext *TestRunContext, dest *ftwhttp.Destination, stageID
 	return nil, fmt.Errorf("can't find log marker. Am I reading the correct log? Log file: %s", runContext.Config.LogFile)
 }
 
-func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, testCase *schema.Test) bool {
+func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, tag *regexp.Regexp, testCase *schema.Test) bool {
 	// never skip enabled explicit inclusions
 	if include != nil {
 		if include.MatchString(testCase.IdString()) {
@@ -267,6 +268,14 @@ func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, testCase *sc
 	}
 
 	result := false
+	// if the test's tags do not match the passed ones
+	// it needs to be skipped
+	if tag != nil {
+		if !utils.MatchSlice(tag, testCase.Tags) {
+			result = true
+		}
+	}
+
 	// if we need to exclude tests, and the ID matches,
 	// it needs to be skipped
 	if exclude != nil {

runner/run_test.go

@@ -305,6 +305,14 @@ func (s *runTestSuite) TestRunTests_Run() {
 		s.Equal(res.Stats.TotalFailed(), 0, "failed to exclude test")
 	})
 
+	s.Run("count test tagged with `tag-1`", func() {
+		res, err := Run(s.cfg, s.ftwTests, RunnerConfig{
+			Tag: regexp.MustCompile("tag-1"),
+		}, s.out)
+		s.Require().NoError(err)
+		s.Equal(res.Stats.TotalFailed(), 0, "failed to incorporate tagged test")
+	})
+
 	s.Run("test exceptions 1", func() {
 		res, err := Run(s.cfg, s.ftwTests, RunnerConfig{
 			Include: regexp.MustCompile("1*"),

runner/types.go

@@ -19,6 +19,8 @@ type RunnerConfig struct {
 	Include *regexp.Regexp
 	// Exclude is a regular expression to filter tests to exclude. If nil, no tests are excluded.
 	Exclude *regexp.Regexp
+	// Tag is a regular expression to filter tests to count the ones tagged with the mathing label. If nil, no impact on test runner.
+	Tag *regexp.Regexp
 	// ShowTime determines whether to show the time taken to run each test.
 	ShowTime bool
 	// ShowOnlyFailed will only output information related to failed tests
@@ -43,6 +45,7 @@ type TestRunContext struct {
 	RunnerConfig   *RunnerConfig
 	Include        *regexp.Regexp
 	Exclude        *regexp.Regexp
+	Tag            *regexp.Regexp
 	ShowTime       bool
 	ShowOnlyFailed bool
 	Output         *output.Output

utils/slice.go

@@ -0,0 +1,18 @@
+// Copyright 2023 OWASP ModSecurity Core Rule Set Project
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"regexp"
+)
+
+func MatchSlice(r *regexp.Regexp, s []string) bool {
+	for _, str := range s {
+		if r.MatchString(str) {
+			return true
+		}
+	}
+
+	return false
+}

utils/slice_test.go

@@ -0,0 +1,27 @@
+// Copyright 2023 OWASP ModSecurity Core Rule Set Project
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type sliceTestSuite struct {
+	suite.Suite
+}
+
+func TestSliceTestSuite(t *testing.T) {
+	suite.Run(t, new(timeTestSuite))
+}
+
+func (s *timeTestSuite) TestMatchSlice() {
+	re := regexp.MustCompile("^cookie$")
+
+	s.True(MatchSlice(re, []string{"cookie", "php"}))
+	s.False(MatchSlice(re, []string{"cooke", "php"}))
+	s.False(MatchSlice(re, []string{"cookies", "php"}))
+}