diff --git a/internal/civisibility/constants/test_tags.go b/internal/civisibility/constants/test_tags.go index d3b30d461b..98c4bd9052 100644 --- a/internal/civisibility/constants/test_tags.go +++ b/internal/civisibility/constants/test_tags.go @@ -78,6 +78,9 @@ const ( // This constant is used to tag test events that are part of a retry execution TestIsRetry = "test.is_retry" + // TestRetryReason indicates the reason for retrying the test + TestRetryReason = "test.retry_reason" + // TestEarlyFlakeDetectionRetryAborted indicates a retry abort reason by the early flake detection feature TestEarlyFlakeDetectionRetryAborted = "test.early_flake.abort_reason" diff --git a/internal/civisibility/integrations/civisibility_features.go b/internal/civisibility/integrations/civisibility_features.go index b85ba53b61..1a7a6caea7 100644 --- a/internal/civisibility/integrations/civisibility_features.go +++ b/internal/civisibility/integrations/civisibility_features.go @@ -51,8 +51,8 @@ var ( // ciVisibilitySettings contains the CI Visibility settings for this session ciVisibilitySettings net.SettingsResponseData - // ciVisibilityEarlyFlakyDetectionSettings contains the CI Visibility Early Flake Detection data for this session - ciVisibilityEarlyFlakyDetectionSettings net.EfdResponseData + // ciVisibilityKnownTests contains the CI Visibility Known Tests data for this session + ciVisibilityKnownTests net.KnownTestsResponseData // ciVisibilityFlakyRetriesSettings contains the CI Visibility Flaky Retries settings for this session ciVisibilityFlakyRetriesSettings FlakyRetriesSetting @@ -121,15 +121,20 @@ func ensureAdditionalFeaturesInitialization(serviceName string) { return } - // if early flake detection is enabled then we run the early flake detection request - if ciVisibilitySettings.EarlyFlakeDetection.Enabled { - ciEfdData, err := ciVisibilityClient.GetEarlyFlakeDetectionData() + // if early flake detection is enabled then we run the known tests request + if ciVisibilitySettings.KnownTestsEnabled { + ciEfdData, err := ciVisibilityClient.GetKnownTests() if err != nil { - log.Error("civisibility: error getting CI visibility early flake detection data: %v", err) + log.Error("civisibility: error getting CI visibility known tests data: %v", err) } else if ciEfdData != nil { - ciVisibilityEarlyFlakyDetectionSettings = *ciEfdData - log.Debug("civisibility: early flake detection data loaded.") + ciVisibilityKnownTests = *ciEfdData + log.Debug("civisibility: known tests data loaded.") } + } else { + // "known_tests_enabled" parameter works as a kill-switch for EFD, so if “known_tests_enabled” is false it + // will disable EFD even if “early_flake_detection.enabled” is set to true (which should not happen normally, + // the backend should disable both of them in that case) + ciVisibilitySettings.EarlyFlakeDetection.Enabled = false } // if flaky test retries is enabled then let's load the flaky retries settings @@ -172,11 +177,11 @@ func GetSettings() *net.SettingsResponseData { return &ciVisibilitySettings } -// GetEarlyFlakeDetectionSettings gets the early flake detection known tests data -func GetEarlyFlakeDetectionSettings() *net.EfdResponseData { +// GetKnownTests gets the known tests data +func GetKnownTests() *net.KnownTestsResponseData { // call to ensure the additional features initialization is completed (service name can be null here) ensureAdditionalFeaturesInitialization("") - return &ciVisibilityEarlyFlakyDetectionSettings + return &ciVisibilityKnownTests } // GetFlakyRetriesSettings gets the flaky retries settings diff --git a/internal/civisibility/integrations/gotesting/coverage/coverage_writer_test.go b/internal/civisibility/integrations/gotesting/coverage/coverage_writer_test.go index f66ea11b5d..ed0f040734 100644 --- a/internal/civisibility/integrations/gotesting/coverage/coverage_writer_test.go +++ b/internal/civisibility/integrations/gotesting/coverage/coverage_writer_test.go @@ -73,7 +73,7 @@ type MockClient struct { SendCoveragePayloadFunc func(ciTestCovPayload io.Reader) error SendCoveragePayloadWithFormatFunc func(ciTestCovPayload io.Reader, format string) error GetSettingsFunc func() (*net.SettingsResponseData, error) - GetEarlyFlakeDetectionDataFunc func() (*net.EfdResponseData, error) + GetKnownTestsFunc func() (*net.KnownTestsResponseData, error) GetCommitsFunc func(localCommits []string) ([]string, error) SendPackFilesFunc func(commitSha string, packFiles []string) (bytes int64, err error) GetSkippableTestsFunc func() (correlationId string, skippables map[string]map[string][]net.SkippableResponseDataAttributes, err error) @@ -91,8 +91,8 @@ func (m *MockClient) GetSettings() (*net.SettingsResponseData, error) { return m.GetSettingsFunc() } -func (m *MockClient) GetEarlyFlakeDetectionData() (*net.EfdResponseData, error) { - return m.GetEarlyFlakeDetectionDataFunc() +func (m *MockClient) GetKnownTests() (*net.KnownTestsResponseData, error) { + return m.GetKnownTestsFunc() } func (m *MockClient) GetCommits(localCommits []string) ([]string, error) { diff --git a/internal/civisibility/integrations/gotesting/instrumentation.go b/internal/civisibility/integrations/gotesting/instrumentation.go index 4d8c0a5738..b163dab80d 100644 --- a/internal/civisibility/integrations/gotesting/instrumentation.go +++ b/internal/civisibility/integrations/gotesting/instrumentation.go @@ -9,7 +9,6 @@ import ( "fmt" "reflect" "runtime" - "slices" "sync" "sync/atomic" "testing" @@ -36,7 +35,9 @@ type ( panicData any // panic data recovered from an internal test execution when using an additional feature wrapper panicStacktrace string // stacktrace from the panic recovered from an internal test isARetry bool // flag to tag if a current test execution is a retry - isANewTest bool // flag to tag if a current test execution is part of a new test (EFD not known test) + isANewTest bool // flag to tag if a current test execution is part of a new test + isEFDExecution bool // flag to tag if a current test execution is part of an EFD execution + isATRExecution bool // flag to tag if a current test execution is part of an ATR execution hasAdditionalFeatureWrapper bool // flag to check if the current execution is part of an additional feature wrapper } @@ -191,20 +192,29 @@ func applyFlakyTestRetriesAdditionalFeature(targetFunc func(*testing.T)) (func(* initialRetryCount: flakyRetrySettings.RetryCount, adjustRetryCount: nil, // No adjustRetryCount shouldRetry: func(ptrToLocalT *testing.T, executionIndex int, remainingRetries int64) bool { - remainingTotalRetries := atomic.AddInt64(&flakyRetrySettings.RemainingTotalRetryCount, -1) // Decide whether to retry - return ptrToLocalT.Failed() && remainingRetries >= 0 && remainingTotalRetries >= 0 + return ptrToLocalT.Failed() && remainingRetries >= 0 && atomic.LoadInt64(&flakyRetrySettings.RemainingTotalRetryCount) >= 0 + }, + perExecution: func(ptrToLocalT *testing.T, executionIndex int, duration time.Duration) { + if executionIndex > 0 { + atomic.AddInt64(&flakyRetrySettings.RemainingTotalRetryCount, -1) + } }, - perExecution: nil, // No perExecution needed onRetryEnd: func(t *testing.T, executionIndex int, lastPtrToLocalT *testing.T) { // Update original `t` with results from last execution tCommonPrivates := getTestPrivateFields(t) + if tCommonPrivates == nil { + panic("getting test private fields failed") + } tCommonPrivates.SetFailed(lastPtrToLocalT.Failed()) tCommonPrivates.SetSkipped(lastPtrToLocalT.Skipped()) // Update parent status if failed if lastPtrToLocalT.Failed() { tParentCommonPrivates := getTestParentPrivateFields(t) + if tParentCommonPrivates == nil { + panic("getting test parent private fields failed") + } tParentCommonPrivates.SetFailed(true) } @@ -218,14 +228,17 @@ func applyFlakyTestRetriesAdditionalFeature(targetFunc func(*testing.T)) (func(* } fmt.Printf(" [ %v after %v retries by Datadog's auto test retries ]\n", status, executionIndex) - } - // Check if total retry count was exceeded - if flakyRetrySettings.RemainingTotalRetryCount < 1 { - fmt.Println(" the maximum number of total retries was exceeded.") + // Check if total retry count was exceeded + if atomic.LoadInt64(&flakyRetrySettings.RemainingTotalRetryCount) < 1 { + fmt.Println(" the maximum number of total retries was exceeded.") + } } }, - execMetaAdjust: nil, // No execMetaAdjust needed + execMetaAdjust: func(execMeta *testExecutionMetadata, executionIndex int) { + // Set the flag ATR execution to true + execMeta.isATRExecution = true + }, }) }, true } @@ -234,89 +247,82 @@ func applyFlakyTestRetriesAdditionalFeature(targetFunc func(*testing.T)) (func(* // applyEarlyFlakeDetectionAdditionalFeature applies the early flake detection feature as a wrapper of a func(*testing.T) func applyEarlyFlakeDetectionAdditionalFeature(testInfo *commonInfo, targetFunc func(*testing.T), settings *net.SettingsResponseData) (func(*testing.T), bool) { - earlyFlakeDetectionData := integrations.GetEarlyFlakeDetectionSettings() - if earlyFlakeDetectionData != nil && - len(earlyFlakeDetectionData.Tests) > 0 { - - // Define is a known test flag - isAKnownTest := false - - // Check if the test is a known test or a new one - if knownSuites, ok := earlyFlakeDetectionData.Tests[testInfo.moduleName]; ok { - if knownTests, ok := knownSuites[testInfo.suiteName]; ok { - if slices.Contains(knownTests, testInfo.testName) { - isAKnownTest = true - } - } - } + isKnown, hasKnownData := isKnownTest(testInfo) + if !hasKnownData || isKnown { + return targetFunc, false + } - // If it's a new test, then we apply the EFD wrapper - if !isAKnownTest { - return func(t *testing.T) { - var testPassCount, testSkipCount, testFailCount int - - runTestWithRetry(&runTestWithRetryOptions{ - targetFunc: targetFunc, - t: t, - initialRetryCount: 0, - adjustRetryCount: func(duration time.Duration) int64 { - slowTestRetriesSettings := settings.EarlyFlakeDetection.SlowTestRetries - durationSecs := duration.Seconds() - if durationSecs < 5 { - return int64(slowTestRetriesSettings.FiveS) - } else if durationSecs < 10 { - return int64(slowTestRetriesSettings.TenS) - } else if durationSecs < 30 { - return int64(slowTestRetriesSettings.ThirtyS) - } else if duration.Minutes() < 5 { - return int64(slowTestRetriesSettings.FiveM) - } - return 0 - }, - shouldRetry: func(ptrToLocalT *testing.T, executionIndex int, remainingRetries int64) bool { - return remainingRetries >= 0 - }, - perExecution: func(ptrToLocalT *testing.T, executionIndex int, duration time.Duration) { - // Collect test results - if ptrToLocalT.Failed() { - testFailCount++ - } else if ptrToLocalT.Skipped() { - testSkipCount++ - } else { - testPassCount++ - } - }, - onRetryEnd: func(t *testing.T, executionIndex int, lastPtrToLocalT *testing.T) { - // Update test status based on collected counts - tCommonPrivates := getTestPrivateFields(t) + // If it's a new test, then we apply the EFD wrapper + return func(t *testing.T) { + var testPassCount, testSkipCount, testFailCount int + + runTestWithRetry(&runTestWithRetryOptions{ + targetFunc: targetFunc, + t: t, + initialRetryCount: 0, + adjustRetryCount: func(duration time.Duration) int64 { + slowTestRetriesSettings := settings.EarlyFlakeDetection.SlowTestRetries + durationSecs := duration.Seconds() + if durationSecs < 5 { + return int64(slowTestRetriesSettings.FiveS) + } else if durationSecs < 10 { + return int64(slowTestRetriesSettings.TenS) + } else if durationSecs < 30 { + return int64(slowTestRetriesSettings.ThirtyS) + } else if duration.Minutes() < 5 { + return int64(slowTestRetriesSettings.FiveM) + } + return 0 + }, + shouldRetry: func(ptrToLocalT *testing.T, executionIndex int, remainingRetries int64) bool { + return remainingRetries >= 0 + }, + perExecution: func(ptrToLocalT *testing.T, executionIndex int, duration time.Duration) { + // Collect test results + if ptrToLocalT.Failed() { + testFailCount++ + } else if ptrToLocalT.Skipped() { + testSkipCount++ + } else { + testPassCount++ + } + }, + onRetryEnd: func(t *testing.T, executionIndex int, lastPtrToLocalT *testing.T) { + // Update test status based on collected counts + tCommonPrivates := getTestPrivateFields(t) + if tCommonPrivates == nil { + panic("getting test private fields failed") + } + status := "passed" + if testPassCount == 0 { + if testSkipCount > 0 { + status = "skipped" + tCommonPrivates.SetSkipped(true) + } + if testFailCount > 0 { + status = "failed" + tCommonPrivates.SetFailed(true) tParentCommonPrivates := getTestParentPrivateFields(t) - status := "passed" - if testPassCount == 0 { - if testSkipCount > 0 { - status = "skipped" - tCommonPrivates.SetSkipped(true) - } - if testFailCount > 0 { - status = "failed" - tCommonPrivates.SetFailed(true) - tParentCommonPrivates.SetFailed(true) - } + if tParentCommonPrivates == nil { + panic("getting test parent private fields failed") } + tParentCommonPrivates.SetFailed(true) + } + } - // Print summary after retries - if executionIndex > 0 { - fmt.Printf(" [ %v after %v retries by Datadog's early flake detection ]\n", status, executionIndex) - } - }, - execMetaAdjust: func(execMeta *testExecutionMetadata, executionIndex int) { - // Set the flag new test to true - execMeta.isANewTest = true - }, - }) - }, true - } - } - return targetFunc, false + // Print summary after retries + if executionIndex > 0 { + fmt.Printf(" [ %v after %v retries by Datadog's early flake detection ]\n", status, executionIndex) + } + }, + execMetaAdjust: func(execMeta *testExecutionMetadata, executionIndex int) { + // Set the flag new test to true + execMeta.isANewTest = true + // Set the flag EFD execution to true + execMeta.isEFDExecution = true + }, + }) + }, true } // runTestWithRetry encapsulates the common retry logic for test functions. @@ -336,7 +342,10 @@ func runTestWithRetry(options *runTestWithRetryOptions) { for { // Clear the matcher subnames map before each execution to avoid subname tests being called "parent/subname#NN" due to retries - getTestContextMatcherPrivateFields(options.t).ClearSubNames() + matcher := getTestContextMatcherPrivateFields(options.t) + if matcher != nil { + matcher.ClearSubNames() + } // Increment execution index executionIndex++ @@ -348,6 +357,12 @@ func runTestWithRetry(options *runTestWithRetryOptions) { // Create a dummy parent so we can run the test using this local copy // without affecting the test parent localTPrivateFields := getTestPrivateFields(ptrToLocalT) + if localTPrivateFields == nil { + panic("getting test private fields failed") + } + if localTPrivateFields.parent == nil { + panic("parent of the test is nil") + } *localTPrivateFields.parent = unsafe.Pointer(&testing.T{}) // Create an execution metadata instance @@ -362,6 +377,12 @@ func runTestWithRetry(options *runTestWithRetryOptions) { if originalExecMeta.isARetry { execMeta.isARetry = true } + if originalExecMeta.isEFDExecution { + execMeta.isEFDExecution = true + } + if originalExecMeta.isATRExecution { + execMeta.isATRExecution = true + } } // If we are in a retry execution, set the `isARetry` flag diff --git a/internal/civisibility/integrations/gotesting/instrumentation_orchestrion.go b/internal/civisibility/integrations/gotesting/instrumentation_orchestrion.go index becf25bed8..8d1baa762c 100644 --- a/internal/civisibility/integrations/gotesting/instrumentation_orchestrion.go +++ b/internal/civisibility/integrations/gotesting/instrumentation_orchestrion.go @@ -159,6 +159,12 @@ func instrumentTestingTFunc(f func(*testing.T)) func(*testing.T) { if parentExecMeta.isARetry { execMeta.isARetry = true } + if parentExecMeta.isEFDExecution { + execMeta.isEFDExecution = true + } + if parentExecMeta.isATRExecution { + execMeta.isATRExecution = true + } } } @@ -175,6 +181,15 @@ func instrumentTestingTFunc(f func(*testing.T)) func(*testing.T) { if execMeta.isARetry { // Set the retry tag test.SetTag(constants.TestIsRetry, "true") + + // If the execution is an EFD execution we tag the test event reason + if execMeta.isEFDExecution { + // Set the EFD as the retry reason + test.SetTag(constants.TestRetryReason, "efd") + } else if execMeta.isATRExecution { + // Set the ATR as the retry reason + test.SetTag(constants.TestRetryReason, "atr") + } } defer func() { diff --git a/internal/civisibility/integrations/gotesting/testcontroller_test.go b/internal/civisibility/integrations/gotesting/testcontroller_test.go index d205c7dcf8..05b2c9063d 100644 --- a/internal/civisibility/integrations/gotesting/testcontroller_test.go +++ b/internal/civisibility/integrations/gotesting/testcontroller_test.go @@ -74,7 +74,19 @@ func TestMain(m *testing.M) { func runFlakyTestRetriesTests(m *testing.M) { // mock the settings api to enable automatic test retries - server := setUpHttpServer(true, false, nil, false, nil) + server := setUpHttpServer(true, true, false, &net.KnownTestsResponseData{ + Tests: net.KnownTestsResponseDataModules{ + "gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting": net.KnownTestsResponseDataSuites{ + "reflections_test.go": []string{ + "TestGetFieldPointerFrom", + "TestGetInternalTestArray", + "TestGetInternalBenchmarkArray", + "TestCommonPrivateFields_AddLevel", + "TestGetBenchmarkPrivateFields", + }, + }, + }, + }, false, nil) defer server.Close() // set a custom retry count @@ -137,6 +149,13 @@ func runFlakyTestRetriesTests(m *testing.M) { // check spans by tag checkSpansByTagName(finishedSpans, constants.TestIsRetry, 6) + trrSpan := checkSpansByTagName(finishedSpans, constants.TestRetryReason, 6)[0] + if trrSpan.Tag(constants.TestRetryReason) != "atr" { + panic(fmt.Sprintf("expected retry reason to be %s, got %s", "atr", trrSpan.Tag(constants.TestRetryReason))) + } + + // check the test is new tag + checkSpansByTagName(finishedSpans, constants.TestIsNew, 27) // check spans by type checkSpansByType(finishedSpans, @@ -153,9 +172,9 @@ func runFlakyTestRetriesTests(m *testing.M) { func runEarlyFlakyTestDetectionTests(m *testing.M) { // mock the settings api to enable automatic test retries - server := setUpHttpServer(false, true, &net.EfdResponseData{ - Tests: net.EfdResponseDataModules{ - "github.com/DataDog/dd-trace-go/v2/internal/civisibility/integrations/gotesting": net.EfdResponseDataSuites{ + server := setUpHttpServer(false, true, true, &net.KnownTestsResponseData{ + Tests: net.KnownTestsResponseDataModules{ + "github.com/DataDog/dd-trace-go/v2/internal/civisibility/integrations/gotesting": net.KnownTestsResponseDataSuites{ "reflections_test.go": []string{ "TestGetFieldPointerFrom", "TestGetInternalTestArray", @@ -227,6 +246,10 @@ func runEarlyFlakyTestDetectionTests(m *testing.M) { // check spans by tag checkSpansByTagName(finishedSpans, constants.TestIsNew, 176) checkSpansByTagName(finishedSpans, constants.TestIsRetry, 160) + trrSpan := checkSpansByTagName(finishedSpans, constants.TestRetryReason, 160)[0] + if trrSpan.Tag(constants.TestRetryReason) != "efd" { + panic(fmt.Sprintf("expected retry reason to be %s, got %s", "efd", trrSpan.Tag(constants.TestRetryReason))) + } // check spans by type checkSpansByType(finishedSpans, @@ -243,9 +266,9 @@ func runEarlyFlakyTestDetectionTests(m *testing.M) { func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) { // mock the settings api to enable automatic test retries - server := setUpHttpServer(true, true, &net.EfdResponseData{ - Tests: net.EfdResponseDataModules{ - "github.com/DataDog/dd-trace-go/v2/internal/civisibility/integrations/gotesting": net.EfdResponseDataSuites{ + server := setUpHttpServer(true, true, true, &net.KnownTestsResponseData{ + Tests: net.KnownTestsResponseDataModules{ + "github.com/DataDog/dd-trace-go/v2/internal/civisibility/integrations/gotesting": net.KnownTestsResponseDataSuites{ "reflections_test.go": []string{ "TestGetFieldPointerFrom", "TestGetInternalTestArray", @@ -355,7 +378,7 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) { func runIntelligentTestRunnerTests(m *testing.M) { // mock the settings api to enable automatic test retries - server := setUpHttpServer(true, false, nil, true, []net.SkippableResponseDataAttributes{ + server := setUpHttpServer(true, true, false, nil, true, []net.SkippableResponseDataAttributes{ { Suite: "testing_test.go", Name: "TestMyTest01", @@ -521,7 +544,7 @@ func checkSpansByType(finishedSpans []*mocktracer.Span, } } -func checkSpansByResourceName(finishedSpans []*mocktracer.Span, resourceName string, count int) []mocktracer.Span { +func checkSpansByResourceName(finishedSpans []*mocktracer.Span, resourceName string, count int) []*mocktracer.Span { spans := getSpansWithResourceName(finishedSpans, resourceName) numOfSpans := len(spans) if numOfSpans != count { @@ -531,7 +554,7 @@ func checkSpansByResourceName(finishedSpans []*mocktracer.Span, resourceName str return spans } -func checkSpansByTagName(finishedSpans []*mocktracer.Span, tagName string, count int) []mocktracer.Span { +func checkSpansByTagName(finishedSpans []*mocktracer.Span, tagName string, count int) []*mocktracer.Span { spans := getSpansWithTagName(finishedSpans, tagName) numOfSpans := len(spans) if numOfSpans != count { @@ -541,7 +564,7 @@ func checkSpansByTagName(finishedSpans []*mocktracer.Span, tagName string, count return spans } -func checkSpansByTagValue(finishedSpans []*mocktracer.Span, tagName, tagValue string, count int) []mocktracer.Span { +func checkSpansByTagValue(finishedSpans []*mocktracer.Span, tagName, tagValue string, count int) []*mocktracer.Span { spans := getSpansWithTagNameAndValue(finishedSpans, tagName, tagValue) numOfSpans := len(spans) if numOfSpans != count { @@ -569,8 +592,10 @@ type ( ) func setUpHttpServer(flakyRetriesEnabled bool, - earlyFlakyDetectionEnabled bool, earlyFlakyDetectionData *net.EfdResponseData, + knownTestsEnabled bool, + earlyFlakyDetectionEnabled bool, earlyFlakyDetectionData *net.KnownTestsResponseData, itrEnabled bool, itrData []net.SkippableResponseDataAttributes) *httptest.Server { + enableKnownTests := knownTestsEnabled || earlyFlakyDetectionEnabled // mock the settings api to enable automatic test retries server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Printf("MockApi received request: %s\n", r.URL.Path) @@ -591,6 +616,7 @@ func setUpHttpServer(flakyRetriesEnabled bool, FlakyTestRetriesEnabled: flakyRetriesEnabled, ItrEnabled: itrEnabled, TestsSkipping: itrEnabled, + KnownTestsEnabled: enableKnownTests, } response.Data.Attributes.EarlyFlakeDetection.Enabled = earlyFlakyDetectionEnabled response.Data.Attributes.EarlyFlakeDetection.SlowTestRetries.FiveS = 10 @@ -600,13 +626,13 @@ func setUpHttpServer(flakyRetriesEnabled bool, fmt.Printf("MockApi sending response: %v\n", response) json.NewEncoder(w).Encode(&response) - } else if earlyFlakyDetectionEnabled && r.URL.Path == "/api/v2/ci/libraries/tests" { + } else if enableKnownTests && r.URL.Path == "/api/v2/ci/libraries/tests" { w.Header().Set("Content-Type", "application/json") response := struct { Data struct { - ID string `json:"id"` - Type string `json:"type"` - Attributes net.EfdResponseData `json:"attributes"` + ID string `json:"id"` + Type string `json:"type"` + Attributes net.KnownTestsResponseData `json:"attributes"` } `json:"data,omitempty"` }{} @@ -652,51 +678,51 @@ func setUpHttpServer(flakyRetriesEnabled bool, return server } -func getSpansWithType(spans []*mocktracer.Span, spanType string) []mocktracer.Span { - var result []mocktracer.Span +func getSpansWithType(spans []*mocktracer.Span, spanType string) []*mocktracer.Span { + var result []*mocktracer.Span for _, span := range spans { if span.Tag(ext.SpanType) == spanType { - result = append(result, *span) + result = append(result, span) } } return result } -func getSpansWithResourceName(spans []*mocktracer.Span, resourceName string) []mocktracer.Span { - var result []mocktracer.Span +func getSpansWithResourceName(spans []*mocktracer.Span, resourceName string) []*mocktracer.Span { + var result []*mocktracer.Span for _, span := range spans { if span.Tag(ext.ResourceName) == resourceName { - result = append(result, *span) + result = append(result, span) } } return result } -func getSpansWithTagName(spans []*mocktracer.Span, tag string) []mocktracer.Span { - var result []mocktracer.Span +func getSpansWithTagName(spans []*mocktracer.Span, tag string) []*mocktracer.Span { + var result []*mocktracer.Span for _, span := range spans { if span.Tag(tag) != nil { - result = append(result, *span) + result = append(result, span) } } return result } -func getSpansWithTagNameAndValue(spans []*mocktracer.Span, tag, value string) []mocktracer.Span { - var result []mocktracer.Span +func getSpansWithTagNameAndValue(spans []*mocktracer.Span, tag, value string) []*mocktracer.Span { + var result []*mocktracer.Span for _, span := range spans { if span.Tag(tag) == value { - result = append(result, *span) + result = append(result, span) } } return result } -func showResourcesNameFromSpans(spans []mocktracer.Span) { +func showResourcesNameFromSpans(spans []*mocktracer.Span) { for i, span := range spans { fmt.Printf(" [%d] = %v\n", i, span.Tag(ext.ResourceName)) } diff --git a/internal/civisibility/integrations/gotesting/testing.go b/internal/civisibility/integrations/gotesting/testing.go index 184061905e..6696c54aba 100644 --- a/internal/civisibility/integrations/gotesting/testing.go +++ b/internal/civisibility/integrations/gotesting/testing.go @@ -9,6 +9,7 @@ import ( "fmt" "reflect" "runtime" + "slices" "sync/atomic" "testing" "time" @@ -168,6 +169,7 @@ func (ddm *M) executeInternalTest(testInfo *testingTInfo) func(*testing.T) { settings := integrations.GetSettings() coverageEnabled := settings.CodeCoverage testSkippedByITR := false + testIsNew := true // Check if the test is going to be skipped by ITR if settings.ItrEnabled && settings.TestsSkipping { @@ -182,6 +184,15 @@ func (ddm *M) executeInternalTest(testInfo *testingTInfo) func(*testing.T) { } } + // Check if the test is known + if settings.KnownTestsEnabled { + testIsKnown, testKnownDataOk := isKnownTest(&testInfo.commonInfo) + testIsNew = testKnownDataOk && !testIsKnown + } else { + // We don't mark any test as new if the feature is disabled + testIsNew = false + } + // Instrument the test function instrumentedFunc := func(t *testing.T) { // Set this func as a helper func of t @@ -204,7 +215,8 @@ func (ddm *M) executeInternalTest(testInfo *testingTInfo) func(*testing.T) { // Set the CI Visibility test to the execution metadata execMeta.test = test - // If the execution is for a new test we tag the test event from early flake detection + // If the execution is for a new test we tag the test event as new + execMeta.isANewTest = execMeta.isANewTest || testIsNew if execMeta.isANewTest { // Set the is new test tag test.SetTag(constants.TestIsNew, "true") @@ -214,6 +226,15 @@ func (ddm *M) executeInternalTest(testInfo *testingTInfo) func(*testing.T) { if execMeta.isARetry { // Set the retry tag test.SetTag(constants.TestIsRetry, "true") + + // If the execution is an EFD execution we tag the test event reason + if execMeta.isEFDExecution { + // Set the EFD as the retry reason + test.SetTag(constants.TestRetryReason, "efd") + } else if execMeta.isATRExecution { + // Set the ATR as the retry reason + test.SetTag(constants.TestRetryReason, "atr") + } } // Check if the test needs to be skipped by ITR @@ -385,6 +406,19 @@ func (ddm *M) instrumentInternalBenchmarks(internalBenchmarks *[]testing.Interna // executeInternalBenchmark wraps the original benchmark function to include CI visibility instrumentation. func (ddm *M) executeInternalBenchmark(benchmarkInfo *testingBInfo) func(*testing.B) { originalFunc := runtime.FuncForPC(reflect.Indirect(reflect.ValueOf(benchmarkInfo.originalFunc)).Pointer()) + + settings := integrations.GetSettings() + testIsNew := true + + // Check if the test is known + if settings.KnownTestsEnabled { + testIsKnown, testKnownDataOk := isKnownTest(&benchmarkInfo.commonInfo) + testIsNew = testKnownDataOk && !testIsKnown + } else { + // We don't mark any test as new if the feature is disabled + testIsNew = false + } + instrumentedInternalFunc := func(b *testing.B) { // decrement level @@ -399,6 +433,12 @@ func (ddm *M) executeInternalBenchmark(benchmarkInfo *testingBInfo) func(*testin test := suite.CreateTest(benchmarkInfo.testName, integrations.WithTestStartTime(startTime)) test.SetTestFunc(originalFunc) + // If the execution is for a new test we tag the test event as new + if testIsNew { + // Set the is new test tag + test.SetTag(constants.TestIsNew, "true") + } + // Run the original benchmark function. var iPfOfB *benchmarkPrivateFields var recoverFunc *func(r any) @@ -528,3 +568,20 @@ func checkModuleAndSuite(module integrations.TestModule, suite integrations.Test module.Close() } } + +// isKnownTest checks if a test is a known test or a new one +func isKnownTest(testInfo *commonInfo) (isKnown bool, hasKnownData bool) { + knownTestsData := integrations.GetKnownTests() + if knownTestsData != nil && len(knownTestsData.Tests) > 0 { + // Check if the test is a known test or a new one + if knownSuites, ok := knownTestsData.Tests[testInfo.moduleName]; ok { + if knownTests, ok := knownSuites[testInfo.suiteName]; ok { + return slices.Contains(knownTests, testInfo.testName), true + } + } + + return false, true + } + + return false, false +} diff --git a/internal/civisibility/utils/net/client.go b/internal/civisibility/utils/net/client.go index 19cf446dbf..6473ca1d5f 100644 --- a/internal/civisibility/utils/net/client.go +++ b/internal/civisibility/utils/net/client.go @@ -38,7 +38,7 @@ type ( // Client is an interface for sending requests to the Datadog backend. Client interface { GetSettings() (*SettingsResponseData, error) - GetEarlyFlakeDetectionData() (*EfdResponseData, error) + GetKnownTests() (*KnownTestsResponseData, error) GetCommits(localCommits []string) ([]string, error) SendPackFiles(commitSha string, packFiles []string) (bytes int64, err error) SendCoveragePayload(ciTestCovPayload io.Reader) error diff --git a/internal/civisibility/utils/net/efd_api.go b/internal/civisibility/utils/net/efd_api.go deleted file mode 100644 index ef2fa83d43..0000000000 --- a/internal/civisibility/utils/net/efd_api.go +++ /dev/null @@ -1,116 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2024 Datadog, Inc. - -package net - -import ( - "fmt" - "time" - - "github.com/DataDog/dd-trace-go/v2/internal/civisibility/utils/telemetry" -) - -const ( - efdRequestType string = "ci_app_libraries_tests_request" - efdURLPath string = "api/v2/ci/libraries/tests" -) - -type ( - efdRequest struct { - Data efdRequestHeader `json:"data"` - } - - efdRequestHeader struct { - ID string `json:"id"` - Type string `json:"type"` - Attributes EfdRequestData `json:"attributes"` - } - - EfdRequestData struct { - Service string `json:"service"` - Env string `json:"env"` - RepositoryURL string `json:"repository_url"` - Configurations testConfigurations `json:"configurations"` - } - - efdResponse struct { - Data struct { - ID string `json:"id"` - Type string `json:"type"` - Attributes EfdResponseData `json:"attributes"` - } `json:"data"` - } - - EfdResponseData struct { - Tests EfdResponseDataModules `json:"tests"` - } - - EfdResponseDataModules map[string]EfdResponseDataSuites - EfdResponseDataSuites map[string][]string -) - -func (c *client) GetEarlyFlakeDetectionData() (*EfdResponseData, error) { - if c.repositoryURL == "" || c.commitSha == "" { - return nil, fmt.Errorf("civisibility.GetEarlyFlakeDetectionData: repository URL and commit SHA are required") - } - - body := efdRequest{ - Data: efdRequestHeader{ - ID: c.id, - Type: efdRequestType, - Attributes: EfdRequestData{ - Service: c.serviceName, - Env: c.environment, - RepositoryURL: c.repositoryURL, - Configurations: c.testConfigurations, - }, - }, - } - - request := c.getPostRequestConfig(efdURLPath, body) - if request.Compressed { - telemetry.EarlyFlakeDetectionRequest(telemetry.CompressedRequestCompressedType) - } else { - telemetry.EarlyFlakeDetectionRequest(telemetry.UncompressedRequestCompressedType) - } - - startTime := time.Now() - response, err := c.handler.SendRequest(*request) - telemetry.EarlyFlakeDetectionRequestMs(float64(time.Since(startTime).Milliseconds())) - - if err != nil { - telemetry.EarlyFlakeDetectionRequestErrors(telemetry.NetworkErrorType) - return nil, fmt.Errorf("sending early flake detection request: %s", err.Error()) - } - - if response.StatusCode < 200 || response.StatusCode >= 300 { - telemetry.EarlyFlakeDetectionRequestErrors(telemetry.GetErrorTypeFromStatusCode(response.StatusCode)) - } - if response.Compressed { - telemetry.EarlyFlakeDetectionResponseBytes(telemetry.CompressedResponseCompressedType, float64(len(response.Body))) - } else { - telemetry.EarlyFlakeDetectionResponseBytes(telemetry.UncompressedResponseCompressedType, float64(len(response.Body))) - } - - var responseObject efdResponse - err = response.Unmarshal(&responseObject) - if err != nil { - return nil, fmt.Errorf("unmarshalling early flake detection data response: %s", err.Error()) - } - - testCount := 0 - if responseObject.Data.Attributes.Tests != nil { - for _, suites := range responseObject.Data.Attributes.Tests { - if suites == nil { - continue - } - for _, tests := range suites { - testCount += len(tests) - } - } - } - telemetry.EarlyFlakeDetectionResponseTests(float64(testCount)) - return &responseObject.Data.Attributes, nil -} diff --git a/internal/civisibility/utils/net/known_tests_api.go b/internal/civisibility/utils/net/known_tests_api.go new file mode 100644 index 0000000000..f6b4c58173 --- /dev/null +++ b/internal/civisibility/utils/net/known_tests_api.go @@ -0,0 +1,116 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024 Datadog, Inc. + +package net + +import ( + "fmt" + "time" + + "github.com/DataDog/dd-trace-go/v2/internal/civisibility/utils/telemetry" +) + +const ( + knownTestsRequestType string = "ci_app_libraries_tests_request" + knownTestsURLPath string = "api/v2/ci/libraries/tests" +) + +type ( + knownTestsRequest struct { + Data knownTestsRequestHeader `json:"data"` + } + + knownTestsRequestHeader struct { + ID string `json:"id"` + Type string `json:"type"` + Attributes KnownTestsRequestData `json:"attributes"` + } + + KnownTestsRequestData struct { + Service string `json:"service"` + Env string `json:"env"` + RepositoryURL string `json:"repository_url"` + Configurations testConfigurations `json:"configurations"` + } + + knownTestsResponse struct { + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Attributes KnownTestsResponseData `json:"attributes"` + } `json:"data"` + } + + KnownTestsResponseData struct { + Tests KnownTestsResponseDataModules `json:"tests"` + } + + KnownTestsResponseDataModules map[string]KnownTestsResponseDataSuites + KnownTestsResponseDataSuites map[string][]string +) + +func (c *client) GetKnownTests() (*KnownTestsResponseData, error) { + if c.repositoryURL == "" || c.commitSha == "" { + return nil, fmt.Errorf("civisibility.GetKnownTests: repository URL and commit SHA are required") + } + + body := knownTestsRequest{ + Data: knownTestsRequestHeader{ + ID: c.id, + Type: knownTestsRequestType, + Attributes: KnownTestsRequestData{ + Service: c.serviceName, + Env: c.environment, + RepositoryURL: c.repositoryURL, + Configurations: c.testConfigurations, + }, + }, + } + + request := c.getPostRequestConfig(knownTestsURLPath, body) + if request.Compressed { + telemetry.KnownTestsRequest(telemetry.CompressedRequestCompressedType) + } else { + telemetry.KnownTestsRequest(telemetry.UncompressedRequestCompressedType) + } + + startTime := time.Now() + response, err := c.handler.SendRequest(*request) + telemetry.KnownTestsRequestMs(float64(time.Since(startTime).Milliseconds())) + + if err != nil { + telemetry.KnownTestsRequestErrors(telemetry.NetworkErrorType) + return nil, fmt.Errorf("sending known tests request: %s", err.Error()) + } + + if response.StatusCode < 200 || response.StatusCode >= 300 { + telemetry.KnownTestsRequestErrors(telemetry.GetErrorTypeFromStatusCode(response.StatusCode)) + } + if response.Compressed { + telemetry.KnownTestsResponseBytes(telemetry.CompressedResponseCompressedType, float64(len(response.Body))) + } else { + telemetry.KnownTestsResponseBytes(telemetry.UncompressedResponseCompressedType, float64(len(response.Body))) + } + + var responseObject knownTestsResponse + err = response.Unmarshal(&responseObject) + if err != nil { + return nil, fmt.Errorf("unmarshalling known tests response: %s", err.Error()) + } + + testCount := 0 + if responseObject.Data.Attributes.Tests != nil { + for _, suites := range responseObject.Data.Attributes.Tests { + if suites == nil { + continue + } + for _, tests := range suites { + testCount += len(tests) + } + } + } + telemetry.KnownTestsResponseTests(float64(testCount)) + return &responseObject.Data.Attributes, nil +} diff --git a/internal/civisibility/utils/net/efd_api_test.go b/internal/civisibility/utils/net/known_tests_api_test.go similarity index 77% rename from internal/civisibility/utils/net/efd_api_test.go rename to internal/civisibility/utils/net/known_tests_api_test.go index 93008d25ce..79c894883f 100644 --- a/internal/civisibility/utils/net/efd_api_test.go +++ b/internal/civisibility/utils/net/known_tests_api_test.go @@ -16,15 +16,15 @@ import ( "github.com/stretchr/testify/assert" ) -func TestEfdApiRequest(t *testing.T) { +func TestKnownTestsApiRequest(t *testing.T) { var c *client - expectedResponse := efdResponse{} + expectedResponse := knownTestsResponse{} expectedResponse.Data.Type = settingsRequestType - expectedResponse.Data.Attributes.Tests = EfdResponseDataModules{ - "MyModule1": EfdResponseDataSuites{ + expectedResponse.Data.Attributes.Tests = KnownTestsResponseDataModules{ + "MyModule1": KnownTestsResponseDataSuites{ "MySuite1": []string{"Test1", "Test2"}, }, - "MyModule2": EfdResponseDataSuites{ + "MyModule2": KnownTestsResponseDataSuites{ "MySuite2": []string{"Test3", "Test4"}, }, } @@ -37,11 +37,11 @@ func TestEfdApiRequest(t *testing.T) { } if r.Header.Get(HeaderContentType) == ContentTypeJSON { - var request efdRequest + var request knownTestsRequest json.Unmarshal(body, &request) assert.Equal(t, c.id, request.Data.ID) - assert.Equal(t, efdRequestType, request.Data.Type) - assert.Equal(t, efdURLPath, r.URL.Path[1:]) + assert.Equal(t, knownTestsRequestType, request.Data.Type) + assert.Equal(t, knownTestsURLPath, r.URL.Path[1:]) assert.Equal(t, c.environment, request.Data.Attributes.Env) assert.Equal(t, c.repositoryURL, request.Data.Attributes.RepositoryURL) assert.Equal(t, c.serviceName, request.Data.Attributes.Service) @@ -62,12 +62,12 @@ func TestEfdApiRequest(t *testing.T) { cInterface := NewClient() c = cInterface.(*client) - efdData, err := cInterface.GetEarlyFlakeDetectionData() + efdData, err := cInterface.GetKnownTests() assert.Nil(t, err) assert.Equal(t, expectedResponse.Data.Attributes, *efdData) } -func TestEfdApiRequestFailToUnmarshal(t *testing.T) { +func TestKnownTestsApiRequestFailToUnmarshal(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "failed to read body", http.StatusBadRequest) })) @@ -80,13 +80,13 @@ func TestEfdApiRequestFailToUnmarshal(t *testing.T) { setCiVisibilityEnv(path, server.URL) cInterface := NewClient() - efdData, err := cInterface.GetEarlyFlakeDetectionData() + efdData, err := cInterface.GetKnownTests() assert.Nil(t, efdData) assert.NotNil(t, err) assert.Contains(t, err.Error(), "cannot unmarshal response") } -func TestEfdApiRequestFailToGet(t *testing.T) { +func TestKnownTestsApiRequestFailToGet(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "internal processing error", http.StatusInternalServerError) })) @@ -99,8 +99,8 @@ func TestEfdApiRequestFailToGet(t *testing.T) { setCiVisibilityEnv(path, server.URL) cInterface := NewClient() - efdData, err := cInterface.GetEarlyFlakeDetectionData() + efdData, err := cInterface.GetKnownTests() assert.Nil(t, efdData) assert.NotNil(t, err) - assert.Contains(t, err.Error(), "sending early flake detection request") + assert.Contains(t, err.Error(), "sending known tests request") } diff --git a/internal/civisibility/utils/net/settings_api.go b/internal/civisibility/utils/net/settings_api.go index 48800b9d33..c6fc68e3b4 100644 --- a/internal/civisibility/utils/net/settings_api.go +++ b/internal/civisibility/utils/net/settings_api.go @@ -62,6 +62,7 @@ type ( ItrEnabled bool `json:"itr_enabled"` RequireGit bool `json:"require_git"` TestsSkipping bool `json:"tests_skipping"` + KnownTestsEnabled bool `json:"known_tests_enabled"` } ) diff --git a/internal/civisibility/utils/telemetry/telemetry_count.go b/internal/civisibility/utils/telemetry/telemetry_count.go index c184cd5016..c71cb2d75b 100644 --- a/internal/civisibility/utils/telemetry/telemetry_count.go +++ b/internal/civisibility/utils/telemetry/telemetry_count.go @@ -199,14 +199,14 @@ func CodeCoverageErrors() { telemetry.GlobalClient.Count(telemetry.NamespaceCiVisibility, "code_coverage.errors", 1.0, nil, true) } -// EarlyFlakeDetectionRequest the number of requests sent to the early flake detection endpoint, tagged by the request compressed type. -func EarlyFlakeDetectionRequest(requestCompressed RequestCompressedType) { - telemetry.GlobalClient.Count(telemetry.NamespaceCiVisibility, "early_flake_detection.request", 1.0, removeEmptyStrings([]string{ +// KnownTestsRequest the number of requests sent to the known tests endpoint, tagged by the request compressed type. +func KnownTestsRequest(requestCompressed RequestCompressedType) { + telemetry.GlobalClient.Count(telemetry.NamespaceCiVisibility, "known_tests.request", 1.0, removeEmptyStrings([]string{ string(requestCompressed), }), true) } -// EarlyFlakeDetectionRequestErrors the number of requests sent to the early flake detection endpoint that errored, tagged by the error type. -func EarlyFlakeDetectionRequestErrors(errorType ErrorType) { - telemetry.GlobalClient.Count(telemetry.NamespaceCiVisibility, "early_flake_detection.request_errors", 1.0, removeEmptyStrings(errorType), true) +// KnownTestsRequestErrors the number of requests sent to the known tests endpoint that errored, tagged by the error type. +func KnownTestsRequestErrors(errorType ErrorType) { + telemetry.GlobalClient.Count(telemetry.NamespaceCiVisibility, "known_tests.request_errors", 1.0, removeEmptyStrings(errorType), true) } diff --git a/internal/civisibility/utils/telemetry/telemetry_distribution.go b/internal/civisibility/utils/telemetry/telemetry_distribution.go index 3bab710a6b..5a7d1d1209 100644 --- a/internal/civisibility/utils/telemetry/telemetry_distribution.go +++ b/internal/civisibility/utils/telemetry/telemetry_distribution.go @@ -86,19 +86,19 @@ func CodeCoverageFiles(value float64) { telemetry.GlobalClient.Record(telemetry.NamespaceCiVisibility, telemetry.MetricKindDist, "code_coverage.files", value, nil, true) } -// EarlyFlakeDetectionRequestMs records the time it takes to get the response of the early flake detection endpoint request in ms by CI Visibility. -func EarlyFlakeDetectionRequestMs(value float64) { - telemetry.GlobalClient.Record(telemetry.NamespaceCiVisibility, telemetry.MetricKindDist, "early_flake_detection.request_ms", value, nil, true) +// KnownTestsRequestMs records the time it takes to get the response of the known tests endpoint request in ms by CI Visibility. +func KnownTestsRequestMs(value float64) { + telemetry.GlobalClient.Record(telemetry.NamespaceCiVisibility, telemetry.MetricKindDist, "known_tests.request_ms", value, nil, true) } -// EarlyFlakeDetectionResponseBytes records the number of bytes received by the endpoint. Tagged with a boolean flag set to true if response body is compressed. -func EarlyFlakeDetectionResponseBytes(responseCompressedType ResponseCompressedType, value float64) { - telemetry.GlobalClient.Record(telemetry.NamespaceCiVisibility, telemetry.MetricKindDist, "early_flake_detection.response_bytes", value, removeEmptyStrings([]string{ +// KnownTestsResponseBytes records the number of bytes received by the endpoint. Tagged with a boolean flag set to true if response body is compressed. +func KnownTestsResponseBytes(responseCompressedType ResponseCompressedType, value float64) { + telemetry.GlobalClient.Record(telemetry.NamespaceCiVisibility, telemetry.MetricKindDist, "known_tests.response_bytes", value, removeEmptyStrings([]string{ (string)(responseCompressedType), }), true) } -// EarlyFlakeDetectionResponseTests records the number of tests in the response of the early flake detection endpoint by CI Visibility. -func EarlyFlakeDetectionResponseTests(value float64) { - telemetry.GlobalClient.Record(telemetry.NamespaceCiVisibility, telemetry.MetricKindDist, "early_flake_detection.response_tests", value, nil, true) +// KnownTestsResponseTests records the number of tests in the response of the known tests endpoint by CI Visibility. +func KnownTestsResponseTests(value float64) { + telemetry.GlobalClient.Record(telemetry.NamespaceCiVisibility, telemetry.MetricKindDist, "known_tests.response_tests", value, nil, true) }