Skip to content

Commit

Permalink
(feat) internal/civisibility: add Known Tests feature and refactor EF…
Browse files Browse the repository at this point in the history 
…D logic
  • Loading branch information
@tonyredondo
tonyredondo committed Feb 3, 2025
1 parent 2d92d3e commit 7265b80
Show file tree
Hide file tree
Showing 6 changed files with 198 additions and 96 deletions.
3 changes: 3 additions & 0 deletions internal/civisibility/constants/test_tags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ const (
// This constant is used to tag test events that are part of a retry execution
TestIsRetry = "test.is_retry"

// TestRetryReason indicates the reason for retrying the test
TestRetryReason = "test.retry_reason"

// TestEarlyFlakeDetectionRetryAborted indicates a retry abort reason by the early flake detection feature
TestEarlyFlakeDetectionRetryAborted = "test.early_flake.abort_reason"

Expand Down
5 changes: 5 additions & 0 deletions internal/civisibility/integrations/civisibility_features.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,11 @@ func ensureAdditionalFeaturesInitialization(serviceName string) {
ciVisibilityKnownTests = *ciEfdData
log.Debug("civisibility: known tests data loaded.")
}
} else {
// "known_tests_enabled" parameter works as a kill-switch for EFD, so if “known_tests_enabled” is false it
// will disable EFD even if “early_flake_detection.enabled” is set to true (which should not happen normally,
// the backend should disable both of them in that case)
ciVisibilitySettings.EarlyFlakeDetection.Enabled = false
}

// if flaky test retries is enabled then let's load the flaky retries settings
Expand Down
175 changes: 86 additions & 89 deletions internal/civisibility/integrations/gotesting/instrumentation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"fmt"
"reflect"
"runtime"
"slices"
"sync"
"sync/atomic"
"testing"
Expand All @@ -36,7 +35,9 @@ type (
panicData any // panic data recovered from an internal test execution when using an additional feature wrapper
panicStacktrace string // stacktrace from the panic recovered from an internal test
isARetry bool // flag to tag if a current test execution is a retry
isANewTest bool // flag to tag if a current test execution is part of a new test (EFD not known test)
isANewTest bool // flag to tag if a current test execution is part of a new test
isEFDExecution bool // flag to tag if a current test execution is part of an EFD execution
isATRExecution bool // flag to tag if a current test execution is part of an ATR execution
hasAdditionalFeatureWrapper bool // flag to check if the current execution is part of an additional feature wrapper
}

Expand Down Expand Up @@ -234,7 +235,10 @@ func applyFlakyTestRetriesAdditionalFeature(targetFunc func(*testing.T)) (func(*
}
}
},
execMetaAdjust: nil, // No execMetaAdjust needed
execMetaAdjust: func(execMeta *testExecutionMetadata, executionIndex int) {
// Set the flag ATR execution to true
execMeta.isATRExecution = true
},
})
}, true
}
Expand All @@ -243,95 +247,82 @@ func applyFlakyTestRetriesAdditionalFeature(targetFunc func(*testing.T)) (func(*

// applyEarlyFlakeDetectionAdditionalFeature applies the early flake detection feature as a wrapper of a func(*testing.T)
func applyEarlyFlakeDetectionAdditionalFeature(testInfo *commonInfo, targetFunc func(*testing.T), settings *net.SettingsResponseData) (func(*testing.T), bool) {
knownTestsData := integrations.GetKnownTests()
if knownTestsData != nil &&
len(knownTestsData.Tests) > 0 {

// Define is a known test flag
isAKnownTest := false

// Check if the test is a known test or a new one
if knownSuites, ok := knownTestsData.Tests[testInfo.moduleName]; ok {
if knownTests, ok := knownSuites[testInfo.suiteName]; ok {
if slices.Contains(knownTests, testInfo.testName) {
isAKnownTest = true
}
}
}
isKnown, hasKnownData := isKnownTest(testInfo)
if !hasKnownData || isKnown {
return targetFunc, false
}

// If it's a new test, then we apply the EFD wrapper
if !isAKnownTest {
return func(t *testing.T) {
var testPassCount, testSkipCount, testFailCount int

runTestWithRetry(&runTestWithRetryOptions{
targetFunc: targetFunc,
t: t,
initialRetryCount: 0,
adjustRetryCount: func(duration time.Duration) int64 {
slowTestRetriesSettings := settings.EarlyFlakeDetection.SlowTestRetries
durationSecs := duration.Seconds()
if durationSecs < 5 {
return int64(slowTestRetriesSettings.FiveS)
} else if durationSecs < 10 {
return int64(slowTestRetriesSettings.TenS)
} else if durationSecs < 30 {
return int64(slowTestRetriesSettings.ThirtyS)
} else if duration.Minutes() < 5 {
return int64(slowTestRetriesSettings.FiveM)
}
return 0
},
shouldRetry: func(ptrToLocalT *testing.T, executionIndex int, remainingRetries int64) bool {
return remainingRetries >= 0
},
perExecution: func(ptrToLocalT *testing.T, executionIndex int, duration time.Duration) {
// Collect test results
if ptrToLocalT.Failed() {
testFailCount++
} else if ptrToLocalT.Skipped() {
testSkipCount++
} else {
testPassCount++
}
},
onRetryEnd: func(t *testing.T, executionIndex int, lastPtrToLocalT *testing.T) {
// Update test status based on collected counts
tCommonPrivates := getTestPrivateFields(t)
if tCommonPrivates == nil {
panic("getting test private fields failed")
}
status := "passed"
if testPassCount == 0 {
if testSkipCount > 0 {
status = "skipped"
tCommonPrivates.SetSkipped(true)
}
if testFailCount > 0 {
status = "failed"
tCommonPrivates.SetFailed(true)
tParentCommonPrivates := getTestParentPrivateFields(t)
if tParentCommonPrivates == nil {
panic("getting test parent private fields failed")
}
tParentCommonPrivates.SetFailed(true)
}
// If it's a new test, then we apply the EFD wrapper
return func(t *testing.T) {
var testPassCount, testSkipCount, testFailCount int

runTestWithRetry(&runTestWithRetryOptions{
targetFunc: targetFunc,
t: t,
initialRetryCount: 0,
adjustRetryCount: func(duration time.Duration) int64 {
slowTestRetriesSettings := settings.EarlyFlakeDetection.SlowTestRetries
durationSecs := duration.Seconds()
if durationSecs < 5 {
return int64(slowTestRetriesSettings.FiveS)
} else if durationSecs < 10 {
return int64(slowTestRetriesSettings.TenS)
} else if durationSecs < 30 {
return int64(slowTestRetriesSettings.ThirtyS)
} else if duration.Minutes() < 5 {
return int64(slowTestRetriesSettings.FiveM)
}
return 0
},
shouldRetry: func(ptrToLocalT *testing.T, executionIndex int, remainingRetries int64) bool {
return remainingRetries >= 0
},
perExecution: func(ptrToLocalT *testing.T, executionIndex int, duration time.Duration) {
// Collect test results
if ptrToLocalT.Failed() {
testFailCount++
} else if ptrToLocalT.Skipped() {
testSkipCount++
} else {
testPassCount++
}
},
onRetryEnd: func(t *testing.T, executionIndex int, lastPtrToLocalT *testing.T) {
// Update test status based on collected counts
tCommonPrivates := getTestPrivateFields(t)
if tCommonPrivates == nil {
panic("getting test private fields failed")
}
status := "passed"
if testPassCount == 0 {
if testSkipCount > 0 {
status = "skipped"
tCommonPrivates.SetSkipped(true)
}
if testFailCount > 0 {
status = "failed"
tCommonPrivates.SetFailed(true)
tParentCommonPrivates := getTestParentPrivateFields(t)
if tParentCommonPrivates == nil {
panic("getting test parent private fields failed")
}
tParentCommonPrivates.SetFailed(true)
}
}

// Print summary after retries
if executionIndex > 0 {
fmt.Printf(" [ %v after %v retries by Datadog's early flake detection ]\n", status, executionIndex)
}
},
execMetaAdjust: func(execMeta *testExecutionMetadata, executionIndex int) {
// Set the flag new test to true
execMeta.isANewTest = true
},
})
}, true
}
}
return targetFunc, false
// Print summary after retries
if executionIndex > 0 {
fmt.Printf(" [ %v after %v retries by Datadog's early flake detection ]\n", status, executionIndex)
}
},
execMetaAdjust: func(execMeta *testExecutionMetadata, executionIndex int) {
// Set the flag new test to true
execMeta.isANewTest = true
// Set the flag EFD execution to true
execMeta.isEFDExecution = true
},
})
}, true
}

// runTestWithRetry encapsulates the common retry logic for test functions.
Expand Down Expand Up @@ -386,6 +377,12 @@ func runTestWithRetry(options *runTestWithRetryOptions) {
if originalExecMeta.isARetry {
execMeta.isARetry = true
}
if originalExecMeta.isEFDExecution {
execMeta.isEFDExecution = true
}
if originalExecMeta.isATRExecution {
execMeta.isATRExecution = true
}
}

// If we are in a retry execution, set the `isARetry` flag
Expand Down
15 changes: 15 additions & 0 deletions internal/civisibility/integrations/gotesting/instrumentation_orchestrion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,12 @@ func instrumentTestingTFunc(f func(*testing.T)) func(*testing.T) {
if parentExecMeta.isARetry {
execMeta.isARetry = true
}
if parentExecMeta.isEFDExecution {
execMeta.isEFDExecution = true
}
if parentExecMeta.isATRExecution {
execMeta.isATRExecution = true
}
}
}

Expand All @@ -186,6 +192,15 @@ func instrumentTestingTFunc(f func(*testing.T)) func(*testing.T) {
if execMeta.isARetry {
// Set the retry tag
test.SetTag(constants.TestIsRetry, "true")

// If the execution is an EFD execution we tag the test event reason
if execMeta.isEFDExecution {
// Set the EFD as the retry reason
test.SetTag(constants.TestRetryReason, "efd")
} else if execMeta.isATRExecution {
// Set the ATR as the retry reason
test.SetTag(constants.TestRetryReason, "atr")
}
}

defer func() {
Expand Down
37 changes: 31 additions & 6 deletions internal/civisibility/integrations/gotesting/testcontroller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,19 @@ func TestMain(m *testing.M) {

func runFlakyTestRetriesTests(m *testing.M) {
// mock the settings api to enable automatic test retries
server := setUpHttpServer(true, false, nil, false, nil)
server := setUpHttpServer(true, true, false, &net.KnownTestsResponseData{
Tests: net.KnownTestsResponseDataModules{
"gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting": net.KnownTestsResponseDataSuites{
"reflections_test.go": []string{
"TestGetFieldPointerFrom",
"TestGetInternalTestArray",
"TestGetInternalBenchmarkArray",
"TestCommonPrivateFields_AddLevel",
"TestGetBenchmarkPrivateFields",
},
},
},
}, false, nil)
defer server.Close()

// set a custom retry count
Expand Down Expand Up @@ -137,6 +149,13 @@ func runFlakyTestRetriesTests(m *testing.M) {

// check spans by tag
checkSpansByTagName(finishedSpans, constants.TestIsRetry, 6)
trrSpan := checkSpansByTagName(finishedSpans, constants.TestRetryReason, 6)[0]
if trrSpan.Tag(constants.TestRetryReason) != "atr" {
panic(fmt.Sprintf("expected retry reason to be %s, got %s", "atr", trrSpan.Tag(constants.TestRetryReason)))
}

// check the test is new tag
checkSpansByTagName(finishedSpans, constants.TestIsNew, 22)

// check spans by type
checkSpansByType(finishedSpans,
Expand All @@ -153,7 +172,7 @@ func runFlakyTestRetriesTests(m *testing.M) {

func runEarlyFlakyTestDetectionTests(m *testing.M) {
// mock the settings api to enable automatic test retries
server := setUpHttpServer(false, true, &net.KnownTestsResponseData{
server := setUpHttpServer(false, true, true, &net.KnownTestsResponseData{
Tests: net.KnownTestsResponseDataModules{
"gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting": net.KnownTestsResponseDataSuites{
"reflections_test.go": []string{
Expand Down Expand Up @@ -227,6 +246,10 @@ func runEarlyFlakyTestDetectionTests(m *testing.M) {
// check spans by tag
checkSpansByTagName(finishedSpans, constants.TestIsNew, 176)
checkSpansByTagName(finishedSpans, constants.TestIsRetry, 160)
trrSpan := checkSpansByTagName(finishedSpans, constants.TestRetryReason, 160)[0]
if trrSpan.Tag(constants.TestRetryReason) != "efd" {
panic(fmt.Sprintf("expected retry reason to be %s, got %s", "efd", trrSpan.Tag(constants.TestRetryReason)))
}

// check spans by type
checkSpansByType(finishedSpans,
Expand All @@ -243,7 +266,7 @@ func runEarlyFlakyTestDetectionTests(m *testing.M) {

func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {
// mock the settings api to enable automatic test retries
server := setUpHttpServer(true, true, &net.KnownTestsResponseData{
server := setUpHttpServer(true, true, true, &net.KnownTestsResponseData{
Tests: net.KnownTestsResponseDataModules{
"gopkg.in/DataDog/dd-trace-go.v1/internal/civisibility/integrations/gotesting": net.KnownTestsResponseDataSuites{
"reflections_test.go": []string{
Expand Down Expand Up @@ -355,7 +378,7 @@ func runFlakyTestRetriesWithEarlyFlakyTestDetectionTests(m *testing.M) {

func runIntelligentTestRunnerTests(m *testing.M) {
// mock the settings api to enable automatic test retries
server := setUpHttpServer(true, false, nil, true, []net.SkippableResponseDataAttributes{
server := setUpHttpServer(true, true, false, nil, true, []net.SkippableResponseDataAttributes{
{
Suite: "testing_test.go",
Name: "TestMyTest01",
Expand Down Expand Up @@ -569,8 +592,10 @@ type (
)

func setUpHttpServer(flakyRetriesEnabled bool,
knownTestsEnabled bool,
earlyFlakyDetectionEnabled bool, earlyFlakyDetectionData *net.KnownTestsResponseData,
itrEnabled bool, itrData []net.SkippableResponseDataAttributes) *httptest.Server {
enableKnownTests := knownTestsEnabled || earlyFlakyDetectionEnabled
// mock the settings api to enable automatic test retries
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Printf("MockApi received request: %s\n", r.URL.Path)
Expand All @@ -591,7 +616,7 @@ func setUpHttpServer(flakyRetriesEnabled bool,
FlakyTestRetriesEnabled: flakyRetriesEnabled,
ItrEnabled: itrEnabled,
TestsSkipping: itrEnabled,
KnownTestsEnabled: earlyFlakyDetectionEnabled,
KnownTestsEnabled: enableKnownTests,
}
response.Data.Attributes.EarlyFlakeDetection.Enabled = earlyFlakyDetectionEnabled
response.Data.Attributes.EarlyFlakeDetection.SlowTestRetries.FiveS = 10
Expand All @@ -601,7 +626,7 @@ func setUpHttpServer(flakyRetriesEnabled bool,

fmt.Printf("MockApi sending response: %v\n", response)
json.NewEncoder(w).Encode(&response)
} else if earlyFlakyDetectionEnabled && r.URL.Path == "/api/v2/ci/libraries/tests" {
} else if enableKnownTests && r.URL.Path == "/api/v2/ci/libraries/tests" {
w.Header().Set("Content-Type", "application/json")
response := struct {
Data struct {
Expand Down
Loading

0 comments on commit 7265b80

Please sign in to comment.