diff --git a/integration-tests/ci-visibility-intake.js b/integration-tests/ci-visibility-intake.js index e00cd6c4fc3..f4b9d443be2 100644 --- a/integration-tests/ci-visibility-intake.js +++ b/integration-tests/ci-visibility-intake.js @@ -12,31 +12,43 @@ const DEFAULT_SETTINGS = { code_coverage: true, tests_skipping: true, itr_enabled: true, + require_git: false, early_flake_detection: { enabled: false, slow_test_retries: { '5s': 3 } + }, + flaky_test_retries_enabled: false, + di_enabled: false, + known_tests_enabled: false, + test_management: { + enabled: false } } const DEFAULT_SUITES_TO_SKIP = [] const DEFAULT_GIT_UPLOAD_STATUS = 200 -const DEFAULT_KNOWN_TESTS_UPLOAD_STATUS = 200 +const DEFAULT_KNOWN_TESTS_RESPONSE_STATUS = 200 const DEFAULT_INFO_RESPONSE = { endpoints: ['/evp_proxy/v2', '/debugger/v1/input'] } const DEFAULT_CORRELATION_ID = '1234' const DEFAULT_KNOWN_TESTS = ['test-suite1.js.test-name1', 'test-suite2.js.test-name2'] +const DEFAULT_QUARANTINED_TESTS = {} +const DEFAULT_QUARANTINED_TESTS_RESPONSE_STATUS = 200 + let settings = DEFAULT_SETTINGS let suitesToSkip = DEFAULT_SUITES_TO_SKIP let gitUploadStatus = DEFAULT_GIT_UPLOAD_STATUS let infoResponse = DEFAULT_INFO_RESPONSE let correlationId = DEFAULT_CORRELATION_ID let knownTests = DEFAULT_KNOWN_TESTS -let knownTestsStatusCode = DEFAULT_KNOWN_TESTS_UPLOAD_STATUS +let knownTestsStatusCode = DEFAULT_KNOWN_TESTS_RESPONSE_STATUS let waitingTime = 0 +let quarantineResponse = DEFAULT_QUARANTINED_TESTS +let quarantineResponseStatusCode = DEFAULT_QUARANTINED_TESTS_RESPONSE_STATUS class FakeCiVisIntake extends FakeAgent { setKnownTestsResponseCode (statusCode) { @@ -71,6 +83,14 @@ class FakeCiVisIntake extends FakeAgent { waitingTime = newWaitingTime } + setQuarantinedTests (newQuarantinedTests) { + quarantineResponse = newQuarantinedTests + } + + setQuarantinedTestsResponseCode (newStatusCode) { + quarantineResponseStatusCode = newStatusCode + } + async start () { const app = express() app.use(bodyParser.raw({ limit: Infinity, type: 'application/msgpack' })) @@ -219,6 +239,25 @@ class FakeCiVisIntake extends FakeAgent { }) }) + app.post([ + '/api/v2/test/libraries/test-management/tests', + '/evp_proxy/:version/api/v2/test/libraries/test-management/tests' + ], (req, res) => { + res.setHeader('content-type', 'application/json') + const data = JSON.stringify({ + data: { + attributes: { + modules: quarantineResponse + } + } + }) + res.status(quarantineResponseStatusCode).send(data) + this.emit('message', { + headers: req.headers, + url: req.url + }) + }) + return new Promise((resolve, reject) => { const timeoutObj = setTimeout(() => { reject(new Error('Intake timed out starting up')) @@ -237,8 +276,10 @@ class FakeCiVisIntake extends FakeAgent { settings = DEFAULT_SETTINGS suitesToSkip = DEFAULT_SUITES_TO_SKIP gitUploadStatus = DEFAULT_GIT_UPLOAD_STATUS - knownTestsStatusCode = DEFAULT_KNOWN_TESTS_UPLOAD_STATUS + knownTestsStatusCode = DEFAULT_KNOWN_TESTS_RESPONSE_STATUS infoResponse = DEFAULT_INFO_RESPONSE + quarantineResponseStatusCode = DEFAULT_QUARANTINED_TESTS_RESPONSE_STATUS + quarantineResponse = DEFAULT_QUARANTINED_TESTS this.removeAllListeners() if (this.waitingTimeoutId) { clearTimeout(this.waitingTimeoutId) diff --git a/integration-tests/ci-visibility/features-quarantine/quarantine.feature b/integration-tests/ci-visibility/features-quarantine/quarantine.feature new file mode 100644 index 00000000000..d837149878a --- /dev/null +++ b/integration-tests/ci-visibility/features-quarantine/quarantine.feature @@ -0,0 +1,4 @@ +Feature: Quarantine + Scenario: Say quarantine + When the greeter says quarantine + Then I should have heard "quarantine" diff --git a/integration-tests/ci-visibility/features-quarantine/support/steps.js b/integration-tests/ci-visibility/features-quarantine/support/steps.js new file mode 100644 index 00000000000..86b1a3aa9b6 --- /dev/null +++ b/integration-tests/ci-visibility/features-quarantine/support/steps.js @@ -0,0 +1,10 @@ +const assert = require('assert') +const { When, Then } = require('@cucumber/cucumber') + +Then('I should have heard {string}', function (expectedResponse) { + assert.equal(this.whatIHeard, 'fail') +}) + +When('the greeter says quarantine', function () { + this.whatIHeard = 'quarantine' +}) diff --git a/integration-tests/ci-visibility/playwright-tests-quarantine/quarantine-test.js b/integration-tests/ci-visibility/playwright-tests-quarantine/quarantine-test.js new file mode 100644 index 00000000000..69287e98ecb --- /dev/null +++ b/integration-tests/ci-visibility/playwright-tests-quarantine/quarantine-test.js @@ -0,0 +1,13 @@ +const { test, expect } = require('@playwright/test') + +test.beforeEach(async ({ page }) => { + await page.goto(process.env.PW_BASE_URL) +}) + +test.describe('quarantine', () => { + test('should quarantine failed test', async ({ page }) => { + await expect(page.locator('.hello-world')).toHaveText([ + 'Hello Warld' + ]) + }) +}) diff --git a/integration-tests/ci-visibility/quarantine/test-quarantine-1.js b/integration-tests/ci-visibility/quarantine/test-quarantine-1.js new file mode 100644 index 00000000000..c75cb4c5b75 --- /dev/null +++ b/integration-tests/ci-visibility/quarantine/test-quarantine-1.js @@ -0,0 +1,11 @@ +const { expect } = require('chai') + +describe('quarantine tests', () => { + it('can quarantine a test', () => { + expect(1 + 2).to.equal(4) + }) + + it('can pass normally', () => { + expect(1 + 2).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/quarantine/test-quarantine-2.js b/integration-tests/ci-visibility/quarantine/test-quarantine-2.js new file mode 100644 index 00000000000..f94386f1b87 --- /dev/null +++ b/integration-tests/ci-visibility/quarantine/test-quarantine-2.js @@ -0,0 +1,11 @@ +const { expect } = require('chai') + +describe('quarantine tests 2', () => { + it('can quarantine a test', () => { + expect(1 + 2).to.equal(3) + }) + + it('can pass normally', () => { + expect(1 + 2).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/run-jest.js b/integration-tests/ci-visibility/run-jest.js index a1f236be7a2..0a8b7b47ce6 100644 --- a/integration-tests/ci-visibility/run-jest.js +++ b/integration-tests/ci-visibility/run-jest.js @@ -31,8 +31,12 @@ if (process.env.COLLECT_COVERAGE_FROM) { jest.runCLI( options, options.projects -).then(() => { +).then((results) => { if (process.send) { process.send('finished') } + if (process.env.SHOULD_CHECK_RESULTS) { + const exitCode = results.results.success ? 0 : 1 + process.exit(exitCode) + } }) diff --git a/integration-tests/ci-visibility/run-mocha.js b/integration-tests/ci-visibility/run-mocha.js index fc767f4051f..19d009ca9a2 100644 --- a/integration-tests/ci-visibility/run-mocha.js +++ b/integration-tests/ci-visibility/run-mocha.js @@ -12,11 +12,14 @@ if (process.env.TESTS_TO_RUN) { mocha.addFile(require.resolve('./test/ci-visibility-test.js')) mocha.addFile(require.resolve('./test/ci-visibility-test-2.js')) } -mocha.run(() => { +mocha.run((failures) => { if (process.send) { process.send('finished') } -}).on('end', () => { + if (process.env.SHOULD_CHECK_RESULTS && failures > 0) { + process.exit(1) + } +}).on('end', (res) => { // eslint-disable-next-line console.log('end event: can add event listeners to mocha') }) diff --git a/integration-tests/ci-visibility/vitest-tests/test-quarantine.mjs b/integration-tests/ci-visibility/vitest-tests/test-quarantine.mjs new file mode 100644 index 00000000000..d48e61fe64d --- /dev/null +++ b/integration-tests/ci-visibility/vitest-tests/test-quarantine.mjs @@ -0,0 +1,11 @@ +import { describe, test, expect } from 'vitest' + +describe('quarantine tests', () => { + test('can quarantine a test', () => { + expect(1 + 2).to.equal(4) + }) + + test('can pass normally', () => { + expect(1 + 2).to.equal(3) + }) +}) diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index 3dfef057fd1..aaebeb9cb41 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -43,7 +43,9 @@ const { DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, DI_DEBUG_ERROR_LINE_SUFFIX, TEST_RETRY_REASON, - DD_TEST_IS_USER_PROVIDED_SERVICE + DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_IS_QUARANTINED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -2029,5 +2031,94 @@ versions.forEach(version => { }).catch(done) }) }) + + context('quarantine', () => { + beforeEach(() => { + receiver.setQuarantinedTests({ + cucumber: { + suites: { + 'ci-visibility/features-quarantine/quarantine.feature': { + tests: { + 'Say quarantine': { + properties: { + quarantined: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isQuarantining) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const failedTest = events.find(event => event.type === 'test').content + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isQuarantining) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + assert.equal(failedTest.resource, 'ci-visibility/features-quarantine/quarantine.feature.Say quarantine') + + assert.equal(failedTest.meta[TEST_STATUS], 'fail') + if (isQuarantining) { + assert.propertyVal(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } else { + assert.notProperty(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED) + } + }) + + const runTest = (done, isQuarantining, extraEnvVars) => { + const testAssertionsPromise = getTestAssertions(isQuarantining) + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-quarantine/*.feature', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + ...extraEnvVars + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', exitCode => { + testAssertionsPromise.then(() => { + if (isQuarantining) { + // even though a test fails, the exit code is 1 because the test is quarantined + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can quarantine tests', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runTest(done, true) + }) + + it('fails if quarantine is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false } }) + + runTest(done, false) + }) + + it('does not enable quarantine tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runTest(done, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + }) }) }) diff --git a/integration-tests/cypress-esm-config.mjs b/integration-tests/cypress-esm-config.mjs index d6f4c2b8e95..1f27d834070 100644 --- a/integration-tests/cypress-esm-config.mjs +++ b/integration-tests/cypress-esm-config.mjs @@ -1,7 +1,7 @@ import cypress from 'cypress' async function runCypress () { - await cypress.run({ + const results = await cypress.run({ config: { defaultCommandTimeout: 1000, e2e: { @@ -39,6 +39,9 @@ async function runCypress () { screenshotOnRunFailure: false } }) + if (results.totalFailed !== 0) { + process.exit(1) + } } runCypress() diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index a2dd81a74f5..f5e071ace60 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -38,7 +38,9 @@ const { TEST_SESSION_NAME, TEST_LEVEL_EVENT_TYPES, TEST_RETRY_REASON, - DD_TEST_IS_USER_PROVIDED_SERVICE + DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_IS_QUARANTINED, + TEST_MANAGEMENT_ENABLED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -1731,5 +1733,104 @@ moduleTypes.forEach(({ }).catch(done) }) }) + + context('quarantine', () => { + beforeEach(() => { + receiver.setQuarantinedTests({ + cypress: { + suites: { + 'cypress/e2e/quarantine.js': { + tests: { + 'quarantine is quarantined': { + properties: { + quarantined: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isQuarantining) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const failedTest = events.find(event => event.type === 'test').content + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isQuarantining) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + assert.equal(failedTest.resource, 'cypress/e2e/quarantine.js.quarantine is quarantined') + + if (isQuarantining) { + // TODO: run instead of skipping, but ignore its result + assert.propertyVal(failedTest.meta, TEST_STATUS, 'skip') + assert.propertyVal(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } else { + assert.propertyVal(failedTest.meta, TEST_STATUS, 'fail') + assert.notProperty(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED) + } + }) + + const runQuarantineTest = (done, isQuarantining, extraEnvVars) => { + const testAssertionsPromise = getTestAssertions(isQuarantining) + + const { + NODE_OPTIONS, + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + const specToRun = 'cypress/e2e/quarantine.js' + + childProcess = exec( + version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}`, + SPEC_PATTERN: specToRun, + ...extraEnvVars + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + if (isQuarantining) { + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can quarantine tests', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest(done, true) + }) + + it('fails if quarantine is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false } }) + + runQuarantineTest(done, false) + }) + + it('does not enable quarantine tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest(done, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + }) }) }) diff --git a/integration-tests/cypress/e2e/quarantine.js b/integration-tests/cypress/e2e/quarantine.js new file mode 100644 index 00000000000..efbae41cd64 --- /dev/null +++ b/integration-tests/cypress/e2e/quarantine.js @@ -0,0 +1,8 @@ +/* eslint-disable */ +describe('quarantine', () => { + it('is quarantined', () => { + cy.visit('/') + .get('.hello-world') + .should('have.text', 'Hello Warld') + }) +}) diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js index 35413ea7e60..489aaa228ff 100644 --- a/integration-tests/jest/jest.spec.js +++ b/integration-tests/jest/jest.spec.js @@ -40,7 +40,9 @@ const { DI_DEBUG_ERROR_FILE_SUFFIX, DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, DI_DEBUG_ERROR_LINE_SUFFIX, - DD_TEST_IS_USER_PROVIDED_SERVICE + DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_IS_QUARANTINED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -2938,4 +2940,130 @@ describe('jest CommonJS', () => { }).catch(done) }) }) + + context('quarantine', () => { + beforeEach(() => { + receiver.setQuarantinedTests({ + jest: { + suites: { + 'ci-visibility/quarantine/test-quarantine-1.js': { + tests: { + 'quarantine tests can quarantine a test': { + properties: { + quarantined: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isQuarantining, isParallel) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isQuarantining) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/quarantine/test-quarantine-1.js.quarantine tests can quarantine a test', + 'ci-visibility/quarantine/test-quarantine-1.js.quarantine tests can pass normally' + ] + ) + + if (isParallel) { + // Parallel mode in jest requires more than a single test suite + // Here we check that the second test suite is actually running, so we can be sure that parallel mode is on + assert.includeMembers(resourceNames, [ + 'ci-visibility/quarantine/test-quarantine-2.js.quarantine tests 2 can quarantine a test', + 'ci-visibility/quarantine/test-quarantine-2.js.quarantine tests 2 can pass normally' + ]) + } + + const failedTest = tests.find( + test => test.meta[TEST_NAME] === 'quarantine tests can quarantine a test' + ) + assert.equal(failedTest.meta[TEST_STATUS], 'fail') + + if (isQuarantining) { + assert.propertyVal(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } else { + assert.notProperty(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED) + } + }) + + const runQuarantineTest = (done, isQuarantining, extraEnvVars = {}, isParallel = false) => { + const testAssertionsPromise = getTestAssertions(isQuarantining, isParallel) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: 'quarantine/test-quarantine-1', + SHOULD_CHECK_RESULTS: '1', + ...extraEnvVars + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', exitCode => { + testAssertionsPromise.then(() => { + if (isQuarantining) { + // even though a test fails, the exit code is 1 because the test is quarantined + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can quarantine tests', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest(done, true) + }) + + it('fails if quarantine is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false } }) + + runQuarantineTest(done, false) + }) + + it('does not enable quarantine tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest(done, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + + it('can quarantine in parallel mode', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest( + done, + true, + { + // we need to run more than 1 suite for parallel mode to kick in + TESTS_TO_RUN: 'quarantine/test-quarantine', + RUN_IN_PARALLEL: true + }, + true + ) + }) + }) }) diff --git a/integration-tests/mocha/mocha.spec.js b/integration-tests/mocha/mocha.spec.js index 86d9491b3f0..8593d438f09 100644 --- a/integration-tests/mocha/mocha.spec.js +++ b/integration-tests/mocha/mocha.spec.js @@ -42,7 +42,9 @@ const { DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, DI_DEBUG_ERROR_LINE_SUFFIX, TEST_RETRY_REASON, - DD_TEST_IS_USER_PROVIDED_SERVICE + DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_IS_QUARANTINED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -2557,4 +2559,108 @@ describe('mocha CommonJS', function () { }).catch(done) }) }) + + context('quarantine', () => { + beforeEach(() => { + receiver.setQuarantinedTests({ + mocha: { + suites: { + 'ci-visibility/quarantine/test-quarantine-1.js': { + tests: { + 'quarantine tests can quarantine a test': { + properties: { + quarantined: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isQuarantining) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isQuarantining) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/quarantine/test-quarantine-1.js.quarantine tests can quarantine a test', + 'ci-visibility/quarantine/test-quarantine-1.js.quarantine tests can pass normally' + ] + ) + + const failedTest = tests.find( + test => test.meta[TEST_NAME] === 'quarantine tests can quarantine a test' + ) + // The test fails but the exit code is 0 if it's quarantined + assert.equal(failedTest.meta[TEST_STATUS], 'fail') + + if (isQuarantining) { + assert.propertyVal(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } else { + assert.notProperty(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED) + } + }) + + const runQuarantineTest = (done, isQuarantining, extraEnvVars = {}) => { + const testAssertionsPromise = getTestAssertions(isQuarantining) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TESTS_TO_RUN: JSON.stringify([ + './quarantine/test-quarantine-1.js' + ]), + SHOULD_CHECK_RESULTS: '1', + ...extraEnvVars + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + if (isQuarantining) { + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can quarantine tests', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest(done, true) + }) + + it('fails if quarantine is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false } }) + + runQuarantineTest(done, false) + }) + + it('does not enable quarantine tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest(done, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + }) }) diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index ee3a05182ad..023381978ee 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -27,7 +27,9 @@ const { TEST_SESSION_NAME, TEST_LEVEL_EVENT_TYPES, TEST_RETRY_REASON, - DD_TEST_IS_USER_PROVIDED_SERVICE + DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_IS_QUARANTINED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') @@ -886,5 +888,98 @@ versions.forEach((version) => { receiverPromise.then(() => done()).catch(done) }) }) + + if (version === 'latest') { + context('quarantine', () => { + beforeEach(() => { + receiver.setQuarantinedTests({ + playwright: { + suites: { + 'quarantine-test.js': { + tests: { + 'quarantine should quarantine failed test': { + properties: { + quarantined: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isQuarantining) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + if (isQuarantining) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const failedTest = events.find(event => event.type === 'test').content + + if (isQuarantining) { + // TODO: manage to run the test + assert.equal(failedTest.meta[TEST_STATUS], 'skip') + assert.propertyVal(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } else { + assert.equal(failedTest.meta[TEST_STATUS], 'fail') + assert.notProperty(failedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED) + } + }) + + const runQuarantineTest = (done, isQuarantining, extraEnvVars) => { + const testAssertionsPromise = getTestAssertions(isQuarantining) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + TEST_DIR: './ci-visibility/playwright-tests-quarantine', + ...extraEnvVars + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + if (isQuarantining) { + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }).catch(done) + }) + } + + it('can quarantine tests', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest(done, true) + }) + + it('fails if quarantine is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false } }) + + runQuarantineTest(done, false) + }) + + it('does not enable quarantine tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest(done, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + }) + } }) }) diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index 56f060ce509..33565590f1b 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -31,7 +31,9 @@ const { DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX, DI_DEBUG_ERROR_LINE_SUFFIX, TEST_RETRY_REASON, - DD_TEST_IS_USER_PROVIDED_SERVICE + DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_IS_QUARANTINED } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') @@ -1334,5 +1336,113 @@ versions.forEach((version) => { }).catch(done) }) }) + + if (version === 'latest') { + context('quarantine', () => { + beforeEach(() => { + receiver.setQuarantinedTests({ + vitest: { + suites: { + 'ci-visibility/vitest-tests/test-quarantine.mjs': { + tests: { + 'quarantine tests can quarantine a test': { + properties: { + quarantined: true + } + } + } + } + } + } + }) + }) + + const getTestAssertions = (isQuarantining) => + receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + + const testSession = events.find(event => event.type === 'test_session_end').content + + if (isQuarantining) { + assert.propertyVal(testSession.meta, TEST_MANAGEMENT_ENABLED, 'true') + } else { + assert.notProperty(testSession.meta, TEST_MANAGEMENT_ENABLED) + } + + const resourceNames = tests.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + 'ci-visibility/vitest-tests/test-quarantine.mjs.quarantine tests can quarantine a test', + 'ci-visibility/vitest-tests/test-quarantine.mjs.quarantine tests can pass normally' + ] + ) + + const quarantinedTest = tests.find( + test => test.meta[TEST_NAME] === 'quarantine tests can quarantine a test' + ) + + if (isQuarantining) { + // TODO: do not flip the status of the test but still ignore failures + assert.equal(quarantinedTest.meta[TEST_STATUS], 'pass') + assert.propertyVal(quarantinedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } else { + assert.equal(quarantinedTest.meta[TEST_STATUS], 'fail') + assert.notProperty(quarantinedTest.meta, TEST_MANAGEMENT_IS_QUARANTINED) + } + }) + + const runQuarantineTest = (done, isQuarantining, extraEnvVars = {}) => { + const testAssertionsPromise = getTestAssertions(isQuarantining) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + TEST_DIR: 'ci-visibility/vitest-tests/test-quarantine*', + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init --no-warnings', + ...extraEnvVars + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', (exitCode) => { + testAssertionsPromise.then(() => { + if (isQuarantining) { + // exit code 0 even though one of the tests failed + assert.equal(exitCode, 0) + } else { + assert.equal(exitCode, 1) + } + done() + }) + }) + } + + it('can quarantine tests', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest(done, true) + }) + + it('fails if quarantine is not enabled', (done) => { + receiver.setSettings({ test_management: { enabled: false } }) + + runQuarantineTest(done, false) + }) + + it('does not enable quarantine tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => { + receiver.setSettings({ test_management: { enabled: true } }) + + runQuarantineTest(done, false, { DD_TEST_MANAGEMENT_ENABLED: '0' }) + }) + }) + } }) }) diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index 639f955cc56..a97b8842938 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -22,6 +22,7 @@ const knownTestsCh = channel('ci:cucumber:known-tests') const skippableSuitesCh = channel('ci:cucumber:test-suite:skippable') const sessionStartCh = channel('ci:cucumber:session:start') const sessionFinishCh = channel('ci:cucumber:session:finish') +const quarantinedTestsCh = channel('ci:cucumber:quarantined-tests') const workerReportTraceCh = channel('ci:cucumber:worker-report:trace') @@ -71,6 +72,8 @@ let earlyFlakeDetectionFaultyThreshold = 0 let isEarlyFlakeDetectionFaulty = false let isFlakyTestRetriesEnabled = false let isKnownTestsEnabled = false +let isQuarantinedTestsEnabled = false +let quarantinedTests = {} let numTestRetries = 0 let knownTests = [] let skippedSuites = [] @@ -117,6 +120,17 @@ function isNewTest (testSuite, testName) { return !testsForSuite.includes(testName) } +function isQuarantinedTest (testSuite, testName) { + return quarantinedTests + ?.cucumber + ?.suites + ?.[testSuite] + ?.tests + ?.[testName] + ?.properties + ?.quarantined +} + function getTestStatusFromRetries (testStatuses) { if (testStatuses.every(status => status === 'fail')) { return 'fail' @@ -293,12 +307,17 @@ function wrapRun (pl, isLatestVersion) { } let isNew = false let isEfdRetry = false + let isQuarantined = false if (isKnownTestsEnabled && status !== 'skip') { const numRetries = numRetriesByPickleId.get(this.pickle.id) isNew = numRetries !== undefined isEfdRetry = numRetries > 0 } + if (isQuarantinedTestsEnabled) { + const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd()) + isQuarantined = isQuarantinedTest(testSuitePath, this.pickle.name) + } const attemptAsyncResource = numAttemptToAsyncResource.get(numAttempt) const error = getErrorFromCucumberResult(result) @@ -307,7 +326,15 @@ function wrapRun (pl, isLatestVersion) { await promises.hitBreakpointPromise } attemptAsyncResource.runInAsyncScope(() => { - testFinishCh.publish({ status, skipReason, error, isNew, isEfdRetry, isFlakyRetry: numAttempt > 0 }) + testFinishCh.publish({ + status, + skipReason, + error, + isNew, + isEfdRetry, + isFlakyRetry: numAttempt > 0, + isQuarantined + }) }) }) return promise @@ -396,6 +423,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin isFlakyTestRetriesEnabled = configurationResponse.libraryConfig?.isFlakyTestRetriesEnabled numTestRetries = configurationResponse.libraryConfig?.flakyTestRetriesCount isKnownTestsEnabled = configurationResponse.libraryConfig?.isKnownTestsEnabled + isQuarantinedTestsEnabled = configurationResponse.libraryConfig?.isQuarantinedTestsEnabled if (isKnownTestsEnabled) { const knownTestsResponse = await getChannelPromise(knownTestsCh) @@ -453,6 +481,15 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin } } + if (isQuarantinedTestsEnabled) { + const quarantinedTestsResponse = await getChannelPromise(quarantinedTestsCh) + if (!quarantinedTestsResponse.err) { + quarantinedTests = quarantinedTestsResponse.quarantinedTests + } else { + isQuarantinedTestsEnabled = false + } + } + const processArgv = process.argv.slice(2).join(' ') const command = process.env.npm_lifecycle_script || `cucumber-js ${processArgv}` @@ -500,6 +537,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin hasForcedToRunSuites: isForcedToRun, isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, + isQuarantinedTestsEnabled, isParallel }) }) @@ -536,6 +574,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } let isNew = false + let isQuarantined = false if (isKnownTestsEnabled) { isNew = isNewTest(testSuitePath, pickle.name) @@ -543,6 +582,10 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa numRetriesByPickleId.set(pickle.id, 0) } } + if (isQuarantinedTestsEnabled) { + isQuarantined = isQuarantinedTest(testSuitePath, pickle.name) + } + // TODO: for >=11 we could use `runTestCaseResult` instead of accumulating results in `lastStatusByPickleId` let runTestCaseResult = await runTestCaseFunction.apply(this, arguments) @@ -557,6 +600,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } let testStatus = lastTestStatus let shouldBePassedByEFD = false + let shouldBePassedByQuarantine = false if (isNew && isEarlyFlakeDetectionEnabled) { /** * If Early Flake Detection (EFD) is enabled the logic is as follows: @@ -573,6 +617,11 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa } } + if (isQuarantinedTestsEnabled && isQuarantined) { + this.success = true + shouldBePassedByQuarantine = true + } + if (!pickleResultByFile[testFileAbsolutePath]) { pickleResultByFile[testFileAbsolutePath] = [testStatus] } else { @@ -604,6 +653,10 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa return shouldBePassedByEFD } + if (isNewerCucumberVersion && isQuarantinedTestsEnabled && isQuarantined) { + return shouldBePassedByQuarantine + } + return runTestCaseResult } } diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index d4f01cf7e5d..da31b18e6d1 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -43,6 +43,7 @@ const testErrCh = channel('ci:jest:test:err') const skippableSuitesCh = channel('ci:jest:test-suite:skippable') const libraryConfigurationCh = channel('ci:jest:library-configuration') const knownTestsCh = channel('ci:jest:known-tests') +const quarantinedTestsCh = channel('ci:jest:quarantined-tests') const itrSkippedSuitesCh = channel('ci:jest:itr:skipped-suites') @@ -70,6 +71,8 @@ let earlyFlakeDetectionFaultyThreshold = 30 let isEarlyFlakeDetectionFaulty = false let hasFilteredSkippableSuites = false let isKnownTestsEnabled = false +let isQuarantinedTestsEnabled = false +let quarantinedTests = {} const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -140,6 +143,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { this.flakyTestRetriesCount = this.testEnvironmentOptions._ddFlakyTestRetriesCount this.isDiEnabled = this.testEnvironmentOptions._ddIsDiEnabled this.isKnownTestsEnabled = this.testEnvironmentOptions._ddIsKnownTestsEnabled + this.isQuarantinedTestsEnabled = this.testEnvironmentOptions._ddIsQuarantinedTestsEnabled if (this.isKnownTestsEnabled) { try { @@ -161,6 +165,18 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { this.global[RETRY_TIMES] = this.flakyTestRetriesCount } } + + if (this.isQuarantinedTestsEnabled) { + try { + const hasQuarantinedTests = !!quarantinedTests.jest + this.quarantinedTestsForThisSuite = hasQuarantinedTests + ? this.getQuarantinedTestsForSuite(quarantinedTests.jest.suites[this.testSuite].tests) + : this.getQuarantinedTestsForSuite(this.testEnvironmentOptions._ddQuarantinedTests) + } catch (e) { + log.error('Error parsing quarantined tests', e) + this.isQuarantinedTestsEnabled = false + } + } } getHasSnapshotTests () { @@ -193,8 +209,25 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { return knownTestsForSuite } + getQuarantinedTestsForSuite (quaratinedTests) { + if (this.quarantinedTestsForThisSuite) { + return this.quarantinedTestsForThisSuite + } + let quarantinedTestsForSuite = quaratinedTests + // If jest is using workers, quarantined tests are serialized to json. + // If jest runs in band, they are not. + if (typeof quarantinedTestsForSuite === 'string') { + quarantinedTestsForSuite = JSON.parse(quarantinedTestsForSuite) + } + return Object.entries(quarantinedTestsForSuite).reduce((acc, [testName, { properties }]) => { + if (properties?.quarantined) { + acc.push(testName) + } + return acc + }, []) + } + // Add the `add_test` event we don't have the test object yet, so - // we use its describe block to get the full name getTestNameFromAddTestEvent (event, state) { const describeSuffix = getJestTestName(state.currentDescribeBlock) const fullTestName = describeSuffix ? `${describeSuffix} ${event.testName}` : event.testName @@ -303,6 +336,12 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } } } + let isQuarantined = false + + if (this.isQuarantinedTestsEnabled) { + const testName = getJestTestName(event.test) + isQuarantined = this.quarantinedTestsForThisSuite?.includes(testName) + } const promises = {} const numRetries = this.global[RETRY_TIMES] @@ -337,7 +376,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { testFinishCh.publish({ status, testStartLine: getTestLineStart(event.test.asyncError, this.testSuite), - promises + isQuarantined }) }) @@ -485,6 +524,7 @@ function cliWrapper (cli, jestVersion) { earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled + isQuarantinedTestsEnabled = libraryConfig.isQuarantinedTestsEnabled } } catch (err) { log.error('Jest library configuration error', err) @@ -532,6 +572,25 @@ function cliWrapper (cli, jestVersion) { } } + if (isQuarantinedTestsEnabled) { + const quarantinedTestsPromise = new Promise((resolve) => { + onDone = resolve + }) + + sessionAsyncResource.runInAsyncScope(() => { + quarantinedTestsCh.publish({ onDone }) + }) + + try { + const { err, quarantinedTests: receivedQuarantinedTests } = await quarantinedTestsPromise + if (!err) { + quarantinedTests = receivedQuarantinedTests + } + } catch (err) { + log.error('Jest quarantined tests error', err) + } + } + const processArgv = process.argv.slice(2).join(' ') sessionAsyncResource.runInAsyncScope(() => { testSessionStartCh.publish({ command: `jest ${processArgv}`, frameworkVersion: jestVersion }) @@ -601,6 +660,7 @@ function cliWrapper (cli, jestVersion) { error, isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, + isQuarantinedTestsEnabled, onDone }) }) @@ -634,6 +694,37 @@ function cliWrapper (cli, jestVersion) { } } + if (isQuarantinedTestsEnabled) { + const failedTests = result + .results + .testResults.flatMap(({ testResults, testFilePath: testSuiteAbsolutePath }) => ( + testResults.map(({ fullName: testName, status }) => ({ testName, testSuiteAbsolutePath, status })) + )) + .filter(({ status }) => status === 'failed') + + let numFailedQuarantinedTests = 0 + + for (const { testName, testSuiteAbsolutePath } of failedTests) { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, result.globalConfig.rootDir) + const isQuarantined = quarantinedTests + ?.jest + ?.suites + ?.[testSuite] + ?.tests + ?.[testName] + ?.properties + ?.quarantined + if (isQuarantined) { + numFailedQuarantinedTests++ + } + } + + // If every test that failed was quarantined, we'll consider the suite passed + if (numFailedQuarantinedTests !== 0 && result.results.numFailedTests === numFailedQuarantinedTests) { + result.results.success = true + } + } + return result }) @@ -825,6 +916,8 @@ addHook({ _ddFlakyTestRetriesCount, _ddIsDiEnabled, _ddIsKnownTestsEnabled, + _ddIsQuarantinedTestsEnabled, + _ddQuarantinedTests, ...restOfTestEnvironmentOptions } = testEnvironmentOptions @@ -936,8 +1029,9 @@ addHook({ }) /* -* This hook does two things: +* This hook does three things: * - Pass known tests to the workers. +* - Pass quarantined tests to the workers. * - Receive trace, coverage and logs payloads from the workers. */ addHook({ @@ -947,7 +1041,7 @@ addHook({ }, (childProcessWorker) => { const ChildProcessWorker = childProcessWorker.default shimmer.wrap(ChildProcessWorker.prototype, 'send', send => function (request) { - if (!isKnownTestsEnabled) { + if (!isKnownTestsEnabled && !isQuarantinedTestsEnabled) { return send.apply(this, arguments) } const [type] = request @@ -967,11 +1061,15 @@ addHook({ const [{ globalConfig, config, path: testSuiteAbsolutePath }] = args const testSuite = getTestSuitePath(testSuiteAbsolutePath, globalConfig.rootDir || process.cwd()) const suiteKnownTests = knownTests.jest?.[testSuite] || [] + + const suiteQuarantinedTests = quarantinedTests.jest?.suites?.[testSuite]?.tests || {} + args[0].config = { ...config, testEnvironmentOptions: { ...config.testEnvironmentOptions, - _ddKnownTests: suiteKnownTests + _ddKnownTests: suiteKnownTests, + _ddQuarantinedTests: suiteQuarantinedTests } } } diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index afa7bfe0fc4..143935da3fb 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -27,6 +27,7 @@ const { getOnPendingHandler, testFileToSuiteAr, newTests, + testsQuarantined, getTestFullName, getRunTestsWrapper } = require('./utils') @@ -61,6 +62,7 @@ const testSuiteCodeCoverageCh = channel('ci:mocha:test-suite:code-coverage') const libraryConfigurationCh = channel('ci:mocha:library-configuration') const knownTestsCh = channel('ci:mocha:known-tests') const skippableSuitesCh = channel('ci:mocha:test-suite:skippable') +const quarantinedTestsCh = channel('ci:mocha:quarantined-tests') const workerReportTraceCh = channel('ci:mocha:worker-report:trace') const testSessionStartCh = channel('ci:mocha:session:start') const testSessionFinishCh = channel('ci:mocha:session:finish') @@ -135,6 +137,18 @@ function getOnEndHandler (isParallel) { } } + // We subtract the errors from quarantined tests from the total number of failures + if (config.isQuarantinedTestsEnabled) { + let numFailedQuarantinedTests = 0 + for (const test of testsQuarantined) { + if (isTestFailed(test)) { + numFailedQuarantinedTests++ + } + } + this.stats.failures -= numFailedQuarantinedTests + this.failures -= numFailedQuarantinedTests + } + if (status === 'fail') { error = new Error(`Failed tests: ${this.failures}.`) } @@ -165,6 +179,7 @@ function getOnEndHandler (isParallel) { error, isEarlyFlakeDetectionEnabled: config.isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty: config.isEarlyFlakeDetectionFaulty, + isQuarantinedTestsEnabled: config.isQuarantinedTestsEnabled, isParallel }) }) @@ -173,6 +188,22 @@ function getOnEndHandler (isParallel) { function getExecutionConfiguration (runner, isParallel, onFinishRequest) { const mochaRunAsyncResource = new AsyncResource('bound-anonymous-fn') + const onReceivedQuarantinedTests = ({ err, quarantinedTests: receivedQuarantinedTests }) => { + if (err) { + config.quarantinedTests = {} + config.isQuarantinedTestsEnabled = false + } else { + config.quarantinedTests = receivedQuarantinedTests + } + if (config.isSuitesSkippingEnabled) { + skippableSuitesCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) + }) + } else { + onFinishRequest() + } + } + const onReceivedSkippableSuites = ({ err, skippableSuites, itrCorrelationId: responseItrCorrelationId }) => { if (err) { suitesToSkip = [] @@ -205,8 +236,11 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { } else { config.knownTests = knownTests } - - if (config.isSuitesSkippingEnabled) { + if (config.isQuarantinedTestsEnabled) { + quarantinedTestsCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedQuarantinedTests) + }) + } else if (config.isSuitesSkippingEnabled) { skippableSuitesCh.publish({ onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) }) @@ -224,15 +258,20 @@ function getExecutionConfiguration (runner, isParallel, onFinishRequest) { config.earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries config.earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold config.isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled - // ITR and auto test retries are not supported in parallel mode yet + // ITR, auto test retries and quarantine are not supported in parallel mode yet config.isSuitesSkippingEnabled = !isParallel && libraryConfig.isSuitesSkippingEnabled config.isFlakyTestRetriesEnabled = !isParallel && libraryConfig.isFlakyTestRetriesEnabled config.flakyTestRetriesCount = !isParallel && libraryConfig.flakyTestRetriesCount + config.isQuarantinedTestsEnabled = !isParallel && libraryConfig.isQuarantinedTestsEnabled if (config.isKnownTestsEnabled) { knownTestsCh.publish({ onDone: mochaRunAsyncResource.bind(onReceivedKnownTests) }) + } else if (config.isQuarantinedTestsEnabled) { + quarantinedTestsCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedQuarantinedTests) + }) } else if (config.isSuitesSkippingEnabled) { skippableSuitesCh.publish({ onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) @@ -357,7 +396,7 @@ addHook({ this.once('end', getOnEndHandler(false)) - this.on('test', getOnTestHandler(true, newTests)) + this.on('test', getOnTestHandler(true)) this.on('test end', getOnTestEndHandler()) @@ -579,6 +618,7 @@ addHook({ const testPath = getTestSuitePath(testSuiteAbsolutePath, process.cwd()) const testSuiteKnownTests = config.knownTests.mocha?.[testPath] || [] + const testSuiteQuarantinedTests = config.quarantinedTests?.modules?.mocha?.suites?.[testPath] || [] // We pass the known tests for the test file to the worker const testFileResult = await run.apply( @@ -589,6 +629,8 @@ addHook({ ...workerArgs, _ddEfdNumRetries: config.earlyFlakeDetectionNumRetries, _ddIsEfdEnabled: config.isEarlyFlakeDetectionEnabled, + _ddIsQuarantinedEnabled: config.isQuarantinedTestsEnabled, + _ddQuarantinedTests: testSuiteQuarantinedTests, _ddKnownTests: { mocha: { [testPath]: testSuiteKnownTests diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index 30710ab645b..40fcbdc4ff7 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -26,6 +26,27 @@ const testToStartLine = new WeakMap() const testFileToSuiteAr = new Map() const wrappedFunctions = new WeakSet() const newTests = {} +const testsQuarantined = new Set() + +function isQuarantinedTest (test, testsToQuarantine) { + const testSuite = getTestSuitePath(test.file, process.cwd()) + const testName = test.fullTitle() + + const isQuarantined = (testsToQuarantine + .mocha + ?.suites + ?.[testSuite] + ?.tests + ?.[testName] + ?.properties + ?.quarantined) ?? false + + if (isQuarantined) { + testsQuarantined.add(test) + } + + return isQuarantined +} function isNewTest (test, knownTests) { const testSuite = getTestSuitePath(test.file, process.cwd()) @@ -171,7 +192,8 @@ function getOnTestHandler (isMain) { file: testSuiteAbsolutePath, title, _ddIsNew: isNew, - _ddIsEfdRetry: isEfdRetry + _ddIsEfdRetry: isEfdRetry, + _ddIsQuarantined: isQuarantined } = test const testInfo = { @@ -187,6 +209,7 @@ function getOnTestHandler (isMain) { testInfo.isNew = isNew testInfo.isEfdRetry = isEfdRetry + testInfo.isQuarantined = isQuarantined // We want to store the result of the new tests if (isNew) { const testFullName = getTestFullName(test) @@ -360,6 +383,15 @@ function getRunTestsWrapper (runTests, config) { } }) } + + if (config.isQuarantinedTestsEnabled) { + suite.tests.forEach(test => { + if (isQuarantinedTest(test, config.quarantinedTests)) { + test._ddIsQuarantined = true + } + }) + } + return runTests.apply(this, arguments) } } @@ -384,5 +416,6 @@ module.exports = { getOnPendingHandler, testFileToSuiteAr, getRunTestsWrapper, - newTests + newTests, + testsQuarantined } diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index 56a9dc75270..88a2b33b498 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -33,6 +33,13 @@ addHook({ delete this.options._ddIsEfdEnabled delete this.options._ddKnownTests delete this.options._ddEfdNumRetries + delete this.options._ddQuarantinedTests + } + if (this.options._ddIsQuarantinedEnabled) { + config.isQuarantinedEnabled = true + config.quarantinedTests = this.options._ddQuarantinedTests + delete this.options._ddIsQuarantinedEnabled + delete this.options._ddQuarantinedTests } return run.apply(this, arguments) }) diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index bcd389dd09f..ee219e5290c 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -13,6 +13,7 @@ const testSessionFinishCh = channel('ci:playwright:session:finish') const libraryConfigurationCh = channel('ci:playwright:library-configuration') const knownTestsCh = channel('ci:playwright:known-tests') +const quarantinedTestsCh = channel('ci:playwright:quarantined-tests') const testSuiteStartCh = channel('ci:playwright:test-suite:start') const testSuiteFinishCh = channel('ci:playwright:test-suite:finish') @@ -41,9 +42,25 @@ let earlyFlakeDetectionNumRetries = 0 let isFlakyTestRetriesEnabled = false let flakyTestRetriesCount = 0 let knownTests = {} +let isQuarantinedTestsEnabled = false +let quarantinedTests = {} let rootDir = '' const MINIMUM_SUPPORTED_VERSION_RANGE_EFD = '>=1.38.0' +function isQuarantineTest (test) { + const testName = getTestFullname(test) + const testSuite = getTestSuitePath(test._requireFile, rootDir) + + return quarantinedTests + ?.playwright + ?.suites + ?.[testSuite] + ?.tests + ?.[testName] + ?.properties + ?.quarantined +} + function isNewTest (test) { const testSuite = getTestSuitePath(test._requireFile, rootDir) const testsForSuite = knownTests?.playwright?.[testSuite] || [] @@ -296,6 +313,7 @@ function testEndHandler (test, annotations, testStatus, error, isTimeout) { error, extraTags: annotationTags, isNew: test._ddIsNew, + isQuarantined: test._ddIsQuarantined, isEfdRetry: test._ddIsEfdRetry }) }) @@ -424,10 +442,12 @@ function runnerHook (runnerExport, playwrightVersion) { earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount + isQuarantinedTestsEnabled = libraryConfig.isQuarantinedTestsEnabled } } catch (e) { isEarlyFlakeDetectionEnabled = false isKnownTestsEnabled = false + isQuarantinedTestsEnabled = false log.error('Playwright session start error', e) } @@ -447,6 +467,20 @@ function runnerHook (runnerExport, playwrightVersion) { } } + if (isQuarantinedTestsEnabled && satisfies(playwrightVersion, MINIMUM_SUPPORTED_VERSION_RANGE_EFD)) { + try { + const { err, quarantinedTests: receivedQuarantinedTests } = await getChannelPromise(quarantinedTestsCh) + if (!err) { + quarantinedTests = receivedQuarantinedTests + } else { + isQuarantinedTestsEnabled = false + } + } catch (err) { + isQuarantinedTestsEnabled = false + log.error('Playwright quarantined tests error', err) + } + } + const projects = getProjectsFromRunner(this) if (isFlakyTestRetriesEnabled && flakyTestRetriesCount > 0) { @@ -479,6 +513,7 @@ function runnerHook (runnerExport, playwrightVersion) { testSessionFinishCh.publish({ status: STATUS_TO_TEST_STATUS[sessionStatus], isEarlyFlakeDetectionEnabled, + isQuarantinedTestsEnabled, onDone }) }) @@ -487,6 +522,8 @@ function runnerHook (runnerExport, playwrightVersion) { startedSuites = [] remainingTestsByFile = {} + // TODO: we can trick playwright into thinking the session passed by returning + // 'passed' here. We might be able to use this for both EFD and Quarantined tests. return runAllTestsReturn }) @@ -557,26 +594,37 @@ addHook({ const oldCreateRootSuite = loadUtilsPackage.createRootSuite async function newCreateRootSuite () { + if (!isKnownTestsEnabled && !isQuarantinedTestsEnabled) { + return oldCreateRootSuite.apply(this, arguments) + } const rootSuite = await oldCreateRootSuite.apply(this, arguments) - if (!isKnownTestsEnabled) { - return rootSuite + + const allTests = rootSuite.allTests() + + if (isQuarantinedTestsEnabled) { + const testsToBeIgnored = allTests.filter(isQuarantineTest) + testsToBeIgnored.forEach(test => { + test._ddIsQuarantined = true + test.expectedStatus = 'skipped' + }) } - const newTests = rootSuite - .allTests() - .filter(isNewTest) - - newTests.forEach(newTest => { - newTest._ddIsNew = true - if (isEarlyFlakeDetectionEnabled && newTest.expectedStatus !== 'skipped') { - const fileSuite = getSuiteType(newTest, 'file') - const projectSuite = getSuiteType(newTest, 'project') - for (let repeatEachIndex = 0; repeatEachIndex < earlyFlakeDetectionNumRetries; repeatEachIndex++) { - const copyFileSuite = deepCloneSuite(fileSuite, isNewTest) - applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1) - projectSuite._addSuite(copyFileSuite) + + if (isKnownTestsEnabled) { + const newTests = allTests.filter(isNewTest) + + newTests.forEach(newTest => { + newTest._ddIsNew = true + if (isEarlyFlakeDetectionEnabled && newTest.expectedStatus !== 'skipped') { + const fileSuite = getSuiteType(newTest, 'file') + const projectSuite = getSuiteType(newTest, 'project') + for (let repeatEachIndex = 0; repeatEachIndex < earlyFlakeDetectionNumRetries; repeatEachIndex++) { + const copyFileSuite = deepCloneSuite(fileSuite, isNewTest) + applyRepeatEachIndex(projectSuite._fullProject, copyFileSuite, repeatEachIndex + 1) + projectSuite._addSuite(copyFileSuite) + } } - } - }) + }) + } return rootSuite } diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js index ebde98b4789..340fd188340 100644 --- a/packages/datadog-instrumentations/src/vitest.js +++ b/packages/datadog-instrumentations/src/vitest.js @@ -9,6 +9,7 @@ const testPassCh = channel('ci:vitest:test:pass') const testErrorCh = channel('ci:vitest:test:error') const testSkipCh = channel('ci:vitest:test:skip') const isNewTestCh = channel('ci:vitest:test:is-new') +const isQuarantinedCh = channel('ci:vitest:test:is-quarantined') // test suite hooks const testSuiteStartCh = channel('ci:vitest:test-suite:start') @@ -21,10 +22,12 @@ const testSessionFinishCh = channel('ci:vitest:session:finish') const libraryConfigurationCh = channel('ci:vitest:library-configuration') const knownTestsCh = channel('ci:vitest:known-tests') const isEarlyFlakeDetectionFaultyCh = channel('ci:vitest:is-early-flake-detection-faulty') +const quarantinedTestsCh = channel('ci:vitest:quarantined-tests') const taskToAsync = new WeakMap() const taskToStatuses = new WeakMap() const newTasks = new WeakSet() +const quarantinedTasks = new WeakSet() let isRetryReasonEfd = false const switchedStatuses = new WeakSet() const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') @@ -46,7 +49,9 @@ function getProvidedContext () { _ddIsDiEnabled, _ddKnownTests: knownTests, _ddEarlyFlakeDetectionNumRetries: numRepeats, - _ddIsKnownTestsEnabled: isKnownTestsEnabled + _ddIsKnownTestsEnabled: isKnownTestsEnabled, + _ddIsQuarantinedTestsEnabled: isQuarantinedTestsEnabled, + _ddQuarantinedTests: quarantinedTests } = globalThis.__vitest_worker__.providedContext return { @@ -54,7 +59,9 @@ function getProvidedContext () { isEarlyFlakeDetectionEnabled: _ddIsEarlyFlakeDetectionEnabled, knownTests, numRepeats, - isKnownTestsEnabled + isKnownTestsEnabled, + isQuarantinedTestsEnabled, + quarantinedTests } } catch (e) { log.error('Vitest workers could not parse provided context, so some features will not work.') @@ -63,7 +70,9 @@ function getProvidedContext () { isEarlyFlakeDetectionEnabled: false, knownTests: {}, numRepeats: 0, - isKnownTestsEnabled: false + isKnownTestsEnabled: false, + isQuarantinedTestsEnabled: false, + quarantinedTests: {} } } } @@ -158,8 +167,10 @@ function getSortWrapper (sort) { let earlyFlakeDetectionNumRetries = 0 let isEarlyFlakeDetectionFaulty = false let isKnownTestsEnabled = false + let isQuarantinedTestsEnabled = false let isDiEnabled = false let knownTests = {} + let quarantinedTests = {} try { const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh) @@ -170,6 +181,7 @@ function getSortWrapper (sort) { earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries isDiEnabled = libraryConfig.isDiEnabled isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled + isQuarantinedTestsEnabled = libraryConfig.isQuarantinedTestsEnabled } } catch (e) { isFlakyTestRetriesEnabled = false @@ -229,6 +241,23 @@ function getSortWrapper (sort) { } } + if (isQuarantinedTestsEnabled) { + const { err, quarantinedTests: receivedQuarantinedTests } = await getChannelPromise(quarantinedTestsCh) + if (!err) { + quarantinedTests = receivedQuarantinedTests + try { + const workspaceProject = this.ctx.getCoreWorkspaceProject() + workspaceProject._provided._ddIsQuarantinedTestsEnabled = isQuarantinedTestsEnabled + workspaceProject._provided._ddQuarantinedTests = quarantinedTests + } catch (e) { + log.warn('Could not send quarantined tests to workers so Quarantine will not work.') + } + } else { + isQuarantinedTestsEnabled = false + log.error('Could not get quarantined tests.') + } + } + let testCodeCoverageLinesTotal if (this.ctx.coverageProvider?.generateCoverage) { @@ -263,6 +292,7 @@ function getSortWrapper (sort) { error, isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, + isQuarantinedTestsEnabled, onFinish }) }) @@ -332,7 +362,7 @@ addHook({ // `onAfterRunTask` is run after all repetitions or attempts are run shimmer.wrap(VitestTestRunner.prototype, 'onAfterRunTask', onAfterRunTask => async function (task) { - const { isEarlyFlakeDetectionEnabled } = getProvidedContext() + const { isEarlyFlakeDetectionEnabled, isQuarantinedTestsEnabled } = getProvidedContext() if (isEarlyFlakeDetectionEnabled && taskToStatuses.has(task)) { const statuses = taskToStatuses.get(task) @@ -345,6 +375,12 @@ addHook({ } } + if (isQuarantinedTestsEnabled) { + if (quarantinedTasks.has(task)) { + task.result.state = 'pass' + } + } + return onAfterRunTask.apply(this, arguments) }) @@ -356,17 +392,34 @@ addHook({ } const testName = getTestName(task) let isNew = false + let isQuarantined = false const { isKnownTestsEnabled, isEarlyFlakeDetectionEnabled, - isDiEnabled + isDiEnabled, + isQuarantinedTestsEnabled, + quarantinedTests } = getProvidedContext() if (isKnownTestsEnabled) { isNew = newTasks.has(task) } + if (isQuarantinedTestsEnabled) { + isQuarantinedCh.publish({ + quarantinedTests, + testSuiteAbsolutePath: task.file.filepath, + testName, + onDone: (isTestQuarantined) => { + isQuarantined = isTestQuarantined + if (isTestQuarantined) { + quarantinedTasks.add(task) + } + } + }) + } + const { retry: numAttempt, repeats: numRepetition } = retryInfo // We finish the previous test here because we know it has failed already @@ -448,7 +501,8 @@ addHook({ isRetry: numAttempt > 0 || numRepetition > 0, isRetryReasonEfd, isNew, - mightHitProbe: isDiEnabled && numAttempt > 0 + mightHitProbe: isDiEnabled && numAttempt > 0, + isQuarantined }) }) return onBeforeTryTask.apply(this, arguments) diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index 3620e063f0f..a79601c6799 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -27,7 +27,9 @@ const { TEST_MODULE_ID, TEST_SUITE, CUCUMBER_IS_PARALLEL, - TEST_RETRY_REASON + TEST_RETRY_REASON, + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_IS_QUARANTINED } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants') @@ -84,6 +86,7 @@ class CucumberPlugin extends CiPlugin { hasForcedToRunSuites, isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, + isQuarantinedTestsEnabled, isParallel }) => { const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {} @@ -110,6 +113,9 @@ class CucumberPlugin extends CiPlugin { if (isParallel) { this.testSessionSpan.setTag(CUCUMBER_IS_PARALLEL, 'true') } + if (isQuarantinedTestsEnabled) { + this.testSessionSpan.setTag(TEST_MANAGEMENT_ENABLED, 'true') + } this.testSessionSpan.setTag(TEST_STATUS, status) this.testModuleSpan.setTag(TEST_STATUS, status) @@ -317,7 +323,8 @@ class CucumberPlugin extends CiPlugin { errorMessage, isNew, isEfdRetry, - isFlakyRetry + isFlakyRetry, + isQuarantined }) => { const span = storage('legacy').getStore().span const statusTag = isStep ? 'step.status' : TEST_STATUS @@ -346,6 +353,10 @@ class CucumberPlugin extends CiPlugin { span.setTag(TEST_IS_RETRY, 'true') } + if (isQuarantined) { + span.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } + span.finish() if (!isStep) { const spanTags = span.context()._tags diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js index 67487e47dbb..470ff290625 100644 --- a/packages/datadog-plugin-cypress/src/cypress-plugin.js +++ b/packages/datadog-plugin-cypress/src/cypress-plugin.js @@ -33,7 +33,9 @@ const { TEST_SESSION_NAME, TEST_LEVEL_EVENT_TYPES, TEST_RETRY_REASON, - DD_TEST_IS_USER_PROVIDED_SERVICE + DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_IS_QUARANTINED, + TEST_MANAGEMENT_ENABLED } = require('../../dd-trace/src/plugins/util/test') const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') const { ORIGIN_KEY, COMPONENT } = require('../../dd-trace/src/constants') @@ -152,6 +154,20 @@ function getKnownTests (tracer, testConfiguration) { }) } +function getQuarantinedTests (tracer, testConfiguration) { + return new Promise(resolve => { + if (!tracer._tracer._exporter?.getQuarantinedTests) { + return resolve({ err: new Error('Test Optimization was not initialized correctly') }) + } + tracer._tracer._exporter.getQuarantinedTests(testConfiguration, (err, quarantinedTests) => { + resolve({ + err, + quarantinedTests + }) + }) + }) +} + function getSuiteStatus (suiteStats) { if (!suiteStats) { return 'skip' @@ -240,7 +256,8 @@ class CypressPlugin { earlyFlakeDetectionNumRetries, isFlakyTestRetriesEnabled, flakyTestRetriesCount, - isKnownTestsEnabled + isKnownTestsEnabled, + isQuarantinedTestsEnabled } } = libraryConfigurationResponse this.isSuitesSkippingEnabled = isSuitesSkippingEnabled @@ -251,12 +268,24 @@ class CypressPlugin { if (isFlakyTestRetriesEnabled) { this.cypressConfig.retries.runMode = flakyTestRetriesCount } + this.isQuarantinedTestsEnabled = isQuarantinedTestsEnabled } return this.cypressConfig }) return this.libraryConfigurationPromise } + getIsQuarantinedTest (testSuite, testName) { + return this.quarantinedTests + ?.cypress + ?.suites + ?.[testSuite] + ?.tests + ?.[testName] + ?.properties + ?.quarantined + } + getTestSuiteSpan ({ testSuite, testSuiteAbsolutePath }) { const testSuiteSpanMetadata = getTestSuiteCommonTags(this.command, this.frameworkVersion, testSuite, TEST_FRAMEWORK_NAME) @@ -351,10 +380,6 @@ class CypressPlugin { }) } - isNewTest (testName, testSuite) { - return !this.knownTestsByTestSuite?.[testSuite]?.includes(testName) - } - async beforeRun (details) { // We need to make sure that the plugin is initialized before running the tests // This is for the case where the user has not returned the promise from the init function @@ -393,6 +418,19 @@ class CypressPlugin { } } + if (this.isQuarantinedTestsEnabled) { + const quarantinedTestsResponse = await getQuarantinedTests( + this.tracer, + this.testConfiguration + ) + if (quarantinedTestsResponse.err) { + log.error('Cypress quarantined tests response error', quarantinedTestsResponse.err) + this.isQuarantinedTestsEnabled = false + } else { + this.quarantinedTests = quarantinedTestsResponse.quarantinedTests + } + } + // `details.specs` are test files details.specs?.forEach(({ absolute, relative }) => { const isUnskippableSuite = isMarkedAsUnskippable({ path: absolute }) @@ -471,6 +509,10 @@ class CypressPlugin { } ) + if (this.isQuarantinedTestsEnabled) { + this.testSessionSpan.setTag(TEST_MANAGEMENT_ENABLED, 'true') + } + this.testModuleSpan.finish() this.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() @@ -545,6 +587,13 @@ class CypressPlugin { if (this.itrCorrelationId) { skippedTestSpan.setTag(ITR_CORRELATION_ID, this.itrCorrelationId) } + + const isQuarantined = this.getIsQuarantinedTest(spec.relative, cypressTestName) + + if (isQuarantined) { + skippedTestSpan.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } + skippedTestSpan.finish() }) @@ -648,6 +697,7 @@ class CypressPlugin { }) const isUnskippable = this.unskippableSuites.includes(testSuite) const isForcedToRun = shouldSkip && isUnskippable + const isQuarantined = this.getIsQuarantinedTest(testSuite, testName) // skip test if (shouldSkip && !isUnskippable) { @@ -656,6 +706,12 @@ class CypressPlugin { return { shouldSkip: true } } + // TODO: I haven't found a way to trick cypress into ignoring a test + // The way we'll implement quarantine in cypress is by skipping the test altogether + if (isQuarantined) { + return { shouldSkip: true } + } + if (!this.activeTestSpan) { this.activeTestSpan = this.getTestSpan({ testName, @@ -681,7 +737,8 @@ class CypressPlugin { testSuiteAbsolutePath, testName, isNew, - isEfdRetry + isEfdRetry, + isQuarantined } = test if (coverage && this.isCodeCoverageEnabled && this.tracer._tracer._exporter?.exportCoverage) { const coverageFiles = getCoveredFilenamesFromCoverage(coverage) @@ -720,6 +777,9 @@ class CypressPlugin { this.activeTestSpan.setTag(TEST_RETRY_REASON, 'efd') } } + if (isQuarantined) { + this.activeTestSpan.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } const finishedTest = { testName, testStatus, diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index 3ec965efdbd..77e9409cebf 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -24,7 +24,9 @@ const { TEST_IS_RUM_ACTIVE, TEST_BROWSER_DRIVER, getFormattedError, - TEST_RETRY_REASON + TEST_RETRY_REASON, + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_IS_QUARANTINED } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const id = require('../../dd-trace/src/id') @@ -106,6 +108,7 @@ class JestPlugin extends CiPlugin { error, isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, + isQuarantinedTestsEnabled, onDone }) => { this.testSessionSpan.setTag(TEST_STATUS, status) @@ -137,6 +140,9 @@ class JestPlugin extends CiPlugin { if (isEarlyFlakeDetectionFaulty) { this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') } + if (isQuarantinedTestsEnabled) { + this.testSessionSpan.setTag(TEST_MANAGEMENT_ENABLED, 'true') + } this.testModuleSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') @@ -166,6 +172,7 @@ class JestPlugin extends CiPlugin { config._ddEarlyFlakeDetectionNumRetries = this.libraryConfig?.earlyFlakeDetectionNumRetries ?? 0 config._ddRepositoryRoot = this.repositoryRoot config._ddIsFlakyTestRetriesEnabled = this.libraryConfig?.isFlakyTestRetriesEnabled ?? false + config._ddIsQuarantinedTestsEnabled = this.libraryConfig?.isQuarantinedTestsEnabled ?? false config._ddFlakyTestRetriesCount = this.libraryConfig?.flakyTestRetriesCount config._ddIsDiEnabled = this.libraryConfig?.isDiEnabled ?? false config._ddIsKnownTestsEnabled = this.libraryConfig?.isKnownTestsEnabled ?? false @@ -325,12 +332,15 @@ class JestPlugin extends CiPlugin { this.activeTestSpan = span }) - this.addSub('ci:jest:test:finish', ({ status, testStartLine }) => { + this.addSub('ci:jest:test:finish', ({ status, testStartLine, isQuarantined }) => { const span = storage('legacy').getStore().span span.setTag(TEST_STATUS, status) if (testStartLine) { span.setTag(TEST_SOURCE_START, testStartLine) } + if (isQuarantined) { + span.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } const spanTags = span.context()._tags this.telemetry.ciVisEvent( diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index 7152aafe8b3..b96517e0ea5 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -31,7 +31,9 @@ const { MOCHA_IS_PARALLEL, TEST_IS_RUM_ACTIVE, TEST_BROWSER_DRIVER, - TEST_RETRY_REASON + TEST_RETRY_REASON, + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_IS_QUARANTINED } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -309,6 +311,7 @@ class MochaPlugin extends CiPlugin { error, isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, + isQuarantinedTestsEnabled, isParallel }) => { if (this.testSessionSpan) { @@ -325,6 +328,10 @@ class MochaPlugin extends CiPlugin { this.testSessionSpan.setTag(MOCHA_IS_PARALLEL, 'true') } + if (isQuarantinedTestsEnabled) { + this.testSessionSpan.setTag(TEST_MANAGEMENT_ENABLED, 'true') + } + addIntelligentTestRunnerSpanTags( this.testSessionSpan, this.testModuleSpan, @@ -397,7 +404,8 @@ class MochaPlugin extends CiPlugin { isNew, isEfdRetry, testStartLine, - isParallel + isParallel, + isQuarantined } = testInfo const testName = removeEfdStringFromTestName(testInfo.testName) @@ -416,6 +424,10 @@ class MochaPlugin extends CiPlugin { extraTags[MOCHA_IS_PARALLEL] = 'true' } + if (isQuarantined) { + extraTags[TEST_MANAGEMENT_IS_QUARANTINED] = 'true' + } + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.sourceRoot) const testSuiteSpan = this._testSuites.get(testSuite) diff --git a/packages/datadog-plugin-playwright/src/index.js b/packages/datadog-plugin-playwright/src/index.js index 5573984ef1b..f75fee37a3d 100644 --- a/packages/datadog-plugin-playwright/src/index.js +++ b/packages/datadog-plugin-playwright/src/index.js @@ -17,6 +17,8 @@ const { TEST_EARLY_FLAKE_ENABLED, TELEMETRY_TEST_SESSION, TEST_RETRY_REASON, + TEST_MANAGEMENT_IS_QUARANTINED, + TEST_MANAGEMENT_ENABLED, TEST_BROWSER_NAME } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') @@ -39,7 +41,12 @@ class PlaywrightPlugin extends CiPlugin { this.numFailedTests = 0 this.numFailedSuites = 0 - this.addSub('ci:playwright:session:finish', ({ status, isEarlyFlakeDetectionEnabled, onDone }) => { + this.addSub('ci:playwright:session:finish', ({ + status, + isEarlyFlakeDetectionEnabled, + isQuarantinedTestsEnabled, + onDone + }) => { this.testModuleSpan.setTag(TEST_STATUS, status) this.testSessionSpan.setTag(TEST_STATUS, status) @@ -57,6 +64,10 @@ class PlaywrightPlugin extends CiPlugin { this.testSessionSpan.setTag('error', error) } + if (isQuarantinedTestsEnabled) { + this.testSessionSpan.setTag(TEST_MANAGEMENT_ENABLED, 'true') + } + this.testModuleSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() @@ -129,7 +140,16 @@ class PlaywrightPlugin extends CiPlugin { this.enter(span, store) }) - this.addSub('ci:playwright:test:finish', ({ testStatus, steps, error, extraTags, isNew, isEfdRetry, isRetry }) => { + this.addSub('ci:playwright:test:finish', ({ + testStatus, + steps, + error, + extraTags, + isNew, + isEfdRetry, + isRetry, + isQuarantined + }) => { const store = storage('legacy').getStore() const span = store && store.span if (!span) return @@ -152,6 +172,9 @@ class PlaywrightPlugin extends CiPlugin { if (isRetry) { span.setTag(TEST_IS_RETRY, 'true') } + if (isQuarantined) { + span.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true') + } steps.forEach(step => { const stepStartTime = step.startTime.getTime() diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index 2aa88f2f38b..6b0b5cabf60 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -18,7 +18,9 @@ const { TEST_IS_NEW, TEST_EARLY_FLAKE_ENABLED, TEST_EARLY_FLAKE_ABORT_REASON, - TEST_RETRY_REASON + TEST_RETRY_REASON, + TEST_MANAGEMENT_ENABLED, + TEST_MANAGEMENT_IS_QUARANTINED } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -48,6 +50,20 @@ class VitestPlugin extends CiPlugin { onDone(!testsForThisTestSuite.includes(testName)) }) + this.addSub('ci:vitest:test:is-quarantined', ({ quarantinedTests, testSuiteAbsolutePath, testName, onDone }) => { + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) + const isQuarantined = quarantinedTests + ?.vitest + ?.suites + ?.[testSuite] + ?.tests + ?.[testName] + ?.properties + ?.quarantined + + onDone(isQuarantined ?? false) + }) + this.addSub('ci:vitest:is-early-flake-detection-faulty', ({ knownTests, testFilepaths, @@ -66,6 +82,7 @@ class VitestPlugin extends CiPlugin { testSuiteAbsolutePath, isRetry, isNew, + isQuarantined, mightHitProbe, isRetryReasonEfd }) => { @@ -84,6 +101,9 @@ class VitestPlugin extends CiPlugin { if (isRetryReasonEfd) { extraTags[TEST_RETRY_REASON] = 'efd' } + if (isQuarantined) { + extraTags[TEST_MANAGEMENT_IS_QUARANTINED] = 'true' + } const span = this.startTestSpan( testName, @@ -257,6 +277,7 @@ class VitestPlugin extends CiPlugin { testCodeCoverageLinesTotal, isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, + isQuarantinedTestsEnabled, onFinish }) => { this.testSessionSpan.setTag(TEST_STATUS, status) @@ -275,6 +296,9 @@ class VitestPlugin extends CiPlugin { if (isEarlyFlakeDetectionFaulty) { this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') } + if (isQuarantinedTestsEnabled) { + this.testSessionSpan.setTag(TEST_MANAGEMENT_ENABLED, 'true') + } this.testModuleSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index 3cbd64afbc2..c738ec68ff1 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -6,6 +6,7 @@ const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata' const { getLibraryConfiguration: getLibraryConfigurationRequest } = require('../requests/get-library-configuration') const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligent-test-runner/get-skippable-suites') const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests') +const { getQuarantinedTests: getQuarantinedTestsRequest } = require('../quarantined-tests/get-quarantined-tests') const log = require('../../log') const AgentInfoExporter = require('../../exporters/common/agent-info-exporter') const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags') @@ -92,6 +93,14 @@ class CiVisibilityExporter extends AgentInfoExporter { ) } + shouldRequestQuarantinedTests () { + return !!( + this._canUseCiVisProtocol && + this._config.isTestManagementEnabled && + this._libraryConfig?.isQuarantinedTestsEnabled + ) + } + shouldRequestLibraryConfiguration () { return this._config.isIntelligentTestRunnerEnabled } @@ -138,6 +147,13 @@ class CiVisibilityExporter extends AgentInfoExporter { getKnownTestsRequest(this.getRequestConfiguration(testConfiguration), callback) } + getQuarantinedTests (testConfiguration, callback) { + if (!this.shouldRequestQuarantinedTests()) { + return callback(null) + } + getQuarantinedTestsRequest(this.getRequestConfiguration(testConfiguration), callback) + } + /** * We can't request library configuration until we know whether we can use the * CI Visibility Protocol, hence the this._canUseCiVisProtocol promise. @@ -197,7 +213,8 @@ class CiVisibilityExporter extends AgentInfoExporter { earlyFlakeDetectionFaultyThreshold, isFlakyTestRetriesEnabled, isDiEnabled, - isKnownTestsEnabled + isKnownTestsEnabled, + isQuarantinedTestsEnabled } = remoteConfiguration return { isCodeCoverageEnabled, @@ -210,7 +227,8 @@ class CiVisibilityExporter extends AgentInfoExporter { isFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled && this._config.isFlakyTestRetriesEnabled, flakyTestRetriesCount: this._config.flakyTestRetriesCount, isDiEnabled: isDiEnabled && this._config.isTestDynamicInstrumentationEnabled, - isKnownTestsEnabled + isKnownTestsEnabled, + isQuarantinedTestsEnabled: isQuarantinedTestsEnabled && this._config.isTestManagementEnabled } } diff --git a/packages/dd-trace/src/ci-visibility/quarantined-tests/get-quarantined-tests.js b/packages/dd-trace/src/ci-visibility/quarantined-tests/get-quarantined-tests.js new file mode 100644 index 00000000000..bc8c40a9c22 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/quarantined-tests/get-quarantined-tests.js @@ -0,0 +1,62 @@ +const request = require('../../exporters/common/request') +const id = require('../../id') + +function getQuarantinedTests ({ + url, + isEvpProxy, + evpProxyPrefix, + isGzipCompatible, + repositoryUrl +}, done) { + const options = { + path: '/api/v2/test/libraries/test-management/tests', + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + timeout: 20000, + url + } + + if (isGzipCompatible) { + options.headers['accept-encoding'] = 'gzip' + } + + if (isEvpProxy) { + options.path = `${evpProxyPrefix}/api/v2/test/libraries/test-management/tests` + options.headers['X-Datadog-EVP-Subdomain'] = 'api' + } else { + const apiKey = process.env.DATADOG_API_KEY || process.env.DD_API_KEY + if (!apiKey) { + return done(new Error('Quarantined tests were not fetched because Datadog API key is not defined.')) + } + + options.headers['dd-api-key'] = apiKey + } + + const data = JSON.stringify({ + data: { + id: id().toString(10), + type: 'ci_app_libraries_tests_request', + attributes: { + repository_url: repositoryUrl + } + } + }) + + request(data, options, (err, res) => { + if (err) { + done(err) + } else { + try { + const { data: { attributes: { modules: quarantinedTests } } } = JSON.parse(res) + + done(null, quarantinedTests) + } catch (err) { + done(err) + } + } + }) +} + +module.exports = { getQuarantinedTests } diff --git a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js index 26d818bcdd2..707e3bb12d4 100644 --- a/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +++ b/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js @@ -94,7 +94,8 @@ function getLibraryConfiguration ({ early_flake_detection: earlyFlakeDetectionConfig, flaky_test_retries_enabled: isFlakyTestRetriesEnabled, di_enabled: isDiEnabled, - known_tests_enabled: isKnownTestsEnabled + known_tests_enabled: isKnownTestsEnabled, + test_management: testManagementConfig } } } = JSON.parse(res) @@ -111,7 +112,9 @@ function getLibraryConfiguration ({ earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD, isFlakyTestRetriesEnabled, isDiEnabled: isDiEnabled && isFlakyTestRetriesEnabled, - isKnownTestsEnabled + isKnownTestsEnabled, + // TODO: should it be test management? + isQuarantinedTestsEnabled: (testManagementConfig?.enabled ?? false) } log.debug(() => `Remote settings: ${JSON.stringify(settings)}`) diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 9c1f5b3bfe6..1a4adbcf18a 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -520,6 +520,8 @@ class Config { this._setValue(defaults, 'legacyBaggageEnabled', true) this._setValue(defaults, 'isTestDynamicInstrumentationEnabled', false) this._setValue(defaults, 'isServiceUserProvided', false) + this._setValue(defaults, 'testManagementAttemptToFixRetries', 20) + this._setValue(defaults, 'isTestManagementEnabled', false) this._setValue(defaults, 'logInjection', false) this._setValue(defaults, 'lookup', undefined) this._setValue(defaults, 'inferredProxyServicesEnabled', false) @@ -1142,7 +1144,9 @@ class Config { DD_CIVISIBILITY_FLAKY_RETRY_COUNT, DD_TEST_SESSION_NAME, DD_AGENTLESS_LOG_SUBMISSION_ENABLED, - DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED + DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED, + DD_TEST_MANAGEMENT_ENABLED, + DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES } = process.env if (DD_CIVISIBILITY_AGENTLESS_URL) { @@ -1162,6 +1166,11 @@ class Config { this._setBoolean(calc, 'ciVisAgentlessLogSubmissionEnabled', isTrue(DD_AGENTLESS_LOG_SUBMISSION_ENABLED)) this._setBoolean(calc, 'isTestDynamicInstrumentationEnabled', isTrue(DD_TEST_DYNAMIC_INSTRUMENTATION_ENABLED)) this._setBoolean(calc, 'isServiceUserProvided', !!this._env.service) + this._setBoolean(calc, 'isTestManagementEnabled', !isFalse(DD_TEST_MANAGEMENT_ENABLED)) + this._setValue(calc, + 'testManagementAttemptToFixRetries', + coalesce(maybeInt(DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES), 20) + ) } this._setString(calc, 'dogstatsd.hostname', this._getHostname()) this._setBoolean(calc, 'isGitUploadEnabled', diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index d08462a813c..173af519e2d 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -50,7 +50,7 @@ module.exports = class CiPlugin extends Plugin { this.addSub(`ci:${this.constructor.id}:library-configuration`, ({ onDone }) => { if (!this.tracer._exporter || !this.tracer._exporter.getLibraryConfiguration) { - return onDone({ err: new Error('CI Visibility was not initialized correctly') }) + return onDone({ err: new Error('Test optimization was not initialized correctly') }) } this.tracer._exporter.getLibraryConfiguration(this.testConfiguration, (err, libraryConfig) => { if (err) { @@ -64,7 +64,7 @@ module.exports = class CiPlugin extends Plugin { this.addSub(`ci:${this.constructor.id}:test-suite:skippable`, ({ onDone }) => { if (!this.tracer._exporter?.getSkippableSuites) { - return onDone({ err: new Error('CI Visibility was not initialized correctly') }) + return onDone({ err: new Error('Test optimization was not initialized correctly') }) } this.tracer._exporter.getSkippableSuites(this.testConfiguration, (err, skippableSuites, itrCorrelationId) => { if (err) { @@ -153,7 +153,7 @@ module.exports = class CiPlugin extends Plugin { this.addSub(`ci:${this.constructor.id}:known-tests`, ({ onDone }) => { if (!this.tracer._exporter?.getKnownTests) { - return onDone({ err: new Error('CI Visibility was not initialized correctly') }) + return onDone({ err: new Error('Test optimization was not initialized correctly') }) } this.tracer._exporter.getKnownTests(this.testConfiguration, (err, knownTests) => { if (err) { @@ -164,6 +164,19 @@ module.exports = class CiPlugin extends Plugin { onDone({ err, knownTests }) }) }) + + this.addSub(`ci:${this.constructor.id}:quarantined-tests`, ({ onDone }) => { + if (!this.tracer._exporter?.getQuarantinedTests) { + return onDone({ err: new Error('Test optimization was not initialized correctly') }) + } + this.tracer._exporter.getQuarantinedTests(this.testConfiguration, (err, quarantinedTests) => { + if (err) { + log.error('Quarantined tests could not be fetched. %s', err.message) + this.libraryConfig.isQuarantinedTestsEnabled = false + } + onDone({ err, quarantinedTests }) + }) + }) } get telemetry () { diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 285a03cc709..676acae1770 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -115,6 +115,9 @@ const DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX = 'snapshot_id' const DI_DEBUG_ERROR_FILE_SUFFIX = 'file' const DI_DEBUG_ERROR_LINE_SUFFIX = 'line' +const TEST_MANAGEMENT_IS_QUARANTINED = 'test.test_management.is_quarantined' +const TEST_MANAGEMENT_ENABLED = 'test.test_management.enabled' + module.exports = { TEST_CODE_OWNERS, TEST_SESSION_NAME, @@ -199,7 +202,9 @@ module.exports = { DI_DEBUG_ERROR_FILE_SUFFIX, DI_DEBUG_ERROR_LINE_SUFFIX, getFormattedError, - DD_TEST_IS_USER_PROVIDED_SERVICE + DD_TEST_IS_USER_PROVIDED_SERVICE, + TEST_MANAGEMENT_IS_QUARANTINED, + TEST_MANAGEMENT_ENABLED } // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19