Skip to content

Commit 874b04f

Browse files
committed
Added integration test stats (#1083)
1 parent a91e537 commit 874b04f

File tree

2 files changed

+44
-11
lines changed

2 files changed

+44
-11
lines changed

test/integration/index.js

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ const xPackYamlFolder = join(esFolder, 'x-pack', 'plugin', 'src', 'test', 'resou
2020

2121
const MAX_API_TIME = 1000 * 90
2222
const MAX_FILE_TIME = 1000 * 30
23-
const MAX_TEST_TIME = 1000 * 2
23+
const MAX_TEST_TIME = 1000 * 3
2424

2525
const ossSkips = {
2626
'cat.indices/10_basic.yml': ['Test cat indices output for closed index (pre 7.2.0)'],
@@ -111,6 +111,12 @@ async function start ({ client, isXPack }) {
111111

112112
log(`Testing ${isXPack ? 'XPack' : 'oss'} api...`)
113113

114+
const stats = {
115+
total: 0,
116+
skip: 0,
117+
pass: 0,
118+
assertions: 0
119+
}
114120
const folders = getAllFiles(isXPack ? xPackYamlFolder : yamlFolder)
115121
.filter(t => !/(README|TODO)/g.test(t))
116122
// we cluster the array based on the folder names,
@@ -172,10 +178,15 @@ async function start ({ client, isXPack }) {
172178
const testTime = now()
173179
const name = Object.keys(test)[0]
174180
if (name === 'setup' || name === 'teardown') continue
175-
if (shouldSkip(isXPack, file, name)) continue
181+
stats.total += 1
182+
if (shouldSkip(isXPack, file, name)) {
183+
stats.skip += 1
184+
continue
185+
}
176186
log(' - ' + name)
177187
try {
178-
await testRunner.run(setupTest, test[name], teardownTest)
188+
await testRunner.run(setupTest, test[name], teardownTest, stats)
189+
stats.pass += 1
179190
} catch (err) {
180191
console.error(err)
181192
process.exit(1)
@@ -202,6 +213,12 @@ async function start ({ client, isXPack }) {
202213
}
203214
}
204215
log(`Total testing time: ${ms(now() - totalTime)}`)
216+
log(`Test stats:
217+
- Total: ${stats.total}
218+
- Skip: ${stats.skip}
219+
- Pass: ${stats.pass}
220+
- Assertions: ${stats.assertions}
221+
`)
205222
}
206223

207224
function log (text) {

test/integration/test-runner.js

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ function build (opts = {}) {
214214
* @oaram {object} teardown (null if not needed)
215215
* @returns {Promise}
216216
*/
217-
async function run (setup, test, teardown) {
217+
async function run (setup, test, teardown, stats) {
218218
// if we should skip a feature in the setup/teardown section
219219
// we should skip the entire test file
220220
const skip = getSkip(setup) || getSkip(teardown)
@@ -236,11 +236,11 @@ function build (opts = {}) {
236236
}
237237
}
238238

239-
if (setup) await exec('Setup', setup)
239+
if (setup) await exec('Setup', setup, stats)
240240

241-
await exec('Test', test)
241+
await exec('Test', test, stats)
242242

243-
if (teardown) await exec('Teardown', teardown)
243+
if (teardown) await exec('Teardown', teardown, stats)
244244

245245
if (isXPack) await cleanupXPack()
246246

@@ -371,9 +371,14 @@ function build (opts = {}) {
371371
* @param {object} the action to perform
372372
* @returns {Promise}
373373
*/
374-
async function doAction (action) {
374+
async function doAction (action, stats) {
375375
const cmd = parseDo(action)
376-
const api = delve(client, cmd.method).bind(client)
376+
try {
377+
var api = delve(client, cmd.method).bind(client)
378+
} catch (err) {
379+
console.error(`\nError: Cannot find the method '${cmd.method}' in the client.\n`)
380+
process.exit(1)
381+
}
377382

378383
const options = { ignore: cmd.params.ignore, headers: action.headers }
379384
if (cmd.params.ignore) delete cmd.params.ignore
@@ -414,10 +419,12 @@ function build (opts = {}) {
414419
warnings = warnings.filter(h => !h.test(/default\snumber\sof\sshards/g))
415420
}
416421

422+
stats.assertions += 1
417423
assert.ok(deepEqual(warnings, action.warnings))
418424
}
419425

420426
if (action.catch) {
427+
stats.assertions += 1
421428
assert.ok(
422429
parseDoError(err, action.catch),
423430
`the error should be: ${action.catch}`
@@ -428,6 +435,7 @@ function build (opts = {}) {
428435
response = err.body
429436
}
430437
} else {
438+
stats.assertions += 1
431439
assert.ifError(err, `should not error: ${cmd.method}`, action)
432440
response = body
433441
}
@@ -439,7 +447,7 @@ function build (opts = {}) {
439447
* @param {object} the actions to perform
440448
* @returns {Promise}
441449
*/
442-
async function exec (name, actions) {
450+
async function exec (name, actions, stats) {
443451
// tap.comment(name)
444452
for (const action of actions) {
445453
if (action.skip) {
@@ -450,7 +458,7 @@ function build (opts = {}) {
450458
}
451459

452460
if (action.do) {
453-
await doAction(fillStashedValues(action.do))
461+
await doAction(fillStashedValues(action.do), stats)
454462
}
455463

456464
if (action.set) {
@@ -464,6 +472,7 @@ function build (opts = {}) {
464472
}
465473

466474
if (action.match) {
475+
stats.assertions += 1
467476
const key = Object.keys(action.match)[0]
468477
match(
469478
// in some cases, the yaml refers to the body with an empty string
@@ -478,6 +487,7 @@ function build (opts = {}) {
478487
}
479488

480489
if (action.lt) {
490+
stats.assertions += 1
481491
const key = Object.keys(action.lt)[0]
482492
lt(
483493
delve(response, fillStashedValues(key)),
@@ -486,6 +496,7 @@ function build (opts = {}) {
486496
}
487497

488498
if (action.gt) {
499+
stats.assertions += 1
489500
const key = Object.keys(action.gt)[0]
490501
gt(
491502
delve(response, fillStashedValues(key)),
@@ -494,6 +505,7 @@ function build (opts = {}) {
494505
}
495506

496507
if (action.lte) {
508+
stats.assertions += 1
497509
const key = Object.keys(action.lte)[0]
498510
lte(
499511
delve(response, fillStashedValues(key)),
@@ -502,6 +514,7 @@ function build (opts = {}) {
502514
}
503515

504516
if (action.gte) {
517+
stats.assertions += 1
505518
const key = Object.keys(action.gte)[0]
506519
gte(
507520
delve(response, fillStashedValues(key)),
@@ -510,6 +523,7 @@ function build (opts = {}) {
510523
}
511524

512525
if (action.length) {
526+
stats.assertions += 1
513527
const key = Object.keys(action.length)[0]
514528
length(
515529
key === '$body' || key === ''
@@ -522,6 +536,7 @@ function build (opts = {}) {
522536
}
523537

524538
if (action.is_true) {
539+
stats.assertions += 1
525540
const isTrue = fillStashedValues(action.is_true)
526541
is_true(
527542
delve(response, isTrue),
@@ -530,6 +545,7 @@ function build (opts = {}) {
530545
}
531546

532547
if (action.is_false) {
548+
stats.assertions += 1
533549
const isFalse = fillStashedValues(action.is_false)
534550
is_false(
535551
delve(response, isFalse),

0 commit comments

Comments
 (0)