fix(junit): report suite hook failures

Author: morgan-coded

lib/plugin/junitReporter.js

@@ -7,6 +7,7 @@ import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom'
 import event from '../event.js'
 import store from '../store.js'
 import output from '../output.js'
+import container from '../container.js'
 
 const defaultConfig = {
   outputName: 'report.xml',
@@ -18,6 +19,7 @@ const defaultConfig = {
 }
 
 const INVALID_XML_CHARS = new RegExp('[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F\\uFFFE\\uFFFF]', 'g')
+const SUITE_HOOK_TITLE = /^"(before all|after all)" hook:/
 
 /**
  *
@@ -66,6 +68,40 @@ export default function (config = {}) {
   config = Object.assign({}, defaultConfig, config)
 
   let written = false
+  let runnerAttached = false
+  const hookFailures = []
+  const seenHookFailures = new Set()
+
+  const attachRunner = () => {
+    if (runnerAttached) return
+
+    const mocha = container.mocha()
+    const runner = mocha && (mocha.runner || mocha.Runner)
+    if (!runner || typeof runner.on !== 'function') return
+
+    runnerAttached = true
+    runner.on('fail', (failed, err) => {
+      if (!failed || failed.type !== 'hook' || !SUITE_HOOK_TITLE.test(failed.title || '')) return
+
+      const suite = failed.parent
+      const suiteTitle = (suite && suite.title) || ''
+      const key = `${suiteTitle}::${failed.title}`
+      if (seenHookFailures.has(key)) return
+      seenHookFailures.add(key)
+
+      hookFailures.push({
+        title: failed.title || 'hook failed',
+        state: 'failed',
+        err: err || failed.err || {},
+        parent: suite,
+        file: failed.file || (suite && suite.file),
+        tags: suiteTitle.match(/@[\\w-]+/g) || [],
+        meta: {},
+        steps: [],
+        duration: failed.duration || 0,
+      })
+    })
+  }
 
   const writeReport = result => {
     if (written) return
@@ -76,25 +112,29 @@ export default function (config = {}) {
     mkdirp.sync(dir)
     const file = path.join(dir, config.outputName)
 
-    fs.writeFileSync(file, buildXml(result, config))
+    fs.writeFileSync(file, buildXml(result, config, hookFailures))
     output.plugin('junitReporter', `JUnit report saved to ${file}`)
   }
 
+  event.dispatcher.on(event.all.before, attachRunner)
+  event.dispatcher.on(event.suite.before, attachRunner)
+  event.dispatcher.on(event.test.before, attachRunner)
   event.dispatcher.on(event.all.result, writeReport)
   event.dispatcher.on(event.workers.result, writeReport)
 }
 
-function buildXml(result, config) {
+function buildXml(result, config, hookFailures = []) {
   const doc = new DOMImplementation().createDocument(null, null, null)
-  const suites = groupBySuite(result.tests)
+  const allTests = result.tests.concat(hookFailures)
+  const suites = groupBySuite(allTests)
 
   const root = doc.createElement('testsuites')
   setAttr(root, 'name', config.testGroupName)
-  setAttr(root, 'tests', result.tests.length)
-  setAttr(root, 'failures', countState(result.tests, 'failed'))
-  setAttr(root, 'skipped', countSkipped(result.tests))
+  setAttr(root, 'tests', allTests.length)
+  setAttr(root, 'failures', countState(allTests, 'failed'))
+  setAttr(root, 'skipped', countSkipped(allTests))
   setAttr(root, 'errors', 0)
-  setAttr(root, 'time', toSeconds(sumDuration(result.tests)))
+  setAttr(root, 'time', toSeconds(sumDuration(allTests)))
   setAttr(root, 'timestamp', toIso(result.stats && result.stats.start))
   doc.appendChild(root)
 

test/unit/junitReporter_test.js

@@ -8,6 +8,7 @@ import xml2js from 'xml2js'
 import junitReporter from '../../lib/plugin/junitReporter.js'
 import event from '../../lib/event.js'
 import store from '../../lib/store.js'
+import container from '../../lib/container.js'
 import Step from '../../lib/step/base.js'
 import MetaStep from '../../lib/step/meta.js'
 
@@ -83,6 +84,22 @@ function parseReport(dir) {
   return new xml2js.Parser().parseStringPromise(fs.readFileSync(path.join(dir, 'report.xml'), 'utf8'))
 }
 
+function stubMochaRunner() {
+  const listeners = {}
+
+  container.append({
+    mocha: {
+      runner: {
+        on(name, fn) {
+          listeners[name] = fn
+        },
+      },
+    },
+  })
+
+  return listeners
+}
+
 describe('JUnit Reporter Plugin', () => {
   let tmpDir
   let prevOutputDir
@@ -91,13 +108,20 @@ describe('JUnit Reporter Plugin', () => {
     prevOutputDir = store._outputDir
     tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cjs-junit-'))
     store.outputDir = tmpDir
+    event.dispatcher.removeAllListeners(event.all.before)
     event.dispatcher.removeAllListeners(event.all.result)
+    event.dispatcher.removeAllListeners(event.suite.before)
+    event.dispatcher.removeAllListeners(event.test.before)
     event.dispatcher.removeAllListeners(event.workers.result)
   })
 
   afterEach(() => {
+    event.dispatcher.removeAllListeners(event.all.before)
     event.dispatcher.removeAllListeners(event.all.result)
+    event.dispatcher.removeAllListeners(event.suite.before)
+    event.dispatcher.removeAllListeners(event.test.before)
     event.dispatcher.removeAllListeners(event.workers.result)
+    container.append({ mocha: {} })
     store._outputDir = prevOutputDir
     fs.rmSync(tmpDir, { recursive: true, force: true })
   })
@@ -198,4 +222,57 @@ describe('JUnit Reporter Plugin', () => {
     const secondMtime = fs.statSync(path.join(tmpDir, 'report.xml')).mtimeMs
     expect(secondMtime).to.equal(firstMtime)
   })
+
+  it('adds suite-level hook failures as failed testcases', async () => {
+    const listeners = stubMochaRunner()
+    junitReporter({})
+    event.dispatcher.emit(event.suite.before)
+
+    const suite = { title: 'Suite hooks @smoke', file: '/tests/hooks_test.js', startedAt: Date.now() }
+    listeners.fail(
+      {
+        type: 'hook',
+        title: '"before all" hook: BeforeSuite for "records suite hook failures"',
+        parent: suite,
+        file: suite.file,
+        duration: 42,
+      },
+      new Error('BeforeSuite failed'),
+    )
+    listeners.fail(
+      {
+        type: 'hook',
+        title: '"before each" hook: Before for "already reported by test"',
+        parent: suite,
+        file: suite.file,
+        duration: 7,
+      },
+      new Error('Before failed'),
+    )
+    listeners.fail(
+      {
+        type: 'hook',
+        title: '"before all" hook: BeforeSuite for "records suite hook failures"',
+        parent: suite,
+        file: suite.file,
+        duration: 42,
+      },
+      new Error('duplicate suite failure'),
+    )
+
+    event.dispatcher.emit(event.all.result, {
+      tests: [],
+      stats: { start: new Date(), passes: 0, failures: 0, pending: 0, tests: 0 },
+    })
+
+    const parsed = await parseReport(tmpDir)
+    expect(parsed.testsuites.$.tests).to.equal('1')
+    expect(parsed.testsuites.$.failures).to.equal('1')
+
+    const suiteEl = parsed.testsuites.testsuite[0]
+    expect(suiteEl.$.name).to.equal('Suite hooks @smoke')
+    expect(suiteEl.$.tests).to.equal('1')
+    expect(suiteEl.testcase[0].$.name).to.contain('"before all" hook')
+    expect(suiteEl.testcase[0].failure[0].$.message).to.contain('BeforeSuite failed')
+  })
 })