fix(runner): Make exit code configurable when tests are failing (#3116)

Fixes #1300

Author: andreaspsson

lib/browser_collection.js

@@ -46,7 +46,24 @@ class BrowserCollection {
  return this.browsers.map((browser) => browser.serialize())
  }
 
- getResults () {
+ calculateExitCode (results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
+ if (results.disconnected || singleRunBrowserNotCaptured) {
+ return 1
+ }
+ if (results.success + results.failed === 0 && !failOnEmptyTestSuite) {
+ return 0
+ }
+ if (results.error) {
+ return 1
+ }
+ if (failOnFailingTestSuite === false) {
+ // Tests executed without infrastructure error, exit with 0 independent of test status.
+ return 0
+ }
+ return results.failed ? 1 : 0
+ }
+
+ getResults (singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
  const results = this.browsers.reduce((previous, current) => {
  previous.success += current.lastResult.success
  previous.failed += current.lastResult.failed
@@ -56,7 +73,7 @@ class BrowserCollection {
  }, {success: 0, failed: 0, error: false, disconnected: false, exitCode: 0})
 
  // compute exit status code
- results.exitCode = results.failed || results.error || results.disconnected ? 1 : 0
+ results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite)
 
  return results
  }

lib/cli.js

@@ -46,6 +46,10 @@ function processArgs (argv, options, fs, path) {
  options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true'
  }
 
+ if (helper.isString(options.failOnFailingTestSuite)) {
+ options.failOnFailingTestSuite = options.failOnFailingTestSuite === 'true'
+ }
+
  if (helper.isString(options.formatError)) {
  let required
  try {
@@ -196,6 +200,8 @@ function describeStart () {
  .describe('report-slower-than', '<integer> Report tests that are slower than given time [ms].')
  .describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
  .describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
+ .describe('fail-on-failing-test-suite', 'Fail on failing test suite.')
+ .describe('no-fail-on-failing-test-suite', 'Do not fail on failing test suite.')
  .describe('help', 'Print usage and options.')
 }
 

lib/server.js

@@ -247,14 +247,7 @@ class Server extends KarmaEventEmitter {
 
  const emitRunCompleteIfAllBrowsersDone = () => {
  if (Object.keys(singleRunDoneBrowsers).every((key) => singleRunDoneBrowsers[key])) {
- const results = singleRunBrowsers.getResults()
- if (singleRunBrowserNotCaptured) {
- results.exitCode = 1
- } else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
- results.exitCode = 0
- this.log.warn('Test suite was empty.')
- }
- this.emit('run_complete', singleRunBrowsers, results)
+ this.emit('run_complete', singleRunBrowsers, singleRunBrowsers.getResults(singleRunBrowserNotCaptured, config.failOnEmptyTestSuite, config.failOnFailingTestSuite))
  }
  }
 

test/unit/browser_collection.spec.js

@@ -258,4 +258,90 @@ describe('BrowserCollection', () => {
  })
  })
  })
+
+ // ============================================================================
+ // server.calculateExitCode
+ // ============================================================================
+ describe('calculateExitCode', () => {
+ const EXIT_CODE_ERROR = 1
+ const EXIT_CODE_SUCCESS = 0
+
+ describe('no tests', () => {
+ const results = {
+ success: 0,
+ failed: 0,
+ error: true
+ }
+ it('shall pass if failOnEmptyTestSuite not is set', () => {
+ const res = collection.calculateExitCode(results)
+ expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+ })
+ it('shall pass if failOnEmptyTestSuite is false', () => {
+ const res = collection.calculateExitCode(results, false, false)
+ expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+ })
+ it('shall fail if failOnEmptyTestSuite is true', () => {
+ const res = collection.calculateExitCode(results, false, true)
+ expect(res).to.be.equal(EXIT_CODE_ERROR)
+ })
+ it('shall fail if failOnFailingTestSuite is set', () => {
+ const res = collection.calculateExitCode(results, false, true, true)
+ expect(res).to.be.equal(EXIT_CODE_ERROR)
+ })
+ })
+ describe('all test passed, no errors', () => {
+ const results = {
+ success: 10,
+ failed: 0,
+ error: false
+ }
+ it('shall fail if singleRunBrowserNotCaptured is true', () => {
+ const res = collection.calculateExitCode(results, true)
+ expect(res).to.be.equal(EXIT_CODE_ERROR)
+ })
+ it('shall pass if failOnEmptyTestSuite not is set', () => {
+ const res = collection.calculateExitCode(results, false)
+ expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+ })
+ it('shall pass if failOnEmptyTestSuite not is set', () => {
+ const res = collection.calculateExitCode(results, false, false)
+ expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+ })
+ it('shall pass if failOnFailingTestSuite is true', () => {
+ const res = collection.calculateExitCode(results, false, true, true)
+ expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+ })
+ it('shall pass if failOnFailingTestSuite is false', () => {
+ const res = collection.calculateExitCode(results, false, true, false)
+ expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+ })
+ })
+ describe('all test passed, with error', () => {
+ const results = {
+ success: 10,
+ failed: 5,
+ error: false
+ }
+ it('shall fail if singleRunBrowserNotCaptured is true', () => {
+ const res = collection.calculateExitCode(results, true)
+ expect(res).to.be.equal(EXIT_CODE_ERROR)
+ })
+ it('shall fail if failOnEmptyTestSuite not is set', () => {
+ const res = collection.calculateExitCode(results, false)
+ expect(res).to.be.equal(EXIT_CODE_ERROR)
+ })
+ it('shall fail if failOnEmptyTestSuite not is set', () => {
+ const res = collection.calculateExitCode(results, false, false)
+ expect(res).to.be.equal(EXIT_CODE_ERROR)
+ })
+ it('shall fail if failOnFailingTestSuite is true', () => {
+ const res = collection.calculateExitCode(results, false, true, true)
+ expect(res).to.be.equal(EXIT_CODE_ERROR)
+ })
+ it('shall pass if failOnFailingTestSuite is false', () => {
+ const res = collection.calculateExitCode(results, false, true, false)
+ expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+ })
+ })
+ })
 })