Skip to content

Commit f5cb2b9

Browse files
Blue Ftgriesser
andauthored
test: Honeycomb system-test reporter (#19855)
* Mocha performance reporter Co-authored-by: Tim Griesser <tgriesser10@gmail.com>
1 parent 10c6e5a commit f5cb2b9

File tree

8 files changed

+277
-26
lines changed

8 files changed

+277
-26
lines changed

circle.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2088,15 +2088,19 @@ linux-workflow: &linux-workflow
20882088
requires:
20892089
- build
20902090
- system-tests-chrome:
2091+
context: test-runner:performance-tracking
20912092
requires:
20922093
- system-tests-node-modules-install
20932094
- system-tests-electron:
2095+
context: test-runner:performance-tracking
20942096
requires:
20952097
- system-tests-node-modules-install
20962098
- system-tests-firefox:
2099+
context: test-runner:performance-tracking
20972100
requires:
20982101
- system-tests-node-modules-install
20992102
- system-tests-non-root:
2103+
context: test-runner:performance-tracking
21002104
executor: non-root-docker-user
21012105
requires:
21022106
- system-tests-node-modules-install

packages/server/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@
147147
"@types/chai-as-promised": "7.1.2",
148148
"@types/chrome": "0.0.101",
149149
"@types/http-proxy": "1.17.4",
150+
"@types/node": "14.14.31",
150151
"awesome-typescript-loader": "5.2.1",
151152
"babel-loader": "8.1.0",
152153
"body-parser": "1.19.0",
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
const path = require('path')
2+
const chalk = require('chalk')
3+
const Libhoney = require('libhoney')
4+
5+
const pkg = require('@packages/root')
6+
const ciProvider = require('@packages/server/lib/util/ci_provider')
7+
const { commitInfo } = require('@cypress/commit-info')
8+
9+
class StatsdReporter {
10+
constructor (runner) {
11+
if (!process.env.HONEYCOMB_API_KEY) {
12+
return
13+
}
14+
15+
console.log(chalk.green('Reporting to honeycomb'))
16+
17+
let branch; let commitSha
18+
19+
this.honey = new Libhoney({
20+
dataset: 'systemtest-performance',
21+
writeKey: process.env.HONEYCOMB_API_KEY,
22+
})
23+
24+
commitInfo().then((commitInformation) => {
25+
const ciInformation = ciProvider.commitParams() || {}
26+
27+
this.branch = commitInformation.branch || ciInformation.branch
28+
this.commitSha = commitInformation.sha || ciInformation.sha
29+
})
30+
31+
runner.on('test', (test) => {
32+
test.wallclockStart = Date.now()
33+
})
34+
35+
runner.on('test end', (test) => {
36+
// Skipped tests never get a 'start' event, but they still get 'test end' somehow.
37+
if (!test.state || test.state === 'skipped') {
38+
return
39+
}
40+
41+
const title = test.titlePath().join(' / ')
42+
// This regex pulls apart a string like `e2e async timeouts / failing1 [electron]`
43+
// into `e2e async timeouts / failing1` and `electron`, letting us use the same
44+
// test name for all browsers, with the browser as a separate field.
45+
// The browser capture group is optional because some tests aren't browser specific,
46+
// in which case it will be undefined and not passed as a field to honeycomb.
47+
const [, testTitle, browser] = title.match(/(.+?)(?: \[([a-z]+)\])?$/)
48+
49+
const honeycombEvent = this.honey.newEvent()
50+
51+
honeycombEvent.timestamp = test.wallclockStart
52+
honeycombEvent.add({
53+
test: testTitle,
54+
specFile: path.basename(test.file),
55+
browser,
56+
state: test.state,
57+
err: test.err && test.err.message,
58+
errStack: test.err && test.err.stack,
59+
durationMs: Date.now() - test.wallclockStart,
60+
mochaDurationMs: test.duration,
61+
branch,
62+
commitSha,
63+
buildUrl: process.env.CIRCLE_BUILD_URL,
64+
platform: process.platform,
65+
arch: process.arch,
66+
version: pkg.version,
67+
})
68+
69+
honeycombEvent.send()
70+
})
71+
}
72+
73+
// If there is no done callback, then mocha-multi-reporter will kill the process without waiting for our honeycomb post to complete.
74+
done (failures, callback) {
75+
if (this.honey) {
76+
this.honey.flush().then(callback)
77+
}
78+
}
79+
}
80+
81+
module.exports = StatsdReporter

system-tests/lib/spec_helper.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ beforeEach(function () {
103103

104104
nock.disableNetConnect()
105105
nock.enableNetConnect(/localhost/)
106+
nock.enableNetConnect(/api.honeycomb.io/)
106107

107108
// always clean up the cache
108109
// before each test

system-tests/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
"human-interval": "1.0.0",
5656
"image-size": "0.8.3",
5757
"lazy-ass": "1.6.0",
58+
"libhoney": "3.0.0",
5859
"lodash": "^4.17.21",
5960
"mocha": "7.1.0",
6061
"mocha-banner": "1.1.2",
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"reporterEnabled": "spec, mocha-junit-reporter, ../../system-tests/lib/performance-reporter.js",
3+
"mochaJunitReporterReporterOptions": {
4+
"mochaFile": "/tmp/cypress/junit/test-results.[hash].xml"
5+
}
6+
}

system-tests/scripts/run.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ commandAndArguments.args.push(
109109
'--reporter',
110110
'mocha-multi-reporters',
111111
'--reporter-options',
112-
'configFile=../../mocha-reporter-config.json',
112+
'configFile=../../system-tests/scripts/mocha-reporter-config.json',
113113
'--extension=js,ts',
114114
// restore mocha 2.x behavior to force end process after spec run
115115
'--exit',

0 commit comments

Comments
 (0)