ci(NODE-6755): add tags to benchmarks (#4419)

Author: W-A-James

test/benchmarks/driver_bench/readme.md

@@ -51,13 +51,30 @@ type BenchmarkModule = {
  run: () => Promise<void>;
  afterEach?: () => Promise<void>;
  after?: () => Promise<void>;
+
+ tags?: string[];
 };
 ```
 
 Just like mocha we have once before and once after as well as before each and after each hooks.
 
 The `driver.mts` module is intended to hold various helpers for setup and teardown and help abstract some of the driver API.
 
+## Benchmark tags
+The `tags` property of `BenchmarkModule` is where a benchmark's tags should be added to facilitate
+performance alerting and filter of results via our internal tools.
+
+Tags are defined in `driver.mts` under the `TAG` enum.
+Whenever a new tag is defined it should be documented in the table below .
+
+| tag variable name | tag string value | purpose |
+|-------------------|------------------------|--------------------------------------------------------------------------------------------------------------------------------------|
+| `TAG.spec` | `'spec-benchmark'` | Special tag that marks a benchmark as a spec-required benchmark |
+| `TAG.alert` | `'alerting-benchmark'` | Special tag that enables our perf monitoring tooling to create alerts when regressions in this benchmark's performance are detected |
+| `TAG.cursor` | `'cursor-benchmark'` | Tag marking a benchmark as being related to cursor performance |
+| `TAG.read` | `'read-benchmark'` | Tag marking a benchmark as being related to read performance |
+| `TAG.write` | `'write-benchmark'` | Tag marking a benchmark as being related to write performance |
+
 ## Wishlist
 
 - Make it so runner can handle: `./lib/suites/multi_bench/grid_fs_upload.mjs` as an argument so shell path autocomplete makes it easier to pick a benchmark

test/benchmarks/driver_bench/src/driver.mts

@@ -7,6 +7,20 @@ import process from 'node:process';
 const __dirname = import.meta.dirname;
 const require = module.createRequire(__dirname);
 
+export const TAG = {
+ // Special tag that marks a benchmark as a spec-required benchmark
+ spec: 'spec-benchmark',
+ // Special tag that enables our perf monitoring tooling to create alerts when regressions in this
+ // benchmark's performance are detected
+ alert: 'alerting-benchmark',
+ // Tag marking a benchmark as being related to cursor performance
+ cursor: 'cursor-benchmark',
+ // Tag marking a benchmark as being related to read performance
+ read: 'read-benchmark',
+ // Tag marking a benchmark as being related to write performance
+ write: 'write-benchmark'
+};
+
 /**
  * The path to the MongoDB Node.js driver.
  * This MUST be set to the directory the driver is installed in
@@ -118,19 +132,23 @@ export const PARALLEL_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'parallel');
 export const TEMP_DIRECTORY = path.resolve(SPEC_DIRECTORY, 'tmp');
 
 export type Metric = {
- name: 'megabytes_per_second';
+ name: 'megabytes_per_second' | 'normalized_throughput';
  value: number;
+ metadata: {
+ improvement_direction: 'up' | 'down';
+ };
 };
 
 export type MetricInfo = {
  info: {
  test_name: string;
  args: Record<string, number>;
+ tags?: string[];
  };
  metrics: Metric[];
 };
 
-export function metrics(test_name: string, result: number): MetricInfo {
+export function metrics(test_name: string, result: number, tags?: string[]): MetricInfo {
  return {
  info: {
  test_name,
@@ -141,9 +159,13 @@ export function metrics(test_name: string, result: number): MetricInfo {
  key,
  typeof value === 'number' ? value : value ? 1 : 0
  ])
- )
+ ),
+ tags
  },
- metrics: [{ name: 'megabytes_per_second', value: result }]
+ // FIXME(NODE-6781): For now all of our metrics are of throughput so their improvement_direction is up,
+ metrics: [
+ { name: 'megabytes_per_second', value: result, metadata: { improvement_direction: 'up' } }
+ ]
  } as const;
 }
 

test/benchmarks/driver_bench/src/main.mts

@@ -85,7 +85,7 @@ console.log(systemInfo());
 
 const runnerPath = path.join(__dirname, 'runner.mjs');
 
-const results: MetricInfo[] = [];
+let results: MetricInfo[] = [];
 
 for (const [suite, benchmarks] of Object.entries(tests)) {
  console.group(snakeToCamel(suite));
@@ -198,6 +198,42 @@ function calculateCompositeBenchmarks(results: MetricInfo[]) {
  return [...results, ...compositeResults];
 }
 
-const finalResults = calculateCompositeBenchmarks(results);
+function calculateNormalizedResults(results: MetricInfo[]): MetricInfo[] {
+ const baselineBench = results.find(r => r.info.test_name === 'cpuBaseline');
+ const pingBench = results.find(r => r.info.test_name === 'ping');
+
+ assert.ok(pingBench, 'ping bench results not found!');
+ assert.ok(baselineBench, 'baseline results not found!');
+ const pingThroughput = pingBench.metrics[0].value;
+ const cpuBaseline = baselineBench.metrics[0].value;
+
+ for (const bench of results) {
+ if (bench.info.test_name === 'cpuBaseline') continue;
+ if (bench.info.test_name === 'ping') {
+ bench.metrics.push({
+ name: 'normalized_throughput',
+ value: bench.metrics[0].value / cpuBaseline,
+ metadata: {
+ improvement_direction: 'up'
+ }
+ });
+ }
+ // Compute normalized_throughput of benchmarks against ping bench
+ else {
+ bench.metrics.push({
+ name: 'normalized_throughput',
+ value: bench.metrics[0].value / pingThroughput,
+ metadata: {
+ improvement_direction: 'up'
+ }
+ });
+ }
+ }
+
+ return results;
+}
+
+results = calculateCompositeBenchmarks(results);
+results = calculateNormalizedResults(results);
 
-await fs.writeFile('results.json', JSON.stringify(finalResults, undefined, 2), 'utf8');
+await fs.writeFile('results.json', JSON.stringify(results, undefined, 2), 'utf8');

test/benchmarks/driver_bench/src/runner.mts

@@ -14,6 +14,7 @@ type BenchmarkModule = {
  run: () => Promise<void>;
  afterEach?: () => Promise<void>;
  after?: () => Promise<void>;
+ tags?: string[];
 };
 
 const benchmarkName = snakeToCamel(path.basename(benchmarkFile, '.mjs'));
@@ -80,6 +81,14 @@ function percentileIndex(percentile: number, count: number) {
 const medianExecution = durations[percentileIndex(50, count)];
 const megabytesPerSecond = benchmark.taskSize / medianExecution;
 
+const tags = benchmark.tags;
+if (
+ tags &&
+ (!Array.isArray(tags) || (tags.length > 0 && !tags.every(t => typeof t === 'string')))
+) {
+ throw new Error('If tags is specified, it MUST be an array of strings');
+}
+
 console.log(
  ' '.repeat(3),
  ...['total time:', totalDuration, 'sec,'],
@@ -91,6 +100,6 @@ console.log(
 
 await fs.writeFile(
  `results_${path.basename(benchmarkFile, '.mjs')}.json`,
- JSON.stringify(metrics(benchmarkName, megabytesPerSecond), undefined, 2) + '\n',
+ JSON.stringify(metrics(benchmarkName, megabytesPerSecond, tags), undefined, 2) + '\n',
  'utf8'
 );

test/benchmarks/driver_bench/src/suites/multi_bench/find_many_and_empty_cursor.mts

@@ -1,8 +1,10 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 16.22;
 
+export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read];
+
 let collection: mongodb.Collection;
 
 export async function before() {

test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_download.mts

@@ -1,10 +1,12 @@
 import { Readable, Writable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 52.43;
 
+export const tags = [TAG.alert, TAG.spec, TAG.cursor, TAG.read];
+
 let bucket: mongodb.GridFSBucket;
 let bin: Uint8Array;
 let _id: mongodb.ObjectId;

test/benchmarks/driver_bench/src/suites/multi_bench/grid_fs_upload.mts

@@ -1,9 +1,10 @@
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 52.43;
+export const tags = [TAG.alert, TAG.spec, TAG.write];
 
 let bucket: mongodb.GridFSBucket;
 let uploadStream: mongodb.GridFSBucketWriteStream;

test/benchmarks/driver_bench/src/suites/multi_bench/large_doc_bulk_insert.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 27.31;
+export const tags = [TAG.alert, TAG.spec, TAG.write];
 
 let collection: mongodb.Collection;
 let documents: any[];

test/benchmarks/driver_bench/src/suites/multi_bench/small_doc_bulk_insert.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 2.75;
+export const tags = [TAG.alert, TAG.spec, TAG.write];
 
 let collection: mongodb.Collection;
 let documents: any[];

test/benchmarks/driver_bench/src/suites/node_specific/aggregate_a_million_documents_and_to_array.mts

@@ -1,6 +1,7 @@
-import { driver, type mongodb } from '../../driver.mjs';
+import { driver, type mongodb, TAG } from '../../driver.mjs';
 
 export const taskSize = 16;
+export const tags = [TAG.alert, TAG.cursor, TAG.read];
 
 let db: mongodb.Db;