Run microbenchmarks in Evergreen

Author: adriandole

.mci.yml

@@ -18,6 +18,7 @@ variables:
 
  mongodb_version:
  version_latest: &version_latest "latest"
+ version_60_perf: &version_60_perf "v6.0-perf"
  version_70: &version_70 "7.0"
  version_60: &version_60 "6.0"
  version_50: &version_50 "5.0"
@@ -428,6 +429,16 @@ functions:
 
  .evergreen/compile.sh -DCMAKE_PREFIX_PATH="$MONGOC_PREFIX" ${cmake_flags} ${poly_flags} $ADDL_OPTS -DCMAKE_INSTALL_PREFIX=install
 
+ "compile_benchmarks":
+ - command: shell.exec
+ params:
+ shell: bash
+ working_dir: "mongo-cxx-driver"
+ script: |
+ cd build
+ cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$(pwd)/../mongoc" -DCMAKE_CXX_STANDARD=20 ..
+ make microbenchmarks -j64
+
  "test":
  - command: shell.exec
  params:
@@ -849,6 +860,30 @@ functions:
  echo "Building Red Hat UBI Docker image"
  make -C extras/docker/redhat-ubi-9.2 nocachebuild test
 
+ "download benchmark data":
+ - command: shell.exec
+ params:
+ shell: bash
+ working_dir: "mongo-cxx-driver"
+ script: |
+ ./etc/microbenchmark-test-data.sh
+
+ "run benchmarks":
+ - command: shell.exec
+ params:
+ shell: bash
+ working_dir: "mongo-cxx-driver"
+ script: |
+ echo "Starting benchmarks"
+ ./build/benchmark/microbenchmarks all
+
+ "send benchmarks":
+ - command: perf.send
+ params:
+ name: perf
+ file: mongo-cxx-driver/results.json
+
+
 #######################################
 # Post Task #
 #######################################
@@ -959,6 +994,16 @@ tasks:
  vars:
  ENABLE_TESTS: OFF
 
+ - name: compile_and_run_benchmarks
+ commands:
+ - func: "setup"
+ - func: "start_mongod"
+ - func: "fetch_c_driver_source"
+ - func: "compile_benchmarks"
+ - func: "download benchmark data"
+ - func: "run benchmarks"
+ - func: "send benchmarks"
+
  - name: compile_macro_guard_tests
  commands:
  - func: "setup"
@@ -1450,6 +1495,15 @@ buildvariants:
  distros:
  - rhel90-large
 
+ - name: benchmarks-rhel9
+ display_name: "Benchmarks (RHEL 9.2)"
+ expansions:
+ mongodb_version: *version_60_perf
+ run_on:
+ - rhel90-dbx-perf-large
+ tasks:
+ - name: compile_and_run_benchmarks
+
  - name: arm-rhel9-release-latest
  display_name: "arm64 RHEL 9 Release (MongoDB Latest)"
  expansions:

benchmark/benchmark_runner.cpp

@@ -14,6 +14,14 @@
 
 #include "benchmark_runner.hpp"
 
+#include <chrono>
+#include <cstdint>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <sstream>
+
 #include "bson/bson_encoding.hpp"
 #include "multi_doc/bulk_insert.hpp"
 #include "multi_doc/find_many.hpp"
@@ -26,7 +34,13 @@
 #include "single_doc/find_one_by_id.hpp"
 #include "single_doc/insert_one.hpp"
 #include "single_doc/run_command.hpp"
+#include <bsoncxx/builder/basic/array.hpp>
+#include <bsoncxx/builder/basic/document.hpp>
+#include <bsoncxx/builder/basic/kvp.hpp>
+#include <bsoncxx/builder/basic/sub_document.hpp>
+#include <bsoncxx/json.hpp>
 #include <bsoncxx/stdx/make_unique.hpp>
+#include <bsoncxx/types.hpp>
 
 namespace benchmark {
 
@@ -47,26 +61,27 @@ benchmark_runner::benchmark_runner(std::set<benchmark_type> types) : _types{type
  _microbenches.push_back(make_unique<run_command>());
  _microbenches.push_back(make_unique<find_one_by_id>("single_and_multi_document/tweet.json"));
  _microbenches.push_back(make_unique<insert_one>(
- "TestSmallDocInsertOne", 2.75, 10000, "single_and_multi_document/small_doc.json"));
+ "TestSmallDocInsertOne", 2.75, iterations, "single_and_multi_document/small_doc.json"));
  _microbenches.push_back(make_unique<insert_one>(
  "TestLargeDocInsertOne", 27.31, 10, "single_and_multi_document/large_doc.json"));
 
  // Multi doc microbenchmarks
  _microbenches.push_back(make_unique<find_many>("single_and_multi_document/tweet.json"));
  _microbenches.push_back(make_unique<bulk_insert>(
- "TestSmallDocBulkInsert", 2.75, 10000, "single_and_multi_document/small_doc.json"));
+ "TestSmallDocBulkInsert", 2.75, iterations, "single_and_multi_document/small_doc.json"));
  _microbenches.push_back(make_unique<bulk_insert>(
  "TestLargeDocBulkInsert", 27.31, 10, "single_and_multi_document/large_doc.json"));
- _microbenches.push_back(
- make_unique<gridfs_upload>("single_and_multi_document/gridfs_large.bin"));
- _microbenches.push_back(
- make_unique<gridfs_download>("single_and_multi_document/gridfs_large.bin"));
+ // CXX-2794: Disable GridFS benchmarks due to long runtime
+ // _microbenches.push_back(
+ // make_unique<gridfs_upload>("single_and_multi_document/gridfs_large.bin"));
+ // _microbenches.push_back(
+ // make_unique<gridfs_download>("single_and_multi_document/gridfs_large.bin"));
 
  // Parallel microbenchmarks
  _microbenches.push_back(make_unique<json_multi_import>("parallel/ldjson_multi"));
  _microbenches.push_back(make_unique<json_multi_export>("parallel/ldjson_multi"));
- _microbenches.push_back(make_unique<gridfs_multi_import>("parallel/gridfs_multi"));
- _microbenches.push_back(make_unique<gridfs_multi_export>("parallel/gridfs_multi"));
+ // _microbenches.push_back(make_unique<gridfs_multi_import>("parallel/gridfs_multi"));
+ // _microbenches.push_back(make_unique<gridfs_multi_export>("parallel/gridfs_multi"));
 
  // Need to remove some
  if (!_types.empty()) {
@@ -89,8 +104,6 @@ benchmark_runner::benchmark_runner(std::set<benchmark_type> types) : _types{type
 }
 
 void benchmark_runner::run_microbenches() {
- mongocxx::instance instance{};
-
  for (std::unique_ptr<microbench>& bench : _microbenches) {
  std::cout << "Starting " << bench->get_name() << "..." << std::endl;
 
@@ -145,20 +158,50 @@ double benchmark_runner::calculate_driver_bench_score() {
  return (calculate_read_bench_score() + calculate_write_bench_score()) / 2.0;
 }
 
-void benchmark_runner::print_scores() {
+void benchmark_runner::write_scores(
+ const std::chrono::time_point<std::chrono::system_clock> start_time) {
  double read = -1;
  double write = -1;
+ const auto end_time = std::chrono::system_clock::now();
+
+ using namespace bsoncxx;
+ using builder::basic::sub_document;
+
+ auto doc = builder::basic::document{};
+ doc.append(kvp("info", [](sub_document subdoc) {
+ subdoc.append(kvp("test_name", "C++ microbenchmarks"));
+ }));
+
+ auto write_time =
+ [](const std::chrono::time_point<std::chrono::system_clock> t) -> std::string {
+ std::time_t t1 = std::chrono::system_clock::to_time_t(t);
+ std::ostringstream oss;
+ oss << std::put_time(std::gmtime(&t1), "%Y-%m-%dT%H:%M:%S") << "+00:00";
+ return oss.str();
+ };
+ doc.append(kvp("created_at", write_time(start_time)));
+ doc.append(kvp("completed_at", write_time(end_time)));
+ doc.append(kvp("artifacts", builder::basic::make_array()));
+
+ auto metrics_array = builder::basic::array{};
+ std::cout << std::endl << "Composite benchmarks:" << std::endl << "===========" << std::endl;
 
  std::cout << "Individual microbenchmark scores:" << std::endl << "===========" << std::endl;
  for (auto&& bench : _microbenches) {
  auto& score = bench->get_results();
+ const auto bench_time = static_cast<double>(score.get_percentile(50).count()) / 1000.0;
 
- std::cout << bench->get_name() << ": "
- << static_cast<double>(score.get_percentile(50).count()) / 1000.0
- << " second(s) | " << score.get_score() << " MB/s" << std::endl;
- }
+ std::cout << bench->get_name() << ": " << bench_time << " seconds | " << score.get_score()
+ << " MB/s" << std::endl;
 
- std::cout << std::endl << "Composite benchmarks:" << std::endl << "===========" << std::endl;
+ auto metric_doc = builder::basic::document{};
+ metric_doc.append(kvp("name", bench->get_name()));
+ metric_doc.append(kvp("type", "THROUGHPUT"));
+ metric_doc.append(kvp("value", score.get_score()));
+ metrics_array.append(metric_doc);
+ }
+ doc.append(kvp("metrics", metrics_array));
+ doc.append(kvp("sub_tests", builder::basic::make_array()));
 
  auto print_comp = [this, &read, &write](benchmark_type type) {
  double avg = calculate_average(type);
@@ -169,7 +212,7 @@ void benchmark_runner::print_scores() {
  write = avg;
  }
 
- std::cout << type_names[type] << " " << avg << " MB/s" << std::endl;
+ std::cout << type_names.at(type) << " " << avg << " MB/s" << std::endl;
  };
 
  if (!_types.empty()) {
@@ -185,5 +228,8 @@ void benchmark_runner::print_scores() {
  if (read > 0 && write > 0) {
  std::cout << "DriverBench: " << (read + write) / 2.0 << " MB/s" << std::endl;
  }
+
+ std::ofstream os{"results.json"};
+ os << '[' << bsoncxx::to_json(doc.view()) << ']';
 }
 } // namespace benchmark

benchmark/benchmark_runner.hpp

@@ -14,10 +14,11 @@
 
 #pragma once
 
-#include <bsoncxx/stdx/optional.hpp>
-#include <mongocxx/instance.hpp>
+#include <chrono>
 
 #include "microbench.hpp"
+#include <bsoncxx/stdx/optional.hpp>
+#include <mongocxx/instance.hpp>
 
 namespace benchmark {
 
@@ -27,7 +28,7 @@ class benchmark_runner {
 
  void run_microbenches();
 
- void print_scores();
+ void write_scores(const std::chrono::time_point<std::chrono::system_clock> start_time);
 
  double calculate_bson_bench_score();
 

benchmark/bson/bson_decoding.hpp

@@ -42,7 +42,7 @@ void bson_decoding::setup() {
 }
 
 void bson_decoding::task() {
- for (std::uint32_t i = 0; i < 10000; i++) {
+ for (std::uint32_t i = 0; i < iterations; i++) {
  // TODO CXX-1241: call bson_as_extended json on _json.
  }
 }

benchmark/bson/bson_encoding.hpp

@@ -16,11 +16,10 @@
 
 #include <iostream>
 
+#include "../microbench.hpp"
 #include <bsoncxx/json.hpp>
 #include <bsoncxx/types.hpp>
 
-#include "../microbench.hpp"
-
 namespace benchmark {
 
 class bson_encoding : public microbench {
@@ -58,7 +57,7 @@ void visit_document(bsoncxx::document::view doc) {
 
 // Mirroring mongo-c-driver's interpretation of the spec.
 void bson_encoding::task() {
- for (std::uint32_t i = 0; i < 10000; i++) {
+ for (std::uint32_t i = 0; i < iterations; i++) {
  visit_document(_doc->view());
  }
 }

benchmark/main.cpp

@@ -12,24 +12,34 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <chrono>
 #include <iostream>
+#include <string_view>
 
 #include "benchmark_runner.hpp"
+#include <mongocxx/instance.hpp>
 
 using namespace benchmark;
 
 int main(int argc, char* argv[]) {
+ mongocxx::instance instance{};
  std::set<benchmark_type> types;
 
  if (argc > 1) {
- for (int x = 1; x < argc; ++x) {
- std::string type{argv[x]};
- auto it = names_types.find(type);
-
- if (it != names_types.end()) {
- types.insert(it->second);
- } else {
- std::cerr << "Invalid benchmark: " << type << std::endl;
+ if (std::string_view(argv[1]) == "all") {
+ for (const auto& [name, type] : names_types) {
+ types.insert(type);
+ }
+ } else {
+ for (int x = 1; x < argc; ++x) {
+ std::string type{argv[x]};
+ auto it = names_types.find(type);
+
+ if (it != names_types.end()) {
+ types.insert(it->second);
+ } else {
+ std::cerr << "Invalid benchmark: " << type << std::endl;
+ }
  }
  }
 
@@ -40,6 +50,8 @@ int main(int argc, char* argv[]) {
  }
 
  benchmark_runner runner{types};
+
+ const auto start_time = std::chrono::system_clock::now();
  runner.run_microbenches();
- runner.print_scores();
+ runner.write_scores(start_time);
 }

benchmark/microbench.cpp

@@ -23,7 +23,7 @@ namespace benchmark {
 
 bool finished_running(const std::chrono::duration<std::uint32_t, std::milli>& curr_time,
  std::uint32_t iter) {
- return (curr_time > maxtime || (curr_time > mintime && iter > MAX_ITER));
+ return (curr_time > maxtime || (curr_time > mintime && iter > max_iter));
 }
 
 void microbench::run() {