CXX-2710 Run microbenchmarks in Evergreen

.mci.yml

@@ -360,6 +360,16 @@ functions:
  CXX: ${cxx_compiler}
  script: .evergreen/compile.sh
 
+ "compile_benchmarks":
+ - command: shell.exec
+ type: setup
+ params:
+ shell: bash
+ working_dir: "mongo-cxx-driver"
+ script: |
+ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$(pwd)/../mongoc" -DCMAKE_CXX_STANDARD=20
+ cmake --build build --target microbenchmarks --parallel 64
+
  "test":
  - command: shell.exec
  params:
@@ -509,6 +519,24 @@ functions:
  make -C extras/docker/bookworm nocachebuild test
  echo "Building Red Hat UBI Docker image"
  make -C extras/docker/redhat-ubi-9.3 nocachebuild test
+
+ "run benchmarks":
+ - command: shell.exec
+ type: setup
+ params:
+ shell: bash
+ working_dir: "mongo-cxx-driver"
+ script: ./etc/microbenchmark-test-data.sh
+ - command: shell.exec
+ type: test
+ params:
+ shell: bash
+ working_dir: "mongo-cxx-driver"
+ script: ./build/benchmark/microbenchmarks all
+ - command: perf.send
+ params:
+ name: perf
+ file: mongo-cxx-driver/results.json
 
 #######################################
 # Post Task #
@@ -620,6 +648,14 @@ tasks:
  vars:
  ENABLE_TESTS: OFF
 
+ - name: compile_and_run_benchmarks
+ commands:
+ - func: "setup"
+ - func: "start_mongod"
+ - func: "fetch_c_driver_source"
+ - func: "compile_benchmarks"
+ - func: "run benchmarks"
+
  - name: compile_macro_guard_tests
  commands:
  - func: "setup"
@@ -1303,6 +1339,14 @@ buildvariants:
  - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
  - name: build_example_with_add_subdirectory
 
+ - name: benchmarks-rhel9
+ display_name: "Benchmarks (RHEL 9.2)"
+ expansions:
+ mongodb_version: "v6.0-perf"
+ run_on: rhel90-dbx-perf-large
+ tasks:
+ - name: compile_and_run_benchmarks
+
  - name: arm-rhel9-release-latest
  display_name: "arm64 RHEL 9 Release (MongoDB Latest)"
  expansions:

benchmark/CMakeLists.txt

@@ -50,7 +50,7 @@ file (GLOB benchmark_DIST_hpps RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.hpp)
 
 set_dist_list (benchmark_DIST
  CMakeLists.txt
- README.txt
+ README.md
  ${BENCHMARK_LIBRARY}
  ${benchmark_DIST_hpps}
 )

benchmark/README.txt → benchmark/README.md

@@ -1,10 +1,16 @@
 This suite implements the benchmarks described in this spec: https://docs.google.com/document/d/1x7dCHHx_owOFYIPZ6VQwPerSWRhdkMSiepyFrNSFlLw/edit#
 
-In order to run the microbenchmarks, first run etc/microbenchmark-test-data.sh to download the data.
+# Compiling
+Configure the C++ driver to build with C++17 or newer and build the `microbenchmarks` target.
 
-In order to run specific tests, just specify their names as arguments. If run with no arguments,
-all benchmarks will be run.
-e.g. build/benchmark/microbenchmarks BSONBench MultiBench
+# Running
+In order to run the microbenchmarks, first run `etc/microbenchmark-test-data.sh` to download the test data.
+
+In order to run specific tests, just specify their names as arguments:
+`build/benchmark/microbenchmarks BSONBench MultiBench`
+
+To run all tests, specify `all` as an argument:
+`build/benchmark/microbenchmarks all`
 
 Full list of options:
 BSONBench
@@ -14,14 +20,9 @@ ReadBench
 WriteBench
 RunCommandBench
 
-Note: make sure you run both the download script and the microbenchmarks binary from the project root.
-
-See the spec for details on these benchmarks.
-
-In case there is trouble running the benchmarks, they were written and will run with: 
- -libbson at commit 1c5b90022bbbdf0fa8095b337a25b218a2651241 
- -mongoc driver at commit a3c8a760ce90f144fd65d8a4a6e3606cbeea2f6b
+Note: run both the download script and the microbenchmarks binary from the project root.
 
+# Notes
 Note that in order to compare against the other drivers, an inMemory mongod instance should be 
 used.
 

benchmark/benchmark_runner.cpp

@@ -14,6 +14,14 @@
 
 #include "benchmark_runner.hpp"
 
+#include <chrono>
+#include <cstdint>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <sstream>
+
 #include "bson/bson_encoding.hpp"
 #include "multi_doc/bulk_insert.hpp"
 #include "multi_doc/find_many.hpp"
@@ -26,7 +34,13 @@
 #include "single_doc/find_one_by_id.hpp"
 #include "single_doc/insert_one.hpp"
 #include "single_doc/run_command.hpp"
+#include <bsoncxx/builder/basic/array.hpp>
+#include <bsoncxx/builder/basic/document.hpp>
+#include <bsoncxx/builder/basic/kvp.hpp>
+#include <bsoncxx/builder/basic/sub_document.hpp>
+#include <bsoncxx/json.hpp>
 #include <bsoncxx/stdx/make_unique.hpp>
+#include <bsoncxx/types.hpp>
 
 namespace benchmark {
 
@@ -47,26 +61,28 @@ benchmark_runner::benchmark_runner(std::set<benchmark_type> types) : _types{type
  _microbenches.push_back(make_unique<run_command>());
  _microbenches.push_back(make_unique<find_one_by_id>("single_and_multi_document/tweet.json"));
  _microbenches.push_back(make_unique<insert_one>(
- "TestSmallDocInsertOne", 2.75, 10000, "single_and_multi_document/small_doc.json"));
+ "TestSmallDocInsertOne", 2.75, iterations, "single_and_multi_document/small_doc.json"));
  _microbenches.push_back(make_unique<insert_one>(
  "TestLargeDocInsertOne", 27.31, 10, "single_and_multi_document/large_doc.json"));
 
  // Multi doc microbenchmarks
  _microbenches.push_back(make_unique<find_many>("single_and_multi_document/tweet.json"));
  _microbenches.push_back(make_unique<bulk_insert>(
- "TestSmallDocBulkInsert", 2.75, 10000, "single_and_multi_document/small_doc.json"));
+ "TestSmallDocBulkInsert", 2.75, iterations, "single_and_multi_document/small_doc.json"));
  _microbenches.push_back(make_unique<bulk_insert>(
  "TestLargeDocBulkInsert", 27.31, 10, "single_and_multi_document/large_doc.json"));
- _microbenches.push_back(
- make_unique<gridfs_upload>("single_and_multi_document/gridfs_large.bin"));
- _microbenches.push_back(
- make_unique<gridfs_download>("single_and_multi_document/gridfs_large.bin"));
+ // CXX-2794: Disable GridFS benchmarks due to long runtime
+ // _microbenches.push_back(
+ // make_unique<gridfs_upload>("single_and_multi_document/gridfs_large.bin"));
+ // _microbenches.push_back(
+ // make_unique<gridfs_download>("single_and_multi_document/gridfs_large.bin"));
 
  // Parallel microbenchmarks
  _microbenches.push_back(make_unique<json_multi_import>("parallel/ldjson_multi"));
  _microbenches.push_back(make_unique<json_multi_export>("parallel/ldjson_multi"));
- _microbenches.push_back(make_unique<gridfs_multi_import>("parallel/gridfs_multi"));
- _microbenches.push_back(make_unique<gridfs_multi_export>("parallel/gridfs_multi"));
+ // CXX-2794: Disable GridFS benchmarks due to long runtime
+ // _microbenches.push_back(make_unique<gridfs_multi_import>("parallel/gridfs_multi"));
+ // _microbenches.push_back(make_unique<gridfs_multi_export>("parallel/gridfs_multi"));
 
  // Need to remove some
  if (!_types.empty()) {
@@ -89,8 +105,7 @@ benchmark_runner::benchmark_runner(std::set<benchmark_type> types) : _types{type
 }
 
 void benchmark_runner::run_microbenches() {
- mongocxx::instance instance{};
-
+ _start_time = std::chrono::system_clock::now();
  for (std::unique_ptr<microbench>& bench : _microbenches) {
  std::cout << "Starting " << bench->get_name() << "..." << std::endl;
 
@@ -103,6 +118,7 @@ void benchmark_runner::run_microbenches() {
  << " second(s) | " << score.get_score() << " MB/s" << std::endl
  << std::endl;
  }
+ _end_time = std::chrono::system_clock::now();
 }
 
 double benchmark_runner::calculate_average(benchmark_type tag) {
@@ -145,20 +161,48 @@ double benchmark_runner::calculate_driver_bench_score() {
  return (calculate_read_bench_score() + calculate_write_bench_score()) / 2.0;
 }
 
-void benchmark_runner::print_scores() {
+void benchmark_runner::write_scores() {
  double read = -1;
  double write = -1;
 
+ using namespace bsoncxx;
+ using builder::basic::sub_document;
+
+ auto doc = builder::basic::document{};
+ doc.append(kvp("info", [](sub_document subdoc) {
+ subdoc.append(kvp("test_name", "C++ Driver microbenchmarks"));
+ }));
+
+ auto write_time =
+ [](const std::chrono::time_point<std::chrono::system_clock> t) -> std::string {
+ std::time_t t1 = std::chrono::system_clock::to_time_t(t);
+ std::ostringstream oss;
+ oss << std::put_time(std::gmtime(&t1), "%Y-%m-%dT%H:%M:%S") << "+00:00";
+ return oss.str();
+ };
+ doc.append(kvp("created_at", write_time(_start_time)));
+ doc.append(kvp("completed_at", write_time(_end_time)));
+ doc.append(kvp("artifacts", builder::basic::make_array()));
+
+ auto metrics_array = builder::basic::array{};
+ std::cout << std::endl << "Composite benchmarks:" << std::endl << "===========" << std::endl;
+
  std::cout << "Individual microbenchmark scores:" << std::endl << "===========" << std::endl;
  for (auto&& bench : _microbenches) {
  auto& score = bench->get_results();
+ const auto bench_time = static_cast<double>(score.get_percentile(50).count()) / 1000.0;
 
- std::cout << bench->get_name() << ": "
- << static_cast<double>(score.get_percentile(50).count()) / 1000.0
- << " second(s) | " << score.get_score() << " MB/s" << std::endl;
- }
+ std::cout << bench->get_name() << ": " << bench_time << " seconds | " << score.get_score()
+ << " MB/s" << std::endl;
 
- std::cout << std::endl << "Composite benchmarks:" << std::endl << "===========" << std::endl;
+ auto metric_doc = builder::basic::document{};
+ metric_doc.append(kvp("name", bench->get_name()));
+ metric_doc.append(kvp("type", "THROUGHPUT"));
+ metric_doc.append(kvp("value", score.get_score()));
+ metrics_array.append(metric_doc);
+ }
+ doc.append(kvp("metrics", metrics_array));
+ doc.append(kvp("sub_tests", builder::basic::make_array()));
 
  auto print_comp = [this, &read, &write](benchmark_type type) {
  double avg = calculate_average(type);
@@ -169,7 +213,7 @@ void benchmark_runner::print_scores() {
  write = avg;
  }
 
- std::cout << type_names[type] << " " << avg << " MB/s" << std::endl;
+ std::cout << type_names.at(type) << " " << avg << " MB/s" << std::endl;
  };
 
  if (!_types.empty()) {
@@ -185,5 +229,8 @@ void benchmark_runner::print_scores() {
  if (read > 0 && write > 0) {
  std::cout << "DriverBench: " << (read + write) / 2.0 << " MB/s" << std::endl;
  }
+
+ std::ofstream os{"results.json"};
+ os << '[' << bsoncxx::to_json(doc.view()) << ']';
 }
 } // namespace benchmark

benchmark/benchmark_runner.hpp

@@ -14,10 +14,11 @@
 
 #pragma once
 
-#include <bsoncxx/stdx/optional.hpp>
-#include <mongocxx/instance.hpp>
+#include <chrono>
 
 #include "microbench.hpp"
+#include <bsoncxx/stdx/optional.hpp>
+#include <mongocxx/instance.hpp>
 
 namespace benchmark {
 
@@ -27,7 +28,7 @@ class benchmark_runner {
 
  void run_microbenches();
 
- void print_scores();
+ void write_scores();
 
  double calculate_bson_bench_score();
 
@@ -46,6 +47,8 @@ class benchmark_runner {
  private:
  double calculate_average(benchmark_type);
 
+ std::chrono::time_point<std::chrono::system_clock> _start_time;
+ std::chrono::time_point<std::chrono::system_clock> _end_time;
  std::vector<std::unique_ptr<microbench>> _microbenches;
  std::set<benchmark_type> _types;
 };

benchmark/bson/bson_decoding.hpp

@@ -42,7 +42,7 @@ void bson_decoding::setup() {
 }
 
 void bson_decoding::task() {
- for (std::uint32_t i = 0; i < 10000; i++) {
+ for (std::uint32_t i = 0; i < iterations; i++) {
  // TODO CXX-1241: call bson_as_extended json on _json.
  }
 }

benchmark/bson/bson_encoding.hpp

@@ -16,11 +16,10 @@
 
 #include <iostream>
 
+#include "../microbench.hpp"
 #include <bsoncxx/json.hpp>
 #include <bsoncxx/types.hpp>
 
-#include "../microbench.hpp"
-
 namespace benchmark {
 
 class bson_encoding : public microbench {
@@ -58,7 +57,7 @@ void visit_document(bsoncxx::document::view doc) {
 
 // Mirroring mongo-c-driver's interpretation of the spec.
 void bson_encoding::task() {
- for (std::uint32_t i = 0; i < 10000; i++) {
+ for (std::uint32_t i = 0; i < iterations; i++) {
  visit_document(_doc->view());
  }
 }

benchmark/main.cpp

@@ -12,34 +12,46 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+static_assert(__cplusplus >= 201703L, "requires C++17 or higher");
+
+#include <chrono>
 #include <iostream>
+#include <stdexcept>
 
 #include "benchmark_runner.hpp"
+#include <bsoncxx/stdx/string_view.hpp>
+#include <mongocxx/instance.hpp>
 
 using namespace benchmark;
 
 int main(int argc, char* argv[]) {
+ mongocxx::instance instance;
  std::set<benchmark_type> types;
 
  if (argc > 1) {
- for (int x = 1; x < argc; ++x) {
- std::string type{argv[x]};
- auto it = names_types.find(type);
-
- if (it != names_types.end()) {
+ if (bsoncxx::stdx::string_view(argv[1]) == "all") {
+ for (const auto& [name, type] : names_types) {
+ types.insert(type);
+ }
+ } else {
+ for (int x = 1; x < argc; ++x) {
+ std::string type{argv[x]};
+ auto it = names_types.find(type);
+
+ if (it == names_types.end()) {
+ throw std::runtime_error("Invalid benchmark: " + type);
+ }
  types.insert(it->second);
- } else {
- std::cerr << "Invalid benchmark: " << type << std::endl;
  }
  }
 
  if (types.empty()) {
- std::cerr << "No valid benchmarks specified. Exiting." << std::endl;
- return 1;
+ throw std::runtime_error("No valid benchmarks specified");
  }
  }
 
  benchmark_runner runner{types};
+
  runner.run_microbenches();
- runner.print_scores();
+ runner.write_scores();
 }