[RP005] Shards global counters btr_cur_n_non_sea and btr_cur_n_sea to minimize cache coherency cost

"btr_cur_n_sea++" and "btr_cur_n_non_sea++" at btr_cur_search_to_nth_level() cause very expensive cost at high concurreency. Shards by each cpu threads (os_getcpu()) to minimize inter-CPU communication. Not for accuracy, just only for performance, not to add kind of memory barrier.

Author: buildup-db

MYSQL_VERSION

@@ -3,4 +3,4 @@ MYSQL_VERSION_MINOR=0
 MYSQL_VERSION_PATCH=42
 MYSQL_VERSION_EXTRA=
 MYSQL_VERSION_STABILITY="LTS"
-MYSQL_RP_REVISION="-RP004"
+MYSQL_RP_REVISION="-RP005"

storage/innobase/btr/btr0cur.cc

@@ -3,6 +3,7 @@
 Copyright (c) 1994, 2025, Oracle and/or its affiliates.
 Copyright (c) 2008, Google Inc.
 Copyright (c) 2012, Facebook Inc.
+Copyright (c) 2025, buildup-db.
 
 Portions of this file contain modifications contributed and copyrighted by
 Google, Inc. Those modifications are gratefully acknowledged and are described
@@ -124,18 +125,20 @@ throughput clearly from about 100000. */
 constexpr uint32_t BTR_CUR_FINE_HISTORY_LENGTH = 100000;
 
 /** Number of searches down the B-tree in btr_cur_search_to_nth_level(). */
-ulint btr_cur_n_non_sea = 0;
+alignas(ut::INNODB_CACHE_LINE_SIZE)
+ std::array<uint64_t, BTR_CUR_COUNTER_SHARDING> btr_cur_n_non_sea{};
 /** Number of successful adaptive hash index lookups in
 btr_cur_search_to_nth_level(). */
-ulint btr_cur_n_sea = 0;
+alignas(ut::INNODB_CACHE_LINE_SIZE)
+ std::array<uint64_t, BTR_CUR_COUNTER_SHARDING> btr_cur_n_sea{};
 /** Old value of btr_cur_n_non_sea. Copied by
 srv_refresh_innodb_monitor_stats(). Referenced by
 srv_printf_innodb_monitor(). */
-ulint btr_cur_n_non_sea_old = 0;
+uint64_t btr_cur_n_non_sea_old = 0;
 /** Old value of btr_cur_n_sea. Copied by
 srv_refresh_innodb_monitor_stats(). Referenced by
 srv_printf_innodb_monitor(). */
-ulint btr_cur_n_sea_old = 0;
+uint64_t btr_cur_n_sea_old = 0;
 
 #ifdef UNIV_DEBUG
 /* Flag to limit optimistic insert records */
@@ -791,11 +794,11 @@ void btr_cur_search_to_nth_level(
  ut_ad(cursor->up_match != ULINT_UNDEFINED || mode != PAGE_CUR_GE);
  ut_ad(cursor->up_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE);
  ut_ad(cursor->low_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE);
- btr_cur_n_sea++;
+ btr_cur_n_sea[BTR_CUR_COUNTER_INDEX]++;
 
  return;
  }
- btr_cur_n_non_sea++;
+ btr_cur_n_non_sea[BTR_CUR_COUNTER_INDEX]++;
  DBUG_EXECUTE_IF("non_ahi_search",
  assert(!strcmp(index->table->name.m_name, "test/t1")););
 

storage/innobase/include/btr0cur.h

@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1994, 2025, Oracle and/or its affiliates.
+Copyright (c) 2025, buildup-db.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License, version 2.0, as published by the
@@ -40,8 +41,11 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #include "btr0types.h"
 #include "dict0dict.h"
 #include "gis0type.h"
+#include "os0numa.h" /* os_getcpu() */
+#include "os0thread.h" /* ut::this_thread_hash */
 #include "page0cur.h"
 #include "univ.i"
+#include "ut0cpu_cache.h" /* ut::INNODB_CACHE_LINE_SIZE */
 
 /** Mode flags for btr_cur operations; these can be ORed */
 enum {
@@ -756,19 +760,27 @@ is still a good change of success a little later. Sleep this many
 milliseconds between retries. */
 constexpr uint32_t BTR_CUR_RETRY_SLEEP_TIME_MS = 50;
 
+/** Global counter sharded for saving CPU cache coherency cost. */
+constexpr size_t BTR_CUR_COUNTER_SHARDING = 128;
+#ifdef HAVE_OS_GETCPU
+#define BTR_CUR_COUNTER_INDEX \
+ (static_cast<size_t>(os_getcpu()) % BTR_CUR_COUNTER_SHARDING)
+#else /* HAVE_OS_GETCPU */
+#define BTR_CUR_COUNTER_INDEX (ut::this_thread_hash % BTR_CUR_COUNTER_SHARDING)
+#endif /* HAVE_OS_GETCPU */
 /** Number of searches down the B-tree in btr_cur_search_to_nth_level(). */
-extern ulint btr_cur_n_non_sea;
+extern std::array<uint64_t, BTR_CUR_COUNTER_SHARDING> btr_cur_n_non_sea;
 /** Number of successful adaptive hash index lookups in
 btr_cur_search_to_nth_level(). */
-extern ulint btr_cur_n_sea;
+extern std::array<uint64_t, BTR_CUR_COUNTER_SHARDING> btr_cur_n_sea;
 /** Old value of btr_cur_n_non_sea. Copied by
 srv_refresh_innodb_monitor_stats(). Referenced by
 srv_printf_innodb_monitor(). */
-extern ulint btr_cur_n_non_sea_old;
+extern uint64_t btr_cur_n_non_sea_old;
 /** Old value of btr_cur_n_sea. Copied by
 srv_refresh_innodb_monitor_stats(). Referenced by
 srv_printf_innodb_monitor(). */
-extern ulint btr_cur_n_sea_old;
+extern uint64_t btr_cur_n_sea_old;
 
 #ifdef UNIV_DEBUG
 /* Flag to limit optimistic insert records */

storage/innobase/srv/srv0mon.cc

@@ -2,6 +2,7 @@
 
 Copyright (c) 2010, 2025, Oracle and/or its affiliates.
 Copyright (c) 2012, Facebook Inc.
+Copyright (c) 2025, buildup-db.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License, version 2.0, as published by the
@@ -1986,11 +1987,17 @@ void srv_mon_process_existing_counter(
  value = log_sys->m_capacity.adaptive_flush_max_age();
  break;
  case MONITOR_OVLD_ADAPTIVE_HASH_SEARCH:
- value = btr_cur_n_sea;
+ value = 0;
+ for (size_t i = 0; i < BTR_CUR_COUNTER_SHARDING; i++) {
+ value += btr_cur_n_sea[i];
+ }
  break;
 
  case MONITOR_OVLD_ADAPTIVE_HASH_SEARCH_BTREE:
- value = btr_cur_n_non_sea;
+ value = 0;
+ for (size_t i = 0; i < BTR_CUR_COUNTER_SHARDING; i++) {
+ value += btr_cur_n_non_sea[i];
+ }
  break;
 
  default:

storage/innobase/srv/srv0srv.cc

@@ -3,6 +3,7 @@
 Copyright (c) 1995, 2025, Oracle and/or its affiliates.
 Copyright (c) 2008, 2009 Google Inc.
 Copyright (c) 2009, Percona Inc.
+Copyright (c) 2025, buildup-db.
 
 Portions of this file contain modifications contributed and copyrighted by
 Google, Inc. Those modifications are gratefully acknowledged and are described
@@ -1298,8 +1299,14 @@ static void srv_refresh_innodb_monitor_stats(void) {
 
  os_aio_refresh_stats();
 
- btr_cur_n_sea_old = btr_cur_n_sea;
- btr_cur_n_non_sea_old = btr_cur_n_non_sea;
+ uint64_t btr_cur_n_sea_sum = 0;
+ uint64_t btr_cur_n_non_sea_sum = 0;
+ for (size_t i = 0; i < BTR_CUR_COUNTER_SHARDING; i++) {
+ btr_cur_n_sea_sum += btr_cur_n_sea[i];
+ btr_cur_n_non_sea_sum += btr_cur_n_non_sea[i];
+ }
+ btr_cur_n_sea_old = btr_cur_n_sea_sum;
+ btr_cur_n_non_sea_old = btr_cur_n_non_sea_sum;
 
  log_refresh_stats(*log_sys);
 
@@ -1450,11 +1457,18 @@ bool srv_printf_innodb_monitor(FILE *file, bool nowait, ulint *trx_start_pos,
  rw_lock_s_unlock(&part.latch);
  }
 
+ uint64_t btr_cur_n_sea_sum = 0;
+ uint64_t btr_cur_n_non_sea_sum = 0;
+ for (size_t i = 0; i < BTR_CUR_COUNTER_SHARDING; i++) {
+ btr_cur_n_sea_sum += btr_cur_n_sea[i];
+ btr_cur_n_non_sea_sum += btr_cur_n_non_sea[i];
+ }
+
  fprintf(file, "%.2f hash searches/s, %.2f non-hash searches/s\n",
- (btr_cur_n_sea - btr_cur_n_sea_old) / time_elapsed,
- (btr_cur_n_non_sea - btr_cur_n_non_sea_old) / time_elapsed);
- btr_cur_n_sea_old = btr_cur_n_sea;
- btr_cur_n_non_sea_old = btr_cur_n_non_sea;
+ (btr_cur_n_sea_sum - btr_cur_n_sea_old) / time_elapsed,
+ (btr_cur_n_non_sea_sum - btr_cur_n_non_sea_old) / time_elapsed);
+ btr_cur_n_sea_old = btr_cur_n_sea_sum;
+ btr_cur_n_non_sea_old = btr_cur_n_non_sea_sum;
 
  fputs(
  "---\n"