MDEV-35049: btr_search_check_free_space_in_heap() is a bottleneck

Let us use implement a simple fixed-size allocator for the adaptive hash index, insted of complicating mem_heap_t or mem_block_info_t. MEM_HEAP_BTR_SEARCH: Remove. mem_block_info_t::free_block(), mem_heap_free_block_free(): Remove. mem_heap_free_top(), mem_heap_get_top(): Remove. btr_sea::partition::spare: Replaces mem_block_info_t::free_block. This keeps one spare block per adaptive hash index partition, to process an insert. We must not wait for buf_pool.mutex while holding any btr_sea::partition::latch. That is why we cache one block for future allocations. This is protected by a new btr_sea::partition::blocks_mutex in order to relieve pressure on btr_sea::partition::latch. btr_sea::partition::prepare_insert(): Replaces btr_search_check_free_space_in_heap(). btr_sea::partition::erase(): Replaces ha_search_and_delete_if_found(). btr_sea::partition::cleanup_after_erase(): Replaces the most part of ha_delete_hash_node(). Unlike the previous implementation, we will retain a spare block for prepare_insert(). This should reduce some contention on buf_pool.mutex. btr_search.n_parts: Replaces btr_ahi_parts. btr_search.enabled: Replaces btr_search_enabled. This must hold whenever buf_block_t::index is set while a thread is holding a btr_sea::partition::latch. dict_index_t::search_info: Remove pointer indirection, and use Atomic_relaxed or Atomic_counter for most fields. btr_search_guess_on_hash(): Let the caller ensure that latch_mode is BTR_MODIFY_LEAF or BTR_SEARCH_LEAF. Release btr_sea::partition::latch before buffer-fixing the block. The page latch that we already acquired is preventing buffer pool eviction. We must validate both block->index and block->page.state while holding part.latch in order to avoid race conditions with buffer page relocation or buf_pool_t::resize(). btr_search_check_guess(): Remove the constant parameter can_only_compare_to_cursor_rec=false. ahi_node: Replaces ha_node_t. This has been tested by running the regression test suite with the adaptive hash index enabled: ./mtr --mysqld=--loose-innodb-adaptive-hash-index=ON Reviewed by: Vladislav Lesin

Author: dr-m

extra/mariabackup/xtrabackup.cc

@@ -1845,8 +1845,8 @@ struct my_option xb_server_options[] =
 #ifdef BTR_CUR_HASH_ADAPT
  {"innodb_adaptive_hash_index", OPT_INNODB_ADAPTIVE_HASH_INDEX,
  "Enable InnoDB adaptive hash index (disabled by default).",
- &btr_search_enabled,
- &btr_search_enabled,
+ &btr_search.enabled,
+ &btr_search.enabled,
  0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 #endif /* BTR_CUR_HASH_ADAPT */
  {"innodb_autoextend_increment", OPT_INNODB_AUTOEXTEND_INCREMENT,
@@ -2402,7 +2402,7 @@ static bool innodb_init_param()
 srv_page_size = 0;
 srv_page_size_shift = 0;
 #ifdef BTR_CUR_HASH_ADAPT
-btr_ahi_parts = 1;
+btr_search.n_parts = 1;
 #endif /* BTR_CUR_HASH_ADAPT */
 
 if (innobase_page_size != (1LL << 14)) {

storage/innobase/CMakeLists.txt

@@ -193,7 +193,6 @@ SET(INNOBASE_SOURCES
 include/btr0pcur.h
 include/btr0pcur.inl
 include/btr0sea.h
-include/btr0sea.inl
 include/btr0types.h
 include/buf0buddy.h
 include/buf0buf.h
@@ -259,8 +258,6 @@ SET(INNOBASE_SOURCES
 include/gis0rtree.inl
 include/gis0type.h
 include/ha_prototypes.h
-include/ha0ha.h
-include/ha0ha.inl
 include/ha0storage.h
 include/ha0storage.inl
 include/handler0alter.h

storage/innobase/btr/btr0btr.cc

@@ -280,7 +280,7 @@ btr_root_block_get(
 #ifndef BTR_CUR_ADAPT
  static constexpr buf_block_t *guess= nullptr;
 #else
- buf_block_t *&guess= btr_search_get_info(index)->root_guess;
+ buf_block_t *&guess= index->search_info.root_guess;
  guess=
 #endif
  block=
@@ -1085,7 +1085,7 @@ dberr_t dict_index_t::clear(que_thr_t *thr)
 #ifndef BTR_CUR_ADAPT
  static constexpr buf_block_t *guess= nullptr;
 #else
- buf_block_t *&guess= btr_search_get_info(this)->root_guess;
+ buf_block_t *&guess= search_info.root_guess;
  guess=
 #endif
  root_block= buf_page_get_gen({table->space_id, page},
@@ -1095,14 +1095,12 @@ dberr_t dict_index_t::clear(que_thr_t *thr)
  {
  btr_free_but_not_root(root_block, mtr.get_log_mode()
 #ifdef BTR_CUR_HASH_ADAPT
- ,n_ahi_pages() != 0
+ ,any_ahi_pages()
 #endif
  );
-
+ btr_search_drop_page_hash_index(root_block, false);
 #ifdef BTR_CUR_HASH_ADAPT
- if (root_block->index)
- btr_search_drop_page_hash_index(root_block, false);
- ut_ad(n_ahi_pages() == 0);
+ ut_ad(!any_ahi_pages());
 #endif
  mtr.memset(root_block, PAGE_HEADER + PAGE_BTR_SEG_LEAF,
  FSEG_HEADER_SIZE, 0);
@@ -1147,7 +1145,7 @@ void btr_drop_temporary_table(const dict_table_t &table)
 #ifndef BTR_CUR_ADAPT
  static constexpr buf_block_t *guess= nullptr;
 #else
- buf_block_t *guess= index->search_info->root_guess;
+ buf_block_t *guess= index->search_info.root_guess;
 #endif
  if (buf_block_t *block= buf_page_get_gen({SRV_TMP_SPACE_ID, index->page},
  0, RW_X_LATCH, guess, BUF_GET,

storage/innobase/btr/btr0cur.cc

@@ -1122,18 +1122,19 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode,
 #ifndef BTR_CUR_ADAPT
  guess= nullptr;
 #else
- btr_search_t *info= btr_search_get_info(index());
+ auto info= &index()->search_info;
  guess= info->root_guess;
 
 # ifdef BTR_CUR_HASH_ADAPT
 # ifdef UNIV_SEARCH_PERF_STAT
  info->n_searches++;
 # endif
- bool ahi_enabled= btr_search_enabled;
- /* We do a dirty read of btr_search_enabled below,
- and btr_search_guess_on_hash() will have to check it again. */
- if (!ahi_enabled);
- else if (btr_search_guess_on_hash(index(), info, tuple, mode,
+ if (latch_mode > BTR_MODIFY_LEAF)
+ /* The adaptive hash index cannot be useful for these searches. */;
+ /* We do a dirty read of btr_search.enabled below,
+ and btr_search_guess_on_hash() will have to check it again. */
+ else if (!btr_search.enabled);
+ else if (btr_search_guess_on_hash(index(), tuple, mode,
  latch_mode, this, mtr))
  {
  /* Search using the hash index succeeded */
@@ -1397,7 +1398,7 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode,
 
  reached_latched_leaf:
 #ifdef BTR_CUR_HASH_ADAPT
- if (ahi_enabled && !(tuple->info_bits & REC_INFO_MIN_REC_FLAG))
+ if (btr_search.enabled && !(tuple->info_bits & REC_INFO_MIN_REC_FLAG))
  {
  if (page_cur_search_with_match_bytes(tuple, mode,
  &up_match, &up_bytes,
@@ -1422,18 +1423,18 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode,
  goto need_opposite_intention;
 
 #ifdef BTR_CUR_HASH_ADAPT
- /* We do a dirty read of btr_search_enabled here. We will
- properly check btr_search_enabled again in
+ /* We do a dirty read of btr_search.enabled here. We will recheck in
  btr_search_build_page_hash_index() before building a page hash
  index, while holding search latch. */
- if (!btr_search_enabled);
+ if (!btr_search.enabled);
  else if (tuple->info_bits & REC_INFO_MIN_REC_FLAG)
  /* This may be a search tuple for btr_pcur_t::restore_position(). */
  ut_ad(tuple->is_metadata() ||
  (tuple->is_metadata(tuple->info_bits ^ REC_STATUS_INSTANT)));
  else if (index()->table->is_temporary());
- else if (!rec_is_metadata(page_cur.rec, *index()))
- btr_search_info_update(index(), this);
+ else if (!rec_is_metadata(page_cur.rec, *index()) &&
+ index()->search_info.hash_analysis_useful())
+ search_info_update();
 #endif /* BTR_CUR_HASH_ADAPT */
 
  goto func_exit;
@@ -1660,18 +1661,18 @@ dberr_t btr_cur_t::pessimistic_search_leaf(const dtuple_t *tuple,
  ut_ad(low_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE);
 
 #ifdef BTR_CUR_HASH_ADAPT
- /* We do a dirty read of btr_search_enabled here. We will
- properly check btr_search_enabled again in
+ /* We do a dirty read of btr_search.enabled here. We will recheck in
  btr_search_build_page_hash_index() before building a page hash
  index, while holding search latch. */
- if (!btr_search_enabled);
+ if (!btr_search.enabled);
  else if (tuple->info_bits & REC_INFO_MIN_REC_FLAG)
  /* This may be a search tuple for btr_pcur_t::restore_position(). */
  ut_ad(tuple->is_metadata() ||
  (tuple->is_metadata(tuple->info_bits ^ REC_STATUS_INSTANT)));
  else if (index()->table->is_temporary());
- else if (!rec_is_metadata(page_cur.rec, *index()))
- btr_search_info_update(index(), this);
+ else if (!rec_is_metadata(page_cur.rec, *index()) &&
+ index()->search_info.hash_analysis_useful())
+ search_info_update();
 #endif /* BTR_CUR_HASH_ADAPT */
  err= DB_SUCCESS;
  }
@@ -1774,8 +1775,7 @@ dberr_t btr_cur_search_to_nth_level(ulint level,
 #ifndef BTR_CUR_ADAPT
  buf_block_t *block= nullptr;
 #else
- btr_search_t *info= btr_search_get_info(index);
- buf_block_t *block= info->root_guess;
+ buf_block_t *block= index->search_info.root_guess;
 #endif /* BTR_CUR_ADAPT */
 
  ut_ad(mtr->memo_contains_flagged(&index->lock,
@@ -2520,12 +2520,10 @@ btr_cur_optimistic_insert(
 ut_ad(flags == BTR_NO_LOCKING_FLAG);
 } else if (index->table->is_temporary()) {
 } else {
-srw_spin_lock* ahi_latch = btr_search_sys.get_latch(*index);
 if (!reorg && cursor->flag == BTR_CUR_HASH) {
-btr_search_update_hash_node_on_insert(
-cursor, ahi_latch);
+btr_search_update_hash_node_on_insert(cursor);
 } else {
-btr_search_update_hash_on_insert(cursor, ahi_latch);
+btr_search_update_hash_on_insert(cursor);
 }
 }
 #endif /* BTR_CUR_HASH_ADAPT */
@@ -2698,8 +2696,7 @@ btr_cur_pessimistic_insert(
 ut_ad(!(flags & BTR_CREATE_FLAG));
 } else if (index->table->is_temporary()) {
 } else {
-btr_search_update_hash_on_insert(
-cursor, btr_search_sys.get_latch(*index));
+btr_search_update_hash_on_insert(cursor);
 }
 #endif /* BTR_CUR_HASH_ADAPT */
 if (inherit && !(flags & BTR_NO_LOCKING_FLAG)) {
@@ -3249,9 +3246,9 @@ btr_cur_update_in_place(
 
 #ifdef BTR_CUR_HASH_ADAPT
 {
-srw_spin_lock* ahi_latch = block->index
-? btr_search_sys.get_latch(*index) : NULL;
-if (ahi_latch) {
+auto part = block->index
+? &btr_search.get_part(*index) : nullptr;
+if (part) {
 /* TO DO: Can we skip this if none of the fields
 index->search_info->curr_n_fields
 are being updated? */
@@ -3269,7 +3266,7 @@ btr_cur_update_in_place(
 btr_search_update_hash_on_delete(cursor);
 }
 
-ahi_latch->wr_lock(SRW_LOCK_CALL);
+part->latch.wr_lock(SRW_LOCK_CALL);
 }
 
 assert_block_ahi_valid(block);
@@ -3279,8 +3276,8 @@ btr_cur_update_in_place(
  mtr);
 
 #ifdef BTR_CUR_HASH_ADAPT
-if (ahi_latch) {
-ahi_latch->wr_unlock();
+if (part) {
+part->latch.wr_unlock();
 }
 }
 #endif /* BTR_CUR_HASH_ADAPT */