BUG#32212353: IMPROVE WRITE SET COLLECTION MEMORY USAGE HANDLING

When collecting write sets for transactions, the server might run out of memory. This failure shall be handled gracefully and the transaction shall fail for lack of resources. ReviewBoard: 25572 (cherry picked from commit be0fb117b842d34a7c3725dfa2f346b33ae4aa73)

Author: pedrolgomes

sql/handler.cc

@@ -7931,31 +7931,39 @@ int binlog_log_row(TABLE* table,
  {
  if (thd->variables.transaction_write_set_extraction != HASH_ALGORITHM_OFF)
  {
- if (before_record && after_record)
+ try
  {
- size_t length= table->s->reclength;
- uchar* temp_image=(uchar*) my_malloc(PSI_NOT_INSTRUMENTED,
- length,
- MYF(MY_WME));
- if (!temp_image)
+ if (before_record && after_record)
  {
- sql_print_error("Out of memory on transaction write set extraction");
- return 1;
- }
- add_pke(table, thd);
+ size_t length= table->s->reclength;
+ uchar* temp_image=(uchar*) my_malloc(PSI_NOT_INSTRUMENTED,
+ length,
+ MYF(MY_WME));
+ if (!temp_image)
+ {
+ sql_print_error("Out of memory on transaction write set extraction");
+ return 1;
+ }
+ add_pke(table, thd);
 
- memcpy(temp_image, table->record[0],(size_t) table->s->reclength);
- memcpy(table->record[0],table->record[1],(size_t) table->s->reclength);
+  memcpy(temp_image, table->record[0],(size_t) table->s->reclength);
+  memcpy(table->record[0],table->record[1],(size_t) table->s->reclength);
 
- add_pke(table, thd);
+  add_pke(table, thd);
 
- memcpy(table->record[0], temp_image, (size_t) table->s->reclength);
+  memcpy(table->record[0], temp_image, (size_t) table->s->reclength);
 
- my_free(temp_image);
+ my_free(temp_image);
+ }
+ else
+ {
+ add_pke(table, thd);
+ }
  }
- else
+ catch (const std::bad_alloc &)
  {
- add_pke(table, thd);
+ my_error(ER_OUT_OF_RESOURCES, MYF(0));
+ return HA_ERR_RBR_LOGGING_FAILED;
  }
  }
  DBUG_DUMP("read_set 10", (uchar*) table->read_set->bitmap,

sql/rpl_transaction_write_set_ctx.cc

@@ -1,4 +1,4 @@
-/* Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2014, 2020, Oracle and/or its affiliates. All rights reserved.
 
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License, version 2.0,
@@ -45,10 +45,10 @@ Rpl_transaction_write_set_ctx::Rpl_transaction_write_set_ctx():
 
 void Rpl_transaction_write_set_ctx::add_write_set(uint64 hash)
 {
- DBUG_ENTER("Rpl_transaction_write_set_ctx::add_write_set");
+ //Do not use DBUG_ENTER here due to memory allocation exceptions
+ DBUG_EXECUTE_IF("add_write_set_no_memory", throw std::bad_alloc(););
  write_set.push_back(hash);
  write_set_unique.insert(hash);
- DBUG_VOID_RETURN;
 }
 
 std::set<uint64>* Rpl_transaction_write_set_ctx::get_write_set()

sql/rpl_write_set_handler.cc

@@ -576,7 +576,6 @@ static void generate_hash_pke(const std::string &pke, uint collation_conversion_
 #endif
 )
 {
- DBUG_ENTER("generate_hash_pke");
  DBUG_ASSERT(thd->variables.transaction_write_set_extraction !=
  HASH_ALGORITHM_OFF);
 
@@ -591,13 +590,11 @@ static void generate_hash_pke(const std::string &pke, uint collation_conversion_
  hash_list.push_back(hash);
 #endif
  DBUG_PRINT("info", ("pke: %s; hash: %llu", pke.c_str(), hash));
- DBUG_VOID_RETURN;
 }
 
 
 void add_pke(TABLE *table, THD *thd)
 {
- DBUG_ENTER("add_pke");
  /*
  The next section extracts the primary key equivalent of the rows that are
  changing during the current transaction.
@@ -896,6 +893,4 @@ void add_pke(TABLE *table, THD *thd)
 
  if (writeset_hashes_added == 0)
  ws_ctx->set_has_missing_keys();
-
- DBUG_VOID_RETURN;
 }