MDEV-16249 CHECKSUM TABLE for a spider table is not parallel and saves all data in memory in the spider head by default (#1328)

add checksum_null for setting null value of checksum

Author: Kentoku

sql/ha_partition.cc

@@ -8200,6 +8200,7 @@ int ha_partition::info(uint flag)
  stats.delete_length= 0;
  stats.check_time= 0;
  stats.checksum= 0;
+ stats.checksum_null= TRUE;
  for (i= bitmap_get_first_set(&m_part_info->read_partitions);
  i < m_tot_parts;
  i= bitmap_get_next_set(&m_part_info->read_partitions, i))
@@ -8213,7 +8214,11 @@ int ha_partition::info(uint flag)
  stats.delete_length+= file->stats.delete_length;
  if (file->stats.check_time > stats.check_time)
  stats.check_time= file->stats.check_time;
- stats.checksum+= file->stats.checksum;
+ if (!file->stats.checksum_null)
+ {
+ stats.checksum+= file->stats.checksum;
+ stats.checksum_null= FALSE;
+ }
  }
  if (stats.records && stats.records < 2 &&
  !(m_file[0]->ha_table_flags() & HA_STATS_RECORDS_IS_EXACT))
@@ -8370,6 +8375,7 @@ void ha_partition::get_dynamic_partition_info(PARTITION_STATS *stat_info,
  stat_info->update_time= file->stats.update_time;
  stat_info->check_time= file->stats.check_time;
  stat_info->check_sum= file->stats.checksum;
+ stat_info->check_sum_null= file->stats.checksum_null;
 }
 
 
@@ -10599,6 +10605,66 @@ void ha_partition::init_table_handle_for_HANDLER()
 }
 
 
+/**
+ Calculate the checksum of the table (all partitions)
+*/
+
+int ha_partition::pre_calculate_checksum()
+{
+ int error;
+ DBUG_ENTER("ha_partition::pre_calculate_checksum");
+ m_pre_calling= TRUE;
+ if ((table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM)))
+ {
+ handler **file= m_file;
+ do
+ {
+ if ((error= (*file)->pre_calculate_checksum()))
+ {
+ DBUG_RETURN(error);
+ }
+ } while (*(++file));
+ }
+ DBUG_RETURN(0);
+}
+
+
+int ha_partition::calculate_checksum()
+{
+ int error;
+ stats.checksum= 0;
+ stats.checksum_null= TRUE;
+
+ DBUG_ENTER("ha_partition::calculate_checksum");
+ if (!m_pre_calling)
+ {
+ if ((error= pre_calculate_checksum()))
+ {
+ m_pre_calling= FALSE;
+ DBUG_RETURN(error);
+ }
+ }
+ m_pre_calling= FALSE;
+ if ((table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM)))
+ {
+ handler **file= m_file;
+ do
+ {
+ if ((error= (*file)->calculate_checksum()))
+ {
+ DBUG_RETURN(error);
+ }
+ if (!(*file)->stats.checksum_null)
+ {
+ stats.checksum+= (*file)->stats.checksum;
+ stats.checksum_null= FALSE;
+ }
+ } while (*(++file));
+ }
+ DBUG_RETURN(0);
+}
+
+
 /****************************************************************************
  MODULE enable/disable indexes
 ****************************************************************************/

sql/ha_partition.h

@@ -1443,6 +1443,9 @@ class ha_partition :public handler
  void append_row_to_str(String &str);
  public:
 
+ virtual int pre_calculate_checksum();
+ virtual int calculate_checksum();
+
  /* Enabled keycache for performance reasons, WL#4571 */
  virtual int assign_to_keycache(THD* thd, HA_CHECK_OPT *check_opt);
  virtual int preload_keys(THD* thd, HA_CHECK_OPT* check_opt);

sql/handler.cc

@@ -4899,6 +4899,7 @@ void handler::get_dynamic_partition_info(PARTITION_STATS *stat_info,
  stat_info->update_time= stats.update_time;
  stat_info->check_time= stats.check_time;
  stat_info->check_sum= stats.checksum;
+ stat_info->check_sum_null= stats.checksum_null;
 }
 
 

sql/handler.h

@@ -1922,6 +1922,7 @@ typedef struct {
  time_t check_time;
  time_t update_time;
  ulonglong check_sum;
+ bool check_sum_null;
 } PARTITION_STATS;
 
 #define UNDEF_NODEGROUP 65535
@@ -2881,6 +2882,7 @@ class ha_statistics
  time_t update_time;
  uint block_size;/* index block size */
  ha_checksum checksum;
+ bool checksum_null;
 
  /*
  number of buffer bytes that native mrr implementation needs,
@@ -2892,7 +2894,7 @@ class ha_statistics
  index_file_length(0), max_index_file_length(0), delete_length(0),
  auto_increment_value(0), records(0), deleted(0), mean_rec_length(0),
  create_time(0), check_time(0), update_time(0), block_size(0),
- mrr_length_per_rec(0)
+ checksum(0), checksum_null(FALSE), mrr_length_per_rec(0)
  {}
 };
 
@@ -3925,6 +3927,7 @@ class handler :public Sql_alloc
  virtual uint max_supported_key_part_length() const { return 255; }
  virtual uint min_record_length(uint options) const { return 1; }
 
+ virtual int pre_calculate_checksum() { return 0; }
  virtual int calculate_checksum();
  virtual bool is_crashed() const { return 0; }
  virtual bool auto_repair(int error) const { return 0; }

sql/sql_show.cc

@@ -5668,7 +5668,9 @@ static int get_schema_tables_record(THD *thd, TABLE_LIST *tables,
  table->field[16]->store_time(&time);
  table->field[16]->set_notnull();
  }
- if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM))
+ if ((file->ha_table_flags() &
+ (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM)) &&
+ !file->stats.checksum_null)
  {
  table->field[18]->store((longlong) file->stats.checksum, TRUE);
  table->field[18]->set_notnull();

sql/sql_table.cc

@@ -10936,7 +10936,7 @@ bool mysql_checksum_table(THD *thd, TABLE_LIST *tables,
  (((t->file->ha_table_flags() & HA_HAS_OLD_CHECKSUM) && thd->variables.old_mode) ||
  ((t->file->ha_table_flags() & HA_HAS_NEW_CHECKSUM) && !thd->variables.old_mode)))
  {
- if (t->file->info(HA_STATUS_VARIABLE))
+ if (t->file->info(HA_STATUS_VARIABLE) || t->file->stats.checksum_null)
  protocol->store_null();
  else
  protocol->store((longlong)t->file->stats.checksum);
@@ -10956,7 +10956,7 @@ bool mysql_checksum_table(THD *thd, TABLE_LIST *tables,
  thd->protocol->remove_last_row();
  goto err;
  }
- if (error)
+ if (error || t->file->stats.checksum_null)
  protocol->store_null();
  else
  protocol->store((longlong)t->file->stats.checksum);

storage/spider/ha_spider.cc

@@ -1,4 +1,5 @@
-/* Copyright (C) 2008-2018 Kentoku Shiba
+/* Copyright (C) 2008-2019 Kentoku Shiba
+ Copyright (C) 2019 MariaDB corp
 
  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
@@ -116,7 +117,7 @@ ha_spider::ha_spider(
  use_fields = FALSE;
 #endif
  use_pre_call = FALSE;
- use_pre_records = FALSE;
+ use_pre_action = FALSE;
 #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS
  do_direct_update = FALSE;
 #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET)
@@ -228,7 +229,7 @@ ha_spider::ha_spider(
  use_fields = FALSE;
 #endif
  use_pre_call = FALSE;
- use_pre_records = FALSE;
+ use_pre_action = FALSE;
 #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS
  do_direct_update = FALSE;
 #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET)
@@ -1805,7 +1806,7 @@ int ha_spider::reset()
  high_priority = FALSE;
  insert_delayed = FALSE;
  use_pre_call = FALSE;
- use_pre_records = FALSE;
+ use_pre_action = FALSE;
  pre_bitmap_checked = FALSE;
  bulk_insert = FALSE;
  clone_bitmap_init = FALSE;
@@ -8760,22 +8761,24 @@ int ha_spider::info(
  }
 
  if (flag & HA_STATUS_TIME)
- stats.update_time = (ulong) share->update_time;
+ stats.update_time = (ulong) share->stat.update_time;
  if (flag & (HA_STATUS_CONST | HA_STATUS_VARIABLE))
  {
- stats.max_data_file_length = share->max_data_file_length;
- stats.create_time = (ulong) share->create_time;
+ stats.max_data_file_length = share->stat.max_data_file_length;
+ stats.create_time = share->stat.create_time;
  stats.block_size = spider_param_block_size(thd);
  }
  if (flag & HA_STATUS_VARIABLE)
  {
- stats.data_file_length = share->data_file_length;
- stats.index_file_length = share->index_file_length;
- stats.records = share->records;
- stats.mean_rec_length = share->mean_rec_length;
- stats.check_time = (ulong) share->check_time;
+ stats.data_file_length = share->stat.data_file_length;
+ stats.index_file_length = share->stat.index_file_length;
+ stats.records = share->stat.records;
+ stats.mean_rec_length = share->stat.mean_rec_length;
+ stats.check_time = share->stat.check_time;
  if (stats.records <= 1 /* && (flag & HA_STATUS_NO_LOCK) */ )
  stats.records = 2;
+ stats.checksum = share->stat.checksum;
+ stats.checksum_null = share->stat.checksum_null;
  }
  if (flag & HA_STATUS_AUTO)
  {
@@ -9014,7 +9017,7 @@ ha_rows ha_spider::records_in_range(
  key_part_map tgt_key_part_map;
  KEY_PART_INFO *key_part;
  Field *field = NULL;
- double rows = (double) share->records;
+ double rows = (double) share->stat.records;
  double weight, rate;
  DBUG_PRINT("info",("spider rows1=%f", rows));
  if (start_key)
@@ -9323,11 +9326,12 @@ int ha_spider::pre_records()
  result_list.casual_read[search_link_idx] =
  spider_param_casual_read(thd, share->casual_read);
  }
- if ((error_num = spider_db_show_records(this, search_link_idx, TRUE)))
+ if ((error_num = spider_db_simple_action(SPIDER_SIMPLE_RECORDS, this,
+ search_link_idx, TRUE)))
  {
  DBUG_RETURN(check_error_mode(error_num));
  }
- use_pre_records = TRUE;
+ use_pre_action = TRUE;
  DBUG_RETURN(0);
 }
 
@@ -9339,14 +9343,14 @@ ha_rows ha_spider::records()
  DBUG_PRINT("info",("spider this=%p", this));
  if (sql_command == SQLCOM_ALTER_TABLE)
  {
- use_pre_records = FALSE;
+ use_pre_action = FALSE;
  DBUG_RETURN(0);
  }
  if (!(share->additional_table_flags & HA_HAS_RECORDS) && !this->result_list.direct_limit_offset)
  {
  DBUG_RETURN(handler::records());
  }
- if (!use_pre_records && !this->result_list.direct_limit_offset)
+ if (!use_pre_action && !this->result_list.direct_limit_offset)
  {
  THD *thd = trx->thd;
  if (
@@ -9357,17 +9361,84 @@ ha_rows ha_spider::records()
  spider_param_casual_read(thd, share->casual_read);
  }
  }
- if ((error_num = spider_db_show_records(this, search_link_idx, FALSE)))
+ if ((error_num = spider_db_simple_action(SPIDER_SIMPLE_RECORDS, this,
+ search_link_idx, FALSE)))
  {
- use_pre_records = FALSE;
+ use_pre_action = FALSE;
  check_error_mode(error_num);
  DBUG_RETURN(HA_POS_ERROR);
  }
- use_pre_records = FALSE;
- share->records = table_rows;
+ use_pre_action = FALSE;
+ share->stat.records = table_rows;
  DBUG_RETURN(table_rows);
 }
 
+#ifdef HA_HAS_CHECKSUM_EXTENDED
+int ha_spider::pre_calculate_checksum()
+{
+ int error_num;
+ backup_error_status();
+ DBUG_ENTER("ha_spider::pre_calculate_checksum");
+ DBUG_PRINT("info",("spider this=%p", this));
+ THD *thd = trx->thd;
+ if (
+ spider_param_sync_autocommit(thd) &&
+ (!thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))
+ ) {
+ result_list.casual_read[search_link_idx] =
+ spider_param_casual_read(thd, share->casual_read);
+ }
+ action_flags = T_EXTEND;
+ if ((error_num = spider_db_simple_action(SPIDER_SIMPLE_CHECKSUM_TABLE, this,
+ search_link_idx, TRUE)))
+ {
+ DBUG_RETURN(check_error_mode(error_num));
+ }
+ use_pre_action = TRUE;
+ DBUG_RETURN(0);
+}
+
+int ha_spider::calculate_checksum()
+{
+ int error_num;
+ backup_error_status();
+ DBUG_ENTER("ha_spider::calculate_checksum");
+ DBUG_PRINT("info",("spider this=%p", this));
+ if (!use_pre_action && !this->result_list.direct_limit_offset)
+ {
+ THD *thd = trx->thd;
+ if (
+ spider_param_sync_autocommit(thd) &&
+ (!thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))
+ ) {
+ result_list.casual_read[search_link_idx] =
+ spider_param_casual_read(thd, share->casual_read);
+ }
+ }
+ action_flags = T_EXTEND;
+ if ((error_num = spider_db_simple_action(SPIDER_SIMPLE_CHECKSUM_TABLE, this,
+ search_link_idx, FALSE)))
+ {
+ use_pre_action = FALSE;
+ DBUG_RETURN(check_error_mode(error_num));
+ }
+ use_pre_action = FALSE;
+ if (checksum_null)
+ {
+ share->stat.checksum_null = TRUE;
+ share->stat.checksum = 0;
+ stats.checksum_null = TRUE;
+ stats.checksum = 0;
+ } else {
+ share->stat.checksum_null = FALSE;
+ share->stat.checksum = checksum_val;
+ stats.checksum_null = FALSE;
+ stats.checksum = checksum_val;
+ }
+ DBUG_RETURN(0);
+}
+#endif
+
 const char *ha_spider::table_type() const
 {
  DBUG_ENTER("ha_spider::table_type");
@@ -11172,8 +11243,9 @@ double ha_spider::scan_time()
  DBUG_ENTER("ha_spider::scan_time");
  DBUG_PRINT("info",("spider this=%p", this));
  DBUG_PRINT("info",("spider scan_time = %.6f",
- share->scan_rate * share->records * share->mean_rec_length + 2));
- DBUG_RETURN(share->scan_rate * share->records * share->mean_rec_length + 2);
+ share->scan_rate * share->stat.records * share->stat.mean_rec_length + 2));
+ DBUG_RETURN(share->scan_rate * share->stat.records *
+ share->stat.mean_rec_length + 2);
 }
 
 double ha_spider::read_time(
@@ -11192,8 +11264,8 @@ double ha_spider::read_time(
  rows / 2 + 2);
  } else {
  DBUG_PRINT("info",("spider read_time = %.6f",
- share->read_rate * share->mean_rec_length * rows + 2));
- DBUG_RETURN(share->read_rate * share->mean_rec_length * rows + 2);
+ share->read_rate * share->stat.mean_rec_length * rows + 2));
+ DBUG_RETURN(share->read_rate * share->stat.mean_rec_length * rows + 2);
  }
 }