MDEV-17223 Assertion `thd->killed != 0' failed in ha_maria::enable_indexes

MDEV-22500 Assertion `thd->killed != 0' failed in ha_maria::enable_indexes For MDEV-17223 the issue was an assert that didn't take into account that we could get duplicate key errors when enablling unique indexes. Fixed by not retrying repair in case of duplicate key error for this case, which avoids the assert. For MDEV-22500 I removed the assert, as it's not critical (just a way to find potential wrong code) and we will anyway get things logged in the error log if this happens. This case cannot triggered an assert in 10.3 but I verified that it would trigger in 10.5 and that this patch fixes it.

Author: montywi

mysql-test/suite/maria/repair.result

@@ -31,3 +31,34 @@ SET max_session_mem_used=50000;
 REPAIR LOCAL TABLE t1 USE_FRM;
 REPAIR LOCAL TABLE t1;
 DROP TABLE t1;
+SET max_session_mem_used=default;
+
+# MDEV-17223 Assertion `thd->killed != 0' failed in
+# ha_maria::enable_indexes
+#
+CREATE OR REPLACE TABLE t1 (c VARCHAR(1024) NOT NULL) ENGINE=Aria ROW_FORMAT FIXED;
+insert into t1 select char(seq) from seq_65_to_256;
+insert into t1 values ("a");
+ALTER TABLE t1 ADD PRIMARY KEY(c(67));
+ERROR 23000: Duplicate entry 'a' for key 'PRIMARY'
+select count(*) from t1;
+count(*)
+193
+drop table t1;
+
+# MDEV-17223 Assertion `thd->killed != 0' failed in
+# ha_maria::enable_indexes
+#
+SET SESSION aria_sort_buffer_size=1023;
+Warnings:
+Warning	1292	Truncated incorrect aria_sort_buffer_size value: '1023'
+CREATE TABLE t2 (c TEXT,INDEX(c(1000))) ENGINE=Aria;
+INSERT INTO t2 select char(seq) from seq_65_to_255;
+SELECT COUNT(*) FROM t2;
+COUNT(*)
+191
+DROP TABLE t2;
+SET SESSION aria_sort_buffer_size=default;
+#
+# End of 10.3 tests
+#

mysql-test/suite/maria/repair.test

@@ -3,6 +3,7 @@
 # as memory usage is different compared to normal server.
 
 --source include/not_embedded.inc
+--source include/have_sequence.inc
 
 #
 # MDEV-11539 test_if_reopen: Assertion `strcmp(share->unique_file_name,filename) || share->last_version' failed upon select from I_S
@@ -41,3 +42,35 @@ REPAIR LOCAL TABLE t1 USE_FRM;
 REPAIR LOCAL TABLE t1;
 --enable_result_log
 DROP TABLE t1;
+SET max_session_mem_used=default;
+
+--echo
+--echo # MDEV-17223 Assertion `thd->killed != 0' failed in
+--echo # ha_maria::enable_indexes
+--echo #
+
+CREATE OR REPLACE TABLE t1 (c VARCHAR(1024) NOT NULL) ENGINE=Aria ROW_FORMAT FIXED;
+insert into t1 select char(seq) from seq_65_to_256;
+insert into t1 values ("a");
+--error ER_DUP_ENTRY
+ALTER TABLE t1 ADD PRIMARY KEY(c(67));
+select count(*) from t1;
+drop table t1;
+
+--echo
+--echo # MDEV-17223 Assertion `thd->killed != 0' failed in
+--echo # ha_maria::enable_indexes
+--echo #
+
+SET SESSION aria_sort_buffer_size=1023;
+CREATE TABLE t2 (c TEXT,INDEX(c(1000))) ENGINE=Aria;
+--disable_warnings
+INSERT INTO t2 select char(seq) from seq_65_to_255;
+--enable_warnings
+SELECT COUNT(*) FROM t2;
+DROP TABLE t2;
+SET SESSION aria_sort_buffer_size=default;
+
+--echo #
+--echo # End of 10.3 tests
+--echo #

storage/maria/ha_maria.cc

@@ -1958,13 +1958,19 @@ int ha_maria::enable_indexes(uint mode)
  param->sort_buffer_length= THDVAR(thd,sort_buffer_size);
  param->stats_method= (enum_handler_stats_method)THDVAR(thd,stats_method);
  param->tmpdir= &mysql_tmpdir_list;
- if ((error= (repair(thd, param, 0) != HA_ADMIN_OK)) && param->retry_repair)
+
+ /*
+ Don't retry repair if we get duplicate key error if
+ create_unique_index_by_sort is enabled
+ This can be set when doing an ALTER TABLE and enabling unique keys
+ */
+ if ((error= (repair(thd, param, 0) != HA_ADMIN_OK)) && param->retry_repair &&
+ (my_errno != HA_ERR_FOUND_DUPP_KEY ||
+ !file->create_unique_index_by_sort))
  {
  sql_print_warning("Warning: Enabling keys got errno %d on %s.%s, "
  "retrying",
  my_errno, param->db_name, param->table_name);
- /* This should never fail normally */
- DBUG_ASSERT(thd->killed != 0);
  /* Repairing by sort failed. Now try standard repair method. */
  param->testflag &= ~T_REP_BY_SORT;
  file->state->records= start_rows;