Skip to content

Commit 410e3c1

Browse files
sujatha-ssujatha-s
committed
MDEV-17515: GTID Replication in optimistic mode deadlock
Problem: ======= In slave_parallel_mode=optimistic configuration, when admin commands and DML operation on the same table are scheduled simultaneously for execution, it results in lock conflict and slave server either hangs due to deadlock or goes down with an assert. Analysis: ======== Admin commands OPTIMIZE, REPAIR and ANALYZE are written to binary log as ordinary transactions. When 'slave_parallel_mode' is 'optimistic' DMLs are allowed to run in parallel. But these locks are not detected by parallel replication deadlock detection-and-handling mechanism. At times they result in deadlock or assertion. Fix: === Flag admin commands as DDL in Gtid_log_event at the time of writing to binary log. Add a new bit EXECUTED_TABLE_ADMIN_CMD to 'm_unsafe_rollback_flags'. During 'mysql_admin_table' command execution it accepts a list of tables to be processed and executes them in a loop. Upon successful execution enable 'EXECUTED_TABLE_ADMIN_CMD' bit in thd->transaction.stmt_unsafe_rollback_flags. Gtid_log_event constructor will notice this flag and mark the current transaction with 'FL_DDL' flag. Gtid_log_events marked as FL_DDL will not be scheduled parallel execution, on the slave. They will execute in isolation to prevent deadlocks. Note: Removed the call to 'trans_commit_implicit' from 'mysql_admin_table' function as 'mysql_execute_command' will take care of invoking 'trans_commit_implicit'.
1 parent 80ae367 commit 410e3c1

File tree

8 files changed

+257
-38
lines changed

8 files changed

+257
-38
lines changed

‎mysql-test/include/commit.inc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -770,7 +770,7 @@ call p_verify_status_increment(2, 0, 2, 0);
770770
commit;
771771
call p_verify_status_increment(0, 0, 0, 0);
772772
check table t1, t2, t3;
773-
call p_verify_status_increment(6, 0, 6, 0);
773+
call p_verify_status_increment(4, 0, 4, 0);
774774
commit;
775775
call p_verify_status_increment(0, 0, 0, 0);
776776
drop view v1;

‎mysql-test/r/commit_1innodb.result‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -873,7 +873,7 @@ Table Op Msg_type Msg_text
873873
test.t1 check status OK
874874
test.t2 check status OK
875875
test.t3 check status OK
876-
call p_verify_status_increment(6, 0, 6, 0);
876+
call p_verify_status_increment(4, 0, 4, 0);
877877
SUCCESS
878878

879879
commit;

‎mysql-test/suite/rpl/r/rpl_mark_optimize_tbl_ddl.result‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
include/rpl_init.inc [topology=1->2]
2+
connection server_1;
3+
FLUSH TABLES;
4+
ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
5+
connection server_2;
6+
SET @save_slave_parallel_threads= @@GLOBAL.slave_parallel_threads;
7+
SET @save_slave_parallel_mode= @@GLOBAL.slave_parallel_mode;
8+
include/stop_slave.inc
9+
SET GLOBAL slave_parallel_threads=2;
10+
SET GLOBAL slave_parallel_mode=optimistic;
11+
include/start_slave.inc
12+
connection server_1;
13+
CREATE TABLE t1(a INT) ENGINE=INNODB;
14+
OPTIMIZE TABLE t1;
15+
Table Op Msg_type Msg_text
16+
test.t1 optimize note Table does not support optimize, doing recreate + analyze instead
17+
test.t1 optimize status OK
18+
INSERT INTO t1 VALUES(1);
19+
INSERT INTO t1 SELECT 1+a FROM t1;
20+
INSERT INTO t1 SELECT 2+a FROM t1;
21+
connection server_2;
22+
#
23+
# Verify that following admin commands are marked as ddl
24+
# 'OPTIMIZE TABLE', 'REPAIR TABLE' and 'ANALYZE TABLE'
25+
#
26+
connection server_1;
27+
OPTIMIZE TABLE t1;
28+
Table Op Msg_type Msg_text
29+
test.t1 optimize note Table does not support optimize, doing recreate + analyze instead
30+
test.t1 optimize status OK
31+
REPAIR TABLE t1;
32+
Table Op Msg_type Msg_text
33+
test.t1 repair note The storage engine for the table doesn't support repair
34+
ANALYZE TABLE t1;
35+
Table Op Msg_type Msg_text
36+
test.t1 analyze status OK
37+
FLUSH LOGS;
38+
FOUND 1 /GTID 0-1-8 ddl/ in mysqlbinlog.out
39+
FOUND 1 /GTID 0-1-9 ddl/ in mysqlbinlog.out
40+
FOUND 1 /GTID 0-1-10 ddl/ in mysqlbinlog.out
41+
#
42+
# Clean up
43+
#
44+
DROP TABLE t1;
45+
connection server_2;
46+
FLUSH LOGS;
47+
#
48+
# Check that ALTER TABLE commands with ANALYZE, OPTIMIZE and REPAIR on
49+
# partitions will be marked as DDL in binary log.
50+
#
51+
connection server_1;
52+
CREATE TABLE t1(id INT) PARTITION BY RANGE (id) (PARTITION p0 VALUES LESS THAN (100),
53+
PARTITION pmax VALUES LESS THAN (MAXVALUE));
54+
INSERT INTO t1 VALUES (1), (10), (100), (1000);
55+
ALTER TABLE t1 ANALYZE PARTITION p0;
56+
Table Op Msg_type Msg_text
57+
test.t1 analyze status OK
58+
ALTER TABLE t1 OPTIMIZE PARTITION p0;
59+
Table Op Msg_type Msg_text
60+
test.t1 optimize status OK
61+
ALTER TABLE t1 REPAIR PARTITION p0;
62+
Table Op Msg_type Msg_text
63+
test.t1 repair status OK
64+
FLUSH LOGS;
65+
FOUND 1 /GTID 0-1-14 ddl/ in mysqlbinlog.out
66+
FOUND 1 /GTID 0-1-15 ddl/ in mysqlbinlog.out
67+
FOUND 1 /GTID 0-1-16 ddl/ in mysqlbinlog.out
68+
#
69+
# Clean up
70+
#
71+
DROP TABLE t1;
72+
connection server_2;
73+
include/stop_slave.inc
74+
SET GLOBAL slave_parallel_threads= @save_slave_parallel_threads;
75+
SET GLOBAL slave_parallel_mode= @save_slave_parallel_mode;
76+
include/start_slave.inc
77+
include/rpl_end.inc

‎mysql-test/suite/rpl/t/rpl_mark_optimize_tbl_ddl.test‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# ==== Purpose ====
2+
#
3+
# Test verifies that there is no deadlock or assertion in
4+
# slave_parallel_mode=optimistic configuration while applying admin command
5+
# like 'OPTIMIZE TABLE', 'REPAIR TABLE' and 'ANALYZE TABLE'.
6+
#
7+
# ==== Implementation ====
8+
#
9+
# Steps:
10+
# 0 - Create a table, execute OPTIMIZE TABLE command on the table followed
11+
# by some DMLS.
12+
# 1 - No assert should happen on slave server.
13+
# 2 - Assert that 'OPTIMIZE TABLE', 'REPAIR TABLE' and 'ANALYZE TABLE' are
14+
# marked as 'DDL' in the binary log.
15+
#
16+
# ==== References ====
17+
#
18+
# MDEV-17515: GTID Replication in optimistic mode deadlock
19+
#
20+
--source include/have_partition.inc
21+
--source include/have_innodb.inc
22+
--let $rpl_topology=1->2
23+
--source include/rpl_init.inc
24+
25+
--connection server_1
26+
FLUSH TABLES;
27+
ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
28+
29+
--connection server_2
30+
SET @save_slave_parallel_threads= @@GLOBAL.slave_parallel_threads;
31+
SET @save_slave_parallel_mode= @@GLOBAL.slave_parallel_mode;
32+
--source include/stop_slave.inc
33+
SET GLOBAL slave_parallel_threads=2;
34+
SET GLOBAL slave_parallel_mode=optimistic;
35+
--source include/start_slave.inc
36+
37+
--connection server_1
38+
CREATE TABLE t1(a INT) ENGINE=INNODB;
39+
OPTIMIZE TABLE t1;
40+
INSERT INTO t1 VALUES(1);
41+
INSERT INTO t1 SELECT 1+a FROM t1;
42+
INSERT INTO t1 SELECT 2+a FROM t1;
43+
--save_master_pos
44+
45+
--connection server_2
46+
--sync_with_master
47+
48+
--echo #
49+
--echo # Verify that following admin commands are marked as ddl
50+
--echo # 'OPTIMIZE TABLE', 'REPAIR TABLE' and 'ANALYZE TABLE'
51+
--echo #
52+
--connection server_1
53+
54+
OPTIMIZE TABLE t1;
55+
--let optimize_gtid= `SELECT @@GLOBAL.gtid_binlog_pos`
56+
57+
REPAIR TABLE t1;
58+
--let repair_gtid= `SELECT @@GLOBAL.gtid_binlog_pos`
59+
60+
ANALYZE TABLE t1;
61+
--let analyze_gtid= `SELECT @@GLOBAL.gtid_binlog_pos`
62+
63+
let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1);
64+
FLUSH LOGS;
65+
66+
--let $MYSQLD_DATADIR= `select @@datadir`
67+
--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$binlog_file > $MYSQLTEST_VARDIR/tmp/mysqlbinlog.out
68+
69+
--let SEARCH_PATTERN= GTID $optimize_gtid ddl
70+
--let SEARCH_FILE= $MYSQLTEST_VARDIR/tmp/mysqlbinlog.out
71+
--source include/search_pattern_in_file.inc
72+
73+
--let SEARCH_PATTERN= GTID $repair_gtid ddl
74+
--let SEARCH_FILE= $MYSQLTEST_VARDIR/tmp/mysqlbinlog.out
75+
--source include/search_pattern_in_file.inc
76+
77+
--let SEARCH_PATTERN= GTID $analyze_gtid ddl
78+
--let SEARCH_FILE= $MYSQLTEST_VARDIR/tmp/mysqlbinlog.out
79+
--source include/search_pattern_in_file.inc
80+
81+
--echo #
82+
--echo # Clean up
83+
--echo #
84+
DROP TABLE t1;
85+
--remove_file $MYSQLTEST_VARDIR/tmp/mysqlbinlog.out
86+
--save_master_pos
87+
88+
--connection server_2
89+
--sync_with_master
90+
FLUSH LOGS;
91+
92+
--echo #
93+
--echo # Check that ALTER TABLE commands with ANALYZE, OPTIMIZE and REPAIR on
94+
--echo # partitions will be marked as DDL in binary log.
95+
--echo #
96+
--connection server_1
97+
CREATE TABLE t1(id INT) PARTITION BY RANGE (id) (PARTITION p0 VALUES LESS THAN (100),
98+
PARTITION pmax VALUES LESS THAN (MAXVALUE));
99+
INSERT INTO t1 VALUES (1), (10), (100), (1000);
100+
101+
ALTER TABLE t1 ANALYZE PARTITION p0;
102+
--let analyze_gtid= `SELECT @@GLOBAL.gtid_binlog_pos`
103+
104+
ALTER TABLE t1 OPTIMIZE PARTITION p0;
105+
--let optimize_gtid= `SELECT @@GLOBAL.gtid_binlog_pos`
106+
107+
ALTER TABLE t1 REPAIR PARTITION p0;
108+
--let repair_gtid= `SELECT @@GLOBAL.gtid_binlog_pos`
109+
110+
let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1);
111+
FLUSH LOGS;
112+
113+
--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$binlog_file > $MYSQLTEST_VARDIR/tmp/mysqlbinlog.out
114+
115+
--let SEARCH_PATTERN= GTID $analyze_gtid ddl
116+
--let SEARCH_FILE= $MYSQLTEST_VARDIR/tmp/mysqlbinlog.out
117+
--source include/search_pattern_in_file.inc
118+
119+
--let SEARCH_PATTERN= GTID $optimize_gtid ddl
120+
--let SEARCH_FILE= $MYSQLTEST_VARDIR/tmp/mysqlbinlog.out
121+
--source include/search_pattern_in_file.inc
122+
123+
--let SEARCH_PATTERN= GTID $repair_gtid ddl
124+
--let SEARCH_FILE= $MYSQLTEST_VARDIR/tmp/mysqlbinlog.out
125+
--source include/search_pattern_in_file.inc
126+
127+
--echo #
128+
--echo # Clean up
129+
--echo #
130+
DROP TABLE t1;
131+
--remove_file $MYSQLTEST_VARDIR/tmp/mysqlbinlog.out
132+
--save_master_pos
133+
134+
--connection server_2
135+
--sync_with_master
136+
137+
--source include/stop_slave.inc
138+
SET GLOBAL slave_parallel_threads= @save_slave_parallel_threads;
139+
SET GLOBAL slave_parallel_mode= @save_slave_parallel_mode;
140+
--source include/start_slave.inc
141+
142+
--source include/rpl_end.inc

‎sql/handler.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1481,7 +1481,17 @@ struct THD_TRANS
14811481
static unsigned int const DROPPED_TEMP_TABLE= 0x04;
14821482
static unsigned int const DID_WAIT= 0x08;
14831483
static unsigned int const DID_DDL= 0x10;
1484+
static unsigned int const EXECUTED_TABLE_ADMIN_CMD= 0x20;
14841485

1486+
void mark_executed_table_admin_cmd()
1487+
{
1488+
DBUG_PRINT("debug", ("mark_executed_table_admin_cmd"));
1489+
m_unsafe_rollback_flags|= EXECUTED_TABLE_ADMIN_CMD;
1490+
}
1491+
bool trans_executed_admin_cmd()
1492+
{
1493+
return (m_unsafe_rollback_flags & EXECUTED_TABLE_ADMIN_CMD) != 0;
1494+
}
14851495
void mark_created_temp_table()
14861496
{
14871497
DBUG_PRINT("debug", ("mark_created_temp_table"));

‎sql/log_event.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7561,8 +7561,10 @@ Gtid_log_event::Gtid_log_event(THD *thd_arg, uint64 seq_no_arg,
75617561
flags2|= FL_WAITED;
75627562
if (thd_arg->transaction.stmt.trans_did_ddl() ||
75637563
thd_arg->transaction.stmt.has_created_dropped_temp_table() ||
7564+
thd_arg->transaction.stmt.trans_executed_admin_cmd() ||
75647565
thd_arg->transaction.all.trans_did_ddl() ||
7565-
thd_arg->transaction.all.has_created_dropped_temp_table())
7566+
thd_arg->transaction.all.has_created_dropped_temp_table() ||
7567+
thd_arg->transaction.all.trans_executed_admin_cmd())
75667568
flags2|= FL_DDL;
75677569
else if (is_transactional && !is_tmp_table)
75687570
flags2|= FL_TRANSACTIONAL;

‎sql/sql_admin.cc‎

Lines changed: 21 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -434,7 +434,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
434434
int (handler::*operator_func)(THD *,
435435
HA_CHECK_OPT *),
436436
int (view_operator_func)(THD *, TABLE_LIST*,
437-
HA_CHECK_OPT *))
437+
HA_CHECK_OPT *),
438+
bool is_cmd_replicated)
438439
{
439440
TABLE_LIST *table;
440441
List<Item> field_list;
@@ -1147,6 +1148,13 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
11471148
break;
11481149
}
11491150
}
1151+
/*
1152+
Admin commands acquire table locks and these locks are not detected by
1153+
parallel replication deadlock detection-and-handling mechanism. Hence
1154+
they must be marked as DDL so that they are not scheduled in parallel
1155+
with conflicting DMLs resulting in deadlock.
1156+
*/
1157+
thd->transaction.stmt.mark_executed_table_admin_cmd();
11501158
if (table->table && !table->view)
11511159
{
11521160
if (table->table->s->tmp_table)
@@ -1182,9 +1190,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
11821190
}
11831191
else
11841192
{
1185-
if (trans_commit_stmt(thd) ||
1186-
(stmt_causes_implicit_commit(thd, CF_IMPLICIT_COMMIT_END) &&
1187-
trans_commit_implicit(thd)))
1193+
if (trans_commit_stmt(thd))
11881194
goto err;
11891195
}
11901196
close_thread_tables(thd);
@@ -1209,6 +1215,11 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
12091215
if (protocol->write())
12101216
goto err;
12111217
}
1218+
if (is_cmd_replicated && !thd->lex->no_write_to_binlog)
1219+
{
1220+
if (write_bin_log(thd, TRUE, thd->query(), thd->query_length()))
1221+
goto err;
1222+
}
12121223

12131224
my_eof(thd);
12141225
thd->resume_subsequent_commits(suspended_wfc);
@@ -1270,7 +1281,7 @@ bool mysql_assign_to_keycache(THD* thd, TABLE_LIST* tables,
12701281
check_opt.key_cache= key_cache;
12711282
DBUG_RETURN(mysql_admin_table(thd, tables, &check_opt,
12721283
"assign_to_keycache", TL_READ_NO_INSERT, 0, 0,
1273-
0, 0, &handler::assign_to_keycache, 0));
1284+
0, 0, &handler::assign_to_keycache, 0, false));
12741285
}
12751286

12761287

@@ -1297,7 +1308,7 @@ bool mysql_preload_keys(THD* thd, TABLE_LIST* tables)
12971308
*/
12981309
DBUG_RETURN(mysql_admin_table(thd, tables, 0,
12991310
"preload_keys", TL_READ_NO_INSERT, 0, 0, 0, 0,
1300-
&handler::preload_keys, 0));
1311+
&handler::preload_keys, 0, false));
13011312
}
13021313

13031314

@@ -1315,15 +1326,7 @@ bool Sql_cmd_analyze_table::execute(THD *thd)
13151326
WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, first_table);
13161327
res= mysql_admin_table(thd, first_table, &m_lex->check_opt,
13171328
"analyze", lock_type, 1, 0, 0, 0,
1318-
&handler::ha_analyze, 0);
1319-
/* ! we write after unlocking the table */
1320-
if (!res && !m_lex->no_write_to_binlog)
1321-
{
1322-
/*
1323-
Presumably, ANALYZE and binlog writing doesn't require synchronization
1324-
*/
1325-
res= write_bin_log(thd, TRUE, thd->query(), thd->query_length());
1326-
}
1329+
&handler::ha_analyze, 0, true);
13271330
m_lex->select_lex.table_list.first= first_table;
13281331
m_lex->query_tables= first_table;
13291332

@@ -1346,7 +1349,7 @@ bool Sql_cmd_check_table::execute(THD *thd)
13461349
goto error; /* purecov: inspected */
13471350
res= mysql_admin_table(thd, first_table, &m_lex->check_opt, "check",
13481351
lock_type, 0, 0, HA_OPEN_FOR_REPAIR, 0,
1349-
&handler::ha_check, &view_check);
1352+
&handler::ha_check, &view_check, false);
13501353

13511354
m_lex->select_lex.table_list.first= first_table;
13521355
m_lex->query_tables= first_table;
@@ -1371,15 +1374,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd)
13711374
mysql_recreate_table(thd, first_table, true) :
13721375
mysql_admin_table(thd, first_table, &m_lex->check_opt,
13731376
"optimize", TL_WRITE, 1, 0, 0, 0,
1374-
&handler::ha_optimize, 0);
1375-
/* ! we write after unlocking the table */
1376-
if (!res && !m_lex->no_write_to_binlog)
1377-
{
1378-
/*
1379-
Presumably, OPTIMIZE and binlog writing doesn't require synchronization
1380-
*/
1381-
res= write_bin_log(thd, TRUE, thd->query(), thd->query_length());
1382-
}
1377+
&handler::ha_optimize, 0, true);
13831378
m_lex->select_lex.table_list.first= first_table;
13841379
m_lex->query_tables= first_table;
13851380

@@ -1404,16 +1399,8 @@ bool Sql_cmd_repair_table::execute(THD *thd)
14041399
TL_WRITE, 1,
14051400
MY_TEST(m_lex->check_opt.sql_flags & TT_USEFRM),
14061401
HA_OPEN_FOR_REPAIR, &prepare_for_repair,
1407-
&handler::ha_repair, &view_repair);
1402+
&handler::ha_repair, &view_repair, true);
14081403

1409-
/* ! we write after unlocking the table */
1410-
if (!res && !m_lex->no_write_to_binlog)
1411-
{
1412-
/*
1413-
Presumably, REPAIR and binlog writing doesn't require synchronization
1414-
*/
1415-
res= write_bin_log(thd, TRUE, thd->query(), thd->query_length());
1416-
}
14171404
m_lex->select_lex.table_list.first= first_table;
14181405
m_lex->query_tables= first_table;
14191406

0 commit comments

Comments
 (0)