MDEV-25975 innodb_disallow_writes causes shutdown to hang

We will remove the parameter innodb_disallow_writes because it is badly designed and implemented. The parameter was never allowed at startup. It was only internally used by Galera snapshot transfer. If a user executed SET GLOBAL innodb_disallow_writes=ON; the server could hang even on subsequent read operations. During Galera snapshot transfer, we will block writes to implement an rsync friendly snapshot, as follows: sst_flush_tables() will acquire a global lock by executing FLUSH TABLES WITH READ LOCK, which will block any writes at the high level. sst_disable_innodb_writes(), invoked via ha_disable_internal_writes(true), will suspend or disable InnoDB background tasks or threads that could initiate writes. As part of this, log_make_checkpoint() will be invoked to ensure that anything in the InnoDB buf_pool.flush_list will be written to the data files. This has the nice side effect that the Galera joiner will avoid crash recovery. The changes to sql/wsrep.cc and to the tests are based on a prototype that was developed by Jan Lindström. Reviewed by: Jan Lindström

Author: dr-m

extra/mariabackup/xtrabackup.cc

@@ -4372,11 +4372,6 @@ static bool xtrabackup_backup_func()
 crc_init();
 recv_sys_init();
 
-#ifdef WITH_INNODB_DISALLOW_WRITES
-srv_allow_writes_event = os_event_create(0);
-os_event_set(srv_allow_writes_event);
-#endif
-
 xb_filters_init();
 
 xb_fil_io_init();
@@ -5837,10 +5832,6 @@ static bool xtrabackup_prepare_func(char** argv)
 log_sys.create();
 recv_recovery_on = true;
 
-#ifdef WITH_INNODB_DISALLOW_WRITES
-srv_allow_writes_event = os_event_create(0);
-os_event_set(srv_allow_writes_event);
-#endif
 dberr_t err = xb_data_files_init();
 if (err != DB_SUCCESS) {
 msg("mariabackup: error: xb_data_files_init() failed "
@@ -5862,9 +5853,6 @@ static bool xtrabackup_prepare_func(char** argv)
 xb_filter_hash_free(inc_dir_tables_hash);
 
 fil_system.close();
-#ifdef WITH_INNODB_DISALLOW_WRITES
-os_event_destroy(srv_allow_writes_event);
-#endif
 innodb_free_param();
 log_sys.close();
 sync_check_close();

mysql-test/include/have_innodb_disallow_writes.inc

@@ -0,0 +1,11 @@
+connection node_1;
+connection node_2;
+connection node_1;
+CREATE TABLE t1 (f1 INT PRIMARY KEY);
+connection node_2;
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*");
+SET DEBUG_SYNC = 'wsrep_before_certification WAIT_FOR continue';
+INSERT INTO t1 VALUES (1);
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_1;
+DROP TABLE t1;

mysql-test/suite/galera/r/galera_bf_abort_shutdown.result

@@ -46,3 +46,6 @@ SHOW TABLES;
 Tables_in_fts
 DROP DATABASE fts;
 connection node_2;
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:");
+Warnings:
+Note	1049	Unknown database 'fts'

mysql-test/suite/galera/r/galera_drop_database.result

@@ -109,6 +109,7 @@ f1	f2
 SELECT EVENT_CATALOG, EVENT_SCHEMA, EVENT_NAME, DEFINER, EVENT_BODY, EVENT_DEFINITION, EVENT_TYPE, EXECUTE_AT, INTERVAL_VALUE, INTERVAL_FIELD, STATUS,ON_COMPLETION, EVENT_COMMENT FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_NAME='one_event';
 EVENT_CATALOG	EVENT_SCHEMA	EVENT_NAME	DEFINER	EVENT_BODY	EVENT_DEFINITION	EVENT_TYPE	EXECUTE_AT	INTERVAL_VALUE	INTERVAL_FIELD	STATUS	ON_COMPLETION	EVENT_COMMENT
 def	test	one_event	root@localhost	SQL	SELECT 123	RECURRING	NULL	10	SECOND	SLAVESIDE_DISABLED	NOT PRESERVE
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*");
 connection node_1;
 SELECT * FROM t1;
 f1	f2

mysql-test/suite/galera/r/galera_events2.result

@@ -39,6 +39,7 @@ f1	f2
 connection node_2;
 Starting server ...
 Starting server ...
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:");
 SELECT * FROM t1;
 f1	f2
 1	a

mysql-test/suite/galera/r/galera_restart_on_unknown_option.result

@@ -0,0 +1,34 @@
+#
+# This test verifies that the server can be shut down even if
+# some of the wsrep transactions are in QUERY_COMMITTING state.
+# In this case the shutdown sequence may do a BF abort for the
+# connection.
+#
+
+--source include/have_innodb.inc
+--source include/galera_cluster.inc
+--source include/have_debug_sync.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--connection node_1
+CREATE TABLE t1 (f1 INT PRIMARY KEY);
+
+--connection node_2
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*");
+SET DEBUG_SYNC = 'wsrep_before_certification WAIT_FOR continue';
+--send INSERT INTO t1 VALUES (1)
+
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--source include/restart_mysqld.inc
+
+# Restore original auto_increment_offset values.
+--let $node_2=node_2a
+--source include/auto_increment_offset_restore.inc
+
+--connection node_1
+
+DROP TABLE t1;

mysql-test/suite/galera/r/galera_var_innodb_disallow_writes.result

@@ -56,6 +56,7 @@ SHOW TABLES;
 DROP DATABASE fts;
 
 --connection node_2
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:");
 --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'fts_t1';
 --source include/wait_condition.inc
 --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'fts_t2';

mysql-test/suite/galera/t/galera_bf_abort_shutdown.test

@@ -137,6 +137,8 @@ SELECT * FROM t1;
 --echo # node_2 Event should be SERVERSIDE_DISABLED
 SELECT EVENT_CATALOG, EVENT_SCHEMA, EVENT_NAME, DEFINER, EVENT_BODY, EVENT_DEFINITION, EVENT_TYPE, EXECUTE_AT, INTERVAL_VALUE, INTERVAL_FIELD, STATUS,ON_COMPLETION, EVENT_COMMENT FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_NAME='one_event';
 
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*");
+
 --connection node_1
 SELECT * FROM t1;
 --echo # node_1 Event should be ENABLED