MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys

Add error handling to btr_validate_index when index root block can't be read because block decryption fails.

Author: janlindstrom

mysql-test/suite/encryption/r/innodb-bad-key-change4.result

@@ -0,0 +1,20 @@
+call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
+call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match.");
+call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
+call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
+call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");
+call mtr.add_suppression("Couldn't load plugins from 'file_key_management*");
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4;
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	Warning	Table test/t1 in tablespace 4 is encrypted but encryption service or used key_id is not available. Can't continue reading table.
+test.t1	check	Warning	Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue checking table.
+test.t1	check	error	Corrupt
+SHOW WARNINGS;
+Level	Code	Message
+DROP TABLE t1;

mysql-test/suite/encryption/t/innodb-bad-key-change4.test

@@ -0,0 +1,88 @@
+--source include/have_innodb.inc
+# embedded does not support restart
+-- source include/not_embedded.inc
+-- source include/not_valgrind.inc
+# Avoid CrashReporter popup on Mac
+-- source include/not_crashrep.inc
+#
+# MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys
+#
+call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
+call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match.");
+call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
+call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
+call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");
+# Suppression for builds where file_key_management plugin is linked statically
+call mtr.add_suppression("Couldn't load plugins from 'file_key_management*");
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys1.txt
+1;770A8A65DA156D24EE2A093277530142
+4;770A8A65DA156D24EE2A093277530143
+EOF
+
+--exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+--let $MYSQLD_TMPDIR = `SELECT @@tmpdir`
+--let $MYSQLD_DATADIR = `SELECT @@datadir`
+
+--disable_query_log
+let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
+let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
+--enable_query_log
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+
+CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4;
+INSERT INTO t1 VALUES (1,'foo'),(2,'bar');
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys2.txt
+1;770A8A65DA156D24EE2A093277530142
+4;770A8A65DA156D24EE2A093277530144
+EOF
+
+--exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+
+CHECK TABLE t1;
+SHOW WARNINGS;
+
+--remove_file $MYSQLTEST_VARDIR/keys1.txt
+--remove_file $MYSQLTEST_VARDIR/keys2.txt
+
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--shutdown_server
+--source include/wait_until_disconnected.inc
+
+--write_file $MYSQLTEST_VARDIR/keys1.txt
+1;770A8A65DA156D24EE2A093277530142
+4;770A8A65DA156D24EE2A093277530143
+EOF
+
+--exec echo "restart:--innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+DROP TABLE t1;
+
+--remove_file $MYSQLTEST_VARDIR/keys1.txt
+
+# reset system
+--disable_query_log
+EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
+EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
+--enable_query_log

storage/innobase/btr/btr0btr.cc

@@ -5123,18 +5123,20 @@ btr_validate_level(
 
 /**************************************************************//**
 Checks the consistency of an index tree.
-@returnTRUE if ok */
+@returnDB_SUCCESS if ok, error code if not */
 UNIV_INTERN
-bool
+dberr_t
 btr_validate_index(
 /*===============*/
 dict_index_t*	index,/*!< in: index */
 const trx_t*	trx)/*!< in: transaction or NULL */
 {
+dberr_t err = DB_SUCCESS;
+
 /* Full Text index are implemented by auxiliary tables,
 not the B-tree */
 if (dict_index_is_online_ddl(index) || (index->type & DICT_FTS)) {
-return(true);
+return(err);
 }
 
 mtr_tmtr;
@@ -5143,21 +5145,27 @@ btr_validate_index(
 
 mtr_x_lock(dict_index_get_lock(index), &mtr);
 
-boolok = true;
 page_t*	root = btr_root_get(index, &mtr);
+
+if (root == NULL && index->table->is_encrypted) {
+err = DB_DECRYPTION_FAILED;
+mtr_commit(&mtr);
+return err;
+}
+
 ulint	n = btr_page_get_level(root, &mtr);
 
 for (ulint i = 0; i <= n; ++i) {
 
 if (!btr_validate_level(index, trx, n - i)) {
-ok = false;
+err = DB_CORRUPTION;
 break;
 }
 }
 
 mtr_commit(&mtr);
 
-return(ok);
+return(err);
 }
 
 /**************************************************************//**

storage/innobase/handler/ha_innodb.cc

@@ -13512,7 +13512,7 @@ ha_innobase::check(
 server_mutex,
  srv_fatal_semaphore_wait_threshold,
 SRV_SEMAPHORE_WAIT_EXTENSION);
-bool valid = btr_validate_index(index, prebuilt->trx);
+dberr_t err = btr_validate_index(index, prebuilt->trx);
 
 /* Restore the fatal lock wait timeout after
 CHECK TABLE. */
@@ -13521,19 +13521,33 @@ ha_innobase::check(
  srv_fatal_semaphore_wait_threshold,
 SRV_SEMAPHORE_WAIT_EXTENSION);
 
-if (!valid) {
+if (err != DB_SUCCESS) {
 is_ok = false;
 
 innobase_format_name(
 index_name, sizeof index_name,
 index->name, TRUE);
-push_warning_printf(
-thd,
-Sql_condition::WARN_LEVEL_WARN,
-ER_NOT_KEYFILE,
-"InnoDB: The B-tree of"
-" index %s is corrupted.",
-index_name);
+
+if (err == DB_DECRYPTION_FAILED) {
+push_warning_printf(
+thd,
+Sql_condition::WARN_LEVEL_WARN,
+ER_NO_SUCH_TABLE,
+"Table %s is encrypted but encryption service or"
+" used key_id is not available. "
+" Can't continue checking table.",
+
+index->table->name);
+} else {
+push_warning_printf(
+thd,
+Sql_condition::WARN_LEVEL_WARN,
+ER_NOT_KEYFILE,
+"InnoDB: The B-tree of"
+" index %s is corrupted.",
+index_name);
+}
+
 continue;
 }
 }

storage/innobase/include/btr0btr.h

@@ -2,7 +2,7 @@
 
 Copyright (c) 1994, 2013, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
-Copyright (c) 2014, SkySQL Ab. All Rights Reserved.
+Copyright (c) 2014, 2015, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -797,9 +797,9 @@ btr_index_rec_validate(
 __attribute__((nonnull, warn_unused_result));
 /**************************************************************//**
 Checks the consistency of an index tree.
-@returnTRUE if ok */
+@returnDB_SUCCESS if ok, error code if not */
 UNIV_INTERN
-bool
+dberr_t
 btr_validate_index(
 /*===============*/
 dict_index_t*index,/*!< in: index */

storage/xtradb/btr/btr0btr.cc

@@ -5159,18 +5159,20 @@ btr_validate_level(
 
 /**************************************************************//**
 Checks the consistency of an index tree.
-@returnTRUE if ok */
+@returnDB_SUCCESS if ok, error code if not */
 UNIV_INTERN
-bool
+dberr_t
 btr_validate_index(
 /*===============*/
 dict_index_t*	index,/*!< in: index */
 const trx_t*	trx)/*!< in: transaction or NULL */
 {
+dberr_t err = DB_SUCCESS;
+
 /* Full Text index are implemented by auxiliary tables,
 not the B-tree */
 if (dict_index_is_online_ddl(index) || (index->type & DICT_FTS)) {
-return(true);
+return(err);
 }
 
 mtr_tmtr;
@@ -5179,28 +5181,33 @@ btr_validate_index(
 
 mtr_x_lock(dict_index_get_lock(index), &mtr);
 
-boolok = true;
 page_t*	root = btr_root_get(index, &mtr);
 
+if (root == NULL && index->table->is_encrypted) {
+err = DB_DECRYPTION_FAILED;
+mtr_commit(&mtr);
+return err;
+}
+
 SRV_CORRUPT_TABLE_CHECK(root,
 {
 mtr_commit(&mtr);
-return(FALSE);
+return(DB_CORRUPTION);
 });
 
 ulint	n = btr_page_get_level(root, &mtr);
 
 for (ulint i = 0; i <= n; ++i) {
 
 if (!btr_validate_level(index, trx, n - i)) {
-ok = false;
+err = DB_CORRUPTION;
 break;
 }
 }
 
 mtr_commit(&mtr);
 
-return(ok);
+return(err);
 }
 
 /**************************************************************//**

storage/xtradb/handler/ha_innodb.cc

@@ -14041,7 +14041,7 @@ ha_innobase::check(
 server_mutex,
  srv_fatal_semaphore_wait_threshold,
 SRV_SEMAPHORE_WAIT_EXTENSION);
-bool valid = btr_validate_index(index, prebuilt->trx);
+dberr_t err = btr_validate_index(index, prebuilt->trx);
 
 /* Restore the fatal lock wait timeout after
 CHECK TABLE. */
@@ -14050,19 +14050,33 @@ ha_innobase::check(
  srv_fatal_semaphore_wait_threshold,
 SRV_SEMAPHORE_WAIT_EXTENSION);
 
-if (!valid) {
+if (err != DB_SUCCESS) {
 is_ok = false;
 
 innobase_format_name(
 index_name, sizeof index_name,
 index->name, TRUE);
-push_warning_printf(
-thd,
-Sql_condition::WARN_LEVEL_WARN,
-ER_NOT_KEYFILE,
-"InnoDB: The B-tree of"
-" index %s is corrupted.",
-index_name);
+
+if (err == DB_DECRYPTION_FAILED) {
+push_warning_printf(
+thd,
+Sql_condition::WARN_LEVEL_WARN,
+ER_NO_SUCH_TABLE,
+"Table %s is encrypted but encryption service or"
+" used key_id is not available. "
+" Can't continue checking table.",
+
+index->table->name);
+} else {
+push_warning_printf(
+thd,
+Sql_condition::WARN_LEVEL_WARN,
+ER_NOT_KEYFILE,
+"InnoDB: The B-tree of"
+" index %s is corrupted.",
+index_name);
+}
+
 continue;
 }
 }

storage/xtradb/include/btr0btr.h

@@ -2,7 +2,7 @@
 
 Copyright (c) 1994, 2013, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
-Copyright (c) 2014, SkySQL Ab. All Rights Reserved.
+Copyright (c) 2014, 2015, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -800,9 +800,9 @@ btr_index_rec_validate(
 __attribute__((nonnull, warn_unused_result));
 /**************************************************************//**
 Checks the consistency of an index tree.
-@returnTRUE if ok */
+@returnDB_SUCCESS if ok, error code if not */
 UNIV_INTERN
-bool
+dberr_t
 btr_validate_index(
 /*===============*/
 dict_index_t*index,/*!< in: index */