MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables

Author: janlindstrom

extra/innochecksum.cc

@@ -62,6 +62,7 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
 #include "fsp0fsp.h" /* fsp_flags_get_page_size() &
  fsp_flags_get_zip_size() */
 #include "ut0crc32.h" /* ut_crc32_init() */
+#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */
 
 #ifdef UNIV_NONINL
 # include "fsp0fsp.ic"
@@ -109,6 +110,8 @@ int n_fil_page_type_xdes;
 int n_fil_page_type_blob;
 int n_fil_page_type_zblob;
 int n_fil_page_type_other;
+int n_fil_page_type_page_compressed;
+int n_fil_page_type_page_compressed_encrypted;
 
 int n_fil_page_max_index_id;
 
@@ -152,6 +155,8 @@ struct per_index_stats {
 
 std::map<unsigned long long, per_index_stats> index_ids;
 
+bool encrypted = false;
+
 /* Get the page size of the filespace from the filespace header. */
 static
 my_bool
@@ -197,6 +202,8 @@ get_page_size(
  {
  compressed= true;
  }
+
+
  return TRUE;
 }
 
@@ -515,6 +522,18 @@ parse_page(
  }
  n_fil_page_type_zblob++;
  break;
+ case FIL_PAGE_PAGE_COMPRESSED:
+ if (per_page_details) {
+ printf("FIL_PAGE_PAGE_COMPRESSED\n");
+ }
+ n_fil_page_type_page_compressed++;
+ break;
+ case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED:
+ if (per_page_details) {
+ printf("FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n");
+ }
+ n_fil_page_type_page_compressed_encrypted++;
+ break;
  default:
  if (per_page_details) {
  printf("FIL_PAGE_TYPE_OTHER\n");
@@ -604,6 +623,8 @@ print_stats()
  printf("%d\tFIL_PAGE_TYPE_XDES\n", n_fil_page_type_xdes);
  printf("%d\tFIL_PAGE_TYPE_BLOB\n", n_fil_page_type_blob);
  printf("%d\tFIL_PAGE_TYPE_ZBLOB\n", n_fil_page_type_zblob);
+ printf("%d\tFIL_PAGE_PAGE_COMPRESSED\n", n_fil_page_type_page_compressed);
+ printf("%d\tFIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n", n_fil_page_type_page_compressed_encrypted);
  printf("%d\tother\n", n_fil_page_type_other);
  printf("%d\tmax index_id\n", n_fil_page_max_index_id);
  printf("undo type: %d insert, %d update, %d other\n",
@@ -791,7 +812,9 @@ int main(int argc, char **argv)
  while (!feof(f))
  {
  int page_ok = 1;
+
  bytes= fread(buf, 1, physical_page_size, f);
+
  if (!bytes && feof(f))
  {
  print_stats();
@@ -809,58 +832,126 @@ int main(int argc, char **argv)
  return 1;
  }
 
- if (compressed) {
- /* compressed pages */
- if (!page_zip_verify_checksum(buf, physical_page_size)) {
- fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct);
- if (!skip_corrupt)
- {
- free(big_buf);
- free(big_xdes);
- return 1;
- }
- page_ok = 0;
- }
+ ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
+ ulint key_version = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+
+ if (key_version && page_type != FIL_PAGE_PAGE_COMPRESSED) {
+ encrypted = true;
  } else {
+ encrypted = false;
+ }
+
+ ulint comp_method = 0;
 
- /* check the "stored log sequence numbers" */
- logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4);
- logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
+ if (encrypted) {
+ comp_method = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE);
+ } else {
+ comp_method = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+ }
+
+ ulint comp_size = mach_read_from_2(buf+FIL_PAGE_DATA);
+ ib_uint32_t encryption_checksum = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
+
+
+ if (page_type == FIL_PAGE_PAGE_COMPRESSED) {
+ /* Page compressed tables do not have any checksum */
  if (debug)
- printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield);
- if (logseq != logseqfield)
- {
- fprintf(stderr, "Fail; page %lu invalid (fails log sequence number check)\n", ct);
- if (!skip_corrupt)
- {
- free(big_buf);
- free(big_xdes);
- return 1;
+fprintf(stderr, "Page %lu page compressed with method %s real_size %lu\n", ct,
+ fil_get_compression_alg_name(comp_method), comp_size);
+ page_ok = 1;
+ } else if (compressed) {
+ /* compressed pages */
+ulint crccsum = page_zip_calc_checksum(buf, physical_page_size, SRV_CHECKSUM_ALGORITHM_CRC32);
+ulint icsum = page_zip_calc_checksum(buf, physical_page_size, SRV_CHECKSUM_ALGORITHM_INNODB);
+
+ if (debug) {
+ if (key_version != 0) {
+ fprintf(stderr,
+ "Page %lu encrypted key_version %lu calculated = %lu; crc32 = %lu; recorded = %u\n",
+ ct, key_version, icsum, crccsum, encryption_checksum);
+ }
+ }
+
+if (encrypted) {
+ if (encryption_checksum != 0 && crccsum != encryption_checksum && icsum != encryption_checksum) {
+ if (debug)
+ fprintf(stderr, "page %lu: compressed: calculated = %lu; crc32 = %lu; recorded = %u\n",
+ ct, icsum, crccsum, encryption_checksum);
+ fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct);
+ }
+} else {
+ if (!page_zip_verify_checksum(buf, physical_page_size)) {
+ fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct);
+ if (!skip_corrupt)
+ {
+ free(big_buf);
+ free(big_xdes);
+ return 1;
+ }
+ page_ok = 0;
+ }
+}
+ } else {
+ if (key_version != 0) {
+ /* Encrypted page */
+ if (debug) {
+ if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+ fprintf(stderr,
+ "Page %lu page compressed with method %s real_size %lu and encrypted key_version %lu checksum %u\n",
+ ct, fil_get_compression_alg_name(comp_method), comp_size, key_version, encryption_checksum);
+ } else {
+ fprintf(stderr,
+ "Page %lu encrypted key_version %lu checksum %u\n",
+ ct, key_version, encryption_checksum);
+ }
  }
- page_ok = 0;
  }
 
- /* check old method of checksumming */
- oldcsum= buf_calc_page_old_checksum(buf);
- oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM);
- if (debug)
- printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield);
- if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum)
- {
- fprintf(stderr, "Fail; page %lu invalid (fails old style checksum)\n", ct);
- if (!skip_corrupt)
+ /* Page compressed tables do not contain FIL tailer */
+ if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && page_type != FIL_PAGE_PAGE_COMPRESSED) {
+ /* check the "stored log sequence numbers" */
+ logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4);
+ logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
+ if (debug)
+ printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield);
+ if (logseq != logseqfield)
  {
- free(big_buf);
- free(big_xdes);
- return 1;
+ fprintf(stderr, "Fail; page %lu invalid (fails log sequence number check)\n", ct);
+ if (!skip_corrupt)
+ {
+ free(big_buf);
+ free(big_xdes);
+ return 1;
+ }
+ page_ok = 0;
+ }
+
+ /* check old method of checksumming */
+ oldcsum= buf_calc_page_old_checksum(buf);
+ oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM);
+ if (debug)
+ printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield);
+ if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum)
+ {
+ fprintf(stderr, "Fail; page %lu invalid (fails old style checksum)\n", ct);
+ if (!skip_corrupt)
+ {
+ free(big_buf);
+ free(big_xdes);
+ return 1;
+ }
+ page_ok = 0;
  }
- page_ok = 0;
  }
 
  /* now check the new method */
  csum= buf_calc_page_new_checksum(buf);
  crc32= buf_calc_page_crc32(buf);
  csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM);
+
+ if (key_version)
+ csumfield = encryption_checksum;
+
  if (debug)
  printf("page %lu: new style: calculated = %lu; crc32 = %lu; recorded = %lu\n",
  ct, csum, crc32, csumfield);
@@ -903,7 +994,10 @@ int main(int argc, char **argv)
  continue;
  }
 
- parse_page(buf, xdes);
+ /* Can't parse compressed or/and encrypted pages */
+ if (page_type != FIL_PAGE_PAGE_COMPRESSED && !encrypted) {
+ parse_page(buf, xdes);
+ }
 
  if (verbose)
  {

mysql-test/suite/encryption/r/innochecksum.result

@@ -0,0 +1,23 @@
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+set global innodb_compression_algorithm = 1;
+# Create and populate a tables
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
+CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1;
+CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+# Write file to make mysql-test-run.pl expect the "crash", but don't
+# start it until it's told to
+# We give 30 seconds to do a clean shutdown because we do not want
+# to redo apply the pages of t1.ibd at the time of recovery.
+# We want SQL to initiate the first access to t1.ibd.
+# Wait until disconnected.
+# Run innochecksum on t1
+# Run innochecksum on t2
+# Run innochecksum on t3
+# Run innochecksum on t4
+# Run innochecksum on t4
+# Write file to make mysql-test-run.pl start up the server again
+# Cleanup
+DROP TABLE t1, t2, t3, t4, t5;

mysql-test/suite/encryption/r/innodb-bad-key-change.result

@@ -49,8 +49,8 @@ SELECT * FROM t2;
 ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB
 SHOW WARNINGS;
 Level	Code	Message
-Warning	192Table test/t2 in tablespace 7 is encrypted but encryption service or used key_id is not available. Can't continue reading table.
-Warning	192Table test/t2 in tablespace 7 is encrypted but encryption service or used key_id is not available. Can't continue reading table.
+Warning	192 is encrypted but encryption service or used key_id is not available. Can't continue reading table.
+Warning	192 is encrypted but encryption service or used key_id is not available. Can't continue reading table.
 Warning	192	Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table.
 Error	1296	Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB
 SELECT * FROM t2 where id = 1;

mysql-test/suite/encryption/t/innochecksum.test

@@ -0,0 +1,101 @@
+#
+# MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables
+#
+
+# Don't test under embedded
+-- source include/not_embedded.inc
+# Require InnoDB
+-- source include/have_innodb.inc
+-- source include/have_file_key_management_plugin.inc
+
+if (!$INNOCHECKSUM) {
+ --echo Need innochecksum binary
+ --die Need innochecksum binary
+}
+
+--disable_query_log
+let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`;
+let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
+let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
+--enable_query_log
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+# zlib
+set global innodb_compression_algorithm = 1;
+
+--echo # Create and populate a tables
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
+CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1;
+CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+
+--disable_query_log
+--let $i = 1000
+while ($i)
+{
+ INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100));
+ dec $i;
+}
+INSERT INTO t2 SELECT * FROM t1;
+INSERT INTO t3 SELECT * FROM t1;
+INSERT INTO t4 SELECT * FROM t1;
+INSERT INTO t5 SELECT * FROM t1;
+--enable_query_log
+
+let $MYSQLD_DATADIR=`select @@datadir`;
+let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd;
+let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd;
+let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd;
+let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd;
+let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd;
+
+--echo # Write file to make mysql-test-run.pl expect the "crash", but don't
+--echo # start it until it's told to
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+
+--echo # We give 30 seconds to do a clean shutdown because we do not want
+--echo # to redo apply the pages of t1.ibd at the time of recovery.
+--echo # We want SQL to initiate the first access to t1.ibd.
+shutdown_server 30;
+
+--echo # Wait until disconnected.
+--source include/wait_until_disconnected.inc
+
+--echo # Run innochecksum on t1
+-- disable_result_log
+--exec $INNOCHECKSUM $t1_IBD
+
+--echo # Run innochecksum on t2
+
+--exec $INNOCHECKSUM $t2_IBD
+
+--echo # Run innochecksum on t3
+
+--exec $INNOCHECKSUM $t3_IBD
+
+--echo # Run innochecksum on t4
+
+--exec $INNOCHECKSUM $t4_IBD
+
+--echo # Run innochecksum on t4
+
+--exec $INNOCHECKSUM $t4_IBD
+
+--enable_result_log
+
+--echo # Write file to make mysql-test-run.pl start up the server again
+--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+--echo # Cleanup
+DROP TABLE t1, t2, t3, t4, t5;
+
+# reset system
+--disable_query_log
+EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig;
+EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
+EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
+--enable_query_log

mysql-test/suite/encryption/t/innodb-bad-key-change.test

@@ -79,6 +79,7 @@ INSERT INTO t2 VALUES ('foobar',1,2);
 
 --error ER_GET_ERRMSG
 SELECT * FROM t2;
+--replace_regex /.*tablespace [0-9]*//
 SHOW WARNINGS;
 --error ER_GET_ERRMSG
 SELECT * FROM t2 where id = 1;