Skip to content

Commit 44b1fb3

Browse files
Aditya Adr-m
Aditya A
authored and
dr-m
committed
WL9513 Bug#23333990 PERSISTENT INDEX STATISTICS UPDATE BEFORE TRANSACTION IS COMMITTED
PROBLEM By design stats estimation always reading uncommitted data. In this scenario an uncommitted transaction has deleted all rows in the table. In Innodb uncommitted delete records are marked as delete but not actually removed from Btree until the transaction has committed or a read view for the rows is present.While calculating persistent stats we were ignoring the delete marked records,since all the records are delete marked we were estimating the number of rows present in the table as zero which leads to bad plans in other transaction operating on the table. Fix Introduced a system variable called innodb_stats_include_delete_marked which when enabled includes delete marked records for stat calculations .
1 parent 6f5f720 commit 44b1fb3

File tree

7 files changed

+223
-3
lines changed

7 files changed

+223
-3
lines changed

‎mysql-test/suite/innodb/r/innodb_stats_del_mark.result‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#
2+
# Bug 23333990 PERSISTENT INDEX STATISTICS UPDATE BEFORE
3+
# TRANSACTION IS COMMITTED
4+
#
5+
"Test 1:- Uncommited delete test"
6+
CREATE TABLE t1 (id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
7+
val INT UNSIGNED NOT NULL,
8+
INDEX (val)) ENGINE=INNODB
9+
STATS_PERSISTENT=1,STATS_AUTO_RECALC=1;
10+
INSERT INTO t1 (val) VALUES (CEIL(RAND()*20));
11+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
12+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
13+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
14+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
15+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
16+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
17+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
18+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
19+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
20+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
21+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
22+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
23+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
24+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
25+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
26+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
27+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
28+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
29+
SELECT COUNT(*) FROM t1;
30+
COUNT(*)
31+
262144
32+
ANALYZE TABLE t1;
33+
Table Op Msg_type Msg_text
34+
test.t1 analyze status OK
35+
connect con1, localhost, root,,;
36+
START TRANSACTION;
37+
DELETE FROM t1;
38+
SELECT COUNT(*) FROM t1;
39+
connection default;
40+
Test correctly estimates the number of rows as > 20000
41+
even when in other uncommmited transaction
42+
all rows have been deleted.
43+
connection con1;
44+
COUNT(*)
45+
0
46+
commit;
47+
connection default;
48+
Test 2:- Insert and rollback test
49+
CREATE TABLE t2 (id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
50+
val INT UNSIGNED NOT NULL,
51+
INDEX (val)) ENGINE=INNODB
52+
STATS_PERSISTENT=1,STATS_AUTO_RECALC=1;
53+
connection con1;
54+
START TRANSACTION;
55+
INSERT INTO t2 (val) VALUES (CEIL(RAND()*20));
56+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
57+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
58+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
59+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
60+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
61+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
62+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
63+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
64+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
65+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
66+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
67+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
68+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
69+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
70+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
71+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
72+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
73+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
74+
SELECT COUNT(*) FROM t2;
75+
connection default;
76+
select count(*) from t2;
77+
count(*)
78+
0
79+
Test correctly estimates the number of rows as > 20000
80+
even when in other uncommited transaction
81+
many rows are inserted.
82+
connection con1;
83+
COUNT(*)
84+
262144
85+
Rollback the insert
86+
rollback;
87+
disconnect con1;
88+
connection default;
89+
Test correctly estimates the number of rows as 1
90+
after rollback.
91+
DROP TABLE t1,t2;

‎mysql-test/suite/innodb/t/innodb_stats_del_mark-master.opt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
--innodb_stats_include_delete_marked=on

‎mysql-test/suite/innodb/t/innodb_stats_del_mark.test‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
--source include/have_innodb.inc
2+
--source include/big_test.inc
3+
4+
--echo #
5+
--echo # Bug 23333990 PERSISTENT INDEX STATISTICS UPDATE BEFORE
6+
--echo # TRANSACTION IS COMMITTED
7+
--echo #
8+
9+
--echo "Test 1:- Uncommited delete test"
10+
CREATE TABLE t1 (id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
11+
val INT UNSIGNED NOT NULL,
12+
INDEX (val)) ENGINE=INNODB
13+
STATS_PERSISTENT=1,STATS_AUTO_RECALC=1;
14+
15+
16+
INSERT INTO t1 (val) VALUES (CEIL(RAND()*20));
17+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
18+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
19+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
20+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
21+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
22+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
23+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
24+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
25+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
26+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
27+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
28+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
29+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
30+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
31+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
32+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
33+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
34+
INSERT INTO t1 (val) SELECT CEIL(RAND()*20) FROM t1;
35+
36+
SELECT COUNT(*) FROM t1;
37+
ANALYZE TABLE t1;
38+
39+
connect(con1, localhost, root,,);
40+
START TRANSACTION;
41+
DELETE FROM t1;
42+
send SELECT COUNT(*) FROM t1;
43+
44+
connection default;
45+
let $row_count= query_get_value(EXPLAIN SELECT * FROM t1 WHERE val=4, rows,1);
46+
if ($row_count > 20000)
47+
{
48+
--echo Test correctly estimates the number of rows as > 20000
49+
--echo even when in other uncommmited transaction
50+
--echo all rows have been deleted.
51+
}
52+
53+
connection con1;
54+
reap;
55+
commit;
56+
57+
connection default;
58+
59+
--echo Test 2:- Insert and rollback test
60+
CREATE TABLE t2 (id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
61+
val INT UNSIGNED NOT NULL,
62+
INDEX (val)) ENGINE=INNODB
63+
STATS_PERSISTENT=1,STATS_AUTO_RECALC=1;
64+
65+
connection con1;
66+
67+
START TRANSACTION;
68+
INSERT INTO t2 (val) VALUES (CEIL(RAND()*20));
69+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
70+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
71+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
72+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
73+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
74+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
75+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
76+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
77+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
78+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
79+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
80+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
81+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
82+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
83+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
84+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
85+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
86+
INSERT INTO t2 (val) SELECT CEIL(RAND()*20) FROM t2;
87+
send SELECT COUNT(*) FROM t2;
88+
89+
connection default;
90+
select count(*) from t2;
91+
let $row_count= query_get_value(EXPLAIN SELECT * FROM t2 WHERE val=4, rows,1);
92+
if ($row_count > 20000)
93+
{
94+
--echo Test correctly estimates the number of rows as > 20000
95+
--echo even when in other uncommited transaction
96+
--echo many rows are inserted.
97+
}
98+
99+
connection con1;
100+
reap;
101+
--echo Rollback the insert
102+
rollback;
103+
disconnect con1;
104+
105+
connection default;
106+
let $row_count= query_get_value(EXPLAIN SELECT * FROM t2 WHERE val=4, rows,1);
107+
if ($row_count <= 1)
108+
{
109+
--echo Test correctly estimates the number of rows as $row_count
110+
--echo after rollback.
111+
}
112+
113+
DROP TABLE t1,t2;

‎storage/innobase/dict/dict0stats.cc‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,7 +1152,8 @@ dict_stats_analyze_index_level(
11521152
them away) which brings non-determinism. We skip only
11531153
leaf-level delete marks because delete marks on
11541154
non-leaf level do not make sense. */
1155-
if (level == 0 &&
1155+
1156+
if (level == 0 && srv_stats_include_delete_marked ? 0:
11561157
rec_get_deleted_flag(
11571158
rec,
11581159
page_is_comp(btr_pcur_get_page(&pcur)))) {
@@ -1176,7 +1177,6 @@ dict_stats_analyze_index_level(
11761177

11771178
continue;
11781179
}
1179-
11801180
rec_offsets = rec_get_offsets(
11811181
rec, index, rec_offsets, n_uniq, &heap);
11821182

@@ -1334,8 +1334,12 @@ enum page_scan_method_t {
13341334
the given page and count the number of
13351335
distinct ones, also ignore delete marked
13361336
records */
1337-
QUIT_ON_FIRST_NON_BORING/* quit when the first record that differs
1337+
QUIT_ON_FIRST_NON_BORING,/* quit when the first record that differs
13381338
from its right neighbor is found */
1339+
COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED/* scan all records on
1340+
the given page and count the number of
1341+
distinct ones, include delete marked
1342+
records */
13391343
};
13401344
/* @} */
13411345

@@ -1608,6 +1612,8 @@ dict_stats_analyze_index_below_cur(
16081612

16091613
offsets_rec = dict_stats_scan_page(
16101614
&rec, offsets1, offsets2, index, page, n_prefix,
1615+
srv_stats_include_delete_marked ?
1616+
COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED:
16111617
COUNT_ALL_NON_BORING_AND_SKIP_DEL_MARKED, n_diff,
16121618
n_external_pages);
16131619

‎storage/innobase/handler/ha_innodb.cc‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20769,6 +20769,12 @@ static MYSQL_SYSVAR_BOOL(use_fallocate, innobase_use_fallocate,
2076920769
"Use posix_fallocate() to allocate files. DEPRECATED, has no effect.",
2077020770
NULL, NULL, FALSE);
2077120771

20772+
static MYSQL_SYSVAR_BOOL(stats_include_delete_marked,
20773+
srv_stats_include_delete_marked,
20774+
PLUGIN_VAR_OPCMDARG,
20775+
"Include delete marked records when calculating persistent statistics",
20776+
NULL, NULL, FALSE);
20777+
2077220778
static MYSQL_SYSVAR_ULONG(io_capacity, srv_io_capacity,
2077320779
PLUGIN_VAR_RQCMDARG,
2077420780
"Number of IOPs the server can do. Tunes the background IO rate",
@@ -21965,6 +21971,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
2196521971
MYSQL_SYSVAR(temp_data_file_path),
2196621972
MYSQL_SYSVAR(data_home_dir),
2196721973
MYSQL_SYSVAR(doublewrite),
21974+
MYSQL_SYSVAR(stats_include_delete_marked),
2196821975
MYSQL_SYSVAR(use_atomic_writes),
2196921976
MYSQL_SYSVAR(use_fallocate),
2197021977
MYSQL_SYSVAR(fast_shutdown),

‎storage/innobase/include/srv0srv.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,7 @@ extern unsigned long long srv_stats_transient_sample_pages;
472472
extern my_boolsrv_stats_persistent;
473473
extern unsigned long longsrv_stats_persistent_sample_pages;
474474
extern my_boolsrv_stats_auto_recalc;
475+
extern my_boolsrv_stats_include_delete_marked;
475476
extern unsigned long longsrv_stats_modified_counter;
476477
extern my_boolsrv_stats_sample_traditional;
477478

‎storage/innobase/srv/srv0srv.cc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,7 @@ this many index pages, there are 2 ways to calculate statistics:
373373
table/index are not found in the innodb database */
374374
unsigned long longsrv_stats_transient_sample_pages = 8;
375375
my_bool srv_stats_persistent = TRUE;
376+
my_bool srv_stats_include_delete_marked = FALSE;
376377
unsigned long longsrv_stats_persistent_sample_pages = 20;
377378
my_bool srv_stats_auto_recalc = TRUE;
378379

0 commit comments

Comments
 (0)