Skip to content

Commit 284ac6f

Browse files
abarkovabarkov
committed
MDEV-27653 long uniques don't work with unicode collations
1 parent 9924466 commit 284ac6f

27 files changed

+668
-66
lines changed

‎mysql-test/main/ctype_utf8.result‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11379,3 +11379,178 @@ a
1137911379
#
1138011380
# End of 10.3 tests
1138111381
#
11382+
#
11383+
# Start of 10.4 tests
11384+
#
11385+
#
11386+
# MDEV-27653 long uniques don't work with unicode collations
11387+
#
11388+
SET NAMES utf8mb3;
11389+
CREATE TABLE t1 (
11390+
a CHAR(30) COLLATE utf8mb3_general_ci,
11391+
UNIQUE KEY(a) USING HASH
11392+
);
11393+
SHOW CREATE TABLE t1;
11394+
Table Create Table
11395+
t1 CREATE TABLE `t1` (
11396+
`a` char(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
11397+
UNIQUE KEY `a` (`a`) USING HASH
11398+
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
11399+
INSERT INTO t1 VALUES ('a');
11400+
INSERT INTO t1 VALUES ('ä');
11401+
ERROR 23000: Duplicate entry 'ä' for key 'a'
11402+
SELECT * FROM t1;
11403+
a
11404+
a
11405+
DROP TABLE t1;
11406+
CREATE TABLE t1 (
11407+
a CHAR(30) COLLATE utf8mb3_general_ci,
11408+
UNIQUE KEY(a(10)) USING HASH
11409+
);
11410+
SHOW CREATE TABLE t1;
11411+
Table Create Table
11412+
t1 CREATE TABLE `t1` (
11413+
`a` char(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
11414+
UNIQUE KEY `a` (`a`(10)) USING HASH
11415+
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
11416+
INSERT INTO t1 VALUES ('a');
11417+
INSERT INTO t1 VALUES ('ä');
11418+
ERROR 23000: Duplicate entry 'ä' for key 'a'
11419+
SELECT * FROM t1;
11420+
a
11421+
a
11422+
DROP TABLE t1;
11423+
CREATE TABLE t1 (
11424+
a VARCHAR(30) COLLATE utf8mb3_general_ci,
11425+
UNIQUE KEY(a) USING HASH
11426+
);
11427+
SHOW CREATE TABLE t1;
11428+
Table Create Table
11429+
t1 CREATE TABLE `t1` (
11430+
`a` varchar(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
11431+
UNIQUE KEY `a` (`a`) USING HASH
11432+
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
11433+
INSERT INTO t1 VALUES ('a');
11434+
INSERT INTO t1 VALUES ('ä');
11435+
ERROR 23000: Duplicate entry 'ä' for key 'a'
11436+
SELECT * FROM t1;
11437+
a
11438+
a
11439+
DROP TABLE t1;
11440+
CREATE TABLE t1 (
11441+
a VARCHAR(30) COLLATE utf8mb3_general_ci,
11442+
UNIQUE KEY(a(10)) USING HASH
11443+
);
11444+
SHOW CREATE TABLE t1;
11445+
Table Create Table
11446+
t1 CREATE TABLE `t1` (
11447+
`a` varchar(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
11448+
UNIQUE KEY `a` (`a`(10)) USING HASH
11449+
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
11450+
INSERT INTO t1 VALUES ('a');
11451+
INSERT INTO t1 VALUES ('ä');
11452+
ERROR 23000: Duplicate entry 'ä' for key 'a'
11453+
SELECT * FROM t1;
11454+
a
11455+
a
11456+
DROP TABLE t1;
11457+
CREATE TABLE t1 (a TEXT COLLATE utf8mb3_general_ci UNIQUE);
11458+
SHOW CREATE TABLE t1;
11459+
Table Create Table
11460+
t1 CREATE TABLE `t1` (
11461+
`a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
11462+
UNIQUE KEY `a` (`a`) USING HASH
11463+
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
11464+
INSERT INTO t1 VALUES ('a');
11465+
INSERT INTO t1 VALUES ('ä');
11466+
ERROR 23000: Duplicate entry 'ä' for key 'a'
11467+
SELECT * FROM t1;
11468+
a
11469+
a
11470+
DROP TABLE t1;
11471+
CREATE TABLE t1 (
11472+
a LONGTEXT COLLATE utf8mb3_general_ci,
11473+
UNIQUE KEY(a(10)) USING HASH
11474+
);
11475+
SHOW CREATE TABLE t1;
11476+
Table Create Table
11477+
t1 CREATE TABLE `t1` (
11478+
`a` longtext CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
11479+
UNIQUE KEY `a` (`a`(10)) USING HASH
11480+
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
11481+
INSERT INTO t1 VALUES ('a');
11482+
INSERT INTO t1 VALUES ('ä');
11483+
ERROR 23000: Duplicate entry 'ä' for key 'a'
11484+
SELECT * FROM t1;
11485+
a
11486+
a
11487+
DROP TABLE t1;
11488+
SHOW CREATE TABLE t1;
11489+
Table Create Table
11490+
t1 CREATE TABLE `t1` (
11491+
`a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
11492+
UNIQUE KEY `a` (`a`) USING HASH
11493+
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
11494+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
11495+
a OCTET_LENGTH(a)
11496+
a 1
11497+
ä 2
11498+
CHECK TABLE t1;
11499+
Table Op Msg_type Msg_text
11500+
test.t1 check error Upgrade required. Please do "REPAIR TABLE `t1`" or dump/reload to fix it!
11501+
INSERT INTO t1 VALUES ('A');
11502+
ERROR 23000: Duplicate entry 'A' for key 'a'
11503+
INSERT INTO t1 VALUES ('Ä');
11504+
ERROR 23000: Duplicate entry 'Ä' for key 'a'
11505+
INSERT INTO t1 VALUES ('Ấ');
11506+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
11507+
a OCTET_LENGTH(a)
11508+
a 1
11509+
ä 2
11510+
Ấ 3
11511+
CHECK TABLE t1;
11512+
Table Op Msg_type Msg_text
11513+
test.t1 check error Upgrade required. Please do "REPAIR TABLE `t1`" or dump/reload to fix it!
11514+
ALTER TABLE t1 FORCE;
11515+
ERROR 23000: Duplicate entry 'ä' for key 'a'
11516+
DELETE FROM t1 WHERE OCTET_LENGTH(a)>1;
11517+
ALTER TABLE t1 FORCE;
11518+
INSERT INTO t1 VALUES ('ä');
11519+
ERROR 23000: Duplicate entry 'ä' for key 'a'
11520+
DROP TABLE t1;
11521+
SHOW CREATE TABLE t1;
11522+
Table Create Table
11523+
t1 CREATE TABLE `t1` (
11524+
`a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
11525+
UNIQUE KEY `a` (`a`) USING HASH
11526+
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
11527+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
11528+
a OCTET_LENGTH(a)
11529+
a 1
11530+
ä 2
11531+
ALTER IGNORE TABLE t1 FORCE;
11532+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
11533+
a OCTET_LENGTH(a)
11534+
a 1
11535+
DROP TABLE t1;
11536+
SHOW CREATE TABLE t1;
11537+
Table Create Table
11538+
t1 CREATE TABLE `t1` (
11539+
`a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
11540+
UNIQUE KEY `a` (`a`) USING HASH
11541+
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
11542+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
11543+
a OCTET_LENGTH(a)
11544+
a 1
11545+
ä 2
11546+
REPAIR TABLE t1;
11547+
Table Op Msg_type Msg_text
11548+
test.t1 repair Warning Number of rows changed from 2 to 1
11549+
test.t1 repair status OK
11550+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
11551+
a OCTET_LENGTH(a)
11552+
a 1
11553+
DROP TABLE t1;
11554+
#
11555+
# End of 10.4 tests
11556+
#

‎mysql-test/main/ctype_utf8.test‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2310,3 +2310,161 @@ VALUES (_latin1 0xDF) UNION VALUES(_utf8'a' COLLATE utf8_bin);
23102310
--echo #
23112311
--echo # End of 10.3 tests
23122312
--echo #
2313+
2314+
2315+
--echo #
2316+
--echo # Start of 10.4 tests
2317+
--echo #
2318+
2319+
--echo #
2320+
--echo # MDEV-27653 long uniques don't work with unicode collations
2321+
--echo #
2322+
2323+
SET NAMES utf8mb3;
2324+
2325+
# CHAR
2326+
2327+
CREATE TABLE t1 (
2328+
a CHAR(30) COLLATE utf8mb3_general_ci,
2329+
UNIQUE KEY(a) USING HASH
2330+
);
2331+
SHOW CREATE TABLE t1;
2332+
INSERT INTO t1 VALUES ('a');
2333+
--error ER_DUP_ENTRY
2334+
INSERT INTO t1 VALUES ('ä');
2335+
SELECT * FROM t1;
2336+
DROP TABLE t1;
2337+
2338+
CREATE TABLE t1 (
2339+
a CHAR(30) COLLATE utf8mb3_general_ci,
2340+
UNIQUE KEY(a(10)) USING HASH
2341+
);
2342+
SHOW CREATE TABLE t1;
2343+
INSERT INTO t1 VALUES ('a');
2344+
--error ER_DUP_ENTRY
2345+
INSERT INTO t1 VALUES ('ä');
2346+
SELECT * FROM t1;
2347+
DROP TABLE t1;
2348+
2349+
2350+
# VARCHAR
2351+
2352+
CREATE TABLE t1 (
2353+
a VARCHAR(30) COLLATE utf8mb3_general_ci,
2354+
UNIQUE KEY(a) USING HASH
2355+
);
2356+
SHOW CREATE TABLE t1;
2357+
INSERT INTO t1 VALUES ('a');
2358+
--error ER_DUP_ENTRY
2359+
INSERT INTO t1 VALUES ('ä');
2360+
SELECT * FROM t1;
2361+
DROP TABLE t1;
2362+
2363+
CREATE TABLE t1 (
2364+
a VARCHAR(30) COLLATE utf8mb3_general_ci,
2365+
UNIQUE KEY(a(10)) USING HASH
2366+
);
2367+
SHOW CREATE TABLE t1;
2368+
INSERT INTO t1 VALUES ('a');
2369+
--error ER_DUP_ENTRY
2370+
INSERT INTO t1 VALUES ('ä');
2371+
SELECT * FROM t1;
2372+
DROP TABLE t1;
2373+
2374+
2375+
# TEXT
2376+
2377+
CREATE TABLE t1 (a TEXT COLLATE utf8mb3_general_ci UNIQUE);
2378+
SHOW CREATE TABLE t1;
2379+
INSERT INTO t1 VALUES ('a');
2380+
--error ER_DUP_ENTRY
2381+
INSERT INTO t1 VALUES ('ä');
2382+
SELECT * FROM t1;
2383+
DROP TABLE t1;
2384+
2385+
CREATE TABLE t1 (
2386+
a LONGTEXT COLLATE utf8mb3_general_ci,
2387+
UNIQUE KEY(a(10)) USING HASH
2388+
);
2389+
SHOW CREATE TABLE t1;
2390+
INSERT INTO t1 VALUES ('a');
2391+
--error ER_DUP_ENTRY
2392+
INSERT INTO t1 VALUES ('ä');
2393+
SELECT * FROM t1;
2394+
DROP TABLE t1;
2395+
2396+
2397+
# Testing upgrade:
2398+
# Prior to MDEV-27653, the UNIQUE HASH function errorneously
2399+
# took into account string octet length.
2400+
# Old tables should still open and work, but with wrong results.
2401+
2402+
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm;
2403+
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD;
2404+
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI;
2405+
SHOW CREATE TABLE t1;
2406+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
2407+
CHECK TABLE t1;
2408+
2409+
# There is already a one byte value 'a' in the table
2410+
--error ER_DUP_ENTRY
2411+
INSERT INTO t1 VALUES ('A');
2412+
2413+
# There is already a two-byte value 'ä' in the table
2414+
--error ER_DUP_ENTRY
2415+
INSERT INTO t1 VALUES ('Ä');
2416+
2417+
# There were no three-byte values in the table so far.
2418+
# The below value violates UNIQUE, but it gets inserted.
2419+
# This is wrong but expected for a pre-MDEV-27653 table.
2420+
INSERT INTO t1 VALUES ('Ấ');
2421+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
2422+
CHECK TABLE t1;
2423+
2424+
# ALTER FORCE fails: it tries to rebuild the table
2425+
# with a correct UNIQUE HASH function, but there are duplicates!
2426+
--error ER_DUP_ENTRY
2427+
ALTER TABLE t1 FORCE;
2428+
2429+
# Let's remove all duplicate values, so only the one-byte 'a' stays.
2430+
# ALTER..FORCE should work after that.
2431+
DELETE FROM t1 WHERE OCTET_LENGTH(a)>1;
2432+
ALTER TABLE t1 FORCE;
2433+
2434+
# Make sure that 'a' and 'ä' cannot co-exists any more,
2435+
# because the table was recreated with a correct UNIQUE HASH function.
2436+
--error ER_DUP_ENTRY
2437+
INSERT INTO t1 VALUES ('ä');
2438+
DROP TABLE t1;
2439+
2440+
#
2441+
# Testing an old table with ALTER IGNORE.
2442+
# The table is expected to rebuild with a new hash function,
2443+
# duplicates go away.
2444+
#
2445+
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm;
2446+
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD;
2447+
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI;
2448+
SHOW CREATE TABLE t1;
2449+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
2450+
ALTER IGNORE TABLE t1 FORCE;
2451+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
2452+
DROP TABLE t1;
2453+
2454+
#
2455+
# Testing an old table with REPAIR.
2456+
# The table is expected to rebuild with a new hash function,
2457+
# duplicates go away.
2458+
#
2459+
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm;
2460+
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD;
2461+
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI;
2462+
SHOW CREATE TABLE t1;
2463+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
2464+
REPAIR TABLE t1;
2465+
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
2466+
DROP TABLE t1;
2467+
2468+
--echo #
2469+
--echo # End of 10.4 tests
2470+
--echo #

‎mysql-test/main/type_timestamp.result‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1320,5 +1320,27 @@ CASE WHEN a THEN DEFAULT(a) END
13201320
DROP TABLE t1;
13211321
SET timestamp=DEFAULT;
13221322
#
1323+
# MDEV-27653 long uniques don't work with unicode collations
1324+
#
1325+
CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH);
1326+
SET time_zone='+00:00';
1327+
INSERT INTO t1 VALUES ('2001-01-01 10:20:30');
1328+
SET time_zone='+01:00';
1329+
INSERT INTO t1 SELECT MAX(a) FROM t1;
1330+
ERROR 23000: Duplicate entry '2001-01-01 11:20:30' for key 'a'
1331+
SELECT * FROM t1;
1332+
a
1333+
2001-01-01 11:20:30
1334+
DROP TABLE t1;
1335+
CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH);
1336+
SET time_zone='+00:00';
1337+
INSERT INTO t1 VALUES ('2001-01-01 10:20:30');
1338+
SET time_zone='+01:00';
1339+
CHECK TABLE t1;
1340+
Table Op Msg_type Msg_text
1341+
test.t1 check status OK
1342+
DROP TABLE t1;
1343+
SET time_zone=DEFAULT;
1344+
#
13231345
# End of 10.4 tests
13241346
#

‎mysql-test/main/type_timestamp.test‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -878,6 +878,27 @@ DROP TABLE t1;
878878
SET timestamp=DEFAULT;
879879

880880

881+
--echo #
882+
--echo # MDEV-27653 long uniques don't work with unicode collations
883+
--echo #
884+
885+
CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH);
886+
SET time_zone='+00:00';
887+
INSERT INTO t1 VALUES ('2001-01-01 10:20:30');
888+
SET time_zone='+01:00';
889+
--error ER_DUP_ENTRY
890+
INSERT INTO t1 SELECT MAX(a) FROM t1;
891+
SELECT * FROM t1;
892+
DROP TABLE t1;
893+
894+
CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH);
895+
SET time_zone='+00:00';
896+
INSERT INTO t1 VALUES ('2001-01-01 10:20:30');
897+
SET time_zone='+01:00';
898+
CHECK TABLE t1;
899+
DROP TABLE t1;
900+
SET time_zone=DEFAULT;
901+
881902
--echo #
882903
--echo # End of 10.4 tests
883904
--echo #

‎mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD‎

40 Bytes
Binary file not shown.

‎mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI‎

2 KB
Binary file not shown.

‎mysql-test/std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm‎

970 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)