MDEV-12350: Heap corruption, overrun buffer, ASAN errors, server crash in my_fill_8bit / filesort

In the function make_sortkey a tmp buffer was defined and in the absence of param->tmp_buffer, tmp buffer used the sort_keys buffer. sort_keys buffer has a length defined in sort_field->length, while param->tmp_buffer is stored in param->rec_length. Make sure to use the appropriate length based on which buffer we are using otherwise we'll overflow. Also added a type cast to size_t during the calculation of the sort keys buffer size to avoid an oveflow if the buffer size exceeds 32 bits.

Author: varunraiko

mysql-test/r/group_by.result

@@ -2558,3 +2558,28 @@ create table t2 (c1 int, c2 int);
 select t1.c1 as c1, t2.c2 as c1 from t1, t2 where t1.c1 < 20 and t2.c2 > 5 group by t1.c1, t2.c2 having t1.c1 < 3;
 c1	c1
 drop table t1, t2;
+SET @old_sort_buff_size = @@sort_buffer_size;
+SET @@sort_buffer_size=256*1024;
+CREATE TABLE t1 (c INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES
+(2011),(1977),(1982),(2027),(2023),(NULL),(NULL),(2004),(1974),(2032),
+(1993),(NULL),(1995),(2034),(NULL),(2009),(1900),(NULL),(2025),(1900),
+(2033),(1900),(2012),(NULL),(2009),(1992),(1974),(1974),(2012),(2028),
+(2007),(2012),(1900),(1983),(1900),(2010),(1987),(1994),(1981),(2032),
+(2010),(1989),(2014),(1900),(1900),(1976),(1978),(2007),(2030),(NULL),
+(2002),(1997),(1900),(NULL),(2000),(2027),(1975),(2026),(1975),(2026),
+(2029),(1977),(1900),(1900),(2031),(1993),(1986),(2012),(1979),(2013),
+(1994),(2014),(2025),(2006),(1971),(1974),(2021),(2011),(NULL),(1991),
+(2001),(1977),(2023),(2012),(1900),(1978),(1998),(NULL),(1988),(1999),
+(2017),(2008),(1976),(1900),(2005),(2030),(2023),(1900),(1978),(1990),
+(1978),(1987),(2030),(1900),(2034),(2006),(2015),(2001),(2019),(2024),
+(2030),(1989),(1997),(2007),(2023),(1994),(1971),(2011),(2011),(2015),
+(1984),(1978),(1979),(1989),(2008),(2030);
+SELECT ExtractValue('<a></a>','/a') AS f1, SPACE(c) AS f2 FROM t1 GROUP BY f1, f2 WITH ROLLUP;
+f1	f2
+NULL
+ 
+NULL
+NULL	NULL
+SET @@sort_buffer_size = @old_sort_buff_size;
+DROP TABLE t1;

mysql-test/t/group_by.test

@@ -1741,6 +1741,32 @@ create table t2 (c1 int, c2 int);
 select t1.c1 as c1, t2.c2 as c1 from t1, t2 where t1.c1 < 20 and t2.c2 > 5 group by t1.c1, t2.c2 having t1.c1 < 3;
 drop table t1, t2;
 
+#
+# MDEV-12350: Heap corruption, overrun buffer, ASAN errors, server crash in my_fill_8bit / filesort
+#
+
+SET @old_sort_buff_size = @@sort_buffer_size;
+SET @@sort_buffer_size=256*1024;
+CREATE TABLE t1 (c INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES
+ (2011),(1977),(1982),(2027),(2023),(NULL),(NULL),(2004),(1974),(2032),
+ (1993),(NULL),(1995),(2034),(NULL),(2009),(1900),(NULL),(2025),(1900),
+ (2033),(1900),(2012),(NULL),(2009),(1992),(1974),(1974),(2012),(2028),
+ (2007),(2012),(1900),(1983),(1900),(2010),(1987),(1994),(1981),(2032),
+ (2010),(1989),(2014),(1900),(1900),(1976),(1978),(2007),(2030),(NULL),
+ (2002),(1997),(1900),(NULL),(2000),(2027),(1975),(2026),(1975),(2026),
+ (2029),(1977),(1900),(1900),(2031),(1993),(1986),(2012),(1979),(2013),
+ (1994),(2014),(2025),(2006),(1971),(1974),(2021),(2011),(NULL),(1991),
+ (2001),(1977),(2023),(2012),(1900),(1978),(1998),(NULL),(1988),(1999),
+ (2017),(2008),(1976),(1900),(2005),(2030),(2023),(1900),(1978),(1990),
+ (1978),(1987),(2030),(1900),(2034),(2006),(2015),(2001),(2019),(2024),
+ (2030),(1989),(1997),(2007),(2023),(1994),(1971),(2011),(2011),(2015),
+ (1984),(1978),(1979),(1989),(2008),(2030);
+ 
+SELECT ExtractValue('<a></a>','/a') AS f1, SPACE(c) AS f2 FROM t1 GROUP BY f1, f2 WITH ROLLUP;
+SET @@sort_buffer_size = @old_sort_buff_size;
+DROP TABLE t1;
+
 #
 # End of MariaDB 5.5 tests
 #

sql/filesort.cc

@@ -210,7 +210,9 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
  {
  ulonglong keys= memory_available / (param.rec_length + sizeof(char*));
  table_sort.keys= (uint) min(num_rows, keys);
- sort_buff_sz= table_sort.keys*(param.rec_length+sizeof(char*));
+ /* Cast to size_t to avoid overflow when result is greater than uint. */
+ sort_buff_sz= ((size_t)table_sort.keys) *
+ (param.rec_length + sizeof(char*));
  set_if_bigger(sort_buff_sz, param.rec_length * MERGEBUFF2); 
 
  DBUG_EXECUTE_IF("make_sort_keys_alloc_fail",
@@ -914,7 +916,8 @@ static void make_sortkey(register SORTPARAM *param,
  if (maybe_null)
  *to++=1;
  char *tmp_buffer= param->tmp_buffer ? param->tmp_buffer : (char*)to;
- String tmp(tmp_buffer, param->sort_length, cs);
+ String tmp(tmp_buffer, param->tmp_buffer ? param->sort_length :
+ sort_field->length, cs);
  String *res= item->str_result(&tmp);
  if (!res)
  {