MDEV-8921 Wrong result for CAST(AVG(double_column) AS SIGNED)

Author: Alexander Barkov

mysql-test/r/func_group.result

@@ -2298,5 +2298,21 @@ id	avg	cast_avg
 2	9223372036854775807.0000	9223372036854775807
 DROP TABLE t1;
 #
+# MDEV-8921 Wrong result for CAST(AVG(double_column) AS SIGNED)
+#
+CREATE TABLE t1 (id INT, a DOUBLE);
+INSERT INTO t1 VALUES (1,0x7FFFFFFFFFFFFFFF),(2,0x7FFFFFFFFFFFFFFF);
+SELECT id, AVG(a) AS avg, CAST(MIN(a) AS SIGNED) AS cast_min,CAST(AVG(a) AS SIGNED) AS cast_avg FROM t1 GROUP BY id HAVING avg!=123 ORDER BY id;
+id	avg	cast_min	cast_avg
+1	9.223372036854776e18	9223372036854775807	9223372036854775807
+2	9.223372036854776e18	9223372036854775807	9223372036854775807
+DROP TABLE t1;
+CREATE TABLE t1 (a DOUBLE);
+INSERT INTO t1 VALUES (0x7FFFFFFFFFFFFFFF);
+SELECT MIN(a), SUM(a), CAST(SUM(a) AS SIGNED), CAST(AVG(a) AS SIGNED) FROM t1;
+MIN(a)	SUM(a)	CAST(SUM(a) AS SIGNED)	CAST(AVG(a) AS SIGNED)
+9.223372036854776e18	9.223372036854776e18	9223372036854775807	9223372036854775807
+DROP TABLE t1;
+#
 # End of 10.1 tests
 #

mysql-test/t/func_group.test

@@ -1581,6 +1581,18 @@ SELECT id, AVG(a) AS avg, CAST(MIN(a) AS SIGNED) AS cast_min FROM t1 GROUP BY id
 SELECT id, AVG(a) AS avg, CAST(AVG(a) AS SIGNED) AS cast_avg FROM t1 GROUP BY id HAVING avg!=123 ORDER BY id;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-8921 Wrong result for CAST(AVG(double_column) AS SIGNED)
+--echo #
+CREATE TABLE t1 (id INT, a DOUBLE);
+INSERT INTO t1 VALUES (1,0x7FFFFFFFFFFFFFFF),(2,0x7FFFFFFFFFFFFFFF);
+SELECT id, AVG(a) AS avg, CAST(MIN(a) AS SIGNED) AS cast_min,CAST(AVG(a) AS SIGNED) AS cast_avg FROM t1 GROUP BY id HAVING avg!=123 ORDER BY id;
+DROP TABLE t1;
+CREATE TABLE t1 (a DOUBLE);
+INSERT INTO t1 VALUES (0x7FFFFFFFFFFFFFFF);
+SELECT MIN(a), SUM(a), CAST(SUM(a) AS SIGNED), CAST(AVG(a) AS SIGNED) FROM t1;
+DROP TABLE t1;
+
 --echo #
 --echo # End of 10.1 tests
 --echo #

sql/item.cc

@@ -359,6 +359,14 @@ longlong Item::val_int_from_date()
 }
 
 
+longlong Item::val_int_from_real()
+{
+ DBUG_ASSERT(fixed == 1);
+ bool error;
+ return double_to_longlong(val_real(), false /*unsigned_flag*/, &error);
+}
+
+
 double Item::val_real_from_date()
 {
  DBUG_ASSERT(fixed == 1);

sql/item.h

@@ -1008,6 +1008,7 @@ class Item: public Value_source,
  my_decimal *val_decimal_from_time(my_decimal *decimal_value);
  longlong val_int_from_decimal();
  longlong val_int_from_date();
+ longlong val_int_from_real();
  double val_real_from_decimal();
  double val_real_from_date();
 

sql/item_sum.cc

@@ -1430,7 +1430,7 @@ longlong Item_sum_sum::val_int()
  &result);
  return result;
  }
- return (longlong) rint(val_real());
+ return val_int_from_real();
 }
 
 

sql/item_sum.h

@@ -704,11 +704,7 @@ class Item_sum_num :public Item_sum
  Item_sum_num(THD *thd, Item_sum_num *item):
  Item_sum(thd, item),is_evaluated(item->is_evaluated) {}
  bool fix_fields(THD *, Item **);
- longlong val_int()
- {
- DBUG_ASSERT(fixed == 1);
- return (longlong) rint(val_real()); /* Real as default */
- }
+ longlong val_int() { return val_int_from_real(); /* Real as default */ }
  String *val_str(String*str);
  my_decimal *val_decimal(my_decimal *);
  void reset_field();
@@ -843,7 +839,7 @@ class Item_sum_avg :public Item_sum_sum
  bool add();
  double val_real();
  // In SPs we might force the "wrong" type with select into a declare variable
- longlong val_int() { return (longlong) rint(val_real()); }
+ longlong val_int() { return val_int_from_real(); }
  my_decimal *val_decimal(my_decimal *);
  String *val_str(String *str);
  void reset_field();
@@ -1093,6 +1089,7 @@ class Item_sum_field :public Item
  decimals= item->decimals;
  max_length= item->max_length;
  unsigned_flag= item->unsigned_flag;
+ fixed= true;
  }
  table_map used_tables() const { return (table_map) 1L; }
  Field *get_tmp_table_field() { DBUG_ASSERT(0); return NULL; }
@@ -1127,7 +1124,7 @@ class Item_avg_field_double :public Item_avg_field
  { }
  enum_field_types field_type() const { return MYSQL_TYPE_DOUBLE; }
  enum Item_result result_type () const { return REAL_RESULT; }
- longlong val_int() { return (longlong) rint(val_real()); }
+ longlong val_int() { return val_int_from_real(); }
  my_decimal *val_decimal(my_decimal *dec) { return val_decimal_from_real(dec); }
  String *val_str(String *str) { return val_string_from_real(str); }
  double val_real();
@@ -1162,8 +1159,7 @@ class Item_variance_field :public Item_sum_field
  { }
  enum Type type() const {return FIELD_VARIANCE_ITEM; }
  double val_real();
- longlong val_int()
- { /* can't be fix_fields()ed */ return (longlong) rint(val_real()); }
+ longlong val_int() { return val_int_from_real(); }
  String *val_str(String *str)
  { return val_string_from_real(str); }
  my_decimal *val_decimal(my_decimal *dec_buf)
@@ -1246,11 +1242,7 @@ class Item_sum_udf_float :public Item_udf_sum
  Item_udf_sum(thd, udf_arg, list) {}
  Item_sum_udf_float(THD *thd, Item_sum_udf_float *item)
  :Item_udf_sum(thd, item) {}
- longlong val_int()
- {
- DBUG_ASSERT(fixed == 1);
- return (longlong) rint(Item_sum_udf_float::val_real());
- }
+ longlong val_int() { return val_int_from_real(); }
  double val_real();
  String *val_str(String*str);
  my_decimal *val_decimal(my_decimal *);