MDEV-21946: Server crash in store_length upon GROUP BY WITH ROLLUP with geometry field

overflow happening when we add suffix length to sortlength.
Make sure that sortlength is at max UINT_MAX32

mysql-test/main/order_by.result

@@ -3850,3 +3850,19 @@ ANALYZE
   }
 }
 drop table t1;
+#
+# MDEV-21946: Server crash in store_length upon GROUP BY WITH ROLLUP with geometry field
+#
+create table t1 ( a longblob);
+insert into t1 select repeat('a', 256);
+insert into t1 select repeat('b', 256);
+insert into t1 select repeat('c', 256);
+insert into t1 select repeat('d', 256);
+SELECT IF( 0, NULL, a ) AS f FROM t1 GROUP BY f WITH ROLLUP;
+f
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc
+dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd
+NULL
+DROP TABLE t1;

mysql-test/main/order_by.test

@@ -2400,3 +2400,16 @@ select * from t1 order by a;
 --source include/analyze-format.inc
 analyze format=json select * from t1 order by a;
 drop table t1;
+
+--echo #
+--echo # MDEV-21946: Server crash in store_length upon GROUP BY WITH ROLLUP with geometry field
+--echo #
+
+create table t1 ( a longblob);
+insert into t1 select repeat('a', 256);
+insert into t1 select repeat('b', 256);
+insert into t1 select repeat('c', 256);
+insert into t1 select repeat('d', 256);
+
+SELECT IF( 0, NULL, a ) AS f FROM t1 GROUP BY f WITH ROLLUP;
+DROP TABLE t1;

sql/filesort.cc

@@ -2105,6 +2105,7 @@ Type_handler_string_result::sort_length(THD *thd,
 {
   CHARSET_INFO *cs;
   sortorder->length= item->max_length;
+  set_if_smaller(sortorder->length, thd->variables.max_sort_length);
   sortorder->original_length= item->max_length;
 
   if (use_strnxfrm((cs= item->collation.collation)))
@@ -2114,9 +2115,13 @@ Type_handler_string_result::sort_length(THD *thd,
   else if (cs == &my_charset_bin)
   {
     /* Store length last to be able to sort blob/varbinary */
-    sortorder->suffix_length= suffix_length(sortorder->length);
+    sortorder->suffix_length= suffix_length(item->max_length);
+    DBUG_ASSERT(sortorder->length <= UINT_MAX32 - sortorder->suffix_length);
     sortorder->length+= sortorder->suffix_length;
-    sortorder->original_length+= sortorder->suffix_length;
+    if (sortorder->original_length >= UINT_MAX32 - sortorder->suffix_length)
+      sortorder->original_length= UINT_MAX32;
+    else
+      sortorder->original_length+= sortorder->suffix_length;
   }
 }