Bug#52164 Assertion failed: param.sort_length, file .\filesort.cc, line 149

The crash happens because of incorrect max_length calculation
in QUOTE function(due to overflow). max_length is set
to 0 and it leads to assert failure.
The fix is to cast expression result to
ulonglong variable and adjust it if the
result exceeds MAX_BLOB_WIDTH.

mysql-test/r/func_str.result

@@ -2587,3 +2587,17 @@ FROM t2 WHERE t2.b = 1 GROUP BY t2.b;
 DECODE((SELECT ENCODE('secret', t1.a) FROM t1,t2 WHERE t1.a = t2.a GROUP BY t1.b LIMIT 1), t2.a)
 secret
 DROP TABLE t1, t2;
+#
+# Bug#52164 Assertion failed: param.sort_length, file .\filesort.cc, line 149
+#
+CREATE TABLE t1 (a LONGBLOB NOT NULL);
+INSERT INTO t1 VALUES (''),('');
+SELECT 1 FROM t1, t1 t2
+ORDER BY QUOTE(t1.a);
+1
+1
+1
+1
+1
+DROP TABLE t1;
+End of 5.1 tests

mysql-test/t/func_str.test

@@ -1352,3 +1352,14 @@ SELECT DECODE((SELECT ENCODE('secret', t1.a) FROM t1,t2 WHERE t1.a = t2.a GROUP
   FROM t2 WHERE t2.b = 1 GROUP BY t2.b;
 
 DROP TABLE t1, t2;
+
+--echo #
+--echo # Bug#52164 Assertion failed: param.sort_length, file .\filesort.cc, line 149
+--echo #
+CREATE TABLE t1 (a LONGBLOB NOT NULL);
+INSERT INTO t1 VALUES (''),('');
+SELECT 1 FROM t1, t1 t2
+ORDER BY QUOTE(t1.a);
+DROP TABLE t1;
+
+--echo End of 5.1 tests

sql/item_strfunc.h

@@ -694,8 +694,9 @@ class Item_func_quote :public Item_str_func
   String *val_str(String *);
   void fix_length_and_dec()
   {
+    ulonglong max_result_length= (ulonglong) args[0]->max_length * 2 + 2;
+    max_length= min(max_result_length, MAX_BLOB_WIDTH);
     collation.set(args[0]->collation);
-    max_length= args[0]->max_length * 2 + 2;
   }
 };