MDEV-34509: UBSAN: call to function option_cmp(my_option*, my_option*) through...

MDEV-34509: UBSAN: call to function option_cmp(my_option*, my_option*) through pointer to incorrect function type

Clang-18 was detecting these calls as undefined behaviour.

Comparison functions to conform to the specification require constant
arguments.

Make this the same for filesort too.

sql/filesort.cc

@@ -2801,8 +2801,8 @@ qsort2_cmp get_packed_keys_compare_ptr()
   suffix_bytes are used only for binary columns.
 */
 
-int SORT_FIELD_ATTR::compare_packed_varstrings(uchar *a, size_t *a_len,
-                                               uchar *b, size_t *b_len)
+int SORT_FIELD_ATTR::compare_packed_varstrings(const uchar *a, size_t *a_len,
+                                               const uchar *b, size_t *b_len)
 {
   int retval;
   size_t a_length, b_length;
@@ -2861,8 +2861,8 @@ int SORT_FIELD_ATTR::compare_packed_varstrings(uchar *a, size_t *a_len,
   packed-value format.
 */
 
-int SORT_FIELD_ATTR::compare_packed_fixed_size_vals(uchar *a, size_t *a_len,
-                                                    uchar *b, size_t *b_len)
+int SORT_FIELD_ATTR::compare_packed_fixed_size_vals(const uchar *a, size_t *a_len,
+                                                    const uchar *b, size_t *b_len)
 {
   if (maybe_null)
   {
@@ -2907,15 +2907,15 @@ int SORT_FIELD_ATTR::compare_packed_fixed_size_vals(uchar *a, size_t *a_len,
 
 */
 
-int compare_packed_sort_keys(void *sort_param,
-                             unsigned char **a_ptr, unsigned char **b_ptr)
+int compare_packed_sort_keys(const void *sort_param,
+                             const void *a_ptr, const void *b_ptr)
 {
   int retval= 0;
   size_t a_len, b_len;
   Sort_param *param= (Sort_param*)sort_param;
   Sort_keys *sort_keys= param->sort_keys;
-  uchar *a= *a_ptr;
-  uchar *b= *b_ptr;
+  const uchar *a= *((const uchar **) a_ptr);
+  const uchar *b= *((const uchar **) b_ptr);
 
   a+= Sort_keys::size_of_length_field;
   b+= Sort_keys::size_of_length_field;

sql/filesort_utils.h

@@ -268,8 +268,8 @@ class Filesort_buffer
   longlong m_idx;
 };
 
-int compare_packed_sort_keys(void *sort_keys, unsigned char **a,
-                             unsigned char **b);
+int compare_packed_sort_keys(const void *sort_param,
+                             const void *a_ptr, const void *b_ptr);
 qsort2_cmp get_packed_keys_compare_ptr();
 
 #endif  // FILESORT_UTILS_INCLUDED

sql/mysqld.cc

@@ -7656,10 +7656,10 @@ static void print_version(void)
 }
 
 /** Compares two options' names, treats - and _ the same */
-static int option_cmp(my_option *a, my_option *b)
+static int option_cmp(const void *a, const void *b)
 {
-  const char *sa= a->name;
-  const char *sb= b->name;
+  const char *sa= static_cast<const my_option *>(a)->name;
+  const char *sb= static_cast<const my_option *>(b)->name;
   for (; *sa || *sb; sa++, sb++)
   {
     if (*sa < *sb)

sql/sql_acl.cc

@@ -680,8 +680,8 @@ static DYNAMIC_ARRAY acl_wild_hosts;
 static Hash_filo<acl_entry> *acl_cache;
 static uint grant_version=0; /* Version of priv tables. incremented by acl_load */
 static privilege_t get_access(TABLE *form, uint fieldnr, uint *next_field=0);
-static int acl_compare(const ACL_ACCESS *a, const ACL_ACCESS *b);
-static int acl_user_compare(const ACL_USER *a, const ACL_USER *b);
+static int acl_compare(const void *a, const void *b);
+static int acl_user_compare(const void *a, const void *b);
 static void rebuild_acl_users();
 static int acl_db_compare(const void *a, const void *b);
 static void rebuild_acl_dbs();
@@ -2998,8 +2998,10 @@ static privilege_t get_access(TABLE *form, uint fieldnr, uint *next_field)
 }
 
 
-static int acl_compare(const ACL_ACCESS *a, const ACL_ACCESS *b)
+static int acl_compare(const void *_a, const void *_b)
 {
+  const ACL_ACCESS *a= static_cast<const ACL_ACCESS *>(_a);
+  const ACL_ACCESS *b= static_cast<const ACL_ACCESS *>(_b);
   if (a->sort > b->sort)
     return -1;
   if (a->sort < b->sort)
@@ -3007,8 +3009,10 @@ static int acl_compare(const ACL_ACCESS *a, const ACL_ACCESS *b)
   return 0;
 }
 
-static int acl_user_compare(const ACL_USER *a, const ACL_USER *b)
+static int acl_user_compare(const void *_a, const void *_b)
 {
+  const ACL_USER *a= static_cast<const ACL_USER *>(_a);
+  const ACL_USER *b= static_cast<const ACL_USER *>(_b);
   int res= strcmp(a->user.str, b->user.str);
   if (res)
     return res;

sql/sql_class.h

@@ -6743,10 +6743,10 @@ struct SORT_FIELD_ATTR
   CHARSET_INFO *cs;
   uint pack_sort_string(uchar *to, const Binary_string *str,
                         CHARSET_INFO *cs) const;
-  int compare_packed_fixed_size_vals(uchar *a, size_t *a_len,
-                                     uchar *b, size_t *b_len);
-  int compare_packed_varstrings(uchar *a, size_t *a_len,
-                                uchar *b, size_t *b_len);
+  int compare_packed_fixed_size_vals(const uchar *a, size_t *a_len,
+                                     const uchar *b, size_t *b_len);
+  int compare_packed_varstrings(const uchar *a, size_t *a_len,
+                                const uchar *b, size_t *b_len);
   bool check_if_packing_possible(THD *thd) const;
   bool is_variable_sized() { return type == VARIABLE_SIZE; }
   void set_length_and_original_length(THD *thd, uint length_arg);