bug in my_strnncoll_utf8 (and friends) fixed

cleanups
better, charset-dependent, ft_max_len_for_sort value

myisam/ftdefs.h

@@ -27,7 +27,7 @@
 #define misc_word_char(X)	((X)=='\'')
 #define word_char(s,X)		(true_word_char(s,X) || misc_word_char(X))
 
-#define FT_MAX_WORD_LEN_FOR_SORT 20
+#define FT_MAX_WORD_LEN_FOR_SORT 31
 
 #define COMPILE_STOPWORDS_IN
 

myisam/mi_check.c

@@ -2020,12 +2020,14 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
 
     if (sort_param.keyinfo->flag & HA_FULLTEXT)
     {
+      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
+                                    sort_param.keyinfo->seg->charset->mbmaxlen;
       sort_info.max_records=
-        (ha_rows) (sort_info.filelength/FT_MAX_WORD_LEN_FOR_SORT+1);
+        (ha_rows) (sort_info.filelength/ft_max_word_len_for_sort+1);
 
       sort_param.key_read=sort_ft_key_read;
       sort_param.key_write=sort_ft_key_write;
-      sort_param.key_length+=FT_MAX_WORD_LEN_FOR_SORT-HA_FT_MAXBYTELEN;
+      sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
     }
     else
     {
@@ -2425,7 +2427,11 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
     total_key_length+=sort_param[i].key_length;
 
     if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
-      sort_param[i].key_length+=FT_MAX_WORD_LEN_FOR_SORT-HA_FT_MAXBYTELEN;
+    {
+      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
+                                    sort_param[i].keyinfo->seg->charset->mbmaxlen;
+      sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
+    }
   }
   sort_info.total_keys=i;
   sort_param[0].master= 1;
@@ -2634,7 +2640,6 @@ static int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
   DBUG_RETURN(sort_write_record(sort_param));
 } /* sort_key_read */
 
-
 static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
 {
   int error;
@@ -3950,25 +3955,28 @@ static ha_checksum mi_byte_checksum(const byte *buf, uint length)
   return crc;
 }
 
-/*
-  Deactive all not unique index that can be recreated fast
-  These include packed keys on which sorting will use more temporary
-  space than the max allowed file length or for which the unpacked keys
-  will take much more space than packed keys.
-  Note that 'rows' may be zero for the case when we don't know how many
-  rows we will put into the file.
- */
-
 static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
 {
   uint key_maxlength=key->maxlength;
   if (key->flag & HA_FULLTEXT)
-    key_maxlength+=FT_MAX_WORD_LEN_FOR_SORT-HA_FT_MAXBYTELEN;
+  {
+    uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
+                                  key->seg->charset->mbmaxlen;
+    key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
+  }
   return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&
 	  ((ulonglong) rows * key_maxlength >
 	   (ulonglong) myisam_max_temp_length));
 }
 
+/*
+  Deactivate all not unique index that can be recreated fast
+  These include packed keys on which sorting will use more temporary
+  space than the max allowed file length or for which the unpacked keys
+  will take much more space than packed keys.
+  Note that 'rows' may be zero for the case when we don't know how many
+  rows we will put into the file.
+ */
 
 void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
 {

myisam/myisamchk.c

@@ -154,7 +154,7 @@ enum options_mc {
   OPT_KEY_CACHE_BLOCK_SIZE, OPT_MYISAM_BLOCK_SIZE,
   OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
   OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
-  OPT_FT_MAX_WORD_LEN, OPT_FT_MAX_WORD_LEN_FOR_SORT, OPT_FT_STOPWORD_FILE,
+  OPT_FT_MAX_WORD_LEN, OPT_FT_STOPWORD_FILE,
   OPT_MAX_RECORD_LENGTH
 };
 

myisam/sort.c

@@ -204,7 +204,7 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
 	reinit_io_cache(&tempfile,READ_CACHE,0L,0,0))
       goto err;					/* purecov: inspected */
     if (!no_messages)
-      puts("  - Last merge and dumping keys\n"); /* purecov: tested */
+      printf("  - Last merge and dumping keys\n"); /* purecov: tested */
     if (merge_index(info,keys,sort_keys,dynamic_element(&buffpek,0,BUFFPEK *),
                     maxbuffer,&tempfile))
       goto err;					/* purecov: inspected */
@@ -219,6 +219,8 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
     uint     keyno=info->key;
     uint     key_length, ref_length=index->s->rec_reflength;
 
+    if (!no_messages)
+      printf("  - Adding exceptions\n"); /* purecov: tested */
     if (flush_io_cache(&tempfile_for_exceptions) ||
 	reinit_io_cache(&tempfile_for_exceptions,READ_CACHE,0L,0,0))
       goto err;

sql/mysql_priv.h

@@ -783,10 +783,10 @@ int key_cmp(KEY_PART_INFO *key_part, const byte *key, uint key_length);
 bool init_errmessage(void);
 void sql_perror(const char *message);
 
-void vprint_msg_to_log( enum loglevel level, const char *format, va_list args );
-void sql_print_error( const char *format, ... );
-void sql_print_warning( const char *format, ...); 
-void sql_print_information( const char *format, ...); 
+void vprint_msg_to_log(enum loglevel level, const char *format, va_list args);
+void sql_print_error(const char *format, ...);
+void sql_print_warning(const char *format, ...);
+void sql_print_information(const char *format, ...);
 
 
 

sql/sql_acl.cc

@@ -251,7 +251,7 @@ my_bool acl_init(THD *org_thd, bool dont_read_acl_tables)
     {
       global_system_variables.old_passwords= 1;
       pthread_mutex_unlock(&LOCK_global_system_variables);
-      sql_print_error("mysql.user table is not updated to new password format; "
+      sql_print_warning("mysql.user table is not updated to new password format; "
                         "Disabling new password usage until "
                         "mysql_fix_privilege_tables is run");
     }

sql/tztime.cc

@@ -1560,7 +1560,7 @@ my_tz_init(THD *org_thd, const char *default_tzname, my_bool bootstrap)
   if (open_tables(thd, tables_buff, &counter) ||
       lock_tables(thd, tables_buff, counter))
   {
-    sql_print_error("Warning: Can't open and lock time zone table: %s "
+    sql_print_warning("Can't open and lock time zone table: %s "
                       "trying to live without them", thd->net.last_error);
     /* We will try emulate that everything is ok */
     return_val= time_zone_tables_exist= 0;
@@ -1740,8 +1740,7 @@ tz_load_from_open_tables(const String *tz_name, TABLE_LIST *tz_tables)
   if (!(alloc_buff= alloc_root(&tz_storage, sizeof(TIME_ZONE_INFO) +
                                tz_name->length() + 1)))
   {
-    sql_print_error("Error: Out of memory while loading time zone "
-                    "description");
+    sql_print_error("Out of memory while loading time zone description");
     return 0;
   }
   tz_info= (TIME_ZONE_INFO *)alloc_buff;
@@ -1770,7 +1769,7 @@ tz_load_from_open_tables(const String *tz_name, TABLE_LIST *tz_tables)
   if (table->file->index_read(table->record[0], (byte*)table->field[0]->ptr,
                               0, HA_READ_KEY_EXACT))
   {
-    sql_print_error("Error: Can't find description of time zone.");
+    sql_print_error("Can't find description of time zone.");
     goto end;
   }
 
@@ -1791,7 +1790,7 @@ tz_load_from_open_tables(const String *tz_name, TABLE_LIST *tz_tables)
   if (table->file->index_read(table->record[0], (byte*)table->field[0]->ptr,
                               0, HA_READ_KEY_EXACT))
   {
-    sql_print_error("Error: Can't find description of time zone.");
+    sql_print_error("Can't find description of time zone.");
     goto end;
   }
 
@@ -1948,8 +1947,7 @@ tz_load_from_open_tables(const String *tz_name, TABLE_LIST *tz_tables)
 #endif
                                sizeof(TRAN_TYPE_INFO) * tz_info->typecnt)))
   {
-    sql_print_error("Error: Out of memory while loading time zone "
-                    "description");
+    sql_print_error("Out of memory while loading time zone description");
     goto end;
   }
 
@@ -1974,12 +1972,12 @@ tz_load_from_open_tables(const String *tz_name, TABLE_LIST *tz_tables)
   */
   if (tz_info->typecnt < 1)
   {
-    sql_print_error("Error: loading time zone without transition types");
+    sql_print_error("loading time zone without transition types");
     goto end;
   }
   if (prepare_tz_info(tz_info, &tz_storage))
   {
-    sql_print_error("Error: Unable to build mktime map for time zone");
+    sql_print_error("Unable to build mktime map for time zone");
     goto end;
   }
 
@@ -1991,7 +1989,7 @@ tz_load_from_open_tables(const String *tz_name, TABLE_LIST *tz_tables)
                             &my_charset_latin1),
        my_hash_insert(&tz_names, (const byte *)tmp_tzname)))
   {
-    sql_print_error("Error: Out of memory while loading time zone");
+    sql_print_error("Out of memory while loading time zone");
     goto end;
   }
 

strings/ctype-utf8.c

@@ -1590,6 +1590,15 @@ static uchar to_upper_utf8[] = {
   240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255
 };
 
+static inline int bincmp(const uchar *s, const uchar *se,
+                         const uchar *t, const uchar *te)
+{
+  int slen=se-s, tlen=te-t;
+  int len=min(slen,tlen);
+  int cmp= memcmp(s,t,len);
+  return cmp ? cmp : slen-tlen;
+}
+
 
 static int my_utf8_uni(CHARSET_INFO *cs __attribute__((unused)),
                        my_wc_t * pwc, const uchar *s, const uchar *e)
@@ -1824,8 +1833,8 @@ static int my_strnncoll_utf8(CHARSET_INFO *cs,
 
     if ( s_res <= 0 || t_res <= 0 )
     {
-      /* Incorrect string, compare by char value */
-      return ((int)s[0]-(int)t[0]); 
+      /* Incorrect string, compare byte by byte value */
+      return bincmp(s, se, t, te);
     }
 
     plane=(s_wc>>8) & 0xFF;
@@ -1889,8 +1898,8 @@ static int my_strnncollsp_utf8(CHARSET_INFO *cs,
 
     if ( s_res <= 0 || t_res <= 0 )
     {
-      /* Incorrect string, compare by char value */
-      return ((int)s[0]-(int)t[0]); 
+      /* Incorrect string, compare byte by byte value */
+      return bincmp(s, se, t, te);
     }
 
     plane=(s_wc>>8) & 0xFF;
@@ -1956,8 +1965,8 @@ static int my_strncasecmp_utf8(CHARSET_INFO *cs,
 
     if ( s_res <= 0 || t_res <= 0 )
     {
-      /* Incorrect string, compare by char value */
-      return ((int)s[0]-(int)t[0]); 
+      /* Incorrect string, compare byte by byte value */
+      return bincmp(s, se, t, te);
     }
 
     plane=(s_wc>>8) & 0xFF;