Merge chilla.local:/home/mydev/mysql-5.1-amain

into  chilla.local:/home/mydev/mysql-5.1-axmrg

mysql-test/r/fulltext.result

@@ -476,3 +476,15 @@ ALTER TABLE t1 DISABLE KEYS;
 SELECT * FROM t1 WHERE MATCH(a) AGAINST('test');
 ERROR HY000: Can't find FULLTEXT index matching the column list
 DROP TABLE t1;
+CREATE TABLE t1(a VARCHAR(20), FULLTEXT(a));
+INSERT INTO t1 VALUES('Offside'),('City Of God');
+SELECT a FROM t1 WHERE MATCH a AGAINST ('+city of*' IN BOOLEAN MODE);
+a
+City Of God
+SELECT a FROM t1 WHERE MATCH a AGAINST ('+city (of*)' IN BOOLEAN MODE);
+a
+City Of God
+SELECT a FROM t1 WHERE MATCH a AGAINST ('+city* of*' IN BOOLEAN MODE);
+a
+City Of God
+DROP TABLE t1;

mysql-test/r/fulltext3.result

@@ -11,3 +11,7 @@ Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 SET NAMES latin1;
 DROP TABLE t1;
+CREATE TABLE t1(a VARCHAR(2) CHARACTER SET big5 COLLATE big5_chinese_ci,
+FULLTEXT(a));
+INSERT INTO t1 VALUES(0xA3C2);
+DROP TABLE t1;

mysql-test/suite/ndb/r/ndb_update.result

 DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+DROP TABLE IF EXISTS t3;
 CREATE TABLE t1 (
 pk1 INT NOT NULL PRIMARY KEY,
 b INT NOT NULL,
@@ -40,3 +42,47 @@ pk1	b	c
 12	2	2
 14	1	1
 DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (a int, b int, KEY (a, b)) ENGINE=ndbcluster;
+CREATE TABLE t2 (a int, b int, UNIQUE KEY (a, b)) ENGINE=ndbcluster;
+CREATE TABLE t3 (a int, b int, PRIMARY KEY (a, b)) ENGINE=ndbcluster;
+INSERT INTO t1 VALUES (1, 2);
+INSERT INTO t1 VALUES (2, 2);
+INSERT INTO t2 VALUES (1, 2);
+INSERT INTO t2 VALUES (2, 2);
+INSERT INTO t3 VALUES (1, 2);
+INSERT INTO t3 VALUES (2, 2);
+UPDATE t1 SET a = 1;
+UPDATE t1 SET a = 1 ORDER BY a;
+UPDATE t2 SET a = 1;
+ERROR 23000: Duplicate entry '' for key '*UNKNOWN*'
+UPDATE t2 SET a = 1 ORDER BY a;
+ERROR 23000: Duplicate entry '' for key '*UNKNOWN*'
+UPDATE t3 SET a = 1;
+ERROR 23000: Duplicate entry '1-2' for key 'PRIMARY'
+UPDATE t3 SET a = 1 ORDER BY a;
+ERROR 23000: Duplicate entry '1-2' for key 'PRIMARY'
+SELECT count(*) FROM t1;
+count(*)
+2
+SELECT count(*) FROM t2;
+count(*)
+2
+SELECT count(*) FROM t3;
+count(*)
+2
+SELECT * FROM t1 ORDER by a;
+a	b
+1	2
+1	2
+SELECT * FROM t2 ORDER by a;
+a	b
+1	2
+2	2
+SELECT * FROM t3 ORDER by a;
+a	b
+1	2
+2	2
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+DROP TABLE IF EXISTS t3;
+End of 5.1 tests

mysql-test/suite/ndb/t/ndb_update.test

@@ -3,10 +3,12 @@
 
 --disable_warnings
 DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+DROP TABLE IF EXISTS t3;
 --enable_warnings
 
 #
-# Basic test of INSERT in NDB 
+# Basic test of UPDATE in NDB 
 #
 
 #
@@ -39,3 +41,49 @@ DROP TABLE IF EXISTS t1;
 --enable_warnings
 
 # End of 4.1 tests
+
+#
+# Bug#28158: table->read_set is set incorrectly,
+#            causing wrong error message in Falcon
+#
+CREATE TABLE t1 (a int, b int, KEY (a, b)) ENGINE=ndbcluster;
+CREATE TABLE t2 (a int, b int, UNIQUE KEY (a, b)) ENGINE=ndbcluster;
+CREATE TABLE t3 (a int, b int, PRIMARY KEY (a, b)) ENGINE=ndbcluster;
+#
+INSERT INTO t1 VALUES (1, 2);
+INSERT INTO t1 VALUES (2, 2);
+#
+INSERT INTO t2 VALUES (1, 2);
+INSERT INTO t2 VALUES (2, 2);
+#
+INSERT INTO t3 VALUES (1, 2);
+INSERT INTO t3 VALUES (2, 2);
+#
+UPDATE t1 SET a = 1;
+UPDATE t1 SET a = 1 ORDER BY a;
+#
+--error ER_DUP_ENTRY
+UPDATE t2 SET a = 1;
+--error ER_DUP_ENTRY
+UPDATE t2 SET a = 1 ORDER BY a;
+#
+--error ER_DUP_ENTRY
+UPDATE t3 SET a = 1;
+--error ER_DUP_ENTRY
+UPDATE t3 SET a = 1 ORDER BY a;
+#
+SELECT count(*) FROM t1;
+SELECT count(*) FROM t2;
+SELECT count(*) FROM t3;
+SELECT * FROM t1 ORDER by a;
+SELECT * FROM t2 ORDER by a;
+SELECT * FROM t3 ORDER by a;
+#
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+DROP TABLE IF EXISTS t3;
+--enable_warnings
+
+--echo End of 5.1 tests
+

mysql-test/suite/parts/r/rpl_partition.result

@@ -10,31 +10,9 @@ select @@global.binlog_format, @@session.binlog_format;
 @@global.binlog_format	ROW
 @@session.binlog_format	ROW
 DROP TABLE IF EXISTS t1, t2, t3;
-Warnings:
-Level	Note
-Code	1051
-Message	Unknown table 't1'
-Level	Note
-Code	1051
-Message	Unknown table 't2'
-Level	Note
-Code	1051
-Message	Unknown table 't3'
 DROP PROCEDURE IF EXISTS p1;
-Warnings:
-Level	Note
-Code	1305
-Message	PROCEDURE p1 does not exist
 DROP PROCEDURE IF EXISTS p2;
-Warnings:
-Level	Note
-Code	1305
-Message	PROCEDURE p2 does not exist
 DROP PROCEDURE IF EXISTS p3;
-Warnings:
-Level	Note
-Code	1305
-Message	PROCEDURE p3 does not exist
 CREATE TABLE t1(id MEDIUMINT NOT NULL AUTO_INCREMENT,
 dt TIMESTAMP, user CHAR(255), uuidf LONGBLOB,
 fkid MEDIUMINT, filler VARCHAR(255),

mysql-test/suite/parts/t/rpl_partition.test

@@ -9,12 +9,12 @@ SET GLOBAL binlog_format = 'ROW';
 SET SESSION binlog_format = 'ROW';
 select @@global.binlog_format, @@session.binlog_format;
 
---disable-warnings
+--disable_warnings
 DROP TABLE IF EXISTS t1, t2, t3;
 DROP PROCEDURE IF EXISTS p1;
 DROP PROCEDURE IF EXISTS p2;
 DROP PROCEDURE IF EXISTS p3;
---enable-warnings
+--enable_warnings
 
 eval CREATE TABLE t1(id MEDIUMINT NOT NULL AUTO_INCREMENT,
                            dt TIMESTAMP, user CHAR(255), uuidf LONGBLOB,

mysql-test/t/ctype_uca.test

@@ -530,7 +530,7 @@ create table t1 (
   a varchar(255),
   key a(a)
 ) character set utf8 collate utf8_czech_ci;
--- In Czech 'ch' is a single letter between 'h' and 'i'
+# In Czech 'ch' is a single letter between 'h' and 'i'
 insert into t1 values
 ('b'),('c'),('d'),('e'),('f'),('g'),('h'),('ch'),('i'),('j');
 select * from t1 where a like 'c%';

mysql-test/t/fulltext.test

@@ -399,4 +399,14 @@ ALTER TABLE t1 DISABLE KEYS;
 SELECT * FROM t1 WHERE MATCH(a) AGAINST('test');
 DROP TABLE t1;
 
+#
+# BUG#29445 - match ... against () never returns
+#
+CREATE TABLE t1(a VARCHAR(20), FULLTEXT(a));
+INSERT INTO t1 VALUES('Offside'),('City Of God');
+SELECT a FROM t1 WHERE MATCH a AGAINST ('+city of*' IN BOOLEAN MODE);
+SELECT a FROM t1 WHERE MATCH a AGAINST ('+city (of*)' IN BOOLEAN MODE);
+SELECT a FROM t1 WHERE MATCH a AGAINST ('+city* of*' IN BOOLEAN MODE);
+DROP TABLE t1;
+
 # End of 4.1 tests

mysql-test/t/fulltext3.test

@@ -22,3 +22,13 @@ DROP TABLE t1;
 
 # End of 5.0 tests
 
+#
+# BUG#29464 - load data infile into table with big5 chinese fulltext index
+#             hangs 100% cpu
+#
+CREATE TABLE t1(a VARCHAR(2) CHARACTER SET big5 COLLATE big5_chinese_ci,
+FULLTEXT(a));
+INSERT INTO t1 VALUES(0xA3C2);
+DROP TABLE t1;
+
+# End of 5.1 tests

mysql-test/t/subselect.test

@@ -2970,7 +2970,7 @@ DROP TABLE t1,t2;
 CREATE TABLE t1 (a INT, b INT);
 INSERT INTO t1 VALUES (1, 2), (1,3), (1,4), (2,1), (2,2);
 
--- returns no rows, when it should
+# returns no rows, when it should
 SELECT a1.a, COUNT(*) FROM t1 a1 WHERE a1.a = 1
 AND EXISTS( SELECT a2.a FROM t1 a2 WHERE a2.a = a1.a)
 GROUP BY a1.a;

sql/sql_class.cc

@@ -411,7 +411,7 @@ THD::THD()
   current_linfo =  0;
   slave_thread = 0;
   bzero(&variables, sizeof(variables));
-  thread_id= variables.pseudo_thread_id= 0;
+  thread_id= 0;
   one_shot_set= 0;
   file_id = 0;
   query_id= 0;
@@ -571,6 +571,12 @@ void THD::init(void)
 					       variables.date_format);
   variables.datetime_format= date_time_format_copy((THD*) 0,
 						   variables.datetime_format);
+  /*
+    variables= global_system_variables above has reset
+    variables.pseudo_thread_id to 0. We need to correct it here to
+    avoid temporary tables replication failure.
+  */
+  variables.pseudo_thread_id= thread_id;
   pthread_mutex_unlock(&LOCK_global_system_variables);
   server_status= SERVER_STATUS_AUTOCOMMIT;
   if (variables.sql_mode & MODE_NO_BACKSLASH_ESCAPES)

sql/sql_update.cc

@@ -83,6 +83,75 @@ static bool check_fields(THD *thd, List<Item> &items)
 }
 
 
+/**
+  @brief Re-read record if more columns are needed for error message.
+
+  @detail If we got a duplicate key error, we want to write an error
+    message containing the value of the duplicate key. If we do not have
+    all fields of the key value in record[0], we need to re-read the
+    record with a proper read_set.
+
+  @param[in] error   error number
+  @param[in] table   table
+*/
+
+static void prepare_record_for_error_message(int error, TABLE *table)
+{
+  Field **field_p;
+  Field *field;
+  uint keynr;
+  MY_BITMAP unique_map; /* Fields in offended unique. */
+  my_bitmap_map unique_map_buf[bitmap_buffer_size(MAX_FIELDS)];
+  DBUG_ENTER("prepare_record_for_error_message");
+
+  /*
+    Only duplicate key errors print the key value.
+    If storage engine does always read all columns, we have the value alraedy.
+  */
+  if ((error != HA_ERR_FOUND_DUPP_KEY) ||
+      !(table->file->ha_table_flags() & HA_PARTIAL_COLUMN_READ))
+    DBUG_VOID_RETURN;
+
+  /*
+    Get the number of the offended index.
+    We will see MAX_KEY if the engine cannot determine the affected index.
+  */
+  if ((keynr= table->file->get_dup_key(error)) >= MAX_KEY)
+    DBUG_VOID_RETURN;
+
+  /* Create unique_map with all fields used by that index. */
+  bitmap_init(&unique_map, unique_map_buf, table->s->fields, FALSE);
+  table->mark_columns_used_by_index_no_reset(keynr, &unique_map);
+
+  /* Subtract read_set and write_set. */
+  bitmap_subtract(&unique_map, table->read_set);
+  bitmap_subtract(&unique_map, table->write_set);
+
+  /*
+    If the unique index uses columns that are neither in read_set
+    nor in write_set, we must re-read the record.
+    Otherwise no need to do anything.
+  */
+  if (bitmap_is_clear_all(&unique_map))
+    DBUG_VOID_RETURN;
+
+  /* Get identifier of last read record into table->file->ref. */
+  table->file->position(table->record[0]);
+  /* Add all fields used by unique index to read_set. */
+  bitmap_union(table->read_set, &unique_map);
+  /* Tell the engine about the new set. */
+  table->file->column_bitmaps_signal();
+  /* Read record that is identified by table->file->ref. */
+  (void) table->file->rnd_pos(table->record[1], table->file->ref);
+  /* Copy the newly read columns into the new record. */
+  for (field_p= table->field; (field= *field_p); field_p++)
+    if (bitmap_is_set(&unique_map, field->field_index))
+      field->copy_from_tmp(table->s->rec_buff_length);
+
+  DBUG_VOID_RETURN;
+}
+
+
 /*
   Process usual UPDATE
 
@@ -470,6 +539,13 @@ int mysql_update(THD *thd,
   else
     will_batch= !table->file->start_bulk_update();
 
+  /*
+    Assure that we can use position()
+    if we need to create an error message.
+  */
+  if (table->file->ha_table_flags() & HA_PARTIAL_COLUMN_READ)
+    table->prepare_for_position();
+
   /*
     We can use compare_record() to optimize away updates if
     the table handler is returning all columns OR if
@@ -573,6 +649,8 @@ int mysql_update(THD *thd,
           */
           if (table->file->is_fatal_error(error, HA_CHECK_DUP_KEY))
             thd->fatal_error(); /* Other handler errors are fatal */
+
+          prepare_record_for_error_message(error, table);
 	  table->file->print_error(error,MYF(0));
 	  error= 1;
 	  break;
@@ -596,13 +674,16 @@ int mysql_update(THD *thd,
         {
  	  if (error)
           {
+            /* purecov: begin inspected */
             /*
               The handler should not report error of duplicate keys if they
               are ignored. This is a requirement on batching handlers.
             */
+            prepare_record_for_error_message(error, table);
             table->file->print_error(error,MYF(0));
             error= 1;
             break;
+            /* purecov: end */
           }
           /*
             Either an error was found and we are ignoring errors or there
@@ -668,9 +749,12 @@ int mysql_update(THD *thd,
       in the batched update.
     */
   {
+    /* purecov: begin inspected */
     thd->fatal_error();
+    prepare_record_for_error_message(loc_error, table);
     table->file->print_error(loc_error,MYF(0));
     error= 1;
+    /* purecov: end */
   }
   else
     updated-= dup_key_found;
@@ -1540,6 +1624,8 @@ bool multi_update::send_data(List<Item> &not_used_values)
             */
             if (table->file->is_fatal_error(error, HA_CHECK_DUP_KEY))
               thd->fatal_error(); /* Other handler errors are fatal */
+
+            prepare_record_for_error_message(error, table);
             table->file->print_error(error,MYF(0));
             DBUG_RETURN(1);
           }
@@ -1676,7 +1762,7 @@ int multi_update::do_updates(bool from_send_error)
   ha_rows org_updated;
   TABLE *table, *tmp_table;
   List_iterator_fast<TABLE> check_opt_it(unupdated_check_opt_tables);
-  DBUG_ENTER("do_updates");
+  DBUG_ENTER("multi_update::do_updates");
 
   do_update= 0;					// Don't retry this function
   if (!found)
@@ -1819,6 +1905,7 @@ err:
   if (!from_send_error)
   {
     thd->fatal_error();
+    prepare_record_for_error_message(local_error, table);
     table->file->print_error(local_error,MYF(0));
   }
 
@@ -1849,6 +1936,7 @@ bool multi_update::send_eof()
 {
   char buff[STRING_BUFFER_USUAL_SIZE];
   ulonglong id;
+  DBUG_ENTER("multi_update::send_eof");
   thd->proc_info="updating reference tables";
 
   /* Does updates for the last n - 1 tables, returns 0 if ok */
@@ -1904,7 +1992,7 @@ bool multi_update::send_eof()
     /* Safety: If we haven't got an error before (can happen in do_updates) */
     my_message(ER_UNKNOWN_ERROR, "An error occured in multi-table update",
 	       MYF(0));
-    return TRUE;
+    DBUG_RETURN(TRUE);
   }
 
   id= thd->arg_of_last_insert_id_function ?
@@ -1914,5 +2002,5 @@ bool multi_update::send_eof()
   thd->row_count_func=
     (thd->client_capabilities & CLIENT_FOUND_ROWS) ? found : updated;
   ::send_ok(thd, (ulong) thd->row_count_func, id, buff);
-  return FALSE;
+  DBUG_RETURN(FALSE);
 }

storage/myisam/ft_boolean_search.c

@@ -23,8 +23,14 @@
   inside plus subtree. max_docid could be used by any word in plus
   subtree, but it could be updated by plus-word only.
 
+  Fulltext "smarter index merge" optimization assumes that rows
+  it gets are ordered by doc_id. That is not the case when we
+  search for a word with truncation operator. It may return
+  rows in random order. Thus we may not use "smarter index merge"
+  optimization with "trunc-words".
+
   The idea is: there is no need to search for docid smaller than
-  biggest docid inside current plus subtree.
+  biggest docid inside current plus subtree or any upper plus subtree.
 
   Examples:
   +word1 word2
@@ -36,6 +42,13 @@
   +(word1 -word2) +(+word3 word4)
     share same max_docid
     max_docid updated by word3
+   +word1 word2 (+word3 word4 (+word5 word6))
+    three subexpressions (including the top-level one),
+    every one has its own max_docid, updated by its plus word.
+    but for the search word6 uses
+    max(word1.max_docid, word3.max_docid, word5.max_docid),
+    while word4 uses, accordingly,
+    max(word1.max_docid, word3.max_docid).
 */
 
 #define FT_CORE
@@ -104,7 +117,7 @@ typedef struct st_ftb_word
 /* ^^^^^^^^^^^^^^^^^^ FTB_{EXPR,WORD} common section */
   my_off_t   docid[2];             /* for index search and for scan */
   my_off_t   key_root;
-  my_off_t  *max_docid;
+  FTB_EXPR  *max_docid_expr;
   MI_KEYDEF *keyinfo;
   struct st_ftb_word *prev;
   float      weight;
@@ -208,7 +221,7 @@ static int ftb_query_add_word(MYSQL_FTPARSER_PARAM *param,
       for (tmp_expr= ftb_param->ftbe; tmp_expr->up; tmp_expr= tmp_expr->up)
         if (! (tmp_expr->flags & FTB_FLAG_YES))
           break;
-      ftbw->max_docid= &tmp_expr->max_docid;
+      ftbw->max_docid_expr= tmp_expr;
       /* fall through */
     case FT_TOKEN_STOPWORD:
       if (! ftb_param->up_quot) break;
@@ -347,11 +360,17 @@ static int _ft2_search(FTB *ftb, FTB_WORD *ftbw, my_bool init_search)
   else
   {
     uint sflag= SEARCH_BIGGER;
-    if (ftbw->docid[0] < *ftbw->max_docid)
+    my_off_t max_docid=0;
+    FTB_EXPR *tmp;
+
+    for (tmp= ftbw->max_docid_expr; tmp; tmp= tmp->up)
+      set_if_bigger(max_docid, tmp->max_docid);
+
+    if (ftbw->docid[0] < max_docid)
     {
       sflag|= SEARCH_SAME;
       _mi_dpointer(info, (uchar *)(ftbw->word + ftbw->len + HA_FT_WLEN),
-                   *ftbw->max_docid);
+                   max_docid);
     }
     r=_mi_search(info, ftbw->keyinfo, (uchar*) lastkey_buf,
                    USE_WHOLE_KEY, sflag, ftbw->key_root);
@@ -430,8 +449,8 @@ static int _ft2_search(FTB *ftb, FTB_WORD *ftbw, my_bool init_search)
     memcpy(lastkey_buf+off, info->lastkey, info->lastkey_length);
   }
   ftbw->docid[0]=info->lastpos;
-  if (ftbw->flags & FTB_FLAG_YES)
-    *ftbw->max_docid= info->lastpos;
+  if (ftbw->flags & FTB_FLAG_YES && !(ftbw->flags & FTB_FLAG_TRUNC))
+    ftbw->max_docid_expr->max_docid= info->lastpos;
   return 0;
 }
 
@@ -474,7 +493,8 @@ static void _ftb_init_index_search(FT_INFO *ftb)
            ftbe->up->flags|= FTB_FLAG_TRUNC, ftbe=ftbe->up)
       {
         if (ftbe->flags & FTB_FLAG_NO ||                     /* 2 */
-             ftbe->up->ythresh - ftbe->up->yweaks >1)        /* 1 */
+            ftbe->up->ythresh - ftbe->up->yweaks >
+            (uint) test(ftbe->flags & FTB_FLAG_YES))         /* 1 */
         {
           FTB_EXPR *top_ftbe=ftbe->up;
           ftbw->docid[0]=HA_OFFSET_ERROR;

storage/myisam/ft_parser.c

@@ -111,7 +111,8 @@ uchar ft_get_word(CHARSET_INFO *cs, uchar **start, uchar *end,
 {
   uchar *doc=*start;
   int ctype;
-  uint mwc, length, mbl;
+  uint mwc, length;
+  int mbl;
 
   param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
   param->weight_adjust= param->wasign= 0;
@@ -119,7 +120,7 @@ uchar ft_get_word(CHARSET_INFO *cs, uchar **start, uchar *end,
 
   while (doc<end)
   {
-    for (; doc < end; doc+= (mbl > 0 ? mbl : 1))
+    for (; doc < end; doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
     {
       mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
       if (true_word_char(ctype, *doc))
@@ -157,7 +158,8 @@ uchar ft_get_word(CHARSET_INFO *cs, uchar **start, uchar *end,
     }
 
     mwc=length=0;
-    for (word->pos= doc; doc < end; length++, doc+= (mbl > 0 ? mbl : 1))
+    for (word->pos= doc; doc < end; length++,
+         doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
     {
       mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
       if (true_word_char(ctype, *doc))
@@ -200,13 +202,14 @@ uchar ft_simple_get_word(CHARSET_INFO *cs, uchar **start, const uchar *end,
                          FT_WORD *word, my_bool skip_stopwords)
 {
   uchar *doc= *start;
-  uint mwc, length, mbl;
+  uint mwc, length;
+  int mbl;
   int ctype;
   DBUG_ENTER("ft_simple_get_word");
 
   do
   {
-    for (;; doc+= (mbl > 0 ? mbl : 1))
+    for (;; doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
     {
       if (doc >= end)
         DBUG_RETURN(0);
@@ -216,7 +219,8 @@ uchar ft_simple_get_word(CHARSET_INFO *cs, uchar **start, const uchar *end,
     }
 
     mwc= length= 0;
-    for (word->pos= doc; doc < end; length++, doc+= (mbl > 0 ? mbl : 1))
+    for (word->pos= doc; doc < end; length++,
+         doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
     {
       mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
       if (true_word_char(ctype, *doc))