Now ftparser does not need to bother about memory management -

it can tell MySQL to make a copy of everything (bug#17123)


include/mysql/plugin.h:
  Now ftparser does not need to bother about memory management -
  it can tell MySQL to make a copy of everything (bug#17123)
  MYSQL_FTFLAGS_NEED_COPY flag
storage/myisam/ft_boolean_search.c:
  param->flags
storage/myisam/ft_nlq_search.c:
  param->flags. ft_parse takes a mem_root as an argument
storage/myisam/ft_parser.c:
  ftparser takes a memroot as an argument. words are copied there, if necessary.
  memroot is reset for every parsing and free'd at the end of the statement.
storage/myisam/ft_update.c:
  ftparser takes a memroot as an argument. words are copied there, if necessary.
  memroot is reset for every parsing and free'd at the end of the statement.
storage/myisam/ftdefs.h:
  ftparser takes a memroot as an argument. words are copied there, if necessary.
  memroot is reset for every parsing and free'd at the end of the statement.
storage/myisam/mi_check.c:
  ftparser takes a memroot as an argument. words are copied there, if necessary
storage/myisam/myisamdef.h:
  memroot for ftparser in MI_INFO and MI_SORT_PARAM
storage/myisam/sort.c:
  free ftparser memroot

include/mysql/plugin.h

@@ -201,6 +201,17 @@ typedef struct st_mysql_ftparser_boolean_info
   char *quot;
 } MYSQL_FTPARSER_BOOLEAN_INFO;
 
+/*
+  The following flag means that buffer with a string (document, word)
+  may be overwritten by the caller before the end of the parsing (that is
+  before st_mysql_ftparser::deinit() call). If one needs the string
+  to survive between two successive calls of the parsing function, she
+  needs to save a copy of it. The flag may be set by MySQL before calling
+  st_mysql_ftparser::parse(), or it may be set by a plugin before calling
+  st_mysql_ftparser_param::mysql_parse() or
+  st_mysql_ftparser_param::mysql_add_word().
+*/
+#define MYSQL_FTFLAGS_NEED_COPY 1
 
 /*
   An argument of the full-text parser plugin. This structure is
@@ -234,8 +245,10 @@ typedef struct st_mysql_ftparser_boolean_info
 
   length: Length of the document or query string, in bytes.
 
+  flags: See MYSQL_FTFLAGS_* constants above.
+
   mode: The parsing mode.  With boolean operators, with stopwords, or
-  nothing.  See MYSQL_FTPARSER_* constants above.
+  nothing.  See  enum_ftparser_mode above.
 */
 
 typedef struct st_mysql_ftparser_param
@@ -250,6 +263,7 @@ typedef struct st_mysql_ftparser_param
   struct charset_info_st *cs;
   char *doc;
   int length;
+  int flags;
   enum enum_ftparser_mode mode;
 } MYSQL_FTPARSER_PARAM;
 

storage/myisam/ft_boolean_search.c

@@ -311,6 +311,7 @@ static void _ftb_parse_query(FTB *ftb, byte *query, uint len,
   param->cs= ftb->charset;
   param->doc= query;
   param->length= len;
+  param->flags= 0;
   param->mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO;
   parser->parse(param);
   DBUG_VOID_RETURN;
@@ -665,6 +666,7 @@ static int _ftb_check_phrase(FTB *ftb, const byte *document, uint len,
   param->cs= ftb->charset;
   param->doc= (byte *)document;
   param->length= len;
+  param->flags= 0;
   param->mode= MYSQL_FTPARSER_WITH_STOPWORDS;
   parser->parse(param);
   DBUG_RETURN(ftb_param.match ? 1 : 0);
@@ -916,6 +918,7 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
   param->mysql_parse= ftb_find_relevance_parse;
   param->mysql_add_word= ftb_find_relevance_add_word;
   param->mysql_ftparam= (void *)&ftb_param;
+  param->flags= 0;
   param->cs= ftb->charset;
   param->mode= MYSQL_FTPARSER_SIMPLE_MODE;
   while (_mi_ft_segiterator(&ftsi))

storage/myisam/ft_nlq_search.c

@@ -235,7 +235,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
             NULL, NULL);
 
   ft_parse_init(&wtree, aio.charset);
-  if (ft_parse(&wtree, query, query_len, 0, parser, ftparser_param))
+  ftparser_param->flags= 0;
+  if (ft_parse(&wtree, query, query_len, parser, ftparser_param,
+               &wtree.mem_root))
     goto err;
 
   if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
@@ -255,7 +257,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
       if (!(*info->read_record)(info,docid,record))
       {
         info->update|= HA_STATE_AKTIV;
-        _mi_ft_parse(&wtree, info, keynr, record, 1, ftparser_param);
+        ftparser_param->flags= MYSQL_FTFLAGS_NEED_COPY;
+        _mi_ft_parse(&wtree, info, keynr, record, ftparser_param,
+                     &wtree.mem_root);
       }
     }
     delete_queue(&best);

storage/myisam/ft_parser.c

@@ -24,14 +24,12 @@ typedef struct st_ft_docstat {
   double sum;
 } FT_DOCSTAT;
 
-
 typedef struct st_my_ft_parser_param
 {
   TREE     *wtree;
-  my_bool with_alloc;
+  MEM_ROOT *mem_root;
 } MY_FT_PARSER_PARAM;
 
-
 static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
 {
   return mi_compare_text(cs, (uchar*) w1->pos, w1->len,
@@ -48,14 +46,14 @@ static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat)
 
 /* transforms tree of words into the array, applying normalization */
 
-FT_WORD * ft_linearize(TREE *wtree)
+FT_WORD * ft_linearize(TREE *wtree, MEM_ROOT *mem_root)
 {
   FT_WORD *wlist,*p;
   FT_DOCSTAT docstat;
   DBUG_ENTER("ft_linearize");
 
-  if ((wlist=(FT_WORD *) my_malloc(sizeof(FT_WORD)*
-				   (1+wtree->elements_in_tree),MYF(0))))
+  if ((wlist=(FT_WORD *) alloc_root(mem_root, sizeof(FT_WORD)*
+                                    (1+wtree->elements_in_tree))))
   {
     docstat.list=wlist;
     docstat.uniq=wtree->elements_in_tree;
@@ -249,12 +247,11 @@ static int ft_add_word(MYSQL_FTPARSER_PARAM *param,
   MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;
   DBUG_ENTER("ft_add_word");
   wtree= ft_param->wtree;
-  if (ft_param->with_alloc)
+  if (param->flags & MYSQL_FTFLAGS_NEED_COPY)
   {
     byte *ptr;
-    /* allocating the data in the tree - to avoid mallocs and frees */
     DBUG_ASSERT(wtree->with_delete == 0);
-    ptr= (byte *)alloc_root(&wtree->mem_root, word_len);
+    ptr= (byte *)alloc_root(ft_param->mem_root, word_len);
     memcpy(ptr, word, word_len);
     w.pos= ptr;
   }
@@ -286,16 +283,16 @@ static int ft_parse_internal(MYSQL_FTPARSER_PARAM *param,
 }
 
 
-int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc,
+int ft_parse(TREE *wtree, byte *doc, int doclen,
                     struct st_mysql_ftparser *parser,
-                    MYSQL_FTPARSER_PARAM *param)
+                    MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)
 {
   MY_FT_PARSER_PARAM my_param;
   DBUG_ENTER("ft_parse");
   DBUG_ASSERT(parser);
 
   my_param.wtree= wtree;
-  my_param.with_alloc= with_alloc;
+  my_param.mem_root= mem_root;
 
   param->mysql_parse= ft_parse_internal;
   param->mysql_add_word= ft_add_word;
@@ -357,6 +354,7 @@ MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info,
     info->ftparser_param= (MYSQL_FTPARSER_PARAM *)
       my_malloc(MAX_PARAM_NR * sizeof(MYSQL_FTPARSER_PARAM) *
                 info->s->ftparsers, MYF(MY_WME|MY_ZEROFILL));
+    init_alloc_root(&info->ft_memroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
     if (! info->ftparser_param)
       return 0;
   }
@@ -388,6 +386,7 @@ MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info,
 void ftparser_call_deinitializer(MI_INFO *info)
 {
   uint i, j, keys= info->s->state.header.keys;
+  free_root(&info->ft_memroot, MYF(0));
   if (! info->ftparser_param)
     return;
   for (i= 0; i < keys; i++)

storage/myisam/ft_update.c

@@ -95,9 +95,8 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi)
 
 /* parses a document i.e. calls ft_parse for every keyseg */
 
-uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
-                  const byte *record, my_bool with_alloc,
-                  MYSQL_FTPARSER_PARAM *param)
+uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, const byte *record,
+                  MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)
 {
   FT_SEG_ITERATOR ftsi;
   struct st_mysql_ftparser *parser;
@@ -110,14 +109,14 @@ uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
   while (_mi_ft_segiterator(&ftsi))
   {
     if (ftsi.pos)
-      if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser,
-                   param))
+      if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, parser, param, mem_root))
         DBUG_RETURN(1);
   }
   DBUG_RETURN(0);
 }
 
-FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record)
+FT_WORD *_mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record,
+                             MEM_ROOT *mem_root)
 {
   TREE ptree;
   MYSQL_FTPARSER_PARAM *param;
@@ -125,10 +124,11 @@ FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record)
   if (! (param= ftparser_call_initializer(info, keynr, 0)))
     DBUG_RETURN(NULL);
   bzero((char*) &ptree, sizeof(ptree));
-  if (_mi_ft_parse(&ptree, info, keynr, record, 0, param))
+  param->flags= 0;
+  if (_mi_ft_parse(&ptree, info, keynr, record, param, mem_root))
     DBUG_RETURN(NULL);
 
-  DBUG_RETURN(ft_linearize(&ptree));
+  DBUG_RETURN(ft_linearize(&ptree, mem_root));
 }
 
 static int _mi_ft_store(MI_INFO *info, uint keynr, byte *keybuf,
@@ -206,10 +206,11 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf,
   int cmp, cmp2;
   DBUG_ENTER("_mi_ft_update");
 
-  if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec)))
-    goto err0;
-  if (!(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec)))
-    goto err1;
+  if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec,
+                                            &info->ft_memroot)) ||
+      !(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec,
+                                            &info->ft_memroot)))
+    goto err;
 
   error=0;
   while(old_word->pos && new_word->pos)
@@ -222,13 +223,13 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf,
     {
       key_length=_ft_make_key(info,keynr,keybuf,old_word,pos);
       if ((error=_mi_ck_delete(info,keynr,(uchar*) keybuf,key_length)))
-        goto err2;
+        goto err;
     }
     if (cmp > 0 || cmp2)
     {
       key_length=_ft_make_key(info,keynr,keybuf,new_word,pos);
       if ((error=_mi_ck_write(info,keynr,(uchar*) keybuf,key_length)))
-        goto err2;
+        goto err;
     }
     if (cmp<=0) old_word++;
     if (cmp>=0) new_word++;
@@ -238,11 +239,8 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf,
  else if (new_word->pos)
    error=_mi_ft_store(info,keynr,keybuf,new_word,pos);
 
-err2:
-    my_free((char*) newlist,MYF(0));
-err1:
-    my_free((char*) oldlist,MYF(0));
-err0:
+err:
+  free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));
   DBUG_RETURN(error);
 }
 
@@ -255,12 +253,13 @@ int _mi_ft_add(MI_INFO *info, uint keynr, byte *keybuf, const byte *record,
   int error= -1;
   FT_WORD *wlist;
   DBUG_ENTER("_mi_ft_add");
+  DBUG_PRINT("enter",("keynr: %d",keynr));
 
-  if ((wlist=_mi_ft_parserecord(info, keynr, record)))
-  {
+  if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot)))
     error=_mi_ft_store(info,keynr,keybuf,wlist,pos);
-    my_free((char*) wlist,MYF(0));
-  }
+
+  free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));
+  DBUG_PRINT("exit",("Return: %d",error));
   DBUG_RETURN(error);
 }
 
@@ -275,11 +274,10 @@ int _mi_ft_del(MI_INFO *info, uint keynr, byte *keybuf, const byte *record,
   DBUG_ENTER("_mi_ft_del");
   DBUG_PRINT("enter",("keynr: %d",keynr));
 
-  if ((wlist=_mi_ft_parserecord(info, keynr, record)))
-  {
+  if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot)))
     error=_mi_ft_erase(info,keynr,keybuf,wlist,pos);
-    my_free((char*) wlist,MYF(0));
-  }
+
+  free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));
   DBUG_PRINT("exit",("Return: %d",error));
   DBUG_RETURN(error);
 }

storage/myisam/ftdefs.h

@@ -30,6 +30,8 @@
 
 #define FT_MAX_WORD_LEN_FOR_SORT 31
 
+#define FTPARSER_MEMROOT_ALLOC_SIZE 65536
+
 #define COMPILE_STOPWORDS_IN
 
 /* Interested readers may consult SMART
@@ -119,12 +121,12 @@ void _mi_ft_segiterator_dummy_init(const byte *, uint, FT_SEG_ITERATOR *);
 uint _mi_ft_segiterator(FT_SEG_ITERATOR *);
 
 void ft_parse_init(TREE *, CHARSET_INFO *);
-int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser,
-             MYSQL_FTPARSER_PARAM *param);
-FT_WORD * ft_linearize(TREE *);
-FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *);
-uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool,
-                  MYSQL_FTPARSER_PARAM *param);
+int ft_parse(TREE *, byte *, int, struct st_mysql_ftparser *parser,
+             MYSQL_FTPARSER_PARAM *, MEM_ROOT *);
+FT_WORD * ft_linearize(TREE *, MEM_ROOT *);
+FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *, MEM_ROOT *);
+uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *,
+                  MYSQL_FTPARSER_PARAM *, MEM_ROOT *);
 
 FT_INFO *ft_init_nlq_search(MI_INFO *, uint, byte *, uint, uint, byte *);
 FT_INFO *ft_init_boolean_search(MI_INFO *, uint, byte *, uint, CHARSET_INFO *);

storage/myisam/mi_check.c

@@ -2117,6 +2117,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
     my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));
 
   sort_param.wordlist=NULL;
+  init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
 
   if (share->data_file_type == DYNAMIC_RECORD)
     length=max(share->base.min_pack_length+1,share->base.min_block_length);
@@ -2200,6 +2201,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
       goto err;
     }
     param->calc_checksum=0;			/* No need to calc glob_crc */
+    free_root(&sort_param.wordroot, MYF(0));
 
     /* Set for next loop */
     sort_info.max_records= (ha_rows) info->state->records;
@@ -2589,6 +2591,7 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
       uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                     sort_param[i].keyinfo->seg->charset->mbmaxlen;
       sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
+      init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
     }
   }
   sort_info.total_keys=i;
@@ -2810,10 +2813,11 @@ static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
   {
     for (;;)
     {
-      my_free((char*) wptr, MYF(MY_ALLOW_ZERO_PTR));
+      free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
       if ((error=sort_get_next_record(sort_param)))
         DBUG_RETURN(error);
-      if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record)))
+      if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,
+                                    &sort_param->wordroot)))
         DBUG_RETURN(1);
       if (wptr->pos)
         break;
@@ -2837,7 +2841,7 @@ static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
 #endif
   if (!wptr->pos)
   {
-    my_free((char*) sort_param->wordlist, MYF(0));
+    free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
     sort_param->wordlist=0;
     error=sort_write_record(sort_param);
   }

storage/myisam/myisamdef.h

@@ -235,6 +235,7 @@ struct st_myisam_info {
   /* accumulate indexfile changes between write's */
   TREE	        *bulk_insert;
   DYNAMIC_ARRAY *ft1_to_ft2;            /* used only in ft1->ft2 conversion */
+  MEM_ROOT      ft_memroot;             /* used by the parser               */
   MYSQL_FTPARSER_PARAM *ftparser_param; /* share info between init/deinit   */
   char *filename;			/* parameter to open filename       */
   uchar *buff,				/* Temp area for key                */
@@ -325,6 +326,7 @@ typedef struct st_mi_sort_param
   uchar **sort_keys;
   byte *rec_buff;
   void *wordlist, *wordptr;
+  MEM_ROOT wordroot;
   char *record;
   MY_TMPDIR *tmpdir;
   int (*key_cmp)(struct st_mi_sort_param *, const void *, const void *);

storage/myisam/sort.c

@@ -447,6 +447,7 @@ pthread_handler_t thr_find_all_keys(void *arg)
   close_cached_file(&info->tempfile_for_exceptions);
 
 ok:
+  free_root(&info->wordroot, MYF(0));
   remove_io_thread(&info->read_cache);
   pthread_mutex_lock(&info->sort_info->mutex);
   info->sort_info->threads_running--;