Skip to content

M
MariaDB
Commit 32711e7b authored by serg@serg.mylan's avatar serg@serg.mylan
Browse files
Options

Now ftparser does not need to bother about memory management - 

it can tell MySQL to make a copy of everything (bug#17123)
parent 60e3b617
Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 60 deletions
include/mysql/plugin.h
View file @ 32711e7b
......@@ -201,6 +201,17 @@ typedef struct st_mysql_ftparser_boolean_info
char *quot;
} MYSQL_FTPARSER_BOOLEAN_INFO;
/*
The following flag means that buffer with a string (document, word)
may be overwritten by the caller before the end of the parsing (that is
before st_mysql_ftparser::deinit() call). If one needs the string
to survive between two successive calls of the parsing function, she
needs to save a copy of it. The flag may be set by MySQL before calling
st_mysql_ftparser::parse(), or it may be set by a plugin before calling
st_mysql_ftparser_param::mysql_parse() or
st_mysql_ftparser_param::mysql_add_word().
*/
#define MYSQL_FTFLAGS_NEED_COPY 1
/*
An argument of the full-text parser plugin. This structure is
......@@ -234,8 +245,10 @@ typedef struct st_mysql_ftparser_boolean_info
length: Length of the document or query string, in bytes.
flags: See MYSQL_FTFLAGS_* constants above.
mode: The parsing mode. With boolean operators, with stopwords, or
nothing. See MYSQL_FTPARSER_* constants above.
nothing. See enum_ftparser_mode above.
*/
typedef struct st_mysql_ftparser_param
......@@ -250,6 +263,7 @@ typedef struct st_mysql_ftparser_param
struct charset_info_st *cs;
char *doc;
int length;
int flags;
enum enum_ftparser_mode mode;
} MYSQL_FTPARSER_PARAM;
......
storage/myisam/ft_boolean_search.c
View file @ 32711e7b
......@@ -311,6 +311,7 @@ static void _ftb_parse_query(FTB *ftb, byte *query, uint len,
param->cs= ftb->charset;
param->doc= query;
param->length= len;
param->flags= 0;
param->mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO;
parser->parse(param);
DBUG_VOID_RETURN;
......@@ -665,6 +666,7 @@ static int _ftb_check_phrase(FTB *ftb, const byte *document, uint len,
param->cs= ftb->charset;
param->doc= (byte *)document;
param->length= len;
param->flags= 0;
param->mode= MYSQL_FTPARSER_WITH_STOPWORDS;
parser->parse(param);
DBUG_RETURN(ftb_param.match ? 1 : 0);
......@@ -916,6 +918,7 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
param->mysql_parse= ftb_find_relevance_parse;
param->mysql_add_word= ftb_find_relevance_add_word;
param->mysql_ftparam= (void *)&ftb_param;
param->flags= 0;
param->cs= ftb->charset;
param->mode= MYSQL_FTPARSER_SIMPLE_MODE;
while (_mi_ft_segiterator(&ftsi))
......
storage/myisam/ft_nlq_search.c
View file @ 32711e7b
......@@ -235,7 +235,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
NULL, NULL);
ft_parse_init(&wtree, aio.charset);
if (ft_parse(&wtree, query, query_len, 0, parser, ftparser_param))
ftparser_param->flags= 0;
if (ft_parse(&wtree, query, query_len, parser, ftparser_param,
&wtree.mem_root))
goto err;
if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
......@@ -255,7 +257,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
if (!(*info->read_record)(info,docid,record))
{
info->update|= HA_STATE_AKTIV;
_mi_ft_parse(&wtree, info, keynr, record, 1, ftparser_param);
ftparser_param->flags= MYSQL_FTFLAGS_NEED_COPY;
_mi_ft_parse(&wtree, info, keynr, record, ftparser_param,
&wtree.mem_root);
}
}
delete_queue(&best);
......
storage/myisam/ft_parser.c
View file @ 32711e7b
......@@ -24,14 +24,12 @@ typedef struct st_ft_docstat {
double sum;
} FT_DOCSTAT;
typedef struct st_my_ft_parser_param
{
TREE *wtree;
my_bool with_alloc;
TREE *wtree;
MEM_ROOT *mem_root;
} MY_FT_PARSER_PARAM;
static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
{
return mi_compare_text(cs, (uchar*) w1->pos, w1->len,
......@@ -48,14 +46,14 @@ static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat)
/* transforms tree of words into the array, applying normalization */
FT_WORD * ft_linearize(TREE *wtree)
FT_WORD * ft_linearize(TREE *wtree, MEM_ROOT *mem_root)
{
FT_WORD *wlist,*p;
FT_DOCSTAT docstat;
DBUG_ENTER("ft_linearize");
if ((wlist=(FT_WORD *) my_malloc(sizeof(FT_WORD)*
(1+wtree->elements_in_tree),MYF(0))))
if ((wlist=(FT_WORD *) alloc_root(mem_root, sizeof(FT_WORD)*
(1+wtree->elements_in_tree))))
{
docstat.list=wlist;
docstat.uniq=wtree->elements_in_tree;
......@@ -249,12 +247,11 @@ static int ft_add_word(MYSQL_FTPARSER_PARAM *param,
MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;
DBUG_ENTER("ft_add_word");
wtree= ft_param->wtree;
if (ft_param->with_alloc)
if (param->flags & MYSQL_FTFLAGS_NEED_COPY)
{
byte *ptr;
/* allocating the data in the tree - to avoid mallocs and frees */
DBUG_ASSERT(wtree->with_delete == 0);
ptr= (byte *)alloc_root(&wtree->mem_root, word_len);
ptr= (byte *)alloc_root(ft_param->mem_root, word_len);
memcpy(ptr, word, word_len);
w.pos= ptr;
}
......@@ -286,16 +283,16 @@ static int ft_parse_internal(MYSQL_FTPARSER_PARAM *param,
}
int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc,
int ft_parse(TREE *wtree, byte *doc, int doclen,
struct st_mysql_ftparser *parser,
MYSQL_FTPARSER_PARAM *param)
MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)
{
MY_FT_PARSER_PARAM my_param;
DBUG_ENTER("ft_parse");
DBUG_ASSERT(parser);
my_param.wtree= wtree;
my_param.with_alloc= with_alloc;
my_param.mem_root= mem_root;
param->mysql_parse= ft_parse_internal;
param->mysql_add_word= ft_add_word;
......@@ -357,6 +354,7 @@ MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info,
info->ftparser_param= (MYSQL_FTPARSER_PARAM *)
my_malloc(MAX_PARAM_NR * sizeof(MYSQL_FTPARSER_PARAM) *
info->s->ftparsers, MYF(MY_WME|MY_ZEROFILL));
init_alloc_root(&info->ft_memroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
if (! info->ftparser_param)
return 0;
}
......@@ -388,6 +386,7 @@ MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info,
void ftparser_call_deinitializer(MI_INFO *info)
{
uint i, j, keys= info->s->state.header.keys;
free_root(&info->ft_memroot, MYF(0));
if (! info->ftparser_param)
return;
for (i= 0; i < keys; i++)
......
storage/myisam/ft_update.c
View file @ 32711e7b
......@@ -77,7 +77,7 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi)
{
uint pack_length= (ftsi->seg->bit_start);
ftsi->len= (pack_length == 1 ? (uint) *(uchar*) ftsi->pos :
uint2korr(ftsi->pos));
uint2korr(ftsi->pos));
ftsi->pos+= pack_length; /* Skip VARCHAR length */
DBUG_RETURN(1);
}
......@@ -95,9 +95,8 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi)
/* parses a document i.e. calls ft_parse for every keyseg */
uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
const byte *record, my_bool with_alloc,
MYSQL_FTPARSER_PARAM *param)
uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, const byte *record,
MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)
{
FT_SEG_ITERATOR ftsi;
struct st_mysql_ftparser *parser;
......@@ -110,14 +109,14 @@ uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
while (_mi_ft_segiterator(&ftsi))
{
if (ftsi.pos)
if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser,
param))
if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, parser, param, mem_root))
DBUG_RETURN(1);
}
DBUG_RETURN(0);
}
FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record)
FT_WORD *_mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record,
MEM_ROOT *mem_root)
{
TREE ptree;
MYSQL_FTPARSER_PARAM *param;
......@@ -125,10 +124,11 @@ FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record)
if (! (param= ftparser_call_initializer(info, keynr, 0)))
DBUG_RETURN(NULL);
bzero((char*) &ptree, sizeof(ptree));
if (_mi_ft_parse(&ptree, info, keynr, record, 0, param))
param->flags= 0;
if (_mi_ft_parse(&ptree, info, keynr, record, param, mem_root))
DBUG_RETURN(NULL);
DBUG_RETURN(ft_linearize(&ptree));
DBUG_RETURN(ft_linearize(&ptree, mem_root));
}
static int _mi_ft_store(MI_INFO *info, uint keynr, byte *keybuf,
......@@ -206,10 +206,11 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf,
int cmp, cmp2;
DBUG_ENTER("_mi_ft_update");
if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec)))
goto err0;
if (!(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec)))
goto err1;
if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec,
&info->ft_memroot)) ||
!(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec,
&info->ft_memroot)))
goto err;
error=0;
while(old_word->pos && new_word->pos)
......@@ -222,13 +223,13 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf,
{
key_length=_ft_make_key(info,keynr,keybuf,old_word,pos);
if ((error=_mi_ck_delete(info,keynr,(uchar*) keybuf,key_length)))
goto err2;
goto err;
}
if (cmp > 0 || cmp2)
{
key_length=_ft_make_key(info,keynr,keybuf,new_word,pos);
if ((error=_mi_ck_write(info,keynr,(uchar*) keybuf,key_length)))
goto err2;
goto err;
}
if (cmp<=0) old_word++;
if (cmp>=0) new_word++;
......@@ -238,11 +239,8 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf,
else if (new_word->pos)
error=_mi_ft_store(info,keynr,keybuf,new_word,pos);
err2:
my_free((char*) newlist,MYF(0));
err1:
my_free((char*) oldlist,MYF(0));
err0:
err:
free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));
DBUG_RETURN(error);
}
......@@ -255,12 +253,13 @@ int _mi_ft_add(MI_INFO *info, uint keynr, byte *keybuf, const byte *record,
int error= -1;
FT_WORD *wlist;
DBUG_ENTER("_mi_ft_add");
DBUG_PRINT("enter",("keynr: %d",keynr));
if ((wlist=_mi_ft_parserecord(info, keynr, record)))
{
if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot)))
error=_mi_ft_store(info,keynr,keybuf,wlist,pos);
my_free((char*) wlist,MYF(0));
}
free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));
DBUG_PRINT("exit",("Return: %d",error));
DBUG_RETURN(error);
}
......@@ -275,11 +274,10 @@ int _mi_ft_del(MI_INFO *info, uint keynr, byte *keybuf, const byte *record,
DBUG_ENTER("_mi_ft_del");
DBUG_PRINT("enter",("keynr: %d",keynr));
if ((wlist=_mi_ft_parserecord(info, keynr, record)))
{
if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot)))
error=_mi_ft_erase(info,keynr,keybuf,wlist,pos);
my_free((char*) wlist,MYF(0));
}
free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));
DBUG_PRINT("exit",("Return: %d",error));
DBUG_RETURN(error);
}
......
storage/myisam/ftdefs.h
View file @ 32711e7b
......@@ -30,6 +30,8 @@
#define FT_MAX_WORD_LEN_FOR_SORT 31
#define FTPARSER_MEMROOT_ALLOC_SIZE 65536
#define COMPILE_STOPWORDS_IN
/* Interested readers may consult SMART
......@@ -119,12 +121,12 @@ void _mi_ft_segiterator_dummy_init(const byte *, uint, FT_SEG_ITERATOR *);
uint _mi_ft_segiterator(FT_SEG_ITERATOR *);
void ft_parse_init(TREE *, CHARSET_INFO *);
int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser,
MYSQL_FTPARSER_PARAM *param);
FT_WORD * ft_linearize(TREE *);
FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *);
uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool,
MYSQL_FTPARSER_PARAM *param);
int ft_parse(TREE *, byte *, int, struct st_mysql_ftparser *parser,
MYSQL_FTPARSER_PARAM *, MEM_ROOT *);
FT_WORD * ft_linearize(TREE *, MEM_ROOT *);
FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *, MEM_ROOT *);
uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *,
MYSQL_FTPARSER_PARAM *, MEM_ROOT *);
FT_INFO *ft_init_nlq_search(MI_INFO *, uint, byte *, uint, uint, byte *);
FT_INFO *ft_init_boolean_search(MI_INFO *, uint, byte *, uint, CHARSET_INFO *);
......
storage/myisam/mi_check.c
View file @ 32711e7b
......@@ -2117,6 +2117,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));
sort_param.wordlist=NULL;
init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
if (share->data_file_type == DYNAMIC_RECORD)
length=max(share->base.min_pack_length+1,share->base.min_block_length);
......@@ -2200,6 +2201,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
goto err;
}
param->calc_checksum=0; /* No need to calc glob_crc */
free_root(&sort_param.wordroot, MYF(0));
/* Set for next loop */
sort_info.max_records= (ha_rows) info->state->records;
......@@ -2589,6 +2591,7 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
sort_param[i].keyinfo->seg->charset->mbmaxlen;
sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
}
}
sort_info.total_keys=i;
......@@ -2810,10 +2813,11 @@ static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
{
for (;;)
{
my_free((char*) wptr, MYF(MY_ALLOW_ZERO_PTR));
free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
if ((error=sort_get_next_record(sort_param)))
DBUG_RETURN(error);
if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record)))
if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,
&sort_param->wordroot)))
DBUG_RETURN(1);
if (wptr->pos)
break;
......@@ -2837,7 +2841,7 @@ static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
#endif
if (!wptr->pos)
{
my_free((char*) sort_param->wordlist, MYF(0));
free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
sort_param->wordlist=0;
error=sort_write_record(sort_param);
}
......
storage/myisam/myisamdef.h
View file @ 32711e7b
......@@ -235,13 +235,14 @@ struct st_myisam_info {
/* accumulate indexfile changes between write's */
TREE *bulk_insert;
DYNAMIC_ARRAY *ft1_to_ft2; /* used only in ft1->ft2 conversion */
MYSQL_FTPARSER_PARAM *ftparser_param; /* share info between init/deinit */
char *filename; /* parameter to open filename */
uchar *buff, /* Temp area for key */
*lastkey,*lastkey2; /* Last used search key */
uchar *first_mbr_key; /* Searhed spatial key */
byte *rec_buff; /* Tempbuff for recordpack */
uchar *int_keypos, /* Save position for next/previous */
MEM_ROOT ft_memroot; /* used by the parser */
MYSQL_FTPARSER_PARAM *ftparser_param; /* share info between init/deinit */
char *filename; /* parameter to open filename */
uchar *buff, /* Temp area for key */
*lastkey,*lastkey2; /* Last used search key */
uchar *first_mbr_key; /* Searhed spatial key */
byte *rec_buff; /* Tempbuff for recordpack */
uchar *int_keypos, /* Save position for next/previous */
*int_maxpos; /* -""- */
uint int_nod_flag; /* -""- */
uint32 int_keytree_version; /* -""- */
......@@ -325,6 +326,7 @@ typedef struct st_mi_sort_param
uchar **sort_keys;
byte *rec_buff;
void *wordlist, *wordptr;
MEM_ROOT wordroot;
char *record;
MY_TMPDIR *tmpdir;
int (*key_cmp)(struct st_mi_sort_param *, const void *, const void *);
......
storage/myisam/sort.c
View file @ 32711e7b
......@@ -447,6 +447,7 @@ pthread_handler_t thr_find_all_keys(void *arg)
close_cached_file(&info->tempfile_for_exceptions);
ok:
free_root(&info->wordroot, MYF(0));
remove_io_thread(&info->read_cache);
pthread_mutex_lock(&info->sort_info->mutex);
info->sort_info->threads_running--;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7