Skip to content

M
mariadb
Commit 3791197f authored by svoj@poseidon.ndb.mysql.com's avatar svoj@poseidon.ndb.mysql.com
Browse files
Options

WL#2575 - Fulltext: Parser plugin for FTS 

WL#2763 - MySQL plugin interface: step 1
Manual merge from CNET tree.
parent 37788da7
Hide whitespace changes
Inline Side-by-side
Showing with 612 additions and 234 deletions
include/ft_global.h
View file @ 3791197f
......@@ -53,6 +53,7 @@ extern ulong ft_min_word_len;
extern ulong ft_max_word_len;
extern ulong ft_query_expansion_limit;
extern char ft_boolean_syntax[15];
extern struct st_mysql_ftparser ft_default_parser;
int ft_init_stopwords(void);
void ft_free_stopwords(void);
......
include/my_global.h
View file @ 3791197f
......@@ -1380,4 +1380,23 @@ do { doubleget_union _tmp; \
#define dlerror() ""
#endif
#ifdef HAVE_DLOPEN
#if defined(__WIN__)
#define dlsym(lib, name) GetProcAddress((HMODULE)lib, name)
#define dlopen(libname, unused) LoadLibraryEx(libname, NULL, 0)
#define dlclose(lib) FreeLibrary((HMODULE)lib)
#elif !defined(OS2)
#include <dlfcn.h>
#endif
#endif
/* FreeBSD 2.2.2 does not define RTLD_NOW) */
#ifndef RTLD_NOW
#define RTLD_NOW 1
#endif
#ifndef HAVE_DLERROR
#define dlerror() ""
#endif
#endif /* my_global_h */
include/myisam.h
View file @ 3791197f
......@@ -32,6 +32,7 @@ extern "C" {
#include "keycache.h"
#endif
#include "my_handler.h"
#include <plugin.h>
/* defines used by myisam-funktions */
......@@ -196,6 +197,7 @@ typedef struct st_mi_keydef /* Key definition with open & info */
uint32 version; /* For concurrent read/write */
HA_KEYSEG *seg,*end;
struct st_mysql_ftparser *parser; /* Fulltext [pre]parser */
int (*bin_search)(struct st_myisam_info *info,struct st_mi_keydef *keyinfo,
uchar *page,uchar *key,
uint key_len,uint comp_flag,uchar * *ret_pos,
......
libmysqld/Makefile.am
View file @ 3791197f
......@@ -20,11 +20,13 @@
MYSQLDATAdir = $(localstatedir)
MYSQLSHAREdir = $(pkgdatadir)
MYSQLBASEdir= $(prefix)
MYSQLLIBdir= $(libdir)
DEFS = -DEMBEDDED_LIBRARY -DMYSQL_SERVER \
-DDEFAULT_MYSQL_HOME="\"$(MYSQLBASEdir)\"" \
-DDATADIR="\"$(MYSQLDATAdir)\"" \
-DSHAREDIR="\"$(MYSQLSHAREdir)\""
-DSHAREDIR="\"$(MYSQLSHAREdir)\"" \
-DLIBDIR="\"$(MYSQLLIBdir)\""
INCLUDES= @bdb_includes@ \
-I$(top_builddir)/include -I$(top_srcdir)/include \
-I$(top_srcdir)/sql -I$(top_srcdir)/sql/examples \
......
mysql-test/r/connect.result
View file @ 3791197f
......@@ -9,6 +9,7 @@ help_keyword
help_relation
help_topic
host
plugin
proc
procs_priv
tables_priv
......@@ -36,6 +37,7 @@ help_keyword
help_relation
help_topic
host
plugin
proc
procs_priv
tables_priv
......@@ -71,6 +73,7 @@ help_keyword
help_relation
help_topic
host
plugin
proc
procs_priv
tables_priv
......
mysql-test/r/information_schema.result
View file @ 3791197f
......@@ -60,6 +60,7 @@ help_keyword
help_relation
help_topic
host
plugin
proc
procs_priv
tables_priv
......@@ -709,7 +710,7 @@ CREATE TABLE t_crashme ( f1 BIGINT);
CREATE VIEW a1 (t_CRASHME) AS SELECT f1 FROM t_crashme GROUP BY f1;
CREATE VIEW a2 AS SELECT t_CRASHME FROM a1;
count(*)
101
102
drop view a2, a1;
drop table t_crashme;
select table_schema,table_name, column_name from
......@@ -779,7 +780,7 @@ flush privileges;
SELECT table_schema, count(*) FROM information_schema.TABLES GROUP BY TABLE_SCHEMA;
table_schema count(*)
information_schema 16
mysql 17
mysql 18
create table t1 (i int, j int);
create trigger trg1 before insert on t1 for each row
begin
......
mysql-test/r/mysqlcheck.result
View file @ 3791197f
......@@ -6,6 +6,7 @@ mysql.help_keyword OK
mysql.help_relation OK
mysql.help_topic OK
mysql.host OK
mysql.plugin OK
mysql.proc OK
mysql.procs_priv OK
mysql.tables_priv OK
......@@ -23,6 +24,7 @@ mysql.help_keyword OK
mysql.help_relation OK
mysql.help_topic OK
mysql.host OK
mysql.plugin OK
mysql.proc OK
mysql.procs_priv OK
mysql.tables_priv OK
......
mysql-test/r/system_mysql_db.result
View file @ 3791197f
......@@ -9,6 +9,7 @@ help_keyword
help_relation
help_topic
host
plugin
proc
procs_priv
tables_priv
......
mysql-test/t/system_mysql_db_fix.test
View file @ 3791197f
......@@ -85,7 +85,7 @@ INSERT INTO user VALUES ('localhost','', '','N','N','N','N','N','N','N','N','
-- disable_query_log
DROP TABLE db, host, user, func, tables_priv, columns_priv, procs_priv, help_category, help_keyword, help_relation, help_topic, proc, time_zone, time_zone_leap_second, time_zone_name, time_zone_transition, time_zone_transition_type;
DROP TABLE db, host, user, func, plugin, tables_priv, columns_priv, procs_priv, help_category, help_keyword, help_relation, help_topic, proc, time_zone, time_zone_leap_second, time_zone_name, time_zone_transition, time_zone_transition_type;
-- enable_query_log
......
scripts/mysql_create_system_tables.sh
View file @ 3791197f
......@@ -39,8 +39,8 @@ c_hc=""
c_hr=""
c_hk=""
i_ht=""
c_tzn="" c_tz="" c_tzt="" c_tztt="" c_tzls=""
i_tzn="" i_tz="" i_tzt="" i_tztt="" i_tzls=""
c_tzn="" c_tz="" c_tzt="" c_tztt="" c_tzls="" c_pl=""
i_tzn="" i_tz="" i_tzt="" i_tztt="" i_tzls="" i_pl=""
c_p="" c_pp=""
# Check for old tables
......@@ -202,6 +202,21 @@ then
c_f="$c_f comment='User defined functions';"
fi
if test ! -f $mdata/plugin.frm
then
if test "$1" = "verbose" ; then
echo "Preparing plugin table" 1>&2;
fi
c_pl="$c_pl CREATE TABLE plugin ("
c_pl="$c_pl name char(64) binary DEFAULT '' NOT NULL,"
c_pl="$c_pl dl char(128) DEFAULT '' NOT NULL,"
c_pl="$c_pl PRIMARY KEY (name)"
c_pl="$c_pl ) engine=MyISAM"
c_pl="$c_pl CHARACTER SET utf8 COLLATE utf8_bin"
c_pl="$c_pl comment='MySQL plugins';"
fi
if test ! -f $mdata/tables_priv.frm
then
if test "$1" = "verbose" ; then
......@@ -741,6 +756,9 @@ $i_u
$c_f
$i_f
$c_pl
$i_pl
$c_t
$c_c
......
scripts/mysql_fix_privilege_tables.sql
View file @ 3791197f
......@@ -19,6 +19,12 @@ CREATE TABLE IF NOT EXISTS func (
PRIMARY KEY (name)
) CHARACTER SET utf8 COLLATE utf8_bin;
CREATE TABLE IF NOT EXISTS plugin (
name char(64) binary DEFAULT '' NOT NULL,
dl char(128) DEFAULT '' NOT NULL,
PRIMARY KEY (name)
) CHARACTER SET utf8 COLLATE utf8_bin;
ALTER TABLE user add File_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL;
-- Detect whether or not we had the Grant_priv column
......
sql/Makefile.am
View file @ 3791197f
......@@ -19,6 +19,7 @@
MYSQLDATAdir = $(localstatedir)
MYSQLSHAREdir = $(pkgdatadir)
MYSQLBASEdir= $(prefix)
MYSQLLIBdir= $(pkglibdir)
INCLUDES = @ZLIB_INCLUDES@ \
@bdb_includes@ @innodb_includes@ @ndbcluster_includes@ \
-I$(top_builddir)/include -I$(top_srcdir)/include \
......@@ -116,6 +117,7 @@ DEFS = -DMYSQL_SERVER \
-DDEFAULT_MYSQL_HOME="\"$(MYSQLBASEdir)\"" \
-DDATADIR="\"$(MYSQLDATAdir)\"" \
-DSHAREDIR="\"$(MYSQLSHAREdir)\"" \
-DLIBDIR="\"$(MYSQLLIBdir)\"" \
@DEFS@
BUILT_SOURCES = sql_yacc.cc sql_yacc.h lex_hash.h
......
sql/ha_myisam.cc
View file @ 3791197f
......@@ -280,6 +280,7 @@ err:
int ha_myisam::open(const char *name, int mode, uint test_if_locked)
{
uint i;
if (!(file=mi_open(name, mode, test_if_locked)))
return (my_errno ? my_errno : -1);
......@@ -292,6 +293,14 @@ int ha_myisam::open(const char *name, int mode, uint test_if_locked)
int_table_flags|=HA_REC_NOT_IN_SEQ;
if (file->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
int_table_flags|=HA_HAS_CHECKSUM;
for (i= 0; i < table->s->keys; i++)
{
struct st_plugin_int *parser= table->key_info[i].parser;
if (table->key_info[i].flags & HA_USES_PARSER)
file->s->keyinfo[i].parser=
(struct st_mysql_ftparser *)parser->plugin->info;
}
return (0);
}
......
sql/lex.h
View file @ 3791197f
......@@ -246,6 +246,7 @@ static SYMBOL symbols[] = {
{ "INSENSITIVE", SYM(INSENSITIVE_SYM)},
{ "INSERT", SYM(INSERT)},
{ "INSERT_METHOD", SYM(INSERT_METHOD)},
{ "INSTALL", SYM(INSTALL_SYM)},
{ "INT", SYM(INT_SYM)},
{ "INT1", SYM(TINYINT)},
{ "INT2", SYM(SMALLINT)},
......@@ -370,6 +371,7 @@ static SYMBOL symbols[] = {
{ "OUTER", SYM(OUTER)},
{ "OUTFILE", SYM(OUTFILE)},
{ "PACK_KEYS", SYM(PACK_KEYS_SYM)},
{ "PARSER", SYM(PARSER_SYM)},
{ "PARTIAL", SYM(PARTIAL)},
#ifdef HAVE_PARTITION_DB
{ "PARTITION", SYM(PARTITION_SYM)},
......@@ -377,6 +379,7 @@ static SYMBOL symbols[] = {
{ "PARTITIONS", SYM(PARTITIONS_SYM)},
{ "PASSWORD", SYM(PASSWORD)},
{ "PHASE", SYM(PHASE_SYM)},
{ "PLUGIN", SYM(PLUGIN_SYM)},
{ "POINT", SYM(POINT_SYM)},
{ "POLYGON", SYM(POLYGON)},
{ "PRECISION", SYM(PRECISION)},
......@@ -454,7 +457,7 @@ static SYMBOL symbols[] = {
{ "SNAPSHOT", SYM(SNAPSHOT_SYM)},
{ "SMALLINT", SYM(SMALLINT)},
{ "SOME", SYM(ANY_SYM)},
{ "SONAME", SYM(UDF_SONAME_SYM)},
{ "SONAME", SYM(SONAME_SYM)},
{ "SOUNDS", SYM(SOUNDS_SYM)},
{ "SPATIAL", SYM(SPATIAL_SYM)},
{ "SPECIFIC", SYM(SPECIFIC_SYM)},
......@@ -525,6 +528,7 @@ static SYMBOL symbols[] = {
{ "UNIQUE", SYM(UNIQUE_SYM)},
{ "UNKNOWN", SYM(UNKNOWN_SYM)},
{ "UNLOCK", SYM(UNLOCK_SYM)},
{ "UNINSTALL", SYM(UNINSTALL_SYM)},
{ "UNSIGNED", SYM(UNSIGNED)},
{ "UNTIL", SYM(UNTIL_SYM)},
{ "UPDATE", SYM(UPDATE_SYM)},
......
sql/mysqld.cc
View file @ 3791197f
......@@ -1072,10 +1072,13 @@ void clean_up(bool print_message)
lex_free(); /* Free some memory */
set_var_free();
free_charsets();
#ifdef HAVE_DLOPEN
if (!opt_noacl)
{
#ifdef HAVE_DLOPEN
udf_free();
#endif
plugin_free();
}
(void) ha_panic(HA_PANIC_CLOSE); /* close all tables and logs */
if (tc_log)
tc_log->close();
......@@ -3413,10 +3416,13 @@ we force server id to 2, but this MySQL server will not act as a slave.");
if (!opt_noacl)
(void) grant_init();
#ifdef HAVE_DLOPEN
if (!opt_noacl)
{
plugin_init();
#ifdef HAVE_DLOPEN
udf_init();
#endif
}
if (opt_bootstrap) /* If running with bootstrap, do not start replication. */
opt_skip_slave_start= 1;
/*
......@@ -4574,7 +4580,8 @@ enum options_mysqld
OPT_TIMED_MUTEXES,
OPT_OLD_STYLE_USER_LIMITS,
OPT_LOG_SLOW_ADMIN_STATEMENTS,
OPT_TABLE_LOCK_WAIT_TIMEOUT
OPT_TABLE_LOCK_WAIT_TIMEOUT,
OPT_PLUGIN_DIR
};
......@@ -5724,6 +5731,10 @@ The minimum value for this variable is 4096.",
(gptr*) &global_system_variables.optimizer_search_depth,
(gptr*) &max_system_variables.optimizer_search_depth,
0, GET_ULONG, OPT_ARG, MAX_TABLES+1, 0, MAX_TABLES+2, 0, 1, 0},
{"plugin_dir", OPT_PLUGIN_DIR,
"Directory for plugins.",
(gptr*) &opt_plugin_dir_ptr, (gptr*) &opt_plugin_dir_ptr, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"preload_buffer_size", OPT_PRELOAD_BUFFER_SIZE,
"The size of the buffer that is allocated when preloading indexes",
(gptr*) &global_system_variables.preload_buff_size,
......@@ -6292,6 +6303,9 @@ static void mysql_init_variables(void)
sizeof(mysql_real_data_home)-1);
mysql_data_home_buff[0]=FN_CURLIB; // all paths are relative from here
mysql_data_home_buff[1]=0;
strmake(opt_plugin_dir, get_relative_path(LIBDIR),
sizeof(opt_plugin_dir) - 1);
opt_plugin_dir_ptr= opt_plugin_dir;
/* Replication parameters */
master_user= (char*) "test";
......@@ -7215,6 +7229,7 @@ static void fix_paths(void)
(void) my_load_path(mysql_home,mysql_home,""); // Resolve current dir
(void) my_load_path(mysql_real_data_home,mysql_real_data_home,mysql_home);
(void) my_load_path(pidfile_name,pidfile_name,mysql_real_data_home);
(void) my_load_path(opt_plugin_dir, opt_plugin_dir_ptr, mysql_home);
char *sharedir=get_relative_path(SHAREDIR);
if (test_if_hard_path(sharedir))
......
sql/set_var.cc
View file @ 3791197f
......@@ -781,6 +781,7 @@ struct show_var_st init_vars[]= {
{sys_optimizer_search_depth.name,(char*) &sys_optimizer_search_depth,
SHOW_SYS},
{"pid_file", (char*) pidfile_name, SHOW_CHAR},
{"plugin_dir", (char*) opt_plugin_dir, SHOW_CHAR},
{"port", (char*) &mysqld_port, SHOW_INT},
{sys_preload_buff_size.name, (char*) &sys_preload_buff_size, SHOW_SYS},
{"protocol_version", (char*) &protocol_version, SHOW_INT},
......
sql/share/errmsg.txt
View file @ 3791197f
......@@ -3006,7 +3006,7 @@ ER_CANT_FIND_DL_ENTRY
cze "Nemohu naj-Bt funkci '%-.64s' v knihovn'"
dan "Kan ikke finde funktionen '%-.64s' i bibliotek'"
nla "Kan functie '%-.64s' niet in library vinden"
eng "Can't find function '%-.64s' in library'"
eng "Can't find symbol '%-.64s' in library'"
jps "function '%-.64s' Cu[Ɍt鎖ł܂",
est "Ei leia funktsiooni '%-.64s' antud teegis"
fre "Impossible de trouver la fonction '%-.64s' dans la bibliothque'"
......@@ -3018,7 +3018,7 @@ ER_CANT_FIND_DL_ENTRY
kor "̹ '%-.64s' Լ ã ϴ."
por "No pode encontrar a funo '%-.64s' na biblioteca"
rum "Nu pot gasi functia '%-.64s' in libraria"
rus " '%-.64s' "
rus " '%-.64s' "
serbian "Ne mogu da pronadjem funkciju '%-.64s' u biblioteci"
slo "Nemem njs funkciu '%-.64s' v kninici'"
spa "No puedo encontrar funcin '%-.64s' en libraria'"
......
sql/sql_class.h
View file @ 3791197f
......@@ -413,11 +413,13 @@ public:
List<key_part_spec> columns;
const char *name;
bool generated;
LEX_STRING *parser_name;
Key(enum Keytype type_par, const char *name_arg, enum ha_key_alg alg_par,
bool generated_arg, List<key_part_spec> &cols)
bool generated_arg, List<key_part_spec> &cols,
LEX_STRING *parser_arg= 0)
:type(type_par), algorithm(alg_par), columns(cols), name(name_arg),
generated(generated_arg)
generated(generated_arg), parser_name(parser_arg)
{}
~Key() {}
/* Equality comparison of keys (ignoring name) */
......
sql/sql_lex.h
View file @ 3791197f
......@@ -91,6 +91,7 @@ enum enum_sql_command {
SQLCOM_CREATE_TRIGGER, SQLCOM_DROP_TRIGGER,
SQLCOM_XA_START, SQLCOM_XA_END, SQLCOM_XA_PREPARE,
SQLCOM_XA_COMMIT, SQLCOM_XA_ROLLBACK, SQLCOM_XA_RECOVER,
SQLCOM_INSTALL_PLUGIN, SQLCOM_UNINSTALL_PLUGIN,
/* This should be the last !!! */
SQLCOM_END
......
sql/sql_parse.cc
View file @ 3791197f
......@@ -4780,6 +4780,15 @@ end_with_restore_list:
case SQLCOM_XA_RECOVER:
res= mysql_xa_recover(thd);
break;
case SQLCOM_INSTALL_PLUGIN:
if (! (res= mysql_install_plugin(thd, &thd->lex->comment,
&thd->lex->ident)))
send_ok(thd);
break;
case SQLCOM_UNINSTALL_PLUGIN:
if (! (res= mysql_uninstall_plugin(thd, &thd->lex->comment)))
send_ok(thd);
break;
default:
DBUG_ASSERT(0); /* Impossible */
send_ok(thd);
......
sql/sql_show.cc
View file @ 3791197f
......@@ -948,6 +948,12 @@ store_create_info(THD *thd, TABLE_LIST *table_list, String *packet)
}
}
packet->append(')');
if (key_info->parser)
{
packet->append(" WITH PARSER ", 13);
append_identifier(thd, packet, key_info->parser->name.str,
key_info->parser->name.length);
}
}
/*
......
sql/sql_table.cc
View file @ 3791197f
......@@ -1087,6 +1087,8 @@ static int mysql_prepare_table(THD *thd, HA_CREATE_INFO *create_info,
break;
case Key::FULLTEXT:
key_info->flags= HA_FULLTEXT;
if ((key_info->parser_name= key->parser_name))
key_info->flags|= HA_USES_PARSER;
break;
case Key::SPATIAL:
#ifdef HAVE_SPATIAL
......
sql/sql_udf.cc
View file @ 3791197f
......@@ -38,36 +38,10 @@
#ifdef HAVE_DLOPEN
extern "C"
{
#if defined(__WIN__)
void* dlsym(void* lib,const char* name)
{
return GetProcAddress((HMODULE)lib,name);
}
void* dlopen(const char* libname,int unused)
{
return LoadLibraryEx(libname,NULL,0);
}
void dlclose(void* lib)
{
FreeLibrary((HMODULE)lib);
}
#elif !defined(OS2)
#include <dlfcn.h>
#endif
#include <stdarg.h>
#include <hash.h>
}
#ifndef RTLD_NOW
#define RTLD_NOW 1 // For FreeBSD 2.2.2
#endif
#ifndef HAVE_DLERROR
#define dlerror() ""
#endif
static bool initialized = 0;
static MEM_ROOT mem;
static HASH udf_hash;
......@@ -194,8 +168,10 @@ void udf_init()
This is done to ensure that only approved dll from the system
directories are used (to make this even remotely secure).
*/
if (strchr(dl_name, '/') ||
IF_WIN(strchr(dl_name, '\\'),0) ||
if (my_strchr(files_charset_info, dl_name,
dl_name + strlen(dl_name), '/') ||
IF_WIN(my_strchr(files_charset_info, dl_name,
dl_name + strlen(dl_name), '\\'),0) ||
strlen(name.str) > NAME_LEN)
{
sql_print_error("Invalid row in mysql.func table for function '%.64s'",
......@@ -214,10 +190,13 @@ void udf_init()
void *dl = find_udf_dl(tmp->dl);
if (dl == NULL)
{
if (!(dl = dlopen(tmp->dl, RTLD_NOW)))
char dlpath[FN_REFLEN];
strxnmov(dlpath, sizeof(dlpath) - 1, opt_plugin_dir, "/", tmp->dl,
NullS);
if (!(dl= dlopen(dlpath, RTLD_NOW)))
{
/* Print warning to log */
sql_print_error(ER(ER_CANT_OPEN_LIBRARY), tmp->dl,errno,dlerror());
sql_print_error(ER(ER_CANT_OPEN_LIBRARY), dlpath, errno, dlerror());
/* Keep the udf in the hash so that we can remove it later */
continue;
}
......@@ -412,7 +391,9 @@ int mysql_create_function(THD *thd,udf_func *udf)
This is done to ensure that only approved dll from the system
directories are used (to make this even remotely secure).
*/
if (strchr(udf->dl, '/') || IF_WIN(strchr(udf->dl, '\\'),0))
if (my_strchr(files_charset_info, udf->dl, udf->dl + strlen(udf->dl), '/') ||
IF_WIN(strchr(files_charset_info, udf->dl,
udf->dl + strlen(udf->dl), '\\'),0))
{
my_message(ER_UDF_NO_PATHS, ER(ER_UDF_NO_PATHS), MYF(0));
DBUG_RETURN(1);
......@@ -431,12 +412,14 @@ int mysql_create_function(THD *thd,udf_func *udf)
}
if (!(dl = find_udf_dl(udf->dl)))
{
if (!(dl = dlopen(udf->dl, RTLD_NOW)))
char dlpath[FN_REFLEN];
strxnmov(dlpath, sizeof(dlpath) - 1, opt_plugin_dir, "/", udf->dl, NullS);
if (!(dl = dlopen(dlpath, RTLD_NOW)))
{
DBUG_PRINT("error",("dlopen of %s failed, error: %d (%s)",
udf->dl,errno,dlerror()));
dlpath, errno, dlerror()));
my_error(ER_CANT_OPEN_LIBRARY, MYF(0),
udf->dl, errno, dlerror());
dlpath, errno, dlerror());
goto err;
}
new_dl=1;
......
sql/sql_yacc.yy
View file @ 3791197f
......@@ -335,6 +335,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
%token INSENSITIVE_SYM
%token INSERT
%token INSERT_METHOD
%token INSTALL_SYM
%token INTERVAL_SYM
%token INTO
%token INT_SYM
......@@ -470,12 +471,14 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
%token OUTFILE
%token OUT_SYM
%token PACK_KEYS_SYM
%token PARSER_SYM
%token PARTIAL
%token PARTITION_SYM
%token PARTITIONS_SYM
%token PASSWORD
%token PARAM_MARKER
%token PHASE_SYM
%token PLUGIN_SYM
%token POINTFROMTEXT
%token POINT_SYM
%token POLYFROMTEXT
......@@ -560,6 +563,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
%token SLAVE
%token SMALLINT
%token SNAPSHOT_SYM
%token SONAME_SYM
%token SOUNDS_SYM
%token SPATIAL_SYM
%token SPECIFIC_SYM
......@@ -621,13 +625,13 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
%token TYPES_SYM
%token TYPE_SYM
%token UDF_RETURNS_SYM
%token UDF_SONAME_SYM
%token ULONGLONG_NUM
%token UNCOMMITTED_SYM
%token UNDEFINED_SYM
%token UNDERSCORE_CHARSET
%token UNDO_SYM
%token UNICODE_SYM
%token UNINSTALL_SYM
%token UNION_SYM
%token UNIQUE_SYM
%token UNIQUE_USERS
......@@ -696,7 +700,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
sp_opt_label BIN_NUM label_ident
%type <lex_str_ptr>
opt_table_alias
opt_table_alias opt_fulltext_parser
%type <table>
table_ident table_ident_nodb references xid
......@@ -844,7 +848,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
statement sp_suid opt_view_list view_list or_replace algorithm
sp_c_chistics sp_a_chistics sp_chistic sp_c_chistic xa
load_data opt_field_or_var_spec fields_or_vars opt_load_data_set_spec
view_user view_suid
install uninstall view_user view_suid
partition_entry
END_OF_INPUT
......@@ -906,6 +910,7 @@ statement:
| handler
| help
| insert
| install
| kill
| load
| lock
......@@ -930,6 +935,7 @@ statement:
| slave
| start
| truncate
| uninstall
| unlock
| update
| use
......@@ -1187,11 +1193,15 @@ create:
lex->col_list.empty();
lex->change=NullS;
}
'(' key_list ')'
'(' key_list ')' opt_fulltext_parser
{
LEX *lex=Lex;
lex->key_list.push_back(new Key($2,$4.str, $5, 0, lex->col_list));
if ($2 != Key::FULLTEXT && $12)
{
yyerror(ER(ER_SYNTAX_ERROR));
YYABORT;
}
lex->key_list.push_back(new Key($2,$4.str,$5,0,lex->col_list,$12));
lex->col_list.empty();
}
| CREATE DATABASE opt_if_not_exists ident
......@@ -1382,7 +1392,7 @@ sp_name:
;
create_function_tail:
RETURNS_SYM udf_type UDF_SONAME_SYM TEXT_STRING_sys
RETURNS_SYM udf_type SONAME_SYM TEXT_STRING_sys
{
LEX *lex=Lex;
lex->sql_command = SQLCOM_CREATE_FUNCTION;
......@@ -3302,10 +3312,15 @@ column_def:
;
key_def:
key_type opt_ident key_alg '(' key_list ')'
key_type opt_ident key_alg '(' key_list ')' opt_fulltext_parser
{
LEX *lex=Lex;
lex->key_list.push_back(new Key($1,$2, $3, 0, lex->col_list));
if ($1 != Key::FULLTEXT && $7)
{
yyerror(ER(ER_SYNTAX_ERROR));
YYABORT;
}
lex->key_list.push_back(new Key($1,$2, $3, 0, lex->col_list, $7));
lex->col_list.empty(); /* Alloced by sql_alloc */
}
| opt_constraint constraint_key_type opt_ident key_alg '(' key_list ')'
......@@ -3340,6 +3355,20 @@ key_def:
}
;
opt_fulltext_parser:
/* empty */ { $$= (LEX_STRING *)0; }
| WITH PARSER_SYM IDENT_sys
{
if (plugin_is_ready(&$3, MYSQL_FTPARSER_PLUGIN))
$$= (LEX_STRING *)sql_memdup(&$3, sizeof(LEX_STRING));
else
{
my_error(ER_FUNCTION_NOT_DEFINED, MYF(0), $3.str);
YYABORT;
}
}
;
opt_check_constraint:
/* empty */
| check_constraint
......@@ -8153,10 +8182,13 @@ keyword:
| FLUSH_SYM {}
| HANDLER_SYM {}
| HELP_SYM {}
| INSTALL_SYM {}
| LANGUAGE_SYM {}
| NO_SYM {}
| OPEN_SYM {}
| PARSER_SYM {}
| PARTITION_SYM {}
| PLUGIN_SYM {}
| PREPARE_SYM {}
| REPAIR {}
| RESET_SYM {}
......@@ -8166,10 +8198,12 @@ keyword:
| SECURITY_SYM {}
| SIGNED_SYM {}
| SLAVE {}
| SONAME_SYM {}
| START_SYM {}
| STOP_SYM {}
| TRUNCATE_SYM {}
| UNICODE_SYM {}
| UNINSTALL_SYM {}
| XA_SYM {}
;
......@@ -9732,4 +9766,19 @@ opt_migrate:
| FOR_SYM MIGRATE_SYM { Lex->xa_opt=XA_FOR_MIGRATE; }
;
install:
INSTALL_SYM PLUGIN_SYM IDENT_sys SONAME_SYM TEXT_STRING_sys
{
LEX *lex= Lex;
lex->sql_command= SQLCOM_INSTALL_PLUGIN;
lex->comment= $3;
lex->ident= $5;
};
uninstall:
UNINSTALL_SYM PLUGIN_SYM IDENT_sys
{
LEX *lex= Lex;
lex->sql_command= SQLCOM_UNINSTALL_PLUGIN;
lex->comment= $3;
};
sql/table.cc
View file @ 3791197f
......@@ -323,6 +323,7 @@ int openfrm(THD *thd, const char *name, const char *alias, uint db_stat,
if (! (share->connect_string.str= strmake_root(&outparam->mem_root,
next_chunk + 2, share->connect_string.length)))
{
DBUG_PRINT("EDS", ("strmake_root failed for connect_string"));
my_free(buff, MYF(0));
goto err;
}
......@@ -1064,9 +1065,17 @@ int openfrm(THD *thd, const char *name, const char *alias, uint db_stat,
int closefrm(register TABLE *table)
{
int error=0;
uint idx;
KEY *key_info;
DBUG_ENTER("closefrm");
if (table->db_stat)
error=table->file->close();
key_info= table->key_info;
for (idx= table->s->keys; idx; idx--, key_info++)
{
if (key_info->flags & HA_USES_PARSER)
plugin_unlock(key_info->parser);
}
my_free((char*) table->alias, MYF(MY_ALLOW_ZERO_PTR));
table->alias= 0;
if (table->field)
......
storage/myisam/ft_boolean_search.c
View file @ 3791197f
......@@ -91,6 +91,7 @@ struct st_ftb_expr
float weight;
float cur_weight;
LIST *phrase; /* phrase words */
LIST *document; /* for phrase search */
uint yesses; /* number of "yes" words matched */
uint nos; /* number of "no" words matched */
uint ythresh; /* number of "yes" words in expr */
......@@ -154,85 +155,160 @@ static int FTB_WORD_cmp_list(CHARSET_INFO *cs, FTB_WORD **a, FTB_WORD **b)
return i;
}
static void _ftb_parse_query(FTB *ftb, byte **start, byte *end,
FTB_EXPR *up, uint depth, byte *up_quot)
typedef struct st_my_ftb_param
{
byte res;
FTB_PARAM param;
FT_WORD w;
FTB_WORD *ftbw;
FTB_EXPR *ftbe;
FTB_EXPR *tmp_expr;
FT_WORD *phrase_word;
LIST *phrase_list;
uint extra=HA_FT_WLEN+ftb->info->s->rec_reflength; /* just a shortcut */
FTB *ftb;
FTB_EXPR *ftbe;
byte *up_quot;
uint depth;
} MY_FTB_PARAM;
static int ftb_query_add_word(void *param, byte *word, uint word_len,
MYSQL_FTPARSER_BOOLEAN_INFO *info)
{
MY_FTB_PARAM *ftb_param= (MY_FTB_PARAM *)param;
FTB_WORD *ftbw;
FTB_EXPR *ftbe, *tmp_expr;
FT_WORD *phrase_word;
LIST *tmp_element;
int r= info->weight_adjust;
float weight= (float)
(info->wasign ? nwghts : wghts)[(r>5)?5:((r<-5)?-5:r)];
switch (info->type) {
case FT_TOKEN_WORD:
ftbw= (FTB_WORD *)alloc_root(&ftb_param->ftb->mem_root,
sizeof(FTB_WORD) +
(info->trunc ? MI_MAX_KEY_BUFF :
word_len * ftb_param->ftb->charset->mbmaxlen +
HA_FT_WLEN +
ftb_param->ftb->info->s->rec_reflength));
ftbw->len= word_len + 1;
ftbw->flags= 0;
ftbw->off= 0;
if (info->yesno > 0) ftbw->flags|= FTB_FLAG_YES;
if (info->yesno < 0) ftbw->flags|= FTB_FLAG_NO;
if (info->trunc) ftbw->flags|= FTB_FLAG_TRUNC;
ftbw->weight= weight;
ftbw->up= ftb_param->ftbe;
ftbw->docid[0]= ftbw->docid[1]= HA_OFFSET_ERROR;
ftbw->ndepth= (info->yesno < 0) + ftb_param->depth;
ftbw->key_root= HA_OFFSET_ERROR;
memcpy(ftbw->word + 1, word, word_len);
ftbw->word[0]= word_len;
if (info->yesno > 0) ftbw->up->ythresh++;
queue_insert(&ftb_param->ftb->queue, (byte *)ftbw);
ftb_param->ftb->with_scan|= (info->trunc & FTB_FLAG_TRUNC);
for (tmp_expr= ftb_param->ftbe; tmp_expr->up; tmp_expr= tmp_expr->up)
if (! (tmp_expr->flags & FTB_FLAG_YES))
break;
ftbw->max_docid= &tmp_expr->max_docid;
/* fall through */
case FT_TOKEN_STOPWORD:
if (! ftb_param->up_quot) break;
phrase_word= (FT_WORD *)alloc_root(&ftb_param->ftb->mem_root, sizeof(FT_WORD));
tmp_element= (LIST *)alloc_root(&ftb_param->ftb->mem_root, sizeof(LIST));
phrase_word->pos= word;
phrase_word->len= word_len;
tmp_element->data= (void *)phrase_word;
ftb_param->ftbe->phrase= list_add(ftb_param->ftbe->phrase, tmp_element);
/* Allocate document list at this point.
It allows to avoid huge amount of allocs/frees for each row.*/
tmp_element= (LIST *)alloc_root(&ftb_param->ftb->mem_root, sizeof(LIST));
tmp_element->data= alloc_root(&ftb_param->ftb->mem_root, sizeof(FT_WORD));
ftb_param->ftbe->document=
list_add(ftb_param->ftbe->document, tmp_element);
break;
case FT_TOKEN_LEFT_PAREN:
ftbe=(FTB_EXPR *)alloc_root(&ftb_param->ftb->mem_root, sizeof(FTB_EXPR));
ftbe->flags= 0;
if (info->yesno > 0) ftbe->flags|= FTB_FLAG_YES;
if (info->yesno < 0) ftbe->flags|= FTB_FLAG_NO;
ftbe->weight= weight;
ftbe->up= ftb_param->ftbe;
ftbe->max_docid= ftbe->ythresh= ftbe->yweaks= 0;
ftbe->docid[0]= ftbe->docid[1]= HA_OFFSET_ERROR;
ftbe->phrase= NULL;
ftbe->document= 0;
if (info->quot) ftb_param->ftb->with_scan|= 2;
if (info->yesno > 0) ftbe->up->ythresh++;
ftb_param->ftbe= ftbe;
ftb_param->depth++;
ftb_param->up_quot= info->quot;
break;
case FT_TOKEN_RIGHT_PAREN:
if (ftb_param->ftbe->document)
{
/* Circuit document list */
for (tmp_element= ftb_param->ftbe->document;
tmp_element->next; tmp_element= tmp_element->next) /* no-op */;
tmp_element->next= ftb_param->ftbe->document;
ftb_param->ftbe->document->prev= tmp_element;
}
info->quot= 0;
if (ftb_param->ftbe->up)
{
DBUG_ASSERT(ftb_param->depth);
ftb_param->ftbe= ftb_param->ftbe->up;
ftb_param->depth--;
ftb_param->up_quot= 0;
}
break;
case FT_TOKEN_EOF:
default:
break;
}
return(0);
}
static int ftb_parse_query_internal(void *param, byte *query, uint len)
{
MY_FTB_PARAM *ftb_param= (MY_FTB_PARAM *)param;
MYSQL_FTPARSER_BOOLEAN_INFO info;
CHARSET_INFO *cs= ftb_param->ftb->charset;
byte **start= &query;
byte *end= query + len;
FT_WORD w;
info.prev= ' ';
info.quot= 0;
while (ft_get_word(cs, start, end, &w, &info))
ftb_query_add_word(param, w.pos, w.len, &info);
return(0);
}
static void _ftb_parse_query(FTB *ftb, byte *query, uint len,
struct st_mysql_ftparser *parser)
{
MYSQL_FTPARSER_PARAM param;
MY_FTB_PARAM ftb_param;
DBUG_ENTER("_ftb_parse_query");
DBUG_ASSERT(parser);
if (ftb->state != UNINITIALIZED)
return;
param.prev=' ';
param.quot= up_quot;
while ((res=ft_get_word(ftb->charset,start,end,&w,&param)))
{
int r=param.plusminus;
float weight= (float) (param.pmsign ? nwghts : wghts)[(r>5)?5:((r<-5)?-5:r)];
switch (res) {
case 1: /* word found */
ftbw=(FTB_WORD *)alloc_root(&ftb->mem_root,
sizeof(FTB_WORD) +
(param.trunc ? MI_MAX_KEY_BUFF :
w.len*ftb->charset->mbmaxlen+extra));
ftbw->len=w.len+1;
ftbw->flags=0;
ftbw->off=0;
if (param.yesno>0) ftbw->flags|=FTB_FLAG_YES;
if (param.yesno<0) ftbw->flags|=FTB_FLAG_NO;
if (param.trunc) ftbw->flags|=FTB_FLAG_TRUNC;
ftbw->weight=weight;
ftbw->up=up;
ftbw->docid[0]=ftbw->docid[1]=HA_OFFSET_ERROR;
ftbw->ndepth= (param.yesno<0) + depth;
ftbw->key_root=HA_OFFSET_ERROR;
memcpy(ftbw->word+1, w.pos, w.len);
ftbw->word[0]=w.len;
if (param.yesno > 0) up->ythresh++;
queue_insert(& ftb->queue, (byte *)ftbw);
ftb->with_scan|=(param.trunc & FTB_FLAG_TRUNC);
for (tmp_expr= up; tmp_expr->up; tmp_expr= tmp_expr->up)
if (! (tmp_expr->flags & FTB_FLAG_YES))
break;
ftbw->max_docid= &tmp_expr->max_docid;
case 4: /* not indexed word (stopword or too short/long) */
if (! up_quot) break;
phrase_word= (FT_WORD *)alloc_root(&ftb->mem_root, sizeof(FT_WORD));
phrase_list= (LIST *)alloc_root(&ftb->mem_root, sizeof(LIST));
phrase_word->pos= w.pos;
phrase_word->len= w.len;
phrase_list->data= (void *)phrase_word;
up->phrase= list_add(up->phrase, phrase_list);
break;
case 2: /* left bracket */
ftbe=(FTB_EXPR *)alloc_root(&ftb->mem_root, sizeof(FTB_EXPR));
ftbe->flags=0;
if (param.yesno>0) ftbe->flags|=FTB_FLAG_YES;
if (param.yesno<0) ftbe->flags|=FTB_FLAG_NO;
ftbe->weight=weight;
ftbe->up=up;
ftbe->max_docid= ftbe->ythresh= ftbe->yweaks= 0;
ftbe->docid[0]=ftbe->docid[1]=HA_OFFSET_ERROR;
ftbe->phrase= NULL;
if (param.quot) ftb->with_scan|=2;
if (param.yesno > 0) up->ythresh++;
_ftb_parse_query(ftb, start, end, ftbe, depth+1, param.quot);
param.quot=0;
break;
case 3: /* right bracket */
if (up_quot) up->phrase= list_reverse(up->phrase);
return;
}
}
return;
ftb_param.ftb= ftb;
ftb_param.depth= 0;
ftb_param.ftbe= ftb->root;
ftb_param.up_quot= 0;
param.mysql_parse= ftb_parse_query_internal;
param.mysql_add_word= ftb_query_add_word;
param.ftparser_state= 0;
param.mysql_ftparam= (void *)&ftb_param;
param.cs= ftb->charset;
param.doc= query;
param.length= len;
param.mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO;
parser->parse(&param);
DBUG_VOID_RETURN;
}
static int _ftb_no_dupes_cmp(void* not_used __attribute__((unused)),
const void *a,const void *b)
......@@ -463,8 +539,11 @@ FT_INFO * ft_init_boolean_search(MI_INFO *info, uint keynr, byte *query,
ftbe->max_docid= ftbe->ythresh= ftbe->yweaks= 0;
ftbe->docid[0]=ftbe->docid[1]=HA_OFFSET_ERROR;
ftbe->phrase= NULL;
ftbe->document= 0;
ftb->root=ftbe;
_ftb_parse_query(ftb, &query, query+query_len, ftbe, 0, NULL);
_ftb_parse_query(ftb, query, query_len, keynr == NO_SUCH_KEY ?
&ft_default_parser :
info->s->keyinfo[keynr].parser);
ftb->list=(FTB_WORD **)alloc_root(&ftb->mem_root,
sizeof(FTB_WORD *)*ftb->queue.elements);
memcpy(ftb->list, ftb->queue.root+1, sizeof(FTB_WORD *)*ftb->queue.elements);
......@@ -480,6 +559,62 @@ err:
}
typedef struct st_my_ftb_phrase_param
{
LIST *phrase;
LIST *document;
CHARSET_INFO *cs;
uint phrase_length;
uint document_length;
uint match;
} MY_FTB_PHRASE_PARAM;
static int ftb_phrase_add_word(void *param, byte *word, uint word_len,
MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
{
MY_FTB_PHRASE_PARAM *phrase_param= (MY_FTB_PHRASE_PARAM *)param;
FT_WORD *w= (FT_WORD *)phrase_param->document->data;
LIST *phrase, *document;
w->pos= word;
w->len= word_len;
phrase_param->document= phrase_param->document->prev;
if (phrase_param->phrase_length > phrase_param->document_length)
{
phrase_param->document_length++;
return 0;
}
/* TODO: rewrite phrase search to avoid
comparing the same word twice. */
for (phrase= phrase_param->phrase, document= phrase_param->document->next;
phrase; phrase= phrase->next, document= document->next)
{
FT_WORD *phrase_word= (FT_WORD *)phrase->data;
FT_WORD *document_word= (FT_WORD *)document->data;
if (my_strnncoll(phrase_param->cs, phrase_word->pos, phrase_word->len,
document_word->pos, document_word->len))
return 0;
}
phrase_param->match++;
return 0;
}
static int ftb_check_phrase_internal(void *param, byte *document, uint len)
{
FT_WORD word;
MY_FTB_PHRASE_PARAM *phrase_param= (MY_FTB_PHRASE_PARAM *)param;
const byte *docend= document + len;
while (ft_simple_get_word(phrase_param->cs, &document, docend, &word, FALSE))
{
ftb_phrase_add_word(param, word.pos, word.len, 0);
if (phrase_param->match)
return 1;
}
return 0;
}
/*
Checks if given buffer matches phrase list.
......@@ -494,32 +629,31 @@ err:
1 is returned if phrase found, 0 else.
*/
static int _ftb_check_phrase(const byte *s0, const byte *e0,
LIST *phrase, CHARSET_INFO *cs)
static int _ftb_check_phrase(const byte *document, uint len,
FTB_EXPR *ftbe, CHARSET_INFO *cs,
struct st_mysql_ftparser *parser)
{
FT_WORD h_word;
const byte *h_start= s0;
DBUG_ENTER("_ftb_strstr");
DBUG_ASSERT(phrase);
while (ft_simple_get_word(cs, (byte **)&h_start, e0, &h_word, FALSE))
{
FT_WORD *n_word;
LIST *phrase_element= phrase;
const byte *h_start1= h_start;
for (;;)
{
n_word= (FT_WORD *)phrase_element->data;
if (my_strnncoll(cs, (const uchar *) h_word.pos, h_word.len,
(const uchar *) n_word->pos, n_word->len))
break;
if (! (phrase_element= phrase_element->next))
DBUG_RETURN(1);
if (! ft_simple_get_word(cs, (byte **)&h_start1, e0, &h_word, FALSE))
DBUG_RETURN(0);
}
}
DBUG_RETURN(0);
MY_FTB_PHRASE_PARAM ftb_param;
MYSQL_FTPARSER_PARAM param;
DBUG_ENTER("_ftb_check_phrase");
DBUG_ASSERT(parser);
ftb_param.phrase= ftbe->phrase;
ftb_param.document= ftbe->document;
ftb_param.cs= cs;
ftb_param.phrase_length= list_length(ftbe->phrase);
ftb_param.document_length= 1;
ftb_param.match= 0;
param.mysql_parse= ftb_check_phrase_internal;
param.mysql_add_word= ftb_phrase_add_word;
param.ftparser_state= 0;
param.mysql_ftparam= (void *)&ftb_param;
param.cs= cs;
param.doc= (byte *)document;
param.length= len;
param.mode= MYSQL_FTPARSER_WITH_STOPWORDS;
parser->parse(&param);
DBUG_RETURN(ftb_param.match ? 1 : 0);
}
......@@ -530,6 +664,9 @@ static void _ftb_climb_the_tree(FTB *ftb, FTB_WORD *ftbw, FT_SEG_ITERATOR *ftsi_
float weight=ftbw->weight;
int yn=ftbw->flags, ythresh, mode=(ftsi_orig != 0);
my_off_t curdoc=ftbw->docid[mode];
struct st_mysql_ftparser *parser= ftb->keynr == NO_SUCH_KEY ?
&ft_default_parser :
ftb->info->s->keyinfo[ftb->keynr].parser;
for (ftbe=ftbw->up; ftbe; ftbe=ftbe->up)
{
......@@ -559,8 +696,8 @@ static void _ftb_climb_the_tree(FTB *ftb, FTB_WORD *ftbw, FT_SEG_ITERATOR *ftsi_
{
if (!ftsi.pos)
continue;
not_found = ! _ftb_check_phrase(ftsi.pos, ftsi.pos+ftsi.len,
ftbe->phrase, ftb->charset);
not_found = ! _ftb_check_phrase(ftsi.pos, ftsi.len,
ftbe, ftb->charset, parser);
}
if (not_found) break;
} /* ftbe->quot */
......@@ -667,14 +804,67 @@ err:
}
float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
typedef struct st_my_ftb_find_param
{
FT_WORD word;
FT_INFO *ftb;
FT_SEG_ITERATOR *ftsi;
} MY_FTB_FIND_PARAM;
static int ftb_find_relevance_add_word(void *param, byte *word, uint len,
MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
{
MY_FTB_FIND_PARAM *ftb_param= (MY_FTB_FIND_PARAM *)param;
FT_INFO *ftb= ftb_param->ftb;
FTB_WORD *ftbw;
int a, b, c;
for (a= 0, b= ftb->queue.elements, c= (a+b)/2; b-a>1; c= (a+b)/2)
{
ftbw= ftb->list[c];
if (mi_compare_text(ftb->charset, (uchar*)word, len,
(uchar*)ftbw->word+1, ftbw->len-1,
(my_bool)(ftbw->flags&FTB_FLAG_TRUNC), 0) > 0)
b= c;
else
a= c;
}
for (; c >= 0; c--)
{
ftbw= ftb->list[c];
if (mi_compare_text(ftb->charset, (uchar*)word, len,
(uchar*)ftbw->word + 1,ftbw->len - 1,
(my_bool)(ftbw->flags & FTB_FLAG_TRUNC), 0))
break;
if (ftbw->docid[1] == ftb->info->lastpos)
continue;
ftbw->docid[1]= ftb->info->lastpos;
_ftb_climb_the_tree(ftb, ftbw, ftb_param->ftsi);
}
return(0);
}
static int ftb_find_relevance_parse(void *param, byte *doc, uint len)
{
FT_INFO *ftb= ((MY_FTB_FIND_PARAM *)param)->ftb;
byte *end= doc + len;
FT_WORD w;
while (ft_simple_get_word(ftb->charset, &doc, end, &w, TRUE))
ftb_find_relevance_add_word(param, w.pos, w.len, 0);
return(0);
}
float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
{
FTB_EXPR *ftbe;
FT_SEG_ITERATOR ftsi, ftsi2;
const byte *end;
my_off_t docid=ftb->info->lastpos;
MY_FTB_FIND_PARAM ftb_param;
MYSQL_FTPARSER_PARAM param;
struct st_mysql_ftparser *parser= ftb->keynr == NO_SUCH_KEY ?
&ft_default_parser :
ftb->info->s->keyinfo[ftb->keynr].parser;
if (docid == HA_OFFSET_ERROR)
return -2.0;
......@@ -702,41 +892,23 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
_mi_ft_segiterator_init(ftb->info, ftb->keynr, record, &ftsi);
memcpy(&ftsi2, &ftsi, sizeof(ftsi));
ftb_param.ftb= ftb;
ftb_param.ftsi= &ftsi2;
param.mysql_parse= ftb_find_relevance_parse;
param.mysql_add_word= ftb_find_relevance_add_word;
param.ftparser_state= 0;
param.mysql_ftparam= (void *)&ftb_param;
param.cs= ftb->charset;
param.mode= MYSQL_FTPARSER_SIMPLE_MODE;
while (_mi_ft_segiterator(&ftsi))
{
if (!ftsi.pos)
continue;
end=ftsi.pos+ftsi.len;
while (ft_simple_get_word(ftb->charset, (byte **) &ftsi.pos,
(byte *) end, &word, TRUE))
{
int a, b, c;
for (a=0, b=ftb->queue.elements, c=(a+b)/2; b-a>1; c=(a+b)/2)
{
ftbw=ftb->list[c];
if (mi_compare_text(ftb->charset, (uchar*) word.pos, word.len,
(uchar*) ftbw->word+1, ftbw->len-1,
(my_bool) (ftbw->flags&FTB_FLAG_TRUNC),0) >0)
b=c;
else
a=c;
}
for (; c>=0; c--)
{
ftbw=ftb->list[c];
if (mi_compare_text(ftb->charset, (uchar*) word.pos, word.len,
(uchar*) ftbw->word+1,ftbw->len-1,
(my_bool) (ftbw->flags&FTB_FLAG_TRUNC),0))
break;
if (ftbw->docid[1] == docid)
continue;
ftbw->docid[1]=docid;
_ftb_climb_the_tree(ftb, ftbw, &ftsi2);
}
}
param.doc= (byte *)ftsi.pos;
param.length= ftsi.len;
parser->parse(&param);
}
ftbe=ftb->root;
if (ftbe->docid[1]==docid && ftbe->cur_weight>0 &&
ftbe->yesses>=ftbe->ythresh && !ftbe->nos)
......
storage/myisam/ft_nlq_search.c
View file @ 3791197f
......@@ -230,7 +230,7 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
NULL, NULL);
ft_parse_init(&wtree, aio.charset);
if (ft_parse(&wtree,query,query_len,0))
if (ft_parse(&wtree, query, query_len, 0, info->s->keyinfo[keynr].parser))
goto err;
if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
......
storage/myisam/ft_parser.c
View file @ 3791197f
......@@ -24,6 +24,14 @@ typedef struct st_ft_docstat {
double sum;
} FT_DOCSTAT;
typedef struct st_my_ft_parser_param
{
TREE *wtree;
my_bool with_alloc;
} MY_FT_PARSER_PARAM;
static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
{
return mi_compare_text(cs, (uchar*) w1->pos, w1->len,
......@@ -102,13 +110,14 @@ my_bool ft_boolean_check_syntax_string(const byte *str)
4 - stopword found
*/
byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
FT_WORD *word, FTB_PARAM *param)
FT_WORD *word, MYSQL_FTPARSER_BOOLEAN_INFO *param)
{
byte *doc=*start;
uint mwc, length, mbl;
param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
param->plusminus=param->pmsign=0;
param->weight_adjust= param->wasign= 0;
param->type= FT_TOKEN_EOF;
while (doc<end)
{
......@@ -119,7 +128,8 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
{
param->quot=doc;
*start=doc+1;
return 3; /* FTB_RBR */
param->type= FT_TOKEN_RIGHT_PAREN;
goto ret;
}
if (!param->quot)
{
......@@ -128,21 +138,22 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
/* param->prev=' '; */
*start=doc+1;
if (*doc == FTB_LQUOT) param->quot=*start;
return (*doc == FTB_RBR)+2;
param->type= (*doc == FTB_RBR ? FT_TOKEN_RIGHT_PAREN : FT_TOKEN_LEFT_PAREN);
goto ret;
}
if (param->prev == ' ')
{
if (*doc == FTB_YES ) { param->yesno=+1; continue; } else
if (*doc == FTB_EGAL) { param->yesno= 0; continue; } else
if (*doc == FTB_NO ) { param->yesno=-1; continue; } else
if (*doc == FTB_INC ) { param->plusminus++; continue; } else
if (*doc == FTB_DEC ) { param->plusminus--; continue; } else
if (*doc == FTB_NEG ) { param->pmsign=!param->pmsign; continue; }
if (*doc == FTB_INC ) { param->weight_adjust++; continue; } else
if (*doc == FTB_DEC ) { param->weight_adjust--; continue; } else
if (*doc == FTB_NEG ) { param->wasign= !param->wasign; continue; }
}
}
param->prev=*doc;
param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
param->plusminus=param->pmsign=0;
param->weight_adjust= param->wasign= 0;
}
mwc=length=0;
......@@ -161,20 +172,24 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
|| param->trunc) && length < ft_max_word_len)
{
*start=doc;
return 1;
param->type= FT_TOKEN_WORD;
goto ret;
}
else if (length) /* make sure length > 0 (if start contains spaces only) */
{
*start= doc;
return 4;
param->type= FT_TOKEN_STOPWORD;
goto ret;
}
}
if (param->quot)
{
param->quot=*start=doc;
return 3; /* FTB_RBR */
param->type= 3; /* FT_RBR */
goto ret;
}
return 0;
ret:
return param->type;
}
byte ft_simple_get_word(CHARSET_INFO *cs, byte **start, const byte *end,
......@@ -220,30 +235,67 @@ void ft_parse_init(TREE *wtree, CHARSET_INFO *cs)
DBUG_VOID_RETURN;
}
int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc)
static int ft_add_word(void *param, byte *word, uint word_len,
MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
{
byte *end=doc+doclen;
TREE *wtree;
FT_WORD w;
DBUG_ENTER("ft_parse");
while (ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
DBUG_ENTER("ft_add_word");
wtree= ((MY_FT_PARSER_PARAM *)param)->wtree;
if (((MY_FT_PARSER_PARAM *)param)->with_alloc)
{
if (with_alloc)
{
byte *ptr;
/* allocating the data in the tree - to avoid mallocs and frees */
DBUG_ASSERT(wtree->with_delete==0);
ptr=(byte *)alloc_root(& wtree->mem_root,w.len);
memcpy(ptr, w.pos, w.len);
w.pos=ptr;
}
if (!tree_insert(wtree, &w, 0, wtree->custom_arg))
goto err;
byte *ptr;
/* allocating the data in the tree - to avoid mallocs and frees */
DBUG_ASSERT(wtree->with_delete == 0);
ptr= (byte *)alloc_root(&wtree->mem_root, word_len);
memcpy(ptr, word, word_len);
w.pos= ptr;
}
else
w.pos= word;
w.len= word_len;
if (!tree_insert(wtree, &w, 0, wtree->custom_arg))
{
delete_tree(wtree);
DBUG_RETURN(1);
}
DBUG_RETURN(0);
}
err:
delete_tree(wtree);
DBUG_RETURN(1);
static int ft_parse_internal(void *param, byte *doc, uint doc_len)
{
byte *end=doc+doc_len;
FT_WORD w;
TREE *wtree;
DBUG_ENTER("ft_parse_internal");
wtree= ((MY_FT_PARSER_PARAM *)param)->wtree;
while (ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
if (ft_add_word(param, w.pos, w.len, 0))
DBUG_RETURN(1);
DBUG_RETURN(0);
}
int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc,
struct st_mysql_ftparser *parser)
{
MYSQL_FTPARSER_PARAM param;
MY_FT_PARSER_PARAM my_param;
DBUG_ENTER("ft_parse");
DBUG_ASSERT(parser);
my_param.wtree= wtree;
my_param.with_alloc= with_alloc;
param.mysql_parse= ft_parse_internal;
param.mysql_add_word= ft_add_word;
param.ftparser_state= 0;
param.mysql_ftparam= &my_param;
param.cs= wtree->custom_arg;
param.doc= doc;
param.length= doclen;
param.mode= MYSQL_FTPARSER_SIMPLE_MODE;
DBUG_RETURN(parser->parse(&param));
}
storage/myisam/ft_static.c
View file @ 3791197f
......@@ -626,3 +626,14 @@ const char *ft_precompiled_stopwords[] = {
#endif
NULL };
static int ft_default_parser_parse(MYSQL_FTPARSER_PARAM *param)
{
return param->mysql_parse(param->mysql_ftparam, param->doc, param->length);
}
struct st_mysql_ftparser ft_default_parser=
{
MYSQL_FTPARSER_INTERFACE_VERSION, ft_default_parser_parse, 0, 0
};
storage/myisam/ft_update.c
View file @ 3791197f
......@@ -99,15 +99,17 @@ uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
const byte *record, my_bool with_alloc)
{
FT_SEG_ITERATOR ftsi;
struct st_mysql_ftparser *parser;
DBUG_ENTER("_mi_ft_parse");
_mi_ft_segiterator_init(info, keynr, record, &ftsi);
ft_parse_init(parsed, info->s->keyinfo[keynr].seg->charset);
parser= info->s->keyinfo[keynr].parser;
while (_mi_ft_segiterator(&ftsi))
{
if (ftsi.pos)
if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc))
if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser))
DBUG_RETURN(1);
}
DBUG_RETURN(0);
......
storage/myisam/ftdefs.h
View file @ 3791197f
......@@ -22,6 +22,7 @@
#include <m_ctype.h>
#include <my_tree.h>
#include <queues.h>
#include <plugin.h>
#define true_word_char(s,X) (my_isalnum(s,X) || (X)=='_')
#define misc_word_char(X) ((X)=='\'')
......@@ -98,20 +99,12 @@ typedef struct st_ft_word {
double weight;
} FT_WORD;
typedef struct st_ftb_param {
byte prev;
int yesno;
int plusminus;
bool pmsign;
bool trunc;
byte *quot;
} FTB_PARAM;
int is_stopword(char *word, uint len);
uint _ft_make_key(MI_INFO *, uint , byte *, FT_WORD *, my_off_t);
byte ft_get_word(CHARSET_INFO *, byte **, byte *, FT_WORD *, FTB_PARAM *);
byte ft_get_word(CHARSET_INFO *, byte **, byte *, FT_WORD *,
MYSQL_FTPARSER_BOOLEAN_INFO *);
byte ft_simple_get_word(CHARSET_INFO *, byte **, const byte *,
FT_WORD *, my_bool);
......@@ -126,7 +119,7 @@ void _mi_ft_segiterator_dummy_init(const byte *, uint, FT_SEG_ITERATOR *);
uint _mi_ft_segiterator(FT_SEG_ITERATOR *);
void ft_parse_init(TREE *, CHARSET_INFO *);
int ft_parse(TREE *, byte *, int, my_bool);
int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser);
FT_WORD * ft_linearize(TREE *);
FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *);
uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool);
......
storage/myisam/mi_open.c
View file @ 3791197f
......@@ -1047,6 +1047,7 @@ char *mi_keydef_read(char *ptr, MI_KEYDEF *keydef)
keydef->block_size = keydef->block_length/MI_MIN_KEY_BLOCK_LENGTH-1;
keydef->underflow_block_length=keydef->block_length/3;
keydef->version = 0; /* Not saved */
keydef->parser = &ft_default_parser;
return ptr;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7