Adding Full Text Search support to partitions

Contains Spiral patches:
007_mariadb-10.2.0.partition_fulltext.diff  MDEV-7705
038_mariadb-10.2.0.partition_fulltext2.diff MDEV-7734

This commit has the following differences compared to the original
patches:

- Added necessary full text search cleanup at the storage engine layer
  that was omitted in the original patch.
- Added test case.
- A lot of code cleanups to make the code notable smaller.
- Changed SQL code to use ha_ft_end() instead of ft_end()

Original author: Kentoku SHIBA
First reviewer:  Jacob Mathew
Second reviewer: Michael Widenius

sql/ha_partition.h

@@ -67,6 +67,17 @@ class Parts_share_refs
   }
 };
 
+class ha_partition;
+
+/* Partition Full Text Search info */
+struct st_partition_ft_info
+{
+  struct _ft_vft        *please;
+  st_partition_ft_info  *next;
+  ha_partition          *file;
+  FT_INFO               **part_ft_info;
+};
+
 
 /**
   Partition specific Handler_share.
@@ -164,6 +175,8 @@ class ha_partition :public handler
   partition_info *m_part_info;          // local reference to partition
   Field **m_part_field_array;           // Part field array locally to save acc
   uchar *m_ordered_rec_buffer;          // Row and key buffer for ord. idx scan
+  st_partition_ft_info *ft_first;
+  st_partition_ft_info *ft_current;
   /*
     Current index.
     When used in key_rec_cmp: If clustered pk, index compare
@@ -229,6 +242,7 @@ class ha_partition :public handler
   bool m_is_sub_partitioned;             // Is subpartitioned
   bool m_ordered_scan_ongoing;
   bool m_rnd_init_and_first;
+  bool m_ft_init_and_first;
 
   /*
     If set, this object was created with ha_partition::clone and doesn't
@@ -283,6 +297,9 @@ class ha_partition :public handler
   enum_monotonicity_info m_part_func_monotonicity_info;
   bool                m_pre_calling;
   bool                m_pre_call_use_parallel;
+  /* Keep track of bulk access requests */
+  bool                bulk_access_executing;
+
   /** keep track of locked partitions */
   MY_BITMAP m_locked_partitions;
   /** Stores shared auto_increment etc. */
@@ -333,6 +350,7 @@ class ha_partition :public handler
                  ha_partition *clone_arg,
                  MEM_ROOT *clone_mem_root_arg);
    ~ha_partition();
+   void ha_partition_init();
   /*
     A partition handler has no characteristics in itself. It only inherits
     those from the underlying handlers. Here we set-up those constants to
@@ -693,7 +711,7 @@ class ha_partition :public handler
   virtual int multi_range_read_next(range_id_t *range_info);
   virtual int multi_range_read_explain_info(uint mrr_mode, char *str,
                                             size_t size);
-
+  uint last_part() { return m_last_part; }
 
 private:
   bool init_record_priority_queue();
@@ -992,7 +1010,7 @@ class ha_partition :public handler
     special file for handling names of partitions, engine types.
     HA_REC_NOT_IN_SEQ is always set for partition handler since we cannot
     guarantee that the records will be returned in sequence.
-    HA_CAN_FULLTEXT, HA_DUPLICATE_POS,
+    HA_DUPLICATE_POS,
     HA_CAN_INSERT_DELAYED, HA_PRIMARY_KEY_REQUIRED_FOR_POSITION is disabled
     until further investigated.
   */
@@ -1205,14 +1223,15 @@ class ha_partition :public handler
     -------------------------------------------------------------------------
     MODULE fulltext index
     -------------------------------------------------------------------------
-    Fulltext stuff not yet.
-    -------------------------------------------------------------------------
-    virtual int ft_init() { return HA_ERR_WRONG_COMMAND; }
-    virtual FT_INFO *ft_init_ext(uint flags,uint inx,const uchar *key,
-    uint keylen)
-    { return NULL; }
-    virtual int ft_read(uchar *buf) { return HA_ERR_WRONG_COMMAND; }
   */
+    void ft_close_search(FT_INFO *handler);
+    virtual int ft_init();
+    virtual int pre_ft_init();
+    virtual void ft_end();
+    virtual int pre_ft_end();
+    virtual FT_INFO *ft_init_ext(uint flags, uint inx, String *key);
+    virtual int ft_read(uchar *buf);
+    virtual int pre_ft_read(bool use_parallel);
 
   /*
      -------------------------------------------------------------------------

sql/handler.h

@@ -3312,7 +3312,9 @@ class handler :public Sql_alloc
   int compare_key(key_range *range);
   int compare_key2(key_range *range) const;
   virtual int ft_init() { return HA_ERR_WRONG_COMMAND; }
-  void ft_end() { ft_handler=NULL; }
+  virtual int pre_ft_init() { return HA_ERR_WRONG_COMMAND; }
+  virtual void ft_end() {}
+  virtual int pre_ft_end() { return 0; }
   virtual FT_INFO *ft_init_ext(uint flags, uint inx,String *key)
     { return NULL; }
 private:
@@ -3335,6 +3337,7 @@ class handler :public Sql_alloc
 
   /* Same as above, but with statistics */
   inline int ha_ft_read(uchar *buf);
+  inline void ha_ft_end() { ft_end(); ft_handler=NULL; }
   int ha_rnd_next(uchar *buf);
   int ha_rnd_pos(uchar *buf, uchar *pos);
   inline int ha_rnd_pos_by_record(uchar *buf);

sql/sql_select.cc

@@ -22144,7 +22144,7 @@ create_sort_index(THD *thd, JOIN *join, JOIN_TAB *tab, Filesort *fsort)
  
   table->file->ha_end_keyread();
   if (tab->type == JT_FT)
-    table->file->ft_end();
+    table->file->ha_ft_end();
   else
     table->file->ha_index_or_rnd_end();
 

storage/spider/mysql-test/spider/include/partition_fulltext_deinit.inc

+--let $MASTER_1_COMMENT_2_1= $MASTER_1_COMMENT_2_1_BACKUP
+--let $MASTER_1_COMMENT_2_2= $MASTER_1_COMMENT_2_2_BACKUP
+--let $CHILD2_1_DROP_TABLES= $CHILD2_1_DROP_TABLES_BACKUP
+--let $CHILD2_1_CREATE_TABLES= $CHILD2_1_CREATE_TABLES_BACKUP
+--let $CHILD2_1_SELECT_TABLES= $CHILD2_1_SELECT_TABLES_BACKUP
+--let $CHILD2_2_DROP_TABLES= $CHILD2_2_DROP_TABLES_BACKUP
+--let $CHILD2_2_CREATE_TABLES= $CHILD2_2_CREATE_TABLES_BACKUP
+--let $CHILD2_2_SELECT_TABLES= $CHILD2_2_SELECT_TABLES_BACKUP
+--let $CHILD2_3_DROP_TABLES= $CHILD2_3_DROP_TABLES_BACKUP
+--let $CHILD2_3_CREATE_TABLES= $CHILD2_3_CREATE_TABLES_BACKUP
+--let $CHILD2_3_SELECT_TABLES= $CHILD2_3_SELECT_TABLES_BACKUP
+--let $OUTPUT_CHILD_GROUP2= $OUTPUT_CHILD_GROUP2_BACKUP
+--let $USE_GENERAL_LOG= $USE_GENERAL_LOG_BACKUP
+--connection master_1
+set session join_cache_level= @old_join_cache_level;
+set session optimizer_switch= @old_optimizer_switch;
+--disable_warnings
+--disable_query_log
+--disable_result_log
+--source ../t/test_deinit.inc
+--enable_result_log
+--enable_query_log
+--enable_warnings

storage/spider/mysql-test/spider/include/partition_fulltext_init.inc

+--disable_warnings
+--disable_query_log
+--disable_result_log
+--source ../t/test_init.inc
+--enable_result_log
+--enable_query_log
+--enable_warnings
+--let $MASTER_1_COMMENT_2_1_BACKUP= $MASTER_1_COMMENT_2_1
+let $MASTER_1_COMMENT_2_1=
+  COMMENT='table "tbl_a", bka_mode "1"'
+  PARTITION BY KEY(pkey) (
+    PARTITION pt1 COMMENT='srv "s_2_1"',
+    PARTITION pt2 COMMENT='srv "s_2_2"',
+    PARTITION pt3 COMMENT='srv "s_2_3"'
+  );
+--let $CHILD2_1_DROP_TABLES_BACKUP= $CHILD2_1_DROP_TABLES
+let $CHILD2_1_DROP_TABLES=
+  DROP TABLE IF EXISTS tbl_a;
+--let $CHILD2_1_CREATE_TABLES_BACKUP= $CHILD2_1_CREATE_TABLES
+let $CHILD2_1_CREATE_TABLES=
+  CREATE TABLE tbl_a (
+    pkey int NOT NULL,
+    words text NOT NULL,
+    PRIMARY KEY (pkey),
+    FULLTEXT (words)
+  ) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
+--let $CHILD2_1_SELECT_TABLES_BACKUP= $CHILD2_1_SELECT_TABLES
+let $CHILD2_1_SELECT_TABLES=
+  SELECT pkey FROM tbl_a ORDER BY pkey;
+let $CHILD2_1_SELECT_ARGUMENT1=
+  SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
+--let $CHILD2_2_DROP_TABLES_BACKUP= $CHILD2_2_DROP_TABLES
+let $CHILD2_2_DROP_TABLES=
+  DROP TABLE IF EXISTS tbl_a;
+--let $CHILD2_2_CREATE_TABLES_BACKUP= $CHILD2_2_CREATE_TABLES
+let $CHILD2_2_CREATE_TABLES=
+  CREATE TABLE tbl_a (
+    pkey int NOT NULL,
+    words text NOT NULL,
+    PRIMARY KEY (pkey),
+    FULLTEXT (words)
+  ) $CHILD2_2_ENGINE $CHILD2_2_CHARSET;
+--let $CHILD2_2_SELECT_TABLES_BACKUP= $CHILD2_2_SELECT_TABLES
+let $CHILD2_2_SELECT_TABLES=
+  SELECT pkey FROM tbl_a ORDER BY pkey;
+let $CHILD2_2_SELECT_ARGUMENT1=
+  SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
+--let $CHILD2_3_DROP_TABLES_BACKUP= $CHILD2_3_DROP_TABLES
+let $CHILD2_3_DROP_TABLES=
+  DROP TABLE IF EXISTS tbl_a;
+--let $CHILD2_3_CREATE_TABLES_BACKUP= $CHILD2_3_CREATE_TABLES
+let $CHILD2_3_CREATE_TABLES=
+  CREATE TABLE tbl_a (
+    pkey int NOT NULL,
+    words text NOT NULL,
+    PRIMARY KEY (pkey),
+    FULLTEXT (words)
+  ) $CHILD2_3_ENGINE $CHILD2_3_CHARSET;
+--let $CHILD2_3_SELECT_TABLES_BACKUP= $CHILD2_3_SELECT_TABLES
+let $CHILD2_3_SELECT_TABLES=
+  SELECT pkey FROM tbl_a ORDER BY pkey;
+let $CHILD2_3_SELECT_ARGUMENT1=
+  SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
+--let $OUTPUT_CHILD_GROUP2_BACKUP= $OUTPUT_CHILD_GROUP2
+--let $OUTPUT_CHILD_GROUP2= 1
+--let $USE_GENERAL_LOG_BACKUP= $USE_GENERAL_LOG
+--let $USE_GENERAL_LOG= 1
+--connection master_1
+set @old_join_cache_level= @@join_cache_level;
+set session join_cache_level= 5;
+set @old_optimizer_switch= @@optimizer_switch;
+set session optimizer_switch= 'mrr=on';

storage/spider/mysql-test/spider/r/partition_fulltext.result

+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3
+child3_1
+child3_2
+child3_3
+connection master_1;
+set @old_join_cache_level= @@join_cache_level;
+set session join_cache_level= 5;
+set @old_optimizer_switch= @@optimizer_switch;
+set session optimizer_switch= 'mrr=on';
+
+drop and create databases
+connection master_1;
+DROP DATABASE IF EXISTS auto_test_local;
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+connection child2_1;
+SET @old_log_output = @@global.log_output;
+SET GLOBAL log_output = 'TABLE,FILE';
+DROP DATABASE IF EXISTS auto_test_remote;
+CREATE DATABASE auto_test_remote;
+USE auto_test_remote;
+connection child2_2;
+SET @old_log_output = @@global.log_output;
+SET GLOBAL log_output = 'TABLE,FILE';
+DROP DATABASE IF EXISTS auto_test_remote2;
+CREATE DATABASE auto_test_remote2;
+USE auto_test_remote2;
+connection child2_3;
+SET @old_log_output = @@global.log_output;
+SET GLOBAL log_output = 'TABLE,FILE';
+DROP DATABASE IF EXISTS auto_test_remote3;
+CREATE DATABASE auto_test_remote3;
+USE auto_test_remote3;
+
+create table and insert
+connection child2_1;
+CHILD2_1_DROP_TABLES
+CHILD2_1_CREATE_TABLES
+TRUNCATE TABLE mysql.general_log;
+connection child2_2;
+CHILD2_2_DROP_TABLES
+CHILD2_2_CREATE_TABLES
+TRUNCATE TABLE mysql.general_log;
+connection child2_3;
+CHILD2_3_DROP_TABLES
+CHILD2_3_CREATE_TABLES
+TRUNCATE TABLE mysql.general_log;
+connection master_1;
+DROP TABLE IF EXISTS tbl_a;
+DROP TABLE IF EXISTS tbl_b;
+CREATE TABLE tbl_a (
+pkey int NOT NULL,
+words text NOT NULL,
+PRIMARY KEY (pkey),
+FULLTEXT (words)
+) MASTER_1_ENGINE MASTER_1_CHARSET MASTER_1_COMMENT_2_1
+INSERT INTO tbl_a (pkey, words) VALUES (0, 'abc'),(1, 'def'),(2, 'ghi'),(3, 'jkl'),(4, 'mno'),(5, 'pqr'),(6, 'stu'),(7, 'vwx');
+
+select test
+connection child2_1;
+TRUNCATE TABLE mysql.general_log;
+connection master_1;
+SELECT pkey, words FROM tbl_a WHERE match(words) against('+ghi' in boolean mode);
+pkey	words
+2	ghi
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
+argument
+select match(`words`)against('+ghi' in boolean mode),`pkey`,`words` from `auto_test_remote`.`tbl_a` where match(`words`)against('+ghi' in boolean mode) and (match(`words`)against('+ghi' in boolean mode))
+SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
+SELECT pkey FROM tbl_a ORDER BY pkey;
+pkey
+4
+5
+connection child2_2;
+SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
+argument
+select match(`words`)against('+ghi' in boolean mode),`pkey`,`words` from `auto_test_remote2`.`tbl_a` where match(`words`)against('+ghi' in boolean mode) and (match(`words`)against('+ghi' in boolean mode))
+SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
+SELECT pkey FROM tbl_a ORDER BY pkey;
+pkey
+0
+1
+6
+7
+connection child2_3;
+SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
+argument
+select match(`words`)against('+ghi' in boolean mode),`pkey`,`words` from `auto_test_remote3`.`tbl_a` where match(`words`)against('+ghi' in boolean mode) and (match(`words`)against('+ghi' in boolean mode))
+SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'
+SELECT pkey FROM tbl_a ORDER BY pkey;
+pkey
+2
+3
+
+deinit
+connection master_1;
+DROP DATABASE IF EXISTS auto_test_local;
+connection child2_1;
+DROP DATABASE IF EXISTS auto_test_remote;
+SET GLOBAL log_output = @old_log_output;
+connection child2_2;
+DROP DATABASE IF EXISTS auto_test_remote2;
+SET GLOBAL log_output = @old_log_output;
+connection child2_3;
+DROP DATABASE IF EXISTS auto_test_remote3;
+SET GLOBAL log_output = @old_log_output;
+connection master_1;
+set session join_cache_level= @old_join_cache_level;
+set session optimizer_switch= @old_optimizer_switch;
+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3
+child3_1
+child3_2
+child3_3
+
+end of test

storage/spider/mysql-test/spider/t/partition_fulltext.test

+--source ../include/partition_fulltext_init.inc
+if (!$HAVE_PARTITION)
+{
+  --source ../include/partition_fulltext_deinit.inc
+  skip Test requires partitioning;
+}
+
+--echo
+--echo drop and create databases
+--connection master_1
+--disable_warnings
+DROP DATABASE IF EXISTS auto_test_local;
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  if ($USE_GENERAL_LOG)
+  {
+    SET @old_log_output = @@global.log_output;
+    SET GLOBAL log_output = 'TABLE,FILE';
+  }
+  DROP DATABASE IF EXISTS auto_test_remote;
+  CREATE DATABASE auto_test_remote;
+  USE auto_test_remote;
+  --connection child2_2
+  if ($USE_GENERAL_LOG)
+  {
+    SET @old_log_output = @@global.log_output;
+    SET GLOBAL log_output = 'TABLE,FILE';
+  }
+  DROP DATABASE IF EXISTS auto_test_remote2;
+  CREATE DATABASE auto_test_remote2;
+  USE auto_test_remote2;
+  --connection child2_3
+  if ($USE_GENERAL_LOG)
+  {
+    SET @old_log_output = @@global.log_output;
+    SET GLOBAL log_output = 'TABLE,FILE';
+  }
+  DROP DATABASE IF EXISTS auto_test_remote3;
+  CREATE DATABASE auto_test_remote3;
+  USE auto_test_remote3;
+}
+--enable_warnings
+
+--echo
+--echo create table and insert
+if ($USE_CHILD_GROUP2)
+{
+  if (!$OUTPUT_CHILD_GROUP2)
+  {
+    --disable_query_log
+    --disable_result_log
+  }
+  --connection child2_1
+  if ($OUTPUT_CHILD_GROUP2)
+  {
+    --disable_query_log
+    echo CHILD2_1_DROP_TABLES;
+    echo CHILD2_1_CREATE_TABLES;
+  }
+  --disable_warnings
+  eval $CHILD2_1_DROP_TABLES;
+  --enable_warnings
+  eval $CHILD2_1_CREATE_TABLES;
+  if ($OUTPUT_CHILD_GROUP2)
+  {
+    --enable_query_log
+  }
+  if ($USE_GENERAL_LOG)
+  {
+    TRUNCATE TABLE mysql.general_log;
+  }
+  --connection child2_2
+  if ($OUTPUT_CHILD_GROUP2)
+  {
+    --disable_query_log
+    echo CHILD2_2_DROP_TABLES;
+    echo CHILD2_2_CREATE_TABLES;
+  }
+  --disable_warnings
+  eval $CHILD2_2_DROP_TABLES;
+  --enable_warnings
+  eval $CHILD2_2_CREATE_TABLES;
+  if ($OUTPUT_CHILD_GROUP2)
+  {
+    --enable_query_log
+  }
+  if ($USE_GENERAL_LOG)
+  {
+    TRUNCATE TABLE mysql.general_log;
+  }
+  --connection child2_3
+  if ($OUTPUT_CHILD_GROUP2)
+  {
+    --disable_query_log
+    echo CHILD2_3_DROP_TABLES;
+    echo CHILD2_3_CREATE_TABLES;
+  }
+  --disable_warnings
+  eval $CHILD2_3_DROP_TABLES;
+  --enable_warnings
+  eval $CHILD2_3_CREATE_TABLES;
+  if ($OUTPUT_CHILD_GROUP2)
+  {
+    --enable_query_log
+  }
+  if ($USE_GENERAL_LOG)
+  {
+    TRUNCATE TABLE mysql.general_log;
+  }
+  if (!$OUTPUT_CHILD_GROUP2)
+  {
+    --enable_query_log
+    --enable_result_log
+  }
+}
+--connection master_1
+--disable_warnings
+DROP TABLE IF EXISTS tbl_a;
+DROP TABLE IF EXISTS tbl_b;
+--enable_warnings
+--disable_query_log
+echo CREATE TABLE tbl_a (
+    pkey int NOT NULL,
+    words text NOT NULL,
+    PRIMARY KEY (pkey),
+    FULLTEXT (words)
+) MASTER_1_ENGINE MASTER_1_CHARSET MASTER_1_COMMENT_2_1;
+eval CREATE TABLE tbl_a (
+    pkey int NOT NULL,
+    words text NOT NULL,
+    PRIMARY KEY (pkey),
+    FULLTEXT (words)
+) $MASTER_1_ENGINE $MASTER_1_CHARSET $MASTER_1_COMMENT_2_1;
+--enable_query_log
+INSERT INTO tbl_a (pkey, words) VALUES (0, 'abc'),(1, 'def'),(2, 'ghi'),(3, 'jkl'),(4, 'mno'),(5, 'pqr'),(6, 'stu'),(7, 'vwx');
+
+--echo
+--echo select test
+if ($USE_CHILD_GROUP2)
+{
+  if (!$OUTPUT_CHILD_GROUP2)
+  {
+    --disable_query_log
+    --disable_result_log
+  }
+  --connection child2_1
+  if ($USE_GENERAL_LOG)
+  {
+    TRUNCATE TABLE mysql.general_log;
+  }
+  if (!$OUTPUT_CHILD_GROUP2)
+  {
+    --enable_query_log
+    --enable_result_log
+  }
+}
+--connection master_1
+SELECT pkey, words FROM tbl_a WHERE match(words) against('+ghi' in boolean mode);
+if ($USE_CHILD_GROUP2)
+{
+  if (!$OUTPUT_CHILD_GROUP2)
+  {
+    --disable_query_log
+    --disable_result_log
+  }
+  --connection child2_1
+  if ($USE_GENERAL_LOG)
+  {
+    eval $CHILD2_1_SELECT_ARGUMENT1;
+  }
+  eval $CHILD2_1_SELECT_TABLES;
+  --connection child2_2
+  if ($USE_GENERAL_LOG)
+  {
+    eval $CHILD2_2_SELECT_ARGUMENT1;
+  }
+  eval $CHILD2_2_SELECT_TABLES;
+  --connection child2_3
+  if ($USE_GENERAL_LOG)
+  {
+    eval $CHILD2_3_SELECT_ARGUMENT1;
+  }
+  eval $CHILD2_3_SELECT_TABLES;
+  if (!$OUTPUT_CHILD_GROUP2)
+  {
+    --enable_query_log
+    --enable_result_log
+  }
+}
+
+--echo
+--echo deinit
+--disable_warnings
+--connection master_1
+DROP DATABASE IF EXISTS auto_test_local;
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  DROP DATABASE IF EXISTS auto_test_remote;
+  if ($USE_GENERAL_LOG)
+  {
+    SET GLOBAL log_output = @old_log_output;
+  }
+  --connection child2_2
+  DROP DATABASE IF EXISTS auto_test_remote2;
+  if ($USE_GENERAL_LOG)
+  {
+    SET GLOBAL log_output = @old_log_output;
+  }
+  --connection child2_3
+  DROP DATABASE IF EXISTS auto_test_remote3;
+  if ($USE_GENERAL_LOG)
+  {
+    SET GLOBAL log_output = @old_log_output;
+  }
+}
+--enable_warnings
+--source ../include/partition_fulltext_deinit.inc
+--echo
+--echo end of test