Skip to content

M
MariaDB
Commit 2c567b2f authored by Thirunarayanan Balathandayuthapani's avatar Thirunarayanan Balathandayuthapani
Browse files
Options

MDEV-30996 insert.. select in presence of full text index freezes all other commits at commit time 

- This patch does the following:
git revert --no-commit 673243c8
git revert --no-commit 6c669b95
git revert --no-commit bacaf2d4
git checkout HEAD mysql-test
git revert --no-commit 1fd7d3a9

Above command reverts MDEV-29277, MDEV-25581, MDEV-29342.

When binlog is enabled, trasaction takes a lot of time to do
sync operation on innodb fts table. This leads to block
of other transaction commit. To avoid this failure, remove
the fulltext sync operation during transaction commit. So
reverted MDEV-25581 related patches.

We filed MDEV-31105 to avoid the memory consumption
problem during fulltext sync operation.
parent 51e62cb3
Hide whitespace changes
Inline Side-by-side
Showing with 552 additions and 109 deletions
mysql-test/suite/innodb_fts/r/concurrent_insert.result
View file @ 2c567b2f
......@@ -19,7 +19,7 @@ INSERT INTO t2 VALUES('mariadb');
connection default;
SET @saved_dbug = @@GLOBAL.debug_dbug;
SET GLOBAL debug_dbug ='+d,fts_instrument_sync_request,ib_optimize_wq_hang';
SET DEBUG_SYNC= 'fts_sync_end
SET DEBUG_SYNC= 'fts_instrument_sync_request
SIGNAL drop_index_start WAIT_FOR sync_op';
INSERT INTO t1 VALUES('Keyword');
connect con1,localhost,root,,,;
......
mysql-test/suite/innodb_fts/r/sync.result
View file @ 2c567b2f
......@@ -11,19 +11,19 @@ INSERT INTO t1(title) VALUES('database');
connection con1;
SET @old_dbug = @@SESSION.debug_dbug;
SET debug_dbug = '+d,fts_instrument_sync_debug';
SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR selected';
SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR selected';
INSERT INTO t1(title) VALUES('mysql database');
connection default;
SET DEBUG_SYNC= 'now WAIT_FOR written';
SET GLOBAL innodb_ft_aux_table="test/t1";
SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE;
WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION
SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE;
WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION
database 2 3 2 2 0
database 2 3 2 3 6
mysql 1 3 2 1 0
mysql 1 3 2 3 0
SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE;
WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION
SET GLOBAL innodb_ft_aux_table=default;
SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database');
FTS_DOC_ID title
......@@ -59,7 +59,7 @@ INSERT INTO t1(title) VALUES('mysql');
INSERT INTO t1(title) VALUES('database');
connection con1;
SET debug_dbug = '+d,fts_instrument_sync_debug';
SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR inserted';
SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR inserted';
INSERT INTO t1(title) VALUES('mysql database');
connection default;
SET DEBUG_SYNC= 'now WAIT_FOR written';
......@@ -70,14 +70,14 @@ SET debug_dbug = @old_dbug;
SET GLOBAL innodb_ft_aux_table="test/t1";
SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE;
WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION
database 4 4 1 4 6
mysql 4 4 1 4 0
SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE;
WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION
database 2 3 2 2 0
database 2 3 2 3 6
mysql 1 3 2 1 0
mysql 1 3 2 3 0
database 4 4 1 4 6
mysql 1 4 3 1 0
mysql 1 4 3 3 0
mysql 1 4 3 4 0
SET GLOBAL innodb_ft_aux_table=default;
SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database');
FTS_DOC_ID title
......
mysql-test/suite/innodb_fts/r/sync_block.result 0 → 100644
View file @ 2c567b2f
SET @old_log_output = @@global.log_output;
SET @old_slow_query_log = @@global.slow_query_log;
SET @old_general_log = @@global.general_log;
SET @old_long_query_time = @@global.long_query_time;
SET @old_debug = @@global.debug_dbug;
SET GLOBAL log_output = 'TABLE';
SET GLOBAL general_log = 1;
SET GLOBAL slow_query_log = 1;
SET GLOBAL long_query_time = 1;
connect con1,localhost,root,,;
connect con2,localhost,root,,;
connection default;
# Case 1: Sync blocks DML(insert) on the same table.
CREATE TABLE t1 (
FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
title VARCHAR(200),
FULLTEXT(title)
) ENGINE = InnoDB;
connection con1;
SET GLOBAL debug_dbug='+d,fts_instrument_sync_debug,fts_instrument_sync_sleep';
SET DEBUG_SYNC= 'fts_sync_begin SIGNAL begin WAIT_FOR continue';
INSERT INTO t1(title) VALUES('mysql database');
connection con2;
SET DEBUG_SYNC= 'now WAIT_FOR begin';
SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database');
connection default;
SET DEBUG_SYNC= 'now SIGNAL continue';
connection con1;
/* connection con1 */ INSERT INTO t1(title) VALUES('mysql database');
connection con2;
/* conneciton con2 */ SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database');
FTS_DOC_ID title
connection default;
# make con1 & con2 show up in mysql.slow_log
SELECT SLEEP(2);
SLEEP(2)
0
# slow log results should only contain INSERT INTO t1.
SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02';
sql_text
INSERT INTO t1(title) VALUES('mysql database')
SET GLOBAL debug_dbug = @old_debug;
TRUNCATE TABLE mysql.slow_log;
DROP TABLE t1;
# Case 2: Sync blocks DML(insert) on other tables.
CREATE TABLE t1 (
FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
title VARCHAR(200),
FULLTEXT(title)
) ENGINE = InnoDB;
CREATE TABLE t2(id INT);
connection con1;
SET GLOBAL debug_dbug='+d,fts_instrument_sync_request,fts_instrument_sync_sleep';
SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL begin WAIT_FOR continue';
INSERT INTO t1(title) VALUES('mysql database');
connection con2;
SET DEBUG_SYNC= 'now WAIT_FOR begin';
INSERT INTO t2 VALUES(1);
connection default;
SET DEBUG_SYNC= 'now SIGNAL continue';
connection con1;
/* connection con1 */ INSERT INTO t1(title) VALUES('mysql database');
connection con2;
/* conneciton con2 */ INSERT INTO t2 VALUES(1);
connection default;
SET DEBUG_SYNC = 'RESET';
# make con1 & con2 show up in mysql.slow_log
SELECT SLEEP(2);
SLEEP(2)
0
# slow log results should be empty here.
SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02';
sql_text
SET GLOBAL debug_dbug = @old_debug;
TRUNCATE TABLE mysql.slow_log;
DROP TABLE t1,t2;
disconnect con1;
disconnect con2;
# Restore slow log settings.
SET GLOBAL log_output = @old_log_output;
SET GLOBAL general_log = @old_general_log;
SET GLOBAL slow_query_log = @old_slow_query_log;
SET GLOBAL long_query_time = @old_long_query_time;
mysql-test/suite/innodb_fts/t/concurrent_insert.test
View file @ 2c567b2f
......@@ -31,7 +31,7 @@ INSERT INTO t2 VALUES('mariadb');
connection default;
SET @saved_dbug = @@GLOBAL.debug_dbug;
SET GLOBAL debug_dbug ='+d,fts_instrument_sync_request,ib_optimize_wq_hang';
SET DEBUG_SYNC= 'fts_sync_end
SET DEBUG_SYNC= 'fts_instrument_sync_request
SIGNAL drop_index_start WAIT_FOR sync_op';
send INSERT INTO t1 VALUES('Keyword');
......
mysql-test/suite/innodb_fts/t/sync.test
View file @ 2c567b2f
......@@ -27,7 +27,7 @@ connection con1;
SET @old_dbug = @@SESSION.debug_dbug;
SET debug_dbug = '+d,fts_instrument_sync_debug';
SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR selected';
SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR selected';
send INSERT INTO t1(title) VALUES('mysql database');
......@@ -74,7 +74,7 @@ connection con1;
SET debug_dbug = '+d,fts_instrument_sync_debug';
SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR inserted';
SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR inserted';
send INSERT INTO t1(title) VALUES('mysql database');
......
mysql-test/suite/innodb_fts/t/sync_block.test 0 → 100644
View file @ 2c567b2f
#
# BUG#22516559 MYSQL INSTANCE STALLS WHEN SYNCING FTS INDEX
#
--source include/have_innodb.inc
--source include/have_debug.inc
--source include/have_debug_sync.inc
--source include/have_log_bin.inc
--source include/count_sessions.inc
SET @old_log_output = @@global.log_output;
SET @old_slow_query_log = @@global.slow_query_log;
SET @old_general_log = @@global.general_log;
SET @old_long_query_time = @@global.long_query_time;
SET @old_debug = @@global.debug_dbug;
SET GLOBAL log_output = 'TABLE';
SET GLOBAL general_log = 1;
SET GLOBAL slow_query_log = 1;
SET GLOBAL long_query_time = 1;
connect (con1,localhost,root,,);
connect (con2,localhost,root,,);
connection default;
--echo # Case 1: Sync blocks DML(insert) on the same table.
CREATE TABLE t1 (
FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
title VARCHAR(200),
FULLTEXT(title)
) ENGINE = InnoDB;
connection con1;
SET GLOBAL debug_dbug='+d,fts_instrument_sync_debug,fts_instrument_sync_sleep';
SET DEBUG_SYNC= 'fts_sync_begin SIGNAL begin WAIT_FOR continue';
send INSERT INTO t1(title) VALUES('mysql database');
connection con2;
SET DEBUG_SYNC= 'now WAIT_FOR begin';
send SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database');
connection default;
SET DEBUG_SYNC= 'now SIGNAL continue';
connection con1;
--echo /* connection con1 */ INSERT INTO t1(title) VALUES('mysql database');
--reap
connection con2;
--echo /* conneciton con2 */ SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database');
--reap
connection default;
-- echo # make con1 & con2 show up in mysql.slow_log
SELECT SLEEP(2);
-- echo # slow log results should only contain INSERT INTO t1.
SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02';
SET GLOBAL debug_dbug = @old_debug;
TRUNCATE TABLE mysql.slow_log;
DROP TABLE t1;
--echo # Case 2: Sync blocks DML(insert) on other tables.
CREATE TABLE t1 (
FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
title VARCHAR(200),
FULLTEXT(title)
) ENGINE = InnoDB;
CREATE TABLE t2(id INT);
connection con1;
SET GLOBAL debug_dbug='+d,fts_instrument_sync_request,fts_instrument_sync_sleep';
SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL begin WAIT_FOR continue';
send INSERT INTO t1(title) VALUES('mysql database');
connection con2;
SET DEBUG_SYNC= 'now WAIT_FOR begin';
send INSERT INTO t2 VALUES(1);
connection default;
SET DEBUG_SYNC= 'now SIGNAL continue';
connection con1;
--echo /* connection con1 */ INSERT INTO t1(title) VALUES('mysql database');
--reap
connection con2;
--echo /* conneciton con2 */ INSERT INTO t2 VALUES(1);
--reap
connection default;
SET DEBUG_SYNC = 'RESET';
-- echo # make con1 & con2 show up in mysql.slow_log
SELECT SLEEP(2);
-- echo # slow log results should be empty here.
SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02';
SET GLOBAL debug_dbug = @old_debug;
TRUNCATE TABLE mysql.slow_log;
DROP TABLE t1,t2;
disconnect con1;
disconnect con2;
--source include/wait_until_count_sessions.inc
-- echo # Restore slow log settings.
SET GLOBAL log_output = @old_log_output;
SET GLOBAL general_log = @old_general_log;
SET GLOBAL slow_query_log = @old_slow_query_log;
SET GLOBAL long_query_time = @old_long_query_time;
storage/innobase/fts/fts0fts.cc
View file @ 2c567b2f
......@@ -38,22 +38,6 @@ Full Text Search interface
#include "dict0stats.h"
#include "btr0pcur.h"
/** The SYNC state of the cache. There is one instance of this struct
associated with each ADD thread. */
struct fts_sync_t {
/** Transaction used for SYNCing the cache to disk */
trx_t *trx;
/** Table with FTS index(es) */
dict_table_t *table;
/** Max size in bytes of the cache */
ulint max_cache_size;
/** The doc id at which the cache was noted as being
full, we use this to set the upper_limit field */
doc_id_t max_doc_id;
/** SYNC start time; only used if fts_enable_diag_print */
time_t start_time;
};
static const ulint FTS_MAX_ID_LEN = 32;
/** Column name from the FTS config table */
......@@ -201,8 +185,15 @@ struct fts_tokenize_param_t {
/** Run SYNC on the table, i.e., write out data from the cache to the
FTS auxiliary INDEX table and clear the cache at the end.
@param[in,out] sync sync state
@param[in] unlock_cache whether unlock cache lock when write node
@param[in] wait whether wait when a sync is in progress
@return DB_SUCCESS if all OK */
static dberr_t fts_sync(fts_sync_t *sync);
static
dberr_t
fts_sync(
fts_sync_t* sync,
bool unlock_cache,
bool wait);
/****************************************************************//**
Release all resources help by the words rb tree e.g., the node ilist. */
......@@ -275,6 +266,7 @@ fts_cache_destroy(fts_cache_t* cache)
mysql_mutex_destroy(&cache->init_lock);
mysql_mutex_destroy(&cache->deleted_lock);
mysql_mutex_destroy(&cache->doc_id_lock);
pthread_cond_destroy(&cache->sync->cond);
if (cache->stopword_info.cached_stopword) {
rbt_free(cache->stopword_info.cached_stopword);
......@@ -574,6 +566,7 @@ fts_index_cache_init(
for (i = 0; i < FTS_NUM_AUX_INDEX; ++i) {
ut_a(index_cache->ins_graph[i] == NULL);
ut_a(index_cache->sel_graph[i] == NULL);
}
}
......@@ -643,6 +636,7 @@ fts_cache_create(
mem_heap_zalloc(heap, sizeof(fts_sync_t)));
cache->sync->table = table;
pthread_cond_init(&cache->sync->cond, nullptr);
/* Create the index cache vector that will hold the inverted indexes. */
cache->indexes = ib_vector_create(
......@@ -968,6 +962,10 @@ fts_cache_index_cache_create(
mem_heap_zalloc(static_cast<mem_heap_t*>(
cache->self_heap->arg), n_bytes));
index_cache->sel_graph = static_cast<que_t**>(
mem_heap_zalloc(static_cast<mem_heap_t*>(
cache->self_heap->arg), n_bytes));
fts_index_cache_init(cache->sync_heap, index_cache);
if (cache->get_docs) {
......@@ -1041,6 +1039,13 @@ fts_cache_clear(
index_cache->ins_graph[j] = NULL;
}
if (index_cache->sel_graph[j] != NULL) {
que_graph_free(index_cache->sel_graph[j]);
index_cache->sel_graph[j] = NULL;
}
}
index_cache->doc_stats = NULL;
......@@ -1333,7 +1338,8 @@ fts_cache_add_doc(
ib_vector_last(word->nodes));
}
if (!fts_node || fts_node->ilist_size > FTS_ILIST_MAX_SIZE
if (fts_node == NULL || fts_node->synced
|| fts_node->ilist_size > FTS_ILIST_MAX_SIZE
|| doc_id < fts_node->last_doc_id) {
fts_node = static_cast<fts_node_t*>(
......@@ -3320,7 +3326,7 @@ fts_add_doc_from_tuple(
if (cache->total_size > fts_max_cache_size / 5
|| fts_need_sync) {
fts_sync(cache->sync);
fts_sync(cache->sync, true, false);
}
mtr_start(&mtr);
......@@ -3356,7 +3362,7 @@ fts_add_doc_by_id(
dict_index_t* fts_id_index;
ibool is_id_cluster;
fts_cache_t* cache = ftt->table->fts->cache;
bool need_sync= false;
ut_ad(cache->get_docs);
/* If Doc ID has been supplied by the user, then the table
......@@ -3496,32 +3502,44 @@ fts_add_doc_by_id(
get_doc->index_cache,
doc_id, doc.tokens);
/** FTS cache sync should happen
frequently. Because user thread
shouldn't hold the cache lock for
longer time. So cache should sync
whenever cache size exceeds 512 KB */
need_sync =
cache->total_size > 512*1024;
bool need_sync = !cache->sync->in_progress
&& (fts_need_sync
|| (cache->total_size
- cache->total_size_at_sync)
> fts_max_cache_size / 10);
if (need_sync) {
cache->total_size_at_sync =
cache->total_size;
}
mysql_mutex_unlock(&table->fts->cache->lock);
DBUG_EXECUTE_IF(
"fts_instrument_sync",
fts_sync_table(table);
fts_optimize_request_sync_table(table);
mysql_mutex_lock(&cache->lock);
if (cache->sync->in_progress)
my_cond_wait(
&cache->sync->cond,
&cache->lock.m_mutex);
mysql_mutex_unlock(&cache->lock);
);
DBUG_EXECUTE_IF(
"fts_instrument_sync_debug",
fts_sync(cache->sync);
fts_sync(cache->sync, true, true);
);
DEBUG_SYNC_C("fts_instrument_sync_request");
DBUG_EXECUTE_IF(
"fts_instrument_sync_request",
need_sync= true;
fts_optimize_request_sync_table(table);
);
if (need_sync) {
fts_optimize_request_sync_table(table);
}
mtr_start(&mtr);
if (i < num_idx - 1) {
......@@ -3547,10 +3565,6 @@ fts_add_doc_by_id(
ut_free(pcur.old_rec_buf);
mem_heap_free(heap);
if (need_sync) {
fts_sync_table(table);
}
}
......@@ -3910,13 +3924,15 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result))
dberr_t
fts_sync_write_words(
trx_t* trx,
fts_index_cache_t* index_cache)
fts_index_cache_t* index_cache,
bool unlock_cache)
{
fts_table_t fts_table;
ulint n_nodes = 0;
ulint n_words = 0;
const ib_rbt_node_t* rbt_node;
dberr_t error = DB_SUCCESS;
ibool print_error = FALSE;
dict_table_t* table = index_cache->index->table;
FTS_INIT_INDEX_TABLE(
......@@ -3947,36 +3963,53 @@ fts_sync_write_words(
fts_table.suffix = fts_get_suffix(selected);
/* We iterate over all the nodes even if there was an error */
for (i = 0; i < ib_vector_size(word->nodes); ++i) {
fts_node_t* fts_node = static_cast<fts_node_t*>(
ib_vector_get(word->nodes, i));
error = fts_write_node(
trx, &index_cache->ins_graph[selected],
&fts_table, &word->text, fts_node);
if (fts_node->synced) {
continue;
} else {
fts_node->synced = true;
}
/*FIXME: we need to handle the error properly. */
if (error == DB_SUCCESS) {
if (unlock_cache) {
mysql_mutex_unlock(
&table->fts->cache->lock);
}
error = fts_write_node(
trx,
&index_cache->ins_graph[selected],
&fts_table, &word->text, fts_node);
DEBUG_SYNC_C("fts_write_node");
DBUG_EXECUTE_IF("fts_write_node_crash",
DEBUG_SYNC_C("fts_write_node");
DBUG_EXECUTE_IF("fts_write_node_crash",
DBUG_SUICIDE(););
DBUG_EXECUTE_IF("fts_instrument_sync_sleep",
DBUG_EXECUTE_IF(
"fts_instrument_sync_sleep",
std::this_thread::sleep_for(
std::chrono::seconds(1)););
if (error != DB_SUCCESS) {
goto err_exit;
if (unlock_cache) {
mysql_mutex_lock(
&table->fts->cache->lock);
}
}
}
n_nodes += ib_vector_size(word->nodes);
if (UNIV_UNLIKELY(error != DB_SUCCESS)) {
err_exit:
if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) {
ib::error() << "(" << error << ") writing"
" word node to FTS auxiliary index table "
<< table->name;
break;
print_error = TRUE;
}
}
......@@ -4035,44 +4068,58 @@ fts_sync_index(
ut_ad(rbt_validate(index_cache->words));
return(fts_sync_write_words(trx, index_cache));
return(fts_sync_write_words(trx, index_cache, sync->unlock_cache));
}
/** Rollback a sync operation
@param[in,out] sync sync state */
/** Check if index cache has been synced completely
@param[in,out] index_cache index cache
@return true if index is synced, otherwise false. */
static
void
fts_sync_rollback(
fts_sync_t* sync)
bool
fts_sync_index_check(
fts_index_cache_t* index_cache)
{
trx_t* trx = sync->trx;
fts_cache_t* cache = sync->table->fts->cache;
for (ulint i = 0; i < ib_vector_size(cache->indexes); ++i) {
ulint j;
fts_index_cache_t* index_cache;
index_cache = static_cast<fts_index_cache_t*>(
ib_vector_get(cache->indexes, i));
const ib_rbt_node_t* rbt_node;
for (j = 0; fts_index_selector[j].value; ++j) {
for (rbt_node = rbt_first(index_cache->words);
rbt_node != NULL;
rbt_node = rbt_next(index_cache->words, rbt_node)) {
if (index_cache->ins_graph[j] != NULL) {
fts_tokenizer_word_t* word;
word = rbt_value(fts_tokenizer_word_t, rbt_node);
que_graph_free(index_cache->ins_graph[j]);
fts_node_t* fts_node;
fts_node = static_cast<fts_node_t*>(ib_vector_last(word->nodes));
index_cache->ins_graph[j] = NULL;
}
if (!fts_node->synced) {
return(false);
}
}
mysql_mutex_unlock(&cache->lock);
return(true);
}
fts_sql_rollback(trx);
/** Reset synced flag in index cache when rollback
@param[in,out] index_cache index cache */
static
void
fts_sync_index_reset(
fts_index_cache_t* index_cache)
{
const ib_rbt_node_t* rbt_node;
/* Avoid assertion in trx_t::free(). */
trx->dict_operation_lock_mode = false;
trx->free();
for (rbt_node = rbt_first(index_cache->words);
rbt_node != NULL;
rbt_node = rbt_next(index_cache->words, rbt_node)) {
fts_tokenizer_word_t* word;
word = rbt_value(fts_tokenizer_word_t, rbt_node);
fts_node_t* fts_node;
fts_node = static_cast<fts_node_t*>(ib_vector_last(word->nodes));
fts_node->synced = false;
}
}
/** Commit the SYNC, change state of processed doc ids etc.
......@@ -4105,20 +4152,19 @@ fts_sync_commit(
sync, cache->deleted_doc_ids);
}
/* We need to do this within the deleted lock since fts_delete() can
attempt to add a deleted doc id to the cache deleted id array. */
fts_cache_clear(cache);
DEBUG_SYNC_C("fts_deleted_doc_ids_clear");
fts_cache_init(cache);
mysql_mutex_unlock(&cache->lock);
if (UNIV_LIKELY(error == DB_SUCCESS)) {
/* We need to do this within the deleted lock
since fts_delete() can attempt to add a deleted
doc id to the cache deleted id array. */
fts_cache_clear(cache);
DEBUG_SYNC_C("fts_deleted_doc_ids_clear");
fts_cache_init(cache);
mysql_mutex_unlock(&cache->lock);
fts_sql_commit(trx);
} else {
fts_sql_rollback(trx);
ib::error() << "(" << error << ") during SYNC of "
"table " << sync->table->name;
fts_sync_rollback(sync);
return error;
}
if (UNIV_UNLIKELY(fts_enable_diag_print) && elapsed_time) {
......@@ -4138,13 +4184,66 @@ fts_sync_commit(
return(error);
}
/** Rollback a sync operation
@param[in,out] sync sync state */
static
void
fts_sync_rollback(
fts_sync_t* sync)
{
trx_t* trx = sync->trx;
fts_cache_t* cache = sync->table->fts->cache;
for (ulint i = 0; i < ib_vector_size(cache->indexes); ++i) {
ulint j;
fts_index_cache_t* index_cache;
index_cache = static_cast<fts_index_cache_t*>(
ib_vector_get(cache->indexes, i));
/* Reset synced flag so nodes will not be skipped
in the next sync, see fts_sync_write_words(). */
fts_sync_index_reset(index_cache);
for (j = 0; fts_index_selector[j].value; ++j) {
if (index_cache->ins_graph[j] != NULL) {
que_graph_free(index_cache->ins_graph[j]);
index_cache->ins_graph[j] = NULL;
}
if (index_cache->sel_graph[j] != NULL) {
que_graph_free(index_cache->sel_graph[j]);
index_cache->sel_graph[j] = NULL;
}
}
}
mysql_mutex_unlock(&cache->lock);
fts_sql_rollback(trx);
/* Avoid assertion in trx_t::free(). */
trx->dict_operation_lock_mode = false;
trx->free();
}
/** Run SYNC on the table, i.e., write out data from the cache to the
FTS auxiliary INDEX table and clear the cache at the end.
@param[in,out] sync sync state
@param[in] unlock_cache whether unlock cache lock when write node
@param[in] wait whether wait when a sync is in progress
@return DB_SUCCESS if all OK */
static dberr_t fts_sync(fts_sync_t *sync)
static
dberr_t
fts_sync(
fts_sync_t* sync,
bool unlock_cache,
bool wait)
{
if (srv_read_only_mode) {
return DB_READ_ONLY;
......@@ -4155,13 +4254,33 @@ static dberr_t fts_sync(fts_sync_t *sync)
fts_cache_t* cache = sync->table->fts->cache;
mysql_mutex_lock(&cache->lock);
/* Check if cache is being synced.
Note: we release cache lock in fts_sync_write_words() to
avoid long wait for the lock by other threads. */
if (sync->in_progress) {
if (!wait) {
mysql_mutex_unlock(&cache->lock);
return(DB_SUCCESS);
}
do {
my_cond_wait(&sync->cond, &cache->lock.m_mutex);
} while (sync->in_progress);
}
sync->unlock_cache = unlock_cache;
sync->in_progress = true;
DEBUG_SYNC_C("fts_sync_begin");
fts_sync_begin(sync);
begin_sync:
const size_t fts_cache_size= fts_max_cache_size;
if (cache->total_size > fts_cache_size) {
/* Avoid the case: sync never finish when
insert/update keeps comming. */
ut_ad(sync->unlock_cache);
sync->unlock_cache = false;
ib::warn() << "Total InnoDB FTS size "
<< cache->total_size << " for the table "
<< cache->sync->table->name
......@@ -4185,23 +4304,52 @@ static dberr_t fts_sync(fts_sync_t *sync)
error = fts_sync_index(sync, index_cache);
if (error != DB_SUCCESS) {
goto err_exit;
goto end_sync;
}
if (!sync->unlock_cache
&& cache->total_size < fts_max_cache_size) {
/* Reset the unlock cache if the value
is less than innodb_ft_cache_size */
sync->unlock_cache = true;
}
}
DBUG_EXECUTE_IF("fts_instrument_sync_interrupted",
sync->interrupted = true;
error = DB_INTERRUPTED;
goto err_exit;
goto end_sync;
);
if (error == DB_SUCCESS) {
/* Make sure all the caches are synced. */
for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
fts_index_cache_t* index_cache;
index_cache = static_cast<fts_index_cache_t*>(
ib_vector_get(cache->indexes, i));
if (index_cache->index->to_be_dropped
|| fts_sync_index_check(index_cache)) {
continue;
}
goto begin_sync;
}
end_sync:
if (error == DB_SUCCESS && !sync->interrupted) {
error = fts_sync_commit(sync);
} else {
err_exit:
fts_sync_rollback(sync);
return error;
}
mysql_mutex_lock(&cache->lock);
ut_ad(sync->in_progress);
sync->interrupted = false;
sync->in_progress = false;
pthread_cond_broadcast(&sync->cond);
mysql_mutex_unlock(&cache->lock);
/* We need to check whether an optimize is required, for that
we make copies of the two variables that control the trigger. These
variables can change behind our back and we don't want to hold the
......@@ -4213,7 +4361,6 @@ static dberr_t fts_sync(fts_sync_t *sync)
mysql_mutex_unlock(&cache->deleted_lock);
DEBUG_SYNC_C("fts_sync_end");
return(error);
}
......@@ -4222,12 +4369,12 @@ FTS auxiliary INDEX table and clear the cache at the end.
@param[in,out] table fts table
@param[in] wait whether wait for existing sync to finish
@return DB_SUCCESS on success, error code on failure. */
dberr_t fts_sync_table(dict_table_t* table)
dberr_t fts_sync_table(dict_table_t* table, bool wait)
{
ut_ad(table->fts);
return table->space && !table->corrupted && table->fts->cache
? fts_sync(table->fts->cache->sync)
? fts_sync(table->fts->cache->sync, !wait, wait)
: DB_SUCCESS;
}
......
storage/innobase/fts/fts0opt.cc
View file @ 2c567b2f
......@@ -83,8 +83,9 @@ enum fts_msg_type_t {
FTS_MSG_ADD_TABLE, /*!< Add table to the optimize thread's
work queue */
FTS_MSG_DEL_TABLE /*!< Remove a table from the optimize
FTS_MSG_DEL_TABLE, /*!< Remove a table from the optimize
threads work queue */
FTS_MSG_SYNC_TABLE /*!< Sync fts cache of a table */
};
/** Compressed list of words that have been read from FTS INDEX
......@@ -2624,6 +2625,36 @@ fts_optimize_remove_table(
mysql_mutex_unlock(&fts_optimize_wq->mutex);
}
/** Send sync fts cache for the table.
@param[in] table table to sync */
void
fts_optimize_request_sync_table(
dict_table_t* table)
{
/* if the optimize system not yet initialized, return */
if (!fts_optimize_wq) {
return;
}
mysql_mutex_lock(&fts_optimize_wq->mutex);
/* FTS optimizer thread is already exited */
if (fts_opt_start_shutdown) {
ib::info() << "Try to sync table " << table->name
<< " after FTS optimize thread exiting.";
} else if (table->fts->sync_message) {
/* If the table already has SYNC message in
fts_optimize_wq queue then ignore it */
} else {
add_msg(fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, table));
table->fts->sync_message = true;
DBUG_EXECUTE_IF("fts_optimize_wq_count_check",
DBUG_ASSERT(fts_optimize_wq->length <= 1000););
}
mysql_mutex_unlock(&fts_optimize_wq->mutex);
}
/** Add a table to fts_slots if it doesn't already exist. */
static bool fts_optimize_new_table(dict_table_t* table)
{
......@@ -2765,8 +2796,7 @@ static void fts_optimize_sync_table(dict_table_t *table,
if (sync_table->fts && sync_table->fts->cache && sync_table->is_accessible())
{
fts_sync_table(sync_table);
fts_sync_table(sync_table, false);
if (process_message)
{
mysql_mutex_lock(&fts_optimize_wq->mutex);
......@@ -2866,6 +2896,24 @@ static void fts_optimize_callback(void *)
--n_tables;
}
break;
case FTS_MSG_SYNC_TABLE:
if (UNIV_UNLIKELY(wsrep_sst_disable_writes)) {
add_msg(msg);
goto retry_later;
}
DBUG_EXECUTE_IF(
"fts_instrument_msg_sync_sleep",
std::this_thread::sleep_for(
std::chrono::milliseconds(
300)););
fts_optimize_sync_table(
static_cast<dict_table_t*>(msg->ptr),
true);
break;
default:
ut_error;
}
......@@ -2998,7 +3046,7 @@ void fts_sync_during_ddl(dict_table_t* table)
if (!sync_message)
return;
fts_sync_table(table);
fts_sync_table(table, false);
mysql_mutex_lock(&fts_optimize_wq->mutex);
table->fts->sync_message = false;
......
storage/innobase/handler/handler0alter.cc
View file @ 2c567b2f
......@@ -11584,8 +11584,12 @@ ha_innobase::commit_inplace_alter_table(
ut_d(dict_table_check_for_dup_indexes(
ctx->new_table, CHECK_ABORTED_OK));
ut_ad(!ctx->new_table->fts
|| fts_check_cached_index(ctx->new_table));
#ifdef UNIV_DEBUG
if (!(ctx->new_table->fts != NULL
&& ctx->new_table->fts->cache->sync->in_progress)) {
ut_a(fts_check_cached_index(ctx->new_table));
}
#endif
}
unlock_and_close_files(deleted, trx);
......
storage/innobase/include/fts0fts.h
View file @ 2c567b2f
......@@ -648,6 +648,12 @@ fts_optimize_remove_table(
void
fts_optimize_shutdown();
/** Send sync fts cache for the table.
@param[in] table table to sync */
void
fts_optimize_request_sync_table(
dict_table_t* table);
/**********************************************************************//**
Take a FTS savepoint. */
void
......@@ -702,8 +708,9 @@ fts_savepoint_rollback_last_stmt(
/** Run SYNC on the table, i.e., write out data from the cache to the
FTS auxiliary INDEX table and clear the cache at the end.
@param[in,out] table fts table
@param[in] wait whether to wait for existing sync to finish
@return DB_SUCCESS on success, error code on failure. */
dberr_t fts_sync_table(dict_table_t* table);
dberr_t fts_sync_table(dict_table_t* table, bool wait = true);
/****************************************************************//**
Create an FTS index cache. */
......
storage/innobase/include/fts0types.h
View file @ 2c567b2f
......@@ -75,6 +75,7 @@ struct fts_index_cache_t {
que_t** ins_graph; /*!< Insert query graphs */
que_t** sel_graph; /*!< Select query graphs */
CHARSET_INFO* charset; /*!< charset */
};
......@@ -86,7 +87,35 @@ struct fts_stopword_t {
CHARSET_INFO* charset; /*!< charset for stopword */
};
struct fts_sync_t;
/** The SYNC state of the cache. There is one instance of this struct
associated with each ADD thread. */
struct fts_sync_t {
trx_t* trx; /*!< The transaction used for SYNCing
the cache to disk */
dict_table_t* table; /*!< Table with FTS index(es) */
ulint max_cache_size; /*!< Max size in bytes of the cache */
ibool cache_full; /*!< flag, when true it indicates that
we need to sync the cache to disk */
ulint lower_index; /*!< the start index of the doc id
vector from where to start adding
documents to the FTS cache */
ulint upper_index; /*!< max index of the doc id vector to
add to the FTS cache */
ibool interrupted; /*!< TRUE if SYNC was interrupted */
doc_id_t min_doc_id; /*!< The smallest doc id added to the
cache. It should equal to
doc_ids[lower_index] */
doc_id_t max_doc_id; /*!< The doc id at which the cache was
noted as being full, we use this to
set the upper_limit field */
time_t start_time; /*!< SYNC start time; only used if
fts_enable_diag_print */
bool in_progress; /*!< flag whether sync is in progress.*/
bool unlock_cache; /*!< flag whether unlock cache when
write fts node */
/** condition variable for in_progress; used with table->fts->cache->lock */
pthread_cond_t cond;
};
/** The cache for the FTS system. It is a memory-based inverted index
that new entries are added to, until it grows over the configured maximum
......@@ -175,6 +204,7 @@ struct fts_node_t {
ulint ilist_size_alloc;
/*!< Allocated size of ilist in
bytes */
bool synced; /*!< flag whether the node is synced */
};
/** A tokenizer word. Contains information about one word. */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7