ha_tokudb_admin.cc 34.9 KB
Newer Older
Sergei Golubchik's avatar
Sergei Golubchik committed
1 2 3 4 5 6 7 8 9
/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "$Id$"
/*======
This file is part of TokuDB


Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.

Sergei Golubchik's avatar
Sergei Golubchik committed
10
    TokuDB is free software: you can redistribute it and/or modify
Sergei Golubchik's avatar
Sergei Golubchik committed
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
    it under the terms of the GNU General Public License, version 2,
    as published by the Free Software Foundation.

    TokuDB is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with TokuDB.  If not, see <http://www.gnu.org/licenses/>.

======= */

#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."

Sergei Golubchik's avatar
Sergei Golubchik committed
26
#include "tokudb_sysvars.h"
Sergei Golubchik's avatar
Sergei Golubchik committed
27 28
#include "toku_time.h"

Sergei Golubchik's avatar
Sergei Golubchik committed
29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
namespace tokudb {
namespace analyze {

class recount_rows_t : public tokudb::background::job_manager_t::job_t {
public:
    void* operator new(size_t sz);
    void operator delete(void* p);

    recount_rows_t(
        bool user_schedued,
        THD* thd,
        TOKUDB_SHARE* share,
        DB_TXN* txn);

    virtual ~recount_rows_t();

    virtual const char* key();
Sergei Golubchik's avatar
Sergei Golubchik committed
46 47 48 49 50
    virtual const char* database();
    virtual const char* table();
    virtual const char* type();
    virtual const char* parameters();
    virtual const char* status();
Sergei Golubchik's avatar
Sergei Golubchik committed
51 52 53 54 55 56 57 58 59 60 61 62 63 64

protected:
    virtual void on_run();

    virtual void on_destroy();

private:
    // to be provided by the initiator of recount rows
    THD*            _thd;
    TOKUDB_SHARE*   _share;
    DB_TXN*         _txn;
    ulonglong       _throttle;

    // for recount rows status reporting
Sergei Golubchik's avatar
Sergei Golubchik committed
65 66
    char            _parameters[256];
    char            _status[1024];
Sergei Golubchik's avatar
Sergei Golubchik committed
67 68 69 70 71 72 73 74 75 76 77 78 79 80
    int             _result;
    ulonglong       _recount_start; // in microseconds
    ulonglong       _total_elapsed_time; // in microseconds

    bool            _local_txn;
    ulonglong       _rows;
    ulonglong       _deleted_rows;
    ulonglong       _ticks;

    static int analyze_recount_rows_progress(
        uint64_t count,
        uint64_t deleted,
        void* extra);
    int analyze_recount_rows_progress(uint64_t count, uint64_t deleted);
Sergei Golubchik's avatar
Sergei Golubchik committed
81 82
};

Sergei Golubchik's avatar
Sergei Golubchik committed
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
void* recount_rows_t::operator new(size_t sz) {
    return tokudb::memory::malloc(sz, MYF(MY_WME|MY_ZEROFILL|MY_FAE));
}
void recount_rows_t::operator delete(void* p) {
    tokudb::memory::free(p);
}
recount_rows_t::recount_rows_t(
    bool user_scheduled,
    THD* thd,
    TOKUDB_SHARE* share,
    DB_TXN* txn) :
    tokudb::background::job_manager_t::job_t(user_scheduled),
    _share(share),
    _result(HA_ADMIN_OK),
    _recount_start(0),
    _total_elapsed_time(0),
    _local_txn(false),
    _rows(0),
    _deleted_rows(0),
    _ticks(0) {

    assert_debug(thd != NULL);
    assert_debug(share != NULL);

    if (tokudb::sysvars::analyze_in_background(thd)) {
        _thd = NULL;
        _txn = NULL;
    } else {
        _thd = thd;
        _txn = txn;
    }

    _throttle = tokudb::sysvars::analyze_throttle(thd);
Sergei Golubchik's avatar
Sergei Golubchik committed
116 117 118 119 120 121

    snprintf(_parameters,
             sizeof(_parameters),
             "TOKUDB_ANALYZE_THROTTLE=%llu;",
             _throttle);
    _status[0] = '\0';
Sergei Golubchik's avatar
Sergei Golubchik committed
122 123 124 125
}
recount_rows_t::~recount_rows_t() {
}
void recount_rows_t::on_run() {
Sergei Golubchik's avatar
Sergei Golubchik committed
126 127 128
    const char* orig_proc_info = NULL;
    if (_thd)
        orig_proc_info = tokudb_thd_get_proc_info(_thd);
Sergei Golubchik's avatar
Sergei Golubchik committed
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
    _recount_start = tokudb::time::microsec();
    _total_elapsed_time = 0;

    if (_txn == NULL) {
        _result = db_env->txn_begin(db_env, NULL, &_txn, DB_READ_UNCOMMITTED);

        if (_result != 0) {
            _txn = NULL;
            _result = HA_ADMIN_FAILED;
            goto error;
        }
        _local_txn = true;
    } else {
        _local_txn = false;
    }

    _result =
        _share->file->recount_rows(
            _share->file,
            analyze_recount_rows_progress,
            this);

    if (_result != 0) {
        if (_local_txn) {
            _txn->abort(_txn);
            _txn = NULL;
        }
        _result = HA_ADMIN_FAILED;
        goto error;
    }

    DB_BTREE_STAT64 dict_stats;
    _result = _share->file->stat64(_share->file, _txn, &dict_stats);
    if (_result == 0) {
        _share->set_row_count(dict_stats.bt_ndata, false);
    }
    if (_result != 0)
        _result = HA_ADMIN_FAILED;

    if (_local_txn) {
        if (_result == HA_ADMIN_OK) {
            _txn->commit(_txn, 0);
        } else {
            _txn->abort(_txn);
        }
        _txn = NULL;
    }

    sql_print_information(
        "tokudb analyze recount rows %d counted %lld",
        _result,
        _share->row_count());
error:
Sergei Golubchik's avatar
Sergei Golubchik committed
182 183
    if(_thd)
        tokudb_thd_set_proc_info(_thd, orig_proc_info);
Sergei Golubchik's avatar
Sergei Golubchik committed
184 185 186 187 188 189 190 191
    return;
}
void recount_rows_t::on_destroy() {
    _share->release();
}
const char* recount_rows_t::key() {
    return _share->full_table_name();
}
Sergei Golubchik's avatar
Sergei Golubchik committed
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
const char* recount_rows_t::database() {
    return _share->database_name();
}
const char* recount_rows_t::table() {
    return _share->table_name();
}
const char* recount_rows_t::type() {
    static const char* type = "TOKUDB_ANALYZE_MODE_RECOUNT_ROWS";
    return type;
}
const char* recount_rows_t::parameters() {
    return _parameters;
}
const char* recount_rows_t::status() {
    return _status;
Sergei Golubchik's avatar
Sergei Golubchik committed
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
}
int recount_rows_t::analyze_recount_rows_progress(
    uint64_t count,
    uint64_t deleted,
    void* extra) {

    recount_rows_t* context = (recount_rows_t*)extra;
    return context->analyze_recount_rows_progress(count, deleted);
}
int recount_rows_t::analyze_recount_rows_progress(
    uint64_t count,
    uint64_t deleted) {

    _rows = count;
    _deleted_rows += deleted;
    deleted > 0 ? _ticks += deleted : _ticks++;

    if (_ticks > 1000) {
        _ticks = 0;
        uint64_t now = tokudb::time::microsec();
        _total_elapsed_time = now - _recount_start;
228
        if ((_thd && thd_kill_level(_thd)) || cancelled()) {
Sergei Golubchik's avatar
Sergei Golubchik committed
229 230 231 232
            // client killed
            return ER_ABORTING_CONNECTION;
        }

Sergei Golubchik's avatar
Sergei Golubchik committed
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
        // rebuild status
        // There is a slight race condition here,
        // _status is used here for tokudb_thd_set_proc_info and it is also used
        // for the status column in i_s.background_job_status.
        // If someone happens to be querying/building the i_s table
        // at the exact same time that the status is being rebuilt here,
        // the i_s table could get some garbage status.
        // This solution is a little heavy handed but it works, it prevents us
        // from changing the status while someone might be immediately observing
        // us and it prevents someone from observing us while we change the
        // status
        tokudb::background::_job_manager->lock();
        snprintf(_status,
                 sizeof(_status),
                 "recount_rows %s.%s counted %llu rows and %llu deleted "
                 "in %llu seconds.",
                 _share->database_name(),
                 _share->table_name(),
                 _rows,
                 _deleted_rows,
                 _total_elapsed_time / tokudb::time::MICROSECONDS);
        tokudb::background::_job_manager->unlock();

Sergei Golubchik's avatar
Sergei Golubchik committed
256
        // report
Sergei Golubchik's avatar
Sergei Golubchik committed
257 258
        if (_thd)
            tokudb_thd_set_proc_info(_thd, _status);
Sergei Golubchik's avatar
Sergei Golubchik committed
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284

        // throttle
        // given the throttle value, lets calculate the maximum number of rows
        // we should have seen so far in a .1 sec resolution
        if (_throttle > 0) {
            uint64_t estimated_rows = _total_elapsed_time / 100000;
            estimated_rows = estimated_rows * (_throttle / 10);
            if (_rows + _deleted_rows > estimated_rows) {
                // sleep for 1/10 of a second
                tokudb::time::sleep_microsec(100000);
            }
        }
    }
    return 0;
}

class standard_t : public tokudb::background::job_manager_t::job_t {
public:
    void* operator new(size_t sz);
    void operator delete(void* p);

    standard_t(bool user_scheduled, THD* thd, TOKUDB_SHARE* share, DB_TXN* txn);

    virtual ~standard_t();

    virtual const char* key(void);
Sergei Golubchik's avatar
Sergei Golubchik committed
285 286 287 288 289
    virtual const char* database();
    virtual const char* table();
    virtual const char* type();
    virtual const char* parameters();
    virtual const char* status();
Sergei Golubchik's avatar
Sergei Golubchik committed
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305

protected:
    virtual void on_run();

    virtual void on_destroy();

private:
    // to be provided by initiator of analyze
    THD*            _thd;
    TOKUDB_SHARE*   _share;
    DB_TXN*         _txn;
    ulonglong       _throttle;      // in microseconds
    ulonglong       _time_limit;    // in microseconds
    double          _delete_fraction;

    // for analyze status reporting, may also use other state
Sergei Golubchik's avatar
Sergei Golubchik committed
306 307
    char            _parameters[256];
    char            _status[1024];
Sergei Golubchik's avatar
Sergei Golubchik committed
308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
    int             _result;
    ulonglong       _analyze_start; // in microseconds
    ulonglong       _total_elapsed_time; // in microseconds

    // for analyze internal use, pretty much these are per-key/index
    ulonglong       _current_key;
    bool            _local_txn;
    ulonglong       _half_time;
    ulonglong       _half_rows;
    ulonglong       _rows;
    ulonglong       _deleted_rows;
    ulonglong       _ticks;
    ulonglong       _analyze_key_start; // in microseconds
    ulonglong       _key_elapsed_time; // in microseconds
    uint            _scan_direction;

    static bool analyze_standard_cursor_callback(
        void* extra,
        uint64_t deleted_rows);
    bool analyze_standard_cursor_callback(uint64_t deleted_rows);

    int analyze_key_progress();
    int analyze_key(uint64_t* rec_per_key_part);
};

void* standard_t::operator new(size_t sz) {
    return tokudb::memory::malloc(sz, MYF(MY_WME|MY_ZEROFILL|MY_FAE));
}
void standard_t::operator delete(void* p) {
    tokudb::memory::free(p);
}
standard_t::standard_t(
    bool user_scheduled,
    THD* thd,
    TOKUDB_SHARE* share,
    DB_TXN* txn) :
    tokudb::background::job_manager_t::job_t(user_scheduled),
    _share(share),
    _result(HA_ADMIN_OK),
    _analyze_start(0),
    _total_elapsed_time(0),
    _current_key(0),
    _local_txn(false),
    _half_time(0),
    _half_rows(0),
    _rows(0),
    _deleted_rows(0),
    _ticks(0),
    _analyze_key_start(0),
    _key_elapsed_time(0),
    _scan_direction(0) {

    assert_debug(thd != NULL);
    assert_debug(share != NULL);

    if (tokudb::sysvars::analyze_in_background(thd)) {
       _thd = NULL;
       _txn = NULL;
    } else {
       _thd = thd;
       _txn = txn;
    }
    _throttle = tokudb::sysvars::analyze_throttle(thd);
    _time_limit =
        tokudb::sysvars::analyze_time(thd) * tokudb::time::MICROSECONDS;
    _delete_fraction = tokudb::sysvars::analyze_delete_fraction(thd);
Sergei Golubchik's avatar
Sergei Golubchik committed
374 375 376 377 378 379 380 381 382 383

    snprintf(_parameters,
             sizeof(_parameters),
             "TOKUDB_ANALYZE_DELETE_FRACTION=%f; "
             "TOKUDB_ANALYZE_TIME=%llu; TOKUDB_ANALYZE_THROTTLE=%llu;",
             _delete_fraction,
             _time_limit / tokudb::time::MICROSECONDS,
             _throttle);

    _status[0] = '\0';
Sergei Golubchik's avatar
Sergei Golubchik committed
384 385 386 387 388 389 390
}
standard_t::~standard_t() {
}
void standard_t::on_run() {
    DB_BTREE_STAT64 stat64;
    uint64_t rec_per_key_part[_share->_max_key_parts];
    uint64_t total_key_parts = 0;
Sergei Golubchik's avatar
Sergei Golubchik committed
391 392 393 394
    const char* orig_proc_info = NULL;
    if (_thd)
        orig_proc_info = tokudb_thd_get_proc_info(_thd);

Sergei Golubchik's avatar
Sergei Golubchik committed
395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
    _analyze_start = tokudb::time::microsec();
    _half_time = _time_limit > 0 ? _time_limit/2 : 0;

    if (_txn == NULL) {
        _result = db_env->txn_begin(db_env, NULL, &_txn, DB_READ_UNCOMMITTED);

        if (_result != 0) {
            _txn = NULL;
            _result = HA_ADMIN_FAILED;
            goto error;
        }
        _local_txn = true;
    } else {
        _local_txn = false;
    }

Sergei Golubchik's avatar
Sergei Golubchik committed
411
    assert_always(_share->key_file[0] != NULL);
Sergei Golubchik's avatar
Sergei Golubchik committed
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
    _result = _share->key_file[0]->stat64(_share->key_file[0], _txn, &stat64);
    if (_result != 0) {
        _result = HA_ADMIN_FAILED;
        goto cleanup;
    }
    _half_rows = stat64.bt_ndata / 2;

    for (ulonglong current_key = 0;
         _result == HA_ADMIN_OK && current_key < _share->_keys;
         current_key++) {

        _current_key = current_key;
        _rows = _deleted_rows = _ticks = 0;
        _result = analyze_key(&rec_per_key_part[total_key_parts]);

        if ((_result != 0 && _result != ETIME) ||
            (_result != 0 && _rows == 0 && _deleted_rows > 0)) {
            _result = HA_ADMIN_FAILED;
        }
        if (_thd && (_result == HA_ADMIN_FAILED ||
Sergei Golubchik's avatar
Sergei Golubchik committed
432
            static_cast<double>(_deleted_rows) >
Sergei Golubchik's avatar
Sergei Golubchik committed
433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
                _delete_fraction * (_rows + _deleted_rows))) {

            char name[256]; int namelen;
            namelen =
                snprintf(
                    name,
                    sizeof(name),
                    "%s.%s.%s",
                    _share->database_name(),
                    _share->table_name(),
                    _share->_key_descriptors[_current_key]._name);
            _thd->protocol->prepare_for_resend();
            _thd->protocol->store(name, namelen,  system_charset_info);
            _thd->protocol->store("analyze", 7, system_charset_info);
            _thd->protocol->store("info", 4, system_charset_info);
            char rowmsg[256];
            int rowmsglen;
            rowmsglen =
                snprintf(
                    rowmsg,
                    sizeof(rowmsg),
                    "rows processed %llu rows deleted %llu",
                    _rows,
                    _deleted_rows);
            _thd->protocol->store(rowmsg, rowmsglen, system_charset_info);
            _thd->protocol->write();

            sql_print_information(
                "tokudb analyze on %.*s %.*s",
                namelen,
                name,
                rowmsglen,
                rowmsg);
        }

        total_key_parts += _share->_key_descriptors[_current_key]._parts;
    }
    if (_result == HA_ADMIN_OK) {
        int error =
            tokudb::set_card_in_status(
                _share->status_block,
                _txn,
                total_key_parts,
                rec_per_key_part);
        if (error)
            _result = HA_ADMIN_FAILED;

        _share->lock();
        _share->update_cardinality_counts(total_key_parts, rec_per_key_part);
        _share->allow_auto_analysis(true);
        _share->unlock();
    }

cleanup:
    if (_local_txn) {
        if (_result == HA_ADMIN_OK) {
            _txn->commit(_txn, 0);
        } else {
            _txn->abort(_txn);
        }
        _txn = NULL;
    }

error:
Sergei Golubchik's avatar
Sergei Golubchik committed
497 498
    if (_thd)
        tokudb_thd_set_proc_info(_thd, orig_proc_info);
Sergei Golubchik's avatar
Sergei Golubchik committed
499 500 501 502 503 504 505 506 507 508 509
    return;
}
void standard_t::on_destroy() {
    _share->lock();
    _share->allow_auto_analysis(false);
    _share->unlock();
    _share->release();
}
const char* standard_t::key() {
    return _share->full_table_name();
}
Sergei Golubchik's avatar
Sergei Golubchik committed
510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
const char* standard_t::database() {
    return _share->database_name();
}
const char* standard_t::table() {
    return _share->table_name();
}
const char* standard_t::type() {
    static const char* type = "TOKUDB_ANALYZE_MODE_STANDARD";
    return type;
}
const char* standard_t::parameters() {
    return _parameters;
}
const char* standard_t::status() {
    return _status;
Sergei Golubchik's avatar
Sergei Golubchik committed
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542
}
bool standard_t::analyze_standard_cursor_callback(
    void* extra,
    uint64_t deleted_rows) {
    standard_t* context = (standard_t*)extra;
    return context->analyze_standard_cursor_callback(deleted_rows);
}
bool standard_t::analyze_standard_cursor_callback(uint64_t deleted_rows) {
    _deleted_rows += deleted_rows;
    _ticks += deleted_rows;
    return analyze_key_progress() != 0;
}
int standard_t::analyze_key_progress(void) {
    if (_ticks > 1000) {
        _ticks = 0;
        uint64_t now = tokudb::time::microsec();
        _total_elapsed_time = now - _analyze_start;
        _key_elapsed_time = now - _analyze_key_start;
543
        if ((_thd && thd_kill_level(_thd)) || cancelled()) {
Sergei Golubchik's avatar
Sergei Golubchik committed
544 545
            // client killed
            return ER_ABORTING_CONNECTION;
Sergei Golubchik's avatar
Sergei Golubchik committed
546 547
        } else if (_time_limit > 0 &&
                   static_cast<uint64_t>(_key_elapsed_time) > _time_limit) {
Sergei Golubchik's avatar
Sergei Golubchik committed
548 549 550 551
            // time limit reached
            return ETIME;
        }

Sergei Golubchik's avatar
Sergei Golubchik committed
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581
        // rebuild status
        // There is a slight race condition here,
        // _status is used here for tokudb_thd_set_proc_info and it is also used
        // for the status column in i_s.background_job_status.
        // If someone happens to be querying/building the i_s table
        // at the exact same time that the status is being rebuilt here,
        // the i_s table could get some garbage status.
        // This solution is a little heavy handed but it works, it prevents us
        // from changing the status while someone might be immediately observing
        // us and it prevents someone from observing us while we change the
        // status.
        static const char* scan_direction_str[] = {"not scanning",
                                                   "scanning forward",
                                                   "scanning backward",
                                                   "scan unknown"};

        const char* scan_direction = NULL;
        switch (_scan_direction) {
            case 0:
                scan_direction = scan_direction_str[0];
                break;
            case DB_NEXT:
                scan_direction = scan_direction_str[1];
                break;
            case DB_PREV:
                scan_direction = scan_direction_str[2];
                break;
            default:
                scan_direction = scan_direction_str[3];
                break;
Sergei Golubchik's avatar
Sergei Golubchik committed
582 583
        }

Sergei Golubchik's avatar
Sergei Golubchik committed
584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611
        float progress_rows = 0.0;
        if (_share->row_count() > 0)
            progress_rows = static_cast<float>(_rows) /
                            static_cast<float>(_share->row_count());
        float progress_time = 0.0;
        if (_time_limit > 0)
            progress_time = static_cast<float>(_key_elapsed_time) /
                            static_cast<float>(_time_limit);
        tokudb::background::_job_manager->lock();
        snprintf(
            _status,
            sizeof(_status),
            "analyze table standard %s.%s.%s %llu of %u %.lf%% rows %.lf%% "
            "time, %s",
            _share->database_name(),
            _share->table_name(),
            _share->_key_descriptors[_current_key]._name,
            _current_key,
            _share->_keys,
            progress_rows * 100.0,
            progress_time * 100.0,
            scan_direction);
        tokudb::background::_job_manager->unlock();

        // report
        if (_thd)
            tokudb_thd_set_proc_info(_thd, _status);

Sergei Golubchik's avatar
Sergei Golubchik committed
612 613 614 615 616 617 618 619 620 621 622 623
        // throttle
        // given the throttle value, lets calculate the maximum number of rows
        // we should have seen so far in a .1 sec resolution
        if (_throttle > 0) {
            uint64_t estimated_rows = _key_elapsed_time / 100000;
            estimated_rows = estimated_rows * (_throttle / 10);
            if (_rows + _deleted_rows > estimated_rows) {
                // sleep for 1/10 of a second
                tokudb::time::sleep_microsec(100000);
            }
        }
    }
Sergei Golubchik's avatar
Sergei Golubchik committed
624 625
    return 0;
}
Sergei Golubchik's avatar
Sergei Golubchik committed
626 627 628
int standard_t::analyze_key(uint64_t* rec_per_key_part) {
    int error = 0;
    DB* db = _share->key_file[_current_key];
Sergei Golubchik's avatar
Sergei Golubchik committed
629
    assert_always(db != NULL);
Sergei Golubchik's avatar
Sergei Golubchik committed
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664
    uint64_t num_key_parts = _share->_key_descriptors[_current_key]._parts;
    uint64_t unique_rows[num_key_parts];
    bool is_unique = _share->_key_descriptors[_current_key]._is_unique;
    DBC* cursor = NULL;
    int close_error = 0;
    DBT key, prev_key;
    bool copy_key = false;

    _analyze_key_start = tokudb::time::microsec();
    _key_elapsed_time = 0;
    _scan_direction = DB_NEXT;

    if (is_unique && num_key_parts == 1) {
        // don't compute for unique keys with a single part. we already know
        // the answer.
        _rows = unique_rows[0] = 1;
        goto done;
    }

    for (uint64_t i = 0; i < num_key_parts; i++)
        unique_rows[i] = 1;

    // stop looking when the entire dictionary was analyzed, or a
    // cap on execution time was reached, or the analyze was killed.
    while (1) {
        if (cursor == NULL) {
            error = db->cursor(db, _txn, &cursor, 0);
            if (error != 0)
                goto done;

            cursor->c_set_check_interrupt_callback(
                cursor,
                analyze_standard_cursor_callback,
                this);

Sergei Golubchik's avatar
Sergei Golubchik committed
665 666
            memset(&key, 0, sizeof(DBT));
            memset(&prev_key, 0, sizeof(DBT));
Sergei Golubchik's avatar
Sergei Golubchik committed
667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746
            copy_key = true;
        }

        error = cursor->c_get(cursor, &key, 0, _scan_direction);
        if (error != 0) {
            if (error == DB_NOTFOUND || error == TOKUDB_INTERRUPTED)
                error = 0; // not an error
            break;
        } else if (cancelled()) {
            error = ER_ABORTING_CONNECTION;
            break;
        }

        _rows++;
        _ticks++;

        // if copy_key is false at this pont, we have some value sitting in
        // prev_key that we can compare to
        // if the comparison reveals a unique key, we must set copy_key to true
        // so the code following can copy he current key into prev_key for the
        // next iteration
        if (copy_key == false) {
            // compare this key with the previous key. ignore
            // appended PK for SK's.
            // TODO if a prefix is different, then all larger keys
            // that include the prefix are also different.
            // TODO if we are comparing the entire primary key or
            // the entire unique secondary key, then the cardinality
            // must be 1, so we can avoid computing it.
            for (uint64_t i = 0; i < num_key_parts; i++) {
                int cmp = tokudb_cmp_dbt_key_parts(db, &prev_key, &key, i+1);
                if (cmp != 0) {
                    unique_rows[i]++;
                    copy_key = true;
                }
            }
        }

        // prev_key = key or prev_key is NULL
        if (copy_key) {
            prev_key.data =
                tokudb::memory::realloc(
                    prev_key.data,
                    key.size,
                    MYF(MY_WME|MY_ZEROFILL|MY_FAE));
            assert_always(prev_key.data);
            prev_key.size = key.size;
            memcpy(prev_key.data, key.data, prev_key.size);
            copy_key = false;
        }

        error = analyze_key_progress();
        if (error == ETIME) {
            error = 0;
            break;
        } else if (error) {
            break;
        }

        // if we have a time limit, are scanning forward and have exceed the
        // _half_time and not passed the _half_rows number of the rows in the
        // index: clean up the keys, close the cursor and reverse direction.
        if (TOKUDB_UNLIKELY(_half_time > 0 &&
            _scan_direction == DB_NEXT &&
            _key_elapsed_time >= _half_time &&
            _rows < _half_rows)) {

            tokudb::memory::free(prev_key.data); prev_key.data = NULL;
            close_error = cursor->c_close(cursor);
            assert_always(close_error == 0);
            cursor = NULL;
            _scan_direction = DB_PREV;
        }
    }
    // cleanup
    if (prev_key.data) tokudb::memory::free(prev_key.data);
    if (cursor) close_error = cursor->c_close(cursor);
    assert_always(close_error == 0);

done:
Sergei Golubchik's avatar
Sergei Golubchik committed
747 748 749 750 751
    // in case we timed out (bunch of deleted records) without hitting a
    // single row
    if (_rows == 0)
        _rows = 1;

Sergei Golubchik's avatar
Sergei Golubchik committed
752 753 754 755 756 757 758 759 760 761
    // return cardinality
    for (uint64_t i = 0; i < num_key_parts; i++) {
        rec_per_key_part[i] = _rows / unique_rows[i];
    }
    return error;
}

} // namespace analyze
} // namespace tokudb

Sergei Golubchik's avatar
Sergei Golubchik committed
762 763

int ha_tokudb::analyze(THD *thd, HA_CHECK_OPT *check_opt) {
Sergei Golubchik's avatar
Sergei Golubchik committed
764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821
    TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name());
    int result = HA_ADMIN_OK;
    tokudb::sysvars::analyze_mode_t mode = tokudb::sysvars::analyze_mode(thd);

    switch (mode) {
    case tokudb::sysvars::TOKUDB_ANALYZE_RECOUNT_ROWS:
        result = share->analyze_recount_rows(thd, transaction);
        break;
    case tokudb::sysvars::TOKUDB_ANALYZE_STANDARD:
        share->lock();
        result = share->analyze_standard(thd, transaction);
        share->unlock();
        break;
    case tokudb::sysvars::TOKUDB_ANALYZE_CANCEL:
        share->cancel_background_jobs();
        break;
    default:
        break; // no-op
    }
    TOKUDB_HANDLER_DBUG_RETURN(result);
}

int TOKUDB_SHARE::analyze_recount_rows(THD* thd,DB_TXN* txn) {
    TOKUDB_HANDLER_DBUG_ENTER("%s", table_name());

    assert_always(thd != NULL);

    int result = HA_ADMIN_OK;

    tokudb::analyze::recount_rows_t* job
        = new tokudb::analyze::recount_rows_t(true, thd, this, txn);
    assert_always(job != NULL);

    // job->destroy will drop the ref
    addref();
    unlock();

    bool ret = tokudb::background::_job_manager->
        run_job(job, tokudb::sysvars::analyze_in_background(thd));

    if (!ret) {
        job->destroy();
        delete job;
        result = HA_ADMIN_FAILED;
    }

    TOKUDB_HANDLER_DBUG_RETURN(result);
}

// on entry, if txn is !NULL, it is a user session invoking ANALYZE directly
// and no lock will be held on 'this', else if txn is NULL it is an auto and
// 'this' will be locked.
int TOKUDB_SHARE::analyze_standard(THD* thd, DB_TXN* txn) {
    TOKUDB_HANDLER_DBUG_ENTER("%s", table_name());

    assert_always(thd != NULL);
    assert_debug(_mutex.is_owned_by_me() == true);

Sergei Golubchik's avatar
Sergei Golubchik committed
822 823 824
    int result = HA_ADMIN_OK;

    // stub out analyze if optimize is remapped to alter recreate + analyze
Sergei Golubchik's avatar
Sergei Golubchik committed
825 826 827 828 829
    // when not auto analyze or if this is an alter
    if ((txn &&
         thd_sql_command(thd) != SQLCOM_ANALYZE &&
         thd_sql_command(thd) != SQLCOM_ALTER_TABLE) ||
        thd_sql_command(thd) == SQLCOM_ALTER_TABLE) {
Sergei Golubchik's avatar
Sergei Golubchik committed
830 831 832
        TOKUDB_HANDLER_DBUG_RETURN(result);
    }

Sergei Golubchik's avatar
Sergei Golubchik committed
833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
    tokudb::analyze::standard_t* job
        = new tokudb::analyze::standard_t(txn == NULL ? false : true, thd,
                                          this, txn);
    assert_always(job != NULL);

    // akin to calling addref, but we know, right here, right now, everything
    // in the share is set up, files open, etc...
    // job->destroy will drop the ref
    _use_count++;

    // don't want any autos kicking off while we are analyzing
    disallow_auto_analysis();

    unlock();

    bool ret =
        tokudb::background::_job_manager->run_job(
            job,
            tokudb::sysvars::analyze_in_background(thd));

    if (!ret) {
        job->destroy();
        delete job;
Sergei Golubchik's avatar
Sergei Golubchik committed
856 857
        result = HA_ADMIN_FAILED;
    }
Sergei Golubchik's avatar
Sergei Golubchik committed
858 859 860

    lock();

Sergei Golubchik's avatar
Sergei Golubchik committed
861 862 863
    TOKUDB_HANDLER_DBUG_RETURN(result);
}

Sergei Golubchik's avatar
Sergei Golubchik committed
864

Sergei Golubchik's avatar
Sergei Golubchik committed
865
typedef struct hot_optimize_context {
Sergei Golubchik's avatar
Sergei Golubchik committed
866
    THD* thd;
Sergei Golubchik's avatar
Sergei Golubchik committed
867
    char* write_status_msg;
Sergei Golubchik's avatar
Sergei Golubchik committed
868
    ha_tokudb* ha;
Sergei Golubchik's avatar
Sergei Golubchik committed
869 870 871 872 873 874 875 876 877 878
    uint progress_stage;
    uint current_table;
    uint num_tables;
    float progress_limit;
    uint64_t progress_last_time;
    uint64_t throttle;
} *HOT_OPTIMIZE_CONTEXT;

static int hot_optimize_progress_fun(void *extra, float progress) {
    HOT_OPTIMIZE_CONTEXT context = (HOT_OPTIMIZE_CONTEXT)extra;
879
    if (thd_kill_level(context->thd)) {
Sergei Golubchik's avatar
Sergei Golubchik committed
880 881 882
        sprintf(
            context->write_status_msg,
            "The process has been killed, aborting hot optimize.");
Sergei Golubchik's avatar
Sergei Golubchik committed
883 884 885
        return ER_ABORTING_CONNECTION;
    }
    float percentage = progress * 100;
Sergei Golubchik's avatar
Sergei Golubchik committed
886 887 888 889 890 891
    sprintf(
        context->write_status_msg,
        "Optimization of index %u of %u about %.lf%% done",
        context->current_table + 1,
        context->num_tables,
        percentage);
Sergei Golubchik's avatar
Sergei Golubchik committed
892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
    thd_proc_info(context->thd, context->write_status_msg);
#ifdef HA_TOKUDB_HAS_THD_PROGRESS
    if (context->progress_stage < context->current_table) {
        // the progress stage is behind the current table, so move up
        // to the next stage and set the progress stage to current.
        thd_progress_next_stage(context->thd);
        context->progress_stage = context->current_table;
    }
    // the percentage we report here is for the current stage/db
    thd_progress_report(context->thd, (unsigned long long) percentage, 100);
#endif

    // throttle the optimize table
    if (context->throttle) {
        uint64_t time_now = toku_current_time_microsec();
        uint64_t dt = time_now - context->progress_last_time;
        uint64_t throttle_time = 1000000ULL / context->throttle;
        if (throttle_time > dt) {
            usleep(throttle_time - dt);
        }
        context->progress_last_time = toku_current_time_microsec();
    }

    // return 1 if progress has reach the progress limit
    return progress >= context->progress_limit;
}

// flatten all DB's in this table, to do so, peform hot optimize on each db
Sergei Golubchik's avatar
Sergei Golubchik committed
920 921
int ha_tokudb::do_optimize(THD* thd) {
    TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name());
Sergei Golubchik's avatar
Sergei Golubchik committed
922
    int error = 0;
Sergei Golubchik's avatar
Sergei Golubchik committed
923
    const char* orig_proc_info = tokudb_thd_get_proc_info(thd);
Sergei Golubchik's avatar
Sergei Golubchik committed
924 925 926 927 928 929 930 931 932 933
    uint curr_num_DBs = table->s->keys + tokudb_test(hidden_primary_key);

#ifdef HA_TOKUDB_HAS_THD_PROGRESS
    // each DB is its own stage. as HOT goes through each db, we'll
    // move on to the next stage.
    thd_progress_init(thd, curr_num_DBs);
#endif

    // for each DB, run optimize and hot_optimize
    for (uint i = 0; i < curr_num_DBs; i++) {
Sergei Golubchik's avatar
Sergei Golubchik committed
934 935 936 937
        // only optimize the index if it matches the optimize_index_name
        // session variable
        const char* optimize_index_name =
            tokudb::sysvars::optimize_index_name(thd);
Sergei Golubchik's avatar
Sergei Golubchik committed
938
        if (optimize_index_name) {
Sergei Golubchik's avatar
Sergei Golubchik committed
939 940 941 942
            const char* this_index_name =
                i >= table_share->keys ?
                    "primary" :
                    table_share->key_info[i].name;
Sergei Golubchik's avatar
Sergei Golubchik committed
943 944 945 946 947 948
            if (strcasecmp(optimize_index_name, this_index_name) != 0) {
                continue;
            }
        }

        DB* db = share->key_file[i];
Sergei Golubchik's avatar
Sergei Golubchik committed
949
        assert_always(db != NULL);
Sergei Golubchik's avatar
Sergei Golubchik committed
950 951 952 953 954 955 956 957 958 959 960 961
        error = db->optimize(db);
        if (error) {
            goto cleanup;
        }

        struct hot_optimize_context hc;
        memset(&hc, 0, sizeof hc);
        hc.thd = thd;
        hc.write_status_msg = this->write_status_msg;
        hc.ha = this;
        hc.current_table = i;
        hc.num_tables = curr_num_DBs;
Sergei Golubchik's avatar
Sergei Golubchik committed
962
        hc.progress_limit = tokudb::sysvars::optimize_index_fraction(thd);
Sergei Golubchik's avatar
Sergei Golubchik committed
963
        hc.progress_last_time = toku_current_time_microsec();
Sergei Golubchik's avatar
Sergei Golubchik committed
964
        hc.throttle = tokudb::sysvars::optimize_throttle(thd);
Sergei Golubchik's avatar
Sergei Golubchik committed
965
        uint64_t loops_run;
Sergei Golubchik's avatar
Sergei Golubchik committed
966 967 968 969 970 971 972 973
        error =
            db->hot_optimize(
                db,
                NULL,
                NULL,
                hot_optimize_progress_fun,
                &hc,
                &loops_run);
Sergei Golubchik's avatar
Sergei Golubchik committed
974 975 976 977 978 979 980 981 982 983 984 985 986 987
        if (error) {
            goto cleanup;
        }
    }
    error = 0;

cleanup:
#ifdef HA_TOKUDB_HAS_THD_PROGRESS
    thd_progress_end(thd);
#endif
    thd_proc_info(thd, orig_proc_info);
    TOKUDB_HANDLER_DBUG_RETURN(error);
}

Sergei Golubchik's avatar
Sergei Golubchik committed
988 989
int ha_tokudb::optimize(THD* thd, HA_CHECK_OPT* check_opt) {
    TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name());
Sergei Golubchik's avatar
Sergei Golubchik committed
990 991 992 993 994 995 996 997 998 999
    int error;
#if TOKU_OPTIMIZE_WITH_RECREATE
    error = HA_ADMIN_TRY_ALTER;
#else
    error = do_optimize(thd);
#endif
    TOKUDB_HANDLER_DBUG_RETURN(error);
}

struct check_context {
Sergei Golubchik's avatar
Sergei Golubchik committed
1000
    THD* thd;
Sergei Golubchik's avatar
Sergei Golubchik committed
1001 1002
};

Sergei Golubchik's avatar
Sergei Golubchik committed
1003 1004
static int ha_tokudb_check_progress(void* extra, float progress) {
    struct check_context* context = (struct check_context*)extra;
Sergei Golubchik's avatar
Sergei Golubchik committed
1005
    int result = 0;
1006
    if (thd_kill_level(context->thd))
Sergei Golubchik's avatar
Sergei Golubchik committed
1007 1008 1009 1010
        result = ER_ABORTING_CONNECTION;
    return result;
}

Sergei Golubchik's avatar
Sergei Golubchik committed
1011
static void ha_tokudb_check_info(THD* thd, TABLE* table, const char* msg) {
Sergei Golubchik's avatar
Sergei Golubchik committed
1012
    if (thd->vio_ok()) {
Sergei Golubchik's avatar
Sergei Golubchik committed
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
        char tablename[
            table->s->db.length + 1 +
            table->s->table_name.length + 1];
        snprintf(
            tablename,
            sizeof(tablename),
            "%.*s.%.*s",
            (int)table->s->db.length,
            table->s->db.str,
            (int)table->s->table_name.length,
            table->s->table_name.str);
Sergei Golubchik's avatar
Sergei Golubchik committed
1024 1025 1026 1027 1028 1029 1030 1031 1032
        thd->protocol->prepare_for_resend();
        thd->protocol->store(tablename, strlen(tablename), system_charset_info);
        thd->protocol->store("check", 5, system_charset_info);
        thd->protocol->store("info", 4, system_charset_info);
        thd->protocol->store(msg, strlen(msg), system_charset_info);
        thd->protocol->write();
    }
}

Sergei Golubchik's avatar
Sergei Golubchik committed
1033 1034 1035
int ha_tokudb::check(THD* thd, HA_CHECK_OPT* check_opt) {
    TOKUDB_HANDLER_DBUG_ENTER("%s", share->table_name());
    const char* orig_proc_info = tokudb_thd_get_proc_info(thd);
Sergei Golubchik's avatar
Sergei Golubchik committed
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
    int result = HA_ADMIN_OK;
    int r;

    int keep_going = 1;
    if (check_opt->flags & T_QUICK) {
        keep_going = 0;
    }
    if (check_opt->flags & T_EXTEND) {
        keep_going = 1;
    }

    r = acquire_table_lock(transaction, lock_write);
    if (r != 0)
        result = HA_ADMIN_INTERNAL_ERROR;
    if (result == HA_ADMIN_OK) {
        uint32_t num_DBs = table_share->keys + tokudb_test(hidden_primary_key);
Sergei Golubchik's avatar
Sergei Golubchik committed
1052 1053 1054 1055 1056 1057 1058 1059
        snprintf(
            write_status_msg,
            sizeof(write_status_msg),
            "%s primary=%d num=%d",
            share->table_name(),
            primary_key,
            num_DBs);
        if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_CHECK))) {
Sergei Golubchik's avatar
Sergei Golubchik committed
1060 1061 1062
            ha_tokudb_check_info(thd, table, write_status_msg);
            time_t now = time(0);
            char timebuf[32];
Sergei Golubchik's avatar
Sergei Golubchik committed
1063 1064 1065 1066
            TOKUDB_HANDLER_TRACE(
                "%.24s %s",
                ctime_r(&now, timebuf),
                write_status_msg);
Sergei Golubchik's avatar
Sergei Golubchik committed
1067 1068
        }
        for (uint i = 0; i < num_DBs; i++) {
Sergei Golubchik's avatar
Sergei Golubchik committed
1069 1070
            DB* db = share->key_file[i];
            assert_always(db != NULL);
Sergei Golubchik's avatar
Sergei Golubchik committed
1071 1072 1073 1074 1075 1076 1077 1078 1079
            const char* kname =
                i == primary_key ? "primary" : table_share->key_info[i].name;
            snprintf(
                write_status_msg,
                sizeof(write_status_msg),
                "%s key=%s %u",
                share->table_name(),
                kname,
                i);
Sergei Golubchik's avatar
Sergei Golubchik committed
1080
            thd_proc_info(thd, write_status_msg);
Sergei Golubchik's avatar
Sergei Golubchik committed
1081
            if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_CHECK))) {
Sergei Golubchik's avatar
Sergei Golubchik committed
1082 1083 1084
                ha_tokudb_check_info(thd, table, write_status_msg);
                time_t now = time(0);
                char timebuf[32];
Sergei Golubchik's avatar
Sergei Golubchik committed
1085 1086 1087 1088
                TOKUDB_HANDLER_TRACE(
                    "%.24s %s",
                    ctime_r(&now, timebuf),
                    write_status_msg);
Sergei Golubchik's avatar
Sergei Golubchik committed
1089 1090
            }
            struct check_context check_context = { thd };
Sergei Golubchik's avatar
Sergei Golubchik committed
1091 1092 1093 1094 1095 1096
            r = db->verify_with_progress(
                db,
                ha_tokudb_check_progress,
                &check_context,
                (tokudb::sysvars::debug & TOKUDB_DEBUG_CHECK) != 0,
                keep_going);
Sergei Golubchik's avatar
Sergei Golubchik committed
1097 1098 1099 1100 1101
            if (r != 0) {
                char msg[32 + strlen(kname)];
                sprintf(msg, "Corrupt %s", kname);
                ha_tokudb_check_info(thd, table, msg);
            }
Sergei Golubchik's avatar
Sergei Golubchik committed
1102 1103 1104 1105 1106 1107 1108 1109
            snprintf(
                write_status_msg,
                sizeof(write_status_msg),
                "%s key=%s %u result=%d",
                share->full_table_name(),
                kname,
                i,
                r);
Sergei Golubchik's avatar
Sergei Golubchik committed
1110
            thd_proc_info(thd, write_status_msg);
Sergei Golubchik's avatar
Sergei Golubchik committed
1111
            if (TOKUDB_UNLIKELY(TOKUDB_DEBUG_FLAGS(TOKUDB_DEBUG_CHECK))) {
Sergei Golubchik's avatar
Sergei Golubchik committed
1112 1113 1114
                ha_tokudb_check_info(thd, table, write_status_msg);
                time_t now = time(0);
                char timebuf[32];
Sergei Golubchik's avatar
Sergei Golubchik committed
1115 1116 1117 1118
                TOKUDB_HANDLER_TRACE(
                    "%.24s %s",
                    ctime_r(&now, timebuf),
                    write_status_msg);
Sergei Golubchik's avatar
Sergei Golubchik committed
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
            }
            if (result == HA_ADMIN_OK && r != 0) {
                result = HA_ADMIN_CORRUPT;
                if (!keep_going)
                    break;
            }
        }
    }
    thd_proc_info(thd, orig_proc_info);
    TOKUDB_HANDLER_DBUG_RETURN(result);
}