rdb_perf_context.cc 9.1 KB
Newer Older
Sergei Petrunia's avatar
Sergei Petrunia committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
/*
   Portions Copyright (c) 2015-Present, Facebook, Inc.
   Portions Copyright (c) 2012, Monty Program Ab

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

/* This C++ file's header file */
#include "./rdb_perf_context.h"

/* C++ system header files */
#include <string>

/* RocksDB header files */
#include "rocksdb/iostats_context.h"
#include "rocksdb/perf_context.h"

/* MyRocks header files */
#include "./ha_rocksdb_proto.h"

namespace myrocks {

// To add a new metric:
//   1. Update the PC enum in rdb_perf_context.h
//   2. Update sections (A), (B), and (C) below
//   3. Update perf_context.test and show_engine.test

Sergei Petrunia's avatar
Sergei Petrunia committed
38 39 40 41 42 43 44 45 46
std::string rdb_pc_stat_types[] = {
    // (A) These should be in the same order as the PC enum
    "USER_KEY_COMPARISON_COUNT",
    "BLOCK_CACHE_HIT_COUNT",
    "BLOCK_READ_COUNT",
    "BLOCK_READ_BYTE",
    "BLOCK_READ_TIME",
    "BLOCK_CHECKSUM_TIME",
    "BLOCK_DECOMPRESS_TIME",
Sergei Petrunia's avatar
Sergei Petrunia committed
47 48 49
    "GET_READ_BYTES",
    "MULTIGET_READ_BYTES",
    "ITER_READ_BYTES",
Sergei Petrunia's avatar
Sergei Petrunia committed
50 51
    "INTERNAL_KEY_SKIPPED_COUNT",
    "INTERNAL_DELETE_SKIPPED_COUNT",
Sergei Petrunia's avatar
Sergei Petrunia committed
52 53
    "INTERNAL_RECENT_SKIPPED_COUNT",
    "INTERNAL_MERGE_COUNT",
Sergei Petrunia's avatar
Sergei Petrunia committed
54 55 56 57 58 59 60
    "GET_SNAPSHOT_TIME",
    "GET_FROM_MEMTABLE_TIME",
    "GET_FROM_MEMTABLE_COUNT",
    "GET_POST_PROCESS_TIME",
    "GET_FROM_OUTPUT_FILES_TIME",
    "SEEK_ON_MEMTABLE_TIME",
    "SEEK_ON_MEMTABLE_COUNT",
Sergei Petrunia's avatar
Sergei Petrunia committed
61 62
    "NEXT_ON_MEMTABLE_COUNT",
    "PREV_ON_MEMTABLE_COUNT",
Sergei Petrunia's avatar
Sergei Petrunia committed
63 64
    "SEEK_CHILD_SEEK_TIME",
    "SEEK_CHILD_SEEK_COUNT",
Sergei Petrunia's avatar
Sergei Petrunia committed
65 66
    "SEEK_MIN_HEAP_TIME",
    "SEEK_MAX_HEAP_TIME",
Sergei Petrunia's avatar
Sergei Petrunia committed
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
    "SEEK_INTERNAL_SEEK_TIME",
    "FIND_NEXT_USER_ENTRY_TIME",
    "WRITE_WAL_TIME",
    "WRITE_MEMTABLE_TIME",
    "WRITE_DELAY_TIME",
    "WRITE_PRE_AND_POST_PROCESS_TIME",
    "DB_MUTEX_LOCK_NANOS",
    "DB_CONDITION_WAIT_NANOS",
    "MERGE_OPERATOR_TIME_NANOS",
    "READ_INDEX_BLOCK_NANOS",
    "READ_FILTER_BLOCK_NANOS",
    "NEW_TABLE_BLOCK_ITER_NANOS",
    "NEW_TABLE_ITERATOR_NANOS",
    "BLOCK_SEEK_NANOS",
    "FIND_TABLE_NANOS",
Sergei Petrunia's avatar
Sergei Petrunia committed
82 83 84 85 86 87
    "BLOOM_MEMTABLE_HIT_COUNT",
    "BLOOM_MEMTABLE_MISS_COUNT",
    "BLOOM_SST_HIT_COUNT",
    "BLOOM_SST_MISS_COUNT",
    "KEY_LOCK_WAIT_TIME",
    "KEY_LOCK_WAIT_COUNT",
Sergei Petrunia's avatar
Sergei Petrunia committed
88 89 90 91 92 93 94 95 96 97 98 99
    "IO_THREAD_POOL_ID",
    "IO_BYTES_WRITTEN",
    "IO_BYTES_READ",
    "IO_OPEN_NANOS",
    "IO_ALLOCATE_NANOS",
    "IO_WRITE_NANOS",
    "IO_READ_NANOS",
    "IO_RANGE_SYNC_NANOS",
    "IO_LOGGER_NANOS"};

#define IO_PERF_RECORD(_field_)                                                \
  do {                                                                         \
Sergei Petrunia's avatar
Sergei Petrunia committed
100 101
    if (rocksdb::get_perf_context()->_field_ > 0)                              \
      counters->m_value[idx] += rocksdb::get_perf_context()->_field_;          \
Sergei Petrunia's avatar
Sergei Petrunia committed
102
    idx++;                                                                     \
Sergei Petrunia's avatar
Sergei Petrunia committed
103
  } while (0)
Sergei Petrunia's avatar
Sergei Petrunia committed
104 105
#define IO_STAT_RECORD(_field_)                                                \
  do {                                                                         \
Sergei Petrunia's avatar
Sergei Petrunia committed
106 107
    if (rocksdb::get_iostats_context()->_field_ > 0)                           \
      counters->m_value[idx] += rocksdb::get_iostats_context()->_field_;       \
Sergei Petrunia's avatar
Sergei Petrunia committed
108
    idx++;                                                                     \
Sergei Petrunia's avatar
Sergei Petrunia committed
109 110
  } while (0)

Sergei Petrunia's avatar
Sergei Petrunia committed
111
static void harvest_diffs(Rdb_atomic_perf_counters *const counters) {
Sergei Petrunia's avatar
Sergei Petrunia committed
112
  // (C) These should be in the same order as the PC enum
Sergei Petrunia's avatar
Sergei Petrunia committed
113
  size_t idx = 0;
Sergei Petrunia's avatar
Sergei Petrunia committed
114 115 116 117 118 119 120
  IO_PERF_RECORD(user_key_comparison_count);
  IO_PERF_RECORD(block_cache_hit_count);
  IO_PERF_RECORD(block_read_count);
  IO_PERF_RECORD(block_read_byte);
  IO_PERF_RECORD(block_read_time);
  IO_PERF_RECORD(block_checksum_time);
  IO_PERF_RECORD(block_decompress_time);
Sergei Petrunia's avatar
Sergei Petrunia committed
121 122 123
  IO_PERF_RECORD(get_read_bytes);
  IO_PERF_RECORD(multiget_read_bytes);
  IO_PERF_RECORD(iter_read_bytes);
Sergei Petrunia's avatar
Sergei Petrunia committed
124 125
  IO_PERF_RECORD(internal_key_skipped_count);
  IO_PERF_RECORD(internal_delete_skipped_count);
Sergei Petrunia's avatar
Sergei Petrunia committed
126 127
  IO_PERF_RECORD(internal_recent_skipped_count);
  IO_PERF_RECORD(internal_merge_count);
Sergei Petrunia's avatar
Sergei Petrunia committed
128 129 130 131 132 133 134
  IO_PERF_RECORD(get_snapshot_time);
  IO_PERF_RECORD(get_from_memtable_time);
  IO_PERF_RECORD(get_from_memtable_count);
  IO_PERF_RECORD(get_post_process_time);
  IO_PERF_RECORD(get_from_output_files_time);
  IO_PERF_RECORD(seek_on_memtable_time);
  IO_PERF_RECORD(seek_on_memtable_count);
Sergei Petrunia's avatar
Sergei Petrunia committed
135 136
  IO_PERF_RECORD(next_on_memtable_count);
  IO_PERF_RECORD(prev_on_memtable_count);
Sergei Petrunia's avatar
Sergei Petrunia committed
137 138 139
  IO_PERF_RECORD(seek_child_seek_time);
  IO_PERF_RECORD(seek_child_seek_count);
  IO_PERF_RECORD(seek_min_heap_time);
Sergei Petrunia's avatar
Sergei Petrunia committed
140
  IO_PERF_RECORD(seek_max_heap_time);
Sergei Petrunia's avatar
Sergei Petrunia committed
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
  IO_PERF_RECORD(seek_internal_seek_time);
  IO_PERF_RECORD(find_next_user_entry_time);
  IO_PERF_RECORD(write_wal_time);
  IO_PERF_RECORD(write_memtable_time);
  IO_PERF_RECORD(write_delay_time);
  IO_PERF_RECORD(write_pre_and_post_process_time);
  IO_PERF_RECORD(db_mutex_lock_nanos);
  IO_PERF_RECORD(db_condition_wait_nanos);
  IO_PERF_RECORD(merge_operator_time_nanos);
  IO_PERF_RECORD(read_index_block_nanos);
  IO_PERF_RECORD(read_filter_block_nanos);
  IO_PERF_RECORD(new_table_block_iter_nanos);
  IO_PERF_RECORD(new_table_iterator_nanos);
  IO_PERF_RECORD(block_seek_nanos);
  IO_PERF_RECORD(find_table_nanos);
Sergei Petrunia's avatar
Sergei Petrunia committed
156 157 158 159 160 161 162
  IO_PERF_RECORD(bloom_memtable_hit_count);
  IO_PERF_RECORD(bloom_memtable_miss_count);
  IO_PERF_RECORD(bloom_sst_hit_count);
  IO_PERF_RECORD(bloom_sst_miss_count);
  IO_PERF_RECORD(key_lock_wait_time);
  IO_PERF_RECORD(key_lock_wait_count);

Sergei Petrunia's avatar
Sergei Petrunia committed
163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178
  IO_STAT_RECORD(thread_pool_id);
  IO_STAT_RECORD(bytes_written);
  IO_STAT_RECORD(bytes_read);
  IO_STAT_RECORD(open_nanos);
  IO_STAT_RECORD(allocate_nanos);
  IO_STAT_RECORD(write_nanos);
  IO_STAT_RECORD(read_nanos);
  IO_STAT_RECORD(range_sync_nanos);
  IO_STAT_RECORD(logger_nanos);
}

#undef IO_PERF_DIFF
#undef IO_STAT_DIFF

static Rdb_atomic_perf_counters rdb_global_perf_counters;

Sergei Petrunia's avatar
Sergei Petrunia committed
179
void rdb_get_global_perf_counters(Rdb_perf_counters *const counters) {
Sergei Petrunia's avatar
Sergei Petrunia committed
180 181 182 183 184
  DBUG_ASSERT(counters != nullptr);

  counters->load(rdb_global_perf_counters);
}

Sergei Petrunia's avatar
Sergei Petrunia committed
185 186 187
void Rdb_perf_counters::load(const Rdb_atomic_perf_counters &atomic_counters) {
  for (int i = 0; i < PC_MAX_IDX; i++) {
    m_value[i] = atomic_counters.m_value[i].load(std::memory_order_relaxed);
Sergei Petrunia's avatar
Sergei Petrunia committed
188 189 190
  }
}

Sergei Petrunia's avatar
Sergei Petrunia committed
191 192 193
bool Rdb_io_perf::start(const uint32_t perf_context_level) {
  const rocksdb::PerfLevel perf_level =
      static_cast<rocksdb::PerfLevel>(perf_context_level);
Sergei Petrunia's avatar
Sergei Petrunia committed
194

Sergei Petrunia's avatar
Sergei Petrunia committed
195
  if (rocksdb::GetPerfLevel() != perf_level) {
Sergei Petrunia's avatar
Sergei Petrunia committed
196 197 198
    rocksdb::SetPerfLevel(perf_level);
  }

Sergei Petrunia's avatar
Sergei Petrunia committed
199
  if (perf_level == rocksdb::kDisable) {
Sergei Petrunia's avatar
Sergei Petrunia committed
200 201 202
    return false;
  }

Sergei Petrunia's avatar
Sergei Petrunia committed
203 204
  rocksdb::get_perf_context()->Reset();
  rocksdb::get_iostats_context()->Reset();
Sergei Petrunia's avatar
Sergei Petrunia committed
205 206 207
  return true;
}

Sergei Petrunia's avatar
Sergei Petrunia committed
208 209 210 211 212 213 214 215 216 217
void Rdb_io_perf::update_bytes_written(const uint32_t perf_context_level,
                                       ulonglong bytes_written) {
  const rocksdb::PerfLevel perf_level =
      static_cast<rocksdb::PerfLevel>(perf_context_level);
  if (perf_level != rocksdb::kDisable && m_shared_io_perf_write) {
    io_write_bytes += bytes_written;
    io_write_requests += 1;
  }
}

Sergei Petrunia's avatar
Sergei Petrunia committed
218 219 220
void Rdb_io_perf::end_and_record(const uint32_t perf_context_level) {
  const rocksdb::PerfLevel perf_level =
      static_cast<rocksdb::PerfLevel>(perf_context_level);
Sergei Petrunia's avatar
Sergei Petrunia committed
221

Sergei Petrunia's avatar
Sergei Petrunia committed
222
  if (perf_level == rocksdb::kDisable) {
Sergei Petrunia's avatar
Sergei Petrunia committed
223 224 225
    return;
  }

Sergei Petrunia's avatar
Sergei Petrunia committed
226
  if (m_atomic_counters) {
Sergei Petrunia's avatar
Sergei Petrunia committed
227 228 229 230
    harvest_diffs(m_atomic_counters);
  }
  harvest_diffs(&rdb_global_perf_counters);

Sergei Petrunia's avatar
Sergei Petrunia committed
231 232 233 234
  if (m_shared_io_perf_read &&
      (rocksdb::get_perf_context()->block_read_byte != 0 ||
       rocksdb::get_perf_context()->block_read_count != 0 ||
       rocksdb::get_perf_context()->block_read_time != 0)) {
Sergei Petrunia's avatar
Sergei Petrunia committed
235 236
    my_io_perf_t io_perf_read;

Sergei Petrunia's avatar
Sergei Petrunia committed
237
    io_perf_read.init();
Sergei Petrunia's avatar
Sergei Petrunia committed
238 239
    io_perf_read.bytes = rocksdb::get_perf_context()->block_read_byte;
    io_perf_read.requests = rocksdb::get_perf_context()->block_read_count;
Sergei Petrunia's avatar
Sergei Petrunia committed
240 241 242 243 244

    /*
      Rocksdb does not distinguish between I/O service and wait time, so just
      use svc time.
     */
Sergei Petrunia's avatar
Sergei Petrunia committed
245
    io_perf_read.svc_time_max = io_perf_read.svc_time =
Sergei Petrunia's avatar
Sergei Petrunia committed
246
        rocksdb::get_perf_context()->block_read_time;
Sergei Petrunia's avatar
Sergei Petrunia committed
247

Sergei Petrunia's avatar
Sergei Petrunia committed
248 249
    m_shared_io_perf_read->sum(io_perf_read);
    m_stats->table_io_perf_read.sum(io_perf_read);
Sergei Petrunia's avatar
Sergei Petrunia committed
250 251
  }

Sergei Petrunia's avatar
Sergei Petrunia committed
252 253 254 255 256 257 258 259 260 261 262 263
  if (m_shared_io_perf_write &&
      (io_write_bytes != 0 || io_write_requests != 0)) {
    my_io_perf_t io_perf_write;
    io_perf_write.init();
    io_perf_write.bytes = io_write_bytes;
    io_perf_write.requests = io_write_requests;
    m_shared_io_perf_write->sum(io_perf_write);
    m_stats->table_io_perf_write.sum(io_perf_write);
    io_write_bytes = 0;
    io_write_requests = 0;
  }

Sergei Petrunia's avatar
Sergei Petrunia committed
264
  if (m_stats) {
Sergei Petrunia's avatar
Sergei Petrunia committed
265 266 267
    if (rocksdb::get_perf_context()->internal_key_skipped_count != 0) {
      m_stats->key_skipped +=
          rocksdb::get_perf_context()->internal_key_skipped_count;
Sergei Petrunia's avatar
Sergei Petrunia committed
268 269
    }

Sergei Petrunia's avatar
Sergei Petrunia committed
270
    if (rocksdb::get_perf_context()->internal_delete_skipped_count != 0) {
Sergei Petrunia's avatar
Sergei Petrunia committed
271
      m_stats->delete_skipped +=
Sergei Petrunia's avatar
Sergei Petrunia committed
272
          rocksdb::get_perf_context()->internal_delete_skipped_count;
Sergei Petrunia's avatar
Sergei Petrunia committed
273 274 275 276
    }
  }
}

Sergei Petrunia's avatar
Sergei Petrunia committed
277
} // namespace myrocks