MDEV-18144: ANALYZE for statement support for PK filters

ANALYZE and ANALYZE FORMAT=JSON structures are changed in the way that they
show additional information when rowid filter is used:

- r_selectivity_pct - the observed filter selectivity
- r_buffer_size - the size of the rowid filter container buffer
- r_filling_time_ms - how long it took to fill rowid filter container

New class Rowid_filter_tracker was added. This class is needed to collect data
about how rowid filter is executed.

mysql-test/include/analyze-format.inc

 # The time on ANALYSE FORMAT=JSON is rather variable
 
---replace_regex /("(r_total_time_ms|r_buffer_size)": )[^, \n]*/\1"REPLACED"/
+--replace_regex /("(r_total_time_ms|r_buffer_size|r_filling_time_ms)": )[^, \n]*/\1"REPLACED"/

mysql-test/main/rowid_filter.test

@@ -46,11 +46,17 @@ SELECT l_orderkey, l_linenumber, l_shipdate, l_quantity FROM lineitem
 
 eval $with_filter EXPLAIN $q1;
 eval $with_filter EXPLAIN FORMAT=JSON $q1;
+eval $with_filter ANALYZE $q1;
+--source include/analyze-format.inc
+eval $with_filter ANALYZE FORMAT=JSON $q1;
 --sorted_result
 eval $with_filter $q1;
 
 eval $without_filter EXPLAIN $q1;
 eval $without_filter EXPLAIN FORMAT=JSON $q1;
+eval $without_filter ANALYZE $q1;
+--source include/analyze-format.inc
+eval $without_filter ANALYZE FORMAT=JSON $q1;
 --sorted_result
 eval $without_filter $q1;
 
@@ -62,11 +68,17 @@ SELECT o_orderkey, l_linenumber, l_shipdate, o_totalprice
 
 eval $with_filter EXPLAIN $q2;
 eval $with_filter EXPLAIN FORMAT=JSON $q2;
+eval $with_filter ANALYZE $q2;
+--source include/analyze-format.inc
+eval $with_filter ANALYZE FORMAT=JSON $q2;
 --sorted_result
 eval $with_filter $q2;
 
 eval $without_filter EXPLAIN $q2;
 eval $without_filter EXPLAIN FORMAT=JSON $q2;
+eval $without_filter ANALYZE $q2;
+--source include/analyze-format.inc
+eval $without_filter ANALYZE FORMAT=JSON $q2;
 --sorted_result
 eval $without_filter $q2;
 
@@ -79,11 +91,17 @@ SELECT o_orderkey, l_linenumber, l_shipdate, l_quantity, o_totalprice
 
 eval $with_filter EXPLAIN $q3;
 eval $with_filter EXPLAIN FORMAT=JSON $q3;
+eval $with_filter ANALYZE $q3;
+--source include/analyze-format.inc
+eval $with_filter ANALYZE FORMAT=JSON $q3;
 --sorted_result
 eval $with_filter $q3;
 
 eval $without_filter EXPLAIN $q3;
 eval $without_filter EXPLAIN FORMAT=JSON $q3;
+eval $without_filter ANALYZE $q3;
+--source include/analyze-format.inc
+eval $without_filter ANALYZE FORMAT=JSON $q3;
 --sorted_result
 eval $without_filter $q3;
 
@@ -95,11 +113,17 @@ SELECT o_orderkey, l_linenumber, l_shipdate, o_totalprice
 
 eval $with_filter EXPLAIN $q4;
 eval $with_filter EXPLAIN FORMAT=JSON $q4;
+eval $with_filter ANALYZE $q4;
+--source include/analyze-format.inc
+eval $with_filter ANALYZE FORMAT=JSON $q4;
 --sorted_result
 eval $with_filter $q4;
 
 eval $without_filter EXPLAIN $q4;
 eval $without_filter EXPLAIN FORMAT=JSON $q4;
+eval $without_filter ANALYZE $q4;
+--source include/analyze-format.inc
+eval $without_filter ANALYZE FORMAT=JSON $q4;
 --sorted_result
 eval $without_filter $q4;
 

sql/handler.h

@@ -3014,6 +3014,7 @@ class handler :public Sql_alloc
   Exec_time_tracker *tracker;
 public:
   void set_time_tracker(Exec_time_tracker *tracker_arg) { tracker=tracker_arg;}
+  Exec_time_tracker *get_time_tracker() { return tracker; }
 
   Item *pushed_idx_cond;
   uint pushed_idx_cond_keyno;  /* The index which the above condition is for */

sql/rowid_filter.cc

@@ -478,6 +478,8 @@ bool Range_rowid_filter::fill()
       file->position(quick->record);
       if (container->add(NULL, (char*) file->ref))
         rc= 1;
+      else
+        tracker->increment_container_elements_count();
     }
   }
 
@@ -488,6 +490,7 @@ bool Range_rowid_filter::fill()
   file->pushed_idx_cond= pushed_idx_cond_save;
   file->pushed_idx_cond_keyno= pushed_idx_cond_keyno_save;
   file->in_range_check_pushed_down= in_range_check_pushed_down_save;
+  tracker->report_container_buff_size(table->file->ref_length);
 
   if (rc != HA_ERR_END_OF_FILE)
     return 1;

sql/rowid_filter.h

@@ -211,6 +211,8 @@ class Rowid_filter : public Sql_alloc
   /* The container to store info the set of elements in the filter */
   Rowid_filter_container *container;
 
+  Rowid_filter_tracker *tracker;
+
 public:
   Rowid_filter(Rowid_filter_container *container_arg)
     : container(container_arg) {}
@@ -230,6 +232,9 @@ class Rowid_filter : public Sql_alloc
   virtual ~Rowid_filter() {}
 
   Rowid_filter_container *get_container() { return container; }
+
+  void set_tracker(Rowid_filter_tracker *track_arg) { tracker= track_arg; }
+  Rowid_filter_tracker *get_tracker() { return tracker; }
 };
 
 
@@ -261,7 +266,12 @@ class Range_rowid_filter: public Rowid_filter
 
   bool build() { return fill(); }
 
-  bool check(char *elem) { return container->check(table, elem); }
+  bool check(char *elem)
+  {
+    bool was_checked= container->check(table, elem);
+    tracker->increment_checked_elements_count(was_checked);
+    return was_checked;
+  }
 
   bool fill();
 

sql/sql_analyze_stmt.h

@@ -284,3 +284,82 @@ class Filesort_tracker : public Sql_alloc
   ulonglong sort_buffer_size;
 };
 
+
+/**
+  A class to collect data about how rowid filter is executed.
+
+  It stores information about how rowid filter container is filled,
+  containers size and observed selectivity.
+
+  The observed selectivity is calculated in this way.
+  Some elements elem_set are checked if they belong to container.
+  Observed selectivity is calculated as the count of elem_set
+  elements that belong to container devided by all elem_set elements.
+*/
+
+class Rowid_filter_tracker : public Sql_alloc
+{
+private:
+  /* A member to track the time to fill the rowid filter */
+  Time_and_counter_tracker time_tracker;
+
+  /* Size of the rowid filter container buffer */
+  size_t container_buff_size;
+
+  /* Count of elements that were used to fill the rowid filter container */
+  uint container_elements;
+
+  /* Elements counts used for observed selectivity calculation */
+  uint n_checks;
+  uint n_positive_checks;
+public:
+  Rowid_filter_tracker(bool do_timing) :
+    time_tracker(do_timing), container_buff_size(0),
+    container_elements(0), n_checks(0), n_positive_checks(0)
+  {}
+
+  inline void start_tracking()
+  {
+    ANALYZE_START_TRACKING(&time_tracker);
+  }
+
+  inline void stop_tracking()
+  {
+    ANALYZE_STOP_TRACKING(&time_tracker);
+  }
+
+  /* Save container buffer size in bytes */
+  inline void report_container_buff_size(uint elem_size)
+  {
+   container_buff_size= container_elements * elem_size / 8;
+  }
+
+  Time_and_counter_tracker *get_time_tracker()
+  {
+    return &time_tracker;
+  }
+
+  double get_time_fill_container_ms()
+  {
+    return time_tracker.get_time_ms();
+  }
+
+  void increment_checked_elements_count(bool was_checked)
+  {
+    n_checks++;
+    if (was_checked)
+     n_positive_checks++;
+  }
+
+  inline void increment_container_elements_count() { container_elements++; }
+
+  uint get_container_elements() { return container_elements; }
+
+  double get_r_selectivity_pct()
+  {
+    return (double)n_positive_checks/(double)n_checks;
+  }
+
+  size_t get_container_buff_size() { return container_buff_size; }
+};
+

sql/sql_class.cc

@@ -2728,7 +2728,7 @@ void THD::make_explain_field_list(List<Item> &field_list, uint8 explain_flags,
   if (is_analyze)
   {
     field_list.push_back(item= new (mem_root)
-                         Item_float(this, "r_rows", 0.1234, 10, 4),
+                         Item_empty_string(this, "r_rows", NAME_CHAR_LEN, cs),
                          mem_root);
     item->maybe_null=1;
   }

sql/sql_explain.cc

@@ -1324,6 +1324,7 @@ int Explain_table_access::print_explain(select_result_sink *output, uint8 explai
     item_list.push_back(item_null, mem_root);
 
   /* `r_rows` */
+  StringBuffer<64> r_rows_str;
   if (is_analyze)
   {
     if (!tracker.has_scans())
@@ -1333,8 +1334,19 @@ int Explain_table_access::print_explain(select_result_sink *output, uint8 explai
     else
     {
       double avg_rows= tracker.get_avg_rows();
-      item_list.push_back(new (mem_root) Item_float(thd, avg_rows, 2),
-                          mem_root);
+      Item_float *fl= new (mem_root) Item_float(thd, avg_rows, 2);
+      String tmp;
+      String *res= fl->val_str(&tmp);
+      r_rows_str.append(res->ptr());
+      if (rowid_filter)
+      {
+        r_rows_str.append(" (");
+        r_rows_str.append_ulonglong(rowid_filter->tracker->get_r_selectivity_pct() * 100.0);
+        r_rows_str.append("%)");
+      }
+      item_list.push_back(new (mem_root)
+                          Item_string_sys(thd, r_rows_str.ptr(),
+                                          r_rows_str.length()), mem_root);
     }
   }
 
@@ -1404,7 +1416,7 @@ int Explain_table_access::print_explain(select_result_sink *output, uint8 explai
       first= false;
     else
       extra_buf.append(STRING_WITH_LEN("; "));
-    extra_buf.append(STRING_WITH_LEN("Using filter"));
+    extra_buf.append(STRING_WITH_LEN("Using rowid filter"));
   }
 
   item_list.push_back(new (mem_root)
@@ -1604,6 +1616,16 @@ void Explain_rowid_filter::print_explain_json(Explain_query *query,
   quick->print_json(writer);
   writer->add_member("rows").add_ll(rows);
   writer->add_member("selectivity_pct").add_double(selectivity * 100.0);
+  if (is_analyze)
+  {
+    writer->add_member("r_rows").add_double(tracker->get_container_elements());
+    writer->add_member("r_selectivity_pct").
+      add_double(tracker->get_r_selectivity_pct() * 100.0);
+    writer->add_member("r_buffer_size").
+      add_double(tracker->get_container_buff_size());
+    writer->add_member("r_filling_time_ms").
+      add_double(tracker->get_time_fill_container_ms());
+  }
   writer->end_object(); // rowid_filter
 }
 
@@ -1742,8 +1764,10 @@ void Explain_table_access::print_explain_json(Explain_query *query,
 
     if (op_tracker.get_loops())
     {
-      writer->add_member("r_total_time_ms").
-              add_double(op_tracker.get_time_ms());
+      double total_time= op_tracker.get_time_ms();
+      if (rowid_filter)
+        total_time+= rowid_filter->tracker->get_time_fill_container_ms();
+      writer->add_member("r_total_time_ms").add_double(total_time);
     }
   }
   

sql/sql_explain.h

@@ -624,6 +624,9 @@ class Explain_rowid_filter : public Sql_alloc
   /* Expected selectivity for the filter */
   double selectivity;
 
+  /* Tracker with the information about how rowid filter is executed */
+  Rowid_filter_tracker *tracker;
+
   void print_explain_json(Explain_query *query, Json_writer *writer,
                           bool is_analyze);
 

sql/sql_select.cc

@@ -12619,6 +12619,18 @@ void JOIN_TAB::build_range_rowid_filter_if_needed()
 {
   if (rowid_filter && !is_rowid_filter_built)
   {
+    /**
+      The same handler object (table->file) is used to build a filter
+      and to perfom a primary table access (by the main query).
+
+      To estimate the time for filter building tracker should be changed
+      and after building of the filter has been finished it should be
+      switched back to the previos tracker.
+    */
+    Exec_time_tracker *table_tracker= table->file->get_time_tracker();
+    Rowid_filter_tracker *rowid_tracker= rowid_filter->get_tracker();
+    table->file->set_time_tracker(rowid_tracker->get_time_tracker());
+    rowid_tracker->start_tracking();
     if (!rowid_filter->build())
     {
       is_rowid_filter_built= true;
@@ -12628,6 +12640,8 @@ void JOIN_TAB::build_range_rowid_filter_if_needed()
       delete rowid_filter;
       rowid_filter= 0;
     }
+    rowid_tracker->stop_tracking();
+    table->file->set_time_tracker(table_tracker);
   }
 }
 
@@ -25408,8 +25422,10 @@ bool JOIN_TAB::save_explain_data(Explain_table_access *eta,
     erf->quick= quick->get_explain(thd->mem_root);
     erf->selectivity= range_rowid_filter_info->selectivity;
     erf->rows= quick->records;
+    if (!(erf->tracker= new Rowid_filter_tracker(thd->lex->analyze_stmt)))
+      return 1;
+    rowid_filter->set_tracker(erf->tracker);
     eta->rowid_filter= erf;
-    //psergey-todo: also do setup for ANALYZE here.
   }
 
   if (tab_type == JT_NEXT)