MDEV-18880: Optimizer trace prints date in hexadecimal

Introduced a print_key_value function to makes sure that the trace prints data in readable format
for readable characters and the rest of the characters are printed as hexadecimal.

mysql-test/main/opt_trace.test

@@ -444,4 +444,58 @@ select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) fr
 
 drop table t1;
 
+--echo #
+--echo # MDEV-18880: Optimizer trace prints date in hexadecimal
+--echo #
+
+CREATE TABLE t1(i INT PRIMARY KEY, b VARCHAR(10) CHARSET BINARY , INDEX i_b(b));
+INSERT INTO t1 VALUES (1, 'ab\n');
+INSERT INTO t1 VALUES (2, NULL);
+set optimizer_trace=1;
+EXPLAIN SELECT * FROM t1 WHERE b='ab\n';
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+
+ALTER TABLE t1 modify column b BINARY(10) AFTER i;
+EXPLAIN SELECT * FROM t1 WHERE b='ab\n';
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+
+ALTER TABLE t1 modify column b VARBINARY(10) AFTER i;
+EXPLAIN SELECT * FROM t1 WHERE b='ab\n';
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+drop table t1;
+
+CREATE TABLE t1(i INT PRIMARY KEY, b CHAR(10), INDEX i_b(b));
+INSERT INTO t1 VALUES (1, 'ab\n');
+INSERT INTO t1 VALUES (2, NULL);
+EXPLAIN SELECT * FROM t1 WHERE b='ab\n';
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+drop table t1;
+
+CREATE TABLE t1(i INT PRIMARY KEY, b blob , INDEX i_b(b));
+INSERT INTO t1 VALUES (1, 'ab\n');
+INSERT INTO t1 VALUES (2, NULL);
+set optimizer_trace=1;
+EXPLAIN SELECT * FROM t1 WHERE b= 'ab\n';
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+drop table t1;
+
+CREATE TABLE t1(i INT PRIMARY KEY, b VARCHAR(10), INDEX i_b(b));
+INSERT INTO t1 VALUES (1, 'ab\n');
+INSERT INTO t1 VALUES (2, 'ab\n');
+set optimizer_trace=1;
+EXPLAIN SELECT * FROM t1 WHERE b='ab\n';
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+drop table t1;
+create table t0(a int);
+insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
+create table one_k (a int);
+insert into one_k select A.a + B.a*10 + C.a*100 from t0 A, t0 B, t0 C;
+create table t1 (start_date date, end_date date, filler char(100), key(start_date, end_date)) ;
+--disable_warnings
+insert into t1 select date_add(now(), interval a day), date_add(now(), interval (a+7) day), 'data' from one_k;
+--enable_warnings
+explain format=json select * from t1 force index(start_date) where start_date >= '2019-02-10' and end_date <'2019-04-01';
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+drop table t1, t0, one_k;
+
 set optimizer_trace='enabled=off';

mysql-test/main/opt_trace_ucs2.result

+create or replace table t1 (col1 char(10) character set ucs2, filler char(100), key(col1)) ;
+insert into t1 values ('a', 'a');
+insert into t1 values ('a', 'a');
+set optimizer_trace=1;
+explain format=json select * from t1 force index(col1) where col1 >='a';
+EXPLAIN
+{
+  "query_block": {
+    "select_id": 1,
+    "table": {
+      "table_name": "t1",
+      "access_type": "range",
+      "possible_keys": ["col1"],
+      "key": "col1",
+      "key_length": "21",
+      "used_key_parts": ["col1"],
+      "rows": 2,
+      "filtered": 100,
+      "index_condition": "t1.col1 >= 'a'"
+    }
+  }
+}
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives'))
+[
+    
+    {
+        "range_scan_alternatives": 
+        [
+            
+            {
+                "index": "col1",
+                "ranges": 
+                [
+                    "(a) <= (col1)"
+                ],
+                "rowid_ordered": false,
+                "using_mrr": false,
+                "index_only": false,
+                "rows": 2,
+                "cost": 3.7609,
+                "chosen": true
+            }
+        ],
+        "analyzing_roworder_intersect": 
+        {
+            "cause": "too few roworder scans"
+        },
+        "analyzing_index_merge_union": 
+        [
+        ]
+    }
+]
+drop table t1;

mysql-test/main/opt_trace_ucs2.test

+--source include/not_embedded.inc
+--source include/have_ucs2.inc
+
+create or replace table t1 (col1 char(10) character set ucs2, filler char(100), key(col1)) ;
+insert into t1 values ('a', 'a');
+insert into t1 values ('a', 'a');
+set optimizer_trace=1;
+explain format=json select * from t1 force index(col1) where col1 >='a';
+select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+drop table t1;

sql/field.cc

@@ -11242,3 +11242,58 @@ bool Field::val_str_nopad(MEM_ROOT *mem_root, LEX_CSTRING *to)
   thd->variables.sql_mode= sql_mode_backup;
   return rc;
 }
+
+
+void Field::print_key_value(String *out, uint32 length)
+{
+  if (charset() == &my_charset_bin)
+    print_key_value_binary(out, ptr, length);
+  else
+    val_str(out);
+}
+
+
+void Field_string::print_key_value(String *out, uint32 length)
+{
+  if (charset() == &my_charset_bin)
+  {
+    size_t len= field_charset->cset->lengthsp(field_charset, (const char*) ptr, length);
+    print_key_value_binary(out, ptr, static_cast<uint32>(len));
+  }
+  else
+  {
+    THD *thd= get_thd();
+    sql_mode_t sql_mode_backup= thd->variables.sql_mode;
+    thd->variables.sql_mode&= ~MODE_PAD_CHAR_TO_FULL_LENGTH;
+    val_str(out,out);
+    thd->variables.sql_mode= sql_mode_backup;
+  }
+}
+
+
+void Field_varstring::print_key_value(String *out, uint32 length)
+{
+  if (charset() == &my_charset_bin)
+    print_key_value_binary(out, get_data(), get_length());
+  else
+    val_str(out,out);
+}
+
+
+void Field_blob::print_key_value(String *out, uint32 length)
+{
+  if (charset() == &my_charset_bin)
+  {
+    uchar *blob;
+    memcpy(&blob, ptr+packlength, sizeof(uchar*));
+    print_key_value_binary(out, blob, get_length());
+  }
+  else
+    val_str(out, out);
+}
+
+
+void Field::print_key_value_binary(String *out, const uchar* key, uint32 length)
+{
+  out->append_semi_hex((const char*)key, length, charset());
+}
\ No newline at end of file

sql/field.h

@@ -1383,6 +1383,8 @@ class Field: public Value_source
   virtual int set_time() { return 1; }
   bool set_warning(Sql_condition::enum_warning_level, unsigned int code,
                    int cuted_increment, ulong current_row=0) const;
+  virtual void print_key_value(String *out, uint32 length);
+  void print_key_value_binary(String *out, const uchar* key, uint32 length);
 protected:
   bool set_warning(unsigned int code, int cuted_increment) const
   {
@@ -3592,6 +3594,7 @@ class Field_string :public Field_longstr {
   { return charset() == &my_charset_bin ? FALSE : TRUE; }
   Field *make_new_field(MEM_ROOT *root, TABLE *new_table, bool keep_type);
   virtual uint get_key_image(uchar *buff,uint length, imagetype type);
+  void print_key_value(String *out, uint32 length);
 private:
   int save_field_metadata(uchar *first_byte);
 };
@@ -3697,6 +3700,7 @@ class Field_varstring :public Field_longstr {
   uint is_equal(Create_field *new_field);
   void hash(ulong *nr, ulong *nr2);
   uint length_size() { return length_bytes; }
+  void print_key_value(String *out, uint32 length);
 private:
   int save_field_metadata(uchar *first_byte);
 };
@@ -4035,6 +4039,7 @@ class Field_blob :public Field_longstr {
   uint32 char_length() const;
   uint32 character_octet_length() const;
   uint is_equal(Create_field *new_field);
+  void print_key_value(String *out, uint32 length);
 
   friend void TABLE::remember_blob_values(String *blob_storage);
   friend void TABLE::restore_blob_values(String *blob_storage);
@@ -4159,6 +4164,10 @@ class Field_geom :public Field_blob {
   geometry_type get_geometry_type() { return geom_type; };
   static geometry_type geometry_type_merge(geometry_type, geometry_type);
   uint get_srid() { return srid; }
+  void print_key_value(String *out, uint32 length)
+  {
+    out->append(STRING_WITH_LEN("unprintable_geometry_value"));
+  }
 };
 
 uint gis_field_options_image(uchar *buff, List<Create_field> &create_fields);
@@ -4466,6 +4475,11 @@ class Field_bit :public Field {
   {
     return get_mm_leaf_int(param, key_part, cond, op, value, true);
   }
+  void print_key_value(String *out, uint32 length)
+  {
+    val_int_as_str(out, 1);
+  }
+
 private:
   virtual size_t do_last_null_byte() const;
   int save_field_metadata(uchar *first_byte);

sql/opt_range.cc

@@ -15835,13 +15835,11 @@ static void print_key_value(String *out, const KEY_PART_INFO *key_part,
     {
       // Byte 0 of a nullable key is the null-byte. If set, key is NULL.
       if (field->real_maybe_null() && *key)
+      {
         out->append(STRING_WITH_LEN("NULL"));
-      else
-        (field->type() == MYSQL_TYPE_GEOMETRY)
-            ? out->append(STRING_WITH_LEN("unprintable_geometry_value"))
-            : out->append(STRING_WITH_LEN("unprintable_blob_value"));
         goto next;
       }
+    }
 
     if (field->real_maybe_null())
     {
@@ -15859,28 +15857,12 @@ static void print_key_value(String *out, const KEY_PART_INFO *key_part,
       store_length--;
     }
 
-    /*
-      Binary data cannot be converted to UTF8 which is what the
-      optimizer trace expects. If the column is binary, the hex
-      representation is printed to the trace instead.
-    */
-    if (field->flags & BINARY_FLAG)
-    {
-      out->append("0x");
-      for (uint i = 0; i < store_length; i++)
-      {
-        out->append(_dig_vec_lower[*(key + i) >> 4]);
-        out->append(_dig_vec_lower[*(key + i) & 0x0F]);
-      }
-      goto next;
-    }
-
     field->set_key_image(key, key_part->length);
-    if (field->type() == MYSQL_TYPE_BIT)
-      (void)field->val_int_as_str(&tmp, 1);  // may change tmp's charset
-    else
-      field->val_str(&tmp);  // may change tmp's charset
+    field->print_key_value(&tmp, key_part->length);
+    if (field->charset() == &my_charset_bin)
       out->append(tmp.ptr(), tmp.length(), tmp.charset());
+    else
+      tmp.print(out, system_charset_info);
 
   next:
     if (key + store_length < key_end)

sql/sql_string.cc

@@ -1196,3 +1196,15 @@ uint convert_to_printable(char *to, size_t to_len,
     *t= '\0';
   return (uint) (t - to);
 }
+
+
+bool String::append_semi_hex(const char *s, uint len, CHARSET_INFO *cs)
+{
+  size_t dst_len= len * 4 + 1; //extra length for the '\0' character
+  if (reserve(dst_len))
+    return true;
+  uint nbytes= convert_to_printable(Ptr + str_length, dst_len, s, len, cs);
+  DBUG_ASSERT((ulonglong) str_length + nbytes < UINT_MAX32);
+  str_length+= nbytes;
+  return false;
+}

sql/sql_string.h

@@ -674,6 +674,7 @@ class Binary_string: public Static_binary_string
 
   int reserve(size_t space_needed)
   {
+    DBUG_ASSERT((ulonglong) str_length + space_needed < UINT_MAX32);
     return realloc(str_length + space_needed);
   }
   int reserve(size_t space_needed, size_t grow_by);
@@ -959,6 +960,8 @@ class String: public Charset, public Binary_string
   {
     return !sortcmp(this, other, cs);
   }
+private:
+  bool append_semi_hex(const char *s, uint len, CHARSET_INFO *cs);
 };