Fix bug lp:894326

The patch also fixes an unrelated compiler warning.

Analysis:
The temporary table created during SJ-materialization
might be used for sorting for a group by operation. The
sort buffers for this internal temporary table were not
cleared by the execution code after each subquery
re-execution. This resulted in a memory leak detected
by valgrind.

Solution:
Cleanup the sort buffers for the semijon tables as well.


sql/item_subselect.cc:
  - Fix a compiler warning and add logic to revert to table
    scan partial match when there are more rows in the materialized
    subquery than there can be bits in the NULL bitmap index used
    for partial matching.
sql/opt_subselect.cc:
  - Fixed a memory leak detected by valgrind

sql/item_subselect.cc

@@ -4024,6 +4024,7 @@ ulonglong subselect_hash_sj_engine::rowid_merge_buff_size(
   uint rowid_length= tmp_table->file->ref_length;
   select_materialize_with_stats *result_sink=
     (select_materialize_with_stats *) result;
+  ha_rows max_null_row;
 
   /* Size of the subselect_rowid_merge_engine::row_num_to_rowid buffer. */
   buff_size= row_count * rowid_length * sizeof(uchar);
@@ -4046,7 +4047,18 @@ ulonglong subselect_hash_sj_engine::rowid_merge_buff_size(
       buff_size+= (row_count - result_sink->get_null_count_of_col(i)) *
                          sizeof(rownum_t);
       /* Add the size of Ordered_key::null_key */
-      buff_size+= bitmap_buffer_size(result_sink->get_max_null_of_col(i));
+      max_null_row= result_sink->get_max_null_of_col(i);
+      if (max_null_row >= UINT_MAX)
+      {
+        /*
+          There can be at most UINT_MAX bits in a MY_BITMAP that is used to
+          store NULLs in an Ordered_key. Return a number of bytes bigger than
+          the maximum allowed memory buffer for partial matching to disable
+          the rowid merge strategy.
+        */
+        return ULONGLONG_MAX;
+      }
+      buff_size+= bitmap_buffer_size(max_null_row);
     }
   }
 
@@ -5588,7 +5600,7 @@ exists_complementing_null_row(MY_BITMAP *keys_to_complement)
 
   return bitmap_exists_intersection((const MY_BITMAP**) null_bitmaps,
                                     count_null_keys,
-                                    highest_min_row, lowest_max_row);
+                                    (uint)highest_min_row, (uint)lowest_max_row);
 }
 
 

sql/opt_subselect.cc

@@ -4075,6 +4075,8 @@ int clear_sj_tmp_tables(JOIN *join)
   {
     if ((res= table->file->ha_delete_all_rows()))
       return res; /* purecov: inspected */
+   free_io_cache(table);
+   filesort_free_buffers(table,0);
   }
 
   SJ_MATERIALIZATION_INFO *sjm;