Fixes some issues in Firstmatch optimization

Allows FirstMatch to handle the case where the fanout of firstmatch tables
is already less than 1.
Also Fixes LooseScan strategy to set position->{records_init, records_out}
(They were set to 0 which also caused assertion failures)

Author: Sergei Petrunia <sergey@mariadb.com>
Reviewer: Monty

sql/opt_subselect.cc

@@ -3655,15 +3655,15 @@ bool Duplicate_weedout_picker::check_qep(JOIN *join,
 
       if (p->table->emb_sj_nest)
       {
-        sj_inner_fanout= COST_MULT(sj_inner_fanout, p->records_read);
+        sj_inner_fanout= COST_MULT(sj_inner_fanout, p->records_out);
         dups_removed_fanout |= p->table->table->map;
       }
       else
       {
+        sj_outer_fanout= COST_MULT(sj_outer_fanout, p->records_out);
         /* Ensure that table supports comparable rowids */
         DBUG_ASSERT(!(p->table->table->file->ha_table_flags() & HA_NON_COMPARABLE_ROWID));
 
-        sj_outer_fanout= COST_MULT(sj_outer_fanout, p->records_read);
         temptable_rec_size += p->table->table->file->ref_length;
       }
     }
@@ -3682,10 +3682,9 @@ bool Duplicate_weedout_picker::check_qep(JOIN *join,
                                                  sj_outer_fanout,
                                                  temptable_rec_size,
                                                  0, 0);
-    double prefix_record_count= join->positions[first_tab].prefix_record_count;
     double write_cost= (one_cost.create +
-                        prefix_record_count * sj_outer_fanout * one_cost.write);
-    double full_lookup_cost= (prefix_record_count * sj_outer_fanout *
+                        first_weedout_table_rec_count * sj_outer_fanout * one_cost.write);
+    double full_lookup_cost= (first_weedout_table_rec_count* sj_outer_fanout *
                               sj_inner_fanout * one_cost.lookup);
     *read_time= dups_cost + write_cost + full_lookup_cost;
     
@@ -3697,7 +3696,7 @@ bool Duplicate_weedout_picker::check_qep(JOIN *join,
       Json_writer_object trace(join->thd);
       trace.
         add("strategy", "DuplicateWeedout").
-        add("prefix_row_count", prefix_record_count).
+        add("prefix_row_count", first_weedout_table_rec_count).
         add("tmp_table_rows", sj_outer_fanout).
         add("sj_inner_fanout", sj_inner_fanout).
         add("rows", *record_count).

sql/opt_subselect.h

@@ -300,6 +300,7 @@ class Loose_scan_opt
     if (best_loose_scan_cost != DBL_MAX)
     {
       pos->records_read=    best_loose_scan_records;
+      pos->records_init= pos->records_out= pos->records_read;
       pos->key=             best_loose_scan_start_key;
       pos->cond_selectivity= 1.0;
       pos->loosescan_picker.loosescan_key=   best_loose_scan_key;

sql/sql_select.cc

@@ -18893,6 +18893,14 @@ void optimize_wo_join_buffering(JOIN *join, uint first_tab, uint last_tab,
   join->cur_sj_inner_tables= save_cur_sj_inner_tables;
 
   *reopt_cost= cost;
+  if (rec_count < *outer_rec_count)
+  {
+    /*
+      The tables inside the subquery produce smaller fanout than outer tables.
+      This can happen in edge cases.
+    */
+    *outer_rec_count= rec_count;
+  }
 }