merge of bug #37943 5.0-bugteam -> 5.1-bugteam

mysql-test/include/index_merge1.inc

@@ -487,3 +487,18 @@ select * from t2 where a=4 or b=4;
 
 drop table t1, t2;
 
+#
+# Bug #37943: Reproducible mysqld crash/sigsegv in sel_trees_can_be_ored
+#
+
+CREATE TABLE t1 (a varchar(8), b set('a','b','c','d','e','f','g','h'),
+                 KEY b(b), KEY a(a));
+INSERT INTO t1 VALUES ('y',''), ('z','');
+
+#should not crash
+SELECT b,a from t1 WHERE (b!='c' AND b!='f' && b!='h') OR 
+  (a='pure-S') OR (a='DE80337a') OR (a='DE80799');
+
+DROP TABLE t1;
+
+--echo End of 5.0 tests

mysql-test/r/index_merge_myisam.result

@@ -520,6 +520,16 @@ a	filler	b
 4	zz	4
 5	qq	4
 drop table t1, t2;
+CREATE TABLE t1 (a varchar(8), b set('a','b','c','d','e','f','g','h'),
+KEY b(b), KEY a(a));
+INSERT INTO t1 VALUES ('y',''), ('z','');
+SELECT b,a from t1 WHERE (b!='c' AND b!='f' && b!='h') OR 
+(a='pure-S') OR (a='DE80337a') OR (a='DE80799');
+b	a
+	y
+	z
+DROP TABLE t1;
+End of 5.0 tests
 #---------------- ROR-index_merge tests -----------------------
 SET SESSION STORAGE_ENGINE = MyISAM;
 drop table if exists  t0,t1,t2;

sql/opt_range.cc

@@ -574,6 +574,7 @@ public:
     keys_map.clear_all();
     bzero((char*) keys,sizeof(keys));
   }
+  SEL_TREE(SEL_TREE *arg, RANGE_OPT_PARAM *param);
   /*
     Note: there may exist SEL_TREE objects with sel_tree->type=KEY and
     keys[i]=0 for all i. (SergeyP: it is not clear whether there is any
@@ -767,6 +768,7 @@ public:
     trees_next(trees),
     trees_end(trees + PREALLOCED_TREES)
   {}
+  SEL_IMERGE (SEL_IMERGE *arg, RANGE_OPT_PARAM *param);
   int or_sel_tree(RANGE_OPT_PARAM *param, SEL_TREE *tree);
   int or_sel_tree_with_checks(RANGE_OPT_PARAM *param, SEL_TREE *new_tree);
   int or_sel_imerge_with_checks(RANGE_OPT_PARAM *param, SEL_IMERGE* imerge);
@@ -883,6 +885,61 @@ int SEL_IMERGE::or_sel_imerge_with_checks(RANGE_OPT_PARAM *param, SEL_IMERGE* im
 }
 
 
+SEL_TREE::SEL_TREE(SEL_TREE *arg, RANGE_OPT_PARAM *param): Sql_alloc()
+{
+  keys_map= arg->keys_map;
+  type= arg->type;
+  for (int idx= 0; idx < MAX_KEY; idx++)
+  {
+    if ((keys[idx]= arg->keys[idx]))
+      keys[idx]->increment_use_count(1);
+  }
+
+  List_iterator<SEL_IMERGE> it(arg->merges);
+  for (SEL_IMERGE *el= it++; el; el= it++)
+  {
+    SEL_IMERGE *merge= new SEL_IMERGE(el, param);
+    if (!merge || merge->trees == merge->trees_next)
+    {
+      merges.empty();
+      return;
+    }
+    merges.push_back (merge);
+  }
+}
+
+
+SEL_IMERGE::SEL_IMERGE (SEL_IMERGE *arg, RANGE_OPT_PARAM *param) : Sql_alloc()
+{
+  uint elements= (arg->trees_end - arg->trees);
+  if (elements > PREALLOCED_TREES)
+  {
+    uint size= elements * sizeof (SEL_TREE **);
+    if (!(trees= (SEL_TREE **)alloc_root(param->mem_root, size)))
+      goto mem_err;
+  }
+  else
+    trees= &trees_prealloced[0];
+
+  trees_next= trees;
+  trees_end= trees + elements;
+
+  for (SEL_TREE **tree = trees, **arg_tree= arg->trees; tree < trees_end; 
+       tree++, arg_tree++)
+  {
+    if (!(*tree= new SEL_TREE(*arg_tree, param)))
+      goto mem_err;
+  }
+
+  return;
+
+mem_err:
+  trees= &trees_prealloced[0];
+  trees_next= trees;
+  trees_end= trees;
+}
+
+
 /*
   Perform AND operation on two index_merge lists and store result in *im1.
 */
@@ -942,10 +999,23 @@ int imerge_list_or_tree(RANGE_OPT_PARAM *param,
 {
   SEL_IMERGE *imerge;
   List_iterator<SEL_IMERGE> it(*im1);
+  bool tree_used= FALSE;
   while ((imerge= it++))
   {
-    if (imerge->or_sel_tree_with_checks(param, tree))
+    SEL_TREE *or_tree;
+    if (tree_used)
+    {
+      or_tree= new SEL_TREE (tree, param);
+      if (!or_tree ||
+          (or_tree->keys_map.is_clear_all() && or_tree->merges.is_empty()))
+        return FALSE;
+    }
+    else
+      or_tree= tree;
+
+    if (imerge->or_sel_tree_with_checks(param, or_tree))
       it.remove();
+    tree_used= TRUE;
   }
   return im1->is_empty();
 }
@@ -5548,6 +5618,8 @@ get_mm_parts(RANGE_OPT_PARAM *param, COND *cond_func, Field *field,
     }
   }
 
+  if (tree && tree->merges.is_empty() && tree->keys_map.is_clear_all())
+    tree= NULL;
   DBUG_RETURN(tree);
 }