MDEV-30964: MAX_SEL_ARG memory exhaustion is not visible in the optimizer trace

Add printing

mysql-test/main/opt_trace.result

@@ -8515,6 +8515,47 @@ SELECT a FROM t1 WHERE (a,b) in (SELECT @c,@d);
 a
 DROP TABLE t1;
 #
+# MDEV-30964: MAX_SEL_ARG memory exhaustion is not visible in the optimizer trace
+#
+create table t1 (
+c1 int,
+c2 int,
+c3 int,
+c4 int,
+c5 int,
+c6 int,
+c7 int,
+c8 int,
+key(c1,c2,c3,c4,c5,c6,c7,c8)
+);
+insert into t1 () values (),(),();
+explain select *
+from t1
+where
+(c1 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16) and c2=1) and
+c3 in (1,2,3,4,5,6,7,8,9,10) and
+c4 in (1,2,3,4,5,6,7,8,9,10) and
+c5 in (1,2,3,4,5,6,7,8,9,10) and
+c6 in (1,2,3,4);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	c1	c1	40	NULL	3	Using where; Using index
+select
+json_detailed(json_extract(trace, '$**.setup_range_conditions'))
+from
+information_schema.optimizer_trace;
+json_detailed(json_extract(trace, '$**.setup_range_conditions'))
+[
+    [
+        {
+            "sel_arg_alloc_limit_hit": 
+            {
+                "alloced_sel_args": 16001
+            }
+        }
+    ]
+]
+drop table t1;
+#
 # MDEV-31085: multi-update using view with optimizer trace enabled
 #
 SET SESSION optimizer_trace = 'enabled=on';

mysql-test/main/opt_trace.test

@@ -677,6 +677,39 @@ INSERT INTO t1 VALUES (0,0);
 SELECT a FROM t1 WHERE (a,b) in (SELECT @c,@d);
 DROP TABLE t1;
 
+
+--echo #
+--echo # MDEV-30964: MAX_SEL_ARG memory exhaustion is not visible in the optimizer trace
+--echo #
+create table t1 (
+  c1 int,
+  c2 int,
+  c3 int,
+  c4 int,
+  c5 int,
+  c6 int,
+  c7 int,
+  c8 int,
+  key(c1,c2,c3,c4,c5,c6,c7,c8)
+);
+insert into t1 () values (),(),();
+
+explain select *
+from t1
+where
+  (c1 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16) and c2=1) and
+  c3 in (1,2,3,4,5,6,7,8,9,10) and
+  c4 in (1,2,3,4,5,6,7,8,9,10) and
+  c5 in (1,2,3,4,5,6,7,8,9,10) and
+  c6 in (1,2,3,4);
+
+select
+  json_detailed(json_extract(trace, '$**.setup_range_conditions'))
+from
+  information_schema.optimizer_trace;
+
+drop table t1;
+
 --echo #
 --echo # MDEV-31085: multi-update using view with optimizer trace enabled
 --echo #

sql/opt_range.cc

@@ -2853,6 +2853,13 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
         Json_writer_array trace_range_summary(thd,
                                               "setup_range_conditions");
         tree= cond->get_mm_tree(&param, &cond);
+        if (thd->trace_started() && 
+            param.alloced_sel_args >= SEL_ARG::MAX_SEL_ARGS)
+        {
+          Json_writer_object wrapper(thd);
+          Json_writer_object obj(thd, "sel_arg_alloc_limit_hit");
+          obj.add("alloced_sel_args", param.alloced_sel_args);
+        }
       }
       if (tree)
       {