Fixed the bug mdev-12556.

When the rows produced on the current iteration are sent to the
temporary table T of the UNION type created for CTE the rows
that were not there simultaneously are sent to the temporary
table D that contains rows for the next iteration. The test
whether a row was in T checks the return code of writing into T.
If just a HEAP table is used for T then the return code is
HA_ERR_FOUND_DUPP_KEY, but if an ARIA table is used for T then
the return code is HA_ERR_FOUND_DUPP_UNIQUE.
The implementation of select_union_recursive::send_data()
erroneously checked only for the first return code. So if an Aria
table was used for T then all rows produced by the current iteration
went to D and and in most cases D grew with each iteration.
Whether T has reached stabilization is detected by
checking whether D is empty. So as a result, the iterations were
never stopped unless a limit for them was set.

Fixed by checking for both HA_ERR_FOUND_DUPP_KEY and
HA_ERR_FOUND_DUPP_UNIQUE as return codes returned by
the function writing a row into the temporary table T.

mysql-test/r/cte_recursive.result

@@ -2771,3 +2771,45 @@ f
 1
 2
 DROP TABLE t1;
+#
+# mdev-12556: recursive execution uses Aria temporary tables 
+#             
+CREATE TABLE t (c1 varchar(255), c2 tinytext);
+INSERT INTO t VALUES ('a','a'),('b','b'),('c','c'),('d','d');
+ANALYZE WITH RECURSIVE cte(f) AS (
+SELECT c1 FROM t
+UNION
+SELECT c1 FROM t, cte
+) SELECT COUNT(*) FROM cte;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	4	4.00	100.00	100.00	
+2	SUBQUERY	t	ALL	NULL	NULL	NULL	NULL	4	4.00	100.00	100.00	
+3	RECURSIVE UNION	t	ALL	NULL	NULL	NULL	NULL	4	4.00	100.00	100.00	
+3	RECURSIVE UNION	<derived2>	ALL	NULL	NULL	NULL	NULL	4	4.00	100.00	100.00	Using join buffer (flat, BNL join)
+NULL	UNION RESULT	<union2,3>	ALL	NULL	NULL	NULL	NULL	NULL	0.00	NULL	NULL	
+WITH RECURSIVE cte(f) AS (
+SELECT c1 FROM t
+UNION
+SELECT c1 FROM t, cte
+) SELECT COUNT(*) FROM cte;
+COUNT(*)
+4
+ANALYZE WITH RECURSIVE cte(f) AS (
+SELECT c2 FROM t
+UNION
+SELECT c2 FROM t, cte
+) SELECT COUNT(*) FROM cte;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
+1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	4	4.00	100.00	100.00	
+2	SUBQUERY	t	ALL	NULL	NULL	NULL	NULL	4	4.00	100.00	100.00	
+3	RECURSIVE UNION	t	ALL	NULL	NULL	NULL	NULL	4	4.00	100.00	100.00	
+3	RECURSIVE UNION	<derived2>	ALL	NULL	NULL	NULL	NULL	4	4.00	100.00	100.00	Using join buffer (flat, BNL join)
+NULL	UNION RESULT	<union2,3>	ALL	NULL	NULL	NULL	NULL	NULL	0.00	NULL	NULL	
+WITH RECURSIVE cte(f) AS (
+SELECT c2 FROM t
+UNION
+SELECT c2 FROM t, cte
+) SELECT COUNT(*) FROM cte;
+COUNT(*)
+4
+DROP TABLE t;

mysql-test/t/cte_recursive.test

@@ -1846,3 +1846,33 @@ cte(f) AS ( SELECT i FROM t1 UNION SELECT f FROM t1, cte WHERE 1=0 )
 SELECT * FROM cte;
 
 DROP TABLE t1;
+
+--echo #
+--echo # mdev-12556: recursive execution uses Aria temporary tables 
+--echo #             
+
+CREATE TABLE t (c1 varchar(255), c2 tinytext);
+INSERT INTO t VALUES ('a','a'),('b','b'),('c','c'),('d','d');
+
+let $q1=
+WITH RECURSIVE cte(f) AS (
+  SELECT c1 FROM t
+  UNION
+  SELECT c1 FROM t, cte
+) SELECT COUNT(*) FROM cte;
+
+let $q2=
+WITH RECURSIVE cte(f) AS (
+  SELECT c2 FROM t
+  UNION
+  SELECT c2 FROM t, cte
+) SELECT COUNT(*) FROM cte;
+
+eval ANALYZE $q1;
+eval $q1;
+
+eval ANALYZE $q2;
+eval $q2;
+
+DROP TABLE t;
+

sql/sql_union.cc

@@ -105,7 +105,8 @@ int select_union_recursive::send_data(List<Item> &values)
 {
   int rc= select_union::send_data(values);
 
-  if (write_err != HA_ERR_FOUND_DUPP_KEY)
+  if (write_err != HA_ERR_FOUND_DUPP_KEY && 
+      write_err != HA_ERR_FOUND_DUPP_UNIQUE)
   { 
     int err;
     if ((err= incr_table->file->ha_write_tmp_row(table->record[0])))