BUG#11869 (cont'd, eliminating "table type doesn't support FULLTEXT" error):

When creating temporary table for UNION, pass TMP_TABLE_FORCE_MYISAM flag to 
create_tmp_table if we will be using fulltext function(s) when reading from the 
temp. table.

mysql-test/r/fulltext_order_by.result

@@ -159,4 +159,14 @@ match(c.beitrag) against ('+abc' in boolean mode)
 order by 
 match(betreff) against ('+abc' in boolean mode) desc;
 text	id	betreff
+(select b.id, b.betreff from t3 b) union 
+(select b.id, b.betreff from t3 b) 
+order by match(betreff) against ('+abc' in boolean mode) desc;
+id	betreff
+select distinct b.id, b.betreff from t3 b 
+order by match(betreff) against ('+abc' in boolean mode) desc;
+id	betreff
+select b.id, b.betreff from t3 b group by b.id+1 
+order by match(betreff) against ('+abc' in boolean mode) desc;
+id	betreff
 drop table t1,t2,t3;

mysql-test/t/fulltext_order_by.test

@@ -133,6 +133,17 @@ where
 order by 
   match(betreff) against ('+abc' in boolean mode) desc;
 
+# BUG#11869 part2: used table type doesn't support FULLTEXT indexes error
+(select b.id, b.betreff from t3 b) union 
+(select b.id, b.betreff from t3 b) 
+order by match(betreff) against ('+abc' in boolean mode) desc;
+
+select distinct b.id, b.betreff from t3 b 
+order by match(betreff) against ('+abc' in boolean mode) desc;
+
+select b.id, b.betreff from t3 b group by b.id+1 
+order by match(betreff) against ('+abc' in boolean mode) desc;
+
 drop table t1,t2,t3;
 
 # End of 4.1 tests

sql/mysql_priv.h

@@ -215,6 +215,12 @@ extern CHARSET_INFO *national_charset_info, *table_alias_charset;
    in the user query has requested */
 #define SELECT_ALL			(1L << 29)
 
+/* 
+  Force the used temporary table to be a MyISAM table (because we will use
+  fulltext functions when reading from it.
+*/
+#define TMP_TABLE_FORCE_MYISAM          (1L << 30) 
+
 /* If set to 0, then the thread will ignore all warnings with level notes.
    Set by executing SET SQL_NOTES=1 */
 #define OPTION_SQL_NOTES                (1L << 31)

sql/sql_select.cc

@@ -5290,7 +5290,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
   /* If result table is small; use a heap */
   if (blob_count || using_unique_constraint ||
       (select_options & (OPTION_BIG_TABLES | SELECT_SMALL_RESULT)) ==
-      OPTION_BIG_TABLES)
+      OPTION_BIG_TABLES ||(select_options & TMP_TABLE_FORCE_MYISAM))
   {
     table->file=get_new_handler(table,table->db_type=DB_TYPE_MYISAM);
     if (group &&

sql/sql_union.cc

@@ -296,15 +296,23 @@ int st_select_lex_unit::prepare(THD *thd_arg, select_result *sel_result,
         goto err;
       }
     }
+    
+    ulong create_options= first_select_in_union()->options | thd_arg->options |
+                          TMP_TABLE_ALL_COLUMNS;
+    /*
+      Force the temporary table to be a MyISAM table if we're going to use
+      fullext functions (MATCH ... AGAINST .. IN BOOLEAN MODE) when reading
+      from it.
+    */
+    if (global_parameters->ftfunc_list->elements)
+      create_options= create_options | TMP_TABLE_FORCE_MYISAM;
 
     union_result->tmp_table_param.field_count= types.elements;
     if (!(table= create_tmp_table(thd_arg,
 				  &union_result->tmp_table_param, types,
 				  (ORDER*) 0, (bool) union_distinct, 1, 
-				  (first_select_in_union()->options |
-				   thd_arg->options |
-				   TMP_TABLE_ALL_COLUMNS),
-				  HA_POS_ERROR, (char *) tmp_table_alias)))
+                                  create_options, HA_POS_ERROR, 
+                                  (char *) tmp_table_alias)))
       goto err;
     table->file->extra(HA_EXTRA_WRITE_CACHE);
     table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);