MDEV-17096 Pushdown of simple derived tables to storage engines

MDEV-17631 select_handler for a full query pushdown

Added comments and file headers for files introduced in these tasks.

sql/derived_handler.cc

+/*
+   Copyright (c) 2018, 2019 MariaDB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */
+
 #include "mariadb.h"
 #include "sql_priv.h"
 #include "sql_select.h"
 #include "derived_handler.h"
 
-void derived_handler::set_derived(TABLE_LIST *tbl)
-{
-  derived= tbl;
-  table= tbl->table;
-  unit= tbl->derived;
-  select= unit->first_select();
-  tmp_table_param= select->next_select() ?
-                   ((select_unit *)(unit->result))->get_tmp_table_param() :
-                   &select->join->tmp_table_param;
-}
+
+/**
+  The methods of the Pushdown_derived class.
+
+  The objects of this class are used for pushdown of the derived tables
+  into engines. The main  method of the class is Pushdown_derived::execute()
+  that initiates execution of the query specifying a derived by a foreign
+  engine, receives the rows of the result set and put them in a temporary
+  table on the server side.
+
+  The method uses only the functions of the derived_handle interface to do
+  this. The constructor of the class gets this interface as a parameter.
+
+  Currently a derived tables pushed into an engine is always materialized.
+  It could be changed if the cases when the tables is used as driving table.
+*/
+
 
 Pushdown_derived::Pushdown_derived(TABLE_LIST *tbl, derived_handler *h)
  : derived(tbl), handler(h)
@@ -20,11 +43,13 @@ Pushdown_derived::Pushdown_derived(TABLE_LIST *tbl, derived_handler *h)
   is_analyze= handler->thd->lex->analyze_stmt;
 }
 
+
 Pushdown_derived::~Pushdown_derived()
 {
   delete handler;
 }
 
+
 int Pushdown_derived::execute()
 {
   int err;
@@ -82,3 +107,21 @@ int Pushdown_derived::execute()
   DBUG_RETURN(-1);                              // Error not sent to client
 }
 
+
+void derived_handler::print_error(int error, myf errflag)
+{
+  my_error(ER_GET_ERRNO, MYF(0), error, hton_name(ht)->str);
+}
+
+
+void derived_handler::set_derived(TABLE_LIST *tbl)
+{
+  derived= tbl;
+  table= tbl->table;
+  unit= tbl->derived;
+  select= unit->first_select();
+  tmp_table_param= select->next_select() ?
+                   ((select_unit *)(unit->result))->get_tmp_table_param() :
+                   &select->join->tmp_table_param;
+}
+

sql/derived_handler.h

+/*
+   Copyright (c) 2016, 2017 MariaDB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
 #ifndef DERIVED_HANDLER_INCLUDED
 #define DERIVED_HANDLER_INCLUDED
 
@@ -8,6 +24,13 @@ class TMP_TABLE_PARAM;
 
 typedef class st_select_lex_unit SELECT_LEX_UNIT;
 
+/**
+  @class derived_handler
+
+  This interface class is to be used for execution of queries that specify
+  derived table by foreign engines
+*/
+
 class derived_handler
 {
 public:
@@ -23,11 +46,12 @@ class derived_handler
   */
   TABLE *table;
 
+  /* The parameters if the temporary table used at its creation */
   TMP_TABLE_PARAM *tmp_table_param;
 
-  SELECT_LEX_UNIT *unit;
+  SELECT_LEX_UNIT *unit;   // Specifies the derived table
 
-  SELECT_LEX *select;
+  SELECT_LEX *select;      // The first select of the specification
 
   derived_handler(THD *thd_arg, handlerton *ht_arg)
     : thd(thd_arg), ht(ht_arg), derived(0),table(0), tmp_table_param(0),
@@ -53,7 +77,7 @@ class derived_handler
   virtual int end_scan()=0;
 
   /* Report errors */
-  virtual void print_error(int error, myf errflag)=0;
+  virtual void print_error(int error, myf errflag);
 
   void set_derived(TABLE_LIST *tbl);
 };

sql/select_handler.cc

+/*
+   Copyright (c) 2018, 2019 MariaDB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */
+
 #include "mariadb.h"
 #include "sql_priv.h"
 #include "sql_select.h"
 #include "select_handler.h"
 
 
+/**
+  The methods of the Pushdown_select class.
+
+  The objects of this class are used for pushdown of the select queries
+  into engines. The main  method of the class is Pushdown_select::execute()
+  that initiates execution of a select query by a foreign engine, receives the
+  rows of the result set, put it in a buffer of a temporary table and send
+  them from the buffer directly into output.
+
+  The method uses the functions of the select_handle interface to do this.
+  It also employes plus some helper functions to create the needed temporary
+  table and to send rows from the temporary table into output.
+  The constructor of the class gets the select_handler interface as a parameter.
+*/
+
+
 Pushdown_select::Pushdown_select(SELECT_LEX *sel, select_handler *h)
   : select(sel), handler(h)
 {
   is_analyze= handler->thd->lex->analyze_stmt;
 }
 
+
 Pushdown_select::~Pushdown_select()
 {
   delete handler;
   select->select_h= NULL;
 }
 
+
 bool Pushdown_select::init()
 {
   List<Item> types;
@@ -38,6 +72,7 @@ bool Pushdown_select::init()
   DBUG_RETURN(false);
 }
 
+
 bool Pushdown_select::send_result_set_metadata()
 {
   THD *thd= handler->thd;
@@ -59,6 +94,7 @@ bool Pushdown_select::send_result_set_metadata()
   DBUG_RETURN(false);
 }
 
+
 bool Pushdown_select::send_data()
 {
   THD *thd= handler->thd;
@@ -83,6 +119,7 @@ bool Pushdown_select::send_data()
   DBUG_RETURN(false);
 }
 
+
 bool Pushdown_select::send_eof()
 {
   THD *thd= handler->thd;
@@ -98,6 +135,7 @@ bool Pushdown_select::send_eof()
   DBUG_RETURN(false);
 }
 
+
 int Pushdown_select::execute()
 {
   int err;
@@ -143,3 +181,8 @@ int Pushdown_select::execute()
   handler->print_error(err, MYF(0));
   DBUG_RETURN(-1);                              // Error not sent to client
 }
+
+void select_handler::print_error(int error, myf errflag)
+{
+  my_error(ER_GET_ERRNO, MYF(0), error, hton_name(ht)->str);
+}

sql/select_handler.h

+/*
+   Copyright (c) 2018, 2019 MariaDB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
 #ifndef SELECT_HANDLER_INCLUDED
 #define SELECT_HANDLER_INCLUDED
 
 #include "mariadb.h"
 #include "sql_priv.h"
 
+/**
+  @class select_handler
+
+  This interface class is to be used for execution of select queries
+  by foreign engines
+*/
+
 class select_handler
 {
  public:
   THD *thd;
   handlerton *ht;
 
-  SELECT_LEX *select;
+  SELECT_LEX *select;  // Select to be excuted
 
   /*
     Temporary table where all results should be stored in record[0]
     The table has a field for every item from the select_lex::item_list.
+    The table is actually never filled. Only its record buffer is used.
   */
   TABLE *table;
 
@@ -42,7 +66,7 @@ class select_handler
   virtual int end_scan() = 0;
 
   /* Report errors */
-  virtual void print_error(int error, myf errflag) = 0;
+  virtual void print_error(int error, myf errflag);
 };
 
 #endif /* SELECT_HANDLER_INCLUDED */

sql/sql_derived.cc

@@ -932,6 +932,7 @@ bool mysql_derived_optimize(THD *thd, LEX *lex, TABLE_LIST *derived)
 
   if (derived->is_materialized_derived() && derived->dt_handler)
   {
+    /* Create an object for execution of the query specifying the table */
     if (!(derived->pushdown_derived=
             new (thd->mem_root) Pushdown_derived(derived, derived->dt_handler)))
     {
@@ -1151,6 +1152,7 @@ bool mysql_derived_fill(THD *thd, LEX *lex, TABLE_LIST *derived)
     int res;
     if (unit->executed)
       DBUG_RETURN(FALSE);
+    /* Execute the query that specifies the derived table by a foreign engine */
     res= derived->pushdown_derived->execute();
     unit->executed= true;
     delete derived->pushdown_derived;
@@ -1457,6 +1459,25 @@ bool pushdown_cond_for_derived(THD *thd, Item *cond, TABLE_LIST *derived)
 }
 
 
+/**
+  @brief
+    Look for provision of the derived_handler interface by a foreign engine
+
+  @param thd   The thread handler
+
+  @details
+    The function looks through its tables of the query that specifies this
+    derived table searching for a table whose handlerton owns a
+    create_derived call-back function. If the call of this function returns
+    a derived_handler interface object then the server will push the query
+    specifying the derived table into this engine.
+    This is a responsibility of the create_derived call-back function to
+    check whether the engine can execute the query.
+
+  @retval the found derived_handler if the search is successful
+          0  otherwise
+*/
+
 derived_handler *TABLE_LIST::find_derived_handler(THD *thd)
 {
   if (!derived || is_recursive_with_table())

sql/sql_lex.h

@@ -1257,7 +1257,9 @@ class st_select_lex: public st_select_lex_node
   table_value_constr *tvc;
   bool in_tvc;
 
+  /* The interface employed to execute the select query by a foreign engine */
   select_handler *select_h;
+  /* The object used to organize execution of the query by a foreign engine */
   Pushdown_select *pushdown_select;
 
   /** System Versioning */

sql/sql_select.cc

@@ -1444,7 +1444,10 @@ int JOIN::optimize()
   if (select_lex->pushdown_select)
   {
     if (!(select_options & SELECT_DESCRIBE))
+    {
+      /* Prepare to execute the query pushed into a foreign engine */
       res= select_lex->pushdown_select->init();
+    }
     with_two_phase_optimization= false;
   }
   else if (optimization_state == JOIN::OPTIMIZATION_PHASE_1_DONE)
@@ -4074,6 +4077,7 @@ void JOIN::exec_inner()
   }
   else if (select_lex->pushdown_select)
   {
+    /* Execute the query pushed into a foreign engine */
     error= select_lex->pushdown_select->execute();
     DBUG_VOID_RETURN;
   }
@@ -4288,9 +4292,11 @@ mysql_select(THD *thd,
     }
   }
 
+  /* Look for a table owned by an engine with the select_handler interface */
   select_lex->select_h= select_lex->find_select_handler(thd);
   if (select_lex->select_h)
   {
+    /* Create a Pushdown_select object for later execution of the query */
     if (!(select_lex->pushdown_select=
       new (thd->mem_root) Pushdown_select(select_lex,
                                           select_lex->select_h)))
@@ -27620,6 +27626,26 @@ Item *remove_pushed_top_conjuncts(THD *thd, Item *cond)
   return cond;
 }
 
+
+/**
+  @brief
+    Look for provision of the select_handler interface by a foreign engine
+
+  @param thd   The thread handler
+
+  @details
+    The function checks that this is an upper level select and if so looks
+    through its tables searching for one whose handlerton owns a
+    create_select call-back function. If the call of this function returns
+    a select_handler interface object then the server will push the select
+    query into this engine.
+    This is a responsibility of the create_select call-back function to
+    check whether the engine can execute the query.
+
+  @retval the found select_handler if the search is successful
+          0  otherwise
+*/
+
 select_handler *SELECT_LEX::find_select_handler(THD *thd)
 {
   if (next_select())

sql/table.h

@@ -2136,8 +2136,14 @@ struct TABLE_LIST
   TABLE_LIST * next_with_rec_ref;
   bool is_derived_with_recursive_reference;
   bool block_handle_derived;
+  /* The interface employed to materialize the table by a foreign engine */
   derived_handler *dt_handler;
+  /* The text of the query specifying the derived table */
   LEX_CSTRING derived_spec;
+  /*
+    The object used to organize execution of the query that specifies
+    the derived table by a foreign engine
+  */
   Pushdown_derived *pushdown_derived;
   ST_SCHEMA_TABLE *schema_table;        /* Information_schema table */
   st_select_lex	*schema_select_lex;

storage/federatedx/federatedx_pushdown.cc

@@ -16,6 +16,25 @@
 
 /* !!! For inclusion into ha_federatedx.cc */
 
+
+/*
+  This is a quick a dirty implemention of the derived_handler and select_handler
+  interfaces to be used to push select queries and the queries specifying
+  derived tables into FEDERATEDX engine.
+  The functions
+    create_federatedx_derived_handler and
+    create_federatedx_select_handler
+  that return the corresponding interfaces for pushdown capabilities do
+  not check a lot of things. In particular they do not check that the tables
+  of the pushed queries belong to the same foreign server.
+
+  The implementation is provided purely for testing purposes.
+  The pushdown capabilities are enabled by turning on the plugin system
+  variable federated_pushdown:
+    set global federated_pushdown=1;
+*/
+
+
 static derived_handler*
 create_federatedx_derived_handler(THD* thd, TABLE_LIST *derived)
 {