Fixed Bug#11764168 "56976: SEVERE DENIAL OF SERVICE IN PREPARED STATEMENTS".

The problem was that server didn't check resulting size of prepared
statement argument which was set using mysql_send_long_data() API.
By calling mysql_send_long_data() several times it was possible
to create overly big string and thus force server to allocate
memory for it. There was no way to limit this allocation.

The solution is to add check for size of result string against
value of max_long_data_size start-up parameter. When intermediate
string exceeds max_long_data_size value an appropriate error message
is emitted.

We can't use existing max_allowed_packet parameter for this purpose
since its value is limited by 1GB and therefore using it as a limit
for data set through mysql_send_long_data() API would have been an
incompatible change. Newly introduced max_long_data_size parameter
gets value from max_allowed_packet parameter unless its value is
specified explicitly. This new parameter is marked as deprecated
and will be eventually replaced by max_allowed_packet parameter.
Value of max_long_data_size parameter can be set only at server
startup.


mysql-test/t/variables.test:
  Added checking for new start-up parameter max_long_data_size.
sql/item.cc:
  Added call to my_message() when accumulated string exceeds
  max_long_data_size value. my_message() calls error handler
  that was installed in mysql_stmt_get_longdata before call
  to Item_param::set_longdata.
  
  The error handler then sets state, last_error and last_errno
  fields for current statement to values which correspond to
  error which was caught.
sql/mysql_priv.h:
  Added max_long_data_size variable declaration.
sql/mysqld.cc:
  Added support for start-up parameter 'max_long_data_size'.
  This parameter limits size of data which can be sent from
  client to server using mysql_send_long_data() API.
sql/set_var.cc:
  Added variable 'max_long_data_size' into list of variables
  displayed by command 'show variables'.
sql/sql_prepare.cc:
  Added error handler class Set_longdata_error_handler.
  This handler is used to catch any errors that can be
  generated during execution of Item_param::set_longdata().
  
  Source code snippet that makes checking for statement's state 
  during statement execution is moved from Prepared_statement::execute()
  to Prepared_statement::execute_loop() in order not to call
  set_parameters() when statement has failed during
  set_long_data() execution. If this hadn't been done
  the call to set_parameters() would have failed.
tests/mysql_client_test.c:
  A testcase for the bug #56976 was added.

mysql-test/r/variables.result

@@ -1540,6 +1540,9 @@ ERROR HY000: Cannot drop default keycache
 SET @@global.key_cache_block_size=0;
 Warnings:
 Warning	1292	Truncated incorrect key_cache_block_size value: '0'
+select @@max_long_data_size;
+@@max_long_data_size
+1048576
 SET @@global.max_binlog_cache_size=DEFAULT;
 SET @@global.max_join_size=DEFAULT;
 SET @@global.key_buffer_size=@kbs;

mysql-test/t/variables.test

@@ -1293,6 +1293,11 @@ SET @@global.max_join_size=0;
 SET @@global.key_buffer_size=0;
 SET @@global.key_cache_block_size=0;
 
+#
+# Bug#56976: added new start-up parameter
+#
+select @@max_long_data_size;
+
 # cleanup
 SET @@global.max_binlog_cache_size=DEFAULT;
 SET @@global.max_join_size=DEFAULT;

sql/item.cc

@@ -2742,6 +2742,16 @@ bool Item_param::set_longdata(const char *str, ulong length)
     (here), and first have to concatenate all pieces together,
     write query to the binary log and only then perform conversion.
   */
+  if (str_value.length() + length > max_long_data_size)
+  {
+    my_message(ER_UNKNOWN_ERROR,
+               "Parameter of prepared statement which is set through "
+               "mysql_send_long_data() is longer than "
+               "'max_long_data_size' bytes",
+               MYF(0));
+    DBUG_RETURN(true);
+  }
+
   if (str_value.append(str, length, &my_charset_bin))
     DBUG_RETURN(TRUE);
   state= LONG_DATA_VALUE;

sql/mysql_priv.h

@@ -1965,6 +1965,7 @@ extern my_bool relay_log_purge, opt_innodb_safe_binlog, opt_innodb;
 extern uint test_flags,select_errors,ha_open_options;
 extern uint protocol_version, mysqld_port, dropping_tables;
 extern uint delay_key_write_options;
+extern ulong max_long_data_size;
 #endif /* MYSQL_SERVER */
 #if defined MYSQL_SERVER || defined INNODB_COMPATIBILITY_HOOKS
 extern MYSQL_PLUGIN_IMPORT uint lower_case_table_names;

sql/mysqld.cc

@@ -409,6 +409,7 @@ TYPELIB log_output_typelib= {array_elements(log_output_names)-1,"",
 
 /* the default log output is log tables */
 static bool lower_case_table_names_used= 0;
+static bool max_long_data_size_used= false;
 static bool volatile select_thread_in_use, signal_thread_in_use;
 static bool volatile ready_to_exit;
 static my_bool opt_debugging= 0, opt_external_locking= 0, opt_console= 0;
@@ -574,6 +575,11 @@ ulong delayed_insert_errors,flush_time;
 ulong specialflag=0;
 ulong binlog_cache_use= 0, binlog_cache_disk_use= 0;
 ulong max_connections, max_connect_errors;
+/*
+  Maximum length of parameter value which can be set through
+  mysql_send_long_data() call.
+*/
+ulong max_long_data_size;
 uint  max_user_connections= 0;
 /**
   Limit of the total number of prepared statements in the server.
@@ -5800,7 +5806,8 @@ enum options_mysqld
   OPT_SLOW_QUERY_LOG_FILE,
   OPT_IGNORE_BUILTIN_INNODB,
   OPT_BINLOG_DIRECT_NON_TRANS_UPDATE,
-  OPT_DEFAULT_CHARACTER_SET_OLD
+  OPT_DEFAULT_CHARACTER_SET_OLD,
+  OPT_MAX_LONG_DATA_SIZE
 };
 
 
@@ -6880,6 +6887,13 @@ thread is in the relay logs.",
     &global_system_variables.max_length_for_sort_data,
     &max_system_variables.max_length_for_sort_data, 0, GET_ULONG,
     REQUIRED_ARG, 1024, 4, 8192*1024L, 0, 1, 0},
+  {"max_long_data_size", OPT_MAX_LONG_DATA_SIZE,
+   "The maximum size of prepared statement parameter which can be provided "
+   "through mysql_send_long_data() API call. "
+   "Deprecated option; use max_allowed_packet instead.",
+   &max_long_data_size,
+   &max_long_data_size, 0, GET_ULONG,
+   REQUIRED_ARG, 1024*1024L, 1024, UINT_MAX32, MALLOC_OVERHEAD, 1, 0},
   {"max_prepared_stmt_count", OPT_MAX_PREPARED_STMT_COUNT,
    "Maximum number of prepared statements in the server.",
    &max_prepared_stmt_count, &max_prepared_stmt_count,
@@ -8688,6 +8702,10 @@ mysqld_get_one_option(int optid,
     }
     break;
 #endif /* defined(ENABLED_DEBUG_SYNC) */
+  case OPT_MAX_LONG_DATA_SIZE:
+    max_long_data_size_used= true;
+    WARN_DEPRECATED(NULL, VER_CELOSIA, "--max_long_data_size", "--max_allowed_packet");
+    break;
   }
   return 0;
 }
@@ -8849,6 +8867,14 @@ static int get_options(int *argc,char **argv)
   else
     pool_of_threads_scheduler(&thread_scheduler);  /* purecov: tested */
 #endif
+
+  /*
+    If max_long_data_size is not specified explicitly use
+    value of max_allowed_packet.
+  */
+  if (!max_long_data_size_used)
+    max_long_data_size= global_system_variables.max_allowed_packet;
+
   return 0;
 }
 

sql/set_var.cc

@@ -394,6 +394,12 @@ static sys_var_thd_ulong	sys_max_seeks_for_key(&vars, "max_seeks_for_key",
 					      &SV::max_seeks_for_key);
 static sys_var_thd_ulong   sys_max_length_for_sort_data(&vars, "max_length_for_sort_data",
                                                  &SV::max_length_for_sort_data);
+static sys_var_const    sys_max_long_data_size(&vars,
+                                               "max_long_data_size",
+                                               OPT_GLOBAL, SHOW_LONG,
+                                               (uchar*)
+                                               &max_long_data_size);
+
 #ifndef TO_BE_DELETED	/* Alias for max_join_size */
 static sys_var_thd_ha_rows	sys_sql_max_join_size(&vars, "sql_max_join_size",
 					      &SV::max_join_size,

sql/sql_prepare.cc

@@ -2730,6 +2730,32 @@ void mysql_sql_stmt_close(THD *thd)
   }
 }
 
+
+class Set_longdata_error_handler : public Internal_error_handler
+{
+public:
+  Set_longdata_error_handler(Prepared_statement *statement)
+    : stmt(statement)
+  { }
+
+public:
+  bool handle_error(uint sql_errno,
+                    const char *message,
+                    MYSQL_ERROR::enum_warning_level level,
+                    THD *)
+  {
+    stmt->state= Query_arena::ERROR;
+    stmt->last_errno= sql_errno;
+    strncpy(stmt->last_error, message, MYSQL_ERRMSG_SIZE);
+
+    return TRUE;
+  }
+
+private:
+  Prepared_statement *stmt;
+};
+
+
 /**
   Handle long data in pieces from client.
 
@@ -2786,16 +2812,19 @@ void mysql_stmt_get_longdata(THD *thd, char *packet, ulong packet_length)
 
   param= stmt->param_array[param_number];
 
+  Set_longdata_error_handler err_handler(stmt);
+  /*
+    Install handler that will catch any errors that can be generated
+    during execution of Item_param::set_longdata() and propagate
+    them to Statement::last_error.
+  */
+  thd->push_internal_handler(&err_handler);
 #ifndef EMBEDDED_LIBRARY
-  if (param->set_longdata(packet, (ulong) (packet_end - packet)))
+  param->set_longdata(packet, (ulong) (packet_end - packet));
 #else
-  if (param->set_longdata(thd->extra_data, thd->extra_length))
+  param->set_longdata(thd->extra_data, thd->extra_length);
 #endif
-  {
-    stmt->state= Query_arena::ERROR;
-    stmt->last_errno= ER_OUTOFMEMORY;
-    sprintf(stmt->last_error, ER(ER_OUTOFMEMORY), 0);
-  }
+  thd->pop_internal_handler();
 
   general_log_print(thd, thd->command, NullS);
 
@@ -3257,6 +3286,13 @@ Prepared_statement::execute_loop(String *expanded_query,
   bool error;
   int reprepare_attempt= 0;
 
+  /* Check if we got an error when sending long data */
+  if (state == Query_arena::ERROR)
+  {
+    my_message(last_errno, last_error, MYF(0));
+    return TRUE;
+  }
+
   if (set_parameters(expanded_query, packet, packet_end))
     return TRUE;
 
@@ -3497,12 +3533,6 @@ bool Prepared_statement::execute(String *expanded_query, bool open_cursor)
 
   status_var_increment(thd->status_var.com_stmt_execute);
 
-  /* Check if we got an error when sending long data */
-  if (state == Query_arena::ERROR)
-  {
-    my_message(last_errno, last_error, MYF(0));
-    return TRUE;
-  }
   if (flags & (uint) IS_IN_USE)
   {
     my_error(ER_PS_NO_RECURSION, MYF(0));

tests/mysql_client_test.c

@@ -18464,6 +18464,56 @@ static void test_bug58036()
 }
 
 
+/*
+  Bug #56976:   Severe Denial Of Service in prepared statements
+*/
+static void test_bug56976()
+{
+  MYSQL_STMT   *stmt;
+  MYSQL_BIND    bind[1];
+  int           rc;
+  const char*   query = "SELECT LENGTH(?)";
+  char *long_buffer;
+  unsigned long i, packet_len = 256 * 1024L;
+  unsigned long dos_len    = 2 * 1024 * 1024L;
+
+  DBUG_ENTER("test_bug56976");
+  myheader("test_bug56976");
+
+  stmt= mysql_stmt_init(mysql);
+  check_stmt(stmt);
+
+  rc= mysql_stmt_prepare(stmt, query, strlen(query));
+  check_execute(stmt, rc);
+
+  memset(bind, 0, sizeof(bind));
+  bind[0].buffer_type = MYSQL_TYPE_TINY_BLOB;
+
+  rc= mysql_stmt_bind_param(stmt, bind);
+  check_execute(stmt, rc);
+
+  long_buffer= (char*) my_malloc(packet_len, MYF(0));
+  DIE_UNLESS(long_buffer);
+
+  memset(long_buffer, 'a', packet_len);
+
+  for (i= 0; i < dos_len / packet_len; i++)
+  {
+    rc= mysql_stmt_send_long_data(stmt, 0, long_buffer, packet_len);
+    check_execute(stmt, rc);
+  }
+
+  my_free(long_buffer, MYF(0));
+  rc= mysql_stmt_execute(stmt);
+
+  DIE_UNLESS(rc && mysql_stmt_errno(stmt) == ER_UNKNOWN_ERROR);
+
+  mysql_stmt_close(stmt);
+
+  DBUG_VOID_RETURN;
+}
+
+
 /*
   Read and parse arguments and MySQL options from my.cnf
 */
@@ -18791,6 +18841,7 @@ static struct my_tests_st my_tests[]= {
   { "test_bug54041", test_bug54041 },
   { "test_bug47485", test_bug47485 },
   { "test_bug58036", test_bug58036 },
+  { "test_bug56976", test_bug56976 },
   { 0, 0 }
 };