MWL#192: Non-blocking client API for libmysqlclient.

All client functions that can block on I/O have alternate _start() and
_cont() versions that do not block but return control back to the
application, which can then issue I/O wait in its own fashion and later
call back into the library to continue the operation.

Works behind the scenes by spawning a co-routine/fiber to run the
blocking operation and suspend it while waiting for I/O. This
co-routine/fiber use is invisible to applications.

For i368/x86_64 on GCC, uses very fast assembler co-routine support. On
Windows uses native Win32 Fibers. Falls back to POSIX ucontext on other
platforms. Assembler routines for more platforms are relatively easy to
add by extending mysys/my_context.c, eg. similar to the Lua lcoco
library.

For testing, mysqltest and mysql_client_test are extended with the
option --non-blocking-api. This causes the programs to use the
non-blocking API for database access. mysql-test-run.pl has a similar
option --non-blocking-api that uses this, as well as additional
testcases.

An example program tests/async_queries.c is included that uses the new
non-blocking API with libevent to show how, in a single-threaded
program, to issue many queries in parallel against a database.


client/async_example.c:
  Fix const warning
  ******
  Fix bug with wrong timeout value for poll().
include/Makefile.am:
  Fix missing include for `make dist`
include/mysql.h:
  Add prototypes for all non-blocking API calls.
include/mysql.h.pp:
  Add prototypes for all non-blocking API calls.
mysys/my_context.c:
  Fix type warning for makecontext() function pointer argument.
sql-common/mysql_async.c:
  Fix crashes in the non-blocking API for functions that can take MYSQL argument
  that is NULL.
tests/Makefile.am:
  Add header file to `make dist`
tests/mysql_client_test.c:
  Replace blocking calls with wrappers around the non-blocking calls, used in
  mysql_client_test to test the new non-blocking API.
tests/nonblock-wrappers.h:
  Replace blocking calls with wrappers around the non-blocking calls, used in
  mysql_client_test to test the new non-blocking API.

.bzrignore

@@ -350,6 +350,7 @@ client/ssl_test
 client/thimble
 client/thread_test
 client/tmp.diff
+client/async_example
 client_debug/*
 client_release/*
 client_test
@@ -681,6 +682,7 @@ libmysqld/unireg.cc
 libmysqld/discover_xt.cc
 libmysqld/ha_pbxt.cc
 libmysqld/myxt_xt.cc
+libmysqld/mysql_async.c
 libmysqltest/*.ds?
 libmysqltest/*.vcproj
 libmysqltest/mytest.c
@@ -1867,6 +1869,7 @@ tests/bug25714
 tests/client_test
 tests/connect_test
 tests/mysql_client_test
+tests/async_queries
 thr_insert_test/*
 thr_test/*
 thread_test
@@ -1952,6 +1955,7 @@ client/sql_list.cc
 client/sql_list.h
 libmysqld/client_plugin.c
 sql/client_plugin.c
+sql/mysql_async.c
 *.dgcov
 libmysqld/create_options.cc
 storage/pbxt/bin/xtstat

client/CMakeLists.txt

@@ -68,5 +68,8 @@ MYSQL_ADD_EXECUTABLE(mysqlslap mysqlslap.c DESTINATION bin)
 SET_SOURCE_FILES_PROPERTIES(mysqlslap.c PROPERTIES COMPILE_FLAGS "-DTHREADS")
 TARGET_LINK_LIBRARIES(mysqlslap mysqlclient mysys zlib wsock32 dbug)
 
+MYSQL_ADD_EXECUTABLE(async_example async_example.c)
+TARGET_LINK_LIBRARIES(async_example mysqlclient mysys zlib wsock32 dbug)
+
 
 MYSQL_ADD_EXECUTABLE(echo echo.c COMPONENT Test)

client/Makefile.am

@@ -51,6 +51,8 @@ bin_PROGRAMS =			mysql \
 				mysqltest \
 				mysql_upgrade
 
+noinst_PROGRAMS = 		async_example
+
 mysql_SOURCES =			mysql.cc readline.cc sql_string.cc \
 				completion_hash.cc
 mysql_LDADD =			@readline_link@ @TERMCAP_LIB@ \
@@ -96,6 +98,11 @@ mysqltest_LDADD =		$(CXXLDFLAGS) $(CLIENT_THREAD_LIBS) \
 mysql_upgrade_SOURCES=          mysql_upgrade.c \
                                 $(top_srcdir)/mysys/my_getpagesize.c
 
+async_example_SOURCES=		async_example.c
+async_example_LDADD =		$(CXXLDFLAGS) $(CLIENT_THREAD_LIBS) \
+				@CLIENT_EXTRA_LDFLAGS@ \
+				$(LIBMYSQLCLIENT_LA)
+
 # Fix for mit-threads
 DEFS =			-DMYSQL_CLIENT_NO_THREADS \
 			-DDEFAULT_MYSQL_HOME='"$(prefix)"' \

client/async_example.c

+/*
+  Copyright 2011 Kristian Nielsen and Monty Program Ab.
+
+  Experiments with non-blocking libmysql.
+
+  This is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 2 of the License, or
+  (at your option) any later version.
+
+  This is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+#ifndef __WIN__
+#include <poll.h>
+#else
+#include <WinSock2.h>
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <mysql.h>
+
+#define SL(s) (s), sizeof(s)
+
+static const char *my_groups[]= { "client", NULL };
+
+static int
+wait_for_mysql(MYSQL *mysql, int status)
+{
+#ifdef __WIN__
+  fd_set rs, ws, es;
+  int res;
+  struct timeval tv, *timeout;
+  my_socket s= mysql_get_socket(mysql);
+  FD_ZERO(&rs);
+  FD_ZERO(&ws);
+  FD_ZERO(&es);
+  if (status & MYSQL_WAIT_READ)
+    FD_SET(s, &rs);
+  if (status & MYSQL_WAIT_WRITE)
+    FD_SET(s, &ws);
+  if (status & MYSQL_WAIT_EXCEPT)
+    FD_SET(s, &es);
+  if (status & MYSQL_WAIT_TIMEOUT)
+  {
+    tv.tv_sec= mysql_get_timeout_value(mysql);
+    tv.tv_usec= 0;
+    timeout= &tv;
+  }
+  else
+    timeout= NULL;
+  res= select(1, &rs, &ws, &es, timeout);
+  if (res == 0)
+    return MYSQL_WAIT_TIMEOUT;
+  else if (res == SOCKET_ERROR)
+  {
+    /*
+      In a real event framework, we should handle errors and re-try the select.
+    */
+    return MYSQL_WAIT_TIMEOUT;
+  }
+  else
+  {
+    int status= 0;
+    if (FD_ISSET(s, &rs))
+      status|= MYSQL_WAIT_READ;
+    if (FD_ISSET(s, &ws))
+      status|= MYSQL_WAIT_WRITE;
+    if (FD_ISSET(s, &es))
+      status|= MYSQL_WAIT_EXCEPT;
+    return status;
+  }
+#else
+  struct pollfd pfd;
+  int timeout;
+  int res;
+
+  pfd.fd= mysql_get_socket(mysql);
+  pfd.events=
+    (status & MYSQL_WAIT_READ ? POLLIN : 0) |
+    (status & MYSQL_WAIT_WRITE ? POLLOUT : 0) |
+    (status & MYSQL_WAIT_EXCEPT ? POLLPRI : 0);
+  if (status & MYSQL_WAIT_TIMEOUT)
+    timeout= 1000*mysql_get_timeout_value(mysql);
+  else
+    timeout= -1;
+  res= poll(&pfd, 1, timeout);
+  if (res == 0)
+    return MYSQL_WAIT_TIMEOUT;
+  else if (res < 0)
+  {
+    /*
+      In a real event framework, we should handle EINTR and re-try the poll.
+    */
+    return MYSQL_WAIT_TIMEOUT;
+  }
+  else
+  {
+    int status= 0;
+    if (pfd.revents & POLLIN)
+      status|= MYSQL_WAIT_READ;
+    if (pfd.revents & POLLOUT)
+      status|= MYSQL_WAIT_WRITE;
+    if (pfd.revents & POLLPRI)
+      status|= MYSQL_WAIT_EXCEPT;
+    return status;
+  }
+#endif
+}
+
+static void
+fatal(MYSQL *mysql, const char *msg)
+{
+  fprintf(stderr, "%s: %s\n", msg, mysql_error(mysql));
+  exit(1);
+}
+
+static void
+doit(const char *host, const char *user, const char *password)
+{
+  int err;
+  MYSQL mysql, *ret;
+  MYSQL_RES *res;
+  MYSQL_ROW row;
+  int status;
+
+  mysql_init(&mysql);
+  mysql_options(&mysql, MYSQL_READ_DEFAULT_GROUP, "myapp");
+
+  /* Returns 0 when done, else flag for what to wait for when need to block. */
+  status= mysql_real_connect_start(&ret, &mysql, host, user, password, NULL,
+                                   0, NULL, 0);
+  while (status)
+  {
+    status= wait_for_mysql(&mysql, status);
+    status= mysql_real_connect_cont(&ret, &mysql, status);
+  }
+
+  if (!ret)
+    fatal(&mysql, "Failed to mysql_real_connect()");
+
+  status= mysql_real_query_start(&err, &mysql, SL("SHOW STATUS"));
+  while (status)
+  {
+    status= wait_for_mysql(&mysql, status);
+    status= mysql_real_query_cont(&err, &mysql, status);
+  }
+  if (err)
+    fatal(&mysql, "mysql_real_query() returns error");
+
+  /* This method cannot block. */
+  res= mysql_use_result(&mysql);
+  if (!res)
+    fatal(&mysql, "mysql_use_result() returns error");
+
+  for (;;)
+  {
+    status= mysql_fetch_row_start(&row, res);
+    while (status)
+    {
+      status= wait_for_mysql(&mysql, status);
+      status= mysql_fetch_row_cont(&row, res, status);
+    }
+    if (!row)
+      break;
+    printf("%s: %s\n", row[0], row[1]);
+  }
+  if (mysql_errno(&mysql))
+    fatal(&mysql, "Got error while retrieving rows");
+  mysql_free_result(res);
+
+  /* I suppose this must be non-blocking too. */
+  mysql_close(&mysql);
+}
+
+int
+main(int argc, char *argv[])
+{
+  int err;
+
+  if (argc != 4)
+  {
+    fprintf(stderr, "Usage: %s <host> <user> <password>\n", argv[0]);
+    exit(1);
+  }
+
+  err= mysql_library_init(argc, argv, (char **)my_groups);
+  if (err)
+  {
+    fprintf(stderr, "Fatal: mysql_library_init() returns error: %d\n", err);
+    exit(1);
+  }
+
+  doit(argv[1], argv[2], argv[3]);
+
+  mysql_library_end();
+
+  return 0;
+}

client/mysqltest.cc

@@ -60,6 +60,12 @@
 #define SIGNAL_FMT "signal %d"
 #endif
 
+static my_bool non_blocking_api_enabled= 0;
+#if !defined(EMBEDDED_LIBRARY)
+#define WRAP_NONBLOCK_ENABLED non_blocking_api_enabled
+#include "../tests/nonblock-wrappers.h"
+#endif
+
 /* Use cygwin for --exec and --system before 5.0 */
 #if MYSQL_VERSION_ID < 50000
 #define USE_CYGWIN
@@ -84,7 +90,7 @@ enum {
   OPT_PS_PROTOCOL, OPT_SP_PROTOCOL, OPT_CURSOR_PROTOCOL, OPT_VIEW_PROTOCOL,
   OPT_MAX_CONNECT_RETRIES, OPT_MAX_CONNECTIONS,
   OPT_MARK_PROGRESS, OPT_LOG_DIR, OPT_TAIL_LINES,
-  OPT_GLOBAL_SUBST, OPT_MY_CONNECT_TIMEOUT
+  OPT_GLOBAL_SUBST, OPT_MY_CONNECT_TIMEOUT, OPT_NON_BLOCKING_API
 };
 
 static int record= 0, opt_sleep= -1;
@@ -305,6 +311,7 @@ enum enum_commands {
   Q_LOWERCASE,
   Q_START_TIMER, Q_END_TIMER,
   Q_CHARACTER_SET, Q_DISABLE_PS_PROTOCOL, Q_ENABLE_PS_PROTOCOL,
+  Q_ENABLE_NON_BLOCKING_API, Q_DISABLE_NON_BLOCKING_API,
   Q_DISABLE_RECONNECT, Q_ENABLE_RECONNECT,
   Q_IF,
   Q_DISABLE_PARSING, Q_ENABLE_PARSING,
@@ -386,6 +393,8 @@ const char *command_names[]=
   "character_set",
   "disable_ps_protocol",
   "enable_ps_protocol",
+  "enable_non_blocking_api",
+  "disable_non_blocking_api",
   "disable_reconnect",
   "enable_reconnect",
   "if",
@@ -5235,7 +5244,10 @@ void do_connect(struct st_command *command)
   int con_port= opt_port;
   char *con_options;
   my_bool con_ssl= 0, con_compress= 0;
-  my_bool con_pipe= 0, con_shm= 0;
+  my_bool con_pipe= 0;
+#ifdef HAVE_SMEM
+  my_bool con_shm= 0;
+#endif
   struct st_connection* con_slot;
 
   static DYNAMIC_STRING ds_connection_name;
@@ -5324,7 +5336,11 @@ void do_connect(struct st_command *command)
     else if (length == 4 && !strncmp(con_options, "PIPE", 4))
       con_pipe= 1;
     else if (length == 3 && !strncmp(con_options, "SHM", 3))
+#ifdef HAVE_SMEM
       con_shm= 1;
+#else
+    { }
+#endif
     else
       die("Illegal option to connect: %.*s", 
           (int) (end - con_options), con_options);
@@ -6146,6 +6162,10 @@ static struct my_option my_long_options[] =
    "Use prepared-statement protocol for communication.",
    &ps_protocol, &ps_protocol, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"non-blocking-api", OPT_NON_BLOCKING_API,
+   "Use the non-blocking client API for communication.",
+   &non_blocking_api_enabled, &non_blocking_api_enabled, 0,
+   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"quiet", 's', "Suppress all normal output.", &silent,
    &silent, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"record", 'r', "Record output of test_file into result file.",
@@ -8130,6 +8150,7 @@ int main(int argc, char **argv)
   next_con= connections + 1;
   
   var_set_int("$PS_PROTOCOL", ps_protocol);
+  var_set_int("$NON_BLOCKING_API", non_blocking_api_enabled);
   var_set_int("$SP_PROTOCOL", sp_protocol);
   var_set_int("$VIEW_PROTOCOL", view_protocol);
   var_set_int("$CURSOR_PROTOCOL", cursor_protocol);
@@ -8538,6 +8559,12 @@ int main(int argc, char **argv)
       case Q_ENABLE_PS_PROTOCOL:
         ps_protocol_enabled= ps_protocol;
         break;
+      case Q_DISABLE_NON_BLOCKING_API:
+        non_blocking_api_enabled= 0;
+        break;
+      case Q_ENABLE_NON_BLOCKING_API:
+        non_blocking_api_enabled= 1;
+        break;
       case Q_DISABLE_RECONNECT:
         set_reconnect(cur_con->mysql, 0);
         break;

dbug/dbug.c

@@ -404,6 +404,27 @@ static CODE_STATE *code_state(void)
   return cs;
 }
 
+void
+dbug_swap_code_state(void **code_state_store)
+{
+  CODE_STATE *cs, **cs_ptr;
+
+  if (!(cs_ptr= (CODE_STATE**) my_thread_var_dbug()))
+    return;
+  cs= *cs_ptr;
+  *cs_ptr= *code_state_store;
+  *code_state_store= cs;
+}
+
+void dbug_free_code_state(void **code_state_store)
+{
+  if (*code_state_store)
+  {
+    free(*code_state_store);
+    *code_state_store= NULL;
+  }
+}
+
 #else /* !THREAD */
 
 static CODE_STATE static_code_state=
@@ -424,6 +445,35 @@ static CODE_STATE *code_state(void)
   return &static_code_state;
 }
 
+void
+dbug_swap_code_state(void **code_state_store)
+{
+  CODE_STATE temp, *cs;
+
+  if (!(cs= *code_state_store))
+  {
+    cs= (CODE_STATE *)DbugMalloc(sizeof(*cs));
+    cs->process= db_process ? db_process : "dbug";
+    cs->func="?func";
+    cs->file="?file";
+    cs->stack=&init_settings;
+    *code_state_store= cs;
+  }
+  memcpy(&temp, cs, sizeof(*cs));
+  memcpy(cs, &static_code_state, sizeof(*cs));
+  memcpy(&static_code_state, &temp, sizeof(*cs));
+}
+
+void
+dbug_free_code_state(void **code_state_store)
+{
+  if (*code_state_store)
+  {
+    free(*code_state_store);
+    *code_state_store= NULL;
+  }
+}
+
 #define pthread_mutex_lock(A) {}
 #define pthread_mutex_unlock(A) {}
 #endif

include/Makefile.am

@@ -46,7 +46,7 @@ noinst_HEADERS =	config-win.h config-netware.h lf.h my_bit.h \
 			atomic/rwlock.h atomic/x86-gcc.h \
 			atomic/generic-msvc.h \
                         atomic/gcc_builtins.h my_libwrap.h my_stacktrace.h \
-                        wqueue.h waiting_threads.h
+                        wqueue.h waiting_threads.h my_context.h
 
 EXTRA_DIST =            mysql.h.pp mysql/plugin_auth.h.pp mysql/client_plugin.h.pp CMakeLists.txt
 

include/my_context.h

+/*
+  Copyright 2011 Kristian Nielsen
+
+  Experiments with non-blocking libmysql.
+
+  This is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 2 of the License, or
+  (at your option) any later version.
+
+  This is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/*
+  Simple API for spawning a co-routine, to be used for async libmysqlclient.
+
+  Idea is that by implementing this interface using whatever facilities are
+  available for given platform, we can use the same code for the generic
+  libmysqlclient-async code.
+
+  (This particular implementation uses Posix ucontext swapcontext().)
+*/
+
+#ifdef __WIN__
+#define MY_CONTEXT_USE_WIN32_FIBERS 1
+#elif defined(__GNUC__) && __GNUC__ >= 3 && defined(__x86_64__)
+#define MY_CONTEXT_USE_X86_64_GCC_ASM
+#elif defined(__GNUC__) && __GNUC__ >= 3 && defined(__i386__)
+#define MY_CONTEXT_USE_I386_GCC_ASM
+#else
+#define MY_CONTEXT_USE_UCONTEXT
+#endif
+
+#ifdef MY_CONTEXT_USE_WIN32_FIBERS
+struct my_context {
+  void (*user_func)(void *);
+  void *user_arg;
+  void *app_fiber;
+  void *lib_fiber;
+  int return_value;
+#ifndef DBUG_OFF
+  void *dbug_state;
+#endif
+};
+#endif
+
+
+#ifdef MY_CONTEXT_USE_UCONTEXT
+#include <ucontext.h>
+
+struct my_context {
+  void (*user_func)(void *);
+  void *user_data;
+  void *stack;
+  size_t stack_size;
+  ucontext_t base_context;
+  ucontext_t spawned_context;
+  int active;
+#ifdef HAVE_VALGRIND_VALGRIND_H
+  unsigned int valgrind_stack_id;
+#endif
+#ifndef DBUG_OFF
+  void *dbug_state;
+#endif
+};
+#endif
+
+
+#ifdef MY_CONTEXT_USE_X86_64_GCC_ASM
+#include <stdint.h>
+
+struct my_context {
+  uint64_t save[9];
+  void *stack_top;
+  void *stack_bot;
+#ifdef HAVE_VALGRIND_VALGRIND_H
+  unsigned int valgrind_stack_id;
+#endif
+#ifndef DBUG_OFF
+  void *dbug_state;
+#endif
+};
+#endif
+
+
+#ifdef MY_CONTEXT_USE_I386_GCC_ASM
+#include <stdint.h>
+
+struct my_context {
+  uint64_t save[7];
+  void *stack_top;
+  void *stack_bot;
+#ifdef HAVE_VALGRIND_VALGRIND_H
+  unsigned int valgrind_stack_id;
+#endif
+#ifndef DBUG_OFF
+  void *dbug_state;
+#endif
+};
+#endif
+
+
+/*
+  Initialize an asynchroneous context object.
+  Returns 0 on success, non-zero on failure.
+*/
+extern int my_context_init(struct my_context *c, size_t stack_size);
+
+/* Free an asynchroneous context object, deallocating any resources used. */
+extern void my_context_destroy(struct my_context *c);
+
+/*
+  Spawn an asynchroneous context. The context will run the supplied user
+  function, passing the supplied user data pointer.
+
+  The context must have been initialised with my_context_init() prior to
+  this call.
+
+  The user function may call my_context_yield(), which will cause this
+  function to return 1. Then later my_context_continue() may be called, which
+  will resume the asynchroneous context by returning from the previous
+  my_context_yield() call.
+
+  When the user function returns, this function returns 0.
+
+  In case of error, -1 is returned.
+*/
+extern int my_context_spawn(struct my_context *c, void (*f)(void *), void *d);
+
+/*
+  Suspend an asynchroneous context started with my_context_spawn.
+
+  When my_context_yield() is called, execution immediately returns from the
+  last my_context_spawn() or my_context_continue() call. Then when later
+  my_context_continue() is called, execution resumes by returning from this
+  my_context_yield() call.
+
+  Returns 0 if ok, -1 in case of error.
+*/
+extern int my_context_yield(struct my_context *c);
+
+/*
+  Resume an asynchroneous context. The context was spawned by
+  my_context_spawn(), and later suspended inside my_context_yield().
+
+  The asynchroneous context may be repeatedly suspended with
+  my_context_yield() and resumed with my_context_continue().
+
+  Each time it is suspended, this function returns 1. When the originally
+  spawned user function returns, this function returns 0.
+
+  In case of error, -1 is returned.
+*/
+extern int my_context_continue(struct my_context *c);
+
+
+struct mysql_async_context {
+  /*
+    This is set to the value that should be returned from foo_start() or
+    foo_cont() when a call is suspended.
+    It is also set to the event(s) that triggered when a suspended call is
+    resumed, eg. whether we woke up due to connection completed or timeout
+    in mysql_real_connect_cont().
+  */
+  unsigned int ret_status;
+  /*
+    This is set to the result of the whole asynchronous operation when it
+    completes. It uses a union, as different calls have different return
+    types.
+  */
+  union {
+    void *r_ptr;
+    const void *r_const_ptr;
+    int r_int;
+    my_bool r_my_bool;
+  } ret_result;
+  /*
+    The timeout value, for suspended calls that need to wake up on a timeout
+    (eg. mysql_real_connect_start().
+  */
+  unsigned int timeout_value;
+  /*
+    This flag is set when we are executing inside some asynchronous call
+    foo_start() or foo_cont(). It is used to decide whether to use the
+    synchronous or asynchronous version of calls that may block such as
+    recv().
+
+    Note that this flag is not set when a call is suspended, eg. after
+    returning from foo_start() and before re-entering foo_cont().
+  */
+  my_bool active;
+  /*
+    This flag is set when an asynchronous operation is in progress, but
+    suspended. Ie. it is set when foo_start() or foo_cont() returns because
+    the operation needs to block, suspending the operation.
+
+    It is used to give an error (rather than crash) if the application
+    attempts to call some foo_cont() method when no suspended operation foo is
+    in progress.
+  */
+  my_bool suspended;
+  /*
+    If non-NULL, this is a pointer to a callback hook that will be invoked with
+    the user data argument just before the context is suspended, and just after
+    it is resumed.
+  */
+  void (*suspend_resume_hook)(my_bool suspend, void *user_data);
+  void *suspend_resume_hook_user_data;
+  /*
+    This is used to save the execution contexts so that we can suspend an
+    operation and switch back to the application context, to resume the
+    suspended context later when the application re-invokes us with
+    foo_cont().
+  */
+  struct my_context async_context;
+};

include/my_dbug.h

@@ -83,6 +83,8 @@ extern	void _db_lock_file_(void);
 extern	void _db_unlock_file_(void);
 extern FILE *_db_fp_(void);
 extern  void _db_flush_();
+extern void dbug_swap_code_state(void **code_state_store);
+extern void dbug_free_code_state(void **code_state_store);
 
 #ifdef __cplusplus
 

include/sql_common.h

@@ -26,11 +26,18 @@ extern const char	*unknown_sqlstate;
 extern const char	*cant_connect_sqlstate;
 extern const char	*not_error_sqlstate;
 
+
+struct mysql_async_context;
+
 struct st_mysql_options_extention {
   char *plugin_dir;
   char *default_auth;
 };
 
+struct st_mysql_extension {
+  struct mysql_async_context *async_context;
+};
+
 typedef struct st_mysql_methods
 {
   my_bool (*read_query_result)(MYSQL *mysql);
@@ -102,6 +109,10 @@ void mysql_client_plugin_deinit();
 struct st_mysql_client_plugin;
 extern struct st_mysql_client_plugin *mysql_client_builtins[];
 
+/* Non-blocking client API. */
+void my_context_install_suspend_resume_hook(struct mysql_async_context *b,
+                                            void (*)(my_bool, void *), void *);
+
 #ifdef	__cplusplus
 }
 #endif

include/violite.h

@@ -194,6 +194,8 @@ struct st_vio
   char                  *read_pos;      /* start of unfetched data in the
                                            read buffer */
   char                  *read_end;      /* end of unfetched data */
+  struct mysql_async_context *async_context; /* For non-blocking API */
+  uint read_timeout, write_timeout;
   /* function pointers. They are similar for socket/SSL/whatever */
   void    (*viodelete)(Vio*);
   int     (*vioerrno)(Vio*);

libmysql/CMakeLists.txt

@@ -95,7 +95,8 @@ SET(CLIENT_SOURCES   ../mysys/array.c ../strings/bchange.c ../strings/bmove.c
                      ../strings/strtoll.c ../strings/strtoull.c ../strings/strxmov.c ../strings/strxnmov.c 
                      ../mysys/thr_mutex.c ../mysys/typelib.c ../vio/vio.c ../vio/viosocket.c 
                      ../vio/viossl.c ../vio/viosslfactories.c ../strings/xml.c ../mysys/mf_qsort.c
-                     ../mysys/my_getsystime.c ../mysys/my_sync.c ../sql-common/client_plugin.c ${LIB_SOURCES})
+                     ../mysys/my_getsystime.c ../mysys/my_sync.c ../sql-common/client_plugin.c
+		     ../sql-common/mysql_async.c ../mysys/my_context.c ${LIB_SOURCES})
 
 # Need to set USE_TLS for building the DLL, since __declspec(thread)
 # approach to thread local storage does not work properly in DLLs.

libmysql/Makefile.shared

@@ -71,9 +71,10 @@ mysysobjects1 =		my_init.lo my_static.lo my_malloc.lo my_realloc.lo \
                         mf_iocache2.lo my_seek.lo my_sleep.lo \
 			my_pread.lo mf_cache.lo md5.lo sha1.lo my_rnd.lo \
 			my_getopt.lo my_port.lo \
-                        my_rename.lo my_chsize.lo my_sync.lo my_getsystime.lo
+                        my_rename.lo my_chsize.lo my_sync.lo my_getsystime.lo \
+			my_context.lo
 sqlobjects =		net.lo
-sql_cmn_objects =	pack.lo client.lo my_time.lo client_plugin.lo
+sql_cmn_objects =	pack.lo client.lo my_time.lo client_plugin.lo mysql_async.lo
 
 # Not needed in the minimum library
 mysysobjects2 =		my_lib.lo mf_qsort.lo

libmysqld/CMakeLists.txt

@@ -104,6 +104,7 @@ SET(LIBMYSQLD_SOURCES  libmysqld.c emb_qcache.cc lib_sql.cc
            ../sql-common/my_user.c ../sql-common/pack.c
            ../sql/password.c ../sql/discover.cc ../sql/derror.cc 
            ../sql/field.cc ../sql/field_conv.cc ../sql-common/client_plugin.c
+           ../sql-common/mysql_async.c
            ../sql/filesort.cc ../sql/gstream.cc ../sql/ha_partition.cc 
            ../sql/handler.cc ../sql/hash_filo.cc ../sql/hostname.cc 
            ../sql/init.cc ../sql/item_buff.cc ../sql/item_cmpfunc.cc 

libmysqld/Makefile.am

@@ -41,7 +41,7 @@ pkglib_LTLIBRARIES =	libmysqld.la
 SUBDIRS =		. examples
 libmysqld_sources=	libmysqld.c lib_sql.cc emb_qcache.cc
 libmysqlsources =	errmsg.c get_password.c libmysql.c client.c pack.c \
-                        my_time.c client_plugin.c
+                        my_time.c client_plugin.c mysql_async.c
 
 noinst_HEADERS =	embedded_priv.h emb_qcache.h
 

mysql-test/mysql-test-run.pl

@@ -185,6 +185,7 @@ my $opt_ps_protocol;
 my $opt_sp_protocol;
 my $opt_cursor_protocol;
 my $opt_view_protocol;
+my $opt_non_blocking_api;
 
 our $opt_debug;
 our $opt_debug_server;
@@ -964,6 +965,7 @@ sub command_line_setup {
              'sp-protocol'              => \$opt_sp_protocol,
              'view-protocol'            => \$opt_view_protocol,
              'cursor-protocol'          => \$opt_cursor_protocol,
+             'non-blocking-api'         => \$opt_non_blocking_api,
              'ssl|with-openssl'         => \$opt_ssl,
              'skip-ssl'                 => \$opt_skip_ssl,
              'compress'                 => \$opt_compress,
@@ -5468,6 +5470,11 @@ sub start_mysqltest ($) {
     mtr_add_arg($args, "--cursor-protocol");
   }
 
+  if ( $opt_non_blocking_api )
+  {
+    mtr_add_arg($args, "--non-blocking-api");
+  }
+
   if ( $opt_strace_client )
   {
     $exe=  $opt_strace_client || "strace";
@@ -5915,6 +5922,7 @@ Options to control what engine/variation to run
                         (implies --ps-protocol)
   view-protocol         Create a view to execute all non updating queries
   sp-protocol           Create a stored procedure to execute all queries
+  non-blocking-api      Use the non-blocking client API
   compress              Use the compressed protocol between client and server
   ssl                   Use ssl protocol between client and server
   skip-ssl              Dont start server with support for ssl connections

mysql-test/r/mysql_client_test_nonblock.result

+SET @old_general_log= @@global.general_log;
+SET @old_slow_query_log= @@global.slow_query_log;
+ok
+SET @@global.general_log= @old_general_log;
+SET @@global.slow_query_log= @old_slow_query_log;

mysql-test/r/non_blocking_api.result

+CREATE TABLE t1 (a INT PRIMARY KEY);
+INSERT INTO t1 VALUES (1);
+SELECT * FROM t1;
+a
+1
+SELECT * FROM t1;
+a
+1
+DROP TABLE t1;

mysql-test/t/mysql_client_test_nonblock-master.opt

+--log=$MYSQLTEST_VARDIR/log/master.log --log-output=FILE,TABLE

mysql-test/t/mysql_client_test_nonblock.test

+# This runs the mysql_client_test using the non-blocking API.
+
+# This test should work in embedded server after we fix mysqltest
+-- source include/not_embedded.inc
+
+SET @old_general_log= @@global.general_log;
+SET @old_slow_query_log= @@global.slow_query_log;
+
+# We run with different binaries for normal and --embedded-server
+#
+# If this test fails with "command "$MYSQL_CLIENT_TEST" failed",
+# you should either run mysql_client_test separartely against a running
+# server or run mysql-test-run --debug mysql_client_test and check
+# var/log/mysql_client_test.trace
+
+--exec echo "$MYSQL_CLIENT_TEST --non-blocking-api" > $MYSQLTEST_VARDIR/log/mysql_client_test.out.log 2>&1
+--exec $MYSQL_CLIENT_TEST --non-blocking-api --getopt-ll-test=25600M >> $MYSQLTEST_VARDIR/log/mysql_client_test.out.log 2>&1
+
+# End of 4.1 tests
+echo ok;
+
+SET @@global.general_log= @old_general_log;
+SET @@global.slow_query_log= @old_slow_query_log;

mysql-test/t/named_pipe.test

@@ -2,6 +2,8 @@
 # in order  to optimize things we skip this test on all
 # other platforms
 --source include/windows.inc
+# Named pipe does not support the non-blocking API.
+--disable_non_blocking_api
 
 # Only run this test if named pipe is avaliable
 let $nmp= query_get_value("SHOW VARIABLES LIKE 'named_pipe'", Value, 1);

mysql-test/t/non_blocking_api.test

+# Test mixing the use of blocking and non-blocking API in a single connection.
+
+--enable_non_blocking_api
+connect (con_nonblock,localhost,root,,test);
+--disable_non_blocking_api
+connect (con_normal,localhost,root,,test);
+
+connection con_nonblock;
+CREATE TABLE t1 (a INT PRIMARY KEY);
+--enable_non_blocking_api
+INSERT INTO t1 VALUES (1);
+--disable_non_blocking_api
+SELECT * FROM t1;
+--enable_non_blocking_api
+SELECT * FROM t1;
+
+connection con_normal;
+DROP TABLE t1;

mysys/CMakeLists.txt

@@ -44,6 +44,7 @@ SET(MYSYS_SOURCES  array.c charset-def.c charset.c checksum.c default.c default_
                                 lf_alloc-pin.c lf_dynarray.c lf_hash.c
                                 my_atomic.c my_getncpus.c my_rnd.c
                                 my_uuid.c wqueue.c waiting_threads.c
+                                my_context.c
 )
 
 IF(NOT SOURCE_SUBLIBS)

mysys/Makefile.am

@@ -58,7 +58,7 @@ libmysys_la_SOURCES =	my_init.c my_getwd.c mf_getdate.c my_mmap.c \
 			my_compare.c my_netware.c my_largepage.c \
 			my_memmem.c stacktrace.c \
 			my_windac.c my_access.c base64.c my_libwrap.c \
-		        wqueue.c
+		        wqueue.c my_context.c
 libmysys_la_LDFLAGS =	$(AM_LDFLAGS) @WRAPLIBS@
 libmysys_la_LIBADD =	$(ZLIB_LIBS)
 

server-tools/instance-manager/CMakeLists.txt

@@ -26,7 +26,8 @@ ADD_EXECUTABLE(mysqlmanager buffer.cc command.cc commands.cc guardian.cc instanc
                             user_management_commands.cc ../../mysys/my_rnd.c
                             ../../sql/net_serv.cc ../../sql-common/pack.c ../../sql/password.c
                             ../../sql/sql_state.c ../../sql-common/client.c ../../libmysql/get_password.c
-                            ../../libmysql/errmsg.c ../../sql-common/client_plugin.c)
+                            ../../libmysql/errmsg.c ../../sql-common/client_plugin.c
+                            ../../sql-common/mysql_async.c)
 
 ADD_DEPENDENCIES(mysqlmanager GenError)
 TARGET_LINK_LIBRARIES(mysqlmanager debug dbug mysys strings taocrypt vio yassl zlib wsock32)

server-tools/instance-manager/Makefile.am

@@ -47,6 +47,7 @@ libnet_a_LIBADD= $(top_builddir)/sql/password.$(OBJEXT) \
 	$(top_builddir)/sql/sql_state.$(OBJEXT) \
 	$(top_builddir)/sql/mini_client_errors.$(OBJEXT)\
 	$(top_builddir)/sql/client.$(OBJEXT) \
+	$(top_builddir)/sql/mysql_async.$(OBJEXT) \
 	$(top_builddir)/sql/client_plugin.$(OBJEXT)
 
 CLEANFILES= net_serv.cc client_settings.h

sql-common/Makefile.am

@@ -14,4 +14,5 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 ## Process this file with automake to create Makefile.in
-EXTRA_DIST = client.c pack.c my_time.c my_user.c client_plugin.c
+EXTRA_DIST = client.c pack.c my_time.c my_user.c client_plugin.c \
+	     mysql_async.c

sql/CMakeLists.txt

@@ -53,6 +53,7 @@ SET (SQL_SOURCE
                log_event_old.cc rpl_record_old.cc
                message.h mf_iocache.cc my_decimal.cc ../sql-common/my_time.c
                mysqld.cc net_serv.cc ../sql-common/client_plugin.c 
+	       ../sql-common/mysql_async.c
                nt_servc.cc nt_servc.h opt_range.cc opt_range.h opt_sum.cc 
                ../sql-common/pack.c parse_file.cc password.c procedure.cc 
                protocol.cc records.cc repl_failsafe.cc rpl_filter.cc set_var.cc 

sql/Makefile.am

@@ -127,7 +127,8 @@ mysqld_SOURCES =	sql_lex.cc sql_handler.cc sql_partition.cc \
 			sql_servers.cc event_parse_data.cc \
                         opt_table_elimination.cc create_options.cc
 
-nodist_mysqld_SOURCES =	mini_client_errors.c pack.c client.c my_time.c my_user.c client_plugin.c
+nodist_mysqld_SOURCES =	mini_client_errors.c pack.c client.c my_time.c \
+			my_user.c client_plugin.c mysql_async.c
 
 libndb_la_CPPFLAGS=	@ndbcluster_includes@
 libndb_la_SOURCES=	ha_ndbcluster.cc \
@@ -169,6 +170,8 @@ link_sources:
 	@LN_CP_F@ $(top_srcdir)/sql-common/pack.c pack.c
 	rm -f client.c
 	@LN_CP_F@ $(top_srcdir)/sql-common/client.c client.c
+	rm -f mysql_async.c
+	@LN_CP_F@ $(top_srcdir)/sql-common/mysql_async.c mysql_async.c
 	rm -f client_plugin.c
 	@LN_CP_F@ $(top_srcdir)/sql-common/client_plugin.c client_plugin.c
 	rm -f my_time.c

tests/Makefile.am

@@ -24,6 +24,7 @@ else
 LIBMYSQLCLIENT_LA =		$(top_builddir)/libmysql/libmysqlclient.la
 endif
 
+noinst_HEADERS = nonblock-wrappers.h
 EXTRA_DIST =		auto_increment.res auto_increment.tst \
 			function.res function.tst lock_test.pl lock_test.res \
 			export.pl big_record.pl \
@@ -36,6 +37,9 @@ EXTRA_DIST =		auto_increment.res auto_increment.tst \
 
 bin_PROGRAMS =		mysql_client_test
 noinst_PROGRAMS =	insert_test select_test thread_test bug25714
+if HAVE_LIBEVENT
+noinst_PROGRAMS +=	async_queries
+endif
 
 INCLUDES =		-I$(top_builddir)/include -I$(top_srcdir)/include \
 			$(openssl_includes)
@@ -52,6 +56,10 @@ select_test_SOURCES=       select_test.c
 insert_test_DEPENDENCIES=	$(LIBRARIES) $(pkglib_LTLIBRARIES)
 select_test_DEPENDENCIES=	$(LIBRARIES) $(pkglib_LTLIBRARIES)
 
+async_queries_SOURCES=	async_queries.c
+async_queries_CFLAGS=	$(AM_CFLAGS) @libevent_includes@
+async_queries_LDADD=	$(LDADD) @libevent_libs@
+
 bug25714_SOURCES=          bug25714.c
 bug25714_DEPENDENCIES=     $(LIBRARIES) $(pkglib_LTLIBRARIES)
 

tests/check_async_queries.pl

+#! /usr/bin/perl
+
+# Read the output of async_queries.c. Run the queries again serially, using
+# the normal (not asynchronous) API. Compare the two results for correctness.
+
+use strict;
+use warnings;
+
+use DBI;
+
+my $D= [];
+
+die "Usage: $0 <host> <user> <password> <database>\n"
+    unless @ARGV == 4;
+
+my $dbh= DBI->connect("DBI:mysql:database=$ARGV[3];host=$ARGV[0]",
+                      $ARGV[1], $ARGV[2],
+                      { RaiseError => 1, PrintError => 0 });
+
+while (<STDIN>) {
+  chomp;
+  if (/^([0-9]+) ! (.*);$/) {
+    my ($index, $query)= ($1, $2);
+    $D->[$index]= { QUERY => $query, OUTPUT => [] };
+  } elsif (/^([0-9]+) - (.*)$/) {
+    my ($index, $data)= ($1, $2);
+    push @{$D->[$index]{OUTPUT}}, $data;
+  } elsif (/^([0-9]+) \| Error: (.*)$/) {
+    my ($index, $errmsg)= ($1, $2);
+    my $rows;
+    my $res= eval {
+      my $stm= $dbh->prepare($D->[$index]{QUERY});
+      $stm->execute();
+      $rows= $stm->fetchall_arrayref();
+      1;
+    };
+    if ($res) {
+      die "Query $index succeeded, but should have failed with error.\nquery=$D->[$index]{QUERY}\nerror=$errmsg\n";
+    }
+    my $errmsg2= $@;
+    if ($errmsg2 =~ /^DBD::.*failed: (.*) at .*$/s) {
+      $errmsg2= $1;
+    } else {
+      die "Unexpected DBD error message format: '$errmsg2'\n";
+    }
+    if ($errmsg2 ne $errmsg) {
+      die "Query $index failed with different error message\nquery=$D->[$index]{QUERY}\nerror1=$errmsg\nerror2=$errmsg2\n";
+    }
+    print "OK $index\n";
+    delete $D->[$index];
+  } elsif (/^([0-9]+) \| EOF$/) {
+    my $index= $1;
+    my $rows;
+    my $res= eval {
+      my $stm= $dbh->prepare($D->[$index]{QUERY});
+      $stm->execute();
+      $rows= $stm->fetchall_arrayref();
+      1;
+    };
+    if (!$res) {
+      die "Query $index failed, but should have succeeded.\nquery=$D->[$index]{QUERY}\nerror=$@\n";
+    }
+    my $result_string= join("\n", sort @{$D->[$index]{OUTPUT}});
+    my $result_string2= join("\n", sort(map(join("\t", map((defined($_) ? $_ : "(null)"), @$_)), @$rows)));
+    if ($result_string ne $result_string2) {
+      die "Query $index result difference.\nquery=$D->[$index]{QUERY}\noutput1=\n$$result_string\noutput2=\n$result_string2\n";
+    }
+    delete $D->[$index];
+  } else {
+    die "Unexpected line: '$_'\n";
+  }
+}
+$dbh->disconnect();

tests/mysql_client_test.c

@@ -36,6 +36,18 @@
 #include <my_handler.h>
 #include <sql_common.h>
 
+/*
+  If non_blocking_api_enabled is true, we will re-define all the blocking
+  API functions as wrappers that call the corresponding non-blocking API
+  and use poll()/select() to wait for them to complete. This way we can get
+  a good coverage testing of the non-blocking API as well.
+*/
+static my_bool non_blocking_api_enabled= 0;
+#if !defined(EMBEDDED_LIBRARY)
+#define WRAP_NONBLOCK_ENABLED non_blocking_api_enabled
+#include "nonblock-wrappers.h"
+#endif
+
 #define VER "2.1"
 #define MAX_TEST_QUERY_LENGTH 300 /* MAX QUERY BUFFER LENGTH */
 #define MAX_KEY MAX_INDEXES
@@ -18614,6 +18626,10 @@ static struct my_option client_test_long_options[] =
 #endif
   {"vardir", 'v', "Data dir for tests.", (char**) &opt_vardir,
    (char**) &opt_vardir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"non-blocking-api", 'n',
+   "Use the non-blocking client API for communication.",
+   &non_blocking_api_enabled, &non_blocking_api_enabled, 0,
+   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"getopt-ll-test", 'g', "Option for testing bug in getopt library",
    &opt_getopt_ll_test, &opt_getopt_ll_test, 0,
    GET_LL, REQUIRED_ARG, 0, 0, LONGLONG_MAX, 0, 0, 0},