Bug#35272: @@global.key_buffer_size = 4294967295 let the server crash

When trying to get the requested amount of memory for the keybuffer,
the out of memory could be signaled if one of the tentative allocations
fail. Later the server would crash (debug assert) when trying to send
a ok packet with a error set.

The solution is only to signal the error if all tentative allocations
for the keybuffer fail.


mysql-test/r/key_cache.result:
  Add test case result for Bug#35272
mysql-test/t/key_cache.test:
  Add test case for Bug#35272
mysys/mf_keycache.c:
  Don't set error on my_large_malloc if allocation fails.
  Set the error if all tentative allocations failed.

mysql-test/r/key_cache.result

@@ -368,3 +368,8 @@ Variable_name	Value
 key_cache_block_size	1536
 SET GLOBAL key_cache_block_size= @bug28478_key_cache_block_size;
 DROP TABLE t1;
+SET @save_key_buffer = @@global.key_buffer_size;
+SET @@global.key_buffer_size = 4294967295;
+SET @@global.key_buffer_size = 9223372036854775807;
+SET @@global.key_buffer_size = @save_key_buffer;
+End of 5.1 tests

mysql-test/t/key_cache.test

@@ -247,3 +247,28 @@ SET GLOBAL key_cache_block_size= @bug28478_key_cache_block_size;
 DROP TABLE t1;
 
 # End of 4.1 tests
+
+#
+# Bug#35272: @@global.key_buffer_size = 4294967295 let the server crash
+#
+
+SET @save_key_buffer = @@global.key_buffer_size;
+
+# Wee try to force Out Of Memory here. key_buffer_size is ULL, so
+# on a 32 bit machine, 4GB is the most we can ask for before the
+# server complains about value/variable mismatch. At the off chance
+# of one of our 64-bit machines actually offering us 4GB, we also
+# accept "no error" (in addition to the expected "out of memory").
+--error 0,ER_OUTOFMEMORY
+SET @@global.key_buffer_size = 4294967295;
+
+# on 32-bit, we get "out of range", on 64-bit, "out of memory".
+--error 0,ER_WRONG_ARGUMENTS,ER_OUTOFMEMORY
+--disable_warnings
+SET @@global.key_buffer_size = 9223372036854775807;
+--enable_warnings
+
+# restore normal value, just in case we got the 4GB or something.
+SET @@global.key_buffer_size = @save_key_buffer;
+
+--echo End of 5.1 tests

mysys/mf_keycache.c

@@ -102,6 +102,7 @@
 */
 
 #include "mysys_priv.h"
+#include "mysys_err.h"
 #include <keycache.h>
 #include "my_static.h"
 #include <m_string.h>
@@ -430,7 +431,7 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
       /* Allocate memory for cache page buffers */
       if ((keycache->block_mem=
 	   my_large_malloc((size_t) blocks * keycache->key_cache_block_size,
-			  MYF(MY_WME))))
+			  MYF(0))))
       {
         /*
 	  Allocate memory for blocks, hash_links and hash entries;
@@ -445,6 +446,7 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
       if (blocks < 8)
       {
         my_errno= ENOMEM;
+        my_error(EE_OUTOFMEMORY, MYF(0), blocks * keycache->key_cache_block_size);
         goto err;
       }
       blocks= blocks / 4*3;