Bug #27119 server crash with integer division by zero during filesort on huge result

Added checks to detect integer overflow and fixed other bugs on the error path.


myisam/sort.c:
  Replaced a break statement with a goto statement so that a failure will instead break
sql/filesort.cc:
  Fixed an allocation routine to detect integer overflow, and as an optimization a check that prevents the number of buffpeks being larger than can possibly fit into memory.
      
  Fixed several unchecked error codes.
      
  Changed an index variable from int to uint to the match the type of the variable it's
  being compared with.
      
      Replaced a break statement with a goto statement so that a failure will instead break
  out of the higher level while-loop, instead of just the nested for-loop.

myisam/sort.c

@@ -773,7 +773,7 @@ static int NEAR_F merge_many_buff(MI_SORT_PARAM *info, uint keys,
     {
       if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
                         buffpek+i,buffpek+i+MERGEBUFF-1))
-        break; /* purecov: inspected */
+        goto cleanup;
     }
     if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
                       buffpek+i,buffpek+ *maxbuffer))
@@ -783,6 +783,7 @@ static int NEAR_F merge_many_buff(MI_SORT_PARAM *info, uint keys,
     temp=from_file; from_file=to_file; to_file=temp;
     *maxbuffer= (int) (lastbuff-buffpek)-1;
   }
+cleanup:
   close_cached_file(to_file);                   /* This holds old result */
   if (to_file == t_file)
     *t_file=t_file2;                            /* Copy result file */

sql/filesort.cc

@@ -249,7 +249,8 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
 	open_cached_file(outfile,mysql_tmpdir,TEMP_PREFIX,READ_RECORD_BUFFER,
 			  MYF(MY_WME)))
       goto err;
-    reinit_io_cache(outfile,WRITE_CACHE,0L,0,0);
+    if (reinit_io_cache(outfile,WRITE_CACHE,0L,0,0))
+      goto err;
 
     /*
       Use also the space previously used by string pointers in sort_buffer
@@ -369,6 +370,8 @@ static BUFFPEK *read_buffpek_from_file(IO_CACHE *buffpek_pointers, uint count)
   ulong length;
   BUFFPEK *tmp;
   DBUG_ENTER("read_buffpek_from_file");
+  if (count > ULONG_MAX/sizeof(BUFFPEK))
+    return 0; /* sizeof(BUFFPEK)*count will overflow */
   tmp=(BUFFPEK*) my_malloc(length=sizeof(BUFFPEK)*count, MYF(MY_WME));
   if (tmp)
   {
@@ -600,6 +603,9 @@ write_keys(SORTPARAM *param, register uchar **sort_keys, uint count,
       open_cached_file(tempfile, mysql_tmpdir, TEMP_PREFIX, DISK_BUFFER_SIZE,
                        MYF(MY_WME)))
     goto err;                                   /* purecov: inspected */
+  /* check we won't have more buffpeks than we can possibly keep in memory */
+  if (my_b_tell(buffpek_pointers) + sizeof(BUFFPEK) > (ulonglong)ULONG_MAX)
+    goto err;
   buffpek.file_pos= my_b_tell(tempfile);
   if ((ha_rows) count > param->max_rows)
     count=(uint) param->max_rows;               /* purecov: inspected */
@@ -907,7 +913,7 @@ static bool save_index(SORTPARAM *param, uchar **sort_keys, uint count,
 int merge_many_buff(SORTPARAM *param, uchar *sort_buffer,
 		    BUFFPEK *buffpek, uint *maxbuffer, IO_CACHE *t_file)
 {
-  register int i;
+  register uint i;
   IO_CACHE t_file2,*from_file,*to_file,*temp;
   BUFFPEK *lastbuff;
   DBUG_ENTER("merge_many_buff");
@@ -922,14 +928,16 @@ int merge_many_buff(SORTPARAM *param, uchar *sort_buffer,
   from_file= t_file ; to_file= &t_file2;
   while (*maxbuffer >= MERGEBUFF2)
   {
-    reinit_io_cache(from_file,READ_CACHE,0L,0,0);
-    reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
+    if (reinit_io_cache(from_file,READ_CACHE,0L,0,0))
+      goto cleanup;
+    if (reinit_io_cache(to_file,WRITE_CACHE,0L,0,0))
+      goto cleanup;
     lastbuff=buffpek;
-    for (i=0 ; i <= (int) *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
+    for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
     {
       if (merge_buffers(param,from_file,to_file,sort_buffer,lastbuff++,
 			buffpek+i,buffpek+i+MERGEBUFF-1,0))
-	break;					/* purecov: inspected */
+      goto cleanup;
     }
     if (merge_buffers(param,from_file,to_file,sort_buffer,lastbuff++,
 		      buffpek+i,buffpek+ *maxbuffer,0))
@@ -941,6 +949,7 @@ int merge_many_buff(SORTPARAM *param, uchar *sort_buffer,
     setup_io_cache(to_file);
     *maxbuffer= (uint) (lastbuff-buffpek)-1;
   }
+cleanup:
   close_cached_file(to_file);			// This holds old result
   if (to_file == t_file)
   {