Modified read buffer to see if performance difference existed.

Re-enabled 8gig unit test. 

storage/archive/archive_test.c

@@ -217,14 +217,13 @@ int main(int argc, char *argv[])
 
   azclose(&writer_handle);
   azclose(&reader_handle);
-  exit(0);
   unlink(TEST_FILENAME);
 
   /* Start size tests */
   printf("About to run 2/4/8 gig tests now, you may want to hit CTRL-C\n");
-  size_test(TWOGIG, 2097152L);
-  size_test(FOURGIG, 4194304L);
-  size_test(EIGHTGIG, 8388608L);
+  size_test(TWOGIG, 2088992L);
+  size_test(FOURGIG, 4177984L);
+  size_test(EIGHTGIG, 8355968L);
 
   return 0;
 }
@@ -234,6 +233,7 @@ int size_test(unsigned long long length, unsigned long long rows_to_test_for)
   azio_stream writer_handle, reader_handle;
   unsigned long long write_length;
   unsigned long long read_length= 0;
+  unsigned long long count;
   unsigned int ret;
   char buffer[BUFFER_LEN];
   int error;
@@ -244,8 +244,10 @@ int size_test(unsigned long long length, unsigned long long rows_to_test_for)
     return 0;
   }
 
-  for (write_length= 0; write_length < length ; write_length+= ret)
+  for (count= 0, write_length= 0; write_length < length ; 
+       write_length+= ret)
   {
+    count++;
     ret= azwrite(&writer_handle, test_string, BUFFER_LEN);
     if (ret != BUFFER_LEN)
     {
@@ -257,7 +259,7 @@ int size_test(unsigned long long length, unsigned long long rows_to_test_for)
       azflush(&writer_handle,  Z_SYNC_FLUSH);
     }
   }
-  assert(write_length == length);
+  assert(write_length != count * BUFFER_LEN); /* Number of rows time BUFFER_LEN */
   azflush(&writer_handle,  Z_SYNC_FLUSH);
 
   printf("Reading back data\n");
@@ -279,7 +281,7 @@ int size_test(unsigned long long length, unsigned long long rows_to_test_for)
     }
   }
 
-  assert(read_length == length);
+  assert(read_length == write_length);
   assert(writer_handle.rows == rows_to_test_for);
   azclose(&writer_handle);
   azclose(&reader_handle);

storage/archive/azio.c

@@ -55,8 +55,8 @@ int az_open (azio_stream *s, const char *path, int Flags, File fd)
   s->stream.zalloc = (alloc_func)0;
   s->stream.zfree = (free_func)0;
   s->stream.opaque = (voidpf)0;
-  memset(s->inbuf, 0, AZ_BUFSIZE);
-  memset(s->outbuf, 0, AZ_BUFSIZE);
+  memset(s->inbuf, 0, AZ_BUFSIZE_READ);
+  memset(s->outbuf, 0, AZ_BUFSIZE_WRITE);
   s->stream.next_in = s->inbuf;
   s->stream.next_out = s->outbuf;
   s->stream.avail_in = s->stream.avail_out = 0;
@@ -109,7 +109,7 @@ int az_open (azio_stream *s, const char *path, int Flags, File fd)
       return Z_NULL;
     }
   }
-  s->stream.avail_out = AZ_BUFSIZE;
+  s->stream.avail_out = AZ_BUFSIZE_WRITE;
 
   errno = 0;
   s->file = fd < 0 ? my_open(path, Flags, MYF(0)) : fd;
@@ -159,7 +159,7 @@ void write_header(azio_stream *s)
   char buffer[AZHEADER_SIZE + AZMETA_BUFFER_SIZE];
   char *ptr= buffer;
 
-  s->block_size= AZ_BUFSIZE;
+  s->block_size= AZ_BUFSIZE_WRITE;
   s->version = (unsigned char)az_magic[1];
   s->minor_version = (unsigned char)az_magic[2];
 
@@ -224,7 +224,7 @@ int get_byte(s)
   if (s->stream.avail_in == 0) 
   {
     errno = 0;
-    s->stream.avail_in = my_read(s->file, (byte *)s->inbuf, AZ_BUFSIZE, MYF(0));
+    s->stream.avail_in = my_read(s->file, (byte *)s->inbuf, AZ_BUFSIZE_READ, MYF(0));
     if (s->stream.avail_in == 0) 
     {
       s->z_eof = 1;
@@ -260,7 +260,7 @@ void check_header(azio_stream *s)
   if (len < 2) {
     if (len) s->inbuf[0] = s->stream.next_in[0];
     errno = 0;
-    len = (uInt)my_read(s->file, (byte *)s->inbuf + len, AZ_BUFSIZE >> len, MYF(0));
+    len = (uInt)my_read(s->file, (byte *)s->inbuf + len, AZ_BUFSIZE_READ >> len, MYF(0));
     if (len == 0) s->z_err = Z_ERRNO;
     s->stream.avail_in += len;
     s->stream.next_in = s->inbuf;
@@ -455,7 +455,7 @@ unsigned int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned int len, int *
     if (s->stream.avail_in == 0 && !s->z_eof) {
 
       errno = 0;
-      s->stream.avail_in = (uInt)my_read(s->file, (byte *)s->inbuf, AZ_BUFSIZE, MYF(0));
+      s->stream.avail_in = (uInt)my_read(s->file, (byte *)s->inbuf, AZ_BUFSIZE_READ, MYF(0));
       if (s->stream.avail_in == 0) 
       {
         s->z_eof = 1;
@@ -522,12 +522,13 @@ unsigned int azwrite (azio_stream *s, voidpc buf, unsigned int len)
     {
 
       s->stream.next_out = s->outbuf;
-      if (my_write(s->file, (byte *)s->outbuf, AZ_BUFSIZE, MYF(0)) != AZ_BUFSIZE) 
+      if (my_write(s->file, (byte *)s->outbuf, AZ_BUFSIZE_WRITE, 
+                   MYF(0)) != AZ_BUFSIZE_WRITE) 
       {
         s->z_err = Z_ERRNO;
         break;
       }
-      s->stream.avail_out = AZ_BUFSIZE;
+      s->stream.avail_out = AZ_BUFSIZE_WRITE;
     }
     s->in += s->stream.avail_in;
     s->out += s->stream.avail_out;
@@ -563,7 +564,7 @@ int do_flush (azio_stream *s, int flush)
 
   for (;;) 
   {
-    len = AZ_BUFSIZE - s->stream.avail_out;
+    len = AZ_BUFSIZE_WRITE - s->stream.avail_out;
 
     if (len != 0) 
     {
@@ -574,7 +575,7 @@ int do_flush (azio_stream *s, int flush)
         return Z_ERRNO;
       }
       s->stream.next_out = s->outbuf;
-      s->stream.avail_out = AZ_BUFSIZE;
+      s->stream.avail_out = AZ_BUFSIZE_WRITE;
     }
     if (done) break;
     s->out += s->stream.avail_out;
@@ -675,8 +676,8 @@ my_off_t azseek (s, offset, whence)
     /* There was a zmemzero here if inbuf was null -Brian */
     while (offset > 0)  
     {
-      uInt size = AZ_BUFSIZE;
-      if (offset < AZ_BUFSIZE) size = (uInt)offset;
+      uInt size = AZ_BUFSIZE_WRITE;
+      if (offset < AZ_BUFSIZE_WRITE) size = (uInt)offset;
 
       size = azwrite(s, s->inbuf, size);
       if (size == 0) return -1L;
@@ -719,8 +720,8 @@ my_off_t azseek (s, offset, whence)
   }
   while (offset > 0)  {
     int error;
-    unsigned int size = AZ_BUFSIZE;
-    if (offset < AZ_BUFSIZE) size = (int)offset;
+    unsigned int size = AZ_BUFSIZE_READ;
+    if (offset < AZ_BUFSIZE_READ) size = (int)offset;
 
     size = azread(s, s->outbuf, size, &error);
     if (error <= 0) return -1L;

storage/archive/azlib.h

@@ -196,7 +196,8 @@ extern "C" {
 /* The deflate compression method (the only one supported in this version) */
 
 #define Z_NULL  0  /* for initializing zalloc, zfree, opaque */
-#define AZ_BUFSIZE 16384
+#define AZ_BUFSIZE_READ 524288
+#define AZ_BUFSIZE_WRITE 16384
 
 
 typedef struct azio_stream {
@@ -204,8 +205,8 @@ typedef struct azio_stream {
   int      z_err;   /* error code for last stream operation */
   int      z_eof;   /* set if end of input file */
   File     file;   /* .gz file */
-  Byte     inbuf[AZ_BUFSIZE];  /* input buffer */
-  Byte     outbuf[AZ_BUFSIZE]; /* output buffer */
+  Byte     inbuf[AZ_BUFSIZE_READ];  /* input buffer */
+  Byte     outbuf[AZ_BUFSIZE_WRITE]; /* output buffer */
   uLong    crc;     /* crc32 of uncompressed data */
   char     *msg;    /* error message */
   int      transparent; /* 1 if input file is not a .gz file */