After merge fixes

Read blocks through page cache in check_block_record()
Don't read first bitmap on ma_open()
Don't require that a files block_size is equal to maria_block_size, if page cache is not setup yet.
Changed ma_test1, ma_test2, maria_chk and maria_pack to always create a page cache.
The above fixes so that ma_test_all now works again



BitKeeper/etc/ignore:
  added storage/maria/unittest/ma_pagecache_consist_1k-t-big storage/maria/unittest/ma_pagecache_consist_1kHC-t-big storage/maria/unittest/ma_pagecache_consist_1kRD-t-big storage/maria/unittest/ma_pagecache_consist_1kWR-t-big storage/maria/unittest/ma_pagecache_consist_64k-t-big storage/maria/unittest/ma_pagecache_consist_64kHC-t-big storage/maria/unittest/ma_pagecache_consist_64kRD-t-big storage/maria/unittest/ma_pagecache_consist_64kWR-t-big storage/maria/unittest/ma_pagecache_single_64k-t-big
include/maria.h:
  Added MARIA_MIN_PAGE_CACHE_SIZE
include/pagecache.h:
  Filedescriptors should be of type File
storage/maria/ma_bitmap.c:
  After merge fixes
  Create dummy bitmap on startup (can't read first bitmap becasue page cache may not be set up yet)
storage/maria/ma_blockrec.c:
  After merge fixes
storage/maria/ma_check.c:
  Use page cache to read rows-in-block rows.
  Don't initialize page cache; It's now done in maria_chk
storage/maria/ma_dynrec.c:
  Trivial code reorganization
storage/maria/ma_open.c:
  Don't give error for conflicting block size if page cache is not initalized.
  (Needed for maria_chk to be able to work on tables with different page sizes)
  After merge fixes
storage/maria/ma_page.c:
  Fix compiler warning
  Remove net needed asserts (Guranteed by ma_create())
storage/maria/ma_pagecache.c:
  Allow one to create a page cache with just one block
  (For trivail scan of table)
  Trivial code simplication
storage/maria/ma_test1.c:
  Always create a page cache (Maria now requires a page cache to work)
storage/maria/ma_test2.c:
  Always create a page cache (Maria now requires a page cache to work)
storage/maria/maria_chk.c:
  Remove command line options --maria_block_size and --pagecache_block_size.
  Set the global maria_block_size from the data file. This allows maria_chk to work with tables of different block sizes.
  Simply DESCRIPT handling; Allows us to remove one indentation level in maria_chk().
  Always initialize page cache if we are doing check/repair.
  (Most of the patch is reindentation of the code)
storage/maria/maria_def.h:
  After merge fix
storage/maria/maria_pack.c:
  Set maria_block_size based on the files block_size.
  Initalize page cache (needed for getting rows-in-blocks to works)

.bzrignore

@@ -2989,3 +2989,12 @@ win/vs71cache.txt
 win/vs8cache.txt
 zlib/*.ds?
 zlib/*.vcproj
+storage/maria/unittest/ma_pagecache_consist_1k-t-big
+storage/maria/unittest/ma_pagecache_consist_1kHC-t-big
+storage/maria/unittest/ma_pagecache_consist_1kRD-t-big
+storage/maria/unittest/ma_pagecache_consist_1kWR-t-big
+storage/maria/unittest/ma_pagecache_consist_64k-t-big
+storage/maria/unittest/ma_pagecache_consist_64kHC-t-big
+storage/maria/unittest/ma_pagecache_consist_64kRD-t-big
+storage/maria/unittest/ma_pagecache_consist_64kWR-t-big
+storage/maria/unittest/ma_pagecache_single_64k-t-big

include/maria.h

@@ -54,6 +54,8 @@ extern "C" {
 #define MARIA_KEY_BLOCK_LENGTH	8192		/* default key block length */
 #define MARIA_MIN_KEY_BLOCK_LENGTH	1024	/* Min key block length */
 #define MARIA_MAX_KEY_BLOCK_LENGTH	32768
+/* Minimal page cache when we only want to be able to scan a table */
+#define MARIA_MIN_PAGE_CACHE_SIZE	65536
 
 /*
   In the following macros '_keyno_' is 0 .. keys-1.

include/pagecache.h

@@ -75,7 +75,7 @@ typedef void *PAGECACHE_PAGE_LINK;
 /* file descriptor for Maria */
 typedef struct st_pagecache_file
 {
-  int file; /* it is for debugging purposes then it will be uint32 file_no */
+  File file;
 } PAGECACHE_FILE;
 
 /* page number for maria */

storage/maria/ma_bitmap.c

@@ -131,7 +131,7 @@ static inline my_bool write_changed_bitmap(MARIA_SHARE *share,
 {
   DBUG_ASSERT(share->pagecache->block_size == bitmap->block_size);
   return (pagecache_write(share->pagecache,
-                          (PAGECACHE_FILE*)&bitmap->file, bitmap->page, 0,
+                          &bitmap->file, bitmap->page, 0,
                           (byte*) bitmap->map, PAGECACHE_PLAIN_PAGE,
                           PAGECACHE_LOCK_LEFT_UNLOCKED,
                           PAGECACHE_PIN_LEFT_UNPINNED,
@@ -168,7 +168,7 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file)
   if (!(bitmap->map= (uchar*) my_malloc(size, MYF(MY_WME))))
     return 1;
 
-  bitmap->file= file;
+  bitmap->file.file= file;
   bitmap->changed= 0;
   bitmap->block_size= share->block_size;
   /* Size needs to be alligned on 6 */
@@ -195,10 +195,15 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file)
   pthread_mutex_init(&share->bitmap.bitmap_lock, MY_MUTEX_INIT_SLOW);
 
   /*
-    Start by reading first page (assume table scan)
-    Later code is simpler if it can assume we always have an active bitmap.
+    We can't read a page yet, as in some case we don't have an active
+    page cache yet.
+    Pretend we have a dummy, full and not changed bitmap page in memory.
  */
-  return _ma_read_bitmap_page(share, bitmap, (ulonglong) 0);
+  
+  bitmap->page= ~(ulonglong) 0;
+  bitmap->used_size= bitmap->total_size;
+  bfill(bitmap->map, share->block_size, 255);
+  return 0;
 }
 
 

storage/maria/ma_blockrec.c

@@ -374,21 +374,20 @@ my_bool _ma_once_init_block_record(MARIA_SHARE *share, File data_file)
 my_bool _ma_once_end_block_record(MARIA_SHARE *share)
 {
   int res= _ma_bitmap_end(share);
-  if (share->bitmap.file >= 0)
+  if (share->bitmap.file.file >= 0)
   {
-    if (flush_pagecache_blocks(share->pagecache, (PAGECACHE_FILE*)&share->bitmap,
-    if (flush_key_blocks(share->key_cache, share->bitmap.file,
-                         share->temporary ? FLUSH_IGNORE_CHANGED :
-                         FLUSH_RELEASE))
+    if (flush_pagecache_blocks(share->pagecache, &share->bitmap.file,
+                               share->temporary ? FLUSH_IGNORE_CHANGED :
+                               FLUSH_RELEASE))
       res= 1;
-    if (my_close(share->bitmap.file, MYF(MY_WME)))
+    if (my_close(share->bitmap.file.file, MYF(MY_WME)))
       res= 1;
     /*
       Trivial assignment to guard against multiple invocations
       (May happen if file are closed but we want to keep the maria object
       around a bit longer)
     */
-    share->bitmap.file= -1;
+    share->bitmap.file.file= -1;
   }
   return res;
 }
@@ -450,7 +449,7 @@ void _ma_end_block_record(MARIA_HA *info)
     The following protects us from doing an extra, not allowed, close
     in maria_close()
   */
-  info->dfile= -1;
+  info->dfile.file= -1;
   DBUG_VOID_RETURN;
 }
 

storage/maria/ma_check.c

@@ -1599,8 +1599,12 @@ static int check_block_record(HA_CHECK *param, MARIA_HA *info, int extend,
     if (((pos / block_size) % info->s->bitmap.pages_covered) == 0)
     {
       /* Bitmap page */
-      if (_ma_read_cache(&param->read_cache, bitmap_buff, pos,
-                       block_size, READING_NEXT))
+      if (pagecache_read(info->s->pagecache,
+                         &info->dfile,
+                         (pos / block_size), 1,
+                         bitmap_buff,
+                         PAGECACHE_PLAIN_PAGE,
+                         PAGECACHE_LOCK_LEFT_UNLOCKED, 0) == 0)
       {
         _ma_check_print_error(param,
                               "Page %9s:  Got error: %d when reading datafile",
@@ -1624,8 +1628,12 @@ static int check_block_record(HA_CHECK *param, MARIA_HA *info, int extend,
       continue;
     }
 
-    if (_ma_read_cache(&param->read_cache, page_buff, pos,
-                       block_size, READING_NEXT))
+    if (pagecache_read(info->s->pagecache,
+                       &info->dfile,
+                       (pos / block_size), 1,
+                       page_buff,
+                       PAGECACHE_PLAIN_PAGE,
+                       PAGECACHE_LOCK_LEFT_UNLOCKED, 0) == 0)
     {
       _ma_check_print_error(param,
                             "Page %9s:  Got error: %d when reading datafile",
@@ -1935,10 +1943,6 @@ int maria_repair(HA_CHECK *param, register MARIA_HA *info,
   if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
     param->testflag|=T_CALC_CHECKSUM;
 
-  if (!param->using_global_keycache)
-    VOID(init_pagecache(maria_pagecache, param->use_buffers, 0, 0,
-                        param->pagecache_block_size));
-
   if (init_io_cache(&param->read_cache, info->dfile.file,
 		    (uint) param->read_buffer_length,
 		    READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
@@ -2308,8 +2312,6 @@ int _ma_flush_blocks(HA_CHECK *param, PAGECACHE *pagecache,
     _ma_check_print_error(param,"%d when trying to write bufferts",my_errno);
     return(1);
   }
-  if (!param->using_global_keycache)
-    end_pagecache(pagecache,1);
   return 0;
 } /* _ma_flush_blocks */
 
@@ -3602,7 +3604,6 @@ static int sort_get_next_record(MARIA_SORT_PARAM *sort_param)
   int parallel_flag;
   uint found_record,b_type,left_length;
   my_off_t pos;
-  byte *to;
   MARIA_BLOCK_INFO block_info;
   MARIA_SORT_INFO *sort_info=sort_param->sort_info;
   HA_CHECK *param=sort_info->param;
@@ -3652,6 +3653,8 @@ static int sort_get_next_record(MARIA_SORT_PARAM *sort_param)
       }
     }
   case DYNAMIC_RECORD:
+  {
+    byte *to;
     LINT_INIT(to);
     pos=sort_param->pos;
     searching=(sort_param->fix_datafile && (param->testflag & T_EXTEND));
@@ -3948,6 +3951,7 @@ static int sort_get_next_record(MARIA_SORT_PARAM *sort_param)
       pos=(sort_param->start_recpos+=MARIA_DYN_ALIGN_SIZE);
       searching=1;
     }
+  }
   case COMPRESSED_RECORD:
     for (searching=0 ;; searching=1, sort_param->pos++)
     {

storage/maria/ma_dynrec.c

@@ -1372,13 +1372,15 @@ int _ma_read_dynamic_record(MARIA_HA *info, byte *buf,
 {
   int block_of_record;
   uint b_type,left_length;
-  byte *to;
   MARIA_BLOCK_INFO block_info;
   File file;
   DBUG_ENTER("_ma_read_dynamic_record");
 
   if (filepos != HA_OFFSET_ERROR)
   {
+    byte *to;
+    uint left_length;
+
     LINT_INIT(to);
     LINT_INIT(left_length);
     file= info->dfile.file;
@@ -1390,13 +1392,14 @@ int _ma_read_dynamic_record(MARIA_HA *info, byte *buf,
       if (filepos == HA_OFFSET_ERROR)
         goto panic;
       if (info->opt_flag & WRITE_CACHE_USED &&
-	  info->rec_cache.pos_in_file < filepos + MARIA_BLOCK_INFO_HEADER_LENGTH &&
+	  (info->rec_cache.pos_in_file < filepos +
+           MARIA_BLOCK_INFO_HEADER_LENGTH) &&
 	  flush_io_cache(&info->rec_cache))
 	goto err;
       info->rec_cache.seek_not_done=1;
-      if ((b_type= _ma_get_block_info(&block_info, file, filepos))
-	  & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
-	     BLOCK_FATAL_ERROR))
+      if ((b_type= _ma_get_block_info(&block_info, file, filepos)) &
+	  (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
+           BLOCK_FATAL_ERROR))
       {
 	if (b_type & (BLOCK_SYNC_ERROR | BLOCK_DELETED))
 	  my_errno=HA_ERR_RECORD_DELETED;

storage/maria/ma_open.c

@@ -239,7 +239,12 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags)
       my_errno=HA_ERR_UNSUPPORTED;
       goto err;
     }
-    if (share->base.block_size != maria_block_size)
+    /*
+      If page cache is not initialized, then assume we will create it
+      after the table is opened!
+    */
+    if (share->base.block_size != maria_block_size &&
+        share_buff.pagecache->inited != 0)
     {
       DBUG_PRINT("error", ("Wrong block size %u; Expected %u",
                            (uint) share->base.block_size,
@@ -301,7 +306,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags)
     strmov(share->index_file_name,  index_name);
     strmov(share->data_file_name,   data_name);
 
-    share->block_size= maria_block_size;
+    share->block_size= share->base.block_size;
     {
       HA_KEYSEG *pos=share->keyparts;
       for (i=0 ; i < keys ; i++)
@@ -553,7 +558,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags)
       goto err;
     }
     if (share->data_file_type == BLOCK_RECORD)
-      info.dfile.file= share->bitmap.file;
+      info.dfile= share->bitmap.file;
     else if (_ma_open_datafile(&info, share, old_info->dfile.file))
       goto err;
     errpos= 5;

storage/maria/ma_page.c

@@ -21,15 +21,15 @@
 
 byte *_ma_fetch_keypage(register MARIA_HA *info, MARIA_KEYDEF *keyinfo,
                         my_off_t page, int level,
-                        byte *buff, int return_buffer)
+                        byte *buff,
+                        int return_buffer __attribute__ ((unused)))
 {
   byte *tmp;
   uint page_size;
   DBUG_ENTER("_ma_fetch_keypage");
   DBUG_PRINT("enter",("page: %ld", (long) page));
 
-  DBUG_ASSERT(info->s->pagecache->block_size == keyinfo->block_length &&
-              info->s->pagecache->block_size == info->s->block_size);
+  DBUG_ASSERT(info->s->pagecache->block_size == keyinfo->block_length);
   /*
     TODO: replace PAGECACHE_PLAIN_PAGE with PAGECACHE_LSN_PAGE when
     LSN on the pages will be implemented
@@ -88,7 +88,6 @@ int _ma_write_keypage(register MARIA_HA *info, register MARIA_KEYDEF *keyinfo,
 #endif
 
   DBUG_ASSERT(info->s->pagecache->block_size == keyinfo->block_length);
-  DBUG_ASSERT(info->s->pagecache->block_size == info->s->block_size);
   /*
     TODO: replace PAGECACHE_PLAIN_PAGE with PAGECACHE_LSN_PAGE when
     LSN on the pages will be implemented

storage/maria/ma_pagecache.c

@@ -675,8 +675,11 @@ int init_pagecache(PAGECACHE *pagecache, my_size_t use_mem,
                             2 * sizeof(PAGECACHE_HASH_LINK) +
                             sizeof(PAGECACHE_HASH_LINK*) *
                             5/4 + block_size));
-  /* It doesn't make sense to have too few blocks (less than 8) */
-  if (blocks >= 8 && pagecache->disk_blocks < 0)
+  /*
+    We need to support page cache with just one block to be able to do
+    scanning of rows-in-block files
+  */
+  if (blocks >= 1)
   {
     for ( ; ; )
     {
@@ -3768,7 +3771,7 @@ my_bool pagecache_collect_changed_blocks_with_lsn(PAGECACHE *pagecache,
                                                   LEX_STRING *str,
                                                   LSN *max_lsn)
 {
-  my_bool error;
+  my_bool error= 0;
   ulong stored_list_size= 0;
   uint file_hash;
   char *ptr;
@@ -3836,7 +3839,7 @@ my_bool pagecache_collect_changed_blocks_with_lsn(PAGECACHE *pagecache,
   ptr= str->str;
   int8store(ptr, stored_list_size);
   ptr+= 8;
-  if (0 == stored_list_size)
+  if (!stored_list_size)
     goto end;
   for (file_hash= 0; file_hash < PAGECACHE_CHANGED_BLOCKS_HASH; file_hash++)
   {
@@ -3858,13 +3861,13 @@ my_bool pagecache_collect_changed_blocks_with_lsn(PAGECACHE *pagecache,
       set_if_bigger(*max_lsn, block->rec_lsn);
     }
   }
-  error= 0;
-  goto end;
-err:
-  error= 1;
 end:
   pagecache_pthread_mutex_unlock(&pagecache->cache_lock);
   DBUG_RETURN(error);
+
+err:
+  error= 1;
+  goto end;
 }
 
 

storage/maria/ma_test1.c

@@ -51,8 +51,8 @@ int main(int argc,char *argv[])
   my_init();
   maria_init();
   get_options(argc,argv);
-  if (pagecacheing)
-    init_pagecache(maria_pagecache, IO_SIZE*16, 0, 0, MARIA_KEY_BLOCK_LENGTH);
+  /* Maria requires that we always have a page cache */
+  init_pagecache(maria_pagecache, IO_SIZE*16, 0, 0, maria_block_size);
 
   exit(run_test("test1"));
 }

storage/maria/ma_test2.c

@@ -50,7 +50,6 @@ static	int verbose=0,testflag=0,
 static int pack_seg=HA_SPACE_PACK,pack_type=HA_PACK_KEY,remove_count=-1;
 static int create_flag= 0, srand_arg= 0;
 static ulong pagecache_size=IO_SIZE*16;
-static uint pagecache_block_size= MARIA_KEY_BLOCK_LENGTH;
 static enum data_file_type record_type= DYNAMIC_RECORD;
 
 static uint keys=MARIA_KEYS,recant=1000;
@@ -219,8 +218,8 @@ int main(int argc, char *argv[])
     goto err;
   if (!silent)
     printf("- Writing key:s\n");
-  if (pagecacheing)
-    init_pagecache(maria_pagecache, pagecache_size, 0, 0, pagecache_block_size);
+  /* Maria requires that we always have a page cache */
+  init_pagecache(maria_pagecache, pagecache_size, 0, 0, maria_block_size);
   if (locking)
     maria_lock_database(file,F_WRLCK);
   if (write_cacheing)
@@ -282,12 +281,11 @@ int main(int argc, char *argv[])
       goto end;
     }
   }
-  /*
-    TODO: uncomment when resize will be implemented
+#ifdef REMOVE_WHEN_WE_HAVE_RESIZE
   if (pagecacheing)
-    resize_pagecache(maria_pagecache, pagecache_block_size,
+    resize_pagecache(maria_pagecache, maria_block_size,
                      pagecache_size * 2, 0, 0);
-  */
+#endif
   if (!silent)
     printf("- Delete\n");
   if (srand_arg)
@@ -862,13 +860,8 @@ end:
     if (rec_pointer_size)
       printf("Record pointer size:  %d\n",rec_pointer_size);
     printf("maria_block_size:    %lu\n", maria_block_size);
-    if (pagecacheing)
-    {
-      puts("Key cache used");
-      printf("pagecache_block_size: %u\n", pagecache_block_size);
-      if (write_cacheing)
-	puts("Key cache resized");
-    }
+    if (write_cacheing)
+      puts("Key cache resized");
     if (write_cacheing)
       puts("Write cacheing used");
     if (write_cacheing)
@@ -960,6 +953,7 @@ static void get_options(int argc, char **argv)
       }
       break;
     case 'e':				/* maria_block_length */
+    case 'E':
       if ((maria_block_size= atoi(++pos)) < MARIA_MIN_KEY_BLOCK_LENGTH ||
 	  maria_block_size > MARIA_MAX_KEY_BLOCK_LENGTH)
       {
@@ -968,15 +962,6 @@ static void get_options(int argc, char **argv)
       }
       maria_block_size= my_round_up_to_next_power(maria_block_size);
       break;
-    case 'E':				/* maria_block_length */
-      if ((pagecache_block_size=atoi(++pos)) < MARIA_MIN_KEY_BLOCK_LENGTH ||
-	  pagecache_block_size > MARIA_MAX_KEY_BLOCK_LENGTH)
-      {
-	fprintf(stderr,"Wrong pagecache_block_size\n");
-	exit(1);
-      }
-      pagecache_block_size= my_round_up_to_next_power(pagecache_block_size);
-      break;
     case 'f':
       if ((first_key=atoi(++pos)) < 0 || first_key >= MARIA_KEYS)
 	first_key=0;

storage/maria/maria_def.h

@@ -190,9 +190,9 @@ typedef struct st_maria_file_bitmap
 {
   uchar *map;
   ulonglong page;                      /* Page number for current bitmap */
-  PAGECACHE_FILE page_cache;
   uint used_size;                      /* Size of bitmap head that is not 0 */
-  my_bool changed;                     /* Set to 1 if page needs to be flushed */
+  my_bool changed;                     /* 1 if page needs to be flushed */
+  PAGECACHE_FILE file;		       /* datafile where bitmap is stored */
 
 #ifdef THREAD
   pthread_mutex_t bitmap_lock;

storage/maria/maria_pack.c

@@ -505,13 +505,21 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table)
   trees=fields=0;
   huff_trees=0;
   huff_counts=0;
+  maria_block_size= isam_file->s->block_size;
 
   /* Create temporary or join file */
-
   if (backup)
     VOID(fn_format(org_name,isam_file->filename,"",MARIA_NAME_DEXT,2));
   else
     VOID(fn_format(org_name,isam_file->filename,"",MARIA_NAME_DEXT,2+4+16));
+
+  if (init_pagecache(maria_pagecache, MARIA_MIN_PAGE_CACHE_SIZE, 0, 0,
+                     maria_block_size) == 0)
+  {
+    fprintf(stderr, "Can't initialize page cache\n");
+    goto err;
+  }
+
   if (!test_only && result_table)
   {
     /* Make a new indexfile based on first file in list */
@@ -681,7 +689,7 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table)
     {
       error|=my_close(isam_file->dfile.file, MYF(MY_WME));
       isam_file->dfile.file= -1;	/* Tell maria_close file is closed */
-      isam_file->s->bitmap.file= -1;
+      isam_file->s->bitmap.file.file= -1;
     }
   }
 
@@ -751,6 +759,7 @@ static int compress(PACK_MRG_INFO *mrg,char *result_table)
   DBUG_RETURN(0);
 
  err:
+  end_pagecache(maria_pagecache, 1);
   free_counts_and_tree_and_queue(huff_trees,trees,huff_counts,fields);
   if (new_file >= 0)
     VOID(my_close(new_file,MYF(0)));