Added applying of undo for updates

Fixed bug in duplicate key handling for block records during repair
All read-row methods now return error number in case of error
Don't calculate checksum for null fields
Fixed bug when running maria_read_log with -o


BUILD/SETUP.sh:
  Added STACK_DIRECTION
BUILD/compile-pentium-debug-max:
  Moved STACK_DIRECTION to SETUP
include/myisam.h:
  Added extra parameter to write_key
storage/maria/ma_blockrec.c:
  Added applying of undo for updates
  Fixed indentation
  Removed some not needed casts
  Fixed wrong logging of CLR record
  Split ma_update_block_record to two functions to be able to reuse it from undo-applying
  Simplify filling of packed fields
  ma_record_block_record) now returns error number on failure
  Sligtly changed log record information for undo-update
storage/maria/ma_check.c:
  Fixed bug in duplicate key handling for block records during repair
storage/maria/ma_checksum.c:
  Don't calculate checksum for null fields
storage/maria/ma_dynrec.c:
  _ma_read_dynamic_reocrd() now returns error number on error
  Rest of the changes are code simplification and indentation fixes
storage/maria/ma_locking.c:
  Added comment
storage/maria/ma_loghandler.c:
  More debugging
  Removed printing of total_record_length as this was always same as record_length
storage/maria/ma_open.c:
  Allocate bitmap for changed fields
storage/maria/ma_packrec.c:
  read_record now returns error number on error
storage/maria/ma_recovery.c:
  Fixed wrong arguments to undo_row_update
storage/maria/ma_statrec.c:
  read_record now returns error number on error (not 1)
  Code simplification
storage/maria/ma_test1.c:
  Added exit possibility after update phase (to test undo of updates)
storage/maria/maria_def.h:
  Include bitmap header file
storage/maria/maria_read_log.c:
  Fixed bug when running with -o

BUILD/SETUP.sh

@@ -170,10 +170,10 @@ max_configs="$SSL_LIBRARY --with-plugins=max --with-embedded-server"
 # CPU and platform specific compilation flags.
 #
 alpha_cflags="$check_cpu_cflags -Wa,-m$cpu_flag"
-amd64_cflags="$check_cpu_cflags"
+amd64_cflags="$check_cpu_cflags  -DSTACK_DIRECTION=-1"
 amd64_cxxflags=""  # If dropping '--with-big-tables', add here  "-DBIG_TABLES"
-pentium_cflags="$check_cpu_cflags"
-pentium64_cflags="$check_cpu_cflags -m64"
+pentium_cflags="$check_cpu_cflags  -DSTACK_DIRECTION=-1"
+pentium64_cflags="$check_cpu_cflags -m64 -DSTACK_DIRECTION=-1"
 ppc_cflags="$check_cpu_cflags"
 sparc_cflags=""
 

BUILD/compile-pentium-debug-max

@@ -4,7 +4,7 @@ path=`dirname $0`
 set -- "$@" --with-debug=full
 . "$path/SETUP.sh"
 
-extra_flags="$pentium_cflags $debug_cflags -DSTACK_DIRECTION=-1"
+extra_flags="$pentium_cflags $debug_cflags"
 extra_configs="$pentium_configs $debug_configs $max_configs $error_inject --with-experimental-collations"
 
 . "$path/FINISH.sh"

include/myisam.h

@@ -364,7 +364,7 @@ typedef struct st_mi_sort_param
   NEAR int (*write_keys)(struct st_mi_sort_param *, register uchar **,
                      uint , struct st_buffpek *, IO_CACHE *);
   NEAR uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint);
-  NEAR int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,char *,
+  NEAR int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,uchar *,
                        uint, uint);
 } MI_SORT_PARAM;
 

storage/maria/ma_check.c

@@ -2165,9 +2165,19 @@ int maria_repair(HA_CHECK *param, register MARIA_HA *info,
 	param->error_printed=1;
 	goto err;
       }
+      /* purecov: begin tested */
+      if (block_record)
+      {
+        sort_info.new_info->state->records--;
+        if ((*sort_info.new_info->s->write_record_abort)(sort_info.new_info))
+        {
+          _ma_check_print_error(param,"Couldn't delete duplicate row");
+          goto err;
+        }
         continue;
       }
-
+      /* purecov: end */
+    }
     if (!block_record && _ma_sort_write_record(&sort_param))
       goto err;
   }

storage/maria/ma_checksum.c

@@ -31,6 +31,9 @@ ha_checksum _ma_checksum(MARIA_HA *info, const uchar *record)
     const uchar *pos= record + column->offset;
     ulong length;
 
+    if (record[column->null_pos] & column->null_bit)
+      continue;                                 /* Null field */
+
     switch (column->type) {
     case FIELD_BLOB:
     {
@@ -45,12 +48,12 @@ ha_checksum _ma_checksum(MARIA_HA *info, const uchar *record)
     }
     case FIELD_VARCHAR:
     {
-      uint pack_length= HA_VARCHAR_PACKLENGTH(column->length-1);
+      uint pack_length= column->fill_length;
       if (pack_length == 1)
         length= (ulong) *(uchar*) pos;
       else
         length= uint2korr(pos);
-      pos+= pack_length;
+      pos+= pack_length;                        /* Skip length information */
       break;
     }
     default:

storage/maria/ma_dynrec.c

@@ -1138,10 +1138,14 @@ my_bool _ma_rec_check(MARIA_HA *info,const uchar *record, uchar *rec_buff,
 }
 
 
+/*
+  @brief Unpacks a record
 
-	/* Unpacks a record */
-	/* Returns -1 and my_errno =HA_ERR_RECORD_DELETED if reclength isn't */
-	/* right. Returns reclength (>0) if ok */
+  @return Recordlength
+  @retval >0  ok
+  @retval MY_FILE_ERROR (== -1)  Error.
+          my_errno is set to HA_ERR_WRONG_IN_RECORD
+*/
 
 ulong _ma_rec_unpack(register MARIA_HA *info, register uchar *to, uchar *from,
 		     ulong found_length)
@@ -1370,8 +1374,9 @@ void _ma_store_blob_length(uchar *pos,uint pack_length,uint length)
 
   RETURN
     0          OK
-    1          Error
+    #          Error number
 */
+
 int _ma_read_dynamic_record(MARIA_HA *info, uchar *buf,
                             MARIA_RECORD_POS filepos)
 {
@@ -1379,12 +1384,12 @@ int _ma_read_dynamic_record(MARIA_HA *info, uchar *buf,
   uint b_type;
   MARIA_BLOCK_INFO block_info;
   File file;
-  DBUG_ENTER("_ma_read_dynamic_record");
-
-  if (filepos != HA_OFFSET_ERROR)
-  {
   uchar *to;
   uint left_length;
+  DBUG_ENTER("_ma_read_dynamic_record");
+
+  if (filepos == HA_OFFSET_ERROR)
+    goto err;
 
   LINT_INIT(to);
   LINT_INIT(left_length);
@@ -1466,16 +1471,15 @@ int _ma_read_dynamic_record(MARIA_HA *info, uchar *buf,
   info->update|= HA_STATE_AKTIV;	/* We have a aktive record */
   fast_ma_writeinfo(info);
   DBUG_RETURN(_ma_rec_unpack(info,buf,info->rec_buff,block_info.rec_len) !=
-		MY_FILE_ERROR ? 0 : 1);
-  }
+              MY_FILE_ERROR ? 0 : my_errno);
+
+err:
   fast_ma_writeinfo(info);
-  DBUG_RETURN(1);			/* Wrong data to read */
+  DBUG_RETURN(my_errno);
 
 panic:
   my_errno=HA_ERR_WRONG_IN_RECORD;
-err:
-  VOID(_ma_writeinfo(info,0));
-  DBUG_RETURN(1);
+  goto err;
 }
 
 	/* compare unique constraint between stored rows */
@@ -1655,7 +1659,7 @@ static my_bool _ma_cmp_buffer(File file, const uchar *buff, my_off_t filepos,
 
   RETURN
     0           OK
-    != 0        Error
+    != 0        Error number
 */
 
 int _ma_read_rnd_dynamic_record(MARIA_HA *info,
@@ -1663,7 +1667,7 @@ int _ma_read_rnd_dynamic_record(MARIA_HA *info,
                                 MARIA_RECORD_POS filepos,
 				my_bool skip_deleted_blocks)
 {
-  int block_of_record, info_read, save_errno;
+  int block_of_record, info_read;
   uint left_len,b_type;
   uchar *to;
   MARIA_BLOCK_INFO block_info;
@@ -1827,9 +1831,8 @@ int _ma_read_rnd_dynamic_record(MARIA_HA *info,
 panic:
   my_errno=HA_ERR_WRONG_IN_RECORD;		/* Something is fatal wrong */
 err:
-  save_errno=my_errno;
-  VOID(_ma_writeinfo(info,0));
-  DBUG_RETURN(my_errno=save_errno);
+  fast_ma_writeinfo(info);
+  DBUG_RETURN(my_errno);
 }
 
 

storage/maria/ma_locking.c

@@ -387,6 +387,9 @@ int _ma_readinfo(register MARIA_HA *info, int lock_type, int check_keybuffer)
 /*
   Every isam-function that uppdates the isam-database MUST end with this
   request
+
+  NOTES
+    my_errno is not changed if this succeeds!
 */
 
 int _ma_writeinfo(register MARIA_HA *info, uint operation)

storage/maria/ma_loghandler.c

@@ -4901,8 +4901,8 @@ my_bool translog_write_record(LSN *lsn,
   int rc;
   uint short_trid= trn->short_id;
   DBUG_ENTER("translog_write_record");
-  DBUG_PRINT("enter", ("type: %u  ShortTrID: %u",
-                       (uint) type, (uint)short_trid));
+  DBUG_PRINT("enter", ("type: %u  ShortTrID: %u  rec_len: %lu",
+                       (uint) type, (uint) short_trid, (ulong) rec_len));
 
   if (tbl_info)
   {
@@ -4995,9 +4995,7 @@ my_bool translog_write_record(LSN *lsn,
     be add
   */
   parts.total_record_length= parts.record_length;
-  DBUG_PRINT("info", ("record length: %lu  %lu",
-                      (ulong) parts.record_length,
-                      (ulong) parts.total_record_length));
+  DBUG_PRINT("info", ("record length: %lu", (ulong) parts.record_length));
 
   /* process this parts */
   if (!(rc= (log_record_type_descriptor[type].prewrite_hook &&

storage/maria/ma_open.c

@@ -94,6 +94,7 @@ static MARIA_HA *maria_clone_internal(MARIA_SHARE *share, int mode,
   int save_errno;
   uint errpos;
   MARIA_HA info,*m_info;
+  my_bitmap_map *changed_fields_bitmap;
   DBUG_ENTER("maria_clone_internal");
 
   errpos= 0;
@@ -120,6 +121,8 @@ static MARIA_HA *maria_clone_internal(MARIA_SHARE *share, int mode,
 		       &info.first_mbr_key, share->base.max_key_length,
 		       &info.maria_rtree_recursion_state,
                        share->have_rtree ? 1024 : 0,
+                       &changed_fields_bitmap,
+                       bitmap_buffer_size(share->base.fields),
 		       NullS))
     goto err;
   errpos= 6;
@@ -144,6 +147,8 @@ static MARIA_HA *maria_clone_internal(MARIA_SHARE *share, int mode,
   info.errkey= -1;
   info.page_changed=1;
   info.keyread_buff= info.buff + share->base.max_key_block_length;
+  bitmap_init(&info.changed_fields, changed_fields_bitmap,
+              share->base.fields, 0);
   if ((*share->init)(&info))
     goto err;
 

storage/maria/ma_packrec.c

@@ -728,8 +728,8 @@ static uint find_longest_bitstream(uint16 *table, uint16 *end)
     buf                 RETURN  The buffer to receive the record.
 
   RETURN
-    0                                   on success
-    HA_ERR_WRONG_IN_RECORD or -1        on error
+    0   On success
+    #   Error number
 */
 
 int _ma_read_pack_record(MARIA_HA *info, uchar *buf, MARIA_RECORD_POS filepos)
@@ -739,7 +739,7 @@ int _ma_read_pack_record(MARIA_HA *info, uchar *buf, MARIA_RECORD_POS filepos)
   DBUG_ENTER("maria_read_pack_record");
 
   if (filepos == HA_OFFSET_ERROR)
-    DBUG_RETURN(-1);			/* _search() didn't find record */
+    DBUG_RETURN(my_errno);          /* _search() didn't find record */
 
   file= info->dfile.file;
   if (_ma_pack_get_block_info(info, &info->bit_buff, &block_info,
@@ -755,7 +755,7 @@ int _ma_read_pack_record(MARIA_HA *info, uchar *buf, MARIA_RECORD_POS filepos)
 panic:
   my_errno=HA_ERR_WRONG_IN_RECORD;
 err:
-  DBUG_RETURN(-1);
+  DBUG_RETURN(my_errno);
 }
 
 
@@ -1598,14 +1598,14 @@ static int _ma_read_mempack_record(MARIA_HA *info, uchar *buf,
   DBUG_ENTER("maria_read_mempack_record");
 
   if (filepos == HA_OFFSET_ERROR)
-    DBUG_RETURN(-1);			/* _search() didn't find record */
+    DBUG_RETURN(my_errno);          /* _search() didn't find record */
 
   if (!(pos= (uchar*) _ma_mempack_get_block_info(info, &info->bit_buff,
                                                 &block_info, &info->rec_buff,
                                                 &info->rec_buff_size,
 						(uchar*) share->file_map+
 						filepos)))
-    DBUG_RETURN(-1);
+    DBUG_RETURN(my_errno);
   DBUG_RETURN(_ma_pack_rec_unpack(info, &info->bit_buff, buf,
                                   pos, block_info.rec_len));
 }

storage/maria/ma_recovery.c

@@ -1045,17 +1045,11 @@ prototype_undo_exec_hook(UNDO_ROW_UPDATE)
   info->trn= trn;
   info->trn->undo_lsn= lsn_korr(rec->header);
 
-  /*
-    For now we skip the page and directory entry. This is to be used
-    later when we mark rows as deleted.
-  */
   error= _ma_apply_undo_row_update(info, rec->lsn,
                                    log_record_buffer.str + LSN_STORE_SIZE +
-                                   FILEID_STORE_SIZE + PAGE_STORE_SIZE +
-                                   DIRPOS_STORE_SIZE,
+                                   FILEID_STORE_SIZE,
                                    rec->record_length -
-                                   (LSN_STORE_SIZE + FILEID_STORE_SIZE +
-                                    PAGE_STORE_SIZE + DIRPOS_STORE_SIZE));
+                                   (LSN_STORE_SIZE + FILEID_STORE_SIZE));
   info->trn= 0;
   return error;
 }

storage/maria/ma_statrec.c

@@ -183,26 +183,25 @@ int _ma_read_static_record(register MARIA_HA *info, register uchar *record,
     if (info->opt_flag & WRITE_CACHE_USED &&
 	info->rec_cache.pos_in_file <= pos &&
 	flush_io_cache(&info->rec_cache))
-      return(-1);
+      return(my_errno);
     info->rec_cache.seek_not_done=1;		/* We have done a seek */
 
     error=info->s->file_read(info,(char*) record,info->s->base.reclength,
-                             pos, MYF(MY_NABP)) != 0;
-    fast_ma_writeinfo(info);
+                             pos, MYF(MY_NABP));
     if (! error)
     {
+      fast_ma_writeinfo(info);
       if (!*record)
       {
-	my_errno=HA_ERR_RECORD_DELETED;
-	return(1);				/* Record is deleted */
+        /* Record is deleted */
+	return ((my_errno=HA_ERR_RECORD_DELETED));
       }
       info->update|= HA_STATE_AKTIV;		/* Record is read */
       return(0);
     }
-    return(-1);					/* Error on read */
   }
   fast_ma_writeinfo(info);			/* No such record */
-  return(-1);
+  return(my_errno);
 }
 
 
@@ -264,13 +263,7 @@ int _ma_read_rnd_static_record(MARIA_HA *info, uchar *buf,
 
   if (! cache_read)			/* No cacheing */
   {
-    if ((error= _ma_read_static_record(info, buf, filepos)))
-    {
-      if (error > 0)
-	error=my_errno=HA_ERR_RECORD_DELETED;
-      else
-	error=my_errno;
-    }
+    error= _ma_read_static_record(info, buf, filepos);
     DBUG_RETURN(error);
   }
 

storage/maria/ma_test1.c

@@ -252,6 +252,9 @@ static int run_test(const char *filename)
     exit(1);
   }
 
+  if (maria_commit(file) || maria_begin(file))
+    goto err;
+
   if (!skip_update)
   {
     if (opt_unique)
@@ -289,7 +292,7 @@ static int run_test(const char *filename)
     found=0;
     while ((error= maria_scan(file,read_record)) == 0)
     {
-      if (update_count-- == 0) { VOID(maria_close(file)) ; exit(0) ; }
+      if (--update_count == 0) { VOID(maria_close(file)) ; exit(0) ; }
       memcpy(record,read_record,rec_length);
       update_record(record);
       if (maria_update(file,read_record,record))
@@ -304,6 +307,18 @@ static int run_test(const char *filename)
     maria_scan_end(file);
   }
 
+  if (die_in_middle_of_transaction == 2)
+  {
+    /*
+      Ensure we get changed pages and log to disk
+      As commit record is not done, the undo entries needs to be rolled back.
+    */
+    _ma_flush_table_files(file, MARIA_FLUSH_DATA, FLUSH_RELEASE,
+                          FLUSH_RELEASE);
+    printf("Dying on request after update without maria_close()\n");
+    exit(1);
+  }
+
   if (!silent)
     printf("- Reopening file\n");
   if (maria_commit(file))
@@ -356,7 +371,7 @@ static int run_test(const char *filename)
       }
     }
 
-    if (die_in_middle_of_transaction == 2)
+    if (die_in_middle_of_transaction == 3)
     {
       /*
         Ensure we get changed pages and log to disk

storage/maria/maria_def.h

@@ -16,8 +16,9 @@
 /* This file is included by all internal maria files */
 
 #include "maria.h"				/* Structs & some defines */
-#include "myisampack.h"				/* packing of keys */
+#include <myisampack.h>				/* packing of keys */
 #include <my_tree.h>
+#include <my_bitmap.h>
 #ifdef THREAD
 #include <my_pthread.h>
 #include <thr_lock.h>
@@ -437,6 +438,7 @@ struct st_maria_info
   PAGECACHE_FILE dfile;			/* The datafile */
   IO_CACHE rec_cache;			/* When cacheing records */
   LIST open_list;
+  MY_BITMAP changed_fields;
   uint opt_flag;			/* Optim. for space/speed */
   uint update;				/* If file changed since open */
   int lastinx;				/* Last used index */

storage/maria/maria_read_log.c

@@ -93,7 +93,8 @@ int main(int argc, char **argv)
   */
 
   fprintf(stdout, "TRACE of the last maria_read_log\n");
-  if (maria_apply_log(lsn, opt_display_and_apply, stdout, TRUE))
+  if (maria_apply_log(lsn, opt_display_and_apply, stdout,
+                      opt_display_and_apply))
     goto err;
   fprintf(stdout, "%s: SUCCESS\n", my_progname);