WL#3072 Maria Recovery

- new program maria_read_log to display and apply log records
found in a Maria log (see file's revision comment)
- minor, misc fixes


storage/maria/Makefile.am:
  new program maria_read_log
storage/maria/ha_maria.cc:
  create control file if missing
storage/maria/ma_blockrec.c:
  0 -> LSN_IMPOSSIBLE; comments
storage/maria/ma_checkpoint.h:
  preparations for Checkpoint module
storage/maria/ma_close.c:
  comment
storage/maria/ma_control_file.c:
  renaming constants.
  Possibility to say "open control file but don't create it if it's
  missing" (used by maria_read_log which does not want to create
  anything)
storage/maria/ma_control_file.h:
  renaming constants
storage/maria/ma_create.c:
  I had duplicated "linkname" and "linkname_ptr", now I see it's not
  needed, reverting. Indeed those variables don't contain interesting
  information; fixing log record accordingly (the links are in
  ci->data/index_file_name). Storing keystart in log record is needed,
  to know at which size we must extend the file if we replay
  LOGREC_CREATE_TABLE.
storage/maria/ma_loghandler.c:
  some structures need to be known to maria_read_log.c, taking
  them to ma_loghandler.h
storage/maria/ma_loghandler.h:
  we have page_store, adding page_korr.
  translog_lock() made public, because Checkpoint will need it (to
  write to control file).
  Some structures moved from ma_loghandler.c because maria_read_log.c
  needs them (needs to know the execute-in-REDO-phase hooks of each
  record).
storage/maria/ma_loghandler_lsn.h:
  constants defined in ma_control_file.h serve everywhere,
  and they relate to LSNs, so putting them in ma_loghandler_lsn.h.
  Stronger constraints in LSN_VALID().
storage/maria/ma_pagecache.c:
  renaming constants
storage/maria/ma_recovery.h:
  copyright
storage/maria/ma_test1.c:
  new prototype
storage/maria/ma_test2.c:
  new prototype
storage/maria/trnman_public.h:
  double-inclusion safe
storage/maria/unittest/ma_control_file-t.c:
  constants renamed, new prototype
storage/maria/unittest/ma_test_loghandler-t.c:
  constants renamed, new prototype
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
  constants renamed, new prototype
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
  constants renamed, new prototype
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
  constants renamed, new prototype
storage/myisam/mi_close.c:
  comment
storage/maria/maria_read_log.c:
  program to read and print log records from a Maria transaction log,
  and optionally apply them to tables. Very basic, early version.
  Should serve as a base for Recovery's code. Designed to be idempotent.
  Create a log by running maria.test, then cd to var/master-data
  and run "maria_read_log --only-display" to see info about records;
  run "maria_read_log --display-and-apply" to also apply the records
  to tables (it's more interesting if you first wipe out the
  tables in var/master-data/test, to see how they get re-created).
  Only a few records are handled by now: LONG_TRANSACTION_ID,
  COMMIT, FILE_ID, REDO_CREATE_TABLE; place is ready for
  REDO_INSERT_ROW_HEAD where I could use Monty's help (search for
  "Monty" in the file). Note: changes to the index pages, index's header
  and bitmap pages are not properly logged yet, so don't expect
  the program to work with that.

storage/maria/Makefile.am

@@ -33,7 +33,7 @@ SUBDIRS =		. unittest
 EXTRA_DIST =		ma_test_all.sh ma_test_all.res ma_ft_stem.c CMakeLists.txt plug.in
 pkgdata_DATA =		ma_test_all ma_test_all.res
 pkglib_LIBRARIES =	libmaria.a
-bin_PROGRAMS =		maria_chk maria_pack maria_ftdump
+bin_PROGRAMS =		maria_chk maria_pack maria_ftdump maria_read_log
 maria_chk_DEPENDENCIES=	$(LIBRARIES)
 # Only reason to link with libmyisam.a here is that it's where some fulltext
 # pieces are (but soon we'll remove fulltext dependencies from Maria).
@@ -49,6 +49,12 @@ maria_pack_LDADD=		@CLIENT_EXTRA_LDFLAGS@ libmaria.a \
 			$(top_builddir)/mysys/libmysys.a \
 			$(top_builddir)/dbug/libdbug.a \
 			$(top_builddir)/strings/libmystrings.a @ZLIB_LIBS@
+maria_read_log_DEPENDENCIES=$(LIBRARIES)
+maria_read_log_LDADD=		@CLIENT_EXTRA_LDFLAGS@ libmaria.a \
+                        $(top_builddir)/storage/myisam/libmyisam.a \
+			$(top_builddir)/mysys/libmysys.a \
+			$(top_builddir)/dbug/libdbug.a \
+			$(top_builddir)/strings/libmystrings.a @ZLIB_LIBS@
 noinst_PROGRAMS =	ma_test1 ma_test2 ma_test3 ma_rt_test ma_sp_test
 noinst_HEADERS =	maria_def.h ma_rt_index.h ma_rt_key.h ma_rt_mbr.h \
 			ma_sp_defs.h ma_fulltext.h ma_ftdefs.h ma_ft_test1.h \

storage/maria/ha_maria.cc

@@ -2241,7 +2241,7 @@ static int ha_maria_init(void *p)
   maria_hton->flags= HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES;
   bzero(maria_log_pagecache, sizeof(*maria_log_pagecache));
   maria_data_root= mysql_real_data_home;
-  res= maria_init() || ma_control_file_create_or_open() ||
+  res= maria_init() || ma_control_file_create_or_open(TRUE) ||
     (init_pagecache(maria_log_pagecache,
                     TRANSLOG_PAGECACHE_SIZE, 0, 0,
                     TRANSLOG_PAGE_SIZE) == 0) ||

storage/maria/ma_blockrec.c

@@ -557,7 +557,8 @@ static my_bool check_if_zero(byte *pos, uint length)
   SYNOPSIS
     _ma_unpin_all_pages()
     info	Maria handler
-    undo_lsn	LSN for undo pages. 0 if we shouldn't write undo (error)
+    undo_lsn	LSN for undo pages. LSN_IMPOSSIBLE if we shouldn't write undo
+                (error)
 
   NOTE
     We unpin pages in the reverse order as they where pinned; This may not
@@ -580,14 +581,15 @@ void _ma_unpin_all_pages(MARIA_HA *info, LSN undo_lsn)
   DBUG_PRINT("info", ("undo_lsn: %lu", (ulong) undo_lsn));
 
   /* True if not disk error */
-  DBUG_ASSERT(undo_lsn != 0 || !info->s->base.transactional);
+  DBUG_ASSERT((undo_lsn != LSN_IMPOSSIBLE) || !info->s->base.transactional);
 
   if (!info->s->base.transactional)
   {
     /*
       If this is a transactional table but with transactionality temporarily
       disabled (like in ALTER TABLE) we need to give a sensible LSN to pages
-      and not 0. If this is not a transactional table it will reduce to 0.
+      and not LSN_IMPOSSIBLE. If this is not a transactional table it will
+      reduce to LSN_IMPOSSIBLE.
     */
     undo_lsn= info->s->state.create_rename_lsn;
   }
@@ -1958,8 +1960,8 @@ static my_bool write_block_record(MARIA_HA *info,
     size_t data_length= (size_t) (data - row_pos->data);
 
     /* Log REDO changes of head page */
-    page_store(log_data+ FILEID_STORE_SIZE, head_block->page);
-    dirpos_store(log_data+ FILEID_STORE_SIZE + PAGE_STORE_SIZE,
+    page_store(log_data + FILEID_STORE_SIZE, head_block->page);
+    dirpos_store(log_data + FILEID_STORE_SIZE + PAGE_STORE_SIZE,
                  row_pos->rownr);
     log_array[TRANSLOG_INTERNAL_PARTS + 0].str=    (char*) log_data;
     log_array[TRANSLOG_INTERNAL_PARTS + 0].length= sizeof(log_data);
@@ -2183,12 +2185,22 @@ static my_bool write_block_record(MARIA_HA *info,
 disk_err:
   /**
      @todo RECOVERY we are going to let dirty pages go to disk while we have
-     logged UNDO, this violates WAL. If we have not written any full pages,
-     all dirty pages are pinned so we could just delete them from the
-     pagecache. Moreover, we have written some REDOs without a closing UNDO,
+     logged UNDO, this violates WAL. We must mark the table corrupted!
+
+     @todo RECOVERY we have written some REDOs without a closing UNDO,
      it's possible that a next operation by this transaction succeeds and then
      Recovery would glue the "orphan REDOs" to the succeeded operation and
-     execute the failed REDOs.
+     execute the failed REDOs. We need some mark "abort this group" in the
+     log, or mark the table corrupted (then user will repair it and thus REDOs
+     will be skipped).
+
+     @todo RECOVERY to not let write errors go unnoticed, pagecache_write()
+     should take a MARIA_HA* in argument, and it it
+     fails when flushing a page to disk it should call
+     (*the_maria_ha->write_error_func)(the_maria_ha)
+     and this hook will mark the table corrupted.
+     Maybe hook should be stored in the pagecache's block structure, or in a
+     hash "file->maria_ha*".
   */
   /* Unpin all pinned pages to not cause problems for disk cache */
   _ma_unpin_all_pages(info, 0);

storage/maria/ma_checkpoint.h

-/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
+/* Copyright (C) 2006,2007 MySQL AB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -21,14 +21,61 @@
 
 /* This is the interface of this module. */
 
-typedef enum enum_checkpoint_level {
-  NONE=-1,
-  INDIRECT, /* just write dirty_pages, transactions table and sync files */
-  MEDIUM, /* also flush all dirty pages which were already dirty at prev checkpoint*/
-  FULL /* also flush all dirty pages */
+typedef enum enum_ma_checkpoint_level {
+  CHECKPOINT_NONE= 0,
+  /* just write dirty_pages, transactions table and sync files */
+  CHECKPOINT_INDIRECT,
+  /* also flush all dirty pages which were already dirty at prev checkpoint */
+  CHECKPOINT_MEDIUM,
+  /* also flush all dirty pages */
+  CHECKPOINT_FULL
 } CHECKPOINT_LEVEL;
 
-void request_asynchronous_checkpoint(CHECKPOINT_LEVEL level);
-my_bool execute_synchronous_checkpoint(CHECKPOINT_LEVEL level);
-my_bool execute_asynchronous_checkpoint_if_any();
-/* that's all that's needed in the interface */
+C_MODE_START
+int ma_checkpoint_init();
+void ma_checkpoint_end();
+int ma_checkpoint_execute(CHECKPOINT_LEVEL level, my_bool no_wait);
+C_MODE_END
+
+/**
+   @brief reads some LSNs with special trickery
+
+   If a 64-bit variable transitions between both halves being zero to both
+   halves being non-zero, and back, this function can be used to do a read of
+   it (without mutex, without atomic load) which always produces a correct
+   (though maybe slightly old) value (even on 32-bit CPUs). The value is at
+   least as new as the latest mutex unlock done by the calling thread.
+   The assumption is that the system sets both 4-byte halves either at the
+   same time, or one after the other (in any order), but NOT some bytes of the
+   first half then some bytes of the second half then the rest of bytes of the
+   first half. With this assumption, the function can detect when it is
+   seeing an inconsistent value.
+
+   @param LSN              pointer to the LSN variable to read
+
+   @return LSN part (most significant byte always 0)
+*/
+#if ( SIZEOF_CHARP >= 8 )
+/* 64-bit CPU, 64-bit reads are atomic */
+#define lsn_read_non_atomic LSN_WITH_FLAGS_TO_LSN
+#else
+static inline LSN lsn_read_non_atomic_32(const volatile LSN *x)
+{
+  /*
+    32-bit CPU, 64-bit reads may give a mixed of old half and new half (old
+    low bits and new high bits, or the contrary).
+  */
+  for (;;) /* loop until no atomicity problems */
+  {
+    /*
+      Remove most significant byte in case this is a LSN_WITH_FLAGS object.
+      Those flags in TRN::first_undo_lsn break the condition on transitions so
+      they must be removed below.
+    */
+    LSN y= LSN_WITH_FLAGS_TO_LSN(*x);
+    if (likely((y == LSN_IMPOSSIBLE) || LSN_VALID(y)))
+      return y;
+  }
+}
+#define lsn_read_non_atomic(x) lsn_read_non_atomic_32(&x)
+#endif

storage/maria/ma_close.c

@@ -85,6 +85,7 @@ int maria_close(register MARIA_HA *info)
         not change the crashed state.
         We can NOT write the state in other cases as other threads
         may be using the file at this point
+        IF using --external-locking, which does not apply to Maria.
       */
       if (share->mode != O_RDONLY && maria_is_crashed(info))
 	_ma_state_info_write(share->kfile.file, &share->state, 1);

storage/maria/ma_control_file.c

@@ -40,15 +40,9 @@
 #define CONTROL_FILE_FILENO_SIZE 4
 #define CONTROL_FILE_SIZE (CONTROL_FILE_FILENO_OFFSET + CONTROL_FILE_FILENO_SIZE)
 
-/*
-  This module owns these two vars.
-  uint32 is always atomically updated, but LSN is 8 bytes, we will need
-  provisions to ensure that it's updated atomically in
-  ma_control_file_write_and_force(). Probably the log mutex could be
-  used. TODO.
-*/
-LSN last_checkpoint_lsn;
-uint32 last_logno;
+/* This module owns these two vars. */
+LSN    last_checkpoint_lsn= LSN_IMPOSSIBLE;
+uint32 last_logno=          FILENO_IMPOSSIBLE;
 
 /**
    @brief If log's lock should be asserted when writing to control file.
@@ -65,16 +59,16 @@ my_bool maria_multi_threaded= FALSE;
 static int control_file_fd= -1;
 
 /*
-  Initialize control file subsystem
-
-  SYNOPSIS
-    ma_control_file_create_or_open()
+  @brief Initialize control file subsystem
 
-  Looks for the control file. If absent, it's a fresh start, creates file.
+  Looks for the control file. If none and creation is requested, creates file.
   If present, reads it to find out last checkpoint's LSN and last log, updates
   the last_checkpoint_lsn and last_logno global variables.
   Called at engine's start.
 
+  @param  create_if_missing
+
+  @note
   The format of the control file is:
   4 bytes: magic string
   4 bytes: checksum of the following bytes
@@ -82,11 +76,11 @@ static int control_file_fd= -1;
   4 bytes: offset in log where last checkpoint is
   4 bytes: number of last log
 
-  RETURN
-    0 - OK
-    1 - Error (in which case the file is left closed)
+  @return Operation status
+    @retval 0      OK
+    @retval 1      Error (in which case the file is left closed)
 */
-CONTROL_FILE_ERROR ma_control_file_create_or_open()
+CONTROL_FILE_ERROR ma_control_file_create_or_open(my_bool create_if_missing)
 {
   char buffer[CONTROL_FILE_SIZE];
   char name[FN_REFLEN];
@@ -115,6 +109,8 @@ CONTROL_FILE_ERROR ma_control_file_create_or_open()
 
   if (create_file)
   {
+    if (!create_if_missing)
+      DBUG_RETURN(CONTROL_FILE_MISSING);
     if ((control_file_fd= my_create(name, 0,
                                     open_flags, MYF(MY_SYNC_DIR))) < 0)
       DBUG_RETURN(CONTROL_FILE_UNKNOWN_ERROR);
@@ -136,8 +132,8 @@ CONTROL_FILE_ERROR ma_control_file_create_or_open()
     */
 
     /* init the file with these "undefined" values */
-    DBUG_RETURN(ma_control_file_write_and_force(CONTROL_FILE_IMPOSSIBLE_LSN,
-                                                CONTROL_FILE_IMPOSSIBLE_FILENO,
+    DBUG_RETURN(ma_control_file_write_and_force(LSN_IMPOSSIBLE,
+                                                FILENO_IMPOSSIBLE,
                                                 CONTROL_FILE_UPDATE_ALL));
   }
 
@@ -315,8 +311,8 @@ int ma_control_file_end()
     As this module owns these variables, closing the module forbids access to
     them (just a safety):
   */
-  last_checkpoint_lsn= CONTROL_FILE_IMPOSSIBLE_LSN;
-  last_logno= CONTROL_FILE_IMPOSSIBLE_FILENO;
+  last_checkpoint_lsn= LSN_IMPOSSIBLE;
+  last_logno= FILENO_IMPOSSIBLE;
 
   DBUG_RETURN(close_error);
 }

storage/maria/ma_control_file.h

@@ -19,27 +19,17 @@
 */
 
 #define CONTROL_FILE_BASE_NAME "maria_control"
-/*
-  indicate absence of the log file number; first log is always number 1, 0 is
-  impossible.
-*/
-#define CONTROL_FILE_IMPOSSIBLE_FILENO 0
-/* logs always have a header */
-#define CONTROL_FILE_IMPOSSIBLE_LOG_OFFSET 0
-/* indicate absence of LSN.  */
-#define CONTROL_FILE_IMPOSSIBLE_LSN ((LSN)0)
 
 /* Here is the interface of this module */
 
 /*
   LSN of the last checkoint
-  (if last_checkpoint_lsn == CONTROL_FILE_IMPOSSIBLE_LSN
-  then there was never a checkpoint)
+  (if last_checkpoint_lsn == LSN_IMPOSSIBLE then there was never a checkpoint)
 */
 extern LSN last_checkpoint_lsn;
 /*
-  Last log number (if last_logno ==
-  CONTROL_FILE_IMPOSSIBLE_FILENO then there is no log file yet)
+  Last log number (if last_logno == FILENO_IMPOSSIBLE then there is no log
+  file yet)
 */
 extern uint32 last_logno;
 
@@ -51,6 +41,7 @@ typedef enum enum_control_file_error {
   CONTROL_FILE_TOO_BIG,
   CONTROL_FILE_BAD_MAGIC_STRING,
   CONTROL_FILE_BAD_CHECKSUM,
+  CONTROL_FILE_MISSING,
   CONTROL_FILE_UNKNOWN_ERROR /* any other error */
 } CONTROL_FILE_ERROR;
 
@@ -63,11 +54,11 @@ extern "C" {
 #endif
 
 /*
-  Looks for the control file. If absent, it's a fresh start, create file.
-  If present, read it to find out last checkpoint's LSN and last log.
+  Looks for the control file. If none and creation was requested, creates file.
+  If present, reads it to find out last checkpoint's LSN and last log.
   Called at engine's start.
 */
-CONTROL_FILE_ERROR ma_control_file_create_or_open();
+CONTROL_FILE_ERROR ma_control_file_create_or_open(my_bool);
 /*
   Write information durably to the control file.
   Called when we have created a new log (after syncing this log's creation)

storage/maria/ma_create.c

@@ -52,8 +52,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
        unique_key_parts,fulltext_keys,offset, not_block_record_extra_length;
   uint max_field_lengths, extra_header_size;
   ulong reclength, real_reclength,min_pack_length;
-  char filename[FN_REFLEN], dlinkname[FN_REFLEN], *dlinkname_ptr= NULL,
-    klinkname[FN_REFLEN], *klinkname_ptr= NULL;
+  char filename[FN_REFLEN], linkname[FN_REFLEN], *linkname_ptr;
   ulong pack_reclength;
   ulonglong tot_length,max_rows, tmp;
   enum en_fieldtype type;
@@ -628,7 +627,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
 
   share.state.dellink = HA_OFFSET_ERROR;
   share.state.first_bitmap_with_space= 0;
-  share.state.create_rename_lsn= 0;
+  share.state.create_rename_lsn= LSN_IMPOSSIBLE;
   share.state.process=	(ulong) getpid();
   share.state.unique=	(ulong) 0;
   share.state.update_count=(ulong) 0;
@@ -721,9 +720,9 @@ int maria_create(const char *name, enum data_file_type datafile_type,
                 MY_UNPACK_FILENAME | (have_iext ? MY_REPLACE_EXT :
                                       MY_APPEND_EXT));
     }
-    fn_format(klinkname, name, "", MARIA_NAME_IEXT,
+    fn_format(linkname, name, "", MARIA_NAME_IEXT,
               MY_UNPACK_FILENAME|MY_APPEND_EXT);
-    klinkname_ptr= klinkname;
+    linkname_ptr= linkname;
     /*
       Don't create the table if the link or file exists to ensure that one
       doesn't accidently destroy another table.
@@ -739,6 +738,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
               (MY_UNPACK_FILENAME |
                (flags & HA_DONT_TOUCH_DATA) ? MY_RETURN_REAL_PATH : 0) |
                 MY_APPEND_EXT);
+    linkname_ptr= NULL;
     /*
       Replace the current file.
       Don't sync dir now if the data file has the same path.
@@ -761,7 +761,7 @@ int maria_create(const char *name, enum data_file_type datafile_type,
     goto err;
   }
 
-  if ((file= my_create_with_symlink(klinkname_ptr, filename, 0, create_mode,
+  if ((file= my_create_with_symlink(linkname_ptr, filename, 0, create_mode,
 				    MYF(MY_WME|create_flag))) < 0)
     goto err;
   errpos=1;
@@ -788,19 +788,20 @@ int maria_create(const char *name, enum data_file_type datafile_type,
                   MY_UNPACK_FILENAME |
                   (have_dext ? MY_REPLACE_EXT : MY_APPEND_EXT));
       }
-      fn_format(dlinkname, name, "",MARIA_NAME_DEXT,
+      fn_format(linkname, name, "",MARIA_NAME_DEXT,
                 MY_UNPACK_FILENAME | MY_APPEND_EXT);
-      dlinkname_ptr= dlinkname;
+      linkname_ptr= linkname;
       create_flag=0;
     }
     else
     {
       fn_format(filename,name,"", MARIA_NAME_DEXT,
                 MY_UNPACK_FILENAME | MY_APPEND_EXT);
+      linkname_ptr= NULL;
       create_flag=MY_DELETE_OLD;
     }
     if ((dfile=
-         my_create_with_symlink(dlinkname_ptr, filename, 0, create_mode,
+         my_create_with_symlink(linkname_ptr, filename, 0, create_mode,
                                 MYF(MY_WME | create_flag | sync_dir))) < 0)
       goto err;
     errpos=3;
@@ -948,15 +949,15 @@ int maria_create(const char *name, enum data_file_type datafile_type,
       not log 1 KB of mostly zeroes if this is a small table.
     */
     char empty_string[]= "";
-    LEX_STRING log_array[TRANSLOG_INTERNAL_PARTS + 3];
+    LEX_STRING log_array[TRANSLOG_INTERNAL_PARTS + 4];
     uint total_rec_length= 0;
     uint i;
-    log_array[TRANSLOG_INTERNAL_PARTS + 0].length= 1 + 2 +
+    log_array[TRANSLOG_INTERNAL_PARTS + 1].length= 1 + 2 + 2 +
       kfile_size_before_extension;
     /* we are needing maybe 64 kB, so don't use the stack */
-    log_data= my_malloc(log_array[TRANSLOG_INTERNAL_PARTS + 0].length, MYF(0));
+    log_data= my_malloc(log_array[TRANSLOG_INTERNAL_PARTS + 1].length, MYF(0));
     if ((log_data == NULL) ||
-        my_pread(file, 1 + 2 + log_data, kfile_size_before_extension,
+        my_pread(file, 1 + 2 + 2 + log_data, kfile_size_before_extension,
                  0, MYF(MY_NABP)))
       goto err_no_lock;
     /*
@@ -965,16 +966,21 @@ int maria_create(const char *name, enum data_file_type datafile_type,
     */
     log_data[0]= test(flags & HA_DONT_TOUCH_DATA);
     int2store(log_data + 1, kfile_size_before_extension);
-    log_array[TRANSLOG_INTERNAL_PARTS + 0].str= log_data;
+    int2store(log_data + 1 + 2, share.base.keystart);
+    log_array[TRANSLOG_INTERNAL_PARTS + 0].str= (char *)name;
+    /* we store the end-zero, for Recovery to just pass it to my_create() */
+    log_array[TRANSLOG_INTERNAL_PARTS + 0].length=
+      strlen(log_array[TRANSLOG_INTERNAL_PARTS + 0].str) + 1;
+    log_array[TRANSLOG_INTERNAL_PARTS + 1].str= log_data;
     /* symlink description is also needed for re-creation by Recovery: */
-    log_array[TRANSLOG_INTERNAL_PARTS + 1].str=
-      dlinkname_ptr ? dlinkname : empty_string;
-    log_array[TRANSLOG_INTERNAL_PARTS + 1].length=
-      strlen(log_array[TRANSLOG_INTERNAL_PARTS + 1].str);
-    log_array[TRANSLOG_INTERNAL_PARTS + 2].str=
-      klinkname_ptr ? klinkname : empty_string;
+    log_array[TRANSLOG_INTERNAL_PARTS + 2].str= (char *)
+      (ci->data_file_name ? ci->data_file_name : empty_string);
     log_array[TRANSLOG_INTERNAL_PARTS + 2].length=
-      strlen(log_array[TRANSLOG_INTERNAL_PARTS + 2].str);
+      strlen(log_array[TRANSLOG_INTERNAL_PARTS + 2].str) + 1;
+    log_array[TRANSLOG_INTERNAL_PARTS + 3].str= (char *)
+      (ci->index_file_name ? ci->index_file_name : empty_string);
+    log_array[TRANSLOG_INTERNAL_PARTS + 3].length=
+      strlen(log_array[TRANSLOG_INTERNAL_PARTS + 3].str) + 1;
     for (i= TRANSLOG_INTERNAL_PARTS;
          i < (sizeof(log_array)/sizeof(log_array[0])); i++)
       total_rec_length+= log_array[i].length;

storage/maria/ma_loghandler.h

+// TODO copyright
+
+#ifndef _ma_loghandler_h
+#define _ma_loghandler_h
+
 /* transaction log default cache size  (TODO: make it global variable) */
 #define TRANSLOG_PAGECACHE_SIZE 1024*1024*2
 /* transaction log default file size  (TODO: make it global variable) */
@@ -20,6 +25,7 @@
 #define TRANSLOG_PAGE_SIZE (8*1024)
 
 #include "ma_loghandler_lsn.h"
+#include "trnman_public.h"
 
 /* short transaction ID type */
 typedef uint16 SHORT_TRANSACTION_ID;
@@ -41,6 +47,10 @@ struct st_maria_share;
 #define page_store(T,A)   int5store(T,A)
 #define dirpos_store(T,A) ((*(uchar*) (T)) = A)
 #define pagerange_store(T,A) int2store(T,A)
+#define fileid_korr(P) uint2korr(P)
+#define page_korr(P)   uint5korr(P)
+#define dirpos_korr(P) (P[0])
+#define pagerange_korr(P) uint2korr(P)
 
 /*
   Length of disk drive sector size (we assume that writing it
@@ -228,10 +238,99 @@ extern translog_size_t translog_read_next_record_header(TRANSLOG_SCANNER_DATA
 							*scanner,
 							TRANSLOG_HEADER_BUFFER
 							*buff);
+extern my_bool translog_lock();
+extern my_bool translog_unlock();
 extern void translog_lock_assert_owner();
 extern TRANSLOG_ADDRESS translog_get_horizon();
 extern int translog_assign_id_to_share(struct st_maria_share *share,
                                        struct st_transaction *trn);
 extern void translog_deassign_id_from_share(struct st_maria_share *share);
 extern my_bool translog_inited;
+
+/*
+  all the rest added because of recovery; should we make
+  ma_loghandler_for_recovery.h ?
+*/
+extern LSN first_lsn_in_log();
+
+/* record parts descriptor */
+struct st_translog_parts
+{
+  /* full record length */
+  translog_size_t record_length;
+  /* full record length with chunk headers */
+  translog_size_t total_record_length;
+  /* current part index */
+  uint current;
+  /* total number of elements in parts */
+  uint elements;
+  /* array of parts (LEX_STRING) */
+  LEX_STRING *parts;
+};
+
+typedef my_bool(*prewrite_rec_hook) (enum translog_record_type type,
+                                     TRN *trn, struct st_maria_share *share,
+                                     struct st_translog_parts *parts);
+
+typedef my_bool(*inwrite_rec_hook) (enum translog_record_type type,
+                                    TRN *trn,
+                                    LSN *lsn,
+                                    struct st_translog_parts *parts);
+
+typedef uint16(*read_rec_hook) (enum translog_record_type type,
+                                uint16 read_length, uchar *read_buff,
+                                byte *decoded_buff);
+
+
+/* record classes */
+enum record_class
+{
+  LOGRECTYPE_NOT_ALLOWED,
+  LOGRECTYPE_VARIABLE_LENGTH,
+  LOGRECTYPE_PSEUDOFIXEDLENGTH,
+  LOGRECTYPE_FIXEDLENGTH
+};
+
+/* C++ can't bear that a variable's name is "class" */
+#ifndef __cplusplus
+/*
+  Descriptor of log record type
+  Note: Don't reorder because of constructs later...
+*/
+typedef struct st_log_record_type_descriptor
+{
+  /* internal class of the record */
+  enum record_class class;
+  /*
+    length for fixed-size record, pseudo-fixed record
+    length with uncompressed LSNs
+  */
+  uint16 fixed_length;
+  /* how much record body (belonged to headers too) read with headers */
+  uint16 read_header_len;
+  /* HOOK for writing the record called before lock */
+  prewrite_rec_hook prewrite_hook;
+  /* HOOK for writing the record called when LSN is known, inside lock */
+  inwrite_rec_hook inwrite_hook;
+  /* HOOK for reading headers */
+  read_rec_hook read_hook;
+  /*
+     For pseudo fixed records number of compressed LSNs followed by
+     system header
+  */
+  int16 compressed_LSN;
+  /*  the rest is for maria_read_log & Recovery */
+  /** @brief for debug error messages or "maria_read_log" command-line tool */
+  const char *name;
+  my_bool record_ends_group;
+  /* a function to execute when we see the record during the REDO phase */
+  int (*record_execute_in_redo_phase)(const TRANSLOG_HEADER_BUFFER *);
+  /* a function to execute when we see the record during the UNDO phase */
+  int (*record_execute_in_undo_phase)(const TRANSLOG_HEADER_BUFFER *);
+} LOG_DESC;
+
+extern LOG_DESC log_record_type_descriptor[LOGREC_NUMBER_OF_TYPES];
+#endif
+
 C_MODE_END
+#endif

storage/maria/ma_loghandler_lsn.h

@@ -24,7 +24,7 @@ typedef TRANSLOG_ADDRESS LSN;
 #define LSN_FILE_NO(L) ((L) >> 32)
 
 /* Gets raw file number part of a LSN/log address */
-#define LSN_FINE_NO_PART(L) ((L) & ((int64)0xFFFFFF00000000LL))
+#define LSN_FILE_NO_PART(L) ((L) & ((int64)0xFFFFFF00000000LL))
 
 /* Gets record offset of a LSN/log address */
 #define LSN_OFFSET(L) ((L) & 0xFFFFFFFFL)
@@ -33,7 +33,9 @@ typedef TRANSLOG_ADDRESS LSN;
 #define MAKE_LSN(F,S) ((((uint64)(F)) << 32) | (S))
 
 /* checks LSN */
-#define LSN_VALID(L) DBUG_ASSERT((L) >= 0 && (L) < (uint64)0xFFFFFFFFFFFFFFLL)
+#define LSN_VALID(L)                                    \
+  ((LSN_FILE_NO_PART(L) != FILENO_IMPOSSIBLE) &&        \
+   (LSN_OFFSET(L) != LOG_OFFSET_IMPOSSIBLE))
 
 /* size of stored LSN on a disk, don't change it! */
 #define LSN_STORE_SIZE 7
@@ -51,7 +53,7 @@ typedef TRANSLOG_ADDRESS LSN;
 /* what we need to add to LSN to increase it on one file */
 #define LSN_ONE_FILE ((int64)0x100000000LL)
 
-#define LSN_REPLACE_OFFSET(L, S) (LSN_FINE_NO_PART(L) | (S))
+#define LSN_REPLACE_OFFSET(L, S) (LSN_FILE_NO_PART(L) | (S))
 
 /*
   an 8-byte type whose most significant byte is used for "flags"; 7
@@ -61,4 +63,7 @@ typedef LSN LSN_WITH_FLAGS;
 #define LSN_WITH_FLAGS_TO_LSN(x)   (x & ULL(0x00FFFFFFFFFFFFFF))
 #define LSN_WITH_FLAGS_TO_FLAGS(x) (x & ULL(0xFF00000000000000))
 
+#define FILENO_IMPOSSIBLE     0 /**< log file's numbering starts at 1 */
+#define LOG_OFFSET_IMPOSSIBLE 0 /**< log always has a header */
+#define LSN_IMPOSSIBLE        0
 #endif

storage/maria/ma_pagecache.c

@@ -587,11 +587,7 @@ static uint pagecache_fwrite(PAGECACHE *pagecache,
     DBUG_PRINT("info", ("Log handler call"));
     /* TODO: integrate with page format */
     lsn= lsn_korr(buffer + PAGE_LSN_OFFSET);
-    /*
-      check CONTROL_FILE_IMPOSSIBLE_FILENO &
-      CONTROL_FILE_IMPOSSIBLE_LOG_OFFSET
-    */
-    DBUG_ASSERT(lsn != 0);
+    DBUG_ASSERT(LSN_VALID(lsn));
     translog_flush(lsn);
   }
   DBUG_RETURN(my_pwrite(filedesc->file, buffer, pagecache->block_size,
@@ -2474,7 +2470,7 @@ static void check_and_set_lsn(LSN lsn, PAGECACHE_BLOCK_LINK *block)
     lock                lock change
     pin                 pin page
     first_REDO_LSN_for_page do not set it if it is zero
-    lsn                 if it is not CONTROL_FILE_IMPOSSIBLE_LSN (0) and it
+    lsn                 if it is not LSN_IMPOSSIBLE (0) and it
                         is bigger then LSN on the page it will be written on
                         the page
 
@@ -2566,7 +2562,7 @@ void pagecache_unlock(PAGECACHE *pagecache,
     pagecache           pointer to a page cache data structure
     file                handler for the file for the block of data to be read
     pageno              number of the block of data in the file
-    lsn                 if it is not CONTROL_FILE_IMPOSSIBLE_LSN (0) and it
+    lsn                 if it is not LSN_IMPOSSIBLE (0) and it
                         is bigger then LSN on the page it will be written on
                         the page
 */
@@ -2635,10 +2631,9 @@ void pagecache_unpin(PAGECACHE *pagecache,
     link                direct link to page (returned by read or write)
     lock                lock change
     pin                 pin page
-    first_REDO_LSN_for_page do not set it if it is zero
-    lsn                 if it is not CONTROL_FILE_IMPOSSIBLE_LSN (0) and it
-                        is bigger then LSN on the page it will be written on
-                        the page
+    first_REDO_LSN_for_page do not set it if it is LSN_IMPOSSIBLE (0)
+    lsn                 if it is not LSN_IMPOSSIBLE and it is bigger then
+                        LSN on the page it will be written on the page
 */
 
 void pagecache_unlock_by_link(PAGECACHE *pagecache,
@@ -2681,7 +2676,7 @@ void pagecache_unlock_by_link(PAGECACHE *pagecache,
   DBUG_ASSERT(pagecache->can_be_used);
 
   inc_counter_for_resize_op(pagecache);
-  if (first_REDO_LSN_for_page)
+  if (first_REDO_LSN_for_page != LSN_IMPOSSIBLE)
   {
     /*
       LOCK_READ_UNLOCK is ok here as the page may have first locked
@@ -2694,10 +2689,8 @@ void pagecache_unlock_by_link(PAGECACHE *pagecache,
     if (block->rec_lsn == 0)
       block->rec_lsn= first_REDO_LSN_for_page;
   }
-  if (lsn != 0)
-  {
+  if (lsn != LSN_IMPOSSIBLE)
     check_and_set_lsn(lsn, block);
-  }
 
   if (make_lock_and_pin(pagecache, block, lock, pin))
     DBUG_ASSERT(0);                           /* should not happend */
@@ -2726,7 +2719,7 @@ void pagecache_unlock_by_link(PAGECACHE *pagecache,
     pagecache_unpin_by_link()
     pagecache           pointer to a page cache data structure
     link                direct link to page (returned by read or write)
-    lsn                 if it is not CONTROL_FILE_IMPOSSIBLE_LSN (0) and it
+    lsn                 if it is not LSN_IMPOSSIBLE (0) and it
                         is bigger then LSN on the page it will be written on
                         the page
 */

storage/maria/ma_recovery.h

-/* Copyright (C) 2006 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
+/* Copyright (C) 2006,2007 MySQL AB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by

storage/maria/ma_test1.c

@@ -60,7 +60,7 @@ int main(int argc,char *argv[])
   if (maria_init() ||
       (init_pagecache(maria_pagecache, IO_SIZE*16, 0, 0,
                       maria_block_size) == 0) ||
-      ma_control_file_create_or_open() ||
+      ma_control_file_create_or_open(TRUE) ||
       (init_pagecache(maria_log_pagecache,
                       TRANSLOG_PAGECACHE_SIZE, 0, 0,
                       TRANSLOG_PAGE_SIZE) == 0) ||

storage/maria/ma_test2.c

@@ -224,7 +224,7 @@ int main(int argc, char *argv[])
   /* Maria requires that we always have a page cache */
   if ((init_pagecache(maria_pagecache, pagecache_size, 0, 0,
 		      maria_block_size) == 0) ||
-      ma_control_file_create_or_open() ||
+      ma_control_file_create_or_open(TRUE) ||
       (init_pagecache(maria_log_pagecache,
 		      TRANSLOG_PAGECACHE_SIZE, 0, 0,
 		      TRANSLOG_PAGE_SIZE) == 0) ||

storage/maria/trnman_public.h

@@ -20,6 +20,9 @@
   to include my_atomic.h in C++ code.
 */
 
+#ifndef _trnman_public_h
+#define _trnman_public_h
+
 #include "ma_loghandler_lsn.h"
 
 C_MODE_START
@@ -52,3 +55,4 @@ my_bool trnman_has_locked_tables(TRN *trn);
 void trnman_reset_locked_tables(TRN *trn);
 
 C_MODE_END
+#endif

storage/maria/unittest/ma_control_file-t.c

@@ -121,8 +121,8 @@ static int delete_file(myf my_flags)
     The error will however be printed on stderr.
   */
   my_delete(file_name, my_flags);
-  expect_checkpoint_lsn= CONTROL_FILE_IMPOSSIBLE_LSN;
-  expect_logno= CONTROL_FILE_IMPOSSIBLE_FILENO;
+  expect_checkpoint_lsn= LSN_IMPOSSIBLE;
+  expect_logno= FILENO_IMPOSSIBLE;
 
   return 0;
 }
@@ -146,9 +146,9 @@ static int verify_module_values_match_expected()
 */
 static int verify_module_values_are_impossible()
 {
-  RET_ERR_UNLESS(last_logno == CONTROL_FILE_IMPOSSIBLE_FILENO);
+  RET_ERR_UNLESS(last_logno == FILENO_IMPOSSIBLE);
   RET_ERR_UNLESS(last_checkpoint_lsn ==
-                 CONTROL_FILE_IMPOSSIBLE_LSN);
+                 LSN_IMPOSSIBLE);
   return 0;
 }
 
@@ -164,7 +164,7 @@ static int close_file()
 
 static int create_or_open_file()
 {
-  RET_ERR_UNLESS(ma_control_file_create_or_open() == CONTROL_FILE_OK);
+  RET_ERR_UNLESS(ma_control_file_create_or_open(TRUE) == CONTROL_FILE_OK);
   /* Check that the module reports expected information */
   RET_ERR_UNLESS(verify_module_values_match_expected() == 0);
   return 0;
@@ -188,7 +188,7 @@ static int test_one_log()
   RET_ERR_UNLESS(create_or_open_file() == CONTROL_FILE_OK);
   objs_to_write= CONTROL_FILE_UPDATE_ONLY_LOGNO;
   expect_logno= 123;
-  RET_ERR_UNLESS(write_file(CONTROL_FILE_IMPOSSIBLE_LSN,
+  RET_ERR_UNLESS(write_file(LSN_IMPOSSIBLE,
                             expect_logno,
                             objs_to_write) == 0);
   RET_ERR_UNLESS(close_file() == 0);
@@ -206,7 +206,7 @@ static int test_five_logs()
   for (i= 0; i<5; i++)
   {
     expect_logno*= 3;
-    RET_ERR_UNLESS(write_file(CONTROL_FILE_IMPOSSIBLE_LSN, expect_logno,
+    RET_ERR_UNLESS(write_file(LSN_IMPOSSIBLE, expect_logno,
                               objs_to_write) == 0);
   }
   RET_ERR_UNLESS(close_file() == 0);
@@ -320,7 +320,7 @@ static int test_bad_magic_string()
   RET_ERR_UNLESS(my_pwrite(fd, "papa", 4, 0, MYF(MY_FNABP |  MY_WME)) == 0);
 
   /* Check that control file module sees the problem */
-  RET_ERR_UNLESS(ma_control_file_create_or_open() ==
+  RET_ERR_UNLESS(ma_control_file_create_or_open(TRUE) ==
              CONTROL_FILE_BAD_MAGIC_STRING);
   /* Restore magic string */
   RET_ERR_UNLESS(my_pwrite(fd, buffer, 4, 0, MYF(MY_FNABP |  MY_WME)) == 0);
@@ -346,7 +346,7 @@ static int test_bad_checksum()
   buffer[0]+= 3; /* mangle checksum */
   RET_ERR_UNLESS(my_pwrite(fd, buffer, 1, 8, MYF(MY_FNABP |  MY_WME)) == 0);
   /* Check that control file module sees the problem */
-  RET_ERR_UNLESS(ma_control_file_create_or_open() ==
+  RET_ERR_UNLESS(ma_control_file_create_or_open(TRUE) ==
                  CONTROL_FILE_BAD_CHECKSUM);
   /* Restore checksum */
   buffer[0]-= 3;
@@ -369,10 +369,11 @@ static int test_bad_size()
                           MYF(MY_WME))) >= 0);
   RET_ERR_UNLESS(my_write(fd, buffer, 10, MYF(MY_FNABP |  MY_WME)) == 0);
   /* Check that control file module sees the problem */
-  RET_ERR_UNLESS(ma_control_file_create_or_open() == CONTROL_FILE_TOO_SMALL);
+  RET_ERR_UNLESS(ma_control_file_create_or_open(TRUE) ==
+                 CONTROL_FILE_TOO_SMALL);
   RET_ERR_UNLESS(my_write(fd, buffer, 30, MYF(MY_FNABP |  MY_WME)) == 0);
   /* Check that control file module sees the problem */
-  RET_ERR_UNLESS(ma_control_file_create_or_open() == CONTROL_FILE_TOO_BIG);
+  RET_ERR_UNLESS(ma_control_file_create_or_open(TRUE) == CONTROL_FILE_TOO_BIG);
   RET_ERR_UNLESS(my_close(fd, MYF(MY_WME)) == 0);
 
   /* Leave a correct control file */

storage/maria/unittest/ma_test_loghandler-t.c

@@ -164,7 +164,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open())
+  if (ma_control_file_create_or_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);
@@ -336,7 +336,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   ma_control_file_end();
 
 
-  if (ma_control_file_create_or_open())
+  if (ma_control_file_create_or_open(TRUE))
   {
     fprintf(stderr, "pass2: Can't init control file (%d)\n", errno);
     exit(1);
@@ -398,7 +398,7 @@ int main(int argc __attribute__((unused)), char *argv[])
                 i, errno);
         goto err;
       }
-      if (rec.lsn == CONTROL_FILE_IMPOSSIBLE_LSN)
+      if (rec.lsn == LSN_IMPOSSIBLE)
       {
         if (i != ITERATIONS)
         {
@@ -477,7 +477,7 @@ int main(int argc __attribute__((unused)), char *argv[])
                 "failed (%d)\n", i, errno);
         goto err;
       }
-      if (rec.lsn == CONTROL_FILE_IMPOSSIBLE_LSN)
+      if (rec.lsn == LSN_IMPOSSIBLE)
       {
         fprintf(stderr, "EOL met at the middle of iteration (first var) %u "
                 "instead of beginning of %u\n", i, ITERATIONS);
@@ -572,7 +572,7 @@ int main(int argc __attribute__((unused)), char *argv[])
                 i, errno);
         goto err;
       }
-      if (rec.lsn == CONTROL_FILE_IMPOSSIBLE_LSN)
+      if (rec.lsn == LSN_IMPOSSIBLE)
       {
         fprintf(stderr, "EOL met at the middle of iteration %u "
                 "instead of beginning of %u\n", i, ITERATIONS);

storage/maria/unittest/ma_test_loghandler_multigroup-t.c

@@ -161,7 +161,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open())
+  if (ma_control_file_create_or_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);
@@ -325,7 +325,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   end_pagecache(&pagecache, 1);
   ma_control_file_end();
 
-  if (ma_control_file_create_or_open())
+  if (ma_control_file_create_or_open(TRUE))
   {
     fprintf(stderr, "pass2: Can't init control file (%d)\n", errno);
     exit(1);
@@ -390,7 +390,7 @@ int main(int argc __attribute__((unused)), char *argv[])
         translog_free_record_header(&rec);
         goto err;
       }
-      if (rec.lsn == CONTROL_FILE_IMPOSSIBLE_LSN)
+      if (rec.lsn == LSN_IMPOSSIBLE)
       {
         if (i != ITERATIONS)
         {
@@ -470,7 +470,7 @@ int main(int argc __attribute__((unused)), char *argv[])
                 "failed (%d)\n", i, errno);
         goto err;
       }
-      if (rec.lsn == CONTROL_FILE_IMPOSSIBLE_LSN)
+      if (rec.lsn == LSN_IMPOSSIBLE)
       {
         fprintf(stderr, "EOL met at the middle of iteration (first var) %u "
                 "instead of beginning of %u\n", i, ITERATIONS);
@@ -568,7 +568,7 @@ int main(int argc __attribute__((unused)), char *argv[])
         translog_free_record_header(&rec);
         goto err;
       }
-      if (rec.lsn == CONTROL_FILE_IMPOSSIBLE_LSN)
+      if (rec.lsn == LSN_IMPOSSIBLE)
       {
         fprintf(stderr, "EOL met at the middle of iteration %u "
                 "instead of beginning of %u\n", i, ITERATIONS);

storage/maria/unittest/ma_test_loghandler_multithread-t.c

@@ -270,7 +270,7 @@ int main(int argc __attribute__((unused)),
 
   my_thread_global_init();
 
-  if (ma_control_file_create_or_open())
+  if (ma_control_file_create_or_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);
@@ -384,7 +384,7 @@ int main(int argc __attribute__((unused)),
         translog_free_record_header(&rec);
         goto err;
       }
-      if (rec.lsn == CONTROL_FILE_IMPOSSIBLE_LSN)
+      if (rec.lsn == LSN_IMPOSSIBLE)
       {
         if (i != WRITERS * ITERATIONS * 2)
         {

storage/maria/unittest/ma_test_loghandler_pagecache-t.c

@@ -56,7 +56,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open())
+  if (ma_control_file_create_or_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);

storage/myisam/mi_close.c

@@ -75,6 +75,7 @@ int mi_close(register MI_INFO *info)
         not change the crashed state.
         We can NOT write the state in other cases as other threads
         may be using the file at this point
+        IF using --external-locking.
       */
       if (share->mode != O_RDONLY && mi_is_crashed(info))
 	mi_state_info_write(share->kfile, &share->state, 1);