Fix for Windows-specific bugs:

- one which led REDO_INSERT_ROW_BLOBS to fail to apply
- one excess close ("-1 file left open")
Don't need maria-path option / environment variable.
Fixes for ma_test_all-t to run under Windows.
Port of ma_test_recovery to Perl, written by Jani.


storage/maria/unittest/ma_test_recovery.expected:
  Rename: storage/maria/ma_test_recovery.expected -> storage/maria/unittest/ma_test_recovery.expected
mysys/my_pread.c:
  Fix for Windows-specific bug (maria_read_log -a failed during
  ma_test_all-t): Windows does not have pread() so the branch setting
  HA_ERR_FILE_TOO_SHORT was not compiled in, broke applying of
  REDO_INSERT_ROW_BLOBS. After fixing that, it appeared that in my
  Windows machine, errno is not changed in case of EOF; as we read it
  we have to reset it at start.
  The changed to readbytes!=-1 is to detect EOF
mysys/my_read.c:
  The change to readbytes!=-1 is to detect EOF
storage/maria/ma_loghandler.c:
  Fix for Windows-specific bug: as we don't open the directory
  we should not close it.
storage/maria/ma_page.c:
  This is C, cannot declare variable after instruction.
storage/maria/ma_test_recovery:
  ma_test_recovery.expected moved
storage/maria/unittest/ma_test_all-t:
  Can now safely guess maria_path so don't need the command-line option
  or environment variable. Port to Windows (.exe, different locations
  of executables); can guess suffix, don't need --suffix.
storage/maria/unittest/ma_test_recovery.pl:
  Perl version of ma_test_recovery, written by Jani. Will deprecate the
  shell version.

mysys/my_pread.c

@@ -47,7 +47,7 @@ size_t my_pread(File Filedes, uchar *Buffer, size_t Count, my_off_t offset,
                 myf MyFlags)
 {
   size_t readbytes;
-  int error= 0;
+  int error= 0, save_errno;
 #ifndef DBUG_OFF
   char llbuf[22];
   DBUG_ENTER("my_pread");
@@ -57,26 +57,25 @@ size_t my_pread(File Filedes, uchar *Buffer, size_t Count, my_off_t offset,
 #endif
   for (;;)
   {
-#ifndef __WIN__
-    errno=0;					/* Linux doesn't reset this */
-#endif
+    errno=0;					/* Linux, Windows don't reset this on EOF/success */
 #ifndef HAVE_PREAD
     pthread_mutex_lock(&my_file_info[Filedes].mutex);
     readbytes= (uint) -1;
     error= (lseek(Filedes, offset, MY_SEEK_SET) == (my_off_t) -1 ||
 	    (readbytes= read(Filedes, Buffer, Count)) != Count);
+    save_errno= errno;
     pthread_mutex_unlock(&my_file_info[Filedes].mutex);
+    if (error)
+    {
+      errno= save_errno;
 #else
     if ((error= ((readbytes= pread(Filedes, Buffer, Count, offset)) != Count)))
     {
+#endif
       my_errno= errno;
-      if (errno == 0 || (readbytes == (size_t) -1 &&
+      if (errno == 0 || (readbytes != (size_t) -1 &&
                          (MyFlags & (MY_NABP | MY_FNABP))))
         my_errno= HA_ERR_FILE_TOO_SHORT;
-    }
-#endif
-    if (error)
-    {
       DBUG_PRINT("warning",("Read only %d bytes off %u from %d, errno: %d",
                             (int) readbytes, (uint) Count,Filedes,my_errno));
 #ifdef THREAD

mysys/my_read.c

@@ -43,11 +43,11 @@ size_t my_read(File Filedes, uchar *Buffer, size_t Count, myf MyFlags)
 
   for (;;)
   {
-    errno= 0;					/* Linux doesn't reset this */
+    errno= 0;					/* Linux, Windows don't reset this on EOF/success */
     if ((readbytes= read(Filedes, Buffer, Count)) != Count)
     {
       my_errno= errno;
-      if (errno == 0 || (readbytes == (size_t) -1 &&
+      if (errno == 0 || (readbytes != (size_t) -1 &&
                          (MyFlags & (MY_NABP | MY_FNABP))))
         my_errno= HA_ERR_FILE_TOO_SHORT;
       DBUG_PRINT("warning",("Read only %d bytes off %lu from %d, errno: %d",

storage/maria/ma_loghandler.c

@@ -3246,6 +3246,7 @@ my_bool translog_init_with_table(const char *directory,
   DBUG_ENTER("translog_init_with_table");
 
   id_to_share= NULL;
+  log_descriptor.directory_fd= -1;
 
   (*init_table_func)();
 
@@ -3914,6 +3915,7 @@ void translog_destroy()
   delete_dynamic(&log_descriptor.open_files);
   delete_dynamic(&log_descriptor.unfinished_files);
 
+  if (log_descriptor.directory_fd >= 0)
     my_close(log_descriptor.directory_fd, MYF(MY_WME));
   my_atomic_rwlock_destroy(&LOCK_id_to_share);
   if (id_to_share != NULL)

storage/maria/ma_page.c

@@ -326,9 +326,12 @@ my_off_t _ma_new(register MARIA_HA *info, int level,
         Next deleted page's number is in the header of the present page
         (single linked list):
       */
+#ifndef DBUG_OFF
+      my_off_t current_key_del;
+#endif
       share->current_key_del= mi_sizekorr(buff+share->keypage_header);
 #ifndef DBUG_OFF
-      my_off_t current_key_del= share->current_key_del;
+      current_key_del= share->current_key_del;
       DBUG_ASSERT(current_key_del != share->state.key_del &&
                   (current_key_del != 0) &&
                   ((current_key_del == HA_OFFSET_ERROR) ||

storage/maria/ma_test_recovery

@@ -219,11 +219,11 @@ fi
 # also note that maria_chk -dvv shows differences for ma_test2 in UNDO phase,
 # this is normal: removing records does not shrink the data/key file,
 # does not put back the "analyzed,optimized keys"(etc) index state.
-diff $maria_path/ma_test_recovery.expected $tmp/ma_test_recovery.output > /dev/null || diff_failed=1
+diff $maria_path/unittest/ma_test_recovery.expected $tmp/ma_test_recovery.output > /dev/null || diff_failed=1
 if [ "$diff_failed" == "1" ]
     then
     echo "UNEXPECTED OUTPUT OF TESTS, FAILED"
-    echo "For more info, do diff $maria_path/ma_test_recovery.expected $tmp/ma_test_recovery.output"
+    echo "For more info, do diff $maria_path/unittest/ma_test_recovery.expected $tmp/ma_test_recovery.output"
     exit 1
     fi
 echo "ALL RECOVERY TESTS OK"

storage/maria/unittest/ma_test_all-t

@@ -13,14 +13,13 @@ $opt_version=         0;
 $opt_help=            0;
 $opt_verbose=         0;
 $opt_abort_on_error=  0;
-$opt_maria_path=      undef();
 $opt_valgrind=        "valgrind --alignment=8 --leak-check=yes";
-$opt_suffix=          "";
 $opt_silent=          "-s";
 $opt_number_of_tests= 0;
 $opt_run_tests=       undef();
 
-my $maria_path= $ENV{'maria_path'};
+my $maria_path;       # path to "storage/maria"
+my $maria_exe_path;   # path to executables (ma_test1, maria_chk etc)
 my $my_progname= $0;
 $my_progname=~ s/.*[\/]//;
 my $runtime_error= 0; # Return 1 if error(s) occur during run
@@ -40,8 +39,8 @@ sub run_tests
   my $nr_tests= 0;
   my $flag_exit= 0;
 
-  if (!GetOptions("help", "version", "verbose", "abort-on-error", "maria-path=s",
-                  "valgrind=s", "suffix=s", "silent=s", "number-of-tests",
+  if (!GetOptions("help", "version", "verbose", "abort-on-error",
+                  "valgrind=s", "silent=s", "number-of-tests",
                   "run-tests=s"))
   {
     $flag_exit= 1;
@@ -51,15 +50,27 @@ sub run_tests
     print "$my_progname version $VER\n";
     exit(0);
   }
-  if (defined($opt_maria_path))
+  $maria_path= dirname($0) . "/..";
+
+  my $suffix= ( $^O =~ /win/i ) ? ".exe" : "";
+  $maria_exe_path= "$maria_path/release";
+  # we use -f, sometimes -x is unexpectedly false in Cygwin
+  if ( ! -f "$maria_exe_path/ma_test1$suffix" )
   {
-    $maria_path= $opt_maria_path;
-  }
-  if (!defined($maria_path) || !length($maria_path))
+    $maria_exe_path= "$maria_path/relwithdebinfo";
+    if ( ! -f "$maria_exe_path/ma_test1$suffix" )
+    {
+      $maria_exe_path= "$maria_path/debug";
+      if ( ! -f "$maria_exe_path/ma_test1$suffix" )
+      {
+        $maria_exe_path= $maria_path;
+        if ( ! -f "$maria_exe_path/ma_test1$suffix" )
         {
-    $maria_path= dirname($0) . '/..';
+           die("Cannot find ma_test1 executable\n");
+        }
+      }
+    }
   }    
-  $ENV{'maria_path'}=$maria_path;
 
   usage() if ($opt_help || $flag_exit);
 
@@ -81,7 +92,7 @@ sub run_tests
   $nr_tests+= run_pack_tests(0, 0, 0, 0, 1) * 4;   #
   $nr_tests+= run_tests_on_warnings_and_errors(0, 0, 0, 1);
   $nr_tests+= run_ma_test_recovery(0, 1);
-  $nr_tests+= run_tests_on_clrs(0, 1);
+  $nr_tests+= run_tests_on_clrs(0, 0, 1);
 
   if ($opt_number_of_tests)
   {
@@ -146,37 +157,37 @@ sub run_tests
   {
     print "Running tests with dynamic row format\n"
   }
-  run_check_tests($opt_suffix, $opt_silent, "", $opt_verbose, 0);
-  run_repair_tests($opt_suffix, $opt_silent, "", $opt_verbose, 0);
-  run_pack_tests($opt_suffix, $opt_silent, "", $opt_verbose, 0);
+  run_check_tests($suffix, $opt_silent, "", $opt_verbose, 0);
+  run_repair_tests($suffix, $opt_silent, "", $opt_verbose, 0);
+  run_pack_tests($suffix, $opt_silent, "", $opt_verbose, 0);
 
   if ($opt_verbose)
   {
     print "\nRunning tests with static row format\n";
   }
-  run_check_tests($opt_suffix, $opt_silent, "-S", $opt_verbose, 0);
-  run_repair_tests($opt_suffix, $opt_silent, "-S", $opt_verbose, 0);
-  run_pack_tests($opt_suffix, $opt_silent, "-S", $opt_verbose, 0);
+  run_check_tests($suffix, $opt_silent, "-S", $opt_verbose, 0);
+  run_repair_tests($suffix, $opt_silent, "-S", $opt_verbose, 0);
+  run_pack_tests($suffix, $opt_silent, "-S", $opt_verbose, 0);
 
   if ($opt_verbose)
   {
     print "\nRunning tests with block row format\n";
   }
-  run_check_tests($opt_suffix, $opt_silent, "-M", $opt_verbose, 0);
-  run_repair_tests($opt_suffix, $opt_silent, "-M", $opt_verbose, 0);
-  run_pack_tests($opt_suffix, $opt_silent, "-M", $opt_verbose, 0);
+  run_check_tests($suffix, $opt_silent, "-M", $opt_verbose, 0);
+  run_repair_tests($suffix, $opt_silent, "-M", $opt_verbose, 0);
+  run_pack_tests($suffix, $opt_silent, "-M", $opt_verbose, 0);
 
   if ($opt_verbose)
   {
     print "\nRunning tests with block row format and transactions\n";
   }
-  run_check_tests($opt_suffix, $opt_silent, "-M -T", $opt_verbose, 0);
-  run_repair_tests($opt_suffix, $opt_silent, "-M -T", $opt_verbose, 0);
-  run_pack_tests($opt_suffix, $opt_silent, "-M -T", $opt_verbose, 0);
+  run_check_tests($suffix, $opt_silent, "-M -T", $opt_verbose, 0);
+  run_repair_tests($suffix, $opt_silent, "-M -T", $opt_verbose, 0);
+  run_pack_tests($suffix, $opt_silent, "-M -T", $opt_verbose, 0);
 
-  run_tests_on_warnings_and_errors($opt_suffix, $opt_silent, $opt_verbose, 0);
+  run_tests_on_warnings_and_errors($suffix, $opt_silent, $opt_verbose, 0);
   run_ma_test_recovery($opt_verbose, 0);
-  run_tests_on_clrs($opt_verbose, 0);
+  run_tests_on_clrs($suffix, $opt_verbose, 0);
 
   exit($runtime_error);
 }
@@ -246,24 +257,24 @@ sub run_check_tests
   for ($i= 0; defined($ma_test1_opt[$i]); $i++)
   {
     unlink <maria_log_control maria_log.*>;
-    ok("$maria_path/ma_test1$suffix $silent $ma_test1_opt[$i][0] $row_type",
+    ok("$maria_exe_path/ma_test1$suffix $silent $ma_test1_opt[$i][0] $row_type",
        $verbose, $i + 1);
-    ok("$maria_path/maria_chk$suffix $ma_test1_opt[$i][1] test1",
+    ok("$maria_exe_path/maria_chk$suffix $ma_test1_opt[$i][1] test1",
        $verbose, $i + 1);
   }
   #
   # These tests are outside the loops. Make sure to include them in
   # nr_tests manually
   #
-  ok("$maria_path/maria_pack$suffix --force -s test1", $verbose, 0);
-  ok("$maria_path/maria_chk$suffix -ess test1", $verbose, 0);
+  ok("$maria_exe_path/maria_pack$suffix --force -s test1", $verbose, 0);
+  ok("$maria_exe_path/maria_chk$suffix -ess test1", $verbose, 0);
 
   for ($i= 0; defined($ma_test2_opt[$i]); $i++)
   {
     unlink <maria_log_control maria_log.*>;
-    ok("$maria_path/ma_test2$suffix $silent $ma_test2_opt[$i][0] $row_type",
+    ok("$maria_exe_path/ma_test2$suffix $silent $ma_test2_opt[$i][0] $row_type",
        $verbose, $i + 1);
-    ok("$maria_path/maria_chk$suffix $ma_test2_opt[$i][1] test2",
+    ok("$maria_exe_path/maria_chk$suffix $ma_test2_opt[$i][1] test2",
        $verbose, $i + 1);
   }
   unlink <maria_log_control maria_log.*>;
@@ -281,35 +292,35 @@ sub run_repair_tests()
   my ($i);
 
   my @t= ($NEW_TEST,
-          "$maria_path/ma_test1$suffix $silent --checksum $row_type",
-          "$maria_path/maria_chk$suffix -se test1",
-          "$maria_path/maria_chk$suffix --silent -re --transaction-log test1",
-          "$maria_path/maria_chk$suffix -rs test1",
-          "$maria_path/maria_chk$suffix -se test1",
-          "$maria_path/maria_chk$suffix -rqs test1",
-          "$maria_path/maria_chk$suffix -se test1",
-          "$maria_path/maria_chk$suffix -rs --correct-checksum test1",
-          "$maria_path/maria_chk$suffix -se test1",
-          "$maria_path/maria_chk$suffix -rqs --correct-checksum test1",
-          "$maria_path/maria_chk$suffix -se test1",
-          "$maria_path/maria_chk$suffix -ros --correct-checksum test1",
-          "$maria_path/maria_chk$suffix -se test1",
-          "$maria_path/maria_chk$suffix -rqos --correct-checksum test1",
-          "$maria_path/maria_chk$suffix -se test1",
-          "$maria_path/maria_chk$suffix -sz test1",
-          "$maria_path/maria_chk$suffix -se test1",
-          "$maria_path/ma_test2$suffix $silent -c -d1 $row_type",
-          "$maria_path/maria_chk$suffix -s --parallel-recover test2",
-          "$maria_path/maria_chk$suffix -se test2",
-          "$maria_path/maria_chk$suffix -s --parallel-recover --quick test2",
-          "$maria_path/maria_chk$suffix -se test2",
-          "$maria_path/ma_test2$suffix $silent -c $row_type",
-          "$maria_path/maria_chk$suffix -se test2",
-          "$maria_path/maria_chk$suffix -sr test2",
-          "$maria_path/maria_chk$suffix -se test2",
-          "$maria_path/ma_test2$suffix $silent -c -t4 -b32768 $row_type",
-          "$maria_path/maria_chk$suffix -s --zerofill test1",
-          "$maria_path/maria_chk$suffix -se test1"
+          "$maria_exe_path/ma_test1$suffix $silent --checksum $row_type",
+          "$maria_exe_path/maria_chk$suffix -se test1",
+          "$maria_exe_path/maria_chk$suffix --silent -re --transaction-log test1",
+          "$maria_exe_path/maria_chk$suffix -rs test1",
+          "$maria_exe_path/maria_chk$suffix -se test1",
+          "$maria_exe_path/maria_chk$suffix -rqs test1",
+          "$maria_exe_path/maria_chk$suffix -se test1",
+          "$maria_exe_path/maria_chk$suffix -rs --correct-checksum test1",
+          "$maria_exe_path/maria_chk$suffix -se test1",
+          "$maria_exe_path/maria_chk$suffix -rqs --correct-checksum test1",
+          "$maria_exe_path/maria_chk$suffix -se test1",
+          "$maria_exe_path/maria_chk$suffix -ros --correct-checksum test1",
+          "$maria_exe_path/maria_chk$suffix -se test1",
+          "$maria_exe_path/maria_chk$suffix -rqos --correct-checksum test1",
+          "$maria_exe_path/maria_chk$suffix -se test1",
+          "$maria_exe_path/maria_chk$suffix -sz test1",
+          "$maria_exe_path/maria_chk$suffix -se test1",
+          "$maria_exe_path/ma_test2$suffix $silent -c -d1 $row_type",
+          "$maria_exe_path/maria_chk$suffix -s --parallel-recover test2",
+          "$maria_exe_path/maria_chk$suffix -se test2",
+          "$maria_exe_path/maria_chk$suffix -s --parallel-recover --quick test2",
+          "$maria_exe_path/maria_chk$suffix -se test2",
+          "$maria_exe_path/ma_test2$suffix $silent -c $row_type",
+          "$maria_exe_path/maria_chk$suffix -se test2",
+          "$maria_exe_path/maria_chk$suffix -sr test2",
+          "$maria_exe_path/maria_chk$suffix -se test2",
+          "$maria_exe_path/ma_test2$suffix $silent -c -t4 -b32768 $row_type",
+          "$maria_exe_path/maria_chk$suffix -s --zerofill test1",
+          "$maria_exe_path/maria_chk$suffix -se test1"
          );
 
   return &count_tests(\@t) if ($count);
@@ -327,49 +338,49 @@ sub run_pack_tests()
   my ($i);
 
   my @t= ($NEW_TEST,
-          "$maria_path/ma_test1$suffix $silent --checksum $row_type",
-          "$maria_path/maria_pack$suffix --force -s test1",
-          "$maria_path/maria_chk$suffix -ess test1",
-          "$maria_path/maria_chk$suffix -rqs test1",
-          "$maria_path/maria_chk$suffix -es test1",
-          "$maria_path/maria_chk$suffix -rs test1",
-          "$maria_path/maria_chk$suffix -es test1",
-          "$maria_path/maria_chk$suffix -rus test1",
-          "$maria_path/maria_chk$suffix -es test1",
+          "$maria_exe_path/ma_test1$suffix $silent --checksum $row_type",
+          "$maria_exe_path/maria_pack$suffix --force -s test1",
+          "$maria_exe_path/maria_chk$suffix -ess test1",
+          "$maria_exe_path/maria_chk$suffix -rqs test1",
+          "$maria_exe_path/maria_chk$suffix -es test1",
+          "$maria_exe_path/maria_chk$suffix -rs test1",
+          "$maria_exe_path/maria_chk$suffix -es test1",
+          "$maria_exe_path/maria_chk$suffix -rus test1",
+          "$maria_exe_path/maria_chk$suffix -es test1",
           $NEW_TEST,
-          "$maria_path/ma_test1$suffix $silent --checksum $row_type",
-          "$maria_path/maria_pack$suffix --force -s test1",
-          "$maria_path/maria_chk$suffix -rus --safe-recover test1",
-          "$maria_path/maria_chk$suffix -es test1",
+          "$maria_exe_path/ma_test1$suffix $silent --checksum $row_type",
+          "$maria_exe_path/maria_pack$suffix --force -s test1",
+          "$maria_exe_path/maria_chk$suffix -rus --safe-recover test1",
+          "$maria_exe_path/maria_chk$suffix -es test1",
           $NEW_TEST,
-          "$maria_path/ma_test1$suffix $silent --checksum -S $row_type",
-          "$maria_path/maria_chk$suffix -se test1",
-          "$maria_path/maria_chk$suffix -ros test1",
-          "$maria_path/maria_chk$suffix -rqs test1",
-          "$maria_path/maria_chk$suffix -se test1",
+          "$maria_exe_path/ma_test1$suffix $silent --checksum -S $row_type",
+          "$maria_exe_path/maria_chk$suffix -se test1",
+          "$maria_exe_path/maria_chk$suffix -ros test1",
+          "$maria_exe_path/maria_chk$suffix -rqs test1",
+          "$maria_exe_path/maria_chk$suffix -se test1",
           $NEW_TEST,
-          "$maria_path/maria_pack$suffix --force -s test1",
-          "$maria_path/maria_chk$suffix -rqs test1",
-          "$maria_path/maria_chk$suffix -es test1",
-          "$maria_path/maria_chk$suffix -rus test1",
-          "$maria_path/maria_chk$suffix -es test1",
+          "$maria_exe_path/maria_pack$suffix --force -s test1",
+          "$maria_exe_path/maria_chk$suffix -rqs test1",
+          "$maria_exe_path/maria_chk$suffix -es test1",
+          "$maria_exe_path/maria_chk$suffix -rus test1",
+          "$maria_exe_path/maria_chk$suffix -es test1",
           $NEW_TEST,
-          "$maria_path/ma_test2$suffix $silent -c -d1 $row_type",
-          "$maria_path/maria_chk$suffix -s --parallel-recover test2",
-          "$maria_path/maria_chk$suffix -se test2",
-          "$maria_path/maria_chk$suffix -s --unpack --parallel-recover test2",
-          "$maria_path/maria_chk$suffix -se test2",
-          "$maria_path/maria_pack$suffix --force -s test1",
-          "$maria_path/maria_chk$suffix -s --unpack --parallel-recover test2",
-          "$maria_path/maria_chk$suffix -se test2",
+          "$maria_exe_path/ma_test2$suffix $silent -c -d1 $row_type",
+          "$maria_exe_path/maria_chk$suffix -s --parallel-recover test2",
+          "$maria_exe_path/maria_chk$suffix -se test2",
+          "$maria_exe_path/maria_chk$suffix -s --unpack --parallel-recover test2",
+          "$maria_exe_path/maria_chk$suffix -se test2",
+          "$maria_exe_path/maria_pack$suffix --force -s test1",
+          "$maria_exe_path/maria_chk$suffix -s --unpack --parallel-recover test2",
+          "$maria_exe_path/maria_chk$suffix -se test2",
           $NEW_TEST,
-          "$maria_path/ma_test1$suffix $silent -c $row_type",
+          "$maria_exe_path/ma_test1$suffix $silent -c $row_type",
 	  "cp test1.MAD test2.MAD",
 	  "cp test1.MAI test2.MAI",
-          "$maria_path/maria_pack$suffix --force -s --join=test3 test1 test2",
-          "$maria_path/maria_chk -s test3",
-          "$maria_path/maria_chk -s --safe-recover test3",
-          "$maria_path/maria_chk -s test3"
+          "$maria_exe_path/maria_pack$suffix --force -s --join=test3 test1 test2",
+          "$maria_exe_path/maria_chk -s test3",
+          "$maria_exe_path/maria_chk -s --safe-recover test3",
+          "$maria_exe_path/maria_chk -s test3"
          );
 
   return &count_tests(\@t) if ($count);
@@ -388,25 +399,25 @@ sub run_tests_on_warnings_and_errors
 
   return 9 if ($count);  # Number of tests in this function, e.g. calls to ok()
 
-  ok("$maria_path/ma_test2$suffix $silent -L -K -W -P -S -R1 -m500",
+  ok("$maria_exe_path/ma_test2$suffix $silent -L -K -W -P -S -R1 -m500",
      $verbose, 0);
-  ok("$maria_path/maria_chk$suffix -sm test2", $verbose, 0);
+  ok("$maria_exe_path/maria_chk$suffix -sm test2", $verbose, 0);
   # ma_test2$suffix $silent -L -K -R1 -m2000 ;  Should give error 135\n
   # In the following a failure is a success and success is a failure
-  $com=  "$maria_path/ma_test2$suffix $silent -L -K -R1 -m2000 ";
+  $com=  "$maria_exe_path/ma_test2$suffix $silent -L -K -R1 -m2000 ";
   $com.= ">ma_test2_message.txt 2>&1";
   ok($com, $verbose, 0, 1);
   ok("cat ma_test2_message.txt", $verbose, 0);
   ok("grep \"Error: 135\" ma_test2_message.txt > /dev/null", $verbose, 0);
-  # maria_path/maria_chk$suffix -sm test2 will warn that
+  # maria_exe_path/maria_chk$suffix -sm test2 will warn that
   # Datafile is almost full
-  ok("$maria_path/maria_chk$suffix -sm test2 >ma_test2_message.txt 2>&1",
+  ok("$maria_exe_path/maria_chk$suffix -sm test2 >ma_test2_message.txt 2>&1",
      $verbose, 0);
   ok("cat ma_test2_message.txt", $verbose, 0);
   ok("grep \"warning: Datafile is almost full\" ma_test2_message.txt>/dev/null",
      $verbose, 0);
   unlink <ma_test2_message.txt>;
-  ok("$maria_path/maria_chk$suffix -ssm test2", $verbose, 0);
+  ok("$maria_exe_path/maria_chk$suffix -ssm test2", $verbose, 0);
 
   return 0;
 }
@@ -420,7 +431,7 @@ sub run_ma_test_recovery
   my ($verbose, $count)= @_;
 
   return 1 if ($count);                # Number of tests in this function
-  ok("$maria_path/ma_test_recovery", $verbose, 0);
+  ok("$maria_path/unittest/ma_test_recovery.pl", $verbose, 0);
   return 0;
 }
 
@@ -430,36 +441,36 @@ sub run_ma_test_recovery
 
 sub run_tests_on_clrs
 {
-  my ($verbose, $count)= @_;
+  my ($suffix, $verbose, $count)= @_;
   my ($i);
 
   my @t= ($NEW_TEST,
-          "$maria_path/ma_test2 -s -L -K -W -P -M -T -c -b -t2 -A1",
+          "$maria_exe_path/ma_test2$suffix -s -L -K -W -P -M -T -c -b -t2 -A1",
           "cp maria_log_control tmp",
-          "$maria_path/maria_read_log -a -s",
-          "$maria_path/maria_chk -s -e test2",
+          "$maria_exe_path/maria_read_log$suffix -a -s",
+          "$maria_exe_path/maria_chk$suffix -s -e test2",
           "cp tmp/maria_log_control .",
           "rm test2.MA?",
-          "$maria_path/maria_read_log -a -s",
-          "$maria_path/maria_chk -s -e test2",
+          "$maria_exe_path/maria_read_log$suffix -a -s",
+          "$maria_exe_path/maria_chk$suffix -s -e test2",
           "rm test2.MA?",
           $NEW_TEST,
-          "$maria_path/ma_test2 -s -L -K -W -P -M -T -c -b -t2 -A1",
-          "$maria_path/maria_read_log -a -s",
-          "$maria_path/maria_chk -s -e test2",
+          "$maria_exe_path/ma_test2$suffix -s -L -K -W -P -M -T -c -b -t2 -A1",
+          "$maria_exe_path/maria_read_log$suffix -a -s",
+          "$maria_exe_path/maria_chk$suffix -s -e test2",
           "rm test2.MA?",
-          "$maria_path/maria_read_log -a -s",
-          "$maria_path/maria_chk -e -s test2",
+          "$maria_exe_path/maria_read_log$suffix -a -s",
+          "$maria_exe_path/maria_chk$suffix -e -s test2",
           "rm test2.MA?",
           $NEW_TEST,
-          "$maria_path/ma_test2 -s -L -K -W -P -M -T -c -b32768 -t4 -A1",
-          "$maria_path/maria_read_log -a -s",
-          "$maria_path/maria_chk -es test2",
-          "$maria_path/maria_read_log -a -s",
-          "$maria_path/maria_chk -es test2",
+          "$maria_exe_path/ma_test2$suffix -s -L -K -W -P -M -T -c -b32768 -t4 -A1",
+          "$maria_exe_path/maria_read_log$suffix -a -s",
+          "$maria_exe_path/maria_chk$suffix -es test2",
+          "$maria_exe_path/maria_read_log$suffix -a -s",
+          "$maria_exe_path/maria_chk$suffix -es test2",
           "rm test2.MA?",
-          "$maria_path/maria_read_log -a -s",
-          "$maria_path/maria_chk -es test2",
+          "$maria_exe_path/maria_read_log$suffix -a -s",
+          "$maria_exe_path/maria_chk$suffix -es test2",
           "rm test2.MA?"
          );
 
@@ -648,16 +659,11 @@ Options
 --help             Show this help and exit.
 --abort-on-error   Abort at once in case of error.
 --number-of-tests  Print the total number of tests and exit.
---maria-path=...   Path to maria test files. You can set this as an
-                   environment variable 'maria_path' also.
-                   (maria_path: '$maria_path')
 --run-tests=...    Test number(s) that should be run. You can give just
                    one number or a range. For example 45..89
                    Use this with caution, because some of the tests
                    might depend on previous ones.
 --silent=...       Silent option passed to ma_test* tests ('$opt_silent')
---suffix=...       Suffix for test files (ma_test1, ma_test2 etc.),
-                   if they have one ('$opt_suffix')
 --valgrind=...     Options for valgrind.
                    ('$opt_valgrind')
 --verbose          Be more verbose. Will print each unittest on a line

storage/maria/unittest/ma_test_recovery.pl

+#!/usr/bin/perl -w
+
+use File::Basename;
+
+$|= 1;
+
+my $silent= "-s";
+my $tmp= "./tmp";
+my $maria_path;       # path to "storage/maria"
+my $maria_exe_path;   # path to executables (ma_test1, maria_chk etc)
+
+$maria_path= dirname($0) . "/..";
+
+$suffix= ( $^O =~ /win/i ) ? ".exe" : "";
+$maria_exe_path= "$maria_path/release";
+# we use -f, sometimes -x is unexpectedly false in Cygwin
+if ( ! -f "$maria_exe_path/ma_test1$suffix" )
+{
+  $maria_exe_path= "$maria_path/relwithdebinfo";
+  if ( ! -f "$maria_exe_path/ma_test1$suffix" )
+  {
+    $maria_exe_path= "$maria_path/debug";
+    if ( ! -f "$maria_exe_path/ma_test1$suffix" )
+    {
+      $maria_exe_path= $maria_path;
+      if ( ! -f "$maria_exe_path/ma_test1$suffix" )
+      {
+         die("Cannot find ma_test1 executable\n");
+      }
+    }
+  }
+}    
+
+
+# test data is always put in the current directory or a tmp subdirectory of it
+
+if (! -d "$tmp")
+{
+  mkdir $tmp;
+}
+
+print "MARIA RECOVERY TESTS\n";
+
+my $res;
+
+$res= `$maria_exe_path/maria_read_log$suffix --help | grep IDENTICAL_PAGES_AFTER_RECOVERY`;
+
+if (length($res))
+{
+  print "Recovery tests require compilation with DBUG\n";
+  print "Aborting test\n";
+  exit(1);
+}
+
+# To not flood the screen, we redirect all the commands below to a text file
+# and just give a final error if their output is not as expected
+
+open (MY_LOG, ">$tmp/ma_test_recovery.output") or die "Can't open log file\n";
+print MY_LOG "Testing the REDO PHASE ALONE\n";
+
+# runs a program inserting/deleting rows, then moves the resulting table
+# elsewhere; applies the log and checks that the data file is
+# identical to the saved original.
+
+my @t= ("ma_test1$suffix $silent -M -T -c",
+	"ma_test2$suffix $silent -L -K -W -P -M -T -c -d500",
+	"ma_test2$suffix $silent -M -T -c -b65000",
+	"ma_test2$suffix $silent -M -T -c -b65000 -d800");
+
+my ($table);
+
+foreach my $prog (@t)
+{
+  unlink <maria_log.* maria_log_control>;
+  my $prog_no_suffix= $prog;
+  $prog_no_suffix=~ s/$suffix//;
+  print MY_LOG "TEST WITH $prog_no_suffix\n";
+  $res= `$maria_exe_path/$prog`;
+  print MY_LOG $res;
+  # derive table's name from program's name
+  if ($prog =~ m/ma_(test[0-9]+).*/)
+  {
+    $table= $1;
+  }
+  $com=  "$maria_exe_path/maria_chk$suffix -dvv $table ";
+  $com.= "| grep -v \"Creation time:\" | grep -v \"file length\" ";
+  $com.= "> $tmp/maria_chk_message.good.txt 2>&1";
+  `$com`;
+  my $checksum=`$maria_exe_path/maria_chk$suffix -dss $table`;
+  `mv $table.MAD $tmp/$table-good.MAD`;
+  `mv $table.MAI $tmp/$table-good.MAI`;
+  apply_log($table, "shouldnotchangelog");
+  `cmp $table.MAD $tmp/$table-good.MAD`;
+  `cmp $table.MAI $tmp/$table-good.MAI`;
+  check_table_is_same($table, $checksum);
+  print MY_LOG "testing idempotency\n";
+  apply_log($table, "shouldnotchangelog");
+  $res= `cmp $table.MAD $tmp/$table-good.MAD`;
+  print MY_LOG $res;
+  $res= `cmp $table.MAI $tmp/$table-good.MAI`;
+  print MY_LOG $res;
+  check_table_is_same($table, $checksum);
+}
+
+print MY_LOG "Testing the REDO AND UNDO PHASE\n";
+# The test programs look like:
+# work; commit (time T1); work; exit-without-commit (time T2)
+# We first run the test program and let it exit after T1's commit.
+# Then we run it again and let it exit at T2. Then we compare
+# and expect identity.
+
+my @take_checkpoints= ("no", "yes");
+my @blobs= ("", "-b32768");
+my @test_undo= (1, 2, 3, 4);
+my @t2= ("ma_test1$suffix $silent -M -T -c -N blob -H1",
+	 "--testflag=1",
+	 "--testflag=2 --test-undo=",
+	 "ma_test1$suffix $silent -M -T -c -N blob -H2",
+	 "--testflag=3",
+	 "--testflag=4 --test-undo=",
+	 "ma_test1$suffix $silent -M -T -c -N blob -H2",
+	 "--testflag=2",
+	 "--testflag=3 --test-undo=",
+	 "ma_test2$suffix $silent -L -K -W -P -M -T -c blob -H1",
+	 "-t1",
+	 "-t2 -A",
+	 "ma_test2$suffix $silent -L -K -W -P -M -T -c blob -H1",
+	 "-t1",
+	 "-t6 -A");
+
+foreach my $take_checkpoint (@take_checkpoints)
+{
+  my ($i, $j, $k, $commit_run_args, $abort_run_args);
+  # we test table without blobs and then table with blobs
+  for ($i= 0; defined($blobs[$i]); $i++)
+  {
+    for ($j= 0; defined($test_undo[$j]); $j++)
+    {
+      # first iteration tests rollback of insert, second tests rollback of delete
+      # -N (create NULL fields) is needed because --test-undo adds it anyway
+      for ($k= 0; defined($t2[$k]); $k+= 3)
+      {
+	$prog= $t2[$k];
+	$prog=~ s/blob/$blobs[$i]/;
+	if ("$take_checkpoint" eq "no")
+	{
+	  $prog=~ s/\s+\-H[0-9]+//;
+	}
+	$commit_run_args= $t2[$k + 1];
+	$abort_run_args= $t2[$k + 2];
+	unlink <maria_log.* maria_log_control>;
+      my $prog_no_suffix= $prog;
+      $prog_no_suffix=~ s/$suffix//;
+	print MY_LOG "TEST WITH $prog_no_suffix $commit_run_args (commit at end)\n";
+	$res= `$maria_exe_path/$prog $commit_run_args`;
+	print MY_LOG $res;
+	# derive table's name from program's name
+	if ($prog =~ m/ma_(test[0-9]+).*/)
+	{
+	  $table= $1;
+	}
+	$com=  "$maria_exe_path/maria_chk$suffix -dvv $table ";
+	$com.= "| grep -v \"Creation time:\" | grep -v \"file length\" ";
+	$com.= "> $tmp/maria_chk_message.good.txt 2>&1";
+	$res= `$com`;
+	print MY_LOG $res;
+	$checksum= `$maria_exe_path/maria_chk$suffix -dss $table`;
+	`mv $table.MAD $tmp/$table-good.MAD`;
+	`mv $table.MAI $tmp/$table-good.MAI`;
+	unlink <maria_log.* maria_log_control>;
+	print MY_LOG "TEST WITH $prog_no_suffix $abort_run_args$test_undo[$j] (additional aborted work)\n";
+	$res= `$maria_exe_path/$prog $abort_run_args$test_undo[$j]`;
+	print MY_LOG $res;
+	`cp $table.MAD $tmp/$table-before_undo.MAD`;
+	`cp $table.MAI $tmp/$table-before_undo.MAI`;
+
+	# The lines below seem unneeded, will be removed soon
+	# We have to copy and restore logs, as running maria_read_log will
+	# change the maria_control_file
+	#    rm -f $tmp/maria_log.* $tmp/maria_log_control
+	#    cp $maria_path/maria_log* $tmp
+
+	if ($test_undo[$j] != 3)
+	{
+	  apply_log($table, "shouldchangelog"); # should undo aborted work
+	}
+        else
+	{
+	  # probably nothing to undo went to log or data file
+	  apply_log($table, "dontknow");
+	}
+	`cp $table.MAD $tmp/$table-after_undo.MAD`;
+	`cp $table.MAI $tmp/$table-after_undo.MAI`;
+
+	# It is impossible to do a "cmp" between .good and .after_undo,
+	# because the UNDO phase generated log
+	# records whose LSN tagged pages. Another reason is that rolling back
+	# INSERT only marks the rows free, does not empty them (optimization), so
+	# traces of the INSERT+rollback remain.
+	
+	check_table_is_same($table, $checksum);
+	print MY_LOG "testing idempotency\n";
+	apply_log($table, "shouldnotchangelog");
+	# We can't do a binary compary as there may have been different number
+	# of calls to compact_page. We can enable this if we first call
+	# maria-check to generate identically compacted pages.
+	#    cmp $table.MAD $tmp/$table-after_undo.MAD
+	$res= `cmp $table.MAI $tmp/$table-after_undo.MAI`;
+	print MY_LOG $res;
+	check_table_is_same($table, $checksum);
+	print MY_LOG "testing applying of CLRs to recreate table\n";
+	unlink <$table.MA?>;
+	#    cp $tmp/maria_log* $maria_path  #unneeded
+	apply_log($table, "shouldnotchangelog");
+	#    cmp $table.MAD $tmp/$table-after_undo.MAD
+	$res= `cmp $table.MAI $tmp/$table-after_undo.MAI`;
+	print MY_LOG $res;
+	check_table_is_same($table, $checksum);
+      }
+      unlink <$table.* $tmp/$table* $tmp/maria_chk_*.txt $tmp/maria_read_log_$table.txt>;
+    }
+  }
+}
+
+if ($? >> 8)
+{
+  print "Some test failed\n";
+  exit(1);
+}
+
+# also note that maria_chk -dvv shows differences for ma_test2 in UNDO phase,
+# this is normal: removing records does not shrink the data/key file,
+# does not put back the "analyzed,optimized keys"(etc) index state.
+`diff -b $maria_path/unittest/ma_test_recovery.expected $tmp/ma_test_recovery.output`;
+if ($? >> 8)
+{
+  print "UNEXPECTED OUTPUT OF TESTS, FAILED\n";
+  print "For more info, do diff -b $maria_path/unittest/ma_test_recovery.expected ";
+  print "$tmp/ma_test_recovery.output\n";
+  exit(1);
+}
+print "ALL RECOVERY TESTS OK\n";
+
+####
+#### check_table_is_same
+####
+
+sub check_table_is_same
+{
+  my ($table, $checksum)= @_;
+  my ($com, $checksum2, $res);
+
+  # Computes checksum of new table and compares to checksum of old table
+  # Shows any difference in table's state (info from the index's header)
+  # Data/key file length is random in ma_test2 (as it uses srand() which
+  # may differ between machines).
+
+  $com=  "$maria_exe_path/maria_chk$suffix -dvv $table | grep -v \"Creation time:\" ";
+  $com.= "| grep -v \"file length\"> $tmp/maria_chk_message.txt 2>&1";
+  $res= `$com`;
+  print MY_LOG $res;
+  $res= `$maria_exe_path/maria_chk$suffix -s -e --read-only $table`;
+  print MY_LOG $res;
+  $checksum2= `$maria_exe_path/maria_chk$suffix -dss $table`;
+  if ("$checksum" ne "$checksum2")
+  {
+    print MY_LOG "checksum differs for $table before and after recovery\n";
+    return 1;
+  }
+
+  $com=  "diff $tmp/maria_chk_message.good.txt $tmp/maria_chk_message.txt ";
+  $com.= "> $tmp/maria_chk_diff.txt || true";
+  $res= `$com`;
+  print MY_LOG $res;
+
+  if (-s "$tmp/maria_chk_diff.txt")
+  {
+    print MY_LOG "Differences in maria_chk -dvv, recovery not yet perfect !\n";
+    print MY_LOG "========DIFF START=======\n";
+    open(MY_FILE, "<$tmp/maria_chk_diff.txt") || die "Can't open file maria_chk_diff.txt\n";
+    while (<MY_FILE>)
+    {
+      print MY_LOG $_;
+    }
+    close(MY_FILE);
+    print MY_LOG "========DIFF END=======\n";
+  }
+}
+
+####
+#### apply_log
+####
+
+sub apply_log
+{
+  my ($table, $shouldchangelog)= @_;
+  my ($log_md5);
+
+  # applies log, can verify if applying did write to log or not
+
+  if ("$shouldchangelog" ne "shouldnotchangelog" &&
+      "$shouldchangelog" ne "shouldchangelog" &&
+      "$shouldchangelog" ne "dontknow" )
+  {
+    print MY_LOG "bad argument '$shouldchangelog'\n";
+    return 1;
+  }
+  $log_md5= `md5sum maria_log.*`;
+
+  print MY_LOG "applying log\n";
+  `$maria_exe_path/maria_read_log$suffix -a > $tmp/maria_read_log_$table.txt`;
+  $log_md5_2= `md5sum maria_log.*`;
+  if ("$log_md5" ne "$log_md5_2" )
+  {
+    if ("$shouldchangelog" eq "shouldnotchangelog")
+    {
+      print MY_LOG "maria_read_log should not have modified the log\n";
+      return 1;
+    }
+  }
+  elsif ("$shouldchangelog" eq "shouldchangelog")
+  {
+    print MY_LOG "maria_read_log should have modified the log\n";
+    return 1;
+  }
+}