Fix for LP#634943 "marked as crashed", "zerofilling" and "wrong data in...

Fix for LP#634943 "marked as crashed", "zerofilling" and "wrong data in bitmap" when recovering Aria tables
This was an interaction of several bugs:
- Tables marked as opened was not properly unmarked on recovery if there was not changes since checkpoint
- zerofill of tables put wrong data in bitmap if directory for page was full
- Tables was thought as 'moved' during recovery if they had a create_lsn bigger than the lsn in the control file.


storage/maria/ha_maria.cc:
  If table is moved and crashed, threat it as crashed.
  (Not a related to this bug, but still good to have fixed)
storage/maria/ma_blockrec.c:
  Make enough_free_entries_on_page() global
storage/maria/ma_blockrec.h:
  Make enough_free_entries_on_page() global
storage/maria/ma_check.c:
  If directory is full, mark page full. Fixes bug in zerofill
storage/maria/ma_open.c:
  Don't marke files as MOVED during recovery if create_trid > trnman_max_trid, as this fails for tables created after checkpoint.
storage/maria/ma_recovery.c:
  Reset open_count in file that was open during crash and was part of checkpoint.
  Fixed wrong warning of 'open count' after recovery of files that was not touched since checkpoint.
storage/maria/maria_chk.c:
  Changed not documented option --log-dir to --logdir
  Document more of the options.
  Clean up output for --help
storage/maria/trnman.c:
  Added DBUG_PRINT

storage/maria/ha_maria.cc

@@ -2006,14 +2006,16 @@ bool ha_maria::check_and_repair(THD *thd)
 
   check_opt.init();
 
-  if (file->s->state.changed & STATE_MOVED)
+  error= 1;
+  if ((file->s->state.changed &
+       (STATE_CRASHED | STATE_CRASHED_ON_REPAIR | STATE_MOVED)) ==
+      STATE_MOVED)
   {
-    sql_print_information("Zerofilling table:   '%s'", table->s->path.str);
+    sql_print_information("Zerofilling moved table:  '%s'",
+                          table->s->path.str);
     if (!(error= zerofill(thd, &check_opt)))
       DBUG_RETURN(0);
   }
-  else
-    error= 1;
 
   /*
     if we got this far - the table is crashed.

storage/maria/ma_blockrec.c

@@ -742,8 +742,8 @@ static my_bool enough_free_entries(uchar *buff, uint block_size,
    @return 1	There is room for more entries on the page
 */
 
-static my_bool enough_free_entries_on_page(MARIA_SHARE *share,
-                                           uchar *page_buff)
+my_bool enough_free_entries_on_page(MARIA_SHARE *share,
+                                    uchar *page_buff)
 {
   enum en_page_type page_type;
   page_type= (enum en_page_type) (page_buff[PAGE_TYPE_OFFSET] &

storage/maria/ma_blockrec.h

@@ -176,6 +176,7 @@ my_bool _ma_compare_block_record(register MARIA_HA *info,
 void    _ma_compact_block_page(uchar *buff, uint block_size, uint rownr,
                                my_bool extend_block, TrID min_read_from,
                                uint min_row_length);
+my_bool enough_free_entries_on_page(MARIA_SHARE *share, uchar *page_buff);
 TRANSLOG_ADDRESS
 maria_page_get_lsn(uchar *page, pgcache_page_no_t page_no, uchar* data_ptr);
 

storage/maria/ma_check.c

@@ -3376,7 +3376,7 @@ static my_bool maria_zerofill_data(HA_CHECK *param, MARIA_HA *info,
     case TAIL_PAGE:
     {
       uint max_entry= (uint) buff[DIR_COUNT_OFFSET];
-      uint offset, dir_start;
+      uint offset, dir_start, empty_space;
       uchar *dir;
 
       if (zero_lsn)
@@ -3389,9 +3389,13 @@ static my_bool maria_zerofill_data(HA_CHECK *param, MARIA_HA *info,
                                is_head_page ? ~(TrID) 0 : 0,
                                is_head_page ?
                                share->base.min_block_length : 0);
+
         /* compactation may have increased free space */
+        empty_space= uint2korr(buff + EMPTY_SPACE_OFFSET);
+        if (!enough_free_entries_on_page(share, buff))
+          empty_space= 0;                         /* Page is full */
         if (_ma_bitmap_set(info, page, is_head_page,
-                           uint2korr(buff + EMPTY_SPACE_OFFSET)))
+                           empty_space))
           goto err;
 
         /* Zerofill the not used part */

storage/maria/ma_open.c

@@ -434,8 +434,14 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags)
         share->base.born_transactional &&
         ((!(open_flags & HA_OPEN_IGNORE_MOVED_STATE) &&
           memcmp(share->base.uuid, maria_uuid, MY_UUID_SIZE)) ||
-         share->state.create_trid > trnman_get_max_trid()))
+         (share->state.create_trid > trnman_get_max_trid() &&
+          !maria_in_recovery)))
     {
+      DBUG_PRINT("warning", ("table is moved from another system.  uuid_diff: %d  create_trid: %lu  max_trid: %lu",
+                            memcmp(share->base.uuid, maria_uuid,
+                                   MY_UUID_SIZE) != 0,
+                             (ulong) share->state.create_trid,
+                             (ulong) trnman_get_max_trid()));
       if (open_flags & HA_OPEN_FOR_REPAIR)
         share->state.changed|= STATE_MOVED;
       else

storage/maria/ma_recovery.c

@@ -3321,7 +3321,12 @@ static int close_all_tables(void)
       state while they were used. As Recovery corrected them, don't alarm the
       user, don't ask for a table check:
     */
-    info->s->state.open_count= 0;
+    if (info->s->state.open_count != 0)
+    {
+      /* let ma_close() mark the table properly closed */
+      info->s->state.open_count= 1;
+      info->s->global_changed= 1;
+    }
     prepare_table_for_close(info, addr);
     error|= maria_close(info);
     pthread_mutex_lock(&THR_LOCK_maria);

storage/maria/maria_chk.c

@@ -271,10 +271,10 @@ static struct my_option my_long_options[] =
    &check_param.keys_in_use,
    0, GET_ULL, REQUIRED_ARG, -1, 0, 0, 0, 0, 0},
   {"datadir", OPT_DATADIR,
-   "Path for control file (and logs if --log-dir not used).",
+   "Path for control file (and logs if --logdir not used).",
    &maria_data_root, 0, 0, GET_STR, REQUIRED_ARG,
    0, 0, 0, 0, 0, 0},
-  {"log-dir", OPT_LOG_DIR,
+  {"logdir", OPT_LOG_DIR,
    "Path for log files.",
    (char**) &opt_log_dir, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"max-record-length", OPT_MAX_RECORD_LENGTH,
@@ -320,9 +320,6 @@ static struct my_option my_long_options[] =
    "Change the collation used by the index",
    (char**) &set_collation_name, 0, 0, GET_STR, REQUIRED_ARG,
    0, 0, 0, 0, 0, 0},
-  {"set-variable", 'O',
-   "Change the value of a variable. Please note that this option is deprecated; you can set variables directly with --variable-name=value.",
-   0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"silent", 's',
    "Only print errors. One can use two -s to make maria_chk very silent.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
@@ -353,7 +350,7 @@ static struct my_option my_long_options[] =
    "properly closed'",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"unpack", 'u',
-   "Unpack file packed with mariapack.",
+   "Unpack file packed with maria_pack.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"verbose", 'v',
    "Print more information. This can be used with --description and --check. Use many -v for more verbosity!",
@@ -372,12 +369,14 @@ static struct my_option my_long_options[] =
     &check_param.use_buffers, &check_param.use_buffers, 0,
     GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT, 1024L*1024L,
     (long) ~0L, (long) MALLOC_OVERHEAD, (long) IO_SIZE, 0},
-  { "read_buffer_size", OPT_READ_BUFFER_SIZE, "",
+  { "read_buffer_size", OPT_READ_BUFFER_SIZE,
+    "Read buffer size for sequential reads during scanning",
     &check_param.read_buffer_length,
     &check_param.read_buffer_length, 0, GET_ULONG, REQUIRED_ARG,
     (long) READ_BUFFER_INIT, (long) MALLOC_OVERHEAD,
     (long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0},
-  { "write_buffer_size", OPT_WRITE_BUFFER_SIZE, "",
+  { "write_buffer_size", OPT_WRITE_BUFFER_SIZE,
+    "Write buffer size for sequential writes during repair of fixed size or dynamic size rows",
     &check_param.write_buffer_length,
     &check_param.write_buffer_length, 0, GET_ULONG, REQUIRED_ARG,
     (long) READ_BUFFER_INIT, (long) MALLOC_OVERHEAD,
@@ -388,7 +387,8 @@ static struct my_option my_long_options[] =
     &check_param.sort_buffer_length, 0, GET_ULONG, REQUIRED_ARG,
     (long) SORT_BUFFER_INIT, (long) (MIN_SORT_BUFFER + MALLOC_OVERHEAD),
     (long) ~0L, (long) MALLOC_OVERHEAD, (long) 1L, 0},
-  { "sort_key_blocks", OPT_SORT_KEY_BLOCKS, "",
+  { "sort_key_blocks", OPT_SORT_KEY_BLOCKS,
+    "Internal buffer for sorting keys; Don't touch :)",
     &check_param.sort_key_blocks,
     &check_param.sort_key_blocks, 0, GET_ULONG, REQUIRED_ARG,
     BUFFERS_WHEN_SORTING, 4L, 100L, 0L, 1L, 0},
@@ -411,7 +411,7 @@ static struct my_option my_long_options[] =
     (char**) &maria_stats_method_str, (char**) &maria_stats_method_str, 0,
     GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   { "zerofill", 'z',
-    "Fill empty space in data and index files with zeroes",
+    "Fill empty space in data and index files with zeroes,",
     0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   { "zerofill-keep-lsn", OPT_ZEROFILL_KEEP_LSN,
     "Like --zerofill but does not zero out LSN of data/index pages;"
@@ -447,10 +447,12 @@ static void usage(void)
   printf("\
   -H, --HELP          Display this help and exit.\n\
   -?, --help          Display this help and exit.\n\
-  -O, --set-variable var=option.\n\
-                      Change the value of a variable. Please note that\n\
-                      this option is deprecated; you can set variables\n\
-                      directly with '--variable-name=value'.\n\
+  --datadir=path      Path for control file (and logs if --logdir not used)\n\
+  --logdir=path       Path for log files\n\
+  --require-control-file  Abort if we can't find/read the maria_log_control\n\
+                          file\n\
+  -s, --silent	      Only print errors.  One can use two -s to make\n\
+		      maria_chk very silent.\n\
   -t, --tmpdir=path   Path for temporary files. Multiple paths can be\n\
                       specified, separated by ");
 #if defined( __WIN__) || defined(__NETWARE__)
@@ -458,12 +460,8 @@ static void usage(void)
 #else
    printf("colon (:)");
 #endif
-                      printf(", they will be used\n\
+   printf(", they will be used\n\
                       in a round-robin fashion.\n\
-  --require-control-file  Abort if we can't find/read the maria_log_control\n\
-                          file\n\
-  -s, --silent	      Only print errors.  One can use two -s to make\n\
-		      maria_chk very silent.\n\
   -v, --verbose       Print more information. This can be used with\n\
                       --description and --check. Use many -v for more verbosity.\n\
   -V, --version       Print version and exit.\n\
@@ -485,10 +483,11 @@ static void usage(void)
   -i, --information   Print statistics information about table that is checked.\n\
   -m, --medium-check  Faster than extend-check, but only finds 99.99% of\n\
 		      all errors.  Should be good enough for most cases.\n\
-  -U  --update-state  Mark tables as crashed if you find any errors.\n\
+  -U, --update-state  Mark tables as crashed if you find any errors.\n\
   -T, --read-only     Don't mark table as checked.\n");
 
-  puts("Recover (repair)/ options (When using '-r' or '-o'):\n\
+  puts("\
+Recover (repair)/ options (When using '--recover' or '--safe-recover'):\n\
   -B, --backup	      Make a backup of the .MAD file as 'filename-time.BAK'.\n\
   --correct-checksum  Correct checksum information for table.\n\
   -D, --data-file-length=#  Max length of data file (when recreating data\n\
@@ -531,7 +530,7 @@ static void usage(void)
 
   puts("Other actions:\n\
   -a, --analyze	      Analyze distribution of keys. Will make some joins in\n\
-		      MySQL faster.  You can check the calculated distribution\n\
+		      MariaDB faster.  You can check the calculated distribution\n\
 		      by using '--description --verbose table_name'.\n\
   --stats_method=name Specifies how index statistics collection code should\n\
                       treat NULLs. Possible values of name are \"nulls_unequal\"\n\
@@ -554,6 +553,13 @@ static void usage(void)
   --zerofill-keep-lsn Like --zerofill but does not zero out LSN of\n\
                       data/index pages.");
 
+  puts("Variables:\n\
+--page_buffer_size=#   Size of page buffer. Used by --safe-repair\n\
+--read_buffer_size=#   Read buffer size for sequential reads during scanning\n\
+--sort_buffer_size=#   Size of sort buffer. Used by --recover\n\
+--sort_key_blocks=#    Internal buffer for sorting keys; Don't touch :)\n\
+--write_buffer_size=#  Write buffer size for sequential writes during repair");
+
   print_defaults("my", load_default_groups);
   my_print_variables(my_long_options);
 }

storage/maria/trnman.c

@@ -145,6 +145,7 @@ static uchar *trn_get_hash_key(const uchar *trn, size_t *len,
 int trnman_init(TrID initial_trid)
 {
   DBUG_ENTER("trnman_init");
+  DBUG_PRINT("enter", ("initial_trid: %lu", (ulong) initial_trid));
 
   short_trid_to_active_trn= (TRN **)my_malloc(SHORT_TRID_MAX*sizeof(TRN*),
                                      MYF(MY_WME|MY_ZEROFILL));