mi_check.c 134 KB
Newer Older
unknown's avatar
unknown committed
1
/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
2

unknown's avatar
unknown committed
3 4 5 6
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.
7

unknown's avatar
unknown committed
8 9 10 11
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
12

unknown's avatar
unknown committed
13 14 15 16
   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

17
/* Describe, check and repair of MyISAM tables */
unknown's avatar
unknown committed
18

19
#include "ftdefs.h"
unknown's avatar
unknown committed
20 21
#include <m_ctype.h>
#include <stdarg.h>
22
#include <my_getopt.h>
23
#ifdef HAVE_SYS_VADVISE_H
unknown's avatar
unknown committed
24 25 26 27 28
#include <sys/vadvise.h>
#endif
#ifdef HAVE_SYS_MMAN_H
#include <sys/mman.h>
#endif
29
#include "rt_index.h"
unknown's avatar
unknown committed
30 31 32 33 34 35 36 37 38

#ifndef USE_RAID
#define my_raid_create(A,B,C,D,E,F,G) my_create(A,B,C,G)
#define my_raid_delete(A,B,C) my_delete(A,B)
#endif

	/* Functions defined in this file */

static int check_k_link(MI_CHECK *param, MI_INFO *info,uint nr);
39
static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
unknown's avatar
unknown committed
40 41 42 43
		     my_off_t page, uchar *buff, ha_rows *keys,
		     ha_checksum *key_checksum, uint level);
static uint isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
static ha_checksum calc_checksum(ha_rows count);
44 45
static int writekeys(MI_CHECK *param, MI_INFO *info,byte *buff,
		     my_off_t filepos);
unknown's avatar
unknown committed
46 47
static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
			  my_off_t pagepos, File new_file);
unknown's avatar
unknown committed
48 49 50 51
static int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key);
static int sort_get_next_record(MI_SORT_PARAM *sort_param);
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
52
static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);
unknown's avatar
unknown committed
53
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
unknown's avatar
unknown committed
54 55
static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
				uchar *key);
56
static int sort_insert_key(MI_SORT_PARAM  *sort_param,
unknown's avatar
unknown committed
57
                           reg1 SORT_KEY_BLOCKS *key_block,
unknown's avatar
unknown committed
58
			   uchar *key, my_off_t prev_block);
unknown's avatar
unknown committed
59
static int sort_delete_record(MI_SORT_PARAM *sort_param);
60
/*static int flush_pending_blocks(MI_CHECK *param);*/
unknown's avatar
unknown committed
61 62 63
static SORT_KEY_BLOCKS	*alloc_key_blocks(MI_CHECK *param, uint blocks,
					  uint buffer_length);
static ha_checksum mi_byte_checksum(const byte *buf, uint length);
unknown's avatar
unknown committed
64
static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
unknown's avatar
unknown committed
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79

void myisamchk_init(MI_CHECK *param)
{
  bzero((gptr) param,sizeof(*param));
  param->opt_follow_links=1;
  param->keys_in_use= ~(ulonglong) 0;
  param->search_after_block=HA_OFFSET_ERROR;
  param->auto_increment_value= 0;
  param->use_buffers=USE_BUFFER_INIT;
  param->read_buffer_length=READ_BUFFER_INIT;
  param->write_buffer_length=READ_BUFFER_INIT;
  param->sort_buffer_length=SORT_BUFFER_INIT;
  param->sort_key_blocks=BUFFERS_WHEN_SORTING;
  param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
  param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
80
  param->start_check_pos=0;
81
  param->max_record_length= LONGLONG_MAX;
82
  param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
83
  param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
unknown's avatar
unknown committed
84 85
}

86 87 88 89 90
	/* Check the status flags for the table */

int chk_status(MI_CHECK *param, register MI_INFO *info)
{
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
91

92 93 94 95 96 97
  if (mi_is_crashed_on_repair(info))
    mi_check_print_warning(param,
			   "Table is marked as crashed and last repair failed");
  else if (mi_is_crashed(info))
    mi_check_print_warning(param,
			   "Table is marked as crashed");
unknown's avatar
unknown committed
98
  if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
99
  {
unknown's avatar
unknown committed
100 101
    /* Don't count this as a real warning, as check can correct this ! */
    uint save=param->warning_printed;
102
    mi_check_print_warning(param,
103 104 105
			   share->state.open_count==1 ? 
			   "%d client is using or hasn't closed the table properly" : 
			   "%d clients are using or haven't closed the table properly",
106
			   share->state.open_count);
unknown's avatar
unknown committed
107 108 109
    /* If this will be fixed by the check, forget the warning */
    if (param->testflag & T_UPDATE_STATE)
      param->warning_printed=save;
110 111 112 113
  }
  return 0;
}

unknown's avatar
unknown committed
114 115 116 117 118
	/* Check delete links */

int chk_del(MI_CHECK *param, register MI_INFO *info, uint test_flag)
{
  reg2 ha_rows i;
119
  uint delete_link_length;
unknown's avatar
unknown committed
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
  my_off_t empty,next_link,old_link;
  char buff[22],buff2[22];
  DBUG_ENTER("chk_del");

  LINT_INIT(old_link);
  param->record_checksum=0;
  delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 :
		      info->s->rec_reflength+1);

  if (!(test_flag & T_SILENT))
    puts("- check record delete-chain");

  next_link=info->s->state.dellink;
  if (info->state->del == 0)
  {
    if (test_flag & T_VERBOSE)
    {
      puts("No recordlinks");
    }
  }
  else
  {
    if (test_flag & T_VERBOSE)
      printf("Recordlinks:    ");
    empty=0;
    for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--)
    {
unknown's avatar
unknown committed
147 148
      if (*killed_ptr(param))
        DBUG_RETURN(1);
unknown's avatar
unknown committed
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
      if (test_flag & T_VERBOSE)
	printf(" %9s",llstr(next_link,buff));
      if (next_link >= info->state->data_file_length)
	goto wrong;
      if (my_pread(info->dfile,(char*) buff,delete_link_length,
		   next_link,MYF(MY_NABP)))
      {
	if (test_flag & T_VERBOSE) puts("");
	mi_check_print_error(param,"Can't read delete-link at filepos: %s",
		    llstr(next_link,buff));
	DBUG_RETURN(1);
      }
      if (*buff != '\0')
      {
	if (test_flag & T_VERBOSE) puts("");
	mi_check_print_error(param,"Record at pos: %s is not remove-marked",
		    llstr(next_link,buff));
	goto wrong;
      }
      if (info->s->options & HA_OPTION_PACK_RECORD)
      {
	my_off_t prev_link=mi_sizekorr(buff+12);
	if (empty && prev_link != old_link)
	{
	  if (test_flag & T_VERBOSE) puts("");
	  mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
	  goto wrong;
	}
	old_link=next_link;
	next_link=mi_sizekorr(buff+4);
	empty+=mi_uint3korr(buff+1);
      }
      else
      {
	param->record_checksum+=(ha_checksum) next_link;
	next_link=_mi_rec_pos(info->s,(uchar*) buff+1);
	empty+=info->s->base.pack_reclength;
      }
    }
188 189
    if (test_flag & T_VERBOSE)
      puts("\n");
unknown's avatar
unknown committed
190 191 192
    if (empty != info->state->empty)
    {
      mi_check_print_warning(param,
193 194 195
			     "Found %s deleted space in delete link chain. Should be %s",
			     llstr(empty,buff2),
			     llstr(info->state->empty,buff));
unknown's avatar
unknown committed
196
    }
197 198 199 200 201
    if (next_link != HA_OFFSET_ERROR)
    {
      mi_check_print_error(param,
			   "Found more than the expected %s deleted rows in delete link chain",
			   llstr(info->state->del, buff));
unknown's avatar
unknown committed
202
      goto wrong;
203 204 205 206 207 208 209 210 211
    }
    if (i != 0)
    {
      mi_check_print_error(param,
			   "Found %s deleted rows in delete link chain. Should be %s",
			   llstr(info->state->del - i, buff2),
			   llstr(info->state->del, buff));
      goto wrong;
    }
unknown's avatar
unknown committed
212 213
  }
  DBUG_RETURN(0);
214

unknown's avatar
unknown committed
215
wrong:
unknown's avatar
unknown committed
216
  param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
217 218 219 220 221 222 223 224 225 226 227
  if (test_flag & T_VERBOSE) puts("");
  mi_check_print_error(param,"record delete-link-chain corrupted");
  DBUG_RETURN(1);
} /* chk_del */


	/* Check delete links in index file */

static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint nr)
{
  my_off_t next_link;
unknown's avatar
unknown committed
228
  uint block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
unknown's avatar
unknown committed
229 230 231 232 233 234 235 236 237 238 239
  ha_rows records;
  char llbuff[21],*buff;
  DBUG_ENTER("check_k_link");

  if (param->testflag & T_VERBOSE)
    printf("block_size %4d:",block_size);

  next_link=info->s->state.key_del[nr];
  records= (ha_rows) (info->state->key_file_length / block_size);
  while (next_link != HA_OFFSET_ERROR && records > 0)
  {
unknown's avatar
unknown committed
240 241
    if (*killed_ptr(param))
      DBUG_RETURN(1);
unknown's avatar
unknown committed
242 243 244 245 246
    if (param->testflag & T_VERBOSE)
      printf("%16s",llstr(next_link,llbuff));
    if (next_link > info->state->key_file_length ||
	next_link & (info->s->blocksize-1))
      DBUG_RETURN(1);
unknown's avatar
unknown committed
247
    if (!(buff=key_cache_read(info->s->key_cache,
248 249
                              info->s->kfile, next_link, DFLT_INIT_HITS,
                              (byte*) info->buff,
unknown's avatar
unknown committed
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
			      myisam_block_size, block_size, 1)))
      DBUG_RETURN(1);
    next_link=mi_sizekorr(buff);
    records--;
    param->key_file_blocks+=block_size;
  }
  if (param->testflag & T_VERBOSE)
  {
    if (next_link != HA_OFFSET_ERROR)
      printf("%16s\n",llstr(next_link,llbuff));
    else
      puts("");
  }
  DBUG_RETURN (next_link != HA_OFFSET_ERROR);
} /* check_k_link */


unknown's avatar
unknown committed
267
	/* Check sizes of files */
unknown's avatar
unknown committed
268 269 270 271 272 273 274 275 276 277

int chk_size(MI_CHECK *param, register MI_INFO *info)
{
  int error=0;
  register my_off_t skr,size;
  char buff[22],buff2[22];
  DBUG_ENTER("chk_size");

  if (!(param->testflag & T_SILENT)) puts("- check file-size");

unknown's avatar
unknown committed
278 279 280
  /* The following is needed if called externally (not from myisamchk) */
  flush_key_blocks(info->s->key_cache,
		   info->s->kfile, FLUSH_FORCE_WRITE);
unknown's avatar
unknown committed
281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315

  size=my_seek(info->s->kfile,0L,MY_SEEK_END,MYF(0));
  if ((skr=(my_off_t) info->state->key_file_length) != size)
  {
    if (skr > size)
    {
      error=1;
      mi_check_print_error(param,
			   "Size of indexfile is: %-8s        Should be: %s",
			   llstr(size,buff), llstr(skr,buff2));
    }
    else
      mi_check_print_warning(param,
			     "Size of indexfile is: %-8s      Should be: %s",
			     llstr(size,buff), llstr(skr,buff2));
  }
  if (!(param->testflag & T_VERY_SILENT) &&
      ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
      ulonglong2double(info->state->key_file_length) >
      ulonglong2double(info->s->base.margin_key_file_length)*0.9)
    mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
			   llstr(info->state->key_file_length,buff),
			   llstr(info->s->base.max_key_file_length-1,buff));

  size=my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
  skr=(my_off_t) info->state->data_file_length;
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
    skr+= MEMMAP_EXTRA_MARGIN;
#ifdef USE_RELOC
  if (info->data_file_type == STATIC_RECORD &&
      skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
    skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
#endif
  if (skr != size)
  {
316
    info->state->data_file_length=size;	/* Skip other errors */
unknown's avatar
unknown committed
317 318 319
    if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
    {
      error=1;
320
      mi_check_print_error(param,"Size of datafile is: %-9s         Should be: %s",
unknown's avatar
unknown committed
321
		    llstr(size,buff), llstr(skr,buff2));
unknown's avatar
unknown committed
322
      param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
323 324 325 326
    }
    else
    {
      mi_check_print_warning(param,
327
			     "Size of datafile is: %-9s       Should be: %s",
unknown's avatar
unknown committed
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345
			     llstr(size,buff), llstr(skr,buff2));
    }
  }
  if (!(param->testflag & T_VERY_SILENT) &&
      !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
      ulonglong2double(info->state->data_file_length) >
      (ulonglong2double(info->s->base.max_data_file_length)*0.9))
    mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
			   llstr(info->state->data_file_length,buff),
			   llstr(info->s->base.max_data_file_length-1,buff2));
  DBUG_RETURN(error);
} /* chk_size */


	/* Check keys */

int chk_key(MI_CHECK *param, register MI_INFO *info)
{
346
  uint key,found_keys=0,full_text_keys=0,result=0;
unknown's avatar
unknown committed
347 348 349 350 351 352 353 354 355
  ha_rows keys;
  ha_checksum old_record_checksum,init_checksum;
  my_off_t all_keydata,all_totaldata,key_totlength,length;
  ulong   *rec_per_key_part;
  MYISAM_SHARE *share=info->s;
  MI_KEYDEF *keyinfo;
  char buff[22],buff2[22];
  DBUG_ENTER("chk_key");

356 357 358 359 360 361 362 363 364 365 366 367
  if (!(param->testflag & T_SILENT))
    puts("- check key delete-chain");

  param->key_file_blocks=info->s->base.keystart;
  for (key=0 ; key < info->s->state.header.max_block_size ; key++)
    if (check_k_link(param,info,key))
    {
      if (param->testflag & T_VERBOSE) puts("");
      mi_check_print_error(param,"key delete-link-chain corrupted");
      DBUG_RETURN(-1);
    }

unknown's avatar
unknown committed
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382
  if (!(param->testflag & T_SILENT)) puts("- check index reference");

  all_keydata=all_totaldata=key_totlength=0;
  old_record_checksum=0;
  init_checksum=param->record_checksum;
  if (!(share->options &
	(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
    old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
      share->base.pack_reclength;
  rec_per_key_part= param->rec_per_key_part;
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
       rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
  {
    param->key_crc[key]=0;
    if (!(((ulonglong) 1 << key) & share->state.key_map))
383 384 385
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
386 387
	     (char*) (share->state.rec_per_key_part +
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
388
	     keyinfo->keysegs*sizeof(*rec_per_key_part));
unknown's avatar
unknown committed
389
      continue;
390
    }
unknown's avatar
unknown committed
391 392 393
    found_keys++;

    param->record_checksum=init_checksum;
394
    
unknown's avatar
unknown committed
395
    bzero((char*) &param->unique_count,sizeof(param->unique_count));
396 397
    bzero((char*) &param->notnull_count,sizeof(param->notnull_count));

unknown's avatar
unknown committed
398 399
    if ((!(param->testflag & T_SILENT)))
      printf ("- check data record references index: %d\n",key+1);
unknown's avatar
unknown committed
400 401
    if (keyinfo->flag & HA_FULLTEXT)
      full_text_keys++;
unknown's avatar
unknown committed
402
    if (share->state.key_root[key] == HA_OFFSET_ERROR &&
403
	(info->state->records == 0 || keyinfo->flag & HA_FULLTEXT))
unknown's avatar
unknown committed
404
      continue;
405 406
    if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
                           DFLT_INIT_HITS,info->buff,0))
unknown's avatar
unknown committed
407 408 409
    {
      mi_check_print_error(param,"Can't read indexpage from filepos: %s",
		  llstr(share->state.key_root[key],buff));
410 411 412 413
      if (!(param->testflag & T_INFO))
	DBUG_RETURN(-1);
      result= -1;
      continue;
unknown's avatar
unknown committed
414 415 416 417 418 419 420 421 422
    }
    param->key_file_blocks+=keyinfo->block_length;
    keys=0;
    param->keydata=param->totaldata=0;
    param->key_blocks=0;
    param->max_level=0;
    if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
		  &keys, param->key_crc+key,1))
      DBUG_RETURN(-1);
423
    if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)))
unknown's avatar
unknown committed
424 425 426 427 428
    {
      if (keys != info->state->records)
      {
	mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
		    llstr(info->state->records,buff2));
429
	if (!(param->testflag & T_INFO))
unknown's avatar
unknown committed
430
	DBUG_RETURN(-1);
431 432
	result= -1;
	continue;
unknown's avatar
unknown committed
433 434 435 436 437 438 439 440 441 442 443 444 445
      }
      if (found_keys - full_text_keys == 1 &&
	  ((share->options &
	    (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
	   (param->testflag & T_DONT_CHECK_CHECKSUM)))
	old_record_checksum=param->record_checksum;
      else if (old_record_checksum != param->record_checksum)
      {
	if (key)
	  mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
		      key+1);
	else
	  mi_check_print_error(param,"Key 1 doesn't point at all records");
446 447 448 449
	if (!(param->testflag & T_INFO))
	  DBUG_RETURN(-1);
	result= -1;
	continue;
unknown's avatar
unknown committed
450 451 452 453
      }
    }
    if ((uint) share->base.auto_key -1 == key)
    {
unknown's avatar
unknown committed
454
      /* Check that auto_increment key is bigger than max key value */
unknown's avatar
unknown committed
455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
      ulonglong save_auto_value=info->s->state.auto_increment;
      info->s->state.auto_increment=0;
      info->lastinx=key;
      _mi_read_key_record(info, 0L, info->rec_buff);
      update_auto_increment(info, info->rec_buff);
      if (info->s->state.auto_increment > save_auto_value)
      {
	mi_check_print_warning(param,
			       "Auto-increment value: %s is smaller than max used value: %s",
			       llstr(save_auto_value,buff2),
			       llstr(info->s->state.auto_increment, buff));
      }
      if (param->testflag & T_AUTO_INC)
      {
	set_if_bigger(info->s->state.auto_increment,
		      param->auto_increment_value);
      }
      else
	info->s->state.auto_increment=save_auto_value;
unknown's avatar
unknown committed
474 475

      /* Check that there isn't a row with auto_increment = 0 in the table */
unknown's avatar
unknown committed
476
      mi_extra(info,HA_EXTRA_KEYREAD,0);
unknown's avatar
unknown committed
477
      bzero(info->lastkey,keyinfo->seg->length);
unknown's avatar
unknown committed
478
      if (!mi_rkey(info, info->rec_buff, key, (const byte*) info->lastkey,
unknown's avatar
unknown committed
479 480 481 482 483 484 485 486
		   keyinfo->seg->length, HA_READ_KEY_EXACT))
      {
	/* Don't count this as a real warning, as myisamchk can't correct it */
	uint save=param->warning_printed;
	mi_check_print_warning(param,
			       "Found row where the auto_increment column has the value 0");
	param->warning_printed=save;
      }
unknown's avatar
unknown committed
487
      mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
unknown's avatar
unknown committed
488 489 490 491 492 493 494 495 496 497 498 499 500 501
    }

    length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
    if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
      printf("Key: %2d:  Keyblocks used: %3d%%  Packed: %4d%%  Max levels: %2d\n",
	     key+1,
	     (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
	     (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
		    my_off_t2double(length)),
	     param->max_level);
    all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;

    if (param->testflag & T_STATISTICS)
      update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
502 503 504
                       param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
                       param->notnull_count: NULL, 
                       (ulonglong)info->state->records);
unknown's avatar
unknown committed
505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
  }
  if (param->testflag & T_INFO)
  {
    if (all_totaldata != 0L && found_keys > 0)
      printf("Total:    Keyblocks used: %3d%%  Packed: %4d%%\n\n",
	     (int) (my_off_t2double(all_keydata)*100.0/
		    my_off_t2double(all_totaldata)),
	     (int) ((my_off_t2double(key_totlength) -
		     my_off_t2double(all_keydata))*100.0/
		     my_off_t2double(key_totlength)));
    else if (all_totaldata != 0L && share->state.key_map)
      puts("");
  }
  if (param->key_file_blocks != info->state->key_file_length &&
      param->keys_in_use != ~(ulonglong) 0)
    mi_check_print_warning(param, "Some data are unreferenced in keyfile");
  if (found_keys != full_text_keys)
    param->record_checksum=old_record_checksum-init_checksum;	/* Remove delete links */
  else
    param->record_checksum=0;
unknown's avatar
unknown committed
525
  DBUG_RETURN(result);
unknown's avatar
unknown committed
526 527
} /* chk_key */

unknown's avatar
unknown committed
528

529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
                     my_off_t page, uchar *buff, ha_rows *keys,
                     ha_checksum *key_checksum, uint level)
{
  char llbuff[22],llbuff2[22];
  if (page > info->state->key_file_length || (page & (info->s->blocksize -1)))
  {
    my_off_t max_length=my_seek(info->s->kfile,0L,MY_SEEK_END,MYF(0));
    mi_check_print_error(param,"Wrong pagepointer: %s at page: %s",
                llstr(page,llbuff),llstr(page,llbuff2));

    if (page+info->s->blocksize > max_length)
      goto err;
    info->state->key_file_length=(max_length &
                                  ~ (my_off_t) (info->s->blocksize-1));
  }
  if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
  {
    mi_check_print_error(param,"Can't read key from filepos: %s",
        llstr(page,llbuff));
    goto err;
  }
  param->key_file_blocks+=keyinfo->block_length;
  if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
    goto err;

  return 0;
err:
  return 1;
}
unknown's avatar
unknown committed
559

560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605

/*
  "Ignore NULLs" statistics collection method: process first index tuple.

  SYNOPSIS
    mi_collect_stats_nonulls_first()
      keyseg   IN     Array of key part descriptions
      notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
                                           tuples that don't contain NULLs)
      key      IN     Key values tuple

  DESCRIPTION
    Process the first index tuple - find out which prefix tuples don't
    contain NULLs, and update the array of notnull counters accordingly.
*/

static
void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, ulonglong *notnull,
                                    uchar *key)
{
  uint first_null, kp;
  first_null= ha_find_null(keyseg, key) - keyseg;
  /*
    All prefix tuples that don't include keypart_{first_null} are not-null
    tuples (and all others aren't), increment counters for them.
  */
  for (kp= 0; kp < first_null; kp++)
    notnull[kp]++;
}


/*
  "Ignore NULLs" statistics collection method: process next index tuple.

  SYNOPSIS
    mi_collect_stats_nonulls_next()
      keyseg   IN     Array of key part descriptions
      notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
                                           tuples that don't contain NULLs)
      prev_key IN     Previous key values tuple
      last_key IN     Next key values tuple

  DESCRIPTION
    Process the next index tuple:
    1. Find out which prefix tuples of last_key don't contain NULLs, and
       update the array of notnull counters accordingly.
606 607 608 609 610
    2. Find the first keypart number where the prev_key and last_key tuples
       are different(A), or last_key has NULL value(B), and return it, so the 
       caller can count number of unique tuples for each key prefix. We don't 
       need (B) to be counted, and that is compensated back in 
       update_key_parts().
611 612 613 614 615 616 617 618 619 620 621

  RETURN
    1 + number of first keypart where values differ or last_key tuple has NULL
*/

static
int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, ulonglong *notnull,
                                  uchar *prev_key, uchar *last_key)
{
  uint diffs[2];
  uint first_null_seg, kp;
622
  HA_KEYSEG *seg;
623

624 625 626 627 628 629 630 631
  /* 
     Find the first keypart where values are different or either of them is
     NULL. We get results in diffs array:
     diffs[0]= 1 + number of first different keypart
     diffs[1]=offset: (last_key + diffs[1]) points to first value in
                      last_key that is NULL or different from corresponding
                      value in prev_key.
  */
632
  ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY, 
633 634
             SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
  seg= keyseg + diffs[0] - 1;
635

636 637 638 639 640 641 642 643 644 645 646 647 648 649
  /* Find first NULL in last_key */
  first_null_seg= ha_find_null(seg, last_key + diffs[1]) - keyseg;
  for (kp= 0; kp < first_null_seg; kp++)
    notnull[kp]++;

  /* 
    Return 1+ number of first key part where values differ. Don't care if
    these were NULLs and not .... We compensate for that in
    update_key_parts.
  */
  return diffs[0];
}


unknown's avatar
unknown committed
650 651 652 653 654 655 656
	/* Check if index is ok */

static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
		     my_off_t page, uchar *buff, ha_rows *keys,
		     ha_checksum *key_checksum, uint level)
{
  int flag;
657
  uint used_length,comp_flag,nod_flag,key_length=0;
unknown's avatar
unknown committed
658 659
  uchar key[MI_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos;
  my_off_t next_page,record;
unknown's avatar
unknown committed
660
  char llbuff[22];
661
  uint diff_pos[2];
unknown's avatar
unknown committed
662 663 664
  DBUG_ENTER("chk_index");
  DBUG_DUMP("buff",(byte*) buff,mi_getint(buff));

665 666 667 668
  /* TODO: implement appropriate check for RTree keys */
  if (keyinfo->flag & HA_SPATIAL)
    DBUG_RETURN(0);

unknown's avatar
unknown committed
669 670
  if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
  {
671
    mi_check_print_error(param,"Not enough memory for keyblock");
unknown's avatar
unknown committed
672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690
    DBUG_RETURN(-1);
  }

  if (keyinfo->flag & HA_NOSAME)
    comp_flag=SEARCH_FIND | SEARCH_UPDATE;	/* Not real duplicates */
  else
    comp_flag=SEARCH_SAME;			/* Keys in positionorder */
  nod_flag=mi_test_if_nod(buff);
  used_length=mi_getint(buff);
  keypos=buff+2+nod_flag;
  endpos=buff+used_length;

  param->keydata+=used_length; param->totaldata+=keyinfo->block_length;	/* INFO */
  param->key_blocks++;
  if (level > param->max_level)
    param->max_level=level;

  if (used_length > keyinfo->block_length)
  {
unknown's avatar
unknown committed
691 692
    mi_check_print_error(param,"Wrong pageinfo at page: %s",
			 llstr(page,llbuff));
unknown's avatar
unknown committed
693 694 695 696
    goto err;
  }
  for ( ;; )
  {
unknown's avatar
unknown committed
697 698
    if (*killed_ptr(param))
      goto err;
699 700
    memcpy((char*) info->lastkey,(char*) key,key_length);
    info->lastkey_length=key_length;
unknown's avatar
unknown committed
701 702 703
    if (nod_flag)
    {
      next_page=_mi_kpos(nod_flag,keypos);
704 705
      if (chk_index_down(param,info,keyinfo,next_page,
                         temp_buff,keys,key_checksum,level+1))
unknown's avatar
unknown committed
706 707 708 709 710 711 712 713 714 715 716 717
	goto err;
    }
    old_keypos=keypos;
    if (keypos >= endpos ||
	(key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
      break;
    if (keypos > endpos)
    {
      mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
      goto err;
    }
    if ((*keys)++ &&
unknown's avatar
unknown committed
718
	(flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
719
			 comp_flag, diff_pos)) >=0)
unknown's avatar
unknown committed
720 721 722 723 724 725 726 727 728 729 730 731 732
    {
      DBUG_DUMP("old",(byte*) info->lastkey, info->lastkey_length);
      DBUG_DUMP("new",(byte*) key, key_length);
      DBUG_DUMP("new_in_page",(char*) old_keypos,(uint) (keypos-old_keypos));

      if (comp_flag & SEARCH_FIND && flag == 0)
	mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
      else
	mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
      goto err;
    }
    if (param->testflag & T_STATISTICS)
    {
733
      if (*keys != 1L)				/* not first_key */
unknown's avatar
unknown committed
734
      {
735 736 737
        if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
          ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
                     SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
738
                     diff_pos);
739 740
        else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
        {
741
          diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg, 
742 743 744
                                                  param->notnull_count,
                                                  info->lastkey, key);
        }
745
	param->unique_count[diff_pos[0]-1]++;
unknown's avatar
unknown committed
746
      }
747 748 749 750 751 752
      else
      {  
        if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
          mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
                                         key);
      }
unknown's avatar
unknown committed
753 754 755 756
    }
    (*key_checksum)+= mi_byte_checksum((byte*) key,
				       key_length- info->s->rec_reflength);
    record= _mi_dpos(info,0,key+key_length);
757 758 759 760 761 762 763 764 765 766 767 768
    if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */
    {
      uint off;
      int  subkeys;
      get_key_full_length_rdonly(off, key);
      subkeys=ft_sintXkorr(key+off);
      if (subkeys < 0)
      {
        ha_rows tmp_keys=0;
        if (chk_index_down(param,info,&info->s->ft2_keyinfo,record,
                           temp_buff,&tmp_keys,key_checksum,1))
          goto err;
769 770
        if (tmp_keys + subkeys)
        {
unknown's avatar
unknown committed
771 772 773 774 775
          mi_check_print_error(param,
                               "Number of words in the 2nd level tree "
                               "does not match the number in the header. "
                               "Parent word in on the page %s, offset %u",
                               llstr(page,llbuff), (uint) (old_keypos-buff));
776 777
          goto err;
        }
778 779 780 781 782
        (*keys)+=tmp_keys-1;
        continue;
      }
      /* fall through */
    }
unknown's avatar
unknown committed
783 784 785
    if (record >= info->state->data_file_length)
    {
#ifndef DBUG_OFF
unknown's avatar
unknown committed
786
      char llbuff2[22], llbuff3[22];
unknown's avatar
unknown committed
787 788 789 790 791
#endif
      mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
      DBUG_PRINT("test",("page: %s  record: %s  filelength: %s",
			 llstr(page,llbuff),llstr(record,llbuff2),
			 llstr(info->state->data_file_length,llbuff3)));
792
      DBUG_DUMP("key",(byte*) key,key_length);
unknown's avatar
unknown committed
793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
      DBUG_DUMP("new_in_page",(char*) old_keypos,(uint) (keypos-old_keypos));
      goto err;
    }
    param->record_checksum+=(ha_checksum) record;
  }
  if (keypos != endpos)
  {
    mi_check_print_error(param,"Keyblock size at page %s is not correct.  Block length: %d  key length: %d",
                llstr(page,llbuff), used_length, (keypos - buff));
    goto err;
  }
  my_afree((byte*) temp_buff);
  DBUG_RETURN(0);
 err:
  my_afree((byte*) temp_buff);
  DBUG_RETURN(1);
} /* chk_index */


	/* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */

static ha_checksum calc_checksum(ha_rows count)
{
  ulonglong sum,a,b;
  DBUG_ENTER("calc_checksum");

  sum=0;
  a=count; b=count+1;
  if (a & 1)
    b>>=1;
  else
    a>>=1;
  while (b)
  {
    if (b & 1)
      sum+=a;
    a<<=1; b>>=1;
  }
  DBUG_PRINT("exit",("sum: %lx",(ulong) sum));
  DBUG_RETURN((ha_checksum) sum);
} /* calc_checksum */


	/* Calc length of key in normal isam */

static uint isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
{
  uint length;
unknown's avatar
unknown committed
841
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
  DBUG_ENTER("isam_key_length");

  length= info->s->rec_reflength;
  for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
    length+= keyseg->length;

  DBUG_PRINT("exit",("length: %d",length));
  DBUG_RETURN(length);
} /* key_length */


	/* Check that record-link is ok */

int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
{
  int	error,got_error,flag;
  uint	key,left_length,b_type,field;
  ha_rows records,del_blocks;
  my_off_t used,empty,pos,splits,start_recpos,
	   del_length,link_used,start_block;
  byte	*record,*to;
  char llbuff[22],llbuff2[22],llbuff3[22];
  ha_checksum intern_record_checksum;
  ha_checksum key_checksum[MI_MAX_POSSIBLE_KEY];
  my_bool static_row_size;
  MI_KEYDEF *keyinfo;
  MI_BLOCK_INFO block_info;
  DBUG_ENTER("chk_data_link");

  if (!(param->testflag & T_SILENT))
  {
    if (extend)
      puts("- check records and index references");
    else
      puts("- check record links");
  }

unknown's avatar
unknown committed
879
  if (!(record= (byte*) my_malloc(info->s->base.pack_reclength,MYF(0))))
unknown's avatar
unknown committed
880
  {
881
    mi_check_print_error(param,"Not enough memory for record");
unknown's avatar
unknown committed
882 883 884 885 886 887 888
    DBUG_RETURN(-1);
  }
  records=del_blocks=0;
  used=link_used=splits=del_length=0;
  intern_record_checksum=param->glob_crc=0;
  LINT_INIT(left_length);  LINT_INIT(start_recpos);  LINT_INIT(to);
  got_error=error=0;
889
  empty=info->s->pack.header_length;
unknown's avatar
unknown committed
890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905

  /* Check how to calculate checksum of rows */
  static_row_size=1;
  if (info->s->data_file_type == COMPRESSED_RECORD)
  {
    for (field=0 ; field < info->s->base.fields ; field++)
    {
      if (info->s->rec[field].base_type == FIELD_BLOB ||
	  info->s->rec[field].base_type == FIELD_VARCHAR)
      {
	static_row_size=0;
	break;
      }
    }
  }

906
  pos=my_b_tell(&param->read_cache);
unknown's avatar
unknown committed
907 908 909
  bzero((char*) key_checksum, info->s->base.keys * sizeof(key_checksum[0]));
  while (pos < info->state->data_file_length)
  {
unknown's avatar
unknown committed
910 911
    if (*killed_ptr(param))
      goto err2;
unknown's avatar
unknown committed
912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
    switch (info->s->data_file_type) {
    case STATIC_RECORD:
      if (my_b_read(&param->read_cache,(byte*) record,
		    info->s->base.pack_reclength))
	goto err;
      start_recpos=pos;
      pos+=info->s->base.pack_reclength;
      splits++;
      if (*record == '\0')
      {
	del_blocks++;
	del_length+=info->s->base.pack_reclength;
	continue;					/* Record removed */
      }
      param->glob_crc+= mi_static_checksum(info,record);
      used+=info->s->base.pack_reclength;
      break;
    case DYNAMIC_RECORD:
      flag=block_info.second_read=0;
      block_info.next_filepos=pos;
      do
      {
	if (_mi_read_cache(&param->read_cache,(byte*) block_info.header,
935 936 937
			   (start_block=block_info.next_filepos),
			   sizeof(block_info.header),
			   (flag ? 0 : READING_NEXT) | READING_HEADER))
unknown's avatar
unknown committed
938 939 940
	  goto err;
	if (start_block & (MI_DYN_ALIGN_SIZE-1))
	{
941 942
	  mi_check_print_error(param,"Wrong aligned block at %s",
			       llstr(start_block,llbuff));
unknown's avatar
unknown committed
943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964
	  goto err2;
	}
	b_type=_mi_get_block_info(&block_info,-1,start_block);
	if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
		      BLOCK_FATAL_ERROR))
	{
	  if (b_type & BLOCK_SYNC_ERROR)
	  {
	    if (flag)
	    {
	      mi_check_print_error(param,"Unexpected byte: %d at link: %s",
			  (int) block_info.header[0],
			  llstr(start_block,llbuff));
	      goto err2;
	    }
	    pos=block_info.filepos+block_info.block_len;
	    goto next;
	  }
	  if (b_type & BLOCK_DELETED)
	  {
	    if (block_info.block_len < info->s->base.min_block_length)
	    {
965 966 967
	      mi_check_print_error(param,
				   "Deleted block with impossible length %lu at %s",
				   block_info.block_len,llstr(pos,llbuff));
unknown's avatar
unknown committed
968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
	      goto err2;
	    }
	    if ((block_info.next_filepos != HA_OFFSET_ERROR &&
		 block_info.next_filepos >= info->state->data_file_length) ||
		(block_info.prev_filepos != HA_OFFSET_ERROR &&
		 block_info.prev_filepos >= info->state->data_file_length))
	    {
	      mi_check_print_error(param,"Delete link points outside datafile at %s",
			  llstr(pos,llbuff));
	      goto err2;
	    }
	    del_blocks++;
	    del_length+=block_info.block_len;
	    pos=block_info.filepos+block_info.block_len;
	    splits++;
	    goto next;
	  }
	  mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
986 987 988
			       block_info.header[0],block_info.header[1],
			       block_info.header[2],
			       llstr(start_block,llbuff));
unknown's avatar
unknown committed
989 990 991 992 993
	  goto err2;
	}
	if (info->state->data_file_length < block_info.filepos+
	    block_info.block_len)
	{
994 995 996
	  mi_check_print_error(param,
			       "Recordlink that points outside datafile at %s",
			       llstr(pos,llbuff));
unknown's avatar
unknown committed
997 998 999 1000 1001 1002 1003 1004 1005 1006
	  got_error=1;
	  break;
	}
	splits++;
	if (!flag++)				/* First block */
	{
	  start_recpos=pos;
	  pos=block_info.filepos+block_info.block_len;
	  if (block_info.rec_len > (uint) info->s->base.max_pack_length)
	  {
1007 1008 1009
	    mi_check_print_error(param,"Found too long record (%lu) at %s",
				 (ulong) block_info.rec_len,
				 llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1010 1011 1012 1013 1014
	    got_error=1;
	    break;
	  }
	  if (info->s->base.blobs)
	  {
1015
	    if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1016
					&info->rec_buff)))
unknown's avatar
unknown committed
1017
	    {
1018 1019 1020 1021
	      mi_check_print_error(param,
				   "Not enough memory (%lu) for blob at %s",
				   (ulong) block_info.rec_len,
				   llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
	      got_error=1;
	      break;
	    }
	  }
	  else
	    to= info->rec_buff;
	  left_length=block_info.rec_len;
	}
	if (left_length < block_info.data_len)
	{
1032 1033 1034 1035 1036
	  mi_check_print_error(param,"Found too long record (%lu) at %s",
			       (ulong) block_info.data_len,
			       llstr(start_recpos,llbuff));
	  got_error=1;
	  break;
unknown's avatar
unknown committed
1037 1038
	}
	if (_mi_read_cache(&param->read_cache,(byte*) to,block_info.filepos,
1039 1040
			   (uint) block_info.data_len,
			   flag == 1 ? READING_NEXT : 0))
unknown's avatar
unknown committed
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
	  goto err;
	to+=block_info.data_len;
	link_used+= block_info.filepos-start_block;
	used+= block_info.filepos - start_block + block_info.data_len;
	empty+=block_info.block_len-block_info.data_len;
	left_length-=block_info.data_len;
	if (left_length)
	{
	  if (b_type & BLOCK_LAST)
	  {
1051 1052 1053 1054 1055
	    mi_check_print_error(param,
				 "Wrong record length %s of %s at %s",
				 llstr(block_info.rec_len-left_length,llbuff),
				 llstr(block_info.rec_len, llbuff2),
				 llstr(start_recpos,llbuff3));
unknown's avatar
unknown committed
1056 1057 1058 1059 1060
	    got_error=1;
	    break;
	  }
	  if (info->state->data_file_length < block_info.next_filepos)
	  {
1061 1062 1063
	    mi_check_print_error(param,
				 "Found next-recordlink that points outside datafile at %s",
				 llstr(block_info.filepos,llbuff));
unknown's avatar
unknown committed
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
	    got_error=1;
	    break;
	  }
	}
      } while (left_length);
      if (! got_error)
      {
	if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
	    MY_FILE_ERROR)
	{
1074 1075
	  mi_check_print_error(param,"Found wrong record at %s",
			       llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1076 1077 1078 1079 1080 1081 1082
	  got_error=1;
	}
	else
	{
	  info->checksum=mi_checksum(info,record);
	  if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
	  {
1083 1084
	    if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
                              test(info->s->calc_checksum)))
unknown's avatar
unknown committed
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099
	    {
	      mi_check_print_error(param,"Found wrong packed record at %s",
			  llstr(start_recpos,llbuff));
	      got_error=1;
	    }
	  }
	  if (!got_error)
	    param->glob_crc+= info->checksum;
	}
      }
      else if (!flag)
	pos=block_info.filepos+block_info.block_len;
      break;
    case COMPRESSED_RECORD:
      if (_mi_read_cache(&param->read_cache,(byte*) block_info.header, pos,
1100
			 info->s->pack.ref_length, READING_NEXT))
unknown's avatar
unknown committed
1101 1102 1103
	goto err;
      start_recpos=pos;
      splits++;
1104
      VOID(_mi_pack_get_block_info(info,&block_info, -1, start_recpos));
unknown's avatar
unknown committed
1105 1106 1107 1108
      pos=block_info.filepos+block_info.rec_len;
      if (block_info.rec_len < (uint) info->s->min_pack_length ||
	  block_info.rec_len > (uint) info->s->max_pack_length)
      {
unknown's avatar
unknown committed
1109 1110 1111
	mi_check_print_error(param,
			     "Found block with wrong recordlength: %d at %s",
			     block_info.rec_len, llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1112 1113 1114 1115
	got_error=1;
	break;
      }
      if (_mi_read_cache(&param->read_cache,(byte*) info->rec_buff,
1116
			block_info.filepos, block_info.rec_len, READING_NEXT))
unknown's avatar
unknown committed
1117 1118 1119
	goto err;
      if (_mi_pack_rec_unpack(info,record,info->rec_buff,block_info.rec_len))
      {
unknown's avatar
unknown committed
1120 1121
	mi_check_print_error(param,"Found wrong record at %s",
			     llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
	got_error=1;
      }
      if (static_row_size)
	param->glob_crc+= mi_static_checksum(info,record);
      else
	param->glob_crc+= mi_checksum(info,record);
      link_used+= (block_info.filepos - start_recpos);
      used+= (pos-start_recpos);
    } /* switch */
    if (! got_error)
    {
      intern_record_checksum+=(ha_checksum) start_recpos;
      records++;
      if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
      {
	printf("%s\r", llstr(records,llbuff)); VOID(fflush(stdout));
      }

      /* Check if keys match the record */

      for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
	   key++,keyinfo++)
      {
	if ((((ulonglong) 1 << key) & info->s->state.key_map))
	{
1147
          if(!(keyinfo->flag & HA_FULLTEXT))
unknown's avatar
unknown committed
1148 1149 1150 1151 1152 1153 1154 1155
	  {
	    uint key_length=_mi_make_key(info,key,info->lastkey,record,
					 start_recpos);
	    if (extend)
	    {
	      /* We don't need to lock the key tree here as we don't allow
		 concurrent threads when running myisamchk
	      */
1156 1157
              int search_result= (keyinfo->flag & HA_SPATIAL) ?
                rtree_find_first(info, key, info->lastkey, key_length,
1158
                                 MBR_EQUAL | MBR_DATA) : 
1159 1160 1161 1162
                _mi_search(info,keyinfo,info->lastkey,key_length,
                           SEARCH_SAME, info->s->state.key_root[key]);
              if (search_result)
              {
1163 1164
                mi_check_print_error(param,"Record at: %10s  "
                                     "Can't find key for index: %2d",
1165 1166 1167 1168
                                     llstr(start_recpos,llbuff),key+1);
                if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
                  goto err2;
              }
unknown's avatar
unknown committed
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214
	    }
	    else
	      key_checksum[key]+=mi_byte_checksum((byte*) info->lastkey,
						  key_length);
	  }
	}
      }
    }
    else
    {
      got_error=0;
      if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
	goto err2;
    }
  next:;				/* Next record */
  }
  if (param->testflag & T_WRITE_LOOP)
  {
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
  }
  if (records != info->state->records)
  {
    mi_check_print_error(param,"Record-count is not ok; is %-10s   Should be: %s",
		llstr(records,llbuff), llstr(info->state->records,llbuff2));
    error=1;
  }
  else if (param->record_checksum &&
	   param->record_checksum != intern_record_checksum)
  {
    mi_check_print_error(param,
			 "Keypointers and record positions doesn't match");
    error=1;
  }
  else if (param->glob_crc != info->s->state.checksum &&
	   (info->s->options &
	    (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)))
  {
    mi_check_print_warning(param,
			   "Record checksum is not the same as checksum stored in the index file\n");
    error=1;
  }
  else if (!extend)
  {
    for (key=0 ; key < info->s->base.keys;  key++)
    {
      if (key_checksum[key] != param->key_crc[key] &&
1215
          !(info->s->keyinfo[key].flag & (HA_FULLTEXT | HA_SPATIAL)))
unknown's avatar
unknown committed
1216 1217 1218 1219 1220 1221 1222 1223
      {
	mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
		    key+1);
	error=1;
      }
    }
  }

1224 1225 1226 1227 1228 1229 1230
  if (del_length != info->state->empty)
  {
    mi_check_print_warning(param,
			   "Found %s deleted space.   Should be %s",
			   llstr(del_length,llbuff2),
			   llstr(info->state->empty,llbuff));
  }
unknown's avatar
unknown committed
1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
  if (used+empty+del_length != info->state->data_file_length)
  {
    mi_check_print_warning(param,
			   "Found %s record-data and %s unused data and %s deleted-data",
			   llstr(used,llbuff),llstr(empty,llbuff2),
			   llstr(del_length,llbuff3));
    mi_check_print_warning(param,
			   "Total %s, Should be: %s",
			   llstr((used+empty+del_length),llbuff),
			   llstr(info->state->data_file_length,llbuff2));
  }
  if (del_blocks != info->state->del)
  {
    mi_check_print_warning(param,
			   "Found %10s deleted blocks       Should be: %s",
			   llstr(del_blocks,llbuff),
			   llstr(info->state->del,llbuff2));
  }
  if (splits != info->s->state.split)
  {
    mi_check_print_warning(param,
			   "Found %10s parts                Should be: %s parts",
			   llstr(splits,llbuff),
			   llstr(info->s->state.split,llbuff2));
  }
  if (param->testflag & T_INFO)
  {
    if (param->warning_printed || param->error_printed)
      puts("");
    if (used != 0 && ! param->error_printed)
    {
      printf("Records:%18s    M.recordlength:%9lu   Packed:%14.0f%%\n",
unknown's avatar
unknown committed
1263
	     llstr(records,llbuff), (long)((used-link_used)/records),
unknown's avatar
unknown committed
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
	     (info->s->base.blobs ? 0.0 :
	      (ulonglong2double((ulonglong) info->s->base.reclength*records)-
	       my_off_t2double(used))/
	      ulonglong2double((ulonglong) info->s->base.reclength*records)*100.0));
      printf("Recordspace used:%9.0f%%   Empty space:%12d%%  Blocks/Record: %6.2f\n",
	     (ulonglong2double(used-link_used)/ulonglong2double(used-link_used+empty)*100.0),
	     (!records ? 100 : (int) (ulonglong2double(del_length+empty)/
				      my_off_t2double(used)*100.0)),
	     ulonglong2double(splits - del_blocks) / records);
    }
    printf("Record blocks:%12s    Delete blocks:%10s\n",
	   llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
    printf("Record data:  %12s    Deleted data: %10s\n",
	   llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
    printf("Lost space:   %12s    Linkdata:     %10s\n",
	   llstr(empty,llbuff),llstr(link_used,llbuff2));
  }
unknown's avatar
unknown committed
1281
  my_free((gptr) record,MYF(0));
unknown's avatar
unknown committed
1282 1283
  DBUG_RETURN (error);
 err:
unknown's avatar
unknown committed
1284
  mi_check_print_error(param,"got error: %d when reading datafile at record: %s",my_errno, llstr(records,llbuff));
unknown's avatar
unknown committed
1285
 err2:
unknown's avatar
unknown committed
1286
  my_free((gptr) record,MYF(0));
unknown's avatar
unknown committed
1287
  param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304
  DBUG_RETURN(1);
} /* chk_data_link */


	/* Recover old table by reading each record and writing all keys */
	/* Save new datafile-name in temp_filename */

int mi_repair(MI_CHECK *param, register MI_INFO *info,
	      my_string name, int rep_quick)
{
  int error,got_error;
  uint i;
  ha_rows start_records,new_header_length;
  my_off_t del;
  File new_file;
  MYISAM_SHARE *share=info->s;
  char llbuff[22],llbuff2[22];
unknown's avatar
unknown committed
1305 1306
  SORT_INFO sort_info;
  MI_SORT_PARAM sort_param;
unknown's avatar
unknown committed
1307 1308
  DBUG_ENTER("mi_repair");

unknown's avatar
unknown committed
1309 1310
  bzero((char *)&sort_info, sizeof(sort_info));
  bzero((char *)&sort_param, sizeof(sort_param));
unknown's avatar
unknown committed
1311
  start_records=info->state->records;
1312 1313
  new_header_length=(param->testflag & T_UNPACK) ? 0L :
    share->pack.header_length;
unknown's avatar
unknown committed
1314 1315
  got_error=1;
  new_file= -1;
unknown's avatar
unknown committed
1316
  sort_param.sort_info=&sort_info;
1317

unknown's avatar
unknown committed
1318 1319
  if (!(param->testflag & T_SILENT))
  {
1320
    printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
unknown's avatar
unknown committed
1321 1322
    printf("Data records: %s\n", llstr(info->state->records,llbuff));
  }
1323
  param->testflag|=T_REP; /* for easy checking */
unknown's avatar
unknown committed
1324

1325 1326 1327
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

unknown's avatar
unknown committed
1328
  if (!param->using_global_keycache)
unknown's avatar
unknown committed
1329 1330
    VOID(init_key_cache(dflt_key_cache, param->key_cache_block_size,
                        param->use_buffers, 0, 0));
unknown's avatar
unknown committed
1331 1332 1333 1334

  if (init_io_cache(&param->read_cache,info->dfile,
		    (uint) param->read_buffer_length,
		    READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1335 1336
  {
    bzero(&info->rec_cache,sizeof(info->rec_cache));
unknown's avatar
unknown committed
1337
    goto err;
1338
  }
unknown's avatar
unknown committed
1339 1340 1341 1342 1343 1344
  if (!rep_quick)
    if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length,
		      WRITE_CACHE, new_header_length, 1,
		      MYF(MY_WME | MY_WAIT_IF_FULL)))
      goto err;
  info->opt_flag|=WRITE_CACHE_USED;
unknown's avatar
unknown committed
1345
  if (!(sort_param.record=(byte*) my_malloc((uint) share->base.pack_reclength,
unknown's avatar
unknown committed
1346 1347
					   MYF(0))) ||
      !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
unknown's avatar
unknown committed
1348
  {
1349
    mi_check_print_error(param, "Not enough memory for extra record");
unknown's avatar
unknown committed
1350 1351 1352 1353 1354
    goto err;
  }

  if (!rep_quick)
  {
1355 1356
    /* Get real path for data file */
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
1357
					   share->data_file_name, "",
unknown's avatar
unknown committed
1358
					   DATA_TMP_EXT, 2+4),
unknown's avatar
unknown committed
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379
				 0,param->tmpfile_createflag,
				 share->base.raid_type,
				 share->base.raid_chunks,
				 share->base.raid_chunksize,
				 MYF(0))) < 0)
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
    if (filecopy(param,new_file,info->dfile,0L,new_header_length,
		 "datafile-header"))
      goto err;
    info->s->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
  }
unknown's avatar
unknown committed
1380 1381 1382 1383 1384 1385
  sort_info.info=info;
  sort_info.param = param;
  sort_param.read_cache=param->read_cache;
  sort_param.pos=sort_param.max_pos=share->pack.header_length;
  sort_param.filepos=new_header_length;
  param->read_cache.end_of_file=sort_info.filelength=
unknown's avatar
unknown committed
1386
    my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
unknown's avatar
unknown committed
1387 1388
  sort_info.dupp=0;
  sort_param.fix_datafile= (my_bool) (! rep_quick);
1389
  sort_param.master=1;
unknown's avatar
unknown committed
1390
  sort_info.max_records= ~(ha_rows) 0;
unknown's avatar
unknown committed
1391

unknown's avatar
unknown committed
1392
  set_data_file_type(&sort_info, share);
unknown's avatar
unknown committed
1393 1394 1395
  del=info->state->del;
  info->state->records=info->state->del=share->state.split=0;
  info->state->empty=0;
1396 1397 1398 1399
  param->glob_crc=0;
  if (param->testflag & T_CALC_CHECKSUM)
    param->calc_checksum=1;

unknown's avatar
unknown committed
1400 1401 1402 1403 1404 1405
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
  for (i=0 ; i < info->s->base.keys ; i++)
    share->state.key_root[i]= HA_OFFSET_ERROR;
  for (i=0 ; i < share->state.header.max_block_size ; i++)
    share->state.key_del[i]=  HA_OFFSET_ERROR;

unknown's avatar
unknown committed
1406 1407 1408 1409
  /*
    I think mi_repair and mi_repair_by_sort should do the same
    (according, e.g. to ha_myisam::repair), but as mi_repair doesn't
    touch key_map it cannot be used to T_CREATE_MISSING_KEYS.
1410
    That is what the next line is for
1411 1412
  */

1413 1414 1415
  if (param->testflag & T_CREATE_MISSING_KEYS)
    share->state.key_map= ((((ulonglong) 1L << share->base.keys)-1) &
			   param->keys_in_use);
1416

unknown's avatar
unknown committed
1417 1418 1419
  info->state->key_file_length=share->base.keystart;

  lock_memory(param);			/* Everything is alloced */
unknown's avatar
unknown committed
1420
  while (!(error=sort_get_next_record(&sort_param)))
unknown's avatar
unknown committed
1421
  {
unknown's avatar
unknown committed
1422
    if (writekeys(param,info,(byte*)sort_param.record,sort_param.filepos))
unknown's avatar
unknown committed
1423
    {
1424 1425
      if (my_errno != HA_ERR_FOUND_DUPP_KEY)
	goto err;
unknown's avatar
unknown committed
1426
      DBUG_DUMP("record",(byte*) sort_param.record,share->base.pack_reclength);
unknown's avatar
unknown committed
1427
      mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
1428
			  info->errkey+1,
unknown's avatar
unknown committed
1429
			  llstr(sort_param.start_recpos,llbuff),
1430
			  llstr(info->dupp_key_pos,llbuff2));
unknown's avatar
unknown committed
1431 1432 1433
      if (param->testflag & T_VERBOSE)
      {
	VOID(_mi_make_key(info,(uint) info->errkey,info->lastkey,
unknown's avatar
unknown committed
1434
			  sort_param.record,0L));
unknown's avatar
unknown committed
1435 1436 1437
	_mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey,
		      USE_WHOLE_KEY);
      }
unknown's avatar
unknown committed
1438
      sort_info.dupp++;
1439
      if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
unknown's avatar
unknown committed
1440
      {
unknown's avatar
unknown committed
1441 1442
        param->testflag|=T_RETRY_WITHOUT_QUICK;
	param->error_printed=1;
unknown's avatar
unknown committed
1443 1444 1445 1446
	goto err;
      }
      continue;
    }
unknown's avatar
unknown committed
1447
    if (sort_write_record(&sort_param))
unknown's avatar
unknown committed
1448 1449
      goto err;
  }
unknown's avatar
unknown committed
1450
  if (error > 0 || write_data_suffix(&sort_info, (my_bool)!rep_quick) ||
unknown's avatar
unknown committed
1451 1452 1453 1454 1455 1456 1457
      flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
    goto err;

  if (param->testflag & T_WRITE_LOOP)
  {
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
  }
1458
  if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
unknown's avatar
unknown committed
1459 1460 1461 1462 1463 1464 1465
  {
    mi_check_print_warning(param,
			   "Can't change size of indexfile, error: %d",
			   my_errno);
    goto err;
  }

unknown's avatar
unknown committed
1466
  if (rep_quick && del+sort_info.dupp != info->state->del)
unknown's avatar
unknown committed
1467 1468 1469 1470
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    got_error=1;
unknown's avatar
unknown committed
1471 1472
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
1473 1474
    goto err;
  }
1475 1476 1477 1478 1479 1480 1481 1482 1483 1484
  if (param->testflag & T_SAFE_REPAIR)
  {
    /* Don't repair if we loosed more than one row */
    if (info->state->records+1 < start_records)
    {
      info->state->records=start_records;
      got_error=1;
      goto err;
    }
  }
unknown's avatar
unknown committed
1485 1486 1487 1488 1489

  if (!rep_quick)
  {
    my_close(info->dfile,MYF(0));
    info->dfile=new_file;
unknown's avatar
unknown committed
1490
    info->state->data_file_length=sort_param.filepos;
unknown's avatar
unknown committed
1491 1492 1493
    share->state.version=(ulong) time((time_t*) 0);	/* Force reopen */
  }
  else
1494
  {
unknown's avatar
unknown committed
1495
    info->state->data_file_length=sort_param.max_pos;
1496
  }
unknown's avatar
unknown committed
1497 1498
  if (param->testflag & T_CALC_CHECKSUM)
    share->state.checksum=param->glob_crc;
unknown's avatar
unknown committed
1499 1500 1501 1502 1503

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
unknown's avatar
unknown committed
1504
    if (sort_info.dupp)
unknown's avatar
unknown committed
1505 1506
      mi_check_print_warning(param,
			     "%s records have been removed",
unknown's avatar
unknown committed
1507
			     llstr(sort_info.dupp,llbuff));
unknown's avatar
unknown committed
1508 1509 1510 1511 1512 1513 1514 1515
  }

  got_error=0;
  /* If invoked by external program that uses thr_lock */
  if (&share->state.state != info->state)
    memcpy( &share->state.state, info->state, sizeof(*info->state));

err:
1516 1517 1518 1519 1520 1521 1522
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
      my_close(new_file,MYF(0));
      info->dfile=new_file= -1;
unknown's avatar
unknown committed
1523
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
1524 1525 1526
			    DATA_TMP_EXT, share->base.raid_chunks,
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
unknown's avatar
unknown committed
1527
	  mi_open_datafile(info,share,-1))
1528 1529 1530
	got_error=1;
    }
  }
unknown's avatar
unknown committed
1531 1532 1533 1534
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d for record at pos %s",my_errno,
unknown's avatar
unknown committed
1535
		  llstr(sort_param.start_recpos,llbuff));
unknown's avatar
unknown committed
1536 1537 1538 1539 1540
    if (new_file >= 0)
    {
      VOID(my_close(new_file,MYF(0)));
      VOID(my_raid_delete(param->temp_filename,info->s->base.raid_chunks,
			  MYF(MY_WME)));
1541
      info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
unknown's avatar
unknown committed
1542
    }
1543
    mi_mark_crashed_on_repair(info);
unknown's avatar
unknown committed
1544
  }
1545 1546
  my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
                            MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
1547 1548
  my_free(sort_param.record,MYF(MY_ALLOW_ZERO_PTR));
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
1549 1550 1551
  VOID(end_io_cache(&param->read_cache));
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
  VOID(end_io_cache(&info->rec_cache));
unknown's avatar
unknown committed
1552
  got_error|=flush_blocks(param, share->key_cache, share->kfile);
unknown's avatar
unknown committed
1553 1554 1555 1556
  if (!got_error && param->testflag & T_UNPACK)
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
unknown's avatar
unknown committed
1557
    share->data_file_type=sort_info.new_data_file_type;
unknown's avatar
unknown committed
1558
  }
1559 1560
  share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
			  STATE_NOT_ANALYZED);
unknown's avatar
unknown committed
1561
  DBUG_RETURN(got_error);
1562
}
unknown's avatar
unknown committed
1563 1564 1565 1566


/* Uppate keyfile when doing repair */

1567 1568
static int writekeys(MI_CHECK *param, register MI_INFO *info, byte *buff,
		     my_off_t filepos)
unknown's avatar
unknown committed
1569 1570 1571 1572 1573 1574 1575 1576 1577 1578
{
  register uint i;
  uchar *key;
  DBUG_ENTER("writekeys");

  key=info->lastkey+info->s->base.max_key_length;
  for (i=0 ; i < info->s->base.keys ; i++)
  {
    if (((ulonglong) 1 << i) & info->s->state.key_map)
    {
1579 1580
      if (info->s->keyinfo[i].flag & HA_FULLTEXT )
      {
1581 1582
        if (_mi_ft_add(info,i,(char*) key,buff,filepos))
	  goto err;
1583
      }
1584
#ifdef HAVE_SPATIAL
1585 1586 1587 1588 1589 1590
      else if (info->s->keyinfo[i].flag & HA_SPATIAL)
      {
	uint key_length=_mi_make_key(info,i,key,buff,filepos);
	if (rtree_insert(info, i, key, key_length))
	  goto err;
      }
1591
#endif /*HAVE_SPATIAL*/
1592 1593 1594
      else
      {
	uint key_length=_mi_make_key(info,i,key,buff,filepos);
1595 1596
	if (_mi_ck_write(info,i,key,key_length))
	  goto err;
1597
      }
unknown's avatar
unknown committed
1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609
    }
  }
  DBUG_RETURN(0);

 err:
  if (my_errno == HA_ERR_FOUND_DUPP_KEY)
  {
    info->errkey=(int) i;			/* This key was found */
    while ( i-- > 0 )
    {
      if (((ulonglong) 1 << i) & info->s->state.key_map)
      {
1610 1611
	if (info->s->keyinfo[i].flag & HA_FULLTEXT)
        {
1612 1613
          if (_mi_ft_del(info,i,(char*) key,buff,filepos))
	    break;
1614 1615 1616 1617
        }
        else
	{
	  uint key_length=_mi_make_key(info,i,key,buff,filepos);
1618 1619
	  if (_mi_ck_delete(info,i,key,key_length))
	    break;
1620
	}
unknown's avatar
unknown committed
1621 1622 1623
      }
    }
  }
1624 1625 1626
  /* Remove checksum that was added to glob_crc in sort_get_next_record */
  if (param->calc_checksum)
    param->glob_crc-= info->checksum;
unknown's avatar
unknown committed
1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659
  DBUG_PRINT("error",("errno: %d",my_errno));
  DBUG_RETURN(-1);
} /* writekeys */


	/* Change all key-pointers that points to a records */

int movepoint(register MI_INFO *info, byte *record, my_off_t oldpos,
	      my_off_t newpos, uint prot_key)
{
  register uint i;
  uchar *key;
  uint key_length;
  DBUG_ENTER("movepoint");

  key=info->lastkey+info->s->base.max_key_length;
  for (i=0 ; i < info->s->base.keys; i++)
  {
    if (i != prot_key && (((ulonglong) 1 << i) & info->s->state.key_map))
    {
      key_length=_mi_make_key(info,i,key,record,oldpos);
      if (info->s->keyinfo[i].flag & HA_NOSAME)
      {					/* Change pointer direct */
	uint nod_flag;
	MI_KEYDEF *keyinfo;
	keyinfo=info->s->keyinfo+i;
	if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
		       (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
		       info->s->state.key_root[i]))
	  DBUG_RETURN(-1);
	nod_flag=mi_test_if_nod(info->buff);
	_mi_dpointer(info,info->int_keypos-nod_flag-
		     info->s->rec_reflength,newpos);
1660 1661
	if (_mi_write_keypage(info,keyinfo,info->last_keypage,
                              DFLT_INIT_HITS,info->buff))
unknown's avatar
unknown committed
1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
	  DBUG_RETURN(-1);
      }
      else
      {					/* Change old key to new */
	if (_mi_ck_delete(info,i,key,key_length))
	  DBUG_RETURN(-1);
	key_length=_mi_make_key(info,i,key,record,newpos);
	if (_mi_ck_write(info,i,key,key_length))
	  DBUG_RETURN(-1);
      }
    }
  }
  DBUG_RETURN(0);
} /* movepoint */


	/* Tell system that we want all memory for our cache */

void lock_memory(MI_CHECK *param __attribute__((unused)))
{
#ifdef SUN_OS				/* Key-cacheing thrases on sun 4.1 */
  if (param->opt_lock_memory)
  {
    int success = mlockall(MCL_CURRENT);	/* or plock(DATLOCK); */
    if (geteuid() == 0 && success != 0)
      mi_check_print_warning(param,
			     "Failed to lock memory. errno %d",my_errno);
  }
#endif
} /* lock_memory */


	/* Flush all changed blocks to disk */

unknown's avatar
unknown committed
1696
int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file)
unknown's avatar
unknown committed
1697
{
1698
  if (flush_key_blocks(key_cache, file, FLUSH_RELEASE))
unknown's avatar
unknown committed
1699 1700 1701 1702 1703
  {
    mi_check_print_error(param,"%d when trying to write bufferts",my_errno);
    return(1);
  }
  if (!param->using_global_keycache)
1704
    end_key_cache(key_cache,1);
unknown's avatar
unknown committed
1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716
  return 0;
} /* flush_blocks */


	/* Sort index for more efficent reads */

int mi_sort_index(MI_CHECK *param, register MI_INFO *info, my_string name)
{
  reg2 uint key;
  reg1 MI_KEYDEF *keyinfo;
  File new_file;
  my_off_t index_pos[MI_MAX_POSSIBLE_KEY];
unknown's avatar
unknown committed
1717
  uint r_locks,w_locks;
1718
  int old_lock;
unknown's avatar
unknown committed
1719
  MYISAM_SHARE *share=info->s;
1720
  MI_STATE_INFO old_state;
1721
  DBUG_ENTER("mi_sort_index");
unknown's avatar
unknown committed
1722 1723 1724 1725

  if (!(param->testflag & T_SILENT))
    printf("- Sorting index for MyISAM-table '%s'\n",name);

1726 1727 1728 1729
  /* Get real path for index file */
  fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
  if ((new_file=my_create(fn_format(param->temp_filename,param->temp_filename,
				    "", INDEX_TMP_EXT,2+4),
unknown's avatar
unknown committed
1730 1731 1732 1733 1734 1735
			  0,param->tmpfile_createflag,MYF(0))) <= 0)
  {
    mi_check_print_error(param,"Can't create new tempfile: '%s'",
			 param->temp_filename);
    DBUG_RETURN(-1);
  }
unknown's avatar
unknown committed
1736 1737
  if (filecopy(param, new_file,share->kfile,0L,
	       (ulong) share->base.keystart, "headerblock"))
unknown's avatar
unknown committed
1738 1739
    goto err;

unknown's avatar
unknown committed
1740 1741
  param->new_file_pos=share->base.keystart;
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
unknown's avatar
unknown committed
1742 1743
       key++,keyinfo++)
  {
unknown's avatar
unknown committed
1744
    if (!(((ulonglong) 1 << key) & share->state.key_map))
unknown's avatar
unknown committed
1745 1746
      continue;

unknown's avatar
unknown committed
1747
    if (share->state.key_root[key] != HA_OFFSET_ERROR)
unknown's avatar
unknown committed
1748
    {
1749
      index_pos[key]=param->new_file_pos;	/* Write first block here */
unknown's avatar
unknown committed
1750
      if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
unknown's avatar
unknown committed
1751 1752 1753 1754 1755 1756 1757 1758
			 new_file))
	goto err;
    }
    else
      index_pos[key]= HA_OFFSET_ERROR;		/* No blocks */
  }

  /* Flush key cache for this file if we are calling this outside myisamchk */
unknown's avatar
unknown committed
1759
  flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
unknown's avatar
unknown committed
1760

unknown's avatar
unknown committed
1761
  share->state.version=(ulong) time((time_t*) 0);
1762 1763 1764 1765 1766
  old_state= share->state;			/* save state if not stored */
  r_locks=   share->r_locks;
  w_locks=   share->w_locks;
  old_lock=  info->lock_type;

1767
	/* Put same locks as old file */
1768
  share->r_locks= share->w_locks= share->tot_locks= 0;
unknown's avatar
unknown committed
1769 1770 1771
  (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
  VOID(my_close(share->kfile,MYF(MY_WME)));
  share->kfile = -1;
1772
  VOID(my_close(new_file,MYF(MY_WME)));
unknown's avatar
unknown committed
1773
  if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
1774
			MYF(0)) ||
unknown's avatar
unknown committed
1775
      mi_open_keyfile(share))
1776
    goto err2;
1777
  info->lock_type= F_UNLCK;			/* Force mi_readinfo to lock */
unknown's avatar
unknown committed
1778
  _mi_readinfo(info,F_WRLCK,0);			/* Will lock the table */
1779 1780 1781
  info->lock_type=  old_lock;
  share->r_locks=   r_locks;
  share->w_locks=   w_locks;
1782
  share->tot_locks= r_locks+w_locks;
1783
  share->state=     old_state;			/* Restore old state */
unknown's avatar
unknown committed
1784 1785 1786 1787 1788 1789 1790 1791

  info->state->key_file_length=param->new_file_pos;
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
  for (key=0 ; key < info->s->base.keys ; key++)
    info->s->state.key_root[key]=index_pos[key];
  for (key=0 ; key < info->s->state.header.max_block_size ; key++)
    info->s->state.key_del[key]=  HA_OFFSET_ERROR;

1792
  info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
unknown's avatar
unknown committed
1793 1794 1795 1796
  DBUG_RETURN(0);

err:
  VOID(my_close(new_file,MYF(MY_WME)));
1797
err2:
unknown's avatar
unknown committed
1798 1799
  VOID(my_delete(param->temp_filename,MYF(MY_WME)));
  DBUG_RETURN(-1);
1800
} /* mi_sort_index */
unknown's avatar
unknown committed
1801 1802 1803 1804 1805 1806 1807


	 /* Sort records recursive using one index */

static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
			  my_off_t pagepos, File new_file)
{
1808
  uint length,nod_flag,used_length, key_length;
unknown's avatar
unknown committed
1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
  uchar *buff,*keypos,*endpos;
  uchar key[MI_MAX_POSSIBLE_KEY_BUFF];
  my_off_t new_page_pos,next_page;
  char llbuff[22];
  DBUG_ENTER("sort_one_index");

  new_page_pos=param->new_file_pos;
  param->new_file_pos+=keyinfo->block_length;

  if (!(buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
  {
1820
    mi_check_print_error(param,"Not enough memory for key block");
unknown's avatar
unknown committed
1821 1822
    DBUG_RETURN(-1);
  }
1823
  if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
unknown's avatar
unknown committed
1824 1825 1826 1827 1828
  {
    mi_check_print_error(param,"Can't read key block from filepos: %s",
		llstr(pagepos,llbuff));
    goto err;
  }
1829
  if ((nod_flag=mi_test_if_nod(buff)) || keyinfo->flag & HA_FULLTEXT)
unknown's avatar
unknown committed
1830 1831 1832 1833 1834 1835 1836 1837 1838 1839
  {
    used_length=mi_getint(buff);
    keypos=buff+2+nod_flag;
    endpos=buff+used_length;
    for ( ;; )
    {
      if (nod_flag)
      {
	next_page=_mi_kpos(nod_flag,keypos);
	_mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
1840
	if (sort_one_index(param,info,keyinfo,next_page,new_file))
unknown's avatar
unknown committed
1841
	{
unknown's avatar
unknown committed
1842
	  DBUG_PRINT("error",
unknown's avatar
unknown committed
1843
		     ("From page: %ld, keyoffset: %lu  used_length: %d",
unknown's avatar
unknown committed
1844 1845
		      (ulong) pagepos, (ulong) (keypos - buff),
		      (int) used_length));
unknown's avatar
unknown committed
1846 1847 1848 1849 1850
	  DBUG_DUMP("buff",(byte*) buff,used_length);
	  goto err;
	}
      }
      if (keypos >= endpos ||
1851
	  (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
unknown's avatar
unknown committed
1852
	break;
1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
      DBUG_ASSERT(keypos <= endpos);
      if (keyinfo->flag & HA_FULLTEXT)
      {
        uint off;
        int  subkeys;
        get_key_full_length_rdonly(off, key);
        subkeys=ft_sintXkorr(key+off);
        if (subkeys < 0)
        {
          next_page= _mi_dpos(info,0,key+key_length);
          _mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength,
                       param->new_file_pos); /* Save new pos */
          if (sort_one_index(param,info,&info->s->ft2_keyinfo,
                             next_page,new_file))
            goto err;
        }
      }
unknown's avatar
unknown committed
1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889
    }
  }

  /* Fill block with zero and write it to the new index file */
  length=mi_getint(buff);
  bzero((byte*) buff+length,keyinfo->block_length-length);
  if (my_pwrite(new_file,(byte*) buff,(uint) keyinfo->block_length,
		new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
  {
    mi_check_print_error(param,"Can't write indexblock, error: %d",my_errno);
    goto err;
  }
  my_afree((gptr) buff);
  DBUG_RETURN(0);
err:
  my_afree((gptr) buff);
  DBUG_RETURN(1);
} /* sort_one_index */


1890 1891 1892 1893 1894 1895 1896 1897
	/*
	  Let temporary file replace old file.
	  This assumes that the new file was created in the same
	  directory as given by realpath(filename).
	  This will ensure that any symlinks that are used will still work.
	  Copy stats from old file to new file, deletes orignal and
	  changes new file name to old file name
	*/
unknown's avatar
unknown committed
1898 1899 1900

int change_to_newfile(const char * filename, const char * old_ext,
		      const char * new_ext,
1901 1902
		      uint raid_chunks __attribute__((unused)),
		      myf MyFlags)
unknown's avatar
unknown committed
1903 1904 1905 1906 1907 1908 1909
{
  char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
#ifdef USE_RAID
  if (raid_chunks)
    return my_raid_redel(fn_format(old_filename,filename,"",old_ext,2+4),
			 fn_format(new_filename,filename,"",new_ext,2+4),
			 raid_chunks,
1910
			 MYF(MY_WME | MY_LINK_WARNING | MyFlags));
unknown's avatar
unknown committed
1911
#endif
1912 1913 1914 1915
  /* Get real path to filename */
  (void) fn_format(old_filename,filename,"",old_ext,2+4+32);
  return my_redel(old_filename,
		  fn_format(new_filename,old_filename,"",new_ext,2+4),
1916
		  MYF(MY_WME | MY_LINK_WARNING | MyFlags));
unknown's avatar
unknown committed
1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973
} /* change_to_newfile */


	/* Locks a whole file */
	/* Gives an error-message if file can't be locked */

int lock_file(MI_CHECK *param, File file, my_off_t start, int lock_type,
	      const char *filetype, const char *filename)
{
  if (my_lock(file,lock_type,start,F_TO_EOF,
	      param->testflag & T_WAIT_FOREVER ? MYF(MY_SEEK_NOT_DONE) :
	      MYF(MY_SEEK_NOT_DONE |  MY_DONT_WAIT)))
  {
    mi_check_print_error(param," %d when locking %s '%s'",my_errno,filetype,filename);
    param->error_printed=2;		/* Don't give that data is crashed */
    return 1;
  }
  return 0;
} /* lock_file */


	/* Copy a block between two files */

int filecopy(MI_CHECK *param, File to,File from,my_off_t start,
	     my_off_t length, const char *type)
{
  char tmp_buff[IO_SIZE],*buff;
  ulong buff_length;
  DBUG_ENTER("filecopy");

  buff_length=(ulong) min(param->write_buffer_length,length);
  if (!(buff=my_malloc(buff_length,MYF(0))))
  {
    buff=tmp_buff; buff_length=IO_SIZE;
  }

  VOID(my_seek(from,start,MY_SEEK_SET,MYF(0)));
  while (length > buff_length)
  {
    if (my_read(from,(byte*) buff,buff_length,MYF(MY_NABP)) ||
	my_write(to,(byte*) buff,buff_length,param->myf_rw))
      goto err;
    length-= buff_length;
  }
  if (my_read(from,(byte*) buff,(uint) length,MYF(MY_NABP)) ||
      my_write(to,(byte*) buff,(uint) length,param->myf_rw))
    goto err;
  if (buff != tmp_buff)
    my_free(buff,MYF(0));
  DBUG_RETURN(0);
err:
  if (buff != tmp_buff)
    my_free(buff,MYF(0));
  mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
		       type,my_errno);
  DBUG_RETURN(1);
}
unknown's avatar
unknown committed
1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989


/*
  Repair table or given index using sorting

  SYNOPSIS
    mi_repair_by_sort()
    param		Repair parameters
    info		MyISAM handler to repair
    name		Name of table (for warnings)
    rep_quick		set to <> 0 if we should not change data file

  RESULT
    0	ok
    <>0	Error
*/
unknown's avatar
unknown committed
1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001

int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
		      const char * name, int rep_quick)
{
  int got_error;
  uint i;
  ulong length;
  ha_rows start_records;
  my_off_t new_header_length,del;
  File new_file;
  MI_SORT_PARAM sort_param;
  MYISAM_SHARE *share=info->s;
2002
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
2003 2004
  ulong   *rec_per_key_part;
  char llbuff[22];
unknown's avatar
unknown committed
2005
  SORT_INFO sort_info;
2006
  ulonglong key_map=share->state.key_map;
unknown's avatar
unknown committed
2007
  DBUG_ENTER("mi_repair_by_sort");
unknown's avatar
unknown committed
2008 2009 2010 2011 2012 2013 2014 2015

  start_records=info->state->records;
  got_error=1;
  new_file= -1;
  new_header_length=(param->testflag & T_UNPACK) ? 0 :
    share->pack.header_length;
  if (!(param->testflag & T_SILENT))
  {
2016
    printf("- recovering (with sort) MyISAM-table '%s'\n",name);
unknown's avatar
unknown committed
2017 2018
    printf("Data records: %s\n", llstr(start_records,llbuff));
  }
2019
  param->testflag|=T_REP; /* for easy checking */
2020

2021 2022 2023
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

unknown's avatar
unknown committed
2024
  bzero((char*)&sort_info,sizeof(sort_info));
2025
  bzero((char *)&sort_param, sizeof(sort_param));
unknown's avatar
unknown committed
2026
  if (!(sort_info.key_block=
unknown's avatar
unknown committed
2027 2028 2029 2030 2031 2032 2033 2034 2035 2036
	alloc_key_blocks(param,
			 (uint) param->sort_key_blocks,
			 share->base.max_key_block_length))
      || init_io_cache(&param->read_cache,info->dfile,
		       (uint) param->read_buffer_length,
		       READ_CACHE,share->pack.header_length,1,MYF(MY_WME)) ||
      (! rep_quick &&
       init_io_cache(&info->rec_cache,info->dfile,
		     (uint) param->write_buffer_length,
		     WRITE_CACHE,new_header_length,1,
2037
		     MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
unknown's avatar
unknown committed
2038
    goto err;
unknown's avatar
unknown committed
2039
  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
unknown's avatar
unknown committed
2040 2041 2042
  info->opt_flag|=WRITE_CACHE_USED;
  info->rec_cache.file=info->dfile;		/* for sort_delete_record */

unknown's avatar
unknown committed
2043
  if (!(sort_param.record=(byte*) my_malloc((uint) share->base.pack_reclength,
unknown's avatar
unknown committed
2044 2045
					   MYF(0))) ||
      !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
unknown's avatar
unknown committed
2046
  {
2047
    mi_check_print_error(param, "Not enough memory for extra record");
unknown's avatar
unknown committed
2048 2049 2050 2051
    goto err;
  }
  if (!rep_quick)
  {
2052 2053
    /* Get real path for data file */
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
2054 2055
					   share->data_file_name, "",
					   DATA_TMP_EXT, 2+4),
unknown's avatar
unknown committed
2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078
				 0,param->tmpfile_createflag,
				 share->base.raid_type,
				 share->base.raid_chunks,
				 share->base.raid_chunksize,
				 MYF(0))) < 0)
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
    if (filecopy(param, new_file,info->dfile,0L,new_header_length,
		 "datafile-header"))
      goto err;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
    share->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
  }

  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2079 2080
  if (!(param->testflag & T_CREATE_MISSING_KEYS))
  {
unknown's avatar
unknown committed
2081 2082 2083 2084
    /*
      Flush key cache for this file if we are calling this outside
      myisamchk
    */
unknown's avatar
unknown committed
2085
    flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
unknown's avatar
unknown committed
2086
    /* Clear the pointers to the given rows */
2087 2088 2089 2090
    for (i=0 ; i < share->base.keys ; i++)
      share->state.key_root[i]= HA_OFFSET_ERROR;
    for (i=0 ; i < share->state.header.max_block_size ; i++)
      share->state.key_del[i]=  HA_OFFSET_ERROR;
unknown's avatar
unknown committed
2091
    info->state->key_file_length=share->base.keystart;
2092 2093
  }
  else
unknown's avatar
unknown committed
2094
  {
unknown's avatar
unknown committed
2095
    if (flush_key_blocks(share->key_cache,share->kfile, FLUSH_FORCE_WRITE))
unknown's avatar
unknown committed
2096
      goto err;
2097
    key_map= ~key_map;				/* Create the missing keys */
unknown's avatar
unknown committed
2098
  }
unknown's avatar
unknown committed
2099

unknown's avatar
unknown committed
2100 2101
  sort_info.info=info;
  sort_info.param = param;
unknown's avatar
unknown committed
2102

unknown's avatar
unknown committed
2103 2104 2105 2106 2107
  set_data_file_type(&sort_info, share);
  sort_param.filepos=new_header_length;
  sort_info.dupp=0;
  sort_info.buff=0;
  param->read_cache.end_of_file=sort_info.filelength=
unknown's avatar
unknown committed
2108 2109
    my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));

unknown's avatar
unknown committed
2110
  sort_param.wordlist=NULL;
2111

unknown's avatar
unknown committed
2112 2113 2114 2115 2116 2117
  if (share->data_file_type == DYNAMIC_RECORD)
    length=max(share->base.min_pack_length+1,share->base.min_block_length);
  else if (share->data_file_type == COMPRESSED_RECORD)
    length=share->base.min_block_length;
  else
    length=share->base.pack_reclength;
unknown's avatar
unknown committed
2118
  sort_info.max_records=
2119
    ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
unknown's avatar
unknown committed
2120
     (ha_rows) (sort_info.filelength/length+1));
unknown's avatar
unknown committed
2121 2122 2123
  sort_param.key_cmp=sort_key_cmp;
  sort_param.lock_in_memory=lock_memory;
  sort_param.tmpdir=param->tmpdir;
unknown's avatar
unknown committed
2124 2125
  sort_param.sort_info=&sort_info;
  sort_param.fix_datafile= (my_bool) (! rep_quick);
2126
  sort_param.master =1;
2127
  
unknown's avatar
unknown committed
2128
  del=info->state->del;
2129 2130 2131
  param->glob_crc=0;
  if (param->testflag & T_CALC_CHECKSUM)
    param->calc_checksum=1;
unknown's avatar
unknown committed
2132 2133

  rec_per_key_part= param->rec_per_key_part;
unknown's avatar
unknown committed
2134 2135
  for (sort_param.key=0 ; sort_param.key < share->base.keys ;
       rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
unknown's avatar
unknown committed
2136
  {
unknown's avatar
unknown committed
2137 2138
    sort_param.read_cache=param->read_cache;
    sort_param.keyinfo=share->keyinfo+sort_param.key;
2139
    sort_param.seg=sort_param.keyinfo->seg;
unknown's avatar
unknown committed
2140
    if (!(((ulonglong) 1 << sort_param.key) & key_map))
2141 2142 2143
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
2144 2145
	     (char*) (share->state.rec_per_key_part +
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
unknown's avatar
unknown committed
2146
	     sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
unknown's avatar
unknown committed
2147
      continue;
2148
    }
unknown's avatar
unknown committed
2149 2150

    if ((!(param->testflag & T_SILENT)))
unknown's avatar
unknown committed
2151 2152
      printf ("- Fixing index %d\n",sort_param.key+1);
    sort_param.max_pos=sort_param.pos=share->pack.header_length;
2153
    keyseg=sort_param.seg;
unknown's avatar
unknown committed
2154
    bzero((char*) sort_param.unique,sizeof(sort_param.unique));
unknown's avatar
unknown committed
2155
    sort_param.key_length=share->rec_reflength;
unknown's avatar
unknown committed
2156
    for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
unknown's avatar
unknown committed
2157
    {
unknown's avatar
unknown committed
2158 2159 2160 2161 2162 2163
      sort_param.key_length+=keyseg[i].length;
      if (keyseg[i].flag & HA_SPACE_PACK)
	sort_param.key_length+=get_pack_length(keyseg[i].length);
      if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH))
	sort_param.key_length+=2 + test(keyseg[i].length >= 127);
      if (keyseg[i].flag & HA_NULL_PART)
unknown's avatar
unknown committed
2164 2165 2166 2167 2168
	sort_param.key_length++;
    }
    info->state->records=info->state->del=share->state.split=0;
    info->state->empty=0;

unknown's avatar
unknown committed
2169
    if (sort_param.keyinfo->flag & HA_FULLTEXT)
2170
    {
2171 2172
      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                    sort_param.keyinfo->seg->charset->mbmaxlen;
unknown's avatar
unknown committed
2173
      sort_info.max_records=
2174
        (ha_rows) (sort_info.filelength/ft_min_word_len+1);
2175 2176

      sort_param.key_read=sort_ft_key_read;
2177
      sort_param.key_write=sort_ft_key_write;
2178
      sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2179 2180
    }
    else
2181
    {
2182
      sort_param.key_read=sort_key_read;
2183 2184
      sort_param.key_write=sort_key_write;
    }
2185

unknown's avatar
unknown committed
2186 2187 2188
    if (_create_index_by_sort(&sort_param,
			      (my_bool) (!(param->testflag & T_VERBOSE)),
			      (uint) param->sort_buffer_length))
2189 2190
    {
      param->retry_repair=1;
unknown's avatar
unknown committed
2191
      goto err;
2192
    }
2193
    param->calc_checksum=0;			/* No need to calc glob_crc */
unknown's avatar
unknown committed
2194 2195

    /* Set for next loop */
unknown's avatar
unknown committed
2196
    sort_info.max_records= (ha_rows) info->state->records;
unknown's avatar
unknown committed
2197 2198

    if (param->testflag & T_STATISTICS)
unknown's avatar
unknown committed
2199
      update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2200 2201
                       param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
                       sort_param.notnull: NULL,(ulonglong) info->state->records);
unknown's avatar
unknown committed
2202
    share->state.key_map|=(ulonglong) 1 << sort_param.key;
unknown's avatar
unknown committed
2203

unknown's avatar
unknown committed
2204
    if (sort_param.fix_datafile)
unknown's avatar
unknown committed
2205
    {
unknown's avatar
unknown committed
2206 2207
      param->read_cache.end_of_file=sort_param.filepos;
      if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
unknown's avatar
unknown committed
2208
	goto err;
2209 2210 2211 2212 2213 2214 2215 2216 2217
      if (param->testflag & T_SAFE_REPAIR)
      {
	/* Don't repair if we loosed more than one row */
	if (info->state->records+1 < start_records)
	{
	  info->state->records=start_records;
	  goto err;
	}
      }
unknown's avatar
unknown committed
2218 2219
      share->state.state.data_file_length = info->state->data_file_length=
	sort_param.filepos;
unknown's avatar
unknown committed
2220 2221 2222 2223
      /* Only whole records */
      share->state.version=(ulong) time((time_t*) 0);
      my_close(info->dfile,MYF(0));
      info->dfile=new_file;
unknown's avatar
unknown committed
2224
      share->data_file_type=sort_info.new_data_file_type;
unknown's avatar
unknown committed
2225
      share->pack.header_length=(ulong) new_header_length;
unknown's avatar
unknown committed
2226
      sort_param.fix_datafile=0;
unknown's avatar
unknown committed
2227 2228
    }
    else
unknown's avatar
unknown committed
2229
      info->state->data_file_length=sort_param.max_pos;
unknown's avatar
unknown committed
2230 2231

    param->read_cache.file=info->dfile;		/* re-init read cache */
unknown's avatar
unknown committed
2232 2233
    reinit_io_cache(&param->read_cache,READ_CACHE,share->pack.header_length,
                    1,1);
unknown's avatar
unknown committed
2234 2235 2236 2237 2238 2239 2240
  }

  if (param->testflag & T_WRITE_LOOP)
  {
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
  }

unknown's avatar
unknown committed
2241
  if (rep_quick && del+sort_info.dupp != info->state->del)
unknown's avatar
unknown committed
2242 2243 2244 2245
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    got_error=1;
unknown's avatar
unknown committed
2246 2247
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
2248 2249 2250
    goto err;
  }

unknown's avatar
unknown committed
2251
  if (rep_quick & T_FORCE_UNIQUENESS)
unknown's avatar
unknown committed
2252 2253 2254 2255 2256 2257 2258 2259 2260
  {
    my_off_t skr=info->state->data_file_length+
      (share->options & HA_OPTION_COMPRESS_RECORD ?
       MEMMAP_EXTRA_MARGIN : 0);
#ifdef USE_RELOC
    if (share->data_file_type == STATIC_RECORD &&
	skr < share->base.reloc*share->base.min_pack_length)
      skr=share->base.reloc*share->base.min_pack_length;
#endif
unknown's avatar
unknown committed
2261
    if (skr != sort_info.filelength && !info->s->base.raid_type)
2262
      if (my_chsize(info->dfile,skr,0,MYF(0)))
unknown's avatar
unknown committed
2263 2264 2265 2266
	mi_check_print_warning(param,
			       "Can't change size of datafile,  error: %d",
			       my_errno);
  }
unknown's avatar
unknown committed
2267
  if (param->testflag & T_CALC_CHECKSUM)
2268 2269
    share->state.checksum=param->glob_crc;

2270
  if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
unknown's avatar
unknown committed
2271 2272 2273 2274 2275 2276 2277 2278
    mi_check_print_warning(param,
			   "Can't change size of indexfile, error: %d",
			   my_errno);

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
unknown's avatar
unknown committed
2279
    if (sort_info.dupp)
unknown's avatar
unknown committed
2280 2281
      mi_check_print_warning(param,
			     "%s records have been removed",
unknown's avatar
unknown committed
2282
			     llstr(sort_info.dupp,llbuff));
unknown's avatar
unknown committed
2283 2284 2285 2286 2287 2288 2289
  }
  got_error=0;

  if (&share->state.state != info->state)
    memcpy( &share->state.state, info->state, sizeof(*info->state));

err:
unknown's avatar
unknown committed
2290
  got_error|= flush_blocks(param, share->key_cache, share->kfile);
2291 2292 2293 2294 2295 2296 2297 2298
  VOID(end_io_cache(&info->rec_cache));
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
      my_close(new_file,MYF(0));
      info->dfile=new_file= -1;
unknown's avatar
unknown committed
2299
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
2300 2301 2302
			    DATA_TMP_EXT, share->base.raid_chunks,
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
unknown's avatar
unknown committed
2303
	  mi_open_datafile(info,share,-1))
2304 2305 2306
	got_error=1;
    }
  }
unknown's avatar
unknown committed
2307 2308 2309 2310 2311 2312 2313
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d when fixing table",my_errno);
    if (new_file >= 0)
    {
      VOID(my_close(new_file,MYF(0)));
2314
      VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
unknown's avatar
unknown committed
2315
			  MYF(MY_WME)));
2316
      if (info->dfile == new_file)
2317
	info->dfile= -1;
unknown's avatar
unknown committed
2318
    }
2319
    mi_mark_crashed_on_repair(info);
unknown's avatar
unknown committed
2320
  }
2321 2322 2323 2324
  else if (key_map == share->state.key_map)
    share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
  share->state.changed|=STATE_NOT_SORTED_PAGES;

2325 2326
  my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
                            MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2327
  my_free(sort_param.record,MYF(MY_ALLOW_ZERO_PTR));
2328
  my_free((gptr) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
2329
  my_free((gptr) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2330
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2331 2332
  VOID(end_io_cache(&param->read_cache));
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2333
  if (!got_error && (param->testflag & T_UNPACK))
unknown's avatar
unknown committed
2334 2335 2336 2337 2338
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
  }
  DBUG_RETURN(got_error);
2339
}
unknown's avatar
unknown committed
2340

unknown's avatar
unknown committed
2341 2342 2343 2344
/*
  Threaded repair of table using sorting

  SYNOPSIS
2345
    mi_repair_parallel()
unknown's avatar
unknown committed
2346 2347 2348 2349 2350 2351 2352 2353
    param		Repair parameters
    info		MyISAM handler to repair
    name		Name of table (for warnings)
    rep_quick		set to <> 0 if we should not change data file

  DESCRIPTION
    Same as mi_repair_by_sort but do it multithreaded
    Each key is handled by a separate thread.
unknown's avatar
unknown committed
2354
    TODO: make a number of threads a parameter
unknown's avatar
unknown committed
2355 2356 2357 2358 2359

  RESULT
    0	ok
    <>0	Error
*/
unknown's avatar
unknown committed
2360

2361
int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
unknown's avatar
unknown committed
2362
			const char * name, int rep_quick)
unknown's avatar
unknown committed
2363
{
2364 2365 2366
#ifndef THREAD
  return mi_repair_by_sort(param, info, name, rep_quick);
#else
unknown's avatar
unknown committed
2367
  int got_error;
unknown's avatar
unknown committed
2368
  uint i,key, total_key_length, istep;
unknown's avatar
unknown committed
2369
  ulong rec_length;
unknown's avatar
unknown committed
2370 2371 2372
  ha_rows start_records;
  my_off_t new_header_length,del;
  File new_file;
unknown's avatar
unknown committed
2373
  MI_SORT_PARAM *sort_param=0;
unknown's avatar
unknown committed
2374 2375
  MYISAM_SHARE *share=info->s;
  ulong   *rec_per_key_part;
2376
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
2377 2378 2379 2380
  char llbuff[22];
  IO_CACHE_SHARE io_share;
  SORT_INFO sort_info;
  ulonglong key_map=share->state.key_map;
2381
  pthread_attr_t thr_attr;
2382
  DBUG_ENTER("mi_repair_parallel");
unknown's avatar
unknown committed
2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393

  start_records=info->state->records;
  got_error=1;
  new_file= -1;
  new_header_length=(param->testflag & T_UNPACK) ? 0 :
    share->pack.header_length;
  if (!(param->testflag & T_SILENT))
  {
    printf("- parallel recovering (with sort) MyISAM-table '%s'\n",name);
    printf("Data records: %s\n", llstr(start_records,llbuff));
  }
2394
  param->testflag|=T_REP; /* for easy checking */
unknown's avatar
unknown committed
2395

2396 2397 2398
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

unknown's avatar
unknown committed
2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414
  bzero((char*)&sort_info,sizeof(sort_info));
  if (!(sort_info.key_block=
	alloc_key_blocks(param,
			 (uint) param->sort_key_blocks,
			 share->base.max_key_block_length))
      || init_io_cache(&param->read_cache,info->dfile,
		       (uint) param->read_buffer_length,
		       READ_CACHE,share->pack.header_length,1,MYF(MY_WME)) ||
      (! rep_quick &&
       init_io_cache(&info->rec_cache,info->dfile,
		     (uint) param->write_buffer_length,
		     WRITE_CACHE,new_header_length,1,
		     MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
    goto err;
  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
  info->opt_flag|=WRITE_CACHE_USED;
2415
  info->rec_cache.file=info->dfile;         /* for sort_delete_record */
unknown's avatar
unknown committed
2416 2417 2418 2419 2420

  if (!rep_quick)
  {
    /* Get real path for data file */
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
2421
					   share->data_file_name, "",
unknown's avatar
unknown committed
2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452
					   DATA_TMP_EXT,
					   2+4),
				 0,param->tmpfile_createflag,
				 share->base.raid_type,
				 share->base.raid_chunks,
				 share->base.raid_chunksize,
				 MYF(0))) < 0)
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
    if (filecopy(param, new_file,info->dfile,0L,new_header_length,
		 "datafile-header"))
      goto err;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
    share->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
  }

  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
  if (!(param->testflag & T_CREATE_MISSING_KEYS))
  {
    /*
      Flush key cache for this file if we are calling this outside
      myisamchk
    */
unknown's avatar
unknown committed
2453
    flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
unknown's avatar
unknown committed
2454 2455 2456 2457 2458 2459 2460 2461 2462
    /* Clear the pointers to the given rows */
    for (i=0 ; i < share->base.keys ; i++)
      share->state.key_root[i]= HA_OFFSET_ERROR;
    for (i=0 ; i < share->state.header.max_block_size ; i++)
      share->state.key_del[i]=  HA_OFFSET_ERROR;
    info->state->key_file_length=share->base.keystart;
  }
  else
  {
unknown's avatar
unknown committed
2463
    if (flush_key_blocks(share->key_cache,share->kfile, FLUSH_FORCE_WRITE))
unknown's avatar
unknown committed
2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477
      goto err;
    key_map= ~key_map;				/* Create the missing keys */
  }

  sort_info.info=info;
  sort_info.param = param;

  set_data_file_type(&sort_info, share);
  sort_info.dupp=0;
  sort_info.buff=0;
  param->read_cache.end_of_file=sort_info.filelength=
    my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));

  if (share->data_file_type == DYNAMIC_RECORD)
unknown's avatar
unknown committed
2478
    rec_length=max(share->base.min_pack_length+1,share->base.min_block_length);
unknown's avatar
unknown committed
2479
  else if (share->data_file_type == COMPRESSED_RECORD)
unknown's avatar
unknown committed
2480
    rec_length=share->base.min_block_length;
unknown's avatar
unknown committed
2481
  else
unknown's avatar
unknown committed
2482
    rec_length=share->base.pack_reclength;
2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494
  /*
    +1 below is required hack for parallel repair mode.
    The info->state->records value, that is compared later
    to sort_info.max_records and cannot exceed it, is
    increased in sort_key_write. In mi_repair_by_sort, sort_key_write
    is called after sort_key_read, where the comparison is performed,
    but in parallel mode master thread can call sort_key_write
    before some other repair thread calls sort_key_read.
    Furthermore I'm not even sure +1 would be enough.
    May be sort_info.max_records shold be always set to max value in
    parallel mode.
  */
unknown's avatar
unknown committed
2495
  sort_info.max_records=
2496
    ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records + 1:
unknown's avatar
unknown committed
2497
     (ha_rows) (sort_info.filelength/rec_length+1));
unknown's avatar
unknown committed
2498 2499 2500 2501 2502 2503 2504 2505

  del=info->state->del;
  param->glob_crc=0;
  if (param->testflag & T_CALC_CHECKSUM)
    param->calc_checksum=1;

  if (!(sort_param=(MI_SORT_PARAM *)
        my_malloc((uint) share->base.keys *
unknown's avatar
unknown committed
2506 2507
		  (sizeof(MI_SORT_PARAM) + share->base.pack_reclength),
		  MYF(MY_ZEROFILL))))
unknown's avatar
unknown committed
2508
  {
2509
    mi_check_print_error(param,"Not enough memory for key!");
unknown's avatar
unknown committed
2510 2511
    goto err;
  }
unknown's avatar
unknown committed
2512
  total_key_length=0;
unknown's avatar
unknown committed
2513 2514 2515 2516
  rec_per_key_part= param->rec_per_key_part;
  info->state->records=info->state->del=share->state.split=0;
  info->state->empty=0;

unknown's avatar
unknown committed
2517 2518
  for (i=key=0, istep=1 ; key < share->base.keys ;
       rec_per_key_part+=sort_param[i].keyinfo->keysegs, i+=istep, key++)
unknown's avatar
unknown committed
2519 2520 2521
  {
    sort_param[i].key=key;
    sort_param[i].keyinfo=share->keyinfo+key;
2522
    sort_param[i].seg=sort_param[i].keyinfo->seg;
unknown's avatar
unknown committed
2523 2524 2525 2526
    if (!(((ulonglong) 1 << key) & key_map))
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
2527 2528
	     (char*) (share->state.rec_per_key_part+
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
unknown's avatar
unknown committed
2529
	     sort_param[i].keyinfo->keysegs*sizeof(*rec_per_key_part));
unknown's avatar
unknown committed
2530
      istep=0;
unknown's avatar
unknown committed
2531 2532
      continue;
    }
unknown's avatar
unknown committed
2533
    istep=1;
unknown's avatar
unknown committed
2534 2535
    if ((!(param->testflag & T_SILENT)))
      printf ("- Fixing index %d\n",key+1);
2536 2537 2538 2539 2540 2541 2542 2543 2544 2545
    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
    {
      sort_param[i].key_read=sort_ft_key_read;
      sort_param[i].key_write=sort_ft_key_write;
    }
    else
    {
      sort_param[i].key_read=sort_key_read;
      sort_param[i].key_write=sort_key_write;
    }
unknown's avatar
unknown committed
2546 2547 2548 2549
    sort_param[i].key_cmp=sort_key_cmp;
    sort_param[i].lock_in_memory=lock_memory;
    sort_param[i].tmpdir=param->tmpdir;
    sort_param[i].sort_info=&sort_info;
2550
    sort_param[i].master=0;
unknown's avatar
unknown committed
2551 2552 2553 2554 2555
    sort_param[i].fix_datafile=0;

    sort_param[i].filepos=new_header_length;
    sort_param[i].max_pos=sort_param[i].pos=share->pack.header_length;

unknown's avatar
unknown committed
2556 2557
    sort_param[i].record= (((char *)(sort_param+share->base.keys))+
			   (share->base.pack_reclength * i));
unknown's avatar
unknown committed
2558 2559 2560 2561 2562
    if (!mi_alloc_rec_buff(info, -1, &sort_param[i].rec_buff))
    {
      mi_check_print_error(param,"Not enough memory!");
      goto err;
    }
unknown's avatar
unknown committed
2563 2564

    sort_param[i].key_length=share->rec_reflength;
2565
    for (keyseg=sort_param[i].seg; keyseg->type != HA_KEYTYPE_END;
unknown's avatar
unknown committed
2566
	 keyseg++)
unknown's avatar
unknown committed
2567 2568 2569 2570 2571 2572 2573 2574 2575
    {
      sort_param[i].key_length+=keyseg->length;
      if (keyseg->flag & HA_SPACE_PACK)
        sort_param[i].key_length+=get_pack_length(keyseg->length);
      if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH))
        sort_param[i].key_length+=2 + test(keyseg->length >= 127);
      if (keyseg->flag & HA_NULL_PART)
        sort_param[i].key_length++;
    }
unknown's avatar
unknown committed
2576
    total_key_length+=sort_param[i].key_length;
unknown's avatar
unknown committed
2577 2578

    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2579 2580 2581 2582 2583
    {
      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                    sort_param[i].keyinfo->seg->charset->mbmaxlen;
      sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
    }
unknown's avatar
unknown committed
2584 2585
  }
  sort_info.total_keys=i;
2586 2587
  sort_param[0].master= 1;
  sort_param[0].fix_datafile= (my_bool)(! rep_quick);
unknown's avatar
unknown committed
2588 2589

  sort_info.got_error=0;
unknown's avatar
unknown committed
2590 2591 2592
  pthread_mutex_init(&sort_info.mutex, MY_MUTEX_INIT_FAST);
  pthread_cond_init(&sort_info.cond, 0);
  pthread_mutex_lock(&sort_info.mutex);
unknown's avatar
unknown committed
2593

unknown's avatar
unknown committed
2594
  init_io_cache_share(&param->read_cache, &io_share, i);
2595 2596 2597
  (void) pthread_attr_init(&thr_attr);
  (void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED);

unknown's avatar
unknown committed
2598
  for (i=0 ; i < sort_info.total_keys ; i++)
unknown's avatar
unknown committed
2599 2600
  {
    sort_param[i].read_cache=param->read_cache;
unknown's avatar
unknown committed
2601 2602 2603 2604 2605 2606
    /*
      two approaches: the same amount of memory for each thread
      or the memory for the same number of keys for each thread...
      In the second one all the threads will fill their sort_buffers
      (and call write_keys) at the same time, putting more stress on i/o.
    */
unknown's avatar
unknown committed
2607
    sort_param[i].sortbuff_size=
unknown's avatar
unknown committed
2608
#ifndef USING_SECOND_APPROACH
unknown's avatar
unknown committed
2609 2610
      param->sort_buffer_length/sort_info.total_keys;
#else
unknown's avatar
unknown committed
2611
      param->sort_buffer_length*sort_param[i].key_length/total_key_length;
unknown's avatar
unknown committed
2612
#endif
2613 2614 2615
    if (pthread_create(&sort_param[i].thr, &thr_attr,
		       thr_find_all_keys,
		       (void *) (sort_param+i)))
unknown's avatar
unknown committed
2616 2617
    {
      mi_check_print_error(param,"Cannot start a repair thread");
unknown's avatar
unknown committed
2618
      remove_io_thread(&param->read_cache);
unknown's avatar
unknown committed
2619 2620 2621 2622 2623
      sort_info.got_error=1;
    }
    else
      sort_info.threads_running++;
  }
2624
  (void) pthread_attr_destroy(&thr_attr);
unknown's avatar
unknown committed
2625 2626 2627

  /* waiting for all threads to finish */
  while (sort_info.threads_running)
unknown's avatar
unknown committed
2628 2629
    pthread_cond_wait(&sort_info.cond, &sort_info.mutex);
  pthread_mutex_unlock(&sort_info.mutex);
unknown's avatar
unknown committed
2630

2631
  if ((got_error= thr_write_keys(sort_param)))
unknown's avatar
unknown committed
2632 2633 2634 2635
  {
    param->retry_repair=1;
    goto err;
  }
unknown's avatar
unknown committed
2636
  got_error=1;				/* Assume the following may go wrong */
unknown's avatar
unknown committed
2637

unknown's avatar
unknown committed
2638
  if (sort_param[0].fix_datafile)
unknown's avatar
unknown committed
2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650
  {
    if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
      goto err;
    if (param->testflag & T_SAFE_REPAIR)
    {
      /* Don't repair if we loosed more than one row */
      if (info->state->records+1 < start_records)
      {
        info->state->records=start_records;
        goto err;
      }
    }
unknown's avatar
unknown committed
2651 2652
    share->state.state.data_file_length= info->state->data_file_length=
      sort_param->filepos;
unknown's avatar
unknown committed
2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682
    /* Only whole records */
    share->state.version=(ulong) time((time_t*) 0);
    my_close(info->dfile,MYF(0));
    info->dfile=new_file;
    share->data_file_type=sort_info.new_data_file_type;
    share->pack.header_length=(ulong) new_header_length;
  }
  else
    info->state->data_file_length=sort_param->max_pos;

  if (rep_quick && del+sort_info.dupp != info->state->del)
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
    goto err;
  }

  if (rep_quick & T_FORCE_UNIQUENESS)
  {
    my_off_t skr=info->state->data_file_length+
      (share->options & HA_OPTION_COMPRESS_RECORD ?
       MEMMAP_EXTRA_MARGIN : 0);
#ifdef USE_RELOC
    if (share->data_file_type == STATIC_RECORD &&
	skr < share->base.reloc*share->base.min_pack_length)
      skr=share->base.reloc*share->base.min_pack_length;
#endif
    if (skr != sort_info.filelength && !info->s->base.raid_type)
2683
      if (my_chsize(info->dfile,skr,0,MYF(0)))
unknown's avatar
unknown committed
2684 2685 2686 2687 2688 2689 2690
	mi_check_print_warning(param,
			       "Can't change size of datafile,  error: %d",
			       my_errno);
  }
  if (param->testflag & T_CALC_CHECKSUM)
    share->state.checksum=param->glob_crc;

2691
  if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
unknown's avatar
unknown committed
2692
    mi_check_print_warning(param,
unknown's avatar
unknown committed
2693
			   "Can't change size of indexfile, error: %d", my_errno);
unknown's avatar
unknown committed
2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
    if (sort_info.dupp)
      mi_check_print_warning(param,
			     "%s records have been removed",
			     llstr(sort_info.dupp,llbuff));
  }
  got_error=0;

  if (&share->state.state != info->state)
2707
    memcpy(&share->state.state, info->state, sizeof(*info->state));
unknown's avatar
unknown committed
2708 2709

err:
unknown's avatar
unknown committed
2710
  got_error|= flush_blocks(param, share->key_cache, share->kfile);
unknown's avatar
unknown committed
2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744
  VOID(end_io_cache(&info->rec_cache));
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
      my_close(new_file,MYF(0));
      info->dfile=new_file= -1;
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
			    DATA_TMP_EXT, share->base.raid_chunks,
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
	  mi_open_datafile(info,share,-1))
	got_error=1;
    }
  }
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d when fixing table",my_errno);
    if (new_file >= 0)
    {
      VOID(my_close(new_file,MYF(0)));
      VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
			  MYF(MY_WME)));
      if (info->dfile == new_file)
	info->dfile= -1;
    }
    mi_mark_crashed_on_repair(info);
  }
  else if (key_map == share->state.key_map)
    share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
  share->state.changed|=STATE_NOT_SORTED_PAGES;

unknown's avatar
unknown committed
2745 2746
  pthread_cond_destroy (&sort_info.cond);
  pthread_mutex_destroy(&sort_info.mutex);
unknown's avatar
unknown committed
2747

2748
  my_free((gptr) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759
  my_free((gptr) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
  my_free((gptr) sort_param,MYF(MY_ALLOW_ZERO_PTR));
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
  VOID(end_io_cache(&param->read_cache));
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
  if (!got_error && (param->testflag & T_UNPACK))
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
  }
  DBUG_RETURN(got_error);
2760
#endif /* THREAD */
unknown's avatar
unknown committed
2761
}
unknown's avatar
unknown committed
2762 2763 2764

	/* Read next record and return next key */

unknown's avatar
unknown committed
2765
static int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
unknown's avatar
unknown committed
2766 2767
{
  int error;
unknown's avatar
unknown committed
2768 2769
  SORT_INFO *sort_info=sort_param->sort_info;
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
2770 2771
  DBUG_ENTER("sort_key_read");

unknown's avatar
unknown committed
2772
  if ((error=sort_get_next_record(sort_param)))
unknown's avatar
unknown committed
2773 2774 2775 2776
    DBUG_RETURN(error);
  if (info->state->records == sort_info->max_records)
  {
    mi_check_print_error(sort_info->param,
2777 2778
			 "Key %d - Found too many records; Can't continue",
                         sort_param->key+1);
unknown's avatar
unknown committed
2779 2780
    DBUG_RETURN(1);
  }
unknown's avatar
unknown committed
2781
  sort_param->real_key_length=
unknown's avatar
unknown committed
2782
    (info->s->rec_reflength+
unknown's avatar
unknown committed
2783 2784
     _mi_make_key(info, sort_param->key, (uchar*) key,
		  sort_param->record, sort_param->filepos));
unknown's avatar
unknown committed
2785 2786 2787 2788
#ifdef HAVE_purify
  bzero(key+sort_param->real_key_length,
	(sort_param->key_length-sort_param->real_key_length));
#endif
unknown's avatar
unknown committed
2789
  DBUG_RETURN(sort_write_record(sort_param));
unknown's avatar
unknown committed
2790 2791
} /* sort_key_read */

unknown's avatar
unknown committed
2792
static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
2793 2794
{
  int error;
unknown's avatar
unknown committed
2795 2796
  SORT_INFO *sort_info=sort_param->sort_info;
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
2797
  FT_WORD *wptr=0;
2798 2799
  DBUG_ENTER("sort_ft_key_read");

unknown's avatar
unknown committed
2800
  if (!sort_param->wordlist)
2801
  {
unknown's avatar
unknown committed
2802
    for (;;)
2803
    {
unknown's avatar
unknown committed
2804
      my_free((char*) wptr, MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2805
      if ((error=sort_get_next_record(sort_param)))
2806
        DBUG_RETURN(error);
unknown's avatar
unknown committed
2807
      if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record)))
2808
        DBUG_RETURN(1);
unknown's avatar
unknown committed
2809 2810
      if (wptr->pos)
        break;
unknown's avatar
unknown committed
2811
      error=sort_write_record(sort_param);
2812
    }
unknown's avatar
unknown committed
2813
    sort_param->wordptr=sort_param->wordlist=wptr;
2814 2815 2816 2817
  }
  else
  {
    error=0;
unknown's avatar
unknown committed
2818
    wptr=(FT_WORD*)(sort_param->wordptr);
2819 2820
  }

unknown's avatar
unknown committed
2821 2822 2823 2824 2825 2826 2827 2828
  sort_param->real_key_length=(info->s->rec_reflength+
			       _ft_make_key(info, sort_param->key,
					    key, wptr++, sort_param->filepos));
#ifdef HAVE_purify
  if (sort_param->key_length > sort_param->real_key_length)
    bzero(key+sort_param->real_key_length,
	  (sort_param->key_length-sort_param->real_key_length));
#endif
2829 2830
  if (!wptr->pos)
  {
unknown's avatar
unknown committed
2831 2832
    my_free((char*) sort_param->wordlist, MYF(0));
    sort_param->wordlist=0;
unknown's avatar
unknown committed
2833
    error=sort_write_record(sort_param);
2834 2835
  }
  else
unknown's avatar
unknown committed
2836
    sort_param->wordptr=(void*)wptr;
2837 2838 2839

  DBUG_RETURN(error);
} /* sort_ft_key_read */
unknown's avatar
unknown committed
2840

unknown's avatar
unknown committed
2841

unknown's avatar
unknown committed
2842 2843 2844
	/* Read next record from file using parameters in sort_info */
	/* Return -1 if end of file, 0 if ok and > 0 if error */

unknown's avatar
unknown committed
2845
static int sort_get_next_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
2846 2847 2848 2849 2850 2851
{
  int searching;
  uint found_record,b_type,left_length;
  my_off_t pos;
  byte *to;
  MI_BLOCK_INFO block_info;
unknown's avatar
unknown committed
2852
  SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
2853
  MI_CHECK *param=sort_info->param;
2854 2855
  MI_INFO *info=sort_info->info;
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
2856 2857 2858
  char llbuff[22],llbuff2[22];
  DBUG_ENTER("sort_get_next_record");

unknown's avatar
unknown committed
2859 2860 2861
  if (*killed_ptr(param))
    DBUG_RETURN(1);

unknown's avatar
unknown committed
2862 2863 2864 2865
  switch (share->data_file_type) {
  case STATIC_RECORD:
    for (;;)
    {
unknown's avatar
unknown committed
2866
      if (my_b_read(&sort_param->read_cache,sort_param->record,
unknown's avatar
unknown committed
2867
		    share->base.pack_reclength))
2868
      {
unknown's avatar
unknown committed
2869
	if (sort_param->read_cache.error)
2870
	  param->out_flag |= O_DATA_LOST;
unknown's avatar
unknown committed
2871 2872
        param->retry_repair=1;
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
2873
	DBUG_RETURN(-1);
2874
      }
unknown's avatar
unknown committed
2875 2876
      sort_param->start_recpos=sort_param->pos;
      if (!sort_param->fix_datafile)
unknown's avatar
unknown committed
2877
      {
unknown's avatar
unknown committed
2878
	sort_param->filepos=sort_param->pos;
2879 2880
        if (sort_param->master)
	  share->state.split++;
unknown's avatar
unknown committed
2881
      }
unknown's avatar
unknown committed
2882 2883
      sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
      if (*sort_param->record)
2884 2885
      {
	if (param->calc_checksum)
2886
	  param->glob_crc+= (info->checksum=
unknown's avatar
unknown committed
2887
			     mi_static_checksum(info,sort_param->record));
unknown's avatar
unknown committed
2888
	DBUG_RETURN(0);
2889
      }
2890
      if (!sort_param->fix_datafile && sort_param->master)
unknown's avatar
unknown committed
2891 2892 2893 2894 2895 2896 2897
      {
	info->state->del++;
	info->state->empty+=share->base.pack_reclength;
      }
    }
  case DYNAMIC_RECORD:
    LINT_INIT(to);
unknown's avatar
unknown committed
2898 2899
    pos=sort_param->pos;
    searching=(sort_param->fix_datafile && (param->testflag & T_EXTEND));
unknown's avatar
unknown committed
2900 2901 2902 2903 2904
    for (;;)
    {
      found_record=block_info.second_read= 0;
      left_length=1;
      if (searching)
unknown's avatar
unknown committed
2905
      {
unknown's avatar
unknown committed
2906
	pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
unknown's avatar
unknown committed
2907
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
2908
	sort_param->start_recpos=pos;
unknown's avatar
unknown committed
2909
      }
unknown's avatar
unknown committed
2910 2911
      do
      {
unknown's avatar
unknown committed
2912 2913
	if (pos > sort_param->max_pos)
	  sort_param->max_pos=pos;
unknown's avatar
unknown committed
2914 2915 2916 2917 2918
	if (pos & (MI_DYN_ALIGN_SIZE-1))
	{
	  if ((param->testflag & T_VERBOSE) || searching == 0)
	    mi_check_print_info(param,"Wrong aligned block at %s",
				llstr(pos,llbuff));
unknown's avatar
unknown committed
2919
	  if (searching)
unknown's avatar
unknown committed
2920 2921 2922 2923 2924
	    goto try_next;
	}
	if (found_record && pos == param->search_after_block)
	  mi_check_print_info(param,"Block: %s used by record at %s",
		     llstr(param->search_after_block,llbuff),
unknown's avatar
unknown committed
2925 2926 2927
		     llstr(sort_param->start_recpos,llbuff2));
	if (_mi_read_cache(&sort_param->read_cache,
                           (byte*) block_info.header,pos,
2928 2929 2930
			   MI_BLOCK_INFO_HEADER_LENGTH,
			   (! found_record ? READING_NEXT : 0) |
			   READING_HEADER))
unknown's avatar
unknown committed
2931 2932 2933 2934 2935
	{
	  if (found_record)
	  {
	    mi_check_print_info(param,
				"Can't read whole record at %s (errno: %d)",
unknown's avatar
unknown committed
2936
				llstr(sort_param->start_recpos,llbuff),errno);
unknown's avatar
unknown committed
2937 2938 2939 2940
	    goto try_next;
	  }
	  DBUG_RETURN(-1);
	}
unknown's avatar
unknown committed
2941
	if (searching && ! sort_param->fix_datafile)
unknown's avatar
unknown committed
2942 2943
	{
	  param->error_printed=1;
unknown's avatar
unknown committed
2944 2945
          param->retry_repair=1;
          param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
2946 2947
	  DBUG_RETURN(1);	/* Something wrong with data */
	}
2948 2949 2950
	b_type=_mi_get_block_info(&block_info,-1,pos);
	if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
	   ((b_type & BLOCK_FIRST) &&
unknown's avatar
unknown committed
2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971
	     (block_info.rec_len < (uint) share->base.min_pack_length ||
	      block_info.rec_len > (uint) share->base.max_pack_length)))
	{
	  uint i;
	  if (param->testflag & T_VERBOSE || searching == 0)
	    mi_check_print_info(param,
				"Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
		       block_info.header[0],block_info.header[1],
		       block_info.header[2],llstr(pos,llbuff));
	  if (found_record)
	    goto try_next;
	  block_info.second_read=0;
	  searching=1;
	  /* Search after block in read header string */
	  for (i=MI_DYN_ALIGN_SIZE ;
	       i < MI_BLOCK_INFO_HEADER_LENGTH ;
	       i+= MI_DYN_ALIGN_SIZE)
	    if (block_info.header[i] >= 1 &&
		block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
	      break;
	  pos+=(ulong) i;
unknown's avatar
unknown committed
2972
	  sort_param->start_recpos=pos;
unknown's avatar
unknown committed
2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006
	  continue;
	}
	if (b_type & BLOCK_DELETED)
	{
	  bool error=0;
	  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
	      share->base.min_block_length)
	  {
	    if (!searching)
	      mi_check_print_info(param,
				  "Deleted block with impossible length %u at %s",
				  block_info.block_len,llstr(pos,llbuff));
	    error=1;
	  }
	  else
	  {
	    if ((block_info.next_filepos != HA_OFFSET_ERROR &&
		 block_info.next_filepos >=
		 info->state->data_file_length) ||
		(block_info.prev_filepos != HA_OFFSET_ERROR &&
		 block_info.prev_filepos >= info->state->data_file_length))
	    {
	      if (!searching)
		mi_check_print_info(param,
				    "Delete link points outside datafile at %s",
				    llstr(pos,llbuff));
	      error=1;
	    }
	  }
	  if (error)
	  {
	    if (found_record)
	      goto try_next;
	    searching=1;
3007
	    pos+= MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3008
	    sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027
	    block_info.second_read=0;
	    continue;
	  }
	}
	else
	{
	  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
	      share->base.min_block_length ||
	      block_info.block_len > (uint) share->base.max_pack_length+
	      MI_SPLIT_LENGTH)
	  {
	    if (!searching)
	      mi_check_print_info(param,
				  "Found block with impossible length %u at %s; Skipped",
				  block_info.block_len+ (uint) (block_info.filepos-pos),
				  llstr(pos,llbuff));
	    if (found_record)
	      goto try_next;
	    searching=1;
3028
	    pos+= MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3029
	    sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3030 3031 3032 3033 3034 3035
	    block_info.second_read=0;
	    continue;
	  }
	}
	if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
	{
3036 3037
          if (!sort_param->fix_datafile && sort_param->master &&
              (b_type & BLOCK_DELETED))
unknown's avatar
unknown committed
3038 3039 3040 3041 3042 3043 3044 3045
	  {
	    info->state->empty+=block_info.block_len;
	    info->state->del++;
	    share->state.split++;
	  }
	  if (found_record)
	    goto try_next;
	  if (searching)
3046 3047
	  {
	    pos+=MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3048
	    sort_param->start_recpos=pos;
3049
	  }
unknown's avatar
unknown committed
3050 3051 3052 3053 3054 3055
	  else
	    pos=block_info.filepos+block_info.block_len;
	  block_info.second_read=0;
	  continue;
	}

3056
	if (!sort_param->fix_datafile && sort_param->master)
unknown's avatar
unknown committed
3057
	  share->state.split++;
unknown's avatar
unknown committed
3058 3059
	if (! found_record++)
	{
unknown's avatar
unknown committed
3060 3061 3062 3063 3064 3065
	  sort_param->find_length=left_length=block_info.rec_len;
	  sort_param->start_recpos=pos;
	  if (!sort_param->fix_datafile)
	    sort_param->filepos=sort_param->start_recpos;
	  if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
	    sort_param->pos=block_info.filepos+1;
unknown's avatar
unknown committed
3066
	  else
unknown's avatar
unknown committed
3067
	    sort_param->pos=block_info.filepos+block_info.block_len;
unknown's avatar
unknown committed
3068 3069
	  if (share->base.blobs)
	  {
3070
	    if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
3071
				       &(sort_param->rec_buff))))
unknown's avatar
unknown committed
3072
	    {
3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086
	      if (param->max_record_length >= block_info.rec_len)
	      {
		mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
				     llstr(sort_param->start_recpos,llbuff),
				     (ulong) block_info.rec_len);
		DBUG_RETURN(1);
	      }
	      else
	      {
		mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
				    llstr(sort_param->start_recpos,llbuff),
				    (ulong) block_info.rec_len);
		goto try_next;
	      }
unknown's avatar
unknown committed
3087 3088 3089
	    }
	  }
	  else
3090
	    to= sort_param->rec_buff;
unknown's avatar
unknown committed
3091 3092 3093
	}
	if (left_length < block_info.data_len || ! block_info.data_len)
	{
3094 3095
	  mi_check_print_info(param,
			      "Found block with too small length at %s; Skipped",
unknown's avatar
unknown committed
3096
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3097 3098 3099
	  goto try_next;
	}
	if (block_info.filepos + block_info.data_len >
unknown's avatar
unknown committed
3100
	    sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3101
	{
3102 3103
	  mi_check_print_info(param,
			      "Found block that points outside data file at %s",
unknown's avatar
unknown committed
3104
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3105 3106
	  goto try_next;
	}
unknown's avatar
unknown committed
3107
	if (_mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
3108 3109
			   block_info.data_len,
			   (found_record == 1 ? READING_NEXT : 0)))
unknown's avatar
unknown committed
3110
	{
3111 3112
	  mi_check_print_info(param,
			      "Read error for block at: %s (error: %d); Skipped",
3113
			      llstr(block_info.filepos,llbuff),my_errno);
unknown's avatar
unknown committed
3114 3115 3116 3117 3118 3119 3120 3121
	  goto try_next;
	}
	left_length-=block_info.data_len;
	to+=block_info.data_len;
	pos=block_info.next_filepos;
	if (pos == HA_OFFSET_ERROR && left_length)
	{
	  mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
unknown's avatar
unknown committed
3122
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3123 3124
	  goto try_next;
	}
unknown's avatar
unknown committed
3125
	if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3126 3127
	{
	  mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
3128
			      llstr(pos,llbuff2),
unknown's avatar
unknown committed
3129
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3130 3131 3132 3133
	  goto try_next;
	}
      } while (left_length);

3134
      if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
unknown's avatar
unknown committed
3135
			 sort_param->find_length) != MY_FILE_ERROR)
unknown's avatar
unknown committed
3136
      {
unknown's avatar
unknown committed
3137
	if (sort_param->read_cache.error < 0)
unknown's avatar
unknown committed
3138
	  DBUG_RETURN(1);
3139
	if (info->s->calc_checksum)
unknown's avatar
unknown committed
3140
	  info->checksum=mi_checksum(info,sort_param->record);
unknown's avatar
unknown committed
3141 3142
	if ((param->testflag & (T_EXTEND | T_REP)) || searching)
	{
unknown's avatar
unknown committed
3143
	  if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
3144 3145 3146
                            sort_param->find_length,
                            (param->testflag & T_QUICK) &&
                            test(info->s->calc_checksum)))
unknown's avatar
unknown committed
3147 3148
	  {
	    mi_check_print_info(param,"Found wrong packed record at %s",
unknown's avatar
unknown committed
3149
				llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3150 3151 3152
	    goto try_next;
	  }
	}
3153 3154
	if (param->calc_checksum)
	  param->glob_crc+= info->checksum;
unknown's avatar
unknown committed
3155 3156
	DBUG_RETURN(0);
      }
unknown's avatar
unknown committed
3157
      if (!searching)
3158 3159 3160
        mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
                            sort_param->key+1,
                            llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3161
    try_next:
unknown's avatar
unknown committed
3162
      pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
unknown's avatar
unknown committed
3163 3164 3165
      searching=1;
    }
  case COMPRESSED_RECORD:
unknown's avatar
unknown committed
3166
    for (searching=0 ;; searching=1, sort_param->pos++)
unknown's avatar
unknown committed
3167
    {
unknown's avatar
unknown committed
3168 3169
      if (_mi_read_cache(&sort_param->read_cache,(byte*) block_info.header,
			 sort_param->pos,
3170
			 share->pack.ref_length,READING_NEXT))
unknown's avatar
unknown committed
3171
	DBUG_RETURN(-1);
unknown's avatar
unknown committed
3172
      if (searching && ! sort_param->fix_datafile)
unknown's avatar
unknown committed
3173 3174
      {
	param->error_printed=1;
unknown's avatar
unknown committed
3175 3176
        param->retry_repair=1;
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3177 3178
	DBUG_RETURN(1);		/* Something wrong with data */
      }
unknown's avatar
unknown committed
3179
      sort_param->start_recpos=sort_param->pos;
3180
      if (_mi_pack_get_block_info(info,&block_info,-1,sort_param->pos))
unknown's avatar
unknown committed
3181 3182
	DBUG_RETURN(-1);
      if (!block_info.rec_len &&
unknown's avatar
unknown committed
3183 3184
	  sort_param->pos + MEMMAP_EXTRA_MARGIN ==
	  sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3185 3186 3187 3188 3189 3190
	DBUG_RETURN(-1);
      if (block_info.rec_len < (uint) share->min_pack_length ||
	  block_info.rec_len > (uint) share->max_pack_length)
      {
	if (! searching)
	  mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n",
3191
			      block_info.rec_len,
unknown's avatar
unknown committed
3192
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3193 3194
	continue;
      }
3195
      if (_mi_read_cache(&sort_param->read_cache,(byte*) sort_param->rec_buff,
3196 3197
			 block_info.filepos, block_info.rec_len,
			 READING_NEXT))
unknown's avatar
unknown committed
3198 3199
      {
	if (! searching)
3200
	  mi_check_print_info(param,"Couldn't read whole record from %s",
unknown's avatar
unknown committed
3201
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3202 3203
	continue;
      }
3204
      if (_mi_pack_rec_unpack(info,sort_param->record,sort_param->rec_buff,
unknown's avatar
unknown committed
3205 3206 3207
			      block_info.rec_len))
      {
	if (! searching)
3208
	  mi_check_print_info(param,"Found wrong record at %s",
unknown's avatar
unknown committed
3209
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3210 3211
	continue;
      }
unknown's avatar
unknown committed
3212 3213
      info->checksum=mi_checksum(info,sort_param->record);
      if (!sort_param->fix_datafile)
unknown's avatar
unknown committed
3214
      {
unknown's avatar
unknown committed
3215
	sort_param->filepos=sort_param->pos;
3216 3217
        if (sort_param->master)
	  share->state.split++;
unknown's avatar
unknown committed
3218
      }
unknown's avatar
unknown committed
3219
      sort_param->max_pos=(sort_param->pos=block_info.filepos+
unknown's avatar
unknown committed
3220 3221
			 block_info.rec_len);
      info->packed_length=block_info.rec_len;
3222 3223
      if (param->calc_checksum)
	param->glob_crc+= info->checksum;
unknown's avatar
unknown committed
3224 3225 3226 3227 3228 3229 3230 3231 3232
      DBUG_RETURN(0);
    }
  }
  DBUG_RETURN(1);		/* Impossible */
}


	/* Write record to new file */

unknown's avatar
unknown committed
3233
int sort_write_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
3234 3235 3236 3237 3238 3239
{
  int flag;
  uint length;
  ulong block_length,reclength;
  byte *from;
  byte block_buff[8];
unknown's avatar
unknown committed
3240
  SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
3241
  MI_CHECK *param=sort_info->param;
unknown's avatar
unknown committed
3242 3243
  MI_INFO *info=sort_info->info;
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
3244 3245
  DBUG_ENTER("sort_write_record");

unknown's avatar
unknown committed
3246
  if (sort_param->fix_datafile)
unknown's avatar
unknown committed
3247 3248 3249
  {
    switch (sort_info->new_data_file_type) {
    case STATIC_RECORD:
unknown's avatar
unknown committed
3250
      if (my_b_write(&info->rec_cache,sort_param->record,
unknown's avatar
unknown committed
3251 3252 3253 3254 3255
		     share->base.pack_reclength))
      {
	mi_check_print_error(param,"%d when writing to datafile",my_errno);
	DBUG_RETURN(1);
      }
unknown's avatar
unknown committed
3256
      sort_param->filepos+=share->base.pack_reclength;
unknown's avatar
unknown committed
3257
      info->s->state.split++;
unknown's avatar
unknown committed
3258
      /* sort_info->param->glob_crc+=mi_static_checksum(info, sort_param->record); */
unknown's avatar
unknown committed
3259 3260 3261
      break;
    case DYNAMIC_RECORD:
      if (! info->blobs)
3262
	from=sort_param->rec_buff;
unknown's avatar
unknown committed
3263 3264 3265 3266
      else
      {
	/* must be sure that local buffer is big enough */
	reclength=info->s->base.pack_reclength+
unknown's avatar
unknown committed
3267
	  _my_calc_total_blob_length(info,sort_param->record)+
unknown's avatar
unknown committed
3268 3269 3270 3271 3272
	  ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
	  MI_DYN_DELETE_BLOCK_HEADER;
	if (sort_info->buff_length < reclength)
	{
	  if (!(sort_info->buff=my_realloc(sort_info->buff, (uint) reclength,
3273 3274
					   MYF(MY_FREE_ON_ERROR |
					       MY_ALLOW_ZERO_PTR))))
unknown's avatar
unknown committed
3275 3276 3277 3278 3279
	    DBUG_RETURN(1);
	  sort_info->buff_length=reclength;
	}
	from=sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
      }
unknown's avatar
unknown committed
3280 3281
      info->checksum=mi_checksum(info,sort_param->record);
      reclength=_mi_rec_pack(info,from,sort_param->record);
unknown's avatar
unknown committed
3282
      flag=0;
unknown's avatar
unknown committed
3283 3284 3285
      /* sort_info->param->glob_crc+=info->checksum; */

      do
unknown's avatar
unknown committed
3286
      {
unknown's avatar
unknown committed
3287 3288 3289 3290 3291 3292 3293 3294
	block_length=reclength+ 3 + test(reclength >= (65520-3));
	if (block_length < share->base.min_block_length)
	  block_length=share->base.min_block_length;
	info->update|=HA_STATE_WRITE_AT_END;
	block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
	if (block_length > MI_MAX_BLOCK_LENGTH)
	  block_length=MI_MAX_BLOCK_LENGTH;
	if (_mi_write_part_record(info,0L,block_length,
unknown's avatar
unknown committed
3295
				  sort_param->filepos+block_length,
unknown's avatar
unknown committed
3296 3297 3298 3299 3300
				  &from,&reclength,&flag))
	{
	  mi_check_print_error(param,"%d when writing to datafile",my_errno);
	  DBUG_RETURN(1);
	}
unknown's avatar
unknown committed
3301
	sort_param->filepos+=block_length;
unknown's avatar
unknown committed
3302 3303
	info->s->state.split++;
      } while (reclength);
unknown's avatar
unknown committed
3304
      /* sort_info->param->glob_crc+=info->checksum; */
unknown's avatar
unknown committed
3305 3306 3307
      break;
    case COMPRESSED_RECORD:
      reclength=info->packed_length;
3308 3309
      length= save_pack_length((uint) share->pack.version, block_buff,
                               reclength);
unknown's avatar
unknown committed
3310
      if (info->s->base.blobs)
3311 3312
	length+= save_pack_length((uint) share->pack.version,
	                          block_buff + length, info->blob_length);
unknown's avatar
unknown committed
3313
      if (my_b_write(&info->rec_cache,block_buff,length) ||
3314
	  my_b_write(&info->rec_cache,(byte*) sort_param->rec_buff,reclength))
unknown's avatar
unknown committed
3315 3316 3317 3318
      {
	mi_check_print_error(param,"%d when writing to datafile",my_errno);
	DBUG_RETURN(1);
      }
unknown's avatar
unknown committed
3319
      /* sort_info->param->glob_crc+=info->checksum; */
unknown's avatar
unknown committed
3320
      sort_param->filepos+=reclength+length;
unknown's avatar
unknown committed
3321
      info->s->state.split++;
unknown's avatar
unknown committed
3322 3323 3324
      break;
    }
  }
3325
  if (sort_param->master)
unknown's avatar
unknown committed
3326
  {
3327 3328 3329 3330 3331
    info->state->records++;
    if ((param->testflag & T_WRITE_LOOP) &&
        (info->state->records % WRITE_COUNT) == 0)
    {
      char llbuff[22];
3332 3333
      printf("%s\r", llstr(info->state->records,llbuff));
      VOID(fflush(stdout));
3334
    }
unknown's avatar
unknown committed
3335 3336 3337 3338 3339 3340 3341
  }
  DBUG_RETURN(0);
} /* sort_write_record */


	/* Compare two keys from _create_index_by_sort */

unknown's avatar
unknown committed
3342 3343
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,
			const void *b)
unknown's avatar
unknown committed
3344
{
3345
  uint not_used[2];
3346
  return (ha_key_cmp(sort_param->seg, *((uchar**) a), *((uchar**) b),
3347
		     USE_WHOLE_KEY, SEARCH_SAME, not_used));
unknown's avatar
unknown committed
3348 3349 3350
} /* sort_key_cmp */


unknown's avatar
unknown committed
3351
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
unknown's avatar
unknown committed
3352
{
3353
  uint diff_pos[2];
unknown's avatar
unknown committed
3354
  char llbuff[22],llbuff2[22];
unknown's avatar
unknown committed
3355
  SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
3356 3357 3358 3359 3360
  MI_CHECK *param= sort_info->param;
  int cmp;

  if (sort_info->key_block->inited)
  {
3361
    cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
unknown's avatar
unknown committed
3362
		   (uchar*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
3363
		   diff_pos);
3364 3365 3366
    if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
      ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
                 (uchar*) a, USE_WHOLE_KEY, 
3367
                 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
3368 3369
    else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
    {
3370 3371 3372 3373
      diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
                                                 sort_param->notnull,
                                                 sort_info->key_block->lastkey,
                                                 (uchar*)a);
3374
    }
3375
    sort_param->unique[diff_pos[0]-1]++;
unknown's avatar
unknown committed
3376 3377 3378
  }
  else
  {
3379
    cmp= -1;
3380 3381 3382
    if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
      mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
                                     (uchar*)a);
unknown's avatar
unknown committed
3383
  }
unknown's avatar
unknown committed
3384
  if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
unknown's avatar
unknown committed
3385 3386 3387
  {
    sort_info->dupp++;
    sort_info->info->lastpos=get_record_for_key(sort_info->info,
3388
						sort_param->keyinfo,
unknown's avatar
unknown committed
3389
						(uchar*) a);
unknown's avatar
unknown committed
3390 3391 3392 3393
    mi_check_print_warning(param,
			   "Duplicate key for record at %10s against record at %10s",
			   llstr(sort_info->info->lastpos,llbuff),
			   llstr(get_record_for_key(sort_info->info,
unknown's avatar
unknown committed
3394
						    sort_param->keyinfo,
unknown's avatar
unknown committed
3395 3396 3397
						    sort_info->key_block->
						    lastkey),
				 llbuff2));
unknown's avatar
unknown committed
3398
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3399
    if (sort_info->param->testflag & T_VERBOSE)
3400
      _mi_print_key(stdout,sort_param->seg,(uchar*) a, USE_WHOLE_KEY);
unknown's avatar
unknown committed
3401
    return (sort_delete_record(sort_param));
unknown's avatar
unknown committed
3402 3403 3404 3405 3406 3407 3408 3409 3410
  }
#ifndef DBUG_OFF
  if (cmp > 0)
  {
    mi_check_print_error(param,
			 "Internal error: Keys are not in order from sort");
    return(1);
  }
#endif
unknown's avatar
unknown committed
3411
  return (sort_insert_key(sort_param,sort_info->key_block,
unknown's avatar
unknown committed
3412
			  (uchar*) a, HA_OFFSET_ERROR));
unknown's avatar
unknown committed
3413 3414
} /* sort_key_write */

3415 3416 3417 3418 3419
int sort_ft_buf_flush(MI_SORT_PARAM *sort_param)
{
  SORT_INFO *sort_info=sort_param->sort_info;
  SORT_KEY_BLOCKS *key_block=sort_info->key_block;
  MYISAM_SHARE *share=sort_info->info->s;
unknown's avatar
unknown committed
3420 3421
  uint val_off, val_len;
  int error;
3422 3423 3424 3425 3426 3427 3428 3429
  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
  uchar *from, *to;

  val_len=share->ft2_keyinfo.keylength;
  get_key_full_length_rdonly(val_off, ft_buf->lastkey);
  to=ft_buf->lastkey+val_off;

  if (ft_buf->buf)
unknown's avatar
unknown committed
3430 3431 3432 3433
  {
    /* flushing first-level tree */
    error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
			  HA_OFFSET_ERROR);
3434 3435 3436 3437 3438
    for (from=to+val_len;
         !error && from < ft_buf->buf;
         from+= val_len)
    {
      memcpy(to, from, val_len);
unknown's avatar
unknown committed
3439 3440
      error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
			    HA_OFFSET_ERROR);
3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493
    }
    return error;
  }
  /* flushing second-level tree keyblocks */
  error=flush_pending_blocks(sort_param);
  /* updating lastkey with second-level tree info */
  ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count);
  _mi_dpointer(sort_info->info, ft_buf->lastkey+val_off+HA_FT_WLEN,
      share->state.key_root[sort_param->key]);
  /* restoring first level tree data in sort_info/sort_param */
  sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks;
  sort_param->keyinfo=share->keyinfo+sort_param->key;
  share->state.key_root[sort_param->key]=HA_OFFSET_ERROR;
  /* writing lastkey in first-level tree */
  return error ? error :
                 sort_insert_key(sort_param,sort_info->key_block,
                                 ft_buf->lastkey,HA_OFFSET_ERROR);
}

static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a)
{
  uint a_len, val_off, val_len, error;
  uchar *p;
  SORT_INFO *sort_info=sort_param->sort_info;
  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
  SORT_KEY_BLOCKS *key_block=sort_info->key_block;

  val_len=HA_FT_WLEN+sort_info->info->s->base.rec_reflength;
  get_key_full_length_rdonly(a_len, (uchar *)a);

  if (!ft_buf)
  {
    /*
      use two-level tree only if key_reflength fits in rec_reflength place
      and row format is NOT static - for _mi_dpointer not to garble offsets
     */
    if ((sort_info->info->s->base.key_reflength <=
         sort_info->info->s->base.rec_reflength) &&
        (sort_info->info->s->options &
          (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
      ft_buf=(SORT_FT_BUF *)my_malloc(sort_param->keyinfo->block_length +
                                      sizeof(SORT_FT_BUF), MYF(MY_WME));

    if (!ft_buf)
    {
      sort_param->key_write=sort_key_write;
      return sort_key_write(sort_param, a);
    }
    sort_info->ft_buf=ft_buf;
    goto word_init_ft_buf; /* no need to duplicate the code */
  }
  get_key_full_length_rdonly(val_off, ft_buf->lastkey);

3494
  if (mi_compare_text(sort_param->seg->charset,
3495
                      ((uchar *)a)+1,a_len-1,
3496
                      ft_buf->lastkey+1,val_off-1, 0, 0)==0)
3497 3498 3499 3500 3501
  {
    if (!ft_buf->buf) /* store in second-level tree */
    {
      ft_buf->count++;
      return sort_insert_key(sort_param,key_block,
3502
                             ((uchar *)a)+a_len, HA_OFFSET_ERROR);
3503 3504 3505
    }

    /* storing the key in the buffer. */
3506
    memcpy (ft_buf->buf, (char *)a+a_len, val_len);
3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526
    ft_buf->buf+=val_len;
    if (ft_buf->buf < ft_buf->end)
      return 0;

    /* converting to two-level tree */
    p=ft_buf->lastkey+val_off;

    while (key_block->inited)
      key_block++;
    sort_info->key_block=key_block;
    sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo;
    ft_buf->count=(ft_buf->buf - p)/val_len;

    /* flushing buffer to second-level tree */
    for (error=0; !error && p < ft_buf->buf; p+= val_len)
      error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR);
    ft_buf->buf=0;
    return error;
  }

3527 3528 3529
  /* flushing buffer */
  if ((error=sort_ft_buf_flush(sort_param)))
    return error;
3530

3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544
word_init_ft_buf:
  a_len+=val_len;
  memcpy(ft_buf->lastkey, a, a_len);
  ft_buf->buf=ft_buf->lastkey+a_len;
  /*
    32 is just a safety margin here
    (at least max(val_len, sizeof(nod_flag)) should be there).
    May be better performance could be achieved if we'd put
      (sort_info->keyinfo->block_length-32)/XXX
      instead.
        TODO: benchmark the best value for XXX.
  */
  ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32);
  return 0;
3545
} /* sort_ft_key_write */
unknown's avatar
unknown committed
3546

3547

unknown's avatar
unknown committed
3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558
	/* get pointer to record from a key */

static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
				   uchar *key)
{
  return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
} /* get_record_for_key */


	/* Insert a key in sort-key-blocks */

unknown's avatar
unknown committed
3559
static int sort_insert_key(MI_SORT_PARAM *sort_param,
unknown's avatar
unknown committed
3560 3561 3562 3563
			   register SORT_KEY_BLOCKS *key_block, uchar *key,
			   my_off_t prev_block)
{
  uint a_length,t_length,nod_flag;
3564
  my_off_t filepos,key_file_length;
unknown's avatar
unknown committed
3565 3566 3567
  uchar *anc_buff,*lastkey;
  MI_KEY_PARAM s_temp;
  MI_INFO *info;
unknown's avatar
unknown committed
3568 3569 3570
  MI_KEYDEF *keyinfo=sort_param->keyinfo;
  SORT_INFO *sort_info= sort_param->sort_info;
  MI_CHECK *param=sort_info->param;
unknown's avatar
unknown committed
3571 3572 3573 3574 3575 3576
  DBUG_ENTER("sort_insert_key");

  anc_buff=key_block->buff;
  info=sort_info->info;
  lastkey=key_block->lastkey;
  nod_flag= (key_block == sort_info->key_block ? 0 :
unknown's avatar
unknown committed
3577
	     info->s->base.key_reflength);
unknown's avatar
unknown committed
3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597

  if (!key_block->inited)
  {
    key_block->inited=1;
    if (key_block == sort_info->key_block_end)
    {
      mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
      DBUG_RETURN(1);
    }
    a_length=2+nod_flag;
    key_block->end_pos=anc_buff+2;
    lastkey=0;					/* No previous key in block */
  }
  else
    a_length=mi_getint(anc_buff);

	/* Save pointer to previous block */
  if (nod_flag)
    _mi_kpointer(info,key_block->end_pos,prev_block);

unknown's avatar
unknown committed
3598 3599 3600 3601
  t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
				(uchar*) 0,lastkey,lastkey,key,
				 &s_temp);
  (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
unknown's avatar
unknown committed
3602 3603 3604
  a_length+=t_length;
  mi_putint(anc_buff,a_length,nod_flag);
  key_block->end_pos+=t_length;
unknown's avatar
unknown committed
3605
  if (a_length <= keyinfo->block_length)
unknown's avatar
unknown committed
3606
  {
unknown's avatar
unknown committed
3607
    VOID(_mi_move_key(keyinfo,key_block->lastkey,key));
unknown's avatar
unknown committed
3608 3609 3610 3611 3612 3613 3614
    key_block->last_length=a_length-t_length;
    DBUG_RETURN(0);
  }

	/* Fill block with end-zero and write filled block */
  mi_putint(anc_buff,key_block->last_length,nod_flag);
  bzero((byte*) anc_buff+key_block->last_length,
unknown's avatar
unknown committed
3615
	keyinfo->block_length- key_block->last_length);
3616
  key_file_length=info->state->key_file_length;
3617
  if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
3618 3619 3620 3621 3622
    DBUG_RETURN(1);

  /* If we read the page from the key cache, we have to write it back to it */
  if (key_file_length == info->state->key_file_length)
  {
3623
    if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
3624 3625 3626
      DBUG_RETURN(1);
  }
  else if (my_pwrite(info->s->kfile,(byte*) anc_buff,
unknown's avatar
unknown committed
3627
		     (uint) keyinfo->block_length,filepos, param->myf_rw))
unknown's avatar
unknown committed
3628 3629 3630 3631
    DBUG_RETURN(1);
  DBUG_DUMP("buff",(byte*) anc_buff,mi_getint(anc_buff));

	/* Write separator-key to block in next level */
unknown's avatar
unknown committed
3632
  if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
unknown's avatar
unknown committed
3633 3634 3635 3636
    DBUG_RETURN(1);

	/* clear old block and write new key in it */
  key_block->inited=0;
unknown's avatar
unknown committed
3637
  DBUG_RETURN(sort_insert_key(sort_param, key_block,key,prev_block));
unknown's avatar
unknown committed
3638 3639 3640 3641 3642
} /* sort_insert_key */


	/* Delete record when we found a duplicated key */

unknown's avatar
unknown committed
3643
static int sort_delete_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
3644 3645 3646 3647
{
  uint i;
  int old_file,error;
  uchar *key;
unknown's avatar
unknown committed
3648 3649 3650
  SORT_INFO *sort_info=sort_param->sort_info;
  MI_CHECK *param=sort_info->param;
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
3651 3652
  DBUG_ENTER("sort_delete_record");

3653
  if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
unknown's avatar
unknown committed
3654 3655
  {
    mi_check_print_error(param,
unknown's avatar
unknown committed
3656
			 "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
unknown's avatar
unknown committed
3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667
    DBUG_RETURN(1);
  }
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
  {
    mi_check_print_error(param,
			 "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
    DBUG_RETURN(1);
  }

  old_file=info->dfile;
  info->dfile=info->rec_cache.file;
unknown's avatar
unknown committed
3668
  if (sort_info->current_key)
unknown's avatar
unknown committed
3669 3670
  {
    key=info->lastkey+info->s->base.max_key_length;
unknown's avatar
unknown committed
3671
    if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
unknown's avatar
unknown committed
3672 3673 3674 3675 3676 3677 3678
	error != HA_ERR_RECORD_DELETED)
    {
      mi_check_print_error(param,"Can't read record to be removed");
      info->dfile=old_file;
      DBUG_RETURN(1);
    }

unknown's avatar
unknown committed
3679
    for (i=0 ; i < sort_info->current_key ; i++)
unknown's avatar
unknown committed
3680
    {
unknown's avatar
unknown committed
3681
      uint key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
unknown's avatar
unknown committed
3682 3683 3684 3685 3686 3687 3688
      if (_mi_ck_delete(info,i,key,key_length))
      {
	mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
	info->dfile=old_file;
	DBUG_RETURN(1);
      }
    }
unknown's avatar
unknown committed
3689
    if (param->calc_checksum)
unknown's avatar
unknown committed
3690
      param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
unknown's avatar
unknown committed
3691 3692 3693 3694 3695 3696 3697 3698 3699
  }
  error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
  info->dfile=old_file;				/* restore actual value */
  info->state->records--;
  DBUG_RETURN(error);
} /* sort_delete_record */

	/* Fix all pending blocks and flush everything to disk */

unknown's avatar
unknown committed
3700
int flush_pending_blocks(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
3701 3702
{
  uint nod_flag,length;
3703
  my_off_t filepos,key_file_length;
unknown's avatar
unknown committed
3704
  SORT_KEY_BLOCKS *key_block;
unknown's avatar
unknown committed
3705
  SORT_INFO *sort_info= sort_param->sort_info;
3706
  myf myf_rw=sort_info->param->myf_rw;
unknown's avatar
unknown committed
3707 3708
  MI_INFO *info=sort_info->info;
  MI_KEYDEF *keyinfo=sort_param->keyinfo;
unknown's avatar
unknown committed
3709 3710 3711 3712 3713 3714 3715 3716 3717 3718
  DBUG_ENTER("flush_pending_blocks");

  filepos= HA_OFFSET_ERROR;			/* if empty file */
  nod_flag=0;
  for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
  {
    key_block->inited=0;
    length=mi_getint(key_block->buff);
    if (nod_flag)
      _mi_kpointer(info,key_block->end_pos,filepos);
3719
    key_file_length=info->state->key_file_length;
unknown's avatar
unknown committed
3720
    bzero((byte*) key_block->buff+length, keyinfo->block_length-length);
3721
    if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
3722 3723 3724 3725 3726
      DBUG_RETURN(1);

    /* If we read the page from the key cache, we have to write it back */
    if (key_file_length == info->state->key_file_length)
    {
3727 3728
      if (_mi_write_keypage(info, keyinfo, filepos,
                            DFLT_INIT_HITS, key_block->buff))
3729 3730 3731
	DBUG_RETURN(1);
    }
    else if (my_pwrite(info->s->kfile,(byte*) key_block->buff,
3732
		       (uint) keyinfo->block_length,filepos, myf_rw))
unknown's avatar
unknown committed
3733 3734 3735 3736
      DBUG_RETURN(1);
    DBUG_DUMP("buff",(byte*) key_block->buff,length);
    nod_flag=1;
  }
unknown's avatar
unknown committed
3737
  info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
unknown's avatar
unknown committed
3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753
  DBUG_RETURN(0);
} /* flush_pending_blocks */

	/* alloc space and pointers for key_blocks */

static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks,
                                         uint buffer_length)
{
  reg1 uint i;
  SORT_KEY_BLOCKS *block;
  DBUG_ENTER("alloc_key_blocks");

  if (!(block=(SORT_KEY_BLOCKS*) my_malloc((sizeof(SORT_KEY_BLOCKS)+
					    buffer_length+IO_SIZE)*blocks,
					   MYF(0))))
  {
3754
    mi_check_print_error(param,"Not enough memory for sort-key-blocks");
unknown's avatar
unknown committed
3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785
    return(0);
  }
  for (i=0 ; i < blocks ; i++)
  {
    block[i].inited=0;
    block[i].buff=(uchar*) (block+blocks)+(buffer_length+IO_SIZE)*i;
  }
  DBUG_RETURN(block);
} /* alloc_key_blocks */


	/* Check if file is almost full */

int test_if_almost_full(MI_INFO *info)
{
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
    return 0;
  return (my_seek(info->s->kfile,0L,MY_SEEK_END,MYF(0))/10*9 >
	  (my_off_t) (info->s->base.max_key_file_length) ||
	  my_seek(info->dfile,0L,MY_SEEK_END,MYF(0))/10*9 >
	  (my_off_t) info->s->base.max_data_file_length);
}

	/* Recreate table with bigger more alloced record-data */

int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename)
{
  int error;
  MI_INFO info;
  MYISAM_SHARE share;
  MI_KEYDEF *keyinfo,*key,*key_end;
unknown's avatar
unknown committed
3786
  HA_KEYSEG *keysegs,*keyseg;
unknown's avatar
unknown committed
3787 3788 3789 3790 3791 3792 3793
  MI_COLUMNDEF *recdef,*rec,*end;
  MI_UNIQUEDEF *uniquedef,*u_ptr,*u_end;
  MI_STATUS_INFO status_info;
  uint unpack,key_parts;
  ha_rows max_records;
  ulonglong file_length,tmp_length;
  MI_CREATE_INFO create_info;
3794
  DBUG_ENTER("recreate_table");
unknown's avatar
unknown committed
3795 3796 3797 3798 3799 3800 3801 3802 3803

  error=1;					/* Default error */
  info= **org_info;
  status_info= (*org_info)->state[0];
  info.state= &status_info;
  share= *(*org_info)->s;
  unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
    (param->testflag & T_UNPACK);
  if (!(keyinfo=(MI_KEYDEF*) my_alloca(sizeof(MI_KEYDEF)*share.base.keys)))
3804
    DBUG_RETURN(0);
unknown's avatar
unknown committed
3805 3806 3807 3808
  memcpy((byte*) keyinfo,(byte*) share.keyinfo,
	 (size_t) (sizeof(MI_KEYDEF)*share.base.keys));

  key_parts= share.base.all_key_parts;
unknown's avatar
unknown committed
3809
  if (!(keysegs=(HA_KEYSEG*) my_alloca(sizeof(HA_KEYSEG)*
unknown's avatar
unknown committed
3810 3811 3812
				       (key_parts+share.base.keys))))
  {
    my_afree((gptr) keyinfo);
3813
    DBUG_RETURN(1);
unknown's avatar
unknown committed
3814 3815 3816 3817 3818 3819
  }
  if (!(recdef=(MI_COLUMNDEF*)
	my_alloca(sizeof(MI_COLUMNDEF)*(share.base.fields+1))))
  {
    my_afree((gptr) keyinfo);
    my_afree((gptr) keysegs);
3820
    DBUG_RETURN(1);
unknown's avatar
unknown committed
3821 3822 3823 3824 3825 3826 3827
  }
  if (!(uniquedef=(MI_UNIQUEDEF*)
	my_alloca(sizeof(MI_UNIQUEDEF)*(share.state.header.uniques+1))))
  {
    my_afree((gptr) recdef);
    my_afree((gptr) keyinfo);
    my_afree((gptr) keysegs);
3828
    DBUG_RETURN(1);
unknown's avatar
unknown committed
3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844
  }

  /* Copy the column definitions */
  memcpy((byte*) recdef,(byte*) share.rec,
	 (size_t) (sizeof(MI_COLUMNDEF)*(share.base.fields+1)));
  for (rec=recdef,end=recdef+share.base.fields; rec != end ; rec++)
  {
    if (unpack && !(share.options & HA_OPTION_PACK_RECORD) &&
	rec->type != FIELD_BLOB &&
	rec->type != FIELD_VARCHAR &&
	rec->type != FIELD_CHECK)
      rec->type=(int) FIELD_NORMAL;
  }

  /* Change the new key to point at the saved key segments */
  memcpy((byte*) keysegs,(byte*) share.keyparts,
unknown's avatar
unknown committed
3845
	 (size_t) (sizeof(HA_KEYSEG)*(key_parts+share.base.keys+
unknown's avatar
unknown committed
3846 3847 3848 3849 3850 3851 3852 3853 3854 3855
				      share.state.header.uniques)));
  keyseg=keysegs;
  for (key=keyinfo,key_end=keyinfo+share.base.keys; key != key_end ; key++)
  {
    key->seg=keyseg;
    for (; keyseg->type ; keyseg++)
    {
      if (param->language)
	keyseg->language=param->language;	/* change language */
    }
3856
    keyseg++;					/* Skip end pointer */
unknown's avatar
unknown committed
3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899
  }

  /* Copy the unique definitions and change them to point at the new key
     segments*/
  memcpy((byte*) uniquedef,(byte*) share.uniqueinfo,
	 (size_t) (sizeof(MI_UNIQUEDEF)*(share.state.header.uniques)));
  for (u_ptr=uniquedef,u_end=uniquedef+share.state.header.uniques;
       u_ptr != u_end ; u_ptr++)
  {
    u_ptr->seg=keyseg;
    keyseg+=u_ptr->keysegs+1;
  }
  if (share.options & HA_OPTION_COMPRESS_RECORD)
    share.base.records=max_records=info.state->records;
  else if (share.base.min_pack_length)
    max_records=(ha_rows) (my_seek(info.dfile,0L,MY_SEEK_END,MYF(0)) /
			   (ulong) share.base.min_pack_length);
  else
    max_records=0;
  unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
    (param->testflag & T_UNPACK);
  share.options&= ~HA_OPTION_TEMP_COMPRESS_RECORD;

  file_length=(ulonglong) my_seek(info.dfile,0L,MY_SEEK_END,MYF(0));
  tmp_length= file_length+file_length/10;
  set_if_bigger(file_length,param->max_data_file_length);
  set_if_bigger(file_length,tmp_length);
  set_if_bigger(file_length,(ulonglong) share.base.max_data_file_length);

  VOID(mi_close(*org_info));
  bzero((char*) &create_info,sizeof(create_info));
  create_info.max_rows=max(max_records,share.base.records);
  create_info.reloc_rows=share.base.reloc;
  create_info.old_options=(share.options |
			   (unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0));

  create_info.data_file_length=file_length;
  create_info.auto_increment=share.state.auto_increment;
  create_info.raid_type=   share.base.raid_type;
  create_info.raid_chunks= share.base.raid_chunks;
  create_info.raid_chunksize= share.base.raid_chunksize;
  create_info.language = (param->language ? param->language :
			  share.state.header.language);
unknown's avatar
unknown committed
3900
  create_info.key_file_length=  status_info.key_file_length;
3901 3902 3903 3904 3905
  /*
    Allow for creating an auto_increment key. This has an effect only if
    an auto_increment key exists in the original table.
  */
  create_info.with_auto_increment= TRUE;
unknown's avatar
unknown committed
3906 3907 3908
  /* We don't have to handle symlinks here because we are using
     HA_DONT_TOUCH_DATA */
  if (mi_create(filename,
unknown's avatar
unknown committed
3909 3910 3911 3912 3913 3914 3915 3916 3917
		share.base.keys - share.state.header.uniques,
		keyinfo, share.base.fields, recdef,
		share.state.header.uniques, uniquedef,
		&create_info,
		HA_DONT_TOUCH_DATA))
  {
    mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno);
    goto end;
  }
unknown's avatar
unknown committed
3918
  *org_info=mi_open(filename,O_RDWR,
unknown's avatar
unknown committed
3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949
		    (param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED :
		    (param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED :
		    HA_OPEN_ABORT_IF_LOCKED);
  if (!*org_info)
  {
    mi_check_print_error(param,"Got error %d when trying to open re-created indexfile",
		my_errno);
    goto end;
  }
  /* We are modifing */
  (*org_info)->s->options&= ~HA_OPTION_READ_ONLY_DATA;
  VOID(_mi_readinfo(*org_info,F_WRLCK,0));
  (*org_info)->state->records=info.state->records;
  if (share.state.create_time)
    (*org_info)->s->state.create_time=share.state.create_time;
  (*org_info)->s->state.unique=(*org_info)->this_unique=
    share.state.unique;
  (*org_info)->s->state.checksum=share.state.checksum;
  (*org_info)->state->del=info.state->del;
  (*org_info)->s->state.dellink=share.state.dellink;
  (*org_info)->state->empty=info.state->empty;
  (*org_info)->state->data_file_length=info.state->data_file_length;
  if (update_state_info(param,*org_info,UPDATE_TIME | UPDATE_STAT |
			UPDATE_OPEN_COUNT))
    goto end;
  error=0;
end:
  my_afree((gptr) uniquedef);
  my_afree((gptr) keyinfo);
  my_afree((gptr) recdef);
  my_afree((gptr) keysegs);
3950
  DBUG_RETURN(error);
unknown's avatar
unknown committed
3951 3952 3953 3954 3955
}


	/* write suffix to data file if neaded */

unknown's avatar
unknown committed
3956
int write_data_suffix(SORT_INFO *sort_info, my_bool fix_datafile)
unknown's avatar
unknown committed
3957
{
unknown's avatar
unknown committed
3958 3959 3960
  MI_INFO *info=sort_info->info;

  if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
unknown's avatar
unknown committed
3961 3962 3963 3964 3965
  {
    char buff[MEMMAP_EXTRA_MARGIN];
    bzero(buff,sizeof(buff));
    if (my_b_write(&info->rec_cache,buff,sizeof(buff)))
    {
unknown's avatar
unknown committed
3966
      mi_check_print_error(sort_info->param,
unknown's avatar
unknown committed
3967
			   "%d when writing to datafile",my_errno);
unknown's avatar
unknown committed
3968 3969
      return 1;
    }
unknown's avatar
unknown committed
3970
    sort_info->param->read_cache.end_of_file+=sizeof(buff);
unknown's avatar
unknown committed
3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987
  }
  return 0;
}

	/* Update state and myisamchk_time of indexfile */

int update_state_info(MI_CHECK *param, MI_INFO *info,uint update)
{
  MYISAM_SHARE *share=info->s;

  if (update & UPDATE_OPEN_COUNT)
  {
    share->state.open_count=0;
    share->global_changed=0;
  }
  if (update & UPDATE_STAT)
  {
3988
    uint i, key_parts= mi_uint2korr(share->state.header.key_parts);
unknown's avatar
unknown committed
3989
    share->state.rec_per_key_rows=info->state->records;
3990
    share->state.changed&= ~STATE_NOT_ANALYZED;
unknown's avatar
unknown committed
3991
    if (info->state->records)
3992
    {
unknown's avatar
unknown committed
3993 3994 3995 3996 3997
      for (i=0; i<key_parts; i++)
      {
        if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
          share->state.changed|= STATE_NOT_ANALYZED;
      }
3998
    }
unknown's avatar
unknown committed
3999 4000 4001 4002 4003 4004 4005 4006 4007
  }
  if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
  {
    if (update & UPDATE_TIME)
    {
      share->state.check_time= (long) time((time_t*) 0);
      if (!share->state.create_time)
	share->state.create_time=share->state.check_time;
    }
unknown's avatar
unknown committed
4008 4009 4010
    /*
      When tables are locked we haven't synched the share state and the
      real state for a while so we better do it here before synching
unknown's avatar
unknown committed
4011 4012
      the share state to disk. Only when table is write locked is it
      necessary to perform this synch.
unknown's avatar
unknown committed
4013
    */
unknown's avatar
unknown committed
4014 4015
    if (info->lock_type == F_WRLCK)
      share->state.state= *info->state;
unknown's avatar
unknown committed
4016 4017
    if (mi_state_info_write(share->kfile,&share->state,1+2))
      goto err;
unknown's avatar
unknown committed
4018
    share->changed=0;
unknown's avatar
unknown committed
4019 4020 4021 4022
  }
  {						/* Force update of status */
    int error;
    uint r_locks=share->r_locks,w_locks=share->w_locks;
4023
    share->r_locks= share->w_locks= share->tot_locks= 0;
unknown's avatar
unknown committed
4024
    error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
4025 4026 4027
    share->r_locks=r_locks;
    share->w_locks=w_locks;
    share->tot_locks=r_locks+w_locks;
unknown's avatar
unknown committed
4028 4029 4030 4031
    if (!error)
      return 0;
  }
err:
unknown's avatar
unknown committed
4032
  mi_check_print_error(param,"%d when updating keyfile",my_errno);
unknown's avatar
unknown committed
4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051
  return 1;
}

	/*
	  Update auto increment value for a table
	  When setting the 'repair_only' flag we only want to change the
	  old auto_increment value if its wrong (smaller than some given key).
	  The reason is that we shouldn't change the auto_increment value
	  for a table without good reason when only doing a repair; If the
	  user have inserted and deleted rows, the auto_increment value
	  may be bigger than the biggest current row and this is ok.

	  If repair_only is not set, we will update the flag to the value in
	  param->auto_increment is bigger than the biggest key.
	*/

void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
			       my_bool repair_only)
{
4052
  byte *record;
4053 4054
  DBUG_ENTER("update_auto_increment_key");

unknown's avatar
unknown committed
4055 4056 4057 4058
  if (!info->s->base.auto_key ||
      !(((ulonglong) 1 << (info->s->base.auto_key-1)
	 & info->s->state.key_map)))
  {
unknown's avatar
unknown committed
4059 4060 4061 4062
    if (!(param->testflag & T_VERY_SILENT))
      mi_check_print_info(param,
			  "Table: %s doesn't have an auto increment key\n",
			  param->isam_file_name);
4063
    DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4064 4065
  }
  if (!(param->testflag & T_SILENT) &&
4066
      !(param->testflag & T_REP))
unknown's avatar
unknown committed
4067
    printf("Updating MyISAM file: %s\n", param->isam_file_name);
4068 4069 4070 4071 4072 4073 4074 4075
  /*
    We have to use an allocated buffer instead of info->rec_buff as 
    _mi_put_key_in_record() may use info->rec_buff
  */
  if (!(record= (byte*) my_malloc((uint) info->s->base.pack_reclength,
				  MYF(0))))
  {
    mi_check_print_error(param,"Not enough memory for extra record");
4076
    DBUG_VOID_RETURN;
4077 4078
  }

unknown's avatar
unknown committed
4079
  mi_extra(info,HA_EXTRA_KEYREAD,0);
4080
  if (mi_rlast(info, record, info->s->base.auto_key-1))
unknown's avatar
unknown committed
4081 4082 4083
  {
    if (my_errno != HA_ERR_END_OF_FILE)
    {
unknown's avatar
unknown committed
4084
      mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4085
      my_free((char*) record, MYF(0));
unknown's avatar
unknown committed
4086
      mi_check_print_error(param,"%d when reading last record",my_errno);
4087
      DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4088 4089 4090 4091 4092 4093 4094 4095 4096
    }
    if (!repair_only)
      info->s->state.auto_increment=param->auto_increment_value;
  }
  else
  {
    ulonglong auto_increment= (repair_only ? info->s->state.auto_increment :
			       param->auto_increment_value);
    info->s->state.auto_increment=0;
4097
    update_auto_increment(info, record);
unknown's avatar
unknown committed
4098 4099
    set_if_bigger(info->s->state.auto_increment,auto_increment);
  }
unknown's avatar
unknown committed
4100
  mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4101
  my_free((char*) record, MYF(0));
unknown's avatar
unknown committed
4102
  update_state_info(param, info, UPDATE_AUTO_INC);
4103
  DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4104 4105
}

4106 4107 4108

/*
  Update statistics for each part of an index
4109

4110 4111
  SYNOPSIS
    update_key_parts()
4112
      keyinfo           IN  Index information (only key->keysegs used)
4113
      rec_per_key_part  OUT Store statistics here
4114 4115
      unique            IN  Array of (#distinct tuples)
      notnull_tuples    IN  Array of (#tuples), or NULL
4116
      records               Number of records in the table
4117 4118 4119 4120 4121 4122 4123

  DESCRIPTION
    This function is called produce index statistics values from unique and 
    notnull_tuples arrays after these arrays were produced with sequential
    index scan (the scan is done in two places: chk_index() and
    sort_key_write()).

4124 4125
    This function handles all 3 index statistics collection methods.

4126
    Unique is an array:
4127 4128 4129 4130 4131 4132 4133 4134 4135 4136
      unique[0]= (#different values of {keypart1}) - 1
      unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
      ...

    For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
      notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
      notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all 
                          keypart{i} are not NULL)
      ...
    For all other statistics collection methods notnull_tuples==NULL.
4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147

    Output is an array:
    rec_per_key_part[k] = 
     = E(#records in the table such that keypart_1=c_1 AND ... AND 
         keypart_k=c_k for arbitrary constants c_1 ... c_k) 
     
     = {assuming that values have uniform distribution and index contains all
        tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
        index tuples}
     
     = #tuples-in-the-index / #distinct-tuples-in-the-index.
4148 4149 4150 4151 4152 4153 4154 4155
    
    The #tuples-in-the-index and #distinct-tuples-in-the-index have different 
    meaning depending on which statistics collection method is used:
    
    MI_STATS_METHOD_*  how are nulls compared?  which tuples are counted?
     NULLS_EQUAL            NULL == NULL           all tuples in table
     NULLS_NOT_EQUAL        NULL != NULL           all tuples in table
     IGNORE_NULLS               n/a             tuples that don't have NULLs
4156
*/
unknown's avatar
unknown committed
4157

unknown's avatar
unknown committed
4158
void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
4159 4160
                      ulonglong *unique, ulonglong *notnull,
                      ulonglong records)
unknown's avatar
unknown committed
4161
{
4162 4163
  ulonglong count=0,tmp, unique_tuples;
  ulonglong tuples= records;
unknown's avatar
unknown committed
4164 4165 4166 4167
  uint parts;
  for (parts=0 ; parts < keyinfo->keysegs  ; parts++)
  {
    count+=unique[parts];
4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183
    unique_tuples= count + 1;    
    if (notnull)
    {
      tuples= notnull[parts];
      /* 
        #(unique_tuples not counting tuples with NULLs) = 
          #(unique_tuples counting tuples with NULLs as different) - 
          #(tuples with NULLs)
      */
      unique_tuples -= (records - notnull[parts]);
    }
    
    if (unique_tuples == 0)
      tmp= 1;
    else if (count == 0)
      tmp= tuples; /* 1 unique tuple */
unknown's avatar
unknown committed
4184
    else
4185 4186 4187 4188 4189 4190
      tmp= (tuples + unique_tuples/2) / unique_tuples;

    /* 
      for some weird keys (e.g. FULLTEXT) tmp can be <1 here. 
      let's ensure it is not
    */
4191
    set_if_bigger(tmp,1);
unknown's avatar
unknown committed
4192 4193
    if (tmp >= (ulonglong) ~(ulong) 0)
      tmp=(ulonglong) ~(ulong) 0;
4194

unknown's avatar
unknown committed
4195 4196 4197 4198 4199 4200
    *rec_per_key_part=(ulong) tmp;
    rec_per_key_part++;
  }
}


unknown's avatar
unknown committed
4201
static ha_checksum mi_byte_checksum(const byte *buf, uint length)
unknown's avatar
unknown committed
4202 4203 4204 4205 4206 4207 4208 4209
{
  ha_checksum crc;
  const byte *end=buf+length;
  for (crc=0; buf != end; buf++)
    crc=((crc << 1) + *((uchar*) buf)) +
      test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
  return crc;
}
4210 4211 4212

static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
{
4213 4214
  uint key_maxlength=key->maxlength;
  if (key->flag & HA_FULLTEXT)
4215 4216 4217 4218 4219
  {
    uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                  key->seg->charset->mbmaxlen;
    key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
  }
4220 4221
  return (key->flag & HA_SPATIAL) ||
          (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&
4222
	  ((ulonglong) rows * key_maxlength >
4223
	   (ulonglong) myisam_max_temp_length));
4224 4225
}

4226 4227 4228 4229 4230 4231 4232 4233
/*
  Deactivate all not unique index that can be recreated fast
  These include packed keys on which sorting will use more temporary
  space than the max allowed file length or for which the unpacked keys
  will take much more space than packed keys.
  Note that 'rows' may be zero for the case when we don't know how many
  rows we will put into the file.
 */
4234

4235
void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
4236 4237
{
  MYISAM_SHARE *share=info->s;
4238 4239 4240 4241 4242 4243
  MI_KEYDEF    *key=share->keyinfo;
  uint          i;

  DBUG_ASSERT(info->state->records == 0 &&
              (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
  for (i=0 ; i < share->base.keys ; i++,key++)
4244
  {
4245 4246
    if (!(key->flag & (HA_NOSAME | HA_SPATIAL | HA_AUTO_KEY)) &&
        ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
4247
    {
4248 4249
      share->state.key_map&= ~ ((ulonglong) 1 << i);
      info->update|= HA_STATE_CHANGED;
4250 4251 4252 4253 4254
    }
  }
}


4255 4256 4257 4258 4259
/*
  Return TRUE if we can use repair by sorting
  One can set the force argument to force to use sorting
  even if the temporary file would be quite big!
*/
4260

unknown's avatar
unknown committed
4261
my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
unknown's avatar
unknown committed
4262
			    ulonglong key_map, my_bool force)
4263 4264 4265
{
  MYISAM_SHARE *share=info->s;
  MI_KEYDEF *key=share->keyinfo;
4266 4267 4268
  uint i;

  /*
unknown's avatar
unknown committed
4269
    mi_repair_by_sort only works if we have at least one key. If we don't
4270 4271 4272
    have any keys, we should use the normal repair.
  */
  if (!key_map)
4273 4274 4275
    return FALSE;				/* Can't use sort */
  for (i=0 ; i < share->base.keys ; i++,key++)
  {
4276
    if (!force && mi_too_big_key_for_sort(key,rows))
4277 4278 4279 4280
      return FALSE;
  }
  return TRUE;
}
4281 4282 4283


static void
unknown's avatar
unknown committed
4284
set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
4285 4286
{
  if ((sort_info->new_data_file_type=share->data_file_type) ==
unknown's avatar
unknown committed
4287
      COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302
  {
    MYISAM_SHARE tmp;

    if (share->options & HA_OPTION_PACK_RECORD)
      sort_info->new_data_file_type = DYNAMIC_RECORD;
    else
      sort_info->new_data_file_type = STATIC_RECORD;

    /* Set delete_function for sort_delete_record() */
    memcpy((char*) &tmp, share, sizeof(*share));
    tmp.options= ~HA_OPTION_COMPRESS_RECORD;
    mi_setup_functions(&tmp);
    share->delete_record=tmp.delete_record;
  }
}