multipath.c 25.7 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9
/*
 * multipath.c : Multiple Devices driver for Linux
 *
 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
 *
 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
 *
 * MULTIPATH management functions.
 *
Linus Torvalds's avatar
Linus Torvalds committed
10
 * derived from raid1.c.
Linus Torvalds's avatar
Linus Torvalds committed
11 12 13 14 15 16 17 18 19 20 21 22 23
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include <linux/module.h>
#include <linux/slab.h>
Linus Torvalds's avatar
Linus Torvalds committed
24
#include <linux/spinlock.h>
Linus Torvalds's avatar
Linus Torvalds committed
25
#include <linux/raid/multipath.h>
Alexander Viro's avatar
Alexander Viro committed
26
#include <linux/buffer_head.h>
Linus Torvalds's avatar
Linus Torvalds committed
27 28 29 30 31
#include <asm/atomic.h>

#define MAJOR_NR MD_MAJOR
#define MD_DRIVER
#define MD_PERSONALITY
Alexander Viro's avatar
Alexander Viro committed
32
#define DEVICE_NR(device) (minor(device))
Linus Torvalds's avatar
Linus Torvalds committed
33 34 35

#define MAX_WORK_PER_DISK 128

Linus Torvalds's avatar
Linus Torvalds committed
36 37 38
#define	NR_RESERVED_BUFS	32


Linus Torvalds's avatar
Linus Torvalds committed
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
/*
 * The following can be used to debug the driver
 */
#define MULTIPATH_DEBUG	0

#if MULTIPATH_DEBUG
#define PRINTK(x...)   printk(x)
#define inline
#define __inline__
#else
#define PRINTK(x...)  do { } while (0)
#endif


static mdk_personality_t multipath_personality;
54
static spinlock_t retry_list_lock = SPIN_LOCK_UNLOCKED;
Linus Torvalds's avatar
Linus Torvalds committed
55 56 57 58 59 60 61 62
struct multipath_bh *multipath_retry_list = NULL, **multipath_retry_tail;

static int multipath_diskop(mddev_t *mddev, mdp_disk_t **d, int state);



static struct multipath_bh *multipath_alloc_mpbh(multipath_conf_t *conf)
{
Linus Torvalds's avatar
Linus Torvalds committed
63
	struct multipath_bh *mp_bh = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
64 65

	do {
66
		spin_lock_irq(&conf->device_lock);
Linus Torvalds's avatar
Linus Torvalds committed
67 68 69 70 71 72
		if (!conf->freer1_blocked && conf->freer1) {
			mp_bh = conf->freer1;
			conf->freer1 = mp_bh->next_mp;
			conf->freer1_cnt--;
			mp_bh->next_mp = NULL;
			mp_bh->state = (1 << MPBH_PreAlloc);
Linus Torvalds's avatar
Linus Torvalds committed
73
		}
74
		spin_unlock_irq(&conf->device_lock);
Linus Torvalds's avatar
Linus Torvalds committed
75 76 77
		if (mp_bh)
			return mp_bh;
		mp_bh = (struct multipath_bh *) kmalloc(sizeof(struct multipath_bh),
Linus Torvalds's avatar
Linus Torvalds committed
78
					GFP_NOIO);
Linus Torvalds's avatar
Linus Torvalds committed
79 80 81
		if (mp_bh) {
			memset(mp_bh, 0, sizeof(*mp_bh));
			return mp_bh;
Linus Torvalds's avatar
Linus Torvalds committed
82
		}
Linus Torvalds's avatar
Linus Torvalds committed
83 84 85 86 87 88
		conf->freer1_blocked = 1;
		wait_disk_event(conf->wait_buffer,
				!conf->freer1_blocked ||
				conf->freer1_cnt > NR_RESERVED_BUFS/2
		    );
		conf->freer1_blocked = 0;
Linus Torvalds's avatar
Linus Torvalds committed
89 90 91
	} while (1);
}

Linus Torvalds's avatar
Linus Torvalds committed
92
static inline void multipath_free_mpbh(struct multipath_bh *mp_bh)
Linus Torvalds's avatar
Linus Torvalds committed
93
{
Linus Torvalds's avatar
Linus Torvalds committed
94
	multipath_conf_t *conf = mddev_to_conf(mp_bh->mddev);
Linus Torvalds's avatar
Linus Torvalds committed
95

Linus Torvalds's avatar
Linus Torvalds committed
96
	if (test_bit(MPBH_PreAlloc, &mp_bh->state)) {
Linus Torvalds's avatar
Linus Torvalds committed
97
		unsigned long flags;
98
		mp_bh->bio = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
99
		spin_lock_irqsave(&conf->device_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
100 101 102
		mp_bh->next_mp = conf->freer1;
		conf->freer1 = mp_bh;
		conf->freer1_cnt++;
Linus Torvalds's avatar
Linus Torvalds committed
103
		spin_unlock_irqrestore(&conf->device_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
104
		wake_up(&conf->wait_buffer);
Linus Torvalds's avatar
Linus Torvalds committed
105
	} else {
Linus Torvalds's avatar
Linus Torvalds committed
106
		kfree(mp_bh);
Linus Torvalds's avatar
Linus Torvalds committed
107 108 109 110 111 112 113 114
	}
}

static int multipath_grow_mpbh (multipath_conf_t *conf, int cnt)
{
	int i = 0;

	while (i < cnt) {
Linus Torvalds's avatar
Linus Torvalds committed
115 116 117
		struct multipath_bh *mp_bh;
		mp_bh = (struct multipath_bh*)kmalloc(sizeof(*mp_bh), GFP_KERNEL);
		if (!mp_bh)
Linus Torvalds's avatar
Linus Torvalds committed
118
			break;
Linus Torvalds's avatar
Linus Torvalds committed
119 120 121
		memset(mp_bh, 0, sizeof(*mp_bh));
		set_bit(MPBH_PreAlloc, &mp_bh->state);
		mp_bh->mddev = conf->mddev;	       
Linus Torvalds's avatar
Linus Torvalds committed
122

Linus Torvalds's avatar
Linus Torvalds committed
123
		multipath_free_mpbh(mp_bh);
Linus Torvalds's avatar
Linus Torvalds committed
124 125 126 127 128 129 130
		i++;
	}
	return i;
}

static void multipath_shrink_mpbh(multipath_conf_t *conf)
{
131
	spin_lock_irq(&conf->device_lock);
Linus Torvalds's avatar
Linus Torvalds committed
132
	while (conf->freer1) {
Linus Torvalds's avatar
Linus Torvalds committed
133 134 135 136
		struct multipath_bh *mp_bh = conf->freer1;
		conf->freer1 = mp_bh->next_mp;
		conf->freer1_cnt--;
		kfree(mp_bh);
Linus Torvalds's avatar
Linus Torvalds committed
137
	}
138
	spin_unlock_irq(&conf->device_lock);
Linus Torvalds's avatar
Linus Torvalds committed
139 140 141
}


142
static int multipath_map (mddev_t *mddev, struct block_device **bdev)
Linus Torvalds's avatar
Linus Torvalds committed
143 144 145 146 147 148 149 150 151 152 153
{
	multipath_conf_t *conf = mddev_to_conf(mddev);
	int i, disks = MD_SB_DISKS;

	/*
	 * Later we do read balancing on the read side 
	 * now we use the first available disk.
	 */

	for (i = 0; i < disks; i++) {
		if (conf->multipaths[i].operational) {
154
			*bdev = conf->multipaths[i].bdev;
Linus Torvalds's avatar
Linus Torvalds committed
155 156 157 158 159 160 161 162
			return (0);
		}
	}

	printk (KERN_ERR "multipath_map(): no more operational IO paths?\n");
	return (-1);
}

Linus Torvalds's avatar
Linus Torvalds committed
163
static void multipath_reschedule_retry (struct multipath_bh *mp_bh)
Linus Torvalds's avatar
Linus Torvalds committed
164 165
{
	unsigned long flags;
Linus Torvalds's avatar
Linus Torvalds committed
166
	mddev_t *mddev = mp_bh->mddev;
Linus Torvalds's avatar
Linus Torvalds committed
167 168
	multipath_conf_t *conf = mddev_to_conf(mddev);

169
	spin_lock_irqsave(&retry_list_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
170 171
	if (multipath_retry_list == NULL)
		multipath_retry_tail = &multipath_retry_list;
Linus Torvalds's avatar
Linus Torvalds committed
172 173 174
	*multipath_retry_tail = mp_bh;
	multipath_retry_tail = &mp_bh->next_mp;
	mp_bh->next_mp = NULL;
175
	spin_unlock_irqrestore(&retry_list_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
176 177 178 179 180 181 182 183 184
	md_wakeup_thread(conf->thread);
}


/*
 * multipath_end_bh_io() is called when we have finished servicing a multipathed
 * operation and are ready to return a success/failure code to the buffer
 * cache layer.
 */
Linus Torvalds's avatar
Linus Torvalds committed
185
static void multipath_end_bh_io (struct multipath_bh *mp_bh, int uptodate)
Linus Torvalds's avatar
Linus Torvalds committed
186
{
187
	struct bio *bio = mp_bh->master_bio;
Linus Torvalds's avatar
Linus Torvalds committed
188

189 190
	bio_endio(bio, uptodate);
	bio_put(mp_bh->bio);
Linus Torvalds's avatar
Linus Torvalds committed
191
	multipath_free_mpbh(mp_bh);
Linus Torvalds's avatar
Linus Torvalds committed
192 193
}

194
void multipath_end_request(struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
195
{
196 197
	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
	struct multipath_bh * mp_bh = (struct multipath_bh *)(bio->bi_private);
Linus Torvalds's avatar
Linus Torvalds committed
198 199 200 201 202

	/*
	 * this branch is our 'one multipath IO has finished' event handler:
	 */
	if (!uptodate)
203
		md_error (mp_bh->mddev, bio->bi_bdev);
Linus Torvalds's avatar
Linus Torvalds committed
204 205 206 207 208 209 210 211 212 213
	else
		/*
		 * Set MPBH_Uptodate in our master buffer_head, so that
		 * we will return a good error code for to the higher
		 * levels even if IO on some other multipathed buffer fails.
		 *
		 * The 'master' represents the complex operation to 
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' buffer_head.
		 */
Linus Torvalds's avatar
Linus Torvalds committed
214
		set_bit (MPBH_Uptodate, &mp_bh->state);
Linus Torvalds's avatar
Linus Torvalds committed
215 216 217

		
	if (uptodate) {
Linus Torvalds's avatar
Linus Torvalds committed
218
		multipath_end_bh_io(mp_bh, uptodate);
Linus Torvalds's avatar
Linus Torvalds committed
219 220 221 222 223
		return;
	}
	/*
	 * oops, IO error:
	 */
224
	printk(KERN_ERR "multipath: %s: rescheduling sector %lu\n", 
225
		 bdev_partition_name(bio->bi_bdev), bio->bi_sector);
Linus Torvalds's avatar
Linus Torvalds committed
226
	multipath_reschedule_retry(mp_bh);
Linus Torvalds's avatar
Linus Torvalds committed
227 228 229 230 231
	return;
}

/*
 * This routine returns the disk from which the requested read should
Linus Torvalds's avatar
Linus Torvalds committed
232
 * be done.
Linus Torvalds's avatar
Linus Torvalds committed
233 234 235 236 237 238 239 240 241 242 243 244 245
 */

static int multipath_read_balance (multipath_conf_t *conf)
{
	int disk;

	for (disk = 0; disk < conf->raid_disks; disk++)	
		if (conf->multipaths[disk].operational)
			return disk;
	BUG();
	return 0;
}

246
static int multipath_make_request (request_queue_t *q, struct bio * bio)
Linus Torvalds's avatar
Linus Torvalds committed
247
{
248
	mddev_t *mddev = q->queuedata;
Linus Torvalds's avatar
Linus Torvalds committed
249
	multipath_conf_t *conf = mddev_to_conf(mddev);
250
	struct bio *real_bio;
Linus Torvalds's avatar
Linus Torvalds committed
251
	struct multipath_bh * mp_bh;
Linus Torvalds's avatar
Linus Torvalds committed
252 253
	struct multipath_info *multipath;

Linus Torvalds's avatar
Linus Torvalds committed
254
	mp_bh = multipath_alloc_mpbh (conf);
Linus Torvalds's avatar
Linus Torvalds committed
255

256
	mp_bh->master_bio = bio;
Linus Torvalds's avatar
Linus Torvalds committed
257
	mp_bh->mddev = mddev;
258
	mp_bh->cmd = bio_data_dir(bio);
Linus Torvalds's avatar
Linus Torvalds committed
259 260 261 262 263 264

	/*
	 * read balancing logic:
	 */
	multipath = conf->multipaths + multipath_read_balance(conf);

265
	real_bio = bio_clone(bio, GFP_NOIO);
266
	real_bio->bi_bdev = multipath->bdev;
267
	real_bio->bi_rw = bio_data_dir(bio);
268 269 270 271
	real_bio->bi_end_io = multipath_end_request;
	real_bio->bi_private = mp_bh;
	mp_bh->bio = real_bio;
	generic_make_request(real_bio);
Linus Torvalds's avatar
Linus Torvalds committed
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
	return 0;
}

static int multipath_status (char *page, mddev_t *mddev)
{
	multipath_conf_t *conf = mddev_to_conf(mddev);
	int sz = 0, i;
	
	sz += sprintf (page+sz, " [%d/%d] [", conf->raid_disks,
						 conf->working_disks);
	for (i = 0; i < conf->raid_disks; i++)
		sz += sprintf (page+sz, "%s",
			conf->multipaths[i].operational ? "U" : "_");
	sz += sprintf (page+sz, "]");
	return sz;
}

#define LAST_DISK KERN_ALERT \
"multipath: only one IO path left and IO error.\n"

#define NO_SPARE_DISK KERN_ALERT \
"multipath: no spare IO path left!\n"

#define DISK_FAILED KERN_ALERT \
"multipath: IO failure on %s, disabling IO path. \n" \
"	Operation continuing on %d IO paths.\n"

static void mark_disk_bad (mddev_t *mddev, int failed)
{
	multipath_conf_t *conf = mddev_to_conf(mddev);
	struct multipath_info *multipath = conf->multipaths+failed;
	mdp_super_t *sb = mddev->sb;

	multipath->operational = 0;
	mark_disk_faulty(sb->disks+multipath->number);
	mark_disk_nonsync(sb->disks+multipath->number);
	mark_disk_inactive(sb->disks+multipath->number);
	sb->active_disks--;
	sb->working_disks--;
	sb->failed_disks++;
	mddev->sb_dirty = 1;
	md_wakeup_thread(conf->thread);
	conf->working_disks--;
	printk (DISK_FAILED, partition_name (multipath->dev),
				 conf->working_disks);
}

/*
 * Careful, this can execute in IRQ contexts as well!
 */
static int multipath_error (mddev_t *mddev, kdev_t dev)
{
	multipath_conf_t *conf = mddev_to_conf(mddev);
	struct multipath_info * multipaths = conf->multipaths;
	int disks = MD_SB_DISKS;
	int other_paths = 1;
	int i;

	if (conf->working_disks == 1) {
		other_paths = 0;
		for (i = 0; i < disks; i++) {
			if (multipaths[i].spare) {
				other_paths = 1;
				break;
			}
		}
	}

	if (!other_paths) {
		/*
		 * Uh oh, we can do nothing if this is our last path, but
		 * first check if this is a queued request for a device
		 * which has just failed.
		 */
		for (i = 0; i < disks; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
347
			if (kdev_same(multipaths[i].dev, dev) && !multipaths[i].operational)
Linus Torvalds's avatar
Linus Torvalds committed
348 349 350 351 352 353 354 355
				return 0;
		}
		printk (LAST_DISK);
	} else {
		/*
		 * Mark disk as unusable
		 */
		for (i = 0; i < disks; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
356
			if (kdev_same(multipaths[i].dev,dev) && multipaths[i].operational) {
Linus Torvalds's avatar
Linus Torvalds committed
357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
				mark_disk_bad(mddev, i);
				break;
			}
		}
		if (!conf->working_disks) {
			int err = 1;
			mdp_disk_t *spare;
			mdp_super_t *sb = mddev->sb;

			spare = get_spare(mddev);
			if (spare) {
				err = multipath_diskop(mddev, &spare, DISKOP_SPARE_WRITE);
				printk("got DISKOP_SPARE_WRITE err: %d. (spare_faulty(): %d)\n", err, disk_faulty(spare));
			}
			if (!err && !disk_faulty(spare)) {
				multipath_diskop(mddev, &spare, DISKOP_SPARE_ACTIVE);
				mark_disk_sync(spare);
				mark_disk_active(spare);
				sb->active_disks++;
				sb->spare_disks--;
			}
		}
	}
	return 0;
}

#undef LAST_DISK
#undef NO_SPARE_DISK
#undef DISK_FAILED


static void print_multipath_conf (multipath_conf_t *conf)
{
	int i;
	struct multipath_info *tmp;

	printk("MULTIPATH conf printout:\n");
	if (!conf) {
		printk("(conf==NULL)\n");
		return;
	}
	printk(" --- wd:%d rd:%d nd:%d\n", conf->working_disks,
			 conf->raid_disks, conf->nr_disks);

	for (i = 0; i < MD_SB_DISKS; i++) {
		tmp = conf->multipaths + i;
		if (tmp->spare || tmp->operational || tmp->number ||
				tmp->raid_disk || tmp->used_slot)
			printk(" disk%d, s:%d, o:%d, n:%d rd:%d us:%d dev:%s\n",
				i, tmp->spare,tmp->operational,
				tmp->number,tmp->raid_disk,tmp->used_slot,
				partition_name(tmp->dev));
	}
}

static int multipath_diskop(mddev_t *mddev, mdp_disk_t **d, int state)
{
	int err = 0;
	int i, failed_disk=-1, spare_disk=-1, removed_disk=-1, added_disk=-1;
	multipath_conf_t *conf = mddev->private;
	struct multipath_info *tmp, *sdisk, *fdisk, *rdisk, *adisk;
	mdp_super_t *sb = mddev->sb;
	mdp_disk_t *failed_desc, *spare_desc, *added_desc;
	mdk_rdev_t *spare_rdev, *failed_rdev;
421
	struct block_device *bdev;
Linus Torvalds's avatar
Linus Torvalds committed
422 423

	print_multipath_conf(conf);
424
	spin_lock_irq(&conf->device_lock);
Linus Torvalds's avatar
Linus Torvalds committed
425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
	/*
	 * find the disk ...
	 */
	switch (state) {

	case DISKOP_SPARE_ACTIVE:

		/*
		 * Find the failed disk within the MULTIPATH configuration ...
		 * (this can only be in the first conf->working_disks part)
		 */
		for (i = 0; i < conf->raid_disks; i++) {
			tmp = conf->multipaths + i;
			if ((!tmp->operational && !tmp->spare) ||
					!tmp->used_slot) {
				failed_disk = i;
				break;
			}
		}
		/*
		 * When we activate a spare disk we _must_ have a disk in
		 * the lower (active) part of the array to replace. 
		 */
		if ((failed_disk == -1) || (failed_disk >= conf->raid_disks)) {
			MD_BUG();
			err = 1;
			goto abort;
		}
		/* fall through */

	case DISKOP_SPARE_WRITE:
	case DISKOP_SPARE_INACTIVE:

		/*
		 * Find the spare disk ... (can only be in the 'high'
		 * area of the array)
		 */
		for (i = conf->raid_disks; i < MD_SB_DISKS; i++) {
			tmp = conf->multipaths + i;
			if (tmp->spare && tmp->number == (*d)->number) {
				spare_disk = i;
				break;
			}
		}
		if (spare_disk == -1) {
			MD_BUG();
			err = 1;
			goto abort;
		}
		break;

	case DISKOP_HOT_REMOVE_DISK:

		for (i = 0; i < MD_SB_DISKS; i++) {
			tmp = conf->multipaths + i;
			if (tmp->used_slot && (tmp->number == (*d)->number)) {
				if (tmp->operational) {
					printk(KERN_ERR "hot-remove-disk, slot %d is identified to be the requested disk (number %d), but is still operational!\n", i, (*d)->number);
					err = -EBUSY;
					goto abort;
				}
				removed_disk = i;
				break;
			}
		}
		if (removed_disk == -1) {
			MD_BUG();
			err = 1;
			goto abort;
		}
		break;

	case DISKOP_HOT_ADD_DISK:

		for (i = conf->raid_disks; i < MD_SB_DISKS; i++) {
			tmp = conf->multipaths + i;
			if (!tmp->used_slot) {
				added_disk = i;
				break;
			}
		}
		if (added_disk == -1) {
			MD_BUG();
			err = 1;
			goto abort;
		}
		break;
	}

	switch (state) {
	/*
	 * Switch the spare disk to write-only mode:
	 */
	case DISKOP_SPARE_WRITE:
		sdisk = conf->multipaths + spare_disk;
		sdisk->operational = 1;
		break;
	/*
	 * Deactivate a spare disk:
	 */
	case DISKOP_SPARE_INACTIVE:
		sdisk = conf->multipaths + spare_disk;
		sdisk->operational = 0;
		break;
	/*
	 * Activate (mark read-write) the (now sync) spare disk,
	 * which means we switch it's 'raid position' (->raid_disk)
	 * with the failed disk. (only the first 'conf->nr_disks'
	 * slots are used for 'real' disks and we must preserve this
	 * property)
	 */
	case DISKOP_SPARE_ACTIVE:
		sdisk = conf->multipaths + spare_disk;
		fdisk = conf->multipaths + failed_disk;

		spare_desc = &sb->disks[sdisk->number];
		failed_desc = &sb->disks[fdisk->number];

		if (spare_desc != *d) {
			MD_BUG();
			err = 1;
			goto abort;
		}

		if (spare_desc->raid_disk != sdisk->raid_disk) {
			MD_BUG();
			err = 1;
			goto abort;
		}
			
		if (sdisk->raid_disk != spare_disk) {
			MD_BUG();
			err = 1;
			goto abort;
		}

		if (failed_desc->raid_disk != fdisk->raid_disk) {
			MD_BUG();
			err = 1;
			goto abort;
		}

		if (fdisk->raid_disk != failed_disk) {
			MD_BUG();
			err = 1;
			goto abort;
		}

		/*
		 * do the switch finally
		 */
		spare_rdev = find_rdev_nr(mddev, spare_desc->number);
		failed_rdev = find_rdev_nr(mddev, failed_desc->number);
		xchg_values(spare_rdev->desc_nr, failed_rdev->desc_nr);
		spare_rdev->alias_device = 0;
		failed_rdev->alias_device = 1;

		xchg_values(*spare_desc, *failed_desc);
		xchg_values(*fdisk, *sdisk);

		/*
		 * (careful, 'failed' and 'spare' are switched from now on)
		 *
		 * we want to preserve linear numbering and we want to
		 * give the proper raid_disk number to the now activated
		 * disk. (this means we switch back these values)
		 */
	
		xchg_values(spare_desc->raid_disk, failed_desc->raid_disk);
		xchg_values(sdisk->raid_disk, fdisk->raid_disk);
		xchg_values(spare_desc->number, failed_desc->number);
		xchg_values(sdisk->number, fdisk->number);

		*d = failed_desc;

600
		if (!sdisk->bdev)
Linus Torvalds's avatar
Linus Torvalds committed
601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624
			sdisk->used_slot = 0;
		/*
		 * this really activates the spare.
		 */
		fdisk->spare = 0;

		/*
		 * if we activate a spare, we definitely replace a
		 * non-operational disk slot in the 'low' area of
		 * the disk array.
		 */

		conf->working_disks++;

		break;

	case DISKOP_HOT_REMOVE_DISK:
		rdisk = conf->multipaths + removed_disk;

		if (rdisk->spare && (removed_disk < conf->raid_disks)) {
			MD_BUG();	
			err = 1;
			goto abort;
		}
625
		bdev = rdisk->bdev;
Linus Torvalds's avatar
Linus Torvalds committed
626
		rdisk->dev = NODEV;
627
		rdisk->bdev = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
628 629
		rdisk->used_slot = 0;
		conf->nr_disks--;
630
		bdput(bdev);
Linus Torvalds's avatar
Linus Torvalds committed
631 632 633 634 635 636 637 638 639 640 641 642 643 644
		break;

	case DISKOP_HOT_ADD_DISK:
		adisk = conf->multipaths + added_disk;
		added_desc = *d;

		if (added_disk != added_desc->number) {
			MD_BUG();	
			err = 1;
			goto abort;
		}

		adisk->number = added_desc->number;
		adisk->raid_disk = added_desc->raid_disk;
Linus Torvalds's avatar
Linus Torvalds committed
645
		adisk->dev = mk_kdev(added_desc->major,added_desc->minor);
646 647
		/* it will be held open by rdev */
		adisk->bdev = bdget(kdev_t_to_nr(adisk->dev));
Linus Torvalds's avatar
Linus Torvalds committed
648 649 650 651 652 653 654 655 656

		adisk->operational = 0;
		adisk->spare = 1;
		adisk->used_slot = 1;
		conf->nr_disks++;

		break;

	default:
657
		MD_BUG();
Linus Torvalds's avatar
Linus Torvalds committed
658 659 660 661
		err = 1;
		goto abort;
	}
abort:
662
	spin_unlock_irq(&conf->device_lock);
Linus Torvalds's avatar
Linus Torvalds committed
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684

	print_multipath_conf(conf);
	return err;
}


#define IO_ERROR KERN_ALERT \
"multipath: %s: unrecoverable IO read error for block %lu\n"

#define REDIRECT_SECTOR KERN_ERR \
"multipath: %s: redirecting sector %lu to another IO path\n"

/*
 * This is a kernel thread which:
 *
 *	1.	Retries failed read operations on working multipaths.
 *	2.	Updates the raid superblock when problems encounter.
 *	3.	Performs writes following reads for array syncronising.
 */

static void multipathd (void *data)
{
Linus Torvalds's avatar
Linus Torvalds committed
685
	struct multipath_bh *mp_bh;
686
	struct bio *bio;
Linus Torvalds's avatar
Linus Torvalds committed
687 688
	unsigned long flags;
	mddev_t *mddev;
689
	struct block_device *bdev;
Linus Torvalds's avatar
Linus Torvalds committed
690 691

	for (;;) {
692
		spin_lock_irqsave(&retry_list_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
693 694
		mp_bh = multipath_retry_list;
		if (!mp_bh)
Linus Torvalds's avatar
Linus Torvalds committed
695
			break;
Linus Torvalds's avatar
Linus Torvalds committed
696
		multipath_retry_list = mp_bh->next_mp;
697
		spin_unlock_irqrestore(&retry_list_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
698

Linus Torvalds's avatar
Linus Torvalds committed
699
		mddev = mp_bh->mddev;
Linus Torvalds's avatar
Linus Torvalds committed
700 701 702 703
		if (mddev->sb_dirty) {
			printk(KERN_INFO "dirty sb detected, updating.\n");
			md_update_sb(mddev);
		}
704
		bio = mp_bh->bio;
705
		bdev = bio->bi_bdev;
Linus Torvalds's avatar
Linus Torvalds committed
706
		
707 708
		multipath_map (mddev, &bio->bi_bdev);
		if (bio->bi_bdev == bdev) {
709
			printk(IO_ERROR,
710
				bdev_partition_name(bio->bi_bdev), bio->bi_sector);
Linus Torvalds's avatar
Linus Torvalds committed
711
			multipath_end_bh_io(mp_bh, 0);
Linus Torvalds's avatar
Linus Torvalds committed
712
		} else {
713
			printk(REDIRECT_SECTOR,
714
				bdev_partition_name(bio->bi_bdev), bio->bi_sector);
715
			generic_make_request(bio);
Linus Torvalds's avatar
Linus Torvalds committed
716 717
		}
	}
718
	spin_unlock_irqrestore(&retry_list_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
}
#undef IO_ERROR
#undef REDIRECT_SECTOR

/*
 * This will catch the scenario in which one of the multipaths was
 * mounted as a normal device rather than as a part of a raid set.
 *
 * check_consistency is very personality-dependent, eg. RAID5 cannot
 * do this check, it uses another method.
 */
static int __check_consistency (mddev_t *mddev, int row)
{
	multipath_conf_t *conf = mddev_to_conf(mddev);
	int disks = MD_SB_DISKS;
734
	struct block_device *bdev;
Linus Torvalds's avatar
Linus Torvalds committed
735
	int i, rc = 0;
736 737 738 739 740 741 742 743
	char *buffer;
	struct page *page = NULL;
	int first = 1;
	int order = PAGE_CACHE_SHIFT-PAGE_SHIFT;

	buffer = (char *) __get_free_pages(GFP_KERNEL, order);
	if (!buffer)
		return rc;
Linus Torvalds's avatar
Linus Torvalds committed
744 745

	for (i = 0; i < disks; i++) {
746 747
		struct address_space *mapping;
		char *p;
Linus Torvalds's avatar
Linus Torvalds committed
748 749 750
		if (!conf->multipaths[i].operational)
			continue;
		printk("(checking disk %d)\n",i);
751
		bdev = conf->multipaths[i].bdev;
752 753 754 755 756
		mapping = bdev->bd_inode->i_mapping;
		page = read_cache_page(mapping, row/(PAGE_CACHE_SIZE/1024),
				(filler_t *)mapping->a_ops->readpage, NULL);
		if (IS_ERR(page)) {
			page = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
757
			break;
758 759 760 761 762 763 764 765 766 767 768
		}
		wait_on_page_locked(page);
		if (!PageUptodate(page))
			break;
		if (PageError(page))
			break;
		p = page_address(page);
		if (first) {
			memcpy(buffer, p, PAGE_CACHE_SIZE);
			first = 0;
		} else if (memcmp(buffer, p, PAGE_CACHE_SIZE)) {
Linus Torvalds's avatar
Linus Torvalds committed
769 770 771
			rc = 1;
			break;
		}
772
		page_cache_release(page);
773 774
		fsync_bdev(bdev);
		invalidate_bdev(bdev, 0);
775
		page = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
776
	}
777 778 779
	if (page) {
		bdev = page->mapping->host->i_bdev;
		page_cache_release(page);
780 781
		fsync_bdev(bdev);
		invalidate_bdev(bdev, 0);
Linus Torvalds's avatar
Linus Torvalds committed
782
	}
783
	free_pages((unsigned long) buffer, order);
Linus Torvalds's avatar
Linus Torvalds committed
784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846
	return rc;
}

static int check_consistency (mddev_t *mddev)
{
	if (__check_consistency(mddev, 0))
/*
 * we do not do this currently, as it's perfectly possible to
 * have an inconsistent array when it's freshly created. Only
 * newly written data has to be consistent.
 */
		return 0;

	return 0;
}

#define INVALID_LEVEL KERN_WARNING \
"multipath: md%d: raid level not set to multipath IO (%d)\n"

#define NO_SB KERN_ERR \
"multipath: disabled IO path %s (couldn't access raid superblock)\n"

#define ERRORS KERN_ERR \
"multipath: disabled IO path %s (errors detected)\n"

#define NOT_IN_SYNC KERN_ERR \
"multipath: making IO path %s a spare path (not in sync)\n"

#define INCONSISTENT KERN_ERR \
"multipath: disabled IO path %s (inconsistent descriptor)\n"

#define ALREADY_RUNNING KERN_ERR \
"multipath: disabled IO path %s (multipath %d already operational)\n"

#define OPERATIONAL KERN_INFO \
"multipath: device %s operational as IO path %d\n"

#define MEM_ERROR KERN_ERR \
"multipath: couldn't allocate memory for md%d\n"

#define SPARE KERN_INFO \
"multipath: spare IO path %s\n"

#define NONE_OPERATIONAL KERN_ERR \
"multipath: no operational IO paths for md%d\n"

#define SB_DIFFERENCES KERN_ERR \
"multipath: detected IO path differences!\n"

#define ARRAY_IS_ACTIVE KERN_INFO \
"multipath: array md%d active with %d out of %d IO paths (%d spare IO paths)\n"

#define THREAD_ERROR KERN_ERR \
"multipath: couldn't allocate thread for md%d\n"

static int multipath_run (mddev_t *mddev)
{
	multipath_conf_t *conf;
	int i, j, disk_idx;
	struct multipath_info *disk, *disk2;
	mdp_super_t *sb = mddev->sb;
	mdp_disk_t *desc, *desc2;
	mdk_rdev_t *rdev, *def_rdev = NULL;
847
	struct list_head *tmp;
Linus Torvalds's avatar
Linus Torvalds committed
848
	int num_rdevs = 0;
Linus Torvalds's avatar
Linus Torvalds committed
849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903

	MOD_INC_USE_COUNT;

	if (sb->level != -4) {
		printk(INVALID_LEVEL, mdidx(mddev), sb->level);
		goto out;
	}
	/*
	 * copy the already verified devices into our private MULTIPATH
	 * bookkeeping area. [whatever we allocate in multipath_run(),
	 * should be freed in multipath_stop()]
	 */

	conf = kmalloc(sizeof(multipath_conf_t), GFP_KERNEL);
	mddev->private = conf;
	if (!conf) {
		printk(MEM_ERROR, mdidx(mddev));
		goto out;
	}
	memset(conf, 0, sizeof(*conf));

	ITERATE_RDEV(mddev,rdev,tmp) {
		if (rdev->faulty) {
			/* this is a "should never happen" case and if it */
			/* ever does happen, a continue; won't help */
			printk(ERRORS, partition_name(rdev->dev));
			continue;
		} else {
			/* this is a "should never happen" case and if it */
			/* ever does happen, a continue; won't help */
			if (!rdev->sb) {
				MD_BUG();
				continue;
			}
		}
		if (rdev->desc_nr == -1) {
			MD_BUG();
			continue;
		}

		desc = &sb->disks[rdev->desc_nr];
		disk_idx = desc->raid_disk;
		disk = conf->multipaths + disk_idx;

		if (!disk_sync(desc))
			printk(NOT_IN_SYNC, partition_name(rdev->dev));

		/*
		 * Mark all disks as spare to start with, then pick our
		 * active disk.  If we have a disk that is marked active
		 * in the sb, then use it, else use the first rdev.
		 */
		disk->number = desc->number;
		disk->raid_disk = desc->raid_disk;
		disk->dev = rdev->dev;
904 905
		disk->bdev = rdev->bdev;
		atomic_inc(&rdev->bdev->bd_count);
Linus Torvalds's avatar
Linus Torvalds committed
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963
		disk->operational = 0;
		disk->spare = 1;
		disk->used_slot = 1;
		mark_disk_sync(desc);

		if (disk_active(desc)) {
			if(!conf->working_disks) {
				printk(OPERATIONAL, partition_name(rdev->dev),
 					desc->raid_disk);
				disk->operational = 1;
				disk->spare = 0;
				conf->working_disks++;
				def_rdev = rdev;
			} else {
				mark_disk_spare(desc);
			}
		} else
			mark_disk_spare(desc);

		if(!num_rdevs++) def_rdev = rdev;
	}
	if(!conf->working_disks && num_rdevs) {
		desc = &sb->disks[def_rdev->desc_nr];
		disk = conf->multipaths + desc->raid_disk;
		printk(OPERATIONAL, partition_name(def_rdev->dev),
			disk->raid_disk);
		disk->operational = 1;
		disk->spare = 0;
		conf->working_disks++;
		mark_disk_active(desc);
	}
	/*
	 * Make sure our active path is in desc spot 0
	 */
	if(def_rdev->desc_nr != 0) {
		rdev = find_rdev_nr(mddev, 0);
		desc = &sb->disks[def_rdev->desc_nr];
		desc2 = sb->disks;
		disk = conf->multipaths + desc->raid_disk;
		disk2 = conf->multipaths + desc2->raid_disk;
		xchg_values(*desc2,*desc);
		xchg_values(*disk2,*disk);
		xchg_values(desc2->number, desc->number);
		xchg_values(disk2->number, disk->number);
		xchg_values(desc2->raid_disk, desc->raid_disk);
		xchg_values(disk2->raid_disk, disk->raid_disk);
		if(rdev) {
			xchg_values(def_rdev->desc_nr,rdev->desc_nr);
		} else {
			def_rdev->desc_nr = 0;
		}
	}
	conf->raid_disks = sb->raid_disks = sb->active_disks = 1;
	conf->nr_disks = sb->nr_disks = sb->working_disks = num_rdevs;
	sb->failed_disks = 0;
	sb->spare_disks = num_rdevs - 1;
	mddev->sb_dirty = 1;
	conf->mddev = mddev;
964
	conf->device_lock = SPIN_LOCK_UNLOCKED;
Linus Torvalds's avatar
Linus Torvalds committed
965 966 967 968 969 970 971 972 973 974 975 976 977

	init_waitqueue_head(&conf->wait_buffer);

	if (!conf->working_disks) {
		printk(NONE_OPERATIONAL, mdidx(mddev));
		goto out_free_conf;
	}


	/* pre-allocate some buffer_head structures.
	 * As a minimum, 1 mpbh and raid_disks buffer_heads
	 * would probably get us by in tight memory situations,
	 * but a few more is probably a good idea.
Linus Torvalds's avatar
Linus Torvalds committed
978 979 980 981
	 * For now, try NR_RESERVED_BUFS mpbh and
	 * NR_RESERVED_BUFS*raid_disks bufferheads
	 * This will allow at least NR_RESERVED_BUFS concurrent
	 * reads or writes even if kmalloc starts failing
Linus Torvalds's avatar
Linus Torvalds committed
982
	 */
Linus Torvalds's avatar
Linus Torvalds committed
983
	if (multipath_grow_mpbh(conf, NR_RESERVED_BUFS) < NR_RESERVED_BUFS) {
Linus Torvalds's avatar
Linus Torvalds committed
984 985 986 987
		printk(MEM_ERROR, mdidx(mddev));
		goto out_free_conf;
	}

Linus Torvalds's avatar
Linus Torvalds committed
988
	if ((sb->state & (1 << MD_SB_CLEAN))) {
Linus Torvalds's avatar
Linus Torvalds committed
989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
		/*
		 * we do sanity checks even if the device says
		 * it's clean ...
		 */
		if (check_consistency(mddev)) {
			printk(SB_DIFFERENCES);
			sb->state &= ~(1 << MD_SB_CLEAN);
		}
	}

	{
		const char * name = "multipathd";

		conf->thread = md_register_thread(multipathd, conf, name);
		if (!conf->thread) {
			printk(THREAD_ERROR, mdidx(mddev));
			goto out_free_conf;
		}
	}

	/*
	 * Regenerate the "device is in sync with the raid set" bit for
	 * each device.
	 */
	for (i = 0; i < MD_SB_DISKS; i++) {
		mark_disk_nonsync(sb->disks+i);
		for (j = 0; j < sb->raid_disks; j++) {
			if (sb->disks[i].number == conf->multipaths[j].number)
				mark_disk_sync(sb->disks+i);
		}
	}

	printk(ARRAY_IS_ACTIVE, mdidx(mddev), sb->active_disks,
			sb->raid_disks, sb->spare_disks);
	/*
	 * Ok, everything is just fine now
	 */
	return 0;

out_free_conf:
	multipath_shrink_mpbh(conf);
1030 1031 1032
	for (i = 0; i < MD_SB_DISKS; i++)
		if (conf->multipaths[i].bdev)
			bdput(conf->multipaths[i].bdev);
Linus Torvalds's avatar
Linus Torvalds committed
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
	kfree(conf);
	mddev->private = NULL;
out:
	MOD_DEC_USE_COUNT;
	return -EIO;
}

#undef INVALID_LEVEL
#undef NO_SB
#undef ERRORS
#undef NOT_IN_SYNC
#undef INCONSISTENT
#undef ALREADY_RUNNING
#undef OPERATIONAL
#undef SPARE
#undef NONE_OPERATIONAL
#undef SB_DIFFERENCES
#undef ARRAY_IS_ACTIVE

static int multipath_stop (mddev_t *mddev)
{
	multipath_conf_t *conf = mddev_to_conf(mddev);
1055
	int i;
Linus Torvalds's avatar
Linus Torvalds committed
1056 1057 1058

	md_unregister_thread(conf->thread);
	multipath_shrink_mpbh(conf);
1059 1060 1061
	for (i = 0; i < MD_SB_DISKS; i++)
		if (conf->multipaths[i].bdev)
			bdput(conf->multipaths[i].bdev);
Linus Torvalds's avatar
Linus Torvalds committed
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
	kfree(conf);
	mddev->private = NULL;
	MOD_DEC_USE_COUNT;
	return 0;
}

static mdk_personality_t multipath_personality=
{
	name:		"multipath",
	make_request:	multipath_make_request,
	run:		multipath_run,
	stop:		multipath_stop,
	status:		multipath_status,
	error_handler:	multipath_error,
	diskop:		multipath_diskop,
};

1079
static int __init multipath_init (void)
Linus Torvalds's avatar
Linus Torvalds committed
1080 1081 1082 1083
{
	return register_md_personality (MULTIPATH, &multipath_personality);
}

1084
static void __exit multipath_exit (void)
Linus Torvalds's avatar
Linus Torvalds committed
1085 1086 1087 1088 1089 1090
{
	unregister_md_personality (MULTIPATH);
}

module_init(multipath_init);
module_exit(multipath_exit);
Linus Torvalds's avatar
Linus Torvalds committed
1091
MODULE_LICENSE("GPL");