[PATCH] partial bio completion notification

Make bio->bi_end_io() take bytes_done and actual error as argument. This
enables partial completion of bio's, which is important for latency
reasons (bio can be huge, for slow media we want page-by-page
completions).

I think I got most of the bi_end_io() functions out there, but I might
have missed a few. For the record, if you don't care about partial
completions and just want to be notified when the entire bio completes,
add a

	if (bio->bi_size)
		return 1;

to the top of your bi_end_io(). It should return 0 on completion.
bio_endio() will decrement bio->bi_size appropriately, it's recommended
for people to go through that. Otherwise they will have to control
BIO_UPTODATE and bi_size decrement themselves, there's really no reason
to do that. I've deliberately avoided doing any functional changes to
any of the end_io functions, as I think that would only make the patch
more complex. It's simple right now, but this being i/o paths I prefer
(as usual) to be careful and take small steps. The mpage_end_io_read()
do-vecs-at-the-time change can come right after this, for instance.

drivers/block/cciss.c

@@ -1567,10 +1567,11 @@ static inline void complete_buffers(struct bio *bio, int status)
 {
 	while (bio) {
 		struct bio *xbh = bio->bi_next; 
+		int nr_sectors = bio_sectors(bio);
 
 		bio->bi_next = NULL; 
-		blk_finished_io(bio_sectors(bio));
-		bio_endio(bio, status);
+		blk_finished_io(len);
+		bio_endio(bio, nr_sectors << 9, status ? 0 : -EIO);
 		bio = xbh;
 	}
 

drivers/block/cpqarray.c

@@ -911,11 +911,13 @@ static inline void complete_buffers(struct bio *bio, int ok)
 {
 	struct bio *xbh;
 	while(bio) {
+		int nr_sectors = bio_sectors(bio);
+
 		xbh = bio->bi_next;
 		bio->bi_next = NULL;
 		
-		blk_finished_io(bio_sectors(bio));
-		bio_endio(bio, ok);
+		blk_finished_io(nr_sectors);
+		bio_endio(bio, nr_sectors << 9, ok ? 0 : -EIO);
 
 		bio = xbh;
 	}

drivers/block/floppy.c

@@ -3846,9 +3846,13 @@ static int check_floppy_change(kdev_t dev)
  * a disk in the drive, and whether that disk is writable.
  */
 
-static void floppy_rb0_complete(struct bio *bio)
+static int floppy_rb0_complete(struct bio *bio, unsigned int bytes_done, int err)
 {
+	if (bio->bi_size)
+		return 1;
+
 	complete((struct completion*)bio->bi_private);
+	return 0;
 }
 
 static int __floppy_read_block_0(struct block_device *bdev)

drivers/block/ll_rw_blk.c

@@ -1576,10 +1576,8 @@ static int __make_request(request_queue_t *q, struct bio *bio)
 		/*
 		 * READA bit set
 		 */
-		if (bio->bi_rw & (1 << BIO_RW_AHEAD)) {
-			set_bit(BIO_RW_BLOCK, &bio->bi_flags);
+		if (bio_flagged(bio, BIO_RW_AHEAD))
 			goto end_io;
-		}
 
 		freereq = get_request_wait(q, rw);
 		spin_lock_irq(q->queue_lock);
@@ -1616,7 +1614,7 @@ static int __make_request(request_queue_t *q, struct bio *bio)
 	return 0;
 
 end_io:
-	bio->bi_end_io(bio);
+	bio_endio(bio, nr_sectors << 9, -EWOULDBLOCK);
 	return 0;
 }
 
@@ -1705,7 +1703,7 @@ void generic_make_request(struct bio *bio)
 			       bdevname(bio->bi_bdev),
 			       (long long) bio->bi_sector);
 end_io:
-			bio->bi_end_io(bio);
+			bio_endio(bio, 0, -EIO);
 			break;
 		}
 
@@ -1825,6 +1823,7 @@ int end_that_request_first(struct request *req, int uptodate, int nr_sectors)
 	total_nsect = 0;
 	while ((bio = req->bio)) {
 		nsect = bio_iovec(bio)->bv_len >> 9;
+		total_nsect += nsect;
 
 		BIO_BUG_ON(bio_iovec(bio)->bv_len > bio->bi_size);
 
@@ -1834,38 +1833,31 @@ int end_that_request_first(struct request *req, int uptodate, int nr_sectors)
 		if (unlikely(nsect > nr_sectors)) {
 			int partial = nr_sectors << 9;
 
-			bio->bi_size -= partial;
 			bio_iovec(bio)->bv_offset += partial;
 			bio_iovec(bio)->bv_len -= partial;
-			blk_recalc_rq_sectors(req, nr_sectors);
+			blk_recalc_rq_sectors(req, total_nsect);
 			blk_recalc_rq_segments(req);
+			bio_endio(bio, partial, !uptodate ? -EIO : 0);
 			return 1;
 		}
 
 		/*
-		 * account transfer
+		 * if bio->bi_end_io returns 0, this bio is done. move on
 		 */
-		bio->bi_size -= bio_iovec(bio)->bv_len;
-		bio->bi_idx++;
+		req->bio = bio->bi_next;
+		if (bio_endio(bio, nsect << 9, !uptodate ? -EIO : 0)) {
+			bio->bi_idx++;
+			req->bio = bio;
+		}
 
 		nr_sectors -= nsect;
-		total_nsect += nsect;
-
-		if (!bio->bi_size) {
-			req->bio = bio->bi_next;
-
-			bio_endio(bio, uptodate);
-
-			total_nsect = 0;
-		}
 
 		if ((bio = req->bio)) {
-			blk_recalc_rq_sectors(req, nsect);
-
 			/*
 			 * end more in this run, or just return 'not-done'
 			 */
 			if (unlikely(nr_sectors <= 0)) {
+				blk_recalc_rq_sectors(req, total_nsect);
 				blk_recalc_rq_segments(req);
 				return 1;
 			}

drivers/block/loop.c

@@ -374,7 +374,7 @@ static int do_bio_filebacked(struct loop_device *lo, struct bio *bio)
 	return ret;
 }
 
-static void loop_end_io_transfer(struct bio *);
+static int loop_end_io_transfer(struct bio *, unsigned int, int);
 static void loop_put_buffer(struct bio *bio)
 {
 	/*
@@ -382,6 +382,7 @@ static void loop_put_buffer(struct bio *bio)
 	 */
 	if (bio && bio->bi_end_io == loop_end_io_transfer) {
 		int i;
+
 		for (i = 0; i < bio->bi_vcnt; i++)
 			__free_page(bio->bi_io_vec[i].bv_page);
 
@@ -432,19 +433,23 @@ static struct bio *loop_get_bio(struct loop_device *lo)
  * bi_end_io context (we don't want to do decrypt of a page with irqs
  * disabled)
  */
-static void loop_end_io_transfer(struct bio *bio)
+static int loop_end_io_transfer(struct bio *bio, unsigned int bytes_done, int err)
 {
 	struct bio *rbh = bio->bi_private;
 	struct loop_device *lo = &loop_dev[minor(to_kdev_t(rbh->bi_bdev->bd_dev))];
-	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 
-	if (!uptodate || bio_rw(bio) == WRITE) {
-		bio_endio(rbh, uptodate);
+	if (bio->bi_size)
+		return 1;
+
+	if (!err || bio_rw(bio) == WRITE) {
+		bio_endio(rbh, rbh->bi_size, err);
 		if (atomic_dec_and_test(&lo->lo_pending))
 			up(&lo->lo_bh_mutex);
 		loop_put_buffer(bio);
 	} else
 		loop_add_bio(lo, bio);
+
+	return 0;
 }
 
 static struct bio *loop_get_buffer(struct loop_device *lo, struct bio *rbh)
@@ -553,7 +558,7 @@ static int loop_make_request(request_queue_t *q, struct bio *old_bio)
 		up(&lo->lo_bh_mutex);
 	loop_put_buffer(new_bio);
 out:
-	bio_io_error(old_bio);
+	bio_io_error(old_bio, old_bio->bi_size);
 	return 0;
 inactive:
 	spin_unlock_irq(&lo->lo_lock);
@@ -569,13 +574,13 @@ static inline void loop_handle_bio(struct loop_device *lo, struct bio *bio)
 	 */
 	if (lo->lo_flags & LO_FLAGS_DO_BMAP) {
 		ret = do_bio_filebacked(lo, bio);
-		bio_endio(bio, !ret);
+		bio_endio(bio, bio->bi_size, ret);
 	} else {
 		struct bio *rbh = bio->bi_private;
 
 		ret = bio_transfer(lo, bio, rbh);
 
-		bio_endio(rbh, !ret);
+		bio_endio(rbh, rbh->bi_size, ret);
 		loop_put_buffer(bio);
 	}
 }

drivers/block/rd.c

@@ -277,11 +277,10 @@ static int rd_make_request(request_queue_t * q, struct bio *sbh)
 	if (rd_blkdev_bio_IO(sbh, minor))
 		goto fail;
 
-	set_bit(BIO_UPTODATE, &sbh->bi_flags);
-	sbh->bi_end_io(sbh);
+	bio_endio(sbh, sbh->bi_size, 0);
 	return 0;
  fail:
-	bio_io_error(sbh);
+	bio_io_error(sbh, sbh->bi_size);
 	return 0;
 } 
 

fs/bio.c

@@ -401,12 +401,16 @@ int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
 	return 0;
 }
 
-static void bio_end_io_kio(struct bio *bio)
+static int bio_end_io_kio(struct bio *bio, unsigned int bytes_done, int error)
 {
 	struct kiobuf *kio = (struct kiobuf *) bio->bi_private;
 
-	end_kio_request(kio, test_bit(BIO_UPTODATE, &bio->bi_flags));
+	if (bio->bi_size)
+		return 1;
+
+	end_kio_request(kio, error);
 	bio_put(bio);
+	return 0;
 }
 
 /**
@@ -519,15 +523,15 @@ void ll_rw_kio(int rw, struct kiobuf *kio, struct block_device *bdev, sector_t s
 	end_kio_request(kio, !err);
 }
 
-void bio_endio(struct bio *bio, int uptodate)
+int bio_endio(struct bio *bio, unsigned int bytes_done, int error)
 {
-	if (uptodate)
+	if (!error)
 		set_bit(BIO_UPTODATE, &bio->bi_flags);
 	else
 		clear_bit(BIO_UPTODATE, &bio->bi_flags);
 
-	if (bio->bi_end_io)
-		bio->bi_end_io(bio);
+	bio->bi_size -= bytes_done;
+	return bio->bi_end_io(bio, bytes_done, error);
 }
 
 static void __init biovec_init_pools(void)

fs/buffer.c

@@ -2413,12 +2413,16 @@ int brw_kiovec(int rw, int nr, struct kiobuf *iovec[],
 	return err ? err : transferred;
 }
 
-static void end_bio_bh_io_sync(struct bio *bio)
+static int end_bio_bh_io_sync(struct bio *bio, unsigned int bytes_done, int err)
 {
 	struct buffer_head *bh = bio->bi_private;
 
+	if (bio->bi_size)
+		return 1;
+
 	bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
 	bio_put(bio);
+	return 0;
 }
 
 int submit_bh(int rw, struct buffer_head * bh)

fs/direct-io.c

@@ -151,17 +151,21 @@ static struct page *dio_get_page(struct dio *dio)
  * During I/O bi_private points at the dio.  After I/O, bi_private is used to
  * implement a singly-linked list of completed BIOs, at dio->bio_list.
  */
-static void dio_bio_end_io(struct bio *bio)
+static int dio_bio_end_io(struct bio *bio, unsigned int bytes_done, int error)
 {
 	struct dio *dio = bio->bi_private;
 	unsigned long flags;
 
+	if (bio->bi_size)
+		return 1;
+
 	spin_lock_irqsave(&dio->bio_list_lock, flags);
 	bio->bi_private = dio->bio_list;
 	dio->bio_list = bio;
 	if (dio->waiter)
 		wake_up_process(dio->waiter);
 	spin_unlock_irqrestore(&dio->bio_list_lock, flags);
+	return 0;
 }
 
 static int

fs/mpage.c

@@ -36,11 +36,14 @@
  * status of that page is hard.  See end_buffer_async_read() for the details.
  * There is no point in duplicating all that complexity.
  */
-static void mpage_end_io_read(struct bio *bio)
+static int mpage_end_io_read(struct bio *bio, unsigned int bytes_done, int err)
 {
 	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 	struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
 
+	if (bio->bi_size)
+		return 1;
+
 	do {
 		struct page *page = bvec->bv_page;
 
@@ -56,13 +59,17 @@ static void mpage_end_io_read(struct bio *bio)
 		unlock_page(page);
 	} while (bvec >= bio->bi_io_vec);
 	bio_put(bio);
+	return 0;
 }
 
-static void mpage_end_io_write(struct bio *bio)
+static int mpage_end_io_write(struct bio *bio, unsigned int bytes_done, int err)
 {
 	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 	struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
 
+	if (bio->bi_size)
+		return 1;
+
 	do {
 		struct page *page = bvec->bv_page;
 
@@ -74,6 +81,7 @@ static void mpage_end_io_write(struct bio *bio)
 		end_page_writeback(page);
 	} while (bvec >= bio->bi_io_vec);
 	bio_put(bio);
+	return 0;
 }
 
 struct bio *mpage_bio_submit(int rw, struct bio *bio)

include/linux/bio.h

@@ -51,7 +51,7 @@ struct bio_vec {
 };
 
 struct bio;
-typedef void (bio_end_io_t) (struct bio *);
+typedef int (bio_end_io_t) (struct bio *, unsigned int, int);
 typedef void (bio_destructor_t) (struct bio *);
 
 /*
@@ -161,7 +161,7 @@ struct bio {
 #define BIO_SEG_BOUNDARY(q, b1, b2) \
 	BIOVEC_SEG_BOUNDARY((q), __BVEC_END((b1)), __BVEC_START((b2)))
 
-#define bio_io_error(bio) bio_endio((bio), 0)
+#define bio_io_error(bio, bytes) bio_endio((bio), (bytes), -EIO)
 
 /*
  * drivers should not use the __ version unless they _really_ want to
@@ -194,7 +194,7 @@ struct bio {
 extern struct bio *bio_alloc(int, int);
 extern void bio_put(struct bio *);
 
-extern void bio_endio(struct bio *, int);
+extern int bio_endio(struct bio *, unsigned int, int);
 struct request_queue;
 extern inline int bio_phys_segments(struct request_queue *, struct bio *);
 extern inline int bio_hw_segments(struct request_queue *, struct bio *);

mm/highmem.c

@@ -291,12 +291,16 @@ static inline void copy_to_high_bio_irq(struct bio *to, struct bio *from)
 	}
 }
 
-static inline void bounce_end_io(struct bio *bio, mempool_t *pool)
+static inline int bounce_end_io(struct bio *bio, unsigned int bytes_done,
+				int error, mempool_t *pool)
 {
 	struct bio *bio_orig = bio->bi_private;
 	struct bio_vec *bvec, *org_vec;
 	int i;
 
+	if (bio->bi_size)
+		return 1;
+
 	if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
 		goto out_eio;
 
@@ -314,38 +318,43 @@ static inline void bounce_end_io(struct bio *bio, mempool_t *pool)
 	}
 
 out_eio:
-	bio_orig->bi_end_io(bio_orig);
+	bio_endio(bio_orig, bytes_done, error);
 	bio_put(bio);
+	return 0;
 }
 
-static void bounce_end_io_write(struct bio *bio)
+static int bounce_end_io_write(struct bio *bio, unsigned int bytes_done,
+			       int error)
 {
-	bounce_end_io(bio, page_pool);
+	return bounce_end_io(bio, bytes_done, error, page_pool);
 }
 
-static void bounce_end_io_write_isa(struct bio *bio)
+static int bounce_end_io_write_isa(struct bio *bio, unsigned int bytes_done,
+				   int error)
 {
-	bounce_end_io(bio, isa_page_pool);
+	return bounce_end_io(bio, bytes_done, error, isa_page_pool);
 }
 
-static inline void __bounce_end_io_read(struct bio *bio, mempool_t *pool)
+static inline int __bounce_end_io_read(struct bio *bio, unsigned int done,
+				       int error, mempool_t *pool)
 {
 	struct bio *bio_orig = bio->bi_private;
 
 	if (test_bit(BIO_UPTODATE, &bio->bi_flags))
 		copy_to_high_bio_irq(bio_orig, bio);
 
-	bounce_end_io(bio, pool);
+	return bounce_end_io(bio, done, error, pool);
 }
 
-static void bounce_end_io_read(struct bio *bio)
+static int bounce_end_io_read(struct bio *bio, unsigned int bytes_done, int err)
 {
-	__bounce_end_io_read(bio, page_pool);
+	return __bounce_end_io_read(bio, bytes_done, err, page_pool);
 }
 
-static void bounce_end_io_read_isa(struct bio *bio)
+static int bounce_end_io_read_isa(struct bio *bio, unsigned int bytes_done,
+				  int err)
 {
-	return __bounce_end_io_read(bio, isa_page_pool);
+	return __bounce_end_io_read(bio, bytes_done, err, isa_page_pool);
 }
 
 void blk_queue_bounce(request_queue_t *q, struct bio **bio_orig)

mm/page_io.c

@@ -47,22 +47,29 @@ get_swap_bio(int gfp_flags, struct page *page, bio_end_io_t end_io)
 	return bio;
 }
 
-static void end_swap_bio_write(struct bio *bio)
+static int end_swap_bio_write(struct bio *bio, unsigned int bytes_done, int err)
 {
 	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 	struct page *page = bio->bi_io_vec[0].bv_page;
 
+	if (bio->bi_size)
+		return 1;
+
 	if (!uptodate)
 		SetPageError(page);
 	end_page_writeback(page);
 	bio_put(bio);
+	return 0;
 }
 
-static void end_swap_bio_read(struct bio *bio)
+static int end_swap_bio_read(struct bio *bio, unsigned int bytes_done, int err)
 {
 	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 	struct page *page = bio->bi_io_vec[0].bv_page;
 
+	if (bio->bi_size)
+		return 1;
+
 	if (!uptodate) {
 		SetPageError(page);
 		ClearPageUptodate(page);
@@ -71,6 +78,7 @@ static void end_swap_bio_read(struct bio *bio)
 	}
 	unlock_page(page);
 	bio_put(bio);
+	return 0;
 }
 
 /*