[BLOCK] reimplement handling of barrier request

Reimplement handling of barrier requests. * Flexible handling to deal with various capabilities of target devices. * Retry support for falling back. * Tagged queues which don't support ordered tag can do ordered. Signed-off-by: Tejun Heo <htejun@gmail.com> Signed-off-by: Jens Axboe <axboe@suse.de>

[BLOCK] reimplement handling of barrier request
Reimplement handling of barrier requests. * Flexible handling to deal with various capabilities of target devices. * Retry support for falling back. * Tagged queues which don't support ordered tag can do ordered. Signed-off-by: Tejun Heo <htejun@gmail.com> Signed-off-by: Jens Axboe <axboe@suse.de>
Tejun Heo · Jens Axboe
1 parent 52d9e67536
Showing 4 changed files with 359 additions and 192 deletions Side-by-side Diff
block/elevator.c
block/ll_rw_blk.c
include/linux/blkdev.h
include/linux/elevator.h
@@ -304,15 +304,7 @@
  
 	rq->flags &= ~REQ_STARTED;
  
-	/*
-	 * if this is the flush, requeue the original instead and drop the flush
-	 */
-	if (rq->flags & REQ_BAR_FLUSH) {
-		clear_bit(QUEUE_FLAG_FLUSH, &q->queue_flags);
-		rq = rq->end_io_data;
-	}
-
-	__elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
+	__elv_add_request(q, rq, ELEVATOR_INSERT_REQUEUE, 0);
 }
  
 static void elv_drain_elevator(request_queue_t *q)
  
@@ -332,8 +324,19 @@
 void __elv_add_request(request_queue_t *q, struct request *rq, int where,
 		       int plug)
 {
+	struct list_head *pos;
+	unsigned ordseq;
+
+	if (q->ordcolor)
+		rq->flags |= REQ_ORDERED_COLOR;
+
 	if (rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)) {
 		/*
+		 * toggle ordered color
+		 */
+		q->ordcolor ^= 1;
+
+		/*
 		 * barriers implicitly indicate back insertion
 		 */
 		if (where == ELEVATOR_INSERT_SORT)
@@ -393,6 +396,30 @@
 		q->elevator->ops->elevator_add_req_fn(q, rq);
 		break;
  
+	case ELEVATOR_INSERT_REQUEUE:
+		/*
+		 * If ordered flush isn't in progress, we do front
+		 * insertion; otherwise, requests should be requeued
+		 * in ordseq order.
+		 */
+		rq->flags |= REQ_SOFTBARRIER;
+
+		if (q->ordseq == 0) {
+			list_add(&rq->queuelist, &q->queue_head);
+			break;
+		}
+
+		ordseq = blk_ordered_req_seq(rq);
+
+		list_for_each(pos, &q->queue_head) {
+			struct request *pos_rq = list_entry_rq(pos);
+			if (ordseq <= blk_ordered_req_seq(pos_rq))
+				break;
+		}
+
+		list_add_tail(&rq->queuelist, pos);
+		break;
+
 	default:
 		printk(KERN_ERR "%s: bad insertion point %d\n",
 		       __FUNCTION__, where);
  
  
@@ -422,25 +449,16 @@
 {
 	struct request *rq;
  
-	if (unlikely(list_empty(&q->queue_head) &&
-		     !q->elevator->ops->elevator_dispatch_fn(q, 0)))
-		return NULL;
+	while (1) {
+		while (!list_empty(&q->queue_head)) {
+			rq = list_entry_rq(q->queue_head.next);
+			if (blk_do_ordered(q, &rq))
+				return rq;
+		}
  
-	rq = list_entry_rq(q->queue_head.next);
-
-	/*
-	 * if this is a barrier write and the device has to issue a
-	 * flush sequence to support it, check how far we are
-	 */
-	if (blk_fs_request(rq) && blk_barrier_rq(rq)) {
-		BUG_ON(q->ordered == QUEUE_ORDERED_NONE);
-
-		if (q->ordered == QUEUE_ORDERED_FLUSH &&
-		    !blk_barrier_preflush(rq))
-			rq = blk_start_pre_flush(q, rq);
+		if (!q->elevator->ops->elevator_dispatch_fn(q, 0))
+			return NULL;
 	}
-
-	return rq;
 }
  
 struct request *elv_next_request(request_queue_t *q)
  
@@ -593,7 +611,21 @@
 	 * request is released from the driver, io must be done
 	 */
 	if (blk_account_rq(rq)) {
+		struct request *first_rq = list_entry_rq(q->queue_head.next);
+
 		q->in_flight--;
+
+		/*
+		 * Check if the queue is waiting for fs requests to be
+		 * drained for flush sequence.
+		 */
+		if (q->ordseq && q->in_flight == 0 &&
+		    blk_ordered_cur_seq(q) == QUEUE_ORDSEQ_DRAIN &&
+		    blk_ordered_req_seq(first_rq) > QUEUE_ORDSEQ_DRAIN) {
+			blk_ordered_complete_seq(q, QUEUE_ORDSEQ_DRAIN, 0);
+			q->request_fn(q);
+		}
+
 		if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn)
 			e->ops->elevator_completed_req_fn(q, rq);
 	}
@@ -290,8 +290,8 @@
  
 /**
  * blk_queue_ordered - does this queue support ordered writes
- * @q:     the request queue
- * @flag:  see below
+ * @q:        the request queue
+ * @ordered:  one of QUEUE_ORDERED_*
  *
  * Description:
  *   For journalled file systems, doing ordered writes on a commit
  
  
@@ -300,28 +300,30 @@
  *   feature should call this function and indicate so.
  *
  **/
-void blk_queue_ordered(request_queue_t *q, int flag)
+int blk_queue_ordered(request_queue_t *q, unsigned ordered,
+		      prepare_flush_fn *prepare_flush_fn)
 {
-	switch (flag) {
-		case QUEUE_ORDERED_NONE:
-			if (q->flush_rq)
-				kmem_cache_free(request_cachep, q->flush_rq);
-			q->flush_rq = NULL;
-			q->ordered = flag;
-			break;
-		case QUEUE_ORDERED_TAG:
-			q->ordered = flag;
-			break;
-		case QUEUE_ORDERED_FLUSH:
-			q->ordered = flag;
-			if (!q->flush_rq)
-				q->flush_rq = kmem_cache_alloc(request_cachep,
-								GFP_KERNEL);
-			break;
-		default:
-			printk("blk_queue_ordered: bad value %d\n", flag);
-			break;
+	if (ordered & (QUEUE_ORDERED_PREFLUSH | QUEUE_ORDERED_POSTFLUSH) &&
+	    prepare_flush_fn == NULL) {
+		printk(KERN_ERR "blk_queue_ordered: prepare_flush_fn required\n");
+		return -EINVAL;
 	}
+
+	if (ordered != QUEUE_ORDERED_NONE &&
+	    ordered != QUEUE_ORDERED_DRAIN &&
+	    ordered != QUEUE_ORDERED_DRAIN_FLUSH &&
+	    ordered != QUEUE_ORDERED_DRAIN_FUA &&
+	    ordered != QUEUE_ORDERED_TAG &&
+	    ordered != QUEUE_ORDERED_TAG_FLUSH &&
+	    ordered != QUEUE_ORDERED_TAG_FUA) {
+		printk(KERN_ERR "blk_queue_ordered: bad value %d\n", ordered);
+		return -EINVAL;
+	}
+
+	q->next_ordered = ordered;
+	q->prepare_flush_fn = prepare_flush_fn;
+
+	return 0;
 }
  
 EXPORT_SYMBOL(blk_queue_ordered);
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
@@ -346,169 +348,267 @@
 /*
  * Cache flushing for ordered writes handling
  */
-static void blk_pre_flush_end_io(struct request *flush_rq, int error)
+inline unsigned blk_ordered_cur_seq(request_queue_t *q)
 {
-	struct request *rq = flush_rq->end_io_data;
+	if (!q->ordseq)
+		return 0;
+	return 1 << ffz(q->ordseq);
+}
+
+unsigned blk_ordered_req_seq(struct request *rq)
+{
 	request_queue_t *q = rq->q;
  
-	elv_completed_request(q, flush_rq);
+	BUG_ON(q->ordseq == 0);
  
-	rq->flags |= REQ_BAR_PREFLUSH;
+	if (rq == &q->pre_flush_rq)
+		return QUEUE_ORDSEQ_PREFLUSH;
+	if (rq == &q->bar_rq)
+		return QUEUE_ORDSEQ_BAR;
+	if (rq == &q->post_flush_rq)
+		return QUEUE_ORDSEQ_POSTFLUSH;
  
-	if (!flush_rq->errors)
-		elv_requeue_request(q, rq);
-	else {
-		q->end_flush_fn(q, flush_rq);
-		clear_bit(QUEUE_FLAG_FLUSH, &q->queue_flags);
-		q->request_fn(q);
-	}
+	if ((rq->flags & REQ_ORDERED_COLOR) ==
+	    (q->orig_bar_rq->flags & REQ_ORDERED_COLOR))
+		return QUEUE_ORDSEQ_DRAIN;
+	else
+		return QUEUE_ORDSEQ_DONE;
 }
  
-static void blk_post_flush_end_io(struct request *flush_rq, int error)
+void blk_ordered_complete_seq(request_queue_t *q, unsigned seq, int error)
 {
-	struct request *rq = flush_rq->end_io_data;
-	request_queue_t *q = rq->q;
+	struct request *rq;
+	int uptodate;
  
-	elv_completed_request(q, flush_rq);
+	if (error && !q->orderr)
+		q->orderr = error;
  
-	rq->flags |= REQ_BAR_POSTFLUSH;
+	BUG_ON(q->ordseq & seq);
+	q->ordseq |= seq;
  
-	q->end_flush_fn(q, flush_rq);
-	clear_bit(QUEUE_FLAG_FLUSH, &q->queue_flags);
-	q->request_fn(q);
+	if (blk_ordered_cur_seq(q) != QUEUE_ORDSEQ_DONE)
+		return;
+
+	/*
+	 * Okay, sequence complete.
+	 */
+	rq = q->orig_bar_rq;
+	uptodate = q->orderr ? q->orderr : 1;
+
+	q->ordseq = 0;
+
+	end_that_request_first(rq, uptodate, rq->hard_nr_sectors);
+	end_that_request_last(rq, uptodate);
 }
  
-struct request *blk_start_pre_flush(request_queue_t *q, struct request *rq)
+static void pre_flush_end_io(struct request *rq, int error)
 {
-	struct request *flush_rq = q->flush_rq;
+	elv_completed_request(rq->q, rq);
+	blk_ordered_complete_seq(rq->q, QUEUE_ORDSEQ_PREFLUSH, error);
+}
  
-	BUG_ON(!blk_barrier_rq(rq));
+static void bar_end_io(struct request *rq, int error)
+{
+	elv_completed_request(rq->q, rq);
+	blk_ordered_complete_seq(rq->q, QUEUE_ORDSEQ_BAR, error);
+}
  
-	if (test_and_set_bit(QUEUE_FLAG_FLUSH, &q->queue_flags))
-		return NULL;
+static void post_flush_end_io(struct request *rq, int error)
+{
+	elv_completed_request(rq->q, rq);
+	blk_ordered_complete_seq(rq->q, QUEUE_ORDSEQ_POSTFLUSH, error);
+}
  
-	rq_init(q, flush_rq);
-	flush_rq->elevator_private = NULL;
-	flush_rq->flags = REQ_BAR_FLUSH;
-	flush_rq->rq_disk = rq->rq_disk;
-	flush_rq->rl = NULL;
+static void queue_flush(request_queue_t *q, unsigned which)
+{
+	struct request *rq;
+	rq_end_io_fn *end_io;
  
+	if (which == QUEUE_ORDERED_PREFLUSH) {
+		rq = &q->pre_flush_rq;
+		end_io = pre_flush_end_io;
+	} else {
+		rq = &q->post_flush_rq;
+		end_io = post_flush_end_io;
+	}
+
+	rq_init(q, rq);
+	rq->flags = REQ_HARDBARRIER;
+	rq->elevator_private = NULL;
+	rq->rq_disk = q->bar_rq.rq_disk;
+	rq->rl = NULL;
+	rq->end_io = end_io;
+	q->prepare_flush_fn(q, rq);
+
+	__elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
+}
+
+static inline struct request *start_ordered(request_queue_t *q,
+					    struct request *rq)
+{
+	q->bi_size = 0;
+	q->orderr = 0;
+	q->ordered = q->next_ordered;
+	q->ordseq |= QUEUE_ORDSEQ_STARTED;
+
 	/*
-	 * prepare_flush returns 0 if no flush is needed, just mark both
-	 * pre and post flush as done in that case
+	 * Prep proxy barrier request.
 	 */
-	if (!q->prepare_flush_fn(q, flush_rq)) {
-		rq->flags |= REQ_BAR_PREFLUSH | REQ_BAR_POSTFLUSH;
-		clear_bit(QUEUE_FLAG_FLUSH, &q->queue_flags);
-		return rq;
-	}
+	blkdev_dequeue_request(rq);
+	q->orig_bar_rq = rq;
+	rq = &q->bar_rq;
+	rq_init(q, rq);
+	rq->flags = bio_data_dir(q->orig_bar_rq->bio);
+	rq->flags |= q->ordered & QUEUE_ORDERED_FUA ? REQ_FUA : 0;
+	rq->elevator_private = NULL;
+	rq->rl = NULL;
+	init_request_from_bio(rq, q->orig_bar_rq->bio);
+	rq->end_io = bar_end_io;
  
 	/*
-	 * some drivers dequeue requests right away, some only after io
-	 * completion. make sure the request is dequeued.
+	 * Queue ordered sequence.  As we stack them at the head, we
+	 * need to queue in reverse order.  Note that we rely on that
+	 * no fs request uses ELEVATOR_INSERT_FRONT and thus no fs
+	 * request gets inbetween ordered sequence.
 	 */
-	if (!list_empty(&rq->queuelist))
-		blkdev_dequeue_request(rq);
+	if (q->ordered & QUEUE_ORDERED_POSTFLUSH)
+		queue_flush(q, QUEUE_ORDERED_POSTFLUSH);
+	else
+		q->ordseq |= QUEUE_ORDSEQ_POSTFLUSH;
  
-	flush_rq->end_io_data = rq;
-	flush_rq->end_io = blk_pre_flush_end_io;
+	__elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
  
-	__elv_add_request(q, flush_rq, ELEVATOR_INSERT_FRONT, 0);
-	return flush_rq;
+	if (q->ordered & QUEUE_ORDERED_PREFLUSH) {
+		queue_flush(q, QUEUE_ORDERED_PREFLUSH);
+		rq = &q->pre_flush_rq;
+	} else
+		q->ordseq |= QUEUE_ORDSEQ_PREFLUSH;
+
+	if ((q->ordered & QUEUE_ORDERED_TAG) || q->in_flight == 0)
+		q->ordseq |= QUEUE_ORDSEQ_DRAIN;
+	else
+		rq = NULL;
+
+	return rq;
 }
  
-static void blk_start_post_flush(request_queue_t *q, struct request *rq)
+int blk_do_ordered(request_queue_t *q, struct request **rqp)
 {
-	struct request *flush_rq = q->flush_rq;
+	struct request *rq = *rqp, *allowed_rq;
+	int is_barrier = blk_fs_request(rq) && blk_barrier_rq(rq);
  
-	BUG_ON(!blk_barrier_rq(rq));
+	if (!q->ordseq) {
+		if (!is_barrier)
+			return 1;
  
-	rq_init(q, flush_rq);
-	flush_rq->elevator_private = NULL;
-	flush_rq->flags = REQ_BAR_FLUSH;
-	flush_rq->rq_disk = rq->rq_disk;
-	flush_rq->rl = NULL;
+		if (q->next_ordered != QUEUE_ORDERED_NONE) {
+			*rqp = start_ordered(q, rq);
+			return 1;
+		} else {
+			/*
+			 * This can happen when the queue switches to
+			 * ORDERED_NONE while this request is on it.
+			 */
+			blkdev_dequeue_request(rq);
+			end_that_request_first(rq, -EOPNOTSUPP,
+					       rq->hard_nr_sectors);
+			end_that_request_last(rq, -EOPNOTSUPP);
+			*rqp = NULL;
+			return 0;
+		}
+	}
  
-	if (q->prepare_flush_fn(q, flush_rq)) {
-		flush_rq->end_io_data = rq;
-		flush_rq->end_io = blk_post_flush_end_io;
+	if (q->ordered & QUEUE_ORDERED_TAG) {
+		if (is_barrier && rq != &q->bar_rq)
+			*rqp = NULL;
+		return 1;
+	}
  
-		__elv_add_request(q, flush_rq, ELEVATOR_INSERT_FRONT, 0);
-		q->request_fn(q);
+	switch (blk_ordered_cur_seq(q)) {
+	case QUEUE_ORDSEQ_PREFLUSH:
+		allowed_rq = &q->pre_flush_rq;
+		break;
+	case QUEUE_ORDSEQ_BAR:
+		allowed_rq = &q->bar_rq;
+		break;
+	case QUEUE_ORDSEQ_POSTFLUSH:
+		allowed_rq = &q->post_flush_rq;
+		break;
+	default:
+		allowed_rq = NULL;
+		break;
 	}
+
+	if (rq != allowed_rq &&
+	    (blk_fs_request(rq) || rq == &q->pre_flush_rq ||
+	     rq == &q->post_flush_rq))
+		*rqp = NULL;
+
+	return 1;
 }
  
-static inline int blk_check_end_barrier(request_queue_t *q, struct request *rq,
-					int sectors)
+static int flush_dry_bio_endio(struct bio *bio, unsigned int bytes, int error)
 {
-	if (sectors > rq->nr_sectors)
-		sectors = rq->nr_sectors;
+	request_queue_t *q = bio->bi_private;
+	struct bio_vec *bvec;
+	int i;
  
-	rq->nr_sectors -= sectors;
-	return rq->nr_sectors;
+	/*
+	 * This is dry run, restore bio_sector and size.  We'll finish
+	 * this request again with the original bi_end_io after an
+	 * error occurs or post flush is complete.
+	 */
+	q->bi_size += bytes;
+
+	if (bio->bi_size)
+		return 1;
+
+	/* Rewind bvec's */
+	bio->bi_idx = 0;
+	bio_for_each_segment(bvec, bio, i) {
+		bvec->bv_len += bvec->bv_offset;
+		bvec->bv_offset = 0;
+	}
+
+	/* Reset bio */
+	set_bit(BIO_UPTODATE, &bio->bi_flags);
+	bio->bi_size = q->bi_size;
+	bio->bi_sector -= (q->bi_size >> 9);
+	q->bi_size = 0;
+
+	return 0;
 }
  
-static int __blk_complete_barrier_rq(request_queue_t *q, struct request *rq,
-				     int sectors, int queue_locked)
+static inline int ordered_bio_endio(struct request *rq, struct bio *bio,
+				    unsigned int nbytes, int error)
 {
-	if (q->ordered != QUEUE_ORDERED_FLUSH)
+	request_queue_t *q = rq->q;
+	bio_end_io_t *endio;
+	void *private;
+
+	if (&q->bar_rq != rq)
 		return 0;
-	if (!blk_fs_request(rq) || !blk_barrier_rq(rq))
-		return 0;
-	if (blk_barrier_postflush(rq))
-		return 0;
  
-	if (!blk_check_end_barrier(q, rq, sectors)) {
-		unsigned long flags = 0;
+	/*
+	 * Okay, this is the barrier request in progress, dry finish it.
+	 */
+	if (error && !q->orderr)
+		q->orderr = error;
  
-		if (!queue_locked)
-			spin_lock_irqsave(q->queue_lock, flags);
+	endio = bio->bi_end_io;
+	private = bio->bi_private;
+	bio->bi_end_io = flush_dry_bio_endio;
+	bio->bi_private = q;
  
-		blk_start_post_flush(q, rq);
+	bio_endio(bio, nbytes, error);
  
-		if (!queue_locked)
-			spin_unlock_irqrestore(q->queue_lock, flags);
-	}
+	bio->bi_end_io = endio;
+	bio->bi_private = private;
  
 	return 1;
 }
  
 /**
- * blk_complete_barrier_rq - complete possible barrier request
- * @q:  the request queue for the device
- * @rq:  the request
- * @sectors:  number of sectors to complete
- *
- * Description:
- *   Used in driver end_io handling to determine whether to postpone
- *   completion of a barrier request until a post flush has been done. This
- *   is the unlocked variant, used if the caller doesn't already hold the
- *   queue lock.
- **/
-int blk_complete_barrier_rq(request_queue_t *q, struct request *rq, int sectors)
-{
-	return __blk_complete_barrier_rq(q, rq, sectors, 0);
-}
-EXPORT_SYMBOL(blk_complete_barrier_rq);
-
-/**
- * blk_complete_barrier_rq_locked - complete possible barrier request
- * @q:  the request queue for the device
- * @rq:  the request
- * @sectors:  number of sectors to complete
- *
- * Description:
- *   See blk_complete_barrier_rq(). This variant must be used if the caller
- *   holds the queue lock.
- **/
-int blk_complete_barrier_rq_locked(request_queue_t *q, struct request *rq,
-				   int sectors)
-{
-	return __blk_complete_barrier_rq(q, rq, sectors, 1);
-}
-EXPORT_SYMBOL(blk_complete_barrier_rq_locked);
-
-/**
  * blk_queue_bounce_limit - set bounce buffer limit for queue
  * @q:  the request queue for the device
  * @dma_addr:   bus address limit
@@ -1047,6 +1147,7 @@
 	"REQ_SORTED",
 	"REQ_SOFTBARRIER",
 	"REQ_HARDBARRIER",
+	"REQ_FUA",
 	"REQ_CMD",
 	"REQ_NOMERGE",
 	"REQ_STARTED",
@@ -1066,6 +1167,7 @@
 	"REQ_PM_SUSPEND",
 	"REQ_PM_RESUME",
 	"REQ_PM_SHUTDOWN",
+	"REQ_ORDERED_COLOR",
 };
  
 void blk_dump_rq_flags(struct request *rq, char *msg)
@@ -1643,8 +1745,6 @@
 	if (q->queue_tags)
 		__blk_queue_free_tags(q);
  
-	blk_queue_ordered(q, QUEUE_ORDERED_NONE);
-
 	kmem_cache_free(requestq_cachep, q);
 }
  
@@ -2714,7 +2814,7 @@
 	spin_lock_prefetch(q->queue_lock);
  
 	barrier = bio_barrier(bio);
-	if (unlikely(barrier) && (q->ordered == QUEUE_ORDERED_NONE)) {
+	if (unlikely(barrier) && (q->next_ordered == QUEUE_ORDERED_NONE)) {
 		err = -EOPNOTSUPP;
 		goto end_io;
 	}
@@ -3075,7 +3175,8 @@
 		if (nr_bytes >= bio->bi_size) {
 			req->bio = bio->bi_next;
 			nbytes = bio->bi_size;
-			bio_endio(bio, nbytes, error);
+			if (!ordered_bio_endio(req, bio, nbytes, error))
+				bio_endio(bio, nbytes, error);
 			next_idx = 0;
 			bio_nbytes = 0;
 		} else {
@@ -3130,7 +3231,8 @@
 	 * if the request wasn't completed, update state
 	 */
 	if (bio_nbytes) {
-		bio_endio(bio, bio_nbytes, error);
+		if (!ordered_bio_endio(req, bio, bio_nbytes, error))
+			bio_endio(bio, bio_nbytes, error);
 		bio->bi_idx += next_idx;
 		bio_iovec(bio)->bv_offset += nr_bytes;
 		bio_iovec(bio)->bv_len -= nr_bytes;
@@ -207,6 +207,7 @@
 	__REQ_SORTED,		/* elevator knows about this request */
 	__REQ_SOFTBARRIER,	/* may not be passed by ioscheduler */
 	__REQ_HARDBARRIER,	/* may not be passed by drive either */
+	__REQ_FUA,		/* forced unit access */
 	__REQ_CMD,		/* is a regular fs rw request */
 	__REQ_NOMERGE,		/* don't touch this for merging */
 	__REQ_STARTED,		/* drive already may have started this one */
@@ -230,9 +231,7 @@
 	__REQ_PM_SUSPEND,	/* suspend request */
 	__REQ_PM_RESUME,	/* resume request */
 	__REQ_PM_SHUTDOWN,	/* shutdown request */
-	__REQ_BAR_PREFLUSH,	/* barrier pre-flush done */
-	__REQ_BAR_POSTFLUSH,	/* barrier post-flush */
-	__REQ_BAR_FLUSH,	/* rq is the flush request */
+	__REQ_ORDERED_COLOR,	/* is before or after barrier */
 	__REQ_NR_BITS,		/* stops here */
 };
  
@@ -241,6 +240,7 @@
 #define REQ_SORTED	(1 << __REQ_SORTED)
 #define REQ_SOFTBARRIER	(1 << __REQ_SOFTBARRIER)
 #define REQ_HARDBARRIER	(1 << __REQ_HARDBARRIER)
+#define REQ_FUA		(1 << __REQ_FUA)
 #define REQ_CMD		(1 << __REQ_CMD)
 #define REQ_NOMERGE	(1 << __REQ_NOMERGE)
 #define REQ_STARTED	(1 << __REQ_STARTED)
@@ -260,9 +260,7 @@
 #define REQ_PM_SUSPEND	(1 << __REQ_PM_SUSPEND)
 #define REQ_PM_RESUME	(1 << __REQ_PM_RESUME)
 #define REQ_PM_SHUTDOWN	(1 << __REQ_PM_SHUTDOWN)
-#define REQ_BAR_PREFLUSH	(1 << __REQ_BAR_PREFLUSH)
-#define REQ_BAR_POSTFLUSH	(1 << __REQ_BAR_POSTFLUSH)
-#define REQ_BAR_FLUSH	(1 << __REQ_BAR_FLUSH)
+#define REQ_ORDERED_COLOR	(1 << __REQ_ORDERED_COLOR)
  
 /*
  * State information carried for REQ_PM_SUSPEND and REQ_PM_RESUME
@@ -292,8 +290,7 @@
 typedef int (merge_bvec_fn) (request_queue_t *, struct bio *, struct bio_vec *);
 typedef void (activity_fn) (void *data, int rw);
 typedef int (issue_flush_fn) (request_queue_t *, struct gendisk *, sector_t *);
-typedef int (prepare_flush_fn) (request_queue_t *, struct request *);
-typedef void (end_flush_fn) (request_queue_t *, struct request *);
+typedef void (prepare_flush_fn) (request_queue_t *, struct request *);
  
 enum blk_queue_state {
 	Queue_down,
@@ -335,7 +332,6 @@
 	activity_fn		*activity_fn;
 	issue_flush_fn		*issue_flush_fn;
 	prepare_flush_fn	*prepare_flush_fn;
-	end_flush_fn		*end_flush_fn;
  
 	/*
 	 * Dispatch queue sorting
  
@@ -420,16 +416,13 @@
 	/*
 	 * reserved for flush operations
 	 */
-	struct request		*flush_rq;
-	unsigned char		ordered;
+	unsigned int		ordered, next_ordered, ordseq;
+	int			orderr, ordcolor;
+	struct request		pre_flush_rq, bar_rq, post_flush_rq;
+	struct request		*orig_bar_rq;
+	unsigned int		bi_size;
 };
  
-enum {
-	QUEUE_ORDERED_NONE,
-	QUEUE_ORDERED_TAG,
-	QUEUE_ORDERED_FLUSH,
-};
-
 #define RQ_INACTIVE		(-1)
 #define RQ_ACTIVE		1
 #define RQ_SCSI_BUSY		0xffff
  
  
@@ -445,12 +438,51 @@
 #define QUEUE_FLAG_REENTER	6	/* Re-entrancy avoidance */
 #define QUEUE_FLAG_PLUGGED	7	/* queue is plugged */
 #define QUEUE_FLAG_ELVSWITCH	8	/* don't use elevator, just do FIFO */
-#define QUEUE_FLAG_FLUSH	9	/* doing barrier flush sequence */
  
+enum {
+	/*
+	 * Hardbarrier is supported with one of the following methods.
+	 *
+	 * NONE		: hardbarrier unsupported
+	 * DRAIN	: ordering by draining is enough
+	 * DRAIN_FLUSH	: ordering by draining w/ pre and post flushes
+	 * DRAIN_FUA	: ordering by draining w/ pre flush and FUA write
+	 * TAG		: ordering by tag is enough
+	 * TAG_FLUSH	: ordering by tag w/ pre and post flushes
+	 * TAG_FUA	: ordering by tag w/ pre flush and FUA write
+	 */
+	QUEUE_ORDERED_NONE	= 0x00,
+	QUEUE_ORDERED_DRAIN	= 0x01,
+	QUEUE_ORDERED_TAG	= 0x02,
+
+	QUEUE_ORDERED_PREFLUSH	= 0x10,
+	QUEUE_ORDERED_POSTFLUSH	= 0x20,
+	QUEUE_ORDERED_FUA	= 0x40,
+
+	QUEUE_ORDERED_DRAIN_FLUSH = QUEUE_ORDERED_DRAIN |
+			QUEUE_ORDERED_PREFLUSH | QUEUE_ORDERED_POSTFLUSH,
+	QUEUE_ORDERED_DRAIN_FUA	= QUEUE_ORDERED_DRAIN |
+			QUEUE_ORDERED_PREFLUSH | QUEUE_ORDERED_FUA,
+	QUEUE_ORDERED_TAG_FLUSH	= QUEUE_ORDERED_TAG |
+			QUEUE_ORDERED_PREFLUSH | QUEUE_ORDERED_POSTFLUSH,
+	QUEUE_ORDERED_TAG_FUA	= QUEUE_ORDERED_TAG |
+			QUEUE_ORDERED_PREFLUSH | QUEUE_ORDERED_FUA,
+
+	/*
+	 * Ordered operation sequence
+	 */
+	QUEUE_ORDSEQ_STARTED	= 0x01,	/* flushing in progress */
+	QUEUE_ORDSEQ_DRAIN	= 0x02,	/* waiting for the queue to be drained */
+	QUEUE_ORDSEQ_PREFLUSH	= 0x04,	/* pre-flushing in progress */
+	QUEUE_ORDSEQ_BAR	= 0x08,	/* original barrier req in progress */
+	QUEUE_ORDSEQ_POSTFLUSH	= 0x10,	/* post-flushing in progress */
+	QUEUE_ORDSEQ_DONE	= 0x20,
+};
+
 #define blk_queue_plugged(q)	test_bit(QUEUE_FLAG_PLUGGED, &(q)->queue_flags)
 #define blk_queue_tagged(q)	test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
 #define blk_queue_stopped(q)	test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
-#define blk_queue_flushing(q)	test_bit(QUEUE_FLAG_FLUSH, &(q)->queue_flags)
+#define blk_queue_flushing(q)	((q)->ordseq)
  
 #define blk_fs_request(rq)	((rq)->flags & REQ_CMD)
 #define blk_pc_request(rq)	((rq)->flags & REQ_BLOCK_PC)
@@ -466,8 +498,7 @@
  
 #define blk_sorted_rq(rq)	((rq)->flags & REQ_SORTED)
 #define blk_barrier_rq(rq)	((rq)->flags & REQ_HARDBARRIER)
-#define blk_barrier_preflush(rq)	((rq)->flags & REQ_BAR_PREFLUSH)
-#define blk_barrier_postflush(rq)	((rq)->flags & REQ_BAR_POSTFLUSH)
+#define blk_fua_rq(rq)		((rq)->flags & REQ_FUA)
  
 #define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)
  
  
@@ -665,11 +696,12 @@
 extern void blk_queue_merge_bvec(request_queue_t *, merge_bvec_fn *);
 extern void blk_queue_dma_alignment(request_queue_t *, int);
 extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
-extern void blk_queue_ordered(request_queue_t *, int);
+extern int blk_queue_ordered(request_queue_t *, unsigned, prepare_flush_fn *);
 extern void blk_queue_issue_flush_fn(request_queue_t *, issue_flush_fn *);
-extern struct request *blk_start_pre_flush(request_queue_t *,struct request *);
-extern int blk_complete_barrier_rq(request_queue_t *, struct request *, int);
-extern int blk_complete_barrier_rq_locked(request_queue_t *, struct request *, int);
+extern int blk_do_ordered(request_queue_t *, struct request **);
+extern unsigned blk_ordered_cur_seq(request_queue_t *);
+extern unsigned blk_ordered_req_seq(struct request *);
+extern void blk_ordered_complete_seq(request_queue_t *, unsigned, int);
  
 extern int blk_rq_map_sg(request_queue_t *, struct request *, struct scatterlist *);
 extern void blk_dump_rq_flags(struct request *, char *);
@@ -130,6 +130,7 @@
 #define ELEVATOR_INSERT_FRONT	1
 #define ELEVATOR_INSERT_BACK	2
 #define ELEVATOR_INSERT_SORT	3
+#define ELEVATOR_INSERT_REQUEUE	4
  
 /*
  * return values from elevator_may_queue_fn
...	...	@@ -304,15 +304,7 @@
304	304
305	305	rq->flags &= ~REQ_STARTED;
306	306
307		- /*
308		- * if this is the flush, requeue the original instead and drop the flush
309		- */
310		- if (rq->flags & REQ_BAR_FLUSH) {
311		- clear_bit(QUEUE_FLAG_FLUSH, &q->queue_flags);
312		- rq = rq->end_io_data;
313		- }
314		-
315		- __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
	307	+ __elv_add_request(q, rq, ELEVATOR_INSERT_REQUEUE, 0);
316	308	}
317	309
318	310	static void elv_drain_elevator(request_queue_t *q)
319	311
...	...	@@ -332,8 +324,19 @@
332	324	void __elv_add_request(request_queue_t q, struct request rq, int where,
333	325	int plug)
334	326	{
	327	+ struct list_head *pos;
	328	+ unsigned ordseq;
	329	+
	330	+ if (q->ordcolor)
	331	+ rq->flags \|= REQ_ORDERED_COLOR;
	332	+
335	333	if (rq->flags & (REQ_SOFTBARRIER \| REQ_HARDBARRIER)) {
336	334	/*
	335	+ * toggle ordered color
	336	+ */
	337	+ q->ordcolor ^= 1;
	338	+
	339	+ /*
337	340	* barriers implicitly indicate back insertion
338	341	*/
339	342	if (where == ELEVATOR_INSERT_SORT)
...	...	@@ -393,6 +396,30 @@
393	396	q->elevator->ops->elevator_add_req_fn(q, rq);
394	397	break;
395	398
	399	+ case ELEVATOR_INSERT_REQUEUE:
	400	+ /*
	401	+ * If ordered flush isn't in progress, we do front
	402	+ * insertion; otherwise, requests should be requeued
	403	+ * in ordseq order.
	404	+ */
	405	+ rq->flags \|= REQ_SOFTBARRIER;
	406	+
	407	+ if (q->ordseq == 0) {
	408	+ list_add(&rq->queuelist, &q->queue_head);
	409	+ break;
	410	+ }
	411	+
	412	+ ordseq = blk_ordered_req_seq(rq);
	413	+
	414	+ list_for_each(pos, &q->queue_head) {
	415	+ struct request *pos_rq = list_entry_rq(pos);
	416	+ if (ordseq <= blk_ordered_req_seq(pos_rq))
	417	+ break;
	418	+ }
	419	+
	420	+ list_add_tail(&rq->queuelist, pos);
	421	+ break;
	422	+
396	423	default:
397	424	printk(KERN_ERR "%s: bad insertion point %d\n",
398	425	__FUNCTION__, where);
399	426
400	427
...	...	@@ -422,25 +449,16 @@
422	449	{
423	450	struct request *rq;
424	451
425		- if (unlikely(list_empty(&q->queue_head) &&
426		- !q->elevator->ops->elevator_dispatch_fn(q, 0)))
427		- return NULL;
	452	+ while (1) {
	453	+ while (!list_empty(&q->queue_head)) {
	454	+ rq = list_entry_rq(q->queue_head.next);
	455	+ if (blk_do_ordered(q, &rq))
	456	+ return rq;
	457	+ }
428	458
429		- rq = list_entry_rq(q->queue_head.next);
430		-
431		- /*
432		- * if this is a barrier write and the device has to issue a
433		- * flush sequence to support it, check how far we are
434		- */
435		- if (blk_fs_request(rq) && blk_barrier_rq(rq)) {
436		- BUG_ON(q->ordered == QUEUE_ORDERED_NONE);
437		-
438		- if (q->ordered == QUEUE_ORDERED_FLUSH &&
439		- !blk_barrier_preflush(rq))
440		- rq = blk_start_pre_flush(q, rq);
	459	+ if (!q->elevator->ops->elevator_dispatch_fn(q, 0))
	460	+ return NULL;
441	461	}
442		-
443		- return rq;
444	462	}
445	463
446	464	struct request elv_next_request(request_queue_t q)
447	465
...	...	@@ -593,7 +611,21 @@
593	611	* request is released from the driver, io must be done
594	612	*/
595	613	if (blk_account_rq(rq)) {
	614	+ struct request *first_rq = list_entry_rq(q->queue_head.next);
	615	+
596	616	q->in_flight--;
	617	+
	618	+ /*
	619	+ * Check if the queue is waiting for fs requests to be
	620	+ * drained for flush sequence.
	621	+ */
	622	+ if (q->ordseq && q->in_flight == 0 &&
	623	+ blk_ordered_cur_seq(q) == QUEUE_ORDSEQ_DRAIN &&
	624	+ blk_ordered_req_seq(first_rq) > QUEUE_ORDSEQ_DRAIN) {
	625	+ blk_ordered_complete_seq(q, QUEUE_ORDSEQ_DRAIN, 0);
	626	+ q->request_fn(q);
	627	+ }
	628	+
597	629	if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn)
598	630	e->ops->elevator_completed_req_fn(q, rq);
599	631	}
...	...	@@ -290,8 +290,8 @@
290	290
291	291	/**
292	292	* blk_queue_ordered - does this queue support ordered writes
293		- * @q: the request queue
294		- * @flag: see below
	293	+ * @q: the request queue
	294	+ * @ordered: one of QUEUE_ORDERED_*
295	295	*
296	296	* Description:
297	297	* For journalled file systems, doing ordered writes on a commit
298	298
299	299
...	...	@@ -300,28 +300,30 @@
300	300	* feature should call this function and indicate so.
301	301	*
302	302	**/
303		-void blk_queue_ordered(request_queue_t *q, int flag)
	303	+int blk_queue_ordered(request_queue_t *q, unsigned ordered,
	304	+ prepare_flush_fn *prepare_flush_fn)
304	305	{
305		- switch (flag) {
306		- case QUEUE_ORDERED_NONE:
307		- if (q->flush_rq)
308		- kmem_cache_free(request_cachep, q->flush_rq);
309		- q->flush_rq = NULL;
310		- q->ordered = flag;
311		- break;
312		- case QUEUE_ORDERED_TAG:
313		- q->ordered = flag;
314		- break;
315		- case QUEUE_ORDERED_FLUSH:
316		- q->ordered = flag;
317		- if (!q->flush_rq)
318		- q->flush_rq = kmem_cache_alloc(request_cachep,
319		- GFP_KERNEL);
320		- break;
321		- default:
322		- printk("blk_queue_ordered: bad value %d\n", flag);
323		- break;
	306	+ if (ordered & (QUEUE_ORDERED_PREFLUSH \| QUEUE_ORDERED_POSTFLUSH) &&
	307	+ prepare_flush_fn == NULL) {
	308	+ printk(KERN_ERR "blk_queue_ordered: prepare_flush_fn required\n");
	309	+ return -EINVAL;
324	310	}
	311	+
	312	+ if (ordered != QUEUE_ORDERED_NONE &&
	313	+ ordered != QUEUE_ORDERED_DRAIN &&
	314	+ ordered != QUEUE_ORDERED_DRAIN_FLUSH &&
	315	+ ordered != QUEUE_ORDERED_DRAIN_FUA &&
	316	+ ordered != QUEUE_ORDERED_TAG &&
	317	+ ordered != QUEUE_ORDERED_TAG_FLUSH &&
	318	+ ordered != QUEUE_ORDERED_TAG_FUA) {
	319	+ printk(KERN_ERR "blk_queue_ordered: bad value %d\n", ordered);
	320	+ return -EINVAL;
	321	+ }
	322	+
	323	+ q->next_ordered = ordered;
	324	+ q->prepare_flush_fn = prepare_flush_fn;
	325	+
	326	+ return 0;
325	327	}
326	328
327	329	EXPORT_SYMBOL(blk_queue_ordered);
328	330
329	331
330	332
331	333
332	334
333	335
334	336
335	337
336	338
337	339
338	340
339	341
340	342
341	343
342	344
343	345
344	346
345	347
346	348
347	349
348	350
349	351
350	352
351	353
352	354
353	355
354	356
355	357
356	358
357	359
358	360
359	361
360	362
361	363
362	364
363	365
364	366
365	367
366	368
...	...	@@ -346,169 +348,267 @@
346	348	/*
347	349	* Cache flushing for ordered writes handling
348	350	*/
349		-static void blk_pre_flush_end_io(struct request *flush_rq, int error)
	351	+inline unsigned blk_ordered_cur_seq(request_queue_t *q)
350	352	{
351		- struct request *rq = flush_rq->end_io_data;
	353	+ if (!q->ordseq)
	354	+ return 0;
	355	+ return 1 << ffz(q->ordseq);
	356	+}
	357	+
	358	+unsigned blk_ordered_req_seq(struct request *rq)
	359	+{
352	360	request_queue_t *q = rq->q;
353	361
354		- elv_completed_request(q, flush_rq);
	362	+ BUG_ON(q->ordseq == 0);
355	363
356		- rq->flags \|= REQ_BAR_PREFLUSH;
	364	+ if (rq == &q->pre_flush_rq)
	365	+ return QUEUE_ORDSEQ_PREFLUSH;
	366	+ if (rq == &q->bar_rq)
	367	+ return QUEUE_ORDSEQ_BAR;
	368	+ if (rq == &q->post_flush_rq)
	369	+ return QUEUE_ORDSEQ_POSTFLUSH;
357	370
358		- if (!flush_rq->errors)
359		- elv_requeue_request(q, rq);
360		- else {
361		- q->end_flush_fn(q, flush_rq);
362		- clear_bit(QUEUE_FLAG_FLUSH, &q->queue_flags);
363		- q->request_fn(q);
364		- }
	371	+ if ((rq->flags & REQ_ORDERED_COLOR) ==
	372	+ (q->orig_bar_rq->flags & REQ_ORDERED_COLOR))
	373	+ return QUEUE_ORDSEQ_DRAIN;
	374	+ else
	375	+ return QUEUE_ORDSEQ_DONE;
365	376	}
366	377
367		-static void blk_post_flush_end_io(struct request *flush_rq, int error)
	378	+void blk_ordered_complete_seq(request_queue_t *q, unsigned seq, int error)
368	379	{
369		- struct request *rq = flush_rq->end_io_data;
370		- request_queue_t *q = rq->q;
	380	+ struct request *rq;
	381	+ int uptodate;
371	382
372		- elv_completed_request(q, flush_rq);
	383	+ if (error && !q->orderr)
	384	+ q->orderr = error;
373	385
374		- rq->flags \|= REQ_BAR_POSTFLUSH;
	386	+ BUG_ON(q->ordseq & seq);
	387	+ q->ordseq \|= seq;
375	388
376		- q->end_flush_fn(q, flush_rq);
377		- clear_bit(QUEUE_FLAG_FLUSH, &q->queue_flags);
378		- q->request_fn(q);
	389	+ if (blk_ordered_cur_seq(q) != QUEUE_ORDSEQ_DONE)
	390	+ return;
	391	+
	392	+ /*
	393	+ * Okay, sequence complete.
	394	+ */
	395	+ rq = q->orig_bar_rq;
	396	+ uptodate = q->orderr ? q->orderr : 1;
	397	+
	398	+ q->ordseq = 0;
	399	+
	400	+ end_that_request_first(rq, uptodate, rq->hard_nr_sectors);
	401	+ end_that_request_last(rq, uptodate);
379	402	}
380	403
381		-struct request blk_start_pre_flush(request_queue_t q, struct request *rq)
	404	+static void pre_flush_end_io(struct request *rq, int error)
382	405	{
383		- struct request *flush_rq = q->flush_rq;
	406	+ elv_completed_request(rq->q, rq);
	407	+ blk_ordered_complete_seq(rq->q, QUEUE_ORDSEQ_PREFLUSH, error);
	408	+}
384	409
385		- BUG_ON(!blk_barrier_rq(rq));
	410	+static void bar_end_io(struct request *rq, int error)
	411	+{
	412	+ elv_completed_request(rq->q, rq);
	413	+ blk_ordered_complete_seq(rq->q, QUEUE_ORDSEQ_BAR, error);
	414	+}
386	415
387		- if (test_and_set_bit(QUEUE_FLAG_FLUSH, &q->queue_flags))
388		- return NULL;
	416	+static void post_flush_end_io(struct request *rq, int error)
	417	+{
	418	+ elv_completed_request(rq->q, rq);
	419	+ blk_ordered_complete_seq(rq->q, QUEUE_ORDSEQ_POSTFLUSH, error);
	420	+}
389	421
390		- rq_init(q, flush_rq);
391		- flush_rq->elevator_private = NULL;
392		- flush_rq->flags = REQ_BAR_FLUSH;
393		- flush_rq->rq_disk = rq->rq_disk;
394		- flush_rq->rl = NULL;
	422	+static void queue_flush(request_queue_t *q, unsigned which)
	423	+{
	424	+ struct request *rq;
	425	+ rq_end_io_fn *end_io;
395	426
	427	+ if (which == QUEUE_ORDERED_PREFLUSH) {
	428	+ rq = &q->pre_flush_rq;
	429	+ end_io = pre_flush_end_io;
	430	+ } else {
	431	+ rq = &q->post_flush_rq;
	432	+ end_io = post_flush_end_io;
	433	+ }
	434	+
	435	+ rq_init(q, rq);
	436	+ rq->flags = REQ_HARDBARRIER;
	437	+ rq->elevator_private = NULL;
	438	+ rq->rq_disk = q->bar_rq.rq_disk;
	439	+ rq->rl = NULL;
	440	+ rq->end_io = end_io;
	441	+ q->prepare_flush_fn(q, rq);
	442	+
	443	+ __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
	444	+}
	445	+
	446	+static inline struct request start_ordered(request_queue_t q,
	447	+ struct request *rq)
	448	+{
	449	+ q->bi_size = 0;
	450	+ q->orderr = 0;
	451	+ q->ordered = q->next_ordered;
	452	+ q->ordseq \|= QUEUE_ORDSEQ_STARTED;
	453	+
396	454	/*
397		- * prepare_flush returns 0 if no flush is needed, just mark both
398		- * pre and post flush as done in that case
	455	+ * Prep proxy barrier request.
399	456	*/
400		- if (!q->prepare_flush_fn(q, flush_rq)) {
401		- rq->flags \|= REQ_BAR_PREFLUSH \| REQ_BAR_POSTFLUSH;
402		- clear_bit(QUEUE_FLAG_FLUSH, &q->queue_flags);
403		- return rq;
404		- }
	457	+ blkdev_dequeue_request(rq);
	458	+ q->orig_bar_rq = rq;
	459	+ rq = &q->bar_rq;
	460	+ rq_init(q, rq);
	461	+ rq->flags = bio_data_dir(q->orig_bar_rq->bio);
	462	+ rq->flags \|= q->ordered & QUEUE_ORDERED_FUA ? REQ_FUA : 0;
	463	+ rq->elevator_private = NULL;
	464	+ rq->rl = NULL;
	465	+ init_request_from_bio(rq, q->orig_bar_rq->bio);
	466	+ rq->end_io = bar_end_io;
405	467
406	468	/*
407		- * some drivers dequeue requests right away, some only after io
408		- * completion. make sure the request is dequeued.
	469	+ * Queue ordered sequence. As we stack them at the head, we
	470	+ * need to queue in reverse order. Note that we rely on that
	471	+ * no fs request uses ELEVATOR_INSERT_FRONT and thus no fs
	472	+ * request gets inbetween ordered sequence.
409	473	*/
410		- if (!list_empty(&rq->queuelist))
411		- blkdev_dequeue_request(rq);
	474	+ if (q->ordered & QUEUE_ORDERED_POSTFLUSH)
	475	+ queue_flush(q, QUEUE_ORDERED_POSTFLUSH);
	476	+ else
	477	+ q->ordseq \|= QUEUE_ORDSEQ_POSTFLUSH;
412	478
413		- flush_rq->end_io_data = rq;
414		- flush_rq->end_io = blk_pre_flush_end_io;
	479	+ __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
415	480
416		- __elv_add_request(q, flush_rq, ELEVATOR_INSERT_FRONT, 0);
417		- return flush_rq;
	481	+ if (q->ordered & QUEUE_ORDERED_PREFLUSH) {
	482	+ queue_flush(q, QUEUE_ORDERED_PREFLUSH);
	483	+ rq = &q->pre_flush_rq;
	484	+ } else
	485	+ q->ordseq \|= QUEUE_ORDSEQ_PREFLUSH;
	486	+
	487	+ if ((q->ordered & QUEUE_ORDERED_TAG) \|\| q->in_flight == 0)
	488	+ q->ordseq \|= QUEUE_ORDSEQ_DRAIN;
	489	+ else
	490	+ rq = NULL;
	491	+
	492	+ return rq;
418	493	}
419	494
420		-static void blk_start_post_flush(request_queue_t q, struct request rq)
	495	+int blk_do_ordered(request_queue_t q, struct request *rqp)
421	496	{
422		- struct request *flush_rq = q->flush_rq;
	497	+ struct request rq = rqp, *allowed_rq;
	498	+ int is_barrier = blk_fs_request(rq) && blk_barrier_rq(rq);
423	499
424		- BUG_ON(!blk_barrier_rq(rq));
	500	+ if (!q->ordseq) {
	501	+ if (!is_barrier)
	502	+ return 1;
425	503
426		- rq_init(q, flush_rq);
427		- flush_rq->elevator_private = NULL;
428		- flush_rq->flags = REQ_BAR_FLUSH;
429		- flush_rq->rq_disk = rq->rq_disk;
430		- flush_rq->rl = NULL;
	504	+ if (q->next_ordered != QUEUE_ORDERED_NONE) {
	505	+ *rqp = start_ordered(q, rq);
	506	+ return 1;
	507	+ } else {
	508	+ /*
	509	+ * This can happen when the queue switches to
	510	+ * ORDERED_NONE while this request is on it.
	511	+ */
	512	+ blkdev_dequeue_request(rq);
	513	+ end_that_request_first(rq, -EOPNOTSUPP,
	514	+ rq->hard_nr_sectors);
	515	+ end_that_request_last(rq, -EOPNOTSUPP);
	516	+ *rqp = NULL;
	517	+ return 0;
	518	+ }
	519	+ }
431	520
432		- if (q->prepare_flush_fn(q, flush_rq)) {
433		- flush_rq->end_io_data = rq;
434		- flush_rq->end_io = blk_post_flush_end_io;
	521	+ if (q->ordered & QUEUE_ORDERED_TAG) {
	522	+ if (is_barrier && rq != &q->bar_rq)
	523	+ *rqp = NULL;
	524	+ return 1;
	525	+ }
435	526
436		- __elv_add_request(q, flush_rq, ELEVATOR_INSERT_FRONT, 0);
437		- q->request_fn(q);
	527	+ switch (blk_ordered_cur_seq(q)) {
	528	+ case QUEUE_ORDSEQ_PREFLUSH:
	529	+ allowed_rq = &q->pre_flush_rq;
	530	+ break;
	531	+ case QUEUE_ORDSEQ_BAR:
	532	+ allowed_rq = &q->bar_rq;
	533	+ break;
	534	+ case QUEUE_ORDSEQ_POSTFLUSH:
	535	+ allowed_rq = &q->post_flush_rq;
	536	+ break;
	537	+ default:
	538	+ allowed_rq = NULL;
	539	+ break;
438	540	}
	541	+
	542	+ if (rq != allowed_rq &&
	543	+ (blk_fs_request(rq) \|\| rq == &q->pre_flush_rq \|\|
	544	+ rq == &q->post_flush_rq))
	545	+ *rqp = NULL;
	546	+
	547	+ return 1;
439	548	}
440	549
441		-static inline int blk_check_end_barrier(request_queue_t q, struct request rq,
442		- int sectors)
	550	+static int flush_dry_bio_endio(struct bio *bio, unsigned int bytes, int error)
443	551	{
444		- if (sectors > rq->nr_sectors)
445		- sectors = rq->nr_sectors;
	552	+ request_queue_t *q = bio->bi_private;
	553	+ struct bio_vec *bvec;
	554	+ int i;
446	555
447		- rq->nr_sectors -= sectors;
448		- return rq->nr_sectors;
	556	+ /*
	557	+ * This is dry run, restore bio_sector and size. We'll finish
	558	+ * this request again with the original bi_end_io after an
	559	+ * error occurs or post flush is complete.
	560	+ */
	561	+ q->bi_size += bytes;
	562	+
	563	+ if (bio->bi_size)
	564	+ return 1;
	565	+
	566	+ /* Rewind bvec's */
	567	+ bio->bi_idx = 0;
	568	+ bio_for_each_segment(bvec, bio, i) {
	569	+ bvec->bv_len += bvec->bv_offset;
	570	+ bvec->bv_offset = 0;
	571	+ }
	572	+
	573	+ /* Reset bio */
	574	+ set_bit(BIO_UPTODATE, &bio->bi_flags);
	575	+ bio->bi_size = q->bi_size;
	576	+ bio->bi_sector -= (q->bi_size >> 9);
	577	+ q->bi_size = 0;
	578	+
	579	+ return 0;
449	580	}
450	581
451		-static int __blk_complete_barrier_rq(request_queue_t q, struct request rq,
452		- int sectors, int queue_locked)
	582	+static inline int ordered_bio_endio(struct request rq, struct bio bio,
	583	+ unsigned int nbytes, int error)
453	584	{
454		- if (q->ordered != QUEUE_ORDERED_FLUSH)
	585	+ request_queue_t *q = rq->q;
	586	+ bio_end_io_t *endio;
	587	+ void *private;
	588	+
	589	+ if (&q->bar_rq != rq)
455	590	return 0;
456		- if (!blk_fs_request(rq) \|\| !blk_barrier_rq(rq))
457		- return 0;
458		- if (blk_barrier_postflush(rq))
459		- return 0;
460	591
461		- if (!blk_check_end_barrier(q, rq, sectors)) {
462		- unsigned long flags = 0;
	592	+ /*
	593	+ * Okay, this is the barrier request in progress, dry finish it.
	594	+ */
	595	+ if (error && !q->orderr)
	596	+ q->orderr = error;
463	597
464		- if (!queue_locked)
465		- spin_lock_irqsave(q->queue_lock, flags);
	598	+ endio = bio->bi_end_io;
	599	+ private = bio->bi_private;
	600	+ bio->bi_end_io = flush_dry_bio_endio;
	601	+ bio->bi_private = q;
466	602
467		- blk_start_post_flush(q, rq);
	603	+ bio_endio(bio, nbytes, error);
468	604
469		- if (!queue_locked)
470		- spin_unlock_irqrestore(q->queue_lock, flags);
471		- }
	605	+ bio->bi_end_io = endio;
	606	+ bio->bi_private = private;
472	607
473	608	return 1;
474	609	}
475	610
476	611	/**
477		- * blk_complete_barrier_rq - complete possible barrier request
478		- * @q: the request queue for the device
479		- * @rq: the request
480		- * @sectors: number of sectors to complete
481		- *
482		- * Description:
483		- * Used in driver end_io handling to determine whether to postpone
484		- * completion of a barrier request until a post flush has been done. This
485		- * is the unlocked variant, used if the caller doesn't already hold the
486		- * queue lock.
487		- **/
488		-int blk_complete_barrier_rq(request_queue_t q, struct request rq, int sectors)
489		-{
490		- return __blk_complete_barrier_rq(q, rq, sectors, 0);
491		-}
492		-EXPORT_SYMBOL(blk_complete_barrier_rq);
493		-
494		-/**
495		- * blk_complete_barrier_rq_locked - complete possible barrier request
496		- * @q: the request queue for the device
497		- * @rq: the request
498		- * @sectors: number of sectors to complete
499		- *
500		- * Description:
501		- * See blk_complete_barrier_rq(). This variant must be used if the caller
502		- * holds the queue lock.
503		- **/
504		-int blk_complete_barrier_rq_locked(request_queue_t q, struct request rq,
505		- int sectors)
506		-{
507		- return __blk_complete_barrier_rq(q, rq, sectors, 1);
508		-}
509		-EXPORT_SYMBOL(blk_complete_barrier_rq_locked);
510		-
511		-/**
512	612	* blk_queue_bounce_limit - set bounce buffer limit for queue
513	613	* @q: the request queue for the device
514	614	* @dma_addr: bus address limit
...	...	@@ -1047,6 +1147,7 @@
1047	1147	"REQ_SORTED",
1048	1148	"REQ_SOFTBARRIER",
1049	1149	"REQ_HARDBARRIER",
	1150	+ "REQ_FUA",
1050	1151	"REQ_CMD",
1051	1152	"REQ_NOMERGE",
1052	1153	"REQ_STARTED",
...	...	@@ -1066,6 +1167,7 @@
1066	1167	"REQ_PM_SUSPEND",
1067	1168	"REQ_PM_RESUME",
1068	1169	"REQ_PM_SHUTDOWN",
	1170	+ "REQ_ORDERED_COLOR",
1069	1171	};
1070	1172
1071	1173	void blk_dump_rq_flags(struct request rq, char msg)
...	...	@@ -1643,8 +1745,6 @@
1643	1745	if (q->queue_tags)
1644	1746	__blk_queue_free_tags(q);
1645	1747
1646		- blk_queue_ordered(q, QUEUE_ORDERED_NONE);
1647		-
1648	1748	kmem_cache_free(requestq_cachep, q);
1649	1749	}
1650	1750
...	...	@@ -2714,7 +2814,7 @@
2714	2814	spin_lock_prefetch(q->queue_lock);
2715	2815
2716	2816	barrier = bio_barrier(bio);
2717		- if (unlikely(barrier) && (q->ordered == QUEUE_ORDERED_NONE)) {
	2817	+ if (unlikely(barrier) && (q->next_ordered == QUEUE_ORDERED_NONE)) {
2718	2818	err = -EOPNOTSUPP;
2719	2819	goto end_io;
2720	2820	}
...	...	@@ -3075,7 +3175,8 @@
3075	3175	if (nr_bytes >= bio->bi_size) {
3076	3176	req->bio = bio->bi_next;
3077	3177	nbytes = bio->bi_size;
3078		- bio_endio(bio, nbytes, error);
	3178	+ if (!ordered_bio_endio(req, bio, nbytes, error))
	3179	+ bio_endio(bio, nbytes, error);
3079	3180	next_idx = 0;
3080	3181	bio_nbytes = 0;
3081	3182	} else {
...	...	@@ -3130,7 +3231,8 @@
3130	3231	* if the request wasn't completed, update state
3131	3232	*/
3132	3233	if (bio_nbytes) {
3133		- bio_endio(bio, bio_nbytes, error);
	3234	+ if (!ordered_bio_endio(req, bio, bio_nbytes, error))
	3235	+ bio_endio(bio, bio_nbytes, error);
3134	3236	bio->bi_idx += next_idx;
3135	3237	bio_iovec(bio)->bv_offset += nr_bytes;
3136	3238	bio_iovec(bio)->bv_len -= nr_bytes;
...	...	@@ -207,6 +207,7 @@
207	207	__REQ_SORTED, /* elevator knows about this request */
208	208	__REQ_SOFTBARRIER, /* may not be passed by ioscheduler */
209	209	__REQ_HARDBARRIER, /* may not be passed by drive either */
	210	+ __REQ_FUA, /* forced unit access */
210	211	__REQ_CMD, /* is a regular fs rw request */
211	212	__REQ_NOMERGE, /* don't touch this for merging */
212	213	__REQ_STARTED, /* drive already may have started this one */
...	...	@@ -230,9 +231,7 @@
230	231	__REQ_PM_SUSPEND, /* suspend request */
231	232	__REQ_PM_RESUME, /* resume request */
232	233	__REQ_PM_SHUTDOWN, /* shutdown request */
233		- __REQ_BAR_PREFLUSH, /* barrier pre-flush done */
234		- __REQ_BAR_POSTFLUSH, /* barrier post-flush */
235		- __REQ_BAR_FLUSH, /* rq is the flush request */
	234	+ __REQ_ORDERED_COLOR, /* is before or after barrier */
236	235	__REQ_NR_BITS, /* stops here */
237	236	};
238	237
...	...	@@ -241,6 +240,7 @@
241	240	#define REQ_SORTED (1 << __REQ_SORTED)
242	241	#define REQ_SOFTBARRIER (1 << __REQ_SOFTBARRIER)
243	242	#define REQ_HARDBARRIER (1 << __REQ_HARDBARRIER)
	243	+#define REQ_FUA (1 << __REQ_FUA)
244	244	#define REQ_CMD (1 << __REQ_CMD)
245	245	#define REQ_NOMERGE (1 << __REQ_NOMERGE)
246	246	#define REQ_STARTED (1 << __REQ_STARTED)
...	...	@@ -260,9 +260,7 @@
260	260	#define REQ_PM_SUSPEND (1 << __REQ_PM_SUSPEND)
261	261	#define REQ_PM_RESUME (1 << __REQ_PM_RESUME)
262	262	#define REQ_PM_SHUTDOWN (1 << __REQ_PM_SHUTDOWN)
263		-#define REQ_BAR_PREFLUSH (1 << __REQ_BAR_PREFLUSH)
264		-#define REQ_BAR_POSTFLUSH (1 << __REQ_BAR_POSTFLUSH)
265		-#define REQ_BAR_FLUSH (1 << __REQ_BAR_FLUSH)
	263	+#define REQ_ORDERED_COLOR (1 << __REQ_ORDERED_COLOR)
266	264
267	265	/*
268	266	* State information carried for REQ_PM_SUSPEND and REQ_PM_RESUME
...	...	@@ -292,8 +290,7 @@
292	290	typedef int (merge_bvec_fn) (request_queue_t , struct bio , struct bio_vec *);
293	291	typedef void (activity_fn) (void *data, int rw);
294	292	typedef int (issue_flush_fn) (request_queue_t , struct gendisk , sector_t *);
295		-typedef int (prepare_flush_fn) (request_queue_t , struct request );
296		-typedef void (end_flush_fn) (request_queue_t , struct request );
	293	+typedef void (prepare_flush_fn) (request_queue_t , struct request );
297	294
298	295	enum blk_queue_state {
299	296	Queue_down,
...	...	@@ -335,7 +332,6 @@
335	332	activity_fn *activity_fn;
336	333	issue_flush_fn *issue_flush_fn;
337	334	prepare_flush_fn *prepare_flush_fn;
338		- end_flush_fn *end_flush_fn;
339	335
340	336	/*
341	337	* Dispatch queue sorting
342	338
...	...	@@ -420,16 +416,13 @@
420	416	/*
421	417	* reserved for flush operations
422	418	*/
423		- struct request *flush_rq;
424		- unsigned char ordered;
	419	+ unsigned int ordered, next_ordered, ordseq;
	420	+ int orderr, ordcolor;
	421	+ struct request pre_flush_rq, bar_rq, post_flush_rq;
	422	+ struct request *orig_bar_rq;
	423	+ unsigned int bi_size;
425	424	};
426	425
427		-enum {
428		- QUEUE_ORDERED_NONE,
429		- QUEUE_ORDERED_TAG,
430		- QUEUE_ORDERED_FLUSH,
431		-};
432		-
433	426	#define RQ_INACTIVE (-1)
434	427	#define RQ_ACTIVE 1
435	428	#define RQ_SCSI_BUSY 0xffff
436	429
437	430
...	...	@@ -445,12 +438,51 @@
445	438	#define QUEUE_FLAG_REENTER 6 /* Re-entrancy avoidance */
446	439	#define QUEUE_FLAG_PLUGGED 7 /* queue is plugged */
447	440	#define QUEUE_FLAG_ELVSWITCH 8 /* don't use elevator, just do FIFO */
448		-#define QUEUE_FLAG_FLUSH 9 /* doing barrier flush sequence */
449	441
	442	+enum {
	443	+ /*
	444	+ * Hardbarrier is supported with one of the following methods.
	445	+ *
	446	+ * NONE : hardbarrier unsupported
	447	+ * DRAIN : ordering by draining is enough
	448	+ * DRAIN_FLUSH : ordering by draining w/ pre and post flushes
	449	+ * DRAIN_FUA : ordering by draining w/ pre flush and FUA write
	450	+ * TAG : ordering by tag is enough
	451	+ * TAG_FLUSH : ordering by tag w/ pre and post flushes
	452	+ * TAG_FUA : ordering by tag w/ pre flush and FUA write
	453	+ */
	454	+ QUEUE_ORDERED_NONE = 0x00,
	455	+ QUEUE_ORDERED_DRAIN = 0x01,
	456	+ QUEUE_ORDERED_TAG = 0x02,
	457	+
	458	+ QUEUE_ORDERED_PREFLUSH = 0x10,
	459	+ QUEUE_ORDERED_POSTFLUSH = 0x20,
	460	+ QUEUE_ORDERED_FUA = 0x40,
	461	+
	462	+ QUEUE_ORDERED_DRAIN_FLUSH = QUEUE_ORDERED_DRAIN \|
	463	+ QUEUE_ORDERED_PREFLUSH \| QUEUE_ORDERED_POSTFLUSH,
	464	+ QUEUE_ORDERED_DRAIN_FUA = QUEUE_ORDERED_DRAIN \|
	465	+ QUEUE_ORDERED_PREFLUSH \| QUEUE_ORDERED_FUA,
	466	+ QUEUE_ORDERED_TAG_FLUSH = QUEUE_ORDERED_TAG \|
	467	+ QUEUE_ORDERED_PREFLUSH \| QUEUE_ORDERED_POSTFLUSH,
	468	+ QUEUE_ORDERED_TAG_FUA = QUEUE_ORDERED_TAG \|
	469	+ QUEUE_ORDERED_PREFLUSH \| QUEUE_ORDERED_FUA,
	470	+
	471	+ /*
	472	+ * Ordered operation sequence
	473	+ */
	474	+ QUEUE_ORDSEQ_STARTED = 0x01, /* flushing in progress */
	475	+ QUEUE_ORDSEQ_DRAIN = 0x02, /* waiting for the queue to be drained */
	476	+ QUEUE_ORDSEQ_PREFLUSH = 0x04, /* pre-flushing in progress */
	477	+ QUEUE_ORDSEQ_BAR = 0x08, /* original barrier req in progress */
	478	+ QUEUE_ORDSEQ_POSTFLUSH = 0x10, /* post-flushing in progress */
	479	+ QUEUE_ORDSEQ_DONE = 0x20,
	480	+};
	481	+
450	482	#define blk_queue_plugged(q) test_bit(QUEUE_FLAG_PLUGGED, &(q)->queue_flags)
451	483	#define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
452	484	#define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
453		-#define blk_queue_flushing(q) test_bit(QUEUE_FLAG_FLUSH, &(q)->queue_flags)
	485	+#define blk_queue_flushing(q) ((q)->ordseq)
454	486
455	487	#define blk_fs_request(rq) ((rq)->flags & REQ_CMD)
456	488	#define blk_pc_request(rq) ((rq)->flags & REQ_BLOCK_PC)
...	...	@@ -466,8 +498,7 @@
466	498
467	499	#define blk_sorted_rq(rq) ((rq)->flags & REQ_SORTED)
468	500	#define blk_barrier_rq(rq) ((rq)->flags & REQ_HARDBARRIER)
469		-#define blk_barrier_preflush(rq) ((rq)->flags & REQ_BAR_PREFLUSH)
470		-#define blk_barrier_postflush(rq) ((rq)->flags & REQ_BAR_POSTFLUSH)
	501	+#define blk_fua_rq(rq) ((rq)->flags & REQ_FUA)
471	502
472	503	#define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
473	504
474	505
...	...	@@ -665,11 +696,12 @@
665	696	extern void blk_queue_merge_bvec(request_queue_t , merge_bvec_fn );
666	697	extern void blk_queue_dma_alignment(request_queue_t *, int);
667	698	extern struct backing_dev_info blk_get_backing_dev_info(struct block_device bdev);
668		-extern void blk_queue_ordered(request_queue_t *, int);
	699	+extern int blk_queue_ordered(request_queue_t , unsigned, prepare_flush_fn );
669	700	extern void blk_queue_issue_flush_fn(request_queue_t , issue_flush_fn );
670		-extern struct request blk_start_pre_flush(request_queue_t ,struct request *);
671		-extern int blk_complete_barrier_rq(request_queue_t , struct request , int);
672		-extern int blk_complete_barrier_rq_locked(request_queue_t , struct request , int);
	701	+extern int blk_do_ordered(request_queue_t , struct request *);
	702	+extern unsigned blk_ordered_cur_seq(request_queue_t *);
	703	+extern unsigned blk_ordered_req_seq(struct request *);
	704	+extern void blk_ordered_complete_seq(request_queue_t *, unsigned, int);
673	705
674	706	extern int blk_rq_map_sg(request_queue_t , struct request , struct scatterlist *);
675	707	extern void blk_dump_rq_flags(struct request , char );
...	...	@@ -130,6 +130,7 @@
130	130	#define ELEVATOR_INSERT_FRONT 1
131	131	#define ELEVATOR_INSERT_BACK 2
132	132	#define ELEVATOR_INSERT_SORT 3
	133	+#define ELEVATOR_INSERT_REQUEUE 4
133	134
134	135	/*
135	136	* return values from elevator_may_queue_fn