block: make queue flags non-atomic

We can save some atomic ops in the IO path, if we clearly define
the rules of how to modify the queue flags.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

diff --git a/block/blk-core.c b/block/blk-core.c
index e447799256d6c0abeb460d61f10e15b6e80030e4..d2f23ec5ebfa5e49e39e1613d6526ef2665df844 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -198,7 +198,8 @@ void blk_plug_device(struct request_queue *q)
        if (blk_queue_stopped(q))
                return;
 
-       if (!test_and_set_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags)) {
+       if (!test_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags)) {
+               __set_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags);
                mod_timer(&q->unplug_timer, jiffies + q->unplug_delay);
                blk_add_trace_generic(q, NULL, 0, BLK_TA_PLUG);
        }
@@ -213,9 +214,10 @@ int blk_remove_plug(struct request_queue *q)
 {
        WARN_ON(!irqs_disabled());
 
-       if (!test_and_clear_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags))
+       if (!test_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags))
                return 0;
 
+       queue_flag_clear(QUEUE_FLAG_PLUGGED, q);
        del_timer(&q->unplug_timer);
        return 1;
 }
@@ -311,15 +313,16 @@ void blk_start_queue(struct request_queue *q)
 {
        WARN_ON(!irqs_disabled());
 
-       clear_bit(QUEUE_FLAG_STOPPED, &q->queue_flags);
+       queue_flag_clear(QUEUE_FLAG_STOPPED, q);
 
        /*
         * one level of recursion is ok and is much faster than kicking
         * the unplug handling
         */
-       if (!test_and_set_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
+       if (!test_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
+               queue_flag_set(QUEUE_FLAG_REENTER, q);
                q->request_fn(q);
-               clear_bit(QUEUE_FLAG_REENTER, &q->queue_flags);
+               queue_flag_clear(QUEUE_FLAG_REENTER, q);
        } else {
                blk_plug_device(q);
                kblockd_schedule_work(&q->unplug_work);
@@ -344,7 +347,7 @@ EXPORT_SYMBOL(blk_start_queue);
 void blk_stop_queue(struct request_queue *q)
 {
        blk_remove_plug(q);
-       set_bit(QUEUE_FLAG_STOPPED, &q->queue_flags);
+       queue_flag_set(QUEUE_FLAG_STOPPED, q);
 }
 EXPORT_SYMBOL(blk_stop_queue);
 
@@ -373,11 +376,8 @@ EXPORT_SYMBOL(blk_sync_queue);
  * blk_run_queue - run a single device queue
  * @q: The queue to run
  */
-void blk_run_queue(struct request_queue *q)
+void __blk_run_queue(struct request_queue *q)
 {
-       unsigned long flags;
-
-       spin_lock_irqsave(q->queue_lock, flags);
        blk_remove_plug(q);
 
        /*
@@ -385,15 +385,28 @@ void blk_run_queue(struct request_queue *q)
         * handling reinvoke the handler shortly if we already got there.
         */
        if (!elv_queue_empty(q)) {
-               if (!test_and_set_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
+               if (!test_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
+                       queue_flag_set(QUEUE_FLAG_REENTER, q);
                        q->request_fn(q);
-                       clear_bit(QUEUE_FLAG_REENTER, &q->queue_flags);
+                       queue_flag_clear(QUEUE_FLAG_REENTER, q);
                } else {
                        blk_plug_device(q);
                        kblockd_schedule_work(&q->unplug_work);
                }
        }
+}
+EXPORT_SYMBOL(__blk_run_queue);
 
+/**
+ * blk_run_queue - run a single device queue
+ * @q: The queue to run
+ */
+void blk_run_queue(struct request_queue *q)
+{
+       unsigned long flags;
+
+       spin_lock_irqsave(q->queue_lock, flags);
+       __blk_run_queue(q);
        spin_unlock_irqrestore(q->queue_lock, flags);
 }
 EXPORT_SYMBOL(blk_run_queue);
@@ -406,7 +419,7 @@ void blk_put_queue(struct request_queue *q)
 void blk_cleanup_queue(struct request_queue *q)
 {
        mutex_lock(&q->sysfs_lock);
-       set_bit(QUEUE_FLAG_DEAD, &q->queue_flags);
+       queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q);
        mutex_unlock(&q->sysfs_lock);
 
        if (q->elevator)

diff --git a/block/blk-merge.c b/block/blk-merge.c
index b5c5c4a9e3f08d051a25d44e92689580d19cbce5..73b23562af20086f3a8623df6e9f32561ae330b5 100644 (file)
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -55,7 +55,7 @@ void blk_recalc_rq_segments(struct request *rq)
        if (!rq->bio)
                return;
 
-       cluster = q->queue_flags & (1 << QUEUE_FLAG_CLUSTER);
+       cluster = test_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
        hw_seg_size = seg_size = 0;
        phys_size = hw_size = nr_phys_segs = nr_hw_segs = 0;
        rq_for_each_segment(bv, rq, iter) {
@@ -128,7 +128,7 @@ EXPORT_SYMBOL(blk_recount_segments);
 static int blk_phys_contig_segment(struct request_queue *q, struct bio *bio,
                                   struct bio *nxt)
 {
-       if (!(q->queue_flags & (1 << QUEUE_FLAG_CLUSTER)))
+       if (!test_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags))
                return 0;
 
        if (!BIOVEC_PHYS_MERGEABLE(__BVEC_END(bio), __BVEC_START(nxt)))
@@ -175,7 +175,7 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,
        int nsegs, cluster;
 
        nsegs = 0;
-       cluster = q->queue_flags & (1 << QUEUE_FLAG_CLUSTER);
+       cluster = test_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
 
        /*
         * for each bio in rq

diff --git a/block/blk-settings.c b/block/blk-settings.c
index 77b51dc37a3c5432af233fcd1ed4a775eaa155da..6089384ab06499becc951478d06dda2e22aa7e00 100644 (file)
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -287,7 +287,7 @@ void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b)
        t->max_segment_size = min(t->max_segment_size, b->max_segment_size);
        t->hardsect_size = max(t->hardsect_size, b->hardsect_size);
        if (!test_bit(QUEUE_FLAG_CLUSTER, &b->queue_flags))
-               clear_bit(QUEUE_FLAG_CLUSTER, &t->queue_flags);
+               queue_flag_clear(QUEUE_FLAG_CLUSTER, t);
 }
 EXPORT_SYMBOL(blk_queue_stack_limits);
 

diff --git a/block/blk-tag.c b/block/blk-tag.c
index 4780a46ce2346898953085a6f4079c7734dcbaf7..e176ddbe599e23796c004e3d0354cbc4c6b337cf 100644 (file)
--- a/block/blk-tag.c
+++ b/block/blk-tag.c
@@ -70,7 +70,7 @@ void __blk_queue_free_tags(struct request_queue *q)
        __blk_free_tags(bqt);
 
        q->queue_tags = NULL;
-       q->queue_flags &= ~(1 << QUEUE_FLAG_QUEUED);
+       queue_flag_clear(QUEUE_FLAG_QUEUED, q);
 }
 
 /**
@@ -98,7 +98,7 @@ EXPORT_SYMBOL(blk_free_tags);
  **/
 void blk_queue_free_tags(struct request_queue *q)
 {
-       clear_bit(QUEUE_FLAG_QUEUED, &q->queue_flags);
+       queue_flag_clear(QUEUE_FLAG_QUEUED, q);
 }
 EXPORT_SYMBOL(blk_queue_free_tags);
 
@@ -188,7 +188,7 @@ int blk_queue_init_tags(struct request_queue *q, int depth,
                rc = blk_queue_resize_tags(q, depth);
                if (rc)
                        return rc;
-               set_bit(QUEUE_FLAG_QUEUED, &q->queue_flags);
+               queue_flag_set(QUEUE_FLAG_QUEUED, q);
                return 0;
        } else
                atomic_inc(&tags->refcnt);
@@ -197,7 +197,7 @@ int blk_queue_init_tags(struct request_queue *q, int depth,
         * assign it, all done
         */
        q->queue_tags = tags;
-       q->queue_flags |= (1 << QUEUE_FLAG_QUEUED);
+       queue_flag_set(QUEUE_FLAG_QUEUED, q);
        INIT_LIST_HEAD(&q->tag_busy_list);
        return 0;
 fail:

diff --git a/block/elevator.c b/block/elevator.c
index 88318c383608defa5018bdfa929918c767c8d7fa..e8a90fe234244b79fad9bfc745aaa84a2fa6260d 100644 (file)
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -1070,7 +1070,7 @@ static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
         */
        spin_lock_irq(q->queue_lock);
 
-       set_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
+       queue_flag_set(QUEUE_FLAG_ELVSWITCH, q);
 
        elv_drain_elevator(q);
 
@@ -1104,7 +1104,10 @@ static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
         * finally exit old elevator and turn off BYPASS.
         */
        elevator_exit(old_elevator);
-       clear_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
+       spin_lock_irq(q->queue_lock);
+       queue_flag_clear(QUEUE_FLAG_ELVSWITCH, q);
+       spin_unlock_irq(q->queue_lock);
+
        return 1;
 
 fail_register:
@@ -1115,7 +1118,11 @@ fail_register:
        elevator_exit(e);
        q->elevator = old_elevator;
        elv_register_queue(q);
-       clear_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
+
+       spin_lock_irq(q->queue_lock);
+       queue_flag_clear(QUEUE_FLAG_ELVSWITCH, q);
+       spin_unlock_irq(q->queue_lock);
+
        return 0;
 }
 

diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index f7f163557aa0f3ea88bd1bf91efd41942b59dfc0..d3a25b027ff9fc5e6331372a252b5e134b6270c4 100644 (file)
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -546,7 +546,7 @@ static void loop_unplug(struct request_queue *q)
 {
        struct loop_device *lo = q->queuedata;
 
-       clear_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags);
+       queue_flag_clear_unlocked(QUEUE_FLAG_PLUGGED, q);
        blk_run_address_space(lo->lo_backing_file->f_mapping);
 }
 

diff --git a/drivers/block/ub.c b/drivers/block/ub.c
index 27bfe72aab59360951b133b0be25c88ba85ea6c4..e322cce8c12d7870d45b2b202c8ff60e9e924c9f 100644 (file)
--- a/drivers/block/ub.c
+++ b/drivers/block/ub.c
@@ -2399,7 +2399,7 @@ static void ub_disconnect(struct usb_interface *intf)
                del_gendisk(lun->disk);
                /*
                 * I wish I could do:
-                *    set_bit(QUEUE_FLAG_DEAD, &q->queue_flags);
+                *    queue_flag_set(QUEUE_FLAG_DEAD, q);
                 * As it is, we rely on our internal poisoning and let
                 * the upper levels to spin furiously failing all the I/O.
                 */

diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index 51be53344214906593ec54dec6b6e8e1c018f369..73326e7c54bfccbf7ba6122b4252ae003bf75e28 100644 (file)
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -873,10 +873,13 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q)
        q->max_hw_sectors = t->limits.max_hw_sectors;
        q->seg_boundary_mask = t->limits.seg_boundary_mask;
        q->bounce_pfn = t->limits.bounce_pfn;
+       /* XXX: the below will probably go bug. must ensure there can be no
+        * concurrency on queue_flags, and use the unlocked versions...
+        */
        if (t->limits.no_cluster)
-               q->queue_flags &= ~(1 << QUEUE_FLAG_CLUSTER);
+               queue_flag_clear(QUEUE_FLAG_CLUSTER, q);
        else
-               q->queue_flags |= (1 << QUEUE_FLAG_CLUSTER);
+               queue_flag_set(QUEUE_FLAG_CLUSTER, q);
 
 }
 

diff --git a/drivers/md/md.c b/drivers/md/md.c
index 87620b705bee01182e37cdb0a95bf825f75d77b2..acd716b657b863c0e4e6ceae949968d0bc56448e 100644 (file)
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -282,7 +282,8 @@ static mddev_t * mddev_find(dev_t unit)
                kfree(new);
                return NULL;
        }
-       set_bit(QUEUE_FLAG_CLUSTER, &new->queue->queue_flags);
+       /* Can be unlocked because the queue is new: no concurrency */
+       queue_flag_set_unlocked(QUEUE_FLAG_CLUSTER, new->queue);
 
        blk_queue_make_request(new->queue, md_fail_request);
 

diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 07103c399fe0c2e7ce840fcdcb7d995bd277ca73..f6600bfb5bde71ffc244260ec6aa1aec4cf59469 100644 (file)
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -1773,7 +1773,7 @@ static int scsi_debug_slave_alloc(struct scsi_device *sdp)
        if (SCSI_DEBUG_OPT_NOISE & scsi_debug_opts)
                printk(KERN_INFO "scsi_debug: slave_alloc <%u %u %u %u>\n",
                       sdp->host->host_no, sdp->channel, sdp->id, sdp->lun);
-       set_bit(QUEUE_FLAG_BIDI, &sdp->request_queue->queue_flags);
+       queue_flag_set_unlocked(QUEUE_FLAG_BIDI, sdp->request_queue);
        return 0;
 }
 

diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 67f412bb4974b91783009fb3404c4a589bb6f6c8..d545ad1cf47a7f94c70369da8c1b6fb4dab0fe9d 100644 (file)
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -536,6 +536,9 @@ static void scsi_run_queue(struct request_queue *q)
               !shost->host_blocked && !shost->host_self_blocked &&
                !((shost->can_queue > 0) &&
                  (shost->host_busy >= shost->can_queue))) {
+
+               int flagset;
+
                /*
                 * As long as shost is accepting commands and we have
                 * starved queues, call blk_run_queue. scsi_request_fn
@@ -549,19 +552,20 @@ static void scsi_run_queue(struct request_queue *q)
                sdev = list_entry(shost->starved_list.next,
                                          struct scsi_device, starved_entry);
                list_del_init(&sdev->starved_entry);
-               spin_unlock_irqrestore(shost->host_lock, flags);
-
+               spin_unlock(shost->host_lock);
+
+               spin_lock(sdev->request_queue->queue_lock);
+               flagset = test_bit(QUEUE_FLAG_REENTER, &q->queue_flags) &&
+                               !test_bit(QUEUE_FLAG_REENTER,
+                                       &sdev->request_queue->queue_flags);
+               if (flagset)
+                       queue_flag_set(QUEUE_FLAG_REENTER, sdev->request_queue);
+               __blk_run_queue(sdev->request_queue);
+               if (flagset)
+                       queue_flag_clear(QUEUE_FLAG_REENTER, sdev->request_queue);
+               spin_unlock(sdev->request_queue->queue_lock);
 
-               if (test_bit(QUEUE_FLAG_REENTER, &q->queue_flags) &&
-                   !test_and_set_bit(QUEUE_FLAG_REENTER,
-                                     &sdev->request_queue->queue_flags)) {
-                       blk_run_queue(sdev->request_queue);
-                       clear_bit(QUEUE_FLAG_REENTER,
-                                 &sdev->request_queue->queue_flags);
-               } else
-                       blk_run_queue(sdev->request_queue);
-
-               spin_lock_irqsave(shost->host_lock, flags);
+               spin_lock(shost->host_lock);
                if (unlikely(!list_empty(&sdev->starved_entry)))
                        /*
                         * sdev lost a race, and was put back on the
@@ -1585,8 +1589,9 @@ struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
 
        blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));
 
+       /* New queue, no concurrency on queue_flags */
        if (!shost->use_clustering)
-               clear_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags);
+               queue_flag_clear_unlocked(QUEUE_FLAG_CLUSTER, q);
 
        /*
         * set a reasonable default alignment on word boundaries: the

diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index 7899e3dda9bfd7ccf357f98ed299c77af8b18013..f4461d35ffb9157d6cc3e634f692ef0937b5a9c5 100644 (file)
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -248,8 +248,7 @@ static int sas_bsg_initialize(struct Scsi_Host *shost, struct sas_rphy *rphy)
        else
                q->queuedata = shost;
 
-       set_bit(QUEUE_FLAG_BIDI, &q->queue_flags);
-
+       queue_flag_set_unlocked(QUEUE_FLAG_BIDI, q);
        return 0;
 }
 

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index c5065e3d2ca9eea3e1a4319d117cdf4d5abbd539..8ca481cd7d73ee4c09f613edf6e4beb8180cfd9f 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -408,6 +408,30 @@ struct request_queue
 #define QUEUE_FLAG_ELVSWITCH   8       /* don't use elevator, just do FIFO */
 #define QUEUE_FLAG_BIDI                9       /* queue supports bidi requests */
 
+static inline void queue_flag_set_unlocked(unsigned int flag,
+                                          struct request_queue *q)
+{
+       __set_bit(flag, &q->queue_flags);
+}
+
+static inline void queue_flag_set(unsigned int flag, struct request_queue *q)
+{
+       WARN_ON_ONCE(!spin_is_locked(q->queue_lock));
+       __set_bit(flag, &q->queue_flags);
+}
+
+static inline void queue_flag_clear_unlocked(unsigned int flag,
+                                            struct request_queue *q)
+{
+       __clear_bit(flag, &q->queue_flags);
+}
+
+static inline void queue_flag_clear(unsigned int flag, struct request_queue *q)
+{
+       WARN_ON_ONCE(!spin_is_locked(q->queue_lock));
+       __clear_bit(flag, &q->queue_flags);
+}
+
 enum {
        /*
         * Hardbarrier is supported with one of the following methods.
@@ -496,17 +520,17 @@ static inline int blk_queue_full(struct request_queue *q, int rw)
 static inline void blk_set_queue_full(struct request_queue *q, int rw)
 {
        if (rw == READ)
-               set_bit(QUEUE_FLAG_READFULL, &q->queue_flags);
+               queue_flag_set(QUEUE_FLAG_READFULL, q);
        else
-               set_bit(QUEUE_FLAG_WRITEFULL, &q->queue_flags);
+               queue_flag_set(QUEUE_FLAG_WRITEFULL, q);
 }
 
 static inline void blk_clear_queue_full(struct request_queue *q, int rw)
 {
        if (rw == READ)
-               clear_bit(QUEUE_FLAG_READFULL, &q->queue_flags);
+               queue_flag_clear(QUEUE_FLAG_READFULL, q);
        else
-               clear_bit(QUEUE_FLAG_WRITEFULL, &q->queue_flags);
+               queue_flag_clear(QUEUE_FLAG_WRITEFULL, q);
 }
 
 
@@ -626,6 +650,7 @@ extern void blk_start_queue(struct request_queue *q);
 extern void blk_stop_queue(struct request_queue *q);
 extern void blk_sync_queue(struct request_queue *q);
 extern void __blk_stop_queue(struct request_queue *q);
+extern void __blk_run_queue(struct request_queue *);
 extern void blk_run_queue(struct request_queue *);
 extern void blk_start_queueing(struct request_queue *);
 extern int blk_rq_map_user(struct request_queue *, struct request *, void __user *, unsigned long);