static void drive_stat_acct(struct request *rq, int nr_sectors, int new_io);
static void init_request_from_bio(struct request *req, struct bio *bio);
static int __make_request(request_queue_t *q, struct bio *bio);
+static struct io_context *current_io_context(gfp_t gfp_flags, int node);
/*
* For the allocated request tables
*/
static kmem_cache_t *iocontext_cachep;
-static wait_queue_head_t congestion_wqh[2] = {
- __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]),
- __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
- };
-
/*
* Controlling structure to kblockd
*/
q->nr_congestion_off = nr;
}
-/*
- * A queue has just exitted congestion. Note this in the global counter of
- * congested queues, and wake up anyone who was waiting for requests to be
- * put back.
- */
-static void clear_queue_congested(request_queue_t *q, int rw)
-{
- enum bdi_state bit;
- wait_queue_head_t *wqh = &congestion_wqh[rw];
-
- bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested;
- clear_bit(bit, &q->backing_dev_info.state);
- smp_mb__after_clear_bit();
- if (waitqueue_active(wqh))
- wake_up(wqh);
-}
-
-/*
- * A queue has just entered congestion. Flag that in the queue's VM-visible
- * state flags and increment the global gounter of congested queues.
- */
-static void set_queue_congested(request_queue_t *q, int rw)
-{
- enum bdi_state bit;
-
- bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested;
- set_bit(bit, &q->backing_dev_info.state);
-}
-
/**
* blk_get_backing_dev_info - get the address of a queue's backing_dev_info
* @bdev: device
ret = &q->backing_dev_info;
return ret;
}
-
EXPORT_SYMBOL(blk_get_backing_dev_info);
void blk_queue_activity_fn(request_queue_t *q, activity_fn *fn, void *data)
q->activity_fn = fn;
q->activity_data = data;
}
-
EXPORT_SYMBOL(blk_queue_activity_fn);
/**
EXPORT_SYMBOL(blk_queue_make_request);
-static inline void rq_init(request_queue_t *q, struct request *rq)
+static void rq_init(request_queue_t *q, struct request *rq)
{
INIT_LIST_HEAD(&rq->queuelist);
INIT_LIST_HEAD(&rq->donelist);
return 0;
}
-static inline int ordered_bio_endio(struct request *rq, struct bio *bio,
- unsigned int nbytes, int error)
+static int ordered_bio_endio(struct request *rq, struct bio *bio,
+ unsigned int nbytes, int error)
{
request_queue_t *q = rq->q;
bio_end_io_t *endio;
**/
struct request *blk_queue_find_tag(request_queue_t *q, int tag)
{
- struct blk_queue_tag *bqt = q->queue_tags;
-
- if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
- return NULL;
-
- return bqt->tag_index[tag];
+ return blk_map_queue_find_tag(q->queue_tags, tag);
}
EXPORT_SYMBOL(blk_queue_find_tag);
BUG();
}
- tag = find_first_zero_bit(bqt->tag_map, bqt->max_depth);
- if (tag >= bqt->max_depth)
- return 1;
+ /*
+ * Protect against shared tag maps, as we may not have exclusive
+ * access to the tag map.
+ */
+ do {
+ tag = find_first_zero_bit(bqt->tag_map, bqt->max_depth);
+ if (tag >= bqt->max_depth)
+ return 1;
- __set_bit(tag, bqt->tag_map);
+ } while (test_and_set_bit(tag, bqt->tag_map));
rq->cmd_flags |= REQ_QUEUED;
rq->tag = tag;
mempool_free(rq, q->rq.rq_pool);
}
-static inline struct request *
-blk_alloc_request(request_queue_t *q, int rw, struct bio *bio,
- int priv, gfp_t gfp_mask)
+static struct request *
+blk_alloc_request(request_queue_t *q, int rw, int priv, gfp_t gfp_mask)
{
struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);
rq->cmd_flags = rw | REQ_ALLOCED;
if (priv) {
- if (unlikely(elv_set_request(q, rq, bio, gfp_mask))) {
+ if (unlikely(elv_set_request(q, rq, gfp_mask))) {
mempool_free(rq, q->rq.rq_pool);
return NULL;
}
struct request_list *rl = &q->rq;
if (rl->count[rw] < queue_congestion_off_threshold(q))
- clear_queue_congested(q, rw);
+ blk_clear_queue_congested(q, rw);
if (rl->count[rw] + 1 <= q->nr_requests) {
if (waitqueue_active(&rl->wait[rw]))
struct io_context *ioc = NULL;
int may_queue, priv;
- may_queue = elv_may_queue(q, rw, bio);
+ may_queue = elv_may_queue(q, rw);
if (may_queue == ELV_MQUEUE_NO)
goto rq_starved;
if (rl->count[rw]+1 >= queue_congestion_on_threshold(q)) {
if (rl->count[rw]+1 >= q->nr_requests) {
- ioc = current_io_context(GFP_ATOMIC);
+ ioc = current_io_context(GFP_ATOMIC, q->node);
/*
* The queue will fill after this allocation, so set
* it as full, and mark this process as "batching".
}
}
}
- set_queue_congested(q, rw);
+ blk_set_queue_congested(q, rw);
}
/*
spin_unlock_irq(q->queue_lock);
- rq = blk_alloc_request(q, rw, bio, priv, gfp_mask);
+ rq = blk_alloc_request(q, rw, priv, gfp_mask);
if (unlikely(!rq)) {
/*
* Allocation failed presumably due to memory. Undo anything
* up to a big batch of them for a small period time.
* See ioc_batching, ioc_set_batching
*/
- ioc = current_io_context(GFP_NOIO);
+ ioc = current_io_context(GFP_NOIO, q->node);
ioc_set_batching(q, ioc);
spin_lock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_get_request);
+/**
+ * blk_start_queueing - initiate dispatch of requests to device
+ * @q: request queue to kick into gear
+ *
+ * This is basically a helper to remove the need to know whether a queue
+ * is plugged or not if someone just wants to initiate dispatch of requests
+ * for this queue.
+ *
+ * The queue lock must be held with interrupts disabled.
+ */
+void blk_start_queueing(request_queue_t *q)
+{
+ if (!blk_queue_plugged(q))
+ q->request_fn(q);
+ else
+ __generic_unplug_device(q);
+}
+EXPORT_SYMBOL(blk_start_queueing);
+
/**
* blk_requeue_request - put a request back on queue
* @q: request queue where request should be inserted
drive_stat_acct(rq, rq->nr_sectors, 1);
__elv_add_request(q, rq, where, 0);
-
- if (blk_queue_plugged(q))
- __generic_unplug_device(q);
- else
- q->request_fn(q);
+ blk_start_queueing(q);
spin_unlock_irqrestore(q->queue_lock, flags);
}
}
EXPORT_SYMBOL(blk_end_sync_rq);
-/**
- * blk_congestion_wait - wait for a queue to become uncongested
- * @rw: READ or WRITE
- * @timeout: timeout in jiffies
- *
- * Waits for up to @timeout jiffies for a queue (any queue) to exit congestion.
- * If no queues are congested then just wait for the next request to be
- * returned.
- */
-long blk_congestion_wait(int rw, long timeout)
-{
- long ret;
- DEFINE_WAIT(wait);
- wait_queue_head_t *wqh = &congestion_wqh[rw];
-
- prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
- ret = io_schedule_timeout(timeout);
- finish_wait(wqh, &wait);
- return ret;
-}
-
-EXPORT_SYMBOL(blk_congestion_wait);
-
-/**
- * blk_congestion_end - wake up sleepers on a congestion queue
- * @rw: READ or WRITE
- */
-void blk_congestion_end(int rw)
-{
- wait_queue_head_t *wqh = &congestion_wqh[rw];
-
- if (waitqueue_active(wqh))
- wake_up(wqh);
-}
-
/*
* Has to be called with the request spinlock acquired
*/
if (bio_sync(bio))
req->cmd_flags |= REQ_RW_SYNC;
+ if (bio_rw_meta(bio))
+ req->cmd_flags |= REQ_RW_META;
req->errors = 0;
req->hard_sector = req->sector = bio->bi_sector;
static int __make_request(request_queue_t *q, struct bio *bio)
{
struct request *req;
- int el_ret, rw, nr_sectors, cur_nr_sectors, barrier, err, sync;
- unsigned short prio;
- sector_t sector;
+ int el_ret, nr_sectors, barrier, err;
+ const unsigned short prio = bio_prio(bio);
+ const int sync = bio_sync(bio);
- sector = bio->bi_sector;
nr_sectors = bio_sectors(bio);
- cur_nr_sectors = bio_cur_sectors(bio);
- prio = bio_prio(bio);
-
- rw = bio_data_dir(bio);
- sync = bio_sync(bio);
/*
* low level driver can indicate that it wants pages above a
*/
blk_queue_bounce(q, &bio);
- spin_lock_prefetch(q->queue_lock);
-
barrier = bio_barrier(bio);
if (unlikely(barrier) && (q->next_ordered == QUEUE_ORDERED_NONE)) {
err = -EOPNOTSUPP;
* not touch req->buffer either...
*/
req->buffer = bio_data(bio);
- req->current_nr_sectors = cur_nr_sectors;
- req->hard_cur_sectors = cur_nr_sectors;
- req->sector = req->hard_sector = sector;
+ req->current_nr_sectors = bio_cur_sectors(bio);
+ req->hard_cur_sectors = req->current_nr_sectors;
+ req->sector = req->hard_sector = bio->bi_sector;
req->nr_sectors = req->hard_nr_sectors += nr_sectors;
req->ioprio = ioprio_best(req->ioprio, prio);
drive_stat_acct(req, nr_sectors, 0);
* Grab a free request. This is might sleep but can not fail.
* Returns with the queue unlocked.
*/
- req = get_request_wait(q, rw, bio);
+ req = get_request_wait(q, bio_data_dir(bio), bio);
/*
* After dropping the lock and possibly sleeping here, our request
{
request_queue_t *q;
sector_t maxsector;
+ sector_t old_sector;
int ret, nr_sectors = bio_sectors(bio);
dev_t old_dev;
* NOTE: we don't repeat the blk_size check for each new device.
* Stacking drivers are expected to know what they are doing.
*/
- maxsector = -1;
+ old_sector = -1;
old_dev = 0;
do {
char b[BDEVNAME_SIZE];
*/
blk_partition_remap(bio);
- if (maxsector != -1)
+ if (old_sector != -1)
blk_add_trace_remap(q, bio, old_dev, bio->bi_sector,
- maxsector);
+ old_sector);
blk_add_trace_bio(q, bio, BLK_TA_QUEUE);
- maxsector = bio->bi_sector;
+ old_sector = bio->bi_sector;
old_dev = bio->bi_bdev->bd_dev;
+ maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
+ if (maxsector) {
+ sector_t sector = bio->bi_sector;
+
+ if (maxsector < nr_sectors ||
+ maxsector - nr_sectors < sector) {
+ /*
+ * This may well happen - partitions are not
+ * checked to make sure they are within the size
+ * of the whole device.
+ */
+ handle_bad_sector(bio);
+ goto end_io;
+ }
+ }
+
ret = q->make_request_fn(q, bio);
} while (ret);
}
/* Called by the exitting task */
void exit_io_context(void)
{
- unsigned long flags;
struct io_context *ioc;
struct cfq_io_context *cic;
- local_irq_save(flags);
task_lock(current);
ioc = current->io_context;
current->io_context = NULL;
- ioc->task = NULL;
task_unlock(current);
- local_irq_restore(flags);
+ ioc->task = NULL;
if (ioc->aic && ioc->aic->exit)
ioc->aic->exit(ioc->aic);
if (ioc->cic_root.rb_node != NULL) {
cic = rb_entry(rb_first(&ioc->cic_root), struct cfq_io_context, rb_node);
cic->exit(ioc);
}
-
+
put_io_context(ioc);
}
* but since the current task itself holds a reference, the context can be
* used in general code, so long as it stays within `current` context.
*/
-struct io_context *current_io_context(gfp_t gfp_flags)
+static struct io_context *current_io_context(gfp_t gfp_flags, int node)
{
struct task_struct *tsk = current;
struct io_context *ret;
if (likely(ret))
return ret;
- ret = kmem_cache_alloc(iocontext_cachep, gfp_flags);
+ ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);
if (ret) {
atomic_set(&ret->refcount, 1);
ret->task = current;
- ret->set_ioprio = NULL;
+ ret->ioprio_changed = 0;
ret->last_waited = jiffies; /* doesn't matter... */
ret->nr_batch_requests = 0; /* because this is 0 */
ret->aic = NULL;
*
* This is always called in the context of the task which submitted the I/O.
*/
-struct io_context *get_io_context(gfp_t gfp_flags)
+struct io_context *get_io_context(gfp_t gfp_flags, int node)
{
struct io_context *ret;
- ret = current_io_context(gfp_flags);
+ ret = current_io_context(gfp_flags, node);
if (likely(ret))
atomic_inc(&ret->refcount);
return ret;
blk_queue_congestion_threshold(q);
if (rl->count[READ] >= queue_congestion_on_threshold(q))
- set_queue_congested(q, READ);
+ blk_set_queue_congested(q, READ);
else if (rl->count[READ] < queue_congestion_off_threshold(q))
- clear_queue_congested(q, READ);
+ blk_clear_queue_congested(q, READ);
if (rl->count[WRITE] >= queue_congestion_on_threshold(q))
- set_queue_congested(q, WRITE);
+ blk_set_queue_congested(q, WRITE);
else if (rl->count[WRITE] < queue_congestion_off_threshold(q))
- clear_queue_congested(q, WRITE);
+ blk_clear_queue_congested(q, WRITE);
if (rl->count[READ] >= q->nr_requests) {
blk_set_queue_full(q, READ);
ssize_t ret = queue_var_store(&ra_kb, page, count);
spin_lock_irq(q->queue_lock);
- if (ra_kb > (q->max_sectors >> 1))
- ra_kb = (q->max_sectors >> 1);
-
q->backing_dev_info.ra_pages = ra_kb >> (PAGE_CACHE_SHIFT - 10);
spin_unlock_irq(q->queue_lock);