/*
* tunables
*/
-static const int cfq_quantum = 4; /* max queue in one round of service */
+/* max queue in one round of service */
+static const int cfq_quantum = 4;
static const int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 };
-static const int cfq_back_max = 16 * 1024; /* maximum backwards seek, in KiB */
-static const int cfq_back_penalty = 2; /* penalty of a backwards seek */
-
+/* maximum backwards seek, in KiB */
+static const int cfq_back_max = 16 * 1024;
+/* penalty of a backwards seek */
+static const int cfq_back_penalty = 2;
static const int cfq_slice_sync = HZ / 10;
static int cfq_slice_async = HZ / 25;
static const int cfq_slice_async_rq = 2;
#define CFQ_SLICE_SCALE (5)
-#define RQ_CIC(rq) ((struct cfq_io_context*)(rq)->elevator_private)
+#define RQ_CIC(rq) \
+ ((struct cfq_io_context *) (rq)->elevator_private)
#define RQ_CFQQ(rq) ((rq)->elevator_private2)
static struct kmem_cache *cfq_pool;
#define CFQ_CFQQ_FNS(name) \
static inline void cfq_mark_cfqq_##name(struct cfq_queue *cfqq) \
{ \
- cfqq->flags |= (1 << CFQ_CFQQ_FLAG_##name); \
+ (cfqq)->flags |= (1 << CFQ_CFQQ_FLAG_##name); \
} \
static inline void cfq_clear_cfqq_##name(struct cfq_queue *cfqq) \
{ \
- cfqq->flags &= ~(1 << CFQ_CFQQ_FLAG_##name); \
+ (cfqq)->flags &= ~(1 << CFQ_CFQQ_FLAG_##name); \
} \
static inline int cfq_cfqq_##name(const struct cfq_queue *cfqq) \
{ \
- return (cfqq->flags & (1 << CFQ_CFQQ_FLAG_##name)) != 0; \
+ return ((cfqq)->flags & (1 << CFQ_CFQQ_FLAG_##name)) != 0; \
}
CFQ_CFQQ_FNS(on_rr);
/*
* follow expired path, else get first next available
*/
- if ((rq = cfq_check_fifo(cfqq)) == NULL)
+ rq = cfq_check_fifo(cfqq);
+ if (rq == NULL)
rq = cfqq->next_rq;
/*
}
/*
- * Call func for each cic attached to this ioc. Returns number of cic's seen.
+ * Call func for each cic attached to this ioc.
*/
-#define CIC_GANG_NR 16
-static unsigned int
+static void
call_for_each_cic(struct io_context *ioc,
void (*func)(struct io_context *, struct cfq_io_context *))
{
- struct cfq_io_context *cics[CIC_GANG_NR];
- unsigned long index = 0;
- unsigned int called = 0;
- int nr;
+ struct cfq_io_context *cic;
+ struct hlist_node *n;
rcu_read_lock();
+ hlist_for_each_entry_rcu(cic, n, &ioc->cic_list, cic_list)
+ func(ioc, cic);
+ rcu_read_unlock();
+}
- do {
- int i;
-
- /*
- * Perhaps there's a better way - this just gang lookups from
- * 0 to the end, restarting after each CIC_GANG_NR from the
- * last key + 1.
- */
- nr = radix_tree_gang_lookup(&ioc->radix_root, (void **) cics,
- index, CIC_GANG_NR);
- if (!nr)
- break;
+static void cfq_cic_free_rcu(struct rcu_head *head)
+{
+ struct cfq_io_context *cic;
- called += nr;
- index = 1 + (unsigned long) cics[nr - 1]->key;
+ cic = container_of(head, struct cfq_io_context, rcu_head);
- for (i = 0; i < nr; i++)
- func(ioc, cics[i]);
- } while (nr == CIC_GANG_NR);
+ kmem_cache_free(cfq_ioc_pool, cic);
+ elv_ioc_count_dec(ioc_count);
- rcu_read_unlock();
+ if (ioc_gone && !elv_ioc_count_read(ioc_count))
+ complete(ioc_gone);
+}
- return called;
+static void cfq_cic_free(struct cfq_io_context *cic)
+{
+ call_rcu(&cic->rcu_head, cfq_cic_free_rcu);
}
static void cic_free_func(struct io_context *ioc, struct cfq_io_context *cic)
spin_lock_irqsave(&ioc->lock, flags);
radix_tree_delete(&ioc->radix_root, cic->dead_key);
+ hlist_del_rcu(&cic->cic_list);
spin_unlock_irqrestore(&ioc->lock, flags);
- kmem_cache_free(cfq_ioc_pool, cic);
+ cfq_cic_free(cic);
}
static void cfq_free_io_context(struct io_context *ioc)
{
- int freed;
-
/*
- * ioc->refcount is zero here, so no more cic's are allowed to be
- * linked into this ioc. So it should be ok to iterate over the known
- * list, we will see all cic's since no new ones are added.
+ * ioc->refcount is zero here, or we are called from elv_unregister(),
+ * so no more cic's are allowed to be linked into this ioc. So it
+ * should be ok to iterate over the known list, we will see all cic's
+ * since no new ones are added.
*/
- freed = call_for_each_cic(ioc, cic_free_func);
-
- elv_ioc_count_mod(ioc_count, -freed);
-
- if (ioc_gone && !elv_ioc_count_read(ioc_count))
- complete(ioc_gone);
+ call_for_each_cic(ioc, cic_free_func);
}
static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
static void __cfq_exit_single_io_context(struct cfq_data *cfqd,
struct cfq_io_context *cic)
{
+ struct io_context *ioc = cic->ioc;
+
list_del_init(&cic->queue_list);
/*
cic->dead_key = (unsigned long) cic->key;
cic->key = NULL;
+ if (ioc->ioc_data == cic)
+ rcu_assign_pointer(ioc->ioc_data, NULL);
+
if (cic->cfqq[ASYNC]) {
cfq_exit_cfqq(cfqd, cic->cfqq[ASYNC]);
cic->cfqq[ASYNC] = NULL;
*/
static void cfq_exit_io_context(struct io_context *ioc)
{
- rcu_assign_pointer(ioc->ioc_data, NULL);
call_for_each_cic(ioc, cfq_exit_single_io_context);
}
if (cic) {
cic->last_end_request = jiffies;
INIT_LIST_HEAD(&cic->queue_list);
+ INIT_HLIST_NODE(&cic->cic_list);
cic->dtor = cfq_free_io_context;
cic->exit = cfq_exit_io_context;
elv_ioc_count_inc(ioc_count);
ioprio_class = IOPRIO_PRIO_CLASS(ioc->ioprio);
switch (ioprio_class) {
- default:
- printk(KERN_ERR "cfq: bad prio %x\n", ioprio_class);
- case IOPRIO_CLASS_NONE:
- /*
- * no prio set, place us in the middle of the BE classes
- */
- cfqq->ioprio = task_nice_ioprio(tsk);
- cfqq->ioprio_class = IOPRIO_CLASS_BE;
- break;
- case IOPRIO_CLASS_RT:
- cfqq->ioprio = task_ioprio(ioc);
- cfqq->ioprio_class = IOPRIO_CLASS_RT;
- break;
- case IOPRIO_CLASS_BE:
- cfqq->ioprio = task_ioprio(ioc);
- cfqq->ioprio_class = IOPRIO_CLASS_BE;
- break;
- case IOPRIO_CLASS_IDLE:
- cfqq->ioprio_class = IOPRIO_CLASS_IDLE;
- cfqq->ioprio = 7;
- cfq_clear_cfqq_idle_window(cfqq);
- break;
+ default:
+ printk(KERN_ERR "cfq: bad prio %x\n", ioprio_class);
+ case IOPRIO_CLASS_NONE:
+ /*
+ * no prio set, place us in the middle of the BE classes
+ */
+ cfqq->ioprio = task_nice_ioprio(tsk);
+ cfqq->ioprio_class = IOPRIO_CLASS_BE;
+ break;
+ case IOPRIO_CLASS_RT:
+ cfqq->ioprio = task_ioprio(ioc);
+ cfqq->ioprio_class = IOPRIO_CLASS_RT;
+ break;
+ case IOPRIO_CLASS_BE:
+ cfqq->ioprio = task_ioprio(ioc);
+ cfqq->ioprio_class = IOPRIO_CLASS_BE;
+ break;
+ case IOPRIO_CLASS_IDLE:
+ cfqq->ioprio_class = IOPRIO_CLASS_IDLE;
+ cfqq->ioprio = 7;
+ cfq_clear_cfqq_idle_window(cfqq);
+ break;
}
/*
static struct cfq_queue **
cfq_async_queue_prio(struct cfq_data *cfqd, int ioprio_class, int ioprio)
{
- switch(ioprio_class) {
+ switch (ioprio_class) {
case IOPRIO_CLASS_RT:
return &cfqd->async_cfqq[0][ioprio];
case IOPRIO_CLASS_BE:
return cfqq;
}
-static void cfq_cic_free(struct cfq_io_context *cic)
-{
- kmem_cache_free(cfq_ioc_pool, cic);
- elv_ioc_count_dec(ioc_count);
-
- if (ioc_gone && !elv_ioc_count_read(ioc_count))
- complete(ioc_gone);
-}
-
/*
* We drop cfq io contexts lazily, so we may find a dead one.
*/
spin_lock_irqsave(&ioc->lock, flags);
- if (ioc->ioc_data == cic)
- rcu_assign_pointer(ioc->ioc_data, NULL);
+ BUG_ON(ioc->ioc_data == cic);
radix_tree_delete(&ioc->radix_root, (unsigned long) cfqd);
+ hlist_del_rcu(&cic->cic_list);
spin_unlock_irqrestore(&ioc->lock, flags);
cfq_cic_free(cic);
spin_lock_irqsave(&ioc->lock, flags);
ret = radix_tree_insert(&ioc->radix_root,
(unsigned long) cfqd, cic);
+ if (!ret)
+ hlist_add_head_rcu(&cic->cic_list, &ioc->cic_list);
spin_unlock_irqrestore(&ioc->lock, flags);
radix_tree_preload_end();
spin_lock_irqsave(cfqd->queue->queue_lock, flags);
- if ((cfqq = cfqd->active_queue) != NULL) {
+ cfqq = cfqd->active_queue;
+ if (cfqq) {
timed_out = 0;
/*
if (!cfq_pool)
goto fail;
- cfq_ioc_pool = KMEM_CACHE(cfq_io_context, SLAB_DESTROY_BY_RCU);
+ cfq_ioc_pool = KMEM_CACHE(cfq_io_context, 0);
if (!cfq_ioc_pool)
goto fail;
return ret; \
}
STORE_FUNCTION(cfq_quantum_store, &cfqd->cfq_quantum, 1, UINT_MAX, 0);
-STORE_FUNCTION(cfq_fifo_expire_sync_store, &cfqd->cfq_fifo_expire[1], 1, UINT_MAX, 1);
-STORE_FUNCTION(cfq_fifo_expire_async_store, &cfqd->cfq_fifo_expire[0], 1, UINT_MAX, 1);
+STORE_FUNCTION(cfq_fifo_expire_sync_store, &cfqd->cfq_fifo_expire[1], 1,
+ UINT_MAX, 1);
+STORE_FUNCTION(cfq_fifo_expire_async_store, &cfqd->cfq_fifo_expire[0], 1,
+ UINT_MAX, 1);
STORE_FUNCTION(cfq_back_seek_max_store, &cfqd->cfq_back_max, 0, UINT_MAX, 0);
-STORE_FUNCTION(cfq_back_seek_penalty_store, &cfqd->cfq_back_penalty, 1, UINT_MAX, 0);
+STORE_FUNCTION(cfq_back_seek_penalty_store, &cfqd->cfq_back_penalty, 1,
+ UINT_MAX, 0);
STORE_FUNCTION(cfq_slice_idle_store, &cfqd->cfq_slice_idle, 0, UINT_MAX, 1);
STORE_FUNCTION(cfq_slice_sync_store, &cfqd->cfq_slice[1], 1, UINT_MAX, 1);
STORE_FUNCTION(cfq_slice_async_store, &cfqd->cfq_slice[0], 1, UINT_MAX, 1);
-STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1, UINT_MAX, 0);
+STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1,
+ UINT_MAX, 0);
#undef STORE_FUNCTION
#define CFQ_ATTR(name) \
smp_wmb();
if (elv_ioc_count_read(ioc_count))
wait_for_completion(ioc_gone);
- synchronize_rcu();
cfq_slab_kill();
}