#define SCHED_FEAT(name, enabled) \
#name ,
-__read_mostly char *sched_feat_names[] = {
+static __read_mostly char *sched_feat_names[] = {
#include "sched_features.h"
NULL
};
#undef SCHED_FEAT
-int sched_feat_open(struct inode *inode, struct file *filp)
+static int sched_feat_open(struct inode *inode, struct file *filp)
{
filp->private_data = inode->i_private;
return 0;
{
struct rq *rq = cpu_rq(smp_processor_id());
+ WARN_ON(!irqs_disabled());
spin_lock(&rq->lock);
__update_rq_clock(rq);
spin_unlock(&rq->lock);
struct rq *rq = cpu_rq(smp_processor_id());
u64 now = sched_clock();
+ WARN_ON(!irqs_disabled());
rq->idle_clock += delta_ns;
/*
* Override the previous timestamp and ignore all
enum {
HRTICK_SET, /* re-programm hrtick_timer */
HRTICK_RESET, /* not a new slice */
+ HRTICK_BLOCK, /* stop hrtick operations */
};
/*
{
if (!sched_feat(HRTICK))
return 0;
+ if (unlikely(test_bit(HRTICK_BLOCK, &rq->hrtick_flags)))
+ return 0;
return hrtimer_is_hres_active(&rq->hrtick_timer);
}
return HRTIMER_NORESTART;
}
-static inline void init_rq_hrtick(struct rq *rq)
+static void hotplug_hrtick_disable(int cpu)
+{
+ struct rq *rq = cpu_rq(cpu);
+ unsigned long flags;
+
+ spin_lock_irqsave(&rq->lock, flags);
+ rq->hrtick_flags = 0;
+ __set_bit(HRTICK_BLOCK, &rq->hrtick_flags);
+ spin_unlock_irqrestore(&rq->lock, flags);
+
+ hrtick_clear(rq);
+}
+
+static void hotplug_hrtick_enable(int cpu)
+{
+ struct rq *rq = cpu_rq(cpu);
+ unsigned long flags;
+
+ spin_lock_irqsave(&rq->lock, flags);
+ __clear_bit(HRTICK_BLOCK, &rq->hrtick_flags);
+ spin_unlock_irqrestore(&rq->lock, flags);
+}
+
+static int
+hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
+{
+ int cpu = (int)(long)hcpu;
+
+ switch (action) {
+ case CPU_UP_CANCELED:
+ case CPU_UP_CANCELED_FROZEN:
+ case CPU_DOWN_PREPARE:
+ case CPU_DOWN_PREPARE_FROZEN:
+ case CPU_DEAD:
+ case CPU_DEAD_FROZEN:
+ hotplug_hrtick_disable(cpu);
+ return NOTIFY_OK;
+
+ case CPU_UP_PREPARE:
+ case CPU_UP_PREPARE_FROZEN:
+ case CPU_DOWN_FAILED:
+ case CPU_DOWN_FAILED_FROZEN:
+ case CPU_ONLINE:
+ case CPU_ONLINE_FROZEN:
+ hotplug_hrtick_enable(cpu);
+ return NOTIFY_OK;
+ }
+
+ return NOTIFY_DONE;
+}
+
+static void init_hrtick(void)
+{
+ hotcpu_notifier(hotplug_hrtick, 0);
+}
+
+static void init_rq_hrtick(struct rq *rq)
{
rq->hrtick_flags = 0;
hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
void hrtick_resched(void)
{
}
+
+static inline void init_hrtick(void)
+{
+}
#endif
/*
{
u64 tmp;
- if (unlikely(!lw->inv_weight))
- lw->inv_weight = (WMULT_CONST-lw->weight/2) / (lw->weight+1);
+ if (!lw->inv_weight)
+ lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)/(lw->weight+1);
tmp = (u64)delta_exec * weight;
/*
aggregate(tg, sd)->task_weight = task_weight;
}
-/*
- * Redistribute tg->shares amongst all tg->cfs_rq[]s.
- */
-static void __aggregate_redistribute_shares(struct task_group *tg)
-{
- int i, max_cpu = smp_processor_id();
- unsigned long rq_weight = 0;
- unsigned long shares, max_shares = 0, shares_rem = tg->shares;
-
- for_each_possible_cpu(i)
- rq_weight += tg->cfs_rq[i]->load.weight;
-
- for_each_possible_cpu(i) {
- /*
- * divide shares proportional to the rq_weights.
- */
- shares = tg->shares * tg->cfs_rq[i]->load.weight;
- shares /= rq_weight + 1;
-
- tg->cfs_rq[i]->shares = shares;
-
- if (shares > max_shares) {
- max_shares = shares;
- max_cpu = i;
- }
- shares_rem -= shares;
- }
-
- /*
- * Ensure it all adds up to tg->shares; we can loose a few
- * due to rounding down when computing the per-cpu shares.
- */
- if (shares_rem)
- tg->cfs_rq[max_cpu]->shares += shares_rem;
-}
-
/*
* Compute the weight of this group on the given cpus.
*/
unsigned long shares = 0;
int i;
-again:
for_each_cpu_mask(i, sd->span)
shares += tg->cfs_rq[i]->shares;
- /*
- * When the span doesn't have any shares assigned, but does have
- * tasks to run do a machine wide rebalance (should be rare).
- */
- if (unlikely(!shares && aggregate(tg, sd)->rq_weight)) {
- __aggregate_redistribute_shares(tg);
- goto again;
- }
+ if ((!shares && aggregate(tg, sd)->rq_weight) || shares > tg->shares)
+ shares = tg->shares;
aggregate(tg, sd)->shares = shares;
}
struct rq *rq = this_rq();
cputime64_t tmp;
- if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0))
- return account_guest_time(p, cputime);
+ if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
+ account_guest_time(p, cputime);
+ return;
+ }
p->stime = cputime_add(p->stime, cputime);
prev->sched_class->put_prev_task(rq, prev);
next = pick_next_task(rq, prev);
- sched_info_switch(prev, next);
-
if (likely(prev != next)) {
+ sched_info_switch(prev, next);
+
rq->nr_switches++;
rq->curr = next;
++*switch_count;
put_online_cpus();
/* XXX: Theoretical race here - CPU may be hotplugged now */
hotcpu_notifier(update_sched_domains, 0);
+ init_hrtick();
/* Move init over to a non-isolated CPU */
if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0)
#else
void __init sched_init_smp(void)
{
-#if defined(CONFIG_NUMA)
- sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
- GFP_KERNEL);
- BUG_ON(sched_group_nodes_bycpu == NULL);
-#endif
sched_init_granularity();
}
#endif /* CONFIG_SMP */
se->my_q = cfs_rq;
se->load.weight = tg->shares;
- se->load.inv_weight = div64_64(1ULL<<32, se->load.weight);
+ se->load.inv_weight = 0;
se->parent = parent;
}
#endif
* we use alloc_bootmem().
*/
if (alloc_size) {
- ptr = (unsigned long)alloc_bootmem_low(alloc_size);
+ ptr = (unsigned long)alloc_bootmem(alloc_size);
#ifdef CONFIG_FAIR_GROUP_SCHED
init_task_group.se = (struct sched_entity **)ptr;
dequeue_entity(cfs_rq, se, 0);
se->load.weight = shares;
- se->load.inv_weight = div64_64((1ULL<<32), shares);
+ se->load.inv_weight = 0;
if (on_rq)
enqueue_entity(cfs_rq, se, 0);
if (runtime == RUNTIME_INF)
return 1ULL << 16;
- return div64_64(runtime << 16, period);
+ return div64_u64(runtime << 16, period);
}
#ifdef CONFIG_CGROUP_SCHED
}
#ifdef CONFIG_FAIR_GROUP_SCHED
-static int cpu_shares_write_uint(struct cgroup *cgrp, struct cftype *cftype,
+static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
u64 shareval)
{
return sched_group_set_shares(cgroup_tg(cgrp), shareval);
}
-static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
{
struct task_group *tg = cgroup_tg(cgrp);
#ifdef CONFIG_RT_GROUP_SCHED
static ssize_t cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
- struct file *file,
- const char __user *userbuf,
- size_t nbytes, loff_t *unused_ppos)
+ s64 val)
{
- char buffer[64];
- int retval = 0;
- s64 val;
- char *end;
-
- if (!nbytes)
- return -EINVAL;
- if (nbytes >= sizeof(buffer))
- return -E2BIG;
- if (copy_from_user(buffer, userbuf, nbytes))
- return -EFAULT;
-
- buffer[nbytes] = 0; /* nul-terminate */
-
- /* strip newline if necessary */
- if (nbytes && (buffer[nbytes-1] == '\n'))
- buffer[nbytes-1] = 0;
- val = simple_strtoll(buffer, &end, 0);
- if (*end)
- return -EINVAL;
-
- /* Pass to subsystem */
- retval = sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
- if (!retval)
- retval = nbytes;
- return retval;
+ return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
}
-static ssize_t cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft,
- struct file *file,
- char __user *buf, size_t nbytes,
- loff_t *ppos)
+static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
{
- char tmp[64];
- long val = sched_group_rt_runtime(cgroup_tg(cgrp));
- int len = sprintf(tmp, "%ld\n", val);
-
- return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
+ return sched_group_rt_runtime(cgroup_tg(cgrp));
}
static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
#ifdef CONFIG_FAIR_GROUP_SCHED
{
.name = "shares",
- .read_uint = cpu_shares_read_uint,
- .write_uint = cpu_shares_write_uint,
+ .read_u64 = cpu_shares_read_u64,
+ .write_u64 = cpu_shares_write_u64,
},
#endif
#ifdef CONFIG_RT_GROUP_SCHED
{
.name = "rt_runtime_us",
- .read = cpu_rt_runtime_read,
- .write = cpu_rt_runtime_write,
+ .read_s64 = cpu_rt_runtime_read,
+ .write_s64 = cpu_rt_runtime_write,
},
{
.name = "rt_period_us",
- .read_uint = cpu_rt_period_read_uint,
- .write_uint = cpu_rt_period_write_uint,
+ .read_u64 = cpu_rt_period_read_uint,
+ .write_u64 = cpu_rt_period_write_uint,
},
#endif
};
static struct cftype files[] = {
{
.name = "usage",
- .read_uint = cpuusage_read,
- .write_uint = cpuusage_write,
+ .read_u64 = cpuusage_read,
+ .write_u64 = cpuusage_write,
},
};
projects / linux-2.6-omap-h63xx.git / blobdiff