stop_machine: add ALL_CPUS option

[linux-2.6-omap-h63xx.git] / kernel / stop_machine.c

diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index b7350bbfb076293dd6d13d7ba060b3aba1f673a9..a473bd0cb71b756f628604a123c800adf814f2fc 100644 (file)
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -22,9 +22,17 @@ enum stopmachine_state {
        STOPMACHINE_WAIT,
        STOPMACHINE_PREPARE,
        STOPMACHINE_DISABLE_IRQ,
+       STOPMACHINE_RUN,
        STOPMACHINE_EXIT,
 };
 
+struct stop_machine_data {
+       int (*fn)(void *);
+       void *data;
+       struct completion done;
+       int run_all;
+} smdata;
+
 static enum stopmachine_state stopmachine_state;
 static unsigned int stopmachine_num_threads;
 static atomic_t stopmachine_thread_ack;
@@ -33,8 +41,10 @@ static int stopmachine(void *cpu)
 {
        int irqs_disabled = 0;
        int prepared = 0;
+       int ran = 0;
+       cpumask_of_cpu_ptr(cpumask, (int)(long)cpu);
 
-       set_cpus_allowed_ptr(current, &cpumask_of_cpu((int)(long)cpu));
+       set_cpus_allowed_ptr(current, cpumask);
 
        /* Ack: we are alive */
        smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */
@@ -57,6 +67,11 @@ static int stopmachine(void *cpu)
                        prepared = 1;
                        smp_mb(); /* Must read state first. */
                        atomic_inc(&stopmachine_thread_ack);
+               } else if (stopmachine_state == STOPMACHINE_RUN && !ran) {
+                       smdata.fn(smdata.data);
+                       ran = 1;
+                       smp_mb(); /* Must read state first. */
+                       atomic_inc(&stopmachine_thread_ack);
                }
                /* Yield in first stage: migration threads need to
                 * help our sisters onto their CPUs. */
@@ -135,11 +150,10 @@ static void restart_machine(void)
        preempt_enable_no_resched();
 }
 
-struct stop_machine_data {
-       int (*fn)(void *);
-       void *data;
-       struct completion done;
-};
+static void run_other_cpus(void)
+{
+       stopmachine_set_state(STOPMACHINE_RUN);
+}
 
 static int do_stop(void *_smdata)
 {
@@ -149,6 +163,8 @@ static int do_stop(void *_smdata)
        ret = stop_machine();
        if (ret == 0) {
                ret = smdata->fn(smdata->data);
+               if (smdata->run_all)
+                       run_other_cpus();
                restart_machine();
        }
 
@@ -172,14 +188,17 @@ struct task_struct *__stop_machine_run(int (*fn)(void *), void *data,
        struct stop_machine_data smdata;
        struct task_struct *p;
 
+       mutex_lock(&stopmachine_mutex);
+
        smdata.fn = fn;
        smdata.data = data;
+       smdata.run_all = (cpu == ALL_CPUS) ? 1 : 0;
        init_completion(&smdata.done);
 
-       mutex_lock(&stopmachine_mutex);
+       smp_wmb(); /* make sure other cpus see smdata updates */
 
        /* If they don't care which CPU fn runs on, bind to any online one. */
-       if (cpu == NR_CPUS)
+       if (cpu == NR_CPUS || cpu == ALL_CPUS)
                cpu = raw_smp_processor_id();
 
        p = kthread_create(do_stop, &smdata, "kstopmachine");
@@ -187,7 +206,7 @@ struct task_struct *__stop_machine_run(int (*fn)(void *), void *data,
                struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 
                /* One high-prio thread per cpu.  We'll do this one. */
-               sched_setscheduler(p, SCHED_FIFO, &param);
+               sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
                kthread_bind(p, cpu);
                wake_up_process(p);
                wait_for_completion(&smdata.done);