Merge branch 'linus' into core/softirq

[linux-2.6-omap-h63xx.git] / kernel / softirq.c

diff --git a/kernel/softirq.c b/kernel/softirq.c
index 31e9f2a4792847388b524d313bc389bd8cd4cf20..2cf2502b7c731d0d1f303286b67d1b19dc44e672 100644 (file)
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -131,23 +131,17 @@ void _local_bh_enable(void)
 
 EXPORT_SYMBOL(_local_bh_enable);
 
-void local_bh_enable(void)
+static inline void _local_bh_enable_ip(unsigned long ip)
 {
+       WARN_ON_ONCE(in_irq() || irqs_disabled());
 #ifdef CONFIG_TRACE_IRQFLAGS
-       unsigned long flags;
-
-       WARN_ON_ONCE(in_irq());
-#endif
-       WARN_ON_ONCE(irqs_disabled());
-
-#ifdef CONFIG_TRACE_IRQFLAGS
-       local_irq_save(flags);
+       local_irq_disable();
 #endif
        /*
         * Are softirqs going to be turned on now:
         */
        if (softirq_count() == SOFTIRQ_OFFSET)
-               trace_softirqs_on((unsigned long)__builtin_return_address(0));
+               trace_softirqs_on(ip);
        /*
         * Keep preemption disabled until we are done with
         * softirq processing:
@@ -159,40 +153,20 @@ void local_bh_enable(void)
 
        dec_preempt_count();
 #ifdef CONFIG_TRACE_IRQFLAGS
-       local_irq_restore(flags);
+       local_irq_enable();
 #endif
        preempt_check_resched();
 }
+
+void local_bh_enable(void)
+{
+       _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
+}
 EXPORT_SYMBOL(local_bh_enable);
 
 void local_bh_enable_ip(unsigned long ip)
 {
-#ifdef CONFIG_TRACE_IRQFLAGS
-       unsigned long flags;
-
-       WARN_ON_ONCE(in_irq());
-
-       local_irq_save(flags);
-#endif
-       /*
-        * Are softirqs going to be turned on now:
-        */
-       if (softirq_count() == SOFTIRQ_OFFSET)
-               trace_softirqs_on(ip);
-       /*
-        * Keep preemption disabled until we are done with
-        * softirq processing:
-        */
-       sub_preempt_count(SOFTIRQ_OFFSET - 1);
-
-       if (unlikely(!in_interrupt() && local_softirq_pending()))
-               do_softirq();
-
-       dec_preempt_count();
-#ifdef CONFIG_TRACE_IRQFLAGS
-       local_irq_restore(flags);
-#endif
-       preempt_check_resched();
+       _local_bh_enable_ip(ip);
 }
 EXPORT_SYMBOL(local_bh_enable_ip);
 
@@ -347,30 +321,29 @@ void raise_softirq(unsigned int nr)
        local_irq_restore(flags);
 }
 
-void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
+void open_softirq(int nr, void (*action)(struct softirq_action *))
 {
-       softirq_vec[nr].data = data;
        softirq_vec[nr].action = action;
 }
 
 /* Tasklets */
 struct tasklet_head
 {
-       struct tasklet_struct *list;
+       struct tasklet_struct *head;
+       struct tasklet_struct **tail;
 };
 
-/* Some compilers disobey section attribute on statics when not
-   initialized -- RR */
-static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
-static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
+static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
+static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 
 void __tasklet_schedule(struct tasklet_struct *t)
 {
        unsigned long flags;
 
        local_irq_save(flags);
-       t->next = __get_cpu_var(tasklet_vec).list;
-       __get_cpu_var(tasklet_vec).list = t;
+       t->next = NULL;
+       *__get_cpu_var(tasklet_vec).tail = t;
+       __get_cpu_var(tasklet_vec).tail = &(t->next);
        raise_softirq_irqoff(TASKLET_SOFTIRQ);
        local_irq_restore(flags);
 }
@@ -382,8 +355,9 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
        unsigned long flags;
 
        local_irq_save(flags);
-       t->next = __get_cpu_var(tasklet_hi_vec).list;
-       __get_cpu_var(tasklet_hi_vec).list = t;
+       t->next = NULL;
+       *__get_cpu_var(tasklet_hi_vec).tail = t;
+       __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
        raise_softirq_irqoff(HI_SOFTIRQ);
        local_irq_restore(flags);
 }
@@ -395,8 +369,9 @@ static void tasklet_action(struct softirq_action *a)
        struct tasklet_struct *list;
 
        local_irq_disable();
-       list = __get_cpu_var(tasklet_vec).list;
-       __get_cpu_var(tasklet_vec).list = NULL;
+       list = __get_cpu_var(tasklet_vec).head;
+       __get_cpu_var(tasklet_vec).head = NULL;
+       __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
        local_irq_enable();
 
        while (list) {
@@ -416,8 +391,9 @@ static void tasklet_action(struct softirq_action *a)
                }
 
                local_irq_disable();
-               t->next = __get_cpu_var(tasklet_vec).list;
-               __get_cpu_var(tasklet_vec).list = t;
+               t->next = NULL;
+               *__get_cpu_var(tasklet_vec).tail = t;
+               __get_cpu_var(tasklet_vec).tail = &(t->next);
                __raise_softirq_irqoff(TASKLET_SOFTIRQ);
                local_irq_enable();
        }
@@ -428,8 +404,9 @@ static void tasklet_hi_action(struct softirq_action *a)
        struct tasklet_struct *list;
 
        local_irq_disable();
-       list = __get_cpu_var(tasklet_hi_vec).list;
-       __get_cpu_var(tasklet_hi_vec).list = NULL;
+       list = __get_cpu_var(tasklet_hi_vec).head;
+       __get_cpu_var(tasklet_hi_vec).head = NULL;
+       __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
        local_irq_enable();
 
        while (list) {
@@ -449,8 +426,9 @@ static void tasklet_hi_action(struct softirq_action *a)
                }
 
                local_irq_disable();
-               t->next = __get_cpu_var(tasklet_hi_vec).list;
-               __get_cpu_var(tasklet_hi_vec).list = t;
+               t->next = NULL;
+               *__get_cpu_var(tasklet_hi_vec).tail = t;
+               __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
                __raise_softirq_irqoff(HI_SOFTIRQ);
                local_irq_enable();
        }
@@ -487,8 +465,17 @@ EXPORT_SYMBOL(tasklet_kill);
 
 void __init softirq_init(void)
 {
-       open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
-       open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
+       int cpu;
+
+       for_each_possible_cpu(cpu) {
+               per_cpu(tasklet_vec, cpu).tail =
+                       &per_cpu(tasklet_vec, cpu).head;
+               per_cpu(tasklet_hi_vec, cpu).tail =
+                       &per_cpu(tasklet_hi_vec, cpu).head;
+       }
+
+       open_softirq(TASKLET_SOFTIRQ, tasklet_action);
+       open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 }
 
 static int ksoftirqd(void * __bind_cpu)
@@ -555,9 +542,12 @@ void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
                return;
 
        /* CPU is dead, so no lock needed. */
-       for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
+       for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
                if (*i == t) {
                        *i = t->next;
+                       /* If this was the tail element, move the tail ptr */
+                       if (*i == NULL)
+                               per_cpu(tasklet_vec, cpu).tail = i;
                        return;
                }
        }
@@ -566,20 +556,24 @@ void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 
 static void takeover_tasklets(unsigned int cpu)
 {
-       struct tasklet_struct **i;
-
        /* CPU is dead, so no lock needed. */
        local_irq_disable();
 
        /* Find end, append list for that CPU. */
-       for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
-       *i = per_cpu(tasklet_vec, cpu).list;
-       per_cpu(tasklet_vec, cpu).list = NULL;
+       if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
+               *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
+               __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
+               per_cpu(tasklet_vec, cpu).head = NULL;
+               per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
+       }
        raise_softirq_irqoff(TASKLET_SOFTIRQ);
 
-       for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
-       *i = per_cpu(tasklet_hi_vec, cpu).list;
-       per_cpu(tasklet_hi_vec, cpu).list = NULL;
+       if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
+               *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
+               __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
+               per_cpu(tasklet_hi_vec, cpu).head = NULL;
+               per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
+       }
        raise_softirq_irqoff(HI_SOFTIRQ);
 
        local_irq_enable();