kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/smp.h>
  25 #include <linux/tick.h>
  26
  27 #include <asm/irq.h>
  28 /*
  29    - No shared variables, all the data are CPU local.
  30    - If a softirq needs serialization, let it serialize itself
  31      by its own spinlocks.
  32    - Even if softirq is serialized, only local cpu is marked for
  33      execution. Hence, we get something sort of weak cpu binding.
  34      Though it is still not clear, will it result in better locality
  35      or will not.
  36
  37    Examples:
  38    - NET RX softirq. It is multithreaded and does not require
  39      any global serialization.
  40    - NET TX softirq. It kicks software netdevice queues, hence
  41      it is logically serialized per device, but this serialization
  42      is invisible to common code.
  43    - Tasklets: serialized wrt itself.
  44  */
  45
  46 #ifndef __ARCH_IRQ_STAT
  47 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  48 EXPORT_SYMBOL(irq_stat);
  49 #endif
  50
  51 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  52
  53 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  54
  55 /*
  56  * we cannot loop indefinitely here to avoid userspace starvation,
  57  * but we also don't want to introduce a worst case 1/HZ latency
  58  * to the pending events, so lets the scheduler to balance
  59  * the softirq load for us.
  60  */
  61 static inline void wakeup_softirqd(void)
  62 {
  63         /* Interrupts are disabled: no need to stop preemption */
  64         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  65
  66         if (tsk && tsk->state != TASK_RUNNING)
  67                 wake_up_process(tsk);
  68 }
  69
  70 /*
  71  * This one is for softirq.c-internal use,
  72  * where hardirqs are disabled legitimately:
  73  */
  74 #ifdef CONFIG_TRACE_IRQFLAGS
  75 static void __local_bh_disable(unsigned long ip)
  76 {
  77         unsigned long flags;
  78
  79         WARN_ON_ONCE(in_irq());
  80
  81         raw_local_irq_save(flags);
  82         add_preempt_count(SOFTIRQ_OFFSET);
  83         /*
  84          * Were softirqs turned off above:
  85          */
  86         if (softirq_count() == SOFTIRQ_OFFSET)
  87                 trace_softirqs_off(ip);
  88         raw_local_irq_restore(flags);
  89 }
  90 #else /* !CONFIG_TRACE_IRQFLAGS */
  91 static inline void __local_bh_disable(unsigned long ip)
  92 {
  93         add_preempt_count(SOFTIRQ_OFFSET);
  94         barrier();
  95 }
  96 #endif /* CONFIG_TRACE_IRQFLAGS */
  97
  98 void local_bh_disable(void)
  99 {
 100         __local_bh_disable((unsigned long)__builtin_return_address(0));
 101 }
 102
 103 EXPORT_SYMBOL(local_bh_disable);
 104
 105 void __local_bh_enable(void)
 106 {
 107         WARN_ON_ONCE(in_irq());
 108
 109         /*
 110          * softirqs should never be enabled by __local_bh_enable(),
 111          * it always nests inside local_bh_enable() sections:
 112          */
 113         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
 114
 115         sub_preempt_count(SOFTIRQ_OFFSET);
 116 }
 117 EXPORT_SYMBOL_GPL(__local_bh_enable);
 118
 119 /*
 120  * Special-case - softirqs can safely be enabled in
 121  * cond_resched_softirq(), or by __do_softirq(),
 122  * without processing still-pending softirqs:
 123  */
 124 void _local_bh_enable(void)
 125 {
 126         WARN_ON_ONCE(in_irq());
 127         WARN_ON_ONCE(!irqs_disabled());
 128
 129         if (softirq_count() == SOFTIRQ_OFFSET)
 130                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 131         sub_preempt_count(SOFTIRQ_OFFSET);
 132 }
 133
 134 EXPORT_SYMBOL(_local_bh_enable);
 135
 136 static inline void _local_bh_enable_ip(unsigned long ip)
 137 {
 138         WARN_ON_ONCE(in_irq() || irqs_disabled());
 139 #ifdef CONFIG_TRACE_IRQFLAGS
 140         local_irq_disable();
 141 #endif
 142         /*
 143          * Are softirqs going to be turned on now:
 144          */
 145         if (softirq_count() == SOFTIRQ_OFFSET)
 146                 trace_softirqs_on(ip);
 147         /*
 148          * Keep preemption disabled until we are done with
 149          * softirq processing:
 150          */
 151         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 152
 153         if (unlikely(!in_interrupt() && local_softirq_pending()))
 154                 do_softirq();
 155
 156         dec_preempt_count();
 157 #ifdef CONFIG_TRACE_IRQFLAGS
 158         local_irq_enable();
 159 #endif
 160         preempt_check_resched();
 161 }
 162
 163 void local_bh_enable(void)
 164 {
 165         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 166 }
 167 EXPORT_SYMBOL(local_bh_enable);
 168
 169 void local_bh_enable_ip(unsigned long ip)
 170 {
 171         _local_bh_enable_ip(ip);
 172 }
 173 EXPORT_SYMBOL(local_bh_enable_ip);
 174
 175 /*
 176  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 177  * and we fall back to softirqd after that.
 178  *
 179  * This number has been established via experimentation.
 180  * The two things to balance is latency against fairness -
 181  * we want to handle softirqs as soon as possible, but they
 182  * should not be able to lock up the box.
 183  */
 184 #define MAX_SOFTIRQ_RESTART 10
 185
 186 asmlinkage void __do_softirq(void)
 187 {
 188         struct softirq_action *h;
 189         __u32 pending;
 190         int max_restart = MAX_SOFTIRQ_RESTART;
 191         int cpu;
 192
 193         pending = local_softirq_pending();
 194         account_system_vtime(current);
 195
 196         __local_bh_disable((unsigned long)__builtin_return_address(0));
 197         trace_softirq_enter();
 198
 199         cpu = smp_processor_id();
 200 restart:
 201         /* Reset the pending bitmask before enabling irqs */
 202         set_softirq_pending(0);
 203
 204         local_irq_enable();
 205
 206         h = softirq_vec;
 207
 208         do {
 209                 if (pending & 1) {
 210                         int prev_count = preempt_count();
 211
 212                         h->action(h);
 213
 214                         if (unlikely(prev_count != preempt_count())) {
 215                                 printk(KERN_ERR "huh, entered softirq %td %p"
 216                                        "with preempt_count %08x,"
 217                                        " exited with %08x?\n", h - softirq_vec,
 218                                        h->action, prev_count, preempt_count());
 219                                 preempt_count() = prev_count;
 220                         }
 221
 222                         rcu_bh_qsctr_inc(cpu);
 223                 }
 224                 h++;
 225                 pending >>= 1;
 226         } while (pending);
 227
 228         local_irq_disable();
 229
 230         pending = local_softirq_pending();
 231         if (pending && --max_restart)
 232                 goto restart;
 233
 234         if (pending)
 235                 wakeup_softirqd();
 236
 237         trace_softirq_exit();
 238
 239         account_system_vtime(current);
 240         _local_bh_enable();
 241 }
 242
 243 #ifndef __ARCH_HAS_DO_SOFTIRQ
 244
 245 asmlinkage void do_softirq(void)
 246 {
 247         __u32 pending;
 248         unsigned long flags;
 249
 250         if (in_interrupt())
 251                 return;
 252
 253         local_irq_save(flags);
 254
 255         pending = local_softirq_pending();
 256
 257         if (pending)
 258                 __do_softirq();
 259
 260         local_irq_restore(flags);
 261 }
 262
 263 #endif
 264
 265 /*
 266  * Enter an interrupt context.
 267  */
 268 void irq_enter(void)
 269 {
 270         int cpu = smp_processor_id();
 271
 272         if (idle_cpu(cpu) && !in_interrupt()) {
 273                 __irq_enter();
 274                 tick_check_idle(cpu);
 275         } else
 276                 __irq_enter();
 277 }
 278
 279 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 280 # define invoke_softirq()       __do_softirq()
 281 #else
 282 # define invoke_softirq()       do_softirq()
 283 #endif
 284
 285 /*
 286  * Exit an interrupt context. Process softirqs if needed and possible:
 287  */
 288 void irq_exit(void)
 289 {
 290         account_system_vtime(current);
 291         trace_hardirq_exit();
 292         sub_preempt_count(IRQ_EXIT_OFFSET);
 293         if (!in_interrupt() && local_softirq_pending())
 294                 invoke_softirq();
 295
 296 #ifdef CONFIG_NO_HZ
 297         /* Make sure that timer wheel updates are propagated */
 298         if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
 299                 tick_nohz_stop_sched_tick(0);
 300         rcu_irq_exit();
 301 #endif
 302         preempt_enable_no_resched();
 303 }
 304
 305 /*
 306  * This function must run with irqs disabled!
 307  */
 308 inline void raise_softirq_irqoff(unsigned int nr)
 309 {
 310         __raise_softirq_irqoff(nr);
 311
 312         /*
 313          * If we're in an interrupt or softirq, we're done
 314          * (this also catches softirq-disabled code). We will
 315          * actually run the softirq once we return from
 316          * the irq or softirq.
 317          *
 318          * Otherwise we wake up ksoftirqd to make sure we
 319          * schedule the softirq soon.
 320          */
 321         if (!in_interrupt())
 322                 wakeup_softirqd();
 323 }
 324
 325 void raise_softirq(unsigned int nr)
 326 {
 327         unsigned long flags;
 328
 329         local_irq_save(flags);
 330         raise_softirq_irqoff(nr);
 331         local_irq_restore(flags);
 332 }
 333
 334 void open_softirq(int nr, void (*action)(struct softirq_action *))
 335 {
 336         softirq_vec[nr].action = action;
 337 }
 338
 339 /* Tasklets */
 340 struct tasklet_head
 341 {
 342         struct tasklet_struct *head;
 343         struct tasklet_struct **tail;
 344 };
 345
 346 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 347 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 348
 349 void __tasklet_schedule(struct tasklet_struct *t)
 350 {
 351         unsigned long flags;
 352
 353         local_irq_save(flags);
 354         t->next = NULL;
 355         *__get_cpu_var(tasklet_vec).tail = t;
 356         __get_cpu_var(tasklet_vec).tail = &(t->next);
 357         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 358         local_irq_restore(flags);
 359 }
 360
 361 EXPORT_SYMBOL(__tasklet_schedule);
 362
 363 void __tasklet_hi_schedule(struct tasklet_struct *t)
 364 {
 365         unsigned long flags;
 366
 367         local_irq_save(flags);
 368         t->next = NULL;
 369         *__get_cpu_var(tasklet_hi_vec).tail = t;
 370         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 371         raise_softirq_irqoff(HI_SOFTIRQ);
 372         local_irq_restore(flags);
 373 }
 374
 375 EXPORT_SYMBOL(__tasklet_hi_schedule);
 376
 377 static void tasklet_action(struct softirq_action *a)
 378 {
 379         struct tasklet_struct *list;
 380
 381         local_irq_disable();
 382         list = __get_cpu_var(tasklet_vec).head;
 383         __get_cpu_var(tasklet_vec).head = NULL;
 384         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
 385         local_irq_enable();
 386
 387         while (list) {
 388                 struct tasklet_struct *t = list;
 389
 390                 list = list->next;
 391
 392                 if (tasklet_trylock(t)) {
 393                         if (!atomic_read(&t->count)) {
 394                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 395                                         BUG();
 396                                 t->func(t->data);
 397                                 tasklet_unlock(t);
 398                                 continue;
 399                         }
 400                         tasklet_unlock(t);
 401                 }
 402
 403                 local_irq_disable();
 404                 t->next = NULL;
 405                 *__get_cpu_var(tasklet_vec).tail = t;
 406                 __get_cpu_var(tasklet_vec).tail = &(t->next);
 407                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 408                 local_irq_enable();
 409         }
 410 }
 411
 412 static void tasklet_hi_action(struct softirq_action *a)
 413 {
 414         struct tasklet_struct *list;
 415
 416         local_irq_disable();
 417         list = __get_cpu_var(tasklet_hi_vec).head;
 418         __get_cpu_var(tasklet_hi_vec).head = NULL;
 419         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
 420         local_irq_enable();
 421
 422         while (list) {
 423                 struct tasklet_struct *t = list;
 424
 425                 list = list->next;
 426
 427                 if (tasklet_trylock(t)) {
 428                         if (!atomic_read(&t->count)) {
 429                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 430                                         BUG();
 431                                 t->func(t->data);
 432                                 tasklet_unlock(t);
 433                                 continue;
 434                         }
 435                         tasklet_unlock(t);
 436                 }
 437
 438                 local_irq_disable();
 439                 t->next = NULL;
 440                 *__get_cpu_var(tasklet_hi_vec).tail = t;
 441                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 442                 __raise_softirq_irqoff(HI_SOFTIRQ);
 443                 local_irq_enable();
 444         }
 445 }
 446
 447
 448 void tasklet_init(struct tasklet_struct *t,
 449                   void (*func)(unsigned long), unsigned long data)
 450 {
 451         t->next = NULL;
 452         t->state = 0;
 453         atomic_set(&t->count, 0);
 454         t->func = func;
 455         t->data = data;
 456 }
 457
 458 EXPORT_SYMBOL(tasklet_init);
 459
 460 void tasklet_kill(struct tasklet_struct *t)
 461 {
 462         if (in_interrupt())
 463                 printk("Attempt to kill tasklet from interrupt\n");
 464
 465         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 466                 do
 467                         yield();
 468                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 469         }
 470         tasklet_unlock_wait(t);
 471         clear_bit(TASKLET_STATE_SCHED, &t->state);
 472 }
 473
 474 EXPORT_SYMBOL(tasklet_kill);
 475
 476 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 477 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 478
 479 static void __local_trigger(struct call_single_data *cp, int softirq)
 480 {
 481         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 482
 483         list_add_tail(&cp->list, head);
 484
 485         /* Trigger the softirq only if the list was previously empty.  */
 486         if (head->next == &cp->list)
 487                 raise_softirq_irqoff(softirq);
 488 }
 489
 490 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 491 static void remote_softirq_receive(void *data)
 492 {
 493         struct call_single_data *cp = data;
 494         unsigned long flags;
 495         int softirq;
 496
 497         softirq = cp->priv;
 498
 499         local_irq_save(flags);
 500         __local_trigger(cp, softirq);
 501         local_irq_restore(flags);
 502 }
 503
 504 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 505 {
 506         if (cpu_online(cpu)) {
 507                 cp->func = remote_softirq_receive;
 508                 cp->info = cp;
 509                 cp->flags = 0;
 510                 cp->priv = softirq;
 511
 512                 __smp_call_function_single(cpu, cp);
 513                 return 0;
 514         }
 515         return 1;
 516 }
 517 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 518 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 519 {
 520         return 1;
 521 }
 522 #endif
 523
 524 /**
 525  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 526  * @cp: private SMP call function data area
 527  * @cpu: the remote cpu
 528  * @this_cpu: the currently executing cpu
 529  * @softirq: the softirq for the work
 530  *
 531  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 532  * done, the work is instead queued up on the local cpu.
 533  *
 534  * Interrupts must be disabled.
 535  */
 536 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 537 {
 538         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 539                 __local_trigger(cp, softirq);
 540 }
 541 EXPORT_SYMBOL(__send_remote_softirq);
 542
 543 /**
 544  * send_remote_softirq - try to schedule softirq work on a remote cpu
 545  * @cp: private SMP call function data area
 546  * @cpu: the remote cpu
 547  * @softirq: the softirq for the work
 548  *
 549  * Like __send_remote_softirq except that disabling interrupts and
 550  * computing the current cpu is done for the caller.
 551  */
 552 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 553 {
 554         unsigned long flags;
 555         int this_cpu;
 556
 557         local_irq_save(flags);
 558         this_cpu = smp_processor_id();
 559         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 560         local_irq_restore(flags);
 561 }
 562 EXPORT_SYMBOL(send_remote_softirq);
 563
 564 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 565                                                unsigned long action, void *hcpu)
 566 {
 567         /*
 568          * If a CPU goes away, splice its entries to the current CPU
 569          * and trigger a run of the softirq
 570          */
 571         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 572                 int cpu = (unsigned long) hcpu;
 573                 int i;
 574
 575                 local_irq_disable();
 576                 for (i = 0; i < NR_SOFTIRQS; i++) {
 577                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 578                         struct list_head *local_head;
 579
 580                         if (list_empty(head))
 581                                 continue;
 582
 583                         local_head = &__get_cpu_var(softirq_work_list[i]);
 584                         list_splice_init(head, local_head);
 585                         raise_softirq_irqoff(i);
 586                 }
 587                 local_irq_enable();
 588         }
 589
 590         return NOTIFY_OK;
 591 }
 592
 593 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 594         .notifier_call  = remote_softirq_cpu_notify,
 595 };
 596
 597 void __init softirq_init(void)
 598 {
 599         int cpu;
 600
 601         for_each_possible_cpu(cpu) {
 602                 int i;
 603
 604                 per_cpu(tasklet_vec, cpu).tail =
 605                         &per_cpu(tasklet_vec, cpu).head;
 606                 per_cpu(tasklet_hi_vec, cpu).tail =
 607                         &per_cpu(tasklet_hi_vec, cpu).head;
 608                 for (i = 0; i < NR_SOFTIRQS; i++)
 609                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 610         }
 611
 612         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 613
 614         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 615         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 616 }
 617
 618 static int ksoftirqd(void * __bind_cpu)
 619 {
 620         set_current_state(TASK_INTERRUPTIBLE);
 621
 622         while (!kthread_should_stop()) {
 623                 preempt_disable();
 624                 if (!local_softirq_pending()) {
 625                         preempt_enable_no_resched();
 626                         schedule();
 627                         preempt_disable();
 628                 }
 629
 630                 __set_current_state(TASK_RUNNING);
 631
 632                 while (local_softirq_pending()) {
 633                         /* Preempt disable stops cpu going offline.
 634                            If already offline, we'll be on wrong CPU:
 635                            don't process */
 636                         if (cpu_is_offline((long)__bind_cpu))
 637                                 goto wait_to_die;
 638                         do_softirq();
 639                         preempt_enable_no_resched();
 640                         cond_resched();
 641                         preempt_disable();
 642                 }
 643                 preempt_enable();
 644                 set_current_state(TASK_INTERRUPTIBLE);
 645         }
 646         __set_current_state(TASK_RUNNING);
 647         return 0;
 648
 649 wait_to_die:
 650         preempt_enable();
 651         /* Wait for kthread_stop */
 652         set_current_state(TASK_INTERRUPTIBLE);
 653         while (!kthread_should_stop()) {
 654                 schedule();
 655                 set_current_state(TASK_INTERRUPTIBLE);
 656         }
 657         __set_current_state(TASK_RUNNING);
 658         return 0;
 659 }
 660
 661 #ifdef CONFIG_HOTPLUG_CPU
 662 /*
 663  * tasklet_kill_immediate is called to remove a tasklet which can already be
 664  * scheduled for execution on @cpu.
 665  *
 666  * Unlike tasklet_kill, this function removes the tasklet
 667  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 668  *
 669  * When this function is called, @cpu must be in the CPU_DEAD state.
 670  */
 671 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 672 {
 673         struct tasklet_struct **i;
 674
 675         BUG_ON(cpu_online(cpu));
 676         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 677
 678         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 679                 return;
 680
 681         /* CPU is dead, so no lock needed. */
 682         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 683                 if (*i == t) {
 684                         *i = t->next;
 685                         /* If this was the tail element, move the tail ptr */
 686                         if (*i == NULL)
 687                                 per_cpu(tasklet_vec, cpu).tail = i;
 688                         return;
 689                 }
 690         }
 691         BUG();
 692 }
 693
 694 static void takeover_tasklets(unsigned int cpu)
 695 {
 696         /* CPU is dead, so no lock needed. */
 697         local_irq_disable();
 698
 699         /* Find end, append list for that CPU. */
 700         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 701                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
 702                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
 703                 per_cpu(tasklet_vec, cpu).head = NULL;
 704                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 705         }
 706         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 707
 708         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 709                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
 710                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
 711                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 712                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 713         }
 714         raise_softirq_irqoff(HI_SOFTIRQ);
 715
 716         local_irq_enable();
 717 }
 718 #endif /* CONFIG_HOTPLUG_CPU */
 719
 720 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 721                                   unsigned long action,
 722                                   void *hcpu)
 723 {
 724         int hotcpu = (unsigned long)hcpu;
 725         struct task_struct *p;
 726
 727         switch (action) {
 728         case CPU_UP_PREPARE:
 729         case CPU_UP_PREPARE_FROZEN:
 730                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 731                 if (IS_ERR(p)) {
 732                         printk("ksoftirqd for %i failed\n", hotcpu);
 733                         return NOTIFY_BAD;
 734                 }
 735                 kthread_bind(p, hotcpu);
 736                 per_cpu(ksoftirqd, hotcpu) = p;
 737                 break;
 738         case CPU_ONLINE:
 739         case CPU_ONLINE_FROZEN:
 740                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 741                 break;
 742 #ifdef CONFIG_HOTPLUG_CPU
 743         case CPU_UP_CANCELED:
 744         case CPU_UP_CANCELED_FROZEN:
 745                 if (!per_cpu(ksoftirqd, hotcpu))
 746                         break;
 747                 /* Unbind so it can run.  Fall thru. */
 748                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 749                              any_online_cpu(cpu_online_map));
 750         case CPU_DEAD:
 751         case CPU_DEAD_FROZEN: {
 752                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 753
 754                 p = per_cpu(ksoftirqd, hotcpu);
 755                 per_cpu(ksoftirqd, hotcpu) = NULL;
 756                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
 757                 kthread_stop(p);
 758                 takeover_tasklets(hotcpu);
 759                 break;
 760         }
 761 #endif /* CONFIG_HOTPLUG_CPU */
 762         }
 763         return NOTIFY_OK;
 764 }
 765
 766 static struct notifier_block __cpuinitdata cpu_nfb = {
 767         .notifier_call = cpu_callback
 768 };
 769
 770 static __init int spawn_ksoftirqd(void)
 771 {
 772         void *cpu = (void *)(long)smp_processor_id();
 773         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 774
 775         BUG_ON(err == NOTIFY_BAD);
 776         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 777         register_cpu_notifier(&cpu_nfb);
 778         return 0;
 779 }
 780 early_initcall(spawn_ksoftirqd);
 781
 782 #ifdef CONFIG_SMP
 783 /*
 784  * Call a function on all processors
 785  */
 786 int on_each_cpu(void (*func) (void *info), void *info, int wait)
 787 {
 788         int ret = 0;
 789
 790         preempt_disable();
 791         ret = smp_call_function(func, info, wait);
 792         local_irq_disable();
 793         func(info);
 794         local_irq_enable();
 795         preempt_enable();
 796         return ret;
 797 }
 798 EXPORT_SYMBOL(on_each_cpu);
 799 #endif