X-Git-Url: http://pilppa.org/gitweb/gitweb.cgi?a=blobdiff_plain;f=kernel%2Firq%2Fhandle.c;h=c815b42d0f5bf12baed5e25eb92c7f2d5e4b4472;hb=a56cdcb662032a732f7c4f35cc5a9acf37759d8c;hp=6d174390f3a039eed8a95bcd9e47dd32fbe0890c;hpb=67fb283e148e9bd761f73691d3173b6eab9ba8db;p=linux-2.6-omap-h63xx.git diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index 6d174390f3a..f51eaee921b 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c @@ -15,16 +15,16 @@ #include #include #include +#include +#include +#include #include "internals.h" -#ifdef CONFIG_TRACE_IRQFLAGS - /* * lockdep: we want to handle all irq_desc locks as a single lock-class: */ -static struct lock_class_key irq_desc_lock_class; -#endif +struct lock_class_key irq_desc_lock_class; /** * handle_bad_irq - handle spurious and unhandled irqs @@ -33,14 +33,25 @@ static struct lock_class_key irq_desc_lock_class; * * Handles spurious and unhandled IRQ's. It also prints a debugmessage. */ -void -handle_bad_irq(unsigned int irq, struct irq_desc *desc) +void handle_bad_irq(unsigned int irq, struct irq_desc *desc) { print_irq_desc(irq, desc); - kstat_irqs_this_cpu(desc)++; + kstat_incr_irqs_this_cpu(irq, desc); ack_bad_irq(irq); } +#if defined(CONFIG_SMP) && defined(CONFIG_GENERIC_HARDIRQS) +static void __init init_irq_default_affinity(void) +{ + alloc_bootmem_cpumask_var(&irq_default_affinity); + cpumask_setall(irq_default_affinity); +} +#else +static void __init init_irq_default_affinity(void) +{ +} +#endif + /* * Linux has a controller-independent interrupt architecture. * Every controller has a 'controller-template', that is used @@ -58,203 +69,163 @@ handle_bad_irq(unsigned int irq, struct irq_desc *desc) int nr_irqs = NR_IRQS; EXPORT_SYMBOL_GPL(nr_irqs); -#ifdef CONFIG_HAVE_DYN_ARRAY +#ifdef CONFIG_SPARSE_IRQ + static struct irq_desc irq_desc_init = { - .irq = -1U, - .status = IRQ_DISABLED, - .chip = &no_irq_chip, + .irq = -1, + .status = IRQ_DISABLED, + .chip = &no_irq_chip, .handle_irq = handle_bad_irq, - .depth = 1, - .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), -#ifdef CONFIG_SMP - .affinity = CPU_MASK_ALL -#endif + .depth = 1, + .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), }; - -static void init_one_irq_desc(struct irq_desc *desc) +void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr) { - memcpy(desc, &irq_desc_init, sizeof(struct irq_desc)); -#ifdef CONFIG_TRACE_IRQFLAGS - lockdep_set_class(&desc->lock, &irq_desc_lock_class); -#endif -} - -extern int after_bootmem; -extern void *__alloc_bootmem_nopanic(unsigned long size, - unsigned long align, - unsigned long goal); - -static void init_kstat_irqs(struct irq_desc *desc, int nr_desc, int nr) -{ - unsigned long bytes, total_bytes; + unsigned long bytes; char *ptr; - int i; - unsigned long phys; + int node; /* Compute how many bytes we need per irq and allocate them */ bytes = nr * sizeof(unsigned int); - total_bytes = bytes * nr_desc; - if (after_bootmem) - ptr = kzalloc(total_bytes, GFP_ATOMIC); - else - ptr = __alloc_bootmem_nopanic(total_bytes, PAGE_SIZE, 0); - if (!ptr) - panic(" can not allocate kstat_irqs\n"); + node = cpu_to_node(cpu); + ptr = kzalloc_node(bytes, GFP_ATOMIC, node); + printk(KERN_DEBUG " alloc kstat_irqs on cpu %d node %d\n", cpu, node); + + if (ptr) + desc->kstat_irqs = (unsigned int *)ptr; +} - phys = __pa(ptr); - printk(KERN_DEBUG "kstat_irqs ==> [%#lx - %#lx]\n", phys, phys + total_bytes); +static void init_one_irq_desc(int irq, struct irq_desc *desc, int cpu) +{ + memcpy(desc, &irq_desc_init, sizeof(struct irq_desc)); - for (i = 0; i < nr_desc; i++) { - desc[i].kstat_irqs = (unsigned int *)ptr; - ptr += bytes; + spin_lock_init(&desc->lock); + desc->irq = irq; +#ifdef CONFIG_SMP + desc->cpu = cpu; +#endif + lockdep_set_class(&desc->lock, &irq_desc_lock_class); + init_kstat_irqs(desc, cpu, nr_cpu_ids); + if (!desc->kstat_irqs) { + printk(KERN_ERR "can not alloc kstat_irqs\n"); + BUG_ON(1); + } + if (!init_alloc_desc_masks(desc, cpu, false)) { + printk(KERN_ERR "can not alloc irq_desc cpumasks\n"); + BUG_ON(1); } + arch_init_chip_data(desc, cpu); } -#ifdef CONFIG_HAVE_SPARSE_IRQ -static struct irq_desc *sparse_irqs_free; -struct irq_desc *sparse_irqs; -#endif +/* + * Protect the sparse_irqs: + */ +DEFINE_SPINLOCK(sparse_irq_lock); + +struct irq_desc **irq_desc_ptrs __read_mostly; + +static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_smp = { + [0 ... NR_IRQS_LEGACY-1] = { + .irq = -1, + .status = IRQ_DISABLED, + .chip = &no_irq_chip, + .handle_irq = handle_bad_irq, + .depth = 1, + .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), + } +}; + +static unsigned int *kstat_irqs_legacy; -static void __init init_work(void *data) +int __init early_irq_init(void) { - struct dyn_array *da = data; + struct irq_desc *desc; + int legacy_count; int i; - struct irq_desc *desc; - - desc = *da->name; - for (i = 0; i < *da->nr; i++) { - init_one_irq_desc(&desc[i]); -#ifndef CONFIG_HAVE_SPARSE_IRQ - desc[i].irq = i; -#endif - } + init_irq_default_affinity(); - /* init kstat_irqs, nr_cpu_ids is ready already */ - init_kstat_irqs(desc, *da->nr, nr_cpu_ids); + /* initialize nr_irqs based on nr_cpu_ids */ + arch_probe_nr_irqs(); + printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d\n", NR_IRQS, nr_irqs); -#ifdef CONFIG_HAVE_SPARSE_IRQ - for (i = 1; i < *da->nr; i++) - desc[i-1].next = &desc[i]; + desc = irq_desc_legacy; + legacy_count = ARRAY_SIZE(irq_desc_legacy); - sparse_irqs_free = sparse_irqs; - sparse_irqs = NULL; -#endif -} + /* allocate irq_desc_ptrs array based on nr_irqs */ + irq_desc_ptrs = alloc_bootmem(nr_irqs * sizeof(void *)); -#ifdef CONFIG_HAVE_SPARSE_IRQ -static int nr_irq_desc = 32; + /* allocate based on nr_cpu_ids */ + /* FIXME: invert kstat_irgs, and it'd be a per_cpu_alloc'd thing */ + kstat_irqs_legacy = alloc_bootmem(NR_IRQS_LEGACY * nr_cpu_ids * + sizeof(int)); -static int __init parse_nr_irq_desc(char *arg) -{ - if (arg) - nr_irq_desc = simple_strtoul(arg, NULL, 0); - return 0; -} + for (i = 0; i < legacy_count; i++) { + desc[i].irq = i; + desc[i].kstat_irqs = kstat_irqs_legacy + i * nr_cpu_ids; + lockdep_set_class(&desc[i].lock, &irq_desc_lock_class); + init_alloc_desc_masks(&desc[i], 0, true); + irq_desc_ptrs[i] = desc + i; + } -early_param("nr_irq_desc", parse_nr_irq_desc); + for (i = legacy_count; i < nr_irqs; i++) + irq_desc_ptrs[i] = NULL; -DEFINE_DYN_ARRAY(sparse_irqs, sizeof(struct irq_desc), nr_irq_desc, PAGE_SIZE, init_work); + return arch_early_irq_init(); +} struct irq_desc *irq_to_desc(unsigned int irq) { - struct irq_desc *desc; + if (irq_desc_ptrs && irq < nr_irqs) + return irq_desc_ptrs[irq]; - desc = sparse_irqs; - while (desc) { - if (desc->irq == irq) - return desc; - - desc = desc->next; - } return NULL; } -struct irq_desc *irq_to_desc_alloc(unsigned int irq) -{ - struct irq_desc *desc, *desc_pri; - int i; - int count = 0; - desc_pri = desc = sparse_irqs; - while (desc) { - if (desc->irq == irq) - return desc; +struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu) +{ + struct irq_desc *desc; + unsigned long flags; + int node; - desc_pri = desc; - desc = desc->next; - count++; + if (irq >= nr_irqs) { + WARN(1, "irq (%d) >= nr_irqs (%d) in irq_to_desc_alloc\n", + irq, nr_irqs); + return NULL; } - /* - * we run out of pre-allocate ones, allocate more - */ - if (!sparse_irqs_free) { - unsigned long phys; - unsigned long total_bytes; - - printk(KERN_DEBUG "try to get more irq_desc %d\n", nr_irq_desc); - - total_bytes = sizeof(struct irq_desc) * nr_irq_desc; - if (after_bootmem) - desc = kzalloc(total_bytes, GFP_ATOMIC); - else - desc = __alloc_bootmem_nopanic(total_bytes, PAGE_SIZE, 0); - - if (!desc) - panic("please boot with nr_irq_desc= %d\n", count * 2); + desc = irq_desc_ptrs[irq]; + if (desc) + return desc; - phys = __pa(desc); - printk(KERN_DEBUG "irq_desc ==> [%#lx - %#lx]\n", phys, phys + total_bytes); + spin_lock_irqsave(&sparse_irq_lock, flags); - for (i = 0; i < nr_irq_desc; i++) - init_one_irq_desc(&desc[i]); + /* We have to check it to avoid races with another CPU */ + desc = irq_desc_ptrs[irq]; + if (desc) + goto out_unlock; - for (i = 1; i < nr_irq_desc; i++) - desc[i-1].next = &desc[i]; + node = cpu_to_node(cpu); + desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node); + printk(KERN_DEBUG " alloc irq_desc for %d on cpu %d node %d\n", + irq, cpu, node); + if (!desc) { + printk(KERN_ERR "can not alloc irq_desc\n"); + BUG_ON(1); + } + init_one_irq_desc(irq, desc, cpu); - /* init kstat_irqs, nr_cpu_ids is ready already */ - init_kstat_irqs(desc, nr_irq_desc, nr_cpu_ids); + irq_desc_ptrs[irq] = desc; - sparse_irqs_free = desc; - } +out_unlock: + spin_unlock_irqrestore(&sparse_irq_lock, flags); - desc = sparse_irqs_free; - sparse_irqs_free = sparse_irqs_free->next; - desc->next = NULL; - if (desc_pri) - desc_pri->next = desc; - else - sparse_irqs = desc; - desc->irq = irq; - printk(KERN_DEBUG "found new irq_desc for irq %d\n", desc->irq); -#ifdef CONFIG_HAVE_SPARSE_IRQ_DEBUG - { - /* dump the results */ - struct irq_desc *desc; - unsigned long phys; - unsigned long bytes = sizeof(struct irq_desc); - unsigned int irqx; - - printk(KERN_DEBUG "=========================== %d\n", irq); - printk(KERN_DEBUG "irq_desc dump after get that for %d\n", irq); - for_each_irq_desc(irqx, desc) { - phys = __pa(desc); - printk(KERN_DEBUG "irq_desc %d ==> [%#lx - %#lx]\n", irqx, phys, phys + bytes); - } - printk(KERN_DEBUG "===========================\n"); - } -#endif return desc; } -#else -struct irq_desc *irq_desc; -DEFINE_DYN_ARRAY(irq_desc, sizeof(struct irq_desc), nr_irqs, PAGE_SIZE, init_work); -#endif - -#else +#else /* !CONFIG_SPARSE_IRQ */ struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { [0 ... NR_IRQS-1] = { @@ -262,28 +233,40 @@ struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { .chip = &no_irq_chip, .handle_irq = handle_bad_irq, .depth = 1, - .lock = __SPIN_LOCK_UNLOCKED(sparse_irqs->lock), -#ifdef CONFIG_SMP - .affinity = CPU_MASK_ALL -#endif + .lock = __SPIN_LOCK_UNLOCKED(irq_desc->lock), } }; -#endif +int __init early_irq_init(void) +{ + struct irq_desc *desc; + int count; + int i; + + init_irq_default_affinity(); + + printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS); + + desc = irq_desc; + count = ARRAY_SIZE(irq_desc); + + for (i = 0; i < count; i++) { + desc[i].irq = i; + init_alloc_desc_masks(&desc[i], 0, true); + } + return arch_early_irq_init(); +} -#ifndef CONFIG_HAVE_SPARSE_IRQ struct irq_desc *irq_to_desc(unsigned int irq) { - if (irq < nr_irqs) - return &irq_desc[irq]; - - return NULL; + return (irq < NR_IRQS) ? irq_desc + irq : NULL; } -struct irq_desc *irq_to_desc_alloc(unsigned int irq) + +struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu) { return irq_to_desc(irq); } -#endif +#endif /* !CONFIG_SPARSE_IRQ */ /* * What should we do if we get a hw irq event on an illegal vector? @@ -291,9 +274,8 @@ struct irq_desc *irq_to_desc_alloc(unsigned int irq) */ static void ack_bad(unsigned int irq) { - struct irq_desc *desc; + struct irq_desc *desc = irq_to_desc(irq); - desc = irq_to_desc(irq); print_irq_desc(irq, desc); ack_bad_irq(irq); } @@ -395,15 +377,19 @@ unsigned int __do_IRQ(unsigned int irq) struct irqaction *action; unsigned int status; - kstat_irqs_this_cpu(desc)++; + kstat_incr_irqs_this_cpu(irq, desc); + if (CHECK_IRQ_PER_CPU(desc->status)) { irqreturn_t action_ret; /* * No locking required for CPU-local interrupts: */ - if (desc->chip->ack) + if (desc->chip->ack) { desc->chip->ack(irq); + /* get new one */ + desc = irq_remap_to_desc(irq, desc); + } if (likely(!(desc->status & IRQ_DISABLED))) { action_ret = handle_IRQ_event(irq, desc->action); if (!noirqdebug) @@ -414,8 +400,10 @@ unsigned int __do_IRQ(unsigned int irq) } spin_lock(&desc->lock); - if (desc->chip->ack) + if (desc->chip->ack) { desc->chip->ack(irq); + desc = irq_remap_to_desc(irq, desc); + } /* * REPLAY is when Linux resends an IRQ that was dropped earlier * WAITING is used by probe to mark irqs that are being tested @@ -482,23 +470,22 @@ out: } #endif - -#ifdef CONFIG_TRACE_IRQFLAGS void early_init_irq_lock_class(void) { -#ifndef CONFIG_HAVE_DYN_ARRAY + struct irq_desc *desc; int i; - for (i = 0; i < nr_irqs; i++) - lockdep_set_class(&irq_desc[i].lock, &irq_desc_lock_class); -#endif + for_each_irq_desc(i, desc) { + lockdep_set_class(&desc->lock, &irq_desc_lock_class); + } } -#endif +#ifdef CONFIG_SPARSE_IRQ unsigned int kstat_irqs_cpu(unsigned int irq, int cpu) { struct irq_desc *desc = irq_to_desc(irq); - return desc->kstat_irqs[cpu]; + return desc ? desc->kstat_irqs[cpu] : 0; } +#endif EXPORT_SYMBOL(kstat_irqs_cpu);