static int wakeup_cpu;
static unsigned wakeup_prio = -1;
-static DEFINE_SPINLOCK(wakeup_lock);
+static raw_spinlock_t wakeup_lock =
+ (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
static void __wakeup_reset(struct trace_array *tr);
+#ifdef CONFIG_FTRACE
+/*
+ * irqsoff uses its own tracer function to keep the overhead down:
+ */
+static void
+wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
+{
+ struct trace_array *tr = wakeup_trace;
+ struct trace_array_cpu *data;
+ unsigned long flags;
+ long disabled;
+ int resched;
+ int cpu;
+
+ if (likely(!wakeup_task))
+ return;
+
+ resched = need_resched();
+ preempt_disable_notrace();
+
+ cpu = raw_smp_processor_id();
+ data = tr->data[cpu];
+ disabled = atomic_inc_return(&data->disabled);
+ if (unlikely(disabled != 1))
+ goto out;
+
+ local_irq_save(flags);
+ __raw_spin_lock(&wakeup_lock);
+
+ if (unlikely(!wakeup_task))
+ goto unlock;
+
+ /*
+ * The task can't disappear because it needs to
+ * wake up first, and we have the wakeup_lock.
+ */
+ if (task_cpu(wakeup_task) != cpu)
+ goto unlock;
+
+ trace_function(tr, data, ip, parent_ip, flags);
+
+ unlock:
+ __raw_spin_unlock(&wakeup_lock);
+ local_irq_restore(flags);
+
+ out:
+ atomic_dec(&data->disabled);
+
+ /*
+ * To prevent recursion from the scheduler, if the
+ * resched flag was set before we entered, then
+ * don't reschedule.
+ */
+ if (resched)
+ preempt_enable_no_resched_notrace();
+ else
+ preempt_enable_notrace();
+}
+
+static struct ftrace_ops trace_ops __read_mostly =
+{
+ .func = wakeup_tracer_call,
+};
+#endif /* CONFIG_FTRACE */
+
/*
* Should this new latency be reported/recorded?
*/
if (next != wakeup_task)
return;
- /* The task we are waitng for is waking up */
+ /* The task we are waiting for is waking up */
data = tr->data[wakeup_cpu];
/* disable local data, not wakeup_cpu data */
if (likely(disabled != 1))
goto out;
- spin_lock_irqsave(&wakeup_lock, flags);
+ local_irq_save(flags);
+ __raw_spin_lock(&wakeup_lock);
/* We could race with grabbing wakeup_lock */
if (unlikely(!tracer_enabled || next != wakeup_task))
out_unlock:
__wakeup_reset(tr);
- spin_unlock_irqrestore(&wakeup_lock, flags);
+ __raw_spin_unlock(&wakeup_lock);
+ local_irq_restore(flags);
out:
atomic_dec(&tr->data[cpu]->disabled);
}
struct trace_array_cpu *data;
int cpu;
- assert_spin_locked(&wakeup_lock);
-
for_each_possible_cpu(cpu) {
data = tr->data[cpu];
tracing_reset(data);
{
unsigned long flags;
- spin_lock_irqsave(&wakeup_lock, flags);
+ local_irq_save(flags);
+ __raw_spin_lock(&wakeup_lock);
__wakeup_reset(tr);
- spin_unlock_irqrestore(&wakeup_lock, flags);
+ __raw_spin_unlock(&wakeup_lock);
+ local_irq_restore(flags);
}
static void
goto out;
/* interrupts should be off from try_to_wake_up */
- spin_lock(&wakeup_lock);
+ __raw_spin_lock(&wakeup_lock);
/* check for races. */
if (!tracer_enabled || p->prio >= wakeup_prio)
CALLER_ADDR1, CALLER_ADDR2, flags);
out_locked:
- spin_unlock(&wakeup_lock);
+ __raw_spin_unlock(&wakeup_lock);
out:
atomic_dec(&tr->data[cpu]->disabled);
}
*/
smp_wmb();
+ register_ftrace_function(&trace_ops);
+
tracer_enabled = 1;
return;
static void stop_wakeup_tracer(struct trace_array *tr)
{
tracer_enabled = 0;
+ unregister_ftrace_function(&trace_ops);
marker_probe_unregister("kernel_sched_schedule",
sched_switch_callback,
&wakeup_trace);