kernel/trace/trace.c

   1 /*
   2  * ring buffer based function tracer
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally taken from the RT patch by:
   8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code from the latency_tracer, that is:
  11  *  Copyright (C) 2004-2006 Ingo Molnar
  12  *  Copyright (C) 2004 William Lee Irwin III
  13  */
  14 #include <linux/utsrelease.h>
  15 #include <linux/kallsyms.h>
  16 #include <linux/seq_file.h>
  17 #include <linux/notifier.h>
  18 #include <linux/debugfs.h>
  19 #include <linux/pagemap.h>
  20 #include <linux/hardirq.h>
  21 #include <linux/linkage.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/ftrace.h>
  24 #include <linux/module.h>
  25 #include <linux/percpu.h>
  26 #include <linux/kdebug.h>
  27 #include <linux/ctype.h>
  28 #include <linux/init.h>
  29 #include <linux/poll.h>
  30 #include <linux/gfp.h>
  31 #include <linux/fs.h>
  32 #include <linux/kprobes.h>
  33 #include <linux/writeback.h>
  34
  35 #include <linux/stacktrace.h>
  36 #include <linux/ring_buffer.h>
  37
  38 #include "trace.h"
  39
  40 #define TRACE_BUFFER_FLAGS      (RB_FL_OVERWRITE)
  41
  42 unsigned long __read_mostly     tracing_max_latency = (cycle_t)ULONG_MAX;
  43 unsigned long __read_mostly     tracing_thresh;
  44
  45 static cpumask_t __read_mostly          tracing_buffer_mask;
  46
  47 #define for_each_tracing_cpu(cpu)       \
  48         for_each_cpu_mask(cpu, tracing_buffer_mask)
  49
  50 static int tracing_disabled = 1;
  51
  52 long
  53 ns2usecs(cycle_t nsec)
  54 {
  55         nsec += 500;
  56         do_div(nsec, 1000);
  57         return nsec;
  58 }
  59
  60 cycle_t ftrace_now(int cpu)
  61 {
  62         u64 ts = ring_buffer_time_stamp(cpu);
  63         ring_buffer_normalize_time_stamp(cpu, &ts);
  64         return ts;
  65 }
  66
  67 /*
  68  * The global_trace is the descriptor that holds the tracing
  69  * buffers for the live tracing. For each CPU, it contains
  70  * a link list of pages that will store trace entries. The
  71  * page descriptor of the pages in the memory is used to hold
  72  * the link list by linking the lru item in the page descriptor
  73  * to each of the pages in the buffer per CPU.
  74  *
  75  * For each active CPU there is a data field that holds the
  76  * pages for the buffer for that CPU. Each CPU has the same number
  77  * of pages allocated for its buffer.
  78  */
  79 static struct trace_array       global_trace;
  80
  81 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
  82
  83 /*
  84  * The max_tr is used to snapshot the global_trace when a maximum
  85  * latency is reached. Some tracers will use this to store a maximum
  86  * trace while it continues examining live traces.
  87  *
  88  * The buffers for the max_tr are set up the same as the global_trace.
  89  * When a snapshot is taken, the link list of the max_tr is swapped
  90  * with the link list of the global_trace and the buffers are reset for
  91  * the global_trace so the tracing can continue.
  92  */
  93 static struct trace_array       max_tr;
  94
  95 static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
  96
  97 /* tracer_enabled is used to toggle activation of a tracer */
  98 static int                      tracer_enabled = 1;
  99
 100 /* function tracing enabled */
 101 int                             ftrace_function_enabled;
 102
 103 /*
 104  * trace_buf_size is the size in bytes that is allocated
 105  * for a buffer. Note, the number of bytes is always rounded
 106  * to page size.
 107  *
 108  * This number is purposely set to a low number of 16384.
 109  * If the dump on oops happens, it will be much appreciated
 110  * to not have to wait for all that output. Anyway this can be
 111  * boot time and run time configurable.
 112  */
 113 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
 114
 115 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
 116
 117 /* trace_types holds a link list of available tracers. */
 118 static struct tracer            *trace_types __read_mostly;
 119
 120 /* current_trace points to the tracer that is currently active */
 121 static struct tracer            *current_trace __read_mostly;
 122
 123 /*
 124  * max_tracer_type_len is used to simplify the allocating of
 125  * buffers to read userspace tracer names. We keep track of
 126  * the longest tracer name registered.
 127  */
 128 static int                      max_tracer_type_len;
 129
 130 /*
 131  * trace_types_lock is used to protect the trace_types list.
 132  * This lock is also used to keep user access serialized.
 133  * Accesses from userspace will grab this lock while userspace
 134  * activities happen inside the kernel.
 135  */
 136 static DEFINE_MUTEX(trace_types_lock);
 137
 138 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
 139 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
 140
 141 /* trace_flags holds iter_ctrl options */
 142 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT;
 143
 144 /**
 145  * trace_wake_up - wake up tasks waiting for trace input
 146  *
 147  * Simply wakes up any task that is blocked on the trace_wait
 148  * queue. These is used with trace_poll for tasks polling the trace.
 149  */
 150 void trace_wake_up(void)
 151 {
 152         /*
 153          * The runqueue_is_locked() can fail, but this is the best we
 154          * have for now:
 155          */
 156         if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
 157                 wake_up(&trace_wait);
 158 }
 159
 160 static int __init set_buf_size(char *str)
 161 {
 162         unsigned long buf_size;
 163         int ret;
 164
 165         if (!str)
 166                 return 0;
 167         ret = strict_strtoul(str, 0, &buf_size);
 168         /* nr_entries can not be zero */
 169         if (ret < 0 || buf_size == 0)
 170                 return 0;
 171         trace_buf_size = buf_size;
 172         return 1;
 173 }
 174 __setup("trace_buf_size=", set_buf_size);
 175
 176 unsigned long nsecs_to_usecs(unsigned long nsecs)
 177 {
 178         return nsecs / 1000;
 179 }
 180
 181 /*
 182  * TRACE_ITER_SYM_MASK masks the options in trace_flags that
 183  * control the output of kernel symbols.
 184  */
 185 #define TRACE_ITER_SYM_MASK \
 186         (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
 187
 188 /* These must match the bit postions in trace_iterator_flags */
 189 static const char *trace_options[] = {
 190         "print-parent",
 191         "sym-offset",
 192         "sym-addr",
 193         "verbose",
 194         "raw",
 195         "hex",
 196         "bin",
 197         "block",
 198         "stacktrace",
 199         "sched-tree",
 200         "ftrace_printk",
 201         NULL
 202 };
 203
 204 /*
 205  * ftrace_max_lock is used to protect the swapping of buffers
 206  * when taking a max snapshot. The buffers themselves are
 207  * protected by per_cpu spinlocks. But the action of the swap
 208  * needs its own lock.
 209  *
 210  * This is defined as a raw_spinlock_t in order to help
 211  * with performance when lockdep debugging is enabled.
 212  */
 213 static raw_spinlock_t ftrace_max_lock =
 214         (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
 215
 216 /*
 217  * Copy the new maximum trace into the separate maximum-trace
 218  * structure. (this way the maximum trace is permanently saved,
 219  * for later retrieval via /debugfs/tracing/latency_trace)
 220  */
 221 static void
 222 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 223 {
 224         struct trace_array_cpu *data = tr->data[cpu];
 225
 226         max_tr.cpu = cpu;
 227         max_tr.time_start = data->preempt_timestamp;
 228
 229         data = max_tr.data[cpu];
 230         data->saved_latency = tracing_max_latency;
 231
 232         memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
 233         data->pid = tsk->pid;
 234         data->uid = tsk->uid;
 235         data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
 236         data->policy = tsk->policy;
 237         data->rt_priority = tsk->rt_priority;
 238
 239         /* record this tasks comm */
 240         tracing_record_cmdline(current);
 241 }
 242
 243 /**
 244  * trace_seq_printf - sequence printing of trace information
 245  * @s: trace sequence descriptor
 246  * @fmt: printf format string
 247  *
 248  * The tracer may use either sequence operations or its own
 249  * copy to user routines. To simplify formating of a trace
 250  * trace_seq_printf is used to store strings into a special
 251  * buffer (@s). Then the output may be either used by
 252  * the sequencer or pulled into another buffer.
 253  */
 254 int
 255 trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
 256 {
 257         int len = (PAGE_SIZE - 1) - s->len;
 258         va_list ap;
 259         int ret;
 260
 261         if (!len)
 262                 return 0;
 263
 264         va_start(ap, fmt);
 265         ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
 266         va_end(ap);
 267
 268         /* If we can't write it all, don't bother writing anything */
 269         if (ret >= len)
 270                 return 0;
 271
 272         s->len += ret;
 273
 274         return len;
 275 }
 276
 277 /**
 278  * trace_seq_puts - trace sequence printing of simple string
 279  * @s: trace sequence descriptor
 280  * @str: simple string to record
 281  *
 282  * The tracer may use either the sequence operations or its own
 283  * copy to user routines. This function records a simple string
 284  * into a special buffer (@s) for later retrieval by a sequencer
 285  * or other mechanism.
 286  */
 287 static int
 288 trace_seq_puts(struct trace_seq *s, const char *str)
 289 {
 290         int len = strlen(str);
 291
 292         if (len > ((PAGE_SIZE - 1) - s->len))
 293                 return 0;
 294
 295         memcpy(s->buffer + s->len, str, len);
 296         s->len += len;
 297
 298         return len;
 299 }
 300
 301 static int
 302 trace_seq_putc(struct trace_seq *s, unsigned char c)
 303 {
 304         if (s->len >= (PAGE_SIZE - 1))
 305                 return 0;
 306
 307         s->buffer[s->len++] = c;
 308
 309         return 1;
 310 }
 311
 312 static int
 313 trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
 314 {
 315         if (len > ((PAGE_SIZE - 1) - s->len))
 316                 return 0;
 317
 318         memcpy(s->buffer + s->len, mem, len);
 319         s->len += len;
 320
 321         return len;
 322 }
 323
 324 #define HEX_CHARS 17
 325 static const char hex2asc[] = "0123456789abcdef";
 326
 327 static int
 328 trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
 329 {
 330         unsigned char hex[HEX_CHARS];
 331         unsigned char *data = mem;
 332         unsigned char byte;
 333         int i, j;
 334
 335         BUG_ON(len >= HEX_CHARS);
 336
 337 #ifdef __BIG_ENDIAN
 338         for (i = 0, j = 0; i < len; i++) {
 339 #else
 340         for (i = len-1, j = 0; i >= 0; i--) {
 341 #endif
 342                 byte = data[i];
 343
 344                 hex[j++] = hex2asc[byte & 0x0f];
 345                 hex[j++] = hex2asc[byte >> 4];
 346         }
 347         hex[j++] = ' ';
 348
 349         return trace_seq_putmem(s, hex, j);
 350 }
 351
 352 static void
 353 trace_seq_reset(struct trace_seq *s)
 354 {
 355         s->len = 0;
 356         s->readpos = 0;
 357 }
 358
 359 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
 360 {
 361         int len;
 362         int ret;
 363
 364         if (s->len <= s->readpos)
 365                 return -EBUSY;
 366
 367         len = s->len - s->readpos;
 368         if (cnt > len)
 369                 cnt = len;
 370         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
 371         if (ret)
 372                 return -EFAULT;
 373
 374         s->readpos += len;
 375         return cnt;
 376 }
 377
 378 static void
 379 trace_print_seq(struct seq_file *m, struct trace_seq *s)
 380 {
 381         int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
 382
 383         s->buffer[len] = 0;
 384         seq_puts(m, s->buffer);
 385
 386         trace_seq_reset(s);
 387 }
 388
 389 /**
 390  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
 391  * @tr: tracer
 392  * @tsk: the task with the latency
 393  * @cpu: The cpu that initiated the trace.
 394  *
 395  * Flip the buffers between the @tr and the max_tr and record information
 396  * about which task was the cause of this latency.
 397  */
 398 void
 399 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 400 {
 401         struct ring_buffer *buf = tr->buffer;
 402
 403         WARN_ON_ONCE(!irqs_disabled());
 404         __raw_spin_lock(&ftrace_max_lock);
 405
 406         tr->buffer = max_tr.buffer;
 407         max_tr.buffer = buf;
 408
 409         ring_buffer_reset(tr->buffer);
 410
 411         __update_max_tr(tr, tsk, cpu);
 412         __raw_spin_unlock(&ftrace_max_lock);
 413 }
 414
 415 /**
 416  * update_max_tr_single - only copy one trace over, and reset the rest
 417  * @tr - tracer
 418  * @tsk - task with the latency
 419  * @cpu - the cpu of the buffer to copy.
 420  *
 421  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
 422  */
 423 void
 424 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 425 {
 426         int ret;
 427
 428         WARN_ON_ONCE(!irqs_disabled());
 429         __raw_spin_lock(&ftrace_max_lock);
 430
 431         ring_buffer_reset(max_tr.buffer);
 432         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
 433
 434         WARN_ON_ONCE(ret);
 435
 436         __update_max_tr(tr, tsk, cpu);
 437         __raw_spin_unlock(&ftrace_max_lock);
 438 }
 439
 440 /**
 441  * register_tracer - register a tracer with the ftrace system.
 442  * @type - the plugin for the tracer
 443  *
 444  * Register a new plugin tracer.
 445  */
 446 int register_tracer(struct tracer *type)
 447 {
 448         struct tracer *t;
 449         int len;
 450         int ret = 0;
 451
 452         if (!type->name) {
 453                 pr_info("Tracer must have a name\n");
 454                 return -1;
 455         }
 456
 457         mutex_lock(&trace_types_lock);
 458         for (t = trace_types; t; t = t->next) {
 459                 if (strcmp(type->name, t->name) == 0) {
 460                         /* already found */
 461                         pr_info("Trace %s already registered\n",
 462                                 type->name);
 463                         ret = -1;
 464                         goto out;
 465                 }
 466         }
 467
 468 #ifdef CONFIG_FTRACE_STARTUP_TEST
 469         if (type->selftest) {
 470                 struct tracer *saved_tracer = current_trace;
 471                 struct trace_array *tr = &global_trace;
 472                 int saved_ctrl = tr->ctrl;
 473                 int i;
 474                 /*
 475                  * Run a selftest on this tracer.
 476                  * Here we reset the trace buffer, and set the current
 477                  * tracer to be this tracer. The tracer can then run some
 478                  * internal tracing to verify that everything is in order.
 479                  * If we fail, we do not register this tracer.
 480                  */
 481                 for_each_tracing_cpu(i) {
 482                         tracing_reset(tr, i);
 483                 }
 484                 current_trace = type;
 485                 tr->ctrl = 0;
 486                 /* the test is responsible for initializing and enabling */
 487                 pr_info("Testing tracer %s: ", type->name);
 488                 ret = type->selftest(type, tr);
 489                 /* the test is responsible for resetting too */
 490                 current_trace = saved_tracer;
 491                 tr->ctrl = saved_ctrl;
 492                 if (ret) {
 493                         printk(KERN_CONT "FAILED!\n");
 494                         goto out;
 495                 }
 496                 /* Only reset on passing, to avoid touching corrupted buffers */
 497                 for_each_tracing_cpu(i) {
 498                         tracing_reset(tr, i);
 499                 }
 500                 printk(KERN_CONT "PASSED\n");
 501         }
 502 #endif
 503
 504         type->next = trace_types;
 505         trace_types = type;
 506         len = strlen(type->name);
 507         if (len > max_tracer_type_len)
 508                 max_tracer_type_len = len;
 509
 510  out:
 511         mutex_unlock(&trace_types_lock);
 512
 513         return ret;
 514 }
 515
 516 void unregister_tracer(struct tracer *type)
 517 {
 518         struct tracer **t;
 519         int len;
 520
 521         mutex_lock(&trace_types_lock);
 522         for (t = &trace_types; *t; t = &(*t)->next) {
 523                 if (*t == type)
 524                         goto found;
 525         }
 526         pr_info("Trace %s not registered\n", type->name);
 527         goto out;
 528
 529  found:
 530         *t = (*t)->next;
 531         if (strlen(type->name) != max_tracer_type_len)
 532                 goto out;
 533
 534         max_tracer_type_len = 0;
 535         for (t = &trace_types; *t; t = &(*t)->next) {
 536                 len = strlen((*t)->name);
 537                 if (len > max_tracer_type_len)
 538                         max_tracer_type_len = len;
 539         }
 540  out:
 541         mutex_unlock(&trace_types_lock);
 542 }
 543
 544 void tracing_reset(struct trace_array *tr, int cpu)
 545 {
 546         ring_buffer_reset_cpu(tr->buffer, cpu);
 547 }
 548
 549 #define SAVED_CMDLINES 128
 550 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
 551 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
 552 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
 553 static int cmdline_idx;
 554 static DEFINE_SPINLOCK(trace_cmdline_lock);
 555
 556 /* temporary disable recording */
 557 atomic_t trace_record_cmdline_disabled __read_mostly;
 558
 559 static void trace_init_cmdlines(void)
 560 {
 561         memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
 562         memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
 563         cmdline_idx = 0;
 564 }
 565
 566 void trace_stop_cmdline_recording(void);
 567
 568 static void trace_save_cmdline(struct task_struct *tsk)
 569 {
 570         unsigned map;
 571         unsigned idx;
 572
 573         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
 574                 return;
 575
 576         /*
 577          * It's not the end of the world if we don't get
 578          * the lock, but we also don't want to spin
 579          * nor do we want to disable interrupts,
 580          * so if we miss here, then better luck next time.
 581          */
 582         if (!spin_trylock(&trace_cmdline_lock))
 583                 return;
 584
 585         idx = map_pid_to_cmdline[tsk->pid];
 586         if (idx >= SAVED_CMDLINES) {
 587                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
 588
 589                 map = map_cmdline_to_pid[idx];
 590                 if (map <= PID_MAX_DEFAULT)
 591                         map_pid_to_cmdline[map] = (unsigned)-1;
 592
 593                 map_pid_to_cmdline[tsk->pid] = idx;
 594
 595                 cmdline_idx = idx;
 596         }
 597
 598         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
 599
 600         spin_unlock(&trace_cmdline_lock);
 601 }
 602
 603 static char *trace_find_cmdline(int pid)
 604 {
 605         char *cmdline = "<...>";
 606         unsigned map;
 607
 608         if (!pid)
 609                 return "<idle>";
 610
 611         if (pid > PID_MAX_DEFAULT)
 612                 goto out;
 613
 614         map = map_pid_to_cmdline[pid];
 615         if (map >= SAVED_CMDLINES)
 616                 goto out;
 617
 618         cmdline = saved_cmdlines[map];
 619
 620  out:
 621         return cmdline;
 622 }
 623
 624 void tracing_record_cmdline(struct task_struct *tsk)
 625 {
 626         if (atomic_read(&trace_record_cmdline_disabled))
 627                 return;
 628
 629         trace_save_cmdline(tsk);
 630 }
 631
 632 void
 633 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags)
 634 {
 635         struct task_struct *tsk = current;
 636         unsigned long pc;
 637
 638         pc = preempt_count();
 639
 640         entry->field.preempt_count      = pc & 0xff;
 641         entry->field.pid                = (tsk) ? tsk->pid : 0;
 642         entry->field.flags =
 643                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
 644                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
 645                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
 646                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
 647 }
 648
 649 void
 650 trace_function(struct trace_array *tr, struct trace_array_cpu *data,
 651                unsigned long ip, unsigned long parent_ip, unsigned long flags)
 652 {
 653         struct ring_buffer_event *event;
 654         struct trace_entry *entry;
 655         unsigned long irq_flags;
 656
 657         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 658                                          &irq_flags);
 659         if (!event)
 660                 return;
 661         entry   = ring_buffer_event_data(event);
 662         tracing_generic_entry_update(entry, flags);
 663         entry->type                     = TRACE_FN;
 664         entry->field.fn.ip              = ip;
 665         entry->field.fn.parent_ip       = parent_ip;
 666         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 667 }
 668
 669 void
 670 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
 671        unsigned long ip, unsigned long parent_ip, unsigned long flags)
 672 {
 673         if (likely(!atomic_read(&data->disabled)))
 674                 trace_function(tr, data, ip, parent_ip, flags);
 675 }
 676
 677 void __trace_stack(struct trace_array *tr,
 678                    struct trace_array_cpu *data,
 679                    unsigned long flags,
 680                    int skip)
 681 {
 682         struct ring_buffer_event *event;
 683         struct trace_entry *entry;
 684         struct stack_trace trace;
 685         unsigned long irq_flags;
 686
 687         if (!(trace_flags & TRACE_ITER_STACKTRACE))
 688                 return;
 689
 690         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 691                                          &irq_flags);
 692         if (!event)
 693                 return;
 694         entry   = ring_buffer_event_data(event);
 695         tracing_generic_entry_update(entry, flags);
 696         entry->type             = TRACE_STACK;
 697
 698         memset(&entry->field.stack, 0, sizeof(entry->field.stack));
 699
 700         trace.nr_entries        = 0;
 701         trace.max_entries       = FTRACE_STACK_ENTRIES;
 702         trace.skip              = skip;
 703         trace.entries           = entry->field.stack.caller;
 704
 705         save_stack_trace(&trace);
 706         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 707 }
 708
 709 void
 710 __trace_special(void *__tr, void *__data,
 711                 unsigned long arg1, unsigned long arg2, unsigned long arg3)
 712 {
 713         struct ring_buffer_event *event;
 714         struct trace_array_cpu *data = __data;
 715         struct trace_array *tr = __tr;
 716         struct trace_entry *entry;
 717         unsigned long irq_flags;
 718
 719         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 720                                          &irq_flags);
 721         if (!event)
 722                 return;
 723         entry   = ring_buffer_event_data(event);
 724         tracing_generic_entry_update(entry, 0);
 725         entry->type                     = TRACE_SPECIAL;
 726         entry->field.special.arg1       = arg1;
 727         entry->field.special.arg2       = arg2;
 728         entry->field.special.arg3       = arg3;
 729         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 730         __trace_stack(tr, data, irq_flags, 4);
 731
 732         trace_wake_up();
 733 }
 734
 735 void
 736 tracing_sched_switch_trace(struct trace_array *tr,
 737                            struct trace_array_cpu *data,
 738                            struct task_struct *prev,
 739                            struct task_struct *next,
 740                            unsigned long flags)
 741 {
 742         struct ring_buffer_event *event;
 743         struct trace_entry *entry;
 744         unsigned long irq_flags;
 745
 746         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 747                                            &irq_flags);
 748         if (!event)
 749                 return;
 750         entry   = ring_buffer_event_data(event);
 751         tracing_generic_entry_update(entry, flags);
 752         entry->type                     = TRACE_CTX;
 753         entry->field.ctx.prev_pid       = prev->pid;
 754         entry->field.ctx.prev_prio      = prev->prio;
 755         entry->field.ctx.prev_state     = prev->state;
 756         entry->field.ctx.next_pid       = next->pid;
 757         entry->field.ctx.next_prio      = next->prio;
 758         entry->field.ctx.next_state     = next->state;
 759         entry->field.ctx.next_cpu       = task_cpu(next);
 760         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 761         __trace_stack(tr, data, flags, 5);
 762 }
 763
 764 void
 765 tracing_sched_wakeup_trace(struct trace_array *tr,
 766                            struct trace_array_cpu *data,
 767                            struct task_struct *wakee,
 768                            struct task_struct *curr,
 769                            unsigned long flags)
 770 {
 771         struct ring_buffer_event *event;
 772         struct trace_entry *entry;
 773         unsigned long irq_flags;
 774
 775         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
 776                                            &irq_flags);
 777         if (!event)
 778                 return;
 779         entry   = ring_buffer_event_data(event);
 780         tracing_generic_entry_update(entry, flags);
 781         entry->type             = TRACE_WAKE;
 782         entry->field.ctx.prev_pid       = curr->pid;
 783         entry->field.ctx.prev_prio      = curr->prio;
 784         entry->field.ctx.prev_state     = curr->state;
 785         entry->field.ctx.next_pid       = wakee->pid;
 786         entry->field.ctx.next_prio      = wakee->prio;
 787         entry->field.ctx.next_state     = wakee->state;
 788         entry->field.ctx.next_cpu       = task_cpu(wakee);
 789         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
 790         __trace_stack(tr, data, flags, 6);
 791
 792         trace_wake_up();
 793 }
 794
 795 void
 796 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
 797 {
 798         struct trace_array *tr = &global_trace;
 799         struct trace_array_cpu *data;
 800         unsigned long flags;
 801         long disabled;
 802         int cpu;
 803
 804         if (tracing_disabled || !tr->ctrl)
 805                 return;
 806
 807         local_irq_save(flags);
 808         cpu = raw_smp_processor_id();
 809         data = tr->data[cpu];
 810         disabled = atomic_inc_return(&data->disabled);
 811
 812         if (likely(disabled == 1))
 813                 __trace_special(tr, data, arg1, arg2, arg3);
 814
 815         atomic_dec(&data->disabled);
 816         local_irq_restore(flags);
 817 }
 818
 819 #ifdef CONFIG_FTRACE
 820 static void
 821 function_trace_call(unsigned long ip, unsigned long parent_ip)
 822 {
 823         struct trace_array *tr = &global_trace;
 824         struct trace_array_cpu *data;
 825         unsigned long flags;
 826         long disabled;
 827         int cpu;
 828
 829         if (unlikely(!ftrace_function_enabled))
 830                 return;
 831
 832         if (skip_trace(ip))
 833                 return;
 834
 835         local_irq_save(flags);
 836         cpu = raw_smp_processor_id();
 837         data = tr->data[cpu];
 838         disabled = atomic_inc_return(&data->disabled);
 839
 840         if (likely(disabled == 1))
 841                 trace_function(tr, data, ip, parent_ip, flags);
 842
 843         atomic_dec(&data->disabled);
 844         local_irq_restore(flags);
 845 }
 846
 847 static struct ftrace_ops trace_ops __read_mostly =
 848 {
 849         .func = function_trace_call,
 850 };
 851
 852 void tracing_start_function_trace(void)
 853 {
 854         ftrace_function_enabled = 0;
 855         register_ftrace_function(&trace_ops);
 856         if (tracer_enabled)
 857                 ftrace_function_enabled = 1;
 858 }
 859
 860 void tracing_stop_function_trace(void)
 861 {
 862         ftrace_function_enabled = 0;
 863         unregister_ftrace_function(&trace_ops);
 864 }
 865 #endif
 866
 867 enum trace_file_type {
 868         TRACE_FILE_LAT_FMT      = 1,
 869 };
 870
 871 static void trace_iterator_increment(struct trace_iterator *iter, int cpu)
 872 {
 873         iter->idx++;
 874         ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
 875 }
 876
 877 static struct trace_entry *
 878 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
 879 {
 880         struct ring_buffer_event *event;
 881         struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
 882
 883         event = ring_buffer_iter_peek(buf_iter, ts);
 884         return event ? ring_buffer_event_data(event) : NULL;
 885 }
 886 static struct trace_entry *
 887 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
 888 {
 889         struct ring_buffer *buffer = iter->tr->buffer;
 890         struct trace_entry *ent, *next = NULL;
 891         u64 next_ts = 0, ts;
 892         int next_cpu = -1;
 893         int cpu;
 894
 895         for_each_tracing_cpu(cpu) {
 896
 897                 if (ring_buffer_empty_cpu(buffer, cpu))
 898                         continue;
 899
 900                 ent = peek_next_entry(iter, cpu, &ts);
 901
 902                 /*
 903                  * Pick the entry with the smallest timestamp:
 904                  */
 905                 if (ent && (!next || ts < next_ts)) {
 906                         next = ent;
 907                         next_cpu = cpu;
 908                         next_ts = ts;
 909                 }
 910         }
 911
 912         if (ent_cpu)
 913                 *ent_cpu = next_cpu;
 914
 915         if (ent_ts)
 916                 *ent_ts = next_ts;
 917
 918         return next;
 919 }
 920
 921 /* Find the next real entry, without updating the iterator itself */
 922 static struct trace_entry *
 923 find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
 924 {
 925         return __find_next_entry(iter, ent_cpu, ent_ts);
 926 }
 927
 928 /* Find the next real entry, and increment the iterator to the next entry */
 929 static void *find_next_entry_inc(struct trace_iterator *iter)
 930 {
 931         iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
 932
 933         if (iter->ent)
 934                 trace_iterator_increment(iter, iter->cpu);
 935
 936         return iter->ent ? iter : NULL;
 937 }
 938
 939 static void trace_consume(struct trace_iterator *iter)
 940 {
 941         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
 942 }
 943
 944 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
 945 {
 946         struct trace_iterator *iter = m->private;
 947         int i = (int)*pos;
 948         void *ent;
 949
 950         (*pos)++;
 951
 952         /* can't go backwards */
 953         if (iter->idx > i)
 954                 return NULL;
 955
 956         if (iter->idx < 0)
 957                 ent = find_next_entry_inc(iter);
 958         else
 959                 ent = iter;
 960
 961         while (ent && iter->idx < i)
 962                 ent = find_next_entry_inc(iter);
 963
 964         iter->pos = *pos;
 965
 966         return ent;
 967 }
 968
 969 static void *s_start(struct seq_file *m, loff_t *pos)
 970 {
 971         struct trace_iterator *iter = m->private;
 972         void *p = NULL;
 973         loff_t l = 0;
 974         int cpu;
 975
 976         mutex_lock(&trace_types_lock);
 977
 978         if (!current_trace || current_trace != iter->trace) {
 979                 mutex_unlock(&trace_types_lock);
 980                 return NULL;
 981         }
 982
 983         atomic_inc(&trace_record_cmdline_disabled);
 984
 985         /* let the tracer grab locks here if needed */
 986         if (current_trace->start)
 987                 current_trace->start(iter);
 988
 989         if (*pos != iter->pos) {
 990                 iter->ent = NULL;
 991                 iter->cpu = 0;
 992                 iter->idx = -1;
 993
 994                 for_each_tracing_cpu(cpu) {
 995                         ring_buffer_iter_reset(iter->buffer_iter[cpu]);
 996                 }
 997
 998                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
 999                         ;
1000
1001         } else {
1002                 l = *pos - 1;
1003                 p = s_next(m, p, &l);
1004         }
1005
1006         return p;
1007 }
1008
1009 static void s_stop(struct seq_file *m, void *p)
1010 {
1011         struct trace_iterator *iter = m->private;
1012
1013         atomic_dec(&trace_record_cmdline_disabled);
1014
1015         /* let the tracer release locks here if needed */
1016         if (current_trace && current_trace == iter->trace && iter->trace->stop)
1017                 iter->trace->stop(iter);
1018
1019         mutex_unlock(&trace_types_lock);
1020 }
1021
1022 #define KRETPROBE_MSG "[unknown/kretprobe'd]"
1023
1024 #ifdef CONFIG_KRETPROBES
1025 static inline int kretprobed(unsigned long addr)
1026 {
1027         return addr == (unsigned long)kretprobe_trampoline;
1028 }
1029 #else
1030 static inline int kretprobed(unsigned long addr)
1031 {
1032         return 0;
1033 }
1034 #endif /* CONFIG_KRETPROBES */
1035
1036 static int
1037 seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1038 {
1039 #ifdef CONFIG_KALLSYMS
1040         char str[KSYM_SYMBOL_LEN];
1041
1042         kallsyms_lookup(address, NULL, NULL, NULL, str);
1043
1044         return trace_seq_printf(s, fmt, str);
1045 #endif
1046         return 1;
1047 }
1048
1049 static int
1050 seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1051                      unsigned long address)
1052 {
1053 #ifdef CONFIG_KALLSYMS
1054         char str[KSYM_SYMBOL_LEN];
1055
1056         sprint_symbol(str, address);
1057         return trace_seq_printf(s, fmt, str);
1058 #endif
1059         return 1;
1060 }
1061
1062 #ifndef CONFIG_64BIT
1063 # define IP_FMT "%08lx"
1064 #else
1065 # define IP_FMT "%016lx"
1066 #endif
1067
1068 static int
1069 seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1070 {
1071         int ret;
1072
1073         if (!ip)
1074                 return trace_seq_printf(s, "0");
1075
1076         if (sym_flags & TRACE_ITER_SYM_OFFSET)
1077                 ret = seq_print_sym_offset(s, "%s", ip);
1078         else
1079                 ret = seq_print_sym_short(s, "%s", ip);
1080
1081         if (!ret)
1082                 return 0;
1083
1084         if (sym_flags & TRACE_ITER_SYM_ADDR)
1085                 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1086         return ret;
1087 }
1088
1089 static void print_lat_help_header(struct seq_file *m)
1090 {
1091         seq_puts(m, "#                  _------=> CPU#            \n");
1092         seq_puts(m, "#                 / _-----=> irqs-off        \n");
1093         seq_puts(m, "#                | / _----=> need-resched    \n");
1094         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1095         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1096         seq_puts(m, "#                |||| /                      \n");
1097         seq_puts(m, "#                |||||     delay             \n");
1098         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1099         seq_puts(m, "#     \\   /      |||||   \\   |   /           \n");
1100 }
1101
1102 static void print_func_help_header(struct seq_file *m)
1103 {
1104         seq_puts(m, "#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n");
1105         seq_puts(m, "#              | |       |          |         |\n");
1106 }
1107
1108
1109 static void
1110 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1111 {
1112         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1113         struct trace_array *tr = iter->tr;
1114         struct trace_array_cpu *data = tr->data[tr->cpu];
1115         struct tracer *type = current_trace;
1116         unsigned long total;
1117         unsigned long entries;
1118         const char *name = "preemption";
1119
1120         if (type)
1121                 name = type->name;
1122
1123         entries = ring_buffer_entries(iter->tr->buffer);
1124         total = entries +
1125                 ring_buffer_overruns(iter->tr->buffer);
1126
1127         seq_printf(m, "%s latency trace v1.1.5 on %s\n",
1128                    name, UTS_RELEASE);
1129         seq_puts(m, "-----------------------------------"
1130                  "---------------------------------\n");
1131         seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
1132                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1133                    nsecs_to_usecs(data->saved_latency),
1134                    entries,
1135                    total,
1136                    tr->cpu,
1137 #if defined(CONFIG_PREEMPT_NONE)
1138                    "server",
1139 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1140                    "desktop",
1141 #elif defined(CONFIG_PREEMPT)
1142                    "preempt",
1143 #else
1144                    "unknown",
1145 #endif
1146                    /* These are reserved for later use */
1147                    0, 0, 0, 0);
1148 #ifdef CONFIG_SMP
1149         seq_printf(m, " #P:%d)\n", num_online_cpus());
1150 #else
1151         seq_puts(m, ")\n");
1152 #endif
1153         seq_puts(m, "    -----------------\n");
1154         seq_printf(m, "    | task: %.16s-%d "
1155                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1156                    data->comm, data->pid, data->uid, data->nice,
1157                    data->policy, data->rt_priority);
1158         seq_puts(m, "    -----------------\n");
1159
1160         if (data->critical_start) {
1161                 seq_puts(m, " => started at: ");
1162                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1163                 trace_print_seq(m, &iter->seq);
1164                 seq_puts(m, "\n => ended at:   ");
1165                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1166                 trace_print_seq(m, &iter->seq);
1167                 seq_puts(m, "\n");
1168         }
1169
1170         seq_puts(m, "\n");
1171 }
1172
1173 static void
1174 lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1175 {
1176         struct trace_field *field = &entry->field;
1177         int hardirq, softirq;
1178         char *comm;
1179
1180         comm = trace_find_cmdline(field->pid);
1181
1182         trace_seq_printf(s, "%8.8s-%-5d ", comm, field->pid);
1183         trace_seq_printf(s, "%3d", cpu);
1184         trace_seq_printf(s, "%c%c",
1185                         (field->flags & TRACE_FLAG_IRQS_OFF) ? 'd' : '.',
1186                         ((field->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1187
1188         hardirq = field->flags & TRACE_FLAG_HARDIRQ;
1189         softirq = field->flags & TRACE_FLAG_SOFTIRQ;
1190         if (hardirq && softirq) {
1191                 trace_seq_putc(s, 'H');
1192         } else {
1193                 if (hardirq) {
1194                         trace_seq_putc(s, 'h');
1195                 } else {
1196                         if (softirq)
1197                                 trace_seq_putc(s, 's');
1198                         else
1199                                 trace_seq_putc(s, '.');
1200                 }
1201         }
1202
1203         if (field->preempt_count)
1204                 trace_seq_printf(s, "%x", field->preempt_count);
1205         else
1206                 trace_seq_puts(s, ".");
1207 }
1208
1209 unsigned long preempt_mark_thresh = 100;
1210
1211 static void
1212 lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1213                     unsigned long rel_usecs)
1214 {
1215         trace_seq_printf(s, " %4lldus", abs_usecs);
1216         if (rel_usecs > preempt_mark_thresh)
1217                 trace_seq_puts(s, "!: ");
1218         else if (rel_usecs > 1)
1219                 trace_seq_puts(s, "+: ");
1220         else
1221                 trace_seq_puts(s, " : ");
1222 }
1223
1224 static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1225
1226 /*
1227  * The message is supposed to contain an ending newline.
1228  * If the printing stops prematurely, try to add a newline of our own.
1229  */
1230 void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1231 {
1232         struct trace_entry *ent;
1233         bool ok = true;
1234
1235         ent = peek_next_entry(iter, iter->cpu, NULL);
1236         if (!ent || ent->type != TRACE_CONT) {
1237                 trace_seq_putc(s, '\n');
1238                 return;
1239         }
1240
1241         do {
1242                 if (ok)
1243                         ok = (trace_seq_printf(s, "%s", ent->cont.buf) > 0);
1244                 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1245                 ent = peek_next_entry(iter, iter->cpu, NULL);
1246         } while (ent && ent->type == TRACE_CONT);
1247
1248         if (!ok)
1249                 trace_seq_putc(s, '\n');
1250 }
1251
1252 static int
1253 print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1254 {
1255         struct trace_seq *s = &iter->seq;
1256         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1257         struct trace_entry *next_entry;
1258         unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1259         struct trace_entry *entry = iter->ent;
1260         struct trace_field *field = &entry->field;
1261         unsigned long abs_usecs;
1262         unsigned long rel_usecs;
1263         u64 next_ts;
1264         char *comm;
1265         int S, T;
1266         int i;
1267         unsigned state;
1268
1269         if (entry->type == TRACE_CONT)
1270                 return 1;
1271
1272         next_entry = find_next_entry(iter, NULL, &next_ts);
1273         if (!next_entry)
1274                 next_ts = iter->ts;
1275         rel_usecs = ns2usecs(next_ts - iter->ts);
1276         abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
1277
1278         if (verbose) {
1279                 comm = trace_find_cmdline(field->pid);
1280                 trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
1281                                  " %ld.%03ldms (+%ld.%03ldms): ",
1282                                  comm,
1283                                  field->pid, cpu, field->flags,
1284                                  field->preempt_count, trace_idx,
1285                                  ns2usecs(iter->ts),
1286                                  abs_usecs/1000,
1287                                  abs_usecs % 1000, rel_usecs/1000,
1288                                  rel_usecs % 1000);
1289         } else {
1290                 lat_print_generic(s, entry, cpu);
1291                 lat_print_timestamp(s, abs_usecs, rel_usecs);
1292         }
1293         switch (entry->type) {
1294         case TRACE_FN:
1295                 seq_print_ip_sym(s, field->fn.ip, sym_flags);
1296                 trace_seq_puts(s, " (");
1297                 if (kretprobed(field->fn.parent_ip))
1298                         trace_seq_puts(s, KRETPROBE_MSG);
1299                 else
1300                         seq_print_ip_sym(s, field->fn.parent_ip, sym_flags);
1301                 trace_seq_puts(s, ")\n");
1302                 break;
1303         case TRACE_CTX:
1304         case TRACE_WAKE:
1305                 T = field->ctx.next_state < sizeof(state_to_char) ?
1306                         state_to_char[field->ctx.next_state] : 'X';
1307
1308                 state = field->ctx.prev_state ?
1309                         __ffs(field->ctx.prev_state) + 1 : 0;
1310                 S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
1311                 comm = trace_find_cmdline(field->ctx.next_pid);
1312                 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1313                                  field->ctx.prev_pid,
1314                                  field->ctx.prev_prio,
1315                                  S, entry->type == TRACE_CTX ? "==>" : "  +",
1316                                  field->ctx.next_cpu,
1317                                  field->ctx.next_pid,
1318                                  field->ctx.next_prio,
1319                                  T, comm);
1320                 break;
1321         case TRACE_SPECIAL:
1322                 trace_seq_printf(s, "# %ld %ld %ld\n",
1323                                  field->special.arg1,
1324                                  field->special.arg2,
1325                                  field->special.arg3);
1326                 break;
1327         case TRACE_STACK:
1328                 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1329                         if (i)
1330                                 trace_seq_puts(s, " <= ");
1331                         seq_print_ip_sym(s, field->stack.caller[i], sym_flags);
1332                 }
1333                 trace_seq_puts(s, "\n");
1334                 break;
1335         case TRACE_PRINT:
1336                 seq_print_ip_sym(s, field->print.ip, sym_flags);
1337                 trace_seq_printf(s, ": %s", field->print.buf);
1338                 if (field->flags & TRACE_FLAG_CONT)
1339                         trace_seq_print_cont(s, iter);
1340                 break;
1341         default:
1342                 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1343         }
1344         return 1;
1345 }
1346
1347 static int print_trace_fmt(struct trace_iterator *iter)
1348 {
1349         struct trace_seq *s = &iter->seq;
1350         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1351         struct trace_entry *entry;
1352         struct trace_field *field;
1353         unsigned long usec_rem;
1354         unsigned long long t;
1355         unsigned long secs;
1356         char *comm;
1357         int ret;
1358         int S, T;
1359         int i;
1360
1361         entry = iter->ent;
1362
1363         if (entry->type == TRACE_CONT)
1364                 return 1;
1365
1366         field = &entry->field;
1367
1368         comm = trace_find_cmdline(iter->ent->field.pid);
1369
1370         t = ns2usecs(iter->ts);
1371         usec_rem = do_div(t, 1000000ULL);
1372         secs = (unsigned long)t;
1373
1374         ret = trace_seq_printf(s, "%16s-%-5d ", comm, field->pid);
1375         if (!ret)
1376                 return 0;
1377         ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
1378         if (!ret)
1379                 return 0;
1380         ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1381         if (!ret)
1382                 return 0;
1383
1384         switch (entry->type) {
1385         case TRACE_FN:
1386                 ret = seq_print_ip_sym(s, field->fn.ip, sym_flags);
1387                 if (!ret)
1388                         return 0;
1389                 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
1390                                                 field->fn.parent_ip) {
1391                         ret = trace_seq_printf(s, " <-");
1392                         if (!ret)
1393                                 return 0;
1394                         if (kretprobed(field->fn.parent_ip))
1395                                 ret = trace_seq_puts(s, KRETPROBE_MSG);
1396                         else
1397                                 ret = seq_print_ip_sym(s,
1398                                                        field->fn.parent_ip,
1399                                                        sym_flags);
1400                         if (!ret)
1401                                 return 0;
1402                 }
1403                 ret = trace_seq_printf(s, "\n");
1404                 if (!ret)
1405                         return 0;
1406                 break;
1407         case TRACE_CTX:
1408         case TRACE_WAKE:
1409                 S = field->ctx.prev_state < sizeof(state_to_char) ?
1410                         state_to_char[field->ctx.prev_state] : 'X';
1411                 T = field->ctx.next_state < sizeof(state_to_char) ?
1412                         state_to_char[field->ctx.next_state] : 'X';
1413                 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
1414                                        field->ctx.prev_pid,
1415                                        field->ctx.prev_prio,
1416                                        S,
1417                                        entry->type == TRACE_CTX ? "==>" : "  +",
1418                                        field->ctx.next_cpu,
1419                                        field->ctx.next_pid,
1420                                        field->ctx.next_prio,
1421                                        T);
1422                 if (!ret)
1423                         return 0;
1424                 break;
1425         case TRACE_SPECIAL:
1426                 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1427                                  field->special.arg1,
1428                                  field->special.arg2,
1429                                  field->special.arg3);
1430                 if (!ret)
1431                         return 0;
1432                 break;
1433         case TRACE_STACK:
1434                 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1435                         if (i) {
1436                                 ret = trace_seq_puts(s, " <= ");
1437                                 if (!ret)
1438                                         return 0;
1439                         }
1440                         ret = seq_print_ip_sym(s, field->stack.caller[i],
1441                                                sym_flags);
1442                         if (!ret)
1443                                 return 0;
1444                 }
1445                 ret = trace_seq_puts(s, "\n");
1446                 if (!ret)
1447                         return 0;
1448                 break;
1449         case TRACE_PRINT:
1450                 seq_print_ip_sym(s, field->print.ip, sym_flags);
1451                 trace_seq_printf(s, ": %s", field->print.buf);
1452                 if (field->flags & TRACE_FLAG_CONT)
1453                         trace_seq_print_cont(s, iter);
1454                 break;
1455         }
1456         return 1;
1457 }
1458
1459 static int print_raw_fmt(struct trace_iterator *iter)
1460 {
1461         struct trace_seq *s = &iter->seq;
1462         struct trace_entry *entry;
1463         struct trace_field *field;
1464         int ret;
1465         int S, T;
1466
1467         entry = iter->ent;
1468
1469         if (entry->type == TRACE_CONT)
1470                 return 1;
1471
1472         field = &entry->field;
1473
1474         ret = trace_seq_printf(s, "%d %d %llu ",
1475                 field->pid, iter->cpu, iter->ts);
1476         if (!ret)
1477                 return 0;
1478
1479         switch (entry->type) {
1480         case TRACE_FN:
1481                 ret = trace_seq_printf(s, "%x %x\n",
1482                                         field->fn.ip,
1483                                         field->fn.parent_ip);
1484                 if (!ret)
1485                         return 0;
1486                 break;
1487         case TRACE_CTX:
1488         case TRACE_WAKE:
1489                 S = field->ctx.prev_state < sizeof(state_to_char) ?
1490                         state_to_char[field->ctx.prev_state] : 'X';
1491                 T = field->ctx.next_state < sizeof(state_to_char) ?
1492                         state_to_char[field->ctx.next_state] : 'X';
1493                 if (entry->type == TRACE_WAKE)
1494                         S = '+';
1495                 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
1496                                        field->ctx.prev_pid,
1497                                        field->ctx.prev_prio,
1498                                        S,
1499                                        field->ctx.next_cpu,
1500                                        field->ctx.next_pid,
1501                                        field->ctx.next_prio,
1502                                        T);
1503                 if (!ret)
1504                         return 0;
1505                 break;
1506         case TRACE_SPECIAL:
1507         case TRACE_STACK:
1508                 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1509                                  field->special.arg1,
1510                                  field->special.arg2,
1511                                  field->special.arg3);
1512                 if (!ret)
1513                         return 0;
1514                 break;
1515         case TRACE_PRINT:
1516                 trace_seq_printf(s, "# %lx %s",
1517                                  field->print.ip, field->print.buf);
1518                 if (field->flags & TRACE_FLAG_CONT)
1519                         trace_seq_print_cont(s, iter);
1520                 break;
1521         }
1522         return 1;
1523 }
1524
1525 #define SEQ_PUT_FIELD_RET(s, x)                         \
1526 do {                                                    \
1527         if (!trace_seq_putmem(s, &(x), sizeof(x)))      \
1528                 return 0;                               \
1529 } while (0)
1530
1531 #define SEQ_PUT_HEX_FIELD_RET(s, x)                     \
1532 do {                                                    \
1533         if (!trace_seq_putmem_hex(s, &(x), sizeof(x)))  \
1534                 return 0;                               \
1535 } while (0)
1536
1537 static int print_hex_fmt(struct trace_iterator *iter)
1538 {
1539         struct trace_seq *s = &iter->seq;
1540         unsigned char newline = '\n';
1541         struct trace_entry *entry;
1542         struct trace_field *field;
1543         int S, T;
1544
1545         entry = iter->ent;
1546
1547         if (entry->type == TRACE_CONT)
1548                 return 1;
1549
1550         field = &entry->field;
1551
1552         SEQ_PUT_HEX_FIELD_RET(s, field->pid);
1553         SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1554         SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1555
1556         switch (entry->type) {
1557         case TRACE_FN:
1558                 SEQ_PUT_HEX_FIELD_RET(s, field->fn.ip);
1559                 SEQ_PUT_HEX_FIELD_RET(s, field->fn.parent_ip);
1560                 break;
1561         case TRACE_CTX:
1562         case TRACE_WAKE:
1563                 S = field->ctx.prev_state < sizeof(state_to_char) ?
1564                         state_to_char[field->ctx.prev_state] : 'X';
1565                 T = field->ctx.next_state < sizeof(state_to_char) ?
1566                         state_to_char[field->ctx.next_state] : 'X';
1567                 if (entry->type == TRACE_WAKE)
1568                         S = '+';
1569                 SEQ_PUT_HEX_FIELD_RET(s, field->ctx.prev_pid);
1570                 SEQ_PUT_HEX_FIELD_RET(s, field->ctx.prev_prio);
1571                 SEQ_PUT_HEX_FIELD_RET(s, S);
1572                 SEQ_PUT_HEX_FIELD_RET(s, field->ctx.next_cpu);
1573                 SEQ_PUT_HEX_FIELD_RET(s, field->ctx.next_pid);
1574                 SEQ_PUT_HEX_FIELD_RET(s, field->ctx.next_prio);
1575                 SEQ_PUT_HEX_FIELD_RET(s, T);
1576                 break;
1577         case TRACE_SPECIAL:
1578         case TRACE_STACK:
1579                 SEQ_PUT_HEX_FIELD_RET(s, field->special.arg1);
1580                 SEQ_PUT_HEX_FIELD_RET(s, field->special.arg2);
1581                 SEQ_PUT_HEX_FIELD_RET(s, field->special.arg3);
1582                 break;
1583         }
1584         SEQ_PUT_FIELD_RET(s, newline);
1585
1586         return 1;
1587 }
1588
1589 static int print_bin_fmt(struct trace_iterator *iter)
1590 {
1591         struct trace_seq *s = &iter->seq;
1592         struct trace_entry *entry;
1593         struct trace_field *field;
1594
1595         entry = iter->ent;
1596
1597         if (entry->type == TRACE_CONT)
1598                 return 1;
1599
1600         field = &entry->field;
1601
1602         SEQ_PUT_FIELD_RET(s, field->pid);
1603         SEQ_PUT_FIELD_RET(s, field->cpu);
1604         SEQ_PUT_FIELD_RET(s, iter->ts);
1605
1606         switch (entry->type) {
1607         case TRACE_FN:
1608                 SEQ_PUT_FIELD_RET(s, field->fn.ip);
1609                 SEQ_PUT_FIELD_RET(s, field->fn.parent_ip);
1610                 break;
1611         case TRACE_CTX:
1612                 SEQ_PUT_FIELD_RET(s, field->ctx.prev_pid);
1613                 SEQ_PUT_FIELD_RET(s, field->ctx.prev_prio);
1614                 SEQ_PUT_FIELD_RET(s, field->ctx.prev_state);
1615                 SEQ_PUT_FIELD_RET(s, field->ctx.next_pid);
1616                 SEQ_PUT_FIELD_RET(s, field->ctx.next_prio);
1617                 SEQ_PUT_FIELD_RET(s, field->ctx.next_state);
1618                 break;
1619         case TRACE_SPECIAL:
1620         case TRACE_STACK:
1621                 SEQ_PUT_FIELD_RET(s, field->special.arg1);
1622                 SEQ_PUT_FIELD_RET(s, field->special.arg2);
1623                 SEQ_PUT_FIELD_RET(s, field->special.arg3);
1624                 break;
1625         }
1626         return 1;
1627 }
1628
1629 static int trace_empty(struct trace_iterator *iter)
1630 {
1631         int cpu;
1632
1633         for_each_tracing_cpu(cpu) {
1634                 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1635                         return 0;
1636         }
1637         return 1;
1638 }
1639
1640 static int print_trace_line(struct trace_iterator *iter)
1641 {
1642         if (iter->trace && iter->trace->print_line)
1643                 return iter->trace->print_line(iter);
1644
1645         if (trace_flags & TRACE_ITER_BIN)
1646                 return print_bin_fmt(iter);
1647
1648         if (trace_flags & TRACE_ITER_HEX)
1649                 return print_hex_fmt(iter);
1650
1651         if (trace_flags & TRACE_ITER_RAW)
1652                 return print_raw_fmt(iter);
1653
1654         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
1655                 return print_lat_fmt(iter, iter->idx, iter->cpu);
1656
1657         return print_trace_fmt(iter);
1658 }
1659
1660 static int s_show(struct seq_file *m, void *v)
1661 {
1662         struct trace_iterator *iter = v;
1663
1664         if (iter->ent == NULL) {
1665                 if (iter->tr) {
1666                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
1667                         seq_puts(m, "#\n");
1668                 }
1669                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1670                         /* print nothing if the buffers are empty */
1671                         if (trace_empty(iter))
1672                                 return 0;
1673                         print_trace_header(m, iter);
1674                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1675                                 print_lat_help_header(m);
1676                 } else {
1677                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1678                                 print_func_help_header(m);
1679                 }
1680         } else {
1681                 print_trace_line(iter);
1682                 trace_print_seq(m, &iter->seq);
1683         }
1684
1685         return 0;
1686 }
1687
1688 static struct seq_operations tracer_seq_ops = {
1689         .start          = s_start,
1690         .next           = s_next,
1691         .stop           = s_stop,
1692         .show           = s_show,
1693 };
1694
1695 static struct trace_iterator *
1696 __tracing_open(struct inode *inode, struct file *file, int *ret)
1697 {
1698         struct trace_iterator *iter;
1699         struct seq_file *m;
1700         int cpu;
1701
1702         if (tracing_disabled) {
1703                 *ret = -ENODEV;
1704                 return NULL;
1705         }
1706
1707         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1708         if (!iter) {
1709                 *ret = -ENOMEM;
1710                 goto out;
1711         }
1712
1713         mutex_lock(&trace_types_lock);
1714         if (current_trace && current_trace->print_max)
1715                 iter->tr = &max_tr;
1716         else
1717                 iter->tr = inode->i_private;
1718         iter->trace = current_trace;
1719         iter->pos = -1;
1720
1721         for_each_tracing_cpu(cpu) {
1722                 iter->buffer_iter[cpu] =
1723                         ring_buffer_read_start(iter->tr->buffer, cpu);
1724                 if (!iter->buffer_iter[cpu])
1725                         goto fail_buffer;
1726         }
1727
1728         /* TODO stop tracer */
1729         *ret = seq_open(file, &tracer_seq_ops);
1730         if (*ret)
1731                 goto fail_buffer;
1732
1733         m = file->private_data;
1734         m->private = iter;
1735
1736         /* stop the trace while dumping */
1737         if (iter->tr->ctrl) {
1738                 tracer_enabled = 0;
1739                 ftrace_function_enabled = 0;
1740         }
1741
1742         if (iter->trace && iter->trace->open)
1743                         iter->trace->open(iter);
1744
1745         mutex_unlock(&trace_types_lock);
1746
1747  out:
1748         return iter;
1749
1750  fail_buffer:
1751         for_each_tracing_cpu(cpu) {
1752                 if (iter->buffer_iter[cpu])
1753                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1754         }
1755         mutex_unlock(&trace_types_lock);
1756
1757         return ERR_PTR(-ENOMEM);
1758 }
1759
1760 int tracing_open_generic(struct inode *inode, struct file *filp)
1761 {
1762         if (tracing_disabled)
1763                 return -ENODEV;
1764
1765         filp->private_data = inode->i_private;
1766         return 0;
1767 }
1768
1769 int tracing_release(struct inode *inode, struct file *file)
1770 {
1771         struct seq_file *m = (struct seq_file *)file->private_data;
1772         struct trace_iterator *iter = m->private;
1773         int cpu;
1774
1775         mutex_lock(&trace_types_lock);
1776         for_each_tracing_cpu(cpu) {
1777                 if (iter->buffer_iter[cpu])
1778                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1779         }
1780
1781         if (iter->trace && iter->trace->close)
1782                 iter->trace->close(iter);
1783
1784         /* reenable tracing if it was previously enabled */
1785         if (iter->tr->ctrl) {
1786                 tracer_enabled = 1;
1787                 /*
1788                  * It is safe to enable function tracing even if it
1789                  * isn't used
1790                  */
1791                 ftrace_function_enabled = 1;
1792         }
1793         mutex_unlock(&trace_types_lock);
1794
1795         seq_release(inode, file);
1796         kfree(iter);
1797         return 0;
1798 }
1799
1800 static int tracing_open(struct inode *inode, struct file *file)
1801 {
1802         int ret;
1803
1804         __tracing_open(inode, file, &ret);
1805
1806         return ret;
1807 }
1808
1809 static int tracing_lt_open(struct inode *inode, struct file *file)
1810 {
1811         struct trace_iterator *iter;
1812         int ret;
1813
1814         iter = __tracing_open(inode, file, &ret);
1815
1816         if (!ret)
1817                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
1818
1819         return ret;
1820 }
1821
1822
1823 static void *
1824 t_next(struct seq_file *m, void *v, loff_t *pos)
1825 {
1826         struct tracer *t = m->private;
1827
1828         (*pos)++;
1829
1830         if (t)
1831                 t = t->next;
1832
1833         m->private = t;
1834
1835         return t;
1836 }
1837
1838 static void *t_start(struct seq_file *m, loff_t *pos)
1839 {
1840         struct tracer *t = m->private;
1841         loff_t l = 0;
1842
1843         mutex_lock(&trace_types_lock);
1844         for (; t && l < *pos; t = t_next(m, t, &l))
1845                 ;
1846
1847         return t;
1848 }
1849
1850 static void t_stop(struct seq_file *m, void *p)
1851 {
1852         mutex_unlock(&trace_types_lock);
1853 }
1854
1855 static int t_show(struct seq_file *m, void *v)
1856 {
1857         struct tracer *t = v;
1858
1859         if (!t)
1860                 return 0;
1861
1862         seq_printf(m, "%s", t->name);
1863         if (t->next)
1864                 seq_putc(m, ' ');
1865         else
1866                 seq_putc(m, '\n');
1867
1868         return 0;
1869 }
1870
1871 static struct seq_operations show_traces_seq_ops = {
1872         .start          = t_start,
1873         .next           = t_next,
1874         .stop           = t_stop,
1875         .show           = t_show,
1876 };
1877
1878 static int show_traces_open(struct inode *inode, struct file *file)
1879 {
1880         int ret;
1881
1882         if (tracing_disabled)
1883                 return -ENODEV;
1884
1885         ret = seq_open(file, &show_traces_seq_ops);
1886         if (!ret) {
1887                 struct seq_file *m = file->private_data;
1888                 m->private = trace_types;
1889         }
1890
1891         return ret;
1892 }
1893
1894 static struct file_operations tracing_fops = {
1895         .open           = tracing_open,
1896         .read           = seq_read,
1897         .llseek         = seq_lseek,
1898         .release        = tracing_release,
1899 };
1900
1901 static struct file_operations tracing_lt_fops = {
1902         .open           = tracing_lt_open,
1903         .read           = seq_read,
1904         .llseek         = seq_lseek,
1905         .release        = tracing_release,
1906 };
1907
1908 static struct file_operations show_traces_fops = {
1909         .open           = show_traces_open,
1910         .read           = seq_read,
1911         .release        = seq_release,
1912 };
1913
1914 /*
1915  * Only trace on a CPU if the bitmask is set:
1916  */
1917 static cpumask_t tracing_cpumask = CPU_MASK_ALL;
1918
1919 /*
1920  * When tracing/tracing_cpu_mask is modified then this holds
1921  * the new bitmask we are about to install:
1922  */
1923 static cpumask_t tracing_cpumask_new;
1924
1925 /*
1926  * The tracer itself will not take this lock, but still we want
1927  * to provide a consistent cpumask to user-space:
1928  */
1929 static DEFINE_MUTEX(tracing_cpumask_update_lock);
1930
1931 /*
1932  * Temporary storage for the character representation of the
1933  * CPU bitmask (and one more byte for the newline):
1934  */
1935 static char mask_str[NR_CPUS + 1];
1936
1937 static ssize_t
1938 tracing_cpumask_read(struct file *filp, char __user *ubuf,
1939                      size_t count, loff_t *ppos)
1940 {
1941         int len;
1942
1943         mutex_lock(&tracing_cpumask_update_lock);
1944
1945         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
1946         if (count - len < 2) {
1947                 count = -EINVAL;
1948                 goto out_err;
1949         }
1950         len += sprintf(mask_str + len, "\n");
1951         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
1952
1953 out_err:
1954         mutex_unlock(&tracing_cpumask_update_lock);
1955
1956         return count;
1957 }
1958
1959 static ssize_t
1960 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
1961                       size_t count, loff_t *ppos)
1962 {
1963         int err, cpu;
1964
1965         mutex_lock(&tracing_cpumask_update_lock);
1966         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
1967         if (err)
1968                 goto err_unlock;
1969
1970         raw_local_irq_disable();
1971         __raw_spin_lock(&ftrace_max_lock);
1972         for_each_tracing_cpu(cpu) {
1973                 /*
1974                  * Increase/decrease the disabled counter if we are
1975                  * about to flip a bit in the cpumask:
1976                  */
1977                 if (cpu_isset(cpu, tracing_cpumask) &&
1978                                 !cpu_isset(cpu, tracing_cpumask_new)) {
1979                         atomic_inc(&global_trace.data[cpu]->disabled);
1980                 }
1981                 if (!cpu_isset(cpu, tracing_cpumask) &&
1982                                 cpu_isset(cpu, tracing_cpumask_new)) {
1983                         atomic_dec(&global_trace.data[cpu]->disabled);
1984                 }
1985         }
1986         __raw_spin_unlock(&ftrace_max_lock);
1987         raw_local_irq_enable();
1988
1989         tracing_cpumask = tracing_cpumask_new;
1990
1991         mutex_unlock(&tracing_cpumask_update_lock);
1992
1993         return count;
1994
1995 err_unlock:
1996         mutex_unlock(&tracing_cpumask_update_lock);
1997
1998         return err;
1999 }
2000
2001 static struct file_operations tracing_cpumask_fops = {
2002         .open           = tracing_open_generic,
2003         .read           = tracing_cpumask_read,
2004         .write          = tracing_cpumask_write,
2005 };
2006
2007 static ssize_t
2008 tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2009                        size_t cnt, loff_t *ppos)
2010 {
2011         char *buf;
2012         int r = 0;
2013         int len = 0;
2014         int i;
2015
2016         /* calulate max size */
2017         for (i = 0; trace_options[i]; i++) {
2018                 len += strlen(trace_options[i]);
2019                 len += 3; /* "no" and space */
2020         }
2021
2022         /* +2 for \n and \0 */
2023         buf = kmalloc(len + 2, GFP_KERNEL);
2024         if (!buf)
2025                 return -ENOMEM;
2026
2027         for (i = 0; trace_options[i]; i++) {
2028                 if (trace_flags & (1 << i))
2029                         r += sprintf(buf + r, "%s ", trace_options[i]);
2030                 else
2031                         r += sprintf(buf + r, "no%s ", trace_options[i]);
2032         }
2033
2034         r += sprintf(buf + r, "\n");
2035         WARN_ON(r >= len + 2);
2036
2037         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2038
2039         kfree(buf);
2040
2041         return r;
2042 }
2043
2044 static ssize_t
2045 tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
2046                         size_t cnt, loff_t *ppos)
2047 {
2048         char buf[64];
2049         char *cmp = buf;
2050         int neg = 0;
2051         int i;
2052
2053         if (cnt >= sizeof(buf))
2054                 return -EINVAL;
2055
2056         if (copy_from_user(&buf, ubuf, cnt))
2057                 return -EFAULT;
2058
2059         buf[cnt] = 0;
2060
2061         if (strncmp(buf, "no", 2) == 0) {
2062                 neg = 1;
2063                 cmp += 2;
2064         }
2065
2066         for (i = 0; trace_options[i]; i++) {
2067                 int len = strlen(trace_options[i]);
2068
2069                 if (strncmp(cmp, trace_options[i], len) == 0) {
2070                         if (neg)
2071                                 trace_flags &= ~(1 << i);
2072                         else
2073                                 trace_flags |= (1 << i);
2074                         break;
2075                 }
2076         }
2077         /*
2078          * If no option could be set, return an error:
2079          */
2080         if (!trace_options[i])
2081                 return -EINVAL;
2082
2083         filp->f_pos += cnt;
2084
2085         return cnt;
2086 }
2087
2088 static struct file_operations tracing_iter_fops = {
2089         .open           = tracing_open_generic,
2090         .read           = tracing_iter_ctrl_read,
2091         .write          = tracing_iter_ctrl_write,
2092 };
2093
2094 static const char readme_msg[] =
2095         "tracing mini-HOWTO:\n\n"
2096         "# mkdir /debug\n"
2097         "# mount -t debugfs nodev /debug\n\n"
2098         "# cat /debug/tracing/available_tracers\n"
2099         "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
2100         "# cat /debug/tracing/current_tracer\n"
2101         "none\n"
2102         "# echo sched_switch > /debug/tracing/current_tracer\n"
2103         "# cat /debug/tracing/current_tracer\n"
2104         "sched_switch\n"
2105         "# cat /debug/tracing/iter_ctrl\n"
2106         "noprint-parent nosym-offset nosym-addr noverbose\n"
2107         "# echo print-parent > /debug/tracing/iter_ctrl\n"
2108         "# echo 1 > /debug/tracing/tracing_enabled\n"
2109         "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2110         "echo 0 > /debug/tracing/tracing_enabled\n"
2111 ;
2112
2113 static ssize_t
2114 tracing_readme_read(struct file *filp, char __user *ubuf,
2115                        size_t cnt, loff_t *ppos)
2116 {
2117         return simple_read_from_buffer(ubuf, cnt, ppos,
2118                                         readme_msg, strlen(readme_msg));
2119 }
2120
2121 static struct file_operations tracing_readme_fops = {
2122         .open           = tracing_open_generic,
2123         .read           = tracing_readme_read,
2124 };
2125
2126 static ssize_t
2127 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2128                   size_t cnt, loff_t *ppos)
2129 {
2130         struct trace_array *tr = filp->private_data;
2131         char buf[64];
2132         int r;
2133
2134         r = sprintf(buf, "%ld\n", tr->ctrl);
2135         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2136 }
2137
2138 static ssize_t
2139 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2140                    size_t cnt, loff_t *ppos)
2141 {
2142         struct trace_array *tr = filp->private_data;
2143         char buf[64];
2144         long val;
2145         int ret;
2146
2147         if (cnt >= sizeof(buf))
2148                 return -EINVAL;
2149
2150         if (copy_from_user(&buf, ubuf, cnt))
2151                 return -EFAULT;
2152
2153         buf[cnt] = 0;
2154
2155         ret = strict_strtoul(buf, 10, &val);
2156         if (ret < 0)
2157                 return ret;
2158
2159         val = !!val;
2160
2161         mutex_lock(&trace_types_lock);
2162         if (tr->ctrl ^ val) {
2163                 if (val)
2164                         tracer_enabled = 1;
2165                 else
2166                         tracer_enabled = 0;
2167
2168                 tr->ctrl = val;
2169
2170                 if (current_trace && current_trace->ctrl_update)
2171                         current_trace->ctrl_update(tr);
2172         }
2173         mutex_unlock(&trace_types_lock);
2174
2175         filp->f_pos += cnt;
2176
2177         return cnt;
2178 }
2179
2180 static ssize_t
2181 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2182                        size_t cnt, loff_t *ppos)
2183 {
2184         char buf[max_tracer_type_len+2];
2185         int r;
2186
2187         mutex_lock(&trace_types_lock);
2188         if (current_trace)
2189                 r = sprintf(buf, "%s\n", current_trace->name);
2190         else
2191                 r = sprintf(buf, "\n");
2192         mutex_unlock(&trace_types_lock);
2193
2194         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2195 }
2196
2197 static ssize_t
2198 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2199                         size_t cnt, loff_t *ppos)
2200 {
2201         struct trace_array *tr = &global_trace;
2202         struct tracer *t;
2203         char buf[max_tracer_type_len+1];
2204         int i;
2205
2206         if (cnt > max_tracer_type_len)
2207                 cnt = max_tracer_type_len;
2208
2209         if (copy_from_user(&buf, ubuf, cnt))
2210                 return -EFAULT;
2211
2212         buf[cnt] = 0;
2213
2214         /* strip ending whitespace. */
2215         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2216                 buf[i] = 0;
2217
2218         mutex_lock(&trace_types_lock);
2219         for (t = trace_types; t; t = t->next) {
2220                 if (strcmp(t->name, buf) == 0)
2221                         break;
2222         }
2223         if (!t || t == current_trace)
2224                 goto out;
2225
2226         if (current_trace && current_trace->reset)
2227                 current_trace->reset(tr);
2228
2229         current_trace = t;
2230         if (t->init)
2231                 t->init(tr);
2232
2233  out:
2234         mutex_unlock(&trace_types_lock);
2235
2236         filp->f_pos += cnt;
2237
2238         return cnt;
2239 }
2240
2241 static ssize_t
2242 tracing_max_lat_read(struct file *filp, char __user *ubuf,
2243                      size_t cnt, loff_t *ppos)
2244 {
2245         unsigned long *ptr = filp->private_data;
2246         char buf[64];
2247         int r;
2248
2249         r = snprintf(buf, sizeof(buf), "%ld\n",
2250                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2251         if (r > sizeof(buf))
2252                 r = sizeof(buf);
2253         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2254 }
2255
2256 static ssize_t
2257 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2258                       size_t cnt, loff_t *ppos)
2259 {
2260         long *ptr = filp->private_data;
2261         char buf[64];
2262         long val;
2263         int ret;
2264
2265         if (cnt >= sizeof(buf))
2266                 return -EINVAL;
2267
2268         if (copy_from_user(&buf, ubuf, cnt))
2269                 return -EFAULT;
2270
2271         buf[cnt] = 0;
2272
2273         ret = strict_strtoul(buf, 10, &val);
2274         if (ret < 0)
2275                 return ret;
2276
2277         *ptr = val * 1000;
2278
2279         return cnt;
2280 }
2281
2282 static atomic_t tracing_reader;
2283
2284 static int tracing_open_pipe(struct inode *inode, struct file *filp)
2285 {
2286         struct trace_iterator *iter;
2287         int cpu;
2288
2289         if (tracing_disabled)
2290                 return -ENODEV;
2291
2292         /* We only allow for reader of the pipe */
2293         if (atomic_inc_return(&tracing_reader) != 1) {
2294                 atomic_dec(&tracing_reader);
2295                 return -EBUSY;
2296         }
2297
2298         /* create a buffer to store the information to pass to userspace */
2299         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2300         if (!iter)
2301                 return -ENOMEM;
2302
2303         mutex_lock(&trace_types_lock);
2304         iter->tr = &global_trace;
2305         iter->trace = current_trace;
2306         filp->private_data = iter;
2307
2308         for_each_tracing_cpu(cpu) {
2309                 iter->buffer_iter[cpu] =
2310                         ring_buffer_read_start(iter->tr->buffer, cpu);
2311                 if (!iter->buffer_iter[cpu])
2312                         goto fail_buffer;
2313         }
2314
2315         if (iter->trace->pipe_open)
2316                 iter->trace->pipe_open(iter);
2317         mutex_unlock(&trace_types_lock);
2318
2319         return 0;
2320
2321  fail_buffer:
2322         for_each_tracing_cpu(cpu) {
2323                 if (iter->buffer_iter[cpu])
2324                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2325         }
2326         mutex_unlock(&trace_types_lock);
2327
2328         return -ENOMEM;
2329 }
2330
2331 static int tracing_release_pipe(struct inode *inode, struct file *file)
2332 {
2333         struct trace_iterator *iter = file->private_data;
2334         int cpu;
2335
2336         for_each_tracing_cpu(cpu) {
2337                 if (iter->buffer_iter[cpu])
2338                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2339         }
2340         kfree(iter);
2341         atomic_dec(&tracing_reader);
2342
2343         return 0;
2344 }
2345
2346 static unsigned int
2347 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2348 {
2349         struct trace_iterator *iter = filp->private_data;
2350
2351         if (trace_flags & TRACE_ITER_BLOCK) {
2352                 /*
2353                  * Always select as readable when in blocking mode
2354                  */
2355                 return POLLIN | POLLRDNORM;
2356         } else {
2357                 if (!trace_empty(iter))
2358                         return POLLIN | POLLRDNORM;
2359                 poll_wait(filp, &trace_wait, poll_table);
2360                 if (!trace_empty(iter))
2361                         return POLLIN | POLLRDNORM;
2362
2363                 return 0;
2364         }
2365 }
2366
2367 /*
2368  * Consumer reader.
2369  */
2370 static ssize_t
2371 tracing_read_pipe(struct file *filp, char __user *ubuf,
2372                   size_t cnt, loff_t *ppos)
2373 {
2374         struct trace_iterator *iter = filp->private_data;
2375         unsigned long flags;
2376 #ifdef CONFIG_FTRACE
2377         int ftrace_save;
2378 #endif
2379         ssize_t sret;
2380
2381         /* return any leftover data */
2382         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2383         if (sret != -EBUSY)
2384                 return sret;
2385         sret = 0;
2386
2387         trace_seq_reset(&iter->seq);
2388
2389         mutex_lock(&trace_types_lock);
2390         if (iter->trace->read) {
2391                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2392                 if (sret)
2393                         goto out;
2394         }
2395
2396         while (trace_empty(iter)) {
2397
2398                 if ((filp->f_flags & O_NONBLOCK)) {
2399                         sret = -EAGAIN;
2400                         goto out;
2401                 }
2402
2403                 /*
2404                  * This is a make-shift waitqueue. The reason we don't use
2405                  * an actual wait queue is because:
2406                  *  1) we only ever have one waiter
2407                  *  2) the tracing, traces all functions, we don't want
2408                  *     the overhead of calling wake_up and friends
2409                  *     (and tracing them too)
2410                  *     Anyway, this is really very primitive wakeup.
2411                  */
2412                 set_current_state(TASK_INTERRUPTIBLE);
2413                 iter->tr->waiter = current;
2414
2415                 mutex_unlock(&trace_types_lock);
2416
2417                 /* sleep for 100 msecs, and try again. */
2418                 schedule_timeout(HZ/10);
2419
2420                 mutex_lock(&trace_types_lock);
2421
2422                 iter->tr->waiter = NULL;
2423
2424                 if (signal_pending(current)) {
2425                         sret = -EINTR;
2426                         goto out;
2427                 }
2428
2429                 if (iter->trace != current_trace)
2430                         goto out;
2431
2432                 /*
2433                  * We block until we read something and tracing is disabled.
2434                  * We still block if tracing is disabled, but we have never
2435                  * read anything. This allows a user to cat this file, and
2436                  * then enable tracing. But after we have read something,
2437                  * we give an EOF when tracing is again disabled.
2438                  *
2439                  * iter->pos will be 0 if we haven't read anything.
2440                  */
2441                 if (!tracer_enabled && iter->pos)
2442                         break;
2443
2444                 continue;
2445         }
2446
2447         /* stop when tracing is finished */
2448         if (trace_empty(iter))
2449                 goto out;
2450
2451         if (cnt >= PAGE_SIZE)
2452                 cnt = PAGE_SIZE - 1;
2453
2454         /* reset all but tr, trace, and overruns */
2455         memset(&iter->seq, 0,
2456                sizeof(struct trace_iterator) -
2457                offsetof(struct trace_iterator, seq));
2458         iter->pos = -1;
2459
2460         /*
2461          * We need to stop all tracing on all CPUS to read the
2462          * the next buffer. This is a bit expensive, but is
2463          * not done often. We fill all what we can read,
2464          * and then release the locks again.
2465          */
2466
2467         local_irq_disable();
2468 #ifdef CONFIG_FTRACE
2469         ftrace_save = ftrace_enabled;
2470         ftrace_enabled = 0;
2471 #endif
2472         smp_wmb();
2473         ring_buffer_lock(iter->tr->buffer, &flags);
2474
2475         while (find_next_entry_inc(iter) != NULL) {
2476                 int ret;
2477                 int len = iter->seq.len;
2478
2479                 ret = print_trace_line(iter);
2480                 if (!ret) {
2481                         /* don't print partial lines */
2482                         iter->seq.len = len;
2483                         break;
2484                 }
2485
2486                 trace_consume(iter);
2487
2488                 if (iter->seq.len >= cnt)
2489                         break;
2490         }
2491
2492         ring_buffer_unlock(iter->tr->buffer, flags);
2493 #ifdef CONFIG_FTRACE
2494         ftrace_enabled = ftrace_save;
2495 #endif
2496         local_irq_enable();
2497
2498         /* Now copy what we have to the user */
2499         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2500         if (iter->seq.readpos >= iter->seq.len)
2501                 trace_seq_reset(&iter->seq);
2502         if (sret == -EBUSY)
2503                 sret = 0;
2504
2505 out:
2506         mutex_unlock(&trace_types_lock);
2507
2508         return sret;
2509 }
2510
2511 static ssize_t
2512 tracing_entries_read(struct file *filp, char __user *ubuf,
2513                      size_t cnt, loff_t *ppos)
2514 {
2515         struct trace_array *tr = filp->private_data;
2516         char buf[64];
2517         int r;
2518
2519         r = sprintf(buf, "%lu\n", tr->entries);
2520         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2521 }
2522
2523 static ssize_t
2524 tracing_entries_write(struct file *filp, const char __user *ubuf,
2525                       size_t cnt, loff_t *ppos)
2526 {
2527         unsigned long val;
2528         char buf[64];
2529         int ret;
2530         struct trace_array *tr = filp->private_data;
2531
2532         if (cnt >= sizeof(buf))
2533                 return -EINVAL;
2534
2535         if (copy_from_user(&buf, ubuf, cnt))
2536                 return -EFAULT;
2537
2538         buf[cnt] = 0;
2539
2540         ret = strict_strtoul(buf, 10, &val);
2541         if (ret < 0)
2542                 return ret;
2543
2544         /* must have at least 1 entry */
2545         if (!val)
2546                 return -EINVAL;
2547
2548         mutex_lock(&trace_types_lock);
2549
2550         if (tr->ctrl) {
2551                 cnt = -EBUSY;
2552                 pr_info("ftrace: please disable tracing"
2553                         " before modifying buffer size\n");
2554                 goto out;
2555         }
2556
2557         if (val != global_trace.entries) {
2558                 ret = ring_buffer_resize(global_trace.buffer, val);
2559                 if (ret < 0) {
2560                         cnt = ret;
2561                         goto out;
2562                 }
2563
2564                 ret = ring_buffer_resize(max_tr.buffer, val);
2565                 if (ret < 0) {
2566                         int r;
2567                         cnt = ret;
2568                         r = ring_buffer_resize(global_trace.buffer,
2569                                                global_trace.entries);
2570                         if (r < 0) {
2571                                 /* AARGH! We are left with different
2572                                  * size max buffer!!!! */
2573                                 WARN_ON(1);
2574                                 tracing_disabled = 1;
2575                         }
2576                         goto out;
2577                 }
2578
2579                 global_trace.entries = val;
2580         }
2581
2582         filp->f_pos += cnt;
2583
2584         /* If check pages failed, return ENOMEM */
2585         if (tracing_disabled)
2586                 cnt = -ENOMEM;
2587  out:
2588         max_tr.entries = global_trace.entries;
2589         mutex_unlock(&trace_types_lock);
2590
2591         return cnt;
2592 }
2593
2594 static int mark_printk(const char *fmt, ...)
2595 {
2596         int ret;
2597         va_list args;
2598         va_start(args, fmt);
2599         ret = trace_vprintk(0, fmt, args);
2600         va_end(args);
2601         return ret;
2602 }
2603
2604 static ssize_t
2605 tracing_mark_write(struct file *filp, const char __user *ubuf,
2606                                         size_t cnt, loff_t *fpos)
2607 {
2608         char *buf;
2609         char *end;
2610         struct trace_array *tr = &global_trace;
2611
2612         if (!tr->ctrl || tracing_disabled)
2613                 return -EINVAL;
2614
2615         if (cnt > TRACE_BUF_SIZE)
2616                 cnt = TRACE_BUF_SIZE;
2617
2618         buf = kmalloc(cnt + 1, GFP_KERNEL);
2619         if (buf == NULL)
2620                 return -ENOMEM;
2621
2622         if (copy_from_user(buf, ubuf, cnt)) {
2623                 kfree(buf);
2624                 return -EFAULT;
2625         }
2626
2627         /* Cut from the first nil or newline. */
2628         buf[cnt] = '\0';
2629         end = strchr(buf, '\n');
2630         if (end)
2631                 *end = '\0';
2632
2633         cnt = mark_printk("%s\n", buf);
2634         kfree(buf);
2635         *fpos += cnt;
2636
2637         return cnt;
2638 }
2639
2640 static struct file_operations tracing_max_lat_fops = {
2641         .open           = tracing_open_generic,
2642         .read           = tracing_max_lat_read,
2643         .write          = tracing_max_lat_write,
2644 };
2645
2646 static struct file_operations tracing_ctrl_fops = {
2647         .open           = tracing_open_generic,
2648         .read           = tracing_ctrl_read,
2649         .write          = tracing_ctrl_write,
2650 };
2651
2652 static struct file_operations set_tracer_fops = {
2653         .open           = tracing_open_generic,
2654         .read           = tracing_set_trace_read,
2655         .write          = tracing_set_trace_write,
2656 };
2657
2658 static struct file_operations tracing_pipe_fops = {
2659         .open           = tracing_open_pipe,
2660         .poll           = tracing_poll_pipe,
2661         .read           = tracing_read_pipe,
2662         .release        = tracing_release_pipe,
2663 };
2664
2665 static struct file_operations tracing_entries_fops = {
2666         .open           = tracing_open_generic,
2667         .read           = tracing_entries_read,
2668         .write          = tracing_entries_write,
2669 };
2670
2671 static struct file_operations tracing_mark_fops = {
2672         .open           = tracing_open_generic,
2673         .write          = tracing_mark_write,
2674 };
2675
2676 #ifdef CONFIG_DYNAMIC_FTRACE
2677
2678 static ssize_t
2679 tracing_read_long(struct file *filp, char __user *ubuf,
2680                   size_t cnt, loff_t *ppos)
2681 {
2682         unsigned long *p = filp->private_data;
2683         char buf[64];
2684         int r;
2685
2686         r = sprintf(buf, "%ld\n", *p);
2687
2688         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2689 }
2690
2691 static struct file_operations tracing_read_long_fops = {
2692         .open           = tracing_open_generic,
2693         .read           = tracing_read_long,
2694 };
2695 #endif
2696
2697 static struct dentry *d_tracer;
2698
2699 struct dentry *tracing_init_dentry(void)
2700 {
2701         static int once;
2702
2703         if (d_tracer)
2704                 return d_tracer;
2705
2706         d_tracer = debugfs_create_dir("tracing", NULL);
2707
2708         if (!d_tracer && !once) {
2709                 once = 1;
2710                 pr_warning("Could not create debugfs directory 'tracing'\n");
2711                 return NULL;
2712         }
2713
2714         return d_tracer;
2715 }
2716
2717 #ifdef CONFIG_FTRACE_SELFTEST
2718 /* Let selftest have access to static functions in this file */
2719 #include "trace_selftest.c"
2720 #endif
2721
2722 static __init int tracer_init_debugfs(void)
2723 {
2724         struct dentry *d_tracer;
2725         struct dentry *entry;
2726
2727         d_tracer = tracing_init_dentry();
2728
2729         entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
2730                                     &global_trace, &tracing_ctrl_fops);
2731         if (!entry)
2732                 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
2733
2734         entry = debugfs_create_file("iter_ctrl", 0644, d_tracer,
2735                                     NULL, &tracing_iter_fops);
2736         if (!entry)
2737                 pr_warning("Could not create debugfs 'iter_ctrl' entry\n");
2738
2739         entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
2740                                     NULL, &tracing_cpumask_fops);
2741         if (!entry)
2742                 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
2743
2744         entry = debugfs_create_file("latency_trace", 0444, d_tracer,
2745                                     &global_trace, &tracing_lt_fops);
2746         if (!entry)
2747                 pr_warning("Could not create debugfs 'latency_trace' entry\n");
2748
2749         entry = debugfs_create_file("trace", 0444, d_tracer,
2750                                     &global_trace, &tracing_fops);
2751         if (!entry)
2752                 pr_warning("Could not create debugfs 'trace' entry\n");
2753
2754         entry = debugfs_create_file("available_tracers", 0444, d_tracer,
2755                                     &global_trace, &show_traces_fops);
2756         if (!entry)
2757                 pr_warning("Could not create debugfs 'available_tracers' entry\n");
2758
2759         entry = debugfs_create_file("current_tracer", 0444, d_tracer,
2760                                     &global_trace, &set_tracer_fops);
2761         if (!entry)
2762                 pr_warning("Could not create debugfs 'current_tracer' entry\n");
2763
2764         entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
2765                                     &tracing_max_latency,
2766                                     &tracing_max_lat_fops);
2767         if (!entry)
2768                 pr_warning("Could not create debugfs "
2769                            "'tracing_max_latency' entry\n");
2770
2771         entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
2772                                     &tracing_thresh, &tracing_max_lat_fops);
2773         if (!entry)
2774                 pr_warning("Could not create debugfs "
2775                            "'tracing_thresh' entry\n");
2776         entry = debugfs_create_file("README", 0644, d_tracer,
2777                                     NULL, &tracing_readme_fops);
2778         if (!entry)
2779                 pr_warning("Could not create debugfs 'README' entry\n");
2780
2781         entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
2782                                     NULL, &tracing_pipe_fops);
2783         if (!entry)
2784                 pr_warning("Could not create debugfs "
2785                            "'trace_pipe' entry\n");
2786
2787         entry = debugfs_create_file("trace_entries", 0644, d_tracer,
2788                                     &global_trace, &tracing_entries_fops);
2789         if (!entry)
2790                 pr_warning("Could not create debugfs "
2791                            "'trace_entries' entry\n");
2792
2793         entry = debugfs_create_file("trace_marker", 0220, d_tracer,
2794                                     NULL, &tracing_mark_fops);
2795         if (!entry)
2796                 pr_warning("Could not create debugfs "
2797                            "'trace_marker' entry\n");
2798
2799 #ifdef CONFIG_DYNAMIC_FTRACE
2800         entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
2801                                     &ftrace_update_tot_cnt,
2802                                     &tracing_read_long_fops);
2803         if (!entry)
2804                 pr_warning("Could not create debugfs "
2805                            "'dyn_ftrace_total_info' entry\n");
2806 #endif
2807 #ifdef CONFIG_SYSPROF_TRACER
2808         init_tracer_sysprof_debugfs(d_tracer);
2809 #endif
2810         return 0;
2811 }
2812
2813 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2814 {
2815         static DEFINE_SPINLOCK(trace_buf_lock);
2816         static char trace_buf[TRACE_BUF_SIZE];
2817
2818         struct ring_buffer_event *event;
2819         struct trace_array *tr = &global_trace;
2820         struct trace_array_cpu *data;
2821         struct trace_entry *entry;
2822         unsigned long flags, irq_flags;
2823         long disabled;
2824         int cpu, len = 0, write, written = 0;
2825
2826         if (!tr->ctrl || tracing_disabled)
2827                 return 0;
2828
2829         local_irq_save(flags);
2830         cpu = raw_smp_processor_id();
2831         data = tr->data[cpu];
2832         disabled = atomic_inc_return(&data->disabled);
2833
2834         if (unlikely(disabled != 1))
2835                 goto out;
2836
2837         spin_lock(&trace_buf_lock);
2838         len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
2839
2840         len = min(len, TRACE_BUF_SIZE-1);
2841         trace_buf[len] = 0;
2842
2843         event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
2844                                          &irq_flags);
2845         if (!event)
2846                 goto out_unlock;
2847         entry   = ring_buffer_event_data(event);
2848         tracing_generic_entry_update(entry, flags);
2849         entry->type                     = TRACE_PRINT;
2850         entry->field.print.ip           = ip;
2851
2852         write = min(len, (int)(TRACE_PRINT_BUF_SIZE-1));
2853
2854         memcpy(&entry->field.print.buf, trace_buf, write);
2855         entry->field.print.buf[write] = 0;
2856         written = write;
2857         ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
2858
2859         if (written != len)
2860                 entry->field.flags |= TRACE_FLAG_CONT;
2861
2862         while (written != len) {
2863                 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
2864                                                  &irq_flags);
2865                 if (!event)
2866                         goto out_unlock;
2867                 entry   = ring_buffer_event_data(event);
2868
2869                 entry->type = TRACE_CONT;
2870                 write = min(len - written, (int)(TRACE_CONT_BUF_SIZE-1));
2871                 memcpy(&entry->cont.buf, trace_buf+written, write);
2872                 entry->cont.buf[write] = 0;
2873                 written += write;
2874                 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
2875         }
2876
2877  out_unlock:
2878         spin_unlock(&trace_buf_lock);
2879
2880  out:
2881         atomic_dec(&data->disabled);
2882         local_irq_restore(flags);
2883
2884         return len;
2885 }
2886 EXPORT_SYMBOL_GPL(trace_vprintk);
2887
2888 int __ftrace_printk(unsigned long ip, const char *fmt, ...)
2889 {
2890         int ret;
2891         va_list ap;
2892
2893         if (!(trace_flags & TRACE_ITER_PRINTK))
2894                 return 0;
2895
2896         va_start(ap, fmt);
2897         ret = trace_vprintk(ip, fmt, ap);
2898         va_end(ap);
2899         return ret;
2900 }
2901 EXPORT_SYMBOL_GPL(__ftrace_printk);
2902
2903 static int trace_panic_handler(struct notifier_block *this,
2904                                unsigned long event, void *unused)
2905 {
2906         ftrace_dump();
2907         return NOTIFY_OK;
2908 }
2909
2910 static struct notifier_block trace_panic_notifier = {
2911         .notifier_call  = trace_panic_handler,
2912         .next           = NULL,
2913         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
2914 };
2915
2916 static int trace_die_handler(struct notifier_block *self,
2917                              unsigned long val,
2918                              void *data)
2919 {
2920         switch (val) {
2921         case DIE_OOPS:
2922                 ftrace_dump();
2923                 break;
2924         default:
2925                 break;
2926         }
2927         return NOTIFY_OK;
2928 }
2929
2930 static struct notifier_block trace_die_notifier = {
2931         .notifier_call = trace_die_handler,
2932         .priority = 200
2933 };
2934
2935 /*
2936  * printk is set to max of 1024, we really don't need it that big.
2937  * Nothing should be printing 1000 characters anyway.
2938  */
2939 #define TRACE_MAX_PRINT         1000
2940
2941 /*
2942  * Define here KERN_TRACE so that we have one place to modify
2943  * it if we decide to change what log level the ftrace dump
2944  * should be at.
2945  */
2946 #define KERN_TRACE              KERN_INFO
2947
2948 static void
2949 trace_printk_seq(struct trace_seq *s)
2950 {
2951         /* Probably should print a warning here. */
2952         if (s->len >= 1000)
2953                 s->len = 1000;
2954
2955         /* should be zero ended, but we are paranoid. */
2956         s->buffer[s->len] = 0;
2957
2958         printk(KERN_TRACE "%s", s->buffer);
2959
2960         trace_seq_reset(s);
2961 }
2962
2963
2964 void ftrace_dump(void)
2965 {
2966         static DEFINE_SPINLOCK(ftrace_dump_lock);
2967         /* use static because iter can be a bit big for the stack */
2968         static struct trace_iterator iter;
2969         static cpumask_t mask;
2970         static int dump_ran;
2971         unsigned long flags, irq_flags;
2972         int cnt = 0;
2973
2974         /* only one dump */
2975         spin_lock_irqsave(&ftrace_dump_lock, flags);
2976         if (dump_ran)
2977                 goto out;
2978
2979         dump_ran = 1;
2980
2981         /* No turning back! */
2982         ftrace_kill_atomic();
2983
2984         printk(KERN_TRACE "Dumping ftrace buffer:\n");
2985
2986         iter.tr = &global_trace;
2987         iter.trace = current_trace;
2988
2989         /*
2990          * We need to stop all tracing on all CPUS to read the
2991          * the next buffer. This is a bit expensive, but is
2992          * not done often. We fill all what we can read,
2993          * and then release the locks again.
2994          */
2995
2996         cpus_clear(mask);
2997
2998         ring_buffer_lock(iter.tr->buffer, &irq_flags);
2999
3000         while (!trace_empty(&iter)) {
3001
3002                 if (!cnt)
3003                         printk(KERN_TRACE "---------------------------------\n");
3004
3005                 cnt++;
3006
3007                 /* reset all but tr, trace, and overruns */
3008                 memset(&iter.seq, 0,
3009                        sizeof(struct trace_iterator) -
3010                        offsetof(struct trace_iterator, seq));
3011                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
3012                 iter.pos = -1;
3013
3014                 if (find_next_entry_inc(&iter) != NULL) {
3015                         print_trace_line(&iter);
3016                         trace_consume(&iter);
3017                 }
3018
3019                 trace_printk_seq(&iter.seq);
3020         }
3021
3022         if (!cnt)
3023                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
3024         else
3025                 printk(KERN_TRACE "---------------------------------\n");
3026
3027         ring_buffer_unlock(iter.tr->buffer, irq_flags);
3028
3029  out:
3030         spin_unlock_irqrestore(&ftrace_dump_lock, flags);
3031 }
3032
3033 __init static int tracer_alloc_buffers(void)
3034 {
3035         struct trace_array_cpu *data;
3036         int i;
3037
3038         /* TODO: make the number of buffers hot pluggable with CPUS */
3039         tracing_buffer_mask = cpu_possible_map;
3040
3041         global_trace.buffer = ring_buffer_alloc(trace_buf_size,
3042                                                    TRACE_BUFFER_FLAGS);
3043         if (!global_trace.buffer) {
3044                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
3045                 WARN_ON(1);
3046                 return 0;
3047         }
3048         global_trace.entries = ring_buffer_size(global_trace.buffer);
3049
3050 #ifdef CONFIG_TRACER_MAX_TRACE
3051         max_tr.buffer = ring_buffer_alloc(trace_buf_size,
3052                                              TRACE_BUFFER_FLAGS);
3053         if (!max_tr.buffer) {
3054                 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
3055                 WARN_ON(1);
3056                 ring_buffer_free(global_trace.buffer);
3057                 return 0;
3058         }
3059         max_tr.entries = ring_buffer_size(max_tr.buffer);
3060         WARN_ON(max_tr.entries != global_trace.entries);
3061 #endif
3062
3063         /* Allocate the first page for all buffers */
3064         for_each_tracing_cpu(i) {
3065                 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
3066                 max_tr.data[i] = &per_cpu(max_data, i);
3067         }
3068
3069         trace_init_cmdlines();
3070
3071         register_tracer(&nop_trace);
3072 #ifdef CONFIG_BOOT_TRACER
3073         register_tracer(&boot_tracer);
3074         current_trace = &boot_tracer;
3075         current_trace->init(&global_trace);
3076 #else
3077         current_trace = &nop_trace;
3078 #endif
3079
3080         /* All seems OK, enable tracing */
3081         global_trace.ctrl = tracer_enabled;
3082         tracing_disabled = 0;
3083
3084         atomic_notifier_chain_register(&panic_notifier_list,
3085                                        &trace_panic_notifier);
3086
3087         register_die_notifier(&trace_die_notifier);
3088
3089         return 0;
3090 }
3091 early_initcall(tracer_alloc_buffers);
3092 fs_initcall(tracer_init_debugfs);