kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/debugfs.h>
  21 #include <linux/hardirq.h>
  22 #include <linux/kthread.h>
  23 #include <linux/uaccess.h>
  24 #include <linux/kprobes.h>
  25 #include <linux/ftrace.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/ctype.h>
  28 #include <linux/list.h>
  29
  30 #include <asm/ftrace.h>
  31
  32 #include "trace.h"
  33
  34 #define FTRACE_WARN_ON(cond)                    \
  35         do {                                    \
  36                 if (WARN_ON(cond))              \
  37                         ftrace_kill();          \
  38         } while (0)
  39
  40 #define FTRACE_WARN_ON_ONCE(cond)               \
  41         do {                                    \
  42                 if (WARN_ON_ONCE(cond))         \
  43                         ftrace_kill();          \
  44         } while (0)
  45
  46 /* ftrace_enabled is a method to turn ftrace on or off */
  47 int ftrace_enabled __read_mostly;
  48 static int last_ftrace_enabled;
  49
  50 /* ftrace_pid_trace >= 0 will only trace threads with this pid */
  51 static int ftrace_pid_trace = -1;
  52
  53 /* Quick disabling of function tracer. */
  54 int function_trace_stop;
  55
  56 /*
  57  * ftrace_disabled is set when an anomaly is discovered.
  58  * ftrace_disabled is much stronger than ftrace_enabled.
  59  */
  60 static int ftrace_disabled __read_mostly;
  61
  62 static DEFINE_SPINLOCK(ftrace_lock);
  63 static DEFINE_MUTEX(ftrace_sysctl_lock);
  64 static DEFINE_MUTEX(ftrace_start_lock);
  65
  66 static struct ftrace_ops ftrace_list_end __read_mostly =
  67 {
  68         .func = ftrace_stub,
  69 };
  70
  71 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  72 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  73 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  74 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  75
  76 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  77 {
  78         struct ftrace_ops *op = ftrace_list;
  79
  80         /* in case someone actually ports this to alpha! */
  81         read_barrier_depends();
  82
  83         while (op != &ftrace_list_end) {
  84                 /* silly alpha */
  85                 read_barrier_depends();
  86                 op->func(ip, parent_ip);
  87                 op = op->next;
  88         };
  89 }
  90
  91 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
  92 {
  93         if (current->pid != ftrace_pid_trace)
  94                 return;
  95
  96         ftrace_pid_function(ip, parent_ip);
  97 }
  98
  99 static void set_ftrace_pid_function(ftrace_func_t func)
 100 {
 101         /* do not set ftrace_pid_function to itself! */
 102         if (func != ftrace_pid_func)
 103                 ftrace_pid_function = func;
 104 }
 105
 106 /**
 107  * clear_ftrace_function - reset the ftrace function
 108  *
 109  * This NULLs the ftrace function and in essence stops
 110  * tracing.  There may be lag
 111  */
 112 void clear_ftrace_function(void)
 113 {
 114         ftrace_trace_function = ftrace_stub;
 115         __ftrace_trace_function = ftrace_stub;
 116         ftrace_pid_function = ftrace_stub;
 117 }
 118
 119 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 120 /*
 121  * For those archs that do not test ftrace_trace_stop in their
 122  * mcount call site, we need to do it from C.
 123  */
 124 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 125 {
 126         if (function_trace_stop)
 127                 return;
 128
 129         __ftrace_trace_function(ip, parent_ip);
 130 }
 131 #endif
 132
 133 static int __register_ftrace_function(struct ftrace_ops *ops)
 134 {
 135         /* should not be called from interrupt context */
 136         spin_lock(&ftrace_lock);
 137
 138         ops->next = ftrace_list;
 139         /*
 140          * We are entering ops into the ftrace_list but another
 141          * CPU might be walking that list. We need to make sure
 142          * the ops->next pointer is valid before another CPU sees
 143          * the ops pointer included into the ftrace_list.
 144          */
 145         smp_wmb();
 146         ftrace_list = ops;
 147
 148         if (ftrace_enabled) {
 149                 ftrace_func_t func;
 150
 151                 if (ops->next == &ftrace_list_end)
 152                         func = ops->func;
 153                 else
 154                         func = ftrace_list_func;
 155
 156                 if (ftrace_pid_trace >= 0) {
 157                         set_ftrace_pid_function(func);
 158                         func = ftrace_pid_func;
 159                 }
 160
 161                 /*
 162                  * For one func, simply call it directly.
 163                  * For more than one func, call the chain.
 164                  */
 165 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 166                 ftrace_trace_function = func;
 167 #else
 168                 __ftrace_trace_function = func;
 169                 ftrace_trace_function = ftrace_test_stop_func;
 170 #endif
 171         }
 172
 173         spin_unlock(&ftrace_lock);
 174
 175         return 0;
 176 }
 177
 178 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 179 {
 180         struct ftrace_ops **p;
 181         int ret = 0;
 182
 183         /* should not be called from interrupt context */
 184         spin_lock(&ftrace_lock);
 185
 186         /*
 187          * If we are removing the last function, then simply point
 188          * to the ftrace_stub.
 189          */
 190         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 191                 ftrace_trace_function = ftrace_stub;
 192                 ftrace_list = &ftrace_list_end;
 193                 goto out;
 194         }
 195
 196         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 197                 if (*p == ops)
 198                         break;
 199
 200         if (*p != ops) {
 201                 ret = -1;
 202                 goto out;
 203         }
 204
 205         *p = (*p)->next;
 206
 207         if (ftrace_enabled) {
 208                 /* If we only have one func left, then call that directly */
 209                 if (ftrace_list->next == &ftrace_list_end) {
 210                         ftrace_func_t func = ftrace_list->func;
 211
 212                         if (ftrace_pid_trace >= 0) {
 213                                 set_ftrace_pid_function(func);
 214                                 func = ftrace_pid_func;
 215                         }
 216 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 217                         ftrace_trace_function = func;
 218 #else
 219                         __ftrace_trace_function = func;
 220 #endif
 221                 }
 222         }
 223
 224  out:
 225         spin_unlock(&ftrace_lock);
 226
 227         return ret;
 228 }
 229
 230 static void ftrace_update_pid_func(void)
 231 {
 232         ftrace_func_t func;
 233
 234         /* should not be called from interrupt context */
 235         spin_lock(&ftrace_lock);
 236
 237         if (ftrace_trace_function == ftrace_stub)
 238                 goto out;
 239
 240         func = ftrace_trace_function;
 241
 242         if (ftrace_pid_trace >= 0) {
 243                 set_ftrace_pid_function(func);
 244                 func = ftrace_pid_func;
 245         } else {
 246                 if (func != ftrace_pid_func)
 247                         goto out;
 248
 249                 set_ftrace_pid_function(func);
 250         }
 251
 252 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 253         ftrace_trace_function = func;
 254 #else
 255         __ftrace_trace_function = func;
 256 #endif
 257
 258  out:
 259         spin_unlock(&ftrace_lock);
 260 }
 261
 262 #ifdef CONFIG_DYNAMIC_FTRACE
 263 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 264 # error Dynamic ftrace depends on MCOUNT_RECORD
 265 #endif
 266
 267 /*
 268  * Since MCOUNT_ADDR may point to mcount itself, we do not want
 269  * to get it confused by reading a reference in the code as we
 270  * are parsing on objcopy output of text. Use a variable for
 271  * it instead.
 272  */
 273 static unsigned long mcount_addr = MCOUNT_ADDR;
 274
 275 enum {
 276         FTRACE_ENABLE_CALLS             = (1 << 0),
 277         FTRACE_DISABLE_CALLS            = (1 << 1),
 278         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 279         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 280         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 281         FTRACE_START_FUNC_RET           = (1 << 5),
 282         FTRACE_STOP_FUNC_RET            = (1 << 6),
 283 };
 284
 285 static int ftrace_filtered;
 286
 287 static LIST_HEAD(ftrace_new_addrs);
 288
 289 static DEFINE_MUTEX(ftrace_regex_lock);
 290
 291 struct ftrace_page {
 292         struct ftrace_page      *next;
 293         unsigned long           index;
 294         struct dyn_ftrace       records[];
 295 };
 296
 297 #define ENTRIES_PER_PAGE \
 298   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 299
 300 /* estimate from running different kernels */
 301 #define NR_TO_INIT              10000
 302
 303 static struct ftrace_page       *ftrace_pages_start;
 304 static struct ftrace_page       *ftrace_pages;
 305
 306 static struct dyn_ftrace *ftrace_free_records;
 307
 308
 309 #ifdef CONFIG_KPROBES
 310
 311 static int frozen_record_count;
 312
 313 static inline void freeze_record(struct dyn_ftrace *rec)
 314 {
 315         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 316                 rec->flags |= FTRACE_FL_FROZEN;
 317                 frozen_record_count++;
 318         }
 319 }
 320
 321 static inline void unfreeze_record(struct dyn_ftrace *rec)
 322 {
 323         if (rec->flags & FTRACE_FL_FROZEN) {
 324                 rec->flags &= ~FTRACE_FL_FROZEN;
 325                 frozen_record_count--;
 326         }
 327 }
 328
 329 static inline int record_frozen(struct dyn_ftrace *rec)
 330 {
 331         return rec->flags & FTRACE_FL_FROZEN;
 332 }
 333 #else
 334 # define freeze_record(rec)                     ({ 0; })
 335 # define unfreeze_record(rec)                   ({ 0; })
 336 # define record_frozen(rec)                     ({ 0; })
 337 #endif /* CONFIG_KPROBES */
 338
 339 static void ftrace_free_rec(struct dyn_ftrace *rec)
 340 {
 341         rec->ip = (unsigned long)ftrace_free_records;
 342         ftrace_free_records = rec;
 343         rec->flags |= FTRACE_FL_FREE;
 344 }
 345
 346 void ftrace_release(void *start, unsigned long size)
 347 {
 348         struct dyn_ftrace *rec;
 349         struct ftrace_page *pg;
 350         unsigned long s = (unsigned long)start;
 351         unsigned long e = s + size;
 352         int i;
 353
 354         if (ftrace_disabled || !start)
 355                 return;
 356
 357         /* should not be called from interrupt context */
 358         spin_lock(&ftrace_lock);
 359
 360         for (pg = ftrace_pages_start; pg; pg = pg->next) {
 361                 for (i = 0; i < pg->index; i++) {
 362                         rec = &pg->records[i];
 363
 364                         if ((rec->ip >= s) && (rec->ip < e))
 365                                 ftrace_free_rec(rec);
 366                 }
 367         }
 368         spin_unlock(&ftrace_lock);
 369 }
 370
 371 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 372 {
 373         struct dyn_ftrace *rec;
 374
 375         /* First check for freed records */
 376         if (ftrace_free_records) {
 377                 rec = ftrace_free_records;
 378
 379                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 380                         FTRACE_WARN_ON_ONCE(1);
 381                         ftrace_free_records = NULL;
 382                         return NULL;
 383                 }
 384
 385                 ftrace_free_records = (void *)rec->ip;
 386                 memset(rec, 0, sizeof(*rec));
 387                 return rec;
 388         }
 389
 390         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 391                 if (!ftrace_pages->next) {
 392                         /* allocate another page */
 393                         ftrace_pages->next =
 394                                 (void *)get_zeroed_page(GFP_KERNEL);
 395                         if (!ftrace_pages->next)
 396                                 return NULL;
 397                 }
 398                 ftrace_pages = ftrace_pages->next;
 399         }
 400
 401         return &ftrace_pages->records[ftrace_pages->index++];
 402 }
 403
 404 static struct dyn_ftrace *
 405 ftrace_record_ip(unsigned long ip)
 406 {
 407         struct dyn_ftrace *rec;
 408
 409         if (ftrace_disabled)
 410                 return NULL;
 411
 412         rec = ftrace_alloc_dyn_node(ip);
 413         if (!rec)
 414                 return NULL;
 415
 416         rec->ip = ip;
 417
 418         list_add(&rec->list, &ftrace_new_addrs);
 419
 420         return rec;
 421 }
 422
 423 static void print_ip_ins(const char *fmt, unsigned char *p)
 424 {
 425         int i;
 426
 427         printk(KERN_CONT "%s", fmt);
 428
 429         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 430                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 431 }
 432
 433 static void ftrace_bug(int failed, unsigned long ip)
 434 {
 435         switch (failed) {
 436         case -EFAULT:
 437                 FTRACE_WARN_ON_ONCE(1);
 438                 pr_info("ftrace faulted on modifying ");
 439                 print_ip_sym(ip);
 440                 break;
 441         case -EINVAL:
 442                 FTRACE_WARN_ON_ONCE(1);
 443                 pr_info("ftrace failed to modify ");
 444                 print_ip_sym(ip);
 445                 print_ip_ins(" actual: ", (unsigned char *)ip);
 446                 printk(KERN_CONT "\n");
 447                 break;
 448         case -EPERM:
 449                 FTRACE_WARN_ON_ONCE(1);
 450                 pr_info("ftrace faulted on writing ");
 451                 print_ip_sym(ip);
 452                 break;
 453         default:
 454                 FTRACE_WARN_ON_ONCE(1);
 455                 pr_info("ftrace faulted on unknown error ");
 456                 print_ip_sym(ip);
 457         }
 458 }
 459
 460
 461 static int
 462 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 463 {
 464         unsigned long ip, fl;
 465         unsigned long ftrace_addr;
 466
 467         ftrace_addr = (unsigned long)ftrace_caller;
 468
 469         ip = rec->ip;
 470
 471         /*
 472          * If this record is not to be traced and
 473          * it is not enabled then do nothing.
 474          *
 475          * If this record is not to be traced and
 476          * it is enabled then disabled it.
 477          *
 478          */
 479         if (rec->flags & FTRACE_FL_NOTRACE) {
 480                 if (rec->flags & FTRACE_FL_ENABLED)
 481                         rec->flags &= ~FTRACE_FL_ENABLED;
 482                 else
 483                         return 0;
 484
 485         } else if (ftrace_filtered && enable) {
 486                 /*
 487                  * Filtering is on:
 488                  */
 489
 490                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
 491
 492                 /* Record is filtered and enabled, do nothing */
 493                 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
 494                         return 0;
 495
 496                 /* Record is not filtered and is not enabled do nothing */
 497                 if (!fl)
 498                         return 0;
 499
 500                 /* Record is not filtered but enabled, disable it */
 501                 if (fl == FTRACE_FL_ENABLED)
 502                         rec->flags &= ~FTRACE_FL_ENABLED;
 503                 else
 504                 /* Otherwise record is filtered but not enabled, enable it */
 505                         rec->flags |= FTRACE_FL_ENABLED;
 506         } else {
 507                 /* Disable or not filtered */
 508
 509                 if (enable) {
 510                         /* if record is enabled, do nothing */
 511                         if (rec->flags & FTRACE_FL_ENABLED)
 512                                 return 0;
 513
 514                         rec->flags |= FTRACE_FL_ENABLED;
 515
 516                 } else {
 517
 518                         /* if record is not enabled do nothing */
 519                         if (!(rec->flags & FTRACE_FL_ENABLED))
 520                                 return 0;
 521
 522                         rec->flags &= ~FTRACE_FL_ENABLED;
 523                 }
 524         }
 525
 526         if (rec->flags & FTRACE_FL_ENABLED)
 527                 return ftrace_make_call(rec, ftrace_addr);
 528         else
 529                 return ftrace_make_nop(NULL, rec, ftrace_addr);
 530 }
 531
 532 static void ftrace_replace_code(int enable)
 533 {
 534         int i, failed;
 535         struct dyn_ftrace *rec;
 536         struct ftrace_page *pg;
 537
 538         for (pg = ftrace_pages_start; pg; pg = pg->next) {
 539                 for (i = 0; i < pg->index; i++) {
 540                         rec = &pg->records[i];
 541
 542                         /*
 543                          * Skip over free records and records that have
 544                          * failed.
 545                          */
 546                         if (rec->flags & FTRACE_FL_FREE ||
 547                             rec->flags & FTRACE_FL_FAILED)
 548                                 continue;
 549
 550                         /* ignore updates to this record's mcount site */
 551                         if (get_kprobe((void *)rec->ip)) {
 552                                 freeze_record(rec);
 553                                 continue;
 554                         } else {
 555                                 unfreeze_record(rec);
 556                         }
 557
 558                         failed = __ftrace_replace_code(rec, enable);
 559                         if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
 560                                 rec->flags |= FTRACE_FL_FAILED;
 561                                 if ((system_state == SYSTEM_BOOTING) ||
 562                                     !core_kernel_text(rec->ip)) {
 563                                         ftrace_free_rec(rec);
 564                                 } else
 565                                         ftrace_bug(failed, rec->ip);
 566                         }
 567                 }
 568         }
 569 }
 570
 571 static int
 572 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
 573 {
 574         unsigned long ip;
 575         int ret;
 576
 577         ip = rec->ip;
 578
 579         ret = ftrace_make_nop(mod, rec, mcount_addr);
 580         if (ret) {
 581                 ftrace_bug(ret, ip);
 582                 rec->flags |= FTRACE_FL_FAILED;
 583                 return 0;
 584         }
 585         return 1;
 586 }
 587
 588 static int __ftrace_modify_code(void *data)
 589 {
 590         int *command = data;
 591
 592         if (*command & FTRACE_ENABLE_CALLS)
 593                 ftrace_replace_code(1);
 594         else if (*command & FTRACE_DISABLE_CALLS)
 595                 ftrace_replace_code(0);
 596
 597         if (*command & FTRACE_UPDATE_TRACE_FUNC)
 598                 ftrace_update_ftrace_func(ftrace_trace_function);
 599
 600         if (*command & FTRACE_START_FUNC_RET)
 601                 ftrace_enable_ftrace_graph_caller();
 602         else if (*command & FTRACE_STOP_FUNC_RET)
 603                 ftrace_disable_ftrace_graph_caller();
 604
 605         return 0;
 606 }
 607
 608 static void ftrace_run_update_code(int command)
 609 {
 610         stop_machine(__ftrace_modify_code, &command, NULL);
 611 }
 612
 613 static ftrace_func_t saved_ftrace_func;
 614 static int ftrace_start_up;
 615
 616 static void ftrace_startup_enable(int command)
 617 {
 618         if (saved_ftrace_func != ftrace_trace_function) {
 619                 saved_ftrace_func = ftrace_trace_function;
 620                 command |= FTRACE_UPDATE_TRACE_FUNC;
 621         }
 622
 623         if (!command || !ftrace_enabled)
 624                 return;
 625
 626         ftrace_run_update_code(command);
 627 }
 628
 629 static void ftrace_startup(int command)
 630 {
 631         if (unlikely(ftrace_disabled))
 632                 return;
 633
 634         mutex_lock(&ftrace_start_lock);
 635         ftrace_start_up++;
 636         command |= FTRACE_ENABLE_CALLS;
 637
 638         ftrace_startup_enable(command);
 639
 640         mutex_unlock(&ftrace_start_lock);
 641 }
 642
 643 static void ftrace_shutdown(int command)
 644 {
 645         if (unlikely(ftrace_disabled))
 646                 return;
 647
 648         mutex_lock(&ftrace_start_lock);
 649         ftrace_start_up--;
 650         if (!ftrace_start_up)
 651                 command |= FTRACE_DISABLE_CALLS;
 652
 653         if (saved_ftrace_func != ftrace_trace_function) {
 654                 saved_ftrace_func = ftrace_trace_function;
 655                 command |= FTRACE_UPDATE_TRACE_FUNC;
 656         }
 657
 658         if (!command || !ftrace_enabled)
 659                 goto out;
 660
 661         ftrace_run_update_code(command);
 662  out:
 663         mutex_unlock(&ftrace_start_lock);
 664 }
 665
 666 static void ftrace_startup_sysctl(void)
 667 {
 668         int command = FTRACE_ENABLE_MCOUNT;
 669
 670         if (unlikely(ftrace_disabled))
 671                 return;
 672
 673         mutex_lock(&ftrace_start_lock);
 674         /* Force update next time */
 675         saved_ftrace_func = NULL;
 676         /* ftrace_start_up is true if we want ftrace running */
 677         if (ftrace_start_up)
 678                 command |= FTRACE_ENABLE_CALLS;
 679
 680         ftrace_run_update_code(command);
 681         mutex_unlock(&ftrace_start_lock);
 682 }
 683
 684 static void ftrace_shutdown_sysctl(void)
 685 {
 686         int command = FTRACE_DISABLE_MCOUNT;
 687
 688         if (unlikely(ftrace_disabled))
 689                 return;
 690
 691         mutex_lock(&ftrace_start_lock);
 692         /* ftrace_start_up is true if ftrace is running */
 693         if (ftrace_start_up)
 694                 command |= FTRACE_DISABLE_CALLS;
 695
 696         ftrace_run_update_code(command);
 697         mutex_unlock(&ftrace_start_lock);
 698 }
 699
 700 static cycle_t          ftrace_update_time;
 701 static unsigned long    ftrace_update_cnt;
 702 unsigned long           ftrace_update_tot_cnt;
 703
 704 static int ftrace_update_code(struct module *mod)
 705 {
 706         struct dyn_ftrace *p, *t;
 707         cycle_t start, stop;
 708
 709         start = ftrace_now(raw_smp_processor_id());
 710         ftrace_update_cnt = 0;
 711
 712         list_for_each_entry_safe(p, t, &ftrace_new_addrs, list) {
 713
 714                 /* If something went wrong, bail without enabling anything */
 715                 if (unlikely(ftrace_disabled))
 716                         return -1;
 717
 718                 list_del_init(&p->list);
 719
 720                 /* convert record (i.e, patch mcount-call with NOP) */
 721                 if (ftrace_code_disable(mod, p)) {
 722                         p->flags |= FTRACE_FL_CONVERTED;
 723                         ftrace_update_cnt++;
 724                 } else
 725                         ftrace_free_rec(p);
 726         }
 727
 728         stop = ftrace_now(raw_smp_processor_id());
 729         ftrace_update_time = stop - start;
 730         ftrace_update_tot_cnt += ftrace_update_cnt;
 731
 732         return 0;
 733 }
 734
 735 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
 736 {
 737         struct ftrace_page *pg;
 738         int cnt;
 739         int i;
 740
 741         /* allocate a few pages */
 742         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
 743         if (!ftrace_pages_start)
 744                 return -1;
 745
 746         /*
 747          * Allocate a few more pages.
 748          *
 749          * TODO: have some parser search vmlinux before
 750          *   final linking to find all calls to ftrace.
 751          *   Then we can:
 752          *    a) know how many pages to allocate.
 753          *     and/or
 754          *    b) set up the table then.
 755          *
 756          *  The dynamic code is still necessary for
 757          *  modules.
 758          */
 759
 760         pg = ftrace_pages = ftrace_pages_start;
 761
 762         cnt = num_to_init / ENTRIES_PER_PAGE;
 763         pr_info("ftrace: allocating %ld entries in %d pages\n",
 764                 num_to_init, cnt + 1);
 765
 766         for (i = 0; i < cnt; i++) {
 767                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 768
 769                 /* If we fail, we'll try later anyway */
 770                 if (!pg->next)
 771                         break;
 772
 773                 pg = pg->next;
 774         }
 775
 776         return 0;
 777 }
 778
 779 enum {
 780         FTRACE_ITER_FILTER      = (1 << 0),
 781         FTRACE_ITER_CONT        = (1 << 1),
 782         FTRACE_ITER_NOTRACE     = (1 << 2),
 783         FTRACE_ITER_FAILURES    = (1 << 3),
 784 };
 785
 786 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
 787
 788 struct ftrace_iterator {
 789         loff_t                  pos;
 790         struct ftrace_page      *pg;
 791         unsigned                idx;
 792         unsigned                flags;
 793         unsigned char           buffer[FTRACE_BUFF_MAX+1];
 794         unsigned                buffer_idx;
 795         unsigned                filtered;
 796 };
 797
 798 static void *
 799 t_next(struct seq_file *m, void *v, loff_t *pos)
 800 {
 801         struct ftrace_iterator *iter = m->private;
 802         struct dyn_ftrace *rec = NULL;
 803
 804         (*pos)++;
 805
 806         /* should not be called from interrupt context */
 807         spin_lock(&ftrace_lock);
 808  retry:
 809         if (iter->idx >= iter->pg->index) {
 810                 if (iter->pg->next) {
 811                         iter->pg = iter->pg->next;
 812                         iter->idx = 0;
 813                         goto retry;
 814                 }
 815         } else {
 816                 rec = &iter->pg->records[iter->idx++];
 817                 if ((rec->flags & FTRACE_FL_FREE) ||
 818
 819                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
 820                      (rec->flags & FTRACE_FL_FAILED)) ||
 821
 822                     ((iter->flags & FTRACE_ITER_FAILURES) &&
 823                      !(rec->flags & FTRACE_FL_FAILED)) ||
 824
 825                     ((iter->flags & FTRACE_ITER_FILTER) &&
 826                      !(rec->flags & FTRACE_FL_FILTER)) ||
 827
 828                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
 829                      !(rec->flags & FTRACE_FL_NOTRACE))) {
 830                         rec = NULL;
 831                         goto retry;
 832                 }
 833         }
 834         spin_unlock(&ftrace_lock);
 835
 836         iter->pos = *pos;
 837
 838         return rec;
 839 }
 840
 841 static void *t_start(struct seq_file *m, loff_t *pos)
 842 {
 843         struct ftrace_iterator *iter = m->private;
 844         void *p = NULL;
 845         loff_t l = -1;
 846
 847         if (*pos > iter->pos)
 848                 *pos = iter->pos;
 849
 850         l = *pos;
 851         p = t_next(m, p, &l);
 852
 853         return p;
 854 }
 855
 856 static void t_stop(struct seq_file *m, void *p)
 857 {
 858 }
 859
 860 static int t_show(struct seq_file *m, void *v)
 861 {
 862         struct ftrace_iterator *iter = m->private;
 863         struct dyn_ftrace *rec = v;
 864         char str[KSYM_SYMBOL_LEN];
 865         int ret = 0;
 866
 867         if (!rec)
 868                 return 0;
 869
 870         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 871
 872         ret = seq_printf(m, "%s\n", str);
 873         if (ret < 0) {
 874                 iter->pos--;
 875                 iter->idx--;
 876         }
 877
 878         return 0;
 879 }
 880
 881 static struct seq_operations show_ftrace_seq_ops = {
 882         .start = t_start,
 883         .next = t_next,
 884         .stop = t_stop,
 885         .show = t_show,
 886 };
 887
 888 static int
 889 ftrace_avail_open(struct inode *inode, struct file *file)
 890 {
 891         struct ftrace_iterator *iter;
 892         int ret;
 893
 894         if (unlikely(ftrace_disabled))
 895                 return -ENODEV;
 896
 897         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
 898         if (!iter)
 899                 return -ENOMEM;
 900
 901         iter->pg = ftrace_pages_start;
 902         iter->pos = 0;
 903
 904         ret = seq_open(file, &show_ftrace_seq_ops);
 905         if (!ret) {
 906                 struct seq_file *m = file->private_data;
 907
 908                 m->private = iter;
 909         } else {
 910                 kfree(iter);
 911         }
 912
 913         return ret;
 914 }
 915
 916 int ftrace_avail_release(struct inode *inode, struct file *file)
 917 {
 918         struct seq_file *m = (struct seq_file *)file->private_data;
 919         struct ftrace_iterator *iter = m->private;
 920
 921         seq_release(inode, file);
 922         kfree(iter);
 923
 924         return 0;
 925 }
 926
 927 static int
 928 ftrace_failures_open(struct inode *inode, struct file *file)
 929 {
 930         int ret;
 931         struct seq_file *m;
 932         struct ftrace_iterator *iter;
 933
 934         ret = ftrace_avail_open(inode, file);
 935         if (!ret) {
 936                 m = (struct seq_file *)file->private_data;
 937                 iter = (struct ftrace_iterator *)m->private;
 938                 iter->flags = FTRACE_ITER_FAILURES;
 939         }
 940
 941         return ret;
 942 }
 943
 944
 945 static void ftrace_filter_reset(int enable)
 946 {
 947         struct ftrace_page *pg;
 948         struct dyn_ftrace *rec;
 949         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
 950         unsigned i;
 951
 952         /* should not be called from interrupt context */
 953         spin_lock(&ftrace_lock);
 954         if (enable)
 955                 ftrace_filtered = 0;
 956         pg = ftrace_pages_start;
 957         while (pg) {
 958                 for (i = 0; i < pg->index; i++) {
 959                         rec = &pg->records[i];
 960                         if (rec->flags & FTRACE_FL_FAILED)
 961                                 continue;
 962                         rec->flags &= ~type;
 963                 }
 964                 pg = pg->next;
 965         }
 966         spin_unlock(&ftrace_lock);
 967 }
 968
 969 static int
 970 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
 971 {
 972         struct ftrace_iterator *iter;
 973         int ret = 0;
 974
 975         if (unlikely(ftrace_disabled))
 976                 return -ENODEV;
 977
 978         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
 979         if (!iter)
 980                 return -ENOMEM;
 981
 982         mutex_lock(&ftrace_regex_lock);
 983         if ((file->f_mode & FMODE_WRITE) &&
 984             !(file->f_flags & O_APPEND))
 985                 ftrace_filter_reset(enable);
 986
 987         if (file->f_mode & FMODE_READ) {
 988                 iter->pg = ftrace_pages_start;
 989                 iter->pos = 0;
 990                 iter->flags = enable ? FTRACE_ITER_FILTER :
 991                         FTRACE_ITER_NOTRACE;
 992
 993                 ret = seq_open(file, &show_ftrace_seq_ops);
 994                 if (!ret) {
 995                         struct seq_file *m = file->private_data;
 996                         m->private = iter;
 997                 } else
 998                         kfree(iter);
 999         } else
1000                 file->private_data = iter;
1001         mutex_unlock(&ftrace_regex_lock);
1002
1003         return ret;
1004 }
1005
1006 static int
1007 ftrace_filter_open(struct inode *inode, struct file *file)
1008 {
1009         return ftrace_regex_open(inode, file, 1);
1010 }
1011
1012 static int
1013 ftrace_notrace_open(struct inode *inode, struct file *file)
1014 {
1015         return ftrace_regex_open(inode, file, 0);
1016 }
1017
1018 static ssize_t
1019 ftrace_regex_read(struct file *file, char __user *ubuf,
1020                        size_t cnt, loff_t *ppos)
1021 {
1022         if (file->f_mode & FMODE_READ)
1023                 return seq_read(file, ubuf, cnt, ppos);
1024         else
1025                 return -EPERM;
1026 }
1027
1028 static loff_t
1029 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1030 {
1031         loff_t ret;
1032
1033         if (file->f_mode & FMODE_READ)
1034                 ret = seq_lseek(file, offset, origin);
1035         else
1036                 file->f_pos = ret = 1;
1037
1038         return ret;
1039 }
1040
1041 enum {
1042         MATCH_FULL,
1043         MATCH_FRONT_ONLY,
1044         MATCH_MIDDLE_ONLY,
1045         MATCH_END_ONLY,
1046 };
1047
1048 static void
1049 ftrace_match(unsigned char *buff, int len, int enable)
1050 {
1051         char str[KSYM_SYMBOL_LEN];
1052         char *search = NULL;
1053         struct ftrace_page *pg;
1054         struct dyn_ftrace *rec;
1055         int type = MATCH_FULL;
1056         unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1057         unsigned i, match = 0, search_len = 0;
1058
1059         for (i = 0; i < len; i++) {
1060                 if (buff[i] == '*') {
1061                         if (!i) {
1062                                 search = buff + i + 1;
1063                                 type = MATCH_END_ONLY;
1064                                 search_len = len - (i + 1);
1065                         } else {
1066                                 if (type == MATCH_END_ONLY) {
1067                                         type = MATCH_MIDDLE_ONLY;
1068                                 } else {
1069                                         match = i;
1070                                         type = MATCH_FRONT_ONLY;
1071                                 }
1072                                 buff[i] = 0;
1073                                 break;
1074                         }
1075                 }
1076         }
1077
1078         /* should not be called from interrupt context */
1079         spin_lock(&ftrace_lock);
1080         if (enable)
1081                 ftrace_filtered = 1;
1082         pg = ftrace_pages_start;
1083         while (pg) {
1084                 for (i = 0; i < pg->index; i++) {
1085                         int matched = 0;
1086                         char *ptr;
1087
1088                         rec = &pg->records[i];
1089                         if (rec->flags & FTRACE_FL_FAILED)
1090                                 continue;
1091                         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1092                         switch (type) {
1093                         case MATCH_FULL:
1094                                 if (strcmp(str, buff) == 0)
1095                                         matched = 1;
1096                                 break;
1097                         case MATCH_FRONT_ONLY:
1098                                 if (memcmp(str, buff, match) == 0)
1099                                         matched = 1;
1100                                 break;
1101                         case MATCH_MIDDLE_ONLY:
1102                                 if (strstr(str, search))
1103                                         matched = 1;
1104                                 break;
1105                         case MATCH_END_ONLY:
1106                                 ptr = strstr(str, search);
1107                                 if (ptr && (ptr[search_len] == 0))
1108                                         matched = 1;
1109                                 break;
1110                         }
1111                         if (matched)
1112                                 rec->flags |= flag;
1113                 }
1114                 pg = pg->next;
1115         }
1116         spin_unlock(&ftrace_lock);
1117 }
1118
1119 static ssize_t
1120 ftrace_regex_write(struct file *file, const char __user *ubuf,
1121                    size_t cnt, loff_t *ppos, int enable)
1122 {
1123         struct ftrace_iterator *iter;
1124         char ch;
1125         size_t read = 0;
1126         ssize_t ret;
1127
1128         if (!cnt || cnt < 0)
1129                 return 0;
1130
1131         mutex_lock(&ftrace_regex_lock);
1132
1133         if (file->f_mode & FMODE_READ) {
1134                 struct seq_file *m = file->private_data;
1135                 iter = m->private;
1136         } else
1137                 iter = file->private_data;
1138
1139         if (!*ppos) {
1140                 iter->flags &= ~FTRACE_ITER_CONT;
1141                 iter->buffer_idx = 0;
1142         }
1143
1144         ret = get_user(ch, ubuf++);
1145         if (ret)
1146                 goto out;
1147         read++;
1148         cnt--;
1149
1150         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1151                 /* skip white space */
1152                 while (cnt && isspace(ch)) {
1153                         ret = get_user(ch, ubuf++);
1154                         if (ret)
1155                                 goto out;
1156                         read++;
1157                         cnt--;
1158                 }
1159
1160                 if (isspace(ch)) {
1161                         file->f_pos += read;
1162                         ret = read;
1163                         goto out;
1164                 }
1165
1166                 iter->buffer_idx = 0;
1167         }
1168
1169         while (cnt && !isspace(ch)) {
1170                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1171                         iter->buffer[iter->buffer_idx++] = ch;
1172                 else {
1173                         ret = -EINVAL;
1174                         goto out;
1175                 }
1176                 ret = get_user(ch, ubuf++);
1177                 if (ret)
1178                         goto out;
1179                 read++;
1180                 cnt--;
1181         }
1182
1183         if (isspace(ch)) {
1184                 iter->filtered++;
1185                 iter->buffer[iter->buffer_idx] = 0;
1186                 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1187                 iter->buffer_idx = 0;
1188         } else
1189                 iter->flags |= FTRACE_ITER_CONT;
1190
1191
1192         file->f_pos += read;
1193
1194         ret = read;
1195  out:
1196         mutex_unlock(&ftrace_regex_lock);
1197
1198         return ret;
1199 }
1200
1201 static ssize_t
1202 ftrace_filter_write(struct file *file, const char __user *ubuf,
1203                     size_t cnt, loff_t *ppos)
1204 {
1205         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1206 }
1207
1208 static ssize_t
1209 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1210                      size_t cnt, loff_t *ppos)
1211 {
1212         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1213 }
1214
1215 static void
1216 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1217 {
1218         if (unlikely(ftrace_disabled))
1219                 return;
1220
1221         mutex_lock(&ftrace_regex_lock);
1222         if (reset)
1223                 ftrace_filter_reset(enable);
1224         if (buf)
1225                 ftrace_match(buf, len, enable);
1226         mutex_unlock(&ftrace_regex_lock);
1227 }
1228
1229 /**
1230  * ftrace_set_filter - set a function to filter on in ftrace
1231  * @buf - the string that holds the function filter text.
1232  * @len - the length of the string.
1233  * @reset - non zero to reset all filters before applying this filter.
1234  *
1235  * Filters denote which functions should be enabled when tracing is enabled.
1236  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1237  */
1238 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1239 {
1240         ftrace_set_regex(buf, len, reset, 1);
1241 }
1242
1243 /**
1244  * ftrace_set_notrace - set a function to not trace in ftrace
1245  * @buf - the string that holds the function notrace text.
1246  * @len - the length of the string.
1247  * @reset - non zero to reset all filters before applying this filter.
1248  *
1249  * Notrace Filters denote which functions should not be enabled when tracing
1250  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1251  * for tracing.
1252  */
1253 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1254 {
1255         ftrace_set_regex(buf, len, reset, 0);
1256 }
1257
1258 static int
1259 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1260 {
1261         struct seq_file *m = (struct seq_file *)file->private_data;
1262         struct ftrace_iterator *iter;
1263
1264         mutex_lock(&ftrace_regex_lock);
1265         if (file->f_mode & FMODE_READ) {
1266                 iter = m->private;
1267
1268                 seq_release(inode, file);
1269         } else
1270                 iter = file->private_data;
1271
1272         if (iter->buffer_idx) {
1273                 iter->filtered++;
1274                 iter->buffer[iter->buffer_idx] = 0;
1275                 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1276         }
1277
1278         mutex_lock(&ftrace_sysctl_lock);
1279         mutex_lock(&ftrace_start_lock);
1280         if (ftrace_start_up && ftrace_enabled)
1281                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1282         mutex_unlock(&ftrace_start_lock);
1283         mutex_unlock(&ftrace_sysctl_lock);
1284
1285         kfree(iter);
1286         mutex_unlock(&ftrace_regex_lock);
1287         return 0;
1288 }
1289
1290 static int
1291 ftrace_filter_release(struct inode *inode, struct file *file)
1292 {
1293         return ftrace_regex_release(inode, file, 1);
1294 }
1295
1296 static int
1297 ftrace_notrace_release(struct inode *inode, struct file *file)
1298 {
1299         return ftrace_regex_release(inode, file, 0);
1300 }
1301
1302 static struct file_operations ftrace_avail_fops = {
1303         .open = ftrace_avail_open,
1304         .read = seq_read,
1305         .llseek = seq_lseek,
1306         .release = ftrace_avail_release,
1307 };
1308
1309 static struct file_operations ftrace_failures_fops = {
1310         .open = ftrace_failures_open,
1311         .read = seq_read,
1312         .llseek = seq_lseek,
1313         .release = ftrace_avail_release,
1314 };
1315
1316 static struct file_operations ftrace_filter_fops = {
1317         .open = ftrace_filter_open,
1318         .read = ftrace_regex_read,
1319         .write = ftrace_filter_write,
1320         .llseek = ftrace_regex_lseek,
1321         .release = ftrace_filter_release,
1322 };
1323
1324 static struct file_operations ftrace_notrace_fops = {
1325         .open = ftrace_notrace_open,
1326         .read = ftrace_regex_read,
1327         .write = ftrace_notrace_write,
1328         .llseek = ftrace_regex_lseek,
1329         .release = ftrace_notrace_release,
1330 };
1331
1332 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
1333 {
1334         struct dentry *entry;
1335
1336         entry = debugfs_create_file("available_filter_functions", 0444,
1337                                     d_tracer, NULL, &ftrace_avail_fops);
1338         if (!entry)
1339                 pr_warning("Could not create debugfs "
1340                            "'available_filter_functions' entry\n");
1341
1342         entry = debugfs_create_file("failures", 0444,
1343                                     d_tracer, NULL, &ftrace_failures_fops);
1344         if (!entry)
1345                 pr_warning("Could not create debugfs 'failures' entry\n");
1346
1347         entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
1348                                     NULL, &ftrace_filter_fops);
1349         if (!entry)
1350                 pr_warning("Could not create debugfs "
1351                            "'set_ftrace_filter' entry\n");
1352
1353         entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
1354                                     NULL, &ftrace_notrace_fops);
1355         if (!entry)
1356                 pr_warning("Could not create debugfs "
1357                            "'set_ftrace_notrace' entry\n");
1358
1359         return 0;
1360 }
1361
1362 static int ftrace_convert_nops(struct module *mod,
1363                                unsigned long *start,
1364                                unsigned long *end)
1365 {
1366         unsigned long *p;
1367         unsigned long addr;
1368         unsigned long flags;
1369
1370         mutex_lock(&ftrace_start_lock);
1371         p = start;
1372         while (p < end) {
1373                 addr = ftrace_call_adjust(*p++);
1374                 /*
1375                  * Some architecture linkers will pad between
1376                  * the different mcount_loc sections of different
1377                  * object files to satisfy alignments.
1378                  * Skip any NULL pointers.
1379                  */
1380                 if (!addr)
1381                         continue;
1382                 ftrace_record_ip(addr);
1383         }
1384
1385         /* disable interrupts to prevent kstop machine */
1386         local_irq_save(flags);
1387         ftrace_update_code(mod);
1388         local_irq_restore(flags);
1389         mutex_unlock(&ftrace_start_lock);
1390
1391         return 0;
1392 }
1393
1394 void ftrace_init_module(struct module *mod,
1395                         unsigned long *start, unsigned long *end)
1396 {
1397         if (ftrace_disabled || start == end)
1398                 return;
1399         ftrace_convert_nops(mod, start, end);
1400 }
1401
1402 extern unsigned long __start_mcount_loc[];
1403 extern unsigned long __stop_mcount_loc[];
1404
1405 void __init ftrace_init(void)
1406 {
1407         unsigned long count, addr, flags;
1408         int ret;
1409
1410         /* Keep the ftrace pointer to the stub */
1411         addr = (unsigned long)ftrace_stub;
1412
1413         local_irq_save(flags);
1414         ftrace_dyn_arch_init(&addr);
1415         local_irq_restore(flags);
1416
1417         /* ftrace_dyn_arch_init places the return code in addr */
1418         if (addr)
1419                 goto failed;
1420
1421         count = __stop_mcount_loc - __start_mcount_loc;
1422
1423         ret = ftrace_dyn_table_alloc(count);
1424         if (ret)
1425                 goto failed;
1426
1427         last_ftrace_enabled = ftrace_enabled = 1;
1428
1429         ret = ftrace_convert_nops(NULL,
1430                                   __start_mcount_loc,
1431                                   __stop_mcount_loc);
1432
1433         return;
1434  failed:
1435         ftrace_disabled = 1;
1436 }
1437
1438 #else
1439
1440 static int __init ftrace_nodyn_init(void)
1441 {
1442         ftrace_enabled = 1;
1443         return 0;
1444 }
1445 device_initcall(ftrace_nodyn_init);
1446
1447 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
1448 static inline void ftrace_startup_enable(int command) { }
1449 /* Keep as macros so we do not need to define the commands */
1450 # define ftrace_startup(command)        do { } while (0)
1451 # define ftrace_shutdown(command)       do { } while (0)
1452 # define ftrace_startup_sysctl()        do { } while (0)
1453 # define ftrace_shutdown_sysctl()       do { } while (0)
1454 #endif /* CONFIG_DYNAMIC_FTRACE */
1455
1456 static ssize_t
1457 ftrace_pid_read(struct file *file, char __user *ubuf,
1458                        size_t cnt, loff_t *ppos)
1459 {
1460         char buf[64];
1461         int r;
1462
1463         if (ftrace_pid_trace >= 0)
1464                 r = sprintf(buf, "%u\n", ftrace_pid_trace);
1465         else
1466                 r = sprintf(buf, "no pid\n");
1467
1468         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1469 }
1470
1471 static ssize_t
1472 ftrace_pid_write(struct file *filp, const char __user *ubuf,
1473                    size_t cnt, loff_t *ppos)
1474 {
1475         char buf[64];
1476         long val;
1477         int ret;
1478
1479         if (cnt >= sizeof(buf))
1480                 return -EINVAL;
1481
1482         if (copy_from_user(&buf, ubuf, cnt))
1483                 return -EFAULT;
1484
1485         buf[cnt] = 0;
1486
1487         ret = strict_strtol(buf, 10, &val);
1488         if (ret < 0)
1489                 return ret;
1490
1491         mutex_lock(&ftrace_start_lock);
1492         if (ret < 0) {
1493                 /* disable pid tracing */
1494                 if (ftrace_pid_trace < 0)
1495                         goto out;
1496                 ftrace_pid_trace = -1;
1497
1498         } else {
1499
1500                 if (ftrace_pid_trace == val)
1501                         goto out;
1502
1503                 ftrace_pid_trace = val;
1504         }
1505
1506         /* update the function call */
1507         ftrace_update_pid_func();
1508         ftrace_startup_enable(0);
1509
1510  out:
1511         mutex_unlock(&ftrace_start_lock);
1512
1513         return cnt;
1514 }
1515
1516 static struct file_operations ftrace_pid_fops = {
1517         .read = ftrace_pid_read,
1518         .write = ftrace_pid_write,
1519 };
1520
1521 static __init int ftrace_init_debugfs(void)
1522 {
1523         struct dentry *d_tracer;
1524         struct dentry *entry;
1525
1526         d_tracer = tracing_init_dentry();
1527         if (!d_tracer)
1528                 return 0;
1529
1530         ftrace_init_dyn_debugfs(d_tracer);
1531
1532         entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
1533                                     NULL, &ftrace_pid_fops);
1534         if (!entry)
1535                 pr_warning("Could not create debugfs "
1536                            "'set_ftrace_pid' entry\n");
1537         return 0;
1538 }
1539
1540 fs_initcall(ftrace_init_debugfs);
1541
1542 /**
1543  * ftrace_kill - kill ftrace
1544  *
1545  * This function should be used by panic code. It stops ftrace
1546  * but in a not so nice way. If you need to simply kill ftrace
1547  * from a non-atomic section, use ftrace_kill.
1548  */
1549 void ftrace_kill(void)
1550 {
1551         ftrace_disabled = 1;
1552         ftrace_enabled = 0;
1553         clear_ftrace_function();
1554 }
1555
1556 /**
1557  * register_ftrace_function - register a function for profiling
1558  * @ops - ops structure that holds the function for profiling.
1559  *
1560  * Register a function to be called by all functions in the
1561  * kernel.
1562  *
1563  * Note: @ops->func and all the functions it calls must be labeled
1564  *       with "notrace", otherwise it will go into a
1565  *       recursive loop.
1566  */
1567 int register_ftrace_function(struct ftrace_ops *ops)
1568 {
1569         int ret;
1570
1571         if (unlikely(ftrace_disabled))
1572                 return -1;
1573
1574         mutex_lock(&ftrace_sysctl_lock);
1575
1576         ret = __register_ftrace_function(ops);
1577         ftrace_startup(0);
1578
1579         mutex_unlock(&ftrace_sysctl_lock);
1580         return ret;
1581 }
1582
1583 /**
1584  * unregister_ftrace_function - unresgister a function for profiling.
1585  * @ops - ops structure that holds the function to unregister
1586  *
1587  * Unregister a function that was added to be called by ftrace profiling.
1588  */
1589 int unregister_ftrace_function(struct ftrace_ops *ops)
1590 {
1591         int ret;
1592
1593         mutex_lock(&ftrace_sysctl_lock);
1594         ret = __unregister_ftrace_function(ops);
1595         ftrace_shutdown(0);
1596         mutex_unlock(&ftrace_sysctl_lock);
1597
1598         return ret;
1599 }
1600
1601 int
1602 ftrace_enable_sysctl(struct ctl_table *table, int write,
1603                      struct file *file, void __user *buffer, size_t *lenp,
1604                      loff_t *ppos)
1605 {
1606         int ret;
1607
1608         if (unlikely(ftrace_disabled))
1609                 return -ENODEV;
1610
1611         mutex_lock(&ftrace_sysctl_lock);
1612
1613         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
1614
1615         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
1616                 goto out;
1617
1618         last_ftrace_enabled = ftrace_enabled;
1619
1620         if (ftrace_enabled) {
1621
1622                 ftrace_startup_sysctl();
1623
1624                 /* we are starting ftrace again */
1625                 if (ftrace_list != &ftrace_list_end) {
1626                         if (ftrace_list->next == &ftrace_list_end)
1627                                 ftrace_trace_function = ftrace_list->func;
1628                         else
1629                                 ftrace_trace_function = ftrace_list_func;
1630                 }
1631
1632         } else {
1633                 /* stopping ftrace calls (just send to ftrace_stub) */
1634                 ftrace_trace_function = ftrace_stub;
1635
1636                 ftrace_shutdown_sysctl();
1637         }
1638
1639  out:
1640         mutex_unlock(&ftrace_sysctl_lock);
1641         return ret;
1642 }
1643
1644 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1645
1646 static atomic_t ftrace_graph_active;
1647
1648 /* The callbacks that hook a function */
1649 trace_func_graph_ret_t ftrace_graph_return =
1650                         (trace_func_graph_ret_t)ftrace_stub;
1651 trace_func_graph_ent_t ftrace_graph_entry =
1652                         (trace_func_graph_ent_t)ftrace_stub;
1653
1654 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
1655 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
1656 {
1657         int i;
1658         int ret = 0;
1659         unsigned long flags;
1660         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
1661         struct task_struct *g, *t;
1662
1663         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
1664                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
1665                                         * sizeof(struct ftrace_ret_stack),
1666                                         GFP_KERNEL);
1667                 if (!ret_stack_list[i]) {
1668                         start = 0;
1669                         end = i;
1670                         ret = -ENOMEM;
1671                         goto free;
1672                 }
1673         }
1674
1675         read_lock_irqsave(&tasklist_lock, flags);
1676         do_each_thread(g, t) {
1677                 if (start == end) {
1678                         ret = -EAGAIN;
1679                         goto unlock;
1680                 }
1681
1682                 if (t->ret_stack == NULL) {
1683                         t->ret_stack = ret_stack_list[start++];
1684                         t->curr_ret_stack = -1;
1685                         atomic_set(&t->trace_overrun, 0);
1686                 }
1687         } while_each_thread(g, t);
1688
1689 unlock:
1690         read_unlock_irqrestore(&tasklist_lock, flags);
1691 free:
1692         for (i = start; i < end; i++)
1693                 kfree(ret_stack_list[i]);
1694         return ret;
1695 }
1696
1697 /* Allocate a return stack for each task */
1698 static int start_graph_tracing(void)
1699 {
1700         struct ftrace_ret_stack **ret_stack_list;
1701         int ret;
1702
1703         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
1704                                 sizeof(struct ftrace_ret_stack *),
1705                                 GFP_KERNEL);
1706
1707         if (!ret_stack_list)
1708                 return -ENOMEM;
1709
1710         do {
1711                 ret = alloc_retstack_tasklist(ret_stack_list);
1712         } while (ret == -EAGAIN);
1713
1714         kfree(ret_stack_list);
1715         return ret;
1716 }
1717
1718 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
1719                         trace_func_graph_ent_t entryfunc)
1720 {
1721         int ret = 0;
1722
1723         mutex_lock(&ftrace_sysctl_lock);
1724
1725         atomic_inc(&ftrace_graph_active);
1726         ret = start_graph_tracing();
1727         if (ret) {
1728                 atomic_dec(&ftrace_graph_active);
1729                 goto out;
1730         }
1731
1732         ftrace_graph_return = retfunc;
1733         ftrace_graph_entry = entryfunc;
1734
1735         ftrace_startup(FTRACE_START_FUNC_RET);
1736
1737 out:
1738         mutex_unlock(&ftrace_sysctl_lock);
1739         return ret;
1740 }
1741
1742 void unregister_ftrace_graph(void)
1743 {
1744         mutex_lock(&ftrace_sysctl_lock);
1745
1746         atomic_dec(&ftrace_graph_active);
1747         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
1748         ftrace_graph_entry = (trace_func_graph_ent_t)ftrace_stub;
1749         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
1750
1751         mutex_unlock(&ftrace_sysctl_lock);
1752 }
1753
1754 /* Allocate a return stack for newly created task */
1755 void ftrace_graph_init_task(struct task_struct *t)
1756 {
1757         if (atomic_read(&ftrace_graph_active)) {
1758                 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
1759                                 * sizeof(struct ftrace_ret_stack),
1760                                 GFP_KERNEL);
1761                 if (!t->ret_stack)
1762                         return;
1763                 t->curr_ret_stack = -1;
1764                 atomic_set(&t->trace_overrun, 0);
1765         } else
1766                 t->ret_stack = NULL;
1767 }
1768
1769 void ftrace_graph_exit_task(struct task_struct *t)
1770 {
1771         struct ftrace_ret_stack *ret_stack = t->ret_stack;
1772
1773         t->ret_stack = NULL;
1774         /* NULL must become visible to IRQs before we free it: */
1775         barrier();
1776
1777         kfree(ret_stack);
1778 }
1779 #endif
1780