2 * linux/arch/x86-64/mm/fault.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
8 #include <linux/signal.h>
9 #include <linux/sched.h>
10 #include <linux/kernel.h>
11 #include <linux/errno.h>
12 #include <linux/string.h>
13 #include <linux/types.h>
14 #include <linux/ptrace.h>
15 #include <linux/mman.h>
17 #include <linux/smp.h>
18 #include <linux/smp_lock.h>
19 #include <linux/interrupt.h>
20 #include <linux/init.h>
21 #include <linux/tty.h>
22 #include <linux/vt_kern.h> /* For unblank_screen() */
23 #include <linux/compiler.h>
24 #include <linux/module.h>
25 #include <linux/kprobes.h>
26 #include <linux/uaccess.h>
28 #include <asm/system.h>
29 #include <asm/pgalloc.h>
31 #include <asm/tlbflush.h>
32 #include <asm/proto.h>
33 #include <asm/kdebug.h>
34 #include <asm-generic/sections.h>
36 /* Page fault error code bits */
37 #define PF_PROT (1<<0) /* or no page found */
38 #define PF_WRITE (1<<1)
39 #define PF_USER (1<<2)
40 #define PF_RSVD (1<<3)
41 #define PF_INSTR (1<<4)
43 static ATOMIC_NOTIFIER_HEAD(notify_page_fault_chain);
45 /* Hook to register for page fault notifications */
46 int register_page_fault_notifier(struct notifier_block *nb)
49 return atomic_notifier_chain_register(¬ify_page_fault_chain, nb);
51 EXPORT_SYMBOL_GPL(register_page_fault_notifier);
53 int unregister_page_fault_notifier(struct notifier_block *nb)
55 return atomic_notifier_chain_unregister(¬ify_page_fault_chain, nb);
57 EXPORT_SYMBOL_GPL(unregister_page_fault_notifier);
59 static inline int notify_page_fault(enum die_val val, const char *str,
60 struct pt_regs *regs, long err, int trap, int sig)
62 struct die_args args = {
69 return atomic_notifier_call_chain(¬ify_page_fault_chain, val, &args);
72 /* Sometimes the CPU reports invalid exceptions on prefetch.
73 Check that here and ignore.
74 Opcode checker based on code by Richard Brunner */
75 static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
76 unsigned long error_code)
81 unsigned char *max_instr;
83 /* If it was a exec fault ignore */
84 if (error_code & PF_INSTR)
87 instr = (unsigned char __user *)convert_rip_to_linear(current, regs);
88 max_instr = instr + 15;
90 if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
93 while (scan_more && instr < max_instr) {
95 unsigned char instr_hi;
96 unsigned char instr_lo;
98 if (probe_kernel_address(instr, opcode))
101 instr_hi = opcode & 0xf0;
102 instr_lo = opcode & 0x0f;
108 /* Values 0x26,0x2E,0x36,0x3E are valid x86
109 prefixes. In long mode, the CPU will signal
110 invalid opcode if some of these prefixes are
111 present so we will never get here anyway */
112 scan_more = ((instr_lo & 7) == 0x6);
116 /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
117 Need to figure out under what instruction mode the
118 instruction was issued ... */
119 /* Could check the LDT for lm, but for now it's good
120 enough to assume that long mode only uses well known
121 segments or kernel. */
122 scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
126 /* 0x64 thru 0x67 are valid prefixes in all modes. */
127 scan_more = (instr_lo & 0xC) == 0x4;
130 /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
131 scan_more = !instr_lo || (instr_lo>>1) == 1;
134 /* Prefetch instruction is 0x0F0D or 0x0F18 */
136 if (probe_kernel_address(instr, opcode))
138 prefetch = (instr_lo == 0xF) &&
139 (opcode == 0x0D || opcode == 0x18);
149 static int bad_address(void *p)
152 return probe_kernel_address((unsigned long *)p, dummy);
155 void dump_pagetable(unsigned long address)
162 asm("movq %%cr3,%0" : "=r" (pgd));
164 pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
165 pgd += pgd_index(address);
166 if (bad_address(pgd)) goto bad;
167 printk("PGD %lx ", pgd_val(*pgd));
168 if (!pgd_present(*pgd)) goto ret;
170 pud = pud_offset(pgd, address);
171 if (bad_address(pud)) goto bad;
172 printk("PUD %lx ", pud_val(*pud));
173 if (!pud_present(*pud)) goto ret;
175 pmd = pmd_offset(pud, address);
176 if (bad_address(pmd)) goto bad;
177 printk("PMD %lx ", pmd_val(*pmd));
178 if (!pmd_present(*pmd)) goto ret;
180 pte = pte_offset_kernel(pmd, address);
181 if (bad_address(pte)) goto bad;
182 printk("PTE %lx", pte_val(*pte));
190 static const char errata93_warning[] =
191 KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
192 KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
193 KERN_ERR "******* Please consider a BIOS update.\n"
194 KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
196 /* Workaround for K8 erratum #93 & buggy BIOS.
197 BIOS SMM functions are required to use a specific workaround
198 to avoid corruption of the 64bit RIP register on C stepping K8.
199 A lot of BIOS that didn't get tested properly miss this.
200 The OS sees this as a page fault with the upper 32bits of RIP cleared.
201 Try to work around it here.
202 Note we only handle faults in kernel here. */
204 static int is_errata93(struct pt_regs *regs, unsigned long address)
207 if (address != regs->rip)
209 if ((address >> 32) != 0)
211 address |= 0xffffffffUL << 32;
212 if ((address >= (u64)_stext && address <= (u64)_etext) ||
213 (address >= MODULES_VADDR && address <= MODULES_END)) {
215 printk(errata93_warning);
224 int unhandled_signal(struct task_struct *tsk, int sig)
228 if (tsk->ptrace & PT_PTRACED)
230 return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
231 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
234 static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
235 unsigned long error_code)
237 unsigned long flags = oops_begin();
238 struct task_struct *tsk;
240 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
241 current->comm, address);
242 dump_pagetable(address);
244 tsk->thread.cr2 = address;
245 tsk->thread.trap_no = 14;
246 tsk->thread.error_code = error_code;
247 __die("Bad pagetable", regs, error_code);
253 * Handle a fault on the vmalloc area
255 * This assumes no large pages in there.
257 static int vmalloc_fault(unsigned long address)
259 pgd_t *pgd, *pgd_ref;
260 pud_t *pud, *pud_ref;
261 pmd_t *pmd, *pmd_ref;
262 pte_t *pte, *pte_ref;
264 /* Copy kernel mappings over when needed. This can also
265 happen within a race in page table update. In the later
268 pgd = pgd_offset(current->mm ?: &init_mm, address);
269 pgd_ref = pgd_offset_k(address);
270 if (pgd_none(*pgd_ref))
273 set_pgd(pgd, *pgd_ref);
275 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
277 /* Below here mismatches are bugs because these lower tables
280 pud = pud_offset(pgd, address);
281 pud_ref = pud_offset(pgd_ref, address);
282 if (pud_none(*pud_ref))
284 if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
286 pmd = pmd_offset(pud, address);
287 pmd_ref = pmd_offset(pud_ref, address);
288 if (pmd_none(*pmd_ref))
290 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
292 pte_ref = pte_offset_kernel(pmd_ref, address);
293 if (!pte_present(*pte_ref))
295 pte = pte_offset_kernel(pmd, address);
296 /* Don't use pte_page here, because the mappings can point
297 outside mem_map, and the NUMA hash lookup cannot handle
299 if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
304 int page_fault_trace = 0;
305 int exception_trace = 1;
308 * This routine handles page faults. It determines the address,
309 * and the problem, and then passes it off to one of the appropriate
312 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
313 unsigned long error_code)
315 struct task_struct *tsk;
316 struct mm_struct *mm;
317 struct vm_area_struct * vma;
318 unsigned long address;
319 const struct exception_table_entry *fixup;
326 prefetchw(&mm->mmap_sem);
328 /* get the address */
329 __asm__("movq %%cr2,%0":"=r" (address));
331 info.si_code = SEGV_MAPERR;
335 * We fault-in kernel-space virtual memory on-demand. The
336 * 'reference' page table is init_mm.pgd.
338 * NOTE! We MUST NOT take any locks for this case. We may
339 * be in an interrupt or a critical region, and should
340 * only copy the information from the master page table,
343 * This verifies that the fault happens in kernel space
344 * (error_code & 4) == 0, and that the fault was not a
345 * protection error (error_code & 9) == 0.
347 if (unlikely(address >= TASK_SIZE64)) {
349 * Don't check for the module range here: its PML4
350 * is always initialized because it's shared with the main
351 * kernel text. Only vmalloc may need PML4 syncups.
353 if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
354 ((address >= VMALLOC_START && address < VMALLOC_END))) {
355 if (vmalloc_fault(address) >= 0)
358 if (notify_page_fault(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
359 SIGSEGV) == NOTIFY_STOP)
362 * Don't take the mm semaphore here. If we fixup a prefetch
363 * fault we could otherwise deadlock.
365 goto bad_area_nosemaphore;
368 if (notify_page_fault(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
369 SIGSEGV) == NOTIFY_STOP)
372 if (likely(regs->eflags & X86_EFLAGS_IF))
375 if (unlikely(page_fault_trace))
376 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
377 regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
379 if (unlikely(error_code & PF_RSVD))
380 pgtable_bad(address, regs, error_code);
383 * If we're in an interrupt or have no user
384 * context, we must not take the fault..
386 if (unlikely(in_atomic() || !mm))
387 goto bad_area_nosemaphore;
390 /* When running in the kernel we expect faults to occur only to
391 * addresses in user space. All other faults represent errors in the
392 * kernel and should generate an OOPS. Unfortunatly, in the case of an
393 * erroneous fault occurring in a code path which already holds mmap_sem
394 * we will deadlock attempting to validate the fault against the
395 * address space. Luckily the kernel only validly references user
396 * space from well defined areas of code, which are listed in the
399 * As the vast majority of faults will be valid we will only perform
400 * the source reference check when there is a possibilty of a deadlock.
401 * Attempt to lock the address space, if we cannot we then validate the
402 * source. If this is invalid we can skip the address space check,
403 * thus avoiding the deadlock.
405 if (!down_read_trylock(&mm->mmap_sem)) {
406 if ((error_code & PF_USER) == 0 &&
407 !search_exception_tables(regs->rip))
408 goto bad_area_nosemaphore;
409 down_read(&mm->mmap_sem);
412 vma = find_vma(mm, address);
415 if (likely(vma->vm_start <= address))
417 if (!(vma->vm_flags & VM_GROWSDOWN))
419 if (error_code & 4) {
420 /* Allow userspace just enough access below the stack pointer
421 * to let the 'enter' instruction work.
423 if (address + 65536 + 32 * sizeof(unsigned long) < regs->rsp)
426 if (expand_stack(vma, address))
429 * Ok, we have a good vm_area for this memory access, so
433 info.si_code = SEGV_ACCERR;
435 switch (error_code & (PF_PROT|PF_WRITE)) {
436 default: /* 3: write, present */
438 case PF_WRITE: /* write, not present */
439 if (!(vma->vm_flags & VM_WRITE))
443 case PF_PROT: /* read, present */
445 case 0: /* read, not present */
446 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
451 * If for any reason at all we couldn't handle the fault,
452 * make sure we exit gracefully rather than endlessly redo
455 switch (handle_mm_fault(mm, vma, address, write)) {
462 case VM_FAULT_SIGBUS:
468 up_read(&mm->mmap_sem);
472 * Something tried to access memory that isn't in our memory map..
473 * Fix it, but check if it's kernel or user first..
476 up_read(&mm->mmap_sem);
478 bad_area_nosemaphore:
479 /* User mode accesses just cause a SIGSEGV */
480 if (error_code & PF_USER) {
481 if (is_prefetch(regs, address, error_code))
484 /* Work around K8 erratum #100 K8 in compat mode
485 occasionally jumps to illegal addresses >4GB. We
486 catch this here in the page fault handler because
487 these addresses are not reachable. Just detect this
488 case and return. Any code segment in LDT is
489 compatibility mode. */
490 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
494 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
496 "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
497 tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
498 tsk->comm, tsk->pid, address, regs->rip,
499 regs->rsp, error_code);
502 tsk->thread.cr2 = address;
503 /* Kernel addresses are always protection faults */
504 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
505 tsk->thread.trap_no = 14;
506 info.si_signo = SIGSEGV;
508 /* info.si_code has been set above */
509 info.si_addr = (void __user *)address;
510 force_sig_info(SIGSEGV, &info, tsk);
516 /* Are we prepared to handle this kernel fault? */
517 fixup = search_exception_tables(regs->rip);
519 regs->rip = fixup->fixup;
524 * Hall of shame of CPU/BIOS bugs.
527 if (is_prefetch(regs, address, error_code))
530 if (is_errata93(regs, address))
534 * Oops. The kernel tried to access some bad page. We'll have to
535 * terminate things with extreme prejudice.
538 flags = oops_begin();
540 if (address < PAGE_SIZE)
541 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
543 printk(KERN_ALERT "Unable to handle kernel paging request");
544 printk(" at %016lx RIP: \n" KERN_ALERT,address);
545 printk_address(regs->rip);
546 dump_pagetable(address);
547 tsk->thread.cr2 = address;
548 tsk->thread.trap_no = 14;
549 tsk->thread.error_code = error_code;
550 __die("Oops", regs, error_code);
551 /* Executive summary in case the body of the oops scrolled away */
552 printk(KERN_EMERG "CR2: %016lx\n", address);
557 * We ran out of memory, or some other thing happened to us that made
558 * us unable to handle the page fault gracefully.
561 up_read(&mm->mmap_sem);
562 if (is_init(current)) {
566 printk("VM: killing process %s\n", tsk->comm);
572 up_read(&mm->mmap_sem);
574 /* Kernel mode? Handle exceptions or die */
575 if (!(error_code & PF_USER))
578 tsk->thread.cr2 = address;
579 tsk->thread.error_code = error_code;
580 tsk->thread.trap_no = 14;
581 info.si_signo = SIGBUS;
583 info.si_code = BUS_ADRERR;
584 info.si_addr = (void __user *)address;
585 force_sig_info(SIGBUS, &info, tsk);
589 DEFINE_SPINLOCK(pgd_lock);
590 struct page *pgd_list;
592 void vmalloc_sync_all(void)
594 /* Note that races in the updates of insync and start aren't
596 insync can only get set bits added, and updates to start are only
597 improving performance (without affecting correctness if undone). */
598 static DECLARE_BITMAP(insync, PTRS_PER_PGD);
599 static unsigned long start = VMALLOC_START & PGDIR_MASK;
600 unsigned long address;
602 for (address = start; address <= VMALLOC_END; address += PGDIR_SIZE) {
603 if (!test_bit(pgd_index(address), insync)) {
604 const pgd_t *pgd_ref = pgd_offset_k(address);
607 if (pgd_none(*pgd_ref))
609 spin_lock(&pgd_lock);
610 for (page = pgd_list; page;
611 page = (struct page *)page->index) {
613 pgd = (pgd_t *)page_address(page) + pgd_index(address);
615 set_pgd(pgd, *pgd_ref);
617 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
619 spin_unlock(&pgd_lock);
620 set_bit(pgd_index(address), insync);
622 if (address == start)
623 start = address + PGDIR_SIZE;
625 /* Check that there is no need to do the same for the modules area. */
626 BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
627 BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
628 (__START_KERNEL & PGDIR_MASK)));
631 static int __init enable_pagefaulttrace(char *str)
633 page_fault_trace = 1;
636 __setup("pagefaulttrace", enable_pagefaulttrace);