2 * linux/arch/x86_64/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
10 * entry.S contains the system-call and fault low-level handling routines.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call.
15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al.
18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved.
25 * - CFI macros are used to generate dwarf2 unwind information for better
26 * backtraces. They don't change any code.
27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29 * There are unfortunately lots of special cases where some registers
30 * not touched. The macro is a big mess that should be cleaned up.
31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32 * Gives a full stack frame.
33 * - ENTRY/END Define functions in the symbol table.
34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35 * frame that is otherwise undefined after a SYSCALL
36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
40 #include <linux/linkage.h>
41 #include <asm/segment.h>
42 #include <asm/cache.h>
43 #include <asm/errno.h>
44 #include <asm/dwarf2.h>
45 #include <asm/calling.h>
46 #include <asm/asm-offsets.h>
48 #include <asm/unistd.h>
49 #include <asm/thread_info.h>
50 #include <asm/hw_irq.h>
52 #include <asm/irqflags.h>
53 #include <asm/paravirt.h>
57 #ifndef CONFIG_PREEMPT
58 #define retint_kernel retint_restore_args
61 #ifdef CONFIG_PARAVIRT
62 ENTRY(native_usergs_sysret64)
65 #endif /* CONFIG_PARAVIRT */
68 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
69 #ifdef CONFIG_TRACE_IRQFLAGS
70 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
78 * C code is not supposed to know about undefined top of stack. Every time
79 * a C function with an pt_regs argument is called from the SYSCALL based
80 * fast path FIXUP_TOP_OF_STACK is needed.
81 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
85 /* %rsp:at FRAMEEND */
86 .macro FIXUP_TOP_OF_STACK tmp
87 movq %gs:pda_oldrsp,\tmp
89 movq $__USER_DS,SS(%rsp)
90 movq $__USER_CS,CS(%rsp)
92 movq R11(%rsp),\tmp /* get eflags */
93 movq \tmp,EFLAGS(%rsp)
96 .macro RESTORE_TOP_OF_STACK tmp,offset=0
97 movq RSP-\offset(%rsp),\tmp
98 movq \tmp,%gs:pda_oldrsp
99 movq EFLAGS-\offset(%rsp),\tmp
100 movq \tmp,R11-\offset(%rsp)
103 .macro FAKE_STACK_FRAME child_rip
104 /* push in order ss, rsp, eflags, cs, rip */
106 pushq $__KERNEL_DS /* ss */
107 CFI_ADJUST_CFA_OFFSET 8
108 /*CFI_REL_OFFSET ss,0*/
110 CFI_ADJUST_CFA_OFFSET 8
112 pushq $(1<<9) /* eflags - interrupts on */
113 CFI_ADJUST_CFA_OFFSET 8
114 /*CFI_REL_OFFSET rflags,0*/
115 pushq $__KERNEL_CS /* cs */
116 CFI_ADJUST_CFA_OFFSET 8
117 /*CFI_REL_OFFSET cs,0*/
118 pushq \child_rip /* rip */
119 CFI_ADJUST_CFA_OFFSET 8
121 pushq %rax /* orig rax */
122 CFI_ADJUST_CFA_OFFSET 8
125 .macro UNFAKE_STACK_FRAME
127 CFI_ADJUST_CFA_OFFSET -(6*8)
130 .macro CFI_DEFAULT_STACK start=1
136 CFI_DEF_CFA_OFFSET SS+8
138 CFI_REL_OFFSET r15,R15
139 CFI_REL_OFFSET r14,R14
140 CFI_REL_OFFSET r13,R13
141 CFI_REL_OFFSET r12,R12
142 CFI_REL_OFFSET rbp,RBP
143 CFI_REL_OFFSET rbx,RBX
144 CFI_REL_OFFSET r11,R11
145 CFI_REL_OFFSET r10,R10
148 CFI_REL_OFFSET rax,RAX
149 CFI_REL_OFFSET rcx,RCX
150 CFI_REL_OFFSET rdx,RDX
151 CFI_REL_OFFSET rsi,RSI
152 CFI_REL_OFFSET rdi,RDI
153 CFI_REL_OFFSET rip,RIP
154 /*CFI_REL_OFFSET cs,CS*/
155 /*CFI_REL_OFFSET rflags,EFLAGS*/
156 CFI_REL_OFFSET rsp,RSP
157 /*CFI_REL_OFFSET ss,SS*/
160 * A newly forked process directly context switches into this.
165 push kernel_eflags(%rip)
166 CFI_ADJUST_CFA_OFFSET 4
167 popf # reset kernel eflags
168 CFI_ADJUST_CFA_OFFSET -4
170 GET_THREAD_INFO(%rcx)
171 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
175 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
176 je int_ret_from_sys_call
177 testl $_TIF_IA32,TI_flags(%rcx)
178 jnz int_ret_from_sys_call
179 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
180 jmp ret_from_sys_call
183 call syscall_trace_leave
184 GET_THREAD_INFO(%rcx)
190 * System call entry. Upto 6 arguments in registers are supported.
192 * SYSCALL does not save anything on the stack and does not change the
198 * rax system call number
200 * rcx return address for syscall/sysret, C arg3
203 * r10 arg3 (--> moved to rcx for C)
206 * r11 eflags for syscall/sysret, temporary for C
207 * r12-r15,rbp,rbx saved by C code, not touched.
209 * Interrupts are off on entry.
210 * Only called from user space.
212 * XXX if we had a free scratch register we could save the RSP into the stack frame
213 * and report it properly in ps. Unfortunately we haven't.
215 * When user can change the frames always force IRET. That is because
216 * it deals with uncanonical addresses better. SYSRET has trouble
217 * with them due to bugs in both AMD and Intel CPUs.
223 CFI_DEF_CFA rsp,PDA_STACKOFFSET
225 /*CFI_REGISTER rflags,r11*/
228 * A hypervisor implementation might want to use a label
229 * after the swapgs, so that it can do the swapgs
230 * for the guest and jump here on syscall.
232 ENTRY(system_call_after_swapgs)
234 movq %rsp,%gs:pda_oldrsp
235 movq %gs:pda_kernelstack,%rsp
237 * No need to follow this irqs off/on section - it's straight
240 ENABLE_INTERRUPTS(CLBR_NONE)
242 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
243 movq %rcx,RIP-ARGOFFSET(%rsp)
244 CFI_REL_OFFSET rip,RIP-ARGOFFSET
245 GET_THREAD_INFO(%rcx)
246 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP), \
249 cmpq $__NR_syscall_max,%rax
252 call *sys_call_table(,%rax,8) # XXX: rip relative
253 movq %rax,RAX-ARGOFFSET(%rsp)
255 * Syscall return path ending with SYSRET (fast path)
256 * Has incomplete stack frame and undefined top of stack.
259 movl $_TIF_ALLWORK_MASK,%edi
263 GET_THREAD_INFO(%rcx)
264 DISABLE_INTERRUPTS(CLBR_NONE)
266 movl TI_flags(%rcx),%edx
271 * sysretq will re-enable interrupts:
274 movq RIP-ARGOFFSET(%rsp),%rcx
276 RESTORE_ARGS 0,-ARG_SKIP,1
277 /*CFI_REGISTER rflags,r11*/
278 movq %gs:pda_oldrsp, %rsp
282 /* Handle reschedules */
283 /* edx: work, edi: workmask */
285 bt $TIF_NEED_RESCHED,%edx
288 ENABLE_INTERRUPTS(CLBR_NONE)
290 CFI_ADJUST_CFA_OFFSET 8
293 CFI_ADJUST_CFA_OFFSET -8
296 /* Handle a signal */
299 ENABLE_INTERRUPTS(CLBR_NONE)
300 testl $_TIF_DO_NOTIFY_MASK,%edx
303 /* Really a signal */
304 /* edx: work flags (arg3) */
305 leaq do_notify_resume(%rip),%rax
306 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
307 xorl %esi,%esi # oldset -> arg2
308 call ptregscall_common
309 1: movl $_TIF_NEED_RESCHED,%edi
310 /* Use IRET because user could have changed frame. This
311 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
312 DISABLE_INTERRUPTS(CLBR_NONE)
317 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
318 jmp ret_from_sys_call
320 /* Do syscall tracing */
323 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
324 FIXUP_TOP_OF_STACK %rdi
326 call syscall_trace_enter
327 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
329 cmpq $__NR_syscall_max,%rax
330 ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */
331 movq %r10,%rcx /* fixup for C */
332 call *sys_call_table(,%rax,8)
333 movq %rax,RAX-ARGOFFSET(%rsp)
334 /* Use IRET because user could have changed frame */
337 * Syscall return path ending with IRET.
338 * Has correct top of stack, but partial stack frame.
340 .globl int_ret_from_sys_call
341 int_ret_from_sys_call:
342 DISABLE_INTERRUPTS(CLBR_NONE)
344 testl $3,CS-ARGOFFSET(%rsp)
345 je retint_restore_args
346 movl $_TIF_ALLWORK_MASK,%edi
347 /* edi: mask to check */
350 GET_THREAD_INFO(%rcx)
351 movl TI_flags(%rcx),%edx
354 andl $~TS_COMPAT,TI_status(%rcx)
357 /* Either reschedule or signal or syscall exit tracking needed. */
358 /* First do a reschedule test. */
359 /* edx: work, edi: workmask */
361 bt $TIF_NEED_RESCHED,%edx
364 ENABLE_INTERRUPTS(CLBR_NONE)
366 CFI_ADJUST_CFA_OFFSET 8
369 CFI_ADJUST_CFA_OFFSET -8
370 DISABLE_INTERRUPTS(CLBR_NONE)
374 /* handle signals and tracing -- both require a full stack frame */
377 ENABLE_INTERRUPTS(CLBR_NONE)
379 /* Check for syscall exit trace */
380 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
383 CFI_ADJUST_CFA_OFFSET 8
384 leaq 8(%rsp),%rdi # &ptregs -> arg1
385 call syscall_trace_leave
387 CFI_ADJUST_CFA_OFFSET -8
388 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
392 testl $_TIF_DO_NOTIFY_MASK,%edx
394 movq %rsp,%rdi # &ptregs -> arg1
395 xorl %esi,%esi # oldset -> arg2
396 call do_notify_resume
397 1: movl $_TIF_NEED_RESCHED,%edi
400 DISABLE_INTERRUPTS(CLBR_NONE)
407 * Certain special system calls that need to save a complete full stack frame.
410 .macro PTREGSCALL label,func,arg
413 leaq \func(%rip),%rax
414 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
415 jmp ptregscall_common
421 PTREGSCALL stub_clone, sys_clone, %r8
422 PTREGSCALL stub_fork, sys_fork, %rdi
423 PTREGSCALL stub_vfork, sys_vfork, %rdi
424 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
425 PTREGSCALL stub_iopl, sys_iopl, %rsi
427 ENTRY(ptregscall_common)
429 CFI_ADJUST_CFA_OFFSET -8
430 CFI_REGISTER rip, r11
433 CFI_REGISTER rip, r15
434 FIXUP_TOP_OF_STACK %r11
436 RESTORE_TOP_OF_STACK %r11
438 CFI_REGISTER rip, r11
441 CFI_ADJUST_CFA_OFFSET 8
442 CFI_REL_OFFSET rip, 0
445 END(ptregscall_common)
450 CFI_ADJUST_CFA_OFFSET -8
451 CFI_REGISTER rip, r11
453 FIXUP_TOP_OF_STACK %r11
456 RESTORE_TOP_OF_STACK %r11
459 jmp int_ret_from_sys_call
464 * sigreturn is special because it needs to restore all registers on return.
465 * This cannot be done with SYSRET, so use the IRET return path instead.
467 ENTRY(stub_rt_sigreturn)
470 CFI_ADJUST_CFA_OFFSET -8
473 FIXUP_TOP_OF_STACK %r11
474 call sys_rt_sigreturn
475 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
477 jmp int_ret_from_sys_call
479 END(stub_rt_sigreturn)
482 * initial frame state for interrupts and exceptions
487 CFI_DEF_CFA rsp,SS+8-\ref
488 /*CFI_REL_OFFSET ss,SS-\ref*/
489 CFI_REL_OFFSET rsp,RSP-\ref
490 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
491 /*CFI_REL_OFFSET cs,CS-\ref*/
492 CFI_REL_OFFSET rip,RIP-\ref
495 /* initial frame state for interrupts (and exceptions without error code) */
496 #define INTR_FRAME _frame RIP
497 /* initial frame state for exceptions with error code (and interrupts with
498 vector already pushed) */
499 #define XCPT_FRAME _frame ORIG_RAX
502 * Interrupt entry/exit.
504 * Interrupt entry points save only callee clobbered registers in fast path.
506 * Entry runs with interrupts off.
509 /* 0(%rsp): interrupt number */
510 .macro interrupt func
513 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
515 CFI_ADJUST_CFA_OFFSET 8
516 CFI_REL_OFFSET rbp, 0
518 CFI_DEF_CFA_REGISTER rbp
522 /* irqcount is used to check if a CPU is already on an interrupt
523 stack or not. While this is essentially redundant with preempt_count
524 it is a little cheaper to use a separate counter in the PDA
525 (short of moving irq_enter into assembly, which would be too
527 1: incl %gs:pda_irqcount
528 cmoveq %gs:pda_irqstackptr,%rsp
529 push %rbp # backlink for old unwinder
531 * We entered an interrupt context - irqs are off:
537 ENTRY(common_interrupt)
540 /* 0(%rsp): oldrsp-ARGOFFSET */
542 DISABLE_INTERRUPTS(CLBR_NONE)
544 decl %gs:pda_irqcount
546 CFI_DEF_CFA_REGISTER rsp
547 CFI_ADJUST_CFA_OFFSET -8
549 GET_THREAD_INFO(%rcx)
550 testl $3,CS-ARGOFFSET(%rsp)
553 /* Interrupt came from user space */
555 * Has a correct top of stack, but a partial stack frame
556 * %rcx: thread info. Interrupts off.
558 retint_with_reschedule:
559 movl $_TIF_WORK_MASK,%edi
562 movl TI_flags(%rcx),%edx
567 retint_swapgs: /* return to user-space */
569 * The iretq could re-enable interrupts:
571 DISABLE_INTERRUPTS(CLBR_ANY)
576 retint_restore_args: /* return to kernel space */
577 DISABLE_INTERRUPTS(CLBR_ANY)
579 * The iretq could re-enable interrupts:
588 .section __ex_table, "a"
589 .quad irq_return, bad_iret
592 #ifdef CONFIG_PARAVIRT
596 .section __ex_table,"a"
597 .quad native_iret, bad_iret
604 * The iret traps when the %cs or %ss being restored is bogus.
605 * We've lost the original trap vector and error code.
606 * #GPF is the most likely one to get for an invalid selector.
607 * So pretend we completed the iret and took the #GPF in user mode.
609 * We are now running with the kernel GS after exception recovery.
610 * But error_entry expects us to have user GS to match the user %cs,
616 jmp general_protection
620 /* edi: workmask, edx: work */
623 bt $TIF_NEED_RESCHED,%edx
626 ENABLE_INTERRUPTS(CLBR_NONE)
628 CFI_ADJUST_CFA_OFFSET 8
631 CFI_ADJUST_CFA_OFFSET -8
632 GET_THREAD_INFO(%rcx)
633 DISABLE_INTERRUPTS(CLBR_NONE)
638 testl $_TIF_DO_NOTIFY_MASK,%edx
641 ENABLE_INTERRUPTS(CLBR_NONE)
643 movq $-1,ORIG_RAX(%rsp)
644 xorl %esi,%esi # oldset
645 movq %rsp,%rdi # &pt_regs
646 call do_notify_resume
648 DISABLE_INTERRUPTS(CLBR_NONE)
650 movl $_TIF_NEED_RESCHED,%edi
651 GET_THREAD_INFO(%rcx)
654 #ifdef CONFIG_PREEMPT
655 /* Returning to kernel space. Check if we need preemption */
656 /* rcx: threadinfo. interrupts off. */
658 cmpl $0,TI_preempt_count(%rcx)
659 jnz retint_restore_args
660 bt $TIF_NEED_RESCHED,TI_flags(%rcx)
661 jnc retint_restore_args
662 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
663 jnc retint_restore_args
664 call preempt_schedule_irq
669 END(common_interrupt)
674 .macro apicinterrupt num,func
677 CFI_ADJUST_CFA_OFFSET 8
683 ENTRY(thermal_interrupt)
684 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
685 END(thermal_interrupt)
687 ENTRY(threshold_interrupt)
688 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
689 END(threshold_interrupt)
692 ENTRY(reschedule_interrupt)
693 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
694 END(reschedule_interrupt)
696 .macro INVALIDATE_ENTRY num
697 ENTRY(invalidate_interrupt\num)
698 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
699 END(invalidate_interrupt\num)
711 ENTRY(call_function_interrupt)
712 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
713 END(call_function_interrupt)
714 ENTRY(irq_move_cleanup_interrupt)
715 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
716 END(irq_move_cleanup_interrupt)
719 ENTRY(apic_timer_interrupt)
720 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
721 END(apic_timer_interrupt)
723 ENTRY(uv_bau_message_intr1)
724 apicinterrupt 220,uv_bau_message_interrupt
725 END(uv_bau_message_intr1)
727 ENTRY(error_interrupt)
728 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
731 ENTRY(spurious_interrupt)
732 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
733 END(spurious_interrupt)
736 * Exception entry points.
740 PARAVIRT_ADJUST_EXCEPTION_FRAME
741 pushq $0 /* push error code/oldrax */
742 CFI_ADJUST_CFA_OFFSET 8
743 pushq %rax /* push real oldrax to the rdi slot */
744 CFI_ADJUST_CFA_OFFSET 8
751 .macro errorentry sym
753 PARAVIRT_ADJUST_EXCEPTION_FRAME
755 CFI_ADJUST_CFA_OFFSET 8
762 /* error code is on the stack already */
763 /* handle NMI like exceptions that can happen everywhere */
764 .macro paranoidentry sym, ist=0, irqtrace=1
768 movl $MSR_GS_BASE,%ecx
776 movq %gs:pda_data_offset, %rbp
779 movq ORIG_RAX(%rsp),%rsi
780 movq $-1,ORIG_RAX(%rsp)
782 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
786 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
788 DISABLE_INTERRUPTS(CLBR_NONE)
795 * "Paranoid" exit path from exception stack.
796 * Paranoid because this is used by NMIs and cannot take
797 * any kernel state for granted.
798 * We don't do kernel preemption checks here, because only
799 * NMI should be common and it does not enable IRQs and
800 * cannot get reschedule ticks.
802 * "trace" is 0 for the NMI handler only, because irq-tracing
803 * is fundamentally NMI-unsafe. (we cannot change the soft and
804 * hard flags at once, atomically)
806 .macro paranoidexit trace=1
807 /* ebx: no swapgs flag */
809 testl %ebx,%ebx /* swapgs needed? */
810 jnz paranoid_restore\trace
812 jnz paranoid_userspace\trace
813 paranoid_swapgs\trace:
818 paranoid_restore\trace:
821 paranoid_userspace\trace:
822 GET_THREAD_INFO(%rcx)
823 movl TI_flags(%rcx),%ebx
824 andl $_TIF_WORK_MASK,%ebx
825 jz paranoid_swapgs\trace
826 movq %rsp,%rdi /* &pt_regs */
828 movq %rax,%rsp /* switch stack for scheduling */
829 testl $_TIF_NEED_RESCHED,%ebx
830 jnz paranoid_schedule\trace
831 movl %ebx,%edx /* arg3: thread flags */
835 ENABLE_INTERRUPTS(CLBR_NONE)
836 xorl %esi,%esi /* arg2: oldset */
837 movq %rsp,%rdi /* arg1: &pt_regs */
838 call do_notify_resume
839 DISABLE_INTERRUPTS(CLBR_NONE)
843 jmp paranoid_userspace\trace
844 paranoid_schedule\trace:
848 ENABLE_INTERRUPTS(CLBR_ANY)
850 DISABLE_INTERRUPTS(CLBR_ANY)
854 jmp paranoid_userspace\trace
859 * Exception entry point. This expects an error code/orig_rax on the stack
860 * and the exception handler in %rax.
862 KPROBE_ENTRY(error_entry)
865 /* rdi slot contains rax, oldrax contains error code */
868 CFI_ADJUST_CFA_OFFSET (14*8)
870 CFI_REL_OFFSET rsi,RSI
871 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
874 CFI_REL_OFFSET rdx,RDX
876 CFI_REL_OFFSET rcx,RCX
877 movq %rsi,10*8(%rsp) /* store rax */
878 CFI_REL_OFFSET rax,RAX
884 CFI_REL_OFFSET r10,R10
886 CFI_REL_OFFSET r11,R11
888 CFI_REL_OFFSET rbx,RBX
890 CFI_REL_OFFSET rbp,RBP
892 CFI_REL_OFFSET r12,R12
894 CFI_REL_OFFSET r13,R13
896 CFI_REL_OFFSET r14,R14
898 CFI_REL_OFFSET r15,R15
906 CFI_REL_OFFSET rdi,RDI
908 movq ORIG_RAX(%rsp),%rsi /* get error code */
909 movq $-1,ORIG_RAX(%rsp)
911 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
915 DISABLE_INTERRUPTS(CLBR_NONE)
917 GET_THREAD_INFO(%rcx)
921 movl TI_flags(%rcx),%edx
922 movl $_TIF_WORK_MASK,%edi
930 /* There are two places in the kernel that can potentially fault with
931 usergs. Handle them here. The exception handlers after
932 iret run with kernel gs again, so don't set the user space flag.
933 B stepping K8s sometimes report an truncated RIP for IRET
934 exceptions returning to compat mode. Check for these here too. */
935 leaq irq_return(%rip),%rcx
938 movl %ecx,%ecx /* zero extend */
941 cmpq $gs_change,RIP(%rsp)
944 KPROBE_END(error_entry)
946 /* Reload gs selector with exception handling */
947 /* edi: new selector */
948 ENTRY(native_load_gs_index)
951 CFI_ADJUST_CFA_OFFSET 8
952 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
956 2: mfence /* workaround */
959 CFI_ADJUST_CFA_OFFSET -8
962 ENDPROC(native_load_gs_index)
964 .section __ex_table,"a"
966 .quad gs_change,bad_gs
969 /* running with kernelgs */
971 SWAPGS /* switch back to user gs */
978 * Create a kernel thread.
980 * C extern interface:
981 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
983 * asm input arguments:
984 * rdi: fn, rsi: arg, rdx: flags
988 FAKE_STACK_FRAME $child_rip
991 # rdi: flags, rsi: usp, rdx: will be &pt_regs
993 orq kernel_thread_flags(%rip),%rdi
1006 * It isn't worth to check for reschedule here,
1007 * so internally to the x86_64 port you can rely on kernel_thread()
1008 * not to reschedule the child before returning, this avoids the need
1009 * of hacks for example to fork off the per-CPU idle tasks.
1010 * [Hopefully no generic code relies on the reschedule -AK]
1016 ENDPROC(kernel_thread)
1019 pushq $0 # fake return address
1022 * Here we are in the child and the registers are set as they were
1023 * at kernel_thread() invocation in the parent.
1035 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1037 * C extern interface:
1038 * extern long execve(char *name, char **argv, char **envp)
1040 * asm input arguments:
1041 * rdi: name, rsi: argv, rdx: envp
1043 * We want to fallback into:
1044 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs *regs)
1046 * do_sys_execve asm fallback arguments:
1047 * rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
1049 ENTRY(kernel_execve)
1055 movq %rax, RAX(%rsp)
1058 je int_ret_from_sys_call
1063 ENDPROC(kernel_execve)
1065 KPROBE_ENTRY(page_fault)
1066 errorentry do_page_fault
1067 KPROBE_END(page_fault)
1069 ENTRY(coprocessor_error)
1070 zeroentry do_coprocessor_error
1071 END(coprocessor_error)
1073 ENTRY(simd_coprocessor_error)
1074 zeroentry do_simd_coprocessor_error
1075 END(simd_coprocessor_error)
1077 ENTRY(device_not_available)
1078 zeroentry math_state_restore
1079 END(device_not_available)
1081 /* runs on exception stack */
1085 CFI_ADJUST_CFA_OFFSET 8
1086 paranoidentry do_debug, DEBUG_STACK
1090 /* runs on exception stack */
1094 CFI_ADJUST_CFA_OFFSET 8
1095 paranoidentry do_nmi, 0, 0
1096 #ifdef CONFIG_TRACE_IRQFLAGS
1107 CFI_ADJUST_CFA_OFFSET 8
1108 paranoidentry do_int3, DEBUG_STACK
1114 zeroentry do_overflow
1122 zeroentry do_invalid_op
1125 ENTRY(coprocessor_segment_overrun)
1126 zeroentry do_coprocessor_segment_overrun
1127 END(coprocessor_segment_overrun)
1129 /* runs on exception stack */
1132 paranoidentry do_double_fault
1138 errorentry do_invalid_TSS
1141 ENTRY(segment_not_present)
1142 errorentry do_segment_not_present
1143 END(segment_not_present)
1145 /* runs on exception stack */
1146 ENTRY(stack_segment)
1148 paranoidentry do_stack_segment
1153 KPROBE_ENTRY(general_protection)
1154 errorentry do_general_protection
1155 KPROBE_END(general_protection)
1157 ENTRY(alignment_check)
1158 errorentry do_alignment_check
1159 END(alignment_check)
1162 zeroentry do_divide_error
1165 ENTRY(spurious_interrupt_bug)
1166 zeroentry do_spurious_interrupt_bug
1167 END(spurious_interrupt_bug)
1169 #ifdef CONFIG_X86_MCE
1170 /* runs on exception stack */
1171 ENTRY(machine_check)
1174 CFI_ADJUST_CFA_OFFSET 8
1175 paranoidentry do_machine_check
1181 /* Call softirq on interrupt stack. Interrupts are off. */
1185 CFI_ADJUST_CFA_OFFSET 8
1186 CFI_REL_OFFSET rbp,0
1188 CFI_DEF_CFA_REGISTER rbp
1189 incl %gs:pda_irqcount
1190 cmove %gs:pda_irqstackptr,%rsp
1191 push %rbp # backlink for old unwinder
1194 CFI_DEF_CFA_REGISTER rsp
1195 CFI_ADJUST_CFA_OFFSET -8
1196 decl %gs:pda_irqcount
1199 ENDPROC(call_softirq)
1201 KPROBE_ENTRY(ignore_sysret)
1206 ENDPROC(ignore_sysret)