2 * linux/arch/x86_64/entry.S
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
10 * entry.S contains the system-call and fault low-level handling routines.
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call.
15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al.
18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved.
25 * - CFI macros are used to generate dwarf2 unwind information for better
26 * backtraces. They don't change any code.
27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29 * There are unfortunately lots of special cases where some registers
30 * not touched. The macro is a big mess that should be cleaned up.
31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32 * Gives a full stack frame.
33 * - ENTRY/END Define functions in the symbol table.
34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35 * frame that is otherwise undefined after a SYSCALL
36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
40 #include <linux/linkage.h>
41 #include <asm/segment.h>
42 #include <asm/cache.h>
43 #include <asm/errno.h>
44 #include <asm/dwarf2.h>
45 #include <asm/calling.h>
46 #include <asm/asm-offsets.h>
48 #include <asm/unistd.h>
49 #include <asm/thread_info.h>
50 #include <asm/hw_irq.h>
52 #include <asm/irqflags.h>
53 #include <asm/paravirt.h>
57 #ifndef CONFIG_PREEMPT
58 #define retint_kernel retint_restore_args
61 #ifdef CONFIG_PARAVIRT
62 ENTRY(native_irq_enable_syscall_ret)
63 movq %gs:pda_oldrsp,%rsp
66 #endif /* CONFIG_PARAVIRT */
69 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
70 #ifdef CONFIG_TRACE_IRQFLAGS
71 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
79 * C code is not supposed to know about undefined top of stack. Every time
80 * a C function with an pt_regs argument is called from the SYSCALL based
81 * fast path FIXUP_TOP_OF_STACK is needed.
82 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
86 /* %rsp:at FRAMEEND */
87 .macro FIXUP_TOP_OF_STACK tmp
88 movq %gs:pda_oldrsp,\tmp
90 movq $__USER_DS,SS(%rsp)
91 movq $__USER_CS,CS(%rsp)
93 movq R11(%rsp),\tmp /* get eflags */
94 movq \tmp,EFLAGS(%rsp)
97 .macro RESTORE_TOP_OF_STACK tmp,offset=0
98 movq RSP-\offset(%rsp),\tmp
99 movq \tmp,%gs:pda_oldrsp
100 movq EFLAGS-\offset(%rsp),\tmp
101 movq \tmp,R11-\offset(%rsp)
104 .macro FAKE_STACK_FRAME child_rip
105 /* push in order ss, rsp, eflags, cs, rip */
108 CFI_ADJUST_CFA_OFFSET 8
109 /*CFI_REL_OFFSET ss,0*/
111 CFI_ADJUST_CFA_OFFSET 8
113 pushq $(1<<9) /* eflags - interrupts on */
114 CFI_ADJUST_CFA_OFFSET 8
115 /*CFI_REL_OFFSET rflags,0*/
116 pushq $__KERNEL_CS /* cs */
117 CFI_ADJUST_CFA_OFFSET 8
118 /*CFI_REL_OFFSET cs,0*/
119 pushq \child_rip /* rip */
120 CFI_ADJUST_CFA_OFFSET 8
122 pushq %rax /* orig rax */
123 CFI_ADJUST_CFA_OFFSET 8
126 .macro UNFAKE_STACK_FRAME
128 CFI_ADJUST_CFA_OFFSET -(6*8)
131 .macro CFI_DEFAULT_STACK start=1
137 CFI_DEF_CFA_OFFSET SS+8
139 CFI_REL_OFFSET r15,R15
140 CFI_REL_OFFSET r14,R14
141 CFI_REL_OFFSET r13,R13
142 CFI_REL_OFFSET r12,R12
143 CFI_REL_OFFSET rbp,RBP
144 CFI_REL_OFFSET rbx,RBX
145 CFI_REL_OFFSET r11,R11
146 CFI_REL_OFFSET r10,R10
149 CFI_REL_OFFSET rax,RAX
150 CFI_REL_OFFSET rcx,RCX
151 CFI_REL_OFFSET rdx,RDX
152 CFI_REL_OFFSET rsi,RSI
153 CFI_REL_OFFSET rdi,RDI
154 CFI_REL_OFFSET rip,RIP
155 /*CFI_REL_OFFSET cs,CS*/
156 /*CFI_REL_OFFSET rflags,EFLAGS*/
157 CFI_REL_OFFSET rsp,RSP
158 /*CFI_REL_OFFSET ss,SS*/
161 * A newly forked process directly context switches into this.
166 push kernel_eflags(%rip)
167 CFI_ADJUST_CFA_OFFSET 4
168 popf # reset kernel eflags
169 CFI_ADJUST_CFA_OFFSET -4
171 GET_THREAD_INFO(%rcx)
172 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
176 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
177 je int_ret_from_sys_call
178 testl $_TIF_IA32,threadinfo_flags(%rcx)
179 jnz int_ret_from_sys_call
180 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
181 jmp ret_from_sys_call
184 call syscall_trace_leave
185 GET_THREAD_INFO(%rcx)
191 * System call entry. Upto 6 arguments in registers are supported.
193 * SYSCALL does not save anything on the stack and does not change the
199 * rax system call number
201 * rcx return address for syscall/sysret, C arg3
204 * r10 arg3 (--> moved to rcx for C)
207 * r11 eflags for syscall/sysret, temporary for C
208 * r12-r15,rbp,rbx saved by C code, not touched.
210 * Interrupts are off on entry.
211 * Only called from user space.
213 * XXX if we had a free scratch register we could save the RSP into the stack frame
214 * and report it properly in ps. Unfortunately we haven't.
216 * When user can change the frames always force IRET. That is because
217 * it deals with uncanonical addresses better. SYSRET has trouble
218 * with them due to bugs in both AMD and Intel CPUs.
224 CFI_DEF_CFA rsp,PDA_STACKOFFSET
226 /*CFI_REGISTER rflags,r11*/
229 * A hypervisor implementation might want to use a label
230 * after the swapgs, so that it can do the swapgs
231 * for the guest and jump here on syscall.
233 ENTRY(system_call_after_swapgs)
235 movq %rsp,%gs:pda_oldrsp
236 movq %gs:pda_kernelstack,%rsp
238 * No need to follow this irqs off/on section - it's straight
241 ENABLE_INTERRUPTS(CLBR_NONE)
243 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
244 movq %rcx,RIP-ARGOFFSET(%rsp)
245 CFI_REL_OFFSET rip,RIP-ARGOFFSET
246 GET_THREAD_INFO(%rcx)
247 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
249 cmpq $__NR_syscall_max,%rax
252 call *sys_call_table(,%rax,8) # XXX: rip relative
253 movq %rax,RAX-ARGOFFSET(%rsp)
255 * Syscall return path ending with SYSRET (fast path)
256 * Has incomplete stack frame and undefined top of stack.
259 movl $_TIF_ALLWORK_MASK,%edi
263 GET_THREAD_INFO(%rcx)
264 DISABLE_INTERRUPTS(CLBR_NONE)
266 movl threadinfo_flags(%rcx),%edx
271 * sysretq will re-enable interrupts:
274 movq RIP-ARGOFFSET(%rsp),%rcx
276 RESTORE_ARGS 0,-ARG_SKIP,1
277 /*CFI_REGISTER rflags,r11*/
278 ENABLE_INTERRUPTS_SYSCALL_RET
281 /* Handle reschedules */
282 /* edx: work, edi: workmask */
284 bt $TIF_NEED_RESCHED,%edx
287 ENABLE_INTERRUPTS(CLBR_NONE)
289 CFI_ADJUST_CFA_OFFSET 8
292 CFI_ADJUST_CFA_OFFSET -8
295 /* Handle a signal */
298 ENABLE_INTERRUPTS(CLBR_NONE)
299 testl $_TIF_DO_NOTIFY_MASK,%edx
302 /* Really a signal */
303 /* edx: work flags (arg3) */
304 leaq do_notify_resume(%rip),%rax
305 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
306 xorl %esi,%esi # oldset -> arg2
307 call ptregscall_common
308 1: movl $_TIF_NEED_RESCHED,%edi
309 /* Use IRET because user could have changed frame. This
310 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
311 DISABLE_INTERRUPTS(CLBR_NONE)
316 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
317 jmp ret_from_sys_call
319 /* Do syscall tracing */
322 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
323 FIXUP_TOP_OF_STACK %rdi
325 call syscall_trace_enter
326 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
328 cmpq $__NR_syscall_max,%rax
329 ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */
330 movq %r10,%rcx /* fixup for C */
331 call *sys_call_table(,%rax,8)
332 movq %rax,RAX-ARGOFFSET(%rsp)
333 /* Use IRET because user could have changed frame */
336 * Syscall return path ending with IRET.
337 * Has correct top of stack, but partial stack frame.
339 .globl int_ret_from_sys_call
340 int_ret_from_sys_call:
341 DISABLE_INTERRUPTS(CLBR_NONE)
343 testl $3,CS-ARGOFFSET(%rsp)
344 je retint_restore_args
345 movl $_TIF_ALLWORK_MASK,%edi
346 /* edi: mask to check */
349 GET_THREAD_INFO(%rcx)
350 movl threadinfo_flags(%rcx),%edx
353 andl $~TS_COMPAT,threadinfo_status(%rcx)
356 /* Either reschedule or signal or syscall exit tracking needed. */
357 /* First do a reschedule test. */
358 /* edx: work, edi: workmask */
360 bt $TIF_NEED_RESCHED,%edx
363 ENABLE_INTERRUPTS(CLBR_NONE)
365 CFI_ADJUST_CFA_OFFSET 8
368 CFI_ADJUST_CFA_OFFSET -8
369 DISABLE_INTERRUPTS(CLBR_NONE)
373 /* handle signals and tracing -- both require a full stack frame */
376 ENABLE_INTERRUPTS(CLBR_NONE)
378 /* Check for syscall exit trace */
379 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
382 CFI_ADJUST_CFA_OFFSET 8
383 leaq 8(%rsp),%rdi # &ptregs -> arg1
384 call syscall_trace_leave
386 CFI_ADJUST_CFA_OFFSET -8
387 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
391 testl $_TIF_DO_NOTIFY_MASK,%edx
393 movq %rsp,%rdi # &ptregs -> arg1
394 xorl %esi,%esi # oldset -> arg2
395 call do_notify_resume
396 1: movl $_TIF_NEED_RESCHED,%edi
399 DISABLE_INTERRUPTS(CLBR_NONE)
406 * Certain special system calls that need to save a complete full stack frame.
409 .macro PTREGSCALL label,func,arg
412 leaq \func(%rip),%rax
413 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
414 jmp ptregscall_common
420 PTREGSCALL stub_clone, sys_clone, %r8
421 PTREGSCALL stub_fork, sys_fork, %rdi
422 PTREGSCALL stub_vfork, sys_vfork, %rdi
423 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
424 PTREGSCALL stub_iopl, sys_iopl, %rsi
426 ENTRY(ptregscall_common)
428 CFI_ADJUST_CFA_OFFSET -8
429 CFI_REGISTER rip, r11
432 CFI_REGISTER rip, r15
433 FIXUP_TOP_OF_STACK %r11
435 RESTORE_TOP_OF_STACK %r11
437 CFI_REGISTER rip, r11
440 CFI_ADJUST_CFA_OFFSET 8
441 CFI_REL_OFFSET rip, 0
444 END(ptregscall_common)
449 CFI_ADJUST_CFA_OFFSET -8
450 CFI_REGISTER rip, r11
452 FIXUP_TOP_OF_STACK %r11
455 RESTORE_TOP_OF_STACK %r11
458 jmp int_ret_from_sys_call
463 * sigreturn is special because it needs to restore all registers on return.
464 * This cannot be done with SYSRET, so use the IRET return path instead.
466 ENTRY(stub_rt_sigreturn)
469 CFI_ADJUST_CFA_OFFSET -8
472 FIXUP_TOP_OF_STACK %r11
473 call sys_rt_sigreturn
474 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
476 jmp int_ret_from_sys_call
478 END(stub_rt_sigreturn)
481 * initial frame state for interrupts and exceptions
486 CFI_DEF_CFA rsp,SS+8-\ref
487 /*CFI_REL_OFFSET ss,SS-\ref*/
488 CFI_REL_OFFSET rsp,RSP-\ref
489 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
490 /*CFI_REL_OFFSET cs,CS-\ref*/
491 CFI_REL_OFFSET rip,RIP-\ref
494 /* initial frame state for interrupts (and exceptions without error code) */
495 #define INTR_FRAME _frame RIP
496 /* initial frame state for exceptions with error code (and interrupts with
497 vector already pushed) */
498 #define XCPT_FRAME _frame ORIG_RAX
501 * Interrupt entry/exit.
503 * Interrupt entry points save only callee clobbered registers in fast path.
505 * Entry runs with interrupts off.
508 /* 0(%rsp): interrupt number */
509 .macro interrupt func
512 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
514 CFI_ADJUST_CFA_OFFSET 8
515 CFI_REL_OFFSET rbp, 0
517 CFI_DEF_CFA_REGISTER rbp
521 /* irqcount is used to check if a CPU is already on an interrupt
522 stack or not. While this is essentially redundant with preempt_count
523 it is a little cheaper to use a separate counter in the PDA
524 (short of moving irq_enter into assembly, which would be too
526 1: incl %gs:pda_irqcount
527 cmoveq %gs:pda_irqstackptr,%rsp
528 push %rbp # backlink for old unwinder
530 * We entered an interrupt context - irqs are off:
536 ENTRY(common_interrupt)
539 /* 0(%rsp): oldrsp-ARGOFFSET */
541 DISABLE_INTERRUPTS(CLBR_NONE)
543 decl %gs:pda_irqcount
545 CFI_DEF_CFA_REGISTER rsp
546 CFI_ADJUST_CFA_OFFSET -8
548 GET_THREAD_INFO(%rcx)
549 testl $3,CS-ARGOFFSET(%rsp)
552 /* Interrupt came from user space */
554 * Has a correct top of stack, but a partial stack frame
555 * %rcx: thread info. Interrupts off.
557 retint_with_reschedule:
558 movl $_TIF_WORK_MASK,%edi
561 movl threadinfo_flags(%rcx),%edx
566 retint_swapgs: /* return to user-space */
568 * The iretq could re-enable interrupts:
570 DISABLE_INTERRUPTS(CLBR_ANY)
575 retint_restore_args: /* return to kernel space */
576 DISABLE_INTERRUPTS(CLBR_ANY)
578 * The iretq could re-enable interrupts:
587 .section __ex_table, "a"
588 .quad irq_return, bad_iret
591 #ifdef CONFIG_PARAVIRT
595 .section __ex_table,"a"
596 .quad native_iret, bad_iret
603 * The iret traps when the %cs or %ss being restored is bogus.
604 * We've lost the original trap vector and error code.
605 * #GPF is the most likely one to get for an invalid selector.
606 * So pretend we completed the iret and took the #GPF in user mode.
608 * We are now running with the kernel GS after exception recovery.
609 * But error_entry expects us to have user GS to match the user %cs,
615 jmp general_protection
619 /* edi: workmask, edx: work */
622 bt $TIF_NEED_RESCHED,%edx
625 ENABLE_INTERRUPTS(CLBR_NONE)
627 CFI_ADJUST_CFA_OFFSET 8
630 CFI_ADJUST_CFA_OFFSET -8
631 GET_THREAD_INFO(%rcx)
632 DISABLE_INTERRUPTS(CLBR_NONE)
637 testl $_TIF_DO_NOTIFY_MASK,%edx
640 ENABLE_INTERRUPTS(CLBR_NONE)
642 movq $-1,ORIG_RAX(%rsp)
643 xorl %esi,%esi # oldset
644 movq %rsp,%rdi # &pt_regs
645 call do_notify_resume
647 DISABLE_INTERRUPTS(CLBR_NONE)
649 movl $_TIF_NEED_RESCHED,%edi
650 GET_THREAD_INFO(%rcx)
653 #ifdef CONFIG_PREEMPT
654 /* Returning to kernel space. Check if we need preemption */
655 /* rcx: threadinfo. interrupts off. */
657 cmpl $0,threadinfo_preempt_count(%rcx)
658 jnz retint_restore_args
659 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
660 jnc retint_restore_args
661 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
662 jnc retint_restore_args
663 call preempt_schedule_irq
668 END(common_interrupt)
673 .macro apicinterrupt num,func
676 CFI_ADJUST_CFA_OFFSET 8
682 ENTRY(thermal_interrupt)
683 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
684 END(thermal_interrupt)
686 ENTRY(threshold_interrupt)
687 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
688 END(threshold_interrupt)
691 ENTRY(reschedule_interrupt)
692 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
693 END(reschedule_interrupt)
695 .macro INVALIDATE_ENTRY num
696 ENTRY(invalidate_interrupt\num)
697 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
698 END(invalidate_interrupt\num)
710 ENTRY(call_function_interrupt)
711 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
712 END(call_function_interrupt)
713 ENTRY(irq_move_cleanup_interrupt)
714 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
715 END(irq_move_cleanup_interrupt)
718 ENTRY(apic_timer_interrupt)
719 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
720 END(apic_timer_interrupt)
722 ENTRY(error_interrupt)
723 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
726 ENTRY(spurious_interrupt)
727 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
728 END(spurious_interrupt)
731 * Exception entry points.
735 pushq $0 /* push error code/oldrax */
736 CFI_ADJUST_CFA_OFFSET 8
737 pushq %rax /* push real oldrax to the rdi slot */
738 CFI_ADJUST_CFA_OFFSET 8
745 .macro errorentry sym
748 CFI_ADJUST_CFA_OFFSET 8
755 /* error code is on the stack already */
756 /* handle NMI like exceptions that can happen everywhere */
757 .macro paranoidentry sym, ist=0, irqtrace=1
761 movl $MSR_GS_BASE,%ecx
769 movq %gs:pda_data_offset, %rbp
772 movq ORIG_RAX(%rsp),%rsi
773 movq $-1,ORIG_RAX(%rsp)
775 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
779 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
781 DISABLE_INTERRUPTS(CLBR_NONE)
788 * "Paranoid" exit path from exception stack.
789 * Paranoid because this is used by NMIs and cannot take
790 * any kernel state for granted.
791 * We don't do kernel preemption checks here, because only
792 * NMI should be common and it does not enable IRQs and
793 * cannot get reschedule ticks.
795 * "trace" is 0 for the NMI handler only, because irq-tracing
796 * is fundamentally NMI-unsafe. (we cannot change the soft and
797 * hard flags at once, atomically)
799 .macro paranoidexit trace=1
800 /* ebx: no swapgs flag */
802 testl %ebx,%ebx /* swapgs needed? */
803 jnz paranoid_restore\trace
805 jnz paranoid_userspace\trace
806 paranoid_swapgs\trace:
811 paranoid_restore\trace:
814 paranoid_userspace\trace:
815 GET_THREAD_INFO(%rcx)
816 movl threadinfo_flags(%rcx),%ebx
817 andl $_TIF_WORK_MASK,%ebx
818 jz paranoid_swapgs\trace
819 movq %rsp,%rdi /* &pt_regs */
821 movq %rax,%rsp /* switch stack for scheduling */
822 testl $_TIF_NEED_RESCHED,%ebx
823 jnz paranoid_schedule\trace
824 movl %ebx,%edx /* arg3: thread flags */
828 ENABLE_INTERRUPTS(CLBR_NONE)
829 xorl %esi,%esi /* arg2: oldset */
830 movq %rsp,%rdi /* arg1: &pt_regs */
831 call do_notify_resume
832 DISABLE_INTERRUPTS(CLBR_NONE)
836 jmp paranoid_userspace\trace
837 paranoid_schedule\trace:
841 ENABLE_INTERRUPTS(CLBR_ANY)
843 DISABLE_INTERRUPTS(CLBR_ANY)
847 jmp paranoid_userspace\trace
852 * Exception entry point. This expects an error code/orig_rax on the stack
853 * and the exception handler in %rax.
855 KPROBE_ENTRY(error_entry)
858 /* rdi slot contains rax, oldrax contains error code */
861 CFI_ADJUST_CFA_OFFSET (14*8)
863 CFI_REL_OFFSET rsi,RSI
864 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
867 CFI_REL_OFFSET rdx,RDX
869 CFI_REL_OFFSET rcx,RCX
870 movq %rsi,10*8(%rsp) /* store rax */
871 CFI_REL_OFFSET rax,RAX
877 CFI_REL_OFFSET r10,R10
879 CFI_REL_OFFSET r11,R11
881 CFI_REL_OFFSET rbx,RBX
883 CFI_REL_OFFSET rbp,RBP
885 CFI_REL_OFFSET r12,R12
887 CFI_REL_OFFSET r13,R13
889 CFI_REL_OFFSET r14,R14
891 CFI_REL_OFFSET r15,R15
899 CFI_REL_OFFSET rdi,RDI
901 movq ORIG_RAX(%rsp),%rsi /* get error code */
902 movq $-1,ORIG_RAX(%rsp)
904 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
908 DISABLE_INTERRUPTS(CLBR_NONE)
910 GET_THREAD_INFO(%rcx)
914 movl threadinfo_flags(%rcx),%edx
915 movl $_TIF_WORK_MASK,%edi
923 /* There are two places in the kernel that can potentially fault with
924 usergs. Handle them here. The exception handlers after
925 iret run with kernel gs again, so don't set the user space flag.
926 B stepping K8s sometimes report an truncated RIP for IRET
927 exceptions returning to compat mode. Check for these here too. */
928 leaq irq_return(%rip),%rbp
931 movl %ebp,%ebp /* zero extend */
934 cmpq $gs_change,RIP(%rsp)
937 KPROBE_END(error_entry)
939 /* Reload gs selector with exception handling */
940 /* edi: new selector */
944 CFI_ADJUST_CFA_OFFSET 8
945 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
949 2: mfence /* workaround */
952 CFI_ADJUST_CFA_OFFSET -8
955 ENDPROC(load_gs_index)
957 .section __ex_table,"a"
959 .quad gs_change,bad_gs
962 /* running with kernelgs */
964 SWAPGS /* switch back to user gs */
971 * Create a kernel thread.
973 * C extern interface:
974 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
976 * asm input arguments:
977 * rdi: fn, rsi: arg, rdx: flags
981 FAKE_STACK_FRAME $child_rip
984 # rdi: flags, rsi: usp, rdx: will be &pt_regs
986 orq kernel_thread_flags(%rip),%rdi
999 * It isn't worth to check for reschedule here,
1000 * so internally to the x86_64 port you can rely on kernel_thread()
1001 * not to reschedule the child before returning, this avoids the need
1002 * of hacks for example to fork off the per-CPU idle tasks.
1003 * [Hopefully no generic code relies on the reschedule -AK]
1009 ENDPROC(kernel_thread)
1012 pushq $0 # fake return address
1015 * Here we are in the child and the registers are set as they were
1016 * at kernel_thread() invocation in the parent.
1028 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1030 * C extern interface:
1031 * extern long execve(char *name, char **argv, char **envp)
1033 * asm input arguments:
1034 * rdi: name, rsi: argv, rdx: envp
1036 * We want to fallback into:
1037 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs *regs)
1039 * do_sys_execve asm fallback arguments:
1040 * rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
1042 ENTRY(kernel_execve)
1048 movq %rax, RAX(%rsp)
1051 je int_ret_from_sys_call
1056 ENDPROC(kernel_execve)
1058 KPROBE_ENTRY(page_fault)
1059 errorentry do_page_fault
1060 KPROBE_END(page_fault)
1062 ENTRY(coprocessor_error)
1063 zeroentry do_coprocessor_error
1064 END(coprocessor_error)
1066 ENTRY(simd_coprocessor_error)
1067 zeroentry do_simd_coprocessor_error
1068 END(simd_coprocessor_error)
1070 ENTRY(device_not_available)
1071 zeroentry math_state_restore
1072 END(device_not_available)
1074 /* runs on exception stack */
1078 CFI_ADJUST_CFA_OFFSET 8
1079 paranoidentry do_debug, DEBUG_STACK
1083 /* runs on exception stack */
1087 CFI_ADJUST_CFA_OFFSET 8
1088 paranoidentry do_nmi, 0, 0
1089 #ifdef CONFIG_TRACE_IRQFLAGS
1100 CFI_ADJUST_CFA_OFFSET 8
1101 paranoidentry do_int3, DEBUG_STACK
1107 zeroentry do_overflow
1115 zeroentry do_invalid_op
1118 ENTRY(coprocessor_segment_overrun)
1119 zeroentry do_coprocessor_segment_overrun
1120 END(coprocessor_segment_overrun)
1122 /* runs on exception stack */
1125 paranoidentry do_double_fault
1131 errorentry do_invalid_TSS
1134 ENTRY(segment_not_present)
1135 errorentry do_segment_not_present
1136 END(segment_not_present)
1138 /* runs on exception stack */
1139 ENTRY(stack_segment)
1141 paranoidentry do_stack_segment
1146 KPROBE_ENTRY(general_protection)
1147 errorentry do_general_protection
1148 KPROBE_END(general_protection)
1150 ENTRY(alignment_check)
1151 errorentry do_alignment_check
1152 END(alignment_check)
1155 zeroentry do_divide_error
1158 ENTRY(spurious_interrupt_bug)
1159 zeroentry do_spurious_interrupt_bug
1160 END(spurious_interrupt_bug)
1162 #ifdef CONFIG_X86_MCE
1163 /* runs on exception stack */
1164 ENTRY(machine_check)
1167 CFI_ADJUST_CFA_OFFSET 8
1168 paranoidentry do_machine_check
1174 /* Call softirq on interrupt stack. Interrupts are off. */
1178 CFI_ADJUST_CFA_OFFSET 8
1179 CFI_REL_OFFSET rbp,0
1181 CFI_DEF_CFA_REGISTER rbp
1182 incl %gs:pda_irqcount
1183 cmove %gs:pda_irqstackptr,%rsp
1184 push %rbp # backlink for old unwinder
1187 CFI_DEF_CFA_REGISTER rsp
1188 CFI_ADJUST_CFA_OFFSET -8
1189 decl %gs:pda_irqcount
1192 ENDPROC(call_softirq)
1194 KPROBE_ENTRY(ignore_sysret)
1199 ENDPROC(ignore_sysret)