ia64/xen-unstable

view linux-2.6-xen-sparse/arch/xen/x86_64/kernel/entry.S @ 7132:b9e5573785bb

Don't define vectors for IPIs we don't use in Xen
Signed-off-by: ian@xensource.com
author iap10@firebug.cl.cam.ac.uk
date Thu Sep 29 16:57:23 2005 +0100 (2005-09-29)
parents 06d84bf87159
children 61b3b357d827
line source
1 /*
2 * linux/arch/x86_64/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7 *
8 * $Id$
9 *
10 * Jun Nakajima <jun.nakajima@intel.com>
11 * Asit Mallick <asit.k.mallick@intel.com>
12 * Modified for Xen
13 */
15 /*
16 * entry.S contains the system-call and fault low-level handling routines.
17 *
18 * NOTE: This code handles signal-recognition, which happens every time
19 * after an interrupt and after each system call.
20 *
21 * Normal syscalls and interrupts don't save a full stack frame, this is
22 * only done for syscall tracing, signals or fork/exec et.al.
23 *
24 * A note on terminology:
25 * - top of stack: Architecture defined interrupt frame from SS to RIP
26 * at the top of the kernel process stack.
27 * - partial stack frame: partially saved registers upto R11.
28 * - full stack frame: Like partial stack frame, but all register saved.
29 *
30 * TODO:
31 * - schedule it carefully for the final hardware.
32 */
34 #define ASSEMBLY 1
35 #include <linux/config.h>
36 #include <linux/linkage.h>
37 #include <asm/segment.h>
38 #include <asm/smp.h>
39 #include <asm/cache.h>
40 #include <asm/errno.h>
41 #include <asm/dwarf2.h>
42 #include <asm/calling.h>
43 #include <asm/asm_offset.h>
44 #include <asm/msr.h>
45 #include <asm/unistd.h>
46 #include <asm/thread_info.h>
47 #include <asm/hw_irq.h>
48 #include <asm/errno.h>
49 #include <asm-xen/xen-public/arch-x86_64.h>
51 #include "irq_vectors.h"
53 #include "xen_entry.S"
55 .code64
57 #ifndef CONFIG_PREEMPT
58 #define retint_kernel retint_restore_args
59 #endif
61 /*
62 * C code is not supposed to know about undefined top of stack. Every time
63 * a C function with an pt_regs argument is called from the SYSCALL based
64 * fast path FIXUP_TOP_OF_STACK is needed.
65 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
66 * manipulation.
67 */
69 /* %rsp:at FRAMEEND */
70 .macro FIXUP_TOP_OF_STACK tmp
71 movq $__USER_CS,CS(%rsp)
72 movq $-1,RCX(%rsp)
73 .endm
75 .macro RESTORE_TOP_OF_STACK tmp,offset=0
76 .endm
78 .macro FAKE_STACK_FRAME child_rip
79 /* push in order ss, rsp, eflags, cs, rip */
80 xorq %rax, %rax
81 pushq %rax /* ss */
82 CFI_ADJUST_CFA_OFFSET 8
83 pushq %rax /* rsp */
84 CFI_ADJUST_CFA_OFFSET 8
85 CFI_OFFSET rip,0
86 pushq $(1<<9) /* eflags - interrupts on */
87 CFI_ADJUST_CFA_OFFSET 8
88 pushq $__KERNEL_CS /* cs */
89 CFI_ADJUST_CFA_OFFSET 8
90 pushq \child_rip /* rip */
91 CFI_ADJUST_CFA_OFFSET 8
92 CFI_OFFSET rip,0
93 pushq %rax /* orig rax */
94 CFI_ADJUST_CFA_OFFSET 8
95 .endm
97 .macro UNFAKE_STACK_FRAME
98 addq $8*6, %rsp
99 CFI_ADJUST_CFA_OFFSET -(6*8)
100 .endm
102 .macro CFI_DEFAULT_STACK
103 CFI_ADJUST_CFA_OFFSET (SS)
104 CFI_OFFSET r15,R15-SS
105 CFI_OFFSET r14,R14-SS
106 CFI_OFFSET r13,R13-SS
107 CFI_OFFSET r12,R12-SS
108 CFI_OFFSET rbp,RBP-SS
109 CFI_OFFSET rbx,RBX-SS
110 CFI_OFFSET r11,R11-SS
111 CFI_OFFSET r10,R10-SS
112 CFI_OFFSET r9,R9-SS
113 CFI_OFFSET r8,R8-SS
114 CFI_OFFSET rax,RAX-SS
115 CFI_OFFSET rcx,RCX-SS
116 CFI_OFFSET rdx,RDX-SS
117 CFI_OFFSET rsi,RSI-SS
118 CFI_OFFSET rdi,RDI-SS
119 CFI_OFFSET rsp,RSP-SS
120 CFI_OFFSET rip,RIP-SS
121 .endm
123 /*
124 * Must be consistent with the definition in arch_x86_64.h:
125 * struct switch_to_user {
126 * u64 rax, r11, rcx, flags, rip, cs, rflags, rsp, ss;
127 * };
128 * #define VGCF_IN_SYSCALL (1<<8)
129 */
130 .macro SWITCH_TO_USER flag
131 subq $8*4,%rsp # reuse rip, cs, rflags, rsp, ss in the stack
132 movq %rax,(%rsp)
133 movq %r11,1*8(%rsp)
134 movq %rcx,2*8(%rsp) # we saved %rcx upon exceptions
135 movq $\flag,3*8(%rsp)
136 movq $__HYPERVISOR_switch_to_user,%rax
137 syscall
138 .endm
140 .macro SWITCH_TO_KERNEL ssoff,adjust=0
141 jc 1f
142 orb $1,\ssoff-\adjust+4(%rsp)
143 1:
144 .endm
146 /*
147 * A newly forked process directly context switches into this.
148 */
149 /* rdi: prev */
150 ENTRY(ret_from_fork)
151 CFI_STARTPROC
152 CFI_DEFAULT_STACK
153 call schedule_tail
154 GET_THREAD_INFO(%rcx)
155 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
156 jnz rff_trace
157 rff_action:
158 RESTORE_REST
159 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
160 je int_ret_from_sys_call
161 testl $_TIF_IA32,threadinfo_flags(%rcx)
162 jnz int_ret_from_sys_call
163 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
164 jmp ret_from_sys_call
165 rff_trace:
166 movq %rsp,%rdi
167 call syscall_trace_leave
168 GET_THREAD_INFO(%rcx)
169 jmp rff_action
170 CFI_ENDPROC
172 /*
173 * System call entry. Upto 6 arguments in registers are supported.
174 *
175 * SYSCALL does not save anything on the stack and does not change the
176 * stack pointer.
177 */
179 /*
180 * Register setup:
181 * rax system call number
182 * rdi arg0
183 * rcx return address for syscall/sysret, C arg3
184 * rsi arg1
185 * rdx arg2
186 * r10 arg3 (--> moved to rcx for C)
187 * r8 arg4
188 * r9 arg5
189 * r11 eflags for syscall/sysret, temporary for C
190 * r12-r15,rbp,rbx saved by C code, not touched.
191 *
192 * Interrupts are off on entry.
193 * Only called from user space.
194 *
195 * XXX if we had a free scratch register we could save the RSP into the stack frame
196 * and report it properly in ps. Unfortunately we haven't.
197 */
199 ENTRY(system_call)
200 CFI_STARTPROC
201 SAVE_ARGS -8,0
202 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
203 XEN_UNBLOCK_EVENTS(%r11)
204 GET_THREAD_INFO(%rcx)
205 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
206 jnz tracesys
207 cmpq $__NR_syscall_max,%rax
208 ja badsys
209 movq %r10,%rcx
210 call *sys_call_table(,%rax,8) # XXX: rip relative
211 movq %rax,RAX-ARGOFFSET(%rsp)
212 /*
213 * Syscall return path ending with SYSRET (fast path)
214 * Has incomplete stack frame and undefined top of stack.
215 */
216 .globl ret_from_sys_call
217 ret_from_sys_call:
218 movl $_TIF_ALLWORK_MASK,%edi
219 /* edi: flagmask */
220 sysret_check:
221 GET_THREAD_INFO(%rcx)
222 XEN_BLOCK_EVENTS(%rsi)
223 movl threadinfo_flags(%rcx),%edx
224 andl %edi,%edx
225 jnz sysret_careful
226 XEN_UNBLOCK_EVENTS(%rsi)
227 RESTORE_ARGS 0,8,0
228 SWITCH_TO_USER VGCF_IN_SYSCALL
230 /* Handle reschedules */
231 /* edx: work, edi: workmask */
232 sysret_careful:
233 bt $TIF_NEED_RESCHED,%edx
234 jnc sysret_signal
235 XEN_BLOCK_EVENTS(%rsi)
236 pushq %rdi
237 call schedule
238 popq %rdi
239 jmp sysret_check
241 /* Handle a signal */
242 sysret_signal:
243 /* sti */
244 XEN_UNBLOCK_EVENTS(%rsi)
245 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
246 jz 1f
248 /* Really a signal */
249 /* edx: work flags (arg3) */
250 leaq do_notify_resume(%rip),%rax
251 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
252 xorl %esi,%esi # oldset -> arg2
253 call ptregscall_common
254 1: movl $_TIF_NEED_RESCHED,%edi
255 jmp sysret_check
257 /* Do syscall tracing */
258 tracesys:
259 SAVE_REST
260 movq $-ENOSYS,RAX(%rsp)
261 FIXUP_TOP_OF_STACK %rdi
262 movq %rsp,%rdi
263 call syscall_trace_enter
264 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
265 RESTORE_REST
266 cmpq $__NR_syscall_max,%rax
267 ja 1f
268 movq %r10,%rcx /* fixup for C */
269 call *sys_call_table(,%rax,8)
270 movq %rax,RAX-ARGOFFSET(%rsp)
271 1: SAVE_REST
272 movq %rsp,%rdi
273 call syscall_trace_leave
274 RESTORE_TOP_OF_STACK %rbx
275 RESTORE_REST
276 jmp ret_from_sys_call
278 badsys:
279 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
280 jmp ret_from_sys_call
282 /*
283 * Syscall return path ending with IRET.
284 * Has correct top of stack, but partial stack frame.
285 */
286 ENTRY(int_ret_from_sys_call)
287 XEN_BLOCK_EVENTS(%rsi)
288 testb $3,CS-ARGOFFSET(%rsp)
289 jnz 1f
290 /* Need to set the proper %ss (not NULL) for ring 3 iretq */
291 movl $__KERNEL_DS,SS-ARGOFFSET(%rsp)
292 jmp retint_restore_args # retrun from ring3 kernel
293 1:
294 movl $_TIF_ALLWORK_MASK,%edi
295 /* edi: mask to check */
296 int_with_check:
297 GET_THREAD_INFO(%rcx)
298 movl threadinfo_flags(%rcx),%edx
299 andl %edi,%edx
300 jnz int_careful
301 jmp retint_restore_args
303 /* Either reschedule or signal or syscall exit tracking needed. */
304 /* First do a reschedule test. */
305 /* edx: work, edi: workmask */
306 int_careful:
307 bt $TIF_NEED_RESCHED,%edx
308 jnc int_very_careful
309 /* sti */
310 XEN_UNBLOCK_EVENTS(%rsi)
311 pushq %rdi
312 call schedule
313 popq %rdi
314 cli
315 jmp int_with_check
317 /* handle signals and tracing -- both require a full stack frame */
318 int_very_careful:
319 /* sti */
320 XEN_UNBLOCK_EVENTS(%rsi)
321 SAVE_REST
322 /* Check for syscall exit trace */
323 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
324 jz int_signal
325 pushq %rdi
326 leaq 8(%rsp),%rdi # &ptregs -> arg1
327 call syscall_trace_leave
328 popq %rdi
329 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
330 cli
331 jmp int_restore_rest
333 int_signal:
334 testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
335 jz 1f
336 movq %rsp,%rdi # &ptregs -> arg1
337 xorl %esi,%esi # oldset -> arg2
338 call do_notify_resume
339 1: movl $_TIF_NEED_RESCHED,%edi
340 int_restore_rest:
341 RESTORE_REST
342 cli
343 jmp int_with_check
344 CFI_ENDPROC
346 /*
347 * Certain special system calls that need to save a complete full stack frame.
348 */
350 .macro PTREGSCALL label,func,arg
351 .globl \label
352 \label:
353 leaq \func(%rip),%rax
354 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
355 jmp ptregscall_common
356 .endm
358 PTREGSCALL stub_clone, sys_clone, %r8
359 PTREGSCALL stub_fork, sys_fork, %rdi
360 PTREGSCALL stub_vfork, sys_vfork, %rdi
361 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
362 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
363 PTREGSCALL stub_iopl, sys_iopl, %rsi
365 ENTRY(ptregscall_common)
366 CFI_STARTPROC
367 popq %r11
368 CFI_ADJUST_CFA_OFFSET -8
369 SAVE_REST
370 movq %r11, %r15
371 FIXUP_TOP_OF_STACK %r11
372 call *%rax
373 RESTORE_TOP_OF_STACK %r11
374 movq %r15, %r11
375 RESTORE_REST
376 pushq %r11
377 CFI_ADJUST_CFA_OFFSET 8
378 ret
379 CFI_ENDPROC
381 ENTRY(stub_execve)
382 CFI_STARTPROC
383 popq %r11
384 CFI_ADJUST_CFA_OFFSET -8
385 SAVE_REST
386 movq %r11, %r15
387 FIXUP_TOP_OF_STACK %r11
388 call sys_execve
389 GET_THREAD_INFO(%rcx)
390 bt $TIF_IA32,threadinfo_flags(%rcx)
391 jc exec_32bit
392 RESTORE_TOP_OF_STACK %r11
393 movq %r15, %r11
394 RESTORE_REST
395 push %r11
396 ret
398 exec_32bit:
399 CFI_ADJUST_CFA_OFFSET REST_SKIP
400 movq %rax,RAX(%rsp)
401 RESTORE_REST
402 jmp int_ret_from_sys_call
403 CFI_ENDPROC
405 /*
406 * sigreturn is special because it needs to restore all registers on return.
407 * This cannot be done with SYSRET, so use the IRET return path instead.
408 */
409 ENTRY(stub_rt_sigreturn)
410 CFI_STARTPROC
411 addq $8, %rsp
412 SAVE_REST
413 movq %rsp,%rdi
414 FIXUP_TOP_OF_STACK %r11
415 call sys_rt_sigreturn
416 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
417 RESTORE_REST
418 jmp int_ret_from_sys_call
419 CFI_ENDPROC
422 /*
423 * Interrupt entry/exit.
424 *
425 * Interrupt entry points save only callee clobbered registers in fast path.
426 *
427 * Entry runs with interrupts off.
428 */
430 /* 0(%rsp): interrupt number */
431 .macro interrupt func
432 CFI_STARTPROC simple
433 CFI_DEF_CFA rsp,(SS-RDI)
434 CFI_REL_OFFSET rsp,(RSP-ORIG_RAX)
435 CFI_REL_OFFSET rip,(RIP-ORIG_RAX)
436 cld
437 #ifdef CONFIG_DEBUG_INFO
438 SAVE_ALL
439 movq %rsp,%rdi
440 /*
441 * Setup a stack frame pointer. This allows gdb to trace
442 * back to the original stack.
443 */
444 movq %rsp,%rbp
445 CFI_DEF_CFA_REGISTER rbp
446 #else
447 SAVE_ARGS
448 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
449 #endif
450 #if 0 /* For Xen we don't need to do this */
451 testl $3,CS(%rdi)
452 je 1f
453 swapgs
454 #endif
455 1: addl $1,%gs:pda_irqcount # RED-PEN should check preempt count
456 movq %gs:pda_irqstackptr,%rax
457 cmoveq %rax,%rsp
458 pushq %rdi # save old stack
459 call \func
460 .endm
462 retint_check:
463 movl threadinfo_flags(%rcx),%edx
464 andl %edi,%edx
465 jnz retint_careful
466 retint_restore_args:
467 movb EVENT_MASK-REST_SKIP(%rsp), %al
468 notb %al # %al == ~saved_mask
469 XEN_GET_VCPU_INFO(%rsi)
470 andb evtchn_upcall_mask(%rsi),%al
471 andb $1,%al # %al == mask & ~saved_mask
472 jnz restore_all_enable_events # != 0 => reenable event delivery
473 XEN_PUT_VCPU_INFO(%rsi)
475 RESTORE_ARGS 0,8,0
476 testb $3,8(%rsp) # check CS
477 jnz user_mode
478 kernel_mode:
479 orb $3,1*8(%rsp)
480 iretq
481 user_mode:
482 SWITCH_TO_USER 0
484 /* edi: workmask, edx: work */
485 retint_careful:
486 bt $TIF_NEED_RESCHED,%edx
487 jnc retint_signal
488 XEN_UNBLOCK_EVENTS(%rsi)
489 /* sti */
490 pushq %rdi
491 call schedule
492 popq %rdi
493 XEN_BLOCK_EVENTS(%rsi)
494 GET_THREAD_INFO(%rcx)
495 /* cli */
496 jmp retint_check
498 retint_signal:
499 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
500 jz retint_restore_args
501 XEN_UNBLOCK_EVENTS(%rsi)
502 SAVE_REST
503 movq $-1,ORIG_RAX(%rsp)
504 xorq %rsi,%rsi # oldset
505 movq %rsp,%rdi # &pt_regs
506 call do_notify_resume
507 RESTORE_REST
508 XEN_BLOCK_EVENTS(%rsi)
509 movl $_TIF_NEED_RESCHED,%edi
510 GET_THREAD_INFO(%rcx)
511 jmp retint_check
513 #ifdef CONFIG_PREEMPT
514 /* Returning to kernel space. Check if we need preemption */
515 /* rcx: threadinfo. interrupts off. */
516 .p2align
517 retint_kernel:
518 cmpl $0,threadinfo_preempt_count(%rcx)
519 jnz retint_restore_args
520 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
521 jnc retint_restore_args
522 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
523 jnc retint_restore_args
524 call preempt_schedule_irq
525 jmp retint_kernel /* check again */
526 #endif
527 CFI_ENDPROC
529 /*
530 * APIC interrupts.
531 */
532 .macro apicinterrupt num,func
533 pushq $\num-256
534 interrupt \func
535 jmp error_entry
536 CFI_ENDPROC
537 .endm
539 #if 0
540 ENTRY(reschedule_interrupt)
541 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
543 ENTRY(invalidate_interrupt)
544 apicinterrupt INVALIDATE_TLB_VECTOR,smp_invalidate_interrupt
546 ENTRY(call_function_interrupt)
547 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
548 #endif
550 #ifdef CONFIG_X86_LOCAL_APIC
551 ENTRY(apic_timer_interrupt)
552 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
554 ENTRY(error_interrupt)
555 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
557 ENTRY(spurious_interrupt)
558 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
559 #endif
561 /*
562 * Exception entry points.
563 */
564 .macro zeroentry sym
565 movq (%rsp),%rcx
566 movq 8(%rsp),%r11
567 addq $0x10,%rsp /* skip rcx and r11 */
568 pushq $0 /* push error code/oldrax */
569 pushq %rax /* push real oldrax to the rdi slot */
570 leaq \sym(%rip),%rax
571 jmp error_entry
572 .endm
574 .macro errorentry sym
575 movq (%rsp),%rcx
576 movq 8(%rsp),%r11
577 addq $0x10,%rsp /* rsp points to the error code */
578 pushq %rax
579 leaq \sym(%rip),%rax
580 jmp error_entry
581 .endm
583 #if 0
584 /* error code is on the stack already */
585 /* handle NMI like exceptions that can happen everywhere */
586 .macro paranoidentry sym
587 movq (%rsp),%rcx
588 movq 8(%rsp),%r11
589 addq $0x10,%rsp /* skip rcx and r11 */
590 SAVE_ALL
591 cld
592 movl $1,%ebx
593 movl $MSR_GS_BASE,%ecx
594 rdmsr
595 testl %edx,%edx
596 js 1f
597 /* swapgs */
598 xorl %ebx,%ebx
599 1: movq %rsp,%rdi
600 movq ORIG_RAX(%rsp),%rsi
601 movq $-1,ORIG_RAX(%rsp)
602 call \sym
603 cli
604 .endm
605 #endif
607 /*
608 * Exception entry point. This expects an error code/orig_rax on the stack
609 * and the exception handler in %rax.
610 */
611 ENTRY(error_entry)
612 CFI_STARTPROC simple
613 CFI_DEF_CFA rsp,(SS-RDI)
614 CFI_REL_OFFSET rsp,(RSP-RDI)
615 CFI_REL_OFFSET rip,(RIP-RDI)
616 /* rdi slot contains rax, oldrax contains error code */
617 cld
618 subq $14*8,%rsp
619 CFI_ADJUST_CFA_OFFSET (14*8)
620 movq %rsi,13*8(%rsp)
621 CFI_REL_OFFSET rsi,RSI
622 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
623 movq %rdx,12*8(%rsp)
624 CFI_REL_OFFSET rdx,RDX
625 movq %rcx,11*8(%rsp)
626 CFI_REL_OFFSET rcx,RCX
627 movq %rsi,10*8(%rsp) /* store rax */
628 CFI_REL_OFFSET rax,RAX
629 movq %r8, 9*8(%rsp)
630 CFI_REL_OFFSET r8,R8
631 movq %r9, 8*8(%rsp)
632 CFI_REL_OFFSET r9,R9
633 movq %r10,7*8(%rsp)
634 CFI_REL_OFFSET r10,R10
635 movq %r11,6*8(%rsp)
636 CFI_REL_OFFSET r11,R11
637 movq %rbx,5*8(%rsp)
638 CFI_REL_OFFSET rbx,RBX
639 movq %rbp,4*8(%rsp)
640 CFI_REL_OFFSET rbp,RBP
641 movq %r12,3*8(%rsp)
642 CFI_REL_OFFSET r12,R12
643 movq %r13,2*8(%rsp)
644 CFI_REL_OFFSET r13,R13
645 movq %r14,1*8(%rsp)
646 CFI_REL_OFFSET r14,R14
647 movq %r15,(%rsp)
648 CFI_REL_OFFSET r15,R15
649 #if 0
650 cmpl $__KERNEL_CS,CS(%rsp)
651 je error_kernelspace
652 #endif
653 error_call_handler:
654 movq %rdi, RDI(%rsp)
655 movq %rsp,%rdi
656 movq ORIG_RAX(%rsp),%rsi # get error code
657 movq $-1,ORIG_RAX(%rsp)
658 call *%rax
659 error_exit:
660 RESTORE_REST
661 /* cli */
662 XEN_BLOCK_EVENTS(%rsi)
663 GET_THREAD_INFO(%rcx)
664 testb $3,CS-ARGOFFSET(%rsp)
665 jz retint_kernel
666 movl threadinfo_flags(%rcx),%edx
667 movl $_TIF_WORK_MASK,%edi
668 andl %edi,%edx
669 jnz retint_careful
670 jmp retint_restore_args
672 error_kernelspace:
673 /*
674 * We need to re-write the logic here because we don't do iretq to
675 * to return to user mode. It's still possible that we get trap/fault
676 * in the kernel (when accessing buffers pointed to by system calls,
677 * for example).
678 *
679 */
680 #if 0
681 incl %ebx
682 /* There are two places in the kernel that can potentially fault with
683 usergs. Handle them here. The exception handlers after
684 iret run with kernel gs again, so don't set the user space flag.
685 B stepping K8s sometimes report an truncated RIP for IRET
686 exceptions returning to compat mode. Check for these here too. */
687 leaq iret_label(%rip),%rbp
688 cmpq %rbp,RIP(%rsp)
689 je error_swapgs
690 movl %ebp,%ebp /* zero extend */
691 cmpq %rbp,RIP(%rsp)
692 je error_swapgs
693 cmpq $gs_change,RIP(%rsp)
694 je error_swapgs
695 jmp error_sti
696 #endif
698 ENTRY(hypervisor_callback)
699 zeroentry do_hypervisor_callback
701 /*
702 * Copied from arch/xen/i386/kernel/entry.S
703 */
704 # A note on the "critical region" in our callback handler.
705 # We want to avoid stacking callback handlers due to events occurring
706 # during handling of the last event. To do this, we keep events disabled
707 # until we've done all processing. HOWEVER, we must enable events before
708 # popping the stack frame (can't be done atomically) and so it would still
709 # be possible to get enough handler activations to overflow the stack.
710 # Although unlikely, bugs of that kind are hard to track down, so we'd
711 # like to avoid the possibility.
712 # So, on entry to the handler we detect whether we interrupted an
713 # existing activation in its critical region -- if so, we pop the current
714 # activation and restart the handler using the previous one.
715 ENTRY(do_hypervisor_callback) # do_hyperviosr_callback(struct *pt_regs)
716 # Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
717 # see the correct pointer to the pt_regs
718 addq $8, %rsp # we don't return, adjust the stack frame
719 11: movb $0, EVENT_MASK(%rsp)
720 call evtchn_do_upcall
721 jmp error_exit
723 ALIGN
724 restore_all_enable_events:
725 XEN_UNBLOCK_EVENTS(%rsi) # %rsi is already set up...
727 scrit: /**** START OF CRITICAL REGION ****/
728 XEN_TEST_PENDING(%rsi)
729 jnz 14f # process more events if necessary...
730 XEN_PUT_VCPU_INFO(%rsi)
731 RESTORE_ARGS 0,8,0
732 testb $3,8(%rsp) # check CS
733 jnz crit_user_mode
734 orb $3,1*8(%rsp)
735 iretq
736 crit_user_mode:
737 SWITCH_TO_USER 0
739 14: XEN_LOCKED_BLOCK_EVENTS(%rsi)
740 XEN_PUT_VCPU_INFO(%rsi)
741 SAVE_REST
742 movq %rsp,%rdi # set the argument again
743 jmp 11b
744 ecrit: /**** END OF CRITICAL REGION ****/
745 # At this point, unlike on x86-32, we don't do the fixup to simplify the
746 # code and the stack frame is more complex on x86-64.
747 # When the kernel is interrupted in the critical section, the kernel
748 # will do IRET in that case, and everything will be restored at that point,
749 # i.e. it just resumes from the next instruction interrupted with the same context.
751 # Hypervisor uses this for application faults while it executes.
752 ENTRY(failsafe_callback)
753 addq $0x10,%rsp /* skip rcx and r11 */
754 1: mov (%rsp),%ds
755 2: mov 8(%rsp),%es
756 3: mov 16(%rsp),%fs
757 4: mov 24(%rsp),%gs
758 addq $0x20,%rsp /* skip the above selectors */
759 SAVE_ALL
760 jmp error_exit
761 .section .fixup,"ax"; \
762 6: movq $0,(%rsp); \
763 jmp 1b; \
764 7: movq $0,8(%rsp); \
765 jmp 2b; \
766 8: movq $0,16(%rsp); \
767 jmp 3b; \
768 9: movq $0,24(%rsp); \
769 jmp 4b; \
770 .previous; \
771 .section __ex_table,"a";\
772 .align 16; \
773 .quad 1b,6b; \
774 .quad 2b,7b; \
775 .quad 3b,8b; \
776 .quad 4b,9b; \
777 .previous
779 #if 0
780 .section __ex_table,"a"
781 .align 8
782 .quad gs_change,bad_gs
783 .previous
784 .section .fixup,"ax"
785 /* running with kernelgs */
786 bad_gs:
787 /* swapgs */ /* switch back to user gs */
788 xorl %eax,%eax
789 movl %eax,%gs
790 jmp 2b
791 .previous
792 #endif
794 /*
795 * Create a kernel thread.
796 *
797 * C extern interface:
798 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
799 *
800 * asm input arguments:
801 * rdi: fn, rsi: arg, rdx: flags
802 */
803 ENTRY(kernel_thread)
804 CFI_STARTPROC
805 FAKE_STACK_FRAME $child_rip
806 SAVE_ALL
808 # rdi: flags, rsi: usp, rdx: will be &pt_regs
809 movq %rdx,%rdi
810 orq kernel_thread_flags(%rip),%rdi
811 movq $-1, %rsi
812 movq %rsp, %rdx
814 xorl %r8d,%r8d
815 xorl %r9d,%r9d
817 # clone now
818 call do_fork
819 movq %rax,RAX(%rsp)
820 xorl %edi,%edi
822 /*
823 * It isn't worth to check for reschedule here,
824 * so internally to the x86_64 port you can rely on kernel_thread()
825 * not to reschedule the child before returning, this avoids the need
826 * of hacks for example to fork off the per-CPU idle tasks.
827 * [Hopefully no generic code relies on the reschedule -AK]
828 */
829 RESTORE_ALL
830 UNFAKE_STACK_FRAME
831 ret
832 CFI_ENDPROC
835 child_rip:
836 /*
837 * Here we are in the child and the registers are set as they were
838 * at kernel_thread() invocation in the parent.
839 */
840 movq %rdi, %rax
841 movq %rsi, %rdi
842 call *%rax
843 # exit
844 xorq %rdi, %rdi
845 call do_exit
847 /*
848 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
849 *
850 * C extern interface:
851 * extern long execve(char *name, char **argv, char **envp)
852 *
853 * asm input arguments:
854 * rdi: name, rsi: argv, rdx: envp
855 *
856 * We want to fallback into:
857 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
858 *
859 * do_sys_execve asm fallback arguments:
860 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
861 */
862 ENTRY(execve)
863 CFI_STARTPROC
864 FAKE_STACK_FRAME $0
865 SAVE_ALL
866 call sys_execve
867 movq %rax, RAX(%rsp)
868 RESTORE_REST
869 testq %rax,%rax
870 jne 1f
871 jmp int_ret_from_sys_call
872 1: RESTORE_ARGS
873 UNFAKE_STACK_FRAME
874 ret
875 CFI_ENDPROC
878 /*
879 * Copy error_entry because of the different stack frame
880 */
881 ENTRY(page_fault)
882 movq (%rsp),%rcx
883 movq 8(%rsp),%r11
884 addq $0x10,%rsp # now %rsp points to %cr2
885 pushq %rax
886 leaq do_page_fault(%rip),%rax
887 cld
888 subq $13*8,%rsp
889 movq %rdx,12*8(%rsp) # save %rdx
890 movq 13*8(%rsp),%rdx # load rax
891 movq %rcx,11*8(%rsp)
892 movq %rdx,10*8(%rsp) # store rax
893 movq %rsi,13*8(%rsp) # now save %rsi
894 movq 14*8(%rsp),%rdx # load %cr2, 3rd argument
895 movq %r8, 9*8(%rsp)
896 movq %r9, 8*8(%rsp)
897 movq %r10,7*8(%rsp)
898 movq %r11,6*8(%rsp)
899 movq %rbx,5*8(%rsp)
900 movq %rbp,4*8(%rsp)
901 movq %r12,3*8(%rsp)
902 movq %r13,2*8(%rsp)
903 movq %r14,1*8(%rsp)
904 movq %r15,(%rsp)
905 #if 0
906 cmpl $__KERNEL_CS,CS(%rsp)
907 je error_kernelspace
908 #endif
909 /*
910 * 1st and 2nd arguments are set by error_call_handler
911 */
912 jmp error_call_handler
914 ENTRY(coprocessor_error)
915 zeroentry do_coprocessor_error
917 ENTRY(simd_coprocessor_error)
918 zeroentry do_simd_coprocessor_error
920 ENTRY(device_not_available)
921 zeroentry math_state_restore
923 /* runs on exception stack */
924 ENTRY(debug)
925 CFI_STARTPROC
926 /* pushq $0
927 CFI_ADJUST_CFA_OFFSET 8 */
928 zeroentry do_debug
929 /* jmp paranoid_exit */
930 CFI_ENDPROC
932 #if 0
933 /* runs on exception stack */
934 ENTRY(nmi)
935 CFI_STARTPROC
936 pushq $-1
937 CFI_ADJUST_CFA_OFFSET 8
938 paranoidentry do_nmi
939 /*
940 * "Paranoid" exit path from exception stack.
941 * Paranoid because this is used by NMIs and cannot take
942 * any kernel state for granted.
943 * We don't do kernel preemption checks here, because only
944 * NMI should be common and it does not enable IRQs and
945 * cannot get reschedule ticks.
946 */
947 /* ebx: no swapgs flag */
948 paranoid_exit:
949 testl %ebx,%ebx /* swapgs needed? */
950 jnz paranoid_restore
951 paranoid_swapgs:
952 /* swapgs */
953 paranoid_restore:
954 RESTORE_ALL 8
955 /* iretq */
956 paranoid_userspace:
957 GET_THREAD_INFO(%rcx)
958 # movl threadinfo_flags(%rcx),%edx
959 # testl $_TIF_NEED_RESCHED,%edx
960 # jnz paranoid_resched
961 # testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
962 # jnz paranoid_signal
963 # jmp paranoid_swapgs
964 #paranoid_resched:
965 #/* sti */
966 # call schedule
967 # jmp paranoid_exit
968 #paranoid_signal:
969 movl threadinfo_flags(%rcx),%ebx
970 andl $_TIF_WORK_MASK,%ebx
971 jz paranoid_swapgs
972 movq %rsp,%rdi /* &pt_regs */
973 call sync_regs
974 movq %rax,%rsp /* switch stack for scheduling */
975 testl $_TIF_NEED_RESCHED,%ebx
976 jnz paranoid_schedule
977 movl %ebx,%edx /* arg3: thread flags */
978 /* sti */
979 # xorl %esi,%esi /* oldset */
980 # movq %rsp,%rdi /* &pt_regs */
981 xorl %esi,%esi /* arg2: oldset */
982 movq %rsp,%rdi /* arg1: &pt_regs */
983 call do_notify_resume
984 # jmp paranoid_exit
985 cli
986 jmp paranoid_userspace
987 paranoid_schedule:
988 sti
989 call schedule
990 cli
991 jmp paranoid_userspace
992 CFI_ENDPROC
993 #endif
995 ENTRY(int3)
996 zeroentry do_int3
998 ENTRY(overflow)
999 zeroentry do_overflow
1001 ENTRY(bounds)
1002 zeroentry do_bounds
1004 ENTRY(invalid_op)
1005 zeroentry do_invalid_op
1007 ENTRY(coprocessor_segment_overrun)
1008 zeroentry do_coprocessor_segment_overrun
1010 ENTRY(reserved)
1011 zeroentry do_reserved
1013 #if 0
1014 /* runs on exception stack */
1015 ENTRY(double_fault)
1016 CFI_STARTPROC
1017 paranoidentry do_double_fault
1018 jmp paranoid_exit
1019 CFI_ENDPROC
1020 #endif
1022 ENTRY(invalid_TSS)
1023 errorentry do_invalid_TSS
1025 ENTRY(segment_not_present)
1026 errorentry do_segment_not_present
1028 /* runs on exception stack */
1029 ENTRY(stack_segment)
1030 CFI_STARTPROC
1031 errorentry do_stack_segment
1032 CFI_ENDPROC
1034 ENTRY(general_protection)
1035 errorentry do_general_protection
1037 ENTRY(alignment_check)
1038 errorentry do_alignment_check
1040 ENTRY(divide_error)
1041 zeroentry do_divide_error
1043 ENTRY(spurious_interrupt_bug)
1044 zeroentry do_spurious_interrupt_bug
1046 #ifdef CONFIG_X86_MCE
1047 /* runs on exception stack */
1048 ENTRY(machine_check)
1049 CFI_STARTPROC
1050 pushq $0
1051 CFI_ADJUST_CFA_OFFSET 8
1052 paranoidentry do_machine_check
1053 jmp paranoid_exit
1054 CFI_ENDPROC
1055 #endif
1057 ENTRY(call_debug)
1058 zeroentry do_call_debug