ia64/xen-unstable

view linux-2.6-xen-sparse/arch/xen/x86_64/kernel/entry.S @ 8574:08d2a74dce81

Fix x86_64 domU build by conditionalizing NMI support on CONFIG_X86_LOCAL_APIC

The NMI handler is not built unless CONFIG_X86_LOCAL_APIC is defined, and
CONFIG_X86_LOCAL_APIC requires a dom0 build.

Signed-off-by: Ian Campbell <Ian.Campbell@XenSource.com>
author Ian.Campbell@xensource.com
date Thu Jan 12 10:37:10 2006 +0000 (2006-01-12)
parents 0ffd94a02318
children 71914b64b577
line source
1 /*
2 * linux/arch/x86_64/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7 *
8 * $Id$
9 *
10 * Jun Nakajima <jun.nakajima@intel.com>
11 * Asit Mallick <asit.k.mallick@intel.com>
12 * Modified for Xen
13 */
15 /*
16 * entry.S contains the system-call and fault low-level handling routines.
17 *
18 * NOTE: This code handles signal-recognition, which happens every time
19 * after an interrupt and after each system call.
20 *
21 * Normal syscalls and interrupts don't save a full stack frame, this is
22 * only done for syscall tracing, signals or fork/exec et.al.
23 *
24 * A note on terminology:
25 * - top of stack: Architecture defined interrupt frame from SS to RIP
26 * at the top of the kernel process stack.
27 * - partial stack frame: partially saved registers upto R11.
28 * - full stack frame: Like partial stack frame, but all register saved.
29 *
30 * TODO:
31 * - schedule it carefully for the final hardware.
32 */
34 #define ASSEMBLY 1
35 #include <linux/config.h>
36 #include <linux/linkage.h>
37 #include <asm/segment.h>
38 #include <asm/smp.h>
39 #include <asm/cache.h>
40 #include <asm/errno.h>
41 #include <asm/dwarf2.h>
42 #include <asm/calling.h>
43 #include <asm/asm_offset.h>
44 #include <asm/msr.h>
45 #include <asm/unistd.h>
46 #include <asm/thread_info.h>
47 #include <asm/hw_irq.h>
48 #include <asm/errno.h>
49 #include <asm-xen/xen-public/arch-x86_64.h>
51 #include "irq_vectors.h"
53 #include "xen_entry.S"
55 .code64
57 #ifndef CONFIG_PREEMPT
58 #define retint_kernel retint_restore_args
59 #endif
61 /*
62 * C code is not supposed to know about undefined top of stack. Every time
63 * a C function with an pt_regs argument is called from the SYSCALL based
64 * fast path FIXUP_TOP_OF_STACK is needed.
65 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
66 * manipulation.
67 */
69 /* %rsp:at FRAMEEND */
70 .macro FIXUP_TOP_OF_STACK tmp
71 movq $__USER_CS,CS(%rsp)
72 movq $-1,RCX(%rsp)
73 .endm
75 .macro RESTORE_TOP_OF_STACK tmp,offset=0
76 .endm
78 .macro FAKE_STACK_FRAME child_rip
79 /* push in order ss, rsp, eflags, cs, rip */
80 xorq %rax, %rax
81 pushq %rax /* ss */
82 CFI_ADJUST_CFA_OFFSET 8
83 pushq %rax /* rsp */
84 CFI_ADJUST_CFA_OFFSET 8
85 CFI_OFFSET rip,0
86 pushq $(1<<9) /* eflags - interrupts on */
87 CFI_ADJUST_CFA_OFFSET 8
88 pushq $__KERNEL_CS /* cs */
89 CFI_ADJUST_CFA_OFFSET 8
90 pushq \child_rip /* rip */
91 CFI_ADJUST_CFA_OFFSET 8
92 CFI_OFFSET rip,0
93 pushq %rax /* orig rax */
94 CFI_ADJUST_CFA_OFFSET 8
95 .endm
97 .macro UNFAKE_STACK_FRAME
98 addq $8*6, %rsp
99 CFI_ADJUST_CFA_OFFSET -(6*8)
100 .endm
102 .macro CFI_DEFAULT_STACK
103 CFI_ADJUST_CFA_OFFSET (SS)
104 CFI_OFFSET r15,R15-SS
105 CFI_OFFSET r14,R14-SS
106 CFI_OFFSET r13,R13-SS
107 CFI_OFFSET r12,R12-SS
108 CFI_OFFSET rbp,RBP-SS
109 CFI_OFFSET rbx,RBX-SS
110 CFI_OFFSET r11,R11-SS
111 CFI_OFFSET r10,R10-SS
112 CFI_OFFSET r9,R9-SS
113 CFI_OFFSET r8,R8-SS
114 CFI_OFFSET rax,RAX-SS
115 CFI_OFFSET rcx,RCX-SS
116 CFI_OFFSET rdx,RDX-SS
117 CFI_OFFSET rsi,RSI-SS
118 CFI_OFFSET rdi,RDI-SS
119 CFI_OFFSET rsp,RSP-SS
120 CFI_OFFSET rip,RIP-SS
121 .endm
123 /*
124 * Must be consistent with the definition in arch-x86_64.h:
125 * struct iret_context {
126 * u64 rax, r11, rcx, flags, rip, cs, rflags, rsp, ss;
127 * };
128 * #define VGCF_IN_SYSCALL (1<<8)
129 */
130 .macro HYPERVISOR_IRET flag
131 subq $8*4,%rsp # reuse rip, cs, rflags, rsp, ss in the stack
132 movq %rax,(%rsp)
133 movq %r11,1*8(%rsp)
134 movq %rcx,2*8(%rsp) # we saved %rcx upon exceptions
135 movq $\flag,3*8(%rsp)
136 movq $__HYPERVISOR_iret,%rax
137 syscall
138 .endm
140 .macro SWITCH_TO_KERNEL ssoff,adjust=0
141 jc 1f
142 orb $1,\ssoff-\adjust+4(%rsp)
143 1:
144 .endm
146 /*
147 * A newly forked process directly context switches into this.
148 */
149 /* rdi: prev */
150 ENTRY(ret_from_fork)
151 CFI_STARTPROC
152 CFI_DEFAULT_STACK
153 call schedule_tail
154 GET_THREAD_INFO(%rcx)
155 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
156 jnz rff_trace
157 rff_action:
158 RESTORE_REST
159 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
160 je int_ret_from_sys_call
161 testl $_TIF_IA32,threadinfo_flags(%rcx)
162 jnz int_ret_from_sys_call
163 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
164 jmp ret_from_sys_call
165 rff_trace:
166 movq %rsp,%rdi
167 call syscall_trace_leave
168 GET_THREAD_INFO(%rcx)
169 jmp rff_action
170 CFI_ENDPROC
172 /*
173 * System call entry. Upto 6 arguments in registers are supported.
174 *
175 * SYSCALL does not save anything on the stack and does not change the
176 * stack pointer.
177 */
179 /*
180 * Register setup:
181 * rax system call number
182 * rdi arg0
183 * rcx return address for syscall/sysret, C arg3
184 * rsi arg1
185 * rdx arg2
186 * r10 arg3 (--> moved to rcx for C)
187 * r8 arg4
188 * r9 arg5
189 * r11 eflags for syscall/sysret, temporary for C
190 * r12-r15,rbp,rbx saved by C code, not touched.
191 *
192 * Interrupts are off on entry.
193 * Only called from user space.
194 *
195 * XXX if we had a free scratch register we could save the RSP into the stack frame
196 * and report it properly in ps. Unfortunately we haven't.
197 */
199 ENTRY(system_call)
200 CFI_STARTPROC
201 SAVE_ARGS -8,0
202 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
203 XEN_UNBLOCK_EVENTS(%r11)
204 GET_THREAD_INFO(%rcx)
205 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
206 jnz tracesys
207 cmpq $__NR_syscall_max,%rax
208 ja badsys
209 movq %r10,%rcx
210 call *sys_call_table(,%rax,8) # XXX: rip relative
211 movq %rax,RAX-ARGOFFSET(%rsp)
212 /*
213 * Syscall return path ending with SYSRET (fast path)
214 * Has incomplete stack frame and undefined top of stack.
215 */
216 .globl ret_from_sys_call
217 ret_from_sys_call:
218 movl $_TIF_ALLWORK_MASK,%edi
219 /* edi: flagmask */
220 sysret_check:
221 GET_THREAD_INFO(%rcx)
222 XEN_BLOCK_EVENTS(%rsi)
223 movl threadinfo_flags(%rcx),%edx
224 andl %edi,%edx
225 jnz sysret_careful
226 XEN_UNBLOCK_EVENTS(%rsi)
227 RESTORE_ARGS 0,8,0
228 HYPERVISOR_IRET VGCF_IN_SYSCALL
230 /* Handle reschedules */
231 /* edx: work, edi: workmask */
232 sysret_careful:
233 bt $TIF_NEED_RESCHED,%edx
234 jnc sysret_signal
235 XEN_BLOCK_EVENTS(%rsi)
236 pushq %rdi
237 call schedule
238 popq %rdi
239 jmp sysret_check
241 /* Handle a signal */
242 sysret_signal:
243 /* sti */
244 XEN_UNBLOCK_EVENTS(%rsi)
245 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
246 jz 1f
248 /* Really a signal */
249 /* edx: work flags (arg3) */
250 leaq do_notify_resume(%rip),%rax
251 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
252 xorl %esi,%esi # oldset -> arg2
253 call ptregscall_common
254 1: movl $_TIF_NEED_RESCHED,%edi
255 jmp sysret_check
257 /* Do syscall tracing */
258 tracesys:
259 SAVE_REST
260 movq $-ENOSYS,RAX(%rsp)
261 FIXUP_TOP_OF_STACK %rdi
262 movq %rsp,%rdi
263 call syscall_trace_enter
264 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
265 RESTORE_REST
266 cmpq $__NR_syscall_max,%rax
267 ja 1f
268 movq %r10,%rcx /* fixup for C */
269 call *sys_call_table(,%rax,8)
270 movq %rax,RAX-ARGOFFSET(%rsp)
271 1: SAVE_REST
272 movq %rsp,%rdi
273 call syscall_trace_leave
274 RESTORE_TOP_OF_STACK %rbx
275 RESTORE_REST
276 jmp ret_from_sys_call
278 badsys:
279 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
280 jmp ret_from_sys_call
282 /*
283 * Syscall return path ending with IRET.
284 * Has correct top of stack, but partial stack frame.
285 */
286 ENTRY(int_ret_from_sys_call)
287 XEN_BLOCK_EVENTS(%rsi)
288 testb $3,CS-ARGOFFSET(%rsp)
289 jnz 1f
290 /* Need to set the proper %ss (not NULL) for ring 3 iretq */
291 movl $__KERNEL_DS,SS-ARGOFFSET(%rsp)
292 jmp retint_restore_args # retrun from ring3 kernel
293 1:
294 movl $_TIF_ALLWORK_MASK,%edi
295 /* edi: mask to check */
296 int_with_check:
297 GET_THREAD_INFO(%rcx)
298 movl threadinfo_flags(%rcx),%edx
299 andl %edi,%edx
300 jnz int_careful
301 jmp retint_restore_args
303 /* Either reschedule or signal or syscall exit tracking needed. */
304 /* First do a reschedule test. */
305 /* edx: work, edi: workmask */
306 int_careful:
307 bt $TIF_NEED_RESCHED,%edx
308 jnc int_very_careful
309 /* sti */
310 XEN_UNBLOCK_EVENTS(%rsi)
311 pushq %rdi
312 call schedule
313 popq %rdi
314 cli
315 jmp int_with_check
317 /* handle signals and tracing -- both require a full stack frame */
318 int_very_careful:
319 /* sti */
320 XEN_UNBLOCK_EVENTS(%rsi)
321 SAVE_REST
322 /* Check for syscall exit trace */
323 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
324 jz int_signal
325 pushq %rdi
326 leaq 8(%rsp),%rdi # &ptregs -> arg1
327 call syscall_trace_leave
328 popq %rdi
329 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
330 cli
331 jmp int_restore_rest
333 int_signal:
334 testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
335 jz 1f
336 movq %rsp,%rdi # &ptregs -> arg1
337 xorl %esi,%esi # oldset -> arg2
338 call do_notify_resume
339 1: movl $_TIF_NEED_RESCHED,%edi
340 int_restore_rest:
341 RESTORE_REST
342 cli
343 jmp int_with_check
344 CFI_ENDPROC
346 /*
347 * Certain special system calls that need to save a complete full stack frame.
348 */
350 .macro PTREGSCALL label,func,arg
351 .globl \label
352 \label:
353 leaq \func(%rip),%rax
354 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
355 jmp ptregscall_common
356 .endm
358 PTREGSCALL stub_clone, sys_clone, %r8
359 PTREGSCALL stub_fork, sys_fork, %rdi
360 PTREGSCALL stub_vfork, sys_vfork, %rdi
361 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
362 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
363 PTREGSCALL stub_iopl, sys_iopl, %rsi
365 ENTRY(ptregscall_common)
366 CFI_STARTPROC
367 popq %r11
368 CFI_ADJUST_CFA_OFFSET -8
369 SAVE_REST
370 movq %r11, %r15
371 FIXUP_TOP_OF_STACK %r11
372 call *%rax
373 RESTORE_TOP_OF_STACK %r11
374 movq %r15, %r11
375 RESTORE_REST
376 pushq %r11
377 CFI_ADJUST_CFA_OFFSET 8
378 ret
379 CFI_ENDPROC
381 ENTRY(stub_execve)
382 CFI_STARTPROC
383 popq %r11
384 CFI_ADJUST_CFA_OFFSET -8
385 SAVE_REST
386 movq %r11, %r15
387 FIXUP_TOP_OF_STACK %r11
388 call sys_execve
389 GET_THREAD_INFO(%rcx)
390 bt $TIF_IA32,threadinfo_flags(%rcx)
391 jc exec_32bit
392 RESTORE_TOP_OF_STACK %r11
393 movq %r15, %r11
394 RESTORE_REST
395 push %r11
396 ret
398 exec_32bit:
399 CFI_ADJUST_CFA_OFFSET REST_SKIP
400 movq %rax,RAX(%rsp)
401 RESTORE_REST
402 jmp int_ret_from_sys_call
403 CFI_ENDPROC
405 /*
406 * sigreturn is special because it needs to restore all registers on return.
407 * This cannot be done with SYSRET, so use the IRET return path instead.
408 */
409 ENTRY(stub_rt_sigreturn)
410 CFI_STARTPROC
411 addq $8, %rsp
412 SAVE_REST
413 movq %rsp,%rdi
414 FIXUP_TOP_OF_STACK %r11
415 call sys_rt_sigreturn
416 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
417 RESTORE_REST
418 jmp int_ret_from_sys_call
419 CFI_ENDPROC
421 /*
422 * Interrupt entry/exit.
423 *
424 * Interrupt entry points save only callee clobbered registers in fast path.
425 *
426 * Entry runs with interrupts off.
427 */
429 /* 0(%rsp): interrupt number */
430 .macro interrupt func
431 CFI_STARTPROC simple
432 CFI_DEF_CFA rsp,(SS-RDI)
433 CFI_REL_OFFSET rsp,(RSP-ORIG_RAX)
434 CFI_REL_OFFSET rip,(RIP-ORIG_RAX)
435 cld
436 #ifdef CONFIG_DEBUG_INFO
437 SAVE_ALL
438 movq %rsp,%rdi
439 /*
440 * Setup a stack frame pointer. This allows gdb to trace
441 * back to the original stack.
442 */
443 movq %rsp,%rbp
444 CFI_DEF_CFA_REGISTER rbp
445 #else
446 SAVE_ARGS
447 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
448 #endif
449 #if 0 /* For Xen we don't need to do this */
450 testl $3,CS(%rdi)
451 je 1f
452 swapgs
453 #endif
454 1: addl $1,%gs:pda_irqcount # RED-PEN should check preempt count
455 movq %gs:pda_irqstackptr,%rax
456 cmoveq %rax,%rsp
457 pushq %rdi # save old stack
458 call \func
459 .endm
461 retint_check:
462 movl threadinfo_flags(%rcx),%edx
463 andl %edi,%edx
464 jnz retint_careful
465 retint_restore_args:
466 movb EVENT_MASK-REST_SKIP(%rsp), %al
467 notb %al # %al == ~saved_mask
468 XEN_GET_VCPU_INFO(%rsi)
469 andb evtchn_upcall_mask(%rsi),%al
470 andb $1,%al # %al == mask & ~saved_mask
471 jnz restore_all_enable_events # != 0 => reenable event delivery
472 XEN_PUT_VCPU_INFO(%rsi)
474 RESTORE_ARGS 0,8,0
475 testb $3,8(%rsp) # check CS
476 jnz user_mode
477 kernel_mode:
478 orb $3,1*8(%rsp)
479 iretq
480 user_mode:
481 HYPERVISOR_IRET 0
483 /* edi: workmask, edx: work */
484 retint_careful:
485 bt $TIF_NEED_RESCHED,%edx
486 jnc retint_signal
487 XEN_UNBLOCK_EVENTS(%rsi)
488 /* sti */
489 pushq %rdi
490 call schedule
491 popq %rdi
492 XEN_BLOCK_EVENTS(%rsi)
493 GET_THREAD_INFO(%rcx)
494 /* cli */
495 jmp retint_check
497 retint_signal:
498 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
499 jz retint_restore_args
500 XEN_UNBLOCK_EVENTS(%rsi)
501 SAVE_REST
502 movq $-1,ORIG_RAX(%rsp)
503 xorq %rsi,%rsi # oldset
504 movq %rsp,%rdi # &pt_regs
505 call do_notify_resume
506 RESTORE_REST
507 XEN_BLOCK_EVENTS(%rsi)
508 movl $_TIF_NEED_RESCHED,%edi
509 GET_THREAD_INFO(%rcx)
510 jmp retint_check
512 #ifdef CONFIG_PREEMPT
513 /* Returning to kernel space. Check if we need preemption */
514 /* rcx: threadinfo. interrupts off. */
515 .p2align
516 retint_kernel:
517 cmpl $0,threadinfo_preempt_count(%rcx)
518 jnz retint_restore_args
519 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
520 jnc retint_restore_args
521 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
522 jnc retint_restore_args
523 call preempt_schedule_irq
524 jmp retint_kernel /* check again */
525 #endif
526 CFI_ENDPROC
528 /*
529 * APIC interrupts.
530 */
531 .macro apicinterrupt num,func
532 pushq $\num-256
533 interrupt \func
534 jmp error_entry
535 CFI_ENDPROC
536 .endm
538 #if 0
539 ENTRY(reschedule_interrupt)
540 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
542 ENTRY(invalidate_interrupt)
543 apicinterrupt INVALIDATE_TLB_VECTOR,smp_invalidate_interrupt
545 ENTRY(call_function_interrupt)
546 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
547 #endif
549 #ifdef CONFIG_X86_LOCAL_APIC
550 ENTRY(apic_timer_interrupt)
551 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
553 ENTRY(error_interrupt)
554 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
556 ENTRY(spurious_interrupt)
557 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
558 #endif
560 /*
561 * Exception entry points.
562 */
563 .macro zeroentry sym
564 movq (%rsp),%rcx
565 movq 8(%rsp),%r11
566 addq $0x10,%rsp /* skip rcx and r11 */
567 pushq $0 /* push error code/oldrax */
568 pushq %rax /* push real oldrax to the rdi slot */
569 leaq \sym(%rip),%rax
570 jmp error_entry
571 .endm
573 .macro errorentry sym
574 movq (%rsp),%rcx
575 movq 8(%rsp),%r11
576 addq $0x10,%rsp /* rsp points to the error code */
577 pushq %rax
578 leaq \sym(%rip),%rax
579 jmp error_entry
580 .endm
582 #if 0
583 /* error code is on the stack already */
584 /* handle NMI like exceptions that can happen everywhere */
585 .macro paranoidentry sym
586 movq (%rsp),%rcx
587 movq 8(%rsp),%r11
588 addq $0x10,%rsp /* skip rcx and r11 */
589 SAVE_ALL
590 cld
591 movl $1,%ebx
592 movl $MSR_GS_BASE,%ecx
593 rdmsr
594 testl %edx,%edx
595 js 1f
596 /* swapgs */
597 xorl %ebx,%ebx
598 1: movq %rsp,%rdi
599 movq ORIG_RAX(%rsp),%rsi
600 movq $-1,ORIG_RAX(%rsp)
601 call \sym
602 cli
603 .endm
604 #endif
606 /*
607 * Exception entry point. This expects an error code/orig_rax on the stack
608 * and the exception handler in %rax.
609 */
610 ENTRY(error_entry)
611 CFI_STARTPROC simple
612 CFI_DEF_CFA rsp,(SS-RDI)
613 CFI_REL_OFFSET rsp,(RSP-RDI)
614 CFI_REL_OFFSET rip,(RIP-RDI)
615 /* rdi slot contains rax, oldrax contains error code */
616 cld
617 subq $14*8,%rsp
618 CFI_ADJUST_CFA_OFFSET (14*8)
619 movq %rsi,13*8(%rsp)
620 CFI_REL_OFFSET rsi,RSI
621 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
622 movq %rdx,12*8(%rsp)
623 CFI_REL_OFFSET rdx,RDX
624 movq %rcx,11*8(%rsp)
625 CFI_REL_OFFSET rcx,RCX
626 movq %rsi,10*8(%rsp) /* store rax */
627 CFI_REL_OFFSET rax,RAX
628 movq %r8, 9*8(%rsp)
629 CFI_REL_OFFSET r8,R8
630 movq %r9, 8*8(%rsp)
631 CFI_REL_OFFSET r9,R9
632 movq %r10,7*8(%rsp)
633 CFI_REL_OFFSET r10,R10
634 movq %r11,6*8(%rsp)
635 CFI_REL_OFFSET r11,R11
636 movq %rbx,5*8(%rsp)
637 CFI_REL_OFFSET rbx,RBX
638 movq %rbp,4*8(%rsp)
639 CFI_REL_OFFSET rbp,RBP
640 movq %r12,3*8(%rsp)
641 CFI_REL_OFFSET r12,R12
642 movq %r13,2*8(%rsp)
643 CFI_REL_OFFSET r13,R13
644 movq %r14,1*8(%rsp)
645 CFI_REL_OFFSET r14,R14
646 movq %r15,(%rsp)
647 CFI_REL_OFFSET r15,R15
648 #if 0
649 cmpl $__KERNEL_CS,CS(%rsp)
650 je error_kernelspace
651 #endif
652 error_call_handler:
653 movq %rdi, RDI(%rsp)
654 movq %rsp,%rdi
655 movq ORIG_RAX(%rsp),%rsi # get error code
656 movq $-1,ORIG_RAX(%rsp)
657 call *%rax
658 error_exit:
659 RESTORE_REST
660 /* cli */
661 XEN_BLOCK_EVENTS(%rsi)
662 GET_THREAD_INFO(%rcx)
663 testb $3,CS-ARGOFFSET(%rsp)
664 jz retint_kernel
665 movl threadinfo_flags(%rcx),%edx
666 movl $_TIF_WORK_MASK,%edi
667 andl %edi,%edx
668 jnz retint_careful
669 jmp retint_restore_args
671 error_kernelspace:
672 /*
673 * We need to re-write the logic here because we don't do iretq to
674 * to return to user mode. It's still possible that we get trap/fault
675 * in the kernel (when accessing buffers pointed to by system calls,
676 * for example).
677 *
678 */
679 #if 0
680 incl %ebx
681 /* There are two places in the kernel that can potentially fault with
682 usergs. Handle them here. The exception handlers after
683 iret run with kernel gs again, so don't set the user space flag.
684 B stepping K8s sometimes report an truncated RIP for IRET
685 exceptions returning to compat mode. Check for these here too. */
686 leaq iret_label(%rip),%rbp
687 cmpq %rbp,RIP(%rsp)
688 je error_swapgs
689 movl %ebp,%ebp /* zero extend */
690 cmpq %rbp,RIP(%rsp)
691 je error_swapgs
692 cmpq $gs_change,RIP(%rsp)
693 je error_swapgs
694 jmp error_sti
695 #endif
697 ENTRY(hypervisor_callback)
698 zeroentry do_hypervisor_callback
700 /*
701 * Copied from arch/xen/i386/kernel/entry.S
702 */
703 # A note on the "critical region" in our callback handler.
704 # We want to avoid stacking callback handlers due to events occurring
705 # during handling of the last event. To do this, we keep events disabled
706 # until we've done all processing. HOWEVER, we must enable events before
707 # popping the stack frame (can't be done atomically) and so it would still
708 # be possible to get enough handler activations to overflow the stack.
709 # Although unlikely, bugs of that kind are hard to track down, so we'd
710 # like to avoid the possibility.
711 # So, on entry to the handler we detect whether we interrupted an
712 # existing activation in its critical region -- if so, we pop the current
713 # activation and restart the handler using the previous one.
714 ENTRY(do_hypervisor_callback) # do_hyperviosr_callback(struct *pt_regs)
715 # Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
716 # see the correct pointer to the pt_regs
717 addq $8, %rsp # we don't return, adjust the stack frame
718 11: movb $0, EVENT_MASK(%rsp)
719 call evtchn_do_upcall
720 jmp error_exit
722 #ifdef CONFIG_X86_LOCAL_APIC
723 ENTRY(nmi)
724 zeroentry do_nmi_callback
725 ENTRY(do_nmi_callback)
726 addq $8, %rsp
727 call do_nmi
728 RESTORE_REST
729 XEN_BLOCK_EVENTS(%rsi)
730 GET_THREAD_INFO(%rcx)
731 jmp retint_restore_args
732 #endif
734 ALIGN
735 restore_all_enable_events:
736 XEN_UNBLOCK_EVENTS(%rsi) # %rsi is already set up...
738 scrit: /**** START OF CRITICAL REGION ****/
739 XEN_TEST_PENDING(%rsi)
740 jnz 14f # process more events if necessary...
741 XEN_PUT_VCPU_INFO(%rsi)
742 RESTORE_ARGS 0,8,0
743 testb $3,8(%rsp) # check CS
744 jnz crit_user_mode
745 orb $3,1*8(%rsp)
746 iretq
747 crit_user_mode:
748 HYPERVISOR_IRET 0
750 14: XEN_LOCKED_BLOCK_EVENTS(%rsi)
751 XEN_PUT_VCPU_INFO(%rsi)
752 SAVE_REST
753 movq %rsp,%rdi # set the argument again
754 jmp 11b
755 ecrit: /**** END OF CRITICAL REGION ****/
756 # At this point, unlike on x86-32, we don't do the fixup to simplify the
757 # code and the stack frame is more complex on x86-64.
758 # When the kernel is interrupted in the critical section, the kernel
759 # will do IRET in that case, and everything will be restored at that point,
760 # i.e. it just resumes from the next instruction interrupted with the same context.
762 # Hypervisor uses this for application faults while it executes.
763 ENTRY(failsafe_callback)
764 addq $0x10,%rsp /* skip rcx and r11 */
765 1: mov (%rsp),%ds
766 2: mov 8(%rsp),%es
767 3: mov 16(%rsp),%fs
768 4: mov 24(%rsp),%gs
769 addq $0x20,%rsp /* skip the above selectors */
770 SAVE_ALL
771 jmp error_exit
772 .section .fixup,"ax"; \
773 6: movq $0,(%rsp); \
774 jmp 1b; \
775 7: movq $0,8(%rsp); \
776 jmp 2b; \
777 8: movq $0,16(%rsp); \
778 jmp 3b; \
779 9: movq $0,24(%rsp); \
780 jmp 4b; \
781 .previous; \
782 .section __ex_table,"a";\
783 .align 16; \
784 .quad 1b,6b; \
785 .quad 2b,7b; \
786 .quad 3b,8b; \
787 .quad 4b,9b; \
788 .previous
790 #if 0
791 .section __ex_table,"a"
792 .align 8
793 .quad gs_change,bad_gs
794 .previous
795 .section .fixup,"ax"
796 /* running with kernelgs */
797 bad_gs:
798 /* swapgs */ /* switch back to user gs */
799 xorl %eax,%eax
800 movl %eax,%gs
801 jmp 2b
802 .previous
803 #endif
805 /*
806 * Create a kernel thread.
807 *
808 * C extern interface:
809 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
810 *
811 * asm input arguments:
812 * rdi: fn, rsi: arg, rdx: flags
813 */
814 ENTRY(kernel_thread)
815 CFI_STARTPROC
816 FAKE_STACK_FRAME $child_rip
817 SAVE_ALL
819 # rdi: flags, rsi: usp, rdx: will be &pt_regs
820 movq %rdx,%rdi
821 orq kernel_thread_flags(%rip),%rdi
822 movq $-1, %rsi
823 movq %rsp, %rdx
825 xorl %r8d,%r8d
826 xorl %r9d,%r9d
828 # clone now
829 call do_fork
830 movq %rax,RAX(%rsp)
831 xorl %edi,%edi
833 /*
834 * It isn't worth to check for reschedule here,
835 * so internally to the x86_64 port you can rely on kernel_thread()
836 * not to reschedule the child before returning, this avoids the need
837 * of hacks for example to fork off the per-CPU idle tasks.
838 * [Hopefully no generic code relies on the reschedule -AK]
839 */
840 RESTORE_ALL
841 UNFAKE_STACK_FRAME
842 ret
843 CFI_ENDPROC
846 child_rip:
847 /*
848 * Here we are in the child and the registers are set as they were
849 * at kernel_thread() invocation in the parent.
850 */
851 movq %rdi, %rax
852 movq %rsi, %rdi
853 call *%rax
854 # exit
855 xorq %rdi, %rdi
856 call do_exit
858 /*
859 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
860 *
861 * C extern interface:
862 * extern long execve(char *name, char **argv, char **envp)
863 *
864 * asm input arguments:
865 * rdi: name, rsi: argv, rdx: envp
866 *
867 * We want to fallback into:
868 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
869 *
870 * do_sys_execve asm fallback arguments:
871 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
872 */
873 ENTRY(execve)
874 CFI_STARTPROC
875 FAKE_STACK_FRAME $0
876 SAVE_ALL
877 call sys_execve
878 movq %rax, RAX(%rsp)
879 RESTORE_REST
880 testq %rax,%rax
881 jne 1f
882 jmp int_ret_from_sys_call
883 1: RESTORE_ARGS
884 UNFAKE_STACK_FRAME
885 ret
886 CFI_ENDPROC
888 ENTRY(page_fault)
889 errorentry do_page_fault
891 ENTRY(coprocessor_error)
892 zeroentry do_coprocessor_error
894 ENTRY(simd_coprocessor_error)
895 zeroentry do_simd_coprocessor_error
897 ENTRY(device_not_available)
898 zeroentry math_state_restore
900 /* runs on exception stack */
901 ENTRY(debug)
902 CFI_STARTPROC
903 /* pushq $0
904 CFI_ADJUST_CFA_OFFSET 8 */
905 zeroentry do_debug
906 /* jmp paranoid_exit */
907 CFI_ENDPROC
909 #if 0
910 /* runs on exception stack */
911 ENTRY(nmi)
912 CFI_STARTPROC
913 pushq $-1
914 CFI_ADJUST_CFA_OFFSET 8
915 paranoidentry do_nmi
916 /*
917 * "Paranoid" exit path from exception stack.
918 * Paranoid because this is used by NMIs and cannot take
919 * any kernel state for granted.
920 * We don't do kernel preemption checks here, because only
921 * NMI should be common and it does not enable IRQs and
922 * cannot get reschedule ticks.
923 */
924 /* ebx: no swapgs flag */
925 paranoid_exit:
926 testl %ebx,%ebx /* swapgs needed? */
927 jnz paranoid_restore
928 testl $3,CS(%rsp)
929 jnz paranoid_userspace
930 paranoid_swapgs:
931 swapgs
932 paranoid_restore:
933 RESTORE_ALL 8
934 iretq
935 paranoid_userspace:
936 GET_THREAD_INFO(%rcx)
937 movl threadinfo_flags(%rcx),%ebx
938 andl $_TIF_WORK_MASK,%ebx
939 jz paranoid_swapgs
940 movq %rsp,%rdi /* &pt_regs */
941 call sync_regs
942 movq %rax,%rsp /* switch stack for scheduling */
943 testl $_TIF_NEED_RESCHED,%ebx
944 jnz paranoid_schedule
945 movl %ebx,%edx /* arg3: thread flags */
946 sti
947 xorl %esi,%esi /* arg2: oldset */
948 movq %rsp,%rdi /* arg1: &pt_regs */
949 call do_notify_resume
950 cli
951 jmp paranoid_userspace
952 paranoid_schedule:
953 sti
954 call schedule
955 cli
956 jmp paranoid_userspace
957 CFI_ENDPROC
958 #endif
960 ENTRY(int3)
961 zeroentry do_int3
963 ENTRY(overflow)
964 zeroentry do_overflow
966 ENTRY(bounds)
967 zeroentry do_bounds
969 ENTRY(invalid_op)
970 zeroentry do_invalid_op
972 ENTRY(coprocessor_segment_overrun)
973 zeroentry do_coprocessor_segment_overrun
975 ENTRY(reserved)
976 zeroentry do_reserved
978 #if 0
979 /* runs on exception stack */
980 ENTRY(double_fault)
981 CFI_STARTPROC
982 paranoidentry do_double_fault
983 jmp paranoid_exit
984 CFI_ENDPROC
985 #endif
987 ENTRY(invalid_TSS)
988 errorentry do_invalid_TSS
990 ENTRY(segment_not_present)
991 errorentry do_segment_not_present
993 /* runs on exception stack */
994 ENTRY(stack_segment)
995 CFI_STARTPROC
996 errorentry do_stack_segment
997 CFI_ENDPROC
999 ENTRY(general_protection)
1000 errorentry do_general_protection
1002 ENTRY(alignment_check)
1003 errorentry do_alignment_check
1005 ENTRY(divide_error)
1006 zeroentry do_divide_error
1008 ENTRY(spurious_interrupt_bug)
1009 zeroentry do_spurious_interrupt_bug
1011 #ifdef CONFIG_X86_MCE
1012 /* runs on exception stack */
1013 ENTRY(machine_check)
1014 CFI_STARTPROC
1015 pushq $0
1016 CFI_ADJUST_CFA_OFFSET 8
1017 paranoidentry do_machine_check
1018 jmp paranoid_exit
1019 CFI_ENDPROC
1020 #endif
1022 ENTRY(call_debug)
1023 zeroentry do_call_debug