ia64/xen-unstable

view xen/arch/x86/x86_64/entry.S @ 4696:e686528abbfc

bitkeeper revision 1.1389.3.1 (42714dabVSywx2XWGjgw2J54ZylwYg)

Ensure block/yield hypercalls always return a sane return code.

Ensure callers of __enter_scheduler take appropriate arch-specific
action if no context switch occurs (callers from arch/x86 do not
expect to return from a call into the scheduler).

This fixes wildly unintuitive behaviour of do_block() for the
VMX team.

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Thu Apr 28 20:55:07 2005 +0000 (2005-04-28)
parents cd690b71434a
children 123bd8c4b408
line source
1 /*
2 * Hypercall and fault low-level handling routines.
3 *
4 * Copyright (c) 2005, K A Fraser
5 */
7 #include <xen/config.h>
8 #include <xen/errno.h>
9 #include <xen/softirq.h>
10 #include <asm/asm_defns.h>
11 #include <asm/apicdef.h>
12 #include <asm/page.h>
13 #include <public/xen.h>
15 #define GET_CURRENT(reg) \
16 movq $STACK_SIZE-8, reg; \
17 orq %rsp, reg; \
18 andq $~7,reg; \
19 movq (reg),reg;
21 ALIGN
22 /* %rbx: struct exec_domain, interrupts disabled */
23 switch_to_kernel:
24 leaq EDOMAIN_trap_bounce(%rbx),%rdx
25 movq EDOMAIN_syscall_addr(%rbx),%rax
26 movq %rax,TRAPBOUNCE_eip(%rdx)
27 movw $0,TRAPBOUNCE_flags(%rdx)
28 call create_bounce_frame
30 /* %rbx: struct exec_domain */
31 restore_all_guest:
32 RESTORE_ALL
33 testw $TRAP_syscall,4(%rsp)
34 jz iret_exit_to_guest
36 addq $8,%rsp
37 popq %rcx # RIP
38 popq %r11 # CS
39 cmpw $__GUEST_CS32,%r11
40 popq %r11 # RFLAGS
41 cli # No interrupts after stack switch
42 popq %rsp # RSP
43 je 1f
44 sysretq
45 1: sysretl
47 ALIGN
48 /* No special register assumptions. */
49 iret_exit_to_guest:
50 addq $8,%rsp
51 FLT1: iretq
53 .section .fixup,"ax"
54 FIX1: popq -15*8-8(%rsp) # error_code/entry_vector
55 SAVE_ALL # 15*8 bytes pushed
56 movq -8(%rsp),%rsi # error_code/entry_vector
57 sti # after stack abuse (-1024(%rsp))
58 pushq $__HYPERVISOR_DS # SS
59 leaq 8(%rsp),%rax
60 pushq %rax # RSP
61 pushf # RFLAGS
62 pushq $__HYPERVISOR_CS # CS
63 leaq DBLFLT1(%rip),%rax
64 pushq %rax # RIP
65 pushq %rsi # error_code/entry_vector
66 jmp error_code
67 DBLFLT1:GET_CURRENT(%rbx)
68 jmp test_all_events
69 failsafe_callback:
70 GET_CURRENT(%rbx)
71 leaq EDOMAIN_trap_bounce(%rbx),%rdx
72 movq EDOMAIN_failsafe_addr(%rbx),%rax
73 movq %rax,TRAPBOUNCE_eip(%rdx)
74 movw $TBF_FAILSAFE,TRAPBOUNCE_flags(%rdx)
75 call create_bounce_frame
76 jmp test_all_events
77 .previous
78 .section __pre_ex_table,"a"
79 .quad FLT1,FIX1
80 .previous
81 .section __ex_table,"a"
82 .quad DBLFLT1,failsafe_callback
83 .previous
85 ALIGN
86 /* No special register assumptions. */
87 restore_all_xen:
88 RESTORE_ALL
89 addq $8,%rsp
90 iretq
92 /*
93 * When entering SYSCALL from kernel mode:
94 * %rax = hypercall vector
95 * %rdi, %rsi, %rdx, %r10, %r8, %9 = hypercall arguments
96 * %r11, %rcx = SYSCALL-saved %rflags and %rip
97 * NB. We must move %r10 to %rcx for C function-calling ABI.
98 *
99 * When entering SYSCALL from user mode:
100 * Vector directly to the registered arch.syscall_addr.
101 *
102 * Initial work is done by per-CPU stack trampolines. At this point %rsp
103 * has been initialised to point at the correct Xen stack, and %rsp, %rflags
104 * and %cs have been saved. All other registers are still to be saved onto
105 * the stack, starting with %rip, and an appropriate %ss must be saved into
106 * the space left by the trampoline.
107 */
108 ALIGN
109 ENTRY(syscall_enter)
110 movl $__GUEST_SS,24(%rsp)
111 pushq %rcx
112 pushq $0
113 movl $TRAP_syscall,4(%rsp)
114 SAVE_ALL
115 GET_CURRENT(%rbx)
116 testb $TF_kernel_mode,EDOMAIN_thread_flags(%rbx)
117 jz switch_to_kernel
119 /*hypercall:*/
120 sti
121 movq %r10,%rcx
122 andq $(NR_hypercalls-1),%rax
123 leaq SYMBOL_NAME(hypercall_table)(%rip),%r10
124 PERFC_INCR(PERFC_hypercalls, %rax)
125 callq *(%r10,%rax,8)
126 movq %rax,UREGS_rax(%rsp) # save the return value
128 /* %rbx: struct exec_domain */
129 test_all_events:
130 cli # tests must not race interrupts
131 /*test_softirqs:*/
132 movl EDOMAIN_processor(%rbx),%eax
133 shl $IRQSTAT_shift,%rax
134 leaq SYMBOL_NAME(irq_stat)(%rip),%rcx
135 testl $~0,(%rcx,%rax,1)
136 jnz process_softirqs
137 /*test_guest_events:*/
138 movq EDOMAIN_vcpu_info(%rbx),%rax
139 testb $0xFF,VCPUINFO_upcall_mask(%rax)
140 jnz restore_all_guest
141 testb $0xFF,VCPUINFO_upcall_pending(%rax)
142 jz restore_all_guest
143 /*process_guest_events:*/
144 sti
145 leaq EDOMAIN_trap_bounce(%rbx),%rdx
146 movq EDOMAIN_event_addr(%rbx),%rax
147 movq %rax,TRAPBOUNCE_eip(%rdx)
148 movw $TBF_INTERRUPT,TRAPBOUNCE_flags(%rdx)
149 call create_bounce_frame
150 movq EDOMAIN_vcpu_info(%rbx),%rax
151 movb $1,VCPUINFO_upcall_mask(%rax) # Upcalls masked during delivery
152 jmp test_all_events
154 #ifdef CONFIG_VMX
155 /*
156 * At VMExit time the processor saves the guest selectors, rsp, rip,
157 * and rflags. Therefore we don't save them, but simply decrement
158 * the kernel stack pointer to make it consistent with the stack frame
159 * at usual interruption time. The rflags of the host is not saved by VMX,
160 * and we set it to the fixed value.
161 *
162 * We also need the room, especially because orig_eax field is used
163 * by do_IRQ(). Compared the cpu_user_regs, we skip pushing for the following:
164 * (10) u64 gs;
165 * (9) u64 fs;
166 * (8) u64 ds;
167 * (7) u64 es;
168 * <- get_stack_bottom() (= HOST_ESP)
169 * (6) u64 ss;
170 * (5) u64 rsp;
171 * (4) u64 rflags;
172 * (3) u64 cs;
173 * (2) u64 rip;
174 * (2/1) u32 entry_vector;
175 * (1/1) u32 error_code;
176 */
177 #define VMX_MONITOR_RFLAGS 0x202 /* IF on */
178 #define NR_SKIPPED_REGS 6 /* See the above explanation */
179 #define VMX_SAVE_ALL_NOSEGREGS \
180 pushq $VMX_MONITOR_RFLAGS; \
181 popfq; \
182 subq $(NR_SKIPPED_REGS*8), %rsp; \
183 pushq %rdi; \
184 pushq %rsi; \
185 pushq %rdx; \
186 pushq %rcx; \
187 pushq %rax; \
188 pushq %r8; \
189 pushq %r9; \
190 pushq %r10; \
191 pushq %r11; \
192 pushq %rbx; \
193 pushq %rbp; \
194 pushq %r12; \
195 pushq %r13; \
196 pushq %r14; \
197 pushq %r15; \
199 ENTRY(vmx_asm_vmexit_handler)
200 /* selectors are restored/saved by VMX */
201 VMX_SAVE_ALL_NOSEGREGS
202 call SYMBOL_NAME(vmx_vmexit_handler)
203 jmp vmx_asm_do_resume
205 ENTRY(vmx_asm_do_launch)
206 popq %r15
207 popq %r14
208 popq %r13
209 popq %r12
210 popq %rbp
211 popq %rbx
212 popq %r11
213 popq %r10
214 popq %r9
215 popq %r8
216 popq %rax
217 popq %rcx
218 popq %rdx
219 popq %rsi
220 popq %rdi
221 addq $(NR_SKIPPED_REGS*8), %rsp
222 /* VMLUANCH */
223 .byte 0x0f,0x01,0xc2
224 pushfq
225 call SYMBOL_NAME(vm_launch_fail)
226 hlt
228 ALIGN
230 ENTRY(vmx_asm_do_resume)
231 vmx_test_all_events:
232 GET_CURRENT(%rbx)
233 /* test_all_events: */
234 cli # tests must not race interrupts
235 /*test_softirqs:*/
236 movl EDOMAIN_processor(%rbx),%eax
237 shl $IRQSTAT_shift,%rax
238 leaq SYMBOL_NAME(irq_stat)(%rip), %rdx
239 testl $~0,(%rdx,%rax,1)
240 jnz vmx_process_softirqs
242 vmx_restore_all_guest:
243 call SYMBOL_NAME(load_cr2)
244 /*
245 * Check if we are going back to VMX-based VM
246 * By this time, all the setups in the VMCS must be complete.
247 */
248 popq %r15
249 popq %r14
250 popq %r13
251 popq %r12
252 popq %rbp
253 popq %rbx
254 popq %r11
255 popq %r10
256 popq %r9
257 popq %r8
258 popq %rax
259 popq %rcx
260 popq %rdx
261 popq %rsi
262 popq %rdi
263 addq $(NR_SKIPPED_REGS*8), %rsp
264 /* VMRESUME */
265 .byte 0x0f,0x01,0xc3
266 pushfq
267 call SYMBOL_NAME(vm_resume_fail)
268 /* Should never reach here */
269 hlt
271 ALIGN
272 vmx_process_softirqs:
273 sti
274 call SYMBOL_NAME(do_softirq)
275 jmp vmx_test_all_events
276 #endif
278 ALIGN
279 /* %rbx: struct exec_domain */
280 process_softirqs:
281 sti
282 call SYMBOL_NAME(do_softirq)
283 jmp test_all_events
285 /* CREATE A BASIC EXCEPTION FRAME ON GUEST OS STACK: */
286 /* { RCX, R11, [DS-GS,] [CR2,] [ERRCODE,] RIP, CS, RFLAGS, RSP, SS } */
287 /* %rdx: trap_bounce, %rbx: struct exec_domain */
288 /* On return only %rbx is guaranteed non-clobbered. */
289 create_bounce_frame:
290 testb $TF_kernel_mode,EDOMAIN_thread_flags(%rbx)
291 jnz 1f
292 /* Push new frame at registered guest-OS stack base. */
293 pushq %rdx
294 movq %rbx,%rdi
295 call SYMBOL_NAME(toggle_guest_mode)
296 popq %rdx
297 movq EDOMAIN_kernel_sp(%rbx),%rsi
298 jmp 2f
299 1: /* In kernel context already: push new frame at existing %rsp. */
300 movq UREGS_rsp+8(%rsp),%rsi
301 andb $0xfc,UREGS_cs+8(%rsp) # Indicate kernel context to guest.
302 2: movq $HYPERVISOR_VIRT_START,%rax
303 cmpq %rax,%rsi
304 jb 1f # In +ve address space? Then okay.
305 movq $HYPERVISOR_VIRT_END+60,%rax
306 cmpq %rax,%rsi
307 jb domain_crash_synchronous # Above Xen private area? Then okay.
308 1: subq $40,%rsi
309 movq UREGS_ss+8(%rsp),%rax
310 FLT2: movq %rax,32(%rsi) # SS
311 movq UREGS_rsp+8(%rsp),%rax
312 FLT3: movq %rax,24(%rsi) # RSP
313 movq UREGS_eflags+8(%rsp),%rax
314 FLT4: movq %rax,16(%rsi) # RFLAGS
315 movq UREGS_cs+8(%rsp),%rax
316 FLT5: movq %rax,8(%rsi) # CS
317 movq UREGS_rip+8(%rsp),%rax
318 FLT6: movq %rax,(%rsi) # RIP
319 movb TRAPBOUNCE_flags(%rdx),%cl
320 testb $TBF_EXCEPTION_ERRCODE,%cl
321 jz 1f
322 subq $8,%rsi
323 movl TRAPBOUNCE_error_code(%rdx),%eax
324 FLT7: movq %rax,(%rsi) # ERROR CODE
325 testb $TBF_EXCEPTION_CR2,%cl
326 jz 2f
327 subq $8,%rsi
328 movq TRAPBOUNCE_cr2(%rdx),%rax
329 FLT8: movq %rax,(%rsi) # CR2
330 1: testb $TBF_FAILSAFE,%cl
331 jz 2f
332 subq $32,%rsi
333 movl %gs,%eax
334 FLT9: movq %rax,24(%rsi) # GS
335 movl %fs,%eax
336 FLT10: movq %rax,16(%rsi) # FS
337 movl %es,%eax
338 FLT11: movq %rax,8(%rsi) # ES
339 movl %ds,%eax
340 FLT12: movq %rax,(%rsi) # DS
341 2: subq $16,%rsi
342 movq UREGS_r11+8(%rsp),%rax
343 FLT13: movq %rax,8(%rsi) # R11
344 movq UREGS_rcx+8(%rsp),%rax
345 FLT14: movq %rax,(%rsi) # RCX
346 /* Rewrite our stack frame and return to guest-OS mode. */
347 /* IA32 Ref. Vol. 3: TF, VM, RF and NT flags are cleared on trap. */
348 movq $TRAP_syscall,UREGS_entry_vector+8(%rsp)
349 andl $0xfffcbeff,UREGS_eflags+8(%rsp)
350 movq $__GUEST_SS,UREGS_ss+8(%rsp)
351 movq %rsi,UREGS_rsp+8(%rsp)
352 movq $__GUEST_CS,UREGS_cs+8(%rsp)
353 movq TRAPBOUNCE_eip(%rdx),%rax
354 movq %rax,UREGS_rip+8(%rsp)
355 movb $0,TRAPBOUNCE_flags(%rdx)
356 ret
357 .section __ex_table,"a"
358 .quad FLT2,domain_crash_synchronous , FLT3,domain_crash_synchronous
359 .quad FLT4,domain_crash_synchronous , FLT5,domain_crash_synchronous
360 .quad FLT6,domain_crash_synchronous , FLT7,domain_crash_synchronous
361 .quad FLT8,domain_crash_synchronous , FLT9,domain_crash_synchronous
362 .quad FLT10,domain_crash_synchronous , FLT11,domain_crash_synchronous
363 .quad FLT12,domain_crash_synchronous , FLT13,domain_crash_synchronous
364 .quad FLT14,domain_crash_synchronous
365 .previous
367 ALIGN
368 /* %rbx: struct exec_domain */
369 process_guest_exception_and_events:
370 leaq EDOMAIN_trap_bounce(%rbx),%rdx
371 testb $TBF_EXCEPTION,TRAPBOUNCE_flags(%rdx)
372 jz test_all_events
373 call create_bounce_frame
374 jmp test_all_events
376 ALIGN
377 /* No special register assumptions. */
378 ENTRY(ret_from_intr)
379 GET_CURRENT(%rbx)
380 testb $3,UREGS_cs(%rsp)
381 jnz test_all_events
382 jmp restore_all_xen
384 ALIGN
385 /* No special register assumptions. */
386 error_code:
387 SAVE_ALL
388 testb $X86_EFLAGS_IF>>8,UREGS_eflags+1(%rsp)
389 jz exception_with_ints_disabled
390 sti
391 movq %rsp,%rdi
392 movl UREGS_entry_vector(%rsp),%eax
393 leaq SYMBOL_NAME(exception_table)(%rip),%rdx
394 GET_CURRENT(%rbx)
395 PERFC_INCR(PERFC_exceptions, %rax)
396 callq *(%rdx,%rax,8)
397 testb $3,UREGS_cs(%rsp)
398 jz restore_all_xen
399 jmp process_guest_exception_and_events
401 /* No special register assumptions. */
402 exception_with_ints_disabled:
403 testb $3,UREGS_cs(%rsp) # interrupts disabled outside Xen?
404 jnz FATAL_exception_with_ints_disabled
405 movq %rsp,%rdi
406 call search_pre_exception_table
407 testq %rax,%rax # no fixup code for faulting EIP?
408 jz FATAL_exception_with_ints_disabled
409 movq %rax,UREGS_rip(%rsp)
410 subq $8,UREGS_rsp(%rsp) # add ec/ev to previous stack frame
411 testb $15,UREGS_rsp(%rsp) # return %rsp is now aligned?
412 jz 1f # then there is a pad quadword already
413 movq %rsp,%rsi
414 subq $8,%rsp
415 movq %rsp,%rdi
416 movq $UREGS_kernel_sizeof/8,%rcx
417 rep; movsq # make room for ec/ev
418 1: movq UREGS_error_code(%rsp),%rax # ec/ev
419 movq %rax,UREGS_kernel_sizeof(%rsp)
420 jmp restore_all_xen # return to fixup code
422 /* No special register assumptions. */
423 FATAL_exception_with_ints_disabled:
424 movl UREGS_entry_vector(%rsp),%edi
425 movq %rsp,%rsi
426 call SYMBOL_NAME(fatal_trap)
427 ud2
429 ENTRY(divide_error)
430 pushq $0
431 movl $TRAP_divide_error,4(%rsp)
432 jmp error_code
434 ENTRY(coprocessor_error)
435 pushq $0
436 movl $TRAP_copro_error,4(%rsp)
437 jmp error_code
439 ENTRY(simd_coprocessor_error)
440 pushq $0
441 movl $TRAP_simd_error,4(%rsp)
442 jmp error_code
444 ENTRY(device_not_available)
445 pushq $0
446 movl $TRAP_no_device,4(%rsp)
447 jmp error_code
449 ENTRY(debug)
450 pushq $0
451 movl $TRAP_debug,4(%rsp)
452 jmp error_code
454 ENTRY(int3)
455 pushq $0
456 movl $TRAP_int3,4(%rsp)
457 jmp error_code
459 ENTRY(overflow)
460 pushq $0
461 movl $TRAP_overflow,4(%rsp)
462 jmp error_code
464 ENTRY(bounds)
465 pushq $0
466 movl $TRAP_bounds,4(%rsp)
467 jmp error_code
469 ENTRY(invalid_op)
470 pushq $0
471 movl $TRAP_invalid_op,4(%rsp)
472 jmp error_code
474 ENTRY(coprocessor_segment_overrun)
475 pushq $0
476 movl $TRAP_copro_seg,4(%rsp)
477 jmp error_code
479 ENTRY(invalid_TSS)
480 movl $TRAP_invalid_tss,4(%rsp)
481 jmp error_code
483 ENTRY(segment_not_present)
484 movl $TRAP_no_segment,4(%rsp)
485 jmp error_code
487 ENTRY(stack_segment)
488 movl $TRAP_stack_error,4(%rsp)
489 jmp error_code
491 ENTRY(general_protection)
492 movl $TRAP_gp_fault,4(%rsp)
493 jmp error_code
495 ENTRY(alignment_check)
496 movl $TRAP_alignment_check,4(%rsp)
497 jmp error_code
499 ENTRY(page_fault)
500 movl $TRAP_page_fault,4(%rsp)
501 jmp error_code
503 ENTRY(machine_check)
504 pushq $0
505 movl $TRAP_machine_check,4(%rsp)
506 jmp error_code
508 ENTRY(spurious_interrupt_bug)
509 pushq $0
510 movl $TRAP_spurious_int,4(%rsp)
511 jmp error_code
513 ENTRY(double_fault)
514 movl $TRAP_double_fault,4(%rsp)
515 jmp error_code
517 ENTRY(nmi)
518 pushq $0
519 SAVE_ALL
520 inb $0x61,%al
521 movl %eax,%esi # reason
522 movq %rsp,%rdi # regs
523 call SYMBOL_NAME(do_nmi)
524 jmp restore_all_xen
526 do_arch_sched_op:
527 # Ensure we return success even if we return via schedule_tail()
528 xorl %eax,%eax
529 movq %rax,UREGS_rax+8(%rsp)
530 jmp SYMBOL_NAME(do_sched_op)
532 .data
534 ENTRY(exception_table)
535 .quad SYMBOL_NAME(do_divide_error)
536 .quad SYMBOL_NAME(do_debug)
537 .quad 0 # nmi
538 .quad SYMBOL_NAME(do_int3)
539 .quad SYMBOL_NAME(do_overflow)
540 .quad SYMBOL_NAME(do_bounds)
541 .quad SYMBOL_NAME(do_invalid_op)
542 .quad SYMBOL_NAME(math_state_restore)
543 .quad SYMBOL_NAME(do_double_fault)
544 .quad SYMBOL_NAME(do_coprocessor_segment_overrun)
545 .quad SYMBOL_NAME(do_invalid_TSS)
546 .quad SYMBOL_NAME(do_segment_not_present)
547 .quad SYMBOL_NAME(do_stack_segment)
548 .quad SYMBOL_NAME(do_general_protection)
549 .quad SYMBOL_NAME(do_page_fault)
550 .quad SYMBOL_NAME(do_spurious_interrupt_bug)
551 .quad SYMBOL_NAME(do_coprocessor_error)
552 .quad SYMBOL_NAME(do_alignment_check)
553 .quad SYMBOL_NAME(do_machine_check)
554 .quad SYMBOL_NAME(do_simd_coprocessor_error)
556 ENTRY(hypercall_table)
557 .quad SYMBOL_NAME(do_set_trap_table) /* 0 */
558 .quad SYMBOL_NAME(do_mmu_update)
559 .quad SYMBOL_NAME(do_set_gdt)
560 .quad SYMBOL_NAME(do_stack_switch)
561 .quad SYMBOL_NAME(do_set_callbacks)
562 .quad SYMBOL_NAME(do_fpu_taskswitch) /* 5 */
563 .quad SYMBOL_NAME(do_arch_sched_op)
564 .quad SYMBOL_NAME(do_dom0_op)
565 .quad SYMBOL_NAME(do_set_debugreg)
566 .quad SYMBOL_NAME(do_get_debugreg)
567 .quad SYMBOL_NAME(do_update_descriptor) /* 10 */
568 .quad SYMBOL_NAME(do_ni_hypercall)
569 .quad SYMBOL_NAME(do_dom_mem_op)
570 .quad SYMBOL_NAME(do_multicall)
571 .quad SYMBOL_NAME(do_update_va_mapping)
572 .quad SYMBOL_NAME(do_set_timer_op) /* 15 */
573 .quad SYMBOL_NAME(do_event_channel_op)
574 .quad SYMBOL_NAME(do_xen_version)
575 .quad SYMBOL_NAME(do_console_io)
576 .quad SYMBOL_NAME(do_physdev_op)
577 .quad SYMBOL_NAME(do_grant_table_op) /* 20 */
578 .quad SYMBOL_NAME(do_vm_assist)
579 .quad SYMBOL_NAME(do_update_va_mapping_otherdomain)
580 .quad SYMBOL_NAME(do_switch_to_user)
581 .quad SYMBOL_NAME(do_boot_vcpu)
582 .quad SYMBOL_NAME(do_set_segment_base) /* 25 */
583 .quad SYMBOL_NAME(do_mmuext_op)
584 .rept NR_hypercalls-((.-hypercall_table)/4)
585 .quad SYMBOL_NAME(do_ni_hypercall)
586 .endr