direct-io.hg

view xen/arch/x86/x86_64/entry.S @ 4533:c62f4ac13428

bitkeeper revision 1.1300 (425ef0b4lXDHXsulXDmrVOvOZAhPXQ)

Do not STI when transitioning from user to kernel mode in
x86/64 Xen. Otherwise, by directly entering restore_all_guest
without first executing test_all_events, we risk ignoring
pending softirq and event-delivery work.
Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Thu Apr 14 22:37:40 2005 +0000 (2005-04-14)
parents 445b12a7221a
children b1cb9f7f34f9
line source
1 /*
2 * Hypercall and fault low-level handling routines.
3 *
4 * Copyright (c) 2005, K A Fraser
5 */
7 #include <xen/config.h>
8 #include <xen/errno.h>
9 #include <xen/softirq.h>
10 #include <asm/asm_defns.h>
11 #include <asm/apicdef.h>
12 #include <asm/page.h>
13 #include <public/xen.h>
15 #define GET_CURRENT(reg) \
16 movq $STACK_SIZE-8, reg; \
17 orq %rsp, reg; \
18 andq $~7,reg; \
19 movq (reg),reg;
21 ALIGN
22 /* %rbx: struct exec_domain, interrupts disabled */
23 switch_to_kernel:
24 leaq EDOMAIN_trap_bounce(%rbx),%rdx
25 movq EDOMAIN_syscall_addr(%rbx),%rax
26 movq %rax,TRAPBOUNCE_eip(%rdx)
27 movw $0,TRAPBOUNCE_flags(%rdx)
28 call create_bounce_frame
30 /* %rbx: struct exec_domain */
31 restore_all_guest:
32 RESTORE_ALL
33 testw $TRAP_syscall,4(%rsp)
34 jz iret_exit_to_guest
36 addq $8,%rsp
37 popq %rcx # RIP
38 popq %r11 # CS
39 cmpw $__GUEST_CS32,%r11
40 popq %r11 # RFLAGS
41 cli # No interrupts after stack switch
42 popq %rsp # RSP
43 je 1f
44 sysretq
45 1: sysretl
47 ALIGN
48 /* No special register assumptions. */
49 iret_exit_to_guest:
50 addq $8,%rsp
51 FLT1: iretq
53 .section .fixup,"ax"
54 FIX1: popq -15*8-8(%rsp) # error_code/entry_vector
55 SAVE_ALL # 15*8 bytes pushed
56 movq -8(%rsp),%rsi # error_code/entry_vector
57 sti # after stack abuse (-1024(%rsp))
58 pushq $__HYPERVISOR_DS # SS
59 leaq 8(%rsp),%rax
60 pushq %rax # RSP
61 pushf # RFLAGS
62 pushq $__HYPERVISOR_CS # CS
63 leaq DBLFLT1(%rip),%rax
64 pushq %rax # RIP
65 pushq %rsi # error_code/entry_vector
66 jmp error_code
67 DBLFLT1:GET_CURRENT(%rbx)
68 jmp test_all_events
69 failsafe_callback:
70 GET_CURRENT(%rbx)
71 leaq EDOMAIN_trap_bounce(%rbx),%rdx
72 movq EDOMAIN_failsafe_addr(%rbx),%rax
73 movq %rax,TRAPBOUNCE_eip(%rdx)
74 movw $TBF_FAILSAFE,TRAPBOUNCE_flags(%rdx)
75 call create_bounce_frame
76 jmp test_all_events
77 .previous
78 .section __pre_ex_table,"a"
79 .quad FLT1,FIX1
80 .previous
81 .section __ex_table,"a"
82 .quad DBLFLT1,failsafe_callback
83 .previous
85 ALIGN
86 /* No special register assumptions. */
87 restore_all_xen:
88 RESTORE_ALL
89 addq $8,%rsp
90 iretq
92 /*
93 * When entering SYSCALL from kernel mode:
94 * %rax = hypercall vector
95 * %rdi, %rsi, %rdx, %r10, %r8, %9 = hypercall arguments
96 * %r11, %rcx = SYSCALL-saved %rflags and %rip
97 * NB. We must move %r10 to %rcx for C function-calling ABI.
98 *
99 * When entering SYSCALL from user mode:
100 * Vector directly to the registered arch.syscall_addr.
101 *
102 * Initial work is done by per-CPU stack trampolines. At this point %rsp
103 * has been initialised to point at the correct Xen stack, and %rsp, %rflags
104 * and %cs have been saved. All other registers are still to be saved onto
105 * the stack, starting with %rip, and an appropriate %ss must be saved into
106 * the space left by the trampoline.
107 */
108 ALIGN
109 ENTRY(syscall_enter)
110 movl $__GUEST_SS,24(%rsp)
111 pushq %rcx
112 pushq $0
113 movl $TRAP_syscall,4(%rsp)
114 SAVE_ALL
115 GET_CURRENT(%rbx)
116 testb $TF_kernel_mode,EDOMAIN_thread_flags(%rbx)
117 jz switch_to_kernel
119 /*hypercall:*/
120 sti
121 movq %r10,%rcx
122 andq $(NR_hypercalls-1),%rax
123 leaq SYMBOL_NAME(hypercall_table)(%rip),%r10
124 PERFC_INCR(PERFC_hypercalls, %rax)
125 callq *(%r10,%rax,8)
126 movq %rax,XREGS_rax(%rsp) # save the return value
128 /* %rbx: struct exec_domain */
129 test_all_events:
130 cli # tests must not race interrupts
131 /*test_softirqs:*/
132 movl EDOMAIN_processor(%rbx),%eax
133 shl $6,%rax # sizeof(irq_cpustat) == 64
134 leaq SYMBOL_NAME(irq_stat)(%rip),%rcx
135 testl $~0,(%rcx,%rax,1)
136 jnz process_softirqs
137 /*test_guest_events:*/
138 movq EDOMAIN_vcpu_info(%rbx),%rax
139 testb $0xFF,VCPUINFO_upcall_mask(%rax)
140 jnz restore_all_guest
141 testb $0xFF,VCPUINFO_upcall_pending(%rax)
142 jz restore_all_guest
143 /*process_guest_events:*/
144 sti
145 leaq EDOMAIN_trap_bounce(%rbx),%rdx
146 movq EDOMAIN_event_addr(%rbx),%rax
147 movq %rax,TRAPBOUNCE_eip(%rdx)
148 movw $TBF_INTERRUPT,TRAPBOUNCE_flags(%rdx)
149 call create_bounce_frame
150 movq EDOMAIN_vcpu_info(%rbx),%rax
151 movb $1,VCPUINFO_upcall_mask(%rax) # Upcalls masked during delivery
152 jmp test_all_events
154 ALIGN
155 /* %rbx: struct exec_domain */
156 process_softirqs:
157 sti
158 call SYMBOL_NAME(do_softirq)
159 jmp test_all_events
161 /* CREATE A BASIC EXCEPTION FRAME ON GUEST OS STACK: */
162 /* { RCX, R11, [DS-GS,] [CR2,] [ERRCODE,] RIP, CS, RFLAGS, RSP, SS } */
163 /* %rdx: trap_bounce, %rbx: struct exec_domain */
164 /* On return only %rbx is guaranteed non-clobbered. */
165 create_bounce_frame:
166 testb $TF_kernel_mode,EDOMAIN_thread_flags(%rbx)
167 jnz 1f
168 /* Push new frame at registered guest-OS stack base. */
169 pushq %rdx
170 movq %rbx,%rdi
171 call SYMBOL_NAME(toggle_guest_mode)
172 popq %rdx
173 movq EDOMAIN_kernel_sp(%rbx),%rsi
174 jmp 2f
175 1: /* In kernel context already: push new frame at existing %rsp. */
176 movq XREGS_rsp+8(%rsp),%rsi
177 andb $0xfc,XREGS_cs+8(%rsp) # Indicate kernel context to guest.
178 2: movq $HYPERVISOR_VIRT_START,%rax
179 cmpq %rax,%rsi
180 jb 1f # In +ve address space? Then okay.
181 movq $HYPERVISOR_VIRT_END+60,%rax
182 cmpq %rax,%rsi
183 jb domain_crash_synchronous # Above Xen private area? Then okay.
184 1: subq $40,%rsi
185 movq XREGS_ss+8(%rsp),%rax
186 FLT2: movq %rax,32(%rsi) # SS
187 movq XREGS_rsp+8(%rsp),%rax
188 FLT3: movq %rax,24(%rsi) # RSP
189 movq XREGS_eflags+8(%rsp),%rax
190 FLT4: movq %rax,16(%rsi) # RFLAGS
191 movq XREGS_cs+8(%rsp),%rax
192 FLT5: movq %rax,8(%rsi) # CS
193 movq XREGS_rip+8(%rsp),%rax
194 FLT6: movq %rax,(%rsi) # RIP
195 movb TRAPBOUNCE_flags(%rdx),%cl
196 testb $TBF_EXCEPTION_ERRCODE,%cl
197 jz 1f
198 subq $8,%rsi
199 movl TRAPBOUNCE_error_code(%rdx),%eax
200 FLT7: movq %rax,(%rsi) # ERROR CODE
201 testb $TBF_EXCEPTION_CR2,%cl
202 jz 2f
203 subq $8,%rsi
204 movq TRAPBOUNCE_cr2(%rdx),%rax
205 FLT8: movq %rax,(%rsi) # CR2
206 1: testb $TBF_FAILSAFE,%cl
207 jz 2f
208 subq $32,%rsi
209 movl %gs,%eax
210 FLT9: movq %rax,24(%rsi) # GS
211 movl %fs,%eax
212 FLT10: movq %rax,16(%rsi) # FS
213 movl %es,%eax
214 FLT11: movq %rax,8(%rsi) # ES
215 movl %ds,%eax
216 FLT12: movq %rax,(%rsi) # DS
217 2: subq $16,%rsi
218 movq XREGS_r11+8(%rsp),%rax
219 FLT13: movq %rax,8(%rsi) # R11
220 movq XREGS_rcx+8(%rsp),%rax
221 FLT14: movq %rax,(%rsi) # RCX
222 /* Rewrite our stack frame and return to guest-OS mode. */
223 /* IA32 Ref. Vol. 3: TF, VM, RF and NT flags are cleared on trap. */
224 movq $TRAP_syscall,XREGS_entry_vector+8(%rsp)
225 andl $0xfffcbeff,XREGS_eflags+8(%rsp)
226 movq $__GUEST_SS,XREGS_ss+8(%rsp)
227 movq %rsi,XREGS_rsp+8(%rsp)
228 movq $__GUEST_CS,XREGS_cs+8(%rsp)
229 movq TRAPBOUNCE_eip(%rdx),%rax
230 movq %rax,XREGS_rip+8(%rsp)
231 movb $0,TRAPBOUNCE_flags(%rdx)
232 ret
233 .section __ex_table,"a"
234 .quad FLT2,domain_crash_synchronous , FLT3,domain_crash_synchronous
235 .quad FLT4,domain_crash_synchronous , FLT5,domain_crash_synchronous
236 .quad FLT6,domain_crash_synchronous , FLT7,domain_crash_synchronous
237 .quad FLT8,domain_crash_synchronous , FLT9,domain_crash_synchronous
238 .quad FLT10,domain_crash_synchronous , FLT11,domain_crash_synchronous
239 .quad FLT12,domain_crash_synchronous , FLT13,domain_crash_synchronous
240 .quad FLT14,domain_crash_synchronous
241 .previous
243 ALIGN
244 /* %rbx: struct exec_domain */
245 process_guest_exception_and_events:
246 leaq EDOMAIN_trap_bounce(%rbx),%rdx
247 testb $TBF_EXCEPTION,TRAPBOUNCE_flags(%rdx)
248 jz test_all_events
249 call create_bounce_frame
250 jmp test_all_events
252 ALIGN
253 /* No special register assumptions. */
254 ENTRY(ret_from_intr)
255 GET_CURRENT(%rbx)
256 testb $3,XREGS_cs(%rsp)
257 jnz test_all_events
258 jmp restore_all_xen
260 ALIGN
261 /* No special register assumptions. */
262 error_code:
263 SAVE_ALL
264 testb $X86_EFLAGS_IF>>8,XREGS_eflags+1(%rsp)
265 jz exception_with_ints_disabled
266 sti
267 movq %rsp,%rdi
268 movl XREGS_entry_vector(%rsp),%eax
269 leaq SYMBOL_NAME(exception_table)(%rip),%rdx
270 GET_CURRENT(%rbx)
271 PERFC_INCR(PERFC_exceptions, %rax)
272 callq *(%rdx,%rax,8)
273 testb $3,XREGS_cs(%rsp)
274 jz restore_all_xen
275 jmp process_guest_exception_and_events
277 /* No special register assumptions. */
278 exception_with_ints_disabled:
279 testb $3,XREGS_cs(%rsp) # interrupts disabled outside Xen?
280 jnz FATAL_exception_with_ints_disabled
281 movq %rsp,%rdi
282 call search_pre_exception_table
283 testq %rax,%rax # no fixup code for faulting EIP?
284 jz FATAL_exception_with_ints_disabled
285 movq %rax,XREGS_rip(%rsp)
286 subq $8,XREGS_rsp(%rsp) # add ec/ev to previous stack frame
287 testb $15,XREGS_rsp(%rsp) # return %rsp is now aligned?
288 jz 1f # then there is a pad quadword already
289 movq %rsp,%rsi
290 subq $8,%rsp
291 movq %rsp,%rdi
292 movq $XREGS_kernel_sizeof/8,%rcx
293 rep; movsq # make room for ec/ev
294 1: movq XREGS_error_code(%rsp),%rax # ec/ev
295 movq %rax,XREGS_kernel_sizeof(%rsp)
296 jmp restore_all_xen # return to fixup code
298 /* No special register assumptions. */
299 FATAL_exception_with_ints_disabled:
300 movl XREGS_entry_vector(%rsp),%edi
301 movq %rsp,%rsi
302 call SYMBOL_NAME(fatal_trap)
303 ud2
305 ENTRY(divide_error)
306 pushq $0
307 movl $TRAP_divide_error,4(%rsp)
308 jmp error_code
310 ENTRY(coprocessor_error)
311 pushq $0
312 movl $TRAP_copro_error,4(%rsp)
313 jmp error_code
315 ENTRY(simd_coprocessor_error)
316 pushq $0
317 movl $TRAP_simd_error,4(%rsp)
318 jmp error_code
320 ENTRY(device_not_available)
321 pushq $0
322 movl $TRAP_no_device,4(%rsp)
323 jmp error_code
325 ENTRY(debug)
326 pushq $0
327 movl $TRAP_debug,4(%rsp)
328 jmp error_code
330 ENTRY(int3)
331 pushq $0
332 movl $TRAP_int3,4(%rsp)
333 jmp error_code
335 ENTRY(overflow)
336 pushq $0
337 movl $TRAP_overflow,4(%rsp)
338 jmp error_code
340 ENTRY(bounds)
341 pushq $0
342 movl $TRAP_bounds,4(%rsp)
343 jmp error_code
345 ENTRY(invalid_op)
346 pushq $0
347 movl $TRAP_invalid_op,4(%rsp)
348 jmp error_code
350 ENTRY(coprocessor_segment_overrun)
351 pushq $0
352 movl $TRAP_copro_seg,4(%rsp)
353 jmp error_code
355 ENTRY(invalid_TSS)
356 movl $TRAP_invalid_tss,4(%rsp)
357 jmp error_code
359 ENTRY(segment_not_present)
360 movl $TRAP_no_segment,4(%rsp)
361 jmp error_code
363 ENTRY(stack_segment)
364 movl $TRAP_stack_error,4(%rsp)
365 jmp error_code
367 ENTRY(general_protection)
368 movl $TRAP_gp_fault,4(%rsp)
369 jmp error_code
371 ENTRY(alignment_check)
372 movl $TRAP_alignment_check,4(%rsp)
373 jmp error_code
375 ENTRY(page_fault)
376 movl $TRAP_page_fault,4(%rsp)
377 jmp error_code
379 ENTRY(machine_check)
380 pushq $0
381 movl $TRAP_machine_check,4(%rsp)
382 jmp error_code
384 ENTRY(spurious_interrupt_bug)
385 pushq $0
386 movl $TRAP_spurious_int,4(%rsp)
387 jmp error_code
389 ENTRY(double_fault)
390 movl $TRAP_double_fault,4(%rsp)
391 jmp error_code
393 ENTRY(nmi)
394 pushq $0
395 SAVE_ALL
396 inb $0x61,%al
397 movl %eax,%esi # reason
398 movq %rsp,%rdi # regs
399 call SYMBOL_NAME(do_nmi)
400 jmp restore_all_xen
402 .data
404 ENTRY(exception_table)
405 .quad SYMBOL_NAME(do_divide_error)
406 .quad SYMBOL_NAME(do_debug)
407 .quad 0 # nmi
408 .quad SYMBOL_NAME(do_int3)
409 .quad SYMBOL_NAME(do_overflow)
410 .quad SYMBOL_NAME(do_bounds)
411 .quad SYMBOL_NAME(do_invalid_op)
412 .quad SYMBOL_NAME(math_state_restore)
413 .quad SYMBOL_NAME(do_double_fault)
414 .quad SYMBOL_NAME(do_coprocessor_segment_overrun)
415 .quad SYMBOL_NAME(do_invalid_TSS)
416 .quad SYMBOL_NAME(do_segment_not_present)
417 .quad SYMBOL_NAME(do_stack_segment)
418 .quad SYMBOL_NAME(do_general_protection)
419 .quad SYMBOL_NAME(do_page_fault)
420 .quad SYMBOL_NAME(do_spurious_interrupt_bug)
421 .quad SYMBOL_NAME(do_coprocessor_error)
422 .quad SYMBOL_NAME(do_alignment_check)
423 .quad SYMBOL_NAME(do_machine_check)
424 .quad SYMBOL_NAME(do_simd_coprocessor_error)
426 ENTRY(hypercall_table)
427 .quad SYMBOL_NAME(do_set_trap_table) /* 0 */
428 .quad SYMBOL_NAME(do_mmu_update)
429 .quad SYMBOL_NAME(do_set_gdt)
430 .quad SYMBOL_NAME(do_stack_switch)
431 .quad SYMBOL_NAME(do_set_callbacks)
432 .quad SYMBOL_NAME(do_fpu_taskswitch) /* 5 */
433 .quad SYMBOL_NAME(do_sched_op)
434 .quad SYMBOL_NAME(do_dom0_op)
435 .quad SYMBOL_NAME(do_set_debugreg)
436 .quad SYMBOL_NAME(do_get_debugreg)
437 .quad SYMBOL_NAME(do_update_descriptor) /* 10 */
438 .quad SYMBOL_NAME(do_ni_hypercall)
439 .quad SYMBOL_NAME(do_dom_mem_op)
440 .quad SYMBOL_NAME(do_multicall)
441 .quad SYMBOL_NAME(do_update_va_mapping)
442 .quad SYMBOL_NAME(do_set_timer_op) /* 15 */
443 .quad SYMBOL_NAME(do_event_channel_op)
444 .quad SYMBOL_NAME(do_xen_version)
445 .quad SYMBOL_NAME(do_console_io)
446 .quad SYMBOL_NAME(do_physdev_op)
447 .quad SYMBOL_NAME(do_grant_table_op) /* 20 */
448 .quad SYMBOL_NAME(do_vm_assist)
449 .quad SYMBOL_NAME(do_update_va_mapping_otherdomain)
450 .quad SYMBOL_NAME(do_switch_to_user)
451 .quad SYMBOL_NAME(do_boot_vcpu)
452 .quad SYMBOL_NAME(do_set_segment_base) /* 25 */
453 .quad SYMBOL_NAME(do_mmuext_op)
454 .rept NR_hypercalls-((.-hypercall_table)/4)
455 .quad SYMBOL_NAME(do_ni_hypercall)
456 .endr