direct-io.hg

view xen/arch/x86/x86_64/compat/entry.S @ 14445:522a1cd17b6d

[XEN] Implement faster int 0x80 handling for compat mode guests.

Using the GPF handler to spot the software interrupt and pass it back
to the guest increases the base syscall time by a factor of 2.7
compared with 32on32 using direct trap to ring 1. (0.3270->0.8680
microseconds, measured with lmbench lat_syscall).

Since the 64 bit IDT can only contain 64 bit segment selectors we
cannot trap directly to compat mode ring 1. However implementing a
dedicated 64 bit ring 0 trap handler allows us to avoid much of the
GPF handler overhead and reduces the overhead to 1.7 times
(0.3270->0.5497 microseconds).

Signed-off-by: Ian Campbell <ian.campbell@xensource.com>
author Ian Campbell <ian.campbell@xensource.com>
date Tue Mar 20 14:33:15 2007 +0000 (2007-03-20)
parents 9e27245a692e
children 96f167771979
line source
1 /*
2 * Compatibility hypercall routines.
3 */
5 #include <xen/config.h>
6 #include <xen/errno.h>
7 #include <xen/softirq.h>
8 #include <asm/asm_defns.h>
9 #include <asm/apicdef.h>
10 #include <asm/page.h>
11 #include <asm/desc.h>
12 #include <public/xen.h>
14 #define GET_GUEST_REGS(reg) \
15 movq $~(STACK_SIZE-1),reg; \
16 andq %rsp,reg; \
17 orq $(STACK_SIZE-CPUINFO_sizeof),reg;
19 #define GET_CURRENT(reg) \
20 movq $STACK_SIZE-8, reg; \
21 orq %rsp, reg; \
22 andq $~7,reg; \
23 movq (reg),reg;
25 ALIGN
26 ENTRY(compat_hypercall)
27 pushq $0
28 movl $TRAP_syscall,4(%rsp)
29 SAVE_ALL
30 GET_CURRENT(%rbx)
32 cmpl $NR_hypercalls,%eax
33 jae compat_bad_hypercall
34 #ifndef NDEBUG
35 /* Deliberately corrupt parameter regs not used by this hypercall. */
36 pushq UREGS_rbx(%rsp); pushq %rcx; pushq %rdx; pushq %rsi; pushq %rdi
37 pushq UREGS_rbp+5*8(%rsp)
38 leaq compat_hypercall_args_table(%rip),%r10
39 movq $6,%rcx
40 subb (%r10,%rax,1),%cl
41 movq %rsp,%rdi
42 movl $0xDEADBEEF,%eax
43 rep stosq
44 popq %r8 ; popq %r9 ; xchgl %r8d,%r9d /* Args 5&6: zero extend */
45 popq %rdx; popq %rcx; xchgl %edx,%ecx /* Args 3&4: zero extend */
46 popq %rdi; popq %rsi; xchgl %edi,%esi /* Args 1&2: zero extend */
47 movl UREGS_rax(%rsp),%eax
48 pushq %rax
49 pushq UREGS_rip+8(%rsp)
50 #else
51 /* Relocate argument registers and zero-extend to 64 bits. */
52 movl %eax,%eax /* Hypercall # */
53 xchgl %ecx,%esi /* Arg 2, Arg 4 */
54 movl %edx,%edx /* Arg 3 */
55 movl %edi,%r8d /* Arg 5 */
56 movl %ebp,%r9d /* Arg 6 */
57 movl UREGS_rbx(%rsp),%edi /* Arg 1 */
58 #endif
59 leaq compat_hypercall_table(%rip),%r10
60 PERFC_INCR(PERFC_hypercalls, %rax)
61 callq *(%r10,%rax,8)
62 #ifndef NDEBUG
63 /* Deliberately corrupt parameter regs used by this hypercall. */
64 popq %r10 # Shadow RIP
65 cmpq %r10,UREGS_rip+8(%rsp)
66 popq %rcx # Shadow hypercall index
67 jne compat_skip_clobber /* If RIP has changed then don't clobber. */
68 leaq compat_hypercall_args_table(%rip),%r10
69 movb (%r10,%rcx,1),%cl
70 movl $0xDEADBEEF,%r10d
71 testb %cl,%cl; jz compat_skip_clobber; movl %r10d,UREGS_rbx(%rsp)
72 cmpb $2, %cl; jb compat_skip_clobber; movl %r10d,UREGS_rcx(%rsp)
73 cmpb $3, %cl; jb compat_skip_clobber; movl %r10d,UREGS_rdx(%rsp)
74 cmpb $4, %cl; jb compat_skip_clobber; movl %r10d,UREGS_rsi(%rsp)
75 cmpb $5, %cl; jb compat_skip_clobber; movl %r10d,UREGS_rdi(%rsp)
76 cmpb $6, %cl; jb compat_skip_clobber; movl %r10d,UREGS_rbp(%rsp)
77 compat_skip_clobber:
78 #endif
79 movl %eax,UREGS_rax(%rsp) # save the return value
81 /* %rbx: struct vcpu */
82 ENTRY(compat_test_all_events)
83 cli # tests must not race interrupts
84 /*compat_test_softirqs:*/
85 movl VCPU_processor(%rbx),%eax
86 shlq $IRQSTAT_shift,%rax
87 leaq irq_stat(%rip),%rcx
88 testl $~0,(%rcx,%rax,1)
89 jnz compat_process_softirqs
90 btrq $_VCPUF_nmi_pending,VCPU_flags(%rbx)
91 jc compat_process_nmi
92 compat_test_guest_events:
93 movq VCPU_vcpu_info(%rbx),%rax
94 testb $0xFF,COMPAT_VCPUINFO_upcall_mask(%rax)
95 jnz compat_restore_all_guest
96 testb $0xFF,COMPAT_VCPUINFO_upcall_pending(%rax)
97 jz compat_restore_all_guest
98 /*compat_process_guest_events:*/
99 sti
100 leaq VCPU_trap_bounce(%rbx),%rdx
101 movl VCPU_event_addr(%rbx),%eax
102 movl %eax,TRAPBOUNCE_eip(%rdx)
103 movl VCPU_event_sel(%rbx),%eax
104 movl %eax,TRAPBOUNCE_cs(%rdx)
105 movw $TBF_INTERRUPT,TRAPBOUNCE_flags(%rdx)
106 call compat_create_bounce_frame
107 jmp compat_test_all_events
109 ALIGN
110 /* %rbx: struct vcpu */
111 compat_process_softirqs:
112 sti
113 call do_softirq
114 jmp compat_test_all_events
116 ALIGN
117 /* %rbx: struct vcpu */
118 compat_process_nmi:
119 movl VCPU_nmi_addr(%rbx),%eax
120 testl %eax,%eax
121 jz compat_test_all_events
122 btsq $_VCPUF_nmi_masked,VCPU_flags(%rbx)
123 jc 1f
124 sti
125 leaq VCPU_trap_bounce(%rbx),%rdx
126 movl %eax,TRAPBOUNCE_eip(%rdx)
127 movl $FLAT_COMPAT_KERNEL_CS,TRAPBOUNCE_cs(%rdx)
128 movw $TBF_INTERRUPT,TRAPBOUNCE_flags(%rdx)
129 call compat_create_bounce_frame
130 jmp compat_test_all_events
131 1:
132 btsq $_VCPUF_nmi_pending,VCPU_flags(%rbx)
133 jmp compat_test_guest_events
135 compat_bad_hypercall:
136 movl $-ENOSYS,UREGS_rax(%rsp)
137 jmp compat_test_all_events
139 /* %rbx: struct vcpu, interrupts disabled */
140 compat_restore_all_guest:
141 RESTORE_ALL
142 addq $8,%rsp
143 .Lft0: iretq
145 .section .fixup,"ax"
146 .Lfx0: popq -15*8-8(%rsp) # error_code/entry_vector
147 SAVE_ALL # 15*8 bytes pushed
148 movq -8(%rsp),%rsi # error_code/entry_vector
149 sti # after stack abuse (-1024(%rsp))
150 pushq $__HYPERVISOR_DS # SS
151 leaq 8(%rsp),%rax
152 pushq %rax # RSP
153 pushfq # RFLAGS
154 pushq $__HYPERVISOR_CS # CS
155 leaq .Ldf0(%rip),%rax
156 pushq %rax # RIP
157 pushq %rsi # error_code/entry_vector
158 jmp handle_exception
159 .Ldf0: GET_CURRENT(%rbx)
160 jmp compat_test_all_events
161 compat_failsafe_callback:
162 GET_CURRENT(%rbx)
163 leaq VCPU_trap_bounce(%rbx),%rdx
164 movl VCPU_failsafe_addr(%rbx),%eax
165 movl %eax,TRAPBOUNCE_eip(%rdx)
166 movl VCPU_failsafe_sel(%rbx),%eax
167 movl %eax,TRAPBOUNCE_cs(%rdx)
168 movw $TBF_FAILSAFE,TRAPBOUNCE_flags(%rdx)
169 btq $_VGCF_failsafe_disables_events,VCPU_guest_context_flags(%rbx)
170 jnc 1f
171 orw $TBF_INTERRUPT,TRAPBOUNCE_flags(%rdx)
172 1:
173 call compat_create_bounce_frame
174 jmp compat_test_all_events
175 .previous
176 .section __pre_ex_table,"a"
177 .quad .Lft0,.Lfx0
178 .previous
179 .section __ex_table,"a"
180 .quad .Ldf0,compat_failsafe_callback
181 .previous
183 /* %rdx: trap_bounce, %rbx: struct vcpu */
184 ENTRY(compat_post_handle_exception)
185 testb $TBF_EXCEPTION,TRAPBOUNCE_flags(%rdx)
186 jz compat_test_all_events
187 call compat_create_bounce_frame
188 jmp compat_test_all_events
190 ENTRY(compat_int80_direct_trap)
191 call compat_create_bounce_frame
192 jmp compat_restore_all_guest
194 /* CREATE A BASIC EXCEPTION FRAME ON GUEST OS (RING-1) STACK: */
195 /* {[ERRCODE,] EIP, CS, EFLAGS, [ESP, SS]} */
196 /* %rdx: trap_bounce, %rbx: struct vcpu */
197 /* On return only %rbx is guaranteed non-clobbered. */
198 compat_create_bounce_frame:
199 mov %fs,%edi
200 testb $2,UREGS_cs+8(%rsp)
201 jz 1f
202 /* Push new frame at registered guest-OS stack base. */
203 movl VCPU_kernel_sp(%rbx),%esi
204 .Lft1: mov VCPU_kernel_ss(%rbx),%fs
205 subl $2*4,%esi
206 movl UREGS_rsp+8(%rsp),%eax
207 .Lft2: movl %eax,%fs:(%rsi)
208 movl UREGS_ss+8(%rsp),%eax
209 .Lft3: movl %eax,%fs:4(%rsi)
210 jmp 2f
211 1: /* In kernel context already: push new frame at existing %rsp. */
212 movl UREGS_rsp+8(%rsp),%esi
213 .Lft4: mov UREGS_ss+8(%rsp),%fs
214 2:
215 movb TRAPBOUNCE_flags(%rdx),%cl
216 subl $3*4,%esi
217 movq VCPU_vcpu_info(%rbx),%rax
218 pushq COMPAT_VCPUINFO_upcall_mask(%rax)
219 testb $TBF_INTERRUPT,%cl
220 setnz %ch # TBF_INTERRUPT -> set upcall mask
221 orb %ch,COMPAT_VCPUINFO_upcall_mask(%rax)
222 popq %rax
223 shll $16,%eax # Bits 16-23: saved_upcall_mask
224 movw UREGS_cs+8(%rsp),%ax # Bits 0-15: CS
225 .Lft5: movl %eax,%fs:4(%rsi) # CS / saved_upcall_mask
226 shrl $16,%eax
227 testb %al,%al # Bits 0-7: saved_upcall_mask
228 setz %ch # %ch == !saved_upcall_mask
229 movl UREGS_eflags+8(%rsp),%eax
230 andl $~X86_EFLAGS_IF,%eax
231 shlb $1,%ch # Bit 9 (EFLAGS.IF)
232 orb %ch,%ah # Fold EFLAGS.IF into %eax
233 .Lft6: movl %eax,%fs:2*4(%rsi) # EFLAGS
234 movl UREGS_rip+8(%rsp),%eax
235 .Lft7: movl %eax,%fs:(%rsi) # EIP
236 testb $TBF_EXCEPTION_ERRCODE,%cl
237 jz 1f
238 subl $4,%esi
239 movl TRAPBOUNCE_error_code(%rdx),%eax
240 .Lft8: movl %eax,%fs:(%rsi) # ERROR CODE
241 1:
242 testb $TBF_FAILSAFE,%cl
243 jz 2f
244 subl $4*4,%esi
245 movl %gs,%eax
246 .Lft9: movl %eax,%fs:3*4(%rsi) # GS
247 .Lft10: movl %edi,%fs:2*4(%rsi) # FS
248 movl %es,%eax
249 .Lft11: movl %eax,%fs:1*4(%rsi) # ES
250 movl %ds,%eax
251 .Lft12: movl %eax,%fs:0*4(%rsi) # DS
252 2:
253 /* Rewrite our stack frame and return to guest-OS mode. */
254 /* IA32 Ref. Vol. 3: TF, VM, RF and NT flags are cleared on trap. */
255 movl $TRAP_syscall,UREGS_entry_vector+8(%rsp)
256 andl $~(X86_EFLAGS_VM|X86_EFLAGS_RF|\
257 X86_EFLAGS_NT|X86_EFLAGS_TF),UREGS_eflags+8(%rsp)
258 mov %fs,UREGS_ss+8(%rsp)
259 movl %esi,UREGS_rsp+8(%rsp)
260 .Lft13: mov %edi,%fs
261 movzwl TRAPBOUNCE_cs(%rdx),%eax
262 /* Null selectors (0-3) are not allowed. */
263 testl $~3,%eax
264 jz domain_crash_synchronous
265 movl %eax,UREGS_cs+8(%rsp)
266 movl TRAPBOUNCE_eip(%rdx),%eax
267 movl %eax,UREGS_rip+8(%rsp)
268 movb $0,TRAPBOUNCE_flags(%rdx)
269 ret
270 .section .fixup,"ax"
271 .Lfx13:
272 xorl %edi,%edi
273 jmp .Lft13
274 .previous
275 .section __ex_table,"a"
276 .quad .Lft1,domain_crash_synchronous , .Lft2,compat_crash_page_fault
277 .quad .Lft3,compat_crash_page_fault_4 , .Lft4,domain_crash_synchronous
278 .quad .Lft5,compat_crash_page_fault_4 , .Lft6,compat_crash_page_fault_8
279 .quad .Lft7,compat_crash_page_fault , .Lft8,compat_crash_page_fault
280 .quad .Lft9,compat_crash_page_fault_12, .Lft10,compat_crash_page_fault_8
281 .quad .Lft11,compat_crash_page_fault_4 , .Lft12,compat_crash_page_fault
282 .quad .Lft13,.Lfx13
283 .previous
285 compat_crash_page_fault_12:
286 addl $4,%esi
287 compat_crash_page_fault_8:
288 addl $4,%esi
289 compat_crash_page_fault_4:
290 addl $4,%esi
291 compat_crash_page_fault:
292 .Lft14: mov %edi,%fs
293 movl %esi,%edi
294 call show_page_walk
295 jmp domain_crash_synchronous
296 .section .fixup,"ax"
297 .Lfx14:
298 xorl %edi,%edi
299 jmp .Lft14
300 .previous
301 .section __ex_table,"a"
302 .quad .Lft14,.Lfx14
303 .previous
305 .section .rodata, "a", @progbits
307 ENTRY(compat_hypercall_table)
308 .quad compat_set_trap_table /* 0 */
309 .quad do_mmu_update
310 .quad compat_set_gdt
311 .quad do_stack_switch
312 .quad compat_set_callbacks
313 .quad do_fpu_taskswitch /* 5 */
314 .quad do_sched_op_compat
315 .quad compat_platform_op
316 .quad do_set_debugreg
317 .quad do_get_debugreg
318 .quad compat_update_descriptor /* 10 */
319 .quad compat_ni_hypercall
320 .quad compat_memory_op
321 .quad compat_multicall
322 .quad compat_update_va_mapping
323 .quad compat_set_timer_op /* 15 */
324 .quad do_event_channel_op_compat
325 .quad compat_xen_version
326 .quad do_console_io
327 .quad compat_physdev_op_compat
328 .quad compat_grant_table_op /* 20 */
329 .quad compat_vm_assist
330 .quad compat_update_va_mapping_otherdomain
331 .quad compat_iret
332 .quad compat_vcpu_op
333 .quad compat_ni_hypercall /* 25 */
334 .quad compat_mmuext_op
335 .quad compat_acm_op
336 .quad compat_nmi_op
337 .quad compat_sched_op
338 .quad compat_callback_op /* 30 */
339 .quad compat_xenoprof_op
340 .quad do_event_channel_op
341 .quad compat_physdev_op
342 .quad do_hvm_op
343 .quad do_sysctl /* 35 */
344 .quad do_domctl
345 .quad compat_kexec_op
346 .rept NR_hypercalls-((.-compat_hypercall_table)/8)
347 .quad compat_ni_hypercall
348 .endr
350 ENTRY(compat_hypercall_args_table)
351 .byte 1 /* compat_set_trap_table */ /* 0 */
352 .byte 4 /* compat_mmu_update */
353 .byte 2 /* compat_set_gdt */
354 .byte 2 /* compat_stack_switch */
355 .byte 4 /* compat_set_callbacks */
356 .byte 1 /* compat_fpu_taskswitch */ /* 5 */
357 .byte 2 /* compat_sched_op_compat */
358 .byte 1 /* compat_platform_op */
359 .byte 2 /* compat_set_debugreg */
360 .byte 1 /* compat_get_debugreg */
361 .byte 4 /* compat_update_descriptor */ /* 10 */
362 .byte 0 /* compat_ni_hypercall */
363 .byte 2 /* compat_memory_op */
364 .byte 2 /* compat_multicall */
365 .byte 4 /* compat_update_va_mapping */
366 .byte 2 /* compat_set_timer_op */ /* 15 */
367 .byte 1 /* compat_event_channel_op_compat */
368 .byte 2 /* compat_xen_version */
369 .byte 3 /* compat_console_io */
370 .byte 1 /* compat_physdev_op_compat */
371 .byte 3 /* compat_grant_table_op */ /* 20 */
372 .byte 2 /* compat_vm_assist */
373 .byte 5 /* compat_update_va_mapping_otherdomain */
374 .byte 0 /* compat_iret */
375 .byte 3 /* compat_vcpu_op */
376 .byte 0 /* compat_ni_hypercall */ /* 25 */
377 .byte 4 /* compat_mmuext_op */
378 .byte 1 /* compat_acm_op */
379 .byte 2 /* compat_nmi_op */
380 .byte 2 /* compat_sched_op */
381 .byte 2 /* compat_callback_op */ /* 30 */
382 .byte 2 /* compat_xenoprof_op */
383 .byte 2 /* compat_event_channel_op */
384 .byte 2 /* compat_physdev_op */
385 .byte 2 /* do_hvm_op */
386 .byte 1 /* do_sysctl */ /* 35 */
387 .byte 1 /* do_domctl */
388 .byte 2 /* compat_kexec_op */
389 .rept NR_hypercalls-(.-compat_hypercall_args_table)
390 .byte 0 /* compat_ni_hypercall */
391 .endr