ia64/xen-unstable

changeset 15078:60240a72e2b2

svm: Rationalise register synchronisation to be similar to our vmx
handling.
1. Do not copy all VMCB register state in cpu_user_regs on every
vmexit.
2. Save/restore RAX inside asm stub (in particular, before STGI on
vmexit).
3. Simplify store/load_cpu_guest_regs() hook functions to synchronise
precisely the same state as VMX.

By my measurements this reduces the round-trip latency for a null
hypercall by around 150 cycles. This is about 3% of the ~5000-cycle
total on my AMD X2 system. Not a great win, but a nice extra on top of
the code rationalisation.
author Keir Fraser <keir@xensource.com>
date Thu May 10 23:49:04 2007 +0100 (2007-05-10)
parents e19ddfa781c5
children 090ca10cb543
files xen/arch/x86/hvm/svm/svm.c xen/arch/x86/hvm/svm/x86_32/exits.S xen/arch/x86/hvm/svm/x86_64/exits.S
line diff
     1.1 --- a/xen/arch/x86/hvm/svm/svm.c	Thu May 10 22:54:43 2007 +0100
     1.2 +++ b/xen/arch/x86/hvm/svm/svm.c	Thu May 10 23:49:04 2007 +0100
     1.3 @@ -110,15 +110,11 @@ static void svm_store_cpu_guest_regs(
     1.4  
     1.5      if ( regs != NULL )
     1.6      {
     1.7 -        regs->eip    = vmcb->rip;
     1.8 +        regs->ss     = vmcb->ss.sel;
     1.9          regs->esp    = vmcb->rsp;
    1.10          regs->eflags = vmcb->rflags;
    1.11          regs->cs     = vmcb->cs.sel;
    1.12 -        regs->ds     = vmcb->ds.sel;
    1.13 -        regs->es     = vmcb->es.sel;
    1.14 -        regs->ss     = vmcb->ss.sel;
    1.15 -        regs->gs     = vmcb->gs.sel;
    1.16 -        regs->fs     = vmcb->fs.sel;
    1.17 +        regs->eip    = vmcb->rip;
    1.18      }
    1.19  
    1.20      if ( crs != NULL )
    1.21 @@ -752,28 +748,10 @@ static void svm_init_hypercall_page(stru
    1.22      *(u16 *)(hypercall_page + (__HYPERVISOR_iret * 32)) = 0x0b0f; /* ud2 */
    1.23  }
    1.24  
    1.25 -static void save_svm_cpu_user_regs(struct vcpu *v, struct cpu_user_regs *ctxt)
    1.26 +static void svm_load_cpu_guest_regs(struct vcpu *v, struct cpu_user_regs *regs)
    1.27  {
    1.28      struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
    1.29  
    1.30 -    ctxt->eax = vmcb->rax;
    1.31 -    ctxt->ss = vmcb->ss.sel;
    1.32 -    ctxt->esp = vmcb->rsp;
    1.33 -    ctxt->eflags = vmcb->rflags;
    1.34 -    ctxt->cs = vmcb->cs.sel;
    1.35 -    ctxt->eip = vmcb->rip;
    1.36 -    
    1.37 -    ctxt->gs = vmcb->gs.sel;
    1.38 -    ctxt->fs = vmcb->fs.sel;
    1.39 -    ctxt->es = vmcb->es.sel;
    1.40 -    ctxt->ds = vmcb->ds.sel;
    1.41 -}
    1.42 -
    1.43 -static void svm_load_cpu_guest_regs(struct vcpu *v, struct cpu_user_regs *regs)
    1.44 -{
    1.45 -    struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
    1.46 -    
    1.47 -    vmcb->rax      = regs->eax;
    1.48      vmcb->ss.sel   = regs->ss;
    1.49      vmcb->rsp      = regs->esp;   
    1.50      vmcb->rflags   = regs->eflags | 2UL;
    1.51 @@ -2242,7 +2220,6 @@ asmlinkage void svm_vmexit_handler(struc
    1.52      int inst_len, rc;
    1.53  
    1.54      exit_reason = vmcb->exitcode;
    1.55 -    save_svm_cpu_user_regs(v, regs);
    1.56  
    1.57      HVMTRACE_2D(VMEXIT, v, vmcb->rip, exit_reason);
    1.58  
     2.1 --- a/xen/arch/x86/hvm/svm/x86_32/exits.S	Thu May 10 22:54:43 2007 +0100
     2.2 +++ b/xen/arch/x86/hvm/svm/x86_32/exits.S	Thu May 10 23:49:04 2007 +0100
     2.3 @@ -16,6 +16,7 @@
     2.4   * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
     2.5   * Place - Suite 330, Boston, MA 02111-1307 USA.
     2.6   */
     2.7 +
     2.8  #include <xen/config.h>
     2.9  #include <xen/errno.h>
    2.10  #include <xen/softirq.h>
    2.11 @@ -25,61 +26,24 @@
    2.12  #include <public/xen.h>
    2.13  
    2.14  #define GET_CURRENT(reg)         \
    2.15 -        movl $STACK_SIZE-4, reg; \
    2.16 -        orl  %esp, reg;          \
    2.17 +        movl $STACK_SIZE-4,reg;  \
    2.18 +        orl  %esp,reg;           \
    2.19          andl $~3,reg;            \
    2.20          movl (reg),reg;
    2.21  
    2.22 -/*
    2.23 - * At VMExit time the processor saves the guest selectors, esp, eip, 
    2.24 - * and eflags. Therefore we don't save them, but simply decrement 
    2.25 - * the kernel stack pointer to make it consistent with the stack frame 
    2.26 - * at usual interruption time. The eflags of the host is not saved by AMD-V, 
    2.27 - * and we set it to the fixed value.
    2.28 - *
    2.29 - * We also need the room, especially because orig_eax field is used 
    2.30 - * by do_IRQ(). Compared the cpu_user_regs, we skip pushing for the following:
    2.31 - *   (10) u32 gs;                 
    2.32 - *   (9)  u32 fs;
    2.33 - *   (8)  u32 ds;
    2.34 - *   (7)  u32 es;
    2.35 - *               <- get_stack_bottom() (= HOST_ESP)
    2.36 - *   (6)  u32 ss;
    2.37 - *   (5)  u32 esp;
    2.38 - *   (4)  u32 eflags;
    2.39 - *   (3)  u32 cs;
    2.40 - *   (2)  u32 eip;
    2.41 - * (2/1)  u16 entry_vector;
    2.42 - * (1/1)  u16 error_code;
    2.43 - * However, get_stack_bottom() actually returns 20 bytes before the real
    2.44 - * bottom of the stack to allow space for:
    2.45 - * domain pointer, DS, ES, FS, GS. Therefore, we effectively skip 6 registers.
    2.46 - */
    2.47 -
    2.48  #define HVM_MONITOR_EFLAGS 0x202 /* IF on */
    2.49 -#define NR_SKIPPED_REGS    6     /* See the above explanation */
    2.50 -#define HVM_SAVE_ALL_NOSEGREGS \
    2.51 -        pushl $HVM_MONITOR_EFLAGS; \
    2.52 -        popf; \
    2.53 -        subl $(NR_SKIPPED_REGS*4), %esp; \
    2.54 -        pushl %eax; \
    2.55 -        pushl %ebp; \
    2.56 -        pushl %edi; \
    2.57 -        pushl %esi; \
    2.58 -        pushl %edx; \
    2.59 -        pushl %ecx; \
    2.60 +#define NR_SKIPPED_REGS    7     /* Skip SS thru EAX */
    2.61 +#define HVM_SAVE_ALL_NOSEGREGS                  \
    2.62 +        pushl $HVM_MONITOR_EFLAGS;              \
    2.63 +        popf;                                   \
    2.64 +        subl $(NR_SKIPPED_REGS*4),%esp;         \
    2.65 +        pushl %ebp;                             \
    2.66 +        pushl %edi;                             \
    2.67 +        pushl %esi;                             \
    2.68 +        pushl %edx;                             \
    2.69 +        pushl %ecx;                             \
    2.70          pushl %ebx;
    2.71  
    2.72 -#define HVM_RESTORE_ALL_NOSEGREGS   \
    2.73 -        popl %ebx;  \
    2.74 -        popl %ecx;  \
    2.75 -        popl %edx;  \
    2.76 -        popl %esi;  \
    2.77 -        popl %edi;  \
    2.78 -        popl %ebp;  \
    2.79 -        popl %eax;  \
    2.80 -        addl $(NR_SKIPPED_REGS*4), %esp
    2.81 -
    2.82  #define VMRUN  .byte 0x0F,0x01,0xD8
    2.83  #define VMLOAD .byte 0x0F,0x01,0xDA
    2.84  #define VMSAVE .byte 0x0F,0x01,0xDB
    2.85 @@ -88,12 +52,10 @@
    2.86  
    2.87  ENTRY(svm_asm_do_resume)
    2.88          GET_CURRENT(%ebx)
    2.89 -        xorl %ecx,%ecx
    2.90 -        notl %ecx
    2.91          cli                             # tests must not race interrupts
    2.92          movl VCPU_processor(%ebx),%eax
    2.93          shl  $IRQSTAT_shift,%eax
    2.94 -        test %ecx,irq_stat(%eax,1)
    2.95 +        testl $~0,irq_stat(%eax,1)
    2.96          jnz  svm_process_softirqs
    2.97          call svm_intr_assist
    2.98          call svm_load_cr2
    2.99 @@ -101,39 +63,35 @@ ENTRY(svm_asm_do_resume)
   2.100          CLGI                
   2.101          sti
   2.102          GET_CURRENT(%ebx)
   2.103 -        movl VCPU_svm_vmcb(%ebx), %ecx
   2.104 -        movl 24(%esp), %eax
   2.105 -        movl %eax, VMCB_rax(%ecx)
   2.106 -        movl VCPU_processor(%ebx), %eax
   2.107 -        movl root_vmcb_pa(,%eax,8), %eax
   2.108 +        movl VCPU_svm_vmcb(%ebx),%ecx
   2.109 +        movl UREGS_eax(%esp),%eax
   2.110 +        movl %eax,VMCB_rax(%ecx)
   2.111 +        movl VCPU_processor(%ebx),%eax
   2.112 +        movl root_vmcb_pa(,%eax,8),%eax
   2.113          VMSAVE
   2.114  
   2.115 -        movl VCPU_svm_vmcb_pa(%ebx), %eax
   2.116 +        movl VCPU_svm_vmcb_pa(%ebx),%eax
   2.117          popl %ebx
   2.118          popl %ecx
   2.119          popl %edx
   2.120          popl %esi
   2.121          popl %edi
   2.122          popl %ebp
   2.123 -
   2.124 -        /* 
   2.125 -         * Skip %eax, we need to have vmcb address in there.
   2.126 -         * Don't worry, EAX is restored through the VMRUN instruction.
   2.127 -         */
   2.128 -        addl $4, %esp       
   2.129 -        addl $(NR_SKIPPED_REGS*4), %esp
   2.130 +        addl $(NR_SKIPPED_REGS*4),%esp
   2.131          VMLOAD
   2.132          VMRUN
   2.133          VMSAVE
   2.134 -        /* eax is the only register we're allowed to touch here... */
   2.135 +
   2.136 +        HVM_SAVE_ALL_NOSEGREGS
   2.137  
   2.138 -        GET_CURRENT(%eax)
   2.139 -
   2.140 -        movl VCPU_processor(%eax), %eax
   2.141 -        movl root_vmcb_pa(,%eax,8), %eax
   2.142 +        GET_CURRENT(%ebx)
   2.143 +        movl VCPU_svm_vmcb(%ebx),%ecx
   2.144 +        movl VMCB_rax(%ecx),%eax
   2.145 +        movl %eax,UREGS_eax(%esp)
   2.146 +        movl VCPU_processor(%ebx),%eax
   2.147 +        movl root_vmcb_pa(,%eax,8),%eax
   2.148          VMLOAD
   2.149  
   2.150 -        HVM_SAVE_ALL_NOSEGREGS
   2.151          STGI
   2.152  .globl svm_stgi_label;
   2.153  svm_stgi_label:
     3.1 --- a/xen/arch/x86/hvm/svm/x86_64/exits.S	Thu May 10 22:54:43 2007 +0100
     3.2 +++ b/xen/arch/x86/hvm/svm/x86_64/exits.S	Thu May 10 23:49:04 2007 +0100
     3.3 @@ -16,6 +16,7 @@
     3.4   * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
     3.5   * Place - Suite 330, Boston, MA 02111-1307 USA.
     3.6   */
     3.7 +
     3.8  #include <xen/config.h>
     3.9  #include <xen/errno.h>
    3.10  #include <xen/softirq.h>
    3.11 @@ -25,72 +26,32 @@
    3.12  #include <public/xen.h>
    3.13  
    3.14  #define GET_CURRENT(reg)         \
    3.15 -        movq $STACK_SIZE-8, reg; \
    3.16 -        orq  %rsp, reg;          \
    3.17 +        movq $STACK_SIZE-8,reg;  \
    3.18 +        orq  %rsp,reg;           \
    3.19          andq $~7,reg;            \
    3.20          movq (reg),reg;
    3.21  
    3.22 -/*
    3.23 - * At VMExit time the processor saves the guest selectors, rsp, rip, 
    3.24 - * and rflags. Therefore we don't save them, but simply decrement 
    3.25 - * the kernel stack pointer to make it consistent with the stack frame 
    3.26 - * at usual interruption time. The rflags of the host is not saved by AMD-V, 
    3.27 - * and we set it to the fixed value.
    3.28 - *
    3.29 - * We also need the room, especially because orig_eax field is used 
    3.30 - * by do_IRQ(). Compared the cpu_user_regs, we skip pushing for the following:
    3.31 - *   (10) u64 gs;                 
    3.32 - *   (9)  u64 fs;
    3.33 - *   (8)  u64 ds;
    3.34 - *   (7)  u64 es;
    3.35 - *               <- get_stack_bottom() (= HOST_ESP)
    3.36 - *   (6)  u64 ss;
    3.37 - *   (5)  u64 rsp;
    3.38 - *   (4)  u64 rflags;
    3.39 - *   (3)  u64 cs;
    3.40 - *   (2)  u64 rip;
    3.41 - * (2/1)  u32 entry_vector;
    3.42 - * (1/1)  u32 error_code;
    3.43 - */
    3.44  #define HVM_MONITOR_RFLAGS 0x202 /* IF on */
    3.45 -#define NR_SKIPPED_REGS    6     /* See the above explanation */
    3.46 -#define HVM_SAVE_ALL_NOSEGREGS \
    3.47 -        pushq $HVM_MONITOR_RFLAGS; \
    3.48 -        popfq; \
    3.49 -        subq $(NR_SKIPPED_REGS*8), %rsp; \
    3.50 -        pushq %rdi; \
    3.51 -        pushq %rsi; \
    3.52 -        pushq %rdx; \
    3.53 -        pushq %rcx; \
    3.54 -        pushq %rax; \
    3.55 -        pushq %r8;  \
    3.56 -        pushq %r9;  \
    3.57 -        pushq %r10; \
    3.58 -        pushq %r11; \
    3.59 -        pushq %rbx; \
    3.60 -        pushq %rbp; \
    3.61 -        pushq %r12; \
    3.62 -        pushq %r13; \
    3.63 -        pushq %r14; \
    3.64 -        pushq %r15; \
    3.65 -
    3.66 -#define HVM_RESTORE_ALL_NOSEGREGS \
    3.67 -        popq %r15; \
    3.68 -        popq %r14; \
    3.69 -        popq %r13; \
    3.70 -        popq %r12; \
    3.71 -        popq %rbp; \
    3.72 -        popq %rbx; \
    3.73 -        popq %r11; \
    3.74 -        popq %r10; \
    3.75 -        popq %r9;  \
    3.76 -        popq %r8;  \
    3.77 -        popq %rax; \
    3.78 -        popq %rcx; \
    3.79 -        popq %rdx; \
    3.80 -        popq %rsi; \
    3.81 -        popq %rdi; \
    3.82 -        addq $(NR_SKIPPED_REGS*8), %rsp; \
    3.83 +#define NR_SKIPPED_REGS    6     /* Skip SS thru error_code */
    3.84 +#define HVM_SAVE_ALL_NOSEGREGS                  \
    3.85 +        pushq $HVM_MONITOR_RFLAGS;              \
    3.86 +        popfq;                                  \
    3.87 +        subq $(NR_SKIPPED_REGS*8),%rsp;         \
    3.88 +        pushq %rdi;                             \
    3.89 +        pushq %rsi;                             \
    3.90 +        pushq %rdx;                             \
    3.91 +        pushq %rcx;                             \
    3.92 +        pushq %rax;                             \
    3.93 +        pushq %r8;                              \
    3.94 +        pushq %r9;                              \
    3.95 +        pushq %r10;                             \
    3.96 +        pushq %r11;                             \
    3.97 +        pushq %rbx;                             \
    3.98 +        pushq %rbp;                             \
    3.99 +        pushq %r12;                             \
   3.100 +        pushq %r13;                             \
   3.101 +        pushq %r14;                             \
   3.102 +        pushq %r15;
   3.103  
   3.104  #define VMRUN  .byte 0x0F,0x01,0xD8
   3.105  #define VMLOAD .byte 0x0F,0x01,0xDA
   3.106 @@ -102,9 +63,9 @@ ENTRY(svm_asm_do_resume)
   3.107          GET_CURRENT(%rbx)
   3.108          cli                             # tests must not race interrupts
   3.109          movl VCPU_processor(%rbx),%eax
   3.110 -        shl  $IRQSTAT_shift, %rax
   3.111 -        leaq irq_stat(%rip), %rdx
   3.112 -        testl $~0, (%rdx, %rax, 1)
   3.113 +        shl  $IRQSTAT_shift,%rax
   3.114 +        leaq irq_stat(%rip),%rdx
   3.115 +        testl $~0,(%rdx,%rax,1)
   3.116          jnz  svm_process_softirqs
   3.117          call svm_intr_assist
   3.118          call svm_load_cr2
   3.119 @@ -112,15 +73,15 @@ ENTRY(svm_asm_do_resume)
   3.120          CLGI                
   3.121          sti
   3.122          GET_CURRENT(%rbx)
   3.123 -        movq VCPU_svm_vmcb(%rbx), %rcx
   3.124 -        movq UREGS_rax(%rsp), %rax
   3.125 -        movq %rax, VMCB_rax(%rcx)
   3.126 -        leaq root_vmcb_pa(%rip), %rax
   3.127 -        movl VCPU_processor(%rbx), %ecx
   3.128 -        movq (%rax,%rcx,8), %rax
   3.129 +        movq VCPU_svm_vmcb(%rbx),%rcx
   3.130 +        movq UREGS_rax(%rsp),%rax
   3.131 +        movq %rax,VMCB_rax(%rcx)
   3.132 +        leaq root_vmcb_pa(%rip),%rax
   3.133 +        movl VCPU_processor(%rbx),%ecx
   3.134 +        movq (%rax,%rcx,8),%rax
   3.135          VMSAVE
   3.136  
   3.137 -        movq VCPU_svm_vmcb_pa(%rbx), %rax
   3.138 +        movq VCPU_svm_vmcb_pa(%rbx),%rax
   3.139          popq %r15
   3.140          popq %r14
   3.141          popq %r13
   3.142 @@ -131,26 +92,26 @@ ENTRY(svm_asm_do_resume)
   3.143          popq %r10
   3.144          popq %r9
   3.145          popq %r8
   3.146 -        /*
   3.147 -         * Skip %rax, we need to have vmcb address in there.
   3.148 -         * Don't worry, RAX is restored through the VMRUN instruction.
   3.149 -         */
   3.150 -        addq $8, %rsp
   3.151 +        addq $8,%rsp /* Skip %rax: restored by VMRUN. */
   3.152          popq %rcx
   3.153          popq %rdx
   3.154          popq %rsi
   3.155          popq %rdi
   3.156 -        addq $(NR_SKIPPED_REGS*8), %rsp
   3.157 +        addq $(NR_SKIPPED_REGS*8),%rsp
   3.158  
   3.159          VMLOAD
   3.160          VMRUN
   3.161          VMSAVE
   3.162 +
   3.163          HVM_SAVE_ALL_NOSEGREGS
   3.164  
   3.165          GET_CURRENT(%rbx)
   3.166 -        leaq root_vmcb_pa(%rip), %rax
   3.167 -        movl VCPU_processor(%rbx), %ecx
   3.168 -        movq (%rax,%rcx,8), %rax
   3.169 +        movq VCPU_svm_vmcb(%rbx),%rcx
   3.170 +        movq VMCB_rax(%rcx),%rax
   3.171 +        movq %rax,UREGS_rax(%rsp)
   3.172 +        leaq root_vmcb_pa(%rip),%rax
   3.173 +        movl VCPU_processor(%rbx),%ecx
   3.174 +        movq (%rax,%rcx,8),%rax
   3.175          VMLOAD
   3.176  
   3.177          STGI