ia64/xen-unstable

changeset 10703:8d501f39286c

[IA64] vDSO paravirtualization: paravirtualize vDSO

paravirtualize vdso areabased on Kevin's pointout and Dan's Idea.
introduce hyperprivop HYPERPRIVOP_RSM_BE and HYPERPRIVOP_GET_PSR.
and paravirtualize vdso area using them.

Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
author awilliam@xenbuild.aw
date Mon Jul 24 13:43:35 2006 -0600 (2006-07-24)
parents 614deef19299
children c8bc76d877e0
files linux-2.6-xen-sparse/arch/ia64/Kconfig linux-2.6-xen-sparse/arch/ia64/kernel/gate.S linux-2.6-xen-sparse/arch/ia64/kernel/gate.lds.S linux-2.6-xen-sparse/arch/ia64/kernel/patch.c linux-2.6-xen-sparse/arch/ia64/xen/hypercall.S linux-2.6-xen-sparse/include/asm-ia64/xen/privop.h xen/arch/ia64/xen/privop.c xen/include/public/arch-ia64.h
line diff
     1.1 --- a/linux-2.6-xen-sparse/arch/ia64/Kconfig	Mon Jul 24 13:04:40 2006 -0600
     1.2 +++ b/linux-2.6-xen-sparse/arch/ia64/Kconfig	Mon Jul 24 13:43:35 2006 -0600
     1.3 @@ -71,6 +71,13 @@ config XEN_IA64_DOM0_NON_VP
     1.4  	help
     1.5  	  dom0 P=M model
     1.6  
     1.7 +config XEN_IA64_VDSO_PARAVIRT
     1.8 +	bool
     1.9 +	depends on XEN && !ITANIUM
    1.10 +	default y
    1.11 +	help
    1.12 +	  vDSO paravirtualization
    1.13 +
    1.14  config SCHED_NO_NO_OMIT_FRAME_POINTER
    1.15  	bool
    1.16  	default y
     2.1 --- a/linux-2.6-xen-sparse/arch/ia64/kernel/gate.S	Mon Jul 24 13:04:40 2006 -0600
     2.2 +++ b/linux-2.6-xen-sparse/arch/ia64/kernel/gate.S	Mon Jul 24 13:43:35 2006 -0600
     2.3 @@ -14,6 +14,9 @@
     2.4  #include <asm/sigcontext.h>
     2.5  #include <asm/system.h>
     2.6  #include <asm/unistd.h>
     2.7 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
     2.8 +# include <asm/privop.h>
     2.9 +#endif
    2.10  
    2.11  /*
    2.12   * We can't easily refer to symbols inside the kernel.  To avoid full runtime relocation,
    2.13 @@ -33,6 +36,52 @@
    2.14  [1:](pr)brl.cond.sptk 0;				\
    2.15  	.xdata4 ".data.patch.brl_fsys_bubble_down", 1b-.
    2.16  
    2.17 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
    2.18 +	// The page in which hyperprivop lives must be pinned by ITR.
    2.19 +	// However vDSO area isn't pinned. So issuing hyperprivop
    2.20 +	// from vDSO page causes trouble that Kevin pointed out.
    2.21 +	// After clearing vpsr.ic, the vcpu is pre-empted and the itlb
    2.22 +	// is flushed. Then vcpu get cpu again, tlb miss fault occures.
    2.23 +	// However it results in nested dtlb fault because vpsr.ic is off.
    2.24 +	// To avoid such a situation, we jump into the kernel text area
    2.25 +	// which is pinned, and then issue hyperprivop and return back
    2.26 +	// to vDSO page.
    2.27 +	// This is Dan Magenheimer's idea.
    2.28 +
    2.29 +	// Currently is_running_on_xen() is defined as running_on_xen.
    2.30 +	// If is_running_on_xen() is a real function, we must update
    2.31 +	// according to it.
    2.32 +	.section ".data.patch.running_on_xen", "a"
    2.33 +	.previous
    2.34 +#define LOAD_RUNNING_ON_XEN(reg)			\
    2.35 +[1:]	movl reg=0;					\
    2.36 +	.xdata4 ".data.patch.running_on_xen", 1b-.
    2.37 +
    2.38 +	.section ".data.patch.brl_xen_rsm_be_i", "a"
    2.39 +	.previous
    2.40 +#define BRL_COND_XEN_RSM_BE_I(pr)			\
    2.41 +[1:](pr)brl.cond.sptk 0;				\
    2.42 +	.xdata4 ".data.patch.brl_xen_rsm_be_i", 1b-.
    2.43 +
    2.44 +	.section ".data.patch.brl_xen_get_psr", "a"
    2.45 +	.previous
    2.46 +#define BRL_COND_XEN_GET_PSR(pr)			\
    2.47 +[1:](pr)brl.cond.sptk 0;				\
    2.48 +	.xdata4 ".data.patch.brl_xen_get_psr", 1b-.
    2.49 +
    2.50 +	.section ".data.patch.brl_xen_ssm_i_0", "a"
    2.51 +	.previous
    2.52 +#define BRL_COND_XEN_SSM_I_0(pr)			\
    2.53 +[1:](pr)brl.cond.sptk 0;				\
    2.54 +	.xdata4 ".data.patch.brl_xen_ssm_i_0", 1b-.
    2.55 +
    2.56 +	.section ".data.patch.brl_xen_ssm_i_1", "a"
    2.57 +	.previous
    2.58 +#define BRL_COND_XEN_SSM_I_1(pr)			\
    2.59 +[1:](pr)brl.cond.sptk 0;				\
    2.60 +	.xdata4 ".data.patch.brl_xen_ssm_i_1", 1b-.
    2.61 +#endif
    2.62 +
    2.63  GLOBAL_ENTRY(__kernel_syscall_via_break)
    2.64  	.prologue
    2.65  	.altrp b6
    2.66 @@ -77,7 +126,39 @@ GLOBAL_ENTRY(__kernel_syscall_via_epc)
    2.67  	epc					// B	causes split-issue
    2.68  }
    2.69  	;;
    2.70 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
    2.71 +	// r20 = 1
    2.72 +	// r22 = &vcpu->evtchn_mask
    2.73 +	// r23 = &vpsr.ic
    2.74 +	// r24 = vcpu->pending_interruption
    2.75 +	// r25 = tmp
    2.76 +	// r28 = &running_on_xen
    2.77 +	// r30 = running_on_xen
    2.78 +	// r31 = tmp
    2.79 +	// p11 = tmp
    2.80 +	// p12 = running_on_xen
    2.81 +	// p13 = !running_on_xen
    2.82 +	// p14 = tmp
    2.83 +	// p15 = tmp
    2.84 +#define isXen	p12
    2.85 +#define isRaw	p13
    2.86 +	LOAD_RUNNING_ON_XEN(r28)
    2.87 +	movl r22=XSI_PSR_I_ADDR
    2.88 +	movl r23=XSI_PSR_IC
    2.89 +	movl r24=XSI_PSR_I_ADDR+(XSI_PEND_OFS-XSI_PSR_I_ADDR_OFS)
    2.90 +	mov r20=1
    2.91 +	;;
    2.92 +	ld4 r30=[r28]
    2.93 +	;;
    2.94 +	cmp.ne isXen,isRaw=r0,r30
    2.95 +	;;
    2.96 +(isRaw)	rsm psr.be | psr.i
    2.97 +	BRL_COND_XEN_RSM_BE_I(isXen)
    2.98 +	.global .vdso_rsm_be_i_ret
    2.99 +.vdso_rsm_be_i_ret:
   2.100 +#else
   2.101  	rsm psr.be | psr.i			// M2 (5 cyc to srlz.d)
   2.102 +#endif
   2.103  	LOAD_FSYSCALL_TABLE(r14)		// X
   2.104  	;;
   2.105  	mov r16=IA64_KR(CURRENT)		// M2 (12 cyc)
   2.106 @@ -85,7 +166,14 @@ GLOBAL_ENTRY(__kernel_syscall_via_epc)
   2.107  	mov r19=NR_syscalls-1			// A
   2.108  	;;
   2.109  	lfetch [r18]				// M0|1
   2.110 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
   2.111 +(isRaw)	mov r29=psr
   2.112 +	BRL_COND_XEN_GET_PSR(isXen)
   2.113 +	.global .vdso_get_psr_ret
   2.114 +.vdso_get_psr_ret:
   2.115 +#else
   2.116  	mov r29=psr				// M2 (12 cyc)
   2.117 +#endif
   2.118  	// If r17 is a NaT, p6 will be zero
   2.119  	cmp.geu p6,p7=r19,r17			// A    (sysnr > 0 && sysnr < 1024+NR_syscalls)?
   2.120  	;;
   2.121 @@ -99,9 +187,21 @@ GLOBAL_ENTRY(__kernel_syscall_via_epc)
   2.122  	;;
   2.123  	nop.m 0
   2.124  (p6)	tbit.z.unc p8,p0=r18,0			// I0 (dual-issues with "mov b7=r18"!)
   2.125 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
   2.126 +	;;
   2.127 +	// p14 = running_on_xen && p8
   2.128 +	// p15 = !running_on_xen && p8
   2.129 +(p8)	cmp.ne.unc p14,p15=r0,r30
   2.130 +	;;
   2.131 +(p15)	ssm psr.i
   2.132 +	BRL_COND_XEN_SSM_I_0(p14)
   2.133 +	.global .vdso_ssm_i_0_ret
   2.134 +.vdso_ssm_i_0_ret:
   2.135 +#else
   2.136  	nop.i 0
   2.137  	;;
   2.138  (p8)	ssm psr.i
   2.139 +#endif
   2.140  (p6)	mov b7=r18				// I0
   2.141  (p8)	br.dptk.many b7				// B
   2.142  
   2.143 @@ -122,9 +222,21 @@ GLOBAL_ENTRY(__kernel_syscall_via_epc)
   2.144  #else
   2.145  	BRL_COND_FSYS_BUBBLE_DOWN(p6)
   2.146  #endif
   2.147 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
   2.148 +(isRaw)	ssm psr.i
   2.149 +	BRL_COND_XEN_SSM_I_1(isXen)
   2.150 +	.global .vdso_ssm_i_1_ret
   2.151 +.vdso_ssm_i_1_ret:
   2.152 +#else
   2.153  	ssm psr.i
   2.154 +#endif
   2.155  	mov r10=-1
   2.156  (p10)	mov r8=EINVAL
   2.157 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
   2.158 +	dv_serialize_data // shut up gas warning.
   2.159 +		          // we know xen_hyper_ssm_i_0 or xen_hyper_ssm_i_1
   2.160 +		          // doesn't change p9 and p10
   2.161 +#endif
   2.162  (p9)	mov r8=ENOSYS
   2.163  	FSYS_RETURN
   2.164  END(__kernel_syscall_via_epc)
     3.1 --- a/linux-2.6-xen-sparse/arch/ia64/kernel/gate.lds.S	Mon Jul 24 13:04:40 2006 -0600
     3.2 +++ b/linux-2.6-xen-sparse/arch/ia64/kernel/gate.lds.S	Mon Jul 24 13:43:35 2006 -0600
     3.3 @@ -43,6 +43,28 @@ SECTIONS
     3.4  				    __start_gate_brl_fsys_bubble_down_patchlist = .;
     3.5  				    *(.data.patch.brl_fsys_bubble_down)
     3.6  				    __end_gate_brl_fsys_bubble_down_patchlist = .;
     3.7 +
     3.8 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
     3.9 +				    __start_gate_running_on_xen_patchlist = .;
    3.10 +				    *(.data.patch.running_on_xen)
    3.11 +				    __end_gate_running_on_xen_patchlist = .;
    3.12 +
    3.13 +				    __start_gate_brl_xen_rsm_be_i_patchlist = .;
    3.14 +				    *(.data.patch.brl_xen_rsm_be_i)
    3.15 +				    __end_gate_brl_xen_rsm_be_i_patchlist = .;
    3.16 +
    3.17 +				    __start_gate_brl_xen_get_psr_patchlist = .;
    3.18 +				    *(.data.patch.brl_xen_get_psr)
    3.19 +				    __end_gate_brl_xen_get_psr_patchlist = .;
    3.20 +
    3.21 +				    __start_gate_brl_xen_ssm_i_0_patchlist = .;
    3.22 +				    *(.data.patch.brl_xen_ssm_i_0)
    3.23 +				    __end_gate_brl_xen_ssm_i_0_patchlist = .;
    3.24 +
    3.25 +				    __start_gate_brl_xen_ssm_i_1_patchlist = .;
    3.26 +				    *(.data.patch.brl_xen_ssm_i_1)
    3.27 +				    __end_gate_brl_xen_ssm_i_1_patchlist = .;
    3.28 +#endif
    3.29    }									:readable
    3.30    .IA_64.unwind_info		: { *(.IA_64.unwind_info*) }
    3.31    .IA_64.unwind			: { *(.IA_64.unwind*) }			:readable :unwind
     4.1 --- a/linux-2.6-xen-sparse/arch/ia64/kernel/patch.c	Mon Jul 24 13:04:40 2006 -0600
     4.2 +++ b/linux-2.6-xen-sparse/arch/ia64/kernel/patch.c	Mon Jul 24 13:43:35 2006 -0600
     4.3 @@ -184,6 +184,73 @@ patch_brl_fsys_bubble_down (unsigned lon
     4.4  	ia64_srlz_i();
     4.5  }
     4.6  
     4.7 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
     4.8 +extern char __start_gate_running_on_xen_patchlist[];
     4.9 +extern char __end_gate_running_on_xen_patchlist[];
    4.10 +
    4.11 +void
    4.12 +patch_running_on_xen(unsigned long start, unsigned long end)
    4.13 +{
    4.14 +	extern int running_on_xen;
    4.15 +	s32 *offp = (s32 *)start;
    4.16 +	u64 ip;
    4.17 +
    4.18 +	while (offp < (s32 *)end) {
    4.19 +		ip = (u64)ia64_imva((char *)offp + *offp);
    4.20 +		ia64_patch_imm64(ip, (u64)&running_on_xen);
    4.21 +		ia64_fc((void *)ip);
    4.22 +		++offp;
    4.23 +	}
    4.24 +	ia64_sync_i();
    4.25 +	ia64_srlz_i();
    4.26 +}
    4.27 +
    4.28 +static void
    4.29 +patch_brl_symaddr(unsigned long start, unsigned long end,
    4.30 +                  unsigned long symaddr)
    4.31 +{
    4.32 +	s32 *offp = (s32 *)start;
    4.33 +	u64 ip;
    4.34 +
    4.35 +	while (offp < (s32 *)end) {
    4.36 +		ip = (u64)offp + *offp;
    4.37 +		ia64_patch_imm60((u64)ia64_imva((void *)ip),
    4.38 +				 (u64)(symaddr - (ip & -16)) / 16);
    4.39 +		ia64_fc((void *)ip);
    4.40 +		++offp;
    4.41 +	}
    4.42 +	ia64_sync_i();
    4.43 +	ia64_srlz_i();
    4.44 +}
    4.45 +
    4.46 +#define EXTERN_PATCHLIST(name)					\
    4.47 +	extern char __start_gate_brl_##name##_patchlist[];	\
    4.48 +	extern char __end_gate_brl_##name##_patchlist[];	\
    4.49 +	extern char name[]
    4.50 +
    4.51 +#define PATCH_BRL_SYMADDR(name)						\
    4.52 +	patch_brl_symaddr((unsigned long)__start_gate_brl_##name##_patchlist, \
    4.53 +	                  (unsigned long)__end_gate_brl_##name##_patchlist,   \
    4.54 +	                  (unsigned long)name)
    4.55 +
    4.56 +static void
    4.57 +patch_brl_in_vdso(void)
    4.58 +{
    4.59 +	EXTERN_PATCHLIST(xen_rsm_be_i);
    4.60 +	EXTERN_PATCHLIST(xen_get_psr);
    4.61 +	EXTERN_PATCHLIST(xen_ssm_i_0);
    4.62 +	EXTERN_PATCHLIST(xen_ssm_i_1);
    4.63 +
    4.64 +	PATCH_BRL_SYMADDR(xen_rsm_be_i);
    4.65 +	PATCH_BRL_SYMADDR(xen_get_psr);
    4.66 +	PATCH_BRL_SYMADDR(xen_ssm_i_0);
    4.67 +	PATCH_BRL_SYMADDR(xen_ssm_i_1);
    4.68 +}
    4.69 +#else
    4.70 +#define patch_running_on_xen(start, end)	do { } while (0)
    4.71 +#define patch_brl_in_vdso()			do { } while (0)
    4.72 +#endif
    4.73 +
    4.74  void
    4.75  ia64_patch_gate (void)
    4.76  {
    4.77 @@ -192,6 +259,10 @@ ia64_patch_gate (void)
    4.78  
    4.79  	patch_fsyscall_table(START(fsyscall), END(fsyscall));
    4.80  	patch_brl_fsys_bubble_down(START(brl_fsys_bubble_down), END(brl_fsys_bubble_down));
    4.81 +#ifdef CONFIG_XEN
    4.82 +	patch_running_on_xen(START(running_on_xen), END(running_on_xen));
    4.83 +	patch_brl_in_vdso();
    4.84 +#endif
    4.85  	ia64_patch_vtop(START(vtop), END(vtop));
    4.86  	ia64_patch_mckinley_e9(START(mckinley_e9), END(mckinley_e9));
    4.87  }
     5.1 --- a/linux-2.6-xen-sparse/arch/ia64/xen/hypercall.S	Mon Jul 24 13:04:40 2006 -0600
     5.2 +++ b/linux-2.6-xen-sparse/arch/ia64/xen/hypercall.S	Mon Jul 24 13:43:35 2006 -0600
     5.3 @@ -351,3 +351,59 @@ GLOBAL_ENTRY(xen_send_ipi)
     5.4          br.ret.sptk.many rp
     5.5          ;;
     5.6  END(xen_send_ipi)
     5.7 +
     5.8 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT
     5.9 +// Those are vdso specialized.
    5.10 +// In fsys mode, call, ret can't be used.
    5.11 +GLOBAL_ENTRY(xen_rsm_be_i)
    5.12 +	ld8 r22=[r22]
    5.13 +	;; 
    5.14 +	st1 [r22]=r20
    5.15 +	st4 [r23]=r0
    5.16 +	XEN_HYPER_RSM_BE
    5.17 +	st4 [r23]=r20
    5.18 +	brl.cond.sptk	.vdso_rsm_be_i_ret
    5.19 +	;; 
    5.20 +END(xen_rsm_be_i)
    5.21 +
    5.22 +GLOBAL_ENTRY(xen_get_psr)
    5.23 +	mov r31=r8
    5.24 +	mov r25=IA64_PSR_IC
    5.25 +	st4 [r23]=r0
    5.26 +	XEN_HYPER_GET_PSR
    5.27 +	;; 
    5.28 +	st4 [r23]=r20
    5.29 +	or r29=r8,r25 // vpsr.ic was cleared for hyperprivop
    5.30 +	mov r8=r31
    5.31 +	brl.cond.sptk	.vdso_get_psr_ret
    5.32 +	;; 
    5.33 +END(xen_get_psr)
    5.34 +
    5.35 +GLOBAL_ENTRY(xen_ssm_i_0)
    5.36 +	st4 [r22]=r20
    5.37 +	ld4 r25=[r24]
    5.38 +	;;
    5.39 +	cmp.ne.unc p11,p0=r0, r25
    5.40 +	;; 
    5.41 +(p11)	st4 [r22]=r0
    5.42 +(p11)	st4 [r23]=r0
    5.43 +(p11)	XEN_HYPER_SSM_I
    5.44 +	
    5.45 +	brl.cond.sptk	.vdso_ssm_i_0_ret
    5.46 +	;; 
    5.47 +END(xen_ssm_i_0)
    5.48 +
    5.49 +GLOBAL_ENTRY(xen_ssm_i_1)
    5.50 +	st4 [r22]=r20
    5.51 +	ld4 r25=[r24]
    5.52 +	;; 
    5.53 +	cmp.ne.unc p11,p0=r0, r25
    5.54 +	;; 
    5.55 +(p11)	st4 [r22]=r0
    5.56 +(p11)	st4 [r23]=r0
    5.57 +(p11)	XEN_HYPER_SSM_I
    5.58 +	;;
    5.59 +	brl.cond.sptk	.vdso_ssm_i_1_ret
    5.60 +	;; 
    5.61 +END(xen_ssm_i_1)
    5.62 +#endif
     6.1 --- a/linux-2.6-xen-sparse/include/asm-ia64/xen/privop.h	Mon Jul 24 13:04:40 2006 -0600
     6.2 +++ b/linux-2.6-xen-sparse/include/asm-ia64/xen/privop.h	Mon Jul 24 13:43:35 2006 -0600
     6.3 @@ -48,6 +48,8 @@
     6.4  #define	XEN_HYPER_GET_PMD		break HYPERPRIVOP_GET_PMD
     6.5  #define	XEN_HYPER_GET_EFLAG		break HYPERPRIVOP_GET_EFLAG
     6.6  #define	XEN_HYPER_SET_EFLAG		break HYPERPRIVOP_SET_EFLAG
     6.7 +#define	XEN_HYPER_RSM_BE		break HYPERPRIVOP_RSM_BE
     6.8 +#define	XEN_HYPER_GET_PSR		break HYPERPRIVOP_GET_PSR
     6.9  
    6.10  #define XSI_IFS			(XSI_BASE + XSI_IFS_OFS)
    6.11  #define XSI_PRECOVER_IFS	(XSI_BASE + XSI_PRECOVER_IFS_OFS)
     7.1 --- a/xen/arch/ia64/xen/privop.c	Mon Jul 24 13:04:40 2006 -0600
     7.2 +++ b/xen/arch/ia64/xen/privop.c	Mon Jul 24 13:43:35 2006 -0600
     7.3 @@ -789,6 +789,13 @@ ia64_hyperprivop(unsigned long iim, REGS
     7.4  	    case HYPERPRIVOP_SET_EFLAG:
     7.5  		(void)vcpu_set_ar(v,24,regs->r8);
     7.6  		return 1;
     7.7 +	    case HYPERPRIVOP_RSM_BE:
     7.8 +		(void)vcpu_reset_psr_sm(v, IA64_PSR_BE);
     7.9 +		return 1;
    7.10 +	    case HYPERPRIVOP_GET_PSR:
    7.11 +		(void)vcpu_get_psr(v, &val);
    7.12 +		regs->r8 = val;
    7.13 +		return 1;
    7.14  	}
    7.15  	return 0;
    7.16  }
     8.1 --- a/xen/include/public/arch-ia64.h	Mon Jul 24 13:04:40 2006 -0600
     8.2 +++ b/xen/include/public/arch-ia64.h	Mon Jul 24 13:43:35 2006 -0600
     8.3 @@ -420,7 +420,9 @@ struct xen_ia64_boot_param {
     8.4  #define HYPERPRIVOP_GET_PMD		0x15
     8.5  #define HYPERPRIVOP_GET_EFLAG		0x16
     8.6  #define HYPERPRIVOP_SET_EFLAG		0x17
     8.7 -#define HYPERPRIVOP_MAX			0x17
     8.8 +#define HYPERPRIVOP_RSM_BE		0x18
     8.9 +#define HYPERPRIVOP_GET_PSR		0x19
    8.10 +#define HYPERPRIVOP_MAX			0x19
    8.11  
    8.12  #endif /* __HYPERVISOR_IF_IA64_H__ */
    8.13