ia64/xen-unstable

changeset 9504:b09e8f46c9f6

[IA64] ptc.ga for SMP-g

Implementation of ptc.ga for SMP-g.

Signed-off-by: Tristan Gingold <tristan.gingold@bull.net>
author awilliam@xenbuild.aw
date Mon Apr 10 14:51:38 2006 -0600 (2006-04-10)
parents bfc00c83f083
children 0a7e619a248f
files xen/arch/ia64/vmx/vmmu.c xen/arch/ia64/xen/process.c xen/arch/ia64/xen/vcpu.c xen/arch/ia64/xen/vhpt.c xen/include/asm-ia64/ia64_int.h xen/include/asm-ia64/tlb.h xen/include/asm-ia64/vhpt.h
line diff
     1.1 --- a/xen/arch/ia64/vmx/vmmu.c	Fri Apr 07 14:06:44 2006 -0600
     1.2 +++ b/xen/arch/ia64/vmx/vmmu.c	Mon Apr 10 14:51:38 2006 -0600
     1.3 @@ -459,7 +459,7 @@ IA64FAULT vmx_vcpu_ptr_d(VCPU *vcpu,UINT
     1.4      va = PAGEALIGN(ifa, ps);
     1.5      index = vtr_find_overlap(vcpu, va, ps, DSIDE_TLB);
     1.6      if (index>=0) {
     1.7 -        vcpu->arch.dtrs[index].p=0;
     1.8 +        vcpu->arch.dtrs[index].pte.p=0;
     1.9          index = vtr_find_overlap(vcpu, va, ps, DSIDE_TLB);
    1.10      }
    1.11      hcb = vmx_vcpu_get_vtlb(vcpu);
    1.12 @@ -476,7 +476,7 @@ IA64FAULT vmx_vcpu_ptr_i(VCPU *vcpu,UINT
    1.13      va = PAGEALIGN(ifa, ps);
    1.14      index = vtr_find_overlap(vcpu, va, ps, ISIDE_TLB);
    1.15      if (index>=0) {
    1.16 -        vcpu->arch.itrs[index].p=0;
    1.17 +        vcpu->arch.itrs[index].pte.p=0;
    1.18          index = vtr_find_overlap(vcpu, va, ps, ISIDE_TLB);
    1.19      }
    1.20      hcb = vmx_vcpu_get_vtlb(vcpu);
     2.1 --- a/xen/arch/ia64/xen/process.c	Fri Apr 07 14:06:44 2006 -0600
     2.2 +++ b/xen/arch/ia64/xen/process.c	Mon Apr 10 14:51:38 2006 -0600
     2.3 @@ -287,12 +287,24 @@ void ia64_do_page_fault (unsigned long a
     2.4  		return;
     2.5  	}
     2.6  
     2.7 + again:
     2.8  	fault = vcpu_translate(current,address,is_data,0,&pteval,&itir,&iha);
     2.9 -	if (fault == IA64_NO_FAULT) {
    2.10 +	if (fault == IA64_NO_FAULT || fault == IA64_USE_TLB) {
    2.11  		pteval = translate_domain_pte(pteval,address,itir);
    2.12  		vcpu_itc_no_srlz(current,is_data?2:1,address,pteval,-1UL,(itir>>2)&0x3f);
    2.13 +		if (fault == IA64_USE_TLB && !current->arch.dtlb.pte.p) {
    2.14 +			/* dtlb has been purged in-between.  This dtlb was
    2.15 +			   matching.  Undo the work.  */
    2.16 +#ifdef VHPT_GLOBAL
    2.17 +			vhpt_flush_address (address, 1);
    2.18 +#endif
    2.19 +			ia64_ptcl(address, 1<<2);
    2.20 +			ia64_srlz_i();
    2.21 +			goto again;
    2.22 +		}
    2.23  		return;
    2.24  	}
    2.25 +
    2.26  	if (!user_mode (regs)) {
    2.27  		/* The fault occurs inside Xen.  */
    2.28  		if (!ia64_done_with_exception(regs)) {
     3.1 --- a/xen/arch/ia64/xen/vcpu.c	Fri Apr 07 14:06:44 2006 -0600
     3.2 +++ b/xen/arch/ia64/xen/vcpu.c	Mon Apr 10 14:51:38 2006 -0600
     3.3 @@ -1253,17 +1253,23 @@ unsigned long recover_to_break_fault_cou
     3.4  int warn_region0_address = 0; // FIXME later: tie to a boot parameter?
     3.5  
     3.6  // FIXME: also need to check && (!trp->key || vcpu_pkr_match(trp->key))
     3.7 +static inline int vcpu_match_tr_entry_no_p(TR_ENTRY *trp, UINT64 ifa, UINT64 rid)
     3.8 +{
     3.9 +	return trp->rid == rid 
    3.10 +		&& ifa >= trp->vadr
    3.11 +		&& ifa <= (trp->vadr + (1L << trp->ps) - 1);
    3.12 +}
    3.13 +
    3.14  static inline int vcpu_match_tr_entry(TR_ENTRY *trp, UINT64 ifa, UINT64 rid)
    3.15  {
    3.16 -	return trp->p && trp->rid == rid 
    3.17 -		&& ifa >= trp->vadr
    3.18 -		&& ifa <= (trp->vadr + (1L << trp->ps) - 1);
    3.19 +	return trp->pte.p && vcpu_match_tr_entry_no_p(trp, ifa, rid);
    3.20  }
    3.21  
    3.22  IA64FAULT vcpu_translate(VCPU *vcpu, UINT64 address, BOOLEAN is_data, BOOLEAN in_tpa, UINT64 *pteval, UINT64 *itir, UINT64 *iha)
    3.23  {
    3.24  	unsigned long region = address >> 61;
    3.25 -	unsigned long pta, pte, rid, rr;
    3.26 +	unsigned long pta, rid, rr;
    3.27 +	union pte_flags pte;
    3.28  	int i;
    3.29  	TR_ENTRY *trp;
    3.30  
    3.31 @@ -1283,6 +1289,7 @@ IA64FAULT vcpu_translate(VCPU *vcpu, UIN
    3.32  			 */           
    3.33  			printk("vcpu_translate: bad physical address: 0x%lx\n",
    3.34  			       address);
    3.35 +
    3.36  		} else {
    3.37  			*pteval = (address & _PAGE_PPN_MASK) | __DIRTY_BITS |
    3.38  			          _PAGE_PL_2 | _PAGE_AR_RWX;
    3.39 @@ -1307,7 +1314,7 @@ IA64FAULT vcpu_translate(VCPU *vcpu, UIN
    3.40  		if (vcpu_quick_region_check(vcpu->arch.dtr_regions,address)) {
    3.41  			for (trp = vcpu->arch.dtrs, i = NDTRS; i; i--, trp++) {
    3.42  				if (vcpu_match_tr_entry(trp,address,rid)) {
    3.43 -					*pteval = trp->page_flags;
    3.44 +					*pteval = trp->pte.val;
    3.45  					*itir = trp->itir;
    3.46  					tr_translate_count++;
    3.47  					return IA64_NO_FAULT;
    3.48 @@ -1320,7 +1327,7 @@ IA64FAULT vcpu_translate(VCPU *vcpu, UIN
    3.49  		if (vcpu_quick_region_check(vcpu->arch.itr_regions,address)) {
    3.50  			for (trp = vcpu->arch.itrs, i = NITRS; i; i--, trp++) {
    3.51  				if (vcpu_match_tr_entry(trp,address,rid)) {
    3.52 -					*pteval = trp->page_flags;
    3.53 +					*pteval = trp->pte.val;
    3.54  					*itir = trp->itir;
    3.55  					tr_translate_count++;
    3.56  					return IA64_NO_FAULT;
    3.57 @@ -1332,12 +1339,14 @@ IA64FAULT vcpu_translate(VCPU *vcpu, UIN
    3.58  	/* check 1-entry TLB */
    3.59  	// FIXME?: check dtlb for inst accesses too, else bad things happen?
    3.60  	trp = &vcpu->arch.dtlb;
    3.61 -	if (/* is_data && */ vcpu_match_tr_entry(trp,address,rid)) {
    3.62 -		if (vcpu->domain==dom0 && !in_tpa) *pteval = trp->page_flags;
    3.63 +	pte = trp->pte;
    3.64 +	if (/* is_data && */ pte.p
    3.65 +	    && vcpu_match_tr_entry_no_p(trp,address,rid)) {
    3.66 +		if (vcpu->domain==dom0 && !in_tpa) *pteval = pte.val;
    3.67  		else *pteval = vcpu->arch.dtlb_pte;
    3.68  		*itir = trp->itir;
    3.69  		dtlb_translate_count++;
    3.70 -		return IA64_NO_FAULT;
    3.71 +		return IA64_USE_TLB;
    3.72  	}
    3.73  
    3.74  	/* check guest VHPT */
    3.75 @@ -1358,7 +1367,8 @@ IA64FAULT vcpu_translate(VCPU *vcpu, UIN
    3.76  	if (((address ^ pta) & ((itir_mask(pta) << 3) >> 3)) == 0)
    3.77  		return (is_data ? IA64_DATA_TLB_VECTOR : IA64_INST_TLB_VECTOR);
    3.78  
    3.79 -	if (__copy_from_user(&pte, (void *)(*iha), sizeof(pte)) != 0)
    3.80 +	if (!__access_ok (*iha)
    3.81 +	    || __copy_from_user(&pte, (void *)(*iha), sizeof(pte)) != 0)
    3.82  		// virtual VHPT walker "missed" in TLB
    3.83  		return IA64_VHPT_FAULT;
    3.84  
    3.85 @@ -1367,12 +1377,12 @@ IA64FAULT vcpu_translate(VCPU *vcpu, UIN
    3.86  	* instead of inserting a not-present translation, this allows
    3.87  	* vectoring directly to the miss handler.
    3.88  	*/
    3.89 -	if (!(pte & _PAGE_P))
    3.90 +	if (!pte.p)
    3.91  		return (is_data ? IA64_DATA_TLB_VECTOR : IA64_INST_TLB_VECTOR);
    3.92  
    3.93  	/* found mapping in guest VHPT! */
    3.94  	*itir = rr & RR_PS_MASK;
    3.95 -	*pteval = pte;
    3.96 +	*pteval = pte.val;
    3.97  	vhpt_translate_count++;
    3.98  	return IA64_NO_FAULT;
    3.99  }
   3.100 @@ -1383,7 +1393,7 @@ IA64FAULT vcpu_tpa(VCPU *vcpu, UINT64 va
   3.101  	IA64FAULT fault;
   3.102  
   3.103  	fault = vcpu_translate(vcpu, vadr, TRUE, TRUE, &pteval, &itir, &iha);
   3.104 -	if (fault == IA64_NO_FAULT)
   3.105 +	if (fault == IA64_NO_FAULT || fault == IA64_USE_TLB)
   3.106  	{
   3.107  		mask = itir_mask(itir);
   3.108  		*padr = (pteval & _PAGE_PPN_MASK & mask) | (vadr & ~mask);
   3.109 @@ -1670,24 +1680,27 @@ IA64FAULT vcpu_set_pkr(VCPU *vcpu, UINT6
   3.110  
   3.111  static inline void vcpu_purge_tr_entry(TR_ENTRY *trp)
   3.112  {
   3.113 -	trp->p = 0;
   3.114 +	trp->pte.val = 0;
   3.115  }
   3.116  
   3.117  static void vcpu_set_tr_entry(TR_ENTRY *trp, UINT64 pte, UINT64 itir, UINT64 ifa)
   3.118  {
   3.119  	UINT64 ps;
   3.120 +	union pte_flags new_pte;
   3.121  
   3.122  	trp->itir = itir;
   3.123  	trp->rid = VCPU(current,rrs[ifa>>61]) & RR_RID_MASK;
   3.124 -	trp->p = 1;
   3.125  	ps = trp->ps;
   3.126 -	trp->page_flags = pte;
   3.127 -	if (trp->pl < 2) trp->pl = 2;
   3.128 +	new_pte.val = pte;
   3.129 +	if (new_pte.pl < 2) new_pte.pl = 2;
   3.130  	trp->vadr = ifa & ~0xfff;
   3.131  	if (ps > 12) { // "ignore" relevant low-order bits
   3.132 -		trp->ppn &= ~((1UL<<(ps-12))-1);
   3.133 +		new_pte.ppn &= ~((1UL<<(ps-12))-1);
   3.134  		trp->vadr &= ~((1UL<<ps)-1);
   3.135  	}
   3.136 +
   3.137 +	/* Atomic write.  */
   3.138 +	trp->pte.val = new_pte.val;
   3.139  }
   3.140  
   3.141  IA64FAULT vcpu_itr_d(VCPU *vcpu, UINT64 slot, UINT64 pte,
   3.142 @@ -1852,19 +1865,6 @@ IA64FAULT vcpu_ptc_g(VCPU *vcpu, UINT64 
   3.143  	return IA64_ILLOP_FAULT;
   3.144  }
   3.145  
   3.146 -#if defined(CONFIG_XEN_SMP) && defined(VHPT_GLOBAL)
   3.147 -struct ptc_ga_args {
   3.148 -	unsigned long vadr;
   3.149 -	unsigned long addr_range;
   3.150 -};
   3.151 -
   3.152 -static void ptc_ga_remote_func (void *varg)
   3.153 -{
   3.154 -	struct ptc_ga_args *args = (struct ptc_ga_args *)varg;
   3.155 -	vhpt_flush_address (args->vadr, args->addr_range);
   3.156 -}
   3.157 -#endif
   3.158 -
   3.159  IA64FAULT vcpu_ptc_ga(VCPU *vcpu,UINT64 vadr,UINT64 addr_range)
   3.160  {
   3.161  	// FIXME: validate not flushing Xen addresses
   3.162 @@ -1875,32 +1875,20 @@ IA64FAULT vcpu_ptc_ga(VCPU *vcpu,UINT64 
   3.163  #ifdef CONFIG_XEN_SMP
   3.164  	struct domain *d = vcpu->domain;
   3.165  	struct vcpu *v;
   3.166 -	struct ptc_ga_args args;
   3.167  
   3.168 -	args.vadr = vadr;
   3.169 -	args.addr_range = addr_range;
   3.170 -
   3.171 -	/* This method is very conservative and should be optimized:
   3.172 -	   - maybe IPI calls can be avoided,
   3.173 -	   - a processor map can be built to avoid duplicate purge
   3.174 -	   - maybe ptc.ga can be replaced by ptc.l+invala.
   3.175 -	   Hopefully, it has no impact when UP.
   3.176 -	*/
   3.177  	for_each_vcpu (d, v) {
   3.178 -		if (v != vcpu) {
   3.179 -			/* Purge tc entry.
   3.180 -			   Can we do this directly ?  Well, this is just a
   3.181 -			   single atomic write.  */
   3.182 -			vcpu_purge_tr_entry(&PSCBX(v,dtlb));
   3.183 -			vcpu_purge_tr_entry(&PSCBX(v,itlb));
   3.184 +		if (v == vcpu)
   3.185 +			continue;
   3.186 +
   3.187 +		/* Purge TC entries.
   3.188 +		   FIXME: clear only if match.  */
   3.189 +		vcpu_purge_tr_entry(&PSCBX(vcpu,dtlb));
   3.190 +		vcpu_purge_tr_entry(&PSCBX(vcpu,itlb));
   3.191 +
   3.192  #ifdef VHPT_GLOBAL
   3.193 -			/* Flush VHPT on remote processors.
   3.194 -			   FIXME: invalidate directly the entries? */
   3.195 -			smp_call_function_single
   3.196 -				(v->processor, &ptc_ga_remote_func,
   3.197 -				 &args, 0, 1);
   3.198 +		/* Invalidate VHPT entries.  */
   3.199 +		vhpt_flush_address_remote (v->processor, vadr, addr_range);
   3.200  #endif
   3.201 -		}
   3.202  	}
   3.203  #endif
   3.204  
     4.1 --- a/xen/arch/ia64/xen/vhpt.c	Fri Apr 07 14:06:44 2006 -0600
     4.2 +++ b/xen/arch/ia64/xen/vhpt.c	Mon Apr 10 14:51:38 2006 -0600
     4.3 @@ -75,6 +75,20 @@ void vhpt_flush_address(unsigned long va
     4.4  		vadr += PAGE_SIZE;
     4.5  	}
     4.6  }
     4.7 +
     4.8 +void vhpt_flush_address_remote(int cpu,
     4.9 +			       unsigned long vadr, unsigned long addr_range)
    4.10 +{
    4.11 +	while ((long)addr_range > 0) {
    4.12 +		/* Get the VHPT entry.  */
    4.13 +		unsigned int off = ia64_thash(vadr) - VHPT_ADDR;
    4.14 +		volatile struct vhpt_lf_entry *v;
    4.15 +		v =__va(per_cpu(vhpt_paddr, cpu) + off);
    4.16 +		v->ti_tag = INVALID_TI_TAG;
    4.17 +		addr_range -= PAGE_SIZE;
    4.18 +		vadr += PAGE_SIZE;
    4.19 +	}
    4.20 +}
    4.21  #endif
    4.22  
    4.23  static void vhpt_map(unsigned long pte)
     5.1 --- a/xen/include/asm-ia64/ia64_int.h	Fri Apr 07 14:06:44 2006 -0600
     5.2 +++ b/xen/include/asm-ia64/ia64_int.h	Mon Apr 10 14:51:38 2006 -0600
     5.3 @@ -38,6 +38,7 @@
     5.4  #define	IA64_RFI_IN_PROGRESS	0x0002
     5.5  #define IA64_RETRY              0x0003
     5.6  #define IA64_FORCED_IFA         0x0004
     5.7 +#define IA64_USE_TLB		0x0005
     5.8  #define	IA64_ILLOP_FAULT	(IA64_GENEX_VECTOR | 0x00)
     5.9  #define	IA64_PRIVOP_FAULT	(IA64_GENEX_VECTOR | 0x10)
    5.10  #define	IA64_PRIVREG_FAULT	(IA64_GENEX_VECTOR | 0x20)
     6.1 --- a/xen/include/asm-ia64/tlb.h	Fri Apr 07 14:06:44 2006 -0600
     6.2 +++ b/xen/include/asm-ia64/tlb.h	Mon Apr 10 14:51:38 2006 -0600
     6.3 @@ -4,23 +4,24 @@
     6.4  #define	NITRS	8
     6.5  #define NDTRS	8
     6.6  
     6.7 +union pte_flags {
     6.8 +    struct {
     6.9 +	    unsigned long p    :  1; // 0
    6.10 +	    unsigned long      :  1; // 1
    6.11 +	    unsigned long ma   :  3; // 2-4
    6.12 +	    unsigned long a    :  1; // 5
    6.13 +	    unsigned long d    :  1; // 6
    6.14 +	    unsigned long pl   :  2; // 7-8
    6.15 +	    unsigned long ar   :  3; // 9-11
    6.16 +	    unsigned long ppn  : 38; // 12-49
    6.17 +	    unsigned long      :  2; // 50-51
    6.18 +	    unsigned long ed   :  1; // 52
    6.19 +    };
    6.20 +    unsigned long val;
    6.21 +};
    6.22 +
    6.23  typedef struct {
    6.24 -    union {
    6.25 -        struct {
    6.26 -            unsigned long p    :  1; // 0
    6.27 -            unsigned long      :  1; // 1
    6.28 -            unsigned long ma   :  3; // 2-4
    6.29 -            unsigned long a    :  1; // 5
    6.30 -            unsigned long d    :  1; // 6
    6.31 -            unsigned long pl   :  2; // 7-8
    6.32 -            unsigned long ar   :  3; // 9-11
    6.33 -            unsigned long ppn  : 38; // 12-49
    6.34 -            unsigned long      :  2; // 50-51
    6.35 -            unsigned long ed   :  1; // 52
    6.36 -        };
    6.37 -        unsigned long page_flags;
    6.38 -    };
    6.39 -
    6.40 +    volatile union pte_flags pte;
    6.41      union {
    6.42          struct {
    6.43              unsigned long      :  2; // 0-1
     7.1 --- a/xen/include/asm-ia64/vhpt.h	Fri Apr 07 14:06:44 2006 -0600
     7.2 +++ b/xen/include/asm-ia64/vhpt.h	Mon Apr 10 14:51:38 2006 -0600
     7.3 @@ -122,6 +122,8 @@ extern void vhpt_init (void);
     7.4  extern void zero_vhpt_stats(void);
     7.5  extern int dump_vhpt_stats(char *buf);
     7.6  extern void vhpt_flush_address(unsigned long vadr, unsigned long addr_range);
     7.7 +extern void vhpt_flush_address_remote(int cpu, unsigned long vadr,
     7.8 +				      unsigned long addr_range);
     7.9  extern void vhpt_multiple_insert(unsigned long vaddr, unsigned long pte,
    7.10  				 unsigned long logps);
    7.11  extern void vhpt_insert (unsigned long vadr, unsigned long ptr,