1.1 --- a/xen/arch/ia64/xen/mm.c	Mon Jun 19 12:13:11 2006 -0600
     1.2 +++ b/xen/arch/ia64/xen/mm.c	Mon Jun 19 12:52:38 2006 -0600
     1.3 @@ -338,10 +338,42 @@ unsigned long translate_domain_mpaddr(un
     1.4  }
     1.5  
     1.6  //XXX !xxx_present() should be used instread of !xxx_none()?
     1.7 +// __assign_new_domain_page(), assign_new_domain_page() and
     1.8 +// assign_new_domain0_page() are used only when domain creation.
     1.9 +// their accesses aren't racy so that returned pte_t doesn't need
    1.10 +// volatile qualifier
    1.11 +static pte_t*
    1.12 +__lookup_alloc_domain_pte(struct domain* d, unsigned long mpaddr)
    1.13 +{
    1.14 +    struct mm_struct *mm = &d->arch.mm;
    1.15 +    pgd_t *pgd;
    1.16 +    pud_t *pud;
    1.17 +    pmd_t *pmd;
    1.18 +
    1.19 +    BUG_ON(mm->pgd == NULL);
    1.20 +    pgd = pgd_offset(mm, mpaddr);
    1.21 +    if (pgd_none(*pgd)) {
    1.22 +        pgd_populate(mm, pgd, pud_alloc_one(mm,mpaddr));
    1.23 +    }
    1.24 +
    1.25 +    pud = pud_offset(pgd, mpaddr);
    1.26 +    if (pud_none(*pud)) {
    1.27 +        pud_populate(mm, pud, pmd_alloc_one(mm,mpaddr));
    1.28 +    }
    1.29 +
    1.30 +    pmd = pmd_offset(pud, mpaddr);
    1.31 +    if (pmd_none(*pmd)) {
    1.32 +        pmd_populate_kernel(mm, pmd, pte_alloc_one_kernel(mm, mpaddr));
    1.33 +    }
    1.34 +
    1.35 +    return pte_offset_map(pmd, mpaddr);
    1.36 +}
    1.37 +
    1.38 +//XXX !xxx_present() should be used instread of !xxx_none()?
    1.39  // pud, pmd, pte page is zero cleared when they are allocated.
    1.40  // Their area must be visible before population so that
    1.41  // cmpxchg must have release semantics.
    1.42 -static pte_t*
    1.43 +static volatile pte_t*
    1.44  lookup_alloc_domain_pte(struct domain* d, unsigned long mpaddr)
    1.45  {
    1.46      struct mm_struct *mm = &d->arch.mm;
    1.47 @@ -384,11 +416,11 @@ lookup_alloc_domain_pte(struct domain* d
    1.48          }
    1.49      }
    1.50  
    1.51 -    return pte_offset_map(pmd, mpaddr);
    1.52 +    return (volatile pte_t*)pte_offset_map(pmd, mpaddr);
    1.53  }
    1.54  
    1.55  //XXX xxx_none() should be used instread of !xxx_present()?
    1.56 -static pte_t*
    1.57 +static volatile pte_t*
    1.58  lookup_noalloc_domain_pte(struct domain* d, unsigned long mpaddr)
    1.59  {
    1.60      struct mm_struct *mm = &d->arch.mm;
    1.61 @@ -409,11 +441,11 @@ lookup_noalloc_domain_pte(struct domain*
    1.62      if (unlikely(!pmd_present(*pmd)))
    1.63          return NULL;
    1.64  
    1.65 -    return pte_offset_map(pmd, mpaddr);
    1.66 +    return (volatile pte_t*)pte_offset_map(pmd, mpaddr);
    1.67  }
    1.68  
    1.69  #ifdef CONFIG_XEN_IA64_DOM0_VP
    1.70 -static pte_t*
    1.71 +static volatile pte_t*
    1.72  lookup_noalloc_domain_pte_none(struct domain* d, unsigned long mpaddr)
    1.73  {
    1.74      struct mm_struct *mm = &d->arch.mm;
    1.75 @@ -434,13 +466,13 @@ lookup_noalloc_domain_pte_none(struct do
    1.76      if (unlikely(pmd_none(*pmd)))
    1.77          return NULL;
    1.78  
    1.79 -    return pte_offset_map(pmd, mpaddr);
    1.80 +    return (volatile pte_t*)pte_offset_map(pmd, mpaddr);
    1.81  }
    1.82  
    1.83  unsigned long
    1.84  ____lookup_domain_mpa(struct domain *d, unsigned long mpaddr)
    1.85  {
    1.86 -    pte_t *pte;
    1.87 +    volatile pte_t *pte;
    1.88  
    1.89      pte = lookup_noalloc_domain_pte(d, mpaddr);
    1.90      if (pte == NULL)
    1.91 @@ -470,7 +502,7 @@ unsigned long
    1.92  
    1.93  unsigned long lookup_domain_mpa(struct domain *d, unsigned long mpaddr)
    1.94  {
    1.95 -    pte_t *pte;
    1.96 +    volatile pte_t *pte;
    1.97  
    1.98  #ifdef CONFIG_DOMAIN0_CONTIGUOUS
    1.99      if (d == dom0) {
   1.100 @@ -486,9 +518,10 @@ unsigned long lookup_domain_mpa(struct d
   1.101  #endif
   1.102      pte = lookup_noalloc_domain_pte(d, mpaddr);
   1.103      if (pte != NULL) {
   1.104 -        if (pte_present(*pte)) {
   1.105 +        pte_t tmp_pte = *pte;// pte is volatile. copy the value.
   1.106 +        if (pte_present(tmp_pte)) {
   1.107  //printk("lookup_domain_page: found mapping for %lx, pte=%lx\n",mpaddr,pte_val(*pte));
   1.108 -            return pte_val(*pte);
   1.109 +            return pte_val(tmp_pte);
   1.110          } else if (VMX_DOMAIN(d->vcpu[0]))
   1.111              return GPFN_INV_MASK;
   1.112      }
   1.113 @@ -530,7 +563,7 @@ void *domain_mpa_to_imva(struct domain *
   1.114  
   1.115  /* Allocate a new page for domain and map it to the specified metaphysical
   1.116     address.  */
   1.117 -struct page_info *
   1.118 +static struct page_info *
   1.119  __assign_new_domain_page(struct domain *d, unsigned long mpaddr, pte_t* pte)
   1.120  {
   1.121      struct page_info *p = NULL;
   1.122 @@ -595,7 +628,7 @@ assign_new_domain_page(struct domain *d,
   1.123      struct page_info *p = NULL;
   1.124      pte_t *pte;
   1.125  
   1.126 -    pte = lookup_alloc_domain_pte(d, mpaddr);
   1.127 +    pte = __lookup_alloc_domain_pte(d, mpaddr);
   1.128      if (pte_none(*pte))
   1.129          p = __assign_new_domain_page(d, mpaddr, pte);
   1.130  
   1.131 @@ -610,7 +643,7 @@ assign_new_domain0_page(struct domain *d
   1.132      pte_t *pte;
   1.133  
   1.134      BUG_ON(d != dom0);
   1.135 -    pte = lookup_alloc_domain_pte(d, mpaddr);
   1.136 +    pte = __lookup_alloc_domain_pte(d, mpaddr);
   1.137      if (pte_none(*pte)) {
   1.138          struct page_info *p = __assign_new_domain_page(d, mpaddr, pte);
   1.139          if (p == NULL) {
   1.140 @@ -622,21 +655,27 @@ assign_new_domain0_page(struct domain *d
   1.141  
   1.142  /* map a physical address to the specified metaphysical addr */
   1.143  // flags: currently only ASSIGN_readonly
   1.144 +// This is called by assign_domain_mmio_page().
   1.145 +// So accessing to pte is racy.
   1.146  void
   1.147  __assign_domain_page(struct domain *d,
   1.148                       unsigned long mpaddr, unsigned long physaddr,
   1.149                       unsigned long flags)
   1.150  {
   1.151 -    pte_t *pte;
   1.152 +    volatile pte_t *pte;
   1.153 +    pte_t old_pte;
   1.154 +    pte_t new_pte;
   1.155 +    pte_t ret_pte;
   1.156      unsigned long arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX;
   1.157  
   1.158      pte = lookup_alloc_domain_pte(d, mpaddr);
   1.159 -    if (pte_none(*pte)) {
   1.160 -        set_pte_rel(pte,
   1.161 -                    pfn_pte(physaddr >> PAGE_SHIFT,
   1.162 -                            __pgprot(__DIRTY_BITS | _PAGE_PL_2 | arflags)));
   1.163 +
   1.164 +    old_pte = __pte(0);
   1.165 +    new_pte = pfn_pte(physaddr >> PAGE_SHIFT,
   1.166 +                      __pgprot(__DIRTY_BITS | _PAGE_PL_2 | arflags));
   1.167 +    ret_pte = ptep_cmpxchg_rel(&d->arch.mm, mpaddr, pte, old_pte, new_pte);
   1.168 +    if (pte_val(ret_pte) == pte_val(old_pte))
   1.169          smp_mb();
   1.170 -    }
   1.171  }
   1.172  
   1.173  /* get_page() and map a physical address to the specified metaphysical addr */
   1.174 @@ -755,7 +794,7 @@ assign_domain_page_replace(struct domain
   1.175                             unsigned long mfn, unsigned long flags)
   1.176  {
   1.177      struct mm_struct *mm = &d->arch.mm;
   1.178 -    pte_t* pte;
   1.179 +    volatile pte_t* pte;
   1.180      pte_t old_pte;
   1.181      pte_t npte;
   1.182      unsigned long arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX;
   1.183 @@ -798,7 +837,7 @@ assign_domain_page_cmpxchg_rel(struct do
   1.184                                 unsigned long flags)
   1.185  {
   1.186      struct mm_struct *mm = &d->arch.mm;
   1.187 -    pte_t* pte;
   1.188 +    volatile pte_t* pte;
   1.189      unsigned long old_mfn;
   1.190      unsigned long old_arflags;
   1.191      pte_t old_pte;
   1.192 @@ -810,9 +849,14 @@ assign_domain_page_cmpxchg_rel(struct do
   1.193      pte = lookup_alloc_domain_pte(d, mpaddr);
   1.194  
   1.195   again:
   1.196 -    old_arflags = pte_val(*pte) & ~_PAGE_PPN_MASK;//XXX
   1.197 +    old_arflags = pte_val(*pte) & ~_PAGE_PPN_MASK;
   1.198      old_mfn = page_to_mfn(old_page);
   1.199      old_pte = pfn_pte(old_mfn, __pgprot(old_arflags));
   1.200 +    if (!pte_present(old_pte)) {
   1.201 +        DPRINTK("%s: old_pte 0x%lx old_arflags 0x%lx old_mfn 0x%lx\n",
   1.202 +                __func__, pte_val(old_pte), old_arflags, old_mfn);
   1.203 +        return -EINVAL;
   1.204 +    }
   1.205  
   1.206      new_arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX;
   1.207      new_mfn = page_to_mfn(new_page);
   1.208 @@ -850,7 +894,7 @@ static void
   1.209  zap_domain_page_one(struct domain *d, unsigned long mpaddr)
   1.210  {
   1.211      struct mm_struct *mm = &d->arch.mm;
   1.212 -    pte_t *pte;
   1.213 +    volatile pte_t *pte;
   1.214      pte_t old_pte;
   1.215      unsigned long mfn;
   1.216      struct page_info *page;
   1.217 @@ -970,10 +1014,12 @@ destroy_grant_host_mapping(unsigned long
   1.218                 unsigned long mfn, unsigned int flags)
   1.219  {
   1.220      struct domain* d = current->domain;
   1.221 -    pte_t* pte;
   1.222 +    volatile pte_t* pte;
   1.223 +    unsigned long cur_arflags;
   1.224 +    pte_t cur_pte;
   1.225 +    pte_t new_pte;
   1.226      pte_t old_pte;
   1.227 -    unsigned long old_mfn = INVALID_MFN;
   1.228 -    struct page_info* old_page;
   1.229 +    struct page_info* page;
   1.230  
   1.231      if (flags & (GNTMAP_application_map | GNTMAP_contains_pte)) {
   1.232          DPRINTK("%s: flags 0x%x\n", __func__, flags);
   1.233 @@ -981,21 +1027,42 @@ destroy_grant_host_mapping(unsigned long
   1.234      }
   1.235  
   1.236      pte = lookup_noalloc_domain_pte(d, gpaddr);
   1.237 -    if (pte == NULL || !pte_present(*pte) || pte_pfn(*pte) != mfn)
   1.238 +    if (pte == NULL) {
   1.239 +        DPRINTK("%s: gpaddr 0x%lx mfn 0x%lx\n", __func__, gpaddr, mfn);
   1.240          return GNTST_general_error;
   1.241 +    }
   1.242  
   1.243 -    // update pte
   1.244 -    old_pte = ptep_get_and_clear(&d->arch.mm, gpaddr, pte);
   1.245 -    if (pte_present(old_pte)) {
   1.246 -        old_mfn = pte_pfn(old_pte);
   1.247 -    } else {
   1.248 + again:
   1.249 +    cur_arflags = pte_val(*pte) & ~_PAGE_PPN_MASK;
   1.250 +    cur_pte = pfn_pte(mfn, __pgprot(cur_arflags));
   1.251 +    if (!pte_present(cur_pte)) {
   1.252 +        DPRINTK("%s: gpaddr 0x%lx mfn 0x%lx cur_pte 0x%lx\n",
   1.253 +                __func__, gpaddr, mfn, pte_val(cur_pte));
   1.254          return GNTST_general_error;
   1.255      }
   1.256 -    domain_page_flush(d, gpaddr, old_mfn, INVALID_MFN);
   1.257 +    new_pte = __pte(0);
   1.258  
   1.259 -    old_page = mfn_to_page(old_mfn);
   1.260 -    BUG_ON(page_get_owner(old_page) == d);//try_to_clear_PGC_allocate(d, page) is not needed.
   1.261 -    put_page(old_page);
   1.262 +    old_pte = ptep_cmpxchg_rel(&d->arch.mm, gpaddr, pte, cur_pte, new_pte);
   1.263 +    if (unlikely(!pte_present(old_pte))) {
   1.264 +        DPRINTK("%s: gpaddr 0x%lx mfn 0x%lx cur_pte 0x%lx old_pte 0x%lx\n",
   1.265 +                __func__, gpaddr, mfn, pte_val(cur_pte), pte_val(old_pte));
   1.266 +        return GNTST_general_error;
   1.267 +    }
   1.268 +    if (unlikely(pte_val(cur_pte) != pte_val(old_pte))) {
   1.269 +        if (pte_pfn(old_pte) == mfn) {
   1.270 +            goto again;
   1.271 +        }
   1.272 +        DPRINTK("%s gpaddr 0x%lx mfn 0x%lx cur_pte 0x%lx old_pte 0x%lx\n",
   1.273 +                __func__, gpaddr, mfn, pte_val(cur_pte), pte_val(old_pte));
   1.274 +        return GNTST_general_error;
   1.275 +    }
   1.276 +    BUG_ON(pte_pfn(old_pte) != mfn);
   1.277 +
   1.278 +    domain_page_flush(d, gpaddr, mfn, INVALID_MFN);
   1.279 +
   1.280 +    page = mfn_to_page(mfn);
   1.281 +    BUG_ON(page_get_owner(page) == d);//try_to_clear_PGC_allocate(d, page) is not needed.
   1.282 +    put_page(page);
   1.283  
   1.284      return GNTST_okay;
   1.285  }
   1.286 @@ -1141,7 +1208,7 @@ domain_page_flush(struct domain* d, unsi
   1.287  int
   1.288  domain_page_mapped(struct domain* d, unsigned long mpaddr)
   1.289  {
   1.290 -    pte_t * pte;
   1.291 +    volatile pte_t * pte;
   1.292  
   1.293      pte = lookup_noalloc_domain_pte(d, mpaddr);
   1.294      if(pte != NULL && !pte_none(*pte))

     2.1 --- a/xen/include/asm-ia64/linux-xen/asm/pgtable.h	Mon Jun 19 12:13:11 2006 -0600
     2.2 +++ b/xen/include/asm-ia64/linux-xen/asm/pgtable.h	Mon Jun 19 12:52:38 2006 -0600
     2.3 @@ -210,7 +210,7 @@ ia64_phys_addr_valid (unsigned long addr
     2.4  #define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval)
     2.5  #ifdef XEN
     2.6  static inline void
     2.7 -set_pte_rel(pte_t* ptep, pte_t pteval)
     2.8 +set_pte_rel(volatile pte_t* ptep, pte_t pteval)
     2.9  {
    2.10  #if CONFIG_SMP
    2.11  	asm volatile ("st8.rel [%0]=%1" ::
    2.12 @@ -402,8 +402,14 @@ ptep_test_and_clear_dirty (struct vm_are
    2.13  }
    2.14  #endif
    2.15  
    2.16 +#ifdef XEN
    2.17 +static inline pte_t
    2.18 +ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
    2.19 +		   volatile pte_t *ptep)
    2.20 +#else
    2.21  static inline pte_t
    2.22  ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
    2.23 +#endif
    2.24  {
    2.25  #ifdef CONFIG_SMP
    2.26  	return __pte(xchg((long *) ptep, 0));
    2.27 @@ -416,7 +422,8 @@ ptep_get_and_clear(struct mm_struct *mm,
    2.28  
    2.29  #ifdef XEN
    2.30  static inline pte_t
    2.31 -ptep_xchg(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t npte)
    2.32 +ptep_xchg(struct mm_struct *mm, unsigned long addr,
    2.33 +	  volatile pte_t *ptep, pte_t npte)
    2.34  {
    2.35  #ifdef CONFIG_SMP
    2.36  	return __pte(xchg((long *) ptep, pte_val(npte)));
    2.37 @@ -428,8 +435,8 @@ ptep_xchg(struct mm_struct *mm, unsigned
    2.38  }
    2.39  
    2.40  static inline pte_t
    2.41 -ptep_cmpxchg_rel(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
    2.42 -		 pte_t old_pte, pte_t new_pte)
    2.43 +ptep_cmpxchg_rel(struct mm_struct *mm, unsigned long addr,
    2.44 +		 volatile pte_t *ptep, pte_t old_pte, pte_t new_pte)
    2.45  {
    2.46  #ifdef CONFIG_SMP
    2.47  	return __pte(cmpxchg_rel(&pte_val(*ptep),