direct-io.hg

changeset 10583:f02d59f9b11f

[IA64] add volatile to p2m table pte entry

add volatile pte entry of the p2m table.
Since p2m table are shared by cpu. volatile is needed.
fix the races in pte access of __assign_domain_page() and
destroy_grant_host_mapping()

Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
author awilliam@xenbuild.aw
date Mon Jun 19 12:52:38 2006 -0600 (2006-06-19)
parents dcb50b04faec
children 7e26d6ffdde7
files xen/arch/ia64/xen/mm.c xen/include/asm-ia64/linux-xen/asm/pgtable.h
line diff
     1.1 --- a/xen/arch/ia64/xen/mm.c	Mon Jun 19 12:13:11 2006 -0600
     1.2 +++ b/xen/arch/ia64/xen/mm.c	Mon Jun 19 12:52:38 2006 -0600
     1.3 @@ -338,10 +338,42 @@ unsigned long translate_domain_mpaddr(un
     1.4  }
     1.5  
     1.6  //XXX !xxx_present() should be used instread of !xxx_none()?
     1.7 +// __assign_new_domain_page(), assign_new_domain_page() and
     1.8 +// assign_new_domain0_page() are used only when domain creation.
     1.9 +// their accesses aren't racy so that returned pte_t doesn't need
    1.10 +// volatile qualifier
    1.11 +static pte_t*
    1.12 +__lookup_alloc_domain_pte(struct domain* d, unsigned long mpaddr)
    1.13 +{
    1.14 +    struct mm_struct *mm = &d->arch.mm;
    1.15 +    pgd_t *pgd;
    1.16 +    pud_t *pud;
    1.17 +    pmd_t *pmd;
    1.18 +
    1.19 +    BUG_ON(mm->pgd == NULL);
    1.20 +    pgd = pgd_offset(mm, mpaddr);
    1.21 +    if (pgd_none(*pgd)) {
    1.22 +        pgd_populate(mm, pgd, pud_alloc_one(mm,mpaddr));
    1.23 +    }
    1.24 +
    1.25 +    pud = pud_offset(pgd, mpaddr);
    1.26 +    if (pud_none(*pud)) {
    1.27 +        pud_populate(mm, pud, pmd_alloc_one(mm,mpaddr));
    1.28 +    }
    1.29 +
    1.30 +    pmd = pmd_offset(pud, mpaddr);
    1.31 +    if (pmd_none(*pmd)) {
    1.32 +        pmd_populate_kernel(mm, pmd, pte_alloc_one_kernel(mm, mpaddr));
    1.33 +    }
    1.34 +
    1.35 +    return pte_offset_map(pmd, mpaddr);
    1.36 +}
    1.37 +
    1.38 +//XXX !xxx_present() should be used instread of !xxx_none()?
    1.39  // pud, pmd, pte page is zero cleared when they are allocated.
    1.40  // Their area must be visible before population so that
    1.41  // cmpxchg must have release semantics.
    1.42 -static pte_t*
    1.43 +static volatile pte_t*
    1.44  lookup_alloc_domain_pte(struct domain* d, unsigned long mpaddr)
    1.45  {
    1.46      struct mm_struct *mm = &d->arch.mm;
    1.47 @@ -384,11 +416,11 @@ lookup_alloc_domain_pte(struct domain* d
    1.48          }
    1.49      }
    1.50  
    1.51 -    return pte_offset_map(pmd, mpaddr);
    1.52 +    return (volatile pte_t*)pte_offset_map(pmd, mpaddr);
    1.53  }
    1.54  
    1.55  //XXX xxx_none() should be used instread of !xxx_present()?
    1.56 -static pte_t*
    1.57 +static volatile pte_t*
    1.58  lookup_noalloc_domain_pte(struct domain* d, unsigned long mpaddr)
    1.59  {
    1.60      struct mm_struct *mm = &d->arch.mm;
    1.61 @@ -409,11 +441,11 @@ lookup_noalloc_domain_pte(struct domain*
    1.62      if (unlikely(!pmd_present(*pmd)))
    1.63          return NULL;
    1.64  
    1.65 -    return pte_offset_map(pmd, mpaddr);
    1.66 +    return (volatile pte_t*)pte_offset_map(pmd, mpaddr);
    1.67  }
    1.68  
    1.69  #ifdef CONFIG_XEN_IA64_DOM0_VP
    1.70 -static pte_t*
    1.71 +static volatile pte_t*
    1.72  lookup_noalloc_domain_pte_none(struct domain* d, unsigned long mpaddr)
    1.73  {
    1.74      struct mm_struct *mm = &d->arch.mm;
    1.75 @@ -434,13 +466,13 @@ lookup_noalloc_domain_pte_none(struct do
    1.76      if (unlikely(pmd_none(*pmd)))
    1.77          return NULL;
    1.78  
    1.79 -    return pte_offset_map(pmd, mpaddr);
    1.80 +    return (volatile pte_t*)pte_offset_map(pmd, mpaddr);
    1.81  }
    1.82  
    1.83  unsigned long
    1.84  ____lookup_domain_mpa(struct domain *d, unsigned long mpaddr)
    1.85  {
    1.86 -    pte_t *pte;
    1.87 +    volatile pte_t *pte;
    1.88  
    1.89      pte = lookup_noalloc_domain_pte(d, mpaddr);
    1.90      if (pte == NULL)
    1.91 @@ -470,7 +502,7 @@ unsigned long
    1.92  
    1.93  unsigned long lookup_domain_mpa(struct domain *d, unsigned long mpaddr)
    1.94  {
    1.95 -    pte_t *pte;
    1.96 +    volatile pte_t *pte;
    1.97  
    1.98  #ifdef CONFIG_DOMAIN0_CONTIGUOUS
    1.99      if (d == dom0) {
   1.100 @@ -486,9 +518,10 @@ unsigned long lookup_domain_mpa(struct d
   1.101  #endif
   1.102      pte = lookup_noalloc_domain_pte(d, mpaddr);
   1.103      if (pte != NULL) {
   1.104 -        if (pte_present(*pte)) {
   1.105 +        pte_t tmp_pte = *pte;// pte is volatile. copy the value.
   1.106 +        if (pte_present(tmp_pte)) {
   1.107  //printk("lookup_domain_page: found mapping for %lx, pte=%lx\n",mpaddr,pte_val(*pte));
   1.108 -            return pte_val(*pte);
   1.109 +            return pte_val(tmp_pte);
   1.110          } else if (VMX_DOMAIN(d->vcpu[0]))
   1.111              return GPFN_INV_MASK;
   1.112      }
   1.113 @@ -530,7 +563,7 @@ void *domain_mpa_to_imva(struct domain *
   1.114  
   1.115  /* Allocate a new page for domain and map it to the specified metaphysical
   1.116     address.  */
   1.117 -struct page_info *
   1.118 +static struct page_info *
   1.119  __assign_new_domain_page(struct domain *d, unsigned long mpaddr, pte_t* pte)
   1.120  {
   1.121      struct page_info *p = NULL;
   1.122 @@ -595,7 +628,7 @@ assign_new_domain_page(struct domain *d,
   1.123      struct page_info *p = NULL;
   1.124      pte_t *pte;
   1.125  
   1.126 -    pte = lookup_alloc_domain_pte(d, mpaddr);
   1.127 +    pte = __lookup_alloc_domain_pte(d, mpaddr);
   1.128      if (pte_none(*pte))
   1.129          p = __assign_new_domain_page(d, mpaddr, pte);
   1.130  
   1.131 @@ -610,7 +643,7 @@ assign_new_domain0_page(struct domain *d
   1.132      pte_t *pte;
   1.133  
   1.134      BUG_ON(d != dom0);
   1.135 -    pte = lookup_alloc_domain_pte(d, mpaddr);
   1.136 +    pte = __lookup_alloc_domain_pte(d, mpaddr);
   1.137      if (pte_none(*pte)) {
   1.138          struct page_info *p = __assign_new_domain_page(d, mpaddr, pte);
   1.139          if (p == NULL) {
   1.140 @@ -622,21 +655,27 @@ assign_new_domain0_page(struct domain *d
   1.141  
   1.142  /* map a physical address to the specified metaphysical addr */
   1.143  // flags: currently only ASSIGN_readonly
   1.144 +// This is called by assign_domain_mmio_page().
   1.145 +// So accessing to pte is racy.
   1.146  void
   1.147  __assign_domain_page(struct domain *d,
   1.148                       unsigned long mpaddr, unsigned long physaddr,
   1.149                       unsigned long flags)
   1.150  {
   1.151 -    pte_t *pte;
   1.152 +    volatile pte_t *pte;
   1.153 +    pte_t old_pte;
   1.154 +    pte_t new_pte;
   1.155 +    pte_t ret_pte;
   1.156      unsigned long arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX;
   1.157  
   1.158      pte = lookup_alloc_domain_pte(d, mpaddr);
   1.159 -    if (pte_none(*pte)) {
   1.160 -        set_pte_rel(pte,
   1.161 -                    pfn_pte(physaddr >> PAGE_SHIFT,
   1.162 -                            __pgprot(__DIRTY_BITS | _PAGE_PL_2 | arflags)));
   1.163 +
   1.164 +    old_pte = __pte(0);
   1.165 +    new_pte = pfn_pte(physaddr >> PAGE_SHIFT,
   1.166 +                      __pgprot(__DIRTY_BITS | _PAGE_PL_2 | arflags));
   1.167 +    ret_pte = ptep_cmpxchg_rel(&d->arch.mm, mpaddr, pte, old_pte, new_pte);
   1.168 +    if (pte_val(ret_pte) == pte_val(old_pte))
   1.169          smp_mb();
   1.170 -    }
   1.171  }
   1.172  
   1.173  /* get_page() and map a physical address to the specified metaphysical addr */
   1.174 @@ -755,7 +794,7 @@ assign_domain_page_replace(struct domain
   1.175                             unsigned long mfn, unsigned long flags)
   1.176  {
   1.177      struct mm_struct *mm = &d->arch.mm;
   1.178 -    pte_t* pte;
   1.179 +    volatile pte_t* pte;
   1.180      pte_t old_pte;
   1.181      pte_t npte;
   1.182      unsigned long arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX;
   1.183 @@ -798,7 +837,7 @@ assign_domain_page_cmpxchg_rel(struct do
   1.184                                 unsigned long flags)
   1.185  {
   1.186      struct mm_struct *mm = &d->arch.mm;
   1.187 -    pte_t* pte;
   1.188 +    volatile pte_t* pte;
   1.189      unsigned long old_mfn;
   1.190      unsigned long old_arflags;
   1.191      pte_t old_pte;
   1.192 @@ -810,9 +849,14 @@ assign_domain_page_cmpxchg_rel(struct do
   1.193      pte = lookup_alloc_domain_pte(d, mpaddr);
   1.194  
   1.195   again:
   1.196 -    old_arflags = pte_val(*pte) & ~_PAGE_PPN_MASK;//XXX
   1.197 +    old_arflags = pte_val(*pte) & ~_PAGE_PPN_MASK;
   1.198      old_mfn = page_to_mfn(old_page);
   1.199      old_pte = pfn_pte(old_mfn, __pgprot(old_arflags));
   1.200 +    if (!pte_present(old_pte)) {
   1.201 +        DPRINTK("%s: old_pte 0x%lx old_arflags 0x%lx old_mfn 0x%lx\n",
   1.202 +                __func__, pte_val(old_pte), old_arflags, old_mfn);
   1.203 +        return -EINVAL;
   1.204 +    }
   1.205  
   1.206      new_arflags = (flags & ASSIGN_readonly)? _PAGE_AR_R: _PAGE_AR_RWX;
   1.207      new_mfn = page_to_mfn(new_page);
   1.208 @@ -850,7 +894,7 @@ static void
   1.209  zap_domain_page_one(struct domain *d, unsigned long mpaddr)
   1.210  {
   1.211      struct mm_struct *mm = &d->arch.mm;
   1.212 -    pte_t *pte;
   1.213 +    volatile pte_t *pte;
   1.214      pte_t old_pte;
   1.215      unsigned long mfn;
   1.216      struct page_info *page;
   1.217 @@ -970,10 +1014,12 @@ destroy_grant_host_mapping(unsigned long
   1.218                 unsigned long mfn, unsigned int flags)
   1.219  {
   1.220      struct domain* d = current->domain;
   1.221 -    pte_t* pte;
   1.222 +    volatile pte_t* pte;
   1.223 +    unsigned long cur_arflags;
   1.224 +    pte_t cur_pte;
   1.225 +    pte_t new_pte;
   1.226      pte_t old_pte;
   1.227 -    unsigned long old_mfn = INVALID_MFN;
   1.228 -    struct page_info* old_page;
   1.229 +    struct page_info* page;
   1.230  
   1.231      if (flags & (GNTMAP_application_map | GNTMAP_contains_pte)) {
   1.232          DPRINTK("%s: flags 0x%x\n", __func__, flags);
   1.233 @@ -981,21 +1027,42 @@ destroy_grant_host_mapping(unsigned long
   1.234      }
   1.235  
   1.236      pte = lookup_noalloc_domain_pte(d, gpaddr);
   1.237 -    if (pte == NULL || !pte_present(*pte) || pte_pfn(*pte) != mfn)
   1.238 +    if (pte == NULL) {
   1.239 +        DPRINTK("%s: gpaddr 0x%lx mfn 0x%lx\n", __func__, gpaddr, mfn);
   1.240          return GNTST_general_error;
   1.241 +    }
   1.242  
   1.243 -    // update pte
   1.244 -    old_pte = ptep_get_and_clear(&d->arch.mm, gpaddr, pte);
   1.245 -    if (pte_present(old_pte)) {
   1.246 -        old_mfn = pte_pfn(old_pte);
   1.247 -    } else {
   1.248 + again:
   1.249 +    cur_arflags = pte_val(*pte) & ~_PAGE_PPN_MASK;
   1.250 +    cur_pte = pfn_pte(mfn, __pgprot(cur_arflags));
   1.251 +    if (!pte_present(cur_pte)) {
   1.252 +        DPRINTK("%s: gpaddr 0x%lx mfn 0x%lx cur_pte 0x%lx\n",
   1.253 +                __func__, gpaddr, mfn, pte_val(cur_pte));
   1.254          return GNTST_general_error;
   1.255      }
   1.256 -    domain_page_flush(d, gpaddr, old_mfn, INVALID_MFN);
   1.257 +    new_pte = __pte(0);
   1.258  
   1.259 -    old_page = mfn_to_page(old_mfn);
   1.260 -    BUG_ON(page_get_owner(old_page) == d);//try_to_clear_PGC_allocate(d, page) is not needed.
   1.261 -    put_page(old_page);
   1.262 +    old_pte = ptep_cmpxchg_rel(&d->arch.mm, gpaddr, pte, cur_pte, new_pte);
   1.263 +    if (unlikely(!pte_present(old_pte))) {
   1.264 +        DPRINTK("%s: gpaddr 0x%lx mfn 0x%lx cur_pte 0x%lx old_pte 0x%lx\n",
   1.265 +                __func__, gpaddr, mfn, pte_val(cur_pte), pte_val(old_pte));
   1.266 +        return GNTST_general_error;
   1.267 +    }
   1.268 +    if (unlikely(pte_val(cur_pte) != pte_val(old_pte))) {
   1.269 +        if (pte_pfn(old_pte) == mfn) {
   1.270 +            goto again;
   1.271 +        }
   1.272 +        DPRINTK("%s gpaddr 0x%lx mfn 0x%lx cur_pte 0x%lx old_pte 0x%lx\n",
   1.273 +                __func__, gpaddr, mfn, pte_val(cur_pte), pte_val(old_pte));
   1.274 +        return GNTST_general_error;
   1.275 +    }
   1.276 +    BUG_ON(pte_pfn(old_pte) != mfn);
   1.277 +
   1.278 +    domain_page_flush(d, gpaddr, mfn, INVALID_MFN);
   1.279 +
   1.280 +    page = mfn_to_page(mfn);
   1.281 +    BUG_ON(page_get_owner(page) == d);//try_to_clear_PGC_allocate(d, page) is not needed.
   1.282 +    put_page(page);
   1.283  
   1.284      return GNTST_okay;
   1.285  }
   1.286 @@ -1141,7 +1208,7 @@ domain_page_flush(struct domain* d, unsi
   1.287  int
   1.288  domain_page_mapped(struct domain* d, unsigned long mpaddr)
   1.289  {
   1.290 -    pte_t * pte;
   1.291 +    volatile pte_t * pte;
   1.292  
   1.293      pte = lookup_noalloc_domain_pte(d, mpaddr);
   1.294      if(pte != NULL && !pte_none(*pte))
     2.1 --- a/xen/include/asm-ia64/linux-xen/asm/pgtable.h	Mon Jun 19 12:13:11 2006 -0600
     2.2 +++ b/xen/include/asm-ia64/linux-xen/asm/pgtable.h	Mon Jun 19 12:52:38 2006 -0600
     2.3 @@ -210,7 +210,7 @@ ia64_phys_addr_valid (unsigned long addr
     2.4  #define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval)
     2.5  #ifdef XEN
     2.6  static inline void
     2.7 -set_pte_rel(pte_t* ptep, pte_t pteval)
     2.8 +set_pte_rel(volatile pte_t* ptep, pte_t pteval)
     2.9  {
    2.10  #if CONFIG_SMP
    2.11  	asm volatile ("st8.rel [%0]=%1" ::
    2.12 @@ -402,8 +402,14 @@ ptep_test_and_clear_dirty (struct vm_are
    2.13  }
    2.14  #endif
    2.15  
    2.16 +#ifdef XEN
    2.17 +static inline pte_t
    2.18 +ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
    2.19 +		   volatile pte_t *ptep)
    2.20 +#else
    2.21  static inline pte_t
    2.22  ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
    2.23 +#endif
    2.24  {
    2.25  #ifdef CONFIG_SMP
    2.26  	return __pte(xchg((long *) ptep, 0));
    2.27 @@ -416,7 +422,8 @@ ptep_get_and_clear(struct mm_struct *mm,
    2.28  
    2.29  #ifdef XEN
    2.30  static inline pte_t
    2.31 -ptep_xchg(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t npte)
    2.32 +ptep_xchg(struct mm_struct *mm, unsigned long addr,
    2.33 +	  volatile pte_t *ptep, pte_t npte)
    2.34  {
    2.35  #ifdef CONFIG_SMP
    2.36  	return __pte(xchg((long *) ptep, pte_val(npte)));
    2.37 @@ -428,8 +435,8 @@ ptep_xchg(struct mm_struct *mm, unsigned
    2.38  }
    2.39  
    2.40  static inline pte_t
    2.41 -ptep_cmpxchg_rel(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
    2.42 -		 pte_t old_pte, pte_t new_pte)
    2.43 +ptep_cmpxchg_rel(struct mm_struct *mm, unsigned long addr,
    2.44 +		 volatile pte_t *ptep, pte_t old_pte, pte_t new_pte)
    2.45  {
    2.46  #ifdef CONFIG_SMP
    2.47  	return __pte(cmpxchg_rel(&pte_val(*ptep),