ia64/xen-unstable

changeset 12664:439051df45f3

[IA64] fix p2m traversing race

Fixed by adding volatile to pgd_t, pud_t, pmd_t where necessary.
The Xen/IA64 p2m table is lockless so that acquire semantics is
necessary when traversing the p2m in order to prevent memory-reodering
the following memory load.

Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
author awilliam@xenbuild.aw
date Thu Nov 30 15:57:42 2006 -0700 (2006-11-30)
parents 4fae646d60da
children 62686b2e13ea
files xen/arch/ia64/xen/mm.c xen/include/asm-ia64/domain.h xen/include/asm-ia64/linux-xen/asm-generic/pgtable-nopud.h xen/include/asm-ia64/linux-xen/asm/pgalloc.h xen/include/asm-ia64/linux-xen/asm/pgtable.h
line diff
     1.1 --- a/xen/arch/ia64/xen/mm.c	Thu Nov 30 15:57:37 2006 -0700
     1.2 +++ b/xen/arch/ia64/xen/mm.c	Thu Nov 30 15:57:42 2006 -0700
     1.3 @@ -249,7 +249,7 @@ try_to_clear_PGC_allocate(struct domain*
     1.4  }
     1.5  
     1.6  static void
     1.7 -mm_teardown_pte(struct domain* d, pte_t* pte, unsigned long offset)
     1.8 +mm_teardown_pte(struct domain* d, volatile pte_t* pte, unsigned long offset)
     1.9  {
    1.10      pte_t old_pte;
    1.11      unsigned long mfn;
    1.12 @@ -286,39 +286,39 @@ mm_teardown_pte(struct domain* d, pte_t*
    1.13  }
    1.14  
    1.15  static void
    1.16 -mm_teardown_pmd(struct domain* d, pmd_t* pmd, unsigned long offset)
    1.17 +mm_teardown_pmd(struct domain* d, volatile pmd_t* pmd, unsigned long offset)
    1.18  {
    1.19      unsigned long i;
    1.20 -    pte_t* pte = pte_offset_map(pmd, offset);
    1.21 +    volatile pte_t* pte = pte_offset_map(pmd, offset);
    1.22  
    1.23      for (i = 0; i < PTRS_PER_PTE; i++, pte++) {
    1.24 -        if (!pte_present(*pte))
    1.25 +        if (!pte_present(*pte)) // acquire semantics
    1.26              continue;
    1.27          mm_teardown_pte(d, pte, offset + (i << PAGE_SHIFT));
    1.28      }
    1.29  }
    1.30  
    1.31  static void
    1.32 -mm_teardown_pud(struct domain* d, pud_t *pud, unsigned long offset)
    1.33 +mm_teardown_pud(struct domain* d, volatile pud_t *pud, unsigned long offset)
    1.34  {
    1.35      unsigned long i;
    1.36 -    pmd_t *pmd = pmd_offset(pud, offset);
    1.37 +    volatile pmd_t *pmd = pmd_offset(pud, offset);
    1.38  
    1.39      for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
    1.40 -        if (!pmd_present(*pmd))
    1.41 +        if (!pmd_present(*pmd)) // acquire semantics
    1.42              continue;
    1.43          mm_teardown_pmd(d, pmd, offset + (i << PMD_SHIFT));
    1.44      }
    1.45  }
    1.46  
    1.47  static void
    1.48 -mm_teardown_pgd(struct domain* d, pgd_t *pgd, unsigned long offset)
    1.49 +mm_teardown_pgd(struct domain* d, volatile pgd_t *pgd, unsigned long offset)
    1.50  {
    1.51      unsigned long i;
    1.52 -    pud_t *pud = pud_offset(pgd, offset);
    1.53 +    volatile pud_t *pud = pud_offset(pgd, offset);
    1.54  
    1.55      for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
    1.56 -        if (!pud_present(*pud))
    1.57 +        if (!pud_present(*pud)) // acquire semantics
    1.58              continue;
    1.59          mm_teardown_pud(d, pud, offset + (i << PUD_SHIFT));
    1.60      }
    1.61 @@ -329,30 +329,32 @@ mm_teardown(struct domain* d)
    1.62  {
    1.63      struct mm_struct* mm = &d->arch.mm;
    1.64      unsigned long i;
    1.65 -    pgd_t* pgd;
    1.66 +    volatile pgd_t* pgd;
    1.67  
    1.68      if (mm->pgd == NULL)
    1.69          return;
    1.70  
    1.71      pgd = pgd_offset(mm, 0);
    1.72      for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
    1.73 -        if (!pgd_present(*pgd))
    1.74 +        if (!pgd_present(*pgd)) // acquire semantics
    1.75              continue;
    1.76          mm_teardown_pgd(d, pgd, i << PGDIR_SHIFT);
    1.77      }
    1.78  }
    1.79  
    1.80  static void
    1.81 -mm_p2m_teardown_pmd(struct domain* d, pmd_t* pmd, unsigned long offset)
    1.82 +mm_p2m_teardown_pmd(struct domain* d, volatile pmd_t* pmd,
    1.83 +                    unsigned long offset)
    1.84  {
    1.85      pte_free_kernel(pte_offset_map(pmd, offset));
    1.86  }
    1.87  
    1.88  static void
    1.89 -mm_p2m_teardown_pud(struct domain* d, pud_t *pud, unsigned long offset)
    1.90 +mm_p2m_teardown_pud(struct domain* d, volatile pud_t *pud,
    1.91 +                    unsigned long offset)
    1.92  {
    1.93      unsigned long i;
    1.94 -    pmd_t *pmd = pmd_offset(pud, offset);
    1.95 +    volatile pmd_t *pmd = pmd_offset(pud, offset);
    1.96  
    1.97      for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
    1.98          if (!pmd_present(*pmd))
    1.99 @@ -363,10 +365,11 @@ mm_p2m_teardown_pud(struct domain* d, pu
   1.100  }
   1.101  
   1.102  static void
   1.103 -mm_p2m_teardown_pgd(struct domain* d, pgd_t *pgd, unsigned long offset)
   1.104 +mm_p2m_teardown_pgd(struct domain* d, volatile pgd_t *pgd,
   1.105 +                    unsigned long offset)
   1.106  {
   1.107      unsigned long i;
   1.108 -    pud_t *pud = pud_offset(pgd, offset);
   1.109 +    volatile pud_t *pud = pud_offset(pgd, offset);
   1.110  
   1.111      for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
   1.112          if (!pud_present(*pud))
   1.113 @@ -381,7 +384,7 @@ mm_p2m_teardown(struct domain* d)
   1.114  {
   1.115      struct mm_struct* mm = &d->arch.mm;
   1.116      unsigned long i;
   1.117 -    pgd_t* pgd;
   1.118 +    volatile pgd_t* pgd;
   1.119  
   1.120      BUG_ON(mm->pgd == NULL);
   1.121      pgd = pgd_offset(mm, 0);
   1.122 @@ -565,38 +568,6 @@ unsigned long translate_domain_mpaddr(un
   1.123  }
   1.124  
   1.125  //XXX !xxx_present() should be used instread of !xxx_none()?
   1.126 -// __assign_new_domain_page(), assign_new_domain_page() and
   1.127 -// assign_new_domain0_page() are used only when domain creation.
   1.128 -// their accesses aren't racy so that returned pte_t doesn't need
   1.129 -// volatile qualifier
   1.130 -static pte_t*
   1.131 -__lookup_alloc_domain_pte(struct domain* d, unsigned long mpaddr)
   1.132 -{
   1.133 -    struct mm_struct *mm = &d->arch.mm;
   1.134 -    pgd_t *pgd;
   1.135 -    pud_t *pud;
   1.136 -    pmd_t *pmd;
   1.137 -
   1.138 -    BUG_ON(mm->pgd == NULL);
   1.139 -    pgd = pgd_offset(mm, mpaddr);
   1.140 -    if (pgd_none(*pgd)) {
   1.141 -        pgd_populate(mm, pgd, pud_alloc_one(mm,mpaddr));
   1.142 -    }
   1.143 -
   1.144 -    pud = pud_offset(pgd, mpaddr);
   1.145 -    if (pud_none(*pud)) {
   1.146 -        pud_populate(mm, pud, pmd_alloc_one(mm,mpaddr));
   1.147 -    }
   1.148 -
   1.149 -    pmd = pmd_offset(pud, mpaddr);
   1.150 -    if (pmd_none(*pmd)) {
   1.151 -        pmd_populate_kernel(mm, pmd, pte_alloc_one_kernel(mm, mpaddr));
   1.152 -    }
   1.153 -
   1.154 -    return pte_offset_map(pmd, mpaddr);
   1.155 -}
   1.156 -
   1.157 -//XXX !xxx_present() should be used instread of !xxx_none()?
   1.158  // pud, pmd, pte page is zero cleared when they are allocated.
   1.159  // Their area must be visible before population so that
   1.160  // cmpxchg must have release semantics.
   1.161 @@ -604,15 +575,15 @@ static volatile pte_t*
   1.162  lookup_alloc_domain_pte(struct domain* d, unsigned long mpaddr)
   1.163  {
   1.164      struct mm_struct *mm = &d->arch.mm;
   1.165 -    pgd_t *pgd;
   1.166 -    pud_t *pud;
   1.167 -    pmd_t *pmd;
   1.168 +    volatile pgd_t *pgd;
   1.169 +    volatile pud_t *pud;
   1.170 +    volatile pmd_t *pmd;
   1.171  
   1.172      BUG_ON(mm->pgd == NULL);
   1.173  
   1.174      pgd = pgd_offset(mm, mpaddr);
   1.175   again_pgd:
   1.176 -    if (unlikely(pgd_none(*pgd))) {
   1.177 +    if (unlikely(pgd_none(*pgd))) { // acquire semantics
   1.178          pud_t *old_pud = NULL;
   1.179          pud = pud_alloc_one(mm, mpaddr);
   1.180          if (unlikely(!pgd_cmpxchg_rel(mm, pgd, old_pud, pud))) {
   1.181 @@ -623,7 +594,7 @@ lookup_alloc_domain_pte(struct domain* d
   1.182  
   1.183      pud = pud_offset(pgd, mpaddr);
   1.184   again_pud:
   1.185 -    if (unlikely(pud_none(*pud))) {
   1.186 +    if (unlikely(pud_none(*pud))) { // acquire semantics
   1.187          pmd_t* old_pmd = NULL;
   1.188          pmd = pmd_alloc_one(mm, mpaddr);
   1.189          if (unlikely(!pud_cmpxchg_rel(mm, pud, old_pmd, pmd))) {
   1.190 @@ -634,7 +605,7 @@ lookup_alloc_domain_pte(struct domain* d
   1.191  
   1.192      pmd = pmd_offset(pud, mpaddr);
   1.193   again_pmd:
   1.194 -    if (unlikely(pmd_none(*pmd))) {
   1.195 +    if (unlikely(pmd_none(*pmd))) { // acquire semantics
   1.196          pte_t* old_pte = NULL;
   1.197          pte_t* pte = pte_alloc_one_kernel(mm, mpaddr);
   1.198          if (unlikely(!pmd_cmpxchg_kernel_rel(mm, pmd, old_pte, pte))) {
   1.199 @@ -643,7 +614,7 @@ lookup_alloc_domain_pte(struct domain* d
   1.200          }
   1.201      }
   1.202  
   1.203 -    return (volatile pte_t*)pte_offset_map(pmd, mpaddr);
   1.204 +    return pte_offset_map(pmd, mpaddr);
   1.205  }
   1.206  
   1.207  //XXX xxx_none() should be used instread of !xxx_present()?
   1.208 @@ -651,48 +622,48 @@ volatile pte_t*
   1.209  lookup_noalloc_domain_pte(struct domain* d, unsigned long mpaddr)
   1.210  {
   1.211      struct mm_struct *mm = &d->arch.mm;
   1.212 -    pgd_t *pgd;
   1.213 -    pud_t *pud;
   1.214 -    pmd_t *pmd;
   1.215 +    volatile pgd_t *pgd;
   1.216 +    volatile pud_t *pud;
   1.217 +    volatile pmd_t *pmd;
   1.218  
   1.219      BUG_ON(mm->pgd == NULL);
   1.220      pgd = pgd_offset(mm, mpaddr);
   1.221 -    if (unlikely(!pgd_present(*pgd)))
   1.222 +    if (unlikely(!pgd_present(*pgd))) // acquire semantics
   1.223          return NULL;
   1.224  
   1.225      pud = pud_offset(pgd, mpaddr);
   1.226 -    if (unlikely(!pud_present(*pud)))
   1.227 +    if (unlikely(!pud_present(*pud))) // acquire semantics
   1.228          return NULL;
   1.229  
   1.230      pmd = pmd_offset(pud, mpaddr);
   1.231 -    if (unlikely(!pmd_present(*pmd)))
   1.232 +    if (unlikely(!pmd_present(*pmd))) // acquire semantics
   1.233          return NULL;
   1.234  
   1.235 -    return (volatile pte_t*)pte_offset_map(pmd, mpaddr);
   1.236 +    return pte_offset_map(pmd, mpaddr);
   1.237  }
   1.238  
   1.239  static volatile pte_t*
   1.240  lookup_noalloc_domain_pte_none(struct domain* d, unsigned long mpaddr)
   1.241  {
   1.242      struct mm_struct *mm = &d->arch.mm;
   1.243 -    pgd_t *pgd;
   1.244 -    pud_t *pud;
   1.245 -    pmd_t *pmd;
   1.246 +    volatile pgd_t *pgd;
   1.247 +    volatile pud_t *pud;
   1.248 +    volatile pmd_t *pmd;
   1.249  
   1.250      BUG_ON(mm->pgd == NULL);
   1.251      pgd = pgd_offset(mm, mpaddr);
   1.252 -    if (unlikely(pgd_none(*pgd)))
   1.253 +    if (unlikely(pgd_none(*pgd))) // acquire semantics
   1.254          return NULL;
   1.255  
   1.256      pud = pud_offset(pgd, mpaddr);
   1.257 -    if (unlikely(pud_none(*pud)))
   1.258 +    if (unlikely(pud_none(*pud))) // acquire semantics
   1.259          return NULL;
   1.260  
   1.261      pmd = pmd_offset(pud, mpaddr);
   1.262 -    if (unlikely(pmd_none(*pmd)))
   1.263 +    if (unlikely(pmd_none(*pmd))) // acquire semantics
   1.264          return NULL;
   1.265  
   1.266 -    return (volatile pte_t*)pte_offset_map(pmd, mpaddr);
   1.267 +    return pte_offset_map(pmd, mpaddr);
   1.268  }
   1.269  
   1.270  unsigned long
   1.271 @@ -783,7 +754,8 @@ xencomm_paddr_to_maddr(unsigned long pad
   1.272  /* Allocate a new page for domain and map it to the specified metaphysical
   1.273     address.  */
   1.274  static struct page_info *
   1.275 -__assign_new_domain_page(struct domain *d, unsigned long mpaddr, pte_t* pte)
   1.276 +__assign_new_domain_page(struct domain *d, unsigned long mpaddr,
   1.277 +                         volatile pte_t* pte)
   1.278  {
   1.279      struct page_info *p;
   1.280      unsigned long maddr;
   1.281 @@ -825,7 +797,7 @@ static struct page_info *
   1.282  struct page_info *
   1.283  assign_new_domain_page(struct domain *d, unsigned long mpaddr)
   1.284  {
   1.285 -    pte_t *pte = __lookup_alloc_domain_pte(d, mpaddr);
   1.286 +    volatile pte_t *pte = lookup_alloc_domain_pte(d, mpaddr);
   1.287  
   1.288      if (!pte_none(*pte))
   1.289          return NULL;
   1.290 @@ -836,10 +808,10 @@ assign_new_domain_page(struct domain *d,
   1.291  void
   1.292  assign_new_domain0_page(struct domain *d, unsigned long mpaddr)
   1.293  {
   1.294 -    pte_t *pte;
   1.295 +    volatile pte_t *pte;
   1.296  
   1.297      BUG_ON(d != dom0);
   1.298 -    pte = __lookup_alloc_domain_pte(d, mpaddr);
   1.299 +    pte = lookup_alloc_domain_pte(d, mpaddr);
   1.300      if (pte_none(*pte)) {
   1.301          struct page_info *p = __assign_new_domain_page(d, mpaddr, pte);
   1.302          if (p == NULL) {
   1.303 @@ -1816,7 +1788,7 @@ domain_page_mapped(struct domain* d, uns
   1.304  void domain_cache_flush (struct domain *d, int sync_only)
   1.305  {
   1.306      struct mm_struct *mm = &d->arch.mm;
   1.307 -    pgd_t *pgd = mm->pgd;
   1.308 +    volatile pgd_t *pgd = mm->pgd;
   1.309      unsigned long maddr;
   1.310      int i,j,k, l;
   1.311      int nbr_page = 0;
   1.312 @@ -1829,22 +1801,22 @@ void domain_cache_flush (struct domain *
   1.313          flush_func = &flush_dcache_range;
   1.314  
   1.315      for (i = 0; i < PTRS_PER_PGD; pgd++, i++) {
   1.316 -        pud_t *pud;
   1.317 -        if (!pgd_present(*pgd))
   1.318 +        volatile pud_t *pud;
   1.319 +        if (!pgd_present(*pgd)) // acquire semantics
   1.320              continue;
   1.321          pud = pud_offset(pgd, 0);
   1.322          for (j = 0; j < PTRS_PER_PUD; pud++, j++) {
   1.323 -            pmd_t *pmd;
   1.324 -            if (!pud_present(*pud))
   1.325 +            volatile pmd_t *pmd;
   1.326 +            if (!pud_present(*pud)) // acquire semantics
   1.327                  continue;
   1.328              pmd = pmd_offset(pud, 0);
   1.329              for (k = 0; k < PTRS_PER_PMD; pmd++, k++) {
   1.330 -                pte_t *pte;
   1.331 -                if (!pmd_present(*pmd))
   1.332 +                volatile pte_t *pte;
   1.333 +                if (!pmd_present(*pmd)) // acquire semantics
   1.334                      continue;
   1.335                  pte = pte_offset_map(pmd, 0);
   1.336                  for (l = 0; l < PTRS_PER_PTE; pte++, l++) {
   1.337 -                    if (!pte_present(*pte))
   1.338 +                    if (!pte_present(*pte)) // acquire semantics
   1.339                          continue;
   1.340                      /* Convert PTE to maddr.  */
   1.341                      maddr = __va_ul (pte_val(*pte)
     2.1 --- a/xen/include/asm-ia64/domain.h	Thu Nov 30 15:57:37 2006 -0700
     2.2 +++ b/xen/include/asm-ia64/domain.h	Thu Nov 30 15:57:42 2006 -0700
     2.3 @@ -42,7 +42,7 @@ extern void panic_domain(struct pt_regs 
     2.4       __attribute__ ((noreturn, format (printf, 2, 3)));
     2.5  
     2.6  struct mm_struct {
     2.7 -	pgd_t * pgd;
     2.8 +	volatile pgd_t * pgd;
     2.9      //	atomic_t mm_users;			/* How many users with user space? */
    2.10  };
    2.11  
     3.1 --- a/xen/include/asm-ia64/linux-xen/asm-generic/pgtable-nopud.h	Thu Nov 30 15:57:37 2006 -0700
     3.2 +++ b/xen/include/asm-ia64/linux-xen/asm-generic/pgtable-nopud.h	Thu Nov 30 15:57:42 2006 -0700
     3.3 @@ -35,10 +35,18 @@ static inline void pgd_clear(pgd_t *pgd)
     3.4   */
     3.5  #define set_pgd(pgdptr, pgdval)			set_pud((pud_t *)(pgdptr), (pud_t) { pgdval })
     3.6  
     3.7 +#ifndef XEN
     3.8  static inline pud_t * pud_offset(pgd_t * pgd, unsigned long address)
     3.9  {
    3.10  	return (pud_t *)pgd;
    3.11  }
    3.12 +#else
    3.13 +static inline volatile pud_t *
    3.14 +pud_offset(volatile pgd_t * pgd, unsigned long address)
    3.15 +{
    3.16 +	return (volatile pud_t *)pgd;
    3.17 +}
    3.18 +#endif
    3.19  
    3.20  #define pud_val(x)				(pgd_val((x).pgd))
    3.21  #define __pud(x)				((pud_t) { __pgd(x) } )
     4.1 --- a/xen/include/asm-ia64/linux-xen/asm/pgalloc.h	Thu Nov 30 15:57:37 2006 -0700
     4.2 +++ b/xen/include/asm-ia64/linux-xen/asm/pgalloc.h	Thu Nov 30 15:57:42 2006 -0700
     4.3 @@ -92,10 +92,17 @@ static inline pgd_t *pgd_alloc(struct mm
     4.4  	return pgtable_quicklist_alloc();
     4.5  }
     4.6  
     4.7 +#ifndef XEN
     4.8  static inline void pgd_free(pgd_t * pgd)
     4.9  {
    4.10  	pgtable_quicklist_free(pgd);
    4.11  }
    4.12 +#else
    4.13 +static inline void pgd_free(volatile pgd_t * pgd)
    4.14 +{
    4.15 +	pgtable_quicklist_free((void*)pgd);
    4.16 +}
    4.17 +#endif
    4.18  
    4.19  static inline void
    4.20  pud_populate(struct mm_struct *mm, pud_t * pud_entry, pmd_t * pmd)
    4.21 @@ -105,8 +112,8 @@ pud_populate(struct mm_struct *mm, pud_t
    4.22  
    4.23  #ifdef XEN
    4.24  static inline int
    4.25 -pud_cmpxchg_rel(struct mm_struct *mm, pud_t * pud_entry,
    4.26 -		pmd_t * old_pmd, pmd_t * new_pmd)
    4.27 +pud_cmpxchg_rel(struct mm_struct *mm, volatile pud_t * pud_entry,
    4.28 +                pmd_t * old_pmd, volatile pmd_t * new_pmd)
    4.29  {
    4.30  #ifdef CONFIG_SMP
    4.31  	unsigned long r;
    4.32 @@ -127,10 +134,17 @@ static inline pmd_t *pmd_alloc_one(struc
    4.33  	return pgtable_quicklist_alloc();
    4.34  }
    4.35  
    4.36 +#ifndef XEN
    4.37  static inline void pmd_free(pmd_t * pmd)
    4.38  {
    4.39  	pgtable_quicklist_free(pmd);
    4.40  }
    4.41 +#else
    4.42 +static inline void pmd_free(volatile pmd_t * pmd)
    4.43 +{
    4.44 +	pgtable_quicklist_free((void*)pmd);
    4.45 +}
    4.46 +#endif
    4.47  
    4.48  #define __pmd_free_tlb(tlb, pmd)	pmd_free(pmd)
    4.49  
    4.50 @@ -150,7 +164,7 @@ pmd_populate_kernel(struct mm_struct *mm
    4.51  
    4.52  #ifdef XEN
    4.53  static inline int
    4.54 -pmd_cmpxchg_kernel_rel(struct mm_struct *mm, pmd_t * pmd_entry,
    4.55 +pmd_cmpxchg_kernel_rel(struct mm_struct *mm, volatile pmd_t * pmd_entry,
    4.56  		       pte_t * old_pte, pte_t * new_pte)
    4.57  {
    4.58  #ifdef CONFIG_SMP
    4.59 @@ -186,12 +200,17 @@ static inline void pte_free(struct page 
    4.60  {
    4.61  	pgtable_quicklist_free(page_address(pte));
    4.62  }
    4.63 -#endif
    4.64  
    4.65  static inline void pte_free_kernel(pte_t * pte)
    4.66  {
    4.67  	pgtable_quicklist_free(pte);
    4.68  }
    4.69 +#else
    4.70 +static inline void pte_free_kernel(volatile pte_t * pte)
    4.71 +{
    4.72 +	pgtable_quicklist_free((void*)pte);
    4.73 +}
    4.74 +#endif
    4.75  
    4.76  #ifndef XEN
    4.77  #define __pte_free_tlb(tlb, pte)	pte_free(pte)
     5.1 --- a/xen/include/asm-ia64/linux-xen/asm/pgtable.h	Thu Nov 30 15:57:37 2006 -0700
     5.2 +++ b/xen/include/asm-ia64/linux-xen/asm/pgtable.h	Thu Nov 30 15:57:42 2006 -0700
     5.3 @@ -380,7 +380,11 @@ pgd_index (unsigned long address)
     5.4  
     5.5  /* The offset in the 1-level directory is given by the 3 region bits
     5.6     (61..63) and the level-1 bits.  */
     5.7 +#ifndef XEN
     5.8  static inline pgd_t*
     5.9 +#else
    5.10 +static inline volatile pgd_t*
    5.11 +#endif
    5.12  pgd_offset (struct mm_struct *mm, unsigned long address)
    5.13  {
    5.14  	return mm->pgd + pgd_index(address);
    5.15 @@ -397,15 +401,24 @@ pgd_offset (struct mm_struct *mm, unsign
    5.16  #define pgd_offset_gate(mm, addr)	pgd_offset_k(addr)
    5.17  
    5.18  /* Find an entry in the second-level page table.. */
    5.19 +#ifndef XEN
    5.20  #define pmd_offset(dir,addr) \
    5.21  	((pmd_t *) pud_page(*(dir)) + (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)))
    5.22 +#else
    5.23 +#define pmd_offset(dir,addr) \
    5.24 +	((volatile pmd_t *) pud_page(*(dir)) + (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)))
    5.25 +#endif
    5.26  
    5.27  /*
    5.28   * Find an entry in the third-level page table.  This looks more complicated than it
    5.29   * should be because some platforms place page tables in high memory.
    5.30   */
    5.31  #define pte_index(addr)	 	(((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
    5.32 +#ifndef XEN
    5.33  #define pte_offset_kernel(dir,addr)	((pte_t *) pmd_page_kernel(*(dir)) + pte_index(addr))
    5.34 +#else
    5.35 +#define pte_offset_kernel(dir,addr)	((volatile pte_t *) pmd_page_kernel(*(dir)) + pte_index(addr))
    5.36 +#endif
    5.37  #define pte_offset_map(dir,addr)	pte_offset_kernel(dir, addr)
    5.38  #define pte_offset_map_nested(dir,addr)	pte_offset_map(dir, addr)
    5.39  #define pte_unmap(pte)			do { } while (0)