1.1 --- a/xen/arch/x86/mm.c	Mon Jun 19 14:03:04 2006 +0100
     1.2 +++ b/xen/arch/x86/mm.c	Mon Jun 19 14:44:34 2006 +0100
     1.3 @@ -108,14 +108,23 @@
     1.4  #include <public/memory.h>
     1.5  
     1.6  #ifdef VERBOSE
     1.7 -#define MEM_LOG(_f, _a...)                           \
     1.8 -  printk("DOM%u: (file=mm.c, line=%d) " _f "\n", \
     1.9 +#define MEM_LOG(_f, _a...)                                  \
    1.10 +  printk("DOM%u: (file=mm.c, line=%d) " _f "\n",            \
    1.11           current->domain->domain_id , __LINE__ , ## _a )
    1.12  #else
    1.13  #define MEM_LOG(_f, _a...) ((void)0)
    1.14  #endif
    1.15  
    1.16  /*
    1.17 + * PTE updates can be done with ordinary writes except:
    1.18 + *  1. Debug builds get extra checking by using CMPXCHG[8B].
    1.19 + *  2. PAE builds perform an atomic 8-byte store with CMPXCHG8B.
    1.20 + */
    1.21 +#if !defined(NDEBUG) || defined(CONFIG_X86_PAE)
    1.22 +#define PTE_UPDATE_WITH_CMPXCHG
    1.23 +#endif
    1.24 +
    1.25 +/*
    1.26   * Both do_mmuext_op() and do_mmu_update():
    1.27   * We steal the m.s.b. of the @count parameter to indicate whether this
    1.28   * invocation of do_mmu_update() is resuming a previously preempted call.
    1.29 @@ -1173,16 +1182,27 @@ static inline int update_l1e(l1_pgentry_
    1.30      intpte_t o = l1e_get_intpte(ol1e);
    1.31      intpte_t n = l1e_get_intpte(nl1e);
    1.32  
    1.33 -    if ( unlikely(cmpxchg_user(pl1e, o, n) != 0) ||
    1.34 -         unlikely(o != l1e_get_intpte(ol1e)) )
    1.35 +    for ( ; ; )
    1.36      {
    1.37 -        MEM_LOG("Failed to update %" PRIpte " -> %" PRIpte
    1.38 -                ": saw %" PRIpte,
    1.39 -                l1e_get_intpte(ol1e),
    1.40 -                l1e_get_intpte(nl1e),
    1.41 -                o);
    1.42 -        return 0;
    1.43 +        if ( unlikely(cmpxchg_user(pl1e, o, n) != 0) )
    1.44 +        {
    1.45 +            MEM_LOG("Failed to update %" PRIpte " -> %" PRIpte
    1.46 +                    ": saw %" PRIpte,
    1.47 +                    l1e_get_intpte(ol1e),
    1.48 +                    l1e_get_intpte(nl1e),
    1.49 +                    o);
    1.50 +            return 0;
    1.51 +        }
    1.52 +
    1.53 +        if ( o == l1e_get_intpte(ol1e) )
    1.54 +            break;
    1.55 +
    1.56 +        /* Allowed to change in Accessed/Dirty flags only. */
    1.57 +        BUG_ON((o ^ l1e_get_intpte(ol1e)) &
    1.58 +               ~(int)(_PAGE_ACCESSED|_PAGE_DIRTY));
    1.59 +        ol1e = l1e_from_intpte(o);
    1.60      }
    1.61 +
    1.62      return 1;
    1.63  #endif
    1.64  }
    1.65 @@ -1235,17 +1255,20 @@ static int mod_l1_entry(l1_pgentry_t *pl
    1.66  #ifndef PTE_UPDATE_WITH_CMPXCHG
    1.67  #define UPDATE_ENTRY(_t,_p,_o,_n) ({ (*(_p) = (_n)); 1; })
    1.68  #else
    1.69 -#define UPDATE_ENTRY(_t,_p,_o,_n) ({                                    \
    1.70 -    intpte_t __o = cmpxchg((intpte_t *)(_p),                            \
    1.71 -                           _t ## e_get_intpte(_o),                      \
    1.72 -                           _t ## e_get_intpte(_n));                     \
    1.73 -    if ( __o != _t ## e_get_intpte(_o) )                                \
    1.74 -        MEM_LOG("Failed to update %" PRIpte " -> %" PRIpte              \
    1.75 -                ": saw %" PRIpte "",                                    \
    1.76 -                (_t ## e_get_intpte(_o)),                               \
    1.77 -                (_t ## e_get_intpte(_n)),                               \
    1.78 -                (__o));                                                 \
    1.79 -    (__o == _t ## e_get_intpte(_o)); })
    1.80 +#define UPDATE_ENTRY(_t,_p,_o,_n) ({                            \
    1.81 +    for ( ; ; )                                                 \
    1.82 +    {                                                           \
    1.83 +        intpte_t __o = cmpxchg((intpte_t *)(_p),                \
    1.84 +                               _t ## e_get_intpte(_o),          \
    1.85 +                               _t ## e_get_intpte(_n));         \
    1.86 +        if ( __o == _t ## e_get_intpte(_o) )                    \
    1.87 +            break;                                              \
    1.88 +        /* Allowed to change in Accessed/Dirty flags only. */   \
    1.89 +        BUG_ON((__o ^ _t ## e_get_intpte(_o)) &                 \
    1.90 +               ~(int)(_PAGE_ACCESSED|_PAGE_DIRTY));             \
    1.91 +        _o = _t ## e_from_intpte(__o);                          \
    1.92 +    }                                                           \
    1.93 +    1; })
    1.94  #endif
    1.95  
    1.96  /* Update the L2 entry at pl2e to new value nl2e. pl2e is within frame pfn. */
    1.97 @@ -2494,7 +2517,7 @@ static int destroy_grant_pte_mapping(
    1.98      }
    1.99  
   1.100      /* Delete pagetable entry. */
   1.101 -    if ( unlikely(__put_user(0, (intpte_t *)va)))
   1.102 +    if ( unlikely(!update_l1e((l1_pgentry_t *)va, ol1e, l1e_empty())) )
   1.103      {
   1.104          MEM_LOG("Cannot delete PTE entry at %p", va);
   1.105          put_page_type(page);
   1.106 @@ -2574,7 +2597,7 @@ static int destroy_grant_va_mapping(
   1.107      }
   1.108  
   1.109      /* Delete pagetable entry. */
   1.110 -    if ( unlikely(__put_user(0, &pl1e->l1)) )
   1.111 +    if ( unlikely(!update_l1e(pl1e, ol1e, l1e_empty())) )
   1.112      {
   1.113          MEM_LOG("Cannot delete PTE entry at %p", (unsigned long *)pl1e);
   1.114          return GNTST_general_error;
   1.115 @@ -3424,8 +3447,9 @@ static int ptwr_emulated_update(
   1.116      }
   1.117      else
   1.118      {
   1.119 -        ol1e  = *pl1e;
   1.120 -        *pl1e = nl1e;
   1.121 +        ol1e = *pl1e;
   1.122 +        if ( !update_l1e(pl1e, ol1e, nl1e) )
   1.123 +            BUG();
   1.124      }
   1.125      unmap_domain_page(pl1e);
   1.126