ia64/xen-unstable

changeset 19089:055c589f4791

x86: No need for CMPXCHG8B on page_info structure.

Updates and checks on count_info and page owner can safely be
non-atomic.

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Mon Jan 26 14:56:19 2009 +0000 (2009-01-26)
parents c9dc7dcacc1d
children 39517e863cc8
files xen/arch/x86/mm.c
line diff
     1.1 --- a/xen/arch/x86/mm.c	Mon Jan 26 11:22:03 2009 +0000
     1.2 +++ b/xen/arch/x86/mm.c	Mon Jan 26 14:56:19 2009 +0000
     1.3 @@ -1927,36 +1927,29 @@ void put_page(struct page_info *page)
     1.4  
     1.5  int get_page(struct page_info *page, struct domain *domain)
     1.6  {
     1.7 -    u32 x, nx, y = page->count_info;
     1.8 -    u32 d, nd = page->u.inuse._domain;
     1.9 -    u32 _domain = pickle_domptr(domain);
    1.10 +    u32 x, y = page->count_info;
    1.11  
    1.12      do {
    1.13 -        x  = y;
    1.14 -        nx = x + 1;
    1.15 -        d  = nd;
    1.16 +        x = y;
    1.17          if ( unlikely((x & PGC_count_mask) == 0) ||  /* Not allocated? */
    1.18               /* Keep one spare reference to be acquired by get_page_light(). */
    1.19 -             unlikely(((nx + 1) & PGC_count_mask) <= 1) || /* Overflow? */
    1.20 -             unlikely(d != _domain) )                /* Wrong owner? */
    1.21 -        {
    1.22 -            if ( !_shadow_mode_refcounts(domain) && !domain->is_dying )
    1.23 -                gdprintk(XENLOG_INFO,
    1.24 -                         "Error pfn %lx: rd=%p, od=%p, caf=%08x, taf=%"
    1.25 -                         PRtype_info "\n",
    1.26 -                         page_to_mfn(page), domain, unpickle_domptr(d),
    1.27 -                         x, page->u.inuse.type_info);
    1.28 -            return 0;
    1.29 -        }
    1.30 -        asm volatile (
    1.31 -            LOCK_PREFIX "cmpxchg8b %2"
    1.32 -            : "=d" (nd), "=a" (y),
    1.33 -            "=m" (*(volatile u64 *)(&page->count_info))
    1.34 -            : "0" (d), "1" (x), "c" (d), "b" (nx) );
    1.35 +             unlikely(((x + 2) & PGC_count_mask) <= 1) ) /* Overflow? */
    1.36 +            goto fail;
    1.37      }
    1.38 -    while ( unlikely(nd != d) || unlikely(y != x) );
    1.39 -
    1.40 -    return 1;
    1.41 +    while ( (y = cmpxchg(&page->count_info, x, x + 1)) != x );
    1.42 +
    1.43 +    if ( likely(page_get_owner(page) == domain) )
    1.44 +        return 1;
    1.45 +
    1.46 +    put_page(page);
    1.47 +
    1.48 + fail:
    1.49 +    if ( !_shadow_mode_refcounts(domain) && !domain->is_dying )
    1.50 +        gdprintk(XENLOG_INFO,
    1.51 +                 "Error pfn %lx: rd=%p, od=%p, caf=%08x, taf=%" PRtype_info,
    1.52 +                 page_to_mfn(page), domain, page_get_owner(page),
    1.53 +                 y, page->u.inuse.type_info);
    1.54 +    return 0;
    1.55  }
    1.56  
    1.57  /*
    1.58 @@ -3478,49 +3471,47 @@ int replace_grant_host_mapping(
    1.59  int steal_page(
    1.60      struct domain *d, struct page_info *page, unsigned int memflags)
    1.61  {
    1.62 -    u32 _d, _nd, x, y;
    1.63 +    u32 x, y;
    1.64  
    1.65      spin_lock(&d->page_alloc_lock);
    1.66  
    1.67 +    if ( is_xen_heap_page(page) || (page_get_owner(page) != d) )
    1.68 +        goto fail;
    1.69 +
    1.70      /*
    1.71 -     * The tricky bit: atomically release ownership while there is just one 
    1.72 -     * benign reference to the page (PGC_allocated). If that reference 
    1.73 -     * disappears then the deallocation routine will safely spin.
    1.74 +     * We require there is just one reference (PGC_allocated). We temporarily
    1.75 +     * drop this reference now so that we can safely swizzle the owner.
    1.76       */
    1.77 -    _d  = pickle_domptr(d);
    1.78 -    _nd = page->u.inuse._domain;
    1.79 -    y   = page->count_info;
    1.80 +    y = page->count_info;
    1.81      do {
    1.82          x = y;
    1.83 -        if ( unlikely((x & (PGC_count_mask|PGC_allocated)) !=
    1.84 -                      (1 | PGC_allocated)) || unlikely(_nd != _d) )
    1.85 -        { 
    1.86 -            MEM_LOG("gnttab_transfer: Bad page %p: ed=%p(%u), sd=%p,"
    1.87 -                    " caf=%08x, taf=%" PRtype_info "\n", 
    1.88 -                    (void *) page_to_mfn(page),
    1.89 -                    d, d->domain_id, unpickle_domptr(_nd), x, 
    1.90 -                    page->u.inuse.type_info);
    1.91 -            spin_unlock(&d->page_alloc_lock);
    1.92 -            return -1;
    1.93 -        }
    1.94 -        asm volatile (
    1.95 -            LOCK_PREFIX "cmpxchg8b %2"
    1.96 -            : "=d" (_nd), "=a" (y),
    1.97 -            "=m" (*(volatile u64 *)(&page->count_info))
    1.98 -            : "0" (_d), "1" (x), "c" (NULL), "b" (x) );
    1.99 -    } while (unlikely(_nd != _d) || unlikely(y != x));
   1.100 -
   1.101 -    /*
   1.102 -     * Unlink from 'd'. At least one reference remains (now anonymous), so 
   1.103 -     * noone else is spinning to try to delete this page from 'd'.
   1.104 -     */
   1.105 +        if ( (x & (PGC_count_mask|PGC_allocated)) != (1 | PGC_allocated) )
   1.106 +            goto fail;
   1.107 +        y = cmpxchg(&page->count_info, x, x & ~PGC_count_mask);
   1.108 +    } while ( y != x );
   1.109 +
   1.110 +    /* Swizzle the owner then reinstate the PGC_allocated reference. */
   1.111 +    page_set_owner(page, NULL);
   1.112 +    y = page->count_info;
   1.113 +    do {
   1.114 +        x = y;
   1.115 +        BUG_ON((x & (PGC_count_mask|PGC_allocated)) != PGC_allocated);
   1.116 +    } while ( (y = cmpxchg(&page->count_info, x, x | 1)) != x );
   1.117 +
   1.118 +    /* Unlink from original owner. */
   1.119      if ( !(memflags & MEMF_no_refcount) )
   1.120          d->tot_pages--;
   1.121      list_del(&page->list);
   1.122  
   1.123      spin_unlock(&d->page_alloc_lock);
   1.124 -
   1.125      return 0;
   1.126 +
   1.127 + fail:
   1.128 +    spin_unlock(&d->page_alloc_lock);
   1.129 +    MEM_LOG("Bad page %p: ed=%p(%u), sd=%p, caf=%08x, taf=%" PRtype_info,
   1.130 +            (void *)page_to_mfn(page), d, d->domain_id,
   1.131 +            page_get_owner(page), page->count_info, page->u.inuse.type_info);
   1.132 +    return -1;
   1.133  }
   1.134  
   1.135  int do_update_va_mapping(unsigned long va, u64 val64,