ia64/xen-unstable

changeset 19128:77393d4de777

[IA64] No need for cmpxchg on page_info structure.

Updates and checks on count_info and page owner can safely be
non-atomic.
This is ia64 counter part of 19088:055c589f4791.

Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
author Isaku Yamahata <yamahata@valinux.co.jp>
date Fri Jan 30 11:12:29 2009 +0900 (2009-01-30)
parents 4edb72e1d0fe
children 971d2d5cb5cb
files xen/arch/ia64/xen/mm.c xen/include/asm-ia64/mm.h
line diff
     1.1 --- a/xen/arch/ia64/xen/mm.c	Fri Jan 30 11:12:03 2009 +0900
     1.2 +++ b/xen/arch/ia64/xen/mm.c	Fri Jan 30 11:12:29 2009 +0900
     1.3 @@ -183,6 +183,9 @@
     1.4  #include <asm/event.h>
     1.5  #include <asm/debugger.h>
     1.6  
     1.7 +
     1.8 +#define MEM_LOG(_f, _a...) gdprintk(XENLOG_WARNING, _f "\n", ## _a)
     1.9 +
    1.10  static void domain_page_flush_and_put(struct domain* d, unsigned long mpaddr,
    1.11                                        volatile pte_t* ptep, pte_t old_pte, 
    1.12                                        struct page_info* page);
    1.13 @@ -2745,8 +2748,7 @@ steal_page(struct domain *d, struct page
    1.14  #if 0 /* if big endian */
    1.15  # error "implement big endian version of steal_page()"
    1.16  #endif
    1.17 -    u32 _d, _nd;
    1.18 -    u64 x, nx, y;
    1.19 +    u32 x, y;
    1.20  
    1.21      if (page_get_owner(page) != d) {
    1.22          gdprintk(XENLOG_INFO, "%s d 0x%p owner 0x%p\n",
    1.23 @@ -2794,61 +2796,64 @@ steal_page(struct domain *d, struct page
    1.24      }
    1.25  
    1.26      spin_lock(&d->page_alloc_lock);
    1.27 +    /* check again */
    1.28 +    if (is_xen_heap_page(page) || page_get_owner(page) != d) {
    1.29 +        goto fail;
    1.30 +    }
    1.31  
    1.32      /*
    1.33 -     * The tricky bit: atomically release ownership while there is just one
    1.34 -     * benign reference to the page (PGC_allocated). If that reference
    1.35 -     * disappears then the deallocation routine will safely spin.
    1.36 +     * We require there is just one reference (PGC_allocated). We temporarily
    1.37 +     * drop this reference now so that we can safely swizzle the owner.
    1.38       */
    1.39 -    _d  = pickle_domptr(d);
    1.40 -    y = *((u64*)&page->count_info);
    1.41 +    y = page->count_info;
    1.42      do {
    1.43          x = y;
    1.44 -        nx = x & 0xffffffff;
    1.45          // page->count_info: untouched
    1.46 -        // page->u.inused._domain = 0;
    1.47 -        _nd = x >> 32;
    1.48  
    1.49          if (unlikely(((x & (PGC_count_mask | PGC_allocated)) !=
    1.50 -                      (1 | PGC_allocated))) ||
    1.51 -            unlikely(_nd != _d)) {
    1.52 -            struct domain* nd = unpickle_domptr(_nd);
    1.53 +                      (1 | PGC_allocated)))) {
    1.54 +            struct domain* nd = page_get_owner(page);
    1.55              if (nd == NULL) {
    1.56                  gdprintk(XENLOG_INFO, "gnttab_transfer: "
    1.57 -                        "Bad page %p: ed=%p(%u) 0x%x, "
    1.58 -                        "sd=%p 0x%x,"
    1.59 -                        " caf=%016lx, taf=%" PRtype_info
    1.60 +                        "Bad page %p: ed=%p(%u), "
    1.61 +                        "sd=%p,"
    1.62 +                        " caf=%016x, taf=%" PRtype_info
    1.63                          " memflags 0x%x\n",
    1.64                          (void *) page_to_mfn(page),
    1.65 -                        d, d->domain_id, _d,
    1.66 -                        nd, _nd,
    1.67 +                        d, d->domain_id,
    1.68 +                        nd,
    1.69                          x,
    1.70                          page->u.inuse.type_info,
    1.71                          memflags);
    1.72              } else {
    1.73                  gdprintk(XENLOG_WARNING, "gnttab_transfer: "
    1.74 -                        "Bad page %p: ed=%p(%u) 0x%x, "
    1.75 -                        "sd=%p(%u) 0x%x,"
    1.76 -                        " caf=%016lx, taf=%" PRtype_info
    1.77 +                        "Bad page %p: ed=%p(%u), "
    1.78 +                        "sd=%p(%u),"
    1.79 +                        " caf=%016x, taf=%" PRtype_info
    1.80                          " memflags 0x%x\n",
    1.81                          (void *) page_to_mfn(page),
    1.82 -                        d, d->domain_id, _d,
    1.83 -                        nd, nd->domain_id, _nd,
    1.84 +                        d, d->domain_id,
    1.85 +                        nd, nd->domain_id,
    1.86                          x,
    1.87                          page->u.inuse.type_info,
    1.88                          memflags);
    1.89              }
    1.90 -            spin_unlock(&d->page_alloc_lock);
    1.91 -            return -1;
    1.92 +            goto fail;
    1.93          }
    1.94  
    1.95 -        y = cmpxchg((u64*)&page->count_info, x, nx);
    1.96 +        y = cmpxchg(&page->count_info, x, x & ~PGC_count_mask);
    1.97      } while (unlikely(y != x));
    1.98  
    1.99 -    /*
   1.100 -     * Unlink from 'd'. At least one reference remains (now anonymous), so
   1.101 -     * noone else is spinning to try to delete this page from 'd'.
   1.102 -     */
   1.103 +    /* Swizzle the owner then reinstate the PGC_allocated reference. */
   1.104 +    page_set_owner(page, NULL);
   1.105 +    y = page->count_info;
   1.106 +    do {
   1.107 +        x = y;
   1.108 +        BUG_ON((x & (PGC_count_mask | PGC_allocated)) != PGC_allocated);
   1.109 +        y = cmpxchg(&page->count_info, x, x | 1);
   1.110 +    } while (unlikely(y != x));
   1.111 +
   1.112 +    /* Unlink from original owner. */
   1.113      if ( !(memflags & MEMF_no_refcount) )
   1.114          d->tot_pages--;
   1.115      list_del(&page->list);
   1.116 @@ -2856,6 +2861,13 @@ steal_page(struct domain *d, struct page
   1.117      spin_unlock(&d->page_alloc_lock);
   1.118      perfc_incr(steal_page);
   1.119      return 0;
   1.120 +
   1.121 + fail:
   1.122 +    spin_unlock(&d->page_alloc_lock);
   1.123 +    MEM_LOG("Bad page %p: ed=%p(%u), sd=%p, caf=%08x, taf=%" PRtype_info,
   1.124 +            (void *)page_to_mfn(page), d, d->domain_id,
   1.125 +            page_get_owner(page), page->count_info, page->u.inuse.type_info);
   1.126 +    return -1;
   1.127  }
   1.128  
   1.129  static void
   1.130 @@ -3043,14 +3055,6 @@ void domain_cache_flush (struct domain *
   1.131      //printk ("domain_cache_flush: %d %d pages\n", d->domain_id, nbr_page);
   1.132  }
   1.133  
   1.134 -#ifdef VERBOSE
   1.135 -#define MEM_LOG(_f, _a...)                           \
   1.136 -  printk("DOM%u: (file=mm.c, line=%d) " _f "\n", \
   1.137 -         current->domain->domain_id , __LINE__ , ## _a )
   1.138 -#else
   1.139 -#define MEM_LOG(_f, _a...) ((void)0)
   1.140 -#endif
   1.141 -
   1.142  static void free_page_type(struct page_info *page, u32 type)
   1.143  {
   1.144  }
     2.1 --- a/xen/include/asm-ia64/mm.h	Fri Jan 30 11:12:03 2009 +0900
     2.2 +++ b/xen/include/asm-ia64/mm.h	Fri Jan 30 11:12:29 2009 +0900
     2.3 @@ -167,24 +167,28 @@ static inline void put_page(struct page_
     2.4  static inline int get_page(struct page_info *page,
     2.5                             struct domain *domain)
     2.6  {
     2.7 -    u64 x, nx, y = *((u64*)&page->count_info);
     2.8 -    u32 _domain = pickle_domptr(domain);
     2.9 +    u32 x, y = page->count_info;
    2.10  
    2.11      do {
    2.12 -	x = y;
    2.13 -	nx = x + 1;
    2.14 -	if (unlikely((x & PGC_count_mask) == 0) ||	/* Not allocated? */
    2.15 -	    unlikely((nx & PGC_count_mask) == 0) ||	/* Count overflow? */
    2.16 -	    unlikely((x >> 32) != _domain)) {		/* Wrong owner? */
    2.17 +        x = y;
    2.18 +        if (unlikely((x & PGC_count_mask) == 0) ||  /* Not allocated? */
    2.19 +            unlikely(((x + 1) & PGC_count_mask) == 0) ) {/* Count overflow? */
    2.20 +            goto fail;
    2.21 +        }
    2.22 +        y = cmpxchg_acq(&page->count_info, x, x + 1);
    2.23 +    } while (unlikely(y != x));
    2.24  
    2.25 -	    gdprintk(XENLOG_INFO, "Error pfn %lx: rd=%p, od=%p, caf=%016lx, taf=%"
    2.26 -		PRtype_info "\n", page_to_mfn(page), domain,
    2.27 -		unpickle_domptr(x >> 32), x, page->u.inuse.type_info);
    2.28 -	    return 0;
    2.29 -	}
    2.30 -    }
    2.31 -    while(unlikely((y = cmpxchg_acq((u64*)&page->count_info, x, nx)) != x));
    2.32 -    return 1;
    2.33 +    if (likely(page_get_owner(page) == domain))
    2.34 +        return 1;
    2.35 +
    2.36 +    put_page(page);
    2.37 +fail:
    2.38 +    /* if (!domain->is_dying) */ /* XXX: header inclusion hell */
    2.39 +    gdprintk(XENLOG_INFO,
    2.40 +             "Error pfn %lx: rd=%p, od=%p, caf=%016x, taf=%" PRtype_info "\n",
    2.41 +             page_to_mfn(page), domain,
    2.42 +             page_get_owner(page), y, page->u.inuse.type_info);
    2.43 +    return 0;
    2.44  }
    2.45  
    2.46  int is_iomem_page(unsigned long mfn);