1.1 --- a/xen/arch/ia64/xen/mm.c	Thu Mar 22 09:30:54 2007 -0600
     1.2 +++ b/xen/arch/ia64/xen/mm.c	Mon Mar 26 09:15:17 2007 -0600
     1.3 @@ -208,46 +208,6 @@ alloc_dom_xen_and_dom_io(void)
     1.4      BUG_ON(dom_io == NULL);
     1.5  }
     1.6  
     1.7 -// heavily depends on the struct page_info layout.
     1.8 -// if (page_get_owner(page) == d &&
     1.9 -//     test_and_clear_bit(_PGC_allocated, &page->count_info)) {
    1.10 -//     put_page(page);
    1.11 -// }
    1.12 -static void
    1.13 -try_to_clear_PGC_allocate(struct domain* d, struct page_info* page)
    1.14 -{
    1.15 -    u32 _d, _nd;
    1.16 -    u64 x, nx, y;
    1.17 -
    1.18 -    _d = pickle_domptr(d);
    1.19 -    y = *((u64*)&page->count_info);
    1.20 -    do {
    1.21 -        x = y;
    1.22 -        _nd = x >> 32;
    1.23 -        nx = x - 1;
    1.24 -        __clear_bit(_PGC_allocated, &nx);
    1.25 -
    1.26 -        if (unlikely(!(x & PGC_allocated)) || unlikely(_nd != _d)) {
    1.27 -            struct domain* nd = unpickle_domptr(_nd);
    1.28 -            if (nd == NULL) {
    1.29 -                gdprintk(XENLOG_INFO, "gnttab_transfer: "
    1.30 -                        "Bad page %p: ed=%p(%u) 0x%x, "
    1.31 -                        "sd=%p 0x%x,"
    1.32 -                        " caf=%016lx, taf=%" PRtype_info "\n",
    1.33 -                        (void *) page_to_mfn(page),
    1.34 -                        d, d->domain_id, _d,
    1.35 -                        nd, _nd,
    1.36 -                        x,
    1.37 -                        page->u.inuse.type_info);
    1.38 -            }
    1.39 -            break;
    1.40 -        }
    1.41 -
    1.42 -        BUG_ON((nx & PGC_count_mask) < 1);
    1.43 -        y = cmpxchg((u64*)&page->count_info, x, nx);
    1.44 -    } while (unlikely(y != x));
    1.45 -}
    1.46 -
    1.47  static void
    1.48  mm_teardown_pte(struct domain* d, volatile pte_t* pte, unsigned long offset)
    1.49  {
    1.50 @@ -267,22 +227,25 @@ mm_teardown_pte(struct domain* d, volati
    1.51      if (!mfn_valid(mfn))
    1.52          return;
    1.53      page = mfn_to_page(mfn);
    1.54 -    // struct page_info corresponding to mfn may exist or not depending
    1.55 -    // on CONFIG_VIRTUAL_FRAME_TABLE.
    1.56 -    // This check is too easy.
    1.57 -    // The right way is to check whether this page is of io area or acpi pages
    1.58 +    // page might be pte page for p2m exposing. check it.
    1.59      if (page_get_owner(page) == NULL) {
    1.60          BUG_ON(page->count_info != 0);
    1.61          return;
    1.62      }
    1.63 +    // struct page_info corresponding to mfn may exist or not depending
    1.64 +    // on CONFIG_VIRTUAL_FRAME_TABLE.
    1.65 +    // The above check is too easy.
    1.66 +    // The right way is to check whether this page is of io area or acpi pages
    1.67  
    1.68      if (pte_pgc_allocated(old_pte)) {
    1.69          BUG_ON(page_get_owner(page) != d);
    1.70          BUG_ON(get_gpfn_from_mfn(mfn) == INVALID_M2P_ENTRY);
    1.71          set_gpfn_from_mfn(mfn, INVALID_M2P_ENTRY);
    1.72 -        try_to_clear_PGC_allocate(d, page);
    1.73 +        if (test_and_clear_bit(_PGC_allocated, &page->count_info))
    1.74 +            put_page(page);
    1.75 +    } else {
    1.76 +        put_page(page);
    1.77      }
    1.78 -    put_page(page);
    1.79  }
    1.80  
    1.81  static void
    1.82 @@ -770,7 +733,6 @@ static struct page_info *
    1.83  {
    1.84      struct page_info *p;
    1.85      unsigned long maddr;
    1.86 -    int ret;
    1.87  
    1.88      BUG_ON(!pte_none(*pte));
    1.89  
    1.90 @@ -791,8 +753,6 @@ static struct page_info *
    1.91                 maddr);
    1.92      }
    1.93  
    1.94 -    ret = get_page(p, d);
    1.95 -    BUG_ON(ret == 0);
    1.96      set_gpfn_from_mfn(page_to_mfn(p), mpaddr >> PAGE_SHIFT);
    1.97      // clear_page() and set_gpfn_from_mfn() become visible before set_pte_rel()
    1.98      // because set_pte_rel() has release semantics
    1.99 @@ -893,11 +853,9 @@ assign_domain_page(struct domain *d,
   1.100                     unsigned long mpaddr, unsigned long physaddr)
   1.101  {
   1.102      struct page_info* page = mfn_to_page(physaddr >> PAGE_SHIFT);
   1.103 -    int ret;
   1.104  
   1.105      BUG_ON((physaddr & GPFN_IO_MASK) != GPFN_MEM);
   1.106 -    ret = get_page(page, d);
   1.107 -    BUG_ON(ret == 0);
   1.108 +    BUG_ON(page->count_info != (PGC_allocated | 1));
   1.109      set_gpfn_from_mfn(physaddr >> PAGE_SHIFT, mpaddr >> PAGE_SHIFT);
   1.110      // because __assign_domain_page() uses set_pte_rel() which has
   1.111      // release semantics, smp_mb() isn't needed.
   1.112 @@ -1086,6 +1044,29 @@ assign_domain_mach_page(struct domain *d
   1.113  }
   1.114  
   1.115  static void
   1.116 +adjust_page_count_info(struct page_info* page)
   1.117 +{
   1.118 +    struct domain* d = page_get_owner(page);
   1.119 +    BUG_ON((page->count_info & PGC_count_mask) != 1);
   1.120 +    if (d != NULL) {
   1.121 +        int ret = get_page(page, d);
   1.122 +        BUG_ON(ret == 0);
   1.123 +    } else {
   1.124 +        u64 x, nx, y;
   1.125 +
   1.126 +        y = *((u64*)&page->count_info);
   1.127 +        do {
   1.128 +            x = y;
   1.129 +            nx = x + 1;
   1.130 +
   1.131 +            BUG_ON((x >> 32) != 0);
   1.132 +            BUG_ON((nx & PGC_count_mask) != 2);
   1.133 +            y = cmpxchg((u64*)&page->count_info, x, nx);
   1.134 +        } while (unlikely(y != x));
   1.135 +    }
   1.136 +}
   1.137 +
   1.138 +static void
   1.139  domain_put_page(struct domain* d, unsigned long mpaddr,
   1.140                  volatile pte_t* ptep, pte_t old_pte, int clear_PGC_allocate)
   1.141  {
   1.142 @@ -1100,8 +1081,19 @@ domain_put_page(struct domain* d, unsign
   1.143              BUG();
   1.144          }
   1.145  
   1.146 -	if (clear_PGC_allocate)
   1.147 -            try_to_clear_PGC_allocate(d, page);
   1.148 +        if (likely(clear_PGC_allocate)) {
   1.149 +            if (!test_and_clear_bit(_PGC_allocated, &page->count_info))
   1.150 +                BUG();
   1.151 +            /* put_page() is done by domain_page_flush_and_put() */
   1.152 +        } else {
   1.153 +            // In this case, page reference count mustn't touched.
   1.154 +            // domain_page_flush_and_put() decrements it, we increment
   1.155 +            // it in advence. This patch is slow path.
   1.156 +            //
   1.157 +            // guest_remove_page(): owner = d, count_info = 1
   1.158 +            // memory_exchange(): owner = NULL, count_info = 1
   1.159 +            adjust_page_count_info(page);
   1.160 +        }
   1.161      }
   1.162      domain_page_flush_and_put(d, mpaddr, ptep, old_pte, page);
   1.163  }
   1.164 @@ -1148,7 +1140,7 @@ static int
   1.165  assign_domain_page_cmpxchg_rel(struct domain* d, unsigned long mpaddr,
   1.166                                 struct page_info* old_page,
   1.167                                 struct page_info* new_page,
   1.168 -                               unsigned long flags)
   1.169 +                               unsigned long flags, int clear_PGC_allocate)
   1.170  {
   1.171      struct mm_struct *mm = &d->arch.mm;
   1.172      volatile pte_t* pte;
   1.173 @@ -1160,6 +1152,7 @@ assign_domain_page_cmpxchg_rel(struct do
   1.174      pte_t new_pte;
   1.175      pte_t ret_pte;
   1.176  
   1.177 +    BUG_ON((flags & ASSIGN_pgc_allocated) == 0);
   1.178      pte = lookup_alloc_domain_pte(d, mpaddr);
   1.179  
   1.180   again:
   1.181 @@ -1200,6 +1193,18 @@ assign_domain_page_cmpxchg_rel(struct do
   1.182      BUG_ON(old_mfn == new_mfn);
   1.183  
   1.184      set_gpfn_from_mfn(old_mfn, INVALID_M2P_ENTRY);
   1.185 +    if (likely(clear_PGC_allocate)) {
   1.186 +        if (!test_and_clear_bit(_PGC_allocated, &old_page->count_info))
   1.187 +            BUG();
   1.188 +    } else {
   1.189 +        int ret;
   1.190 +        // adjust for count_info for domain_page_flush_and_put()
   1.191 +        // This is slow path.
   1.192 +        BUG_ON(!test_bit(_PGC_allocated, &old_page->count_info));
   1.193 +        BUG_ON(d == NULL);
   1.194 +        ret = get_page(old_page, d);
   1.195 +        BUG_ON(ret == 0);
   1.196 +    }
   1.197  
   1.198      domain_page_flush_and_put(d, mpaddr, pte, old_pte, old_page);
   1.199      perfc_incrc(assign_domain_pge_cmpxchg_rel);
   1.200 @@ -1207,7 +1212,8 @@ assign_domain_page_cmpxchg_rel(struct do
   1.201  }
   1.202  
   1.203  static void
   1.204 -zap_domain_page_one(struct domain *d, unsigned long mpaddr, unsigned long mfn)
   1.205 +zap_domain_page_one(struct domain *d, unsigned long mpaddr,
   1.206 +                    int clear_PGC_allocate, unsigned long mfn)
   1.207  {
   1.208      struct mm_struct *mm = &d->arch.mm;
   1.209      volatile pte_t *pte;
   1.210 @@ -1258,12 +1264,8 @@ zap_domain_page_one(struct domain *d, un
   1.211      page = mfn_to_page(mfn);
   1.212      BUG_ON((page->count_info & PGC_count_mask) == 0);
   1.213  
   1.214 -    // exchange_memory() calls
   1.215 -    //   steal_page()
   1.216 -    //     page owner is set to NULL
   1.217 -    //   guest_physmap_remove_page()
   1.218 -    //     zap_domain_page_one()
   1.219 -    domain_put_page(d, mpaddr, pte, old_pte, (page_get_owner(page) != NULL));
   1.220 +    BUG_ON(clear_PGC_allocate && (page_get_owner(page) == NULL));
   1.221 +    domain_put_page(d, mpaddr, pte, old_pte, clear_PGC_allocate);
   1.222      perfc_incrc(zap_dcomain_page_one);
   1.223  }
   1.224  
   1.225 @@ -1276,7 +1278,7 @@ dom0vp_zap_physmap(struct domain *d, uns
   1.226          return -ENOSYS;
   1.227      }
   1.228  
   1.229 -    zap_domain_page_one(d, gpfn << PAGE_SHIFT, INVALID_MFN);
   1.230 +    zap_domain_page_one(d, gpfn << PAGE_SHIFT, 1, INVALID_MFN);
   1.231      perfc_incrc(dom0vp_zap_physmap);
   1.232      return 0;
   1.233  }
   1.234 @@ -1563,6 +1565,7 @@ destroy_grant_host_mapping(unsigned long
   1.235      /* try_to_clear_PGC_allocate(d, page) is not needed. */
   1.236      BUG_ON(page_get_owner(page) == d &&
   1.237             get_gpfn_from_mfn(mfn) == gpfn);
   1.238 +    BUG_ON(pte_pgc_allocated(old_pte));
   1.239      domain_page_flush_and_put(d, gpaddr, pte, old_pte, page);
   1.240  
   1.241      perfc_incrc(destroy_grant_host_mapping);
   1.242 @@ -1606,9 +1609,6 @@ steal_page(struct domain *d, struct page
   1.243          // assign_domain_page_cmpxchg_rel() has release semantics
   1.244          // so smp_mb() isn't needed.
   1.245  
   1.246 -        ret = get_page(new, d);
   1.247 -        BUG_ON(ret == 0);
   1.248 -
   1.249          gpfn = get_gpfn_from_mfn(page_to_mfn(page));
   1.250          if (gpfn == INVALID_M2P_ENTRY) {
   1.251              free_domheap_page(new);
   1.252 @@ -1621,7 +1621,7 @@ steal_page(struct domain *d, struct page
   1.253  
   1.254          ret = assign_domain_page_cmpxchg_rel(d, gpfn << PAGE_SHIFT, page, new,
   1.255                                               ASSIGN_writable |
   1.256 -                                             ASSIGN_pgc_allocated);
   1.257 +                                             ASSIGN_pgc_allocated, 0);
   1.258          if (ret < 0) {
   1.259              gdprintk(XENLOG_INFO, "assign_domain_page_cmpxchg_rel failed %d\n",
   1.260                      ret);
   1.261 @@ -1648,18 +1648,8 @@ steal_page(struct domain *d, struct page
   1.262          // page->u.inused._domain = 0;
   1.263          _nd = x >> 32;
   1.264  
   1.265 -        if (
   1.266 -            // when !MEMF_no_refcount, page might be put_page()'d or
   1.267 -            // it will be put_page()'d later depending on queued.
   1.268 -            unlikely(!(memflags & MEMF_no_refcount) &&
   1.269 -                     ((x & (PGC_count_mask | PGC_allocated)) !=
   1.270 +        if (unlikely(((x & (PGC_count_mask | PGC_allocated)) !=
   1.271                        (1 | PGC_allocated))) ||
   1.272 -            // when MEMF_no_refcount, page isn't de-assigned from
   1.273 -            // this domain yet. So count_info = 2
   1.274 -            unlikely((memflags & MEMF_no_refcount) &&
   1.275 -                     ((x & (PGC_count_mask | PGC_allocated)) !=
   1.276 -                      (2 | PGC_allocated))) ||
   1.277 -
   1.278              unlikely(_nd != _d)) {
   1.279              struct domain* nd = unpickle_domptr(_nd);
   1.280              if (nd == NULL) {
   1.281 @@ -1711,11 +1701,8 @@ void
   1.282  guest_physmap_add_page(struct domain *d, unsigned long gpfn,
   1.283                         unsigned long mfn)
   1.284  {
   1.285 -    int ret;
   1.286 -
   1.287      BUG_ON(!mfn_valid(mfn));
   1.288 -    ret = get_page(mfn_to_page(mfn), d);
   1.289 -    BUG_ON(ret == 0);
   1.290 +    BUG_ON(mfn_to_page(mfn)->count_info != (PGC_allocated | 1));
   1.291      set_gpfn_from_mfn(mfn, gpfn);
   1.292      smp_mb();
   1.293      assign_domain_page_replace(d, gpfn << PAGE_SHIFT, mfn,
   1.294 @@ -1731,7 +1718,7 @@ guest_physmap_remove_page(struct domain 
   1.295                            unsigned long mfn)
   1.296  {
   1.297      BUG_ON(mfn == 0);//XXX
   1.298 -    zap_domain_page_one(d, gpfn << PAGE_SHIFT, mfn);
   1.299 +    zap_domain_page_one(d, gpfn << PAGE_SHIFT, 0, mfn);
   1.300      perfc_incrc(guest_physmap_remove_page);
   1.301  }
   1.302  
   1.303 @@ -2142,27 +2129,9 @@ arch_memory_op(int op, XEN_GUEST_HANDLE(
   1.304  
   1.305          /* Unmap from old location, if any. */
   1.306          gpfn = get_gpfn_from_mfn(mfn);
   1.307 -        if (gpfn != INVALID_M2P_ENTRY) {
   1.308 -            /*
   1.309 -             * guest_physmap_remove_page() (for IPF) descrements page
   1.310 -             * counter and unset PGC_allocated flag,
   1.311 -             * so pre-increment page counter and post-set flag inserted
   1.312 -             */
   1.313 -            /* pre-increment page counter */
   1.314 -            if (!get_page(mfn_to_page(mfn), d))
   1.315 -                goto out;
   1.316 -
   1.317 +        if (gpfn != INVALID_M2P_ENTRY)
   1.318              guest_physmap_remove_page(d, gpfn, mfn);
   1.319  
   1.320 -            /* post-set PGC_allocated flag */
   1.321 -            if ((mfn_to_page(mfn)->count_info & PGC_count_mask) != 1) {
   1.322 -                /* no one but us is using this page */
   1.323 -                put_page(mfn_to_page(mfn));
   1.324 -                goto out;
   1.325 -            }
   1.326 -            set_bit(_PGC_allocated, &mfn_to_page(mfn)->count_info);
   1.327 -        }
   1.328 -
   1.329          /* Map at new location. */
   1.330          guest_physmap_add_page(d, xatp.gpfn, mfn);
   1.331  

     2.1 --- a/xen/include/asm-ia64/mm.h	Thu Mar 22 09:30:54 2007 -0600
     2.2 +++ b/xen/include/asm-ia64/mm.h	Mon Mar 26 09:15:17 2007 -0600
     2.3 @@ -212,7 +212,7 @@ static inline int get_page_and_type(stru
     2.4  
     2.5  static inline int page_is_removable(struct page_info *page)
     2.6  {
     2.7 -    return ((page->count_info & PGC_count_mask) == 2);
     2.8 +    return ((page->count_info & PGC_count_mask) == 1);
     2.9  }
    2.10  
    2.11  #define	set_machinetophys(_mfn, _pfn) do { } while(0);