1.1 --- a/xen/arch/x86/mm.c	Tue Jan 27 11:54:22 2009 +0000
     1.2 +++ b/xen/arch/x86/mm.c	Tue Jan 27 16:02:21 2009 +0000
     1.3 @@ -205,11 +205,6 @@ void __init init_frametable(void)
     1.4      }
     1.5  
     1.6      memset(frame_table, 0, nr_pages << PAGE_SHIFT);
     1.7 -
     1.8 -#if defined(__x86_64__)
     1.9 -    for ( i = 0; i < max_page; i ++ )
    1.10 -        spin_lock_init(&frame_table[i].lock);
    1.11 -#endif
    1.12  }
    1.13  
    1.14  void __init arch_init_memory(void)
    1.15 @@ -330,11 +325,7 @@ void share_xen_page_with_guest(
    1.16  
    1.17      page_set_owner(page, d);
    1.18      wmb(); /* install valid domain ptr before updating refcnt. */
    1.19 -#ifdef __i386__
    1.20 -    ASSERT(page->count_info == 0);
    1.21 -#else
    1.22      ASSERT((page->count_info & ~PGC_xen_heap) == 0);
    1.23 -#endif
    1.24  
    1.25      /* Only add to the allocation list if the domain isn't dying. */
    1.26      if ( !d->is_dying )
    1.27 @@ -1543,24 +1534,31 @@ static int free_l4_table(struct page_inf
    1.28  #define free_l4_table(page, preemptible) (-EINVAL)
    1.29  #endif
    1.30  
    1.31 -static void page_lock(struct page_info *page)
    1.32 +static int page_lock(struct page_info *page)
    1.33  {
    1.34 -#if defined(__i386__)
    1.35 -    while ( unlikely(test_and_set_bit(_PGC_locked, &page->count_info)) )
    1.36 -        while ( test_bit(_PGC_locked, &page->count_info) )
    1.37 +    unsigned long x, nx;
    1.38 +
    1.39 +    do {
    1.40 +        while ( (x = page->u.inuse.type_info) & PGT_locked )
    1.41              cpu_relax();
    1.42 -#else
    1.43 -    spin_lock(&page->lock);
    1.44 -#endif
    1.45 +        nx = x + (1 | PGT_locked);
    1.46 +        if ( !(x & PGT_validated) ||
    1.47 +             !(x & PGT_count_mask) ||
    1.48 +             !(nx & PGT_count_mask) )
    1.49 +            return 0;
    1.50 +    } while ( cmpxchg(&page->u.inuse.type_info, x, nx) != x );
    1.51 +
    1.52 +    return 1;
    1.53  }
    1.54  
    1.55  static void page_unlock(struct page_info *page)
    1.56  {
    1.57 -#if defined(__i386__)
    1.58 -    clear_bit(_PGC_locked, &page->count_info);
    1.59 -#else
    1.60 -    spin_unlock(&page->lock);
    1.61 -#endif
    1.62 +    unsigned long x, nx, y = page->u.inuse.type_info;
    1.63 +
    1.64 +    do {
    1.65 +        x = y;
    1.66 +        nx = x - (1 | PGT_locked);
    1.67 +    } while ( (y = cmpxchg(&page->u.inuse.type_info, x, nx)) != x );
    1.68  }
    1.69  
    1.70  /* How to write an entry to the guest pagetables.
    1.71 @@ -1623,19 +1621,15 @@ static int mod_l1_entry(l1_pgentry_t *pl
    1.72      struct vcpu *curr = current;
    1.73      struct domain *d = curr->domain;
    1.74      unsigned long mfn;
    1.75 -    struct page_info *l1pg = mfn_to_page(gl1mfn);
    1.76      p2m_type_t p2mt;
    1.77      int rc = 1;
    1.78  
    1.79 -    page_lock(l1pg);
    1.80 -
    1.81      if ( unlikely(__copy_from_user(&ol1e, pl1e, sizeof(ol1e)) != 0) )
    1.82 -        return page_unlock(l1pg), 0;
    1.83 +        return 0;
    1.84  
    1.85      if ( unlikely(paging_mode_refcounts(d)) )
    1.86      {
    1.87          rc = UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, curr, preserve_ad);
    1.88 -        page_unlock(l1pg);
    1.89          return rc;
    1.90      }
    1.91  
    1.92 @@ -1644,13 +1638,12 @@ static int mod_l1_entry(l1_pgentry_t *pl
    1.93          /* Translate foreign guest addresses. */
    1.94          mfn = mfn_x(gfn_to_mfn(FOREIGNDOM, l1e_get_pfn(nl1e), &p2mt));
    1.95          if ( !p2m_is_ram(p2mt) || unlikely(mfn == INVALID_MFN) )
    1.96 -            return page_unlock(l1pg), 0;
    1.97 +            return 0;
    1.98          ASSERT((mfn & ~(PADDR_MASK >> PAGE_SHIFT)) == 0);
    1.99          nl1e = l1e_from_pfn(mfn, l1e_get_flags(nl1e));
   1.100  
   1.101          if ( unlikely(l1e_get_flags(nl1e) & l1_disallow_mask(d)) )
   1.102          {
   1.103 -            page_unlock(l1pg);
   1.104              MEM_LOG("Bad L1 flags %x",
   1.105                      l1e_get_flags(nl1e) & l1_disallow_mask(d));
   1.106              return 0;
   1.107 @@ -1662,12 +1655,11 @@ static int mod_l1_entry(l1_pgentry_t *pl
   1.108              adjust_guest_l1e(nl1e, d);
   1.109              rc = UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, curr,
   1.110                                preserve_ad);
   1.111 -            page_unlock(l1pg);
   1.112              return rc;
   1.113          }
   1.114  
   1.115          if ( unlikely(!get_page_from_l1e(nl1e, FOREIGNDOM)) )
   1.116 -            return page_unlock(l1pg), 0;
   1.117 +            return 0;
   1.118          
   1.119          adjust_guest_l1e(nl1e, d);
   1.120          if ( unlikely(!UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, curr,
   1.121 @@ -1680,11 +1672,9 @@ static int mod_l1_entry(l1_pgentry_t *pl
   1.122      else if ( unlikely(!UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, curr,
   1.123                                       preserve_ad)) )
   1.124      {
   1.125 -        page_unlock(l1pg);
   1.126          return 0;
   1.127      }
   1.128  
   1.129 -    page_unlock(l1pg);
   1.130      put_page_from_l1e(ol1e, d);
   1.131      return rc;
   1.132  }
   1.133 @@ -1694,13 +1684,13 @@ static int mod_l1_entry(l1_pgentry_t *pl
   1.134  static int mod_l2_entry(l2_pgentry_t *pl2e, 
   1.135                          l2_pgentry_t nl2e, 
   1.136                          unsigned long pfn,
   1.137 -                        unsigned long type,
   1.138                          int preserve_ad)
   1.139  {
   1.140      l2_pgentry_t ol2e;
   1.141      struct vcpu *curr = current;
   1.142      struct domain *d = curr->domain;
   1.143      struct page_info *l2pg = mfn_to_page(pfn);
   1.144 +    unsigned long type = l2pg->u.inuse.type_info;
   1.145      int rc = 1;
   1.146  
   1.147      if ( unlikely(!is_guest_l2_slot(d, type, pgentry_ptr_to_slot(pl2e))) )
   1.148 @@ -1709,16 +1699,13 @@ static int mod_l2_entry(l2_pgentry_t *pl
   1.149          return 0;
   1.150      }
   1.151  
   1.152 -    page_lock(l2pg);
   1.153 -
   1.154      if ( unlikely(__copy_from_user(&ol2e, pl2e, sizeof(ol2e)) != 0) )
   1.155 -        return page_unlock(l2pg), 0;
   1.156 +        return 0;
   1.157  
   1.158      if ( l2e_get_flags(nl2e) & _PAGE_PRESENT )
   1.159      {
   1.160          if ( unlikely(l2e_get_flags(nl2e) & L2_DISALLOW_MASK) )
   1.161          {
   1.162 -            page_unlock(l2pg);
   1.163              MEM_LOG("Bad L2 flags %x",
   1.164                      l2e_get_flags(nl2e) & L2_DISALLOW_MASK);
   1.165              return 0;
   1.166 @@ -1729,12 +1716,11 @@ static int mod_l2_entry(l2_pgentry_t *pl
   1.167          {
   1.168              adjust_guest_l2e(nl2e, d);
   1.169              rc = UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn, curr, preserve_ad);
   1.170 -            page_unlock(l2pg);
   1.171              return rc;
   1.172          }
   1.173  
   1.174          if ( unlikely(get_page_from_l2e(nl2e, pfn, d) < 0) )
   1.175 -            return page_unlock(l2pg), 0;
   1.176 +            return 0;
   1.177  
   1.178          adjust_guest_l2e(nl2e, d);
   1.179          if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn, curr,
   1.180 @@ -1747,11 +1733,9 @@ static int mod_l2_entry(l2_pgentry_t *pl
   1.181      else if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn, curr,
   1.182                                       preserve_ad)) )
   1.183      {
   1.184 -        page_unlock(l2pg);
   1.185          return 0;
   1.186      }
   1.187  
   1.188 -    page_unlock(l2pg);
   1.189      put_page_from_l2e(ol2e, pfn);
   1.190      return rc;
   1.191  }
   1.192 @@ -1766,7 +1750,6 @@ static int mod_l3_entry(l3_pgentry_t *pl
   1.193      l3_pgentry_t ol3e;
   1.194      struct vcpu *curr = current;
   1.195      struct domain *d = curr->domain;
   1.196 -    struct page_info *l3pg = mfn_to_page(pfn);
   1.197      int rc = 0;
   1.198  
   1.199      if ( unlikely(!is_guest_l3_slot(pgentry_ptr_to_slot(pl3e))) )
   1.200 @@ -1782,16 +1765,13 @@ static int mod_l3_entry(l3_pgentry_t *pl
   1.201      if ( is_pv_32bit_domain(d) && (pgentry_ptr_to_slot(pl3e) >= 3) )
   1.202          return -EINVAL;
   1.203  
   1.204 -    page_lock(l3pg);
   1.205 -
   1.206      if ( unlikely(__copy_from_user(&ol3e, pl3e, sizeof(ol3e)) != 0) )
   1.207 -        return page_unlock(l3pg), -EFAULT;
   1.208 +        return -EFAULT;
   1.209  
   1.210      if ( l3e_get_flags(nl3e) & _PAGE_PRESENT )
   1.211      {
   1.212          if ( unlikely(l3e_get_flags(nl3e) & l3_disallow_mask(d)) )
   1.213          {
   1.214 -            page_unlock(l3pg);
   1.215              MEM_LOG("Bad L3 flags %x",
   1.216                      l3e_get_flags(nl3e) & l3_disallow_mask(d));
   1.217              return -EINVAL;
   1.218 @@ -1802,13 +1782,12 @@ static int mod_l3_entry(l3_pgentry_t *pl
   1.219          {
   1.220              adjust_guest_l3e(nl3e, d);
   1.221              rc = UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn, curr, preserve_ad);
   1.222 -            page_unlock(l3pg);
   1.223              return rc ? 0 : -EFAULT;
   1.224          }
   1.225  
   1.226          rc = get_page_from_l3e(nl3e, pfn, d, 0, preemptible);
   1.227          if ( unlikely(rc < 0) )
   1.228 -            return page_unlock(l3pg), rc;
   1.229 +            return rc;
   1.230          rc = 0;
   1.231  
   1.232          adjust_guest_l3e(nl3e, d);
   1.233 @@ -1822,7 +1801,6 @@ static int mod_l3_entry(l3_pgentry_t *pl
   1.234      else if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn, curr,
   1.235                                       preserve_ad)) )
   1.236      {
   1.237 -        page_unlock(l3pg);
   1.238          return -EFAULT;
   1.239      }
   1.240  
   1.241 @@ -1834,7 +1812,6 @@ static int mod_l3_entry(l3_pgentry_t *pl
   1.242          pae_flush_pgd(pfn, pgentry_ptr_to_slot(pl3e), nl3e);
   1.243      }
   1.244  
   1.245 -    page_unlock(l3pg);
   1.246      put_page_from_l3e(ol3e, pfn, 0, 0);
   1.247      return rc;
   1.248  }
   1.249 @@ -1851,7 +1828,6 @@ static int mod_l4_entry(l4_pgentry_t *pl
   1.250      struct vcpu *curr = current;
   1.251      struct domain *d = curr->domain;
   1.252      l4_pgentry_t ol4e;
   1.253 -    struct page_info *l4pg = mfn_to_page(pfn);
   1.254      int rc = 0;
   1.255  
   1.256      if ( unlikely(!is_guest_l4_slot(d, pgentry_ptr_to_slot(pl4e))) )
   1.257 @@ -1860,16 +1836,13 @@ static int mod_l4_entry(l4_pgentry_t *pl
   1.258          return -EINVAL;
   1.259      }
   1.260  
   1.261 -    page_lock(l4pg);
   1.262 -
   1.263      if ( unlikely(__copy_from_user(&ol4e, pl4e, sizeof(ol4e)) != 0) )
   1.264 -        return page_unlock(l4pg), -EFAULT;
   1.265 +        return -EFAULT;
   1.266  
   1.267      if ( l4e_get_flags(nl4e) & _PAGE_PRESENT )
   1.268      {
   1.269          if ( unlikely(l4e_get_flags(nl4e) & L4_DISALLOW_MASK) )
   1.270          {
   1.271 -            page_unlock(l4pg);
   1.272              MEM_LOG("Bad L4 flags %x",
   1.273                      l4e_get_flags(nl4e) & L4_DISALLOW_MASK);
   1.274              return -EINVAL;
   1.275 @@ -1880,13 +1853,12 @@ static int mod_l4_entry(l4_pgentry_t *pl
   1.276          {
   1.277              adjust_guest_l4e(nl4e, d);
   1.278              rc = UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn, curr, preserve_ad);
   1.279 -            page_unlock(l4pg);
   1.280              return rc ? 0 : -EFAULT;
   1.281          }
   1.282  
   1.283          rc = get_page_from_l4e(nl4e, pfn, d, 0, preemptible);
   1.284          if ( unlikely(rc < 0) )
   1.285 -            return page_unlock(l4pg), rc;
   1.286 +            return rc;
   1.287          rc = 0;
   1.288  
   1.289          adjust_guest_l4e(nl4e, d);
   1.290 @@ -1900,11 +1872,9 @@ static int mod_l4_entry(l4_pgentry_t *pl
   1.291      else if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn, curr,
   1.292                                       preserve_ad)) )
   1.293      {
   1.294 -        page_unlock(l4pg);
   1.295          return -EFAULT;
   1.296      }
   1.297  
   1.298 -    page_unlock(l4pg);
   1.299      put_page_from_l4e(ol4e, pfn, 0, 0);
   1.300      return rc;
   1.301  }
   1.302 @@ -2963,7 +2933,6 @@ int do_mmu_update(
   1.303      unsigned int cmd, done = 0;
   1.304      struct vcpu *v = current;
   1.305      struct domain *d = v->domain;
   1.306 -    unsigned long type_info;
   1.307      struct domain_mmap_cache mapcache;
   1.308  
   1.309      if ( unlikely(count & MMU_UPDATE_PREEMPTED) )
   1.310 @@ -3035,24 +3004,9 @@ int do_mmu_update(
   1.311                            (unsigned long)(req.ptr & ~PAGE_MASK));
   1.312              page = mfn_to_page(mfn);
   1.313  
   1.314 -            switch ( (type_info = page->u.inuse.type_info) & PGT_type_mask )
   1.315 -            {
   1.316 -            case PGT_l1_page_table:
   1.317 -            case PGT_l2_page_table:
   1.318 -            case PGT_l3_page_table:
   1.319 -            case PGT_l4_page_table:
   1.320 +            if ( page_lock(page) )
   1.321              {
   1.322 -                if ( paging_mode_refcounts(d) )
   1.323 -                {
   1.324 -                    MEM_LOG("mmu update on auto-refcounted domain!");
   1.325 -                    break;
   1.326 -                }
   1.327 -
   1.328 -                if ( unlikely(!get_page_type(
   1.329 -                    page, type_info & (PGT_type_mask|PGT_pae_xen_l2))) )
   1.330 -                    goto not_a_pt;
   1.331 -
   1.332 -                switch ( type_info & PGT_type_mask )
   1.333 +                switch ( page->u.inuse.type_info & PGT_type_mask )
   1.334                  {
   1.335                  case PGT_l1_page_table:
   1.336                  {
   1.337 @@ -3064,7 +3018,7 @@ int do_mmu_update(
   1.338                  case PGT_l2_page_table:
   1.339                  {
   1.340                      l2_pgentry_t l2e = l2e_from_intpte(req.val);
   1.341 -                    okay = mod_l2_entry(va, l2e, mfn, type_info,
   1.342 +                    okay = mod_l2_entry(va, l2e, mfn,
   1.343                                          cmd == MMU_PT_UPDATE_PRESERVE_AD);
   1.344                  }
   1.345                  break;
   1.346 @@ -3086,31 +3040,23 @@ int do_mmu_update(
   1.347                  }
   1.348                  break;
   1.349  #endif
   1.350 +                case PGT_writable_page:
   1.351 +                    perfc_incr(writable_mmu_updates);
   1.352 +                    okay = paging_write_guest_entry(v, va, req.val, _mfn(mfn));
   1.353 +                    break;
   1.354                  }
   1.355 -
   1.356 -                put_page_type(page);
   1.357 +                page_unlock(page);
   1.358                  if ( rc == -EINTR )
   1.359                      rc = -EAGAIN;
   1.360              }
   1.361 -            break;
   1.362 -
   1.363 -            default:
   1.364 -            not_a_pt:
   1.365 +            else if ( get_page_type(page, PGT_writable_page) )
   1.366              {
   1.367 -                if ( unlikely(!get_page_type(page, PGT_writable_page)) )
   1.368 -                    break;
   1.369 -
   1.370                  perfc_incr(writable_mmu_updates);
   1.371 -
   1.372                  okay = paging_write_guest_entry(v, va, req.val, _mfn(mfn));
   1.373 -
   1.374                  put_page_type(page);
   1.375              }
   1.376 -            break;
   1.377 -            }
   1.378  
   1.379              unmap_domain_page_with_cache(va, &mapcache);
   1.380 -
   1.381              put_page(page);
   1.382              break;
   1.383  
   1.384 @@ -3189,7 +3135,6 @@ static int create_grant_pte_mapping(
   1.385      void *va;
   1.386      unsigned long gmfn, mfn;
   1.387      struct page_info *page;
   1.388 -    unsigned long type;
   1.389      l1_pgentry_t ol1e;
   1.390      struct domain *d = v->domain;
   1.391  
   1.392 @@ -3210,21 +3155,23 @@ static int create_grant_pte_mapping(
   1.393      va = (void *)((unsigned long)va + ((unsigned long)pte_addr & ~PAGE_MASK));
   1.394      page = mfn_to_page(mfn);
   1.395  
   1.396 -    type = page->u.inuse.type_info & PGT_type_mask;
   1.397 -    if ( (type != PGT_l1_page_table) || !get_page_type(page, type) )
   1.398 +    if ( !page_lock(page) )
   1.399      {
   1.400 -        MEM_LOG("Grant map attempted to update a non-L1 page");
   1.401          rc = GNTST_general_error;
   1.402          goto failed;
   1.403      }
   1.404  
   1.405 -    page_lock(page);
   1.406 +    if ( (page->u.inuse.type_info & PGT_type_mask) != PGT_l1_page_table )
   1.407 +    {
   1.408 +        page_unlock(page);
   1.409 +        rc = GNTST_general_error;
   1.410 +        goto failed;
   1.411 +    }
   1.412  
   1.413      ol1e = *(l1_pgentry_t *)va;
   1.414      if ( !UPDATE_ENTRY(l1, (l1_pgentry_t *)va, ol1e, nl1e, mfn, v, 0) )
   1.415      {
   1.416          page_unlock(page);
   1.417 -        put_page_type(page);
   1.418          rc = GNTST_general_error;
   1.419          goto failed;
   1.420      } 
   1.421 @@ -3234,8 +3181,6 @@ static int create_grant_pte_mapping(
   1.422      if ( !paging_mode_refcounts(d) )
   1.423          put_page_from_l1e(ol1e, d);
   1.424  
   1.425 -    put_page_type(page);
   1.426 - 
   1.427   failed:
   1.428      unmap_domain_page(va);
   1.429      put_page(page);
   1.430 @@ -3250,7 +3195,6 @@ static int destroy_grant_pte_mapping(
   1.431      void *va;
   1.432      unsigned long gmfn, mfn;
   1.433      struct page_info *page;
   1.434 -    unsigned long type;
   1.435      l1_pgentry_t ol1e;
   1.436  
   1.437      gmfn = addr >> PAGE_SHIFT;
   1.438 @@ -3266,15 +3210,18 @@ static int destroy_grant_pte_mapping(
   1.439      va = (void *)((unsigned long)va + ((unsigned long)addr & ~PAGE_MASK));
   1.440      page = mfn_to_page(mfn);
   1.441  
   1.442 -    type = page->u.inuse.type_info & PGT_type_mask;
   1.443 -    if ( (type != PGT_l1_page_table) || !get_page_type(page, type) )
   1.444 +    if ( !page_lock(page) )
   1.445      {
   1.446 -        MEM_LOG("Grant map attempted to update a non-L1 page");
   1.447          rc = GNTST_general_error;
   1.448          goto failed;
   1.449      }
   1.450  
   1.451 -    page_lock(page);
   1.452 +    if ( (page->u.inuse.type_info & PGT_type_mask) != PGT_l1_page_table )
   1.453 +    {
   1.454 +        page_unlock(page);
   1.455 +        rc = GNTST_general_error;
   1.456 +        goto failed;
   1.457 +    }
   1.458  
   1.459      ol1e = *(l1_pgentry_t *)va;
   1.460      
   1.461 @@ -3284,7 +3231,6 @@ static int destroy_grant_pte_mapping(
   1.462          page_unlock(page);
   1.463          MEM_LOG("PTE entry %lx for address %"PRIx64" doesn't match frame %lx",
   1.464                  (unsigned long)l1e_get_intpte(ol1e), addr, frame);
   1.465 -        put_page_type(page);
   1.466          rc = GNTST_general_error;
   1.467          goto failed;
   1.468      }
   1.469 @@ -3298,13 +3244,11 @@ static int destroy_grant_pte_mapping(
   1.470      {
   1.471          page_unlock(page);
   1.472          MEM_LOG("Cannot delete PTE entry at %p", va);
   1.473 -        put_page_type(page);
   1.474          rc = GNTST_general_error;
   1.475          goto failed;
   1.476      }
   1.477  
   1.478      page_unlock(page);
   1.479 -    put_page_type(page);
   1.480  
   1.481   failed:
   1.482      unmap_domain_page(va);
   1.483 @@ -3332,21 +3276,40 @@ static int create_grant_va_mapping(
   1.484          MEM_LOG("Could not find L1 PTE for address %lx", va);
   1.485          return GNTST_general_error;
   1.486      }
   1.487 +
   1.488 +    if ( !get_page_from_pagenr(gl1mfn, current->domain) )
   1.489 +    {
   1.490 +        guest_unmap_l1e(v, pl1e);
   1.491 +        return GNTST_general_error;
   1.492 +    }
   1.493 +
   1.494      l1pg = mfn_to_page(gl1mfn);
   1.495 -    page_lock(l1pg);
   1.496 +    if ( !page_lock(l1pg) )
   1.497 +    {
   1.498 +        put_page(l1pg);
   1.499 +        guest_unmap_l1e(v, pl1e);
   1.500 +        return GNTST_general_error;
   1.501 +    }
   1.502 +
   1.503 +    if ( (l1pg->u.inuse.type_info & PGT_type_mask) != PGT_l1_page_table )
   1.504 +    {
   1.505 +        page_unlock(l1pg);
   1.506 +        put_page(l1pg);
   1.507 +        guest_unmap_l1e(v, pl1e);
   1.508 +        return GNTST_general_error;
   1.509 +    }
   1.510 +
   1.511      ol1e = *pl1e;
   1.512      okay = UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, v, 0);
   1.513 +
   1.514      page_unlock(l1pg);
   1.515 +    put_page(l1pg);
   1.516      guest_unmap_l1e(v, pl1e);
   1.517 -    pl1e = NULL;
   1.518 -
   1.519 -    if ( !okay )
   1.520 -            return GNTST_general_error;
   1.521 -
   1.522 -    if ( !paging_mode_refcounts(d) )
   1.523 +
   1.524 +    if ( okay && !paging_mode_refcounts(d) )
   1.525          put_page_from_l1e(ol1e, d);
   1.526  
   1.527 -    return GNTST_okay;
   1.528 +    return okay ? GNTST_okay : GNTST_general_error;
   1.529  }
   1.530  
   1.531  static int replace_grant_va_mapping(
   1.532 @@ -3364,31 +3327,48 @@ static int replace_grant_va_mapping(
   1.533          return GNTST_general_error;
   1.534      }
   1.535  
   1.536 +    if ( !get_page_from_pagenr(gl1mfn, current->domain) )
   1.537 +    {
   1.538 +        rc = GNTST_general_error;
   1.539 +        goto out;
   1.540 +    }
   1.541 +
   1.542      l1pg = mfn_to_page(gl1mfn);
   1.543 -    page_lock(l1pg);
   1.544 +    if ( !page_lock(l1pg) )
   1.545 +    {
   1.546 +        rc = GNTST_general_error;
   1.547 +        put_page(l1pg);
   1.548 +        goto out;
   1.549 +    }
   1.550 +
   1.551 +    if ( (l1pg->u.inuse.type_info & PGT_type_mask) != PGT_l1_page_table )
   1.552 +    {
   1.553 +        rc = GNTST_general_error;
   1.554 +        goto unlock_and_out;
   1.555 +    }
   1.556 +
   1.557      ol1e = *pl1e;
   1.558  
   1.559      /* Check that the virtual address supplied is actually mapped to frame. */
   1.560      if ( unlikely(l1e_get_pfn(ol1e) != frame) )
   1.561      {
   1.562 -        page_unlock(l1pg);
   1.563          MEM_LOG("PTE entry %lx for address %lx doesn't match frame %lx",
   1.564                  l1e_get_pfn(ol1e), addr, frame);
   1.565          rc = GNTST_general_error;
   1.566 -        goto out;
   1.567 +        goto unlock_and_out;
   1.568      }
   1.569  
   1.570      /* Delete pagetable entry. */
   1.571      if ( unlikely(!UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, v, 0)) )
   1.572      {
   1.573 -        page_unlock(l1pg);
   1.574          MEM_LOG("Cannot delete PTE entry at %p", (unsigned long *)pl1e);
   1.575          rc = GNTST_general_error;
   1.576 -        goto out;
   1.577 +        goto unlock_and_out;
   1.578      }
   1.579  
   1.580 + unlock_and_out:
   1.581      page_unlock(l1pg);
   1.582 -
   1.583 +    put_page(l1pg);
   1.584   out:
   1.585      guest_unmap_l1e(v, pl1e);
   1.586      return rc;
   1.587 @@ -3450,20 +3430,42 @@ int replace_grant_host_mapping(
   1.588          return GNTST_general_error;
   1.589      }
   1.590  
   1.591 +    if ( !get_page_from_pagenr(gl1mfn, current->domain) )
   1.592 +    {
   1.593 +        guest_unmap_l1e(curr, pl1e);
   1.594 +        return GNTST_general_error;
   1.595 +    }
   1.596 +
   1.597      l1pg = mfn_to_page(gl1mfn);
   1.598 -    page_lock(l1pg);
   1.599 +    if ( !page_lock(l1pg) )
   1.600 +    {
   1.601 +        put_page(l1pg);
   1.602 +        guest_unmap_l1e(curr, pl1e);
   1.603 +        return GNTST_general_error;
   1.604 +    }
   1.605 +
   1.606 +    if ( (l1pg->u.inuse.type_info & PGT_type_mask) != PGT_l1_page_table )
   1.607 +    {
   1.608 +        page_unlock(l1pg);
   1.609 +        put_page(l1pg);
   1.610 +        guest_unmap_l1e(curr, pl1e);
   1.611 +        return GNTST_general_error;
   1.612 +    }
   1.613 +
   1.614      ol1e = *pl1e;
   1.615  
   1.616      if ( unlikely(!UPDATE_ENTRY(l1, pl1e, ol1e, l1e_empty(),
   1.617                                  gl1mfn, curr, 0)) )
   1.618      {
   1.619          page_unlock(l1pg);
   1.620 +        put_page(l1pg);
   1.621          MEM_LOG("Cannot delete PTE entry at %p", (unsigned long *)pl1e);
   1.622          guest_unmap_l1e(curr, pl1e);
   1.623          return GNTST_general_error;
   1.624      }
   1.625  
   1.626      page_unlock(l1pg);
   1.627 +    put_page(l1pg);
   1.628      guest_unmap_l1e(curr, pl1e);
   1.629  
   1.630      rc = replace_grant_va_mapping(addr, frame, ol1e, curr);
   1.631 @@ -3525,28 +3527,45 @@ int do_update_va_mapping(unsigned long v
   1.632      l1_pgentry_t   val = l1e_from_intpte(val64);
   1.633      struct vcpu   *v   = current;
   1.634      struct domain *d   = v->domain;
   1.635 +    struct page_info *gl1pg;
   1.636      l1_pgentry_t  *pl1e;
   1.637      unsigned long  vmask, bmap_ptr, gl1mfn;
   1.638      cpumask_t      pmask;
   1.639 -    int            rc  = 0;
   1.640 +    int            rc;
   1.641  
   1.642      perfc_incr(calls_to_update_va);
   1.643  
   1.644 -    if ( unlikely(!access_ok(va, 1) && !paging_mode_external(d)) )
   1.645 -        return -EINVAL;
   1.646 -
   1.647      rc = xsm_update_va_mapping(d, FOREIGNDOM, val);
   1.648      if ( rc )
   1.649          return rc;
   1.650  
   1.651 +    rc = -EINVAL;
   1.652      pl1e = guest_map_l1e(v, va, &gl1mfn);
   1.653 -
   1.654 -    if ( unlikely(!pl1e || !mod_l1_entry(pl1e, val, gl1mfn, 0)) )
   1.655 -        rc = -EINVAL;
   1.656 -
   1.657 +    if ( unlikely(!pl1e || !get_page_from_pagenr(gl1mfn, d)) )
   1.658 +        goto out;
   1.659 +
   1.660 +    gl1pg = mfn_to_page(gl1mfn);
   1.661 +    if ( !page_lock(gl1pg) )
   1.662 +    {
   1.663 +        put_page(gl1pg);
   1.664 +        goto out;
   1.665 +    }
   1.666 +
   1.667 +    if ( (gl1pg->u.inuse.type_info & PGT_type_mask) != PGT_l1_page_table )
   1.668 +    {
   1.669 +        page_unlock(gl1pg);
   1.670 +        put_page(gl1pg);
   1.671 +        goto out;
   1.672 +    }
   1.673 +
   1.674 +    rc = mod_l1_entry(pl1e, val, gl1mfn, 0) ? 0 : -EINVAL;
   1.675 +
   1.676 +    page_unlock(gl1pg);
   1.677 +    put_page(gl1pg);
   1.678 +
   1.679 + out:
   1.680      if ( pl1e )
   1.681          guest_unmap_l1e(v, pl1e);
   1.682 -    pl1e = NULL;
   1.683  
   1.684      process_deferred_ops();
   1.685  
   1.686 @@ -4223,15 +4242,25 @@ int ptwr_do_page_fault(struct vcpu *v, u
   1.687  
   1.688      /* Attempt to read the PTE that maps the VA being accessed. */
   1.689      guest_get_eff_l1e(v, addr, &pte);
   1.690 -    page = l1e_get_page(pte);
   1.691  
   1.692      /* We are looking only for read-only mappings of p.t. pages. */
   1.693      if ( ((l1e_get_flags(pte) & (_PAGE_PRESENT|_PAGE_RW)) != _PAGE_PRESENT) ||
   1.694 -         !mfn_valid(l1e_get_pfn(pte)) ||
   1.695 -         ((page->u.inuse.type_info & PGT_type_mask) != PGT_l1_page_table) ||
   1.696 -         ((page->u.inuse.type_info & PGT_count_mask) == 0) ||
   1.697 -         (page_get_owner(page) != d) )
   1.698 +         !get_page_from_pagenr(l1e_get_pfn(pte), d) )
   1.699 +        goto bail;
   1.700 +
   1.701 +    page = l1e_get_page(pte);
   1.702 +    if ( !page_lock(page) )
   1.703 +    {
   1.704 +        put_page(page);
   1.705          goto bail;
   1.706 +    }
   1.707 +
   1.708 +    if ( (page->u.inuse.type_info & PGT_type_mask) != PGT_l1_page_table )
   1.709 +    {
   1.710 +        page_unlock(page);
   1.711 +        put_page(page);
   1.712 +        goto bail;
   1.713 +    }
   1.714  
   1.715      ptwr_ctxt.ctxt.regs = regs;
   1.716      ptwr_ctxt.ctxt.force_writeback = 0;
   1.717 @@ -4240,9 +4269,11 @@ int ptwr_do_page_fault(struct vcpu *v, u
   1.718      ptwr_ctxt.cr2 = addr;
   1.719      ptwr_ctxt.pte = pte;
   1.720  
   1.721 -    page_lock(page);
   1.722      rc = x86_emulate(&ptwr_ctxt.ctxt, &ptwr_emulate_ops);
   1.723 +
   1.724      page_unlock(page);
   1.725 +    put_page(page);
   1.726 +
   1.727      if ( rc == X86EMUL_UNHANDLEABLE )
   1.728          goto bail;
   1.729  

     2.1 --- a/xen/arch/x86/mm/shadow/common.c	Tue Jan 27 11:54:22 2009 +0000
     2.2 +++ b/xen/arch/x86/mm/shadow/common.c	Tue Jan 27 16:02:21 2009 +0000
     2.3 @@ -1722,9 +1722,6 @@ shadow_free_p2m_page(struct domain *d, s
     2.4      /* Free should not decrement domain's total allocation, since 
     2.5       * these pages were allocated without an owner. */
     2.6      page_set_owner(pg, NULL); 
     2.7 -#if defined(__x86_64__)
     2.8 -    spin_lock_init(&pg->lock);
     2.9 -#endif
    2.10      free_domheap_pages(pg, 0);
    2.11      d->arch.paging.shadow.p2m_pages--;
    2.12      perfc_decr(shadow_alloc_count);
    2.13 @@ -1833,14 +1830,6 @@ static unsigned int sh_set_allocation(st
    2.14              sp = list_entry(d->arch.paging.shadow.freelists[order].next,
    2.15                              struct shadow_page_info, list);
    2.16              list_del(&sp->list);
    2.17 -#if defined(__x86_64__)
    2.18 -            /*
    2.19 -             * Re-instate lock field which we overwrite with shadow_page_info.
    2.20 -             * This was safe, since the lock is only used on guest pages.
    2.21 -             */
    2.22 -            for ( j = 0; j < 1U << order; j++ )
    2.23 -                spin_lock_init(&((struct page_info *)sp)[j].lock);
    2.24 -#endif
    2.25              d->arch.paging.shadow.free_pages -= 1 << order;
    2.26              d->arch.paging.shadow.total_pages -= 1 << order;
    2.27              free_domheap_pages((struct page_info *)sp, order);

     3.1 --- a/xen/include/asm-x86/mm.h	Tue Jan 27 11:54:22 2009 +0000
     3.2 +++ b/xen/include/asm-x86/mm.h	Tue Jan 27 16:02:21 2009 +0000
     3.3 @@ -46,10 +46,6 @@ struct page_info
     3.4  
     3.5      } u;
     3.6  
     3.7 -#if defined(__x86_64__)
     3.8 -    spinlock_t lock;
     3.9 -#endif
    3.10 -
    3.11      union {
    3.12          /*
    3.13           * Timestamp from 'TLB clock', used to avoid extra safety flushes.
    3.14 @@ -127,23 +123,20 @@ struct page_info
    3.15  /* Has this page been *partially* validated for use as its current type? */
    3.16  #define _PGT_partial      PG_shift(7)
    3.17  #define PGT_partial       PG_mask(1, 7)
    3.18 + /* Page is locked? */
    3.19 +#define _PGT_locked       PG_shift(8)
    3.20 +#define PGT_locked        PG_mask(1, 8)
    3.21  
    3.22   /* Count of uses of this frame as its current type. */
    3.23 -#define PGT_count_width   PG_shift(7)
    3.24 +#define PGT_count_width   PG_shift(8)
    3.25  #define PGT_count_mask    ((1UL<<PGT_count_width)-1)
    3.26  
    3.27   /* Cleared when the owning guest 'frees' this page. */
    3.28  #define _PGC_allocated    PG_shift(1)
    3.29  #define PGC_allocated     PG_mask(1, 1)
    3.30 -#if defined(__i386__)
    3.31 - /* Page is locked? */
    3.32 -# define _PGC_locked      PG_shift(2)
    3.33 -# define PGC_locked       PG_mask(1, 2)
    3.34 -#else
    3.35   /* Page is Xen heap? */
    3.36 -# define _PGC_xen_heap    PG_shift(2)
    3.37 -# define PGC_xen_heap     PG_mask(1, 2)
    3.38 -#endif
    3.39 +#define _PGC_xen_heap     PG_shift(2)
    3.40 +#define PGC_xen_heap      PG_mask(1, 2)
    3.41   /* Set when is using a page as a page table */
    3.42  #define _PGC_page_table   PG_shift(3)
    3.43  #define PGC_page_table    PG_mask(1, 3)

     4.1 --- a/xen/include/asm-x86/paging.h	Tue Jan 27 11:54:22 2009 +0000
     4.2 +++ b/xen/include/asm-x86/paging.h	Tue Jan 27 16:02:21 2009 +0000
     4.3 @@ -336,7 +336,7 @@ void paging_dump_vcpu_info(struct vcpu *
     4.4   * Access to the guest pagetables */
     4.5  
     4.6  /* Get a mapping of a PV guest's l1e for this virtual address. */
     4.7 -static inline void *
     4.8 +static inline l1_pgentry_t *
     4.9  guest_map_l1e(struct vcpu *v, unsigned long addr, unsigned long *gl1mfn)
    4.10  {
    4.11      l2_pgentry_t l2e;
    4.12 @@ -354,15 +354,14 @@ guest_map_l1e(struct vcpu *v, unsigned l
    4.13           != _PAGE_PRESENT )
    4.14          return NULL;
    4.15      *gl1mfn = l2e_get_pfn(l2e);
    4.16 -    return &__linear_l1_table[l1_linear_offset(addr)];
    4.17 +    return (l1_pgentry_t *)map_domain_page(*gl1mfn) + l1_table_offset(addr);
    4.18  }
    4.19  
    4.20  /* Pull down the mapping we got from guest_map_l1e() */
    4.21  static inline void
    4.22  guest_unmap_l1e(struct vcpu *v, void *p)
    4.23  {
    4.24 -    if ( unlikely(paging_mode_translate(v->domain)) )
    4.25 -        unmap_domain_page(p);
    4.26 +    unmap_domain_page(p);
    4.27  }
    4.28  
    4.29  /* Read the guest's l1e that maps this address. */