ia64/xen-unstable

changeset 19098:20f94c2757b8

x86-64: also use 1G page mappings for M2P table

Also, specify the node for the respective page table allocations.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
author Keir Fraser <keir.fraser@citrix.com>
date Tue Jan 27 11:23:30 2009 +0000 (2009-01-27)
parents 70793b2ff15e
children 89dd2bd6031b
files xen/arch/x86/x86_64/mm.c
line diff
     1.1 --- a/xen/arch/x86/x86_64/mm.c	Tue Jan 27 10:25:33 2009 +0000
     1.2 +++ b/xen/arch/x86/x86_64/mm.c	Tue Jan 27 11:23:30 2009 +0000
     1.3 @@ -30,6 +30,7 @@
     1.4  #include <asm/fixmap.h>
     1.5  #include <asm/hypercall.h>
     1.6  #include <asm/msr.h>
     1.7 +#include <asm/numa.h>
     1.8  #include <public/memory.h>
     1.9  
    1.10  #ifdef CONFIG_COMPAT
    1.11 @@ -105,6 +106,7 @@ l2_pgentry_t *virt_to_xen_l2e(unsigned l
    1.12  void __init paging_init(void)
    1.13  {
    1.14      unsigned long i, mpt_size, va;
    1.15 +    unsigned int memflags;
    1.16      l3_pgentry_t *l3_ro_mpt;
    1.17      l2_pgentry_t *l2_ro_mpt = NULL;
    1.18      struct page_info *l1_pg, *l2_pg, *l3_pg;
    1.19 @@ -125,7 +127,36 @@ void __init paging_init(void)
    1.20      mpt_size &= ~((1UL << L2_PAGETABLE_SHIFT) - 1UL);
    1.21      for ( i = 0; i < (mpt_size >> L2_PAGETABLE_SHIFT); i++ )
    1.22      {
    1.23 -        if ( (l1_pg = alloc_domheap_pages(NULL, PAGETABLE_ORDER, 0)) == NULL )
    1.24 +        BUILD_BUG_ON(RO_MPT_VIRT_START & ((1UL << L3_PAGETABLE_SHIFT) - 1));
    1.25 +        va = RO_MPT_VIRT_START + (i << L2_PAGETABLE_SHIFT);
    1.26 +        memflags = MEMF_node(phys_to_nid(i <<
    1.27 +            (L2_PAGETABLE_SHIFT - 3 + PAGE_SHIFT)));
    1.28 +
    1.29 +        if ( cpu_has_page1gb &&
    1.30 +             !((unsigned long)l2_ro_mpt & ~PAGE_MASK) &&
    1.31 +             (mpt_size >> L3_PAGETABLE_SHIFT) > (i >> PAGETABLE_ORDER) &&
    1.32 +             (l1_pg = alloc_domheap_pages(NULL, 2 * PAGETABLE_ORDER,
    1.33 +                                          memflags)) != NULL )
    1.34 +        {
    1.35 +            map_pages_to_xen(
    1.36 +                RDWR_MPT_VIRT_START + (i << L2_PAGETABLE_SHIFT),
    1.37 +                page_to_mfn(l1_pg),
    1.38 +                1UL << (2 * PAGETABLE_ORDER),
    1.39 +                PAGE_HYPERVISOR);
    1.40 +            memset((void *)(RDWR_MPT_VIRT_START + (i << L2_PAGETABLE_SHIFT)),
    1.41 +                   0x77, 1UL << L3_PAGETABLE_SHIFT);
    1.42 +
    1.43 +            ASSERT(!l2_table_offset(va));
    1.44 +            /* NB. Cannot be GLOBAL as shadow_mode_translate reuses this area. */
    1.45 +            l3e_write(&l3_ro_mpt[l3_table_offset(va)],
    1.46 +                l3e_from_page(l1_pg,
    1.47 +                    /*_PAGE_GLOBAL|*/_PAGE_PSE|_PAGE_USER|_PAGE_PRESENT));
    1.48 +            i += (1UL << PAGETABLE_ORDER) - 1;
    1.49 +            continue;
    1.50 +        }
    1.51 +
    1.52 +        if ( (l1_pg = alloc_domheap_pages(NULL, PAGETABLE_ORDER,
    1.53 +                                          memflags)) == NULL )
    1.54              goto nomem;
    1.55          map_pages_to_xen(
    1.56              RDWR_MPT_VIRT_START + (i << L2_PAGETABLE_SHIFT),
    1.57 @@ -136,14 +167,13 @@ void __init paging_init(void)
    1.58                 1UL << L2_PAGETABLE_SHIFT);
    1.59          if ( !((unsigned long)l2_ro_mpt & ~PAGE_MASK) )
    1.60          {
    1.61 -            if ( (l2_pg = alloc_domheap_page(NULL, 0)) == NULL )
    1.62 +            if ( (l2_pg = alloc_domheap_page(NULL, memflags)) == NULL )
    1.63                  goto nomem;
    1.64 -            va = RO_MPT_VIRT_START + (i << L2_PAGETABLE_SHIFT);
    1.65              l2_ro_mpt = page_to_virt(l2_pg);
    1.66              clear_page(l2_ro_mpt);
    1.67              l3e_write(&l3_ro_mpt[l3_table_offset(va)],
    1.68                        l3e_from_page(l2_pg, __PAGE_HYPERVISOR | _PAGE_USER));
    1.69 -            l2_ro_mpt += l2_table_offset(va);
    1.70 +            ASSERT(!l2_table_offset(va));
    1.71          }
    1.72          /* NB. Cannot be GLOBAL as shadow_mode_translate reuses this area. */
    1.73          l2e_write(l2_ro_mpt, l2e_from_page(
    1.74 @@ -172,7 +202,10 @@ void __init paging_init(void)
    1.75          m2p_compat_vstart = MACH2PHYS_COMPAT_VIRT_END - mpt_size;
    1.76      for ( i = 0; i < (mpt_size >> L2_PAGETABLE_SHIFT); i++ )
    1.77      {
    1.78 -        if ( (l1_pg = alloc_domheap_pages(NULL, PAGETABLE_ORDER, 0)) == NULL )
    1.79 +        memflags = MEMF_node(phys_to_nid(i <<
    1.80 +            (L2_PAGETABLE_SHIFT - 2 + PAGE_SHIFT)));
    1.81 +        if ( (l1_pg = alloc_domheap_pages(NULL, PAGETABLE_ORDER,
    1.82 +                                          memflags)) == NULL )
    1.83              goto nomem;
    1.84          map_pages_to_xen(
    1.85              RDWR_COMPAT_MPT_VIRT_START + (i << L2_PAGETABLE_SHIFT),
    1.86 @@ -221,25 +254,36 @@ void __init zap_low_mappings(void)
    1.87  
    1.88  void __init subarch_init_memory(void)
    1.89  {
    1.90 -    unsigned long i, v, m2p_start_mfn;
    1.91 +    unsigned long i, n, v, m2p_start_mfn;
    1.92      l3_pgentry_t l3e;
    1.93      l2_pgentry_t l2e;
    1.94  
    1.95 +    BUILD_BUG_ON(RDWR_MPT_VIRT_START & ((1UL << L3_PAGETABLE_SHIFT) - 1));
    1.96 +    BUILD_BUG_ON(RDWR_MPT_VIRT_END   & ((1UL << L3_PAGETABLE_SHIFT) - 1));
    1.97      /* M2P table is mappable read-only by privileged domains. */
    1.98      for ( v  = RDWR_MPT_VIRT_START;
    1.99            v != RDWR_MPT_VIRT_END;
   1.100 -          v += 1 << L2_PAGETABLE_SHIFT )
   1.101 +          v += n << PAGE_SHIFT )
   1.102      {
   1.103 +        n = L2_PAGETABLE_ENTRIES * L1_PAGETABLE_ENTRIES;
   1.104          l3e = l4e_to_l3e(idle_pg_table[l4_table_offset(v)])[
   1.105              l3_table_offset(v)];
   1.106          if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
   1.107              continue;
   1.108 -        l2e = l3e_to_l2e(l3e)[l2_table_offset(v)];
   1.109 -        if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
   1.110 -            continue;
   1.111 -        m2p_start_mfn = l2e_get_pfn(l2e);
   1.112 +        if ( !(l3e_get_flags(l3e) & _PAGE_PSE) )
   1.113 +        {
   1.114 +            n = L1_PAGETABLE_ENTRIES;
   1.115 +            l2e = l3e_to_l2e(l3e)[l2_table_offset(v)];
   1.116 +            if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
   1.117 +                continue;
   1.118 +            m2p_start_mfn = l2e_get_pfn(l2e);
   1.119 +        }
   1.120 +        else
   1.121 +        {
   1.122 +            m2p_start_mfn = l3e_get_pfn(l3e);
   1.123 +        }
   1.124  
   1.125 -        for ( i = 0; i < L1_PAGETABLE_ENTRIES; i++ )
   1.126 +        for ( i = 0; i < n; i++ )
   1.127          {
   1.128              struct page_info *page = mfn_to_page(m2p_start_mfn + i);
   1.129              share_xen_page_with_privileged_guests(page, XENSHARE_readonly);
   1.130 @@ -283,18 +327,29 @@ long subarch_memory_op(int op, XEN_GUEST
   1.131          if ( copy_from_guest(&xmml, arg, 1) )
   1.132              return -EFAULT;
   1.133  
   1.134 +        BUILD_BUG_ON(RDWR_MPT_VIRT_START & ((1UL << L3_PAGETABLE_SHIFT) - 1));
   1.135 +        BUILD_BUG_ON(RDWR_MPT_VIRT_END   & ((1UL << L3_PAGETABLE_SHIFT) - 1));
   1.136          for ( i = 0, v = RDWR_MPT_VIRT_START;
   1.137                (i != xmml.max_extents) && (v != RDWR_MPT_VIRT_END);
   1.138 -              i++, v += 1 << 21 )
   1.139 +              i++, v += 1UL << L2_PAGETABLE_SHIFT )
   1.140          {
   1.141              l3e = l4e_to_l3e(idle_pg_table[l4_table_offset(v)])[
   1.142                  l3_table_offset(v)];
   1.143              if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
   1.144                  break;
   1.145 -            l2e = l3e_to_l2e(l3e)[l2_table_offset(v)];
   1.146 -            if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
   1.147 -                break;
   1.148 -            mfn = l2e_get_pfn(l2e) + l1_table_offset(v);
   1.149 +            if ( !(l3e_get_flags(l3e) & _PAGE_PSE) )
   1.150 +            {
   1.151 +                l2e = l3e_to_l2e(l3e)[l2_table_offset(v)];
   1.152 +                if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
   1.153 +                    break;
   1.154 +                mfn = l2e_get_pfn(l2e);
   1.155 +            }
   1.156 +            else
   1.157 +            {
   1.158 +                mfn = l3e_get_pfn(l3e)
   1.159 +                    + (l2_table_offset(v) << PAGETABLE_ORDER);
   1.160 +            }
   1.161 +            ASSERT(!l1_table_offset(v));
   1.162              if ( copy_to_guest_offset(xmml.extent_start, i, &mfn, 1) )
   1.163                  return -EFAULT;
   1.164          }