ia64/xen-unstable

changeset 5361:9d9e48be101d

bitkeeper revision 1.1690 (42a5a8ffLN-D5XsB4IKFmZTjaHRH-Q)

More PAE cleanups, this time for the page-type management.
Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Tue Jun 07 14:02:39 2005 +0000 (2005-06-07)
parents 2d8e63df504a
children 731cd57862e5
files xen/arch/x86/mm.c xen/include/asm-x86/x86_32/page-3level.h
line diff
     1.1 --- a/xen/arch/x86/mm.c	Tue Jun 07 11:46:09 2005 +0000
     1.2 +++ b/xen/arch/x86/mm.c	Tue Jun 07 14:02:39 2005 +0000
     1.3 @@ -685,53 +685,78 @@ static int alloc_l1_table(struct pfn_inf
     1.4  }
     1.5  
     1.6  #ifdef CONFIG_X86_PAE
     1.7 -static inline int fixup_pae_linear_mappings(l3_pgentry_t *pl3e)
     1.8 +static int create_pae_xen_mappings(l3_pgentry_t *pl3e)
     1.9  {
    1.10 -    l2_pgentry_t *pl2e;
    1.11 -    unsigned long vaddr;
    1.12 -    int i,idx;
    1.13 -
    1.14 -    while ((unsigned long)pl3e & ~PAGE_MASK)
    1.15 -        pl3e--;
    1.16 -
    1.17 -    if (!(l3e_get_flags(pl3e[3]) & _PAGE_PRESENT)) {
    1.18 -        printk("Installing a L3 PAE pt without L2 in slot #3 isn't going to fly ...\n");
    1.19 +    struct pfn_info *page;
    1.20 +    l2_pgentry_t    *pl2e;
    1.21 +    l3_pgentry_t     l3e3;
    1.22 +    int              i;
    1.23 +
    1.24 +    pl3e = (l3_pgentry_t *)((unsigned long)pl3e & PAGE_MASK);
    1.25 +
    1.26 +    /* 3rd L3 slot contains L2 with Xen-private mappings. It *must* exist. */
    1.27 +    l3e3 = pl3e[3];
    1.28 +    if ( !(l3e_get_flags(l3e3) & _PAGE_PRESENT) )
    1.29 +    {
    1.30 +        MEM_LOG("PAE L3 3rd slot is empty");
    1.31          return 0;
    1.32      }
    1.33  
    1.34 -    pl2e = map_domain_page(l3e_get_pfn(pl3e[3]));
    1.35 -    for (i = 0; i < 4; i++) {
    1.36 -        vaddr = LINEAR_PT_VIRT_START + (i << L2_PAGETABLE_SHIFT);
    1.37 -        idx = (vaddr >> L2_PAGETABLE_SHIFT) & (L2_PAGETABLE_ENTRIES-1);
    1.38 -        if (l3e_get_flags(pl3e[i]) & _PAGE_PRESENT) {
    1.39 -            pl2e[idx] = l2e_from_paddr(l3e_get_paddr(pl3e[i]),
    1.40 -                                       __PAGE_HYPERVISOR);
    1.41 -        } else
    1.42 -            pl2e[idx] = l2e_empty();
    1.43 +    /*
    1.44 +     * The Xen-private mappings include linear mappings. The L2 thus cannot
    1.45 +     * be shared by multiple L3 tables. The test here is adequate because:
    1.46 +     *  1. Cannot appear in slots != 3 because the page would then then have
    1.47 +     *     unknown va backpointer, which get_page_type() explicitly disallows.
    1.48 +     *  2. Cannot appear in another page table's L3:
    1.49 +     *     a. alloc_l3_table() calls this function and this check will fail
    1.50 +     *     b. mod_l3_entry() disallows updates to slot 3 in an existing table
    1.51 +     */
    1.52 +    page = l3e_get_page(l3e3);
    1.53 +    BUG_ON(page->u.inuse.type_info & PGT_pinned);
    1.54 +    BUG_ON((page->u.inuse.type_info & PGT_count_mask) == 0);
    1.55 +    if ( (page->u.inuse.type_info & PGT_count_mask) != 1 )
    1.56 +    {
    1.57 +        MEM_LOG("PAE L3 3rd slot is shared");
    1.58 +        return 0;
    1.59      }
    1.60 +
    1.61 +    /* Xen private mappings. */
    1.62 +    pl2e = map_domain_page(l3e_get_pfn(l3e3));
    1.63 +    memcpy(&pl2e[L2_PAGETABLE_FIRST_XEN_SLOT & (L2_PAGETABLE_ENTRIES-1)],
    1.64 +           &idle_pg_table_l2[L2_PAGETABLE_FIRST_XEN_SLOT],
    1.65 +           L2_PAGETABLE_XEN_SLOTS * sizeof(l2_pgentry_t));
    1.66 +    for ( i = 0; i < (PERDOMAIN_MBYTES >> (L2_PAGETABLE_SHIFT - 20)); i++ )
    1.67 +        pl2e[l2_table_offset(PERDOMAIN_VIRT_START) + i] =
    1.68 +            l2e_from_page(
    1.69 +                virt_to_page(page_get_owner(page)->arch.mm_perdomain_pt) + i,
    1.70 +                __PAGE_HYPERVISOR);
    1.71 +    for ( i = 0; i < (LINEARPT_MBYTES >> (L2_PAGETABLE_SHIFT - 20)); i++ )
    1.72 +        pl2e[l2_table_offset(LINEAR_PT_VIRT_START) + i] =
    1.73 +            (l3e_get_flags(pl3e[i]) & _PAGE_PRESENT) ?
    1.74 +            l2e_from_pfn(l3e_get_pfn(pl3e[i]), __PAGE_HYPERVISOR) :
    1.75 +            l2e_empty();
    1.76      unmap_domain_page(pl2e);
    1.77  
    1.78      return 1;
    1.79  }
    1.80  
    1.81 -static inline unsigned long fixup_pae_vaddr(unsigned long l2vaddr,
    1.82 -                                            unsigned long l2type)
    1.83 +static inline int l1_backptr(
    1.84 +    unsigned long *backptr, unsigned long offset_in_l2, unsigned long l2_type)
    1.85  {
    1.86 -    unsigned long l3vaddr;
    1.87 -    
    1.88 -    if ((l2type & PGT_va_mask) == PGT_va_unknown) {
    1.89 -        printk("%s: hooking one l2 pt into multiple l3 slots isn't allowed, sorry\n",
    1.90 -               __FUNCTION__);
    1.91 -        domain_crash();
    1.92 -    }
    1.93 -    l3vaddr = ((l2type & PGT_va_mask) >> PGT_va_shift)
    1.94 -        << L3_PAGETABLE_SHIFT;
    1.95 -    return l3vaddr + l2vaddr;
    1.96 +    unsigned long l2_backptr = l2_type & PGT_va_mask;
    1.97 +    BUG_ON(l2_backptr == PGT_va_unknown);
    1.98 +    if ( l2_backptr == PGT_va_mutable )
    1.99 +        return 0;
   1.100 +    *backptr = 
   1.101 +        ((l2_backptr >> PGT_va_shift) << L3_PAGETABLE_SHIFT) | 
   1.102 +        (offset_in_l2 << L2_PAGETABLE_SHIFT);
   1.103 +    return 1;
   1.104  }
   1.105  
   1.106  #else
   1.107 -# define fixup_pae_linear_mappings(unused) (1)
   1.108 -# define fixup_pae_vaddr(vaddr, type) (vaddr)
   1.109 +# define create_pae_xen_mappings(pl3e) (1)
   1.110 +# define l1_backptr(bp,l2o,l2t) \
   1.111 +    ({ *(bp) = (l2o) << L2_PAGETABLE_SHIFT; 1; })
   1.112  #endif
   1.113  
   1.114  static int alloc_l2_table(struct pfn_info *page, unsigned int type)
   1.115 @@ -742,18 +767,18 @@ static int alloc_l2_table(struct pfn_inf
   1.116      l2_pgentry_t  *pl2e;
   1.117      int            i;
   1.118  
   1.119 -    // See the code in shadow_promote() to understand why this is here...
   1.120 +    /* See the code in shadow_promote() to understand why this is here. */
   1.121      if ( (PGT_base_page_table == PGT_l2_page_table) &&
   1.122           unlikely(shadow_mode_refcounts(d)) )
   1.123          return 1;
   1.124 -    ASSERT( !shadow_mode_refcounts(d) );
   1.125 -   
   1.126 +    ASSERT(!shadow_mode_refcounts(d));
   1.127      
   1.128      pl2e = map_domain_page(pfn);
   1.129  
   1.130 -    for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ ) {
   1.131 -        vaddr = i << L2_PAGETABLE_SHIFT;
   1.132 -        vaddr = fixup_pae_vaddr(vaddr,type);
   1.133 +    for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ )
   1.134 +    {
   1.135 +        if ( !l1_backptr(&vaddr, i, type) )
   1.136 +            goto fail;
   1.137          if ( is_guest_l2_slot(type, i) &&
   1.138               unlikely(!get_page_from_l2e(pl2e[i], pfn, d, vaddr)) )
   1.139              goto fail;
   1.140 @@ -771,24 +796,6 @@ static int alloc_l2_table(struct pfn_inf
   1.141              virt_to_page(page_get_owner(page)->arch.mm_perdomain_pt),
   1.142              __PAGE_HYPERVISOR);
   1.143  #endif
   1.144 -#if CONFIG_PAGING_LEVELS == 3
   1.145 -    if (3 == ((type & PGT_va_mask) >> PGT_va_shift)) {
   1.146 -        unsigned long v,src,dst;
   1.147 -        void *virt;
   1.148 -        /* Xen private mappings. */
   1.149 -        dst = L2_PAGETABLE_FIRST_XEN_SLOT & (L2_PAGETABLE_ENTRIES-1);
   1.150 -        src = L2_PAGETABLE_FIRST_XEN_SLOT;
   1.151 -        memcpy(&pl2e[dst], &idle_pg_table_l2[src],
   1.152 -               L2_PAGETABLE_XEN_SLOTS * sizeof(l2_pgentry_t));
   1.153 -        for (v = PERDOMAIN_VIRT_START; v < PERDOMAIN_VIRT_END;
   1.154 -             v += (1 << L2_PAGETABLE_SHIFT)) {
   1.155 -            dst = (v >> L2_PAGETABLE_SHIFT) & (L2_PAGETABLE_ENTRIES-1);
   1.156 -            virt = page_get_owner(page)->arch.mm_perdomain_pt + (v-PERDOMAIN_VIRT_START);
   1.157 -            pl2e[dst] = l2e_from_page(virt_to_page(virt), __PAGE_HYPERVISOR);
   1.158 -        }
   1.159 -        /* see fixup_pae_linear_mappings() for linear pagetables */
   1.160 -    }
   1.161 -#endif
   1.162  
   1.163      unmap_domain_page(pl2e);
   1.164      return 1;
   1.165 @@ -804,7 +811,6 @@ static int alloc_l2_table(struct pfn_inf
   1.166  
   1.167  
   1.168  #if CONFIG_PAGING_LEVELS >= 3
   1.169 -
   1.170  static int alloc_l3_table(struct pfn_info *page)
   1.171  {
   1.172      struct domain *d = page_get_owner(page);
   1.173 @@ -813,18 +819,20 @@ static int alloc_l3_table(struct pfn_inf
   1.174      l3_pgentry_t  *pl3e;
   1.175      int            i;
   1.176  
   1.177 -    ASSERT( !shadow_mode_refcounts(d) );
   1.178 +    ASSERT(!shadow_mode_refcounts(d));
   1.179  
   1.180      pl3e = map_domain_page(pfn);
   1.181 -    for ( i = 0; i < L3_PAGETABLE_ENTRIES; i++ ) {
   1.182 +    for ( i = 0; i < L3_PAGETABLE_ENTRIES; i++ )
   1.183 +    {
   1.184          vaddr = i << L3_PAGETABLE_SHIFT;
   1.185          if ( is_guest_l3_slot(i) &&
   1.186               unlikely(!get_page_from_l3e(pl3e[i], pfn, d, vaddr)) )
   1.187              goto fail;
   1.188      }
   1.189  
   1.190 -    if (!fixup_pae_linear_mappings(pl3e))
   1.191 +    if ( !create_pae_xen_mappings(pl3e) )
   1.192          goto fail;
   1.193 +
   1.194      unmap_domain_page(pl3e);
   1.195      return 1;
   1.196  
   1.197 @@ -836,11 +844,11 @@ static int alloc_l3_table(struct pfn_inf
   1.198      unmap_domain_page(pl3e);
   1.199      return 0;
   1.200  }
   1.201 -
   1.202 +#else
   1.203 +#define alloc_l3_table(page) (0)
   1.204  #endif
   1.205  
   1.206  #if CONFIG_PAGING_LEVELS >= 4
   1.207 -
   1.208  static int alloc_l4_table(struct pfn_info *page)
   1.209  {
   1.210      struct domain *d = page_get_owner(page);
   1.211 @@ -848,12 +856,11 @@ static int alloc_l4_table(struct pfn_inf
   1.212      l4_pgentry_t  *pl4e = page_to_virt(page);
   1.213      int            i;
   1.214  
   1.215 -    // See the code in shadow_promote() to understand why this is here...
   1.216 +    /* See the code in shadow_promote() to understand why this is here. */
   1.217      if ( (PGT_base_page_table == PGT_l4_page_table) &&
   1.218           shadow_mode_refcounts(d) )
   1.219          return 1;
   1.220 -
   1.221 -    ASSERT( !shadow_mode_refcounts(d) );
   1.222 +    ASSERT(!shadow_mode_refcounts(d));
   1.223  
   1.224      for ( i = 0; i < L4_PAGETABLE_ENTRIES; i++ )
   1.225          if ( is_guest_l4_slot(i) &&
   1.226 @@ -880,8 +887,9 @@ static int alloc_l4_table(struct pfn_inf
   1.227  
   1.228      return 0;
   1.229  }
   1.230 -
   1.231 -#endif /* __x86_64__ */
   1.232 +#else
   1.233 +#define alloc_l4_table(page) (0)
   1.234 +#endif
   1.235  
   1.236  
   1.237  static void free_l1_table(struct pfn_info *page)
   1.238 @@ -909,10 +917,9 @@ static void free_l2_table(struct pfn_inf
   1.239  
   1.240      pl2e = map_domain_page(pfn);
   1.241  
   1.242 -    for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ ) {
   1.243 +    for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ )
   1.244          if ( is_guest_l2_slot(page->u.inuse.type_info, i) )
   1.245              put_page_from_l2e(pl2e[i], pfn);
   1.246 -    }
   1.247  
   1.248      unmap_domain_page(pl2e);
   1.249  }
   1.250 @@ -1060,10 +1067,8 @@ static int mod_l2_entry(l2_pgentry_t *pl
   1.251          if ( !l2e_has_changed(ol2e, nl2e, _PAGE_PRESENT))
   1.252              return UPDATE_ENTRY(l2, pl2e, ol2e, nl2e);
   1.253  
   1.254 -        vaddr = (((unsigned long)pl2e & ~PAGE_MASK) / sizeof(l2_pgentry_t))
   1.255 -            << L2_PAGETABLE_SHIFT;
   1.256 -        vaddr = fixup_pae_vaddr(vaddr,type);
   1.257 -        if ( unlikely(!get_page_from_l2e(nl2e, pfn, current->domain, vaddr)) )
   1.258 +        if ( unlikely(!l1_backptr(&vaddr, pgentry_ptr_to_slot(pl2e), type)) ||
   1.259 +             unlikely(!get_page_from_l2e(nl2e, pfn, current->domain, vaddr)) )
   1.260              return 0;
   1.261  
   1.262          if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e)) )
   1.263 @@ -1099,6 +1104,15 @@ static int mod_l3_entry(l3_pgentry_t *pl
   1.264          return 0;
   1.265      }
   1.266  
   1.267 +#ifdef CONFIG_PAE
   1.268 +    /*
   1.269 +     * Disallow updates to final L3 slot. It contains Xen mappings, and it
   1.270 +     * would be a pain to ensure they remain continuously valid throughout.
   1.271 +     */
   1.272 +    if ( pgentry_ptr_to_slot(pl3e) >= 3 )
   1.273 +        return 0;
   1.274 +#endif
   1.275 +
   1.276      if ( unlikely(__copy_from_user(&ol3e, pl3e, sizeof(ol3e)) != 0) )
   1.277          return 0;
   1.278  
   1.279 @@ -1120,9 +1134,9 @@ static int mod_l3_entry(l3_pgentry_t *pl
   1.280          if ( unlikely(!get_page_from_l3e(nl3e, pfn, current->domain, vaddr)) )
   1.281              return 0;
   1.282  
   1.283 -        if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e) ||
   1.284 -                      !fixup_pae_linear_mappings(pl3e)) )
   1.285 +        if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e)) )
   1.286          {
   1.287 +            BUG_ON(!create_pae_xen_mappings(pl3e));
   1.288              put_page_from_l3e(nl3e, pfn);
   1.289              return 0;
   1.290          }
   1.291 @@ -1131,9 +1145,11 @@ static int mod_l3_entry(l3_pgentry_t *pl
   1.292          return 1;
   1.293      }
   1.294  
   1.295 -    if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e) ||
   1.296 -                  !fixup_pae_linear_mappings(pl3e)) )
   1.297 +    if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e)) )
   1.298 +    {
   1.299 +        BUG_ON(!create_pae_xen_mappings(pl3e));
   1.300          return 0;
   1.301 +    }
   1.302  
   1.303      put_page_from_l3e(ol3e, pfn);
   1.304      return 1;
   1.305 @@ -1202,14 +1218,10 @@ int alloc_page_type(struct pfn_info *pag
   1.306          return alloc_l1_table(page);
   1.307      case PGT_l2_page_table:
   1.308          return alloc_l2_table(page, type);
   1.309 -#if CONFIG_PAGING_LEVELS >= 3
   1.310      case PGT_l3_page_table:
   1.311          return alloc_l3_table(page);
   1.312 -#endif
   1.313 -#if CONFIG_PAGING_LEVELS >= 4
   1.314      case PGT_l4_page_table:
   1.315          return alloc_l4_table(page);
   1.316 -#endif
   1.317      case PGT_gdt_page:
   1.318      case PGT_ldt_page:
   1.319          return alloc_segdesc_page(page);
   1.320 @@ -1388,13 +1400,14 @@ int get_page_type(struct pfn_info *page,
   1.321                  else if ( ((type & PGT_va_mask) != PGT_va_mutable) &&
   1.322                            ((type & PGT_va_mask) != (x & PGT_va_mask)) )
   1.323                  {
   1.324 -                    /* This table is may be mapped at multiple locations. */
   1.325 +#ifdef CONFIG_PAE
   1.326 +                    /* We use backptr as extra typing. Cannot be unknown. */
   1.327 +                    if ( (type & PGT_type_mask) == PGT_l2_page_table )
   1.328 +                        return 0;
   1.329 +#endif
   1.330 +                    /* This table is possibly mapped at multiple locations. */
   1.331                      nx &= ~PGT_va_mask;
   1.332                      nx |= PGT_va_unknown;
   1.333 -#if 0 /* debug */
   1.334 -                    printk("%s: pfn %lx type %x -> %x (tag as unknown)\n",
   1.335 -                           __FUNCTION__,page_to_pfn(page),x,nx);
   1.336 -#endif
   1.337                  }
   1.338              }
   1.339              if ( unlikely(!(x & PGT_validated)) )
   1.340 @@ -1658,21 +1671,19 @@ int do_mmuext_op(
   1.341              
   1.342              break;
   1.343  
   1.344 +#ifndef CONFIG_PAE /* Unsafe on PAE because of Xen-private mappings. */
   1.345          case MMUEXT_PIN_L2_TABLE:
   1.346              type = PGT_l2_page_table;
   1.347              goto pin_page;
   1.348 -
   1.349 -#if CONFIG_PAGING_LEVELS >= 3
   1.350 +#endif
   1.351 +
   1.352          case MMUEXT_PIN_L3_TABLE:
   1.353              type = PGT_l3_page_table;
   1.354              goto pin_page;
   1.355 -#endif
   1.356 -
   1.357 -#if CONFIG_PAGING_LEVELS >= 4
   1.358 +
   1.359          case MMUEXT_PIN_L4_TABLE:
   1.360              type = PGT_l4_page_table;
   1.361              goto pin_page;
   1.362 -#endif
   1.363  
   1.364          case MMUEXT_UNPIN_TABLE:
   1.365              if ( unlikely(!(okay = get_page_from_pagenr(op.mfn, FOREIGNDOM))) )
     2.1 --- a/xen/include/asm-x86/x86_32/page-3level.h	Tue Jun 07 11:46:09 2005 +0000
     2.2 +++ b/xen/include/asm-x86/x86_32/page-3level.h	Tue Jun 07 14:02:39 2005 +0000
     2.3 @@ -47,11 +47,11 @@ typedef l3_pgentry_t root_pgentry_t;
     2.4  #define PGT_root_page_table       PGT_l3_page_table
     2.5  
     2.6  /* misc */
     2.7 -#define is_guest_l1_slot(_s)    (1)
     2.8 -#define is_guest_l2_slot(_t,_s) \
     2.9 -    ((3 != (((_t) & PGT_va_mask) >> PGT_va_shift)) || \
    2.10 -     ((_s) < (L2_PAGETABLE_FIRST_XEN_SLOT & (L2_PAGETABLE_ENTRIES-1))))
    2.11 -#define is_guest_l3_slot(_s)    (1)
    2.12 +#define is_guest_l1_slot(s)    (1)
    2.13 +#define is_guest_l2_slot(t,s)                                              \
    2.14 +    ( ((((t) & PGT_va_mask) >> PGT_va_shift) != 3) ||                      \
    2.15 +      ((s) < (L2_PAGETABLE_FIRST_XEN_SLOT & (L2_PAGETABLE_ENTRIES - 1))) )
    2.16 +#define is_guest_l3_slot(s)    (1)
    2.17  
    2.18  /*
    2.19   * PTE pfn and flags: