ia64/xen-unstable

changeset 8565:57c50578414d

Add a per-vcpu lock-free cache of mappings to map_domain_page.
Improve mapping slot search by using a free bitmap.
Removed map_domain_pages (multi-page mapping functionality).
Noone uses it, nor should they use it in future,

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Wed Jan 11 19:18:21 2006 +0100 (2006-01-11)
parents 1c70b9d81731
children 228c96d95c80 6ccee759e34f
files xen/arch/x86/domain.c xen/arch/x86/x86_32/domain_page.c xen/include/asm-x86/domain.h xen/include/xen/domain_page.h
line diff
     1.1 --- a/xen/arch/x86/domain.c	Wed Jan 11 16:26:04 2006 +0000
     1.2 +++ b/xen/arch/x86/domain.c	Wed Jan 11 19:18:21 2006 +0100
     1.3 @@ -288,9 +288,7 @@ int arch_do_createdomain(struct vcpu *v)
     1.4  
     1.5  #if defined(__i386__)
     1.6  
     1.7 -    d->arch.mapcache.l1tab = d->arch.mm_perdomain_pt +
     1.8 -        (GDT_LDT_MBYTES << (20 - PAGE_SHIFT));
     1.9 -    spin_lock_init(&d->arch.mapcache.lock);
    1.10 +    mapcache_init(d);
    1.11  
    1.12  #else /* __x86_64__ */
    1.13  
     2.1 --- a/xen/arch/x86/x86_32/domain_page.c	Wed Jan 11 16:26:04 2006 +0000
     2.2 +++ b/xen/arch/x86/x86_32/domain_page.c	Wed Jan 11 19:18:21 2006 +0100
     2.3 @@ -20,33 +20,16 @@
     2.4  #include <asm/flushtlb.h>
     2.5  #include <asm/hardirq.h>
     2.6  
     2.7 -#define MAPCACHE_ORDER    10
     2.8 -#define MAPCACHE_ENTRIES  (1 << MAPCACHE_ORDER)
     2.9 -
    2.10 -/* Use a spare PTE bit to mark entries ready for recycling. */
    2.11 -#define READY_FOR_TLB_FLUSH (1<<10)
    2.12 -
    2.13 -static void flush_all_ready_maps(void)
    2.14 -{
    2.15 -    struct mapcache *cache = &current->domain->arch.mapcache;
    2.16 -    unsigned int i;
    2.17 -
    2.18 -    for ( i = 0; i < MAPCACHE_ENTRIES; i++ )
    2.19 -        if ( (l1e_get_flags(cache->l1tab[i]) & READY_FOR_TLB_FLUSH) )
    2.20 -            cache->l1tab[i] = l1e_empty();
    2.21 -}
    2.22 -
    2.23 -void *map_domain_pages(unsigned long pfn, unsigned int order)
    2.24 +void *map_domain_page(unsigned long pfn)
    2.25  {
    2.26      unsigned long va;
    2.27 -    unsigned int idx, i, flags, vcpu = current->vcpu_id;
    2.28 +    unsigned int idx, i, vcpu = current->vcpu_id;
    2.29      struct domain *d;
    2.30      struct mapcache *cache;
    2.31 -#ifndef NDEBUG
    2.32 -    unsigned int flush_count = 0;
    2.33 -#endif
    2.34 +    struct vcpu_maphash_entry *hashent;
    2.35  
    2.36      ASSERT(!in_irq());
    2.37 +
    2.38      perfc_incrc(map_domain_page_count);
    2.39  
    2.40      /* If we are the idle domain, ensure that we run on our own page tables. */
    2.41 @@ -56,6 +39,16 @@ void *map_domain_pages(unsigned long pfn
    2.42  
    2.43      cache = &d->arch.mapcache;
    2.44  
    2.45 +    hashent = &cache->vcpu_maphash[vcpu].hash[MAPHASH_HASHFN(pfn)];
    2.46 +    if ( hashent->pfn == pfn )
    2.47 +    {
    2.48 +        idx = hashent->idx;
    2.49 +        hashent->refcnt++;
    2.50 +        ASSERT(hashent->refcnt != 0);
    2.51 +        ASSERT(l1e_get_pfn(cache->l1tab[idx]) == pfn);
    2.52 +        goto out;
    2.53 +    }
    2.54 +
    2.55      spin_lock(&cache->lock);
    2.56  
    2.57      /* Has some other CPU caused a wrap? We must flush if so. */
    2.58 @@ -70,45 +63,97 @@ void *map_domain_pages(unsigned long pfn
    2.59          }
    2.60      }
    2.61  
    2.62 -    do {
    2.63 -        idx = cache->cursor = (cache->cursor + 1) & (MAPCACHE_ENTRIES - 1);
    2.64 -        if ( unlikely(idx == 0) )
    2.65 +    idx = find_next_zero_bit(cache->inuse, MAPCACHE_ENTRIES, cache->cursor);
    2.66 +    if ( unlikely(idx >= MAPCACHE_ENTRIES) )
    2.67 +    {
    2.68 +        /* /First/, clean the garbage map and update the inuse list. */
    2.69 +        for ( i = 0; i < ARRAY_SIZE(cache->garbage); i++ )
    2.70          {
    2.71 -            ASSERT(flush_count++ == 0);
    2.72 -            flush_all_ready_maps();
    2.73 -            perfc_incrc(domain_page_tlb_flush);
    2.74 -            local_flush_tlb();
    2.75 -            cache->shadow_epoch[vcpu] = ++cache->epoch;
    2.76 -            cache->tlbflush_timestamp = tlbflush_current_time();
    2.77 +            unsigned long x = xchg(&cache->garbage[i], 0);
    2.78 +            cache->inuse[i] &= ~x;
    2.79          }
    2.80  
    2.81 -        flags = 0;
    2.82 -        for ( i = 0; i < (1U << order); i++ )
    2.83 -            flags |= l1e_get_flags(cache->l1tab[idx+i]);
    2.84 +        /* /Second/, flush TLBs. */
    2.85 +        perfc_incrc(domain_page_tlb_flush);
    2.86 +        local_flush_tlb();
    2.87 +        cache->shadow_epoch[vcpu] = ++cache->epoch;
    2.88 +        cache->tlbflush_timestamp = tlbflush_current_time();
    2.89 +
    2.90 +        idx = find_first_zero_bit(cache->inuse, MAPCACHE_ENTRIES);
    2.91 +        ASSERT(idx < MAPCACHE_ENTRIES);
    2.92      }
    2.93 -    while ( flags & _PAGE_PRESENT );
    2.94  
    2.95 -    for ( i = 0; i < (1U << order); i++ )
    2.96 -        cache->l1tab[idx+i] = l1e_from_pfn(pfn+i, __PAGE_HYPERVISOR);
    2.97 +    set_bit(idx, cache->inuse);
    2.98 +    cache->cursor = idx + 1;
    2.99  
   2.100      spin_unlock(&cache->lock);
   2.101  
   2.102 +    cache->l1tab[idx] = l1e_from_pfn(pfn, __PAGE_HYPERVISOR);
   2.103 +
   2.104 + out:
   2.105      va = MAPCACHE_VIRT_START + (idx << PAGE_SHIFT);
   2.106      return (void *)va;
   2.107  }
   2.108  
   2.109 -void unmap_domain_pages(void *va, unsigned int order)
   2.110 +void unmap_domain_page(void *va)
   2.111  {
   2.112 -    unsigned int idx, i;
   2.113 +    unsigned int idx;
   2.114      struct mapcache *cache = &current->domain->arch.mapcache;
   2.115 +    unsigned long pfn;
   2.116 +    struct vcpu_maphash_entry *hashent;
   2.117 +
   2.118 +    ASSERT(!in_irq());
   2.119  
   2.120      ASSERT((void *)MAPCACHE_VIRT_START <= va);
   2.121      ASSERT(va < (void *)MAPCACHE_VIRT_END);
   2.122  
   2.123      idx = ((unsigned long)va - MAPCACHE_VIRT_START) >> PAGE_SHIFT;
   2.124 +    pfn = l1e_get_pfn(cache->l1tab[idx]);
   2.125 +    hashent = &cache->vcpu_maphash[current->vcpu_id].hash[MAPHASH_HASHFN(pfn)];
   2.126  
   2.127 -    for ( i = 0; i < (1U << order); i++ )
   2.128 -        l1e_add_flags(cache->l1tab[idx+i], READY_FOR_TLB_FLUSH);
   2.129 +    if ( hashent->idx == idx )
   2.130 +    {
   2.131 +        ASSERT(hashent->pfn == pfn);
   2.132 +        ASSERT(hashent->refcnt != 0);
   2.133 +        hashent->refcnt--;
   2.134 +    }
   2.135 +    else if ( hashent->refcnt == 0 )
   2.136 +    {
   2.137 +        if ( hashent->idx != MAPHASHENT_NOTINUSE )
   2.138 +        {
   2.139 +            /* /First/, zap the PTE. */
   2.140 +            ASSERT(l1e_get_pfn(cache->l1tab[hashent->idx]) == hashent->pfn);
   2.141 +            cache->l1tab[hashent->idx] = l1e_empty();
   2.142 +            /* /Second/, mark as garbage. */
   2.143 +            set_bit(hashent->idx, cache->garbage);
   2.144 +        }
   2.145 +
   2.146 +        /* Add newly-freed mapping to the maphash. */
   2.147 +        hashent->pfn = pfn;
   2.148 +        hashent->idx = idx;
   2.149 +    }
   2.150 +    else
   2.151 +    {
   2.152 +        /* /First/, zap the PTE. */
   2.153 +        cache->l1tab[idx] = l1e_empty();
   2.154 +        /* /Second/, mark as garbage. */
   2.155 +        set_bit(idx, cache->garbage);
   2.156 +    }
   2.157 +}
   2.158 +
   2.159 +void mapcache_init(struct domain *d)
   2.160 +{
   2.161 +    unsigned int i, j;
   2.162 +
   2.163 +    d->arch.mapcache.l1tab = d->arch.mm_perdomain_pt +
   2.164 +        (GDT_LDT_MBYTES << (20 - PAGE_SHIFT));
   2.165 +    spin_lock_init(&d->arch.mapcache.lock);
   2.166 +
   2.167 +    /* Mark all maphash entries as not in use. */
   2.168 +    for ( i = 0; i < MAX_VIRT_CPUS; i++ )
   2.169 +        for ( j = 0; j < MAPHASH_ENTRIES; j++ )
   2.170 +            d->arch.mapcache.vcpu_maphash[i].hash[j].idx =
   2.171 +                MAPHASHENT_NOTINUSE;
   2.172  }
   2.173  
   2.174  #define GLOBALMAP_BITS (IOREMAP_MBYTES << (20 - PAGE_SHIFT))
   2.175 @@ -128,15 +173,10 @@ void *map_domain_page_global(unsigned lo
   2.176  
   2.177      spin_lock(&globalmap_lock);
   2.178  
   2.179 -    for ( ; ; )
   2.180 +    idx = find_next_zero_bit(inuse, GLOBALMAP_BITS, inuse_cursor);
   2.181 +    va = IOREMAP_VIRT_START + (idx << PAGE_SHIFT);
   2.182 +    if ( unlikely(va >= FIXADDR_START) )
   2.183      {
   2.184 -        idx = find_next_zero_bit(inuse, GLOBALMAP_BITS, inuse_cursor);
   2.185 -        va = IOREMAP_VIRT_START + (idx << PAGE_SHIFT);
   2.186 -
   2.187 -        /* End of round? If not then we're done in this loop. */
   2.188 -        if ( va < FIXADDR_START )
   2.189 -            break;
   2.190 -
   2.191          /* /First/, clean the garbage map and update the inuse list. */
   2.192          for ( i = 0; i < ARRAY_SIZE(garbage); i++ )
   2.193          {
   2.194 @@ -147,7 +187,9 @@ void *map_domain_page_global(unsigned lo
   2.195          /* /Second/, flush all TLBs to get rid of stale garbage mappings. */
   2.196          flush_tlb_all();
   2.197  
   2.198 -        inuse_cursor = 0;
   2.199 +        idx = find_first_zero_bit(inuse, GLOBALMAP_BITS);
   2.200 +        va = IOREMAP_VIRT_START + (idx << PAGE_SHIFT);
   2.201 +        ASSERT(va < FIXADDR_START);
   2.202      }
   2.203  
   2.204      set_bit(idx, inuse);
     3.1 --- a/xen/include/asm-x86/domain.h	Wed Jan 11 16:26:04 2006 +0000
     3.2 +++ b/xen/include/asm-x86/domain.h	Wed Jan 11 19:18:21 2006 +0100
     3.3 @@ -13,14 +13,41 @@ struct trap_bounce {
     3.4      unsigned long  eip;
     3.5  };
     3.6  
     3.7 +#define MAPHASH_ENTRIES 8
     3.8 +#define MAPHASH_HASHFN(pfn) ((pfn) & (MAPHASH_ENTRIES-1))
     3.9 +#define MAPHASHENT_NOTINUSE ((u16)~0U)
    3.10 +struct vcpu_maphash {
    3.11 +    struct vcpu_maphash_entry {
    3.12 +        unsigned long pfn;
    3.13 +        uint16_t      idx;
    3.14 +        uint16_t      refcnt;
    3.15 +    } hash[MAPHASH_ENTRIES];
    3.16 +} __cacheline_aligned;
    3.17 +
    3.18 +#define MAPCACHE_ORDER   10
    3.19 +#define MAPCACHE_ENTRIES (1 << MAPCACHE_ORDER)
    3.20  struct mapcache {
    3.21 +    /* The PTEs that provide the mappings, and a cursor into the array. */
    3.22      l1_pgentry_t *l1tab;
    3.23      unsigned int cursor;
    3.24 +
    3.25 +    /* Protects map_domain_page(). */
    3.26 +    spinlock_t lock;
    3.27 +
    3.28 +    /* Garbage mappings are flushed from TLBs in batches called 'epochs'. */
    3.29      unsigned int epoch, shadow_epoch[MAX_VIRT_CPUS];
    3.30      u32 tlbflush_timestamp;
    3.31 -    spinlock_t lock;
    3.32 +
    3.33 +    /* Which mappings are in use, and which are garbage to reap next epoch? */
    3.34 +    unsigned long inuse[BITS_TO_LONGS(MAPCACHE_ENTRIES)];
    3.35 +    unsigned long garbage[BITS_TO_LONGS(MAPCACHE_ENTRIES)];
    3.36 +
    3.37 +    /* Lock-free per-VCPU hash of recently-used mappings. */
    3.38 +    struct vcpu_maphash vcpu_maphash[MAX_VIRT_CPUS];
    3.39  };
    3.40  
    3.41 +extern void mapcache_init(struct domain *d);
    3.42 +
    3.43  struct arch_domain
    3.44  {
    3.45      l1_pgentry_t *mm_perdomain_pt;
     4.1 --- a/xen/include/xen/domain_page.h	Wed Jan 11 16:26:04 2006 +0000
     4.2 +++ b/xen/include/xen/domain_page.h	Wed Jan 11 19:18:21 2006 +0100
     4.3 @@ -10,24 +10,19 @@
     4.4  #include <xen/config.h>
     4.5  #include <xen/mm.h>
     4.6  
     4.7 -#define map_domain_page(pfn)   map_domain_pages(pfn,0)
     4.8 -#define unmap_domain_page(va)  unmap_domain_pages(va,0)
     4.9 -
    4.10  #ifdef CONFIG_DOMAIN_PAGE
    4.11  
    4.12  /*
    4.13 - * Maps a given range of page frames, returning the mapped virtual address. The
    4.14 - * pages are now accessible within the current VCPU until a corresponding
    4.15 - * call to unmap_domain_page().
    4.16 + * Map a given page frame, returning the mapped virtual address. The page is
    4.17 + * then accessible within the current VCPU until a corresponding unmap call.
    4.18   */
    4.19 -extern void *map_domain_pages(unsigned long pfn, unsigned int order);
    4.20 +extern void *map_domain_page(unsigned long pfn);
    4.21  
    4.22  /*
    4.23 - * Pass a VA within the first page of a range previously mapped in the context
    4.24 - * of the currently-executing VCPU via a call to map_domain_pages(). Those
    4.25 - * pages will then be removed from the mapping lists.
    4.26 + * Pass a VA within a page previously mapped in the context of the
    4.27 + * currently-executing VCPU via a call to map_domain_pages().
    4.28   */
    4.29 -extern void unmap_domain_pages(void *va, unsigned int order);
    4.30 +extern void unmap_domain_page(void *va);
    4.31  
    4.32  /*
    4.33   * Similar to the above calls, except the mapping is accessible in all
    4.34 @@ -97,8 +92,8 @@ domain_mmap_cache_destroy(struct domain_
    4.35  
    4.36  #else /* !CONFIG_DOMAIN_PAGE */
    4.37  
    4.38 -#define map_domain_pages(pfn,order)         phys_to_virt((pfn)<<PAGE_SHIFT)
    4.39 -#define unmap_domain_pages(va,order)        ((void)((void)(va),(void)(order)))
    4.40 +#define map_domain_page(pfn)                phys_to_virt((pfn)<<PAGE_SHIFT)
    4.41 +#define unmap_domain_page(va)               ((void)(va))
    4.42  
    4.43  #define map_domain_page_global(pfn)         phys_to_virt((pfn)<<PAGE_SHIFT)
    4.44  #define unmap_domain_page_global(va)        ((void)(va))