ia64/xen-unstable

changeset 4403:cb602e6ad404

bitkeeper revision 1.1236.1.174 (424bf73dRPXTiuGP-qQmSHwGKGOyRQ)

Improved TLB flushing of subsets of CPUs. Can now do remote invlpg
as well as complete flush.
Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Thu Mar 31 13:12:29 2005 +0000 (2005-03-31)
parents fa851e5b369e
children 198cfa01d71b
files xen/arch/x86/mm.c xen/arch/x86/mtrr/generic.c xen/arch/x86/smp.c xen/arch/x86/x86_32/mm.c xen/arch/x86/x86_64/mm.c xen/common/grant_table.c xen/include/asm-x86/flushtlb.h xen/include/asm-x86/page.h
line diff
     1.1 --- a/xen/arch/x86/mm.c	Thu Mar 31 11:00:46 2005 +0000
     1.2 +++ b/xen/arch/x86/mm.c	Thu Mar 31 13:12:29 2005 +0000
     1.3 @@ -1476,23 +1476,34 @@ int do_mmuext_op(
     1.4              break;
     1.5      
     1.6          case MMUEXT_INVLPG_LOCAL:
     1.7 -            __flush_tlb_one(op.linear_addr);
     1.8 +            local_flush_tlb_one(op.linear_addr);
     1.9              break;
    1.10  
    1.11          case MMUEXT_TLB_FLUSH_MULTI:
    1.12 -            flush_tlb_mask(d->cpuset); /* XXX KAF XXX */
    1.13 +        case MMUEXT_INVLPG_MULTI:
    1.14 +        {
    1.15 +            unsigned long inset = op.cpuset, outset = 0;
    1.16 +            while ( inset != 0 )
    1.17 +            {
    1.18 +                unsigned int vcpu = find_first_set_bit(inset);
    1.19 +                inset &= ~(1UL<<vcpu);
    1.20 +                if ( (vcpu < MAX_VIRT_CPUS) &&
    1.21 +                     ((ed = d->exec_domain[vcpu]) != NULL) )
    1.22 +                    outset |= 1UL << ed->processor;
    1.23 +            }
    1.24 +            if ( op.cmd == MMUEXT_TLB_FLUSH_MULTI )
    1.25 +                flush_tlb_mask(outset & d->cpuset);
    1.26 +            else
    1.27 +                flush_tlb_one_mask(outset & d->cpuset, op.linear_addr);
    1.28              break;
    1.29 -    
    1.30 -        case MMUEXT_INVLPG_MULTI:
    1.31 -            flush_tlb_mask(d->cpuset); /* XXX KAF XXX */
    1.32 -            break;
    1.33 +        }
    1.34  
    1.35          case MMUEXT_TLB_FLUSH_ALL:
    1.36              flush_tlb_mask(d->cpuset);
    1.37              break;
    1.38      
    1.39          case MMUEXT_INVLPG_ALL:
    1.40 -            flush_tlb_mask(d->cpuset); /* XXX KAF XXX */
    1.41 +            flush_tlb_one_mask(d->cpuset, op.linear_addr);
    1.42              break;
    1.43  
    1.44          case MMUEXT_FLUSH_CACHE:
    1.45 @@ -2029,10 +2040,10 @@ int do_update_va_mapping(unsigned long v
    1.46          percpu_info[cpu].deferred_ops &= ~DOP_FLUSH_TLB;
    1.47          break;
    1.48      case UVMF_INVLPG_LOCAL:
    1.49 -        __flush_tlb_one(va);
    1.50 +        local_flush_tlb_one(va);
    1.51          break;
    1.52      case UVMF_INVLPG_ALL:
    1.53 -        flush_tlb_mask(d->cpuset); /* XXX KAF XXX */
    1.54 +        flush_tlb_one_mask(d->cpuset, va);
    1.55          break;
    1.56      }
    1.57  
    1.58 @@ -2317,7 +2328,7 @@ void ptwr_flush(const int which)
    1.59  
    1.60      /* Ensure that there are no stale writable mappings in any TLB. */
    1.61      /* NB. INVLPG is a serialising instruction: flushes pending updates. */
    1.62 -    __flush_tlb_one(l1va); /* XXX Multi-CPU guests? */
    1.63 +    local_flush_tlb_one(l1va); /* XXX Multi-CPU guests? */
    1.64      PTWR_PRINTK("[%c] disconnected_l1va at %p now %p\n",
    1.65                  PTWR_PRINT_WHICH, ptep, pte);
    1.66  
    1.67 @@ -2636,7 +2647,7 @@ int ptwr_do_page_fault(unsigned long add
    1.68           likely(!shadow_mode_enabled(ed->domain)) )
    1.69      {
    1.70          *pl2e = mk_l2_pgentry(l2e & ~_PAGE_PRESENT);
    1.71 -        flush_tlb(); /* XXX Multi-CPU guests? */
    1.72 +        local_flush_tlb(); /* XXX Multi-CPU guests? */
    1.73      }
    1.74      
    1.75      /* Temporarily map the L1 page, and make a copy of it. */
     2.1 --- a/xen/arch/x86/mtrr/generic.c	Thu Mar 31 11:00:46 2005 +0000
     2.2 +++ b/xen/arch/x86/mtrr/generic.c	Thu Mar 31 13:12:29 2005 +0000
     2.3 @@ -261,7 +261,7 @@ static void prepare_set(void)
     2.4  	}
     2.5  
     2.6  	/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
     2.7 -	__flush_tlb();
     2.8 +	local_flush_tlb();
     2.9  
    2.10  	/*  Save MTRR state */
    2.11  	rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
    2.12 @@ -273,7 +273,7 @@ static void prepare_set(void)
    2.13  static void post_set(void)
    2.14  {
    2.15  	/*  Flush TLBs (no need to flush caches - they are disabled)  */
    2.16 -	__flush_tlb();
    2.17 +	local_flush_tlb();
    2.18  
    2.19  	/* Intel (P6) standard MTRRs */
    2.20  	wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
     3.1 --- a/xen/arch/x86/smp.c	Thu Mar 31 11:00:46 2005 +0000
     3.2 +++ b/xen/arch/x86/smp.c	Thu Mar 31 13:12:29 2005 +0000
     3.3 @@ -148,17 +148,20 @@ static inline void send_IPI_allbutself(i
     3.4  }
     3.5  
     3.6  static spinlock_t flush_lock = SPIN_LOCK_UNLOCKED;
     3.7 -static unsigned long flush_cpumask;
     3.8 +static unsigned long flush_cpumask, flush_va;
     3.9  
    3.10  asmlinkage void smp_invalidate_interrupt(void)
    3.11  {
    3.12      ack_APIC_irq();
    3.13      perfc_incrc(ipis);
    3.14 -    local_flush_tlb();
    3.15 +    if ( flush_va == FLUSHVA_ALL )
    3.16 +        local_flush_tlb();
    3.17 +    else
    3.18 +        local_flush_tlb_one(flush_va);
    3.19      clear_bit(smp_processor_id(), &flush_cpumask);
    3.20  }
    3.21  
    3.22 -void flush_tlb_mask(unsigned long mask)
    3.23 +void __flush_tlb_mask(unsigned long mask, unsigned long va)
    3.24  {
    3.25      ASSERT(local_irq_is_enabled());
    3.26      
    3.27 @@ -172,6 +175,7 @@ void flush_tlb_mask(unsigned long mask)
    3.28      {
    3.29          spin_lock(&flush_lock);
    3.30          flush_cpumask = mask;
    3.31 +        flush_va      = va;
    3.32          send_IPI_mask(mask, INVALIDATE_TLB_VECTOR);
    3.33          while ( flush_cpumask != 0 )
    3.34              cpu_relax();
    3.35 @@ -190,6 +194,7 @@ void new_tlbflush_clock_period(void)
    3.36          spin_lock(&flush_lock);
    3.37          flush_cpumask  = (1UL << smp_num_cpus) - 1;
    3.38          flush_cpumask &= ~(1UL << smp_processor_id());
    3.39 +        flush_va       = FLUSHVA_ALL;
    3.40          send_IPI_allbutself(INVALIDATE_TLB_VECTOR);
    3.41          while ( flush_cpumask != 0 )
    3.42              cpu_relax();
    3.43 @@ -203,13 +208,13 @@ void new_tlbflush_clock_period(void)
    3.44  
    3.45  static void flush_tlb_all_pge_ipi(void *info)
    3.46  {
    3.47 -    __flush_tlb_pge();
    3.48 +    local_flush_tlb_pge();
    3.49  }
    3.50  
    3.51  void flush_tlb_all_pge(void)
    3.52  {
    3.53      smp_call_function(flush_tlb_all_pge_ipi, 0, 1, 1);
    3.54 -    __flush_tlb_pge();
    3.55 +    local_flush_tlb_pge();
    3.56  }
    3.57  
    3.58  void smp_send_event_check_mask(unsigned long cpu_mask)
     4.1 --- a/xen/arch/x86/x86_32/mm.c	Thu Mar 31 11:00:46 2005 +0000
     4.2 +++ b/xen/arch/x86/x86_32/mm.c	Thu Mar 31 13:12:29 2005 +0000
     4.3 @@ -48,7 +48,7 @@ int map_pages(
     4.4          {
     4.5              /* Super-page mapping. */
     4.6              if ( (l2_pgentry_val(*pl2e) & _PAGE_PRESENT) )
     4.7 -                __flush_tlb_pge();
     4.8 +                local_flush_tlb_pge();
     4.9              *pl2e = mk_l2_pgentry(p|flags|_PAGE_PSE);
    4.10  
    4.11              v += 1 << L2_PAGETABLE_SHIFT;
    4.12 @@ -66,7 +66,7 @@ int map_pages(
    4.13              }
    4.14              pl1e = l2_pgentry_to_l1(*pl2e) + l1_table_offset(v);
    4.15              if ( (l1_pgentry_val(*pl1e) & _PAGE_PRESENT) )
    4.16 -                __flush_tlb_one(v);
    4.17 +                local_flush_tlb_one(v);
    4.18              *pl1e = mk_l1_pgentry(p|flags);
    4.19  
    4.20              v += 1 << L1_PAGETABLE_SHIFT;
     5.1 --- a/xen/arch/x86/x86_64/mm.c	Thu Mar 31 11:00:46 2005 +0000
     5.2 +++ b/xen/arch/x86/x86_64/mm.c	Thu Mar 31 13:12:29 2005 +0000
     5.3 @@ -89,7 +89,7 @@ int map_pages(
     5.4          {
     5.5              /* Super-page mapping. */
     5.6              if ( (l2_pgentry_val(*pl2e) & _PAGE_PRESENT) )
     5.7 -                __flush_tlb_pge();
     5.8 +                local_flush_tlb_pge();
     5.9              *pl2e = mk_l2_pgentry(p|flags|_PAGE_PSE);
    5.10  
    5.11              v += 1 << L2_PAGETABLE_SHIFT;
    5.12 @@ -107,7 +107,7 @@ int map_pages(
    5.13              }
    5.14              pl1e = l2_pgentry_to_l1(*pl2e) + l1_table_offset(v);
    5.15              if ( (l1_pgentry_val(*pl1e) & _PAGE_PRESENT) )
    5.16 -                __flush_tlb_one(v);
    5.17 +                local_flush_tlb_one(v);
    5.18              *pl1e = mk_l1_pgentry(p|flags);
    5.19  
    5.20              v += 1 << L1_PAGETABLE_SHIFT;
     6.1 --- a/xen/common/grant_table.c	Thu Mar 31 11:00:46 2005 +0000
     6.2 +++ b/xen/common/grant_table.c	Thu Mar 31 13:12:29 2005 +0000
     6.3 @@ -102,7 +102,8 @@ static int
     6.4      if ( ((host_virt_addr != 0) || (flags & GNTMAP_host_map) ) &&
     6.5           unlikely(!__addr_ok(host_virt_addr)))
     6.6      {
     6.7 -        DPRINTK("Bad virtual address (%x) or flags (%x).\n", host_virt_addr, flags);
     6.8 +        DPRINTK("Bad virtual address (%x) or flags (%x).\n",
     6.9 +                host_virt_addr, flags);
    6.10          (void)__put_user(GNTST_bad_virt_addr, &uop->handle);
    6.11          return GNTST_bad_gntref;
    6.12      }
    6.13 @@ -332,8 +333,10 @@ static int
    6.14           */
    6.15      }
    6.16  
    6.17 -    /* Only make the maptrack live _after_ writing the pte, in case
    6.18 -     * we overwrite the same frame number, causing a maptrack walk to find it */
    6.19 +    /*
    6.20 +     * Only make the maptrack live _after_ writing the pte, in case we 
    6.21 +     * overwrite the same frame number, causing a maptrack walk to find it.
    6.22 +     */
    6.23      ld->grant_table->maptrack[handle].domid         = dom;
    6.24      ld->grant_table->maptrack[handle].ref_and_flags =
    6.25          (ref << MAPTRACK_REF_SHIFT) | (flags & MAPTRACK_GNTMAP_MASK);
    6.26 @@ -364,13 +367,14 @@ gnttab_map_grant_ref(
    6.27      unsigned long va = 0;
    6.28  
    6.29      for ( i = 0; i < count; i++ )
    6.30 -        if ( __gnttab_map_grant_ref(&uop[i], &va) == 0)
    6.31 +        if ( __gnttab_map_grant_ref(&uop[i], &va) == 0 )
    6.32              flush++;
    6.33  
    6.34 +    /* XXX KAF: I think we are probably flushing too much here. */
    6.35      if ( flush == 1 )
    6.36 -        __flush_tlb_one(va);
    6.37 +        flush_tlb_one_mask(current->domain->cpuset, va);
    6.38      else if ( flush != 0 )
    6.39 -        local_flush_tlb();
    6.40 +        flush_tlb_mask(current->domain->cpuset);
    6.41  
    6.42      return 0;
    6.43  }
    6.44 @@ -457,7 +461,7 @@ static int
    6.45          unsigned long   _ol1e;
    6.46  
    6.47          pl1e = &linear_pg_table[l1_linear_offset(virt)];
    6.48 -                                                                                            
    6.49 +
    6.50          if ( unlikely(__get_user(_ol1e, (unsigned long *)pl1e) != 0) )
    6.51          {
    6.52              DPRINTK("Could not find PTE entry for address %x\n", virt);
    6.53 @@ -526,14 +530,13 @@ gnttab_unmap_grant_ref(
    6.54      unsigned long va = 0;
    6.55  
    6.56      for ( i = 0; i < count; i++ )
    6.57 -        if ( __gnttab_unmap_grant_ref(&uop[i], &va) == 0)
    6.58 +        if ( __gnttab_unmap_grant_ref(&uop[i], &va) == 0 )
    6.59              flush++;
    6.60  
    6.61      if ( flush == 1 )
    6.62 -        __flush_tlb_one(va);
    6.63 -
    6.64 +        flush_tlb_one_mask(current->domain->cpuset, va);
    6.65      else if ( flush != 0 )
    6.66 -        local_flush_tlb();
    6.67 +        flush_tlb_mask(current->domain->cpuset);
    6.68  
    6.69      return 0;
    6.70  }
     7.1 --- a/xen/include/asm-x86/flushtlb.h	Thu Mar 31 11:00:46 2005 +0000
     7.2 +++ b/xen/include/asm-x86/flushtlb.h	Thu Mar 31 13:12:29 2005 +0000
     7.3 @@ -77,46 +77,35 @@ static inline unsigned long read_cr3(voi
     7.4  /* Write pagetable base and implicitly tick the tlbflush clock. */
     7.5  extern void write_cr3(unsigned long cr3);
     7.6  
     7.7 -/*
     7.8 - * TLB flushing:
     7.9 - *
    7.10 - *  - flush_tlb() flushes the current mm struct TLBs
    7.11 - *  - flush_tlb_all() flushes all processes TLBs
    7.12 - *  - flush_tlb_pgtables(mm, start, end) flushes a range of page tables
    7.13 - *
    7.14 - * ..but the i386 has somewhat limited tlb flushing capabilities,
    7.15 - * and page-granular flushes are available only on i486 and up.
    7.16 - */
    7.17 -
    7.18 -#define __flush_tlb()                                             \
    7.19 +#define local_flush_tlb()                                         \
    7.20      do {                                                          \
    7.21          unsigned long cr3 = read_cr3();                           \
    7.22          write_cr3(cr3);                                           \
    7.23      } while ( 0 )
    7.24  
    7.25 -#ifndef CONFIG_SMP
    7.26 +#define local_flush_tlb_pge()                                     \
    7.27 +    do {                                                          \
    7.28 +        __pge_off();                                              \
    7.29 +        local_flush_tlb();                                        \
    7.30 +        __pge_on();                                               \
    7.31 +    } while ( 0 )
    7.32  
    7.33 -#define flush_tlb()               __flush_tlb()
    7.34 -#define flush_tlb_all()           __flush_tlb()
    7.35 -#define flush_tlb_all_pge()       __flush_tlb_pge()
    7.36 -#define local_flush_tlb()         __flush_tlb()
    7.37 -#define flush_tlb_cpu(_cpu)       __flush_tlb()
    7.38 -#define flush_tlb_mask(_mask)     __flush_tlb()
    7.39 -#define try_flush_tlb_mask(_mask) __flush_tlb()
    7.40 +#define local_flush_tlb_one(__addr) \
    7.41 +    __asm__ __volatile__("invlpg %0": :"m" (*(char *) (__addr)))
    7.42  
    7.43 -#else
    7.44 +#define flush_tlb_all()     flush_tlb_mask((1 << smp_num_cpus) - 1)
    7.45  
    7.46 +#ifndef CONFIG_SMP
    7.47 +#define flush_tlb_all_pge()          local_flush_tlb_pge()
    7.48 +#define flush_tlb_mask(_mask)        local_flush_tlb()
    7.49 +#define flush_tlb_one_mask(_mask,_v) local_flush_tlb_one(_v)
    7.50 +#else
    7.51  #include <xen/smp.h>
    7.52 -
    7.53 -extern int try_flush_tlb_mask(unsigned long mask);
    7.54 -extern void flush_tlb_mask(unsigned long mask);
    7.55 +#define FLUSHVA_ALL (~0UL)
    7.56  extern void flush_tlb_all_pge(void);
    7.57 -
    7.58 -#define flush_tlb()	    __flush_tlb()
    7.59 -#define flush_tlb_all()     flush_tlb_mask((1 << smp_num_cpus) - 1)
    7.60 -#define local_flush_tlb()   __flush_tlb()
    7.61 -#define flush_tlb_cpu(_cpu) flush_tlb_mask(1 << (_cpu))
    7.62 -
    7.63 +extern void __flush_tlb_mask(unsigned long mask, unsigned long va);
    7.64 +#define flush_tlb_mask(_mask)        __flush_tlb_mask(_mask,FLUSHVA_ALL)
    7.65 +#define flush_tlb_one_mask(_mask,_v) __flush_tlb_mask(_mask,_v)
    7.66  #endif
    7.67  
    7.68  #endif /* __FLUSHTLB_H__ */
     8.1 --- a/xen/include/asm-x86/page.h	Thu Mar 31 11:00:46 2005 +0000
     8.2 +++ b/xen/include/asm-x86/page.h	Thu Mar 31 13:12:29 2005 +0000
     8.3 @@ -78,8 +78,6 @@ extern root_pgentry_t idle_pg_table[ROOT
     8.4  
     8.5  extern void paging_init(void);
     8.6  
     8.7 -/* Flush global pages as well. */
     8.8 -
     8.9  #define __pge_off()                                                     \
    8.10      do {                                                                \
    8.11          __asm__ __volatile__(                                           \
    8.12 @@ -94,17 +92,6 @@ extern void paging_init(void);
    8.13              : : "r" (mmu_cr4_features) );                               \
    8.14      } while ( 0 )
    8.15  
    8.16 -
    8.17 -#define __flush_tlb_pge()                                               \
    8.18 -    do {                                                                \
    8.19 -        __pge_off();                                                    \
    8.20 -        __flush_tlb();                                                  \
    8.21 -        __pge_on();                                                     \
    8.22 -    } while ( 0 )
    8.23 -
    8.24 -#define __flush_tlb_one(__addr) \
    8.25 -    __asm__ __volatile__("invlpg %0": :"m" (*(char *) (__addr)))
    8.26 -
    8.27  #endif /* !__ASSEMBLY__ */
    8.28  
    8.29