ia64/xen-unstable

changeset 16138:9488d3166553

x86: consolidate/enhance TLB flushing interface

Folding into a single local handler and a single SMP multiplexor as
well as adding capability to also flush caches through the same
interfaces (a subsequent patch will make use of this).

Once at changing cpuinfo_x86, this patch also removes several unused
fields apparently inherited from Linux.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
Signed-off-by: Keir Fraser <keir@xensource.com>
author Keir Fraser <keir@xensource.com>
date Tue Oct 16 17:31:37 2007 +0100 (2007-10-16)
parents 1f893d055c6f
children 96930bc060a6
files xen/arch/x86/cpu/amd.c xen/arch/x86/cpu/common.c xen/arch/x86/cpu/cyrix.c xen/arch/x86/cpu/intel.c xen/arch/x86/cpu/mtrr/generic.c xen/arch/x86/domain.c xen/arch/x86/domain_build.c xen/arch/x86/flushtlb.c xen/arch/x86/mm.c xen/arch/x86/mm/p2m.c xen/arch/x86/mm/shadow/multi.c xen/arch/x86/setup.c xen/arch/x86/smp.c xen/arch/x86/smpboot.c xen/arch/x86/x86_32/domain_page.c xen/arch/x86/x86_32/mm.c xen/arch/x86/x86_64/compat/mm.c xen/arch/x86/x86_64/mm.c xen/include/asm-x86/cpufeature.h xen/include/asm-x86/flushtlb.h xen/include/asm-x86/mach-default/smpboot_hooks.h xen/include/asm-x86/processor.h
line diff
     1.1 --- a/xen/arch/x86/cpu/amd.c	Tue Oct 16 10:27:55 2007 +0100
     1.2 +++ b/xen/arch/x86/cpu/amd.c	Tue Oct 16 17:31:37 2007 +0100
     1.3 @@ -373,6 +373,11 @@ static void __init init_amd(struct cpuin
     1.4  	if ((smp_processor_id() == 1) && c1_ramping_may_cause_clock_drift(c))
     1.5  		disable_c1_ramping();
     1.6  
     1.7 +	/* Support INVLPG of superpages? */
     1.8 +	__set_bit(2, &c->invlpg_works_ok);
     1.9 +	if ( cpu_has(c, X86_FEATURE_PAGE1GB) )
    1.10 +		__set_bit(3, &c->invlpg_works_ok);
    1.11 +
    1.12  	start_svm(c);
    1.13  }
    1.14  
     2.1 --- a/xen/arch/x86/cpu/common.c	Tue Oct 16 10:27:55 2007 +0100
     2.2 +++ b/xen/arch/x86/cpu/common.c	Tue Oct 16 17:31:37 2007 +0100
     2.3 @@ -229,7 +229,6 @@ static void __init early_cpu_detect(void
     2.4  void __devinit generic_identify(struct cpuinfo_x86 * c)
     2.5  {
     2.6  	u32 tfms, xlvl;
     2.7 -	int junk;
     2.8  
     2.9  	if (have_cpuid_p()) {
    2.10  		/* Get vendor name */
    2.11 @@ -244,8 +243,8 @@ void __devinit generic_identify(struct c
    2.12  	
    2.13  		/* Intel-defined flags: level 0x00000001 */
    2.14  		if ( c->cpuid_level >= 0x00000001 ) {
    2.15 -			u32 capability, excap;
    2.16 -			cpuid(0x00000001, &tfms, &junk, &excap, &capability);
    2.17 +			u32 capability, excap, ebx;
    2.18 +			cpuid(0x00000001, &tfms, &ebx, &excap, &capability);
    2.19  			c->x86_capability[0] = capability;
    2.20  			c->x86_capability[4] = excap;
    2.21  			c->x86 = (tfms >> 8) & 15;
    2.22 @@ -255,6 +254,8 @@ void __devinit generic_identify(struct c
    2.23  				c->x86_model += ((tfms >> 16) & 0xF) << 4;
    2.24  			} 
    2.25  			c->x86_mask = tfms & 15;
    2.26 +			if ( cpu_has(c, X86_FEATURE_CLFLSH) )
    2.27 +				c->x86_clflush_size = ((ebx >> 8) & 0xff) * 8;
    2.28  		} else {
    2.29  			/* Have CPUID level 0 only - unheard of */
    2.30  			c->x86 = 4;
    2.31 @@ -313,6 +314,7 @@ void __devinit identify_cpu(struct cpuin
    2.32  	c->x86_vendor_id[0] = '\0'; /* Unset */
    2.33  	c->x86_model_id[0] = '\0';  /* Unset */
    2.34  	c->x86_max_cores = 1;
    2.35 +	c->x86_clflush_size = 0;
    2.36  	memset(&c->x86_capability, 0, sizeof c->x86_capability);
    2.37  
    2.38  	if (!have_cpuid_p()) {
     3.1 --- a/xen/arch/x86/cpu/cyrix.c	Tue Oct 16 10:27:55 2007 +0100
     3.2 +++ b/xen/arch/x86/cpu/cyrix.c	Tue Oct 16 17:31:37 2007 +0100
     3.3 @@ -239,7 +239,7 @@ static void __init init_cyrix(struct cpu
     3.4  		/* Emulate MTRRs using Cyrix's ARRs. */
     3.5  		set_bit(X86_FEATURE_CYRIX_ARR, c->x86_capability);
     3.6  		/* 6x86's contain this bug */
     3.7 -		c->coma_bug = 1;
     3.8 +		/*c->coma_bug = 1;*/
     3.9  		break;
    3.10  
    3.11  	case 4: /* MediaGX/GXm or Geode GXM/GXLV/GX1 */
    3.12 @@ -272,7 +272,7 @@ static void __init init_cyrix(struct cpu
    3.13  		}
    3.14  		else
    3.15  		{
    3.16 -			c->coma_bug = 1;      /* 6x86MX, it has the bug. */
    3.17 +			/*c->coma_bug = 1;*/      /* 6x86MX, it has the bug. */
    3.18  		}
    3.19  		tmp = (!(dir0_lsn & 7) || dir0_lsn & 1) ? 2 : 0;
    3.20  		Cx86_cb[tmp] = cyrix_model_mult2[dir0_lsn & 7];
    3.21 @@ -287,7 +287,7 @@ static void __init init_cyrix(struct cpu
    3.22  		switch (dir0_lsn) {
    3.23  		case 0xd:  /* either a 486SLC or DLC w/o DEVID */
    3.24  			dir0_msn = 0;
    3.25 -			p = Cx486_name[(c->hard_math) ? 1 : 0];
    3.26 +			p = Cx486_name[/*(c->hard_math) ? 1 : 0*/1];
    3.27  			break;
    3.28  
    3.29  		case 0xe:  /* a 486S A step */
     4.1 --- a/xen/arch/x86/cpu/intel.c	Tue Oct 16 10:27:55 2007 +0100
     4.2 +++ b/xen/arch/x86/cpu/intel.c	Tue Oct 16 17:31:37 2007 +0100
     4.3 @@ -123,6 +123,18 @@ static void __devinit init_intel(struct 
     4.4  	if ((c->x86<<8 | c->x86_model<<4 | c->x86_mask) < 0x633)
     4.5  		clear_bit(X86_FEATURE_SEP, c->x86_capability);
     4.6  
     4.7 +	/* Supports INVLPG of superpages? */
     4.8 +	__set_bit(2, &c->invlpg_works_ok);
     4.9 +	if (/* PentiumPro erratum 30 */
    4.10 +	    (c->x86 == 6 && c->x86_model == 1 && c->x86_mask < 9) ||
    4.11 +	    /* Dual-Core Intel Xeon 3000/5100 series erratum 89/90 */
    4.12 +	    /* Quad-Core Intel Xeon 3200/5300 series erratum 89/88 */
    4.13 +	    /* Intel Core2 erratum 89 */
    4.14 +	    (c->x86 == 6 && c->x86_model == 15 ) ||
    4.15 +	    /* Dual-Core Intel Xeon LV/ULV erratum 75 */
    4.16 +	    (c->x86 == 6 && c->x86_model == 14 ))
    4.17 +		__clear_bit(2, &c->invlpg_works_ok);
    4.18 +
    4.19  	/* Names for the Pentium II/Celeron processors 
    4.20  	   detectable only by also checking the cache size.
    4.21  	   Dixon is NOT a Celeron. */
     5.1 --- a/xen/arch/x86/cpu/mtrr/generic.c	Tue Oct 16 10:27:55 2007 +0100
     5.2 +++ b/xen/arch/x86/cpu/mtrr/generic.c	Tue Oct 16 17:31:37 2007 +0100
     5.3 @@ -313,7 +313,7 @@ static void prepare_set(void)
     5.4  	}
     5.5  
     5.6  	/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
     5.7 -	local_flush_tlb();
     5.8 +	flush_tlb_local();
     5.9  
    5.10  	/*  Save MTRR state */
    5.11  	rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
    5.12 @@ -325,7 +325,7 @@ static void prepare_set(void)
    5.13  static void post_set(void)
    5.14  {
    5.15  	/*  Flush TLBs (no need to flush caches - they are disabled)  */
    5.16 -	local_flush_tlb();
    5.17 +	flush_tlb_local();
    5.18  
    5.19  	/* Intel (P6) standard MTRRs */
    5.20  	mtrr_wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
     6.1 --- a/xen/arch/x86/domain.c	Tue Oct 16 10:27:55 2007 +0100
     6.2 +++ b/xen/arch/x86/domain.c	Tue Oct 16 17:31:37 2007 +0100
     6.3 @@ -1299,7 +1299,7 @@ void context_switch(struct vcpu *prev, s
     6.4          {
     6.5              uint64_t efer = read_efer();
     6.6  
     6.7 -            local_flush_tlb_one(GDT_VIRT_START(next) +
     6.8 +            flush_tlb_one_local(GDT_VIRT_START(next) +
     6.9                                  FIRST_RESERVED_GDT_BYTE);
    6.10  
    6.11              if ( !is_pv_32on64_vcpu(next) == !(efer & EFER_SCE) )
     7.1 --- a/xen/arch/x86/domain_build.c	Tue Oct 16 10:27:55 2007 +0100
     7.2 +++ b/xen/arch/x86/domain_build.c	Tue Oct 16 17:31:37 2007 +0100
     7.3 @@ -347,7 +347,7 @@ int __init construct_dom0(
     7.4          for ( i = 0; i < MAX_VIRT_CPUS; i++ )
     7.5              d->arch.mm_perdomain_pt[((i << GDT_LDT_VCPU_SHIFT) +
     7.6                                       FIRST_RESERVED_GDT_PAGE)] = gdt_l1e;
     7.7 -        local_flush_tlb_one(GDT_LDT_VIRT_START + FIRST_RESERVED_GDT_BYTE);
     7.8 +        flush_tlb_one_local(GDT_LDT_VIRT_START + FIRST_RESERVED_GDT_BYTE);
     7.9      }
    7.10  #endif
    7.11      if ( parms.pae == PAEKERN_extended_cr3 )
     8.1 --- a/xen/arch/x86/flushtlb.c	Tue Oct 16 10:27:55 2007 +0100
     8.2 +++ b/xen/arch/x86/flushtlb.c	Tue Oct 16 17:31:37 2007 +0100
     8.3 @@ -84,10 +84,10 @@ void write_cr3(unsigned long cr3)
     8.4  
     8.5  #ifdef USER_MAPPINGS_ARE_GLOBAL
     8.6      __pge_off();
     8.7 -    __asm__ __volatile__ ( "mov %0, %%cr3" : : "r" (cr3) : "memory" );
     8.8 +    asm volatile ( "mov %0, %%cr3" : : "r" (cr3) : "memory" );
     8.9      __pge_on();
    8.10  #else
    8.11 -    __asm__ __volatile__ ( "mov %0, %%cr3" : : "r" (cr3) : "memory" );
    8.12 +    asm volatile ( "mov %0, %%cr3" : : "r" (cr3) : "memory" );
    8.13  #endif
    8.14  
    8.15      post_flush(t);
    8.16 @@ -95,26 +95,69 @@ void write_cr3(unsigned long cr3)
    8.17      local_irq_restore(flags);
    8.18  }
    8.19  
    8.20 -void local_flush_tlb(void)
    8.21 +void flush_area_local(const void *va, unsigned int flags)
    8.22  {
    8.23 -    unsigned long flags;
    8.24 -    u32 t;
    8.25 +    const struct cpuinfo_x86 *c = &current_cpu_data;
    8.26 +    unsigned int level = flags & FLUSH_LEVEL_MASK;
    8.27 +    unsigned long irqfl;
    8.28 +
    8.29 +    ASSERT(level < CONFIG_PAGING_LEVELS);
    8.30  
    8.31      /* This non-reentrant function is sometimes called in interrupt context. */
    8.32 -    local_irq_save(flags);
    8.33 -
    8.34 -    t = pre_flush();
    8.35 -
    8.36 -    hvm_flush_guest_tlbs();
    8.37 +    local_irq_save(irqfl);
    8.38  
    8.39 -#ifdef USER_MAPPINGS_ARE_GLOBAL
    8.40 -    __pge_off();
    8.41 -    __pge_on();
    8.42 -#else
    8.43 -    __asm__ __volatile__ ( "mov %0, %%cr3" : : "r" (read_cr3()) : "memory" );
    8.44 +    if ( flags & (FLUSH_TLB|FLUSH_TLB_GLOBAL) )
    8.45 +    {
    8.46 +        if ( (level != 0) && test_bit(level, &c->invlpg_works_ok) )
    8.47 +        {
    8.48 +            asm volatile ( "invlpg %0"
    8.49 +                           : : "m" (*(const char *)(va)) : "memory" );
    8.50 +        }
    8.51 +        else
    8.52 +        {
    8.53 +            u32 t = pre_flush();
    8.54 +
    8.55 +            hvm_flush_guest_tlbs();
    8.56 +
    8.57 +#ifndef USER_MAPPINGS_ARE_GLOBAL
    8.58 +            if ( !(flags & FLUSH_TLB_GLOBAL) ||
    8.59 +                 !(mmu_cr4_features & X86_CR4_PGE) )
    8.60 +            {
    8.61 +                asm volatile ( "mov %0, %%cr3"
    8.62 +                               : : "r" (read_cr3()) : "memory" );
    8.63 +            }
    8.64 +            else
    8.65  #endif
    8.66 +            {
    8.67 +                __pge_off();
    8.68 +                barrier();
    8.69 +                __pge_on();
    8.70 +            }
    8.71  
    8.72 -    post_flush(t);
    8.73 +            post_flush(t);
    8.74 +        }
    8.75 +    }
    8.76  
    8.77 -    local_irq_restore(flags);
    8.78 +    if ( flags & FLUSH_CACHE )
    8.79 +    {
    8.80 +        unsigned long i, sz;
    8.81 +
    8.82 +        sz = level ? (1UL << ((level - 1) * PAGETABLE_ORDER)) : ULONG_MAX;
    8.83 +
    8.84 +        if ( c->x86_clflush_size && c->x86_cache_size &&
    8.85 +             (sz < (c->x86_cache_size >> (PAGE_SHIFT - 10))) )
    8.86 +        {
    8.87 +            sz <<= PAGE_SHIFT;
    8.88 +            va = (const void *)((unsigned long)va & ~(sz - 1));
    8.89 +            for ( i = 0; i < sz; i += c->x86_clflush_size )
    8.90 +                 asm volatile ( "clflush %0"
    8.91 +                                : : "m" (((const char *)va)[i]) );
    8.92 +        }
    8.93 +        else
    8.94 +        {
    8.95 +            wbinvd();
    8.96 +        }
    8.97 +    }
    8.98 +
    8.99 +    local_irq_restore(irqfl);
   8.100  }
     9.1 --- a/xen/arch/x86/mm.c	Tue Oct 16 10:27:55 2007 +0100
     9.2 +++ b/xen/arch/x86/mm.c	Tue Oct 16 17:31:37 2007 +0100
     9.3 @@ -372,7 +372,7 @@ void make_cr3(struct vcpu *v, unsigned l
     9.4      /* First check the previous high mapping can't be in the TLB. 
     9.5       * (i.e. have we loaded CR3 since we last did this?) */
     9.6      if ( unlikely(this_cpu(make_cr3_timestamp) == this_cpu(tlbflush_time)) )
     9.7 -        local_flush_tlb_one(fix_to_virt(FIX_PAE_HIGHMEM_0 + cpu));
     9.8 +        flush_tlb_one_local(fix_to_virt(FIX_PAE_HIGHMEM_0 + cpu));
     9.9      highmem_l3tab = (l3_pgentry_t *)fix_to_virt(FIX_PAE_HIGHMEM_0 + cpu);
    9.10      lowmem_l3tab  = cache->table[cache->inuse_idx];
    9.11      memcpy(lowmem_l3tab, highmem_l3tab, sizeof(cache->table[0]));
    9.12 @@ -1886,7 +1886,7 @@ static void process_deferred_ops(void)
    9.13          if ( deferred_ops & DOP_FLUSH_ALL_TLBS )
    9.14              flush_tlb_mask(d->domain_dirty_cpumask);
    9.15          else
    9.16 -            local_flush_tlb();
    9.17 +            flush_tlb_local();
    9.18      }
    9.19  
    9.20      if ( deferred_ops & DOP_RELOAD_LDT )
    9.21 @@ -2172,7 +2172,7 @@ int do_mmuext_op(
    9.22          case MMUEXT_INVLPG_LOCAL:
    9.23              if ( !paging_mode_enabled(d) 
    9.24                   || paging_invlpg(v, op.arg1.linear_addr) != 0 )
    9.25 -                local_flush_tlb_one(op.arg1.linear_addr);
    9.26 +                flush_tlb_one_local(op.arg1.linear_addr);
    9.27              break;
    9.28  
    9.29          case MMUEXT_TLB_FLUSH_MULTI:
    9.30 @@ -2848,7 +2848,7 @@ int do_update_va_mapping(unsigned long v
    9.31          switch ( (bmap_ptr = flags & ~UVMF_FLUSHTYPE_MASK) )
    9.32          {
    9.33          case UVMF_LOCAL:
    9.34 -            local_flush_tlb();
    9.35 +            flush_tlb_local();
    9.36              break;
    9.37          case UVMF_ALL:
    9.38              flush_tlb_mask(d->domain_dirty_cpumask);
    9.39 @@ -2870,7 +2870,7 @@ int do_update_va_mapping(unsigned long v
    9.40          case UVMF_LOCAL:
    9.41              if ( !paging_mode_enabled(d) 
    9.42                   || (paging_invlpg(current, va) != 0) ) 
    9.43 -                local_flush_tlb_one(va);
    9.44 +                flush_tlb_one_local(va);
    9.45              break;
    9.46          case UVMF_ALL:
    9.47              flush_tlb_one_mask(d->domain_dirty_cpumask, va);
    9.48 @@ -2989,7 +2989,7 @@ long do_set_gdt(XEN_GUEST_HANDLE(ulong) 
    9.49      LOCK_BIGLOCK(current->domain);
    9.50  
    9.51      if ( (ret = set_gdt(current, frames, entries)) == 0 )
    9.52 -        local_flush_tlb();
    9.53 +        flush_tlb_local();
    9.54  
    9.55      UNLOCK_BIGLOCK(current->domain);
    9.56  
    9.57 @@ -3544,7 +3544,8 @@ int map_pages_to_xen(
    9.58  
    9.59              if ( (l2e_get_flags(ol2e) & _PAGE_PRESENT) )
    9.60              {
    9.61 -                local_flush_tlb_pge();
    9.62 +                flush_area_local((const void *)virt,
    9.63 +                                 FLUSH_TLB_GLOBAL|FLUSH_LEVEL(2));
    9.64                  if ( !(l2e_get_flags(ol2e) & _PAGE_PSE) )
    9.65                      free_xen_pagetable(mfn_to_virt(l2e_get_pfn(ol2e)));
    9.66              }
    9.67 @@ -3572,14 +3573,15 @@ int map_pages_to_xen(
    9.68                                             l2e_get_flags(*pl2e) & ~_PAGE_PSE));
    9.69                  l2e_write_atomic(pl2e, l2e_from_pfn(virt_to_mfn(pl1e),
    9.70                                                      __PAGE_HYPERVISOR));
    9.71 -                local_flush_tlb_pge();
    9.72 +                flush_area_local((const void *)virt,
    9.73 +                                 FLUSH_TLB_GLOBAL|FLUSH_LEVEL(2));
    9.74              }
    9.75  
    9.76              pl1e  = l2e_to_l1e(*pl2e) + l1_table_offset(virt);
    9.77              ol1e  = *pl1e;
    9.78              l1e_write_atomic(pl1e, l1e_from_pfn(mfn, flags));
    9.79              if ( (l1e_get_flags(ol1e) & _PAGE_PRESENT) )
    9.80 -                local_flush_tlb_one(virt);
    9.81 +                flush_tlb_one_local(virt);
    9.82  
    9.83              virt    += 1UL << L1_PAGETABLE_SHIFT;
    9.84              mfn     += 1UL;
    9.85 @@ -3655,7 +3657,7 @@ void destroy_xen_mappings(unsigned long 
    9.86          }
    9.87      }
    9.88  
    9.89 -    flush_tlb_all_pge();
    9.90 +    flush_all(FLUSH_TLB_GLOBAL);
    9.91  }
    9.92  
    9.93  void __set_fixmap(
    10.1 --- a/xen/arch/x86/mm/p2m.c	Tue Oct 16 10:27:55 2007 +0100
    10.2 +++ b/xen/arch/x86/mm/p2m.c	Tue Oct 16 17:31:37 2007 +0100
    10.3 @@ -493,7 +493,7 @@ static void audit_p2m(struct domain *d)
    10.4      test_linear = ( (d == current->domain)
    10.5                      && !pagetable_is_null(current->arch.monitor_table) );
    10.6      if ( test_linear )
    10.7 -        local_flush_tlb();
    10.8 +        flush_tlb_local();
    10.9  
   10.10      /* Audit part one: walk the domain's page allocation list, checking
   10.11       * the m2p entries. */
    11.1 --- a/xen/arch/x86/mm/shadow/multi.c	Tue Oct 16 10:27:55 2007 +0100
    11.2 +++ b/xen/arch/x86/mm/shadow/multi.c	Tue Oct 16 17:31:37 2007 +0100
    11.3 @@ -3089,7 +3089,7 @@ sh_invlpg(struct vcpu *v, unsigned long 
    11.4      if ( mfn_to_shadow_page(shadow_l2e_get_mfn(sl2e))->type
    11.5           == SH_type_fl1_shadow )
    11.6      {
    11.7 -        local_flush_tlb();
    11.8 +        flush_tlb_local();
    11.9          return 0;
   11.10      }
   11.11  
    12.1 --- a/xen/arch/x86/setup.c	Tue Oct 16 10:27:55 2007 +0100
    12.2 +++ b/xen/arch/x86/setup.c	Tue Oct 16 17:31:37 2007 +0100
    12.3 @@ -114,7 +114,7 @@ struct tss_struct init_tss[NR_CPUS];
    12.4  
    12.5  char __attribute__ ((__section__(".bss.stack_aligned"))) cpu0_stack[STACK_SIZE];
    12.6  
    12.7 -struct cpuinfo_x86 boot_cpu_data = { 0, 0, 0, 0, -1, 1, 0, 0, -1 };
    12.8 +struct cpuinfo_x86 boot_cpu_data = { 0, 0, 0, 0, 1, -1 };
    12.9  
   12.10  #if CONFIG_PAGING_LEVELS > 2
   12.11  unsigned long mmu_cr4_features = X86_CR4_PSE | X86_CR4_PGE | X86_CR4_PAE;
    13.1 --- a/xen/arch/x86/smp.c	Tue Oct 16 10:27:55 2007 +0100
    13.2 +++ b/xen/arch/x86/smp.c	Tue Oct 16 17:31:37 2007 +0100
    13.3 @@ -164,34 +164,28 @@ void send_IPI_mask_phys(cpumask_t mask, 
    13.4  
    13.5  static DEFINE_SPINLOCK(flush_lock);
    13.6  static cpumask_t flush_cpumask;
    13.7 -static unsigned long flush_va;
    13.8 +static const void *flush_va;
    13.9 +static unsigned int flush_flags;
   13.10  
   13.11  fastcall void smp_invalidate_interrupt(void)
   13.12  {
   13.13      ack_APIC_irq();
   13.14      perfc_incr(ipis);
   13.15      irq_enter();
   13.16 -    if ( !__sync_lazy_execstate() )
   13.17 -    {
   13.18 -        if ( flush_va == FLUSHVA_ALL )
   13.19 -            local_flush_tlb();
   13.20 -        else
   13.21 -            local_flush_tlb_one(flush_va);
   13.22 -    }
   13.23 +    if ( !__sync_lazy_execstate() ||
   13.24 +         (flush_flags & (FLUSH_TLB_GLOBAL | FLUSH_CACHE)) )
   13.25 +        flush_area_local(flush_va, flush_flags);
   13.26      cpu_clear(smp_processor_id(), flush_cpumask);
   13.27      irq_exit();
   13.28  }
   13.29  
   13.30 -void __flush_tlb_mask(cpumask_t mask, unsigned long va)
   13.31 +void flush_area_mask(cpumask_t mask, const void *va, unsigned int flags)
   13.32  {
   13.33      ASSERT(local_irq_is_enabled());
   13.34      
   13.35      if ( cpu_isset(smp_processor_id(), mask) )
   13.36      {
   13.37 -        if ( va == FLUSHVA_ALL )
   13.38 -            local_flush_tlb();
   13.39 -        else
   13.40 -            local_flush_tlb_one(va);
   13.41 +        flush_area_local(va, flags);
   13.42          cpu_clear(smp_processor_id(), mask);
   13.43      }
   13.44  
   13.45 @@ -200,6 +194,7 @@ void __flush_tlb_mask(cpumask_t mask, un
   13.46          spin_lock(&flush_lock);
   13.47          flush_cpumask = mask;
   13.48          flush_va      = va;
   13.49 +        flush_flags   = flags;
   13.50          send_IPI_mask(mask, INVALIDATE_TLB_VECTOR);
   13.51          while ( !cpus_empty(flush_cpumask) )
   13.52              cpu_relax();
   13.53 @@ -215,24 +210,13 @@ void new_tlbflush_clock_period(void)
   13.54      /* Flush everyone else. We definitely flushed just before entry. */
   13.55      allbutself = cpu_online_map;
   13.56      cpu_clear(smp_processor_id(), allbutself);
   13.57 -    __flush_tlb_mask(allbutself, FLUSHVA_ALL);
   13.58 +    flush_mask(allbutself, FLUSH_TLB);
   13.59  
   13.60      /* No need for atomicity: we are the only possible updater. */
   13.61      ASSERT(tlbflush_clock == 0);
   13.62      tlbflush_clock++;
   13.63  }
   13.64  
   13.65 -static void flush_tlb_all_pge_ipi(void *info)
   13.66 -{
   13.67 -    local_flush_tlb_pge();
   13.68 -}
   13.69 -
   13.70 -void flush_tlb_all_pge(void)
   13.71 -{
   13.72 -    smp_call_function(flush_tlb_all_pge_ipi, 0, 1, 1);
   13.73 -    local_flush_tlb_pge();
   13.74 -}
   13.75 -
   13.76  void smp_send_event_check_mask(cpumask_t mask)
   13.77  {
   13.78      cpu_clear(smp_processor_id(), mask);
    14.1 --- a/xen/arch/x86/smpboot.c	Tue Oct 16 10:27:55 2007 +0100
    14.2 +++ b/xen/arch/x86/smpboot.c	Tue Oct 16 17:31:37 2007 +0100
    14.3 @@ -518,7 +518,7 @@ void __devinit start_secondary(void *unu
    14.4  	 * low-memory mappings have been cleared, flush them from
    14.5  	 * the local TLBs too.
    14.6  	 */
    14.7 -	local_flush_tlb();
    14.8 +	flush_tlb_local();
    14.9  
   14.10  	/* This must be done before setting cpu_online_map */
   14.11  	set_cpu_sibling_map(raw_smp_processor_id());
    15.1 --- a/xen/arch/x86/x86_32/domain_page.c	Tue Oct 16 10:27:55 2007 +0100
    15.2 +++ b/xen/arch/x86/x86_32/domain_page.c	Tue Oct 16 17:31:37 2007 +0100
    15.3 @@ -78,7 +78,7 @@ void *map_domain_page(unsigned long mfn)
    15.4          if ( NEED_FLUSH(this_cpu(tlbflush_time), dcache->tlbflush_timestamp) )
    15.5          {
    15.6              perfc_incr(domain_page_tlb_flush);
    15.7 -            local_flush_tlb();
    15.8 +            flush_tlb_local();
    15.9          }
   15.10      }
   15.11  
   15.12 @@ -94,7 +94,7 @@ void *map_domain_page(unsigned long mfn)
   15.13  
   15.14          /* /Second/, flush TLBs. */
   15.15          perfc_incr(domain_page_tlb_flush);
   15.16 -        local_flush_tlb();
   15.17 +        flush_tlb_local();
   15.18          vcache->shadow_epoch = ++dcache->epoch;
   15.19          dcache->tlbflush_timestamp = tlbflush_current_time();
   15.20  
    16.1 --- a/xen/arch/x86/x86_32/mm.c	Tue Oct 16 10:27:55 2007 +0100
    16.2 +++ b/xen/arch/x86/x86_32/mm.c	Tue Oct 16 17:31:37 2007 +0100
    16.3 @@ -152,7 +152,7 @@ void __init zap_low_mappings(l2_pgentry_
    16.4      /* Now zap mappings in the idle pagetables. */
    16.5      destroy_xen_mappings(0, HYPERVISOR_VIRT_START);
    16.6  
    16.7 -    flush_tlb_all_pge();
    16.8 +    flush_all(FLUSH_TLB_GLOBAL);
    16.9  
   16.10      /* Replace with mapping of the boot trampoline only. */
   16.11      map_pages_to_xen(BOOT_TRAMPOLINE, BOOT_TRAMPOLINE >> PAGE_SHIFT,
    17.1 --- a/xen/arch/x86/x86_64/compat/mm.c	Tue Oct 16 10:27:55 2007 +0100
    17.2 +++ b/xen/arch/x86/x86_64/compat/mm.c	Tue Oct 16 17:31:37 2007 +0100
    17.3 @@ -31,7 +31,7 @@ int compat_set_gdt(XEN_GUEST_HANDLE(uint
    17.4      LOCK_BIGLOCK(current->domain);
    17.5  
    17.6      if ( (ret = set_gdt(current, frames, entries)) == 0 )
    17.7 -        local_flush_tlb();
    17.8 +        flush_tlb_local();
    17.9  
   17.10      UNLOCK_BIGLOCK(current->domain);
   17.11  
    18.1 --- a/xen/arch/x86/x86_64/mm.c	Tue Oct 16 10:27:55 2007 +0100
    18.2 +++ b/xen/arch/x86/x86_64/mm.c	Tue Oct 16 17:31:37 2007 +0100
    18.3 @@ -205,7 +205,7 @@ void __init zap_low_mappings(void)
    18.4  
    18.5      /* Remove aliased mapping of first 1:1 PML4 entry. */
    18.6      l4e_write(&idle_pg_table[0], l4e_empty());
    18.7 -    local_flush_tlb_pge();
    18.8 +    flush_local(FLUSH_TLB_GLOBAL);
    18.9  
   18.10      /* Replace with mapping of the boot trampoline only. */
   18.11      map_pages_to_xen(BOOT_TRAMPOLINE, BOOT_TRAMPOLINE >> PAGE_SHIFT,
    19.1 --- a/xen/include/asm-x86/cpufeature.h	Tue Oct 16 10:27:55 2007 +0100
    19.2 +++ b/xen/include/asm-x86/cpufeature.h	Tue Oct 16 17:31:37 2007 +0100
    19.3 @@ -50,6 +50,7 @@
    19.4  #define X86_FEATURE_NX		(1*32+20) /* Execute Disable */
    19.5  #define X86_FEATURE_MMXEXT	(1*32+22) /* AMD MMX extensions */
    19.6  #define X86_FEATURE_FFXSR       (1*32+25) /* FFXSR instruction optimizations */
    19.7 +#define X86_FEATURE_PAGE1GB	(1*32+26) /* 1Gb large page support */
    19.8  #define X86_FEATURE_RDTSCP	(1*32+27) /* RDTSCP */
    19.9  #define X86_FEATURE_LM		(1*32+29) /* Long Mode (x86-64) */
   19.10  #define X86_FEATURE_3DNOWEXT	(1*32+30) /* AMD 3DNow! extensions */
   19.11 @@ -143,6 +144,7 @@
   19.12  #define cpu_has_cyrix_arr	boot_cpu_has(X86_FEATURE_CYRIX_ARR)
   19.13  #define cpu_has_centaur_mcr	boot_cpu_has(X86_FEATURE_CENTAUR_MCR)
   19.14  #define cpu_has_clflush		boot_cpu_has(X86_FEATURE_CLFLSH)
   19.15 +#define cpu_has_page1gb		0
   19.16  #else /* __x86_64__ */
   19.17  #define cpu_has_vme		0
   19.18  #define cpu_has_de		1
   19.19 @@ -166,6 +168,7 @@
   19.20  #define cpu_has_cyrix_arr	0
   19.21  #define cpu_has_centaur_mcr	0
   19.22  #define cpu_has_clflush		boot_cpu_has(X86_FEATURE_CLFLSH)
   19.23 +#define cpu_has_page1gb		boot_cpu_has(X86_FEATURE_PAGE1GB)
   19.24  #endif
   19.25  
   19.26  #define cpu_has_ffxsr           ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) \
    20.1 --- a/xen/include/asm-x86/flushtlb.h	Tue Oct 16 10:27:55 2007 +0100
    20.2 +++ b/xen/include/asm-x86/flushtlb.h	Tue Oct 16 17:31:37 2007 +0100
    20.3 @@ -57,7 +57,7 @@ do {                                    
    20.4              cpu_clear(cpu, mask);                                       \
    20.5  } while ( 0 )
    20.6  
    20.7 -extern void new_tlbflush_clock_period(void);
    20.8 +void new_tlbflush_clock_period(void);
    20.9  
   20.10  /* Read pagetable base. */
   20.11  static inline unsigned long read_cr3(void)
   20.12 @@ -69,34 +69,53 @@ static inline unsigned long read_cr3(voi
   20.13  }
   20.14  
   20.15  /* Write pagetable base and implicitly tick the tlbflush clock. */
   20.16 -extern void write_cr3(unsigned long cr3);
   20.17 -
   20.18 -/* Flush guest mappings from the TLB and implicitly tick the tlbflush clock. */
   20.19 -extern void local_flush_tlb(void);
   20.20 -
   20.21 -#define local_flush_tlb_pge()                                     \
   20.22 -    do {                                                          \
   20.23 -        __pge_off();                                              \
   20.24 -        local_flush_tlb();                                        \
   20.25 -        __pge_on();                                               \
   20.26 -    } while ( 0 )
   20.27 +void write_cr3(unsigned long cr3);
   20.28  
   20.29 -#define local_flush_tlb_one(__addr) \
   20.30 -    __asm__ __volatile__("invlpg %0": :"m" (*(char *) (__addr)))
   20.31 -
   20.32 -#define flush_tlb_all()     flush_tlb_mask(cpu_online_map)
   20.33 +/* flush_* flag fields: */
   20.34 + /*
   20.35 +  * Area to flush:
   20.36 +  *  0 -> flush entire address space
   20.37 +  *  1 -> 4kB area containing specified virtual address
   20.38 +  *  2 -> 4MB/2MB area containing specified virtual address
   20.39 +  *  3 -> 1GB area containing specified virtual address (x86/64 only)
   20.40 +  */
   20.41 +#define FLUSH_LEVEL_MASK 0x0f
   20.42 +#define FLUSH_LEVEL(x)   (x)
   20.43 + /* Flush TLBs (or parts thereof) */
   20.44 +#define FLUSH_TLB        0x10
   20.45 + /* Flush TLBs (or parts thereof) including global mappings */
   20.46 +#define FLUSH_TLB_GLOBAL 0x20
   20.47 + /* Flush data caches */
   20.48 +#define FLUSH_CACHE      0x40
   20.49  
   20.50 -#ifndef CONFIG_SMP
   20.51 -#define flush_tlb_all_pge()        local_flush_tlb_pge()
   20.52 -#define flush_tlb_mask(mask)       local_flush_tlb()
   20.53 -#define flush_tlb_one_mask(mask,v) local_flush_tlb_one(_v)
   20.54 -#else
   20.55 -#include <xen/smp.h>
   20.56 -#define FLUSHVA_ALL (~0UL)
   20.57 -extern void flush_tlb_all_pge(void);
   20.58 -extern void __flush_tlb_mask(cpumask_t mask, unsigned long va);
   20.59 -#define flush_tlb_mask(mask)       __flush_tlb_mask(mask,FLUSHVA_ALL)
   20.60 -#define flush_tlb_one_mask(mask,v) __flush_tlb_mask(mask,(unsigned long)(v))
   20.61 -#endif
   20.62 +/* Flush local TLBs/caches. */
   20.63 +void flush_area_local(const void *va, unsigned int flags);
   20.64 +#define flush_local(flags) flush_area_local(NULL, flags)
   20.65 +
   20.66 +/* Flush specified CPUs' TLBs/caches */
   20.67 +void flush_area_mask(cpumask_t, const void *va, unsigned int flags);
   20.68 +#define flush_mask(mask, flags) flush_area_mask(mask, NULL, flags)
   20.69 +
   20.70 +/* Flush all CPUs' TLBs/caches */
   20.71 +#define flush_area_all(va, flags) flush_area_mask(cpu_online_map, va, flags)
   20.72 +#define flush_all(flags) flush_mask(cpu_online_map, flags)
   20.73 +
   20.74 +/* Flush local TLBs */
   20.75 +#define flush_tlb_local()                       \
   20.76 +    flush_local(FLUSH_TLB)
   20.77 +#define flush_tlb_one_local(v)                  \
   20.78 +    flush_area_local((const void *)(v), FLUSH_TLB|1)
   20.79 +
   20.80 +/* Flush specified CPUs' TLBs */
   20.81 +#define flush_tlb_mask(mask)                    \
   20.82 +    flush_mask(mask, FLUSH_TLB)
   20.83 +#define flush_tlb_one_mask(mask,v)              \
   20.84 +    flush_area_mask(mask, (const void *)(v), FLUSH_TLB|1)
   20.85 +
   20.86 +/* Flush all CPUs' TLBs */
   20.87 +#define flush_tlb_all()                         \
   20.88 +    flush_tlb_mask(cpu_online_map)
   20.89 +#define flush_tlb_one_all(v)                    \
   20.90 +    flush_tlb_one_mask(cpu_online_map, v)
   20.91  
   20.92  #endif /* __FLUSHTLB_H__ */
    21.1 --- a/xen/include/asm-x86/mach-default/smpboot_hooks.h	Tue Oct 16 10:27:55 2007 +0100
    21.2 +++ b/xen/include/asm-x86/mach-default/smpboot_hooks.h	Tue Oct 16 17:31:37 2007 +0100
    21.3 @@ -9,7 +9,7 @@ static inline void smpboot_clear_io_apic
    21.4  static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip)
    21.5  {
    21.6  	CMOS_WRITE(0xa, 0xf);
    21.7 -	local_flush_tlb();
    21.8 +	flush_tlb_local();
    21.9  	Dprintk("1.\n");
   21.10  	*((volatile unsigned short *) TRAMPOLINE_HIGH) = start_eip >> 4;
   21.11  	Dprintk("2.\n");
   21.12 @@ -22,7 +22,7 @@ static inline void smpboot_restore_warm_
   21.13  	/*
   21.14  	 * Install writable page 0 entry to set BIOS data area.
   21.15  	 */
   21.16 -	local_flush_tlb();
   21.17 +	flush_tlb_local();
   21.18  
   21.19  	/*
   21.20  	 * Paranoid:  Set warm reset code and vector here back
    22.1 --- a/xen/include/asm-x86/processor.h	Tue Oct 16 10:27:55 2007 +0100
    22.2 +++ b/xen/include/asm-x86/processor.h	Tue Oct 16 17:31:37 2007 +0100
    22.3 @@ -160,28 +160,22 @@ struct vcpu;
    22.4  #endif
    22.5  
    22.6  struct cpuinfo_x86 {
    22.7 -    __u8 x86;		/* CPU family */
    22.8 -    __u8 x86_vendor;	/* CPU vendor */
    22.9 +    __u8 x86;            /* CPU family */
   22.10 +    __u8 x86_vendor;     /* CPU vendor */
   22.11      __u8 x86_model;
   22.12      __u8 x86_mask;
   22.13 -    char wp_works_ok;	/* It doesn't on 386's */
   22.14 -    char hlt_works_ok;	/* Problems on some 486Dx4's and old 386's */
   22.15 -    char hard_math;
   22.16 -    char rfu;
   22.17 -    int  cpuid_level;	/* Maximum supported CPUID level, -1=no CPUID */
   22.18 +    __u8 invlpg_works_ok;
   22.19 +    int  cpuid_level;    /* Maximum supported CPUID level, -1=no CPUID */
   22.20      unsigned int x86_capability[NCAPINTS];
   22.21      char x86_vendor_id[16];
   22.22      char x86_model_id[64];
   22.23 -    int  x86_cache_size;  /* in KB - valid for CPUS which support this call  */
   22.24 -    int  x86_cache_alignment;	/* In bytes */
   22.25 -    char fdiv_bug;
   22.26 -    char f00f_bug;
   22.27 -    char coma_bug;
   22.28 -    char pad0;
   22.29 +    int  x86_cache_size; /* in KB - valid for CPUS which support this call  */
   22.30 +    int  x86_cache_alignment;    /* In bytes */
   22.31      int  x86_power;
   22.32      unsigned char x86_max_cores; /* cpuid returned max cores value */
   22.33 -    unsigned char booted_cores; /* number of cores as seen by OS */
   22.34 +    unsigned char booted_cores;  /* number of cores as seen by OS */
   22.35      unsigned char apicid;
   22.36 +    unsigned short x86_clflush_size;
   22.37  } __cacheline_aligned;
   22.38  
   22.39  /*