From: Keir Fraser Date: Tue, 16 Oct 2007 16:31:37 +0000 (+0100) Subject: x86: consolidate/enhance TLB flushing interface X-Git-Tag: 3.2.0-rc1~228^2~29 X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=19435c10abf79a343c267baaa4e16d349ee93333;p=xen.git x86: consolidate/enhance TLB flushing interface Folding into a single local handler and a single SMP multiplexor as well as adding capability to also flush caches through the same interfaces (a subsequent patch will make use of this). Once at changing cpuinfo_x86, this patch also removes several unused fields apparently inherited from Linux. Signed-off-by: Jan Beulich Signed-off-by: Keir Fraser --- diff --git a/xen/arch/x86/cpu/amd.c b/xen/arch/x86/cpu/amd.c index 13fec06717..4ed95ea6a7 100644 --- a/xen/arch/x86/cpu/amd.c +++ b/xen/arch/x86/cpu/amd.c @@ -373,6 +373,11 @@ static void __init init_amd(struct cpuinfo_x86 *c) if ((smp_processor_id() == 1) && c1_ramping_may_cause_clock_drift(c)) disable_c1_ramping(); + /* Support INVLPG of superpages? */ + __set_bit(2, &c->invlpg_works_ok); + if ( cpu_has(c, X86_FEATURE_PAGE1GB) ) + __set_bit(3, &c->invlpg_works_ok); + start_svm(c); } diff --git a/xen/arch/x86/cpu/common.c b/xen/arch/x86/cpu/common.c index 6cedb8fd71..c333ac36f2 100644 --- a/xen/arch/x86/cpu/common.c +++ b/xen/arch/x86/cpu/common.c @@ -229,7 +229,6 @@ static void __init early_cpu_detect(void) void __devinit generic_identify(struct cpuinfo_x86 * c) { u32 tfms, xlvl; - int junk; if (have_cpuid_p()) { /* Get vendor name */ @@ -244,8 +243,8 @@ void __devinit generic_identify(struct cpuinfo_x86 * c) /* Intel-defined flags: level 0x00000001 */ if ( c->cpuid_level >= 0x00000001 ) { - u32 capability, excap; - cpuid(0x00000001, &tfms, &junk, &excap, &capability); + u32 capability, excap, ebx; + cpuid(0x00000001, &tfms, &ebx, &excap, &capability); c->x86_capability[0] = capability; c->x86_capability[4] = excap; c->x86 = (tfms >> 8) & 15; @@ -255,6 +254,8 @@ void __devinit generic_identify(struct cpuinfo_x86 * c) c->x86_model += ((tfms >> 16) & 0xF) << 4; } c->x86_mask = tfms & 15; + if ( cpu_has(c, X86_FEATURE_CLFLSH) ) + c->x86_clflush_size = ((ebx >> 8) & 0xff) * 8; } else { /* Have CPUID level 0 only - unheard of */ c->x86 = 4; @@ -313,6 +314,7 @@ void __devinit identify_cpu(struct cpuinfo_x86 *c) c->x86_vendor_id[0] = '\0'; /* Unset */ c->x86_model_id[0] = '\0'; /* Unset */ c->x86_max_cores = 1; + c->x86_clflush_size = 0; memset(&c->x86_capability, 0, sizeof c->x86_capability); if (!have_cpuid_p()) { diff --git a/xen/arch/x86/cpu/cyrix.c b/xen/arch/x86/cpu/cyrix.c index 705afbf18f..66e7e75143 100644 --- a/xen/arch/x86/cpu/cyrix.c +++ b/xen/arch/x86/cpu/cyrix.c @@ -239,7 +239,7 @@ static void __init init_cyrix(struct cpuinfo_x86 *c) /* Emulate MTRRs using Cyrix's ARRs. */ set_bit(X86_FEATURE_CYRIX_ARR, c->x86_capability); /* 6x86's contain this bug */ - c->coma_bug = 1; + /*c->coma_bug = 1;*/ break; case 4: /* MediaGX/GXm or Geode GXM/GXLV/GX1 */ @@ -272,7 +272,7 @@ static void __init init_cyrix(struct cpuinfo_x86 *c) } else { - c->coma_bug = 1; /* 6x86MX, it has the bug. */ + /*c->coma_bug = 1;*/ /* 6x86MX, it has the bug. */ } tmp = (!(dir0_lsn & 7) || dir0_lsn & 1) ? 2 : 0; Cx86_cb[tmp] = cyrix_model_mult2[dir0_lsn & 7]; @@ -287,7 +287,7 @@ static void __init init_cyrix(struct cpuinfo_x86 *c) switch (dir0_lsn) { case 0xd: /* either a 486SLC or DLC w/o DEVID */ dir0_msn = 0; - p = Cx486_name[(c->hard_math) ? 1 : 0]; + p = Cx486_name[/*(c->hard_math) ? 1 : 0*/1]; break; case 0xe: /* a 486S A step */ diff --git a/xen/arch/x86/cpu/intel.c b/xen/arch/x86/cpu/intel.c index 5fcad2403a..c3daf2987d 100644 --- a/xen/arch/x86/cpu/intel.c +++ b/xen/arch/x86/cpu/intel.c @@ -123,6 +123,18 @@ static void __devinit init_intel(struct cpuinfo_x86 *c) if ((c->x86<<8 | c->x86_model<<4 | c->x86_mask) < 0x633) clear_bit(X86_FEATURE_SEP, c->x86_capability); + /* Supports INVLPG of superpages? */ + __set_bit(2, &c->invlpg_works_ok); + if (/* PentiumPro erratum 30 */ + (c->x86 == 6 && c->x86_model == 1 && c->x86_mask < 9) || + /* Dual-Core Intel Xeon 3000/5100 series erratum 89/90 */ + /* Quad-Core Intel Xeon 3200/5300 series erratum 89/88 */ + /* Intel Core2 erratum 89 */ + (c->x86 == 6 && c->x86_model == 15 ) || + /* Dual-Core Intel Xeon LV/ULV erratum 75 */ + (c->x86 == 6 && c->x86_model == 14 )) + __clear_bit(2, &c->invlpg_works_ok); + /* Names for the Pentium II/Celeron processors detectable only by also checking the cache size. Dixon is NOT a Celeron. */ diff --git a/xen/arch/x86/cpu/mtrr/generic.c b/xen/arch/x86/cpu/mtrr/generic.c index ac597f3742..d654aa6fb4 100644 --- a/xen/arch/x86/cpu/mtrr/generic.c +++ b/xen/arch/x86/cpu/mtrr/generic.c @@ -313,7 +313,7 @@ static void prepare_set(void) } /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ - local_flush_tlb(); + flush_tlb_local(); /* Save MTRR state */ rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi); @@ -325,7 +325,7 @@ static void prepare_set(void) static void post_set(void) { /* Flush TLBs (no need to flush caches - they are disabled) */ - local_flush_tlb(); + flush_tlb_local(); /* Intel (P6) standard MTRRs */ mtrr_wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi); diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index 7aad9ed0cb..12b76a8a02 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -1299,7 +1299,7 @@ void context_switch(struct vcpu *prev, struct vcpu *next) { uint64_t efer = read_efer(); - local_flush_tlb_one(GDT_VIRT_START(next) + + flush_tlb_one_local(GDT_VIRT_START(next) + FIRST_RESERVED_GDT_BYTE); if ( !is_pv_32on64_vcpu(next) == !(efer & EFER_SCE) ) diff --git a/xen/arch/x86/domain_build.c b/xen/arch/x86/domain_build.c index c8deceef83..0fe3b2adc5 100644 --- a/xen/arch/x86/domain_build.c +++ b/xen/arch/x86/domain_build.c @@ -347,7 +347,7 @@ int __init construct_dom0( for ( i = 0; i < MAX_VIRT_CPUS; i++ ) d->arch.mm_perdomain_pt[((i << GDT_LDT_VCPU_SHIFT) + FIRST_RESERVED_GDT_PAGE)] = gdt_l1e; - local_flush_tlb_one(GDT_LDT_VIRT_START + FIRST_RESERVED_GDT_BYTE); + flush_tlb_one_local(GDT_LDT_VIRT_START + FIRST_RESERVED_GDT_BYTE); } #endif if ( parms.pae == PAEKERN_extended_cr3 ) diff --git a/xen/arch/x86/flushtlb.c b/xen/arch/x86/flushtlb.c index f7c06f30ef..69aa7fa21d 100644 --- a/xen/arch/x86/flushtlb.c +++ b/xen/arch/x86/flushtlb.c @@ -84,10 +84,10 @@ void write_cr3(unsigned long cr3) #ifdef USER_MAPPINGS_ARE_GLOBAL __pge_off(); - __asm__ __volatile__ ( "mov %0, %%cr3" : : "r" (cr3) : "memory" ); + asm volatile ( "mov %0, %%cr3" : : "r" (cr3) : "memory" ); __pge_on(); #else - __asm__ __volatile__ ( "mov %0, %%cr3" : : "r" (cr3) : "memory" ); + asm volatile ( "mov %0, %%cr3" : : "r" (cr3) : "memory" ); #endif post_flush(t); @@ -95,26 +95,69 @@ void write_cr3(unsigned long cr3) local_irq_restore(flags); } -void local_flush_tlb(void) +void flush_area_local(const void *va, unsigned int flags) { - unsigned long flags; - u32 t; + const struct cpuinfo_x86 *c = ¤t_cpu_data; + unsigned int level = flags & FLUSH_LEVEL_MASK; + unsigned long irqfl; - /* This non-reentrant function is sometimes called in interrupt context. */ - local_irq_save(flags); + ASSERT(level < CONFIG_PAGING_LEVELS); - t = pre_flush(); - - hvm_flush_guest_tlbs(); - -#ifdef USER_MAPPINGS_ARE_GLOBAL - __pge_off(); - __pge_on(); -#else - __asm__ __volatile__ ( "mov %0, %%cr3" : : "r" (read_cr3()) : "memory" ); + /* This non-reentrant function is sometimes called in interrupt context. */ + local_irq_save(irqfl); + + if ( flags & (FLUSH_TLB|FLUSH_TLB_GLOBAL) ) + { + if ( (level != 0) && test_bit(level, &c->invlpg_works_ok) ) + { + asm volatile ( "invlpg %0" + : : "m" (*(const char *)(va)) : "memory" ); + } + else + { + u32 t = pre_flush(); + + hvm_flush_guest_tlbs(); + +#ifndef USER_MAPPINGS_ARE_GLOBAL + if ( !(flags & FLUSH_TLB_GLOBAL) || + !(mmu_cr4_features & X86_CR4_PGE) ) + { + asm volatile ( "mov %0, %%cr3" + : : "r" (read_cr3()) : "memory" ); + } + else #endif + { + __pge_off(); + barrier(); + __pge_on(); + } + + post_flush(t); + } + } - post_flush(t); + if ( flags & FLUSH_CACHE ) + { + unsigned long i, sz; + + sz = level ? (1UL << ((level - 1) * PAGETABLE_ORDER)) : ULONG_MAX; + + if ( c->x86_clflush_size && c->x86_cache_size && + (sz < (c->x86_cache_size >> (PAGE_SHIFT - 10))) ) + { + sz <<= PAGE_SHIFT; + va = (const void *)((unsigned long)va & ~(sz - 1)); + for ( i = 0; i < sz; i += c->x86_clflush_size ) + asm volatile ( "clflush %0" + : : "m" (((const char *)va)[i]) ); + } + else + { + wbinvd(); + } + } - local_irq_restore(flags); + local_irq_restore(irqfl); } diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index 94f8f89182..fa6bac5a89 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -372,7 +372,7 @@ void make_cr3(struct vcpu *v, unsigned long mfn) /* First check the previous high mapping can't be in the TLB. * (i.e. have we loaded CR3 since we last did this?) */ if ( unlikely(this_cpu(make_cr3_timestamp) == this_cpu(tlbflush_time)) ) - local_flush_tlb_one(fix_to_virt(FIX_PAE_HIGHMEM_0 + cpu)); + flush_tlb_one_local(fix_to_virt(FIX_PAE_HIGHMEM_0 + cpu)); highmem_l3tab = (l3_pgentry_t *)fix_to_virt(FIX_PAE_HIGHMEM_0 + cpu); lowmem_l3tab = cache->table[cache->inuse_idx]; memcpy(lowmem_l3tab, highmem_l3tab, sizeof(cache->table[0])); @@ -1886,7 +1886,7 @@ static void process_deferred_ops(void) if ( deferred_ops & DOP_FLUSH_ALL_TLBS ) flush_tlb_mask(d->domain_dirty_cpumask); else - local_flush_tlb(); + flush_tlb_local(); } if ( deferred_ops & DOP_RELOAD_LDT ) @@ -2172,7 +2172,7 @@ int do_mmuext_op( case MMUEXT_INVLPG_LOCAL: if ( !paging_mode_enabled(d) || paging_invlpg(v, op.arg1.linear_addr) != 0 ) - local_flush_tlb_one(op.arg1.linear_addr); + flush_tlb_one_local(op.arg1.linear_addr); break; case MMUEXT_TLB_FLUSH_MULTI: @@ -2848,7 +2848,7 @@ int do_update_va_mapping(unsigned long va, u64 val64, switch ( (bmap_ptr = flags & ~UVMF_FLUSHTYPE_MASK) ) { case UVMF_LOCAL: - local_flush_tlb(); + flush_tlb_local(); break; case UVMF_ALL: flush_tlb_mask(d->domain_dirty_cpumask); @@ -2870,7 +2870,7 @@ int do_update_va_mapping(unsigned long va, u64 val64, case UVMF_LOCAL: if ( !paging_mode_enabled(d) || (paging_invlpg(current, va) != 0) ) - local_flush_tlb_one(va); + flush_tlb_one_local(va); break; case UVMF_ALL: flush_tlb_one_mask(d->domain_dirty_cpumask, va); @@ -2989,7 +2989,7 @@ long do_set_gdt(XEN_GUEST_HANDLE(ulong) frame_list, unsigned int entries) LOCK_BIGLOCK(current->domain); if ( (ret = set_gdt(current, frames, entries)) == 0 ) - local_flush_tlb(); + flush_tlb_local(); UNLOCK_BIGLOCK(current->domain); @@ -3544,7 +3544,8 @@ int map_pages_to_xen( if ( (l2e_get_flags(ol2e) & _PAGE_PRESENT) ) { - local_flush_tlb_pge(); + flush_area_local((const void *)virt, + FLUSH_TLB_GLOBAL|FLUSH_LEVEL(2)); if ( !(l2e_get_flags(ol2e) & _PAGE_PSE) ) free_xen_pagetable(mfn_to_virt(l2e_get_pfn(ol2e))); } @@ -3572,14 +3573,15 @@ int map_pages_to_xen( l2e_get_flags(*pl2e) & ~_PAGE_PSE)); l2e_write_atomic(pl2e, l2e_from_pfn(virt_to_mfn(pl1e), __PAGE_HYPERVISOR)); - local_flush_tlb_pge(); + flush_area_local((const void *)virt, + FLUSH_TLB_GLOBAL|FLUSH_LEVEL(2)); } pl1e = l2e_to_l1e(*pl2e) + l1_table_offset(virt); ol1e = *pl1e; l1e_write_atomic(pl1e, l1e_from_pfn(mfn, flags)); if ( (l1e_get_flags(ol1e) & _PAGE_PRESENT) ) - local_flush_tlb_one(virt); + flush_tlb_one_local(virt); virt += 1UL << L1_PAGETABLE_SHIFT; mfn += 1UL; @@ -3655,7 +3657,7 @@ void destroy_xen_mappings(unsigned long s, unsigned long e) } } - flush_tlb_all_pge(); + flush_all(FLUSH_TLB_GLOBAL); } void __set_fixmap( diff --git a/xen/arch/x86/mm/p2m.c b/xen/arch/x86/mm/p2m.c index fb0394282a..0dfdaaceb5 100644 --- a/xen/arch/x86/mm/p2m.c +++ b/xen/arch/x86/mm/p2m.c @@ -493,7 +493,7 @@ static void audit_p2m(struct domain *d) test_linear = ( (d == current->domain) && !pagetable_is_null(current->arch.monitor_table) ); if ( test_linear ) - local_flush_tlb(); + flush_tlb_local(); /* Audit part one: walk the domain's page allocation list, checking * the m2p entries. */ diff --git a/xen/arch/x86/mm/shadow/multi.c b/xen/arch/x86/mm/shadow/multi.c index c64d9552fe..49c01dcce4 100644 --- a/xen/arch/x86/mm/shadow/multi.c +++ b/xen/arch/x86/mm/shadow/multi.c @@ -3089,7 +3089,7 @@ sh_invlpg(struct vcpu *v, unsigned long va) if ( mfn_to_shadow_page(shadow_l2e_get_mfn(sl2e))->type == SH_type_fl1_shadow ) { - local_flush_tlb(); + flush_tlb_local(); return 0; } diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c index f1f161597c..c0e62fdada 100644 --- a/xen/arch/x86/setup.c +++ b/xen/arch/x86/setup.c @@ -114,7 +114,7 @@ struct tss_struct init_tss[NR_CPUS]; char __attribute__ ((__section__(".bss.stack_aligned"))) cpu0_stack[STACK_SIZE]; -struct cpuinfo_x86 boot_cpu_data = { 0, 0, 0, 0, -1, 1, 0, 0, -1 }; +struct cpuinfo_x86 boot_cpu_data = { 0, 0, 0, 0, 1, -1 }; #if CONFIG_PAGING_LEVELS > 2 unsigned long mmu_cr4_features = X86_CR4_PSE | X86_CR4_PGE | X86_CR4_PAE; diff --git a/xen/arch/x86/smp.c b/xen/arch/x86/smp.c index 660c26436f..1d575e678e 100644 --- a/xen/arch/x86/smp.c +++ b/xen/arch/x86/smp.c @@ -164,34 +164,28 @@ void send_IPI_mask_phys(cpumask_t mask, int vector) static DEFINE_SPINLOCK(flush_lock); static cpumask_t flush_cpumask; -static unsigned long flush_va; +static const void *flush_va; +static unsigned int flush_flags; fastcall void smp_invalidate_interrupt(void) { ack_APIC_irq(); perfc_incr(ipis); irq_enter(); - if ( !__sync_lazy_execstate() ) - { - if ( flush_va == FLUSHVA_ALL ) - local_flush_tlb(); - else - local_flush_tlb_one(flush_va); - } + if ( !__sync_lazy_execstate() || + (flush_flags & (FLUSH_TLB_GLOBAL | FLUSH_CACHE)) ) + flush_area_local(flush_va, flush_flags); cpu_clear(smp_processor_id(), flush_cpumask); irq_exit(); } -void __flush_tlb_mask(cpumask_t mask, unsigned long va) +void flush_area_mask(cpumask_t mask, const void *va, unsigned int flags) { ASSERT(local_irq_is_enabled()); if ( cpu_isset(smp_processor_id(), mask) ) { - if ( va == FLUSHVA_ALL ) - local_flush_tlb(); - else - local_flush_tlb_one(va); + flush_area_local(va, flags); cpu_clear(smp_processor_id(), mask); } @@ -200,6 +194,7 @@ void __flush_tlb_mask(cpumask_t mask, unsigned long va) spin_lock(&flush_lock); flush_cpumask = mask; flush_va = va; + flush_flags = flags; send_IPI_mask(mask, INVALIDATE_TLB_VECTOR); while ( !cpus_empty(flush_cpumask) ) cpu_relax(); @@ -215,24 +210,13 @@ void new_tlbflush_clock_period(void) /* Flush everyone else. We definitely flushed just before entry. */ allbutself = cpu_online_map; cpu_clear(smp_processor_id(), allbutself); - __flush_tlb_mask(allbutself, FLUSHVA_ALL); + flush_mask(allbutself, FLUSH_TLB); /* No need for atomicity: we are the only possible updater. */ ASSERT(tlbflush_clock == 0); tlbflush_clock++; } -static void flush_tlb_all_pge_ipi(void *info) -{ - local_flush_tlb_pge(); -} - -void flush_tlb_all_pge(void) -{ - smp_call_function(flush_tlb_all_pge_ipi, 0, 1, 1); - local_flush_tlb_pge(); -} - void smp_send_event_check_mask(cpumask_t mask) { cpu_clear(smp_processor_id(), mask); diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c index 90846524a9..9ffa664789 100644 --- a/xen/arch/x86/smpboot.c +++ b/xen/arch/x86/smpboot.c @@ -518,7 +518,7 @@ void __devinit start_secondary(void *unused) * low-memory mappings have been cleared, flush them from * the local TLBs too. */ - local_flush_tlb(); + flush_tlb_local(); /* This must be done before setting cpu_online_map */ set_cpu_sibling_map(raw_smp_processor_id()); diff --git a/xen/arch/x86/x86_32/domain_page.c b/xen/arch/x86/x86_32/domain_page.c index fc6e1125d0..ff160c43e2 100644 --- a/xen/arch/x86/x86_32/domain_page.c +++ b/xen/arch/x86/x86_32/domain_page.c @@ -78,7 +78,7 @@ void *map_domain_page(unsigned long mfn) if ( NEED_FLUSH(this_cpu(tlbflush_time), dcache->tlbflush_timestamp) ) { perfc_incr(domain_page_tlb_flush); - local_flush_tlb(); + flush_tlb_local(); } } @@ -94,7 +94,7 @@ void *map_domain_page(unsigned long mfn) /* /Second/, flush TLBs. */ perfc_incr(domain_page_tlb_flush); - local_flush_tlb(); + flush_tlb_local(); vcache->shadow_epoch = ++dcache->epoch; dcache->tlbflush_timestamp = tlbflush_current_time(); diff --git a/xen/arch/x86/x86_32/mm.c b/xen/arch/x86/x86_32/mm.c index 9b3780f0c9..a0c751eaf5 100644 --- a/xen/arch/x86/x86_32/mm.c +++ b/xen/arch/x86/x86_32/mm.c @@ -152,7 +152,7 @@ void __init zap_low_mappings(l2_pgentry_t *dom0_l2) /* Now zap mappings in the idle pagetables. */ destroy_xen_mappings(0, HYPERVISOR_VIRT_START); - flush_tlb_all_pge(); + flush_all(FLUSH_TLB_GLOBAL); /* Replace with mapping of the boot trampoline only. */ map_pages_to_xen(BOOT_TRAMPOLINE, BOOT_TRAMPOLINE >> PAGE_SHIFT, diff --git a/xen/arch/x86/x86_64/compat/mm.c b/xen/arch/x86/x86_64/compat/mm.c index a9c136323f..6fc1206c5f 100644 --- a/xen/arch/x86/x86_64/compat/mm.c +++ b/xen/arch/x86/x86_64/compat/mm.c @@ -31,7 +31,7 @@ int compat_set_gdt(XEN_GUEST_HANDLE(uint) frame_list, unsigned int entries) LOCK_BIGLOCK(current->domain); if ( (ret = set_gdt(current, frames, entries)) == 0 ) - local_flush_tlb(); + flush_tlb_local(); UNLOCK_BIGLOCK(current->domain); diff --git a/xen/arch/x86/x86_64/mm.c b/xen/arch/x86/x86_64/mm.c index 596bf5cecd..af551a05d7 100644 --- a/xen/arch/x86/x86_64/mm.c +++ b/xen/arch/x86/x86_64/mm.c @@ -205,7 +205,7 @@ void __init zap_low_mappings(void) /* Remove aliased mapping of first 1:1 PML4 entry. */ l4e_write(&idle_pg_table[0], l4e_empty()); - local_flush_tlb_pge(); + flush_local(FLUSH_TLB_GLOBAL); /* Replace with mapping of the boot trampoline only. */ map_pages_to_xen(BOOT_TRAMPOLINE, BOOT_TRAMPOLINE >> PAGE_SHIFT, diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h index b281c5f780..ec16c923ff 100644 --- a/xen/include/asm-x86/cpufeature.h +++ b/xen/include/asm-x86/cpufeature.h @@ -50,6 +50,7 @@ #define X86_FEATURE_NX (1*32+20) /* Execute Disable */ #define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */ #define X86_FEATURE_FFXSR (1*32+25) /* FFXSR instruction optimizations */ +#define X86_FEATURE_PAGE1GB (1*32+26) /* 1Gb large page support */ #define X86_FEATURE_RDTSCP (1*32+27) /* RDTSCP */ #define X86_FEATURE_LM (1*32+29) /* Long Mode (x86-64) */ #define X86_FEATURE_3DNOWEXT (1*32+30) /* AMD 3DNow! extensions */ @@ -143,6 +144,7 @@ #define cpu_has_cyrix_arr boot_cpu_has(X86_FEATURE_CYRIX_ARR) #define cpu_has_centaur_mcr boot_cpu_has(X86_FEATURE_CENTAUR_MCR) #define cpu_has_clflush boot_cpu_has(X86_FEATURE_CLFLSH) +#define cpu_has_page1gb 0 #else /* __x86_64__ */ #define cpu_has_vme 0 #define cpu_has_de 1 @@ -166,6 +168,7 @@ #define cpu_has_cyrix_arr 0 #define cpu_has_centaur_mcr 0 #define cpu_has_clflush boot_cpu_has(X86_FEATURE_CLFLSH) +#define cpu_has_page1gb boot_cpu_has(X86_FEATURE_PAGE1GB) #endif #define cpu_has_ffxsr ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) \ diff --git a/xen/include/asm-x86/flushtlb.h b/xen/include/asm-x86/flushtlb.h index 03d132797c..97a60e6b25 100644 --- a/xen/include/asm-x86/flushtlb.h +++ b/xen/include/asm-x86/flushtlb.h @@ -57,7 +57,7 @@ do { \ cpu_clear(cpu, mask); \ } while ( 0 ) -extern void new_tlbflush_clock_period(void); +void new_tlbflush_clock_period(void); /* Read pagetable base. */ static inline unsigned long read_cr3(void) @@ -69,34 +69,53 @@ static inline unsigned long read_cr3(void) } /* Write pagetable base and implicitly tick the tlbflush clock. */ -extern void write_cr3(unsigned long cr3); - -/* Flush guest mappings from the TLB and implicitly tick the tlbflush clock. */ -extern void local_flush_tlb(void); - -#define local_flush_tlb_pge() \ - do { \ - __pge_off(); \ - local_flush_tlb(); \ - __pge_on(); \ - } while ( 0 ) - -#define local_flush_tlb_one(__addr) \ - __asm__ __volatile__("invlpg %0": :"m" (*(char *) (__addr))) - -#define flush_tlb_all() flush_tlb_mask(cpu_online_map) - -#ifndef CONFIG_SMP -#define flush_tlb_all_pge() local_flush_tlb_pge() -#define flush_tlb_mask(mask) local_flush_tlb() -#define flush_tlb_one_mask(mask,v) local_flush_tlb_one(_v) -#else -#include -#define FLUSHVA_ALL (~0UL) -extern void flush_tlb_all_pge(void); -extern void __flush_tlb_mask(cpumask_t mask, unsigned long va); -#define flush_tlb_mask(mask) __flush_tlb_mask(mask,FLUSHVA_ALL) -#define flush_tlb_one_mask(mask,v) __flush_tlb_mask(mask,(unsigned long)(v)) -#endif +void write_cr3(unsigned long cr3); + +/* flush_* flag fields: */ + /* + * Area to flush: + * 0 -> flush entire address space + * 1 -> 4kB area containing specified virtual address + * 2 -> 4MB/2MB area containing specified virtual address + * 3 -> 1GB area containing specified virtual address (x86/64 only) + */ +#define FLUSH_LEVEL_MASK 0x0f +#define FLUSH_LEVEL(x) (x) + /* Flush TLBs (or parts thereof) */ +#define FLUSH_TLB 0x10 + /* Flush TLBs (or parts thereof) including global mappings */ +#define FLUSH_TLB_GLOBAL 0x20 + /* Flush data caches */ +#define FLUSH_CACHE 0x40 + +/* Flush local TLBs/caches. */ +void flush_area_local(const void *va, unsigned int flags); +#define flush_local(flags) flush_area_local(NULL, flags) + +/* Flush specified CPUs' TLBs/caches */ +void flush_area_mask(cpumask_t, const void *va, unsigned int flags); +#define flush_mask(mask, flags) flush_area_mask(mask, NULL, flags) + +/* Flush all CPUs' TLBs/caches */ +#define flush_area_all(va, flags) flush_area_mask(cpu_online_map, va, flags) +#define flush_all(flags) flush_mask(cpu_online_map, flags) + +/* Flush local TLBs */ +#define flush_tlb_local() \ + flush_local(FLUSH_TLB) +#define flush_tlb_one_local(v) \ + flush_area_local((const void *)(v), FLUSH_TLB|1) + +/* Flush specified CPUs' TLBs */ +#define flush_tlb_mask(mask) \ + flush_mask(mask, FLUSH_TLB) +#define flush_tlb_one_mask(mask,v) \ + flush_area_mask(mask, (const void *)(v), FLUSH_TLB|1) + +/* Flush all CPUs' TLBs */ +#define flush_tlb_all() \ + flush_tlb_mask(cpu_online_map) +#define flush_tlb_one_all(v) \ + flush_tlb_one_mask(cpu_online_map, v) #endif /* __FLUSHTLB_H__ */ diff --git a/xen/include/asm-x86/mach-default/smpboot_hooks.h b/xen/include/asm-x86/mach-default/smpboot_hooks.h index 104de09d4d..5b4dac9518 100644 --- a/xen/include/asm-x86/mach-default/smpboot_hooks.h +++ b/xen/include/asm-x86/mach-default/smpboot_hooks.h @@ -9,7 +9,7 @@ static inline void smpboot_clear_io_apic_irqs(void) static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip) { CMOS_WRITE(0xa, 0xf); - local_flush_tlb(); + flush_tlb_local(); Dprintk("1.\n"); *((volatile unsigned short *) TRAMPOLINE_HIGH) = start_eip >> 4; Dprintk("2.\n"); @@ -22,7 +22,7 @@ static inline void smpboot_restore_warm_reset_vector(void) /* * Install writable page 0 entry to set BIOS data area. */ - local_flush_tlb(); + flush_tlb_local(); /* * Paranoid: Set warm reset code and vector here back diff --git a/xen/include/asm-x86/processor.h b/xen/include/asm-x86/processor.h index 9acdd6a1d9..8ed9ecb67c 100644 --- a/xen/include/asm-x86/processor.h +++ b/xen/include/asm-x86/processor.h @@ -160,28 +160,22 @@ struct vcpu; #endif struct cpuinfo_x86 { - __u8 x86; /* CPU family */ - __u8 x86_vendor; /* CPU vendor */ + __u8 x86; /* CPU family */ + __u8 x86_vendor; /* CPU vendor */ __u8 x86_model; __u8 x86_mask; - char wp_works_ok; /* It doesn't on 386's */ - char hlt_works_ok; /* Problems on some 486Dx4's and old 386's */ - char hard_math; - char rfu; - int cpuid_level; /* Maximum supported CPUID level, -1=no CPUID */ + __u8 invlpg_works_ok; + int cpuid_level; /* Maximum supported CPUID level, -1=no CPUID */ unsigned int x86_capability[NCAPINTS]; char x86_vendor_id[16]; char x86_model_id[64]; - int x86_cache_size; /* in KB - valid for CPUS which support this call */ - int x86_cache_alignment; /* In bytes */ - char fdiv_bug; - char f00f_bug; - char coma_bug; - char pad0; + int x86_cache_size; /* in KB - valid for CPUS which support this call */ + int x86_cache_alignment; /* In bytes */ int x86_power; unsigned char x86_max_cores; /* cpuid returned max cores value */ - unsigned char booted_cores; /* number of cores as seen by OS */ + unsigned char booted_cores; /* number of cores as seen by OS */ unsigned char apicid; + unsigned short x86_clflush_size; } __cacheline_aligned; /*