ia64/xen-unstable

changeset 4406:c0a3bd2e46cb

bitkeeper revision 1.1236.1.176 (424c2d3fy4zylRn8ctgTwLuwa-5b1Q)

Fix SMP TLB flushing in XenLinux 2.6.
Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Thu Mar 31 17:02:55 2005 +0000 (2005-03-31)
parents 198cfa01d71b
children e0ebda45aae6
files linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h
line diff
     1.1 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c	Thu Mar 31 15:44:24 2005 +0000
     1.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/pci-dma.c	Thu Mar 31 17:02:55 2005 +0000
     1.3 @@ -13,6 +13,7 @@
     1.4  #include <linux/pci.h>
     1.5  #include <linux/version.h>
     1.6  #include <asm/io.h>
     1.7 +#include <asm/tlbflush.h>
     1.8  #include <asm-xen/balloon.h>
     1.9  
    1.10  #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
    1.11 @@ -78,9 +79,9 @@ xen_contig_memory(unsigned long vstart, 
    1.12  		phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
    1.13  			pfn+i;
    1.14  	}
    1.15 -	xen_tlb_flush();
    1.16 +	flush_tlb_all();
    1.17  
    1.18 -        balloon_unlock(flags);
    1.19 +	balloon_unlock(flags);
    1.20  }
    1.21  
    1.22  #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
     2.1 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c	Thu Mar 31 15:44:24 2005 +0000
     2.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c	Thu Mar 31 17:02:55 2005 +0000
     2.3 @@ -203,6 +203,7 @@ inline void send_IPI_mask_sequence(cpuma
     2.4  
     2.5  #include <mach_ipi.h> /* must come after the send_IPI functions above for inlining */
     2.6  
     2.7 +#if 0 /* XEN */
     2.8  /*
     2.9   *	Smarter SMP flushing macros. 
    2.10   *		c/o Linus Torvalds.
    2.11 @@ -441,6 +442,22 @@ void flush_tlb_all(void)
    2.12  	on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
    2.13  }
    2.14  
    2.15 +#else
    2.16 +
    2.17 +irqreturn_t smp_invalidate_interrupt(int irq, void *dev_id,
    2.18 +				     struct pt_regs *regs)
    2.19 +{ return 0; }
    2.20 +void flush_tlb_current_task(void)
    2.21 +{ xen_tlb_flush_mask(current->mm->cpu_vm_mask); }
    2.22 +void flush_tlb_mm(struct mm_struct * mm)
    2.23 +{ xen_tlb_flush_mask(mm->cpu_vm_mask); }
    2.24 +void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
    2.25 +{ xen_invlpg_mask(vma->vm_mm->cpu_vm_mask, va); }
    2.26 +void flush_tlb_all(void)
    2.27 +{ xen_tlb_flush_all(); }
    2.28 +
    2.29 +#endif /* XEN */
    2.30 +
    2.31  /*
    2.32   * this function sends a 'reschedule' IPI to another CPU.
    2.33   * it goes straight through and wastes no time serializing
     3.1 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c	Thu Mar 31 15:44:24 2005 +0000
     3.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c	Thu Mar 31 17:02:55 2005 +0000
     3.3 @@ -33,6 +33,7 @@
     3.4  #include <linux/vmalloc.h>
     3.5  #include <asm/page.h>
     3.6  #include <asm/pgtable.h>
     3.7 +#include <asm/tlbflush.h>
     3.8  #include <asm-xen/hypervisor.h>
     3.9  #include <asm-xen/balloon.h>
    3.10  #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0)
    3.11 @@ -94,6 +95,42 @@ void xen_invlpg(unsigned long ptr)
    3.12      BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    3.13  }
    3.14  
    3.15 +#ifdef CONFIG_SMP
    3.16 +
    3.17 +void xen_tlb_flush_all(void)
    3.18 +{
    3.19 +    struct mmuext_op op;
    3.20 +    op.cmd = MMUEXT_TLB_FLUSH_ALL;
    3.21 +    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    3.22 +}
    3.23 +
    3.24 +void xen_tlb_flush_mask(cpumask_t mask)
    3.25 +{
    3.26 +    struct mmuext_op op;
    3.27 +    op.cmd = MMUEXT_TLB_FLUSH_MULTI;
    3.28 +    op.cpuset = mask.bits[0];
    3.29 +    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    3.30 +}
    3.31 +
    3.32 +void xen_invlpg_all(unsigned long ptr)
    3.33 +{
    3.34 +    struct mmuext_op op;
    3.35 +    op.cmd = MMUEXT_INVLPG_ALL;
    3.36 +    op.linear_addr = ptr & PAGE_MASK;
    3.37 +    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    3.38 +}
    3.39 +
    3.40 +void xen_invlpg_mask(cpumask_t mask, unsigned long ptr)
    3.41 +{
    3.42 +    struct mmuext_op op;
    3.43 +    op.cmd = MMUEXT_INVLPG_MULTI;
    3.44 +    op.cpuset = mask.bits[0];
    3.45 +    op.linear_addr = ptr & PAGE_MASK;
    3.46 +    BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    3.47 +}
    3.48 +
    3.49 +#endif /* CONFIG_SMP */
    3.50 +
    3.51  void xen_pgd_pin(unsigned long ptr)
    3.52  {
    3.53      struct mmuext_op op;
    3.54 @@ -169,7 +206,7 @@ unsigned long allocate_empty_lowmem_regi
    3.55          phys_to_machine_mapping[__pa(vstart)>>PAGE_SHIFT] = INVALID_P2M_ENTRY;
    3.56      }
    3.57  
    3.58 -    xen_tlb_flush();
    3.59 +    flush_tlb_all();
    3.60  
    3.61      balloon_put_pages(pfn_array, 1 << order);
    3.62  
     4.1 --- a/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c	Thu Mar 31 15:44:24 2005 +0000
     4.2 +++ b/linux-2.6.11-xen-sparse/drivers/xen/balloon/balloon.c	Thu Mar 31 17:02:55 2005 +0000
     4.3 @@ -260,8 +260,7 @@ static void balloon_process(void *unused
     4.4  
     4.5          /* Ensure that ballooned highmem pages don't have cached mappings. */
     4.6          kmap_flush_unused();
     4.7 -
     4.8 -        xen_tlb_flush();
     4.9 +        flush_tlb_all();
    4.10  
    4.11          /* No more mappings: invalidate pages in P2M and add to balloon. */
    4.12          for ( i = 0; i < debt; i++ )
     5.1 --- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h	Thu Mar 31 15:44:24 2005 +0000
     5.2 +++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/pgtable.h	Thu Mar 31 17:02:55 2005 +0000
     5.3 @@ -407,7 +407,8 @@ extern void noexec_setup(const char *str
     5.4  	do {								  \
     5.5  		if (__dirty) {						  \
     5.6  		        if ( likely((__vma)->vm_mm == current->mm) ) {    \
     5.7 -			    HYPERVISOR_update_va_mapping((__address), (__entry), UVMF_INVLPG_LOCAL); \
     5.8 +			    HYPERVISOR_update_va_mapping((__address), (__entry), 0); \
     5.9 +			    flush_tlb_page((__vma), (__address));         \
    5.10  			} else {                                          \
    5.11                              xen_l1_entry_update((__ptep), (__entry).pte_low); \
    5.12  			    flush_tlb_page((__vma), (__address));         \
     6.1 --- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h	Thu Mar 31 15:44:24 2005 +0000
     6.2 +++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h	Thu Mar 31 17:02:55 2005 +0000
     6.3 @@ -6,23 +6,11 @@
     6.4  #include <asm/processor.h>
     6.5  
     6.6  #define __flush_tlb() xen_tlb_flush()
     6.7 -
     6.8 -/*
     6.9 - * Global pages have to be flushed a bit differently. Not a real
    6.10 - * performance problem because this does not happen often.
    6.11 - */
    6.12  #define __flush_tlb_global() xen_tlb_flush()
    6.13 +#define __flush_tlb_all() xen_tlb_flush()
    6.14  
    6.15  extern unsigned long pgkern_mask;
    6.16  
    6.17 -# define __flush_tlb_all()						\
    6.18 -	do {								\
    6.19 -		if (cpu_has_pge)					\
    6.20 -			__flush_tlb_global();				\
    6.21 -		else							\
    6.22 -			__flush_tlb();					\
    6.23 -	} while (0)
    6.24 -
    6.25  #define cpu_has_invlpg	(boot_cpu_data.x86 > 3)
    6.26  
    6.27  #define __flush_tlb_single(addr) xen_invlpg(addr)
     7.1 --- a/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h	Thu Mar 31 15:44:24 2005 +0000
     7.2 +++ b/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h	Thu Mar 31 17:02:55 2005 +0000
     7.3 @@ -83,6 +83,14 @@ void xen_pte_unpin(unsigned long ptr);
     7.4  void xen_set_ldt(unsigned long ptr, unsigned long bytes);
     7.5  void xen_machphys_update(unsigned long mfn, unsigned long pfn);
     7.6  
     7.7 +#ifdef CONFIG_SMP
     7.8 +#include <linux/cpumask.h>
     7.9 +void xen_tlb_flush_all(void);
    7.10 +void xen_invlpg_all(unsigned long ptr);
    7.11 +void xen_tlb_flush_mask(cpumask_t mask);
    7.12 +void xen_invlpg_mask(cpumask_t mask, unsigned long ptr);
    7.13 +#endif
    7.14 +
    7.15  #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)
    7.16  /* 
    7.17  ** XXX SMH: 2.4 doesn't have percpu.h (or support SMP guests) so just