direct-io.hg

changeset 4657:9d8f47890534

bitkeeper revision 1.1383 (426d2078vvMiOXQcJ98uhha5uIt0Sw)

Enormously better process destruction performance. By not running on
pagetables when we are destroying them we massively reduce TLB flushes
and increase writable p.t. batches. This patch would probably improve
even native Linux performance, because of the rabid ZAP_BLOCK_SIZE
changes to unmap_vmas() that were introduced to 2.6.11. :-)
Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Mon Apr 25 16:53:12 2005 +0000 (2005-04-25)
parents 7b4421af1180
children d528142a1daf
files linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/hypervisor.c linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/mmu_context.h linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/tlbflush.h linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h
line diff
     1.1 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c	Mon Apr 25 15:10:07 2005 +0000
     1.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smp.c	Mon Apr 25 16:53:12 2005 +0000
     1.3 @@ -448,11 +448,11 @@ irqreturn_t smp_invalidate_interrupt(int
     1.4  				     struct pt_regs *regs)
     1.5  { return 0; }
     1.6  void flush_tlb_current_task(void)
     1.7 -{ xen_tlb_flush_mask(current->mm->cpu_vm_mask); }
     1.8 +{ xen_tlb_flush_mask(&current->mm->cpu_vm_mask); }
     1.9  void flush_tlb_mm(struct mm_struct * mm)
    1.10 -{ xen_tlb_flush_mask(mm->cpu_vm_mask); }
    1.11 +{ xen_tlb_flush_mask(&mm->cpu_vm_mask); }
    1.12  void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
    1.13 -{ xen_invlpg_mask(vma->vm_mm->cpu_vm_mask, va); }
    1.14 +{ xen_invlpg_mask(&vma->vm_mm->cpu_vm_mask, va); }
    1.15  void flush_tlb_all(void)
    1.16  { xen_tlb_flush_all(); }
    1.17  
     2.1 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c	Mon Apr 25 15:10:07 2005 +0000
     2.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/mm/hypervisor.c	Mon Apr 25 16:53:12 2005 +0000
     2.3 @@ -106,11 +106,13 @@ void xen_tlb_flush_all(void)
     2.4      BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
     2.5  }
     2.6  
     2.7 -void xen_tlb_flush_mask(cpumask_t mask)
     2.8 +void xen_tlb_flush_mask(cpumask_t *mask)
     2.9  {
    2.10      struct mmuext_op op;
    2.11 +    if ( cpus_empty(*mask) )
    2.12 +        return;
    2.13      op.cmd = MMUEXT_TLB_FLUSH_MULTI;
    2.14 -    op.cpuset = mask.bits;
    2.15 +    op.cpuset = mask->bits;
    2.16      BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    2.17  }
    2.18  
    2.19 @@ -122,11 +124,13 @@ void xen_invlpg_all(unsigned long ptr)
    2.20      BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    2.21  }
    2.22  
    2.23 -void xen_invlpg_mask(cpumask_t mask, unsigned long ptr)
    2.24 +void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr)
    2.25  {
    2.26      struct mmuext_op op;
    2.27 +    if ( cpus_empty(*mask) )
    2.28 +        return;
    2.29      op.cmd = MMUEXT_INVLPG_MULTI;
    2.30 -    op.cpuset = mask.bits;
    2.31 +    op.cpuset = mask->bits;
    2.32      op.linear_addr = ptr & PAGE_MASK;
    2.33      BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    2.34  }
     3.1 --- a/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/hypervisor.c	Mon Apr 25 15:10:07 2005 +0000
     3.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/x86_64/mm/hypervisor.c	Mon Apr 25 16:53:12 2005 +0000
     3.3 @@ -116,11 +116,13 @@ void xen_tlb_flush_all(void)
     3.4      BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
     3.5  }
     3.6  
     3.7 -void xen_tlb_flush_mask(cpumask_t mask)
     3.8 +void xen_tlb_flush_mask(cpumask_t *mask)
     3.9  {
    3.10      struct mmuext_op op;
    3.11 +    if ( cpus_empty(*mask) )
    3.12 +        return;
    3.13      op.cmd = MMUEXT_TLB_FLUSH_MULTI;
    3.14 -    op.cpuset = mask.bits[0];
    3.15 +    op.cpuset = mask->bits;
    3.16      BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    3.17  }
    3.18  
    3.19 @@ -132,11 +134,13 @@ void xen_invlpg_all(unsigned long ptr)
    3.20      BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    3.21  }
    3.22  
    3.23 -void xen_invlpg_mask(cpumask_t mask, unsigned long ptr)
    3.24 +void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr)
    3.25  {
    3.26      struct mmuext_op op;
    3.27 +    if ( cpus_empty(*mask) )
    3.28 +        return;
    3.29      op.cmd = MMUEXT_INVLPG_MULTI;
    3.30 -    op.cpuset = mask.bits[0];
    3.31 +    op.cpuset = mask->bits;
    3.32      op.linear_addr = ptr & PAGE_MASK;
    3.33      BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
    3.34  }
     4.1 --- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/mmu_context.h	Mon Apr 25 15:10:07 2005 +0000
     4.2 +++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/mmu_context.h	Mon Apr 25 16:53:12 2005 +0000
     4.3 @@ -16,7 +16,7 @@ void destroy_context(struct mm_struct *m
     4.4  
     4.5  static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
     4.6  {
     4.7 -#if 0 /* XEN */
     4.8 +#if 0 /* XEN: no lazy tlb */
     4.9  	unsigned cpu = smp_processor_id();
    4.10  	if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
    4.11  		per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_LAZY;
    4.12 @@ -51,7 +51,7 @@ static inline void switch_mm(struct mm_s
    4.13  	if (likely(prev != next)) {
    4.14  		/* stop flush ipis for the previous mm */
    4.15  		cpu_clear(cpu, prev->cpu_vm_mask);
    4.16 -#if 0 /* XEN */
    4.17 +#if 0 /* XEN: no lazy tlb */
    4.18  		per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_OK;
    4.19  		per_cpu(cpu_tlbstate, cpu).active_mm = next;
    4.20  #endif
    4.21 @@ -76,7 +76,7 @@ static inline void switch_mm(struct mm_s
    4.22  
    4.23  		BUG_ON(HYPERVISOR_mmuext_op(_op, op-_op, NULL, DOMID_SELF));
    4.24  	}
    4.25 -#if 0 /* XEN */
    4.26 +#if 0 /* XEN: no lazy tlb */
    4.27  	else {
    4.28  		per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_OK;
    4.29  		BUG_ON(per_cpu(cpu_tlbstate, cpu).active_mm != next);
    4.30 @@ -92,8 +92,17 @@ static inline void switch_mm(struct mm_s
    4.31  #endif
    4.32  }
    4.33  
    4.34 -#define deactivate_mm(tsk, mm) \
    4.35 -	asm("movl %0,%%fs ; movl %0,%%gs": :"r" (0))
    4.36 +/*
    4.37 + * XEN: We aggressively remove defunct pgd from cr3. We execute unmap_vmas()
    4.38 + * *much* faster this way, as no tlb flushes means much bigger wrpt batches.
    4.39 + */
    4.40 +#define deactivate_mm(tsk, mm) do {					\
    4.41 +	asm("movl %0,%%fs ; movl %0,%%gs": :"r" (0));			\
    4.42 +	if ((mm) && cpu_isset(smp_processor_id(), (mm)->cpu_vm_mask)) {	\
    4.43 +		cpu_clear(smp_processor_id(), (mm)->cpu_vm_mask);	\
    4.44 +		load_cr3(swapper_pg_dir);				\
    4.45 +	}								\
    4.46 +} while (0)
    4.47  
    4.48  #define activate_mm(prev, next) do {		\
    4.49  	switch_mm((prev),(next),NULL);		\
     5.1 --- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h	Mon Apr 25 15:10:07 2005 +0000
     5.2 +++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/tlbflush.h	Mon Apr 25 16:53:12 2005 +0000
     5.3 @@ -40,21 +40,24 @@ extern unsigned long pgkern_mask;
     5.4  
     5.5  static inline void flush_tlb_mm(struct mm_struct *mm)
     5.6  {
     5.7 -	if (mm == current->active_mm)
     5.8 +	/* XEN: cpu_vm_mask is more accurate than active_mm. */
     5.9 +	if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask))
    5.10  		__flush_tlb();
    5.11  }
    5.12  
    5.13  static inline void flush_tlb_page(struct vm_area_struct *vma,
    5.14  	unsigned long addr)
    5.15  {
    5.16 -	if (vma->vm_mm == current->active_mm)
    5.17 +	/* XEN: cpu_vm_mask is more accurate than active_mm. */
    5.18 +	if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask))
    5.19  		__flush_tlb_one(addr);
    5.20  }
    5.21  
    5.22  static inline void flush_tlb_range(struct vm_area_struct *vma,
    5.23  	unsigned long start, unsigned long end)
    5.24  {
    5.25 -	if (vma->vm_mm == current->active_mm)
    5.26 +	/* XEN: cpu_vm_mask is more accurate than active_mm. */
    5.27 +	if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask))
    5.28  		__flush_tlb();
    5.29  }
    5.30  
     6.1 --- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/tlbflush.h	Mon Apr 25 15:10:07 2005 +0000
     6.2 +++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-x86_64/tlbflush.h	Mon Apr 25 16:53:12 2005 +0000
     6.3 @@ -44,21 +44,24 @@ extern unsigned long pgkern_mask;
     6.4  
     6.5  static inline void flush_tlb_mm(struct mm_struct *mm)
     6.6  {
     6.7 -	if (mm == current->active_mm)
     6.8 +	/* XEN: cpu_vm_mask is more accurate than active_mm. */
     6.9 +	if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask))
    6.10  		__flush_tlb();
    6.11  }
    6.12  
    6.13  static inline void flush_tlb_page(struct vm_area_struct *vma,
    6.14  	unsigned long addr)
    6.15  {
    6.16 -	if (vma->vm_mm == current->active_mm)
    6.17 +	/* XEN: cpu_vm_mask is more accurate than active_mm. */
    6.18 +	if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask))
    6.19  		__flush_tlb_one(addr);
    6.20  }
    6.21  
    6.22  static inline void flush_tlb_range(struct vm_area_struct *vma,
    6.23  	unsigned long start, unsigned long end)
    6.24  {
    6.25 -	if (vma->vm_mm == current->active_mm)
    6.26 +	/* XEN: cpu_vm_mask is more accurate than active_mm. */
    6.27 +	if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask))
    6.28  		__flush_tlb();
    6.29  }
    6.30  
     7.1 --- a/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h	Mon Apr 25 15:10:07 2005 +0000
     7.2 +++ b/linux-2.6.11-xen-sparse/include/asm-xen/hypervisor.h	Mon Apr 25 16:53:12 2005 +0000
     7.3 @@ -110,8 +110,8 @@ void xen_machphys_update(unsigned long m
     7.4  #include <linux/cpumask.h>
     7.5  void xen_tlb_flush_all(void);
     7.6  void xen_invlpg_all(unsigned long ptr);
     7.7 -void xen_tlb_flush_mask(cpumask_t mask);
     7.8 -void xen_invlpg_mask(cpumask_t mask, unsigned long ptr);
     7.9 +void xen_tlb_flush_mask(cpumask_t *mask);
    7.10 +void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr);
    7.11  #endif
    7.12  
    7.13  #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,0)