direct-io.hg

changeset 4417:5fbc250ab25b

bitkeeper revision 1.1236.1.181 (424d132aYORx7PllUl-nlTBMndGn4g)

Improved lazy state switching -- flush the state switch through on the
first flush IPI received. This avoids needing to receive any further
ones at little cost (needed to flush TLB anyway). The main extra cost
will be that, when switching back to a guest, we won't save the
ctxt switch. But we may have saved an unbounded number of IPI flushes.
Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Fri Apr 01 09:23:54 2005 +0000 (2005-04-01)
parents 70ce80d7e30c
children 941598a91ed0
files xen/arch/ia64/xenmisc.c xen/arch/x86/domain.c xen/arch/x86/mm.c xen/arch/x86/smp.c xen/include/xen/sched.h xen/include/xen/smp.h
line diff
     1.1 --- a/xen/arch/ia64/xenmisc.c	Fri Apr 01 08:56:32 2005 +0000
     1.2 +++ b/xen/arch/ia64/xenmisc.c	Fri Apr 01 09:23:54 2005 +0000
     1.3 @@ -53,7 +53,6 @@ platform_is_hp_ski(void)
     1.4  }
     1.5  
     1.6  /* calls in xen/common code that are unused on ia64 */
     1.7 -void synchronise_execution_state(unsigned long cpu_mask) { }
     1.8  
     1.9  int grant_table_create(struct domain *d) { return 0; }
    1.10  void grant_table_destroy(struct domain *d)
     2.1 --- a/xen/arch/x86/domain.c	Fri Apr 01 08:56:32 2005 +0000
     2.2 +++ b/xen/arch/x86/domain.c	Fri Apr 01 09:23:54 2005 +0000
     2.3 @@ -744,12 +744,21 @@ long do_switch_to_user(void)
     2.4      return stu.rax; 
     2.5  }
     2.6  
     2.7 +#define switch_kernel_stack(_n,_c) ((void)0)
     2.8 +
     2.9  #elif defined(__i386__)
    2.10  
    2.11  #define load_segments(_p, _n) ((void)0)
    2.12  #define save_segments(_p)     ((void)0)
    2.13  #define clear_segments()      ((void)0)
    2.14  
    2.15 +static inline void switch_kernel_stack(struct exec_domain *n, unsigned int cpu)
    2.16 +{
    2.17 +    struct tss_struct *tss = &init_tss[cpu];
    2.18 +    tss->esp1 = n->arch.kernel_sp;
    2.19 +    tss->ss1  = n->arch.kernel_ss;
    2.20 +}
    2.21 +
    2.22  #endif
    2.23  
    2.24  #define loaddebug(_ed,_reg) \
    2.25 @@ -772,42 +781,36 @@ static void __context_switch(void)
    2.26          save_segments(p);
    2.27      }
    2.28  
    2.29 -    memcpy(stack_ec,
    2.30 -           &n->arch.user_ctxt,
    2.31 -           sizeof(*stack_ec));
    2.32 -
    2.33 -    /* Maybe switch the debug registers. */
    2.34 -    if ( unlikely(n->arch.debugreg[7]) )
    2.35 +    if ( !is_idle_task(n->domain) )
    2.36      {
    2.37 -        loaddebug(&n->arch, 0);
    2.38 -        loaddebug(&n->arch, 1);
    2.39 -        loaddebug(&n->arch, 2);
    2.40 -        loaddebug(&n->arch, 3);
    2.41 -        /* no 4 and 5 */
    2.42 -        loaddebug(&n->arch, 6);
    2.43 -        loaddebug(&n->arch, 7);
    2.44 -    }
    2.45 +        memcpy(stack_ec,
    2.46 +               &n->arch.user_ctxt,
    2.47 +               sizeof(*stack_ec));
    2.48  
    2.49 -    if ( !VMX_DOMAIN(n) )
    2.50 -    {
    2.51 -        SET_FAST_TRAP(&n->arch);
    2.52 -
    2.53 -#ifdef __i386__
    2.54 +        /* Maybe switch the debug registers. */
    2.55 +        if ( unlikely(n->arch.debugreg[7]) )
    2.56          {
    2.57 -            /* Switch the kernel ring-1 stack. */
    2.58 -            struct tss_struct *tss = &init_tss[cpu];
    2.59 -            tss->esp1 = n->arch.kernel_sp;
    2.60 -            tss->ss1  = n->arch.kernel_ss;
    2.61 +            loaddebug(&n->arch, 0);
    2.62 +            loaddebug(&n->arch, 1);
    2.63 +            loaddebug(&n->arch, 2);
    2.64 +            loaddebug(&n->arch, 3);
    2.65 +            /* no 4 and 5 */
    2.66 +            loaddebug(&n->arch, 6);
    2.67 +            loaddebug(&n->arch, 7);
    2.68          }
    2.69 -#endif
    2.70 +
    2.71 +        if ( !VMX_DOMAIN(n) )
    2.72 +        {
    2.73 +            SET_FAST_TRAP(&n->arch);
    2.74 +            switch_kernel_stack(n, cpu);
    2.75 +        }
    2.76      }
    2.77  
    2.78      set_bit(cpu, &n->domain->cpuset);
    2.79      write_ptbase(n);
    2.80 +    __asm__ __volatile__ ( "lgdt %0" : "=m" (*n->arch.gdt) );
    2.81      clear_bit(cpu, &p->domain->cpuset);
    2.82  
    2.83 -    __asm__ __volatile__ ( "lgdt %0" : "=m" (*n->arch.gdt) );
    2.84 -
    2.85      percpu_ctxt[cpu].curr_ed = n;
    2.86  }
    2.87  
    2.88 @@ -820,7 +823,7 @@ void context_switch(struct exec_domain *
    2.89  
    2.90      set_current(next);
    2.91  
    2.92 -    if ( ((realprev = percpu_ctxt[smp_processor_id()]. curr_ed) == next) || 
    2.93 +    if ( ((realprev = percpu_ctxt[smp_processor_id()].curr_ed) == next) || 
    2.94           is_idle_task(next->domain) )
    2.95      {
    2.96          local_irq_enable();
    2.97 @@ -851,18 +854,14 @@ void context_switch(struct exec_domain *
    2.98      BUG();
    2.99  }
   2.100  
   2.101 -static void __synchronise_lazy_execstate(void *unused)
   2.102 +int __sync_lazy_execstate(void)
   2.103  {
   2.104 -    if ( percpu_ctxt[smp_processor_id()].curr_ed != current )
   2.105 -    {
   2.106 -        __context_switch();
   2.107 -        load_LDT(current);
   2.108 -        clear_segments();
   2.109 -    }
   2.110 -}
   2.111 -void synchronise_lazy_execstate(unsigned long cpuset)
   2.112 -{
   2.113 -    smp_subset_call_function(__synchronise_lazy_execstate, NULL, 1, cpuset);
   2.114 +    if ( percpu_ctxt[smp_processor_id()].curr_ed == current )
   2.115 +        return 0;
   2.116 +    __context_switch();
   2.117 +    load_LDT(current);
   2.118 +    clear_segments();
   2.119 +    return 1;
   2.120  }
   2.121  
   2.122  unsigned long __hypercall_create_continuation(
     3.1 --- a/xen/arch/x86/mm.c	Fri Apr 01 08:56:32 2005 +0000
     3.2 +++ b/xen/arch/x86/mm.c	Fri Apr 01 09:23:54 2005 +0000
     3.3 @@ -2827,7 +2827,8 @@ void audit_domain(struct domain *d)
     3.4  
     3.5      if ( d != current->domain )
     3.6          domain_pause(d);
     3.7 -    synchronise_lazy_execstate(~0UL);
     3.8 +
     3.9 +    sync_lazy_execstate_all();
    3.10  
    3.11      printk("pt base=%lx sh_info=%x\n",
    3.12             pagetable_val(d->exec_domain[0]->arch.guest_table)>>PAGE_SHIFT,
     4.1 --- a/xen/arch/x86/smp.c	Fri Apr 01 08:56:32 2005 +0000
     4.2 +++ b/xen/arch/x86/smp.c	Fri Apr 01 09:23:54 2005 +0000
     4.3 @@ -154,10 +154,13 @@ asmlinkage void smp_invalidate_interrupt
     4.4  {
     4.5      ack_APIC_irq();
     4.6      perfc_incrc(ipis);
     4.7 -    if ( flush_va == FLUSHVA_ALL )
     4.8 -        local_flush_tlb();
     4.9 -    else
    4.10 -        local_flush_tlb_one(flush_va);
    4.11 +    if ( !__sync_lazy_execstate() )
    4.12 +    {
    4.13 +        if ( flush_va == FLUSHVA_ALL )
    4.14 +            local_flush_tlb();
    4.15 +        else
    4.16 +            local_flush_tlb_one(flush_va);
    4.17 +    }
    4.18      clear_bit(smp_processor_id(), &flush_cpumask);
    4.19  }
    4.20  
    4.21 @@ -278,45 +281,6 @@ int smp_call_function(
    4.22      return 0;
    4.23  }
    4.24  
    4.25 -/* Run a function on a subset of CPUs (may include local CPU). */
    4.26 -int smp_subset_call_function(
    4.27 -    void (*func) (void *info), void *info, int wait, unsigned long cpuset)
    4.28 -{
    4.29 -    struct call_data_struct data;
    4.30 -
    4.31 -    ASSERT(local_irq_is_enabled());
    4.32 -
    4.33 -    if ( cpuset & (1UL << smp_processor_id()) )
    4.34 -    {
    4.35 -        local_irq_disable();
    4.36 -        (*func)(info);
    4.37 -        local_irq_enable();
    4.38 -    }
    4.39 -
    4.40 -    cpuset &= ((1UL << smp_num_cpus) - 1) & ~(1UL << smp_processor_id());
    4.41 -    if ( cpuset == 0 )
    4.42 -        return 0;
    4.43 -
    4.44 -    data.func = func;
    4.45 -    data.info = info;
    4.46 -    data.started = data.finished = 0;
    4.47 -    data.wait = wait;
    4.48 -
    4.49 -    spin_lock(&call_lock);
    4.50 -
    4.51 -    call_data = &data;
    4.52 -    wmb();
    4.53 -
    4.54 -    send_IPI_mask(cpuset, CALL_FUNCTION_VECTOR);
    4.55 -
    4.56 -    while ( (wait ? data.finished : data.started) != cpuset )
    4.57 -        cpu_relax();
    4.58 -
    4.59 -    spin_unlock(&call_lock);
    4.60 -
    4.61 -    return 0;
    4.62 -}
    4.63 -
    4.64  static void stop_this_cpu (void *dummy)
    4.65  {
    4.66      clear_bit(smp_processor_id(), &cpu_online_map);
     5.1 --- a/xen/include/xen/sched.h	Fri Apr 01 08:56:32 2005 +0000
     5.2 +++ b/xen/include/xen/sched.h	Fri Apr 01 09:23:54 2005 +0000
     5.3 @@ -255,7 +255,9 @@ void domain_sleep(struct exec_domain *d)
     5.4   * Force loading of currently-executing domain state on the specified set
     5.5   * of CPUs. This is used to counteract lazy state switching where required.
     5.6   */
     5.7 -void synchronise_lazy_execstate(unsigned long cpuset);
     5.8 +#define sync_lazy_execstate_cpuset(_cpuset) flush_tlb_mask(_cpuset)
     5.9 +#define sync_lazy_execstate_all()           flush_tlb_all()
    5.10 +extern int __sync_lazy_execstate(void);
    5.11  
    5.12  extern void context_switch(
    5.13      struct exec_domain *prev, 
    5.14 @@ -337,7 +339,7 @@ static inline void exec_domain_pause(str
    5.15      ASSERT(ed != current);
    5.16      atomic_inc(&ed->pausecnt);
    5.17      domain_sleep(ed);
    5.18 -    synchronise_lazy_execstate(ed->domain->cpuset & (1UL << ed->processor));
    5.19 +    sync_lazy_execstate_cpuset(ed->domain->cpuset & (1UL << ed->processor));
    5.20  }
    5.21  
    5.22  static inline void domain_pause(struct domain *d)
    5.23 @@ -351,7 +353,7 @@ static inline void domain_pause(struct d
    5.24          domain_sleep(ed);
    5.25      }
    5.26  
    5.27 -    synchronise_lazy_execstate(d->cpuset);
    5.28 +    sync_lazy_execstate_cpuset(d->cpuset);
    5.29  }
    5.30  
    5.31  static inline void exec_domain_unpause(struct exec_domain *ed)
    5.32 @@ -386,7 +388,7 @@ static inline void domain_pause_by_syste
    5.33              domain_sleep(ed);
    5.34      }
    5.35  
    5.36 -    synchronise_lazy_execstate(d->cpuset);
    5.37 +    sync_lazy_execstate_cpuset(d->cpuset);
    5.38  }
    5.39  
    5.40  static inline void domain_unpause_by_systemcontroller(struct domain *d)
     6.1 --- a/xen/include/xen/smp.h	Fri Apr 01 08:56:32 2005 +0000
     6.2 +++ b/xen/include/xen/smp.h	Fri Apr 01 09:23:54 2005 +0000
     6.3 @@ -45,8 +45,6 @@ extern void smp_commence(void);
     6.4   */
     6.5  extern int smp_call_function(
     6.6      void (*func) (void *info), void *info, int retry, int wait);
     6.7 -extern int smp_subset_call_function(
     6.8 -    void (*func) (void *info), void *info, int wait, unsigned long cpuset);
     6.9  
    6.10  /*
    6.11   * True once the per process idle is forked
    6.12 @@ -87,7 +85,6 @@ extern volatile int smp_msg_id;
    6.13  #define cpu_logical_map(cpu)			0
    6.14  #define cpu_number_map(cpu)			0
    6.15  #define smp_call_function(func,info,retry,wait)	0
    6.16 -#define smp_subset_call_function(f,i,w,c)	({ if ( (c&1) ) (*f)(i); 0; })
    6.17  #define cpu_online_map				1
    6.18  
    6.19  #endif