ia64/xen-unstable

changeset 19260:58edfadb2d87

x86: Minimize the TSC drift between pCPUs
Signed-off-by: Xiaowei Yang <xiaowei.yang@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Mon Mar 02 13:56:47 2009 +0000 (2009-03-02)
parents 507b264f0a21
children c62b453f27d5
files xen/arch/x86/time.c
line diff
     1.1 --- a/xen/arch/x86/time.c	Mon Mar 02 11:23:23 2009 +0000
     1.2 +++ b/xen/arch/x86/time.c	Mon Mar 02 13:56:47 2009 +0000
     1.3 @@ -1087,38 +1087,69 @@ static void local_time_calibration(void)
     1.4   */
     1.5  struct calibration_rendezvous {
     1.6      cpumask_t cpu_calibration_map;
     1.7 -    atomic_t nr_cpus;
     1.8 +    atomic_t count_start;
     1.9 +    atomic_t count_end;
    1.10      s_time_t master_stime;
    1.11      u64 master_tsc_stamp;
    1.12  };
    1.13  
    1.14 +#define NR_LOOPS 5
    1.15 +
    1.16  static void time_calibration_rendezvous(void *_r)
    1.17  {
    1.18 +    int i;
    1.19      struct cpu_calibration *c = &this_cpu(cpu_calibration);
    1.20      struct calibration_rendezvous *r = _r;
    1.21      unsigned int total_cpus = cpus_weight(r->cpu_calibration_map);
    1.22  
    1.23 -    if ( smp_processor_id() == 0 )
    1.24 +    /* 
    1.25 +     * Loop is used here to get rid of the cache's side effect to enlarge
    1.26 +     * the TSC difference among CPUs.
    1.27 +     */
    1.28 +    for ( i = 0; i < NR_LOOPS; i++ )
    1.29      {
    1.30 -        while ( atomic_read(&r->nr_cpus) != (total_cpus - 1) )
    1.31 -            cpu_relax();
    1.32 -        r->master_stime = read_platform_stime();
    1.33 -        rdtscll(r->master_tsc_stamp);
    1.34 -        mb(); /* write r->master_* /then/ signal */
    1.35 -        atomic_inc(&r->nr_cpus);
    1.36 -        c->local_tsc_stamp = r->master_tsc_stamp;
    1.37 -    }
    1.38 -    else
    1.39 -    {
    1.40 -        atomic_inc(&r->nr_cpus);
    1.41 -        while ( atomic_read(&r->nr_cpus) != total_cpus )
    1.42 -            cpu_relax();
    1.43 -        mb(); /* receive signal /then/ read r->master_* */
    1.44 -        if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) )
    1.45 -            wrmsrl(MSR_IA32_TSC, r->master_tsc_stamp);
    1.46 -        rdtscll(c->local_tsc_stamp);
    1.47 +        if ( smp_processor_id() == 0 )
    1.48 +        {
    1.49 +            while ( atomic_read(&r->count_start) != (total_cpus - 1) )
    1.50 +                mb();
    1.51 +   
    1.52 +            if ( r->master_stime == 0 )
    1.53 +            {
    1.54 +                r->master_stime = read_platform_stime();
    1.55 +                if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) )
    1.56 +                    rdtscll(r->master_tsc_stamp);
    1.57 +            }
    1.58 +            atomic_set(&r->count_end, 0);
    1.59 +            wmb();
    1.60 +            atomic_inc(&r->count_start);
    1.61 +    
    1.62 +            if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && 
    1.63 +                 i == NR_LOOPS - 1 )
    1.64 +                write_tsc((u32)r->master_tsc_stamp, (u32)(r->master_tsc_stamp >> 32));
    1.65 +    
    1.66 +            while (atomic_read(&r->count_end) != total_cpus - 1)
    1.67 +                mb();
    1.68 +            atomic_set(&r->count_start, 0);
    1.69 +            wmb();
    1.70 +            atomic_inc(&r->count_end);
    1.71 +        }
    1.72 +        else
    1.73 +        {
    1.74 +            atomic_inc(&r->count_start);
    1.75 +            while ( atomic_read(&r->count_start) != total_cpus )
    1.76 +                mb();
    1.77 +    
    1.78 +            if ( boot_cpu_has(X86_FEATURE_CONSTANT_TSC) && 
    1.79 +                 i == NR_LOOPS - 1 )
    1.80 +                write_tsc((u32)r->master_tsc_stamp, (u32)(r->master_tsc_stamp >> 32));
    1.81 +    
    1.82 +            atomic_inc(&r->count_end);
    1.83 +            while (atomic_read(&r->count_end) != total_cpus)
    1.84 +                mb();
    1.85 +        }
    1.86      }
    1.87  
    1.88 +    rdtscll(c->local_tsc_stamp);
    1.89      c->stime_local_stamp = get_s_time();
    1.90      c->stime_master_stamp = r->master_stime;
    1.91  
    1.92 @@ -1129,7 +1160,9 @@ static void time_calibration(void *unuse
    1.93  {
    1.94      struct calibration_rendezvous r = {
    1.95          .cpu_calibration_map = cpu_online_map,
    1.96 -        .nr_cpus = ATOMIC_INIT(0)
    1.97 +        .count_start = ATOMIC_INIT(0),
    1.98 +        .count_end = ATOMIC_INIT(0),
    1.99 +        .master_stime = 0
   1.100      };
   1.101  
   1.102      /* @wait=1 because we must wait for all cpus before freeing @r. */