1.1 --- a/xen/arch/x86/time.c	Mon Jun 11 14:56:50 2007 +0100
     1.2 +++ b/xen/arch/x86/time.c	Mon Jun 11 15:10:23 2007 +0100
     1.3 @@ -57,25 +57,29 @@ struct cpu_time {
     1.4      struct timer calibration_timer;
     1.5  };
     1.6  
     1.7 +struct platform_timesource {
     1.8 +    char *name;
     1.9 +    u64 frequency;
    1.10 +    u32 (*read_counter)(void);
    1.11 +    int counter_bits;
    1.12 +};
    1.13 +
    1.14  static DEFINE_PER_CPU(struct cpu_time, cpu_time);
    1.15  
    1.16  /*
    1.17   * Protected by platform_timer_lock, which must be acquired with interrupts
    1.18 - * disabled because pit_overflow() is called from PIT ch0 interrupt context.
    1.19 + * disabled because plt_overflow() is called from PIT ch0 interrupt context.
    1.20   */
    1.21  static s_time_t stime_platform_stamp;
    1.22  static u64 platform_timer_stamp;
    1.23 -static struct time_scale platform_timer_scale;
    1.24  static DEFINE_SPINLOCK(platform_timer_lock);
    1.25 -static u64 (*read_platform_count)(void);
    1.26  
    1.27  /*
    1.28 - * Folding 16-bit PIT into 64-bit software counter is a really critical
    1.29 - * operation! We therefore do it directly in PIT ch0 interrupt handler,
    1.30 - * based on this flag.
    1.31 + * Folding platform timer into 64-bit software counter is a really critical
    1.32 + * operation! We therefore do it directly in PIT ch0 interrupt handler.
    1.33   */
    1.34 -static int using_pit;
    1.35 -static void pit_overflow(void);
    1.36 +static u32 plt_overflow_jiffies;
    1.37 +static void plt_overflow(void);
    1.38  
    1.39  /*
    1.40   * 32-bit division of integer dividend and integer divisor yielding
    1.41 @@ -153,8 +157,8 @@ void timer_interrupt(int irq, void *dev_
    1.42      if ( !cpu_has_apic )
    1.43          raise_softirq(TIMER_SOFTIRQ);
    1.44  
    1.45 -    if ( using_pit )
    1.46 -        pit_overflow();
    1.47 +    if ( --plt_overflow_jiffies == 0 )
    1.48 +        plt_overflow();
    1.49  }
    1.50  
    1.51  static struct irqaction irq0 = { timer_interrupt, "timer", NULL};
    1.52 @@ -281,76 +285,34 @@ static char *freq_string(u64 freq)
    1.53   * PLATFORM TIMER 1: PROGRAMMABLE INTERVAL TIMER (LEGACY PIT)
    1.54   */
    1.55  
    1.56 -/* Protected by platform_timer_lock. */
    1.57 -static u64 pit_counter64;
    1.58 -static u16 pit_stamp;
    1.59 -
    1.60 -static u16 pit_read_counter(void)
    1.61 +static u32 read_pit_count(void)
    1.62  {
    1.63      u16 count;
    1.64      ASSERT(spin_is_locked(&platform_timer_lock));
    1.65      outb(0x80, PIT_MODE);
    1.66      count  = inb(PIT_CH2);
    1.67      count |= inb(PIT_CH2) << 8;
    1.68 -    return count;
    1.69 -}
    1.70 -
    1.71 -static void pit_overflow(void)
    1.72 -{
    1.73 -    u16 counter;
    1.74 -
    1.75 -    spin_lock_irq(&platform_timer_lock);
    1.76 -    counter = pit_read_counter();
    1.77 -    pit_counter64 += (u16)(pit_stamp - counter);
    1.78 -    pit_stamp = counter;
    1.79 -    spin_unlock_irq(&platform_timer_lock);
    1.80 +    return ~count;
    1.81  }
    1.82  
    1.83 -static u64 read_pit_count(void)
    1.84 -{
    1.85 -    return pit_counter64 + (u16)(pit_stamp - pit_read_counter());
    1.86 -}
    1.87 -
    1.88 -static void init_pit(void)
    1.89 +static void init_pit(struct platform_timesource *pts)
    1.90  {
    1.91 -    read_platform_count = read_pit_count;
    1.92 -
    1.93 -    pit_overflow();
    1.94 -    platform_timer_stamp = pit_counter64;
    1.95 -    set_time_scale(&platform_timer_scale, CLOCK_TICK_RATE);
    1.96 -
    1.97 -    printk("Platform timer is %s PIT\n", freq_string(CLOCK_TICK_RATE));
    1.98 -    using_pit = 1;
    1.99 +    pts->name = "PIT";
   1.100 +    pts->frequency = CLOCK_TICK_RATE;
   1.101 +    pts->read_counter = read_pit_count;
   1.102 +    pts->counter_bits = 16;
   1.103  }
   1.104  
   1.105  /************************************************************
   1.106   * PLATFORM TIMER 2: HIGH PRECISION EVENT TIMER (HPET)
   1.107   */
   1.108  
   1.109 -/* Protected by platform_timer_lock. */
   1.110 -static u64 hpet_counter64, hpet_overflow_period;
   1.111 -static u32 hpet_stamp;
   1.112 -static struct timer hpet_overflow_timer;
   1.113 -
   1.114 -static void hpet_overflow(void *unused)
   1.115 +static u32 read_hpet_count(void)
   1.116  {
   1.117 -    u32 counter;
   1.118 -
   1.119 -    spin_lock_irq(&platform_timer_lock);
   1.120 -    counter = hpet_read32(HPET_COUNTER);
   1.121 -    hpet_counter64 += (u32)(counter - hpet_stamp);
   1.122 -    hpet_stamp = counter;
   1.123 -    spin_unlock_irq(&platform_timer_lock);
   1.124 -
   1.125 -    set_timer(&hpet_overflow_timer, NOW() + hpet_overflow_period);
   1.126 +    return hpet_read32(HPET_COUNTER);
   1.127  }
   1.128  
   1.129 -static u64 read_hpet_count(void)
   1.130 -{
   1.131 -    return hpet_counter64 + (u32)(hpet_read32(HPET_COUNTER) - hpet_stamp);
   1.132 -}
   1.133 -
   1.134 -static int init_hpet(void)
   1.135 +static int init_hpet(struct platform_timesource *pts)
   1.136  {
   1.137      u64 hpet_rate;
   1.138      u32 hpet_id, hpet_period, cfg;
   1.139 @@ -391,29 +353,13 @@ static int init_hpet(void)
   1.140      cfg |= HPET_CFG_ENABLE;
   1.141      hpet_write32(cfg, HPET_CFG);
   1.142  
   1.143 -    read_platform_count = read_hpet_count;
   1.144 -
   1.145      hpet_rate = 1000000000000000ULL; /* 10^15 */
   1.146      (void)do_div(hpet_rate, hpet_period);
   1.147 -    set_time_scale(&platform_timer_scale, hpet_rate);
   1.148  
   1.149 -    /* Trigger overflow avoidance roughly when counter increments 2^31. */
   1.150 -    if ( (hpet_rate >> 31) != 0 )
   1.151 -    {
   1.152 -        hpet_overflow_period = MILLISECS(1000);
   1.153 -        (void)do_div(hpet_overflow_period, (u32)(hpet_rate >> 31) + 1);
   1.154 -    }
   1.155 -    else
   1.156 -    {
   1.157 -        hpet_overflow_period = MILLISECS(1000) << 31;
   1.158 -        (void)do_div(hpet_overflow_period, (u32)hpet_rate);
   1.159 -    }
   1.160 -
   1.161 -    init_timer(&hpet_overflow_timer, hpet_overflow, NULL, 0);
   1.162 -    hpet_overflow(NULL);
   1.163 -    platform_timer_stamp = hpet_counter64;
   1.164 -
   1.165 -    printk("Platform timer is %s HPET\n", freq_string(hpet_rate));
   1.166 +    pts->name = "HPET";
   1.167 +    pts->frequency = hpet_rate;
   1.168 +    pts->read_counter = read_hpet_count;
   1.169 +    pts->counter_bits = 32;
   1.170  
   1.171      return 1;
   1.172  }
   1.173 @@ -435,28 +381,12 @@ int use_cyclone;
   1.174  #define CYCLONE_MPCS_OFFSET 0x51A8
   1.175  #define CYCLONE_TIMER_FREQ  100000000
   1.176  
   1.177 -/* Protected by platform_timer_lock. */
   1.178 -static u64 cyclone_counter64;
   1.179 -static u32 cyclone_stamp;
   1.180 -static struct timer cyclone_overflow_timer;
   1.181 -static volatile u32 *cyclone_timer; /* Cyclone MPMC0 register */
   1.182 -
   1.183 -static void cyclone_overflow(void *unused)
   1.184 -{
   1.185 -    u32 counter;
   1.186 +/* Cyclone MPMC0 register. */
   1.187 +static volatile u32 *cyclone_timer;
   1.188  
   1.189 -    spin_lock_irq(&platform_timer_lock);
   1.190 -    counter = *cyclone_timer;
   1.191 -    cyclone_counter64 += (u32)(counter - cyclone_stamp);
   1.192 -    cyclone_stamp = counter;
   1.193 -    spin_unlock_irq(&platform_timer_lock);
   1.194 -
   1.195 -    set_timer(&cyclone_overflow_timer, NOW() + MILLISECS(20000));
   1.196 -}
   1.197 -
   1.198 -static u64 read_cyclone_count(void)
   1.199 +static u32 read_cyclone_count(void)
   1.200  {
   1.201 -    return cyclone_counter64 + (u32)(*cyclone_timer - cyclone_stamp);
   1.202 +    return *cyclone_timer;
   1.203  }
   1.204  
   1.205  static volatile u32 *map_cyclone_reg(unsigned long regaddr)
   1.206 @@ -467,7 +397,7 @@ static volatile u32 *map_cyclone_reg(uns
   1.207      return (volatile u32 *)(fix_to_virt(FIX_CYCLONE_TIMER) + offset);
   1.208  }
   1.209  
   1.210 -static int init_cyclone(void)
   1.211 +static int init_cyclone(struct platform_timesource *pts)
   1.212  {
   1.213      u32 base;
   1.214      
   1.215 @@ -487,15 +417,10 @@ static int init_cyclone(void)
   1.216      *(map_cyclone_reg(base + CYCLONE_MPCS_OFFSET)) = 1;
   1.217      cyclone_timer = map_cyclone_reg(base + CYCLONE_MPMC_OFFSET);
   1.218  
   1.219 -    read_platform_count = read_cyclone_count;
   1.220 -
   1.221 -    init_timer(&cyclone_overflow_timer, cyclone_overflow, NULL, 0);
   1.222 -    cyclone_overflow(NULL);
   1.223 -    platform_timer_stamp = cyclone_counter64;
   1.224 -    set_time_scale(&platform_timer_scale, CYCLONE_TIMER_FREQ);
   1.225 -
   1.226 -    printk("Platform timer is %s IBM Cyclone\n",
   1.227 -           freq_string(CYCLONE_TIMER_FREQ));
   1.228 +    pts->name = "IBM Cyclone";
   1.229 +    pts->frequency = CYCLONE_TIMER_FREQ;
   1.230 +    pts->read_counter = read_cyclone_count;
   1.231 +    pts->counter_bits = 32;
   1.232  
   1.233      return 1;
   1.234  }
   1.235 @@ -506,50 +431,23 @@ static int init_cyclone(void)
   1.236  
   1.237  u32 pmtmr_ioport;
   1.238  
   1.239 -/* Protected by platform_timer_lock. */
   1.240 -static u64 pmtimer_counter64;
   1.241 -static u32 pmtimer_stamp;
   1.242 -static struct timer pmtimer_overflow_timer;
   1.243 -
   1.244  /* ACPI PM timer ticks at 3.579545 MHz. */
   1.245  #define ACPI_PM_FREQUENCY 3579545
   1.246  
   1.247 -/* Deltas are 24-bit unsigned values, as counter may be only 24 bits wide. */
   1.248 -#define pmtimer_delta(c) ((u32)(((c) - pmtimer_stamp) & ((1U<<24)-1)))
   1.249 -
   1.250 -static void pmtimer_overflow(void *unused)
   1.251 +static u32 read_pmtimer_count(void)
   1.252  {
   1.253 -    u32 counter;
   1.254 -
   1.255 -    spin_lock_irq(&platform_timer_lock);
   1.256 -    counter = inl(pmtmr_ioport);
   1.257 -    pmtimer_counter64 += pmtimer_delta(counter);
   1.258 -    pmtimer_stamp = counter;
   1.259 -    spin_unlock_irq(&platform_timer_lock);
   1.260 -
   1.261 -    /* Trigger overflow avoidance roughly when counter increments 2^23. */
   1.262 -    set_timer(&pmtimer_overflow_timer, NOW() + MILLISECS(2000));
   1.263 +    return inl(pmtmr_ioport);
   1.264  }
   1.265  
   1.266 -static u64 read_pmtimer_count(void)
   1.267 -{
   1.268 -    return pmtimer_counter64 + pmtimer_delta(inl(pmtmr_ioport));
   1.269 -}
   1.270 -
   1.271 -static int init_pmtimer(void)
   1.272 +static int init_pmtimer(struct platform_timesource *pts)
   1.273  {
   1.274      if ( pmtmr_ioport == 0 )
   1.275          return 0;
   1.276  
   1.277 -    read_platform_count = read_pmtimer_count;
   1.278 -
   1.279 -    init_timer(&pmtimer_overflow_timer, pmtimer_overflow, NULL, 0);
   1.280 -    pmtimer_overflow(NULL);
   1.281 -    platform_timer_stamp = pmtimer_counter64;
   1.282 -    set_time_scale(&platform_timer_scale, ACPI_PM_FREQUENCY);
   1.283 -
   1.284 -    printk("Platform timer is %s ACPI PM Timer\n",
   1.285 -           freq_string(ACPI_PM_FREQUENCY));
   1.286 +    pts->name = "ACPI PM Timer";
   1.287 +    pts->frequency = ACPI_PM_FREQUENCY;
   1.288 +    pts->read_counter = read_pmtimer_count;
   1.289 +    pts->counter_bits = 24;
   1.290  
   1.291      return 1;
   1.292  }
   1.293 @@ -558,21 +456,43 @@ static int init_pmtimer(void)
   1.294   * GENERIC PLATFORM TIMER INFRASTRUCTURE
   1.295   */
   1.296  
   1.297 +static struct platform_timesource plt_src; /* details of chosen timesource  */
   1.298 +static u32 plt_mask;             /* hardware-width mask                     */
   1.299 +static u32 plt_overflow_period;  /* jiffies between calls to plt_overflow() */
   1.300 +static struct time_scale plt_scale; /* scale: platform counter -> nanosecs  */
   1.301 +
   1.302 +/* Protected by platform_timer_lock. */
   1.303 +static u64 plt_count64;          /* 64-bit platform counter stamp           */
   1.304 +static u32 plt_count;            /* hardware-width platform counter stamp   */
   1.305 +
   1.306 +static void plt_overflow(void)
   1.307 +{
   1.308 +    u32 count;
   1.309 +    unsigned long flags;
   1.310 +
   1.311 +    spin_lock_irqsave(&platform_timer_lock, flags);
   1.312 +    count = plt_src.read_counter();
   1.313 +    plt_count64 += (count - plt_count) & plt_mask;
   1.314 +    plt_count = count;
   1.315 +    plt_overflow_jiffies = plt_overflow_period;
   1.316 +    spin_unlock_irqrestore(&platform_timer_lock, flags);
   1.317 +}
   1.318 +
   1.319  static s_time_t __read_platform_stime(u64 platform_time)
   1.320  {
   1.321      u64 diff = platform_time - platform_timer_stamp;
   1.322      ASSERT(spin_is_locked(&platform_timer_lock));
   1.323 -    return (stime_platform_stamp + scale_delta(diff, &platform_timer_scale));
   1.324 +    return (stime_platform_stamp + scale_delta(diff, &plt_scale));
   1.325  }
   1.326  
   1.327  static s_time_t read_platform_stime(void)
   1.328  {
   1.329 -    u64 counter;
   1.330 +    u64 count;
   1.331      s_time_t stime;
   1.332  
   1.333      spin_lock_irq(&platform_timer_lock);
   1.334 -    counter = read_platform_count();
   1.335 -    stime   = __read_platform_stime(counter);
   1.336 +    count = plt_count64 + ((plt_src.read_counter() - plt_count) & plt_mask);
   1.337 +    stime = __read_platform_stime(count);
   1.338      spin_unlock_irq(&platform_timer_lock);
   1.339  
   1.340      return stime;
   1.341 @@ -580,42 +500,60 @@ static s_time_t read_platform_stime(void
   1.342  
   1.343  static void platform_time_calibration(void)
   1.344  {
   1.345 -    u64 counter;
   1.346 +    u64 count;
   1.347      s_time_t stamp;
   1.348  
   1.349      spin_lock_irq(&platform_timer_lock);
   1.350 -    counter = read_platform_count();
   1.351 -    stamp   = __read_platform_stime(counter);
   1.352 +    count = plt_count64 + ((plt_src.read_counter() - plt_count) & plt_mask);
   1.353 +    stamp = __read_platform_stime(count);
   1.354      stime_platform_stamp = stamp;
   1.355 -    platform_timer_stamp = counter;
   1.356 +    platform_timer_stamp = count;
   1.357      spin_unlock_irq(&platform_timer_lock);
   1.358  }
   1.359  
   1.360  static void init_platform_timer(void)
   1.361  {
   1.362 +    struct platform_timesource *pts = &plt_src;
   1.363 +    u64 overflow_period;
   1.364 +    int rc = -1;
   1.365 +
   1.366      if ( opt_clocksource[0] != '\0' )
   1.367      {
   1.368 -        int rc = -1;
   1.369 -
   1.370          if ( !strcmp(opt_clocksource, "pit") )
   1.371 -            rc = (init_pit(), 1);
   1.372 +            rc = (init_pit(pts), 1);
   1.373          else if ( !strcmp(opt_clocksource, "hpet") )
   1.374 -            rc = init_hpet();
   1.375 +            rc = init_hpet(pts);
   1.376          else if ( !strcmp(opt_clocksource, "cyclone") )
   1.377 -            rc = init_cyclone();
   1.378 +            rc = init_cyclone(pts);
   1.379          else if ( !strcmp(opt_clocksource, "acpi") )
   1.380 -            rc = init_pmtimer();
   1.381 +            rc = init_pmtimer(pts);
   1.382  
   1.383 -        if ( rc == 1 )
   1.384 -            return;
   1.385 -
   1.386 -        printk("WARNING: %s clocksource '%s'.\n",
   1.387 -               (rc == 0) ? "Could not initialise" : "Unrecognised",
   1.388 -               opt_clocksource);
   1.389 +        if ( rc <= 0 )
   1.390 +            printk("WARNING: %s clocksource '%s'.\n",
   1.391 +                   (rc == 0) ? "Could not initialise" : "Unrecognised",
   1.392 +                   opt_clocksource);
   1.393      }
   1.394  
   1.395 -    if ( !init_cyclone() && !init_hpet() && !init_pmtimer() )
   1.396 -        init_pit();
   1.397 +    if ( (rc <= 0) &&
   1.398 +         !init_cyclone(pts) &&
   1.399 +         !init_hpet(pts) &&
   1.400 +         !init_pmtimer(pts) )
   1.401 +        init_pit(pts);
   1.402 +
   1.403 +    plt_mask = (u32)~0u >> (32 - pts->counter_bits);
   1.404 +
   1.405 +    set_time_scale(&plt_scale, pts->frequency);
   1.406 +
   1.407 +    overflow_period = scale_delta(1ull << (pts->counter_bits-1), &plt_scale);
   1.408 +    do_div(overflow_period, MILLISECS(1000/HZ));
   1.409 +    plt_overflow_period = overflow_period;
   1.410 +    plt_overflow();
   1.411 +    printk("Platform timer overflows in %d jiffies.\n", plt_overflow_period);
   1.412 +
   1.413 +    platform_timer_stamp = plt_count64;
   1.414 +
   1.415 +    printk("Platform timer is %s %s\n",
   1.416 +           freq_string(pts->frequency), pts->name);
   1.417  }
   1.418  
   1.419