ia64/xen-unstable

view xen/arch/x86/time.c @ 7124:f069a06e650f

Fix fixed-point long multiplication used in time delta
calculations. The carry flag was clobbered before it was
added to the result.

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Wed Sep 28 22:18:30 2005 +0100 (2005-09-28)
parents 06d84bf87159
children 61b3b357d827
line source
1 /******************************************************************************
2 * arch/x86/time.c
3 *
4 * Per-CPU time calibration and management.
5 *
6 * Copyright (c) 2002-2005, K A Fraser
7 *
8 * Portions from Linux are:
9 * Copyright (c) 1991, 1992, 1995 Linus Torvalds
10 */
12 #include <xen/config.h>
13 #include <xen/errno.h>
14 #include <xen/event.h>
15 #include <xen/sched.h>
16 #include <xen/lib.h>
17 #include <xen/config.h>
18 #include <xen/init.h>
19 #include <xen/time.h>
20 #include <xen/ac_timer.h>
21 #include <xen/smp.h>
22 #include <xen/irq.h>
23 #include <xen/softirq.h>
24 #include <asm/io.h>
25 #include <asm/msr.h>
26 #include <asm/mpspec.h>
27 #include <asm/processor.h>
28 #include <asm/fixmap.h>
29 #include <asm/mc146818rtc.h>
30 #include <asm/div64.h>
31 #include <asm/hpet.h>
32 #include <io_ports.h>
34 /* opt_hpet_force: If true, force HPET configuration via PCI space. */
35 /* NB. This is a gross hack. Mainly useful for HPET testing. */
36 static int opt_hpet_force = 0;
37 boolean_param("hpet_force", opt_hpet_force);
39 #define EPOCH MILLISECS(1000)
41 unsigned long cpu_khz; /* CPU clock frequency in kHz. */
42 unsigned long hpet_address;
43 spinlock_t rtc_lock = SPIN_LOCK_UNLOCKED;
44 int timer_ack = 0;
45 unsigned long volatile jiffies;
46 static u32 wc_sec, wc_nsec; /* UTC time at last 'time update'. */
47 static spinlock_t wc_lock = SPIN_LOCK_UNLOCKED;
49 struct time_scale {
50 int shift;
51 u32 mul_frac;
52 };
54 struct cpu_time {
55 u64 local_tsc_stamp;
56 s_time_t stime_local_stamp;
57 s_time_t stime_master_stamp;
58 struct time_scale tsc_scale;
59 struct ac_timer calibration_timer;
60 } __cacheline_aligned;
62 static struct cpu_time cpu_time[NR_CPUS];
64 /* Protected by platform_timer_lock. */
65 static s_time_t stime_platform_stamp;
66 static u64 platform_timer_stamp;
67 static struct time_scale platform_timer_scale;
68 static spinlock_t platform_timer_lock = SPIN_LOCK_UNLOCKED;
69 static u64 (*read_platform_count)(void);
71 /*
72 * 32-bit division of integer dividend and integer divisor yielding
73 * 32-bit fractional quotient.
74 */
75 static inline u32 div_frac(u32 dividend, u32 divisor)
76 {
77 u32 quotient, remainder;
78 ASSERT(dividend < divisor);
79 __asm__ (
80 "divl %4"
81 : "=a" (quotient), "=d" (remainder)
82 : "0" (0), "1" (dividend), "r" (divisor) );
83 return quotient;
84 }
86 /*
87 * 32-bit multiplication of multiplicand and fractional multiplier
88 * yielding 32-bit product (radix point at same position as in multiplicand).
89 */
90 static inline u32 mul_frac(u32 multiplicand, u32 multiplier)
91 {
92 u32 product_int, product_frac;
93 __asm__ (
94 "mul %3"
95 : "=a" (product_frac), "=d" (product_int)
96 : "0" (multiplicand), "r" (multiplier) );
97 return product_int;
98 }
100 /*
101 * Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction,
102 * yielding a 64-bit result.
103 */
104 static inline u64 scale_delta(u64 delta, struct time_scale *scale)
105 {
106 u64 product;
107 #ifdef CONFIG_X86_32
108 u32 tmp1, tmp2;
109 #endif
111 if ( scale->shift < 0 )
112 delta >>= -scale->shift;
113 else
114 delta <<= scale->shift;
116 #ifdef CONFIG_X86_32
117 __asm__ (
118 "mul %5 ; "
119 "mov %4,%%eax ; "
120 "mov %%edx,%4 ; "
121 "mul %5 ; "
122 "xor %5,%5 ; "
123 "add %4,%%eax ; "
124 "adc %5,%%edx ; "
125 : "=A" (product), "=r" (tmp1), "=r" (tmp2)
126 : "a" ((u32)delta), "1" ((u32)(delta >> 32)), "2" (scale->mul_frac) );
127 #else
128 __asm__ (
129 "mul %%rdx ; shrd $32,%%rdx,%%rax"
130 : "=a" (product) : "0" (delta), "d" ((u64)scale->mul_frac) );
131 #endif
133 return product;
134 }
136 void timer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
137 {
138 if ( timer_ack )
139 {
140 extern spinlock_t i8259A_lock;
141 spin_lock(&i8259A_lock);
142 outb(0x0c, 0x20);
143 /* Ack the IRQ; AEOI will end it automatically. */
144 inb(0x20);
145 spin_unlock(&i8259A_lock);
146 }
148 /* Update jiffies counter. */
149 (*(unsigned long *)&jiffies)++;
151 /* Rough hack to allow accurate timers to sort-of-work with no APIC. */
152 if ( !cpu_has_apic )
153 raise_softirq(AC_TIMER_SOFTIRQ);
154 }
156 static struct irqaction irq0 = { timer_interrupt, "timer", NULL};
158 /* ------ Calibrate the TSC -------
159 * Return processor ticks per second / CALIBRATE_FRAC.
160 */
162 #define CLOCK_TICK_RATE 1193180 /* system crystal frequency (Hz) */
163 #define CALIBRATE_FRAC 20 /* calibrate over 50ms */
164 #define CALIBRATE_LATCH ((CLOCK_TICK_RATE+(CALIBRATE_FRAC/2))/CALIBRATE_FRAC)
166 static u64 calibrate_boot_tsc(void)
167 {
168 u64 start, end;
169 unsigned long count;
171 /* Set the Gate high, disable speaker */
172 outb((inb(0x61) & ~0x02) | 0x01, 0x61);
174 /*
175 * Now let's take care of CTC channel 2
176 *
177 * Set the Gate high, program CTC channel 2 for mode 0, (interrupt on
178 * terminal count mode), binary count, load 5 * LATCH count, (LSB and MSB)
179 * to begin countdown.
180 */
181 outb(0xb0, PIT_MODE); /* binary, mode 0, LSB/MSB, Ch 2 */
182 outb(CALIBRATE_LATCH & 0xff, PIT_CH2); /* LSB of count */
183 outb(CALIBRATE_LATCH >> 8, PIT_CH2); /* MSB of count */
185 rdtscll(start);
186 for ( count = 0; (inb(0x61) & 0x20) == 0; count++ )
187 continue;
188 rdtscll(end);
190 /* Error if the CTC doesn't behave itself. */
191 if ( count == 0 )
192 return 0;
194 return ((end - start) * (u64)CALIBRATE_FRAC);
195 }
197 static void set_time_scale(struct time_scale *ts, u64 ticks_per_sec)
198 {
199 u64 tps64 = ticks_per_sec;
200 u32 tps32;
201 int shift = 0;
203 while ( tps64 > (MILLISECS(1000)*2) )
204 {
205 tps64 >>= 1;
206 shift--;
207 }
209 tps32 = (u32)tps64;
210 while ( tps32 < (u32)MILLISECS(1000) )
211 {
212 tps32 <<= 1;
213 shift++;
214 }
216 ts->mul_frac = div_frac(MILLISECS(1000), tps32);
217 ts->shift = shift;
218 }
220 static atomic_t tsc_calibrate_gang = ATOMIC_INIT(0);
221 static unsigned int tsc_calibrate_status = 0;
223 void calibrate_tsc_bp(void)
224 {
225 while ( atomic_read(&tsc_calibrate_gang) != (num_booting_cpus() - 1) )
226 mb();
228 outb(CALIBRATE_LATCH & 0xff, PIT_CH2);
229 outb(CALIBRATE_LATCH >> 8, PIT_CH2);
231 tsc_calibrate_status = 1;
232 wmb();
234 while ( (inb(0x61) & 0x20) == 0 )
235 continue;
237 tsc_calibrate_status = 2;
238 wmb();
240 while ( atomic_read(&tsc_calibrate_gang) != 0 )
241 mb();
242 }
244 void calibrate_tsc_ap(void)
245 {
246 u64 t1, t2, ticks_per_sec;
248 atomic_inc(&tsc_calibrate_gang);
250 while ( tsc_calibrate_status < 1 )
251 mb();
253 rdtscll(t1);
255 while ( tsc_calibrate_status < 2 )
256 mb();
258 rdtscll(t2);
260 ticks_per_sec = (t2 - t1) * (u64)CALIBRATE_FRAC;
261 set_time_scale(&cpu_time[smp_processor_id()].tsc_scale, ticks_per_sec);
263 atomic_dec(&tsc_calibrate_gang);
264 }
266 static char *freq_string(u64 freq)
267 {
268 static char s[20];
269 unsigned int x, y;
270 y = (unsigned int)do_div(freq, 1000000) / 1000;
271 x = (unsigned int)freq;
272 sprintf(s, "%u.%03uMHz", x, y);
273 return s;
274 }
276 /************************************************************
277 * PLATFORM TIMER 1: PROGRAMMABLE INTERVAL TIMER (LEGACY PIT)
278 */
280 /* Protected by platform_timer_lock. */
281 static u64 pit_counter64;
282 static u16 pit_stamp;
283 static struct ac_timer pit_overflow_timer;
285 static u16 pit_read_counter(void)
286 {
287 u16 count;
288 ASSERT(spin_is_locked(&platform_timer_lock));
289 outb(0x80, PIT_MODE);
290 count = inb(PIT_CH2);
291 count |= inb(PIT_CH2) << 8;
292 return count;
293 }
295 static void pit_overflow(void *unused)
296 {
297 u16 counter;
299 spin_lock(&platform_timer_lock);
300 counter = pit_read_counter();
301 pit_counter64 += (u16)(pit_stamp - counter);
302 pit_stamp = counter;
303 spin_unlock(&platform_timer_lock);
305 set_ac_timer(&pit_overflow_timer, NOW() + MILLISECS(20));
306 }
308 static u64 read_pit_count(void)
309 {
310 return pit_counter64 + (u16)(pit_stamp - pit_read_counter());
311 }
313 static int init_pit(void)
314 {
315 read_platform_count = read_pit_count;
317 init_ac_timer(&pit_overflow_timer, pit_overflow, NULL, 0);
318 pit_overflow(NULL);
319 platform_timer_stamp = pit_counter64;
320 set_time_scale(&platform_timer_scale, CLOCK_TICK_RATE);
322 printk("Platform timer is %s PIT\n", freq_string(CLOCK_TICK_RATE));
324 return 1;
325 }
327 /************************************************************
328 * PLATFORM TIMER 2: HIGH PRECISION EVENT TIMER (HPET)
329 */
331 /* Protected by platform_timer_lock. */
332 static u64 hpet_counter64, hpet_overflow_period;
333 static u32 hpet_stamp;
334 static struct ac_timer hpet_overflow_timer;
336 static void hpet_overflow(void *unused)
337 {
338 u32 counter;
340 spin_lock(&platform_timer_lock);
341 counter = hpet_read32(HPET_COUNTER);
342 hpet_counter64 += (u32)(counter - hpet_stamp);
343 hpet_stamp = counter;
344 spin_unlock(&platform_timer_lock);
346 set_ac_timer(&hpet_overflow_timer, NOW() + hpet_overflow_period);
347 }
349 static u64 read_hpet_count(void)
350 {
351 return hpet_counter64 + (u32)(hpet_read32(HPET_COUNTER) - hpet_stamp);
352 }
354 static int init_hpet(void)
355 {
356 u64 hpet_rate;
357 u32 hpet_id, hpet_period, cfg;
358 int i;
360 if ( (hpet_address == 0) && opt_hpet_force )
361 {
362 outl(0x800038a0, 0xcf8);
363 outl(0xff000001, 0xcfc);
364 outl(0x800038a0, 0xcf8);
365 hpet_address = inl(0xcfc) & 0xfffffffe;
366 printk("WARNING: Forcibly enabled HPET at %#lx.\n", hpet_address);
367 }
369 if ( hpet_address == 0 )
370 return 0;
372 set_fixmap_nocache(FIX_HPET_BASE, hpet_address);
374 hpet_id = hpet_read32(HPET_ID);
375 if ( hpet_id == 0 )
376 {
377 printk("BAD HPET vendor id.\n");
378 return 0;
379 }
381 /* Check for sane period (100ps <= period <= 100ns). */
382 hpet_period = hpet_read32(HPET_PERIOD);
383 if ( (hpet_period > 100000000) || (hpet_period < 100000) )
384 {
385 printk("BAD HPET period %u.\n", hpet_period);
386 return 0;
387 }
389 cfg = hpet_read32(HPET_CFG);
390 cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY);
391 hpet_write32(cfg, HPET_CFG);
393 for ( i = 0; i <= ((hpet_id >> 8) & 31); i++ )
394 {
395 cfg = hpet_read32(HPET_T0_CFG + i*0x20);
396 cfg &= ~HPET_TN_ENABLE;
397 hpet_write32(cfg & ~HPET_TN_ENABLE, HPET_T0_CFG);
398 }
400 cfg = hpet_read32(HPET_CFG);
401 cfg |= HPET_CFG_ENABLE;
402 hpet_write32(cfg, HPET_CFG);
404 read_platform_count = read_hpet_count;
406 hpet_rate = 1000000000000000ULL; /* 10^15 */
407 (void)do_div(hpet_rate, hpet_period);
408 set_time_scale(&platform_timer_scale, hpet_rate);
410 /* Trigger overflow avoidance roughly when counter increments 2^31. */
411 if ( (hpet_rate >> 31) != 0 )
412 {
413 hpet_overflow_period = MILLISECS(1000);
414 (void)do_div(hpet_overflow_period, (u32)(hpet_rate >> 31) + 1);
415 }
416 else
417 {
418 hpet_overflow_period = MILLISECS(1000) << 31;
419 (void)do_div(hpet_overflow_period, (u32)hpet_rate);
420 }
422 init_ac_timer(&hpet_overflow_timer, hpet_overflow, NULL, 0);
423 hpet_overflow(NULL);
424 platform_timer_stamp = hpet_counter64;
426 printk("Platform timer is %s HPET\n", freq_string(hpet_rate));
428 return 1;
429 }
431 /************************************************************
432 * PLATFORM TIMER 3: IBM 'CYCLONE' TIMER
433 */
435 int use_cyclone;
437 /*
438 * Although the counter is read via a 64-bit register, I believe it is actually
439 * a 40-bit counter. Since this will wrap, I read only the low 32 bits and
440 * periodically fold into a 64-bit software counter, just as for PIT and HPET.
441 */
442 #define CYCLONE_CBAR_ADDR 0xFEB00CD0
443 #define CYCLONE_PMCC_OFFSET 0x51A0
444 #define CYCLONE_MPMC_OFFSET 0x51D0
445 #define CYCLONE_MPCS_OFFSET 0x51A8
446 #define CYCLONE_TIMER_FREQ 100000000
448 /* Protected by platform_timer_lock. */
449 static u64 cyclone_counter64;
450 static u32 cyclone_stamp;
451 static struct ac_timer cyclone_overflow_timer;
452 static volatile u32 *cyclone_timer; /* Cyclone MPMC0 register */
454 static void cyclone_overflow(void *unused)
455 {
456 u32 counter;
458 spin_lock(&platform_timer_lock);
459 counter = *cyclone_timer;
460 cyclone_counter64 += (u32)(counter - cyclone_stamp);
461 cyclone_stamp = counter;
462 spin_unlock(&platform_timer_lock);
464 set_ac_timer(&cyclone_overflow_timer, NOW() + MILLISECS(20000));
465 }
467 static u64 read_cyclone_count(void)
468 {
469 return cyclone_counter64 + (u32)(*cyclone_timer - cyclone_stamp);
470 }
472 static volatile u32 *map_cyclone_reg(unsigned long regaddr)
473 {
474 unsigned long pageaddr = regaddr & PAGE_MASK;
475 unsigned long offset = regaddr & ~PAGE_MASK;
476 set_fixmap_nocache(FIX_CYCLONE_TIMER, pageaddr);
477 return (volatile u32 *)(fix_to_virt(FIX_CYCLONE_TIMER) + offset);
478 }
480 static int init_cyclone(void)
481 {
482 u32 base;
484 if ( !use_cyclone )
485 return 0;
487 /* Find base address. */
488 base = *(map_cyclone_reg(CYCLONE_CBAR_ADDR));
489 if ( base == 0 )
490 {
491 printk(KERN_ERR "Cyclone: Could not find valid CBAR value.\n");
492 return 0;
493 }
495 /* Enable timer and map the counter register. */
496 *(map_cyclone_reg(base + CYCLONE_PMCC_OFFSET)) = 1;
497 *(map_cyclone_reg(base + CYCLONE_MPCS_OFFSET)) = 1;
498 cyclone_timer = map_cyclone_reg(base + CYCLONE_MPMC_OFFSET);
500 read_platform_count = read_cyclone_count;
502 init_ac_timer(&cyclone_overflow_timer, cyclone_overflow, NULL, 0);
503 cyclone_overflow(NULL);
504 platform_timer_stamp = cyclone_counter64;
505 set_time_scale(&platform_timer_scale, CYCLONE_TIMER_FREQ);
507 printk("Platform timer is %s IBM Cyclone\n",
508 freq_string(CYCLONE_TIMER_FREQ));
510 return 1;
511 }
513 /************************************************************
514 * GENERIC PLATFORM TIMER INFRASTRUCTURE
515 */
517 static s_time_t __read_platform_stime(u64 platform_time)
518 {
519 u64 diff = platform_time - platform_timer_stamp;
520 ASSERT(spin_is_locked(&platform_timer_lock));
521 return (stime_platform_stamp + scale_delta(diff, &platform_timer_scale));
522 }
524 static s_time_t read_platform_stime(void)
525 {
526 u64 counter;
527 s_time_t stime;
529 spin_lock(&platform_timer_lock);
530 counter = read_platform_count();
531 stime = __read_platform_stime(counter);
532 spin_unlock(&platform_timer_lock);
534 return stime;
535 }
537 static void platform_time_calibration(void)
538 {
539 u64 counter;
540 s_time_t stamp;
542 spin_lock(&platform_timer_lock);
543 counter = read_platform_count();
544 stamp = __read_platform_stime(counter);
545 stime_platform_stamp = stamp;
546 platform_timer_stamp = counter;
547 spin_unlock(&platform_timer_lock);
548 }
550 static void init_platform_timer(void)
551 {
552 if ( !init_cyclone() && !init_hpet() )
553 BUG_ON(!init_pit());
554 }
557 /***************************************************************************
558 * CMOS Timer functions
559 ***************************************************************************/
561 /* Converts Gregorian date to seconds since 1970-01-01 00:00:00.
562 * Assumes input in normal date format, i.e. 1980-12-31 23:59:59
563 * => year=1980, mon=12, day=31, hour=23, min=59, sec=59.
564 *
565 * [For the Julian calendar (which was used in Russia before 1917,
566 * Britain & colonies before 1752, anywhere else before 1582,
567 * and is still in use by some communities) leave out the
568 * -year/100+year/400 terms, and add 10.]
569 *
570 * This algorithm was first published by Gauss (I think).
571 *
572 * WARNING: this function will overflow on 2106-02-07 06:28:16 on
573 * machines were long is 32-bit! (However, as time_t is signed, we
574 * will already get problems at other places on 2038-01-19 03:14:08)
575 */
576 static inline unsigned long
577 mktime (unsigned int year, unsigned int mon,
578 unsigned int day, unsigned int hour,
579 unsigned int min, unsigned int sec)
580 {
581 /* 1..12 -> 11,12,1..10: put Feb last since it has a leap day. */
582 if ( 0 >= (int) (mon -= 2) )
583 {
584 mon += 12;
585 year -= 1;
586 }
588 return ((((unsigned long)(year/4 - year/100 + year/400 + 367*mon/12 + day)+
589 year*365 - 719499
590 )*24 + hour /* now have hours */
591 )*60 + min /* now have minutes */
592 )*60 + sec; /* finally seconds */
593 }
595 static unsigned long __get_cmos_time(void)
596 {
597 unsigned int year, mon, day, hour, min, sec;
599 sec = CMOS_READ(RTC_SECONDS);
600 min = CMOS_READ(RTC_MINUTES);
601 hour = CMOS_READ(RTC_HOURS);
602 day = CMOS_READ(RTC_DAY_OF_MONTH);
603 mon = CMOS_READ(RTC_MONTH);
604 year = CMOS_READ(RTC_YEAR);
606 if ( !(CMOS_READ(RTC_CONTROL) & RTC_DM_BINARY) || RTC_ALWAYS_BCD )
607 {
608 BCD_TO_BIN(sec);
609 BCD_TO_BIN(min);
610 BCD_TO_BIN(hour);
611 BCD_TO_BIN(day);
612 BCD_TO_BIN(mon);
613 BCD_TO_BIN(year);
614 }
616 if ( (year += 1900) < 1970 )
617 year += 100;
619 return mktime(year, mon, day, hour, min, sec);
620 }
622 static unsigned long get_cmos_time(void)
623 {
624 unsigned long res, flags;
625 int i;
627 spin_lock_irqsave(&rtc_lock, flags);
629 /* read RTC exactly on falling edge of update flag */
630 for ( i = 0 ; i < 1000000 ; i++ ) /* may take up to 1 second... */
631 if ( (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) )
632 break;
633 for ( i = 0 ; i < 1000000 ; i++ ) /* must try at least 2.228 ms */
634 if ( !(CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) )
635 break;
637 res = __get_cmos_time();
639 spin_unlock_irqrestore(&rtc_lock, flags);
640 return res;
641 }
643 /***************************************************************************
644 * System Time
645 ***************************************************************************/
647 s_time_t get_s_time(void)
648 {
649 struct cpu_time *t = &cpu_time[smp_processor_id()];
650 u64 tsc, delta;
651 s_time_t now;
653 rdtscll(tsc);
654 delta = tsc - t->local_tsc_stamp;
655 now = t->stime_local_stamp + scale_delta(delta, &t->tsc_scale);
657 return now;
658 }
660 static inline void version_update_begin(u32 *version)
661 {
662 /* Explicitly OR with 1 just in case version number gets out of sync. */
663 *version = (*version + 1) | 1;
664 wmb();
665 }
667 static inline void version_update_end(u32 *version)
668 {
669 wmb();
670 (*version)++;
671 }
673 static inline void __update_dom_time(struct vcpu *v)
674 {
675 struct cpu_time *t = &cpu_time[smp_processor_id()];
676 struct vcpu_time_info *u = &v->domain->shared_info->vcpu_time[v->vcpu_id];
678 version_update_begin(&u->version);
680 u->tsc_timestamp = t->local_tsc_stamp;
681 u->system_time = t->stime_local_stamp;
682 u->tsc_to_system_mul = t->tsc_scale.mul_frac;
683 u->tsc_shift = (s8)t->tsc_scale.shift;
685 version_update_end(&u->version);
686 }
688 void update_dom_time(struct vcpu *v)
689 {
690 if ( v->domain->shared_info->vcpu_time[v->vcpu_id].tsc_timestamp !=
691 cpu_time[smp_processor_id()].local_tsc_stamp )
692 __update_dom_time(v);
693 }
695 /* Set clock to <secs,usecs> after 00:00:00 UTC, 1 January, 1970. */
696 void do_settime(unsigned long secs, unsigned long nsecs, u64 system_time_base)
697 {
698 u64 x;
699 u32 y, _wc_sec, _wc_nsec;
700 struct domain *d;
701 shared_info_t *s;
703 x = (secs * 1000000000ULL) + (u64)nsecs - system_time_base;
704 y = do_div(x, 1000000000);
706 wc_sec = _wc_sec = (u32)x;
707 wc_nsec = _wc_nsec = (u32)y;
709 read_lock(&domlist_lock);
710 spin_lock(&wc_lock);
712 for_each_domain ( d )
713 {
714 s = d->shared_info;
715 version_update_begin(&s->wc_version);
716 s->wc_sec = _wc_sec;
717 s->wc_nsec = _wc_nsec;
718 version_update_end(&s->wc_version);
719 }
721 spin_unlock(&wc_lock);
722 read_unlock(&domlist_lock);
723 }
725 void init_domain_time(struct domain *d)
726 {
727 spin_lock(&wc_lock);
728 version_update_begin(&d->shared_info->wc_version);
729 d->shared_info->wc_sec = wc_sec;
730 d->shared_info->wc_nsec = wc_nsec;
731 version_update_end(&d->shared_info->wc_version);
732 spin_unlock(&wc_lock);
733 }
735 static void local_time_calibration(void *unused)
736 {
737 unsigned int cpu = smp_processor_id();
739 /*
740 * System timestamps, extrapolated from local and master oscillators,
741 * taken during this calibration and the previous calibration.
742 */
743 s_time_t prev_local_stime, curr_local_stime;
744 s_time_t prev_master_stime, curr_master_stime;
746 /* TSC timestamps taken during this calibration and prev calibration. */
747 u64 prev_tsc, curr_tsc;
749 /*
750 * System time and TSC ticks elapsed during the previous calibration
751 * 'epoch'. These values are down-shifted to fit in 32 bits.
752 */
753 u64 stime_elapsed64, tsc_elapsed64;
754 u32 stime_elapsed32, tsc_elapsed32;
756 /* The accumulated error in the local estimate. */
757 u64 local_stime_err;
759 /* Error correction to slow down a fast local clock. */
760 u32 error_factor = 0;
762 /* Calculated TSC shift to ensure 32-bit scale multiplier. */
763 int tsc_shift = 0;
765 /* The overall calibration scale multiplier. */
766 u32 calibration_mul_frac;
768 prev_tsc = cpu_time[cpu].local_tsc_stamp;
769 prev_local_stime = cpu_time[cpu].stime_local_stamp;
770 prev_master_stime = cpu_time[cpu].stime_master_stamp;
772 /* Disable IRQs to get 'instantaneous' current timestamps. */
773 local_irq_disable();
774 rdtscll(curr_tsc);
775 curr_local_stime = get_s_time();
776 curr_master_stime = read_platform_stime();
777 local_irq_enable();
779 #if 0
780 printk("PRE%d: tsc=%lld stime=%lld master=%lld\n",
781 cpu, prev_tsc, prev_local_stime, prev_master_stime);
782 printk("CUR%d: tsc=%lld stime=%lld master=%lld -> %lld\n",
783 cpu, curr_tsc, curr_local_stime, curr_master_stime,
784 curr_master_stime - curr_local_stime);
785 #endif
787 /* Local time warps forward if it lags behind master time. */
788 if ( curr_local_stime < curr_master_stime )
789 curr_local_stime = curr_master_stime;
791 stime_elapsed64 = curr_master_stime - prev_master_stime;
792 tsc_elapsed64 = curr_tsc - prev_tsc;
794 /*
795 * Weirdness can happen if we lose sync with the platform timer.
796 * We could be smarter here: resync platform timer with local timer?
797 */
798 if ( ((s64)stime_elapsed64 < (EPOCH / 2)) )
799 goto out;
801 /*
802 * Calculate error-correction factor. This only slows down a fast local
803 * clock (slow clocks are warped forwards). The scale factor is clamped
804 * to >= 0.5.
805 */
806 if ( curr_local_stime != curr_master_stime )
807 {
808 local_stime_err = curr_local_stime - curr_master_stime;
809 if ( local_stime_err > EPOCH )
810 local_stime_err = EPOCH;
811 error_factor = div_frac(EPOCH, EPOCH + (u32)local_stime_err);
812 }
814 /*
815 * We require 0 < stime_elapsed < 2^31.
816 * This allows us to binary shift a 32-bit tsc_elapsed such that:
817 * stime_elapsed < tsc_elapsed <= 2*stime_elapsed
818 */
819 while ( ((u32)stime_elapsed64 != stime_elapsed64) ||
820 ((s32)stime_elapsed64 < 0) )
821 {
822 stime_elapsed64 >>= 1;
823 tsc_elapsed64 >>= 1;
824 }
826 /* stime_master_diff now fits in a 32-bit word. */
827 stime_elapsed32 = (u32)stime_elapsed64;
829 /* tsc_elapsed <= 2*stime_elapsed */
830 while ( tsc_elapsed64 > (stime_elapsed32 * 2) )
831 {
832 tsc_elapsed64 >>= 1;
833 tsc_shift--;
834 }
836 /* Local difference must now fit in 32 bits. */
837 ASSERT((u32)tsc_elapsed64 == tsc_elapsed64);
838 tsc_elapsed32 = (u32)tsc_elapsed64;
840 /* tsc_elapsed > stime_elapsed */
841 ASSERT(tsc_elapsed32 != 0);
842 while ( tsc_elapsed32 <= stime_elapsed32 )
843 {
844 tsc_elapsed32 <<= 1;
845 tsc_shift++;
846 }
848 calibration_mul_frac = div_frac(stime_elapsed32, tsc_elapsed32);
849 if ( error_factor != 0 )
850 calibration_mul_frac = mul_frac(calibration_mul_frac, error_factor);
852 #if 0
853 printk("---%d: %08x %08x %d\n", cpu,
854 error_factor, calibration_mul_frac, tsc_shift);
855 #endif
857 /* Record new timestamp information. */
858 cpu_time[cpu].tsc_scale.mul_frac = calibration_mul_frac;
859 cpu_time[cpu].tsc_scale.shift = tsc_shift;
860 cpu_time[cpu].local_tsc_stamp = curr_tsc;
861 cpu_time[cpu].stime_local_stamp = curr_local_stime;
862 cpu_time[cpu].stime_master_stamp = curr_master_stime;
864 out:
865 set_ac_timer(&cpu_time[cpu].calibration_timer, NOW() + EPOCH);
867 if ( cpu == 0 )
868 platform_time_calibration();
869 }
871 void init_percpu_time(void)
872 {
873 unsigned int cpu = smp_processor_id();
874 unsigned long flags;
875 s_time_t now;
877 local_irq_save(flags);
878 rdtscll(cpu_time[cpu].local_tsc_stamp);
879 now = (cpu == 0) ? 0 : read_platform_stime();
880 local_irq_restore(flags);
882 cpu_time[cpu].stime_master_stamp = now;
883 cpu_time[cpu].stime_local_stamp = now;
885 init_ac_timer(&cpu_time[cpu].calibration_timer,
886 local_time_calibration, NULL, cpu);
887 set_ac_timer(&cpu_time[cpu].calibration_timer, NOW() + EPOCH);
888 }
890 /* Late init function (after all CPUs are booted). */
891 int __init init_xen_time(void)
892 {
893 wc_sec = get_cmos_time();
895 local_irq_disable();
897 init_percpu_time();
899 stime_platform_stamp = 0;
900 init_platform_timer();
902 local_irq_enable();
904 return 0;
905 }
908 /* Early init function. */
909 void __init early_time_init(void)
910 {
911 u64 tmp = calibrate_boot_tsc();
913 set_time_scale(&cpu_time[0].tsc_scale, tmp);
915 do_div(tmp, 1000);
916 cpu_khz = (unsigned long)tmp;
917 printk("Detected %lu.%03lu MHz processor.\n",
918 cpu_khz / 1000, cpu_khz % 1000);
920 setup_irq(0, &irq0);
921 }
923 /*
924 * Local variables:
925 * mode: C
926 * c-set-style: "BSD"
927 * c-basic-offset: 4
928 * tab-width: 4
929 * indent-tabs-mode: nil
930 * End:
931 */