ia64/xen-unstable

view xen/arch/x86/time.c @ 16098:f6a06b2eefc5

x86: Fix time-scale calculation to avoid #DE fault.
Signed-off-by: Keir Fraser <keir@xensource.com>
author Keir Fraser <keir@xensource.com>
date Thu Oct 11 19:23:40 2007 +0100 (2007-10-11)
parents 2d761ca771fb
children 4fcc8b64c2b5
line source
1 /******************************************************************************
2 * arch/x86/time.c
3 *
4 * Per-CPU time calibration and management.
5 *
6 * Copyright (c) 2002-2005, K A Fraser
7 *
8 * Portions from Linux are:
9 * Copyright (c) 1991, 1992, 1995 Linus Torvalds
10 */
12 #include <xen/config.h>
13 #include <xen/errno.h>
14 #include <xen/event.h>
15 #include <xen/sched.h>
16 #include <xen/lib.h>
17 #include <xen/config.h>
18 #include <xen/init.h>
19 #include <xen/time.h>
20 #include <xen/timer.h>
21 #include <xen/smp.h>
22 #include <xen/irq.h>
23 #include <xen/softirq.h>
24 #include <asm/io.h>
25 #include <asm/msr.h>
26 #include <asm/mpspec.h>
27 #include <asm/processor.h>
28 #include <asm/fixmap.h>
29 #include <asm/mc146818rtc.h>
30 #include <asm/div64.h>
31 #include <asm/hpet.h>
32 #include <io_ports.h>
34 /* opt_clocksource: Force clocksource to one of: pit, hpet, cyclone, acpi. */
35 static char opt_clocksource[10];
36 string_param("clocksource", opt_clocksource);
38 #define EPOCH MILLISECS(1000)
40 unsigned long cpu_khz; /* CPU clock frequency in kHz. */
41 unsigned long hpet_address;
42 DEFINE_SPINLOCK(rtc_lock);
43 volatile unsigned long jiffies;
44 static u32 wc_sec, wc_nsec; /* UTC time at last 'time update'. */
45 static DEFINE_SPINLOCK(wc_lock);
47 struct time_scale {
48 int shift;
49 u32 mul_frac;
50 };
52 struct cpu_time {
53 u64 local_tsc_stamp;
54 s_time_t stime_local_stamp;
55 s_time_t stime_master_stamp;
56 struct time_scale tsc_scale;
57 struct timer calibration_timer;
58 };
60 struct platform_timesource {
61 char *name;
62 u64 frequency;
63 u32 (*read_counter)(void);
64 int counter_bits;
65 };
67 static DEFINE_PER_CPU(struct cpu_time, cpu_time);
69 /*
70 * Protected by platform_timer_lock, which must be acquired with interrupts
71 * disabled because plt_overflow() is called from PIT ch0 interrupt context.
72 */
73 static s_time_t stime_platform_stamp;
74 static u64 platform_timer_stamp;
75 static DEFINE_SPINLOCK(platform_timer_lock);
77 /*
78 * Folding platform timer into 64-bit software counter is a really critical
79 * operation! We therefore do it directly in PIT ch0 interrupt handler.
80 */
81 static u32 plt_overflow_jiffies;
82 static void plt_overflow(void);
84 /*
85 * 32-bit division of integer dividend and integer divisor yielding
86 * 32-bit fractional quotient.
87 */
88 static inline u32 div_frac(u32 dividend, u32 divisor)
89 {
90 u32 quotient, remainder;
91 ASSERT(dividend < divisor);
92 asm (
93 "divl %4"
94 : "=a" (quotient), "=d" (remainder)
95 : "0" (0), "1" (dividend), "r" (divisor) );
96 return quotient;
97 }
99 /*
100 * 32-bit multiplication of multiplicand and fractional multiplier
101 * yielding 32-bit product (radix point at same position as in multiplicand).
102 */
103 static inline u32 mul_frac(u32 multiplicand, u32 multiplier)
104 {
105 u32 product_int, product_frac;
106 asm (
107 "mul %3"
108 : "=a" (product_frac), "=d" (product_int)
109 : "0" (multiplicand), "r" (multiplier) );
110 return product_int;
111 }
113 /*
114 * Scale a 64-bit delta by scaling and multiplying by a 32-bit fraction,
115 * yielding a 64-bit result.
116 */
117 static inline u64 scale_delta(u64 delta, struct time_scale *scale)
118 {
119 u64 product;
120 #ifdef CONFIG_X86_32
121 u32 tmp1, tmp2;
122 #endif
124 if ( scale->shift < 0 )
125 delta >>= -scale->shift;
126 else
127 delta <<= scale->shift;
129 #ifdef CONFIG_X86_32
130 asm (
131 "mul %5 ; "
132 "mov %4,%%eax ; "
133 "mov %%edx,%4 ; "
134 "mul %5 ; "
135 "xor %5,%5 ; "
136 "add %4,%%eax ; "
137 "adc %5,%%edx ; "
138 : "=A" (product), "=r" (tmp1), "=r" (tmp2)
139 : "a" ((u32)delta), "1" ((u32)(delta >> 32)), "2" (scale->mul_frac) );
140 #else
141 asm (
142 "mul %%rdx ; shrd $32,%%rdx,%%rax"
143 : "=a" (product) : "0" (delta), "d" ((u64)scale->mul_frac) );
144 #endif
146 return product;
147 }
149 void timer_interrupt(int irq, void *dev_id, struct cpu_user_regs *regs)
150 {
151 ASSERT(local_irq_is_enabled());
153 /* Update jiffies counter. */
154 (*(volatile unsigned long *)&jiffies)++;
156 /* Rough hack to allow accurate timers to sort-of-work with no APIC. */
157 if ( !cpu_has_apic )
158 raise_softirq(TIMER_SOFTIRQ);
160 if ( --plt_overflow_jiffies == 0 )
161 plt_overflow();
162 }
164 static struct irqaction irq0 = { timer_interrupt, "timer", NULL};
166 /* ------ Calibrate the TSC -------
167 * Return processor ticks per second / CALIBRATE_FRAC.
168 */
170 #define CLOCK_TICK_RATE 1193180 /* system crystal frequency (Hz) */
171 #define CALIBRATE_FRAC 20 /* calibrate over 50ms */
172 #define CALIBRATE_LATCH ((CLOCK_TICK_RATE+(CALIBRATE_FRAC/2))/CALIBRATE_FRAC)
174 static u64 init_pit_and_calibrate_tsc(void)
175 {
176 u64 start, end;
177 unsigned long count;
179 /* Set PIT channel 0 to HZ Hz. */
180 #define LATCH (((CLOCK_TICK_RATE)+(HZ/2))/HZ)
181 outb_p(0x34, PIT_MODE); /* binary, mode 2, LSB/MSB, ch 0 */
182 outb_p(LATCH & 0xff, PIT_CH0); /* LSB */
183 outb(LATCH >> 8, PIT_CH0); /* MSB */
185 /* Set the Gate high, disable speaker */
186 outb((inb(0x61) & ~0x02) | 0x01, 0x61);
188 /*
189 * Now let's take care of CTC channel 2
190 *
191 * Set the Gate high, program CTC channel 2 for mode 0, (interrupt on
192 * terminal count mode), binary count, load 5 * LATCH count, (LSB and MSB)
193 * to begin countdown.
194 */
195 outb(0xb0, PIT_MODE); /* binary, mode 0, LSB/MSB, Ch 2 */
196 outb(CALIBRATE_LATCH & 0xff, PIT_CH2); /* LSB of count */
197 outb(CALIBRATE_LATCH >> 8, PIT_CH2); /* MSB of count */
199 rdtscll(start);
200 for ( count = 0; (inb(0x61) & 0x20) == 0; count++ )
201 continue;
202 rdtscll(end);
204 /* Error if the CTC doesn't behave itself. */
205 if ( count == 0 )
206 return 0;
208 return ((end - start) * (u64)CALIBRATE_FRAC);
209 }
211 static void set_time_scale(struct time_scale *ts, u64 ticks_per_sec)
212 {
213 u64 tps64 = ticks_per_sec;
214 u32 tps32;
215 int shift = 0;
217 ASSERT(tps64 != 0);
219 while ( tps64 > (MILLISECS(1000)*2) )
220 {
221 tps64 >>= 1;
222 shift--;
223 }
225 tps32 = (u32)tps64;
226 while ( tps32 <= (u32)MILLISECS(1000) )
227 {
228 tps32 <<= 1;
229 shift++;
230 }
232 ts->mul_frac = div_frac(MILLISECS(1000), tps32);
233 ts->shift = shift;
234 }
236 static atomic_t tsc_calibrate_gang = ATOMIC_INIT(0);
237 static unsigned int tsc_calibrate_status = 0;
239 void calibrate_tsc_bp(void)
240 {
241 while ( atomic_read(&tsc_calibrate_gang) != (num_booting_cpus() - 1) )
242 mb();
244 outb(CALIBRATE_LATCH & 0xff, PIT_CH2);
245 outb(CALIBRATE_LATCH >> 8, PIT_CH2);
247 tsc_calibrate_status = 1;
248 wmb();
250 while ( (inb(0x61) & 0x20) == 0 )
251 continue;
253 tsc_calibrate_status = 2;
254 wmb();
256 while ( atomic_read(&tsc_calibrate_gang) != 0 )
257 mb();
258 }
260 void calibrate_tsc_ap(void)
261 {
262 u64 t1, t2, ticks_per_sec;
264 atomic_inc(&tsc_calibrate_gang);
266 while ( tsc_calibrate_status < 1 )
267 mb();
269 rdtscll(t1);
271 while ( tsc_calibrate_status < 2 )
272 mb();
274 rdtscll(t2);
276 ticks_per_sec = (t2 - t1) * (u64)CALIBRATE_FRAC;
277 set_time_scale(&this_cpu(cpu_time).tsc_scale, ticks_per_sec);
279 atomic_dec(&tsc_calibrate_gang);
280 }
282 static char *freq_string(u64 freq)
283 {
284 static char s[20];
285 unsigned int x, y;
286 y = (unsigned int)do_div(freq, 1000000) / 1000;
287 x = (unsigned int)freq;
288 snprintf(s, sizeof(s), "%u.%03uMHz", x, y);
289 return s;
290 }
292 /************************************************************
293 * PLATFORM TIMER 1: PROGRAMMABLE INTERVAL TIMER (LEGACY PIT)
294 */
296 static u32 read_pit_count(void)
297 {
298 u16 count;
299 ASSERT(spin_is_locked(&platform_timer_lock));
300 outb(0x80, PIT_MODE);
301 count = inb(PIT_CH2);
302 count |= inb(PIT_CH2) << 8;
303 return ~count;
304 }
306 static void init_pit(struct platform_timesource *pts)
307 {
308 pts->name = "PIT";
309 pts->frequency = CLOCK_TICK_RATE;
310 pts->read_counter = read_pit_count;
311 pts->counter_bits = 16;
312 }
314 /************************************************************
315 * PLATFORM TIMER 2: HIGH PRECISION EVENT TIMER (HPET)
316 */
318 static u32 read_hpet_count(void)
319 {
320 return hpet_read32(HPET_COUNTER);
321 }
323 static int init_hpet(struct platform_timesource *pts)
324 {
325 u64 hpet_rate;
326 u32 hpet_id, hpet_period, cfg;
327 int i;
329 if ( hpet_address == 0 )
330 return 0;
332 set_fixmap_nocache(FIX_HPET_BASE, hpet_address);
334 hpet_id = hpet_read32(HPET_ID);
335 if ( hpet_id == 0 )
336 {
337 printk("BAD HPET vendor id.\n");
338 return 0;
339 }
341 /* Check for sane period (100ps <= period <= 100ns). */
342 hpet_period = hpet_read32(HPET_PERIOD);
343 if ( (hpet_period > 100000000) || (hpet_period < 100000) )
344 {
345 printk("BAD HPET period %u.\n", hpet_period);
346 return 0;
347 }
349 cfg = hpet_read32(HPET_CFG);
350 cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY);
351 hpet_write32(cfg, HPET_CFG);
353 for ( i = 0; i <= ((hpet_id >> 8) & 31); i++ )
354 {
355 cfg = hpet_read32(HPET_T0_CFG + i*0x20);
356 cfg &= ~HPET_TN_ENABLE;
357 hpet_write32(cfg & ~HPET_TN_ENABLE, HPET_T0_CFG);
358 }
360 cfg = hpet_read32(HPET_CFG);
361 cfg |= HPET_CFG_ENABLE;
362 hpet_write32(cfg, HPET_CFG);
364 hpet_rate = 1000000000000000ULL; /* 10^15 */
365 (void)do_div(hpet_rate, hpet_period);
367 pts->name = "HPET";
368 pts->frequency = hpet_rate;
369 pts->read_counter = read_hpet_count;
370 pts->counter_bits = 32;
372 return 1;
373 }
375 /************************************************************
376 * PLATFORM TIMER 3: IBM 'CYCLONE' TIMER
377 */
379 int use_cyclone;
381 /*
382 * Although the counter is read via a 64-bit register, I believe it is actually
383 * a 40-bit counter. Since this will wrap, I read only the low 32 bits and
384 * periodically fold into a 64-bit software counter, just as for PIT and HPET.
385 */
386 #define CYCLONE_CBAR_ADDR 0xFEB00CD0
387 #define CYCLONE_PMCC_OFFSET 0x51A0
388 #define CYCLONE_MPMC_OFFSET 0x51D0
389 #define CYCLONE_MPCS_OFFSET 0x51A8
390 #define CYCLONE_TIMER_FREQ 100000000
392 /* Cyclone MPMC0 register. */
393 static volatile u32 *cyclone_timer;
395 static u32 read_cyclone_count(void)
396 {
397 return *cyclone_timer;
398 }
400 static volatile u32 *map_cyclone_reg(unsigned long regaddr)
401 {
402 unsigned long pageaddr = regaddr & PAGE_MASK;
403 unsigned long offset = regaddr & ~PAGE_MASK;
404 set_fixmap_nocache(FIX_CYCLONE_TIMER, pageaddr);
405 return (volatile u32 *)(fix_to_virt(FIX_CYCLONE_TIMER) + offset);
406 }
408 static int init_cyclone(struct platform_timesource *pts)
409 {
410 u32 base;
412 if ( !use_cyclone )
413 return 0;
415 /* Find base address. */
416 base = *(map_cyclone_reg(CYCLONE_CBAR_ADDR));
417 if ( base == 0 )
418 {
419 printk(KERN_ERR "Cyclone: Could not find valid CBAR value.\n");
420 return 0;
421 }
423 /* Enable timer and map the counter register. */
424 *(map_cyclone_reg(base + CYCLONE_PMCC_OFFSET)) = 1;
425 *(map_cyclone_reg(base + CYCLONE_MPCS_OFFSET)) = 1;
426 cyclone_timer = map_cyclone_reg(base + CYCLONE_MPMC_OFFSET);
428 pts->name = "IBM Cyclone";
429 pts->frequency = CYCLONE_TIMER_FREQ;
430 pts->read_counter = read_cyclone_count;
431 pts->counter_bits = 32;
433 return 1;
434 }
436 /************************************************************
437 * PLATFORM TIMER 4: ACPI PM TIMER
438 */
440 u32 pmtmr_ioport;
442 /* ACPI PM timer ticks at 3.579545 MHz. */
443 #define ACPI_PM_FREQUENCY 3579545
445 static u32 read_pmtimer_count(void)
446 {
447 return inl(pmtmr_ioport);
448 }
450 static int init_pmtimer(struct platform_timesource *pts)
451 {
452 if ( pmtmr_ioport == 0 )
453 return 0;
455 pts->name = "ACPI PM Timer";
456 pts->frequency = ACPI_PM_FREQUENCY;
457 pts->read_counter = read_pmtimer_count;
458 pts->counter_bits = 24;
460 return 1;
461 }
463 /************************************************************
464 * GENERIC PLATFORM TIMER INFRASTRUCTURE
465 */
467 static struct platform_timesource plt_src; /* details of chosen timesource */
468 static u32 plt_mask; /* hardware-width mask */
469 static u32 plt_overflow_period; /* jiffies between calls to plt_overflow() */
470 static struct time_scale plt_scale; /* scale: platform counter -> nanosecs */
472 /* Protected by platform_timer_lock. */
473 static u64 plt_count64; /* 64-bit platform counter stamp */
474 static u32 plt_count; /* hardware-width platform counter stamp */
476 static void plt_overflow(void)
477 {
478 u32 count;
479 unsigned long flags;
481 spin_lock_irqsave(&platform_timer_lock, flags);
482 count = plt_src.read_counter();
483 plt_count64 += (count - plt_count) & plt_mask;
484 plt_count = count;
485 plt_overflow_jiffies = plt_overflow_period;
486 spin_unlock_irqrestore(&platform_timer_lock, flags);
487 }
489 static s_time_t __read_platform_stime(u64 platform_time)
490 {
491 u64 diff = platform_time - platform_timer_stamp;
492 ASSERT(spin_is_locked(&platform_timer_lock));
493 return (stime_platform_stamp + scale_delta(diff, &plt_scale));
494 }
496 static s_time_t read_platform_stime(void)
497 {
498 u64 count;
499 s_time_t stime;
500 unsigned long flags;
502 spin_lock_irqsave(&platform_timer_lock, flags);
503 count = plt_count64 + ((plt_src.read_counter() - plt_count) & plt_mask);
504 stime = __read_platform_stime(count);
505 spin_unlock_irqrestore(&platform_timer_lock, flags);
507 return stime;
508 }
510 static void platform_time_calibration(void)
511 {
512 u64 count;
513 s_time_t stamp;
514 unsigned long flags;
516 spin_lock_irqsave(&platform_timer_lock, flags);
517 count = plt_count64 + ((plt_src.read_counter() - plt_count) & plt_mask);
518 stamp = __read_platform_stime(count);
519 stime_platform_stamp = stamp;
520 platform_timer_stamp = count;
521 spin_unlock_irqrestore(&platform_timer_lock, flags);
522 }
524 static void resume_platform_timer(void)
525 {
526 /* No change in platform_stime across suspend/resume. */
527 platform_timer_stamp = plt_count64;
528 plt_count = plt_src.read_counter();
529 }
531 static void init_platform_timer(void)
532 {
533 struct platform_timesource *pts = &plt_src;
534 u64 overflow_period;
535 int rc = -1;
537 if ( opt_clocksource[0] != '\0' )
538 {
539 if ( !strcmp(opt_clocksource, "pit") )
540 rc = (init_pit(pts), 1);
541 else if ( !strcmp(opt_clocksource, "hpet") )
542 rc = init_hpet(pts);
543 else if ( !strcmp(opt_clocksource, "cyclone") )
544 rc = init_cyclone(pts);
545 else if ( !strcmp(opt_clocksource, "acpi") )
546 rc = init_pmtimer(pts);
548 if ( rc <= 0 )
549 printk("WARNING: %s clocksource '%s'.\n",
550 (rc == 0) ? "Could not initialise" : "Unrecognised",
551 opt_clocksource);
552 }
554 if ( (rc <= 0) &&
555 !init_cyclone(pts) &&
556 !init_hpet(pts) &&
557 !init_pmtimer(pts) )
558 init_pit(pts);
560 plt_mask = (u32)~0u >> (32 - pts->counter_bits);
562 set_time_scale(&plt_scale, pts->frequency);
564 overflow_period = scale_delta(1ull << (pts->counter_bits-1), &plt_scale);
565 do_div(overflow_period, MILLISECS(1000/HZ));
566 plt_overflow_period = overflow_period;
567 plt_overflow();
568 printk("Platform timer overflows in %d jiffies.\n", plt_overflow_period);
570 platform_timer_stamp = plt_count64;
572 printk("Platform timer is %s %s\n",
573 freq_string(pts->frequency), pts->name);
574 }
577 /***************************************************************************
578 * CMOS Timer functions
579 ***************************************************************************/
581 /* Converts Gregorian date to seconds since 1970-01-01 00:00:00.
582 * Assumes input in normal date format, i.e. 1980-12-31 23:59:59
583 * => year=1980, mon=12, day=31, hour=23, min=59, sec=59.
584 *
585 * [For the Julian calendar (which was used in Russia before 1917,
586 * Britain & colonies before 1752, anywhere else before 1582,
587 * and is still in use by some communities) leave out the
588 * -year/100+year/400 terms, and add 10.]
589 *
590 * This algorithm was first published by Gauss (I think).
591 *
592 * WARNING: this function will overflow on 2106-02-07 06:28:16 on
593 * machines were long is 32-bit! (However, as time_t is signed, we
594 * will already get problems at other places on 2038-01-19 03:14:08)
595 */
596 unsigned long
597 mktime (unsigned int year, unsigned int mon,
598 unsigned int day, unsigned int hour,
599 unsigned int min, unsigned int sec)
600 {
601 /* 1..12 -> 11,12,1..10: put Feb last since it has a leap day. */
602 if ( 0 >= (int) (mon -= 2) )
603 {
604 mon += 12;
605 year -= 1;
606 }
608 return ((((unsigned long)(year/4 - year/100 + year/400 + 367*mon/12 + day)+
609 year*365 - 719499
610 )*24 + hour /* now have hours */
611 )*60 + min /* now have minutes */
612 )*60 + sec; /* finally seconds */
613 }
615 static unsigned long __get_cmos_time(void)
616 {
617 unsigned int year, mon, day, hour, min, sec;
619 sec = CMOS_READ(RTC_SECONDS);
620 min = CMOS_READ(RTC_MINUTES);
621 hour = CMOS_READ(RTC_HOURS);
622 day = CMOS_READ(RTC_DAY_OF_MONTH);
623 mon = CMOS_READ(RTC_MONTH);
624 year = CMOS_READ(RTC_YEAR);
626 if ( !(CMOS_READ(RTC_CONTROL) & RTC_DM_BINARY) || RTC_ALWAYS_BCD )
627 {
628 BCD_TO_BIN(sec);
629 BCD_TO_BIN(min);
630 BCD_TO_BIN(hour);
631 BCD_TO_BIN(day);
632 BCD_TO_BIN(mon);
633 BCD_TO_BIN(year);
634 }
636 if ( (year += 1900) < 1970 )
637 year += 100;
639 return mktime(year, mon, day, hour, min, sec);
640 }
642 static unsigned long get_cmos_time(void)
643 {
644 unsigned long res, flags;
645 int i;
647 spin_lock_irqsave(&rtc_lock, flags);
649 /* read RTC exactly on falling edge of update flag */
650 for ( i = 0 ; i < 1000000 ; i++ ) /* may take up to 1 second... */
651 if ( (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) )
652 break;
653 for ( i = 0 ; i < 1000000 ; i++ ) /* must try at least 2.228 ms */
654 if ( !(CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) )
655 break;
657 res = __get_cmos_time();
659 spin_unlock_irqrestore(&rtc_lock, flags);
660 return res;
661 }
663 /***************************************************************************
664 * System Time
665 ***************************************************************************/
667 s_time_t get_s_time(void)
668 {
669 struct cpu_time *t = &this_cpu(cpu_time);
670 u64 tsc, delta;
671 s_time_t now;
673 rdtscll(tsc);
674 delta = tsc - t->local_tsc_stamp;
675 now = t->stime_local_stamp + scale_delta(delta, &t->tsc_scale);
677 return now;
678 }
680 static inline void version_update_begin(u32 *version)
681 {
682 /* Explicitly OR with 1 just in case version number gets out of sync. */
683 *version = (*version + 1) | 1;
684 wmb();
685 }
687 static inline void version_update_end(u32 *version)
688 {
689 wmb();
690 (*version)++;
691 }
693 void update_vcpu_system_time(struct vcpu *v)
694 {
695 struct cpu_time *t;
696 struct vcpu_time_info *u;
698 if ( v->vcpu_info == NULL )
699 return;
701 t = &this_cpu(cpu_time);
702 u = &vcpu_info(v, time);
704 if ( u->tsc_timestamp == t->local_tsc_stamp )
705 return;
707 version_update_begin(&u->version);
709 u->tsc_timestamp = t->local_tsc_stamp;
710 u->system_time = t->stime_local_stamp;
711 u->tsc_to_system_mul = t->tsc_scale.mul_frac;
712 u->tsc_shift = (s8)t->tsc_scale.shift;
714 version_update_end(&u->version);
715 }
717 void update_domain_wallclock_time(struct domain *d)
718 {
719 spin_lock(&wc_lock);
720 version_update_begin(&shared_info(d, wc_version));
721 shared_info(d, wc_sec) = wc_sec + d->time_offset_seconds;
722 shared_info(d, wc_nsec) = wc_nsec;
723 version_update_end(&shared_info(d, wc_version));
724 spin_unlock(&wc_lock);
725 }
727 int cpu_frequency_change(u64 freq)
728 {
729 struct cpu_time *t = &this_cpu(cpu_time);
730 u64 curr_tsc;
732 local_irq_disable();
733 rdtscll(curr_tsc);
734 t->local_tsc_stamp = curr_tsc;
735 t->stime_local_stamp = get_s_time();
736 t->stime_master_stamp = read_platform_stime();
737 set_time_scale(&t->tsc_scale, freq);
738 local_irq_enable();
740 /* A full epoch should pass before we check for deviation. */
741 set_timer(&t->calibration_timer, NOW() + EPOCH);
742 if ( smp_processor_id() == 0 )
743 platform_time_calibration();
745 return 0;
746 }
748 /* Set clock to <secs,usecs> after 00:00:00 UTC, 1 January, 1970. */
749 void do_settime(unsigned long secs, unsigned long nsecs, u64 system_time_base)
750 {
751 u64 x;
752 u32 y, _wc_sec, _wc_nsec;
753 struct domain *d;
755 x = (secs * 1000000000ULL) + (u64)nsecs - system_time_base;
756 y = do_div(x, 1000000000);
758 spin_lock(&wc_lock);
759 wc_sec = _wc_sec = (u32)x;
760 wc_nsec = _wc_nsec = (u32)y;
761 spin_unlock(&wc_lock);
763 rcu_read_lock(&domlist_read_lock);
764 for_each_domain ( d )
765 update_domain_wallclock_time(d);
766 rcu_read_unlock(&domlist_read_lock);
767 }
769 static void local_time_calibration(void *unused)
770 {
771 struct cpu_time *t = &this_cpu(cpu_time);
773 /*
774 * System timestamps, extrapolated from local and master oscillators,
775 * taken during this calibration and the previous calibration.
776 */
777 s_time_t prev_local_stime, curr_local_stime;
778 s_time_t prev_master_stime, curr_master_stime;
780 /* TSC timestamps taken during this calibration and prev calibration. */
781 u64 prev_tsc, curr_tsc;
783 /*
784 * System time and TSC ticks elapsed during the previous calibration
785 * 'epoch'. These values are down-shifted to fit in 32 bits.
786 */
787 u64 stime_elapsed64, tsc_elapsed64;
788 u32 stime_elapsed32, tsc_elapsed32;
790 /* The accumulated error in the local estimate. */
791 u64 local_stime_err;
793 /* Error correction to slow down a fast local clock. */
794 u32 error_factor = 0;
796 /* Calculated TSC shift to ensure 32-bit scale multiplier. */
797 int tsc_shift = 0;
799 /* The overall calibration scale multiplier. */
800 u32 calibration_mul_frac;
802 prev_tsc = t->local_tsc_stamp;
803 prev_local_stime = t->stime_local_stamp;
804 prev_master_stime = t->stime_master_stamp;
806 /* Disable IRQs to get 'instantaneous' current timestamps. */
807 local_irq_disable();
808 rdtscll(curr_tsc);
809 curr_local_stime = get_s_time();
810 curr_master_stime = read_platform_stime();
811 local_irq_enable();
813 #if 0
814 printk("PRE%d: tsc=%"PRIu64" stime=%"PRIu64" master=%"PRIu64"\n",
815 smp_processor_id(), prev_tsc, prev_local_stime, prev_master_stime);
816 printk("CUR%d: tsc=%"PRIu64" stime=%"PRIu64" master=%"PRIu64
817 " -> %"PRId64"\n",
818 smp_processor_id(), curr_tsc, curr_local_stime, curr_master_stime,
819 curr_master_stime - curr_local_stime);
820 #endif
822 /* Local time warps forward if it lags behind master time. */
823 if ( curr_local_stime < curr_master_stime )
824 curr_local_stime = curr_master_stime;
826 stime_elapsed64 = curr_master_stime - prev_master_stime;
827 tsc_elapsed64 = curr_tsc - prev_tsc;
829 /*
830 * Weirdness can happen if we lose sync with the platform timer.
831 * We could be smarter here: resync platform timer with local timer?
832 */
833 if ( ((s64)stime_elapsed64 < (EPOCH / 2)) )
834 goto out;
836 /*
837 * Calculate error-correction factor. This only slows down a fast local
838 * clock (slow clocks are warped forwards). The scale factor is clamped
839 * to >= 0.5.
840 */
841 if ( curr_local_stime != curr_master_stime )
842 {
843 local_stime_err = curr_local_stime - curr_master_stime;
844 if ( local_stime_err > EPOCH )
845 local_stime_err = EPOCH;
846 error_factor = div_frac(EPOCH, EPOCH + (u32)local_stime_err);
847 }
849 /*
850 * We require 0 < stime_elapsed < 2^31.
851 * This allows us to binary shift a 32-bit tsc_elapsed such that:
852 * stime_elapsed < tsc_elapsed <= 2*stime_elapsed
853 */
854 while ( ((u32)stime_elapsed64 != stime_elapsed64) ||
855 ((s32)stime_elapsed64 < 0) )
856 {
857 stime_elapsed64 >>= 1;
858 tsc_elapsed64 >>= 1;
859 }
861 /* stime_master_diff now fits in a 32-bit word. */
862 stime_elapsed32 = (u32)stime_elapsed64;
864 /* tsc_elapsed <= 2*stime_elapsed */
865 while ( tsc_elapsed64 > (stime_elapsed32 * 2) )
866 {
867 tsc_elapsed64 >>= 1;
868 tsc_shift--;
869 }
871 /* Local difference must now fit in 32 bits. */
872 ASSERT((u32)tsc_elapsed64 == tsc_elapsed64);
873 tsc_elapsed32 = (u32)tsc_elapsed64;
875 /* tsc_elapsed > stime_elapsed */
876 ASSERT(tsc_elapsed32 != 0);
877 while ( tsc_elapsed32 <= stime_elapsed32 )
878 {
879 tsc_elapsed32 <<= 1;
880 tsc_shift++;
881 }
883 calibration_mul_frac = div_frac(stime_elapsed32, tsc_elapsed32);
884 if ( error_factor != 0 )
885 calibration_mul_frac = mul_frac(calibration_mul_frac, error_factor);
887 #if 0
888 printk("---%d: %08x %08x %d\n", smp_processor_id(),
889 error_factor, calibration_mul_frac, tsc_shift);
890 #endif
892 /* Record new timestamp information, atomically w.r.t. interrupts. */
893 local_irq_disable();
894 t->tsc_scale.mul_frac = calibration_mul_frac;
895 t->tsc_scale.shift = tsc_shift;
896 t->local_tsc_stamp = curr_tsc;
897 t->stime_local_stamp = curr_local_stime;
898 t->stime_master_stamp = curr_master_stime;
899 local_irq_enable();
901 update_vcpu_system_time(current);
903 out:
904 set_timer(&t->calibration_timer, NOW() + EPOCH);
906 if ( smp_processor_id() == 0 )
907 platform_time_calibration();
908 }
910 void init_percpu_time(void)
911 {
912 struct cpu_time *t = &this_cpu(cpu_time);
913 unsigned long flags;
914 s_time_t now;
916 local_irq_save(flags);
917 rdtscll(t->local_tsc_stamp);
918 now = !plt_src.read_counter ? 0 : read_platform_stime();
919 local_irq_restore(flags);
921 t->stime_master_stamp = now;
922 t->stime_local_stamp = now;
924 init_timer(&t->calibration_timer, local_time_calibration,
925 NULL, smp_processor_id());
926 set_timer(&t->calibration_timer, NOW() + EPOCH);
927 }
929 /* Late init function (after all CPUs are booted). */
930 int __init init_xen_time(void)
931 {
932 wc_sec = get_cmos_time();
934 local_irq_disable();
936 init_percpu_time();
938 stime_platform_stamp = 0;
939 init_platform_timer();
941 local_irq_enable();
943 return 0;
944 }
947 /* Early init function. */
948 void __init early_time_init(void)
949 {
950 u64 tmp = init_pit_and_calibrate_tsc();
952 set_time_scale(&this_cpu(cpu_time).tsc_scale, tmp);
954 do_div(tmp, 1000);
955 cpu_khz = (unsigned long)tmp;
956 printk("Detected %lu.%03lu MHz processor.\n",
957 cpu_khz / 1000, cpu_khz % 1000);
959 setup_irq(0, &irq0);
960 }
962 void send_timer_event(struct vcpu *v)
963 {
964 send_guest_vcpu_virq(v, VIRQ_TIMER);
965 }
967 /* Return secs after 00:00:00 localtime, 1 January, 1970. */
968 unsigned long get_localtime(struct domain *d)
969 {
970 return wc_sec + (wc_nsec + NOW()) / 1000000000ULL
971 + d->time_offset_seconds;
972 }
974 int time_suspend(void)
975 {
976 /* Better to cancel calibration timer for accuracy. */
977 kill_timer(&this_cpu(cpu_time).calibration_timer);
979 return 0;
980 }
982 int time_resume(void)
983 {
984 u64 tmp = init_pit_and_calibrate_tsc();
986 set_time_scale(&this_cpu(cpu_time).tsc_scale, tmp);
988 resume_platform_timer();
989 do_settime(get_cmos_time(), 0, read_platform_stime());
991 init_percpu_time();
993 if ( !is_idle_vcpu(current) )
994 update_vcpu_system_time(current);
996 return 0;
997 }
999 int dom0_pit_access(struct ioreq *ioreq)
1001 /* Is Xen using Channel 2? Then disallow direct dom0 access. */
1002 if ( plt_src.read_counter == read_pit_count )
1003 return 0;
1005 switch ( ioreq->addr )
1007 case PIT_CH2:
1008 if ( ioreq->dir == IOREQ_READ )
1009 ioreq->data = inb(PIT_CH2);
1010 else
1011 outb(ioreq->data, PIT_CH2);
1012 return 1;
1014 case PIT_MODE:
1015 if ( ioreq->dir == IOREQ_READ )
1016 return 0; /* urk! */
1017 switch ( ioreq->data & 0xc0 )
1019 case 0xc0: /* Read Back */
1020 if ( ioreq->data & 0x08 ) /* Select Channel 2? */
1021 outb(ioreq->data & 0xf8, PIT_MODE);
1022 if ( !(ioreq->data & 0x06) ) /* Select Channel 0/1? */
1023 return 1; /* no - we're done */
1024 /* Filter Channel 2 and reserved bit 0. */
1025 ioreq->data &= ~0x09;
1026 return 0; /* emulate ch0/1 readback */
1027 case 0x80: /* Select Counter 2 */
1028 outb(ioreq->data, PIT_MODE);
1029 return 1;
1032 case 0x61:
1033 if ( ioreq->dir == IOREQ_READ )
1034 ioreq->data = inb(0x61);
1035 else
1036 outb((inb(0x61) & ~3) | (ioreq->data & 3), 0x61);
1037 return 1;
1040 return 0;
1043 struct tm wallclock_time(void)
1045 uint64_t seconds;
1047 if ( !wc_sec )
1048 return (struct tm) { 0 };
1050 seconds = NOW() + (wc_sec * 1000000000ull) + wc_nsec;
1051 do_div(seconds, 1000000000);
1052 return gmtime(seconds);
1055 /*
1056 * Local variables:
1057 * mode: C
1058 * c-set-style: "BSD"
1059 * c-basic-offset: 4
1060 * tab-width: 4
1061 * indent-tabs-mode: nil
1062 * End:
1063 */