ia64/xen-unstable

view xen/arch/x86/hpet.c @ 19493:4da7f15e9126

x86 hpet: two more fixes for c/s 19419

vector_channel[], as its name already says, is vector-, not
irq-indexed.

hpet_assign_irq() sits not only in the boot path, but also in the
resume one. Short of knowing why this is, simply checking whether a
vector was already assigned prevents leaking previously assigned ones.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
author Keir Fraser <keir.fraser@citrix.com>
date Wed Apr 01 14:04:46 2009 +0100 (2009-04-01)
parents 443b3bb740e2
children 573e6673e9e0
line source
1 /******************************************************************************
2 * arch/x86/hpet.c
3 *
4 * HPET management.
5 */
7 #include <xen/config.h>
8 #include <xen/errno.h>
9 #include <xen/time.h>
10 #include <xen/timer.h>
11 #include <xen/smp.h>
12 #include <xen/softirq.h>
13 #include <xen/irq.h>
14 #include <asm/fixmap.h>
15 #include <asm/div64.h>
16 #include <asm/hpet.h>
17 #include <asm/msi.h>
18 #include <mach_apic.h>
20 #define MAX_DELTA_NS MILLISECS(10*1000)
21 #define MIN_DELTA_NS MICROSECS(20)
23 #define MAX_HPET_NUM 32
25 #define HPET_EVT_USED_BIT 2
26 #define HPET_EVT_USED (1 << HPET_EVT_USED_BIT)
28 struct hpet_event_channel
29 {
30 unsigned long mult;
31 int shift;
32 s_time_t next_event;
33 cpumask_t cpumask;
34 spinlock_t lock;
35 void (*event_handler)(struct hpet_event_channel *);
37 unsigned int idx; /* physical channel idx */
38 int cpu; /* msi target */
39 unsigned int vector;/* msi vector */
40 unsigned int flags; /* HPET_EVT_x */
41 } __cacheline_aligned;
42 static struct hpet_event_channel legacy_hpet_event;
43 static struct hpet_event_channel hpet_events[MAX_HPET_NUM];
44 static unsigned int num_hpets_used; /* msi hpet channels used for broadcast */
46 DEFINE_PER_CPU(struct hpet_event_channel *, cpu_bc_channel);
48 static int vector_channel[NR_VECTORS] = {[0 ... NR_VECTORS-1] = -1};
50 #define vector_to_channel(vector) vector_channel[vector]
52 unsigned long hpet_address;
54 void msi_compose_msg(struct pci_dev *pdev, int vector, struct msi_msg *msg);
56 /* force_hpet_broadcast: if true, force using hpet_broadcast to fix lapic stop
57 issue for deep C state with pit disabled */
58 int force_hpet_broadcast;
59 boolean_param("hpetbroadcast", force_hpet_broadcast);
61 /*
62 * Calculate a multiplication factor for scaled math, which is used to convert
63 * nanoseconds based values to clock ticks:
64 *
65 * clock_ticks = (nanoseconds * factor) >> shift.
66 *
67 * div_sc is the rearranged equation to calculate a factor from a given clock
68 * ticks / nanoseconds ratio:
69 *
70 * factor = (clock_ticks << shift) / nanoseconds
71 */
72 static inline unsigned long div_sc(unsigned long ticks, unsigned long nsec,
73 int shift)
74 {
75 uint64_t tmp = ((uint64_t)ticks) << shift;
77 do_div(tmp, nsec);
78 return (unsigned long) tmp;
79 }
81 /*
82 * Convert nanoseconds based values to clock ticks:
83 *
84 * clock_ticks = (nanoseconds * factor) >> shift.
85 */
86 static inline unsigned long ns2ticks(unsigned long nsec, int shift,
87 unsigned long factor)
88 {
89 uint64_t tmp = ((uint64_t)nsec * factor) >> shift;
91 return (unsigned long) tmp;
92 }
94 static int hpet_next_event(unsigned long delta, int timer)
95 {
96 uint32_t cnt, cmp;
97 unsigned long flags;
99 local_irq_save(flags);
100 cnt = hpet_read32(HPET_COUNTER);
101 cmp = cnt + delta;
102 hpet_write32(cmp, HPET_Tn_CMP(timer));
103 cmp = hpet_read32(HPET_COUNTER);
104 local_irq_restore(flags);
106 /* Are we within two ticks of the deadline passing? Then we may miss. */
107 return ((cmp + 2 - cnt) > delta) ? -ETIME : 0;
108 }
110 static int reprogram_hpet_evt_channel(
111 struct hpet_event_channel *ch,
112 s_time_t expire, s_time_t now, int force)
113 {
114 int64_t delta;
115 int ret;
117 if ( unlikely(expire < 0) )
118 {
119 printk(KERN_DEBUG "reprogram: expire < 0\n");
120 return -ETIME;
121 }
123 delta = expire - now;
124 if ( (delta <= 0) && !force )
125 return -ETIME;
127 ch->next_event = expire;
129 if ( expire == STIME_MAX )
130 {
131 /* We assume it will take a long time for the timer to wrap. */
132 hpet_write32(0, HPET_Tn_CMP(ch->idx));
133 return 0;
134 }
136 delta = min_t(int64_t, delta, MAX_DELTA_NS);
137 delta = max_t(int64_t, delta, MIN_DELTA_NS);
138 delta = ns2ticks(delta, ch->shift, ch->mult);
140 ret = hpet_next_event(delta, ch->idx);
141 while ( ret && force )
142 {
143 delta += delta;
144 ret = hpet_next_event(delta, ch->idx);
145 }
147 return ret;
148 }
150 static int evt_do_broadcast(cpumask_t mask)
151 {
152 int ret = 0, cpu = smp_processor_id();
154 if ( cpu_isset(cpu, mask) )
155 {
156 cpu_clear(cpu, mask);
157 raise_softirq(TIMER_SOFTIRQ);
158 ret = 1;
159 }
161 if ( !cpus_empty(mask) )
162 {
163 cpumask_raise_softirq(mask, TIMER_SOFTIRQ);
164 ret = 1;
165 }
166 return ret;
167 }
169 static void handle_hpet_broadcast(struct hpet_event_channel *ch)
170 {
171 cpumask_t mask;
172 s_time_t now, next_event;
173 int cpu;
175 spin_lock_irq(&ch->lock);
177 again:
178 ch->next_event = STIME_MAX;
179 next_event = STIME_MAX;
180 mask = (cpumask_t)CPU_MASK_NONE;
181 now = NOW();
183 /* find all expired events */
184 for_each_cpu_mask(cpu, ch->cpumask)
185 {
186 if ( per_cpu(timer_deadline, cpu) <= now )
187 cpu_set(cpu, mask);
188 else if ( per_cpu(timer_deadline, cpu) < next_event )
189 next_event = per_cpu(timer_deadline, cpu);
190 }
192 /* wakeup the cpus which have an expired event. */
193 evt_do_broadcast(mask);
195 if ( next_event != STIME_MAX )
196 {
197 if ( reprogram_hpet_evt_channel(ch, next_event, now, 0) )
198 goto again;
199 }
200 spin_unlock_irq(&ch->lock);
201 }
203 static void hpet_interrupt_handler(int vector, void *data,
204 struct cpu_user_regs *regs)
205 {
206 struct hpet_event_channel *ch = (struct hpet_event_channel *)data;
207 if ( !ch->event_handler )
208 {
209 printk(XENLOG_WARNING "Spurious HPET timer interrupt on HPET timer %d\n", ch->idx);
210 return;
211 }
213 ch->event_handler(ch);
214 }
216 static void hpet_msi_unmask(unsigned int vector)
217 {
218 unsigned long cfg;
219 int ch_idx = vector_to_channel(vector);
220 struct hpet_event_channel *ch;
222 BUG_ON(ch_idx < 0);
223 ch = &hpet_events[ch_idx];
225 cfg = hpet_read32(HPET_Tn_CFG(ch->idx));
226 cfg |= HPET_TN_FSB;
227 hpet_write32(cfg, HPET_Tn_CFG(ch->idx));
228 }
230 static void hpet_msi_mask(unsigned int vector)
231 {
232 unsigned long cfg;
233 int ch_idx = vector_to_channel(vector);
234 struct hpet_event_channel *ch;
236 BUG_ON(ch_idx < 0);
237 ch = &hpet_events[ch_idx];
239 cfg = hpet_read32(HPET_Tn_CFG(ch->idx));
240 cfg &= ~HPET_TN_FSB;
241 hpet_write32(cfg, HPET_Tn_CFG(ch->idx));
242 }
244 static void hpet_msi_write(unsigned int vector, struct msi_msg *msg)
245 {
246 int ch_idx = vector_to_channel(vector);
247 struct hpet_event_channel *ch;
249 BUG_ON(ch_idx < 0);
250 ch = &hpet_events[ch_idx];
252 hpet_write32(msg->data, HPET_Tn_ROUTE(ch->idx));
253 hpet_write32(msg->address_lo, HPET_Tn_ROUTE(ch->idx) + 4);
254 }
256 static void hpet_msi_read(unsigned int vector, struct msi_msg *msg)
257 {
258 int ch_idx = vector_to_channel(vector);
259 struct hpet_event_channel *ch;
261 BUG_ON(ch_idx < 0);
262 ch = &hpet_events[ch_idx];
264 msg->data = hpet_read32(HPET_Tn_ROUTE(ch->idx));
265 msg->address_lo = hpet_read32(HPET_Tn_ROUTE(ch->idx) + 4);
266 msg->address_hi = 0;
267 }
269 static unsigned int hpet_msi_startup(unsigned int vector)
270 {
271 hpet_msi_unmask(vector);
272 return 0;
273 }
275 static void hpet_msi_shutdown(unsigned int vector)
276 {
277 hpet_msi_mask(vector);
278 }
280 static void hpet_msi_ack(unsigned int vector)
281 {
282 ack_APIC_irq();
283 }
285 static void hpet_msi_end(unsigned int vector)
286 {
287 }
289 static void hpet_msi_set_affinity(unsigned int vector, cpumask_t mask)
290 {
291 struct msi_msg msg;
292 unsigned int dest;
293 cpumask_t tmp;
295 cpus_and(tmp, mask, cpu_online_map);
296 if ( cpus_empty(tmp) )
297 mask = TARGET_CPUS;
299 dest = cpu_mask_to_apicid(mask);
301 hpet_msi_read(vector, &msg);
303 msg.data &= ~MSI_DATA_VECTOR_MASK;
304 msg.data |= MSI_DATA_VECTOR(vector);
305 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
306 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
308 hpet_msi_write(vector, &msg);
309 irq_desc[vector].affinity = mask;
310 }
312 /*
313 * IRQ Chip for MSI HPET Devices,
314 */
315 static struct hw_interrupt_type hpet_msi_type = {
316 .typename = "HPET-MSI",
317 .startup = hpet_msi_startup,
318 .shutdown = hpet_msi_shutdown,
319 .enable = hpet_msi_unmask,
320 .disable = hpet_msi_mask,
321 .ack = hpet_msi_ack,
322 .end = hpet_msi_end,
323 .set_affinity = hpet_msi_set_affinity,
324 };
326 static int hpet_setup_msi_irq(unsigned int vector)
327 {
328 int ret;
329 struct msi_msg msg;
330 struct hpet_event_channel *ch = &hpet_events[vector_to_channel(vector)];
332 irq_desc[vector].handler = &hpet_msi_type;
333 ret = request_irq_vector(vector, hpet_interrupt_handler,
334 0, "HPET", ch);
335 if ( ret < 0 )
336 return ret;
338 msi_compose_msg(NULL, vector, &msg);
339 hpet_msi_write(vector, &msg);
341 return 0;
342 }
344 static int hpet_assign_irq(struct hpet_event_channel *ch)
345 {
346 int vector;
348 if ( ch->vector )
349 return 0;
351 if ( (vector = assign_irq_vector(AUTO_ASSIGN_IRQ)) < 0 )
352 return vector;
354 vector_channel[vector] = ch - &hpet_events[0];
356 if ( hpet_setup_msi_irq(vector) )
357 {
358 free_irq_vector(vector);
359 vector_channel[vector] = -1;
360 return -EINVAL;
361 }
363 ch->vector = vector;
364 return 0;
365 }
367 static int hpet_fsb_cap_lookup(void)
368 {
369 unsigned int id;
370 unsigned int num_chs, num_chs_used;
371 int i;
373 id = hpet_read32(HPET_ID);
375 num_chs = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT);
376 num_chs++; /* Value read out starts from 0 */
378 num_chs_used = 0;
379 for ( i = 0; i < num_chs; i++ )
380 {
381 struct hpet_event_channel *ch = &hpet_events[num_chs_used];
382 unsigned long cfg = hpet_read32(HPET_Tn_CFG(i));
384 /* Only consider HPET timer with MSI support */
385 if ( !(cfg & HPET_TN_FSB_CAP) )
386 continue;
388 ch->flags = 0;
389 ch->idx = i;
391 if ( hpet_assign_irq(ch) )
392 continue;
394 /* set default irq affinity */
395 ch->cpu = num_chs_used;
396 per_cpu(cpu_bc_channel, ch->cpu) = ch;
397 irq_desc[ch->vector].handler->
398 set_affinity(ch->vector, cpumask_of_cpu(ch->cpu));
400 num_chs_used++;
402 if ( num_chs_used == num_possible_cpus() )
403 break;
404 }
406 printk(XENLOG_INFO
407 "HPET: %d timers in total, %d timers will be used for broadcast\n",
408 num_chs, num_chs_used);
410 return num_chs_used;
411 }
413 static int next_channel;
414 static spinlock_t next_lock = SPIN_LOCK_UNLOCKED;
416 static struct hpet_event_channel *hpet_get_channel(int cpu)
417 {
418 int i;
419 int next;
420 struct hpet_event_channel *ch;
422 spin_lock(&next_lock);
423 next = next_channel = (next_channel + 1) % num_hpets_used;
424 spin_unlock(&next_lock);
426 /* try unused channel first */
427 for ( i = next; i < next + num_hpets_used; i++ )
428 {
429 ch = &hpet_events[i % num_hpets_used];
430 if ( !test_and_set_bit(HPET_EVT_USED_BIT, &ch->flags) )
431 {
432 ch->cpu = cpu;
433 return ch;
434 }
435 }
437 /* share a in-use channel */
438 ch = &hpet_events[next];
439 if ( !test_and_set_bit(HPET_EVT_USED_BIT, &ch->flags) )
440 ch->cpu = cpu;
442 return ch;
443 }
445 static void hpet_attach_channel_share(int cpu, struct hpet_event_channel *ch)
446 {
447 per_cpu(cpu_bc_channel, cpu) = ch;
449 /* try to be the channel owner again while holding the lock */
450 if ( !test_and_set_bit(HPET_EVT_USED_BIT, &ch->flags) )
451 ch->cpu = cpu;
453 if ( ch->cpu != cpu )
454 return;
456 /* set irq affinity */
457 irq_desc[ch->vector].handler->
458 set_affinity(ch->vector, cpumask_of_cpu(ch->cpu));
459 }
461 static void hpet_detach_channel_share(int cpu)
462 {
463 struct hpet_event_channel *ch = per_cpu(cpu_bc_channel, cpu);
465 per_cpu(cpu_bc_channel, cpu) = NULL;
467 if ( cpu != ch->cpu )
468 return;
470 if ( cpus_empty(ch->cpumask) )
471 {
472 ch->cpu = -1;
473 clear_bit(HPET_EVT_USED_BIT, &ch->flags);
474 return;
475 }
477 ch->cpu = first_cpu(ch->cpumask);
478 /* set irq affinity */
479 irq_desc[ch->vector].handler->
480 set_affinity(ch->vector, cpumask_of_cpu(ch->cpu));
481 }
483 static void (*hpet_attach_channel)(int cpu, struct hpet_event_channel *ch);
484 static void (*hpet_detach_channel)(int cpu);
486 void hpet_broadcast_init(void)
487 {
488 u64 hpet_rate;
489 u32 hpet_id, cfg;
490 int i;
492 hpet_rate = hpet_setup();
493 if ( hpet_rate == 0 )
494 return;
496 num_hpets_used = hpet_fsb_cap_lookup();
497 if ( num_hpets_used > 0 )
498 {
499 /* Stop HPET legacy interrupts */
500 cfg = hpet_read32(HPET_CFG);
501 cfg &= ~HPET_CFG_LEGACY;
502 hpet_write32(cfg, HPET_CFG);
504 for ( i = 0; i < num_hpets_used; i++ )
505 {
506 /* set HPET Tn as oneshot */
507 cfg = hpet_read32(HPET_Tn_CFG(hpet_events[i].idx));
508 cfg &= ~HPET_TN_PERIODIC;
509 cfg |= HPET_TN_ENABLE | HPET_TN_32BIT;
510 hpet_write32(cfg, HPET_Tn_CFG(hpet_events[i].idx));
512 hpet_events[i].mult = div_sc((unsigned long)hpet_rate,
513 1000000000ul, 32);
514 hpet_events[i].shift = 32;
515 hpet_events[i].next_event = STIME_MAX;
516 hpet_events[i].event_handler = handle_hpet_broadcast;
517 spin_lock_init(&hpet_events[i].lock);
518 }
520 if ( num_hpets_used < num_possible_cpus() )
521 {
522 hpet_attach_channel = hpet_attach_channel_share;
523 hpet_detach_channel = hpet_detach_channel_share;
524 }
526 return;
527 }
529 hpet_id = hpet_read32(HPET_ID);
530 if ( !(hpet_id & HPET_ID_LEGSUP) || !force_hpet_broadcast )
531 return;
533 /* Start HPET legacy interrupts */
534 cfg = hpet_read32(HPET_CFG);
535 cfg |= HPET_CFG_LEGACY;
536 hpet_write32(cfg, HPET_CFG);
538 /* set HPET T0 as oneshot */
539 cfg = hpet_read32(HPET_T0_CFG);
540 cfg &= ~HPET_TN_PERIODIC;
541 cfg |= HPET_TN_ENABLE | HPET_TN_32BIT;
542 hpet_write32(cfg, HPET_T0_CFG);
544 /*
545 * The period is a femto seconds value. We need to calculate the scaled
546 * math multiplication factor for nanosecond to hpet tick conversion.
547 */
548 legacy_hpet_event.mult = div_sc((unsigned long)hpet_rate, 1000000000ul, 32);
549 legacy_hpet_event.shift = 32;
550 legacy_hpet_event.next_event = STIME_MAX;
551 legacy_hpet_event.event_handler = handle_hpet_broadcast;
552 legacy_hpet_event.idx = 0;
553 legacy_hpet_event.flags = 0;
554 spin_lock_init(&legacy_hpet_event.lock);
556 for_each_cpu(i)
557 per_cpu(cpu_bc_channel, i) = &legacy_hpet_event;
558 }
560 void hpet_broadcast_enter(void)
561 {
562 int cpu = smp_processor_id();
563 struct hpet_event_channel *ch = per_cpu(cpu_bc_channel, cpu);
565 if ( !ch )
566 ch = hpet_get_channel(cpu);
567 BUG_ON( !ch );
569 ASSERT(!local_irq_is_enabled());
570 spin_lock(&ch->lock);
572 if ( hpet_attach_channel )
573 hpet_attach_channel(cpu, ch);
575 disable_APIC_timer();
577 cpu_set(cpu, ch->cpumask);
579 /* reprogram if current cpu expire time is nearer */
580 if ( this_cpu(timer_deadline) < ch->next_event )
581 reprogram_hpet_evt_channel(ch, this_cpu(timer_deadline), NOW(), 1);
583 spin_unlock(&ch->lock);
584 }
586 void hpet_broadcast_exit(void)
587 {
588 int cpu = smp_processor_id();
589 struct hpet_event_channel *ch = per_cpu(cpu_bc_channel, cpu);
591 BUG_ON( !ch );
593 spin_lock_irq(&ch->lock);
595 if ( cpu_test_and_clear(cpu, ch->cpumask) )
596 {
597 /* Cancel any outstanding LAPIC event and re-enable interrupts. */
598 reprogram_timer(0);
599 enable_APIC_timer();
601 /* Reprogram the deadline; trigger timer work now if it has passed. */
602 if ( !reprogram_timer(per_cpu(timer_deadline, cpu)) )
603 raise_softirq(TIMER_SOFTIRQ);
605 if ( cpus_empty(ch->cpumask) && ch->next_event != STIME_MAX )
606 reprogram_hpet_evt_channel(ch, STIME_MAX, 0, 0);
607 }
609 if ( hpet_detach_channel )
610 hpet_detach_channel(cpu);
612 spin_unlock_irq(&ch->lock);
613 }
615 int hpet_broadcast_is_available(void)
616 {
617 return (legacy_hpet_event.event_handler == handle_hpet_broadcast
618 || num_hpets_used > 0);
619 }
621 int hpet_legacy_irq_tick(void)
622 {
623 if ( !legacy_hpet_event.event_handler )
624 return 0;
625 legacy_hpet_event.event_handler(&legacy_hpet_event);
626 return 1;
627 }
629 u64 hpet_setup(void)
630 {
631 static u64 hpet_rate;
632 static u32 system_reset_latch;
633 u32 hpet_id, hpet_period, cfg;
634 int i;
636 if ( system_reset_latch == system_reset_counter )
637 return hpet_rate;
638 system_reset_latch = system_reset_counter;
640 if ( hpet_address == 0 )
641 return 0;
643 set_fixmap_nocache(FIX_HPET_BASE, hpet_address);
645 hpet_id = hpet_read32(HPET_ID);
646 if ( (hpet_id & HPET_ID_REV) == 0 )
647 {
648 printk("BAD HPET revision id.\n");
649 return 0;
650 }
652 /* Check for sane period (100ps <= period <= 100ns). */
653 hpet_period = hpet_read32(HPET_PERIOD);
654 if ( (hpet_period > 100000000) || (hpet_period < 100000) )
655 {
656 printk("BAD HPET period %u.\n", hpet_period);
657 return 0;
658 }
660 cfg = hpet_read32(HPET_CFG);
661 cfg &= ~(HPET_CFG_ENABLE | HPET_CFG_LEGACY);
662 hpet_write32(cfg, HPET_CFG);
664 for ( i = 0; i <= ((hpet_id >> 8) & 31); i++ )
665 {
666 cfg = hpet_read32(HPET_Tn_CFG(i));
667 cfg &= ~HPET_TN_ENABLE;
668 hpet_write32(cfg, HPET_Tn_CFG(i));
669 }
671 cfg = hpet_read32(HPET_CFG);
672 cfg |= HPET_CFG_ENABLE;
673 hpet_write32(cfg, HPET_CFG);
675 hpet_rate = 1000000000000000ULL; /* 10^15 */
676 (void)do_div(hpet_rate, hpet_period);
678 return hpet_rate;
679 }