ia64/xen-unstable

view xen/arch/x86/irq.c @ 9776:72f9c751d3ea

Replace &foo[0] with foo where the latter seems cleaner
(which is usually, and particularly when its an argument
to one of the bitops functions).

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Wed Apr 19 18:32:20 2006 +0100 (2006-04-19)
parents b550a93c6459
children 42a8e3101c6c
line source
1 /******************************************************************************
2 * arch/x86/irq.c
3 *
4 * Portions of this file are:
5 * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar
6 */
8 #include <xen/config.h>
9 #include <xen/init.h>
10 #include <xen/errno.h>
11 #include <xen/event.h>
12 #include <xen/irq.h>
13 #include <xen/perfc.h>
14 #include <xen/sched.h>
15 #include <xen/keyhandler.h>
16 #include <asm/current.h>
17 #include <asm/smpboot.h>
19 /* opt_noirqbalance: If true, software IRQ balancing/affinity is disabled. */
20 int opt_noirqbalance = 0;
21 boolean_param("noirqbalance", opt_noirqbalance);
23 irq_desc_t irq_desc[NR_IRQS];
25 static void __do_IRQ_guest(int vector);
27 void no_action(int cpl, void *dev_id, struct cpu_user_regs *regs) { }
29 static void enable_none(unsigned int vector) { }
30 static unsigned int startup_none(unsigned int vector) { return 0; }
31 static void disable_none(unsigned int vector) { }
32 static void ack_none(unsigned int vector)
33 {
34 ack_bad_irq(vector);
35 }
37 #define shutdown_none disable_none
38 #define end_none enable_none
40 struct hw_interrupt_type no_irq_type = {
41 "none",
42 startup_none,
43 shutdown_none,
44 enable_none,
45 disable_none,
46 ack_none,
47 end_none
48 };
50 atomic_t irq_err_count;
52 asmlinkage void do_IRQ(struct cpu_user_regs *regs)
53 {
54 unsigned int vector = regs->entry_vector;
55 irq_desc_t *desc = &irq_desc[vector];
56 struct irqaction *action;
58 perfc_incrc(irqs);
60 spin_lock(&desc->lock);
61 desc->handler->ack(vector);
63 if ( likely(desc->status & IRQ_GUEST) )
64 {
65 __do_IRQ_guest(vector);
66 spin_unlock(&desc->lock);
67 return;
68 }
70 desc->status &= ~IRQ_REPLAY;
71 desc->status |= IRQ_PENDING;
73 /*
74 * Since we set PENDING, if another processor is handling a different
75 * instance of this same irq, the other processor will take care of it.
76 */
77 if ( desc->status & (IRQ_DISABLED | IRQ_INPROGRESS) )
78 goto out;
80 desc->status |= IRQ_INPROGRESS;
82 action = desc->action;
83 while ( desc->status & IRQ_PENDING )
84 {
85 desc->status &= ~IRQ_PENDING;
86 irq_enter();
87 spin_unlock_irq(&desc->lock);
88 action->handler(vector_to_irq(vector), action->dev_id, regs);
89 spin_lock_irq(&desc->lock);
90 irq_exit();
91 }
93 desc->status &= ~IRQ_INPROGRESS;
95 out:
96 desc->handler->end(vector);
97 spin_unlock(&desc->lock);
98 }
100 void free_irq(unsigned int irq)
101 {
102 unsigned int vector = irq_to_vector(irq);
103 irq_desc_t *desc = &irq_desc[vector];
104 unsigned long flags;
106 spin_lock_irqsave(&desc->lock,flags);
107 desc->action = NULL;
108 desc->depth = 1;
109 desc->status |= IRQ_DISABLED;
110 desc->handler->shutdown(irq);
111 spin_unlock_irqrestore(&desc->lock,flags);
113 /* Wait to make sure it's not being used on another CPU */
114 do { smp_mb(); } while ( desc->status & IRQ_INPROGRESS );
115 }
117 int setup_irq(unsigned int irq, struct irqaction *new)
118 {
119 unsigned int vector = irq_to_vector(irq);
120 irq_desc_t *desc = &irq_desc[vector];
121 unsigned long flags;
123 spin_lock_irqsave(&desc->lock,flags);
125 if ( desc->action != NULL )
126 {
127 spin_unlock_irqrestore(&desc->lock,flags);
128 return -EBUSY;
129 }
131 desc->action = new;
132 desc->depth = 0;
133 desc->status &= ~IRQ_DISABLED;
134 desc->handler->startup(vector);
136 spin_unlock_irqrestore(&desc->lock,flags);
138 return 0;
139 }
142 /*
143 * HANDLING OF GUEST-BOUND PHYSICAL IRQS
144 */
146 #define IRQ_MAX_GUESTS 7
147 typedef struct {
148 u8 nr_guests;
149 u8 in_flight;
150 u8 shareable;
151 u8 ack_type;
152 #define ACKTYPE_NONE 0 /* No final acknowledgement is required */
153 #define ACKTYPE_UNMASK 1 /* Unmask PIC hardware (from any CPU) */
154 #define ACKTYPE_EOI 2 /* EOI on the CPU that was interrupted */
155 cpumask_t cpu_eoi_map; /* CPUs that need to EOI this interrupt */
156 struct domain *guest[IRQ_MAX_GUESTS];
157 } irq_guest_action_t;
159 /*
160 * Stack of interrupts awaiting EOI on each CPU. These must be popped in
161 * order, as only the current highest-priority pending irq can be EOIed.
162 */
163 static struct {
164 u8 vector; /* Vector awaiting EOI */
165 u8 ready; /* Ready for EOI now? */
166 } pending_eoi[NR_CPUS][NR_VECTORS] __cacheline_aligned;
167 #define pending_eoi_sp(cpu) (pending_eoi[cpu][NR_VECTORS-1].vector)
169 static void __do_IRQ_guest(int vector)
170 {
171 unsigned int irq = vector_to_irq(vector);
172 irq_desc_t *desc = &irq_desc[vector];
173 irq_guest_action_t *action = (irq_guest_action_t *)desc->action;
174 struct domain *d;
175 int i, sp, cpu = smp_processor_id();
177 if ( unlikely(action->nr_guests == 0) )
178 {
179 /* An interrupt may slip through while freeing an ACKTYPE_EOI irq. */
180 ASSERT(action->ack_type == ACKTYPE_EOI);
181 ASSERT(desc->status & IRQ_DISABLED);
182 desc->handler->end(vector);
183 return;
184 }
186 if ( action->ack_type == ACKTYPE_EOI )
187 {
188 sp = pending_eoi_sp(cpu);
189 ASSERT((sp == 0) || (pending_eoi[cpu][sp-1].vector < vector));
190 ASSERT(sp < (NR_VECTORS-1));
191 pending_eoi[cpu][sp].vector = vector;
192 pending_eoi[cpu][sp].ready = 0;
193 pending_eoi_sp(cpu) = sp+1;
194 cpu_set(cpu, action->cpu_eoi_map);
195 }
197 for ( i = 0; i < action->nr_guests; i++ )
198 {
199 d = action->guest[i];
200 if ( (action->ack_type != ACKTYPE_NONE) &&
201 !test_and_set_bit(irq, d->pirq_mask) )
202 action->in_flight++;
203 send_guest_pirq(d, irq);
204 }
205 }
207 /* Flush all ready EOIs from the top of this CPU's pending-EOI stack. */
208 static void flush_ready_eoi(void *unused)
209 {
210 irq_desc_t *desc;
211 int vector, sp, cpu = smp_processor_id();
213 ASSERT(!local_irq_is_enabled());
215 sp = pending_eoi_sp(cpu);
217 while ( (--sp >= 0) && pending_eoi[cpu][sp].ready )
218 {
219 vector = pending_eoi[cpu][sp].vector;
220 desc = &irq_desc[vector];
221 spin_lock(&desc->lock);
222 desc->handler->end(vector);
223 spin_unlock(&desc->lock);
224 }
226 pending_eoi_sp(cpu) = sp+1;
227 }
229 static void __set_eoi_ready(irq_desc_t *desc)
230 {
231 irq_guest_action_t *action = (irq_guest_action_t *)desc->action;
232 int vector, sp, cpu = smp_processor_id();
234 vector = desc - irq_desc;
236 if ( !(desc->status & IRQ_GUEST) ||
237 (action->in_flight != 0) ||
238 !cpu_test_and_clear(cpu, action->cpu_eoi_map) )
239 return;
241 sp = pending_eoi_sp(cpu);
242 do {
243 ASSERT(sp > 0);
244 } while ( pending_eoi[cpu][--sp].vector != vector );
245 ASSERT(!pending_eoi[cpu][sp].ready);
246 pending_eoi[cpu][sp].ready = 1;
247 }
249 /* Mark specified IRQ as ready-for-EOI (if it really is) and attempt to EOI. */
250 static void set_eoi_ready(void *data)
251 {
252 irq_desc_t *desc = data;
254 ASSERT(!local_irq_is_enabled());
256 spin_lock(&desc->lock);
257 __set_eoi_ready(desc);
258 spin_unlock(&desc->lock);
260 flush_ready_eoi(NULL);
261 }
263 /*
264 * Forcibly flush all pending EOIs on this CPU by emulating end-of-ISR
265 * notifications from guests. The caller of this function must ensure that
266 * all CPUs execute flush_ready_eoi().
267 */
268 static void flush_all_pending_eoi(void *unused)
269 {
270 irq_desc_t *desc;
271 irq_guest_action_t *action;
272 int i, vector, sp, cpu = smp_processor_id();
274 ASSERT(!local_irq_is_enabled());
276 sp = pending_eoi_sp(cpu);
277 while ( --sp >= 0 )
278 {
279 if ( pending_eoi[cpu][sp].ready )
280 continue;
281 vector = pending_eoi[cpu][sp].vector;
282 desc = &irq_desc[vector];
283 spin_lock(&desc->lock);
284 action = (irq_guest_action_t *)desc->action;
285 ASSERT(action->ack_type == ACKTYPE_EOI);
286 ASSERT(desc->status & IRQ_GUEST);
287 for ( i = 0; i < action->nr_guests; i++ )
288 clear_bit(vector_to_irq(vector), action->guest[i]->pirq_mask);
289 action->in_flight = 0;
290 spin_unlock(&desc->lock);
291 }
293 flush_ready_eoi(NULL);
294 }
296 int pirq_guest_unmask(struct domain *d)
297 {
298 irq_desc_t *desc;
299 irq_guest_action_t *action;
300 cpumask_t cpu_eoi_map = CPU_MASK_NONE;
301 unsigned int pirq, cpu = smp_processor_id();
302 shared_info_t *s = d->shared_info;
304 for ( pirq = find_first_bit(d->pirq_mask, NR_PIRQS);
305 pirq < NR_PIRQS;
306 pirq = find_next_bit(d->pirq_mask, NR_PIRQS, pirq+1) )
307 {
308 desc = &irq_desc[irq_to_vector(pirq)];
309 action = (irq_guest_action_t *)desc->action;
311 spin_lock_irq(&desc->lock);
313 if ( !test_bit(d->pirq_to_evtchn[pirq], s->evtchn_mask) &&
314 test_and_clear_bit(pirq, d->pirq_mask) )
315 {
316 ASSERT(action->ack_type != ACKTYPE_NONE);
317 if ( --action->in_flight == 0 )
318 {
319 if ( action->ack_type == ACKTYPE_UNMASK )
320 desc->handler->end(irq_to_vector(pirq));
321 cpu_eoi_map = action->cpu_eoi_map;
322 }
323 }
325 if ( cpu_test_and_clear(cpu, cpu_eoi_map) )
326 {
327 __set_eoi_ready(desc);
328 spin_unlock(&desc->lock);
329 flush_ready_eoi(NULL);
330 local_irq_enable();
331 }
332 else
333 {
334 spin_unlock_irq(&desc->lock);
335 }
337 if ( !cpus_empty(cpu_eoi_map) )
338 {
339 on_selected_cpus(cpu_eoi_map, set_eoi_ready, desc, 1, 0);
340 cpu_eoi_map = CPU_MASK_NONE;
341 }
342 }
344 return 0;
345 }
347 extern int ioapic_ack_new;
348 int pirq_acktype(int irq)
349 {
350 irq_desc_t *desc;
351 unsigned int vector;
353 vector = irq_to_vector(irq);
354 if ( vector == 0 )
355 return ACKTYPE_NONE;
357 desc = &irq_desc[vector];
359 /*
360 * Edge-triggered IO-APIC interrupts need no final acknowledgement:
361 * we ACK early during interrupt processing.
362 */
363 if ( !strcmp(desc->handler->typename, "IO-APIC-edge") )
364 return ACKTYPE_NONE;
366 /* Legacy PIC interrupts can be acknowledged from any CPU. */
367 if ( !strcmp(desc->handler->typename, "XT-PIC") )
368 return ACKTYPE_UNMASK;
370 /*
371 * Level-triggered IO-APIC interrupts need to be acknowledged on the CPU
372 * on which they were received. This is because we tickle the LAPIC to EOI.
373 */
374 if ( !strcmp(desc->handler->typename, "IO-APIC-level") )
375 return ioapic_ack_new ? ACKTYPE_EOI : ACKTYPE_UNMASK;
377 BUG();
378 return 0;
379 }
381 int pirq_guest_bind(struct vcpu *v, int irq, int will_share)
382 {
383 unsigned int vector;
384 irq_desc_t *desc;
385 irq_guest_action_t *action;
386 unsigned long flags;
387 int rc = 0;
388 cpumask_t cpumask = CPU_MASK_NONE;
390 if ( (irq < 0) || (irq >= NR_IRQS) )
391 return -EINVAL;
393 retry:
394 vector = irq_to_vector(irq);
395 if ( vector == 0 )
396 return -EINVAL;
398 desc = &irq_desc[vector];
400 spin_lock_irqsave(&desc->lock, flags);
402 action = (irq_guest_action_t *)desc->action;
404 if ( !(desc->status & IRQ_GUEST) )
405 {
406 if ( desc->action != NULL )
407 {
408 DPRINTK("Cannot bind IRQ %d to guest. In use by '%s'.\n",
409 irq, desc->action->name);
410 rc = -EBUSY;
411 goto out;
412 }
414 action = xmalloc(irq_guest_action_t);
415 if ( (desc->action = (struct irqaction *)action) == NULL )
416 {
417 DPRINTK("Cannot bind IRQ %d to guest. Out of memory.\n", irq);
418 rc = -ENOMEM;
419 goto out;
420 }
422 action->nr_guests = 0;
423 action->in_flight = 0;
424 action->shareable = will_share;
425 action->ack_type = pirq_acktype(irq);
426 action->cpu_eoi_map = CPU_MASK_NONE;
428 desc->depth = 0;
429 desc->status |= IRQ_GUEST;
430 desc->status &= ~IRQ_DISABLED;
431 desc->handler->startup(vector);
433 /* Attempt to bind the interrupt target to the correct CPU. */
434 cpu_set(v->processor, cpumask);
435 if ( !opt_noirqbalance && (desc->handler->set_affinity != NULL) )
436 desc->handler->set_affinity(vector, cpumask);
437 }
438 else if ( !will_share || !action->shareable )
439 {
440 DPRINTK("Cannot bind IRQ %d to guest. Will not share with others.\n",
441 irq);
442 rc = -EBUSY;
443 goto out;
444 }
445 else if ( action->nr_guests == 0 )
446 {
447 /*
448 * Indicates that an ACKTYPE_EOI interrupt is being released.
449 * Wait for that to happen before continuing.
450 */
451 ASSERT(action->ack_type == ACKTYPE_EOI);
452 ASSERT(desc->status & IRQ_DISABLED);
453 spin_unlock_irqrestore(&desc->lock, flags);
454 cpu_relax();
455 goto retry;
456 }
458 if ( action->nr_guests == IRQ_MAX_GUESTS )
459 {
460 DPRINTK("Cannot bind IRQ %d to guest. Already at max share.\n", irq);
461 rc = -EBUSY;
462 goto out;
463 }
465 action->guest[action->nr_guests++] = v->domain;
467 out:
468 spin_unlock_irqrestore(&desc->lock, flags);
469 return rc;
470 }
472 int pirq_guest_unbind(struct domain *d, int irq)
473 {
474 unsigned int vector = irq_to_vector(irq);
475 irq_desc_t *desc = &irq_desc[vector];
476 irq_guest_action_t *action;
477 cpumask_t cpu_eoi_map;
478 unsigned long flags;
479 int i;
481 BUG_ON(vector == 0);
483 spin_lock_irqsave(&desc->lock, flags);
485 action = (irq_guest_action_t *)desc->action;
487 i = 0;
488 while ( action->guest[i] && (action->guest[i] != d) )
489 i++;
490 memmove(&action->guest[i], &action->guest[i+1], IRQ_MAX_GUESTS-i-1);
491 action->nr_guests--;
493 switch ( action->ack_type )
494 {
495 case ACKTYPE_UNMASK:
496 if ( test_and_clear_bit(irq, d->pirq_mask) &&
497 (--action->in_flight == 0) )
498 desc->handler->end(vector);
499 break;
500 case ACKTYPE_EOI:
501 /* NB. If #guests == 0 then we clear the eoi_map later on. */
502 if ( test_and_clear_bit(irq, d->pirq_mask) &&
503 (--action->in_flight == 0) &&
504 (action->nr_guests != 0) )
505 {
506 cpu_eoi_map = action->cpu_eoi_map;
507 spin_unlock_irqrestore(&desc->lock, flags);
508 on_selected_cpus(cpu_eoi_map, set_eoi_ready, desc, 1, 0);
509 spin_lock_irqsave(&desc->lock, flags);
510 }
511 break;
512 }
514 BUG_ON(test_bit(irq, d->pirq_mask));
516 if ( action->nr_guests != 0 )
517 goto out;
519 BUG_ON(action->in_flight != 0);
521 /* Disabling IRQ before releasing the desc_lock avoids an IRQ storm. */
522 desc->depth = 1;
523 desc->status |= IRQ_DISABLED;
524 desc->handler->disable(vector);
526 /*
527 * We may have a EOI languishing anywhere in one of the per-CPU
528 * EOI stacks. Forcibly flush the stack on every CPU where this might
529 * be the case.
530 */
531 cpu_eoi_map = action->cpu_eoi_map;
532 if ( !cpus_empty(cpu_eoi_map) )
533 {
534 BUG_ON(action->ack_type != ACKTYPE_EOI);
535 spin_unlock_irqrestore(&desc->lock, flags);
536 on_selected_cpus(cpu_eoi_map, flush_all_pending_eoi, NULL, 1, 1);
537 on_selected_cpus(cpu_online_map, flush_ready_eoi, NULL, 1, 1);
538 spin_lock_irqsave(&desc->lock, flags);
539 }
541 BUG_ON(!cpus_empty(action->cpu_eoi_map));
543 desc->action = NULL;
544 xfree(action);
545 desc->status &= ~IRQ_GUEST;
546 desc->handler->shutdown(vector);
548 out:
549 spin_unlock_irqrestore(&desc->lock, flags);
550 return 0;
551 }
553 extern void dump_ioapic_irq_info(void);
555 static void dump_irqs(unsigned char key)
556 {
557 int i, irq, vector;
558 irq_desc_t *desc;
559 irq_guest_action_t *action;
560 struct domain *d;
561 unsigned long flags;
563 printk("Guest interrupt information:\n");
565 for ( irq = 0; irq < NR_IRQS; irq++ )
566 {
567 vector = irq_to_vector(irq);
568 if ( vector == 0 )
569 continue;
571 desc = &irq_desc[vector];
573 spin_lock_irqsave(&desc->lock, flags);
575 if ( desc->status & IRQ_GUEST )
576 {
577 action = (irq_guest_action_t *)desc->action;
579 printk(" IRQ%3d Vec%3d: type=%-15s status=%08x "
580 "in-flight=%d domain-list=",
581 irq, vector, desc->handler->typename,
582 desc->status, action->in_flight);
584 for ( i = 0; i < action->nr_guests; i++ )
585 {
586 d = action->guest[i];
587 printk("%u(%c%c%c%c)",
588 d->domain_id,
589 (test_bit(d->pirq_to_evtchn[irq],
590 d->shared_info->evtchn_pending) ?
591 'P' : '-'),
592 (test_bit(d->pirq_to_evtchn[irq]/BITS_PER_LONG,
593 &d->shared_info->vcpu_info[0].
594 evtchn_pending_sel) ?
595 'S' : '-'),
596 (test_bit(d->pirq_to_evtchn[irq],
597 d->shared_info->evtchn_mask) ?
598 'M' : '-'),
599 (test_bit(irq, d->pirq_mask) ?
600 'M' : '-'));
601 if ( i != action->nr_guests )
602 printk(",");
603 }
605 printk("\n");
606 }
608 spin_unlock_irqrestore(&desc->lock, flags);
609 }
611 dump_ioapic_irq_info();
612 }
614 static int __init setup_dump_irqs(void)
615 {
616 register_keyhandler('i', dump_irqs, "dump interrupt bindings");
617 return 0;
618 }
619 __initcall(setup_dump_irqs);
621 static struct timer end_irq_timer[NR_CPUS];
623 /*
624 * force_intack: Forcibly emit all pending EOIs on each CPU every second.
625 * Mainly useful for debugging or poking lazy guests ISRs.
626 */
628 static void end_irq_timeout(void *unused)
629 {
630 int cpu = smp_processor_id();
632 local_irq_disable();
633 flush_all_pending_eoi(NULL);
634 local_irq_enable();
636 on_selected_cpus(cpu_online_map, flush_ready_eoi, NULL, 1, 0);
638 set_timer(&end_irq_timer[cpu], NOW() + MILLISECS(1000));
639 }
641 static void __init __setup_irq_timeout(void *unused)
642 {
643 int cpu = smp_processor_id();
644 init_timer(&end_irq_timer[cpu], end_irq_timeout, NULL, cpu);
645 set_timer(&end_irq_timer[cpu], NOW() + MILLISECS(1000));
646 }
648 static int force_intack;
649 boolean_param("force_intack", force_intack);
651 static int __init setup_irq_timeout(void)
652 {
653 if ( force_intack )
654 on_each_cpu(__setup_irq_timeout, NULL, 1, 1);
655 return 0;
656 }
657 __initcall(setup_irq_timeout);