ia64/linux-2.6.18-xen.hg

view drivers/pci/msi.c @ 882:8dec4aa9b8b9

PCI pass through: PCIe IO space multiplexing

This is required for more than 16 HVM domain to boot from
PCIe pass through device.

Linux as dom0 exclusively assigns IO space to downstream PCI bridges
and the assignment unit of PCI bridge IO space is 4K. So the only up
to 16 PCIe device can be accessed via IO space within 64K IO ports.
PCI expansion ROM BIOS often uses IO port access to boot from the
device, so on virtualized environment, it means only up to 16 guest
domain can boot from pass-through device.

This patch allows PCIe IO space sharing of pass-through device.
- reassign IO space of PCIe devices specified by
"guestiomuldev=[<segment>:]<bus>:<dev>[,[<segment:><bus>:dev]][,...]"
to be shared.
This is implemented as Linux PCI quirk fixup.

The sharing unit is PCIe switch. Ie IO space of the end point
devices under the same switch will be shared. If there are more than
one switches, two areas of IO space will be used.

- And the driver which arbitrates the accesses to the multiplexed PCIe
IO space. Later qemu-dm will use this.

Limitation:
IO port of IO shared devices can't be accessed from dom0 Linux device
driver. But this wouldn't be a big issue because PCIe specification
discourages the use of IO space and recommends that IO space should be
used only for bootable device with ROM code. OS device driver should
work without IO space access.

Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
author Keir Fraser <keir.fraser@citrix.com>
date Thu May 28 09:57:49 2009 +0100 (2009-05-28)
parents 831230e53067
children
line source
1 /*
2 * File: msi.c
3 * Purpose: PCI Message Signaled Interrupt (MSI)
4 *
5 * Copyright (C) 2003-2004 Intel
6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
7 */
9 #include <linux/mm.h>
10 #include <linux/irq.h>
11 #include <linux/interrupt.h>
12 #include <linux/init.h>
13 #include <linux/ioport.h>
14 #include <linux/smp_lock.h>
15 #include <linux/pci.h>
16 #include <linux/proc_fs.h>
18 #include <asm/errno.h>
19 #include <asm/io.h>
20 #include <asm/smp.h>
22 #include "pci.h"
23 #include "msi.h"
25 static DEFINE_SPINLOCK(msi_lock);
26 static struct msi_desc* msi_desc[NR_IRQS] = { [0 ... NR_IRQS-1] = NULL };
27 static kmem_cache_t* msi_cachep;
29 static int pci_msi_enable = 1;
30 static int last_alloc_vector;
31 static int nr_released_vectors;
32 static int nr_reserved_vectors = NR_HP_RESERVED_VECTORS;
33 static int nr_msix_devices;
35 #ifndef CONFIG_X86_IO_APIC
36 int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1};
37 #endif
39 static struct msi_ops *msi_ops;
41 int
42 msi_register(struct msi_ops *ops)
43 {
44 msi_ops = ops;
45 return 0;
46 }
48 static void msi_cache_ctor(void *p, kmem_cache_t *cache, unsigned long flags)
49 {
50 memset(p, 0, sizeof(struct msi_desc));
51 }
53 static int msi_cache_init(void)
54 {
55 msi_cachep = kmem_cache_create("msi_cache",
56 sizeof(struct msi_desc),
57 0, SLAB_HWCACHE_ALIGN, msi_cache_ctor, NULL);
58 if (!msi_cachep)
59 return -ENOMEM;
61 return 0;
62 }
64 static void msi_set_mask_bit(unsigned int vector, int flag)
65 {
66 struct msi_desc *entry;
68 entry = (struct msi_desc *)msi_desc[vector];
69 if (!entry || !entry->dev || !entry->mask_base)
70 return;
71 switch (entry->msi_attrib.type) {
72 case PCI_CAP_ID_MSI:
73 {
74 int pos;
75 u32 mask_bits;
77 pos = (long)entry->mask_base;
78 pci_read_config_dword(entry->dev, pos, &mask_bits);
79 mask_bits &= ~(1);
80 mask_bits |= flag;
81 pci_write_config_dword(entry->dev, pos, mask_bits);
82 break;
83 }
84 case PCI_CAP_ID_MSIX:
85 {
86 int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
87 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
88 writel(flag, entry->mask_base + offset);
89 break;
90 }
91 default:
92 break;
93 }
94 }
96 #ifdef CONFIG_SMP
97 static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask)
98 {
99 struct msi_desc *entry;
100 u32 address_hi, address_lo;
101 unsigned int irq = vector;
102 unsigned int dest_cpu = first_cpu(cpu_mask);
104 entry = (struct msi_desc *)msi_desc[vector];
105 if (!entry || !entry->dev)
106 return;
108 switch (entry->msi_attrib.type) {
109 case PCI_CAP_ID_MSI:
110 {
111 int pos = pci_find_capability(entry->dev, PCI_CAP_ID_MSI);
113 if (!pos)
114 return;
116 pci_read_config_dword(entry->dev, msi_upper_address_reg(pos),
117 &address_hi);
118 pci_read_config_dword(entry->dev, msi_lower_address_reg(pos),
119 &address_lo);
121 msi_ops->target(vector, dest_cpu, &address_hi, &address_lo);
123 pci_write_config_dword(entry->dev, msi_upper_address_reg(pos),
124 address_hi);
125 pci_write_config_dword(entry->dev, msi_lower_address_reg(pos),
126 address_lo);
127 set_native_irq_info(irq, cpu_mask);
128 break;
129 }
130 case PCI_CAP_ID_MSIX:
131 {
132 int offset_hi =
133 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
134 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET;
135 int offset_lo =
136 entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
137 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET;
139 address_hi = readl(entry->mask_base + offset_hi);
140 address_lo = readl(entry->mask_base + offset_lo);
142 msi_ops->target(vector, dest_cpu, &address_hi, &address_lo);
144 writel(address_hi, entry->mask_base + offset_hi);
145 writel(address_lo, entry->mask_base + offset_lo);
146 set_native_irq_info(irq, cpu_mask);
147 break;
148 }
149 default:
150 break;
151 }
152 }
153 #else
154 #define set_msi_affinity NULL
155 #endif /* CONFIG_SMP */
157 static void mask_MSI_irq(unsigned int vector)
158 {
159 msi_set_mask_bit(vector, 1);
160 }
162 static void unmask_MSI_irq(unsigned int vector)
163 {
164 msi_set_mask_bit(vector, 0);
165 }
167 static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector)
168 {
169 struct msi_desc *entry;
170 unsigned long flags;
172 spin_lock_irqsave(&msi_lock, flags);
173 entry = msi_desc[vector];
174 if (!entry || !entry->dev) {
175 spin_unlock_irqrestore(&msi_lock, flags);
176 return 0;
177 }
178 entry->msi_attrib.state = 1; /* Mark it active */
179 spin_unlock_irqrestore(&msi_lock, flags);
181 return 0; /* never anything pending */
182 }
184 static unsigned int startup_msi_irq_w_maskbit(unsigned int vector)
185 {
186 startup_msi_irq_wo_maskbit(vector);
187 unmask_MSI_irq(vector);
188 return 0; /* never anything pending */
189 }
191 static void shutdown_msi_irq(unsigned int vector)
192 {
193 struct msi_desc *entry;
194 unsigned long flags;
196 spin_lock_irqsave(&msi_lock, flags);
197 entry = msi_desc[vector];
198 if (entry && entry->dev)
199 entry->msi_attrib.state = 0; /* Mark it not active */
200 spin_unlock_irqrestore(&msi_lock, flags);
201 }
203 static void end_msi_irq_wo_maskbit(unsigned int vector)
204 {
205 move_native_irq(vector);
206 ack_APIC_irq();
207 }
209 static void end_msi_irq_w_maskbit(unsigned int vector)
210 {
211 move_native_irq(vector);
212 unmask_MSI_irq(vector);
213 ack_APIC_irq();
214 }
216 static void do_nothing(unsigned int vector)
217 {
218 }
220 /*
221 * Interrupt Type for MSI-X PCI/PCI-X/PCI-Express Devices,
222 * which implement the MSI-X Capability Structure.
223 */
224 static struct hw_interrupt_type msix_irq_type = {
225 .typename = "PCI-MSI-X",
226 .startup = startup_msi_irq_w_maskbit,
227 .shutdown = shutdown_msi_irq,
228 .enable = unmask_MSI_irq,
229 .disable = mask_MSI_irq,
230 .ack = mask_MSI_irq,
231 .end = end_msi_irq_w_maskbit,
232 .set_affinity = set_msi_affinity
233 };
235 /*
236 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices,
237 * which implement the MSI Capability Structure with
238 * Mask-and-Pending Bits.
239 */
240 static struct hw_interrupt_type msi_irq_w_maskbit_type = {
241 .typename = "PCI-MSI",
242 .startup = startup_msi_irq_w_maskbit,
243 .shutdown = shutdown_msi_irq,
244 .enable = unmask_MSI_irq,
245 .disable = mask_MSI_irq,
246 .ack = mask_MSI_irq,
247 .end = end_msi_irq_w_maskbit,
248 .set_affinity = set_msi_affinity
249 };
251 /*
252 * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices,
253 * which implement the MSI Capability Structure without
254 * Mask-and-Pending Bits.
255 */
256 static struct hw_interrupt_type msi_irq_wo_maskbit_type = {
257 .typename = "PCI-MSI",
258 .startup = startup_msi_irq_wo_maskbit,
259 .shutdown = shutdown_msi_irq,
260 .enable = do_nothing,
261 .disable = do_nothing,
262 .ack = do_nothing,
263 .end = end_msi_irq_wo_maskbit,
264 .set_affinity = set_msi_affinity
265 };
267 static int msi_free_vector(struct pci_dev* dev, int vector, int reassign);
268 static int assign_msi_vector(void)
269 {
270 static int new_vector_avail = 1;
271 int vector;
272 unsigned long flags;
274 /*
275 * msi_lock is provided to ensure that successful allocation of MSI
276 * vector is assigned unique among drivers.
277 */
278 spin_lock_irqsave(&msi_lock, flags);
280 if (!new_vector_avail) {
281 int free_vector = 0;
283 /*
284 * vector_irq[] = -1 indicates that this specific vector is:
285 * - assigned for MSI (since MSI have no associated IRQ) or
286 * - assigned for legacy if less than 16, or
287 * - having no corresponding 1:1 vector-to-IOxAPIC IRQ mapping
288 * vector_irq[] = 0 indicates that this vector, previously
289 * assigned for MSI, is freed by hotplug removed operations.
290 * This vector will be reused for any subsequent hotplug added
291 * operations.
292 * vector_irq[] > 0 indicates that this vector is assigned for
293 * IOxAPIC IRQs. This vector and its value provides a 1-to-1
294 * vector-to-IOxAPIC IRQ mapping.
295 */
296 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
297 if (vector_irq[vector] != 0)
298 continue;
299 free_vector = vector;
300 if (!msi_desc[vector])
301 break;
302 else
303 continue;
304 }
305 if (!free_vector) {
306 spin_unlock_irqrestore(&msi_lock, flags);
307 return -EBUSY;
308 }
309 vector_irq[free_vector] = -1;
310 nr_released_vectors--;
311 spin_unlock_irqrestore(&msi_lock, flags);
312 if (msi_desc[free_vector] != NULL) {
313 struct pci_dev *dev;
314 int tail;
316 /* free all linked vectors before re-assign */
317 do {
318 spin_lock_irqsave(&msi_lock, flags);
319 dev = msi_desc[free_vector]->dev;
320 tail = msi_desc[free_vector]->link.tail;
321 spin_unlock_irqrestore(&msi_lock, flags);
322 msi_free_vector(dev, tail, 1);
323 } while (free_vector != tail);
324 }
326 return free_vector;
327 }
328 vector = assign_irq_vector(AUTO_ASSIGN);
329 last_alloc_vector = vector;
330 if (vector == LAST_DEVICE_VECTOR)
331 new_vector_avail = 0;
333 spin_unlock_irqrestore(&msi_lock, flags);
334 return vector;
335 }
337 static int get_new_vector(void)
338 {
339 int vector = assign_msi_vector();
341 if (vector > 0)
342 set_intr_gate(vector, interrupt[vector]);
344 return vector;
345 }
347 static int msi_init(void)
348 {
349 static int status = -ENOMEM;
351 if (!status)
352 return status;
354 if (pci_msi_quirk) {
355 pci_msi_enable = 0;
356 printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n");
357 status = -EINVAL;
358 return status;
359 }
361 status = msi_arch_init();
362 if (status < 0) {
363 pci_msi_enable = 0;
364 printk(KERN_WARNING
365 "PCI: MSI arch init failed. MSI disabled.\n");
366 return status;
367 }
369 if (! msi_ops) {
370 printk(KERN_WARNING
371 "PCI: MSI ops not registered. MSI disabled.\n");
372 status = -EINVAL;
373 return status;
374 }
376 last_alloc_vector = assign_irq_vector(AUTO_ASSIGN);
377 status = msi_cache_init();
378 if (status < 0) {
379 pci_msi_enable = 0;
380 printk(KERN_WARNING "PCI: MSI cache init failed\n");
381 return status;
382 }
384 if (last_alloc_vector < 0) {
385 pci_msi_enable = 0;
386 printk(KERN_WARNING "PCI: No interrupt vectors available for MSI\n");
387 status = -EBUSY;
388 return status;
389 }
390 vector_irq[last_alloc_vector] = 0;
391 nr_released_vectors++;
393 return status;
394 }
396 static int get_msi_vector(struct pci_dev *dev)
397 {
398 return get_new_vector();
399 }
401 static struct msi_desc* alloc_msi_entry(void)
402 {
403 struct msi_desc *entry;
405 entry = kmem_cache_alloc(msi_cachep, SLAB_KERNEL);
406 if (!entry)
407 return NULL;
409 memset(entry, 0, sizeof(struct msi_desc));
410 entry->link.tail = entry->link.head = 0; /* single message */
411 entry->dev = NULL;
413 return entry;
414 }
416 static void attach_msi_entry(struct msi_desc *entry, int vector)
417 {
418 unsigned long flags;
420 spin_lock_irqsave(&msi_lock, flags);
421 msi_desc[vector] = entry;
422 spin_unlock_irqrestore(&msi_lock, flags);
423 }
425 static void irq_handler_init(int cap_id, int pos, int mask)
426 {
427 unsigned long flags;
429 spin_lock_irqsave(&irq_desc[pos].lock, flags);
430 if (cap_id == PCI_CAP_ID_MSIX)
431 irq_desc[pos].chip = &msix_irq_type;
432 else {
433 if (!mask)
434 irq_desc[pos].chip = &msi_irq_wo_maskbit_type;
435 else
436 irq_desc[pos].chip = &msi_irq_w_maskbit_type;
437 }
438 spin_unlock_irqrestore(&irq_desc[pos].lock, flags);
439 }
441 static void enable_msi_mode(struct pci_dev *dev, int pos, int type)
442 {
443 u16 control;
445 pci_read_config_word(dev, msi_control_reg(pos), &control);
446 if (type == PCI_CAP_ID_MSI) {
447 /* Set enabled bits to single MSI & enable MSI_enable bit */
448 msi_enable(control, 1);
449 pci_write_config_word(dev, msi_control_reg(pos), control);
450 dev->msi_enabled = 1;
451 } else {
452 msix_enable(control);
453 pci_write_config_word(dev, msi_control_reg(pos), control);
454 dev->msix_enabled = 1;
455 }
456 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
457 /* PCI Express Endpoint device detected */
458 pci_intx(dev, 0); /* disable intx */
459 }
460 }
462 void disable_msi_mode(struct pci_dev *dev, int pos, int type)
463 {
464 u16 control;
466 pci_read_config_word(dev, msi_control_reg(pos), &control);
467 if (type == PCI_CAP_ID_MSI) {
468 /* Set enabled bits to single MSI & enable MSI_enable bit */
469 msi_disable(control);
470 pci_write_config_word(dev, msi_control_reg(pos), control);
471 dev->msi_enabled = 0;
472 } else {
473 msix_disable(control);
474 pci_write_config_word(dev, msi_control_reg(pos), control);
475 dev->msix_enabled = 0;
476 }
477 if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
478 /* PCI Express Endpoint device detected */
479 pci_intx(dev, 1); /* enable intx */
480 }
481 }
483 static int msi_lookup_vector(struct pci_dev *dev, int type)
484 {
485 int vector;
486 unsigned long flags;
488 spin_lock_irqsave(&msi_lock, flags);
489 for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
490 if (!msi_desc[vector] || msi_desc[vector]->dev != dev ||
491 msi_desc[vector]->msi_attrib.type != type ||
492 msi_desc[vector]->msi_attrib.default_vector != dev->irq)
493 continue;
494 spin_unlock_irqrestore(&msi_lock, flags);
495 /* This pre-assigned MSI vector for this device
496 already exits. Override dev->irq with this vector */
497 dev->irq = vector;
498 return 0;
499 }
500 spin_unlock_irqrestore(&msi_lock, flags);
502 return -EACCES;
503 }
505 void pci_scan_msi_device(struct pci_dev *dev)
506 {
507 if (!dev)
508 return;
510 if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0)
511 nr_msix_devices++;
512 else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0)
513 nr_reserved_vectors++;
514 }
516 #ifdef CONFIG_PM
517 int pci_save_msi_state(struct pci_dev *dev)
518 {
519 int pos, i = 0;
520 u16 control;
521 struct pci_cap_saved_state *save_state;
522 u32 *cap;
524 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
525 if (pos <= 0 || dev->no_msi)
526 return 0;
528 pci_read_config_word(dev, msi_control_reg(pos), &control);
529 if (!(control & PCI_MSI_FLAGS_ENABLE))
530 return 0;
532 save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u32) * 5,
533 GFP_KERNEL);
534 if (!save_state) {
535 printk(KERN_ERR "Out of memory in pci_save_msi_state\n");
536 return -ENOMEM;
537 }
538 cap = &save_state->data[0];
540 pci_read_config_dword(dev, pos, &cap[i++]);
541 control = cap[0] >> 16;
542 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, &cap[i++]);
543 if (control & PCI_MSI_FLAGS_64BIT) {
544 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, &cap[i++]);
545 pci_read_config_dword(dev, pos + PCI_MSI_DATA_64, &cap[i++]);
546 } else
547 pci_read_config_dword(dev, pos + PCI_MSI_DATA_32, &cap[i++]);
548 if (control & PCI_MSI_FLAGS_MASKBIT)
549 pci_read_config_dword(dev, pos + PCI_MSI_MASK_BIT, &cap[i++]);
550 save_state->cap_nr = PCI_CAP_ID_MSI;
551 pci_add_saved_cap(dev, save_state);
552 return 0;
553 }
555 void pci_restore_msi_state(struct pci_dev *dev)
556 {
557 int i = 0, pos;
558 u16 control;
559 struct pci_cap_saved_state *save_state;
560 u32 *cap;
562 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSI);
563 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
564 if (!save_state || pos <= 0)
565 return;
566 cap = &save_state->data[0];
568 control = cap[i++] >> 16;
569 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, cap[i++]);
570 if (control & PCI_MSI_FLAGS_64BIT) {
571 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, cap[i++]);
572 pci_write_config_dword(dev, pos + PCI_MSI_DATA_64, cap[i++]);
573 } else
574 pci_write_config_dword(dev, pos + PCI_MSI_DATA_32, cap[i++]);
575 if (control & PCI_MSI_FLAGS_MASKBIT)
576 pci_write_config_dword(dev, pos + PCI_MSI_MASK_BIT, cap[i++]);
577 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control);
578 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
579 pci_remove_saved_cap(save_state);
580 kfree(save_state);
581 }
583 int pci_save_msix_state(struct pci_dev *dev)
584 {
585 int pos;
586 int temp;
587 int vector, head, tail = 0;
588 u16 control;
589 struct pci_cap_saved_state *save_state;
591 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
592 if (pos <= 0 || dev->no_msi)
593 return 0;
595 /* save the capability */
596 pci_read_config_word(dev, msi_control_reg(pos), &control);
597 if (!(control & PCI_MSIX_FLAGS_ENABLE))
598 return 0;
599 save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u16),
600 GFP_KERNEL);
601 if (!save_state) {
602 printk(KERN_ERR "Out of memory in pci_save_msix_state\n");
603 return -ENOMEM;
604 }
605 *((u16 *)&save_state->data[0]) = control;
607 /* save the table */
608 temp = dev->irq;
609 if (msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
610 kfree(save_state);
611 return -EINVAL;
612 }
614 vector = head = dev->irq;
615 while (head != tail) {
616 int j;
617 void __iomem *base;
618 struct msi_desc *entry;
620 entry = msi_desc[vector];
621 base = entry->mask_base;
622 j = entry->msi_attrib.entry_nr;
624 entry->address_lo_save =
625 readl(base + j * PCI_MSIX_ENTRY_SIZE +
626 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
627 entry->address_hi_save =
628 readl(base + j * PCI_MSIX_ENTRY_SIZE +
629 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
630 entry->data_save =
631 readl(base + j * PCI_MSIX_ENTRY_SIZE +
632 PCI_MSIX_ENTRY_DATA_OFFSET);
634 tail = msi_desc[vector]->link.tail;
635 vector = tail;
636 }
637 dev->irq = temp;
639 save_state->cap_nr = PCI_CAP_ID_MSIX;
640 pci_add_saved_cap(dev, save_state);
641 return 0;
642 }
644 void pci_restore_msix_state(struct pci_dev *dev)
645 {
646 u16 save;
647 int pos;
648 int vector, head, tail = 0;
649 void __iomem *base;
650 int j;
651 struct msi_desc *entry;
652 int temp;
653 struct pci_cap_saved_state *save_state;
655 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSIX);
656 if (!save_state)
657 return;
658 save = *((u16 *)&save_state->data[0]);
659 pci_remove_saved_cap(save_state);
660 kfree(save_state);
662 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
663 if (pos <= 0)
664 return;
666 /* route the table */
667 temp = dev->irq;
668 if (msi_lookup_vector(dev, PCI_CAP_ID_MSIX))
669 return;
670 vector = head = dev->irq;
671 while (head != tail) {
672 entry = msi_desc[vector];
673 base = entry->mask_base;
674 j = entry->msi_attrib.entry_nr;
676 writel(entry->address_lo_save,
677 base + j * PCI_MSIX_ENTRY_SIZE +
678 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
679 writel(entry->address_hi_save,
680 base + j * PCI_MSIX_ENTRY_SIZE +
681 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
682 writel(entry->data_save,
683 base + j * PCI_MSIX_ENTRY_SIZE +
684 PCI_MSIX_ENTRY_DATA_OFFSET);
686 tail = msi_desc[vector]->link.tail;
687 vector = tail;
688 }
689 dev->irq = temp;
691 pci_write_config_word(dev, msi_control_reg(pos), save);
692 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
693 }
694 #endif
696 static int msi_register_init(struct pci_dev *dev, struct msi_desc *entry)
697 {
698 int status;
699 u32 address_hi;
700 u32 address_lo;
701 u32 data;
702 int pos, vector = dev->irq;
703 u16 control;
705 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
706 pci_read_config_word(dev, msi_control_reg(pos), &control);
708 /* Configure MSI capability structure */
709 status = msi_ops->setup(dev, vector, &address_hi, &address_lo, &data);
710 if (status < 0)
711 return status;
713 pci_write_config_dword(dev, msi_lower_address_reg(pos), address_lo);
714 if (is_64bit_address(control)) {
715 pci_write_config_dword(dev,
716 msi_upper_address_reg(pos), address_hi);
717 pci_write_config_word(dev,
718 msi_data_reg(pos, 1), data);
719 } else
720 pci_write_config_word(dev,
721 msi_data_reg(pos, 0), data);
722 if (entry->msi_attrib.maskbit) {
723 unsigned int maskbits, temp;
724 /* All MSIs are unmasked by default, Mask them all */
725 pci_read_config_dword(dev,
726 msi_mask_bits_reg(pos, is_64bit_address(control)),
727 &maskbits);
728 temp = (1 << multi_msi_capable(control));
729 temp = ((temp - 1) & ~temp);
730 maskbits |= temp;
731 pci_write_config_dword(dev,
732 msi_mask_bits_reg(pos, is_64bit_address(control)),
733 maskbits);
734 }
736 return 0;
737 }
739 /**
740 * msi_capability_init - configure device's MSI capability structure
741 * @dev: pointer to the pci_dev data structure of MSI device function
742 *
743 * Setup the MSI capability structure of device function with a single
744 * MSI vector, regardless of device function is capable of handling
745 * multiple messages. A return of zero indicates the successful setup
746 * of an entry zero with the new MSI vector or non-zero for otherwise.
747 **/
748 static int msi_capability_init(struct pci_dev *dev)
749 {
750 int status;
751 struct msi_desc *entry;
752 int pos, vector;
753 u16 control;
755 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
756 pci_read_config_word(dev, msi_control_reg(pos), &control);
757 /* MSI Entry Initialization */
758 entry = alloc_msi_entry();
759 if (!entry)
760 return -ENOMEM;
762 vector = get_msi_vector(dev);
763 if (vector < 0) {
764 kmem_cache_free(msi_cachep, entry);
765 return -EBUSY;
766 }
767 entry->link.head = vector;
768 entry->link.tail = vector;
769 entry->msi_attrib.type = PCI_CAP_ID_MSI;
770 entry->msi_attrib.state = 0; /* Mark it not active */
771 entry->msi_attrib.entry_nr = 0;
772 entry->msi_attrib.maskbit = is_mask_bit_support(control);
773 entry->msi_attrib.default_vector = dev->irq; /* Save IOAPIC IRQ */
774 dev->irq = vector;
775 entry->dev = dev;
776 if (is_mask_bit_support(control)) {
777 entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos,
778 is_64bit_address(control));
779 }
780 /* Replace with MSI handler */
781 irq_handler_init(PCI_CAP_ID_MSI, vector, entry->msi_attrib.maskbit);
782 /* Configure MSI capability structure */
783 status = msi_register_init(dev, entry);
784 if (status != 0) {
785 dev->irq = entry->msi_attrib.default_vector;
786 kmem_cache_free(msi_cachep, entry);
787 return status;
788 }
790 attach_msi_entry(entry, vector);
791 /* Set MSI enabled bits */
792 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
794 return 0;
795 }
797 /**
798 * msix_capability_init - configure device's MSI-X capability
799 * @dev: pointer to the pci_dev data structure of MSI-X device function
800 * @entries: pointer to an array of struct msix_entry entries
801 * @nvec: number of @entries
802 *
803 * Setup the MSI-X capability structure of device function with a
804 * single MSI-X vector. A return of zero indicates the successful setup of
805 * requested MSI-X entries with allocated vectors or non-zero for otherwise.
806 **/
807 static int msix_capability_init(struct pci_dev *dev,
808 struct msix_entry *entries, int nvec)
809 {
810 struct msi_desc *head = NULL, *tail = NULL, *entry = NULL;
811 u32 address_hi;
812 u32 address_lo;
813 u32 data;
814 int status;
815 int vector, pos, i, j, nr_entries, temp = 0;
816 unsigned long phys_addr;
817 u32 table_offset;
818 u16 control;
819 u8 bir;
820 void __iomem *base;
822 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
823 /* Request & Map MSI-X table region */
824 pci_read_config_word(dev, msi_control_reg(pos), &control);
825 nr_entries = multi_msix_capable(control);
827 pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset);
828 bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
829 table_offset &= ~PCI_MSIX_FLAGS_BIRMASK;
830 phys_addr = pci_resource_start (dev, bir) + table_offset;
831 base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
832 if (base == NULL)
833 return -ENOMEM;
835 /* MSI-X Table Initialization */
836 for (i = 0; i < nvec; i++) {
837 entry = alloc_msi_entry();
838 if (!entry)
839 break;
840 vector = get_msi_vector(dev);
841 if (vector < 0) {
842 kmem_cache_free(msi_cachep, entry);
843 break;
844 }
846 j = entries[i].entry;
847 entries[i].vector = vector;
848 entry->msi_attrib.type = PCI_CAP_ID_MSIX;
849 entry->msi_attrib.state = 0; /* Mark it not active */
850 entry->msi_attrib.entry_nr = j;
851 entry->msi_attrib.maskbit = 1;
852 entry->msi_attrib.default_vector = dev->irq;
853 entry->dev = dev;
854 entry->mask_base = base;
855 if (!head) {
856 entry->link.head = vector;
857 entry->link.tail = vector;
858 head = entry;
859 } else {
860 entry->link.head = temp;
861 entry->link.tail = tail->link.tail;
862 tail->link.tail = vector;
863 head->link.head = vector;
864 }
865 temp = vector;
866 tail = entry;
867 /* Replace with MSI-X handler */
868 irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
869 /* Configure MSI-X capability structure */
870 status = msi_ops->setup(dev, vector,
871 &address_hi,
872 &address_lo,
873 &data);
874 if (status < 0)
875 break;
877 writel(address_lo,
878 base + j * PCI_MSIX_ENTRY_SIZE +
879 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
880 writel(address_hi,
881 base + j * PCI_MSIX_ENTRY_SIZE +
882 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
883 writel(data,
884 base + j * PCI_MSIX_ENTRY_SIZE +
885 PCI_MSIX_ENTRY_DATA_OFFSET);
886 attach_msi_entry(entry, vector);
887 }
888 if (i != nvec) {
889 i--;
890 for (; i >= 0; i--) {
891 vector = (entries + i)->vector;
892 msi_free_vector(dev, vector, 0);
893 (entries + i)->vector = 0;
894 }
895 return -EBUSY;
896 }
897 /* Set MSI-X enabled bits */
898 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
900 return 0;
901 }
903 /**
904 * pci_enable_msi - configure device's MSI capability structure
905 * @dev: pointer to the pci_dev data structure of MSI device function
906 *
907 * Setup the MSI capability structure of device function with
908 * a single MSI vector upon its software driver call to request for
909 * MSI mode enabled on its hardware device function. A return of zero
910 * indicates the successful setup of an entry zero with the new MSI
911 * vector or non-zero for otherwise.
912 **/
913 int pci_enable_msi(struct pci_dev* dev)
914 {
915 struct pci_bus *bus;
916 int pos, temp, status = -EINVAL;
917 u16 control;
919 if (!pci_msi_enable || !dev)
920 return status;
922 if (dev->no_msi)
923 return status;
925 for (bus = dev->bus; bus; bus = bus->parent)
926 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
927 return -EINVAL;
929 temp = dev->irq;
931 status = msi_init();
932 if (status < 0)
933 return status;
935 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
936 if (!pos)
937 return -EINVAL;
939 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
940 /* Lookup Sucess */
941 unsigned long flags;
943 pci_read_config_word(dev, msi_control_reg(pos), &control);
944 if (control & PCI_MSI_FLAGS_ENABLE)
945 return 0; /* Already in MSI mode */
946 spin_lock_irqsave(&msi_lock, flags);
947 if (!vector_irq[dev->irq]) {
948 msi_desc[dev->irq]->msi_attrib.state = 0;
949 vector_irq[dev->irq] = -1;
950 nr_released_vectors--;
951 spin_unlock_irqrestore(&msi_lock, flags);
952 status = msi_register_init(dev, msi_desc[dev->irq]);
953 if (status == 0)
954 enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
955 return status;
956 }
957 spin_unlock_irqrestore(&msi_lock, flags);
958 dev->irq = temp;
959 }
960 /* Check whether driver already requested for MSI-X vectors */
961 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
962 if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
963 printk(KERN_INFO "PCI: %s: Can't enable MSI. "
964 "Device already has MSI-X vectors assigned\n",
965 pci_name(dev));
966 dev->irq = temp;
967 return -EINVAL;
968 }
969 status = msi_capability_init(dev);
970 if (!status) {
971 if (!pos)
972 nr_reserved_vectors--; /* Only MSI capable */
973 else if (nr_msix_devices > 0)
974 nr_msix_devices--; /* Both MSI and MSI-X capable,
975 but choose enabling MSI */
976 }
978 return status;
979 }
981 void pci_disable_msi(struct pci_dev* dev)
982 {
983 struct msi_desc *entry;
984 int pos, default_vector;
985 u16 control;
986 unsigned long flags;
988 if (!pci_msi_enable)
989 return;
990 if (!dev)
991 return;
993 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
994 if (!pos)
995 return;
997 pci_read_config_word(dev, msi_control_reg(pos), &control);
998 if (!(control & PCI_MSI_FLAGS_ENABLE))
999 return;
1001 spin_lock_irqsave(&msi_lock, flags);
1002 entry = msi_desc[dev->irq];
1003 if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) {
1004 spin_unlock_irqrestore(&msi_lock, flags);
1005 return;
1007 if (entry->msi_attrib.state) {
1008 spin_unlock_irqrestore(&msi_lock, flags);
1009 printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without "
1010 "free_irq() on MSI vector %d\n",
1011 pci_name(dev), dev->irq);
1012 BUG_ON(entry->msi_attrib.state > 0);
1013 } else {
1014 vector_irq[dev->irq] = 0; /* free it */
1015 nr_released_vectors++;
1016 default_vector = entry->msi_attrib.default_vector;
1017 spin_unlock_irqrestore(&msi_lock, flags);
1018 /* Restore dev->irq to its default pin-assertion vector */
1019 dev->irq = default_vector;
1020 disable_msi_mode(dev, pci_find_capability(dev, PCI_CAP_ID_MSI),
1021 PCI_CAP_ID_MSI);
1025 static int msi_free_vector(struct pci_dev* dev, int vector, int reassign)
1027 struct msi_desc *entry;
1028 int head, entry_nr, type;
1029 void __iomem *base;
1030 unsigned long flags;
1032 msi_ops->teardown(vector);
1034 spin_lock_irqsave(&msi_lock, flags);
1035 entry = msi_desc[vector];
1036 if (!entry || entry->dev != dev) {
1037 spin_unlock_irqrestore(&msi_lock, flags);
1038 return -EINVAL;
1040 type = entry->msi_attrib.type;
1041 entry_nr = entry->msi_attrib.entry_nr;
1042 head = entry->link.head;
1043 base = entry->mask_base;
1044 msi_desc[entry->link.head]->link.tail = entry->link.tail;
1045 msi_desc[entry->link.tail]->link.head = entry->link.head;
1046 entry->dev = NULL;
1047 if (!reassign) {
1048 vector_irq[vector] = 0;
1049 nr_released_vectors++;
1051 msi_desc[vector] = NULL;
1052 spin_unlock_irqrestore(&msi_lock, flags);
1054 kmem_cache_free(msi_cachep, entry);
1056 if (type == PCI_CAP_ID_MSIX) {
1057 if (!reassign)
1058 writel(1, base +
1059 entry_nr * PCI_MSIX_ENTRY_SIZE +
1060 PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
1062 if (head == vector)
1063 iounmap(base);
1066 return 0;
1069 static int reroute_msix_table(int head, struct msix_entry *entries, int *nvec)
1071 int vector = head, tail = 0;
1072 int i, j = 0, nr_entries = 0;
1073 void __iomem *base;
1074 unsigned long flags;
1076 spin_lock_irqsave(&msi_lock, flags);
1077 while (head != tail) {
1078 nr_entries++;
1079 tail = msi_desc[vector]->link.tail;
1080 if (entries[0].entry == msi_desc[vector]->msi_attrib.entry_nr)
1081 j = vector;
1082 vector = tail;
1084 if (*nvec > nr_entries) {
1085 spin_unlock_irqrestore(&msi_lock, flags);
1086 *nvec = nr_entries;
1087 return -EINVAL;
1089 vector = ((j > 0) ? j : head);
1090 for (i = 0; i < *nvec; i++) {
1091 j = msi_desc[vector]->msi_attrib.entry_nr;
1092 msi_desc[vector]->msi_attrib.state = 0; /* Mark it not active */
1093 vector_irq[vector] = -1; /* Mark it busy */
1094 nr_released_vectors--;
1095 entries[i].vector = vector;
1096 if (j != (entries + i)->entry) {
1097 base = msi_desc[vector]->mask_base;
1098 msi_desc[vector]->msi_attrib.entry_nr =
1099 (entries + i)->entry;
1100 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE +
1101 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET), base +
1102 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
1103 PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
1104 writel( readl(base + j * PCI_MSIX_ENTRY_SIZE +
1105 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET), base +
1106 (entries + i)->entry * PCI_MSIX_ENTRY_SIZE +
1107 PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
1108 writel( (readl(base + j * PCI_MSIX_ENTRY_SIZE +
1109 PCI_MSIX_ENTRY_DATA_OFFSET) & 0xff00) | vector,
1110 base + (entries+i)->entry*PCI_MSIX_ENTRY_SIZE +
1111 PCI_MSIX_ENTRY_DATA_OFFSET);
1113 vector = msi_desc[vector]->link.tail;
1115 spin_unlock_irqrestore(&msi_lock, flags);
1117 return 0;
1120 /**
1121 * pci_enable_msix - configure device's MSI-X capability structure
1122 * @dev: pointer to the pci_dev data structure of MSI-X device function
1123 * @entries: pointer to an array of MSI-X entries
1124 * @nvec: number of MSI-X vectors requested for allocation by device driver
1126 * Setup the MSI-X capability structure of device function with the number
1127 * of requested vectors upon its software driver call to request for
1128 * MSI-X mode enabled on its hardware device function. A return of zero
1129 * indicates the successful configuration of MSI-X capability structure
1130 * with new allocated MSI-X vectors. A return of < 0 indicates a failure.
1131 * Or a return of > 0 indicates that driver request is exceeding the number
1132 * of vectors available. Driver should use the returned value to re-send
1133 * its request.
1134 **/
1135 int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
1137 struct pci_bus *bus;
1138 int status, pos, nr_entries, free_vectors;
1139 int i, j, temp;
1140 u16 control;
1141 unsigned long flags;
1143 if (!pci_msi_enable || !dev || !entries)
1144 return -EINVAL;
1146 if (dev->no_msi)
1147 return -EINVAL;
1149 for (bus = dev->bus; bus; bus = bus->parent)
1150 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
1151 return -EINVAL;
1153 status = msi_init();
1154 if (status < 0)
1155 return status;
1157 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
1158 if (!pos)
1159 return -EINVAL;
1161 pci_read_config_word(dev, msi_control_reg(pos), &control);
1162 if (control & PCI_MSIX_FLAGS_ENABLE)
1163 return -EINVAL; /* Already in MSI-X mode */
1165 nr_entries = multi_msix_capable(control);
1166 if (nvec > nr_entries)
1167 return -EINVAL;
1169 /* Check for any invalid entries */
1170 for (i = 0; i < nvec; i++) {
1171 if (entries[i].entry >= nr_entries)
1172 return -EINVAL; /* invalid entry */
1173 for (j = i + 1; j < nvec; j++) {
1174 if (entries[i].entry == entries[j].entry)
1175 return -EINVAL; /* duplicate entry */
1178 temp = dev->irq;
1179 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
1180 /* Lookup Sucess */
1181 nr_entries = nvec;
1182 /* Reroute MSI-X table */
1183 if (reroute_msix_table(dev->irq, entries, &nr_entries)) {
1184 /* #requested > #previous-assigned */
1185 dev->irq = temp;
1186 return nr_entries;
1188 dev->irq = temp;
1189 enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
1190 return 0;
1192 /* Check whether driver already requested for MSI vector */
1193 if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 &&
1194 !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
1195 printk(KERN_INFO "PCI: %s: Can't enable MSI-X. "
1196 "Device already has an MSI vector assigned\n",
1197 pci_name(dev));
1198 dev->irq = temp;
1199 return -EINVAL;
1202 spin_lock_irqsave(&msi_lock, flags);
1203 /*
1204 * msi_lock is provided to ensure that enough vectors resources are
1205 * available before granting.
1206 */
1207 free_vectors = pci_vector_resources(last_alloc_vector,
1208 nr_released_vectors);
1209 /* Ensure that each MSI/MSI-X device has one vector reserved by
1210 default to avoid any MSI-X driver to take all available
1211 resources */
1212 free_vectors -= nr_reserved_vectors;
1213 /* Find the average of free vectors among MSI-X devices */
1214 if (nr_msix_devices > 0)
1215 free_vectors /= nr_msix_devices;
1216 spin_unlock_irqrestore(&msi_lock, flags);
1218 if (nvec > free_vectors) {
1219 if (free_vectors > 0)
1220 return free_vectors;
1221 else
1222 return -EBUSY;
1225 status = msix_capability_init(dev, entries, nvec);
1226 if (!status && nr_msix_devices > 0)
1227 nr_msix_devices--;
1229 return status;
1232 void pci_disable_msix(struct pci_dev* dev)
1234 int pos, temp;
1235 u16 control;
1237 if (!pci_msi_enable)
1238 return;
1239 if (!dev)
1240 return;
1242 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
1243 if (!pos)
1244 return;
1246 pci_read_config_word(dev, msi_control_reg(pos), &control);
1247 if (!(control & PCI_MSIX_FLAGS_ENABLE))
1248 return;
1250 temp = dev->irq;
1251 if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
1252 int state, vector, head, tail = 0, warning = 0;
1253 unsigned long flags;
1255 vector = head = dev->irq;
1256 spin_lock_irqsave(&msi_lock, flags);
1257 while (head != tail) {
1258 state = msi_desc[vector]->msi_attrib.state;
1259 if (state)
1260 warning = 1;
1261 else {
1262 vector_irq[vector] = 0; /* free it */
1263 nr_released_vectors++;
1265 tail = msi_desc[vector]->link.tail;
1266 vector = tail;
1268 spin_unlock_irqrestore(&msi_lock, flags);
1269 if (warning) {
1270 dev->irq = temp;
1271 printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without "
1272 "free_irq() on all MSI-X vectors\n",
1273 pci_name(dev));
1274 BUG_ON(warning > 0);
1275 } else {
1276 dev->irq = temp;
1277 disable_msi_mode(dev,
1278 pci_find_capability(dev, PCI_CAP_ID_MSIX),
1279 PCI_CAP_ID_MSIX);
1285 /**
1286 * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state
1287 * @dev: pointer to the pci_dev data structure of MSI(X) device function
1289 * Being called during hotplug remove, from which the device function
1290 * is hot-removed. All previous assigned MSI/MSI-X vectors, if
1291 * allocated for this device function, are reclaimed to unused state,
1292 * which may be used later on.
1293 **/
1294 void msi_remove_pci_irq_vectors(struct pci_dev* dev)
1296 int state, pos, temp;
1297 unsigned long flags;
1299 if (!pci_msi_enable || !dev)
1300 return;
1302 temp = dev->irq; /* Save IOAPIC IRQ */
1303 pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
1304 if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
1305 spin_lock_irqsave(&msi_lock, flags);
1306 state = msi_desc[dev->irq]->msi_attrib.state;
1307 spin_unlock_irqrestore(&msi_lock, flags);
1308 if (state) {
1309 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
1310 "called without free_irq() on MSI vector %d\n",
1311 pci_name(dev), dev->irq);
1312 BUG_ON(state > 0);
1313 } else /* Release MSI vector assigned to this device */
1314 msi_free_vector(dev, dev->irq, 0);
1315 dev->irq = temp; /* Restore IOAPIC IRQ */
1317 pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
1318 if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
1319 int vector, head, tail = 0, warning = 0;
1320 void __iomem *base = NULL;
1322 vector = head = dev->irq;
1323 while (head != tail) {
1324 spin_lock_irqsave(&msi_lock, flags);
1325 state = msi_desc[vector]->msi_attrib.state;
1326 tail = msi_desc[vector]->link.tail;
1327 base = msi_desc[vector]->mask_base;
1328 spin_unlock_irqrestore(&msi_lock, flags);
1329 if (state)
1330 warning = 1;
1331 else if (vector != head) /* Release MSI-X vector */
1332 msi_free_vector(dev, vector, 0);
1333 vector = tail;
1335 msi_free_vector(dev, vector, 0);
1336 if (warning) {
1337 iounmap(base);
1338 printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
1339 "called without free_irq() on all MSI-X vectors\n",
1340 pci_name(dev));
1341 BUG_ON(warning > 0);
1343 dev->irq = temp; /* Restore IOAPIC IRQ */
1347 void pci_no_msi(void)
1349 pci_msi_enable = 0;
1352 EXPORT_SYMBOL(pci_enable_msi);
1353 EXPORT_SYMBOL(pci_disable_msi);
1354 EXPORT_SYMBOL(pci_enable_msix);
1355 EXPORT_SYMBOL(pci_disable_msix);