ia64/xen-unstable

view xen/drivers/passthrough/vtd/iommu.c @ 17443:80ba1b427032

x86: Emulate accesses to PCI window registers cf8/cfc to synchronise
with accesses by teh hypervisor itself. All users of cf8/cfc go
through new access functions which take the appropriate spinlock.

Based on a patch by Haitao Shan <haitao.shan@intel.com>

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri Apr 11 13:19:55 2008 +0100 (2008-04-11)
parents 5b7a3e040683
children da4042899fd2
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Ashok Raj <ashok.raj@intel.com>
18 * Copyright (C) Shaohua Li <shaohua.li@intel.com>
19 * Copyright (C) Allen Kay <allen.m.kay@intel.com> - adapted to xen
20 */
22 #include <xen/irq.h>
23 #include <xen/sched.h>
24 #include <xen/xmalloc.h>
25 #include <xen/domain_page.h>
26 #include <xen/iommu.h>
27 #include <xen/numa.h>
28 #include <xen/time.h>
29 #include <xen/pci.h>
30 #include <asm/paging.h>
31 #include "iommu.h"
32 #include "dmar.h"
33 #include "../pci_regs.h"
34 #include "msi.h"
35 #include "extern.h"
36 #include "vtd.h"
38 #define domain_iommu_domid(d) ((d)->arch.hvm_domain.hvm_iommu.iommu_domid)
40 static spinlock_t domid_bitmap_lock; /* protect domain id bitmap */
41 static int domid_bitmap_size; /* domain id bitmap size in bits */
42 static unsigned long *domid_bitmap; /* iommu domain id bitmap */
44 #define DID_FIELD_WIDTH 16
45 #define DID_HIGH_OFFSET 8
46 static void context_set_domain_id(struct context_entry *context,
47 struct domain *d)
48 {
49 unsigned long flags;
50 domid_t iommu_domid = domain_iommu_domid(d);
52 if ( iommu_domid == 0 )
53 {
54 spin_lock_irqsave(&domid_bitmap_lock, flags);
55 iommu_domid = find_first_zero_bit(domid_bitmap, domid_bitmap_size);
56 set_bit(iommu_domid, domid_bitmap);
57 spin_unlock_irqrestore(&domid_bitmap_lock, flags);
58 d->arch.hvm_domain.hvm_iommu.iommu_domid = iommu_domid;
59 }
61 context->hi &= (1 << DID_HIGH_OFFSET) - 1;
62 context->hi |= iommu_domid << DID_HIGH_OFFSET;
63 }
65 static void iommu_domid_release(struct domain *d)
66 {
67 domid_t iommu_domid = domain_iommu_domid(d);
69 if ( iommu_domid != 0 )
70 {
71 d->arch.hvm_domain.hvm_iommu.iommu_domid = 0;
72 clear_bit(iommu_domid, domid_bitmap);
73 }
74 }
76 static struct intel_iommu *alloc_intel_iommu(void)
77 {
78 struct intel_iommu *intel;
80 intel = xmalloc(struct intel_iommu);
81 if ( !intel )
82 {
83 gdprintk(XENLOG_ERR VTDPREFIX,
84 "Allocate intel_iommu failed.\n");
85 return NULL;
86 }
87 memset(intel, 0, sizeof(struct intel_iommu));
89 spin_lock_init(&intel->qi_ctrl.qinval_lock);
90 spin_lock_init(&intel->qi_ctrl.qinval_poll_lock);
92 spin_lock_init(&intel->ir_ctrl.iremap_lock);
94 return intel;
95 }
97 static void free_intel_iommu(struct intel_iommu *intel)
98 {
99 if ( intel )
100 {
101 xfree(intel);
102 intel = NULL;
103 }
104 }
106 struct qi_ctrl *iommu_qi_ctrl(struct iommu *iommu)
107 {
108 if ( !iommu )
109 return NULL;
111 if ( !iommu->intel )
112 {
113 iommu->intel = alloc_intel_iommu();
114 if ( !iommu->intel )
115 {
116 dprintk(XENLOG_ERR VTDPREFIX,
117 "iommu_qi_ctrl: Allocate iommu->intel failed.\n");
118 return NULL;
119 }
120 }
122 return &(iommu->intel->qi_ctrl);
123 }
125 struct ir_ctrl *iommu_ir_ctrl(struct iommu *iommu)
126 {
127 if ( !iommu )
128 return NULL;
130 if ( !iommu->intel )
131 {
132 iommu->intel = alloc_intel_iommu();
133 if ( !iommu->intel )
134 {
135 dprintk(XENLOG_ERR VTDPREFIX,
136 "iommu_ir_ctrl: Allocate iommu->intel failed.\n");
137 return NULL;
138 }
139 }
141 return &(iommu->intel->ir_ctrl);
142 }
144 struct iommu_flush *iommu_get_flush(struct iommu *iommu)
145 {
146 if ( !iommu )
147 return NULL;
149 if ( !iommu->intel )
150 {
151 iommu->intel = alloc_intel_iommu();
152 if ( !iommu->intel )
153 {
154 dprintk(XENLOG_ERR VTDPREFIX,
155 "iommu_get_flush: Allocate iommu->intel failed.\n");
156 return NULL;
157 }
158 }
160 return &(iommu->intel->flush);
161 }
163 unsigned int clflush_size;
164 void clflush_cache_range(void *adr, int size)
165 {
166 int i;
167 for ( i = 0; i < size; i += clflush_size )
168 clflush(adr + i);
169 }
171 static void __iommu_flush_cache(struct iommu *iommu, void *addr, int size)
172 {
173 if ( !ecap_coherent(iommu->ecap) )
174 clflush_cache_range(addr, size);
175 }
177 void iommu_flush_cache_entry(struct iommu *iommu, void *addr)
178 {
179 __iommu_flush_cache(iommu, addr, 8);
180 }
182 void iommu_flush_cache_page(struct iommu *iommu, void *addr)
183 {
184 __iommu_flush_cache(iommu, addr, PAGE_SIZE_4K);
185 }
187 int nr_iommus;
188 /* context entry handling */
189 static u64 bus_to_context_maddr(struct iommu *iommu, u8 bus)
190 {
191 struct root_entry *root, *root_entries;
192 unsigned long flags;
193 u64 maddr;
195 spin_lock_irqsave(&iommu->lock, flags);
196 root_entries = (struct root_entry *)map_vtd_domain_page(iommu->root_maddr);
197 root = &root_entries[bus];
198 if ( !root_present(*root) )
199 {
200 maddr = alloc_pgtable_maddr();
201 if ( maddr == 0 )
202 {
203 spin_unlock_irqrestore(&iommu->lock, flags);
204 return 0;
205 }
206 set_root_value(*root, maddr);
207 set_root_present(*root);
208 iommu_flush_cache_entry(iommu, root);
209 }
210 maddr = (u64) get_context_addr(*root);
211 unmap_vtd_domain_page(root_entries);
212 spin_unlock_irqrestore(&iommu->lock, flags);
213 return maddr;
214 }
216 static int device_context_mapped(struct iommu *iommu, u8 bus, u8 devfn)
217 {
218 struct root_entry *root, *root_entries;
219 struct context_entry *context;
220 u64 context_maddr;
221 int ret;
222 unsigned long flags;
224 spin_lock_irqsave(&iommu->lock, flags);
225 root_entries = (struct root_entry *)map_vtd_domain_page(iommu->root_maddr);
226 root = &root_entries[bus];
227 if ( !root_present(*root) )
228 {
229 ret = 0;
230 goto out;
231 }
232 context_maddr = get_context_addr(*root);
233 context = (struct context_entry *)map_vtd_domain_page(context_maddr);
234 ret = context_present(context[devfn]);
235 unmap_vtd_domain_page(context);
236 out:
237 unmap_vtd_domain_page(root_entries);
238 spin_unlock_irqrestore(&iommu->lock, flags);
239 return ret;
240 }
242 static u64 addr_to_dma_page_maddr(struct domain *domain, u64 addr)
243 {
244 struct hvm_iommu *hd = domain_hvm_iommu(domain);
245 struct acpi_drhd_unit *drhd;
246 struct iommu *iommu;
247 int addr_width = agaw_to_width(hd->agaw);
248 struct dma_pte *parent, *pte = NULL;
249 int level = agaw_to_level(hd->agaw);
250 int offset;
251 unsigned long flags;
252 u64 pte_maddr = 0;
253 u64 *vaddr = NULL;
255 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
256 iommu = drhd->iommu;
258 addr &= (((u64)1) << addr_width) - 1;
259 spin_lock_irqsave(&hd->mapping_lock, flags);
260 if ( hd->pgd_maddr == 0 )
261 {
262 hd->pgd_maddr = alloc_pgtable_maddr();
263 if ( hd->pgd_maddr == 0 )
264 return 0;
265 }
267 parent = (struct dma_pte *)map_vtd_domain_page(hd->pgd_maddr);
268 while ( level > 1 )
269 {
270 offset = address_level_offset(addr, level);
271 pte = &parent[offset];
273 if ( dma_pte_addr(*pte) == 0 )
274 {
275 u64 maddr = alloc_pgtable_maddr();
276 dma_set_pte_addr(*pte, maddr);
277 vaddr = map_vtd_domain_page(maddr);
278 if ( !vaddr )
279 {
280 unmap_vtd_domain_page(parent);
281 spin_unlock_irqrestore(&hd->mapping_lock, flags);
282 return 0;
283 }
285 /*
286 * high level table always sets r/w, last level
287 * page table control read/write
288 */
289 dma_set_pte_readable(*pte);
290 dma_set_pte_writable(*pte);
291 iommu_flush_cache_entry(iommu, pte);
292 }
293 else
294 {
295 vaddr = map_vtd_domain_page(pte->val);
296 if ( !vaddr )
297 {
298 unmap_vtd_domain_page(parent);
299 spin_unlock_irqrestore(&hd->mapping_lock, flags);
300 return 0;
301 }
302 }
304 unmap_vtd_domain_page(parent);
305 if ( level == 2 )
306 {
307 pte_maddr = pte->val & PAGE_MASK_4K;
308 unmap_vtd_domain_page(vaddr);
309 break;
310 }
312 parent = (struct dma_pte *)vaddr;
313 vaddr = NULL;
314 level--;
315 }
317 spin_unlock_irqrestore(&hd->mapping_lock, flags);
318 return pte_maddr;
319 }
321 /* return address's page at specific level */
322 static u64 dma_addr_level_page_maddr(
323 struct domain *domain, u64 addr, int level)
324 {
325 struct hvm_iommu *hd = domain_hvm_iommu(domain);
326 struct dma_pte *parent, *pte = NULL;
327 int total = agaw_to_level(hd->agaw);
328 int offset;
329 u64 pg_maddr = hd->pgd_maddr;
331 if ( pg_maddr == 0 )
332 return 0;
334 parent = (struct dma_pte *)map_vtd_domain_page(pg_maddr);
335 while ( level <= total )
336 {
337 offset = address_level_offset(addr, total);
338 pte = &parent[offset];
339 if ( dma_pte_addr(*pte) == 0 )
340 break;
342 pg_maddr = pte->val & PAGE_MASK_4K;
343 unmap_vtd_domain_page(parent);
345 if ( level == total )
346 return pg_maddr;
348 parent = map_vtd_domain_page(pte->val);
349 total--;
350 }
352 unmap_vtd_domain_page(parent);
353 return 0;
354 }
356 static void iommu_flush_write_buffer(struct iommu *iommu)
357 {
358 u32 val;
359 unsigned long flag;
360 s_time_t start_time;
362 if ( !cap_rwbf(iommu->cap) )
363 return;
364 val = iommu->gcmd | DMA_GCMD_WBF;
366 spin_lock_irqsave(&iommu->register_lock, flag);
367 dmar_writel(iommu->reg, DMAR_GCMD_REG, val);
369 /* Make sure hardware complete it */
370 start_time = NOW();
371 for ( ; ; )
372 {
373 val = dmar_readl(iommu->reg, DMAR_GSTS_REG);
374 if ( !(val & DMA_GSTS_WBFS) )
375 break;
376 if ( NOW() > start_time + DMAR_OPERATION_TIMEOUT )
377 panic("DMAR hardware is malfunctional,"
378 " please disable IOMMU\n");
379 cpu_relax();
380 }
381 spin_unlock_irqrestore(&iommu->register_lock, flag);
382 }
384 /* return value determine if we need a write buffer flush */
385 static int flush_context_reg(
386 void *_iommu,
387 u16 did, u16 source_id, u8 function_mask, u64 type,
388 int non_present_entry_flush)
389 {
390 struct iommu *iommu = (struct iommu *) _iommu;
391 u64 val = 0;
392 unsigned long flag;
393 s_time_t start_time;
395 /*
396 * In the non-present entry flush case, if hardware doesn't cache
397 * non-present entry we do nothing and if hardware cache non-present
398 * entry, we flush entries of domain 0 (the domain id is used to cache
399 * any non-present entries)
400 */
401 if ( non_present_entry_flush )
402 {
403 if ( !cap_caching_mode(iommu->cap) )
404 return 1;
405 else
406 did = 0;
407 }
409 /* use register invalidation */
410 switch ( type )
411 {
412 case DMA_CCMD_GLOBAL_INVL:
413 val = DMA_CCMD_GLOBAL_INVL;
414 break;
415 case DMA_CCMD_DOMAIN_INVL:
416 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
417 break;
418 case DMA_CCMD_DEVICE_INVL:
419 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
420 |DMA_CCMD_SID(source_id)|DMA_CCMD_FM(function_mask);
421 break;
422 default:
423 BUG();
424 }
425 val |= DMA_CCMD_ICC;
427 spin_lock_irqsave(&iommu->register_lock, flag);
428 dmar_writeq(iommu->reg, DMAR_CCMD_REG, val);
430 /* Make sure hardware complete it */
431 start_time = NOW();
432 for ( ; ; )
433 {
434 val = dmar_readq(iommu->reg, DMAR_CCMD_REG);
435 if ( !(val & DMA_CCMD_ICC) )
436 break;
437 if ( NOW() > start_time + DMAR_OPERATION_TIMEOUT )
438 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
439 cpu_relax();
440 }
441 spin_unlock_irqrestore(&iommu->register_lock, flag);
442 /* flush context entry will implictly flush write buffer */
443 return 0;
444 }
446 static int inline iommu_flush_context_global(
447 struct iommu *iommu, int non_present_entry_flush)
448 {
449 struct iommu_flush *flush = iommu_get_flush(iommu);
450 return flush->context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
451 non_present_entry_flush);
452 }
454 static int inline iommu_flush_context_domain(
455 struct iommu *iommu, u16 did, int non_present_entry_flush)
456 {
457 struct iommu_flush *flush = iommu_get_flush(iommu);
458 return flush->context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
459 non_present_entry_flush);
460 }
462 static int inline iommu_flush_context_device(
463 struct iommu *iommu, u16 did, u16 source_id,
464 u8 function_mask, int non_present_entry_flush)
465 {
466 struct iommu_flush *flush = iommu_get_flush(iommu);
467 return flush->context(iommu, did, source_id, function_mask,
468 DMA_CCMD_DEVICE_INVL,
469 non_present_entry_flush);
470 }
472 /* return value determine if we need a write buffer flush */
473 static int flush_iotlb_reg(void *_iommu, u16 did,
474 u64 addr, unsigned int size_order, u64 type,
475 int non_present_entry_flush)
476 {
477 struct iommu *iommu = (struct iommu *) _iommu;
478 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
479 u64 val = 0, val_iva = 0;
480 unsigned long flag;
481 s_time_t start_time;
483 /*
484 * In the non-present entry flush case, if hardware doesn't cache
485 * non-present entry we do nothing and if hardware cache non-present
486 * entry, we flush entries of domain 0 (the domain id is used to cache
487 * any non-present entries)
488 */
489 if ( non_present_entry_flush )
490 {
491 if ( !cap_caching_mode(iommu->cap) )
492 return 1;
493 else
494 did = 0;
495 }
497 /* use register invalidation */
498 switch ( type )
499 {
500 case DMA_TLB_GLOBAL_FLUSH:
501 /* global flush doesn't need set IVA_REG */
502 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
503 break;
504 case DMA_TLB_DSI_FLUSH:
505 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
506 break;
507 case DMA_TLB_PSI_FLUSH:
508 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
509 /* Note: always flush non-leaf currently */
510 val_iva = size_order | addr;
511 break;
512 default:
513 BUG();
514 }
515 /* Note: set drain read/write */
516 if ( cap_read_drain(iommu->cap) )
517 val |= DMA_TLB_READ_DRAIN;
518 if ( cap_write_drain(iommu->cap) )
519 val |= DMA_TLB_WRITE_DRAIN;
521 spin_lock_irqsave(&iommu->register_lock, flag);
522 /* Note: Only uses first TLB reg currently */
523 if ( val_iva )
524 dmar_writeq(iommu->reg, tlb_offset, val_iva);
525 dmar_writeq(iommu->reg, tlb_offset + 8, val);
527 /* Make sure hardware complete it */
528 start_time = NOW();
529 for ( ; ; )
530 {
531 val = dmar_readq(iommu->reg, tlb_offset + 8);
532 if ( !(val & DMA_TLB_IVT) )
533 break;
534 if ( NOW() > start_time + DMAR_OPERATION_TIMEOUT )
535 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
536 cpu_relax();
537 }
538 spin_unlock_irqrestore(&iommu->register_lock, flag);
540 /* check IOTLB invalidation granularity */
541 if ( DMA_TLB_IAIG(val) == 0 )
542 printk(KERN_ERR VTDPREFIX "IOMMU: flush IOTLB failed\n");
543 if ( DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type) )
544 printk(KERN_ERR VTDPREFIX "IOMMU: tlb flush request %x, actual %x\n",
545 (u32)DMA_TLB_IIRG(type), (u32)DMA_TLB_IAIG(val));
546 /* flush context entry will implictly flush write buffer */
547 return 0;
548 }
550 static int inline iommu_flush_iotlb_global(struct iommu *iommu,
551 int non_present_entry_flush)
552 {
553 struct iommu_flush *flush = iommu_get_flush(iommu);
554 return flush->iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
555 non_present_entry_flush);
556 }
558 static int inline iommu_flush_iotlb_dsi(struct iommu *iommu, u16 did,
559 int non_present_entry_flush)
560 {
561 struct iommu_flush *flush = iommu_get_flush(iommu);
562 return flush->iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
563 non_present_entry_flush);
564 }
566 static int inline get_alignment(u64 base, unsigned int size)
567 {
568 int t = 0;
569 u64 end;
571 end = base + size - 1;
572 while ( base != end )
573 {
574 t++;
575 base >>= 1;
576 end >>= 1;
577 }
578 return t;
579 }
581 static int inline iommu_flush_iotlb_psi(
582 struct iommu *iommu, u16 did,
583 u64 addr, unsigned int pages, int non_present_entry_flush)
584 {
585 unsigned int align;
586 struct iommu_flush *flush = iommu_get_flush(iommu);
588 BUG_ON(addr & (~PAGE_MASK_4K));
589 BUG_ON(pages == 0);
591 /* Fallback to domain selective flush if no PSI support */
592 if ( !cap_pgsel_inv(iommu->cap) )
593 return iommu_flush_iotlb_dsi(iommu, did,
594 non_present_entry_flush);
596 /*
597 * PSI requires page size is 2 ^ x, and the base address is naturally
598 * aligned to the size
599 */
600 align = get_alignment(addr >> PAGE_SHIFT_4K, pages);
601 /* Fallback to domain selective flush if size is too big */
602 if ( align > cap_max_amask_val(iommu->cap) )
603 return iommu_flush_iotlb_dsi(iommu, did,
604 non_present_entry_flush);
606 addr >>= PAGE_SHIFT_4K + align;
607 addr <<= PAGE_SHIFT_4K + align;
609 return flush->iotlb(iommu, did, addr, align,
610 DMA_TLB_PSI_FLUSH, non_present_entry_flush);
611 }
613 void iommu_flush_all(void)
614 {
615 struct acpi_drhd_unit *drhd;
616 struct iommu *iommu;
618 wbinvd();
619 for_each_drhd_unit ( drhd )
620 {
621 iommu = drhd->iommu;
622 iommu_flush_context_global(iommu, 0);
623 iommu_flush_iotlb_global(iommu, 0);
624 }
625 }
627 /* clear one page's page table */
628 static void dma_pte_clear_one(struct domain *domain, u64 addr)
629 {
630 struct acpi_drhd_unit *drhd;
631 struct iommu *iommu;
632 struct dma_pte *page = NULL, *pte = NULL;
633 u64 pg_maddr;
635 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
637 /* get last level pte */
638 pg_maddr = dma_addr_level_page_maddr(domain, addr, 1);
639 if ( pg_maddr == 0 )
640 return;
641 page = (struct dma_pte *)map_vtd_domain_page(pg_maddr);
642 pte = page + address_level_offset(addr, 1);
643 if ( pte )
644 {
645 dma_clear_pte(*pte);
646 iommu_flush_cache_entry(drhd->iommu, pte);
648 for_each_drhd_unit ( drhd )
649 {
650 iommu = drhd->iommu;
651 if ( cap_caching_mode(iommu->cap) )
652 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(domain),
653 addr, 1, 0);
654 else if (cap_rwbf(iommu->cap))
655 iommu_flush_write_buffer(iommu);
656 }
657 }
658 unmap_vtd_domain_page(page);
659 }
661 /* clear last level pte, a tlb flush should be followed */
662 static void dma_pte_clear_range(struct domain *domain, u64 start, u64 end)
663 {
664 struct hvm_iommu *hd = domain_hvm_iommu(domain);
665 int addr_width = agaw_to_width(hd->agaw);
667 start &= (((u64)1) << addr_width) - 1;
668 end &= (((u64)1) << addr_width) - 1;
669 /* in case it's partial page */
670 start = PAGE_ALIGN_4K(start);
671 end &= PAGE_MASK_4K;
673 /* we don't need lock here, nobody else touches the iova range */
674 while ( start < end )
675 {
676 dma_pte_clear_one(domain, start);
677 start += PAGE_SIZE_4K;
678 }
679 }
681 /* free page table pages. last level pte should already be cleared */
682 void dma_pte_free_pagetable(struct domain *domain, u64 start, u64 end)
683 {
684 struct acpi_drhd_unit *drhd;
685 struct hvm_iommu *hd = domain_hvm_iommu(domain);
686 struct iommu *iommu;
687 int addr_width = agaw_to_width(hd->agaw);
688 struct dma_pte *page, *pte;
689 int total = agaw_to_level(hd->agaw);
690 int level;
691 u32 tmp;
692 u64 pg_maddr;
694 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
695 iommu = drhd->iommu;
697 start &= (((u64)1) << addr_width) - 1;
698 end &= (((u64)1) << addr_width) - 1;
700 /* we don't need lock here, nobody else touches the iova range */
701 level = 2;
702 while ( level <= total )
703 {
704 tmp = align_to_level(start, level);
705 if ( (tmp >= end) || ((tmp + level_size(level)) > end) )
706 return;
708 while ( tmp < end )
709 {
710 pg_maddr = dma_addr_level_page_maddr(domain, tmp, level);
711 if ( pg_maddr == 0 )
712 return;
713 page = (struct dma_pte *)map_vtd_domain_page(pg_maddr);
714 pte = page + address_level_offset(tmp, level);
715 dma_clear_pte(*pte);
716 iommu_flush_cache_entry(iommu, pte);
717 unmap_vtd_domain_page(page);
718 free_pgtable_maddr(pg_maddr);
720 tmp += level_size(level);
721 }
722 level++;
723 }
725 /* free pgd */
726 if ( start == 0 && end == ((((u64)1) << addr_width) - 1) )
727 {
728 free_pgtable_maddr(hd->pgd_maddr);
729 hd->pgd_maddr = 0;
730 }
731 }
733 /* iommu handling */
734 static int iommu_set_root_entry(struct iommu *iommu)
735 {
736 u32 cmd, sts;
737 unsigned long flags;
739 if ( iommu == NULL )
740 {
741 gdprintk(XENLOG_ERR VTDPREFIX,
742 "iommu_set_root_entry: iommu == NULL\n");
743 return -EINVAL;
744 }
746 if ( iommu->root_maddr != 0 )
747 {
748 free_pgtable_maddr(iommu->root_maddr);
749 iommu->root_maddr = 0;
750 }
752 spin_lock_irqsave(&iommu->register_lock, flags);
754 iommu->root_maddr = alloc_pgtable_maddr();
755 if ( iommu->root_maddr == 0 )
756 return -ENOMEM;
758 dmar_writeq(iommu->reg, DMAR_RTADDR_REG, iommu->root_maddr);
759 cmd = iommu->gcmd | DMA_GCMD_SRTP;
760 dmar_writel(iommu->reg, DMAR_GCMD_REG, cmd);
762 /* Make sure hardware complete it */
763 for ( ; ; )
764 {
765 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
766 if ( sts & DMA_GSTS_RTPS )
767 break;
768 cpu_relax();
769 }
771 spin_unlock_irqrestore(&iommu->register_lock, flags);
773 return 0;
774 }
776 static int iommu_enable_translation(struct iommu *iommu)
777 {
778 u32 sts;
779 unsigned long flags;
781 dprintk(XENLOG_INFO VTDPREFIX,
782 "iommu_enable_translation: iommu->reg = %p\n", iommu->reg);
783 spin_lock_irqsave(&iommu->register_lock, flags);
784 iommu->gcmd |= DMA_GCMD_TE;
785 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
786 /* Make sure hardware complete it */
787 for ( ; ; )
788 {
789 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
790 if ( sts & DMA_GSTS_TES )
791 break;
792 cpu_relax();
793 }
795 /* Disable PMRs when VT-d engine takes effect per spec definition */
796 disable_pmr(iommu);
797 spin_unlock_irqrestore(&iommu->register_lock, flags);
798 return 0;
799 }
801 int iommu_disable_translation(struct iommu *iommu)
802 {
803 u32 sts;
804 unsigned long flags;
806 spin_lock_irqsave(&iommu->register_lock, flags);
807 iommu->gcmd &= ~ DMA_GCMD_TE;
808 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
810 /* Make sure hardware complete it */
811 for ( ; ; )
812 {
813 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
814 if ( !(sts & DMA_GSTS_TES) )
815 break;
816 cpu_relax();
817 }
818 spin_unlock_irqrestore(&iommu->register_lock, flags);
819 return 0;
820 }
822 static struct iommu *vector_to_iommu[NR_VECTORS];
823 static int iommu_page_fault_do_one(struct iommu *iommu, int type,
824 u8 fault_reason, u16 source_id, u32 addr)
825 {
826 dprintk(XENLOG_WARNING VTDPREFIX,
827 "iommu_fault:%s: %x:%x.%x addr %x REASON %x iommu->reg = %p\n",
828 (type ? "DMA Read" : "DMA Write"), (source_id >> 8),
829 PCI_SLOT(source_id & 0xFF), PCI_FUNC(source_id & 0xFF), addr,
830 fault_reason, iommu->reg);
832 if ( fault_reason < 0x20 )
833 print_vtd_entries(current->domain, iommu, (source_id >> 8),
834 (source_id & 0xff), (addr >> PAGE_SHIFT));
836 return 0;
837 }
839 static void iommu_fault_status(u32 fault_status)
840 {
841 if ( fault_status & DMA_FSTS_PFO )
842 dprintk(XENLOG_ERR VTDPREFIX,
843 "iommu_fault_status: Fault Overflow\n");
844 else if ( fault_status & DMA_FSTS_PPF )
845 dprintk(XENLOG_ERR VTDPREFIX,
846 "iommu_fault_status: Primary Pending Fault\n");
847 else if ( fault_status & DMA_FSTS_AFO )
848 dprintk(XENLOG_ERR VTDPREFIX,
849 "iommu_fault_status: Advanced Fault Overflow\n");
850 else if ( fault_status & DMA_FSTS_APF )
851 dprintk(XENLOG_ERR VTDPREFIX,
852 "iommu_fault_status: Advanced Pending Fault\n");
853 else if ( fault_status & DMA_FSTS_IQE )
854 dprintk(XENLOG_ERR VTDPREFIX,
855 "iommu_fault_status: Invalidation Queue Error\n");
856 else if ( fault_status & DMA_FSTS_ICE )
857 dprintk(XENLOG_ERR VTDPREFIX,
858 "iommu_fault_status: Invalidation Completion Error\n");
859 else if ( fault_status & DMA_FSTS_ITE )
860 dprintk(XENLOG_ERR VTDPREFIX,
861 "iommu_fault_status: Invalidation Time-out Error\n");
862 }
864 #define PRIMARY_FAULT_REG_LEN (16)
865 static void iommu_page_fault(int vector, void *dev_id,
866 struct cpu_user_regs *regs)
867 {
868 struct iommu *iommu = dev_id;
869 int reg, fault_index;
870 u32 fault_status;
871 unsigned long flags;
873 dprintk(XENLOG_WARNING VTDPREFIX,
874 "iommu_page_fault: iommu->reg = %p\n", iommu->reg);
876 spin_lock_irqsave(&iommu->register_lock, flags);
877 fault_status = dmar_readl(iommu->reg, DMAR_FSTS_REG);
878 spin_unlock_irqrestore(&iommu->register_lock, flags);
880 iommu_fault_status(fault_status);
882 /* FIXME: ignore advanced fault log */
883 if ( !(fault_status & DMA_FSTS_PPF) )
884 return;
885 fault_index = dma_fsts_fault_record_index(fault_status);
886 reg = cap_fault_reg_offset(iommu->cap);
887 for ( ; ; )
888 {
889 u8 fault_reason;
890 u16 source_id;
891 u32 guest_addr, data;
892 int type;
894 /* highest 32 bits */
895 spin_lock_irqsave(&iommu->register_lock, flags);
896 data = dmar_readl(iommu->reg, reg +
897 fault_index * PRIMARY_FAULT_REG_LEN + 12);
898 if ( !(data & DMA_FRCD_F) )
899 {
900 spin_unlock_irqrestore(&iommu->register_lock, flags);
901 break;
902 }
904 fault_reason = dma_frcd_fault_reason(data);
905 type = dma_frcd_type(data);
907 data = dmar_readl(iommu->reg, reg +
908 fault_index * PRIMARY_FAULT_REG_LEN + 8);
909 source_id = dma_frcd_source_id(data);
911 guest_addr = dmar_readq(iommu->reg, reg +
912 fault_index * PRIMARY_FAULT_REG_LEN);
913 guest_addr = dma_frcd_page_addr(guest_addr);
914 /* clear the fault */
915 dmar_writel(iommu->reg, reg +
916 fault_index * PRIMARY_FAULT_REG_LEN + 12, DMA_FRCD_F);
917 spin_unlock_irqrestore(&iommu->register_lock, flags);
919 iommu_page_fault_do_one(iommu, type, fault_reason,
920 source_id, guest_addr);
922 fault_index++;
923 if ( fault_index > cap_num_fault_regs(iommu->cap) )
924 fault_index = 0;
925 }
927 /* clear primary fault overflow */
928 if ( fault_status & DMA_FSTS_PFO )
929 {
930 spin_lock_irqsave(&iommu->register_lock, flags);
931 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
932 spin_unlock_irqrestore(&iommu->register_lock, flags);
933 }
934 }
936 static void dma_msi_unmask(unsigned int vector)
937 {
938 struct iommu *iommu = vector_to_iommu[vector];
939 unsigned long flags;
941 /* unmask it */
942 spin_lock_irqsave(&iommu->register_lock, flags);
943 dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
944 spin_unlock_irqrestore(&iommu->register_lock, flags);
945 }
947 static void dma_msi_mask(unsigned int vector)
948 {
949 unsigned long flags;
950 struct iommu *iommu = vector_to_iommu[vector];
952 /* mask it */
953 spin_lock_irqsave(&iommu->register_lock, flags);
954 dmar_writel(iommu->reg, DMAR_FECTL_REG, DMA_FECTL_IM);
955 spin_unlock_irqrestore(&iommu->register_lock, flags);
956 }
958 static unsigned int dma_msi_startup(unsigned int vector)
959 {
960 dma_msi_unmask(vector);
961 return 0;
962 }
964 static void dma_msi_end(unsigned int vector)
965 {
966 dma_msi_unmask(vector);
967 ack_APIC_irq();
968 }
970 static void dma_msi_data_init(struct iommu *iommu, int vector)
971 {
972 u32 msi_data = 0;
973 unsigned long flags;
975 /* Fixed, edge, assert mode. Follow MSI setting */
976 msi_data |= vector & 0xff;
977 msi_data |= 1 << 14;
979 spin_lock_irqsave(&iommu->register_lock, flags);
980 dmar_writel(iommu->reg, DMAR_FEDATA_REG, msi_data);
981 spin_unlock_irqrestore(&iommu->register_lock, flags);
982 }
984 static void dma_msi_addr_init(struct iommu *iommu, int phy_cpu)
985 {
986 u64 msi_address;
987 unsigned long flags;
989 /* Physical, dedicated cpu. Follow MSI setting */
990 msi_address = (MSI_ADDRESS_HEADER << (MSI_ADDRESS_HEADER_SHIFT + 8));
991 msi_address |= MSI_PHYSICAL_MODE << 2;
992 msi_address |= MSI_REDIRECTION_HINT_MODE << 3;
993 msi_address |= phy_cpu << MSI_TARGET_CPU_SHIFT;
995 spin_lock_irqsave(&iommu->register_lock, flags);
996 dmar_writel(iommu->reg, DMAR_FEADDR_REG, (u32)msi_address);
997 dmar_writel(iommu->reg, DMAR_FEUADDR_REG, (u32)(msi_address >> 32));
998 spin_unlock_irqrestore(&iommu->register_lock, flags);
999 }
1001 static void dma_msi_set_affinity(unsigned int vector, cpumask_t dest)
1003 struct iommu *iommu = vector_to_iommu[vector];
1004 dma_msi_addr_init(iommu, cpu_physical_id(first_cpu(dest)));
1007 static struct hw_interrupt_type dma_msi_type = {
1008 .typename = "DMA_MSI",
1009 .startup = dma_msi_startup,
1010 .shutdown = dma_msi_mask,
1011 .enable = dma_msi_unmask,
1012 .disable = dma_msi_mask,
1013 .ack = dma_msi_mask,
1014 .end = dma_msi_end,
1015 .set_affinity = dma_msi_set_affinity,
1016 };
1018 int iommu_set_interrupt(struct iommu *iommu)
1020 int vector, ret;
1022 vector = assign_irq_vector(AUTO_ASSIGN);
1023 vector_to_iommu[vector] = iommu;
1025 /* VT-d fault is a MSI, make irq == vector */
1026 irq_vector[vector] = vector;
1027 vector_irq[vector] = vector;
1029 if ( !vector )
1031 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: no vectors\n");
1032 return -EINVAL;
1035 irq_desc[vector].handler = &dma_msi_type;
1036 ret = request_irq(vector, iommu_page_fault, 0, "dmar", iommu);
1037 if ( ret )
1038 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: can't request irq\n");
1039 return vector;
1042 struct iommu *iommu_alloc(void *hw_data)
1044 struct acpi_drhd_unit *drhd = (struct acpi_drhd_unit *) hw_data;
1045 struct iommu *iommu;
1047 if ( nr_iommus > MAX_IOMMUS )
1049 gdprintk(XENLOG_ERR VTDPREFIX,
1050 "IOMMU: nr_iommus %d > MAX_IOMMUS\n", nr_iommus);
1051 return NULL;
1054 iommu = xmalloc(struct iommu);
1055 if ( !iommu )
1056 return NULL;
1057 memset(iommu, 0, sizeof(struct iommu));
1059 set_fixmap_nocache(FIX_IOMMU_REGS_BASE_0 + nr_iommus, drhd->address);
1060 iommu->reg = (void *) fix_to_virt(FIX_IOMMU_REGS_BASE_0 + nr_iommus);
1062 printk("iommu_alloc: iommu->reg = %p drhd->address = %lx\n",
1063 iommu->reg, drhd->address);
1065 nr_iommus++;
1067 if ( !iommu->reg )
1069 printk(KERN_ERR VTDPREFIX "IOMMU: can't mapping the region\n");
1070 goto error;
1073 iommu->cap = dmar_readq(iommu->reg, DMAR_CAP_REG);
1074 iommu->ecap = dmar_readq(iommu->reg, DMAR_ECAP_REG);
1076 printk("iommu_alloc: cap = %"PRIx64"\n",iommu->cap);
1077 printk("iommu_alloc: ecap = %"PRIx64"\n", iommu->ecap);
1079 spin_lock_init(&iommu->lock);
1080 spin_lock_init(&iommu->register_lock);
1082 iommu->intel = alloc_intel_iommu();
1084 drhd->iommu = iommu;
1085 return iommu;
1086 error:
1087 xfree(iommu);
1088 return NULL;
1091 static void free_iommu(struct iommu *iommu)
1093 if ( !iommu )
1094 return;
1095 if ( iommu->root_maddr != 0 )
1097 free_pgtable_maddr(iommu->root_maddr);
1098 iommu->root_maddr = 0;
1100 if ( iommu->reg )
1101 iounmap(iommu->reg);
1102 free_intel_iommu(iommu->intel);
1103 free_irq(iommu->vector);
1104 xfree(iommu);
1107 #define guestwidth_to_adjustwidth(gaw) ({ \
1108 int agaw, r = (gaw - 12) % 9; \
1109 agaw = (r == 0) ? gaw : (gaw + 9 - r); \
1110 if ( agaw > 64 ) \
1111 agaw = 64; \
1112 agaw; })
1114 int intel_iommu_domain_init(struct domain *domain)
1116 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1117 struct iommu *iommu = NULL;
1118 int guest_width = DEFAULT_DOMAIN_ADDRESS_WIDTH;
1119 int adjust_width, agaw;
1120 unsigned long sagaw;
1121 struct acpi_drhd_unit *drhd;
1123 if ( !vtd_enabled || list_empty(&acpi_drhd_units) )
1124 return 0;
1126 for_each_drhd_unit ( drhd )
1127 iommu = drhd->iommu ? : iommu_alloc(drhd);
1129 /* calculate AGAW */
1130 if ( guest_width > cap_mgaw(iommu->cap) )
1131 guest_width = cap_mgaw(iommu->cap);
1132 adjust_width = guestwidth_to_adjustwidth(guest_width);
1133 agaw = width_to_agaw(adjust_width);
1134 /* FIXME: hardware doesn't support it, choose a bigger one? */
1135 sagaw = cap_sagaw(iommu->cap);
1136 if ( !test_bit(agaw, &sagaw) )
1138 gdprintk(XENLOG_ERR VTDPREFIX,
1139 "IOMMU: hardware doesn't support the agaw\n");
1140 agaw = find_next_bit(&sagaw, 5, agaw);
1141 if ( agaw >= 5 )
1142 return -ENODEV;
1144 hd->agaw = agaw;
1145 return 0;
1148 static int domain_context_mapping_one(
1149 struct domain *domain,
1150 struct iommu *iommu,
1151 u8 bus, u8 devfn)
1153 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1154 struct context_entry *context, *context_entries;
1155 unsigned long flags;
1156 int ret = 0;
1157 u64 maddr;
1159 maddr = bus_to_context_maddr(iommu, bus);
1160 context_entries = (struct context_entry *)map_vtd_domain_page(maddr);
1161 context = &context_entries[devfn];
1162 if ( !context )
1164 unmap_vtd_domain_page(context_entries);
1165 gdprintk(XENLOG_ERR VTDPREFIX,
1166 "domain_context_mapping_one:context == NULL:"
1167 "bdf = %x:%x:%x\n",
1168 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1169 return -ENOMEM;
1172 if ( context_present(*context) )
1174 unmap_vtd_domain_page(context_entries);
1175 gdprintk(XENLOG_WARNING VTDPREFIX,
1176 "domain_context_mapping_one:context present:bdf=%x:%x:%x\n",
1177 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1178 return 0;
1181 spin_lock_irqsave(&iommu->lock, flags);
1182 /*
1183 * domain_id 0 is not valid on Intel's IOMMU, force domain_id to
1184 * be 1 based as required by intel's iommu hw.
1185 */
1186 context_set_domain_id(context, domain);
1187 context_set_address_width(*context, hd->agaw);
1189 if ( ecap_pass_thru(iommu->ecap) )
1190 context_set_translation_type(*context, CONTEXT_TT_PASS_THRU);
1191 #ifdef CONTEXT_PASSTHRU
1192 else
1194 #endif
1195 ASSERT(hd->pgd_maddr != 0);
1196 context_set_address_root(*context, hd->pgd_maddr);
1197 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1198 #ifdef CONTEXT_PASSTHRU
1200 #endif
1202 context_set_fault_enable(*context);
1203 context_set_present(*context);
1204 iommu_flush_cache_entry(iommu, context);
1206 gdprintk(XENLOG_INFO VTDPREFIX,
1207 "domain_context_mapping_one-%x:%x:%x-*context=%"PRIx64":%"PRIx64
1208 " hd->pgd_maddr=%"PRIx64"\n",
1209 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1210 context->hi, context->lo, hd->pgd_maddr);
1212 unmap_vtd_domain_page(context_entries);
1214 if ( iommu_flush_context_device(iommu, domain_iommu_domid(domain),
1215 (((u16)bus) << 8) | devfn,
1216 DMA_CCMD_MASK_NOBIT, 1) )
1217 iommu_flush_write_buffer(iommu);
1218 else
1219 iommu_flush_iotlb_dsi(iommu, domain_iommu_domid(domain), 0);
1220 spin_unlock_irqrestore(&iommu->lock, flags);
1221 return ret;
1224 static int __pci_find_next_cap(u8 bus, unsigned int devfn, u8 pos, int cap)
1226 u8 id;
1227 int ttl = 48;
1229 while ( ttl-- )
1231 pos = pci_conf_read8(bus, PCI_SLOT(devfn), PCI_FUNC(devfn), pos);
1232 if ( pos < 0x40 )
1233 break;
1235 pos &= ~3;
1236 id = pci_conf_read8(bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1237 pos + PCI_CAP_LIST_ID);
1239 if ( id == 0xff )
1240 break;
1241 if ( id == cap )
1242 return pos;
1244 pos += PCI_CAP_LIST_NEXT;
1246 return 0;
1249 #define PCI_BASE_CLASS_BRIDGE 0x06
1250 #define PCI_CLASS_BRIDGE_PCI 0x0604
1252 #define DEV_TYPE_PCIe_ENDPOINT 1
1253 #define DEV_TYPE_PCI_BRIDGE 2
1254 #define DEV_TYPE_PCI 3
1256 int pdev_type(struct pci_dev *dev)
1258 u16 class_device;
1259 u16 status;
1261 class_device = pci_conf_read16(dev->bus, PCI_SLOT(dev->devfn),
1262 PCI_FUNC(dev->devfn), PCI_CLASS_DEVICE);
1263 if ( class_device == PCI_CLASS_BRIDGE_PCI )
1264 return DEV_TYPE_PCI_BRIDGE;
1266 status = pci_conf_read16(dev->bus, PCI_SLOT(dev->devfn),
1267 PCI_FUNC(dev->devfn), PCI_STATUS);
1269 if ( !(status & PCI_STATUS_CAP_LIST) )
1270 return DEV_TYPE_PCI;
1272 if ( __pci_find_next_cap(dev->bus, dev->devfn,
1273 PCI_CAPABILITY_LIST, PCI_CAP_ID_EXP) )
1274 return DEV_TYPE_PCIe_ENDPOINT;
1276 return DEV_TYPE_PCI;
1279 #define MAX_BUSES 256
1280 struct pci_dev bus2bridge[MAX_BUSES];
1282 static int domain_context_mapping(
1283 struct domain *domain,
1284 struct iommu *iommu,
1285 struct pci_dev *pdev)
1287 int ret = 0;
1288 int dev, func, sec_bus, sub_bus;
1289 u32 type;
1291 type = pdev_type(pdev);
1292 switch ( type )
1294 case DEV_TYPE_PCI_BRIDGE:
1295 sec_bus = pci_conf_read8(
1296 pdev->bus, PCI_SLOT(pdev->devfn),
1297 PCI_FUNC(pdev->devfn), PCI_SECONDARY_BUS);
1299 if ( bus2bridge[sec_bus].bus == 0 )
1301 bus2bridge[sec_bus].bus = pdev->bus;
1302 bus2bridge[sec_bus].devfn = pdev->devfn;
1305 sub_bus = pci_conf_read8(
1306 pdev->bus, PCI_SLOT(pdev->devfn),
1307 PCI_FUNC(pdev->devfn), PCI_SUBORDINATE_BUS);
1309 if ( sec_bus != sub_bus )
1310 gdprintk(XENLOG_WARNING VTDPREFIX,
1311 "context_context_mapping: nested PCI bridge not "
1312 "supported: bdf = %x:%x:%x sec_bus = %x sub_bus = %x\n",
1313 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1314 sec_bus, sub_bus);
1315 break;
1316 case DEV_TYPE_PCIe_ENDPOINT:
1317 gdprintk(XENLOG_INFO VTDPREFIX,
1318 "domain_context_mapping:PCIe : bdf = %x:%x:%x\n",
1319 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1320 ret = domain_context_mapping_one(domain, iommu,
1321 (u8)(pdev->bus), (u8)(pdev->devfn));
1322 break;
1323 case DEV_TYPE_PCI:
1324 gdprintk(XENLOG_INFO VTDPREFIX,
1325 "domain_context_mapping:PCI: bdf = %x:%x:%x\n",
1326 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1328 if ( pdev->bus == 0 )
1329 ret = domain_context_mapping_one(
1330 domain, iommu, (u8)(pdev->bus), (u8)(pdev->devfn));
1331 else
1333 if ( bus2bridge[pdev->bus].bus != 0 )
1334 gdprintk(XENLOG_WARNING VTDPREFIX,
1335 "domain_context_mapping:bus2bridge"
1336 "[%d].bus != 0\n", pdev->bus);
1338 ret = domain_context_mapping_one(
1339 domain, iommu,
1340 (u8)(bus2bridge[pdev->bus].bus),
1341 (u8)(bus2bridge[pdev->bus].devfn));
1343 /* now map everything behind the PCI bridge */
1344 for ( dev = 0; dev < 32; dev++ )
1346 for ( func = 0; func < 8; func++ )
1348 ret = domain_context_mapping_one(
1349 domain, iommu,
1350 pdev->bus, (u8)PCI_DEVFN(dev, func));
1351 if ( ret )
1352 return ret;
1356 break;
1357 default:
1358 gdprintk(XENLOG_ERR VTDPREFIX,
1359 "domain_context_mapping:unknown type : bdf = %x:%x:%x\n",
1360 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1361 ret = -EINVAL;
1362 break;
1365 return ret;
1368 static int domain_context_unmap_one(
1369 struct domain *domain,
1370 struct iommu *iommu,
1371 u8 bus, u8 devfn)
1373 struct context_entry *context, *context_entries;
1374 unsigned long flags;
1375 u64 maddr;
1377 maddr = bus_to_context_maddr(iommu, bus);
1378 context_entries = (struct context_entry *)map_vtd_domain_page(maddr);
1379 context = &context_entries[devfn];
1380 if ( !context )
1382 unmap_vtd_domain_page(context_entries);
1383 gdprintk(XENLOG_ERR VTDPREFIX,
1384 "domain_context_unmap_one-%x:%x:%x- context == NULL:return\n",
1385 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1386 return -ENOMEM;
1389 if ( !context_present(*context) )
1391 unmap_vtd_domain_page(context_entries);
1392 gdprintk(XENLOG_WARNING VTDPREFIX,
1393 "domain_context_unmap_one-%x:%x:%x- "
1394 "context NOT present:return\n",
1395 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1396 return 0;
1399 gdprintk(XENLOG_INFO VTDPREFIX,
1400 "domain_context_unmap_one: bdf = %x:%x:%x\n",
1401 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1403 spin_lock_irqsave(&iommu->lock, flags);
1404 context_clear_present(*context);
1405 context_clear_entry(*context);
1406 iommu_flush_cache_entry(iommu, context);
1407 iommu_flush_context_global(iommu, 0);
1408 iommu_flush_iotlb_global(iommu, 0);
1409 unmap_vtd_domain_page(context_entries);
1410 spin_unlock_irqrestore(&iommu->lock, flags);
1412 return 0;
1415 static int domain_context_unmap(
1416 struct domain *domain,
1417 struct iommu *iommu,
1418 struct pci_dev *pdev)
1420 int ret = 0;
1421 int dev, func, sec_bus, sub_bus;
1422 u32 type;
1424 type = pdev_type(pdev);
1425 switch ( type )
1427 case DEV_TYPE_PCI_BRIDGE:
1428 sec_bus = pci_conf_read8(
1429 pdev->bus, PCI_SLOT(pdev->devfn),
1430 PCI_FUNC(pdev->devfn), PCI_SECONDARY_BUS);
1431 sub_bus = pci_conf_read8(
1432 pdev->bus, PCI_SLOT(pdev->devfn),
1433 PCI_FUNC(pdev->devfn), PCI_SUBORDINATE_BUS);
1435 gdprintk(XENLOG_INFO VTDPREFIX,
1436 "domain_context_unmap:BRIDGE:%x:%x:%x "
1437 "sec_bus=%x sub_bus=%x\n",
1438 pdev->bus, PCI_SLOT(pdev->devfn),
1439 PCI_FUNC(pdev->devfn), sec_bus, sub_bus);
1440 break;
1441 case DEV_TYPE_PCIe_ENDPOINT:
1442 gdprintk(XENLOG_INFO VTDPREFIX,
1443 "domain_context_unmap:PCIe : bdf = %x:%x:%x\n",
1444 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1445 ret = domain_context_unmap_one(domain, iommu,
1446 (u8)(pdev->bus), (u8)(pdev->devfn));
1447 break;
1448 case DEV_TYPE_PCI:
1449 gdprintk(XENLOG_INFO VTDPREFIX,
1450 "domain_context_unmap:PCI: bdf = %x:%x:%x\n",
1451 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1452 if ( pdev->bus == 0 )
1453 ret = domain_context_unmap_one(
1454 domain, iommu,
1455 (u8)(pdev->bus), (u8)(pdev->devfn));
1456 else
1458 if ( bus2bridge[pdev->bus].bus != 0 )
1459 gdprintk(XENLOG_WARNING VTDPREFIX,
1460 "domain_context_unmap:"
1461 "bus2bridge[%d].bus != 0\n", pdev->bus);
1463 ret = domain_context_unmap_one(domain, iommu,
1464 (u8)(bus2bridge[pdev->bus].bus),
1465 (u8)(bus2bridge[pdev->bus].devfn));
1467 /* Unmap everything behind the PCI bridge */
1468 for ( dev = 0; dev < 32; dev++ )
1470 for ( func = 0; func < 8; func++ )
1472 ret = domain_context_unmap_one(
1473 domain, iommu,
1474 pdev->bus, (u8)PCI_DEVFN(dev, func));
1475 if ( ret )
1476 return ret;
1480 break;
1481 default:
1482 gdprintk(XENLOG_ERR VTDPREFIX,
1483 "domain_context_unmap:unknown type: bdf = %x:%x:%x\n",
1484 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1485 ret = -EINVAL;
1486 break;
1489 return ret;
1492 void reassign_device_ownership(
1493 struct domain *source,
1494 struct domain *target,
1495 u8 bus, u8 devfn)
1497 struct hvm_iommu *source_hd = domain_hvm_iommu(source);
1498 struct hvm_iommu *target_hd = domain_hvm_iommu(target);
1499 struct pci_dev *pdev;
1500 struct acpi_drhd_unit *drhd;
1501 struct iommu *iommu;
1502 int status;
1503 unsigned long flags;
1505 gdprintk(XENLOG_INFO VTDPREFIX,
1506 "reassign_device-%x:%x:%x- source = %d target = %d\n",
1507 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1508 source->domain_id, target->domain_id);
1510 pdev_flr(bus, devfn);
1512 for_each_pdev( source, pdev )
1514 if ( (pdev->bus != bus) || (pdev->devfn != devfn) )
1515 continue;
1517 drhd = acpi_find_matched_drhd_unit(pdev);
1518 iommu = drhd->iommu;
1519 domain_context_unmap(source, iommu, pdev);
1521 /* Move pci device from the source domain to target domain. */
1522 spin_lock_irqsave(&source_hd->iommu_list_lock, flags);
1523 spin_lock_irqsave(&target_hd->iommu_list_lock, flags);
1524 list_move(&pdev->list, &target_hd->pdev_list);
1525 spin_unlock_irqrestore(&target_hd->iommu_list_lock, flags);
1526 spin_unlock_irqrestore(&source_hd->iommu_list_lock, flags);
1528 status = domain_context_mapping(target, iommu, pdev);
1529 if ( status != 0 )
1530 gdprintk(XENLOG_ERR VTDPREFIX, "domain_context_mapping failed\n");
1532 break;
1536 void return_devices_to_dom0(struct domain *d)
1538 struct hvm_iommu *hd = domain_hvm_iommu(d);
1539 struct pci_dev *pdev;
1541 while ( !list_empty(&hd->pdev_list) )
1543 pdev = list_entry(hd->pdev_list.next, typeof(*pdev), list);
1544 dprintk(XENLOG_INFO VTDPREFIX,
1545 "return_devices_to_dom0: bdf = %x:%x:%x\n",
1546 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1547 reassign_device_ownership(d, dom0, pdev->bus, pdev->devfn);
1550 #ifdef VTD_DEBUG
1551 for_each_pdev ( dom0, pdev )
1552 dprintk(XENLOG_INFO VTDPREFIX,
1553 "return_devices_to_dom0:%x: bdf = %x:%x:%x\n",
1554 dom0->domain_id, pdev->bus,
1555 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1556 #endif
1559 void iommu_domain_teardown(struct domain *d)
1561 if ( list_empty(&acpi_drhd_units) )
1562 return;
1564 iommu_domid_release(d);
1565 iommu_free_pgd(d);
1566 return_devices_to_dom0(d);
1569 static int domain_context_mapped(struct pci_dev *pdev)
1571 struct acpi_drhd_unit *drhd;
1572 struct iommu *iommu;
1573 int ret;
1575 for_each_drhd_unit ( drhd )
1577 iommu = drhd->iommu;
1578 ret = device_context_mapped(iommu, pdev->bus, pdev->devfn);
1579 if ( ret )
1580 return ret;
1583 return 0;
1586 int intel_iommu_map_page(
1587 struct domain *d, unsigned long gfn, unsigned long mfn)
1589 struct acpi_drhd_unit *drhd;
1590 struct iommu *iommu;
1591 struct dma_pte *page = NULL, *pte = NULL;
1592 u64 pg_maddr;
1594 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1595 iommu = drhd->iommu;
1597 #ifdef CONTEXT_PASSTHRU
1598 /* do nothing if dom0 and iommu supports pass thru */
1599 if ( ecap_pass_thru(iommu->ecap) && (d->domain_id == 0) )
1600 return 0;
1601 #endif
1603 pg_maddr = addr_to_dma_page_maddr(d, gfn << PAGE_SHIFT_4K);
1604 if ( pg_maddr == 0 )
1605 return -ENOMEM;
1606 page = (struct dma_pte *)map_vtd_domain_page(pg_maddr);
1607 pte = page + (gfn & LEVEL_MASK);
1608 dma_set_pte_addr(*pte, (paddr_t)mfn << PAGE_SHIFT_4K);
1609 dma_set_pte_prot(*pte, DMA_PTE_READ | DMA_PTE_WRITE);
1610 iommu_flush_cache_entry(iommu, pte);
1611 unmap_vtd_domain_page(page);
1613 for_each_drhd_unit ( drhd )
1615 iommu = drhd->iommu;
1616 if ( cap_caching_mode(iommu->cap) )
1617 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(d),
1618 (paddr_t)gfn << PAGE_SHIFT_4K, 1, 0);
1619 else if ( cap_rwbf(iommu->cap) )
1620 iommu_flush_write_buffer(iommu);
1623 return 0;
1626 int intel_iommu_unmap_page(struct domain *d, unsigned long gfn)
1628 struct acpi_drhd_unit *drhd;
1629 struct iommu *iommu;
1631 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1632 iommu = drhd->iommu;
1634 #ifdef CONTEXT_PASSTHRU
1635 /* do nothing if dom0 and iommu supports pass thru */
1636 if ( ecap_pass_thru(iommu->ecap) && (d->domain_id == 0) )
1637 return 0;
1638 #endif
1640 dma_pte_clear_one(d, (paddr_t)gfn << PAGE_SHIFT_4K);
1642 return 0;
1645 int iommu_page_mapping(struct domain *domain, paddr_t iova,
1646 void *hpa, size_t size, int prot)
1648 struct acpi_drhd_unit *drhd;
1649 struct iommu *iommu;
1650 unsigned long start_pfn, end_pfn;
1651 struct dma_pte *page = NULL, *pte = NULL;
1652 int index;
1653 u64 pg_maddr;
1655 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1656 iommu = drhd->iommu;
1657 if ( (prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0 )
1658 return -EINVAL;
1659 iova = (iova >> PAGE_SHIFT_4K) << PAGE_SHIFT_4K;
1660 start_pfn = (unsigned long)(((unsigned long) hpa) >> PAGE_SHIFT_4K);
1661 end_pfn = (unsigned long)
1662 ((PAGE_ALIGN_4K(((unsigned long)hpa) + size)) >> PAGE_SHIFT_4K);
1663 index = 0;
1664 while ( start_pfn < end_pfn )
1666 pg_maddr = addr_to_dma_page_maddr(domain, iova + PAGE_SIZE_4K * index);
1667 if ( pg_maddr == 0 )
1668 return -ENOMEM;
1669 page = (struct dma_pte *)map_vtd_domain_page(pg_maddr);
1670 pte = page + (start_pfn & LEVEL_MASK);
1671 dma_set_pte_addr(*pte, start_pfn << PAGE_SHIFT_4K);
1672 dma_set_pte_prot(*pte, prot);
1673 iommu_flush_cache_entry(iommu, pte);
1674 unmap_vtd_domain_page(page);
1675 start_pfn++;
1676 index++;
1679 for_each_drhd_unit ( drhd )
1681 iommu = drhd->iommu;
1682 if ( cap_caching_mode(iommu->cap) )
1683 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(domain),
1684 iova, index, 0);
1685 else if ( cap_rwbf(iommu->cap) )
1686 iommu_flush_write_buffer(iommu);
1689 return 0;
1692 int iommu_page_unmapping(struct domain *domain, paddr_t addr, size_t size)
1694 dma_pte_clear_range(domain, addr, addr + size);
1696 return 0;
1699 void iommu_flush(struct domain *d, unsigned long gfn, u64 *p2m_entry)
1701 struct acpi_drhd_unit *drhd;
1702 struct iommu *iommu = NULL;
1703 struct dma_pte *pte = (struct dma_pte *) p2m_entry;
1705 for_each_drhd_unit ( drhd )
1707 iommu = drhd->iommu;
1708 if ( cap_caching_mode(iommu->cap) )
1709 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(d),
1710 (paddr_t)gfn << PAGE_SHIFT_4K, 1, 0);
1711 else if ( cap_rwbf(iommu->cap) )
1712 iommu_flush_write_buffer(iommu);
1715 iommu_flush_cache_entry(iommu, pte);
1718 static int iommu_prepare_rmrr_dev(
1719 struct domain *d,
1720 struct acpi_rmrr_unit *rmrr,
1721 struct pci_dev *pdev)
1723 struct acpi_drhd_unit *drhd;
1724 unsigned long size;
1725 int ret;
1727 /* page table init */
1728 size = rmrr->end_address - rmrr->base_address + 1;
1729 ret = iommu_page_mapping(d, rmrr->base_address,
1730 (void *)rmrr->base_address, size,
1731 DMA_PTE_READ|DMA_PTE_WRITE);
1732 if ( ret )
1733 return ret;
1735 if ( domain_context_mapped(pdev) == 0 )
1737 drhd = acpi_find_matched_drhd_unit(pdev);
1738 ret = domain_context_mapping(d, drhd->iommu, pdev);
1739 if ( !ret )
1740 return 0;
1743 return ret;
1746 void __init setup_dom0_devices(void)
1748 struct hvm_iommu *hd = domain_hvm_iommu(dom0);
1749 struct acpi_drhd_unit *drhd;
1750 struct pci_dev *pdev;
1751 int bus, dev, func, ret;
1752 u32 l;
1754 #ifdef DEBUG_VTD_CONTEXT_ENTRY
1755 for ( bus = 0; bus < 256; bus++ )
1757 for ( dev = 0; dev < 32; dev++ )
1759 for ( func = 0; func < 8; func++ )
1761 struct context_entry *context;
1762 struct pci_dev device;
1764 device.bus = bus;
1765 device.devfn = PCI_DEVFN(dev, func);
1766 drhd = acpi_find_matched_drhd_unit(&device);
1767 context = device_to_context_entry(drhd->iommu,
1768 bus, PCI_DEVFN(dev, func));
1769 if ( (context->lo != 0) || (context->hi != 0) )
1770 dprintk(XENLOG_INFO VTDPREFIX,
1771 "setup_dom0_devices-%x:%x:%x- context not 0\n",
1772 bus, dev, func);
1776 #endif
1778 for ( bus = 0; bus < 256; bus++ )
1780 for ( dev = 0; dev < 32; dev++ )
1782 for ( func = 0; func < 8; func++ )
1784 l = pci_conf_read32(bus, dev, func, PCI_VENDOR_ID);
1785 /* some broken boards return 0 or ~0 if a slot is empty: */
1786 if ( (l == 0xffffffff) || (l == 0x00000000) ||
1787 (l == 0x0000ffff) || (l == 0xffff0000) )
1788 continue;
1789 pdev = xmalloc(struct pci_dev);
1790 pdev->bus = bus;
1791 pdev->devfn = PCI_DEVFN(dev, func);
1792 list_add_tail(&pdev->list, &hd->pdev_list);
1794 drhd = acpi_find_matched_drhd_unit(pdev);
1795 ret = domain_context_mapping(dom0, drhd->iommu, pdev);
1796 if ( ret != 0 )
1797 gdprintk(XENLOG_ERR VTDPREFIX,
1798 "domain_context_mapping failed\n");
1803 for_each_pdev ( dom0, pdev )
1804 dprintk(XENLOG_INFO VTDPREFIX,
1805 "setup_dom0_devices: bdf = %x:%x:%x\n",
1806 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1809 void clear_fault_bits(struct iommu *iommu)
1811 u64 val;
1813 val = dmar_readq(
1814 iommu->reg,
1815 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+0x8);
1816 dmar_writeq(
1817 iommu->reg,
1818 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+8,
1819 val);
1820 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_FAULTS);
1823 static int init_vtd_hw(void)
1825 struct acpi_drhd_unit *drhd;
1826 struct iommu *iommu;
1827 struct iommu_flush *flush = NULL;
1828 int vector;
1829 int ret;
1831 for_each_drhd_unit ( drhd )
1833 iommu = drhd->iommu;
1834 ret = iommu_set_root_entry(iommu);
1835 if ( ret )
1837 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: set root entry failed\n");
1838 return -EIO;
1841 vector = iommu_set_interrupt(iommu);
1842 dma_msi_data_init(iommu, vector);
1843 dma_msi_addr_init(iommu, cpu_physical_id(first_cpu(cpu_online_map)));
1844 iommu->vector = vector;
1845 clear_fault_bits(iommu);
1846 dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
1848 /* initialize flush functions */
1849 flush = iommu_get_flush(iommu);
1850 flush->context = flush_context_reg;
1851 flush->iotlb = flush_iotlb_reg;
1853 return 0;
1856 static int init_vtd2_hw(void)
1858 struct acpi_drhd_unit *drhd;
1859 struct iommu *iommu;
1861 for_each_drhd_unit ( drhd )
1863 iommu = drhd->iommu;
1864 if ( qinval_setup(iommu) != 0 )
1865 dprintk(XENLOG_ERR VTDPREFIX,
1866 "Queued Invalidation hardware not found\n");
1869 for_each_drhd_unit ( drhd )
1871 iommu = drhd->iommu;
1872 if ( intremap_setup(iommu) != 0 )
1873 dprintk(XENLOG_ERR VTDPREFIX,
1874 "Interrupt Remapping hardware not found\n");
1876 return 0;
1879 static int enable_vtd_translation(void)
1881 struct acpi_drhd_unit *drhd;
1882 struct iommu *iommu;
1884 for_each_drhd_unit ( drhd )
1886 iommu = drhd->iommu;
1887 if ( iommu_enable_translation(iommu) )
1888 return -EIO;
1890 return 0;
1893 static void setup_dom0_rmrr(void)
1895 struct acpi_rmrr_unit *rmrr;
1896 struct pci_dev *pdev;
1897 int ret;
1899 for_each_rmrr_device ( rmrr, pdev )
1900 ret = iommu_prepare_rmrr_dev(dom0, rmrr, pdev);
1901 if ( ret )
1902 gdprintk(XENLOG_ERR VTDPREFIX,
1903 "IOMMU: mapping reserved region failed\n");
1904 end_for_each_rmrr_device ( rmrr, pdev )
1907 int iommu_setup(void)
1909 struct hvm_iommu *hd = domain_hvm_iommu(dom0);
1910 struct acpi_drhd_unit *drhd;
1911 struct iommu *iommu;
1912 unsigned long i;
1914 if ( !vtd_enabled )
1915 return 0;
1917 spin_lock_init(&domid_bitmap_lock);
1918 INIT_LIST_HEAD(&hd->pdev_list);
1920 /* setup clflush size */
1921 clflush_size = get_clflush_size();
1923 /* Allocate IO page directory page for the domain. */
1924 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1925 iommu = drhd->iommu;
1927 /* Allocate domain id bitmap, and set bit 0 as reserved */
1928 domid_bitmap_size = cap_ndoms(iommu->cap);
1929 domid_bitmap = xmalloc_array(unsigned long,
1930 BITS_TO_LONGS(domid_bitmap_size));
1931 if ( domid_bitmap == NULL )
1932 goto error;
1933 memset(domid_bitmap, 0, domid_bitmap_size / 8);
1934 set_bit(0, domid_bitmap);
1936 /* setup 1:1 page table for dom0 */
1937 for ( i = 0; i < max_page; i++ )
1938 iommu_map_page(dom0, i, i);
1940 init_vtd_hw();
1941 setup_dom0_devices();
1942 setup_dom0_rmrr();
1943 iommu_flush_all();
1944 enable_vtd_translation();
1945 init_vtd2_hw();
1947 return 0;
1949 error:
1950 printk("iommu_setup() failed\n");
1951 for_each_drhd_unit ( drhd )
1953 iommu = drhd->iommu;
1954 free_iommu(iommu);
1956 return -EIO;
1959 /*
1960 * If the device isn't owned by dom0, it means it already
1961 * has been assigned to other domain, or it's not exist.
1962 */
1963 int device_assigned(u8 bus, u8 devfn)
1965 struct pci_dev *pdev;
1967 for_each_pdev( dom0, pdev )
1968 if ( (pdev->bus == bus ) && (pdev->devfn == devfn) )
1969 return 0;
1971 return 1;
1974 int intel_iommu_assign_device(struct domain *d, u8 bus, u8 devfn)
1976 struct acpi_rmrr_unit *rmrr;
1977 struct pci_dev *pdev;
1978 int ret = 0;
1980 if ( list_empty(&acpi_drhd_units) )
1981 return ret;
1983 gdprintk(XENLOG_INFO VTDPREFIX,
1984 "assign_device: bus = %x dev = %x func = %x\n",
1985 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1987 reassign_device_ownership(dom0, d, bus, devfn);
1989 /* Setup rmrr identify mapping */
1990 for_each_rmrr_device( rmrr, pdev )
1991 if ( pdev->bus == bus && pdev->devfn == devfn )
1993 /* FIXME: Because USB RMRR conflicts with guest bios region,
1994 * ignore USB RMRR temporarily.
1995 */
1996 if ( is_usb_device(pdev) )
1997 return 0;
1999 ret = iommu_prepare_rmrr_dev(d, rmrr, pdev);
2000 if ( ret )
2002 gdprintk(XENLOG_ERR VTDPREFIX,
2003 "IOMMU: mapping reserved region failed\n");
2004 return ret;
2007 end_for_each_rmrr_device(rmrr, pdev)
2009 return ret;
2012 u8 iommu_state[MAX_IOMMU_REGS * MAX_IOMMUS];
2013 int iommu_suspend(void)
2015 struct acpi_drhd_unit *drhd;
2016 struct iommu *iommu;
2017 int i = 0;
2019 iommu_flush_all();
2021 for_each_drhd_unit ( drhd )
2023 iommu = drhd->iommu;
2024 iommu_state[DMAR_RTADDR_REG * i] =
2025 (u64) dmar_readq(iommu->reg, DMAR_RTADDR_REG);
2026 iommu_state[DMAR_FECTL_REG * i] =
2027 (u32) dmar_readl(iommu->reg, DMAR_FECTL_REG);
2028 iommu_state[DMAR_FEDATA_REG * i] =
2029 (u32) dmar_readl(iommu->reg, DMAR_FEDATA_REG);
2030 iommu_state[DMAR_FEADDR_REG * i] =
2031 (u32) dmar_readl(iommu->reg, DMAR_FEADDR_REG);
2032 iommu_state[DMAR_FEUADDR_REG * i] =
2033 (u32) dmar_readl(iommu->reg, DMAR_FEUADDR_REG);
2034 iommu_state[DMAR_PLMBASE_REG * i] =
2035 (u32) dmar_readl(iommu->reg, DMAR_PLMBASE_REG);
2036 iommu_state[DMAR_PLMLIMIT_REG * i] =
2037 (u32) dmar_readl(iommu->reg, DMAR_PLMLIMIT_REG);
2038 iommu_state[DMAR_PHMBASE_REG * i] =
2039 (u64) dmar_readq(iommu->reg, DMAR_PHMBASE_REG);
2040 iommu_state[DMAR_PHMLIMIT_REG * i] =
2041 (u64) dmar_readq(iommu->reg, DMAR_PHMLIMIT_REG);
2042 i++;
2045 return 0;
2048 int iommu_resume(void)
2050 struct acpi_drhd_unit *drhd;
2051 struct iommu *iommu;
2052 int i = 0;
2054 iommu_flush_all();
2056 init_vtd_hw();
2057 for_each_drhd_unit ( drhd )
2059 iommu = drhd->iommu;
2060 dmar_writeq( iommu->reg, DMAR_RTADDR_REG,
2061 (u64) iommu_state[DMAR_RTADDR_REG * i]);
2062 dmar_writel(iommu->reg, DMAR_FECTL_REG,
2063 (u32) iommu_state[DMAR_FECTL_REG * i]);
2064 dmar_writel(iommu->reg, DMAR_FEDATA_REG,
2065 (u32) iommu_state[DMAR_FEDATA_REG * i]);
2066 dmar_writel(iommu->reg, DMAR_FEADDR_REG,
2067 (u32) iommu_state[DMAR_FEADDR_REG * i]);
2068 dmar_writel(iommu->reg, DMAR_FEUADDR_REG,
2069 (u32) iommu_state[DMAR_FEUADDR_REG * i]);
2070 dmar_writel(iommu->reg, DMAR_PLMBASE_REG,
2071 (u32) iommu_state[DMAR_PLMBASE_REG * i]);
2072 dmar_writel(iommu->reg, DMAR_PLMLIMIT_REG,
2073 (u32) iommu_state[DMAR_PLMLIMIT_REG * i]);
2074 dmar_writeq(iommu->reg, DMAR_PHMBASE_REG,
2075 (u64) iommu_state[DMAR_PHMBASE_REG * i]);
2076 dmar_writeq(iommu->reg, DMAR_PHMLIMIT_REG,
2077 (u64) iommu_state[DMAR_PHMLIMIT_REG * i]);
2079 if ( iommu_enable_translation(iommu) )
2080 return -EIO;
2081 i++;
2083 return 0;
2086 struct iommu_ops intel_iommu_ops = {
2087 .init = intel_iommu_domain_init,
2088 .assign_device = intel_iommu_assign_device,
2089 .teardown = iommu_domain_teardown,
2090 .map_page = intel_iommu_map_page,
2091 .unmap_page = intel_iommu_unmap_page,
2092 .reassign_device = reassign_device_ownership,
2093 };
2095 /*
2096 * Local variables:
2097 * mode: C
2098 * c-set-style: "BSD"
2099 * c-basic-offset: 4
2100 * tab-width: 4
2101 * indent-tabs-mode: nil
2102 * End:
2103 */