ia64/xen-unstable

view xen/arch/x86/hvm/vmx/vtd/intel-iommu.c @ 15939:39c85fa942aa

VTD: Small fix to iommu initialisation.
Signed-off-by: Allen Kay <allen.m.kay@intel.com>
Signed-off-by: Guy Zana <guy@neocleus.com>
author Keir Fraser <keir@xensource.com>
date Thu Sep 20 09:42:01 2007 +0100 (2007-09-20)
parents 45548c83daef
children 7bd5b1f55308
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Ashok Raj <ashok.raj@intel.com>
18 * Copyright (C) Shaohua Li <shaohua.li@intel.com>
19 * Copyright (C) Allen Kay <allen.m.kay@intel.com> - adapted to xen
20 */
22 #include <xen/init.h>
23 #include <xen/irq.h>
24 #include <xen/spinlock.h>
25 #include <xen/sched.h>
26 #include <xen/xmalloc.h>
27 #include <xen/domain_page.h>
28 #include <asm/delay.h>
29 #include <asm/string.h>
30 #include <asm/mm.h>
31 #include <asm/iommu.h>
32 #include <asm/hvm/vmx/intel-iommu.h>
33 #include "dmar.h"
34 #include "pci-direct.h"
35 #include "pci_regs.h"
36 #include "msi.h"
38 #define VTDPREFIX
39 extern void print_iommu_regs(struct acpi_drhd_unit *drhd);
40 extern void print_vtd_entries(struct domain *d, int bus, int devfn,
41 unsigned long gmfn);
43 #define DMAR_OPERATION_TIMEOUT (HZ*60) /* 1m */
45 #define time_after(a,b) \
46 (typecheck(unsigned long, a) && \
47 typecheck(unsigned long, b) && \
48 ((long)(b) - (long)(a) < 0))
50 unsigned int x86_clflush_size;
51 void clflush_cache_range(void *adr, int size)
52 {
53 int i;
54 for (i = 0; i < size; i += x86_clflush_size)
55 clflush(adr + i);
56 }
58 static void __iommu_flush_cache(struct iommu *iommu, void *addr, int size)
59 {
60 if (!ecap_coherent(iommu->ecap))
61 clflush_cache_range(addr, size);
62 }
64 #define iommu_flush_cache_entry(iommu, addr) \
65 __iommu_flush_cache(iommu, addr, 8)
66 #define iommu_flush_cache_page(iommu, addr) \
67 __iommu_flush_cache(iommu, addr, PAGE_SIZE_4K)
69 int nr_iommus;
70 /* context entry handling */
71 static struct context_entry * device_to_context_entry(struct iommu *iommu,
72 u8 bus, u8 devfn)
73 {
74 struct root_entry *root;
75 struct context_entry *context;
76 unsigned long phy_addr;
77 unsigned long flags;
79 spin_lock_irqsave(&iommu->lock, flags);
80 root = &iommu->root_entry[bus];
81 if (!root_present(*root)) {
82 phy_addr = (unsigned long) alloc_xenheap_page();
83 if (!phy_addr) {
84 spin_unlock_irqrestore(&iommu->lock, flags);
85 return NULL;
86 }
87 memset((void *) phy_addr, 0, PAGE_SIZE);
88 iommu_flush_cache_page(iommu, (void *)phy_addr);
89 phy_addr = virt_to_maddr((void *)phy_addr);
90 set_root_value(*root, phy_addr);
91 set_root_present(*root);
92 iommu_flush_cache_entry(iommu, root);
93 }
94 phy_addr = (unsigned long) get_context_addr(*root);
95 context = (struct context_entry *)maddr_to_virt(phy_addr);
96 spin_unlock_irqrestore(&iommu->lock, flags);
97 return &context[devfn];
98 }
100 static int device_context_mapped(struct iommu *iommu, u8 bus, u8 devfn)
101 {
102 struct root_entry *root;
103 struct context_entry *context;
104 unsigned long phy_addr;
105 int ret;
106 unsigned long flags;
108 spin_lock_irqsave(&iommu->lock, flags);
109 root = &iommu->root_entry[bus];
110 if (!root_present(*root)) {
111 ret = 0;
112 goto out;
113 }
114 phy_addr = get_context_addr(*root);
115 context = (struct context_entry *)maddr_to_virt(phy_addr);
116 ret = context_present(context[devfn]);
117 out:
118 spin_unlock_irqrestore(&iommu->lock, flags);
119 return ret;
120 }
122 /* page table handling */
123 #define LEVEL_STRIDE (9)
124 #define LEVEL_MASK ((1 << LEVEL_STRIDE) - 1)
125 #define agaw_to_level(val) ((val) + 2)
126 #define agaw_to_width(val) (30 + val * LEVEL_STRIDE)
127 #define width_to_agaw(w) ((w - 30)/LEVEL_STRIDE)
128 #define level_to_offset_bits(l) (12 + (l - 1) * LEVEL_STRIDE)
129 #define address_level_offset(addr, level) \
130 ((addr >> level_to_offset_bits(level)) & LEVEL_MASK)
131 #define level_mask(l) (((u64)(-1)) << level_to_offset_bits(l))
132 #define level_size(l) (1 << level_to_offset_bits(l))
133 #define align_to_level(addr, l) ((addr + level_size(l) - 1) & level_mask(l))
134 static struct dma_pte * addr_to_dma_pte(struct domain *domain, u64 addr)
135 {
136 struct hvm_iommu *hd = domain_hvm_iommu(domain);
137 struct acpi_drhd_unit *drhd;
138 struct iommu *iommu;
139 int addr_width = agaw_to_width(hd->agaw);
140 struct dma_pte *parent, *pte = NULL, *pgd;
141 int level = agaw_to_level(hd->agaw);
142 int offset;
143 unsigned long flags;
145 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
146 iommu = drhd->iommu;
148 addr &= (((u64)1) << addr_width) - 1;
149 spin_lock_irqsave(&hd->mapping_lock, flags);
150 if (!hd->pgd) {
151 pgd = (struct dma_pte *)alloc_xenheap_page();
152 if (!pgd && !hd->pgd) {
153 spin_unlock_irqrestore(&hd->mapping_lock, flags);
154 return NULL;
155 }
156 memset((u8*)pgd, 0, PAGE_SIZE);
157 if (!hd->pgd)
158 hd->pgd = pgd;
159 else /* somebody is fast */
160 free_xenheap_page((void *) pgd);
161 }
162 parent = hd->pgd;
163 while (level > 0) {
164 u8 *tmp;
165 offset = address_level_offset(addr, level);
166 pte = &parent[offset];
167 if (level == 1)
168 break;
169 if (dma_pte_addr(*pte) == 0) {
170 tmp = alloc_xenheap_page();
171 if (tmp == NULL)
172 gdprintk(XENLOG_ERR VTDPREFIX,
173 "addr_to_dma_pte: tmp == NULL\n");
175 memset(tmp, 0, PAGE_SIZE);
176 iommu_flush_cache_page(iommu, tmp);
178 if (!tmp && dma_pte_addr(*pte) == 0) {
179 spin_unlock_irqrestore(&hd->mapping_lock, flags);
180 return NULL;
181 }
182 if (dma_pte_addr(*pte) == 0) {
183 dma_set_pte_addr(*pte,
184 virt_to_maddr(tmp));
185 /*
186 * high level table always sets r/w, last level
187 * page table control read/write
188 */
189 dma_set_pte_readable(*pte);
190 dma_set_pte_writable(*pte);
191 iommu_flush_cache_entry(iommu, pte);
192 } else /* somebody is fast */
193 free_xenheap_page(tmp);
194 }
195 parent = maddr_to_virt(dma_pte_addr(*pte));
196 level--;
197 }
198 spin_unlock_irqrestore(&hd->mapping_lock, flags);
199 return pte;
200 }
202 /* return address's pte at specific level */
203 static struct dma_pte *dma_addr_level_pte(struct domain *domain, u64 addr,
204 int level)
205 {
206 struct hvm_iommu *hd = domain_hvm_iommu(domain);
207 struct dma_pte *parent, *pte = NULL;
208 int total = agaw_to_level(hd->agaw);
209 int offset;
211 parent = hd->pgd;
212 while (level <= total) {
213 offset = address_level_offset(addr, total);
214 pte = &parent[offset];
215 if (level == total)
216 return pte;
218 if (dma_pte_addr(*pte) == 0)
219 break;
220 parent = maddr_to_virt(dma_pte_addr(*pte));
221 total--;
222 }
223 return NULL;
224 }
226 static void iommu_flush_write_buffer(struct iommu *iommu)
227 {
228 u32 val;
229 unsigned long flag;
230 unsigned long start_time;
232 if (!cap_rwbf(iommu->cap))
233 return;
234 val = iommu->gcmd | DMA_GCMD_WBF;
236 spin_lock_irqsave(&iommu->register_lock, flag);
237 dmar_writel(iommu->reg, DMAR_GCMD_REG, val);
239 /* Make sure hardware complete it */
240 start_time = jiffies;
241 while (1) {
242 val = dmar_readl(iommu->reg, DMAR_GSTS_REG);
243 if (!(val & DMA_GSTS_WBFS))
244 break;
245 if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT))
246 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
247 cpu_relax();
248 }
249 spin_unlock_irqrestore(&iommu->register_lock, flag);
250 }
252 /* return value determine if we need a write buffer flush */
253 static int __iommu_flush_context(struct iommu *iommu,
254 u16 did, u16 source_id, u8 function_mask, u64 type,
255 int non_present_entry_flush)
256 {
257 u64 val = 0;
258 unsigned long flag;
259 unsigned long start_time;
261 /*
262 * In the non-present entry flush case, if hardware doesn't cache
263 * non-present entry we do nothing and if hardware cache non-present
264 * entry, we flush entries of domain 0 (the domain id is used to cache
265 * any non-present entries)
266 */
267 if (non_present_entry_flush) {
268 if (!cap_caching_mode(iommu->cap))
269 return 1;
270 else
271 did = 0;
272 }
274 /* use register invalidation */
275 switch (type)
276 {
277 case DMA_CCMD_GLOBAL_INVL:
278 val = DMA_CCMD_GLOBAL_INVL;
279 break;
280 case DMA_CCMD_DOMAIN_INVL:
281 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
282 break;
283 case DMA_CCMD_DEVICE_INVL:
284 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
285 |DMA_CCMD_SID(source_id)|DMA_CCMD_FM(function_mask);
286 break;
287 default:
288 BUG();
289 }
290 val |= DMA_CCMD_ICC;
292 spin_lock_irqsave(&iommu->register_lock, flag);
293 dmar_writeq(iommu->reg, DMAR_CCMD_REG, val);
295 /* Make sure hardware complete it */
296 start_time = jiffies;
297 while (1) {
298 val = dmar_readq(iommu->reg, DMAR_CCMD_REG);
299 if (!(val & DMA_CCMD_ICC))
300 break;
301 if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT))
302 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
303 cpu_relax();
304 }
305 spin_unlock_irqrestore(&iommu->register_lock, flag);
306 /* flush context entry will implictly flush write buffer */
307 return 0;
308 }
310 static int inline iommu_flush_context_global(struct iommu *iommu,
311 int non_present_entry_flush)
312 {
313 return __iommu_flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
314 non_present_entry_flush);
315 }
317 static int inline iommu_flush_context_domain(struct iommu *iommu, u16 did,
318 int non_present_entry_flush)
319 {
320 return __iommu_flush_context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
321 non_present_entry_flush);
322 }
324 static int inline iommu_flush_context_device(struct iommu *iommu,
325 u16 did, u16 source_id, u8 function_mask, int non_present_entry_flush)
326 {
327 return __iommu_flush_context(iommu, did, source_id, function_mask,
328 DMA_CCMD_DEVICE_INVL, non_present_entry_flush);
329 }
331 /* return value determine if we need a write buffer flush */
332 static int __iommu_flush_iotlb(struct iommu *iommu, u16 did,
333 u64 addr, unsigned int size_order, u64 type,
334 int non_present_entry_flush)
335 {
336 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
337 u64 val = 0, val_iva = 0;
338 unsigned long flag;
339 unsigned long start_time;
341 /*
342 * In the non-present entry flush case, if hardware doesn't cache
343 * non-present entry we do nothing and if hardware cache non-present
344 * entry, we flush entries of domain 0 (the domain id is used to cache
345 * any non-present entries)
346 */
347 if (non_present_entry_flush) {
348 if (!cap_caching_mode(iommu->cap))
349 return 1;
350 else
351 did = 0;
352 }
354 /* use register invalidation */
355 switch (type) {
356 case DMA_TLB_GLOBAL_FLUSH:
357 /* global flush doesn't need set IVA_REG */
358 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
359 break;
360 case DMA_TLB_DSI_FLUSH:
361 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
362 break;
363 case DMA_TLB_PSI_FLUSH:
364 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
365 /* Note: always flush non-leaf currently */
366 val_iva = size_order | addr;
367 break;
368 default:
369 BUG();
370 }
371 /* Note: set drain read/write */
372 #if 0
373 /*
374 * This is probably to be super secure.. Looks like we can
375 * ignore it without any impact.
376 */
377 if (cap_read_drain(iommu->cap))
378 val |= DMA_TLB_READ_DRAIN;
379 #endif
380 if (cap_write_drain(iommu->cap))
381 val |= DMA_TLB_WRITE_DRAIN;
383 spin_lock_irqsave(&iommu->register_lock, flag);
384 /* Note: Only uses first TLB reg currently */
385 if (val_iva)
386 dmar_writeq(iommu->reg, tlb_offset, val_iva);
387 dmar_writeq(iommu->reg, tlb_offset + 8, val);
389 /* Make sure hardware complete it */
390 start_time = jiffies;
391 while (1) {
392 val = dmar_readq(iommu->reg, tlb_offset + 8);
393 if (!(val & DMA_TLB_IVT))
394 break;
395 if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT))
396 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
397 cpu_relax();
398 }
399 spin_unlock_irqrestore(&iommu->register_lock, flag);
401 /* check IOTLB invalidation granularity */
402 if (DMA_TLB_IAIG(val) == 0)
403 printk(KERN_ERR VTDPREFIX "IOMMU: flush IOTLB failed\n");
404 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
405 printk(KERN_ERR VTDPREFIX "IOMMU: tlb flush request %x, actual %x\n",
406 (u32)DMA_TLB_IIRG(type), (u32)DMA_TLB_IAIG(val));
407 /* flush context entry will implictly flush write buffer */
408 return 0;
409 }
411 static int inline iommu_flush_iotlb_global(struct iommu *iommu,
412 int non_present_entry_flush)
413 {
414 return __iommu_flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
415 non_present_entry_flush);
416 }
418 static int inline iommu_flush_iotlb_dsi(struct iommu *iommu, u16 did,
419 int non_present_entry_flush)
420 {
421 return __iommu_flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
422 non_present_entry_flush);
423 }
425 static int inline get_alignment(u64 base, unsigned int size)
426 {
427 int t = 0;
428 u64 end;
430 end = base + size - 1;
431 while (base != end) {
432 t++;
433 base >>= 1;
434 end >>= 1;
435 }
436 return t;
437 }
439 static int inline iommu_flush_iotlb_psi(struct iommu *iommu, u16 did,
440 u64 addr, unsigned int pages, int non_present_entry_flush)
441 {
442 unsigned int align;
444 BUG_ON(addr & (~PAGE_MASK_4K));
445 BUG_ON(pages == 0);
447 /* Fallback to domain selective flush if no PSI support */
448 if (!cap_pgsel_inv(iommu->cap))
449 return iommu_flush_iotlb_dsi(iommu, did,
450 non_present_entry_flush);
452 /*
453 * PSI requires page size is 2 ^ x, and the base address is naturally
454 * aligned to the size
455 */
456 align = get_alignment(addr >> PAGE_SHIFT_4K, pages);
457 /* Fallback to domain selective flush if size is too big */
458 if (align > cap_max_amask_val(iommu->cap))
459 return iommu_flush_iotlb_dsi(iommu, did,
460 non_present_entry_flush);
462 addr >>= PAGE_SHIFT_4K + align;
463 addr <<= PAGE_SHIFT_4K + align;
465 return __iommu_flush_iotlb(iommu, did, addr, align,
466 DMA_TLB_PSI_FLUSH, non_present_entry_flush);
467 }
469 void flush_all(void)
470 {
471 struct acpi_drhd_unit *drhd;
472 struct iommu *iommu;
473 int i = 0;
475 wbinvd();
476 for_each_drhd_unit(drhd) {
477 iommu = drhd->iommu;
478 iommu_flush_context_global(iommu, 0);
479 iommu_flush_iotlb_global(iommu, 0);
480 i++;
481 }
482 }
484 /* clear one page's page table */
485 static void dma_pte_clear_one(struct domain *domain, u64 addr)
486 {
487 struct acpi_drhd_unit *drhd;
488 struct iommu *iommu;
489 struct dma_pte *pte = NULL;
491 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
493 /* get last level pte */
494 pte = dma_addr_level_pte(domain, addr, 1);
496 if (pte) {
497 dma_clear_pte(*pte);
498 iommu_flush_cache_entry(drhd->iommu, pte);
500 for_each_drhd_unit(drhd) {
501 iommu = drhd->iommu;
502 if (cap_caching_mode(iommu->cap))
503 {
504 iommu_flush_iotlb_psi(iommu, domain->domain_id, addr, 1, 0);
505 }
506 else if (cap_rwbf(iommu->cap))
507 iommu_flush_write_buffer(iommu);
508 }
509 }
510 }
512 /* clear last level pte, a tlb flush should be followed */
513 static void dma_pte_clear_range(struct domain *domain, u64 start, u64 end)
514 {
515 struct hvm_iommu *hd = domain_hvm_iommu(domain);
516 int addr_width = agaw_to_width(hd->agaw);
518 start &= (((u64)1) << addr_width) - 1;
519 end &= (((u64)1) << addr_width) - 1;
520 /* in case it's partial page */
521 start = PAGE_ALIGN_4K(start);
522 end &= PAGE_MASK_4K;
524 /* we don't need lock here, nobody else touches the iova range */
525 while (start < end) {
526 dma_pte_clear_one(domain, start);
527 start += PAGE_SIZE_4K;
528 }
529 }
531 /* free page table pages. last level pte should already be cleared */
532 // static void dma_pte_free_pagetable(struct domain *domain, u64 start, u64 end)
533 void dma_pte_free_pagetable(struct domain *domain, u64 start, u64 end)
534 {
535 struct acpi_drhd_unit *drhd;
536 struct hvm_iommu *hd = domain_hvm_iommu(domain);
537 struct iommu *iommu;
538 int addr_width = agaw_to_width(hd->agaw);
539 struct dma_pte *pte;
540 int total = agaw_to_level(hd->agaw);
541 int level;
542 u32 tmp;
544 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
545 iommu = drhd->iommu;
547 start &= (((u64)1) << addr_width) - 1;
548 end &= (((u64)1) << addr_width) - 1;
550 /* we don't need lock here, nobody else touches the iova range */
551 level = 2;
552 while (level <= total) {
553 tmp = align_to_level(start, level);
554 if (tmp >= end || (tmp + level_size(level) > end))
555 return;
557 while (tmp < end) {
558 pte = dma_addr_level_pte(domain, tmp, level);
559 if (pte) {
560 free_xenheap_page((void *) maddr_to_virt(dma_pte_addr(*pte)));
561 dma_clear_pte(*pte);
562 iommu_flush_cache_entry(iommu, pte);
563 }
564 tmp += level_size(level);
565 }
566 level++;
567 }
568 /* free pgd */
569 if (start == 0 && end == ((((u64)1) << addr_width) - 1)) {
570 free_xenheap_page((void *)hd->pgd);
571 hd->pgd = NULL;
572 }
573 }
575 /* iommu handling */
576 static int iommu_set_root_entry(struct iommu *iommu)
577 {
578 void *addr;
579 u32 cmd, sts;
580 struct root_entry *root;
581 unsigned long flags;
583 if (iommu == NULL)
584 gdprintk(XENLOG_ERR VTDPREFIX,
585 "iommu_set_root_entry: iommu == NULL\n");
587 spin_lock_irqsave(&iommu->lock, flags);
588 if (!iommu->root_entry) {
589 spin_unlock_irqrestore(&iommu->lock, flags);
590 root = (struct root_entry *)alloc_xenheap_page();
591 memset((u8*)root, 0, PAGE_SIZE);
592 iommu_flush_cache_page(iommu, root);
593 spin_lock_irqsave(&iommu->lock, flags);
595 if (!root && !iommu->root_entry) {
596 spin_unlock_irqrestore(&iommu->lock, flags);
597 return -ENOMEM;
598 }
600 if (!iommu->root_entry)
601 iommu->root_entry = root;
602 else /* somebody is fast */
603 free_xenheap_page((void *)root);
604 }
605 spin_unlock_irqrestore(&iommu->lock, flags);
607 addr = iommu->root_entry;
608 spin_lock_irqsave(&iommu->register_lock, flags);
609 dmar_writeq(iommu->reg, DMAR_RTADDR_REG, virt_to_maddr(addr));
610 cmd = iommu->gcmd | DMA_GCMD_SRTP;
611 dmar_writel(iommu->reg, DMAR_GCMD_REG, cmd);
613 /* Make sure hardware complete it */
614 while (1) {
615 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
616 if (sts & DMA_GSTS_RTPS)
617 break;
618 cpu_relax();
619 }
620 spin_unlock_irqrestore(&iommu->register_lock, flags);
622 return 0;
623 }
625 static int iommu_enable_translation(struct iommu *iommu)
626 {
627 u32 sts;
628 unsigned long flags;
630 dprintk(XENLOG_INFO VTDPREFIX,
631 "iommu_enable_translation: enabling vt-d translation\n");
632 spin_lock_irqsave(&iommu->register_lock, flags);
633 iommu->gcmd |= DMA_GCMD_TE;
634 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
635 /* Make sure hardware complete it */
636 while (1) {
637 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
638 if (sts & DMA_GSTS_TES) {
639 break;
640 }
641 cpu_relax();
642 }
643 spin_unlock_irqrestore(&iommu->register_lock, flags);
644 return 0;
645 }
647 int iommu_disable_translation(struct iommu *iommu)
648 {
649 u32 sts;
650 unsigned long flags;
652 spin_lock_irqsave(&iommu->register_lock, flags);
653 iommu->gcmd &= ~ DMA_GCMD_TE;
654 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
656 /* Make sure hardware complete it */
657 while(1) {
658 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
659 if (!(sts & DMA_GSTS_TES))
660 break;
661 cpu_relax();
662 }
663 spin_unlock_irqrestore(&iommu->register_lock, flags);
664 return 0;
665 }
667 static struct iommu *vector_to_iommu[NR_VECTORS];
668 static int iommu_page_fault_do_one(struct iommu *iommu, int type,
669 u8 fault_reason, u16 source_id, u32 addr)
670 {
671 dprintk(XENLOG_WARNING VTDPREFIX,
672 "iommu_page_fault:%s: DEVICE %x:%x.%x addr %x REASON %x\n",
673 (type ? "DMA Read" : "DMA Write"),
674 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
675 PCI_FUNC(source_id & 0xFF), addr, fault_reason);
677 print_vtd_entries(current->domain, (source_id >> 8),(source_id & 0xff),
678 (addr >> PAGE_SHIFT));
679 return 0;
680 }
682 #define PRIMARY_FAULT_REG_LEN (16)
683 static void iommu_page_fault(int vector, void *dev_id,
684 struct cpu_user_regs *regs)
685 {
686 struct iommu *iommu = dev_id;
687 int reg, fault_index;
688 u32 fault_status;
689 unsigned long flags;
691 dprintk(XENLOG_WARNING VTDPREFIX,
692 "iommu_page_fault: iommu->reg = %p\n", iommu->reg);
694 spin_lock_irqsave(&iommu->register_lock, flags);
695 fault_status = dmar_readl(iommu->reg, DMAR_FSTS_REG);
696 spin_unlock_irqrestore(&iommu->register_lock, flags);
698 /* FIXME: ignore advanced fault log */
699 if (!(fault_status & DMA_FSTS_PPF))
700 return;
701 fault_index = dma_fsts_fault_record_index(fault_status);
702 reg = cap_fault_reg_offset(iommu->cap);
703 while (1) {
704 u8 fault_reason;
705 u16 source_id;
706 u32 guest_addr;
707 int type;
708 u32 data;
710 /* highest 32 bits */
711 spin_lock_irqsave(&iommu->register_lock, flags);
712 data = dmar_readl(iommu->reg, reg +
713 fault_index * PRIMARY_FAULT_REG_LEN + 12);
714 if (!(data & DMA_FRCD_F)) {
715 spin_unlock_irqrestore(&iommu->register_lock, flags);
716 break;
717 }
719 fault_reason = dma_frcd_fault_reason(data);
720 type = dma_frcd_type(data);
722 data = dmar_readl(iommu->reg, reg +
723 fault_index * PRIMARY_FAULT_REG_LEN + 8);
724 source_id = dma_frcd_source_id(data);
726 guest_addr = dmar_readq(iommu->reg, reg +
727 fault_index * PRIMARY_FAULT_REG_LEN);
728 guest_addr = dma_frcd_page_addr(guest_addr);
729 /* clear the fault */
730 dmar_writel(iommu->reg, reg +
731 fault_index * PRIMARY_FAULT_REG_LEN + 12, DMA_FRCD_F);
732 spin_unlock_irqrestore(&iommu->register_lock, flags);
734 iommu_page_fault_do_one(iommu, type, fault_reason,
735 source_id, guest_addr);
737 fault_index++;
738 if (fault_index > cap_num_fault_regs(iommu->cap))
739 fault_index = 0;
740 }
741 /* clear primary fault overflow */
742 if (fault_status & DMA_FSTS_PFO) {
743 spin_lock_irqsave(&iommu->register_lock, flags);
744 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
745 spin_unlock_irqrestore(&iommu->register_lock, flags);
746 }
747 return;
748 }
750 static void dma_msi_unmask(unsigned int vector)
751 {
752 struct iommu *iommu = vector_to_iommu[vector];
753 unsigned long flags;
755 /* unmask it */
756 spin_lock_irqsave(&iommu->register_lock, flags);
757 dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
758 spin_unlock_irqrestore(&iommu->register_lock, flags);
759 }
761 static void dma_msi_mask(unsigned int vector)
762 {
763 unsigned long flags;
764 struct iommu *iommu = vector_to_iommu[vector];
766 /* mask it */
767 spin_lock_irqsave(&iommu->register_lock, flags);
768 dmar_writel(iommu->reg, DMAR_FECTL_REG, DMA_FECTL_IM);
769 spin_unlock_irqrestore(&iommu->register_lock, flags);
770 }
772 static unsigned int dma_msi_startup(unsigned int vector)
773 {
774 dma_msi_unmask(vector);
775 return 0;
776 }
778 static void dma_msi_end(unsigned int vector)
779 {
780 dma_msi_unmask(vector);
781 ack_APIC_irq();
782 }
784 static void dma_msi_data_init(struct iommu *iommu, int vector)
785 {
786 u32 msi_data = 0;
787 unsigned long flags;
789 /* Fixed, edge, assert mode. Follow MSI setting */
790 msi_data |= vector & 0xff;
791 msi_data |= 1 << 14;
793 spin_lock_irqsave(&iommu->register_lock, flags);
794 dmar_writel(iommu->reg, DMAR_FEDATA_REG, msi_data);
795 spin_unlock_irqrestore(&iommu->register_lock, flags);
796 }
798 static void dma_msi_addr_init(struct iommu *iommu, int phy_cpu)
799 {
800 u64 msi_address;
801 unsigned long flags;
803 /* Physical, dedicated cpu. Follow MSI setting */
804 msi_address = (MSI_ADDRESS_HEADER << (MSI_ADDRESS_HEADER_SHIFT + 8));
805 msi_address |= MSI_PHYSICAL_MODE << 2;
806 msi_address |= MSI_REDIRECTION_HINT_MODE << 3;
807 msi_address |= phy_cpu << MSI_TARGET_CPU_SHIFT;
809 spin_lock_irqsave(&iommu->register_lock, flags);
810 dmar_writel(iommu->reg, DMAR_FEADDR_REG, (u32)msi_address);
811 dmar_writel(iommu->reg, DMAR_FEUADDR_REG, (u32)(msi_address >> 32));
812 spin_unlock_irqrestore(&iommu->register_lock, flags);
813 }
815 static void dma_msi_set_affinity(unsigned int vector, cpumask_t dest)
816 {
817 struct iommu *iommu = vector_to_iommu[vector];
818 dma_msi_addr_init(iommu, cpu_physical_id(first_cpu(dest)));
819 }
821 static struct hw_interrupt_type dma_msi_type = {
822 .typename = "DMA_MSI",
823 .startup = dma_msi_startup,
824 .shutdown = dma_msi_mask,
825 .enable = dma_msi_unmask,
826 .disable = dma_msi_mask,
827 .ack = dma_msi_mask,
828 .end = dma_msi_end,
829 .set_affinity = dma_msi_set_affinity,
830 };
832 int iommu_set_interrupt(struct iommu *iommu)
833 {
834 int vector, ret;
836 vector = assign_irq_vector(AUTO_ASSIGN);
837 vector_to_iommu[vector] = iommu;
839 /* VT-d fault is a MSI, make irq == vector */
840 irq_vector[vector] = vector;
841 vector_irq[vector] = vector;
843 if (!vector) {
844 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: no vectors\n");
845 return -EINVAL;
846 }
848 irq_desc[vector].handler = &dma_msi_type;
849 ret = request_irq(vector, iommu_page_fault, 0, "dmar", iommu);
850 if (ret)
851 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: can't request irq\n");
852 return vector;
853 }
855 struct iommu *iommu_alloc(void *hw_data)
856 {
857 struct acpi_drhd_unit *drhd = (struct acpi_drhd_unit *) hw_data;
858 struct iommu *iommu;
860 if (nr_iommus > MAX_IOMMUS) {
861 gdprintk(XENLOG_ERR VTDPREFIX,
862 "IOMMU: nr_iommus %d > MAX_IOMMUS\n", nr_iommus);
863 return NULL;
864 }
866 iommu = xmalloc(struct iommu);
867 if (!iommu)
868 return NULL;
869 memset(iommu, 0, sizeof(struct iommu));
871 set_fixmap_nocache(FIX_IOMMU_REGS_BASE_0 + nr_iommus, drhd->address);
872 iommu->reg = (void *) fix_to_virt(FIX_IOMMU_REGS_BASE_0 + nr_iommus);
873 dprintk(XENLOG_INFO VTDPREFIX,
874 "iommu_alloc: iommu->reg = %p drhd->address = %lx\n",
875 iommu->reg, drhd->address);
876 nr_iommus++;
878 if (!iommu->reg) {
879 printk(KERN_ERR VTDPREFIX "IOMMU: can't mapping the region\n");
880 goto error;
881 }
883 iommu->cap = dmar_readq(iommu->reg, DMAR_CAP_REG);
884 iommu->ecap = dmar_readq(iommu->reg, DMAR_ECAP_REG);
886 spin_lock_init(&iommu->lock);
887 spin_lock_init(&iommu->register_lock);
889 drhd->iommu = iommu;
890 return iommu;
891 error:
892 xfree(iommu);
893 return NULL;
894 }
896 static void free_iommu(struct iommu *iommu)
897 {
898 if (!iommu)
899 return;
900 if (iommu->root_entry)
901 free_xenheap_page((void *)iommu->root_entry);
902 if (iommu->reg)
903 iounmap(iommu->reg);
904 free_irq(iommu->vector);
905 xfree(iommu);
906 }
908 #define guestwidth_to_adjustwidth(gaw) ({ \
909 int agaw; \
910 int r = (gaw - 12) % 9; \
911 if (r == 0) \
912 agaw = gaw; \
913 else \
914 agaw = gaw + 9 - r; \
915 if (agaw > 64) \
916 agaw = 64; \
917 agaw; })
918 int iommu_domain_init(struct domain *domain)
919 {
920 struct hvm_iommu *hd = domain_hvm_iommu(domain);
921 struct iommu *iommu = NULL;
922 int guest_width = DEFAULT_DOMAIN_ADDRESS_WIDTH;
923 int adjust_width, agaw;
924 unsigned long sagaw;
925 struct acpi_drhd_unit *drhd;
927 spin_lock_init(&hd->mapping_lock);
928 spin_lock_init(&hd->iommu_list_lock);
929 INIT_LIST_HEAD(&hd->pdev_list);
930 INIT_LIST_HEAD(&hd->g2m_ioport_list);
932 if ( !vtd_enabled || list_empty(&acpi_drhd_units) )
933 return 0;
935 for_each_drhd_unit ( drhd )
936 iommu = drhd->iommu ? : iommu_alloc(drhd);
938 /* calculate AGAW */
939 if (guest_width > cap_mgaw(iommu->cap))
940 guest_width = cap_mgaw(iommu->cap);
941 adjust_width = guestwidth_to_adjustwidth(guest_width);
942 agaw = width_to_agaw(adjust_width);
943 /* FIXME: hardware doesn't support it, choose a bigger one? */
944 sagaw = cap_sagaw(iommu->cap);
945 if ( !test_bit(agaw, &sagaw) )
946 {
947 gdprintk(XENLOG_ERR VTDPREFIX,
948 "IOMMU: hardware doesn't support the agaw\n");
949 agaw = find_next_bit(&sagaw, 5, agaw);
950 if (agaw >= 5)
951 return -ENODEV;
952 }
953 hd->agaw = agaw;
954 return 0;
955 }
957 static int domain_context_mapping_one(
958 struct domain *domain,
959 struct iommu *iommu,
960 u8 bus, u8 devfn)
961 {
962 struct hvm_iommu *hd = domain_hvm_iommu(domain);
963 struct context_entry *context;
964 unsigned long flags;
965 int ret = 0;
967 context = device_to_context_entry(iommu, bus, devfn);
968 if (!context) {
969 gdprintk(XENLOG_INFO VTDPREFIX,
970 "domain_context_mapping_one:context == NULL:bdf = %x:%x:%x \n",
971 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
972 return -ENOMEM;
973 }
974 spin_lock_irqsave(&iommu->lock, flags);
975 if (context_present(*context)) {
976 spin_unlock_irqrestore(&iommu->lock, flags);
977 gdprintk(XENLOG_INFO VTDPREFIX,
978 "domain_context_mapping_one:context present:bdf=%x:%x:%x\n",
979 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
980 return 0;
981 }
983 #ifdef VTD_DEBUG
984 dprintk(XENLOG_INFO VTDPREFIX,
985 "context_mapping_one_1-%x:%x:%x-*context = %lx %lx\n",
986 bus, PCI_SLOT(devfn), PCI_FUNC(devfn), context->hi, context->lo);
987 #endif
989 /*
990 * domain_id 0 is not valid on Intel's IOMMU, force domain_id to
991 * be 1 based as required by intel's iommu hw.
992 */
993 context_set_domain_id(*context, domain->domain_id);
994 context_set_address_width(*context, hd->agaw);
996 if (ecap_pass_thru(iommu->ecap))
997 context_set_translation_type(*context, CONTEXT_TT_PASS_THRU);
998 else {
999 context_set_address_root(*context, virt_to_maddr(hd->pgd));
1000 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1003 context_set_fault_enable(*context);
1004 context_set_present(*context);
1005 iommu_flush_cache_entry(iommu, context);
1007 #ifdef VTD_DEBUG
1008 dprintk(XENLOG_INFO VTDPREFIX,
1009 "context_mapping_one_2-%x:%x:%x-*context=%lx %lx hd->pgd = %p\n",
1010 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1011 context->hi, context->lo, hd->pgd);
1012 #endif
1014 if (iommu_flush_context_device(iommu, domain->domain_id,
1015 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT, 1))
1016 iommu_flush_write_buffer(iommu);
1017 else
1018 iommu_flush_iotlb_dsi(iommu, domain->domain_id, 0);
1019 spin_unlock_irqrestore(&iommu->lock, flags);
1020 return ret;
1023 static int __pci_find_next_cap(u8 bus, unsigned int devfn, u8 pos, int cap)
1025 u8 id;
1026 int ttl = 48;
1028 while (ttl--) {
1029 pos = read_pci_config_byte(bus, PCI_SLOT(devfn), PCI_FUNC(devfn), pos);
1030 if (pos < 0x40)
1031 break;
1032 pos &= ~3;
1033 id = read_pci_config_byte(bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1034 pos + PCI_CAP_LIST_ID);
1036 if (id == 0xff)
1037 break;
1038 if (id == cap)
1039 return pos;
1040 pos += PCI_CAP_LIST_NEXT;
1042 return 0;
1045 #define PCI_BASE_CLASS_BRIDGE 0x06
1046 #define PCI_CLASS_BRIDGE_PCI 0x0604
1048 #define DEV_TYPE_PCIe_ENDPOINT 1
1049 #define DEV_TYPE_PCI_BRIDGE 2
1050 #define DEV_TYPE_PCI 3
1052 int pdev_type(struct pci_dev *dev)
1054 u16 class_device;
1055 u16 status;
1057 class_device = read_pci_config_16(dev->bus, PCI_SLOT(dev->devfn),
1058 PCI_FUNC(dev->devfn), PCI_CLASS_DEVICE);
1059 if (class_device == PCI_CLASS_BRIDGE_PCI)
1060 return DEV_TYPE_PCI_BRIDGE;
1062 status = read_pci_config_16(dev->bus, PCI_SLOT(dev->devfn),
1063 PCI_FUNC(dev->devfn), PCI_STATUS);
1065 if (!(status & PCI_STATUS_CAP_LIST))
1066 return DEV_TYPE_PCI;
1068 if (__pci_find_next_cap(dev->bus, dev->devfn, PCI_CAPABILITY_LIST, PCI_CAP_ID_EXP))
1069 return DEV_TYPE_PCIe_ENDPOINT;
1071 return DEV_TYPE_PCI;
1074 #define MAX_BUSES 256
1075 struct pci_dev bus2bridge[MAX_BUSES];
1077 static int domain_context_mapping(
1078 struct domain *domain,
1079 struct iommu *iommu,
1080 struct pci_dev *pdev)
1082 int ret = 0;
1083 int dev, func, sec_bus, sub_bus;
1084 u32 type;
1086 type = pdev_type(pdev);
1087 if (type == DEV_TYPE_PCI_BRIDGE) {
1088 sec_bus = read_pci_config_byte(pdev->bus, PCI_SLOT(pdev->devfn),
1089 PCI_FUNC(pdev->devfn), PCI_SECONDARY_BUS);
1091 if (bus2bridge[sec_bus].bus == 0) {
1092 bus2bridge[sec_bus].bus = pdev->bus;
1093 bus2bridge[sec_bus].devfn = pdev->devfn;
1096 sub_bus = read_pci_config_byte(pdev->bus, PCI_SLOT(pdev->devfn),
1097 PCI_FUNC(pdev->devfn), PCI_SUBORDINATE_BUS);
1099 if (sec_bus != sub_bus) {
1100 dprintk(XENLOG_INFO VTDPREFIX,
1101 "context_mapping: nested PCI bridge not supported\n");
1102 dprintk(XENLOG_INFO VTDPREFIX,
1103 " bdf = %x:%x:%x sec_bus = %x sub_bus = %x\n",
1104 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1105 sec_bus, sub_bus);
1109 if (type == DEV_TYPE_PCIe_ENDPOINT) {
1110 gdprintk(XENLOG_INFO VTDPREFIX,
1111 "domain_context_mapping:PCIe : bdf = %x:%x:%x\n",
1112 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1113 ret = domain_context_mapping_one(domain, iommu,
1114 (u8)(pdev->bus), (u8) (pdev->devfn));
1117 /* PCI devices */
1118 if (type == DEV_TYPE_PCI) {
1119 gdprintk(XENLOG_INFO VTDPREFIX,
1120 "domain_context_mapping:PCI: bdf = %x:%x:%x\n",
1121 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1123 if (pdev->bus == 0)
1124 ret = domain_context_mapping_one(domain, iommu,
1125 (u8)(pdev->bus), (u8) (pdev->devfn));
1126 else {
1127 if (bus2bridge[pdev->bus].bus != 0)
1128 gdprintk(XENLOG_ERR VTDPREFIX,
1129 "domain_context_mapping:bus2bridge[pdev->bus].bus==0\n");
1131 ret = domain_context_mapping_one(domain, iommu,
1132 (u8)(bus2bridge[pdev->bus].bus),
1133 (u8)(bus2bridge[pdev->bus].devfn));
1135 /* now map everything behind the PCI bridge */
1136 for (dev = 0; dev < 32; dev++) {
1137 for (func = 0; func < 8; func++) {
1138 ret = domain_context_mapping_one(domain, iommu,
1139 pdev->bus, (u8)PCI_DEVFN(dev, func));
1140 if (ret)
1141 return ret;
1146 return ret;
1149 static int domain_context_unmap_one(
1150 struct domain *domain,
1151 struct iommu *iommu,
1152 u8 bus, u8 devfn)
1154 struct context_entry *context;
1155 unsigned long flags;
1157 context = device_to_context_entry(iommu, bus, devfn);
1158 if (!context) {
1159 gdprintk(XENLOG_INFO VTDPREFIX,
1160 "domain_context_unmap_one-%x:%x:%x- context == NULL:return\n",
1161 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1162 return -ENOMEM;
1164 spin_lock_irqsave(&iommu->lock, flags);
1165 if (!context_present(*context)) {
1166 spin_unlock_irqrestore(&iommu->lock, flags);
1167 gdprintk(XENLOG_INFO VTDPREFIX,
1168 "domain_context_unmap_one-%x:%x:%x- context NOT present:return\n",
1169 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1170 return 0;
1172 gdprintk(XENLOG_INFO VTDPREFIX,
1173 "domain_context_unmap_one_1:bdf = %x:%x:%x\n",
1174 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1176 context_clear_present(*context);
1177 context_clear_entry(*context);
1178 iommu_flush_cache_entry(iommu, context);
1179 iommu_flush_context_global(iommu, 0);
1180 iommu_flush_iotlb_global(iommu, 0);
1181 spin_unlock_irqrestore(&iommu->lock, flags);
1183 gdprintk(XENLOG_INFO VTDPREFIX,
1184 "domain_context_unmap_one_2:bdf = %x:%x:%x\n",
1185 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1187 return 0;
1190 static int domain_context_unmap(
1191 struct domain *domain,
1192 struct iommu *iommu,
1193 struct pci_dev *pdev)
1195 int ret = 0;
1196 int dev, func, sec_bus, sub_bus;
1197 u32 type;
1199 type = pdev_type(pdev);
1200 if (type == DEV_TYPE_PCI_BRIDGE) {
1201 sec_bus = read_pci_config_byte(pdev->bus, PCI_SLOT(pdev->devfn),
1202 PCI_FUNC(pdev->devfn), PCI_SECONDARY_BUS);
1203 sub_bus = read_pci_config_byte(pdev->bus, PCI_SLOT(pdev->devfn),
1204 PCI_FUNC(pdev->devfn), PCI_SUBORDINATE_BUS);
1206 gdprintk(XENLOG_INFO VTDPREFIX,
1207 "domain_context_unmap:BRIDGE:%x:%x:%x sec_bus=%x sub_bus=%x\n",
1208 pdev->bus, PCI_SLOT(pdev->devfn),
1209 PCI_FUNC(pdev->devfn), sec_bus, sub_bus);
1212 if (type == DEV_TYPE_PCIe_ENDPOINT) {
1213 gdprintk(XENLOG_INFO VTDPREFIX,
1214 "domain_context_unmap:PCIe : bdf = %x:%x:%x\n",
1215 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1216 ret = domain_context_unmap_one(domain, iommu,
1217 (u8)(pdev->bus), (u8) (pdev->devfn));
1220 /* PCI devices */
1221 if (type == DEV_TYPE_PCI) {
1222 gdprintk(XENLOG_INFO VTDPREFIX,
1223 "domain_context_unmap:PCI: bdf = %x:%x:%x\n",
1224 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1225 if (pdev->bus == 0)
1226 ret = domain_context_unmap_one(domain, iommu,
1227 (u8)(pdev->bus), (u8) (pdev->devfn));
1228 else {
1229 if (bus2bridge[pdev->bus].bus != 0)
1230 gdprintk(XENLOG_INFO VTDPREFIX,
1231 "domain_context_mapping:bus2bridge[pdev->bus].bus==0\n");
1233 ret = domain_context_unmap_one(domain, iommu,
1234 (u8)(bus2bridge[pdev->bus].bus),
1235 (u8)(bus2bridge[pdev->bus].devfn));
1237 /* now map everything behind the PCI bridge */
1238 for (dev = 0; dev < 32; dev++) {
1239 for (func = 0; func < 8; func++) {
1240 ret = domain_context_unmap_one(domain, iommu,
1241 pdev->bus, (u8)PCI_DEVFN(dev, func));
1242 if (ret)
1243 return ret;
1248 return ret;
1251 void reassign_device_ownership(
1252 struct domain *source,
1253 struct domain *target,
1254 u8 bus, u8 devfn)
1256 struct hvm_iommu *source_hd = domain_hvm_iommu(source);
1257 struct hvm_iommu *target_hd = domain_hvm_iommu(target);
1258 struct pci_dev *pdev;
1259 struct acpi_drhd_unit *drhd;
1260 struct iommu *iommu;
1261 int status;
1262 unsigned long flags;
1264 gdprintk(XENLOG_ERR VTDPREFIX,
1265 "reassign_device-%x:%x:%x- source = %d target = %d\n",
1266 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1267 source->domain_id, target->domain_id);
1269 for_each_pdev(source, pdev) {
1270 if ( (pdev->bus != bus) || (pdev->devfn != devfn) )
1271 continue;
1273 pdev->bus = bus;
1274 pdev->devfn = devfn;
1275 drhd = acpi_find_matched_drhd_unit(pdev);
1276 iommu = drhd->iommu;
1277 domain_context_unmap(source, iommu, pdev);
1279 /*
1280 * move pci device from the source domain to target domain.
1281 */
1282 spin_lock_irqsave(&source_hd->iommu_list_lock, flags);
1283 spin_lock_irqsave(&target_hd->iommu_list_lock, flags);
1284 list_move(&pdev->list, &target_hd->pdev_list);
1285 spin_unlock_irqrestore(&target_hd->iommu_list_lock, flags);
1286 spin_unlock_irqrestore(&source_hd->iommu_list_lock, flags);
1288 status = domain_context_mapping(target, iommu, pdev);
1289 if (status != 0)
1290 gdprintk(XENLOG_ERR VTDPREFIX, "domain_context_mapping failed\n");
1292 /*
1293 * We are done.
1294 */
1295 break;
1299 void return_devices_to_dom0(struct domain *d)
1301 struct hvm_iommu *hd = domain_hvm_iommu(d);
1302 struct pci_dev *pdev;
1304 while (!list_empty(&hd->pdev_list)) {
1305 pdev = list_entry(hd->pdev_list.next, typeof(*pdev), list);
1306 dprintk(XENLOG_INFO VTDPREFIX,
1307 "return_devices_to_dom0: bdf = %x:%x:%x\n",
1308 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1309 reassign_device_ownership(d, dom0, pdev->bus, pdev->devfn);
1312 #ifdef VTD_DEBUG
1313 for_each_pdev(dom0, pdev) {
1314 dprintk(XENLOG_INFO VTDPREFIX,
1315 "return_devices_to_dom0:%x: bdf = %x:%x:%x\n",
1316 dom0->domain_id, pdev->bus,
1317 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1319 #endif
1322 void iommu_domain_teardown(struct domain *d)
1324 if (list_empty(&acpi_drhd_units))
1325 return;
1327 #if CONFIG_PAGING_LEVELS == 3
1329 struct hvm_iommu *hd = domain_hvm_iommu(d);
1330 int level = agaw_to_level(hd->agaw);
1331 struct dma_pte *pgd = NULL;
1333 switch (level)
1335 case VTD_PAGE_TABLE_LEVEL_3:
1336 if ( hd->pgd )
1337 free_xenheap_page((void *)hd->pgd);
1338 break;
1339 case VTD_PAGE_TABLE_LEVEL_4:
1340 if ( hd->pgd )
1342 pgd = hd->pgd;
1343 if ( pgd[0].val != 0 )
1344 free_xenheap_page((void*)maddr_to_virt(
1345 dma_pte_addr(pgd[0])));
1347 break;
1348 default:
1349 gdprintk(XENLOG_ERR VTDPREFIX,
1350 "Unsupported p2m table sharing level!\n");
1351 break;
1354 #endif
1355 return_devices_to_dom0(d);
1358 static int domain_context_mapped(struct domain *domain, struct pci_dev *pdev)
1360 struct acpi_drhd_unit *drhd;
1361 struct iommu *iommu;
1362 int ret;
1364 for_each_drhd_unit(drhd) {
1365 iommu = drhd->iommu;
1366 ret = device_context_mapped(iommu, pdev->bus, pdev->devfn);
1367 if (ret)
1368 return ret;
1370 return 0;
1373 int iommu_map_page(struct domain *d, paddr_t gfn, paddr_t mfn)
1375 struct acpi_drhd_unit *drhd;
1376 struct iommu *iommu;
1377 struct dma_pte *pte = NULL;
1379 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1380 iommu = drhd->iommu;
1382 /* do nothing if dom0 and iommu supports pass thru */
1383 if (ecap_pass_thru(iommu->ecap) && (d->domain_id == 0))
1384 return 0;
1386 pte = addr_to_dma_pte(d, gfn << PAGE_SHIFT_4K);
1387 if (!pte)
1388 return -ENOMEM;
1389 dma_set_pte_addr(*pte, mfn << PAGE_SHIFT_4K);
1390 dma_set_pte_prot(*pte, DMA_PTE_READ | DMA_PTE_WRITE);
1391 iommu_flush_cache_entry(iommu, pte);
1393 for_each_drhd_unit(drhd) {
1394 iommu = drhd->iommu;
1395 if (cap_caching_mode(iommu->cap))
1396 iommu_flush_iotlb_psi(iommu, d->domain_id,
1397 gfn << PAGE_SHIFT_4K, 1, 0);
1398 else if (cap_rwbf(iommu->cap))
1399 iommu_flush_write_buffer(iommu);
1401 return 0;
1404 int iommu_unmap_page(struct domain *d, dma_addr_t gfn)
1406 struct acpi_drhd_unit *drhd;
1407 struct iommu *iommu;
1408 struct dma_pte *pte = NULL;
1410 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1411 iommu = drhd->iommu;
1413 /* do nothing if dom0 and iommu supports pass thru */
1414 if (ecap_pass_thru(iommu->ecap) && (d->domain_id == 0))
1415 return 0;
1417 /* get last level pte */
1418 pte = dma_addr_level_pte(d, gfn << PAGE_SHIFT_4K, 1);
1419 dma_pte_clear_one(d, gfn << PAGE_SHIFT_4K);
1421 return 0;
1424 int iommu_page_mapping(struct domain *domain, dma_addr_t iova,
1425 void *hpa, size_t size, int prot)
1427 struct acpi_drhd_unit *drhd;
1428 struct iommu *iommu;
1429 unsigned long start_pfn, end_pfn;
1430 struct dma_pte *pte = NULL;
1431 int index;
1433 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1434 iommu = drhd->iommu;
1435 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1436 return -EINVAL;
1437 iova = (iova >> PAGE_SHIFT_4K) << PAGE_SHIFT_4K;
1438 start_pfn = (unsigned long)(((unsigned long) hpa) >> PAGE_SHIFT_4K);
1439 end_pfn = (unsigned long)
1440 ((PAGE_ALIGN_4K(((unsigned long)hpa) + size)) >> PAGE_SHIFT_4K);
1441 index = 0;
1442 while (start_pfn < end_pfn) {
1443 pte = addr_to_dma_pte(domain, iova + PAGE_SIZE_4K * index);
1444 if (!pte)
1445 return -ENOMEM;
1446 dma_set_pte_addr(*pte, start_pfn << PAGE_SHIFT_4K);
1447 dma_set_pte_prot(*pte, prot);
1448 iommu_flush_cache_entry(iommu, pte);
1449 start_pfn++;
1450 index++;
1453 for_each_drhd_unit(drhd) {
1454 iommu = drhd->iommu;
1455 if (cap_caching_mode(iommu->cap))
1456 iommu_flush_iotlb_psi(iommu, domain->domain_id, iova, size, 0);
1457 else if (cap_rwbf(iommu->cap))
1458 iommu_flush_write_buffer(iommu);
1460 return 0;
1463 int iommu_page_unmapping(struct domain *domain, dma_addr_t addr, size_t size)
1465 struct dma_pte *pte = NULL;
1467 /* get last level pte */
1468 pte = dma_addr_level_pte(domain, addr, 1);
1469 dma_pte_clear_range(domain, addr, addr + size);
1471 return 0;
1474 void iommu_flush(struct domain *d, dma_addr_t gfn, u64 *p2m_entry)
1476 struct acpi_drhd_unit *drhd;
1477 struct iommu *iommu = NULL;
1478 struct dma_pte *pte = (struct dma_pte *) p2m_entry;
1480 for_each_drhd_unit(drhd) {
1481 iommu = drhd->iommu;
1482 if (cap_caching_mode(iommu->cap))
1483 iommu_flush_iotlb_psi(iommu, d->domain_id,
1484 gfn << PAGE_SHIFT_4K, 1, 0);
1485 else if (cap_rwbf(iommu->cap))
1486 iommu_flush_write_buffer(iommu);
1488 iommu_flush_cache_entry(iommu, pte);
1491 int
1492 prepare_device(struct domain *domain, struct pci_dev dev)
1494 return 0;
1497 static int iommu_prepare_rmrr_dev(
1498 struct domain *d,
1499 struct acpi_rmrr_unit *rmrr,
1500 struct pci_dev *pdev)
1502 struct acpi_drhd_unit *drhd;
1503 unsigned long size;
1504 int ret;
1506 /* page table init */
1507 size = rmrr->end_address - rmrr->base_address + 1;
1508 ret = iommu_page_mapping(d, rmrr->base_address,
1509 (void *)rmrr->base_address, size,
1510 DMA_PTE_READ|DMA_PTE_WRITE);
1511 if (ret)
1512 return ret;
1514 if (domain_context_mapped(d, pdev) == 0) {
1515 drhd = acpi_find_matched_drhd_unit(pdev);
1516 ret = domain_context_mapping(d, drhd->iommu, pdev);
1517 if (!ret)
1518 return 0;
1520 return ret;
1523 void __init setup_dom0_devices(void)
1525 struct hvm_iommu *hd = domain_hvm_iommu(dom0);
1526 struct acpi_drhd_unit *drhd;
1527 struct pci_dev *pdev;
1528 int bus, dev, func;
1529 u32 l;
1530 u8 hdr_type;
1531 int ret;
1533 #ifdef DEBUG_VTD_CONTEXT_ENTRY
1534 for (bus = 0; bus < 256; bus++) {
1535 for (dev = 0; dev < 32; dev++) {
1536 for (func = 0; func < 8; func++) {
1537 struct context_entry *context;
1538 struct pci_dev device;
1540 device.bus = bus;
1541 device.devfn = PCI_DEVFN(dev, func);
1542 drhd = acpi_find_matched_drhd_unit(&device);
1543 context = device_to_context_entry(drhd->iommu,
1544 bus, PCI_DEVFN(dev, func));
1545 if ((context->lo != 0) || (context->hi != 0))
1546 dprintk(XENLOG_INFO VTDPREFIX,
1547 "setup_dom0_devices-%x:%x:%x- context not 0\n",
1548 bus, dev, func);
1552 #endif
1554 for (bus = 0; bus < 256; bus++) {
1555 for (dev = 0; dev < 32; dev++) {
1556 for (func = 0; func < 8; func++) {
1557 l = read_pci_config(bus, dev, func, PCI_VENDOR_ID);
1558 /* some broken boards return 0 or ~0 if a slot is empty: */
1559 if (l == 0xffffffff || l == 0x00000000 ||
1560 l == 0x0000ffff || l == 0xffff0000)
1561 continue;
1562 pdev = xmalloc(struct pci_dev);
1563 pdev->bus = bus;
1564 pdev->devfn = PCI_DEVFN(dev, func);
1565 list_add_tail(&pdev->list, &hd->pdev_list);
1567 drhd = acpi_find_matched_drhd_unit(pdev);
1568 ret = domain_context_mapping(dom0, drhd->iommu, pdev);
1569 if (ret != 0)
1570 gdprintk(XENLOG_ERR VTDPREFIX,
1571 "domain_context_mapping failed\n");
1573 hdr_type = read_pci_config(bus, dev, func, PCI_HEADER_TYPE);
1574 // if ((hdr_type & 0x8) == 0)
1575 // break;
1579 for_each_pdev(dom0, pdev) {
1580 dprintk(XENLOG_INFO VTDPREFIX,
1581 "setup_dom0_devices: bdf = %x:%x:%x\n",
1582 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1586 void clear_fault_bit(struct iommu *iommu)
1588 u64 val;
1590 val = dmar_readq(
1591 iommu->reg,
1592 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+0x8);
1593 dmar_writeq(
1594 iommu->reg,
1595 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+8,
1596 val);
1597 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
1600 /*
1601 * Called from ACPI discovery code, once all DMAR's and RMRR's are done
1602 * scanning, we need to run through and initialize as much of it as necessary
1603 */
1604 int vtd_enable = 1;
1605 static void setup_vtd_enable(char *s)
1607 if ( !strcmp(s, "0") )
1608 vtd_enable = 0;
1609 else if ( !strcmp(s, "1") )
1610 vtd_enable = 1;
1611 else
1612 dprintk(XENLOG_INFO VTDPREFIX,
1613 "Unknown vtd_enable value specified: '%s'\n", s);
1614 dprintk(XENLOG_INFO VTDPREFIX, "vtd_enable = %x\n", vtd_enable);
1616 custom_param("vtd", setup_vtd_enable);
1618 static int init_vtd_hw(void)
1620 struct acpi_drhd_unit *drhd;
1621 struct iommu *iommu;
1622 int ret;
1624 for_each_drhd_unit(drhd) {
1625 iommu = drhd->iommu;
1626 ret = iommu_set_root_entry(iommu);
1627 if (ret) {
1628 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: set root entry failed\n");
1629 return -EIO;
1632 return 0;
1635 static int enable_vtd_translation(void)
1637 struct acpi_drhd_unit *drhd;
1638 struct iommu *iommu;
1639 int vector = 0;
1641 for_each_drhd_unit(drhd) {
1642 iommu = drhd->iommu;
1643 vector = iommu_set_interrupt(iommu);
1644 dma_msi_data_init(iommu, vector);
1645 dma_msi_addr_init(iommu, cpu_physical_id(first_cpu(cpu_online_map)));
1646 iommu->vector = vector;
1647 clear_fault_bit(iommu);
1648 if (vtd_enable && iommu_enable_translation(iommu))
1649 return -EIO;
1651 return 0;
1654 static void setup_dom0_rmrr(void)
1656 struct acpi_rmrr_unit *rmrr;
1657 struct pci_dev *pdev;
1658 int ret;
1660 for_each_rmrr_device(rmrr, pdev)
1661 ret = iommu_prepare_rmrr_dev(dom0, rmrr, pdev);
1662 if (ret)
1663 gdprintk(XENLOG_ERR VTDPREFIX,
1664 "IOMMU: mapping reserved region failed\n");
1665 end_for_each_rmrr_device(rmrr, pdev)
1668 int iommu_setup(void)
1670 struct hvm_iommu *hd = domain_hvm_iommu(dom0);
1671 struct acpi_drhd_unit *drhd;
1672 struct iommu *iommu;
1673 unsigned long i;
1675 if (!vtd_enabled)
1676 return 0;
1678 INIT_LIST_HEAD(&hd->pdev_list);
1680 /* start from scratch */
1681 flush_all();
1683 /* setup clflush size */
1684 x86_clflush_size = ((cpuid_ebx(1) >> 8) & 0xff) * 8;
1686 /*
1687 * allocate IO page directory page for the domain.
1688 */
1689 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1690 iommu = drhd->iommu;
1692 /* setup 1:1 page table for dom0 */
1693 for (i = 0; i < max_page; i++)
1694 iommu_map_page(dom0, i, i);
1696 if (init_vtd_hw())
1697 goto error;
1698 setup_dom0_devices();
1699 setup_dom0_rmrr();
1700 if (enable_vtd_translation())
1701 goto error;
1703 return 0;
1705 error:
1706 printk("iommu_setup() failed\n");
1707 for_each_drhd_unit(drhd) {
1708 iommu = drhd->iommu;
1709 free_iommu(iommu);
1711 return -EIO;
1714 int assign_device(struct domain *d, u8 bus, u8 devfn)
1716 struct hvm_iommu *hd = domain_hvm_iommu(d);
1717 struct acpi_rmrr_unit *rmrr;
1718 struct pci_dev *pdev;
1719 int ret = 0;
1721 if (list_empty(&acpi_drhd_units))
1722 return ret;
1724 dprintk(XENLOG_INFO VTDPREFIX,
1725 "assign_device: bus = %x dev = %x func = %x\n",
1726 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1728 reassign_device_ownership(dom0, d, bus, devfn);
1730 /* setup rmrr identify mapping just once per domain */
1731 if (list_empty(&hd->pdev_list))
1732 for_each_rmrr_device(rmrr, pdev)
1733 ret = iommu_prepare_rmrr_dev(d, rmrr, pdev);
1734 if (ret)
1735 gdprintk(XENLOG_ERR VTDPREFIX,
1736 "IOMMU: mapping reserved region failed\n");
1737 end_for_each_rmrr_device(rmrr, pdev)
1738 return ret;
1741 void iommu_set_pgd(struct domain *d)
1743 struct hvm_iommu *hd = domain_hvm_iommu(d);
1744 unsigned long p2m_table;
1746 if (hd->pgd) {
1747 gdprintk(XENLOG_INFO VTDPREFIX,
1748 "iommu_set_pgd_1: hd->pgd = %p\n", hd->pgd);
1749 hd->pgd = NULL;
1751 p2m_table = mfn_x(pagetable_get_mfn(d->arch.phys_table));
1753 #if CONFIG_PAGING_LEVELS == 3
1754 if ( !hd->pgd )
1756 int level = agaw_to_level(hd->agaw);
1757 struct dma_pte *pmd = NULL;
1758 struct dma_pte *pgd = NULL;
1759 struct dma_pte *pte = NULL;
1760 l3_pgentry_t *l3e;
1761 unsigned long flags;
1762 int i;
1764 spin_lock_irqsave(&hd->mapping_lock, flags);
1765 if (!hd->pgd) {
1766 pgd = (struct dma_pte *)alloc_xenheap_page();
1767 memset((u8*)pgd, 0, PAGE_SIZE);
1768 if (!hd->pgd)
1769 hd->pgd = pgd;
1770 else /* somebody is fast */
1771 free_xenheap_page((void *) pgd);
1774 l3e = map_domain_page(p2m_table);
1775 switch(level)
1777 case VTD_PAGE_TABLE_LEVEL_3: /* Weybridge */
1778 /* We only support 8 entries for the PAE L3 p2m table */
1779 for ( i = 0; i < 8 ; i++ )
1781 /* Don't create new L2 entry, use ones from p2m table */
1782 pgd[i].val = l3e[i].l3 | _PAGE_PRESENT | _PAGE_RW;
1784 break;
1786 case VTD_PAGE_TABLE_LEVEL_4: /* Stoakley */
1787 /* We allocate one more page for the top vtd page table. */
1788 pmd = (struct dma_pte *)alloc_xenheap_page();
1789 memset((u8*)pmd, 0, PAGE_SIZE);
1790 pte = &pgd[0];
1791 dma_set_pte_addr(*pte, virt_to_maddr(pmd));
1792 dma_set_pte_readable(*pte);
1793 dma_set_pte_writable(*pte);
1795 for ( i = 0; i < 8; i++ )
1797 /* Don't create new L2 entry, use ones from p2m table */
1798 pmd[i].val = l3e[i].l3 | _PAGE_PRESENT | _PAGE_RW;
1800 break;
1801 default:
1802 gdprintk(XENLOG_ERR VTDPREFIX,
1803 "iommu_set_pgd:Unsupported p2m table sharing level!\n");
1804 break;
1806 unmap_domain_page(l3e);
1807 spin_unlock_irqrestore(&hd->mapping_lock, flags);
1809 #elif CONFIG_PAGING_LEVELS == 4
1810 if ( !hd->pgd )
1812 int level = agaw_to_level(hd->agaw);
1813 l3_pgentry_t *l3e;
1814 mfn_t pgd_mfn;
1816 switch (level)
1818 case VTD_PAGE_TABLE_LEVEL_3:
1819 l3e = map_domain_page(p2m_table);
1820 if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
1822 gdprintk(XENLOG_ERR VTDPREFIX,
1823 "iommu_set_pgd: second level wasn't there\n");
1824 unmap_domain_page(l3e);
1825 return;
1827 pgd_mfn = _mfn(l3e_get_pfn(*l3e));
1828 unmap_domain_page(l3e);
1829 hd->pgd = maddr_to_virt(pagetable_get_paddr(
1830 pagetable_from_mfn(pgd_mfn)));
1831 break;
1833 case VTD_PAGE_TABLE_LEVEL_4:
1834 pgd_mfn = _mfn(p2m_table);
1835 hd->pgd = maddr_to_virt(pagetable_get_paddr(
1836 pagetable_from_mfn(pgd_mfn)));
1837 break;
1838 default:
1839 gdprintk(XENLOG_ERR VTDPREFIX,
1840 "iommu_set_pgd:Unsupported p2m table sharing level!\n");
1841 break;
1844 #endif
1845 gdprintk(XENLOG_INFO VTDPREFIX,
1846 "iommu_set_pgd: hd->pgd = %p\n", hd->pgd);
1850 u8 iommu_state[MAX_IOMMU_REGS * MAX_IOMMUS];
1851 int iommu_suspend(void)
1853 struct acpi_drhd_unit *drhd;
1854 struct iommu *iommu;
1855 int i = 0;
1857 if (!vtd_enable)
1858 return 0;
1860 flush_all();
1861 for_each_drhd_unit(drhd) {
1862 iommu = drhd->iommu;
1863 iommu_state[DMAR_RTADDR_REG * i] =
1864 (u64) dmar_readq(iommu->reg, DMAR_RTADDR_REG);
1865 iommu_state[DMAR_FECTL_REG * i] =
1866 (u32) dmar_readl(iommu->reg, DMAR_FECTL_REG);
1867 iommu_state[DMAR_FEDATA_REG * i] =
1868 (u32) dmar_readl(iommu->reg, DMAR_FEDATA_REG);
1869 iommu_state[DMAR_FEADDR_REG * i] =
1870 (u32) dmar_readl(iommu->reg, DMAR_FEADDR_REG);
1871 iommu_state[DMAR_FEUADDR_REG * i] =
1872 (u32) dmar_readl(iommu->reg, DMAR_FEUADDR_REG);
1873 iommu_state[DMAR_PLMBASE_REG * i] =
1874 (u32) dmar_readl(iommu->reg, DMAR_PLMBASE_REG);
1875 iommu_state[DMAR_PLMLIMIT_REG * i] =
1876 (u32) dmar_readl(iommu->reg, DMAR_PLMLIMIT_REG);
1877 iommu_state[DMAR_PHMBASE_REG * i] =
1878 (u64) dmar_readq(iommu->reg, DMAR_PHMBASE_REG);
1879 iommu_state[DMAR_PHMLIMIT_REG * i] =
1880 (u64) dmar_readq(iommu->reg, DMAR_PHMLIMIT_REG);
1881 i++;
1884 return 0;
1887 int iommu_resume(void)
1889 struct acpi_drhd_unit *drhd;
1890 struct iommu *iommu;
1891 int i = 0;
1893 if (!vtd_enable)
1894 return 0;
1896 flush_all();
1898 init_vtd_hw();
1899 for_each_drhd_unit(drhd) {
1900 iommu = drhd->iommu;
1901 dmar_writeq( iommu->reg, DMAR_RTADDR_REG,
1902 (u64) iommu_state[DMAR_RTADDR_REG * i]);
1903 dmar_writel(iommu->reg, DMAR_FECTL_REG,
1904 (u32) iommu_state[DMAR_FECTL_REG * i]);
1905 dmar_writel(iommu->reg, DMAR_FEDATA_REG,
1906 (u32) iommu_state[DMAR_FEDATA_REG * i]);
1907 dmar_writel(iommu->reg, DMAR_FEADDR_REG,
1908 (u32) iommu_state[DMAR_FEADDR_REG * i]);
1909 dmar_writel(iommu->reg, DMAR_FEUADDR_REG,
1910 (u32) iommu_state[DMAR_FEUADDR_REG * i]);
1911 dmar_writel(iommu->reg, DMAR_PLMBASE_REG,
1912 (u32) iommu_state[DMAR_PLMBASE_REG * i]);
1913 dmar_writel(iommu->reg, DMAR_PLMLIMIT_REG,
1914 (u32) iommu_state[DMAR_PLMLIMIT_REG * i]);
1915 dmar_writeq(iommu->reg, DMAR_PHMBASE_REG,
1916 (u64) iommu_state[DMAR_PHMBASE_REG * i]);
1917 dmar_writeq(iommu->reg, DMAR_PHMLIMIT_REG,
1918 (u64) iommu_state[DMAR_PHMLIMIT_REG * i]);
1920 if (iommu_enable_translation(iommu))
1921 return -EIO;
1922 i++;
1924 return 0;