ia64/xen-unstable

view xen/arch/x86/hvm/vmx/vtd/qinval.c @ 16830:cc5bb500df5f

vtd: Enable queued invalidation method if such HW support is
detected. Otherwise, register invalidation method is used.

Signed-off-by: Allen Kay <allen.m.kay@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Tue Jan 22 09:48:51 2008 +0000 (2008-01-22)
parents
children
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Allen Kay <allen.m.kay@intel.com>
18 * Copyright (C) Xiaohui Xin <xiaohui.xin@intel.com>
19 */
22 #include <xen/init.h>
23 #include <xen/irq.h>
24 #include <xen/spinlock.h>
25 #include <xen/sched.h>
26 #include <xen/xmalloc.h>
27 #include <xen/domain_page.h>
28 #include <asm/delay.h>
29 #include <asm/string.h>
30 #include <asm/iommu.h>
31 #include <asm/hvm/vmx/intel-iommu.h>
32 #include "dmar.h"
33 #include "vtd.h"
34 #include "pci-direct.h"
35 #include "pci_regs.h"
36 #include "msi.h"
37 #include "extern.h"
39 static void print_qi_regs(struct iommu *iommu)
40 {
41 u64 val;
43 val = dmar_readq(iommu->reg, DMAR_IQA_REG);
44 printk("DMAR_IAQ_REG = %"PRIx64"\n", val);
46 val = dmar_readq(iommu->reg, DMAR_IQH_REG);
47 printk("DMAR_IAH_REG = %"PRIx64"\n", val);
49 val = dmar_readq(iommu->reg, DMAR_IQT_REG);
50 printk("DMAR_IAT_REG = %"PRIx64"\n", val);
51 }
53 static int qinval_next_index(struct iommu *iommu)
54 {
55 u64 val;
56 val = dmar_readq(iommu->reg, DMAR_IQT_REG);
57 return (val >> 4);
58 }
60 static int qinval_update_qtail(struct iommu *iommu, int index)
61 {
62 u64 val;
64 /* Need an ASSERT to insure that we have got register lock */
65 val = (index < (QINVAL_ENTRY_NR-1)) ? (index + 1) : 0;
66 dmar_writeq(iommu->reg, DMAR_IQT_REG, (val << 4));
67 return 0;
68 }
70 static int gen_cc_inv_dsc(struct iommu *iommu, int index,
71 u16 did, u16 source_id, u8 function_mask, u8 granu)
72 {
73 u64 *ptr64;
74 unsigned long flags;
75 struct qinval_entry * qinval_entry = NULL;
76 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
78 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
79 qinval_entry = &qi_ctrl->qinval[index];
80 qinval_entry->q.cc_inv_dsc.lo.type = TYPE_INVAL_CONTEXT;
81 qinval_entry->q.cc_inv_dsc.lo.granu = granu;
82 qinval_entry->q.cc_inv_dsc.lo.res_1 = 0;
83 qinval_entry->q.cc_inv_dsc.lo.did = did;
84 qinval_entry->q.cc_inv_dsc.lo.sid = source_id;
85 qinval_entry->q.cc_inv_dsc.lo.fm = function_mask;
86 qinval_entry->q.cc_inv_dsc.lo.res_2 = 0;
87 qinval_entry->q.cc_inv_dsc.hi.res = 0;
88 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
90 ptr64 = (u64 *)qinval_entry;
91 return 0;
92 }
94 int queue_invalidate_context(struct iommu *iommu,
95 u16 did, u16 source_id, u8 function_mask, u8 granu)
96 {
97 int ret = -1;
98 unsigned long flags;
99 int index = -1;
101 spin_lock_irqsave(&iommu->register_lock, flags);
102 index = qinval_next_index(iommu);
103 if (index == -1)
104 return -EBUSY;
105 ret = gen_cc_inv_dsc(iommu, index, did, source_id,
106 function_mask, granu);
107 ret |= qinval_update_qtail(iommu, index);
108 spin_unlock_irqrestore(&iommu->register_lock, flags);
109 return ret;
110 }
112 static int gen_iotlb_inv_dsc(struct iommu *iommu, int index,
113 u8 granu, u8 dr, u8 dw, u16 did, u8 am, u8 ih, u64 addr)
114 {
115 unsigned long flags;
116 struct qinval_entry * qinval_entry = NULL;
117 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
119 if ( index == -1 )
120 return -1;
121 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
123 qinval_entry = &qi_ctrl->qinval[index];
124 qinval_entry->q.iotlb_inv_dsc.lo.type = TYPE_INVAL_IOTLB;
125 qinval_entry->q.iotlb_inv_dsc.lo.granu = granu;
126 qinval_entry->q.iotlb_inv_dsc.lo.dr = 0;
127 qinval_entry->q.iotlb_inv_dsc.lo.dw = 0;
128 qinval_entry->q.iotlb_inv_dsc.lo.res_1 = 0;
129 qinval_entry->q.iotlb_inv_dsc.lo.did = did;
130 qinval_entry->q.iotlb_inv_dsc.lo.res_2 = 0;
132 qinval_entry->q.iotlb_inv_dsc.hi.am = am;
133 qinval_entry->q.iotlb_inv_dsc.hi.ih = ih;
134 qinval_entry->q.iotlb_inv_dsc.hi.res_1 = 0;
135 qinval_entry->q.iotlb_inv_dsc.hi.addr = addr;
137 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
138 return 0;
139 }
141 int queue_invalidate_iotlb(struct iommu *iommu,
142 u8 granu, u8 dr, u8 dw, u16 did, u8 am, u8 ih, u64 addr)
143 {
144 int ret = -1;
145 unsigned long flags;
146 int index = -1;
148 spin_lock_irqsave(&iommu->register_lock, flags);
150 index = qinval_next_index(iommu);
151 ret = gen_iotlb_inv_dsc(iommu, index, granu, dr, dw, did,
152 am, ih, addr);
153 ret |= qinval_update_qtail(iommu, index);
154 spin_unlock_irqrestore(&iommu->register_lock, flags);
155 return ret;
156 }
158 static int gen_wait_dsc(struct iommu *iommu, int index,
159 u8 iflag, u8 sw, u8 fn, u32 sdata, volatile u32 *saddr)
160 {
161 u64 *ptr64;
162 unsigned long flags;
163 struct qinval_entry * qinval_entry = NULL;
164 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
166 if ( index == -1 )
167 return -1;
168 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
169 qinval_entry = &qi_ctrl->qinval[index];
170 qinval_entry->q.inv_wait_dsc.lo.type = TYPE_INVAL_WAIT;
171 qinval_entry->q.inv_wait_dsc.lo.iflag = iflag;
172 qinval_entry->q.inv_wait_dsc.lo.sw = sw;
173 qinval_entry->q.inv_wait_dsc.lo.fn = fn;
174 qinval_entry->q.inv_wait_dsc.lo.res_1 = 0;
175 qinval_entry->q.inv_wait_dsc.lo.sdata = sdata;
176 qinval_entry->q.inv_wait_dsc.hi.res_1 = 0;
177 qinval_entry->q.inv_wait_dsc.hi.saddr = virt_to_maddr(saddr) >> 2;
178 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
179 ptr64 = (u64 *)qinval_entry;
180 return 0;
181 }
183 static int queue_invalidate_wait(struct iommu *iommu,
184 u8 iflag, u8 sw, u8 fn, u32 sdata, volatile u32 *saddr)
185 {
186 unsigned long flags;
187 unsigned long start_time;
188 int index = -1;
189 int ret = -1;
190 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
192 spin_lock_irqsave(&qi_ctrl->qinval_poll_lock, flags);
193 spin_lock_irqsave(&iommu->register_lock, flags);
194 index = qinval_next_index(iommu);
195 if (*saddr == 1)
196 *saddr = 0;
197 ret = gen_wait_dsc(iommu, index, iflag, sw, fn, sdata, saddr);
198 ret |= qinval_update_qtail(iommu, index);
199 spin_unlock_irqrestore(&iommu->register_lock, flags);
201 /* Now we don't support interrupt method */
202 if ( sw )
203 {
204 /* In case all wait descriptor writes to same addr with same data */
205 start_time = jiffies;
206 while ( *saddr != 1 ) {
207 if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT)) {
208 print_qi_regs(iommu);
209 panic("queue invalidate wait descriptor was not executed\n");
210 }
211 cpu_relax();
212 }
213 }
214 spin_unlock_irqrestore(&qi_ctrl->qinval_poll_lock, flags);
215 return ret;
216 }
218 int invalidate_sync(struct iommu *iommu)
219 {
220 int ret = -1;
221 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
223 if (qi_ctrl->qinval)
224 {
225 ret = queue_invalidate_wait(iommu,
226 0, 1, 1, 1, &qi_ctrl->qinval_poll_status);
227 return ret;
228 }
229 return 0;
230 }
232 static int gen_dev_iotlb_inv_dsc(struct iommu *iommu, int index,
233 u32 max_invs_pend, u16 sid, u16 size, u64 addr)
234 {
235 unsigned long flags;
236 struct qinval_entry * qinval_entry = NULL;
237 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
239 if ( index == -1 )
240 return -1;
241 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
243 qinval_entry = &qi_ctrl->qinval[index];
244 qinval_entry->q.dev_iotlb_inv_dsc.lo.type = TYPE_INVAL_DEVICE_IOTLB;
245 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_1 = 0;
246 qinval_entry->q.dev_iotlb_inv_dsc.lo.max_invs_pend = max_invs_pend;
247 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_2 = 0;
248 qinval_entry->q.dev_iotlb_inv_dsc.lo.sid = sid;
249 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_3 = 0;
251 qinval_entry->q.dev_iotlb_inv_dsc.hi.size = size;
252 qinval_entry->q.dev_iotlb_inv_dsc.hi.addr = addr;
254 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
255 return 0;
256 }
258 int queue_invalidate_device_iotlb(struct iommu *iommu,
259 u32 max_invs_pend, u16 sid, u16 size, u64 addr)
260 {
261 int ret = -1;
262 unsigned long flags;
263 int index = -1;
265 spin_lock_irqsave(&iommu->register_lock, flags);
266 index = qinval_next_index(iommu);
267 ret = gen_dev_iotlb_inv_dsc(iommu, index, max_invs_pend,
268 sid, size, addr);
269 ret |= qinval_update_qtail(iommu, index);
270 spin_unlock_irqrestore(&iommu->register_lock, flags);
271 return ret;
272 }
274 static int gen_iec_inv_dsc(struct iommu *iommu, int index,
275 u8 granu, u8 im, u16 iidx)
276 {
277 unsigned long flags;
278 struct qinval_entry * qinval_entry = NULL;
279 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
281 if ( index == -1 )
282 return -1;
283 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
285 qinval_entry = &qi_ctrl->qinval[index];
286 qinval_entry->q.iec_inv_dsc.lo.type = TYPE_INVAL_IEC;
287 qinval_entry->q.iec_inv_dsc.lo.granu = granu;
288 qinval_entry->q.iec_inv_dsc.lo.res_1 = 0;
289 qinval_entry->q.iec_inv_dsc.lo.im = im;
290 qinval_entry->q.iec_inv_dsc.lo.iidx = iidx;
291 qinval_entry->q.iec_inv_dsc.lo.res_2 = 0;
292 qinval_entry->q.iec_inv_dsc.hi.res = 0;
294 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
295 return 0;
296 }
298 int queue_invalidate_iec(struct iommu *iommu, u8 granu, u8 im, u16 iidx)
299 {
300 int ret;
301 unsigned long flags;
302 int index = -1;
304 spin_lock_irqsave(&iommu->register_lock, flags);
305 index = qinval_next_index(iommu);
306 ret = gen_iec_inv_dsc(iommu, index, granu, im, iidx);
307 ret |= qinval_update_qtail(iommu, index);
308 spin_unlock_irqrestore(&iommu->register_lock, flags);
309 return ret;
310 }
312 u64 iec_cap;
313 int __iommu_flush_iec(struct iommu *iommu, u8 granu, u8 im, u16 iidx)
314 {
315 int ret;
316 ret = queue_invalidate_iec(iommu, granu, im, iidx);
317 ret |= invalidate_sync(iommu);
319 /*
320 * reading vt-d architecture register will ensure
321 * draining happens in implementation independent way.
322 */
323 iec_cap = dmar_readq(iommu->reg, DMAR_CAP_REG);
324 return ret;
325 }
327 int iommu_flush_iec_global(struct iommu *iommu)
328 {
329 return __iommu_flush_iec(iommu, IEC_GLOBAL_INVL, 0, 0);
330 }
332 int iommu_flush_iec_index(struct iommu *iommu, u8 im, u16 iidx)
333 {
334 return __iommu_flush_iec(iommu, IEC_INDEX_INVL, im, iidx);
335 }
337 static int flush_context_qi(
338 void *_iommu, u16 did, u16 sid, u8 fm, u64 type,
339 int non_present_entry_flush)
340 {
341 int ret = 0;
342 struct iommu *iommu = (struct iommu *)_iommu;
343 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
345 /*
346 * In the non-present entry flush case, if hardware doesn't cache
347 * non-present entry we do nothing and if hardware cache non-present
348 * entry, we flush entries of domain 0 (the domain id is used to cache
349 * any non-present entries)
350 */
351 if ( non_present_entry_flush )
352 {
353 if ( !cap_caching_mode(iommu->cap) )
354 return 1;
355 else
356 did = 0;
357 }
359 if (qi_ctrl->qinval)
360 {
361 ret = queue_invalidate_context(iommu, did, sid, fm,
362 type >> DMA_CCMD_INVL_GRANU_OFFSET);
363 ret |= invalidate_sync(iommu);
364 }
365 return ret;
366 }
368 static int flush_iotlb_qi(
369 void *_iommu, u16 did,
370 u64 addr, unsigned int size_order, u64 type,
371 int non_present_entry_flush)
372 {
373 u8 dr = 0, dw = 0;
374 int ret = 0;
375 struct iommu *iommu = (struct iommu *)_iommu;
376 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
378 /*
379 * In the non-present entry flush case, if hardware doesn't cache
380 * non-present entry we do nothing and if hardware cache non-present
381 * entry, we flush entries of domain 0 (the domain id is used to cache
382 * any non-present entries)
383 */
384 if ( non_present_entry_flush )
385 {
386 if ( !cap_caching_mode(iommu->cap) )
387 return 1;
388 else
389 did = 0;
390 }
392 if (qi_ctrl->qinval) {
393 /* use queued invalidation */
394 if (cap_write_drain(iommu->cap))
395 dw = 1;
396 if (cap_read_drain(iommu->cap))
397 dr = 1;
398 /* Need to conside the ih bit later */
399 ret = queue_invalidate_iotlb(iommu,
400 (type >> DMA_TLB_FLUSH_GRANU_OFFSET), dr,
401 dw, did, (u8)size_order, 0, addr);
402 ret |= invalidate_sync(iommu);
403 }
404 return ret;
405 }
407 int qinval_setup(struct iommu *iommu)
408 {
409 unsigned long start_time;
410 u64 paddr;
411 u32 status = 0;
412 struct qi_ctrl *qi_ctrl;
413 struct iommu_flush *flush;
415 qi_ctrl = iommu_qi_ctrl(iommu);
416 flush = iommu_get_flush(iommu);
418 if ( !ecap_queued_inval(iommu->ecap) )
419 return -ENODEV;
421 if (qi_ctrl->qinval == NULL) {
422 qi_ctrl->qinval = alloc_xenheap_page();
423 if (qi_ctrl->qinval == NULL)
424 panic("Cannot allocate memory for qi_ctrl->qinval\n");
425 memset((u8*)qi_ctrl->qinval, 0, PAGE_SIZE_4K);
426 flush->context = flush_context_qi;
427 flush->iotlb = flush_iotlb_qi;
428 }
429 paddr = virt_to_maddr(qi_ctrl->qinval);
431 /* Setup Invalidation Queue Address(IQA) register with the
432 * address of the page we just allocated. QS field at
433 * bits[2:0] to indicate size of queue is one 4KB page.
434 * That's 256 entries. Queued Head (IQH) and Queue Tail (IQT)
435 * registers are automatically reset to 0 with write
436 * to IQA register.
437 */
438 dmar_writeq(iommu->reg, DMAR_IQA_REG, paddr);
440 /* enable queued invalidation hardware */
441 iommu->gcmd |= DMA_GCMD_QIE;
442 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
444 /* Make sure hardware complete it */
445 start_time = jiffies;
446 while (1) {
447 status = dmar_readl(iommu->reg, DMAR_GSTS_REG);
448 if (status & DMA_GSTS_QIES)
449 break;
450 if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT))
451 panic("Cannot set QIE field for queue invalidation\n");
452 cpu_relax();
453 }
454 status = 0;
455 return status;
456 }