ia64/xen-unstable

view xen/drivers/passthrough/vtd/qinval.c @ 19734:4fb8a6c993e2

VT-d: correct way to submit command to GCMD register

Per VT-d spec, software should submit only one "incremental" command
at a time to Global Command reigster. Current implementation uses a
variable (gcmd) to record the state of Global Status register. It's
error prone.

Signed-off-by: Weidong Han <weidong.han@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri Jun 05 09:29:42 2009 +0100 (2009-06-05)
parents a69daf23602a
children cc07094a02e4
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Allen Kay <allen.m.kay@intel.com>
18 * Copyright (C) Xiaohui Xin <xiaohui.xin@intel.com>
19 */
22 #include <xen/sched.h>
23 #include <xen/iommu.h>
24 #include <xen/time.h>
25 #include <xen/pci.h>
26 #include <xen/pci_regs.h>
27 #include "iommu.h"
28 #include "dmar.h"
29 #include "vtd.h"
30 #include "extern.h"
32 int qinval_enabled;
34 static void print_qi_regs(struct iommu *iommu)
35 {
36 u64 val;
38 val = dmar_readq(iommu->reg, DMAR_IQA_REG);
39 printk("DMAR_IQA_REG = %"PRIx64"\n", val);
41 val = dmar_readq(iommu->reg, DMAR_IQH_REG);
42 printk("DMAR_IQH_REG = %"PRIx64"\n", val);
44 val = dmar_readq(iommu->reg, DMAR_IQT_REG);
45 printk("DMAR_IQT_REG = %"PRIx64"\n", val);
46 }
48 static int qinval_next_index(struct iommu *iommu)
49 {
50 u64 val;
51 val = dmar_readq(iommu->reg, DMAR_IQT_REG);
52 return (val >> 4);
53 }
55 static int qinval_update_qtail(struct iommu *iommu, int index)
56 {
57 u64 val;
59 /* Need an ASSERT to insure that we have got register lock */
60 val = (index < (QINVAL_ENTRY_NR-1)) ? (index + 1) : 0;
61 dmar_writeq(iommu->reg, DMAR_IQT_REG, (val << 4));
62 return 0;
63 }
65 static int gen_cc_inv_dsc(struct iommu *iommu, int index,
66 u16 did, u16 source_id, u8 function_mask, u8 granu)
67 {
68 unsigned long flags;
69 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
70 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
72 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
73 qinval_entries =
74 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
75 qinval_entry = &qinval_entries[index];
76 qinval_entry->q.cc_inv_dsc.lo.type = TYPE_INVAL_CONTEXT;
77 qinval_entry->q.cc_inv_dsc.lo.granu = granu;
78 qinval_entry->q.cc_inv_dsc.lo.res_1 = 0;
79 qinval_entry->q.cc_inv_dsc.lo.did = did;
80 qinval_entry->q.cc_inv_dsc.lo.sid = source_id;
81 qinval_entry->q.cc_inv_dsc.lo.fm = function_mask;
82 qinval_entry->q.cc_inv_dsc.lo.res_2 = 0;
83 qinval_entry->q.cc_inv_dsc.hi.res = 0;
85 unmap_vtd_domain_page(qinval_entries);
86 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
88 return 0;
89 }
91 int queue_invalidate_context(struct iommu *iommu,
92 u16 did, u16 source_id, u8 function_mask, u8 granu)
93 {
94 int ret = -1;
95 unsigned long flags;
96 int index = -1;
98 spin_lock_irqsave(&iommu->register_lock, flags);
99 index = qinval_next_index(iommu);
100 if ( index == -1 )
101 return -EBUSY;
102 ret = gen_cc_inv_dsc(iommu, index, did, source_id,
103 function_mask, granu);
104 ret |= qinval_update_qtail(iommu, index);
105 spin_unlock_irqrestore(&iommu->register_lock, flags);
106 return ret;
107 }
109 static int gen_iotlb_inv_dsc(struct iommu *iommu, int index,
110 u8 granu, u8 dr, u8 dw, u16 did, u8 am, u8 ih, u64 addr)
111 {
112 unsigned long flags;
113 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
114 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
116 if ( index == -1 )
117 return -1;
118 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
120 qinval_entries =
121 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
122 qinval_entry = &qinval_entries[index];
123 qinval_entry->q.iotlb_inv_dsc.lo.type = TYPE_INVAL_IOTLB;
124 qinval_entry->q.iotlb_inv_dsc.lo.granu = granu;
125 qinval_entry->q.iotlb_inv_dsc.lo.dr = 0;
126 qinval_entry->q.iotlb_inv_dsc.lo.dw = 0;
127 qinval_entry->q.iotlb_inv_dsc.lo.res_1 = 0;
128 qinval_entry->q.iotlb_inv_dsc.lo.did = did;
129 qinval_entry->q.iotlb_inv_dsc.lo.res_2 = 0;
131 qinval_entry->q.iotlb_inv_dsc.hi.am = am;
132 qinval_entry->q.iotlb_inv_dsc.hi.ih = ih;
133 qinval_entry->q.iotlb_inv_dsc.hi.res_1 = 0;
134 qinval_entry->q.iotlb_inv_dsc.hi.addr = addr;
136 unmap_vtd_domain_page(qinval_entries);
137 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
138 return 0;
139 }
141 int queue_invalidate_iotlb(struct iommu *iommu,
142 u8 granu, u8 dr, u8 dw, u16 did, u8 am, u8 ih, u64 addr)
143 {
144 int ret = -1;
145 unsigned long flags;
146 int index = -1;
148 spin_lock_irqsave(&iommu->register_lock, flags);
150 index = qinval_next_index(iommu);
151 ret = gen_iotlb_inv_dsc(iommu, index, granu, dr, dw, did,
152 am, ih, addr);
153 ret |= qinval_update_qtail(iommu, index);
154 spin_unlock_irqrestore(&iommu->register_lock, flags);
155 return ret;
156 }
158 static int gen_wait_dsc(struct iommu *iommu, int index,
159 u8 iflag, u8 sw, u8 fn, u32 sdata, volatile u32 *saddr)
160 {
161 unsigned long flags;
162 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
163 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
165 if ( index == -1 )
166 return -1;
167 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
168 qinval_entries =
169 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
170 qinval_entry = &qinval_entries[index];
171 qinval_entry->q.inv_wait_dsc.lo.type = TYPE_INVAL_WAIT;
172 qinval_entry->q.inv_wait_dsc.lo.iflag = iflag;
173 qinval_entry->q.inv_wait_dsc.lo.sw = sw;
174 qinval_entry->q.inv_wait_dsc.lo.fn = fn;
175 qinval_entry->q.inv_wait_dsc.lo.res_1 = 0;
176 qinval_entry->q.inv_wait_dsc.lo.sdata = sdata;
177 qinval_entry->q.inv_wait_dsc.hi.res_1 = 0;
178 qinval_entry->q.inv_wait_dsc.hi.saddr = virt_to_maddr(saddr) >> 2;
179 unmap_vtd_domain_page(qinval_entries);
180 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
181 return 0;
182 }
184 static int queue_invalidate_wait(struct iommu *iommu,
185 u8 iflag, u8 sw, u8 fn, u32 sdata, volatile u32 *saddr)
186 {
187 unsigned long flags;
188 s_time_t start_time;
189 int index = -1;
190 int ret = -1;
191 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
193 spin_lock_irqsave(&qi_ctrl->qinval_poll_lock, flags);
194 spin_lock(&iommu->register_lock);
195 index = qinval_next_index(iommu);
196 if ( *saddr == 1 )
197 *saddr = 0;
198 ret = gen_wait_dsc(iommu, index, iflag, sw, fn, sdata, saddr);
199 ret |= qinval_update_qtail(iommu, index);
200 spin_unlock(&iommu->register_lock);
202 /* Now we don't support interrupt method */
203 if ( sw )
204 {
205 /* In case all wait descriptor writes to same addr with same data */
206 start_time = NOW();
207 while ( *saddr != 1 )
208 {
209 if ( NOW() > (start_time + DMAR_OPERATION_TIMEOUT) )
210 {
211 print_qi_regs(iommu);
212 panic("queue invalidate wait descriptor was not executed\n");
213 }
214 cpu_relax();
215 }
216 }
217 spin_unlock_irqrestore(&qi_ctrl->qinval_poll_lock, flags);
218 return ret;
219 }
221 int invalidate_sync(struct iommu *iommu)
222 {
223 int ret = -1;
224 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
226 if ( qi_ctrl->qinval_maddr != 0 )
227 {
228 ret = queue_invalidate_wait(iommu,
229 0, 1, 1, 1, &qi_ctrl->qinval_poll_status);
230 return ret;
231 }
232 return 0;
233 }
235 static int gen_dev_iotlb_inv_dsc(struct iommu *iommu, int index,
236 u32 max_invs_pend, u16 sid, u16 size, u64 addr)
237 {
238 unsigned long flags;
239 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
240 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
242 if ( index == -1 )
243 return -1;
244 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
246 qinval_entries =
247 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
248 qinval_entry = &qinval_entries[index];
249 qinval_entry->q.dev_iotlb_inv_dsc.lo.type = TYPE_INVAL_DEVICE_IOTLB;
250 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_1 = 0;
251 qinval_entry->q.dev_iotlb_inv_dsc.lo.max_invs_pend = max_invs_pend;
252 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_2 = 0;
253 qinval_entry->q.dev_iotlb_inv_dsc.lo.sid = sid;
254 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_3 = 0;
256 qinval_entry->q.dev_iotlb_inv_dsc.hi.size = size;
257 qinval_entry->q.dev_iotlb_inv_dsc.hi.res_1 = 0;
258 qinval_entry->q.dev_iotlb_inv_dsc.hi.addr = addr >> PAGE_SHIFT_4K;
260 unmap_vtd_domain_page(qinval_entries);
261 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
262 return 0;
263 }
265 int qinval_device_iotlb(struct iommu *iommu,
266 u32 max_invs_pend, u16 sid, u16 size, u64 addr)
267 {
268 int ret = -1;
269 unsigned long flags;
270 int index = -1;
272 spin_lock_irqsave(&iommu->register_lock, flags);
273 index = qinval_next_index(iommu);
274 ret = gen_dev_iotlb_inv_dsc(iommu, index, max_invs_pend,
275 sid, size, addr);
276 ret |= qinval_update_qtail(iommu, index);
277 spin_unlock_irqrestore(&iommu->register_lock, flags);
278 return ret;
279 }
281 static int gen_iec_inv_dsc(struct iommu *iommu, int index,
282 u8 granu, u8 im, u16 iidx)
283 {
284 unsigned long flags;
285 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
286 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
288 if ( index == -1 )
289 return -1;
290 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
292 qinval_entries =
293 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
294 qinval_entry = &qinval_entries[index];
295 qinval_entry->q.iec_inv_dsc.lo.type = TYPE_INVAL_IEC;
296 qinval_entry->q.iec_inv_dsc.lo.granu = granu;
297 qinval_entry->q.iec_inv_dsc.lo.res_1 = 0;
298 qinval_entry->q.iec_inv_dsc.lo.im = im;
299 qinval_entry->q.iec_inv_dsc.lo.iidx = iidx;
300 qinval_entry->q.iec_inv_dsc.lo.res_2 = 0;
301 qinval_entry->q.iec_inv_dsc.hi.res = 0;
303 unmap_vtd_domain_page(qinval_entries);
304 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
305 return 0;
306 }
308 int queue_invalidate_iec(struct iommu *iommu, u8 granu, u8 im, u16 iidx)
309 {
310 int ret;
311 unsigned long flags;
312 int index = -1;
314 spin_lock_irqsave(&iommu->register_lock, flags);
315 index = qinval_next_index(iommu);
316 ret = gen_iec_inv_dsc(iommu, index, granu, im, iidx);
317 ret |= qinval_update_qtail(iommu, index);
318 spin_unlock_irqrestore(&iommu->register_lock, flags);
319 return ret;
320 }
322 int __iommu_flush_iec(struct iommu *iommu, u8 granu, u8 im, u16 iidx)
323 {
324 int ret;
325 ret = queue_invalidate_iec(iommu, granu, im, iidx);
326 ret |= invalidate_sync(iommu);
328 /*
329 * reading vt-d architecture register will ensure
330 * draining happens in implementation independent way.
331 */
332 (void)dmar_readq(iommu->reg, DMAR_CAP_REG);
333 return ret;
334 }
336 int iommu_flush_iec_global(struct iommu *iommu)
337 {
338 return __iommu_flush_iec(iommu, IEC_GLOBAL_INVL, 0, 0);
339 }
341 int iommu_flush_iec_index(struct iommu *iommu, u8 im, u16 iidx)
342 {
343 return __iommu_flush_iec(iommu, IEC_INDEX_INVL, im, iidx);
344 }
346 static int flush_context_qi(
347 void *_iommu, u16 did, u16 sid, u8 fm, u64 type,
348 int flush_non_present_entry)
349 {
350 int ret = 0;
351 struct iommu *iommu = (struct iommu *)_iommu;
352 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
354 /*
355 * In the non-present entry flush case, if hardware doesn't cache
356 * non-present entry we do nothing and if hardware cache non-present
357 * entry, we flush entries of domain 0 (the domain id is used to cache
358 * any non-present entries)
359 */
360 if ( flush_non_present_entry )
361 {
362 if ( !cap_caching_mode(iommu->cap) )
363 return 1;
364 else
365 did = 0;
366 }
368 if ( qi_ctrl->qinval_maddr != 0 )
369 {
370 ret = queue_invalidate_context(iommu, did, sid, fm,
371 type >> DMA_CCMD_INVL_GRANU_OFFSET);
372 ret |= invalidate_sync(iommu);
373 }
374 return ret;
375 }
377 static int flush_iotlb_qi(
378 void *_iommu, u16 did,
379 u64 addr, unsigned int size_order, u64 type,
380 int flush_non_present_entry, int flush_dev_iotlb)
381 {
382 u8 dr = 0, dw = 0;
383 int ret = 0;
384 struct iommu *iommu = (struct iommu *)_iommu;
385 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
387 /*
388 * In the non-present entry flush case, if hardware doesn't cache
389 * non-present entry we do nothing and if hardware cache non-present
390 * entry, we flush entries of domain 0 (the domain id is used to cache
391 * any non-present entries)
392 */
393 if ( flush_non_present_entry )
394 {
395 if ( !cap_caching_mode(iommu->cap) )
396 return 1;
397 else
398 did = 0;
399 }
401 if ( qi_ctrl->qinval_maddr != 0 )
402 {
403 /* use queued invalidation */
404 if (cap_write_drain(iommu->cap))
405 dw = 1;
406 if (cap_read_drain(iommu->cap))
407 dr = 1;
408 /* Need to conside the ih bit later */
409 ret = queue_invalidate_iotlb(iommu,
410 (type >> DMA_TLB_FLUSH_GRANU_OFFSET), dr,
411 dw, did, (u8)size_order, 0, addr);
412 if ( flush_dev_iotlb )
413 ret |= dev_invalidate_iotlb(iommu, did, addr, size_order, type);
414 ret |= invalidate_sync(iommu);
415 }
416 return ret;
417 }
419 int enable_qinval(struct iommu *iommu)
420 {
421 struct qi_ctrl *qi_ctrl;
422 struct iommu_flush *flush;
423 u32 sts;
425 qi_ctrl = iommu_qi_ctrl(iommu);
426 flush = iommu_get_flush(iommu);
428 ASSERT(ecap_queued_inval(iommu->ecap) && iommu_qinval);
430 if ( qi_ctrl->qinval_maddr == 0 )
431 {
432 qi_ctrl->qinval_maddr = alloc_pgtable_maddr(NULL, NUM_QINVAL_PAGES);
433 if ( qi_ctrl->qinval_maddr == 0 )
434 {
435 dprintk(XENLOG_WARNING VTDPREFIX,
436 "Cannot allocate memory for qi_ctrl->qinval_maddr\n");
437 return -ENOMEM;
438 }
439 }
441 flush->context = flush_context_qi;
442 flush->iotlb = flush_iotlb_qi;
444 /* Setup Invalidation Queue Address(IQA) register with the
445 * address of the page we just allocated. QS field at
446 * bits[2:0] to indicate size of queue is one 4KB page.
447 * That's 256 entries. Queued Head (IQH) and Queue Tail (IQT)
448 * registers are automatically reset to 0 with write
449 * to IQA register.
450 */
451 qi_ctrl->qinval_maddr |= IQA_REG_QS;
452 dmar_writeq(iommu->reg, DMAR_IQA_REG, qi_ctrl->qinval_maddr);
454 dmar_writeq(iommu->reg, DMAR_IQT_REG, 0);
456 /* enable queued invalidation hardware */
457 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
458 dmar_writel(iommu->reg, DMAR_GCMD_REG, sts | DMA_GCMD_QIE);
460 /* Make sure hardware complete it */
461 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
462 (sts & DMA_GSTS_QIES), sts);
464 qinval_enabled = 1;
465 return 0;
466 }
468 void disable_qinval(struct iommu *iommu)
469 {
470 u32 sts;
472 ASSERT(ecap_queued_inval(iommu->ecap) && iommu_qinval);
474 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
475 dmar_writel(iommu->reg, DMAR_GCMD_REG, sts & (~DMA_GCMD_QIE));
477 /* Make sure hardware complete it */
478 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
479 !(sts & DMA_GSTS_QIES), sts);
480 }