ia64/xen-unstable

view xen/drivers/passthrough/vtd/qinval.c @ 19673:f3bed18decfc

[VTD] laying the ground work for ATS

These changes lay the ground work for ATS enabling in Xen. It will be
followed by patch which enables PCI MMCFG which is needed for actual
enabling of ATS functionality.

Signed-off-by: Allen Kay <allen.m.kay@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri May 29 09:19:30 2009 +0100 (2009-05-29)
parents 57b733f66531
children 42fe00c6f8b4
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Allen Kay <allen.m.kay@intel.com>
18 * Copyright (C) Xiaohui Xin <xiaohui.xin@intel.com>
19 */
22 #include <xen/sched.h>
23 #include <xen/iommu.h>
24 #include <xen/time.h>
25 #include <xen/pci.h>
26 #include <xen/pci_regs.h>
27 #include "iommu.h"
28 #include "dmar.h"
29 #include "vtd.h"
30 #include "extern.h"
32 int qinval_enabled;
34 static void print_qi_regs(struct iommu *iommu)
35 {
36 u64 val;
38 val = dmar_readq(iommu->reg, DMAR_IQA_REG);
39 printk("DMAR_IQA_REG = %"PRIx64"\n", val);
41 val = dmar_readq(iommu->reg, DMAR_IQH_REG);
42 printk("DMAR_IQH_REG = %"PRIx64"\n", val);
44 val = dmar_readq(iommu->reg, DMAR_IQT_REG);
45 printk("DMAR_IQT_REG = %"PRIx64"\n", val);
46 }
48 static int qinval_next_index(struct iommu *iommu)
49 {
50 u64 val;
51 val = dmar_readq(iommu->reg, DMAR_IQT_REG);
52 return (val >> 4);
53 }
55 static int qinval_update_qtail(struct iommu *iommu, int index)
56 {
57 u64 val;
59 /* Need an ASSERT to insure that we have got register lock */
60 val = (index < (QINVAL_ENTRY_NR-1)) ? (index + 1) : 0;
61 dmar_writeq(iommu->reg, DMAR_IQT_REG, (val << 4));
62 return 0;
63 }
65 static int gen_cc_inv_dsc(struct iommu *iommu, int index,
66 u16 did, u16 source_id, u8 function_mask, u8 granu)
67 {
68 unsigned long flags;
69 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
70 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
72 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
73 qinval_entries =
74 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
75 qinval_entry = &qinval_entries[index];
76 qinval_entry->q.cc_inv_dsc.lo.type = TYPE_INVAL_CONTEXT;
77 qinval_entry->q.cc_inv_dsc.lo.granu = granu;
78 qinval_entry->q.cc_inv_dsc.lo.res_1 = 0;
79 qinval_entry->q.cc_inv_dsc.lo.did = did;
80 qinval_entry->q.cc_inv_dsc.lo.sid = source_id;
81 qinval_entry->q.cc_inv_dsc.lo.fm = function_mask;
82 qinval_entry->q.cc_inv_dsc.lo.res_2 = 0;
83 qinval_entry->q.cc_inv_dsc.hi.res = 0;
85 unmap_vtd_domain_page(qinval_entries);
86 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
88 return 0;
89 }
91 int queue_invalidate_context(struct iommu *iommu,
92 u16 did, u16 source_id, u8 function_mask, u8 granu)
93 {
94 int ret = -1;
95 unsigned long flags;
96 int index = -1;
98 spin_lock_irqsave(&iommu->register_lock, flags);
99 index = qinval_next_index(iommu);
100 if ( index == -1 )
101 return -EBUSY;
102 ret = gen_cc_inv_dsc(iommu, index, did, source_id,
103 function_mask, granu);
104 ret |= qinval_update_qtail(iommu, index);
105 spin_unlock_irqrestore(&iommu->register_lock, flags);
106 return ret;
107 }
109 static int gen_iotlb_inv_dsc(struct iommu *iommu, int index,
110 u8 granu, u8 dr, u8 dw, u16 did, u8 am, u8 ih, u64 addr)
111 {
112 unsigned long flags;
113 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
114 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
116 if ( index == -1 )
117 return -1;
118 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
120 qinval_entries =
121 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
122 qinval_entry = &qinval_entries[index];
123 qinval_entry->q.iotlb_inv_dsc.lo.type = TYPE_INVAL_IOTLB;
124 qinval_entry->q.iotlb_inv_dsc.lo.granu = granu;
125 qinval_entry->q.iotlb_inv_dsc.lo.dr = 0;
126 qinval_entry->q.iotlb_inv_dsc.lo.dw = 0;
127 qinval_entry->q.iotlb_inv_dsc.lo.res_1 = 0;
128 qinval_entry->q.iotlb_inv_dsc.lo.did = did;
129 qinval_entry->q.iotlb_inv_dsc.lo.res_2 = 0;
131 qinval_entry->q.iotlb_inv_dsc.hi.am = am;
132 qinval_entry->q.iotlb_inv_dsc.hi.ih = ih;
133 qinval_entry->q.iotlb_inv_dsc.hi.res_1 = 0;
134 qinval_entry->q.iotlb_inv_dsc.hi.addr = addr;
136 unmap_vtd_domain_page(qinval_entries);
137 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
138 return 0;
139 }
141 int queue_invalidate_iotlb(struct iommu *iommu,
142 u8 granu, u8 dr, u8 dw, u16 did, u8 am, u8 ih, u64 addr)
143 {
144 int ret = -1;
145 unsigned long flags;
146 int index = -1;
148 spin_lock_irqsave(&iommu->register_lock, flags);
150 index = qinval_next_index(iommu);
151 ret = gen_iotlb_inv_dsc(iommu, index, granu, dr, dw, did,
152 am, ih, addr);
153 ret |= qinval_update_qtail(iommu, index);
154 spin_unlock_irqrestore(&iommu->register_lock, flags);
155 return ret;
156 }
158 static int gen_wait_dsc(struct iommu *iommu, int index,
159 u8 iflag, u8 sw, u8 fn, u32 sdata, volatile u32 *saddr)
160 {
161 unsigned long flags;
162 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
163 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
165 if ( index == -1 )
166 return -1;
167 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
168 qinval_entries =
169 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
170 qinval_entry = &qinval_entries[index];
171 qinval_entry->q.inv_wait_dsc.lo.type = TYPE_INVAL_WAIT;
172 qinval_entry->q.inv_wait_dsc.lo.iflag = iflag;
173 qinval_entry->q.inv_wait_dsc.lo.sw = sw;
174 qinval_entry->q.inv_wait_dsc.lo.fn = fn;
175 qinval_entry->q.inv_wait_dsc.lo.res_1 = 0;
176 qinval_entry->q.inv_wait_dsc.lo.sdata = sdata;
177 qinval_entry->q.inv_wait_dsc.hi.res_1 = 0;
178 qinval_entry->q.inv_wait_dsc.hi.saddr = virt_to_maddr(saddr) >> 2;
179 unmap_vtd_domain_page(qinval_entries);
180 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
181 return 0;
182 }
184 static int queue_invalidate_wait(struct iommu *iommu,
185 u8 iflag, u8 sw, u8 fn, u32 sdata, volatile u32 *saddr)
186 {
187 unsigned long flags;
188 s_time_t start_time;
189 int index = -1;
190 int ret = -1;
191 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
193 spin_lock_irqsave(&qi_ctrl->qinval_poll_lock, flags);
194 spin_lock(&iommu->register_lock);
195 index = qinval_next_index(iommu);
196 if ( *saddr == 1 )
197 *saddr = 0;
198 ret = gen_wait_dsc(iommu, index, iflag, sw, fn, sdata, saddr);
199 ret |= qinval_update_qtail(iommu, index);
200 spin_unlock(&iommu->register_lock);
202 /* Now we don't support interrupt method */
203 if ( sw )
204 {
205 /* In case all wait descriptor writes to same addr with same data */
206 start_time = NOW();
207 while ( *saddr != 1 )
208 {
209 if ( NOW() > (start_time + DMAR_OPERATION_TIMEOUT) )
210 {
211 print_qi_regs(iommu);
212 panic("queue invalidate wait descriptor was not executed\n");
213 }
214 cpu_relax();
215 }
216 }
217 spin_unlock_irqrestore(&qi_ctrl->qinval_poll_lock, flags);
218 return ret;
219 }
221 int invalidate_sync(struct iommu *iommu)
222 {
223 int ret = -1;
224 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
226 if ( qi_ctrl->qinval_maddr != 0 )
227 {
228 ret = queue_invalidate_wait(iommu,
229 0, 1, 1, 1, &qi_ctrl->qinval_poll_status);
230 return ret;
231 }
232 return 0;
233 }
235 static int gen_dev_iotlb_inv_dsc(struct iommu *iommu, int index,
236 u32 max_invs_pend, u16 sid, u16 size, u64 addr)
237 {
238 unsigned long flags;
239 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
240 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
242 if ( index == -1 )
243 return -1;
244 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
246 qinval_entries =
247 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
248 qinval_entry = &qinval_entries[index];
249 qinval_entry->q.dev_iotlb_inv_dsc.lo.type = TYPE_INVAL_DEVICE_IOTLB;
250 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_1 = 0;
251 qinval_entry->q.dev_iotlb_inv_dsc.lo.max_invs_pend = max_invs_pend;
252 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_2 = 0;
253 qinval_entry->q.dev_iotlb_inv_dsc.lo.sid = sid;
254 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_3 = 0;
256 qinval_entry->q.dev_iotlb_inv_dsc.hi.size = size;
257 qinval_entry->q.dev_iotlb_inv_dsc.hi.res_1 = 0;
258 qinval_entry->q.dev_iotlb_inv_dsc.hi.addr = addr >> PAGE_SHIFT_4K;
260 unmap_vtd_domain_page(qinval_entries);
261 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
262 return 0;
263 }
265 int qinval_device_iotlb(struct iommu *iommu,
266 u32 max_invs_pend, u16 sid, u16 size, u64 addr)
267 {
268 int ret = -1;
269 unsigned long flags;
270 int index = -1;
272 spin_lock_irqsave(&iommu->register_lock, flags);
273 index = qinval_next_index(iommu);
274 ret = gen_dev_iotlb_inv_dsc(iommu, index, max_invs_pend,
275 sid, size, addr);
276 ret |= qinval_update_qtail(iommu, index);
277 spin_unlock_irqrestore(&iommu->register_lock, flags);
278 return ret;
279 }
281 static int gen_iec_inv_dsc(struct iommu *iommu, int index,
282 u8 granu, u8 im, u16 iidx)
283 {
284 unsigned long flags;
285 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
286 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
288 if ( index == -1 )
289 return -1;
290 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
292 qinval_entries =
293 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
294 qinval_entry = &qinval_entries[index];
295 qinval_entry->q.iec_inv_dsc.lo.type = TYPE_INVAL_IEC;
296 qinval_entry->q.iec_inv_dsc.lo.granu = granu;
297 qinval_entry->q.iec_inv_dsc.lo.res_1 = 0;
298 qinval_entry->q.iec_inv_dsc.lo.im = im;
299 qinval_entry->q.iec_inv_dsc.lo.iidx = iidx;
300 qinval_entry->q.iec_inv_dsc.lo.res_2 = 0;
301 qinval_entry->q.iec_inv_dsc.hi.res = 0;
303 unmap_vtd_domain_page(qinval_entries);
304 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
305 return 0;
306 }
308 int queue_invalidate_iec(struct iommu *iommu, u8 granu, u8 im, u16 iidx)
309 {
310 int ret;
311 unsigned long flags;
312 int index = -1;
314 spin_lock_irqsave(&iommu->register_lock, flags);
315 index = qinval_next_index(iommu);
316 ret = gen_iec_inv_dsc(iommu, index, granu, im, iidx);
317 ret |= qinval_update_qtail(iommu, index);
318 spin_unlock_irqrestore(&iommu->register_lock, flags);
319 return ret;
320 }
322 int __iommu_flush_iec(struct iommu *iommu, u8 granu, u8 im, u16 iidx)
323 {
324 int ret;
325 ret = queue_invalidate_iec(iommu, granu, im, iidx);
326 ret |= invalidate_sync(iommu);
328 /*
329 * reading vt-d architecture register will ensure
330 * draining happens in implementation independent way.
331 */
332 (void)dmar_readq(iommu->reg, DMAR_CAP_REG);
333 return ret;
334 }
336 int iommu_flush_iec_global(struct iommu *iommu)
337 {
338 return __iommu_flush_iec(iommu, IEC_GLOBAL_INVL, 0, 0);
339 }
341 int iommu_flush_iec_index(struct iommu *iommu, u8 im, u16 iidx)
342 {
343 return __iommu_flush_iec(iommu, IEC_INDEX_INVL, im, iidx);
344 }
346 static int flush_context_qi(
347 void *_iommu, u16 did, u16 sid, u8 fm, u64 type,
348 int flush_non_present_entry)
349 {
350 int ret = 0;
351 struct iommu *iommu = (struct iommu *)_iommu;
352 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
354 /*
355 * In the non-present entry flush case, if hardware doesn't cache
356 * non-present entry we do nothing and if hardware cache non-present
357 * entry, we flush entries of domain 0 (the domain id is used to cache
358 * any non-present entries)
359 */
360 if ( flush_non_present_entry )
361 {
362 if ( !cap_caching_mode(iommu->cap) )
363 return 1;
364 else
365 did = 0;
366 }
368 if ( qi_ctrl->qinval_maddr != 0 )
369 {
370 ret = queue_invalidate_context(iommu, did, sid, fm,
371 type >> DMA_CCMD_INVL_GRANU_OFFSET);
372 ret |= invalidate_sync(iommu);
373 }
374 return ret;
375 }
377 static int flush_iotlb_qi(
378 void *_iommu, u16 did,
379 u64 addr, unsigned int size_order, u64 type,
380 int flush_non_present_entry, int flush_dev_iotlb)
381 {
382 u8 dr = 0, dw = 0;
383 int ret = 0;
384 struct iommu *iommu = (struct iommu *)_iommu;
385 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
387 /*
388 * In the non-present entry flush case, if hardware doesn't cache
389 * non-present entry we do nothing and if hardware cache non-present
390 * entry, we flush entries of domain 0 (the domain id is used to cache
391 * any non-present entries)
392 */
393 if ( flush_non_present_entry )
394 {
395 if ( !cap_caching_mode(iommu->cap) )
396 return 1;
397 else
398 did = 0;
399 }
401 if ( qi_ctrl->qinval_maddr != 0 )
402 {
403 /* use queued invalidation */
404 if (cap_write_drain(iommu->cap))
405 dw = 1;
406 if (cap_read_drain(iommu->cap))
407 dr = 1;
408 /* Need to conside the ih bit later */
409 ret = queue_invalidate_iotlb(iommu,
410 (type >> DMA_TLB_FLUSH_GRANU_OFFSET), dr,
411 dw, did, (u8)size_order, 0, addr);
412 #if defined(NOT_YET)
413 if ( flush_dev_iotlb )
414 ret |= dev_invalidate_iotlb(iommu, did, addr, size_order, type);
415 #endif
416 ret |= invalidate_sync(iommu);
417 }
418 return ret;
419 }
421 int enable_qinval(struct iommu *iommu)
422 {
423 s_time_t start_time;
424 struct qi_ctrl *qi_ctrl;
425 struct iommu_flush *flush;
427 qi_ctrl = iommu_qi_ctrl(iommu);
428 flush = iommu_get_flush(iommu);
430 ASSERT(ecap_queued_inval(iommu->ecap) && iommu_qinval);
432 if ( qi_ctrl->qinval_maddr == 0 )
433 {
434 qi_ctrl->qinval_maddr = alloc_pgtable_maddr(NULL, NUM_QINVAL_PAGES);
435 if ( qi_ctrl->qinval_maddr == 0 )
436 {
437 dprintk(XENLOG_WARNING VTDPREFIX,
438 "Cannot allocate memory for qi_ctrl->qinval_maddr\n");
439 return -ENOMEM;
440 }
441 }
443 flush->context = flush_context_qi;
444 flush->iotlb = flush_iotlb_qi;
446 /* Setup Invalidation Queue Address(IQA) register with the
447 * address of the page we just allocated. QS field at
448 * bits[2:0] to indicate size of queue is one 4KB page.
449 * That's 256 entries. Queued Head (IQH) and Queue Tail (IQT)
450 * registers are automatically reset to 0 with write
451 * to IQA register.
452 */
453 qi_ctrl->qinval_maddr |= IQA_REG_QS;
454 dmar_writeq(iommu->reg, DMAR_IQA_REG, qi_ctrl->qinval_maddr);
456 dmar_writeq(iommu->reg, DMAR_IQT_REG, 0);
458 /* enable queued invalidation hardware */
459 iommu->gcmd |= DMA_GCMD_QIE;
460 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
462 /* Make sure hardware complete it */
463 start_time = NOW();
464 while ( !(dmar_readl(iommu->reg, DMAR_GSTS_REG) & DMA_GSTS_QIES) )
465 {
466 if ( NOW() > (start_time + DMAR_OPERATION_TIMEOUT) )
467 panic("Cannot set QIE field for queue invalidation\n");
468 cpu_relax();
469 }
471 qinval_enabled = 1;
472 return 0;
473 }
475 void disable_qinval(struct iommu *iommu)
476 {
477 s_time_t start_time;
479 ASSERT(ecap_queued_inval(iommu->ecap) && iommu_qinval);
481 iommu->gcmd &= ~DMA_GCMD_QIE;
482 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
484 /* Make sure hardware complete it */
485 start_time = NOW();
486 while ( dmar_readl(iommu->reg, DMAR_GSTS_REG) & DMA_GSTS_QIES )
487 {
488 if ( NOW() > (start_time + DMAR_OPERATION_TIMEOUT) )
489 panic("Cannot clear QIE field for queue invalidation\n");
490 cpu_relax();
491 }
492 }