ia64/xen-unstable

view xen/drivers/passthrough/vtd/qinval.c @ 17443:80ba1b427032

x86: Emulate accesses to PCI window registers cf8/cfc to synchronise
with accesses by teh hypervisor itself. All users of cf8/cfc go
through new access functions which take the appropriate spinlock.

Based on a patch by Haitao Shan <haitao.shan@intel.com>

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri Apr 11 13:19:55 2008 +0100 (2008-04-11)
parents 5b7a3e040683
children 8bced3d8a907
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Allen Kay <allen.m.kay@intel.com>
18 * Copyright (C) Xiaohui Xin <xiaohui.xin@intel.com>
19 */
22 #include <xen/sched.h>
23 #include <xen/iommu.h>
24 #include <xen/time.h>
25 #include <xen/pci.h>
26 #include "iommu.h"
27 #include "dmar.h"
28 #include "vtd.h"
29 #include "../pci_regs.h"
30 #include "msi.h"
31 #include "extern.h"
33 static void print_qi_regs(struct iommu *iommu)
34 {
35 u64 val;
37 val = dmar_readq(iommu->reg, DMAR_IQA_REG);
38 printk("DMAR_IAQ_REG = %"PRIx64"\n", val);
40 val = dmar_readq(iommu->reg, DMAR_IQH_REG);
41 printk("DMAR_IAH_REG = %"PRIx64"\n", val);
43 val = dmar_readq(iommu->reg, DMAR_IQT_REG);
44 printk("DMAR_IAT_REG = %"PRIx64"\n", val);
45 }
47 static int qinval_next_index(struct iommu *iommu)
48 {
49 u64 val;
50 val = dmar_readq(iommu->reg, DMAR_IQT_REG);
51 return (val >> 4);
52 }
54 static int qinval_update_qtail(struct iommu *iommu, int index)
55 {
56 u64 val;
58 /* Need an ASSERT to insure that we have got register lock */
59 val = (index < (QINVAL_ENTRY_NR-1)) ? (index + 1) : 0;
60 dmar_writeq(iommu->reg, DMAR_IQT_REG, (val << 4));
61 return 0;
62 }
64 static int gen_cc_inv_dsc(struct iommu *iommu, int index,
65 u16 did, u16 source_id, u8 function_mask, u8 granu)
66 {
67 unsigned long flags;
68 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
69 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
71 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
72 qinval_entries =
73 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
74 qinval_entry = &qinval_entries[index];
75 qinval_entry->q.cc_inv_dsc.lo.type = TYPE_INVAL_CONTEXT;
76 qinval_entry->q.cc_inv_dsc.lo.granu = granu;
77 qinval_entry->q.cc_inv_dsc.lo.res_1 = 0;
78 qinval_entry->q.cc_inv_dsc.lo.did = did;
79 qinval_entry->q.cc_inv_dsc.lo.sid = source_id;
80 qinval_entry->q.cc_inv_dsc.lo.fm = function_mask;
81 qinval_entry->q.cc_inv_dsc.lo.res_2 = 0;
82 qinval_entry->q.cc_inv_dsc.hi.res = 0;
84 unmap_vtd_domain_page(qinval_entries);
85 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
87 return 0;
88 }
90 int queue_invalidate_context(struct iommu *iommu,
91 u16 did, u16 source_id, u8 function_mask, u8 granu)
92 {
93 int ret = -1;
94 unsigned long flags;
95 int index = -1;
97 spin_lock_irqsave(&iommu->register_lock, flags);
98 index = qinval_next_index(iommu);
99 if ( index == -1 )
100 return -EBUSY;
101 ret = gen_cc_inv_dsc(iommu, index, did, source_id,
102 function_mask, granu);
103 ret |= qinval_update_qtail(iommu, index);
104 spin_unlock_irqrestore(&iommu->register_lock, flags);
105 return ret;
106 }
108 static int gen_iotlb_inv_dsc(struct iommu *iommu, int index,
109 u8 granu, u8 dr, u8 dw, u16 did, u8 am, u8 ih, u64 addr)
110 {
111 unsigned long flags;
112 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
113 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
115 if ( index == -1 )
116 return -1;
117 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
119 qinval_entries =
120 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
121 qinval_entry = &qinval_entries[index];
122 qinval_entry->q.iotlb_inv_dsc.lo.type = TYPE_INVAL_IOTLB;
123 qinval_entry->q.iotlb_inv_dsc.lo.granu = granu;
124 qinval_entry->q.iotlb_inv_dsc.lo.dr = 0;
125 qinval_entry->q.iotlb_inv_dsc.lo.dw = 0;
126 qinval_entry->q.iotlb_inv_dsc.lo.res_1 = 0;
127 qinval_entry->q.iotlb_inv_dsc.lo.did = did;
128 qinval_entry->q.iotlb_inv_dsc.lo.res_2 = 0;
130 qinval_entry->q.iotlb_inv_dsc.hi.am = am;
131 qinval_entry->q.iotlb_inv_dsc.hi.ih = ih;
132 qinval_entry->q.iotlb_inv_dsc.hi.res_1 = 0;
133 qinval_entry->q.iotlb_inv_dsc.hi.addr = addr;
135 unmap_vtd_domain_page(qinval_entries);
136 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
137 return 0;
138 }
140 int queue_invalidate_iotlb(struct iommu *iommu,
141 u8 granu, u8 dr, u8 dw, u16 did, u8 am, u8 ih, u64 addr)
142 {
143 int ret = -1;
144 unsigned long flags;
145 int index = -1;
147 spin_lock_irqsave(&iommu->register_lock, flags);
149 index = qinval_next_index(iommu);
150 ret = gen_iotlb_inv_dsc(iommu, index, granu, dr, dw, did,
151 am, ih, addr);
152 ret |= qinval_update_qtail(iommu, index);
153 spin_unlock_irqrestore(&iommu->register_lock, flags);
154 return ret;
155 }
157 static int gen_wait_dsc(struct iommu *iommu, int index,
158 u8 iflag, u8 sw, u8 fn, u32 sdata, volatile u32 *saddr)
159 {
160 unsigned long flags;
161 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
162 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
164 if ( index == -1 )
165 return -1;
166 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
167 qinval_entries =
168 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
169 qinval_entry = &qinval_entries[index];
170 qinval_entry->q.inv_wait_dsc.lo.type = TYPE_INVAL_WAIT;
171 qinval_entry->q.inv_wait_dsc.lo.iflag = iflag;
172 qinval_entry->q.inv_wait_dsc.lo.sw = sw;
173 qinval_entry->q.inv_wait_dsc.lo.fn = fn;
174 qinval_entry->q.inv_wait_dsc.lo.res_1 = 0;
175 qinval_entry->q.inv_wait_dsc.lo.sdata = sdata;
176 qinval_entry->q.inv_wait_dsc.hi.res_1 = 0;
177 qinval_entry->q.inv_wait_dsc.hi.saddr = virt_to_maddr(saddr) >> 2;
178 unmap_vtd_domain_page(qinval_entries);
179 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
180 return 0;
181 }
183 static int queue_invalidate_wait(struct iommu *iommu,
184 u8 iflag, u8 sw, u8 fn, u32 sdata, volatile u32 *saddr)
185 {
186 unsigned long flags;
187 s_time_t start_time;
188 int index = -1;
189 int ret = -1;
190 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
192 spin_lock_irqsave(&qi_ctrl->qinval_poll_lock, flags);
193 spin_lock_irqsave(&iommu->register_lock, flags);
194 index = qinval_next_index(iommu);
195 if ( *saddr == 1 )
196 *saddr = 0;
197 ret = gen_wait_dsc(iommu, index, iflag, sw, fn, sdata, saddr);
198 ret |= qinval_update_qtail(iommu, index);
199 spin_unlock_irqrestore(&iommu->register_lock, flags);
201 /* Now we don't support interrupt method */
202 if ( sw )
203 {
204 /* In case all wait descriptor writes to same addr with same data */
205 start_time = NOW();
206 while ( *saddr != 1 )
207 {
208 if ( NOW() > (start_time + DMAR_OPERATION_TIMEOUT) )
209 {
210 print_qi_regs(iommu);
211 panic("queue invalidate wait descriptor was not executed\n");
212 }
213 cpu_relax();
214 }
215 }
216 spin_unlock_irqrestore(&qi_ctrl->qinval_poll_lock, flags);
217 return ret;
218 }
220 int invalidate_sync(struct iommu *iommu)
221 {
222 int ret = -1;
223 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
225 if ( qi_ctrl->qinval_maddr == 0 )
226 {
227 ret = queue_invalidate_wait(iommu,
228 0, 1, 1, 1, &qi_ctrl->qinval_poll_status);
229 return ret;
230 }
231 return 0;
232 }
234 static int gen_dev_iotlb_inv_dsc(struct iommu *iommu, int index,
235 u32 max_invs_pend, u16 sid, u16 size, u64 addr)
236 {
237 unsigned long flags;
238 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
239 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
241 if ( index == -1 )
242 return -1;
243 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
245 qinval_entries =
246 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
247 qinval_entry = &qinval_entries[index];
248 qinval_entry->q.dev_iotlb_inv_dsc.lo.type = TYPE_INVAL_DEVICE_IOTLB;
249 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_1 = 0;
250 qinval_entry->q.dev_iotlb_inv_dsc.lo.max_invs_pend = max_invs_pend;
251 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_2 = 0;
252 qinval_entry->q.dev_iotlb_inv_dsc.lo.sid = sid;
253 qinval_entry->q.dev_iotlb_inv_dsc.lo.res_3 = 0;
255 qinval_entry->q.dev_iotlb_inv_dsc.hi.size = size;
256 qinval_entry->q.dev_iotlb_inv_dsc.hi.addr = addr;
258 unmap_vtd_domain_page(qinval_entries);
259 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
260 return 0;
261 }
263 int queue_invalidate_device_iotlb(struct iommu *iommu,
264 u32 max_invs_pend, u16 sid, u16 size, u64 addr)
265 {
266 int ret = -1;
267 unsigned long flags;
268 int index = -1;
270 spin_lock_irqsave(&iommu->register_lock, flags);
271 index = qinval_next_index(iommu);
272 ret = gen_dev_iotlb_inv_dsc(iommu, index, max_invs_pend,
273 sid, size, addr);
274 ret |= qinval_update_qtail(iommu, index);
275 spin_unlock_irqrestore(&iommu->register_lock, flags);
276 return ret;
277 }
279 static int gen_iec_inv_dsc(struct iommu *iommu, int index,
280 u8 granu, u8 im, u16 iidx)
281 {
282 unsigned long flags;
283 struct qinval_entry *qinval_entry = NULL, *qinval_entries;
284 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
286 if ( index == -1 )
287 return -1;
288 spin_lock_irqsave(&qi_ctrl->qinval_lock, flags);
290 qinval_entries =
291 (struct qinval_entry *)map_vtd_domain_page(qi_ctrl->qinval_maddr);
292 qinval_entry = &qinval_entries[index];
293 qinval_entry->q.iec_inv_dsc.lo.type = TYPE_INVAL_IEC;
294 qinval_entry->q.iec_inv_dsc.lo.granu = granu;
295 qinval_entry->q.iec_inv_dsc.lo.res_1 = 0;
296 qinval_entry->q.iec_inv_dsc.lo.im = im;
297 qinval_entry->q.iec_inv_dsc.lo.iidx = iidx;
298 qinval_entry->q.iec_inv_dsc.lo.res_2 = 0;
299 qinval_entry->q.iec_inv_dsc.hi.res = 0;
301 unmap_vtd_domain_page(qinval_entries);
302 spin_unlock_irqrestore(&qi_ctrl->qinval_lock, flags);
303 return 0;
304 }
306 int queue_invalidate_iec(struct iommu *iommu, u8 granu, u8 im, u16 iidx)
307 {
308 int ret;
309 unsigned long flags;
310 int index = -1;
312 spin_lock_irqsave(&iommu->register_lock, flags);
313 index = qinval_next_index(iommu);
314 ret = gen_iec_inv_dsc(iommu, index, granu, im, iidx);
315 ret |= qinval_update_qtail(iommu, index);
316 spin_unlock_irqrestore(&iommu->register_lock, flags);
317 return ret;
318 }
320 u64 iec_cap;
321 int __iommu_flush_iec(struct iommu *iommu, u8 granu, u8 im, u16 iidx)
322 {
323 int ret;
324 ret = queue_invalidate_iec(iommu, granu, im, iidx);
325 ret |= invalidate_sync(iommu);
327 /*
328 * reading vt-d architecture register will ensure
329 * draining happens in implementation independent way.
330 */
331 iec_cap = dmar_readq(iommu->reg, DMAR_CAP_REG);
332 return ret;
333 }
335 int iommu_flush_iec_global(struct iommu *iommu)
336 {
337 return __iommu_flush_iec(iommu, IEC_GLOBAL_INVL, 0, 0);
338 }
340 int iommu_flush_iec_index(struct iommu *iommu, u8 im, u16 iidx)
341 {
342 return __iommu_flush_iec(iommu, IEC_INDEX_INVL, im, iidx);
343 }
345 static int flush_context_qi(
346 void *_iommu, u16 did, u16 sid, u8 fm, u64 type,
347 int non_present_entry_flush)
348 {
349 int ret = 0;
350 struct iommu *iommu = (struct iommu *)_iommu;
351 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
353 /*
354 * In the non-present entry flush case, if hardware doesn't cache
355 * non-present entry we do nothing and if hardware cache non-present
356 * entry, we flush entries of domain 0 (the domain id is used to cache
357 * any non-present entries)
358 */
359 if ( non_present_entry_flush )
360 {
361 if ( !cap_caching_mode(iommu->cap) )
362 return 1;
363 else
364 did = 0;
365 }
367 if ( qi_ctrl->qinval_maddr != 0 )
368 {
369 ret = queue_invalidate_context(iommu, did, sid, fm,
370 type >> DMA_CCMD_INVL_GRANU_OFFSET);
371 ret |= invalidate_sync(iommu);
372 }
373 return ret;
374 }
376 static int flush_iotlb_qi(
377 void *_iommu, u16 did,
378 u64 addr, unsigned int size_order, u64 type,
379 int non_present_entry_flush)
380 {
381 u8 dr = 0, dw = 0;
382 int ret = 0;
383 struct iommu *iommu = (struct iommu *)_iommu;
384 struct qi_ctrl *qi_ctrl = iommu_qi_ctrl(iommu);
386 /*
387 * In the non-present entry flush case, if hardware doesn't cache
388 * non-present entry we do nothing and if hardware cache non-present
389 * entry, we flush entries of domain 0 (the domain id is used to cache
390 * any non-present entries)
391 */
392 if ( non_present_entry_flush )
393 {
394 if ( !cap_caching_mode(iommu->cap) )
395 return 1;
396 else
397 did = 0;
398 }
400 if ( qi_ctrl->qinval_maddr != 0 )
401 {
402 /* use queued invalidation */
403 if (cap_write_drain(iommu->cap))
404 dw = 1;
405 if (cap_read_drain(iommu->cap))
406 dr = 1;
407 /* Need to conside the ih bit later */
408 ret = queue_invalidate_iotlb(iommu,
409 (type >> DMA_TLB_FLUSH_GRANU_OFFSET), dr,
410 dw, did, (u8)size_order, 0, addr);
411 ret |= invalidate_sync(iommu);
412 }
413 return ret;
414 }
416 int qinval_setup(struct iommu *iommu)
417 {
418 s_time_t start_time;
419 u32 status = 0;
420 struct qi_ctrl *qi_ctrl;
421 struct iommu_flush *flush;
423 qi_ctrl = iommu_qi_ctrl(iommu);
424 flush = iommu_get_flush(iommu);
426 if ( !ecap_queued_inval(iommu->ecap) )
427 return -ENODEV;
429 if ( qi_ctrl->qinval_maddr == 0 )
430 {
431 qi_ctrl->qinval_maddr = alloc_pgtable_maddr();
432 if ( qi_ctrl->qinval_maddr == 0 )
433 panic("Cannot allocate memory for qi_ctrl->qinval_maddr\n");
434 flush->context = flush_context_qi;
435 flush->iotlb = flush_iotlb_qi;
436 }
438 /* Setup Invalidation Queue Address(IQA) register with the
439 * address of the page we just allocated. QS field at
440 * bits[2:0] to indicate size of queue is one 4KB page.
441 * That's 256 entries. Queued Head (IQH) and Queue Tail (IQT)
442 * registers are automatically reset to 0 with write
443 * to IQA register.
444 */
445 dmar_writeq(iommu->reg, DMAR_IQA_REG, qi_ctrl->qinval_maddr);
447 /* enable queued invalidation hardware */
448 iommu->gcmd |= DMA_GCMD_QIE;
449 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
451 /* Make sure hardware complete it */
452 start_time = NOW();
453 for ( ; ; )
454 {
455 status = dmar_readl(iommu->reg, DMAR_GSTS_REG);
456 if ( status & DMA_GSTS_QIES )
457 break;
458 if ( NOW() > (start_time + DMAR_OPERATION_TIMEOUT) )
459 panic("Cannot set QIE field for queue invalidation\n");
460 cpu_relax();
461 }
462 status = 0;
463 return status;
464 }