direct-io.hg

view xen/arch/ia64/vmx/vtlb.c @ 11301:50837fb04807

[IA64] vti bugs fix

Bug fixes:
- Do not read long-format vhpt as short-format.
- Avoid infinite loop in vtlb_purge.

Signed-off-by: Tristan Gingold <tristan.gingold@bull.net>
author awilliam@xenbuild.aw
date Mon Aug 14 11:46:40 2006 -0600 (2006-08-14)
parents 561df7d9cecc
children 45cd04201495
line source
2 /* -*- Mode:C; c-basic-offset:4; tab-width:4; indent-tabs-mode:nil -*- */
3 /*
4 * vtlb.c: guest virtual tlb handling module.
5 * Copyright (c) 2004, Intel Corporation.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
18 * Place - Suite 330, Boston, MA 02111-1307 USA.
19 *
20 * Yaozu Dong (Eddie Dong) (Eddie.dong@intel.com)
21 * XiaoYan Feng (Fleming Feng) (Fleming.feng@intel.com)
22 */
24 #include <linux/sched.h>
25 #include <asm/tlb.h>
26 #include <asm/mm.h>
27 #include <asm/vmx_mm_def.h>
28 #include <asm/gcc_intrin.h>
29 #include <linux/interrupt.h>
30 #include <asm/vmx_vcpu.h>
31 #include <asm/vmx_phy_mode.h>
32 #include <asm/vmmu.h>
33 #include <asm/tlbflush.h>
34 #include <asm/regionreg.h>
35 #define MAX_CCH_LENGTH 40
37 thash_data_t *__alloc_chain(thash_cb_t *);
39 static void cch_mem_init(thash_cb_t *hcb)
40 {
41 int num;
42 thash_data_t *p;
44 hcb->cch_freelist = p = hcb->cch_buf;
45 num = (hcb->cch_sz/sizeof(thash_data_t))-1;
46 do{
47 p->next =p+1;
48 p++;
49 num--;
50 }while(num);
51 p->next = NULL;
52 }
54 static thash_data_t *cch_alloc(thash_cb_t *hcb)
55 {
56 thash_data_t *p;
57 if ( (p = hcb->cch_freelist) != NULL ) {
58 hcb->cch_freelist = p->next;
59 }
60 return p;
61 }
63 /*
64 * Check to see if the address rid:va is translated by the TLB
65 */
67 static inline int __is_tr_translated(thash_data_t *trp, u64 rid, u64 va)
68 {
69 return ((trp->p) && (trp->rid == rid) && ((va-trp->vadr)<PSIZE(trp->ps)));
70 }
72 /*
73 * Only for GUEST TR format.
74 */
75 static int
76 __is_tr_overlap(thash_data_t *trp, u64 rid, u64 sva, u64 eva)
77 {
78 uint64_t sa1, ea1;
80 if (!trp->p || trp->rid != rid ) {
81 return 0;
82 }
83 sa1 = trp->vadr;
84 ea1 = sa1 + PSIZE(trp->ps) -1;
85 eva -= 1;
86 if ( (sva>ea1) || (sa1>eva) )
87 return 0;
88 else
89 return 1;
91 }
93 thash_data_t *__vtr_lookup(VCPU *vcpu, u64 va, int is_data)
94 {
96 thash_data_t *trp;
97 int i;
98 u64 rid;
99 vcpu_get_rr(vcpu, va, &rid);
100 rid = rid&RR_RID_MASK;;
101 if (is_data) {
102 if (vcpu_quick_region_check(vcpu->arch.dtr_regions,va)) {
103 for (trp =(thash_data_t *) vcpu->arch.dtrs,i=0; i<NDTRS; i++, trp++) {
104 if (__is_tr_translated(trp, rid, va)) {
105 return trp;
106 }
107 }
108 }
109 }
110 else {
111 if (vcpu_quick_region_check(vcpu->arch.itr_regions,va)) {
112 for (trp =(thash_data_t *) vcpu->arch.itrs,i=0; i<NITRS; i++, trp++) {
113 if (__is_tr_translated(trp, rid, va)) {
114 return trp;
115 }
116 }
117 }
118 }
119 return NULL;
120 }
123 static void thash_recycle_cch(thash_cb_t *hcb, thash_data_t *hash)
124 {
125 thash_data_t *p, *q;
126 int i=0;
128 p=hash;
129 for(i=0; i < MAX_CCN_DEPTH; i++){
130 p=p->next;
131 }
132 q=hash->next;
133 hash->len=0;
134 hash->next=0;
135 p->next=hcb->cch_freelist;
136 hcb->cch_freelist=q;
137 }
142 static void vmx_vhpt_insert(thash_cb_t *hcb, u64 pte, u64 itir, u64 ifa)
143 {
144 u64 tag ,len;
145 ia64_rr rr;
146 thash_data_t *head, *cch;
147 pte = pte & ~PAGE_FLAGS_RV_MASK;
148 rr.rrval = ia64_get_rr(ifa);
149 head = (thash_data_t *)ia64_thash(ifa);
150 tag = ia64_ttag(ifa);
151 if( INVALID_VHPT(head) ) {
152 len = head->len;
153 head->page_flags = pte;
154 head->len = len;
155 head->itir = rr.ps << 2;
156 head->etag = tag;
157 return;
158 }
160 if(head->len>=MAX_CCN_DEPTH){
161 thash_recycle_cch(hcb, head);
162 cch = cch_alloc(hcb);
163 }
164 else{
165 cch = __alloc_chain(hcb);
166 }
167 *cch = *head;
168 head->page_flags=pte;
169 head->itir = rr.ps << 2;
170 head->etag=tag;
171 head->next = cch;
172 head->len = cch->len+1;
173 cch->len = 0;
174 return;
175 }
177 void thash_vhpt_insert(VCPU *v, u64 pte, u64 itir, u64 va)
178 {
179 u64 phy_pte;
180 phy_pte=translate_phy_pte(v, &pte, itir, va);
181 vmx_vhpt_insert(vcpu_get_vhpt(v), phy_pte, itir, va);
182 }
183 /*
184 * vhpt lookup
185 */
187 thash_data_t * vhpt_lookup(u64 va)
188 {
189 thash_data_t *hash, *head;
190 u64 tag, pte;
191 head = (thash_data_t *)ia64_thash(va);
192 hash=head;
193 tag = ia64_ttag(va);
194 do{
195 if(hash->etag == tag)
196 break;
197 hash=hash->next;
198 }while(hash);
199 if(hash && hash!=head){
200 pte = hash->page_flags;
201 hash->page_flags = head->page_flags;
202 head->page_flags = pte;
203 tag = hash->etag;
204 hash->etag = head->etag;
205 head->etag = tag;
206 head->len = hash->len;
207 hash->len=0;
208 return head;
209 }
210 return hash;
211 }
213 u64 guest_vhpt_lookup(u64 iha, u64 *pte)
214 {
215 u64 ret;
216 thash_data_t * data;
217 PTA vpta;
219 data = vhpt_lookup(iha);
220 if (data == NULL) {
221 data = vtlb_lookup(current, iha, DSIDE_TLB);
222 if (data != NULL)
223 thash_vhpt_insert(current, data->page_flags, data->itir ,iha);
224 }
226 /* VHPT long format is not read. */
227 vmx_vcpu_get_pta(current, &vpta.val);
228 if (vpta.vf == 1) {
229 *pte = 0;
230 return 0;
231 }
233 asm volatile ("rsm psr.ic|psr.i;;"
234 "srlz.d;;"
235 "ld8.s r9=[%1];;"
236 "tnat.nz p6,p7=r9;;"
237 "(p6) mov %0=1;"
238 "(p6) mov r9=r0;"
239 "(p7) mov %0=r0;"
240 "(p7) st8 [%2]=r9;;"
241 "ssm psr.ic;;"
242 "srlz.d;;"
243 "ssm psr.i;;"
244 : "=r"(ret) : "r"(iha), "r"(pte):"memory");
245 return ret;
246 }
249 /*
250 * purge software guest tlb
251 */
253 void vtlb_purge(VCPU *v, u64 va, u64 ps)
254 {
255 thash_data_t *cur;
256 u64 start, end, curadr, size, psbits, tag, def_size;
257 ia64_rr vrr;
258 thash_cb_t *hcb = &v->arch.vtlb;
259 vcpu_get_rr(v, va, &vrr.rrval);
260 psbits = VMX(v, psbits[(va >> 61)]);
261 size = PSIZE(ps);
262 start = va & (-size);
263 end = start + size;
264 while (psbits) {
265 curadr = start;
266 ps = __ffs(psbits);
267 psbits &= ~(1UL << ps);
268 def_size = PSIZE(ps);
269 vrr.ps = ps;
270 /* Be careful about overflow. */
271 while (curadr < end && curadr >= start) {
272 cur = vsa_thash(hcb->pta, curadr, vrr.rrval, &tag);
273 while (cur) {
274 if (cur->etag == tag && cur->ps == ps)
275 cur->etag = 1UL << 63;
276 cur = cur->next;
277 }
278 curadr += def_size;
279 }
280 }
281 }
284 /*
285 * purge VHPT and machine TLB
286 */
287 static void vhpt_purge(VCPU *v, u64 va, u64 ps)
288 {
289 //thash_cb_t *hcb = &v->arch.vhpt;
290 thash_data_t *cur;
291 u64 start, end, size, tag;
292 ia64_rr rr;
293 size = PSIZE(ps);
294 start = va & (-size);
295 end = start + size;
296 rr.rrval = ia64_get_rr(va);
297 size = PSIZE(rr.ps);
298 while(start < end){
299 cur = (thash_data_t *)ia64_thash(start);
300 tag = ia64_ttag(start);
301 while (cur) {
302 if (cur->etag == tag)
303 cur->etag = 1UL << 63;
304 cur = cur->next;
305 }
306 start += size;
307 }
308 machine_tlb_purge(va, ps);
309 }
311 /*
312 * Recycle all collisions chain in VTLB or VHPT.
313 *
314 */
315 void thash_recycle_cch_all(thash_cb_t *hcb)
316 {
317 int num;
318 thash_data_t *head;
319 head=hcb->hash;
320 num = (hcb->hash_sz/sizeof(thash_data_t));
321 do{
322 head->len = 0;
323 head->next = 0;
324 head++;
325 num--;
326 }while(num);
327 cch_mem_init(hcb);
328 }
331 thash_data_t *__alloc_chain(thash_cb_t *hcb)
332 {
333 thash_data_t *cch;
335 cch = cch_alloc(hcb);
336 if(cch == NULL){
337 thash_recycle_cch_all(hcb);
338 cch = cch_alloc(hcb);
339 }
340 return cch;
341 }
343 /*
344 * Insert an entry into hash TLB or VHPT.
345 * NOTES:
346 * 1: When inserting VHPT to thash, "va" is a must covered
347 * address by the inserted machine VHPT entry.
348 * 2: The format of entry is always in TLB.
349 * 3: The caller need to make sure the new entry will not overlap
350 * with any existed entry.
351 */
352 void vtlb_insert(VCPU *v, u64 pte, u64 itir, u64 va)
353 {
354 thash_data_t *hash_table, *cch;
355 /* int flag; */
356 ia64_rr vrr;
357 /* u64 gppn, ppns, ppne; */
358 u64 tag, len;
359 thash_cb_t *hcb = &v->arch.vtlb;
360 vcpu_get_rr(v, va, &vrr.rrval);
361 #ifdef VTLB_DEBUG
362 if (vrr.ps != itir_ps(itir)) {
363 // machine_tlb_insert(hcb->vcpu, entry);
364 panic_domain(NULL, "not preferred ps with va: 0x%lx vrr.ps=%d ps=%ld\n",
365 va, vrr.ps, itir_ps(itir));
366 return;
367 }
368 #endif
369 vrr.ps = itir_ps(itir);
370 VMX(v, psbits[va >> 61]) |= (1UL << vrr.ps);
371 hash_table = vsa_thash(hcb->pta, va, vrr.rrval, &tag);
372 if( INVALID_TLB(hash_table) ) {
373 len = hash_table->len;
374 hash_table->page_flags = pte;
375 hash_table->len = len;
376 hash_table->itir=itir;
377 hash_table->etag=tag;
378 return;
379 }
380 if (hash_table->len>=MAX_CCN_DEPTH){
381 thash_recycle_cch(hcb, hash_table);
382 cch = cch_alloc(hcb);
383 }
384 else {
385 cch = __alloc_chain(hcb);
386 }
387 *cch = *hash_table;
388 hash_table->page_flags = pte;
389 hash_table->itir=itir;
390 hash_table->etag=tag;
391 hash_table->next = cch;
392 hash_table->len = cch->len + 1;
393 cch->len = 0;
394 return ;
395 }
398 int vtr_find_overlap(VCPU *vcpu, u64 va, u64 ps, int is_data)
399 {
400 thash_data_t *trp;
401 int i;
402 u64 end, rid;
403 vcpu_get_rr(vcpu, va, &rid);
404 rid = rid&RR_RID_MASK;;
405 end = va + PSIZE(ps);
406 if (is_data) {
407 if (vcpu_quick_region_check(vcpu->arch.dtr_regions,va)) {
408 for (trp =(thash_data_t *) vcpu->arch.dtrs,i=0; i<NDTRS; i++, trp++) {
409 if (__is_tr_overlap(trp, rid, va, end )) {
410 return i;
411 }
412 }
413 }
414 }
415 else {
416 if (vcpu_quick_region_check(vcpu->arch.itr_regions,va)) {
417 for (trp =(thash_data_t *) vcpu->arch.itrs,i=0; i<NITRS; i++, trp++) {
418 if (__is_tr_overlap(trp, rid, va, end )) {
419 return i;
420 }
421 }
422 }
423 }
424 return -1;
425 }
427 /*
428 * Purge entries in VTLB and VHPT
429 */
430 void thash_purge_entries(VCPU *v, u64 va, u64 ps)
431 {
432 if(vcpu_quick_region_check(v->arch.tc_regions,va))
433 vtlb_purge(v, va, ps);
434 vhpt_purge(v, va, ps);
435 }
437 u64 translate_phy_pte(VCPU *v, u64 *pte, u64 itir, u64 va)
438 {
439 u64 ps, ps_mask, paddr, maddr;
440 // ia64_rr rr;
441 union pte_flags phy_pte;
442 ps = itir_ps(itir);
443 ps_mask = ~((1UL << ps) - 1);
444 phy_pte.val = *pte;
445 paddr = *pte;
446 paddr = ((paddr & _PAGE_PPN_MASK) & ps_mask) | (va & ~ps_mask);
447 maddr = lookup_domain_mpa(v->domain, paddr, NULL);
448 if (maddr & GPFN_IO_MASK) {
449 *pte |= VTLB_PTE_IO;
450 return -1;
451 }
452 // rr.rrval = ia64_get_rr(va);
453 // ps = rr.ps;
454 maddr = ((maddr & _PAGE_PPN_MASK) & PAGE_MASK) | (paddr & ~PAGE_MASK);
455 phy_pte.ppn = maddr >> ARCH_PAGE_SHIFT;
456 return phy_pte.val;
457 }
460 /*
461 * Purge overlap TCs and then insert the new entry to emulate itc ops.
462 * Notes: Only TC entry can purge and insert.
463 */
464 void thash_purge_and_insert(VCPU *v, u64 pte, u64 itir, u64 ifa, int type)
465 {
466 u64 ps;//, va;
467 u64 phy_pte;
468 ia64_rr vrr, mrr;
469 ps = itir_ps(itir);
470 vcpu_get_rr(current, ifa, &vrr.rrval);
471 mrr.rrval = ia64_get_rr(ifa);
472 // if (vrr.ps != itir_ps(itir)) {
473 // printf("not preferred ps with va: 0x%lx vrr.ps=%d ps=%ld\n",
474 // ifa, vrr.ps, itir_ps(itir));
475 // }
476 if(VMX_DOMAIN(v)){
477 /* Ensure WB attribute if pte is related to a normal mem page,
478 * which is required by vga acceleration since qemu maps shared
479 * vram buffer with WB.
480 */
481 if (!(pte & VTLB_PTE_IO) && ((pte & _PAGE_MA_MASK) != _PAGE_MA_NAT))
482 pte &= ~_PAGE_MA_MASK;
484 phy_pte = translate_phy_pte(v, &pte, itir, ifa);
485 vtlb_purge(v, ifa, ps);
486 vhpt_purge(v, ifa, ps);
487 if (ps == mrr.ps) {
488 if(!(pte&VTLB_PTE_IO)){
489 vmx_vhpt_insert(&v->arch.vhpt, phy_pte, itir, ifa);
490 }
491 else{
492 vtlb_insert(v, pte, itir, ifa);
493 vcpu_quick_region_set(PSCBX(v,tc_regions),ifa);
494 }
495 }
496 else if (ps > mrr.ps) {
497 vtlb_insert(v, pte, itir, ifa);
498 vcpu_quick_region_set(PSCBX(v,tc_regions),ifa);
499 if(!(pte&VTLB_PTE_IO)){
500 vmx_vhpt_insert(&v->arch.vhpt, phy_pte, itir, ifa);
501 }
502 }
503 else {
504 u64 psr;
505 phy_pte &= ~PAGE_FLAGS_RV_MASK;
506 psr = ia64_clear_ic();
507 ia64_itc(type + 1, ifa, phy_pte, ps);
508 ia64_set_psr(psr);
509 ia64_srlz_i();
510 // ps < mrr.ps, this is not supported
511 // panic_domain(NULL, "%s: ps (%lx) < mrr.ps \n", __func__, ps);
512 }
513 }
514 else{
515 phy_pte = translate_phy_pte(v, &pte, itir, ifa);
516 if(ps!=PAGE_SHIFT){
517 vtlb_insert(v, pte, itir, ifa);
518 vcpu_quick_region_set(PSCBX(v,tc_regions),ifa);
519 }
520 machine_tlb_purge(ifa, ps);
521 vmx_vhpt_insert(&v->arch.vhpt, phy_pte, itir, ifa);
522 }
523 }
525 /*
526 * Purge all TCs or VHPT entries including those in Hash table.
527 *
528 */
530 //TODO: add sections.
531 void thash_purge_all(VCPU *v)
532 {
533 int num;
534 thash_data_t *head;
535 thash_cb_t *vtlb,*vhpt;
536 vtlb =&v->arch.vtlb;
537 vhpt =&v->arch.vhpt;
539 for (num = 0; num < 8; num++)
540 VMX(v, psbits[num]) = 0;
542 head=vtlb->hash;
543 num = (vtlb->hash_sz/sizeof(thash_data_t));
544 do{
545 head->page_flags = 0;
546 head->etag = 1UL<<63;
547 head->itir = 0;
548 head->next = 0;
549 head++;
550 num--;
551 }while(num);
552 cch_mem_init(vtlb);
554 head=vhpt->hash;
555 num = (vhpt->hash_sz/sizeof(thash_data_t));
556 do{
557 head->page_flags = 0;
558 head->etag = 1UL<<63;
559 head->next = 0;
560 head++;
561 num--;
562 }while(num);
563 cch_mem_init(vhpt);
564 local_flush_tlb_all();
565 }
568 /*
569 * Lookup the hash table and its collision chain to find an entry
570 * covering this address rid:va or the entry.
571 *
572 * INPUT:
573 * in: TLB format for both VHPT & TLB.
574 */
576 thash_data_t *vtlb_lookup(VCPU *v, u64 va,int is_data)
577 {
578 thash_data_t *cch;
579 u64 psbits, ps, tag;
580 ia64_rr vrr;
581 thash_cb_t * hcb= &v->arch.vtlb;
583 cch = __vtr_lookup(v, va, is_data);;
584 if ( cch ) return cch;
586 if(vcpu_quick_region_check(v->arch.tc_regions,va)==0)
587 return NULL;
588 psbits = VMX(v, psbits[(va >> 61)]);
589 vcpu_get_rr(v,va,&vrr.rrval);
590 while (psbits) {
591 ps = __ffs(psbits);
592 psbits &= ~(1UL << ps);
593 vrr.ps = ps;
594 cch = vsa_thash(hcb->pta, va, vrr.rrval, &tag);
595 do {
596 if (cch->etag == tag && cch->ps == ps)
597 return cch;
598 cch = cch->next;
599 } while(cch);
600 }
601 return NULL;
602 }
605 /*
606 * Initialize internal control data before service.
607 */
608 void thash_init(thash_cb_t *hcb, u64 sz)
609 {
610 int num;
611 thash_data_t *head, *p;
613 hcb->pta.val = (unsigned long)hcb->hash;
614 hcb->pta.vf = 1;
615 hcb->pta.ve = 1;
616 hcb->pta.size = sz;
617 hcb->cch_rec_head = hcb->hash;
619 head=hcb->hash;
620 num = (hcb->hash_sz/sizeof(thash_data_t));
621 do{
622 head->itir = PAGE_SHIFT<<2;
623 head->etag = 1UL<<63;
624 head->next = 0;
625 head++;
626 num--;
627 }while(num);
629 hcb->cch_freelist = p = hcb->cch_buf;
630 num = (hcb->cch_sz/sizeof(thash_data_t))-1;
631 do{
632 p->itir = PAGE_SHIFT<<2;
633 p->next =p+1;
634 p++;
635 num--;
636 }while(num);
637 p->itir = PAGE_SHIFT<<2;
638 p->next = NULL;
639 }