ia64/xen-unstable

view xen/arch/ia64/vmx/vtlb.c @ 11823:2a9c0f4682ed

[IA64] Correctly not handle VHPT long format.

Signed-off-by: Tristan Gingold <tristan.gingold@bull.net>
author awilliam@xenbuild.aw
date Wed Oct 18 22:07:06 2006 -0600 (2006-10-18)
parents 261b95f114a2
children 9e9d8696fb55
line source
2 /* -*- Mode:C; c-basic-offset:4; tab-width:4; indent-tabs-mode:nil -*- */
3 /*
4 * vtlb.c: guest virtual tlb handling module.
5 * Copyright (c) 2004, Intel Corporation.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
18 * Place - Suite 330, Boston, MA 02111-1307 USA.
19 *
20 * Yaozu Dong (Eddie Dong) (Eddie.dong@intel.com)
21 * XiaoYan Feng (Fleming Feng) (Fleming.feng@intel.com)
22 */
24 #include <linux/sched.h>
25 #include <asm/tlb.h>
26 #include <xen/mm.h>
27 #include <asm/vmx_mm_def.h>
28 #include <asm/gcc_intrin.h>
29 #include <linux/interrupt.h>
30 #include <asm/vmx_vcpu.h>
31 #include <asm/vmx_phy_mode.h>
32 #include <asm/vmmu.h>
33 #include <asm/tlbflush.h>
34 #include <asm/regionreg.h>
35 #define MAX_CCH_LENGTH 40
37 thash_data_t *__alloc_chain(thash_cb_t *);
39 static void cch_mem_init(thash_cb_t *hcb)
40 {
41 int num;
42 thash_data_t *p;
44 hcb->cch_freelist = p = hcb->cch_buf;
45 num = (hcb->cch_sz/sizeof(thash_data_t))-1;
46 do{
47 p->next =p+1;
48 p++;
49 num--;
50 }while(num);
51 p->next = NULL;
52 }
54 static thash_data_t *cch_alloc(thash_cb_t *hcb)
55 {
56 thash_data_t *p;
57 if ( (p = hcb->cch_freelist) != NULL ) {
58 hcb->cch_freelist = p->next;
59 }
60 return p;
61 }
63 /*
64 * Check to see if the address rid:va is translated by the TLB
65 */
67 static inline int __is_tr_translated(thash_data_t *trp, u64 rid, u64 va)
68 {
69 return ((trp->p) && (trp->rid == rid) && ((va-trp->vadr)<PSIZE(trp->ps)));
70 }
72 /*
73 * Only for GUEST TR format.
74 */
75 static int
76 __is_tr_overlap(thash_data_t *trp, u64 rid, u64 sva, u64 eva)
77 {
78 uint64_t sa1, ea1;
80 if (!trp->p || trp->rid != rid ) {
81 return 0;
82 }
83 sa1 = trp->vadr;
84 ea1 = sa1 + PSIZE(trp->ps) -1;
85 eva -= 1;
86 if ( (sva>ea1) || (sa1>eva) )
87 return 0;
88 else
89 return 1;
91 }
93 thash_data_t *__vtr_lookup(VCPU *vcpu, u64 va, int is_data)
94 {
96 thash_data_t *trp;
97 int i;
98 u64 rid;
99 vcpu_get_rr(vcpu, va, &rid);
100 rid = rid&RR_RID_MASK;;
101 if (is_data) {
102 if (vcpu_quick_region_check(vcpu->arch.dtr_regions,va)) {
103 for (trp =(thash_data_t *) vcpu->arch.dtrs,i=0; i<NDTRS; i++, trp++) {
104 if (__is_tr_translated(trp, rid, va)) {
105 return trp;
106 }
107 }
108 }
109 }
110 else {
111 if (vcpu_quick_region_check(vcpu->arch.itr_regions,va)) {
112 for (trp =(thash_data_t *) vcpu->arch.itrs,i=0; i<NITRS; i++, trp++) {
113 if (__is_tr_translated(trp, rid, va)) {
114 return trp;
115 }
116 }
117 }
118 }
119 return NULL;
120 }
123 static void thash_recycle_cch(thash_cb_t *hcb, thash_data_t *hash)
124 {
125 thash_data_t *p, *q;
126 int i=0;
128 p=hash;
129 for(i=0; i < MAX_CCN_DEPTH; i++){
130 p=p->next;
131 }
132 q=hash->next;
133 hash->len=0;
134 hash->next=0;
135 p->next=hcb->cch_freelist;
136 hcb->cch_freelist=q;
137 }
142 static void vmx_vhpt_insert(thash_cb_t *hcb, u64 pte, u64 itir, u64 ifa)
143 {
144 u64 tag ,len;
145 ia64_rr rr;
146 thash_data_t *head, *cch;
147 pte = pte & ~PAGE_FLAGS_RV_MASK;
148 rr.rrval = ia64_get_rr(ifa);
149 head = (thash_data_t *)ia64_thash(ifa);
150 tag = ia64_ttag(ifa);
151 cch = head;
152 while (cch) {
153 if (INVALID_VHPT(cch)) {
154 len = cch->len;
155 cch->page_flags = pte;
156 cch->len = len;
157 cch->itir = rr.ps << 2;
158 cch->etag = tag;
159 return;
160 }
161 cch = cch->next;
162 }
164 if(head->len>=MAX_CCN_DEPTH){
165 thash_recycle_cch(hcb, head);
166 cch = cch_alloc(hcb);
167 }
168 else{
169 cch = __alloc_chain(hcb);
170 }
171 *cch = *head;
172 head->page_flags=pte;
173 head->itir = rr.ps << 2;
174 head->etag=tag;
175 head->next = cch;
176 head->len = cch->len+1;
177 cch->len = 0;
178 return;
179 }
181 void thash_vhpt_insert(VCPU *v, u64 pte, u64 itir, u64 va)
182 {
183 u64 phy_pte;
184 phy_pte=translate_phy_pte(v, &pte, itir, va);
185 vmx_vhpt_insert(vcpu_get_vhpt(v), phy_pte, itir, va);
186 }
187 /*
188 * vhpt lookup
189 */
191 thash_data_t * vhpt_lookup(u64 va)
192 {
193 thash_data_t *hash, *head;
194 u64 tag, pte;
195 head = (thash_data_t *)ia64_thash(va);
196 hash=head;
197 tag = ia64_ttag(va);
198 do{
199 if(hash->etag == tag)
200 break;
201 hash=hash->next;
202 }while(hash);
203 if(hash && hash!=head){
204 pte = hash->page_flags;
205 hash->page_flags = head->page_flags;
206 head->page_flags = pte;
207 tag = hash->etag;
208 hash->etag = head->etag;
209 head->etag = tag;
210 head->len = hash->len;
211 hash->len=0;
212 return head;
213 }
214 return hash;
215 }
217 u64 guest_vhpt_lookup(u64 iha, u64 *pte)
218 {
219 u64 ret;
220 thash_data_t * data;
222 data = vhpt_lookup(iha);
223 if (data == NULL) {
224 data = vtlb_lookup(current, iha, DSIDE_TLB);
225 if (data != NULL)
226 thash_vhpt_insert(current, data->page_flags, data->itir ,iha);
227 }
229 asm volatile ("rsm psr.ic|psr.i;;"
230 "srlz.d;;"
231 "ld8.s r9=[%1];;"
232 "tnat.nz p6,p7=r9;;"
233 "(p6) mov %0=1;"
234 "(p6) mov r9=r0;"
235 "(p7) mov %0=r0;"
236 "(p7) st8 [%2]=r9;;"
237 "ssm psr.ic;;"
238 "srlz.d;;"
239 "ssm psr.i;;"
240 : "=r"(ret) : "r"(iha), "r"(pte):"memory");
241 return ret;
242 }
244 /*
245 * purge software guest tlb
246 */
248 void vtlb_purge(VCPU *v, u64 va, u64 ps)
249 {
250 thash_data_t *cur;
251 u64 start, curadr, size, psbits, tag, rr_ps, num;
252 ia64_rr vrr;
253 thash_cb_t *hcb = &v->arch.vtlb;
255 vcpu_get_rr(v, va, &vrr.rrval);
256 psbits = VMX(v, psbits[(va >> 61)]);
257 start = va & ~((1UL << ps) - 1);
258 while (psbits) {
259 curadr = start;
260 rr_ps = __ffs(psbits);
261 psbits &= ~(1UL << rr_ps);
262 num = 1UL << ((ps < rr_ps) ? 0 : (ps - rr_ps));
263 size = PSIZE(rr_ps);
264 vrr.ps = rr_ps;
265 while (num) {
266 cur = vsa_thash(hcb->pta, curadr, vrr.rrval, &tag);
267 while (cur) {
268 if (cur->etag == tag && cur->ps == rr_ps)
269 cur->etag = 1UL << 63;
270 cur = cur->next;
271 }
272 curadr += size;
273 num--;
274 }
275 }
276 }
279 /*
280 * purge VHPT and machine TLB
281 */
282 static void vhpt_purge(VCPU *v, u64 va, u64 ps)
283 {
284 //thash_cb_t *hcb = &v->arch.vhpt;
285 thash_data_t *cur;
286 u64 start, size, tag, num;
287 ia64_rr rr;
289 start = va & ~((1UL << ps) - 1);
290 rr.rrval = ia64_get_rr(va);
291 size = PSIZE(rr.ps);
292 num = 1UL << ((ps < rr.ps) ? 0 : (ps - rr.ps));
293 while (num) {
294 cur = (thash_data_t *)ia64_thash(start);
295 tag = ia64_ttag(start);
296 while (cur) {
297 if (cur->etag == tag)
298 cur->etag = 1UL << 63;
299 cur = cur->next;
300 }
301 start += size;
302 num--;
303 }
304 machine_tlb_purge(va, ps);
305 }
307 /*
308 * Recycle all collisions chain in VTLB or VHPT.
309 *
310 */
311 void thash_recycle_cch_all(thash_cb_t *hcb)
312 {
313 int num;
314 thash_data_t *head;
315 head=hcb->hash;
316 num = (hcb->hash_sz/sizeof(thash_data_t));
317 do{
318 head->len = 0;
319 head->next = 0;
320 head++;
321 num--;
322 }while(num);
323 cch_mem_init(hcb);
324 }
327 thash_data_t *__alloc_chain(thash_cb_t *hcb)
328 {
329 thash_data_t *cch;
331 cch = cch_alloc(hcb);
332 if(cch == NULL){
333 thash_recycle_cch_all(hcb);
334 cch = cch_alloc(hcb);
335 }
336 return cch;
337 }
339 /*
340 * Insert an entry into hash TLB or VHPT.
341 * NOTES:
342 * 1: When inserting VHPT to thash, "va" is a must covered
343 * address by the inserted machine VHPT entry.
344 * 2: The format of entry is always in TLB.
345 * 3: The caller need to make sure the new entry will not overlap
346 * with any existed entry.
347 */
348 void vtlb_insert(VCPU *v, u64 pte, u64 itir, u64 va)
349 {
350 thash_data_t *hash_table, *cch;
351 /* int flag; */
352 ia64_rr vrr;
353 /* u64 gppn, ppns, ppne; */
354 u64 tag, len;
355 thash_cb_t *hcb = &v->arch.vtlb;
356 vcpu_get_rr(v, va, &vrr.rrval);
357 vrr.ps = itir_ps(itir);
358 VMX(v, psbits[va >> 61]) |= (1UL << vrr.ps);
359 hash_table = vsa_thash(hcb->pta, va, vrr.rrval, &tag);
360 cch = hash_table;
361 while (cch) {
362 if (INVALID_TLB(cch)) {
363 len = cch->len;
364 cch->page_flags = pte;
365 cch->len = len;
366 cch->itir=itir;
367 cch->etag=tag;
368 return;
369 }
370 cch = cch->next;
371 }
372 if (hash_table->len>=MAX_CCN_DEPTH){
373 thash_recycle_cch(hcb, hash_table);
374 cch = cch_alloc(hcb);
375 }
376 else {
377 cch = __alloc_chain(hcb);
378 }
379 *cch = *hash_table;
380 hash_table->page_flags = pte;
381 hash_table->itir=itir;
382 hash_table->etag=tag;
383 hash_table->next = cch;
384 hash_table->len = cch->len + 1;
385 cch->len = 0;
386 return ;
387 }
390 int vtr_find_overlap(VCPU *vcpu, u64 va, u64 ps, int is_data)
391 {
392 thash_data_t *trp;
393 int i;
394 u64 end, rid;
395 vcpu_get_rr(vcpu, va, &rid);
396 rid = rid&RR_RID_MASK;;
397 end = va + PSIZE(ps);
398 if (is_data) {
399 if (vcpu_quick_region_check(vcpu->arch.dtr_regions,va)) {
400 for (trp =(thash_data_t *) vcpu->arch.dtrs,i=0; i<NDTRS; i++, trp++) {
401 if (__is_tr_overlap(trp, rid, va, end )) {
402 return i;
403 }
404 }
405 }
406 }
407 else {
408 if (vcpu_quick_region_check(vcpu->arch.itr_regions,va)) {
409 for (trp =(thash_data_t *) vcpu->arch.itrs,i=0; i<NITRS; i++, trp++) {
410 if (__is_tr_overlap(trp, rid, va, end )) {
411 return i;
412 }
413 }
414 }
415 }
416 return -1;
417 }
419 /*
420 * Purge entries in VTLB and VHPT
421 */
422 void thash_purge_entries(VCPU *v, u64 va, u64 ps)
423 {
424 if(vcpu_quick_region_check(v->arch.tc_regions,va))
425 vtlb_purge(v, va, ps);
426 vhpt_purge(v, va, ps);
427 }
429 u64 translate_phy_pte(VCPU *v, u64 *pte, u64 itir, u64 va)
430 {
431 u64 ps, ps_mask, paddr, maddr;
432 // ia64_rr rr;
433 union pte_flags phy_pte;
434 ps = itir_ps(itir);
435 ps_mask = ~((1UL << ps) - 1);
436 phy_pte.val = *pte;
437 paddr = *pte;
438 paddr = ((paddr & _PAGE_PPN_MASK) & ps_mask) | (va & ~ps_mask);
439 maddr = lookup_domain_mpa(v->domain, paddr, NULL);
440 if (maddr & GPFN_IO_MASK) {
441 *pte |= VTLB_PTE_IO;
442 return -1;
443 }
444 // rr.rrval = ia64_get_rr(va);
445 // ps = rr.ps;
446 maddr = ((maddr & _PAGE_PPN_MASK) & PAGE_MASK) | (paddr & ~PAGE_MASK);
447 phy_pte.ppn = maddr >> ARCH_PAGE_SHIFT;
448 return phy_pte.val;
449 }
452 /*
453 * Purge overlap TCs and then insert the new entry to emulate itc ops.
454 * Notes: Only TC entry can purge and insert.
455 */
456 void thash_purge_and_insert(VCPU *v, u64 pte, u64 itir, u64 ifa, int type)
457 {
458 u64 ps;//, va;
459 u64 phy_pte;
460 ia64_rr vrr, mrr;
461 ps = itir_ps(itir);
462 vcpu_get_rr(current, ifa, &vrr.rrval);
463 mrr.rrval = ia64_get_rr(ifa);
464 if(VMX_DOMAIN(v)){
465 /* Ensure WB attribute if pte is related to a normal mem page,
466 * which is required by vga acceleration since qemu maps shared
467 * vram buffer with WB.
468 */
469 if (!(pte & VTLB_PTE_IO) && ((pte & _PAGE_MA_MASK) != _PAGE_MA_NAT))
470 pte &= ~_PAGE_MA_MASK;
472 phy_pte = translate_phy_pte(v, &pte, itir, ifa);
473 vtlb_purge(v, ifa, ps);
474 vhpt_purge(v, ifa, ps);
475 if (ps == mrr.ps) {
476 if(!(pte&VTLB_PTE_IO)){
477 vmx_vhpt_insert(&v->arch.vhpt, phy_pte, itir, ifa);
478 }
479 else{
480 vtlb_insert(v, pte, itir, ifa);
481 vcpu_quick_region_set(PSCBX(v,tc_regions),ifa);
482 }
483 }
484 else if (ps > mrr.ps) {
485 vtlb_insert(v, pte, itir, ifa);
486 vcpu_quick_region_set(PSCBX(v,tc_regions),ifa);
487 if(!(pte&VTLB_PTE_IO)){
488 vmx_vhpt_insert(&v->arch.vhpt, phy_pte, itir, ifa);
489 }
490 }
491 else {
492 u64 psr;
493 phy_pte &= ~PAGE_FLAGS_RV_MASK;
494 psr = ia64_clear_ic();
495 ia64_itc(type + 1, ifa, phy_pte, ps);
496 ia64_set_psr(psr);
497 ia64_srlz_i();
498 // ps < mrr.ps, this is not supported
499 // panic_domain(NULL, "%s: ps (%lx) < mrr.ps \n", __func__, ps);
500 }
501 }
502 else{
503 phy_pte = translate_phy_pte(v, &pte, itir, ifa);
504 if(ps!=PAGE_SHIFT){
505 vtlb_insert(v, pte, itir, ifa);
506 vcpu_quick_region_set(PSCBX(v,tc_regions),ifa);
507 }
508 machine_tlb_purge(ifa, ps);
509 vmx_vhpt_insert(&v->arch.vhpt, phy_pte, itir, ifa);
510 }
511 }
513 /*
514 * Purge all TCs or VHPT entries including those in Hash table.
515 *
516 */
518 //TODO: add sections.
519 void thash_purge_all(VCPU *v)
520 {
521 int num;
522 thash_data_t *head;
523 thash_cb_t *vtlb,*vhpt;
524 vtlb =&v->arch.vtlb;
525 vhpt =&v->arch.vhpt;
527 for (num = 0; num < 8; num++)
528 VMX(v, psbits[num]) = 0;
530 head=vtlb->hash;
531 num = (vtlb->hash_sz/sizeof(thash_data_t));
532 do{
533 head->page_flags = 0;
534 head->etag = 1UL<<63;
535 head->itir = 0;
536 head->next = 0;
537 head++;
538 num--;
539 }while(num);
540 cch_mem_init(vtlb);
542 head=vhpt->hash;
543 num = (vhpt->hash_sz/sizeof(thash_data_t));
544 do{
545 head->page_flags = 0;
546 head->etag = 1UL<<63;
547 head->next = 0;
548 head++;
549 num--;
550 }while(num);
551 cch_mem_init(vhpt);
552 local_flush_tlb_all();
553 }
556 /*
557 * Lookup the hash table and its collision chain to find an entry
558 * covering this address rid:va or the entry.
559 *
560 * INPUT:
561 * in: TLB format for both VHPT & TLB.
562 */
564 thash_data_t *vtlb_lookup(VCPU *v, u64 va,int is_data)
565 {
566 thash_data_t *cch;
567 u64 psbits, ps, tag;
568 ia64_rr vrr;
569 thash_cb_t * hcb= &v->arch.vtlb;
571 cch = __vtr_lookup(v, va, is_data);;
572 if ( cch ) return cch;
574 if(vcpu_quick_region_check(v->arch.tc_regions,va)==0)
575 return NULL;
576 psbits = VMX(v, psbits[(va >> 61)]);
577 vcpu_get_rr(v,va,&vrr.rrval);
578 while (psbits) {
579 ps = __ffs(psbits);
580 psbits &= ~(1UL << ps);
581 vrr.ps = ps;
582 cch = vsa_thash(hcb->pta, va, vrr.rrval, &tag);
583 do {
584 if (cch->etag == tag && cch->ps == ps)
585 return cch;
586 cch = cch->next;
587 } while(cch);
588 }
589 return NULL;
590 }
593 /*
594 * Initialize internal control data before service.
595 */
596 void thash_init(thash_cb_t *hcb, u64 sz)
597 {
598 int num;
599 thash_data_t *head, *p;
601 hcb->pta.val = (unsigned long)hcb->hash;
602 hcb->pta.vf = 1;
603 hcb->pta.ve = 1;
604 hcb->pta.size = sz;
605 hcb->cch_rec_head = hcb->hash;
607 head=hcb->hash;
608 num = (hcb->hash_sz/sizeof(thash_data_t));
609 do{
610 head->itir = PAGE_SHIFT<<2;
611 head->etag = 1UL<<63;
612 head->next = 0;
613 head++;
614 num--;
615 }while(num);
617 hcb->cch_freelist = p = hcb->cch_buf;
618 num = (hcb->cch_sz/sizeof(thash_data_t))-1;
619 do{
620 p->itir = PAGE_SHIFT<<2;
621 p->next =p+1;
622 p++;
623 num--;
624 }while(num);
625 p->itir = PAGE_SHIFT<<2;
626 p->next = NULL;
627 }