direct-io.hg

view xen/arch/ia64/xen/vcpu.c @ 13487:2056bc71fa55

[IA64] Let Xen handle identity mapping for xenolinux region 7

During kernel build in dom0, half of the resulting slow reflections
are alt DTLB misses. Since we know Linux identity maps region 7,
Xen can handle the miss itself. This reduces the system time component
of the kernel build by nearly 15%.

Signed-off-by: Anthony Xu <anthony.xu@intel.com>
author awilliam@xenbuild2.aw
date Wed Jan 17 19:48:02 2007 -0700 (2007-01-17)
parents 1d72428a0fab
children d12ea0bfecce
line source
1 /*
2 * Virtualized CPU functions
3 *
4 * Copyright (C) 2004-2005 Hewlett-Packard Co.
5 * Dan Magenheimer (dan.magenheimer@hp.com)
6 *
7 */
9 #include <linux/sched.h>
10 #include <public/xen.h>
11 #include <xen/mm.h>
12 #include <asm/ia64_int.h>
13 #include <asm/vcpu.h>
14 #include <asm/regionreg.h>
15 #include <asm/tlb.h>
16 #include <asm/processor.h>
17 #include <asm/delay.h>
18 #include <asm/vmx_vcpu.h>
19 #include <asm/vhpt.h>
20 #include <asm/tlbflush.h>
21 #include <asm/privop.h>
22 #include <xen/event.h>
23 #include <asm/vmx_phy_mode.h>
24 #include <asm/bundle.h>
25 #include <asm/privop_stat.h>
26 #include <asm/uaccess.h>
27 #include <asm/p2m_entry.h>
28 #include <asm/tlb_track.h>
30 /* FIXME: where these declarations should be there ? */
31 extern void getreg(unsigned long regnum, unsigned long *val, int *nat,
32 struct pt_regs *regs);
33 extern void setreg(unsigned long regnum, unsigned long val, int nat,
34 struct pt_regs *regs);
35 extern void getfpreg(unsigned long regnum, struct ia64_fpreg *fpval,
36 struct pt_regs *regs);
38 extern void setfpreg(unsigned long regnum, struct ia64_fpreg *fpval,
39 struct pt_regs *regs);
41 typedef union {
42 struct ia64_psr ia64_psr;
43 unsigned long i64;
44 } PSR;
46 // this def for vcpu_regs won't work if kernel stack is present
47 //#define vcpu_regs(vcpu) ((struct pt_regs *) vcpu->arch.regs
49 #define IA64_PTA_SZ_BIT 2
50 #define IA64_PTA_VF_BIT 8
51 #define IA64_PTA_BASE_BIT 15
52 #define IA64_PTA_LFMT (1UL << IA64_PTA_VF_BIT)
53 #define IA64_PTA_SZ(x) (x##UL << IA64_PTA_SZ_BIT)
55 unsigned long vcpu_verbose = 0;
57 /**************************************************************************
58 VCPU general register access routines
59 **************************************************************************/
60 #ifdef XEN
61 u64 vcpu_get_gr(VCPU * vcpu, unsigned long reg)
62 {
63 REGS *regs = vcpu_regs(vcpu);
64 u64 val;
66 if (!reg)
67 return 0;
68 getreg(reg, &val, 0, regs); // FIXME: handle NATs later
69 return val;
70 }
72 IA64FAULT vcpu_get_gr_nat(VCPU * vcpu, unsigned long reg, u64 * val)
73 {
74 REGS *regs = vcpu_regs(vcpu);
75 int nat;
77 getreg(reg, val, &nat, regs); // FIXME: handle NATs later
78 if (nat)
79 return IA64_NAT_CONSUMPTION_VECTOR;
80 return 0;
81 }
83 // returns:
84 // IA64_ILLOP_FAULT if the register would cause an Illegal Operation fault
85 // IA64_NO_FAULT otherwise
86 IA64FAULT vcpu_set_gr(VCPU * vcpu, unsigned long reg, u64 value, int nat)
87 {
88 REGS *regs = vcpu_regs(vcpu);
89 long sof = (regs->cr_ifs) & 0x7f;
91 if (!reg)
92 return IA64_ILLOP_FAULT;
93 if (reg >= sof + 32)
94 return IA64_ILLOP_FAULT;
95 setreg(reg, value, nat, regs); // FIXME: handle NATs later
96 return IA64_NO_FAULT;
97 }
99 IA64FAULT
100 vcpu_get_fpreg(VCPU * vcpu, unsigned long reg, struct ia64_fpreg * val)
101 {
102 REGS *regs = vcpu_regs(vcpu);
103 getfpreg(reg, val, regs); // FIXME: handle NATs later
104 return IA64_NO_FAULT;
105 }
107 IA64FAULT
108 vcpu_set_fpreg(VCPU * vcpu, unsigned long reg, struct ia64_fpreg * val)
109 {
110 REGS *regs = vcpu_regs(vcpu);
111 if (reg > 1)
112 setfpreg(reg, val, regs); // FIXME: handle NATs later
113 return IA64_NO_FAULT;
114 }
116 #else
117 // returns:
118 // IA64_ILLOP_FAULT if the register would cause an Illegal Operation fault
119 // IA64_NO_FAULT otherwise
120 IA64FAULT vcpu_set_gr(VCPU * vcpu, unsigned long reg, u64 value)
121 {
122 REGS *regs = vcpu_regs(vcpu);
123 long sof = (regs->cr_ifs) & 0x7f;
125 if (!reg)
126 return IA64_ILLOP_FAULT;
127 if (reg >= sof + 32)
128 return IA64_ILLOP_FAULT;
129 setreg(reg, value, 0, regs); // FIXME: handle NATs later
130 return IA64_NO_FAULT;
131 }
133 #endif
135 void vcpu_init_regs(struct vcpu *v)
136 {
137 struct pt_regs *regs;
139 regs = vcpu_regs(v);
140 if (VMX_DOMAIN(v)) {
141 /* dt/rt/it:1;i/ic:1, si:1, vm/bn:1, ac:1 */
142 /* Need to be expanded as macro */
143 regs->cr_ipsr = 0x501008826008;
144 } else {
145 regs->cr_ipsr = ia64_getreg(_IA64_REG_PSR)
146 | IA64_PSR_BITS_TO_SET | IA64_PSR_BN;
147 regs->cr_ipsr &= ~(IA64_PSR_BITS_TO_CLEAR
148 | IA64_PSR_RI | IA64_PSR_IS);
149 // domain runs at PL2
150 regs->cr_ipsr |= 2UL << IA64_PSR_CPL0_BIT;
151 }
152 regs->cr_ifs = 1UL << 63; /* or clear? */
153 regs->ar_fpsr = FPSR_DEFAULT;
155 if (VMX_DOMAIN(v)) {
156 vmx_init_all_rr(v);
157 /* Virtual processor context setup */
158 VCPU(v, vpsr) = IA64_PSR_BN;
159 VCPU(v, dcr) = 0;
160 } else {
161 init_all_rr(v);
162 regs->ar_rsc |= (2 << 2); /* force PL2/3 */
163 VCPU(v, banknum) = 1;
164 VCPU(v, metaphysical_mode) = 1;
165 VCPU(v, interrupt_mask_addr) =
166 (unsigned char *)v->domain->arch.shared_info_va +
167 INT_ENABLE_OFFSET(v);
168 VCPU(v, itv) = (1 << 16); /* timer vector masked */
169 }
171 v->arch.domain_itm_last = -1L;
172 }
174 /**************************************************************************
175 VCPU privileged application register access routines
176 **************************************************************************/
178 void vcpu_load_kernel_regs(VCPU * vcpu)
179 {
180 ia64_set_kr(0, VCPU(vcpu, krs[0]));
181 ia64_set_kr(1, VCPU(vcpu, krs[1]));
182 ia64_set_kr(2, VCPU(vcpu, krs[2]));
183 ia64_set_kr(3, VCPU(vcpu, krs[3]));
184 ia64_set_kr(4, VCPU(vcpu, krs[4]));
185 ia64_set_kr(5, VCPU(vcpu, krs[5]));
186 ia64_set_kr(6, VCPU(vcpu, krs[6]));
187 ia64_set_kr(7, VCPU(vcpu, krs[7]));
188 }
190 /* GCC 4.0.2 seems not to be able to suppress this call!. */
191 #define ia64_setreg_unknown_kr() return IA64_ILLOP_FAULT
193 IA64FAULT vcpu_set_ar(VCPU * vcpu, u64 reg, u64 val)
194 {
195 if (reg == 44)
196 return vcpu_set_itc(vcpu, val);
197 else if (reg == 27)
198 return IA64_ILLOP_FAULT;
199 else if (reg == 24)
200 printk("warning: setting ar.eflg is a no-op; no IA-32 "
201 "support\n");
202 else if (reg > 7)
203 return IA64_ILLOP_FAULT;
204 else {
205 PSCB(vcpu, krs[reg]) = val;
206 ia64_set_kr(reg, val);
207 }
208 return IA64_NO_FAULT;
209 }
211 IA64FAULT vcpu_get_ar(VCPU * vcpu, u64 reg, u64 * val)
212 {
213 if (reg == 24)
214 printk("warning: getting ar.eflg is a no-op; no IA-32 "
215 "support\n");
216 else if (reg > 7)
217 return IA64_ILLOP_FAULT;
218 else
219 *val = PSCB(vcpu, krs[reg]);
220 return IA64_NO_FAULT;
221 }
223 /**************************************************************************
224 VCPU processor status register access routines
225 **************************************************************************/
227 void vcpu_set_metaphysical_mode(VCPU * vcpu, BOOLEAN newmode)
228 {
229 /* only do something if mode changes */
230 if (!!newmode ^ !!PSCB(vcpu, metaphysical_mode)) {
231 PSCB(vcpu, metaphysical_mode) = newmode;
232 if (newmode)
233 set_metaphysical_rr0();
234 else if (PSCB(vcpu, rrs[0]) != -1)
235 set_one_rr(0, PSCB(vcpu, rrs[0]));
236 }
237 }
239 IA64FAULT vcpu_reset_psr_dt(VCPU * vcpu)
240 {
241 vcpu_set_metaphysical_mode(vcpu, TRUE);
242 return IA64_NO_FAULT;
243 }
245 IA64FAULT vcpu_reset_psr_sm(VCPU * vcpu, u64 imm24)
246 {
247 struct ia64_psr psr, imm, *ipsr;
248 REGS *regs = vcpu_regs(vcpu);
250 //PRIVOP_COUNT_ADDR(regs,_RSM);
251 // TODO: All of these bits need to be virtualized
252 // TODO: Only allowed for current vcpu
253 __asm__ __volatile("mov %0=psr;;":"=r"(psr)::"memory");
254 ipsr = (struct ia64_psr *)&regs->cr_ipsr;
255 imm = *(struct ia64_psr *)&imm24;
256 // interrupt flag
257 if (imm.i)
258 vcpu->vcpu_info->evtchn_upcall_mask = 1;
259 if (imm.ic)
260 PSCB(vcpu, interrupt_collection_enabled) = 0;
261 // interrupt collection flag
262 //if (imm.ic) PSCB(vcpu,interrupt_delivery_enabled) = 0;
263 // just handle psr.up and psr.pp for now
264 if (imm24 & ~(IA64_PSR_BE | IA64_PSR_PP | IA64_PSR_UP | IA64_PSR_SP |
265 IA64_PSR_I | IA64_PSR_IC | IA64_PSR_DT |
266 IA64_PSR_DFL | IA64_PSR_DFH))
267 return IA64_ILLOP_FAULT;
268 if (imm.dfh)
269 ipsr->dfh = 0;
270 if (imm.dfl)
271 ipsr->dfl = 0;
272 if (imm.pp) {
273 ipsr->pp = 1;
274 psr.pp = 1; // priv perf ctrs always enabled
275 PSCB(vcpu, vpsr_pp) = 0; // but fool the domain if it gets psr
276 }
277 if (imm.up) {
278 ipsr->up = 0;
279 psr.up = 0;
280 }
281 if (imm.sp) {
282 ipsr->sp = 0;
283 psr.sp = 0;
284 }
285 if (imm.be)
286 ipsr->be = 0;
287 if (imm.dt)
288 vcpu_set_metaphysical_mode(vcpu, TRUE);
289 __asm__ __volatile(";; mov psr.l=%0;; srlz.d"::"r"(psr):"memory");
290 return IA64_NO_FAULT;
291 }
293 IA64FAULT vcpu_set_psr_dt(VCPU * vcpu)
294 {
295 vcpu_set_metaphysical_mode(vcpu, FALSE);
296 return IA64_NO_FAULT;
297 }
299 IA64FAULT vcpu_set_psr_i(VCPU * vcpu)
300 {
301 vcpu->vcpu_info->evtchn_upcall_mask = 0;
302 PSCB(vcpu, interrupt_collection_enabled) = 1;
303 return IA64_NO_FAULT;
304 }
306 IA64FAULT vcpu_set_psr_sm(VCPU * vcpu, u64 imm24)
307 {
308 struct ia64_psr psr, imm, *ipsr;
309 REGS *regs = vcpu_regs(vcpu);
310 u64 mask, enabling_interrupts = 0;
312 //PRIVOP_COUNT_ADDR(regs,_SSM);
313 // TODO: All of these bits need to be virtualized
314 __asm__ __volatile("mov %0=psr;;":"=r"(psr)::"memory");
315 imm = *(struct ia64_psr *)&imm24;
316 ipsr = (struct ia64_psr *)&regs->cr_ipsr;
317 // just handle psr.sp,pp and psr.i,ic (and user mask) for now
318 mask =
319 IA64_PSR_PP | IA64_PSR_SP | IA64_PSR_I | IA64_PSR_IC | IA64_PSR_UM |
320 IA64_PSR_DT | IA64_PSR_DFL | IA64_PSR_DFH;
321 if (imm24 & ~mask)
322 return IA64_ILLOP_FAULT;
323 if (imm.dfh)
324 ipsr->dfh = 1;
325 if (imm.dfl)
326 ipsr->dfl = 1;
327 if (imm.pp) {
328 ipsr->pp = 1;
329 psr.pp = 1;
330 PSCB(vcpu, vpsr_pp) = 1;
331 }
332 if (imm.sp) {
333 ipsr->sp = 1;
334 psr.sp = 1;
335 }
336 if (imm.i) {
337 if (vcpu->vcpu_info->evtchn_upcall_mask) {
338 //printk("vcpu_set_psr_sm: psr.ic 0->1\n");
339 enabling_interrupts = 1;
340 }
341 vcpu->vcpu_info->evtchn_upcall_mask = 0;
342 }
343 if (imm.ic)
344 PSCB(vcpu, interrupt_collection_enabled) = 1;
345 // TODO: do this faster
346 if (imm.mfl) {
347 ipsr->mfl = 1;
348 psr.mfl = 1;
349 }
350 if (imm.mfh) {
351 ipsr->mfh = 1;
352 psr.mfh = 1;
353 }
354 if (imm.ac) {
355 ipsr->ac = 1;
356 psr.ac = 1;
357 }
358 if (imm.up) {
359 ipsr->up = 1;
360 psr.up = 1;
361 }
362 if (imm.be) {
363 printk("*** DOMAIN TRYING TO TURN ON BIG-ENDIAN!!!\n");
364 return IA64_ILLOP_FAULT;
365 }
366 if (imm.dt)
367 vcpu_set_metaphysical_mode(vcpu, FALSE);
368 __asm__ __volatile(";; mov psr.l=%0;; srlz.d"::"r"(psr):"memory");
369 if (enabling_interrupts &&
370 vcpu_check_pending_interrupts(vcpu) != SPURIOUS_VECTOR)
371 PSCB(vcpu, pending_interruption) = 1;
372 return IA64_NO_FAULT;
373 }
375 IA64FAULT vcpu_set_psr_l(VCPU * vcpu, u64 val)
376 {
377 struct ia64_psr psr, newpsr, *ipsr;
378 REGS *regs = vcpu_regs(vcpu);
379 u64 enabling_interrupts = 0;
381 // TODO: All of these bits need to be virtualized
382 __asm__ __volatile("mov %0=psr;;":"=r"(psr)::"memory");
383 newpsr = *(struct ia64_psr *)&val;
384 ipsr = (struct ia64_psr *)&regs->cr_ipsr;
385 // just handle psr.up and psr.pp for now
386 //if (val & ~(IA64_PSR_PP | IA64_PSR_UP | IA64_PSR_SP))
387 // return IA64_ILLOP_FAULT;
388 // however trying to set other bits can't be an error as it is in ssm
389 if (newpsr.dfh)
390 ipsr->dfh = 1;
391 if (newpsr.dfl)
392 ipsr->dfl = 1;
393 if (newpsr.pp) {
394 ipsr->pp = 1;
395 psr.pp = 1;
396 PSCB(vcpu, vpsr_pp) = 1;
397 } else {
398 ipsr->pp = 1;
399 psr.pp = 1;
400 PSCB(vcpu, vpsr_pp) = 0;
401 }
402 if (newpsr.up) {
403 ipsr->up = 1;
404 psr.up = 1;
405 }
406 if (newpsr.sp) {
407 ipsr->sp = 1;
408 psr.sp = 1;
409 }
410 if (newpsr.i) {
411 if (vcpu->vcpu_info->evtchn_upcall_mask)
412 enabling_interrupts = 1;
413 vcpu->vcpu_info->evtchn_upcall_mask = 0;
414 }
415 if (newpsr.ic)
416 PSCB(vcpu, interrupt_collection_enabled) = 1;
417 if (newpsr.mfl) {
418 ipsr->mfl = 1;
419 psr.mfl = 1;
420 }
421 if (newpsr.mfh) {
422 ipsr->mfh = 1;
423 psr.mfh = 1;
424 }
425 if (newpsr.ac) {
426 ipsr->ac = 1;
427 psr.ac = 1;
428 }
429 if (newpsr.up) {
430 ipsr->up = 1;
431 psr.up = 1;
432 }
433 if (newpsr.dt && newpsr.rt)
434 vcpu_set_metaphysical_mode(vcpu, FALSE);
435 else
436 vcpu_set_metaphysical_mode(vcpu, TRUE);
437 if (newpsr.be) {
438 printk("*** DOMAIN TRYING TO TURN ON BIG-ENDIAN!!!\n");
439 return IA64_ILLOP_FAULT;
440 }
441 if (enabling_interrupts &&
442 vcpu_check_pending_interrupts(vcpu) != SPURIOUS_VECTOR)
443 PSCB(vcpu, pending_interruption) = 1;
444 return IA64_NO_FAULT;
445 }
447 IA64FAULT vcpu_get_psr(VCPU * vcpu, u64 * pval)
448 {
449 REGS *regs = vcpu_regs(vcpu);
450 struct ia64_psr newpsr;
452 newpsr = *(struct ia64_psr *)&regs->cr_ipsr;
453 if (!vcpu->vcpu_info->evtchn_upcall_mask)
454 newpsr.i = 1;
455 else
456 newpsr.i = 0;
457 if (PSCB(vcpu, interrupt_collection_enabled))
458 newpsr.ic = 1;
459 else
460 newpsr.ic = 0;
461 if (PSCB(vcpu, metaphysical_mode))
462 newpsr.dt = 0;
463 else
464 newpsr.dt = 1;
465 if (PSCB(vcpu, vpsr_pp))
466 newpsr.pp = 1;
467 else
468 newpsr.pp = 0;
469 *pval = *(unsigned long *)&newpsr;
470 *pval &= (MASK(0, 32) | MASK(35, 2));
471 return IA64_NO_FAULT;
472 }
474 BOOLEAN vcpu_get_psr_ic(VCPU * vcpu)
475 {
476 return !!PSCB(vcpu, interrupt_collection_enabled);
477 }
479 BOOLEAN vcpu_get_psr_i(VCPU * vcpu)
480 {
481 return !vcpu->vcpu_info->evtchn_upcall_mask;
482 }
484 u64 vcpu_get_ipsr_int_state(VCPU * vcpu, u64 prevpsr)
485 {
486 u64 dcr = PSCBX(vcpu, dcr);
487 PSR psr;
489 //printk("*** vcpu_get_ipsr_int_state (0x%016lx)...\n",prevpsr);
490 psr.i64 = prevpsr;
491 psr.ia64_psr.be = 0;
492 if (dcr & IA64_DCR_BE)
493 psr.ia64_psr.be = 1;
494 psr.ia64_psr.pp = 0;
495 if (dcr & IA64_DCR_PP)
496 psr.ia64_psr.pp = 1;
497 psr.ia64_psr.ic = PSCB(vcpu, interrupt_collection_enabled);
498 psr.ia64_psr.i = !vcpu->vcpu_info->evtchn_upcall_mask;
499 psr.ia64_psr.bn = PSCB(vcpu, banknum);
500 psr.ia64_psr.dt = 1;
501 psr.ia64_psr.it = 1;
502 psr.ia64_psr.rt = 1;
503 if (psr.ia64_psr.cpl == 2)
504 psr.ia64_psr.cpl = 0; // !!!! fool domain
505 // psr.pk = 1;
506 //printk("returns 0x%016lx...\n",psr.i64);
507 return psr.i64;
508 }
510 /**************************************************************************
511 VCPU control register access routines
512 **************************************************************************/
514 IA64FAULT vcpu_get_dcr(VCPU * vcpu, u64 * pval)
515 {
516 //verbose("vcpu_get_dcr: called @%p\n",PSCB(vcpu,iip));
517 // Reads of cr.dcr on Xen always have the sign bit set, so
518 // a domain can differentiate whether it is running on SP or not
519 *pval = PSCBX(vcpu, dcr) | 0x8000000000000000L;
520 return IA64_NO_FAULT;
521 }
523 IA64FAULT vcpu_get_iva(VCPU * vcpu, u64 * pval)
524 {
525 if (VMX_DOMAIN(vcpu))
526 *pval = PSCB(vcpu, iva) & ~0x7fffL;
527 else
528 *pval = PSCBX(vcpu, iva) & ~0x7fffL;
530 return IA64_NO_FAULT;
531 }
533 IA64FAULT vcpu_get_pta(VCPU * vcpu, u64 * pval)
534 {
535 *pval = PSCB(vcpu, pta);
536 return IA64_NO_FAULT;
537 }
539 IA64FAULT vcpu_get_ipsr(VCPU * vcpu, u64 * pval)
540 {
541 //REGS *regs = vcpu_regs(vcpu);
542 //*pval = regs->cr_ipsr;
543 *pval = PSCB(vcpu, ipsr);
544 return IA64_NO_FAULT;
545 }
547 IA64FAULT vcpu_get_isr(VCPU * vcpu, u64 * pval)
548 {
549 *pval = PSCB(vcpu, isr);
550 return IA64_NO_FAULT;
551 }
553 IA64FAULT vcpu_get_iip(VCPU * vcpu, u64 * pval)
554 {
555 //REGS *regs = vcpu_regs(vcpu);
556 //*pval = regs->cr_iip;
557 *pval = PSCB(vcpu, iip);
558 return IA64_NO_FAULT;
559 }
561 IA64FAULT vcpu_get_ifa(VCPU * vcpu, u64 * pval)
562 {
563 PRIVOP_COUNT_ADDR(vcpu_regs(vcpu), privop_inst_get_ifa);
564 *pval = PSCB(vcpu, ifa);
565 return IA64_NO_FAULT;
566 }
568 unsigned long vcpu_get_rr_ps(VCPU * vcpu, u64 vadr)
569 {
570 ia64_rr rr;
572 rr.rrval = PSCB(vcpu, rrs)[vadr >> 61];
573 return rr.ps;
574 }
576 unsigned long vcpu_get_rr_rid(VCPU * vcpu, u64 vadr)
577 {
578 ia64_rr rr;
580 rr.rrval = PSCB(vcpu, rrs)[vadr >> 61];
581 return rr.rid;
582 }
584 unsigned long vcpu_get_itir_on_fault(VCPU * vcpu, u64 ifa)
585 {
586 ia64_rr rr;
588 rr.rrval = 0;
589 rr.ps = vcpu_get_rr_ps(vcpu, ifa);
590 rr.rid = vcpu_get_rr_rid(vcpu, ifa);
591 return rr.rrval;
592 }
594 IA64FAULT vcpu_get_itir(VCPU * vcpu, u64 * pval)
595 {
596 u64 val = PSCB(vcpu, itir);
597 *pval = val;
598 return IA64_NO_FAULT;
599 }
601 IA64FAULT vcpu_get_iipa(VCPU * vcpu, u64 * pval)
602 {
603 u64 val = PSCB(vcpu, iipa);
604 // SP entry code does not save iipa yet nor does it get
605 // properly delivered in the pscb
606 // printk("*** vcpu_get_iipa: cr.iipa not fully implemented yet!!\n");
607 *pval = val;
608 return IA64_NO_FAULT;
609 }
611 IA64FAULT vcpu_get_ifs(VCPU * vcpu, u64 * pval)
612 {
613 //PSCB(vcpu,ifs) = PSCB(vcpu)->regs.cr_ifs;
614 //*pval = PSCB(vcpu,regs).cr_ifs;
615 *pval = PSCB(vcpu, ifs);
616 return IA64_NO_FAULT;
617 }
619 IA64FAULT vcpu_get_iim(VCPU * vcpu, u64 * pval)
620 {
621 u64 val = PSCB(vcpu, iim);
622 *pval = val;
623 return IA64_NO_FAULT;
624 }
626 IA64FAULT vcpu_get_iha(VCPU * vcpu, u64 * pval)
627 {
628 PRIVOP_COUNT_ADDR(vcpu_regs(vcpu), privop_inst_thash);
629 *pval = PSCB(vcpu, iha);
630 return IA64_NO_FAULT;
631 }
633 IA64FAULT vcpu_set_dcr(VCPU * vcpu, u64 val)
634 {
635 // Reads of cr.dcr on SP always have the sign bit set, so
636 // a domain can differentiate whether it is running on SP or not
637 // Thus, writes of DCR should ignore the sign bit
638 //verbose("vcpu_set_dcr: called\n");
639 PSCBX(vcpu, dcr) = val & ~0x8000000000000000L;
640 return IA64_NO_FAULT;
641 }
643 IA64FAULT vcpu_set_iva(VCPU * vcpu, u64 val)
644 {
645 if (VMX_DOMAIN(vcpu))
646 PSCB(vcpu, iva) = val & ~0x7fffL;
647 else
648 PSCBX(vcpu, iva) = val & ~0x7fffL;
650 return IA64_NO_FAULT;
651 }
653 IA64FAULT vcpu_set_pta(VCPU * vcpu, u64 val)
654 {
655 if (val & IA64_PTA_LFMT) {
656 printk("*** No support for VHPT long format yet!!\n");
657 return IA64_ILLOP_FAULT;
658 }
659 if (val & (0x3f << 9)) /* reserved fields */
660 return IA64_RSVDREG_FAULT;
661 if (val & 2) /* reserved fields */
662 return IA64_RSVDREG_FAULT;
663 PSCB(vcpu, pta) = val;
664 return IA64_NO_FAULT;
665 }
667 IA64FAULT vcpu_set_ipsr(VCPU * vcpu, u64 val)
668 {
669 PSCB(vcpu, ipsr) = val;
670 return IA64_NO_FAULT;
671 }
673 IA64FAULT vcpu_set_isr(VCPU * vcpu, u64 val)
674 {
675 PSCB(vcpu, isr) = val;
676 return IA64_NO_FAULT;
677 }
679 IA64FAULT vcpu_set_iip(VCPU * vcpu, u64 val)
680 {
681 PSCB(vcpu, iip) = val;
682 return IA64_NO_FAULT;
683 }
685 IA64FAULT vcpu_increment_iip(VCPU * vcpu)
686 {
687 REGS *regs = vcpu_regs(vcpu);
688 struct ia64_psr *ipsr = (struct ia64_psr *)&regs->cr_ipsr;
689 if (ipsr->ri == 2) {
690 ipsr->ri = 0;
691 regs->cr_iip += 16;
692 } else
693 ipsr->ri++;
694 return IA64_NO_FAULT;
695 }
697 IA64FAULT vcpu_decrement_iip(VCPU * vcpu)
698 {
699 REGS *regs = vcpu_regs(vcpu);
700 struct ia64_psr *ipsr = (struct ia64_psr *)&regs->cr_ipsr;
702 if (ipsr->ri == 0) {
703 ipsr->ri = 2;
704 regs->cr_iip -= 16;
705 } else
706 ipsr->ri--;
708 return IA64_NO_FAULT;
709 }
711 IA64FAULT vcpu_set_ifa(VCPU * vcpu, u64 val)
712 {
713 PSCB(vcpu, ifa) = val;
714 return IA64_NO_FAULT;
715 }
717 IA64FAULT vcpu_set_itir(VCPU * vcpu, u64 val)
718 {
719 PSCB(vcpu, itir) = val;
720 return IA64_NO_FAULT;
721 }
723 IA64FAULT vcpu_set_iipa(VCPU * vcpu, u64 val)
724 {
725 // SP entry code does not save iipa yet nor does it get
726 // properly delivered in the pscb
727 // printk("*** vcpu_set_iipa: cr.iipa not fully implemented yet!!\n");
728 PSCB(vcpu, iipa) = val;
729 return IA64_NO_FAULT;
730 }
732 IA64FAULT vcpu_set_ifs(VCPU * vcpu, u64 val)
733 {
734 //REGS *regs = vcpu_regs(vcpu);
735 PSCB(vcpu, ifs) = val;
736 return IA64_NO_FAULT;
737 }
739 IA64FAULT vcpu_set_iim(VCPU * vcpu, u64 val)
740 {
741 PSCB(vcpu, iim) = val;
742 return IA64_NO_FAULT;
743 }
745 IA64FAULT vcpu_set_iha(VCPU * vcpu, u64 val)
746 {
747 PSCB(vcpu, iha) = val;
748 return IA64_NO_FAULT;
749 }
751 /**************************************************************************
752 VCPU interrupt control register access routines
753 **************************************************************************/
755 void vcpu_pend_unspecified_interrupt(VCPU * vcpu)
756 {
757 PSCB(vcpu, pending_interruption) = 1;
758 }
760 void vcpu_pend_interrupt(VCPU * vcpu, u64 vector)
761 {
762 if (vector & ~0xff) {
763 printk("vcpu_pend_interrupt: bad vector\n");
764 return;
765 }
767 if (vcpu->arch.event_callback_ip) {
768 printk("Deprecated interface. Move to new event based "
769 "solution\n");
770 return;
771 }
773 if (VMX_DOMAIN(vcpu)) {
774 set_bit(vector, VCPU(vcpu, irr));
775 } else {
776 set_bit(vector, PSCBX(vcpu, irr));
777 PSCB(vcpu, pending_interruption) = 1;
778 }
779 }
781 #define IA64_TPR_MMI 0x10000
782 #define IA64_TPR_MIC 0x000f0
784 /* checks to see if a VCPU has any unmasked pending interrupts
785 * if so, returns the highest, else returns SPURIOUS_VECTOR */
786 /* NOTE: Since this gets called from vcpu_get_ivr() and the
787 * semantics of "mov rx=cr.ivr" ignore the setting of the psr.i bit,
788 * this routine also ignores pscb.interrupt_delivery_enabled
789 * and this must be checked independently; see vcpu_deliverable interrupts() */
790 u64 vcpu_check_pending_interrupts(VCPU * vcpu)
791 {
792 u64 *p, *r, bits, bitnum, mask, i, vector;
794 if (vcpu->arch.event_callback_ip)
795 return SPURIOUS_VECTOR;
797 /* Always check pending event, since guest may just ack the
798 * event injection without handle. Later guest may throw out
799 * the event itself.
800 */
801 check_start:
802 if (event_pending(vcpu) &&
803 !test_bit(vcpu->domain->shared_info->arch.evtchn_vector,
804 &PSCBX(vcpu, insvc[0])))
805 vcpu_pend_interrupt(vcpu,
806 vcpu->domain->shared_info->arch.
807 evtchn_vector);
809 p = &PSCBX(vcpu, irr[3]);
810 r = &PSCBX(vcpu, insvc[3]);
811 for (i = 3 ;; p--, r--, i--) {
812 bits = *p;
813 if (bits)
814 break; // got a potential interrupt
815 if (*r) {
816 // nothing in this word which is pending+inservice
817 // but there is one inservice which masks lower
818 return SPURIOUS_VECTOR;
819 }
820 if (i == 0) {
821 // checked all bits... nothing pending+inservice
822 return SPURIOUS_VECTOR;
823 }
824 }
825 // have a pending,deliverable interrupt... see if it is masked
826 bitnum = ia64_fls(bits);
827 //printk("XXXXXXX vcpu_check_pending_interrupts: got bitnum=%p...\n",bitnum);
828 vector = bitnum + (i * 64);
829 mask = 1L << bitnum;
830 /* sanity check for guest timer interrupt */
831 if (vector == (PSCB(vcpu, itv) & 0xff)) {
832 uint64_t now = ia64_get_itc();
833 if (now < PSCBX(vcpu, domain_itm)) {
834 // printk("Ooops, pending guest timer before its due\n");
835 PSCBX(vcpu, irr[i]) &= ~mask;
836 goto check_start;
837 }
838 }
839 //printk("XXXXXXX vcpu_check_pending_interrupts: got vector=%p...\n",vector);
840 if (*r >= mask) {
841 // masked by equal inservice
842 //printk("but masked by equal inservice\n");
843 return SPURIOUS_VECTOR;
844 }
845 if (PSCB(vcpu, tpr) & IA64_TPR_MMI) {
846 // tpr.mmi is set
847 //printk("but masked by tpr.mmi\n");
848 return SPURIOUS_VECTOR;
849 }
850 if (((PSCB(vcpu, tpr) & IA64_TPR_MIC) + 15) >= vector) {
851 //tpr.mic masks class
852 //printk("but masked by tpr.mic\n");
853 return SPURIOUS_VECTOR;
854 }
855 //printk("returned to caller\n");
856 return vector;
857 }
859 u64 vcpu_deliverable_interrupts(VCPU * vcpu)
860 {
861 return (vcpu_get_psr_i(vcpu) &&
862 vcpu_check_pending_interrupts(vcpu) != SPURIOUS_VECTOR);
863 }
865 u64 vcpu_deliverable_timer(VCPU * vcpu)
866 {
867 return (vcpu_get_psr_i(vcpu) &&
868 vcpu_check_pending_interrupts(vcpu) == PSCB(vcpu, itv));
869 }
871 IA64FAULT vcpu_get_lid(VCPU * vcpu, u64 * pval)
872 {
873 /* Use EID=0, ID=vcpu_id. */
874 *pval = vcpu->vcpu_id << 24;
875 return IA64_NO_FAULT;
876 }
878 IA64FAULT vcpu_get_ivr(VCPU * vcpu, u64 * pval)
879 {
880 int i;
881 u64 vector, mask;
883 #define HEARTBEAT_FREQ 16 // period in seconds
884 #ifdef HEARTBEAT_FREQ
885 #define N_DOMS 16 // period in seconds
886 #if 0
887 static long count[N_DOMS] = { 0 };
888 #endif
889 static long nonclockcount[N_DOMS] = { 0 };
890 unsigned domid = vcpu->domain->domain_id;
891 #endif
892 #ifdef IRQ_DEBUG
893 static char firstivr = 1;
894 static char firsttime[256];
895 if (firstivr) {
896 int i;
897 for (i = 0; i < 256; i++)
898 firsttime[i] = 1;
899 firstivr = 0;
900 }
901 #endif
903 vector = vcpu_check_pending_interrupts(vcpu);
904 if (vector == SPURIOUS_VECTOR) {
905 PSCB(vcpu, pending_interruption) = 0;
906 *pval = vector;
907 return IA64_NO_FAULT;
908 }
909 #ifdef HEARTBEAT_FREQ
910 if (domid >= N_DOMS)
911 domid = N_DOMS - 1;
912 #if 0
913 if (vector == (PSCB(vcpu, itv) & 0xff)) {
914 if (!(++count[domid] & ((HEARTBEAT_FREQ * 1024) - 1))) {
915 printk("Dom%d heartbeat... ticks=%lx,nonticks=%lx\n",
916 domid, count[domid], nonclockcount[domid]);
917 //count[domid] = 0;
918 //dump_runq();
919 }
920 }
921 #endif
922 else
923 nonclockcount[domid]++;
924 #endif
925 // now have an unmasked, pending, deliverable vector!
926 // getting ivr has "side effects"
927 #ifdef IRQ_DEBUG
928 if (firsttime[vector]) {
929 printk("*** First get_ivr on vector=%lu,itc=%lx\n",
930 vector, ia64_get_itc());
931 firsttime[vector] = 0;
932 }
933 #endif
934 /* if delivering a timer interrupt, remember domain_itm, which
935 * needs to be done before clearing irr
936 */
937 if (vector == (PSCB(vcpu, itv) & 0xff)) {
938 PSCBX(vcpu, domain_itm_last) = PSCBX(vcpu, domain_itm);
939 }
941 i = vector >> 6;
942 mask = 1L << (vector & 0x3f);
943 //printk("ZZZZZZ vcpu_get_ivr: setting insvc mask for vector %lu\n",vector);
944 PSCBX(vcpu, insvc[i]) |= mask;
945 PSCBX(vcpu, irr[i]) &= ~mask;
946 //PSCB(vcpu,pending_interruption)--;
947 *pval = vector;
948 return IA64_NO_FAULT;
949 }
951 IA64FAULT vcpu_get_tpr(VCPU * vcpu, u64 * pval)
952 {
953 *pval = PSCB(vcpu, tpr);
954 return IA64_NO_FAULT;
955 }
957 IA64FAULT vcpu_get_eoi(VCPU * vcpu, u64 * pval)
958 {
959 *pval = 0L; // reads of eoi always return 0
960 return IA64_NO_FAULT;
961 }
963 IA64FAULT vcpu_get_irr0(VCPU * vcpu, u64 * pval)
964 {
965 *pval = PSCBX(vcpu, irr[0]);
966 return IA64_NO_FAULT;
967 }
969 IA64FAULT vcpu_get_irr1(VCPU * vcpu, u64 * pval)
970 {
971 *pval = PSCBX(vcpu, irr[1]);
972 return IA64_NO_FAULT;
973 }
975 IA64FAULT vcpu_get_irr2(VCPU * vcpu, u64 * pval)
976 {
977 *pval = PSCBX(vcpu, irr[2]);
978 return IA64_NO_FAULT;
979 }
981 IA64FAULT vcpu_get_irr3(VCPU * vcpu, u64 * pval)
982 {
983 *pval = PSCBX(vcpu, irr[3]);
984 return IA64_NO_FAULT;
985 }
987 IA64FAULT vcpu_get_itv(VCPU * vcpu, u64 * pval)
988 {
989 *pval = PSCB(vcpu, itv);
990 return IA64_NO_FAULT;
991 }
993 IA64FAULT vcpu_get_pmv(VCPU * vcpu, u64 * pval)
994 {
995 *pval = PSCB(vcpu, pmv);
996 return IA64_NO_FAULT;
997 }
999 IA64FAULT vcpu_get_cmcv(VCPU * vcpu, u64 * pval)
1001 *pval = PSCB(vcpu, cmcv);
1002 return IA64_NO_FAULT;
1005 IA64FAULT vcpu_get_lrr0(VCPU * vcpu, u64 * pval)
1007 // fix this when setting values other than m-bit is supported
1008 printk("vcpu_get_lrr0: Unmasked interrupts unsupported\n");
1009 *pval = (1L << 16);
1010 return IA64_NO_FAULT;
1013 IA64FAULT vcpu_get_lrr1(VCPU * vcpu, u64 * pval)
1015 // fix this when setting values other than m-bit is supported
1016 printk("vcpu_get_lrr1: Unmasked interrupts unsupported\n");
1017 *pval = (1L << 16);
1018 return IA64_NO_FAULT;
1021 IA64FAULT vcpu_set_lid(VCPU * vcpu, u64 val)
1023 printk("vcpu_set_lid: Setting cr.lid is unsupported\n");
1024 return IA64_ILLOP_FAULT;
1027 IA64FAULT vcpu_set_tpr(VCPU * vcpu, u64 val)
1029 if (val & 0xff00)
1030 return IA64_RSVDREG_FAULT;
1031 PSCB(vcpu, tpr) = val;
1032 /* This can unmask interrupts. */
1033 if (vcpu_check_pending_interrupts(vcpu) != SPURIOUS_VECTOR)
1034 PSCB(vcpu, pending_interruption) = 1;
1035 return IA64_NO_FAULT;
1038 IA64FAULT vcpu_set_eoi(VCPU * vcpu, u64 val)
1040 u64 *p, bits, vec, bitnum;
1041 int i;
1043 p = &PSCBX(vcpu, insvc[3]);
1044 for (i = 3; (i >= 0) && !(bits = *p); i--, p--)
1046 if (i < 0) {
1047 printk("Trying to EOI interrupt when none are in-service.\n");
1048 return IA64_NO_FAULT;
1050 bitnum = ia64_fls(bits);
1051 vec = bitnum + (i * 64);
1052 /* clear the correct bit */
1053 bits &= ~(1L << bitnum);
1054 *p = bits;
1055 /* clearing an eoi bit may unmask another pending interrupt... */
1056 if (!vcpu->vcpu_info->evtchn_upcall_mask) { // but only if enabled...
1057 // worry about this later... Linux only calls eoi
1058 // with interrupts disabled
1059 printk("Trying to EOI interrupt with interrupts enabled\n");
1061 if (vcpu_check_pending_interrupts(vcpu) != SPURIOUS_VECTOR)
1062 PSCB(vcpu, pending_interruption) = 1;
1063 //printk("YYYYY vcpu_set_eoi: Successful\n");
1064 return IA64_NO_FAULT;
1067 IA64FAULT vcpu_set_lrr0(VCPU * vcpu, u64 val)
1069 if (!(val & (1L << 16))) {
1070 printk("vcpu_set_lrr0: Unmasked interrupts unsupported\n");
1071 return IA64_ILLOP_FAULT;
1073 // no place to save this state but nothing to do anyway
1074 return IA64_NO_FAULT;
1077 IA64FAULT vcpu_set_lrr1(VCPU * vcpu, u64 val)
1079 if (!(val & (1L << 16))) {
1080 printk("vcpu_set_lrr0: Unmasked interrupts unsupported\n");
1081 return IA64_ILLOP_FAULT;
1083 // no place to save this state but nothing to do anyway
1084 return IA64_NO_FAULT;
1087 IA64FAULT vcpu_set_itv(VCPU * vcpu, u64 val)
1089 /* Check reserved fields. */
1090 if (val & 0xef00)
1091 return IA64_ILLOP_FAULT;
1092 PSCB(vcpu, itv) = val;
1093 if (val & 0x10000) {
1094 /* Disable itm. */
1095 PSCBX(vcpu, domain_itm) = 0;
1096 } else
1097 vcpu_set_next_timer(vcpu);
1098 return IA64_NO_FAULT;
1101 IA64FAULT vcpu_set_pmv(VCPU * vcpu, u64 val)
1103 if (val & 0xef00) /* reserved fields */
1104 return IA64_RSVDREG_FAULT;
1105 PSCB(vcpu, pmv) = val;
1106 return IA64_NO_FAULT;
1109 IA64FAULT vcpu_set_cmcv(VCPU * vcpu, u64 val)
1111 if (val & 0xef00) /* reserved fields */
1112 return IA64_RSVDREG_FAULT;
1113 PSCB(vcpu, cmcv) = val;
1114 return IA64_NO_FAULT;
1117 /**************************************************************************
1118 VCPU temporary register access routines
1119 **************************************************************************/
1120 u64 vcpu_get_tmp(VCPU * vcpu, u64 index)
1122 if (index > 7)
1123 return 0;
1124 return PSCB(vcpu, tmp[index]);
1127 void vcpu_set_tmp(VCPU * vcpu, u64 index, u64 val)
1129 if (index <= 7)
1130 PSCB(vcpu, tmp[index]) = val;
1133 /**************************************************************************
1134 Interval timer routines
1135 **************************************************************************/
1137 BOOLEAN vcpu_timer_disabled(VCPU * vcpu)
1139 u64 itv = PSCB(vcpu, itv);
1140 return (!itv || !!(itv & 0x10000));
1143 BOOLEAN vcpu_timer_inservice(VCPU * vcpu)
1145 u64 itv = PSCB(vcpu, itv);
1146 return test_bit(itv, PSCBX(vcpu, insvc));
1149 BOOLEAN vcpu_timer_expired(VCPU * vcpu)
1151 unsigned long domain_itm = PSCBX(vcpu, domain_itm);
1152 unsigned long now = ia64_get_itc();
1154 if (!domain_itm)
1155 return FALSE;
1156 if (now < domain_itm)
1157 return FALSE;
1158 if (vcpu_timer_disabled(vcpu))
1159 return FALSE;
1160 return TRUE;
1163 void vcpu_safe_set_itm(unsigned long val)
1165 unsigned long epsilon = 100;
1166 unsigned long flags;
1167 u64 now = ia64_get_itc();
1169 local_irq_save(flags);
1170 while (1) {
1171 //printk("*** vcpu_safe_set_itm: Setting itm to %lx, itc=%lx\n",val,now);
1172 ia64_set_itm(val);
1173 if (val > (now = ia64_get_itc()))
1174 break;
1175 val = now + epsilon;
1176 epsilon <<= 1;
1178 local_irq_restore(flags);
1181 void vcpu_set_next_timer(VCPU * vcpu)
1183 u64 d = PSCBX(vcpu, domain_itm);
1184 //u64 s = PSCBX(vcpu,xen_itm);
1185 u64 s = local_cpu_data->itm_next;
1186 u64 now = ia64_get_itc();
1188 /* gloss over the wraparound problem for now... we know it exists
1189 * but it doesn't matter right now */
1191 if (is_idle_domain(vcpu->domain)) {
1192 // printk("****** vcpu_set_next_timer called during idle!!\n");
1193 vcpu_safe_set_itm(s);
1194 return;
1196 //s = PSCBX(vcpu,xen_itm);
1197 if (d && (d > now) && (d < s)) {
1198 vcpu_safe_set_itm(d);
1199 //using_domain_as_itm++;
1200 } else {
1201 vcpu_safe_set_itm(s);
1202 //using_xen_as_itm++;
1206 IA64FAULT vcpu_set_itm(VCPU * vcpu, u64 val)
1208 //UINT now = ia64_get_itc();
1210 //if (val < now) val = now + 1000;
1211 //printk("*** vcpu_set_itm: called with %lx\n",val);
1212 PSCBX(vcpu, domain_itm) = val;
1213 vcpu_set_next_timer(vcpu);
1214 return IA64_NO_FAULT;
1217 IA64FAULT vcpu_set_itc(VCPU * vcpu, u64 val)
1219 #define DISALLOW_SETTING_ITC_FOR_NOW
1220 #ifdef DISALLOW_SETTING_ITC_FOR_NOW
1221 static int did_print;
1222 if (!did_print) {
1223 printk("vcpu_set_itc: Setting ar.itc is currently disabled "
1224 "(this message is only displayed once)\n");
1225 did_print = 1;
1227 #else
1228 u64 oldnow = ia64_get_itc();
1229 u64 olditm = PSCBX(vcpu, domain_itm);
1230 unsigned long d = olditm - oldnow;
1231 unsigned long x = local_cpu_data->itm_next - oldnow;
1233 u64 newnow = val, min_delta;
1235 local_irq_disable();
1236 if (olditm) {
1237 printk("**** vcpu_set_itc(%lx): vitm changed to %lx\n", val,
1238 newnow + d);
1239 PSCBX(vcpu, domain_itm) = newnow + d;
1241 local_cpu_data->itm_next = newnow + x;
1242 d = PSCBX(vcpu, domain_itm);
1243 x = local_cpu_data->itm_next;
1245 ia64_set_itc(newnow);
1246 if (d && (d > newnow) && (d < x)) {
1247 vcpu_safe_set_itm(d);
1248 //using_domain_as_itm++;
1249 } else {
1250 vcpu_safe_set_itm(x);
1251 //using_xen_as_itm++;
1253 local_irq_enable();
1254 #endif
1255 return IA64_NO_FAULT;
1258 IA64FAULT vcpu_get_itm(VCPU * vcpu, u64 * pval)
1260 //FIXME: Implement this
1261 printk("vcpu_get_itm: Getting cr.itm is unsupported... continuing\n");
1262 return IA64_NO_FAULT;
1263 //return IA64_ILLOP_FAULT;
1266 IA64FAULT vcpu_get_itc(VCPU * vcpu, u64 * pval)
1268 //TODO: Implement this
1269 printk("vcpu_get_itc: Getting ar.itc is unsupported\n");
1270 return IA64_ILLOP_FAULT;
1273 void vcpu_pend_timer(VCPU * vcpu)
1275 u64 itv = PSCB(vcpu, itv) & 0xff;
1277 if (vcpu_timer_disabled(vcpu))
1278 return;
1279 //if (vcpu_timer_inservice(vcpu)) return;
1280 if (PSCBX(vcpu, domain_itm_last) == PSCBX(vcpu, domain_itm)) {
1281 // already delivered an interrupt for this so
1282 // don't deliver another
1283 return;
1285 if (vcpu->arch.event_callback_ip) {
1286 /* A small window may occur when injecting vIRQ while related
1287 * handler has not been registered. Don't fire in such case.
1288 */
1289 if (vcpu->virq_to_evtchn[VIRQ_ITC]) {
1290 send_guest_vcpu_virq(vcpu, VIRQ_ITC);
1291 PSCBX(vcpu, domain_itm_last) = PSCBX(vcpu, domain_itm);
1293 } else
1294 vcpu_pend_interrupt(vcpu, itv);
1297 // returns true if ready to deliver a timer interrupt too early
1298 u64 vcpu_timer_pending_early(VCPU * vcpu)
1300 u64 now = ia64_get_itc();
1301 u64 itm = PSCBX(vcpu, domain_itm);
1303 if (vcpu_timer_disabled(vcpu))
1304 return 0;
1305 if (!itm)
1306 return 0;
1307 return (vcpu_deliverable_timer(vcpu) && (now < itm));
1310 /**************************************************************************
1311 Privileged operation emulation routines
1312 **************************************************************************/
1314 static void vcpu_force_tlb_miss(VCPU * vcpu, u64 ifa)
1316 PSCB(vcpu, ifa) = ifa;
1317 PSCB(vcpu, itir) = vcpu_get_itir_on_fault(vcpu, ifa);
1318 vcpu_thash(current, ifa, &PSCB(current, iha));
1321 IA64FAULT vcpu_force_inst_miss(VCPU * vcpu, u64 ifa)
1323 vcpu_force_tlb_miss(vcpu, ifa);
1324 return vcpu_get_rr_ve(vcpu, ifa) ? IA64_INST_TLB_VECTOR :
1325 IA64_ALT_INST_TLB_VECTOR;
1328 IA64FAULT vcpu_force_data_miss(VCPU * vcpu, u64 ifa)
1330 vcpu_force_tlb_miss(vcpu, ifa);
1331 return vcpu_get_rr_ve(vcpu, ifa) ? IA64_DATA_TLB_VECTOR :
1332 IA64_ALT_DATA_TLB_VECTOR;
1335 IA64FAULT vcpu_rfi(VCPU * vcpu)
1337 // TODO: Only allowed for current vcpu
1338 PSR psr;
1339 u64 int_enable, ifs;
1340 REGS *regs = vcpu_regs(vcpu);
1342 psr.i64 = PSCB(vcpu, ipsr);
1343 if (psr.ia64_psr.cpl < 3)
1344 psr.ia64_psr.cpl = 2;
1345 int_enable = psr.ia64_psr.i;
1346 if (psr.ia64_psr.ic)
1347 PSCB(vcpu, interrupt_collection_enabled) = 1;
1348 if (psr.ia64_psr.dt && psr.ia64_psr.rt && psr.ia64_psr.it)
1349 vcpu_set_metaphysical_mode(vcpu, FALSE);
1350 else
1351 vcpu_set_metaphysical_mode(vcpu, TRUE);
1352 psr.ia64_psr.ic = 1;
1353 psr.ia64_psr.i = 1;
1354 psr.ia64_psr.dt = 1;
1355 psr.ia64_psr.rt = 1;
1356 psr.ia64_psr.it = 1;
1357 psr.ia64_psr.bn = 1;
1358 //psr.pk = 1; // checking pkeys shouldn't be a problem but seems broken
1359 if (psr.ia64_psr.be) {
1360 printk("*** DOMAIN TRYING TO TURN ON BIG-ENDIAN!!!\n");
1361 return IA64_ILLOP_FAULT;
1364 ifs = PSCB(vcpu, ifs);
1365 if (ifs & 0x8000000000000000UL)
1366 regs->cr_ifs = ifs;
1368 regs->cr_ipsr = psr.i64;
1369 regs->cr_iip = PSCB(vcpu, iip);
1370 PSCB(vcpu, interrupt_collection_enabled) = 1;
1371 vcpu_bsw1(vcpu);
1372 vcpu->vcpu_info->evtchn_upcall_mask = !int_enable;
1373 return IA64_NO_FAULT;
1376 IA64FAULT vcpu_cover(VCPU * vcpu)
1378 // TODO: Only allowed for current vcpu
1379 REGS *regs = vcpu_regs(vcpu);
1381 if (!PSCB(vcpu, interrupt_collection_enabled)) {
1382 PSCB(vcpu, ifs) = regs->cr_ifs;
1384 regs->cr_ifs = 0;
1385 return IA64_NO_FAULT;
1388 IA64FAULT vcpu_thash(VCPU * vcpu, u64 vadr, u64 * pval)
1390 u64 pta = PSCB(vcpu, pta);
1391 u64 pta_sz = (pta & IA64_PTA_SZ(0x3f)) >> IA64_PTA_SZ_BIT;
1392 u64 pta_base = pta & ~((1UL << IA64_PTA_BASE_BIT) - 1);
1393 u64 Mask = (1L << pta_sz) - 1;
1394 u64 Mask_60_15 = (Mask >> 15) & 0x3fffffffffff;
1395 u64 compMask_60_15 = ~Mask_60_15;
1396 u64 rr_ps = vcpu_get_rr_ps(vcpu, vadr);
1397 u64 VHPT_offset = (vadr >> rr_ps) << 3;
1398 u64 VHPT_addr1 = vadr & 0xe000000000000000L;
1399 u64 VHPT_addr2a =
1400 ((pta_base >> 15) & 0x3fffffffffff) & compMask_60_15;
1401 u64 VHPT_addr2b =
1402 ((VHPT_offset >> 15) & 0x3fffffffffff) & Mask_60_15;
1403 u64 VHPT_addr3 = VHPT_offset & 0x7fff;
1404 u64 VHPT_addr = VHPT_addr1 | ((VHPT_addr2a | VHPT_addr2b) << 15) |
1405 VHPT_addr3;
1407 //verbose("vcpu_thash: vadr=%p, VHPT_addr=%p\n",vadr,VHPT_addr);
1408 *pval = VHPT_addr;
1409 return IA64_NO_FAULT;
1412 IA64FAULT vcpu_ttag(VCPU * vcpu, u64 vadr, u64 * padr)
1414 printk("vcpu_ttag: ttag instruction unsupported\n");
1415 return IA64_ILLOP_FAULT;
1418 int warn_region0_address = 0; // FIXME later: tie to a boot parameter?
1420 /* Return TRUE iff [b1,e1] and [b2,e2] partially or fully overlaps. */
1421 static inline int range_overlap(u64 b1, u64 e1, u64 b2, u64 e2)
1423 return (b1 <= e2) && (e1 >= b2);
1426 /* Crash domain if [base, base + page_size] and Xen virtual space overlaps.
1427 Note: LSBs of base inside page_size are ignored. */
1428 static inline void
1429 check_xen_space_overlap(const char *func, u64 base, u64 page_size)
1431 /* Overlaps can occur only in region 7.
1432 (This is an optimization to bypass all the checks). */
1433 if (REGION_NUMBER(base) != 7)
1434 return;
1436 /* Mask LSBs of base. */
1437 base &= ~(page_size - 1);
1439 /* FIXME: ideally an MCA should be generated... */
1440 if (range_overlap(HYPERVISOR_VIRT_START, HYPERVISOR_VIRT_END,
1441 base, base + page_size)
1442 || range_overlap(current->domain->arch.shared_info_va,
1443 current->domain->arch.shared_info_va
1444 + XSI_SIZE + XMAPPEDREGS_SIZE,
1445 base, base + page_size))
1446 panic_domain(NULL, "%s on Xen virtual space (%lx)\n",
1447 func, base);
1450 // FIXME: also need to check && (!trp->key || vcpu_pkr_match(trp->key))
1451 static inline int vcpu_match_tr_entry_no_p(TR_ENTRY * trp, u64 ifa,
1452 u64 rid)
1454 return trp->rid == rid
1455 && ifa >= trp->vadr && ifa <= (trp->vadr + (1L << trp->ps) - 1);
1458 static inline int vcpu_match_tr_entry(TR_ENTRY * trp, u64 ifa, u64 rid)
1460 return trp->pte.p && vcpu_match_tr_entry_no_p(trp, ifa, rid);
1463 static inline int
1464 vcpu_match_tr_entry_range(TR_ENTRY * trp, u64 rid, u64 b, u64 e)
1466 return trp->rid == rid
1467 && trp->pte.p
1468 && range_overlap(b, e, trp->vadr, trp->vadr + (1L << trp->ps) - 1);
1472 static TR_ENTRY *vcpu_tr_lookup(VCPU * vcpu, unsigned long va, u64 rid,
1473 BOOLEAN is_data)
1475 unsigned char *regions;
1476 TR_ENTRY *trp;
1477 int tr_max;
1478 int i;
1480 if (is_data) {
1481 // data
1482 regions = &vcpu->arch.dtr_regions;
1483 trp = vcpu->arch.dtrs;
1484 tr_max = sizeof(vcpu->arch.dtrs) / sizeof(vcpu->arch.dtrs[0]);
1485 } else {
1486 // instruction
1487 regions = &vcpu->arch.itr_regions;
1488 trp = vcpu->arch.itrs;
1489 tr_max = sizeof(vcpu->arch.itrs) / sizeof(vcpu->arch.itrs[0]);
1492 if (!vcpu_quick_region_check(*regions, va)) {
1493 return NULL;
1495 for (i = 0; i < tr_max; i++, trp++) {
1496 if (vcpu_match_tr_entry(trp, va, rid)) {
1497 return trp;
1500 return NULL;
1503 // return value
1504 // 0: failure
1505 // 1: success
1506 int
1507 vcpu_get_domain_bundle(VCPU * vcpu, REGS * regs, u64 gip,
1508 IA64_BUNDLE * bundle)
1510 u64 gpip; // guest pseudo phyiscal ip
1511 unsigned long vaddr;
1512 struct page_info *page;
1514 again:
1515 #if 0
1516 // Currently xen doesn't track psr.it bits.
1517 // it assumes always psr.it = 1.
1518 if (!(VCPU(vcpu, vpsr) & IA64_PSR_IT)) {
1519 gpip = gip;
1520 } else
1521 #endif
1523 unsigned long region = REGION_NUMBER(gip);
1524 unsigned long rr = PSCB(vcpu, rrs)[region];
1525 unsigned long rid = rr & RR_RID_MASK;
1526 BOOLEAN swap_rr0;
1527 TR_ENTRY *trp;
1529 // vcpu->arch.{i, d}tlb are volatile,
1530 // copy its value to the variable, tr, before use.
1531 TR_ENTRY tr;
1533 trp = vcpu_tr_lookup(vcpu, gip, rid, 0);
1534 if (trp != NULL) {
1535 tr = *trp;
1536 goto found;
1538 // When it failed to get a bundle, itlb miss is reflected.
1539 // Last itc.i value is cached to PSCBX(vcpu, itlb).
1540 tr = PSCBX(vcpu, itlb);
1541 if (vcpu_match_tr_entry(&tr, gip, rid)) {
1542 //dprintk(XENLOG_WARNING,
1543 // "%s gip 0x%lx gpip 0x%lx\n", __func__,
1544 // gip, gpip);
1545 goto found;
1547 trp = vcpu_tr_lookup(vcpu, gip, rid, 1);
1548 if (trp != NULL) {
1549 tr = *trp;
1550 goto found;
1552 #if 0
1553 tr = PSCBX(vcpu, dtlb);
1554 if (vcpu_match_tr_entry(&tr, gip, rid)) {
1555 goto found;
1557 #endif
1559 // try to access gip with guest virtual address
1560 // This may cause tlb miss. see vcpu_translate(). Be careful!
1561 swap_rr0 = (!region && PSCB(vcpu, metaphysical_mode));
1562 if (swap_rr0) {
1563 set_one_rr(0x0, PSCB(vcpu, rrs[0]));
1565 *bundle = __get_domain_bundle(gip);
1566 if (swap_rr0) {
1567 set_metaphysical_rr0();
1569 if (bundle->i64[0] == 0 && bundle->i64[1] == 0) {
1570 dprintk(XENLOG_INFO, "%s gip 0x%lx\n", __func__, gip);
1571 return 0;
1573 return 1;
1575 found:
1576 gpip = ((tr.pte.ppn >> (tr.ps - 12)) << tr.ps) |
1577 (gip & ((1 << tr.ps) - 1));
1580 vaddr = (unsigned long)domain_mpa_to_imva(vcpu->domain, gpip);
1581 page = virt_to_page(vaddr);
1582 if (get_page(page, vcpu->domain) == 0) {
1583 if (page_get_owner(page) != vcpu->domain) {
1584 // This page might be a page granted by another
1585 // domain.
1586 panic_domain(regs, "domain tries to execute foreign "
1587 "domain page which might be mapped by "
1588 "grant table.\n");
1590 goto again;
1592 *bundle = *((IA64_BUNDLE *) vaddr);
1593 put_page(page);
1594 return 1;
1597 IA64FAULT vcpu_translate(VCPU * vcpu, u64 address, BOOLEAN is_data,
1598 u64 * pteval, u64 * itir, u64 * iha)
1600 unsigned long region = address >> 61;
1601 unsigned long pta, rid, rr;
1602 union pte_flags pte;
1603 TR_ENTRY *trp;
1605 if (PSCB(vcpu, metaphysical_mode) && !(!is_data && region)) {
1606 // dom0 may generate an uncacheable physical address (msb=1)
1607 if (region && ((region != 4) || (vcpu->domain != dom0))) {
1608 // FIXME: This seems to happen even though it shouldn't. Need to track
1609 // this down, but since it has been apparently harmless, just flag it for now
1610 // panic_domain(vcpu_regs(vcpu),
1612 /*
1613 * Guest may execute itc.d and rfi with psr.dt=0
1614 * When VMM try to fetch opcode, tlb miss may happen,
1615 * At this time PSCB(vcpu,metaphysical_mode)=1,
1616 * region=5,VMM need to handle this tlb miss as if
1617 * PSCB(vcpu,metaphysical_mode)=0
1618 */
1619 printk("vcpu_translate: bad physical address: 0x%lx "
1620 "at %lx\n", address, vcpu_regs(vcpu)->cr_iip);
1622 } else {
1623 *pteval = (address & _PAGE_PPN_MASK) |
1624 __DIRTY_BITS | _PAGE_PL_2 | _PAGE_AR_RWX;
1625 *itir = PAGE_SHIFT << 2;
1626 perfc_incrc(phys_translate);
1627 return IA64_NO_FAULT;
1629 } else if (!region && warn_region0_address) {
1630 REGS *regs = vcpu_regs(vcpu);
1631 unsigned long viip = PSCB(vcpu, iip);
1632 unsigned long vipsr = PSCB(vcpu, ipsr);
1633 unsigned long iip = regs->cr_iip;
1634 unsigned long ipsr = regs->cr_ipsr;
1635 printk("vcpu_translate: bad address 0x%lx, viip=0x%lx, "
1636 "vipsr=0x%lx, iip=0x%lx, ipsr=0x%lx continuing\n",
1637 address, viip, vipsr, iip, ipsr);
1640 rr = PSCB(vcpu, rrs)[region];
1641 rid = rr & RR_RID_MASK;
1642 if (is_data) {
1643 trp = vcpu_tr_lookup(vcpu, address, rid, 1);
1644 if (trp != NULL) {
1645 *pteval = trp->pte.val;
1646 *itir = trp->itir;
1647 perfc_incrc(tr_translate);
1648 return IA64_NO_FAULT;
1651 // FIXME?: check itr's for data accesses too, else bad things happen?
1652 /* else */ {
1653 trp = vcpu_tr_lookup(vcpu, address, rid, 0);
1654 if (trp != NULL) {
1655 *pteval = trp->pte.val;
1656 *itir = trp->itir;
1657 perfc_incrc(tr_translate);
1658 return IA64_NO_FAULT;
1662 /* check 1-entry TLB */
1663 // FIXME?: check dtlb for inst accesses too, else bad things happen?
1664 trp = &vcpu->arch.dtlb;
1665 pte = trp->pte;
1666 if ( /* is_data && */ pte.p
1667 && vcpu_match_tr_entry_no_p(trp, address, rid)) {
1668 *pteval = pte.val;
1669 *itir = trp->itir;
1670 perfc_incrc(dtlb_translate);
1671 return IA64_USE_TLB;
1674 /* check guest VHPT */
1675 pta = PSCB(vcpu, pta);
1676 if (pta & IA64_PTA_VF) { /* long format VHPT - not implemented */
1677 panic_domain(vcpu_regs(vcpu), "can't do long format VHPT\n");
1678 //return is_data ? IA64_DATA_TLB_VECTOR:IA64_INST_TLB_VECTOR;
1681 *itir = rr & (RR_RID_MASK | RR_PS_MASK);
1682 // note: architecturally, iha is optionally set for alt faults but
1683 // xenlinux depends on it so should document it as part of PV interface
1684 vcpu_thash(vcpu, address, iha);
1685 if (!(rr & RR_VE_MASK) || !(pta & IA64_PTA_VE)) {
1686 REGS *regs = vcpu_regs(vcpu);
1687 // NOTE: This is specific code for linux kernel
1688 // We assume region 7 is identity mapped
1689 if (region == 7 && ia64_psr(regs)->cpl == 2) {
1690 pte.val = address & _PAGE_PPN_MASK;
1691 pte.val = pte.val | pgprot_val(PAGE_KERNEL);
1692 goto out;
1694 return is_data ? IA64_ALT_DATA_TLB_VECTOR :
1695 IA64_ALT_INST_TLB_VECTOR;
1698 /* avoid recursively walking (short format) VHPT */
1699 if (((address ^ pta) & ((itir_mask(pta) << 3) >> 3)) == 0)
1700 return is_data ? IA64_DATA_TLB_VECTOR : IA64_INST_TLB_VECTOR;
1702 if (!__access_ok(*iha)
1703 || __copy_from_user(&pte, (void *)(*iha), sizeof(pte)) != 0)
1704 // virtual VHPT walker "missed" in TLB
1705 return IA64_VHPT_FAULT;
1707 /*
1708 * Optimisation: this VHPT walker aborts on not-present pages
1709 * instead of inserting a not-present translation, this allows
1710 * vectoring directly to the miss handler.
1711 */
1712 if (!pte.p)
1713 return is_data ? IA64_DATA_TLB_VECTOR : IA64_INST_TLB_VECTOR;
1715 /* found mapping in guest VHPT! */
1716 out:
1717 *itir = rr & RR_PS_MASK;
1718 *pteval = pte.val;
1719 perfc_incrc(vhpt_translate);
1720 return IA64_NO_FAULT;
1723 IA64FAULT vcpu_tpa(VCPU * vcpu, u64 vadr, u64 * padr)
1725 u64 pteval, itir, mask, iha;
1726 IA64FAULT fault;
1728 fault = vcpu_translate(vcpu, vadr, TRUE, &pteval, &itir, &iha);
1729 if (fault == IA64_NO_FAULT || fault == IA64_USE_TLB) {
1730 mask = itir_mask(itir);
1731 *padr = (pteval & _PAGE_PPN_MASK & mask) | (vadr & ~mask);
1732 return IA64_NO_FAULT;
1734 return vcpu_force_data_miss(vcpu, vadr);
1737 IA64FAULT vcpu_tak(VCPU * vcpu, u64 vadr, u64 * key)
1739 printk("vcpu_tak: tak instruction unsupported\n");
1740 return IA64_ILLOP_FAULT;
1741 // HACK ALERT: tak does a thash for now
1742 //return vcpu_thash(vcpu,vadr,key);
1745 /**************************************************************************
1746 VCPU debug breakpoint register access routines
1747 **************************************************************************/
1749 IA64FAULT vcpu_set_dbr(VCPU * vcpu, u64 reg, u64 val)
1751 // TODO: unimplemented DBRs return a reserved register fault
1752 // TODO: Should set Logical CPU state, not just physical
1753 ia64_set_dbr(reg, val);
1754 return IA64_NO_FAULT;
1757 IA64FAULT vcpu_set_ibr(VCPU * vcpu, u64 reg, u64 val)
1759 // TODO: unimplemented IBRs return a reserved register fault
1760 // TODO: Should set Logical CPU state, not just physical
1761 ia64_set_ibr(reg, val);
1762 return IA64_NO_FAULT;
1765 IA64FAULT vcpu_get_dbr(VCPU * vcpu, u64 reg, u64 * pval)
1767 // TODO: unimplemented DBRs return a reserved register fault
1768 u64 val = ia64_get_dbr(reg);
1769 *pval = val;
1770 return IA64_NO_FAULT;
1773 IA64FAULT vcpu_get_ibr(VCPU * vcpu, u64 reg, u64 * pval)
1775 // TODO: unimplemented IBRs return a reserved register fault
1776 u64 val = ia64_get_ibr(reg);
1777 *pval = val;
1778 return IA64_NO_FAULT;
1781 /**************************************************************************
1782 VCPU performance monitor register access routines
1783 **************************************************************************/
1785 IA64FAULT vcpu_set_pmc(VCPU * vcpu, u64 reg, u64 val)
1787 // TODO: Should set Logical CPU state, not just physical
1788 // NOTE: Writes to unimplemented PMC registers are discarded
1789 #ifdef DEBUG_PFMON
1790 printk("vcpu_set_pmc(%x,%lx)\n", reg, val);
1791 #endif
1792 ia64_set_pmc(reg, val);
1793 return IA64_NO_FAULT;
1796 IA64FAULT vcpu_set_pmd(VCPU * vcpu, u64 reg, u64 val)
1798 // TODO: Should set Logical CPU state, not just physical
1799 // NOTE: Writes to unimplemented PMD registers are discarded
1800 #ifdef DEBUG_PFMON
1801 printk("vcpu_set_pmd(%x,%lx)\n", reg, val);
1802 #endif
1803 ia64_set_pmd(reg, val);
1804 return IA64_NO_FAULT;
1807 IA64FAULT vcpu_get_pmc(VCPU * vcpu, u64 reg, u64 * pval)
1809 // NOTE: Reads from unimplemented PMC registers return zero
1810 u64 val = (u64) ia64_get_pmc(reg);
1811 #ifdef DEBUG_PFMON
1812 printk("%lx=vcpu_get_pmc(%x)\n", val, reg);
1813 #endif
1814 *pval = val;
1815 return IA64_NO_FAULT;
1818 IA64FAULT vcpu_get_pmd(VCPU * vcpu, u64 reg, u64 * pval)
1820 // NOTE: Reads from unimplemented PMD registers return zero
1821 u64 val = (u64) ia64_get_pmd(reg);
1822 #ifdef DEBUG_PFMON
1823 printk("%lx=vcpu_get_pmd(%x)\n", val, reg);
1824 #endif
1825 *pval = val;
1826 return IA64_NO_FAULT;
1829 /**************************************************************************
1830 VCPU banked general register access routines
1831 **************************************************************************/
1832 #define vcpu_bsw0_unat(i,b0unat,b1unat,runat,IA64_PT_REGS_R16_SLOT) \
1833 do{ \
1834 __asm__ __volatile__ ( \
1835 ";;extr.u %0 = %3,%6,16;;\n" \
1836 "dep %1 = %0, %1, 0, 16;;\n" \
1837 "st8 [%4] = %1\n" \
1838 "extr.u %0 = %2, 16, 16;;\n" \
1839 "dep %3 = %0, %3, %6, 16;;\n" \
1840 "st8 [%5] = %3\n" \
1841 ::"r"(i),"r"(*b1unat),"r"(*b0unat),"r"(*runat),"r"(b1unat), \
1842 "r"(runat),"i"(IA64_PT_REGS_R16_SLOT):"memory"); \
1843 }while(0)
1845 IA64FAULT vcpu_bsw0(VCPU * vcpu)
1847 // TODO: Only allowed for current vcpu
1848 REGS *regs = vcpu_regs(vcpu);
1849 unsigned long *r = &regs->r16;
1850 unsigned long *b0 = &PSCB(vcpu, bank0_regs[0]);
1851 unsigned long *b1 = &PSCB(vcpu, bank1_regs[0]);
1852 unsigned long *runat = &regs->eml_unat;
1853 unsigned long *b0unat = &PSCB(vcpu, vbnat);
1854 unsigned long *b1unat = &PSCB(vcpu, vnat);
1856 unsigned long i;
1858 if (VMX_DOMAIN(vcpu)) {
1859 if (VCPU(vcpu, vpsr) & IA64_PSR_BN) {
1860 for (i = 0; i < 16; i++) {
1861 *b1++ = *r;
1862 *r++ = *b0++;
1864 vcpu_bsw0_unat(i, b0unat, b1unat, runat,
1865 IA64_PT_REGS_R16_SLOT);
1866 VCPU(vcpu, vpsr) &= ~IA64_PSR_BN;
1868 } else {
1869 if (PSCB(vcpu, banknum)) {
1870 for (i = 0; i < 16; i++) {
1871 *b1++ = *r;
1872 *r++ = *b0++;
1874 vcpu_bsw0_unat(i, b0unat, b1unat, runat,
1875 IA64_PT_REGS_R16_SLOT);
1876 PSCB(vcpu, banknum) = 0;
1879 return IA64_NO_FAULT;
1882 #define vcpu_bsw1_unat(i, b0unat, b1unat, runat, IA64_PT_REGS_R16_SLOT) \
1883 do { \
1884 __asm__ __volatile__ (";;extr.u %0 = %3,%6,16;;\n" \
1885 "dep %1 = %0, %1, 16, 16;;\n" \
1886 "st8 [%4] = %1\n" \
1887 "extr.u %0 = %2, 0, 16;;\n" \
1888 "dep %3 = %0, %3, %6, 16;;\n" \
1889 "st8 [%5] = %3\n" \
1890 ::"r"(i), "r"(*b0unat), "r"(*b1unat), \
1891 "r"(*runat), "r"(b0unat), "r"(runat), \
1892 "i"(IA64_PT_REGS_R16_SLOT): "memory"); \
1893 } while(0)
1895 IA64FAULT vcpu_bsw1(VCPU * vcpu)
1897 // TODO: Only allowed for current vcpu
1898 REGS *regs = vcpu_regs(vcpu);
1899 unsigned long *r = &regs->r16;
1900 unsigned long *b0 = &PSCB(vcpu, bank0_regs[0]);
1901 unsigned long *b1 = &PSCB(vcpu, bank1_regs[0]);
1902 unsigned long *runat = &regs->eml_unat;
1903 unsigned long *b0unat = &PSCB(vcpu, vbnat);
1904 unsigned long *b1unat = &PSCB(vcpu, vnat);
1906 unsigned long i;
1908 if (VMX_DOMAIN(vcpu)) {
1909 if (!(VCPU(vcpu, vpsr) & IA64_PSR_BN)) {
1910 for (i = 0; i < 16; i++) {
1911 *b0++ = *r;
1912 *r++ = *b1++;
1914 vcpu_bsw1_unat(i, b0unat, b1unat, runat,
1915 IA64_PT_REGS_R16_SLOT);
1916 VCPU(vcpu, vpsr) |= IA64_PSR_BN;
1918 } else {
1919 if (!PSCB(vcpu, banknum)) {
1920 for (i = 0; i < 16; i++) {
1921 *b0++ = *r;
1922 *r++ = *b1++;
1924 vcpu_bsw1_unat(i, b0unat, b1unat, runat,
1925 IA64_PT_REGS_R16_SLOT);
1926 PSCB(vcpu, banknum) = 1;
1929 return IA64_NO_FAULT;
1932 /**************************************************************************
1933 VCPU cpuid access routines
1934 **************************************************************************/
1936 IA64FAULT vcpu_get_cpuid(VCPU * vcpu, u64 reg, u64 * pval)
1938 // FIXME: This could get called as a result of a rsvd-reg fault
1939 // if reg > 3
1940 switch (reg) {
1941 case 0:
1942 memcpy(pval, "Xen/ia64", 8);
1943 break;
1944 case 1:
1945 *pval = 0;
1946 break;
1947 case 2:
1948 *pval = 0;
1949 break;
1950 case 3:
1951 *pval = ia64_get_cpuid(3);
1952 break;
1953 case 4:
1954 *pval = ia64_get_cpuid(4);
1955 break;
1956 default:
1957 if (reg > (ia64_get_cpuid(3) & 0xff))
1958 return IA64_RSVDREG_FAULT;
1959 *pval = ia64_get_cpuid(reg);
1960 break;
1962 return IA64_NO_FAULT;
1965 /**************************************************************************
1966 VCPU region register access routines
1967 **************************************************************************/
1969 unsigned long vcpu_get_rr_ve(VCPU * vcpu, u64 vadr)
1971 ia64_rr rr;
1973 rr.rrval = PSCB(vcpu, rrs)[vadr >> 61];
1974 return rr.ve;
1977 IA64FAULT vcpu_set_rr(VCPU * vcpu, u64 reg, u64 val)
1979 PSCB(vcpu, rrs)[reg >> 61] = val;
1980 // warning: set_one_rr() does it "live"
1981 set_one_rr(reg, val);
1982 return IA64_NO_FAULT;
1985 IA64FAULT vcpu_get_rr(VCPU * vcpu, u64 reg, u64 * pval)
1987 if (VMX_DOMAIN(vcpu))
1988 *pval = VMX(vcpu, vrr[reg >> 61]);
1989 else
1990 *pval = PSCB(vcpu, rrs)[reg >> 61];
1992 return IA64_NO_FAULT;
1995 /**************************************************************************
1996 VCPU protection key register access routines
1997 **************************************************************************/
1999 IA64FAULT vcpu_get_pkr(VCPU * vcpu, u64 reg, u64 * pval)
2001 #ifndef PKR_USE_FIXED
2002 printk("vcpu_get_pkr: called, not implemented yet\n");
2003 return IA64_ILLOP_FAULT;
2004 #else
2005 u64 val = (u64) ia64_get_pkr(reg);
2006 *pval = val;
2007 return IA64_NO_FAULT;
2008 #endif
2011 IA64FAULT vcpu_set_pkr(VCPU * vcpu, u64 reg, u64 val)
2013 #ifndef PKR_USE_FIXED
2014 printk("vcpu_set_pkr: called, not implemented yet\n");
2015 return IA64_ILLOP_FAULT;
2016 #else
2017 // if (reg >= NPKRS)
2018 // return IA64_ILLOP_FAULT;
2019 vcpu->pkrs[reg] = val;
2020 ia64_set_pkr(reg, val);
2021 return IA64_NO_FAULT;
2022 #endif
2025 /**************************************************************************
2026 VCPU translation register access routines
2027 **************************************************************************/
2029 static void
2030 vcpu_set_tr_entry_rid(TR_ENTRY * trp, u64 pte,
2031 u64 itir, u64 ifa, u64 rid)
2033 u64 ps;
2034 union pte_flags new_pte;
2036 trp->itir = itir;
2037 trp->rid = rid;
2038 ps = trp->ps;
2039 new_pte.val = pte;
2040 if (new_pte.pl < 2)
2041 new_pte.pl = 2;
2042 trp->vadr = ifa & ~0xfff;
2043 if (ps > 12) { // "ignore" relevant low-order bits
2044 new_pte.ppn &= ~((1UL << (ps - 12)) - 1);
2045 trp->vadr &= ~((1UL << ps) - 1);
2048 /* Atomic write. */
2049 trp->pte.val = new_pte.val;
2052 static inline void
2053 vcpu_set_tr_entry(TR_ENTRY * trp, u64 pte, u64 itir, u64 ifa)
2055 vcpu_set_tr_entry_rid(trp, pte, itir, ifa,
2056 VCPU(current, rrs[ifa >> 61]) & RR_RID_MASK);
2059 IA64FAULT vcpu_itr_d(VCPU * vcpu, u64 slot, u64 pte,
2060 u64 itir, u64 ifa)
2062 TR_ENTRY *trp;
2064 if (slot >= NDTRS)
2065 return IA64_RSVDREG_FAULT;
2067 vcpu_purge_tr_entry(&PSCBX(vcpu, dtlb));
2069 trp = &PSCBX(vcpu, dtrs[slot]);
2070 //printk("***** itr.d: setting slot %d: ifa=%p\n",slot,ifa);
2071 vcpu_set_tr_entry(trp, pte, itir, ifa);
2072 vcpu_quick_region_set(PSCBX(vcpu, dtr_regions), ifa);
2074 /*
2075 * FIXME According to spec, vhpt should be purged, but this
2076 * incurs considerable performance loss, since it is safe for
2077 * linux not to purge vhpt, vhpt purge is disabled until a
2078 * feasible way is found.
2080 * vcpu_flush_tlb_vhpt_range(ifa & itir_mask(itir), itir_ps(itir));
2081 */
2083 return IA64_NO_FAULT;
2086 IA64FAULT vcpu_itr_i(VCPU * vcpu, u64 slot, u64 pte,
2087 u64 itir, u64 ifa)
2089 TR_ENTRY *trp;
2091 if (slot >= NITRS)
2092 return IA64_RSVDREG_FAULT;
2094 vcpu_purge_tr_entry(&PSCBX(vcpu, itlb));
2096 trp = &PSCBX(vcpu, itrs[slot]);
2097 //printk("***** itr.i: setting slot %d: ifa=%p\n",slot,ifa);
2098 vcpu_set_tr_entry(trp, pte, itir, ifa);
2099 vcpu_quick_region_set(PSCBX(vcpu, itr_regions), ifa);
2101 /*
2102 * FIXME According to spec, vhpt should be purged, but this
2103 * incurs considerable performance loss, since it is safe for
2104 * linux not to purge vhpt, vhpt purge is disabled until a
2105 * feasible way is found.
2107 * vcpu_flush_tlb_vhpt_range(ifa & itir_mask(itir), itir_ps(itir));
2108 */
2110 return IA64_NO_FAULT;
2113 IA64FAULT vcpu_set_itr(VCPU * vcpu, u64 slot, u64 pte,
2114 u64 itir, u64 ifa, u64 rid)
2116 TR_ENTRY *trp;
2118 if (slot >= NITRS)
2119 return IA64_RSVDREG_FAULT;
2120 trp = &PSCBX(vcpu, itrs[slot]);
2121 vcpu_set_tr_entry_rid(trp, pte, itir, ifa, rid);
2123 /* Recompute the itr_region. */
2124 vcpu->arch.itr_regions = 0;
2125 for (trp = vcpu->arch.itrs; trp < &vcpu->arch.itrs[NITRS]; trp++)
2126 if (trp->pte.p)
2127 vcpu_quick_region_set(vcpu->arch.itr_regions,
2128 trp->vadr);
2129 return IA64_NO_FAULT;
2132 IA64FAULT vcpu_set_dtr(VCPU * vcpu, u64 slot, u64 pte,
2133 u64 itir, u64 ifa, u64 rid)
2135 TR_ENTRY *trp;
2137 if (slot >= NDTRS)
2138 return IA64_RSVDREG_FAULT;
2139 trp = &PSCBX(vcpu, dtrs[slot]);
2140 vcpu_set_tr_entry_rid(trp, pte, itir, ifa, rid);
2142 /* Recompute the dtr_region. */
2143 vcpu->arch.dtr_regions = 0;
2144 for (trp = vcpu->arch.dtrs; trp < &vcpu->arch.dtrs[NDTRS]; trp++)
2145 if (trp->pte.p)
2146 vcpu_quick_region_set(vcpu->arch.dtr_regions,
2147 trp->vadr);
2148 return IA64_NO_FAULT;
2151 /**************************************************************************
2152 VCPU translation cache access routines
2153 **************************************************************************/
2155 void
2156 vcpu_itc_no_srlz(VCPU * vcpu, u64 IorD, u64 vaddr, u64 pte,
2157 u64 mp_pte, u64 logps, struct p2m_entry *entry)
2159 unsigned long psr;
2160 unsigned long ps = (vcpu->domain == dom0) ? logps : PAGE_SHIFT;
2162 check_xen_space_overlap("itc", vaddr, 1UL << logps);
2164 // FIXME, must be inlined or potential for nested fault here!
2165 if ((vcpu->domain == dom0) && (logps < PAGE_SHIFT))
2166 panic_domain(NULL, "vcpu_itc_no_srlz: domain trying to use "
2167 "smaller page size!\n");
2169 BUG_ON(logps > PAGE_SHIFT);
2170 vcpu_tlb_track_insert_or_dirty(vcpu, vaddr, entry);
2171 psr = ia64_clear_ic();
2172 pte &= ~(_PAGE_RV2 | _PAGE_RV1); // Mask out the reserved bits.
2173 ia64_itc(IorD, vaddr, pte, ps); // FIXME: look for bigger mappings
2174 ia64_set_psr(psr);
2175 // ia64_srlz_i(); // no srls req'd, will rfi later
2176 #ifdef VHPT_GLOBAL
2177 if (vcpu->domain == dom0 && ((vaddr >> 61) == 7)) {
2178 // FIXME: this is dangerous... vhpt_flush_address ensures these
2179 // addresses never get flushed. More work needed if this
2180 // ever happens.
2181 //printk("vhpt_insert(%p,%p,%p)\n",vaddr,pte,1L<<logps);
2182 if (logps > PAGE_SHIFT)
2183 vhpt_multiple_insert(vaddr, pte, logps);
2184 else
2185 vhpt_insert(vaddr, pte, logps << 2);
2187 // even if domain pagesize is larger than PAGE_SIZE, just put
2188 // PAGE_SIZE mapping in the vhpt for now, else purging is complicated
2189 else
2190 vhpt_insert(vaddr, pte, PAGE_SHIFT << 2);
2191 #endif
2194 IA64FAULT vcpu_itc_d(VCPU * vcpu, u64 pte, u64 itir, u64 ifa)
2196 unsigned long pteval, logps = itir_ps(itir);
2197 BOOLEAN swap_rr0 = (!(ifa >> 61) && PSCB(vcpu, metaphysical_mode));
2198 struct p2m_entry entry;
2200 if (logps < PAGE_SHIFT)
2201 panic_domain(NULL, "vcpu_itc_d: domain trying to use "
2202 "smaller page size!\n");
2204 again:
2205 //itir = (itir & ~0xfc) | (PAGE_SHIFT<<2); // ignore domain's pagesize
2206 pteval = translate_domain_pte(pte, ifa, itir, &logps, &entry);
2207 if (!pteval)
2208 return IA64_ILLOP_FAULT;
2209 if (swap_rr0)
2210 set_one_rr(0x0, PSCB(vcpu, rrs[0]));
2211 vcpu_itc_no_srlz(vcpu, 2, ifa, pteval, pte, logps, &entry);
2212 if (swap_rr0)
2213 set_metaphysical_rr0();
2214 if (p2m_entry_retry(&entry)) {
2215 vcpu_flush_tlb_vhpt_range(ifa, logps);
2216 goto again;
2218 vcpu_set_tr_entry(&PSCBX(vcpu, dtlb), pte, itir, ifa);
2219 return IA64_NO_FAULT;
2222 IA64FAULT vcpu_itc_i(VCPU * vcpu, u64 pte, u64 itir, u64 ifa)
2224 unsigned long pteval, logps = itir_ps(itir);
2225 BOOLEAN swap_rr0 = (!(ifa >> 61) && PSCB(vcpu, metaphysical_mode));
2226 struct p2m_entry entry;
2228 if (logps < PAGE_SHIFT)
2229 panic_domain(NULL, "vcpu_itc_i: domain trying to use "
2230 "smaller page size!\n");
2231 again:
2232 //itir = (itir & ~0xfc) | (PAGE_SHIFT<<2); // ignore domain's pagesize
2233 pteval = translate_domain_pte(pte, ifa, itir, &logps, &entry);
2234 if (!pteval)
2235 return IA64_ILLOP_FAULT;
2236 if (swap_rr0)
2237 set_one_rr(0x0, PSCB(vcpu, rrs[0]));
2238 vcpu_itc_no_srlz(vcpu, 1, ifa, pteval, pte, logps, &entry);
2239 if (swap_rr0)
2240 set_metaphysical_rr0();
2241 if (p2m_entry_retry(&entry)) {
2242 vcpu_flush_tlb_vhpt_range(ifa, logps);
2243 goto again;
2245 vcpu_set_tr_entry(&PSCBX(vcpu, itlb), pte, itir, ifa);
2246 return IA64_NO_FAULT;
2249 IA64FAULT vcpu_ptc_l(VCPU * vcpu, u64 vadr, u64 log_range)
2251 BUG_ON(vcpu != current);
2253 check_xen_space_overlap("ptc_l", vadr, 1UL << log_range);
2255 /* Purge TC */
2256 vcpu_purge_tr_entry(&PSCBX(vcpu, dtlb));
2257 vcpu_purge_tr_entry(&PSCBX(vcpu, itlb));
2259 /* Purge all tlb and vhpt */
2260 vcpu_flush_tlb_vhpt_range(vadr, log_range);
2262 return IA64_NO_FAULT;
2265 // At privlvl=0, fc performs no access rights or protection key checks, while
2266 // at privlvl!=0, fc performs access rights checks as if it were a 1-byte
2267 // read but no protection key check. Thus in order to avoid an unexpected
2268 // access rights fault, we have to translate the virtual address to a
2269 // physical address (possibly via a metaphysical address) and do the fc
2270 // on the physical address, which is guaranteed to flush the same cache line
2271 IA64FAULT vcpu_fc(VCPU * vcpu, u64 vadr)
2273 // TODO: Only allowed for current vcpu
2274 u64 mpaddr, paddr;
2275 IA64FAULT fault;
2277 again:
2278 fault = vcpu_tpa(vcpu, vadr, &mpaddr);
2279 if (fault == IA64_NO_FAULT) {
2280 struct p2m_entry entry;
2281 paddr = translate_domain_mpaddr(mpaddr, &entry);
2282 ia64_fc(__va(paddr));
2283 if (p2m_entry_retry(&entry))
2284 goto again;
2286 return fault;
2289 IA64FAULT vcpu_ptc_e(VCPU * vcpu, u64 vadr)
2291 // Note that this only needs to be called once, i.e. the
2292 // architected loop to purge the entire TLB, should use
2293 // base = stride1 = stride2 = 0, count0 = count 1 = 1
2295 vcpu_flush_vtlb_all(current);
2297 return IA64_NO_FAULT;
2300 IA64FAULT vcpu_ptc_g(VCPU * vcpu, u64 vadr, u64 addr_range)
2302 printk("vcpu_ptc_g: called, not implemented yet\n");
2303 return IA64_ILLOP_FAULT;
2306 IA64FAULT vcpu_ptc_ga(VCPU * vcpu, u64 vadr, u64 addr_range)
2308 // FIXME: validate not flushing Xen addresses
2309 // if (Xen address) return(IA64_ILLOP_FAULT);
2310 // FIXME: ??breaks if domain PAGE_SIZE < Xen PAGE_SIZE
2311 //printk("######## vcpu_ptc_ga(%p,%p) ##############\n",vadr,addr_range);
2313 check_xen_space_overlap("ptc_ga", vadr, addr_range);
2315 domain_flush_vtlb_range(vcpu->domain, vadr, addr_range);
2317 return IA64_NO_FAULT;
2320 IA64FAULT vcpu_ptr_d(VCPU * vcpu, u64 vadr, u64 log_range)
2322 unsigned long region = vadr >> 61;
2323 u64 addr_range = 1UL << log_range;
2324 unsigned long rid, rr;
2325 int i;
2326 TR_ENTRY *trp;
2328 BUG_ON(vcpu != current);
2329 check_xen_space_overlap("ptr_d", vadr, 1UL << log_range);
2331 rr = PSCB(vcpu, rrs)[region];
2332 rid = rr & RR_RID_MASK;
2334 /* Purge TC */
2335 vcpu_purge_tr_entry(&PSCBX(vcpu, dtlb));
2337 /* Purge tr and recompute dtr_regions. */
2338 vcpu->arch.dtr_regions = 0;
2339 for (trp = vcpu->arch.dtrs, i = NDTRS; i; i--, trp++)
2340 if (vcpu_match_tr_entry_range
2341 (trp, rid, vadr, vadr + addr_range))
2342 vcpu_purge_tr_entry(trp);
2343 else if (trp->pte.p)
2344 vcpu_quick_region_set(vcpu->arch.dtr_regions,
2345 trp->vadr);
2347 vcpu_flush_tlb_vhpt_range(vadr, log_range);
2349 return IA64_NO_FAULT;
2352 IA64FAULT vcpu_ptr_i(VCPU * vcpu, u64 vadr, u64 log_range)
2354 unsigned long region = vadr >> 61;
2355 u64 addr_range = 1UL << log_range;
2356 unsigned long rid, rr;
2357 int i;
2358 TR_ENTRY *trp;
2360 BUG_ON(vcpu != current);
2361 check_xen_space_overlap("ptr_i", vadr, 1UL << log_range);
2363 rr = PSCB(vcpu, rrs)[region];
2364 rid = rr & RR_RID_MASK;
2366 /* Purge TC */
2367 vcpu_purge_tr_entry(&PSCBX(vcpu, itlb));
2369 /* Purge tr and recompute itr_regions. */
2370 vcpu->arch.itr_regions = 0;
2371 for (trp = vcpu->arch.itrs, i = NITRS; i; i--, trp++)
2372 if (vcpu_match_tr_entry_range
2373 (trp, rid, vadr, vadr + addr_range))
2374 vcpu_purge_tr_entry(trp);
2375 else if (trp->pte.p)
2376 vcpu_quick_region_set(vcpu->arch.itr_regions,
2377 trp->vadr);
2379 vcpu_flush_tlb_vhpt_range(vadr, log_range);
2381 return IA64_NO_FAULT;