ia64/xen-unstable

view xen/arch/ia64/xen/hyperprivop.S @ 9756:14a34d811e81

[IA64] introduce P2M conversion

introduce P2M conversion functions necessary for dom0vp model.

Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
author awilliam@ldap.hp.com
date Tue Apr 25 13:06:57 2006 -0600 (2006-04-25)
parents 0a7e619a248f
children 4174856876f9
line source
1 /*
2 * arch/ia64/kernel/hyperprivop.S
3 *
4 * Copyright (C) 2005 Hewlett-Packard Co
5 * Dan Magenheimer <dan.magenheimer@hp.com>
6 */
8 #include <linux/config.h>
10 #include <asm/asmmacro.h>
11 #include <asm/kregs.h>
12 #include <asm/offsets.h>
13 #include <asm/processor.h>
14 #include <asm/system.h>
15 #include <asm/debugger.h>
16 #include <public/arch-ia64.h>
19 #define _PAGE_PPN_MASK 0x0003fffffffff000 //asm/pgtable.h doesn't do assembly
20 #define PAGE_PHYS 0x0010000000000761 //__pgprot(__DIRTY_BITS|_PAGE_PL_2|_PAGE_AR_RWX)
21 #define _PAGE_PL_2 (2<<7)
23 #if 1 // change to 0 to turn off all fast paths
24 #define FAST_HYPERPRIVOPS
25 #define FAST_HYPERPRIVOP_CNT
26 #define FAST_REFLECT_CNT
27 //#define FAST_TICK // mostly working (unat problems) but default off for now
28 //#define FAST_TLB_MISS_REFLECT // mostly working but default off for now
29 #ifdef CONFIG_XEN_IA64_DOM0_VP
30 #undef FAST_ITC //XXX CONFIG_XEN_IA64_DOM0_VP
31 // TODO fast_itc doesn't suport dom0 vp yet.
32 #else
33 //#define FAST_ITC // working but default off for now
34 #endif
35 #define FAST_BREAK
36 #ifndef CONFIG_XEN_IA64_DOM0_VP
37 # define FAST_ACCESS_REFLECT
38 #else
39 # undef FAST_ACCESS_REFLECT //XXX CONFIG_XEN_IA64_DOM0_VP
40 // TODO fast_access_reflect
41 // doesn't support dom0 vp yet.
42 #endif
43 #define FAST_RFI
44 #define FAST_SSM_I
45 #define FAST_PTC_GA
46 #undef RFI_TO_INTERRUPT // not working yet
47 #endif
49 #define XEN_HYPER_RFI 0x1
50 #define XEN_HYPER_RSM_DT 0x2
51 #define XEN_HYPER_SSM_DT 0x3
52 #define XEN_HYPER_COVER 0x4
53 #define XEN_HYPER_ITC_D 0x5
54 #define XEN_HYPER_ITC_I 0x6
55 #define XEN_HYPER_SSM_I 0x7
56 #define XEN_HYPER_GET_IVR 0x8
57 #define XEN_HYPER_GET_TPR 0x9
58 #define XEN_HYPER_SET_TPR 0xa
59 #define XEN_HYPER_EOI 0xb
60 #define XEN_HYPER_SET_ITM 0xc
61 #define XEN_HYPER_THASH 0xd
62 #define XEN_HYPER_PTC_GA 0xe
63 #define XEN_HYPER_ITR_D 0xf
64 #define XEN_HYPER_GET_RR 0x10
65 #define XEN_HYPER_SET_RR 0x11
66 #define XEN_HYPER_SET_KR 0x12
68 #ifdef CONFIG_SMP
69 #warning "FIXME: ptc.ga instruction requires spinlock for SMP"
70 #undef FAST_PTC_GA
71 #endif
73 // FIXME: turn off for now... but NaTs may crash Xen so re-enable soon!
74 #define HANDLE_AR_UNAT
76 // FIXME: This is defined in include/asm-ia64/hw_irq.h but this
77 // doesn't appear to be include'able from assembly?
78 #define IA64_TIMER_VECTOR 0xef
80 // Should be included from common header file (also in process.c)
81 // NO PSR_CLR IS DIFFERENT! (CPL)
82 #define IA64_PSR_CPL1 (__IA64_UL(1) << IA64_PSR_CPL1_BIT)
83 #define IA64_PSR_CPL0 (__IA64_UL(1) << IA64_PSR_CPL0_BIT)
84 // note IA64_PSR_PK removed from following, why is this necessary?
85 #define DELIVER_PSR_SET (IA64_PSR_IC | IA64_PSR_I | \
86 IA64_PSR_DT | IA64_PSR_RT | IA64_PSR_CPL1 | \
87 IA64_PSR_IT | IA64_PSR_BN)
89 #define DELIVER_PSR_CLR (IA64_PSR_AC | IA64_PSR_DFL | IA64_PSR_DFH | \
90 IA64_PSR_SP | IA64_PSR_DI | IA64_PSR_SI | \
91 IA64_PSR_DB | IA64_PSR_LP | IA64_PSR_TB | \
92 IA64_PSR_MC | IA64_PSR_IS | \
93 IA64_PSR_ID | IA64_PSR_DA | IA64_PSR_DD | \
94 IA64_PSR_SS | IA64_PSR_RI | IA64_PSR_ED | IA64_PSR_IA)
96 // Note: not hand-scheduled for now
97 // Registers at entry
98 // r16 == cr.isr
99 // r17 == cr.iim
100 // r18 == XSI_PSR_IC_OFS
101 // r19 == vpsr.ic
102 // r31 == pr
103 GLOBAL_ENTRY(fast_hyperprivop)
104 #ifndef FAST_HYPERPRIVOPS // see beginning of file
105 br.sptk.many dispatch_break_fault ;;
106 #endif
107 // HYPERPRIVOP_SSM_I?
108 // assumes domain interrupts pending, so just do it
109 cmp.eq p7,p6=XEN_HYPER_SSM_I,r17
110 (p7) br.sptk.many hyper_ssm_i;;
112 // FIXME. This algorithm gives up (goes to the slow path) if there
113 // are ANY interrupts pending, even if they are currently
114 // undeliverable. This should be improved later...
115 adds r20=XSI_PEND_OFS-XSI_PSR_IC_OFS,r18 ;;
116 ld4 r20=[r20] ;;
117 cmp.eq p7,p0=r0,r20
118 (p7) br.cond.sptk.many 1f
119 movl r20=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
120 ld8 r20=[r20];;
121 adds r21=IA64_VCPU_IRR0_OFFSET,r20;
122 adds r22=IA64_VCPU_IRR0_OFFSET+8,r20;;
123 ld8 r23=[r21],16; ld8 r24=[r22],16;;
124 ld8 r21=[r21]; ld8 r22=[r22];;
125 or r23=r23,r24; or r21=r21,r22;;
126 or r20=r23,r21;;
127 1: // when we get to here r20=~=interrupts pending
129 // HYPERPRIVOP_RFI?
130 cmp.eq p7,p6=XEN_HYPER_RFI,r17
131 (p7) br.sptk.many hyper_rfi;;
133 // HYPERPRIVOP_GET_IVR?
134 cmp.eq p7,p6=XEN_HYPER_GET_IVR,r17
135 (p7) br.sptk.many hyper_get_ivr;;
137 cmp.ne p7,p0=r20,r0
138 (p7) br.spnt.many dispatch_break_fault ;;
140 // HYPERPRIVOP_COVER?
141 cmp.eq p7,p6=XEN_HYPER_COVER,r17
142 (p7) br.sptk.many hyper_cover;;
144 // HYPERPRIVOP_SSM_DT?
145 cmp.eq p7,p6=XEN_HYPER_SSM_DT,r17
146 (p7) br.sptk.many hyper_ssm_dt;;
148 // HYPERPRIVOP_RSM_DT?
149 cmp.eq p7,p6=XEN_HYPER_RSM_DT,r17
150 (p7) br.sptk.many hyper_rsm_dt;;
152 // HYPERPRIVOP_GET_TPR?
153 cmp.eq p7,p6=XEN_HYPER_GET_TPR,r17
154 (p7) br.sptk.many hyper_get_tpr;;
156 // HYPERPRIVOP_SET_TPR?
157 cmp.eq p7,p6=XEN_HYPER_SET_TPR,r17
158 (p7) br.sptk.many hyper_set_tpr;;
160 // HYPERPRIVOP_EOI?
161 cmp.eq p7,p6=XEN_HYPER_EOI,r17
162 (p7) br.sptk.many hyper_eoi;;
164 // HYPERPRIVOP_SET_ITM?
165 cmp.eq p7,p6=XEN_HYPER_SET_ITM,r17
166 (p7) br.sptk.many hyper_set_itm;;
168 // HYPERPRIVOP_SET_RR?
169 cmp.eq p7,p6=XEN_HYPER_SET_RR,r17
170 (p7) br.sptk.many hyper_set_rr;;
172 // HYPERPRIVOP_GET_RR?
173 cmp.eq p7,p6=XEN_HYPER_GET_RR,r17
174 (p7) br.sptk.many hyper_get_rr;;
176 // HYPERPRIVOP_PTC_GA?
177 cmp.eq p7,p6=XEN_HYPER_PTC_GA,r17
178 (p7) br.sptk.many hyper_ptc_ga;;
180 // HYPERPRIVOP_ITC_D?
181 cmp.eq p7,p6=XEN_HYPER_ITC_D,r17
182 (p7) br.sptk.many hyper_itc_d;;
184 // HYPERPRIVOP_ITC_I?
185 cmp.eq p7,p6=XEN_HYPER_ITC_I,r17
186 (p7) br.sptk.many hyper_itc_i;;
188 // HYPERPRIVOP_THASH?
189 cmp.eq p7,p6=XEN_HYPER_THASH,r17
190 (p7) br.sptk.many hyper_thash;;
192 // HYPERPRIVOP_SET_KR?
193 cmp.eq p7,p6=XEN_HYPER_SET_KR,r17
194 (p7) br.sptk.many hyper_set_kr;;
196 // if not one of the above, give up for now and do it the slow way
197 br.sptk.many dispatch_break_fault ;;
200 // give up for now if: ipsr.be==1, ipsr.pp==1
201 // from reflect_interruption, don't need to:
202 // - printf first extint (debug only)
203 // - check for interrupt collection enabled (routine will force on)
204 // - set ifa (not valid for extint)
205 // - set iha (not valid for extint)
206 // - set itir (not valid for extint)
207 // DO need to
208 // - increment the HYPER_SSM_I fast_hyperprivop counter
209 // - set shared_mem iip to instruction after HYPER_SSM_I
210 // - set cr.iip to guest iva+0x3000
211 // - set shared_mem ipsr to [vcpu_get_ipsr_int_state]
212 // be = pp = bn = 0; dt = it = rt = 1; cpl = 3 or 0;
213 // i = shared_mem interrupt_delivery_enabled
214 // ic = shared_mem interrupt_collection_enabled
215 // ri = instruction after HYPER_SSM_I
216 // all other bits unchanged from real cr.ipsr
217 // - set cr.ipsr (DELIVER_PSR_SET/CLEAR, don't forget cpl!)
218 // - set shared_mem isr: isr.ei to instr following HYPER_SSM_I
219 // and isr.ri to cr.isr.ri (all other bits zero)
220 // - cover and set shared_mem precover_ifs to cr.ifs
221 // ^^^ MISSED THIS FOR fast_break??
222 // - set shared_mem ifs and incomplete_regframe to 0
223 // - set shared_mem interrupt_delivery_enabled to 0
224 // - set shared_mem interrupt_collection_enabled to 0
225 // - set r31 to SHAREDINFO_ADDR
226 // - virtual bank switch 0
227 // maybe implement later
228 // - verify that there really IS a deliverable interrupt pending
229 // - set shared_mem iva
230 // needs to be done but not implemented (in reflect_interruption)
231 // - set shared_mem iipa
232 // don't know for sure
233 // - set shared_mem unat
234 // r16 == cr.isr
235 // r17 == cr.iim
236 // r18 == XSI_PSR_IC
237 // r19 == vpsr.ic
238 // r31 == pr
239 ENTRY(hyper_ssm_i)
240 #ifndef FAST_SSM_I
241 br.spnt.few dispatch_break_fault ;;
242 #endif
243 // give up for now if: ipsr.be==1, ipsr.pp==1
244 mov r30=cr.ipsr;;
245 mov r29=cr.iip;;
246 extr.u r21=r30,IA64_PSR_BE_BIT,1 ;;
247 cmp.ne p7,p0=r21,r0
248 (p7) br.sptk.many dispatch_break_fault ;;
249 extr.u r21=r30,IA64_PSR_PP_BIT,1 ;;
250 cmp.ne p7,p0=r21,r0
251 (p7) br.sptk.many dispatch_break_fault ;;
252 #ifdef FAST_HYPERPRIVOP_CNT
253 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_SSM_I);;
254 ld8 r21=[r20];;
255 adds r21=1,r21;;
256 st8 [r20]=r21;;
257 #endif
258 // set shared_mem iip to instruction after HYPER_SSM_I
259 extr.u r20=r30,41,2 ;;
260 cmp.eq p6,p7=2,r20 ;;
261 (p6) mov r20=0
262 (p6) adds r29=16,r29
263 (p7) adds r20=1,r20 ;;
264 dep r30=r20,r30,41,2;; // adjust cr.ipsr.ri but don't save yet
265 adds r21=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
266 st8 [r21]=r29 ;;
267 // set shared_mem isr
268 extr.u r16=r16,38,1;; // grab cr.isr.ir bit
269 dep r16=r16,r0,38,1 ;; // insert into cr.isr (rest of bits zero)
270 dep r16=r20,r16,41,2 ;; // deposit cr.isr.ri
271 adds r21=XSI_ISR_OFS-XSI_PSR_IC_OFS,r18 ;;
272 st8 [r21]=r16 ;;
273 // set cr.ipsr
274 mov r29=r30 ;;
275 movl r28=DELIVER_PSR_SET;;
276 movl r27=~DELIVER_PSR_CLR;;
277 or r29=r29,r28;;
278 and r29=r29,r27;;
279 mov cr.ipsr=r29;;
280 // set shared_mem ipsr (from ipsr in r30 with ipsr.ri already set)
281 extr.u r29=r30,IA64_PSR_CPL0_BIT,2;;
282 cmp.eq p6,p7=3,r29;;
283 (p6) dep r30=-1,r30,IA64_PSR_CPL0_BIT,2
284 (p7) dep r30=0,r30,IA64_PSR_CPL0_BIT,2
285 ;;
286 // FOR SSM_I ONLY, also turn on psr.i and psr.ic
287 movl r28=(IA64_PSR_DT|IA64_PSR_IT|IA64_PSR_RT|IA64_PSR_I|IA64_PSR_IC);;
288 // movl r27=~(IA64_PSR_BE|IA64_PSR_PP|IA64_PSR_BN);;
289 movl r27=~(IA64_PSR_BE|IA64_PSR_BN);;
290 or r30=r30,r28;;
291 and r30=r30,r27;;
292 mov r20=1
293 adds r22=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r18
294 adds r21=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18 ;;
295 ld8 r22=[r22]
296 st8 [r21]=r30 ;;
297 // set shared_mem interrupt_delivery_enabled to 0
298 // set shared_mem interrupt_collection_enabled to 0
299 st1 [r22]=r20;;
300 st4 [r18]=r0;;
301 // cover and set shared_mem precover_ifs to cr.ifs
302 // set shared_mem ifs and incomplete_regframe to 0
303 cover ;;
304 mov r20=cr.ifs;;
305 adds r21=XSI_INCOMPL_REGFR_OFS-XSI_PSR_IC_OFS,r18 ;;
306 st4 [r21]=r0 ;;
307 adds r21=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
308 st8 [r21]=r0 ;;
309 adds r21=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
310 st8 [r21]=r20 ;;
311 // leave cr.ifs alone for later rfi
312 // set iip to go to domain IVA break instruction vector
313 movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
314 ld8 r22=[r22];;
315 adds r22=IA64_VCPU_IVA_OFFSET,r22;;
316 ld8 r23=[r22];;
317 movl r24=0x3000;;
318 add r24=r24,r23;;
319 mov cr.iip=r24;;
320 // OK, now all set to go except for switch to virtual bank0
321 mov r30=r2; mov r29=r3;;
322 adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18;
323 adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18;;
324 bsw.1;;
325 // FIXME?: ar.unat is not really handled correctly,
326 // but may not matter if the OS is NaT-clean
327 .mem.offset 0,0; st8.spill [r2]=r16,16;
328 .mem.offset 8,0; st8.spill [r3]=r17,16 ;;
329 .mem.offset 0,0; st8.spill [r2]=r18,16;
330 .mem.offset 8,0; st8.spill [r3]=r19,16 ;;
331 .mem.offset 0,0; st8.spill [r2]=r20,16;
332 .mem.offset 8,0; st8.spill [r3]=r21,16 ;;
333 .mem.offset 0,0; st8.spill [r2]=r22,16;
334 .mem.offset 8,0; st8.spill [r3]=r23,16 ;;
335 .mem.offset 0,0; st8.spill [r2]=r24,16;
336 .mem.offset 8,0; st8.spill [r3]=r25,16 ;;
337 .mem.offset 0,0; st8.spill [r2]=r26,16;
338 .mem.offset 8,0; st8.spill [r3]=r27,16 ;;
339 .mem.offset 0,0; st8.spill [r2]=r28,16;
340 .mem.offset 8,0; st8.spill [r3]=r29,16 ;;
341 .mem.offset 0,0; st8.spill [r2]=r30,16;
342 .mem.offset 8,0; st8.spill [r3]=r31,16 ;;
343 movl r31=XSI_IPSR;;
344 bsw.0 ;;
345 mov r2=r30; mov r3=r29;;
346 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
347 st4 [r20]=r0 ;;
348 mov pr=r31,-1 ;;
349 rfi
350 ;;
352 // reflect domain clock interrupt
353 // r31 == pr
354 // r30 == cr.ivr
355 // r29 == rp
356 GLOBAL_ENTRY(fast_tick_reflect)
357 #ifndef FAST_TICK // see beginning of file
358 br.cond.sptk.many rp;;
359 #endif
360 mov r28=IA64_TIMER_VECTOR;;
361 cmp.ne p6,p0=r28,r30
362 (p6) br.cond.spnt.few rp;;
363 movl r20=THIS_CPU(cpu_info)+IA64_CPUINFO_ITM_NEXT_OFFSET;;
364 ld8 r26=[r20];;
365 mov r27=ar.itc;;
366 adds r27=200,r27;; // safety margin
367 cmp.ltu p6,p0=r26,r27
368 (p6) br.cond.spnt.few rp;;
369 mov r17=cr.ipsr;;
370 // slow path if: ipsr.be==1, ipsr.pp==1
371 extr.u r21=r17,IA64_PSR_BE_BIT,1 ;;
372 cmp.ne p6,p0=r21,r0
373 (p6) br.cond.spnt.few rp;;
374 extr.u r21=r17,IA64_PSR_PP_BIT,1 ;;
375 cmp.ne p6,p0=r21,r0
376 (p6) br.cond.spnt.few rp;;
377 // definitely have a domain tick
378 mov cr.eoi=r0;;
379 mov rp=r29;;
380 mov cr.itm=r26;; // ensure next tick
381 #ifdef FAST_REFLECT_CNT
382 movl r20=fast_reflect_count+((0x3000>>8)*8);;
383 ld8 r21=[r20];;
384 adds r21=1,r21;;
385 st8 [r20]=r21;;
386 #endif
387 // vcpu_pend_timer(current)
388 movl r18=XSI_PSR_IC;;
389 adds r20=XSI_ITV_OFS-XSI_PSR_IC_OFS,r18 ;;
390 ld8 r20=[r20];;
391 cmp.eq p6,p0=r20,r0 // if cr.itv==0 done
392 (p6) br.cond.spnt.few fast_tick_reflect_done;;
393 tbit.nz p6,p0=r20,16;; // check itv.m (discard) bit
394 (p6) br.cond.spnt.few fast_tick_reflect_done;;
395 extr.u r27=r20,0,6 // r27 has low 6 bits of itv.vector
396 extr.u r26=r20,6,2;; // r26 has irr index of itv.vector
397 movl r19=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
398 ld8 r19=[r19];;
399 adds r22=IA64_VCPU_DOMAIN_ITM_LAST_OFFSET,r19
400 adds r23=IA64_VCPU_DOMAIN_ITM_OFFSET,r19;;
401 ld8 r24=[r22];;
402 ld8 r23=[r23];;
403 cmp.eq p6,p0=r23,r24 // skip if this tick already delivered
404 (p6) br.cond.spnt.few fast_tick_reflect_done;;
405 // set irr bit
406 adds r21=IA64_VCPU_IRR0_OFFSET,r19;
407 shl r26=r26,3;;
408 add r21=r21,r26;;
409 mov r25=1;;
410 shl r22=r25,r27;;
411 ld8 r23=[r21];;
412 or r22=r22,r23;;
413 st8 [r21]=r22;;
414 // set PSCB(pending_interruption)!
415 adds r20=XSI_PEND_OFS-XSI_PSR_IC_OFS,r18 ;;
416 st4 [r20]=r25;;
418 // if interrupted at pl0, we're done
419 extr.u r16=r17,IA64_PSR_CPL0_BIT,2;;
420 cmp.eq p6,p0=r16,r0;;
421 (p6) br.cond.spnt.few fast_tick_reflect_done;;
422 // if guest vpsr.i is off, we're done
423 adds r21=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r18 ;;
424 ld8 r21=[r21];;
425 ld1 r21=[r21];;
426 cmp.eq p0,p6=r21,r0
427 (p6) br.cond.spnt.few fast_tick_reflect_done;;
429 // OK, we have a clock tick to deliver to the active domain!
430 // so deliver to iva+0x3000
431 // r17 == cr.ipsr
432 // r18 == XSI_PSR_IC
433 // r19 == IA64_KR(CURRENT)
434 // r31 == pr
435 mov r16=cr.isr;;
436 mov r29=cr.iip;;
437 adds r21=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
438 st8 [r21]=r29 ;;
439 // set shared_mem isr
440 extr.u r16=r16,38,1;; // grab cr.isr.ir bit
441 dep r16=r16,r0,38,1 ;; // insert into cr.isr (rest of bits zero)
442 extr.u r20=r17,41,2 ;; // get ipsr.ri
443 dep r16=r20,r16,41,2 ;; // deposit cr.isr.ei
444 adds r21=XSI_ISR_OFS-XSI_PSR_IC_OFS,r18 ;;
445 st8 [r21]=r16 ;;
446 // set cr.ipsr (make sure cpl==2!)
447 mov r29=r17 ;;
448 movl r28=DELIVER_PSR_SET;;
449 movl r27=~(DELIVER_PSR_CLR|IA64_PSR_CPL0);;
450 or r29=r29,r28;;
451 and r29=r29,r27;;
452 mov cr.ipsr=r29;;
453 // set shared_mem ipsr (from ipsr in r17 with ipsr.ri already set)
454 extr.u r29=r17,IA64_PSR_CPL0_BIT,2;;
455 cmp.eq p6,p7=3,r29;;
456 (p6) dep r17=-1,r17,IA64_PSR_CPL0_BIT,2
457 (p7) dep r17=0,r17,IA64_PSR_CPL0_BIT,2
458 ;;
459 movl r28=(IA64_PSR_DT|IA64_PSR_IT|IA64_PSR_RT);;
460 movl r27=~(IA64_PSR_BE|IA64_PSR_PP|IA64_PSR_BN|IA64_PSR_I|IA64_PSR_IC);;
461 dep r21=-1,r21,IA64_PSR_CPL1_BIT,1 ;;
462 or r17=r17,r28;;
463 and r17=r17,r27;;
464 ld4 r16=[r18],XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS;;
465 cmp.ne p6,p0=r16,r0;;
466 ld8 r16=[r18],XSI_PSR_IC_OFS-XSI_PSR_I_ADDR_OFS
467 (p6) dep r17=-1,r17,IA64_PSR_IC_BIT,1 ;;
468 ld1 r16=[r16];;
469 cmp.eq p6,p0=r16,r0;;
470 (p6) dep r17=-1,r17,IA64_PSR_I_BIT,1 ;;
471 mov r20=1
472 adds r22=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r18
473 adds r21=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18 ;;
474 ld8 r22=[r22]
475 st8 [r21]=r17 ;;
476 // set shared_mem interrupt_delivery_enabled to 0
477 // set shared_mem interrupt_collection_enabled to 0
478 st1 [r22]=r20;;
479 st4 [r18]=r0;;
480 // cover and set shared_mem precover_ifs to cr.ifs
481 // set shared_mem ifs and incomplete_regframe to 0
482 cover ;;
483 mov r20=cr.ifs;;
484 adds r21=XSI_INCOMPL_REGFR_OFS-XSI_PSR_IC_OFS,r18 ;;
485 st4 [r21]=r0 ;;
486 adds r21=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
487 st8 [r21]=r0 ;;
488 adds r21=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
489 st8 [r21]=r20 ;;
490 // leave cr.ifs alone for later rfi
491 // set iip to go to domain IVA break instruction vector
492 adds r22=IA64_VCPU_IVA_OFFSET,r19;;
493 ld8 r23=[r22];;
494 movl r24=0x3000;;
495 add r24=r24,r23;;
496 mov cr.iip=r24;;
497 // OK, now all set to go except for switch to virtual bank0
498 mov r30=r2; mov r29=r3;;
499 #ifdef HANDLE_AR_UNAT
500 mov r28=ar.unat;
501 #endif
502 adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18;
503 adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18;;
504 bsw.1;;
505 .mem.offset 0,0; st8.spill [r2]=r16,16;
506 .mem.offset 8,0; st8.spill [r3]=r17,16 ;;
507 .mem.offset 0,0; st8.spill [r2]=r18,16;
508 .mem.offset 8,0; st8.spill [r3]=r19,16 ;;
509 .mem.offset 0,0; st8.spill [r2]=r20,16;
510 .mem.offset 8,0; st8.spill [r3]=r21,16 ;;
511 .mem.offset 0,0; st8.spill [r2]=r22,16;
512 .mem.offset 8,0; st8.spill [r3]=r23,16 ;;
513 .mem.offset 0,0; st8.spill [r2]=r24,16;
514 .mem.offset 8,0; st8.spill [r3]=r25,16 ;;
515 .mem.offset 0,0; st8.spill [r2]=r26,16;
516 .mem.offset 8,0; st8.spill [r3]=r27,16 ;;
517 .mem.offset 0,0; st8.spill [r2]=r28,16;
518 .mem.offset 8,0; st8.spill [r3]=r29,16 ;;
519 .mem.offset 0,0; st8.spill [r2]=r30,16;
520 .mem.offset 8,0; st8.spill [r3]=r31,16 ;;
521 #ifdef HANDLE_AR_UNAT
522 // r16~r23 are preserved regsin bank0 regs, we need to restore them,
523 // r24~r31 are scratch regs, we don't need to handle NaT bit,
524 // because OS handler must assign it before access it
525 ld8 r16=[r2],16;
526 ld8 r17=[r3],16;;
527 ld8 r18=[r2],16;
528 ld8 r19=[r3],16;;
529 ld8 r20=[r2],16;
530 ld8 r21=[r3],16;;
531 ld8 r22=[r2],16;
532 ld8 r23=[r3],16;;
533 #endif
534 movl r31=XSI_IPSR;;
535 bsw.0 ;;
536 mov r24=ar.unat;
537 mov r2=r30; mov r3=r29;;
538 #ifdef HANDLE_AR_UNAT
539 mov ar.unat=r28;
540 #endif
541 adds r25=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18 ;
542 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
543 st8 [r25]=r24;
544 st4 [r20]=r0 ;;
545 fast_tick_reflect_done:
546 mov pr=r31,-1 ;;
547 rfi
548 END(fast_tick_reflect)
550 // reflect domain breaks directly to domain
551 // r16 == cr.isr
552 // r17 == cr.iim
553 // r18 == XSI_PSR_IC
554 // r19 == vpsr.ic
555 // r31 == pr
556 GLOBAL_ENTRY(fast_break_reflect)
557 #ifndef FAST_BREAK // see beginning of file
558 br.sptk.many dispatch_break_fault ;;
559 #endif
560 mov r30=cr.ipsr;;
561 mov r29=cr.iip;;
562 extr.u r21=r30,IA64_PSR_BE_BIT,1 ;;
563 cmp.ne p7,p0=r21,r0 ;;
564 (p7) br.spnt.few dispatch_break_fault ;;
565 extr.u r21=r30,IA64_PSR_PP_BIT,1 ;;
566 cmp.ne p7,p0=r21,r0 ;;
567 (p7) br.spnt.few dispatch_break_fault ;;
568 movl r20=IA64_PSR_CPL ;;
569 and r22=r20,r30 ;;
570 cmp.ne p7,p0=r22,r0
571 (p7) br.spnt.many 1f ;;
572 cmp.eq p7,p0=r17,r0
573 (p7) br.spnt.few dispatch_break_fault ;;
574 #ifdef CRASH_DEBUG
575 movl r21=CDB_BREAK_NUM ;;
576 cmp.eq p7,p0=r17,r21
577 (p7) br.spnt.few dispatch_break_fault ;;
578 #endif
579 1:
580 #if 1 /* special handling in case running on simulator */
581 movl r20=first_break;;
582 ld4 r23=[r20];;
583 movl r21=0x80001;
584 movl r22=0x80002;;
585 cmp.ne p7,p0=r23,r0;;
586 (p7) br.spnt.few dispatch_break_fault ;;
587 cmp.eq p7,p0=r21,r17;
588 (p7) br.spnt.few dispatch_break_fault ;;
589 cmp.eq p7,p0=r22,r17;
590 (p7) br.spnt.few dispatch_break_fault ;;
591 #endif
592 movl r20=0x2c00;
593 // save iim in shared_info
594 adds r21=XSI_IIM_OFS-XSI_PSR_IC_OFS,r18 ;;
595 st8 [r21]=r17;;
596 // fall through
599 // reflect to domain ivt+r20
600 // sets up isr,iip,ipsr,ifs (FIXME: do iipa too)
601 // r16 == cr.isr
602 // r18 == XSI_PSR_IC
603 // r20 == offset into ivt
604 // r29 == iip
605 // r30 == ipsr
606 // r31 == pr
607 ENTRY(fast_reflect)
608 #ifdef FAST_REFLECT_CNT
609 movl r22=fast_reflect_count;
610 shr r23=r20,5;;
611 add r22=r22,r23;;
612 ld8 r21=[r22];;
613 adds r21=1,r21;;
614 st8 [r22]=r21;;
615 #endif
616 // save iip in shared_info (DON'T POINT TO NEXT INSTRUCTION!)
617 adds r21=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
618 st8 [r21]=r29,XSI_ISR_OFS-XSI_IIP_OFS;;
619 // set shared_mem isr
620 st8 [r21]=r16 ;;
621 // set cr.ipsr
622 adds r21=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r18
623 mov r29=r30 ;;
624 ld8 r21=[r21]
625 movl r28=DELIVER_PSR_SET;;
626 movl r27=~(DELIVER_PSR_CLR|IA64_PSR_CPL0);;
627 or r29=r29,r28;;
628 and r29=r29,r27;;
629 mov cr.ipsr=r29;;
630 // set shared_mem ipsr (from ipsr in r30 with ipsr.ri already set)
631 extr.u r29=r30,IA64_PSR_CPL0_BIT,2;;
632 cmp.eq p6,p7=3,r29;;
633 (p6) dep r30=-1,r30,IA64_PSR_CPL0_BIT,2
634 (p7) dep r30=0,r30,IA64_PSR_CPL0_BIT,2
635 ;;
636 movl r28=(IA64_PSR_DT|IA64_PSR_IT|IA64_PSR_RT);;
637 movl r27=~(IA64_PSR_BE|IA64_PSR_PP|IA64_PSR_BN);;
638 or r30=r30,r28;;
639 and r30=r30,r27;;
640 // also set shared_mem ipsr.i and ipsr.ic appropriately
641 ld1 r22=[r21]
642 ld4 r24=[r18];;
643 cmp4.eq p6,p7=r24,r0;;
644 (p6) dep r30=0,r30,IA64_PSR_IC_BIT,1
645 (p7) dep r30=-1,r30,IA64_PSR_IC_BIT,1 ;;
646 mov r24=r21
647 cmp.ne p6,p7=r22,r0;;
648 (p6) dep r30=0,r30,IA64_PSR_I_BIT,1
649 (p7) dep r30=-1,r30,IA64_PSR_I_BIT,1 ;;
650 mov r22=1
651 adds r21=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18 ;;
652 st8 [r21]=r30 ;;
653 // set shared_mem interrupt_delivery_enabled to 0
654 // set shared_mem interrupt_collection_enabled to 0
655 st1 [r24]=r22
656 st4 [r18]=r0;;
657 // cover and set shared_mem precover_ifs to cr.ifs
658 // set shared_mem ifs and incomplete_regframe to 0
659 cover ;;
660 mov r24=cr.ifs;;
661 adds r21=XSI_INCOMPL_REGFR_OFS-XSI_PSR_IC_OFS,r18 ;;
662 st4 [r21]=r0 ;;
663 adds r21=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
664 st8 [r21]=r0 ;;
665 adds r21=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
666 st8 [r21]=r24 ;;
667 // FIXME: need to save iipa and isr to be arch-compliant
668 // set iip to go to domain IVA break instruction vector
669 movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
670 ld8 r22=[r22];;
671 adds r22=IA64_VCPU_IVA_OFFSET,r22;;
672 ld8 r23=[r22];;
673 add r20=r20,r23;;
674 mov cr.iip=r20;;
675 // OK, now all set to go except for switch to virtual bank0
676 mov r30=r2; mov r29=r3;;
677 #ifdef HANDLE_AR_UNAT
678 mov r28=ar.unat;
679 #endif
680 adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18;
681 adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18;;
682 bsw.1;;
683 .mem.offset 0,0; st8.spill [r2]=r16,16;
684 .mem.offset 8,0; st8.spill [r3]=r17,16 ;;
685 .mem.offset 0,0; st8.spill [r2]=r18,16;
686 .mem.offset 8,0; st8.spill [r3]=r19,16 ;;
687 .mem.offset 0,0; st8.spill [r2]=r20,16;
688 .mem.offset 8,0; st8.spill [r3]=r21,16 ;;
689 .mem.offset 0,0; st8.spill [r2]=r22,16;
690 .mem.offset 8,0; st8.spill [r3]=r23,16 ;;
691 .mem.offset 0,0; st8.spill [r2]=r24,16;
692 .mem.offset 8,0; st8.spill [r3]=r25,16 ;;
693 .mem.offset 0,0; st8.spill [r2]=r26,16;
694 .mem.offset 8,0; st8.spill [r3]=r27,16 ;;
695 .mem.offset 0,0; st8.spill [r2]=r28,16;
696 .mem.offset 8,0; st8.spill [r3]=r29,16 ;;
697 .mem.offset 0,0; st8.spill [r2]=r30,16;
698 .mem.offset 8,0; st8.spill [r3]=r31,16 ;;
699 #ifdef HANDLE_AR_UNAT
700 // r16~r23 are preserved regsin bank0 regs, we need to restore them,
701 // r24~r31 are scratch regs, we don't need to handle NaT bit,
702 // because OS handler must assign it before access it
703 ld8 r16=[r2],16;
704 ld8 r17=[r3],16;;
705 ld8 r18=[r2],16;
706 ld8 r19=[r3],16;;
707 ld8 r20=[r2],16;
708 ld8 r21=[r3],16;;
709 ld8 r22=[r2],16;
710 ld8 r23=[r3],16;;
711 #endif
712 movl r31=XSI_IPSR;;
713 bsw.0 ;;
714 mov r24=ar.unat;
715 mov r2=r30; mov r3=r29;;
716 #ifdef HANDLE_AR_UNAT
717 mov ar.unat=r28;
718 #endif
719 adds r25=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18 ;
720 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
721 st8 [r25]=r24;
722 st4 [r20]=r0 ;;
723 mov pr=r31,-1 ;;
724 rfi
725 ;;
727 // reflect access faults (0x2400,0x2800,0x5300) directly to domain
728 // r16 == isr
729 // r17 == ifa
730 // r19 == reflect number (only pass-thru to dispatch_reflection)
731 // r20 == offset into ivt
732 // r31 == pr
733 GLOBAL_ENTRY(fast_access_reflect)
734 #ifndef FAST_ACCESS_REFLECT // see beginning of file
735 br.spnt.few dispatch_reflection ;;
736 #endif
737 mov r30=cr.ipsr;;
738 mov r29=cr.iip;;
739 extr.u r21=r30,IA64_PSR_BE_BIT,1 ;;
740 cmp.ne p7,p0=r21,r0
741 (p7) br.spnt.few dispatch_reflection ;;
742 extr.u r21=r30,IA64_PSR_PP_BIT,1 ;;
743 cmp.ne p7,p0=r21,r0
744 (p7) br.spnt.few dispatch_reflection ;;
745 extr.u r21=r30,IA64_PSR_CPL0_BIT,2 ;;
746 cmp.eq p7,p0=r21,r0
747 (p7) br.spnt.few dispatch_reflection ;;
748 movl r18=XSI_PSR_IC;;
749 ld4 r21=[r18];;
750 cmp.eq p7,p0=r0,r21
751 (p7) br.spnt.few dispatch_reflection ;;
752 // set shared_mem ifa, FIXME: should we validate it?
753 mov r17=cr.ifa;;
754 adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
755 st8 [r21]=r17 ;;
756 // get rr[ifa] and save to itir in shared memory (extra bits ignored)
757 shr.u r22=r17,61
758 adds r23=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18
759 adds r21=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
760 shladd r22=r22,3,r21;;
761 ld8 r22=[r22];;
762 and r22=~3,r22;;
763 st8 [r23]=r22;;
764 br.cond.sptk.many fast_reflect;;
766 // when we get to here, VHPT_CCHAIN_LOOKUP has failed and everything
767 // is as it was at the time of original miss. We want to preserve that
768 // so if we get a nested fault, we can just branch to page_fault
769 GLOBAL_ENTRY(fast_tlb_miss_reflect)
770 #ifndef FAST_TLB_MISS_REFLECT // see beginning of file
771 br.spnt.few page_fault ;;
772 #endif
773 mov r31=pr
774 mov r30=cr.ipsr
775 mov r29=cr.iip
776 mov r16=cr.isr
777 mov r17=cr.ifa;;
778 // for now, always take slow path for region 0 (e.g. metaphys mode)
779 extr.u r21=r17,61,3;;
780 cmp.eq p7,p0=r0,r21
781 (p7) br.spnt.few page_fault ;;
782 // always take slow path for PL0 (e.g. __copy_from_user)
783 extr.u r21=r30,IA64_PSR_CPL0_BIT,2 ;;
784 cmp.eq p7,p0=r21,r0
785 (p7) br.spnt.few page_fault ;;
786 // slow path if strange ipsr or isr bits set
787 extr.u r21=r30,IA64_PSR_BE_BIT,1 ;;
788 cmp.ne p7,p0=r21,r0
789 (p7) br.spnt.few page_fault ;;
790 extr.u r21=r30,IA64_PSR_PP_BIT,1 ;;
791 cmp.ne p7,p0=r21,r0
792 (p7) br.spnt.few page_fault ;;
793 movl r21=IA64_ISR_IR|IA64_ISR_SP|IA64_ISR_NA ;;
794 and r21=r16,r21;;
795 cmp.ne p7,p0=r0,r21
796 (p7) br.spnt.few page_fault ;;
797 // also take slow path if virtual psr.ic=0
798 movl r18=XSI_PSR_IC;;
799 ld4 r21=[r18];;
800 cmp.eq p7,p0=r0,r21
801 (p7) br.spnt.few page_fault ;;
802 // OK, if we get to here, we are doing a fast vcpu_translate. Need to:
803 // 1) look in the virtual TR's (pinned), if not there
804 // 2) look in the 1-entry TLB (pinned), if not there
805 // 3) check the domain VHPT (NOT pinned, accesses domain memory!)
806 // If we find it in any of these places, we need to effectively do
807 // a hyper_itc_i/d
809 // short-term hack for now, if in region 5-7, take slow path
810 // since all Linux TRs are in region 5 or 7, we need not check TRs
811 extr.u r21=r17,61,3;;
812 cmp.le p7,p0=5,r21
813 (p7) br.spnt.few page_fault ;;
814 fast_tlb_no_tr_match:
815 movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
816 ld8 r27=[r27];;
817 tbit.nz p6,p7=r16,IA64_ISR_X_BIT;;
818 (p6) adds r25=IA64_VCPU_ITLB_OFFSET,r27;;
819 (p7) adds r25=IA64_VCPU_DTLB_OFFSET,r27;;
820 ld8 r20=[r25],8;;
821 tbit.z p7,p0=r20,0;; // present?
822 (p7) br.cond.spnt.few 1f;;
823 // if ifa is in range of tlb, don't bother to check rid, go slow path
824 ld8 r21=[r25],8;;
825 mov r23=1
826 extr.u r21=r21,2,6;;
827 shl r22=r23,r21
828 ld8 r21=[r25],8;;
829 cmp.ltu p7,p0=r17,r21
830 (p7) br.cond.sptk.many 1f;
831 add r21=r22,r21;;
832 cmp.ltu p7,p0=r17,r21
833 (p7) br.cond.spnt.few page_fault;;
835 1: // check the guest VHPT
836 adds r19 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18;;
837 ld8 r19=[r19];;
838 tbit.nz p7,p0=r19,IA64_PTA_VF_BIT;; // long format VHPT
839 (p7) br.cond.spnt.few page_fault;;
840 // if (!rr.ve || !(pta & IA64_PTA_VE)) take slow way for now
841 // FIXME: later, we deliver an alt_d/i vector after thash and itir
842 tbit.z p7,p0=r19,IA64_PTA_VE_BIT;; //
843 (p7) br.cond.spnt.few page_fault;;
844 extr.u r25=r17,61,3;;
845 adds r21=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
846 shl r25=r25,3;;
847 add r21=r21,r25;;
848 ld8 r22=[r21];;
849 tbit.z p7,p0=r22,0
850 (p7) br.cond.spnt.few page_fault;;
852 // compute and save away itir (r22 & RR_PS_MASK)
853 movl r21=0xfc;;
854 and r22=r22,r21;;
855 adds r21=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
856 st8 [r21]=r22;;
858 // save away ifa
859 adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
860 st8 [r21]=r17;;
861 // see vcpu_thash to save away iha
862 shr.u r20 = r17, 61
863 addl r25 = 1, r0
864 movl r30 = 0xe000000000000000
865 ;;
866 and r21 = r30, r17 // VHPT_Addr1
867 ;;
868 shladd r28 = r20, 3, r18
869 adds r19 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18
870 ;;
871 adds r27 = XSI_RR0_OFS-XSI_PSR_IC_OFS, r28
872 addl r28 = 32767, r0
873 ld8 r24 = [r19] // pta
874 ;;
875 ld8 r23 = [r27] // rrs[vadr>>61]
876 extr.u r26 = r24, 2, 6
877 ;;
878 extr.u r22 = r23, 2, 6
879 shl r30 = r25, r26
880 ;;
881 shr.u r19 = r17, r22
882 shr.u r29 = r24, 15
883 ;;
884 adds r30 = -1, r30
885 ;;
886 shladd r27 = r19, 3, r0
887 extr.u r26 = r30, 15, 46
888 ;;
889 andcm r24 = r29, r26
890 and r19 = r28, r27
891 shr.u r25 = r27, 15
892 ;;
893 and r23 = r26, r25
894 ;;
895 or r22 = r24, r23
896 ;;
897 dep.z r20 = r22, 15, 46
898 ;;
899 or r30 = r20, r21
900 ;;
901 //or r8 = r19, r30
902 or r19 = r19, r30
903 ;;
904 adds r23=XSI_IHA_OFS-XSI_PSR_IC_OFS,r18 ;;
905 st8 [r23]=r19;;
906 // done with thash, check guest VHPT
908 adds r20 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18;;
909 ld8 r24 = [r20];; // pta
910 // avoid recursively walking the VHPT
911 // if (((r17=address ^ r24=pta) & ((itir_mask(pta) << 3) >> 3)) != 0) {
912 mov r20=-8
913 xor r21=r17,r24
914 extr.u r24=r24,2,6;;
915 shl r20=r20,r24;;
916 shr.u r20=r20,3;;
917 and r21=r20,r21;;
918 cmp.eq p7,p0=r21,r0
919 (p7) br.cond.spnt.few 1f;;
920 // __copy_from_user(&pte, r19=(void *)(*iha), sizeof(pte)=8)
921 // prepare for possible nested dtlb fault
922 mov r29=b0
923 movl r30=guest_vhpt_miss;;
924 // now go fetch the entry from the guest VHPT
925 ld8 r20=[r19];;
926 // if we wind up here, we successfully loaded the VHPT entry
928 // this VHPT walker aborts on non-present pages instead
929 // of inserting a not-present translation, this allows
930 // vectoring directly to the miss handler
931 tbit.z p7,p0=r20,0
932 (p7) br.cond.spnt.few page_not_present;;
934 #ifdef FAST_REFLECT_CNT
935 movl r21=fast_vhpt_translate_count;;
936 ld8 r22=[r21];;
937 adds r22=1,r22;;
938 st8 [r21]=r22;;
939 #endif
941 // prepare for fast_insert(PSCB(ifa),PSCB(itir),r16=pte)
942 // r16 == pte
943 // r17 == bit0: 1=inst, 0=data; bit1: 1=itc, 0=vcpu_translate
944 // r18 == XSI_PSR_IC_OFS
945 // r24 == ps
946 // r29 == saved value of b0 in case of recovery
947 // r30 == recovery ip if failure occurs
948 // r31 == pr
949 tbit.nz p6,p7=r16,IA64_ISR_X_BIT;;
950 (p6) mov r17=1;;
951 (p7) mov r17=0;;
952 mov r16=r20
953 mov r29=b0 ;;
954 movl r30=recover_and_page_fault ;;
955 adds r21=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
956 ld8 r24=[r21];;
957 extr.u r24=r24,2,6;;
958 // IFA already in PSCB
959 br.cond.sptk.many fast_insert;;
961 // we get here if fast_insert fails (e.g. due to metaphysical lookup)
962 ENTRY(recover_and_page_fault)
963 #ifdef FAST_REFLECT_CNT
964 movl r21=recover_to_page_fault_count;;
965 ld8 r22=[r21];;
966 adds r22=1,r22;;
967 st8 [r21]=r22;;
968 #endif
969 mov b0=r29;;
970 br.cond.sptk.many page_fault;;
972 // if we wind up here, we missed in guest VHPT so recover
973 // from nested dtlb fault and reflect a tlb fault to the guest
974 guest_vhpt_miss:
975 mov b0=r29;;
976 // fault = IA64_VHPT_FAULT
977 mov r20=r0
978 br.cond.sptk.many 1f;
980 // if we get to here, we are ready to reflect
981 // need to set up virtual ifa, iha, itir (fast_reflect handles
982 // virtual isr, iip, ipsr, ifs
983 // see vcpu_get_itir_on_fault: get ps,rid,(FIXME key) from rr[ifa]
984 page_not_present:
985 tbit.nz p6,p7=r16,IA64_ISR_X_BIT;;
986 (p6) movl r20=0x400;;
987 (p7) movl r20=0x800;;
989 1: extr.u r25=r17,61,3;;
990 adds r21=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
991 shl r25=r25,3;;
992 add r21=r21,r25;;
993 ld8 r22=[r21];;
994 extr.u r22=r22,2,30;;
995 dep.z r22=r22,2,30;;
996 adds r23=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
997 st8 [r23]=r22;;
999 // fast reflect expects
1000 // r16 == cr.isr
1001 // r18 == XSI_PSR_IC
1002 // r20 == offset into ivt
1003 // r29 == iip
1004 // r30 == ipsr
1005 // r31 == pr
1006 //mov r16=cr.isr
1007 mov r29=cr.iip
1008 mov r30=cr.ipsr
1009 br.sptk.many fast_reflect;;
1010 END(fast_tlb_miss_reflect)
1012 // ensure that, if giving up, registers at entry to fast_hyperprivop unchanged
1013 ENTRY(hyper_rfi)
1014 #ifndef FAST_RFI
1015 br.spnt.few dispatch_break_fault ;;
1016 #endif
1017 // if no interrupts pending, proceed
1018 mov r30=r0
1019 cmp.eq p7,p0=r20,r0
1020 (p7) br.sptk.many 1f
1021 ;;
1022 adds r20=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18 ;;
1023 ld8 r21=[r20];; // r21 = vcr.ipsr
1024 extr.u r22=r21,IA64_PSR_I_BIT,1 ;;
1025 mov r30=r22
1026 // r30 determines whether we might deliver an immediate extint
1027 1:
1028 adds r20=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18 ;;
1029 ld8 r21=[r20];; // r21 = vcr.ipsr
1030 extr.u r22=r21,IA64_PSR_BE_BIT,1 ;;
1031 // if turning on psr.be, give up for now and do it the slow way
1032 cmp.ne p7,p0=r22,r0
1033 (p7) br.spnt.few dispatch_break_fault ;;
1034 // if (!(vpsr.dt && vpsr.rt && vpsr.it)), do it the slow way
1035 movl r20=(IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_IT);;
1036 and r22=r20,r21
1037 ;;
1038 cmp.ne p7,p0=r22,r20
1039 (p7) br.spnt.few dispatch_break_fault ;;
1040 // if was in metaphys mode, do it the slow way (FIXME later?)
1041 adds r20=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18 ;;
1042 ld4 r20=[r20];;
1043 cmp.ne p7,p0=r20,r0
1044 (p7) br.spnt.few dispatch_break_fault ;;
1045 // if domain hasn't already done virtual bank switch
1046 // do it the slow way (FIXME later?)
1047 #if 0
1048 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
1049 ld4 r20=[r20];;
1050 cmp.eq p7,p0=r20,r0
1051 (p7) br.spnt.few dispatch_break_fault ;;
1052 #endif
1053 // validate vcr.iip, if in Xen range, do it the slow way
1054 adds r20=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
1055 ld8 r22=[r20];;
1056 movl r23=XEN_VIRT_SPACE_LOW
1057 movl r24=XEN_VIRT_SPACE_HIGH ;;
1058 cmp.ltu p0,p7=r22,r23 ;; // if !(iip<low) &&
1059 (p7) cmp.geu p0,p7=r22,r24 ;; // !(iip>=high)
1060 (p7) br.spnt.few dispatch_break_fault ;;
1061 #ifndef RFI_TO_INTERRUPT // see beginning of file
1062 cmp.ne p6,p0=r30,r0
1063 (p6) br.cond.spnt.few dispatch_break_fault ;;
1064 #endif
1066 1: // OK now, let's do an rfi.
1067 #ifdef FAST_HYPERPRIVOP_CNT
1068 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_RFI);;
1069 ld8 r23=[r20];;
1070 adds r23=1,r23;;
1071 st8 [r20]=r23;;
1072 #endif
1073 #ifdef RFI_TO_INTERRUPT
1074 // maybe do an immediate interrupt delivery?
1075 cmp.ne p6,p0=r30,r0
1076 (p6) br.cond.spnt.few rfi_check_extint;;
1077 #endif
1079 just_do_rfi:
1080 // r18=&vpsr.i|vpsr.ic, r21==vpsr, r22=vcr.iip
1081 mov cr.iip=r22;;
1082 adds r20=XSI_INCOMPL_REGFR_OFS-XSI_PSR_IC_OFS,r18 ;;
1083 st4 [r20]=r0 ;;
1084 adds r20=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1085 ld8 r20=[r20];;
1086 dep r20=0,r20,38,25;; // ensure ifs has no reserved bits set
1087 mov cr.ifs=r20 ;;
1088 // ipsr.cpl == (vcr.ipsr.cpl == 0) 2 : 3;
1089 adds r20=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r18
1090 dep r21=-1,r21,IA64_PSR_CPL1_BIT,1 ;;
1091 // vpsr.i = vcr.ipsr.i; vpsr.ic = vcr.ipsr.ic
1092 ld8 r20=[r20]
1093 mov r19=1
1094 extr.u r23=r21,IA64_PSR_I_BIT,1 ;;
1095 cmp.ne p7,p6=r23,r0 ;;
1096 // not done yet
1097 (p7) st1 [r20]=r0
1098 (p6) st1 [r20]=r19;;
1099 extr.u r23=r21,IA64_PSR_IC_BIT,1 ;;
1100 cmp.ne p7,p6=r23,r0 ;;
1101 (p7) st4 [r18]=r19;;
1102 (p6) st4 [r18]=r0;;
1103 // force on psr.ic, i, dt, rt, it, bn
1104 movl r20=(IA64_PSR_I|IA64_PSR_IC|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_IT|IA64_PSR_BN)
1105 ;;
1106 or r21=r21,r20
1107 ;;
1108 mov cr.ipsr=r21
1109 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
1110 ld4 r21=[r20];;
1111 cmp.ne p7,p0=r21,r0 // domain already did "bank 1 switch?"
1112 (p7) br.cond.spnt.few 1f;
1113 // OK, now all set to go except for switch to virtual bank1
1114 mov r22=1;; st4 [r20]=r22;
1115 mov r30=r2; mov r29=r3;;
1116 mov r17=ar.unat;;
1117 adds r16=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18
1118 adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18;
1119 adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18;;
1120 ld8 r16=[r16];;
1121 mov ar.unat=r16;;
1122 bsw.1;;
1123 // FIXME?: ar.unat is not really handled correctly,
1124 // but may not matter if the OS is NaT-clean
1125 .mem.offset 0,0; ld8.fill r16=[r2],16 ;
1126 .mem.offset 8,0; ld8.fill r17=[r3],16 ;;
1127 .mem.offset 0,0; ld8.fill r18=[r2],16 ;
1128 .mem.offset 0,0; ld8.fill r19=[r3],16 ;;
1129 .mem.offset 8,0; ld8.fill r20=[r2],16 ;
1130 .mem.offset 8,0; ld8.fill r21=[r3],16 ;;
1131 .mem.offset 8,0; ld8.fill r22=[r2],16 ;
1132 .mem.offset 8,0; ld8.fill r23=[r3],16 ;;
1133 .mem.offset 8,0; ld8.fill r24=[r2],16 ;
1134 .mem.offset 8,0; ld8.fill r25=[r3],16 ;;
1135 .mem.offset 8,0; ld8.fill r26=[r2],16 ;
1136 .mem.offset 8,0; ld8.fill r27=[r3],16 ;;
1137 .mem.offset 8,0; ld8.fill r28=[r2],16 ;
1138 .mem.offset 8,0; ld8.fill r29=[r3],16 ;;
1139 .mem.offset 8,0; ld8.fill r30=[r2],16 ;
1140 .mem.offset 8,0; ld8.fill r31=[r3],16 ;;
1141 bsw.0 ;;
1142 mov ar.unat=r17;;
1143 mov r2=r30; mov r3=r29;;
1144 1: mov pr=r31,-1
1145 ;;
1146 rfi
1147 ;;
1149 #ifdef RFI_TO_INTERRUPT
1150 GLOBAL_ENTRY(rfi_check_extint)
1151 //br.sptk.many dispatch_break_fault ;;
1153 // r18=&vpsr.i|vpsr.ic, r21==vpsr, r22=vcr.iip
1154 // make sure none of these get trashed in case going to just_do_rfi
1155 movl r30=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1156 ld8 r30=[r30];;
1157 adds r24=IA64_VCPU_INSVC3_OFFSET,r30;;
1158 mov r25=192
1159 adds r16=IA64_VCPU_IRR3_OFFSET,r30;;
1160 ld8 r23=[r16];;
1161 cmp.eq p6,p0=r23,r0;;
1162 (p6) adds r16=-8,r16;;
1163 (p6) adds r24=-8,r24;;
1164 (p6) adds r25=-64,r25;;
1165 (p6) ld8 r23=[r16];;
1166 (p6) cmp.eq p6,p0=r23,r0;;
1167 (p6) adds r16=-8,r16;;
1168 (p6) adds r24=-8,r24;;
1169 (p6) adds r25=-64,r25;;
1170 (p6) ld8 r23=[r16];;
1171 (p6) cmp.eq p6,p0=r23,r0;;
1172 (p6) adds r16=-8,r16;;
1173 (p6) adds r24=-8,r24;;
1174 (p6) adds r25=-64,r25;;
1175 (p6) ld8 r23=[r16];;
1176 (p6) cmp.eq p6,p0=r23,r0;;
1177 cmp.eq p6,p0=r23,r0
1178 (p6) br.cond.spnt.few just_do_rfi; // this is actually an error
1179 // r16 points to non-zero element of irr, r23 has value
1180 // r24 points to corr element of insvc, r25 has elt*64
1181 ld8 r26=[r24];;
1182 cmp.geu p6,p0=r26,r23
1183 (p6) br.cond.spnt.many just_do_rfi;
1185 // not masked by insvc, get vector number
1186 shr.u r26=r23,1;;
1187 or r26=r23,r26;;
1188 shr.u r27=r26,2;;
1189 or r26=r26,r27;;
1190 shr.u r27=r26,4;;
1191 or r26=r26,r27;;
1192 shr.u r27=r26,8;;
1193 or r26=r26,r27;;
1194 shr.u r27=r26,16;;
1195 or r26=r26,r27;;
1196 shr.u r27=r26,32;;
1197 or r26=r26,r27;;
1198 andcm r26=0xffffffffffffffff,r26;;
1199 popcnt r26=r26;;
1200 sub r26=63,r26;;
1201 // r26 now contains the bit index (mod 64)
1202 mov r27=1;;
1203 shl r27=r27,r26;;
1204 // r27 now contains the (within the proper word) bit mask
1205 add r26=r25,r26
1206 // r26 now contains the vector [0..255]
1207 adds r20=XSI_TPR_OFS-XSI_PSR_IC_OFS,r18 ;;
1208 ld8 r20=[r20] ;;
1209 extr.u r28=r20,16,1
1210 extr.u r29=r20,4,4 ;;
1211 cmp.ne p6,p0=r28,r0 // if tpr.mmi is set, just rfi
1212 (p6) br.cond.spnt.few just_do_rfi;;
1213 shl r29=r29,4;;
1214 adds r29=15,r29;;
1215 cmp.ge p6,p0=r29,r26 // if tpr masks interrupt, just rfi
1216 (p6) br.cond.spnt.few just_do_rfi;;
1218 // this doesn't work yet (dies early after getting to user mode)
1219 // but happens relatively infrequently, so fix it later.
1220 // NOTE that these will be counted incorrectly for now (for privcnt output)
1221 GLOBAL_ENTRY(rfi_with_interrupt)
1222 #if 1
1223 br.sptk.many dispatch_break_fault ;;
1224 #endif
1226 // OK, have an unmasked vector, so deliver extint to vcr.iva+0x3000
1227 // r18 == XSI_PSR_IC
1228 // r21 == vipsr (ipsr in shared_mem)
1229 // r30 == IA64_KR(CURRENT)
1230 // r31 == pr
1231 mov r17=cr.ipsr;;
1232 mov r16=cr.isr;;
1233 // set shared_mem isr
1234 extr.u r16=r16,38,1;; // grab cr.isr.ir bit
1235 dep r16=r16,r0,38,1 ;; // insert into cr.isr (rest of bits zero)
1236 extr.u r20=r21,41,2 ;; // get v(!)psr.ri
1237 dep r16=r20,r16,41,2 ;; // deposit cr.isr.ei
1238 adds r22=XSI_ISR_OFS-XSI_PSR_IC_OFS,r18 ;;
1239 st8 [r22]=r16,XSI_PSR_I_ADDR_OFS-XSI_ISR_OFS ;;
1240 // set cr.ipsr (make sure cpl==2!)
1241 mov r29=r17 ;;
1242 movl r28=DELIVER_PSR_SET;;
1243 mov r20=1
1244 ld8 r22=[r22]
1245 movl r27=~(DELIVER_PSR_CLR|IA64_PSR_CPL0);;
1246 or r29=r29,r28;;
1247 and r29=r29,r27;;
1248 mov cr.ipsr=r29;;
1249 // v.ipsr and v.iip are already set (and v.iip validated) as rfi target
1250 // set shared_mem interrupt_delivery_enabled to 0
1251 // set shared_mem interrupt_collection_enabled to 0
1252 st1 [r22]=r20
1253 st4 [r18]=r0;;
1254 // cover and set shared_mem precover_ifs to cr.ifs
1255 // set shared_mem ifs and incomplete_regframe to 0
1256 #if 0
1257 cover ;;
1258 mov r20=cr.ifs;;
1259 adds r22=XSI_INCOMPL_REG_OFS-XSI_PSR_IC_OFS,r18 ;;
1260 st4 [r22]=r0 ;;
1261 adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1262 st8 [r22]=r0 ;;
1263 adds r22=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1264 st8 [r22]=r20 ;;
1265 // leave cr.ifs alone for later rfi
1266 #else
1267 adds r22=XSI_INCOMPL_REG_OFS-XSI_PSR_IC_OFS,r18 ;;
1268 st4 [r22]=r0 ;;
1269 adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1270 ld8 r20=[r22];;
1271 st8 [r22]=r0 ;;
1272 adds r22=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1273 st8 [r22]=r20 ;;
1274 #endif
1275 // set iip to go to domain IVA break instruction vector
1276 adds r22=IA64_VCPU_IVA_OFFSET,r30;;
1277 ld8 r23=[r22];;
1278 movl r24=0x3000;;
1279 add r24=r24,r23;;
1280 mov cr.iip=r24;;
1281 #if 0
1282 // OK, now all set to go except for switch to virtual bank0
1283 mov r30=r2; mov r29=r3;;
1284 adds r2=XSI_BANK1_OFS-XSI_PSR_IC_OFS,r18;
1285 adds r3=(XSI_BANK1_OFS+8)-XSI_PSR_IC_OFS,r18;;
1286 bsw.1;;
1287 // FIXME: need to handle ar.unat!
1288 .mem.offset 0,0; st8.spill [r2]=r16,16;
1289 .mem.offset 8,0; st8.spill [r3]=r17,16 ;;
1290 .mem.offset 0,0; st8.spill [r2]=r18,16;
1291 .mem.offset 8,0; st8.spill [r3]=r19,16 ;;
1292 .mem.offset 0,0; st8.spill [r2]=r20,16;
1293 .mem.offset 8,0; st8.spill [r3]=r21,16 ;;
1294 .mem.offset 0,0; st8.spill [r2]=r22,16;
1295 .mem.offset 8,0; st8.spill [r3]=r23,16 ;;
1296 .mem.offset 0,0; st8.spill [r2]=r24,16;
1297 .mem.offset 8,0; st8.spill [r3]=r25,16 ;;
1298 .mem.offset 0,0; st8.spill [r2]=r26,16;
1299 .mem.offset 8,0; st8.spill [r3]=r27,16 ;;
1300 .mem.offset 0,0; st8.spill [r2]=r28,16;
1301 .mem.offset 8,0; st8.spill [r3]=r29,16 ;;
1302 .mem.offset 0,0; st8.spill [r2]=r30,16;
1303 .mem.offset 8,0; st8.spill [r3]=r31,16 ;;
1304 movl r31=XSI_IPSR;;
1305 bsw.0 ;;
1306 mov r2=r30; mov r3=r29;;
1307 #else
1308 bsw.1;;
1309 movl r31=XSI_IPSR;;
1310 bsw.0 ;;
1311 #endif
1312 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
1313 st4 [r20]=r0 ;;
1314 mov pr=r31,-1 ;;
1315 rfi
1316 #endif // RFI_TO_INTERRUPT
1318 ENTRY(hyper_cover)
1319 #ifdef FAST_HYPERPRIVOP_CNT
1320 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_COVER);;
1321 ld8 r21=[r20];;
1322 adds r21=1,r21;;
1323 st8 [r20]=r21;;
1324 #endif
1325 mov r24=cr.ipsr
1326 mov r25=cr.iip;;
1327 // skip test for vpsr.ic.. it's a prerequisite for hyperprivops
1328 cover ;;
1329 adds r20=XSI_INCOMPL_REGFR_OFS-XSI_PSR_IC_OFS,r18 ;;
1330 mov r30=cr.ifs;;
1331 adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18
1332 ld4 r21=[r20] ;;
1333 cmp.eq p6,p7=r21,r0 ;;
1334 (p6) st8 [r22]=r30;;
1335 (p7) st4 [r20]=r0;;
1336 mov cr.ifs=r0;;
1337 // adjust return address to skip over break instruction
1338 extr.u r26=r24,41,2 ;;
1339 cmp.eq p6,p7=2,r26 ;;
1340 (p6) mov r26=0
1341 (p6) adds r25=16,r25
1342 (p7) adds r26=1,r26
1343 ;;
1344 dep r24=r26,r24,41,2
1345 ;;
1346 mov cr.ipsr=r24
1347 mov cr.iip=r25
1348 mov pr=r31,-1 ;;
1349 rfi
1350 ;;
1352 // return from metaphysical mode (meta=1) to virtual mode (meta=0)
1353 ENTRY(hyper_ssm_dt)
1354 #ifdef FAST_HYPERPRIVOP_CNT
1355 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_SSM_DT);;
1356 ld8 r21=[r20];;
1357 adds r21=1,r21;;
1358 st8 [r20]=r21;;
1359 #endif
1360 mov r24=cr.ipsr
1361 mov r25=cr.iip;;
1362 adds r20=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18 ;;
1363 ld4 r21=[r20];;
1364 cmp.eq p7,p0=r21,r0 // meta==0?
1365 (p7) br.spnt.many 1f ;; // already in virtual mode
1366 movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1367 ld8 r22=[r22];;
1368 adds r22=IA64_VCPU_META_SAVED_RR0_OFFSET,r22;;
1369 ld8 r23=[r22];;
1370 mov rr[r0]=r23;;
1371 srlz.i;;
1372 st4 [r20]=r0 ;;
1373 // adjust return address to skip over break instruction
1374 1: extr.u r26=r24,41,2 ;;
1375 cmp.eq p6,p7=2,r26 ;;
1376 (p6) mov r26=0
1377 (p6) adds r25=16,r25
1378 (p7) adds r26=1,r26
1379 ;;
1380 dep r24=r26,r24,41,2
1381 ;;
1382 mov cr.ipsr=r24
1383 mov cr.iip=r25
1384 mov pr=r31,-1 ;;
1385 rfi
1386 ;;
1388 // go to metaphysical mode (meta=1) from virtual mode (meta=0)
1389 ENTRY(hyper_rsm_dt)
1390 #ifdef FAST_HYPERPRIVOP_CNT
1391 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_RSM_DT);;
1392 ld8 r21=[r20];;
1393 adds r21=1,r21;;
1394 st8 [r20]=r21;;
1395 #endif
1396 mov r24=cr.ipsr
1397 mov r25=cr.iip;;
1398 adds r20=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18 ;;
1399 ld4 r21=[r20];;
1400 cmp.ne p7,p0=r21,r0 // meta==0?
1401 (p7) br.spnt.many 1f ;; // already in metaphysical mode
1402 movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1403 ld8 r22=[r22];;
1404 adds r22=IA64_VCPU_META_RR0_OFFSET,r22;;
1405 ld8 r23=[r22];;
1406 mov rr[r0]=r23;;
1407 srlz.i;;
1408 adds r21=1,r0 ;;
1409 st4 [r20]=r21 ;;
1410 // adjust return address to skip over break instruction
1411 1: extr.u r26=r24,41,2 ;;
1412 cmp.eq p6,p7=2,r26 ;;
1413 (p6) mov r26=0
1414 (p6) adds r25=16,r25
1415 (p7) adds r26=1,r26
1416 ;;
1417 dep r24=r26,r24,41,2
1418 ;;
1419 mov cr.ipsr=r24
1420 mov cr.iip=r25
1421 mov pr=r31,-1 ;;
1422 rfi
1423 ;;
1425 ENTRY(hyper_get_tpr)
1426 #ifdef FAST_HYPERPRIVOP_CNT
1427 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_GET_TPR);;
1428 ld8 r21=[r20];;
1429 adds r21=1,r21;;
1430 st8 [r20]=r21;;
1431 #endif
1432 mov r24=cr.ipsr
1433 mov r25=cr.iip;;
1434 adds r20=XSI_TPR_OFS-XSI_PSR_IC_OFS,r18 ;;
1435 ld8 r8=[r20];;
1436 extr.u r26=r24,41,2 ;;
1437 cmp.eq p6,p7=2,r26 ;;
1438 (p6) mov r26=0
1439 (p6) adds r25=16,r25
1440 (p7) adds r26=1,r26
1441 ;;
1442 dep r24=r26,r24,41,2
1443 ;;
1444 mov cr.ipsr=r24
1445 mov cr.iip=r25
1446 mov pr=r31,-1 ;;
1447 rfi
1448 ;;
1449 END(hyper_get_tpr)
1451 // if we get to here, there are no interrupts pending so we
1452 // can change virtual tpr to any value without fear of provoking
1453 // (or accidentally missing) delivering an interrupt
1454 ENTRY(hyper_set_tpr)
1455 #ifdef FAST_HYPERPRIVOP_CNT
1456 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_SET_TPR);;
1457 ld8 r21=[r20];;
1458 adds r21=1,r21;;
1459 st8 [r20]=r21;;
1460 #endif
1461 mov r24=cr.ipsr
1462 mov r25=cr.iip;;
1463 movl r27=0xff00;;
1464 adds r20=XSI_TPR_OFS-XSI_PSR_IC_OFS,r18 ;;
1465 andcm r8=r8,r27;;
1466 st8 [r20]=r8;;
1467 extr.u r26=r24,41,2 ;;
1468 cmp.eq p6,p7=2,r26 ;;
1469 (p6) mov r26=0
1470 (p6) adds r25=16,r25
1471 (p7) adds r26=1,r26
1472 ;;
1473 dep r24=r26,r24,41,2
1474 ;;
1475 mov cr.ipsr=r24
1476 mov cr.iip=r25
1477 mov pr=r31,-1 ;;
1478 rfi
1479 ;;
1480 END(hyper_set_tpr)
1482 ENTRY(hyper_get_ivr)
1483 #ifdef FAST_HYPERPRIVOP_CNT
1484 movl r22=fast_hyperpriv_cnt+(8*XEN_HYPER_GET_IVR);;
1485 ld8 r21=[r22];;
1486 adds r21=1,r21;;
1487 st8 [r22]=r21;;
1488 #endif
1489 mov r8=15;;
1490 // when we get to here r20=~=interrupts pending
1491 cmp.eq p7,p0=r20,r0;;
1492 (p7) adds r20=XSI_PEND_OFS-XSI_PSR_IC_OFS,r18 ;;
1493 (p7) st4 [r20]=r0;;
1494 (p7) br.spnt.many 1f ;;
1495 movl r30=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1496 ld8 r30=[r30];;
1497 adds r24=IA64_VCPU_INSVC3_OFFSET,r30;;
1498 mov r25=192
1499 adds r22=IA64_VCPU_IRR3_OFFSET,r30;;
1500 ld8 r23=[r22];;
1501 cmp.eq p6,p0=r23,r0;;
1502 (p6) adds r22=-8,r22;;
1503 (p6) adds r24=-8,r24;;
1504 (p6) adds r25=-64,r25;;
1505 (p6) ld8 r23=[r22];;
1506 (p6) cmp.eq p6,p0=r23,r0;;
1507 (p6) adds r22=-8,r22;;
1508 (p6) adds r24=-8,r24;;
1509 (p6) adds r25=-64,r25;;
1510 (p6) ld8 r23=[r22];;
1511 (p6) cmp.eq p6,p0=r23,r0;;
1512 (p6) adds r22=-8,r22;;
1513 (p6) adds r24=-8,r24;;
1514 (p6) adds r25=-64,r25;;
1515 (p6) ld8 r23=[r22];;
1516 (p6) cmp.eq p6,p0=r23,r0;;
1517 cmp.eq p6,p0=r23,r0
1518 (p6) br.cond.spnt.few 1f; // this is actually an error
1519 // r22 points to non-zero element of irr, r23 has value
1520 // r24 points to corr element of insvc, r25 has elt*64
1521 ld8 r26=[r24];;
1522 cmp.geu p6,p0=r26,r23
1523 (p6) br.cond.spnt.many 1f;
1524 // not masked by insvc, get vector number
1525 shr.u r26=r23,1;;
1526 or r26=r23,r26;;
1527 shr.u r27=r26,2;;
1528 or r26=r26,r27;;
1529 shr.u r27=r26,4;;
1530 or r26=r26,r27;;
1531 shr.u r27=r26,8;;
1532 or r26=r26,r27;;
1533 shr.u r27=r26,16;;
1534 or r26=r26,r27;;
1535 shr.u r27=r26,32;;
1536 or r26=r26,r27;;
1537 andcm r26=0xffffffffffffffff,r26;;
1538 popcnt r26=r26;;
1539 sub r26=63,r26;;
1540 // r26 now contains the bit index (mod 64)
1541 mov r27=1;;
1542 shl r27=r27,r26;;
1543 // r27 now contains the (within the proper word) bit mask
1544 add r26=r25,r26
1545 // r26 now contains the vector [0..255]
1546 adds r20=XSI_TPR_OFS-XSI_PSR_IC_OFS,r18 ;;
1547 ld8 r20=[r20] ;;
1548 extr.u r28=r20,16,1
1549 extr.u r29=r20,4,4 ;;
1550 cmp.ne p6,p0=r28,r0 // if tpr.mmi is set, return SPURIOUS
1551 (p6) br.cond.spnt.few 1f;
1552 shl r29=r29,4;;
1553 adds r29=15,r29;;
1554 cmp.ge p6,p0=r29,r26
1555 (p6) br.cond.spnt.few 1f;
1556 // OK, have an unmasked vector to process/return
1557 ld8 r25=[r24];;
1558 or r25=r25,r27;;
1559 st8 [r24]=r25;;
1560 ld8 r25=[r22];;
1561 andcm r25=r25,r27;;
1562 st8 [r22]=r25;;
1563 mov r8=r26;;
1564 // if its a clock tick, remember itm to avoid delivering it twice
1565 adds r20=XSI_ITV_OFS-XSI_PSR_IC_OFS,r18 ;;
1566 ld8 r20=[r20];;
1567 extr.u r20=r20,0,8;;
1568 cmp.eq p6,p0=r20,r8
1569 adds r22=IA64_VCPU_DOMAIN_ITM_LAST_OFFSET,r30
1570 adds r23=IA64_VCPU_DOMAIN_ITM_OFFSET,r30;;
1571 ld8 r23=[r23];;
1572 (p6) st8 [r22]=r23;;
1573 // all done
1574 1: mov r24=cr.ipsr
1575 mov r25=cr.iip;;
1576 extr.u r26=r24,41,2 ;;
1577 cmp.eq p6,p7=2,r26 ;;
1578 (p6) mov r26=0
1579 (p6) adds r25=16,r25
1580 (p7) adds r26=1,r26
1581 ;;
1582 dep r24=r26,r24,41,2
1583 ;;
1584 mov cr.ipsr=r24
1585 mov cr.iip=r25
1586 mov pr=r31,-1 ;;
1587 rfi
1588 ;;
1589 END(hyper_get_ivr)
1591 ENTRY(hyper_eoi)
1592 // when we get to here r20=~=interrupts pending
1593 cmp.ne p7,p0=r20,r0
1594 (p7) br.spnt.many dispatch_break_fault ;;
1595 #ifdef FAST_HYPERPRIVOP_CNT
1596 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_EOI);;
1597 ld8 r21=[r20];;
1598 adds r21=1,r21;;
1599 st8 [r20]=r21;;
1600 #endif
1601 movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1602 ld8 r22=[r22];;
1603 adds r22=IA64_VCPU_INSVC3_OFFSET,r22;;
1604 ld8 r23=[r22];;
1605 cmp.eq p6,p0=r23,r0;;
1606 (p6) adds r22=-8,r22;;
1607 (p6) ld8 r23=[r22];;
1608 (p6) cmp.eq p6,p0=r23,r0;;
1609 (p6) adds r22=-8,r22;;
1610 (p6) ld8 r23=[r22];;
1611 (p6) cmp.eq p6,p0=r23,r0;;
1612 (p6) adds r22=-8,r22;;
1613 (p6) ld8 r23=[r22];;
1614 (p6) cmp.eq p6,p0=r23,r0;;
1615 cmp.eq p6,p0=r23,r0
1616 (p6) br.cond.spnt.few 1f; // this is actually an error
1617 // r22 points to non-zero element of insvc, r23 has value
1618 shr.u r24=r23,1;;
1619 or r24=r23,r24;;
1620 shr.u r25=r24,2;;
1621 or r24=r24,r25;;
1622 shr.u r25=r24,4;;
1623 or r24=r24,r25;;
1624 shr.u r25=r24,8;;
1625 or r24=r24,r25;;
1626 shr.u r25=r24,16;;
1627 or r24=r24,r25;;
1628 shr.u r25=r24,32;;
1629 or r24=r24,r25;;
1630 andcm r24=0xffffffffffffffff,r24;;
1631 popcnt r24=r24;;
1632 sub r24=63,r24;;
1633 // r24 now contains the bit index
1634 mov r25=1;;
1635 shl r25=r25,r24;;
1636 andcm r23=r23,r25;;
1637 st8 [r22]=r23;;
1638 1: mov r24=cr.ipsr
1639 mov r25=cr.iip;;
1640 extr.u r26=r24,41,2 ;;
1641 cmp.eq p6,p7=2,r26 ;;
1642 (p6) mov r26=0
1643 (p6) adds r25=16,r25
1644 (p7) adds r26=1,r26
1645 ;;
1646 dep r24=r26,r24,41,2
1647 ;;
1648 mov cr.ipsr=r24
1649 mov cr.iip=r25
1650 mov pr=r31,-1 ;;
1651 rfi
1652 ;;
1653 END(hyper_eoi)
1655 ENTRY(hyper_set_itm)
1656 // when we get to here r20=~=interrupts pending
1657 cmp.ne p7,p0=r20,r0
1658 (p7) br.spnt.many dispatch_break_fault ;;
1659 #ifdef FAST_HYPERPRIVOP_CNT
1660 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_SET_ITM);;
1661 ld8 r21=[r20];;
1662 adds r21=1,r21;;
1663 st8 [r20]=r21;;
1664 #endif
1665 movl r20=THIS_CPU(cpu_info)+IA64_CPUINFO_ITM_NEXT_OFFSET;;
1666 ld8 r21=[r20];;
1667 movl r20=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1668 ld8 r20=[r20];;
1669 adds r20=IA64_VCPU_DOMAIN_ITM_OFFSET,r20;;
1670 st8 [r20]=r8;;
1671 cmp.geu p6,p0=r21,r8;;
1672 (p6) mov r21=r8;;
1673 // now "safe set" cr.itm=r21
1674 mov r23=100;;
1675 2: mov cr.itm=r21;;
1676 srlz.d;;
1677 mov r22=ar.itc ;;
1678 cmp.leu p6,p0=r21,r22;;
1679 add r21=r21,r23;;
1680 shl r23=r23,1;;
1681 (p6) br.cond.spnt.few 2b;;
1682 1: mov r24=cr.ipsr
1683 mov r25=cr.iip;;
1684 extr.u r26=r24,41,2 ;;
1685 cmp.eq p6,p7=2,r26 ;;
1686 (p6) mov r26=0
1687 (p6) adds r25=16,r25
1688 (p7) adds r26=1,r26
1689 ;;
1690 dep r24=r26,r24,41,2
1691 ;;
1692 mov cr.ipsr=r24
1693 mov cr.iip=r25
1694 mov pr=r31,-1 ;;
1695 rfi
1696 ;;
1697 END(hyper_set_itm)
1699 ENTRY(hyper_get_rr)
1700 #ifdef FAST_HYPERPRIVOP_CNT
1701 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_GET_RR);;
1702 ld8 r21=[r20];;
1703 adds r21=1,r21;;
1704 st8 [r20]=r21;;
1705 #endif
1706 extr.u r25=r8,61,3;;
1707 adds r20=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
1708 shl r25=r25,3;;
1709 add r20=r20,r25;;
1710 ld8 r8=[r20];;
1711 1: mov r24=cr.ipsr
1712 mov r25=cr.iip;;
1713 extr.u r26=r24,41,2 ;;
1714 cmp.eq p6,p7=2,r26 ;;
1715 (p6) mov r26=0
1716 (p6) adds r25=16,r25
1717 (p7) adds r26=1,r26
1718 ;;
1719 dep r24=r26,r24,41,2
1720 ;;
1721 mov cr.ipsr=r24
1722 mov cr.iip=r25
1723 mov pr=r31,-1 ;;
1724 rfi
1725 ;;
1726 END(hyper_get_rr)
1728 ENTRY(hyper_set_rr)
1729 extr.u r25=r8,61,3;;
1730 cmp.leu p7,p0=7,r25 // punt on setting rr7
1731 (p7) br.spnt.many dispatch_break_fault ;;
1732 #ifdef FAST_HYPERPRIVOP_CNT
1733 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_SET_RR);;
1734 ld8 r21=[r20];;
1735 adds r21=1,r21;;
1736 st8 [r20]=r21;;
1737 #endif
1738 extr.u r26=r9,8,24 // r26 = r9.rid
1739 movl r20=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1740 ld8 r20=[r20];;
1741 adds r21=IA64_VCPU_STARTING_RID_OFFSET,r20;;
1742 ld4 r22=[r21];;
1743 adds r21=IA64_VCPU_ENDING_RID_OFFSET,r20;;
1744 ld4 r23=[r21];;
1745 adds r24=IA64_VCPU_META_SAVED_RR0_OFFSET,r20;;
1746 add r22=r26,r22;;
1747 cmp.geu p6,p0=r22,r23 // if r9.rid + starting_rid >= ending_rid
1748 (p6) br.cond.spnt.few 1f; // this is an error, but just ignore/return
1749 // r21=starting_rid
1750 adds r20=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
1751 shl r25=r25,3;;
1752 add r20=r20,r25;;
1753 st8 [r20]=r9;; // store away exactly what was passed
1754 // but adjust value actually placed in rr[r8]
1755 // r22 contains adjusted rid, "mangle" it (see regionreg.c)
1756 // and set ps to PAGE_SHIFT and ve to 1
1757 extr.u r27=r22,0,8
1758 extr.u r28=r22,8,8
1759 extr.u r29=r22,16,8;;
1760 dep.z r23=PAGE_SHIFT,2,6;;
1761 dep r23=-1,r23,0,1;; // mangling is swapping bytes 1 & 3
1762 dep r23=r27,r23,24,8;;
1763 dep r23=r28,r23,16,8;;
1764 dep r23=r29,r23,8,8
1765 cmp.eq p6,p0=r25,r0;; // if rr0, save for metaphysical
1766 (p6) st8 [r24]=r23
1767 mov rr[r8]=r23;;
1768 // done, mosey on back
1769 1: mov r24=cr.ipsr
1770 mov r25=cr.iip;;
1771 extr.u r26=r24,41,2 ;;
1772 cmp.eq p6,p7=2,r26 ;;
1773 (p6) mov r26=0
1774 (p6) adds r25=16,r25
1775 (p7) adds r26=1,r26
1776 ;;
1777 dep r24=r26,r24,41,2
1778 ;;
1779 mov cr.ipsr=r24
1780 mov cr.iip=r25
1781 mov pr=r31,-1 ;;
1782 rfi
1783 ;;
1784 END(hyper_set_rr)
1786 ENTRY(hyper_set_kr)
1787 extr.u r25=r8,3,61;;
1788 cmp.ne p7,p0=r0,r25 // if kr# > 7, go slow way
1789 (p7) br.spnt.many dispatch_break_fault ;;
1790 #ifdef FAST_HYPERPRIVOP_CNT
1791 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_SET_KR);;
1792 ld8 r21=[r20];;
1793 adds r21=1,r21;;
1794 st8 [r20]=r21;;
1795 #endif
1796 adds r21=XSI_KR0_OFS-XSI_PSR_IC_OFS,r18 ;;
1797 shl r20=r8,3;;
1798 add r22=r20,r21;;
1799 st8 [r22]=r9;;
1800 cmp.eq p7,p0=r8,r0
1801 adds r8=-1,r8;;
1802 (p7) mov ar0=r9;;
1803 cmp.eq p7,p0=r8,r0
1804 adds r8=-1,r8;;
1805 (p7) mov ar1=r9;;
1806 cmp.eq p7,p0=r8,r0
1807 adds r8=-1,r8;;
1808 (p7) mov ar2=r9;;
1809 cmp.eq p7,p0=r8,r0
1810 adds r8=-1,r8;;
1811 (p7) mov ar3=r9;;
1812 cmp.eq p7,p0=r8,r0
1813 adds r8=-1,r8;;
1814 (p7) mov ar4=r9;;
1815 cmp.eq p7,p0=r8,r0
1816 adds r8=-1,r8;;
1817 (p7) mov ar5=r9;;
1818 cmp.eq p7,p0=r8,r0
1819 adds r8=-1,r8;;
1820 (p7) mov ar6=r9;;
1821 cmp.eq p7,p0=r8,r0
1822 adds r8=-1,r8;;
1823 (p7) mov ar7=r9;;
1824 // done, mosey on back
1825 1: mov r24=cr.ipsr
1826 mov r25=cr.iip;;
1827 extr.u r26=r24,41,2 ;;
1828 cmp.eq p6,p7=2,r26 ;;
1829 (p6) mov r26=0
1830 (p6) adds r25=16,r25
1831 (p7) adds r26=1,r26
1832 ;;
1833 dep r24=r26,r24,41,2
1834 ;;
1835 mov cr.ipsr=r24
1836 mov cr.iip=r25
1837 mov pr=r31,-1 ;;
1838 rfi
1839 ;;
1840 END(hyper_set_kr)
1842 // this routine was derived from optimized assembly output from
1843 // vcpu_thash so it is dense and difficult to read but it works
1844 // On entry:
1845 // r18 == XSI_PSR_IC
1846 // r31 == pr
1847 GLOBAL_ENTRY(hyper_thash)
1848 #ifdef FAST_HYPERPRIVOP_CNT
1849 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_THASH);;
1850 ld8 r21=[r20];;
1851 adds r21=1,r21;;
1852 st8 [r20]=r21;;
1853 #endif
1854 shr.u r20 = r8, 61
1855 addl r25 = 1, r0
1856 movl r17 = 0xe000000000000000
1857 ;;
1858 and r21 = r17, r8 // VHPT_Addr1
1859 ;;
1860 shladd r28 = r20, 3, r18
1861 adds r19 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18
1862 ;;
1863 adds r27 = XSI_RR0_OFS-XSI_PSR_IC_OFS, r28
1864 addl r28 = 32767, r0
1865 ld8 r24 = [r19] // pta
1866 ;;
1867 ld8 r23 = [r27] // rrs[vadr>>61]
1868 extr.u r26 = r24, 2, 6
1869 ;;
1870 extr.u r22 = r23, 2, 6
1871 shl r30 = r25, r26
1872 ;;
1873 shr.u r19 = r8, r22
1874 shr.u r29 = r24, 15
1875 ;;
1876 adds r17 = -1, r30
1877 ;;
1878 shladd r27 = r19, 3, r0
1879 extr.u r26 = r17, 15, 46
1880 ;;
1881 andcm r24 = r29, r26
1882 and r19 = r28, r27
1883 shr.u r25 = r27, 15
1884 ;;
1885 and r23 = r26, r25
1886 ;;
1887 or r22 = r24, r23
1888 ;;
1889 dep.z r20 = r22, 15, 46
1890 ;;
1891 or r16 = r20, r21
1892 ;;
1893 or r8 = r19, r16
1894 // done, update iip/ipsr to next instruction
1895 mov r24=cr.ipsr
1896 mov r25=cr.iip;;
1897 extr.u r26=r24,41,2 ;;
1898 cmp.eq p6,p7=2,r26 ;;
1899 (p6) mov r26=0
1900 (p6) adds r25=16,r25
1901 (p7) adds r26=1,r26
1902 ;;
1903 dep r24=r26,r24,41,2
1904 ;;
1905 mov cr.ipsr=r24
1906 mov cr.iip=r25
1907 mov pr=r31,-1 ;;
1908 rfi
1909 ;;
1910 END(hyper_thash)
1912 ENTRY(hyper_ptc_ga)
1913 #ifndef FAST_PTC_GA
1914 br.spnt.few dispatch_break_fault ;;
1915 #endif
1916 // FIXME: validate not flushing Xen addresses
1917 #ifdef FAST_HYPERPRIVOP_CNT
1918 movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_PTC_GA);;
1919 ld8 r21=[r20];;
1920 adds r21=1,r21;;
1921 st8 [r20]=r21;;
1922 #endif
1923 mov r28=r8
1924 extr.u r19=r9,2,6 // addr_range=1<<((r9&0xfc)>>2)
1925 mov r20=1
1926 shr.u r24=r8,61
1927 addl r27=56,r0 // PAGE_SHIFT<<2 (for ptc.ga)
1928 movl r26=0x8000000000000000 // INVALID_TI_TAG
1929 mov r30=ar.lc
1930 ;;
1931 shl r19=r20,r19
1932 cmp.eq p7,p0=7,r24
1933 (p7) br.spnt.many dispatch_break_fault ;; // slow way for rr7
1934 ;;
1935 cmp.le p7,p0=r19,r0 // skip flush if size<=0
1936 (p7) br.cond.dpnt 2f ;;
1937 extr.u r24=r19,0,PAGE_SHIFT
1938 shr.u r23=r19,PAGE_SHIFT ;; // repeat loop for n pages
1939 cmp.ne p7,p0=r24,r0 ;;
1940 (p7) adds r23=1,r23 ;; // n_pages<size<n_pages+1? extra iter
1941 mov ar.lc=r23
1942 movl r29=PAGE_SIZE;;
1943 1:
1944 thash r25=r28 ;;
1945 adds r25=16,r25 ;;
1946 ld8 r24=[r25] ;;
1947 // FIXME: should check if tag matches, not just blow it away
1948 or r24=r26,r24 ;; // vhpt_entry->ti_tag = 1
1949 st8 [r25]=r24
1950 ptc.ga r28,r27 ;;
1951 srlz.i ;;
1952 add r28=r29,r28
1953 br.cloop.sptk.few 1b
1954 ;;
1955 2:
1956 mov ar.lc=r30 ;;
1957 mov r29=cr.ipsr
1958 mov r30=cr.iip;;
1959 movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1960 ld8 r27=[r27];;
1961 adds r25=IA64_VCPU_DTLB_OFFSET,r27
1962 adds r26=IA64_VCPU_ITLB_OFFSET,r27;;
1963 ld8 r24=[r25]
1964 ld8 r27=[r26] ;;
1965 and r24=-2,r24
1966 and r27=-2,r27 ;;
1967 st8 [r25]=r24 // set 1-entry i/dtlb as not present
1968 st8 [r26]=r27 ;;
1969 // increment to point to next instruction
1970 extr.u r26=r29,41,2 ;;
1971 cmp.eq p6,p7=2,r26 ;;
1972 (p6) mov r26=0
1973 (p6) adds r30=16,r30
1974 (p7) adds r26=1,r26
1975 ;;
1976 dep r29=r26,r29,41,2
1977 ;;
1978 mov cr.ipsr=r29
1979 mov cr.iip=r30
1980 mov pr=r31,-1 ;;
1981 rfi
1982 ;;
1983 END(hyper_ptc_ga)
1985 // recovery block for hyper_itc metaphysical memory lookup
1986 ENTRY(recover_and_dispatch_break_fault)
1987 #ifdef FAST_REFLECT_CNT
1988 movl r21=recover_to_break_fault_count;;
1989 ld8 r22=[r21];;
1990 adds r22=1,r22;;
1991 st8 [r21]=r22;;
1992 #endif
1993 mov b0=r29 ;;
1994 br.sptk.many dispatch_break_fault;;
1996 // Registers at entry
1997 // r17 = break immediate (XEN_HYPER_ITC_D or I)
1998 // r18 == XSI_PSR_IC_OFS
1999 // r31 == pr
2000 GLOBAL_ENTRY(hyper_itc)
2001 ENTRY(hyper_itc_i)
2002 // fall through, hyper_itc_d handles both i and d
2003 ENTRY(hyper_itc_d)
2004 #ifndef FAST_ITC
2005 br.sptk.many dispatch_break_fault ;;
2006 #endif
2007 // ensure itir.ps >= xen's pagesize
2008 adds r23=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
2009 ld8 r23=[r23];;
2010 extr.u r24=r23,2,6;; // r24==logps
2011 cmp.gt p7,p0=PAGE_SHIFT,r24
2012 (p7) br.spnt.many dispatch_break_fault ;;
2013 adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
2014 ld8 r21=[r21];;
2015 // for now, punt on region0 inserts
2016 extr.u r21=r21,61,3;;
2017 cmp.eq p7,p0=r21,r0
2018 (p7) br.spnt.many dispatch_break_fault ;;
2019 movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
2020 ld8 r27=[r27];;
2021 adds r27=IA64_VCPU_DOMAIN_OFFSET,r27;;
2022 ld8 r27=[r27];;
2023 // FIXME: is the global var dom0 always pinned? assume so for now
2024 movl r28=dom0;;
2025 ld8 r28=[r28];;
2026 // FIXME: for now, only handle dom0 (see lookup_domain_mpa below)
2027 cmp.ne p7,p0=r27,r28
2028 (p7) br.spnt.many dispatch_break_fault ;;
2029 #ifdef FAST_HYPERPRIVOP_CNT
2030 cmp.eq p6,p7=XEN_HYPER_ITC_D,r17;;
2031 (p6) movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_ITC_D);;
2032 (p7) movl r20=fast_hyperpriv_cnt+(8*XEN_HYPER_ITC_I);;
2033 ld8 r21=[r20];;
2034 adds r21=1,r21;;
2035 st8 [r20]=r21;;
2036 #endif
2037 (p6) mov r17=2;;
2038 (p7) mov r17=3;;
2039 mov r29=b0 ;;
2040 movl r30=recover_and_dispatch_break_fault ;;
2041 mov r16=r8;;
2042 // fall through
2045 // fast_insert(PSCB(ifa),r24=ps,r16=pte)
2046 // r16 == pte
2047 // r17 == bit0: 1=inst, 0=data; bit1: 1=itc, 0=vcpu_translate
2048 // r18 == XSI_PSR_IC_OFS
2049 // r24 == ps
2050 // r29 == saved value of b0 in case of recovery
2051 // r30 == recovery ip if failure occurs
2052 // r31 == pr
2053 GLOBAL_ENTRY(fast_insert)
2054 // translate_domain_pte(r16=pteval,PSCB(ifa)=address,r24=itir)
2055 mov r19=1;;
2056 shl r20=r19,r24;;
2057 adds r20=-1,r20;; // r20 == mask
2058 movl r19=_PAGE_PPN_MASK;;
2059 and r22=r16,r19;; // r22 == pteval & _PAGE_PPN_MASK
2060 andcm r19=r22,r20;;
2061 adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
2062 ld8 r21=[r21];;
2063 and r20=r21,r20;;
2064 or r19=r19,r20;; // r19 == mpaddr
2065 // FIXME: for now, just do domain0 and skip mpaddr range checks
2066 dep r20=r0,r19,0,PAGE_SHIFT
2067 movl r21=PAGE_PHYS ;;
2068 or r20=r20,r21 ;; // r20==return value from lookup_domain_mpa
2069 // r16=pteval,r20=pteval2
2070 movl r19=_PAGE_PPN_MASK
2071 movl r21=_PAGE_PL_2;;
2072 andcm r25=r16,r19;; // r25==pteval & ~_PAGE_PPN_MASK
2073 and r22=r20,r19;;
2074 or r22=r22,r21;;
2075 or r22=r22,r25;; // r22==return value from translate_domain_pte
2076 // done with translate_domain_pte
2077 // now do vcpu_itc_no_srlz(vcpu,IorD,ifa,r22=pte,r16=mppte,r24=logps)
2078 // FIXME: for now, just domain0 and skip range check
2079 // psr.ic already cleared
2080 // NOTE: r24 still contains ps (from above)
2081 shladd r24=r24,2,r0;;
2082 mov cr.itir=r24;;
2083 adds r23=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
2084 ld8 r23=[r23];;
2085 mov cr.ifa=r23;;
2086 tbit.z p6,p7=r17,0;;
2087 (p6) itc.d r22;;
2088 (p7) itc.i r22;;
2089 dv_serialize_data
2090 // FIXME: how do I make assembler warnings go away here?
2091 // vhpt_insert(r23=vaddr,r22=pte,r24=logps<<2)
2092 thash r28=r23
2093 or r26=1,r22;;
2094 ttag r21=r23
2095 adds r25=8,r28
2096 mov r19=r28;;
2097 st8 [r25]=r24
2098 adds r20=16,r28;;
2099 st8 [r19]=r26
2100 st8 [r20]=r21;;
2101 // vcpu_set_tr_entry(trp,r22=pte|1,r24=itir,r23=ifa)
2102 // TR_ENTRY = {page_flags,itir,addr,rid}
2103 tbit.z p6,p7=r17,0;;
2104 movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
2105 ld8 r27=[r27];;
2106 adds r28=IA64_VCPU_STARTING_RID_OFFSET,r27
2107 (p6) adds r27=IA64_VCPU_DTLB_OFFSET,r27
2108 (p7) adds r27=IA64_VCPU_ITLB_OFFSET,r27;;
2109 st8 [r27]=r22,8;; // page_flags: already has pl >= 2 and p==1
2110 st8 [r27]=r24,8;; // itir
2111 mov r19=-4096;;
2112 and r23=r23,r19;;
2113 st8 [r27]=r23,8;; // ifa & ~0xfff
2114 adds r29 = XSI_RR0_OFS-XSI_PSR_IC_OFS,r18
2115 extr.u r25=r23,61,3;;
2116 shladd r29=r25,3,r29;;
2117 ld8 r29=[r29];;
2118 movl r20=0xffffff00;;
2119 and r29=r29,r20;;
2120 st8 [r27]=r29,-8;; // rid
2121 //if ps > 12
2122 cmp.eq p7,p0=12<<2,r24
2123 (p7) br.cond.sptk.many 1f;;
2124 // if (ps > 12) {
2125 // trp->ppn &= ~((1UL<<(ps-12))-1); trp->vadr &= ~((1UL<<ps)-1); }
2126 extr.u r29=r24,2,6
2127 mov r28=1;;
2128 shl r26=r28,r29;;
2129 adds r29=-12,r29;;
2130 shl r25=r28,r29;;
2131 mov r29=-1
2132 adds r26=-1,r26
2133 adds r25=-1,r25;;
2134 andcm r26=r29,r26 // ~((1UL<<ps)-1)
2135 andcm r25=r29,r25;; // ~((1UL<<(ps-12))-1)
2136 ld8 r29=[r27];;
2137 and r29=r29,r26;;
2138 st8 [r27]=r29,-16;;
2139 ld8 r29=[r27];;
2140 extr.u r28=r29,12,38;;
2141 movl r26=0xfffc000000000fff;;
2142 and r29=r29,r26
2143 and r28=r28,r25;;
2144 shl r28=r28,12;;
2145 or r29=r29,r28;;
2146 st8 [r27]=r29;;
2147 1: // done with vcpu_set_tr_entry
2148 //PSCBX(vcpu,i/dtlb_pte) = mp_pte
2149 movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
2150 ld8 r27=[r27];;
2151 tbit.z p6,p7=r17,0;;
2152 (p6) adds r27=IA64_VCPU_DTLB_PTE_OFFSET,r27
2153 (p7) adds r27=IA64_VCPU_ITLB_PTE_OFFSET,r27;;
2154 st8 [r27]=r16;;
2155 // done with vcpu_itc_no_srlz
2157 // if hyper_itc, increment to point to next instruction
2158 tbit.z p7,p0=r17,1
2159 (p7) br.cond.sptk.few no_inc_iip;;
2161 mov r29=cr.ipsr
2162 mov r30=cr.iip;;
2163 extr.u r26=r29,41,2 ;;
2164 cmp.eq p6,p7=2,r26 ;;
2165 (p6) mov r26=0
2166 (p6) adds r30=16,r30
2167 (p7) adds r26=1,r26
2168 ;;
2169 dep r29=r26,r29,41,2
2170 ;;
2171 mov cr.ipsr=r29
2172 mov cr.iip=r30;;
2174 no_inc_iip:
2175 mov pr=r31,-1 ;;
2176 rfi
2177 ;;
2178 END(fast_insert)