ia64/xen-unstable

view xen/arch/ia64/xen/hyperprivop.S @ 14798:605d534f9635

[IA64] Fix hyper_get_psr

It didn't return correct psr.i value.

Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
author Alex Williamson <alex.williamson@hp.com>
date Thu Apr 12 10:26:42 2007 -0600 (2007-04-12)
parents 7158623a1b3d
children e7295db88664
line source
1 /*
2 * arch/ia64/kernel/hyperprivop.S
3 *
4 * Copyright (C) 2005 Hewlett-Packard Co
5 * Dan Magenheimer <dan.magenheimer@hp.com>
6 */
8 #include <linux/config.h>
10 #include <asm/asmmacro.h>
11 #include <asm/kregs.h>
12 #include <asm/offsets.h>
13 #include <asm/processor.h>
14 #include <asm/system.h>
15 #include <asm/debugger.h>
16 #include <asm/asm-xsi-offsets.h>
17 #include <public/xen.h>
20 #define _PAGE_PPN_MASK 0x0003fffffffff000 //asm/pgtable.h doesn't do assembly
21 #define PAGE_PHYS 0x0010000000000761 //__pgprot(__DIRTY_BITS|
22 // _PAGE_PL_2|_PAGE_AR_RWX)
23 #define _PAGE_PL_2 (2<<7)
25 #if 1 // change to 0 to turn off all fast paths
26 # define FAST_HYPERPRIVOPS
27 # ifdef PERF_COUNTERS
28 # define FAST_HYPERPRIVOP_CNT
29 # define FAST_HYPERPRIVOP_PERFC(N) PERFC(fast_hyperprivop + N)
30 # define FAST_REFLECT_CNT
31 # endif
33 //#define FAST_TICK // mostly working (unat problems) but default off for now
34 //#define FAST_TLB_MISS_REFLECT // mostly working but default off for now
35 # undef FAST_ITC //XXX TODO fast_itc doesn't support dom0 vp yet
36 # define FAST_BREAK
37 # undef FAST_ACCESS_REFLECT //XXX TODO fast_access_reflect
38 // doesn't support dom0 vp yet.
39 # define FAST_RFI
40 // TODO: Since we use callback to deliver interrupt,
41 // FAST_SSM_I needs to be rewritten.
42 # define FAST_SSM_I
43 # define FAST_PTC_GA
44 # undef RFI_TO_INTERRUPT // not working yet
45 #endif
47 #ifdef CONFIG_SMP
48 //#warning "FIXME: ptc.ga instruction requires spinlock for SMP"
49 #undef FAST_PTC_GA
50 #endif
52 // FIXME: turn off for now... but NaTs may crash Xen so re-enable soon!
53 #define HANDLE_AR_UNAT
55 // FIXME: This is defined in include/asm-ia64/hw_irq.h but this
56 // doesn't appear to be include'able from assembly?
57 #define IA64_TIMER_VECTOR 0xef
59 // Should be included from common header file (also in process.c)
60 // NO PSR_CLR IS DIFFERENT! (CPL)
61 #define IA64_PSR_CPL1 (__IA64_UL(1) << IA64_PSR_CPL1_BIT)
62 #define IA64_PSR_CPL0 (__IA64_UL(1) << IA64_PSR_CPL0_BIT)
63 // note IA64_PSR_PK removed from following, why is this necessary?
64 #define DELIVER_PSR_SET (IA64_PSR_IC | IA64_PSR_I | \
65 IA64_PSR_DT | IA64_PSR_RT | IA64_PSR_CPL1 | \
66 IA64_PSR_IT | IA64_PSR_BN)
68 #define DELIVER_PSR_CLR (IA64_PSR_AC | IA64_PSR_DFL | IA64_PSR_DFH | \
69 IA64_PSR_SP | IA64_PSR_DI | IA64_PSR_SI | \
70 IA64_PSR_DB | IA64_PSR_LP | IA64_PSR_TB | \
71 IA64_PSR_MC | IA64_PSR_IS | \
72 IA64_PSR_ID | IA64_PSR_DA | IA64_PSR_DD | \
73 IA64_PSR_SS | IA64_PSR_RI | IA64_PSR_ED | IA64_PSR_IA)
75 // Note: not hand-scheduled for now
76 // Registers at entry
77 // r16 == cr.isr
78 // r17 == cr.iim
79 // r18 == XSI_PSR_IC_OFS
80 // r19 == vpsr.ic
81 // r31 == pr
82 GLOBAL_ENTRY(fast_hyperprivop)
83 // HYPERPRIVOP_SSM_I?
84 // assumes domain interrupts pending, so just do it
85 cmp.eq p7,p6=HYPERPRIVOP_SSM_I,r17
86 (p7) br.sptk.many hyper_ssm_i;;
88 // Check pending event indication
89 adds r20=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS, r18;;
90 ld8 r20=[r20]
91 ;;
92 ld1 r22=[r20],-1 // evtchn_upcall_mask
93 ;;
94 ld1 r20=[r20] // evtchn_upcall_pending
95 ;;
97 // HYPERPRIVOP_RFI?
98 cmp.eq p7,p6=HYPERPRIVOP_RFI,r17
99 (p7) br.sptk.many hyper_rfi
100 ;;
101 #ifndef FAST_HYPERPRIVOPS // see beginning of file
102 br.sptk.many dispatch_break_fault ;;
103 #endif
104 // if event enabled and there are pending events
105 cmp.ne p7,p0=r20,r0
106 ;;
107 cmp.eq.and p7,p0=r22,r0
108 (p7) br.spnt.many dispatch_break_fault
109 ;;
111 // HYPERPRIVOP_COVER?
112 cmp.eq p7,p6=HYPERPRIVOP_COVER,r17
113 (p7) br.sptk.many hyper_cover
114 ;;
116 // HYPERPRIVOP_SSM_DT?
117 cmp.eq p7,p6=HYPERPRIVOP_SSM_DT,r17
118 (p7) br.sptk.many hyper_ssm_dt
119 ;;
121 // HYPERPRIVOP_RSM_DT?
122 cmp.eq p7,p6=HYPERPRIVOP_RSM_DT,r17
123 (p7) br.sptk.many hyper_rsm_dt
124 ;;
126 // HYPERPRIVOP_SET_ITM?
127 cmp.eq p7,p6=HYPERPRIVOP_SET_ITM,r17
128 (p7) br.sptk.many hyper_set_itm
129 ;;
131 // HYPERPRIVOP_SET_RR?
132 cmp.eq p7,p6=HYPERPRIVOP_SET_RR,r17
133 (p7) br.sptk.many hyper_set_rr
134 ;;
136 // HYPERPRIVOP_GET_RR?
137 cmp.eq p7,p6=HYPERPRIVOP_GET_RR,r17
138 (p7) br.sptk.many hyper_get_rr
139 ;;
141 // HYPERPRIVOP_GET_PSR?
142 cmp.eq p7,p6=HYPERPRIVOP_GET_PSR,r17
143 (p7) br.sptk.many hyper_get_psr
144 ;;
146 // HYPERPRIVOP_PTC_GA?
147 cmp.eq p7,p6=HYPERPRIVOP_PTC_GA,r17
148 (p7) br.sptk.many hyper_ptc_ga
149 ;;
151 // HYPERPRIVOP_ITC_D?
152 cmp.eq p7,p6=HYPERPRIVOP_ITC_D,r17
153 (p7) br.sptk.many hyper_itc_d
154 ;;
156 // HYPERPRIVOP_ITC_I?
157 cmp.eq p7,p6=HYPERPRIVOP_ITC_I,r17
158 (p7) br.sptk.many hyper_itc_i
159 ;;
161 // HYPERPRIVOP_THASH?
162 cmp.eq p7,p6=HYPERPRIVOP_THASH,r17
163 (p7) br.sptk.many hyper_thash
164 ;;
166 // HYPERPRIVOP_SET_KR?
167 cmp.eq p7,p6=HYPERPRIVOP_SET_KR,r17
168 (p7) br.sptk.many hyper_set_kr
169 ;;
171 // if not one of the above, give up for now and do it the slow way
172 br.sptk.many dispatch_break_fault
173 ;;
174 END(fast_hyperprivop)
176 // give up for now if: ipsr.be==1, ipsr.pp==1
177 // from reflect_interruption, don't need to:
178 // - printk first extint (debug only)
179 // - check for interrupt collection enabled (routine will force on)
180 // - set ifa (not valid for extint)
181 // - set iha (not valid for extint)
182 // - set itir (not valid for extint)
183 // DO need to
184 // - increment the HYPER_SSM_I fast_hyperprivop counter
185 // - set shared_mem iip to instruction after HYPER_SSM_I
186 // - set cr.iip to guest iva+0x3000
187 // - set shared_mem ipsr to [vcpu_get_ipsr_int_state]
188 // be = pp = bn = 0; dt = it = rt = 1; cpl = 3 or 0;
189 // i = shared_mem interrupt_delivery_enabled
190 // ic = shared_mem interrupt_collection_enabled
191 // ri = instruction after HYPER_SSM_I
192 // all other bits unchanged from real cr.ipsr
193 // - set cr.ipsr (DELIVER_PSR_SET/CLEAR, don't forget cpl!)
194 // - set shared_mem isr: isr.ei to instr following HYPER_SSM_I
195 // and isr.ri to cr.isr.ri (all other bits zero)
196 // - cover and set shared_mem precover_ifs to cr.ifs
197 // ^^^ MISSED THIS FOR fast_break??
198 // - set shared_mem interrupt_delivery_enabled to 0
199 // - set shared_mem interrupt_collection_enabled to 0
200 // - set r31 to SHAREDINFO_ADDR
201 // - virtual bank switch 0
202 // maybe implement later
203 // - verify that there really IS a deliverable interrupt pending
204 // - set shared_mem iva
205 // needs to be done but not implemented (in reflect_interruption)
206 // - set shared_mem iipa
207 // don't know for sure
208 // - set shared_mem unat
209 // r16 == cr.isr
210 // r17 == cr.iim
211 // r18 == XSI_PSR_IC
212 // r19 == vpsr.ic
213 // r31 == pr
214 ENTRY(hyper_ssm_i)
215 #ifndef FAST_SSM_I
216 br.spnt.few dispatch_break_fault ;;
217 #endif
218 // give up for now if: ipsr.be==1, ipsr.pp==1
219 mov r30=cr.ipsr;;
220 mov r29=cr.iip;;
221 extr.u r21=r30,IA64_PSR_BE_BIT,1 ;;
222 cmp.ne p7,p0=r21,r0
223 (p7) br.sptk.many dispatch_break_fault ;;
224 extr.u r21=r30,IA64_PSR_PP_BIT,1 ;;
225 cmp.ne p7,p0=r21,r0
226 (p7) br.sptk.many dispatch_break_fault ;;
227 #ifdef FAST_HYPERPRIVOP_CNT
228 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SSM_I);;
229 ld4 r21=[r20];;
230 adds r21=1,r21;;
231 st4 [r20]=r21;;
232 #endif
233 // set shared_mem iip to instruction after HYPER_SSM_I
234 extr.u r20=r30,41,2 ;;
235 cmp.eq p6,p7=2,r20 ;;
236 (p6) mov r20=0
237 (p6) adds r29=16,r29
238 (p7) adds r20=1,r20 ;;
239 dep r30=r20,r30,41,2;; // adjust cr.ipsr.ri but don't save yet
240 adds r21=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
241 st8 [r21]=r29 ;;
242 // set shared_mem isr
243 extr.u r16=r16,38,1;; // grab cr.isr.ir bit
244 dep r16=r16,r0,38,1 ;; // insert into cr.isr (rest of bits zero)
245 dep r16=r20,r16,41,2 ;; // deposit cr.isr.ri
246 adds r21=XSI_ISR_OFS-XSI_PSR_IC_OFS,r18 ;;
247 st8 [r21]=r16 ;;
248 // set cr.ipsr
249 mov r29=r30 ;;
250 movl r28=DELIVER_PSR_SET;;
251 movl r27=~DELIVER_PSR_CLR;;
252 or r29=r29,r28;;
253 and r29=r29,r27;;
254 // set hpsr_dfh to ipsr
255 adds r28=XSI_HPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
256 ld1 r28=[r28];;
257 dep r29=r28,r29,IA64_PSR_DFH_BIT,1;;
258 mov cr.ipsr=r29;;
259 // set shared_mem ipsr (from ipsr in r30 with ipsr.ri already set)
260 extr.u r29=r30,IA64_PSR_CPL0_BIT,2;;
261 cmp.eq p6,p7=3,r29;;
262 (p6) dep r30=-1,r30,IA64_PSR_CPL0_BIT,2
263 (p7) dep r30=0,r30,IA64_PSR_CPL0_BIT,2
264 ;;
265 // FOR SSM_I ONLY, also turn on psr.i and psr.ic
266 movl r28=(IA64_PSR_DT|IA64_PSR_IT|IA64_PSR_RT|IA64_PSR_I|IA64_PSR_IC);;
267 // movl r27=~(IA64_PSR_BE|IA64_PSR_PP|IA64_PSR_BN);;
268 movl r27=~(IA64_PSR_BE|IA64_PSR_BN);;
269 or r30=r30,r28;;
270 and r30=r30,r27;;
271 mov r20=1
272 movl r22=THIS_CPU(current_psr_i_addr)
273 adds r21=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18 ;;
274 ld8 r22=[r22]
275 ;;
276 adds r27=XSI_VPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
277 ld1 r28=[r27];;
278 st1 [r27]=r0
279 dep r30=r28,r30,IA64_PSR_DFH_BIT,1
280 ;;
281 st8 [r21]=r30;;
282 // set shared_mem interrupt_delivery_enabled to 0
283 // set shared_mem interrupt_collection_enabled to 0
284 st1 [r22]=r20
285 st4 [r18]=r0
286 // cover and set shared_mem precover_ifs to cr.ifs
287 // set shared_mem ifs to 0
288 cover ;;
289 mov r20=cr.ifs;;
290 adds r21=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
291 st8 [r21]=r0 ;;
292 adds r21=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
293 st8 [r21]=r20 ;;
294 // leave cr.ifs alone for later rfi
295 // set iip to go to event callback handler
296 movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
297 ld8 r22=[r22];;
298 adds r22=IA64_VCPU_EVENT_CALLBACK_IP_OFFSET,r22;;
299 ld8 r24=[r22];;
300 mov cr.iip=r24;;
301 // OK, now all set to go except for switch to virtual bank0
302 mov r30=r2
303 mov r29=r3
304 ;;
305 adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18;
306 adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18;;
307 // temporarily save ar.unat
308 mov r28=ar.unat
309 bsw.1;;
310 // FIXME?: ar.unat is not really handled correctly,
311 // but may not matter if the OS is NaT-clean
312 .mem.offset 0,0; st8.spill [r2]=r16,16;
313 .mem.offset 8,0; st8.spill [r3]=r17,16 ;;
314 .mem.offset 0,0; st8.spill [r2]=r18,16;
315 .mem.offset 8,0; st8.spill [r3]=r19,16 ;;
316 .mem.offset 0,0; st8.spill [r2]=r20,16;
317 .mem.offset 8,0; st8.spill [r3]=r21,16 ;;
318 .mem.offset 0,0; st8.spill [r2]=r22,16;
319 .mem.offset 8,0; st8.spill [r3]=r23,16 ;;
320 .mem.offset 0,0; st8.spill [r2]=r24,16;
321 .mem.offset 8,0; st8.spill [r3]=r25,16 ;;
322 .mem.offset 0,0; st8.spill [r2]=r26,16;
323 .mem.offset 8,0; st8.spill [r3]=r27,16 ;;
324 .mem.offset 0,0; st8.spill [r2]=r28,16;
325 .mem.offset 8,0; st8.spill [r3]=r29,16 ;;
326 .mem.offset 0,0; st8.spill [r2]=r30,16;
327 .mem.offset 8,0; st8.spill [r3]=r31,16 ;;
328 bsw.0 ;;
329 mov r27=ar.unat
330 adds r26=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18 ;;
331 //save bank1 ar.unat
332 st8 [r26]=r27
333 //restore ar.unat
334 mov ar.unat=r28
335 mov r2=r30
336 mov r3=r29
337 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
338 st4 [r20]=r0 ;;
339 mov pr=r31,-1 ;;
340 rfi
341 ;;
342 END(hyper_ssm_i)
344 // reflect domain clock interrupt
345 // r31 == pr
346 // r30 == cr.ivr
347 // r29 == rp
348 GLOBAL_ENTRY(fast_tick_reflect)
349 #ifndef FAST_TICK // see beginning of file
350 br.cond.sptk.many rp;;
351 #endif
352 mov r28=IA64_TIMER_VECTOR;;
353 cmp.ne p6,p0=r28,r30
354 (p6) br.cond.spnt.few rp;;
355 movl r20=THIS_CPU(cpu_info)+IA64_CPUINFO_ITM_NEXT_OFFSET;;
356 ld8 r26=[r20];;
357 mov r27=ar.itc;;
358 adds r27=200,r27;; // safety margin
359 cmp.ltu p6,p0=r26,r27
360 (p6) br.cond.spnt.few rp;;
361 mov r17=cr.ipsr;;
362 // slow path if: ipsr.be==1, ipsr.pp==1
363 extr.u r21=r17,IA64_PSR_BE_BIT,1 ;;
364 cmp.ne p6,p0=r21,r0
365 (p6) br.cond.spnt.few rp;;
366 extr.u r21=r17,IA64_PSR_PP_BIT,1 ;;
367 cmp.ne p6,p0=r21,r0
368 (p6) br.cond.spnt.few rp;;
369 // definitely have a domain tick
370 mov cr.eoi=r0;;
371 mov rp=r29;;
372 mov cr.itm=r26;; // ensure next tick
373 #ifdef FAST_REFLECT_CNT
374 movl r20=PERFC(fast_reflect + (0x3000>>8));;
375 ld4 r21=[r20];;
376 adds r21=1,r21;;
377 st4 [r20]=r21;;
378 #endif
379 // vcpu_pend_timer(current)
380 movl r18=THIS_CPU(current_psr_ic_addr)
381 ;;
382 ld8 r18=[r18]
383 ;;
384 adds r20=XSI_ITV_OFS-XSI_PSR_IC_OFS,r18 ;;
385 ld8 r20=[r20];;
386 cmp.eq p6,p0=r20,r0 // if cr.itv==0 done
387 (p6) br.cond.spnt.few fast_tick_reflect_done;;
388 tbit.nz p6,p0=r20,16;; // check itv.m (discard) bit
389 (p6) br.cond.spnt.few fast_tick_reflect_done;;
390 extr.u r27=r20,0,6 // r27 has low 6 bits of itv.vector
391 extr.u r26=r20,6,2;; // r26 has irr index of itv.vector
392 movl r19=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
393 ld8 r19=[r19];;
394 adds r22=IA64_VCPU_DOMAIN_ITM_LAST_OFFSET,r19
395 adds r23=IA64_VCPU_DOMAIN_ITM_OFFSET,r19;;
396 ld8 r24=[r22];;
397 ld8 r23=[r23];;
398 cmp.eq p6,p0=r23,r24 // skip if this tick already delivered
399 (p6) br.cond.spnt.few fast_tick_reflect_done;;
400 // set irr bit
401 adds r21=IA64_VCPU_IRR0_OFFSET,r19;
402 shl r26=r26,3;;
403 add r21=r21,r26;;
404 mov r25=1;;
405 shl r22=r25,r27;;
406 ld8 r23=[r21];;
407 or r22=r22,r23;;
408 st8 [r21]=r22;;
409 // set evtchn_upcall_pending!
410 adds r20=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r18;;
411 ld8 r20=[r20];;
412 adds r20=-1,r20;; // evtchn_upcall_pending
413 st1 [r20]=r25;;
414 // if interrupted at pl0, we're done
415 extr.u r16=r17,IA64_PSR_CPL0_BIT,2;;
416 cmp.eq p6,p0=r16,r0;;
417 (p6) br.cond.spnt.few fast_tick_reflect_done;;
418 // if guest vpsr.i is off, we're done
419 movl r21=THIS_CPU(current_psr_i_addr);;
420 ld8 r21=[r21];;
421 ld1 r21=[r21];;
422 cmp.eq p0,p6=r21,r0
423 (p6) br.cond.spnt.few fast_tick_reflect_done;;
425 // OK, we have a clock tick to deliver to the active domain!
426 // so deliver to iva+0x3000
427 // r17 == cr.ipsr
428 // r18 == XSI_PSR_IC
429 // r19 == IA64_KR(CURRENT)
430 // r31 == pr
431 mov r16=cr.isr;;
432 mov r29=cr.iip;;
433 adds r21=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
434 st8 [r21]=r29 ;;
435 // set shared_mem isr
436 extr.u r16=r16,38,1;; // grab cr.isr.ir bit
437 dep r16=r16,r0,38,1 ;; // insert into cr.isr (rest of bits zero)
438 extr.u r20=r17,41,2 ;; // get ipsr.ri
439 dep r16=r20,r16,41,2 ;; // deposit cr.isr.ei
440 adds r21=XSI_ISR_OFS-XSI_PSR_IC_OFS,r18 ;;
441 st8 [r21]=r16 ;;
442 // set cr.ipsr (make sure cpl==2!)
443 mov r29=r17 ;;
444 movl r28=DELIVER_PSR_SET;;
445 movl r27=~(DELIVER_PSR_CLR|IA64_PSR_CPL0);;
446 or r29=r29,r28;;
447 and r29=r29,r27;;
448 mov cr.ipsr=r29;;
449 // set shared_mem ipsr (from ipsr in r17 with ipsr.ri already set)
450 extr.u r29=r17,IA64_PSR_CPL0_BIT,2;;
451 cmp.eq p6,p7=3,r29;;
452 (p6) dep r17=-1,r17,IA64_PSR_CPL0_BIT,2
453 (p7) dep r17=0,r17,IA64_PSR_CPL0_BIT,2
454 ;;
455 movl r28=(IA64_PSR_DT|IA64_PSR_IT|IA64_PSR_RT);;
456 movl r27=~(IA64_PSR_BE|IA64_PSR_PP|IA64_PSR_BN|IA64_PSR_I|IA64_PSR_IC);;
457 dep r21=-1,r21,IA64_PSR_CPL1_BIT,1 ;;
458 or r17=r17,r28;;
459 and r17=r17,r27;;
460 ld4 r16=[r18];;
461 cmp.ne p6,p0=r16,r0;;
462 movl r22=THIS_CPU(current_psr_i_addr);;
463 ld8 r22=[r22]
464 (p6) dep r17=-1,r17,IA64_PSR_IC_BIT,1 ;;
465 ld1 r16=[r22];;
466 cmp.eq p6,p0=r16,r0;;
467 (p6) dep r17=-1,r17,IA64_PSR_I_BIT,1 ;;
468 mov r20=1
469 adds r21=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18 ;;
470 st8 [r21]=r17 ;;
471 // set shared_mem interrupt_delivery_enabled to 0
472 // set shared_mem interrupt_collection_enabled to 0
473 st1 [r22]=r20;;
474 st4 [r18]=r0;;
475 // cover and set shared_mem precover_ifs to cr.ifs
476 // set shared_mem ifs to 0
477 cover ;;
478 mov r20=cr.ifs;;
479 adds r21=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
480 st8 [r21]=r0 ;;
481 adds r21=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
482 st8 [r21]=r20 ;;
483 // leave cr.ifs alone for later rfi
484 // set iip to go to domain IVA break instruction vector
485 adds r22=IA64_VCPU_IVA_OFFSET,r19;;
486 ld8 r23=[r22];;
487 movl r24=0x3000;;
488 add r24=r24,r23;;
489 mov cr.iip=r24;;
490 // OK, now all set to go except for switch to virtual bank0
491 mov r30=r2
492 mov r29=r3
493 #ifdef HANDLE_AR_UNAT
494 mov r28=ar.unat;
495 #endif
496 ;;
497 adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18
498 adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18
499 ;;
500 bsw.1;;
501 .mem.offset 0,0; st8.spill [r2]=r16,16;
502 .mem.offset 8,0; st8.spill [r3]=r17,16 ;;
503 .mem.offset 0,0; st8.spill [r2]=r18,16;
504 .mem.offset 8,0; st8.spill [r3]=r19,16 ;;
505 .mem.offset 0,0; st8.spill [r2]=r20,16;
506 .mem.offset 8,0; st8.spill [r3]=r21,16 ;;
507 .mem.offset 0,0; st8.spill [r2]=r22,16;
508 .mem.offset 8,0; st8.spill [r3]=r23,16 ;;
509 .mem.offset 0,0; st8.spill [r2]=r24,16;
510 .mem.offset 8,0; st8.spill [r3]=r25,16 ;;
511 .mem.offset 0,0; st8.spill [r2]=r26,16;
512 .mem.offset 8,0; st8.spill [r3]=r27,16 ;;
513 .mem.offset 0,0; st8.spill [r2]=r28,16;
514 .mem.offset 8,0; st8.spill [r3]=r29,16 ;;
515 .mem.offset 0,0; st8.spill [r2]=r30,16;
516 .mem.offset 8,0; st8.spill [r3]=r31,16 ;;
517 #ifdef HANDLE_AR_UNAT
518 // r16~r23 are preserved regsin bank0 regs, we need to restore them,
519 // r24~r31 are scratch regs, we don't need to handle NaT bit,
520 // because OS handler must assign it before access it
521 ld8 r16=[r2],16;
522 ld8 r17=[r3],16;;
523 ld8 r18=[r2],16;
524 ld8 r19=[r3],16;;
525 ld8 r20=[r2],16;
526 ld8 r21=[r3],16;;
527 ld8 r22=[r2],16;
528 ld8 r23=[r3],16;;
529 #endif
530 ;;
531 bsw.0 ;;
532 mov r24=ar.unat;
533 mov r2=r30
534 mov r3=r29
535 #ifdef HANDLE_AR_UNAT
536 mov ar.unat=r28;
537 #endif
538 ;;
539 adds r25=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18 ;
540 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
541 st8 [r25]=r24;
542 st4 [r20]=r0 ;;
543 fast_tick_reflect_done:
544 mov pr=r31,-1 ;;
545 rfi
546 END(fast_tick_reflect)
548 // reflect domain breaks directly to domain
549 // r16 == cr.isr
550 // r17 == cr.iim
551 // r18 == XSI_PSR_IC
552 // r19 == vpsr.ic
553 // r31 == pr
554 GLOBAL_ENTRY(fast_break_reflect)
555 #ifndef FAST_BREAK // see beginning of file
556 br.sptk.many dispatch_break_fault ;;
557 #endif
558 mov r30=cr.ipsr;;
559 mov r29=cr.iip;;
560 extr.u r21=r30,IA64_PSR_BE_BIT,1 ;;
561 cmp.ne p7,p0=r21,r0 ;;
562 (p7) br.spnt.few dispatch_break_fault ;;
563 extr.u r21=r30,IA64_PSR_PP_BIT,1 ;;
564 cmp.ne p7,p0=r21,r0 ;;
565 (p7) br.spnt.few dispatch_break_fault ;;
566 movl r20=IA64_PSR_CPL ;;
567 and r22=r20,r30 ;;
568 cmp.ne p7,p0=r22,r0
569 (p7) br.spnt.many 1f ;;
570 cmp.eq p7,p0=r17,r0
571 (p7) br.spnt.few dispatch_break_fault ;;
572 #ifdef CRASH_DEBUG
573 movl r21=CDB_BREAK_NUM ;;
574 cmp.eq p7,p0=r17,r21
575 (p7) br.spnt.few dispatch_break_fault ;;
576 #endif
577 1:
578 #if 1 /* special handling in case running on simulator */
579 movl r20=first_break;;
580 ld4 r23=[r20];;
581 movl r21=0x80001;
582 movl r22=0x80002;;
583 cmp.ne p7,p0=r23,r0;;
584 (p7) br.spnt.few dispatch_break_fault ;;
585 cmp.eq p7,p0=r21,r17;
586 (p7) br.spnt.few dispatch_break_fault ;;
587 cmp.eq p7,p0=r22,r17;
588 (p7) br.spnt.few dispatch_break_fault ;;
589 #endif
590 movl r20=0x2c00;
591 // save iim in shared_info
592 adds r21=XSI_IIM_OFS-XSI_PSR_IC_OFS,r18 ;;
593 st8 [r21]=r17;;
594 // fall through
595 END(fast_break_reflect)
597 // reflect to domain ivt+r20
598 // sets up isr,iip,ipsr,ifs (FIXME: do iipa too)
599 // r16 == cr.isr
600 // r18 == XSI_PSR_IC
601 // r20 == offset into ivt
602 // r29 == iip
603 // r30 == ipsr
604 // r31 == pr
605 ENTRY(fast_reflect)
606 #ifdef FAST_REFLECT_CNT
607 movl r22=PERFC(fast_reflect);
608 shr r23=r20,8-2;;
609 add r22=r22,r23;;
610 ld4 r21=[r22];;
611 adds r21=1,r21;;
612 st4 [r22]=r21;;
613 #endif
614 // save iip in shared_info (DON'T POINT TO NEXT INSTRUCTION!)
615 adds r21=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
616 st8 [r21]=r29,XSI_ISR_OFS-XSI_IIP_OFS;;
617 // set shared_mem isr
618 st8 [r21]=r16 ;;
619 // set cr.ipsr
620 movl r21=THIS_CPU(current_psr_i_addr)
621 mov r29=r30 ;;
622 ld8 r21=[r21]
623 movl r28=DELIVER_PSR_SET;;
624 movl r27=~(DELIVER_PSR_CLR|IA64_PSR_CPL0);;
625 or r29=r29,r28;;
626 and r29=r29,r27;;
627 // set hpsr_dfh to ipsr
628 adds r28=XSI_HPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
629 ld1 r28=[r28];;
630 dep r29=r28,r29,IA64_PSR_DFH_BIT,1;;
631 mov cr.ipsr=r29;;
632 // set shared_mem ipsr (from ipsr in r30 with ipsr.ri already set)
633 extr.u r29=r30,IA64_PSR_CPL0_BIT,2;;
634 cmp.eq p6,p7=3,r29;;
635 (p6) dep r30=-1,r30,IA64_PSR_CPL0_BIT,2
636 (p7) dep r30=0,r30,IA64_PSR_CPL0_BIT,2
637 ;;
638 movl r28=(IA64_PSR_DT|IA64_PSR_IT|IA64_PSR_RT);;
639 movl r27=~(IA64_PSR_BE|IA64_PSR_PP|IA64_PSR_BN);;
640 or r30=r30,r28;;
641 and r30=r30,r27;;
642 // also set shared_mem ipsr.i and ipsr.ic appropriately
643 ld1 r22=[r21]
644 ld4 r24=[r18];;
645 cmp4.eq p6,p7=r24,r0;;
646 (p6) dep r30=0,r30,IA64_PSR_IC_BIT,1
647 (p7) dep r30=-1,r30,IA64_PSR_IC_BIT,1 ;;
648 mov r24=r21
649 cmp.ne p6,p7=r22,r0;;
650 (p6) dep r30=0,r30,IA64_PSR_I_BIT,1
651 (p7) dep r30=-1,r30,IA64_PSR_I_BIT,1 ;;
652 mov r22=1
653 adds r21=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18
654 ;;
655 adds r27=XSI_VPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
656 ld1 r28=[r27];;
657 st1 [r27]=r0
658 dep r30=r28,r30,IA64_PSR_DFH_BIT,1
659 ;;
660 st8 [r21]=r30 ;;
661 // set shared_mem interrupt_delivery_enabled to 0
662 // set shared_mem interrupt_collection_enabled to 0
663 st1 [r24]=r22
664 st4 [r18]=r0;;
665 // cover and set shared_mem precover_ifs to cr.ifs
666 // set shared_mem ifs to 0
667 cover ;;
668 mov r24=cr.ifs;;
669 adds r21=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
670 st8 [r21]=r0 ;;
671 adds r21=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
672 st8 [r21]=r24 ;;
673 // FIXME: need to save iipa and isr to be arch-compliant
674 // set iip to go to domain IVA break instruction vector
675 movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
676 ld8 r22=[r22];;
677 adds r22=IA64_VCPU_IVA_OFFSET,r22;;
678 ld8 r23=[r22];;
679 add r20=r20,r23;;
680 mov cr.iip=r20;;
681 // OK, now all set to go except for switch to virtual bank0
682 mov r30=r2
683 mov r29=r3
684 #ifdef HANDLE_AR_UNAT
685 mov r28=ar.unat;
686 #endif
687 adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18;
688 adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18
689 ;;
690 bsw.1;;
691 .mem.offset 0,0; st8.spill [r2]=r16,16;
692 .mem.offset 8,0; st8.spill [r3]=r17,16 ;;
693 .mem.offset 0,0; st8.spill [r2]=r18,16;
694 .mem.offset 8,0; st8.spill [r3]=r19,16 ;;
695 .mem.offset 0,0; st8.spill [r2]=r20,16;
696 .mem.offset 8,0; st8.spill [r3]=r21,16 ;;
697 .mem.offset 0,0; st8.spill [r2]=r22,16;
698 .mem.offset 8,0; st8.spill [r3]=r23,16 ;;
699 .mem.offset 0,0; st8.spill [r2]=r24,16;
700 .mem.offset 8,0; st8.spill [r3]=r25,16 ;;
701 .mem.offset 0,0; st8.spill [r2]=r26,16;
702 .mem.offset 8,0; st8.spill [r3]=r27,16 ;;
703 .mem.offset 0,0; st8.spill [r2]=r28,16;
704 .mem.offset 8,0; st8.spill [r3]=r29,16 ;;
705 .mem.offset 0,0; st8.spill [r2]=r30,16;
706 .mem.offset 8,0; st8.spill [r3]=r31,16 ;;
707 #ifdef HANDLE_AR_UNAT
708 // r16~r23 are preserved regs in bank0 regs, we need to restore them,
709 // r24~r31 are scratch regs, we don't need to handle NaT bit,
710 // because OS handler must assign it before access it
711 ld8 r16=[r2],16;
712 ld8 r17=[r3],16;;
713 ld8 r18=[r2],16;
714 ld8 r19=[r3],16;;
715 ld8 r20=[r2],16;
716 ld8 r21=[r3],16;;
717 ld8 r22=[r2],16;
718 ld8 r23=[r3],16;;
719 #endif
720 ;;
721 bsw.0 ;;
722 mov r24=ar.unat;
723 mov r2=r30
724 mov r3=r29
725 #ifdef HANDLE_AR_UNAT
726 mov ar.unat=r28;
727 #endif
728 ;;
729 adds r25=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18 ;
730 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
731 st8 [r25]=r24;
732 st4 [r20]=r0 ;;
733 mov pr=r31,-1 ;;
734 rfi
735 ;;
736 END(fast_reflect)
738 // reflect access faults (0x2400,0x2800,0x5300) directly to domain
739 // r16 == isr
740 // r17 == ifa
741 // r19 == reflect number (only pass-thru to dispatch_reflection)
742 // r20 == offset into ivt
743 // r31 == pr
744 GLOBAL_ENTRY(fast_access_reflect)
745 #ifndef FAST_ACCESS_REFLECT // see beginning of file
746 br.spnt.few dispatch_reflection ;;
747 #endif
748 mov r30=cr.ipsr;;
749 mov r29=cr.iip;;
750 extr.u r21=r30,IA64_PSR_BE_BIT,1 ;;
751 cmp.ne p7,p0=r21,r0
752 (p7) br.spnt.few dispatch_reflection ;;
753 extr.u r21=r30,IA64_PSR_PP_BIT,1 ;;
754 cmp.ne p7,p0=r21,r0
755 (p7) br.spnt.few dispatch_reflection ;;
756 extr.u r21=r30,IA64_PSR_CPL0_BIT,2 ;;
757 cmp.eq p7,p0=r21,r0
758 (p7) br.spnt.few dispatch_reflection ;;
759 movl r18=THIS_CPU(current_psr_ic_addr);;
760 ld8 r18=[r18];;
761 ld4 r21=[r18];;
762 cmp.eq p7,p0=r0,r21
763 (p7) br.spnt.few dispatch_reflection ;;
764 // set shared_mem ifa, FIXME: should we validate it?
765 mov r17=cr.ifa;;
766 adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
767 st8 [r21]=r17 ;;
768 // get rr[ifa] and save to itir in shared memory (extra bits ignored)
769 shr.u r22=r17,61
770 adds r23=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18
771 adds r21=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
772 shladd r22=r22,3,r21;;
773 ld8 r22=[r22];;
774 and r22=~3,r22;;
775 st8 [r23]=r22;;
776 br.cond.sptk.many fast_reflect;;
777 END(fast_access_reflect)
779 // when we get to here, VHPT_CCHAIN_LOOKUP has failed and everything
780 // is as it was at the time of original miss. We want to preserve that
781 // so if we get a nested fault, we can just branch to page_fault
782 GLOBAL_ENTRY(fast_tlb_miss_reflect)
783 #ifndef FAST_TLB_MISS_REFLECT // see beginning of file
784 br.spnt.few page_fault ;;
785 #else
786 mov r31=pr
787 mov r30=cr.ipsr
788 mov r29=cr.iip
789 mov r16=cr.isr
790 mov r17=cr.ifa;;
791 // for now, always take slow path for region 0 (e.g. metaphys mode)
792 extr.u r21=r17,61,3;;
793 cmp.eq p7,p0=r0,r21
794 (p7) br.spnt.few page_fault ;;
795 // always take slow path for PL0 (e.g. __copy_from_user)
796 extr.u r21=r30,IA64_PSR_CPL0_BIT,2 ;;
797 cmp.eq p7,p0=r21,r0
798 (p7) br.spnt.few page_fault ;;
799 // slow path if strange ipsr or isr bits set
800 extr.u r21=r30,IA64_PSR_BE_BIT,1 ;;
801 cmp.ne p7,p0=r21,r0
802 (p7) br.spnt.few page_fault ;;
803 extr.u r21=r30,IA64_PSR_PP_BIT,1 ;;
804 cmp.ne p7,p0=r21,r0
805 (p7) br.spnt.few page_fault ;;
806 movl r21=IA64_ISR_IR|IA64_ISR_SP|IA64_ISR_NA ;;
807 and r21=r16,r21;;
808 cmp.ne p7,p0=r0,r21
809 (p7) br.spnt.few page_fault ;;
810 // also take slow path if virtual psr.ic=0
811 movl r18=XSI_PSR_IC;;
812 ld4 r21=[r18];;
813 cmp.eq p7,p0=r0,r21
814 (p7) br.spnt.few page_fault ;;
815 // OK, if we get to here, we are doing a fast vcpu_translate. Need to:
816 // 1) look in the virtual TR's (pinned), if not there
817 // 2) look in the 1-entry TLB (pinned), if not there
818 // 3) check the domain VHPT (NOT pinned, accesses domain memory!)
819 // If we find it in any of these places, we need to effectively do
820 // a hyper_itc_i/d
822 // short-term hack for now, if in region 5-7, take slow path
823 // since all Linux TRs are in region 5 or 7, we need not check TRs
824 extr.u r21=r17,61,3;;
825 cmp.le p7,p0=5,r21
826 (p7) br.spnt.few page_fault ;;
827 fast_tlb_no_tr_match:
828 movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
829 ld8 r27=[r27];;
830 tbit.nz p6,p7=r16,IA64_ISR_X_BIT;;
831 (p6) adds r25=IA64_VCPU_ITLB_OFFSET,r27;;
832 (p7) adds r25=IA64_VCPU_DTLB_OFFSET,r27;;
833 ld8 r20=[r25],8;;
834 tbit.z p7,p0=r20,0;; // present?
835 (p7) br.cond.spnt.few 1f;;
836 // if ifa is in range of tlb, don't bother to check rid, go slow path
837 ld8 r21=[r25],8;;
838 mov r23=1
839 extr.u r21=r21,2,6;;
840 shl r22=r23,r21
841 ld8 r21=[r25],8;;
842 cmp.ltu p7,p0=r17,r21
843 (p7) br.cond.sptk.many 1f;
844 add r21=r22,r21;;
845 cmp.ltu p7,p0=r17,r21
846 (p7) br.cond.spnt.few page_fault;;
848 1: // check the guest VHPT
849 adds r19 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18;;
850 ld8 r19=[r19];;
851 tbit.nz p7,p0=r19,IA64_PTA_VF_BIT;; // long format VHPT
852 (p7) br.cond.spnt.few page_fault;;
853 // if (!rr.ve || !(pta & IA64_PTA_VE)) take slow way for now
854 // FIXME: later, we deliver an alt_d/i vector after thash and itir
855 tbit.z p7,p0=r19,IA64_PTA_VE_BIT;; //
856 (p7) br.cond.spnt.few page_fault;;
857 extr.u r25=r17,61,3;;
858 adds r21=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
859 shl r25=r25,3;;
860 add r21=r21,r25;;
861 ld8 r22=[r21];;
862 tbit.z p7,p0=r22,0
863 (p7) br.cond.spnt.few page_fault;;
865 // compute and save away itir (r22 & RR_PS_MASK)
866 movl r21=0xfc;;
867 and r22=r22,r21;;
868 adds r21=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
869 st8 [r21]=r22;;
871 // save away ifa
872 adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
873 st8 [r21]=r17;;
874 // see vcpu_thash to save away iha
875 shr.u r20 = r17, 61
876 addl r25 = 1, r0
877 movl r30 = 0xe000000000000000
878 ;;
879 and r21 = r30, r17 // VHPT_Addr1
880 ;;
881 shladd r28 = r20, 3, r18
882 adds r19 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18
883 ;;
884 adds r27 = XSI_RR0_OFS-XSI_PSR_IC_OFS, r28
885 addl r28 = 32767, r0
886 ld8 r24 = [r19] // pta
887 ;;
888 ld8 r23 = [r27] // rrs[vadr>>61]
889 extr.u r26 = r24, 2, 6
890 ;;
891 extr.u r22 = r23, 2, 6
892 shl r30 = r25, r26
893 ;;
894 shr.u r19 = r17, r22
895 shr.u r29 = r24, 15
896 ;;
897 adds r30 = -1, r30
898 ;;
899 shladd r27 = r19, 3, r0
900 extr.u r26 = r30, 15, 46
901 ;;
902 andcm r24 = r29, r26
903 and r19 = r28, r27
904 shr.u r25 = r27, 15
905 ;;
906 and r23 = r26, r25
907 ;;
908 or r22 = r24, r23
909 ;;
910 dep.z r20 = r22, 15, 46
911 ;;
912 or r30 = r20, r21
913 ;;
914 //or r8 = r19, r30
915 or r19 = r19, r30
916 ;;
917 adds r23=XSI_IHA_OFS-XSI_PSR_IC_OFS,r18 ;;
918 st8 [r23]=r19;;
919 // done with thash, check guest VHPT
921 adds r20 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18;;
922 ld8 r24 = [r20];; // pta
923 // avoid recursively walking the VHPT
924 // if (((r17=address ^ r24=pta) & ((itir_mask(pta) << 3) >> 3)) != 0) {
925 mov r20=-8
926 xor r21=r17,r24
927 extr.u r24=r24,2,6;;
928 shl r20=r20,r24;;
929 shr.u r20=r20,3;;
930 and r21=r20,r21;;
931 cmp.eq p7,p0=r21,r0
932 (p7) br.cond.spnt.few 1f;;
933 // __copy_from_user(&pte, r19=(void *)(*iha), sizeof(pte)=8)
934 // prepare for possible nested dtlb fault
935 mov r29=b0
936 movl r30=guest_vhpt_miss;;
937 // now go fetch the entry from the guest VHPT
938 ld8 r20=[r19];;
939 // if we wind up here, we successfully loaded the VHPT entry
941 // this VHPT walker aborts on non-present pages instead
942 // of inserting a not-present translation, this allows
943 // vectoring directly to the miss handler
944 tbit.z p7,p0=r20,0
945 (p7) br.cond.spnt.few page_not_present;;
947 #ifdef FAST_REFLECT_CNT
948 movl r21=PERFC(fast_vhpt_translate);;
949 ld4 r22=[r21];;
950 adds r22=1,r22;;
951 st4 [r21]=r22;;
952 #endif
954 // prepare for fast_insert(PSCB(ifa),PSCB(itir),r16=pte)
955 // r16 == pte
956 // r17 == bit0: 1=inst, 0=data; bit1: 1=itc, 0=vcpu_translate
957 // r18 == XSI_PSR_IC_OFS
958 // r24 == ps
959 // r29 == saved value of b0 in case of recovery
960 // r30 == recovery ip if failure occurs
961 // r31 == pr
962 tbit.nz p6,p7=r16,IA64_ISR_X_BIT;;
963 (p6) mov r17=1;;
964 (p7) mov r17=0;;
965 mov r16=r20
966 mov r29=b0 ;;
967 movl r30=recover_and_page_fault ;;
968 adds r21=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
969 ld8 r24=[r21];;
970 extr.u r24=r24,2,6;;
971 // IFA already in PSCB
972 br.cond.sptk.many fast_insert;;
973 END(fast_tlb_miss_reflect)
975 // we get here if fast_insert fails (e.g. due to metaphysical lookup)
976 ENTRY(recover_and_page_fault)
977 #ifdef PERF_COUNTERS
978 movl r21=PERFC(recover_to_page_fault);;
979 ld4 r22=[r21];;
980 adds r22=1,r22;;
981 st4 [r21]=r22;;
982 #endif
983 mov b0=r29;;
984 br.cond.sptk.many page_fault;;
986 // if we wind up here, we missed in guest VHPT so recover
987 // from nested dtlb fault and reflect a tlb fault to the guest
988 guest_vhpt_miss:
989 mov b0=r29;;
990 // fault = IA64_VHPT_FAULT
991 mov r20=r0
992 br.cond.sptk.many 1f;
994 // if we get to here, we are ready to reflect
995 // need to set up virtual ifa, iha, itir (fast_reflect handles
996 // virtual isr, iip, ipsr, ifs
997 // see vcpu_get_itir_on_fault: get ps,rid,(FIXME key) from rr[ifa]
998 page_not_present:
999 tbit.nz p6,p7=r16,IA64_ISR_X_BIT;;
1000 (p6) movl r20=0x400;;
1001 (p7) movl r20=0x800;;
1003 1: extr.u r25=r17,61,3;;
1004 adds r21=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
1005 shl r25=r25,3;;
1006 add r21=r21,r25;;
1007 ld8 r22=[r21];;
1008 extr.u r22=r22,2,30;;
1009 dep.z r22=r22,2,30;;
1010 adds r23=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
1011 st8 [r23]=r22;;
1013 // fast reflect expects
1014 // r16 == cr.isr
1015 // r18 == XSI_PSR_IC
1016 // r20 == offset into ivt
1017 // r29 == iip
1018 // r30 == ipsr
1019 // r31 == pr
1020 //mov r16=cr.isr
1021 mov r29=cr.iip
1022 mov r30=cr.ipsr
1023 br.sptk.many fast_reflect;;
1024 #endif
1025 END(fast_tlb_miss_reflect)
1027 ENTRY(slow_vcpu_rfi)
1028 adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18;;
1029 ld8 r22=[r22];;
1030 tbit.z p6,p0=r22,63
1031 (p6) br.spnt.few dispatch_break_fault ;;
1032 // if vips is valid, discard current register frame
1033 // don't need dorfirfi any more
1034 alloc r22=ar.pfs,0,0,0,0
1035 br.spnt.few dispatch_break_fault
1036 ;;
1037 END(slow_vcpu_rfi)
1039 // ensure that, if giving up, registers at entry to fast_hyperprivop unchanged
1040 ENTRY(hyper_rfi)
1041 #ifndef FAST_RFI
1042 br.spnt.few slow_vcpu_rfi ;;
1043 #endif
1044 // if no interrupts pending, proceed
1045 mov r30=r0
1046 cmp.eq p7,p0=r20,r0
1047 (p7) br.sptk.many 1f
1048 ;;
1049 adds r20=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18 ;;
1050 ld8 r21=[r20];; // r21 = vcr.ipsr
1051 extr.u r22=r21,IA64_PSR_I_BIT,1 ;;
1052 mov r30=r22;;
1053 // r30 determines whether we might deliver an immediate extint
1054 #ifndef RFI_TO_INTERRUPT // see beginning of file
1055 cmp.ne p6,p0=r30,r0
1056 (p6) br.cond.spnt.few slow_vcpu_rfi ;;
1057 #endif
1058 1:
1059 adds r20=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18 ;;
1060 ld8 r21=[r20];; // r21 = vcr.ipsr
1061 extr.u r22=r21,IA64_PSR_BE_BIT,1 ;;
1062 // if turning on psr.be, give up for now and do it the slow way
1063 cmp.ne p7,p0=r22,r0
1064 (p7) br.spnt.few slow_vcpu_rfi ;;
1065 // if (!(vpsr.dt && vpsr.rt && vpsr.it)), do it the slow way
1066 movl r20=(IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_IT);;
1067 and r22=r20,r21
1068 ;;
1069 cmp.ne p7,p0=r22,r20
1070 (p7) br.spnt.few slow_vcpu_rfi ;;
1071 // if was in metaphys mode, do it the slow way (FIXME later?)
1072 adds r20=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18 ;;
1073 ld4 r20=[r20];;
1074 cmp.ne p7,p0=r20,r0
1075 (p7) br.spnt.few slow_vcpu_rfi ;;
1076 // if domain hasn't already done virtual bank switch
1077 // do it the slow way (FIXME later?)
1078 #if 0
1079 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
1080 ld4 r20=[r20];;
1081 cmp.eq p7,p0=r20,r0
1082 (p7) br.spnt.few slow_vcpu_rfi ;;
1083 #endif
1084 // validate vcr.iip, if in Xen range, do it the slow way
1085 adds r20=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
1086 ld8 r22=[r20];;
1087 movl r23=HYPERVISOR_VIRT_START
1088 movl r24=HYPERVISOR_VIRT_END;;
1089 cmp.ltu p0,p7=r22,r23 ;; // if !(iip<low) &&
1090 (p7) cmp.geu p0,p7=r22,r24 ;; // !(iip>=high)
1091 (p7) br.spnt.few slow_vcpu_rfi ;;
1093 1: // OK now, let's do an rfi.
1094 #ifdef FAST_HYPERPRIVOP_CNT
1095 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_RFI);;
1096 ld4 r23=[r20];;
1097 adds r23=1,r23;;
1098 st4 [r20]=r23;;
1099 #endif
1100 #ifdef RFI_TO_INTERRUPT
1101 // maybe do an immediate interrupt delivery?
1102 cmp.ne p6,p0=r30,r0
1103 (p6) br.cond.spnt.few rfi_check_extint;;
1104 #endif
1106 just_do_rfi:
1107 // r18=&vpsr.i|vpsr.ic, r21==vpsr, r22=vcr.iip
1108 mov cr.iip=r22;;
1109 adds r20=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1110 ld8 r20=[r20];;
1111 dep r20=0,r20,38,25;; // ensure ifs has no reserved bits set
1112 mov cr.ifs=r20 ;;
1113 // ipsr.cpl == (vcr.ipsr.cpl == 0) 2 : 3;
1114 movl r20=THIS_CPU(current_psr_i_addr)
1115 dep r21=-1,r21,IA64_PSR_CPL1_BIT,1 ;;
1116 // vpsr.i = vcr.ipsr.i; vpsr.ic = vcr.ipsr.ic
1117 ld8 r20=[r20]
1118 mov r19=1
1119 extr.u r23=r21,IA64_PSR_I_BIT,1 ;;
1120 cmp.ne p7,p6=r23,r0 ;;
1121 // not done yet
1122 (p7) st1 [r20]=r0
1123 (p6) st1 [r20]=r19;;
1124 extr.u r23=r21,IA64_PSR_IC_BIT,1 ;;
1125 cmp.ne p7,p6=r23,r0 ;;
1126 (p7) st4 [r18]=r19;;
1127 (p6) st4 [r18]=r0;;
1128 // force on psr.ic, i, dt, rt, it, bn
1129 movl r20=(IA64_PSR_I|IA64_PSR_IC|IA64_PSR_DT|IA64_PSR_RT| \
1130 IA64_PSR_IT|IA64_PSR_BN)
1131 ;;
1132 or r21=r21,r20
1133 ;;
1134 adds r20=XSI_VPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
1135 tbit.z p8,p9 = r21, IA64_PSR_DFH_BIT
1136 ;;
1137 (p9) mov r27=1;;
1138 (p9) st1 [r20]=r27
1139 ;;
1140 (p8) st1 [r20]=r0
1141 (p8) adds r20=XSI_HPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
1142 (p8) ld1 r27=[r20]
1143 ;;
1144 (p8) dep r21=r27,r21, IA64_PSR_DFH_BIT, 1
1145 ;;
1146 mov cr.ipsr=r21
1147 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
1148 ld4 r21=[r20];;
1149 cmp.ne p7,p0=r21,r0 // domain already did "bank 1 switch?"
1150 (p7) br.cond.spnt.few 1f;
1151 // OK, now all set to go except for switch to virtual bank1
1152 mov r22=1;; st4 [r20]=r22;
1153 mov r30=r2; mov r29=r3;;
1154 mov r17=ar.unat;;
1155 adds r16=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18
1156 adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18;
1157 adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18;;
1158 ld8 r16=[r16];;
1159 mov ar.unat=r16;;
1160 bsw.1;;
1161 // FIXME?: ar.unat is not really handled correctly,
1162 // but may not matter if the OS is NaT-clean
1163 .mem.offset 0,0; ld8.fill r16=[r2],16 ;
1164 .mem.offset 8,0; ld8.fill r17=[r3],16 ;;
1165 .mem.offset 0,0; ld8.fill r18=[r2],16 ;
1166 .mem.offset 0,0; ld8.fill r19=[r3],16 ;;
1167 .mem.offset 8,0; ld8.fill r20=[r2],16 ;
1168 .mem.offset 8,0; ld8.fill r21=[r3],16 ;;
1169 .mem.offset 8,0; ld8.fill r22=[r2],16 ;
1170 .mem.offset 8,0; ld8.fill r23=[r3],16 ;;
1171 .mem.offset 8,0; ld8.fill r24=[r2],16 ;
1172 .mem.offset 8,0; ld8.fill r25=[r3],16 ;;
1173 .mem.offset 8,0; ld8.fill r26=[r2],16 ;
1174 .mem.offset 8,0; ld8.fill r27=[r3],16 ;;
1175 .mem.offset 8,0; ld8.fill r28=[r2],16 ;
1176 .mem.offset 8,0; ld8.fill r29=[r3],16 ;;
1177 .mem.offset 8,0; ld8.fill r30=[r2],16 ;
1178 .mem.offset 8,0; ld8.fill r31=[r3],16 ;;
1179 bsw.0 ;;
1180 mov ar.unat=r17;;
1181 mov r2=r30; mov r3=r29;;
1182 1: mov pr=r31,-1
1183 ;;
1184 rfi
1185 ;;
1186 END(hyper_rfi)
1188 #ifdef RFI_TO_INTERRUPT
1189 ENTRY(rfi_check_extint)
1190 //br.sptk.many dispatch_break_fault ;;
1192 // r18=&vpsr.i|vpsr.ic, r21==vpsr, r22=vcr.iip
1193 // make sure none of these get trashed in case going to just_do_rfi
1194 movl r30=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1195 ld8 r30=[r30];;
1196 adds r24=IA64_VCPU_INSVC3_OFFSET,r30;;
1197 mov r25=192
1198 adds r16=IA64_VCPU_IRR3_OFFSET,r30;;
1199 ld8 r23=[r16];;
1200 cmp.eq p6,p0=r23,r0;;
1201 (p6) adds r16=-8,r16;;
1202 (p6) adds r24=-8,r24;;
1203 (p6) adds r25=-64,r25;;
1204 (p6) ld8 r23=[r16];;
1205 (p6) cmp.eq p6,p0=r23,r0;;
1206 (p6) adds r16=-8,r16;;
1207 (p6) adds r24=-8,r24;;
1208 (p6) adds r25=-64,r25;;
1209 (p6) ld8 r23=[r16];;
1210 (p6) cmp.eq p6,p0=r23,r0;;
1211 (p6) adds r16=-8,r16;;
1212 (p6) adds r24=-8,r24;;
1213 (p6) adds r25=-64,r25;;
1214 (p6) ld8 r23=[r16];;
1215 (p6) cmp.eq p6,p0=r23,r0;;
1216 cmp.eq p6,p0=r23,r0
1217 (p6) br.cond.spnt.few just_do_rfi; // this is actually an error
1218 // r16 points to non-zero element of irr, r23 has value
1219 // r24 points to corr element of insvc, r25 has elt*64
1220 ld8 r26=[r24];;
1221 cmp.geu p6,p0=r26,r23
1222 (p6) br.cond.spnt.many just_do_rfi;
1224 // not masked by insvc, get vector number
1225 shr.u r26=r23,1;;
1226 or r26=r23,r26;;
1227 shr.u r27=r26,2;;
1228 or r26=r26,r27;;
1229 shr.u r27=r26,4;;
1230 or r26=r26,r27;;
1231 shr.u r27=r26,8;;
1232 or r26=r26,r27;;
1233 shr.u r27=r26,16;;
1234 or r26=r26,r27;;
1235 shr.u r27=r26,32;;
1236 or r26=r26,r27;;
1237 andcm r26=0xffffffffffffffff,r26;;
1238 popcnt r26=r26;;
1239 sub r26=63,r26;;
1240 // r26 now contains the bit index (mod 64)
1241 mov r27=1;;
1242 shl r27=r27,r26;;
1243 // r27 now contains the (within the proper word) bit mask
1244 add r26=r25,r26
1245 // r26 now contains the vector [0..255]
1246 adds r20=XSI_TPR_OFS-XSI_PSR_IC_OFS,r18 ;;
1247 ld8 r20=[r20] ;;
1248 extr.u r28=r20,16,1
1249 extr.u r29=r20,4,4 ;;
1250 cmp.ne p6,p0=r28,r0 // if tpr.mmi is set, just rfi
1251 (p6) br.cond.spnt.few just_do_rfi;;
1252 shl r29=r29,4;;
1253 adds r29=15,r29;;
1254 cmp.ge p6,p0=r29,r26 // if tpr masks interrupt, just rfi
1255 (p6) br.cond.spnt.few just_do_rfi;;
1256 END(rfi_check_extint)
1258 // this doesn't work yet (dies early after getting to user mode)
1259 // but happens relatively infrequently, so fix it later.
1260 // NOTE that these will be counted incorrectly for now (for privcnt output)
1261 ENTRY(rfi_with_interrupt)
1262 #if 1
1263 br.sptk.many dispatch_break_fault ;;
1264 #endif
1266 // OK, have an unmasked vector, so deliver extint to vcr.iva+0x3000
1267 // r18 == XSI_PSR_IC
1268 // r21 == vipsr (ipsr in shared_mem)
1269 // r30 == IA64_KR(CURRENT)
1270 // r31 == pr
1271 mov r17=cr.ipsr;;
1272 mov r16=cr.isr;;
1273 // set shared_mem isr
1274 extr.u r16=r16,38,1;; // grab cr.isr.ir bit
1275 dep r16=r16,r0,38,1 ;; // insert into cr.isr (rest of bits zero)
1276 extr.u r20=r21,41,2 ;; // get v(!)psr.ri
1277 dep r16=r20,r16,41,2 ;; // deposit cr.isr.ei
1278 adds r22=XSI_ISR_OFS-XSI_PSR_IC_OFS,r18 ;;
1279 st8 [r22]=r16;;
1280 movl r22=THIS_CPU(current_psr_i_addr)
1281 // set cr.ipsr (make sure cpl==2!)
1282 mov r29=r17
1283 movl r28=DELIVER_PSR_SET;;
1284 mov r20=1;;
1285 ld8 r22=[r22]
1286 movl r27=~(DELIVER_PSR_CLR|IA64_PSR_CPL0)
1287 or r29=r29,r28;;
1288 and r29=r29,r27;;
1289 mov cr.ipsr=r29;;
1290 // v.ipsr and v.iip are already set (and v.iip validated) as rfi target
1291 // set shared_mem interrupt_delivery_enabled to 0
1292 // set shared_mem interrupt_collection_enabled to 0
1293 st1 [r22]=r20
1294 st4 [r18]=r0;;
1295 // cover and set shared_mem precover_ifs to cr.ifs
1296 // set shared_mem ifs to 0
1297 #if 0
1298 cover ;;
1299 mov r20=cr.ifs;;
1300 adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1301 st8 [r22]=r0 ;;
1302 adds r22=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1303 st8 [r22]=r20 ;;
1304 // leave cr.ifs alone for later rfi
1305 #else
1306 adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1307 ld8 r20=[r22];;
1308 st8 [r22]=r0 ;;
1309 adds r22=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1310 st8 [r22]=r20 ;;
1311 #endif
1312 // set iip to go to domain IVA break instruction vector
1313 adds r22=IA64_VCPU_IVA_OFFSET,r30;;
1314 ld8 r23=[r22];;
1315 movl r24=0x3000;;
1316 add r24=r24,r23;;
1317 mov cr.iip=r24;;
1318 #if 0
1319 // OK, now all set to go except for switch to virtual bank0
1320 mov r30=r2; mov r29=r3;;
1321 adds r2=XSI_BANK1_OFS-XSI_PSR_IC_OFS,r18;
1322 adds r3=(XSI_BANK1_OFS+8)-XSI_PSR_IC_OFS,r18;;
1323 bsw.1;;
1324 // FIXME: need to handle ar.unat!
1325 .mem.offset 0,0; st8.spill [r2]=r16,16;
1326 .mem.offset 8,0; st8.spill [r3]=r17,16 ;;
1327 .mem.offset 0,0; st8.spill [r2]=r18,16;
1328 .mem.offset 8,0; st8.spill [r3]=r19,16 ;;
1329 .mem.offset 0,0; st8.spill [r2]=r20,16;
1330 .mem.offset 8,0; st8.spill [r3]=r21,16 ;;
1331 .mem.offset 0,0; st8.spill [r2]=r22,16;
1332 .mem.offset 8,0; st8.spill [r3]=r23,16 ;;
1333 .mem.offset 0,0; st8.spill [r2]=r24,16;
1334 .mem.offset 8,0; st8.spill [r3]=r25,16 ;;
1335 .mem.offset 0,0; st8.spill [r2]=r26,16;
1336 .mem.offset 8,0; st8.spill [r3]=r27,16 ;;
1337 .mem.offset 0,0; st8.spill [r2]=r28,16;
1338 .mem.offset 8,0; st8.spill [r3]=r29,16 ;;
1339 .mem.offset 0,0; st8.spill [r2]=r30,16;
1340 .mem.offset 8,0; st8.spill [r3]=r31,16 ;;
1341 bsw.0 ;;
1342 mov r2=r30; mov r3=r29;;
1343 #endif
1344 adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
1345 st4 [r20]=r0 ;;
1346 mov pr=r31,-1 ;;
1347 rfi
1348 END(rfi_with_interrupt)
1349 #endif // RFI_TO_INTERRUPT
1351 ENTRY(hyper_cover)
1352 #ifdef FAST_HYPERPRIVOP_CNT
1353 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_COVER);;
1354 ld4 r21=[r20];;
1355 adds r21=1,r21;;
1356 st4 [r20]=r21;;
1357 #endif
1358 mov r24=cr.ipsr
1359 mov r25=cr.iip;;
1360 // skip test for vpsr.ic.. it's a prerequisite for hyperprivops
1361 cover ;;
1362 mov r30=cr.ifs
1363 adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18;;
1364 st8 [r22]=r30;;
1365 mov cr.ifs=r0;;
1366 // adjust return address to skip over break instruction
1367 extr.u r26=r24,41,2 ;;
1368 cmp.eq p6,p7=2,r26 ;;
1369 (p6) mov r26=0
1370 (p6) adds r25=16,r25
1371 (p7) adds r26=1,r26
1372 ;;
1373 dep r24=r26,r24,41,2
1374 ;;
1375 mov cr.ipsr=r24
1376 mov cr.iip=r25
1377 mov pr=r31,-1 ;;
1378 rfi
1379 ;;
1380 END(hyper_cover)
1382 // return from metaphysical mode (meta=1) to virtual mode (meta=0)
1383 ENTRY(hyper_ssm_dt)
1384 #ifdef FAST_HYPERPRIVOP_CNT
1385 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SSM_DT);;
1386 ld4 r21=[r20];;
1387 adds r21=1,r21;;
1388 st4 [r20]=r21;;
1389 #endif
1390 mov r24=cr.ipsr
1391 mov r25=cr.iip;;
1392 adds r20=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18 ;;
1393 ld4 r21=[r20];;
1394 cmp.eq p7,p0=r21,r0 // meta==0?
1395 (p7) br.spnt.many 1f ;; // already in virtual mode
1396 movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1397 ld8 r22=[r22];;
1398 adds r22=IA64_VCPU_META_SAVED_RR0_OFFSET,r22;;
1399 ld8 r23=[r22];;
1400 mov rr[r0]=r23;;
1401 srlz.i;;
1402 st4 [r20]=r0 ;;
1403 // adjust return address to skip over break instruction
1404 1: extr.u r26=r24,41,2 ;;
1405 cmp.eq p6,p7=2,r26 ;;
1406 (p6) mov r26=0
1407 (p6) adds r25=16,r25
1408 (p7) adds r26=1,r26
1409 ;;
1410 dep r24=r26,r24,41,2
1411 ;;
1412 mov cr.ipsr=r24
1413 mov cr.iip=r25
1414 mov pr=r31,-1 ;;
1415 rfi
1416 ;;
1417 END(hyper_ssm_dt)
1419 // go to metaphysical mode (meta=1) from virtual mode (meta=0)
1420 ENTRY(hyper_rsm_dt)
1421 #ifdef FAST_HYPERPRIVOP_CNT
1422 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_RSM_DT);;
1423 ld4 r21=[r20];;
1424 adds r21=1,r21;;
1425 st4 [r20]=r21;;
1426 #endif
1427 mov r24=cr.ipsr
1428 mov r25=cr.iip;;
1429 adds r20=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18 ;;
1430 ld4 r21=[r20];;
1431 cmp.ne p7,p0=r21,r0 // meta==0?
1432 (p7) br.spnt.many 1f ;; // already in metaphysical mode
1433 movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1434 ld8 r22=[r22];;
1435 adds r22=IA64_VCPU_META_RR0_OFFSET,r22;;
1436 ld8 r23=[r22];;
1437 mov rr[r0]=r23;;
1438 srlz.i;;
1439 adds r21=1,r0 ;;
1440 st4 [r20]=r21 ;;
1441 // adjust return address to skip over break instruction
1442 1: extr.u r26=r24,41,2 ;;
1443 cmp.eq p6,p7=2,r26 ;;
1444 (p6) mov r26=0
1445 (p6) adds r25=16,r25
1446 (p7) adds r26=1,r26
1447 ;;
1448 dep r24=r26,r24,41,2
1449 ;;
1450 mov cr.ipsr=r24
1451 mov cr.iip=r25
1452 mov pr=r31,-1 ;;
1453 rfi
1454 ;;
1455 END(hyper_rsm_dt)
1457 ENTRY(hyper_set_itm)
1458 // when we get to here r20=~=interrupts pending
1459 cmp.ne p7,p0=r20,r0
1460 (p7) br.spnt.many dispatch_break_fault ;;
1461 #ifdef FAST_HYPERPRIVOP_CNT
1462 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SET_ITM);;
1463 ld4 r21=[r20];;
1464 adds r21=1,r21;;
1465 st4 [r20]=r21;;
1466 #endif
1467 movl r20=THIS_CPU(cpu_info)+IA64_CPUINFO_ITM_NEXT_OFFSET;;
1468 ld8 r21=[r20];;
1469 movl r20=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1470 ld8 r20=[r20];;
1471 adds r20=IA64_VCPU_DOMAIN_ITM_OFFSET,r20;;
1472 st8 [r20]=r8;;
1473 cmp.geu p6,p0=r21,r8;;
1474 (p6) mov r21=r8;;
1475 // now "safe set" cr.itm=r21
1476 mov r23=100;;
1477 2: mov cr.itm=r21;;
1478 srlz.d;;
1479 mov r22=ar.itc ;;
1480 cmp.leu p6,p0=r21,r22;;
1481 add r21=r21,r23;;
1482 shl r23=r23,1;;
1483 (p6) br.cond.spnt.few 2b;;
1484 1: mov r24=cr.ipsr
1485 mov r25=cr.iip;;
1486 extr.u r26=r24,41,2 ;;
1487 cmp.eq p6,p7=2,r26 ;;
1488 (p6) mov r26=0
1489 (p6) adds r25=16,r25
1490 (p7) adds r26=1,r26
1491 ;;
1492 dep r24=r26,r24,41,2
1493 ;;
1494 mov cr.ipsr=r24
1495 mov cr.iip=r25
1496 mov pr=r31,-1 ;;
1497 rfi
1498 ;;
1499 END(hyper_set_itm)
1501 ENTRY(hyper_get_psr)
1502 #ifdef FAST_HYPERPRIVOP_CNT
1503 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_GET_PSR);;
1504 ld4 r21=[r20];;
1505 adds r21=1,r21;;
1506 st4 [r20]=r21;;
1507 #endif
1508 mov r24=cr.ipsr
1509 movl r8=0x18ffffffff;;
1510 // only return PSR{36:35,31:0}
1511 and r8=r8,r24
1512 // set vpsr.ic
1513 ld4 r21=[r18];;
1514 dep r8=r21,r8,IA64_PSR_IC_BIT,1
1515 // set vpsr.pp
1516 adds r20=XSI_VPSR_PP_OFS-XSI_PSR_IC_OFS,r18 ;;
1517 ld1 r21=[r20];;
1518 dep r8=r21,r8,IA64_PSR_PP_BIT,1
1519 // set vpsr.dt
1520 adds r20=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18 ;;
1521 ld4 r21=[r20];;
1522 cmp.ne p6,p0=r21,r0
1523 ;;
1524 (p6) dep.z r8=r8,IA64_PSR_DT_BIT,1
1525 // set vpsr.i
1526 adds r20=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r18 ;;
1527 ld8 r20=[r20];;
1528 ld1 r21=[r20];;
1529 cmp.eq p8,p9=r0,r21
1530 ;;
1531 (p8) dep r8=-1,r8,IA64_PSR_I_BIT,1
1532 (p9) dep r8=0,r8,IA64_PSR_I_BIT,1
1533 // set vpsr.dfh
1534 adds r20=XSI_VPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
1535 ld1 r21=[r20];;
1536 dep r8=r21,r8,IA64_PSR_DFH_BIT,1
1537 ;;
1538 mov r25=cr.iip
1539 extr.u r26=r24,41,2 ;;
1540 cmp.eq p6,p7=2,r26 ;;
1541 (p6) mov r26=0
1542 (p6) adds r25=16,r25
1543 (p7) adds r26=1,r26
1544 ;;
1545 dep r24=r26,r24,41,2
1546 ;;
1547 mov cr.ipsr=r24
1548 mov cr.iip=r25
1549 mov pr=r31,-1 ;;
1550 rfi
1551 ;;
1552 END(hyper_get_psr)
1555 ENTRY(hyper_get_rr)
1556 #ifdef FAST_HYPERPRIVOP_CNT
1557 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_GET_RR);;
1558 ld4 r21=[r20];;
1559 adds r21=1,r21;;
1560 st4 [r20]=r21;;
1561 #endif
1562 extr.u r25=r8,61,3;;
1563 adds r20=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
1564 shl r25=r25,3;;
1565 add r20=r20,r25;;
1566 ld8 r8=[r20];;
1567 1: mov r24=cr.ipsr
1568 mov r25=cr.iip;;
1569 extr.u r26=r24,41,2 ;;
1570 cmp.eq p6,p7=2,r26 ;;
1571 (p6) mov r26=0
1572 (p6) adds r25=16,r25
1573 (p7) adds r26=1,r26
1574 ;;
1575 dep r24=r26,r24,41,2
1576 ;;
1577 mov cr.ipsr=r24
1578 mov cr.iip=r25
1579 mov pr=r31,-1 ;;
1580 rfi
1581 ;;
1582 END(hyper_get_rr)
1584 ENTRY(hyper_set_rr)
1585 extr.u r25=r8,61,3;;
1586 cmp.leu p7,p0=7,r25 // punt on setting rr7
1587 (p7) br.spnt.many dispatch_break_fault ;;
1588 #ifdef FAST_HYPERPRIVOP_CNT
1589 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SET_RR);;
1590 ld4 r21=[r20];;
1591 adds r21=1,r21;;
1592 st4 [r20]=r21;;
1593 #endif
1594 extr.u r26=r9,8,24 // r26 = r9.rid
1595 movl r20=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1596 ld8 r20=[r20];;
1597 adds r21=IA64_VCPU_STARTING_RID_OFFSET,r20;;
1598 ld4 r22=[r21];;
1599 adds r21=IA64_VCPU_ENDING_RID_OFFSET,r20;;
1600 ld4 r23=[r21];;
1601 adds r24=IA64_VCPU_META_SAVED_RR0_OFFSET,r20;;
1602 add r22=r26,r22;;
1603 cmp.geu p6,p0=r22,r23 // if r9.rid + starting_rid >= ending_rid
1604 (p6) br.cond.spnt.few 1f; // this is an error, but just ignore/return
1605 // r21=starting_rid
1606 adds r20=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
1607 shl r25=r25,3;;
1608 add r20=r20,r25;;
1609 st8 [r20]=r9;; // store away exactly what was passed
1610 // but adjust value actually placed in rr[r8]
1611 // r22 contains adjusted rid, "mangle" it (see regionreg.c)
1612 // and set ps to PAGE_SHIFT and ve to 1
1613 extr.u r27=r22,0,8
1614 extr.u r28=r22,8,8
1615 extr.u r29=r22,16,8;;
1616 dep.z r23=PAGE_SHIFT,2,6;;
1617 dep r23=-1,r23,0,1;; // mangling is swapping bytes 1 & 3
1618 dep r23=r27,r23,24,8;;
1619 dep r23=r28,r23,16,8;;
1620 dep r23=r29,r23,8,8
1621 cmp.eq p6,p0=r25,r0;; // if rr0, save for metaphysical
1622 (p6) st8 [r24]=r23
1623 mov rr[r8]=r23;;
1624 // done, mosey on back
1625 1: mov r24=cr.ipsr
1626 mov r25=cr.iip;;
1627 extr.u r26=r24,41,2 ;;
1628 cmp.eq p6,p7=2,r26 ;;
1629 (p6) mov r26=0
1630 (p6) adds r25=16,r25
1631 (p7) adds r26=1,r26
1632 ;;
1633 dep r24=r26,r24,41,2
1634 ;;
1635 mov cr.ipsr=r24
1636 mov cr.iip=r25
1637 mov pr=r31,-1 ;;
1638 rfi
1639 ;;
1640 END(hyper_set_rr)
1642 ENTRY(hyper_set_kr)
1643 extr.u r25=r8,3,61;;
1644 cmp.ne p7,p0=r0,r25 // if kr# > 7, go slow way
1645 (p7) br.spnt.many dispatch_break_fault ;;
1646 #ifdef FAST_HYPERPRIVOP_CNT
1647 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SET_KR);;
1648 ld4 r21=[r20];;
1649 adds r21=1,r21;;
1650 st4 [r20]=r21;;
1651 #endif
1652 adds r21=XSI_KR0_OFS-XSI_PSR_IC_OFS,r18 ;;
1653 shl r20=r8,3;;
1654 add r22=r20,r21;;
1655 st8 [r22]=r9;;
1656 cmp.eq p7,p0=r8,r0
1657 adds r8=-1,r8;;
1658 (p7) mov ar0=r9;;
1659 cmp.eq p7,p0=r8,r0
1660 adds r8=-1,r8;;
1661 (p7) mov ar1=r9;;
1662 cmp.eq p7,p0=r8,r0
1663 adds r8=-1,r8;;
1664 (p7) mov ar2=r9;;
1665 cmp.eq p7,p0=r8,r0
1666 adds r8=-1,r8;;
1667 (p7) mov ar3=r9;;
1668 cmp.eq p7,p0=r8,r0
1669 adds r8=-1,r8;;
1670 (p7) mov ar4=r9;;
1671 cmp.eq p7,p0=r8,r0
1672 adds r8=-1,r8;;
1673 (p7) mov ar5=r9;;
1674 cmp.eq p7,p0=r8,r0
1675 adds r8=-1,r8;;
1676 (p7) mov ar6=r9;;
1677 cmp.eq p7,p0=r8,r0
1678 adds r8=-1,r8;;
1679 (p7) mov ar7=r9;;
1680 // done, mosey on back
1681 1: mov r24=cr.ipsr
1682 mov r25=cr.iip;;
1683 extr.u r26=r24,41,2 ;;
1684 cmp.eq p6,p7=2,r26 ;;
1685 (p6) mov r26=0
1686 (p6) adds r25=16,r25
1687 (p7) adds r26=1,r26
1688 ;;
1689 dep r24=r26,r24,41,2
1690 ;;
1691 mov cr.ipsr=r24
1692 mov cr.iip=r25
1693 mov pr=r31,-1 ;;
1694 rfi
1695 ;;
1696 END(hyper_set_kr)
1698 // this routine was derived from optimized assembly output from
1699 // vcpu_thash so it is dense and difficult to read but it works
1700 // On entry:
1701 // r18 == XSI_PSR_IC
1702 // r31 == pr
1703 ENTRY(hyper_thash)
1704 #ifdef FAST_HYPERPRIVOP_CNT
1705 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_THASH);;
1706 ld4 r21=[r20];;
1707 adds r21=1,r21;;
1708 st4 [r20]=r21;;
1709 #endif
1710 shr.u r20 = r8, 61
1711 addl r25 = 1, r0
1712 movl r17 = 0xe000000000000000
1713 ;;
1714 and r21 = r17, r8 // VHPT_Addr1
1715 ;;
1716 shladd r28 = r20, 3, r18
1717 adds r19 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18
1718 ;;
1719 adds r27 = XSI_RR0_OFS-XSI_PSR_IC_OFS, r28
1720 addl r28 = 32767, r0
1721 ld8 r24 = [r19] // pta
1722 ;;
1723 ld8 r23 = [r27] // rrs[vadr>>61]
1724 extr.u r26 = r24, 2, 6
1725 ;;
1726 extr.u r22 = r23, 2, 6
1727 shl r30 = r25, r26
1728 ;;
1729 shr.u r19 = r8, r22
1730 shr.u r29 = r24, 15
1731 ;;
1732 adds r17 = -1, r30
1733 ;;
1734 shladd r27 = r19, 3, r0
1735 extr.u r26 = r17, 15, 46
1736 ;;
1737 andcm r24 = r29, r26
1738 and r19 = r28, r27
1739 shr.u r25 = r27, 15
1740 ;;
1741 and r23 = r26, r25
1742 ;;
1743 or r22 = r24, r23
1744 ;;
1745 dep.z r20 = r22, 15, 46
1746 ;;
1747 or r16 = r20, r21
1748 ;;
1749 or r8 = r19, r16
1750 // done, update iip/ipsr to next instruction
1751 mov r24=cr.ipsr
1752 mov r25=cr.iip;;
1753 extr.u r26=r24,41,2 ;;
1754 cmp.eq p6,p7=2,r26 ;;
1755 (p6) mov r26=0
1756 (p6) adds r25=16,r25
1757 (p7) adds r26=1,r26
1758 ;;
1759 dep r24=r26,r24,41,2
1760 ;;
1761 mov cr.ipsr=r24
1762 mov cr.iip=r25
1763 mov pr=r31,-1 ;;
1764 rfi
1765 ;;
1766 END(hyper_thash)
1768 ENTRY(hyper_ptc_ga)
1769 #ifndef FAST_PTC_GA
1770 br.spnt.few dispatch_break_fault ;;
1771 #endif
1772 // FIXME: validate not flushing Xen addresses
1773 #ifdef FAST_HYPERPRIVOP_CNT
1774 movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_PTC_GA);;
1775 ld4 r21=[r20];;
1776 adds r21=1,r21;;
1777 st4 [r20]=r21;;
1778 #endif
1779 mov r28=r8
1780 extr.u r19=r9,2,6 // addr_range=1<<((r9&0xfc)>>2)
1781 mov r20=1
1782 shr.u r24=r8,61
1783 addl r27=56,r0 // PAGE_SHIFT<<2 (for ptc.ga)
1784 movl r26=0x8000000000000000 // INVALID_TI_TAG
1785 mov r30=ar.lc
1786 ;;
1787 shl r19=r20,r19
1788 cmp.eq p7,p0=7,r24
1789 (p7) br.spnt.many dispatch_break_fault ;; // slow way for rr7
1790 ;;
1791 cmp.le p7,p0=r19,r0 // skip flush if size<=0
1792 (p7) br.cond.dpnt 2f ;;
1793 extr.u r24=r19,0,PAGE_SHIFT
1794 shr.u r23=r19,PAGE_SHIFT ;; // repeat loop for n pages
1795 cmp.ne p7,p0=r24,r0 ;;
1796 (p7) adds r23=1,r23 ;; // n_pages<size<n_pages+1? extra iter
1797 mov ar.lc=r23
1798 movl r29=PAGE_SIZE;;
1799 1:
1800 thash r25=r28 ;;
1801 adds r25=16,r25 ;;
1802 ld8 r24=[r25] ;;
1803 // FIXME: should check if tag matches, not just blow it away
1804 or r24=r26,r24 ;; // vhpt_entry->ti_tag = 1
1805 st8 [r25]=r24
1806 ptc.ga r28,r27 ;;
1807 srlz.i ;;
1808 add r28=r29,r28
1809 br.cloop.sptk.few 1b
1810 ;;
1811 2:
1812 mov ar.lc=r30 ;;
1813 mov r29=cr.ipsr
1814 mov r30=cr.iip;;
1815 movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1816 ld8 r27=[r27];;
1817 adds r25=IA64_VCPU_DTLB_OFFSET,r27
1818 adds r26=IA64_VCPU_ITLB_OFFSET,r27;;
1819 ld8 r24=[r25]
1820 ld8 r27=[r26] ;;
1821 and r24=-2,r24
1822 and r27=-2,r27 ;;
1823 st8 [r25]=r24 // set 1-entry i/dtlb as not present
1824 st8 [r26]=r27 ;;
1825 // increment to point to next instruction
1826 extr.u r26=r29,41,2 ;;
1827 cmp.eq p6,p7=2,r26 ;;
1828 (p6) mov r26=0
1829 (p6) adds r30=16,r30
1830 (p7) adds r26=1,r26
1831 ;;
1832 dep r29=r26,r29,41,2
1833 ;;
1834 mov cr.ipsr=r29
1835 mov cr.iip=r30
1836 mov pr=r31,-1 ;;
1837 rfi
1838 ;;
1839 END(hyper_ptc_ga)
1841 // recovery block for hyper_itc metaphysical memory lookup
1842 ENTRY(recover_and_dispatch_break_fault)
1843 #ifdef PERF_COUNTERS
1844 movl r21=PERFC(recover_to_break_fault);;
1845 ld4 r22=[r21];;
1846 adds r22=1,r22;;
1847 st4 [r21]=r22;;
1848 #endif
1849 mov b0=r29 ;;
1850 br.sptk.many dispatch_break_fault;;
1851 END(recover_and_dispatch_break_fault)
1853 // Registers at entry
1854 // r17 = break immediate (HYPERPRIVOP_ITC_D or I)
1855 // r18 == XSI_PSR_IC_OFS
1856 // r31 == pr
1857 ENTRY(hyper_itc)
1858 hyper_itc_i:
1859 // fall through, hyper_itc_d handles both i and d
1860 hyper_itc_d:
1861 #ifndef FAST_ITC
1862 br.sptk.many dispatch_break_fault ;;
1863 #else
1864 // ensure itir.ps >= xen's pagesize
1865 adds r23=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
1866 ld8 r23=[r23];;
1867 extr.u r24=r23,2,6;; // r24==logps
1868 cmp.gt p7,p0=PAGE_SHIFT,r24
1869 (p7) br.spnt.many dispatch_break_fault ;;
1870 adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
1871 ld8 r21=[r21];;
1872 // for now, punt on region0 inserts
1873 extr.u r21=r21,61,3;;
1874 cmp.eq p7,p0=r21,r0
1875 (p7) br.spnt.many dispatch_break_fault ;;
1876 movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1877 ld8 r27=[r27];;
1878 adds r27=IA64_VCPU_DOMAIN_OFFSET,r27;;
1879 ld8 r27=[r27];;
1880 // FIXME: is the global var dom0 always pinned? assume so for now
1881 movl r28=dom0;;
1882 ld8 r28=[r28];;
1883 // FIXME: for now, only handle dom0 (see lookup_domain_mpa below)
1884 cmp.ne p7,p0=r27,r28
1885 (p7) br.spnt.many dispatch_break_fault ;;
1886 #ifdef FAST_HYPERPRIVOP_CNT
1887 cmp.eq p6,p7=HYPERPRIVOP_ITC_D,r17;;
1888 (p6) movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_ITC_D);;
1889 (p7) movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_ITC_I);;
1890 ld4 r21=[r20];;
1891 adds r21=1,r21;;
1892 st4 [r20]=r21;;
1893 #endif
1894 (p6) mov r17=2;;
1895 (p7) mov r17=3;;
1896 mov r29=b0 ;;
1897 movl r30=recover_and_dispatch_break_fault ;;
1898 mov r16=r8;;
1899 // fall through
1900 #endif
1901 END(hyper_itc)
1903 #if defined(FAST_ITC) || defined (FAST_TLB_MISS_REFLECT)
1905 // fast_insert(PSCB(ifa),r24=ps,r16=pte)
1906 // r16 == pte
1907 // r17 == bit0: 1=inst, 0=data; bit1: 1=itc, 0=vcpu_translate
1908 // r18 == XSI_PSR_IC_OFS
1909 // r24 == ps
1910 // r29 == saved value of b0 in case of recovery
1911 // r30 == recovery ip if failure occurs
1912 // r31 == pr
1913 ENTRY(fast_insert)
1914 // translate_domain_pte(r16=pteval,PSCB(ifa)=address,r24=itir)
1915 mov r19=1;;
1916 shl r20=r19,r24;;
1917 adds r20=-1,r20;; // r20 == mask
1918 movl r19=_PAGE_PPN_MASK;;
1919 and r22=r16,r19;; // r22 == pteval & _PAGE_PPN_MASK
1920 andcm r19=r22,r20;;
1921 adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
1922 ld8 r21=[r21];;
1923 and r20=r21,r20;;
1924 or r19=r19,r20;; // r19 == mpaddr
1925 // FIXME: for now, just do domain0 and skip mpaddr range checks
1926 dep r20=r0,r19,0,PAGE_SHIFT
1927 movl r21=PAGE_PHYS ;;
1928 or r20=r20,r21 ;; // r20==return value from lookup_domain_mpa
1929 // r16=pteval,r20=pteval2
1930 movl r19=_PAGE_PPN_MASK
1931 movl r21=_PAGE_PL_2;;
1932 andcm r25=r16,r19;; // r25==pteval & ~_PAGE_PPN_MASK
1933 and r22=r20,r19;;
1934 or r22=r22,r21;;
1935 or r22=r22,r25;; // r22==return value from translate_domain_pte
1936 // done with translate_domain_pte
1937 // now do vcpu_itc_no_srlz(vcpu,IorD,ifa,r22=pte,r16=mppte,r24=logps)
1938 // FIXME: for now, just domain0 and skip range check
1939 // psr.ic already cleared
1940 // NOTE: r24 still contains ps (from above)
1941 shladd r24=r24,2,r0;;
1942 mov cr.itir=r24;;
1943 adds r23=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
1944 ld8 r23=[r23];;
1945 mov cr.ifa=r23;;
1946 tbit.z p6,p7=r17,0;;
1947 (p6) itc.d r22;;
1948 (p7) itc.i r22;;
1949 dv_serialize_data
1950 // vhpt_insert(r23=vaddr,r22=pte,r24=logps<<2)
1951 thash r28=r23
1952 or r26=1,r22;;
1953 ttag r21=r23
1954 adds r25=8,r28
1955 mov r19=r28;;
1956 st8 [r25]=r24
1957 adds r20=16,r28;;
1958 st8 [r19]=r26
1959 st8 [r20]=r21;;
1960 // vcpu_set_tr_entry(trp,r22=pte|1,r24=itir,r23=ifa)
1961 // TR_ENTRY = {page_flags,itir,addr,rid}
1962 tbit.z p6,p7=r17,0;;
1963 movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1964 ld8 r27=[r27];;
1965 adds r28=IA64_VCPU_STARTING_RID_OFFSET,r27
1966 (p6) adds r27=IA64_VCPU_DTLB_OFFSET,r27
1967 (p7) adds r27=IA64_VCPU_ITLB_OFFSET,r27;;
1968 st8 [r27]=r22,8;; // page_flags: already has pl >= 2 and p==1
1969 st8 [r27]=r24,8;; // itir
1970 mov r19=-4096;;
1971 and r23=r23,r19;;
1972 st8 [r27]=r23,8;; // ifa & ~0xfff
1973 adds r29 = XSI_RR0_OFS-XSI_PSR_IC_OFS,r18
1974 extr.u r25=r23,61,3;;
1975 shladd r29=r25,3,r29;;
1976 ld8 r29=[r29];;
1977 movl r20=0xffffff00;;
1978 and r29=r29,r20;;
1979 st8 [r27]=r29,-8;; // rid
1980 //if ps > 12
1981 cmp.eq p7,p0=12<<2,r24
1982 (p7) br.cond.sptk.many 1f;;
1983 // if (ps > 12) {
1984 // trp->ppn &= ~((1UL<<(ps-12))-1); trp->vadr &= ~((1UL<<ps)-1); }
1985 extr.u r29=r24,2,6
1986 mov r28=1;;
1987 shl r26=r28,r29;;
1988 adds r29=-12,r29;;
1989 shl r25=r28,r29;;
1990 mov r29=-1
1991 adds r26=-1,r26
1992 adds r25=-1,r25;;
1993 andcm r26=r29,r26 // ~((1UL<<ps)-1)
1994 andcm r25=r29,r25;; // ~((1UL<<(ps-12))-1)
1995 ld8 r29=[r27];;
1996 and r29=r29,r26;;
1997 st8 [r27]=r29,-16;;
1998 ld8 r29=[r27];;
1999 extr.u r28=r29,12,38;;
2000 movl r26=0xfffc000000000fff;;
2001 and r29=r29,r26
2002 and r28=r28,r25;;
2003 shl r28=r28,12;;
2004 or r29=r29,r28;;
2005 st8 [r27]=r29;;
2006 1: // done with vcpu_set_tr_entry
2007 //PSCBX(vcpu,i/dtlb_pte) = mp_pte
2008 movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
2009 ld8 r27=[r27];;
2010 tbit.z p6,p7=r17,0;;
2011 (p6) adds r27=IA64_VCPU_DTLB_PTE_OFFSET,r27
2012 (p7) adds r27=IA64_VCPU_ITLB_PTE_OFFSET,r27;;
2013 st8 [r27]=r16;;
2014 // done with vcpu_itc_no_srlz
2016 // if hyper_itc, increment to point to next instruction
2017 tbit.z p7,p0=r17,1
2018 (p7) br.cond.sptk.few no_inc_iip;;
2020 mov r29=cr.ipsr
2021 mov r30=cr.iip;;
2022 extr.u r26=r29,41,2 ;;
2023 cmp.eq p6,p7=2,r26 ;;
2024 (p6) mov r26=0
2025 (p6) adds r30=16,r30
2026 (p7) adds r26=1,r26
2027 ;;
2028 dep r29=r26,r29,41,2
2029 ;;
2030 mov cr.ipsr=r29
2031 mov cr.iip=r30;;
2033 no_inc_iip:
2034 mov pr=r31,-1 ;;
2035 rfi
2036 ;;
2037 END(fast_insert)
2038 #endif