ia64/xen-unstable

view linux-2.6-xen-sparse/arch/ia64/xen/xenentry.S @ 9296:f85bb99187bf

Update interface documentation to include sched_op_new hypercall
and clean up the style a bit. Also clean up the sched_op_new
description in the sched.h public header.

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Wed Mar 15 19:19:22 2006 +0100 (2006-03-15)
parents 3b9c2c410b14
children 2b6e531dab38
line source
1 /*
2 * ia64/xen/entry.S
3 *
4 * Alternate kernel routines for Xen. Heavily leveraged from
5 * ia64/kernel/entry.S
6 *
7 * Copyright (C) 2005 Hewlett-Packard Co
8 * Dan Magenheimer <dan.magenheimer@.hp.com>
9 */
11 #include <linux/config.h>
13 #include <asm/asmmacro.h>
14 #include <asm/cache.h>
15 #include <asm/errno.h>
16 #include <asm/kregs.h>
17 #include <asm/asm-offsets.h>
18 #include <asm/pgtable.h>
19 #include <asm/percpu.h>
20 #include <asm/processor.h>
21 #include <asm/thread_info.h>
22 #include <asm/unistd.h>
24 #ifdef CONFIG_XEN
25 #include "xenminstate.h"
26 #else
27 #include "minstate.h"
28 #endif
30 /*
31 * prev_task <- ia64_switch_to(struct task_struct *next)
32 * With Ingo's new scheduler, interrupts are disabled when this routine gets
33 * called. The code starting at .map relies on this. The rest of the code
34 * doesn't care about the interrupt masking status.
35 */
36 #ifdef CONFIG_XEN
37 GLOBAL_ENTRY(xen_switch_to)
38 .prologue
39 alloc r16=ar.pfs,1,0,0,0
40 movl r22=running_on_xen;;
41 ld4 r22=[r22];;
42 cmp.eq p7,p0=r22,r0
43 (p7) br.cond.sptk.many __ia64_switch_to;;
44 #else
45 GLOBAL_ENTRY(ia64_switch_to)
46 .prologue
47 alloc r16=ar.pfs,1,0,0,0
48 #endif
49 DO_SAVE_SWITCH_STACK
50 .body
52 adds r22=IA64_TASK_THREAD_KSP_OFFSET,r13
53 movl r25=init_task
54 mov r27=IA64_KR(CURRENT_STACK)
55 adds r21=IA64_TASK_THREAD_KSP_OFFSET,in0
56 dep r20=0,in0,61,3 // physical address of "next"
57 ;;
58 st8 [r22]=sp // save kernel stack pointer of old task
59 shr.u r26=r20,IA64_GRANULE_SHIFT
60 cmp.eq p7,p6=r25,in0
61 ;;
62 #ifdef CONFIG_XEN
63 movl r8=XSI_PSR_IC
64 ;;
65 st4 [r8]=r0 // force psr.ic off for hyperprivop(s)
66 ;;
67 #endif
68 /*
69 * If we've already mapped this task's page, we can skip doing it again.
70 */
71 (p6) cmp.eq p7,p6=r26,r27
72 (p6) br.cond.dpnt .map
73 ;;
74 .done:
75 #ifdef CONFIG_XEN
76 // psr.ic already off
77 // update "current" application register
78 mov r8=IA64_KR_CURRENT
79 mov r9=in0;;
80 XEN_HYPER_SET_KR
81 ld8 sp=[r21] // load kernel stack pointer of new task
82 movl r27=XSI_PSR_IC
83 mov r8=1
84 ;;
85 st4 [r27]=r8 // psr.ic back on
86 ;;
87 #else
88 (p6) ssm psr.ic // if we had to map, reenable the psr.ic bit FIRST!!!
89 ;;
90 (p6) srlz.d
91 ld8 sp=[r21] // load kernel stack pointer of new task
92 mov IA64_KR(CURRENT)=in0 // update "current" application register
93 #endif
94 mov r8=r13 // return pointer to previously running task
95 mov r13=in0 // set "current" pointer
96 ;;
97 DO_LOAD_SWITCH_STACK
99 #ifdef CONFIG_SMP
100 sync.i // ensure "fc"s done by this CPU are visible on other CPUs
101 #endif
102 br.ret.sptk.many rp // boogie on out in new context
104 .map:
105 #ifdef CONFIG_XEN
106 // psr.ic already off
107 #else
108 rsm psr.ic // interrupts (psr.i) are already disabled here
109 #endif
110 movl r25=PAGE_KERNEL
111 ;;
112 srlz.d
113 or r23=r25,r20 // construct PA | page properties
114 mov r25=IA64_GRANULE_SHIFT<<2
115 ;;
116 #ifdef CONFIG_XEN
117 movl r8=XSI_ITIR
118 ;;
119 st8 [r8]=r25
120 ;;
121 movl r8=XSI_IFA
122 ;;
123 st8 [r8]=in0 // VA of next task...
124 ;;
125 mov r25=IA64_TR_CURRENT_STACK
126 // remember last page we mapped...
127 mov r8=IA64_KR_CURRENT_STACK
128 mov r9=r26;;
129 XEN_HYPER_SET_KR;;
130 #else
131 mov cr.itir=r25
132 mov cr.ifa=in0 // VA of next task...
133 ;;
134 mov r25=IA64_TR_CURRENT_STACK
135 mov IA64_KR(CURRENT_STACK)=r26 // remember last page we mapped...
136 #endif
137 ;;
138 itr.d dtr[r25]=r23 // wire in new mapping...
139 br.cond.sptk .done
140 #ifdef CONFIG_XEN
141 END(xen_switch_to)
142 #else
143 END(ia64_switch_to)
144 #endif
146 /*
147 * Invoke a system call, but do some tracing before and after the call.
148 * We MUST preserve the current register frame throughout this routine
149 * because some system calls (such as ia64_execve) directly
150 * manipulate ar.pfs.
151 */
152 #ifdef CONFIG_XEN
153 GLOBAL_ENTRY(xen_trace_syscall)
154 PT_REGS_UNWIND_INFO(0)
155 movl r16=running_on_xen;;
156 ld4 r16=[r16];;
157 cmp.eq p7,p0=r16,r0
158 (p7) br.cond.sptk.many __ia64_trace_syscall;;
159 #else
160 GLOBAL_ENTRY(ia64_trace_syscall)
161 PT_REGS_UNWIND_INFO(0)
162 #endif
163 /*
164 * We need to preserve the scratch registers f6-f11 in case the system
165 * call is sigreturn.
166 */
167 adds r16=PT(F6)+16,sp
168 adds r17=PT(F7)+16,sp
169 ;;
170 stf.spill [r16]=f6,32
171 stf.spill [r17]=f7,32
172 ;;
173 stf.spill [r16]=f8,32
174 stf.spill [r17]=f9,32
175 ;;
176 stf.spill [r16]=f10
177 stf.spill [r17]=f11
178 br.call.sptk.many rp=syscall_trace_enter // give parent a chance to catch syscall args
179 adds r16=PT(F6)+16,sp
180 adds r17=PT(F7)+16,sp
181 ;;
182 ldf.fill f6=[r16],32
183 ldf.fill f7=[r17],32
184 ;;
185 ldf.fill f8=[r16],32
186 ldf.fill f9=[r17],32
187 ;;
188 ldf.fill f10=[r16]
189 ldf.fill f11=[r17]
190 // the syscall number may have changed, so re-load it and re-calculate the
191 // syscall entry-point:
192 adds r15=PT(R15)+16,sp // r15 = &pt_regs.r15 (syscall #)
193 ;;
194 ld8 r15=[r15]
195 mov r3=NR_syscalls - 1
196 ;;
197 adds r15=-1024,r15
198 movl r16=sys_call_table
199 ;;
200 shladd r20=r15,3,r16 // r20 = sys_call_table + 8*(syscall-1024)
201 cmp.leu p6,p7=r15,r3
202 ;;
203 (p6) ld8 r20=[r20] // load address of syscall entry point
204 (p7) movl r20=sys_ni_syscall
205 ;;
206 mov b6=r20
207 br.call.sptk.many rp=b6 // do the syscall
208 .strace_check_retval:
209 cmp.lt p6,p0=r8,r0 // syscall failed?
210 adds r2=PT(R8)+16,sp // r2 = &pt_regs.r8
211 adds r3=PT(R10)+16,sp // r3 = &pt_regs.r10
212 mov r10=0
213 (p6) br.cond.sptk strace_error // syscall failed ->
214 ;; // avoid RAW on r10
215 .strace_save_retval:
216 .mem.offset 0,0; st8.spill [r2]=r8 // store return value in slot for r8
217 .mem.offset 8,0; st8.spill [r3]=r10 // clear error indication in slot for r10
218 br.call.sptk.many rp=syscall_trace_leave // give parent a chance to catch return value
219 .ret3: br.cond.sptk .work_pending_syscall_end
221 strace_error:
222 ld8 r3=[r2] // load pt_regs.r8
223 sub r9=0,r8 // negate return value to get errno value
224 ;;
225 cmp.ne p6,p0=r3,r0 // is pt_regs.r8!=0?
226 adds r3=16,r2 // r3=&pt_regs.r10
227 ;;
228 (p6) mov r10=-1
229 (p6) mov r8=r9
230 br.cond.sptk .strace_save_retval
231 #ifdef CONFIG_XEN
232 END(xen_trace_syscall)
233 #else
234 END(ia64_trace_syscall)
235 #endif
237 /*
238 * ia64_leave_syscall(): Same as ia64_leave_kernel, except that it doesn't
239 * need to switch to bank 0 and doesn't restore the scratch registers.
240 * To avoid leaking kernel bits, the scratch registers are set to
241 * the following known-to-be-safe values:
242 *
243 * r1: restored (global pointer)
244 * r2: cleared
245 * r3: 1 (when returning to user-level)
246 * r8-r11: restored (syscall return value(s))
247 * r12: restored (user-level stack pointer)
248 * r13: restored (user-level thread pointer)
249 * r14: cleared
250 * r15: restored (syscall #)
251 * r16-r17: cleared
252 * r18: user-level b6
253 * r19: cleared
254 * r20: user-level ar.fpsr
255 * r21: user-level b0
256 * r22: cleared
257 * r23: user-level ar.bspstore
258 * r24: user-level ar.rnat
259 * r25: user-level ar.unat
260 * r26: user-level ar.pfs
261 * r27: user-level ar.rsc
262 * r28: user-level ip
263 * r29: user-level psr
264 * r30: user-level cfm
265 * r31: user-level pr
266 * f6-f11: cleared
267 * pr: restored (user-level pr)
268 * b0: restored (user-level rp)
269 * b6: restored
270 * b7: cleared
271 * ar.unat: restored (user-level ar.unat)
272 * ar.pfs: restored (user-level ar.pfs)
273 * ar.rsc: restored (user-level ar.rsc)
274 * ar.rnat: restored (user-level ar.rnat)
275 * ar.bspstore: restored (user-level ar.bspstore)
276 * ar.fpsr: restored (user-level ar.fpsr)
277 * ar.ccv: cleared
278 * ar.csd: cleared
279 * ar.ssd: cleared
280 */
281 #ifdef CONFIG_XEN
282 GLOBAL_ENTRY(xen_leave_syscall)
283 PT_REGS_UNWIND_INFO(0)
284 movl r22=running_on_xen;;
285 ld4 r22=[r22];;
286 cmp.eq p7,p0=r22,r0
287 (p7) br.cond.sptk.many __ia64_leave_syscall;;
288 #else
289 ENTRY(ia64_leave_syscall)
290 PT_REGS_UNWIND_INFO(0)
291 #endif
292 /*
293 * work.need_resched etc. mustn't get changed by this CPU before it returns to
294 * user- or fsys-mode, hence we disable interrupts early on.
295 *
296 * p6 controls whether current_thread_info()->flags needs to be check for
297 * extra work. We always check for extra work when returning to user-level.
298 * With CONFIG_PREEMPT, we also check for extra work when the preempt_count
299 * is 0. After extra work processing has been completed, execution
300 * resumes at .work_processed_syscall with p6 set to 1 if the extra-work-check
301 * needs to be redone.
302 */
303 #ifdef CONFIG_PREEMPT
304 rsm psr.i // disable interrupts
305 cmp.eq pLvSys,p0=r0,r0 // pLvSys=1: leave from syscall
306 (pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13
307 ;;
308 .pred.rel.mutex pUStk,pKStk
309 (pKStk) ld4 r21=[r20] // r21 <- preempt_count
310 (pUStk) mov r21=0 // r21 <- 0
311 ;;
312 cmp.eq p6,p0=r21,r0 // p6 <- pUStk || (preempt_count == 0)
313 #else /* !CONFIG_PREEMPT */
314 #ifdef CONFIG_XEN
315 movl r2=XSI_PSR_I
316 ;;
317 (pUStk) st4 [r2]=r0
318 #else
319 (pUStk) rsm psr.i
320 #endif
321 cmp.eq pLvSys,p0=r0,r0 // pLvSys=1: leave from syscall
322 (pUStk) cmp.eq.unc p6,p0=r0,r0 // p6 <- pUStk
323 #endif
324 .work_processed_syscall:
325 adds r2=PT(LOADRS)+16,r12
326 adds r3=PT(AR_BSPSTORE)+16,r12
327 adds r18=TI_FLAGS+IA64_TASK_SIZE,r13
328 ;;
329 (p6) ld4 r31=[r18] // load current_thread_info()->flags
330 ld8 r19=[r2],PT(B6)-PT(LOADRS) // load ar.rsc value for "loadrs"
331 mov b7=r0 // clear b7
332 ;;
333 ld8 r23=[r3],PT(R11)-PT(AR_BSPSTORE) // load ar.bspstore (may be garbage)
334 ld8 r18=[r2],PT(R9)-PT(B6) // load b6
335 (p6) and r15=TIF_WORK_MASK,r31 // any work other than TIF_SYSCALL_TRACE?
336 ;;
337 mov r16=ar.bsp // M2 get existing backing store pointer
338 (p6) cmp4.ne.unc p6,p0=r15, r0 // any special work pending?
339 (p6) br.cond.spnt .work_pending_syscall
340 ;;
341 // start restoring the state saved on the kernel stack (struct pt_regs):
342 ld8 r9=[r2],PT(CR_IPSR)-PT(R9)
343 ld8 r11=[r3],PT(CR_IIP)-PT(R11)
344 mov f6=f0 // clear f6
345 ;;
346 invala // M0|1 invalidate ALAT
347 #ifdef CONFIG_XEN
348 movl r29=XSI_PSR_IC
349 ;;
350 st8 [r29]=r0 // note: clears both vpsr.i and vpsr.ic!
351 ;;
352 #else
353 rsm psr.i | psr.ic // M2 initiate turning off of interrupt and interruption collection
354 #endif
355 mov f9=f0 // clear f9
357 ld8 r29=[r2],16 // load cr.ipsr
358 ld8 r28=[r3],16 // load cr.iip
359 mov f8=f0 // clear f8
360 ;;
361 ld8 r30=[r2],16 // M0|1 load cr.ifs
362 mov.m ar.ssd=r0 // M2 clear ar.ssd
363 cmp.eq p9,p0=r0,r0 // set p9 to indicate that we should restore cr.ifs
364 ;;
365 ld8 r25=[r3],16 // M0|1 load ar.unat
366 mov.m ar.csd=r0 // M2 clear ar.csd
367 mov r22=r0 // clear r22
368 ;;
369 ld8 r26=[r2],PT(B0)-PT(AR_PFS) // M0|1 load ar.pfs
370 (pKStk) mov r22=psr // M2 read PSR now that interrupts are disabled
371 mov f10=f0 // clear f10
372 ;;
373 ld8 r21=[r2],PT(AR_RNAT)-PT(B0) // load b0
374 ld8 r27=[r3],PT(PR)-PT(AR_RSC) // load ar.rsc
375 mov f11=f0 // clear f11
376 ;;
377 ld8 r24=[r2],PT(AR_FPSR)-PT(AR_RNAT) // load ar.rnat (may be garbage)
378 ld8 r31=[r3],PT(R1)-PT(PR) // load predicates
379 (pUStk) add r14=IA64_TASK_THREAD_ON_USTACK_OFFSET,r13
380 ;;
381 ld8 r20=[r2],PT(R12)-PT(AR_FPSR) // load ar.fpsr
382 ld8.fill r1=[r3],16 // load r1
383 (pUStk) mov r17=1
384 ;;
385 srlz.d // M0 ensure interruption collection is off
386 ld8.fill r13=[r3],16
387 mov f7=f0 // clear f7
388 ;;
389 ld8.fill r12=[r2] // restore r12 (sp)
390 ld8.fill r15=[r3] // restore r15
391 addl r3=THIS_CPU(ia64_phys_stacked_size_p8),r0
392 ;;
393 (pUStk) ld4 r3=[r3] // r3 = cpu_data->phys_stacked_size_p8
394 (pUStk) st1 [r14]=r17
395 mov b6=r18 // I0 restore b6
396 ;;
397 mov r14=r0 // clear r14
398 shr.u r18=r19,16 // I0|1 get byte size of existing "dirty" partition
399 (pKStk) br.cond.dpnt.many skip_rbs_switch
401 mov.m ar.ccv=r0 // clear ar.ccv
402 (pNonSys) br.cond.dpnt.many dont_preserve_current_frame
403 br.cond.sptk.many rbs_switch
404 #ifdef CONFIG_XEN
405 END(xen_leave_syscall)
406 #else
407 END(ia64_leave_syscall)
408 #endif
410 #ifdef CONFIG_XEN
411 GLOBAL_ENTRY(xen_leave_kernel)
412 PT_REGS_UNWIND_INFO(0)
413 movl r22=running_on_xen;;
414 ld4 r22=[r22];;
415 cmp.eq p7,p0=r22,r0
416 (p7) br.cond.sptk.many __ia64_leave_kernel;;
417 #else
418 GLOBAL_ENTRY(ia64_leave_kernel)
419 PT_REGS_UNWIND_INFO(0)
420 #endif
421 /*
422 * work.need_resched etc. mustn't get changed by this CPU before it returns to
423 * user- or fsys-mode, hence we disable interrupts early on.
424 *
425 * p6 controls whether current_thread_info()->flags needs to be check for
426 * extra work. We always check for extra work when returning to user-level.
427 * With CONFIG_PREEMPT, we also check for extra work when the preempt_count
428 * is 0. After extra work processing has been completed, execution
429 * resumes at .work_processed_syscall with p6 set to 1 if the extra-work-check
430 * needs to be redone.
431 */
432 #ifdef CONFIG_PREEMPT
433 rsm psr.i // disable interrupts
434 cmp.eq p0,pLvSys=r0,r0 // pLvSys=0: leave from kernel
435 (pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13
436 ;;
437 .pred.rel.mutex pUStk,pKStk
438 (pKStk) ld4 r21=[r20] // r21 <- preempt_count
439 (pUStk) mov r21=0 // r21 <- 0
440 ;;
441 cmp.eq p6,p0=r21,r0 // p6 <- pUStk || (preempt_count == 0)
442 #else
443 #ifdef CONFIG_XEN
444 (pUStk) movl r17=XSI_PSR_I
445 ;;
446 (pUStk) st4 [r17]=r0
447 ;;
448 #else
449 (pUStk) rsm psr.i
450 #endif
451 cmp.eq p0,pLvSys=r0,r0 // pLvSys=0: leave from kernel
452 (pUStk) cmp.eq.unc p6,p0=r0,r0 // p6 <- pUStk
453 #endif
454 .work_processed_kernel:
455 adds r17=TI_FLAGS+IA64_TASK_SIZE,r13
456 ;;
457 (p6) ld4 r31=[r17] // load current_thread_info()->flags
458 adds r21=PT(PR)+16,r12
459 ;;
461 lfetch [r21],PT(CR_IPSR)-PT(PR)
462 adds r2=PT(B6)+16,r12
463 adds r3=PT(R16)+16,r12
464 ;;
465 lfetch [r21]
466 ld8 r28=[r2],8 // load b6
467 adds r29=PT(R24)+16,r12
469 ld8.fill r16=[r3],PT(AR_CSD)-PT(R16)
470 adds r30=PT(AR_CCV)+16,r12
471 (p6) and r19=TIF_WORK_MASK,r31 // any work other than TIF_SYSCALL_TRACE?
472 ;;
473 ld8.fill r24=[r29]
474 ld8 r15=[r30] // load ar.ccv
475 (p6) cmp4.ne.unc p6,p0=r19, r0 // any special work pending?
476 ;;
477 ld8 r29=[r2],16 // load b7
478 ld8 r30=[r3],16 // load ar.csd
479 (p6) br.cond.spnt .work_pending
480 ;;
481 ld8 r31=[r2],16 // load ar.ssd
482 ld8.fill r8=[r3],16
483 ;;
484 ld8.fill r9=[r2],16
485 ld8.fill r10=[r3],PT(R17)-PT(R10)
486 ;;
487 ld8.fill r11=[r2],PT(R18)-PT(R11)
488 ld8.fill r17=[r3],16
489 ;;
490 ld8.fill r18=[r2],16
491 ld8.fill r19=[r3],16
492 ;;
493 ld8.fill r20=[r2],16
494 ld8.fill r21=[r3],16
495 mov ar.csd=r30
496 mov ar.ssd=r31
497 ;;
498 #ifdef CONFIG_XEN
499 movl r22=XSI_PSR_IC
500 ;;
501 st8 [r22]=r0 // note: clears both vpsr.i and vpsr.ic!
502 ;;
503 #else
504 rsm psr.i | psr.ic // initiate turning off of interrupt and interruption collection
505 #endif
506 invala // invalidate ALAT
507 ;;
508 ld8.fill r22=[r2],24
509 ld8.fill r23=[r3],24
510 mov b6=r28
511 ;;
512 ld8.fill r25=[r2],16
513 ld8.fill r26=[r3],16
514 mov b7=r29
515 ;;
516 ld8.fill r27=[r2],16
517 ld8.fill r28=[r3],16
518 ;;
519 ld8.fill r29=[r2],16
520 ld8.fill r30=[r3],24
521 ;;
522 ld8.fill r31=[r2],PT(F9)-PT(R31)
523 adds r3=PT(F10)-PT(F6),r3
524 ;;
525 ldf.fill f9=[r2],PT(F6)-PT(F9)
526 ldf.fill f10=[r3],PT(F8)-PT(F10)
527 ;;
528 ldf.fill f6=[r2],PT(F7)-PT(F6)
529 ;;
530 ldf.fill f7=[r2],PT(F11)-PT(F7)
531 ldf.fill f8=[r3],32
532 ;;
533 srlz.i // ensure interruption collection is off
534 mov ar.ccv=r15
535 ;;
536 ldf.fill f11=[r2]
537 #ifdef CONFIG_XEN
538 ;;
539 // r16-r31 all now hold bank1 values
540 movl r2=XSI_BANK1_R16
541 movl r3=XSI_BANK1_R16+8
542 ;;
543 st8.spill [r2]=r16,16
544 st8.spill [r3]=r17,16
545 ;;
546 st8.spill [r2]=r18,16
547 st8.spill [r3]=r19,16
548 ;;
549 st8.spill [r2]=r20,16
550 st8.spill [r3]=r21,16
551 ;;
552 st8.spill [r2]=r22,16
553 st8.spill [r3]=r23,16
554 ;;
555 st8.spill [r2]=r24,16
556 st8.spill [r3]=r25,16
557 ;;
558 st8.spill [r2]=r26,16
559 st8.spill [r3]=r27,16
560 ;;
561 st8.spill [r2]=r28,16
562 st8.spill [r3]=r29,16
563 ;;
564 st8.spill [r2]=r30,16
565 st8.spill [r3]=r31,16
566 ;;
567 movl r2=XSI_BANKNUM;;
568 st4 [r2]=r0;
569 #else
570 bsw.0 // switch back to bank 0 (no stop bit required beforehand...)
571 #endif
572 ;;
573 (pUStk) mov r18=IA64_KR(CURRENT)// M2 (12 cycle read latency)
574 adds r16=PT(CR_IPSR)+16,r12
575 adds r17=PT(CR_IIP)+16,r12
577 (pKStk) mov r22=psr // M2 read PSR now that interrupts are disabled
578 nop.i 0
579 nop.i 0
580 ;;
581 ld8 r29=[r16],16 // load cr.ipsr
582 ld8 r28=[r17],16 // load cr.iip
583 ;;
584 ld8 r30=[r16],16 // load cr.ifs
585 ld8 r25=[r17],16 // load ar.unat
586 ;;
587 ld8 r26=[r16],16 // load ar.pfs
588 ld8 r27=[r17],16 // load ar.rsc
589 cmp.eq p9,p0=r0,r0 // set p9 to indicate that we should restore cr.ifs
590 ;;
591 ld8 r24=[r16],16 // load ar.rnat (may be garbage)
592 ld8 r23=[r17],16 // load ar.bspstore (may be garbage)
593 ;;
594 ld8 r31=[r16],16 // load predicates
595 ld8 r21=[r17],16 // load b0
596 ;;
597 ld8 r19=[r16],16 // load ar.rsc value for "loadrs"
598 ld8.fill r1=[r17],16 // load r1
599 ;;
600 ld8.fill r12=[r16],16
601 ld8.fill r13=[r17],16
602 (pUStk) adds r18=IA64_TASK_THREAD_ON_USTACK_OFFSET,r18
603 ;;
604 ld8 r20=[r16],16 // ar.fpsr
605 ld8.fill r15=[r17],16
606 ;;
607 ld8.fill r14=[r16],16
608 ld8.fill r2=[r17]
609 (pUStk) mov r17=1
610 ;;
611 ld8.fill r3=[r16]
612 (pUStk) st1 [r18]=r17 // restore current->thread.on_ustack
613 shr.u r18=r19,16 // get byte size of existing "dirty" partition
614 ;;
615 mov r16=ar.bsp // get existing backing store pointer
616 addl r17=THIS_CPU(ia64_phys_stacked_size_p8),r0
617 ;;
618 ld4 r17=[r17] // r17 = cpu_data->phys_stacked_size_p8
619 (pKStk) br.cond.dpnt skip_rbs_switch
621 /*
622 * Restore user backing store.
623 *
624 * NOTE: alloc, loadrs, and cover can't be predicated.
625 */
626 (pNonSys) br.cond.dpnt dont_preserve_current_frame
628 rbs_switch:
629 #ifdef CONFIG_XEN
630 XEN_HYPER_COVER;
631 #else
632 cover // add current frame into dirty partition and set cr.ifs
633 #endif
634 ;;
635 mov r19=ar.bsp // get new backing store pointer
636 sub r16=r16,r18 // krbs = old bsp - size of dirty partition
637 cmp.ne p9,p0=r0,r0 // clear p9 to skip restore of cr.ifs
638 ;;
639 sub r19=r19,r16 // calculate total byte size of dirty partition
640 add r18=64,r18 // don't force in0-in7 into memory...
641 ;;
642 shl r19=r19,16 // shift size of dirty partition into loadrs position
643 ;;
644 dont_preserve_current_frame:
645 /*
646 * To prevent leaking bits between the kernel and user-space,
647 * we must clear the stacked registers in the "invalid" partition here.
648 * Not pretty, but at least it's fast (3.34 registers/cycle on Itanium,
649 * 5 registers/cycle on McKinley).
650 */
651 # define pRecurse p6
652 # define pReturn p7
653 #ifdef CONFIG_ITANIUM
654 # define Nregs 10
655 #else
656 # define Nregs 14
657 #endif
658 alloc loc0=ar.pfs,2,Nregs-2,2,0
659 shr.u loc1=r18,9 // RNaTslots <= floor(dirtySize / (64*8))
660 sub r17=r17,r18 // r17 = (physStackedSize + 8) - dirtySize
661 ;;
662 mov ar.rsc=r19 // load ar.rsc to be used for "loadrs"
663 shladd in0=loc1,3,r17
664 mov in1=0
665 ;;
666 TEXT_ALIGN(32)
667 rse_clear_invalid:
668 #ifdef CONFIG_ITANIUM
669 // cycle 0
670 { .mii
671 alloc loc0=ar.pfs,2,Nregs-2,2,0
672 cmp.lt pRecurse,p0=Nregs*8,in0 // if more than Nregs regs left to clear, (re)curse
673 add out0=-Nregs*8,in0
674 }{ .mfb
675 add out1=1,in1 // increment recursion count
676 nop.f 0
677 nop.b 0 // can't do br.call here because of alloc (WAW on CFM)
678 ;;
679 }{ .mfi // cycle 1
680 mov loc1=0
681 nop.f 0
682 mov loc2=0
683 }{ .mib
684 mov loc3=0
685 mov loc4=0
686 (pRecurse) br.call.sptk.many b0=rse_clear_invalid
688 }{ .mfi // cycle 2
689 mov loc5=0
690 nop.f 0
691 cmp.ne pReturn,p0=r0,in1 // if recursion count != 0, we need to do a br.ret
692 }{ .mib
693 mov loc6=0
694 mov loc7=0
695 (pReturn) br.ret.sptk.many b0
696 }
697 #else /* !CONFIG_ITANIUM */
698 alloc loc0=ar.pfs,2,Nregs-2,2,0
699 cmp.lt pRecurse,p0=Nregs*8,in0 // if more than Nregs regs left to clear, (re)curse
700 add out0=-Nregs*8,in0
701 add out1=1,in1 // increment recursion count
702 mov loc1=0
703 mov loc2=0
704 ;;
705 mov loc3=0
706 mov loc4=0
707 mov loc5=0
708 mov loc6=0
709 mov loc7=0
710 (pRecurse) br.call.sptk.few b0=rse_clear_invalid
711 ;;
712 mov loc8=0
713 mov loc9=0
714 cmp.ne pReturn,p0=r0,in1 // if recursion count != 0, we need to do a br.ret
715 mov loc10=0
716 mov loc11=0
717 (pReturn) br.ret.sptk.many b0
718 #endif /* !CONFIG_ITANIUM */
719 # undef pRecurse
720 # undef pReturn
721 ;;
722 alloc r17=ar.pfs,0,0,0,0 // drop current register frame
723 ;;
724 loadrs
725 ;;
726 skip_rbs_switch:
727 mov ar.unat=r25 // M2
728 (pKStk) extr.u r22=r22,21,1 // I0 extract current value of psr.pp from r22
729 (pLvSys)mov r19=r0 // A clear r19 for leave_syscall, no-op otherwise
730 ;;
731 (pUStk) mov ar.bspstore=r23 // M2
732 (pKStk) dep r29=r22,r29,21,1 // I0 update ipsr.pp with psr.pp
733 (pLvSys)mov r16=r0 // A clear r16 for leave_syscall, no-op otherwise
734 ;;
735 #ifdef CONFIG_XEN
736 movl r25=XSI_IPSR
737 ;;
738 st8[r25]=r29,XSI_IFS-XSI_IPSR
739 ;;
740 #else
741 mov cr.ipsr=r29 // M2
742 #endif
743 mov ar.pfs=r26 // I0
744 (pLvSys)mov r17=r0 // A clear r17 for leave_syscall, no-op otherwise
746 #ifdef CONFIG_XEN
747 (p9) st8 [r25]=r30
748 ;;
749 adds r25=XSI_IIP-XSI_IFS,r25
750 ;;
751 #else
752 (p9) mov cr.ifs=r30 // M2
753 #endif
754 mov b0=r21 // I0
755 (pLvSys)mov r18=r0 // A clear r18 for leave_syscall, no-op otherwise
757 mov ar.fpsr=r20 // M2
758 #ifdef CONFIG_XEN
759 st8 [r25]=r28
760 #else
761 mov cr.iip=r28 // M2
762 #endif
763 nop 0
764 ;;
765 (pUStk) mov ar.rnat=r24 // M2 must happen with RSE in lazy mode
766 nop 0
767 (pLvSys)mov r2=r0
769 mov ar.rsc=r27 // M2
770 mov pr=r31,-1 // I0
771 #ifdef CONFIG_XEN
772 ;;
773 XEN_HYPER_RFI;
774 #else
775 rfi // B
776 #endif
778 /*
779 * On entry:
780 * r20 = &current->thread_info->pre_count (if CONFIG_PREEMPT)
781 * r31 = current->thread_info->flags
782 * On exit:
783 * p6 = TRUE if work-pending-check needs to be redone
784 */
785 .work_pending_syscall:
786 add r2=-8,r2
787 add r3=-8,r3
788 ;;
789 st8 [r2]=r8
790 st8 [r3]=r10
791 .work_pending:
792 tbit.nz p6,p0=r31,TIF_SIGDELAYED // signal delayed from MCA/INIT/NMI/PMI context?
793 (p6) br.cond.sptk.few .sigdelayed
794 ;;
795 tbit.z p6,p0=r31,TIF_NEED_RESCHED // current_thread_info()->need_resched==0?
796 (p6) br.cond.sptk.few .notify
797 #ifdef CONFIG_PREEMPT
798 (pKStk) dep r21=-1,r0,PREEMPT_ACTIVE_BIT,1
799 ;;
800 (pKStk) st4 [r20]=r21
801 ssm psr.i // enable interrupts
802 #endif
803 br.call.spnt.many rp=schedule
804 .ret9: cmp.eq p6,p0=r0,r0 // p6 <- 1
805 #ifdef CONFIG_XEN
806 movl r2=XSI_PSR_I
807 ;;
808 st4 [r2]=r0
809 #else
810 rsm psr.i // disable interrupts
811 #endif
812 ;;
813 #ifdef CONFIG_PREEMPT
814 (pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13
815 ;;
816 (pKStk) st4 [r20]=r0 // preempt_count() <- 0
817 #endif
818 (pLvSys)br.cond.sptk.few .work_pending_syscall_end
819 br.cond.sptk.many .work_processed_kernel // re-check
821 .notify:
822 (pUStk) br.call.spnt.many rp=notify_resume_user
823 .ret10: cmp.ne p6,p0=r0,r0 // p6 <- 0
824 (pLvSys)br.cond.sptk.few .work_pending_syscall_end
825 br.cond.sptk.many .work_processed_kernel // don't re-check
827 // There is a delayed signal that was detected in MCA/INIT/NMI/PMI context where
828 // it could not be delivered. Deliver it now. The signal might be for us and
829 // may set TIF_SIGPENDING, so redrive ia64_leave_* after processing the delayed
830 // signal.
832 .sigdelayed:
833 br.call.sptk.many rp=do_sigdelayed
834 cmp.eq p6,p0=r0,r0 // p6 <- 1, always re-check
835 (pLvSys)br.cond.sptk.few .work_pending_syscall_end
836 br.cond.sptk.many .work_processed_kernel // re-check
838 .work_pending_syscall_end:
839 adds r2=PT(R8)+16,r12
840 adds r3=PT(R10)+16,r12
841 ;;
842 ld8 r8=[r2]
843 ld8 r10=[r3]
844 br.cond.sptk.many .work_processed_syscall // re-check
846 #ifdef CONFIG_XEN
847 END(xen_leave_kernel)
848 #else
849 END(ia64_leave_kernel)
850 #endif