direct-io.hg

changeset 13485:7a2c224a9252

[IA64] Add fsys.S to sparse tree

From 2.6.16.33

Signed-off-by: Alex Williamson <alex.williamson@hp.com>
author awilliam@xenbuild2.aw
date Tue Jan 16 11:51:45 2007 -0700 (2007-01-16)
parents 1d72428a0fab
children 43115ffc6635
files linux-2.6-xen-sparse/arch/ia64/kernel/fsys.S
line diff
     1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/linux-2.6-xen-sparse/arch/ia64/kernel/fsys.S	Tue Jan 16 11:51:45 2007 -0700
     1.3 @@ -0,0 +1,884 @@
     1.4 +/*
     1.5 + * This file contains the light-weight system call handlers (fsyscall-handlers).
     1.6 + *
     1.7 + * Copyright (C) 2003 Hewlett-Packard Co
     1.8 + * 	David Mosberger-Tang <davidm@hpl.hp.com>
     1.9 + *
    1.10 + * 25-Sep-03 davidm	Implement fsys_rt_sigprocmask().
    1.11 + * 18-Feb-03 louisk	Implement fsys_gettimeofday().
    1.12 + * 28-Feb-03 davidm	Fixed several bugs in fsys_gettimeofday().  Tuned it some more,
    1.13 + *			probably broke it along the way... ;-)
    1.14 + * 13-Jul-04 clameter   Implement fsys_clock_gettime and revise fsys_gettimeofday to make
    1.15 + *                      it capable of using memory based clocks without falling back to C code.
    1.16 + */
    1.17 +
    1.18 +#include <asm/asmmacro.h>
    1.19 +#include <asm/errno.h>
    1.20 +#include <asm/asm-offsets.h>
    1.21 +#include <asm/percpu.h>
    1.22 +#include <asm/thread_info.h>
    1.23 +#include <asm/sal.h>
    1.24 +#include <asm/signal.h>
    1.25 +#include <asm/system.h>
    1.26 +#include <asm/unistd.h>
    1.27 +
    1.28 +#include "entry.h"
    1.29 +
    1.30 +/*
    1.31 + * See Documentation/ia64/fsys.txt for details on fsyscalls.
    1.32 + *
    1.33 + * On entry to an fsyscall handler:
    1.34 + *   r10	= 0 (i.e., defaults to "successful syscall return")
    1.35 + *   r11	= saved ar.pfs (a user-level value)
    1.36 + *   r15	= system call number
    1.37 + *   r16	= "current" task pointer (in normal kernel-mode, this is in r13)
    1.38 + *   r32-r39	= system call arguments
    1.39 + *   b6		= return address (a user-level value)
    1.40 + *   ar.pfs	= previous frame-state (a user-level value)
    1.41 + *   PSR.be	= cleared to zero (i.e., little-endian byte order is in effect)
    1.42 + *   all other registers may contain values passed in from user-mode
    1.43 + *
    1.44 + * On return from an fsyscall handler:
    1.45 + *   r11	= saved ar.pfs (as passed into the fsyscall handler)
    1.46 + *   r15	= system call number (as passed into the fsyscall handler)
    1.47 + *   r32-r39	= system call arguments (as passed into the fsyscall handler)
    1.48 + *   b6		= return address (as passed into the fsyscall handler)
    1.49 + *   ar.pfs	= previous frame-state (as passed into the fsyscall handler)
    1.50 + */
    1.51 +
    1.52 +ENTRY(fsys_ni_syscall)
    1.53 +	.prologue
    1.54 +	.altrp b6
    1.55 +	.body
    1.56 +	mov r8=ENOSYS
    1.57 +	mov r10=-1
    1.58 +	FSYS_RETURN
    1.59 +END(fsys_ni_syscall)
    1.60 +
    1.61 +ENTRY(fsys_getpid)
    1.62 +	.prologue
    1.63 +	.altrp b6
    1.64 +	.body
    1.65 +	add r9=TI_FLAGS+IA64_TASK_SIZE,r16
    1.66 +	;;
    1.67 +	ld4 r9=[r9]
    1.68 +	add r8=IA64_TASK_TGID_OFFSET,r16
    1.69 +	;;
    1.70 +	and r9=TIF_ALLWORK_MASK,r9
    1.71 +	ld4 r8=[r8]				// r8 = current->tgid
    1.72 +	;;
    1.73 +	cmp.ne p8,p0=0,r9
    1.74 +(p8)	br.spnt.many fsys_fallback_syscall
    1.75 +	FSYS_RETURN
    1.76 +END(fsys_getpid)
    1.77 +
    1.78 +ENTRY(fsys_getppid)
    1.79 +	.prologue
    1.80 +	.altrp b6
    1.81 +	.body
    1.82 +	add r17=IA64_TASK_GROUP_LEADER_OFFSET,r16
    1.83 +	;;
    1.84 +	ld8 r17=[r17]				// r17 = current->group_leader
    1.85 +	add r9=TI_FLAGS+IA64_TASK_SIZE,r16
    1.86 +	;;
    1.87 +
    1.88 +	ld4 r9=[r9]
    1.89 +	add r17=IA64_TASK_REAL_PARENT_OFFSET,r17 // r17 = &current->group_leader->real_parent
    1.90 +	;;
    1.91 +	and r9=TIF_ALLWORK_MASK,r9
    1.92 +
    1.93 +1:	ld8 r18=[r17]				// r18 = current->group_leader->real_parent
    1.94 +	;;
    1.95 +	cmp.ne p8,p0=0,r9
    1.96 +	add r8=IA64_TASK_TGID_OFFSET,r18	// r8 = &current->group_leader->real_parent->tgid
    1.97 +	;;
    1.98 +
    1.99 +	/*
   1.100 +	 * The .acq is needed to ensure that the read of tgid has returned its data before
   1.101 +	 * we re-check "real_parent".
   1.102 +	 */
   1.103 +	ld4.acq r8=[r8]				// r8 = current->group_leader->real_parent->tgid
   1.104 +#ifdef CONFIG_SMP
   1.105 +	/*
   1.106 +	 * Re-read current->group_leader->real_parent.
   1.107 +	 */
   1.108 +	ld8 r19=[r17]				// r19 = current->group_leader->real_parent
   1.109 +(p8)	br.spnt.many fsys_fallback_syscall
   1.110 +	;;
   1.111 +	cmp.ne p6,p0=r18,r19			// did real_parent change?
   1.112 +	mov r19=0			// i must not leak kernel bits...
   1.113 +(p6)	br.cond.spnt.few 1b			// yes -> redo the read of tgid and the check
   1.114 +	;;
   1.115 +	mov r17=0			// i must not leak kernel bits...
   1.116 +	mov r18=0			// i must not leak kernel bits...
   1.117 +#else
   1.118 +	mov r17=0			// i must not leak kernel bits...
   1.119 +	mov r18=0			// i must not leak kernel bits...
   1.120 +	mov r19=0			// i must not leak kernel bits...
   1.121 +#endif
   1.122 +	FSYS_RETURN
   1.123 +END(fsys_getppid)
   1.124 +
   1.125 +ENTRY(fsys_set_tid_address)
   1.126 +	.prologue
   1.127 +	.altrp b6
   1.128 +	.body
   1.129 +	add r9=TI_FLAGS+IA64_TASK_SIZE,r16
   1.130 +	;;
   1.131 +	ld4 r9=[r9]
   1.132 +	tnat.z p6,p7=r32		// check argument register for being NaT
   1.133 +	;;
   1.134 +	and r9=TIF_ALLWORK_MASK,r9
   1.135 +	add r8=IA64_TASK_PID_OFFSET,r16
   1.136 +	add r18=IA64_TASK_CLEAR_CHILD_TID_OFFSET,r16
   1.137 +	;;
   1.138 +	ld4 r8=[r8]
   1.139 +	cmp.ne p8,p0=0,r9
   1.140 +	mov r17=-1
   1.141 +	;;
   1.142 +(p6)	st8 [r18]=r32
   1.143 +(p7)	st8 [r18]=r17
   1.144 +(p8)	br.spnt.many fsys_fallback_syscall
   1.145 +	;;
   1.146 +	mov r17=0			// i must not leak kernel bits...
   1.147 +	mov r18=0			// i must not leak kernel bits...
   1.148 +	FSYS_RETURN
   1.149 +END(fsys_set_tid_address)
   1.150 +
   1.151 +/*
   1.152 + * Ensure that the time interpolator structure is compatible with the asm code
   1.153 + */
   1.154 +#if IA64_TIME_INTERPOLATOR_SOURCE_OFFSET !=0 || IA64_TIME_INTERPOLATOR_SHIFT_OFFSET != 2 \
   1.155 +	|| IA64_TIME_INTERPOLATOR_JITTER_OFFSET != 3 || IA64_TIME_INTERPOLATOR_NSEC_OFFSET != 4
   1.156 +#error fsys_gettimeofday incompatible with changes to struct time_interpolator
   1.157 +#endif
   1.158 +#define CLOCK_REALTIME 0
   1.159 +#define CLOCK_MONOTONIC 1
   1.160 +#define CLOCK_DIVIDE_BY_1000 0x4000
   1.161 +#define CLOCK_ADD_MONOTONIC 0x8000
   1.162 +
   1.163 +ENTRY(fsys_gettimeofday)
   1.164 +	.prologue
   1.165 +	.altrp b6
   1.166 +	.body
   1.167 +	mov r31 = r32
   1.168 +	tnat.nz p6,p0 = r33		// guard against NaT argument
   1.169 +(p6)    br.cond.spnt.few .fail_einval
   1.170 +	mov r30 = CLOCK_DIVIDE_BY_1000
   1.171 +	;;
   1.172 +.gettime:
   1.173 +	// Register map
   1.174 +	// Incoming r31 = pointer to address where to place result
   1.175 +	//          r30 = flags determining how time is processed
   1.176 +	// r2,r3 = temp r4-r7 preserved
   1.177 +	// r8 = result nanoseconds
   1.178 +	// r9 = result seconds
   1.179 +	// r10 = temporary storage for clock difference
   1.180 +	// r11 = preserved: saved ar.pfs
   1.181 +	// r12 = preserved: memory stack
   1.182 +	// r13 = preserved: thread pointer
   1.183 +	// r14 = address of mask / mask
   1.184 +	// r15 = preserved: system call number
   1.185 +	// r16 = preserved: current task pointer
   1.186 +	// r17 = wall to monotonic use
   1.187 +	// r18 = time_interpolator->offset
   1.188 +	// r19 = address of wall_to_monotonic
   1.189 +	// r20 = pointer to struct time_interpolator / pointer to time_interpolator->address
   1.190 +	// r21 = shift factor
   1.191 +	// r22 = address of time interpolator->last_counter
   1.192 +	// r23 = address of time_interpolator->last_cycle
   1.193 +	// r24 = adress of time_interpolator->offset
   1.194 +	// r25 = last_cycle value
   1.195 +	// r26 = last_counter value
   1.196 +	// r27 = pointer to xtime
   1.197 +	// r28 = sequence number at the beginning of critcal section
   1.198 +	// r29 = address of seqlock
   1.199 +	// r30 = time processing flags / memory address
   1.200 +	// r31 = pointer to result
   1.201 +	// Predicates
   1.202 +	// p6,p7 short term use
   1.203 +	// p8 = timesource ar.itc
   1.204 +	// p9 = timesource mmio64
   1.205 +	// p10 = timesource mmio32
   1.206 +	// p11 = timesource not to be handled by asm code
   1.207 +	// p12 = memory time source ( = p9 | p10)
   1.208 +	// p13 = do cmpxchg with time_interpolator_last_cycle
   1.209 +	// p14 = Divide by 1000
   1.210 +	// p15 = Add monotonic
   1.211 +	//
   1.212 +	// Note that instructions are optimized for McKinley. McKinley can process two
   1.213 +	// bundles simultaneously and therefore we continuously try to feed the CPU
   1.214 +	// two bundles and then a stop.
   1.215 +	tnat.nz p6,p0 = r31	// branch deferred since it does not fit into bundle structure
   1.216 +	mov pr = r30,0xc000	// Set predicates according to function
   1.217 +	add r2 = TI_FLAGS+IA64_TASK_SIZE,r16
   1.218 +	movl r20 = time_interpolator
   1.219 +	;;
   1.220 +	ld8 r20 = [r20]		// get pointer to time_interpolator structure
   1.221 +	movl r29 = xtime_lock
   1.222 +	ld4 r2 = [r2]		// process work pending flags
   1.223 +	movl r27 = xtime
   1.224 +	;;	// only one bundle here
   1.225 +	ld8 r21 = [r20]		// first quad with control information
   1.226 +	and r2 = TIF_ALLWORK_MASK,r2
   1.227 +(p6)    br.cond.spnt.few .fail_einval	// deferred branch
   1.228 +	;;
   1.229 +	add r10 = IA64_TIME_INTERPOLATOR_ADDRESS_OFFSET,r20
   1.230 +	extr r3 = r21,32,32	// time_interpolator->nsec_per_cyc
   1.231 +	extr r8 = r21,0,16	// time_interpolator->source
   1.232 +	cmp.ne p6, p0 = 0, r2	// Fallback if work is scheduled
   1.233 +(p6)    br.cond.spnt.many fsys_fallback_syscall
   1.234 +	;;
   1.235 +	cmp.eq p8,p12 = 0,r8	// Check for cpu timer
   1.236 +	cmp.eq p9,p0 = 1,r8	// MMIO64 ?
   1.237 +	extr r2 = r21,24,8	// time_interpolator->jitter
   1.238 +	cmp.eq p10,p0 = 2,r8	// MMIO32 ?
   1.239 +	cmp.ltu p11,p0 = 2,r8	// function or other clock
   1.240 +(p11)	br.cond.spnt.many fsys_fallback_syscall
   1.241 +	;;
   1.242 +	setf.sig f7 = r3	// Setup for scaling of counter
   1.243 +(p15)	movl r19 = wall_to_monotonic
   1.244 +(p12)	ld8 r30 = [r10]
   1.245 +	cmp.ne p13,p0 = r2,r0	// need jitter compensation?
   1.246 +	extr r21 = r21,16,8	// shift factor
   1.247 +	;;
   1.248 +.time_redo:
   1.249 +	.pred.rel.mutex p8,p9,p10
   1.250 +	ld4.acq r28 = [r29]	// xtime_lock.sequence. Must come first for locking purposes
   1.251 +(p8)	mov r2 = ar.itc		// CPU_TIMER. 36 clocks latency!!!
   1.252 +	add r22 = IA64_TIME_INTERPOLATOR_LAST_COUNTER_OFFSET,r20
   1.253 +(p9)	ld8 r2 = [r30]		// readq(ti->address). Could also have latency issues..
   1.254 +(p10)	ld4 r2 = [r30]		// readw(ti->address)
   1.255 +(p13)	add r23 = IA64_TIME_INTERPOLATOR_LAST_CYCLE_OFFSET,r20
   1.256 +	;;			// could be removed by moving the last add upward
   1.257 +	ld8 r26 = [r22]		// time_interpolator->last_counter
   1.258 +(p13)	ld8 r25 = [r23]		// time interpolator->last_cycle
   1.259 +	add r24 = IA64_TIME_INTERPOLATOR_OFFSET_OFFSET,r20
   1.260 +(p15)	ld8 r17 = [r19],IA64_TIMESPEC_TV_NSEC_OFFSET
   1.261 + 	ld8 r9 = [r27],IA64_TIMESPEC_TV_NSEC_OFFSET
   1.262 +	add r14 = IA64_TIME_INTERPOLATOR_MASK_OFFSET, r20
   1.263 +	;;
   1.264 +	ld8 r18 = [r24]		// time_interpolator->offset
   1.265 +	ld8 r8 = [r27],-IA64_TIMESPEC_TV_NSEC_OFFSET	// xtime.tv_nsec
   1.266 +(p13)	sub r3 = r25,r2	// Diff needed before comparison (thanks davidm)
   1.267 +	;;
   1.268 +	ld8 r14 = [r14]		// time_interpolator->mask
   1.269 +(p13)	cmp.gt.unc p6,p7 = r3,r0	// check if it is less than last. p6,p7 cleared
   1.270 +	sub r10 = r2,r26	// current_counter - last_counter
   1.271 +	;;
   1.272 +(p6)	sub r10 = r25,r26	// time we got was less than last_cycle
   1.273 +(p7)	mov ar.ccv = r25	// more than last_cycle. Prep for cmpxchg
   1.274 +	;;
   1.275 +	and r10 = r10,r14	// Apply mask
   1.276 +	;;
   1.277 +	setf.sig f8 = r10
   1.278 +	nop.i 123
   1.279 +	;;
   1.280 +(p7)	cmpxchg8.rel r3 = [r23],r2,ar.ccv
   1.281 +EX(.fail_efault, probe.w.fault r31, 3)	// This takes 5 cycles and we have spare time
   1.282 +	xmpy.l f8 = f8,f7	// nsec_per_cyc*(counter-last_counter)
   1.283 +(p15)	add r9 = r9,r17		// Add wall to monotonic.secs to result secs
   1.284 +	;;
   1.285 +(p15)	ld8 r17 = [r19],-IA64_TIMESPEC_TV_NSEC_OFFSET
   1.286 +(p7)	cmp.ne p7,p0 = r25,r3	// if cmpxchg not successful redo
   1.287 +	// simulate tbit.nz.or p7,p0 = r28,0
   1.288 +	and r28 = ~1,r28	// Make sequence even to force retry if odd
   1.289 +	getf.sig r2 = f8
   1.290 +	mf
   1.291 +	add r8 = r8,r18		// Add time interpolator offset
   1.292 +	;;
   1.293 +	ld4 r10 = [r29]		// xtime_lock.sequence
   1.294 +(p15)	add r8 = r8, r17	// Add monotonic.nsecs to nsecs
   1.295 +	shr.u r2 = r2,r21
   1.296 +	;;		// overloaded 3 bundles!
   1.297 +	// End critical section.
   1.298 +	add r8 = r8,r2		// Add xtime.nsecs
   1.299 +	cmp4.ne.or p7,p0 = r28,r10
   1.300 +(p7)	br.cond.dpnt.few .time_redo	// sequence number changed ?
   1.301 +	// Now r8=tv->tv_nsec and r9=tv->tv_sec
   1.302 +	mov r10 = r0
   1.303 +	movl r2 = 1000000000
   1.304 +	add r23 = IA64_TIMESPEC_TV_NSEC_OFFSET, r31
   1.305 +(p14)	movl r3 = 2361183241434822607	// Prep for / 1000 hack
   1.306 +	;;
   1.307 +.time_normalize:
   1.308 +	mov r21 = r8
   1.309 +	cmp.ge p6,p0 = r8,r2
   1.310 +(p14)	shr.u r20 = r8, 3		// We can repeat this if necessary just wasting some time
   1.311 +	;;
   1.312 +(p14)	setf.sig f8 = r20
   1.313 +(p6)	sub r8 = r8,r2
   1.314 +(p6)	add r9 = 1,r9			// two nops before the branch.
   1.315 +(p14)	setf.sig f7 = r3		// Chances for repeats are 1 in 10000 for gettod
   1.316 +(p6)	br.cond.dpnt.few .time_normalize
   1.317 +	;;
   1.318 +	// Divided by 8 though shift. Now divide by 125
   1.319 +	// The compiler was able to do that with a multiply
   1.320 +	// and a shift and we do the same
   1.321 +EX(.fail_efault, probe.w.fault r23, 3)		// This also costs 5 cycles
   1.322 +(p14)	xmpy.hu f8 = f8, f7			// xmpy has 5 cycles latency so use it...
   1.323 +	;;
   1.324 +	mov r8 = r0
   1.325 +(p14)	getf.sig r2 = f8
   1.326 +	;;
   1.327 +(p14)	shr.u r21 = r2, 4
   1.328 +	;;
   1.329 +EX(.fail_efault, st8 [r31] = r9)
   1.330 +EX(.fail_efault, st8 [r23] = r21)
   1.331 +	FSYS_RETURN
   1.332 +.fail_einval:
   1.333 +	mov r8 = EINVAL
   1.334 +	mov r10 = -1
   1.335 +	FSYS_RETURN
   1.336 +.fail_efault:
   1.337 +	mov r8 = EFAULT
   1.338 +	mov r10 = -1
   1.339 +	FSYS_RETURN
   1.340 +END(fsys_gettimeofday)
   1.341 +
   1.342 +ENTRY(fsys_clock_gettime)
   1.343 +	.prologue
   1.344 +	.altrp b6
   1.345 +	.body
   1.346 +	cmp4.ltu p6, p0 = CLOCK_MONOTONIC, r32
   1.347 +	// Fallback if this is not CLOCK_REALTIME or CLOCK_MONOTONIC
   1.348 +(p6)	br.spnt.few fsys_fallback_syscall
   1.349 +	mov r31 = r33
   1.350 +	shl r30 = r32,15
   1.351 +	br.many .gettime
   1.352 +END(fsys_clock_gettime)
   1.353 +
   1.354 +/*
   1.355 + * long fsys_rt_sigprocmask (int how, sigset_t *set, sigset_t *oset, size_t sigsetsize).
   1.356 + */
   1.357 +#if _NSIG_WORDS != 1
   1.358 +# error Sorry, fsys_rt_sigprocmask() needs to be updated for _NSIG_WORDS != 1.
   1.359 +#endif
   1.360 +ENTRY(fsys_rt_sigprocmask)
   1.361 +	.prologue
   1.362 +	.altrp b6
   1.363 +	.body
   1.364 +
   1.365 +	add r2=IA64_TASK_BLOCKED_OFFSET,r16
   1.366 +	add r9=TI_FLAGS+IA64_TASK_SIZE,r16
   1.367 +	cmp4.ltu p6,p0=SIG_SETMASK,r32
   1.368 +
   1.369 +	cmp.ne p15,p0=r0,r34			// oset != NULL?
   1.370 +	tnat.nz p8,p0=r34
   1.371 +	add r31=IA64_TASK_SIGHAND_OFFSET,r16
   1.372 +	;;
   1.373 +	ld8 r3=[r2]				// read/prefetch current->blocked
   1.374 +	ld4 r9=[r9]
   1.375 +	tnat.nz.or p6,p0=r35
   1.376 +
   1.377 +	cmp.ne.or p6,p0=_NSIG_WORDS*8,r35
   1.378 +	tnat.nz.or p6,p0=r32
   1.379 +(p6)	br.spnt.few .fail_einval		// fail with EINVAL
   1.380 +	;;
   1.381 +#ifdef CONFIG_SMP
   1.382 +	ld8 r31=[r31]				// r31 <- current->sighand
   1.383 +#endif
   1.384 +	and r9=TIF_ALLWORK_MASK,r9
   1.385 +	tnat.nz.or p8,p0=r33
   1.386 +	;;
   1.387 +	cmp.ne p7,p0=0,r9
   1.388 +	cmp.eq p6,p0=r0,r33			// set == NULL?
   1.389 +	add r31=IA64_SIGHAND_SIGLOCK_OFFSET,r31	// r31 <- current->sighand->siglock
   1.390 +(p8)	br.spnt.few .fail_efault		// fail with EFAULT
   1.391 +(p7)	br.spnt.many fsys_fallback_syscall	// got pending kernel work...
   1.392 +(p6)	br.dpnt.many .store_mask		// -> short-circuit to just reading the signal mask
   1.393 +
   1.394 +	/* Argh, we actually have to do some work and _update_ the signal mask: */
   1.395 +
   1.396 +EX(.fail_efault, probe.r.fault r33, 3)		// verify user has read-access to *set
   1.397 +EX(.fail_efault, ld8 r14=[r33])			// r14 <- *set
   1.398 +	mov r17=(1 << (SIGKILL - 1)) | (1 << (SIGSTOP - 1))
   1.399 +	;;
   1.400 +
   1.401 +	rsm psr.i				// mask interrupt delivery
   1.402 +	mov ar.ccv=0
   1.403 +	andcm r14=r14,r17			// filter out SIGKILL & SIGSTOP
   1.404 +
   1.405 +#ifdef CONFIG_SMP
   1.406 +	mov r17=1
   1.407 +	;;
   1.408 +	cmpxchg4.acq r18=[r31],r17,ar.ccv	// try to acquire the lock
   1.409 +	mov r8=EINVAL			// default to EINVAL
   1.410 +	;;
   1.411 +	ld8 r3=[r2]			// re-read current->blocked now that we hold the lock
   1.412 +	cmp4.ne p6,p0=r18,r0
   1.413 +(p6)	br.cond.spnt.many .lock_contention
   1.414 +	;;
   1.415 +#else
   1.416 +	ld8 r3=[r2]			// re-read current->blocked now that we hold the lock
   1.417 +	mov r8=EINVAL			// default to EINVAL
   1.418 +#endif
   1.419 +	add r18=IA64_TASK_PENDING_OFFSET+IA64_SIGPENDING_SIGNAL_OFFSET,r16
   1.420 +	add r19=IA64_TASK_SIGNAL_OFFSET,r16
   1.421 +	cmp4.eq p6,p0=SIG_BLOCK,r32
   1.422 +	;;
   1.423 +	ld8 r19=[r19]			// r19 <- current->signal
   1.424 +	cmp4.eq p7,p0=SIG_UNBLOCK,r32
   1.425 +	cmp4.eq p8,p0=SIG_SETMASK,r32
   1.426 +	;;
   1.427 +	ld8 r18=[r18]			// r18 <- current->pending.signal
   1.428 +	.pred.rel.mutex p6,p7,p8
   1.429 +(p6)	or r14=r3,r14			// SIG_BLOCK
   1.430 +(p7)	andcm r14=r3,r14		// SIG_UNBLOCK
   1.431 +
   1.432 +(p8)	mov r14=r14			// SIG_SETMASK
   1.433 +(p6)	mov r8=0			// clear error code
   1.434 +	// recalc_sigpending()
   1.435 +	add r17=IA64_SIGNAL_GROUP_STOP_COUNT_OFFSET,r19
   1.436 +
   1.437 +	add r19=IA64_SIGNAL_SHARED_PENDING_OFFSET+IA64_SIGPENDING_SIGNAL_OFFSET,r19
   1.438 +	;;
   1.439 +	ld4 r17=[r17]		// r17 <- current->signal->group_stop_count
   1.440 +(p7)	mov r8=0		// clear error code
   1.441 +
   1.442 +	ld8 r19=[r19]		// r19 <- current->signal->shared_pending
   1.443 +	;;
   1.444 +	cmp4.gt p6,p7=r17,r0	// p6/p7 <- (current->signal->group_stop_count > 0)?
   1.445 +(p8)	mov r8=0		// clear error code
   1.446 +
   1.447 +	or r18=r18,r19		// r18 <- current->pending | current->signal->shared_pending
   1.448 +	;;
   1.449 +	// r18 <- (current->pending | current->signal->shared_pending) & ~current->blocked:
   1.450 +	andcm r18=r18,r14
   1.451 +	add r9=TI_FLAGS+IA64_TASK_SIZE,r16
   1.452 +	;;
   1.453 +
   1.454 +(p7)	cmp.ne.or.andcm p6,p7=r18,r0		// p6/p7 <- signal pending
   1.455 +	mov r19=0					// i must not leak kernel bits...
   1.456 +(p6)	br.cond.dpnt.many .sig_pending
   1.457 +	;;
   1.458 +
   1.459 +1:	ld4 r17=[r9]				// r17 <- current->thread_info->flags
   1.460 +	;;
   1.461 +	mov ar.ccv=r17
   1.462 +	and r18=~_TIF_SIGPENDING,r17		// r18 <- r17 & ~(1 << TIF_SIGPENDING)
   1.463 +	;;
   1.464 +
   1.465 +	st8 [r2]=r14				// update current->blocked with new mask
   1.466 +	cmpxchg4.acq r8=[r9],r18,ar.ccv		// current->thread_info->flags <- r18
   1.467 +	;;
   1.468 +	cmp.ne p6,p0=r17,r8			// update failed?
   1.469 +(p6)	br.cond.spnt.few 1b			// yes -> retry
   1.470 +
   1.471 +#ifdef CONFIG_SMP
   1.472 +	st4.rel [r31]=r0			// release the lock
   1.473 +#endif
   1.474 +	ssm psr.i
   1.475 +	;;
   1.476 +
   1.477 +	srlz.d					// ensure psr.i is set again
   1.478 +	mov r18=0					// i must not leak kernel bits...
   1.479 +
   1.480 +.store_mask:
   1.481 +EX(.fail_efault, (p15) probe.w.fault r34, 3)	// verify user has write-access to *oset
   1.482 +EX(.fail_efault, (p15) st8 [r34]=r3)
   1.483 +	mov r2=0					// i must not leak kernel bits...
   1.484 +	mov r3=0					// i must not leak kernel bits...
   1.485 +	mov r8=0				// return 0
   1.486 +	mov r9=0					// i must not leak kernel bits...
   1.487 +	mov r14=0					// i must not leak kernel bits...
   1.488 +	mov r17=0					// i must not leak kernel bits...
   1.489 +	mov r31=0					// i must not leak kernel bits...
   1.490 +	FSYS_RETURN
   1.491 +
   1.492 +.sig_pending:
   1.493 +#ifdef CONFIG_SMP
   1.494 +	st4.rel [r31]=r0			// release the lock
   1.495 +#endif
   1.496 +	ssm psr.i
   1.497 +	;;
   1.498 +	srlz.d
   1.499 +	br.sptk.many fsys_fallback_syscall	// with signal pending, do the heavy-weight syscall
   1.500 +
   1.501 +#ifdef CONFIG_SMP
   1.502 +.lock_contention:
   1.503 +	/* Rather than spinning here, fall back on doing a heavy-weight syscall.  */
   1.504 +	ssm psr.i
   1.505 +	;;
   1.506 +	srlz.d
   1.507 +	br.sptk.many fsys_fallback_syscall
   1.508 +#endif
   1.509 +END(fsys_rt_sigprocmask)
   1.510 +
   1.511 +ENTRY(fsys_fallback_syscall)
   1.512 +	.prologue
   1.513 +	.altrp b6
   1.514 +	.body
   1.515 +	/*
   1.516 +	 * We only get here from light-weight syscall handlers.  Thus, we already
   1.517 +	 * know that r15 contains a valid syscall number.  No need to re-check.
   1.518 +	 */
   1.519 +	adds r17=-1024,r15
   1.520 +	movl r14=sys_call_table
   1.521 +	;;
   1.522 +	rsm psr.i
   1.523 +	shladd r18=r17,3,r14
   1.524 +	;;
   1.525 +	ld8 r18=[r18]				// load normal (heavy-weight) syscall entry-point
   1.526 +	mov r29=psr				// read psr (12 cyc load latency)
   1.527 +	mov r27=ar.rsc
   1.528 +	mov r21=ar.fpsr
   1.529 +	mov r26=ar.pfs
   1.530 +END(fsys_fallback_syscall)
   1.531 +	/* FALL THROUGH */
   1.532 +GLOBAL_ENTRY(fsys_bubble_down)
   1.533 +	.prologue
   1.534 +	.altrp b6
   1.535 +	.body
   1.536 +	/*
   1.537 +	 * We get here for syscalls that don't have a lightweight
   1.538 +	 * handler.  For those, we need to bubble down into the kernel
   1.539 +	 * and that requires setting up a minimal pt_regs structure,
   1.540 +	 * and initializing the CPU state more or less as if an
   1.541 +	 * interruption had occurred.  To make syscall-restarts work,
   1.542 +	 * we setup pt_regs such that cr_iip points to the second
   1.543 +	 * instruction in syscall_via_break.  Decrementing the IP
   1.544 +	 * hence will restart the syscall via break and not
   1.545 +	 * decrementing IP will return us to the caller, as usual.
   1.546 +	 * Note that we preserve the value of psr.pp rather than
   1.547 +	 * initializing it from dcr.pp.  This makes it possible to
   1.548 +	 * distinguish fsyscall execution from other privileged
   1.549 +	 * execution.
   1.550 +	 *
   1.551 +	 * On entry:
   1.552 +	 *	- normal fsyscall handler register usage, except
   1.553 +	 *	  that we also have:
   1.554 +	 *	- r18: address of syscall entry point
   1.555 +	 *	- r21: ar.fpsr
   1.556 +	 *	- r26: ar.pfs
   1.557 +	 *	- r27: ar.rsc
   1.558 +	 *	- r29: psr
   1.559 +	 *
   1.560 +	 * We used to clear some PSR bits here but that requires slow
   1.561 +	 * serialization.  Fortuntely, that isn't really necessary.
   1.562 +	 * The rationale is as follows: we used to clear bits
   1.563 +	 * ~PSR_PRESERVED_BITS in PSR.L.  Since
   1.564 +	 * PSR_PRESERVED_BITS==PSR.{UP,MFL,MFH,PK,DT,PP,SP,RT,IC}, we
   1.565 +	 * ended up clearing PSR.{BE,AC,I,DFL,DFH,DI,DB,SI,TB}.
   1.566 +	 * However,
   1.567 +	 *
   1.568 +	 * PSR.BE : already is turned off in __kernel_syscall_via_epc()
   1.569 +	 * PSR.AC : don't care (kernel normally turns PSR.AC on)
   1.570 +	 * PSR.I  : already turned off by the time fsys_bubble_down gets
   1.571 +	 *	    invoked
   1.572 +	 * PSR.DFL: always 0 (kernel never turns it on)
   1.573 +	 * PSR.DFH: don't care --- kernel never touches f32-f127 on its own
   1.574 +	 *	    initiative
   1.575 +	 * PSR.DI : always 0 (kernel never turns it on)
   1.576 +	 * PSR.SI : always 0 (kernel never turns it on)
   1.577 +	 * PSR.DB : don't care --- kernel never enables kernel-level
   1.578 +	 *	    breakpoints
   1.579 +	 * PSR.TB : must be 0 already; if it wasn't zero on entry to
   1.580 +	 *          __kernel_syscall_via_epc, the branch to fsys_bubble_down
   1.581 +	 *          will trigger a taken branch; the taken-trap-handler then
   1.582 +	 *          converts the syscall into a break-based system-call.
   1.583 +	 */
   1.584 +	/*
   1.585 +	 * Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.
   1.586 +	 * The rest we have to synthesize.
   1.587 +	 */
   1.588 +#	define PSR_ONE_BITS		((3 << IA64_PSR_CPL0_BIT)	\
   1.589 +					 | (0x1 << IA64_PSR_RI_BIT)	\
   1.590 +					 | IA64_PSR_BN | IA64_PSR_I)
   1.591 +
   1.592 +	invala					// M0|1
   1.593 +	movl r14=ia64_ret_from_syscall		// X
   1.594 +
   1.595 +	nop.m 0
   1.596 +	movl r28=__kernel_syscall_via_break	// X	create cr.iip
   1.597 +	;;
   1.598 +
   1.599 +	mov r2=r16				// A    get task addr to addl-addressable register
   1.600 +	adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // A
   1.601 +	mov r31=pr				// I0   save pr (2 cyc)
   1.602 +	;;
   1.603 +	st1 [r16]=r0				// M2|3 clear current->thread.on_ustack flag
   1.604 +	addl r22=IA64_RBS_OFFSET,r2		// A    compute base of RBS
   1.605 +	add r3=TI_FLAGS+IA64_TASK_SIZE,r2	// A
   1.606 +	;;
   1.607 +	ld4 r3=[r3]				// M0|1 r3 = current_thread_info()->flags
   1.608 +	lfetch.fault.excl.nt1 [r22]		// M0|1 prefetch register backing-store
   1.609 +	nop.i 0
   1.610 +	;;
   1.611 +	mov ar.rsc=0				// M2   set enforced lazy mode, pl 0, LE, loadrs=0
   1.612 +	nop.m 0
   1.613 +	nop.i 0
   1.614 +	;;
   1.615 +	mov r23=ar.bspstore			// M2 (12 cyc) save ar.bspstore
   1.616 +	mov.m r24=ar.rnat			// M2 (5 cyc) read ar.rnat (dual-issues!)
   1.617 +	nop.i 0
   1.618 +	;;
   1.619 +	mov ar.bspstore=r22			// M2 (6 cyc) switch to kernel RBS
   1.620 +	movl r8=PSR_ONE_BITS			// X
   1.621 +	;;
   1.622 +	mov r25=ar.unat				// M2 (5 cyc) save ar.unat
   1.623 +	mov r19=b6				// I0   save b6 (2 cyc)
   1.624 +	mov r20=r1				// A    save caller's gp in r20
   1.625 +	;;
   1.626 +	or r29=r8,r29				// A    construct cr.ipsr value to save
   1.627 +	mov b6=r18				// I0   copy syscall entry-point to b6 (7 cyc)
   1.628 +	addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2 // A compute base of memory stack
   1.629 +
   1.630 +	mov r18=ar.bsp				// M2   save (kernel) ar.bsp (12 cyc)
   1.631 +	cmp.ne pKStk,pUStk=r0,r0		// A    set pKStk <- 0, pUStk <- 1
   1.632 +	br.call.sptk.many b7=ia64_syscall_setup	// B
   1.633 +	;;
   1.634 +	mov ar.rsc=0x3				// M2   set eager mode, pl 0, LE, loadrs=0
   1.635 +	mov rp=r14				// I0   set the real return addr
   1.636 +	and r3=_TIF_SYSCALL_TRACEAUDIT,r3	// A
   1.637 +	;;
   1.638 +	ssm psr.i				// M2   we're on kernel stacks now, reenable irqs
   1.639 +	cmp.eq p8,p0=r3,r0			// A
   1.640 +(p10)	br.cond.spnt.many ia64_ret_from_syscall	// B    return if bad call-frame or r15 is a NaT
   1.641 +
   1.642 +	nop.m 0
   1.643 +(p8)	br.call.sptk.many b6=b6			// B    (ignore return address)
   1.644 +	br.cond.spnt ia64_trace_syscall		// B
   1.645 +END(fsys_bubble_down)
   1.646 +
   1.647 +	.rodata
   1.648 +	.align 8
   1.649 +	.globl fsyscall_table
   1.650 +
   1.651 +	data8 fsys_bubble_down
   1.652 +fsyscall_table:
   1.653 +	data8 fsys_ni_syscall
   1.654 +	data8 0				// exit			// 1025
   1.655 +	data8 0				// read
   1.656 +	data8 0				// write
   1.657 +	data8 0				// open
   1.658 +	data8 0				// close
   1.659 +	data8 0				// creat		// 1030
   1.660 +	data8 0				// link
   1.661 +	data8 0				// unlink
   1.662 +	data8 0				// execve
   1.663 +	data8 0				// chdir
   1.664 +	data8 0				// fchdir		// 1035
   1.665 +	data8 0				// utimes
   1.666 +	data8 0				// mknod
   1.667 +	data8 0				// chmod
   1.668 +	data8 0				// chown
   1.669 +	data8 0				// lseek		// 1040
   1.670 +	data8 fsys_getpid		// getpid
   1.671 +	data8 fsys_getppid		// getppid
   1.672 +	data8 0				// mount
   1.673 +	data8 0				// umount
   1.674 +	data8 0				// setuid		// 1045
   1.675 +	data8 0				// getuid
   1.676 +	data8 0				// geteuid
   1.677 +	data8 0				// ptrace
   1.678 +	data8 0				// access
   1.679 +	data8 0				// sync			// 1050
   1.680 +	data8 0				// fsync
   1.681 +	data8 0				// fdatasync
   1.682 +	data8 0				// kill
   1.683 +	data8 0				// rename
   1.684 +	data8 0				// mkdir		// 1055
   1.685 +	data8 0				// rmdir
   1.686 +	data8 0				// dup
   1.687 +	data8 0				// pipe
   1.688 +	data8 0				// times
   1.689 +	data8 0				// brk			// 1060
   1.690 +	data8 0				// setgid
   1.691 +	data8 0				// getgid
   1.692 +	data8 0				// getegid
   1.693 +	data8 0				// acct
   1.694 +	data8 0				// ioctl		// 1065
   1.695 +	data8 0				// fcntl
   1.696 +	data8 0				// umask
   1.697 +	data8 0				// chroot
   1.698 +	data8 0				// ustat
   1.699 +	data8 0				// dup2			// 1070
   1.700 +	data8 0				// setreuid
   1.701 +	data8 0				// setregid
   1.702 +	data8 0				// getresuid
   1.703 +	data8 0				// setresuid
   1.704 +	data8 0				// getresgid		// 1075
   1.705 +	data8 0				// setresgid
   1.706 +	data8 0				// getgroups
   1.707 +	data8 0				// setgroups
   1.708 +	data8 0				// getpgid
   1.709 +	data8 0				// setpgid		// 1080
   1.710 +	data8 0				// setsid
   1.711 +	data8 0				// getsid
   1.712 +	data8 0				// sethostname
   1.713 +	data8 0				// setrlimit
   1.714 +	data8 0				// getrlimit		// 1085
   1.715 +	data8 0				// getrusage
   1.716 +	data8 fsys_gettimeofday		// gettimeofday
   1.717 +	data8 0				// settimeofday
   1.718 +	data8 0				// select
   1.719 +	data8 0				// poll			// 1090
   1.720 +	data8 0				// symlink
   1.721 +	data8 0				// readlink
   1.722 +	data8 0				// uselib
   1.723 +	data8 0				// swapon
   1.724 +	data8 0				// swapoff		// 1095
   1.725 +	data8 0				// reboot
   1.726 +	data8 0				// truncate
   1.727 +	data8 0				// ftruncate
   1.728 +	data8 0				// fchmod
   1.729 +	data8 0				// fchown		// 1100
   1.730 +	data8 0				// getpriority
   1.731 +	data8 0				// setpriority
   1.732 +	data8 0				// statfs
   1.733 +	data8 0				// fstatfs
   1.734 +	data8 0				// gettid		// 1105
   1.735 +	data8 0				// semget
   1.736 +	data8 0				// semop
   1.737 +	data8 0				// semctl
   1.738 +	data8 0				// msgget
   1.739 +	data8 0				// msgsnd		// 1110
   1.740 +	data8 0				// msgrcv
   1.741 +	data8 0				// msgctl
   1.742 +	data8 0				// shmget
   1.743 +	data8 0				// shmat
   1.744 +	data8 0				// shmdt		// 1115
   1.745 +	data8 0				// shmctl
   1.746 +	data8 0				// syslog
   1.747 +	data8 0				// setitimer
   1.748 +	data8 0				// getitimer
   1.749 +	data8 0					 		// 1120
   1.750 +	data8 0
   1.751 +	data8 0
   1.752 +	data8 0				// vhangup
   1.753 +	data8 0				// lchown
   1.754 +	data8 0				// remap_file_pages	// 1125
   1.755 +	data8 0				// wait4
   1.756 +	data8 0				// sysinfo
   1.757 +	data8 0				// clone
   1.758 +	data8 0				// setdomainname
   1.759 +	data8 0				// newuname		// 1130
   1.760 +	data8 0				// adjtimex
   1.761 +	data8 0
   1.762 +	data8 0				// init_module
   1.763 +	data8 0				// delete_module
   1.764 +	data8 0							// 1135
   1.765 +	data8 0
   1.766 +	data8 0				// quotactl
   1.767 +	data8 0				// bdflush
   1.768 +	data8 0				// sysfs
   1.769 +	data8 0				// personality		// 1140
   1.770 +	data8 0				// afs_syscall
   1.771 +	data8 0				// setfsuid
   1.772 +	data8 0				// setfsgid
   1.773 +	data8 0				// getdents
   1.774 +	data8 0				// flock		// 1145
   1.775 +	data8 0				// readv
   1.776 +	data8 0				// writev
   1.777 +	data8 0				// pread64
   1.778 +	data8 0				// pwrite64
   1.779 +	data8 0				// sysctl		// 1150
   1.780 +	data8 0				// mmap
   1.781 +	data8 0				// munmap
   1.782 +	data8 0				// mlock
   1.783 +	data8 0				// mlockall
   1.784 +	data8 0				// mprotect		// 1155
   1.785 +	data8 0				// mremap
   1.786 +	data8 0				// msync
   1.787 +	data8 0				// munlock
   1.788 +	data8 0				// munlockall
   1.789 +	data8 0				// sched_getparam	// 1160
   1.790 +	data8 0				// sched_setparam
   1.791 +	data8 0				// sched_getscheduler
   1.792 +	data8 0				// sched_setscheduler
   1.793 +	data8 0				// sched_yield
   1.794 +	data8 0				// sched_get_priority_max	// 1165
   1.795 +	data8 0				// sched_get_priority_min
   1.796 +	data8 0				// sched_rr_get_interval
   1.797 +	data8 0				// nanosleep
   1.798 +	data8 0				// nfsservctl
   1.799 +	data8 0				// prctl		// 1170
   1.800 +	data8 0				// getpagesize
   1.801 +	data8 0				// mmap2
   1.802 +	data8 0				// pciconfig_read
   1.803 +	data8 0				// pciconfig_write
   1.804 +	data8 0				// perfmonctl		// 1175
   1.805 +	data8 0				// sigaltstack
   1.806 +	data8 0				// rt_sigaction
   1.807 +	data8 0				// rt_sigpending
   1.808 +	data8 fsys_rt_sigprocmask	// rt_sigprocmask
   1.809 +	data8 0				// rt_sigqueueinfo	// 1180
   1.810 +	data8 0				// rt_sigreturn
   1.811 +	data8 0				// rt_sigsuspend
   1.812 +	data8 0				// rt_sigtimedwait
   1.813 +	data8 0				// getcwd
   1.814 +	data8 0				// capget		// 1185
   1.815 +	data8 0				// capset
   1.816 +	data8 0				// sendfile
   1.817 +	data8 0
   1.818 +	data8 0
   1.819 +	data8 0				// socket		// 1190
   1.820 +	data8 0				// bind
   1.821 +	data8 0				// connect
   1.822 +	data8 0				// listen
   1.823 +	data8 0				// accept
   1.824 +	data8 0				// getsockname		// 1195
   1.825 +	data8 0				// getpeername
   1.826 +	data8 0				// socketpair
   1.827 +	data8 0				// send
   1.828 +	data8 0				// sendto
   1.829 +	data8 0				// recv			// 1200
   1.830 +	data8 0				// recvfrom
   1.831 +	data8 0				// shutdown
   1.832 +	data8 0				// setsockopt
   1.833 +	data8 0				// getsockopt
   1.834 +	data8 0				// sendmsg		// 1205
   1.835 +	data8 0				// recvmsg
   1.836 +	data8 0				// pivot_root
   1.837 +	data8 0				// mincore
   1.838 +	data8 0				// madvise
   1.839 +	data8 0				// newstat		// 1210
   1.840 +	data8 0				// newlstat
   1.841 +	data8 0				// newfstat
   1.842 +	data8 0				// clone2
   1.843 +	data8 0				// getdents64
   1.844 +	data8 0				// getunwind		// 1215
   1.845 +	data8 0				// readahead
   1.846 +	data8 0				// setxattr
   1.847 +	data8 0				// lsetxattr
   1.848 +	data8 0				// fsetxattr
   1.849 +	data8 0				// getxattr		// 1220
   1.850 +	data8 0				// lgetxattr
   1.851 +	data8 0				// fgetxattr
   1.852 +	data8 0				// listxattr
   1.853 +	data8 0				// llistxattr
   1.854 +	data8 0				// flistxattr		// 1225
   1.855 +	data8 0				// removexattr
   1.856 +	data8 0				// lremovexattr
   1.857 +	data8 0				// fremovexattr
   1.858 +	data8 0				// tkill
   1.859 +	data8 0				// futex		// 1230
   1.860 +	data8 0				// sched_setaffinity
   1.861 +	data8 0				// sched_getaffinity
   1.862 +	data8 fsys_set_tid_address	// set_tid_address
   1.863 +	data8 0				// fadvise64_64
   1.864 +	data8 0				// tgkill		// 1235
   1.865 +	data8 0				// exit_group
   1.866 +	data8 0				// lookup_dcookie
   1.867 +	data8 0				// io_setup
   1.868 +	data8 0				// io_destroy
   1.869 +	data8 0				// io_getevents		// 1240
   1.870 +	data8 0				// io_submit
   1.871 +	data8 0				// io_cancel
   1.872 +	data8 0				// epoll_create
   1.873 +	data8 0				// epoll_ctl
   1.874 +	data8 0				// epoll_wait		// 1245
   1.875 +	data8 0				// restart_syscall
   1.876 +	data8 0				// semtimedop
   1.877 +	data8 0				// timer_create
   1.878 +	data8 0				// timer_settime
   1.879 +	data8 0				// timer_gettime 	// 1250
   1.880 +	data8 0				// timer_getoverrun
   1.881 +	data8 0				// timer_delete
   1.882 +	data8 0				// clock_settime
   1.883 +	data8 fsys_clock_gettime	// clock_gettime
   1.884 +
   1.885 +	// fill in zeros for the remaining entries
   1.886 +	.zero:
   1.887 +	.space fsyscall_table + 8*NR_syscalls - .zero, 0