direct-io.hg

view xen/include/asm-x86/x86_32/uaccess.h @ 4638:7c4ca91ec3ce

bitkeeper revision 1.1368 (42692780QS8e5vEYZOY8vFfJQySx9Q)

Force efficient implementation of __put_user().
Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Fri Apr 22 16:34:08 2005 +0000 (2005-04-22)
parents dfe18db08708
children 4736c86802ad
line source
1 #ifndef __i386_UACCESS_H
2 #define __i386_UACCESS_H
4 /*
5 * User space memory access functions
6 */
7 #include <xen/config.h>
8 #include <xen/errno.h>
9 #include <xen/prefetch.h>
10 #include <xen/string.h>
12 #define __user
14 /*
15 * movsl can be slow when source and dest are not both 8-byte aligned
16 */
17 #ifdef CONFIG_X86_INTEL_USERCOPY
18 extern struct movsl_mask {
19 int mask;
20 } __cacheline_aligned movsl_mask;
21 #endif
23 #define __addr_ok(addr) ((unsigned long)(addr) < HYPERVISOR_VIRT_START)
25 /*
26 * Test whether a block of memory is a valid user space address.
27 * Returns 0 if the range is valid, nonzero otherwise.
28 *
29 * This is equivalent to the following test:
30 * (u33)addr + (u33)size >= (u33)HYPERVISOR_VIRT_START
31 */
32 #define __range_not_ok(addr,size) ({ \
33 unsigned long flag,sum; \
34 asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \
35 :"=&r" (flag), "=r" (sum) \
36 :"1" (addr),"g" ((int)(size)),"r" (HYPERVISOR_VIRT_START)); \
37 flag; })
39 #define access_ok(addr,size) (likely(__range_not_ok(addr,size) == 0))
41 #define array_access_ok(addr,count,size) \
42 (likely(count < (~0UL/size)) && access_ok(addr,count*size))
44 extern long __get_user_bad(void);
45 extern void __put_user_bad(void);
47 /**
48 * get_user: - Get a simple variable from user space.
49 * @x: Variable to store result.
50 * @ptr: Source address, in user space.
51 *
52 * Context: User context only. This function may sleep.
53 *
54 * This macro copies a single simple variable from user space to kernel
55 * space. It supports simple types like char and int, but not larger
56 * data types like structures or arrays.
57 *
58 * @ptr must have pointer-to-simple-variable type, and the result of
59 * dereferencing @ptr must be assignable to @x without a cast.
60 *
61 * Returns zero on success, or -EFAULT on error.
62 * On error, the variable @x is set to zero.
63 */
64 #define get_user(x,ptr) \
65 __get_user_check((x),(ptr),sizeof(*(ptr)))
67 /**
68 * put_user: - Write a simple value into user space.
69 * @x: Value to copy to user space.
70 * @ptr: Destination address, in user space.
71 *
72 * Context: User context only. This function may sleep.
73 *
74 * This macro copies a single simple value from kernel space to user
75 * space. It supports simple types like char and int, but not larger
76 * data types like structures or arrays.
77 *
78 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
79 * to the result of dereferencing @ptr.
80 *
81 * Returns zero on success, or -EFAULT on error.
82 */
83 #define put_user(x,ptr) \
84 __put_user_check((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr)))
87 /**
88 * __get_user: - Get a simple variable from user space, with less checking.
89 * @x: Variable to store result.
90 * @ptr: Source address, in user space.
91 *
92 * Context: User context only. This function may sleep.
93 *
94 * This macro copies a single simple variable from user space to kernel
95 * space. It supports simple types like char and int, but not larger
96 * data types like structures or arrays.
97 *
98 * @ptr must have pointer-to-simple-variable type, and the result of
99 * dereferencing @ptr must be assignable to @x without a cast.
100 *
101 * Caller must check the pointer with access_ok() before calling this
102 * function.
103 *
104 * Returns zero on success, or -EFAULT on error.
105 * On error, the variable @x is set to zero.
106 */
107 #define __get_user(x,ptr) \
108 __get_user_nocheck((x),(ptr),sizeof(*(ptr)))
111 /**
112 * __put_user: - Write a simple value into user space, with less checking.
113 * @x: Value to copy to user space.
114 * @ptr: Destination address, in user space.
115 *
116 * Context: User context only. This function may sleep.
117 *
118 * This macro copies a single simple value from kernel space to user
119 * space. It supports simple types like char and int, but not larger
120 * data types like structures or arrays.
121 *
122 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
123 * to the result of dereferencing @ptr.
124 *
125 * Caller must check the pointer with access_ok() before calling this
126 * function.
127 *
128 * Returns zero on success, or -EFAULT on error.
129 */
130 #define __put_user(x,ptr) \
131 __put_user_nocheck((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr)))
133 #define __put_user_nocheck(x,ptr,size) \
134 ({ \
135 long __pu_err; \
136 __put_user_size((x),(ptr),(size),__pu_err,-EFAULT); \
137 __pu_err; \
138 })
140 #define __put_user_check(x,ptr,size) \
141 ({ \
142 long __pu_err = -EFAULT; \
143 __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
144 if (__addr_ok(__pu_addr)) \
145 __put_user_size((x),__pu_addr,(size),__pu_err,-EFAULT); \
146 __pu_err; \
147 })
149 #define __put_user_u64(x, addr, err) \
150 __asm__ __volatile__( \
151 "1: movl %%eax,0(%2)\n" \
152 "2: movl %%edx,4(%2)\n" \
153 "3:\n" \
154 ".section .fixup,\"ax\"\n" \
155 "4: movl %3,%0\n" \
156 " jmp 3b\n" \
157 ".previous\n" \
158 ".section __ex_table,\"a\"\n" \
159 " .align 4\n" \
160 " .long 1b,4b\n" \
161 " .long 2b,4b\n" \
162 ".previous" \
163 : "=r"(err) \
164 : "A" (x), "r" (addr), "i"(-EFAULT), "0"(err))
166 #define __put_user_size(x,ptr,size,retval,errret) \
167 do { \
168 retval = 0; \
169 switch (size) { \
170 case 1: __put_user_asm(x,ptr,retval,"b","b","iq",errret);break; \
171 case 2: __put_user_asm(x,ptr,retval,"w","w","ir",errret);break; \
172 case 4: __put_user_asm(x,ptr,retval,"l","","ir",errret); break; \
173 case 8: __put_user_u64((__typeof__(*ptr))(x),ptr,retval); break;\
174 default: __put_user_bad(); \
175 } \
176 } while (0)
178 struct __large_struct { unsigned long buf[100]; };
179 #define __m(x) (*(struct __large_struct *)(x))
181 /*
182 * Tell gcc we read from memory instead of writing: this is because
183 * we do not write to any memory gcc knows about, so there are no
184 * aliasing issues.
185 */
186 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
187 __asm__ __volatile__( \
188 "1: mov"itype" %"rtype"1,%2\n" \
189 "2:\n" \
190 ".section .fixup,\"ax\"\n" \
191 "3: movl %3,%0\n" \
192 " jmp 2b\n" \
193 ".previous\n" \
194 ".section __ex_table,\"a\"\n" \
195 " .align 4\n" \
196 " .long 1b,3b\n" \
197 ".previous" \
198 : "=r"(err) \
199 : ltype (x), "m"(__m(addr)), "i"(errret), "0"(err))
202 #define __get_user_nocheck(x,ptr,size) \
203 ({ \
204 long __gu_err, __gu_val; \
205 __get_user_size(__gu_val,(ptr),(size),__gu_err,-EFAULT);\
206 (x) = (__typeof__(*(ptr)))__gu_val; \
207 __gu_err; \
208 })
210 #define __get_user_check(x,ptr,size) \
211 ({ \
212 long __gu_err, __gu_val; \
213 __typeof__(*(ptr)) __user *__gu_addr = (ptr); \
214 __get_user_size(__gu_val,__gu_addr,(size),__gu_err,-EFAULT); \
215 (x) = (__typeof__(*(ptr)))__gu_val; \
216 if (!__addr_ok(__gu_addr)) __gu_err = -EFAULT; \
217 __gu_err; \
218 })
220 #define __get_user_size(x,ptr,size,retval,errret) \
221 do { \
222 retval = 0; \
223 switch (size) { \
224 case 1: __get_user_asm(x,ptr,retval,"b","b","=q",errret);break; \
225 case 2: __get_user_asm(x,ptr,retval,"w","w","=r",errret);break; \
226 case 4: __get_user_asm(x,ptr,retval,"l","","=r",errret);break; \
227 default: (x) = __get_user_bad(); \
228 } \
229 } while (0)
231 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
232 __asm__ __volatile__( \
233 "1: mov"itype" %2,%"rtype"1\n" \
234 "2:\n" \
235 ".section .fixup,\"ax\"\n" \
236 "3: movl %3,%0\n" \
237 " xor"itype" %"rtype"1,%"rtype"1\n" \
238 " jmp 2b\n" \
239 ".previous\n" \
240 ".section __ex_table,\"a\"\n" \
241 " .align 4\n" \
242 " .long 1b,3b\n" \
243 ".previous" \
244 : "=r"(err), ltype (x) \
245 : "m"(__m(addr)), "i"(errret), "0"(err))
248 unsigned long __copy_to_user_ll(void __user *to, const void *from, unsigned long n);
249 unsigned long __copy_from_user_ll(void *to, const void __user *from, unsigned long n);
251 /*
252 * Here we special-case 1, 2 and 4-byte copy_*_user invocations. On a fault
253 * we return the initial request size (1, 2 or 4), as copy_*_user should do.
254 * If a store crosses a page boundary and gets a fault, the x86 will not write
255 * anything, so this is accurate.
256 */
258 /**
259 * __copy_to_user: - Copy a block of data into user space, with less checking.
260 * @to: Destination address, in user space.
261 * @from: Source address, in kernel space.
262 * @n: Number of bytes to copy.
263 *
264 * Context: User context only. This function may sleep.
265 *
266 * Copy data from kernel space to user space. Caller must check
267 * the specified block with access_ok() before calling this function.
268 *
269 * Returns number of bytes that could not be copied.
270 * On success, this will be zero.
271 */
272 static always_inline unsigned long
273 __copy_to_user(void __user *to, const void *from, unsigned long n)
274 {
275 if (__builtin_constant_p(n)) {
276 unsigned long ret;
278 switch (n) {
279 case 1:
280 __put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret, 1);
281 return ret;
282 case 2:
283 __put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret, 2);
284 return ret;
285 case 4:
286 __put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret, 4);
287 return ret;
288 }
289 }
290 return __copy_to_user_ll(to, from, n);
291 }
293 /**
294 * __copy_from_user: - Copy a block of data from user space, with less checking.
295 * @to: Destination address, in kernel space.
296 * @from: Source address, in user space.
297 * @n: Number of bytes to copy.
298 *
299 * Context: User context only. This function may sleep.
300 *
301 * Copy data from user space to kernel space. Caller must check
302 * the specified block with access_ok() before calling this function.
303 *
304 * Returns number of bytes that could not be copied.
305 * On success, this will be zero.
306 *
307 * If some data could not be copied, this function will pad the copied
308 * data to the requested size using zero bytes.
309 */
310 static always_inline unsigned long
311 __copy_from_user(void *to, const void __user *from, unsigned long n)
312 {
313 if (__builtin_constant_p(n)) {
314 unsigned long ret;
316 switch (n) {
317 case 1:
318 __get_user_size(*(u8 *)to, from, 1, ret, 1);
319 return ret;
320 case 2:
321 __get_user_size(*(u16 *)to, from, 2, ret, 2);
322 return ret;
323 case 4:
324 __get_user_size(*(u32 *)to, from, 4, ret, 4);
325 return ret;
326 }
327 }
328 return __copy_from_user_ll(to, from, n);
329 }
331 unsigned long copy_to_user(void __user *to, const void *from, unsigned long n);
332 unsigned long copy_from_user(void *to,
333 const void __user *from, unsigned long n);
335 unsigned long clear_user(void __user *mem, unsigned long len);
336 unsigned long __clear_user(void __user *mem, unsigned long len);
338 #endif /* __i386_UACCESS_H */