win-pvdrivers

view liblfds.6/src/abstraction/abstraction_dcas.c @ 766:6300617040e0

Big changes - not ready for production use.
Removed all the custom DMA handling code as it was completely incompatible with the Windows verifier.
Added liblfds (using the lock free stack) from liblfds.org so that grant's can be obtained at DIRQL.
Fixed xennet and xenvbd to support the changes.
xenusb and xenscsi almost certainly will not yet work after the changes.
author James Harper <james.harper@bendigoit.com.au>
date Sun Jan 31 21:28:42 2010 +1100 (2010-01-31)
parents
children
line source
1 #include "abstraction_internal.h"
7 /****************************************************************************/
8 #if (defined _WIN64 && defined _MSC_VER)
10 /* TRD : 64 bit Windows (user-mode or kernel) on any CPU with the Microsoft C compiler
12 _WIN64 indicates 64 bit Windows
13 _MSC_VER indicates Microsoft C compiler
14 */
16 INLINE unsigned char abstraction_dcas( volatile atom_t *destination, atom_t *exchange, atom_t *compare )
17 {
18 unsigned char
19 cas_result;
21 assert( destination != NULL );
22 assert( exchange != NULL );
23 assert( compare != NULL );
25 cas_result = _InterlockedCompareExchange128( (volatile __int64 *) destination, (__int64) *(exchange+1), (__int64) *exchange, (__int64 *) compare );
27 return( cas_result );
28 }
30 #endif
36 /****************************************************************************/
37 #if (!defined _WIN64 && defined _WIN32 && defined _MSC_VER)
39 /* TRD : 32 bit Windows (user-mode or kernel) on any CPU with the Microsoft C compiler
41 (!defined _WIN64 && defined _WIN32) indicates 32 bit Windows
42 _MSC_VER indicates Microsoft C compiler
43 */
45 INLINE unsigned char abstraction_dcas( volatile atom_t *destination, atom_t *exchange, atom_t *compare )
46 {
47 __int64
48 original_compare;
50 assert( destination != NULL );
51 assert( exchange != NULL );
52 assert( compare != NULL );
54 *(__int64 *) &original_compare = *(__int64 *) compare;
56 *(__int64 *) compare = _InterlockedCompareExchange64( (volatile __int64 *) destination, *(__int64 *) exchange, *(__int64 *) compare );
58 return( (unsigned char) (*(__int64 *) compare == *(__int64 *) &original_compare) );
59 }
61 #endif
67 /****************************************************************************/
68 #if (defined __x86_64__ && __GNUC__ && !defined __pic__)
70 /* TRD : any OS on x64 with GCC for statically linked code
72 __x86_64__ indicates x64
73 __GNUC__ indicates GCC
74 */
76 INLINE unsigned char abstraction_dcas( volatile atom_t *destination, atom_t *exchange, atom_t *compare )
77 {
78 unsigned char
79 cas_result;
81 assert( destination != NULL );
82 assert( exchange != NULL );
83 assert( compare != NULL );
85 __asm__ __volatile__
86 (
87 "lock;" // make cmpxchg16b atomic
88 "cmpxchg16b %0;" // cmpxchg16b sets ZF on success
89 "setz %3;" // if ZF set, set cas_result to 1
91 // output
92 : "+m" (*(volatile atom_t (*)[2]) destination), "+a" (*compare), "+d" (*(compare+1)), "=q" (cas_result)
94 // input
95 : "b" (*exchange), "c" (*(exchange+1))
97 // clobbered
98 : "cc", "memory"
99 );
101 return( cas_result );
102 }
104 #endif
111 /****************************************************************************/
112 #if (defined __i686__ && __GNUC__ && !defined __pic__)
114 /* TRD : any OS on x86 with GCC for statically linked code
116 __i686__ indicates x86
117 __GNUC__ indicates GCC
118 */
120 INLINE unsigned char abstraction_dcas( volatile atom_t *destination, atom_t *exchange, atom_t *compare )
121 {
122 unsigned char
123 cas_result;
125 assert( destination != NULL );
126 assert( exchange != NULL );
127 assert( compare != NULL );
129 __asm__ __volatile__
130 (
131 "lock;" // make cmpxchg8b atomic
132 "cmpxchg8b %0;" // cmpxchg8b sets ZF on success
133 "setz %3;" // if ZF set, set cas_result to 1
135 // output
136 : "+m" (*(volatile atom_t (*)[2]) destination), "+a" (*compare), "+d" (*(compare+1)), "=q" (cas_result)
138 // input
139 : "b" (*exchange), "c" (*(exchange+1))
141 // clobbered
142 : "cc", "memory"
143 );
145 return( cas_result );
146 }
148 #endif
154 /****************************************************************************/
155 #if (defined __x86_64__ && __GNUC__ && defined __pic__)
157 /* TRD : any OS on x64 with GCC for position independent code (e.g. a shared object)
159 __x86_64__ indicates x64
160 __GNUC__ indicates GCC
161 */
163 INLINE unsigned char abstraction_dcas( volatile atom_t *destination, atom_t *exchange, atom_t *compare )
164 {
165 unsigned char
166 cas_result;
168 assert( destination != NULL );
169 assert( exchange != NULL );
170 assert( compare != NULL );
172 /* TRD : with a shared object, we cannot clobber RBX
173 as such, we borrow RSI - we load half of the exchange value into it
174 then swap it with RBX
175 then do the compare-and-swap
176 then swap the original value of RBX back from RSI
177 */
179 __asm__ __volatile__
180 (
181 "xchg %%rsi, %%rbx;" // swap RBI and RBX
182 "lock;" // make cmpxchg16b atomic
183 "cmpxchg16b %0;" // cmpxchg16b sets ZF on success
184 "setz %3;" // if ZF set, set cas_result to 1
185 "xchg %%rbx, %%rsi;" // re-swap RBI and RBX
187 // output
188 : "+m" (*(volatile atom_t (*)[2]) destination), "+a" (*compare), "+d" (*(compare+1)), "=q" (cas_result)
190 // input
191 : "S" (*exchange), "c" (*(exchange+1))
193 // clobbered
194 : "cc", "memory"
195 );
197 return( cas_result );
198 }
200 #endif
207 /****************************************************************************/
208 #if (defined __i686__ && __GNUC__ && defined __pic__)
210 /* TRD : any OS on x86 with GCC for position independent code (e.g. a shared object)
212 __i686__ indicates x86
213 __GNUC__ indicates GCC
214 */
216 INLINE unsigned char abstraction_dcas( volatile atom_t *destination, atom_t *exchange, atom_t *compare )
217 {
218 unsigned char
219 cas_result;
221 assert( destination != NULL );
222 assert( exchange != NULL );
223 assert( compare != NULL );
225 /* TRD : with a shared object, we cannot clobber EBX
226 as such, we borrow ESI - we load half of the exchange value into it
227 then swap it with EBX
228 then do the compare-and-swap
229 then swap the original value of EBX back from ESI
230 */
232 __asm__ __volatile__
233 (
234 "xchg %%esi, %%ebx;" // swap EBI and EBX
235 "lock;" // make cmpxchg8b atomic
236 "cmpxchg8b %0;" // cmpxchg8b sets ZF on success
237 "setz %3;" // if ZF set, set cas_result to 1
238 "xchg %%ebx, %%esi;" // re-swap EBI and EBX
240 // output
241 : "+m" (*(volatile atom_t (*)[2]) destination), "+a" (*compare), "+d" (*(compare+1)), "=q" (cas_result)
243 // input
244 : "S" (*exchange), "c" (*(exchange+1))
246 // clobbered
247 : "cc", "memory"
248 );
250 return( cas_result );
251 }
253 #endif
259 /****************************************************************************/
260 #if (defined __arm__ && __GNUC__)
262 /* TRD : any OS on any ARM with GCC
264 Remember however we need to set into compare the original value of destination.
266 __arm__ indicates ARM
267 __GNUC__ indicates GCC
268 */
270 INLINE unsigned char abstraction_dcas( volatile atom_t *destination, atom_t *exchange, atom_t *compare )
271 {
272 atom_t
273 *local_compare = compare,
274 stored_flag = 1;
276 register atom_t
277 local_exchange_a __asm("r2"),
278 local_exchange_b __asm("r3"),
279 local_compare_a __asm("r4"),
280 local_compare_b __asm("r5"),
281 original_destination_a __asm("r6"),
282 original_destination_b __asm("r7");
284 assert( destination != NULL );
285 assert( exchange != NULL );
286 assert( compare != NULL );
288 /* TRD : some notes
290 the double word ldr and str instructions require contigous registers
291 where the first register is an even number
293 honouring this requirement requires us to specifically specify
294 the registers to use (which is why we're using register __asm("rN")
295 in the declerations above
297 the arguments to the function occupy registers r0, r1 and r2
299 we can use up to and including r8, but r9 can have a frame pointer in it
301 so we make a copy of compare (freeing up r2, so we can use it for a double
302 word load) but use destination (r0) and exchange (r1) directly
304 note LDRD and STRD became available in armv6k
306 apologies for the trickery with the mcr register variable - the code runs
307 out of registers on armv6k
308 */
310 __asm__ __volatile__
311 (
312 " mov %[stored_flag], #1;" // put 1 into stored_flag
313 " mov %[local_exchange_a], #0;" // borrow local_exchange_a for mcr, to save a register
314 " mcr p15, 0, %[local_exchange_a], c7, c10, 5;" // memory barrier (ARM v6 compatible)
315 " ldrd %[local_exchange_a], %[local_exchange_b], [%[exchange]];" // load exchange into local_exchange_a and local_exchange_b (which are r2 and r3, respectively)
316 " ldrd %[local_compare_a], %[local_compare_b], [%[local_compare]];" // load compare into local_compare_a and local_compare_b (which are r4 and r5, respectively)
317 "atomic_dcas:;"
318 " ldrexd %[original_destination_a], %[original_destination_b], [%[destination]];" // load destination into original_destination_a and original_destination_b (which are r6 and r7, respectively)
319 " teq %[original_destination_a], %[local_compare_a];" // compare the first word of destination with the first word of compare
320 " teqeq %[original_destination_b], %[local_compare_b];" // if they're equal, compare the second word of destination with the second word of compare
321 " bne exit;" // if either word of destination does not match its respective word of compare, exit
322 " strexd %[stored_flag], %[local_exchange_a], %[local_exchange_b], [%[destination]];" // if both words were equal, try to store local_exchange_a and local_exchange_b into *destination (on success, strexed puts 0 into stored_flag)
323 " teq %[stored_flag], #0;" // check if stored_flag is 0
324 " bne atomic_dcas;" // if not 0, retry (someone else touched *destination after we loaded but before we stored)
325 "exit:;"
326 " strd %[original_destination_a], %[original_destination_b], [%[local_compare]];" // whether or not the CAS swapped, we always write the original value of destination into *compare
327 " mov %[local_exchange_a], #0;" // borrow local_exchange_a for mcr, to save a register
328 " mcr p15, 0, %[local_exchange_a], c7, c10, 5;" // memory barrier (ARM v6 compatible)
330 // output
331 : "+m" (*(volatile atom_t (*)[2]) destination), "+m" (*(atom_t (*)[2]) local_compare),
332 [stored_flag] "+&r" (stored_flag),
333 [original_destination_a] "+&r" (original_destination_a), [original_destination_b] "+&r" (original_destination_b),
334 [local_compare_a] "+&r" (local_compare_a), [local_compare_b] "+&r" (local_compare_b),
335 [local_exchange_a] "+&r" (local_exchange_a), [local_exchange_b] "+&r" (local_exchange_b)
337 // input
338 : "m" (*(atom_t (*)[2]) exchange),
339 [destination] "r" (destination),
340 [local_compare] "r" (local_compare),
341 [exchange] "r" (exchange)
343 // clobbered
344 : "cc", "memory" // memory is clobbered because we issue a memory barrier
345 );
347 /* TRD : stored_flag is set to 0 on store, 1 on fail
348 we need to return 1 on success, 0 on fail
349 */
351 return( (unsigned char) !stored_flag );
352 }
354 #endif