1 #include "lfds600_abstraction_internal.h"
7 /****************************************************************************/
8 #if (defined _WIN64 && defined _MSC_VER)
10 /* TRD : 64 bit Windows (user-mode or kernel) on any CPU with the Microsoft C compiler
12 _WIN64 indicates 64 bit Windows
13 _MSC_VER indicates Microsoft C compiler
16 LFDS600_INLINE unsigned char lfds600_abstraction_dcas( volatile lfds600_atom_t *destination, lfds600_atom_t *exchange, lfds600_atom_t *compare )
21 assert( destination != NULL );
22 assert( exchange != NULL );
23 assert( compare != NULL );
25 cas_result = _InterlockedCompareExchange128( (volatile __int64 *) destination, (__int64) *(exchange+1), (__int64) *exchange, (__int64 *) compare );
36 /****************************************************************************/
37 #if (!defined _WIN64 && defined _WIN32 && defined _MSC_VER)
39 /* TRD : 32 bit Windows (user-mode or kernel) on any CPU with the Microsoft C compiler
41 (!defined _WIN64 && defined _WIN32) indicates 32 bit Windows
42 _MSC_VER indicates Microsoft C compiler
45 LFDS600_INLINE unsigned char lfds600_abstraction_dcas( volatile lfds600_atom_t *destination, lfds600_atom_t *exchange, lfds600_atom_t *compare )
50 assert( destination != NULL );
51 assert( exchange != NULL );
52 assert( compare != NULL );
54 *(__int64 *) &original_compare = *(__int64 *) compare;
56 *(__int64 *) compare = _InterlockedCompareExchange64( (volatile __int64 *) destination, *(__int64 *) exchange, *(__int64 *) compare );
58 return( (unsigned char) (*(__int64 *) compare == *(__int64 *) &original_compare) );
67 /****************************************************************************/
68 #if (defined __x86_64__ && __GNUC__ && !defined __pic__)
70 /* TRD : any OS on x64 with GCC for statically linked code
72 __x86_64__ indicates x64
73 __GNUC__ indicates GCC
76 LFDS600_INLINE unsigned char lfds600_abstraction_dcas( volatile lfds600_atom_t *destination, lfds600_atom_t *exchange, lfds600_atom_t *compare )
81 assert( destination != NULL );
82 assert( exchange != NULL );
83 assert( compare != NULL );
87 "lock;" // make cmpxchg16b atomic
88 "cmpxchg16b %0;" // cmpxchg16b sets ZF on success
89 "setz %3;" // if ZF set, set cas_result to 1
92 : "+m" (*(volatile lfds600_atom_t (*)[2]) destination), "+a" (*compare), "+d" (*(compare+1)), "=q" (cas_result)
95 : "b" (*exchange), "c" (*(exchange+1))
101 return( cas_result );
111 /****************************************************************************/
112 #if (defined __i686__ && __GNUC__ && !defined __pic__)
114 /* TRD : any OS on x86 with GCC for statically linked code
116 __i686__ indicates x86
117 __GNUC__ indicates GCC
120 LFDS600_INLINE unsigned char lfds600_abstraction_dcas( volatile lfds600_atom_t *destination, lfds600_atom_t *exchange, lfds600_atom_t *compare )
125 assert( destination != NULL );
126 assert( exchange != NULL );
127 assert( compare != NULL );
131 "lock;" // make cmpxchg8b atomic
132 "cmpxchg8b %0;" // cmpxchg8b sets ZF on success
133 "setz %3;" // if ZF set, set cas_result to 1
136 : "+m" (*(volatile lfds600_atom_t (*)[2]) destination), "+a" (*compare), "+d" (*(compare+1)), "=q" (cas_result)
139 : "b" (*exchange), "c" (*(exchange+1))
145 return( cas_result );
154 /****************************************************************************/
155 #if (defined __x86_64__ && __GNUC__ && defined __pic__)
157 /* TRD : any OS on x64 with GCC for position independent code (e.g. a shared object)
159 __x86_64__ indicates x64
160 __GNUC__ indicates GCC
163 LFDS600_INLINE unsigned char lfds600_abstraction_dcas( volatile lfds600_atom_t *destination, lfds600_atom_t *exchange, lfds600_atom_t *compare )
168 assert( destination != NULL );
169 assert( exchange != NULL );
170 assert( compare != NULL );
172 /* TRD : with a shared object, we cannot clobber RBX
173 as such, we borrow RSI - we load half of the exchange value into it
174 then swap it with RBX
175 then do the compare-and-swap
176 then swap the original value of RBX back from RSI
181 "xchg %%rsi, %%rbx;" // swap RBI and RBX
182 "lock;" // make cmpxchg16b atomic
183 "cmpxchg16b %0;" // cmpxchg16b sets ZF on success
184 "setz %3;" // if ZF set, set cas_result to 1
185 "xchg %%rbx, %%rsi;" // re-swap RBI and RBX
188 : "+m" (*(volatile lfds600_atom_t (*)[2]) destination), "+a" (*compare), "+d" (*(compare+1)), "=q" (cas_result)
191 : "S" (*exchange), "c" (*(exchange+1))
197 return( cas_result );
207 /****************************************************************************/
208 #if (defined __i686__ && __GNUC__ && defined __pic__)
210 /* TRD : any OS on x86 with GCC for position independent code (e.g. a shared object)
212 __i686__ indicates x86
213 __GNUC__ indicates GCC
216 LFDS600_INLINE unsigned char lfds600_abstraction_dcas( volatile lfds600_atom_t *destination, lfds600_atom_t *exchange, lfds600_atom_t *compare )
221 assert( destination != NULL );
222 assert( exchange != NULL );
223 assert( compare != NULL );
225 /* TRD : with a shared object, we cannot clobber EBX
226 as such, we borrow ESI - we load half of the exchange value into it
227 then swap it with EBX
228 then do the compare-and-swap
229 then swap the original value of EBX back from ESI
234 "xchg %%esi, %%ebx;" // swap EBI and EBX
235 "lock;" // make cmpxchg8b atomic
236 "cmpxchg8b %0;" // cmpxchg8b sets ZF on success
237 "setz %3;" // if ZF set, set cas_result to 1
238 "xchg %%ebx, %%esi;" // re-swap EBI and EBX
241 : "+m" (*(volatile lfds600_atom_t (*)[2]) destination), "+a" (*compare), "+d" (*(compare+1)), "=q" (cas_result)
244 : "S" (*exchange), "c" (*(exchange+1))
250 return( cas_result );
259 /****************************************************************************/
260 #if (defined __arm__ && __GNUC__)
262 /* TRD : any OS on any ARM with GCC
264 Remember however we need to set into compare the original value of destination.
266 __arm__ indicates ARM
267 __GNUC__ indicates GCC
270 LFDS600_INLINE unsigned char lfds600_abstraction_dcas( volatile lfds600_atom_t *destination, lfds600_atom_t *exchange, lfds600_atom_t *compare )
273 *local_compare = compare,
276 register lfds600_atom_t
277 local_exchange_a __asm("r2"),
278 local_exchange_b __asm("r3"),
279 local_compare_a __asm("r4"),
280 local_compare_b __asm("r5"),
281 original_destination_a __asm("r6"),
282 original_destination_b __asm("r7");
284 assert( destination != NULL );
285 assert( exchange != NULL );
286 assert( compare != NULL );
290 the double word ldr and str instructions require contigous registers
291 where the first register is an even number
293 honouring this requirement requires us to specifically specify
294 the registers to use (which is why we're using register __asm("rN")
295 in the declerations above
297 the arguments to the function occupy registers r0, r1 and r2
299 we can use up to and including r8, but r9 can have a frame pointer in it
301 so we make a copy of compare (freeing up r2, so we can use it for a double
302 word load) but use destination (r0) and exchange (r1) directly
304 note LDRD and STRD became available in armv6k
306 apologies for the trickery with the mcr register variable - the code runs
307 out of registers on armv6k
312 " mov %[stored_flag], #1;" // put 1 into stored_flag
313 " mov %[local_exchange_a], #0;" // borrow local_exchange_a for mcr, to save a register
314 " mcr p15, 0, %[local_exchange_a], c7, c10, 5;" // memory barrier (ARM v6 compatible)
315 " ldrd %[local_exchange_a], %[local_exchange_b], [%[exchange]];" // load exchange into local_exchange_a and local_exchange_b (which are r2 and r3, respectively)
316 " ldrd %[local_compare_a], %[local_compare_b], [%[local_compare]];" // load compare into local_compare_a and local_compare_b (which are r4 and r5, respectively)
318 " ldrexd %[original_destination_a], %[original_destination_b], [%[destination]];" // load destination into original_destination_a and original_destination_b (which are r6 and r7, respectively)
319 " teq %[original_destination_a], %[local_compare_a];" // compare the first word of destination with the first word of compare
320 " teqeq %[original_destination_b], %[local_compare_b];" // if they're equal, compare the second word of destination with the second word of compare
321 " bne exit;" // if either word of destination does not match its respective word of compare, exit
322 " strexd %[stored_flag], %[local_exchange_a], %[local_exchange_b], [%[destination]];" // if both words were equal, try to store local_exchange_a and local_exchange_b into *destination (on success, strexed puts 0 into stored_flag)
323 " teq %[stored_flag], #0;" // check if stored_flag is 0
324 " bne atomic_dcas;" // if not 0, retry (someone else touched *destination after we loaded but before we stored)
326 " strd %[original_destination_a], %[original_destination_b], [%[local_compare]];" // whether or not the CAS swapped, we always write the original value of destination into *compare
327 " mov %[local_exchange_a], #0;" // borrow local_exchange_a for mcr, to save a register
328 " mcr p15, 0, %[local_exchange_a], c7, c10, 5;" // memory barrier (ARM v6 compatible)
331 : "+m" (*(volatile lfds600_atom_t (*)[2]) destination), "+m" (*(lfds600_atom_t (*)[2]) local_compare),
332 [stored_flag] "+&r" (stored_flag),
333 [original_destination_a] "+&r" (original_destination_a), [original_destination_b] "+&r" (original_destination_b),
334 [local_compare_a] "+&r" (local_compare_a), [local_compare_b] "+&r" (local_compare_b),
335 [local_exchange_a] "+&r" (local_exchange_a), [local_exchange_b] "+&r" (local_exchange_b)
338 : "m" (*(lfds600_atom_t (*)[2]) exchange),
339 [destination] "r" (destination),
340 [local_compare] "r" (local_compare),
341 [exchange] "r" (exchange)
344 : "cc", "memory" // memory is clobbered because we issue a memory barrier
347 /* TRD : stored_flag is set to 0 on store, 1 on fail
348 we need to return 1 on success, 0 on fail
351 return( (unsigned char) !stored_flag );