ia64/xen-unstable

view patches/linux-2.6.12/smp-alts.patch @ 8066:43e25c2653ee

Also, make a small tweak to the 12_block_attach test so that we don't leave
a DomU running with a block device attached.

Signed-off-by: Dan Smith <danms@us.ibm.com>
author emellor@leeni.uk.xensource.com
date Fri Nov 25 23:19:23 2005 +0000 (2005-11-25)
parents 06d84bf87159
children
line source
1 diff -Naur linux-2.6.12/arch/i386/Kconfig linux-2.6.12.post/arch/i386/Kconfig
2 --- linux-2.6.12/arch/i386/Kconfig 2005-06-17 15:48:29.000000000 -0400
3 +++ linux-2.6.12.post/arch/i386/Kconfig 2005-07-25 05:51:21.000000000 -0400
4 @@ -487,6 +487,19 @@
6 If you don't know what to do here, say N.
8 +config SMP_ALTERNATIVES
9 + bool "SMP alternatives support (EXPERIMENTAL)"
10 + depends on SMP && EXPERIMENTAL
11 + help
12 + Try to reduce the overhead of running an SMP kernel on a uniprocessor
13 + host slightly by replacing certain key instruction sequences
14 + according to whether we currently have more than one CPU available.
15 + This should provide a noticeable boost to performance when
16 + running SMP kernels on UP machines, and have negligible impact
17 + when running on an true SMP host.
18 +
19 + If unsure, say N.
20 +
21 config NR_CPUS
22 int "Maximum number of CPUs (2-255)"
23 range 2 255
24 diff -Naur linux-2.6.12/arch/i386/kernel/Makefile linux-2.6.12.post/arch/i386/kernel/Makefile
25 --- linux-2.6.12/arch/i386/kernel/Makefile 2005-06-17 15:48:29.000000000 -0400
26 +++ linux-2.6.12.post/arch/i386/kernel/Makefile 2005-07-25 05:51:21.000000000 -0400
27 @@ -33,6 +33,7 @@
28 obj-$(CONFIG_HPET_TIMER) += time_hpet.o
29 obj-$(CONFIG_EFI) += efi.o efi_stub.o
30 obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
31 +obj-$(CONFIG_SMP_ALTERNATIVES) += smpalts.o
33 EXTRA_AFLAGS := -traditional
35 diff -Naur linux-2.6.12/arch/i386/kernel/smpalts.c linux-2.6.12.post/arch/i386/kernel/smpalts.c
36 --- linux-2.6.12/arch/i386/kernel/smpalts.c 1969-12-31 19:00:00.000000000 -0500
37 +++ linux-2.6.12.post/arch/i386/kernel/smpalts.c 2005-07-25 05:51:21.000000000 -0400
38 @@ -0,0 +1,85 @@
39 +#include <linux/kernel.h>
40 +#include <asm/system.h>
41 +#include <asm/smp_alt.h>
42 +#include <asm/processor.h>
43 +#include <asm/string.h>
44 +
45 +struct smp_replacement_record {
46 + unsigned char targ_size;
47 + unsigned char smp1_size;
48 + unsigned char smp2_size;
49 + unsigned char up_size;
50 + unsigned char feature;
51 + unsigned char data[0];
52 +};
53 +
54 +struct smp_alternative_record {
55 + void *targ_start;
56 + struct smp_replacement_record *repl;
57 +};
58 +
59 +extern struct smp_alternative_record __start_smp_alternatives_table,
60 + __stop_smp_alternatives_table;
61 +extern unsigned long __init_begin, __init_end;
62 +
63 +void prepare_for_smp(void)
64 +{
65 + struct smp_alternative_record *r;
66 + printk(KERN_INFO "Enabling SMP...\n");
67 + for (r = &__start_smp_alternatives_table;
68 + r != &__stop_smp_alternatives_table;
69 + r++) {
70 + BUG_ON(r->repl->targ_size < r->repl->smp1_size);
71 + BUG_ON(r->repl->targ_size < r->repl->smp2_size);
72 + BUG_ON(r->repl->targ_size < r->repl->up_size);
73 + if (system_state == SYSTEM_RUNNING &&
74 + r->targ_start >= (void *)&__init_begin &&
75 + r->targ_start < (void *)&__init_end)
76 + continue;
77 + if (r->repl->feature != (unsigned char)-1 &&
78 + boot_cpu_has(r->repl->feature)) {
79 + memcpy(r->targ_start,
80 + r->repl->data + r->repl->smp1_size,
81 + r->repl->smp2_size);
82 + memset(r->targ_start + r->repl->smp2_size,
83 + 0x90,
84 + r->repl->targ_size - r->repl->smp2_size);
85 + } else {
86 + memcpy(r->targ_start,
87 + r->repl->data,
88 + r->repl->smp1_size);
89 + memset(r->targ_start + r->repl->smp1_size,
90 + 0x90,
91 + r->repl->targ_size - r->repl->smp1_size);
92 + }
93 + }
94 + /* Paranoia */
95 + asm volatile ("jmp 1f\n1:");
96 + mb();
97 +}
98 +
99 +void unprepare_for_smp(void)
100 +{
101 + struct smp_alternative_record *r;
102 + printk(KERN_INFO "Disabling SMP...\n");
103 + for (r = &__start_smp_alternatives_table;
104 + r != &__stop_smp_alternatives_table;
105 + r++) {
106 + BUG_ON(r->repl->targ_size < r->repl->smp1_size);
107 + BUG_ON(r->repl->targ_size < r->repl->smp2_size);
108 + BUG_ON(r->repl->targ_size < r->repl->up_size);
109 + if (system_state == SYSTEM_RUNNING &&
110 + r->targ_start >= (void *)&__init_begin &&
111 + r->targ_start < (void *)&__init_end)
112 + continue;
113 + memcpy(r->targ_start,
114 + r->repl->data + r->repl->smp1_size + r->repl->smp2_size,
115 + r->repl->up_size);
116 + memset(r->targ_start + r->repl->up_size,
117 + 0x90,
118 + r->repl->targ_size - r->repl->up_size);
119 + }
120 + /* Paranoia */
121 + asm volatile ("jmp 1f\n1:");
122 + mb();
123 +}
124 diff -Naur linux-2.6.12/arch/i386/kernel/smpboot.c linux-2.6.12.post/arch/i386/kernel/smpboot.c
125 --- linux-2.6.12/arch/i386/kernel/smpboot.c 2005-06-17 15:48:29.000000000 -0400
126 +++ linux-2.6.12.post/arch/i386/kernel/smpboot.c 2005-07-25 05:51:21.000000000 -0400
127 @@ -1001,6 +1001,11 @@
128 if (max_cpus <= cpucount+1)
129 continue;
131 +#ifdef CONFIG_SMP_ALTERNATIVES
132 + if (kicked == 1)
133 + prepare_for_smp();
134 +#endif
135 +
136 if (do_boot_cpu(apicid))
137 printk("CPU #%d not responding - cannot use it.\n",
138 apicid);
139 @@ -1130,6 +1135,11 @@
140 return -EIO;
141 }
143 +#ifdef CONFIG_SMP_ALTERNATIVES
144 + if (num_online_cpus() == 1)
145 + prepare_for_smp();
146 +#endif
147 +
148 local_irq_enable();
149 /* Unleash the CPU! */
150 cpu_set(cpu, smp_commenced_mask);
151 diff -Naur linux-2.6.12/arch/i386/kernel/vmlinux.lds.S linux-2.6.12.post/arch/i386/kernel/vmlinux.lds.S
152 --- linux-2.6.12/arch/i386/kernel/vmlinux.lds.S 2005-06-17 15:48:29.000000000 -0400
153 +++ linux-2.6.12.post/arch/i386/kernel/vmlinux.lds.S 2005-07-25 05:51:21.000000000 -0400
154 @@ -30,6 +30,13 @@
155 __ex_table : { *(__ex_table) }
156 __stop___ex_table = .;
158 + . = ALIGN(16);
159 + __start_smp_alternatives_table = .;
160 + __smp_alternatives : { *(__smp_alternatives) }
161 + __stop_smp_alternatives_table = .;
162 +
163 + __smp_replacements : { *(__smp_replacements) }
164 +
165 RODATA
167 /* writeable */
168 diff -Naur linux-2.6.12/include/asm-i386/atomic.h linux-2.6.12.post/include/asm-i386/atomic.h
169 --- linux-2.6.12/include/asm-i386/atomic.h 2005-06-17 15:48:29.000000000 -0400
170 +++ linux-2.6.12.post/include/asm-i386/atomic.h 2005-07-25 05:51:21.000000000 -0400
171 @@ -4,18 +4,13 @@
172 #include <linux/config.h>
173 #include <linux/compiler.h>
174 #include <asm/processor.h>
175 +#include <asm/smp_alt.h>
177 /*
178 * Atomic operations that C can't guarantee us. Useful for
179 * resource counting etc..
180 */
182 -#ifdef CONFIG_SMP
183 -#define LOCK "lock ; "
184 -#else
185 -#define LOCK ""
186 -#endif
187 -
188 /*
189 * Make sure gcc doesn't try to be clever and move things around
190 * on us. We need to use _exactly_ the address the user gave us,
191 diff -Naur linux-2.6.12/include/asm-i386/bitops.h linux-2.6.12.post/include/asm-i386/bitops.h
192 --- linux-2.6.12/include/asm-i386/bitops.h 2005-06-17 15:48:29.000000000 -0400
193 +++ linux-2.6.12.post/include/asm-i386/bitops.h 2005-07-25 05:51:21.000000000 -0400
194 @@ -7,6 +7,7 @@
196 #include <linux/config.h>
197 #include <linux/compiler.h>
198 +#include <asm/smp_alt.h>
200 /*
201 * These have to be done with inline assembly: that way the bit-setting
202 @@ -16,12 +17,6 @@
203 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
204 */
206 -#ifdef CONFIG_SMP
207 -#define LOCK_PREFIX "lock ; "
208 -#else
209 -#define LOCK_PREFIX ""
210 -#endif
211 -
212 #define ADDR (*(volatile long *) addr)
214 /**
215 @@ -41,7 +36,7 @@
216 */
217 static inline void set_bit(int nr, volatile unsigned long * addr)
218 {
219 - __asm__ __volatile__( LOCK_PREFIX
220 + __asm__ __volatile__( LOCK
221 "btsl %1,%0"
222 :"=m" (ADDR)
223 :"Ir" (nr));
224 @@ -76,7 +71,7 @@
225 */
226 static inline void clear_bit(int nr, volatile unsigned long * addr)
227 {
228 - __asm__ __volatile__( LOCK_PREFIX
229 + __asm__ __volatile__( LOCK
230 "btrl %1,%0"
231 :"=m" (ADDR)
232 :"Ir" (nr));
233 @@ -121,7 +116,7 @@
234 */
235 static inline void change_bit(int nr, volatile unsigned long * addr)
236 {
237 - __asm__ __volatile__( LOCK_PREFIX
238 + __asm__ __volatile__( LOCK
239 "btcl %1,%0"
240 :"=m" (ADDR)
241 :"Ir" (nr));
242 @@ -140,7 +135,7 @@
243 {
244 int oldbit;
246 - __asm__ __volatile__( LOCK_PREFIX
247 + __asm__ __volatile__( LOCK
248 "btsl %2,%1\n\tsbbl %0,%0"
249 :"=r" (oldbit),"=m" (ADDR)
250 :"Ir" (nr) : "memory");
251 @@ -180,7 +175,7 @@
252 {
253 int oldbit;
255 - __asm__ __volatile__( LOCK_PREFIX
256 + __asm__ __volatile__( LOCK
257 "btrl %2,%1\n\tsbbl %0,%0"
258 :"=r" (oldbit),"=m" (ADDR)
259 :"Ir" (nr) : "memory");
260 @@ -231,7 +226,7 @@
261 {
262 int oldbit;
264 - __asm__ __volatile__( LOCK_PREFIX
265 + __asm__ __volatile__( LOCK
266 "btcl %2,%1\n\tsbbl %0,%0"
267 :"=r" (oldbit),"=m" (ADDR)
268 :"Ir" (nr) : "memory");
269 diff -Naur linux-2.6.12/include/asm-i386/rwsem.h linux-2.6.12.post/include/asm-i386/rwsem.h
270 --- linux-2.6.12/include/asm-i386/rwsem.h 2005-06-17 15:48:29.000000000 -0400
271 +++ linux-2.6.12.post/include/asm-i386/rwsem.h 2005-07-25 05:51:21.000000000 -0400
272 @@ -40,6 +40,7 @@
274 #include <linux/list.h>
275 #include <linux/spinlock.h>
276 +#include <asm/smp_alt.h>
278 struct rwsem_waiter;
280 @@ -99,7 +100,7 @@
281 {
282 __asm__ __volatile__(
283 "# beginning down_read\n\t"
284 -LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */
285 +LOCK " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */
286 " js 2f\n\t" /* jump if we weren't granted the lock */
287 "1:\n\t"
288 LOCK_SECTION_START("")
289 @@ -130,7 +131,7 @@
290 " movl %1,%2\n\t"
291 " addl %3,%2\n\t"
292 " jle 2f\n\t"
293 -LOCK_PREFIX " cmpxchgl %2,%0\n\t"
294 +LOCK " cmpxchgl %2,%0\n\t"
295 " jnz 1b\n\t"
296 "2:\n\t"
297 "# ending __down_read_trylock\n\t"
298 @@ -150,7 +151,7 @@
299 tmp = RWSEM_ACTIVE_WRITE_BIAS;
300 __asm__ __volatile__(
301 "# beginning down_write\n\t"
302 -LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */
303 +LOCK " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */
304 " testl %%edx,%%edx\n\t" /* was the count 0 before? */
305 " jnz 2f\n\t" /* jump if we weren't granted the lock */
306 "1:\n\t"
307 @@ -188,7 +189,7 @@
308 __s32 tmp = -RWSEM_ACTIVE_READ_BIAS;
309 __asm__ __volatile__(
310 "# beginning __up_read\n\t"
311 -LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */
312 +LOCK " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */
313 " js 2f\n\t" /* jump if the lock is being waited upon */
314 "1:\n\t"
315 LOCK_SECTION_START("")
316 @@ -214,7 +215,7 @@
317 __asm__ __volatile__(
318 "# beginning __up_write\n\t"
319 " movl %2,%%edx\n\t"
320 -LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
321 +LOCK " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
322 " jnz 2f\n\t" /* jump if the lock is being waited upon */
323 "1:\n\t"
324 LOCK_SECTION_START("")
325 @@ -239,7 +240,7 @@
326 {
327 __asm__ __volatile__(
328 "# beginning __downgrade_write\n\t"
329 -LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
330 +LOCK " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
331 " js 2f\n\t" /* jump if the lock is being waited upon */
332 "1:\n\t"
333 LOCK_SECTION_START("")
334 @@ -263,7 +264,7 @@
335 static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
336 {
337 __asm__ __volatile__(
338 -LOCK_PREFIX "addl %1,%0"
339 +LOCK "addl %1,%0"
340 : "=m"(sem->count)
341 : "ir"(delta), "m"(sem->count));
342 }
343 @@ -276,7 +277,7 @@
344 int tmp = delta;
346 __asm__ __volatile__(
347 -LOCK_PREFIX "xadd %0,(%2)"
348 +LOCK "xadd %0,(%2)"
349 : "+r"(tmp), "=m"(sem->count)
350 : "r"(sem), "m"(sem->count)
351 : "memory");
352 diff -Naur linux-2.6.12/include/asm-i386/smp_alt.h linux-2.6.12.post/include/asm-i386/smp_alt.h
353 --- linux-2.6.12/include/asm-i386/smp_alt.h 1969-12-31 19:00:00.000000000 -0500
354 +++ linux-2.6.12.post/include/asm-i386/smp_alt.h 2005-07-25 05:51:21.000000000 -0400
355 @@ -0,0 +1,32 @@
356 +#ifndef __ASM_SMP_ALT_H__
357 +#define __ASM_SMP_ALT_H__
358 +
359 +#include <linux/config.h>
360 +
361 +#ifdef CONFIG_SMP
362 +#if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE)
363 +#define LOCK \
364 + "6677: nop\n" \
365 + ".section __smp_alternatives,\"a\"\n" \
366 + ".long 6677b\n" \
367 + ".long 6678f\n" \
368 + ".previous\n" \
369 + ".section __smp_replacements,\"a\"\n" \
370 + "6678: .byte 1\n" \
371 + ".byte 1\n" \
372 + ".byte 0\n" \
373 + ".byte 1\n" \
374 + ".byte -1\n" \
375 + "lock\n" \
376 + "nop\n" \
377 + ".previous\n"
378 +void prepare_for_smp(void);
379 +void unprepare_for_smp(void);
380 +#else
381 +#define LOCK "lock ; "
382 +#endif
383 +#else
384 +#define LOCK ""
385 +#endif
386 +
387 +#endif /* __ASM_SMP_ALT_H__ */
388 diff -Naur linux-2.6.12/include/asm-i386/spinlock.h linux-2.6.12.post/include/asm-i386/spinlock.h
389 --- linux-2.6.12/include/asm-i386/spinlock.h 2005-06-17 15:48:29.000000000 -0400
390 +++ linux-2.6.12.post/include/asm-i386/spinlock.h 2005-07-25 05:51:21.000000000 -0400
391 @@ -6,6 +6,7 @@
392 #include <asm/page.h>
393 #include <linux/config.h>
394 #include <linux/compiler.h>
395 +#include <asm/smp_alt.h>
397 asmlinkage int printk(const char * fmt, ...)
398 __attribute__ ((format (printf, 1, 2)));
399 @@ -47,8 +48,9 @@
400 #define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x))
402 #define spin_lock_string \
403 - "\n1:\t" \
404 - "lock ; decb %0\n\t" \
405 + "1:\n" \
406 + LOCK \
407 + "decb %0\n\t" \
408 "jns 3f\n" \
409 "2:\t" \
410 "rep;nop\n\t" \
411 @@ -58,8 +60,9 @@
412 "3:\n\t"
414 #define spin_lock_string_flags \
415 - "\n1:\t" \
416 - "lock ; decb %0\n\t" \
417 + "1:\n" \
418 + LOCK \
419 + "decb %0\n\t" \
420 "jns 4f\n\t" \
421 "2:\t" \
422 "testl $0x200, %1\n\t" \
423 @@ -121,10 +124,34 @@
424 static inline int _raw_spin_trylock(spinlock_t *lock)
425 {
426 char oldval;
427 +#ifdef CONFIG_SMP_ALTERNATIVES
428 __asm__ __volatile__(
429 - "xchgb %b0,%1"
430 + "1:movb %1,%b0\n"
431 + "movb $0,%1\n"
432 + "2:"
433 + ".section __smp_alternatives,\"a\"\n"
434 + ".long 1b\n"
435 + ".long 3f\n"
436 + ".previous\n"
437 + ".section __smp_replacements,\"a\"\n"
438 + "3: .byte 2b - 1b\n"
439 + ".byte 5f-4f\n"
440 + ".byte 0\n"
441 + ".byte 6f-5f\n"
442 + ".byte -1\n"
443 + "4: xchgb %b0,%1\n"
444 + "5: movb %1,%b0\n"
445 + "movb $0,%1\n"
446 + "6:\n"
447 + ".previous\n"
448 :"=q" (oldval), "=m" (lock->slock)
449 :"0" (0) : "memory");
450 +#else
451 + __asm__ __volatile__(
452 + "xchgb %b0,%1\n"
453 + :"=q" (oldval), "=m" (lock->slock)
454 + :"0" (0) : "memory");
455 +#endif
456 return oldval > 0;
457 }
459 @@ -225,8 +252,8 @@
460 __build_write_lock(rw, "__write_lock_failed");
461 }
463 -#define _raw_read_unlock(rw) asm volatile("lock ; incl %0" :"=m" ((rw)->lock) : : "memory")
464 -#define _raw_write_unlock(rw) asm volatile("lock ; addl $" RW_LOCK_BIAS_STR ",%0":"=m" ((rw)->lock) : : "memory")
465 +#define _raw_read_unlock(rw) asm volatile(LOCK "incl %0" :"=m" ((rw)->lock) : : "memory")
466 +#define _raw_write_unlock(rw) asm volatile(LOCK "addl $" RW_LOCK_BIAS_STR ",%0":"=m" ((rw)->lock) : : "memory")
468 static inline int _raw_read_trylock(rwlock_t *lock)
469 {
470 diff -Naur linux-2.6.12/include/asm-i386/system.h linux-2.6.12.post/include/asm-i386/system.h
471 --- linux-2.6.12/include/asm-i386/system.h 2005-06-17 15:48:29.000000000 -0400
472 +++ linux-2.6.12.post/include/asm-i386/system.h 2005-07-25 05:51:21.000000000 -0400
473 @@ -5,7 +5,7 @@
474 #include <linux/kernel.h>
475 #include <asm/segment.h>
476 #include <asm/cpufeature.h>
477 -#include <linux/bitops.h> /* for LOCK_PREFIX */
478 +#include <asm/smp_alt.h>
480 #ifdef __KERNEL__
482 @@ -249,19 +249,19 @@
483 unsigned long prev;
484 switch (size) {
485 case 1:
486 - __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
487 + __asm__ __volatile__(LOCK "cmpxchgb %b1,%2"
488 : "=a"(prev)
489 : "q"(new), "m"(*__xg(ptr)), "0"(old)
490 : "memory");
491 return prev;
492 case 2:
493 - __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
494 + __asm__ __volatile__(LOCK "cmpxchgw %w1,%2"
495 : "=a"(prev)
496 : "q"(new), "m"(*__xg(ptr)), "0"(old)
497 : "memory");
498 return prev;
499 case 4:
500 - __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
501 + __asm__ __volatile__(LOCK "cmpxchgl %1,%2"
502 : "=a"(prev)
503 : "q"(new), "m"(*__xg(ptr)), "0"(old)
504 : "memory");
505 @@ -425,11 +425,55 @@
506 #endif
508 #ifdef CONFIG_SMP
509 -#define smp_mb() mb()
510 -#define smp_rmb() rmb()
511 #define smp_wmb() wmb()
512 -#define smp_read_barrier_depends() read_barrier_depends()
513 +#if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE)
514 +#define smp_alt_mb(instr) \
515 +__asm__ __volatile__("6667:\nnop\nnop\nnop\nnop\nnop\nnop\n6668:\n" \
516 + ".section __smp_alternatives,\"a\"\n" \
517 + ".long 6667b\n" \
518 + ".long 6673f\n" \
519 + ".previous\n" \
520 + ".section __smp_replacements,\"a\"\n" \
521 + "6673:.byte 6668b-6667b\n" \
522 + ".byte 6670f-6669f\n" \
523 + ".byte 6671f-6670f\n" \
524 + ".byte 0\n" \
525 + ".byte %c0\n" \
526 + "6669:lock;addl $0,0(%%esp)\n" \
527 + "6670:" instr "\n" \
528 + "6671:\n" \
529 + ".previous\n" \
530 + : \
531 + : "i" (X86_FEATURE_XMM2) \
532 + : "memory")
533 +#define smp_rmb() smp_alt_mb("lfence")
534 +#define smp_mb() smp_alt_mb("mfence")
535 +#define set_mb(var, value) do { \
536 +unsigned long __set_mb_temp; \
537 +__asm__ __volatile__("6667:movl %1, %0\n6668:\n" \
538 + ".section __smp_alternatives,\"a\"\n" \
539 + ".long 6667b\n" \
540 + ".long 6673f\n" \
541 + ".previous\n" \
542 + ".section __smp_replacements,\"a\"\n" \
543 + "6673: .byte 6668b-6667b\n" \
544 + ".byte 6670f-6669f\n" \
545 + ".byte 0\n" \
546 + ".byte 6671f-6670f\n" \
547 + ".byte -1\n" \
548 + "6669: xchg %1, %0\n" \
549 + "6670:movl %1, %0\n" \
550 + "6671:\n" \
551 + ".previous\n" \
552 + : "=m" (var), "=r" (__set_mb_temp) \
553 + : "1" (value) \
554 + : "memory"); } while (0)
555 +#else
556 +#define smp_rmb() rmb()
557 +#define smp_mb() mb()
558 #define set_mb(var, value) do { xchg(&var, value); } while (0)
559 +#endif
560 +#define smp_read_barrier_depends() read_barrier_depends()
561 #else
562 #define smp_mb() barrier()
563 #define smp_rmb() barrier()