ia64/xen-unstable

view patches/linux-2.6.16.29/smp-alts.patch @ 11729:3e9fcbee3c09

[IA64] fix sparse tree build with p2m exposure disabled

Signed-off-by: Alex Williamson <alex.williamson@hp.com>
author awilliam@xenbuild.aw
date Wed Oct 04 22:13:06 2006 -0600 (2006-10-04)
parents 041be3f6b38e
children
line source
1 diff -pruN ../orig-linux-2.6.16.29/arch/i386/Kconfig ./arch/i386/Kconfig
2 --- ../orig-linux-2.6.16.29/arch/i386/Kconfig 2006-09-12 19:02:10.000000000 +0100
3 +++ ./arch/i386/Kconfig 2006-09-19 14:05:48.000000000 +0100
4 @@ -202,6 +202,19 @@ config SMP
6 If you don't know what to do here, say N.
8 +config SMP_ALTERNATIVES
9 + bool "SMP alternatives support (EXPERIMENTAL)"
10 + depends on SMP && EXPERIMENTAL
11 + help
12 + Try to reduce the overhead of running an SMP kernel on a uniprocessor
13 + host slightly by replacing certain key instruction sequences
14 + according to whether we currently have more than one CPU available.
15 + This should provide a noticeable boost to performance when
16 + running SMP kernels on UP machines, and have negligible impact
17 + when running on an true SMP host.
18 +
19 + If unsure, say N.
20 +
21 config NR_CPUS
22 int "Maximum number of CPUs (2-255)"
23 range 2 255
24 diff -pruN ../orig-linux-2.6.16.29/arch/i386/kernel/Makefile ./arch/i386/kernel/Makefile
25 --- ../orig-linux-2.6.16.29/arch/i386/kernel/Makefile 2006-09-12 19:02:10.000000000 +0100
26 +++ ./arch/i386/kernel/Makefile 2006-09-19 14:05:48.000000000 +0100
27 @@ -37,6 +37,7 @@ obj-$(CONFIG_EFI) += efi.o efi_stub.o
28 obj-$(CONFIG_DOUBLEFAULT) += doublefault.o
29 obj-$(CONFIG_VM86) += vm86.o
30 obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
31 +obj-$(CONFIG_SMP_ALTERNATIVES) += smpalts.o
33 EXTRA_AFLAGS := -traditional
35 diff -pruN ../orig-linux-2.6.16.29/arch/i386/kernel/smpalts.c ./arch/i386/kernel/smpalts.c
36 --- ../orig-linux-2.6.16.29/arch/i386/kernel/smpalts.c 1970-01-01 01:00:00.000000000 +0100
37 +++ ./arch/i386/kernel/smpalts.c 2006-09-19 14:05:48.000000000 +0100
38 @@ -0,0 +1,85 @@
39 +#include <linux/kernel.h>
40 +#include <asm/system.h>
41 +#include <asm/smp_alt.h>
42 +#include <asm/processor.h>
43 +#include <asm/string.h>
44 +
45 +struct smp_replacement_record {
46 + unsigned char targ_size;
47 + unsigned char smp1_size;
48 + unsigned char smp2_size;
49 + unsigned char up_size;
50 + unsigned char feature;
51 + unsigned char data[0];
52 +};
53 +
54 +struct smp_alternative_record {
55 + void *targ_start;
56 + struct smp_replacement_record *repl;
57 +};
58 +
59 +extern struct smp_alternative_record __start_smp_alternatives_table,
60 + __stop_smp_alternatives_table;
61 +extern unsigned long __init_begin, __init_end;
62 +
63 +void prepare_for_smp(void)
64 +{
65 + struct smp_alternative_record *r;
66 + printk(KERN_INFO "Enabling SMP...\n");
67 + for (r = &__start_smp_alternatives_table;
68 + r != &__stop_smp_alternatives_table;
69 + r++) {
70 + BUG_ON(r->repl->targ_size < r->repl->smp1_size);
71 + BUG_ON(r->repl->targ_size < r->repl->smp2_size);
72 + BUG_ON(r->repl->targ_size < r->repl->up_size);
73 + if (system_state == SYSTEM_RUNNING &&
74 + r->targ_start >= (void *)&__init_begin &&
75 + r->targ_start < (void *)&__init_end)
76 + continue;
77 + if (r->repl->feature != (unsigned char)-1 &&
78 + boot_cpu_has(r->repl->feature)) {
79 + memcpy(r->targ_start,
80 + r->repl->data + r->repl->smp1_size,
81 + r->repl->smp2_size);
82 + memset(r->targ_start + r->repl->smp2_size,
83 + 0x90,
84 + r->repl->targ_size - r->repl->smp2_size);
85 + } else {
86 + memcpy(r->targ_start,
87 + r->repl->data,
88 + r->repl->smp1_size);
89 + memset(r->targ_start + r->repl->smp1_size,
90 + 0x90,
91 + r->repl->targ_size - r->repl->smp1_size);
92 + }
93 + }
94 + /* Paranoia */
95 + asm volatile ("jmp 1f\n1:");
96 + mb();
97 +}
98 +
99 +void unprepare_for_smp(void)
100 +{
101 + struct smp_alternative_record *r;
102 + printk(KERN_INFO "Disabling SMP...\n");
103 + for (r = &__start_smp_alternatives_table;
104 + r != &__stop_smp_alternatives_table;
105 + r++) {
106 + BUG_ON(r->repl->targ_size < r->repl->smp1_size);
107 + BUG_ON(r->repl->targ_size < r->repl->smp2_size);
108 + BUG_ON(r->repl->targ_size < r->repl->up_size);
109 + if (system_state == SYSTEM_RUNNING &&
110 + r->targ_start >= (void *)&__init_begin &&
111 + r->targ_start < (void *)&__init_end)
112 + continue;
113 + memcpy(r->targ_start,
114 + r->repl->data + r->repl->smp1_size + r->repl->smp2_size,
115 + r->repl->up_size);
116 + memset(r->targ_start + r->repl->up_size,
117 + 0x90,
118 + r->repl->targ_size - r->repl->up_size);
119 + }
120 + /* Paranoia */
121 + asm volatile ("jmp 1f\n1:");
122 + mb();
123 +}
124 diff -pruN ../orig-linux-2.6.16.29/arch/i386/kernel/smpboot.c ./arch/i386/kernel/smpboot.c
125 --- ../orig-linux-2.6.16.29/arch/i386/kernel/smpboot.c 2006-09-12 19:02:10.000000000 +0100
126 +++ ./arch/i386/kernel/smpboot.c 2006-09-19 14:05:48.000000000 +0100
127 @@ -1218,6 +1218,11 @@ static void __init smp_boot_cpus(unsigne
128 if (max_cpus <= cpucount+1)
129 continue;
131 +#ifdef CONFIG_SMP_ALTERNATIVES
132 + if (kicked == 1)
133 + prepare_for_smp();
134 +#endif
135 +
136 if (((cpu = alloc_cpu_id()) <= 0) || do_boot_cpu(apicid, cpu))
137 printk("CPU #%d not responding - cannot use it.\n",
138 apicid);
139 @@ -1396,6 +1401,11 @@ int __devinit __cpu_up(unsigned int cpu)
140 return -EIO;
141 }
143 +#ifdef CONFIG_SMP_ALTERNATIVES
144 + if (num_online_cpus() == 1)
145 + prepare_for_smp();
146 +#endif
147 +
148 local_irq_enable();
149 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
150 /* Unleash the CPU! */
151 diff -pruN ../orig-linux-2.6.16.29/arch/i386/kernel/vmlinux.lds.S ./arch/i386/kernel/vmlinux.lds.S
152 --- ../orig-linux-2.6.16.29/arch/i386/kernel/vmlinux.lds.S 2006-09-12 19:02:10.000000000 +0100
153 +++ ./arch/i386/kernel/vmlinux.lds.S 2006-09-19 14:05:48.000000000 +0100
154 @@ -34,6 +34,13 @@ SECTIONS
155 __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) { *(__ex_table) }
156 __stop___ex_table = .;
158 + . = ALIGN(16);
159 + __start_smp_alternatives_table = .;
160 + __smp_alternatives : { *(__smp_alternatives) }
161 + __stop_smp_alternatives_table = .;
162 +
163 + __smp_replacements : { *(__smp_replacements) }
164 +
165 RODATA
167 /* writeable */
168 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/atomic.h ./include/asm-i386/atomic.h
169 --- ../orig-linux-2.6.16.29/include/asm-i386/atomic.h 2006-09-12 19:02:10.000000000 +0100
170 +++ ./include/asm-i386/atomic.h 2006-09-19 14:05:48.000000000 +0100
171 @@ -4,18 +4,13 @@
172 #include <linux/config.h>
173 #include <linux/compiler.h>
174 #include <asm/processor.h>
175 +#include <asm/smp_alt.h>
177 /*
178 * Atomic operations that C can't guarantee us. Useful for
179 * resource counting etc..
180 */
182 -#ifdef CONFIG_SMP
183 -#define LOCK "lock ; "
184 -#else
185 -#define LOCK ""
186 -#endif
187 -
188 /*
189 * Make sure gcc doesn't try to be clever and move things around
190 * on us. We need to use _exactly_ the address the user gave us,
191 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/bitops.h ./include/asm-i386/bitops.h
192 --- ../orig-linux-2.6.16.29/include/asm-i386/bitops.h 2006-09-12 19:02:10.000000000 +0100
193 +++ ./include/asm-i386/bitops.h 2006-09-19 14:05:48.000000000 +0100
194 @@ -7,6 +7,7 @@
196 #include <linux/config.h>
197 #include <linux/compiler.h>
198 +#include <asm/smp_alt.h>
200 /*
201 * These have to be done with inline assembly: that way the bit-setting
202 @@ -16,12 +17,6 @@
203 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
204 */
206 -#ifdef CONFIG_SMP
207 -#define LOCK_PREFIX "lock ; "
208 -#else
209 -#define LOCK_PREFIX ""
210 -#endif
211 -
212 #define ADDR (*(volatile long *) addr)
214 /**
215 @@ -41,7 +36,7 @@
216 */
217 static inline void set_bit(int nr, volatile unsigned long * addr)
218 {
219 - __asm__ __volatile__( LOCK_PREFIX
220 + __asm__ __volatile__( LOCK
221 "btsl %1,%0"
222 :"+m" (ADDR)
223 :"Ir" (nr));
224 @@ -76,7 +71,7 @@ static inline void __set_bit(int nr, vol
225 */
226 static inline void clear_bit(int nr, volatile unsigned long * addr)
227 {
228 - __asm__ __volatile__( LOCK_PREFIX
229 + __asm__ __volatile__( LOCK
230 "btrl %1,%0"
231 :"+m" (ADDR)
232 :"Ir" (nr));
233 @@ -121,7 +116,7 @@ static inline void __change_bit(int nr,
234 */
235 static inline void change_bit(int nr, volatile unsigned long * addr)
236 {
237 - __asm__ __volatile__( LOCK_PREFIX
238 + __asm__ __volatile__( LOCK
239 "btcl %1,%0"
240 :"+m" (ADDR)
241 :"Ir" (nr));
242 @@ -140,7 +135,7 @@ static inline int test_and_set_bit(int n
243 {
244 int oldbit;
246 - __asm__ __volatile__( LOCK_PREFIX
247 + __asm__ __volatile__( LOCK
248 "btsl %2,%1\n\tsbbl %0,%0"
249 :"=r" (oldbit),"+m" (ADDR)
250 :"Ir" (nr) : "memory");
251 @@ -180,7 +175,7 @@ static inline int test_and_clear_bit(int
252 {
253 int oldbit;
255 - __asm__ __volatile__( LOCK_PREFIX
256 + __asm__ __volatile__( LOCK
257 "btrl %2,%1\n\tsbbl %0,%0"
258 :"=r" (oldbit),"+m" (ADDR)
259 :"Ir" (nr) : "memory");
260 @@ -231,7 +226,7 @@ static inline int test_and_change_bit(in
261 {
262 int oldbit;
264 - __asm__ __volatile__( LOCK_PREFIX
265 + __asm__ __volatile__( LOCK
266 "btcl %2,%1\n\tsbbl %0,%0"
267 :"=r" (oldbit),"+m" (ADDR)
268 :"Ir" (nr) : "memory");
269 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/futex.h ./include/asm-i386/futex.h
270 --- ../orig-linux-2.6.16.29/include/asm-i386/futex.h 2006-09-12 19:02:10.000000000 +0100
271 +++ ./include/asm-i386/futex.h 2006-09-19 14:05:48.000000000 +0100
272 @@ -28,7 +28,7 @@
273 "1: movl %2, %0\n\
274 movl %0, %3\n" \
275 insn "\n" \
276 -"2: " LOCK_PREFIX "cmpxchgl %3, %2\n\
277 +"2: " LOCK "cmpxchgl %3, %2\n\
278 jnz 1b\n\
279 3: .section .fixup,\"ax\"\n\
280 4: mov %5, %1\n\
281 @@ -68,7 +68,7 @@ futex_atomic_op_inuser (int encoded_op,
282 #endif
283 switch (op) {
284 case FUTEX_OP_ADD:
285 - __futex_atomic_op1(LOCK_PREFIX "xaddl %0, %2", ret,
286 + __futex_atomic_op1(LOCK "xaddl %0, %2", ret,
287 oldval, uaddr, oparg);
288 break;
289 case FUTEX_OP_OR:
290 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/rwsem.h ./include/asm-i386/rwsem.h
291 --- ../orig-linux-2.6.16.29/include/asm-i386/rwsem.h 2006-09-12 19:02:10.000000000 +0100
292 +++ ./include/asm-i386/rwsem.h 2006-09-19 14:05:48.000000000 +0100
293 @@ -40,6 +40,7 @@
295 #include <linux/list.h>
296 #include <linux/spinlock.h>
297 +#include <asm/smp_alt.h>
299 struct rwsem_waiter;
301 @@ -99,7 +100,7 @@ static inline void __down_read(struct rw
302 {
303 __asm__ __volatile__(
304 "# beginning down_read\n\t"
305 -LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */
306 +LOCK " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */
307 " js 2f\n\t" /* jump if we weren't granted the lock */
308 "1:\n\t"
309 LOCK_SECTION_START("")
310 @@ -130,7 +131,7 @@ static inline int __down_read_trylock(st
311 " movl %1,%2\n\t"
312 " addl %3,%2\n\t"
313 " jle 2f\n\t"
314 -LOCK_PREFIX " cmpxchgl %2,%0\n\t"
315 +LOCK " cmpxchgl %2,%0\n\t"
316 " jnz 1b\n\t"
317 "2:\n\t"
318 "# ending __down_read_trylock\n\t"
319 @@ -150,7 +151,7 @@ static inline void __down_write(struct r
320 tmp = RWSEM_ACTIVE_WRITE_BIAS;
321 __asm__ __volatile__(
322 "# beginning down_write\n\t"
323 -LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */
324 +LOCK " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */
325 " testl %%edx,%%edx\n\t" /* was the count 0 before? */
326 " jnz 2f\n\t" /* jump if we weren't granted the lock */
327 "1:\n\t"
328 @@ -188,7 +189,7 @@ static inline void __up_read(struct rw_s
329 __s32 tmp = -RWSEM_ACTIVE_READ_BIAS;
330 __asm__ __volatile__(
331 "# beginning __up_read\n\t"
332 -LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */
333 +LOCK " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */
334 " js 2f\n\t" /* jump if the lock is being waited upon */
335 "1:\n\t"
336 LOCK_SECTION_START("")
337 @@ -214,7 +215,7 @@ static inline void __up_write(struct rw_
338 __asm__ __volatile__(
339 "# beginning __up_write\n\t"
340 " movl %2,%%edx\n\t"
341 -LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
342 +LOCK " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
343 " jnz 2f\n\t" /* jump if the lock is being waited upon */
344 "1:\n\t"
345 LOCK_SECTION_START("")
346 @@ -239,7 +240,7 @@ static inline void __downgrade_write(str
347 {
348 __asm__ __volatile__(
349 "# beginning __downgrade_write\n\t"
350 -LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
351 +LOCK " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
352 " js 2f\n\t" /* jump if the lock is being waited upon */
353 "1:\n\t"
354 LOCK_SECTION_START("")
355 @@ -263,7 +264,7 @@ LOCK_PREFIX " addl %2,(%%eax)\n\t"
356 static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
357 {
358 __asm__ __volatile__(
359 -LOCK_PREFIX "addl %1,%0"
360 +LOCK "addl %1,%0"
361 : "=m"(sem->count)
362 : "ir"(delta), "m"(sem->count));
363 }
364 @@ -276,7 +277,7 @@ static inline int rwsem_atomic_update(in
365 int tmp = delta;
367 __asm__ __volatile__(
368 -LOCK_PREFIX "xadd %0,(%2)"
369 +LOCK "xadd %0,(%2)"
370 : "+r"(tmp), "=m"(sem->count)
371 : "r"(sem), "m"(sem->count)
372 : "memory");
373 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/smp_alt.h ./include/asm-i386/smp_alt.h
374 --- ../orig-linux-2.6.16.29/include/asm-i386/smp_alt.h 1970-01-01 01:00:00.000000000 +0100
375 +++ ./include/asm-i386/smp_alt.h 2006-09-19 14:05:48.000000000 +0100
376 @@ -0,0 +1,32 @@
377 +#ifndef __ASM_SMP_ALT_H__
378 +#define __ASM_SMP_ALT_H__
379 +
380 +#include <linux/config.h>
381 +
382 +#ifdef CONFIG_SMP
383 +#if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE)
384 +#define LOCK \
385 + "6677: nop\n" \
386 + ".section __smp_alternatives,\"a\"\n" \
387 + ".long 6677b\n" \
388 + ".long 6678f\n" \
389 + ".previous\n" \
390 + ".section __smp_replacements,\"a\"\n" \
391 + "6678: .byte 1\n" \
392 + ".byte 1\n" \
393 + ".byte 0\n" \
394 + ".byte 1\n" \
395 + ".byte -1\n" \
396 + "lock\n" \
397 + "nop\n" \
398 + ".previous\n"
399 +void prepare_for_smp(void);
400 +void unprepare_for_smp(void);
401 +#else
402 +#define LOCK "lock ; "
403 +#endif
404 +#else
405 +#define LOCK ""
406 +#endif
407 +
408 +#endif /* __ASM_SMP_ALT_H__ */
409 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/spinlock.h ./include/asm-i386/spinlock.h
410 --- ../orig-linux-2.6.16.29/include/asm-i386/spinlock.h 2006-09-12 19:02:10.000000000 +0100
411 +++ ./include/asm-i386/spinlock.h 2006-09-19 14:05:48.000000000 +0100
412 @@ -6,6 +6,7 @@
413 #include <asm/page.h>
414 #include <linux/config.h>
415 #include <linux/compiler.h>
416 +#include <asm/smp_alt.h>
418 /*
419 * Your basic SMP spinlocks, allowing only a single CPU anywhere
420 @@ -23,7 +24,8 @@
422 #define __raw_spin_lock_string \
423 "\n1:\t" \
424 - "lock ; decb %0\n\t" \
425 + LOCK \
426 + "decb %0\n\t" \
427 "jns 3f\n" \
428 "2:\t" \
429 "rep;nop\n\t" \
430 @@ -34,7 +36,8 @@
432 #define __raw_spin_lock_string_flags \
433 "\n1:\t" \
434 - "lock ; decb %0\n\t" \
435 + LOCK \
436 + "decb %0\n\t" \
437 "jns 4f\n\t" \
438 "2:\t" \
439 "testl $0x200, %1\n\t" \
440 @@ -65,10 +68,34 @@ static inline void __raw_spin_lock_flags
441 static inline int __raw_spin_trylock(raw_spinlock_t *lock)
442 {
443 char oldval;
444 +#ifdef CONFIG_SMP_ALTERNATIVES
445 __asm__ __volatile__(
446 - "xchgb %b0,%1"
447 + "1:movb %1,%b0\n"
448 + "movb $0,%1\n"
449 + "2:"
450 + ".section __smp_alternatives,\"a\"\n"
451 + ".long 1b\n"
452 + ".long 3f\n"
453 + ".previous\n"
454 + ".section __smp_replacements,\"a\"\n"
455 + "3: .byte 2b - 1b\n"
456 + ".byte 5f-4f\n"
457 + ".byte 0\n"
458 + ".byte 6f-5f\n"
459 + ".byte -1\n"
460 + "4: xchgb %b0,%1\n"
461 + "5: movb %1,%b0\n"
462 + "movb $0,%1\n"
463 + "6:\n"
464 + ".previous\n"
465 :"=q" (oldval), "=m" (lock->slock)
466 :"0" (0) : "memory");
467 +#else
468 + __asm__ __volatile__(
469 + "xchgb %b0,%1\n"
470 + :"=q" (oldval), "=m" (lock->slock)
471 + :"0" (0) : "memory");
472 +#endif
473 return oldval > 0;
474 }
476 @@ -178,12 +205,12 @@ static inline int __raw_write_trylock(ra
478 static inline void __raw_read_unlock(raw_rwlock_t *rw)
479 {
480 - asm volatile("lock ; incl %0" :"=m" (rw->lock) : : "memory");
481 + asm volatile(LOCK "incl %0" :"=m" (rw->lock) : : "memory");
482 }
484 static inline void __raw_write_unlock(raw_rwlock_t *rw)
485 {
486 - asm volatile("lock ; addl $" RW_LOCK_BIAS_STR ", %0"
487 + asm volatile(LOCK "addl $" RW_LOCK_BIAS_STR ", %0"
488 : "=m" (rw->lock) : : "memory");
489 }
491 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/system.h ./include/asm-i386/system.h
492 --- ../orig-linux-2.6.16.29/include/asm-i386/system.h 2006-09-12 19:02:10.000000000 +0100
493 +++ ./include/asm-i386/system.h 2006-09-19 14:05:48.000000000 +0100
494 @@ -5,7 +5,7 @@
495 #include <linux/kernel.h>
496 #include <asm/segment.h>
497 #include <asm/cpufeature.h>
498 -#include <linux/bitops.h> /* for LOCK_PREFIX */
499 +#include <asm/smp_alt.h>
501 #ifdef __KERNEL__
503 @@ -271,19 +271,19 @@ static inline unsigned long __cmpxchg(vo
504 unsigned long prev;
505 switch (size) {
506 case 1:
507 - __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
508 + __asm__ __volatile__(LOCK "cmpxchgb %b1,%2"
509 : "=a"(prev)
510 : "q"(new), "m"(*__xg(ptr)), "0"(old)
511 : "memory");
512 return prev;
513 case 2:
514 - __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
515 + __asm__ __volatile__(LOCK "cmpxchgw %w1,%2"
516 : "=a"(prev)
517 : "r"(new), "m"(*__xg(ptr)), "0"(old)
518 : "memory");
519 return prev;
520 case 4:
521 - __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
522 + __asm__ __volatile__(LOCK "cmpxchgl %1,%2"
523 : "=a"(prev)
524 : "r"(new), "m"(*__xg(ptr)), "0"(old)
525 : "memory");
526 @@ -336,7 +336,7 @@ static inline unsigned long long __cmpxc
527 unsigned long long new)
528 {
529 unsigned long long prev;
530 - __asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3"
531 + __asm__ __volatile__(LOCK "cmpxchg8b %3"
532 : "=A"(prev)
533 : "b"((unsigned long)new),
534 "c"((unsigned long)(new >> 32)),
535 @@ -503,11 +503,55 @@ struct alt_instr {
536 #endif
538 #ifdef CONFIG_SMP
539 +#if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE)
540 +#define smp_alt_mb(instr) \
541 +__asm__ __volatile__("6667:\nnop\nnop\nnop\nnop\nnop\nnop\n6668:\n" \
542 + ".section __smp_alternatives,\"a\"\n" \
543 + ".long 6667b\n" \
544 + ".long 6673f\n" \
545 + ".previous\n" \
546 + ".section __smp_replacements,\"a\"\n" \
547 + "6673:.byte 6668b-6667b\n" \
548 + ".byte 6670f-6669f\n" \
549 + ".byte 6671f-6670f\n" \
550 + ".byte 0\n" \
551 + ".byte %c0\n" \
552 + "6669:lock;addl $0,0(%%esp)\n" \
553 + "6670:" instr "\n" \
554 + "6671:\n" \
555 + ".previous\n" \
556 + : \
557 + : "i" (X86_FEATURE_XMM2) \
558 + : "memory")
559 +#define smp_rmb() smp_alt_mb("lfence")
560 +#define smp_mb() smp_alt_mb("mfence")
561 +#define set_mb(var, value) do { \
562 +unsigned long __set_mb_temp; \
563 +__asm__ __volatile__("6667:movl %1, %0\n6668:\n" \
564 + ".section __smp_alternatives,\"a\"\n" \
565 + ".long 6667b\n" \
566 + ".long 6673f\n" \
567 + ".previous\n" \
568 + ".section __smp_replacements,\"a\"\n" \
569 + "6673: .byte 6668b-6667b\n" \
570 + ".byte 6670f-6669f\n" \
571 + ".byte 0\n" \
572 + ".byte 6671f-6670f\n" \
573 + ".byte -1\n" \
574 + "6669: xchg %1, %0\n" \
575 + "6670:movl %1, %0\n" \
576 + "6671:\n" \
577 + ".previous\n" \
578 + : "=m" (var), "=r" (__set_mb_temp) \
579 + : "1" (value) \
580 + : "memory"); } while (0)
581 +#else
582 #define smp_mb() mb()
583 #define smp_rmb() rmb()
584 +#define set_mb(var, value) do { (void) xchg(&var, value); } while (0)
585 +#endif
586 #define smp_wmb() wmb()
587 #define smp_read_barrier_depends() read_barrier_depends()
588 -#define set_mb(var, value) do { (void) xchg(&var, value); } while (0)
589 #else
590 #define smp_mb() barrier()
591 #define smp_rmb() barrier()