direct-io.hg

view patches/linux-2.6.16.29/smp-alts.patch @ 11750:d845c9522d9e

[HVM][SVM] Check if SVM is disabled by the BIOS before enabling it.

Newer BIOS implementations will be able to disable the SVM feature,
although an additional test of an MSR (VMCR 0xC0010114 bit 4) is
necessary (set equals disabled). Bit 4 of MSR 0xc0010114 returns 0
(SVM enabled) on machines with older BIOS' without the SVM disable
feature support.

Signed-off-by: Wei Huang <wei.huang2@amd.com>=20
Signed-off-by: Tom Woller <thomas.woller@amd.com>=20
author kfraser@localhost.localdomain
date Thu Oct 12 16:12:10 2006 +0100 (2006-10-12)
parents 041be3f6b38e
children
line source
1 diff -pruN ../orig-linux-2.6.16.29/arch/i386/Kconfig ./arch/i386/Kconfig
2 --- ../orig-linux-2.6.16.29/arch/i386/Kconfig 2006-09-12 19:02:10.000000000 +0100
3 +++ ./arch/i386/Kconfig 2006-09-19 14:05:48.000000000 +0100
4 @@ -202,6 +202,19 @@ config SMP
6 If you don't know what to do here, say N.
8 +config SMP_ALTERNATIVES
9 + bool "SMP alternatives support (EXPERIMENTAL)"
10 + depends on SMP && EXPERIMENTAL
11 + help
12 + Try to reduce the overhead of running an SMP kernel on a uniprocessor
13 + host slightly by replacing certain key instruction sequences
14 + according to whether we currently have more than one CPU available.
15 + This should provide a noticeable boost to performance when
16 + running SMP kernels on UP machines, and have negligible impact
17 + when running on an true SMP host.
18 +
19 + If unsure, say N.
20 +
21 config NR_CPUS
22 int "Maximum number of CPUs (2-255)"
23 range 2 255
24 diff -pruN ../orig-linux-2.6.16.29/arch/i386/kernel/Makefile ./arch/i386/kernel/Makefile
25 --- ../orig-linux-2.6.16.29/arch/i386/kernel/Makefile 2006-09-12 19:02:10.000000000 +0100
26 +++ ./arch/i386/kernel/Makefile 2006-09-19 14:05:48.000000000 +0100
27 @@ -37,6 +37,7 @@ obj-$(CONFIG_EFI) += efi.o efi_stub.o
28 obj-$(CONFIG_DOUBLEFAULT) += doublefault.o
29 obj-$(CONFIG_VM86) += vm86.o
30 obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
31 +obj-$(CONFIG_SMP_ALTERNATIVES) += smpalts.o
33 EXTRA_AFLAGS := -traditional
35 diff -pruN ../orig-linux-2.6.16.29/arch/i386/kernel/smpalts.c ./arch/i386/kernel/smpalts.c
36 --- ../orig-linux-2.6.16.29/arch/i386/kernel/smpalts.c 1970-01-01 01:00:00.000000000 +0100
37 +++ ./arch/i386/kernel/smpalts.c 2006-09-19 14:05:48.000000000 +0100
38 @@ -0,0 +1,85 @@
39 +#include <linux/kernel.h>
40 +#include <asm/system.h>
41 +#include <asm/smp_alt.h>
42 +#include <asm/processor.h>
43 +#include <asm/string.h>
44 +
45 +struct smp_replacement_record {
46 + unsigned char targ_size;
47 + unsigned char smp1_size;
48 + unsigned char smp2_size;
49 + unsigned char up_size;
50 + unsigned char feature;
51 + unsigned char data[0];
52 +};
53 +
54 +struct smp_alternative_record {
55 + void *targ_start;
56 + struct smp_replacement_record *repl;
57 +};
58 +
59 +extern struct smp_alternative_record __start_smp_alternatives_table,
60 + __stop_smp_alternatives_table;
61 +extern unsigned long __init_begin, __init_end;
62 +
63 +void prepare_for_smp(void)
64 +{
65 + struct smp_alternative_record *r;
66 + printk(KERN_INFO "Enabling SMP...\n");
67 + for (r = &__start_smp_alternatives_table;
68 + r != &__stop_smp_alternatives_table;
69 + r++) {
70 + BUG_ON(r->repl->targ_size < r->repl->smp1_size);
71 + BUG_ON(r->repl->targ_size < r->repl->smp2_size);
72 + BUG_ON(r->repl->targ_size < r->repl->up_size);
73 + if (system_state == SYSTEM_RUNNING &&
74 + r->targ_start >= (void *)&__init_begin &&
75 + r->targ_start < (void *)&__init_end)
76 + continue;
77 + if (r->repl->feature != (unsigned char)-1 &&
78 + boot_cpu_has(r->repl->feature)) {
79 + memcpy(r->targ_start,
80 + r->repl->data + r->repl->smp1_size,
81 + r->repl->smp2_size);
82 + memset(r->targ_start + r->repl->smp2_size,
83 + 0x90,
84 + r->repl->targ_size - r->repl->smp2_size);
85 + } else {
86 + memcpy(r->targ_start,
87 + r->repl->data,
88 + r->repl->smp1_size);
89 + memset(r->targ_start + r->repl->smp1_size,
90 + 0x90,
91 + r->repl->targ_size - r->repl->smp1_size);
92 + }
93 + }
94 + /* Paranoia */
95 + asm volatile ("jmp 1f\n1:");
96 + mb();
97 +}
98 +
99 +void unprepare_for_smp(void)
100 +{
101 + struct smp_alternative_record *r;
102 + printk(KERN_INFO "Disabling SMP...\n");
103 + for (r = &__start_smp_alternatives_table;
104 + r != &__stop_smp_alternatives_table;
105 + r++) {
106 + BUG_ON(r->repl->targ_size < r->repl->smp1_size);
107 + BUG_ON(r->repl->targ_size < r->repl->smp2_size);
108 + BUG_ON(r->repl->targ_size < r->repl->up_size);
109 + if (system_state == SYSTEM_RUNNING &&
110 + r->targ_start >= (void *)&__init_begin &&
111 + r->targ_start < (void *)&__init_end)
112 + continue;
113 + memcpy(r->targ_start,
114 + r->repl->data + r->repl->smp1_size + r->repl->smp2_size,
115 + r->repl->up_size);
116 + memset(r->targ_start + r->repl->up_size,
117 + 0x90,
118 + r->repl->targ_size - r->repl->up_size);
119 + }
120 + /* Paranoia */
121 + asm volatile ("jmp 1f\n1:");
122 + mb();
123 +}
124 diff -pruN ../orig-linux-2.6.16.29/arch/i386/kernel/smpboot.c ./arch/i386/kernel/smpboot.c
125 --- ../orig-linux-2.6.16.29/arch/i386/kernel/smpboot.c 2006-09-12 19:02:10.000000000 +0100
126 +++ ./arch/i386/kernel/smpboot.c 2006-09-19 14:05:48.000000000 +0100
127 @@ -1218,6 +1218,11 @@ static void __init smp_boot_cpus(unsigne
128 if (max_cpus <= cpucount+1)
129 continue;
131 +#ifdef CONFIG_SMP_ALTERNATIVES
132 + if (kicked == 1)
133 + prepare_for_smp();
134 +#endif
135 +
136 if (((cpu = alloc_cpu_id()) <= 0) || do_boot_cpu(apicid, cpu))
137 printk("CPU #%d not responding - cannot use it.\n",
138 apicid);
139 @@ -1396,6 +1401,11 @@ int __devinit __cpu_up(unsigned int cpu)
140 return -EIO;
141 }
143 +#ifdef CONFIG_SMP_ALTERNATIVES
144 + if (num_online_cpus() == 1)
145 + prepare_for_smp();
146 +#endif
147 +
148 local_irq_enable();
149 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
150 /* Unleash the CPU! */
151 diff -pruN ../orig-linux-2.6.16.29/arch/i386/kernel/vmlinux.lds.S ./arch/i386/kernel/vmlinux.lds.S
152 --- ../orig-linux-2.6.16.29/arch/i386/kernel/vmlinux.lds.S 2006-09-12 19:02:10.000000000 +0100
153 +++ ./arch/i386/kernel/vmlinux.lds.S 2006-09-19 14:05:48.000000000 +0100
154 @@ -34,6 +34,13 @@ SECTIONS
155 __ex_table : AT(ADDR(__ex_table) - LOAD_OFFSET) { *(__ex_table) }
156 __stop___ex_table = .;
158 + . = ALIGN(16);
159 + __start_smp_alternatives_table = .;
160 + __smp_alternatives : { *(__smp_alternatives) }
161 + __stop_smp_alternatives_table = .;
162 +
163 + __smp_replacements : { *(__smp_replacements) }
164 +
165 RODATA
167 /* writeable */
168 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/atomic.h ./include/asm-i386/atomic.h
169 --- ../orig-linux-2.6.16.29/include/asm-i386/atomic.h 2006-09-12 19:02:10.000000000 +0100
170 +++ ./include/asm-i386/atomic.h 2006-09-19 14:05:48.000000000 +0100
171 @@ -4,18 +4,13 @@
172 #include <linux/config.h>
173 #include <linux/compiler.h>
174 #include <asm/processor.h>
175 +#include <asm/smp_alt.h>
177 /*
178 * Atomic operations that C can't guarantee us. Useful for
179 * resource counting etc..
180 */
182 -#ifdef CONFIG_SMP
183 -#define LOCK "lock ; "
184 -#else
185 -#define LOCK ""
186 -#endif
187 -
188 /*
189 * Make sure gcc doesn't try to be clever and move things around
190 * on us. We need to use _exactly_ the address the user gave us,
191 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/bitops.h ./include/asm-i386/bitops.h
192 --- ../orig-linux-2.6.16.29/include/asm-i386/bitops.h 2006-09-12 19:02:10.000000000 +0100
193 +++ ./include/asm-i386/bitops.h 2006-09-19 14:05:48.000000000 +0100
194 @@ -7,6 +7,7 @@
196 #include <linux/config.h>
197 #include <linux/compiler.h>
198 +#include <asm/smp_alt.h>
200 /*
201 * These have to be done with inline assembly: that way the bit-setting
202 @@ -16,12 +17,6 @@
203 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
204 */
206 -#ifdef CONFIG_SMP
207 -#define LOCK_PREFIX "lock ; "
208 -#else
209 -#define LOCK_PREFIX ""
210 -#endif
211 -
212 #define ADDR (*(volatile long *) addr)
214 /**
215 @@ -41,7 +36,7 @@
216 */
217 static inline void set_bit(int nr, volatile unsigned long * addr)
218 {
219 - __asm__ __volatile__( LOCK_PREFIX
220 + __asm__ __volatile__( LOCK
221 "btsl %1,%0"
222 :"+m" (ADDR)
223 :"Ir" (nr));
224 @@ -76,7 +71,7 @@ static inline void __set_bit(int nr, vol
225 */
226 static inline void clear_bit(int nr, volatile unsigned long * addr)
227 {
228 - __asm__ __volatile__( LOCK_PREFIX
229 + __asm__ __volatile__( LOCK
230 "btrl %1,%0"
231 :"+m" (ADDR)
232 :"Ir" (nr));
233 @@ -121,7 +116,7 @@ static inline void __change_bit(int nr,
234 */
235 static inline void change_bit(int nr, volatile unsigned long * addr)
236 {
237 - __asm__ __volatile__( LOCK_PREFIX
238 + __asm__ __volatile__( LOCK
239 "btcl %1,%0"
240 :"+m" (ADDR)
241 :"Ir" (nr));
242 @@ -140,7 +135,7 @@ static inline int test_and_set_bit(int n
243 {
244 int oldbit;
246 - __asm__ __volatile__( LOCK_PREFIX
247 + __asm__ __volatile__( LOCK
248 "btsl %2,%1\n\tsbbl %0,%0"
249 :"=r" (oldbit),"+m" (ADDR)
250 :"Ir" (nr) : "memory");
251 @@ -180,7 +175,7 @@ static inline int test_and_clear_bit(int
252 {
253 int oldbit;
255 - __asm__ __volatile__( LOCK_PREFIX
256 + __asm__ __volatile__( LOCK
257 "btrl %2,%1\n\tsbbl %0,%0"
258 :"=r" (oldbit),"+m" (ADDR)
259 :"Ir" (nr) : "memory");
260 @@ -231,7 +226,7 @@ static inline int test_and_change_bit(in
261 {
262 int oldbit;
264 - __asm__ __volatile__( LOCK_PREFIX
265 + __asm__ __volatile__( LOCK
266 "btcl %2,%1\n\tsbbl %0,%0"
267 :"=r" (oldbit),"+m" (ADDR)
268 :"Ir" (nr) : "memory");
269 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/futex.h ./include/asm-i386/futex.h
270 --- ../orig-linux-2.6.16.29/include/asm-i386/futex.h 2006-09-12 19:02:10.000000000 +0100
271 +++ ./include/asm-i386/futex.h 2006-09-19 14:05:48.000000000 +0100
272 @@ -28,7 +28,7 @@
273 "1: movl %2, %0\n\
274 movl %0, %3\n" \
275 insn "\n" \
276 -"2: " LOCK_PREFIX "cmpxchgl %3, %2\n\
277 +"2: " LOCK "cmpxchgl %3, %2\n\
278 jnz 1b\n\
279 3: .section .fixup,\"ax\"\n\
280 4: mov %5, %1\n\
281 @@ -68,7 +68,7 @@ futex_atomic_op_inuser (int encoded_op,
282 #endif
283 switch (op) {
284 case FUTEX_OP_ADD:
285 - __futex_atomic_op1(LOCK_PREFIX "xaddl %0, %2", ret,
286 + __futex_atomic_op1(LOCK "xaddl %0, %2", ret,
287 oldval, uaddr, oparg);
288 break;
289 case FUTEX_OP_OR:
290 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/rwsem.h ./include/asm-i386/rwsem.h
291 --- ../orig-linux-2.6.16.29/include/asm-i386/rwsem.h 2006-09-12 19:02:10.000000000 +0100
292 +++ ./include/asm-i386/rwsem.h 2006-09-19 14:05:48.000000000 +0100
293 @@ -40,6 +40,7 @@
295 #include <linux/list.h>
296 #include <linux/spinlock.h>
297 +#include <asm/smp_alt.h>
299 struct rwsem_waiter;
301 @@ -99,7 +100,7 @@ static inline void __down_read(struct rw
302 {
303 __asm__ __volatile__(
304 "# beginning down_read\n\t"
305 -LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */
306 +LOCK " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */
307 " js 2f\n\t" /* jump if we weren't granted the lock */
308 "1:\n\t"
309 LOCK_SECTION_START("")
310 @@ -130,7 +131,7 @@ static inline int __down_read_trylock(st
311 " movl %1,%2\n\t"
312 " addl %3,%2\n\t"
313 " jle 2f\n\t"
314 -LOCK_PREFIX " cmpxchgl %2,%0\n\t"
315 +LOCK " cmpxchgl %2,%0\n\t"
316 " jnz 1b\n\t"
317 "2:\n\t"
318 "# ending __down_read_trylock\n\t"
319 @@ -150,7 +151,7 @@ static inline void __down_write(struct r
320 tmp = RWSEM_ACTIVE_WRITE_BIAS;
321 __asm__ __volatile__(
322 "# beginning down_write\n\t"
323 -LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */
324 +LOCK " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */
325 " testl %%edx,%%edx\n\t" /* was the count 0 before? */
326 " jnz 2f\n\t" /* jump if we weren't granted the lock */
327 "1:\n\t"
328 @@ -188,7 +189,7 @@ static inline void __up_read(struct rw_s
329 __s32 tmp = -RWSEM_ACTIVE_READ_BIAS;
330 __asm__ __volatile__(
331 "# beginning __up_read\n\t"
332 -LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */
333 +LOCK " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */
334 " js 2f\n\t" /* jump if the lock is being waited upon */
335 "1:\n\t"
336 LOCK_SECTION_START("")
337 @@ -214,7 +215,7 @@ static inline void __up_write(struct rw_
338 __asm__ __volatile__(
339 "# beginning __up_write\n\t"
340 " movl %2,%%edx\n\t"
341 -LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
342 +LOCK " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
343 " jnz 2f\n\t" /* jump if the lock is being waited upon */
344 "1:\n\t"
345 LOCK_SECTION_START("")
346 @@ -239,7 +240,7 @@ static inline void __downgrade_write(str
347 {
348 __asm__ __volatile__(
349 "# beginning __downgrade_write\n\t"
350 -LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
351 +LOCK " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
352 " js 2f\n\t" /* jump if the lock is being waited upon */
353 "1:\n\t"
354 LOCK_SECTION_START("")
355 @@ -263,7 +264,7 @@ LOCK_PREFIX " addl %2,(%%eax)\n\t"
356 static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
357 {
358 __asm__ __volatile__(
359 -LOCK_PREFIX "addl %1,%0"
360 +LOCK "addl %1,%0"
361 : "=m"(sem->count)
362 : "ir"(delta), "m"(sem->count));
363 }
364 @@ -276,7 +277,7 @@ static inline int rwsem_atomic_update(in
365 int tmp = delta;
367 __asm__ __volatile__(
368 -LOCK_PREFIX "xadd %0,(%2)"
369 +LOCK "xadd %0,(%2)"
370 : "+r"(tmp), "=m"(sem->count)
371 : "r"(sem), "m"(sem->count)
372 : "memory");
373 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/smp_alt.h ./include/asm-i386/smp_alt.h
374 --- ../orig-linux-2.6.16.29/include/asm-i386/smp_alt.h 1970-01-01 01:00:00.000000000 +0100
375 +++ ./include/asm-i386/smp_alt.h 2006-09-19 14:05:48.000000000 +0100
376 @@ -0,0 +1,32 @@
377 +#ifndef __ASM_SMP_ALT_H__
378 +#define __ASM_SMP_ALT_H__
379 +
380 +#include <linux/config.h>
381 +
382 +#ifdef CONFIG_SMP
383 +#if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE)
384 +#define LOCK \
385 + "6677: nop\n" \
386 + ".section __smp_alternatives,\"a\"\n" \
387 + ".long 6677b\n" \
388 + ".long 6678f\n" \
389 + ".previous\n" \
390 + ".section __smp_replacements,\"a\"\n" \
391 + "6678: .byte 1\n" \
392 + ".byte 1\n" \
393 + ".byte 0\n" \
394 + ".byte 1\n" \
395 + ".byte -1\n" \
396 + "lock\n" \
397 + "nop\n" \
398 + ".previous\n"
399 +void prepare_for_smp(void);
400 +void unprepare_for_smp(void);
401 +#else
402 +#define LOCK "lock ; "
403 +#endif
404 +#else
405 +#define LOCK ""
406 +#endif
407 +
408 +#endif /* __ASM_SMP_ALT_H__ */
409 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/spinlock.h ./include/asm-i386/spinlock.h
410 --- ../orig-linux-2.6.16.29/include/asm-i386/spinlock.h 2006-09-12 19:02:10.000000000 +0100
411 +++ ./include/asm-i386/spinlock.h 2006-09-19 14:05:48.000000000 +0100
412 @@ -6,6 +6,7 @@
413 #include <asm/page.h>
414 #include <linux/config.h>
415 #include <linux/compiler.h>
416 +#include <asm/smp_alt.h>
418 /*
419 * Your basic SMP spinlocks, allowing only a single CPU anywhere
420 @@ -23,7 +24,8 @@
422 #define __raw_spin_lock_string \
423 "\n1:\t" \
424 - "lock ; decb %0\n\t" \
425 + LOCK \
426 + "decb %0\n\t" \
427 "jns 3f\n" \
428 "2:\t" \
429 "rep;nop\n\t" \
430 @@ -34,7 +36,8 @@
432 #define __raw_spin_lock_string_flags \
433 "\n1:\t" \
434 - "lock ; decb %0\n\t" \
435 + LOCK \
436 + "decb %0\n\t" \
437 "jns 4f\n\t" \
438 "2:\t" \
439 "testl $0x200, %1\n\t" \
440 @@ -65,10 +68,34 @@ static inline void __raw_spin_lock_flags
441 static inline int __raw_spin_trylock(raw_spinlock_t *lock)
442 {
443 char oldval;
444 +#ifdef CONFIG_SMP_ALTERNATIVES
445 __asm__ __volatile__(
446 - "xchgb %b0,%1"
447 + "1:movb %1,%b0\n"
448 + "movb $0,%1\n"
449 + "2:"
450 + ".section __smp_alternatives,\"a\"\n"
451 + ".long 1b\n"
452 + ".long 3f\n"
453 + ".previous\n"
454 + ".section __smp_replacements,\"a\"\n"
455 + "3: .byte 2b - 1b\n"
456 + ".byte 5f-4f\n"
457 + ".byte 0\n"
458 + ".byte 6f-5f\n"
459 + ".byte -1\n"
460 + "4: xchgb %b0,%1\n"
461 + "5: movb %1,%b0\n"
462 + "movb $0,%1\n"
463 + "6:\n"
464 + ".previous\n"
465 :"=q" (oldval), "=m" (lock->slock)
466 :"0" (0) : "memory");
467 +#else
468 + __asm__ __volatile__(
469 + "xchgb %b0,%1\n"
470 + :"=q" (oldval), "=m" (lock->slock)
471 + :"0" (0) : "memory");
472 +#endif
473 return oldval > 0;
474 }
476 @@ -178,12 +205,12 @@ static inline int __raw_write_trylock(ra
478 static inline void __raw_read_unlock(raw_rwlock_t *rw)
479 {
480 - asm volatile("lock ; incl %0" :"=m" (rw->lock) : : "memory");
481 + asm volatile(LOCK "incl %0" :"=m" (rw->lock) : : "memory");
482 }
484 static inline void __raw_write_unlock(raw_rwlock_t *rw)
485 {
486 - asm volatile("lock ; addl $" RW_LOCK_BIAS_STR ", %0"
487 + asm volatile(LOCK "addl $" RW_LOCK_BIAS_STR ", %0"
488 : "=m" (rw->lock) : : "memory");
489 }
491 diff -pruN ../orig-linux-2.6.16.29/include/asm-i386/system.h ./include/asm-i386/system.h
492 --- ../orig-linux-2.6.16.29/include/asm-i386/system.h 2006-09-12 19:02:10.000000000 +0100
493 +++ ./include/asm-i386/system.h 2006-09-19 14:05:48.000000000 +0100
494 @@ -5,7 +5,7 @@
495 #include <linux/kernel.h>
496 #include <asm/segment.h>
497 #include <asm/cpufeature.h>
498 -#include <linux/bitops.h> /* for LOCK_PREFIX */
499 +#include <asm/smp_alt.h>
501 #ifdef __KERNEL__
503 @@ -271,19 +271,19 @@ static inline unsigned long __cmpxchg(vo
504 unsigned long prev;
505 switch (size) {
506 case 1:
507 - __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
508 + __asm__ __volatile__(LOCK "cmpxchgb %b1,%2"
509 : "=a"(prev)
510 : "q"(new), "m"(*__xg(ptr)), "0"(old)
511 : "memory");
512 return prev;
513 case 2:
514 - __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
515 + __asm__ __volatile__(LOCK "cmpxchgw %w1,%2"
516 : "=a"(prev)
517 : "r"(new), "m"(*__xg(ptr)), "0"(old)
518 : "memory");
519 return prev;
520 case 4:
521 - __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
522 + __asm__ __volatile__(LOCK "cmpxchgl %1,%2"
523 : "=a"(prev)
524 : "r"(new), "m"(*__xg(ptr)), "0"(old)
525 : "memory");
526 @@ -336,7 +336,7 @@ static inline unsigned long long __cmpxc
527 unsigned long long new)
528 {
529 unsigned long long prev;
530 - __asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3"
531 + __asm__ __volatile__(LOCK "cmpxchg8b %3"
532 : "=A"(prev)
533 : "b"((unsigned long)new),
534 "c"((unsigned long)(new >> 32)),
535 @@ -503,11 +503,55 @@ struct alt_instr {
536 #endif
538 #ifdef CONFIG_SMP
539 +#if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE)
540 +#define smp_alt_mb(instr) \
541 +__asm__ __volatile__("6667:\nnop\nnop\nnop\nnop\nnop\nnop\n6668:\n" \
542 + ".section __smp_alternatives,\"a\"\n" \
543 + ".long 6667b\n" \
544 + ".long 6673f\n" \
545 + ".previous\n" \
546 + ".section __smp_replacements,\"a\"\n" \
547 + "6673:.byte 6668b-6667b\n" \
548 + ".byte 6670f-6669f\n" \
549 + ".byte 6671f-6670f\n" \
550 + ".byte 0\n" \
551 + ".byte %c0\n" \
552 + "6669:lock;addl $0,0(%%esp)\n" \
553 + "6670:" instr "\n" \
554 + "6671:\n" \
555 + ".previous\n" \
556 + : \
557 + : "i" (X86_FEATURE_XMM2) \
558 + : "memory")
559 +#define smp_rmb() smp_alt_mb("lfence")
560 +#define smp_mb() smp_alt_mb("mfence")
561 +#define set_mb(var, value) do { \
562 +unsigned long __set_mb_temp; \
563 +__asm__ __volatile__("6667:movl %1, %0\n6668:\n" \
564 + ".section __smp_alternatives,\"a\"\n" \
565 + ".long 6667b\n" \
566 + ".long 6673f\n" \
567 + ".previous\n" \
568 + ".section __smp_replacements,\"a\"\n" \
569 + "6673: .byte 6668b-6667b\n" \
570 + ".byte 6670f-6669f\n" \
571 + ".byte 0\n" \
572 + ".byte 6671f-6670f\n" \
573 + ".byte -1\n" \
574 + "6669: xchg %1, %0\n" \
575 + "6670:movl %1, %0\n" \
576 + "6671:\n" \
577 + ".previous\n" \
578 + : "=m" (var), "=r" (__set_mb_temp) \
579 + : "1" (value) \
580 + : "memory"); } while (0)
581 +#else
582 #define smp_mb() mb()
583 #define smp_rmb() rmb()
584 +#define set_mb(var, value) do { (void) xchg(&var, value); } while (0)
585 +#endif
586 #define smp_wmb() wmb()
587 #define smp_read_barrier_depends() read_barrier_depends()
588 -#define set_mb(var, value) do { (void) xchg(&var, value); } while (0)
589 #else
590 #define smp_mb() barrier()
591 #define smp_rmb() barrier()