direct-io.hg

changeset 5466:88198072cbef

bitkeeper revision 1.1713.1.13 (42b16610ojQcHPRC6Ao_1CKXClza9A)

Slightly disgusting hack to avoid using lots of lock instructions on a uniprocessor
machine just because we happened to compile with CONFIG_SMP. Essentially, we
make a big table of all of the instruction sequences which differ in ``easy''
ways between UP and SMP kernels, and then select which one to use at run time.

Signed-off-by: Steven Smith <sos22@cam.ac.uk>
author sos22@douglas.cl.cam.ac.uk
date Thu Jun 16 11:44:16 2005 +0000 (2005-06-16)
parents 7a449c0b860f
children f494f01b62a8 d4ad447e470b
files .rootkeys linux-2.6.11-xen-sparse/arch/xen/i386/Kconfig linux-2.6.11-xen-sparse/arch/xen/i386/kernel/Makefile linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/system.h patches/linux-2.6.11/smp-alts.patch
line diff
     1.1 --- a/.rootkeys	Thu Jun 16 01:47:29 2005 +0000
     1.2 +++ b/.rootkeys	Thu Jun 16 11:44:16 2005 +0000
     1.3 @@ -470,6 +470,7 @@ 42372652KCUP-IOH9RN19YQmGhs4aA patches/l
     1.4  428359d4b3fDYtazwXi4UUmSWaOUew patches/linux-2.6.11/linux-2.6.11.12.patch
     1.5  4296fb998LGSWCcljGKbOCUv3h9uRQ patches/linux-2.6.11/net-csum.patch
     1.6  429ae875I9ZrqrRDjGD34IC2kzDREw patches/linux-2.6.11/rcu-nohz.patch
     1.7 +42b165fcilFTNezi9NIsG2ecLZVU0w patches/linux-2.6.11/smp-alts.patch
     1.8  429ba3007184K-y6WHQ6KgY65-lEIQ patches/linux-2.6.11/udp-frag.patch
     1.9  424f001e_M1Tnxc52rDrmCLelnDWMQ patches/linux-2.6.11/x86_64-linux.patch
    1.10  3f776bd1Hy9rn69ntXBhPReUFw9IEA tools/Makefile
     2.1 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/Kconfig	Thu Jun 16 01:47:29 2005 +0000
     2.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/Kconfig	Thu Jun 16 11:44:16 2005 +0000
     2.3 @@ -372,6 +372,19 @@ config SMP
     2.4  
     2.5  	  If you don't know what to do here, say N.
     2.6  
     2.7 +config SMP_ALTERNATIVES
     2.8 +        bool "SMP alternatives support (EXPERIMENTAL)"
     2.9 +        depends on SMP && EXPERIMENTAL
    2.10 +        help
    2.11 +          Try to reduce the overhead of running an SMP kernel on a uniprocessor
    2.12 +          host slightly by replacing certain key instruction sequences
    2.13 +          according to whether we currently have more than one CPU available.
    2.14 +          This should provide a noticeable boost to performance when
    2.15 +          running SMP kernels on UP machines, and have negligible impact
    2.16 +          when running on an true SMP host.
    2.17 +
    2.18 +          If unsure, say N.
    2.19 +
    2.20  config NR_CPUS
    2.21  	int "Maximum number of CPUs (2-255)"
    2.22  	range 2 255
     3.1 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/Makefile	Thu Jun 16 01:47:29 2005 +0000
     3.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/Makefile	Thu Jun 16 11:44:16 2005 +0000
     3.3 @@ -42,6 +42,7 @@ c-obj-$(CONFIG_ACPI_SRAT) 	+= srat.o
     3.4  c-obj-$(CONFIG_HPET_TIMER) 	+= time_hpet.o
     3.5  c-obj-$(CONFIG_EFI) 		+= efi.o efi_stub.o
     3.6  c-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
     3.7 +c-obj-$(CONFIG_SMP_ALTERNATIVES)+= smpalts.o
     3.8  
     3.9  EXTRA_AFLAGS   := -traditional
    3.10  
     4.1 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c	Thu Jun 16 01:47:29 2005 +0000
     4.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c	Thu Jun 16 11:44:16 2005 +0000
     4.3 @@ -54,6 +54,8 @@
     4.4  #include <asm/desc.h>
     4.5  #include <asm/arch_hooks.h>
     4.6  
     4.7 +#include <asm/smp_alt.h>
     4.8 +
     4.9  #ifndef CONFIG_X86_IO_APIC
    4.10  #define Dprintk(args...)
    4.11  #endif
    4.12 @@ -1186,6 +1188,10 @@ static void __init smp_boot_cpus(unsigne
    4.13  		if (max_cpus <= cpucount+1)
    4.14  			continue;
    4.15  
    4.16 +#ifdef CONFIG_SMP_ALTERNATIVES
    4.17 +		if (kicked == 1)
    4.18 +			prepare_for_smp();
    4.19 +#endif
    4.20  		if (do_boot_cpu(cpu))
    4.21  			printk("CPU #%d not responding - cannot use it.\n",
    4.22  								cpu);
    4.23 @@ -1301,6 +1307,11 @@ void __devinit smp_prepare_boot_cpu(void
    4.24  /* must be called with the cpucontrol mutex held */
    4.25  static int __devinit cpu_enable(unsigned int cpu)
    4.26  {
    4.27 +#ifdef CONFIG_SMP_ALTERNATIVES
    4.28 +	if (num_online_cpus() == 1)
    4.29 +		prepare_for_smp();
    4.30 +#endif
    4.31 +
    4.32  	/* get the target out of its holding state */
    4.33  	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
    4.34  	wmb();
    4.35 @@ -1340,6 +1351,12 @@ int __cpu_disable(void)
    4.36  	fixup_irqs(map);
    4.37  	/* It's now safe to remove this processor from the online map */
    4.38  	cpu_clear(cpu, cpu_online_map);
    4.39 +
    4.40 +#ifdef CONFIG_SMP_ALTERNATIVES
    4.41 +	if (num_online_cpus() == 1)
    4.42 +		unprepare_for_smp();
    4.43 +#endif
    4.44 +
    4.45  	return 0;
    4.46  }
    4.47  
     5.1 --- a/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/system.h	Thu Jun 16 01:47:29 2005 +0000
     5.2 +++ b/linux-2.6.11-xen-sparse/include/asm-xen/asm-i386/system.h	Thu Jun 16 11:44:16 2005 +0000
     5.3 @@ -8,6 +8,7 @@
     5.4  #include <asm/segment.h>
     5.5  #include <asm/cpufeature.h>
     5.6  #include <asm-xen/hypervisor.h>
     5.7 +#include <asm/smp_alt.h>
     5.8  
     5.9  #ifdef __KERNEL__
    5.10  
    5.11 @@ -251,19 +252,19 @@ static inline unsigned long __cmpxchg(vo
    5.12  	unsigned long prev;
    5.13  	switch (size) {
    5.14  	case 1:
    5.15 -		__asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
    5.16 +		__asm__ __volatile__(LOCK "cmpxchgb %b1,%2"
    5.17  				     : "=a"(prev)
    5.18  				     : "q"(new), "m"(*__xg(ptr)), "0"(old)
    5.19  				     : "memory");
    5.20  		return prev;
    5.21  	case 2:
    5.22 -		__asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
    5.23 +		__asm__ __volatile__(LOCK "cmpxchgw %w1,%2"
    5.24  				     : "=a"(prev)
    5.25  				     : "q"(new), "m"(*__xg(ptr)), "0"(old)
    5.26  				     : "memory");
    5.27  		return prev;
    5.28  	case 4:
    5.29 -		__asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
    5.30 +		__asm__ __volatile__(LOCK "cmpxchgl %1,%2"
    5.31  				     : "=a"(prev)
    5.32  				     : "q"(new), "m"(*__xg(ptr)), "0"(old)
    5.33  				     : "memory");
    5.34 @@ -427,11 +428,55 @@ struct alt_instr {
    5.35  #endif
    5.36  
    5.37  #ifdef CONFIG_SMP
    5.38 -#define smp_mb()	mb()
    5.39 +#define smp_wmb()	wmb()
    5.40 +#if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE)
    5.41 +#define smp_alt_mb(instr)                                           \
    5.42 +__asm__ __volatile__("6667:\nnop\nnop\nnop\nnop\nnop\nnop\n6668:\n" \
    5.43 +		     ".section __smp_alternatives,\"a\"\n"          \
    5.44 +		     ".long 6667b\n"                                \
    5.45 +                     ".long 6673f\n"                                \
    5.46 +		     ".previous\n"                                  \
    5.47 +		     ".section __smp_replacements,\"a\"\n"          \
    5.48 +		     "6673:.byte 6668b-6667b\n"                     \
    5.49 +		     ".byte 6670f-6669f\n"                          \
    5.50 +		     ".byte 6671f-6670f\n"                          \
    5.51 +                     ".byte 0\n"                                    \
    5.52 +		     ".byte %c0\n"                                  \
    5.53 +		     "6669:lock;addl $0,0(%%esp)\n"                 \
    5.54 +		     "6670:" instr "\n"                             \
    5.55 +		     "6671:\n"                                      \
    5.56 +		     ".previous\n"                                  \
    5.57 +		     :                                              \
    5.58 +		     : "i" (X86_FEATURE_XMM2)                       \
    5.59 +		     : "memory")
    5.60 +#define smp_rmb() smp_alt_mb("lfence")
    5.61 +#define smp_mb()  smp_alt_mb("mfence")
    5.62 +#define set_mb(var, value) do {                                     \
    5.63 +unsigned long __set_mb_temp;                                        \
    5.64 +__asm__ __volatile__("6667:movl %1, %0\n6668:\n"                    \
    5.65 +		     ".section __smp_alternatives,\"a\"\n"          \
    5.66 +		     ".long 6667b\n"                                \
    5.67 +		     ".long 6673f\n"                                \
    5.68 +		     ".previous\n"                                  \
    5.69 +		     ".section __smp_replacements,\"a\"\n"          \
    5.70 +		     "6673: .byte 6668b-6667b\n"                    \
    5.71 +		     ".byte 6670f-6669f\n"                          \
    5.72 +		     ".byte 0\n"                                    \
    5.73 +		     ".byte 6671f-6670f\n"                          \
    5.74 +		     ".byte -1\n"                                   \
    5.75 +		     "6669: xchg %1, %0\n"                          \
    5.76 +		     "6670:movl %1, %0\n"                           \
    5.77 +		     "6671:\n"                                      \
    5.78 +		     ".previous\n"                                  \
    5.79 +		     : "=m" (var), "=r" (__set_mb_temp)             \
    5.80 +		     : "1" (value)                                  \
    5.81 +		     : "memory"); } while (0)
    5.82 +#else
    5.83  #define smp_rmb()	rmb()
    5.84 -#define smp_wmb()	wmb()
    5.85 +#define smp_mb()	mb()
    5.86 +#define set_mb(var, value) do { xchg(&var, value); } while (0)
    5.87 +#endif
    5.88  #define smp_read_barrier_depends()	read_barrier_depends()
    5.89 -#define set_mb(var, value) do { xchg(&var, value); } while (0)
    5.90  #else
    5.91  #define smp_mb()	barrier()
    5.92  #define smp_rmb()	barrier()
     6.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     6.2 +++ b/patches/linux-2.6.11/smp-alts.patch	Thu Jun 16 11:44:16 2005 +0000
     6.3 @@ -0,0 +1,554 @@
     6.4 +diff -Naur linux-2.6.11/arch/i386/Kconfig linux-2.6.11.post/arch/i386/Kconfig
     6.5 +--- linux-2.6.11/arch/i386/Kconfig	2005-03-02 07:37:49.000000000 +0000
     6.6 ++++ linux-2.6.11.post/arch/i386/Kconfig	2005-06-10 13:42:35.000000000 +0100
     6.7 +@@ -481,6 +481,19 @@
     6.8 + 
     6.9 + 	  If you don't know what to do here, say N.
    6.10 + 
    6.11 ++config SMP_ALTERNATIVES
    6.12 ++	bool "SMP alternatives support (EXPERIMENTAL)"
    6.13 ++	depends on SMP && EXPERIMENTAL
    6.14 ++	help
    6.15 ++	  Try to reduce the overhead of running an SMP kernel on a uniprocessor
    6.16 ++	  host slightly by replacing certain key instruction sequences
    6.17 ++	  according to whether we currently have more than one CPU available.
    6.18 ++	  This should provide a noticeable boost to performance when
    6.19 ++	  running SMP kernels on UP machines, and have negligible impact
    6.20 ++	  when running on an true SMP host.
    6.21 ++
    6.22 ++          If unsure, say N.
    6.23 ++	  
    6.24 + config NR_CPUS
    6.25 + 	int "Maximum number of CPUs (2-255)"
    6.26 + 	range 2 255
    6.27 +diff -Naur linux-2.6.11/arch/i386/kernel/Makefile linux-2.6.11.post/arch/i386/kernel/Makefile
    6.28 +--- linux-2.6.11/arch/i386/kernel/Makefile	2005-03-02 07:37:49.000000000 +0000
    6.29 ++++ linux-2.6.11.post/arch/i386/kernel/Makefile	2005-06-16 11:16:18.555332435 +0100
    6.30 +@@ -32,6 +32,7 @@
    6.31 + obj-$(CONFIG_HPET_TIMER) 	+= time_hpet.o
    6.32 + obj-$(CONFIG_EFI) 		+= efi.o efi_stub.o
    6.33 + obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
    6.34 ++obj-$(CONFIG_SMP_ALTERNATIVES)  += smpalts.o
    6.35 + 
    6.36 + EXTRA_AFLAGS   := -traditional
    6.37 + 
    6.38 +diff -Naur linux-2.6.11/arch/i386/kernel/smpalts.c linux-2.6.11.post/arch/i386/kernel/smpalts.c
    6.39 +--- linux-2.6.11/arch/i386/kernel/smpalts.c	1970-01-01 01:00:00.000000000 +0100
    6.40 ++++ linux-2.6.11.post/arch/i386/kernel/smpalts.c	2005-06-16 11:23:39.300902424 +0100
    6.41 +@@ -0,0 +1,76 @@
    6.42 ++#include <linux/kernel.h>
    6.43 ++#include <asm/system.h>
    6.44 ++#include <asm/smp_alt.h>
    6.45 ++#include <asm/processor.h>
    6.46 ++#include <asm/string.h>
    6.47 ++
    6.48 ++struct smp_replacement_record {
    6.49 ++	unsigned char targ_size;
    6.50 ++	unsigned char smp1_size;
    6.51 ++	unsigned char smp2_size;
    6.52 ++	unsigned char up_size;
    6.53 ++	unsigned char feature;
    6.54 ++	unsigned char data[0];
    6.55 ++};
    6.56 ++
    6.57 ++struct smp_alternative_record {
    6.58 ++	void *targ_start;
    6.59 ++	struct smp_replacement_record *repl;
    6.60 ++};
    6.61 ++
    6.62 ++extern struct smp_alternative_record __start_smp_alternatives_table,
    6.63 ++  __stop_smp_alternatives_table;
    6.64 ++
    6.65 ++void prepare_for_smp(void)
    6.66 ++{
    6.67 ++	struct smp_alternative_record *r;
    6.68 ++	printk(KERN_INFO "Enabling SMP...\n");
    6.69 ++	for (r = &__start_smp_alternatives_table;
    6.70 ++	     r != &__stop_smp_alternatives_table;
    6.71 ++	     r++) {
    6.72 ++		BUG_ON(r->repl->targ_size < r->repl->smp1_size);
    6.73 ++		BUG_ON(r->repl->targ_size < r->repl->smp2_size);
    6.74 ++		BUG_ON(r->repl->targ_size < r->repl->up_size);
    6.75 ++		if (r->repl->feature != (unsigned char)-1 &&
    6.76 ++		    boot_cpu_has(r->repl->feature)) {
    6.77 ++			memcpy(r->targ_start,
    6.78 ++			       r->repl->data + r->repl->smp1_size,
    6.79 ++			       r->repl->smp2_size);
    6.80 ++			memset(r->targ_start + r->repl->smp2_size,
    6.81 ++			       0x90,
    6.82 ++			       r->repl->targ_size - r->repl->smp2_size);
    6.83 ++		} else {
    6.84 ++			memcpy(r->targ_start,
    6.85 ++			       r->repl->data,
    6.86 ++			       r->repl->smp1_size);
    6.87 ++			memset(r->targ_start + r->repl->smp1_size,
    6.88 ++			       0x90,
    6.89 ++			       r->repl->targ_size - r->repl->smp1_size);
    6.90 ++		}
    6.91 ++	}
    6.92 ++	/* Paranoia */
    6.93 ++	asm volatile ("jmp 1f\n1:");
    6.94 ++	mb();
    6.95 ++}
    6.96 ++
    6.97 ++void unprepare_for_smp(void)
    6.98 ++{
    6.99 ++	struct smp_alternative_record *r;
   6.100 ++	printk(KERN_INFO "Disabling SMP...\n");
   6.101 ++	for (r = &__start_smp_alternatives_table;
   6.102 ++	     r != &__stop_smp_alternatives_table;
   6.103 ++	     r++) {
   6.104 ++		BUG_ON(r->repl->targ_size < r->repl->smp1_size);
   6.105 ++		BUG_ON(r->repl->targ_size < r->repl->smp2_size);
   6.106 ++		BUG_ON(r->repl->targ_size < r->repl->up_size);
   6.107 ++		memcpy(r->targ_start,
   6.108 ++		       r->repl->data + r->repl->smp1_size + r->repl->smp2_size,
   6.109 ++		       r->repl->up_size);
   6.110 ++		memset(r->targ_start + r->repl->up_size,
   6.111 ++		       0x90,
   6.112 ++		       r->repl->targ_size - r->repl->up_size);
   6.113 ++	}
   6.114 ++	/* Paranoia */
   6.115 ++	asm volatile ("jmp 1f\n1:");
   6.116 ++	mb();
   6.117 ++}
   6.118 +diff -Naur linux-2.6.11/arch/i386/kernel/smpboot.c linux-2.6.11.post/arch/i386/kernel/smpboot.c
   6.119 +--- linux-2.6.11/arch/i386/kernel/smpboot.c	2005-03-02 07:38:09.000000000 +0000
   6.120 ++++ linux-2.6.11.post/arch/i386/kernel/smpboot.c	2005-06-16 11:17:09.287064617 +0100
   6.121 +@@ -1003,6 +1003,11 @@
   6.122 + 		if (max_cpus <= cpucount+1)
   6.123 + 			continue;
   6.124 + 
   6.125 ++#ifdef CONFIG_SMP_ALTERNATIVES
   6.126 ++		if (kicked == 1)
   6.127 ++			prepare_for_smp();
   6.128 ++#endif
   6.129 ++
   6.130 + 		if (do_boot_cpu(apicid))
   6.131 + 			printk("CPU #%d not responding - cannot use it.\n",
   6.132 + 								apicid);
   6.133 +@@ -1118,6 +1123,11 @@
   6.134 + 		return -EIO;
   6.135 + 	}
   6.136 + 
   6.137 ++#ifdef CONFIG_SMP_ALTERNATIVES
   6.138 ++	if (num_online_cpus() == 1)
   6.139 ++		prepare_for_smp();
   6.140 ++#endif
   6.141 ++
   6.142 + 	local_irq_enable();
   6.143 + 	/* Unleash the CPU! */
   6.144 + 	cpu_set(cpu, smp_commenced_mask);
   6.145 +diff -Naur linux-2.6.11/arch/i386/kernel/vmlinux.lds.S linux-2.6.11.post/arch/i386/kernel/vmlinux.lds.S
   6.146 +--- linux-2.6.11/arch/i386/kernel/vmlinux.lds.S	2005-03-02 07:38:37.000000000 +0000
   6.147 ++++ linux-2.6.11.post/arch/i386/kernel/vmlinux.lds.S	2005-06-10 11:14:14.000000000 +0100
   6.148 +@@ -30,6 +30,13 @@
   6.149 +   __ex_table : { *(__ex_table) }
   6.150 +   __stop___ex_table = .;
   6.151 + 
   6.152 ++  . = ALIGN(16);
   6.153 ++  __start_smp_alternatives_table = .;
   6.154 ++  __smp_alternatives : { *(__smp_alternatives) }
   6.155 ++  __stop_smp_alternatives_table = .;
   6.156 ++
   6.157 ++  __smp_replacements : { *(__smp_replacements) }
   6.158 ++
   6.159 +   RODATA
   6.160 + 
   6.161 +   /* writeable */
   6.162 +diff -Naur linux-2.6.11/include/asm-i386/atomic.h linux-2.6.11.post/include/asm-i386/atomic.h
   6.163 +--- linux-2.6.11/include/asm-i386/atomic.h	2005-03-02 07:37:51.000000000 +0000
   6.164 ++++ linux-2.6.11.post/include/asm-i386/atomic.h	2005-06-13 10:10:39.000000000 +0100
   6.165 +@@ -4,18 +4,13 @@
   6.166 + #include <linux/config.h>
   6.167 + #include <linux/compiler.h>
   6.168 + #include <asm/processor.h>
   6.169 ++#include <asm/smp_alt.h>
   6.170 + 
   6.171 + /*
   6.172 +  * Atomic operations that C can't guarantee us.  Useful for
   6.173 +  * resource counting etc..
   6.174 +  */
   6.175 + 
   6.176 +-#ifdef CONFIG_SMP
   6.177 +-#define LOCK "lock ; "
   6.178 +-#else
   6.179 +-#define LOCK ""
   6.180 +-#endif
   6.181 +-
   6.182 + /*
   6.183 +  * Make sure gcc doesn't try to be clever and move things around
   6.184 +  * on us. We need to use _exactly_ the address the user gave us,
   6.185 +diff -Naur linux-2.6.11/include/asm-i386/bitops.h linux-2.6.11.post/include/asm-i386/bitops.h
   6.186 +--- linux-2.6.11/include/asm-i386/bitops.h	2005-03-02 07:38:12.000000000 +0000
   6.187 ++++ linux-2.6.11.post/include/asm-i386/bitops.h	2005-06-13 10:11:54.000000000 +0100
   6.188 +@@ -7,6 +7,7 @@
   6.189 + 
   6.190 + #include <linux/config.h>
   6.191 + #include <linux/compiler.h>
   6.192 ++#include <asm/smp_alt.h>
   6.193 + 
   6.194 + /*
   6.195 +  * These have to be done with inline assembly: that way the bit-setting
   6.196 +@@ -16,12 +17,6 @@
   6.197 +  * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
   6.198 +  */
   6.199 + 
   6.200 +-#ifdef CONFIG_SMP
   6.201 +-#define LOCK_PREFIX "lock ; "
   6.202 +-#else
   6.203 +-#define LOCK_PREFIX ""
   6.204 +-#endif
   6.205 +-
   6.206 + #define ADDR (*(volatile long *) addr)
   6.207 + 
   6.208 + /**
   6.209 +@@ -41,7 +36,7 @@
   6.210 +  */
   6.211 + static inline void set_bit(int nr, volatile unsigned long * addr)
   6.212 + {
   6.213 +-	__asm__ __volatile__( LOCK_PREFIX
   6.214 ++	__asm__ __volatile__( LOCK
   6.215 + 		"btsl %1,%0"
   6.216 + 		:"=m" (ADDR)
   6.217 + 		:"Ir" (nr));
   6.218 +@@ -76,7 +71,7 @@
   6.219 +  */
   6.220 + static inline void clear_bit(int nr, volatile unsigned long * addr)
   6.221 + {
   6.222 +-	__asm__ __volatile__( LOCK_PREFIX
   6.223 ++	__asm__ __volatile__( LOCK
   6.224 + 		"btrl %1,%0"
   6.225 + 		:"=m" (ADDR)
   6.226 + 		:"Ir" (nr));
   6.227 +@@ -121,7 +116,7 @@
   6.228 +  */
   6.229 + static inline void change_bit(int nr, volatile unsigned long * addr)
   6.230 + {
   6.231 +-	__asm__ __volatile__( LOCK_PREFIX
   6.232 ++	__asm__ __volatile__( LOCK
   6.233 + 		"btcl %1,%0"
   6.234 + 		:"=m" (ADDR)
   6.235 + 		:"Ir" (nr));
   6.236 +@@ -140,7 +135,7 @@
   6.237 + {
   6.238 + 	int oldbit;
   6.239 + 
   6.240 +-	__asm__ __volatile__( LOCK_PREFIX
   6.241 ++	__asm__ __volatile__( LOCK
   6.242 + 		"btsl %2,%1\n\tsbbl %0,%0"
   6.243 + 		:"=r" (oldbit),"=m" (ADDR)
   6.244 + 		:"Ir" (nr) : "memory");
   6.245 +@@ -180,7 +175,7 @@
   6.246 + {
   6.247 + 	int oldbit;
   6.248 + 
   6.249 +-	__asm__ __volatile__( LOCK_PREFIX
   6.250 ++	__asm__ __volatile__( LOCK
   6.251 + 		"btrl %2,%1\n\tsbbl %0,%0"
   6.252 + 		:"=r" (oldbit),"=m" (ADDR)
   6.253 + 		:"Ir" (nr) : "memory");
   6.254 +@@ -231,7 +226,7 @@
   6.255 + {
   6.256 + 	int oldbit;
   6.257 + 
   6.258 +-	__asm__ __volatile__( LOCK_PREFIX
   6.259 ++	__asm__ __volatile__( LOCK
   6.260 + 		"btcl %2,%1\n\tsbbl %0,%0"
   6.261 + 		:"=r" (oldbit),"=m" (ADDR)
   6.262 + 		:"Ir" (nr) : "memory");
   6.263 +diff -Naur linux-2.6.11/include/asm-i386/rwsem.h linux-2.6.11.post/include/asm-i386/rwsem.h
   6.264 +--- linux-2.6.11/include/asm-i386/rwsem.h	2005-03-02 07:38:08.000000000 +0000
   6.265 ++++ linux-2.6.11.post/include/asm-i386/rwsem.h	2005-06-13 10:13:06.000000000 +0100
   6.266 +@@ -40,6 +40,7 @@
   6.267 + 
   6.268 + #include <linux/list.h>
   6.269 + #include <linux/spinlock.h>
   6.270 ++#include <asm/smp_alt.h>
   6.271 + 
   6.272 + struct rwsem_waiter;
   6.273 + 
   6.274 +@@ -99,7 +100,7 @@
   6.275 + {
   6.276 + 	__asm__ __volatile__(
   6.277 + 		"# beginning down_read\n\t"
   6.278 +-LOCK_PREFIX	"  incl      (%%eax)\n\t" /* adds 0x00000001, returns the old value */
   6.279 ++LOCK	        "  incl      (%%eax)\n\t" /* adds 0x00000001, returns the old value */
   6.280 + 		"  js        2f\n\t" /* jump if we weren't granted the lock */
   6.281 + 		"1:\n\t"
   6.282 + 		LOCK_SECTION_START("")
   6.283 +@@ -130,7 +131,7 @@
   6.284 + 		"  movl	     %1,%2\n\t"
   6.285 + 		"  addl      %3,%2\n\t"
   6.286 + 		"  jle	     2f\n\t"
   6.287 +-LOCK_PREFIX	"  cmpxchgl  %2,%0\n\t"
   6.288 ++LOCK	        "  cmpxchgl  %2,%0\n\t"
   6.289 + 		"  jnz	     1b\n\t"
   6.290 + 		"2:\n\t"
   6.291 + 		"# ending __down_read_trylock\n\t"
   6.292 +@@ -150,7 +151,7 @@
   6.293 + 	tmp = RWSEM_ACTIVE_WRITE_BIAS;
   6.294 + 	__asm__ __volatile__(
   6.295 + 		"# beginning down_write\n\t"
   6.296 +-LOCK_PREFIX	"  xadd      %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */
   6.297 ++LOCK	        "  xadd      %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */
   6.298 + 		"  testl     %%edx,%%edx\n\t" /* was the count 0 before? */
   6.299 + 		"  jnz       2f\n\t" /* jump if we weren't granted the lock */
   6.300 + 		"1:\n\t"
   6.301 +@@ -188,7 +189,7 @@
   6.302 + 	__s32 tmp = -RWSEM_ACTIVE_READ_BIAS;
   6.303 + 	__asm__ __volatile__(
   6.304 + 		"# beginning __up_read\n\t"
   6.305 +-LOCK_PREFIX	"  xadd      %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */
   6.306 ++LOCK	        "  xadd      %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */
   6.307 + 		"  js        2f\n\t" /* jump if the lock is being waited upon */
   6.308 + 		"1:\n\t"
   6.309 + 		LOCK_SECTION_START("")
   6.310 +@@ -214,7 +215,7 @@
   6.311 + 	__asm__ __volatile__(
   6.312 + 		"# beginning __up_write\n\t"
   6.313 + 		"  movl      %2,%%edx\n\t"
   6.314 +-LOCK_PREFIX	"  xaddl     %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
   6.315 ++LOCK	        "  xaddl     %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
   6.316 + 		"  jnz       2f\n\t" /* jump if the lock is being waited upon */
   6.317 + 		"1:\n\t"
   6.318 + 		LOCK_SECTION_START("")
   6.319 +@@ -239,7 +240,7 @@
   6.320 + {
   6.321 + 	__asm__ __volatile__(
   6.322 + 		"# beginning __downgrade_write\n\t"
   6.323 +-LOCK_PREFIX	"  addl      %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
   6.324 ++LOCK	        "  addl      %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
   6.325 + 		"  js        2f\n\t" /* jump if the lock is being waited upon */
   6.326 + 		"1:\n\t"
   6.327 + 		LOCK_SECTION_START("")
   6.328 +@@ -263,7 +264,7 @@
   6.329 + static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
   6.330 + {
   6.331 + 	__asm__ __volatile__(
   6.332 +-LOCK_PREFIX	"addl %1,%0"
   6.333 ++LOCK	          "addl %1,%0"
   6.334 + 		: "=m"(sem->count)
   6.335 + 		: "ir"(delta), "m"(sem->count));
   6.336 + }
   6.337 +@@ -276,7 +277,7 @@
   6.338 + 	int tmp = delta;
   6.339 + 
   6.340 + 	__asm__ __volatile__(
   6.341 +-LOCK_PREFIX	"xadd %0,(%2)"
   6.342 ++LOCK  	          "xadd %0,(%2)"
   6.343 + 		: "+r"(tmp), "=m"(sem->count)
   6.344 + 		: "r"(sem), "m"(sem->count)
   6.345 + 		: "memory");
   6.346 +diff -Naur linux-2.6.11/include/asm-i386/smp_alt.h linux-2.6.11.post/include/asm-i386/smp_alt.h
   6.347 +--- linux-2.6.11/include/asm-i386/smp_alt.h	1970-01-01 01:00:00.000000000 +0100
   6.348 ++++ linux-2.6.11.post/include/asm-i386/smp_alt.h	2005-06-16 11:16:50.109433206 +0100
   6.349 +@@ -0,0 +1,32 @@
   6.350 ++#ifndef __ASM_SMP_ALT_H__
   6.351 ++#define __ASM_SMP_ALT_H__
   6.352 ++
   6.353 ++#include <linux/config.h>
   6.354 ++
   6.355 ++#ifdef CONFIG_SMP
   6.356 ++#if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE)
   6.357 ++#define LOCK \
   6.358 ++        "6677: nop\n" \
   6.359 ++	".section __smp_alternatives,\"a\"\n" \
   6.360 ++	".long 6677b\n" \
   6.361 ++	".long 6678f\n" \
   6.362 ++	".previous\n" \
   6.363 ++	".section __smp_replacements,\"a\"\n" \
   6.364 ++	"6678: .byte 1\n" \
   6.365 ++	".byte 1\n" \
   6.366 ++	".byte 0\n" \
   6.367 ++        ".byte 1\n" \
   6.368 ++	".byte -1\n" \
   6.369 ++	"lock\n" \
   6.370 ++	"nop\n" \
   6.371 ++	".previous\n"
   6.372 ++void prepare_for_smp(void);
   6.373 ++void unprepare_for_smp(void);
   6.374 ++#else
   6.375 ++#define LOCK "lock ; "
   6.376 ++#endif
   6.377 ++#else
   6.378 ++#define LOCK ""
   6.379 ++#endif
   6.380 ++
   6.381 ++#endif /* __ASM_SMP_ALT_H__ */
   6.382 +diff -Naur linux-2.6.11/include/asm-i386/spinlock.h linux-2.6.11.post/include/asm-i386/spinlock.h
   6.383 +--- linux-2.6.11/include/asm-i386/spinlock.h	2005-03-02 07:37:50.000000000 +0000
   6.384 ++++ linux-2.6.11.post/include/asm-i386/spinlock.h	2005-06-13 14:13:52.000000000 +0100
   6.385 +@@ -6,6 +6,7 @@
   6.386 + #include <asm/page.h>
   6.387 + #include <linux/config.h>
   6.388 + #include <linux/compiler.h>
   6.389 ++#include <asm/smp_alt.h>
   6.390 + 
   6.391 + asmlinkage int printk(const char * fmt, ...)
   6.392 + 	__attribute__ ((format (printf, 1, 2)));
   6.393 +@@ -47,8 +48,9 @@
   6.394 + #define spin_unlock_wait(x)	do { barrier(); } while(spin_is_locked(x))
   6.395 + 
   6.396 + #define spin_lock_string \
   6.397 +-	"\n1:\t" \
   6.398 +-	"lock ; decb %0\n\t" \
   6.399 ++        "1:\n" \
   6.400 ++	LOCK \
   6.401 ++	"decb %0\n\t" \
   6.402 + 	"jns 3f\n" \
   6.403 + 	"2:\t" \
   6.404 + 	"rep;nop\n\t" \
   6.405 +@@ -58,8 +60,9 @@
   6.406 + 	"3:\n\t"
   6.407 + 
   6.408 + #define spin_lock_string_flags \
   6.409 +-	"\n1:\t" \
   6.410 +-	"lock ; decb %0\n\t" \
   6.411 ++        "1:\n" \
   6.412 ++	LOCK \
   6.413 ++	"decb %0\n\t" \
   6.414 + 	"jns 4f\n\t" \
   6.415 + 	"2:\t" \
   6.416 + 	"testl $0x200, %1\n\t" \
   6.417 +@@ -121,10 +124,34 @@
   6.418 + static inline int _raw_spin_trylock(spinlock_t *lock)
   6.419 + {
   6.420 + 	char oldval;
   6.421 ++#ifdef CONFIG_SMP_ALTERNATIVES
   6.422 + 	__asm__ __volatile__(
   6.423 +-		"xchgb %b0,%1"
   6.424 ++		"1:movb %1,%b0\n"
   6.425 ++		"movb $0,%1\n"
   6.426 ++		"2:"
   6.427 ++		".section __smp_alternatives,\"a\"\n"
   6.428 ++		".long 1b\n"
   6.429 ++		".long 3f\n"
   6.430 ++		".previous\n"
   6.431 ++		".section __smp_replacements,\"a\"\n"
   6.432 ++		"3: .byte 2b - 1b\n"
   6.433 ++		".byte 5f-4f\n"
   6.434 ++		".byte 0\n"
   6.435 ++		".byte 6f-5f\n"
   6.436 ++		".byte -1\n"
   6.437 ++		"4: xchgb %b0,%1\n"
   6.438 ++		"5: movb %1,%b0\n"
   6.439 ++		"movb $0,%1\n"
   6.440 ++		"6:\n"
   6.441 ++		".previous\n"
   6.442 + 		:"=q" (oldval), "=m" (lock->slock)
   6.443 + 		:"0" (0) : "memory");
   6.444 ++#else
   6.445 ++	__asm__ __volatile__(
   6.446 ++		"xchgb %b0,%1\n"
   6.447 ++		:"=q" (oldval), "=m" (lock->slock)
   6.448 ++		:"0" (0) : "memory");
   6.449 ++#endif
   6.450 + 	return oldval > 0;
   6.451 + }
   6.452 + 
   6.453 +@@ -225,8 +252,8 @@
   6.454 + 	__build_write_lock(rw, "__write_lock_failed");
   6.455 + }
   6.456 + 
   6.457 +-#define _raw_read_unlock(rw)		asm volatile("lock ; incl %0" :"=m" ((rw)->lock) : : "memory")
   6.458 +-#define _raw_write_unlock(rw)	asm volatile("lock ; addl $" RW_LOCK_BIAS_STR ",%0":"=m" ((rw)->lock) : : "memory")
   6.459 ++#define _raw_read_unlock(rw)	asm volatile(LOCK "incl %0" :"=m" ((rw)->lock) : : "memory")
   6.460 ++#define _raw_write_unlock(rw)	asm volatile(LOCK "addl $" RW_LOCK_BIAS_STR ",%0":"=m" ((rw)->lock) : : "memory")
   6.461 + 
   6.462 + static inline int _raw_read_trylock(rwlock_t *lock)
   6.463 + {
   6.464 +diff -Naur linux-2.6.11/include/asm-i386/system.h linux-2.6.11.post/include/asm-i386/system.h
   6.465 +--- linux-2.6.11/include/asm-i386/system.h	2005-03-02 07:37:30.000000000 +0000
   6.466 ++++ linux-2.6.11.post/include/asm-i386/system.h	2005-06-15 13:21:40.000000000 +0100
   6.467 +@@ -5,7 +5,7 @@
   6.468 + #include <linux/kernel.h>
   6.469 + #include <asm/segment.h>
   6.470 + #include <asm/cpufeature.h>
   6.471 +-#include <linux/bitops.h> /* for LOCK_PREFIX */
   6.472 ++#include <asm/smp_alt.h>
   6.473 + 
   6.474 + #ifdef __KERNEL__
   6.475 + 
   6.476 +@@ -249,19 +249,19 @@
   6.477 + 	unsigned long prev;
   6.478 + 	switch (size) {
   6.479 + 	case 1:
   6.480 +-		__asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
   6.481 ++		__asm__ __volatile__(LOCK "cmpxchgb %b1,%2"
   6.482 + 				     : "=a"(prev)
   6.483 + 				     : "q"(new), "m"(*__xg(ptr)), "0"(old)
   6.484 + 				     : "memory");
   6.485 + 		return prev;
   6.486 + 	case 2:
   6.487 +-		__asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
   6.488 ++		__asm__ __volatile__(LOCK "cmpxchgw %w1,%2"
   6.489 + 				     : "=a"(prev)
   6.490 + 				     : "q"(new), "m"(*__xg(ptr)), "0"(old)
   6.491 + 				     : "memory");
   6.492 + 		return prev;
   6.493 + 	case 4:
   6.494 +-		__asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
   6.495 ++		__asm__ __volatile__(LOCK "cmpxchgl %1,%2"
   6.496 + 				     : "=a"(prev)
   6.497 + 				     : "q"(new), "m"(*__xg(ptr)), "0"(old)
   6.498 + 				     : "memory");
   6.499 +@@ -425,11 +425,55 @@
   6.500 + #endif
   6.501 + 
   6.502 + #ifdef CONFIG_SMP
   6.503 +-#define smp_mb()	mb()
   6.504 +-#define smp_rmb()	rmb()
   6.505 + #define smp_wmb()	wmb()
   6.506 +-#define smp_read_barrier_depends()	read_barrier_depends()
   6.507 ++#if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE)
   6.508 ++#define smp_alt_mb(instr)                                           \
   6.509 ++__asm__ __volatile__("6667:\nnop\nnop\nnop\nnop\nnop\nnop\n6668:\n" \
   6.510 ++		     ".section __smp_alternatives,\"a\"\n"          \
   6.511 ++		     ".long 6667b\n"                                \
   6.512 ++                     ".long 6673f\n"                                \
   6.513 ++		     ".previous\n"                                  \
   6.514 ++		     ".section __smp_replacements,\"a\"\n"          \
   6.515 ++		     "6673:.byte 6668b-6667b\n"                     \
   6.516 ++		     ".byte 6670f-6669f\n"                          \
   6.517 ++		     ".byte 6671f-6670f\n"                          \
   6.518 ++                     ".byte 0\n"                                    \
   6.519 ++		     ".byte %c0\n"                                  \
   6.520 ++		     "6669:lock;addl $0,0(%%esp)\n"                 \
   6.521 ++		     "6670:" instr "\n"                             \
   6.522 ++		     "6671:\n"                                      \
   6.523 ++		     ".previous\n"                                  \
   6.524 ++		     :                                              \
   6.525 ++		     : "i" (X86_FEATURE_XMM2)                       \
   6.526 ++		     : "memory")
   6.527 ++#define smp_rmb() smp_alt_mb("lfence")
   6.528 ++#define smp_mb()  smp_alt_mb("mfence")
   6.529 ++#define set_mb(var, value) do {                                     \
   6.530 ++unsigned long __set_mb_temp;                                        \
   6.531 ++__asm__ __volatile__("6667:movl %1, %0\n6668:\n"                    \
   6.532 ++		     ".section __smp_alternatives,\"a\"\n"          \
   6.533 ++		     ".long 6667b\n"                                \
   6.534 ++		     ".long 6673f\n"                                \
   6.535 ++		     ".previous\n"                                  \
   6.536 ++		     ".section __smp_replacements,\"a\"\n"          \
   6.537 ++		     "6673: .byte 6668b-6667b\n"                    \
   6.538 ++		     ".byte 6670f-6669f\n"                          \
   6.539 ++		     ".byte 0\n"                                    \
   6.540 ++		     ".byte 6671f-6670f\n"                          \
   6.541 ++		     ".byte -1\n"                                   \
   6.542 ++		     "6669: xchg %1, %0\n"                          \
   6.543 ++		     "6670:movl %1, %0\n"                           \
   6.544 ++		     "6671:\n"                                      \
   6.545 ++		     ".previous\n"                                  \
   6.546 ++		     : "=m" (var), "=r" (__set_mb_temp)             \
   6.547 ++		     : "1" (value)                                  \
   6.548 ++		     : "memory"); } while (0)
   6.549 ++#else
   6.550 ++#define smp_rmb()	rmb()
   6.551 ++#define smp_mb()	mb()
   6.552 + #define set_mb(var, value) do { xchg(&var, value); } while (0)
   6.553 ++#endif
   6.554 ++#define smp_read_barrier_depends()	read_barrier_depends()
   6.555 + #else
   6.556 + #define smp_mb()	barrier()
   6.557 + #define smp_rmb()	barrier()