ia64/xen-unstable

view xen/arch/x86/smp.c @ 9353:eb2a2529f96c

New SMP IPI interface function called on_selected_cpus(), currently implemented
only for x86. The x86 implementation of smp_call_function has been simplified
and is now based on on_slected_cpus().

Signed-off-by: Nitin A Kamble <nitin.a.kamble@intel.com>
Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Tue Mar 21 11:28:03 2006 +0100 (2006-03-21)
parents b9b411b50587
children b7295a83206e
line source
1 /*
2 * Intel SMP support routines.
3 *
4 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
5 * (c) 1998-99, 2000 Ingo Molnar <mingo@redhat.com>
6 *
7 * This code is released under the GNU General Public License version 2 or
8 * later.
9 */
11 #include <xen/config.h>
12 #include <xen/irq.h>
13 #include <xen/sched.h>
14 #include <xen/delay.h>
15 #include <xen/perfc.h>
16 #include <xen/spinlock.h>
17 #include <asm/current.h>
18 #include <asm/smp.h>
19 #include <asm/mc146818rtc.h>
20 #include <asm/flushtlb.h>
21 #include <asm/smpboot.h>
22 #include <asm/hardirq.h>
23 #include <mach_apic.h>
25 /*
26 * Some notes on x86 processor bugs affecting SMP operation:
27 *
28 * Pentium, Pentium Pro, II, III (and all CPUs) have bugs.
29 * The Linux implications for SMP are handled as follows:
30 *
31 * Pentium III / [Xeon]
32 * None of the E1AP-E3AP errata are visible to the user.
33 *
34 * E1AP. see PII A1AP
35 * E2AP. see PII A2AP
36 * E3AP. see PII A3AP
37 *
38 * Pentium II / [Xeon]
39 * None of the A1AP-A3AP errata are visible to the user.
40 *
41 * A1AP. see PPro 1AP
42 * A2AP. see PPro 2AP
43 * A3AP. see PPro 7AP
44 *
45 * Pentium Pro
46 * None of 1AP-9AP errata are visible to the normal user,
47 * except occasional delivery of 'spurious interrupt' as trap #15.
48 * This is very rare and a non-problem.
49 *
50 * 1AP. Linux maps APIC as non-cacheable
51 * 2AP. worked around in hardware
52 * 3AP. fixed in C0 and above steppings microcode update.
53 * Linux does not use excessive STARTUP_IPIs.
54 * 4AP. worked around in hardware
55 * 5AP. symmetric IO mode (normal Linux operation) not affected.
56 * 'noapic' mode has vector 0xf filled out properly.
57 * 6AP. 'noapic' mode might be affected - fixed in later steppings
58 * 7AP. We do not assume writes to the LVT deassering IRQs
59 * 8AP. We do not enable low power mode (deep sleep) during MP bootup
60 * 9AP. We do not use mixed mode
61 */
63 /*
64 * The following functions deal with sending IPIs between CPUs.
65 */
67 static inline int __prepare_ICR (unsigned int shortcut, int vector)
68 {
69 return APIC_DM_FIXED | shortcut | vector | APIC_DEST_LOGICAL;
70 }
72 static inline int __prepare_ICR2 (unsigned int mask)
73 {
74 return SET_APIC_DEST_FIELD(mask);
75 }
77 void __send_IPI_shortcut(unsigned int shortcut, int vector)
78 {
79 /*
80 * Subtle. In the case of the 'never do double writes' workaround
81 * we have to lock out interrupts to be safe. As we don't care
82 * of the value read we use an atomic rmw access to avoid costly
83 * cli/sti. Otherwise we use an even cheaper single atomic write
84 * to the APIC.
85 */
86 unsigned int cfg;
88 /*
89 * Wait for idle.
90 */
91 apic_wait_icr_idle();
93 /*
94 * No need to touch the target chip field
95 */
96 cfg = __prepare_ICR(shortcut, vector);
98 /*
99 * Send the IPI. The write to APIC_ICR fires this off.
100 */
101 apic_write_around(APIC_ICR, cfg);
102 }
104 void send_IPI_self(int vector)
105 {
106 __send_IPI_shortcut(APIC_DEST_SELF, vector);
107 }
109 /*
110 * This is only used on smaller machines.
111 */
112 void send_IPI_mask_bitmask(cpumask_t cpumask, int vector)
113 {
114 unsigned long mask = cpus_addr(cpumask)[0];
115 unsigned long cfg;
116 unsigned long flags;
118 local_irq_save(flags);
120 /*
121 * Wait for idle.
122 */
123 apic_wait_icr_idle();
125 /*
126 * prepare target chip field
127 */
128 cfg = __prepare_ICR2(mask);
129 apic_write_around(APIC_ICR2, cfg);
131 /*
132 * program the ICR
133 */
134 cfg = __prepare_ICR(0, vector);
136 /*
137 * Send the IPI. The write to APIC_ICR fires this off.
138 */
139 apic_write_around(APIC_ICR, cfg);
141 local_irq_restore(flags);
142 }
144 inline void send_IPI_mask_sequence(cpumask_t mask, int vector)
145 {
146 unsigned long cfg, flags;
147 unsigned int query_cpu;
149 /*
150 * Hack. The clustered APIC addressing mode doesn't allow us to send
151 * to an arbitrary mask, so I do a unicasts to each CPU instead. This
152 * should be modified to do 1 message per cluster ID - mbligh
153 */
155 local_irq_save(flags);
157 for (query_cpu = 0; query_cpu < NR_CPUS; ++query_cpu) {
158 if (cpu_isset(query_cpu, mask)) {
160 /*
161 * Wait for idle.
162 */
163 apic_wait_icr_idle();
165 /*
166 * prepare target chip field
167 */
168 cfg = __prepare_ICR2(cpu_to_logical_apicid(query_cpu));
169 apic_write_around(APIC_ICR2, cfg);
171 /*
172 * program the ICR
173 */
174 cfg = __prepare_ICR(0, vector);
176 /*
177 * Send the IPI. The write to APIC_ICR fires this off.
178 */
179 apic_write_around(APIC_ICR, cfg);
180 }
181 }
182 local_irq_restore(flags);
183 }
185 #include <mach_ipi.h>
187 static spinlock_t flush_lock = SPIN_LOCK_UNLOCKED;
188 static cpumask_t flush_cpumask;
189 static unsigned long flush_va;
191 fastcall void smp_invalidate_interrupt(void)
192 {
193 ack_APIC_irq();
194 perfc_incrc(ipis);
195 if ( !__sync_lazy_execstate() )
196 {
197 if ( flush_va == FLUSHVA_ALL )
198 local_flush_tlb();
199 else
200 local_flush_tlb_one(flush_va);
201 }
202 cpu_clear(smp_processor_id(), flush_cpumask);
203 }
205 void __flush_tlb_mask(cpumask_t mask, unsigned long va)
206 {
207 ASSERT(local_irq_is_enabled());
209 if ( cpu_isset(smp_processor_id(), mask) )
210 {
211 local_flush_tlb();
212 cpu_clear(smp_processor_id(), mask);
213 }
215 if ( !cpus_empty(mask) )
216 {
217 spin_lock(&flush_lock);
218 flush_cpumask = mask;
219 flush_va = va;
220 send_IPI_mask(mask, INVALIDATE_TLB_VECTOR);
221 while ( !cpus_empty(flush_cpumask) )
222 cpu_relax();
223 spin_unlock(&flush_lock);
224 }
225 }
227 /* Call with no locks held and interrupts enabled (e.g., softirq context). */
228 void new_tlbflush_clock_period(void)
229 {
230 ASSERT(local_irq_is_enabled());
232 /* Flush everyone else. We definitely flushed just before entry. */
233 if ( num_online_cpus() > 1 )
234 {
235 spin_lock(&flush_lock);
236 flush_cpumask = cpu_online_map;
237 flush_va = FLUSHVA_ALL;
238 send_IPI_allbutself(INVALIDATE_TLB_VECTOR);
239 cpu_clear(smp_processor_id(), flush_cpumask);
240 while ( !cpus_empty(flush_cpumask) )
241 cpu_relax();
242 spin_unlock(&flush_lock);
243 }
245 /* No need for atomicity: we are the only possible updater. */
246 ASSERT(tlbflush_clock == 0);
247 tlbflush_clock++;
248 }
250 static void flush_tlb_all_pge_ipi(void *info)
251 {
252 local_flush_tlb_pge();
253 }
255 void flush_tlb_all_pge(void)
256 {
257 smp_call_function(flush_tlb_all_pge_ipi, 0, 1, 1);
258 local_flush_tlb_pge();
259 }
261 void smp_send_event_check_mask(cpumask_t mask)
262 {
263 cpu_clear(smp_processor_id(), mask);
264 if ( !cpus_empty(mask) )
265 send_IPI_mask(mask, EVENT_CHECK_VECTOR);
266 }
268 /*
269 * Structure and data for smp_call_function()/on_selected_cpus().
270 */
272 struct call_data_struct {
273 void (*func) (void *info);
274 void *info;
275 int wait;
276 atomic_t started;
277 atomic_t finished;
278 cpumask_t selected;
279 };
281 static DEFINE_SPINLOCK(call_lock);
282 static struct call_data_struct *call_data;
284 int smp_call_function(
285 void (*func) (void *info),
286 void *info,
287 int retry,
288 int wait)
289 {
290 cpumask_t allbutself = cpu_online_map;
291 cpu_clear(smp_processor_id(), allbutself);
292 return on_selected_cpus(allbutself, func, info, retry, wait);
293 }
295 extern int on_selected_cpus(
296 cpumask_t selected,
297 void (*func) (void *info),
298 void *info,
299 int retry,
300 int wait)
301 {
302 struct call_data_struct data;
303 unsigned int nr_cpus = cpus_weight(selected);
305 ASSERT(local_irq_is_enabled());
307 data.func = func;
308 data.info = info;
309 data.wait = wait;
310 atomic_set(&data.started, 0);
311 atomic_set(&data.finished, 0);
312 data.selected = selected;
314 spin_lock(&call_lock);
316 call_data = &data;
317 wmb();
319 send_IPI_mask(selected, CALL_FUNCTION_VECTOR);
321 while ( atomic_read(wait ? &data.finished : &data.started) != nr_cpus )
322 cpu_relax();
324 spin_unlock(&call_lock);
326 return 0;
327 }
329 static void stop_this_cpu (void *dummy)
330 {
331 clear_bit(smp_processor_id(), &cpu_online_map);
333 disable_local_APIC();
335 for ( ; ; )
336 __asm__ __volatile__ ( "hlt" );
337 }
339 void smp_send_stop(void)
340 {
341 /* Stop all other CPUs in the system. */
342 smp_call_function(stop_this_cpu, NULL, 1, 0);
344 local_irq_disable();
345 disable_local_APIC();
346 local_irq_enable();
347 }
349 fastcall void smp_event_check_interrupt(struct cpu_user_regs *regs)
350 {
351 ack_APIC_irq();
352 perfc_incrc(ipis);
353 }
355 fastcall void smp_call_function_interrupt(struct cpu_user_regs *regs)
356 {
357 void (*func)(void *info) = call_data->func;
358 void *info = call_data->info;
360 ack_APIC_irq();
361 perfc_incrc(ipis);
363 if ( !cpu_isset(smp_processor_id(), call_data->selected) )
364 return;
366 if ( call_data->wait )
367 {
368 (*func)(info);
369 mb();
370 atomic_inc(&call_data->finished);
371 }
372 else
373 {
374 mb();
375 atomic_inc(&call_data->started);
376 (*func)(info);
377 }
378 }