ia64/xen-unstable

view xen/arch/i386/smp.c @ 945:db2e1ea917df

bitkeeper revision 1.596.1.3 (3fb3b41eWUoRU0H8A0jEX5roXjxKkA)

Many files:
Greatly simplified Xen softirqs. They are now only executed in outermost Xen activation; they are never called within an irq context.
author kaf24@scramble.cl.cam.ac.uk
date Thu Nov 13 16:41:02 2003 +0000 (2003-11-13)
parents 6ef75cc014dc
children 7a554cbf0f58
line source
1 /*
2 * Intel SMP support routines.
3 *
4 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
5 * (c) 1998-99, 2000 Ingo Molnar <mingo@redhat.com>
6 *
7 * This code is released under the GNU General Public License version 2 or
8 * later.
9 */
11 #include <xeno/irq.h>
12 #include <xeno/sched.h>
13 #include <xeno/delay.h>
14 #include <xeno/spinlock.h>
15 #include <asm/smp.h>
16 #include <asm/mc146818rtc.h>
17 #include <asm/pgalloc.h>
18 #include <asm/smpboot.h>
20 #ifdef CONFIG_SMP
22 /*
23 * Some notes on x86 processor bugs affecting SMP operation:
24 *
25 * Pentium, Pentium Pro, II, III (and all CPUs) have bugs.
26 * The Linux implications for SMP are handled as follows:
27 *
28 * Pentium III / [Xeon]
29 * None of the E1AP-E3AP errata are visible to the user.
30 *
31 * E1AP. see PII A1AP
32 * E2AP. see PII A2AP
33 * E3AP. see PII A3AP
34 *
35 * Pentium II / [Xeon]
36 * None of the A1AP-A3AP errata are visible to the user.
37 *
38 * A1AP. see PPro 1AP
39 * A2AP. see PPro 2AP
40 * A3AP. see PPro 7AP
41 *
42 * Pentium Pro
43 * None of 1AP-9AP errata are visible to the normal user,
44 * except occasional delivery of 'spurious interrupt' as trap #15.
45 * This is very rare and a non-problem.
46 *
47 * 1AP. Linux maps APIC as non-cacheable
48 * 2AP. worked around in hardware
49 * 3AP. fixed in C0 and above steppings microcode update.
50 * Linux does not use excessive STARTUP_IPIs.
51 * 4AP. worked around in hardware
52 * 5AP. symmetric IO mode (normal Linux operation) not affected.
53 * 'noapic' mode has vector 0xf filled out properly.
54 * 6AP. 'noapic' mode might be affected - fixed in later steppings
55 * 7AP. We do not assume writes to the LVT deassering IRQs
56 * 8AP. We do not enable low power mode (deep sleep) during MP bootup
57 * 9AP. We do not use mixed mode
58 */
60 /*
61 * the following functions deal with sending IPIs between CPUs.
62 *
63 * We use 'broadcast', CPU->CPU IPIs and self-IPIs too.
64 */
66 static inline int __prepare_ICR (unsigned int shortcut, int vector)
67 {
68 return APIC_DM_FIXED | shortcut | vector | APIC_DEST_LOGICAL;
69 }
71 static inline int __prepare_ICR2 (unsigned int mask)
72 {
73 return SET_APIC_DEST_FIELD(mask);
74 }
76 static inline void __send_IPI_shortcut(unsigned int shortcut, int vector)
77 {
78 /*
79 * Subtle. In the case of the 'never do double writes' workaround
80 * we have to lock out interrupts to be safe. As we don't care
81 * of the value read we use an atomic rmw access to avoid costly
82 * cli/sti. Otherwise we use an even cheaper single atomic write
83 * to the APIC.
84 */
85 unsigned int cfg;
87 /*
88 * Wait for idle.
89 */
90 apic_wait_icr_idle();
92 /*
93 * No need to touch the target chip field
94 */
95 cfg = __prepare_ICR(shortcut, vector);
97 /*
98 * Send the IPI. The write to APIC_ICR fires this off.
99 */
100 apic_write_around(APIC_ICR, cfg);
101 }
103 void send_IPI_self(int vector)
104 {
105 __send_IPI_shortcut(APIC_DEST_SELF, vector);
106 }
108 static inline void send_IPI_mask_bitmask(int mask, int vector)
109 {
110 unsigned long cfg;
111 unsigned long flags;
113 __save_flags(flags);
114 __cli();
117 /*
118 * Wait for idle.
119 */
120 apic_wait_icr_idle();
122 /*
123 * prepare target chip field
124 */
125 cfg = __prepare_ICR2(mask);
126 apic_write_around(APIC_ICR2, cfg);
128 /*
129 * program the ICR
130 */
131 cfg = __prepare_ICR(0, vector);
133 /*
134 * Send the IPI. The write to APIC_ICR fires this off.
135 */
136 apic_write_around(APIC_ICR, cfg);
138 __restore_flags(flags);
139 }
141 static inline void send_IPI_mask_sequence(int mask, int vector)
142 {
143 unsigned long cfg, flags;
144 unsigned int query_cpu, query_mask;
146 __save_flags(flags);
147 __cli();
149 for (query_cpu = 0; query_cpu < NR_CPUS; ++query_cpu) {
150 query_mask = 1 << query_cpu;
151 if (query_mask & mask) {
153 /*
154 * Wait for idle.
155 */
156 apic_wait_icr_idle();
158 /*
159 * prepare target chip field
160 */
161 cfg = __prepare_ICR2(cpu_to_logical_apicid(query_cpu));
162 apic_write_around(APIC_ICR2, cfg);
164 /*
165 * program the ICR
166 */
167 cfg = __prepare_ICR(0, vector);
169 /*
170 * Send the IPI. The write to APIC_ICR fires this off.
171 */
172 apic_write_around(APIC_ICR, cfg);
173 }
174 }
175 __restore_flags(flags);
176 }
178 static inline void send_IPI_mask(int mask, int vector)
179 {
180 send_IPI_mask_bitmask(mask, vector);
181 }
183 static inline void send_IPI_allbutself(int vector)
184 {
185 /*
186 * if there are no other CPUs in the system then
187 * we get an APIC send error if we try to broadcast.
188 * thus we have to avoid sending IPIs in this case.
189 */
190 if (!(smp_num_cpus > 1))
191 return;
193 __send_IPI_shortcut(APIC_DEST_ALLBUT, vector);
194 }
196 static inline void send_IPI_all(int vector)
197 {
198 __send_IPI_shortcut(APIC_DEST_ALLINC, vector);
199 }
201 /*
202 * ********* XEN NOTICE **********
203 * I've left the following comments lying around as they look liek they might
204 * be useful to get multiprocessor guest OSes going. However, I suspect the
205 * issues we face will be quite different so I've ripped out all the
206 * TLBSTATE logic (I didn't understand it anyway :-). These comments do
207 * not apply to Xen, therefore! -- Keir (8th Oct 2003).
208 */
209 /*
210 * Smarter SMP flushing macros.
211 * c/o Linus Torvalds.
212 *
213 * These mean you can really definitely utterly forget about
214 * writing to user space from interrupts. (Its not allowed anyway).
215 *
216 * Optimizations Manfred Spraul <manfred@colorfullife.com>
217 *
218 * The flush IPI assumes that a thread switch happens in this order:
219 * [cpu0: the cpu that switches]
220 * 1) switch_mm() either 1a) or 1b)
221 * 1a) thread switch to a different mm
222 * 1a1) clear_bit(cpu, &old_mm.cpu_vm_mask);
223 * Stop ipi delivery for the old mm. This is not synchronized with
224 * the other cpus, but smp_invalidate_interrupt ignore flush ipis
225 * for the wrong mm, and in the worst case we perform a superflous
226 * tlb flush.
227 * 1a2) set cpu_tlbstate to TLBSTATE_OK
228 * Now the smp_invalidate_interrupt won't call leave_mm if cpu0
229 * was in lazy tlb mode.
230 * 1a3) update cpu_tlbstate[].active_mm
231 * Now cpu0 accepts tlb flushes for the new mm.
232 * 1a4) set_bit(cpu, &new_mm.cpu_vm_mask);
233 * Now the other cpus will send tlb flush ipis.
234 * 1a4) change cr3.
235 * 1b) thread switch without mm change
236 * cpu_tlbstate[].active_mm is correct, cpu0 already handles
237 * flush ipis.
238 * 1b1) set cpu_tlbstate to TLBSTATE_OK
239 * 1b2) test_and_set the cpu bit in cpu_vm_mask.
240 * Atomically set the bit [other cpus will start sending flush ipis],
241 * and test the bit.
242 * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
243 * 2) switch %%esp, ie current
244 *
245 * The interrupt must handle 2 special cases:
246 * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
247 * - the cpu performs speculative tlb reads, i.e. even if the cpu only
248 * runs in kernel space, the cpu could load tlb entries for user space
249 * pages.
250 *
251 * The good news is that cpu_tlbstate is local to each cpu, no
252 * write/read ordering problems.
253 *
254 * TLB flush IPI:
255 *
256 * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
257 * 2) Leave the mm if we are in the lazy tlb mode.
258 */
260 static volatile unsigned long flush_cpumask;
261 static spinlock_t tlbstate_lock = SPIN_LOCK_UNLOCKED;
262 #define FLUSH_ALL 0xffffffff
264 asmlinkage void smp_invalidate_interrupt(void)
265 {
266 ack_APIC_irq();
267 if (test_and_clear_bit(smp_processor_id(), &flush_cpumask))
268 local_flush_tlb();
269 }
271 void flush_tlb_others(unsigned long cpumask)
272 {
273 spin_lock(&tlbstate_lock);
274 atomic_set_mask(cpumask, &flush_cpumask);
275 send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR);
276 while (flush_cpumask) continue;
277 spin_unlock(&tlbstate_lock);
278 }
280 static inline void do_flush_tlb_all_local(void)
281 {
282 __flush_tlb_all();
283 }
285 static void flush_tlb_all_ipi(void* info)
286 {
287 do_flush_tlb_all_local();
288 }
290 void flush_tlb_all(void)
291 {
292 smp_call_function (flush_tlb_all_ipi,0,1,1);
294 do_flush_tlb_all_local();
295 }
297 void smp_send_event_check_mask(unsigned long cpu_mask)
298 {
299 send_IPI_mask(cpu_mask, EVENT_CHECK_VECTOR);
300 }
302 /*
303 * Structure and data for smp_call_function(). This is designed to minimise
304 * static memory requirements. It also looks cleaner.
305 */
306 static spinlock_t call_lock = SPIN_LOCK_UNLOCKED;
308 struct call_data_struct {
309 void (*func) (void *info);
310 void *info;
311 atomic_t started;
312 atomic_t finished;
313 int wait;
314 };
316 static struct call_data_struct * call_data;
318 /*
319 * this function sends a 'generic call function' IPI to all other CPUs
320 * in the system.
321 */
323 int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
324 int wait)
325 /*
326 * [SUMMARY] Run a function on all other CPUs.
327 * <func> The function to run. This must be fast and non-blocking.
328 * <info> An arbitrary pointer to pass to the function.
329 * <nonatomic> currently unused.
330 * <wait> If true, wait (atomically) until function has completed on other CPUs.
331 * [RETURNS] 0 on success, else a negative status code. Does not return until
332 * remote CPUs are nearly ready to execute <<func>> or are or have executed.
333 *
334 * You must not call this function with disabled interrupts or from a
335 * hardware interrupt handler, or bottom halfs.
336 */
337 {
338 struct call_data_struct data;
339 int cpus = smp_num_cpus-1;
341 if (!cpus)
342 return 0;
344 data.func = func;
345 data.info = info;
346 atomic_set(&data.started, 0);
347 data.wait = wait;
348 if (wait)
349 atomic_set(&data.finished, 0);
351 spin_lock(&call_lock);
352 call_data = &data;
353 wmb();
354 /* Send a message to all other CPUs and wait for them to respond */
355 send_IPI_allbutself(CALL_FUNCTION_VECTOR);
357 /* Wait for response */
358 while (atomic_read(&data.started) != cpus)
359 barrier();
361 if (wait)
362 while (atomic_read(&data.finished) != cpus)
363 barrier();
365 spin_unlock(&call_lock);
367 return 0;
368 }
370 static void stop_this_cpu (void * dummy)
371 {
372 /*
373 * Remove this CPU:
374 */
375 clear_bit(smp_processor_id(), &cpu_online_map);
376 __cli();
377 disable_local_APIC();
378 for(;;) __asm__("hlt");
379 }
381 /*
382 * this function calls the 'stop' function on all other CPUs in the system.
383 */
385 void smp_send_stop(void)
386 {
387 smp_call_function(stop_this_cpu, NULL, 1, 0);
388 smp_num_cpus = 1;
390 __cli();
391 disable_local_APIC();
392 __sti();
393 }
395 /*
396 * Nothing to do, as all the work is done automatically when
397 * we return from the interrupt.
398 */
399 asmlinkage void smp_event_check_interrupt(void)
400 {
401 ack_APIC_irq();
402 }
404 asmlinkage void smp_call_function_interrupt(void)
405 {
406 void (*func) (void *info) = call_data->func;
407 void *info = call_data->info;
408 int wait = call_data->wait;
410 ack_APIC_irq();
411 /*
412 * Notify initiating CPU that I've grabbed the data and am
413 * about to execute the function
414 */
415 mb();
416 atomic_inc(&call_data->started);
417 /*
418 * At this point the info structure may be out of scope unless wait==1
419 */
420 (*func)(info);
421 if (wait) {
422 mb();
423 atomic_inc(&call_data->finished);
424 }
425 }
427 #endif /* CONFIG_SMP */